* [FFmpeg-devel] [PR] libswscale: SwsGraph changes (PR #21786)
@ 2026-02-18 13:04 Niklas Haas via ffmpeg-devel
0 siblings, 0 replies; only message in thread
From: Niklas Haas via ffmpeg-devel @ 2026-02-18 13:04 UTC (permalink / raw)
To: ffmpeg-devel; +Cc: Niklas Haas
PR #21786 opened by Niklas Haas (haasn)
URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/21786
Patch URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/21786.patch
More preliminary changes, backported from my WIP swscale branch. This bundles most of the changes related to SwsGraph.
>From a24dd51566a1e114d46c793ac8256bae974d4ee6 Mon Sep 17 00:00:00 2001
From: Niklas Haas <git@haasn.dev>
Date: Mon, 9 Feb 2026 15:27:38 +0100
Subject: [PATCH 01/11] swscale/graph: check output plane pointer instead of
pixel format
To see if the output buffers are allocated or not.
Signed-off-by: Niklas Haas <git@haasn.dev>
---
libswscale/graph.c | 9 ++++-----
1 file changed, 4 insertions(+), 5 deletions(-)
diff --git a/libswscale/graph.c b/libswscale/graph.c
index 9d9ca53b00..10b4a6a259 100644
--- a/libswscale/graph.c
+++ b/libswscale/graph.c
@@ -38,7 +38,7 @@
static int pass_alloc_output(SwsPass *pass)
{
- if (!pass || pass->output.fmt != AV_PIX_FMT_NONE)
+ if (!pass || pass->output.data[0])
return 0;
pass->output.fmt = pass->format;
return av_image_alloc(pass->output.data, pass->output.linesize, pass->width,
@@ -680,7 +680,7 @@ static void sws_graph_worker(void *priv, int jobnr, int threadnr, int nb_jobs,
SwsGraph *graph = priv;
const SwsPass *pass = graph->exec.pass;
const SwsImg *input = pass->input ? &pass->input->output : &graph->exec.input;
- const SwsImg *output = pass->output.fmt != AV_PIX_FMT_NONE ? &pass->output : &graph->exec.output;
+ const SwsImg *output = pass->output.data[0] ? &pass->output : &graph->exec.output;
const int slice_y = jobnr * pass->slice_h;
const int slice_h = FFMIN(pass->slice_h, pass->height - slice_y);
@@ -737,8 +737,7 @@ void ff_sws_graph_free(SwsGraph **pgraph)
SwsPass *pass = graph->passes[i];
if (pass->free)
pass->free(pass->priv);
- if (pass->output.fmt != AV_PIX_FMT_NONE)
- av_free(pass->output.data[0]);
+ av_free(pass->output.data[0]);
av_free(pass);
}
av_free(graph->passes);
@@ -804,7 +803,7 @@ void ff_sws_graph_run(SwsGraph *graph, uint8_t *const out_data[4],
const SwsPass *pass = graph->passes[i];
graph->exec.pass = pass;
if (pass->setup) {
- pass->setup(pass->output.fmt != AV_PIX_FMT_NONE ? &pass->output : out,
+ pass->setup(pass->output.data[0] ? &pass->output : out,
pass->input ? &pass->input->output : in, pass);
}
avpriv_slicethread_execute(graph->slicethread, pass->num_slices, 0);
--
2.52.0
>From cae52ccbeec44977a2a3ba3a87ad3a346375a63f Mon Sep 17 00:00:00 2001
From: Niklas Haas <git@haasn.dev>
Date: Sat, 7 Feb 2026 14:51:35 +0100
Subject: [PATCH 02/11] swscale/graph: simplify ff_sws_graph_run() API
There's little reason not to directly take an SwsImg here; it's already an
internally visible struct.
Signed-off-by: Niklas Haas <git@haasn.dev>
---
libswscale/graph.c | 28 ++++++++++------------------
libswscale/graph.h | 9 +++------
libswscale/swscale.c | 27 ++++++++++++++-------------
3 files changed, 27 insertions(+), 37 deletions(-)
diff --git a/libswscale/graph.c b/libswscale/graph.c
index 10b4a6a259..522f9bbb70 100644
--- a/libswscale/graph.c
+++ b/libswscale/graph.c
@@ -173,7 +173,7 @@ static void setup_legacy_swscale(const SwsImg *out, const SwsImg *in,
const SwsPass *pass)
{
const SwsGraph *graph = pass->graph;
- const SwsImg *in_orig = &graph->exec.input;
+ const SwsImg *in_orig = graph->exec.input;
SwsContext *sws = pass->priv;
SwsInternal *c = sws_internal(sws);
if (sws->flags & SWS_BITEXACT && sws->dither == SWS_DITHER_ED && c->dither_error[0]) {
@@ -679,8 +679,8 @@ static void sws_graph_worker(void *priv, int jobnr, int threadnr, int nb_jobs,
{
SwsGraph *graph = priv;
const SwsPass *pass = graph->exec.pass;
- const SwsImg *input = pass->input ? &pass->input->output : &graph->exec.input;
- const SwsImg *output = pass->output.data[0] ? &pass->output : &graph->exec.output;
+ const SwsImg *input = pass->input ? &pass->input->output : graph->exec.input;
+ const SwsImg *output = pass->output.data[0] ? &pass->output : graph->exec.output;
const int slice_y = jobnr * pass->slice_h;
const int slice_h = FFMIN(pass->slice_h, pass->height - slice_y);
@@ -701,9 +701,6 @@ int ff_sws_graph_create(SwsContext *ctx, const SwsFormat *dst, const SwsFormat *
graph->field = field;
graph->opts_copy = *ctx;
- graph->exec.input.fmt = src->format;
- graph->exec.output.fmt = dst->format;
-
ret = avpriv_slicethread_create(&graph->slicethread, (void *) graph,
sws_graph_worker, NULL, ctx->threads);
if (ret == AVERROR(ENOSYS))
@@ -787,24 +784,19 @@ void ff_sws_graph_update_metadata(SwsGraph *graph, const SwsColor *color)
ff_color_update_dynamic(&graph->src.color, color);
}
-void ff_sws_graph_run(SwsGraph *graph, uint8_t *const out_data[4],
- const int out_linesize[4],
- const uint8_t *const in_data[4],
- const int in_linesize[4])
+void ff_sws_graph_run(SwsGraph *graph, const SwsImg *output, const SwsImg *input)
{
- SwsImg *out = &graph->exec.output;
- SwsImg *in = &graph->exec.input;
- memcpy(out->data, out_data, sizeof(out->data));
- memcpy(out->linesize, out_linesize, sizeof(out->linesize));
- memcpy(in->data, in_data, sizeof(in->data));
- memcpy(in->linesize, in_linesize, sizeof(in->linesize));
+ av_assert0(output->fmt == graph->dst.format);
+ av_assert0(input->fmt == graph->src.format);
+ graph->exec.output = output;
+ graph->exec.input = input;
for (int i = 0; i < graph->num_passes; i++) {
const SwsPass *pass = graph->passes[i];
graph->exec.pass = pass;
if (pass->setup) {
- pass->setup(pass->output.data[0] ? &pass->output : out,
- pass->input ? &pass->input->output : in, pass);
+ pass->setup(pass->output.data[0] ? &pass->output : output,
+ pass->input ? &pass->input->output : input, pass);
}
avpriv_slicethread_execute(graph->slicethread, pass->num_slices, 0);
}
diff --git a/libswscale/graph.h b/libswscale/graph.h
index b829bac88c..44944a3c35 100644
--- a/libswscale/graph.h
+++ b/libswscale/graph.h
@@ -131,8 +131,8 @@ typedef struct SwsGraph {
/** Temporary execution state inside ff_sws_graph_run */
struct {
const SwsPass *pass; /* current filter pass */
- SwsImg input;
- SwsImg output;
+ const SwsImg *input; /* arguments passed to ff_sws_graph_run() */
+ const SwsImg *output;
} exec;
} SwsGraph;
@@ -182,9 +182,6 @@ int ff_sws_graph_reinit(SwsContext *ctx, const SwsFormat *dst, const SwsFormat *
/**
* Dispatch the filter graph on a single field. Internally threaded.
*/
-void ff_sws_graph_run(SwsGraph *graph, uint8_t *const out_data[4],
- const int out_linesize[4],
- const uint8_t *const in_data[4],
- const int in_linesize[4]);
+void ff_sws_graph_run(SwsGraph *graph, const SwsImg *output, const SwsImg *input);
#endif /* SWSCALE_GRAPH_H */
diff --git a/libswscale/swscale.c b/libswscale/swscale.c
index 219382505c..4ae0f25b9d 100644
--- a/libswscale/swscale.c
+++ b/libswscale/swscale.c
@@ -1317,24 +1317,26 @@ int sws_receive_slice(SwsContext *sws, unsigned int slice_start,
dst, c->frame_dst->linesize, slice_start, slice_height);
}
-static void get_frame_pointers(const AVFrame *frame, uint8_t *data[4],
- int linesize[4], int field)
+static SwsImg get_frame_img(const AVFrame *frame, int field)
{
+ SwsImg img = {0};
+
+ img.fmt = frame->format;
for (int i = 0; i < 4; i++) {
- data[i] = frame->data[i];
- linesize[i] = frame->linesize[i];
+ img.data[i] = frame->data[i];
+ img.linesize[i] = frame->linesize[i];
}
if (!(frame->flags & AV_FRAME_FLAG_INTERLACED)) {
av_assert1(!field);
- return;
+ return img;
}
if (field == FIELD_BOTTOM) {
/* Odd rows, offset by one line */
const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(frame->format);
for (int i = 0; i < 4; i++) {
- data[i] += linesize[i];
+ img.data[i] += img.linesize[i];
if (desc->flags & AV_PIX_FMT_FLAG_PAL)
break;
}
@@ -1342,7 +1344,9 @@ static void get_frame_pointers(const AVFrame *frame, uint8_t *data[4],
/* Take only every second line */
for (int i = 0; i < 4; i++)
- linesize[i] <<= 1;
+ img.linesize[i] <<= 1;
+
+ return img;
}
/* Subset of av_frame_ref() that only references (video) data buffers */
@@ -1409,12 +1413,9 @@ int sws_scale_frame(SwsContext *sws, AVFrame *dst, const AVFrame *src)
for (int field = 0; field < 2; field++) {
SwsGraph *graph = c->graph[field];
- uint8_t *dst_data[4], *src_data[4];
- int dst_linesize[4], src_linesize[4];
- get_frame_pointers(dst, dst_data, dst_linesize, field);
- get_frame_pointers(src, src_data, src_linesize, field);
- ff_sws_graph_run(graph, dst_data, dst_linesize,
- (const uint8_t **) src_data, src_linesize);
+ SwsImg input = get_frame_img(src, field);
+ SwsImg output = get_frame_img(dst, field);
+ ff_sws_graph_run(graph, &output, &input);
if (!graph->dst.interlaced)
break;
}
--
2.52.0
>From cefc7c67c25b4e5e17c7890a386b6beaefebee5b Mon Sep 17 00:00:00 2001
From: Niklas Haas <git@haasn.dev>
Date: Wed, 11 Feb 2026 19:38:58 +0100
Subject: [PATCH 03/11] swscale/graph: add internal image alloc helper
This is pointless at the moment, but will become necessary once we
start adding things like refcounted and/or partially allocated plane buffers.
Preempt some of those changes by also adding a buffer reference to SwsImg.
Signed-off-by: Niklas Haas <git@haasn.dev>
---
libswscale/graph.c | 67 +++++++++++++++++++++++++++++++++++++++++++---
libswscale/graph.h | 8 ++++++
2 files changed, 71 insertions(+), 4 deletions(-)
diff --git a/libswscale/graph.c b/libswscale/graph.c
index 522f9bbb70..d921ea9026 100644
--- a/libswscale/graph.c
+++ b/libswscale/graph.c
@@ -36,13 +36,71 @@
#include "graph.h"
#include "ops.h"
+int ff_sws_img_alloc(SwsImg *img, int width, int height)
+{
+ const int align = HAVE_SIMD_ALIGN_64 ? 64 : 32;
+ const int padding = align;
+
+ int ret = av_image_check_size(width, height, 0, NULL);
+ if (ret < 0)
+ return ret;
+
+ int linesize[4];
+ ret = av_image_fill_linesizes(linesize, img->fmt, FFALIGN(width, 8));
+ if (ret < 0)
+ return ret;
+
+ ptrdiff_t linesize1[4];
+ for (int i = 0; i < FF_ARRAY_ELEMS(linesize1); i++) {
+ if (img->linesize[i]) {
+ linesize[i] = img->linesize[i];
+ linesize1[i] = FFALIGN(linesize[i], align);
+ } else {
+ linesize1[i] = linesize[i] = FFALIGN(linesize[i], align);
+ }
+ }
+
+ size_t sizes[4];
+ ret = av_image_fill_plane_sizes(sizes, img->fmt, height, linesize1);
+ if (ret < 0)
+ return ret;
+
+ size_t total_size = 0;
+ for (int i = 0; i < FF_ARRAY_ELEMS(sizes); i++) {
+ size_t padded_size = sizes[i] + align + padding;
+ total_size += padded_size;
+ if (padded_size < sizes[i] || total_size < padded_size)
+ return AVERROR(EINVAL); /* overflow */
+ }
+
+ AVBufferRef *buf = av_buffer_alloc(total_size);
+ if (!buf)
+ return AVERROR(ENOMEM);
+
+ av_assert0(!img->buf[0]);
+ img->buf[0] = buf;
+
+ uintptr_t ptr = (uintptr_t) buf->data;
+ for (int i = 0; i < 4; i++) {
+ if (!sizes[i])
+ break;
+ ptr = FFALIGN(ptr, align);
+ img->data[i] = (uint8_t *) ptr;
+ img->linesize[i] = linesize[i];
+ ptr += sizes[i] + padding;
+ }
+
+ return 0;
+}
+
static int pass_alloc_output(SwsPass *pass)
{
- if (!pass || pass->output.data[0])
+ if (!pass || pass->output.buf[0])
return 0;
+
pass->output.fmt = pass->format;
- return av_image_alloc(pass->output.data, pass->output.linesize, pass->width,
- pass->num_slices * pass->slice_h, pass->format, 64);
+ return ff_sws_img_alloc(&pass->output, pass->width,
+ pass->num_slices * pass->slice_h);
}
SwsPass *ff_sws_graph_add_pass(SwsGraph *graph, enum AVPixelFormat fmt,
@@ -734,7 +792,8 @@ void ff_sws_graph_free(SwsGraph **pgraph)
SwsPass *pass = graph->passes[i];
if (pass->free)
pass->free(pass->priv);
- av_free(pass->output.data[0]);
+ for (int n = 0; n < FF_ARRAY_ELEMS(pass->output.buf); n++)
+ av_buffer_unref(&pass->output.buf[n]);
av_free(pass);
}
av_free(graph->passes);
diff --git a/libswscale/graph.h b/libswscale/graph.h
index 44944a3c35..26f7b969ee 100644
--- a/libswscale/graph.h
+++ b/libswscale/graph.h
@@ -34,6 +34,7 @@ typedef struct SwsImg {
enum AVPixelFormat fmt;
uint8_t *data[4]; /* points to y=0 */
int linesize[4];
+ AVBufferRef *buf[4]; /* buffer references for `data` */
} SwsImg;
static av_always_inline av_const int ff_fmt_vshift(enum AVPixelFormat fmt, int plane)
@@ -50,6 +51,13 @@ static av_const inline SwsImg ff_sws_img_shift(const SwsImg *base, const int y)
return img;
}
+/**
+ * Allocate data pointers for an SwsImg.
+ *
+ * Returns 0 or a negative error code.
+ */
+int ff_sws_img_alloc(SwsImg *img, int width, int height);
+
typedef struct SwsPass SwsPass;
typedef struct SwsGraph SwsGraph;
--
2.52.0
>From 2d7aa9d13920a822f9c424428b2e92c80a90be74 Mon Sep 17 00:00:00 2001
From: Niklas Haas <git@haasn.dev>
Date: Wed, 11 Feb 2026 20:21:20 +0100
Subject: [PATCH 04/11] swscale/graph: remove unneeded field
This is wholly redundant with pass->output.fmt, especially now that we
use buffer refs to track the allocation state.
Signed-off-by: Niklas Haas <git@haasn.dev>
---
libswscale/graph.c | 6 ++----
libswscale/graph.h | 1 -
2 files changed, 2 insertions(+), 5 deletions(-)
diff --git a/libswscale/graph.c b/libswscale/graph.c
index d921ea9026..735ec74c0b 100644
--- a/libswscale/graph.c
+++ b/libswscale/graph.c
@@ -98,7 +98,6 @@ static int pass_alloc_output(SwsPass *pass)
if (!pass || pass->output.buf[0])
return 0;
- pass->output.fmt = pass->format;
return ff_sws_img_alloc(&pass->output, pass->width,
pass->num_slices * pass->slice_h);
}
@@ -115,11 +114,10 @@ SwsPass *ff_sws_graph_add_pass(SwsGraph *graph, enum AVPixelFormat fmt,
pass->graph = graph;
pass->run = run;
pass->priv = priv;
- pass->format = fmt;
pass->width = width;
pass->height = height;
pass->input = input;
- pass->output.fmt = AV_PIX_FMT_NONE;
+ pass->output.fmt = fmt;
ret = pass_alloc_output(input);
if (ret < 0) {
@@ -709,7 +707,7 @@ static int init_passes(SwsGraph *graph)
ret = adapt_colors(graph, src, dst, pass, &pass);
if (ret < 0)
return ret;
- src.format = pass ? pass->format : src.format;
+ src.format = pass ? pass->output.fmt : src.format;
src.color = dst.color;
if (!ff_fmt_equal(&src, &dst)) {
diff --git a/libswscale/graph.h b/libswscale/graph.h
index 26f7b969ee..50ad08fcfa 100644
--- a/libswscale/graph.h
+++ b/libswscale/graph.h
@@ -82,7 +82,6 @@ struct SwsPass {
* are always equal to (or smaller than, for the last slice) `slice_h`.
*/
sws_filter_run_t run;
- enum AVPixelFormat format; /* new pixel format */
int width, height; /* new output size */
int slice_h; /* filter granularity */
int num_slices;
--
2.52.0
>From c5aa158c26589fbbf665248a6f68cf485fc64950 Mon Sep 17 00:00:00 2001
From: Niklas Haas <git@haasn.dev>
Date: Tue, 10 Feb 2026 18:18:09 +0100
Subject: [PATCH 05/11] swscale/graph: set up palette using current input image
Using the original input image here is completely wrong - the palette could
have been set to anything else in the meantime. At best, we would want to
use the original input to add_legacy_sws_pass(), but it's impossible for this
to differ from the per-pass input. The only time legacy subpasses are added
is when using cascaded contexts, but in this case, the only context actually
reading from the palette format would be the first one.
I'm not entirely sure why this code was originally written this way, but
I'm reasonably confident that it's not at all necessary. Tested extensively
on both FATE, the self-test, and real-world files.
Signed-off-by: Niklas Haas <git@haasn.dev>
---
libswscale/graph.c | 4 +---
1 file changed, 1 insertion(+), 3 deletions(-)
diff --git a/libswscale/graph.c b/libswscale/graph.c
index 735ec74c0b..e048d8b859 100644
--- a/libswscale/graph.c
+++ b/libswscale/graph.c
@@ -228,8 +228,6 @@ static void free_legacy_swscale(void *priv)
static void setup_legacy_swscale(const SwsImg *out, const SwsImg *in,
const SwsPass *pass)
{
- const SwsGraph *graph = pass->graph;
- const SwsImg *in_orig = graph->exec.input;
SwsContext *sws = pass->priv;
SwsInternal *c = sws_internal(sws);
if (sws->flags & SWS_BITEXACT && sws->dither == SWS_DITHER_ED && c->dither_error[0]) {
@@ -238,7 +236,7 @@ static void setup_legacy_swscale(const SwsImg *out, const SwsImg *in,
}
if (usePal(sws->src_format))
- ff_update_palette(c, (const uint32_t *) in_orig->data[1]);
+ ff_update_palette(c, (const uint32_t *) in->data[1]);
}
static inline SwsContext *slice_ctx(const SwsPass *pass, int y)
--
2.52.0
>From 09b3ab29e4387abefa99ea3eb436f460450ca2f0 Mon Sep 17 00:00:00 2001
From: Niklas Haas <git@haasn.dev>
Date: Tue, 10 Feb 2026 18:22:57 +0100
Subject: [PATCH 06/11] swscale/graph: store current pass input instead of
global args
The global args to ff_sws_graph_run() really shouldn't matter inside thread
workers. If they ever do, it indicates a leaky abstraction. The only reason
it was needed in the first place was because of the way the input/output
buffers implicitly defaulted to the global args.
However, we can solve this much more elegantly by just calculating it in
ff_sws_graph_run() directly and storing the computed SwsImg inside the
execution state.
Signed-off-by: Niklas Haas <git@haasn.dev>
---
libswscale/graph.c | 32 ++++++++++++++++++++++----------
libswscale/graph.h | 9 ++++++---
2 files changed, 28 insertions(+), 13 deletions(-)
diff --git a/libswscale/graph.c b/libswscale/graph.c
index e048d8b859..cef386a88b 100644
--- a/libswscale/graph.c
+++ b/libswscale/graph.c
@@ -733,12 +733,10 @@ static void sws_graph_worker(void *priv, int jobnr, int threadnr, int nb_jobs,
{
SwsGraph *graph = priv;
const SwsPass *pass = graph->exec.pass;
- const SwsImg *input = pass->input ? &pass->input->output : graph->exec.input;
- const SwsImg *output = pass->output.data[0] ? &pass->output : graph->exec.output;
const int slice_y = jobnr * pass->slice_h;
const int slice_h = FFMIN(pass->slice_h, pass->height - slice_y);
- pass->run(output, input, slice_y, slice_h, pass);
+ pass->run(&graph->exec.output, &graph->exec.input, slice_y, slice_h, pass);
}
int ff_sws_graph_create(SwsContext *ctx, const SwsFormat *dst, const SwsFormat *src,
@@ -839,20 +837,34 @@ void ff_sws_graph_update_metadata(SwsGraph *graph, const SwsColor *color)
ff_color_update_dynamic(&graph->src.color, color);
}
+static SwsImg pass_output(const SwsPass *pass, const SwsImg *fallback)
+{
+ if (!pass)
+ return *fallback;
+
+ SwsImg img = pass->output;
+ for (int i = 0; i < FF_ARRAY_ELEMS(img.data); i++) {
+ if (!img.data[i]) {
+ img.data[i] = fallback->data[i];
+ img.linesize[i] = fallback->linesize[i];
+ }
+ }
+
+ return img;
+}
+
void ff_sws_graph_run(SwsGraph *graph, const SwsImg *output, const SwsImg *input)
{
av_assert0(output->fmt == graph->dst.format);
av_assert0(input->fmt == graph->src.format);
- graph->exec.output = output;
- graph->exec.input = input;
for (int i = 0; i < graph->num_passes; i++) {
const SwsPass *pass = graph->passes[i];
- graph->exec.pass = pass;
- if (pass->setup) {
- pass->setup(pass->output.data[0] ? &pass->output : output,
- pass->input ? &pass->input->output : input, pass);
- }
+ graph->exec.pass = pass;
+ graph->exec.input = pass_output(pass->input, input);
+ graph->exec.output = pass_output(pass, output);
+ if (pass->setup)
+ pass->setup(&graph->exec.output, &graph->exec.input, pass);
avpriv_slicethread_execute(graph->slicethread, pass->num_slices, 0);
}
}
diff --git a/libswscale/graph.h b/libswscale/graph.h
index 50ad08fcfa..51dd222a4b 100644
--- a/libswscale/graph.h
+++ b/libswscale/graph.h
@@ -135,11 +135,14 @@ typedef struct SwsGraph {
SwsFormat src, dst;
int field;
- /** Temporary execution state inside ff_sws_graph_run */
+ /**
+ * Temporary execution state inside ff_sws_graph_run(); used to pass
+ * data to worker threads.
+ */
struct {
const SwsPass *pass; /* current filter pass */
- const SwsImg *input; /* arguments passed to ff_sws_graph_run() */
- const SwsImg *output;
+ SwsImg input; /* current filter pass input/output */
+ SwsImg output;
} exec;
} SwsGraph;
--
2.52.0
>From a2b107461d8aa043af801201a309017dacd70698 Mon Sep 17 00:00:00 2001
From: Niklas Haas <git@haasn.dev>
Date: Wed, 11 Feb 2026 20:16:09 +0100
Subject: [PATCH 07/11] swscale/graph: switch SwsPass.output to refstruct
Allows multiple passes to share a single output buffer reference. We always
allocate an output buffer so that subpasses can share the same output buffer
reference while still allowing that reference to implicitly point to the
final output image.
Sponsored-by: Sovereign Tech Fund
Signed-off-by: Niklas Haas <git@haasn.dev>
---
libswscale/graph.c | 32 ++++++++++++++++++++++++--------
libswscale/graph.h | 2 +-
2 files changed, 25 insertions(+), 9 deletions(-)
diff --git a/libswscale/graph.c b/libswscale/graph.c
index cef386a88b..5f8e1c3ee4 100644
--- a/libswscale/graph.c
+++ b/libswscale/graph.c
@@ -25,6 +25,7 @@
#include "libavutil/mem.h"
#include "libavutil/opt.h"
#include "libavutil/pixdesc.h"
+#include "libavutil/refstruct.h"
#include "libavutil/slicethread.h"
#include "libswscale/swscale.h"
@@ -95,13 +96,20 @@ int ff_sws_img_alloc(SwsImg *img, int width, int height)
static int pass_alloc_output(SwsPass *pass)
{
- if (!pass || pass->output.buf[0])
+ if (!pass || pass->output->buf[0])
return 0;
- return ff_sws_img_alloc(&pass->output, pass->width,
+ return ff_sws_img_alloc(pass->output, pass->width,
pass->num_slices * pass->slice_h);
}
+static void free_img(AVRefStructOpaque opaque, void *obj)
+{
+ SwsImg *img = obj;
+ for (int i = 0; i < FF_ARRAY_ELEMS(img->buf); i++)
+ av_buffer_unref(&img->buf[i]);
+}
+
SwsPass *ff_sws_graph_add_pass(SwsGraph *graph, enum AVPixelFormat fmt,
int width, int height, SwsPass *input,
int align, void *priv, sws_filter_run_t run)
@@ -117,7 +125,10 @@ SwsPass *ff_sws_graph_add_pass(SwsGraph *graph, enum AVPixelFormat fmt,
pass->width = width;
pass->height = height;
pass->input = input;
- pass->output.fmt = fmt;
+ pass->output = av_refstruct_alloc_ext(sizeof(*pass->output), 0, NULL, free_img);
+ if (!pass->output)
+ goto fail;
+ pass->output->fmt = fmt;
ret = pass_alloc_output(input);
if (ret < 0) {
@@ -136,8 +147,14 @@ SwsPass *ff_sws_graph_add_pass(SwsGraph *graph, enum AVPixelFormat fmt,
ret = av_dynarray_add_nofree(&graph->passes, &graph->num_passes, pass);
if (ret < 0)
- av_freep(&pass);
+ goto fail;
+
return pass;
+
+fail:
+ av_refstruct_unref(&pass->output);
+ av_free(pass);
+ return NULL;
}
/* Wrapper around ff_sws_graph_add_pass() that chains a pass "in-place" */
@@ -705,7 +722,7 @@ static int init_passes(SwsGraph *graph)
ret = adapt_colors(graph, src, dst, pass, &pass);
if (ret < 0)
return ret;
- src.format = pass ? pass->output.fmt : src.format;
+ src.format = pass ? pass->output->fmt : src.format;
src.color = dst.color;
if (!ff_fmt_equal(&src, &dst)) {
@@ -786,8 +803,7 @@ void ff_sws_graph_free(SwsGraph **pgraph)
SwsPass *pass = graph->passes[i];
if (pass->free)
pass->free(pass->priv);
- for (int n = 0; n < FF_ARRAY_ELEMS(pass->output.buf); n++)
- av_buffer_unref(&pass->output.buf[n]);
+ av_refstruct_unref(&pass->output);
av_free(pass);
}
av_free(graph->passes);
@@ -842,7 +858,7 @@ static SwsImg pass_output(const SwsPass *pass, const SwsImg *fallback)
if (!pass)
return *fallback;
- SwsImg img = pass->output;
+ SwsImg img = *pass->output;
for (int i = 0; i < FF_ARRAY_ELEMS(img.data); i++) {
if (!img.data[i]) {
img.data[i] = fallback->data[i];
diff --git a/libswscale/graph.h b/libswscale/graph.h
index 51dd222a4b..b9709c903d 100644
--- a/libswscale/graph.h
+++ b/libswscale/graph.h
@@ -95,7 +95,7 @@ struct SwsPass {
/**
* Filter output buffer. Allocated on demand and freed automatically.
*/
- SwsImg output;
+ SwsImg *output; /* refstruct */
/**
* Called once from the main thread before running the filter. Optional.
--
2.52.0
>From 8f093b1e3757db5bbf2dcd482414882af547a3d6 Mon Sep 17 00:00:00 2001
From: Niklas Haas <git@haasn.dev>
Date: Tue, 10 Feb 2026 14:58:28 +0100
Subject: [PATCH 08/11] swscale/graph: omit memcpy() if src and dst are
identical
This allows already referenced planes to be skipped, in the case of e.g.
only some of the output planes being sucessfully referenced. Also avoids
what is technically UB, if the user happens to call ff_sws_graph_run() after
already having ref'd an image.
Signed-off-by: Niklas Haas <git@haasn.dev>
---
libswscale/graph.c | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/libswscale/graph.c b/libswscale/graph.c
index 5f8e1c3ee4..19839507a6 100644
--- a/libswscale/graph.c
+++ b/libswscale/graph.c
@@ -178,7 +178,9 @@ static void run_copy(const SwsImg *out_base, const SwsImg *in_base,
const int lines = h >> ff_fmt_vshift(in.fmt, i);
av_assert1(in.data[i]);
- if (in.linesize[i] == out.linesize[i]) {
+ if (in.data[i] == out.data[i]) {
+ av_assert1(in.linesize[i] == out.linesize[i]);
+ } else if (in.linesize[i] == out.linesize[i]) {
memcpy(out.data[i], in.data[i], lines * out.linesize[i]);
} else {
const int linesize = FFMIN(out.linesize[i], in.linesize[i]);
--
2.52.0
>From 00dec159440556fc6faf22ffa5571bd2d677f27e Mon Sep 17 00:00:00 2001
From: Niklas Haas <git@haasn.dev>
Date: Tue, 10 Feb 2026 22:32:09 +0100
Subject: [PATCH 09/11] swscale/graph: add a function to allow re-using output
buffers
Used for plane splitting, among other things. (e.g. plane passthrough)
Sponsored-by: Sovereign Tech Fund
Signed-off-by: Niklas Haas <git@haasn.dev>
---
libswscale/graph.c | 8 ++++++++
libswscale/graph.h | 8 ++++++++
2 files changed, 16 insertions(+)
diff --git a/libswscale/graph.c b/libswscale/graph.c
index 19839507a6..84530d39c9 100644
--- a/libswscale/graph.c
+++ b/libswscale/graph.c
@@ -157,6 +157,14 @@ fail:
return NULL;
}
+void ff_sws_pass_link_output(SwsPass *pass, const SwsPass *other)
+{
+ if (!other)
+ return;
+
+ av_refstruct_replace(&pass->output, other->output);
+}
+
/* Wrapper around ff_sws_graph_add_pass() that chains a pass "in-place" */
static int pass_append(SwsGraph *graph, enum AVPixelFormat fmt, int w, int h,
SwsPass **pass, int align, void *priv, sws_filter_run_t run)
diff --git a/libswscale/graph.h b/libswscale/graph.h
index b9709c903d..271ba16a9b 100644
--- a/libswscale/graph.h
+++ b/libswscale/graph.h
@@ -170,6 +170,14 @@ SwsPass *ff_sws_graph_add_pass(SwsGraph *graph, enum AVPixelFormat fmt,
int width, int height, SwsPass *input,
int align, void *priv, sws_filter_run_t run);
+/**
+ * Link the output buffers to a different pass, rather than allocating
+ * new image buffers. This allows re-using the same buffer for multiple passes,
+ * e.g. in the case of in-place passes or partial passes that modify different
+ * planes.
+ **/
+void ff_sws_pass_link_output(SwsPass *pass, const SwsPass *other);
+
/**
* Uninitialize any state associate with this filter graph and free it.
*/
--
2.52.0
>From b20782de6af0bf92f7f904571d6313e6f37b3457 Mon Sep 17 00:00:00 2001
From: Niklas Haas <git@haasn.dev>
Date: Thu, 12 Feb 2026 12:40:16 +0100
Subject: [PATCH 10/11] swscale/graph: allow using ff_sws_img_alloc on
partially allocated imgs
Useful when plane copying, in which case some planes might already be
allocated and should instead be ref'd to the existing allocation.
We need to be extremely careful to manage buffer references here; because
the caller may have already allocated extra buffer references that are no
longer needed at this point. (e.g. as a result of previous passes being
optimized into a refcopy that makes a previously used plane now redundant)
Signed-off-by: Niklas Haas <git@haasn.dev>
---
libswscale/graph.c | 55 +++++++++++++++++++++++++++++++++++++++++++---
1 file changed, 52 insertions(+), 3 deletions(-)
diff --git a/libswscale/graph.c b/libswscale/graph.c
index 84530d39c9..68620a797f 100644
--- a/libswscale/graph.c
+++ b/libswscale/graph.c
@@ -37,6 +37,46 @@
#include "graph.h"
#include "ops.h"
+/* Find the buffer index corresponding to `data`, or -1 */
+static int data_buf_idx(const SwsImg *img, const uint8_t *data)
+{
+ if (!data)
+ return -1;
+
+ for (int i = 0; i < FF_ARRAY_ELEMS(img->buf); i++) {
+ const AVBufferRef *buf = img->buf[i];
+ if (!buf)
+ break;
+ if (data >= buf->data && data < buf->data + buf->size)
+ return i;
+ }
+
+ return -1;
+}
+
+/* free all unused buffers and compress the rest; returns the number of bufs */
+static int img_compress(SwsImg *img)
+{
+ bool used[4] = {0};
+ for (int i = 0; i < FF_ARRAY_ELEMS(img->data); i++) {
+ int idx = data_buf_idx(img, img->data[i]);
+ if (idx < 0)
+ continue;
+ used[idx] = true;
+ }
+
+ int nb_bufs = 0;
+ for (int i = 0; i < FF_ARRAY_ELEMS(img->buf); i++) {
+ if (used[i]) {
+ img->buf[nb_bufs++] = img->buf[i];
+ } else {
+ av_buffer_unref(&img->buf[i]);
+ }
+ }
+
+ return nb_bufs;
+}
+
int ff_sws_img_alloc(SwsImg *img, int width, int height)
{
const int align = HAVE_SIMD_ALIGN_64 ? 64 : 32;
@@ -68,23 +108,32 @@ int ff_sws_img_alloc(SwsImg *img, int width, int height)
size_t total_size = 0;
for (int i = 0; i < FF_ARRAY_ELEMS(sizes); i++) {
+ if (img->data[i])
+ continue; /* image buffer already allocated for this plane */
size_t padded_size = sizes[i] + align + padding;
total_size += padded_size;
if (padded_size < sizes[i] || total_size < padded_size)
return AVERROR(EINVAL); /* overflow */
}
+ if (!total_size)
+ return 0;
+
AVBufferRef *buf = av_buffer_alloc(total_size);
if (!buf)
return AVERROR(ENOMEM);
- av_assert0(!img->buf[0]);
- img->buf[0] = buf;
+ const int idx_buf = img_compress(img);
+ av_assert0(idx_buf < FF_ARRAY_ELEMS(img->buf));
+ av_assert0(!img->buf[idx_buf]);
+ img->buf[idx_buf] = buf;
uintptr_t ptr = (uintptr_t) buf->data;
for (int i = 0; i < 4; i++) {
if (!sizes[i])
break;
+ if (img->data[i])
+ continue;
ptr = FFALIGN(ptr, align);
img->data[i] = (uint8_t *) ptr;
img->linesize[i] = linesize[i];
@@ -96,7 +145,7 @@ int ff_sws_img_alloc(SwsImg *img, int width, int height)
static int pass_alloc_output(SwsPass *pass)
{
- if (!pass || pass->output->buf[0])
+ if (!pass)
return 0;
return ff_sws_img_alloc(pass->output, pass->width,
--
2.52.0
>From 7c6b86ed951fe53b62f4fa7d318612bc3eb27bdf Mon Sep 17 00:00:00 2001
From: Niklas Haas <git@haasn.dev>
Date: Thu, 12 Feb 2026 13:05:09 +0100
Subject: [PATCH 11/11] swscale/swscale: switch to internal alloc helper
The ultimate purpose of this will be to allow for partially ref'd frames,
where we may still need to allocate buffers for the remaining planes. Do this
on an SwsImg representing the whole frame, even for interlaced content. This
requires splitting up the AVFrame <-> SwsImg helpers into separate helpers for
frames and fields.
Separated from the following commit for purposes of bisection in case any
bugs can be traced back to the new allocator specifically.
Signed-off-by: Niklas Haas <git@haasn.dev>
---
libswscale/swscale.c | 34 ++++++++++++++++++++++++++++++----
1 file changed, 30 insertions(+), 4 deletions(-)
diff --git a/libswscale/swscale.c b/libswscale/swscale.c
index 4ae0f25b9d..de206c540a 100644
--- a/libswscale/swscale.c
+++ b/libswscale/swscale.c
@@ -1317,7 +1317,7 @@ int sws_receive_slice(SwsContext *sws, unsigned int slice_start,
dst, c->frame_dst->linesize, slice_start, slice_height);
}
-static SwsImg get_frame_img(const AVFrame *frame, int field)
+static SwsImg get_frame_img(const AVFrame *frame)
{
SwsImg img = {0};
@@ -1325,8 +1325,16 @@ static SwsImg get_frame_img(const AVFrame *frame, int field)
for (int i = 0; i < 4; i++) {
img.data[i] = frame->data[i];
img.linesize[i] = frame->linesize[i];
+ img.buf[i] = frame->buf[i];
}
+ return img;
+}
+
+static SwsImg get_field_img(const AVFrame *frame, int field)
+{
+ SwsImg img = get_frame_img(frame);
+
if (!(frame->flags & AV_FRAME_FLAG_INTERLACED)) {
av_assert1(!field);
return img;
@@ -1366,6 +1374,24 @@ static int frame_ref(AVFrame *dst, const AVFrame *src)
return 0;
}
+static int frame_alloc_partial(AVFrame *frame)
+{
+ /* Re-use SwsGraph helpers */
+ SwsImg img = get_frame_img(frame);
+ int ret = ff_sws_img_alloc(&img, frame->width, frame->height);
+ if (ret < 0)
+ return ret;
+
+ /* Reflect change back to AVFrame */
+ frame->extended_data = frame->data;
+ for (int i = 0; i < FF_ARRAY_ELEMS(img.data); i++) {
+ frame->data[i] = img.data[i];
+ frame->linesize[i] = img.linesize[i];
+ frame->buf[i] = img.buf[i];
+ }
+ return 0;
+}
+
int sws_scale_frame(SwsContext *sws, AVFrame *dst, const AVFrame *src)
{
int ret;
@@ -1406,15 +1432,15 @@ int sws_scale_frame(SwsContext *sws, AVFrame *dst, const AVFrame *src)
return ret;
} else {
if (!dst->data[0]) {
- ret = av_frame_get_buffer(dst, 0);
+ ret = frame_alloc_partial(dst);
if (ret < 0)
return ret;
}
for (int field = 0; field < 2; field++) {
SwsGraph *graph = c->graph[field];
- SwsImg input = get_frame_img(src, field);
- SwsImg output = get_frame_img(dst, field);
+ SwsImg input = get_field_img(src, field);
+ SwsImg output = get_field_img(dst, field);
ff_sws_graph_run(graph, &output, &input);
if (!graph->dst.interlaced)
break;
--
2.52.0
_______________________________________________
ffmpeg-devel mailing list -- ffmpeg-devel@ffmpeg.org
To unsubscribe send an email to ffmpeg-devel-leave@ffmpeg.org
^ permalink raw reply [flat|nested] only message in thread
only message in thread, other threads:[~2026-02-18 13:05 UTC | newest]
Thread overview: (only message) (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2026-02-18 13:04 [FFmpeg-devel] [PR] libswscale: SwsGraph changes (PR #21786) Niklas Haas via ffmpeg-devel
Git Inbox Mirror of the ffmpeg-devel mailing list - see https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
This inbox may be cloned and mirrored by anyone:
git clone --mirror https://master.gitmailbox.com/ffmpegdev/0 ffmpegdev/git/0.git
# If you have public-inbox 1.1+ installed, you may
# initialize and index your mirror using the following commands:
public-inbox-init -V2 ffmpegdev ffmpegdev/ https://master.gitmailbox.com/ffmpegdev \
ffmpegdev@gitmailbox.com
public-inbox-index ffmpegdev
Example config snippet for mirrors.
AGPL code for this site: git clone https://public-inbox.org/public-inbox.git