From: James Almer via ffmpeg-devel <ffmpeg-devel@ffmpeg.org>
To: ffmpeg-devel@ffmpeg.org
Cc: James Almer <code@ffmpeg.org>
Subject: [FFmpeg-devel] [PATCH] fftools/ffmpeg: Support merging HEIF tiled images automatically (PR #20727)
Date: Mon, 20 Oct 2025 12:58:25 -0000
Message-ID: <176096510595.62.11860224245626366185@bf907ddaa564> (raw)
PR #20727 opened by James Almer (jamrial)
URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/20727
Patch URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/20727.patch
>From a7d280cec74796ebd6581ad263202e4526b32e3c Mon Sep 17 00:00:00 2001
From: James Almer <jamrial@gmail.com>
Date: Sat, 18 Oct 2025 20:07:15 -0300
Subject: [PATCH 01/12] avcodec/avcodec: add helpers to convert between packet
and frame side data
They will be used in following commits.
Signed-off-by: James Almer <jamrial@gmail.com>
---
libavcodec/avcodec.c | 50 ++++++++++++++++++++++++++++++++++++++++++++
libavcodec/avcodec.h | 27 ++++++++++++++++++++++++
2 files changed, 77 insertions(+)
diff --git a/libavcodec/avcodec.c b/libavcodec/avcodec.c
index 0355b7c338..db24b4ed52 100644
--- a/libavcodec/avcodec.c
+++ b/libavcodec/avcodec.c
@@ -815,3 +815,53 @@ int avcodec_get_supported_config(const AVCodecContext *avctx, const AVCodec *cod
return ff_default_get_supported_config(avctx, codec, config, flags, out, out_num);
}
}
+
+int av_packet_side_data_from_frame(AVPacketSideData **psd, int *pnb_sd,
+ const AVFrameSideData *src, unsigned int flags)
+{
+ AVPacketSideData *sd = NULL;
+
+ for (unsigned j = 0; ff_sd_global_map[j].packet < AV_PKT_DATA_NB; j++) {
+ if (ff_sd_global_map[j].frame != src->type)
+ continue;
+
+ sd = av_packet_side_data_new(psd, pnb_sd, ff_sd_global_map[j].packet,
+ src->size, 0);
+
+ if (!sd)
+ return AVERROR(ENOMEM);
+
+ memcpy(sd->data, src->data, src->size);
+ break;
+ }
+
+ if (!sd)
+ return AVERROR(EINVAL);
+
+ return 0;
+}
+
+int av_packet_side_data_to_frame(AVFrameSideData ***psd, int *pnb_sd,
+ const AVPacketSideData *src, unsigned int flags)
+{
+ AVFrameSideData *sd = NULL;
+
+ for (unsigned j = 0; ff_sd_global_map[j].packet < AV_PKT_DATA_NB; j++) {
+ if (ff_sd_global_map[j].packet != src->type)
+ continue;
+
+ sd = av_frame_side_data_new(psd, pnb_sd, ff_sd_global_map[j].frame,
+ src->size, flags);
+
+ if (!sd)
+ return AVERROR(ENOMEM);
+
+ memcpy(sd->data, src->data, src->size);
+ break;
+ }
+
+ if (!sd)
+ return AVERROR(EINVAL);
+
+ return 0;
+}
diff --git a/libavcodec/avcodec.h b/libavcodec/avcodec.h
index 83a4e56e22..d84ffbd8d5 100644
--- a/libavcodec/avcodec.h
+++ b/libavcodec/avcodec.h
@@ -2948,6 +2948,33 @@ void av_fast_padded_mallocz(void *ptr, unsigned int *size, size_t min_size);
*/
int avcodec_is_open(AVCodecContext *s);
+/**
+ * Add a new packet side data entry to an array based on existing frame
+ * side data, if a matching type exists for packet side data.
+ *
+ * @param flags Currently unused. Must be 0.
+ * @retval >= 0 Success
+ * @retval AVERROR(EINVAL) The frame side data type does not have a matching
+ * packet side data type.
+ * @retval AVERROR(ENOMEM) Failed to add a side data entry to the array, or
+ * similar.
+ */
+int av_packet_side_data_from_frame(AVPacketSideData **sd, int *nb_sd,
+ const AVFrameSideData *src, unsigned int flags);
+/**
+ * Add a new frame side data entry to an array based on existing packet
+ * side data, if a matching type exists for frame side data.
+ *
+ * @param flags Some combination of AV_FRAME_SIDE_DATA_FLAG_* flags,
+ * or 0.
+ * @retval >= 0 Success
+ * @retval AVERROR(EINVAL) The packet side data type does not have a matching
+ * frame side data type.
+ * @retval AVERROR(ENOMEM) Failed to add a side data entry to the array, or
+ * similar.
+ */
+int av_packet_side_data_to_frame(AVFrameSideData ***sd, int *nb_sd,
+ const AVPacketSideData *src, unsigned int flags);
/**
* @}
*/
--
2.49.1
>From 5d236d4166d28d5927d2c3c745e9545ddd78b5a8 Mon Sep 17 00:00:00 2001
From: James Almer <jamrial@gmail.com>
Date: Sat, 18 Oct 2025 20:28:46 -0300
Subject: [PATCH 02/12] fftools/ffmpeg_filter: fix passing certain parameters
to inputs bound to a filtergraph output
Certain parameters, like calculated framerate, are unavailable when connecting
the output of a filtergraph to the input of another.
This fixes command lines like
ffmpeg -lavfi "testsrc=rate=1:duration=1[out0]" -filter_complex "[out0]null[out1]" -map [out1] -y out.png
Signed-off-by: James Almer <jamrial@gmail.com>
---
fftools/ffmpeg_filter.c | 34 ++++++++++++++++++++++++++++++++++
1 file changed, 34 insertions(+)
diff --git a/fftools/ffmpeg_filter.c b/fftools/ffmpeg_filter.c
index c1c8eeb2d8..cb4c792cb0 100644
--- a/fftools/ffmpeg_filter.c
+++ b/fftools/ffmpeg_filter.c
@@ -109,6 +109,9 @@ typedef struct InputFilterPriv {
// used to hold submitted input
AVFrame *frame;
+ // For inputs bound to a filtergraph output
+ OutputFilter *ofilter_src;
+
// source data type: AVMEDIA_TYPE_SUBTITLE for sub2video,
// same as type otherwise
enum AVMediaType type_src;
@@ -928,6 +931,8 @@ static int ofilter_bind_ifilter(OutputFilter *ofilter, InputFilterPriv *ifp,
if (!ofilter->output_name)
return AVERROR(EINVAL);
+ ifp->ofilter_src = ofilter;
+
av_strlcatf(ofp->log_name, sizeof(ofp->log_name), "->%s", ofilter->output_name);
return 0;
@@ -2155,6 +2160,27 @@ static int ifilter_parameters_from_frame(InputFilter *ifilter, const AVFrame *fr
return 0;
}
+static int ifilter_parameters_from_ofilter(InputFilter *ifilter, OutputFilter *ofilter)
+{
+ const OutputFilterPriv *ofp = ofp_from_ofilter(ofilter);
+ InputFilterPriv *ifp = ifp_from_ifilter(ifilter);
+
+ if (!ifp->opts.framerate.num) {
+ ifp->opts.framerate = ofp->fps.framerate;
+ if (ifp->opts.framerate.num > 0 && ifp->opts.framerate.den > 0)
+ ifp->opts.flags |= IFILTER_FLAG_CFR;
+ }
+
+ for (int i = 0; i < ofp->nb_side_data; i++) {
+ int ret = av_frame_side_data_clone(&ifp->side_data, &ifp->nb_side_data,
+ ofp->side_data[i], AV_FRAME_SIDE_DATA_FLAG_REPLACE);
+ if (ret < 0)
+ return ret;
+ }
+
+ return 0;
+}
+
int filtergraph_is_simple(const FilterGraph *fg)
{
const FilterGraphPriv *fgp = cfgp_from_cfg(fg);
@@ -2935,6 +2961,14 @@ static int send_frame(FilterGraph *fg, FilterGraphThread *fgt,
ret = ifilter_parameters_from_frame(ifilter, frame);
if (ret < 0)
return ret;
+
+ /* Inputs bound to a filtergraph output will have some fields unset.
+ * Handle them here */
+ if (ifp->ofilter_src) {
+ ret = ifilter_parameters_from_ofilter(ifilter, ifp->ofilter_src);
+ if (ret < 0)
+ return ret;
+ }
}
/* (re)init the graph if possible, otherwise buffer the frame and return */
--
2.49.1
>From 97b58b7ca9de1e759257bdb76a75516f45468dba Mon Sep 17 00:00:00 2001
From: James Almer <jamrial@gmail.com>
Date: Sat, 18 Oct 2025 20:38:39 -0300
Subject: [PATCH 03/12] fftools: pass global side data through using a separate
array
This keeps global and per frame side data clearly separated, and actually
propagates the former as it comes out from the buffersink filter.
Signed-off-by: James Almer <jamrial@gmail.com>
---
fftools/ffmpeg.c | 13 +++++++++++++
fftools/ffmpeg.h | 3 +++
fftools/ffmpeg_enc.c | 9 +++------
fftools/ffmpeg_filter.c | 20 ++++++++++++++------
4 files changed, 33 insertions(+), 12 deletions(-)
diff --git a/fftools/ffmpeg.c b/fftools/ffmpeg.c
index cbeefb71d6..32289112a8 100644
--- a/fftools/ffmpeg.c
+++ b/fftools/ffmpeg.c
@@ -400,6 +400,7 @@ static void frame_data_free(void *opaque, uint8_t *data)
{
FrameData *fd = (FrameData *)data;
+ av_frame_side_data_free(&fd->side_data, &fd->nb_side_data);
avcodec_parameters_free(&fd->par_enc);
av_free(data);
@@ -429,6 +430,8 @@ static int frame_data_ensure(AVBufferRef **dst, int writable)
memcpy(fd, fd_src, sizeof(*fd));
fd->par_enc = NULL;
+ fd->side_data = NULL;
+ fd->nb_side_data = 0;
if (fd_src->par_enc) {
int ret = 0;
@@ -444,6 +447,16 @@ static int frame_data_ensure(AVBufferRef **dst, int writable)
}
}
+ if (fd_src->nb_side_data) {
+ int ret = clone_side_data(&fd->side_data, &fd->nb_side_data,
+ fd_src->side_data, fd_src->nb_side_data, 0);
+ if (ret < 0) {
+ av_buffer_unref(dst);
+ av_buffer_unref(&src);
+ return ret;
+ }
+ }
+
av_buffer_unref(&src);
} else {
fd->dec.frame_num = UINT64_MAX;
diff --git a/fftools/ffmpeg.h b/fftools/ffmpeg.h
index c479a0351d..a3d7f7ab34 100644
--- a/fftools/ffmpeg.h
+++ b/fftools/ffmpeg.h
@@ -698,6 +698,9 @@ typedef struct FrameData {
int64_t wallclock[LATENCY_PROBE_NB];
AVCodecParameters *par_enc;
+
+ AVFrameSideData **side_data;
+ int nb_side_data;
} FrameData;
extern InputFile **input_files;
diff --git a/fftools/ffmpeg_enc.c b/fftools/ffmpeg_enc.c
index 84e7e0ca0e..978b247fc2 100644
--- a/fftools/ffmpeg_enc.c
+++ b/fftools/ffmpeg_enc.c
@@ -205,15 +205,12 @@ int enc_open(void *opaque, const AVFrame *frame)
av_assert0(frame->opaque_ref);
fd = (FrameData*)frame->opaque_ref->data;
- for (int i = 0; i < frame->nb_side_data; i++) {
- const AVSideDataDescriptor *desc = av_frame_side_data_desc(frame->side_data[i]->type);
-
- if (!(desc->props & AV_SIDE_DATA_PROP_GLOBAL))
- continue;
+ for (int i = 0; i < fd->nb_side_data; i++) {
+ const AVSideDataDescriptor *desc = av_frame_side_data_desc(fd->side_data[i]->type);
ret = av_frame_side_data_clone(&enc_ctx->decoded_side_data,
&enc_ctx->nb_decoded_side_data,
- frame->side_data[i],
+ fd->side_data[i],
AV_FRAME_SIDE_DATA_FLAG_UNIQUE);
if (ret < 0)
return ret;
diff --git a/fftools/ffmpeg_filter.c b/fftools/ffmpeg_filter.c
index cb4c792cb0..7a4b8417de 100644
--- a/fftools/ffmpeg_filter.c
+++ b/fftools/ffmpeg_filter.c
@@ -2129,7 +2129,8 @@ static int ifilter_parameters_from_frame(InputFilter *ifilter, const AVFrame *fr
for (int i = 0; i < frame->nb_side_data; i++) {
const AVSideDataDescriptor *desc = av_frame_side_data_desc(frame->side_data[i]->type);
- if (!(desc->props & AV_SIDE_DATA_PROP_GLOBAL))
+ if (!(desc->props & AV_SIDE_DATA_PROP_GLOBAL) ||
+ frame->side_data[i]->type == AV_FRAME_DATA_DISPLAYMATRIX)
continue;
ret = av_frame_side_data_clone(&ifp->side_data,
@@ -2502,16 +2503,17 @@ static int close_output(OutputFilterPriv *ofp, FilterGraphThread *fgt)
if (ret < 0)
return ret;
}
- av_frame_side_data_free(&frame->side_data, &frame->nb_side_data);
- ret = clone_side_data(&frame->side_data, &frame->nb_side_data,
- ofp->side_data, ofp->nb_side_data, 0);
- if (ret < 0)
- return ret;
fd = frame_data(frame);
if (!fd)
return AVERROR(ENOMEM);
+ av_frame_side_data_free(&fd->side_data, &fd->nb_side_data);
+ ret = clone_side_data(&fd->side_data, &fd->nb_side_data,
+ ofp->side_data, ofp->nb_side_data, 0);
+ if (ret < 0)
+ return ret;
+
fd->frame_rate_filter = ofp->fps.framerate;
av_assert0(!frame->buf[0]);
@@ -2666,6 +2668,12 @@ static int fg_output_step(OutputFilterPriv *ofp, FilterGraphThread *fgt,
return AVERROR(ENOMEM);
}
+ av_frame_side_data_free(&fd->side_data, &fd->nb_side_data);
+ ret = clone_side_data(&fd->side_data, &fd->nb_side_data,
+ ofp->side_data, ofp->nb_side_data, 0);
+ if (ret < 0)
+ return ret;
+
fd->wallclock[LATENCY_PROBE_FILTER_POST] = av_gettime_relative();
// only use bits_per_raw_sample passed through from the decoder
--
2.49.1
>From f4ee7115cc1e9e265cacbd7fab6624d9eab0e92e Mon Sep 17 00:00:00 2001
From: James Almer <jamrial@gmail.com>
Date: Sat, 18 Oct 2025 20:46:14 -0300
Subject: [PATCH 04/12] fftools/ffmpeg_opt: add helpers to match stream groups
Will be used to check for specifiers that match a given stream group and not
a stream within one.
Signed-off-by: James Almer <jamrial@gmail.com>
---
fftools/cmdutils.c | 71 +++++++++++++++++++++++++++++++++++
fftools/cmdutils.h | 4 ++
fftools/ffmpeg.h | 10 +++++
fftools/ffmpeg_opt.c | 89 ++++++++++++++++++++++++++++++++++++++++++++
4 files changed, 174 insertions(+)
diff --git a/fftools/cmdutils.c b/fftools/cmdutils.c
index dc093b0bd3..6a7962d84d 100644
--- a/fftools/cmdutils.c
+++ b/fftools/cmdutils.c
@@ -1350,6 +1350,77 @@ int check_stream_specifier(AVFormatContext *s, AVStream *st, const char *spec)
return ret;
}
+unsigned stream_group_specifier_match(const StreamSpecifier *ss,
+ const AVFormatContext *s, const AVStreamGroup *stg,
+ void *logctx)
+{
+ int start_stream_group = 0, nb_stream_groups;
+ int nb_matched = 0;
+
+ if (ss->idx >= 0)
+ return 0;
+
+ switch (ss->stream_list) {
+ case STREAM_LIST_STREAM_ID:
+ case STREAM_LIST_ALL:
+ case STREAM_LIST_PROGRAM:
+ return 0;
+ case STREAM_LIST_GROUP_ID:
+ // <n-th> stream with given ID makes no sense and should be impossible to request
+ av_assert0(ss->idx < 0);
+ // return early if we know for sure the stream does not match
+ if (stg->id != ss->list_id)
+ return 0;
+ start_stream_group = stg->index;
+ nb_stream_groups = stg->index + 1;
+ break;
+ case STREAM_LIST_GROUP_IDX:
+ start_stream_group = ss->list_id >= 0 ? 0 : stg->index;
+ nb_stream_groups = stg->index + 1;
+ break;
+ default: av_assert0(0);
+ }
+
+ for (int i = start_stream_group; i < nb_stream_groups; i++) {
+ const AVStreamGroup *candidate = s->stream_groups[i];
+
+ if (ss->meta_key) {
+ const AVDictionaryEntry *tag = av_dict_get(candidate->metadata,
+ ss->meta_key, NULL, 0);
+
+ if (!tag)
+ continue;
+ if (ss->meta_val && strcmp(tag->value, ss->meta_val))
+ continue;
+ }
+
+ if (ss->usable_only) {
+ switch (candidate->type) {
+ case AV_STREAM_GROUP_PARAMS_TILE_GRID: {
+ const AVStreamGroupTileGrid *tg = candidate->params.tile_grid;
+ if (!tg->coded_width || !tg->coded_height || !tg->nb_tiles ||
+ !tg->width || !tg->height || !tg->nb_tiles)
+ continue;
+ break;
+ }
+ default:
+ continue;
+ }
+ }
+
+ if (ss->disposition &&
+ (candidate->disposition & ss->disposition) != ss->disposition)
+ continue;
+
+ if (stg == candidate)
+ return ss->list_id < 0 || ss->list_id == nb_matched;
+
+ nb_matched++;
+ }
+
+ return 0;
+}
+
int filter_codec_opts(const AVDictionary *opts, enum AVCodecID codec_id,
AVFormatContext *s, AVStream *st, const AVCodec *codec,
AVDictionary **dst, AVDictionary **opts_used)
diff --git a/fftools/cmdutils.h b/fftools/cmdutils.h
index ad020f893a..c74a4b429e 100644
--- a/fftools/cmdutils.h
+++ b/fftools/cmdutils.h
@@ -158,6 +158,10 @@ unsigned stream_specifier_match(const StreamSpecifier *ss,
const AVFormatContext *s, const AVStream *st,
void *logctx);
+unsigned stream_group_specifier_match(const StreamSpecifier *ss,
+ const AVFormatContext *s, const AVStreamGroup *stg,
+ void *logctx);
+
void stream_specifier_uninit(StreamSpecifier *ss);
typedef struct SpecifierOpt {
diff --git a/fftools/ffmpeg.h b/fftools/ffmpeg.h
index a3d7f7ab34..27b3ca7be6 100644
--- a/fftools/ffmpeg.h
+++ b/fftools/ffmpeg.h
@@ -136,6 +136,7 @@ typedef struct StreamMap {
int disabled; /* 1 is this mapping is disabled by a negative map */
int file_index;
int stream_index;
+ int group_index;
char *linklabel; /* name of an output link, for mapping lavfi outputs */
ViewSpecifier vs;
@@ -932,6 +933,15 @@ void opt_match_per_stream_int64(void *logctx, const SpecifierOptList *sol,
void opt_match_per_stream_dbl(void *logctx, const SpecifierOptList *sol,
AVFormatContext *fc, AVStream *st, double *out);
+void opt_match_per_stream_group_str(void *logctx, const SpecifierOptList *sol,
+ AVFormatContext *fc, AVStreamGroup *stg, const char **out);
+void opt_match_per_stream_group_int(void *logctx, const SpecifierOptList *sol,
+ AVFormatContext *fc, AVStreamGroup *stg, int *out);
+void opt_match_per_stream_group_int64(void *logctx, const SpecifierOptList *sol,
+ AVFormatContext *fc, AVStreamGroup *stg, int64_t *out);
+void opt_match_per_stream_group_dbl(void *logctx, const SpecifierOptList *sol,
+ AVFormatContext *fc, AVStreamGroup *stg, double *out);
+
int view_specifier_parse(const char **pspec, ViewSpecifier *vs);
int muxer_thread(void *arg);
diff --git a/fftools/ffmpeg_opt.c b/fftools/ffmpeg_opt.c
index 926a8bda00..a9c870d94f 100644
--- a/fftools/ffmpeg_opt.c
+++ b/fftools/ffmpeg_opt.c
@@ -242,6 +242,70 @@ OPT_MATCH_PER_STREAM(int, int, OPT_TYPE_INT, i);
OPT_MATCH_PER_STREAM(int64, int64_t, OPT_TYPE_INT64, i64);
OPT_MATCH_PER_STREAM(dbl, double, OPT_TYPE_DOUBLE, dbl);
+static unsigned opt_match_per_stream_group(void *logctx, enum OptionType type,
+ const SpecifierOptList *sol,
+ AVFormatContext *fc, AVStreamGroup *stg)
+{
+ int matches = 0, match_idx = -1;
+
+ av_assert0((type == sol->type) || !sol->nb_opt);
+
+ for (int i = 0; i < sol->nb_opt; i++) {
+ const StreamSpecifier *ss = &sol->opt[i].stream_spec;
+
+ if (stream_group_specifier_match(ss, fc, stg, logctx)) {
+ match_idx = i;
+ matches++;
+ }
+ }
+
+ if (matches > 1 && sol->opt_canon) {
+ const SpecifierOpt *so = &sol->opt[match_idx];
+ const char *spec = so->specifier && so->specifier[0] ? so->specifier : "";
+
+ char namestr[128] = "";
+ char optval_buf[32];
+ const char *optval = optval_buf;
+
+ snprintf(namestr, sizeof(namestr), "-%s", sol->opt_canon->name);
+ if (sol->opt_canon->flags & OPT_HAS_ALT) {
+ const char * const *names_alt = sol->opt_canon->u1.names_alt;
+ for (int i = 0; names_alt[i]; i++)
+ av_strlcatf(namestr, sizeof(namestr), "/-%s", names_alt[i]);
+ }
+
+ switch (sol->type) {
+ case OPT_TYPE_STRING: optval = so->u.str; break;
+ case OPT_TYPE_INT: snprintf(optval_buf, sizeof(optval_buf), "%d", so->u.i); break;
+ case OPT_TYPE_INT64: snprintf(optval_buf, sizeof(optval_buf), "%"PRId64, so->u.i64); break;
+ case OPT_TYPE_FLOAT: snprintf(optval_buf, sizeof(optval_buf), "%f", so->u.f); break;
+ case OPT_TYPE_DOUBLE: snprintf(optval_buf, sizeof(optval_buf), "%f", so->u.dbl); break;
+ default: av_assert0(0);
+ }
+
+ av_log(logctx, AV_LOG_WARNING, "Multiple %s options specified for "
+ "stream group %d, only the last option '-%s%s%s %s' will be used.\n",
+ namestr, stg->index, sol->opt_canon->name, spec[0] ? ":" : "",
+ spec, optval);
+ }
+
+ return match_idx + 1;
+}
+
+#define OPT_MATCH_PER_STREAM_GROUP(name, type, opt_type, m) \
+void opt_match_per_stream_group_ ## name(void *logctx, const SpecifierOptList *sol, \
+ AVFormatContext *fc, AVStreamGroup *stg, type *out) \
+{ \
+ unsigned ret = opt_match_per_stream_group(logctx, opt_type, sol, fc, stg); \
+ if (ret > 0) \
+ *out = sol->opt[ret - 1].u.m; \
+}
+
+OPT_MATCH_PER_STREAM_GROUP(str, const char *, OPT_TYPE_STRING, str);
+OPT_MATCH_PER_STREAM_GROUP(int, int, OPT_TYPE_INT, i);
+OPT_MATCH_PER_STREAM_GROUP(int64, int64_t, OPT_TYPE_INT64, i64);
+OPT_MATCH_PER_STREAM_GROUP(dbl, double, OPT_TYPE_DOUBLE, dbl);
+
int view_specifier_parse(const char **pspec, ViewSpecifier *vs)
{
const char *spec = *pspec;
@@ -504,8 +568,10 @@ static int opt_map(void *optctx, const char *opt, const char *arg)
}
if (arg[0] == '[') {
+ ViewSpecifier vs;
/* this mapping refers to lavfi output */
const char *c = arg + 1;
+ char *endptr;
ret = GROW_ARRAY(o->stream_maps, o->nb_stream_maps);
if (ret < 0)
@@ -518,6 +584,27 @@ static int opt_map(void *optctx, const char *opt, const char *arg)
ret = AVERROR(EINVAL);
goto fail;
}
+
+ arg++;
+
+ m->group_index = -1;
+ file_idx = strtol(arg, &endptr, 0);
+ if (file_idx >= nb_input_files || file_idx < 0)
+ goto end;
+
+ arg = endptr;
+ ret = stream_specifier_parse(&ss, *arg == ':' ? arg + 1 : arg, 1, NULL);
+ if (ret < 0)
+ goto end;
+
+ arg = ss.remainder ? ss.remainder : "";
+ ret = view_specifier_parse(&arg, &vs);
+ if (ret < 0 || (*arg && strcmp(arg, "]")))
+ goto end;
+
+ m->file_index = file_idx;
+ m->stream_index = ss.idx;
+ m->group_index = ss.stream_list == STREAM_LIST_GROUP_IDX ? ss.list_id : -1;
} else {
ViewSpecifier vs;
char *endptr;
@@ -583,6 +670,7 @@ static int opt_map(void *optctx, const char *opt, const char *arg)
m->file_index = file_idx;
m->stream_index = i;
+ m->group_index = ss.stream_list == STREAM_LIST_GROUP_IDX ? ss.list_id : -1;
m->vs = vs;
}
}
@@ -602,6 +690,7 @@ static int opt_map(void *optctx, const char *opt, const char *arg)
goto fail;
}
}
+end:
ret = 0;
fail:
stream_specifier_uninit(&ss);
--
2.49.1
>From 7664d0424638f052576c5d06e89efc08ad9a3be1 Mon Sep 17 00:00:00 2001
From: James Almer <jamrial@gmail.com>
Date: Sat, 18 Oct 2025 20:58:21 -0300
Subject: [PATCH 05/12] fftools/ffmpeg_demux: add InputStreamGroup to store
stream groups
Preparatory work for upcoming changes. Make ffmpeg keep track of stream groups
internally.
Signed-off-by: James Almer <jamrial@gmail.com>
---
fftools/ffmpeg.h | 15 +++++++++
fftools/ffmpeg_demux.c | 73 ++++++++++++++++++++++++++++++++++++++++++
2 files changed, 88 insertions(+)
diff --git a/fftools/ffmpeg.h b/fftools/ffmpeg.h
index 27b3ca7be6..30f39b23a9 100644
--- a/fftools/ffmpeg.h
+++ b/fftools/ffmpeg.h
@@ -492,6 +492,17 @@ typedef struct InputStream {
int nb_filters;
} InputStream;
+typedef struct InputStreamGroup {
+ const AVClass *class;
+
+ /* parent source */
+ struct InputFile *file;
+
+ int index;
+
+ AVStreamGroup *stg;
+} InputStreamGroup;
+
typedef struct InputFile {
const AVClass *class;
@@ -513,6 +524,10 @@ typedef struct InputFile {
* if new streams appear dynamically during demuxing */
InputStream **streams;
int nb_streams;
+
+ /* stream groups that ffmpeg is aware of; */
+ InputStreamGroup **stream_groups;
+ int nb_stream_groups;
} InputFile;
enum forced_keyframes_const {
diff --git a/fftools/ffmpeg_demux.c b/fftools/ffmpeg_demux.c
index d2f0017aeb..6c9834746f 100644
--- a/fftools/ffmpeg_demux.c
+++ b/fftools/ffmpeg_demux.c
@@ -104,6 +104,13 @@ typedef struct DemuxStream {
int64_t lag;
} DemuxStream;
+typedef struct DemuxStreamGroup {
+ InputStreamGroup istg;
+
+ // name used for logging
+ char log_name[32];
+} DemuxStreamGroup;
+
typedef struct Demuxer {
InputFile f;
@@ -886,6 +893,16 @@ static void ist_free(InputStream **pist)
av_freep(pist);
}
+static void istg_free(InputStreamGroup **pistg)
+{
+ InputStreamGroup *istg = *pistg;
+
+ if (!istg)
+ return;
+
+ av_freep(pistg);
+}
+
void ifile_close(InputFile **pf)
{
InputFile *f = *pf;
@@ -901,6 +918,10 @@ void ifile_close(InputFile **pf)
ist_free(&f->streams[i]);
av_freep(&f->streams);
+ for (int i = 0; i < f->nb_stream_groups; i++)
+ istg_free(&f->stream_groups[i]);
+ av_freep(&f->stream_groups);
+
avformat_close_input(&f->ctx);
av_packet_free(&d->pkt_heartbeat);
@@ -1586,6 +1607,51 @@ static int ist_add(const OptionsContext *o, Demuxer *d, AVStream *st, AVDictiona
return 0;
}
+static const char *input_stream_group_item_name(void *obj)
+{
+ const DemuxStreamGroup *dsg = obj;
+
+ return dsg->log_name;
+}
+
+static const AVClass input_stream_group_class = {
+ .class_name = "InputStreamGroup",
+ .version = LIBAVUTIL_VERSION_INT,
+ .item_name = input_stream_group_item_name,
+ .category = AV_CLASS_CATEGORY_DEMUXER,
+};
+
+static DemuxStreamGroup *demux_stream_group_alloc(Demuxer *d, AVStreamGroup *stg)
+{
+ InputFile *f = &d->f;
+ DemuxStreamGroup *dsg;
+
+ dsg = allocate_array_elem(&f->stream_groups, sizeof(*dsg), &f->nb_stream_groups);
+ if (!dsg)
+ return NULL;
+
+ dsg->istg.stg = stg;
+ dsg->istg.file = f;
+ dsg->istg.index = stg->index;
+ dsg->istg.class = &input_stream_group_class;
+
+ snprintf(dsg->log_name, sizeof(dsg->log_name), "istg#%d:%d/%s",
+ d->f.index, stg->index, avformat_stream_group_name(stg->type));
+
+ return dsg;
+}
+
+static int istg_add(const OptionsContext *o, Demuxer *d, AVStreamGroup *stg)
+{
+ DemuxStreamGroup *dsg;
+
+ dsg = demux_stream_group_alloc(d, stg);
+ if (!dsg)
+ return AVERROR(ENOMEM);
+
+ return 0;
+}
+
static int dump_attachment(InputStream *ist, const char *filename)
{
AVStream *st = ist->st;
@@ -1954,6 +2020,13 @@ int ifile_open(const OptionsContext *o, const char *filename, Scheduler *sch)
}
}
+ /* Add all the stream groups from the given input file to the demuxer */
+ for (int i = 0; i < ic->nb_stream_groups; i++) {
+ ret = istg_add(o, d, ic->stream_groups[i]);
+ if (ret < 0)
+ return ret;
+ }
+
/* dump the file content */
av_dump_format(ic, f->index, filename, 0);
--
2.49.1
>From 35c9e4b1eb2f824e3201ab448db65c392b67f316 Mon Sep 17 00:00:00 2001
From: James Almer <jamrial@gmail.com>
Date: Sat, 18 Oct 2025 21:02:21 -0300
Subject: [PATCH 06/12] fftools/ffmpeg_mux_init: allow creating streams from
filtergraphs created out of stream groups
Several formats are being designed where more than one independent video or
audio stream within a container are part of what should be a single combined
output. This is the case for HEIF (Defining a grid where the decoded output
of several separate video streams are to be placed to generate a single output
image) and IAMF (Defining audio streams where channels are present in separate
coded stream).
AVStreamGroup was designed and implemented in libavformat to convey this
information, but the actual merging is left to the caller.
This change allows the FFmpeg CLI to take said information, parse it, and
create filtergraphs to merge the streams, making the combined output be usable
automatically further in the process.
Signed-off-by: James Almer <jamrial@gmail.com>
---
fftools/ffmpeg.h | 1 +
fftools/ffmpeg_enc.c | 2 --
fftools/ffmpeg_mux_init.c | 46 +++++++++++++++++++++++++++++++++++++++
3 files changed, 47 insertions(+), 2 deletions(-)
diff --git a/fftools/ffmpeg.h b/fftools/ffmpeg.h
index 30f39b23a9..7057fd8cc3 100644
--- a/fftools/ffmpeg.h
+++ b/fftools/ffmpeg.h
@@ -500,6 +500,7 @@ typedef struct InputStreamGroup {
int index;
+ FilterGraph *fg;
AVStreamGroup *stg;
} InputStreamGroup;
diff --git a/fftools/ffmpeg_enc.c b/fftools/ffmpeg_enc.c
index 978b247fc2..4c10ba011c 100644
--- a/fftools/ffmpeg_enc.c
+++ b/fftools/ffmpeg_enc.c
@@ -206,8 +206,6 @@ int enc_open(void *opaque, const AVFrame *frame)
fd = (FrameData*)frame->opaque_ref->data;
for (int i = 0; i < fd->nb_side_data; i++) {
- const AVSideDataDescriptor *desc = av_frame_side_data_desc(fd->side_data[i]->type);
-
ret = av_frame_side_data_clone(&enc_ctx->decoded_side_data,
&enc_ctx->nb_decoded_side_data,
fd->side_data[i],
diff --git a/fftools/ffmpeg_mux_init.c b/fftools/ffmpeg_mux_init.c
index e6c10edd62..3c31035d7c 100644
--- a/fftools/ffmpeg_mux_init.c
+++ b/fftools/ffmpeg_mux_init.c
@@ -1598,6 +1598,7 @@ fail:
static int map_auto_video(Muxer *mux, const OptionsContext *o)
{
AVFormatContext *oc = mux->fc;
+ InputStreamGroup *best_istg = NULL;
InputStream *best_ist = NULL;
int64_t best_score = 0;
int qcr;
@@ -1609,8 +1610,35 @@ static int map_auto_video(Muxer *mux, const OptionsContext *o)
qcr = avformat_query_codec(oc->oformat, oc->oformat->video_codec, 0);
for (int j = 0; j < nb_input_files; j++) {
InputFile *ifile = input_files[j];
+ InputStreamGroup *file_best_istg = NULL;
InputStream *file_best_ist = NULL;
int64_t file_best_score = 0;
+ for (int i = 0; i < ifile->nb_stream_groups; i++) {
+ InputStreamGroup *istg = ifile->stream_groups[i];
+ AVStreamGroupTileGrid *tg = NULL;
+ int64_t score = 0;
+
+ if (!istg->fg)
+ continue;
+
+ for (int j = 0; j < istg->stg->nb_streams; j++) {
+ AVStream *st = istg->stg->streams[j];
+
+ if (st->event_flags & AVSTREAM_EVENT_FLAG_NEW_PACKETS) {
+ score = 100000000;
+ break;
+ }
+ }
+
+ tg = istg->stg->params.tile_grid;
+ score += tg->width * (int64_t)tg->height
+ + 5000000*!!(istg->stg->disposition & AV_DISPOSITION_DEFAULT);
+
+ if (score > file_best_score) {
+ file_best_score = score;
+ file_best_istg = istg;
+ }
+ }
for (int i = 0; i < ifile->nb_streams; i++) {
InputStream *ist = ifile->streams[i];
int64_t score;
@@ -1630,6 +1658,14 @@ static int map_auto_video(Muxer *mux, const OptionsContext *o)
continue;
file_best_score = score;
file_best_ist = ist;
+ file_best_istg = NULL;
+ }
+ }
+ if (file_best_istg) {
+ if (file_best_score > best_score) {
+ best_score = file_best_score;
+ best_istg = file_best_istg;
+ best_ist = NULL;
}
}
if (file_best_ist) {
@@ -1639,9 +1675,19 @@ static int map_auto_video(Muxer *mux, const OptionsContext *o)
if (file_best_score > best_score) {
best_score = file_best_score;
best_ist = file_best_ist;
+ best_istg = NULL;
}
}
}
+ if (best_istg) {
+ FilterGraph *fg = best_istg->fg;
+ OutputFilter *ofilter = fg->outputs[0];
+
+ av_assert0(fg->nb_outputs == 1);
+ av_log(mux, AV_LOG_VERBOSE, "Creating output stream from stream group derived complex filtergraph %d.\n", fg->index);
+
+ return ost_add(mux, o, AVMEDIA_TYPE_VIDEO, NULL, ofilter, NULL, NULL);
+ }
if (best_ist)
return ost_add(mux, o, AVMEDIA_TYPE_VIDEO, best_ist, NULL, NULL, NULL);
--
2.49.1
>From d0d2642f9dc34d6dfaf098a34d3bb58f6d38c9b5 Mon Sep 17 00:00:00 2001
From: James Almer <jamrial@gmail.com>
Date: Sat, 18 Oct 2025 20:08:49 -0300
Subject: [PATCH 07/12] fftools/ffmpeg_sched: add a function to remove a
filtergraph from the scheduler
For the purpose of merging streams in a stream group, a filtergraph can't be
created once we know it will be used. Therefore, allow filtergraphs to be
removed from the scheduler after being added.
Signed-off-by: James Almer <jamrial@gmail.com>
---
fftools/ffmpeg_sched.c | 21 +++++++++++++++++++++
fftools/ffmpeg_sched.h | 2 ++
2 files changed, 23 insertions(+)
diff --git a/fftools/ffmpeg_sched.c b/fftools/ffmpeg_sched.c
index d08f4a061d..db91fbd3a7 100644
--- a/fftools/ffmpeg_sched.c
+++ b/fftools/ffmpeg_sched.c
@@ -405,6 +405,9 @@ static int task_start(SchTask *task)
{
int ret;
+ if (!task->parent)
+ return 0;
+
av_log(task->func_arg, AV_LOG_VERBOSE, "Starting thread...\n");
av_assert0(!task->thread_running);
@@ -454,6 +457,21 @@ static int64_t trailing_dts(const Scheduler *sch, int count_finished)
return min_dts == INT64_MAX ? AV_NOPTS_VALUE : min_dts;
}
+void sch_remove_filtergraph(Scheduler *sch, int idx)
+{
+ SchFilterGraph *fg = &sch->filters[idx];
+
+ av_assert0(!fg->task.thread_running);
+ memset(&fg->task, 0, sizeof(fg->task));
+
+ tq_free(&fg->queue);
+
+ av_freep(&fg->inputs);
+ fg->nb_inputs = 0;
+ av_freep(&fg->outputs);
+ fg->nb_outputs = 0;
+}
+
void sch_free(Scheduler **psch)
{
Scheduler *sch = *psch;
@@ -2630,6 +2648,9 @@ static int task_stop(Scheduler *sch, SchTask *task)
int ret;
void *thread_ret;
+ if (!task->parent)
+ return 0;
+
if (!task->thread_running)
return task_cleanup(sch, task->node);
diff --git a/fftools/ffmpeg_sched.h b/fftools/ffmpeg_sched.h
index 0c01f558e4..2cf3034437 100644
--- a/fftools/ffmpeg_sched.h
+++ b/fftools/ffmpeg_sched.h
@@ -205,6 +205,8 @@ int sch_add_dec_output(Scheduler *sch, unsigned dec_idx);
int sch_add_filtergraph(Scheduler *sch, unsigned nb_inputs, unsigned nb_outputs,
SchThreadFunc func, void *ctx);
+void sch_remove_filtergraph(Scheduler *sch, int idx);
+
/**
* Add a muxer to the scheduler.
*
--
2.49.1
>From 20a2ef363b178454bdaaa818d89c24ff1459790c Mon Sep 17 00:00:00 2001
From: James Almer <jamrial@gmail.com>
Date: Sat, 18 Oct 2025 21:06:25 -0300
Subject: [PATCH 08/12] fftools/ffmpeg_filter: allow removing filtergraphs if
it contains unbound outputs
Actual practical implementation of the previous commit.
Signed-off-by: James Almer <jamrial@gmail.com>
---
fftools/ffmpeg.h | 5 ++++
fftools/ffmpeg_filter.c | 57 ++++++++++++++++++++++++++++++++---------
2 files changed, 50 insertions(+), 12 deletions(-)
diff --git a/fftools/ffmpeg.h b/fftools/ffmpeg.h
index 7057fd8cc3..8d0ce6f897 100644
--- a/fftools/ffmpeg.h
+++ b/fftools/ffmpeg.h
@@ -403,6 +403,11 @@ typedef struct FilterGraph {
OutputFilter **outputs;
int nb_outputs;
+ // true when the filtergraph is created internally for
+ // purposes like stream group merging. Meant to be freed
+ // if unbound.
+ int is_internal;
+
const char *graph_desc;
struct AVBPrint graph_print_buf;
} FilterGraph;
diff --git a/fftools/ffmpeg_filter.c b/fftools/ffmpeg_filter.c
index 7a4b8417de..1b0ab00351 100644
--- a/fftools/ffmpeg_filter.c
+++ b/fftools/ffmpeg_filter.c
@@ -193,6 +193,8 @@ typedef struct OutputFilterPriv {
void *log_parent;
char log_name[32];
+ int needed;
+
/* desired output stream properties */
int format;
int width, height;
@@ -813,7 +815,7 @@ int ofilter_bind_enc(OutputFilter *ofilter, unsigned sched_idx_enc,
av_assert0(!opts->enc ||
ofilter->type == opts->enc->type);
- ofilter->bound = 1;
+ ofp->needed = ofilter->bound = 1;
av_freep(&ofilter->linklabel);
ofp->flags = opts->flags;
@@ -924,7 +926,7 @@ static int ofilter_bind_ifilter(OutputFilter *ofilter, InputFilterPriv *ifp,
av_assert0(!ofilter->bound);
av_assert0(ofilter->type == ifp->ifilter.type);
- ofilter->bound = 1;
+ ofp->needed = ofilter->bound = 1;
av_freep(&ofilter->linklabel);
ofilter->output_name = av_strdup(opts->name);
@@ -1264,7 +1266,7 @@ int fg_create_simple(FilterGraph **pfg,
return 0;
}
-static int fg_complex_bind_input(FilterGraph *fg, InputFilter *ifilter)
+static int fg_complex_bind_input(FilterGraph *fg, InputFilter *ifilter, int commit)
{
InputFilterPriv *ifp = ifp_from_ifilter(ifilter);
InputStream *ist = NULL;
@@ -1315,15 +1317,20 @@ static int fg_complex_bind_input(FilterGraph *fg, InputFilter *ifilter)
if (!ofilter->bound && ofilter->linklabel &&
!strcmp(ofilter->linklabel, ifilter->linklabel)) {
+ if (commit) {
av_log(fg, AV_LOG_VERBOSE,
"Binding input with label '%s' to filtergraph output %d:%d\n",
ifilter->linklabel, i, j);
ret = ifilter_bind_fg(ifp, fg_src, j);
- if (ret < 0)
+ if (ret < 0) {
av_log(fg, AV_LOG_ERROR, "Error binding filtergraph input %s\n",
ifilter->linklabel);
return ret;
+ }
+ } else
+ ofp_from_ofilter(ofilter)->needed = 1;
+ return 0;
}
}
}
@@ -1371,6 +1378,7 @@ static int fg_complex_bind_input(FilterGraph *fg, InputFilter *ifilter)
}
ist = input_files[file_idx]->streams[st->index];
+ if (commit)
av_log(fg, AV_LOG_VERBOSE,
"Binding input with label '%s' to input stream %d:%d\n",
ifilter->linklabel, ist->file->index, ist->index);
@@ -1384,12 +1392,14 @@ static int fg_complex_bind_input(FilterGraph *fg, InputFilter *ifilter)
return AVERROR(EINVAL);
}
+ if (commit)
av_log(fg, AV_LOG_VERBOSE,
"Binding unlabeled input %d to input stream %d:%d\n",
ifilter->index, ist->file->index, ist->index);
}
av_assert0(ist);
+ if (commit) {
ret = ifilter_bind_ist(ifilter, ist, &vs);
if (ret < 0) {
av_log(fg, AV_LOG_ERROR,
@@ -1397,11 +1407,12 @@ static int fg_complex_bind_input(FilterGraph *fg, InputFilter *ifilter)
ifilter->name);
return ret;
}
+ }
return 0;
}
-static int bind_inputs(FilterGraph *fg)
+static int bind_inputs(FilterGraph *fg, int commit)
{
// bind filtergraph inputs to input streams or other filtergraphs
for (int i = 0; i < fg->nb_inputs; i++) {
@@ -1411,7 +1422,7 @@ static int bind_inputs(FilterGraph *fg)
if (ifp->bound)
continue;
- ret = fg_complex_bind_input(fg, &ifp->ifilter);
+ ret = fg_complex_bind_input(fg, &ifp->ifilter, commit);
if (ret < 0)
return ret;
}
@@ -1424,27 +1435,49 @@ int fg_finalise_bindings(void)
int ret;
for (int i = 0; i < nb_filtergraphs; i++) {
- ret = bind_inputs(filtergraphs[i]);
+ ret = bind_inputs(filtergraphs[i], 0);
if (ret < 0)
return ret;
}
// check that all outputs were bound
- for (int i = 0; i < nb_filtergraphs; i++) {
+ for (int i = nb_filtergraphs - 1; i >= 0; i--) {
FilterGraph *fg = filtergraphs[i];
+ FilterGraphPriv *fgp = fgp_from_fg(filtergraphs[i]);
for (int j = 0; j < fg->nb_outputs; j++) {
OutputFilter *output = fg->outputs[j];
- if (!output->bound) {
- av_log(fg, AV_LOG_FATAL,
- "Filter '%s' has output %d (%s) unconnected\n",
+ if (!ofp_from_ofilter(output)->needed) {
+ if (!fg->is_internal) {
+ av_log(fg, AV_LOG_FATAL,
+ "Filter '%s' has output %d (%s) unconnected\n",
+ output->name, j,
+ output->linklabel ? (const char *)output->linklabel : "unlabeled");
+ return AVERROR(EINVAL);
+ }
+
+ av_log(fg, AV_LOG_DEBUG,
+ "Internal filter '%s' has output %d (%s) unconnected. Removing graph\n",
output->name, j,
output->linklabel ? (const char *)output->linklabel : "unlabeled");
- return AVERROR(EINVAL);
+ sch_remove_filtergraph(fgp->sch, fgp->sch_idx);
+ fg_free(&filtergraphs[i]);
+ nb_filtergraphs--;
+ if (nb_filtergraphs > 0)
+ memmove(&filtergraphs[i],
+ &filtergraphs[i + 1],
+ (nb_filtergraphs - i) * sizeof(*filtergraphs));
+ break;
}
}
}
+ for (int i = 0; i < nb_filtergraphs; i++) {
+ ret = bind_inputs(filtergraphs[i], 1);
+ if (ret < 0)
+ return ret;
+ }
+
return 0;
}
--
2.49.1
>From 7720ce6643feb109e53fe3e0395eae161ef548b1 Mon Sep 17 00:00:00 2001
From: James Almer <jamrial@gmail.com>
Date: Sat, 18 Oct 2025 21:40:24 -0300
Subject: [PATCH 09/12] fftools/ffmpeg_filter: reindent after the previous
commit
Signed-off-by: James Almer <jamrial@gmail.com>
---
fftools/ffmpeg_filter.c | 44 ++++++++++++++++++++---------------------
1 file changed, 22 insertions(+), 22 deletions(-)
diff --git a/fftools/ffmpeg_filter.c b/fftools/ffmpeg_filter.c
index 1b0ab00351..a17f4ba4ca 100644
--- a/fftools/ffmpeg_filter.c
+++ b/fftools/ffmpeg_filter.c
@@ -1318,16 +1318,16 @@ static int fg_complex_bind_input(FilterGraph *fg, InputFilter *ifilter, int comm
if (!ofilter->bound && ofilter->linklabel &&
!strcmp(ofilter->linklabel, ifilter->linklabel)) {
if (commit) {
- av_log(fg, AV_LOG_VERBOSE,
- "Binding input with label '%s' to filtergraph output %d:%d\n",
- ifilter->linklabel, i, j);
+ av_log(fg, AV_LOG_VERBOSE,
+ "Binding input with label '%s' to filtergraph output %d:%d\n",
+ ifilter->linklabel, i, j);
- ret = ifilter_bind_fg(ifp, fg_src, j);
- if (ret < 0) {
- av_log(fg, AV_LOG_ERROR, "Error binding filtergraph input %s\n",
- ifilter->linklabel);
- return ret;
- }
+ ret = ifilter_bind_fg(ifp, fg_src, j);
+ if (ret < 0) {
+ av_log(fg, AV_LOG_ERROR, "Error binding filtergraph input %s\n",
+ ifilter->linklabel);
+ return ret;
+ }
} else
ofp_from_ofilter(ofilter)->needed = 1;
return 0;
@@ -1379,9 +1379,9 @@ static int fg_complex_bind_input(FilterGraph *fg, InputFilter *ifilter, int comm
ist = input_files[file_idx]->streams[st->index];
if (commit)
- av_log(fg, AV_LOG_VERBOSE,
- "Binding input with label '%s' to input stream %d:%d\n",
- ifilter->linklabel, ist->file->index, ist->index);
+ av_log(fg, AV_LOG_VERBOSE,
+ "Binding input with label '%s' to input stream %d:%d\n",
+ ifilter->linklabel, ist->file->index, ist->index);
} else {
ist = ist_find_unused(type);
if (!ist) {
@@ -1393,20 +1393,20 @@ static int fg_complex_bind_input(FilterGraph *fg, InputFilter *ifilter, int comm
}
if (commit)
- av_log(fg, AV_LOG_VERBOSE,
- "Binding unlabeled input %d to input stream %d:%d\n",
- ifilter->index, ist->file->index, ist->index);
+ av_log(fg, AV_LOG_VERBOSE,
+ "Binding unlabeled input %d to input stream %d:%d\n",
+ ifilter->index, ist->file->index, ist->index);
}
av_assert0(ist);
if (commit) {
- ret = ifilter_bind_ist(ifilter, ist, &vs);
- if (ret < 0) {
- av_log(fg, AV_LOG_ERROR,
- "Error binding an input stream to complex filtergraph input %s.\n",
- ifilter->name);
- return ret;
- }
+ ret = ifilter_bind_ist(ifilter, ist, &vs);
+ if (ret < 0) {
+ av_log(fg, AV_LOG_ERROR,
+ "Error binding an input stream to complex filtergraph input %s.\n",
+ ifilter->name);
+ return ret;
+ }
}
return 0;
--
2.49.1
>From 2d619f4424d1db6f8f30dbdecf2759c1ab96ed1d Mon Sep 17 00:00:00 2001
From: James Almer <jamrial@gmail.com>
Date: Sat, 18 Oct 2025 21:37:33 -0300
Subject: [PATCH 10/12] fftools/ffmpeg_filter: allow binding unlabeled
filtergraphs
Signed-off-by: James Almer <jamrial@gmail.com>
---
fftools/ffmpeg_filter.c | 27 +++++++++++++++++++++++++++
1 file changed, 27 insertions(+)
diff --git a/fftools/ffmpeg_filter.c b/fftools/ffmpeg_filter.c
index a17f4ba4ca..20b12dc130 100644
--- a/fftools/ffmpeg_filter.c
+++ b/fftools/ffmpeg_filter.c
@@ -1383,6 +1383,33 @@ static int fg_complex_bind_input(FilterGraph *fg, InputFilter *ifilter, int comm
"Binding input with label '%s' to input stream %d:%d\n",
ifilter->linklabel, ist->file->index, ist->index);
} else {
+ // try finding an unbound filtergraph output
+ for (int i = 0; i < nb_filtergraphs; i++) {
+ FilterGraph *fg_src = filtergraphs[i];
+
+ if (fg == fg_src)
+ continue;
+
+ for (int j = 0; j < fg_src->nb_outputs; j++) {
+ OutputFilter *ofilter = fg_src->outputs[j];
+
+ if (!ofilter->bound) {
+ if (commit) {
+ av_log(fg, AV_LOG_VERBOSE,
+ "Binding unlabeled filtergraph input to filtergraph output %d:%d\n", i, j);
+
+ ret = ifilter_bind_fg(ifp, fg_src, j);
+ if (ret < 0) {
+ av_log(fg, AV_LOG_ERROR, "Error binding filtergraph input %d:%d\n", i, j);
+ return ret;
+ }
+ } else
+ ofp_from_ofilter(ofilter)->needed = 1;
+ return 0;
+ }
+ }
+ }
+
ist = ist_find_unused(type);
if (!ist) {
av_log(fg, AV_LOG_FATAL,
--
2.49.1
>From ef30f5e6e82ddb6ba284b777c082b4484ee08d79 Mon Sep 17 00:00:00 2001
From: James Almer <jamrial@gmail.com>
Date: Sat, 18 Oct 2025 21:20:43 -0300
Subject: [PATCH 11/12] fftools/ffmpeg_demux: create a filtegraph to merge HEIF
tiles automatically
Examples:
ffmpeg -i fate-suite/heif-conformance/C007.heic out.png
ffmpeg -i fate-suite/heif-conformance/C007.heic -map [0:g:0] out.png
ffmpeg -i fate-suite/heif-conformance/C007.heic -filter_complex "[0:g:0]scale=w=1920:h=1080[out]" -map [out] out.png
Signed-off-by: James Almer <jamrial@gmail.com>
---
fftools/ffmpeg.h | 7 +++-
fftools/ffmpeg_demux.c | 71 +++++++++++++++++++++++++++++++++++++++++
fftools/ffmpeg_filter.c | 21 +++++++++---
fftools/ffmpeg_opt.c | 2 +-
4 files changed, 94 insertions(+), 7 deletions(-)
diff --git a/fftools/ffmpeg.h b/fftools/ffmpeg.h
index 8d0ce6f897..c566a3bb5c 100644
--- a/fftools/ffmpeg.h
+++ b/fftools/ffmpeg.h
@@ -300,6 +300,7 @@ enum OFilterFlags {
// produce 24-bit audio
OFILTER_FLAG_AUDIO_24BIT = (1 << 1),
OFILTER_FLAG_AUTOSCALE = (1 << 2),
+ OFILTER_FLAG_AUTOROTATE = (1 << 3),
};
typedef struct OutputFilterOptions {
@@ -348,6 +349,9 @@ typedef struct OutputFilterOptions {
const enum AVColorRange *color_ranges;
const enum AVAlphaMode *alpha_modes;
+ AVFrameSideData **side_data;
+ int nb_side_data;
+
// for simple filtergraphs only, view specifier passed
// along to the decoder
const ViewSpecifier *vs;
@@ -827,7 +831,8 @@ int ofilter_bind_enc(OutputFilter *ofilter,
* @param graph_desc Graph description; an av_malloc()ed string, filtergraph
* takes ownership of it.
*/
-int fg_create(FilterGraph **pfg, char *graph_desc, Scheduler *sch);
+int fg_create(FilterGraph **pfg, char *graph_desc, Scheduler *sch,
+ const OutputFilterOptions *opts);
void fg_free(FilterGraph **pfg);
diff --git a/fftools/ffmpeg_demux.c b/fftools/ffmpeg_demux.c
index 6c9834746f..7c9597bdba 100644
--- a/fftools/ffmpeg_demux.c
+++ b/fftools/ffmpeg_demux.c
@@ -1641,14 +1641,85 @@ static DemuxStreamGroup *demux_stream_group_alloc(Demuxer *d, AVStreamGroup *stg
return dsg;
}
+static int istg_parse_tile_grid(Demuxer *d, InputStreamGroup *istg, int autorotate)
+{
+ InputFile *f = &d->f;
+ const AVStreamGroup *stg = istg->stg;
+ const AVStreamGroupTileGrid *tg = stg->params.tile_grid;
+ AVBPrint bp;
+ char *graph_str;
+ int ret;
+
+ OutputFilterOptions opts = {
+ .flags = OFILTER_FLAG_AUTOROTATE * !!autorotate,
+ };
+
+ if (tg->nb_tiles == 1)
+ return 0;
+
+ av_bprint_init(&bp, 0, AV_BPRINT_SIZE_UNLIMITED);
+ for (int i = 0; i < tg->nb_tiles; i++)
+ av_bprintf(&bp, "[%d:g:%d:%d]", f->index, stg->index, tg->offsets[i].idx);
+ av_bprintf(&bp, "xstack=inputs=%d:layout=", tg->nb_tiles);
+ for (int i = 0; i < tg->nb_tiles - 1; i++)
+ av_bprintf(&bp, "%d_%d|", tg->offsets[i].horizontal,
+ tg->offsets[i].vertical);
+ av_bprintf(&bp, "%d_%d:fill=0x%02X%02X%02X@0x%02X", tg->offsets[tg->nb_tiles - 1].horizontal,
+ tg->offsets[tg->nb_tiles - 1].vertical,
+ tg->background[0], tg->background[1],
+ tg->background[2], tg->background[3]);
+ if (tg->coded_width != tg->width || tg->coded_height != tg->height)
+ av_bprintf(&bp, ",crop=w=%d:h=%d:x=%d:y=%d", tg->width, tg->height,
+ tg->horizontal_offset, tg->vertical_offset);
+ av_bprintf(&bp, "[%d:g:%d]", f->index, stg->index);
+ ret = av_bprint_finalize(&bp, &graph_str);
+ if (ret < 0)
+ return ret;
+
+ for (int i = 0; i < tg->nb_coded_side_data; i++) {
+ const AVPacketSideData *sd = &tg->coded_side_data[i];
+
+ ret = av_packet_side_data_to_frame(&opts.side_data, &opts.nb_side_data, sd, 0);
+ if (ret < 0 && ret != AVERROR(EINVAL))
+ return ret;
+ }
+
+ ret = fg_create(NULL, graph_str, d->sch, &opts);
+ if (ret < 0)
+ return ret;
+
+ istg->fg = filtergraphs[nb_filtergraphs-1];
+ istg->fg->is_internal = 1;
+
+ return 0;
+}
+
static int istg_add(const OptionsContext *o, Demuxer *d, AVStreamGroup *stg)
{
+ AVFormatContext *ic = d->f.ctx;
DemuxStreamGroup *dsg;
+ InputStreamGroup *istg;
+ int autorotate = 1;
+ int ret;
dsg = demux_stream_group_alloc(d, stg);
if (!dsg)
return AVERROR(ENOMEM);
+ istg = &dsg->istg;
+
+ opt_match_per_stream_group_int(istg, &o->autorotate, ic, stg, &autorotate);
+
+ switch (stg->type) {
+ case AV_STREAM_GROUP_PARAMS_TILE_GRID:
+ ret = istg_parse_tile_grid(d, istg, autorotate);
+ if (ret < 0)
+ return ret;
+ break;
+ default:
+ break;
+ }
+
return 0;
}
diff --git a/fftools/ffmpeg_filter.c b/fftools/ffmpeg_filter.c
index 20b12dc130..9450257a7c 100644
--- a/fftools/ffmpeg_filter.c
+++ b/fftools/ffmpeg_filter.c
@@ -818,7 +818,7 @@ int ofilter_bind_enc(OutputFilter *ofilter, unsigned sched_idx_enc,
ofp->needed = ofilter->bound = 1;
av_freep(&ofilter->linklabel);
- ofp->flags = opts->flags;
+ ofp->flags |= opts->flags;
ofp->ts_offset = opts->ts_offset;
ofp->enc_timebase = opts->output_tb;
@@ -1078,7 +1078,8 @@ static const AVClass fg_class = {
.category = AV_CLASS_CATEGORY_FILTER,
};
-int fg_create(FilterGraph **pfg, char *graph_desc, Scheduler *sch)
+int fg_create(FilterGraph **pfg, char *graph_desc, Scheduler *sch,
+ const OutputFilterOptions *opts)
{
FilterGraphPriv *fgp;
FilterGraph *fg;
@@ -1175,11 +1176,13 @@ int fg_create(FilterGraph **pfg, char *graph_desc, Scheduler *sch)
const enum AVMediaType type = avfilter_pad_get_type(cur->filter_ctx->output_pads,
cur->pad_idx);
OutputFilter *const ofilter = ofilter_alloc(fg, type);
+ OutputFilterPriv *ofp;
if (!ofilter) {
ret = AVERROR(ENOMEM);
goto fail;
}
+ ofp = ofp_from_ofilter(ofilter);
ofilter->linklabel = cur->name;
cur->name = NULL;
@@ -1189,6 +1192,15 @@ int fg_create(FilterGraph **pfg, char *graph_desc, Scheduler *sch)
ret = AVERROR(ENOMEM);
goto fail;
}
+
+ // opts should only be needed in this function to fill fields from filtergraphs
+ // whose output is meant to be treated as if it was stream, e.g. merged HEIF
+ // tile groups.
+ if (opts) {
+ ofp->flags = opts->flags;
+ ofp->side_data = opts->side_data;
+ ofp->nb_side_data = opts->nb_side_data;
+ }
}
if (!fg->nb_outputs) {
@@ -1225,7 +1237,7 @@ int fg_create_simple(FilterGraph **pfg,
FilterGraphPriv *fgp;
int ret;
- ret = fg_create(pfg, graph_desc, sch);
+ ret = fg_create(pfg, graph_desc, sch, NULL);
if (ret < 0)
return ret;
fg = *pfg;
@@ -2100,12 +2112,11 @@ static int configure_filtergraph(FilterGraph *fg, FilterGraphThread *fgt)
ret = av_buffersink_get_ch_layout(sink, &ofp->ch_layout);
if (ret < 0)
goto fail;
- av_frame_side_data_free(&ofp->side_data, &ofp->nb_side_data);
sd = av_buffersink_get_side_data(sink, &nb_sd);
if (nb_sd)
for (int j = 0; j < nb_sd; j++) {
ret = av_frame_side_data_clone(&ofp->side_data, &ofp->nb_side_data,
- sd[j], 0);
+ sd[j], AV_FRAME_SIDE_DATA_FLAG_REPLACE);
if (ret < 0) {
av_frame_side_data_free(&ofp->side_data, &ofp->nb_side_data);
goto fail;
diff --git a/fftools/ffmpeg_opt.c b/fftools/ffmpeg_opt.c
index a9c870d94f..e17fa21fd8 100644
--- a/fftools/ffmpeg_opt.c
+++ b/fftools/ffmpeg_opt.c
@@ -1496,7 +1496,7 @@ int ffmpeg_parse_options(int argc, char **argv, Scheduler *sch)
/* create complex filtergraphs */
for (int i = 0; i < go.nb_filtergraphs; i++) {
- ret = fg_create(NULL, go.filtergraphs[i], sch);
+ ret = fg_create(NULL, go.filtergraphs[i], sch, NULL);
go.filtergraphs[i] = NULL;
if (ret < 0)
goto fail;
--
2.49.1
>From 872ffd4b38055c604e4d250255bdf9c87c0c2c34 Mon Sep 17 00:00:00 2001
From: James Almer <jamrial@gmail.com>
Date: Sat, 18 Oct 2025 21:23:29 -0300
Subject: [PATCH 12/12] fftools/ffmpeg_filter: handle display matrix side data
in stream group filtergraphs
Signed-off-by: James Almer <jamrial@gmail.com>
---
fftools/ffmpeg_filter.c | 43 +++++++++++++++++++++++++++++++++++++++++
1 file changed, 43 insertions(+)
diff --git a/fftools/ffmpeg_filter.c b/fftools/ffmpeg_filter.c
index 9450257a7c..3783cd41c4 100644
--- a/fftools/ffmpeg_filter.c
+++ b/fftools/ffmpeg_filter.c
@@ -228,6 +228,8 @@ typedef struct OutputFilterPriv {
const enum AVColorRange *color_ranges;
const enum AVAlphaMode *alpha_modes;
+ int32_t displaymatrix[9];
+
AVRational enc_timebase;
int64_t trim_start_us;
int64_t trim_duration_us;
@@ -1200,6 +1202,11 @@ int fg_create(FilterGraph **pfg, char *graph_desc, Scheduler *sch,
ofp->flags = opts->flags;
ofp->side_data = opts->side_data;
ofp->nb_side_data = opts->nb_side_data;
+
+ const AVFrameSideData *sd = av_frame_side_data_get(ofp->side_data, ofp->nb_side_data,
+ AV_FRAME_DATA_DISPLAYMATRIX);
+ if (sd)
+ memcpy(ofp->displaymatrix, sd->data, sizeof(ofp->displaymatrix));
}
}
@@ -1638,6 +1645,42 @@ static int configure_output_video_filter(FilterGraphPriv *fgp, AVFilterGraph *gr
pad_idx = 0;
}
+ if (ofp->flags & OFILTER_FLAG_AUTOROTATE) {
+ int32_t *displaymatrix = ofp->displaymatrix;
+ double theta;
+
+ theta = get_rotation(displaymatrix);
+
+ if (fabs(theta - 90) < 1.0) {
+ ret = insert_filter(&last_filter, &pad_idx, "transpose",
+ displaymatrix[3] > 0 ? "cclock_flip" : "clock");
+ } else if (fabs(theta - 180) < 1.0) {
+ if (displaymatrix[0] < 0) {
+ ret = insert_filter(&last_filter, &pad_idx, "hflip", NULL);
+ if (ret < 0)
+ return ret;
+ }
+ if (displaymatrix[4] < 0) {
+ ret = insert_filter(&last_filter, &pad_idx, "vflip", NULL);
+ }
+ } else if (fabs(theta - 270) < 1.0) {
+ ret = insert_filter(&last_filter, &pad_idx, "transpose",
+ displaymatrix[3] < 0 ? "clock_flip" : "cclock");
+ } else if (fabs(theta) > 1.0) {
+ char rotate_buf[64];
+ snprintf(rotate_buf, sizeof(rotate_buf), "%f*PI/180", theta);
+ ret = insert_filter(&last_filter, &pad_idx, "rotate", rotate_buf);
+ } else if (fabs(theta) < 1.0) {
+ if (displaymatrix && displaymatrix[4] < 0) {
+ ret = insert_filter(&last_filter, &pad_idx, "vflip", NULL);
+ }
+ }
+ if (ret < 0)
+ return ret;
+
+ av_frame_side_data_remove(&ofp->side_data, &ofp->nb_side_data, AV_FRAME_DATA_DISPLAYMATRIX);
+ }
+
av_assert0(!(ofp->flags & OFILTER_FLAG_DISABLE_CONVERT) ||
ofp->format != AV_PIX_FMT_NONE || !ofp->formats);
av_bprint_init(&bprint, 0, AV_BPRINT_SIZE_UNLIMITED);
--
2.49.1
_______________________________________________
ffmpeg-devel mailing list -- ffmpeg-devel@ffmpeg.org
To unsubscribe send an email to ffmpeg-devel-leave@ffmpeg.org
reply other threads:[~2025-10-20 12:59 UTC|newest]
Thread overview: [no followups] expand[flat|nested] mbox.gz Atom feed
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=176096510595.62.11860224245626366185@bf907ddaa564 \
--to=ffmpeg-devel@ffmpeg.org \
--cc=code@ffmpeg.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Git Inbox Mirror of the ffmpeg-devel mailing list - see https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
This inbox may be cloned and mirrored by anyone:
git clone --mirror https://master.gitmailbox.com/ffmpegdev/0 ffmpegdev/git/0.git
# If you have public-inbox 1.1+ installed, you may
# initialize and index your mirror using the following commands:
public-inbox-init -V2 ffmpegdev ffmpegdev/ https://master.gitmailbox.com/ffmpegdev \
ffmpegdev@gitmailbox.com
public-inbox-index ffmpegdev
Example config snippet for mirrors.
AGPL code for this site: git clone https://public-inbox.org/public-inbox.git