[FFmpeg-devel] [PATCH] vf_libplacebo: output empty frames during gaps in the input timeline (PR #20223)

[FFmpeg-devel] [PATCH] vf_libplacebo: output empty frames during gaps in the input timeline (PR #20223)

[Niklas Haas]

PR #20223 opened by Niklas Haas (haasn)
URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/20223
Patch URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/20223.patch


From 3b7db1f9eea5ea9294988ec5c079ee9896d610db Mon Sep 17 00:00:00 2001
From: Niklas Haas <git@haasn.dev>
Date: Tue, 12 Aug 2025 12:15:54 +0200
Subject: [PATCH 1/3] avfilter/vf_libplacebo: allow rendering empty frames

When using libplacebo to composite multiple streams with complex PTS
values, it's possible for there to be a "hole" in the output stream; in
particular in constant FPS mode. This commit refactors output_frame() to
allow it to handle the case of there being no active input.
---
 libavfilter/vf_libplacebo.c | 17 ++++++++++-------
 1 file changed, 10 insertions(+), 7 deletions(-)

diff --git a/libavfilter/vf_libplacebo.c b/libavfilter/vf_libplacebo.c
index 01ce9d56b7..1cdf9b67cf 100644
--- a/libavfilter/vf_libplacebo.c
+++ b/libavfilter/vf_libplacebo.c
@@ -949,22 +949,21 @@ static int output_frame(AVFilterContext *ctx, int64_t pts)
         nb_visible++;
     }
 
-    /* It should be impossible to call output_frame() without at least one
-     * valid nonempty frame mix */
-    av_assert1(nb_visible > 0);
-
     out = ff_get_video_buffer(outlink, outlink->w, outlink->h);
     if (!out)
         return AVERROR(ENOMEM);
 
-    RET(av_frame_copy_props(out, ref));
     out->pts = pts;
+    if (s->fps.num)
+        out->duration = 1;
+    if (!ref)
+        goto props_done;
+
+    RET(av_frame_copy_props(out, ref));
     out->width = outlink->w;
     out->height = outlink->h;
     out->colorspace = outlink->colorspace;
     out->color_range = outlink->color_range;
-    if (s->fps.num)
-        out->duration = 1;
     if (s->deinterlace)
         out->flags &= ~(AV_FRAME_FLAG_INTERLACED | AV_FRAME_FLAG_TOP_FIELD_FIRST);
 
@@ -1000,6 +999,7 @@ static int output_frame(AVFilterContext *ctx, int64_t pts)
         out->sample_aspect_ratio = av_mul_q(ref->sample_aspect_ratio, stretch);
     }
 
+props_done:
     /* Map, render and unmap output frame */
     if (outdesc->flags & AV_PIX_FMT_FLAG_HWACCEL) {
         ok = pl_map_avframe_ex(s->gpu, &target, pl_avframe_params(
@@ -1068,6 +1068,9 @@ static int output_frame(AVFilterContext *ctx, int64_t pts)
         target.crop = orig_target.crop = (struct pl_rect2df) {0};
         pl_render_image(s->linear_rr, &target, &orig_target, &opts->params);
         target = orig_target;
+    } else if (!ref) {
+        /* Render an empty image to clear the frame to the desired fill color */
+        pl_render_image(s->linear_rr, NULL, &target, &opts->params);
     }
 
     if (outdesc->flags & AV_PIX_FMT_FLAG_HWACCEL) {
-- 
2.49.1


From da55b6b89c82e2ce3c1280ce9c055aa14918f565 Mon Sep 17 00:00:00 2001
From: Niklas Haas <git@haasn.dev>
Date: Tue, 12 Aug 2025 12:17:41 +0200
Subject: [PATCH 2/3] avfilter/vf_libplacebo: refactor status handling

Instead of having a single s->status field to track the most recently
EOF'd stream, track the number of active streams directly and only query
the most recent status at the end.

The reason this worked before is because we implicitly relied on the
assumption that `ok` would always be true until all streams are EOF, but
because it's possible to have gaps in the timeline when mixing multiple
streams, this is not always the case in principle.

In practice, this fixes a bug where the filter would early-exit (EOF)
too soon, when any input reached EOF and there is a gap in the timeline.
---
 libavfilter/vf_libplacebo.c | 24 +++++++++++++++---------
 1 file changed, 15 insertions(+), 9 deletions(-)

diff --git a/libavfilter/vf_libplacebo.c b/libavfilter/vf_libplacebo.c
index 1cdf9b67cf..1ee206d2df 100644
--- a/libavfilter/vf_libplacebo.c
+++ b/libavfilter/vf_libplacebo.c
@@ -170,8 +170,7 @@ typedef struct LibplaceboContext {
     /* input state */
     LibplaceboInput *inputs;
     int nb_inputs;
-    int64_t status_pts; ///< tracks status of most recently used input
-    int status;
+    int nb_active;
 
     /* settings */
     char *out_format_string;
@@ -767,6 +766,7 @@ static int init_vulkan(AVFilterContext *avctx, const AVVulkanDeviceContext *hwct
         return AVERROR(ENOMEM);
     for (int i = 0; i < s->nb_inputs; i++)
         RET(input_init(avctx, &s->inputs[i], i));
+    s->nb_active = s->nb_inputs;
     s->linear_rr = pl_renderer_create(s->log, s->gpu);
 
     /* fall through */
@@ -1164,11 +1164,7 @@ static int handle_input(AVFilterContext *ctx, LibplaceboInput *input)
         pl_queue_push(input->queue, NULL); /* Signal EOF to pl_queue */
         input->status = status;
         input->status_pts = pts;
-        if (!s->status || pts >= s->status_pts) {
-            /* Also propagate to output unless overwritten by later status change */
-            s->status = status;
-            s->status_pts = pts;
-        }
+        s->nb_active--;
     }
 
     return 0;
@@ -1252,8 +1248,18 @@ static int libplacebo_activate(AVFilterContext *ctx)
             for (int i = 0; i < s->nb_inputs; i++)
                 drain_input_pts(&s->inputs[i], out_pts);
             return output_frame(ctx, out_pts);
-        } else if (s->status) {
-            ff_outlink_set_status(outlink, s->status, s->status_pts);
+        } else if (s->nb_active == 0) {
+            /* Forward most recent status */
+            int status = s->inputs[0].status;
+            int64_t status_pts = s->inputs[0].status_pts;
+            for (int i = 1; i < s->nb_inputs; i++) {
+                const LibplaceboInput *in = &s->inputs[i];
+                if (in->status_pts > status_pts) {
+                    status = s->inputs[i].status;
+                    status_pts = s->inputs[i].status_pts;
+                }
+            }
+            ff_outlink_set_status(outlink, status, status_pts);
             return 0;
         }
 
-- 
2.49.1


From bc01df24c5ec535820c9d89a0e95b8468fb60407 Mon Sep 17 00:00:00 2001
From: Niklas Haas <git@haasn.dev>
Date: Tue, 12 Aug 2025 12:20:11 +0200
Subject: [PATCH 3/3] avfilter/vf_libplacebo: also output a frame during input
 gaps

In constant FPS mode, it's possible for there no be *no* input active at
the current timestamp, even though there would be active inputs later in
the timeline (nb_active > 0). This would previously hit the AVERROR_BUG case.

With this change, we can instead output an empty frame.
---
 libavfilter/vf_libplacebo.c | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/libavfilter/vf_libplacebo.c b/libavfilter/vf_libplacebo.c
index 1ee206d2df..98e8062927 100644
--- a/libavfilter/vf_libplacebo.c
+++ b/libavfilter/vf_libplacebo.c
@@ -1241,6 +1241,10 @@ static int libplacebo_activate(AVFilterContext *ctx)
             }
         }
 
+        /* In constant FPS mode, we can also output an empty frame if there is
+         * a gap in the input timeline and we still have active streams */
+        ok |= s->fps.num && s->nb_active > 0;
+
         if (retry) {
             return 0;
         } else if (ok) {
-- 
2.49.1

_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".