[FFmpeg-devel] [PATCH 17/24] fftools/ffmpeg: move sub2video handling to ffmpeg_filter

[Anton Khirnov <anton@khirnov.net>]

Make all relevant state per-filtergraph input, rather than per-input
stream. Refactor the code to make it work and avoid leaking memory when
a single subtitle stream is sent to multiple filters.
---
 fftools/ffmpeg.c        | 133 ++--------------------------
 fftools/ffmpeg.h        |   7 +-
 fftools/ffmpeg_dec.c    |   9 +-
 fftools/ffmpeg_demux.c  |   1 -
 fftools/ffmpeg_filter.c | 189 +++++++++++++++++++++++++++++++++++-----
 5 files changed, 177 insertions(+), 162 deletions(-)

diff --git a/fftools/ffmpeg.c b/fftools/ffmpeg.c
index e9e60407d2..36b4becaf2 100644
--- a/fftools/ffmpeg.c
+++ b/fftools/ffmpeg.c
@@ -147,139 +147,20 @@ static int restore_tty;
    This is a temporary solution until libavfilter gets real subtitles support.
  */
 
-static int sub2video_get_blank_frame(InputStream *ist)
-{
-    int ret;
-    AVFrame *frame = ist->sub2video.frame;
-
-    av_frame_unref(frame);
-    frame->width  = ist->sub2video.w;
-    frame->height = ist->sub2video.h;
-    frame->format = AV_PIX_FMT_RGB32;
-    if ((ret = av_frame_get_buffer(frame, 0)) < 0)
-        return ret;
-    memset(frame->data[0], 0, frame->height * frame->linesize[0]);
-    return 0;
-}
-
-static void sub2video_copy_rect(uint8_t *dst, int dst_linesize, int w, int h,
-                                AVSubtitleRect *r)
-{
-    uint32_t *pal, *dst2;
-    uint8_t *src, *src2;
-    int x, y;
-
-    if (r->type != SUBTITLE_BITMAP) {
-        av_log(NULL, AV_LOG_WARNING, "sub2video: non-bitmap subtitle\n");
-        return;
-    }
-    if (r->x < 0 || r->x + r->w > w || r->y < 0 || r->y + r->h > h) {
-        av_log(NULL, AV_LOG_WARNING, "sub2video: rectangle (%d %d %d %d) overflowing %d %d\n",
-            r->x, r->y, r->w, r->h, w, h
-        );
-        return;
-    }
-
-    dst += r->y * dst_linesize + r->x * 4;
-    src = r->data[0];
-    pal = (uint32_t *)r->data[1];
-    for (y = 0; y < r->h; y++) {
-        dst2 = (uint32_t *)dst;
-        src2 = src;
-        for (x = 0; x < r->w; x++)
-            *(dst2++) = pal[*(src2++)];
-        dst += dst_linesize;
-        src += r->linesize[0];
-    }
-}
-
-static void sub2video_push_ref(InputStream *ist, int64_t pts)
-{
-    AVFrame *frame = ist->sub2video.frame;
-    int i;
-    int ret;
-
-    av_assert1(frame->data[0]);
-    ist->sub2video.last_pts = frame->pts = pts;
-    for (i = 0; i < ist->nb_filters; i++) {
-        ret = av_buffersrc_add_frame_flags(ist->filters[i]->filter, frame,
-                                           AV_BUFFERSRC_FLAG_KEEP_REF |
-                                           AV_BUFFERSRC_FLAG_PUSH);
-        if (ret != AVERROR_EOF && ret < 0)
-            av_log(NULL, AV_LOG_WARNING, "Error while add the frame to buffer source(%s).\n",
-                   av_err2str(ret));
-    }
-}
-
-void sub2video_update(InputStream *ist, int64_t heartbeat_pts,
-                      const AVSubtitle *sub)
-{
-    AVFrame *frame = ist->sub2video.frame;
-    int8_t *dst;
-    int     dst_linesize;
-    int num_rects, i;
-    int64_t pts, end_pts;
-
-    if (!frame)
-        return;
-    if (sub) {
-        pts       = av_rescale_q(sub->pts + sub->start_display_time * 1000LL,
-                                 AV_TIME_BASE_Q, ist->st->time_base);
-        end_pts   = av_rescale_q(sub->pts + sub->end_display_time   * 1000LL,
-                                 AV_TIME_BASE_Q, ist->st->time_base);
-        num_rects = sub->num_rects;
-    } else {
-        /* If we are initializing the system, utilize current heartbeat
-           PTS as the start time, and show until the following subpicture
-           is received. Otherwise, utilize the previous subpicture's end time
-           as the fall-back value. */
-        pts       = ist->sub2video.initialize ?
-                    heartbeat_pts : ist->sub2video.end_pts;
-        end_pts   = INT64_MAX;
-        num_rects = 0;
-    }
-    if (sub2video_get_blank_frame(ist) < 0) {
-        av_log(NULL, AV_LOG_ERROR,
-               "Impossible to get a blank canvas.\n");
-        return;
-    }
-    dst          = frame->data    [0];
-    dst_linesize = frame->linesize[0];
-    for (i = 0; i < num_rects; i++)
-        sub2video_copy_rect(dst, dst_linesize, frame->width, frame->height, sub->rects[i]);
-    sub2video_push_ref(ist, pts);
-    ist->sub2video.end_pts = end_pts;
-    ist->sub2video.initialize = 0;
-}
-
 static void sub2video_heartbeat(InputFile *infile, int64_t pts, AVRational tb)
 {
-    int i, j, nb_reqs;
-    int64_t pts2;
-
     /* When a frame is read from a file, examine all sub2video streams in
        the same file and send the sub2video frame again. Otherwise, decoded
        video frames could be accumulating in the filter graph while a filter
        (possibly overlay) is desperately waiting for a subtitle frame. */
-    for (i = 0; i < infile->nb_streams; i++) {
-        InputStream *ist2 = infile->streams[i];
-        if (!ist2->sub2video.frame)
-            continue;
-        /* subtitles seem to be usually muxed ahead of other streams;
-           if not, subtracting a larger time here is necessary */
-        pts2 = av_rescale_q(pts, tb, ist2->st->time_base) - 1;
-        /* do not send the heartbeat frame if the subtitle is already ahead */
-        if (pts2 <= ist2->sub2video.last_pts)
+    for (int i = 0; i < infile->nb_streams; i++) {
+        InputStream *ist = infile->streams[i];
+
+        if (ist->dec_ctx->codec_type != AVMEDIA_TYPE_SUBTITLE)
             continue;
-        if (pts2 >= ist2->sub2video.end_pts || ist2->sub2video.initialize)
-            /* if we have hit the end of the current displayed subpicture,
-               or if we need to initialize the system, update the
-               overlayed subpicture and its start/end times */
-            sub2video_update(ist2, pts2 + 1, NULL);
-        for (j = 0, nb_reqs = 0; j < ist2->nb_filters; j++)
-            nb_reqs += av_buffersrc_get_nb_failed_requests(ist2->filters[j]->filter);
-        if (nb_reqs)
-            sub2video_push_ref(ist2, pts2);
+
+        for (int j = 0; j < ist->nb_filters; j++)
+            ifilter_sub2video_heartbeat(ist->filters[j], pts, tb);
     }
 }
 
diff --git a/fftools/ffmpeg.h b/fftools/ffmpeg.h
index d4aff5cc7c..75695d3fb5 100644
--- a/fftools/ffmpeg.h
+++ b/fftools/ffmpeg.h
@@ -372,11 +372,7 @@ typedef struct InputStream {
     } prev_sub;
 
     struct sub2video {
-        int64_t last_pts;
-        int64_t end_pts;
-        AVFrame *frame;
         int w, h;
-        unsigned int initialize; ///< marks if sub2video_update should force an initialization
     } sub2video;
 
     /* decoded data from this stream goes into all those filters
@@ -741,13 +737,12 @@ int init_simple_filtergraph(InputStream *ist, OutputStream *ost,
                             char *graph_desc);
 int init_complex_filtergraph(FilterGraph *fg);
 
-void sub2video_update(InputStream *ist, int64_t heartbeat_pts,
-                      const AVSubtitle *sub);
 int copy_av_subtitle(AVSubtitle *dst, const AVSubtitle *src);
 
 int ifilter_send_frame(InputFilter *ifilter, AVFrame *frame, int keep_reference);
 int ifilter_send_eof(InputFilter *ifilter, int64_t pts, AVRational tb);
 int ifilter_sub2video(InputFilter *ifilter, const AVSubtitle *sub);
+void ifilter_sub2video_heartbeat(InputFilter *ifilter, int64_t pts, AVRational tb);
 
 /**
  * Set up fallback filtering parameters from a decoder context. They will only
diff --git a/fftools/ffmpeg_dec.c b/fftools/ffmpeg_dec.c
index a7acaf67c2..30959c64b7 100644
--- a/fftools/ffmpeg_dec.c
+++ b/fftools/ffmpeg_dec.c
@@ -321,13 +321,8 @@ static int video_frame_process(InputStream *ist, AVFrame *frame)
 
 static void sub2video_flush(InputStream *ist)
 {
-    int i;
-    int ret;
-
-    if (ist->sub2video.end_pts < INT64_MAX)
-        sub2video_update(ist, INT64_MAX, NULL);
-    for (i = 0; i < ist->nb_filters; i++) {
-        ret = av_buffersrc_add_frame(ist->filters[i]->filter, NULL);
+    for (int i = 0; i < ist->nb_filters; i++) {
+        int ret = ifilter_sub2video(ist->filters[i], NULL);
         if (ret != AVERROR_EOF && ret < 0)
             av_log(NULL, AV_LOG_WARNING, "Flush the frame error.\n");
     }
diff --git a/fftools/ffmpeg_demux.c b/fftools/ffmpeg_demux.c
index 817ccbbedc..5c15b8bad3 100644
--- a/fftools/ffmpeg_demux.c
+++ b/fftools/ffmpeg_demux.c
@@ -817,7 +817,6 @@ static void ist_free(InputStream **pist)
 
     av_dict_free(&ist->decoder_opts);
     avsubtitle_free(&ist->prev_sub.subtitle);
-    av_frame_free(&ist->sub2video.frame);
     av_freep(&ist->filters);
     av_freep(&ist->outputs);
     av_freep(&ist->hwaccel_device);
diff --git a/fftools/ffmpeg_filter.c b/fftools/ffmpeg_filter.c
index 670e697f69..ce2a914745 100644
--- a/fftools/ffmpeg_filter.c
+++ b/fftools/ffmpeg_filter.c
@@ -107,6 +107,14 @@ typedef struct InputFilterPriv {
     struct {
         ///< queue of AVSubtitle* before filter init
         AVFifo *queue;
+
+        AVFrame *frame;
+
+        int64_t last_pts;
+        int64_t end_pts;
+
+        ///< marks if sub2video_update should force an initialization
+        unsigned int initialize;
     } sub2video;
 } InputFilterPriv;
 
@@ -115,6 +123,111 @@ static InputFilterPriv *ifp_from_ifilter(InputFilter *ifilter)
     return (InputFilterPriv*)ifilter;
 }
 
+static int sub2video_get_blank_frame(InputFilterPriv *ifp)
+{
+    AVFrame *frame = ifp->sub2video.frame;
+    int ret;
+
+    av_frame_unref(frame);
+
+    frame->width  = ifp->width;
+    frame->height = ifp->height;
+    frame->format = ifp->format;
+
+    ret = av_frame_get_buffer(frame, 0);
+    if (ret < 0)
+        return ret;
+
+    memset(frame->data[0], 0, frame->height * frame->linesize[0]);
+
+    return 0;
+}
+
+static void sub2video_copy_rect(uint8_t *dst, int dst_linesize, int w, int h,
+                                AVSubtitleRect *r)
+{
+    uint32_t *pal, *dst2;
+    uint8_t *src, *src2;
+    int x, y;
+
+    if (r->type != SUBTITLE_BITMAP) {
+        av_log(NULL, AV_LOG_WARNING, "sub2video: non-bitmap subtitle\n");
+        return;
+    }
+    if (r->x < 0 || r->x + r->w > w || r->y < 0 || r->y + r->h > h) {
+        av_log(NULL, AV_LOG_WARNING, "sub2video: rectangle (%d %d %d %d) overflowing %d %d\n",
+            r->x, r->y, r->w, r->h, w, h
+        );
+        return;
+    }
+
+    dst += r->y * dst_linesize + r->x * 4;
+    src = r->data[0];
+    pal = (uint32_t *)r->data[1];
+    for (y = 0; y < r->h; y++) {
+        dst2 = (uint32_t *)dst;
+        src2 = src;
+        for (x = 0; x < r->w; x++)
+            *(dst2++) = pal[*(src2++)];
+        dst += dst_linesize;
+        src += r->linesize[0];
+    }
+}
+
+static void sub2video_push_ref(InputFilterPriv *ifp, int64_t pts)
+{
+    AVFrame *frame = ifp->sub2video.frame;
+    int ret;
+
+    av_assert1(frame->data[0]);
+    ifp->sub2video.last_pts = frame->pts = pts;
+    ret = av_buffersrc_add_frame_flags(ifp->ifilter.filter, frame,
+                                       AV_BUFFERSRC_FLAG_KEEP_REF |
+                                       AV_BUFFERSRC_FLAG_PUSH);
+    if (ret != AVERROR_EOF && ret < 0)
+        av_log(NULL, AV_LOG_WARNING, "Error while add the frame to buffer source(%s).\n",
+               av_err2str(ret));
+}
+
+static void sub2video_update(InputFilterPriv *ifp, int64_t heartbeat_pts,
+                             const AVSubtitle *sub)
+{
+    AVFrame *frame = ifp->sub2video.frame;
+    int8_t *dst;
+    int     dst_linesize;
+    int num_rects, i;
+    int64_t pts, end_pts;
+
+    if (sub) {
+        pts       = av_rescale_q(sub->pts + sub->start_display_time * 1000LL,
+                                 AV_TIME_BASE_Q, ifp->time_base);
+        end_pts   = av_rescale_q(sub->pts + sub->end_display_time   * 1000LL,
+                                 AV_TIME_BASE_Q, ifp->time_base);
+        num_rects = sub->num_rects;
+    } else {
+        /* If we are initializing the system, utilize current heartbeat
+           PTS as the start time, and show until the following subpicture
+           is received. Otherwise, utilize the previous subpicture's end time
+           as the fall-back value. */
+        pts       = ifp->sub2video.initialize ?
+                    heartbeat_pts : ifp->sub2video.end_pts;
+        end_pts   = INT64_MAX;
+        num_rects = 0;
+    }
+    if (sub2video_get_blank_frame(ifp) < 0) {
+        av_log(NULL, AV_LOG_ERROR,
+               "Impossible to get a blank canvas.\n");
+        return;
+    }
+    dst          = frame->data    [0];
+    dst_linesize = frame->linesize[0];
+    for (i = 0; i < num_rects; i++)
+        sub2video_copy_rect(dst, dst_linesize, frame->width, frame->height, sub->rects[i]);
+    sub2video_push_ref(ifp, pts);
+    ifp->sub2video.end_pts = end_pts;
+    ifp->sub2video.initialize = 0;
+}
+
 // FIXME: YUV420P etc. are actually supported with full color range,
 // yet the latter information isn't available here.
 static const enum AVPixelFormat *get_compliance_normal_pix_fmts(const AVCodec *codec, const enum AVPixelFormat default_formats[])
@@ -465,6 +578,12 @@ static int ifilter_bind_ist(InputFilter *ifilter, InputStream *ist)
     ifp->ist             = ist;
     ifp->type_src        = ist->st->codecpar->codec_type;
 
+    if (ifp->type_src == AVMEDIA_TYPE_SUBTITLE) {
+        ifp->sub2video.frame = av_frame_alloc();
+        if (!ifp->sub2video.frame)
+            return AVERROR(ENOMEM);
+    }
+
     return 0;
 }
 
@@ -610,6 +729,7 @@ void fg_free(FilterGraph **pfg)
                 avsubtitle_free(&sub);
             av_fifo_freep2(&ifp->sub2video.queue);
         }
+        av_frame_free(&ifp->sub2video.frame);
 
         av_channel_layout_uninit(&ifp->fallback.ch_layout);
 
@@ -1108,20 +1228,15 @@ void check_filter_outputs(void)
     }
 }
 
-static int sub2video_prepare(InputStream *ist, InputFilter *ifilter)
+static void sub2video_prepare(InputFilterPriv *ifp)
 {
-    ist->sub2video.frame = av_frame_alloc();
-    if (!ist->sub2video.frame)
-        return AVERROR(ENOMEM);
-    ist->sub2video.last_pts = INT64_MIN;
-    ist->sub2video.end_pts  = INT64_MIN;
+    ifp->sub2video.last_pts = INT64_MIN;
+    ifp->sub2video.end_pts  = INT64_MIN;
 
     /* sub2video structure has been (re-)initialized.
        Mark it as such so that the system will be
        initialized with the first received heartbeat. */
-    ist->sub2video.initialize = 1;
-
-    return 0;
+    ifp->sub2video.initialize = 1;
 }
 
 static int configure_input_video_filter(FilterGraph *fg, InputFilter *ifilter,
@@ -1156,11 +1271,8 @@ static int configure_input_video_filter(FilterGraph *fg, InputFilter *ifilter,
     if (!fr.num)
         fr = ist->framerate_guessed;
 
-    if (ist->dec_ctx->codec_type == AVMEDIA_TYPE_SUBTITLE) {
-        ret = sub2video_prepare(ist, ifilter);
-        if (ret < 0)
-            goto fail;
-    }
+    if (ifp->type_src == AVMEDIA_TYPE_SUBTITLE)
+        sub2video_prepare(ifp);
 
     ifp->time_base =  ist->framerate.num ? av_inv_q(ist->framerate) :
                                            ist->st->time_base;
@@ -1466,12 +1578,13 @@ int configure_filtergraph(FilterGraph *fg)
 
     /* process queued up subtitle packets */
     for (i = 0; i < fg->nb_inputs; i++) {
-        InputFilterPriv *ifp = ifp_from_ifilter(fg->inputs[i]);
-        InputStream     *ist = ifp->ist;
-        if (ifp->sub2video.queue && ist->sub2video.frame) {
+        InputFilter *ifilter = fg->inputs[i];
+        InputFilterPriv *ifp = ifp_from_ifilter(ifilter);
+
+        if (ifp->type_src == AVMEDIA_TYPE_SUBTITLE && ifp->sub2video.queue) {
             AVSubtitle tmp;
             while (av_fifo_read(ifp->sub2video.queue, &tmp, 1) >= 0) {
-                sub2video_update(ist, INT64_MIN, &tmp);
+                sub2video_update(ifp, INT64_MIN, &tmp);
                 avsubtitle_free(&tmp);
             }
         }
@@ -1620,15 +1733,47 @@ int reap_filters(int flush)
     return 0;
 }
 
+void ifilter_sub2video_heartbeat(InputFilter *ifilter, int64_t pts, AVRational tb)
+{
+    InputFilterPriv *ifp = ifp_from_ifilter(ifilter);
+    int64_t pts2;
+
+    if (!ifilter->graph->graph)
+        return;
+
+    /* subtitles seem to be usually muxed ahead of other streams;
+       if not, subtracting a larger time here is necessary */
+    pts2 = av_rescale_q(pts, tb, ifp->time_base) - 1;
+
+    /* do not send the heartbeat frame if the subtitle is already ahead */
+    if (pts2 <= ifp->sub2video.last_pts)
+        return;
+
+    if (pts2 >= ifp->sub2video.end_pts || ifp->sub2video.initialize)
+        /* if we have hit the end of the current displayed subpicture,
+           or if we need to initialize the system, update the
+           overlayed subpicture and its start/end times */
+        sub2video_update(ifp, pts2 + 1, NULL);
+
+    if (av_buffersrc_get_nb_failed_requests(ifilter->filter))
+        sub2video_push_ref(ifp, pts2);
+}
+
 int ifilter_sub2video(InputFilter *ifilter, const AVSubtitle *subtitle)
 {
     InputFilterPriv *ifp = ifp_from_ifilter(ifilter);
-    InputStream     *ist = ifp->ist;
     int ret;
 
-    if (ist->sub2video.frame) {
-        sub2video_update(ist, INT64_MIN, subtitle);
-    } else {
+    if (ifilter->graph->graph) {
+        if (!subtitle) {
+            if (ifp->sub2video.end_pts < INT64_MAX)
+                sub2video_update(ifp, INT64_MAX, NULL);
+
+            return av_buffersrc_add_frame(ifilter->filter, NULL);
+        }
+
+        sub2video_update(ifp, INT64_MIN, subtitle);
+    } else if (subtitle) {
         AVSubtitle sub;
 
         if (!ifp->sub2video.queue)
-- 
2.40.1

_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".