[FFmpeg-devel] [PATCH 1/2] ffmpeg: refactor post-decoding steps for subtitles into a function

[FFmpeg-devel] [PATCH 1/2] ffmpeg: refactor post-decoding steps for subtitles into a function

[Jan Ekström]

From: Jan Ekström <jan.ekstrom@24i.com>

This enables us to later call this when generating additional
subtitles for splitting purposes.

Signed-off-by: Jan Ekström <jan.ekstrom@24i.com>
---
 fftools/ffmpeg.c | 50 ++++++++++++++++++++++++++++--------------------
 1 file changed, 29 insertions(+), 21 deletions(-)

diff --git a/fftools/ffmpeg.c b/fftools/ffmpeg.c
index bdeff9a12e..1697891a99 100644
--- a/fftools/ffmpeg.c
+++ b/fftools/ffmpeg.c
@@ -2482,27 +2482,16 @@ fail:
     return err < 0 ? err : ret;
 }
 
-static int transcode_subtitles(InputStream *ist, AVPacket *pkt, int *got_output,
-                               int *decode_failed)
+static int encode_mux_subtitles(InputStream *ist, AVSubtitle *subtitle, int *got_output)
 {
-    AVSubtitle subtitle;
+    int ret = 0;
     int free_sub = 1;
-    int i, ret = avcodec_decode_subtitle2(ist->dec_ctx,
-                                          &subtitle, got_output, pkt);
 
-    check_decode_result(NULL, got_output, ret);
-
-    if (ret < 0 || !*got_output) {
-        *decode_failed = 1;
-        if (!pkt->size)
-            sub2video_flush(ist);
-        return ret;
-    }
 
     if (ist->fix_sub_duration) {
         int end = 1;
         if (ist->prev_sub.got_output) {
-            end = av_rescale(subtitle.pts - ist->prev_sub.subtitle.pts,
+            end = av_rescale(subtitle->pts - ist->prev_sub.subtitle.pts,
                              1000, AV_TIME_BASE);
             if (end < ist->prev_sub.subtitle.end_display_time) {
                 av_log(ist->dec_ctx, AV_LOG_DEBUG,
@@ -2514,7 +2503,7 @@ static int transcode_subtitles(InputStream *ist, AVPacket *pkt, int *got_output,
         }
         FFSWAP(int,        *got_output, ist->prev_sub.got_output);
         FFSWAP(int,        ret,         ist->prev_sub.ret);
-        FFSWAP(AVSubtitle, subtitle,    ist->prev_sub.subtitle);
+        FFSWAP(AVSubtitle, *subtitle,   ist->prev_sub.subtitle);
         if (end <= 0)
             goto out;
     }
@@ -2523,7 +2512,7 @@ static int transcode_subtitles(InputStream *ist, AVPacket *pkt, int *got_output,
         return ret;
 
     if (ist->sub2video.frame) {
-        sub2video_update(ist, INT64_MIN, &subtitle);
+        sub2video_update(ist, INT64_MIN, subtitle);
     } else if (ist->nb_filters) {
         if (!ist->sub2video.sub_queue)
             ist->sub2video.sub_queue = av_fifo_alloc(8 * sizeof(AVSubtitle));
@@ -2534,31 +2523,50 @@ static int transcode_subtitles(InputStream *ist, AVPacket *pkt, int *got_output,
             if (ret < 0)
                 exit_program(1);
         }
-        av_fifo_generic_write(ist->sub2video.sub_queue, &subtitle, sizeof(subtitle), NULL);
+        av_fifo_generic_write(ist->sub2video.sub_queue, subtitle, sizeof(*subtitle), NULL);
         free_sub = 0;
     }
 
-    if (!subtitle.num_rects)
+    if (!subtitle->num_rects)
         goto out;
 
     ist->frames_decoded++;
 
-    for (i = 0; i < nb_output_streams; i++) {
+    for (int i = 0; i < nb_output_streams; i++) {
         OutputStream *ost = output_streams[i];
 
         if (!check_output_constraints(ist, ost) || !ost->encoding_needed
             || ost->enc->type != AVMEDIA_TYPE_SUBTITLE)
             continue;
 
-        do_subtitle_out(output_files[ost->file_index], ost, &subtitle);
+        do_subtitle_out(output_files[ost->file_index], ost, subtitle);
     }
 
 out:
     if (free_sub)
-        avsubtitle_free(&subtitle);
+        avsubtitle_free(subtitle);
     return ret;
 }
 
+static int transcode_subtitles(InputStream *ist, AVPacket *pkt, int *got_output,
+                               int *decode_failed)
+{
+    AVSubtitle subtitle;
+    int ret = avcodec_decode_subtitle2(ist->dec_ctx,
+                                       &subtitle, got_output, pkt);
+
+    check_decode_result(NULL, got_output, ret);
+
+    if (ret < 0 || !*got_output) {
+        *decode_failed = 1;
+        if (!pkt->size)
+            sub2video_flush(ist);
+        return ret;
+    }
+
+    return encode_mux_subtitles(ist, &subtitle, got_output);
+}
+
 static int send_filter_eof(InputStream *ist)
 {
     int i, ret;
-- 
2.33.1

_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".

[FFmpeg-devel] [PATCH 2/2] ffmpeg: add video heartbeat capability to fix_sub_duration

[Jan Ekström]

From: Jan Ekström <jan.ekstrom@24i.com>

Splits the currently handled subtitle at random access point
packets that can be configured to follow a specific output stream.

This way the subtitle - which is known to be shown at this time
can be split and passed to muxer before its full duration is
yet known.

Signed-off-by: Jan Ekström <jan.ekstrom@24i.com>
---
 doc/ffmpeg.texi      |  11 ++++
 fftools/ffmpeg.c     | 142 +++++++++++++++++++++++++++++++++++++++++++
 fftools/ffmpeg.h     |   8 +++
 fftools/ffmpeg_opt.c |   9 +++
 4 files changed, 170 insertions(+)

diff --git a/doc/ffmpeg.texi b/doc/ffmpeg.texi
index 164419cad3..7ceb5532ce 100644
--- a/doc/ffmpeg.texi
+++ b/doc/ffmpeg.texi
@@ -1297,6 +1297,17 @@ List all hardware acceleration components enabled in this build of ffmpeg.
 Actual runtime availability depends on the hardware and its suitable driver
 being installed.
 
+@item -fix_sub_duration_heartbeat[:@var{stream_specifier}]
+Set a specific output video stream as the heartbeat stream according to which
+to split and push through currently in-progress subtitle upon receipt of a
+random access packet.
+
+This lowers the latency of subtitles for which the end packet or the following
+subtitle has not yet been received.
+
+Requires @option{-fix_sub_duration} to be set for the relevant input subtitle
+stream for this to have any effect.
+
 @end table
 
 @section Audio Options
diff --git a/fftools/ffmpeg.c b/fftools/ffmpeg.c
index 1697891a99..f8cc11d1f4 100644
--- a/fftools/ffmpeg.c
+++ b/fftools/ffmpeg.c
@@ -126,6 +126,7 @@ typedef struct BenchmarkTimeStamps {
     int64_t sys_usec;
 } BenchmarkTimeStamps;
 
+static int trigger_fix_sub_duration_heartbeat(OutputStream *ost, const AVPacket *pkt);
 static void do_video_stats(OutputStream *ost, int frame_size);
 static BenchmarkTimeStamps get_benchmark_time_stamps(void);
 static int64_t getmaxrss(void);
@@ -1382,6 +1383,13 @@ static void do_video_out(OutputFile *of,
                     av_ts2str(pkt->dts), av_ts2timestr(pkt->dts, &ost->mux_timebase));
             }
 
+            if ((ret = trigger_fix_sub_duration_heartbeat(ost, pkt)) < 0) {
+                av_log(NULL, AV_LOG_ERROR,
+                       "Subtitle heartbeat logic failed in %s! (%s)\n",
+                       __func__, av_err2str(ret));
+                exit_program(1);
+            }
+
             frame_size = pkt->size;
             output_packet(of, pkt, ost, 0);
 
@@ -2091,6 +2099,16 @@ static void do_streamcopy(InputStream *ist, OutputStream *ost, const AVPacket *p
 
     ost->sync_opts += opkt->duration;
 
+    if (ost->st->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) {
+        int ret;
+        if ((ret = trigger_fix_sub_duration_heartbeat(ost, pkt)) < 0) {
+            av_log(NULL, AV_LOG_ERROR,
+                   "Subtitle heartbeat logic failed in %s! (%s)\n",
+                   __func__, av_err2str(ret));
+            exit_program(1);
+        }
+    }
+
     output_packet(of, opkt, ost, 0);
 }
 
@@ -2548,6 +2566,130 @@ out:
     return ret;
 }
 
+static int copy_av_subtitle(AVSubtitle *dst, AVSubtitle *src)
+{
+    int ret = AVERROR_BUG;
+    AVSubtitle tmp = {
+        .format = src->format,
+        .start_display_time = src->start_display_time,
+        .end_display_time = src->end_display_time,
+        .num_rects = 0,
+        .rects = NULL,
+        .pts = src->pts
+    };
+
+    if (!src->num_rects)
+        goto success;
+
+    if (!(tmp.rects = av_calloc(src->num_rects, sizeof(*tmp.rects))))
+        return AVERROR(ENOMEM);
+
+    for (int i = 0; i < src->num_rects; i++) {
+        AVSubtitleRect *src_rect = src->rects[i];
+        AVSubtitleRect *dst_rect;
+
+        if (!(dst_rect = tmp.rects[i] = av_mallocz(sizeof(*tmp.rects[0])))) {
+            ret = AVERROR(ENOMEM);
+            goto cleanup;
+        }
+
+        tmp.num_rects++;
+
+        dst_rect->type      = src_rect->type;
+        dst_rect->flags     = src_rect->flags;
+
+        dst_rect->x         = src_rect->x;
+        dst_rect->y         = src_rect->y;
+        dst_rect->w         = src_rect->w;
+        dst_rect->h         = src_rect->h;
+        dst_rect->nb_colors = src_rect->nb_colors;
+
+        if (src_rect->text)
+            if (!(dst_rect->text = av_strdup(src_rect->text))) {
+                ret = AVERROR(ENOMEM);
+                goto cleanup;
+            }
+
+        if (src_rect->ass)
+            if (!(dst_rect->ass = av_strdup(src_rect->ass))) {
+                ret = AVERROR(ENOMEM);
+                goto cleanup;
+            }
+
+        for (int j = 0; j < 4; j++) {
+            // SUBTITLE_BITMAP images are special in the sense that they
+            // are like PAL8 images. first pointer to data, second to
+            // palette. This makes the size calculation match this.
+            size_t buf_size = src_rect->type == SUBTITLE_BITMAP && j == 1 ?
+                              AVPALETTE_SIZE :
+                              src_rect->h * src_rect->linesize[j];
+
+            if (!src_rect->data[j])
+                continue;
+
+            if (!(dst_rect->data[j] = av_memdup(src_rect->data[j], buf_size))) {
+                ret = AVERROR(ENOMEM);
+                goto cleanup;
+            }
+            dst_rect->linesize[j] = src_rect->linesize[j];
+        }
+    }
+
+success:
+    *dst = tmp;
+
+    return 0;
+
+cleanup:
+    avsubtitle_free(&tmp);
+
+    return ret;
+}
+
+static int fix_sub_duration_heartbeat(InputStream *ist, int64_t signal_pts)
+{
+    int ret = AVERROR_BUG;
+    int got_output = 1;
+    AVSubtitle *prev_subtitle = &ist->prev_sub.subtitle;
+    AVSubtitle subtitle;
+
+    if (!ist->fix_sub_duration || !prev_subtitle->num_rects ||
+        signal_pts <= prev_subtitle->pts)
+        return 0;
+
+    if ((ret = copy_av_subtitle(&subtitle, prev_subtitle)) < 0)
+        return ret;
+
+    subtitle.pts = signal_pts;
+
+    return encode_mux_subtitles(ist, &subtitle, &got_output);
+}
+
+static int trigger_fix_sub_duration_heartbeat(OutputStream *ost, const AVPacket *pkt)
+{
+    int64_t signal_pts = av_rescale_q(pkt->pts, ost->mux_timebase,
+                                      AV_TIME_BASE_Q);
+
+    if (!ost->fix_sub_duration_heartbeat || !(pkt->flags & AV_PKT_FLAG_KEY))
+        // we are only interested in heartbeats on streams configured, and
+        // only on random access points.
+        return 0;
+
+    for (int index = 0; index < nb_input_streams; index++) {
+        InputStream *subtitle_ist = input_streams[index];
+        int ret = AVERROR_BUG;
+
+        if (!subtitle_ist->decoding_needed ||
+            subtitle_ist->dec_ctx->codec_type != AVMEDIA_TYPE_SUBTITLE)
+            continue;
+
+        if ((ret = fix_sub_duration_heartbeat(subtitle_ist, signal_pts)) < 0)
+            return ret;
+    }
+
+    return 0;
+}
+
 static int transcode_subtitles(InputStream *ist, AVPacket *pkt, int *got_output,
                                int *decode_failed)
 {
diff --git a/fftools/ffmpeg.h b/fftools/ffmpeg.h
index 9b200b806a..b1e67a94e9 100644
--- a/fftools/ffmpeg.h
+++ b/fftools/ffmpeg.h
@@ -204,6 +204,8 @@ typedef struct OptionsContext {
     int        nb_reinit_filters;
     SpecifierOpt *fix_sub_duration;
     int        nb_fix_sub_duration;
+    SpecifierOpt *fix_sub_duration_heartbeat;
+    int        nb_fix_sub_duration_heartbeat;
     SpecifierOpt *canvas_sizes;
     int        nb_canvas_sizes;
     SpecifierOpt *pass;
@@ -571,6 +573,12 @@ typedef struct OutputStream {
 
     /* frame encode sum of squared error values */
     int64_t error[4];
+
+    /*
+     * bool on whether this stream should be utilized for splitting
+     * subtitles utilizing fix_sub_duration at random access points.
+     */
+    unsigned int fix_sub_duration_heartbeat;
 } OutputStream;
 
 typedef struct OutputFile {
diff --git a/fftools/ffmpeg_opt.c b/fftools/ffmpeg_opt.c
index 9c820ab73f..64a08749ac 100644
--- a/fftools/ffmpeg_opt.c
+++ b/fftools/ffmpeg_opt.c
@@ -86,6 +86,7 @@ static const char *const opt_name_filters[]                   = {"filter", "af",
 static const char *const opt_name_filter_scripts[]            = {"filter_script", NULL};
 static const char *const opt_name_reinit_filters[]            = {"reinit_filter", NULL};
 static const char *const opt_name_fix_sub_duration[]          = {"fix_sub_duration", NULL};
+static const char *const opt_name_fix_sub_duration_heartbeat[] = {"fix_sub_duration_heartbeat", NULL};
 static const char *const opt_name_canvas_sizes[]              = {"canvas_size", NULL};
 static const char *const opt_name_pass[]                      = {"pass", NULL};
 static const char *const opt_name_passlogfiles[]              = {"passlogfile", NULL};
@@ -1600,6 +1601,9 @@ static OutputStream *new_output_stream(OptionsContext *o, AVFormatContext *oc, e
     MATCH_PER_STREAM_OPT(bits_per_raw_sample, i, ost->bits_per_raw_sample,
                          oc, st);
 
+    MATCH_PER_STREAM_OPT(fix_sub_duration_heartbeat, i, ost->fix_sub_duration_heartbeat,
+                         oc, st);
+
     if (oc->oformat->flags & AVFMT_GLOBALHEADER)
         ost->enc_ctx->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;
 
@@ -3793,6 +3797,11 @@ const OptionDef options[] = {
     { "autoscale",        HAS_ARG | OPT_BOOL | OPT_SPEC |
                           OPT_EXPERT | OPT_OUTPUT,                               { .off = OFFSET(autoscale) },
         "automatically insert a scale filter at the end of the filter graph" },
+    { "fix_sub_duration_heartbeat", OPT_VIDEO | OPT_BOOL | OPT_EXPERT |
+                                    OPT_SPEC | OPT_OUTPUT,                       { .off = OFFSET(fix_sub_duration_heartbeat) },
+        "set this video output stream to be a heartbeat stream for "
+        "fix_sub_duration, according to which subtitles should be split at "
+        "random access points" },
 
     /* audio options */
     { "aframes",        OPT_AUDIO | HAS_ARG  | OPT_PERFILE | OPT_OUTPUT,           { .func_arg = opt_audio_frames },
-- 
2.33.1

_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".