[FFmpeg-devel] [RFC/PATCH] decklink_enc: Add support for playout of EIA-608 caption codec

[FFmpeg-devel] [RFC/PATCH] decklink_enc: Add support for playout of EIA-608 caption codec

[Devin Heitmueller]

Today the decklink SDI output supports putting out captions over
VANC when the caption data is packet side data.  However some
container formats such as MP4 support a dedicated subtitle stream,
and we end up receiving packets containing the 608 tuples separate
from the video frames.

Make use of a simple intermediate queue to stash the 608 tuples,
and then attach them to the video frame side data such that it can
be inserted into VANC via the subsequent call to construct_cc().
Unlike with packet side data that carries the captions associated
with that one frame, individual c608 packets can contain tuples for
entire batches of frames, so we need to pace out how quickly they are
inserted into the video based on the framerate.

Example usage:
./ffmpeg -i file.mp4 -map 0:0 -map 0:1 -map 0:3 -codec:2 copy -vcodec v210 -f decklink 'DeckLink Duo (4)'

Note that the user needs to specify -codec copy for the
c608 subtitle track, since there is no decoding/encoding
happening.

The CC fifo code here overlaps with some of the functionality
found in the libavfilter/ccfifo.[c/h], but that code can't
be referenced as it is currently private to libavfilter.  If
the ccfifo stuff gets moved to libavutil then the routine here
can be simplified.

Signed-off-by: Devin Heitmueller <dheitmueller@ltnglobal.com>
---
 libavdevice/decklink_common.h |  2 +
 libavdevice/decklink_enc.cpp  | 94 ++++++++++++++++++++++++++++++++++++++++++-
 libavdevice/decklink_enc_c.c  |  2 +-
 3 files changed, 95 insertions(+), 3 deletions(-)

diff --git a/libavdevice/decklink_common.h b/libavdevice/decklink_common.h
index 99afcd4..51ad5da 100644
--- a/libavdevice/decklink_common.h
+++ b/libavdevice/decklink_common.h
@@ -31,6 +31,7 @@
 
 extern "C" {
 #include "libavcodec/packet_internal.h"
+#include "libavutil/fifo.h"
 }
 #include "libavutil/thread.h"
 #include "decklink_common_c.h"
@@ -114,6 +115,7 @@ struct decklink_ctx {
 
     /* Output VANC queue */
     AVPacketQueue vanc_queue;
+    AVFifo *cc_fifo;
 
     /* Streams present */
     int audio;
diff --git a/libavdevice/decklink_enc.cpp b/libavdevice/decklink_enc.cpp
index d7357a0..23f00c8 100644
--- a/libavdevice/decklink_enc.cpp
+++ b/libavdevice/decklink_enc.cpp
@@ -345,6 +345,30 @@ static int decklink_setup_data(AVFormatContext *avctx, AVStream *st)
     return ret;
 }
 
+static int decklink_setup_subtitle(AVFormatContext *avctx, AVStream *st)
+{
+    struct decklink_cctx *cctx = (struct decklink_cctx *)avctx->priv_data;
+    struct decklink_ctx *ctx = (struct decklink_ctx *)cctx->ctx;
+
+    int ret = -1;
+
+    switch(st->codecpar->codec_id) {
+#if CONFIG_LIBKLVANC
+    case AV_CODEC_ID_EIA_608:
+        if (!ctx->cc_fifo)
+            ctx->cc_fifo = av_fifo_alloc2(128, 3, 0);
+        if (ctx->cc_fifo)
+            ret = 0;
+        break;
+#endif
+    default:
+        av_log(avctx, AV_LOG_ERROR, "Unsupported subtitle codec specified\n");
+        break;
+    }
+
+    return ret;
+}
+
 av_cold int ff_decklink_write_trailer(AVFormatContext *avctx)
 {
     struct decklink_cctx *cctx = (struct decklink_cctx *)avctx->priv_data;
@@ -371,6 +395,7 @@ av_cold int ff_decklink_write_trailer(AVFormatContext *avctx)
     klvanc_context_destroy(ctx->vanc_ctx);
 #endif
     avpacket_queue_end(&ctx->vanc_queue);
+    av_fifo_freep2(&ctx->cc_fifo);
 
     av_freep(&cctx->ctx);
 
@@ -523,6 +548,54 @@ out:
         free(afd_words);
 }
 
+/* Parse any EIA-608 subtitles sitting on the queue, and write packet side data
+   that will later be handled by construct_cc... */
+static void parse_608subs(AVFormatContext *avctx, struct decklink_ctx *ctx, AVPacket *pkt)
+{
+    int cc_count = 0;
+    int cc_filled = 0;
+
+    if (!ctx->cc_fifo)
+        return;
+
+    /* 29.97 and 59.94 are really the only two framerates we care about */
+    if (ctx->bmd_tb_den == 30000 && ctx->bmd_tb_num == 1001) {
+        cc_count = 20;
+    } else if (ctx->bmd_tb_den == 60000 && ctx->bmd_tb_num == 1001) {
+        cc_count = 10;
+    }
+    if (cc_count > 0) {
+        uint8_t *cc_buf = av_packet_new_side_data(pkt, AV_PKT_DATA_A53_CC, cc_count * 3);
+        if (!cc_buf)
+            return;
+        int len = av_fifo_can_read(ctx->cc_fifo);
+        if (len > 0) {
+            uint8_t buf[3];
+            av_fifo_read(ctx->cc_fifo, buf, 1);
+            av_log(avctx, AV_LOG_ERROR, "Got a 608 packet!  size=%d\n", len);
+            memcpy(cc_buf, buf, 3);
+            cc_filled++;
+            if (cc_count == 20 && len > 1) {
+                /* If we're at 30 FPS, we need to insert a second tuple if one is
+                   available and it's for CC3/CC4) */
+                av_fifo_peek(ctx->cc_fifo, buf, 1, 0);
+                if (buf[0] == 0xfd) {
+                    av_fifo_read(ctx->cc_fifo, buf, 1);
+                    av_log(avctx, AV_LOG_ERROR, "Got a 608 packet!  size=%d\n", len);
+                    memcpy(cc_buf, buf, 3);
+                    cc_filled++;
+                }
+            }
+        }
+        /* Pad out the rest of the field */
+        for(int i = cc_filled; i < cc_count; i++) {
+            cc_buf[i*3] = 0xfa;
+            cc_buf[i*3+1] = 0x00;
+            cc_buf[i*3+2] = 0x00;
+        }
+    }
+}
+
 static int decklink_construct_vanc(AVFormatContext *avctx, struct decklink_ctx *ctx,
                                    AVPacket *pkt, decklink_frame *frame,
                                    AVStream *st)
@@ -533,6 +606,7 @@ static int decklink_construct_vanc(AVFormatContext *avctx, struct decklink_ctx *
     if (!ctx->supports_vanc)
         return 0;
 
+    parse_608subs(avctx, ctx, pkt);
     construct_cc(avctx, ctx, pkt, &vanc_lines);
     construct_afd(avctx, ctx, pkt, &vanc_lines, st);
 
@@ -793,6 +867,16 @@ static int decklink_write_data_packet(AVFormatContext *avctx, AVPacket *pkt)
     return 0;
 }
 
+static int decklink_write_subtitle_packet(AVFormatContext *avctx, AVPacket *pkt)
+{
+    struct decklink_cctx *cctx = (struct decklink_cctx *)avctx->priv_data;
+    struct decklink_ctx *ctx = (struct decklink_ctx *)cctx->ctx;
+
+    av_fifo_write(ctx->cc_fifo, pkt->data, pkt->size / 3);
+
+    return 0;
+}
+
 extern "C" {
 
 av_cold int ff_decklink_write_header(AVFormatContext *avctx)
@@ -860,6 +944,9 @@ av_cold int ff_decklink_write_header(AVFormatContext *avctx)
         } else if (c->codec_type == AVMEDIA_TYPE_DATA) {
             if (decklink_setup_data(avctx, st))
                 goto error;
+        } else if (c->codec_type == AVMEDIA_TYPE_SUBTITLE) {
+            if (decklink_setup_subtitle(avctx, st))
+                goto error;
         } else {
             av_log(avctx, AV_LOG_ERROR, "Unsupported stream type.\n");
             goto error;
@@ -870,7 +957,8 @@ av_cold int ff_decklink_write_header(AVFormatContext *avctx)
     for (n = 0; n < avctx->nb_streams; n++) {
         AVStream *st = avctx->streams[n];
         AVCodecParameters *c = st->codecpar;
-        if (c->codec_type == AVMEDIA_TYPE_DATA)
+        if (c->codec_type == AVMEDIA_TYPE_DATA ||
+            c->codec_type == AVMEDIA_TYPE_SUBTITLE)
             avpriv_set_pts_info(st, 64, ctx->bmd_tb_num, ctx->bmd_tb_den);
     }
     avpacket_queue_init (avctx, &ctx->vanc_queue);
@@ -890,8 +978,10 @@ int ff_decklink_write_packet(AVFormatContext *avctx, AVPacket *pkt)
         return decklink_write_video_packet(avctx, pkt);
     else if (st->codecpar->codec_type == AVMEDIA_TYPE_AUDIO)
         return decklink_write_audio_packet(avctx, pkt);
-    else if (st->codecpar->codec_type == AVMEDIA_TYPE_DATA) {
+    else if (st->codecpar->codec_type == AVMEDIA_TYPE_DATA)
         return decklink_write_data_packet(avctx, pkt);
+    else if (st->codecpar->codec_type == AVMEDIA_TYPE_SUBTITLE) {
+        return decklink_write_subtitle_packet(avctx, pkt);
     }
 
     return AVERROR(EIO);
diff --git a/libavdevice/decklink_enc_c.c b/libavdevice/decklink_enc_c.c
index 7aab660..d593cca 100644
--- a/libavdevice/decklink_enc_c.c
+++ b/libavdevice/decklink_enc_c.c
@@ -78,7 +78,7 @@ const FFOutputFormat ff_decklink_muxer = {
     .p.long_name      = NULL_IF_CONFIG_SMALL("Blackmagic DeckLink output"),
     .p.audio_codec    = AV_CODEC_ID_PCM_S16LE,
     .p.video_codec    = AV_CODEC_ID_WRAPPED_AVFRAME,
-    .p.subtitle_codec = AV_CODEC_ID_NONE,
+    .p.subtitle_codec = AV_CODEC_ID_EIA_608,
     .p.flags          = AVFMT_NOFILE,
     .p.priv_class     = &decklink_muxer_class,
     .get_device_list = ff_decklink_list_output_devices,
-- 
1.8.3.1

_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".