Re: [FFmpeg-devel] [PATCH v3 2/2] decklink_enc: add support for SMPTE 2038 VANC packet output

[Marton Balint <cus@passwd.hu>]

On Fri, 30 Jun 2023, Devin Heitmueller wrote:

> Support decoding and embedding VANC packets delivered via SMPTE 2038
> into the SDI output.  We leverage an intermediate queue because
> data packets are announced separately from video but we need to embed
> the data into the video frame when it is output.
>
> Note that this patch has some additional abstraction for data
> streams in general as opposed to just SMPTE 2038 packets.  This is
> because subsequent patches will introduce support for other
> data codecs.

Thanks, will apply with some minor cosmetic change and avdevice micro 
version bump.

Regards,
Marton

>
> Thanks to Marton Balint for review/feedback.
>
> Signed-off-by: Devin Heitmueller <dheitmueller@ltnglobal.com>
> ---
> doc/outdevs.texi                |  5 +++
> libavdevice/decklink_common.cpp | 16 +++++++
> libavdevice/decklink_common.h   |  4 ++
> libavdevice/decklink_common_c.h |  1 +
> libavdevice/decklink_enc.cpp    | 94 ++++++++++++++++++++++++++++++++++++++++-
> libavdevice/decklink_enc_c.c    |  1 +
> 6 files changed, 120 insertions(+), 1 deletion(-)
>
> diff --git a/doc/outdevs.texi b/doc/outdevs.texi
> index 93391db..f0484bb 100644
> --- a/doc/outdevs.texi
> +++ b/doc/outdevs.texi
> @@ -235,6 +235,11 @@ Enable SMPTE Level A mode on the used output.
> Must be @samp{unset}, @samp{true} or @samp{false}.
> Defaults to @option{unset}.
>
> +@item vanc_queue_size
> +Sets maximum output buffer size in bytes for VANC data. If the buffering reaches this value,
> +outgoing VANC data will be dropped.
> +Defaults to @samp{1048576}.
> +
> @end table
>
> @subsection Examples
> diff --git a/libavdevice/decklink_common.cpp b/libavdevice/decklink_common.cpp
> index 5e8d612..6eefb6f1 100644
> --- a/libavdevice/decklink_common.cpp
> +++ b/libavdevice/decklink_common.cpp
> @@ -485,6 +485,22 @@ int ff_decklink_packet_queue_get(DecklinkPacketQueue *q, AVPacket *pkt, int bloc
>     return ret;
> }
>
> +int64_t ff_decklink_packet_queue_peekpts(DecklinkPacketQueue *q)
> +{
> +    PacketListEntry *pkt1;
> +    int64_t pts = -1;
> +
> +    pthread_mutex_lock(&q->mutex);
> +    pkt1 = q->pkt_list.head;
> +    if (pkt1) {
> +        pts = pkt1->pkt.pts;
> +    }
> +    pthread_mutex_unlock(&q->mutex);
> +
> +    return pts;
> +}
> +
> +
> int ff_decklink_list_devices(AVFormatContext *avctx,
>                              struct AVDeviceInfoList *device_list,
>                              int show_inputs, int show_outputs)
> diff --git a/libavdevice/decklink_common.h b/libavdevice/decklink_common.h
> index 53e9983..34ab1b9 100644
> --- a/libavdevice/decklink_common.h
> +++ b/libavdevice/decklink_common.h
> @@ -115,6 +115,9 @@ struct decklink_ctx {
>
>     CCFifo cc_fifo;      ///< closed captions
>
> +    /* Output VANC queue */
> +    DecklinkPacketQueue vanc_queue;
> +
>     /* Streams present */
>     int audio;
>     int video;
> @@ -241,5 +244,6 @@ void ff_decklink_packet_queue_end(DecklinkPacketQueue *q);
> unsigned long long ff_decklink_packet_queue_size(DecklinkPacketQueue *q);
> int ff_decklink_packet_queue_put(DecklinkPacketQueue *q, AVPacket *pkt);
> int ff_decklink_packet_queue_get(DecklinkPacketQueue *q, AVPacket *pkt, int block);
> +int64_t ff_decklink_packet_queue_peekpts(DecklinkPacketQueue *q);
>
> #endif /* AVDEVICE_DECKLINK_COMMON_H */
> diff --git a/libavdevice/decklink_common_c.h b/libavdevice/decklink_common_c.h
> index 75896ad..9c55d89 100644
> --- a/libavdevice/decklink_common_c.h
> +++ b/libavdevice/decklink_common_c.h
> @@ -63,6 +63,7 @@ struct decklink_cctx {
>     char *format_code;
>     int raw_format;
>     int64_t queue_size;
> +    int64_t vanc_queue_size;
>     int copyts;
>     int64_t timestamp_align;
>     int timing_offset;
> diff --git a/libavdevice/decklink_enc.cpp b/libavdevice/decklink_enc.cpp
> index 6906cb0..11e67c4 100644
> --- a/libavdevice/decklink_enc.cpp
> +++ b/libavdevice/decklink_enc.cpp
> @@ -345,6 +345,25 @@ static int decklink_setup_subtitle(AVFormatContext *avctx, AVStream *st)
>     return ret;
> }
>
> +static int decklink_setup_data(AVFormatContext *avctx, AVStream *st)
> +{
> +    int ret = -1;
> +
> +    switch(st->codecpar->codec_id) {
> +#if CONFIG_LIBKLVANC
> +    case AV_CODEC_ID_SMPTE_2038:
> +        /* No specific setup required */
> +        ret = 0;
> +        break;
> +#endif
> +    default:
> +        av_log(avctx, AV_LOG_ERROR, "Unsupported data codec specified\n");
> +        break;
> +    }
> +
> +    return ret;
> +}
> +
> av_cold int ff_decklink_write_trailer(AVFormatContext *avctx)
> {
>     struct decklink_cctx *cctx = (struct decklink_cctx *)avctx->priv_data;
> @@ -370,6 +389,7 @@ av_cold int ff_decklink_write_trailer(AVFormatContext *avctx)
> #if CONFIG_LIBKLVANC
>     klvanc_context_destroy(ctx->vanc_ctx);
> #endif
> +    ff_decklink_packet_queue_end(&ctx->vanc_queue);
>
>     ff_ccfifo_uninit(&ctx->cc_fifo);
>     av_freep(&cctx->ctx);
> @@ -552,6 +572,58 @@ static int decklink_construct_vanc(AVFormatContext *avctx, struct decklink_ctx *
>     construct_cc(avctx, ctx, pkt, &vanc_lines);
>     construct_afd(avctx, ctx, pkt, &vanc_lines, st);
>
> +    /* See if there any pending data packets to process */
> +    while (ff_decklink_packet_queue_size(&ctx->vanc_queue) > 0) {
> +        AVStream *vanc_st;
> +        AVPacket vanc_pkt;
> +        int64_t pts;
> +
> +        pts = ff_decklink_packet_queue_peekpts(&ctx->vanc_queue);
> +        if (pts > ctx->last_pts) {
> +            /* We haven't gotten to the video frame we are supposed to inject
> +               the oldest VANC packet into yet, so leave it on the queue... */
> +            break;
> +        }
> +
> +        ret = ff_decklink_packet_queue_get(&ctx->vanc_queue, &vanc_pkt, 1);
> +        if (vanc_pkt.pts + 1 < ctx->last_pts) {
> +            av_log(avctx, AV_LOG_WARNING, "VANC packet too old, throwing away\n");
> +            av_packet_unref(&vanc_pkt);
> +            continue;
> +        }
> +
> +        vanc_st = avctx->streams[vanc_pkt.stream_index];
> +        if (vanc_st->codecpar->codec_id == AV_CODEC_ID_SMPTE_2038) {
> +            struct klvanc_smpte2038_anc_data_packet_s *pkt_2038 = 0;
> +
> +            klvanc_smpte2038_parse_pes_payload(vanc_pkt.data, vanc_pkt.size, &pkt_2038);
> +            if (pkt_2038 == NULL) {
> +                av_log(avctx, AV_LOG_ERROR, "failed to decode SMPTE 2038 PES packet");
> +                av_packet_unref(&vanc_pkt);
> +                continue;
> +            }
> +            for (int i = 0; i < pkt_2038->lineCount; i++) {
> +                struct klvanc_smpte2038_anc_data_line_s *l = &pkt_2038->lines[i];
> +                uint16_t *vancWords = NULL;
> +                uint16_t vancWordCount;
> +
> +                if (klvanc_smpte2038_convert_line_to_words(l, &vancWords,
> +                                                           &vancWordCount) < 0)
> +                    break;
> +
> +                ret = klvanc_line_insert(ctx->vanc_ctx, &vanc_lines, vancWords,
> +                                         vancWordCount, l->line_number, 0);
> +                free(vancWords);
> +                if (ret != 0) {
> +                    av_log(avctx, AV_LOG_ERROR, "VANC line insertion failed\n");
> +                    break;
> +                }
> +            }
> +            klvanc_smpte2038_anc_data_packet_free(pkt_2038);
> +        }
> +        av_packet_unref(&vanc_pkt);
> +    }
> +
>     IDeckLinkVideoFrameAncillary *vanc;
>     int result = ctx->dlo->CreateAncillaryData(bmdFormat10BitYUV, &vanc);
>     if (result != S_OK) {
> @@ -750,6 +822,18 @@ static int decklink_write_subtitle_packet(AVFormatContext *avctx, AVPacket *pkt)
>     return 0;
> }
>
> +static int decklink_write_data_packet(AVFormatContext *avctx, AVPacket *pkt)
> +{
> +    struct decklink_cctx *cctx = (struct decklink_cctx *)avctx->priv_data;
> +    struct decklink_ctx *ctx = (struct decklink_ctx *)cctx->ctx;
> +
> +    if (ff_decklink_packet_queue_put(&ctx->vanc_queue, pkt) < 0) {
> +        av_log(avctx, AV_LOG_WARNING, "Failed to queue DATA packet\n");
> +    }
> +
> +    return 0;
> +}
> +
> extern "C" {
>
> av_cold int ff_decklink_write_header(AVFormatContext *avctx)
> @@ -814,6 +898,9 @@ av_cold int ff_decklink_write_header(AVFormatContext *avctx)
>         } else if (c->codec_type == AVMEDIA_TYPE_VIDEO) {
>             if (decklink_setup_video(avctx, st))
>                 goto error;
> +        } else if (c->codec_type == AVMEDIA_TYPE_DATA) {
> +            if (decklink_setup_data(avctx, st))
> +                goto error;
>         } else if (c->codec_type == AVMEDIA_TYPE_SUBTITLE) {
>             if (decklink_setup_subtitle(avctx, st))
>                 goto error;
> @@ -823,13 +910,16 @@ av_cold int ff_decklink_write_header(AVFormatContext *avctx)
>         }
>     }
>
> +    /* Reconfigure the data/subtitle stream clocks to match the video */
>     for (n = 0; n < avctx->nb_streams; n++) {
>         AVStream *st = avctx->streams[n];
>         AVCodecParameters *c = st->codecpar;
>
> -        if(c->codec_type == AVMEDIA_TYPE_SUBTITLE)
> +        if(c->codec_type == AVMEDIA_TYPE_DATA ||
> +           c->codec_type == AVMEDIA_TYPE_SUBTITLE)
>             avpriv_set_pts_info(st, 64, ctx->bmd_tb_num, ctx->bmd_tb_den);
>     }
> +    ff_decklink_packet_queue_init(avctx, &ctx->vanc_queue, cctx->vanc_queue_size);
>
>     ret = ff_ccfifo_init(&ctx->cc_fifo, av_make_q(ctx->bmd_tb_den, ctx->bmd_tb_num), avctx);
>     if (ret < 0) {
> @@ -852,6 +942,8 @@ int ff_decklink_write_packet(AVFormatContext *avctx, AVPacket *pkt)
>         return decklink_write_video_packet(avctx, pkt);
>     else if (st->codecpar->codec_type == AVMEDIA_TYPE_AUDIO)
>         return decklink_write_audio_packet(avctx, pkt);
> +    else if (st->codecpar->codec_type == AVMEDIA_TYPE_DATA)
> +        return decklink_write_data_packet(avctx, pkt);
>     else if (st->codecpar->codec_type == AVMEDIA_TYPE_SUBTITLE)
>         return decklink_write_subtitle_packet(avctx, pkt);
>
> diff --git a/libavdevice/decklink_enc_c.c b/libavdevice/decklink_enc_c.c
> index 0a3984b..25ffe77 100644
> --- a/libavdevice/decklink_enc_c.c
> +++ b/libavdevice/decklink_enc_c.c
> @@ -32,6 +32,7 @@ static const AVOption options[] = {
>     { "list_devices", "use ffmpeg -sinks decklink instead", OFFSET(list_devices), AV_OPT_TYPE_BOOL, { .i64 = 0   }, 0, 1, ENC | AV_OPT_FLAG_DEPRECATED},
>     { "list_formats", "list supported formats"  , OFFSET(list_formats), AV_OPT_TYPE_INT   , { .i64 = 0   }, 0, 1, ENC },
>     { "preroll"     , "video preroll in seconds", OFFSET(preroll     ), AV_OPT_TYPE_DOUBLE, { .dbl = 0.5 }, 0, 5, ENC },
> +    { "vanc_queue_size", "VANC queue buffer size", OFFSET(vanc_queue_size), AV_OPT_TYPE_INT64, { .i64 = (1024 * 1024)}, 0, INT64_MAX, ENC },
> #if BLACKMAGIC_DECKLINK_API_VERSION >= 0x0b000000
>     { "duplex_mode" , "duplex mode"             , OFFSET(duplex_mode ), AV_OPT_TYPE_INT   , { .i64 = 0   }, 0, 5, ENC, "duplex_mode"},
> #else
> -- 
> 1.8.3.1
>
> _______________________________________________
> ffmpeg-devel mailing list
> ffmpeg-devel@ffmpeg.org
> https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
>
> To unsubscribe, visit link above, or email
> ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
>
_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".