Re: [FFmpeg-devel] [PATCH v5 1/6] ccfifo: Properly handle CEA-708 captions through framerate conversion

[Lance Wang <lance.lmwang@gmail.com>]

On Fri, May 5, 2023 at 5:07 AM Devin Heitmueller <
devin.heitmueller@ltnglobal.com> wrote:

> When transcoding video that contains 708 closed captions, the
> caption data is tied to the frames as side data.  Simply dropping
> or adding frames to change the framerate will result in loss of
> data, so the caption data needs to be preserved and reformatted.
>
> For example, without this patch converting 720p59 to 1080i59
> would result in loss of 50% of the caption bytes, resulting in
> garbled 608 captions and 708 probably wouldn't render at all.
> Further, the frames that are there will have an illegal
> cc_count for the target framerate, so some decoders may ignore
> the packets entirely.
>
> Extract the 608 and 708 tuples and insert them onto queues.  Then
> after dropping/adding frames, re-write the tuples back into the
> resulting frames at the appropriate rate given the target
> framerate.  This includes both having the correct cc_count as
> well as clocking out the 608 pairs at the appropriate rate.
>
> Thanks for Lance Wang <lance.lmwang@gmail.com> and Anton
> Khirnov <anton@khirnov.net> for providing review/feedback.
>
> Signed-off-by: Devin Heitmueller <dheitmueller@ltnglobal.com>
> ---
>  libavfilter/Makefile |   1 +
>  libavfilter/ccfifo.c | 222
> +++++++++++++++++++++++++++++++++++++++++++++++++++
>  libavfilter/ccfifo.h | 110 +++++++++++++++++++++++++
>  3 files changed, 333 insertions(+)
>  create mode 100644 libavfilter/ccfifo.c
>  create mode 100644 libavfilter/ccfifo.h
>
> diff --git a/libavfilter/Makefile b/libavfilter/Makefile
> index 482aeaf..68c8f14 100644
> --- a/libavfilter/Makefile
> +++ b/libavfilter/Makefile
> @@ -14,6 +14,7 @@ OBJS = allfilters.o
>                \
>         buffersink.o                                                     \
>         buffersrc.o                                                      \
>         colorspace.o                                                     \
> +       ccfifo.o                                                         \
>         drawutils.o                                                      \
>         fifo.o                                                           \
>         formats.o                                                        \
> diff --git a/libavfilter/ccfifo.c b/libavfilter/ccfifo.c
> new file mode 100644
> index 0000000..357f764
> --- /dev/null
> +++ b/libavfilter/ccfifo.c
> @@ -0,0 +1,222 @@
> +/*
> + * CEA-708 Closed Captioning FIFO
> + * Copyright (c) 2023 LTN Global Communications
> + *
> + * Author: Devin Heitmueller <dheitmueller@ltnglobal.com>
> + *
> + * This file is part of FFmpeg.
> + *
> + * FFmpeg is free software; you can redistribute it and/or
> + * modify it under the terms of the GNU Lesser General Public
> + * License as published by the Free Software Foundation; either
> + * version 2.1 of the License, or (at your option) any later version.
> + *
> + * FFmpeg is distributed in the hope that it will be useful,
> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
> + * Lesser General Public License for more details.
> + *
> + * You should have received a copy of the GNU Lesser General Public
> + * License along with FFmpeg; if not, write to the Free Software
> + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
> 02110-1301 USA
> + */
> +
> +#include "ccfifo.h"
> +
> +struct AVCCFifo {
> +    AVFifo *cc_608_fifo;
> +    AVFifo *cc_708_fifo;
> +    AVRational framerate;
> +    int expected_cc_count;
> +    int expected_608;
> +    int cc_detected;
> +    int passthrough;
> +    int passthrough_warning;
> +    void *log_ctx;
> +};
> +
> +#define MAX_CC_ELEMENTS 128
> +#define CC_BYTES_PER_ENTRY 3
> +
> +struct cc_lookup {
> +    int num;
> +    int den;
> +    int cc_count;
> +    int num_608;
> +};
> +
> +const static struct cc_lookup cc_lookup_vals[] = {
> +    { 15, 1, 40, 4 },
> +    { 24, 1, 25, 3 },
> +    { 24000, 1001, 25, 3 },
> +    { 30, 1, 20, 2 },
> +    { 30000, 1001, 20, 2},
> +    { 60, 1, 10, 1 },
> +    { 60000, 1001, 10, 1},
> +};
> +
> +void ff_ccfifo_freep(AVCCFifo **ccf)
> +{
> +    AVCCFifo *tmp = *ccf;
> +    av_fifo_freep2(&tmp->cc_608_fifo);
> +    av_fifo_freep2(&tmp->cc_708_fifo);
> +    av_freep(*ccf);
> +}
> +
> +AVCCFifo *ff_ccfifo_alloc(AVRational framerate, void *log_ctx)
> +{
> +    AVCCFifo *ccf;
> +    int i;
> +
> +    ccf = av_mallocz(sizeof(*ccf));
> +    if (!ccf)
> +        return NULL;
> +
> +    ccf->log_ctx = log_ctx;
> +    ccf->framerate = framerate;
> +
> +    if (!(ccf->cc_708_fifo = av_fifo_alloc2(MAX_CC_ELEMENTS,
> CC_BYTES_PER_ENTRY, 0)))
> +        goto error;
> +
> +    if (!(ccf->cc_608_fifo = av_fifo_alloc2(MAX_CC_ELEMENTS,
> CC_BYTES_PER_ENTRY, 0)))
> +        goto error;
> +
> +    /* Based on the target FPS, figure out the expected cc_count and
> number of
> +       608 tuples per packet.  See ANSI/CTA-708-E Sec 4.3.6.1. */
> +    for (i = 0; i < FF_ARRAY_ELEMS(cc_lookup_vals); i++) {
> +        if (framerate.num == cc_lookup_vals[i].num &&
> +            framerate.den == cc_lookup_vals[i].den) {
> +            ccf->expected_cc_count = cc_lookup_vals[i].cc_count;
> +            ccf->expected_608 = cc_lookup_vals[i].num_608;
> +            break;
> +        }
> +    }
> +
> +    if (ccf->expected_608 == 0) {
> +        /* We didn't find an output frame we support.  We'll let the call
> succeed
> +           and the FIFO to be allocated, but the extract/inject functions
> will simply
> +           leave everything the way it is */
> +        ccf->passthrough = 1;
> +    }
> +
> +    return ccf;
> +
> +error:
> +    ff_ccfifo_freep(&ccf);
> +    return NULL;
> +}
> +
> +int ff_ccfifo_getoutputsize(AVCCFifo *ccf)
> +{
> +    return ccf->expected_cc_count * CC_BYTES_PER_ENTRY;
> +}
> +
> +int ff_ccfifo_ccdetected(AVCCFifo *ccf)
> +{
> +    return ccf->cc_detected;
> +}
> +
> +int ff_ccfifo_injectbytes(AVCCFifo *ccf, uint8_t *cc_data, size_t len)
> +{
> +    int cc_608_tuples = 0;
> +    int cc_708_tuples = 0;
> +    int cc_filled = 0;
> +
> +    if (ccf->passthrough) {
> +        return 0;
> +    }
> +
> +    if (len < ff_ccfifo_getoutputsize(ccf)) {
> +        return AVERROR(EINVAL);
> +    }
> +
> +    /* Insert any available data from the 608 FIFO */
> +    if (ccf->expected_608 <= av_fifo_can_read(ccf->cc_608_fifo))
> +        cc_608_tuples = ccf->expected_608;
> +    else
> +        cc_608_tuples = av_fifo_can_read(ccf->cc_608_fifo);
> +    av_fifo_read(ccf->cc_608_fifo, cc_data, cc_608_tuples);
> +    cc_filled += cc_608_tuples;
> +
> +    /* Insert any available data from the 708 FIFO */
> +    if ((ccf->expected_cc_count - cc_filled) <=
> av_fifo_can_read(ccf->cc_708_fifo))
> +        cc_708_tuples = ccf->expected_cc_count - cc_filled;
> +    else
> +        cc_708_tuples = av_fifo_can_read(ccf->cc_708_fifo);
> +    av_fifo_read(ccf->cc_708_fifo, &cc_data[cc_filled *
> CC_BYTES_PER_ENTRY], cc_708_tuples);
> +    cc_filled += cc_708_tuples;
> +
> +    /* Insert 708 padding into any remaining fields */
> +    while (cc_filled < ccf->expected_cc_count) {
> +        cc_data[cc_filled * CC_BYTES_PER_ENTRY]     = 0xfa;
> +        cc_data[cc_filled * CC_BYTES_PER_ENTRY + 1] = 0x00;
> +        cc_data[cc_filled * CC_BYTES_PER_ENTRY + 2] = 0x00;
> +        cc_filled++;
> +    }
> +
> +    return 0;
> +}
> +
> +int ff_ccfifo_inject(AVCCFifo *ccf, AVFrame *frame)
> +{
> +    AVFrameSideData *sd;
> +    int ret;
> +
> +    if (ccf->passthrough == 1 || ccf->cc_detected == 0)
> +        return 0;
> +
> +    sd = av_frame_new_side_data(frame, AV_FRAME_DATA_A53_CC,
> +                                ff_ccfifo_getoutputsize(ccf));
> +    if (sd) {
> +        ret = ff_ccfifo_injectbytes(ccf, sd->data, sd->size);
> +        if (ret < 0) {
> +            av_frame_remove_side_data(frame, AV_FRAME_DATA_A53_CC);
> +            return AVERROR(ENOMEM);
>

prefer to return ret from ff_ccfifo_injectbytes()


> +        }
> +    }
> +
> +    return 0;
> +}
> +
> +int ff_ccfifo_extractbytes(AVCCFifo *ccf, uint8_t *cc_bytes, size_t len)
> +{
> +    int cc_count = len / CC_BYTES_PER_ENTRY;
> +
> +    if (ccf->passthrough == 1) {
> +        av_log_once(ccf->log_ctx, AV_LOG_WARNING, AV_LOG_DEBUG,
> &ccf->passthrough_warning,
> +                    "cc_fifo cannot transcode captions fps=%d/%d\n",
> +                    ccf->framerate.num, ccf->framerate.den);
> +        return 0;
> +    }
> +
> +    ccf->cc_detected = 1;
> +
> +    for (int i = 0; i < cc_count; i++) {
> +        /* See ANSI/CTA-708-E Sec 4.3, Table 3 */
> +        uint8_t cc_valid = (cc_bytes[CC_BYTES_PER_ENTRY*i] & 0x04) >> 2;
> +        uint8_t cc_type = cc_bytes[CC_BYTES_PER_ENTRY*i] & 0x03;
> +        if (cc_type == 0x00 || cc_type == 0x01) {
> +            av_fifo_write(ccf->cc_608_fifo,
> &cc_bytes[CC_BYTES_PER_ENTRY*i], 1);
> +        } else if (cc_valid && (cc_type == 0x02 || cc_type == 0x03)) {
> +            av_fifo_write(ccf->cc_708_fifo,
> &cc_bytes[CC_BYTES_PER_ENTRY*i], 1);
> +        }
> +    }
> +    return 0;
> +}
> +
> +/* Read the A53 side data, discard padding, and put 608/708 into
> +   queues so we can ensure they get into the output frames at
> +   the correct rate... */
> +int ff_ccfifo_extract(AVCCFifo *ccf, AVFrame *frame)
> +{
> +    AVFrameSideData *side_data = av_frame_get_side_data(frame,
> AV_FRAME_DATA_A53_CC);
> +    if (side_data) {
> +        ff_ccfifo_extractbytes(ccf, side_data->data, side_data->size);
> +
> +        /* Remove the side data, as we will re-create it on the
> +           output as needed */
> +        if (!ccf->passthrough)
> +            av_frame_remove_side_data(frame, AV_FRAME_DATA_A53_CC);
> +    }
> +    return 0;
> +}
> diff --git a/libavfilter/ccfifo.h b/libavfilter/ccfifo.h
> new file mode 100644
> index 0000000..44c9245
> --- /dev/null
> +++ b/libavfilter/ccfifo.h
> @@ -0,0 +1,110 @@
> +/*
> + * CEA-708 Closed Captioning FIFO
> + * Copyright (c) 2023 LTN Global Communications
> + *
> + * Author: Devin Heitmueller <dheitmueller@ltnglobal.com>
> + *
> + * This file is part of FFmpeg.
> + *
> + * FFmpeg is free software; you can redistribute it and/or
> + * modify it under the terms of the GNU Lesser General Public
> + * License as published by the Free Software Foundation; either
> + * version 2.1 of the License, or (at your option) any later version.
> + *
> + * FFmpeg is distributed in the hope that it will be useful,
> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
> + * Lesser General Public License for more details.
> + *
> + * You should have received a copy of the GNU Lesser General Public
> + * License along with FFmpeg; if not, write to the Free Software
> + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
> 02110-1301 USA
> + */
> +
> +/**
> + * @file
> + * CC FIFO Buffer
> + */
> +
> +#ifndef AVFILTER_CCFIFO_H
> +#define AVFILTER_CCFIFO_H
> +
> +#include "libavutil/avutil.h"
> +#include "libavutil/frame.h"
> +#include "libavutil/fifo.h"
> +
> +typedef struct AVCCFifo AVCCFifo;
> +
> +/**
> + * Allocate an AVCCFifo.
> + *
> + * @param framerate   output framerate
> + * @param log_ctx     used for any av_log() calls
> + * @return            newly allocated AVCCFifo, or NULL on error
> + */
> +AVCCFifo *ff_ccfifo_alloc(AVRational framerate, void *log_ctx);
> +
> +/**
> + * Free an AVCCFifo
> + *
> + * @param ccf Pointer to the pointer to the AVCCFifo which should be freed
> + * @note `*ptr = NULL` is safe and leads to no action.
> + */
> +void ff_ccfifo_freep(AVCCFifo **ccf);
> +
> +
> +/**
> + * Extract CC data from an AVFrame
> + *
> + * Extract CC bytes from the AVFrame, insert them into our queue, and
> + * remove the side data from the AVFrame.  The side data is removed
> + * as it will be re-inserted at the appropriate rate later in the
> + * filter.
> + *
> + * @param af          AVCCFifo to write to
> + * @param frame       AVFrame with the video frame to operate on
> + * @return            Zero on success, or negative AVERROR
> + *                    code on failure.
> + */
> +int ff_ccfifo_extract(AVCCFifo *ccf, AVFrame *frame);
> +
> +/**
> + *Just like ff_ccfifo_extract(), but takes the raw bytes instead of an
> AVFrame
> + */
> +int ff_ccfifo_extractbytes(AVCCFifo *ccf, uint8_t *data, size_t len);
> +
> +/**
> + * Provide the size in bytes of an output buffer to allocate
> + *
> + * Ask for how many bytes the output will contain, so the caller can
> allocate
> + * an appropriately sized buffer and pass it to ff_ccfifo_injectbytes()
> + *
> + */
> +int ff_ccfifo_getoutputsize(AVCCFifo *ccf);
> +
> +/**
> + * Insert CC data from the FIFO into an AVFrame (as side data)
> + *
> + * Dequeue the appropriate number of CC tuples based on the
> + * frame rate, and insert them into the AVFrame
> + *
> + * @param af          AVCCFifo to read from
> + * @param frame       AVFrame with the video frame to operate on
> + * @return            Zero on success, or negative AVERROR
> + *                    code on failure.
> + */
> +int ff_ccfifo_inject(AVCCFifo *ccf, AVFrame *frame);
> +
> +/**
> + * Just like ff_ccfifo_inject(), but takes the raw bytes to insert the CC
> data
> + * int rather than an AVFrame
> + */
> +int ff_ccfifo_injectbytes(AVCCFifo *ccf, uint8_t *data, size_t len);
> +
> +/**
> + * Returns 1 if captions have been found as a prior call
> + * to ff_ccfifo_extract() or ff_ccfifo_extractbytes()
> + */
> +int ff_ccfifo_ccdetected(AVCCFifo *ccf);
> +
> +#endif /* AVFILTER_CCFIFO_H */
> --
> 1.8.3.1
>
> _______________________________________________
> ffmpeg-devel mailing list
> ffmpeg-devel@ffmpeg.org
> https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
>
> To unsubscribe, visit link above, or email
> ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
>
_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".