[FFmpeg-devel] [PATCH] avcodec/av1dec: parse and export Metadata OBUs

[James Almer <jamrial@gmail.com>]

This includes Mastering Display, Content light level, and some ITU-T T35
metadata like closed captions and HDR10+.

Signed-off-by: James Almer <jamrial@gmail.com>
---
 libavcodec/av1dec.c | 163 ++++++++++++++++++++++++++++++++++++++++++++
 libavcodec/av1dec.h |   8 +++
 2 files changed, 171 insertions(+)

diff --git a/libavcodec/av1dec.c b/libavcodec/av1dec.c
index d83c902f1f..dda9a22cf2 100644
--- a/libavcodec/av1dec.c
+++ b/libavcodec/av1dec.c
@@ -21,13 +21,16 @@
 #include "config_components.h"
 
 #include "libavutil/film_grain_params.h"
+#include "libavutil/mastering_display_metadata.h"
 #include "libavutil/pixdesc.h"
 #include "libavutil/opt.h"
 #include "avcodec.h"
 #include "av1dec.h"
+#include "atsc_a53.h"
 #include "bytestream.h"
 #include "codec_internal.h"
 #include "decode.h"
+#include "dynamic_hdr10_plus.h"
 #include "hwconfig.h"
 #include "profiles.h"
 #include "thread.h"
@@ -645,6 +648,7 @@ fail:
 static av_cold int av1_decode_free(AVCodecContext *avctx)
 {
     AV1DecContext *s = avctx->priv_data;
+    AV1RawMetadataITUTT35 itut_t35;
 
     for (int i = 0; i < FF_ARRAY_ELEMS(s->ref); i++) {
         av1_frame_unref(avctx, &s->ref[i]);
@@ -655,8 +659,14 @@ static av_cold int av1_decode_free(AVCodecContext *avctx)
 
     av_buffer_unref(&s->seq_ref);
     av_buffer_unref(&s->header_ref);
+    av_buffer_unref(&s->cll_ref);
+    av_buffer_unref(&s->mdcv_ref);
     av_freep(&s->tile_group_info);
 
+    while (s->itut_t35_fifo && av_fifo_read(s->itut_t35_fifo, &itut_t35, 1) >= 0)
+        av_buffer_unref(&itut_t35.payload_ref);
+    av_fifo_freep2(&s->itut_t35_fifo);
+
     ff_cbs_fragment_free(&s->current_obu);
     ff_cbs_close(&s->cbc);
 
@@ -771,6 +781,11 @@ static av_cold int av1_decode_init(AVCodecContext *avctx)
     if (ret < 0)
         return ret;
 
+    s->itut_t35_fifo = av_fifo_alloc2(1, sizeof(AV1RawMetadataITUTT35),
+                                      AV_FIFO_FLAG_AUTO_GROW);
+    if (!s->itut_t35_fifo)
+        return AVERROR(ENOMEM);
+
     av_opt_set_int(s->cbc->priv_data, "operating_point", s->operating_point, 0);
 
     if (avctx->extradata && avctx->extradata_size) {
@@ -852,6 +867,106 @@ fail:
     return ret;
 }
 
+static int export_itut_t35(AVCodecContext *avctx, AVFrame *frame,
+                           AV1RawMetadataITUTT35 *itut_t35)
+{
+    GetByteContext gb;
+    int ret, provider_code;
+
+    bytestream2_init(&gb, itut_t35->payload, itut_t35->payload_size);
+
+    provider_code = bytestream2_get_be16(&gb);
+    switch (provider_code) {
+    case 0x31: { // atsc_provider_code
+        uint32_t user_identifier = bytestream2_get_be32(&gb);
+        switch (user_identifier) {
+        case MKBETAG('G', 'A', '9', '4'): { // closed captions
+            AVBufferRef *buf = NULL;
+
+            ret = ff_parse_a53_cc(&buf, gb.buffer, bytestream2_get_bytes_left(&gb));
+            if (ret < 0)
+                return ret;
+            if (!ret)
+                break;
+
+            if (!av_frame_new_side_data_from_buf(frame, AV_FRAME_DATA_A53_CC, buf))
+                av_buffer_unref(&buf);
+
+            avctx->properties |= FF_CODEC_PROPERTY_CLOSED_CAPTIONS;
+            break;
+        }
+        default: // ignore unsupported identifiers
+            break;
+        }
+    }
+    case 0x3C: { // smpte_provider_code
+        AVDynamicHDRPlus *hdrplus;
+        int provider_oriented_code = bytestream2_get_be16(&gb);
+        int application_identifier = bytestream2_get_byte(&gb);
+
+        if (provider_oriented_code != 1 || application_identifier != 4)
+            break;
+
+        hdrplus = av_dynamic_hdr_plus_create_side_data(frame);
+        if (!hdrplus)
+            return AVERROR(ENOMEM);
+
+        ret = ff_parse_itu_t_t35_to_dynamic_hdr10_plus(hdrplus, gb.buffer,
+                                                       bytestream2_get_bytes_left(&gb));
+        if (ret < 0)
+            return ret;
+        break;
+    }
+    default: // ignore unsupported provider codes
+        break;
+    }
+
+    return 0;
+}
+
+static int export_metadata(AVCodecContext *avctx, AVFrame *frame)
+{
+    AV1DecContext *s = avctx->priv_data;
+    AV1RawMetadataITUTT35 itut_t35;
+    int ret = 0;
+
+    if (s->mdcv) {
+        AVMasteringDisplayMetadata *mastering = av_mastering_display_metadata_create_side_data(frame);
+        if (!mastering)
+            return AVERROR(ENOMEM);
+
+        for (int i = 0; i < 3; i++) {
+            mastering->display_primaries[i][0] = av_make_q(s->mdcv->primary_chromaticity_x[i], 1 << 16);
+            mastering->display_primaries[i][1] = av_make_q(s->mdcv->primary_chromaticity_y[i], 1 << 16);
+        }
+        mastering->white_point[0] = av_make_q(s->mdcv->white_point_chromaticity_x, 1 << 16);
+        mastering->white_point[1] = av_make_q(s->mdcv->white_point_chromaticity_y, 1 << 16);
+
+        mastering->max_luminance = av_make_q(s->mdcv->luminance_max, 1 << 8);
+        mastering->min_luminance = av_make_q(s->mdcv->luminance_min, 1 << 14);
+
+        mastering->has_primaries = 1;
+        mastering->has_luminance = 1;
+    }
+
+    if (s->cll) {
+        AVContentLightMetadata *light = av_content_light_metadata_create_side_data(frame);
+        if (!light)
+            return AVERROR(ENOMEM);
+
+        light->MaxCLL = s->cll->max_cll;
+        light->MaxFALL = s->cll->max_fall;
+    }
+
+    while (av_fifo_read(s->itut_t35_fifo, &itut_t35, 1) >= 0) {
+        if (ret >= 0)
+            ret = export_itut_t35(avctx, frame, &itut_t35);
+        av_buffer_unref(&itut_t35.payload_ref);
+    }
+
+    return ret;
+}
+
 static int export_film_grain(AVCodecContext *avctx, AVFrame *frame)
 {
     AV1DecContext *s = avctx->priv_data;
@@ -928,6 +1043,10 @@ static int set_output_frame(AVCodecContext *avctx, AVFrame *frame,
     if (ret < 0)
         return ret;
 
+    ret = export_metadata(avctx, frame);
+    if (ret < 0)
+        return ret;
+
     if (avctx->export_side_data & AV_CODEC_EXPORT_DATA_FILM_GRAIN) {
         ret = export_film_grain(avctx, frame);
         if (ret < 0) {
@@ -1174,6 +1293,45 @@ static int av1_decode_frame(AVCodecContext *avctx, AVFrame *frame,
         case AV1_OBU_TEMPORAL_DELIMITER:
         case AV1_OBU_PADDING:
         case AV1_OBU_METADATA:
+            switch (obu->obu.metadata.metadata_type) {
+            case AV1_METADATA_TYPE_HDR_CLL:
+                av_buffer_unref(&s->cll_ref);
+                s->cll_ref = av_buffer_ref(unit->content_ref);
+                if (!s->cll_ref) {
+                    s->cll = NULL;
+                    ret = AVERROR(ENOMEM);
+                    goto end;
+                }
+                s->cll = &obu->obu.metadata.metadata.hdr_cll;
+                break;
+            case AV1_METADATA_TYPE_HDR_MDCV:
+                av_buffer_unref(&s->mdcv_ref);
+                s->mdcv_ref = av_buffer_ref(unit->content_ref);
+                if (!s->mdcv_ref) {
+                    s->mdcv = NULL;
+                    ret = AVERROR(ENOMEM);
+                    goto end;
+                }
+                s->mdcv = &obu->obu.metadata.metadata.hdr_mdcv;
+                break;
+            case AV1_METADATA_TYPE_ITUT_T35: {
+                AV1RawMetadataITUTT35 itut_t35;
+                memcpy(&itut_t35, &obu->obu.metadata.metadata.itut_t35, sizeof(itut_t35));
+                itut_t35.payload_ref = av_buffer_ref(obu->obu.metadata.metadata.itut_t35.payload_ref);
+                if (!itut_t35.payload_ref) {
+                    ret = AVERROR(ENOMEM);
+                    goto end;
+                }
+                ret = av_fifo_write(s->itut_t35_fifo, &itut_t35, 1);
+                if (ret < 0) {
+                    av_buffer_unref(&itut_t35.payload_ref);
+                    goto end;
+                }
+                break;
+            }
+            default:
+                break;
+            }
             break;
         default:
             av_log(avctx, AV_LOG_DEBUG,
@@ -1218,6 +1376,7 @@ end:
 static void av1_decode_flush(AVCodecContext *avctx)
 {
     AV1DecContext *s = avctx->priv_data;
+    AV1RawMetadataITUTT35 itut_t35;
 
     for (int i = 0; i < FF_ARRAY_ELEMS(s->ref); i++)
         av1_frame_unref(avctx, &s->ref[i]);
@@ -1226,6 +1385,10 @@ static void av1_decode_flush(AVCodecContext *avctx)
     s->operating_point_idc = 0;
     s->raw_frame_header = NULL;
     s->raw_seq = NULL;
+    s->cll = NULL;
+    s->mdcv = NULL;
+    while (s->itut_t35_fifo && av_fifo_read(s->itut_t35_fifo, &itut_t35, 1) >= 0)
+        av_buffer_unref(&itut_t35.payload_ref);
 
     ff_cbs_flush(s->cbc);
 }
diff --git a/libavcodec/av1dec.h b/libavcodec/av1dec.h
index 82c7084e99..cef899f81f 100644
--- a/libavcodec/av1dec.h
+++ b/libavcodec/av1dec.h
@@ -23,6 +23,7 @@
 
 #include <stdint.h>
 
+#include "libavutil/fifo.h"
 #include "libavutil/buffer.h"
 #include "libavutil/frame.h"
 #include "libavutil/pixfmt.h"
@@ -73,6 +74,13 @@ typedef struct AV1DecContext {
     AVBufferRef *header_ref;
     AV1RawFrameHeader *raw_frame_header;
     TileGroupInfo *tile_group_info;
+
+    AVBufferRef *cll_ref;
+    AV1RawMetadataHDRCLL *cll;
+    AVBufferRef *mdcv_ref;
+    AV1RawMetadataHDRMDCV *mdcv;
+    AVFifo *itut_t35_fifo;
+
     uint16_t tile_num;
     uint16_t tg_start;
     uint16_t tg_end;
-- 
2.39.2

_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".