[FFmpeg-devel] [PATCH] avcodec/hevc_metadata_bsf: add HDR SEI message support

[Miklos Juhasz via ffmpeg-devel <ffmpeg-devel@ffmpeg.org>]

Add options to insert HDR metadata into HEVC streams.

The following options are added:
- master_display: For Mastering Display Colour Volume SEI.
- max_cll: For Content Light Level Information SEI.

The filter is idempotent and will replace any existing SEI messages
of the same type. It fails when the option values are invalid.

Static metadata like Mastering Display Colour Volume and Content Light
Level should only be present at random access points. This change
ensures that the filter first removes any existing HDR SEI messages
from all access units, and then inserts the new messages only on
access units containing IRAP frames or parameter sets.

This prevents conflicting SEI messages in the stream and adheres
more closely to the standard's intention for this type of metadata.

When removing HDR SEI messages that are the only messages in a SEI NAL unit,
empty NAL units would be left behind causing bitstream corruption.
Precise cleanup is added that only removes SEI NAL units that become empty
specifically due to HDR message removal, preventing the corruption while
avoiding unintended side effects.

- Track which SEI units contain HDR messages before deletion
- Remove only those units that become empty due to our specific changes
- Handle both PREFIX and SUFFIX SEI units to mirror CBS deletion behavior
- Add proper error handling for memory allocation failures

Signed-off-by: Miklos Juhasz <juhasz.miklos.1@gmail.com>
---
libavcodec/bsf/h265_metadata.c | 174 +++++++++++++++++++++++++++++++--
1 file changed, 168 insertions(+), 6 deletions(-)

diff --git a/libavcodec/bsf/h265_metadata.c b/libavcodec/bsf/h265_metadata.c
index 4b0601beee..074dfd0a74 100644
--- a/libavcodec/bsf/h265_metadata.c
+++ b/libavcodec/bsf/h265_metadata.c
@@ -17,6 +17,7 @@
 */

#include "libavutil/common.h"
+#include "libavutil/mem.h"
#include "libavutil/opt.h"

#include "bsf.h"
@@ -26,6 +27,7 @@
#include "cbs_h265.h"
#include "h2645data.h"
#include "h265_profile_level.h"
+#include "cbs_sei.h"

#include "hevc/hevc.h"

@@ -65,6 +67,14 @@ typedef struct H265MetadataContext {
    int level;
    int level_guess;
    int level_warned;
+
+    // HDR metadata fields
+    char *master_display;
+    char *max_cll;
+    SEIRawMasteringDisplayColourVolume sei_mastering_display;
+    int has_master_display;
+    SEIRawContentLightLevelInfo sei_content_light_level;
+    int has_content_light_level;
} H265MetadataContext;


@@ -412,11 +422,33 @@ static int h265_metadata_update_sps(AVBSFContext *bsf,
    return 0;
}

+static int parse_master_display_option(const char *str, SEIRawMasteringDisplayColourVolume *mdcv) {
+    // Format: G(x,y)B(x,y)R(x,y)WP(x,y)L(max,min)
+    // Example: G(13250,34500)B(7500,3000)R(34000,16000)WP(15635,16450)L(10000000,1)
+    int ret = sscanf(str,
+        "G(%hu,%hu)B(%hu,%hu)R(%hu,%hu)WP(%hu,%hu)L(%u,%u)",
+        &mdcv->display_primaries_x[1], &mdcv->display_primaries_y[1],
+        &mdcv->display_primaries_x[2], &mdcv->display_primaries_y[2],
+        &mdcv->display_primaries_x[0], &mdcv->display_primaries_y[0],
+        &mdcv->white_point_x, &mdcv->white_point_y,
+        &mdcv->max_display_mastering_luminance, &mdcv->min_display_mastering_luminance);
+    return ret == 10 ? 0 : AVERROR(EINVAL);
+}
+static int parse_max_cll_option(const char *str, SEIRawContentLightLevelInfo *clli) {
+    // Format: MaxCLL,MaxFALL
+    // Example: 1000,400
+    int ret = sscanf(str, "%hu,%hu", &clli->max_content_light_level, &clli->max_pic_average_light_level);
+    return ret == 2 ? 0 : AVERROR(EINVAL);
+}
+
+static int h265_metadata_remove_existing_hdr_sei(AVBSFContext *bsf, CodedBitstreamFragment *au);
+
static int h265_metadata_update_fragment(AVBSFContext *bsf, AVPacket *pkt,
                                         CodedBitstreamFragment *au)
{
    H265MetadataContext *ctx = bsf->priv_data;
    int err, i;
+    int is_random_access_point = 0;

    // If an AUD is present, it must be the first NAL unit.
    if (au->nb_units && au->units[0].type == HEVC_NAL_AUD) {
@@ -465,21 +497,119 @@ static int h265_metadata_update_fragment(AVBSFContext *bsf, AVPacket *pkt,
        h265_metadata_guess_level(bsf, au);

    for (i = 0; i < au->nb_units; i++) {
-        if (au->units[i].type == HEVC_NAL_VPS) {
-            err = h265_metadata_update_vps(bsf, au->units[i].content);
+        const CodedBitstreamUnit *unit = &au->units[i];
+        if (unit->type == HEVC_NAL_VPS) {
+            err = h265_metadata_update_vps(bsf, unit->content);
            if (err < 0)
                return err;
-        }
-        if (au->units[i].type == HEVC_NAL_SPS) {
-            err = h265_metadata_update_sps(bsf, au->units[i].content);
+            is_random_access_point = 1;
+        } else if (unit->type == HEVC_NAL_SPS) {
+            err = h265_metadata_update_sps(bsf, unit->content);
            if (err < 0)
                return err;
+            is_random_access_point = 1;
+        } else if (unit->type >= HEVC_NAL_BLA_W_LP && unit->type <= HEVC_NAL_RSV_IRAP_VCL23) {
+            is_random_access_point = 1;
+        }
+    }
+
+    // If we want to set HDR metadata, we must first remove any
+    // pre-existing HDR metadata from ALL access units to avoid conflicts.
+    if (ctx->has_master_display || ctx->has_content_light_level) {
+        err = h265_metadata_remove_existing_hdr_sei(bsf, au);
+        if (err < 0)
+            return err;
+    }
+
+    // Now, if this specific AU is an IRAP or has parameter sets,
+    // insert the new SEI messages.
+    if (is_random_access_point) {
+        if (ctx->has_master_display) {
+            err = ff_cbs_sei_add_message(ctx->common.output, au, 1,
+                SEI_TYPE_MASTERING_DISPLAY_COLOUR_VOLUME,
+                &ctx->sei_mastering_display, NULL);
+            if (err < 0) {
+                av_log(bsf, AV_LOG_ERROR, "Failed to insert Mastering Display SEI.\n");
+                return err;
+            }
+        }
+        if (ctx->has_content_light_level) {
+            err = ff_cbs_sei_add_message(ctx->common.output, au, 1,
+                SEI_TYPE_CONTENT_LIGHT_LEVEL_INFO,
+                &ctx->sei_content_light_level, NULL);
+            if (err < 0) {
+                av_log(bsf, AV_LOG_ERROR, "Failed to insert Content Light Level SEI.\n");
+                return err;
+            }
        }
    }

    return 0;
}

+static int h265_metadata_remove_existing_hdr_sei(AVBSFContext *bsf, CodedBitstreamFragment *au)
+{
+    H265MetadataContext *ctx = bsf->priv_data;
+    int i, j;
+    int *had_hdr;
+    
+    had_hdr = av_calloc(au->nb_units, sizeof(int));
+    if (!had_hdr)
+        return AVERROR(ENOMEM);
+    
+    // Pre-scan: identify PREFIX and SUFFIX SEI units with HDR messages we plan to remove
+    for (i = 0; i < au->nb_units; i++) {
+        CodedBitstreamUnit *unit = &au->units[i];
+        H265RawSEI *sei;
+        
+        if (unit->type != HEVC_NAL_SEI_PREFIX && unit->type != HEVC_NAL_SEI_SUFFIX)
+            continue;
+            
+        sei = unit->content;
+        if (!sei)
+            continue;
+            
+        // Check if this unit contains HDR messages we plan to remove
+        for (j = 0; j < sei->message_list.nb_messages; j++) {
+            uint32_t payload_type = sei->message_list.messages[j].payload_type;
+            if ((ctx->has_master_display && payload_type == SEI_TYPE_MASTERING_DISPLAY_COLOUR_VOLUME) ||
+                (ctx->has_content_light_level && payload_type == SEI_TYPE_CONTENT_LIGHT_LEVEL_INFO)) {
+                had_hdr[i] = 1;
+                break;
+            }
+        }
+    }
+    
+    // Remove HDR SEI messages
+    if (ctx->has_master_display) {
+        ff_cbs_sei_delete_message_type(ctx->common.output, au,
+            SEI_TYPE_MASTERING_DISPLAY_COLOUR_VOLUME);
+    }
+    if (ctx->has_content_light_level) {
+        ff_cbs_sei_delete_message_type(ctx->common.output, au,
+            SEI_TYPE_CONTENT_LIGHT_LEVEL_INFO);
+    }
+    
+    // Clean up only PREFIX and SUFFIX SEI units we know contained HDR and are now empty.
+    // This prevents leaving behind empty SEI NAL units which would cause
+    // bitstream corruption, while ensuring we only remove units that became
+    // empty due to our specific HDR message removal.
+    // Process in reverse order to maintain correct indices during deletion.
+    for (i = au->nb_units - 1; i >= 0; i--) {
+        if (had_hdr[i]) {
+            CodedBitstreamUnit *unit = &au->units[i];
+            H265RawSEI *sei = unit->content;
+            
+            if (sei && sei->message_list.nb_messages == 0) {
+                ff_cbs_delete_unit(au, i);
+            }
+        }
+    }
+    
+    av_free(had_hdr);
+    return 0;
+}
+
static const CBSBSFType h265_metadata_type = {
    .codec_id        = AV_CODEC_ID_HEVC,
    .fragment_name   = "access unit",
@@ -489,7 +619,34 @@ static const CBSBSFType h265_metadata_type = {

static int h265_metadata_init(AVBSFContext *bsf)
{
-    return ff_cbs_bsf_generic_init(bsf, &h265_metadata_type);
+    H265MetadataContext *ctx = bsf->priv_data;
+    int ret;
+
+    ret = ff_cbs_bsf_generic_init(bsf, &h265_metadata_type);
+    if (ret < 0)
+        return ret;
+
+    // Parse HDR options
+    ctx->has_master_display = 0;
+    ctx->has_content_light_level = 0;
+    if (ctx->master_display) {
+        if (parse_master_display_option(ctx->master_display, &ctx->sei_mastering_display) == 0) {
+            ctx->has_master_display = 1;
+        } else {
+            av_log(bsf, AV_LOG_ERROR, "Invalid master_display option format.\n");
+            return AVERROR(EINVAL);
+        }
+    }
+    if (ctx->max_cll) {
+        if (parse_max_cll_option(ctx->max_cll, &ctx->sei_content_light_level) == 0) {
+            ctx->has_content_light_level = 1;
+        } else {
+            av_log(bsf, AV_LOG_ERROR, "Invalid max_cll option format.\n");
+            return AVERROR(EINVAL);
+        }
+    }
+
+    return ret;
}

#define OFFSET(x) offsetof(H265MetadataContext, x)
@@ -574,6 +731,11 @@ static const AVOption h265_metadata_options[] = {
    { LEVEL("8.5", 255) },
#undef LEVEL

+    { "master_display", "Set mastering display colour volume SEI (G(x,y)B(x,y)R(x,y)WP(x,y)L(max,min))",
+      OFFSET(master_display), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, FLAGS },
+    { "max_cll", "Set content light level info SEI (MaxCLL,MaxFALL)",
+      OFFSET(max_cll), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, FLAGS },
+
    { NULL }
};

-- 
2.39.5 (Apple Git-154)

_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".