[FFmpeg-devel] [RFC][PATCH 3/3] decklink: Add support for output of HDR metadata

[Devin Heitmueller <devin.heitmueller@ltnglobal.com>]

Add HDR support to the decklink output for cards that support such
functionality.  This includes setting the EOTF, the colorspace,
the mastering info, and the content light level info.  Both the
Payload Identification HANC data as well as the SMPTE ST 2108-1
VANC data are being set.

Tested with in-house content as well as samples from 4kmedia.org.
Testing was done with the Decklink 8K Pro and the Duo2 with 12.5.1
firmware, as well as with the Duo2 with 10.11.2 (before it supported
HDR) to ensure there are no regressions.

Signed-off-by: Devin Heitmueller <dheitmueller@ltnglobal.com>
---
 libavdevice/decklink_common.cpp |  12 +++
 libavdevice/decklink_common.h   |   2 +
 libavdevice/decklink_enc.cpp    | 204 ++++++++++++++++++++++++++++++++++++++--
 3 files changed, 211 insertions(+), 7 deletions(-)

diff --git a/libavdevice/decklink_common.cpp b/libavdevice/decklink_common.cpp
index 47de7ef..c1bcb82 100644
--- a/libavdevice/decklink_common.cpp
+++ b/libavdevice/decklink_common.cpp
@@ -251,6 +251,18 @@ int ff_decklink_set_configs(AVFormatContext *avctx,
         }
     }
 
+    DECKLINK_BOOL hdr_supported;
+    if (ctx->attr->GetFlag(BMDDeckLinkSupportsHDRMetadata, &hdr_supported) == S_OK) {
+        if (hdr_supported)
+            ctx->supports_hdr = 1;
+    }
+
+    DECKLINK_BOOL colorspace_supported;
+    if (ctx->attr->GetFlag(BMDDeckLinkSupportsColorspaceMetadata, &colorspace_supported) == S_OK) {
+        if (colorspace_supported)
+            ctx->supports_colorspace = 1;
+    }
+
     return 0;
 }
 
diff --git a/libavdevice/decklink_common.h b/libavdevice/decklink_common.h
index 34ab1b9..d50007f 100644
--- a/libavdevice/decklink_common.h
+++ b/libavdevice/decklink_common.h
@@ -109,6 +109,8 @@ struct decklink_ctx {
     int bmd_height;
     int bmd_field_dominance;
     int supports_vanc;
+    int supports_hdr;
+    int supports_colorspace;
 
     /* Capture buffer queue */
     DecklinkPacketQueue queue;
diff --git a/libavdevice/decklink_enc.cpp b/libavdevice/decklink_enc.cpp
index ffd0ad9..92d6bbe 100644
--- a/libavdevice/decklink_enc.cpp
+++ b/libavdevice/decklink_enc.cpp
@@ -35,6 +35,7 @@ extern "C" {
 #include "libavcodec/bytestream.h"
 #include "libavutil/internal.h"
 #include "libavutil/imgutils.h"
+#include "libavutil/mastering_display_metadata.h"
 #include "avdevice.h"
 }
 
@@ -47,13 +48,13 @@ extern "C" {
 #endif
 
 /* DeckLink callback class declaration */
-class decklink_frame : public IDeckLinkVideoFrame
+class decklink_frame : public IDeckLinkVideoFrame, public IDeckLinkVideoFrameMetadataExtensions
 {
 public:
     decklink_frame(struct decklink_ctx *ctx, AVFrame *avframe, AVCodecID codec_id, int height, int width) :
         _ctx(ctx), _avframe(avframe), _avpacket(NULL), _codec_id(codec_id), _ancillary(NULL), _height(height), _width(width),  _refs(1) { }
     decklink_frame(struct decklink_ctx *ctx, AVPacket *avpacket, AVCodecID codec_id, int height, int width) :
-        _ctx(ctx), _avframe(NULL), _avpacket(avpacket), _codec_id(codec_id), _ancillary(NULL), _height(height), _width(width), _refs(1) { }
+        _ctx(ctx), _avframe(NULL), _avpacket(avpacket), _codec_id(codec_id), _ancillary(NULL), _height(height), _width(width), _colorspace(AVCOL_SPC_BT709), _eotf(AVCOL_TRC_BT709), hdr(NULL), lighting(NULL), _refs(1) { }
     virtual long           STDMETHODCALLTYPE GetWidth      (void)          { return _width; }
     virtual long           STDMETHODCALLTYPE GetHeight     (void)          { return _height; }
     virtual long           STDMETHODCALLTYPE GetRowBytes   (void)
@@ -72,10 +73,14 @@ public:
     }
     virtual BMDFrameFlags  STDMETHODCALLTYPE GetFlags      (void)
     {
-       if (_codec_id == AV_CODEC_ID_WRAPPED_AVFRAME)
-           return _avframe->linesize[0] < 0 ? bmdFrameFlagFlipVertical : bmdFrameFlagDefault;
-       else
-           return bmdFrameFlagDefault;
+        if (_codec_id == AV_CODEC_ID_WRAPPED_AVFRAME) {
+            return _avframe->linesize[0] < 0 ? bmdFrameFlagFlipVertical : bmdFrameFlagDefault;
+        } else {
+            if (_ctx->supports_hdr && (hdr || lighting))
+                return bmdFrameFlagDefault | bmdFrameContainsHDRMetadata;
+            else
+                return bmdFrameFlagDefault;
+        }
     }
 
     virtual HRESULT        STDMETHODCALLTYPE GetBytes      (void **buffer)
@@ -110,7 +115,176 @@ public:
         _ancillary->AddRef();
         return S_OK;
     }
-    virtual HRESULT STDMETHODCALLTYPE QueryInterface(REFIID iid, LPVOID *ppv) { return E_NOINTERFACE; }
+
+    virtual HRESULT STDMETHODCALLTYPE SetMetadata(enum AVColorSpace colorspace, enum AVColorTransferCharacteristic eotf)
+    {
+        _colorspace = colorspace;
+        _eotf = eotf;
+        return S_OK;
+    }
+
+    // IDeckLinkVideoFrameMetadataExtensions interface
+    virtual HRESULT GetInt(BMDDeckLinkFrameMetadataID metadataID, int64_t* value)
+    {
+        HRESULT result = S_OK;
+
+        switch (metadataID) {
+        case bmdDeckLinkFrameMetadataHDRElectroOpticalTransferFunc:
+            /* See CTA-861-G Sec 6.9 Dynamic Range and Mastering */
+
+            switch(_eotf) {
+            case AVCOL_TRC_SMPTEST2084:
+                /* PQ */
+                *value = 2;
+               break;
+            case AVCOL_TRC_ARIB_STD_B67:
+                /* Also known as "HLG" */
+                *value = 3;
+                break;
+            case AVCOL_TRC_SMPTE170M:
+            case AVCOL_TRC_SMPTE240M:
+            case AVCOL_TRC_BT709:
+            default:
+                /* SDR */
+                *value = 0;
+               break;
+            }
+            break;
+
+        case bmdDeckLinkFrameMetadataColorspace:
+            if (!_ctx->supports_colorspace) {
+                result = E_NOTIMPL;
+                break;
+            }
+            switch(_colorspace) {
+            case AVCOL_SPC_BT470BG:
+            case AVCOL_SPC_SMPTE170M:
+            case AVCOL_SPC_SMPTE240M:
+                *value = bmdColorspaceRec601;
+                break;
+            case AVCOL_SPC_BT2020_CL:
+            case AVCOL_SPC_BT2020_NCL:
+                *value = bmdColorspaceRec2020;
+                break;
+            case AVCOL_SPC_BT709:
+            default:
+                *value = bmdColorspaceRec709;
+                break;
+            }
+            break;
+        default:
+            result = E_INVALIDARG;
+        }
+
+        return result;
+    }
+    virtual HRESULT GetFloat(BMDDeckLinkFrameMetadataID metadataID, double* value)
+    {
+        *value = 0;
+
+        switch (metadataID) {
+        case bmdDeckLinkFrameMetadataHDRDisplayPrimariesRedX:
+            if (hdr && hdr->has_primaries)
+                *value = av_q2d(hdr->display_primaries[0][0]);
+            break;
+        case bmdDeckLinkFrameMetadataHDRDisplayPrimariesRedY:
+            if (hdr && hdr->has_primaries)
+                *value = av_q2d(hdr->display_primaries[0][1]);
+            break;
+        case bmdDeckLinkFrameMetadataHDRDisplayPrimariesGreenX:
+            if (hdr && hdr->has_primaries)
+                *value = av_q2d(hdr->display_primaries[1][0]);
+            break;
+        case bmdDeckLinkFrameMetadataHDRDisplayPrimariesGreenY:
+            if (hdr && hdr->has_primaries)
+                *value = av_q2d(hdr->display_primaries[1][1]);
+            break;
+        case bmdDeckLinkFrameMetadataHDRDisplayPrimariesBlueX:
+            if (hdr && hdr->has_primaries)
+                *value = av_q2d(hdr->display_primaries[2][0]);
+            break;
+        case bmdDeckLinkFrameMetadataHDRDisplayPrimariesBlueY:
+            if (hdr && hdr->has_primaries)
+                *value = av_q2d(hdr->display_primaries[2][1]);
+            break;
+        case bmdDeckLinkFrameMetadataHDRWhitePointX:
+            if (hdr && hdr->has_primaries)
+                *value = av_q2d(hdr->white_point[0]);
+            break;
+        case bmdDeckLinkFrameMetadataHDRWhitePointY:
+            if (hdr && hdr->has_primaries)
+                *value = av_q2d(hdr->white_point[1]);
+            break;
+        case bmdDeckLinkFrameMetadataHDRMaxDisplayMasteringLuminance:
+            if (hdr && hdr->has_luminance)
+                *value = av_q2d(hdr->max_luminance);
+            break;
+        case bmdDeckLinkFrameMetadataHDRMinDisplayMasteringLuminance:
+            if (hdr && hdr->has_luminance)
+                *value = av_q2d(hdr->min_luminance);
+            break;
+        case bmdDeckLinkFrameMetadataHDRMaximumContentLightLevel:
+            if (lighting)
+                *value = (float) lighting->MaxCLL;
+            else
+                *value = 0;
+            break;
+        case bmdDeckLinkFrameMetadataHDRMaximumFrameAverageLightLevel:
+            if (lighting)
+                *value = (float) lighting->MaxFALL;
+            else
+                *value = 0;
+            break;
+        default:
+            return E_INVALIDARG;
+        }
+
+        return S_OK;
+    }
+
+    virtual HRESULT GetFlag(BMDDeckLinkFrameMetadataID metadataID, bool* value)
+    {
+        *value = false;
+        return E_INVALIDARG;
+    }
+    virtual HRESULT GetString(BMDDeckLinkFrameMetadataID metadataID, const char** value)
+    {
+        *value = nullptr;
+        return E_INVALIDARG;
+    }
+    virtual HRESULT GetBytes(BMDDeckLinkFrameMetadataID metadataID, void* buffer, uint32_t* bufferSize)
+    {
+        *bufferSize = 0;
+        return E_INVALIDARG;
+    }
+
+    virtual HRESULT STDMETHODCALLTYPE QueryInterface(REFIID iid, LPVOID *ppv)
+    {
+        CFUUIDBytes             iunknown;
+        HRESULT                 result          = S_OK;
+
+        if (!ppv)
+            return E_INVALIDARG;
+
+        *ppv = NULL;
+
+        iunknown = CFUUIDGetUUIDBytes(IUnknownUUID);
+        if (memcmp(&iid, &iunknown, sizeof(REFIID)) == 0) {
+            *ppv = this;
+            AddRef();
+        } else if (memcmp(&iid, &IID_IDeckLinkVideoFrame, sizeof(REFIID)) == 0) {
+            *ppv = static_cast<IDeckLinkVideoFrame*>(this);
+            AddRef();
+        } else if (memcmp(&iid, &IID_IDeckLinkVideoFrameMetadataExtensions, sizeof(REFIID)) == 0) {
+            *ppv = static_cast<IDeckLinkVideoFrameMetadataExtensions*>(this);
+            AddRef();
+        } else {
+            result = E_NOINTERFACE;
+        }
+
+        return result;
+    }
+
     virtual ULONG   STDMETHODCALLTYPE AddRef(void)                            { return ++_refs; }
     virtual ULONG   STDMETHODCALLTYPE Release(void)
     {
@@ -132,6 +306,10 @@ public:
     IDeckLinkVideoFrameAncillary *_ancillary;
     int _height;
     int _width;
+    enum AVColorSpace _colorspace;
+    enum AVColorTransferCharacteristic _eotf;
+    const AVMasteringDisplayMetadata *hdr;
+    const AVContentLightMetadata *lighting;
 
 private:
     std::atomic<int>  _refs;
@@ -726,6 +904,18 @@ static int decklink_write_video_packet(AVFormatContext *avctx, AVPacket *pkt)
         return AVERROR(EIO);
     }
 
+    /* Set frame metadata properties */
+    size_t size;
+    const AVMasteringDisplayMetadata *hdr = (const AVMasteringDisplayMetadata *) av_packet_get_side_data(pkt, AV_PKT_DATA_MASTERING_DISPLAY_METADATA, &size);
+    if (hdr && size > 0)
+        frame->hdr = hdr;
+
+    const AVContentLightMetadata *lighting = (const AVContentLightMetadata *) av_packet_get_side_data(pkt, AV_PKT_DATA_CONTENT_LIGHT_LEVEL, &size);
+    if (hdr && size > 0)
+        frame->lighting = lighting;
+
+    frame->SetMetadata(st->codecpar->color_space, st->codecpar->color_trc);
+
     /* Always keep at most one second of frames buffered. */
     pthread_mutex_lock(&ctx->mutex);
     while (ctx->frames_buffer_available_spots == 0) {
-- 
1.8.3.1

_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".