[FFmpeg-devel] [PATCH 1/2] lavc/qsvdec: update HDR side data on output AVFrame for AV1 decoding

[FFmpeg-devel] [PATCH 1/2] lavc/qsvdec: update HDR side data on output AVFrame for AV1 decoding

[Xiang, Haihao]

From: Haihao Xiang <haihao.xiang@intel.com>

The SDK may provide HDR metadata for HDR streams via mfxExtBuffer
attached on output mfxFrameSurface1

Signed-off-by: Haihao Xiang <haihao.xiang@intel.com>
---
 libavcodec/qsvdec.c | 48 ++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 47 insertions(+), 1 deletion(-)

diff --git a/libavcodec/qsvdec.c b/libavcodec/qsvdec.c
index df0d49bc10..7741baff06 100644
--- a/libavcodec/qsvdec.c
+++ b/libavcodec/qsvdec.c
@@ -538,7 +538,8 @@ static int alloc_frame(AVCodecContext *avctx, QSVContext *q, QSVFrame *frame)
 #endif
 
 #if QSV_VERSION_ATLEAST(1, 35)
-    if (QSV_RUNTIME_VERSION_ATLEAST(q->ver, 1, 35) && avctx->codec_id == AV_CODEC_ID_HEVC) {
+    if ((QSV_RUNTIME_VERSION_ATLEAST(q->ver, 1, 35) && avctx->codec_id == AV_CODEC_ID_HEVC) ||
+        (QSV_RUNTIME_VERSION_ATLEAST(q->ver, 2, 9) && avctx->codec_id == AV_CODEC_ID_AV1)) {
         frame->mdcv.Header.BufferId = MFX_EXTBUFF_MASTERING_DISPLAY_COLOUR_VOLUME;
         frame->mdcv.Header.BufferSz = sizeof(frame->mdcv);
         // The data in mdcv is valid when this flag is 1
@@ -742,6 +743,45 @@ static int qsv_export_hdr_side_data(AVCodecContext *avctx, mfxExtMasteringDispla
     return 0;
 }
 
+static int qsv_export_hdr_side_data_av1(AVCodecContext *avctx, mfxExtMasteringDisplayColourVolume *mdcv,
+                                        mfxExtContentLightLevelInfo *clli, AVFrame *frame)
+{
+    if (mdcv->InsertPayloadToggle) {
+        AVMasteringDisplayMetadata *mastering = av_mastering_display_metadata_create_side_data(frame);
+        const int chroma_den   = 1 << 16;
+        const int max_luma_den = 1 << 8;
+        const int min_luma_den = 1 << 14;
+
+        if (!mastering)
+            return AVERROR(ENOMEM);
+
+        for (int i = 0; i < 3; i++) {
+            mastering->display_primaries[i][0] = av_make_q(mdcv->DisplayPrimariesX[i], chroma_den);
+            mastering->display_primaries[i][1] = av_make_q(mdcv->DisplayPrimariesY[i], chroma_den);
+        }
+
+        mastering->white_point[0] = av_make_q(mdcv->WhitePointX, chroma_den);
+        mastering->white_point[1] = av_make_q(mdcv->WhitePointY, chroma_den);
+
+        mastering->max_luminance = av_make_q(mdcv->MaxDisplayMasteringLuminance, max_luma_den);
+        mastering->min_luminance = av_make_q(mdcv->MinDisplayMasteringLuminance, min_luma_den);
+
+        mastering->has_luminance = 1;
+        mastering->has_primaries = 1;
+    }
+
+    if (clli->InsertPayloadToggle) {
+        AVContentLightMetadata *light = av_content_light_metadata_create_side_data(frame);
+        if (!light)
+            return AVERROR(ENOMEM);
+
+        light->MaxCLL  = clli->MaxContentLightLevel;
+        light->MaxFALL = clli->MaxPicAverageLightLevel;
+    }
+
+    return 0;
+}
+
 #endif
 
 static int qsv_decode(AVCodecContext *avctx, QSVContext *q,
@@ -874,6 +914,12 @@ static int qsv_decode(AVCodecContext *avctx, QSVContext *q,
             if (ret < 0)
                 return ret;
         }
+
+        if (QSV_RUNTIME_VERSION_ATLEAST(q->ver, 2, 9) && avctx->codec_id == AV_CODEC_ID_AV1) {
+            ret = qsv_export_hdr_side_data_av1(avctx, &aframe.frame->mdcv, &aframe.frame->clli, frame);
+            if (ret < 0)
+                return ret;
+        }
 #endif
 
         frame->repeat_pict =
-- 
2.34.1

_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".

[FFmpeg-devel] [PATCH 2/2] lavc/qsvenc_av1: accept HDR metadata if have

[Xiang, Haihao]

From: Haihao Xiang <haihao.xiang@intel.com>

The sdk av1 encoder can accept HDR metadata via mfxEncodeCtrl::ExtParam.

Signed-off-by: Haihao Xiang <haihao.xiang@intel.com>
---
 libavcodec/qsvenc_av1.c | 73 +++++++++++++++++++++++++++++++++++++++++
 1 file changed, 73 insertions(+)

diff --git a/libavcodec/qsvenc_av1.c b/libavcodec/qsvenc_av1.c
index 33727bb07e..56002746b9 100644
--- a/libavcodec/qsvenc_av1.c
+++ b/libavcodec/qsvenc_av1.c
@@ -25,6 +25,8 @@
 #include <mfxvideo.h>
 
 #include "libavutil/common.h"
+#include "libavutil/mastering_display_metadata.h"
+#include "libavutil/mem.h"
 #include "libavutil/opt.h"
 
 #include "avcodec.h"
@@ -39,6 +41,75 @@ typedef struct QSVAV1EncContext {
     QSVEncContext qsv;
 } QSVAV1EncContext;
 
+static int qsv_av1_set_encode_ctrl(AVCodecContext *avctx,
+                                   const AVFrame *frame, mfxEncodeCtrl *enc_ctrl)
+{
+    QSVAV1EncContext *q = avctx->priv_data;
+    AVFrameSideData *sd;
+
+    if (!frame || !QSV_RUNTIME_VERSION_ATLEAST(q->qsv.ver, 2, 11))
+        return 0;
+
+    sd = av_frame_get_side_data(frame, AV_FRAME_DATA_MASTERING_DISPLAY_METADATA);
+    if (sd) {
+        AVMasteringDisplayMetadata *mdm = (AVMasteringDisplayMetadata *)sd->data;
+        if (mdm->has_primaries && mdm->has_luminance) {
+            const int chroma_den   = 1 << 16;
+            const int max_luma_den = 1 << 8;
+            const int min_luma_den = 1 << 14;
+            mfxExtMasteringDisplayColourVolume *mdcv = av_mallocz(sizeof(*mdcv));
+            if (!mdcv)
+                return AVERROR(ENOMEM);
+
+            mdcv->Header.BufferId = MFX_EXTBUFF_MASTERING_DISPLAY_COLOUR_VOLUME;
+            mdcv->Header.BufferSz = sizeof(*mdcv);
+
+            for (int i = 0; i < 3; i++) {
+                mdcv->DisplayPrimariesX[i] =
+                    av_rescale(mdm->display_primaries[i][0].num, chroma_den,
+                               mdm->display_primaries[i][0].den);
+                mdcv->DisplayPrimariesY[i] =
+                    av_rescale(mdm->display_primaries[i][1].num, chroma_den,
+                               mdm->display_primaries[i][1].den);
+            }
+
+            mdcv->WhitePointX =
+                av_rescale(mdm->white_point[0].num, chroma_den,
+                           mdm->white_point[0].den);
+            mdcv->WhitePointY =
+                av_rescale(mdm->white_point[1].num, chroma_den,
+                           mdm->white_point[1].den);
+
+            mdcv->MaxDisplayMasteringLuminance =
+                av_rescale(mdm->max_luminance.num, max_luma_den,
+                           mdm->max_luminance.den);
+            mdcv->MinDisplayMasteringLuminance =
+                av_rescale(mdm->min_luminance.num, min_luma_den,
+                           mdm->min_luminance.den);
+
+            enc_ctrl->ExtParam[enc_ctrl->NumExtParam++] = (mfxExtBuffer *)mdcv;
+        }
+    }
+
+    sd = av_frame_get_side_data(frame, AV_FRAME_DATA_CONTENT_LIGHT_LEVEL);
+    if (sd) {
+        AVContentLightMetadata *clm = (AVContentLightMetadata *)sd->data;
+        mfxExtContentLightLevelInfo *clli = av_mallocz(sizeof(*clli));
+        if (!clli)
+            return AVERROR(ENOMEM);
+
+        clli->Header.BufferId = MFX_EXTBUFF_CONTENT_LIGHT_LEVEL_INFO;
+        clli->Header.BufferSz = sizeof(*clli);
+
+        clli->MaxContentLightLevel          = clm->MaxCLL;
+        clli->MaxPicAverageLightLevel       = clm->MaxFALL;
+
+        enc_ctrl->ExtParam[enc_ctrl->NumExtParam++] = (mfxExtBuffer *)clli;
+    }
+
+    return 0;
+}
+
 static av_cold int qsv_enc_init(AVCodecContext *avctx)
 {
     QSVAV1EncContext *q = avctx->priv_data;
@@ -61,6 +132,8 @@ static av_cold int qsv_enc_init(AVCodecContext *avctx)
            return ret;
     }
 
+    q->qsv.set_encode_ctrl_cb = qsv_av1_set_encode_ctrl;
+
     return ff_qsv_enc_init(avctx, &q->qsv);
 }
 
-- 
2.34.1

_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".

Re: [FFmpeg-devel] [PATCH 1/2] lavc/qsvdec: update HDR side data on output AVFrame for AV1 decoding

[Xiang, Haihao]

On Wo, 2024-05-22 at 12:15 +0800, Xiang, Haihao wrote:
> From: Haihao Xiang <haihao.xiang@intel.com>
> 
> The SDK may provide HDR metadata for HDR streams via mfxExtBuffer
> attached on output mfxFrameSurface1
> 
> Signed-off-by: Haihao Xiang <haihao.xiang@intel.com>
> ---
>  libavcodec/qsvdec.c | 48 ++++++++++++++++++++++++++++++++++++++++++++-
>  1 file changed, 47 insertions(+), 1 deletion(-)
> 
> diff --git a/libavcodec/qsvdec.c b/libavcodec/qsvdec.c
> index df0d49bc10..7741baff06 100644
> --- a/libavcodec/qsvdec.c
> +++ b/libavcodec/qsvdec.c
> @@ -538,7 +538,8 @@ static int alloc_frame(AVCodecContext *avctx, QSVContext
> *q, QSVFrame *frame)
>  #endif
>  
>  #if QSV_VERSION_ATLEAST(1, 35)
> -    if (QSV_RUNTIME_VERSION_ATLEAST(q->ver, 1, 35) && avctx->codec_id ==
> AV_CODEC_ID_HEVC) {
> +    if ((QSV_RUNTIME_VERSION_ATLEAST(q->ver, 1, 35) && avctx->codec_id ==
> AV_CODEC_ID_HEVC) ||
> +        (QSV_RUNTIME_VERSION_ATLEAST(q->ver, 2, 9) && avctx->codec_id ==
> AV_CODEC_ID_AV1)) {
>          frame->mdcv.Header.BufferId =
> MFX_EXTBUFF_MASTERING_DISPLAY_COLOUR_VOLUME;
>          frame->mdcv.Header.BufferSz = sizeof(frame->mdcv);
>          // The data in mdcv is valid when this flag is 1
> @@ -742,6 +743,45 @@ static int qsv_export_hdr_side_data(AVCodecContext
> *avctx, mfxExtMasteringDispla
>      return 0;
>  }
>  
> +static int qsv_export_hdr_side_data_av1(AVCodecContext *avctx,
> mfxExtMasteringDisplayColourVolume *mdcv,
> +                                        mfxExtContentLightLevelInfo *clli,
> AVFrame *frame)
> +{
> +    if (mdcv->InsertPayloadToggle) {
> +        AVMasteringDisplayMetadata *mastering =
> av_mastering_display_metadata_create_side_data(frame);
> +        const int chroma_den   = 1 << 16;
> +        const int max_luma_den = 1 << 8;
> +        const int min_luma_den = 1 << 14;
> +
> +        if (!mastering)
> +            return AVERROR(ENOMEM);
> +
> +        for (int i = 0; i < 3; i++) {
> +            mastering->display_primaries[i][0] = av_make_q(mdcv-
> >DisplayPrimariesX[i], chroma_den);
> +            mastering->display_primaries[i][1] = av_make_q(mdcv-
> >DisplayPrimariesY[i], chroma_den);
> +        }
> +
> +        mastering->white_point[0] = av_make_q(mdcv->WhitePointX, chroma_den);
> +        mastering->white_point[1] = av_make_q(mdcv->WhitePointY, chroma_den);
> +
> +        mastering->max_luminance = av_make_q(mdcv-
> >MaxDisplayMasteringLuminance, max_luma_den);
> +        mastering->min_luminance = av_make_q(mdcv-
> >MinDisplayMasteringLuminance, min_luma_den);
> +
> +        mastering->has_luminance = 1;
> +        mastering->has_primaries = 1;
> +    }
> +
> +    if (clli->InsertPayloadToggle) {
> +        AVContentLightMetadata *light =
> av_content_light_metadata_create_side_data(frame);
> +        if (!light)
> +            return AVERROR(ENOMEM);
> +
> +        light->MaxCLL  = clli->MaxContentLightLevel;
> +        light->MaxFALL = clli->MaxPicAverageLightLevel;
> +    }
> +
> +    return 0;
> +}
> +
>  #endif
>  
>  static int qsv_decode(AVCodecContext *avctx, QSVContext *q,
> @@ -874,6 +914,12 @@ static int qsv_decode(AVCodecContext *avctx, QSVContext
> *q,
>              if (ret < 0)
>                  return ret;
>          }
> +
> +        if (QSV_RUNTIME_VERSION_ATLEAST(q->ver, 2, 9) && avctx->codec_id ==
> AV_CODEC_ID_AV1) {
> +            ret = qsv_export_hdr_side_data_av1(avctx, &aframe.frame->mdcv,
> &aframe.frame->clli, frame);
> +            if (ret < 0)
> +                return ret;
> +        }
>  #endif
>  
>          frame->repeat_pict =

Will apply,

-Haihao


_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".

Re: [FFmpeg-devel] [PATCH 1/2] lavc/qsvdec: update HDR side data on output AVFrame for AV1 decoding

[Andreas Rheinhardt]

Xiang, Haihao:
> +static int qsv_export_hdr_side_data_av1(AVCodecContext *avctx, mfxExtMasteringDisplayColourVolume *mdcv,
> +                                        mfxExtContentLightLevelInfo *clli, AVFrame *frame)
> +{
> +    if (mdcv->InsertPayloadToggle) {
> +        AVMasteringDisplayMetadata *mastering = av_mastering_display_metadata_create_side_data(frame);
> +        const int chroma_den   = 1 << 16;
> +        const int max_luma_den = 1 << 8;
> +        const int min_luma_den = 1 << 14;
> +        int i;

Iterators with loop scope should be preferred if possible (like here).

> +
> +        if (!mastering)
> +            return AVERROR(ENOMEM);
> +
> +        for (i = 0; i < 3; i++) {
> +            mastering->display_primaries[i][0] = av_make_q(mdcv->DisplayPrimariesX[i], chroma_den);
> +            mastering->display_primaries[i][1] = av_make_q(mdcv->DisplayPrimariesY[i], chroma_den);
> +        }
> +
> +        mastering->white_point[0] = av_make_q(mdcv->WhitePointX, chroma_den);
> +        mastering->white_point[1] = av_make_q(mdcv->WhitePointY, chroma_den);
> +
> +        mastering->max_luminance = av_make_q(mdcv->MaxDisplayMasteringLuminance, max_luma_den);
> +        mastering->min_luminance = av_make_q(mdcv->MinDisplayMasteringLuminance, min_luma_den);
> +
> +        mastering->has_luminance = 1;
> +        mastering->has_primaries = 1;
> +    }
> +

_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".

[FFmpeg-devel] [PATCH 1/2] lavc/qsvdec: update HDR side data on output AVFrame for AV1 decoding

[Xiang, Haihao]

From: Haihao Xiang <haihao.xiang@intel.com>

The SDK may provides HDR metadata for HDR streams via mfxExtBuffer
attached on output mfxFrameSurface1

Signed-off-by: Haihao Xiang <haihao.xiang@intel.com>
---
 libavcodec/qsvdec.c | 49 ++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 48 insertions(+), 1 deletion(-)

diff --git a/libavcodec/qsvdec.c b/libavcodec/qsvdec.c
index df0d49bc10..7ffc7f0571 100644
--- a/libavcodec/qsvdec.c
+++ b/libavcodec/qsvdec.c
@@ -538,7 +538,8 @@ static int alloc_frame(AVCodecContext *avctx, QSVContext *q, QSVFrame *frame)
 #endif
 
 #if QSV_VERSION_ATLEAST(1, 35)
-    if (QSV_RUNTIME_VERSION_ATLEAST(q->ver, 1, 35) && avctx->codec_id == AV_CODEC_ID_HEVC) {
+    if ((QSV_RUNTIME_VERSION_ATLEAST(q->ver, 1, 35) && avctx->codec_id == AV_CODEC_ID_HEVC) ||
+        (QSV_RUNTIME_VERSION_ATLEAST(q->ver, 2, 9) && avctx->codec_id == AV_CODEC_ID_AV1)) {
         frame->mdcv.Header.BufferId = MFX_EXTBUFF_MASTERING_DISPLAY_COLOUR_VOLUME;
         frame->mdcv.Header.BufferSz = sizeof(frame->mdcv);
         // The data in mdcv is valid when this flag is 1
@@ -742,6 +743,46 @@ static int qsv_export_hdr_side_data(AVCodecContext *avctx, mfxExtMasteringDispla
     return 0;
 }
 
+static int qsv_export_hdr_side_data_av1(AVCodecContext *avctx, mfxExtMasteringDisplayColourVolume *mdcv,
+                                        mfxExtContentLightLevelInfo *clli, AVFrame *frame)
+{
+    if (mdcv->InsertPayloadToggle) {
+        AVMasteringDisplayMetadata *mastering = av_mastering_display_metadata_create_side_data(frame);
+        const int chroma_den   = 1 << 16;
+        const int max_luma_den = 1 << 8;
+        const int min_luma_den = 1 << 14;
+        int i;
+
+        if (!mastering)
+            return AVERROR(ENOMEM);
+
+        for (i = 0; i < 3; i++) {
+            mastering->display_primaries[i][0] = av_make_q(mdcv->DisplayPrimariesX[i], chroma_den);
+            mastering->display_primaries[i][1] = av_make_q(mdcv->DisplayPrimariesY[i], chroma_den);
+        }
+
+        mastering->white_point[0] = av_make_q(mdcv->WhitePointX, chroma_den);
+        mastering->white_point[1] = av_make_q(mdcv->WhitePointY, chroma_den);
+
+        mastering->max_luminance = av_make_q(mdcv->MaxDisplayMasteringLuminance, max_luma_den);
+        mastering->min_luminance = av_make_q(mdcv->MinDisplayMasteringLuminance, min_luma_den);
+
+        mastering->has_luminance = 1;
+        mastering->has_primaries = 1;
+    }
+
+    if (clli->InsertPayloadToggle) {
+        AVContentLightMetadata *light = av_content_light_metadata_create_side_data(frame);
+        if (!light)
+            return AVERROR(ENOMEM);
+
+        light->MaxCLL  = clli->MaxContentLightLevel;
+        light->MaxFALL = clli->MaxPicAverageLightLevel;
+    }
+
+    return 0;
+}
+
 #endif
 
 static int qsv_decode(AVCodecContext *avctx, QSVContext *q,
@@ -874,6 +915,12 @@ static int qsv_decode(AVCodecContext *avctx, QSVContext *q,
             if (ret < 0)
                 return ret;
         }
+
+        if (QSV_RUNTIME_VERSION_ATLEAST(q->ver, 2, 9) && avctx->codec_id == AV_CODEC_ID_AV1) {
+            ret = qsv_export_hdr_side_data_av1(avctx, &aframe.frame->mdcv, &aframe.frame->clli, frame);
+            if (ret < 0)
+                return ret;
+        }
 #endif
 
         frame->repeat_pict =
-- 
2.34.1

_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".