Re: [FFmpeg-devel] [PATCH v2 7/9] avcodec/nvenc: add MV-HEVC encoding support

[James Almer <jamrial@gmail.com>]

[-- Attachment #1.1.1: Type: text/plain, Size: 10136 bytes --]

On 1/30/2025 8:23 PM, Timo Rothenpieler wrote:
> From: Diego de Souza <ddesouza@nvidia.com>
> 
> Added support for MV-HEVC encoding for stereoscopic videos (2 views
> only). Compatible with the framepack filter when using the
> AV_STEREO3D_FRAMESEQUENCE format.
> 
> Signed-off-by: Diego de Souza <ddesouza@nvidia.com>
> ---
>   libavcodec/nvenc.c      | 70 +++++++++++++++++++++++++++++++++++++++++
>   libavcodec/nvenc.h      |  8 +++++
>   libavcodec/nvenc_hevc.c |  5 ++-
>   3 files changed, 82 insertions(+), 1 deletion(-)
> 
> diff --git a/libavcodec/nvenc.c b/libavcodec/nvenc.c
> index 3403fa8996..ad53eaaf98 100644
> --- a/libavcodec/nvenc.c
> +++ b/libavcodec/nvenc.c
> @@ -35,6 +35,7 @@
>   #include "libavutil/mem.h"
>   #include "libavutil/pixdesc.h"
>   #include "libavutil/mathematics.h"
> +#include "libavutil/stereo3d.h"
>   #include "atsc_a53.h"
>   #include "codec_desc.h"
>   #include "encode.h"
> @@ -656,6 +657,14 @@ static int nvenc_check_capabilities(AVCodecContext *avctx)
>   
>       ctx->support_dyn_bitrate = nvenc_check_cap(avctx, NV_ENC_CAPS_SUPPORT_DYN_BITRATE_CHANGE);
>   
> +#ifdef NVENC_HAVE_MVHEVC
> +    ctx->multiview_supported = nvenc_check_cap(avctx, NV_ENC_CAPS_SUPPORT_MVHEVC_ENCODE) > 0;
> +    if(ctx->profile == NV_ENC_HEVC_PROFILE_MULTIVIEW_MAIN && !ctx->multiview_supported) {
> +        av_log(avctx, AV_LOG_WARNING, "Multiview not supported by the device\n");
> +        return AVERROR(ENOSYS);
> +    }
> +#endif
> +
>       return 0;
>   }
>   
> @@ -1475,6 +1484,13 @@ static av_cold int nvenc_setup_hevc_config(AVCodecContext *avctx)
>   
>       hevc->outputPictureTimingSEI = 1;
>   
> +#ifdef NVENC_HAVE_MVHEVC
> +    if (ctx->multiview_supported && ctx->profile == NV_ENC_HEVC_PROFILE_MAIN &&
> +        (av_frame_side_data_get(avctx->decoded_side_data, avctx->nb_decoded_side_data, AV_FRAME_DATA_STEREO3D) ||

Stereo 3D side data being present in the encoder global side data array 
is not guarantee that it's there to signal stereoscopic video as it 
could have a type of AV_STEREO3D_2D, for example.

You need to ensure it's either AV_STEREO3D_UNSPEC, or 
AV_STEREO3D_FRAMESEQUENCE (which afaik we don't export at the global 
level, only for frames containing one of the two views).

> +         av_frame_side_data_get(avctx->decoded_side_data, avctx->nb_decoded_side_data, AV_FRAME_DATA_VIEW_ID)))

VIEW_ID is not a global type, so it should not be there are at all.

> +        ctx->profile = NV_ENC_HEVC_PROFILE_MULTIVIEW_MAIN;
> +#endif
> +
>       switch (ctx->profile) {
>       case NV_ENC_HEVC_PROFILE_MAIN:
>           cc->profileGUID = NV_ENC_HEVC_PROFILE_MAIN_GUID;
> @@ -1488,6 +1504,16 @@ static av_cold int nvenc_setup_hevc_config(AVCodecContext *avctx)
>           cc->profileGUID = NV_ENC_HEVC_PROFILE_FREXT_GUID;
>           avctx->profile  = AV_PROFILE_HEVC_REXT;
>           break;
> +#ifdef NVENC_HAVE_MVHEVC
> +    case NV_ENC_HEVC_PROFILE_MULTIVIEW_MAIN:
> +        cc->profileGUID = NV_ENC_HEVC_PROFILE_MAIN_GUID;
> +        avctx->profile  = AV_PROFILE_HEVC_MULTIVIEW_MAIN;
> +        ctx->multiview  = 1;
> +
> +        hevc->enableMVHEVC = 1;
> +        hevc->outputHevc3DReferenceDisplayInfo = 1;
> +        break;
> +#endif
>       }
>   
>       // force setting profile as main10 if input is 10 bit or if it should be encoded as 10 bit
> @@ -1502,6 +1528,13 @@ static av_cold int nvenc_setup_hevc_config(AVCodecContext *avctx)
>           avctx->profile = AV_PROFILE_HEVC_REXT;
>       }
>   
> +#ifdef NVENC_HAVE_MVHEVC
> +    if (ctx->multiview && avctx->profile != AV_PROFILE_HEVC_MULTIVIEW_MAIN) {
> +        av_log(avctx, AV_LOG_ERROR, "Multiview encoding only works for Main profile content.\n");
> +        return AVERROR(EINVAL);
> +    }
> +#endif
> +
>       hevc->chromaFormatIDC = IS_YUV444(ctx->data_pix_fmt) ? 3 : IS_YUV422(ctx->data_pix_fmt) ? 2 : 1;
>   
>   #ifdef NVENC_HAVE_NEW_BIT_DEPTH_API
> @@ -2469,6 +2502,9 @@ static int nvenc_set_timestamp(AVCodecContext *avctx,
>   
>       // This can be more than necessary, but we don't know the real reorder delay.
>       delay = FFMAX(ctx->encode_config.frameIntervalP - 1, 0);
> +#ifdef NVENC_HAVE_MVHEVC
> +    delay *= ctx->multiview ? 2 : 1;
> +#endif
>       if (ctx->output_frame_num >= delay) {
>           pkt->dts = timestamp_queue_dequeue(ctx->timestamp_list);
>           ctx->output_frame_num++;
> @@ -2875,6 +2911,9 @@ static int nvenc_send_frame(AVCodecContext *avctx, const AVFrame *frame)
>       int res, res2;
>       int sei_count = 0;
>       int i;
> +#ifdef NVENC_HAVE_MVHEVC
> +    HEVC_3D_REFERENCE_DISPLAY_INFO ref_disp_info = { 0 };
> +#endif
>   
>       NvencContext *ctx = avctx->priv_data;
>       NvencDynLoadFunctions *dl_fn = &ctx->nvenc_dload_funcs;
> @@ -2952,6 +2991,37 @@ static int nvenc_send_frame(AVCodecContext *avctx, const AVFrame *frame)
>       if (res < 0)
>           return res;
>   
> +#ifdef NVENC_HAVE_MVHEVC
> +    if (ctx->multiview) {
> +        const AVFrameSideData *sd_stereo3d = av_frame_get_side_data(frame, AV_FRAME_DATA_STEREO3D);
> +        const AVFrameSideData *sd_view_id = av_frame_get_side_data(frame, AV_FRAME_DATA_VIEW_ID);
> +        int view_ids_reversed = 0;
> +
> +        if (sd_view_id)
> +            ctx->next_view_id = *(int*)sd_view_id->data;
> +
> +        if (sd_stereo3d) {
> +            const AVStereo3D *stereo = (const AVStereo3D*)sd_stereo3d->data;
> +
> +            if (stereo->type == AV_STEREO3D_FRAMESEQUENCE) {
> +                if (!sd_view_id)
> +                    ctx->next_view_id = (stereo->view == AV_STEREO3D_VIEW_LEFT) ? 0 : 1;
> +                if (stereo->view == AV_STEREO3D_VIEW_LEFT && ctx->next_view_id)
> +                    view_ids_reversed = 1;
> +            } else
> +                av_log(avctx, AV_LOG_ERROR, "Stereo format %d not supported! Only AV_STEREO3D_FRAMESEQUENCE is supported!\n", stereo->type);
> +        }
> +
> +        pic_params.codecPicParams.hevcPicParams.viewId = ctx->next_view_id;
> +        ctx->next_view_id ^= 1;
> +
> +        ref_disp_info.precRefDisplayWidth = 31;
> +        ref_disp_info.leftViewId[0] = view_ids_reversed ? 1 : 0;
> +        ref_disp_info.rightViewId[0] = view_ids_reversed ? 0 : 1;
> +        pic_params.codecPicParams.hevcPicParams.p3DReferenceDisplayInfo = &ref_disp_info;
> +    }
> +#endif
> +
>       nv_status = p_nvenc->nvEncEncodePicture(ctx->nvencoder, &pic_params);
>   
>       for (i = 0; i < sei_count; i++)
> diff --git a/libavcodec/nvenc.h b/libavcodec/nvenc.h
> index 6f7f8d4e7f..d085b08260 100644
> --- a/libavcodec/nvenc.h
> +++ b/libavcodec/nvenc.h
> @@ -99,6 +99,7 @@ typedef void ID3D11Device;
>   #define NVENC_HAVE_422_SUPPORT
>   #define NVENC_HAVE_AV1_UHQ_TUNING
>   #define NVENC_HAVE_H264_AND_AV1_TEMPORAL_FILTER
> +#define NVENC_HAVE_MVHEVC
>   #endif
>   
>   typedef struct NvencSurface
> @@ -172,6 +173,11 @@ enum {
>       NV_ENC_HEVC_PROFILE_MAIN,
>       NV_ENC_HEVC_PROFILE_MAIN_10,
>       NV_ENC_HEVC_PROFILE_REXT,
> +#ifdef NVENC_HAVE_MVHEVC
> +    NV_ENC_HEVC_PROFILE_MULTIVIEW_MAIN,
> +#endif
> +
> +    NV_ENC_HEVC_PROFILE_COUNT
>   };
>   
>   enum {
> @@ -245,6 +251,7 @@ typedef struct NvencContext
>       void *nvencoder;
>   
>       uint32_t frame_idx_counter;
> +    uint32_t next_view_id;
>   
>       int preset;
>       int profile;
> @@ -299,6 +306,7 @@ typedef struct NvencContext
>       int lookahead_level;
>       int unidir_b;
>       int split_encode_mode;
> +    int multiview, multiview_supported;
>   } NvencContext;
>   
>   int ff_nvenc_encode_init(AVCodecContext *avctx);
> diff --git a/libavcodec/nvenc_hevc.c b/libavcodec/nvenc_hevc.c
> index 5696e14dd4..3c08563c1f 100644
> --- a/libavcodec/nvenc_hevc.c
> +++ b/libavcodec/nvenc_hevc.c
> @@ -60,10 +60,13 @@ static const AVOption options[] = {
>       { "ull",         "Ultra low latency",                   0,                    AV_OPT_TYPE_CONST, { .i64 = NV_ENC_TUNING_INFO_ULTRA_LOW_LATENCY },        0, 0, VE, .unit = "tune" },
>       { "lossless",    "Lossless",                            0,                    AV_OPT_TYPE_CONST, { .i64 = NV_ENC_TUNING_INFO_LOSSLESS },                 0, 0, VE, .unit = "tune" },
>   #endif
> -    { "profile",      "Set the encoding profile",           OFFSET(profile),      AV_OPT_TYPE_INT,   { .i64 = NV_ENC_HEVC_PROFILE_MAIN }, NV_ENC_HEVC_PROFILE_MAIN, AV_PROFILE_HEVC_REXT, VE, .unit = "profile" },
> +    { "profile",      "Set the encoding profile",           OFFSET(profile),      AV_OPT_TYPE_INT,   { .i64 = NV_ENC_HEVC_PROFILE_MAIN }, NV_ENC_HEVC_PROFILE_MAIN, NV_ENC_HEVC_PROFILE_COUNT - 1, VE, .unit = "profile" },
>       { "main",         "",                                   0,                    AV_OPT_TYPE_CONST, { .i64 = NV_ENC_HEVC_PROFILE_MAIN },    0, 0, VE, .unit = "profile" },
>       { "main10",       "",                                   0,                    AV_OPT_TYPE_CONST, { .i64 = NV_ENC_HEVC_PROFILE_MAIN_10 }, 0, 0, VE, .unit = "profile" },
>       { "rext",         "",                                   0,                    AV_OPT_TYPE_CONST, { .i64 = NV_ENC_HEVC_PROFILE_REXT },    0, 0, VE, .unit = "profile" },
> +#ifdef NVENC_HAVE_MVHEVC
> +    { "mv",           "",                                   0,                    AV_OPT_TYPE_CONST, { .i64 = NV_ENC_HEVC_PROFILE_MULTIVIEW_MAIN }, 0, 0, VE, .unit = "profile" },
> +#endif
>       { "level",        "Set the encoding level restriction", OFFSET(level),        AV_OPT_TYPE_INT,   { .i64 = NV_ENC_LEVEL_AUTOSELECT }, NV_ENC_LEVEL_AUTOSELECT, NV_ENC_LEVEL_HEVC_62, VE, .unit = "level" },
>       { "auto",         "",                                   0,                    AV_OPT_TYPE_CONST, { .i64 = NV_ENC_LEVEL_AUTOSELECT },  0, 0, VE,  .unit = "level" },
>       { "1",            "",                                   0,                    AV_OPT_TYPE_CONST, { .i64 = NV_ENC_LEVEL_HEVC_1 },      0, 0, VE,  .unit = "level" },


[-- Attachment #1.2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 495 bytes --]

[-- Attachment #2: Type: text/plain, Size: 251 bytes --]

_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".