Re: [FFmpeg-devel] [PATCH] lavc/vvc: Support pps_mixed_nalu_types_in_pic_flag

[Nuo Mi <nuomi2021-at-gmail.com@ffmpeg.org>]

On Sun, Jun 1, 2025 at 9:45 PM Frank Plowman <post@frankplowman.com> wrote:

> Add support for the pps_mixed_nalu_types_in_pic_flag, which permits
> using different NALU types for different VCL NAL units of a single
> picture.
>
> Fixes decoding of the only two Main 10 conformance bitstreams which
> could not previously be decoded correctly: MNUT_A_Nokia and
> MNUT_B_Nokia.
>
Hi Frank,
Thank you for the patch.
This one-line patch addresses the issue, but a deeper solution may be
needed.
Currently, we inherit HEVC’s DPB logic, which forces a new seq_decode on
IDR frames.
However, for VVC with mixed NAL units, this assumption doesn’t hold.
The seq_decode logic (which isn’t in the spec) might need to be removed for
a long-term fix

diff --git a/libavcodec/vvc/dec.c b/libavcodec/vvc/dec.c
index aba31b94fe..6501415402 100644
--- a/libavcodec/vvc/dec.c
+++ b/libavcodec/vvc/dec.c
@@ -722,7 +722,7 @@ static int frame_context_setup(VVCFrameContext *fc,
VVCContext *s)
             return ret;
     }

-    if (IS_IDR(s)) {
+    if (IS_IDR(s) && !fc->ps.pps->r->pps_mixed_nalu_types_in_pic_flag) {
         s->seq_decode = (s->seq_decode + 1) & 0xff;
         ff_vvc_clear_refs(fc);
     }


>
> Signed-off-by: Frank Plowman <post@frankplowman.com>
> ---
>  libavcodec/vaapi_vvc.c |  2 +-
>  libavcodec/vvc.h       | 11 +++++
>  libavcodec/vvc/dec.c   | 99 +++++++++++++++++++++++++++++++++++++++++-
>  libavcodec/vvc/dec.h   |  2 +-
>  libavcodec/vvc/ps.h    | 10 ++---
>  5 files changed, 116 insertions(+), 8 deletions(-)
>
> diff --git a/libavcodec/vaapi_vvc.c b/libavcodec/vaapi_vvc.c
> index 908db7bfab..436a097a61 100644
> --- a/libavcodec/vaapi_vvc.c
> +++ b/libavcodec/vaapi_vvc.c
> @@ -236,7 +236,7 @@ static int vaapi_vvc_start_frame(AVCodecContext
>   *avctx,
>              .ph_deblocking_filter_disabled_flag    =
> ph->ph_deblocking_filter_disabled_flag,
>          },
>          .PicMiscFlags.fields = {
> -            .IntraPicFlag = pps->pps_mixed_nalu_types_in_pic_flag ? 0 :
> IS_IRAP(h) ? 1 : 0,
> +            .IntraPicFlag = IS_IRAP(h) ? 1 : 0,
>          }
>      };
>
> diff --git a/libavcodec/vvc.h b/libavcodec/vvc.h
> index 5490ddb4c8..d59b22e38d 100644
> --- a/libavcodec/vvc.h
> +++ b/libavcodec/vvc.h
> @@ -66,6 +66,17 @@ enum VVCSliceType {
>      VVC_SLICE_TYPE_I = 2,
>  };
>
> +enum VVCPictureType {
> +    VVC_PICTURE_TYPE_UNSPEC,
> +    VVC_PICTURE_TYPE_CRA,
> +    VVC_PICTURE_TYPE_GDR,
> +    VVC_PICTURE_TYPE_IDR,
> +    VVC_PICTURE_TYPE_RADL,
> +    VVC_PICTURE_TYPE_RASL,
> +    VVC_PICTURE_TYPE_STSA,
> +    VVC_PICTURE_TYPE_TRAILING,
> +};
> +
>  enum VVCAPSType {
>      VVC_ASP_TYPE_ALF     = 0,
>      VVC_ASP_TYPE_LMCS    = 1,
> diff --git a/libavcodec/vvc/dec.c b/libavcodec/vvc/dec.c
> index 381b42c421..f131a6e7eb 100644
> --- a/libavcodec/vvc/dec.c
> +++ b/libavcodec/vvc/dec.c
> @@ -459,6 +459,98 @@ static void smvd_ref_idx(const VVCFrameContext *fc,
> SliceContext *sc)
>      }
>  }
>
> +static int get_picture_type(VVCContext *s, int nb_nalus)
> +{
> +    const CodedBitstreamH266Context *h266 = s->cbc->priv_data;
> +    const H266RawPPS *pps = h266->pps[h266->ph->ph_pic_parameter_set_id];
> +    bool has_nut[VVC_RSV_IRAP_11 /* Final VCL NUT */ + 1] = {false};
> +    int num_nuts = 0;
> +
> +    for (int i = 0; i < nb_nalus; i++) {
> +        const H2645NAL *nal = h266->common.read_packet.nals + i;
> +        switch (nal->type) {
> +            case VVC_TRAIL_NUT:
> +            case VVC_STSA_NUT:
> +            case VVC_RADL_NUT:
> +            case VVC_RASL_NUT:
> +            case VVC_RSV_VCL_4:
> +            case VVC_RSV_VCL_5:
> +            case VVC_RSV_VCL_6:
> +            case VVC_IDR_W_RADL:
> +            case VVC_IDR_N_LP:
> +            case VVC_CRA_NUT:
> +            case VVC_GDR_NUT:
> +            case VVC_RSV_IRAP_11:
> +                if (!has_nut[nal->type]) {
> +                    has_nut[nal->type] = true;
> +                    num_nuts++;
> +                }
> +                break;
> +            default:  // Non-VCL NALU
> +                continue;
> +        }
> +    }
> +
> +    if (!pps->pps_mixed_nalu_types_in_pic_flag && num_nuts > 1) {
> +        const char *msg = "pps_mixed_nalu_types_in_pic_flag is 0, yet
> picture contains mixed NALU types.\n";
> +        if (s->avctx->strict_std_compliance >= FF_COMPLIANCE_STRICT) {
> +            av_log(s->avctx, AV_LOG_ERROR, "%s", msg);
> +            return AVERROR_INVALIDDATA;
> +        } else {
> +            av_log(s->avctx, AV_LOG_WARNING, "%s", msg);
> +        }
> +    } else if (pps->pps_mixed_nalu_types_in_pic_flag && num_nuts == 1) {
> +        const char *msg = "pps_mixed_nalu_types_in_pic_flag is 1, yet
> picture contains only a single NALU type.\n";
> +        if (s->avctx->strict_std_compliance >= FF_COMPLIANCE_STRICT) {
> +            av_log(s->avctx, AV_LOG_ERROR, "%s", msg);
> +            return AVERROR_INVALIDDATA;
> +        } else {
> +            av_log(s->avctx, AV_LOG_WARNING, "%s", msg);
> +        }
> +    }
> +
> +    if (num_nuts == 1) {
> +        for (enum VVCNALUnitType nut = 0; nut < VVC_RSV_IRAP_11 + 1;
> nut++) {
> +            if (has_nut[nut]) {
> +                switch (nut) {
> +                    case VVC_CRA_NUT:
> +                        return VVC_PICTURE_TYPE_CRA;
> +                    case VVC_GDR_NUT:
> +                        return VVC_PICTURE_TYPE_GDR;
> +                    case VVC_IDR_W_RADL:
> +                    case VVC_IDR_N_LP:
> +                        return VVC_PICTURE_TYPE_IDR;
> +                    case VVC_RADL_NUT:
> +                        return VVC_PICTURE_TYPE_RADL;
> +                    case VVC_RASL_NUT:
> +                        return VVC_PICTURE_TYPE_RASL;
> +                    case VVC_STSA_NUT:
> +                        return VVC_PICTURE_TYPE_STSA;
> +                    case VVC_TRAIL_NUT:
> +                        return VVC_PICTURE_TYPE_TRAILING;
> +                    case VVC_RSV_VCL_4:
> +                    case VVC_RSV_VCL_5:
> +                    case VVC_RSV_VCL_6:
> +                    case VVC_RSV_IRAP_11:
> +                        av_log(s->avctx, AV_LOG_ERROR, "Unsupported VCL
> NUT: %d\n", nut);
> +                        return AVERROR_PATCHWELCOME;
> +                    default:  // Non-VCL NUT; should be unreachable
> +                        av_assert0(0);
> +                }
> +            }
> +        }
> +    }
> +
> +    // The only picture type which does not require all VCL NALUs to have
> +    // the same type is the RASL picture, which contains only RASL and
> RADL
> +    // VCL NALUs.
> +    if (num_nuts == 2 && has_nut[VVC_RASL_NUT] && has_nut[VVC_RADL_NUT]) {
> +        return VVC_PICTURE_TYPE_RASL;
> +    }
> +
> +    return VVC_PICTURE_TYPE_UNSPEC;
> +}
> +
>  static void eps_free(SliceContext *slice)
>  {
>      av_freep(&slice->eps);
> @@ -951,7 +1043,6 @@ static int decode_slice(VVCContext *s,
> VVCFrameContext *fc, AVBufferRef *buf_ref
>
>      sc = fc->slices[fc->nb_slices];
>
> -    s->vcl_unit_type = nal->type;
>      if (is_first_slice) {
>          ret = frame_setup(fc, s);
>          if (ret < 0)
> @@ -1050,6 +1141,12 @@ static int decode_nal_units(VVCContext *s,
> VVCFrameContext *fc, AVPacket *avpkt)
>          av_log(s->avctx, AV_LOG_ERROR, "Failed to read packet.\n");
>          return ret;
>      }
> +
> +    ret = get_picture_type(s, frame->nb_units);
> +    if (ret < 0)
> +        return ret;
> +    s->picture_type = ret;
> +
>      /* decode the NAL units */
>      for (int i = 0; i < frame->nb_units; i++) {
>          const H2645NAL *nal            = h266->common.read_packet.nals +
> i;
> diff --git a/libavcodec/vvc/dec.h b/libavcodec/vvc/dec.h
> index 5f8065b38b..2a05227836 100644
> --- a/libavcodec/vvc/dec.h
> +++ b/libavcodec/vvc/dec.h
> @@ -230,7 +230,7 @@ typedef struct VVCContext {
>      int eos;                ///< current packet contains an EOS/EOB NAL
>      int last_eos;           ///< last packet contains an EOS/EOB NAL
>
> -    enum VVCNALUnitType vcl_unit_type;
> +    enum VVCPictureType picture_type;
>      int no_output_before_recovery_flag; ///< NoOutputBeforeRecoveryFlag
>      int gdr_recovery_point_poc;         ///< recoveryPointPocVal
>      int film_grain_warning_shown;
> diff --git a/libavcodec/vvc/ps.h b/libavcodec/vvc/ps.h
> index 3ec2238c17..1c6d2712d6 100644
> --- a/libavcodec/vvc/ps.h
> +++ b/libavcodec/vvc/ps.h
> @@ -26,14 +26,14 @@
>  #include "libavcodec/cbs_h266.h"
>  #include "libavcodec/vvc.h"
>
> -#define IS_IDR(s)  ((s)->vcl_unit_type == VVC_IDR_W_RADL ||
> (s)->vcl_unit_type == VVC_IDR_N_LP)
> -#define IS_CRA(s)  ((s)->vcl_unit_type == VVC_CRA_NUT)
> +#define IS_IDR(s)  ((s)->picture_type == VVC_PICTURE_TYPE_IDR)
> +#define IS_CRA(s)  ((s)->picture_type == VVC_PICTURE_TYPE_CRA)
>  #define IS_IRAP(s) (IS_IDR(s) || IS_CRA(s))
> -#define IS_GDR(s)  ((s)->vcl_unit_type == VVC_GDR_NUT)
> +#define IS_GDR(s)  ((s)->picture_type == VVC_PICTURE_TYPE_GDR)
>  #define IS_CVSS(s) (IS_IRAP(s)|| IS_GDR(s))
>  #define IS_CLVSS(s) (IS_CVSS(s) && s->no_output_before_recovery_flag)
> -#define IS_RASL(s) ((s)->vcl_unit_type == VVC_RASL_NUT)
> -#define IS_RADL(s) ((s)->vcl_unit_type == VVC_RADL_NUT)
> +#define IS_RASL(s) ((s)->picture_type == VVC_PICTURE_TYPE_RASL)
> +#define IS_RADL(s) ((s)->picture_type == VVC_PICTURE_TYPE_RADL)
>
>  #define IS_I(rsh) ((rsh)->sh_slice_type == VVC_SLICE_TYPE_I)
>  #define IS_P(rsh) ((rsh)->sh_slice_type == VVC_SLICE_TYPE_P)
> --
> 2.47.0
>
>
_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".