[FFmpeg-devel] [PATCH] lavc/vvc: Support pps_mixed_nalu_types_in_pic_flag

[Frank Plowman <post@frankplowman.com>]

Add support for the pps_mixed_nalu_types_in_pic_flag, which permits
using different NALU types for different VCL NAL units of a single
picture.

Fixes decoding of the only two Main 10 conformance bitstreams which
could not previously be decoded correctly: MNUT_A_Nokia and
MNUT_B_Nokia.

Signed-off-by: Frank Plowman <post@frankplowman.com>
---
 libavcodec/vaapi_vvc.c |  2 +-
 libavcodec/vvc.h       | 11 +++++
 libavcodec/vvc/dec.c   | 99 +++++++++++++++++++++++++++++++++++++++++-
 libavcodec/vvc/dec.h   |  2 +-
 libavcodec/vvc/ps.h    | 10 ++---
 5 files changed, 116 insertions(+), 8 deletions(-)

diff --git a/libavcodec/vaapi_vvc.c b/libavcodec/vaapi_vvc.c
index 908db7bfab..436a097a61 100644
--- a/libavcodec/vaapi_vvc.c
+++ b/libavcodec/vaapi_vvc.c
@@ -236,7 +236,7 @@ static int vaapi_vvc_start_frame(AVCodecContext          *avctx,
             .ph_deblocking_filter_disabled_flag    = ph->ph_deblocking_filter_disabled_flag,
         },
         .PicMiscFlags.fields = {
-            .IntraPicFlag = pps->pps_mixed_nalu_types_in_pic_flag ? 0 : IS_IRAP(h) ? 1 : 0,
+            .IntraPicFlag = IS_IRAP(h) ? 1 : 0,
         }
     };
 
diff --git a/libavcodec/vvc.h b/libavcodec/vvc.h
index 5490ddb4c8..d59b22e38d 100644
--- a/libavcodec/vvc.h
+++ b/libavcodec/vvc.h
@@ -66,6 +66,17 @@ enum VVCSliceType {
     VVC_SLICE_TYPE_I = 2,
 };
 
+enum VVCPictureType {
+    VVC_PICTURE_TYPE_UNSPEC,
+    VVC_PICTURE_TYPE_CRA,
+    VVC_PICTURE_TYPE_GDR,
+    VVC_PICTURE_TYPE_IDR,
+    VVC_PICTURE_TYPE_RADL,
+    VVC_PICTURE_TYPE_RASL,
+    VVC_PICTURE_TYPE_STSA,
+    VVC_PICTURE_TYPE_TRAILING,
+};
+
 enum VVCAPSType {
     VVC_ASP_TYPE_ALF     = 0,
     VVC_ASP_TYPE_LMCS    = 1,
diff --git a/libavcodec/vvc/dec.c b/libavcodec/vvc/dec.c
index 381b42c421..f131a6e7eb 100644
--- a/libavcodec/vvc/dec.c
+++ b/libavcodec/vvc/dec.c
@@ -459,6 +459,98 @@ static void smvd_ref_idx(const VVCFrameContext *fc, SliceContext *sc)
     }
 }
 
+static int get_picture_type(VVCContext *s, int nb_nalus)
+{
+    const CodedBitstreamH266Context *h266 = s->cbc->priv_data;
+    const H266RawPPS *pps = h266->pps[h266->ph->ph_pic_parameter_set_id];
+    bool has_nut[VVC_RSV_IRAP_11 /* Final VCL NUT */ + 1] = {false};
+    int num_nuts = 0;
+
+    for (int i = 0; i < nb_nalus; i++) {
+        const H2645NAL *nal = h266->common.read_packet.nals + i;
+        switch (nal->type) {
+            case VVC_TRAIL_NUT:
+            case VVC_STSA_NUT:
+            case VVC_RADL_NUT:
+            case VVC_RASL_NUT:
+            case VVC_RSV_VCL_4:
+            case VVC_RSV_VCL_5:
+            case VVC_RSV_VCL_6:
+            case VVC_IDR_W_RADL:
+            case VVC_IDR_N_LP:
+            case VVC_CRA_NUT:
+            case VVC_GDR_NUT:
+            case VVC_RSV_IRAP_11:
+                if (!has_nut[nal->type]) {
+                    has_nut[nal->type] = true;
+                    num_nuts++;
+                }
+                break;
+            default:  // Non-VCL NALU
+                continue;
+        }
+    }
+
+    if (!pps->pps_mixed_nalu_types_in_pic_flag && num_nuts > 1) {
+        const char *msg = "pps_mixed_nalu_types_in_pic_flag is 0, yet picture contains mixed NALU types.\n";
+        if (s->avctx->strict_std_compliance >= FF_COMPLIANCE_STRICT) {
+            av_log(s->avctx, AV_LOG_ERROR, "%s", msg);
+            return AVERROR_INVALIDDATA;
+        } else {
+            av_log(s->avctx, AV_LOG_WARNING, "%s", msg);
+        }
+    } else if (pps->pps_mixed_nalu_types_in_pic_flag && num_nuts == 1) {
+        const char *msg = "pps_mixed_nalu_types_in_pic_flag is 1, yet picture contains only a single NALU type.\n";
+        if (s->avctx->strict_std_compliance >= FF_COMPLIANCE_STRICT) {
+            av_log(s->avctx, AV_LOG_ERROR, "%s", msg);
+            return AVERROR_INVALIDDATA;
+        } else {
+            av_log(s->avctx, AV_LOG_WARNING, "%s", msg);
+        }
+    }
+
+    if (num_nuts == 1) {
+        for (enum VVCNALUnitType nut = 0; nut < VVC_RSV_IRAP_11 + 1; nut++) {
+            if (has_nut[nut]) {
+                switch (nut) {
+                    case VVC_CRA_NUT:
+                        return VVC_PICTURE_TYPE_CRA;
+                    case VVC_GDR_NUT:
+                        return VVC_PICTURE_TYPE_GDR;
+                    case VVC_IDR_W_RADL:
+                    case VVC_IDR_N_LP:
+                        return VVC_PICTURE_TYPE_IDR;
+                    case VVC_RADL_NUT:
+                        return VVC_PICTURE_TYPE_RADL;
+                    case VVC_RASL_NUT:
+                        return VVC_PICTURE_TYPE_RASL;
+                    case VVC_STSA_NUT:
+                        return VVC_PICTURE_TYPE_STSA;
+                    case VVC_TRAIL_NUT:
+                        return VVC_PICTURE_TYPE_TRAILING;
+                    case VVC_RSV_VCL_4:
+                    case VVC_RSV_VCL_5:
+                    case VVC_RSV_VCL_6:
+                    case VVC_RSV_IRAP_11:
+                        av_log(s->avctx, AV_LOG_ERROR, "Unsupported VCL NUT: %d\n", nut);
+                        return AVERROR_PATCHWELCOME;
+                    default:  // Non-VCL NUT; should be unreachable
+                        av_assert0(0);
+                }
+            }
+        }
+    }
+
+    // The only picture type which does not require all VCL NALUs to have
+    // the same type is the RASL picture, which contains only RASL and RADL
+    // VCL NALUs.
+    if (num_nuts == 2 && has_nut[VVC_RASL_NUT] && has_nut[VVC_RADL_NUT]) {
+        return VVC_PICTURE_TYPE_RASL;
+    }
+
+    return VVC_PICTURE_TYPE_UNSPEC;
+}
+
 static void eps_free(SliceContext *slice)
 {
     av_freep(&slice->eps);
@@ -951,7 +1043,6 @@ static int decode_slice(VVCContext *s, VVCFrameContext *fc, AVBufferRef *buf_ref
 
     sc = fc->slices[fc->nb_slices];
 
-    s->vcl_unit_type = nal->type;
     if (is_first_slice) {
         ret = frame_setup(fc, s);
         if (ret < 0)
@@ -1050,6 +1141,12 @@ static int decode_nal_units(VVCContext *s, VVCFrameContext *fc, AVPacket *avpkt)
         av_log(s->avctx, AV_LOG_ERROR, "Failed to read packet.\n");
         return ret;
     }
+
+    ret = get_picture_type(s, frame->nb_units);
+    if (ret < 0)
+        return ret;
+    s->picture_type = ret;
+
     /* decode the NAL units */
     for (int i = 0; i < frame->nb_units; i++) {
         const H2645NAL *nal            = h266->common.read_packet.nals + i;
diff --git a/libavcodec/vvc/dec.h b/libavcodec/vvc/dec.h
index 5f8065b38b..2a05227836 100644
--- a/libavcodec/vvc/dec.h
+++ b/libavcodec/vvc/dec.h
@@ -230,7 +230,7 @@ typedef struct VVCContext {
     int eos;                ///< current packet contains an EOS/EOB NAL
     int last_eos;           ///< last packet contains an EOS/EOB NAL
 
-    enum VVCNALUnitType vcl_unit_type;
+    enum VVCPictureType picture_type;
     int no_output_before_recovery_flag; ///< NoOutputBeforeRecoveryFlag
     int gdr_recovery_point_poc;         ///< recoveryPointPocVal
     int film_grain_warning_shown;
diff --git a/libavcodec/vvc/ps.h b/libavcodec/vvc/ps.h
index 3ec2238c17..1c6d2712d6 100644
--- a/libavcodec/vvc/ps.h
+++ b/libavcodec/vvc/ps.h
@@ -26,14 +26,14 @@
 #include "libavcodec/cbs_h266.h"
 #include "libavcodec/vvc.h"
 
-#define IS_IDR(s)  ((s)->vcl_unit_type == VVC_IDR_W_RADL || (s)->vcl_unit_type == VVC_IDR_N_LP)
-#define IS_CRA(s)  ((s)->vcl_unit_type == VVC_CRA_NUT)
+#define IS_IDR(s)  ((s)->picture_type == VVC_PICTURE_TYPE_IDR)
+#define IS_CRA(s)  ((s)->picture_type == VVC_PICTURE_TYPE_CRA)
 #define IS_IRAP(s) (IS_IDR(s) || IS_CRA(s))
-#define IS_GDR(s)  ((s)->vcl_unit_type == VVC_GDR_NUT)
+#define IS_GDR(s)  ((s)->picture_type == VVC_PICTURE_TYPE_GDR)
 #define IS_CVSS(s) (IS_IRAP(s)|| IS_GDR(s))
 #define IS_CLVSS(s) (IS_CVSS(s) && s->no_output_before_recovery_flag)
-#define IS_RASL(s) ((s)->vcl_unit_type == VVC_RASL_NUT)
-#define IS_RADL(s) ((s)->vcl_unit_type == VVC_RADL_NUT)
+#define IS_RASL(s) ((s)->picture_type == VVC_PICTURE_TYPE_RASL)
+#define IS_RADL(s) ((s)->picture_type == VVC_PICTURE_TYPE_RADL)
 
 #define IS_I(rsh) ((rsh)->sh_slice_type == VVC_SLICE_TYPE_I)
 #define IS_P(rsh) ((rsh)->sh_slice_type == VVC_SLICE_TYPE_P)
-- 
2.47.0

_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".