[FFmpeg-devel] [PATCH] avcodec/hevc_mp4toannexb: prepend extradata before the leading PS for an IRAP

[James Almer <jamrial@gmail.com>]

Parameter sets may be coded in the packet before an IRAP (as is the case for
the hev1 ISO-BMFF brand), and they should have priority as they may override
the extradata ones.
As such, prepend the extradata PS NALUs to the packet PS NALUs if they are
present before an IRAP, instead of prepending them to the IRAP slice.

Should fix ticket #11458.

Signed-off-by: James Almer <jamrial@gmail.com>
---
 libavcodec/bsf/hevc_mp4toannexb.c | 34 +++++++++++++++++++++++++++++--
 1 file changed, 32 insertions(+), 2 deletions(-)

diff --git a/libavcodec/bsf/hevc_mp4toannexb.c b/libavcodec/bsf/hevc_mp4toannexb.c
index f281185769..60f38ac414 100644
--- a/libavcodec/bsf/hevc_mp4toannexb.c
+++ b/libavcodec/bsf/hevc_mp4toannexb.c
@@ -126,6 +126,7 @@ static int hevc_mp4toannexb_filter(AVBSFContext *ctx, AVPacket *out)
     GetByteContext gb;
 
     int got_irap = 0;
+    int got_ps = 0, seen_irap_ps = 0;
     int i, ret = 0;
 
     ret = ff_bsf_get_packet(ctx, &in);
@@ -140,10 +141,37 @@ static int hevc_mp4toannexb_filter(AVBSFContext *ctx, AVPacket *out)
 
     bytestream2_init(&gb, in->data, in->size);
 
+    while (!got_irap && bytestream2_get_bytes_left(&gb)) {
+        uint32_t nalu_size = 0;
+        int      nalu_type;
+
+        if (bytestream2_get_bytes_left(&gb) < s->length_size) {
+            ret = AVERROR_INVALIDDATA;
+            goto fail;
+        }
+        for (i = 0; i < s->length_size; i++)
+            nalu_size = (nalu_size << 8) | bytestream2_get_byte(&gb);
+
+        if (nalu_size < 2 || nalu_size > bytestream2_get_bytes_left(&gb)) {
+            ret = AVERROR_INVALIDDATA;
+            goto fail;
+        }
+
+        nalu_type = (bytestream2_peek_byte(&gb) >> 1) & 0x3f;
+        bytestream2_skip(&gb, nalu_size);
+        got_irap |= nalu_type >= HEVC_NAL_BLA_W_LP &&
+                    nalu_type <= HEVC_NAL_RSV_IRAP_VCL23;
+        got_ps   |= nalu_type >= HEVC_NAL_VPS && nalu_type <= HEVC_NAL_PPS;
+    }
+    seen_irap_ps = got_irap && got_ps;
+    got_irap = got_ps = 0;
+
+    bytestream2_init(&gb, in->data, in->size);
+
     while (bytestream2_get_bytes_left(&gb)) {
         uint32_t nalu_size = 0;
         int      nalu_type;
-        int is_irap, add_extradata, extra_size, prev_size;
+        int is_irap, is_ps, add_extradata, extra_size, prev_size;
 
         if (bytestream2_get_bytes_left(&gb) < s->length_size) {
             ret = AVERROR_INVALIDDATA;
@@ -162,9 +190,11 @@ static int hevc_mp4toannexb_filter(AVBSFContext *ctx, AVPacket *out)
         /* prepend extradata to IRAP frames */
         is_irap = nalu_type >= HEVC_NAL_BLA_W_LP &&
                   nalu_type <= HEVC_NAL_RSV_IRAP_VCL23;
-        add_extradata = is_irap && !got_irap;
+        is_ps   = nalu_type >= HEVC_NAL_VPS && nalu_type <= HEVC_NAL_PPS && seen_irap_ps;
+        add_extradata = (is_ps || is_irap) && !got_ps && !got_irap;
         extra_size    = add_extradata * ctx->par_out->extradata_size;
         got_irap     |= is_irap;
+        got_ps       |= is_ps;
 
         if (FFMIN(INT_MAX, SIZE_MAX) < 4ULL + nalu_size + extra_size) {
             ret = AVERROR_INVALIDDATA;
-- 
2.48.1

_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".