* [FFmpeg-devel] [PATCH v3 3/5] avcodec/h264_mp4toannexb_bsf: fix missing PS before IDR frames
[not found] <20231117172441.288316-1-quinkblack@foxmail.com>
2023-11-17 17:24 ` [FFmpeg-devel] [PATCH v3 2/5] avcodec/h264_mp4toannexb_bsf: remove pass padding size as argument Zhao Zhili
@ 2023-11-17 17:24 ` Zhao Zhili
2023-11-20 3:09 ` Zhao Zhili
2023-11-17 17:24 ` [FFmpeg-devel] [PATCH v3 4/5] avcodec/h264_mp4toannexb_bsf: process new extradata Zhao Zhili
2023-11-17 17:24 ` [FFmpeg-devel] [PATCH v3 5/5] fate/h264: move mp4toannexb_ticket5927 test to fate-h264 Zhao Zhili
3 siblings, 1 reply; 5+ messages in thread
From: Zhao Zhili @ 2023-11-17 17:24 UTC (permalink / raw)
To: ffmpeg-devel; +Cc: Zhao Zhili
From: Zhao Zhili <zhilizhao@tencent.com>
If there is a single group of SPS/PPS before an IDR frame, but no
SPS/PPS after that, we will miss the chance to reset
idr_sps_seen/idr_pps_seen. No SPS/PPS are inserted afterwards.
This patch saves in-band SPS/PPS and insert them before IDR frames
when necessary.
---
libavcodec/h264_mp4toannexb_bsf.c | 83 ++++++++++++++++++++++++++++---
tests/fate/h264.mak | 8 ++-
2 files changed, 84 insertions(+), 7 deletions(-)
diff --git a/libavcodec/h264_mp4toannexb_bsf.c b/libavcodec/h264_mp4toannexb_bsf.c
index 846671abb6..4073c780c5 100644
--- a/libavcodec/h264_mp4toannexb_bsf.c
+++ b/libavcodec/h264_mp4toannexb_bsf.c
@@ -36,6 +36,8 @@ typedef struct H264BSFContext {
uint8_t *pps;
int sps_size;
int pps_size;
+ unsigned sps_buf_size;
+ unsigned pps_buf_size;
uint8_t length_size;
uint8_t new_idr;
uint8_t idr_sps_seen;
@@ -130,16 +132,33 @@ pps:
memset(out + total_size, 0, padding);
if (pps_offset) {
- s->sps = out;
+ uint8_t *sps;
+
s->sps_size = pps_offset;
+ sps = av_fast_realloc(s->sps, &s->sps_buf_size, s->sps_size);
+ if (!sps) {
+ av_free(out);
+ return AVERROR(ENOMEM);
+ }
+ s->sps = sps;
+ memcpy(s->sps, out, s->sps_size);
} else {
av_log(ctx, AV_LOG_WARNING,
"Warning: SPS NALU missing or invalid. "
"The resulting stream may not play.\n");
}
if (pps_offset < total_size) {
- s->pps = out + pps_offset;
+ uint8_t *pps;
+
s->pps_size = total_size - pps_offset;
+ pps = av_fast_realloc(s->pps, &s->pps_buf_size, s->pps_size);
+ if (!pps) {
+ av_freep(&s->sps);
+ av_free(out);
+ return AVERROR(ENOMEM);
+ }
+ s->pps = pps;
+ memcpy(s->pps, out + pps_offset, s->pps_size);
} else {
av_log(ctx, AV_LOG_WARNING,
"Warning: PPS NALU missing or invalid. "
@@ -153,6 +172,35 @@ pps:
return length_size;
}
+static int h264_mp4toannexb_save_ps(uint8_t **dst, int *dst_size,
+ unsigned *dst_buf_size,
+ const uint8_t *nal, uint32_t nal_size,
+ int first)
+{
+ static const uint8_t nalu_header[4] = { 0, 0, 0, 1 };
+ const int start_code_size = sizeof(nalu_header);
+ uint8_t *ptr;
+ uint32_t size;
+
+ if (first)
+ size = 0;
+ else
+ size = *dst_size;
+
+ ptr = av_fast_realloc(*dst, dst_buf_size, size + nal_size + start_code_size);
+ if (!ptr)
+ return AVERROR(ENOMEM);
+
+ memcpy(ptr + size, nalu_header, start_code_size);
+ size += start_code_size;
+ memcpy(ptr + size, nal, nal_size);
+ size += nal_size;
+
+ *dst = ptr;
+ *dst_size = size;
+ return 0;
+}
+
static int h264_mp4toannexb_init(AVBSFContext *ctx)
{
H264BSFContext *s = ctx->priv_data;
@@ -211,6 +259,9 @@ static int h264_mp4toannexb_filter(AVBSFContext *ctx, AVPacket *opkt)
if (j) \
av_log(__VA_ARGS__)
for (int j = 0; j < 2; j++) {
+ int sps_count = 0;
+ int pps_count = 0;
+
buf = in->data;
new_idr = s->new_idr;
sps_seen = s->idr_sps_seen;
@@ -241,8 +292,18 @@ static int h264_mp4toannexb_filter(AVBSFContext *ctx, AVPacket *opkt)
if (unit_type == H264_NAL_SPS) {
sps_seen = new_idr = 1;
+ if (!j) {
+ h264_mp4toannexb_save_ps(&s->sps, &s->sps_size, &s->sps_buf_size,
+ buf, nal_size, !sps_count);
+ sps_count++;
+ }
} else if (unit_type == H264_NAL_PPS) {
pps_seen = new_idr = 1;
+ if (!j) {
+ h264_mp4toannexb_save_ps(&s->pps, &s->pps_size, &s->pps_buf_size,
+ buf, nal_size, !pps_count);
+ pps_count++;
+ }
/* if SPS has not been seen yet, prepend the AVCC one to PPS */
if (!sps_seen) {
if (!s->sps_size) {
@@ -262,9 +323,10 @@ static int h264_mp4toannexb_filter(AVBSFContext *ctx, AVPacket *opkt)
/* prepend only to the first type 5 NAL unit of an IDR picture, if no sps/pps are already present */
if (new_idr && unit_type == H264_NAL_IDR_SLICE && !sps_seen && !pps_seen) {
- if (ctx->par_out->extradata)
- count_or_copy(&out, &out_size, ctx->par_out->extradata,
- ctx->par_out->extradata_size, PS_OUT_OF_BAND, j);
+ if (s->sps_size)
+ count_or_copy(&out, &out_size, s->sps, s->sps_size, PS_OUT_OF_BAND, j);
+ if (s->pps_size)
+ count_or_copy(&out, &out_size, s->pps, s->pps_size, PS_OUT_OF_BAND, j);
new_idr = 0;
/* if only SPS has been seen, also insert PPS */
} else if (new_idr && unit_type == H264_NAL_IDR_SLICE && sps_seen && !pps_seen) {
@@ -280,7 +342,7 @@ static int h264_mp4toannexb_filter(AVBSFContext *ctx, AVPacket *opkt)
else
ps = PS_NONE;
count_or_copy(&out, &out_size, buf, nal_size, ps, j);
- if (!new_idr && unit_type == H264_NAL_SLICE) {
+ if (unit_type == H264_NAL_SLICE) {
new_idr = 1;
sps_seen = 0;
pps_seen = 0;
@@ -320,6 +382,14 @@ fail:
return ret;
}
+static void h264_mp4toannexb_close(AVBSFContext *ctx)
+{
+ H264BSFContext *s = ctx->priv_data;
+
+ av_freep(&s->sps);
+ av_freep(&s->pps);
+}
+
static void h264_mp4toannexb_flush(AVBSFContext *ctx)
{
H264BSFContext *s = ctx->priv_data;
@@ -339,5 +409,6 @@ const FFBitStreamFilter ff_h264_mp4toannexb_bsf = {
.priv_data_size = sizeof(H264BSFContext),
.init = h264_mp4toannexb_init,
.filter = h264_mp4toannexb_filter,
+ .close = h264_mp4toannexb_close,
.flush = h264_mp4toannexb_flush,
};
diff --git a/tests/fate/h264.mak b/tests/fate/h264.mak
index 0ff3d6df55..c7e0d0a84e 100644
--- a/tests/fate/h264.mak
+++ b/tests/fate/h264.mak
@@ -224,7 +224,9 @@ FATE_H264-$(call FRAMECRC, MOV, H264) += fate-h264-unescaped-extradata
# this sample contains field-coded frames, with both fields in a single packet
FATE_H264-$(call FRAMECRC, MOV, H264) += fate-h264-twofields-packet
-FATE_H264-$(call DEMMUX, MOV, H264, H264_MP4TOANNEXB_BSF) += fate-h264-bsf-mp4toannexb
+FATE_H264-$(call DEMMUX, MOV, H264, H264_MP4TOANNEXB_BSF) += fate-h264-bsf-mp4toannexb \
+ fate-h264-bsf-mp4toannexb-2
+
FATE_H264-$(call FRAMECRC, MATROSKA, H264) += fate-h264-direct-bff
FATE_H264-$(call FRAMECRC, FLV, H264, SCALE_FILTER) += fate-h264-brokensps-2580
FATE_H264-$(call FRAMECRC, MXF, H264, PCM_S24LE_DECODER SCALE_FILTER ARESAMPLE_FILTER) += fate-h264-xavc-4389
@@ -426,6 +428,10 @@ fate-h264-conformance-sva_nl1_b: CMD = framecrc -i $(TARGET_SAM
fate-h264-conformance-sva_nl2_e: CMD = framecrc -i $(TARGET_SAMPLES)/h264-conformance/SVA_NL2_E.264
fate-h264-bsf-mp4toannexb: CMD = md5 -i $(TARGET_SAMPLES)/h264/interlaced_crop.mp4 -c:v copy -f h264
+# First IDR is prefixed by SPS/PPS
+fate-h264-bsf-mp4toannexb-2: CMD = md5 -i $(TARGET_SAMPLES)/h264/ps_prefix_first_idr.mp4 -c:v copy -f h264
+fate-h264-bsf-mp4toannexb-2: CMP = oneline
+fate-h264-bsf-mp4toannexb-2: REF = cffcfa6a2d0b58c9de1f5785f099f41d
fate-h264-crop-to-container: CMD = framemd5 -i $(TARGET_SAMPLES)/h264/crop-to-container-dims-canon.mov
fate-h264-direct-bff: CMD = framecrc -i $(TARGET_SAMPLES)/h264/direct-bff.mkv
--
2.25.1
_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
^ permalink raw reply [flat|nested] 5+ messages in thread* [FFmpeg-devel] [PATCH v3 4/5] avcodec/h264_mp4toannexb_bsf: process new extradata
[not found] <20231117172441.288316-1-quinkblack@foxmail.com>
2023-11-17 17:24 ` [FFmpeg-devel] [PATCH v3 2/5] avcodec/h264_mp4toannexb_bsf: remove pass padding size as argument Zhao Zhili
2023-11-17 17:24 ` [FFmpeg-devel] [PATCH v3 3/5] avcodec/h264_mp4toannexb_bsf: fix missing PS before IDR frames Zhao Zhili
@ 2023-11-17 17:24 ` Zhao Zhili
2023-11-17 17:24 ` [FFmpeg-devel] [PATCH v3 5/5] fate/h264: move mp4toannexb_ticket5927 test to fate-h264 Zhao Zhili
3 siblings, 0 replies; 5+ messages in thread
From: Zhao Zhili @ 2023-11-17 17:24 UTC (permalink / raw)
To: ffmpeg-devel; +Cc: Zhao Zhili
From: Zhao Zhili <zhilizhao@tencent.com>
For fate-h264_mp4toannexb_ticket5927 and
fate-h264_mp4toannexb_ticket5927_2, they work by accident
previously. The sample file has two 'avc1' entries, and video
samples use the second one. It means packets should be decoded with
new extradata in side data. Before this patch, only extradata was
kept in the output, new extradata has been dropped. The output can
be decoded because the two extradata are almost the same, except
level indication. This patch fixed the issue, and add another
fate test.
---
libavcodec/h264_mp4toannexb_bsf.c | 37 ++++++++++++-------
tests/fate/h264.mak | 4 +-
| 9 +++++
tests/ref/fate/h264_mp4toannexb_ticket5927 | 8 ++--
tests/ref/fate/h264_mp4toannexb_ticket5927_2 | 8 ++--
5 files changed, 43 insertions(+), 23 deletions(-)
create mode 100644 tests/ref/fate/h264-bsf-mp4toannexb-new-extradata
diff --git a/libavcodec/h264_mp4toannexb_bsf.c b/libavcodec/h264_mp4toannexb_bsf.c
index 4073c780c5..120241c892 100644
--- a/libavcodec/h264_mp4toannexb_bsf.c
+++ b/libavcodec/h264_mp4toannexb_bsf.c
@@ -80,7 +80,8 @@ static void count_or_copy(uint8_t **out, uint64_t *out_size,
*out_size += start_code_size + in_size;
}
-static int h264_extradata_to_annexb(AVBSFContext *ctx)
+static int h264_extradata_to_annexb(AVBSFContext *ctx,
+ uint8_t *extradata, int extradata_size)
{
H264BSFContext *s = ctx->priv_data;
GetByteContext ogb, *gb = &ogb;
@@ -91,7 +92,7 @@ static int h264_extradata_to_annexb(AVBSFContext *ctx)
const int padding = AV_INPUT_BUFFER_PADDING_SIZE;
int length_size, pps_offset = 0;
- bytestream2_init(gb, ctx->par_in->extradata, ctx->par_in->extradata_size);
+ bytestream2_init(gb, extradata, extradata_size);
bytestream2_skipu(gb, 4);
@@ -169,7 +170,13 @@ pps:
ctx->par_out->extradata = out;
ctx->par_out->extradata_size = total_size;
- return length_size;
+ s->length_size = length_size;
+ s->new_idr = 1;
+ s->idr_sps_seen = 0;
+ s->idr_pps_seen = 0;
+ s->extradata_parsed = 1;
+
+ return 0;
}
static int h264_mp4toannexb_save_ps(uint8_t **dst, int *dst_size,
@@ -203,9 +210,7 @@ static int h264_mp4toannexb_save_ps(uint8_t **dst, int *dst_size,
static int h264_mp4toannexb_init(AVBSFContext *ctx)
{
- H264BSFContext *s = ctx->priv_data;
int extra_size = ctx->par_in->extradata_size;
- int ret;
/* retrieve sps and pps NAL units from extradata */
if (!extra_size ||
@@ -214,15 +219,9 @@ static int h264_mp4toannexb_init(AVBSFContext *ctx)
av_log(ctx, AV_LOG_VERBOSE,
"The input looks like it is Annex B already\n");
} else if (extra_size >= 7) {
- ret = h264_extradata_to_annexb(ctx);
- if (ret < 0)
- return ret;
-
- s->length_size = ret;
- s->new_idr = 1;
- s->idr_sps_seen = 0;
- s->idr_pps_seen = 0;
- s->extradata_parsed = 1;
+ return h264_extradata_to_annexb(ctx,
+ ctx->par_in->extradata,
+ ctx->par_in->extradata_size);
} else {
av_log(ctx, AV_LOG_ERROR, "Invalid extradata size: %d\n", extra_size);
return AVERROR_INVALIDDATA;
@@ -241,11 +240,21 @@ static int h264_mp4toannexb_filter(AVBSFContext *ctx, AVPacket *opkt)
uint8_t *out;
uint64_t out_size;
int ret;
+ size_t extradata_size;
+ uint8_t *extradata;
ret = ff_bsf_get_packet(ctx, &in);
if (ret < 0)
return ret;
+ extradata = av_packet_get_side_data(in, AV_PKT_DATA_NEW_EXTRADATA,
+ &extradata_size);
+ if (extradata) {
+ ret = h264_extradata_to_annexb(ctx, extradata, extradata_size);
+ if (ret < 0)
+ goto fail;
+ }
+
/* nothing to filter */
if (!s->extradata_parsed) {
av_packet_move_ref(opkt, in);
diff --git a/tests/fate/h264.mak b/tests/fate/h264.mak
index c7e0d0a84e..010dd9abf6 100644
--- a/tests/fate/h264.mak
+++ b/tests/fate/h264.mak
@@ -225,7 +225,8 @@ FATE_H264-$(call FRAMECRC, MOV, H264) += fate-h264-unescaped-extradata
FATE_H264-$(call FRAMECRC, MOV, H264) += fate-h264-twofields-packet
FATE_H264-$(call DEMMUX, MOV, H264, H264_MP4TOANNEXB_BSF) += fate-h264-bsf-mp4toannexb \
- fate-h264-bsf-mp4toannexb-2
+ fate-h264-bsf-mp4toannexb-2 \
+ fate-h264-bsf-mp4toannexb-new-extradata \
FATE_H264-$(call FRAMECRC, MATROSKA, H264) += fate-h264-direct-bff
FATE_H264-$(call FRAMECRC, FLV, H264, SCALE_FILTER) += fate-h264-brokensps-2580
@@ -432,6 +433,7 @@ fate-h264-bsf-mp4toannexb: CMD = md5 -i $(TARGET_SAMPLES)
fate-h264-bsf-mp4toannexb-2: CMD = md5 -i $(TARGET_SAMPLES)/h264/ps_prefix_first_idr.mp4 -c:v copy -f h264
fate-h264-bsf-mp4toannexb-2: CMP = oneline
fate-h264-bsf-mp4toannexb-2: REF = cffcfa6a2d0b58c9de1f5785f099f41d
+fate-h264-bsf-mp4toannexb-new-extradata: CMD = stream_remux mov $(TARGET_SAMPLES)/h264/extradata-reload-multi-stsd.mov "" h264 "-map 0:v"
fate-h264-crop-to-container: CMD = framemd5 -i $(TARGET_SAMPLES)/h264/crop-to-container-dims-canon.mov
fate-h264-direct-bff: CMD = framecrc -i $(TARGET_SAMPLES)/h264/direct-bff.mkv
--git a/tests/ref/fate/h264-bsf-mp4toannexb-new-extradata b/tests/ref/fate/h264-bsf-mp4toannexb-new-extradata
new file mode 100644
index 0000000000..a0423ad295
--- /dev/null
+++ b/tests/ref/fate/h264-bsf-mp4toannexb-new-extradata
@@ -0,0 +1,9 @@
+#tb 0: 1/25
+#media_type 0: video
+#codec_id 0: rawvideo
+#dimensions 0: 256x128
+#sar 0: 1/1
+0, 0, 0, 1, 49152, 0x08745db9
+0, 1, 1, 1, 49152, 0x96bf5e58
+0, 2, 2, 1, 49152, 0x8fe31b6d
+0, 3, 3, 1, 49152, 0x0b621cc3
diff --git a/tests/ref/fate/h264_mp4toannexb_ticket5927 b/tests/ref/fate/h264_mp4toannexb_ticket5927
index 95e35c4d80..eb0b5cf283 100644
--- a/tests/ref/fate/h264_mp4toannexb_ticket5927
+++ b/tests/ref/fate/h264_mp4toannexb_ticket5927
@@ -1,12 +1,12 @@
-a3b02fd09392e01619cebc959d4d9ff2 *tests/data/fate/h264_mp4toannexb_ticket5927.h264
+edddeef7901b2bd8d55625b8105b579f *tests/data/fate/h264_mp4toannexb_ticket5927.h264
595583 tests/data/fate/h264_mp4toannexb_ticket5927.h264
-#extradata 0: 33, 0x84fe08f8
+#extradata 0: 33, 0x84e308f7
#tb 0: 1/1200000
#media_type 0: video
#codec_id 0: h264
#dimensions 0: 1920x1080
#sar 0: 0/1
-0, -48000, -9223372036854775808, 48000, 247993, 0x1ce821ea
+0, -48000, -9223372036854775808, 48000, 247993, 0x541321e9
0, 0, -9223372036854775808, 48000, 43354, 0xa05dca6f, F=0x0
0, 48000, -9223372036854775808, 48000, 11423, 0x5e8086dd, F=0x0
0, 96000, -9223372036854775808, 48000, 50798, 0x145fbe4f, F=0x0
@@ -18,4 +18,4 @@ a3b02fd09392e01619cebc959d4d9ff2 *tests/data/fate/h264_mp4toannexb_ticket5927.h2
0, 384000, -9223372036854775808, 48000, 54483, 0xefead99f, F=0x0
0, 432000, -9223372036854775808, 48000, 13705, 0x23cd27e8, F=0x0
0, 480000, -9223372036854775808, 48000, 22308, 0x4093b5af, F=0x0
-0, 528000, -9223372036854775808, 48000, 6369, 0x858b2aa1
+0, 528000, -9223372036854775808, 48000, 6369, 0x6cca2aa0
diff --git a/tests/ref/fate/h264_mp4toannexb_ticket5927_2 b/tests/ref/fate/h264_mp4toannexb_ticket5927_2
index 8db6a7e54a..8c3613ee79 100644
--- a/tests/ref/fate/h264_mp4toannexb_ticket5927_2
+++ b/tests/ref/fate/h264_mp4toannexb_ticket5927_2
@@ -1,12 +1,12 @@
-a3b02fd09392e01619cebc959d4d9ff2 *tests/data/fate/h264_mp4toannexb_ticket5927_2.h264
+edddeef7901b2bd8d55625b8105b579f *tests/data/fate/h264_mp4toannexb_ticket5927_2.h264
595583 tests/data/fate/h264_mp4toannexb_ticket5927_2.h264
-#extradata 0: 33, 0x84fe08f8
+#extradata 0: 33, 0x84e308f7
#tb 0: 1/1200000
#media_type 0: video
#codec_id 0: h264
#dimensions 0: 1920x1080
#sar 0: 0/1
-0, -48000, -9223372036854775808, 48000, 247993, 0x1ce821ea
+0, -48000, -9223372036854775808, 48000, 247993, 0x541321e9
0, 0, -9223372036854775808, 48000, 43354, 0xa05dca6f, F=0x0
0, 48000, -9223372036854775808, 48000, 11423, 0x5e8086dd, F=0x0
0, 96000, -9223372036854775808, 48000, 50798, 0x145fbe4f, F=0x0
@@ -18,4 +18,4 @@ a3b02fd09392e01619cebc959d4d9ff2 *tests/data/fate/h264_mp4toannexb_ticket5927_2.
0, 384000, -9223372036854775808, 48000, 54483, 0xefead99f, F=0x0
0, 432000, -9223372036854775808, 48000, 13705, 0x23cd27e8, F=0x0
0, 480000, -9223372036854775808, 48000, 22308, 0x4093b5af, F=0x0
-0, 528000, -9223372036854775808, 48000, 6369, 0x858b2aa1
+0, 528000, -9223372036854775808, 48000, 6369, 0x6cca2aa0
--
2.25.1
_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
^ permalink raw reply [flat|nested] 5+ messages in thread