* [FFmpeg-devel] [PATCH] ffmpeg: pass encoder init AVFrame side data to output AVStream
@ 2023-03-05 11:17 Jan Ekström
2023-03-05 18:16 ` Jan Ekström
0 siblings, 1 reply; 2+ messages in thread
From: Jan Ekström @ 2023-03-05 11:17 UTC (permalink / raw)
To: ffmpeg-devel
This enables passing through various side data during encoding,
which is not yet in AVCodecContext/AVCodecParameters, but is read
from AVStream side data in muxers.
Additionally, add a FATE test that demonstrates PNG->J2K MP4
transcoding with the ICC profile getting passed through.
---
fftools/ffmpeg.c | 6 ++
fftools/ffmpeg.h | 1 +
fftools/ffmpeg_mux.c | 47 +++++++++++++
tests/fate/ffmpeg.mak | 4 ++
tests/ref/fate/ffmpeg-side-data-to-avstreams | 72 ++++++++++++++++++++
5 files changed, 130 insertions(+)
create mode 100644 tests/ref/fate/ffmpeg-side-data-to-avstreams
diff --git a/fftools/ffmpeg.c b/fftools/ffmpeg.c
index d721a5e721..134258b825 100644
--- a/fftools/ffmpeg.c
+++ b/fftools/ffmpeg.c
@@ -3113,6 +3113,12 @@ static int init_output_stream_encode(OutputStream *ost, AVFrame *frame)
av_pix_fmt_desc_get(enc_ctx->pix_fmt)->comp[0].depth);
if (frame) {
+ if (!(ost->side_data_frame = av_frame_alloc()))
+ return AVERROR(ENOMEM);
+
+ if ((ret = av_frame_copy_props(ost->side_data_frame, frame)) < 0)
+ return ret;
+
enc_ctx->color_range = frame->color_range;
enc_ctx->color_primaries = frame->color_primaries;
enc_ctx->color_trc = frame->color_trc;
diff --git a/fftools/ffmpeg.h b/fftools/ffmpeg.h
index 4d4433f5ba..21dd0f3a9e 100644
--- a/fftools/ffmpeg.h
+++ b/fftools/ffmpeg.h
@@ -593,6 +593,7 @@ typedef struct OutputStream {
AVFrame *filtered_frame;
AVFrame *last_frame;
AVFrame *sq_frame;
+ AVFrame *side_data_frame;
AVPacket *pkt;
int64_t last_dropped;
int64_t last_nb0_frames[3];
diff --git a/fftools/ffmpeg_mux.c b/fftools/ffmpeg_mux.c
index cf58051949..13da9ab8da 100644
--- a/fftools/ffmpeg_mux.c
+++ b/fftools/ffmpeg_mux.c
@@ -580,6 +580,49 @@ static int bsf_init(MuxStream *ms)
return 0;
}
+static int avframe_side_data_to_avstream(AVStream *stream, const AVFrame *frame)
+{
+ static const struct sd_mapping {
+ enum AVPacketSideDataType packet;
+ enum AVFrameSideDataType frame;
+ } sd_list[] = {
+ { AV_PKT_DATA_REPLAYGAIN , AV_FRAME_DATA_REPLAYGAIN },
+ { AV_PKT_DATA_DISPLAYMATRIX, AV_FRAME_DATA_DISPLAYMATRIX },
+ { AV_PKT_DATA_SPHERICAL, AV_FRAME_DATA_SPHERICAL },
+ { AV_PKT_DATA_STEREO3D, AV_FRAME_DATA_STEREO3D },
+ { AV_PKT_DATA_AUDIO_SERVICE_TYPE, AV_FRAME_DATA_AUDIO_SERVICE_TYPE },
+ { AV_PKT_DATA_MASTERING_DISPLAY_METADATA, AV_FRAME_DATA_MASTERING_DISPLAY_METADATA },
+ { AV_PKT_DATA_CONTENT_LIGHT_LEVEL, AV_FRAME_DATA_CONTENT_LIGHT_LEVEL },
+ { AV_PKT_DATA_A53_CC, AV_FRAME_DATA_A53_CC },
+ { AV_PKT_DATA_ICC_PROFILE, AV_FRAME_DATA_ICC_PROFILE },
+ { AV_PKT_DATA_S12M_TIMECODE, AV_FRAME_DATA_S12M_TIMECODE },
+ { AV_PKT_DATA_DYNAMIC_HDR10_PLUS, AV_FRAME_DATA_DYNAMIC_HDR_PLUS },
+ };
+
+ if (!frame || !frame->nb_side_data)
+ return 0;
+
+ if (!stream)
+ return AVERROR(EINVAL);
+
+ for (int i = 0; i < FF_ARRAY_ELEMS(sd_list); i++) {
+ const struct sd_mapping mapping = sd_list[i];
+ uint8_t *packet_sd = NULL;
+ AVFrameSideData *sd = av_frame_get_side_data(frame, mapping.frame);
+ if (!sd)
+ continue;
+
+ packet_sd = av_stream_new_side_data(stream, mapping.packet,
+ sd->size);
+ if (!packet_sd)
+ return AVERROR(ENOMEM);
+
+ memcpy(packet_sd, sd->data, sd->size);
+ }
+
+ return 0;
+}
+
int of_stream_init(OutputFile *of, OutputStream *ost)
{
Muxer *mux = mux_from_of(of);
@@ -589,6 +632,9 @@ int of_stream_init(OutputFile *of, OutputStream *ost)
if (ost->sq_idx_mux >= 0)
sq_set_tb(mux->sq_mux, ost->sq_idx_mux, ost->mux_timebase);
+ if ((ret = avframe_side_data_to_avstream(ost->st, ost->side_data_frame)) < 0)
+ return ret;
+
/* initialize bitstream filters for the output stream
* needs to be done here, because the codec id for streamcopy is not
* known until now */
@@ -666,6 +712,7 @@ static void ost_free(OutputStream **post)
av_frame_free(&ost->filtered_frame);
av_frame_free(&ost->sq_frame);
av_frame_free(&ost->last_frame);
+ av_frame_free(&ost->side_data_frame);
av_packet_free(&ost->pkt);
av_dict_free(&ost->encoder_opts);
diff --git a/tests/fate/ffmpeg.mak b/tests/fate/ffmpeg.mak
index 0f33c2a0ed..3d6ef2a8fb 100644
--- a/tests/fate/ffmpeg.mak
+++ b/tests/fate/ffmpeg.mak
@@ -224,3 +224,7 @@ FATE_TIME_BASE-$(call PARSERDEMDEC, MPEGVIDEO, MPEGPS, MPEG2VIDEO, MPEGVIDEO_DEM
fate-time_base: CMD = md5 -i $(TARGET_SAMPLES)/mpeg2/dvd_single_frame.vob -an -sn -c:v copy -r 25 -time_base 1001:30000 -fflags +bitexact -f mxf
FATE_SAMPLES_FFMPEG-yes += $(FATE_TIME_BASE-yes)
+
+FATE_SAMPLES_FFMPEG-$(call TRANSCODE, JPEG2000 PNG, MP4 IMAGE_PNG_PIPE) += fate-ffmpeg-side-data-to-avstreams
+fate-ffmpeg-side-data-to-avstreams: CMD = transcode png_pipe $(TARGET_SAMPLES)/png1/lena-int_rgb24.png\
+ mp4 "-c jpeg2000" "" "-show_streams"
diff --git a/tests/ref/fate/ffmpeg-side-data-to-avstreams b/tests/ref/fate/ffmpeg-side-data-to-avstreams
new file mode 100644
index 0000000000..7681861c94
--- /dev/null
+++ b/tests/ref/fate/ffmpeg-side-data-to-avstreams
@@ -0,0 +1,72 @@
+95be5b4c97ba8989fb738df35cb8b7f2 *tests/data/fate/ffmpeg-side-data-to-avstreams.mp4
+36472 tests/data/fate/ffmpeg-side-data-to-avstreams.mp4
+#tb 0: 1/25
+#media_type 0: video
+#codec_id 0: rawvideo
+#dimensions 0: 128x128
+#sar 0: 1/1
+0, 0, 0, 1, 49152, 0xf6fe3b30
+[STREAM]
+index=0
+codec_name=jpeg2000
+profile=0
+codec_type=video
+codec_tag_string=mp4v
+codec_tag=0x7634706d
+width=128
+height=128
+coded_width=128
+coded_height=128
+closed_captions=0
+film_grain=0
+has_b_frames=0
+sample_aspect_ratio=1:1
+display_aspect_ratio=1:1
+pix_fmt=rgb24
+level=-99
+color_range=unknown
+color_space=unknown
+color_transfer=unknown
+color_primaries=unknown
+chroma_location=unspecified
+field_order=bt
+refs=1
+id=0x1
+r_frame_rate=25/1
+avg_frame_rate=25/1
+time_base=1/12800
+start_pts=0
+start_time=0.000000
+duration_ts=512
+duration=0.040000
+bit_rate=6503600
+max_bit_rate=N/A
+bits_per_raw_sample=8
+nb_frames=1
+nb_read_frames=N/A
+nb_read_packets=N/A
+DISPOSITION:default=1
+DISPOSITION:dub=0
+DISPOSITION:original=0
+DISPOSITION:comment=0
+DISPOSITION:lyrics=0
+DISPOSITION:karaoke=0
+DISPOSITION:forced=0
+DISPOSITION:hearing_impaired=0
+DISPOSITION:visual_impaired=0
+DISPOSITION:clean_effects=0
+DISPOSITION:attached_pic=0
+DISPOSITION:timed_thumbnails=0
+DISPOSITION:captions=0
+DISPOSITION:descriptions=0
+DISPOSITION:metadata=0
+DISPOSITION:dependent=0
+DISPOSITION:still_image=0
+TAG:language=und
+TAG:handler_name=VideoHandler
+TAG:vendor_id=[0][0][0][0]
+TAG:encoder=Lavc jpeg2000
+[SIDE_DATA]
+side_data_type=ICC Profile
+[/SIDE_DATA]
+[/STREAM]
--
2.39.2
_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
^ permalink raw reply [flat|nested] 2+ messages in thread
* Re: [FFmpeg-devel] [PATCH] ffmpeg: pass encoder init AVFrame side data to output AVStream
2023-03-05 11:17 [FFmpeg-devel] [PATCH] ffmpeg: pass encoder init AVFrame side data to output AVStream Jan Ekström
@ 2023-03-05 18:16 ` Jan Ekström
0 siblings, 0 replies; 2+ messages in thread
From: Jan Ekström @ 2023-03-05 18:16 UTC (permalink / raw)
To: ffmpeg-devel
On Sun, Mar 5, 2023 at 1:17 PM Jan Ekström <jeebjp@gmail.com> wrote:
>
> This enables passing through various side data during encoding,
> which is not yet in AVCodecContext/AVCodecParameters, but is read
> from AVStream side data in muxers.
>
> Additionally, add a FATE test that demonstrates PNG->J2K MP4
> transcoding with the ICC profile getting passed through.
> ---
Just tested locally and going PNG->MP4 (RGB H.264)->PNG actually does
work with ffmpeg.c now with regards to ICC side data :) So something
like this does actually help with round-tripping (most likely since
the side data goes from the AVStream to the first AVPacket and thus
into the first AVFrame, which then gets picked up by pngenc),
Currently this is only limited to video, as audio cannot work with the
avfilter peek API, as the peeked frame gets buffered, and thus any
further adjustment of audio frame size will fail, as the buffered
frame gets output first (with whatever the original requested size
was).
> fftools/ffmpeg.c | 6 ++
> fftools/ffmpeg.h | 1 +
> fftools/ffmpeg_mux.c | 47 +++++++++++++
> tests/fate/ffmpeg.mak | 4 ++
> tests/ref/fate/ffmpeg-side-data-to-avstreams | 72 ++++++++++++++++++++
> 5 files changed, 130 insertions(+)
> create mode 100644 tests/ref/fate/ffmpeg-side-data-to-avstreams
>
> diff --git a/fftools/ffmpeg.c b/fftools/ffmpeg.c
> index d721a5e721..134258b825 100644
> --- a/fftools/ffmpeg.c
> +++ b/fftools/ffmpeg.c
> @@ -3113,6 +3113,12 @@ static int init_output_stream_encode(OutputStream *ost, AVFrame *frame)
> av_pix_fmt_desc_get(enc_ctx->pix_fmt)->comp[0].depth);
>
> if (frame) {
> + if (!(ost->side_data_frame = av_frame_alloc()))
> + return AVERROR(ENOMEM);
> +
> + if ((ret = av_frame_copy_props(ost->side_data_frame, frame)) < 0)
> + return ret;
> +
> enc_ctx->color_range = frame->color_range;
> enc_ctx->color_primaries = frame->color_primaries;
> enc_ctx->color_trc = frame->color_trc;
> diff --git a/fftools/ffmpeg.h b/fftools/ffmpeg.h
> index 4d4433f5ba..21dd0f3a9e 100644
> --- a/fftools/ffmpeg.h
> +++ b/fftools/ffmpeg.h
> @@ -593,6 +593,7 @@ typedef struct OutputStream {
> AVFrame *filtered_frame;
> AVFrame *last_frame;
> AVFrame *sq_frame;
> + AVFrame *side_data_frame;
> AVPacket *pkt;
> int64_t last_dropped;
> int64_t last_nb0_frames[3];
> diff --git a/fftools/ffmpeg_mux.c b/fftools/ffmpeg_mux.c
> index cf58051949..13da9ab8da 100644
> --- a/fftools/ffmpeg_mux.c
> +++ b/fftools/ffmpeg_mux.c
> @@ -580,6 +580,49 @@ static int bsf_init(MuxStream *ms)
> return 0;
> }
>
> +static int avframe_side_data_to_avstream(AVStream *stream, const AVFrame *frame)
> +{
> + static const struct sd_mapping {
> + enum AVPacketSideDataType packet;
> + enum AVFrameSideDataType frame;
> + } sd_list[] = {
> + { AV_PKT_DATA_REPLAYGAIN , AV_FRAME_DATA_REPLAYGAIN },
> + { AV_PKT_DATA_DISPLAYMATRIX, AV_FRAME_DATA_DISPLAYMATRIX },
> + { AV_PKT_DATA_SPHERICAL, AV_FRAME_DATA_SPHERICAL },
> + { AV_PKT_DATA_STEREO3D, AV_FRAME_DATA_STEREO3D },
> + { AV_PKT_DATA_AUDIO_SERVICE_TYPE, AV_FRAME_DATA_AUDIO_SERVICE_TYPE },
> + { AV_PKT_DATA_MASTERING_DISPLAY_METADATA, AV_FRAME_DATA_MASTERING_DISPLAY_METADATA },
> + { AV_PKT_DATA_CONTENT_LIGHT_LEVEL, AV_FRAME_DATA_CONTENT_LIGHT_LEVEL },
> + { AV_PKT_DATA_A53_CC, AV_FRAME_DATA_A53_CC },
> + { AV_PKT_DATA_ICC_PROFILE, AV_FRAME_DATA_ICC_PROFILE },
> + { AV_PKT_DATA_S12M_TIMECODE, AV_FRAME_DATA_S12M_TIMECODE },
> + { AV_PKT_DATA_DYNAMIC_HDR10_PLUS, AV_FRAME_DATA_DYNAMIC_HDR_PLUS },
> + };
For the record, this listing was taken from
ff_decode_frame_props_from_pkt, and probably at the end of the day
should be in a "get one from the other" or "iterate me all the
mappings" sort of API in avcodec.
Then this function adding stuff to AVStream could be in avformat
without the list being duplicated.
This thing is here in this patch mostly to prove that this kind of
passing through additional side data would help.
Jan
_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
^ permalink raw reply [flat|nested] 2+ messages in thread
end of thread, other threads:[~2023-03-05 18:17 UTC | newest]
Thread overview: 2+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2023-03-05 11:17 [FFmpeg-devel] [PATCH] ffmpeg: pass encoder init AVFrame side data to output AVStream Jan Ekström
2023-03-05 18:16 ` Jan Ekström
Git Inbox Mirror of the ffmpeg-devel mailing list - see https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
This inbox may be cloned and mirrored by anyone:
git clone --mirror https://master.gitmailbox.com/ffmpegdev/0 ffmpegdev/git/0.git
# If you have public-inbox 1.1+ installed, you may
# initialize and index your mirror using the following commands:
public-inbox-init -V2 ffmpegdev ffmpegdev/ https://master.gitmailbox.com/ffmpegdev \
ffmpegdev@gitmailbox.com
public-inbox-index ffmpegdev
Example config snippet for mirrors.
AGPL code for this site: git clone https://public-inbox.org/public-inbox.git