From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from ffbox0-bg.mplayerhq.hu (ffbox0-bg.ffmpeg.org [79.124.17.100]) by master.gitmailbox.com (Postfix) with ESMTP id E076B481BB for ; Tue, 12 Dec 2023 12:10:26 +0000 (UTC) Received: from [127.0.1.1] (localhost [127.0.0.1]) by ffbox0-bg.mplayerhq.hu (Postfix) with ESMTP id 5BFED68D16D; Tue, 12 Dec 2023 14:10:16 +0200 (EET) Received: from mail1.khirnov.net (quelana.khirnov.net [94.230.150.81]) by ffbox0-bg.mplayerhq.hu (Postfix) with ESMTPS id 2FFE368D102 for ; Tue, 12 Dec 2023 14:10:08 +0200 (EET) Received: from localhost (mail1.khirnov.net [IPv6:::1]) by mail1.khirnov.net (Postfix) with ESMTP id B657A1BA1 for ; Tue, 12 Dec 2023 13:10:07 +0100 (CET) Received: from mail1.khirnov.net ([IPv6:::1]) by localhost (mail1.khirnov.net [IPv6:::1]) (amavis, port 10024) with ESMTP id oFHbWecRcWST for ; Tue, 12 Dec 2023 13:10:06 +0100 (CET) Received: from libav.khirnov.net (libav.khirnov.net [IPv6:2a00:c500:561:201::7]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits) server-digest SHA256 client-signature RSA-PSS (2048 bits) client-digest SHA256) (Client CN "libav.khirnov.net", Issuer "smtp.khirnov.net SMTP CA" (verified OK)) by mail1.khirnov.net (Postfix) with ESMTPS id BD066122F for ; Tue, 12 Dec 2023 13:10:06 +0100 (CET) Received: from libav.khirnov.net (libav.khirnov.net [IPv6:::1]) by libav.khirnov.net (Postfix) with ESMTP id A1CBB3A0585 for ; Tue, 12 Dec 2023 13:09:59 +0100 (CET) From: Anton Khirnov To: ffmpeg-devel@ffmpeg.org Date: Tue, 12 Dec 2023 13:09:57 +0100 Message-ID: <20231212120958.21313-2-anton@khirnov.net> X-Mailer: git-send-email 2.42.0 In-Reply-To: <20231212120958.21313-1-anton@khirnov.net> References: <20231212120958.21313-1-anton@khirnov.net> MIME-Version: 1.0 Subject: [FFmpeg-devel] [PATCH 2/3] fftools/ffmpeg: attach wallclock timing information to packets and frames X-BeenThere: ffmpeg-devel@ffmpeg.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: FFmpeg development discussions and patches List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Reply-To: FFmpeg development discussions and patches Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit Errors-To: ffmpeg-devel-bounces@ffmpeg.org Sender: "ffmpeg-devel" Archived-At: List-Archive: List-Post: Will become useful in following commits. --- fftools/ffmpeg.c | 3 +++ fftools/ffmpeg.h | 13 +++++++++++++ fftools/ffmpeg_dec.c | 14 ++++++++++++-- fftools/ffmpeg_demux.c | 26 ++++++++++++-------------- fftools/ffmpeg_enc.c | 16 ++++++++++++++++ fftools/ffmpeg_filter.c | 9 +++++++++ 6 files changed, 65 insertions(+), 16 deletions(-) diff --git a/fftools/ffmpeg.c b/fftools/ffmpeg.c index e7ff9a6d6f..42e675dcb9 100644 --- a/fftools/ffmpeg.c +++ b/fftools/ffmpeg.c @@ -416,6 +416,9 @@ static int frame_data_ensure(AVBufferRef **dst, int writable) fd->dec.frame_num = UINT64_MAX; fd->dec.pts = AV_NOPTS_VALUE; + + for (unsigned i = 0; i < FF_ARRAY_ELEMS(fd->wallclock); i++) + fd->wallclock[i] = INT64_MIN; } else if (writable) return av_buffer_make_writable(dst); diff --git a/fftools/ffmpeg.h b/fftools/ffmpeg.h index aafb35538e..7f84f3266d 100644 --- a/fftools/ffmpeg.h +++ b/fftools/ffmpeg.h @@ -94,6 +94,17 @@ enum PacketOpaque { PKT_OPAQUE_FIX_SUB_DURATION, }; +enum LatencyProbe { + LATENCY_PROBE_DEMUX, + LATENCY_PROBE_DEC_PRE, + LATENCY_PROBE_DEC_POST, + LATENCY_PROBE_FILTER_PRE, + LATENCY_PROBE_FILTER_POST, + LATENCY_PROBE_ENC_PRE, + LATENCY_PROBE_ENC_POST, + LATENCY_PROBE_NB, +}; + typedef struct HWDevice { const char *name; enum AVHWDeviceType type; @@ -631,6 +642,8 @@ typedef struct FrameData { AVRational frame_rate_filter; int bits_per_raw_sample; + + int64_t wallclock[LATENCY_PROBE_NB]; } FrameData; extern InputFile **input_files; diff --git a/fftools/ffmpeg_dec.c b/fftools/ffmpeg_dec.c index 5dde82a276..ce5aa33f73 100644 --- a/fftools/ffmpeg_dec.c +++ b/fftools/ffmpeg_dec.c @@ -22,6 +22,7 @@ #include "libavutil/log.h" #include "libavutil/pixdesc.h" #include "libavutil/pixfmt.h" +#include "libavutil/time.h" #include "libavutil/timestamp.h" #include "libavcodec/avcodec.h" @@ -475,6 +476,13 @@ static int packet_decode(InputStream *ist, AVPacket *pkt, AVFrame *frame) pkt->dts = AV_NOPTS_VALUE; } + if (pkt) { + FrameData *fd = packet_data(pkt); + if (!fd) + return AVERROR(ENOMEM); + fd->wallclock[LATENCY_PROBE_DEC_PRE] = av_gettime_relative(); + } + ret = avcodec_send_packet(dec, pkt); if (ret < 0 && !(ret == AVERROR_EOF && !pkt)) { // In particular, we don't expect AVERROR(EAGAIN), because we read all @@ -528,8 +536,6 @@ static int packet_decode(InputStream *ist, AVPacket *pkt, AVFrame *frame) return AVERROR_INVALIDDATA; } - - av_assert0(!frame->opaque_ref); fd = frame_data(frame); if (!fd) { av_frame_unref(frame); @@ -540,6 +546,8 @@ static int packet_decode(InputStream *ist, AVPacket *pkt, AVFrame *frame) fd->dec.frame_num = dec->frame_num - 1; fd->bits_per_raw_sample = dec->bits_per_raw_sample; + fd->wallclock[LATENCY_PROBE_DEC_POST] = av_gettime_relative(); + frame->time_base = dec->pkt_timebase; if (dec->codec_type == AVMEDIA_TYPE_AUDIO) { @@ -932,6 +940,8 @@ int dec_open(InputStream *ist, Scheduler *sch, unsigned sch_idx) if (ist->st->disposition & AV_DISPOSITION_ATTACHED_PIC) av_dict_set(&ist->decoder_opts, "threads", "1", 0); + av_dict_set(&ist->decoder_opts, "flags", "+copy_opaque", AV_DICT_MULTIKEY); + ret = hw_device_setup_for_decode(ist); if (ret < 0) { av_log(ist, AV_LOG_ERROR, diff --git a/fftools/ffmpeg_demux.c b/fftools/ffmpeg_demux.c index 392b447338..d2466e885d 100644 --- a/fftools/ffmpeg_demux.c +++ b/fftools/ffmpeg_demux.c @@ -271,7 +271,7 @@ static void ts_discontinuity_process(Demuxer *d, InputStream *ist, ts_discontinuity_detect(d, ist, pkt); } -static int ist_dts_update(DemuxStream *ds, AVPacket *pkt) +static int ist_dts_update(DemuxStream *ds, AVPacket *pkt, FrameData *fd) { InputStream *ist = &ds->ist; const AVCodecParameters *par = ist->par; @@ -326,21 +326,12 @@ static int ist_dts_update(DemuxStream *ds, AVPacket *pkt) break; } - av_assert0(!pkt->opaque_ref); - if (ds->streamcopy_needed) { - FrameData *fd; - - fd = packet_data(pkt); - if (!fd) - return AVERROR(ENOMEM); - - fd->dts_est = ds->dts; - } + fd->dts_est = ds->dts; return 0; } -static int ts_fixup(Demuxer *d, AVPacket *pkt) +static int ts_fixup(Demuxer *d, AVPacket *pkt, FrameData *fd) { InputFile *ifile = &d->f; InputStream *ist = ifile->streams[pkt->stream_index]; @@ -424,7 +415,7 @@ static int ts_fixup(Demuxer *d, AVPacket *pkt) ts_discontinuity_process(d, ist, pkt); // update estimated/predicted dts - ret = ist_dts_update(ds, pkt); + ret = ist_dts_update(ds, pkt, fd); if (ret < 0) return ret; @@ -436,9 +427,14 @@ static int input_packet_process(Demuxer *d, AVPacket *pkt, unsigned *send_flags) InputFile *f = &d->f; InputStream *ist = f->streams[pkt->stream_index]; DemuxStream *ds = ds_from_ist(ist); + FrameData *fd; int ret = 0; - ret = ts_fixup(d, pkt); + fd = packet_data(pkt); + if (!fd) + return AVERROR(ENOMEM); + + ret = ts_fixup(d, pkt, fd); if (ret < 0) return ret; @@ -455,6 +451,8 @@ static int input_packet_process(Demuxer *d, AVPacket *pkt, unsigned *send_flags) ds->data_size += pkt->size; ds->nb_packets++; + fd->wallclock[LATENCY_PROBE_DEMUX] = av_gettime_relative(); + if (debug_ts) { av_log(NULL, AV_LOG_INFO, "demuxer+ffmpeg -> ist_index:%d:%d type:%s pkt_pts:%s pkt_pts_time:%s pkt_dts:%s pkt_dts_time:%s duration:%s duration_time:%s off:%s off_time:%s\n", f->index, pkt->stream_index, diff --git a/fftools/ffmpeg_enc.c b/fftools/ffmpeg_enc.c index 320df2dee7..43e8b077eb 100644 --- a/fftools/ffmpeg_enc.c +++ b/fftools/ffmpeg_enc.c @@ -34,6 +34,7 @@ #include "libavutil/log.h" #include "libavutil/pixdesc.h" #include "libavutil/rational.h" +#include "libavutil/time.h" #include "libavutil/timestamp.h" #include "libavcodec/avcodec.h" @@ -615,6 +616,14 @@ static int encode_frame(OutputFile *of, OutputStream *ost, AVFrame *frame, int ret; if (frame) { + FrameData *fd = frame_data(frame); + + fd = frame_data(frame); + if (!fd) + return AVERROR(ENOMEM); + + fd->wallclock[LATENCY_PROBE_ENC_PRE] = av_gettime_relative(); + if (ost->enc_stats_pre.io) enc_stats_write(ost, &ost->enc_stats_pre, frame, NULL, ost->frames_encoded); @@ -644,6 +653,8 @@ static int encode_frame(OutputFile *of, OutputStream *ost, AVFrame *frame, } while (1) { + FrameData *fd; + av_packet_unref(pkt); ret = avcodec_receive_packet(enc, pkt); @@ -665,6 +676,11 @@ static int encode_frame(OutputFile *of, OutputStream *ost, AVFrame *frame, return ret; } + fd = packet_data(pkt); + if (!fd) + return AVERROR(ENOMEM); + fd->wallclock[LATENCY_PROBE_ENC_POST] = av_gettime_relative(); + if (enc->codec_type == AVMEDIA_TYPE_VIDEO) { ret = update_video_stats(ost, pkt, !!vstats_filename); if (ret < 0) diff --git a/fftools/ffmpeg_filter.c b/fftools/ffmpeg_filter.c index ada235b084..21f823be12 100644 --- a/fftools/ffmpeg_filter.c +++ b/fftools/ffmpeg_filter.c @@ -36,6 +36,7 @@ #include "libavutil/pixfmt.h" #include "libavutil/imgutils.h" #include "libavutil/samplefmt.h" +#include "libavutil/time.h" #include "libavutil/timestamp.h" // FIXME private header, used for mid_pred() @@ -2364,6 +2365,8 @@ static int fg_output_step(OutputFilterPriv *ofp, FilterGraphThread *fgt, return AVERROR(ENOMEM); } + fd->wallclock[LATENCY_PROBE_FILTER_POST] = av_gettime_relative(); + // only use bits_per_raw_sample passed through from the decoder // if the filtergraph did not touch the frame data if (!fgp->is_meta) @@ -2576,6 +2579,7 @@ static int send_frame(FilterGraph *fg, FilterGraphThread *fgt, InputFilter *ifilter, AVFrame *frame) { InputFilterPriv *ifp = ifp_from_ifilter(ifilter); + FrameData *fd; AVFrameSideData *sd; int need_reinit, ret; @@ -2651,6 +2655,11 @@ static int send_frame(FilterGraph *fg, FilterGraphThread *fgt, ) #endif + fd = frame_data(frame); + if (!fd) + return AVERROR(ENOMEM); + fd->wallclock[LATENCY_PROBE_FILTER_PRE] = av_gettime_relative(); + ret = av_buffersrc_add_frame_flags(ifp->filter, frame, AV_BUFFERSRC_FLAG_PUSH); if (ret < 0) { -- 2.42.0 _______________________________________________ ffmpeg-devel mailing list ffmpeg-devel@ffmpeg.org https://ffmpeg.org/mailman/listinfo/ffmpeg-devel To unsubscribe, visit link above, or email ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".