Re: [FFmpeg-devel] [PATCH] avfilter/alimiter:add latency compensation

[Wang Cao <wangcao-at-google.com@ffmpeg.org>]

On Thu, May 5, 2022 at 2:14 PM Wang Cao <wangcao@google.com> wrote:

> Also added 2 FATE tests to verify delay is compenated correctly
>
> Signed-off-by: Wang Cao <wangcao@google.com>
> ---
>  doc/filters.texi            |  5 +++
>  libavfilter/af_alimiter.c   | 90 +++++++++++++++++++++++++++++++++++++
>  tests/fate/filter-audio.mak | 24 ++++++++--
>  3 files changed, 116 insertions(+), 3 deletions(-)
>
> diff --git a/doc/filters.texi b/doc/filters.texi
> index a161754233..75a43edd88 100644
> --- a/doc/filters.texi
> +++ b/doc/filters.texi
> @@ -1978,6 +1978,11 @@ in release time while 1 produces higher release
> times.
>  @item level
>  Auto level output signal. Default is enabled.
>  This normalizes audio back to 0dB if enabled.
> +
> +@item latency
> +Compensate the delay introduced by using the lookahead buffer set with
> attack
> +parameter. Also flush the valid audio data in the lookahead buffer when
> the
> +stream hits EOF
>  @end table
>
>  Depending on picked setting it is recommended to upsample input 2x or 4x
> times
> diff --git a/libavfilter/af_alimiter.c b/libavfilter/af_alimiter.c
> index 133f98f165..01265758d7 100644
> --- a/libavfilter/af_alimiter.c
> +++ b/libavfilter/af_alimiter.c
> @@ -26,6 +26,7 @@
>
>  #include "libavutil/channel_layout.h"
>  #include "libavutil/common.h"
> +#include "libavutil/fifo.h"
>  #include "libavutil/opt.h"
>
>  #include "audio.h"
> @@ -33,6 +34,11 @@
>  #include "formats.h"
>  #include "internal.h"
>
> +typedef struct MetaItem {
> +    int64_t pts;
> +    int nb_samples;
> +} MetaItem;
> +
>  typedef struct AudioLimiterContext {
>      const AVClass *class;
>
> @@ -55,6 +61,14 @@ typedef struct AudioLimiterContext {
>      int *nextpos;
>      double *nextdelta;
>
> +    int in_trim;
> +    int out_pad;
> +    int64_t next_in_pts;
> +    int64_t next_out_pts;
> +    int latency;
> +
> +    AVFifo *fifo;
> +
>      double delta;
>      int nextiter;
>      int nextlen;
> @@ -73,6 +87,7 @@ static const AVOption alimiter_options[] = {
>      { "asc",       "enable asc",       OFFSET(auto_release),
> AV_OPT_TYPE_BOOL,   {.i64=0},      0,    1, AF },
>      { "asc_level", "set asc level",    OFFSET(asc_coeff),
> AV_OPT_TYPE_DOUBLE, {.dbl=0.5},    0,    1, AF },
>      { "level",     "auto level",       OFFSET(auto_level),
>  AV_OPT_TYPE_BOOL,   {.i64=1},      0,    1, AF },
> +    { "latency",   "compensate delay", OFFSET(latency),
> AV_OPT_TYPE_BOOL,   {.i64=0},      0,    1, AF },
>      { NULL }
>  };
>
> @@ -129,6 +144,11 @@ static int filter_frame(AVFilterLink *inlink, AVFrame
> *in)
>      AVFrame *out;
>      double *buf;
>      int n, c, i;
> +    int new_out_samples;
> +    int64_t out_duration;
> +    int64_t in_duration;
> +    int64_t in_pts;
> +    MetaItem meta;
>
>      if (av_frame_is_writable(in)) {
>          out = in;
> @@ -269,12 +289,69 @@ static int filter_frame(AVFilterLink *inlink,
> AVFrame *in)
>          dst += channels;
>      }
>
> +    in_duration = av_rescale_q(in->nb_samples,  inlink->time_base,
> av_make_q(1,  in->sample_rate));
> +    in_pts = in->pts;
> +    meta = (MetaItem){ in->pts, in->nb_samples };
> +    av_fifo_write(s->fifo, &meta, 1);
>      if (in != out)
>          av_frame_free(&in);
>
> +    new_out_samples = out->nb_samples;
> +    if (s->in_trim > 0) {
> +        int trim = FFMIN(new_out_samples, s->in_trim);
> +        new_out_samples -= trim;
> +        s->in_trim -= trim;
> +    }
> +
> +    if (new_out_samples <= 0) {
> +        av_frame_free(&out);
> +        return 0;
> +    } else if (new_out_samples < out->nb_samples) {
> +        int offset = out->nb_samples - new_out_samples;
> +        memmove(out->extended_data[0], out->extended_data[0] +
> sizeof(double) * offset * out->ch_layout.nb_channels,
> +                sizeof(double) * new_out_samples *
> out->ch_layout.nb_channels);
> +        out->nb_samples = new_out_samples;
> +        s->in_trim = 0;
> +    }
> +
> +    av_fifo_read(s->fifo, &meta, 1);
> +
> +    out_duration = av_rescale_q(out->nb_samples, inlink->time_base,
> av_make_q(1, out->sample_rate));
> +    in_duration  = av_rescale_q(meta.nb_samples, inlink->time_base,
> av_make_q(1, out->sample_rate));
> +    in_pts       = meta.pts;
> +
> +    if (s->next_out_pts != AV_NOPTS_VALUE && out->pts != s->next_out_pts
> &&
> +        s->next_in_pts  != AV_NOPTS_VALUE && in_pts   == s->next_in_pts) {
> +        out->pts = s->next_out_pts;
> +    } else {
> +        out->pts = in_pts;
> +    }
> +    s->next_in_pts  = in_pts   + in_duration;
> +    s->next_out_pts = out->pts + out_duration;
> +
>      return ff_filter_frame(outlink, out);
>  }
>
> +static int request_frame(AVFilterLink* outlink)
> +{
> +    AVFilterContext *ctx = outlink->src;
> +    AudioLimiterContext *s = (AudioLimiterContext*)ctx->priv;
> +    int ret;
> +
> +    ret = ff_request_frame(ctx->inputs[0]);
> +
> +    if (ret == AVERROR_EOF && s->out_pad > 0) {
> +        AVFrame *frame = ff_get_audio_buffer(outlink, FFMIN(1024,
> s->out_pad));
> +        if (!frame)
> +            return AVERROR(ENOMEM);
> +
> +        s->out_pad -= frame->nb_samples;
> +        frame->pts = s->next_in_pts;
> +        return filter_frame(ctx->inputs[0], frame);
> +    }
> +    return ret;
> +}
> +
>  static int config_input(AVFilterLink *inlink)
>  {
>      AVFilterContext *ctx = inlink->dst;
> @@ -294,6 +371,16 @@ static int config_input(AVFilterLink *inlink)
>      memset(s->nextpos, -1, obuffer_size * sizeof(*s->nextpos));
>      s->buffer_size = inlink->sample_rate * s->attack *
> inlink->ch_layout.nb_channels;
>      s->buffer_size -= s->buffer_size % inlink->ch_layout.nb_channels;
> +    if (s->latency) {
> +        s->in_trim = s->out_pad = s->buffer_size /
> inlink->ch_layout.nb_channels - 1;
> +    }
> +    s->next_out_pts = AV_NOPTS_VALUE;
> +    s->next_in_pts  = AV_NOPTS_VALUE;
> +
> +    s->fifo = av_fifo_alloc2(8, sizeof(MetaItem), AV_FIFO_FLAG_AUTO_GROW);
> +    if (!s->fifo) {
> +        return AVERROR(ENOMEM);
> +    }
>
>      if (s->buffer_size <= 0) {
>          av_log(ctx, AV_LOG_ERROR, "Attack is too small.\n");
> @@ -310,6 +397,8 @@ static av_cold void uninit(AVFilterContext *ctx)
>      av_freep(&s->buffer);
>      av_freep(&s->nextdelta);
>      av_freep(&s->nextpos);
> +
> +    av_fifo_freep2(&s->fifo);
>  }
>
>  static const AVFilterPad alimiter_inputs[] = {
> @@ -325,6 +414,7 @@ static const AVFilterPad alimiter_outputs[] = {
>      {
>          .name = "default",
>          .type = AVMEDIA_TYPE_AUDIO,
> +        .request_frame = request_frame,
>      },
>  };
>
> diff --git a/tests/fate/filter-audio.mak b/tests/fate/filter-audio.mak
> index eff32b9f81..e33ffdf37f 100644
> --- a/tests/fate/filter-audio.mak
> +++ b/tests/fate/filter-audio.mak
> @@ -63,11 +63,29 @@ fate-filter-agate: tests/data/asynth-44100-2.wav
>  fate-filter-agate: SRC = $(TARGET_PATH)/tests/data/asynth-44100-2.wav
>  fate-filter-agate: CMD = framecrc -i $(SRC) -af
> aresample,agate=level_in=10:range=0:threshold=1:ratio=1:attack=1:knee=1:makeup=4,aresample
>
> -FATE_AFILTER-$(call FILTERDEMDECENCMUX, AFADE, WAV, PCM_S16LE, PCM_S16LE,
> WAV) += fate-filter-alimiter
> -fate-filter-alimiter: tests/data/asynth-44100-2.wav
> -fate-filter-alimiter: SRC = $(TARGET_PATH)/tests/data/asynth-44100-2.wav
> +tests/data/filter-alimiter-passthrough: TAG = GEN
> +tests/data/filter-alimiter-passthrough: ffmpeg$(PROGSSUF)$(EXESUF) |
> tests/data
> +       $(M)$(TARGET_EXEC) $(TARGET_PATH)/$< -nostdin \
> +       -i $(TARGET_PATH)/tests/data/asynth-44100-2.wav -af aresample -f
> crc $(TARGET_PATH)/$@ -y 2>/dev/null
> +
> +FATE_ALIMITER += fate-filter-alimiter-passthrough-default-attack
> +fate-filter-alimiter-passthrough-default-attack:
> tests/data/filter-alimiter-passthrough
> +fate-filter-alimiter-passthrough-default-attack: REF =
> $(TARGET_PATH)/tests/data/filter-alimiter-passthrough
> +fate-filter-alimiter-passthrough-default-attack: CMD = crc -i $(SRC) -af
> aresample,alimiter=level_in=1:level_out=1:limit=1:level=0:latency=1,aresample
> +
> +FATE_ALIMITER += fate-filter-alimiter-passthrough-large-attack
> +fate-filter-alimiter-passthrough-large-attack:
> tests/data/filter-alimiter-passthrough
> +fate-filter-alimiter-passthrough-large-attack: REF =
> $(TARGET_PATH)/tests/data/filter-alimiter-passthrough
> +fate-filter-alimiter-passthrough-large-attack: CMD = crc -i $(SRC) -af
> aresample,alimiter=level_in=1:level_out=1:limit=1:level=0:latency=1:attack=80,aresample
> +
> +FATE_ALIMITER += fate-filter-alimiter
>  fate-filter-alimiter: CMD = framecrc -i $(SRC) -af
> aresample,alimiter=level_in=1:level_out=2:limit=0.2,aresample
>
> +$(FATE_ALIMITER): tests/data/asynth-44100-2.wav
> +$(FATE_ALIMITER): SRC = $(TARGET_PATH)/tests/data/asynth-44100-2.wav
> +
> +FATE_AFILTER-$(call FILTERDEMDECENCMUX, ATRIM, WAV, PCM_S16LE, PCM_S16LE,
> WAV) += $(FATE_ALIMITER)
> +
>  FATE_AFILTER-$(call FILTERDEMDECENCMUX, AMERGE, WAV, PCM_S16LE,
> PCM_S16LE, WAV) += fate-filter-amerge
>  fate-filter-amerge: tests/data/asynth-44100-1.wav
>  fate-filter-amerge: SRC = $(TARGET_PATH)/tests/data/asynth-44100-1.wav
> --
> 2.36.0.512.ge40c2bad7a-goog
>
> Hi folks, I don't know how FATE failed on the server but this specific
"make fate-filter-alimiter" and
"make tests/data/filter-alimiter-passthrough" passed on my local machine.
The code is basically referenced from af_ladspa. Can you advice on this
implementation? Thanks!
-- 

Wang Cao |  Software Engineer |  wangcao@google.com |  650-203-7807
_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".