Re: [FFmpeg-devel] [PATCH] Optimization: support for libx264's mb_info

[Stefano Sabatini <stefasab@gmail.com>]

On date Monday 2023-05-29 17:56:55 +0000, Carotti, Elias wrote:
[...]
> From 7cb97ee977197e310a932b2d7a53bf5c6e99990e Mon Sep 17 00:00:00 2001
> From: Elias Carotti <eliascrt@amazon.it>
> Date: Wed, 19 Apr 2023 11:49:39 +0200
> Subject: [PATCH] Add support for libx264's MB_INFO
> 
> libx264's x264_image_properties_t, which is passed to the encoding function,
> contains a field to pass down information on the portions of the frame which
> changed with respect to the previous one (used for prediction) to mark
> unchanged macroblocks P_SKIP.
> ---
>  libavcodec/libx264.c        | 81 +++++++++++++++++++++++++++++++++
>  libavutil/Makefile          |  4 ++
>  libavutil/frame.h           | 10 +++++
>  libavutil/video_hint_info.c | 89 +++++++++++++++++++++++++++++++++++++
>  libavutil/video_hint_info.h | 87 ++++++++++++++++++++++++++++++++++++
>  5 files changed, 271 insertions(+)
>  create mode 100644 libavutil/video_hint_info.c
>  create mode 100644 libavutil/video_hint_info.h
> 
> diff --git a/libavcodec/libx264.c b/libavcodec/libx264.c
> index 5736f1efa7..32fa80d0a1 100644
> --- a/libavcodec/libx264.c
> +++ b/libavcodec/libx264.c
> @@ -30,6 +30,7 @@
>  #include "libavutil/stereo3d.h"
>  #include "libavutil/time.h"
>  #include "libavutil/intreadwrite.h"
> +#include "libavutil/video_hint_info.h"
>  #include "avcodec.h"
>  #include "codec_internal.h"
>  #include "encode.h"
> @@ -48,6 +49,9 @@
>  // from x264.h, for quant_offsets, Macroblocks are 16x16
>  // blocks of pixels (with respect to the luma plane)
>  #define MB_SIZE 16
> +#define MB_LSIZE 4
> +#define MB_FLOOR(x)      ((x) >> (MB_LSIZE))
> +#define MB_CEIL(x)       MB_FLOOR((x) + (MB_SIZE - 1))
>  
>  typedef struct X264Opaque {
>  #if FF_API_REORDERED_OPAQUE
> @@ -123,6 +127,8 @@ typedef struct X264Context {
>       * encounter a frame with ROI side data.
>       */
>      int roi_warned;
> +
> +    int mb_info;
>  } X264Context;
>  
>  static void X264_log(void *p, int level, const char *fmt, va_list args)
> @@ -295,6 +301,7 @@ static void free_picture(x264_picture_t *pic)
>          av_free(pic->extra_sei.payloads[i].payload);
>      av_freep(&pic->extra_sei.payloads);
>      av_freep(&pic->prop.quant_offsets);
> +    av_freep(&pic->prop.mb_info);
>      pic->extra_sei.num_payloads = 0;
>  }
>  
> @@ -320,6 +327,64 @@ static enum AVPixelFormat csp_to_pixfmt(int csp)
>      return AV_PIX_FMT_NONE;
>  }
>  
> +static int setup_mb_info(AVCodecContext *ctx, x264_picture_t *pic,
> +                         const AVFrame *frame,
> +                         const AVVideoHint *info)
> +{

> +    int mb_width = (frame->width + MB_SIZE - 1) / MB_SIZE;
> +    int mb_height = (frame->height + MB_SIZE - 1) / MB_SIZE;


> +
> +    const AVVideoRect *mbinfo_rects;
> +    int nb_rects;
> +    uint8_t *mbinfo;
> +
> +    mbinfo_rects = (const AVVideoRect *)av_video_hint_rects(info);
> +    nb_rects = info->nb_rects;
> +
> +    mbinfo = av_calloc(mb_width * mb_height, sizeof(*mbinfo));
> +    if (!mbinfo)
> +        return AVERROR(ENOMEM);
> +

> +    if (info->type == AV_VIDEO_HINT_CHANGED) {
> +        /* Sets the default as constant, i.e. P_SKIP-able, then selectively resets the flag */
> +        memset(mbinfo, X264_MBINFO_CONSTANT, sizeof(*mbinfo) * mb_width * mb_height);
> +
> +        for (int i = 0; i < nb_rects; i++) {
> +            int min_y = MB_FLOOR(mbinfo_rects->y);
> +            int max_y = MB_CEIL(mbinfo_rects->y + mbinfo_rects->height);
> +            int min_x = MB_FLOOR(mbinfo_rects->x);
> +            int max_x = MB_CEIL(mbinfo_rects->x + mbinfo_rects->width);
> +
> +            for (int mb_y = min_y; mb_y < max_y; ++mb_y) {
> +                memset(mbinfo + mb_y * mb_width + min_x, 0, max_x - min_x);
> +            }
> +
> +            mbinfo_rects++;
> +        }
> +    } else {
> +        /* Sets the default as changed, i.e. *not* P_SKIP-able, then selectively sets the flag */
> +        memset(mbinfo, 0, sizeof(*mbinfo) * mb_width * mb_height);
> +
> +        for (int i = 0; i < nb_rects; i++) {
> +            int min_y = MB_CEIL(mbinfo_rects->y);
> +            int max_y = MB_FLOOR(mbinfo_rects->y + mbinfo_rects->height);
> +            int min_x = MB_CEIL(mbinfo_rects->x);
> +            int max_x = MB_FLOOR(mbinfo_rects->x + mbinfo_rects->width);
> +
> +            for (int mb_y = min_y; mb_y < max_y; ++mb_y) {
> +                memset(mbinfo + mb_y * mb_width + min_x, X264_MBINFO_CONSTANT, max_x - min_x);
> +            }
> +
> +            mbinfo_rects++;

you can merge the two loops by adding an inline function to compute
the mins and maxs, and then setting:

mbinfo_filler     = AV_VIDEO_HINT_CHANGED ? X264_MBINFO_CONSTANT : 0;
mbinfo_marker     = AV_VIDEO_HINT_CHANGED ? 0 : X264_MBINFO_CONSTANT;
compute_coords_fn = AV_VIDEO_HINT_CHANGED ? compute_changed_coords : compute_constant_coords;
   
memset(mbinfo, mbinfo_filler, sizeof(*mbinfo) * mb_width * mb_height);
for (int i = 0; i < nb_rects; i++) {
    compute_coords_fn...
    for (int mb_y = min_y; mb_y < max_y; ++mb_y) {
        memset(mbinfo + mb_y * mb_width + min_x, mbinfo_marker, max_x - min_x);
    }

    mbinfo_rects++;
}

> +        }
> +    }
> +
> +    pic->prop.mb_info = mbinfo;
> +    pic->prop.mb_info_free = av_free;
> +
> +    return 0;
> +}
> +
>  static int setup_roi(AVCodecContext *ctx, x264_picture_t *pic, int bit_depth,
>                       const AVFrame *frame, const uint8_t *data, size_t size)
>  {
> @@ -404,6 +469,7 @@ static int setup_frame(AVCodecContext *ctx, const AVFrame *frame,
>      int64_t wallclock = 0;
>      int bit_depth, ret;
>      AVFrameSideData *sd;
> +    AVFrameSideData *mbinfo_sd;
>  
>      *ppic = NULL;
>      if (!frame)
> @@ -499,6 +565,17 @@ FF_ENABLE_DEPRECATION_WARNINGS
>              goto fail;
>      }
>  
> +    mbinfo_sd = av_frame_get_side_data(frame, AV_FRAME_DATA_VIDEO_HINT);
> +    if (mbinfo_sd) {
> +        int ret = setup_mb_info(ctx, pic, frame, (const AVVideoHint *)mbinfo_sd->data);
> +        if (ret < 0) {
> +            /* No need to fail here, this is not fatal. We just proceed with no
> +             * mb_info and log a message */
> +
> +            av_log(ctx, AV_LOG_WARNING, "mb_info setup failure\n");
> +        }
> +    }
> +
>      if (x4->udu_sei) {
>          for (int j = 0; j < frame->nb_side_data; j++) {
>              AVFrameSideData *side_data = frame->side_data[j];
> @@ -1102,6 +1179,9 @@ FF_ENABLE_DEPRECATION_WARNINGS
>          }
>      }
>  
> +    x4->params.analyse.b_mb_info = x4->mb_info;
> +    x4->params.analyse.b_fast_pskip = 1;
> +
>      // update AVCodecContext with x264 parameters
>      avctx->has_b_frames = x4->params.i_bframe ?
>          x4->params.i_bframe_pyramid ? 2 : 1 : 0;
> @@ -1311,6 +1391,7 @@ static const AVOption options[] = {
>      { "noise_reduction", "Noise reduction",                               OFFSET(noise_reduction), AV_OPT_TYPE_INT, { .i64 = -1 }, INT_MIN, INT_MAX, VE },
>      { "udu_sei",      "Use user data unregistered SEI if available",      OFFSET(udu_sei),  AV_OPT_TYPE_BOOL,   { .i64 = 0 }, 0, 1, VE },
>      { "x264-params",  "Override the x264 configuration using a :-separated list of key=value parameters", OFFSET(x264_params), AV_OPT_TYPE_DICT, { 0 }, 0, 0, VE },
> +    { "mb_info",      "Set mb_info data through AVSideData, only useful when used from the API", OFFSET(mb_info), AV_OPT_TYPE_BOOL, { .i64 = 0 }, 0, 1, VE },
>      { NULL },
>  };
>  
> diff --git a/libavutil/Makefile b/libavutil/Makefile
> index dc9012f9a8..bb5ecc3235 100644
> --- a/libavutil/Makefile
> +++ b/libavutil/Makefile
> @@ -91,6 +91,7 @@ HEADERS = adler32.h                                                     \
>            tea.h                                                         \
>            tx.h                                                          \
>            film_grain_params.h                                           \
> +          video_hint_info.h                                                     \

>  
>  ARCH_HEADERS = bswap.h                                                  \
>                 intmath.h                                                \
> @@ -196,6 +197,7 @@ OBJS-$(CONFIG_VAAPI)                    += hwcontext_vaapi.o
>  OBJS-$(CONFIG_VIDEOTOOLBOX)             += hwcontext_videotoolbox.o
>  OBJS-$(CONFIG_VDPAU)                    += hwcontext_vdpau.o
>  OBJS-$(CONFIG_VULKAN)                   += hwcontext_vulkan.o
> +OBJS-$(CONFIG_LIBX264)                  += video_hint_info.o

order

>  
>  OBJS-$(!CONFIG_VULKAN)                  += hwcontext_stub.o
>  
> @@ -219,6 +221,8 @@ SKIPHEADERS-$(CONFIG_VULKAN)           += hwcontext_vulkan.h vulkan.h   \
>                                            vulkan_functions.h            \
>                                            vulkan_loader.h
>  
> +SKIPHEADERS-$(CONFIG_LIBX264)  	       += video_hint_info.h
> +
>  TESTPROGS = adler32                                                     \
>              aes                                                         \
>              aes_ctr                                                     \
> diff --git a/libavutil/frame.h b/libavutil/frame.h
> index a491315f25..0e765e5499 100644
> --- a/libavutil/frame.h
> +++ b/libavutil/frame.h
> @@ -214,6 +214,16 @@ enum AVFrameSideDataType {
>       * Ambient viewing environment metadata, as defined by H.274.
>       */
>      AV_FRAME_DATA_AMBIENT_VIEWING_ENVIRONMENT,
> +
> +    /**
> +     * Provide macro block encoder-specific hinting information for the encoder
> +     * processing.  It can be used to pass information about which macroblock
> +     * can be skipped because it hasn't changed from the corresponding one in
> +     * the previous frame. This is useful for applications which know in
> +     * advance this information to speed up real-time encoding.  Currently only
> +     * used by libx264.
> +     */
> +    AV_FRAME_DATA_VIDEO_HINT,
>  };
>  
>  enum AVActiveFormatDescription {
> diff --git a/libavutil/video_hint_info.c b/libavutil/video_hint_info.c
> new file mode 100644
> index 0000000000..0a763962da
> --- /dev/null
> +++ b/libavutil/video_hint_info.c
> @@ -0,0 +1,89 @@
> +/*
> + * Copyright 2023 Elias Carotti <eliascrt at amazon dot it>
> + *
> + * This file is part of FFmpeg.
> + *
> + * FFmpeg is free software; you can redistribute it and/or
> + * modify it under the terms of the GNU Lesser General Public
> + * License as published by the Free Software Foundation; either
> + * version 2.1 of the License, or (at your option) any later version.
> + *
> + * FFmpeg is distributed in the hope that it will be useful,
> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
> + * Lesser General Public License for more details.
> + *
> + * You should have received a copy of the GNU Lesser General Public
> + * License along with FFmpeg; if not, write to the Free Software
> + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
> + */
> +
> +#include <string.h>
> +
> +#include "avstring.h"
> +#include "frame.h"
> +#include "macros.h"
> +#include "mem.h"
> +#include "video_hint_info.h"
> +
> +AVVideoHint *av_video_hint_alloc(AVVideoRect *rects,

> +                                          size_t nb_rects,
> +                                          VideoHintType type,
> +                                          size_t* out_size)

align to (

> +{
> +    struct TestStruct {
> +        AVVideoHint    p;
> +        AVVideoRect        b;
> +    };

nit: weird align (also use more meaningful names - e.g. hint and rect)

> +    const size_t blocks_offset = offsetof(struct TestStruct, b);
> +    size_t size = blocks_offset;

> +    AVVideoHint *par;

AVVideoHint *hint?

> +
> +    *out_size = 0;

> +    if (nb_rects > (SIZE_MAX - size) / sizeof(AVVideoRect))

why SIZE_MAX - size?

> +        return NULL;

> +    size += sizeof(AVVideoRect) * nb_rects;
> +
> +    par = av_mallocz(size);
> +    if (!par)
> +        return NULL;
> +
> +    par->type          = type;
> +    par->nb_rects      = nb_rects;
> +    par->blocks_offset = blocks_offset;
> +
> +    /* Just copies the rects over the newly allocated buffer */
> +    memcpy((uint8_t *)par + blocks_offset, rects, sizeof(AVVideoRect) * nb_rects);
> +
> +    *out_size = size;
> +
> +    return par;
> +}
> +
> +AVVideoHint *av_video_hint_create_side_data(AVFrame *frame,
> +                                            AVVideoRect *rects,
> +                                            size_t num_rects,
> +                                            VideoHintType type)
> +{

> +    AVVideoHint *par;

use a more meaningful name (I think "par" is a leftover of an older
API)

> +    AVBufferRef *buf;
> +    size_t size = 0;
> +
> +    par = av_video_hint_alloc(rects, num_rects, type, &size);
> +    if (!par)
> +        return NULL;
> +
> +    buf = av_buffer_create((uint8_t *)par, size, NULL, NULL, 0);
> +    if (!buf) {
> +        av_freep(&par);
> +        return NULL;
> +    }
> +
> +    if (!av_frame_new_side_data_from_buf(frame, AV_FRAME_DATA_VIDEO_HINT, buf)) {
> +        av_buffer_unref(&buf);
> +        return NULL;
> +    }
> +
> +    return par;
> +}
> +
> diff --git a/libavutil/video_hint_info.h b/libavutil/video_hint_info.h
> new file mode 100644
> index 0000000000..3b04e0c40e
> --- /dev/null
> +++ b/libavutil/video_hint_info.h
> @@ -0,0 +1,87 @@
> +/**
> + * Copyright 2023 Elias Carotti <eliascrt at amazon dot it>
> + *
> + * This file is part of FFmpeg.
> + *
> + * FFmpeg is free software; you can redistribute it and/or
> + * modify it under the terms of the GNU Lesser General Public
> + * License as published by the Free Software Foundation; either
> + * version 2.1 of the License, or (at your option) any later version.
> + *
> + * FFmpeg is distributed in the hope that it will be useful,
> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
> + * Lesser General Public License for more details.
> + *
> + * You should have received a copy of the GNU Lesser General Public
> + * License along with FFmpeg; if not, write to the Free Software
> + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
> + */
> +
> +#ifndef AVUTIL_VIDEO_HINT_INFO_H
> +#define AVUTIL_VIDEO_HINT_INFO_H
> +
> +#include <stddef.h>
> +#include <stdint.h>
> +#include "libavutil/avassert.h"
> +#include "libavutil/frame.h"
> +
> +typedef struct AVVideoRect {
> +    uint32_t x, y;
> +    uint32_t width, height;
> +} AVVideoRect;
> +

> +typedef enum VideoHintType {
> +    /* blocks delimit the constant areas (unchanged), default is changed */
> +    AV_VIDEO_HINT_CONSTANT,
> +
> +    /* blocks delimit the constant areas (changed), default is not changed */
> +    AV_VIDEO_HINT_CHANGED,
> +} VideoHintType;

missing AV prefix

> +
> +typedef struct AVVideoHint {
> +    /**
> +     * Number of blocks in the array.
> +     *
> +     * May be 0, in which case no per-block information is present. In this case
> +     * the values of blocks_offset / block_size are unspecified and should not
> +     * be accessed.
> +     */
> +    int nb_rects;
> +
> +    /**
> +     * Offset in bytes from the beginning of this structure at which the array
> +     * of blocks starts.
> +     */
> +    size_t blocks_offset;
> +
> +    VideoHintType type;
> +} AVVideoHint;
> +
> +static av_always_inline AVVideoRect*
> +av_video_hint_rects(const AVVideoHint *par)
> +{
> +    return (AVVideoRect *)((uint8_t *)par + par->blocks_offset);
> +}
> +
> +/**
> + * Allocate memory for a vector of AVVideoRect in the given AVFrame
> + * {@code frame} as AVFrameSideData of type AV_FRAME_DATA_VIDEO_HINT_INFO.
> + * The side data contains a list of rectangles for the portions of the frame
> + * which changed from the last encoded one (and the remainder are assumed to be
> + * changed), or, alternately (depending on the type parameter) the unchanged
> + * ones (and the remanining ones are those which changed).
> + * Macroblocks will thus be hinted either to be P_SKIP-ped or go through the
> + * regular encoding procedure.
> + */
> +AVVideoHint *av_video_hint_create_side_data(AVFrame *frame,
> +                                            AVVideoRect *rects,
> +                                            size_t num_rects,
> +                                            VideoHintType type);
> +
> +AVVideoHint *av_video_hint_alloc(AVVideoRect *rects,
> +                                 size_t nb_rects,
> +                                 VideoHintType type,
> +                                 size_t *out_size);
> +

> +#endif /* AVUTIL__VIDEO_HINT_INFO_H */

AVUTIL_VIDEO...

> -- 
> 2.34.1
> 

> _______________________________________________
> ffmpeg-devel mailing list
> ffmpeg-devel@ffmpeg.org
> https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
> 
> To unsubscribe, visit link above, or email
> ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".

_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".