From 8288d2bd36ffed29140d46c42b6f5515a9058836 Mon Sep 17 00:00:00 2001 From: Elias Carotti Date: Wed, 19 Apr 2023 11:49:39 +0200 Subject: [PATCH] Add support for libx264's MB_INFO libx264's x264_image_properties_t, which is passed to the encoding function, contains a field to pass down information on the portions of the frame which changed with respect to the previous one (used for prediction) to mark unchanged macroblocks P_SKIP. --- libavcodec/libx264.c | 94 +++++++++++++++++++++++++++++++++++++ libavutil/Makefile | 4 ++ libavutil/frame.h | 10 ++++ libavutil/video_hint_info.c | 89 +++++++++++++++++++++++++++++++++++ libavutil/video_hint_info.h | 87 ++++++++++++++++++++++++++++++++++ 5 files changed, 284 insertions(+) create mode 100644 libavutil/video_hint_info.c create mode 100644 libavutil/video_hint_info.h diff --git a/libavcodec/libx264.c b/libavcodec/libx264.c index 5736f1efa7..2cf7755eec 100644 --- a/libavcodec/libx264.c +++ b/libavcodec/libx264.c @@ -30,6 +30,7 @@ #include "libavutil/stereo3d.h" #include "libavutil/time.h" #include "libavutil/intreadwrite.h" +#include "libavutil/video_hint_info.h" #include "avcodec.h" #include "codec_internal.h" #include "encode.h" @@ -48,6 +49,13 @@ // from x264.h, for quant_offsets, Macroblocks are 16x16 // blocks of pixels (with respect to the luma plane) #define MB_SIZE 16 +#define MB_LSIZE 4 +#define MB_FLOOR(x) ((x) >> (MB_LSIZE)) +#define MB_CEIL(x) MB_FLOOR((x) + (MB_SIZE - 1)) + +typedef void (*AVMBInfoComputeCoords)(const AVVideoRect *rect, + int *min_x, int *max_x, + int *min_y, int *max_y); typedef struct X264Opaque { #if FF_API_REORDERED_OPAQUE @@ -123,6 +131,8 @@ typedef struct X264Context { * encounter a frame with ROI side data. */ int roi_warned; + + int mb_info; } X264Context; static void X264_log(void *p, int level, const char *fmt, va_list args) @@ -295,6 +305,7 @@ static void free_picture(x264_picture_t *pic) av_free(pic->extra_sei.payloads[i].payload); av_freep(&pic->extra_sei.payloads); av_freep(&pic->prop.quant_offsets); + av_freep(&pic->prop.mb_info); pic->extra_sei.num_payloads = 0; } @@ -320,6 +331,73 @@ static enum AVPixelFormat csp_to_pixfmt(int csp) return AV_PIX_FMT_NONE; } +static void mbinfo_compute_changed_coords(const AVVideoRect *rect, + int *min_x, + int *max_x, + int *min_y, + int *max_y) +{ + *min_y = MB_FLOOR(rect->y); + *max_y = MB_CEIL(rect->y + rect->height); + *min_x = MB_FLOOR(rect->x); + *max_x = MB_CEIL(rect->x + rect->width); +} + +static void mbinfo_compute_constant_coords(const AVVideoRect *rect, + int *min_x, + int *max_x, + int *min_y, + int *max_y) +{ + *min_y = MB_CEIL(rect->y); + *max_y = MB_FLOOR(rect->y + rect->height); + *min_x = MB_CEIL(rect->x); + *max_x = MB_FLOOR(rect->x + rect->width); +} + +static int setup_mb_info(AVCodecContext *ctx, x264_picture_t *pic, + const AVFrame *frame, + const AVVideoHint *info) +{ + int mb_width = (frame->width + MB_SIZE - 1) / MB_SIZE; + int mb_height = (frame->height + MB_SIZE - 1) / MB_SIZE; + int mbinfo_filler; + int mbinfo_marker; + AVMBInfoComputeCoords compute_coords_fn; + + const AVVideoRect *mbinfo_rects; + int nb_rects; + uint8_t *mbinfo; + + mbinfo_rects = (const AVVideoRect *)av_video_hint_rects(info); + nb_rects = info->nb_rects; + + mbinfo = av_calloc(mb_width * mb_height, sizeof(*mbinfo)); + if (!mbinfo) + return AVERROR(ENOMEM); + + mbinfo_filler = (info->type == AV_VIDEO_HINT_CHANGED) ? X264_MBINFO_CONSTANT : 0; + mbinfo_marker = (info->type == AV_VIDEO_HINT_CHANGED) ? 0 : X264_MBINFO_CONSTANT; + compute_coords_fn = (info->type == AV_VIDEO_HINT_CHANGED) ? mbinfo_compute_changed_coords : mbinfo_compute_constant_coords; + + memset(mbinfo, mbinfo_filler, sizeof(*mbinfo) * mb_width * mb_height); + for (int i = 0; i < nb_rects; i++) { + int min_x, max_x, min_y, max_y; + + (*compute_coords_fn)(mbinfo_rects, &min_x, &max_x, &min_y, &max_y); + for (int mb_y = min_y; mb_y < max_y; ++mb_y) { + memset(mbinfo + mb_y * mb_width + min_x, mbinfo_marker, max_x - min_x); + } + + mbinfo_rects++; + } + + pic->prop.mb_info = mbinfo; + pic->prop.mb_info_free = av_free; + + return 0; +} + static int setup_roi(AVCodecContext *ctx, x264_picture_t *pic, int bit_depth, const AVFrame *frame, const uint8_t *data, size_t size) { @@ -404,6 +482,7 @@ static int setup_frame(AVCodecContext *ctx, const AVFrame *frame, int64_t wallclock = 0; int bit_depth, ret; AVFrameSideData *sd; + AVFrameSideData *mbinfo_sd; *ppic = NULL; if (!frame) @@ -499,6 +578,17 @@ FF_ENABLE_DEPRECATION_WARNINGS goto fail; } + mbinfo_sd = av_frame_get_side_data(frame, AV_FRAME_DATA_VIDEO_HINT); + if (mbinfo_sd) { + int ret = setup_mb_info(ctx, pic, frame, (const AVVideoHint *)mbinfo_sd->data); + if (ret < 0) { + /* No need to fail here, this is not fatal. We just proceed with no + * mb_info and log a message */ + + av_log(ctx, AV_LOG_WARNING, "mb_info setup failure\n"); + } + } + if (x4->udu_sei) { for (int j = 0; j < frame->nb_side_data; j++) { AVFrameSideData *side_data = frame->side_data[j]; @@ -1102,6 +1192,9 @@ FF_ENABLE_DEPRECATION_WARNINGS } } + x4->params.analyse.b_mb_info = x4->mb_info; + x4->params.analyse.b_fast_pskip = 1; + // update AVCodecContext with x264 parameters avctx->has_b_frames = x4->params.i_bframe ? x4->params.i_bframe_pyramid ? 2 : 1 : 0; @@ -1311,6 +1404,7 @@ static const AVOption options[] = { { "noise_reduction", "Noise reduction", OFFSET(noise_reduction), AV_OPT_TYPE_INT, { .i64 = -1 }, INT_MIN, INT_MAX, VE }, { "udu_sei", "Use user data unregistered SEI if available", OFFSET(udu_sei), AV_OPT_TYPE_BOOL, { .i64 = 0 }, 0, 1, VE }, { "x264-params", "Override the x264 configuration using a :-separated list of key=value parameters", OFFSET(x264_params), AV_OPT_TYPE_DICT, { 0 }, 0, 0, VE }, + { "mb_info", "Set mb_info data through AVSideData, only useful when used from the API", OFFSET(mb_info), AV_OPT_TYPE_BOOL, { .i64 = 0 }, 0, 1, VE }, { NULL }, }; diff --git a/libavutil/Makefile b/libavutil/Makefile index dc9012f9a8..1dad526946 100644 --- a/libavutil/Makefile +++ b/libavutil/Makefile @@ -91,6 +91,7 @@ HEADERS = adler32.h \ tea.h \ tx.h \ film_grain_params.h \ + video_hint_info.h ARCH_HEADERS = bswap.h \ intmath.h \ @@ -188,6 +189,7 @@ OBJS-$(CONFIG_CUDA) += hwcontext_cuda.o OBJS-$(CONFIG_D3D11VA) += hwcontext_d3d11va.o OBJS-$(CONFIG_DXVA2) += hwcontext_dxva2.o OBJS-$(CONFIG_LIBDRM) += hwcontext_drm.o +OBJS-$(CONFIG_LIBX264) += video_hint_info.o OBJS-$(CONFIG_MACOS_KPERF) += macos_kperf.o OBJS-$(CONFIG_MEDIACODEC) += hwcontext_mediacodec.o OBJS-$(CONFIG_OPENCL) += hwcontext_opencl.o @@ -219,6 +221,8 @@ SKIPHEADERS-$(CONFIG_VULKAN) += hwcontext_vulkan.h vulkan.h \ vulkan_functions.h \ vulkan_loader.h +SKIPHEADERS-$(CONFIG_LIBX264) += video_hint_info.h + TESTPROGS = adler32 \ aes \ aes_ctr \ diff --git a/libavutil/frame.h b/libavutil/frame.h index a491315f25..0e765e5499 100644 --- a/libavutil/frame.h +++ b/libavutil/frame.h @@ -214,6 +214,16 @@ enum AVFrameSideDataType { * Ambient viewing environment metadata, as defined by H.274. */ AV_FRAME_DATA_AMBIENT_VIEWING_ENVIRONMENT, + + /** + * Provide macro block encoder-specific hinting information for the encoder + * processing. It can be used to pass information about which macroblock + * can be skipped because it hasn't changed from the corresponding one in + * the previous frame. This is useful for applications which know in + * advance this information to speed up real-time encoding. Currently only + * used by libx264. + */ + AV_FRAME_DATA_VIDEO_HINT, }; enum AVActiveFormatDescription { diff --git a/libavutil/video_hint_info.c b/libavutil/video_hint_info.c new file mode 100644 index 0000000000..c920bd6232 --- /dev/null +++ b/libavutil/video_hint_info.c @@ -0,0 +1,89 @@ +/* + * Copyright 2023 Elias Carotti + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include + +#include "avstring.h" +#include "frame.h" +#include "macros.h" +#include "mem.h" +#include "video_hint_info.h" + +AVVideoHint *av_video_hint_alloc(AVVideoRect *rects, + size_t nb_rects, + AVVideoHintType type, + size_t* out_size) +{ + struct TestStruct { + AVVideoHint hint; + AVVideoRect rect; + }; + const size_t blocks_offset = offsetof(struct TestStruct, rect); + size_t size = blocks_offset; + AVVideoHint *hint; + + *out_size = 0; + if (nb_rects > (SIZE_MAX - size) / sizeof(AVVideoRect)) + return NULL; + size += sizeof(AVVideoRect) * nb_rects; + + hint = av_mallocz(size); + if (!hint) + return NULL; + + hint->type = type; + hint->nb_rects = nb_rects; + hint->blocks_offset = blocks_offset; + + /* Just copies the rects over the newly allocated buffer */ + memcpy((uint8_t *)hint + blocks_offset, rects, sizeof(AVVideoRect) * nb_rects); + + *out_size = size; + + return hint; +} + +AVVideoHint *av_video_hint_create_side_data(AVFrame *frame, + AVVideoRect *rects, + size_t num_rects, + AVVideoHintType type) +{ + AVVideoHint *hint; + AVBufferRef *buf; + size_t size = 0; + + hint = av_video_hint_alloc(rects, num_rects, type, &size); + if (!hint) + return NULL; + + buf = av_buffer_create((uint8_t *)hint, size, NULL, NULL, 0); + if (!buf) { + av_freep(&hint); + return NULL; + } + + if (!av_frame_new_side_data_from_buf(frame, AV_FRAME_DATA_VIDEO_HINT, buf)) { + av_buffer_unref(&buf); + return NULL; + } + + return hint; +} + diff --git a/libavutil/video_hint_info.h b/libavutil/video_hint_info.h new file mode 100644 index 0000000000..2844398d18 --- /dev/null +++ b/libavutil/video_hint_info.h @@ -0,0 +1,87 @@ +/** + * Copyright 2023 Elias Carotti + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifndef AVUTIL_VIDEO_HINT_INFO_H +#define AVUTIL_VIDEO_HINT_INFO_H + +#include +#include +#include "libavutil/avassert.h" +#include "libavutil/frame.h" + +typedef struct AVVideoRect { + uint32_t x, y; + uint32_t width, height; +} AVVideoRect; + +typedef enum AVVideoHintType { + /* blocks delimit the constant areas (unchanged), default is changed */ + AV_VIDEO_HINT_CONSTANT, + + /* blocks delimit the constant areas (changed), default is not changed */ + AV_VIDEO_HINT_CHANGED, +} AVVideoHintType; + +typedef struct AVVideoHint { + /** + * Number of blocks in the array. + * + * May be 0, in which case no per-block information is present. In this case + * the values of blocks_offset / block_size are unspecified and should not + * be accessed. + */ + int nb_rects; + + /** + * Offset in bytes from the beginning of this structure at which the array + * of blocks starts. + */ + size_t blocks_offset; + + AVVideoHintType type; +} AVVideoHint; + +static av_always_inline AVVideoRect* +av_video_hint_rects(const AVVideoHint *par) +{ + return (AVVideoRect *)((uint8_t *)par + par->blocks_offset); +} + +/** + * Allocate memory for a vector of AVVideoRect in the given AVFrame + * {@code frame} as AVFrameSideData of type AV_FRAME_DATA_VIDEO_HINT_INFO. + * The side data contains a list of rectangles for the portions of the frame + * which changed from the last encoded one (and the remainder are assumed to be + * changed), or, alternately (depending on the type parameter) the unchanged + * ones (and the remanining ones are those which changed). + * Macroblocks will thus be hinted either to be P_SKIP-ped or go through the + * regular encoding procedure. + */ +AVVideoHint *av_video_hint_create_side_data(AVFrame *frame, + AVVideoRect *rects, + size_t num_rects, + AVVideoHintType type); + +AVVideoHint *av_video_hint_alloc(AVVideoRect *rects, + size_t nb_rects, + AVVideoHintType type, + size_t *out_size); + +#endif /* AVUTIL_VIDEO_HINT_INFO_H */ -- 2.34.1