From d4b00df7b07447fccd7f5b5c2005fcdb7383717d Mon Sep 17 00:00:00 2001 From: Elias Carotti Date: Tue, 20 Jun 2023 19:28:04 +0200 Subject: [PATCH 1/2] lavu: add AVVideoHint API Add side data type to provide hint to the video encoders about unchanged portions of each frame. --- doc/APIchanges | 3 ++ libavutil/Makefile | 4 ++ libavutil/frame.h | 10 +++++ libavutil/version.h | 4 +- libavutil/video_hint.c | 89 ++++++++++++++++++++++++++++++++++++++++++ libavutil/video_hint.h | 87 +++++++++++++++++++++++++++++++++++++++++ 6 files changed, 195 insertions(+), 2 deletions(-) create mode 100644 libavutil/video_hint.c create mode 100644 libavutil/video_hint.h diff --git a/doc/APIchanges b/doc/APIchanges index f040211f7d..bfe04556d2 100644 --- a/doc/APIchanges +++ b/doc/APIchanges @@ -2,6 +2,9 @@ The last version increases of all libraries were on 2023-02-09 API changes, most recent first: +2023-06-21 - xxxxxxxxxx - lavu 58.14.100 - video_hint.h + Add AVVideoHint API. + 2023-05-29 - xxxxxxxxxx - lavc 60.16.100 - avcodec.h codec_id.h Add AV_CODEC_ID_EVC, FF_PROFILE_EVC_BASELINE, and FF_PROFILE_EVC_MAIN. diff --git a/libavutil/Makefile b/libavutil/Makefile index bd9c6f9e32..a0b43faa9f 100644 --- a/libavutil/Makefile +++ b/libavutil/Makefile @@ -91,6 +91,7 @@ HEADERS = adler32.h \ tea.h \ tx.h \ film_grain_params.h \ + video_hint.h ARCH_HEADERS = bswap.h \ intmath.h \ @@ -188,6 +189,7 @@ OBJS-$(CONFIG_CUDA) += hwcontext_cuda.o OBJS-$(CONFIG_D3D11VA) += hwcontext_d3d11va.o OBJS-$(CONFIG_DXVA2) += hwcontext_dxva2.o OBJS-$(CONFIG_LIBDRM) += hwcontext_drm.o +OBJS-$(CONFIG_LIBX264) += video_hint.o OBJS-$(CONFIG_MACOS_KPERF) += macos_kperf.o OBJS-$(CONFIG_MEDIACODEC) += hwcontext_mediacodec.o OBJS-$(CONFIG_OPENCL) += hwcontext_opencl.o @@ -219,6 +221,8 @@ SKIPHEADERS-$(CONFIG_VULKAN) += hwcontext_vulkan.h vulkan.h \ vulkan_functions.h \ vulkan_loader.h +SKIPHEADERS-$(CONFIG_LIBX264) += video_hint.h + TESTPROGS = adler32 \ aes \ aes_ctr \ diff --git a/libavutil/frame.h b/libavutil/frame.h index a491315f25..0e765e5499 100644 --- a/libavutil/frame.h +++ b/libavutil/frame.h @@ -214,6 +214,16 @@ enum AVFrameSideDataType { * Ambient viewing environment metadata, as defined by H.274. */ AV_FRAME_DATA_AMBIENT_VIEWING_ENVIRONMENT, + + /** + * Provide macro block encoder-specific hinting information for the encoder + * processing. It can be used to pass information about which macroblock + * can be skipped because it hasn't changed from the corresponding one in + * the previous frame. This is useful for applications which know in + * advance this information to speed up real-time encoding. Currently only + * used by libx264. + */ + AV_FRAME_DATA_VIDEO_HINT, }; enum AVActiveFormatDescription { diff --git a/libavutil/version.h b/libavutil/version.h index 17a6d296a6..24af520e08 100644 --- a/libavutil/version.h +++ b/libavutil/version.h @@ -79,8 +79,8 @@ */ #define LIBAVUTIL_VERSION_MAJOR 58 -#define LIBAVUTIL_VERSION_MINOR 13 -#define LIBAVUTIL_VERSION_MICRO 101 +#define LIBAVUTIL_VERSION_MINOR 14 +#define LIBAVUTIL_VERSION_MICRO 100 #define LIBAVUTIL_VERSION_INT AV_VERSION_INT(LIBAVUTIL_VERSION_MAJOR, \ LIBAVUTIL_VERSION_MINOR, \ diff --git a/libavutil/video_hint.c b/libavutil/video_hint.c new file mode 100644 index 0000000000..a638891d45 --- /dev/null +++ b/libavutil/video_hint.c @@ -0,0 +1,89 @@ +/* + * Copyright 2023 Elias Carotti + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include + +#include "avstring.h" +#include "frame.h" +#include "macros.h" +#include "mem.h" +#include "video_hint.h" + +AVVideoHint *av_video_hint_alloc(AVVideoRect *rects, + size_t nb_rects, + AVVideoHintType type, + size_t* out_size) +{ + struct TestStruct { + AVVideoHint hint; + AVVideoRect rect; + }; + const size_t blocks_offset = offsetof(struct TestStruct, rect); + size_t size = blocks_offset; + AVVideoHint *hint; + + *out_size = 0; + if (nb_rects > (SIZE_MAX - size) / sizeof(AVVideoRect)) + return NULL; + size += sizeof(AVVideoRect) * nb_rects; + + hint = av_mallocz(size); + if (!hint) + return NULL; + + hint->type = type; + hint->nb_rects = nb_rects; + hint->blocks_offset = blocks_offset; + + /* Just copies the rects over the newly allocated buffer */ + memcpy((uint8_t *)hint + blocks_offset, rects, sizeof(AVVideoRect) * nb_rects); + + *out_size = size; + + return hint; +} + +AVVideoHint *av_video_hint_create_side_data(AVFrame *frame, + AVVideoRect *rects, + size_t num_rects, + AVVideoHintType type) +{ + AVVideoHint *hint; + AVBufferRef *buf; + size_t size = 0; + + hint = av_video_hint_alloc(rects, num_rects, type, &size); + if (!hint) + return NULL; + + buf = av_buffer_create((uint8_t *)hint, size, NULL, NULL, 0); + if (!buf) { + av_freep(&hint); + return NULL; + } + + if (!av_frame_new_side_data_from_buf(frame, AV_FRAME_DATA_VIDEO_HINT, buf)) { + av_buffer_unref(&buf); + return NULL; + } + + return hint; +} + diff --git a/libavutil/video_hint.h b/libavutil/video_hint.h new file mode 100644 index 0000000000..f7d113fb90 --- /dev/null +++ b/libavutil/video_hint.h @@ -0,0 +1,87 @@ +/** + * Copyright 2023 Elias Carotti + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifndef AVUTIL_VIDEO_HINT_H +#define AVUTIL_VIDEO_HINT_H + +#include +#include +#include "libavutil/avassert.h" +#include "libavutil/frame.h" + +typedef struct AVVideoRect { + uint32_t x, y; + uint32_t width, height; +} AVVideoRect; + +typedef enum AVVideoHintType { + /* blocks delimit the constant areas (unchanged), default is changed */ + AV_VIDEO_HINT_CONSTANT, + + /* blocks delimit the constant areas (changed), default is not changed */ + AV_VIDEO_HINT_CHANGED, +} AVVideoHintType; + +typedef struct AVVideoHint { + /** + * Number of blocks in the array. + * + * May be 0, in which case no per-block information is present. In this case + * the values of blocks_offset / block_size are unspecified and should not + * be accessed. + */ + int nb_rects; + + /** + * Offset in bytes from the beginning of this structure at which the array + * of blocks starts. + */ + size_t blocks_offset; + + AVVideoHintType type; +} AVVideoHint; + +static av_always_inline AVVideoRect* +av_video_hint_rects(const AVVideoHint *par) +{ + return (AVVideoRect *)((uint8_t *)par + par->blocks_offset); +} + +/** + * Allocate memory for a vector of AVVideoRect in the given AVFrame + * {@code frame} as AVFrameSideData of type AV_FRAME_DATA_VIDEO_HINT_INFO. + * The side data contains a list of rectangles for the portions of the frame + * which changed from the last encoded one (and the remainder are assumed to be + * changed), or, alternately (depending on the type parameter) the unchanged + * ones (and the remanining ones are those which changed). + * Macroblocks will thus be hinted either to be P_SKIP-ped or go through the + * regular encoding procedure. + */ +AVVideoHint *av_video_hint_create_side_data(AVFrame *frame, + AVVideoRect *rects, + size_t num_rects, + AVVideoHintType type); + +AVVideoHint *av_video_hint_alloc(AVVideoRect *rects, + size_t nb_rects, + AVVideoHintType type, + size_t *out_size); + +#endif /* AVUTIL_VIDEO_HINT_H */ -- 2.34.1