From: "Черненко Руслан" <ractyfree@gmail.com>
To: ffmpeg-devel@ffmpeg.org
Subject: [FFmpeg-devel] [PATCH 1/2] WIP avcodec/videotoolbox: add AV1 hardware acceleration
Date: Mon, 6 May 2024 23:44:21 +0300
Message-ID: <1e1173f1-9ba6-40cc-b491-e05ca2261e7f@gmail.com> (raw)
This patch adds support for hw accelerated AV1 decoding on new Apple
Silicon M3 CPUs. It's registered on the trac [1]
The first patch is the patch I used to proceed working on this feature
from Jan Ekström <jeebjp@gmail.com>. I took his commit from his branch
on github [2].
This is my first patch into ffmpeg.
[1] https://trac.ffmpeg.org/ticket/1064
[2]
https://github.com/jeeb/ffmpeg/commit/7d0ec776e148d77314ae7fc1419cef1b6d7d201e
Signed-off-by: Chernenko Ruslan <ractyfree@gmail.com>
---
configure | 4 ++
libavcodec/Makefile | 1 +
libavcodec/av1dec.c | 10 ++++
libavcodec/hwaccels.h | 1 +
libavcodec/videotoolbox.c | 14 +++++
libavcodec/videotoolbox_av1.c | 99 +++++++++++++++++++++++++++++++++++
libavcodec/vt_internal.h | 1 +
7 files changed, 130 insertions(+)
create mode 100644 libavcodec/videotoolbox_av1.c
diff --git a/configure b/configure
index 8101b4fce6..661d956480 100755
--- a/configure
+++ b/configure
@@ -2461,6 +2461,7 @@ TYPES_LIST="
kCMVideoCodecType_HEVC
kCMVideoCodecType_HEVCWithAlpha
kCMVideoCodecType_VP9
+ kCMVideoCodecType_AV1
kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange
kCVPixelFormatType_422YpCbCr8BiPlanarVideoRange
kCVPixelFormatType_422YpCbCr10BiPlanarVideoRange
@@ -3155,6 +3156,8 @@ av1_vaapi_hwaccel_deps="vaapi
VADecPictureParameterBufferAV1_bit_depth_idx"
av1_vaapi_hwaccel_select="av1_decoder"
av1_vdpau_hwaccel_deps="vdpau VdpPictureInfoAV1"
av1_vdpau_hwaccel_select="av1_decoder"
+av1_videotoolbox_hwaccel_deps="videotoolbox"
+av1_videotoolbox_hwaccel_select="av1_decoder"
av1_vulkan_hwaccel_deps="vulkan"
av1_vulkan_hwaccel_select="av1_decoder"
h263_vaapi_hwaccel_deps="vaapi"
@@ -6672,6 +6675,7 @@ enabled videotoolbox && {
check_func_headers CoreMedia/CMFormatDescription.h
kCMVideoCodecType_HEVC "-framework CoreMedia"
check_func_headers CoreMedia/CMFormatDescription.h
kCMVideoCodecType_HEVCWithAlpha "-framework CoreMedia"
check_func_headers CoreMedia/CMFormatDescription.h
kCMVideoCodecType_VP9 "-framework CoreMedia"
+ check_func_headers CoreMedia/CMFormatDescription.h
kCMVideoCodecType_AV1 "-framework CoreMedia"
check_func_headers CoreVideo/CVPixelBuffer.h
kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange "-framework CoreVideo"
check_func_headers CoreVideo/CVPixelBuffer.h
kCVPixelFormatType_422YpCbCr8BiPlanarVideoRange "-framework CoreVideo"
check_func_headers CoreVideo/CVPixelBuffer.h
kCVPixelFormatType_422YpCbCr10BiPlanarVideoRange "-framework CoreVideo"
diff --git a/libavcodec/Makefile b/libavcodec/Makefile
index cff6347bdb..c044db5eb6 100644
--- a/libavcodec/Makefile
+++ b/libavcodec/Makefile
@@ -1002,6 +1002,7 @@ OBJS-$(CONFIG_AV1_D3D12VA_HWACCEL) +=
dxva2_av1.o d3d12va_av1.o
OBJS-$(CONFIG_AV1_NVDEC_HWACCEL) += nvdec_av1.o
OBJS-$(CONFIG_AV1_VAAPI_HWACCEL) += vaapi_av1.o
OBJS-$(CONFIG_AV1_VDPAU_HWACCEL) += vdpau_av1.o
+OBJS-$(CONFIG_AV1_VIDEOTOOLBOX_HWACCEL) += videotoolbox_av1.o
OBJS-$(CONFIG_AV1_VULKAN_HWACCEL) += vulkan_decode.o vulkan_av1.o
OBJS-$(CONFIG_H263_VAAPI_HWACCEL) += vaapi_mpeg4.o
OBJS-$(CONFIG_H263_VIDEOTOOLBOX_HWACCEL) += videotoolbox.o
diff --git a/libavcodec/av1dec.c b/libavcodec/av1dec.c
index 79a30a114d..2ac0d0af4a 100644
--- a/libavcodec/av1dec.c
+++ b/libavcodec/av1dec.c
@@ -541,6 +541,7 @@ static int get_pixel_format(AVCodecContext *avctx)
CONFIG_AV1_NVDEC_HWACCEL + \
CONFIG_AV1_VAAPI_HWACCEL + \
CONFIG_AV1_VDPAU_HWACCEL + \
+ CONFIG_AV1_VIDEOTOOLBOX_HWACCEL + \
CONFIG_AV1_VULKAN_HWACCEL)
enum AVPixelFormat pix_fmts[HWACCEL_MAX + 2], *fmtp = pix_fmts;
@@ -568,6 +569,9 @@ static int get_pixel_format(AVCodecContext *avctx)
#if CONFIG_AV1_VDPAU_HWACCEL
*fmtp++ = AV_PIX_FMT_VDPAU;
#endif
+#if CONFIG_AV1_VIDEOTOOLBOX_HWACCEL
+ *fmtp++ = AV_PIX_FMT_VIDEOTOOLBOX;
+#endif
#if CONFIG_AV1_VULKAN_HWACCEL
*fmtp++ = AV_PIX_FMT_VULKAN;
#endif
@@ -592,6 +596,9 @@ static int get_pixel_format(AVCodecContext *avctx)
#if CONFIG_AV1_VDPAU_HWACCEL
*fmtp++ = AV_PIX_FMT_VDPAU;
#endif
+#if CONFIG_AV1_VIDEOTOOLBOX_HWACCEL
+ *fmtp++ = AV_PIX_FMT_VIDEOTOOLBOX;
+#endif
#if CONFIG_AV1_VULKAN_HWACCEL
*fmtp++ = AV_PIX_FMT_VULKAN;
#endif
@@ -1581,6 +1588,9 @@ const FFCodec ff_av1_decoder = {
#if CONFIG_AV1_VDPAU_HWACCEL
HWACCEL_VDPAU(av1),
#endif
+#if CONFIG_AV1_VIDEOTOOLBOX_HWACCEL
+ HWACCEL_VIDEOTOOLBOX(av1),
+#endif
#if CONFIG_AV1_VULKAN_HWACCEL
HWACCEL_VULKAN(av1),
#endif
diff --git a/libavcodec/hwaccels.h b/libavcodec/hwaccels.h
index 5171e4c7d7..2b9bdc8fc9 100644
--- a/libavcodec/hwaccels.h
+++ b/libavcodec/hwaccels.h
@@ -26,6 +26,7 @@ extern const struct FFHWAccel ff_av1_dxva2_hwaccel;
extern const struct FFHWAccel ff_av1_nvdec_hwaccel;
extern const struct FFHWAccel ff_av1_vaapi_hwaccel;
extern const struct FFHWAccel ff_av1_vdpau_hwaccel;
+extern const struct FFHWAccel ff_av1_videotoolbox_hwaccel;
extern const struct FFHWAccel ff_av1_vulkan_hwaccel;
extern const struct FFHWAccel ff_h263_vaapi_hwaccel;
extern const struct FFHWAccel ff_h263_videotoolbox_hwaccel;
diff --git a/libavcodec/videotoolbox.c b/libavcodec/videotoolbox.c
index d6990a39c0..4dbcb1ab68 100644
--- a/libavcodec/videotoolbox.c
+++ b/libavcodec/videotoolbox.c
@@ -56,6 +56,10 @@ enum { kCMVideoCodecType_HEVC = 'hvc1' };
enum { kCMVideoCodecType_VP9 = 'vp09' };
#endif
+#if !HAVE_KCMVIDEOCODECTYPE_AV1
+enum { kCMVideoCodecType_AV1 = 'av01' };
+#endif
+
#define VIDEOTOOLBOX_ESDS_EXTRADATA_PADDING 12
typedef struct VTHWFrame {
@@ -846,6 +850,13 @@ static CFDictionaryRef
videotoolbox_decoder_config_create(CMVideoCodecType codec
if (data)
CFDictionarySetValue(avc_info, CFSTR("vpcC"), data);
break;
+#endif
+#if CONFIG_AV1_VIDEOTOOLBOX_HWACCEL
+ case kCMVideoCodecType_AV1 :
+ data = ff_videotoolbox_av1c_extradata_create(avctx);
+ if (data)
+ CFDictionarySetValue(avc_info, CFSTR("av1C"), data);
+ break;
#endif
default:
break;
@@ -912,6 +923,9 @@ static int videotoolbox_start(AVCodecContext *avctx)
case AV_CODEC_ID_VP9 :
videotoolbox->cm_codec_type = kCMVideoCodecType_VP9;
break;
+ case AV_CODEC_ID_AV1 :
+ videotoolbox->cm_codec_type = kCMVideoCodecType_AV1;
+ break;
default :
break;
}
diff --git a/libavcodec/videotoolbox_av1.c b/libavcodec/videotoolbox_av1.c
new file mode 100644
index 0000000000..7f7270c466
--- /dev/null
+++ b/libavcodec/videotoolbox_av1.c
@@ -0,0 +1,99 @@
+/*
+ * Videotoolbox hardware acceleration for AV1
+ * Copyright (c) 2023 Jan Ekström
+ *
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
02110-1301 USA
+ */
+
+#include "libavformat/avio.h"
+#include "libavformat/avio_internal.h"
+#define CALLED_FROM_AVCODEC 1
+#include "libavformat/av1.c"
+#undef CALLED_FROM_AVCODEC
+
+#include "libavutil/avutil.h"
+#include "libavutil/frame.h"
+#include "libavutil/pixfmt.h"
+
+#include "av1dec.h"
+#include "codec_id.h"
+#include "hwaccel_internal.h"
+#include "internal.h"
+#include "vt_internal.h"
+
+CFDataRef ff_videotoolbox_av1c_extradata_create(AVCodecContext *avctx)
+{
+ AVIOContext *pb;
+ uint8_t *buf;
+ CFDataRef data = NULL;
+ int buf_size = 0;
+ int ret = avio_open_dyn_buf(&pb);
+ if (ret < 0)
+ return NULL;
+
+ ret = ff_isom_write_av1c(pb, avctx->extradata,
avctx->extradata_size, 1);
+ if (ret < 0)
+ goto fail;
+
+ buf_size = avio_get_dyn_buf(pb, &buf);
+ if (buf_size)
+ data = CFDataCreate(kCFAllocatorDefault, buf, buf_size);
+
+fail:
+ ffio_free_dyn_buf(&pb);
+
+ return data;
+}
+
+static int videotoolbox_av1_start_frame(AVCodecContext *avctx,
+ const uint8_t *buffer,
+ uint32_t size)
+{
+ return 0;
+}
+
+static int videotoolbox_av1_decode_slice(AVCodecContext *avctx,
+ const uint8_t *buffer,
+ uint32_t size)
+{
+ VTContext *vtctx = avctx->internal->hwaccel_priv_data;
+
+ return ff_videotoolbox_buffer_copy(vtctx, buffer, size);
+}
+
+static int videotoolbox_av1_end_frame(AVCodecContext *avctx)
+{
+ const AV1DecContext *s = avctx->priv_data;
+ AVFrame *frame = s->cur_frame.f;
+
+ return ff_videotoolbox_common_end_frame(avctx, frame);
+}
+
+const FFHWAccel ff_av1_videotoolbox_hwaccel = {
+ .p.name = "av1_videotoolbox",
+ .p.type = AVMEDIA_TYPE_VIDEO,
+ .p.id = AV_CODEC_ID_AV1,
+ .p.pix_fmt = AV_PIX_FMT_VIDEOTOOLBOX,
+ .alloc_frame = ff_videotoolbox_alloc_frame,
+ .start_frame = videotoolbox_av1_start_frame,
+ .decode_slice = videotoolbox_av1_decode_slice,
+ .end_frame = videotoolbox_av1_end_frame,
+ .frame_params = ff_videotoolbox_frame_params,
+ .init = ff_videotoolbox_common_init,
+ .uninit = ff_videotoolbox_uninit,
+ .priv_data_size = sizeof(VTContext),
+};
diff --git a/libavcodec/vt_internal.h b/libavcodec/vt_internal.h
index 9502d7c7dc..427348f206 100644
--- a/libavcodec/vt_internal.h
+++ b/libavcodec/vt_internal.h
@@ -64,6 +64,7 @@ int ff_videotoolbox_h264_decode_slice(AVCodecContext
*avctx,
const uint8_t *buffer,
uint32_t size);
int ff_videotoolbox_common_end_frame(AVCodecContext *avctx, AVFrame
*frame);
+CFDataRef ff_videotoolbox_av1c_extradata_create(AVCodecContext *avctx);
CFDataRef ff_videotoolbox_avcc_extradata_create(AVCodecContext *avctx);
CFDataRef ff_videotoolbox_hvcc_extradata_create(AVCodecContext *avctx);
CFDataRef ff_videotoolbox_vpcc_extradata_create(AVCodecContext *avctx);
--
2.39.3 (Apple Git-146)
_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
reply other threads:[~2024-05-06 20:45 UTC|newest]
Thread overview: [no followups] expand[flat|nested] mbox.gz Atom feed
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=1e1173f1-9ba6-40cc-b491-e05ca2261e7f@gmail.com \
--to=ractyfree@gmail.com \
--cc=ffmpeg-devel@ffmpeg.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Git Inbox Mirror of the ffmpeg-devel mailing list - see https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
This inbox may be cloned and mirrored by anyone:
git clone --mirror https://master.gitmailbox.com/ffmpegdev/0 ffmpegdev/git/0.git
# If you have public-inbox 1.1+ installed, you may
# initialize and index your mirror using the following commands:
public-inbox-init -V2 ffmpegdev ffmpegdev/ https://master.gitmailbox.com/ffmpegdev \
ffmpegdev@gitmailbox.com
public-inbox-index ffmpegdev
Example config snippet for mirrors.
AGPL code for this site: git clone https://public-inbox.org/public-inbox.git