* [FFmpeg-devel] [PATCH v3 2/2] fftools/ffplay: add hwaccel decoding support
[not found] <20231028181341.13090-1-quinkblack@foxmail.com>
@ 2023-10-28 18:13 ` Zhao Zhili
2023-10-28 19:32 ` Lynne
0 siblings, 1 reply; 7+ messages in thread
From: Zhao Zhili @ 2023-10-28 18:13 UTC (permalink / raw)
To: ffmpeg-devel; +Cc: Zhao Zhili
From: Zhao Zhili <zhilizhao@tencent.com>
---
v3: shared vulkan instance between libplacebo and hwcontext
fftools/ffplay.c | 43 ++++++
fftools/ffplay_renderer.c | 316 +++++++++++++++++++++++++++++++++++++-
fftools/ffplay_renderer.h | 2 +
3 files changed, 355 insertions(+), 6 deletions(-)
diff --git a/fftools/ffplay.c b/fftools/ffplay.c
index 305d72d8b8..29b37a5e46 100644
--- a/fftools/ffplay.c
+++ b/fftools/ffplay.c
@@ -352,6 +352,7 @@ static int autorotate = 1;
static int find_stream_info = 1;
static int filter_nbthreads = 0;
static int enable_vulkan = 0;
+static const char *hwaccel = NULL;
/* current context */
static int is_full_screen;
@@ -2557,6 +2558,37 @@ static int audio_open(void *opaque, AVChannelLayout *wanted_channel_layout, int
return spec.size;
}
+static int create_hwaccel(AVBufferRef **device_ctx)
+{
+ enum AVHWDeviceType type;
+ int ret;
+ AVBufferRef *vk_dev;
+
+ *device_ctx = NULL;
+
+ if (!hwaccel)
+ return 0;
+
+ type = av_hwdevice_find_type_by_name(hwaccel);
+ if (type == AV_HWDEVICE_TYPE_NONE)
+ return AVERROR(ENOTSUP);
+
+ ret = vk_renderer_get_hw_dev(vk_renderer, &vk_dev);
+ if (ret < 0)
+ return ret;
+
+ ret = av_hwdevice_ctx_create_derived(device_ctx, type, vk_dev, 0);
+ if (!ret)
+ return 0;
+
+ if (ret != AVERROR(ENOSYS))
+ return ret;
+
+ av_log(NULL, AV_LOG_WARNING, "Derive %s from vulkan not supported.\n", hwaccel);
+ ret = av_hwdevice_ctx_create(device_ctx, type, NULL, NULL, 0);
+ return ret;
+}
+
/* open a given stream. Return 0 if OK */
static int stream_component_open(VideoState *is, int stream_index)
{
@@ -2624,6 +2656,12 @@ static int stream_component_open(VideoState *is, int stream_index)
av_dict_set(&opts, "flags", "+copy_opaque", AV_DICT_MULTIKEY);
+ if (avctx->codec_type == AVMEDIA_TYPE_VIDEO) {
+ ret = create_hwaccel(&avctx->hw_device_ctx);
+ if (ret < 0)
+ goto fail;
+ }
+
if ((ret = avcodec_open2(avctx, codec, &opts)) < 0) {
goto fail;
}
@@ -3625,6 +3663,7 @@ static const OptionDef options[] = {
"read and decode the streams to fill missing information with heuristics" },
{ "filter_threads", HAS_ARG | OPT_INT | OPT_EXPERT, { &filter_nbthreads }, "number of filter threads per graph" },
{ "enable_vulkan", OPT_BOOL, { &enable_vulkan }, "enable vulkan render" },
+ { "hwaccel", HAS_ARG | OPT_STRING | OPT_EXPERT, { &hwaccel }, "use HW accelerated decoding" },
{ NULL, },
};
@@ -3739,6 +3778,10 @@ int main(int argc, char **argv)
#ifdef SDL_HINT_VIDEO_X11_NET_WM_BYPASS_COMPOSITOR
SDL_SetHint(SDL_HINT_VIDEO_X11_NET_WM_BYPASS_COMPOSITOR, "0");
#endif
+ if (hwaccel && !enable_vulkan) {
+ av_log(NULL, AV_LOG_INFO, "Enable vulkan renderer to support hwaccel %s\n", hwaccel);
+ enable_vulkan = 1;
+ }
if (enable_vulkan) {
vk_renderer = vk_get_renderer();
if (vk_renderer) {
diff --git a/fftools/ffplay_renderer.c b/fftools/ffplay_renderer.c
index 49e5516d5d..497d758ec7 100644
--- a/fftools/ffplay_renderer.c
+++ b/fftools/ffplay_renderer.c
@@ -35,6 +35,8 @@ struct VkRenderer {
int (*create)(VkRenderer *renderer, SDL_Window *window);
+ int (*get_hw_dev)(VkRenderer *renderer, AVBufferRef **dev);
+
int (*display)(VkRenderer *renderer, AVFrame *frame);
int (*resize)(VkRenderer *renderer, int width, int height);
@@ -54,6 +56,13 @@ typedef struct RendererContext {
pl_tex tex[4];
pl_log vk_log;
+
+ AVBufferRef *hw_device_ref;
+ AVBufferRef *hw_frame_ref;
+ enum AVPixelFormat *transfer_formats;
+ AVHWFramesConstraints *constraints;
+
+ AVFrame *vk_frame;
} RendererContext;
static void vk_log_cb(void *log_priv, enum pl_log_level level, const char *msg) {
@@ -74,35 +83,46 @@ static void vk_log_cb(void *log_priv, enum pl_log_level level, const char *msg)
static int create(VkRenderer *renderer, SDL_Window *window)
{
int ret = 0;
- unsigned ext_num = 0;
+ unsigned num_ext = 0;
const char **ext = NULL;
int w, h;
struct pl_log_params vk_log_params = {
.log_cb = vk_log_cb,
- .log_level = PL_LOG_WARN,
+ .log_level = PL_LOG_DEBUG,
.log_priv = renderer,
};
RendererContext *ctx = (RendererContext *)renderer;
- if (!SDL_Vulkan_GetInstanceExtensions(window, &ext_num, NULL)) {
+ static const char *opt_exts[] = {
+ VK_EXT_DESCRIPTOR_BUFFER_EXTENSION_NAME,
+ VK_EXT_SHADER_ATOMIC_FLOAT_EXTENSION_NAME,
+ VK_KHR_VIDEO_DECODE_QUEUE_EXTENSION_NAME,
+ VK_KHR_VIDEO_DECODE_H264_EXTENSION_NAME,
+ VK_KHR_VIDEO_DECODE_H265_EXTENSION_NAME,
+ VK_KHR_VIDEO_QUEUE_EXTENSION_NAME,
+ "VK_MESA_video_decode_av1",
+ };
+ const int num_opt_exts = FF_ARRAY_ELEMS(opt_exts);
+
+ if (!SDL_Vulkan_GetInstanceExtensions(window, &num_ext, NULL)) {
av_log(NULL, AV_LOG_FATAL, "Failed to get vulkan extensions: %s\n", SDL_GetError());
return AVERROR_EXTERNAL;
}
- ext = av_calloc(ext_num, sizeof(*ext));
+ ext = av_calloc(num_ext, sizeof(*ext));
if (!ext) {
ret = AVERROR(ENOMEM);
goto out;
}
- SDL_Vulkan_GetInstanceExtensions(window, &ext_num, ext);
+ SDL_Vulkan_GetInstanceExtensions(window, &num_ext, ext);
ctx->vk_log = pl_log_create(PL_API_VER, &vk_log_params);
ctx->vk_inst = pl_vk_inst_create(ctx->vk_log, pl_vk_inst_params(
.get_proc_addr = SDL_Vulkan_GetVkGetInstanceProcAddr(),
.debug = false,
.extensions = ext,
- .num_extensions = ext_num,
+ .num_extensions = num_ext
));
if (!ctx->vk_inst) {
ret = AVERROR_EXTERNAL;
@@ -119,6 +139,9 @@ static int create(VkRenderer *renderer, SDL_Window *window)
.get_proc_addr = ctx->vk_inst->get_proc_addr,
.surface = ctx->vk_surface,
.allow_software = false,
+ .opt_extensions = opt_exts,
+ .num_opt_extensions = num_opt_exts,
+ .extra_queues = VK_QUEUE_VIDEO_DECODE_BIT_KHR,
));
if (!ctx->pl_vk) {
ret = AVERROR_EXTERNAL;
@@ -137,6 +160,13 @@ static int create(VkRenderer *renderer, SDL_Window *window)
ret = AVERROR_EXTERNAL;
goto out;
}
+
+ ctx->vk_frame = av_frame_alloc();
+ if (!ctx->vk_frame) {
+ ret = AVERROR(ENOMEM);
+ goto out;
+ }
+
ret = 0;
out:
@@ -144,6 +174,264 @@ out:
return ret;
}
+static void lock_queue(struct AVHWDeviceContext *dev_ctx, uint32_t queue_family, uint32_t index)
+{
+ RendererContext *ctx = dev_ctx->user_opaque;
+ pl_vulkan vk = ctx->pl_vk;
+ vk->lock_queue(vk, queue_family, index);
+}
+
+static void unlock_queue(struct AVHWDeviceContext *dev_ctx, uint32_t queue_family, uint32_t index)
+{
+ RendererContext *ctx = dev_ctx->user_opaque;
+ pl_vulkan vk = ctx->pl_vk;
+ vk->unlock_queue(vk, queue_family, index);
+}
+
+static int get_decode_queue(VkRenderer *renderer, int *index, int *count)
+{
+ RendererContext *ctx = (RendererContext *) renderer;
+ VkQueueFamilyProperties *queue_family_prop = NULL;
+ uint32_t num_queue_family_prop = 0;
+ PFN_vkGetPhysicalDeviceQueueFamilyProperties vkGetPhysicalDeviceQueueFamilyProperties;
+
+ *index = -1;
+ *count = 0;
+ vkGetPhysicalDeviceQueueFamilyProperties = (PFN_vkGetPhysicalDeviceQueueFamilyProperties)
+ ctx->vk_inst->get_proc_addr(ctx->vk_inst->instance,
+ "vkGetPhysicalDeviceQueueFamilyProperties");
+ vkGetPhysicalDeviceQueueFamilyProperties(ctx->pl_vk->phys_device, &num_queue_family_prop, NULL);
+ if (!num_queue_family_prop)
+ return AVERROR_EXTERNAL;
+
+ queue_family_prop = av_calloc(num_queue_family_prop, sizeof(*queue_family_prop));
+ if (!queue_family_prop)
+ return AVERROR(ENOMEM);
+
+ vkGetPhysicalDeviceQueueFamilyProperties(ctx->pl_vk->phys_device, &num_queue_family_prop, queue_family_prop);
+
+ for (int i = 0; i < num_queue_family_prop; i++) {
+ if (queue_family_prop[i].queueFlags & VK_QUEUE_VIDEO_DECODE_BIT_KHR) {
+ *index = i;
+ *count = queue_family_prop[i].queueCount;
+ break;
+ }
+ }
+ av_free(queue_family_prop);
+
+ return 0;
+}
+
+static int get_hw_dev(VkRenderer *renderer, AVBufferRef **dev)
+{
+ RendererContext *ctx = (RendererContext *)renderer;
+ AVHWDeviceContext *device_ctx;
+ AVVulkanDeviceContext *vk_dev_ctx;
+ int decode_index;
+ int decode_count;
+ int ret;
+
+ *dev = NULL;
+ if (ctx->hw_device_ref) {
+ *dev = ctx->hw_device_ref;
+ return 0;
+ }
+
+ ctx->hw_device_ref = av_hwdevice_ctx_alloc(AV_HWDEVICE_TYPE_VULKAN);
+ if (!ctx->hw_device_ref) {
+ return AVERROR(ENOMEM);
+ }
+
+ device_ctx = (AVHWDeviceContext *)ctx->hw_device_ref->data;
+ device_ctx->user_opaque = ctx;
+
+ vk_dev_ctx = device_ctx->hwctx;
+ vk_dev_ctx->lock_queue = lock_queue,
+ vk_dev_ctx->unlock_queue = unlock_queue;
+
+ vk_dev_ctx->get_proc_addr = ctx->vk_inst->get_proc_addr;
+
+ vk_dev_ctx->inst = ctx->vk_inst->instance;
+ vk_dev_ctx->phys_dev = ctx->pl_vk->phys_device;
+ vk_dev_ctx->act_dev = ctx->pl_vk->device;
+
+ vk_dev_ctx->device_features = *ctx->pl_vk->features;
+
+ vk_dev_ctx->enabled_inst_extensions = ctx->vk_inst->extensions;
+ vk_dev_ctx->nb_enabled_inst_extensions = ctx->vk_inst->num_extensions;
+
+ vk_dev_ctx->enabled_dev_extensions = ctx->pl_vk->extensions;
+ vk_dev_ctx->nb_enabled_dev_extensions = ctx->pl_vk->num_extensions;
+
+ vk_dev_ctx->queue_family_index = ctx->pl_vk->queue_graphics.index;
+ vk_dev_ctx->nb_graphics_queues = ctx->pl_vk->queue_graphics.count;
+
+ vk_dev_ctx->queue_family_tx_index = ctx->pl_vk->queue_transfer.index;
+ vk_dev_ctx->nb_tx_queues = ctx->pl_vk->queue_transfer.count;
+
+ vk_dev_ctx->queue_family_comp_index = ctx->pl_vk->queue_compute.index;
+ vk_dev_ctx->nb_comp_queues = ctx->pl_vk->queue_compute.count;
+
+ ret = get_decode_queue(renderer, &decode_index, &decode_count);
+ if (ret < 0)
+ return ret;
+
+ vk_dev_ctx->queue_family_decode_index = decode_index;
+ vk_dev_ctx->nb_decode_queues = decode_count;
+
+ ret = av_hwdevice_ctx_init(ctx->hw_device_ref);
+ if (ret < 0)
+ return ret;
+
+ *dev = ctx->hw_device_ref;
+ return 0;
+}
+
+static int create_hw_frame(VkRenderer *renderer, AVFrame *frame)
+{
+ RendererContext *ctx = (RendererContext *)renderer;
+ AVHWFramesContext *src_hw_frame = (AVHWFramesContext *)frame->hw_frames_ctx->data;
+ AVHWFramesContext *hw_frame;
+ AVVulkanFramesContext *vk_frame_ctx;
+ int ret;
+
+ if (ctx->hw_frame_ref) {
+ hw_frame = (AVHWFramesContext *)ctx->hw_frame_ref->data;
+
+ if (hw_frame->width == frame->width &&
+ hw_frame->height == frame->height &&
+ hw_frame->sw_format == src_hw_frame->sw_format)
+ return 0;
+
+ av_buffer_unref(&ctx->hw_frame_ref);
+ }
+
+ if (!ctx->constraints) {
+ ctx->constraints = av_hwdevice_get_hwframe_constraints(ctx->hw_device_ref, NULL);
+ if (!ctx->constraints)
+ return AVERROR(ENOMEM);
+ }
+
+ // Check constraints and skip create hwframe.
+ // Don't take it as error since we can fallback to memory copy from GPU to CPU.
+ if ((ctx->constraints->max_width && ctx->constraints->max_width < frame->width) ||
+ (ctx->constraints->max_height && ctx->constraints->max_height < frame->height) ||
+ (ctx->constraints->min_width && ctx->constraints->min_width > frame->width) ||
+ (ctx->constraints->min_height && ctx->constraints->min_height > frame->height))
+ return 0;
+
+ if (ctx->constraints->valid_sw_formats) {
+ enum AVPixelFormat *sw_formats = ctx->constraints->valid_sw_formats;
+ while (*sw_formats != AV_PIX_FMT_NONE) {
+ if (*sw_formats == src_hw_frame->sw_format)
+ break;
+ sw_formats++;
+ }
+ if (*sw_formats == AV_PIX_FMT_NONE)
+ return 0;
+ }
+
+ ctx->hw_frame_ref = av_hwframe_ctx_alloc(ctx->hw_device_ref);
+ if (!ctx->hw_frame_ref)
+ return AVERROR(ENOMEM);
+
+ hw_frame = (AVHWFramesContext *)ctx->hw_frame_ref->data;
+ hw_frame->format = AV_PIX_FMT_VULKAN;
+ hw_frame->sw_format = src_hw_frame->sw_format;
+ hw_frame->width = frame->width;
+ hw_frame->height = frame->height;
+
+ if (frame->format == AV_PIX_FMT_CUDA) {
+ vk_frame_ctx = hw_frame->hwctx;
+ vk_frame_ctx->flags = AV_VK_FRAME_FLAG_DISABLE_MULTIPLANE;
+ }
+
+ ret = av_hwframe_ctx_init(ctx->hw_frame_ref);
+ if (ret < 0) {
+ av_log(renderer, AV_LOG_ERROR, "Create hwframe context failed, %s\n", av_err2str(ret));
+ return ret;
+ }
+
+ av_hwframe_transfer_get_formats(ctx->hw_frame_ref, AV_HWFRAME_TRANSFER_DIRECTION_TO,
+ &ctx->transfer_formats, 0);
+
+ return 0;
+}
+
+static inline int check_hw_transfer(RendererContext *ctx, AVFrame *frame)
+{
+ if (!ctx->hw_frame_ref || !ctx->transfer_formats)
+ return 0;
+
+ for (int i = 0; ctx->transfer_formats[i] != AV_PIX_FMT_NONE; i++)
+ if (ctx->transfer_formats[i] == frame->format)
+ return 1;
+
+ return 0;
+}
+
+static int transfer_frame(VkRenderer *renderer, AVFrame *frame)
+{
+ RendererContext *ctx = (RendererContext *)renderer;
+ int ret;
+
+ if (!frame->hw_frames_ctx)
+ return 0;
+
+ if (frame->format == AV_PIX_FMT_VULKAN)
+ return 0;
+
+ ret = create_hw_frame(renderer, frame);
+ if (ret < 0)
+ return ret;
+
+ // Try map data first
+ av_frame_unref(ctx->vk_frame);
+ if (ctx->hw_frame_ref) {
+ ctx->vk_frame->hw_frames_ctx = av_buffer_ref(ctx->hw_frame_ref);
+ ctx->vk_frame->format = AV_PIX_FMT_VULKAN;
+ }
+ ret = av_hwframe_map(ctx->vk_frame, frame, AV_HWFRAME_MAP_READ);
+ if (!ret)
+ goto out;
+
+ if (ret != AVERROR(ENOSYS)) {
+ av_log(NULL, AV_LOG_FATAL, "Map data to vulkan failed: %s\n", av_err2str(ret));
+ return ret;
+ }
+
+ // Try transfer to vulkan
+ if (check_hw_transfer(ctx, frame)) {
+ av_frame_unref(ctx->vk_frame);
+ av_hwframe_get_buffer(ctx->hw_frame_ref, ctx->vk_frame, 0);
+ ret = av_hwframe_transfer_data(ctx->vk_frame, frame, 1);
+ if (!ret)
+ goto out;
+
+ if (ret < 0 && ret != AVERROR(ENOSYS)) {
+ av_log(NULL, AV_LOG_FATAL, "Transfer data to vulkan failed: %s\n", av_err2str(ret));
+ return ret;
+ }
+ }
+
+ // Transfer to CPU
+ av_frame_unref(ctx->vk_frame);
+ ret = av_hwframe_transfer_data(ctx->vk_frame, frame, 0);
+ if (ret < 0) {
+ av_log(NULL, AV_LOG_FATAL, "Transfer data to CPU failed: %s\n", av_err2str(ret));
+ return ret;
+ }
+
+out:
+ ret = av_frame_copy_props(ctx->vk_frame, frame);
+ if (ret < 0)
+ return ret;
+ av_frame_unref(frame);
+ av_frame_move_ref(frame, ctx->vk_frame);
+
+ return 0;
+}
+
static int display(VkRenderer *renderer, AVFrame *frame)
{
struct pl_swapchain_frame swap_frame = {0};
@@ -152,6 +440,10 @@ static int display(VkRenderer *renderer, AVFrame *frame)
RendererContext *ctx = (RendererContext *)renderer;
int ret = 0;
+ ret = transfer_frame(renderer, frame);
+ if (ret < 0)
+ return ret;
+
if (!pl_map_avframe_ex(ctx->pl_vk->gpu, &pl_frame, pl_avframe_params(
.frame = frame,
.tex = ctx->tex))) {
@@ -198,6 +490,12 @@ static void destroy(VkRenderer *renderer)
PFN_vkDestroySurfaceKHR vkDestroySurfaceKHR;
RendererContext *ctx = (RendererContext *)renderer;
+ av_freep(&ctx->transfer_formats);
+ av_hwframe_constraints_free(&ctx->constraints);
+ av_buffer_unref(&ctx->hw_frame_ref);
+ av_buffer_unref(&ctx->hw_device_ref);
+ av_frame_free(&ctx->vk_frame);
+
if (ctx->pl_vk) {
for (int i = 0; i < FF_ARRAY_ELEMS(ctx->tex); i++)
pl_tex_destroy(ctx->pl_vk->gpu, &ctx->tex[i]);
@@ -233,6 +531,7 @@ VkRenderer *vk_get_renderer(void)
renderer = &ctx->api;
renderer->class = &vulkan_renderer_class;
+ renderer->get_hw_dev = get_hw_dev;
renderer->create = create;
renderer->display = display;
renderer->resize = resize;
@@ -255,6 +554,11 @@ int vk_renderer_create(VkRenderer *renderer, SDL_Window *window)
return renderer->create(renderer, window);
}
+int vk_renderer_get_hw_dev(VkRenderer *renderer, AVBufferRef **dev)
+{
+ return renderer->get_hw_dev(renderer, dev);
+}
+
int vk_renderer_display(VkRenderer *renderer, AVFrame *frame)
{
return renderer->display(renderer, frame);
diff --git a/fftools/ffplay_renderer.h b/fftools/ffplay_renderer.h
index 4029f154a9..b982ba0192 100644
--- a/fftools/ffplay_renderer.h
+++ b/fftools/ffplay_renderer.h
@@ -29,6 +29,8 @@ VkRenderer *vk_get_renderer(void);
int vk_renderer_create(VkRenderer *renderer, SDL_Window *window);
+int vk_renderer_get_hw_dev(VkRenderer *renderer, AVBufferRef **dev);
+
int vk_renderer_display(VkRenderer *renderer, AVFrame *frame);
int vk_renderer_resize(VkRenderer *renderer, int width, int height);
--
2.42.0
_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
^ permalink raw reply [flat|nested] 7+ messages in thread
* Re: [FFmpeg-devel] [PATCH v3 2/2] fftools/ffplay: add hwaccel decoding support
2023-10-28 18:13 ` [FFmpeg-devel] [PATCH v3 2/2] fftools/ffplay: add hwaccel decoding support Zhao Zhili
@ 2023-10-28 19:32 ` Lynne
2023-10-29 2:23 ` [FFmpeg-devel] [PATCH v4 1/2] fftools/ffplay: add vulkan renderer via libplacebo Zhao Zhili
` (2 more replies)
0 siblings, 3 replies; 7+ messages in thread
From: Lynne @ 2023-10-28 19:32 UTC (permalink / raw)
To: FFmpeg development discussions and patches
Oct 28, 2023, 20:14 by quinkblack@foxmail.com:
> From: Zhao Zhili <zhilizhao@tencent.com>
>
> ---
> v3: shared vulkan instance between libplacebo and hwcontext
>
> fftools/ffplay.c | 43 ++++++
> fftools/ffplay_renderer.c | 316 +++++++++++++++++++++++++++++++++++++-
> fftools/ffplay_renderer.h | 2 +
> 3 files changed, 355 insertions(+), 6 deletions(-)
>
> diff --git a/fftools/ffplay.c b/fftools/ffplay.c
> index 305d72d8b8..29b37a5e46 100644
> --- a/fftools/ffplay.c
> +++ b/fftools/ffplay.c
> @@ -352,6 +352,7 @@ static int autorotate = 1;
> static int find_stream_info = 1;
> static int filter_nbthreads = 0;
> static int enable_vulkan = 0;
> +static const char *hwaccel = NULL;
>
> /* current context */
> static int is_full_screen;
> @@ -2557,6 +2558,37 @@ static int audio_open(void *opaque, AVChannelLayout *wanted_channel_layout, int
> return spec.size;
> }
>
> +static int create_hwaccel(AVBufferRef **device_ctx)
> +{
> + enum AVHWDeviceType type;
> + int ret;
> + AVBufferRef *vk_dev;
> +
> + *device_ctx = NULL;
> +
> + if (!hwaccel)
> + return 0;
> +
> + type = av_hwdevice_find_type_by_name(hwaccel);
> + if (type == AV_HWDEVICE_TYPE_NONE)
> + return AVERROR(ENOTSUP);
> +
> + ret = vk_renderer_get_hw_dev(vk_renderer, &vk_dev);
> + if (ret < 0)
> + return ret;
> +
> + ret = av_hwdevice_ctx_create_derived(device_ctx, type, vk_dev, 0);
> + if (!ret)
> + return 0;
> +
> + if (ret != AVERROR(ENOSYS))
> + return ret;
> +
> + av_log(NULL, AV_LOG_WARNING, "Derive %s from vulkan not supported.\n", hwaccel);
> + ret = av_hwdevice_ctx_create(device_ctx, type, NULL, NULL, 0);
> + return ret;
> +}
> +
> /* open a given stream. Return 0 if OK */
> static int stream_component_open(VideoState *is, int stream_index)
> {
> @@ -2624,6 +2656,12 @@ static int stream_component_open(VideoState *is, int stream_index)
>
> av_dict_set(&opts, "flags", "+copy_opaque", AV_DICT_MULTIKEY);
>
> + if (avctx->codec_type == AVMEDIA_TYPE_VIDEO) {
> + ret = create_hwaccel(&avctx->hw_device_ctx);
> + if (ret < 0)
> + goto fail;
> + }
> +
> if ((ret = avcodec_open2(avctx, codec, &opts)) < 0) {
> goto fail;
> }
> @@ -3625,6 +3663,7 @@ static const OptionDef options[] = {
> "read and decode the streams to fill missing information with heuristics" },
> { "filter_threads", HAS_ARG | OPT_INT | OPT_EXPERT, { &filter_nbthreads }, "number of filter threads per graph" },
> { "enable_vulkan", OPT_BOOL, { &enable_vulkan }, "enable vulkan render" },
> + { "hwaccel", HAS_ARG | OPT_STRING | OPT_EXPERT, { &hwaccel }, "use HW accelerated decoding" },
> { NULL, },
> };
>
> @@ -3739,6 +3778,10 @@ int main(int argc, char **argv)
> #ifdef SDL_HINT_VIDEO_X11_NET_WM_BYPASS_COMPOSITOR
> SDL_SetHint(SDL_HINT_VIDEO_X11_NET_WM_BYPASS_COMPOSITOR, "0");
> #endif
> + if (hwaccel && !enable_vulkan) {
> + av_log(NULL, AV_LOG_INFO, "Enable vulkan renderer to support hwaccel %s\n", hwaccel);
> + enable_vulkan = 1;
> + }
> if (enable_vulkan) {
> vk_renderer = vk_get_renderer();
> if (vk_renderer) {
> diff --git a/fftools/ffplay_renderer.c b/fftools/ffplay_renderer.c
> index 49e5516d5d..497d758ec7 100644
> --- a/fftools/ffplay_renderer.c
> +++ b/fftools/ffplay_renderer.c
> @@ -35,6 +35,8 @@ struct VkRenderer {
>
> int (*create)(VkRenderer *renderer, SDL_Window *window);
>
> + int (*get_hw_dev)(VkRenderer *renderer, AVBufferRef **dev);
> +
> int (*display)(VkRenderer *renderer, AVFrame *frame);
>
> int (*resize)(VkRenderer *renderer, int width, int height);
> @@ -54,6 +56,13 @@ typedef struct RendererContext {
> pl_tex tex[4];
>
> pl_log vk_log;
> +
> + AVBufferRef *hw_device_ref;
> + AVBufferRef *hw_frame_ref;
> + enum AVPixelFormat *transfer_formats;
> + AVHWFramesConstraints *constraints;
> +
> + AVFrame *vk_frame;
> } RendererContext;
>
> static void vk_log_cb(void *log_priv, enum pl_log_level level, const char *msg) {
> @@ -74,35 +83,46 @@ static void vk_log_cb(void *log_priv, enum pl_log_level level, const char *msg)
> static int create(VkRenderer *renderer, SDL_Window *window)
> {
> int ret = 0;
> - unsigned ext_num = 0;
> + unsigned num_ext = 0;
> const char **ext = NULL;
> int w, h;
> struct pl_log_params vk_log_params = {
> .log_cb = vk_log_cb,
> - .log_level = PL_LOG_WARN,
> + .log_level = PL_LOG_DEBUG,
> .log_priv = renderer,
> };
> RendererContext *ctx = (RendererContext *)renderer;
>
> - if (!SDL_Vulkan_GetInstanceExtensions(window, &ext_num, NULL)) {
> + static const char *opt_exts[] = {
> + VK_EXT_DESCRIPTOR_BUFFER_EXTENSION_NAME,
> + VK_EXT_SHADER_ATOMIC_FLOAT_EXTENSION_NAME,
> + VK_KHR_VIDEO_DECODE_QUEUE_EXTENSION_NAME,
> + VK_KHR_VIDEO_DECODE_H264_EXTENSION_NAME,
> + VK_KHR_VIDEO_DECODE_H265_EXTENSION_NAME,
> + VK_KHR_VIDEO_QUEUE_EXTENSION_NAME,
> + "VK_MESA_video_decode_av1",
> + };
> + const int num_opt_exts = FF_ARRAY_ELEMS(opt_exts);
> +
> + if (!SDL_Vulkan_GetInstanceExtensions(window, &num_ext, NULL)) {
> av_log(NULL, AV_LOG_FATAL, "Failed to get vulkan extensions: %s\n", SDL_GetError());
> return AVERROR_EXTERNAL;
>
That's not a complete list. Look at optional_device_exts.
There are also device features that both we and libplacebo would like
to be enabled. Look into vulkan_device_create_internal(), the list there
is up to date. FFmpeg requires timeline semaphores, and descriptor_buffer
for lavfi filters to work.
Is there no option to let libavutil create the hwdevice for both SDL and libplacebo?
That way, that code path can also be tested.
_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
^ permalink raw reply [flat|nested] 7+ messages in thread
* [FFmpeg-devel] [PATCH v4 1/2] fftools/ffplay: add vulkan renderer via libplacebo
2023-10-28 19:32 ` Lynne
@ 2023-10-29 2:23 ` Zhao Zhili
[not found] ` <20231029022347.15243-1-quinkblack@foxmail.com>
2023-10-29 2:31 ` [FFmpeg-devel] [PATCH v3 " Zhao Zhili
2 siblings, 0 replies; 7+ messages in thread
From: Zhao Zhili @ 2023-10-29 2:23 UTC (permalink / raw)
To: ffmpeg-devel; +Cc: Zhao Zhili
From: Zhao Zhili <zhilizhao@tencent.com>
---
configure | 2 +-
fftools/Makefile | 2 +
fftools/ffplay.c | 46 ++++++-
fftools/ffplay_renderer.c | 271 ++++++++++++++++++++++++++++++++++++++
fftools/ffplay_renderer.h | 38 ++++++
5 files changed, 353 insertions(+), 6 deletions(-)
create mode 100644 fftools/ffplay_renderer.c
create mode 100644 fftools/ffplay_renderer.h
diff --git a/configure b/configure
index f494da204c..55329a5037 100755
--- a/configure
+++ b/configure
@@ -3903,7 +3903,7 @@ ffmpeg_select="aformat_filter anull_filter atrim_filter format_filter
ffmpeg_suggest="ole32 psapi shell32"
ffplay_deps="avcodec avformat avfilter swscale swresample sdl2"
ffplay_select="crop_filter transpose_filter hflip_filter vflip_filter rotate_filter"
-ffplay_suggest="shell32"
+ffplay_suggest="shell32 libplacebo vulkan"
ffprobe_deps="avcodec avformat"
ffprobe_suggest="shell32"
diff --git a/fftools/Makefile b/fftools/Makefile
index 56820e6bc8..3c763e3db9 100644
--- a/fftools/Makefile
+++ b/fftools/Makefile
@@ -22,6 +22,8 @@ OBJS-ffmpeg += \
fftools/sync_queue.o \
fftools/thread_queue.o \
+OBJS-ffplay += fftools/ffplay_renderer.o
+
define DOFFTOOL
OBJS-$(1) += fftools/cmdutils.o fftools/opt_common.o fftools/$(1).o $(OBJS-$(1)-yes)
ifdef HAVE_GNU_WINDRES
diff --git a/fftools/ffplay.c b/fftools/ffplay.c
index d8c69e10bc..305d72d8b8 100644
--- a/fftools/ffplay.c
+++ b/fftools/ffplay.c
@@ -58,6 +58,7 @@
#include <SDL_thread.h>
#include "cmdutils.h"
+#include "ffplay_renderer.h"
#include "opt_common.h"
const char program_name[] = "ffplay";
@@ -350,6 +351,7 @@ static char *afilters = NULL;
static int autorotate = 1;
static int find_stream_info = 1;
static int filter_nbthreads = 0;
+static int enable_vulkan = 0;
/* current context */
static int is_full_screen;
@@ -362,6 +364,8 @@ static SDL_Renderer *renderer;
static SDL_RendererInfo renderer_info = {0};
static SDL_AudioDeviceID audio_dev;
+static VkRenderer *vk_renderer;
+
static const struct TextureFormatEntry {
enum AVPixelFormat format;
int texture_fmt;
@@ -954,6 +958,11 @@ static void video_image_display(VideoState *is)
SDL_Rect rect;
vp = frame_queue_peek_last(&is->pictq);
+ if (vk_renderer) {
+ vk_renderer_display(vk_renderer, vp->frame);
+ return;
+ }
+
if (is->subtitle_st) {
if (frame_queue_nb_remaining(&is->subpq) > 0) {
sp = frame_queue_peek(&is->subpq);
@@ -1289,6 +1298,8 @@ static void do_exit(VideoState *is)
}
if (renderer)
SDL_DestroyRenderer(renderer);
+ if (vk_renderer)
+ vk_renderer_destroy(vk_renderer);
if (window)
SDL_DestroyWindow(window);
uninit_opts();
@@ -3447,6 +3458,8 @@ static void event_loop(VideoState *cur_stream)
SDL_DestroyTexture(cur_stream->vis_texture);
cur_stream->vis_texture = NULL;
}
+ if (vk_renderer)
+ vk_renderer_resize(vk_renderer, screen_width, screen_height);
case SDL_WINDOWEVENT_EXPOSED:
cur_stream->force_refresh = 1;
}
@@ -3611,6 +3624,7 @@ static const OptionDef options[] = {
{ "find_stream_info", OPT_BOOL | OPT_INPUT | OPT_EXPERT, { &find_stream_info },
"read and decode the streams to fill missing information with heuristics" },
{ "filter_threads", HAS_ARG | OPT_INT | OPT_EXPERT, { &filter_nbthreads }, "number of filter threads per graph" },
+ { "enable_vulkan", OPT_BOOL, { &enable_vulkan }, "enable vulkan render" },
{ NULL, },
};
@@ -3725,9 +3739,31 @@ int main(int argc, char **argv)
#ifdef SDL_HINT_VIDEO_X11_NET_WM_BYPASS_COMPOSITOR
SDL_SetHint(SDL_HINT_VIDEO_X11_NET_WM_BYPASS_COMPOSITOR, "0");
#endif
+ if (enable_vulkan) {
+ vk_renderer = vk_get_renderer();
+ if (vk_renderer) {
+#if SDL_VERSION_ATLEAST(2, 0, 6)
+ flags |= SDL_WINDOW_VULKAN;
+#endif
+ } else {
+ av_log(NULL, AV_LOG_WARNING, "Doesn't support vulkan renderer, fallback to SDL renderer\n");
+ enable_vulkan = 0;
+ }
+ }
window = SDL_CreateWindow(program_name, SDL_WINDOWPOS_UNDEFINED, SDL_WINDOWPOS_UNDEFINED, default_width, default_height, flags);
SDL_SetHint(SDL_HINT_RENDER_SCALE_QUALITY, "linear");
- if (window) {
+ if (!window) {
+ av_log(NULL, AV_LOG_FATAL, "Failed to create window: %s", SDL_GetError());
+ do_exit(NULL);
+ }
+
+ if (vk_renderer) {
+ ret = vk_renderer_create(vk_renderer, window);
+ if (ret < 0) {
+ av_log(NULL, AV_LOG_FATAL, "Failed to create vulkan renderer, %s\n", av_err2str(ret));
+ do_exit(NULL);
+ }
+ } else {
renderer = SDL_CreateRenderer(window, -1, SDL_RENDERER_ACCELERATED | SDL_RENDERER_PRESENTVSYNC);
if (!renderer) {
av_log(NULL, AV_LOG_WARNING, "Failed to initialize a hardware accelerated renderer: %s\n", SDL_GetError());
@@ -3737,10 +3773,10 @@ int main(int argc, char **argv)
if (!SDL_GetRendererInfo(renderer, &renderer_info))
av_log(NULL, AV_LOG_VERBOSE, "Initialized %s renderer.\n", renderer_info.name);
}
- }
- if (!window || !renderer || !renderer_info.num_texture_formats) {
- av_log(NULL, AV_LOG_FATAL, "Failed to create window or renderer: %s", SDL_GetError());
- do_exit(NULL);
+ if (!renderer || !renderer_info.num_texture_formats) {
+ av_log(NULL, AV_LOG_FATAL, "Failed to create window or renderer: %s", SDL_GetError());
+ do_exit(NULL);
+ }
}
}
diff --git a/fftools/ffplay_renderer.c b/fftools/ffplay_renderer.c
new file mode 100644
index 0000000000..49e5516d5d
--- /dev/null
+++ b/fftools/ffplay_renderer.c
@@ -0,0 +1,271 @@
+/*
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include "config.h"
+#include "ffplay_renderer.h"
+
+#define HAVE_VULKAN_RENDERER (SDL_VERSION_ATLEAST(2, 0, 6) && CONFIG_LIBPLACEBO)
+
+#if HAVE_VULKAN_RENDERER
+
+#include <libplacebo/vulkan.h>
+#include <libplacebo/utils/frame_queue.h>
+#include <libplacebo/utils/libav.h>
+#include <SDL_vulkan.h>
+
+#endif
+
+struct VkRenderer {
+ const AVClass *class;
+
+ int (*create)(VkRenderer *renderer, SDL_Window *window);
+
+ int (*display)(VkRenderer *renderer, AVFrame *frame);
+
+ int (*resize)(VkRenderer *renderer, int width, int height);
+
+ void (*destroy)(VkRenderer *renderer);
+};
+
+#if HAVE_VULKAN_RENDERER
+
+typedef struct RendererContext {
+ VkRenderer api;
+ pl_vk_inst vk_inst;
+ pl_vulkan pl_vk;
+ pl_swapchain swapchain;
+ VkSurfaceKHR vk_surface;
+ pl_renderer renderer;
+ pl_tex tex[4];
+
+ pl_log vk_log;
+} RendererContext;
+
+static void vk_log_cb(void *log_priv, enum pl_log_level level, const char *msg) {
+ static const int level_map[] = {
+ AV_LOG_QUIET,
+ AV_LOG_FATAL,
+ AV_LOG_ERROR,
+ AV_LOG_WARNING,
+ AV_LOG_INFO,
+ AV_LOG_DEBUG,
+ AV_LOG_TRACE,
+ };
+
+ if (level > 0 && level < FF_ARRAY_ELEMS(level_map))
+ av_log(log_priv, level_map[level], "%s\n", msg);
+}
+
+static int create(VkRenderer *renderer, SDL_Window *window)
+{
+ int ret = 0;
+ unsigned ext_num = 0;
+ const char **ext = NULL;
+ int w, h;
+ struct pl_log_params vk_log_params = {
+ .log_cb = vk_log_cb,
+ .log_level = PL_LOG_WARN,
+ .log_priv = renderer,
+ };
+ RendererContext *ctx = (RendererContext *)renderer;
+
+ if (!SDL_Vulkan_GetInstanceExtensions(window, &ext_num, NULL)) {
+ av_log(NULL, AV_LOG_FATAL, "Failed to get vulkan extensions: %s\n", SDL_GetError());
+ return AVERROR_EXTERNAL;
+ }
+
+ ext = av_calloc(ext_num, sizeof(*ext));
+ if (!ext) {
+ ret = AVERROR(ENOMEM);
+ goto out;
+ }
+
+ SDL_Vulkan_GetInstanceExtensions(window, &ext_num, ext);
+
+ ctx->vk_log = pl_log_create(PL_API_VER, &vk_log_params);
+ ctx->vk_inst = pl_vk_inst_create(ctx->vk_log, pl_vk_inst_params(
+ .get_proc_addr = SDL_Vulkan_GetVkGetInstanceProcAddr(),
+ .debug = false,
+ .extensions = ext,
+ .num_extensions = ext_num,
+ ));
+ if (!ctx->vk_inst) {
+ ret = AVERROR_EXTERNAL;
+ goto out;
+ }
+
+ if (!SDL_Vulkan_CreateSurface(window, ctx->vk_inst->instance, &ctx->vk_surface)) {
+ ret = AVERROR_EXTERNAL;
+ goto out;
+ }
+
+ ctx->pl_vk = pl_vulkan_create(ctx->vk_log, pl_vulkan_params(
+ .instance = ctx->vk_inst->instance,
+ .get_proc_addr = ctx->vk_inst->get_proc_addr,
+ .surface = ctx->vk_surface,
+ .allow_software = false,
+ ));
+ if (!ctx->pl_vk) {
+ ret = AVERROR_EXTERNAL;
+ goto out;
+ }
+
+ ctx->swapchain = pl_vulkan_create_swapchain(ctx->pl_vk,
+ pl_vulkan_swapchain_params(.surface = ctx->vk_surface,
+ .present_mode = VK_PRESENT_MODE_FIFO_KHR));
+
+ SDL_Vulkan_GetDrawableSize(window, &w, &h);
+ pl_swapchain_resize(ctx->swapchain, &w, &h);
+
+ ctx->renderer = pl_renderer_create(ctx->vk_log, ctx->pl_vk->gpu);
+ if (!ctx->renderer) {
+ ret = AVERROR_EXTERNAL;
+ goto out;
+ }
+ ret = 0;
+
+out:
+ av_free(ext);
+ return ret;
+}
+
+static int display(VkRenderer *renderer, AVFrame *frame)
+{
+ struct pl_swapchain_frame swap_frame = {0};
+ struct pl_frame pl_frame = {0};
+ struct pl_frame target = {0};
+ RendererContext *ctx = (RendererContext *)renderer;
+ int ret = 0;
+
+ if (!pl_map_avframe_ex(ctx->pl_vk->gpu, &pl_frame, pl_avframe_params(
+ .frame = frame,
+ .tex = ctx->tex))) {
+ av_log(NULL, AV_LOG_ERROR, "pl_map_avframe_ex failed\n");
+ return AVERROR_EXTERNAL;
+ }
+
+ if (!pl_swapchain_start_frame(ctx->swapchain, &swap_frame)) {
+ av_log(NULL, AV_LOG_ERROR, "start frame failed\n");
+ ret = AVERROR_EXTERNAL;
+ goto out;
+ }
+
+ pl_frame_from_swapchain(&target, &swap_frame);
+ if (!pl_render_image(ctx->renderer, &pl_frame, &target, &pl_render_default_params)) {
+ av_log(NULL, AV_LOG_ERROR, "pl_render_image failed\n");
+ ret = AVERROR_EXTERNAL;
+ goto out;
+ }
+
+ if (!pl_swapchain_submit_frame(ctx->swapchain)) {
+ av_log(NULL, AV_LOG_ERROR, "pl_swapchain_submit_frame failed\n");
+ ret = AVERROR_EXTERNAL;
+ goto out;
+ }
+ pl_swapchain_swap_buffers(ctx->swapchain);
+
+out:
+ pl_unmap_avframe(ctx->pl_vk->gpu, &pl_frame);
+ return ret;
+}
+
+static int resize(VkRenderer *renderer, int width, int height)
+{
+ RendererContext *ctx = (RendererContext *)renderer;
+
+ if (!pl_swapchain_resize(ctx->swapchain, &width, &height))
+ return AVERROR_EXTERNAL;
+ return 0;
+}
+
+static void destroy(VkRenderer *renderer)
+{
+ PFN_vkDestroySurfaceKHR vkDestroySurfaceKHR;
+ RendererContext *ctx = (RendererContext *)renderer;
+
+ if (ctx->pl_vk) {
+ for (int i = 0; i < FF_ARRAY_ELEMS(ctx->tex); i++)
+ pl_tex_destroy(ctx->pl_vk->gpu, &ctx->tex[i]);
+ pl_renderer_destroy(&ctx->renderer);
+ pl_swapchain_destroy(&ctx->swapchain);
+ pl_vulkan_destroy(&ctx->pl_vk);
+ }
+
+ if (ctx->vk_surface) {
+ vkDestroySurfaceKHR = (PFN_vkDestroySurfaceKHR)
+ ctx->vk_inst->get_proc_addr(ctx->vk_inst->instance, "vkDestroySurfaceKHR");
+ vkDestroySurfaceKHR(ctx->vk_inst->instance, ctx->vk_surface, NULL);
+ ctx->vk_surface = NULL;
+ }
+
+ pl_vk_inst_destroy(&ctx->vk_inst);
+ pl_log_destroy(&ctx->vk_log);
+}
+
+static const AVClass vulkan_renderer_class = {
+ .class_name = "Vulkan Renderer",
+ .item_name = av_default_item_name,
+ .version = LIBAVUTIL_VERSION_INT,
+};
+
+VkRenderer *vk_get_renderer(void)
+{
+ RendererContext *ctx = av_mallocz(sizeof(*ctx));
+ VkRenderer *renderer;
+
+ if (!ctx)
+ return NULL;
+
+ renderer = &ctx->api;
+ renderer->class = &vulkan_renderer_class;
+ renderer->create = create;
+ renderer->display = display;
+ renderer->resize = resize;
+ renderer->destroy = destroy;
+
+ return renderer;
+}
+
+#else
+
+VkRenderer *vk_get_renderer(void)
+{
+ return NULL;
+}
+
+#endif
+
+int vk_renderer_create(VkRenderer *renderer, SDL_Window *window)
+{
+ return renderer->create(renderer, window);
+}
+
+int vk_renderer_display(VkRenderer *renderer, AVFrame *frame)
+{
+ return renderer->display(renderer, frame);
+}
+
+int vk_renderer_resize(VkRenderer *renderer, int width, int height)
+{
+ return renderer->resize(renderer, width, height);
+}
+
+void vk_renderer_destroy(VkRenderer *renderer)
+{
+ renderer->destroy(renderer);
+}
diff --git a/fftools/ffplay_renderer.h b/fftools/ffplay_renderer.h
new file mode 100644
index 0000000000..4029f154a9
--- /dev/null
+++ b/fftools/ffplay_renderer.h
@@ -0,0 +1,38 @@
+/*
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#ifndef FFTOOLS_FFPLAY_RENDERER_H
+#define FFTOOLS_FFPLAY_RENDERER_H
+
+#include <SDL.h>
+
+#include "libavutil/frame.h"
+
+typedef struct VkRenderer VkRenderer;
+
+VkRenderer *vk_get_renderer(void);
+
+int vk_renderer_create(VkRenderer *renderer, SDL_Window *window);
+
+int vk_renderer_display(VkRenderer *renderer, AVFrame *frame);
+
+int vk_renderer_resize(VkRenderer *renderer, int width, int height);
+
+void vk_renderer_destroy(VkRenderer *renderer);
+
+#endif /* FFTOOLS_FFPLAY_RENDERER_H */
--
2.34.1
_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
^ permalink raw reply [flat|nested] 7+ messages in thread
* [FFmpeg-devel] [PATCH v4 2/2] fftools/ffplay: add hwaccel decoding support
[not found] ` <20231029022347.15243-1-quinkblack@foxmail.com>
@ 2023-10-29 2:23 ` Zhao Zhili
0 siblings, 0 replies; 7+ messages in thread
From: Zhao Zhili @ 2023-10-29 2:23 UTC (permalink / raw)
To: ffmpeg-devel; +Cc: Zhao Zhili
From: Zhao Zhili <zhilizhao@tencent.com>
---
v4: add more optional extensions
v3: shared vulkan instance between libplacebo and hwcontext
fftools/ffplay.c | 43 +++++
fftools/ffplay_renderer.c | 393 ++++++++++++++++++++++++++++++++++++--
fftools/ffplay_renderer.h | 2 +
3 files changed, 418 insertions(+), 20 deletions(-)
diff --git a/fftools/ffplay.c b/fftools/ffplay.c
index 305d72d8b8..29b37a5e46 100644
--- a/fftools/ffplay.c
+++ b/fftools/ffplay.c
@@ -352,6 +352,7 @@ static int autorotate = 1;
static int find_stream_info = 1;
static int filter_nbthreads = 0;
static int enable_vulkan = 0;
+static const char *hwaccel = NULL;
/* current context */
static int is_full_screen;
@@ -2557,6 +2558,37 @@ static int audio_open(void *opaque, AVChannelLayout *wanted_channel_layout, int
return spec.size;
}
+static int create_hwaccel(AVBufferRef **device_ctx)
+{
+ enum AVHWDeviceType type;
+ int ret;
+ AVBufferRef *vk_dev;
+
+ *device_ctx = NULL;
+
+ if (!hwaccel)
+ return 0;
+
+ type = av_hwdevice_find_type_by_name(hwaccel);
+ if (type == AV_HWDEVICE_TYPE_NONE)
+ return AVERROR(ENOTSUP);
+
+ ret = vk_renderer_get_hw_dev(vk_renderer, &vk_dev);
+ if (ret < 0)
+ return ret;
+
+ ret = av_hwdevice_ctx_create_derived(device_ctx, type, vk_dev, 0);
+ if (!ret)
+ return 0;
+
+ if (ret != AVERROR(ENOSYS))
+ return ret;
+
+ av_log(NULL, AV_LOG_WARNING, "Derive %s from vulkan not supported.\n", hwaccel);
+ ret = av_hwdevice_ctx_create(device_ctx, type, NULL, NULL, 0);
+ return ret;
+}
+
/* open a given stream. Return 0 if OK */
static int stream_component_open(VideoState *is, int stream_index)
{
@@ -2624,6 +2656,12 @@ static int stream_component_open(VideoState *is, int stream_index)
av_dict_set(&opts, "flags", "+copy_opaque", AV_DICT_MULTIKEY);
+ if (avctx->codec_type == AVMEDIA_TYPE_VIDEO) {
+ ret = create_hwaccel(&avctx->hw_device_ctx);
+ if (ret < 0)
+ goto fail;
+ }
+
if ((ret = avcodec_open2(avctx, codec, &opts)) < 0) {
goto fail;
}
@@ -3625,6 +3663,7 @@ static const OptionDef options[] = {
"read and decode the streams to fill missing information with heuristics" },
{ "filter_threads", HAS_ARG | OPT_INT | OPT_EXPERT, { &filter_nbthreads }, "number of filter threads per graph" },
{ "enable_vulkan", OPT_BOOL, { &enable_vulkan }, "enable vulkan render" },
+ { "hwaccel", HAS_ARG | OPT_STRING | OPT_EXPERT, { &hwaccel }, "use HW accelerated decoding" },
{ NULL, },
};
@@ -3739,6 +3778,10 @@ int main(int argc, char **argv)
#ifdef SDL_HINT_VIDEO_X11_NET_WM_BYPASS_COMPOSITOR
SDL_SetHint(SDL_HINT_VIDEO_X11_NET_WM_BYPASS_COMPOSITOR, "0");
#endif
+ if (hwaccel && !enable_vulkan) {
+ av_log(NULL, AV_LOG_INFO, "Enable vulkan renderer to support hwaccel %s\n", hwaccel);
+ enable_vulkan = 1;
+ }
if (enable_vulkan) {
vk_renderer = vk_get_renderer();
if (vk_renderer) {
diff --git a/fftools/ffplay_renderer.c b/fftools/ffplay_renderer.c
index 49e5516d5d..b898f69d26 100644
--- a/fftools/ffplay_renderer.c
+++ b/fftools/ffplay_renderer.c
@@ -16,6 +16,9 @@
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
+#define VK_NO_PROTOTYPES
+#define VK_ENABLE_BETA_EXTENSIONS
+
#include "config.h"
#include "ffplay_renderer.h"
@@ -35,6 +38,8 @@ struct VkRenderer {
int (*create)(VkRenderer *renderer, SDL_Window *window);
+ int (*get_hw_dev)(VkRenderer *renderer, AVBufferRef **dev);
+
int (*display)(VkRenderer *renderer, AVFrame *frame);
int (*resize)(VkRenderer *renderer, int width, int height);
@@ -54,9 +59,18 @@ typedef struct RendererContext {
pl_tex tex[4];
pl_log vk_log;
+
+ AVBufferRef *hw_device_ref;
+ AVBufferRef *hw_frame_ref;
+ enum AVPixelFormat *transfer_formats;
+ AVHWFramesConstraints *constraints;
+
+ AVFrame *vk_frame;
} RendererContext;
-static void vk_log_cb(void *log_priv, enum pl_log_level level, const char *msg) {
+static void vk_log_cb(void *log_priv, enum pl_log_level level,
+ const char *msg)
+{
static const int level_map[] = {
AV_LOG_QUIET,
AV_LOG_FATAL,
@@ -71,45 +85,77 @@ static void vk_log_cb(void *log_priv, enum pl_log_level level, const char *msg)
av_log(log_priv, level_map[level], "%s\n", msg);
}
+// Should keep sync with optional_device_exts inside hwcontext_vulkan.c
+static const char *optional_device_exts[] = {
+ /* Misc or required by other extensions */
+ VK_KHR_PORTABILITY_SUBSET_EXTENSION_NAME,
+ VK_KHR_PUSH_DESCRIPTOR_EXTENSION_NAME,
+ VK_KHR_SAMPLER_YCBCR_CONVERSION_EXTENSION_NAME,
+ VK_EXT_DESCRIPTOR_BUFFER_EXTENSION_NAME,
+ VK_EXT_PHYSICAL_DEVICE_DRM_EXTENSION_NAME,
+ VK_EXT_SHADER_ATOMIC_FLOAT_EXTENSION_NAME,
+ VK_KHR_COOPERATIVE_MATRIX_EXTENSION_NAME,
+
+ /* Imports/exports */
+ VK_KHR_EXTERNAL_MEMORY_FD_EXTENSION_NAME,
+ VK_EXT_EXTERNAL_MEMORY_DMA_BUF_EXTENSION_NAME,
+ VK_EXT_IMAGE_DRM_FORMAT_MODIFIER_EXTENSION_NAME,
+ VK_KHR_EXTERNAL_SEMAPHORE_FD_EXTENSION_NAME,
+ VK_EXT_EXTERNAL_MEMORY_HOST_EXTENSION_NAME,
+#ifdef _WIN32
+ VK_KHR_EXTERNAL_MEMORY_WIN32_EXTENSION_NAME,
+ VK_KHR_EXTERNAL_SEMAPHORE_WIN32_EXTENSION_NAME,
+#endif
+
+ /* Video encoding/decoding */
+ VK_KHR_VIDEO_QUEUE_EXTENSION_NAME,
+ VK_KHR_VIDEO_DECODE_QUEUE_EXTENSION_NAME,
+ VK_KHR_VIDEO_DECODE_H264_EXTENSION_NAME,
+ VK_KHR_VIDEO_DECODE_H265_EXTENSION_NAME,
+ "VK_MESA_video_decode_av1",
+};
+
static int create(VkRenderer *renderer, SDL_Window *window)
{
int ret = 0;
- unsigned ext_num = 0;
+ unsigned num_ext = 0;
const char **ext = NULL;
int w, h;
struct pl_log_params vk_log_params = {
.log_cb = vk_log_cb,
- .log_level = PL_LOG_WARN,
+ .log_level = PL_LOG_DEBUG,
.log_priv = renderer,
};
- RendererContext *ctx = (RendererContext *)renderer;
+ RendererContext *ctx = (RendererContext *) renderer;
- if (!SDL_Vulkan_GetInstanceExtensions(window, &ext_num, NULL)) {
- av_log(NULL, AV_LOG_FATAL, "Failed to get vulkan extensions: %s\n", SDL_GetError());
+ if (!SDL_Vulkan_GetInstanceExtensions(window, &num_ext, NULL)) {
+ av_log(NULL, AV_LOG_FATAL, "Failed to get vulkan extensions: %s\n",
+ SDL_GetError());
return AVERROR_EXTERNAL;
}
- ext = av_calloc(ext_num, sizeof(*ext));
+ ext = av_calloc(num_ext, sizeof(*ext));
if (!ext) {
ret = AVERROR(ENOMEM);
goto out;
}
- SDL_Vulkan_GetInstanceExtensions(window, &ext_num, ext);
+ SDL_Vulkan_GetInstanceExtensions(window, &num_ext, ext);
ctx->vk_log = pl_log_create(PL_API_VER, &vk_log_params);
ctx->vk_inst = pl_vk_inst_create(ctx->vk_log, pl_vk_inst_params(
.get_proc_addr = SDL_Vulkan_GetVkGetInstanceProcAddr(),
.debug = false,
.extensions = ext,
- .num_extensions = ext_num,
+ .num_extensions = num_ext
));
if (!ctx->vk_inst) {
ret = AVERROR_EXTERNAL;
goto out;
}
- if (!SDL_Vulkan_CreateSurface(window, ctx->vk_inst->instance, &ctx->vk_surface)) {
+ if (!SDL_Vulkan_CreateSurface(window, ctx->vk_inst->instance,
+ &ctx->vk_surface)) {
ret = AVERROR_EXTERNAL;
goto out;
}
@@ -119,15 +165,20 @@ static int create(VkRenderer *renderer, SDL_Window *window)
.get_proc_addr = ctx->vk_inst->get_proc_addr,
.surface = ctx->vk_surface,
.allow_software = false,
+ .opt_extensions = optional_device_exts,
+ .num_opt_extensions = FF_ARRAY_ELEMS(optional_device_exts),
+ .extra_queues = VK_QUEUE_VIDEO_DECODE_BIT_KHR,
));
if (!ctx->pl_vk) {
ret = AVERROR_EXTERNAL;
goto out;
}
- ctx->swapchain = pl_vulkan_create_swapchain(ctx->pl_vk,
- pl_vulkan_swapchain_params(.surface = ctx->vk_surface,
- .present_mode = VK_PRESENT_MODE_FIFO_KHR));
+ ctx->swapchain = pl_vulkan_create_swapchain(
+ ctx->pl_vk,
+ pl_vulkan_swapchain_params(
+ .surface = ctx->vk_surface,
+ .present_mode = VK_PRESENT_MODE_FIFO_KHR));
SDL_Vulkan_GetDrawableSize(window, &w, &h);
pl_swapchain_resize(ctx->swapchain, &w, &h);
@@ -137,21 +188,309 @@ static int create(VkRenderer *renderer, SDL_Window *window)
ret = AVERROR_EXTERNAL;
goto out;
}
+
+ ctx->vk_frame = av_frame_alloc();
+ if (!ctx->vk_frame) {
+ ret = AVERROR(ENOMEM);
+ goto out;
+ }
+
ret = 0;
-out:
+ out:
av_free(ext);
return ret;
}
+static void lock_queue(struct AVHWDeviceContext *dev_ctx,
+ uint32_t queue_family, uint32_t index)
+{
+ RendererContext *ctx = dev_ctx->user_opaque;
+ pl_vulkan vk = ctx->pl_vk;
+ vk->lock_queue(vk, queue_family, index);
+}
+
+static void unlock_queue(struct AVHWDeviceContext *dev_ctx,
+ uint32_t queue_family,
+ uint32_t index)
+{
+ RendererContext *ctx = dev_ctx->user_opaque;
+ pl_vulkan vk = ctx->pl_vk;
+ vk->unlock_queue(vk, queue_family, index);
+}
+
+static int get_decode_queue(VkRenderer *renderer, int *index, int *count)
+{
+ RendererContext *ctx = (RendererContext *) renderer;
+ VkQueueFamilyProperties *queue_family_prop = NULL;
+ uint32_t num_queue_family_prop = 0;
+ PFN_vkGetPhysicalDeviceQueueFamilyProperties get_queue_family_prop;
+ PFN_vkGetInstanceProcAddr get_proc_addr = ctx->vk_inst->get_proc_addr;
+
+ *index = -1;
+ *count = 0;
+ get_queue_family_prop = (PFN_vkGetPhysicalDeviceQueueFamilyProperties)
+ get_proc_addr(ctx->vk_inst->instance,
+ "vkGetPhysicalDeviceQueueFamilyProperties");
+ get_queue_family_prop(ctx->pl_vk->phys_device,
+ &num_queue_family_prop, NULL);
+ if (!num_queue_family_prop)
+ return AVERROR_EXTERNAL;
+
+ queue_family_prop = av_calloc(num_queue_family_prop,
+ sizeof(*queue_family_prop));
+ if (!queue_family_prop)
+ return AVERROR(ENOMEM);
+
+ get_queue_family_prop(ctx->pl_vk->phys_device,
+ &num_queue_family_prop,
+ queue_family_prop);
+
+ for (int i = 0; i < num_queue_family_prop; i++) {
+ if (queue_family_prop[i].queueFlags & VK_QUEUE_VIDEO_DECODE_BIT_KHR) {
+ *index = i;
+ *count = queue_family_prop[i].queueCount;
+ break;
+ }
+ }
+ av_free(queue_family_prop);
+
+ return 0;
+}
+
+static int get_hw_dev(VkRenderer *renderer, AVBufferRef **dev)
+{
+ RendererContext *ctx = (RendererContext *) renderer;
+ AVHWDeviceContext *device_ctx;
+ AVVulkanDeviceContext *vk_dev_ctx;
+ int decode_index;
+ int decode_count;
+ int ret;
+
+ *dev = NULL;
+ if (ctx->hw_device_ref) {
+ *dev = ctx->hw_device_ref;
+ return 0;
+ }
+
+ ctx->hw_device_ref = av_hwdevice_ctx_alloc(AV_HWDEVICE_TYPE_VULKAN);
+ if (!ctx->hw_device_ref) {
+ return AVERROR(ENOMEM);
+ }
+
+ device_ctx = (AVHWDeviceContext *) ctx->hw_device_ref->data;
+ device_ctx->user_opaque = ctx;
+
+ vk_dev_ctx = device_ctx->hwctx;
+ vk_dev_ctx->lock_queue = lock_queue,
+ vk_dev_ctx->unlock_queue = unlock_queue;
+
+ vk_dev_ctx->get_proc_addr = ctx->vk_inst->get_proc_addr;
+
+ vk_dev_ctx->inst = ctx->vk_inst->instance;
+ vk_dev_ctx->phys_dev = ctx->pl_vk->phys_device;
+ vk_dev_ctx->act_dev = ctx->pl_vk->device;
+
+ vk_dev_ctx->device_features = *ctx->pl_vk->features;
+
+ vk_dev_ctx->enabled_inst_extensions = ctx->vk_inst->extensions;
+ vk_dev_ctx->nb_enabled_inst_extensions = ctx->vk_inst->num_extensions;
+
+ vk_dev_ctx->enabled_dev_extensions = ctx->pl_vk->extensions;
+ vk_dev_ctx->nb_enabled_dev_extensions = ctx->pl_vk->num_extensions;
+
+ vk_dev_ctx->queue_family_index = ctx->pl_vk->queue_graphics.index;
+ vk_dev_ctx->nb_graphics_queues = ctx->pl_vk->queue_graphics.count;
+
+ vk_dev_ctx->queue_family_tx_index = ctx->pl_vk->queue_transfer.index;
+ vk_dev_ctx->nb_tx_queues = ctx->pl_vk->queue_transfer.count;
+
+ vk_dev_ctx->queue_family_comp_index = ctx->pl_vk->queue_compute.index;
+ vk_dev_ctx->nb_comp_queues = ctx->pl_vk->queue_compute.count;
+
+ ret = get_decode_queue(renderer, &decode_index, &decode_count);
+ if (ret < 0)
+ return ret;
+
+ vk_dev_ctx->queue_family_decode_index = decode_index;
+ vk_dev_ctx->nb_decode_queues = decode_count;
+
+ ret = av_hwdevice_ctx_init(ctx->hw_device_ref);
+ if (ret < 0)
+ return ret;
+
+ *dev = ctx->hw_device_ref;
+ return 0;
+}
+
+static int create_hw_frame(VkRenderer *renderer, AVFrame *frame)
+{
+ RendererContext *ctx = (RendererContext *) renderer;
+ AVHWFramesContext *src_hw_frame = (AVHWFramesContext *)
+ frame->hw_frames_ctx->data;
+ AVHWFramesContext *hw_frame;
+ AVVulkanFramesContext *vk_frame_ctx;
+ int ret;
+
+ if (ctx->hw_frame_ref) {
+ hw_frame = (AVHWFramesContext *) ctx->hw_frame_ref->data;
+
+ if (hw_frame->width == frame->width &&
+ hw_frame->height == frame->height &&
+ hw_frame->sw_format == src_hw_frame->sw_format)
+ return 0;
+
+ av_buffer_unref(&ctx->hw_frame_ref);
+ }
+
+ if (!ctx->constraints) {
+ ctx->constraints = av_hwdevice_get_hwframe_constraints(
+ ctx->hw_device_ref, NULL);
+ if (!ctx->constraints)
+ return AVERROR(ENOMEM);
+ }
+
+ // Check constraints and skip create hwframe. Don't take it as error since
+ // we can fallback to memory copy from GPU to CPU.
+ if ((ctx->constraints->max_width &&
+ ctx->constraints->max_width < frame->width) ||
+ (ctx->constraints->max_height &&
+ ctx->constraints->max_height < frame->height) ||
+ (ctx->constraints->min_width &&
+ ctx->constraints->min_width > frame->width) ||
+ (ctx->constraints->min_height &&
+ ctx->constraints->min_height > frame->height))
+ return 0;
+
+ if (ctx->constraints->valid_sw_formats) {
+ enum AVPixelFormat *sw_formats = ctx->constraints->valid_sw_formats;
+ while (*sw_formats != AV_PIX_FMT_NONE) {
+ if (*sw_formats == src_hw_frame->sw_format)
+ break;
+ sw_formats++;
+ }
+ if (*sw_formats == AV_PIX_FMT_NONE)
+ return 0;
+ }
+
+ ctx->hw_frame_ref = av_hwframe_ctx_alloc(ctx->hw_device_ref);
+ if (!ctx->hw_frame_ref)
+ return AVERROR(ENOMEM);
+
+ hw_frame = (AVHWFramesContext *) ctx->hw_frame_ref->data;
+ hw_frame->format = AV_PIX_FMT_VULKAN;
+ hw_frame->sw_format = src_hw_frame->sw_format;
+ hw_frame->width = frame->width;
+ hw_frame->height = frame->height;
+
+ if (frame->format == AV_PIX_FMT_CUDA) {
+ vk_frame_ctx = hw_frame->hwctx;
+ vk_frame_ctx->flags = AV_VK_FRAME_FLAG_DISABLE_MULTIPLANE;
+ }
+
+ ret = av_hwframe_ctx_init(ctx->hw_frame_ref);
+ if (ret < 0) {
+ av_log(renderer, AV_LOG_ERROR, "Create hwframe context failed, %s\n",
+ av_err2str(ret));
+ return ret;
+ }
+
+ av_hwframe_transfer_get_formats(ctx->hw_frame_ref,
+ AV_HWFRAME_TRANSFER_DIRECTION_TO,
+ &ctx->transfer_formats, 0);
+
+ return 0;
+}
+
+static inline int check_hw_transfer(RendererContext *ctx, AVFrame *frame)
+{
+ if (!ctx->hw_frame_ref || !ctx->transfer_formats)
+ return 0;
+
+ for (int i = 0; ctx->transfer_formats[i] != AV_PIX_FMT_NONE; i++)
+ if (ctx->transfer_formats[i] == frame->format)
+ return 1;
+
+ return 0;
+}
+
+static int transfer_frame(VkRenderer *renderer, AVFrame *frame)
+{
+ RendererContext *ctx = (RendererContext *) renderer;
+ int ret;
+
+ if (!frame->hw_frames_ctx)
+ return 0;
+
+ if (frame->format == AV_PIX_FMT_VULKAN)
+ return 0;
+
+ ret = create_hw_frame(renderer, frame);
+ if (ret < 0)
+ return ret;
+
+ // Try map data first
+ av_frame_unref(ctx->vk_frame);
+ if (ctx->hw_frame_ref) {
+ ctx->vk_frame->hw_frames_ctx = av_buffer_ref(ctx->hw_frame_ref);
+ ctx->vk_frame->format = AV_PIX_FMT_VULKAN;
+ }
+ ret = av_hwframe_map(ctx->vk_frame, frame, AV_HWFRAME_MAP_READ);
+ if (!ret)
+ goto out;
+
+ if (ret != AVERROR(ENOSYS)) {
+ av_log(NULL, AV_LOG_FATAL, "Map data to vulkan failed: %s\n",
+ av_err2str(ret));
+ return ret;
+ }
+
+ // Try transfer to vulkan
+ if (check_hw_transfer(ctx, frame)) {
+ av_frame_unref(ctx->vk_frame);
+ av_hwframe_get_buffer(ctx->hw_frame_ref, ctx->vk_frame, 0);
+ ret = av_hwframe_transfer_data(ctx->vk_frame, frame, 1);
+ if (!ret)
+ goto out;
+
+ if (ret < 0 && ret != AVERROR(ENOSYS)) {
+ av_log(NULL, AV_LOG_FATAL, "Transfer data to vulkan failed: %s\n",
+ av_err2str(ret));
+ return ret;
+ }
+ }
+
+ // Transfer to CPU
+ av_frame_unref(ctx->vk_frame);
+ ret = av_hwframe_transfer_data(ctx->vk_frame, frame, 0);
+ if (ret < 0) {
+ av_log(NULL, AV_LOG_FATAL, "Transfer data to CPU failed: %s\n",
+ av_err2str(ret));
+ return ret;
+ }
+
+ out:
+ ret = av_frame_copy_props(ctx->vk_frame, frame);
+ if (ret < 0)
+ return ret;
+ av_frame_unref(frame);
+ av_frame_move_ref(frame, ctx->vk_frame);
+
+ return 0;
+}
+
static int display(VkRenderer *renderer, AVFrame *frame)
{
struct pl_swapchain_frame swap_frame = {0};
struct pl_frame pl_frame = {0};
struct pl_frame target = {0};
- RendererContext *ctx = (RendererContext *)renderer;
+ RendererContext *ctx = (RendererContext *) renderer;
int ret = 0;
+ ret = transfer_frame(renderer, frame);
+ if (ret < 0)
+ return ret;
+
if (!pl_map_avframe_ex(ctx->pl_vk->gpu, &pl_frame, pl_avframe_params(
.frame = frame,
.tex = ctx->tex))) {
@@ -166,7 +505,8 @@ static int display(VkRenderer *renderer, AVFrame *frame)
}
pl_frame_from_swapchain(&target, &swap_frame);
- if (!pl_render_image(ctx->renderer, &pl_frame, &target, &pl_render_default_params)) {
+ if (!pl_render_image(ctx->renderer, &pl_frame, &target,
+ &pl_render_default_params)) {
av_log(NULL, AV_LOG_ERROR, "pl_render_image failed\n");
ret = AVERROR_EXTERNAL;
goto out;
@@ -179,14 +519,14 @@ static int display(VkRenderer *renderer, AVFrame *frame)
}
pl_swapchain_swap_buffers(ctx->swapchain);
-out:
+ out:
pl_unmap_avframe(ctx->pl_vk->gpu, &pl_frame);
return ret;
}
static int resize(VkRenderer *renderer, int width, int height)
{
- RendererContext *ctx = (RendererContext *)renderer;
+ RendererContext *ctx = (RendererContext *) renderer;
if (!pl_swapchain_resize(ctx->swapchain, &width, &height))
return AVERROR_EXTERNAL;
@@ -196,7 +536,13 @@ static int resize(VkRenderer *renderer, int width, int height)
static void destroy(VkRenderer *renderer)
{
PFN_vkDestroySurfaceKHR vkDestroySurfaceKHR;
- RendererContext *ctx = (RendererContext *)renderer;
+ RendererContext *ctx = (RendererContext *) renderer;
+
+ av_freep(&ctx->transfer_formats);
+ av_hwframe_constraints_free(&ctx->constraints);
+ av_buffer_unref(&ctx->hw_frame_ref);
+ av_buffer_unref(&ctx->hw_device_ref);
+ av_frame_free(&ctx->vk_frame);
if (ctx->pl_vk) {
for (int i = 0; i < FF_ARRAY_ELEMS(ctx->tex); i++)
@@ -208,7 +554,8 @@ static void destroy(VkRenderer *renderer)
if (ctx->vk_surface) {
vkDestroySurfaceKHR = (PFN_vkDestroySurfaceKHR)
- ctx->vk_inst->get_proc_addr(ctx->vk_inst->instance, "vkDestroySurfaceKHR");
+ ctx->vk_inst->get_proc_addr(ctx->vk_inst->instance,
+ "vkDestroySurfaceKHR");
vkDestroySurfaceKHR(ctx->vk_inst->instance, ctx->vk_surface, NULL);
ctx->vk_surface = NULL;
}
@@ -233,6 +580,7 @@ VkRenderer *vk_get_renderer(void)
renderer = &ctx->api;
renderer->class = &vulkan_renderer_class;
+ renderer->get_hw_dev = get_hw_dev;
renderer->create = create;
renderer->display = display;
renderer->resize = resize;
@@ -255,6 +603,11 @@ int vk_renderer_create(VkRenderer *renderer, SDL_Window *window)
return renderer->create(renderer, window);
}
+int vk_renderer_get_hw_dev(VkRenderer *renderer, AVBufferRef **dev)
+{
+ return renderer->get_hw_dev(renderer, dev);
+}
+
int vk_renderer_display(VkRenderer *renderer, AVFrame *frame)
{
return renderer->display(renderer, frame);
diff --git a/fftools/ffplay_renderer.h b/fftools/ffplay_renderer.h
index 4029f154a9..b982ba0192 100644
--- a/fftools/ffplay_renderer.h
+++ b/fftools/ffplay_renderer.h
@@ -29,6 +29,8 @@ VkRenderer *vk_get_renderer(void);
int vk_renderer_create(VkRenderer *renderer, SDL_Window *window);
+int vk_renderer_get_hw_dev(VkRenderer *renderer, AVBufferRef **dev);
+
int vk_renderer_display(VkRenderer *renderer, AVFrame *frame);
int vk_renderer_resize(VkRenderer *renderer, int width, int height);
--
2.34.1
_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
^ permalink raw reply [flat|nested] 7+ messages in thread
* Re: [FFmpeg-devel] [PATCH v3 2/2] fftools/ffplay: add hwaccel decoding support
2023-10-28 19:32 ` Lynne
2023-10-29 2:23 ` [FFmpeg-devel] [PATCH v4 1/2] fftools/ffplay: add vulkan renderer via libplacebo Zhao Zhili
[not found] ` <20231029022347.15243-1-quinkblack@foxmail.com>
@ 2023-10-29 2:31 ` Zhao Zhili
2023-10-29 3:18 ` Lynne
2023-10-29 11:12 ` Niklas Haas
2 siblings, 2 replies; 7+ messages in thread
From: Zhao Zhili @ 2023-10-29 2:31 UTC (permalink / raw)
To: ffmpeg-devel
On 2023/10/29 03:32, Lynne wrote:
> Oct 28, 2023, 20:14 byquinkblack@foxmail.com:
>
>> From: Zhao Zhili<zhilizhao@tencent.com>
>>
>> ---
>> v3: shared vulkan instance between libplacebo and hwcontext
>>
>> fftools/ffplay.c | 43 ++++++
>> fftools/ffplay_renderer.c | 316 +++++++++++++++++++++++++++++++++++++-
>> fftools/ffplay_renderer.h | 2 +
>> 3 files changed, 355 insertions(+), 6 deletions(-)
>>
>> diff --git a/fftools/ffplay.c b/fftools/ffplay.c
>> index 305d72d8b8..29b37a5e46 100644
>> --- a/fftools/ffplay.c
>> +++ b/fftools/ffplay.c
>> @@ -352,6 +352,7 @@ static int autorotate = 1;
>> static int find_stream_info = 1;
>> static int filter_nbthreads = 0;
>> static int enable_vulkan = 0;
>> +static const char *hwaccel = NULL;
>>
>> /* current context */
>> static int is_full_screen;
>> @@ -2557,6 +2558,37 @@ static int audio_open(void *opaque, AVChannelLayout *wanted_channel_layout, int
>> return spec.size;
>> }
>>
>> +static int create_hwaccel(AVBufferRef **device_ctx)
>> +{
>> + enum AVHWDeviceType type;
>> + int ret;
>> + AVBufferRef *vk_dev;
>> +
>> + *device_ctx = NULL;
>> +
>> + if (!hwaccel)
>> + return 0;
>> +
>> + type = av_hwdevice_find_type_by_name(hwaccel);
>> + if (type == AV_HWDEVICE_TYPE_NONE)
>> + return AVERROR(ENOTSUP);
>> +
>> + ret = vk_renderer_get_hw_dev(vk_renderer, &vk_dev);
>> + if (ret < 0)
>> + return ret;
>> +
>> + ret = av_hwdevice_ctx_create_derived(device_ctx, type, vk_dev, 0);
>> + if (!ret)
>> + return 0;
>> +
>> + if (ret != AVERROR(ENOSYS))
>> + return ret;
>> +
>> + av_log(NULL, AV_LOG_WARNING, "Derive %s from vulkan not supported.\n", hwaccel);
>> + ret = av_hwdevice_ctx_create(device_ctx, type, NULL, NULL, 0);
>> + return ret;
>> +}
>> +
>> /* open a given stream. Return 0 if OK */
>> static int stream_component_open(VideoState *is, int stream_index)
>> {
>> @@ -2624,6 +2656,12 @@ static int stream_component_open(VideoState *is, int stream_index)
>>
>> av_dict_set(&opts, "flags", "+copy_opaque", AV_DICT_MULTIKEY);
>>
>> + if (avctx->codec_type == AVMEDIA_TYPE_VIDEO) {
>> + ret = create_hwaccel(&avctx->hw_device_ctx);
>> + if (ret < 0)
>> + goto fail;
>> + }
>> +
>> if ((ret = avcodec_open2(avctx, codec, &opts)) < 0) {
>> goto fail;
>> }
>> @@ -3625,6 +3663,7 @@ static const OptionDef options[] = {
>> "read and decode the streams to fill missing information with heuristics" },
>> { "filter_threads", HAS_ARG | OPT_INT | OPT_EXPERT, { &filter_nbthreads }, "number of filter threads per graph" },
>> { "enable_vulkan", OPT_BOOL, { &enable_vulkan }, "enable vulkan render" },
>> + { "hwaccel", HAS_ARG | OPT_STRING | OPT_EXPERT, { &hwaccel }, "use HW accelerated decoding" },
>> { NULL, },
>> };
>>
>> @@ -3739,6 +3778,10 @@ int main(int argc, char **argv)
>> #ifdef SDL_HINT_VIDEO_X11_NET_WM_BYPASS_COMPOSITOR
>> SDL_SetHint(SDL_HINT_VIDEO_X11_NET_WM_BYPASS_COMPOSITOR, "0");
>> #endif
>> + if (hwaccel && !enable_vulkan) {
>> + av_log(NULL, AV_LOG_INFO, "Enable vulkan renderer to support hwaccel %s\n", hwaccel);
>> + enable_vulkan = 1;
>> + }
>> if (enable_vulkan) {
>> vk_renderer = vk_get_renderer();
>> if (vk_renderer) {
>> diff --git a/fftools/ffplay_renderer.c b/fftools/ffplay_renderer.c
>> index 49e5516d5d..497d758ec7 100644
>> --- a/fftools/ffplay_renderer.c
>> +++ b/fftools/ffplay_renderer.c
>> @@ -35,6 +35,8 @@ struct VkRenderer {
>>
>> int (*create)(VkRenderer *renderer, SDL_Window *window);
>>
>> + int (*get_hw_dev)(VkRenderer *renderer, AVBufferRef **dev);
>> +
>> int (*display)(VkRenderer *renderer, AVFrame *frame);
>>
>> int (*resize)(VkRenderer *renderer, int width, int height);
>> @@ -54,6 +56,13 @@ typedef struct RendererContext {
>> pl_tex tex[4];
>>
>> pl_log vk_log;
>> +
>> + AVBufferRef *hw_device_ref;
>> + AVBufferRef *hw_frame_ref;
>> + enum AVPixelFormat *transfer_formats;
>> + AVHWFramesConstraints *constraints;
>> +
>> + AVFrame *vk_frame;
>> } RendererContext;
>>
>> static void vk_log_cb(void *log_priv, enum pl_log_level level, const char *msg) {
>> @@ -74,35 +83,46 @@ static void vk_log_cb(void *log_priv, enum pl_log_level level, const char *msg)
>> static int create(VkRenderer *renderer, SDL_Window *window)
>> {
>> int ret = 0;
>> - unsigned ext_num = 0;
>> + unsigned num_ext = 0;
>> const char **ext = NULL;
>> int w, h;
>> struct pl_log_params vk_log_params = {
>> .log_cb = vk_log_cb,
>> - .log_level = PL_LOG_WARN,
>> + .log_level = PL_LOG_DEBUG,
>> .log_priv = renderer,
>> };
>> RendererContext *ctx = (RendererContext *)renderer;
>>
>> - if (!SDL_Vulkan_GetInstanceExtensions(window, &ext_num, NULL)) {
>> + static const char *opt_exts[] = {
>> + VK_EXT_DESCRIPTOR_BUFFER_EXTENSION_NAME,
>> + VK_EXT_SHADER_ATOMIC_FLOAT_EXTENSION_NAME,
>> + VK_KHR_VIDEO_DECODE_QUEUE_EXTENSION_NAME,
>> + VK_KHR_VIDEO_DECODE_H264_EXTENSION_NAME,
>> + VK_KHR_VIDEO_DECODE_H265_EXTENSION_NAME,
>> + VK_KHR_VIDEO_QUEUE_EXTENSION_NAME,
>> + "VK_MESA_video_decode_av1",
>> + };
>> + const int num_opt_exts = FF_ARRAY_ELEMS(opt_exts);
>> +
>> + if (!SDL_Vulkan_GetInstanceExtensions(window, &num_ext, NULL)) {
>> av_log(NULL, AV_LOG_FATAL, "Failed to get vulkan extensions: %s\n", SDL_GetError());
>> return AVERROR_EXTERNAL;
>>
> That's not a complete list. Look at optional_device_exts.
> There are also device features that both we and libplacebo would like
> to be enabled. Look into vulkan_device_create_internal(), the list there
> is up to date. FFmpeg requires timeline semaphores, and descriptor_buffer
> for lavfi filters to work.
Fixed in patch v4.
Maybe we should export optional_device_exts in API.
>
> Is there no option to let libavutil create the hwdevice for both SDL and libplacebo?
> That way, that code path can also be tested.
libplacebo doesn't support that:
// Creates a new vulkan device based on the given parameters and initializes
// a new GPU. This function will internally initialize a VkDevice. There is
// currently no way to share a vulkan device with the caller. If `params` is
// left as NULL, it defaults to &pl_vulkan_default_params.
//
// Thread-safety: Safe
PL_API pl_vulkan pl_vulkan_create(pl_log log, const struct
pl_vulkan_params *params);
I think share instance or vkImage without support from vulkan spec is
hard to implementation,
if not impossible.
https://www.yosoygames.com.ar/wp/2022/07/vulkan-integration-between-multiple-libraries-e-g-qt-ogrenext/
> _______________________________________________
> ffmpeg-devel mailing list
> ffmpeg-devel@ffmpeg.org
> https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
>
> To unsubscribe, visit link above, or email
> ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
^ permalink raw reply [flat|nested] 7+ messages in thread
* Re: [FFmpeg-devel] [PATCH v3 2/2] fftools/ffplay: add hwaccel decoding support
2023-10-29 2:31 ` [FFmpeg-devel] [PATCH v3 " Zhao Zhili
@ 2023-10-29 3:18 ` Lynne
2023-10-29 11:12 ` Niklas Haas
1 sibling, 0 replies; 7+ messages in thread
From: Lynne @ 2023-10-29 3:18 UTC (permalink / raw)
To: FFmpeg development discussions and patches
Oct 29, 2023, 03:31 by quinkblack@foxmail.com:
>
> On 2023/10/29 03:32, Lynne wrote:
>
>> Oct 28, 2023, 20:14 byquinkblack@foxmail.com:
>>
>
> Fixed in patch v4.
>
> Maybe we should export optional_device_exts in API.
>
Patch welcome.
>>
>> Is there no option to let libavutil create the hwdevice for both SDL and libplacebo?
>> That way, that code path can also be tested.
>>
>
> libplacebo doesn't support that:
>
It does. It's how vf_libplacebo runs, and I also use it in my sample library (txproto).
You should do it this way.
> // Creates a new vulkan device based on the given parameters and initializes
> // a new GPU. This function will internally initialize a VkDevice. There is
> // currently no way to share a vulkan device with the caller. If `params` is
> // left as NULL, it defaults to &pl_vulkan_default_params.
> //
> // Thread-safety: Safe
> PL_API pl_vulkan pl_vulkan_create(pl_log log, const struct pl_vulkan_params *params);
>
> I think share instance or vkImage without support from vulkan spec is hard to implementation,
>
> if not impossible.
>
> https://www.yosoygames.com.ar/wp/2022/07/vulkan-integration-between-multiple-libraries-e-g-qt-ogrenext/
>
That's why we have locking callbacks for AVVkFrame, and for VkQueues.
It's all done for you, as long as you use the pl_map_avframe_ex().
_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
^ permalink raw reply [flat|nested] 7+ messages in thread
* Re: [FFmpeg-devel] [PATCH v3 2/2] fftools/ffplay: add hwaccel decoding support
2023-10-29 2:31 ` [FFmpeg-devel] [PATCH v3 " Zhao Zhili
2023-10-29 3:18 ` Lynne
@ 2023-10-29 11:12 ` Niklas Haas
1 sibling, 0 replies; 7+ messages in thread
From: Niklas Haas @ 2023-10-29 11:12 UTC (permalink / raw)
To: ffmpeg-devel
On Sun, 29 Oct 2023 10:31:17 +0800 Zhao Zhili <quinkblack@foxmail.com> wrote:
> libplacebo doesn't support that:
>
> // Creates a new vulkan device based on the given parameters and initializes
> // a new GPU. This function will internally initialize a VkDevice. There is
> // currently no way to share a vulkan device with the caller. If `params` is
> // left as NULL, it defaults to &pl_vulkan_default_params.
> //
> // Thread-safety: Safe
> PL_API pl_vulkan pl_vulkan_create(pl_log log, const struct
> pl_vulkan_params *params);
Pretty sure this comment is ancient and wrong/misleading, I removed it
in master.
_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
^ permalink raw reply [flat|nested] 7+ messages in thread
end of thread, other threads:[~2023-10-29 11:12 UTC | newest]
Thread overview: 7+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
[not found] <20231028181341.13090-1-quinkblack@foxmail.com>
2023-10-28 18:13 ` [FFmpeg-devel] [PATCH v3 2/2] fftools/ffplay: add hwaccel decoding support Zhao Zhili
2023-10-28 19:32 ` Lynne
2023-10-29 2:23 ` [FFmpeg-devel] [PATCH v4 1/2] fftools/ffplay: add vulkan renderer via libplacebo Zhao Zhili
[not found] ` <20231029022347.15243-1-quinkblack@foxmail.com>
2023-10-29 2:23 ` [FFmpeg-devel] [PATCH v4 2/2] fftools/ffplay: add hwaccel decoding support Zhao Zhili
2023-10-29 2:31 ` [FFmpeg-devel] [PATCH v3 " Zhao Zhili
2023-10-29 3:18 ` Lynne
2023-10-29 11:12 ` Niklas Haas
Git Inbox Mirror of the ffmpeg-devel mailing list - see https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
This inbox may be cloned and mirrored by anyone:
git clone --mirror https://master.gitmailbox.com/ffmpegdev/0 ffmpegdev/git/0.git
# If you have public-inbox 1.1+ installed, you may
# initialize and index your mirror using the following commands:
public-inbox-init -V2 ffmpegdev ffmpegdev/ https://master.gitmailbox.com/ffmpegdev \
ffmpegdev@gitmailbox.com
public-inbox-index ffmpegdev
Example config snippet for mirrors.
AGPL code for this site: git clone https://public-inbox.org/public-inbox.git