* [FFmpeg-devel] [PATCH v3 2/2] fftools/ffplay: add hwaccel decoding support [not found] <20231028181341.13090-1-quinkblack@foxmail.com> @ 2023-10-28 18:13 ` Zhao Zhili 2023-10-28 19:32 ` Lynne 0 siblings, 1 reply; 7+ messages in thread From: Zhao Zhili @ 2023-10-28 18:13 UTC (permalink / raw) To: ffmpeg-devel; +Cc: Zhao Zhili From: Zhao Zhili <zhilizhao@tencent.com> --- v3: shared vulkan instance between libplacebo and hwcontext fftools/ffplay.c | 43 ++++++ fftools/ffplay_renderer.c | 316 +++++++++++++++++++++++++++++++++++++- fftools/ffplay_renderer.h | 2 + 3 files changed, 355 insertions(+), 6 deletions(-) diff --git a/fftools/ffplay.c b/fftools/ffplay.c index 305d72d8b8..29b37a5e46 100644 --- a/fftools/ffplay.c +++ b/fftools/ffplay.c @@ -352,6 +352,7 @@ static int autorotate = 1; static int find_stream_info = 1; static int filter_nbthreads = 0; static int enable_vulkan = 0; +static const char *hwaccel = NULL; /* current context */ static int is_full_screen; @@ -2557,6 +2558,37 @@ static int audio_open(void *opaque, AVChannelLayout *wanted_channel_layout, int return spec.size; } +static int create_hwaccel(AVBufferRef **device_ctx) +{ + enum AVHWDeviceType type; + int ret; + AVBufferRef *vk_dev; + + *device_ctx = NULL; + + if (!hwaccel) + return 0; + + type = av_hwdevice_find_type_by_name(hwaccel); + if (type == AV_HWDEVICE_TYPE_NONE) + return AVERROR(ENOTSUP); + + ret = vk_renderer_get_hw_dev(vk_renderer, &vk_dev); + if (ret < 0) + return ret; + + ret = av_hwdevice_ctx_create_derived(device_ctx, type, vk_dev, 0); + if (!ret) + return 0; + + if (ret != AVERROR(ENOSYS)) + return ret; + + av_log(NULL, AV_LOG_WARNING, "Derive %s from vulkan not supported.\n", hwaccel); + ret = av_hwdevice_ctx_create(device_ctx, type, NULL, NULL, 0); + return ret; +} + /* open a given stream. Return 0 if OK */ static int stream_component_open(VideoState *is, int stream_index) { @@ -2624,6 +2656,12 @@ static int stream_component_open(VideoState *is, int stream_index) av_dict_set(&opts, "flags", "+copy_opaque", AV_DICT_MULTIKEY); + if (avctx->codec_type == AVMEDIA_TYPE_VIDEO) { + ret = create_hwaccel(&avctx->hw_device_ctx); + if (ret < 0) + goto fail; + } + if ((ret = avcodec_open2(avctx, codec, &opts)) < 0) { goto fail; } @@ -3625,6 +3663,7 @@ static const OptionDef options[] = { "read and decode the streams to fill missing information with heuristics" }, { "filter_threads", HAS_ARG | OPT_INT | OPT_EXPERT, { &filter_nbthreads }, "number of filter threads per graph" }, { "enable_vulkan", OPT_BOOL, { &enable_vulkan }, "enable vulkan render" }, + { "hwaccel", HAS_ARG | OPT_STRING | OPT_EXPERT, { &hwaccel }, "use HW accelerated decoding" }, { NULL, }, }; @@ -3739,6 +3778,10 @@ int main(int argc, char **argv) #ifdef SDL_HINT_VIDEO_X11_NET_WM_BYPASS_COMPOSITOR SDL_SetHint(SDL_HINT_VIDEO_X11_NET_WM_BYPASS_COMPOSITOR, "0"); #endif + if (hwaccel && !enable_vulkan) { + av_log(NULL, AV_LOG_INFO, "Enable vulkan renderer to support hwaccel %s\n", hwaccel); + enable_vulkan = 1; + } if (enable_vulkan) { vk_renderer = vk_get_renderer(); if (vk_renderer) { diff --git a/fftools/ffplay_renderer.c b/fftools/ffplay_renderer.c index 49e5516d5d..497d758ec7 100644 --- a/fftools/ffplay_renderer.c +++ b/fftools/ffplay_renderer.c @@ -35,6 +35,8 @@ struct VkRenderer { int (*create)(VkRenderer *renderer, SDL_Window *window); + int (*get_hw_dev)(VkRenderer *renderer, AVBufferRef **dev); + int (*display)(VkRenderer *renderer, AVFrame *frame); int (*resize)(VkRenderer *renderer, int width, int height); @@ -54,6 +56,13 @@ typedef struct RendererContext { pl_tex tex[4]; pl_log vk_log; + + AVBufferRef *hw_device_ref; + AVBufferRef *hw_frame_ref; + enum AVPixelFormat *transfer_formats; + AVHWFramesConstraints *constraints; + + AVFrame *vk_frame; } RendererContext; static void vk_log_cb(void *log_priv, enum pl_log_level level, const char *msg) { @@ -74,35 +83,46 @@ static void vk_log_cb(void *log_priv, enum pl_log_level level, const char *msg) static int create(VkRenderer *renderer, SDL_Window *window) { int ret = 0; - unsigned ext_num = 0; + unsigned num_ext = 0; const char **ext = NULL; int w, h; struct pl_log_params vk_log_params = { .log_cb = vk_log_cb, - .log_level = PL_LOG_WARN, + .log_level = PL_LOG_DEBUG, .log_priv = renderer, }; RendererContext *ctx = (RendererContext *)renderer; - if (!SDL_Vulkan_GetInstanceExtensions(window, &ext_num, NULL)) { + static const char *opt_exts[] = { + VK_EXT_DESCRIPTOR_BUFFER_EXTENSION_NAME, + VK_EXT_SHADER_ATOMIC_FLOAT_EXTENSION_NAME, + VK_KHR_VIDEO_DECODE_QUEUE_EXTENSION_NAME, + VK_KHR_VIDEO_DECODE_H264_EXTENSION_NAME, + VK_KHR_VIDEO_DECODE_H265_EXTENSION_NAME, + VK_KHR_VIDEO_QUEUE_EXTENSION_NAME, + "VK_MESA_video_decode_av1", + }; + const int num_opt_exts = FF_ARRAY_ELEMS(opt_exts); + + if (!SDL_Vulkan_GetInstanceExtensions(window, &num_ext, NULL)) { av_log(NULL, AV_LOG_FATAL, "Failed to get vulkan extensions: %s\n", SDL_GetError()); return AVERROR_EXTERNAL; } - ext = av_calloc(ext_num, sizeof(*ext)); + ext = av_calloc(num_ext, sizeof(*ext)); if (!ext) { ret = AVERROR(ENOMEM); goto out; } - SDL_Vulkan_GetInstanceExtensions(window, &ext_num, ext); + SDL_Vulkan_GetInstanceExtensions(window, &num_ext, ext); ctx->vk_log = pl_log_create(PL_API_VER, &vk_log_params); ctx->vk_inst = pl_vk_inst_create(ctx->vk_log, pl_vk_inst_params( .get_proc_addr = SDL_Vulkan_GetVkGetInstanceProcAddr(), .debug = false, .extensions = ext, - .num_extensions = ext_num, + .num_extensions = num_ext )); if (!ctx->vk_inst) { ret = AVERROR_EXTERNAL; @@ -119,6 +139,9 @@ static int create(VkRenderer *renderer, SDL_Window *window) .get_proc_addr = ctx->vk_inst->get_proc_addr, .surface = ctx->vk_surface, .allow_software = false, + .opt_extensions = opt_exts, + .num_opt_extensions = num_opt_exts, + .extra_queues = VK_QUEUE_VIDEO_DECODE_BIT_KHR, )); if (!ctx->pl_vk) { ret = AVERROR_EXTERNAL; @@ -137,6 +160,13 @@ static int create(VkRenderer *renderer, SDL_Window *window) ret = AVERROR_EXTERNAL; goto out; } + + ctx->vk_frame = av_frame_alloc(); + if (!ctx->vk_frame) { + ret = AVERROR(ENOMEM); + goto out; + } + ret = 0; out: @@ -144,6 +174,264 @@ out: return ret; } +static void lock_queue(struct AVHWDeviceContext *dev_ctx, uint32_t queue_family, uint32_t index) +{ + RendererContext *ctx = dev_ctx->user_opaque; + pl_vulkan vk = ctx->pl_vk; + vk->lock_queue(vk, queue_family, index); +} + +static void unlock_queue(struct AVHWDeviceContext *dev_ctx, uint32_t queue_family, uint32_t index) +{ + RendererContext *ctx = dev_ctx->user_opaque; + pl_vulkan vk = ctx->pl_vk; + vk->unlock_queue(vk, queue_family, index); +} + +static int get_decode_queue(VkRenderer *renderer, int *index, int *count) +{ + RendererContext *ctx = (RendererContext *) renderer; + VkQueueFamilyProperties *queue_family_prop = NULL; + uint32_t num_queue_family_prop = 0; + PFN_vkGetPhysicalDeviceQueueFamilyProperties vkGetPhysicalDeviceQueueFamilyProperties; + + *index = -1; + *count = 0; + vkGetPhysicalDeviceQueueFamilyProperties = (PFN_vkGetPhysicalDeviceQueueFamilyProperties) + ctx->vk_inst->get_proc_addr(ctx->vk_inst->instance, + "vkGetPhysicalDeviceQueueFamilyProperties"); + vkGetPhysicalDeviceQueueFamilyProperties(ctx->pl_vk->phys_device, &num_queue_family_prop, NULL); + if (!num_queue_family_prop) + return AVERROR_EXTERNAL; + + queue_family_prop = av_calloc(num_queue_family_prop, sizeof(*queue_family_prop)); + if (!queue_family_prop) + return AVERROR(ENOMEM); + + vkGetPhysicalDeviceQueueFamilyProperties(ctx->pl_vk->phys_device, &num_queue_family_prop, queue_family_prop); + + for (int i = 0; i < num_queue_family_prop; i++) { + if (queue_family_prop[i].queueFlags & VK_QUEUE_VIDEO_DECODE_BIT_KHR) { + *index = i; + *count = queue_family_prop[i].queueCount; + break; + } + } + av_free(queue_family_prop); + + return 0; +} + +static int get_hw_dev(VkRenderer *renderer, AVBufferRef **dev) +{ + RendererContext *ctx = (RendererContext *)renderer; + AVHWDeviceContext *device_ctx; + AVVulkanDeviceContext *vk_dev_ctx; + int decode_index; + int decode_count; + int ret; + + *dev = NULL; + if (ctx->hw_device_ref) { + *dev = ctx->hw_device_ref; + return 0; + } + + ctx->hw_device_ref = av_hwdevice_ctx_alloc(AV_HWDEVICE_TYPE_VULKAN); + if (!ctx->hw_device_ref) { + return AVERROR(ENOMEM); + } + + device_ctx = (AVHWDeviceContext *)ctx->hw_device_ref->data; + device_ctx->user_opaque = ctx; + + vk_dev_ctx = device_ctx->hwctx; + vk_dev_ctx->lock_queue = lock_queue, + vk_dev_ctx->unlock_queue = unlock_queue; + + vk_dev_ctx->get_proc_addr = ctx->vk_inst->get_proc_addr; + + vk_dev_ctx->inst = ctx->vk_inst->instance; + vk_dev_ctx->phys_dev = ctx->pl_vk->phys_device; + vk_dev_ctx->act_dev = ctx->pl_vk->device; + + vk_dev_ctx->device_features = *ctx->pl_vk->features; + + vk_dev_ctx->enabled_inst_extensions = ctx->vk_inst->extensions; + vk_dev_ctx->nb_enabled_inst_extensions = ctx->vk_inst->num_extensions; + + vk_dev_ctx->enabled_dev_extensions = ctx->pl_vk->extensions; + vk_dev_ctx->nb_enabled_dev_extensions = ctx->pl_vk->num_extensions; + + vk_dev_ctx->queue_family_index = ctx->pl_vk->queue_graphics.index; + vk_dev_ctx->nb_graphics_queues = ctx->pl_vk->queue_graphics.count; + + vk_dev_ctx->queue_family_tx_index = ctx->pl_vk->queue_transfer.index; + vk_dev_ctx->nb_tx_queues = ctx->pl_vk->queue_transfer.count; + + vk_dev_ctx->queue_family_comp_index = ctx->pl_vk->queue_compute.index; + vk_dev_ctx->nb_comp_queues = ctx->pl_vk->queue_compute.count; + + ret = get_decode_queue(renderer, &decode_index, &decode_count); + if (ret < 0) + return ret; + + vk_dev_ctx->queue_family_decode_index = decode_index; + vk_dev_ctx->nb_decode_queues = decode_count; + + ret = av_hwdevice_ctx_init(ctx->hw_device_ref); + if (ret < 0) + return ret; + + *dev = ctx->hw_device_ref; + return 0; +} + +static int create_hw_frame(VkRenderer *renderer, AVFrame *frame) +{ + RendererContext *ctx = (RendererContext *)renderer; + AVHWFramesContext *src_hw_frame = (AVHWFramesContext *)frame->hw_frames_ctx->data; + AVHWFramesContext *hw_frame; + AVVulkanFramesContext *vk_frame_ctx; + int ret; + + if (ctx->hw_frame_ref) { + hw_frame = (AVHWFramesContext *)ctx->hw_frame_ref->data; + + if (hw_frame->width == frame->width && + hw_frame->height == frame->height && + hw_frame->sw_format == src_hw_frame->sw_format) + return 0; + + av_buffer_unref(&ctx->hw_frame_ref); + } + + if (!ctx->constraints) { + ctx->constraints = av_hwdevice_get_hwframe_constraints(ctx->hw_device_ref, NULL); + if (!ctx->constraints) + return AVERROR(ENOMEM); + } + + // Check constraints and skip create hwframe. + // Don't take it as error since we can fallback to memory copy from GPU to CPU. + if ((ctx->constraints->max_width && ctx->constraints->max_width < frame->width) || + (ctx->constraints->max_height && ctx->constraints->max_height < frame->height) || + (ctx->constraints->min_width && ctx->constraints->min_width > frame->width) || + (ctx->constraints->min_height && ctx->constraints->min_height > frame->height)) + return 0; + + if (ctx->constraints->valid_sw_formats) { + enum AVPixelFormat *sw_formats = ctx->constraints->valid_sw_formats; + while (*sw_formats != AV_PIX_FMT_NONE) { + if (*sw_formats == src_hw_frame->sw_format) + break; + sw_formats++; + } + if (*sw_formats == AV_PIX_FMT_NONE) + return 0; + } + + ctx->hw_frame_ref = av_hwframe_ctx_alloc(ctx->hw_device_ref); + if (!ctx->hw_frame_ref) + return AVERROR(ENOMEM); + + hw_frame = (AVHWFramesContext *)ctx->hw_frame_ref->data; + hw_frame->format = AV_PIX_FMT_VULKAN; + hw_frame->sw_format = src_hw_frame->sw_format; + hw_frame->width = frame->width; + hw_frame->height = frame->height; + + if (frame->format == AV_PIX_FMT_CUDA) { + vk_frame_ctx = hw_frame->hwctx; + vk_frame_ctx->flags = AV_VK_FRAME_FLAG_DISABLE_MULTIPLANE; + } + + ret = av_hwframe_ctx_init(ctx->hw_frame_ref); + if (ret < 0) { + av_log(renderer, AV_LOG_ERROR, "Create hwframe context failed, %s\n", av_err2str(ret)); + return ret; + } + + av_hwframe_transfer_get_formats(ctx->hw_frame_ref, AV_HWFRAME_TRANSFER_DIRECTION_TO, + &ctx->transfer_formats, 0); + + return 0; +} + +static inline int check_hw_transfer(RendererContext *ctx, AVFrame *frame) +{ + if (!ctx->hw_frame_ref || !ctx->transfer_formats) + return 0; + + for (int i = 0; ctx->transfer_formats[i] != AV_PIX_FMT_NONE; i++) + if (ctx->transfer_formats[i] == frame->format) + return 1; + + return 0; +} + +static int transfer_frame(VkRenderer *renderer, AVFrame *frame) +{ + RendererContext *ctx = (RendererContext *)renderer; + int ret; + + if (!frame->hw_frames_ctx) + return 0; + + if (frame->format == AV_PIX_FMT_VULKAN) + return 0; + + ret = create_hw_frame(renderer, frame); + if (ret < 0) + return ret; + + // Try map data first + av_frame_unref(ctx->vk_frame); + if (ctx->hw_frame_ref) { + ctx->vk_frame->hw_frames_ctx = av_buffer_ref(ctx->hw_frame_ref); + ctx->vk_frame->format = AV_PIX_FMT_VULKAN; + } + ret = av_hwframe_map(ctx->vk_frame, frame, AV_HWFRAME_MAP_READ); + if (!ret) + goto out; + + if (ret != AVERROR(ENOSYS)) { + av_log(NULL, AV_LOG_FATAL, "Map data to vulkan failed: %s\n", av_err2str(ret)); + return ret; + } + + // Try transfer to vulkan + if (check_hw_transfer(ctx, frame)) { + av_frame_unref(ctx->vk_frame); + av_hwframe_get_buffer(ctx->hw_frame_ref, ctx->vk_frame, 0); + ret = av_hwframe_transfer_data(ctx->vk_frame, frame, 1); + if (!ret) + goto out; + + if (ret < 0 && ret != AVERROR(ENOSYS)) { + av_log(NULL, AV_LOG_FATAL, "Transfer data to vulkan failed: %s\n", av_err2str(ret)); + return ret; + } + } + + // Transfer to CPU + av_frame_unref(ctx->vk_frame); + ret = av_hwframe_transfer_data(ctx->vk_frame, frame, 0); + if (ret < 0) { + av_log(NULL, AV_LOG_FATAL, "Transfer data to CPU failed: %s\n", av_err2str(ret)); + return ret; + } + +out: + ret = av_frame_copy_props(ctx->vk_frame, frame); + if (ret < 0) + return ret; + av_frame_unref(frame); + av_frame_move_ref(frame, ctx->vk_frame); + + return 0; +} + static int display(VkRenderer *renderer, AVFrame *frame) { struct pl_swapchain_frame swap_frame = {0}; @@ -152,6 +440,10 @@ static int display(VkRenderer *renderer, AVFrame *frame) RendererContext *ctx = (RendererContext *)renderer; int ret = 0; + ret = transfer_frame(renderer, frame); + if (ret < 0) + return ret; + if (!pl_map_avframe_ex(ctx->pl_vk->gpu, &pl_frame, pl_avframe_params( .frame = frame, .tex = ctx->tex))) { @@ -198,6 +490,12 @@ static void destroy(VkRenderer *renderer) PFN_vkDestroySurfaceKHR vkDestroySurfaceKHR; RendererContext *ctx = (RendererContext *)renderer; + av_freep(&ctx->transfer_formats); + av_hwframe_constraints_free(&ctx->constraints); + av_buffer_unref(&ctx->hw_frame_ref); + av_buffer_unref(&ctx->hw_device_ref); + av_frame_free(&ctx->vk_frame); + if (ctx->pl_vk) { for (int i = 0; i < FF_ARRAY_ELEMS(ctx->tex); i++) pl_tex_destroy(ctx->pl_vk->gpu, &ctx->tex[i]); @@ -233,6 +531,7 @@ VkRenderer *vk_get_renderer(void) renderer = &ctx->api; renderer->class = &vulkan_renderer_class; + renderer->get_hw_dev = get_hw_dev; renderer->create = create; renderer->display = display; renderer->resize = resize; @@ -255,6 +554,11 @@ int vk_renderer_create(VkRenderer *renderer, SDL_Window *window) return renderer->create(renderer, window); } +int vk_renderer_get_hw_dev(VkRenderer *renderer, AVBufferRef **dev) +{ + return renderer->get_hw_dev(renderer, dev); +} + int vk_renderer_display(VkRenderer *renderer, AVFrame *frame) { return renderer->display(renderer, frame); diff --git a/fftools/ffplay_renderer.h b/fftools/ffplay_renderer.h index 4029f154a9..b982ba0192 100644 --- a/fftools/ffplay_renderer.h +++ b/fftools/ffplay_renderer.h @@ -29,6 +29,8 @@ VkRenderer *vk_get_renderer(void); int vk_renderer_create(VkRenderer *renderer, SDL_Window *window); +int vk_renderer_get_hw_dev(VkRenderer *renderer, AVBufferRef **dev); + int vk_renderer_display(VkRenderer *renderer, AVFrame *frame); int vk_renderer_resize(VkRenderer *renderer, int width, int height); -- 2.42.0 _______________________________________________ ffmpeg-devel mailing list ffmpeg-devel@ffmpeg.org https://ffmpeg.org/mailman/listinfo/ffmpeg-devel To unsubscribe, visit link above, or email ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe". ^ permalink raw reply [flat|nested] 7+ messages in thread
* Re: [FFmpeg-devel] [PATCH v3 2/2] fftools/ffplay: add hwaccel decoding support 2023-10-28 18:13 ` [FFmpeg-devel] [PATCH v3 2/2] fftools/ffplay: add hwaccel decoding support Zhao Zhili @ 2023-10-28 19:32 ` Lynne 2023-10-29 2:23 ` [FFmpeg-devel] [PATCH v4 1/2] fftools/ffplay: add vulkan renderer via libplacebo Zhao Zhili ` (2 more replies) 0 siblings, 3 replies; 7+ messages in thread From: Lynne @ 2023-10-28 19:32 UTC (permalink / raw) To: FFmpeg development discussions and patches Oct 28, 2023, 20:14 by quinkblack@foxmail.com: > From: Zhao Zhili <zhilizhao@tencent.com> > > --- > v3: shared vulkan instance between libplacebo and hwcontext > > fftools/ffplay.c | 43 ++++++ > fftools/ffplay_renderer.c | 316 +++++++++++++++++++++++++++++++++++++- > fftools/ffplay_renderer.h | 2 + > 3 files changed, 355 insertions(+), 6 deletions(-) > > diff --git a/fftools/ffplay.c b/fftools/ffplay.c > index 305d72d8b8..29b37a5e46 100644 > --- a/fftools/ffplay.c > +++ b/fftools/ffplay.c > @@ -352,6 +352,7 @@ static int autorotate = 1; > static int find_stream_info = 1; > static int filter_nbthreads = 0; > static int enable_vulkan = 0; > +static const char *hwaccel = NULL; > > /* current context */ > static int is_full_screen; > @@ -2557,6 +2558,37 @@ static int audio_open(void *opaque, AVChannelLayout *wanted_channel_layout, int > return spec.size; > } > > +static int create_hwaccel(AVBufferRef **device_ctx) > +{ > + enum AVHWDeviceType type; > + int ret; > + AVBufferRef *vk_dev; > + > + *device_ctx = NULL; > + > + if (!hwaccel) > + return 0; > + > + type = av_hwdevice_find_type_by_name(hwaccel); > + if (type == AV_HWDEVICE_TYPE_NONE) > + return AVERROR(ENOTSUP); > + > + ret = vk_renderer_get_hw_dev(vk_renderer, &vk_dev); > + if (ret < 0) > + return ret; > + > + ret = av_hwdevice_ctx_create_derived(device_ctx, type, vk_dev, 0); > + if (!ret) > + return 0; > + > + if (ret != AVERROR(ENOSYS)) > + return ret; > + > + av_log(NULL, AV_LOG_WARNING, "Derive %s from vulkan not supported.\n", hwaccel); > + ret = av_hwdevice_ctx_create(device_ctx, type, NULL, NULL, 0); > + return ret; > +} > + > /* open a given stream. Return 0 if OK */ > static int stream_component_open(VideoState *is, int stream_index) > { > @@ -2624,6 +2656,12 @@ static int stream_component_open(VideoState *is, int stream_index) > > av_dict_set(&opts, "flags", "+copy_opaque", AV_DICT_MULTIKEY); > > + if (avctx->codec_type == AVMEDIA_TYPE_VIDEO) { > + ret = create_hwaccel(&avctx->hw_device_ctx); > + if (ret < 0) > + goto fail; > + } > + > if ((ret = avcodec_open2(avctx, codec, &opts)) < 0) { > goto fail; > } > @@ -3625,6 +3663,7 @@ static const OptionDef options[] = { > "read and decode the streams to fill missing information with heuristics" }, > { "filter_threads", HAS_ARG | OPT_INT | OPT_EXPERT, { &filter_nbthreads }, "number of filter threads per graph" }, > { "enable_vulkan", OPT_BOOL, { &enable_vulkan }, "enable vulkan render" }, > + { "hwaccel", HAS_ARG | OPT_STRING | OPT_EXPERT, { &hwaccel }, "use HW accelerated decoding" }, > { NULL, }, > }; > > @@ -3739,6 +3778,10 @@ int main(int argc, char **argv) > #ifdef SDL_HINT_VIDEO_X11_NET_WM_BYPASS_COMPOSITOR > SDL_SetHint(SDL_HINT_VIDEO_X11_NET_WM_BYPASS_COMPOSITOR, "0"); > #endif > + if (hwaccel && !enable_vulkan) { > + av_log(NULL, AV_LOG_INFO, "Enable vulkan renderer to support hwaccel %s\n", hwaccel); > + enable_vulkan = 1; > + } > if (enable_vulkan) { > vk_renderer = vk_get_renderer(); > if (vk_renderer) { > diff --git a/fftools/ffplay_renderer.c b/fftools/ffplay_renderer.c > index 49e5516d5d..497d758ec7 100644 > --- a/fftools/ffplay_renderer.c > +++ b/fftools/ffplay_renderer.c > @@ -35,6 +35,8 @@ struct VkRenderer { > > int (*create)(VkRenderer *renderer, SDL_Window *window); > > + int (*get_hw_dev)(VkRenderer *renderer, AVBufferRef **dev); > + > int (*display)(VkRenderer *renderer, AVFrame *frame); > > int (*resize)(VkRenderer *renderer, int width, int height); > @@ -54,6 +56,13 @@ typedef struct RendererContext { > pl_tex tex[4]; > > pl_log vk_log; > + > + AVBufferRef *hw_device_ref; > + AVBufferRef *hw_frame_ref; > + enum AVPixelFormat *transfer_formats; > + AVHWFramesConstraints *constraints; > + > + AVFrame *vk_frame; > } RendererContext; > > static void vk_log_cb(void *log_priv, enum pl_log_level level, const char *msg) { > @@ -74,35 +83,46 @@ static void vk_log_cb(void *log_priv, enum pl_log_level level, const char *msg) > static int create(VkRenderer *renderer, SDL_Window *window) > { > int ret = 0; > - unsigned ext_num = 0; > + unsigned num_ext = 0; > const char **ext = NULL; > int w, h; > struct pl_log_params vk_log_params = { > .log_cb = vk_log_cb, > - .log_level = PL_LOG_WARN, > + .log_level = PL_LOG_DEBUG, > .log_priv = renderer, > }; > RendererContext *ctx = (RendererContext *)renderer; > > - if (!SDL_Vulkan_GetInstanceExtensions(window, &ext_num, NULL)) { > + static const char *opt_exts[] = { > + VK_EXT_DESCRIPTOR_BUFFER_EXTENSION_NAME, > + VK_EXT_SHADER_ATOMIC_FLOAT_EXTENSION_NAME, > + VK_KHR_VIDEO_DECODE_QUEUE_EXTENSION_NAME, > + VK_KHR_VIDEO_DECODE_H264_EXTENSION_NAME, > + VK_KHR_VIDEO_DECODE_H265_EXTENSION_NAME, > + VK_KHR_VIDEO_QUEUE_EXTENSION_NAME, > + "VK_MESA_video_decode_av1", > + }; > + const int num_opt_exts = FF_ARRAY_ELEMS(opt_exts); > + > + if (!SDL_Vulkan_GetInstanceExtensions(window, &num_ext, NULL)) { > av_log(NULL, AV_LOG_FATAL, "Failed to get vulkan extensions: %s\n", SDL_GetError()); > return AVERROR_EXTERNAL; > That's not a complete list. Look at optional_device_exts. There are also device features that both we and libplacebo would like to be enabled. Look into vulkan_device_create_internal(), the list there is up to date. FFmpeg requires timeline semaphores, and descriptor_buffer for lavfi filters to work. Is there no option to let libavutil create the hwdevice for both SDL and libplacebo? That way, that code path can also be tested. _______________________________________________ ffmpeg-devel mailing list ffmpeg-devel@ffmpeg.org https://ffmpeg.org/mailman/listinfo/ffmpeg-devel To unsubscribe, visit link above, or email ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe". ^ permalink raw reply [flat|nested] 7+ messages in thread
* [FFmpeg-devel] [PATCH v4 1/2] fftools/ffplay: add vulkan renderer via libplacebo 2023-10-28 19:32 ` Lynne @ 2023-10-29 2:23 ` Zhao Zhili [not found] ` <20231029022347.15243-1-quinkblack@foxmail.com> 2023-10-29 2:31 ` [FFmpeg-devel] [PATCH v3 " Zhao Zhili 2 siblings, 0 replies; 7+ messages in thread From: Zhao Zhili @ 2023-10-29 2:23 UTC (permalink / raw) To: ffmpeg-devel; +Cc: Zhao Zhili From: Zhao Zhili <zhilizhao@tencent.com> --- configure | 2 +- fftools/Makefile | 2 + fftools/ffplay.c | 46 ++++++- fftools/ffplay_renderer.c | 271 ++++++++++++++++++++++++++++++++++++++ fftools/ffplay_renderer.h | 38 ++++++ 5 files changed, 353 insertions(+), 6 deletions(-) create mode 100644 fftools/ffplay_renderer.c create mode 100644 fftools/ffplay_renderer.h diff --git a/configure b/configure index f494da204c..55329a5037 100755 --- a/configure +++ b/configure @@ -3903,7 +3903,7 @@ ffmpeg_select="aformat_filter anull_filter atrim_filter format_filter ffmpeg_suggest="ole32 psapi shell32" ffplay_deps="avcodec avformat avfilter swscale swresample sdl2" ffplay_select="crop_filter transpose_filter hflip_filter vflip_filter rotate_filter" -ffplay_suggest="shell32" +ffplay_suggest="shell32 libplacebo vulkan" ffprobe_deps="avcodec avformat" ffprobe_suggest="shell32" diff --git a/fftools/Makefile b/fftools/Makefile index 56820e6bc8..3c763e3db9 100644 --- a/fftools/Makefile +++ b/fftools/Makefile @@ -22,6 +22,8 @@ OBJS-ffmpeg += \ fftools/sync_queue.o \ fftools/thread_queue.o \ +OBJS-ffplay += fftools/ffplay_renderer.o + define DOFFTOOL OBJS-$(1) += fftools/cmdutils.o fftools/opt_common.o fftools/$(1).o $(OBJS-$(1)-yes) ifdef HAVE_GNU_WINDRES diff --git a/fftools/ffplay.c b/fftools/ffplay.c index d8c69e10bc..305d72d8b8 100644 --- a/fftools/ffplay.c +++ b/fftools/ffplay.c @@ -58,6 +58,7 @@ #include <SDL_thread.h> #include "cmdutils.h" +#include "ffplay_renderer.h" #include "opt_common.h" const char program_name[] = "ffplay"; @@ -350,6 +351,7 @@ static char *afilters = NULL; static int autorotate = 1; static int find_stream_info = 1; static int filter_nbthreads = 0; +static int enable_vulkan = 0; /* current context */ static int is_full_screen; @@ -362,6 +364,8 @@ static SDL_Renderer *renderer; static SDL_RendererInfo renderer_info = {0}; static SDL_AudioDeviceID audio_dev; +static VkRenderer *vk_renderer; + static const struct TextureFormatEntry { enum AVPixelFormat format; int texture_fmt; @@ -954,6 +958,11 @@ static void video_image_display(VideoState *is) SDL_Rect rect; vp = frame_queue_peek_last(&is->pictq); + if (vk_renderer) { + vk_renderer_display(vk_renderer, vp->frame); + return; + } + if (is->subtitle_st) { if (frame_queue_nb_remaining(&is->subpq) > 0) { sp = frame_queue_peek(&is->subpq); @@ -1289,6 +1298,8 @@ static void do_exit(VideoState *is) } if (renderer) SDL_DestroyRenderer(renderer); + if (vk_renderer) + vk_renderer_destroy(vk_renderer); if (window) SDL_DestroyWindow(window); uninit_opts(); @@ -3447,6 +3458,8 @@ static void event_loop(VideoState *cur_stream) SDL_DestroyTexture(cur_stream->vis_texture); cur_stream->vis_texture = NULL; } + if (vk_renderer) + vk_renderer_resize(vk_renderer, screen_width, screen_height); case SDL_WINDOWEVENT_EXPOSED: cur_stream->force_refresh = 1; } @@ -3611,6 +3624,7 @@ static const OptionDef options[] = { { "find_stream_info", OPT_BOOL | OPT_INPUT | OPT_EXPERT, { &find_stream_info }, "read and decode the streams to fill missing information with heuristics" }, { "filter_threads", HAS_ARG | OPT_INT | OPT_EXPERT, { &filter_nbthreads }, "number of filter threads per graph" }, + { "enable_vulkan", OPT_BOOL, { &enable_vulkan }, "enable vulkan render" }, { NULL, }, }; @@ -3725,9 +3739,31 @@ int main(int argc, char **argv) #ifdef SDL_HINT_VIDEO_X11_NET_WM_BYPASS_COMPOSITOR SDL_SetHint(SDL_HINT_VIDEO_X11_NET_WM_BYPASS_COMPOSITOR, "0"); #endif + if (enable_vulkan) { + vk_renderer = vk_get_renderer(); + if (vk_renderer) { +#if SDL_VERSION_ATLEAST(2, 0, 6) + flags |= SDL_WINDOW_VULKAN; +#endif + } else { + av_log(NULL, AV_LOG_WARNING, "Doesn't support vulkan renderer, fallback to SDL renderer\n"); + enable_vulkan = 0; + } + } window = SDL_CreateWindow(program_name, SDL_WINDOWPOS_UNDEFINED, SDL_WINDOWPOS_UNDEFINED, default_width, default_height, flags); SDL_SetHint(SDL_HINT_RENDER_SCALE_QUALITY, "linear"); - if (window) { + if (!window) { + av_log(NULL, AV_LOG_FATAL, "Failed to create window: %s", SDL_GetError()); + do_exit(NULL); + } + + if (vk_renderer) { + ret = vk_renderer_create(vk_renderer, window); + if (ret < 0) { + av_log(NULL, AV_LOG_FATAL, "Failed to create vulkan renderer, %s\n", av_err2str(ret)); + do_exit(NULL); + } + } else { renderer = SDL_CreateRenderer(window, -1, SDL_RENDERER_ACCELERATED | SDL_RENDERER_PRESENTVSYNC); if (!renderer) { av_log(NULL, AV_LOG_WARNING, "Failed to initialize a hardware accelerated renderer: %s\n", SDL_GetError()); @@ -3737,10 +3773,10 @@ int main(int argc, char **argv) if (!SDL_GetRendererInfo(renderer, &renderer_info)) av_log(NULL, AV_LOG_VERBOSE, "Initialized %s renderer.\n", renderer_info.name); } - } - if (!window || !renderer || !renderer_info.num_texture_formats) { - av_log(NULL, AV_LOG_FATAL, "Failed to create window or renderer: %s", SDL_GetError()); - do_exit(NULL); + if (!renderer || !renderer_info.num_texture_formats) { + av_log(NULL, AV_LOG_FATAL, "Failed to create window or renderer: %s", SDL_GetError()); + do_exit(NULL); + } } } diff --git a/fftools/ffplay_renderer.c b/fftools/ffplay_renderer.c new file mode 100644 index 0000000000..49e5516d5d --- /dev/null +++ b/fftools/ffplay_renderer.c @@ -0,0 +1,271 @@ +/* + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "config.h" +#include "ffplay_renderer.h" + +#define HAVE_VULKAN_RENDERER (SDL_VERSION_ATLEAST(2, 0, 6) && CONFIG_LIBPLACEBO) + +#if HAVE_VULKAN_RENDERER + +#include <libplacebo/vulkan.h> +#include <libplacebo/utils/frame_queue.h> +#include <libplacebo/utils/libav.h> +#include <SDL_vulkan.h> + +#endif + +struct VkRenderer { + const AVClass *class; + + int (*create)(VkRenderer *renderer, SDL_Window *window); + + int (*display)(VkRenderer *renderer, AVFrame *frame); + + int (*resize)(VkRenderer *renderer, int width, int height); + + void (*destroy)(VkRenderer *renderer); +}; + +#if HAVE_VULKAN_RENDERER + +typedef struct RendererContext { + VkRenderer api; + pl_vk_inst vk_inst; + pl_vulkan pl_vk; + pl_swapchain swapchain; + VkSurfaceKHR vk_surface; + pl_renderer renderer; + pl_tex tex[4]; + + pl_log vk_log; +} RendererContext; + +static void vk_log_cb(void *log_priv, enum pl_log_level level, const char *msg) { + static const int level_map[] = { + AV_LOG_QUIET, + AV_LOG_FATAL, + AV_LOG_ERROR, + AV_LOG_WARNING, + AV_LOG_INFO, + AV_LOG_DEBUG, + AV_LOG_TRACE, + }; + + if (level > 0 && level < FF_ARRAY_ELEMS(level_map)) + av_log(log_priv, level_map[level], "%s\n", msg); +} + +static int create(VkRenderer *renderer, SDL_Window *window) +{ + int ret = 0; + unsigned ext_num = 0; + const char **ext = NULL; + int w, h; + struct pl_log_params vk_log_params = { + .log_cb = vk_log_cb, + .log_level = PL_LOG_WARN, + .log_priv = renderer, + }; + RendererContext *ctx = (RendererContext *)renderer; + + if (!SDL_Vulkan_GetInstanceExtensions(window, &ext_num, NULL)) { + av_log(NULL, AV_LOG_FATAL, "Failed to get vulkan extensions: %s\n", SDL_GetError()); + return AVERROR_EXTERNAL; + } + + ext = av_calloc(ext_num, sizeof(*ext)); + if (!ext) { + ret = AVERROR(ENOMEM); + goto out; + } + + SDL_Vulkan_GetInstanceExtensions(window, &ext_num, ext); + + ctx->vk_log = pl_log_create(PL_API_VER, &vk_log_params); + ctx->vk_inst = pl_vk_inst_create(ctx->vk_log, pl_vk_inst_params( + .get_proc_addr = SDL_Vulkan_GetVkGetInstanceProcAddr(), + .debug = false, + .extensions = ext, + .num_extensions = ext_num, + )); + if (!ctx->vk_inst) { + ret = AVERROR_EXTERNAL; + goto out; + } + + if (!SDL_Vulkan_CreateSurface(window, ctx->vk_inst->instance, &ctx->vk_surface)) { + ret = AVERROR_EXTERNAL; + goto out; + } + + ctx->pl_vk = pl_vulkan_create(ctx->vk_log, pl_vulkan_params( + .instance = ctx->vk_inst->instance, + .get_proc_addr = ctx->vk_inst->get_proc_addr, + .surface = ctx->vk_surface, + .allow_software = false, + )); + if (!ctx->pl_vk) { + ret = AVERROR_EXTERNAL; + goto out; + } + + ctx->swapchain = pl_vulkan_create_swapchain(ctx->pl_vk, + pl_vulkan_swapchain_params(.surface = ctx->vk_surface, + .present_mode = VK_PRESENT_MODE_FIFO_KHR)); + + SDL_Vulkan_GetDrawableSize(window, &w, &h); + pl_swapchain_resize(ctx->swapchain, &w, &h); + + ctx->renderer = pl_renderer_create(ctx->vk_log, ctx->pl_vk->gpu); + if (!ctx->renderer) { + ret = AVERROR_EXTERNAL; + goto out; + } + ret = 0; + +out: + av_free(ext); + return ret; +} + +static int display(VkRenderer *renderer, AVFrame *frame) +{ + struct pl_swapchain_frame swap_frame = {0}; + struct pl_frame pl_frame = {0}; + struct pl_frame target = {0}; + RendererContext *ctx = (RendererContext *)renderer; + int ret = 0; + + if (!pl_map_avframe_ex(ctx->pl_vk->gpu, &pl_frame, pl_avframe_params( + .frame = frame, + .tex = ctx->tex))) { + av_log(NULL, AV_LOG_ERROR, "pl_map_avframe_ex failed\n"); + return AVERROR_EXTERNAL; + } + + if (!pl_swapchain_start_frame(ctx->swapchain, &swap_frame)) { + av_log(NULL, AV_LOG_ERROR, "start frame failed\n"); + ret = AVERROR_EXTERNAL; + goto out; + } + + pl_frame_from_swapchain(&target, &swap_frame); + if (!pl_render_image(ctx->renderer, &pl_frame, &target, &pl_render_default_params)) { + av_log(NULL, AV_LOG_ERROR, "pl_render_image failed\n"); + ret = AVERROR_EXTERNAL; + goto out; + } + + if (!pl_swapchain_submit_frame(ctx->swapchain)) { + av_log(NULL, AV_LOG_ERROR, "pl_swapchain_submit_frame failed\n"); + ret = AVERROR_EXTERNAL; + goto out; + } + pl_swapchain_swap_buffers(ctx->swapchain); + +out: + pl_unmap_avframe(ctx->pl_vk->gpu, &pl_frame); + return ret; +} + +static int resize(VkRenderer *renderer, int width, int height) +{ + RendererContext *ctx = (RendererContext *)renderer; + + if (!pl_swapchain_resize(ctx->swapchain, &width, &height)) + return AVERROR_EXTERNAL; + return 0; +} + +static void destroy(VkRenderer *renderer) +{ + PFN_vkDestroySurfaceKHR vkDestroySurfaceKHR; + RendererContext *ctx = (RendererContext *)renderer; + + if (ctx->pl_vk) { + for (int i = 0; i < FF_ARRAY_ELEMS(ctx->tex); i++) + pl_tex_destroy(ctx->pl_vk->gpu, &ctx->tex[i]); + pl_renderer_destroy(&ctx->renderer); + pl_swapchain_destroy(&ctx->swapchain); + pl_vulkan_destroy(&ctx->pl_vk); + } + + if (ctx->vk_surface) { + vkDestroySurfaceKHR = (PFN_vkDestroySurfaceKHR) + ctx->vk_inst->get_proc_addr(ctx->vk_inst->instance, "vkDestroySurfaceKHR"); + vkDestroySurfaceKHR(ctx->vk_inst->instance, ctx->vk_surface, NULL); + ctx->vk_surface = NULL; + } + + pl_vk_inst_destroy(&ctx->vk_inst); + pl_log_destroy(&ctx->vk_log); +} + +static const AVClass vulkan_renderer_class = { + .class_name = "Vulkan Renderer", + .item_name = av_default_item_name, + .version = LIBAVUTIL_VERSION_INT, +}; + +VkRenderer *vk_get_renderer(void) +{ + RendererContext *ctx = av_mallocz(sizeof(*ctx)); + VkRenderer *renderer; + + if (!ctx) + return NULL; + + renderer = &ctx->api; + renderer->class = &vulkan_renderer_class; + renderer->create = create; + renderer->display = display; + renderer->resize = resize; + renderer->destroy = destroy; + + return renderer; +} + +#else + +VkRenderer *vk_get_renderer(void) +{ + return NULL; +} + +#endif + +int vk_renderer_create(VkRenderer *renderer, SDL_Window *window) +{ + return renderer->create(renderer, window); +} + +int vk_renderer_display(VkRenderer *renderer, AVFrame *frame) +{ + return renderer->display(renderer, frame); +} + +int vk_renderer_resize(VkRenderer *renderer, int width, int height) +{ + return renderer->resize(renderer, width, height); +} + +void vk_renderer_destroy(VkRenderer *renderer) +{ + renderer->destroy(renderer); +} diff --git a/fftools/ffplay_renderer.h b/fftools/ffplay_renderer.h new file mode 100644 index 0000000000..4029f154a9 --- /dev/null +++ b/fftools/ffplay_renderer.h @@ -0,0 +1,38 @@ +/* + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifndef FFTOOLS_FFPLAY_RENDERER_H +#define FFTOOLS_FFPLAY_RENDERER_H + +#include <SDL.h> + +#include "libavutil/frame.h" + +typedef struct VkRenderer VkRenderer; + +VkRenderer *vk_get_renderer(void); + +int vk_renderer_create(VkRenderer *renderer, SDL_Window *window); + +int vk_renderer_display(VkRenderer *renderer, AVFrame *frame); + +int vk_renderer_resize(VkRenderer *renderer, int width, int height); + +void vk_renderer_destroy(VkRenderer *renderer); + +#endif /* FFTOOLS_FFPLAY_RENDERER_H */ -- 2.34.1 _______________________________________________ ffmpeg-devel mailing list ffmpeg-devel@ffmpeg.org https://ffmpeg.org/mailman/listinfo/ffmpeg-devel To unsubscribe, visit link above, or email ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe". ^ permalink raw reply [flat|nested] 7+ messages in thread
[parent not found: <20231029022347.15243-1-quinkblack@foxmail.com>]
* [FFmpeg-devel] [PATCH v4 2/2] fftools/ffplay: add hwaccel decoding support [not found] ` <20231029022347.15243-1-quinkblack@foxmail.com> @ 2023-10-29 2:23 ` Zhao Zhili 0 siblings, 0 replies; 7+ messages in thread From: Zhao Zhili @ 2023-10-29 2:23 UTC (permalink / raw) To: ffmpeg-devel; +Cc: Zhao Zhili From: Zhao Zhili <zhilizhao@tencent.com> --- v4: add more optional extensions v3: shared vulkan instance between libplacebo and hwcontext fftools/ffplay.c | 43 +++++ fftools/ffplay_renderer.c | 393 ++++++++++++++++++++++++++++++++++++-- fftools/ffplay_renderer.h | 2 + 3 files changed, 418 insertions(+), 20 deletions(-) diff --git a/fftools/ffplay.c b/fftools/ffplay.c index 305d72d8b8..29b37a5e46 100644 --- a/fftools/ffplay.c +++ b/fftools/ffplay.c @@ -352,6 +352,7 @@ static int autorotate = 1; static int find_stream_info = 1; static int filter_nbthreads = 0; static int enable_vulkan = 0; +static const char *hwaccel = NULL; /* current context */ static int is_full_screen; @@ -2557,6 +2558,37 @@ static int audio_open(void *opaque, AVChannelLayout *wanted_channel_layout, int return spec.size; } +static int create_hwaccel(AVBufferRef **device_ctx) +{ + enum AVHWDeviceType type; + int ret; + AVBufferRef *vk_dev; + + *device_ctx = NULL; + + if (!hwaccel) + return 0; + + type = av_hwdevice_find_type_by_name(hwaccel); + if (type == AV_HWDEVICE_TYPE_NONE) + return AVERROR(ENOTSUP); + + ret = vk_renderer_get_hw_dev(vk_renderer, &vk_dev); + if (ret < 0) + return ret; + + ret = av_hwdevice_ctx_create_derived(device_ctx, type, vk_dev, 0); + if (!ret) + return 0; + + if (ret != AVERROR(ENOSYS)) + return ret; + + av_log(NULL, AV_LOG_WARNING, "Derive %s from vulkan not supported.\n", hwaccel); + ret = av_hwdevice_ctx_create(device_ctx, type, NULL, NULL, 0); + return ret; +} + /* open a given stream. Return 0 if OK */ static int stream_component_open(VideoState *is, int stream_index) { @@ -2624,6 +2656,12 @@ static int stream_component_open(VideoState *is, int stream_index) av_dict_set(&opts, "flags", "+copy_opaque", AV_DICT_MULTIKEY); + if (avctx->codec_type == AVMEDIA_TYPE_VIDEO) { + ret = create_hwaccel(&avctx->hw_device_ctx); + if (ret < 0) + goto fail; + } + if ((ret = avcodec_open2(avctx, codec, &opts)) < 0) { goto fail; } @@ -3625,6 +3663,7 @@ static const OptionDef options[] = { "read and decode the streams to fill missing information with heuristics" }, { "filter_threads", HAS_ARG | OPT_INT | OPT_EXPERT, { &filter_nbthreads }, "number of filter threads per graph" }, { "enable_vulkan", OPT_BOOL, { &enable_vulkan }, "enable vulkan render" }, + { "hwaccel", HAS_ARG | OPT_STRING | OPT_EXPERT, { &hwaccel }, "use HW accelerated decoding" }, { NULL, }, }; @@ -3739,6 +3778,10 @@ int main(int argc, char **argv) #ifdef SDL_HINT_VIDEO_X11_NET_WM_BYPASS_COMPOSITOR SDL_SetHint(SDL_HINT_VIDEO_X11_NET_WM_BYPASS_COMPOSITOR, "0"); #endif + if (hwaccel && !enable_vulkan) { + av_log(NULL, AV_LOG_INFO, "Enable vulkan renderer to support hwaccel %s\n", hwaccel); + enable_vulkan = 1; + } if (enable_vulkan) { vk_renderer = vk_get_renderer(); if (vk_renderer) { diff --git a/fftools/ffplay_renderer.c b/fftools/ffplay_renderer.c index 49e5516d5d..b898f69d26 100644 --- a/fftools/ffplay_renderer.c +++ b/fftools/ffplay_renderer.c @@ -16,6 +16,9 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA */ +#define VK_NO_PROTOTYPES +#define VK_ENABLE_BETA_EXTENSIONS + #include "config.h" #include "ffplay_renderer.h" @@ -35,6 +38,8 @@ struct VkRenderer { int (*create)(VkRenderer *renderer, SDL_Window *window); + int (*get_hw_dev)(VkRenderer *renderer, AVBufferRef **dev); + int (*display)(VkRenderer *renderer, AVFrame *frame); int (*resize)(VkRenderer *renderer, int width, int height); @@ -54,9 +59,18 @@ typedef struct RendererContext { pl_tex tex[4]; pl_log vk_log; + + AVBufferRef *hw_device_ref; + AVBufferRef *hw_frame_ref; + enum AVPixelFormat *transfer_formats; + AVHWFramesConstraints *constraints; + + AVFrame *vk_frame; } RendererContext; -static void vk_log_cb(void *log_priv, enum pl_log_level level, const char *msg) { +static void vk_log_cb(void *log_priv, enum pl_log_level level, + const char *msg) +{ static const int level_map[] = { AV_LOG_QUIET, AV_LOG_FATAL, @@ -71,45 +85,77 @@ static void vk_log_cb(void *log_priv, enum pl_log_level level, const char *msg) av_log(log_priv, level_map[level], "%s\n", msg); } +// Should keep sync with optional_device_exts inside hwcontext_vulkan.c +static const char *optional_device_exts[] = { + /* Misc or required by other extensions */ + VK_KHR_PORTABILITY_SUBSET_EXTENSION_NAME, + VK_KHR_PUSH_DESCRIPTOR_EXTENSION_NAME, + VK_KHR_SAMPLER_YCBCR_CONVERSION_EXTENSION_NAME, + VK_EXT_DESCRIPTOR_BUFFER_EXTENSION_NAME, + VK_EXT_PHYSICAL_DEVICE_DRM_EXTENSION_NAME, + VK_EXT_SHADER_ATOMIC_FLOAT_EXTENSION_NAME, + VK_KHR_COOPERATIVE_MATRIX_EXTENSION_NAME, + + /* Imports/exports */ + VK_KHR_EXTERNAL_MEMORY_FD_EXTENSION_NAME, + VK_EXT_EXTERNAL_MEMORY_DMA_BUF_EXTENSION_NAME, + VK_EXT_IMAGE_DRM_FORMAT_MODIFIER_EXTENSION_NAME, + VK_KHR_EXTERNAL_SEMAPHORE_FD_EXTENSION_NAME, + VK_EXT_EXTERNAL_MEMORY_HOST_EXTENSION_NAME, +#ifdef _WIN32 + VK_KHR_EXTERNAL_MEMORY_WIN32_EXTENSION_NAME, + VK_KHR_EXTERNAL_SEMAPHORE_WIN32_EXTENSION_NAME, +#endif + + /* Video encoding/decoding */ + VK_KHR_VIDEO_QUEUE_EXTENSION_NAME, + VK_KHR_VIDEO_DECODE_QUEUE_EXTENSION_NAME, + VK_KHR_VIDEO_DECODE_H264_EXTENSION_NAME, + VK_KHR_VIDEO_DECODE_H265_EXTENSION_NAME, + "VK_MESA_video_decode_av1", +}; + static int create(VkRenderer *renderer, SDL_Window *window) { int ret = 0; - unsigned ext_num = 0; + unsigned num_ext = 0; const char **ext = NULL; int w, h; struct pl_log_params vk_log_params = { .log_cb = vk_log_cb, - .log_level = PL_LOG_WARN, + .log_level = PL_LOG_DEBUG, .log_priv = renderer, }; - RendererContext *ctx = (RendererContext *)renderer; + RendererContext *ctx = (RendererContext *) renderer; - if (!SDL_Vulkan_GetInstanceExtensions(window, &ext_num, NULL)) { - av_log(NULL, AV_LOG_FATAL, "Failed to get vulkan extensions: %s\n", SDL_GetError()); + if (!SDL_Vulkan_GetInstanceExtensions(window, &num_ext, NULL)) { + av_log(NULL, AV_LOG_FATAL, "Failed to get vulkan extensions: %s\n", + SDL_GetError()); return AVERROR_EXTERNAL; } - ext = av_calloc(ext_num, sizeof(*ext)); + ext = av_calloc(num_ext, sizeof(*ext)); if (!ext) { ret = AVERROR(ENOMEM); goto out; } - SDL_Vulkan_GetInstanceExtensions(window, &ext_num, ext); + SDL_Vulkan_GetInstanceExtensions(window, &num_ext, ext); ctx->vk_log = pl_log_create(PL_API_VER, &vk_log_params); ctx->vk_inst = pl_vk_inst_create(ctx->vk_log, pl_vk_inst_params( .get_proc_addr = SDL_Vulkan_GetVkGetInstanceProcAddr(), .debug = false, .extensions = ext, - .num_extensions = ext_num, + .num_extensions = num_ext )); if (!ctx->vk_inst) { ret = AVERROR_EXTERNAL; goto out; } - if (!SDL_Vulkan_CreateSurface(window, ctx->vk_inst->instance, &ctx->vk_surface)) { + if (!SDL_Vulkan_CreateSurface(window, ctx->vk_inst->instance, + &ctx->vk_surface)) { ret = AVERROR_EXTERNAL; goto out; } @@ -119,15 +165,20 @@ static int create(VkRenderer *renderer, SDL_Window *window) .get_proc_addr = ctx->vk_inst->get_proc_addr, .surface = ctx->vk_surface, .allow_software = false, + .opt_extensions = optional_device_exts, + .num_opt_extensions = FF_ARRAY_ELEMS(optional_device_exts), + .extra_queues = VK_QUEUE_VIDEO_DECODE_BIT_KHR, )); if (!ctx->pl_vk) { ret = AVERROR_EXTERNAL; goto out; } - ctx->swapchain = pl_vulkan_create_swapchain(ctx->pl_vk, - pl_vulkan_swapchain_params(.surface = ctx->vk_surface, - .present_mode = VK_PRESENT_MODE_FIFO_KHR)); + ctx->swapchain = pl_vulkan_create_swapchain( + ctx->pl_vk, + pl_vulkan_swapchain_params( + .surface = ctx->vk_surface, + .present_mode = VK_PRESENT_MODE_FIFO_KHR)); SDL_Vulkan_GetDrawableSize(window, &w, &h); pl_swapchain_resize(ctx->swapchain, &w, &h); @@ -137,21 +188,309 @@ static int create(VkRenderer *renderer, SDL_Window *window) ret = AVERROR_EXTERNAL; goto out; } + + ctx->vk_frame = av_frame_alloc(); + if (!ctx->vk_frame) { + ret = AVERROR(ENOMEM); + goto out; + } + ret = 0; -out: + out: av_free(ext); return ret; } +static void lock_queue(struct AVHWDeviceContext *dev_ctx, + uint32_t queue_family, uint32_t index) +{ + RendererContext *ctx = dev_ctx->user_opaque; + pl_vulkan vk = ctx->pl_vk; + vk->lock_queue(vk, queue_family, index); +} + +static void unlock_queue(struct AVHWDeviceContext *dev_ctx, + uint32_t queue_family, + uint32_t index) +{ + RendererContext *ctx = dev_ctx->user_opaque; + pl_vulkan vk = ctx->pl_vk; + vk->unlock_queue(vk, queue_family, index); +} + +static int get_decode_queue(VkRenderer *renderer, int *index, int *count) +{ + RendererContext *ctx = (RendererContext *) renderer; + VkQueueFamilyProperties *queue_family_prop = NULL; + uint32_t num_queue_family_prop = 0; + PFN_vkGetPhysicalDeviceQueueFamilyProperties get_queue_family_prop; + PFN_vkGetInstanceProcAddr get_proc_addr = ctx->vk_inst->get_proc_addr; + + *index = -1; + *count = 0; + get_queue_family_prop = (PFN_vkGetPhysicalDeviceQueueFamilyProperties) + get_proc_addr(ctx->vk_inst->instance, + "vkGetPhysicalDeviceQueueFamilyProperties"); + get_queue_family_prop(ctx->pl_vk->phys_device, + &num_queue_family_prop, NULL); + if (!num_queue_family_prop) + return AVERROR_EXTERNAL; + + queue_family_prop = av_calloc(num_queue_family_prop, + sizeof(*queue_family_prop)); + if (!queue_family_prop) + return AVERROR(ENOMEM); + + get_queue_family_prop(ctx->pl_vk->phys_device, + &num_queue_family_prop, + queue_family_prop); + + for (int i = 0; i < num_queue_family_prop; i++) { + if (queue_family_prop[i].queueFlags & VK_QUEUE_VIDEO_DECODE_BIT_KHR) { + *index = i; + *count = queue_family_prop[i].queueCount; + break; + } + } + av_free(queue_family_prop); + + return 0; +} + +static int get_hw_dev(VkRenderer *renderer, AVBufferRef **dev) +{ + RendererContext *ctx = (RendererContext *) renderer; + AVHWDeviceContext *device_ctx; + AVVulkanDeviceContext *vk_dev_ctx; + int decode_index; + int decode_count; + int ret; + + *dev = NULL; + if (ctx->hw_device_ref) { + *dev = ctx->hw_device_ref; + return 0; + } + + ctx->hw_device_ref = av_hwdevice_ctx_alloc(AV_HWDEVICE_TYPE_VULKAN); + if (!ctx->hw_device_ref) { + return AVERROR(ENOMEM); + } + + device_ctx = (AVHWDeviceContext *) ctx->hw_device_ref->data; + device_ctx->user_opaque = ctx; + + vk_dev_ctx = device_ctx->hwctx; + vk_dev_ctx->lock_queue = lock_queue, + vk_dev_ctx->unlock_queue = unlock_queue; + + vk_dev_ctx->get_proc_addr = ctx->vk_inst->get_proc_addr; + + vk_dev_ctx->inst = ctx->vk_inst->instance; + vk_dev_ctx->phys_dev = ctx->pl_vk->phys_device; + vk_dev_ctx->act_dev = ctx->pl_vk->device; + + vk_dev_ctx->device_features = *ctx->pl_vk->features; + + vk_dev_ctx->enabled_inst_extensions = ctx->vk_inst->extensions; + vk_dev_ctx->nb_enabled_inst_extensions = ctx->vk_inst->num_extensions; + + vk_dev_ctx->enabled_dev_extensions = ctx->pl_vk->extensions; + vk_dev_ctx->nb_enabled_dev_extensions = ctx->pl_vk->num_extensions; + + vk_dev_ctx->queue_family_index = ctx->pl_vk->queue_graphics.index; + vk_dev_ctx->nb_graphics_queues = ctx->pl_vk->queue_graphics.count; + + vk_dev_ctx->queue_family_tx_index = ctx->pl_vk->queue_transfer.index; + vk_dev_ctx->nb_tx_queues = ctx->pl_vk->queue_transfer.count; + + vk_dev_ctx->queue_family_comp_index = ctx->pl_vk->queue_compute.index; + vk_dev_ctx->nb_comp_queues = ctx->pl_vk->queue_compute.count; + + ret = get_decode_queue(renderer, &decode_index, &decode_count); + if (ret < 0) + return ret; + + vk_dev_ctx->queue_family_decode_index = decode_index; + vk_dev_ctx->nb_decode_queues = decode_count; + + ret = av_hwdevice_ctx_init(ctx->hw_device_ref); + if (ret < 0) + return ret; + + *dev = ctx->hw_device_ref; + return 0; +} + +static int create_hw_frame(VkRenderer *renderer, AVFrame *frame) +{ + RendererContext *ctx = (RendererContext *) renderer; + AVHWFramesContext *src_hw_frame = (AVHWFramesContext *) + frame->hw_frames_ctx->data; + AVHWFramesContext *hw_frame; + AVVulkanFramesContext *vk_frame_ctx; + int ret; + + if (ctx->hw_frame_ref) { + hw_frame = (AVHWFramesContext *) ctx->hw_frame_ref->data; + + if (hw_frame->width == frame->width && + hw_frame->height == frame->height && + hw_frame->sw_format == src_hw_frame->sw_format) + return 0; + + av_buffer_unref(&ctx->hw_frame_ref); + } + + if (!ctx->constraints) { + ctx->constraints = av_hwdevice_get_hwframe_constraints( + ctx->hw_device_ref, NULL); + if (!ctx->constraints) + return AVERROR(ENOMEM); + } + + // Check constraints and skip create hwframe. Don't take it as error since + // we can fallback to memory copy from GPU to CPU. + if ((ctx->constraints->max_width && + ctx->constraints->max_width < frame->width) || + (ctx->constraints->max_height && + ctx->constraints->max_height < frame->height) || + (ctx->constraints->min_width && + ctx->constraints->min_width > frame->width) || + (ctx->constraints->min_height && + ctx->constraints->min_height > frame->height)) + return 0; + + if (ctx->constraints->valid_sw_formats) { + enum AVPixelFormat *sw_formats = ctx->constraints->valid_sw_formats; + while (*sw_formats != AV_PIX_FMT_NONE) { + if (*sw_formats == src_hw_frame->sw_format) + break; + sw_formats++; + } + if (*sw_formats == AV_PIX_FMT_NONE) + return 0; + } + + ctx->hw_frame_ref = av_hwframe_ctx_alloc(ctx->hw_device_ref); + if (!ctx->hw_frame_ref) + return AVERROR(ENOMEM); + + hw_frame = (AVHWFramesContext *) ctx->hw_frame_ref->data; + hw_frame->format = AV_PIX_FMT_VULKAN; + hw_frame->sw_format = src_hw_frame->sw_format; + hw_frame->width = frame->width; + hw_frame->height = frame->height; + + if (frame->format == AV_PIX_FMT_CUDA) { + vk_frame_ctx = hw_frame->hwctx; + vk_frame_ctx->flags = AV_VK_FRAME_FLAG_DISABLE_MULTIPLANE; + } + + ret = av_hwframe_ctx_init(ctx->hw_frame_ref); + if (ret < 0) { + av_log(renderer, AV_LOG_ERROR, "Create hwframe context failed, %s\n", + av_err2str(ret)); + return ret; + } + + av_hwframe_transfer_get_formats(ctx->hw_frame_ref, + AV_HWFRAME_TRANSFER_DIRECTION_TO, + &ctx->transfer_formats, 0); + + return 0; +} + +static inline int check_hw_transfer(RendererContext *ctx, AVFrame *frame) +{ + if (!ctx->hw_frame_ref || !ctx->transfer_formats) + return 0; + + for (int i = 0; ctx->transfer_formats[i] != AV_PIX_FMT_NONE; i++) + if (ctx->transfer_formats[i] == frame->format) + return 1; + + return 0; +} + +static int transfer_frame(VkRenderer *renderer, AVFrame *frame) +{ + RendererContext *ctx = (RendererContext *) renderer; + int ret; + + if (!frame->hw_frames_ctx) + return 0; + + if (frame->format == AV_PIX_FMT_VULKAN) + return 0; + + ret = create_hw_frame(renderer, frame); + if (ret < 0) + return ret; + + // Try map data first + av_frame_unref(ctx->vk_frame); + if (ctx->hw_frame_ref) { + ctx->vk_frame->hw_frames_ctx = av_buffer_ref(ctx->hw_frame_ref); + ctx->vk_frame->format = AV_PIX_FMT_VULKAN; + } + ret = av_hwframe_map(ctx->vk_frame, frame, AV_HWFRAME_MAP_READ); + if (!ret) + goto out; + + if (ret != AVERROR(ENOSYS)) { + av_log(NULL, AV_LOG_FATAL, "Map data to vulkan failed: %s\n", + av_err2str(ret)); + return ret; + } + + // Try transfer to vulkan + if (check_hw_transfer(ctx, frame)) { + av_frame_unref(ctx->vk_frame); + av_hwframe_get_buffer(ctx->hw_frame_ref, ctx->vk_frame, 0); + ret = av_hwframe_transfer_data(ctx->vk_frame, frame, 1); + if (!ret) + goto out; + + if (ret < 0 && ret != AVERROR(ENOSYS)) { + av_log(NULL, AV_LOG_FATAL, "Transfer data to vulkan failed: %s\n", + av_err2str(ret)); + return ret; + } + } + + // Transfer to CPU + av_frame_unref(ctx->vk_frame); + ret = av_hwframe_transfer_data(ctx->vk_frame, frame, 0); + if (ret < 0) { + av_log(NULL, AV_LOG_FATAL, "Transfer data to CPU failed: %s\n", + av_err2str(ret)); + return ret; + } + + out: + ret = av_frame_copy_props(ctx->vk_frame, frame); + if (ret < 0) + return ret; + av_frame_unref(frame); + av_frame_move_ref(frame, ctx->vk_frame); + + return 0; +} + static int display(VkRenderer *renderer, AVFrame *frame) { struct pl_swapchain_frame swap_frame = {0}; struct pl_frame pl_frame = {0}; struct pl_frame target = {0}; - RendererContext *ctx = (RendererContext *)renderer; + RendererContext *ctx = (RendererContext *) renderer; int ret = 0; + ret = transfer_frame(renderer, frame); + if (ret < 0) + return ret; + if (!pl_map_avframe_ex(ctx->pl_vk->gpu, &pl_frame, pl_avframe_params( .frame = frame, .tex = ctx->tex))) { @@ -166,7 +505,8 @@ static int display(VkRenderer *renderer, AVFrame *frame) } pl_frame_from_swapchain(&target, &swap_frame); - if (!pl_render_image(ctx->renderer, &pl_frame, &target, &pl_render_default_params)) { + if (!pl_render_image(ctx->renderer, &pl_frame, &target, + &pl_render_default_params)) { av_log(NULL, AV_LOG_ERROR, "pl_render_image failed\n"); ret = AVERROR_EXTERNAL; goto out; @@ -179,14 +519,14 @@ static int display(VkRenderer *renderer, AVFrame *frame) } pl_swapchain_swap_buffers(ctx->swapchain); -out: + out: pl_unmap_avframe(ctx->pl_vk->gpu, &pl_frame); return ret; } static int resize(VkRenderer *renderer, int width, int height) { - RendererContext *ctx = (RendererContext *)renderer; + RendererContext *ctx = (RendererContext *) renderer; if (!pl_swapchain_resize(ctx->swapchain, &width, &height)) return AVERROR_EXTERNAL; @@ -196,7 +536,13 @@ static int resize(VkRenderer *renderer, int width, int height) static void destroy(VkRenderer *renderer) { PFN_vkDestroySurfaceKHR vkDestroySurfaceKHR; - RendererContext *ctx = (RendererContext *)renderer; + RendererContext *ctx = (RendererContext *) renderer; + + av_freep(&ctx->transfer_formats); + av_hwframe_constraints_free(&ctx->constraints); + av_buffer_unref(&ctx->hw_frame_ref); + av_buffer_unref(&ctx->hw_device_ref); + av_frame_free(&ctx->vk_frame); if (ctx->pl_vk) { for (int i = 0; i < FF_ARRAY_ELEMS(ctx->tex); i++) @@ -208,7 +554,8 @@ static void destroy(VkRenderer *renderer) if (ctx->vk_surface) { vkDestroySurfaceKHR = (PFN_vkDestroySurfaceKHR) - ctx->vk_inst->get_proc_addr(ctx->vk_inst->instance, "vkDestroySurfaceKHR"); + ctx->vk_inst->get_proc_addr(ctx->vk_inst->instance, + "vkDestroySurfaceKHR"); vkDestroySurfaceKHR(ctx->vk_inst->instance, ctx->vk_surface, NULL); ctx->vk_surface = NULL; } @@ -233,6 +580,7 @@ VkRenderer *vk_get_renderer(void) renderer = &ctx->api; renderer->class = &vulkan_renderer_class; + renderer->get_hw_dev = get_hw_dev; renderer->create = create; renderer->display = display; renderer->resize = resize; @@ -255,6 +603,11 @@ int vk_renderer_create(VkRenderer *renderer, SDL_Window *window) return renderer->create(renderer, window); } +int vk_renderer_get_hw_dev(VkRenderer *renderer, AVBufferRef **dev) +{ + return renderer->get_hw_dev(renderer, dev); +} + int vk_renderer_display(VkRenderer *renderer, AVFrame *frame) { return renderer->display(renderer, frame); diff --git a/fftools/ffplay_renderer.h b/fftools/ffplay_renderer.h index 4029f154a9..b982ba0192 100644 --- a/fftools/ffplay_renderer.h +++ b/fftools/ffplay_renderer.h @@ -29,6 +29,8 @@ VkRenderer *vk_get_renderer(void); int vk_renderer_create(VkRenderer *renderer, SDL_Window *window); +int vk_renderer_get_hw_dev(VkRenderer *renderer, AVBufferRef **dev); + int vk_renderer_display(VkRenderer *renderer, AVFrame *frame); int vk_renderer_resize(VkRenderer *renderer, int width, int height); -- 2.34.1 _______________________________________________ ffmpeg-devel mailing list ffmpeg-devel@ffmpeg.org https://ffmpeg.org/mailman/listinfo/ffmpeg-devel To unsubscribe, visit link above, or email ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe". ^ permalink raw reply [flat|nested] 7+ messages in thread
* Re: [FFmpeg-devel] [PATCH v3 2/2] fftools/ffplay: add hwaccel decoding support 2023-10-28 19:32 ` Lynne 2023-10-29 2:23 ` [FFmpeg-devel] [PATCH v4 1/2] fftools/ffplay: add vulkan renderer via libplacebo Zhao Zhili [not found] ` <20231029022347.15243-1-quinkblack@foxmail.com> @ 2023-10-29 2:31 ` Zhao Zhili 2023-10-29 3:18 ` Lynne 2023-10-29 11:12 ` Niklas Haas 2 siblings, 2 replies; 7+ messages in thread From: Zhao Zhili @ 2023-10-29 2:31 UTC (permalink / raw) To: ffmpeg-devel On 2023/10/29 03:32, Lynne wrote: > Oct 28, 2023, 20:14 byquinkblack@foxmail.com: > >> From: Zhao Zhili<zhilizhao@tencent.com> >> >> --- >> v3: shared vulkan instance between libplacebo and hwcontext >> >> fftools/ffplay.c | 43 ++++++ >> fftools/ffplay_renderer.c | 316 +++++++++++++++++++++++++++++++++++++- >> fftools/ffplay_renderer.h | 2 + >> 3 files changed, 355 insertions(+), 6 deletions(-) >> >> diff --git a/fftools/ffplay.c b/fftools/ffplay.c >> index 305d72d8b8..29b37a5e46 100644 >> --- a/fftools/ffplay.c >> +++ b/fftools/ffplay.c >> @@ -352,6 +352,7 @@ static int autorotate = 1; >> static int find_stream_info = 1; >> static int filter_nbthreads = 0; >> static int enable_vulkan = 0; >> +static const char *hwaccel = NULL; >> >> /* current context */ >> static int is_full_screen; >> @@ -2557,6 +2558,37 @@ static int audio_open(void *opaque, AVChannelLayout *wanted_channel_layout, int >> return spec.size; >> } >> >> +static int create_hwaccel(AVBufferRef **device_ctx) >> +{ >> + enum AVHWDeviceType type; >> + int ret; >> + AVBufferRef *vk_dev; >> + >> + *device_ctx = NULL; >> + >> + if (!hwaccel) >> + return 0; >> + >> + type = av_hwdevice_find_type_by_name(hwaccel); >> + if (type == AV_HWDEVICE_TYPE_NONE) >> + return AVERROR(ENOTSUP); >> + >> + ret = vk_renderer_get_hw_dev(vk_renderer, &vk_dev); >> + if (ret < 0) >> + return ret; >> + >> + ret = av_hwdevice_ctx_create_derived(device_ctx, type, vk_dev, 0); >> + if (!ret) >> + return 0; >> + >> + if (ret != AVERROR(ENOSYS)) >> + return ret; >> + >> + av_log(NULL, AV_LOG_WARNING, "Derive %s from vulkan not supported.\n", hwaccel); >> + ret = av_hwdevice_ctx_create(device_ctx, type, NULL, NULL, 0); >> + return ret; >> +} >> + >> /* open a given stream. Return 0 if OK */ >> static int stream_component_open(VideoState *is, int stream_index) >> { >> @@ -2624,6 +2656,12 @@ static int stream_component_open(VideoState *is, int stream_index) >> >> av_dict_set(&opts, "flags", "+copy_opaque", AV_DICT_MULTIKEY); >> >> + if (avctx->codec_type == AVMEDIA_TYPE_VIDEO) { >> + ret = create_hwaccel(&avctx->hw_device_ctx); >> + if (ret < 0) >> + goto fail; >> + } >> + >> if ((ret = avcodec_open2(avctx, codec, &opts)) < 0) { >> goto fail; >> } >> @@ -3625,6 +3663,7 @@ static const OptionDef options[] = { >> "read and decode the streams to fill missing information with heuristics" }, >> { "filter_threads", HAS_ARG | OPT_INT | OPT_EXPERT, { &filter_nbthreads }, "number of filter threads per graph" }, >> { "enable_vulkan", OPT_BOOL, { &enable_vulkan }, "enable vulkan render" }, >> + { "hwaccel", HAS_ARG | OPT_STRING | OPT_EXPERT, { &hwaccel }, "use HW accelerated decoding" }, >> { NULL, }, >> }; >> >> @@ -3739,6 +3778,10 @@ int main(int argc, char **argv) >> #ifdef SDL_HINT_VIDEO_X11_NET_WM_BYPASS_COMPOSITOR >> SDL_SetHint(SDL_HINT_VIDEO_X11_NET_WM_BYPASS_COMPOSITOR, "0"); >> #endif >> + if (hwaccel && !enable_vulkan) { >> + av_log(NULL, AV_LOG_INFO, "Enable vulkan renderer to support hwaccel %s\n", hwaccel); >> + enable_vulkan = 1; >> + } >> if (enable_vulkan) { >> vk_renderer = vk_get_renderer(); >> if (vk_renderer) { >> diff --git a/fftools/ffplay_renderer.c b/fftools/ffplay_renderer.c >> index 49e5516d5d..497d758ec7 100644 >> --- a/fftools/ffplay_renderer.c >> +++ b/fftools/ffplay_renderer.c >> @@ -35,6 +35,8 @@ struct VkRenderer { >> >> int (*create)(VkRenderer *renderer, SDL_Window *window); >> >> + int (*get_hw_dev)(VkRenderer *renderer, AVBufferRef **dev); >> + >> int (*display)(VkRenderer *renderer, AVFrame *frame); >> >> int (*resize)(VkRenderer *renderer, int width, int height); >> @@ -54,6 +56,13 @@ typedef struct RendererContext { >> pl_tex tex[4]; >> >> pl_log vk_log; >> + >> + AVBufferRef *hw_device_ref; >> + AVBufferRef *hw_frame_ref; >> + enum AVPixelFormat *transfer_formats; >> + AVHWFramesConstraints *constraints; >> + >> + AVFrame *vk_frame; >> } RendererContext; >> >> static void vk_log_cb(void *log_priv, enum pl_log_level level, const char *msg) { >> @@ -74,35 +83,46 @@ static void vk_log_cb(void *log_priv, enum pl_log_level level, const char *msg) >> static int create(VkRenderer *renderer, SDL_Window *window) >> { >> int ret = 0; >> - unsigned ext_num = 0; >> + unsigned num_ext = 0; >> const char **ext = NULL; >> int w, h; >> struct pl_log_params vk_log_params = { >> .log_cb = vk_log_cb, >> - .log_level = PL_LOG_WARN, >> + .log_level = PL_LOG_DEBUG, >> .log_priv = renderer, >> }; >> RendererContext *ctx = (RendererContext *)renderer; >> >> - if (!SDL_Vulkan_GetInstanceExtensions(window, &ext_num, NULL)) { >> + static const char *opt_exts[] = { >> + VK_EXT_DESCRIPTOR_BUFFER_EXTENSION_NAME, >> + VK_EXT_SHADER_ATOMIC_FLOAT_EXTENSION_NAME, >> + VK_KHR_VIDEO_DECODE_QUEUE_EXTENSION_NAME, >> + VK_KHR_VIDEO_DECODE_H264_EXTENSION_NAME, >> + VK_KHR_VIDEO_DECODE_H265_EXTENSION_NAME, >> + VK_KHR_VIDEO_QUEUE_EXTENSION_NAME, >> + "VK_MESA_video_decode_av1", >> + }; >> + const int num_opt_exts = FF_ARRAY_ELEMS(opt_exts); >> + >> + if (!SDL_Vulkan_GetInstanceExtensions(window, &num_ext, NULL)) { >> av_log(NULL, AV_LOG_FATAL, "Failed to get vulkan extensions: %s\n", SDL_GetError()); >> return AVERROR_EXTERNAL; >> > That's not a complete list. Look at optional_device_exts. > There are also device features that both we and libplacebo would like > to be enabled. Look into vulkan_device_create_internal(), the list there > is up to date. FFmpeg requires timeline semaphores, and descriptor_buffer > for lavfi filters to work. Fixed in patch v4. Maybe we should export optional_device_exts in API. > > Is there no option to let libavutil create the hwdevice for both SDL and libplacebo? > That way, that code path can also be tested. libplacebo doesn't support that: // Creates a new vulkan device based on the given parameters and initializes // a new GPU. This function will internally initialize a VkDevice. There is // currently no way to share a vulkan device with the caller. If `params` is // left as NULL, it defaults to &pl_vulkan_default_params. // // Thread-safety: Safe PL_API pl_vulkan pl_vulkan_create(pl_log log, const struct pl_vulkan_params *params); I think share instance or vkImage without support from vulkan spec is hard to implementation, if not impossible. https://www.yosoygames.com.ar/wp/2022/07/vulkan-integration-between-multiple-libraries-e-g-qt-ogrenext/ > _______________________________________________ > ffmpeg-devel mailing list > ffmpeg-devel@ffmpeg.org > https://ffmpeg.org/mailman/listinfo/ffmpeg-devel > > To unsubscribe, visit link above, or email > ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe". _______________________________________________ ffmpeg-devel mailing list ffmpeg-devel@ffmpeg.org https://ffmpeg.org/mailman/listinfo/ffmpeg-devel To unsubscribe, visit link above, or email ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe". ^ permalink raw reply [flat|nested] 7+ messages in thread
* Re: [FFmpeg-devel] [PATCH v3 2/2] fftools/ffplay: add hwaccel decoding support 2023-10-29 2:31 ` [FFmpeg-devel] [PATCH v3 " Zhao Zhili @ 2023-10-29 3:18 ` Lynne 2023-10-29 11:12 ` Niklas Haas 1 sibling, 0 replies; 7+ messages in thread From: Lynne @ 2023-10-29 3:18 UTC (permalink / raw) To: FFmpeg development discussions and patches Oct 29, 2023, 03:31 by quinkblack@foxmail.com: > > On 2023/10/29 03:32, Lynne wrote: > >> Oct 28, 2023, 20:14 byquinkblack@foxmail.com: >> > > Fixed in patch v4. > > Maybe we should export optional_device_exts in API. > Patch welcome. >> >> Is there no option to let libavutil create the hwdevice for both SDL and libplacebo? >> That way, that code path can also be tested. >> > > libplacebo doesn't support that: > It does. It's how vf_libplacebo runs, and I also use it in my sample library (txproto). You should do it this way. > // Creates a new vulkan device based on the given parameters and initializes > // a new GPU. This function will internally initialize a VkDevice. There is > // currently no way to share a vulkan device with the caller. If `params` is > // left as NULL, it defaults to &pl_vulkan_default_params. > // > // Thread-safety: Safe > PL_API pl_vulkan pl_vulkan_create(pl_log log, const struct pl_vulkan_params *params); > > I think share instance or vkImage without support from vulkan spec is hard to implementation, > > if not impossible. > > https://www.yosoygames.com.ar/wp/2022/07/vulkan-integration-between-multiple-libraries-e-g-qt-ogrenext/ > That's why we have locking callbacks for AVVkFrame, and for VkQueues. It's all done for you, as long as you use the pl_map_avframe_ex(). _______________________________________________ ffmpeg-devel mailing list ffmpeg-devel@ffmpeg.org https://ffmpeg.org/mailman/listinfo/ffmpeg-devel To unsubscribe, visit link above, or email ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe". ^ permalink raw reply [flat|nested] 7+ messages in thread
* Re: [FFmpeg-devel] [PATCH v3 2/2] fftools/ffplay: add hwaccel decoding support 2023-10-29 2:31 ` [FFmpeg-devel] [PATCH v3 " Zhao Zhili 2023-10-29 3:18 ` Lynne @ 2023-10-29 11:12 ` Niklas Haas 1 sibling, 0 replies; 7+ messages in thread From: Niklas Haas @ 2023-10-29 11:12 UTC (permalink / raw) To: ffmpeg-devel On Sun, 29 Oct 2023 10:31:17 +0800 Zhao Zhili <quinkblack@foxmail.com> wrote: > libplacebo doesn't support that: > > // Creates a new vulkan device based on the given parameters and initializes > // a new GPU. This function will internally initialize a VkDevice. There is > // currently no way to share a vulkan device with the caller. If `params` is > // left as NULL, it defaults to &pl_vulkan_default_params. > // > // Thread-safety: Safe > PL_API pl_vulkan pl_vulkan_create(pl_log log, const struct > pl_vulkan_params *params); Pretty sure this comment is ancient and wrong/misleading, I removed it in master. _______________________________________________ ffmpeg-devel mailing list ffmpeg-devel@ffmpeg.org https://ffmpeg.org/mailman/listinfo/ffmpeg-devel To unsubscribe, visit link above, or email ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe". ^ permalink raw reply [flat|nested] 7+ messages in thread
end of thread, other threads:[~2023-10-29 11:12 UTC | newest]
Thread overview: 7+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
[not found] <20231028181341.13090-1-quinkblack@foxmail.com>
2023-10-28 18:13 ` [FFmpeg-devel] [PATCH v3 2/2] fftools/ffplay: add hwaccel decoding support Zhao Zhili
2023-10-28 19:32 ` Lynne
2023-10-29 2:23 ` [FFmpeg-devel] [PATCH v4 1/2] fftools/ffplay: add vulkan renderer via libplacebo Zhao Zhili
[not found] ` <20231029022347.15243-1-quinkblack@foxmail.com>
2023-10-29 2:23 ` [FFmpeg-devel] [PATCH v4 2/2] fftools/ffplay: add hwaccel decoding support Zhao Zhili
2023-10-29 2:31 ` [FFmpeg-devel] [PATCH v3 " Zhao Zhili
2023-10-29 3:18 ` Lynne
2023-10-29 11:12 ` Niklas Haas
Git Inbox Mirror of the ffmpeg-devel mailing list - see https://ffmpeg.org/mailman/listinfo/ffmpeg-devel This inbox may be cloned and mirrored by anyone: git clone --mirror https://master.gitmailbox.com/ffmpegdev/0 ffmpegdev/git/0.git # If you have public-inbox 1.1+ installed, you may # initialize and index your mirror using the following commands: public-inbox-init -V2 ffmpegdev ffmpegdev/ https://master.gitmailbox.com/ffmpegdev \ ffmpegdev@gitmailbox.com public-inbox-index ffmpegdev Example config snippet for mirrors. AGPL code for this site: git clone https://public-inbox.org/public-inbox.git