Re: [FFmpeg-devel] [PATCH v3 2/2] fftools/ffplay: add hwaccel decoding support

[Zhao Zhili <quinkblack@foxmail.com>]

On 2023/10/29 03:32, Lynne wrote:
> Oct 28, 2023, 20:14 byquinkblack@foxmail.com:
>
>> From: Zhao Zhili<zhilizhao@tencent.com>
>>
>> ---
>> v3: shared vulkan instance between libplacebo and hwcontext
>>
>>   fftools/ffplay.c          |  43 ++++++
>>   fftools/ffplay_renderer.c | 316 +++++++++++++++++++++++++++++++++++++-
>>   fftools/ffplay_renderer.h |   2 +
>>   3 files changed, 355 insertions(+), 6 deletions(-)
>>
>> diff --git a/fftools/ffplay.c b/fftools/ffplay.c
>> index 305d72d8b8..29b37a5e46 100644
>> --- a/fftools/ffplay.c
>> +++ b/fftools/ffplay.c
>> @@ -352,6 +352,7 @@ static int autorotate = 1;
>>   static int find_stream_info = 1;
>>   static int filter_nbthreads = 0;
>>   static int enable_vulkan = 0;
>> +static const char *hwaccel = NULL;
>>   
>>   /* current context */
>>   static int is_full_screen;
>> @@ -2557,6 +2558,37 @@ static int audio_open(void *opaque, AVChannelLayout *wanted_channel_layout, int
>>   return spec.size;
>>   }
>>   
>> +static int create_hwaccel(AVBufferRef **device_ctx)
>> +{
>> +    enum AVHWDeviceType type;
>> +    int ret;
>> +    AVBufferRef *vk_dev;
>> +
>> +    *device_ctx = NULL;
>> +
>> +    if (!hwaccel)
>> +        return 0;
>> +
>> +    type = av_hwdevice_find_type_by_name(hwaccel);
>> +    if (type == AV_HWDEVICE_TYPE_NONE)
>> +        return AVERROR(ENOTSUP);
>> +
>> +    ret = vk_renderer_get_hw_dev(vk_renderer, &vk_dev);
>> +    if (ret < 0)
>> +        return ret;
>> +
>> +    ret = av_hwdevice_ctx_create_derived(device_ctx, type, vk_dev, 0);
>> +    if (!ret)
>> +        return 0;
>> +
>> +    if (ret != AVERROR(ENOSYS))
>> +        return ret;
>> +
>> +    av_log(NULL, AV_LOG_WARNING, "Derive %s from vulkan not supported.\n", hwaccel);
>> +    ret = av_hwdevice_ctx_create(device_ctx, type, NULL, NULL, 0);
>> +    return ret;
>> +}
>> +
>>   /* open a given stream. Return 0 if OK */
>>   static int stream_component_open(VideoState *is, int stream_index)
>>   {
>> @@ -2624,6 +2656,12 @@ static int stream_component_open(VideoState *is, int stream_index)
>>   
>>   av_dict_set(&opts, "flags", "+copy_opaque", AV_DICT_MULTIKEY);
>>   
>> +    if (avctx->codec_type == AVMEDIA_TYPE_VIDEO) {
>> +        ret = create_hwaccel(&avctx->hw_device_ctx);
>> +        if (ret < 0)
>> +            goto fail;
>> +    }
>> +
>>   if ((ret = avcodec_open2(avctx, codec, &opts)) < 0) {
>>   goto fail;
>>   }
>> @@ -3625,6 +3663,7 @@ static const OptionDef options[] = {
>>   "read and decode the streams to fill missing information with heuristics" },
>>   { "filter_threads", HAS_ARG | OPT_INT | OPT_EXPERT, { &filter_nbthreads }, "number of filter threads per graph" },
>>   { "enable_vulkan", OPT_BOOL, { &enable_vulkan }, "enable vulkan render" },
>> +    { "hwaccel", HAS_ARG | OPT_STRING | OPT_EXPERT, { &hwaccel }, "use HW accelerated decoding" },
>>   { NULL, },
>>   };
>>   
>> @@ -3739,6 +3778,10 @@ int main(int argc, char **argv)
>>   #ifdef SDL_HINT_VIDEO_X11_NET_WM_BYPASS_COMPOSITOR
>>   SDL_SetHint(SDL_HINT_VIDEO_X11_NET_WM_BYPASS_COMPOSITOR, "0");
>>   #endif
>> +        if (hwaccel && !enable_vulkan) {
>> +            av_log(NULL, AV_LOG_INFO, "Enable vulkan renderer to support hwaccel %s\n", hwaccel);
>> +            enable_vulkan = 1;
>> +        }
>>   if (enable_vulkan) {
>>   vk_renderer = vk_get_renderer();
>>   if (vk_renderer) {
>> diff --git a/fftools/ffplay_renderer.c b/fftools/ffplay_renderer.c
>> index 49e5516d5d..497d758ec7 100644
>> --- a/fftools/ffplay_renderer.c
>> +++ b/fftools/ffplay_renderer.c
>> @@ -35,6 +35,8 @@ struct VkRenderer {
>>   
>>   int (*create)(VkRenderer *renderer, SDL_Window *window);
>>   
>> +    int (*get_hw_dev)(VkRenderer *renderer, AVBufferRef **dev);
>> +
>>   int (*display)(VkRenderer *renderer, AVFrame *frame);
>>   
>>   int (*resize)(VkRenderer *renderer, int width, int height);
>> @@ -54,6 +56,13 @@ typedef struct RendererContext {
>>   pl_tex tex[4];
>>   
>>   pl_log vk_log;
>> +
>> +    AVBufferRef *hw_device_ref;
>> +    AVBufferRef *hw_frame_ref;
>> +    enum AVPixelFormat *transfer_formats;
>> +    AVHWFramesConstraints *constraints;
>> +
>> +    AVFrame *vk_frame;
>>   } RendererContext;
>>   
>>   static void vk_log_cb(void *log_priv, enum pl_log_level level, const char *msg) {
>> @@ -74,35 +83,46 @@ static void vk_log_cb(void *log_priv, enum pl_log_level level, const char *msg)
>>   static int create(VkRenderer *renderer, SDL_Window *window)
>>   {
>>   int ret = 0;
>> -    unsigned ext_num = 0;
>> +    unsigned num_ext = 0;
>>   const char **ext = NULL;
>>   int w, h;
>>   struct pl_log_params vk_log_params = {
>>   .log_cb = vk_log_cb,
>> -            .log_level = PL_LOG_WARN,
>> +            .log_level = PL_LOG_DEBUG,
>>   .log_priv = renderer,
>>   };
>>   RendererContext *ctx = (RendererContext *)renderer;
>>   
>> -    if (!SDL_Vulkan_GetInstanceExtensions(window, &ext_num, NULL)) {
>> +    static const char *opt_exts[] = {
>> +        VK_EXT_DESCRIPTOR_BUFFER_EXTENSION_NAME,
>> +        VK_EXT_SHADER_ATOMIC_FLOAT_EXTENSION_NAME,
>> +        VK_KHR_VIDEO_DECODE_QUEUE_EXTENSION_NAME,
>> +        VK_KHR_VIDEO_DECODE_H264_EXTENSION_NAME,
>> +        VK_KHR_VIDEO_DECODE_H265_EXTENSION_NAME,
>> +        VK_KHR_VIDEO_QUEUE_EXTENSION_NAME,
>> +        "VK_MESA_video_decode_av1",
>> +    };
>> +    const int num_opt_exts = FF_ARRAY_ELEMS(opt_exts);
>> +
>> +    if (!SDL_Vulkan_GetInstanceExtensions(window, &num_ext, NULL)) {
>>   av_log(NULL, AV_LOG_FATAL, "Failed to get vulkan extensions: %s\n", SDL_GetError());
>>   return AVERROR_EXTERNAL;
>>
> That's not a complete list. Look at optional_device_exts.
> There are also device features that both we and libplacebo would like
> to be enabled. Look into vulkan_device_create_internal(), the list there
> is up to date. FFmpeg requires timeline semaphores, and descriptor_buffer
> for lavfi filters to work.

Fixed in patch v4.

Maybe we should export optional_device_exts in API.

>
> Is there no option to let libavutil create the hwdevice for both SDL and libplacebo?
> That way, that code path can also be tested.

libplacebo doesn't support that:

// Creates a new vulkan device based on the given parameters and initializes
// a new GPU. This function will internally initialize a VkDevice. There is
// currently no way to share a vulkan device with the caller. If `params` is
// left as NULL, it defaults to &pl_vulkan_default_params.
//
// Thread-safety: Safe
PL_API pl_vulkan pl_vulkan_create(pl_log log, const struct 
pl_vulkan_params *params);

I think share instance or vkImage without support from vulkan spec is 
hard to implementation,

if not impossible.

https://www.yosoygames.com.ar/wp/2022/07/vulkan-integration-between-multiple-libraries-e-g-qt-ogrenext/

> _______________________________________________
> ffmpeg-devel mailing list
> ffmpeg-devel@ffmpeg.org
> https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
>
> To unsubscribe, visit link above, or email
> ffmpeg-devel-request@ffmpeg.org  with subject "unsubscribe".
_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".