[FFmpeg-devel] [RFC PATCH 1/2] fftools/ffplay: add vulkan renderer via libplacebo

[Zhao Zhili <quinkblack@foxmail.com>]

From: Zhao Zhili <zhilizhao@tencent.com>

---
 configure                 |   2 +-
 fftools/Makefile          |   2 +
 fftools/ffplay.c          |  44 ++++++-
 fftools/ffplay_renderer.c | 244 ++++++++++++++++++++++++++++++++++++++
 fftools/ffplay_renderer.h |  36 ++++++
 5 files changed, 322 insertions(+), 6 deletions(-)
 create mode 100644 fftools/ffplay_renderer.c
 create mode 100644 fftools/ffplay_renderer.h

diff --git a/configure b/configure
index 7af1d943e5..c505372184 100755
--- a/configure
+++ b/configure
@@ -3904,7 +3904,7 @@ ffmpeg_select="aformat_filter anull_filter atrim_filter format_filter
 ffmpeg_suggest="ole32 psapi shell32"
 ffplay_deps="avcodec avformat avfilter swscale swresample sdl2"
 ffplay_select="crop_filter transpose_filter hflip_filter vflip_filter rotate_filter"
-ffplay_suggest="shell32"
+ffplay_suggest="shell32 libplacebo vulkan"
 ffprobe_deps="avcodec avformat"
 ffprobe_suggest="shell32"
 
diff --git a/fftools/Makefile b/fftools/Makefile
index 56820e6bc8..3c763e3db9 100644
--- a/fftools/Makefile
+++ b/fftools/Makefile
@@ -22,6 +22,8 @@ OBJS-ffmpeg +=                  \
     fftools/sync_queue.o        \
     fftools/thread_queue.o      \
 
+OBJS-ffplay += fftools/ffplay_renderer.o
+
 define DOFFTOOL
 OBJS-$(1) += fftools/cmdutils.o fftools/opt_common.o fftools/$(1).o $(OBJS-$(1)-yes)
 ifdef HAVE_GNU_WINDRES
diff --git a/fftools/ffplay.c b/fftools/ffplay.c
index d8c69e10bc..3ae9c47509 100644
--- a/fftools/ffplay.c
+++ b/fftools/ffplay.c
@@ -58,6 +58,7 @@
 #include <SDL_thread.h>
 
 #include "cmdutils.h"
+#include "ffplay_renderer.h"
 #include "opt_common.h"
 
 const char program_name[] = "ffplay";
@@ -350,6 +351,7 @@ static char *afilters = NULL;
 static int autorotate = 1;
 static int find_stream_info = 1;
 static int filter_nbthreads = 0;
+static int enable_vulkan = 0;
 
 /* current context */
 static int is_full_screen;
@@ -362,6 +364,8 @@ static SDL_Renderer *renderer;
 static SDL_RendererInfo renderer_info = {0};
 static SDL_AudioDeviceID audio_dev;
 
+static VkRenderer *vk_renderer;
+
 static const struct TextureFormatEntry {
     enum AVPixelFormat format;
     int texture_fmt;
@@ -954,6 +958,11 @@ static void video_image_display(VideoState *is)
     SDL_Rect rect;
 
     vp = frame_queue_peek_last(&is->pictq);
+    if (vk_renderer) {
+        vk_renderer_display(vk_renderer, vp->frame);
+        return;
+    }
+
     if (is->subtitle_st) {
         if (frame_queue_nb_remaining(&is->subpq) > 0) {
             sp = frame_queue_peek(&is->subpq);
@@ -1289,6 +1298,8 @@ static void do_exit(VideoState *is)
     }
     if (renderer)
         SDL_DestroyRenderer(renderer);
+    if (vk_renderer)
+        vk_renderer_destroy(vk_renderer);
     if (window)
         SDL_DestroyWindow(window);
     uninit_opts();
@@ -3611,6 +3622,7 @@ static const OptionDef options[] = {
     { "find_stream_info", OPT_BOOL | OPT_INPUT | OPT_EXPERT, { &find_stream_info },
         "read and decode the streams to fill missing information with heuristics" },
     { "filter_threads", HAS_ARG | OPT_INT | OPT_EXPERT, { &filter_nbthreads }, "number of filter threads per graph" },
+    { "enable_vulkan", OPT_BOOL, { &enable_vulkan }, "enable vulkan render" },
     { NULL, },
 };
 
@@ -3725,9 +3737,31 @@ int main(int argc, char **argv)
 #ifdef SDL_HINT_VIDEO_X11_NET_WM_BYPASS_COMPOSITOR
         SDL_SetHint(SDL_HINT_VIDEO_X11_NET_WM_BYPASS_COMPOSITOR, "0");
 #endif
+        if (enable_vulkan) {
+            vk_renderer = vk_get_renderer();
+            if (vk_renderer) {
+#if SDL_VERSION_ATLEAST(2, 0, 6) 
+                flags |= SDL_WINDOW_VULKAN;
+#endif
+            } else {
+                av_log(NULL, AV_LOG_WARNING, "Doesn't support vulkan renderer, fallback to SDL renderer\n");
+                enable_vulkan = 0;
+            }
+        }
         window = SDL_CreateWindow(program_name, SDL_WINDOWPOS_UNDEFINED, SDL_WINDOWPOS_UNDEFINED, default_width, default_height, flags);
         SDL_SetHint(SDL_HINT_RENDER_SCALE_QUALITY, "linear");
-        if (window) {
+        if (!window) {
+            av_log(NULL, AV_LOG_FATAL, "Failed to create window: %s", SDL_GetError());
+            do_exit(NULL);
+        }
+
+        if (vk_renderer) {
+            ret = vk_renderer_create(vk_renderer, window);
+            if (ret < 0) {
+                av_log(NULL, AV_LOG_FATAL, "Failed to create vulkan renderer, %s\n", av_err2str(ret));
+                do_exit(NULL);
+            }
+        } else {
             renderer = SDL_CreateRenderer(window, -1, SDL_RENDERER_ACCELERATED | SDL_RENDERER_PRESENTVSYNC);
             if (!renderer) {
                 av_log(NULL, AV_LOG_WARNING, "Failed to initialize a hardware accelerated renderer: %s\n", SDL_GetError());
@@ -3737,10 +3771,10 @@ int main(int argc, char **argv)
                 if (!SDL_GetRendererInfo(renderer, &renderer_info))
                     av_log(NULL, AV_LOG_VERBOSE, "Initialized %s renderer.\n", renderer_info.name);
             }
-        }
-        if (!window || !renderer || !renderer_info.num_texture_formats) {
-            av_log(NULL, AV_LOG_FATAL, "Failed to create window or renderer: %s", SDL_GetError());
-            do_exit(NULL);
+            if (!renderer || !renderer_info.num_texture_formats) {
+                av_log(NULL, AV_LOG_FATAL, "Failed to create window or renderer: %s", SDL_GetError());
+                do_exit(NULL);
+            }
         }
     }
 
diff --git a/fftools/ffplay_renderer.c b/fftools/ffplay_renderer.c
new file mode 100644
index 0000000000..b03729c064
--- /dev/null
+++ b/fftools/ffplay_renderer.c
@@ -0,0 +1,244 @@
+/*
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include "config.h"
+#include "ffplay_renderer.h"
+
+#define HAVE_VULKAN_RENDERER (SDL_VERSION_ATLEAST(2, 0, 6) && CONFIG_LIBPLACEBO)
+
+struct VkRenderer {
+    const AVClass *class;
+
+    int (*create)(VkRenderer *renderer, SDL_Window *window);
+
+    int (*display)(VkRenderer *renderer, AVFrame *frame);
+
+    void (*destroy)(VkRenderer *renderer);
+};
+
+#if HAVE_VULKAN_RENDERER
+
+#include <libplacebo/vulkan.h>
+#include <libplacebo/utils/frame_queue.h>
+#include <libplacebo/utils/libav.h>
+#include <SDL_vulkan.h>
+
+typedef struct RendererContext {
+    VkRenderer api;
+    pl_vk_inst vk_inst;
+    pl_vulkan pl_vk;
+    pl_swapchain swapchain;
+    VkSurfaceKHR vk_surface;
+    pl_renderer renderer;
+    pl_tex tex[4];
+
+    pl_log vk_log;
+} RendererContext;
+
+static void vk_log_cb(void *log_priv, enum pl_log_level level, const char *msg) {
+    static const int level_map[] = {
+            AV_LOG_QUIET,
+            AV_LOG_FATAL,
+            AV_LOG_ERROR,
+            AV_LOG_WARNING,
+            AV_LOG_INFO,
+            AV_LOG_DEBUG,
+            AV_LOG_TRACE,
+    };
+
+    if (level > 0 && level < FF_ARRAY_ELEMS(level_map))
+        av_log(log_priv, level_map[level], "%s\n", msg);
+}
+
+static int create(VkRenderer *renderer, SDL_Window *window)
+{
+    int ret = 0;
+    unsigned ext_num = 0;
+    const char **ext = NULL;
+    int w, h;
+    struct pl_log_params vk_log_params = {
+            .log_cb = vk_log_cb,
+            .log_level = PL_LOG_WARN,
+            .log_priv = renderer,
+    };
+    RendererContext *ctx = (RendererContext *)renderer;
+
+    if (!SDL_Vulkan_GetInstanceExtensions(window, &ext_num, NULL)) {
+        av_log(NULL, AV_LOG_FATAL, "Failed to get vulkan extensions: %s\n", SDL_GetError());
+        return AVERROR_EXTERNAL;
+    }
+
+    ext = av_calloc(ext_num, sizeof(*ext));
+    if (!ext) {
+        ret = AVERROR(ENOMEM);
+        goto out;
+    }
+
+    SDL_Vulkan_GetInstanceExtensions(window, &ext_num, ext);
+
+    ctx->vk_log = pl_log_create(PL_API_VER, &vk_log_params);
+    ctx->vk_inst = pl_vk_inst_create(ctx->vk_log, pl_vk_inst_params(
+            .get_proc_addr = SDL_Vulkan_GetVkGetInstanceProcAddr(),
+            .debug = false,
+            .extensions = ext,
+            .num_extensions = ext_num,
+    ));
+    if (!ctx->vk_inst) {
+        ret = AVERROR_EXTERNAL;
+        goto out;
+    }
+
+    if (!SDL_Vulkan_CreateSurface(window, ctx->vk_inst->instance, &ctx->vk_surface)) {
+        ret = AVERROR_EXTERNAL;
+        goto out;
+    }
+
+    ctx->pl_vk = pl_vulkan_create(ctx->vk_log, pl_vulkan_params(
+            .instance = ctx->vk_inst->instance,
+            .get_proc_addr = ctx->vk_inst->get_proc_addr,
+            .surface = ctx->vk_surface,
+            .allow_software = false,
+    ));
+    if (!ctx->pl_vk) {
+        ret = AVERROR_EXTERNAL;
+        goto out;
+    }
+
+    ctx->swapchain = pl_vulkan_create_swapchain(ctx->pl_vk,
+            pl_vulkan_swapchain_params(.surface = ctx->vk_surface,
+                                           .present_mode = VK_PRESENT_MODE_FIFO_KHR));
+
+    SDL_Vulkan_GetDrawableSize(window, &w, &h);
+    pl_swapchain_resize(ctx->swapchain, &w, &h);
+
+    ctx->renderer = pl_renderer_create(ctx->vk_log, ctx->pl_vk->gpu);
+    if (!ctx->renderer) {
+        ret = AVERROR_EXTERNAL;
+        goto out;
+    }
+    ret = 0;
+
+out:
+    av_free(ext);
+    return ret;
+}
+
+static int display(VkRenderer *renderer, AVFrame *frame)
+{
+    struct pl_swapchain_frame swap_frame = {0};
+    struct pl_frame pl_frame = {0};
+    struct pl_frame target = {0};
+    RendererContext *ctx = (RendererContext *)renderer;
+
+    if (!pl_swapchain_start_frame(ctx->swapchain, &swap_frame)) {
+        av_log(NULL, AV_LOG_ERROR, "start frame failed\n");
+        return AVERROR_EXTERNAL;
+    }
+
+    if (!pl_map_avframe_ex(ctx->pl_vk->gpu, &pl_frame, pl_avframe_params(
+            .frame = frame,
+            .tex = ctx->tex,
+            ))) {
+        av_log(NULL, AV_LOG_ERROR, "pl_map_avframe_ex failed\n");
+        return AVERROR_EXTERNAL;
+    }
+
+    pl_frame_from_swapchain(&target, &swap_frame);
+    if (!pl_render_image(ctx->renderer, &pl_frame, &target, &pl_render_default_params)) {
+        av_log(NULL, AV_LOG_ERROR, "pl_render_image failed\n");
+        return AVERROR_EXTERNAL;
+    }
+
+    if (!pl_swapchain_submit_frame(ctx->swapchain)) {
+        av_log(NULL, AV_LOG_ERROR, "pl_swapchain_submit_frame failed\n");
+        return AVERROR_EXTERNAL;
+    }
+    pl_swapchain_swap_buffers(ctx->swapchain);
+
+    pl_unmap_avframe(ctx->pl_vk->gpu, &pl_frame);
+    return 0;
+}
+
+static void destroy(VkRenderer *renderer)
+{
+    PFN_vkDestroySurfaceKHR vkDestroySurfaceKHR;
+    RendererContext *ctx = (RendererContext *)renderer;
+
+    for (int i = 0; i < FF_ARRAY_ELEMS(ctx->tex); i++)
+        pl_tex_destroy(ctx->pl_vk->gpu, &ctx->tex[i]);
+    pl_renderer_destroy(&ctx->renderer);
+    pl_swapchain_destroy(&ctx->swapchain);
+    pl_vulkan_destroy(&ctx->pl_vk);
+
+    if (ctx->vk_surface) {
+        vkDestroySurfaceKHR = (PFN_vkDestroySurfaceKHR)
+                ctx->vk_inst->get_proc_addr(ctx->vk_inst->instance, "vkDestroySurfaceKHR");
+        vkDestroySurfaceKHR(ctx->vk_inst->instance, ctx->vk_surface, NULL);
+        ctx->vk_surface = NULL;
+    }
+
+    pl_vk_inst_destroy(&ctx->vk_inst);
+    pl_log_destroy(&ctx->vk_log);
+}
+
+static const AVClass vulkan_renderer_class = {
+        .class_name = "Vulkan Renderer",
+        .item_name  = av_default_item_name,
+        .version    = LIBAVUTIL_VERSION_INT,
+};
+
+VkRenderer *vk_get_renderer(void)
+{
+    RendererContext *ctx = av_mallocz(sizeof(*ctx));
+    VkRenderer *renderer;
+
+    if (!ctx)
+        return NULL;
+
+    renderer = &ctx->api;
+    renderer->class = &vulkan_renderer_class;
+    renderer->create = create;
+    renderer->display = display;
+    renderer->destroy = destroy;
+
+    return renderer;
+}
+
+#else
+
+VkRenderer *vk_get_renderer(void)
+{
+    return NULL;
+}
+
+#endif
+
+int vk_renderer_create(VkRenderer *renderer, SDL_Window *window)
+{
+    return renderer->create(renderer, window);
+}
+
+int vk_renderer_display(VkRenderer *renderer, AVFrame *frame)
+{
+    return renderer->display(renderer, frame);
+}
+
+void vk_renderer_destroy(VkRenderer *renderer)
+{
+    renderer->destroy(renderer);
+}
diff --git a/fftools/ffplay_renderer.h b/fftools/ffplay_renderer.h
new file mode 100644
index 0000000000..aa46d27896
--- /dev/null
+++ b/fftools/ffplay_renderer.h
@@ -0,0 +1,36 @@
+/*
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#ifndef FFTOOLS_FFPLAY_RENDERER_H
+#define FFTOOLS_FFPLAY_RENDERER_H
+
+#include <SDL.h>
+
+#include "libavutil/frame.h"
+
+typedef struct VkRenderer VkRenderer;
+
+VkRenderer *vk_get_renderer(void);
+
+int vk_renderer_create(VkRenderer *renderer, SDL_Window *window);
+
+int vk_renderer_display(VkRenderer *renderer, AVFrame *frame);
+
+void vk_renderer_destroy(VkRenderer *renderer);
+
+#endif /* FFTOOLS_FFPLAY_RENDERER_H */
-- 
2.34.1

_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".