[FFmpeg-devel] [PR] swscale: add preliminary support for Vulkan (PR #22265)

[Lynne via ffmpeg-devel <ffmpeg-devel@ffmpeg.org>]

PR #22265 opened by Lynne
URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/22265
Patch URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/22265.patch

This PR implements preliminary support for Vulkan. It adds a new API to attach a hardware device (not specific to Vulkan, lets support for other APIs too). It initializes a SPIR-V compiler. And finally, it creates a hardware frames context (and frame pool) to output Vulkan frames.

The code is only active and compiled if CONFIG_UNSTABLE has been set.

The plan for SPIR-V is to use a new SPIR-V assembler I wrote: https://code.ffmpeg.org/Lynne/spindle
libglslang/libshaderc will only be used as a fallback during development/debugging and will be removed once the new SPIR-V assembler is used.


>From ab754c5e8b9344dc59bed78fee6448680f506784 Mon Sep 17 00:00:00 2001
From: Lynne <dev@lynne.ee>
Date: Sat, 7 Feb 2026 21:51:44 +0100
Subject: [PATCH 1/5] swscale: add a function to attach a Vulkan device

Necessary for filtering.

Sponsored-by: Sovereign Tech Fund
---
 libswscale/Makefile           |  2 ++
 libswscale/swscale.c          |  1 +
 libswscale/swscale.h          |  6 ++++++
 libswscale/swscale_internal.h |  8 ++++++++
 libswscale/utils.c            | 18 ++++++++++++++++++
 libswscale/vulkan.c           | 19 +++++++++++++++++++
 6 files changed, 54 insertions(+)
 create mode 100644 libswscale/vulkan.c

diff --git a/libswscale/Makefile b/libswscale/Makefile
index bde1144897..dc4323fef5 100644
--- a/libswscale/Makefile
+++ b/libswscale/Makefile
@@ -33,6 +33,8 @@ OBJS-$(CONFIG_UNSTABLE) +=                              \
        ops_memcpy.o                                     \
        ops_optimizer.o                                  \
 
+OBJS-$(CONFIG_VULKAN) += vulkan.o
+
 # Objects duplicated from other libraries for shared builds
 SHLIBOBJS                    += log2_tab.o half2float.o
 
diff --git a/libswscale/swscale.c b/libswscale/swscale.c
index 219382505c..77d935572a 100644
--- a/libswscale/swscale.c
+++ b/libswscale/swscale.c
@@ -34,6 +34,7 @@
 #include "config.h"
 #include "swscale_internal.h"
 #include "swscale.h"
+#include "libavutil/hwcontext_vulkan.h"
 
 DECLARE_ALIGNED(8, const uint8_t, ff_dither_8x8_128)[9][8] = {
     {  36, 68,  60, 92,  34, 66,  58, 90, },
diff --git a/libswscale/swscale.h b/libswscale/swscale.h
index 495d500f14..0847480d7f 100644
--- a/libswscale/swscale.h
+++ b/libswscale/swscale.h
@@ -262,6 +262,12 @@ SwsContext *sws_alloc_context(void);
  */
 void sws_free_context(SwsContext **ctx);
 
+/**
+ * Attach a hardware device to the context.
+ * Required to be called if either the input or output frames are hardware.
+ */
+int sws_context_attach_hwcontext(SwsContext *ctx, AVBufferRef *hwctx);
+
 /***************************
  * Supported frame formats *
  ***************************/
diff --git a/libswscale/swscale_internal.h b/libswscale/swscale_internal.h
index 5c58272664..d5ec961d60 100644
--- a/libswscale/swscale_internal.h
+++ b/libswscale/swscale_internal.h
@@ -37,6 +37,8 @@
 #include "libavutil/pixfmt.h"
 #include "libavutil/pixdesc.h"
 #include "libavutil/slicethread.h"
+#include "libavutil/vulkan.h"
+
 #if HAVE_ALTIVEC
 #include "libavutil/ppc/util_altivec.h"
 #endif
@@ -326,6 +328,10 @@ typedef void (*planarX2_YV12_fn)(uint8_t *dst, uint8_t *dst2,
 struct SwsSlice;
 struct SwsFilterDescriptor;
 
+typedef struct SwsInternalVulkan {
+    FFVulkanContext ctx;
+} SwsInternalVulkan;
+
 /* This struct should be aligned on at least a 32-byte boundary. */
 struct SwsInternal {
     /* Currently active user-facing options. Also contains AVClass */
@@ -701,6 +707,8 @@ struct SwsInternal {
     int          color_conversion_warned;
 
     Half2FloatTables *h2f_tables;
+
+    SwsInternalVulkan vk;
 };
 //FIXME check init (where 0)
 
diff --git a/libswscale/utils.c b/libswscale/utils.c
index 52095ab2c7..b710af5891 100644
--- a/libswscale/utils.c
+++ b/libswscale/utils.c
@@ -57,6 +57,7 @@
 #include "libavutil/x86/asm.h"
 #include "libavutil/x86/cpu.h"
 #include "libavutil/loongarch/cpu.h"
+#include "config_components.h"
 
 #include "rgb2rgb.h"
 #include "swscale.h"
@@ -2252,6 +2253,20 @@ fail:
     return NULL;
 }
 
+int sws_context_attach_hwcontext(SwsContext *sws, AVBufferRef *hwctx)
+{
+#if CONFIG_VULKAN
+    SwsInternal *c = sws_internal(sws);
+    int err = ff_vk_init(&c->vk.ctx, sws, hwctx, NULL);
+    if (err < 0)
+        return err;
+
+    return 0;
+#else
+    return AVERROR(ENOTSUP);
+#endif
+}
+
 void sws_freeContext(SwsContext *sws)
 {
     SwsInternal *c = sws_internal(sws);
@@ -2330,6 +2345,9 @@ void sws_freeContext(SwsContext *sws)
     av_freep(&c->xyz_scratch);
 
     ff_free_filters(c);
+#if CONFIG_VULKAN
+    ff_vk_uninit(&c->vk.ctx);
+#endif
 
     av_free(c);
 }
diff --git a/libswscale/vulkan.c b/libswscale/vulkan.c
new file mode 100644
index 0000000000..fc8a1fa47b
--- /dev/null
+++ b/libswscale/vulkan.c
@@ -0,0 +1,19 @@
+/*
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include "libavutil/vulkan.c"
-- 
2.52.0


>From f6bb5c18e8db74c1b4a0b8b4b45114113492ae89 Mon Sep 17 00:00:00 2001
From: Lynne <dev@lynne.ee>
Date: Sun, 8 Feb 2026 11:21:52 +0100
Subject: [PATCH 2/5] swscale: initialize a GLSL compilation context, if
 supported

Sponsored-by: Sovereign Tech Fund
---
 libavutil/vulkan_spirv.h      |  1 +
 libswscale/Makefile           |  2 ++
 libswscale/swscale_internal.h |  2 ++
 libswscale/utils.c            | 18 ++++++++++++++++++
 libswscale/vulkan_glslang.c   | 19 +++++++++++++++++++
 libswscale/vulkan_shaderc.c   | 19 +++++++++++++++++++
 6 files changed, 61 insertions(+)
 create mode 100644 libswscale/vulkan_glslang.c
 create mode 100644 libswscale/vulkan_shaderc.c

diff --git a/libavutil/vulkan_spirv.h b/libavutil/vulkan_spirv.h
index c13b50f8e7..0bdcf8962f 100644
--- a/libavutil/vulkan_spirv.h
+++ b/libavutil/vulkan_spirv.h
@@ -22,6 +22,7 @@
 #include "vulkan.h"
 
 #include "config.h"
+#include "config_components.h"
 
 typedef struct FFVkSPIRVCompiler {
     void *priv;
diff --git a/libswscale/Makefile b/libswscale/Makefile
index dc4323fef5..217d8e4e14 100644
--- a/libswscale/Makefile
+++ b/libswscale/Makefile
@@ -34,6 +34,8 @@ OBJS-$(CONFIG_UNSTABLE) +=                              \
        ops_optimizer.o                                  \
 
 OBJS-$(CONFIG_VULKAN) += vulkan.o
+OBJS-$(CONFIG_LIBGLSLANG) += vulkan_glslang.o
+OBJS-$(CONFIG_LIBSHADERC) += vulkan_shaderc.o
 
 # Objects duplicated from other libraries for shared builds
 SHLIBOBJS                    += log2_tab.o half2float.o
diff --git a/libswscale/swscale_internal.h b/libswscale/swscale_internal.h
index d5ec961d60..94e6f21959 100644
--- a/libswscale/swscale_internal.h
+++ b/libswscale/swscale_internal.h
@@ -38,6 +38,7 @@
 #include "libavutil/pixdesc.h"
 #include "libavutil/slicethread.h"
 #include "libavutil/vulkan.h"
+#include "libavutil/vulkan_spirv.h"
 
 #if HAVE_ALTIVEC
 #include "libavutil/ppc/util_altivec.h"
@@ -330,6 +331,7 @@ struct SwsFilterDescriptor;
 
 typedef struct SwsInternalVulkan {
     FFVulkanContext ctx;
+    FFVkSPIRVCompiler *spvc;
 } SwsInternalVulkan;
 
 /* This struct should be aligned on at least a 32-byte boundary. */
diff --git a/libswscale/utils.c b/libswscale/utils.c
index b710af5891..5716a0fb7c 100644
--- a/libswscale/utils.c
+++ b/libswscale/utils.c
@@ -2261,6 +2261,20 @@ int sws_context_attach_hwcontext(SwsContext *sws, AVBufferRef *hwctx)
     if (err < 0)
         return err;
 
+#if CONFIG_LIBSHADERC
+    c->vk.spvc = ff_vk_shaderc_init();
+    if (!c->vk.spvc) {
+        ff_vk_uninit(&c->vk.ctx);
+        return AVERROR(ENOMEM);
+    }
+#elif CONFIG_LIBGLSLANG
+    c->vk.spvc = ff_vk_glslang_init();
+    if (!c->vk.spvc) {
+        ff_vk_uninit(&c->vk.ctx);
+        return AVERROR(ENOMEM);
+    }
+#endif
+
     return 0;
 #else
     return AVERROR(ENOTSUP);
@@ -2348,6 +2362,10 @@ void sws_freeContext(SwsContext *sws)
 #if CONFIG_VULKAN
     ff_vk_uninit(&c->vk.ctx);
 #endif
+#if CONFIG_LIBSHADERC || CONFIG_LIBGLSLANG
+    if (c->vk.spvc)
+        c->vk.spvc->uninit(&c->vk.spvc);
+#endif
 
     av_free(c);
 }
diff --git a/libswscale/vulkan_glslang.c b/libswscale/vulkan_glslang.c
new file mode 100644
index 0000000000..9aa41567a3
--- /dev/null
+++ b/libswscale/vulkan_glslang.c
@@ -0,0 +1,19 @@
+/*
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include "libavutil/vulkan_glslang.c"
diff --git a/libswscale/vulkan_shaderc.c b/libswscale/vulkan_shaderc.c
new file mode 100644
index 0000000000..9f60bf4dfd
--- /dev/null
+++ b/libswscale/vulkan_shaderc.c
@@ -0,0 +1,19 @@
+/*
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include "libavutil/vulkan_shaderc.c"
-- 
2.52.0


>From 2e5ca59857b180602139ecb0e41a74b43660a792 Mon Sep 17 00:00:00 2001
From: Lynne <dev@lynne.ee>
Date: Sun, 15 Feb 2026 16:24:21 +0100
Subject: [PATCH 3/5] lavu/vulkan: don't redefine RET, if already defined

RET is a very common name for a define that we use in some other parts
of our codebase.

Sponsored-by: Sovereign Tech Fund
---
 libavutil/vulkan.h | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/libavutil/vulkan.h b/libavutil/vulkan.h
index 9d1a53e2be..806c5c879d 100644
--- a/libavutil/vulkan.h
+++ b/libavutil/vulkan.h
@@ -65,11 +65,13 @@
     } while (0)
 
 /* Helper, pretty much every Vulkan return value needs to be checked */
+#ifndef RET
 #define RET(x)                                                                 \
     do {                                                                       \
         if ((err = (x)) < 0)                                                   \
             goto fail;                                                         \
     } while (0)
+#endif
 
 /* Convenience macros for specialization lists */
 #define SPEC_LIST_MAX 256
-- 
2.52.0


>From 8df59b043ad1ac0a7eea4bffc02a1cc4e84eee7c Mon Sep 17 00:00:00 2001
From: Lynne <dev@lynne.ee>
Date: Sun, 15 Feb 2026 17:59:04 +0100
Subject: [PATCH 4/5] swscale: add support for outputting Vulkan frames

Sponsored-by: Sovereign Tech Fund
---
 libswscale/swscale.c          | 47 ++++++++++++++++++++++++++++++++---
 libswscale/swscale_internal.h |  2 ++
 libswscale/utils.c            |  3 +++
 3 files changed, 48 insertions(+), 4 deletions(-)

diff --git a/libswscale/swscale.c b/libswscale/swscale.c
index 77d935572a..0448e9654f 100644
--- a/libswscale/swscale.c
+++ b/libswscale/swscale.c
@@ -1210,6 +1210,32 @@ void sws_frame_end(SwsContext *sws)
     c->src_ranges.nb_ranges = 0;
 }
 
+static int sws_hwframe_ref_init(SwsContext *sws)
+{
+    int err;
+    SwsInternal *c = sws_internal(sws);
+
+    av_buffer_unref(&c->vk.hwframe_ref);
+    c->vk.hwframe_ref = av_hwframe_ctx_alloc(c->vk.ctx.device_ref);
+    if (!c->vk.hwframe_ref)
+        return AVERROR(ENOMEM);
+
+    AVHWFramesContext *hwfc = (AVHWFramesContext *)c->vk.hwframe_ref->data;
+    hwfc->format = AV_PIX_FMT_VULKAN;
+    hwfc->sw_format = sws->dst_format;
+    hwfc->width = sws->dst_w;
+    hwfc->height = sws->dst_h;
+
+    err = av_hwframe_ctx_init(c->vk.hwframe_ref);
+    if (err) {
+        av_log(sws, AV_LOG_ERROR, "AVHWFramesContext init failed: %s!\n",
+               av_err2str(err));
+        return err;
+    }
+
+    return 0;
+}
+
 int sws_frame_start(SwsContext *sws, AVFrame *dst, const AVFrame *src)
 {
     SwsInternal *c = sws_internal(sws);
@@ -1222,11 +1248,24 @@ int sws_frame_start(SwsContext *sws, AVFrame *dst, const AVFrame *src)
     if (!dst->buf[0]) {
         dst->width  = sws->dst_w;
         dst->height = sws->dst_h;
-        dst->format = sws->dst_format;
 
-        ret = av_frame_get_buffer(dst, 0);
-        if (ret < 0)
-            return ret;
+        if (c->vk.initialized) {
+            dst->format = AV_PIX_FMT_VULKAN;
+            if (!c->vk.hwframe_ref) {
+                ret = sws_hwframe_ref_init(sws);
+                if (ret < 0)
+                    return ret;
+            }
+            ret = av_hwframe_get_buffer(c->vk.hwframe_ref, dst, 0);
+            if (ret < 0)
+                return ret;
+        } else {
+            dst->format = sws->dst_format;
+            ret = av_frame_get_buffer(dst, 0);
+            if (ret < 0)
+                return ret;
+        }
+
         allocated = 1;
     }
 
diff --git a/libswscale/swscale_internal.h b/libswscale/swscale_internal.h
index 94e6f21959..5af48b5149 100644
--- a/libswscale/swscale_internal.h
+++ b/libswscale/swscale_internal.h
@@ -332,6 +332,8 @@ struct SwsFilterDescriptor;
 typedef struct SwsInternalVulkan {
     FFVulkanContext ctx;
     FFVkSPIRVCompiler *spvc;
+    AVBufferRef *hwframe_ref;
+    int initialized;
 } SwsInternalVulkan;
 
 /* This struct should be aligned on at least a 32-byte boundary. */
diff --git a/libswscale/utils.c b/libswscale/utils.c
index 5716a0fb7c..fa184dbdbb 100644
--- a/libswscale/utils.c
+++ b/libswscale/utils.c
@@ -2275,6 +2275,8 @@ int sws_context_attach_hwcontext(SwsContext *sws, AVBufferRef *hwctx)
     }
 #endif
 
+    c->vk.initialized = 1;
+
     return 0;
 #else
     return AVERROR(ENOTSUP);
@@ -2360,6 +2362,7 @@ void sws_freeContext(SwsContext *sws)
 
     ff_free_filters(c);
 #if CONFIG_VULKAN
+    av_buffer_unref(&c->vk.hwframe_ref);
     ff_vk_uninit(&c->vk.ctx);
 #endif
 #if CONFIG_LIBSHADERC || CONFIG_LIBGLSLANG
-- 
2.52.0


>From 7ee58801be5644de8a21294b4df7e0a63d10a2c6 Mon Sep 17 00:00:00 2001
From: Lynne <dev@lynne.ee>
Date: Mon, 23 Feb 2026 15:24:26 +0100
Subject: [PATCH 5/5] swscale: only enable the Vulkan code if CONFIG_UNSTABLE
 is set

Sponsored-by: Sovereign Tech Fund
---
 libswscale/swscale.c          | 7 ++++++-
 libswscale/swscale_internal.h | 4 ++++
 libswscale/utils.c            | 4 +++-
 3 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/libswscale/swscale.c b/libswscale/swscale.c
index 0448e9654f..820520e839 100644
--- a/libswscale/swscale.c
+++ b/libswscale/swscale.c
@@ -1210,6 +1210,7 @@ void sws_frame_end(SwsContext *sws)
     c->src_ranges.nb_ranges = 0;
 }
 
+#if CONFIG_UNSTABLE
 static int sws_hwframe_ref_init(SwsContext *sws)
 {
     int err;
@@ -1235,6 +1236,7 @@ static int sws_hwframe_ref_init(SwsContext *sws)
 
     return 0;
 }
+#endif
 
 int sws_frame_start(SwsContext *sws, AVFrame *dst, const AVFrame *src)
 {
@@ -1249,6 +1251,7 @@ int sws_frame_start(SwsContext *sws, AVFrame *dst, const AVFrame *src)
         dst->width  = sws->dst_w;
         dst->height = sws->dst_h;
 
+#if CONFIG_UNSTABLE
         if (c->vk.initialized) {
             dst->format = AV_PIX_FMT_VULKAN;
             if (!c->vk.hwframe_ref) {
@@ -1259,7 +1262,9 @@ int sws_frame_start(SwsContext *sws, AVFrame *dst, const AVFrame *src)
             ret = av_hwframe_get_buffer(c->vk.hwframe_ref, dst, 0);
             if (ret < 0)
                 return ret;
-        } else {
+        } else
+#endif
+        {
             dst->format = sws->dst_format;
             ret = av_frame_get_buffer(dst, 0);
             if (ret < 0)
diff --git a/libswscale/swscale_internal.h b/libswscale/swscale_internal.h
index 5af48b5149..a671b5d3fe 100644
--- a/libswscale/swscale_internal.h
+++ b/libswscale/swscale_internal.h
@@ -329,12 +329,14 @@ typedef void (*planarX2_YV12_fn)(uint8_t *dst, uint8_t *dst2,
 struct SwsSlice;
 struct SwsFilterDescriptor;
 
+#if CONFIG_UNSTABLE
 typedef struct SwsInternalVulkan {
     FFVulkanContext ctx;
     FFVkSPIRVCompiler *spvc;
     AVBufferRef *hwframe_ref;
     int initialized;
 } SwsInternalVulkan;
+#endif
 
 /* This struct should be aligned on at least a 32-byte boundary. */
 struct SwsInternal {
@@ -712,7 +714,9 @@ struct SwsInternal {
 
     Half2FloatTables *h2f_tables;
 
+#if CONFIG_UNSTABLE
     SwsInternalVulkan vk;
+#endif
 };
 //FIXME check init (where 0)
 
diff --git a/libswscale/utils.c b/libswscale/utils.c
index fa184dbdbb..7b518596f3 100644
--- a/libswscale/utils.c
+++ b/libswscale/utils.c
@@ -2255,7 +2255,7 @@ fail:
 
 int sws_context_attach_hwcontext(SwsContext *sws, AVBufferRef *hwctx)
 {
-#if CONFIG_VULKAN
+#if CONFIG_UNSTABLE && CONFIG_VULKAN
     SwsInternal *c = sws_internal(sws);
     int err = ff_vk_init(&c->vk.ctx, sws, hwctx, NULL);
     if (err < 0)
@@ -2361,6 +2361,7 @@ void sws_freeContext(SwsContext *sws)
     av_freep(&c->xyz_scratch);
 
     ff_free_filters(c);
+#if CONFIG_UNSTABLE
 #if CONFIG_VULKAN
     av_buffer_unref(&c->vk.hwframe_ref);
     ff_vk_uninit(&c->vk.ctx);
@@ -2368,6 +2369,7 @@ void sws_freeContext(SwsContext *sws)
 #if CONFIG_LIBSHADERC || CONFIG_LIBGLSLANG
     if (c->vk.spvc)
         c->vk.spvc->uninit(&c->vk.spvc);
+#endif
 #endif
 
     av_free(c);
-- 
2.52.0

_______________________________________________
ffmpeg-devel mailing list -- ffmpeg-devel@ffmpeg.org
To unsubscribe send an email to ffmpeg-devel-leave@ffmpeg.org