[FFmpeg-devel] [PATCH v2 11/13] scale_vulkan: add support for basic Debayering

[Lynne <dev@lynne.ee>]

---
 libavfilter/vf_scale_vulkan.c   |  60 +++++++++++++++++--
 libavfilter/vulkan/Makefile     |   1 +
 libavfilter/vulkan/debayer.comp | 102 ++++++++++++++++++++++++++++++++
 3 files changed, 159 insertions(+), 4 deletions(-)
 create mode 100644 libavfilter/vulkan/debayer.comp

diff --git a/libavfilter/vf_scale_vulkan.c b/libavfilter/vf_scale_vulkan.c
index 46b1476933..5cb09ac385 100644
--- a/libavfilter/vf_scale_vulkan.c
+++ b/libavfilter/vf_scale_vulkan.c
@@ -27,6 +27,8 @@
 #include "colorspace.h"
 #include "video.h"
 
+extern const char *ff_source_debayer_comp;
+
 enum ScalerFunc {
     F_BILINEAR = 0,
     F_NEAREST,
@@ -34,6 +36,13 @@ enum ScalerFunc {
     F_NB,
 };
 
+enum DebayerFunc {
+    DB_BILINEAR = 0,
+    DB_BILINEAR_HQ,
+
+    DB_NB,
+};
+
 typedef struct ScaleVulkanContext {
     FFVulkanContext vkctx;
 
@@ -58,6 +67,7 @@ typedef struct ScaleVulkanContext {
 
     enum ScalerFunc scaler;
     enum AVColorRange out_range;
+    enum DebayerFunc debayer;
 } ScaleVulkanContext;
 
 static const char scale_bilinear[] = {
@@ -184,6 +194,25 @@ static int init_scale_shader(ScaleVulkanContext *s, FFVulkanShader *shd,
     return 0;
 }
 
+static int init_debayer_shader(ScaleVulkanContext *s, FFVulkanShader *shd,
+                               FFVulkanDescriptorSetBinding *desc, AVFrame *in)
+{
+    GLSLD(ff_source_debayer_comp);
+
+    GLSLC(0, void main(void));
+    GLSLC(0, {              );
+    if (s->debayer == DB_BILINEAR)
+        GLSLC(1, debayer_bilinear(););
+    else if (s->debayer == DB_BILINEAR_HQ)
+        GLSLC(1, debayer_bilinear_hq(););
+    GLSLC(0, }              );
+
+    shd->lg_size[0] <<= 1;
+    shd->lg_size[1] <<= 1;
+
+    return 0;
+}
+
 static av_cold int init_filter(AVFilterContext *ctx, AVFrame *in)
 {
     int err;
@@ -197,6 +226,7 @@ static av_cold int init_filter(AVFilterContext *ctx, AVFrame *in)
     FFVkSPIRVCompiler *spv;
     FFVulkanDescriptorSetBinding *desc;
 
+    int debayer = s->vkctx.input_format == AV_PIX_FMT_BAYER_RGGB16;
     int in_planes = av_pix_fmt_count_planes(s->vkctx.input_format);
 
     switch (s->scaler) {
@@ -222,7 +252,10 @@ static av_cold int init_filter(AVFilterContext *ctx, AVFrame *in)
     }
 
     RET(ff_vk_exec_pool_init(vkctx, s->qf, &s->e, s->qf->num*4, 0, 0, 0, NULL));
-    RET(ff_vk_init_sampler(vkctx, &s->sampler, 0, sampler_mode));
+
+    if (!debayer)
+        RET(ff_vk_init_sampler(vkctx, &s->sampler, 0, sampler_mode));
+
     RET(ff_vk_shader_init(vkctx, &s->shd, "scale",
                           VK_SHADER_STAGE_COMPUTE_BIT,
                           NULL, 0,
@@ -232,7 +265,13 @@ static av_cold int init_filter(AVFilterContext *ctx, AVFrame *in)
     desc = (FFVulkanDescriptorSetBinding []) {
         {
             .name       = "input_img",
-            .type       = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER,
+            .type       = debayer ?
+                          VK_DESCRIPTOR_TYPE_STORAGE_IMAGE :
+                          VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER,
+            .mem_layout = debayer ?
+                          ff_vk_shader_rep_fmt(s->vkctx.input_format, FF_VK_REP_FLOAT) :
+                          NULL,
+            .mem_quali  = "readonly",
             .dimensions = 2,
             .elems      = in_planes,
             .stages     = VK_SHADER_STAGE_COMPUTE_BIT,
@@ -263,7 +302,10 @@ static av_cold int init_filter(AVFilterContext *ctx, AVFrame *in)
     ff_vk_shader_add_push_const(&s->shd, 0, sizeof(s->opts),
                                 VK_SHADER_STAGE_COMPUTE_BIT);
 
-    err = init_scale_shader(s, shd, desc, in);
+    if (debayer)
+        err = init_debayer_shader(s, shd, desc, in);
+    else
+        err = init_scale_shader(s, shd, desc, in);
     if (err < 0)
         goto fail;
 
@@ -361,7 +403,14 @@ static int scale_vulkan_config_output(AVFilterLink *outlink)
         s->vkctx.output_format = s->vkctx.input_format;
     }
 
-    if (s->vkctx.output_format != s->vkctx.input_format) {
+    if (s->vkctx.input_format == AV_PIX_FMT_BAYER_RGGB16) {
+        if (s->vkctx.output_format == s->vkctx.input_format) {
+            s->vkctx.output_format = AV_PIX_FMT_RGBA64;
+        } else if (!ff_vk_mt_is_np_rgb(s->vkctx.output_format)) {
+            av_log(avctx, AV_LOG_ERROR, "Unsupported output format for debayer\n");
+            return AVERROR(EINVAL);
+        }
+    } else if (s->vkctx.output_format != s->vkctx.input_format) {
         if (!ff_vk_mt_is_np_rgb(s->vkctx.input_format)) {
             av_log(avctx, AV_LOG_ERROR, "Unsupported input format for conversion\n");
             return AVERROR(EINVAL);
@@ -406,6 +455,9 @@ static const AVOption scale_vulkan_options[] = {
     { "scaler", "Scaler function", OFFSET(scaler), AV_OPT_TYPE_INT, {.i64 = F_BILINEAR}, 0, F_NB, .flags = FLAGS, .unit = "scaler" },
         { "bilinear", "Bilinear interpolation (fastest)", 0, AV_OPT_TYPE_CONST, {.i64 = F_BILINEAR}, 0, 0, .flags = FLAGS, .unit = "scaler" },
         { "nearest", "Nearest (useful for pixel art)", 0, AV_OPT_TYPE_CONST, {.i64 = F_NEAREST}, 0, 0, .flags = FLAGS, .unit = "scaler" },
+    { "debayer", "Debayer algorithm to use", OFFSET(debayer), AV_OPT_TYPE_INT, {.i64 = DB_BILINEAR_HQ}, 0, DB_NB, .flags = FLAGS, .unit = "debayer" },
+        { "bilinear", "Bilinear debayering (fastest)", 0, AV_OPT_TYPE_CONST, {.i64 = DB_BILINEAR}, 0, 0, .flags = FLAGS, .unit = "debayer" },
+        { "bilinear_hq", "Bilinear debayering (high quality)", 0, AV_OPT_TYPE_CONST, {.i64 = DB_BILINEAR_HQ}, 0, 0, .flags = FLAGS, .unit = "debayer" },
     { "format", "Output video format (software format of hardware frames)", OFFSET(out_format_string), AV_OPT_TYPE_STRING, .flags = FLAGS },
     { "out_range", "Output colour range (from 0 to 2) (default 0)", OFFSET(out_range), AV_OPT_TYPE_INT, {.i64 = AVCOL_RANGE_UNSPECIFIED}, AVCOL_RANGE_UNSPECIFIED, AVCOL_RANGE_JPEG, .flags = FLAGS, .unit = "range" },
         { "full", "Full range", 0, AV_OPT_TYPE_CONST, { .i64 = AVCOL_RANGE_JPEG }, 0, 0, FLAGS, .unit = "range" },
diff --git a/libavfilter/vulkan/Makefile b/libavfilter/vulkan/Makefile
index 573eee32c7..c77aaf4f6b 100644
--- a/libavfilter/vulkan/Makefile
+++ b/libavfilter/vulkan/Makefile
@@ -4,6 +4,7 @@ clean::
 	$(RM) $(GEN_CLEANSUFFIXES:%=libavfilter/vulkan/%)
 
 OBJS-$(CONFIG_BWDIF_VULKAN_FILTER) += vulkan/bwdif.o
+OBJS-$(CONFIG_SCALE_VULKAN_FILTER) += vulkan/debayer.o
 
 VULKAN = $(subst $(SRC_PATH)/,,$(wildcard $(SRC_PATH)/libavfilter/vulkan/*.comp))
 .SECONDARY: $(VULKAN:.comp=.c)
diff --git a/libavfilter/vulkan/debayer.comp b/libavfilter/vulkan/debayer.comp
new file mode 100644
index 0000000000..c86c2f5eec
--- /dev/null
+++ b/libavfilter/vulkan/debayer.comp
@@ -0,0 +1,102 @@
+/*
+ *
+ * Copyright (c) 2025 Lynne <dev@lynne.ee>
+ *
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#define LD(xo, yo) \
+    (imageLoad(input_img[0], pos + ivec2((xo), (yo))).r)
+
+void debayer_bilinear(void)
+{
+    ivec2 pos = ivec2(gl_GlobalInvocationID.xy) << 1;
+
+    /* R basis */
+    vec4 tl = vec4(LD(0, 0),
+                   (LD(1, 0) + LD(-1, 0) + LD(0, 1) + LD(0, -1)) / 4.0f,
+                   (LD(-1, -1) + LD(1, 1) + LD(-1, 1) + LD(1, -1)) / 4.0f,
+                   1.0f);
+    imageStore(output_img[0], pos, tl);
+
+    /* G1 basis */
+    vec4 tr = vec4((LD(2, 0) + LD(0, 0)) / 2.0f,
+                   LD(1, 0),
+                   (LD(1, 1) + LD(1, -1)) / 2.0f,
+                   1.0f);
+    imageStore(output_img[0], pos + ivec2(1, 0), tr);
+
+    /* G2 basis */
+    vec4 bl = vec4((LD(0, 2) + LD(0, 0)) / 2.0f,
+                   LD(0, 1),
+                   (LD(1, 1) + LD(-1, 1)) / 2.0f,
+                   1.0f);
+    imageStore(output_img[0], pos + ivec2(0, 1), bl);
+
+    /* B basis */
+    vec4 br = vec4((LD(0, 0) + LD(2, 2) + LD(0, 2) + LD(2, 0)) / 4.0f,
+                   (LD(2, 1) + LD(0, 1) + LD(1, 2) + LD(1, 0)) / 4.0f,
+                   LD(1, 1),
+                   1.0f);
+    imageStore(output_img[0], pos + ivec2(1, 1), br);
+}
+
+void debayer_bilinear_hq(void)
+{
+    ivec2 pos = ivec2(gl_GlobalInvocationID.xy) << 1;
+
+    /* R basis */
+    vec4 tl = vec4(LD(0, 0),
+                   (4.0f*LD(0, 0) + 2.0f*(LD(0, -1) + LD(0, 1) + LD(-1, 0) + LD(1, 0)) -
+                    (LD(0, -2) + LD(0, 2) + LD(-2, 0) + LD(2, 0))) / 8.0f,
+                   (12.0f*LD(0, 0) + 4.0f*(LD(-1, -1) + LD(-1, 1) + LD(1, -1) + LD(1, 1)) -
+                    3.0f*(LD(0, -2) + LD(0, 2) + LD(-2, 0) + LD(2, 0))) / 16.0f,
+                   1.0f);
+    imageStore(output_img[0], pos, tl);
+
+    /* G1 basis */
+    vec4 tr = vec4((10.0f*LD(1, 0) + 8.0f*(LD(0, 0) + LD(2, 0)) -
+                    2.0f*(LD(0, -1) + LD(2, 1) + LD(0, 1) + LD(2, -1) + LD(-1, 0) + LD(3, 0)) +
+                    LD(1, -2) + LD(1, 2)) / 16.0f,
+                   LD(1, 0),
+                   (10.0f*LD(1, 0) + 8.0f*(LD(1, -1) + LD(1, 1)) -
+                    2.0f*(LD(0, -1) + LD(0, 1) + LD(2, -1) + LD(2, 1) + LD(1, -2) + LD(1, 2)) +
+                    LD(-1, 0) + LD(3, 0)) / 16.0f,
+                   1.0f);
+    imageStore(output_img[0], pos + ivec2(1, 0), tr);
+
+
+    /* G2 basis */
+    vec4 bl = vec4((10.0f*LD(0, 1) + 8.0f*(LD(0, 0) + LD(0, 2)) -
+                   2.0f*(LD(-1, 0) + LD(-1, 2) + LD(1, 0) + LD(1, 2) + LD(0, -1) + LD(0, 3)) +
+                   LD(-2, 1) + LD(2, 1)) / 16.0f,
+                   LD(0, 1),
+                   (10.0f*LD(0, 1) + 8.0f*(LD(-1, 1) + LD(1, 1)) -
+                   2.0f*(LD(-1, 0) + LD(1, 2) + LD(-1, 2) + LD(1, 0) + LD(-2, 1) + LD(2, 1)) +
+                    LD(0, -1) + LD(0, 3)) / 16.0f,
+                   1.0f);
+    imageStore(output_img[0], pos + ivec2(0, 1), bl);
+
+    /* B basis */
+    vec4 br = vec4((12.0f*LD(1, 1) + 4.0f*(LD(0, 0) + LD(0, 2) + LD(2, 0) + LD(2, 2)) -
+                    3.0f*(LD(1, -1) + LD(1, 3) + LD(-1, 1) + LD(3, 1))) / 16.0f,
+                   (4.0f*LD(1, 1) + 2.0f*(LD(1, 0) + LD(1, 2) + LD(0, 1) + LD(2, 1)) -
+                    (LD(1, -1) + LD(1, 3) + LD(-1, 1) + LD(3, 1))) / 8.0f,
+                   LD(1, 1),
+                   1.0f);
+    imageStore(output_img[0], pos + ivec2(1, 1), br);
+}
-- 
2.50.0
_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".