Re: [FFmpeg-devel] [PATCH] libavfilter: add vf_xfade_vulkan

[Paul B Mahol <onemda@gmail.com>]

On Tue, May 30, 2023 at 3:34 AM Marvin Scholz <epirat07@gmail.com> wrote:

> This is an initial version of vf_xfade_vulkan based
> on vf_xfade_opencl, for now only fade and wipeleft
> transitions are supported.
> ---
>  libavfilter/Makefile          |   1 +
>  libavfilter/allfilters.c      |   1 +
>  libavfilter/vf_xfade_vulkan.c | 441 ++++++++++++++++++++++++++++++++++
>  3 files changed, 443 insertions(+)
>  create mode 100644 libavfilter/vf_xfade_vulkan.c
>
> diff --git a/libavfilter/Makefile b/libavfilter/Makefile
> index 18935b1616..ff149a3733 100644
> --- a/libavfilter/Makefile
> +++ b/libavfilter/Makefile
> @@ -552,6 +552,7 @@ OBJS-$(CONFIG_XBR_FILTER)                    +=
> vf_xbr.o
>  OBJS-$(CONFIG_XCORRELATE_FILTER)             += vf_convolve.o framesync.o
>  OBJS-$(CONFIG_XFADE_FILTER)                  += vf_xfade.o
>  OBJS-$(CONFIG_XFADE_OPENCL_FILTER)           += vf_xfade_opencl.o
> opencl.o opencl/xfade.o
> +OBJS-$(CONFIG_XFADE_VULKAN_FILTER)           += vf_xfade_vulkan.o
> vulkan.o vulkan_filter.o
>  OBJS-$(CONFIG_XMEDIAN_FILTER)                += vf_xmedian.o framesync.o
>  OBJS-$(CONFIG_XSTACK_FILTER)                 += vf_stack.o framesync.o
>  OBJS-$(CONFIG_YADIF_FILTER)                  += vf_yadif.o yadif_common.o
> diff --git a/libavfilter/allfilters.c b/libavfilter/allfilters.c
> index f1f781101b..6593e4eb83 100644
> --- a/libavfilter/allfilters.c
> +++ b/libavfilter/allfilters.c
> @@ -519,6 +519,7 @@ extern const AVFilter ff_vf_xbr;
>  extern const AVFilter ff_vf_xcorrelate;
>  extern const AVFilter ff_vf_xfade;
>  extern const AVFilter ff_vf_xfade_opencl;
> +extern const AVFilter ff_vf_xfade_vulkan;
>  extern const AVFilter ff_vf_xmedian;
>  extern const AVFilter ff_vf_xstack;
>  extern const AVFilter ff_vf_yadif;
> diff --git a/libavfilter/vf_xfade_vulkan.c b/libavfilter/vf_xfade_vulkan.c
> new file mode 100644
> index 0000000000..4a47c68fb4
> --- /dev/null
> +++ b/libavfilter/vf_xfade_vulkan.c
> @@ -0,0 +1,441 @@
> +/*
> + * This file is part of FFmpeg.
> + *
> + * FFmpeg is free software; you can redistribute it and/or
> + * modify it under the terms of the GNU Lesser General Public
> + * License as published by the Free Software Foundation; either
> + * version 2.1 of the License, or (at your option) any later version.
> + *
> + * FFmpeg is distributed in the hope that it will be useful,
> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
> + * Lesser General Public License for more details.
> + *
> + * You should have received a copy of the GNU Lesser General Public
> + * License along with FFmpeg; if not, write to the Free Software
> + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
> 02110-1301 USA
> + */
> +
> +#include "libavutil/random_seed.h"
> +#include "libavutil/opt.h"
> +#include "vulkan_filter.h"
> +#include "vulkan_spirv.h"
> +#include "filters.h"
> +#include "internal.h"
> +
> +#define IN_A 0
> +#define IN_B 1
> +
> +enum XFadeTransitions {
> +    FADE,
> +    WIPELEFT,
> +    NB_TRANSITIONS,
> +};
> +
> +typedef struct XFadeParameters {
> +    float progress;
> +} XFadeParameters;
> +
> +typedef struct XFadeVulkanContext {
> +    FFVulkanContext     vkctx;
> +
> +    int                 transition;
> +    int64_t             duration;
> +    int64_t             offset;
> +
> +    int                 initialized;
> +    FFVulkanPipeline    pl;
> +    FFVkExecPool        e;
> +    FFVkQueueFamilyCtx  qf;
> +    FFVkSPIRVShader     shd;
> +    VkSampler           sampler;
> +
> +    int64_t             duration_pts;
> +    int64_t             offset_pts;
> +    int64_t             first_pts;
> +    int64_t             last_pts;
> +    int64_t             pts;
> +    int                 xfade_is_over;
> +    int                 need_second;
> +    int                 eof[2];
> +    AVFrame             *xf[2];
> +} XFadeVulkanContext;
> +
> +static const char transition_fade[] = {
> +    C(0, void transition(int idx, ivec2 pos, float progress)
>      )
> +    C(0, {
>      )
> +    C(1,     vec4 a = texture(a_images[idx], pos);
>      )
> +    C(1,     vec4 b = texture(b_images[idx], pos);
>      )
> +    C(1,     imageStore(output_images[idx], pos, mix(b, a, progress));
>      )
> +    C(0, }
>      )
> +};
> +
> +static const char transition_wipeleft[] = {
> +    C(0, void transition(int idx, ivec2 pos, float progress)
>      )
> +    C(0, {
>      )
> +    C(1,     ivec2 size = imageSize(output_images[idx]);
>      )
> +    C(1,     int  s = int(size.x * progress);
>       )
> +    C(1,     vec4 a = texture(a_images[idx], pos);
>      )
> +    C(1,     vec4 b = texture(b_images[idx], pos);
>      )
> +    C(1,     imageStore(output_images[idx], pos, pos.x > s ? b : a);
>      )
> +    C(0, }
>      )
> +};
> +
> +static av_cold int init_filter(AVFilterContext *avctx)
> +{
> +    int err = 0;
> +    uint8_t *spv_data;
> +    size_t spv_len;
> +    void *spv_opaque = NULL;
> +    XFadeVulkanContext *s = avctx->priv;
> +    FFVulkanContext *vkctx = &s->vkctx;
> +    const int planes = av_pix_fmt_count_planes(s->vkctx.output_format);
> +    FFVkSPIRVShader *shd = &s->shd;
> +    FFVkSPIRVCompiler *spv;
> +    FFVulkanDescriptorSetBinding *desc;
> +
> +    spv = ff_vk_spirv_init();
> +    if (!spv) {
> +        av_log(avctx, AV_LOG_ERROR, "Unable to initialize SPIR-V
> compiler!\n");
> +        return AVERROR_EXTERNAL;
> +    }
> +
> +    ff_vk_qf_init(vkctx, &s->qf, VK_QUEUE_COMPUTE_BIT);
> +    RET(ff_vk_exec_pool_init(vkctx, &s->qf, &s->e, s->qf.nb_queues*4, 0,
> 0, 0, NULL));
> +    RET(ff_vk_init_sampler(vkctx, &s->sampler, 1, VK_FILTER_NEAREST));
> +    RET(ff_vk_shader_init(&s->pl, &s->shd, "xfade_compute",
> +                          VK_SHADER_STAGE_COMPUTE_BIT, 0));
> +
> +    ff_vk_shader_set_compute_sizes(&s->shd, 32, 32, 1);
> +
> +    desc = (FFVulkanDescriptorSetBinding []) {
> +        {
> +            .name       = "a_images",
> +            .type       = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER,
> +            .dimensions = 2,
> +            .elems      = planes,
> +            .stages     = VK_SHADER_STAGE_COMPUTE_BIT,
> +            .samplers   = DUP_SAMPLER(s->sampler),
> +        },
> +        {
> +            .name       = "b_images",
> +            .type       = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER,
> +            .dimensions = 2,
> +            .elems      = planes,
> +            .stages     = VK_SHADER_STAGE_COMPUTE_BIT,
> +            .samplers   = DUP_SAMPLER(s->sampler),
> +        },
> +        {
> +            .name       = "output_images",
> +            .type       = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE,
> +            .mem_layout = ff_vk_shader_rep_fmt(s->vkctx.output_format),
> +            .mem_quali  = "writeonly",
> +            .dimensions = 2,
> +            .elems      = planes,
> +            .stages     = VK_SHADER_STAGE_COMPUTE_BIT,
> +        },
> +    };
> +
> +    RET(ff_vk_pipeline_descriptor_set_add(vkctx, &s->pl, shd, desc, 3, 0,
> 0));
> +
> +    GLSLC(0, layout(push_constant, std430) uniform pushConstants {
>          );
> +    GLSLC(1,    float progress;
>           );
> +    GLSLC(0, };
>           );
> +
> +    ff_vk_add_push_constant(&s->pl, 0, sizeof(XFadeParameters),
> +                            VK_SHADER_STAGE_COMPUTE_BIT);
> +
> +    switch (s->transition) {
> +        case FADE:
> +            GLSLD(transition_fade);
> +            break;
> +        case WIPELEFT:
> +            GLSLD(transition_wipeleft);
> +            break;
> +        default:
> +            err = AVERROR_BUG;
> +            goto fail;
> +    }
> +
> +    GLSLC(0, void main()
> );
> +    GLSLC(0, {
> );
> +    GLSLC(1,     ivec2 pos = ivec2(gl_GlobalInvocationID.xy);
>  );
> +    GLSLF(1,     int planes = %i;
> ,planes);
> +    GLSLC(1,     for (int i = 0; i < planes; i++) {
>  );
> +    GLSLC(2,        transition(i, pos, progress);
>  );
> +    GLSLC(1,     }
> );
> +    GLSLC(0, }
> );
> +
> +    RET(spv->compile_shader(spv, avctx, shd, &spv_data, &spv_len, "main",
> +                            &spv_opaque));
> +    RET(ff_vk_shader_create(vkctx, shd, spv_data, spv_len, "main"));
> +
> +    RET(ff_vk_init_compute_pipeline(vkctx, &s->pl, shd));
> +    RET(ff_vk_exec_pipeline_register(vkctx, &s->e, &s->pl));
> +
> +    s->initialized = 1;
> +
> +fail:
> +    if (spv_opaque)
> +        spv->free_shader(spv, &spv_opaque);
> +    if (spv)
> +        spv->uninit(&spv);
> +
> +    return err;
> +}
> +
> +static int xfade_frame(AVFilterContext *avctx, AVFrame *a, AVFrame *b)
> +{
> +    int err;
> +    AVFilterLink *outlink = avctx->outputs[0];
> +    XFadeVulkanContext *s = avctx->priv;
> +    AVFrame *frame_a = s->xf[IN_A];
> +    AVFrame *frame_b = s->xf[IN_B];
> +    float progress;
> +
> +    AVFrame *output = ff_get_video_buffer(outlink, outlink->w,
> outlink->h);
> +    if (!output) {
> +        err = AVERROR(ENOMEM);
> +        goto fail;
> +    }
> +
> +    if (!s->initialized) {
> +        AVHWFramesContext *a_fc =
> (AVHWFramesContext*)frame_a->hw_frames_ctx->data;
> +        AVHWFramesContext *b_fc =
> (AVHWFramesContext*)frame_b->hw_frames_ctx->data;
> +        if (a_fc->sw_format != b_fc->sw_format) {
> +            av_log(avctx, AV_LOG_ERROR,
> +                   "Currently the sw format of the first video neede to
> match the second!\n");
> +            return AVERROR(EINVAL);
> +        }
> +        RET(init_filter(avctx));
> +    }
> +
> +    RET(av_frame_copy_props(output, frame_a));
> +    output->pts = s->pts;
> +
> +    progress = av_clipf(
> +        1.f - ((float)(s->pts - s->first_pts - s->offset_pts) /
> s->duration_pts),
> +        0.f, 1.f);
> +
> +    RET(ff_vk_filter_process_Nin(&s->vkctx, &s->e, &s->pl, output,
> +                                (AVFrame *[]){ frame_a, frame_b }, 2,
> s->sampler,
> +                                &(XFadeParameters){ progress },
> sizeof(XFadeParameters)));
> +
> +    return ff_filter_frame(outlink, output);
> +
> +fail:
> +    av_frame_free(&output);
> +    return err;
> +}
> +
> +static int config_props_output(AVFilterLink *outlink)
> +{
> +    int err;
> +    AVFilterContext *avctx = outlink->src;
> +    XFadeVulkanContext *s = avctx->priv;
> +    AVFilterLink *inlink0 = avctx->inputs[IN_A];
> +    AVFilterLink *inlink1 = avctx->inputs[IN_B];
> +
> +    if (inlink0->w != inlink1->w || inlink0->h != inlink1->h) {
> +        av_log(avctx, AV_LOG_ERROR, "First input link %s parameters "
> +               "(size %dx%d) do not match the corresponding "
> +               "second input link %s parameters (size %dx%d)\n",
> +               avctx->input_pads[IN_A].name, inlink0->w, inlink0->h,
> +               avctx->input_pads[IN_B].name, inlink1->w, inlink1->h);
> +        return AVERROR(EINVAL);
> +    }
> +
> +    if (inlink0->time_base.num != inlink1->time_base.num ||
> +        inlink0->time_base.den != inlink1->time_base.den) {
> +        av_log(avctx, AV_LOG_ERROR, "First input link %s timebase "
> +               "(%d/%d) do not match the corresponding "
> +               "second input link %s timebase (%d/%d)\n",
> +               avctx->input_pads[0].name, inlink0->time_base.num,
> inlink0->time_base.den,
> +               avctx->input_pads[1].name, inlink1->time_base.num,
> inlink1->time_base.den);
> +        return AVERROR(EINVAL);
> +    }
> +
> +    s->first_pts = s->last_pts = s->pts = AV_NOPTS_VALUE;
> +
> +    outlink->time_base = inlink0->time_base;
> +    outlink->sample_aspect_ratio = inlink0->sample_aspect_ratio;
> +    outlink->frame_rate = inlink0->frame_rate;
> +
> +    if (s->duration)
> +        s->duration_pts = av_rescale_q(s->duration, AV_TIME_BASE_Q,
> outlink->time_base);
> +    if (s->offset)
> +        s->offset_pts = av_rescale_q(s->offset, AV_TIME_BASE_Q,
> outlink->time_base);
> +
> +    RET(ff_vk_filter_config_output(outlink));
> +
> +fail:
> +    return err;
> +}
> +
> +static int activate(AVFilterContext *avctx)
> +{
> +    XFadeVulkanContext *s = avctx->priv;
> +    AVFilterLink *outlink = avctx->outputs[0];
> +    AVFrame *in = NULL;
> +    int ret = 0, status;
> +    int64_t pts;
> +
> +    FF_FILTER_FORWARD_STATUS_BACK_ALL(outlink, avctx);
> +
> +    if (s->xfade_is_over) {
> +        ret = ff_inlink_consume_frame(avctx->inputs[1], &in);
> +        if (ret < 0) {
> +            return ret;
> +        } else if (ret > 0) {
> +            in->pts = (in->pts - s->last_pts) + s->pts;
> +            return ff_filter_frame(outlink, in);
> +        } else if (ff_inlink_acknowledge_status(avctx->inputs[1],
> &status, &pts)) {
> +            ff_outlink_set_status(outlink, status, s->pts);
> +            return 0;
> +        } else if (!ret) {
> +            if (ff_outlink_frame_wanted(outlink)) {
> +                ff_inlink_request_frame(avctx->inputs[1]);
> +                return 0;
> +            }
> +        }
> +    }
> +
> +    if (ff_inlink_queued_frames(avctx->inputs[0]) > 0) {
> +        s->xf[0] = ff_inlink_peek_frame(avctx->inputs[0], 0);
> +        if (s->xf[0]) {
> +            if (s->first_pts == AV_NOPTS_VALUE) {
> +                s->first_pts = s->xf[0]->pts;
> +            }
> +            s->pts = s->xf[0]->pts;
> +            if (s->first_pts + s->offset_pts > s->xf[0]->pts) {
> +                s->xf[0] = NULL;
> +                s->need_second = 0;
> +                ff_inlink_consume_frame(avctx->inputs[0], &in);
> +                return ff_filter_frame(outlink, in);
> +            }
> +
> +            s->need_second = 1;
> +        }
> +    }
> +
> +    if (s->xf[0] && ff_inlink_queued_frames(avctx->inputs[1]) > 0) {
> +        ff_inlink_consume_frame(avctx->inputs[0], &s->xf[0]);
> +        ff_inlink_consume_frame(avctx->inputs[1], &s->xf[1]);
> +
> +        s->last_pts = s->xf[1]->pts;
> +        s->pts = s->xf[0]->pts;
> +        if (s->xf[0]->pts - (s->first_pts + s->offset_pts) >
> s->duration_pts)
> +            s->xfade_is_over = 1;
> +        ret = xfade_frame(avctx, s->xf[0], s->xf[1]);
> +        av_frame_free(&s->xf[0]);
> +        av_frame_free(&s->xf[1]);
> +        return ret;
> +    }
> +
> +    if (ff_inlink_queued_frames(avctx->inputs[0]) > 0 &&
> +        ff_inlink_queued_frames(avctx->inputs[1]) > 0) {
> +        ff_filter_set_ready(avctx, 100);
> +        return 0;
> +    }
> +
> +    if (ff_outlink_frame_wanted(outlink)) {
> +        if (!s->eof[0] && ff_outlink_get_status(avctx->inputs[0])) {
>

This is not correct, yes I'm aware filters have it, but it should not be
introduced again.
To get EOF status of inlink only use ack call only.
Will fix vf_xfade*.c in master if time permits.


> +            s->eof[0] = 1;
> +            s->xfade_is_over = 1;
> +        }
> +        if (!s->eof[1] && ff_outlink_get_status(avctx->inputs[1])) {
> +            s->eof[1] = 1;
> +        }
> +        if (!s->eof[0] && !s->xf[0])
> +            ff_inlink_request_frame(avctx->inputs[0]);
> +        if (!s->eof[1] && (s->need_second || s->eof[0]))
> +            ff_inlink_request_frame(avctx->inputs[1]);
> +        if (s->eof[0] && s->eof[1] && (
> +            ff_inlink_queued_frames(avctx->inputs[0]) <= 0 ||
> +            ff_inlink_queued_frames(avctx->inputs[1]) <= 0))
> +            ff_outlink_set_status(outlink, AVERROR_EOF, AV_NOPTS_VALUE);
> +        return 0;
> +    }
> +
> +    return FFERROR_NOT_READY;
> +}
> +
> +static av_cold void uninit(AVFilterContext *avctx)
> +{
> +    XFadeVulkanContext *s = avctx->priv;
> +    FFVulkanContext *vkctx = &s->vkctx;
> +    FFVulkanFunctions *vk = &vkctx->vkfn;
> +
> +    ff_vk_exec_pool_free(vkctx, &s->e);
> +    ff_vk_pipeline_free(vkctx, &s->pl);
> +    ff_vk_shader_free(vkctx, &s->shd);
> +
> +    if (s->sampler)
> +        vk->DestroySampler(vkctx->hwctx->act_dev, s->sampler,
> +                           vkctx->hwctx->alloc);
> +
> +    ff_vk_uninit(&s->vkctx);
> +
> +    s->initialized = 0;
> +}
> +
> +static AVFrame *get_video_buffer(AVFilterLink *inlink, int w, int h)
> +{
> +    XFadeVulkanContext *s = inlink->dst->priv;
> +
> +    return s->xfade_is_over || !s->need_second ?
> +        ff_null_get_video_buffer   (inlink, w, h) :
> +        ff_default_get_video_buffer(inlink, w, h);
> +}
> +
> +#define OFFSET(x) offsetof(XFadeVulkanContext, x)
> +#define FLAGS (AV_OPT_FLAG_FILTERING_PARAM | AV_OPT_FLAG_VIDEO_PARAM)
> +
> +static const AVOption xfade_vulkan_options[] = {
> +    { "transition", "set cross fade transition", OFFSET(transition),
> AV_OPT_TYPE_INT, {.i64=FADE}, 0, NB_TRANSITIONS-1, FLAGS, "transition" },
> +        { "fade",      "fade transition", 0, AV_OPT_TYPE_CONST,
> {.i64=FADE}, 0, 0, FLAGS, "transition" },
> +        { "wipeleft",  "wipe left transition", 0, AV_OPT_TYPE_CONST,
> {.i64=WIPELEFT}, 0, 0, FLAGS, "transition" },
> +    { "duration", "set cross fade duration", OFFSET(duration),
> AV_OPT_TYPE_DURATION, {.i64=1000000}, 0, 60000000, FLAGS },
> +    { "offset",   "set cross fade start relative to first input stream",
> OFFSET(offset), AV_OPT_TYPE_DURATION, {.i64=0}, INT64_MIN, INT64_MAX, FLAGS
> },
> +    { NULL }
> +};
> +
> +AVFILTER_DEFINE_CLASS(xfade_vulkan);
> +
> +static const AVFilterPad xfade_vulkan_inputs[] = {
> +    {
> +        .name             = "main",
> +        .type             = AVMEDIA_TYPE_VIDEO,
> +        .get_buffer.video = get_video_buffer,
> +        .config_props     = &ff_vk_filter_config_input,
> +    },
> +    {
> +        .name             = "xfade",
> +        .type             = AVMEDIA_TYPE_VIDEO,
> +        .get_buffer.video = get_video_buffer,
> +        .config_props     = &ff_vk_filter_config_input,
> +    },
> +};
> +
> +static const AVFilterPad xfade_vulkan_outputs[] = {
> +    {
> +        .name          = "default",
> +        .type          = AVMEDIA_TYPE_VIDEO,
> +        .config_props  = &config_props_output,
> +    },
> +};
> +
> +const AVFilter ff_vf_xfade_vulkan = {
> +    .name            = "xfade_vulkan",
> +    .description     = NULL_IF_CONFIG_SMALL("Cross fade one video with
> another video."),
> +    .priv_size       = sizeof(XFadeVulkanContext),
> +    .init            = &ff_vk_filter_init,
> +    .uninit          = &uninit,
> +    .activate        = &activate,
> +    FILTER_INPUTS(xfade_vulkan_inputs),
> +    FILTER_OUTPUTS(xfade_vulkan_outputs),
> +    FILTER_SINGLE_PIXFMT(AV_PIX_FMT_VULKAN),
> +    .priv_class      = &xfade_vulkan_class,
> +    .flags_internal  = FF_FILTER_FLAG_HWFRAME_AWARE,
> +    .flags           = AVFILTER_FLAG_HWDEVICE,
> +};
> --
> 2.37.0 (Apple Git-136)
>
> _______________________________________________
> ffmpeg-devel mailing list
> ffmpeg-devel@ffmpeg.org
> https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
>
> To unsubscribe, visit link above, or email
> ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
>
_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".