Re: [FFmpeg-devel] libavcodec: add Vulkan common video decoding code

[Anton Khirnov <anton@khirnov.net>]

> commit adb671b921d006255597ac126f85adb05f9d6677
> Author: Lynne <dev@lynne.ee>
> Date:   Mon Jan 16 07:23:27 2023 +0100
> 
>     libavcodec: add Vulkan common video decoding code
> 
> diff --git a/libavcodec/Makefile b/libavcodec/Makefile
> index d99f7bd25a..362ea31e3e 100644
> --- a/libavcodec/Makefile
> +++ b/libavcodec/Makefile
> @@ -1289,7 +1289,7 @@ SKIPHEADERS-$(CONFIG_XVMC)             += xvmc.h
>  SKIPHEADERS-$(CONFIG_VAAPI)            += vaapi_decode.h vaapi_hevc.h vaapi_encode.h
>  SKIPHEADERS-$(CONFIG_VDPAU)            += vdpau.h vdpau_internal.h
>  SKIPHEADERS-$(CONFIG_VIDEOTOOLBOX)     += videotoolbox.h vt_internal.h
> -SKIPHEADERS-$(CONFIG_VULKAN)           += vulkan.h vulkan_video.h
> +SKIPHEADERS-$(CONFIG_VULKAN)           += vulkan.h vulkan_video.h vulkan_decode.h
>  SKIPHEADERS-$(CONFIG_V4L2_M2M)         += v4l2_buffers.h v4l2_context.h v4l2_m2m.h
>  SKIPHEADERS-$(CONFIG_ZLIB)             += zlib_wrapper.h
>  
> diff --git a/libavcodec/vulkan_decode.c b/libavcodec/vulkan_decode.c
> new file mode 100644
> index 0000000000..d07b9aa3eb
> --- /dev/null
> +++ b/libavcodec/vulkan_decode.c
> @@ -0,0 +1,1182 @@
> +/*
> + * This file is part of FFmpeg.
> + *
> + * FFmpeg is free software; you can redistribute it and/or
> + * modify it under the terms of the GNU Lesser General Public
> + * License as published by the Free Software Foundation; either
> + * version 2.1 of the License, or (at your option) any later version.
> + *
> + * FFmpeg is distributed in the hope that it will be useful,
> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
> + * Lesser General Public License for more details.
> + *
> + * You should have received a copy of the GNU Lesser General Public
> + * License along with FFmpeg; if not, write to the Free Software
> + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
> + */
> +
> +#include "vulkan_video.h"
> +#include "vulkan_decode.h"
> +#include "config_components.h"
> +#include "libavutil/hwcontext_internal.h"

what for?

> +static AVFrame *vk_get_dpb_pool(FFVulkanDecodeShared *ctx)
> +{
> +    AVFrame *avf = av_frame_alloc();
> +    AVHWFramesContext *dpb_frames = (AVHWFramesContext *)ctx->dpb_hwfc_ref->data;
> +    if (!avf)
> +        return NULL;
> +
> +    avf->hw_frames_ctx = av_buffer_ref(ctx->dpb_hwfc_ref);
> +    if (!avf->hw_frames_ctx)
> +        av_frame_free(&avf);
> +    avf->buf[0] = av_buffer_pool_get(dpb_frames->pool);
> +    if (!avf->buf[0])
> +        av_frame_free(&avf);
> +    avf->data[0] = avf->buf[0]->data;

Why is this not av_hwframe_get_buffer()?

> +void ff_vk_decode_free_frame(FFVulkanDecodeContext *dec, FFVulkanDecodePicture *vp)
> +{
> +    FFVulkanFunctions *vk;
> +    VkSemaphoreWaitInfo sem_wait;
> +    FFVulkanDecodeShared *ctx;
> +
> +    // TODO: investigate why this happens
> +    if (!dec || !dec->shared_ref) {

My guess is that this is called from a different thread than the one
whose hwaccel_priv_data you gave to ff_hwaccel_frame_priv_alloc().
You have to attach everything you need to hwaccel_priv_buf itself.

> +/* Since to even get decoder capabilities, we have to initialize quite a lot,
> + * this function does initialization and saves it to hwaccel_priv_data if
> + * available. */
> +static int vulkan_decode_check_init(AVCodecContext *avctx, AVBufferRef *frames_ref,
> +                                    VulkanVideoProfile *profile_data,
> +                                    int *width_align, int *height_align,
> +                                    enum AVPixelFormat *pix_fmt, VkFormat *vk_fmt,
> +                                    int *dpb_dedicate)
> +{
> +    VkResult ret;
> +    int err, max_level;
> +    const struct FFVkCodecMap *vk_codec = &ff_vk_codec_map[avctx->codec_id];
> +    AVHWFramesContext *frames = (AVHWFramesContext *)frames_ref->data;
> +    AVHWDeviceContext *device = (AVHWDeviceContext *)frames->device_ref->data;
> +    AVVulkanDeviceContext *hwctx = device->hwctx;
> +    enum AVPixelFormat source_format;
> +    enum AVPixelFormat best_format;
> +    VkFormat best_vkfmt;
> +    int base_profile, cur_profile = avctx->profile;
> +
> +    int dedicated_dpb;
> +    int layered_dpb;
> +
> +    FFVulkanDecodeContext *dec = avctx->internal->hwaccel_priv_data;
> +    FFVulkanDecodeShared *ctx = (FFVulkanDecodeShared *)dec->shared_ref->data;
> +
> +    FFVulkanExtensions local_extensions = 0x0;
> +    FFVulkanExtensions *extensions = ctx ? &ctx->s.extensions : &local_extensions;
> +    FFVulkanFunctions local_vk = { 0 };
> +    FFVulkanFunctions *vk = ctx ? &ctx->s.vkfn : &local_vk;
> +    VkVideoCapabilitiesKHR local_caps = { 0 };
> +    VkVideoCapabilitiesKHR *caps = ctx ? &ctx->common.caps : &local_caps;
> +    VkVideoDecodeCapabilitiesKHR local_dec_caps = { 0 };
> +    VkVideoDecodeCapabilitiesKHR *dec_caps = ctx ? &ctx->dec_caps : &local_dec_caps;
> +
> +    VkVideoDecodeH264ProfileInfoKHR local_h264_profile = { 0 };
> +    VkVideoDecodeH264ProfileInfoKHR *h264_profile = profile_data ?
> +                                                    &profile_data->h264_profile :
> +                                                    &local_h264_profile;
> +
> +    VkVideoDecodeH264ProfileInfoKHR local_h265_profile = { 0 };
> +    VkVideoDecodeH264ProfileInfoKHR *h265_profile = profile_data ?
> +                                                    &profile_data->h265_profile :
> +                                                    &local_h265_profile;
> +
> +    VkVideoDecodeUsageInfoKHR local_usage = { 0 };
> +    VkVideoDecodeUsageInfoKHR *usage = profile_data ?
> +                                       &profile_data->usage : &local_usage;
> +    VkVideoProfileInfoKHR local_profile = { 0 };
> +    VkVideoProfileInfoKHR *profile = profile_data ?
> +                                     &profile_data->profile : &local_profile;
> +    VkVideoProfileListInfoKHR local_profile_list = { 0 };
> +    VkVideoProfileListInfoKHR *profile_list = profile_data ?
> +                                              &profile_data->profile_list :
> +                                              &local_profile_list;
> +
> +    VkPhysicalDeviceVideoFormatInfoKHR fmt_info = {
> +        .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VIDEO_FORMAT_INFO_KHR,
> +        .pNext = profile_list,
> +    };
> +    VkVideoDecodeH264CapabilitiesKHR h264_caps = {
> +        .sType = VK_STRUCTURE_TYPE_VIDEO_DECODE_H264_CAPABILITIES_KHR,
> +    };
> +    VkVideoDecodeH265CapabilitiesKHR h265_caps = {
> +        .sType = VK_STRUCTURE_TYPE_VIDEO_DECODE_H265_CAPABILITIES_KHR,
> +    };
> +    VkVideoFormatPropertiesKHR *ret_info;
> +    uint32_t nb_out_fmts = 0;
> +
> +    const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(avctx->sw_pix_fmt);
> +    if (!desc)
> +        return AVERROR(EINVAL);
> +
> +    if (ctx && ctx->init)
> +        return 0;
> +
> +    if (!vk_codec->decode_op)
> +        return AVERROR(EINVAL);
> +
> +    *extensions = ff_vk_extensions_to_mask(hwctx->enabled_dev_extensions,
> +                                           hwctx->nb_enabled_dev_extensions);
> +
> +    if (!(*extensions & FF_VK_EXT_VIDEO_DECODE_QUEUE)) {
> +        av_log(avctx, AV_LOG_ERROR, "Device does not support the %s extension!\n",
> +               VK_KHR_VIDEO_DECODE_QUEUE_EXTENSION_NAME);
> +        return AVERROR(ENOSYS);
> +    } else if (!vk_codec->decode_extension) {
> +        av_log(avctx, AV_LOG_ERROR, "Unsupported codec for Vulkan decoding: %s!\n",
> +               avcodec_get_name(avctx->codec_id));
> +        return AVERROR(ENOSYS);
> +    } else if (!(vk_codec->decode_extension & *extensions)) {
> +        av_log(avctx, AV_LOG_ERROR, "Device does not support decoding %s!\n",
> +               avcodec_get_name(avctx->codec_id));
> +        return AVERROR(ENOSYS);
> +    }
> +
> +    err = ff_vk_load_functions(device, vk, *extensions, 1, 1);
> +    if (err < 0)
> +        return err;
> +
> +repeat:
> +    if (avctx->codec_id == AV_CODEC_ID_H264) {
> +        base_profile = FF_PROFILE_H264_CONSTRAINED_BASELINE;
> +        dec_caps->pNext = &h264_caps;
> +        usage->pNext = h264_profile;
> +        h264_profile->sType = VK_STRUCTURE_TYPE_VIDEO_DECODE_H264_PROFILE_INFO_KHR;
> +        h264_profile->stdProfileIdc = cur_profile;
> +        h264_profile->pictureLayout = avctx->field_order == AV_FIELD_UNKNOWN ||
> +                                      avctx->field_order == AV_FIELD_PROGRESSIVE ?
> +                                      VK_VIDEO_DECODE_H264_PICTURE_LAYOUT_PROGRESSIVE_KHR :
> +                                      VK_VIDEO_DECODE_H264_PICTURE_LAYOUT_INTERLACED_INTERLEAVED_LINES_BIT_KHR;
> +    } else if (avctx->codec_id == AV_CODEC_ID_H265) {
> +        base_profile = FF_PROFILE_HEVC_MAIN;
> +        dec_caps->pNext = &h265_caps;
> +        usage->pNext = h265_profile;
> +        h265_profile->sType = VK_STRUCTURE_TYPE_VIDEO_DECODE_H265_PROFILE_INFO_KHR;
> +        h265_profile->stdProfileIdc = cur_profile;
> +    }
> +
> +    usage->sType           = VK_STRUCTURE_TYPE_VIDEO_DECODE_USAGE_INFO_KHR;
> +    usage->videoUsageHints = VK_VIDEO_DECODE_USAGE_DEFAULT_KHR;
> +
> +    profile->sType               = VK_STRUCTURE_TYPE_VIDEO_PROFILE_INFO_KHR;
> +    profile->pNext               = usage;
> +    profile->videoCodecOperation = vk_codec->decode_op;
> +    profile->chromaSubsampling   = ff_vk_subsampling_from_av_desc(desc);
> +    profile->lumaBitDepth        = ff_vk_depth_from_av_depth(desc->comp[0].depth);
> +    profile->chromaBitDepth      = profile->lumaBitDepth;
> +
> +    profile_list->sType        = VK_STRUCTURE_TYPE_VIDEO_PROFILE_LIST_INFO_KHR;
> +    profile_list->profileCount = 1;
> +    profile_list->pProfiles    = profile;
> +
> +    /* Get the capabilities of the decoder for the given profile */
> +    caps->sType = VK_STRUCTURE_TYPE_VIDEO_CAPABILITIES_KHR;
> +    caps->pNext = dec_caps;
> +    dec_caps->sType = VK_STRUCTURE_TYPE_VIDEO_DECODE_CAPABILITIES_KHR;
> +    /* dec_caps->pNext already filled in */
> +
> +    ret = vk->GetPhysicalDeviceVideoCapabilitiesKHR(hwctx->phys_dev, profile,
> +                                                    caps);
> +    if (ret == VK_ERROR_VIDEO_PROFILE_OPERATION_NOT_SUPPORTED_KHR &&
> +        avctx->flags & AV_HWACCEL_FLAG_ALLOW_PROFILE_MISMATCH &&
> +        cur_profile != base_profile) {
> +        cur_profile = base_profile;
> +        av_log(avctx, AV_LOG_VERBOSE, "%s profile %s not supported, attempting "
> +               "again with profile %s\n",
> +               avcodec_get_name(avctx->codec_id),
> +               avcodec_profile_name(avctx->codec_id, avctx->profile),
> +               avcodec_profile_name(avctx->codec_id, base_profile));
> +        goto repeat;

This function is long and ugly enough even without backward gotos. What
would Dijkstra say?

> +#endif /* AVCODEC_VULKAN_DECODE_H */
> diff --git a/libavutil/vulkan.c b/libavutil/vulkan.c
> index 0ab90c8f3c..db47956198 100644
> --- a/libavutil/vulkan.c
> +++ b/libavutil/vulkan.c
> @@ -510,8 +510,8 @@ void ff_vk_exec_discard_deps(FFVulkanContext *s, FFVkExecContext *e)
>              AVVkFrame *vkf = (AVVkFrame *)f->data[0];
>              vkfc->unlock_frame(hwfc, vkf);
>              e->frame_locked[j] = 0;
> -            e->frame_update[j] = 0;
>          }
> +        e->frame_update[j] = 0;

unrelated?

-- 
Anton Khirnov
_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".