From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from ffbox0-bg.mplayerhq.hu (ffbox0-bg.ffmpeg.org [79.124.17.100]) by master.gitmailbox.com (Postfix) with ESMTPS id 2030A45F91 for ; Sun, 19 Jan 2025 10:38:59 +0000 (UTC) Received: from [127.0.1.1] (localhost [127.0.0.1]) by ffbox0-bg.mplayerhq.hu (Postfix) with ESMTP id A632068B5C9; Sun, 19 Jan 2025 12:38:32 +0200 (EET) Received: from vidala.pars.ee (vidala.pars.ee [116.203.72.101]) by ffbox0-bg.mplayerhq.hu (Postfix) with ESMTPS id 4B3E968B4B4 for ; Sun, 19 Jan 2025 12:38:31 +0200 (EET) DKIM-Signature: v=1; a=rsa-sha256; s=202405r; d=lynne.ee; c=relaxed/relaxed; h=Message-ID:Date:Subject:To:From; t=1737283110; bh=oYg46qqCIXXgE/YuSLuKq1O 6L/a4n4s9pp4VnZc5R6A=; b=Bwee17Z7+hTkErJKlLE6P2cl9uVm9dVeisfBUKQ99hToPpd/uM gAu+uAPgaOhr12lc2KImg6fbEFBf/uKq1R619Y+oxwcdnmuURkHh68ICvxMbpaqgIV5q/r0Cwau 91F4ffrmDf9hk6BZFy3cMZQSfd0JgyxIbuXhLBeAqU9IZ5OlwA6vz9phTQeWqw43pWjC/oREm1h Qymbvf5Z0Y1tYHiSA5KoEzhuV2GsSHNaclgEHwXTFHGwvvBx9o6sdRkfWOKFPDzixpIIFt7vnn/ MOX1ejq5g2JYHSw47Jm8yxpLnwTmNZi2Xb11O/HPRSDcYZ365Y7IHY7uyehx2Dl+4aw==; DKIM-Signature: v=1; a=ed25519-sha256; s=202405e; d=lynne.ee; c=relaxed/relaxed; h=Message-ID:Date:Subject:To:From; t=1737283110; bh=oYg46qqCIXXgE/YuSLuKq1O 6L/a4n4s9pp4VnZc5R6A=; b=ICPsmfjOLyaZTGcGTWOiO6B9KDFKNq5Pi2atmpieFPlnjAjqqp +RBh2ZwqVZ5GY87rdv7YRNslHPaR2BU3CUDg==; From: Lynne To: ffmpeg-devel@ffmpeg.org Date: Sun, 19 Jan 2025 19:38:15 +0900 Message-ID: <20250119103826.1225044-1-dev@lynne.ee> X-Mailer: git-send-email 2.47.1 In-Reply-To: <20250119103640.1224690-1-dev@lynne.ee> References: <20250119103640.1224690-1-dev@lynne.ee> MIME-Version: 1.0 Subject: [FFmpeg-devel] [PATCH 11/12] vulkan_decode: support software-defined decoders X-BeenThere: ffmpeg-devel@ffmpeg.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: FFmpeg development discussions and patches List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Reply-To: FFmpeg development discussions and patches Cc: Lynne Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit Errors-To: ffmpeg-devel-bounces@ffmpeg.org Sender: "ffmpeg-devel" Archived-At: List-Archive: List-Post: --- libavcodec/vulkan_decode.c | 192 +++++++++++++++++++++++++++---------- libavcodec/vulkan_decode.h | 10 ++ 2 files changed, 152 insertions(+), 50 deletions(-) diff --git a/libavcodec/vulkan_decode.c b/libavcodec/vulkan_decode.c index c57998108c..87132651e2 100644 --- a/libavcodec/vulkan_decode.c +++ b/libavcodec/vulkan_decode.c @@ -24,6 +24,9 @@ #include "libavutil/mem.h" #include "libavutil/vulkan_loader.h" +#define DECODER_IS_SDR(codec_id) \ + ((codec_id) == AV_CODEC_ID_FFV1) + #if CONFIG_H264_VULKAN_HWACCEL extern const FFVulkanDecodeDescriptor ff_vk_dec_h264_desc; #endif @@ -63,7 +66,9 @@ static const VkVideoProfileInfoKHR *get_video_profile(FFVulkanDecodeShared *ctx, codec_id == AV_CODEC_ID_H264 ? VK_STRUCTURE_TYPE_VIDEO_DECODE_H264_PROFILE_INFO_KHR : codec_id == AV_CODEC_ID_HEVC ? VK_STRUCTURE_TYPE_VIDEO_DECODE_H265_PROFILE_INFO_KHR : codec_id == AV_CODEC_ID_AV1 ? VK_STRUCTURE_TYPE_VIDEO_DECODE_AV1_PROFILE_INFO_KHR : - 0; + VK_STRUCTURE_TYPE_MAX_ENUM; + if (profile_struct_type == VK_STRUCTURE_TYPE_MAX_ENUM) + return NULL; profile_list = ff_vk_find_struct(ctx->s.hwfc->create_pnext, VK_STRUCTURE_TYPE_VIDEO_PROFILE_LIST_INFO_KHR); @@ -119,6 +124,20 @@ static AVFrame *vk_get_dpb_pool(FFVulkanDecodeShared *ctx) return avf; } +static void init_frame(FFVulkanDecodeContext *dec, FFVulkanDecodePicture *vkpic) +{ + FFVulkanDecodeShared *ctx = dec->shared_ctx; + FFVulkanFunctions *vk = &ctx->s.vkfn; + + vkpic->dpb_frame = NULL; + vkpic->img_view_ref = VK_NULL_HANDLE; + vkpic->img_view_out = VK_NULL_HANDLE; + vkpic->img_view_dest = VK_NULL_HANDLE; + + vkpic->destroy_image_view = vk->DestroyImageView; + vkpic->wait_semaphores = vk->WaitSemaphores; +} + int ff_vk_decode_prepare_frame(FFVulkanDecodeContext *dec, AVFrame *pic, FFVulkanDecodePicture *vkpic, int is_current, int alloc_dpb) @@ -134,13 +153,7 @@ int ff_vk_decode_prepare_frame(FFVulkanDecodeContext *dec, AVFrame *pic, if (vkpic->img_view_ref) return 0; - vkpic->dpb_frame = NULL; - vkpic->img_view_ref = VK_NULL_HANDLE; - vkpic->img_view_out = VK_NULL_HANDLE; - vkpic->img_view_dest = VK_NULL_HANDLE; - - vkpic->destroy_image_view = vk->DestroyImageView; - vkpic->wait_semaphores = vk->WaitSemaphores; + init_frame(dec, vkpic); if (ctx->common.layered_dpb && alloc_dpb) { vkpic->img_view_ref = ctx->common.layered_view; @@ -183,6 +196,59 @@ int ff_vk_decode_prepare_frame(FFVulkanDecodeContext *dec, AVFrame *pic, return 0; } +int ff_vk_decode_prepare_frame_sdr(FFVulkanDecodeContext *dec, AVFrame *pic, + FFVulkanDecodePicture *vkpic, int is_current, + enum FFVkShaderRepFormat rep_fmt, int alloc_dpb) +{ + int err; + FFVulkanDecodeShared *ctx = dec->shared_ctx; + + vkpic->slices_size = 0; + + if (vkpic->img_view_ref) + return 0; + + init_frame(dec, vkpic); + + if (ctx->common.layered_dpb && alloc_dpb) { + vkpic->img_view_ref = ctx->common.layered_view; + vkpic->img_aspect_ref = ctx->common.layered_aspect; + } else if (alloc_dpb) { + AVHWFramesContext *dpb_frames = (AVHWFramesContext *)ctx->common.dpb_hwfc_ref->data; + AVVulkanFramesContext *dpb_hwfc = dpb_frames->hwctx; + + vkpic->dpb_frame = vk_get_dpb_pool(ctx); + if (!vkpic->dpb_frame) + return AVERROR(ENOMEM); + + err = ff_vk_create_imageview(&ctx->s, + &vkpic->img_view_ref, &vkpic->img_aspect_ref, + vkpic->dpb_frame, 0, rep_fmt); + if (err < 0) + return err; + + vkpic->img_view_dest = vkpic->img_view_ref; + } + + if (!alloc_dpb || is_current) { + AVHWFramesContext *frames = (AVHWFramesContext *)pic->hw_frames_ctx->data; + AVVulkanFramesContext *hwfc = frames->hwctx; + + err = ff_vk_create_imageview(&ctx->s, + &vkpic->img_view_out, &vkpic->img_aspect, + pic, 0, rep_fmt); + if (err < 0) + return err; + + if (!alloc_dpb) { + vkpic->img_view_ref = vkpic->img_view_out; + vkpic->img_aspect_ref = vkpic->img_aspect; + } + } + + return 0; +} + int ff_vk_decode_add_slice(AVCodecContext *avctx, FFVulkanDecodePicture *vp, const uint8_t *data, size_t size, int add_startcode, uint32_t *nb_slices, const uint32_t **offsets) @@ -223,9 +289,14 @@ int ff_vk_decode_add_slice(AVCodecContext *avctx, FFVulkanDecodePicture *vp, buf_size = 2 << av_log2(buf_size); err = ff_vk_get_pooled_buffer(&ctx->s, &ctx->buf_pool, &new_ref, + DECODER_IS_SDR(avctx->codec_id) ? + (VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | + VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT) : VK_BUFFER_USAGE_VIDEO_DECODE_SRC_BIT_KHR, ctx->s.hwfc->create_pnext, buf_size, - VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT); + VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | + (DECODER_IS_SDR(avctx->codec_id) ? + VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT : 0x0)); if (err < 0) return err; @@ -276,6 +347,10 @@ void ff_vk_decode_flush(AVCodecContext *avctx) VkCommandBuffer cmd_buf; FFVkExecContext *exec; + /* Non-video queues do not need to be reset */ + if (!(get_codecdesc(avctx->codec_id)->decode_op)) + return; + exec = ff_vk_exec_get(&ctx->s, &ctx->exec_pool); ff_vk_exec_start(&ctx->s, exec); cmd_buf = exec->buf; @@ -551,6 +626,7 @@ static int vulkan_decode_bootstrap(AVCodecContext *avctx, AVBufferRef *frames_re { int err; FFVulkanDecodeContext *dec = avctx->internal->hwaccel_priv_data; + const FFVulkanDecodeDescriptor *vk_desc = get_codecdesc(avctx->codec_id); AVHWFramesContext *frames = (AVHWFramesContext *)frames_ref->data; AVHWDeviceContext *device = (AVHWDeviceContext *)frames->device_ref->data; AVVulkanDeviceContext *hwctx = device->hwctx; @@ -569,11 +645,13 @@ static int vulkan_decode_bootstrap(AVCodecContext *avctx, AVBufferRef *frames_re ctx->s.extensions = ff_vk_extensions_to_mask(hwctx->enabled_dev_extensions, hwctx->nb_enabled_dev_extensions); - if (!(ctx->s.extensions & FF_VK_EXT_VIDEO_DECODE_QUEUE)) { - av_log(avctx, AV_LOG_ERROR, "Device does not support the %s extension!\n", - VK_KHR_VIDEO_DECODE_QUEUE_EXTENSION_NAME); - av_refstruct_unref(&dec->shared_ctx); - return AVERROR(ENOSYS); + if (vk_desc->queue_flags & VK_QUEUE_VIDEO_DECODE_BIT_KHR) { + if (!(ctx->s.extensions & FF_VK_EXT_VIDEO_DECODE_QUEUE)) { + av_log(avctx, AV_LOG_ERROR, "Device does not support the %s extension!\n", + VK_KHR_VIDEO_DECODE_QUEUE_EXTENSION_NAME); + av_refstruct_unref(&dec->shared_ctx); + return AVERROR(ENOSYS); + } } err = ff_vk_load_functions(device, &ctx->s.vkfn, ctx->s.extensions, 1, 1); @@ -927,53 +1005,61 @@ static void free_profile_data(AVHWFramesContext *hwfc) int ff_vk_frame_params(AVCodecContext *avctx, AVBufferRef *hw_frames_ctx) { - VkFormat vkfmt; + VkFormat vkfmt = VK_FORMAT_UNDEFINED; int err, dedicated_dpb; AVHWFramesContext *frames_ctx = (AVHWFramesContext*)hw_frames_ctx->data; AVVulkanFramesContext *hwfc = frames_ctx->hwctx; FFVulkanDecodeContext *dec = avctx->internal->hwaccel_priv_data; - FFVulkanDecodeProfileData *prof; - FFVulkanDecodeShared *ctx; - - frames_ctx->sw_format = AV_PIX_FMT_NONE; + FFVulkanDecodeProfileData *prof = NULL; err = vulkan_decode_bootstrap(avctx, hw_frames_ctx); if (err < 0) return err; - prof = av_mallocz(sizeof(FFVulkanDecodeProfileData)); - if (!prof) - return AVERROR(ENOMEM); + frames_ctx->sw_format = avctx->sw_pix_fmt; - err = vulkan_decode_get_profile(avctx, hw_frames_ctx, - &frames_ctx->sw_format, &vkfmt, - prof, &dedicated_dpb); - if (err < 0) { - av_free(prof); - return err; - } + if (!DECODER_IS_SDR(avctx->codec_id)) { + prof = av_mallocz(sizeof(FFVulkanDecodeProfileData)); + if (!prof) + return AVERROR(ENOMEM); + + err = vulkan_decode_get_profile(avctx, hw_frames_ctx, + &frames_ctx->sw_format, &vkfmt, + prof, &dedicated_dpb); + if (err < 0) { + av_free(prof); + return err; + } - frames_ctx->user_opaque = prof; - frames_ctx->free = free_profile_data; + frames_ctx->user_opaque = prof; + frames_ctx->free = free_profile_data; + + hwfc->create_pnext = &prof->profile_list; + } frames_ctx->width = avctx->coded_width; frames_ctx->height = avctx->coded_height; frames_ctx->format = AV_PIX_FMT_VULKAN; hwfc->format[0] = vkfmt; - hwfc->create_pnext = &prof->profile_list; hwfc->tiling = VK_IMAGE_TILING_OPTIMAL; hwfc->usage = VK_IMAGE_USAGE_TRANSFER_SRC_BIT | - VK_IMAGE_USAGE_SAMPLED_BIT | - VK_IMAGE_USAGE_VIDEO_DECODE_DST_BIT_KHR; + VK_IMAGE_USAGE_SAMPLED_BIT; - if (!dec->dedicated_dpb) - hwfc->usage |= VK_IMAGE_USAGE_VIDEO_DECODE_DPB_BIT_KHR; + if (prof) { + FFVulkanDecodeShared *ctx; - ctx = dec->shared_ctx; - if (ctx->s.extensions & (FF_VK_EXT_VIDEO_ENCODE_QUEUE | - FF_VK_EXT_VIDEO_MAINTENANCE_1)) - hwfc->usage |= VK_IMAGE_USAGE_VIDEO_ENCODE_SRC_BIT_KHR; + hwfc->usage |= VK_IMAGE_USAGE_VIDEO_DECODE_DST_BIT_KHR; + if (!dec->dedicated_dpb) + hwfc->usage |= VK_IMAGE_USAGE_VIDEO_DECODE_DPB_BIT_KHR; + + ctx = dec->shared_ctx; + if (ctx->s.extensions & (FF_VK_EXT_VIDEO_ENCODE_QUEUE | + FF_VK_EXT_VIDEO_MAINTENANCE_1)) + hwfc->usage |= VK_IMAGE_USAGE_VIDEO_ENCODE_SRC_BIT_KHR; + } else if (DECODER_IS_SDR(avctx->codec_id)) { + hwfc->usage |= VK_IMAGE_USAGE_STORAGE_BIT; + } return err; } @@ -1075,8 +1161,10 @@ int ff_vk_decode_init(AVCodecContext *avctx) if (err < 0) return err; + vk_desc = get_codecdesc(avctx->codec_id); + profile = get_video_profile(ctx, avctx->codec_id); - if (!profile) { + if ((vk_desc->queue_flags & VK_QUEUE_VIDEO_DECODE_BIT_KHR) && !profile) { av_log(avctx, AV_LOG_ERROR, "Video profile missing from frames context!"); return AVERROR(EINVAL); } @@ -1109,9 +1197,11 @@ int ff_vk_decode_init(AVCodecContext *avctx) if (err < 0) goto fail; - err = ff_vk_video_common_init(avctx, s, &ctx->common, &session_create); - if (err < 0) - goto fail; + if (profile) { + err = ff_vk_video_common_init(avctx, s, &ctx->common, &session_create); + if (err < 0) + goto fail; + } /* If doing an out-of-place decoding, create a DPB pool */ if (dec->dedicated_dpb || avctx->codec_id == AV_CODEC_ID_AV1) { @@ -1163,12 +1253,14 @@ int ff_vk_decode_init(AVCodecContext *avctx) } session_params_create.videoSession = ctx->common.session; - ret = vk->CreateVideoSessionParametersKHR(s->hwctx->act_dev, &session_params_create, - s->hwctx->alloc, &ctx->empty_session_params); - if (ret != VK_SUCCESS) { - av_log(avctx, AV_LOG_ERROR, "Unable to create empty Vulkan video session parameters: %s!\n", - ff_vk_ret2str(ret)); - return AVERROR_EXTERNAL; + if (profile) { + ret = vk->CreateVideoSessionParametersKHR(s->hwctx->act_dev, &session_params_create, + s->hwctx->alloc, &ctx->empty_session_params); + if (ret != VK_SUCCESS) { + av_log(avctx, AV_LOG_ERROR, "Unable to create empty Vulkan video session parameters: %s!\n", + ff_vk_ret2str(ret)); + return AVERROR_EXTERNAL; + } } driver_props = &dec->shared_ctx->s.driver_props; diff --git a/libavcodec/vulkan_decode.h b/libavcodec/vulkan_decode.h index 1d89db323f..9a11a80f95 100644 --- a/libavcodec/vulkan_decode.h +++ b/libavcodec/vulkan_decode.h @@ -56,6 +56,9 @@ typedef struct FFVulkanDecodeShared { VkVideoDecodeCapabilitiesKHR dec_caps; VkVideoSessionParametersKHR empty_session_params; + + /* Software-defined decoder context */ + void *sd_ctx; } FFVulkanDecodeShared; typedef struct FFVulkanDecodeContext { @@ -141,6 +144,13 @@ int ff_vk_decode_prepare_frame(FFVulkanDecodeContext *dec, AVFrame *pic, FFVulkanDecodePicture *vkpic, int is_current, int alloc_dpb); +/** + * Software-defined decoder version of ff_vk_decode_prepare_frame. + */ +int ff_vk_decode_prepare_frame_sdr(FFVulkanDecodeContext *dec, AVFrame *pic, + FFVulkanDecodePicture *vkpic, int is_current, + enum FFVkShaderRepFormat rep_fmt, int alloc_dpb); + /** * Add slice data to frame. */ -- 2.47.1 _______________________________________________ ffmpeg-devel mailing list ffmpeg-devel@ffmpeg.org https://ffmpeg.org/mailman/listinfo/ffmpeg-devel To unsubscribe, visit link above, or email ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".