From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from ffbox0-bg.mplayerhq.hu (ffbox0-bg.ffmpeg.org [79.124.17.100]) by master.gitmailbox.com (Postfix) with ESMTPS id 61BED4DF8B for ; Fri, 7 Mar 2025 00:03:45 +0000 (UTC) Received: from [127.0.1.1] (localhost [127.0.0.1]) by ffbox0-bg.mplayerhq.hu (Postfix) with ESMTP id 2EB1168F466; Fri, 7 Mar 2025 02:03:41 +0200 (EET) Received: from vidala.pars.ee (vidala.pars.ee [116.203.72.101]) by ffbox0-bg.mplayerhq.hu (Postfix) with ESMTPS id B1EFD68F455 for ; Fri, 7 Mar 2025 02:03:34 +0200 (EET) DKIM-Signature: v=1; a=rsa-sha256; s=202405r; d=lynne.ee; c=relaxed/relaxed; h=Message-ID:Date:Subject:To:From; t=1741305813; bh=lLfVMGVqJLh/ejK3fTBvuSw g/bu0j+5DHYgeQcGrbE8=; b=ii0i1PlFGqoQURediJk+JQIhsY9vwaiEAFmxDQDJk0ghPirmFC poFYv6A474BVRTH8F+i9LNWie1zlt7yh32ab2/M1ELMVgNMXsdKlIWrDKuJs2TzFjIy9cdwqC0X liymeJzIub40PRv/EQ1CfxouoVzHlwEiZr27KqGGLTf1k/MRIe51XuMcHSDyd+JTFbbH64JO5Vm dMbZgur62Bo0ZuoJI8VgveFsYMmnBeXuppiQjI9DOijUvw/QcOp36Ubt1AElkUsnQ/fz1UQwLAF skE6UuuwFTziU5emQkoScySFv0t3QjNDDXBw1tCJrgzMJNI9OSgVpfoQ6t25ASWeQ+Q==; DKIM-Signature: v=1; a=ed25519-sha256; s=202405e; d=lynne.ee; c=relaxed/relaxed; h=Message-ID:Date:Subject:To:From; t=1741305813; bh=lLfVMGVqJLh/ejK3fTBvuSw g/bu0j+5DHYgeQcGrbE8=; b=Sp/GgNvapGV5eJvSvf9Nux/00i3LndV+7vEIBQp5u37cTl3RyY D536d+Dg1/CW1d9v2ecUg1mmUjPyF2VWbhAQ==; From: Lynne To: ffmpeg-devel@ffmpeg.org Date: Fri, 7 Mar 2025 01:03:16 +0100 Message-ID: <20250307000322.241060-1-dev@lynne.ee> X-Mailer: git-send-email 2.47.2 MIME-Version: 1.0 Subject: [FFmpeg-devel] [PATCH] hwcontext_vulkan: add support for mapping multiplane images into CUDA X-BeenThere: ffmpeg-devel@ffmpeg.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: FFmpeg development discussions and patches List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Reply-To: FFmpeg development discussions and patches Cc: Lynne Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit Errors-To: ffmpeg-devel-bounces@ffmpeg.org Sender: "ffmpeg-devel" Archived-At: List-Archive: List-Post: This patch refactors the CUDA import code to allow for Vulkan images with multiple planes to be mapped. --- libavutil/hwcontext_vulkan.c | 262 +++++++++++++++++++++-------------- 1 file changed, 160 insertions(+), 102 deletions(-) diff --git a/libavutil/hwcontext_vulkan.c b/libavutil/hwcontext_vulkan.c index 10521ce685..fcff34b5e2 100644 --- a/libavutil/hwcontext_vulkan.c +++ b/libavutil/hwcontext_vulkan.c @@ -3409,6 +3409,130 @@ fail: #endif #if CONFIG_CUDA +static int export_mem_to_cuda(AVHWDeviceContext *ctx, + AVHWDeviceContext *cuda_cu, CudaFunctions *cu, + AVVkFrameInternal *dst_int, int idx, + VkDeviceMemory mem, size_t size) +{ + VkResult ret; + VulkanDevicePriv *p = ctx->hwctx; + AVVulkanDeviceContext *hwctx = &p->p; + FFVulkanFunctions *vk = &p->vkctx.vkfn; + +#ifdef _WIN32 + CUDA_EXTERNAL_MEMORY_HANDLE_DESC ext_desc = { + .type = IsWindows8OrGreater() + ? CU_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_WIN32 + : CU_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_WIN32_KMT, + .size = size, + }; + VkMemoryGetWin32HandleInfoKHR export_info = { + .sType = VK_STRUCTURE_TYPE_MEMORY_GET_WIN32_HANDLE_INFO_KHR, + .memory = mem, + .handleType = IsWindows8OrGreater() + ? VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_WIN32_BIT + : VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_WIN32_KMT_BIT, + }; + + ret = vk->GetMemoryWin32HandleKHR(hwctx->act_dev, &export_info, + &ext_desc.handle.win32.handle); + if (ret != VK_SUCCESS) { + av_log(ctx, AV_LOG_ERROR, "Unable to export the image as a Win32 Handle: %s!\n", + ff_vk_ret2str(ret)); + return AVERROR_EXTERNAL; + } + dst_int->ext_mem_handle[idx] = ext_desc.handle.win32.handle; +#else + CUDA_EXTERNAL_MEMORY_HANDLE_DESC ext_desc = { + .type = CU_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_FD, + .size = size, + }; + VkMemoryGetFdInfoKHR export_info = { + .sType = VK_STRUCTURE_TYPE_MEMORY_GET_FD_INFO_KHR, + .memory = mem, + .handleType = VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_FD_BIT_KHR, + }; + + ret = vk->GetMemoryFdKHR(hwctx->act_dev, &export_info, + &ext_desc.handle.fd); + if (ret != VK_SUCCESS) { + av_log(ctx, AV_LOG_ERROR, "Unable to export the image as a FD: %s!\n", + ff_vk_ret2str(ret)); + return AVERROR_EXTERNAL; + } +#endif + + ret = CHECK_CU(cu->cuImportExternalMemory(&dst_int->ext_mem[idx], &ext_desc)); + if (ret < 0) { +#ifndef _WIN32 + close(ext_desc.handle.fd); +#endif + return AVERROR_EXTERNAL; + } + + return 0; +} + +static int export_sem_to_cuda(AVHWDeviceContext *ctx, + AVHWDeviceContext *cuda_cu, CudaFunctions *cu, + AVVkFrameInternal *dst_int, int idx, + VkSemaphore sem) +{ + VkResult ret; + VulkanDevicePriv *p = ctx->hwctx; + AVVulkanDeviceContext *hwctx = &p->p; + FFVulkanFunctions *vk = &p->vkctx.vkfn; + +#ifdef _WIN32 + VkSemaphoreGetWin32HandleInfoKHR sem_export = { + .sType = VK_STRUCTURE_TYPE_SEMAPHORE_GET_WIN32_HANDLE_INFO_KHR, + .semaphore = sem, + .handleType = IsWindows8OrGreater() + ? VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_WIN32_BIT + : VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_WIN32_KMT_BIT, + }; + CUDA_EXTERNAL_SEMAPHORE_HANDLE_DESC ext_sem_desc = { + .type = 10 /* TODO: CU_EXTERNAL_SEMAPHORE_HANDLE_TYPE_TIMELINE_SEMAPHORE_WIN32 */, + }; +#else + VkSemaphoreGetFdInfoKHR sem_export = { + .sType = VK_STRUCTURE_TYPE_SEMAPHORE_GET_FD_INFO_KHR, + .semaphore = sem, + .handleType = VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT, + }; + CUDA_EXTERNAL_SEMAPHORE_HANDLE_DESC ext_sem_desc = { + .type = 9 /* TODO: CU_EXTERNAL_SEMAPHORE_HANDLE_TYPE_TIMELINE_SEMAPHORE_FD */, + }; +#endif + +#ifdef _WIN32 + ret = vk->GetSemaphoreWin32HandleKHR(hwctx->act_dev, &sem_export, + &ext_sem_desc.handle.win32.handle); +#else + ret = vk->GetSemaphoreFdKHR(hwctx->act_dev, &sem_export, + &ext_sem_desc.handle.fd); +#endif + if (ret != VK_SUCCESS) { + av_log(ctx, AV_LOG_ERROR, "Failed to export semaphore: %s\n", + ff_vk_ret2str(ret)); + return AVERROR_EXTERNAL; + } +#ifdef _WIN32 + dst_int->ext_sem_handle[idx] = ext_sem_desc.handle.win32.handle; +#endif + + ret = CHECK_CU(cu->cuImportExternalSemaphore(&dst_int->cu_sem[idx], + &ext_sem_desc)); + if (ret < 0) { +#ifndef _WIN32 + close(ext_sem_desc.handle.fd); +#endif + return AVERROR_EXTERNAL; + } + + return 0; +} + static int vulkan_export_to_cuda(AVHWFramesContext *hwfc, AVBufferRef *cuda_hwfc, const AVFrame *frame) @@ -3423,6 +3547,7 @@ static int vulkan_export_to_cuda(AVHWFramesContext *hwfc, VulkanDevicePriv *p = ctx->hwctx; AVVulkanDeviceContext *hwctx = &p->p; FFVulkanFunctions *vk = &p->vkctx.vkfn; + int nb_images; AVHWFramesContext *cuda_fc = (AVHWFramesContext*)cuda_hwfc->data; AVHWDeviceContext *cuda_cu = cuda_fc->device_ctx; @@ -3436,13 +3561,42 @@ static int vulkan_export_to_cuda(AVHWFramesContext *hwfc, dst_int = dst_f->internal; if (!dst_int->cuda_fc_ref) { + size_t offsets[3] = { 0 }; + dst_int->cuda_fc_ref = av_buffer_ref(cuda_hwfc); if (!dst_int->cuda_fc_ref) return AVERROR(ENOMEM); + nb_images = ff_vk_count_images(dst_f); + for (int i = 0; i < nb_images; i++) { + err = export_mem_to_cuda(ctx, cuda_cu, cu, dst_int, i, + dst_f->mem[i], dst_f->size[i]); + if (err < 0) + goto fail; + + err = export_sem_to_cuda(ctx, cuda_cu, cu, dst_int, i, + dst_f->sem[i]); + if (err < 0) + goto fail; + } + + if (nb_images != planes) { + for (int i = 0; i < planes; i++) { + VkImageSubresource subres = { + .aspectMask = i == 2 ? VK_IMAGE_ASPECT_MEMORY_PLANE_2_BIT_EXT : + i == 1 ? VK_IMAGE_ASPECT_MEMORY_PLANE_1_BIT_EXT : + VK_IMAGE_ASPECT_MEMORY_PLANE_0_BIT_EXT + }; + VkSubresourceLayout layout = { 0 }; + vk->GetImageSubresourceLayout(hwctx->act_dev, dst_f->img[FFMIN(i, nb_images - 1)], + &subres, &layout); + offsets[i] = layout.offset; + } + } + for (int i = 0; i < planes; i++) { CUDA_EXTERNAL_MEMORY_MIPMAPPED_ARRAY_DESC tex_desc = { - .offset = 0, + .offset = offsets[i], .arrayDesc = { .Depth = 0, .Format = cufmt, @@ -3453,84 +3607,12 @@ static int vulkan_export_to_cuda(AVHWFramesContext *hwfc, }; int p_w, p_h; -#ifdef _WIN32 - CUDA_EXTERNAL_MEMORY_HANDLE_DESC ext_desc = { - .type = IsWindows8OrGreater() - ? CU_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_WIN32 - : CU_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_WIN32_KMT, - .size = dst_f->size[i], - }; - VkMemoryGetWin32HandleInfoKHR export_info = { - .sType = VK_STRUCTURE_TYPE_MEMORY_GET_WIN32_HANDLE_INFO_KHR, - .memory = dst_f->mem[i], - .handleType = IsWindows8OrGreater() - ? VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_WIN32_BIT - : VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_WIN32_KMT_BIT, - }; - VkSemaphoreGetWin32HandleInfoKHR sem_export = { - .sType = VK_STRUCTURE_TYPE_SEMAPHORE_GET_WIN32_HANDLE_INFO_KHR, - .semaphore = dst_f->sem[i], - .handleType = IsWindows8OrGreater() - ? VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_WIN32_BIT - : VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_WIN32_KMT_BIT, - }; - CUDA_EXTERNAL_SEMAPHORE_HANDLE_DESC ext_sem_desc = { - .type = 10 /* TODO: CU_EXTERNAL_SEMAPHORE_HANDLE_TYPE_TIMELINE_SEMAPHORE_WIN32 */, - }; - - ret = vk->GetMemoryWin32HandleKHR(hwctx->act_dev, &export_info, - &ext_desc.handle.win32.handle); - if (ret != VK_SUCCESS) { - av_log(hwfc, AV_LOG_ERROR, "Unable to export the image as a Win32 Handle: %s!\n", - ff_vk_ret2str(ret)); - err = AVERROR_EXTERNAL; - goto fail; - } - dst_int->ext_mem_handle[i] = ext_desc.handle.win32.handle; -#else - CUDA_EXTERNAL_MEMORY_HANDLE_DESC ext_desc = { - .type = CU_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_FD, - .size = dst_f->size[i], - }; - VkMemoryGetFdInfoKHR export_info = { - .sType = VK_STRUCTURE_TYPE_MEMORY_GET_FD_INFO_KHR, - .memory = dst_f->mem[i], - .handleType = VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_FD_BIT_KHR, - }; - VkSemaphoreGetFdInfoKHR sem_export = { - .sType = VK_STRUCTURE_TYPE_SEMAPHORE_GET_FD_INFO_KHR, - .semaphore = dst_f->sem[i], - .handleType = VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT, - }; - CUDA_EXTERNAL_SEMAPHORE_HANDLE_DESC ext_sem_desc = { - .type = 9 /* TODO: CU_EXTERNAL_SEMAPHORE_HANDLE_TYPE_TIMELINE_SEMAPHORE_FD */, - }; - - ret = vk->GetMemoryFdKHR(hwctx->act_dev, &export_info, - &ext_desc.handle.fd); - if (ret != VK_SUCCESS) { - av_log(hwfc, AV_LOG_ERROR, "Unable to export the image as a FD: %s!\n", - ff_vk_ret2str(ret)); - err = AVERROR_EXTERNAL; - goto fail; - } -#endif - - ret = CHECK_CU(cu->cuImportExternalMemory(&dst_int->ext_mem[i], &ext_desc)); - if (ret < 0) { -#ifndef _WIN32 - close(ext_desc.handle.fd); -#endif - err = AVERROR_EXTERNAL; - goto fail; - } - get_plane_wh(&p_w, &p_h, hwfc->sw_format, hwfc->width, hwfc->height, i); tex_desc.arrayDesc.Width = p_w; tex_desc.arrayDesc.Height = p_h; ret = CHECK_CU(cu->cuExternalMemoryGetMappedMipmappedArray(&dst_int->cu_mma[i], - dst_int->ext_mem[i], + dst_int->ext_mem[FFMIN(i, nb_images - 1)], &tex_desc)); if (ret < 0) { err = AVERROR_EXTERNAL; @@ -3544,32 +3626,6 @@ static int vulkan_export_to_cuda(AVHWFramesContext *hwfc, goto fail; } -#ifdef _WIN32 - ret = vk->GetSemaphoreWin32HandleKHR(hwctx->act_dev, &sem_export, - &ext_sem_desc.handle.win32.handle); -#else - ret = vk->GetSemaphoreFdKHR(hwctx->act_dev, &sem_export, - &ext_sem_desc.handle.fd); -#endif - if (ret != VK_SUCCESS) { - av_log(ctx, AV_LOG_ERROR, "Failed to export semaphore: %s\n", - ff_vk_ret2str(ret)); - err = AVERROR_EXTERNAL; - goto fail; - } -#ifdef _WIN32 - dst_int->ext_sem_handle[i] = ext_sem_desc.handle.win32.handle; -#endif - - ret = CHECK_CU(cu->cuImportExternalSemaphore(&dst_int->cu_sem[i], - &ext_sem_desc)); - if (ret < 0) { -#ifndef _WIN32 - close(ext_sem_desc.handle.fd); -#endif - err = AVERROR_EXTERNAL; - goto fail; - } } } @@ -4313,6 +4369,7 @@ static int vulkan_transfer_data_to_cuda(AVHWFramesContext *hwfc, AVFrame *dst, VulkanFramesPriv *fp = hwfc->hwctx; const int planes = av_pix_fmt_count_planes(hwfc->sw_format); const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(hwfc->sw_format); + int nb_images; AVHWFramesContext *cuda_fc = (AVHWFramesContext*)dst->hw_frames_ctx->data; AVHWDeviceContext *cuda_cu = cuda_fc->device_ctx; @@ -4323,6 +4380,7 @@ static int vulkan_transfer_data_to_cuda(AVHWFramesContext *hwfc, AVFrame *dst, CUDA_EXTERNAL_SEMAPHORE_SIGNAL_PARAMS s_s_par[AV_NUM_DATA_POINTERS] = { 0 }; dst_f = (AVVkFrame *)src->data[0]; + nb_images = ff_vk_count_images(dst_f); err = prepare_frame(hwfc, &fp->upload_exec, dst_f, PREP_MODE_EXTERNAL_EXPORT); if (err < 0) @@ -4346,7 +4404,7 @@ static int vulkan_transfer_data_to_cuda(AVHWFramesContext *hwfc, AVFrame *dst, } err = CHECK_CU(cu->cuWaitExternalSemaphoresAsync(dst_int->cu_sem, s_w_par, - planes, cuda_dev->stream)); + nb_images, cuda_dev->stream)); if (err < 0) goto fail; @@ -4373,7 +4431,7 @@ static int vulkan_transfer_data_to_cuda(AVHWFramesContext *hwfc, AVFrame *dst, } err = CHECK_CU(cu->cuSignalExternalSemaphoresAsync(dst_int->cu_sem, s_s_par, - planes, cuda_dev->stream)); + nb_images, cuda_dev->stream)); if (err < 0) goto fail; -- 2.47.2 _______________________________________________ ffmpeg-devel mailing list ffmpeg-devel@ffmpeg.org https://ffmpeg.org/mailman/listinfo/ffmpeg-devel To unsubscribe, visit link above, or email ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".