[FFmpeg-devel] [PATCH] WIP: avutil/hwcontext_vaapi: fix use fourcc not supported by devices (PR #20899)

[Zhao Zhili via ffmpeg-devel <ffmpeg-devel@ffmpeg.org>]

PR #20899 opened by Zhao Zhili (quink)
URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/20899
Patch URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/20899.patch

avutil/hwcontext_vaapi: fix use fourcc not supported by devices

1. A AVPixelFormat can map to multiple VA_FOURCCs, while
vaapi_format_from_pix_fmt() only returns the first item matched
before this patch.
2. vaapi_frames_init() use vaapi_format_from_pix_fmt() to get the
first item. fourcc in this item may not be supported by the device.

There is a real issue. av_hwdevice_get_hwframe_constraints returned
AV_PIX_FMT_YUV422P is a supported sw_format, while it's only supported
by VA_FOURCC_YV16, but vaapi_frames_init try to use VA_FOURCC_422H
later.

This patch makes vaapi_format_from_pix_fmt return all matched items
iteratively, then use strict check in vaapi_frames_init to get the
right fourcc.


>From 6bb427af54bf5453b998c3c12a389b087217aeda Mon Sep 17 00:00:00 2001
From: Zhao Zhili <zhilizhao@tencent.com>
Date: Wed, 12 Nov 2025 21:56:36 +0800
Subject: [PATCH 1/2] avutil/hwcontext_vaapi: fix error code for invalid
 argument

Also replace goto by return directly, as there is nothing to
cleanup.
---
 libavutil/hwcontext_vaapi.c | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/libavutil/hwcontext_vaapi.c b/libavutil/hwcontext_vaapi.c
index 753dcf8905..256363eb97 100644
--- a/libavutil/hwcontext_vaapi.c
+++ b/libavutil/hwcontext_vaapi.c
@@ -222,9 +222,12 @@ static int vaapi_get_image_format(AVHWDeviceContext *hwdev,
     const VAAPIFormatDescriptor *desc;
     int i;
 
+    if (!image_format)
+        return AVERROR(EINVAL);
+
     desc = vaapi_format_from_pix_fmt(pix_fmt);
-    if (!desc || !image_format)
-        goto fail;
+    if (!desc)
+        return AVERROR(ENOSYS);
 
     for (i = 0; i < ctx->nb_formats; i++) {
         if (ctx->formats[i].fourcc == desc->fourcc) {
@@ -233,7 +236,6 @@ static int vaapi_get_image_format(AVHWDeviceContext *hwdev,
         }
     }
 
-fail:
     return AVERROR(ENOSYS);
 }
 
-- 
2.49.1


>From b71c350f9f176dab871c2a89a60a7c58909c81f4 Mon Sep 17 00:00:00 2001
From: Zhao Zhili <zhilizhao@tencent.com>
Date: Wed, 12 Nov 2025 22:35:45 +0800
Subject: [PATCH 2/2] avutil/hwcontext_vaapi: fix use fourcc not supported by
 devices

1. A AVPixelFormat can map to multiple VA_FOURCCs, while
vaapi_format_from_pix_fmt() only returns the first item matched
before this patch.
2. vaapi_frames_init() use vaapi_format_from_pix_fmt() to get the
first item. fourcc in this item may not be supported by the device.

There is a real issue. av_hwdevice_get_hwframe_constraints returned
AV_PIX_FMT_YUV422P is a supported sw_format, while it's only supported
by VA_FOURCC_YV16, but vaapi_frames_init try to use VA_FOURCC_422H
later.

This patch makes vaapi_format_from_pix_fmt return all matched items
iteratively, then use strict check in vaapi_frames_init to get the
right fourcc.
---
 libavutil/hwcontext_vaapi.c | 72 ++++++++++++++++++++++---------------
 1 file changed, 43 insertions(+), 29 deletions(-)

diff --git a/libavutil/hwcontext_vaapi.c b/libavutil/hwcontext_vaapi.c
index 256363eb97..4f3502797b 100644
--- a/libavutil/hwcontext_vaapi.c
+++ b/libavutil/hwcontext_vaapi.c
@@ -195,12 +195,17 @@ static const VAAPIFormatDescriptor *
 }
 
 static const VAAPIFormatDescriptor *
-    vaapi_format_from_pix_fmt(enum AVPixelFormat pix_fmt)
+    vaapi_format_from_pix_fmt(enum AVPixelFormat pix_fmt, const VAAPIFormatDescriptor *prev)
 {
-    int i;
-    for (i = 0; i < FF_ARRAY_ELEMS(vaapi_format_map); i++)
-        if (vaapi_format_map[i].pix_fmt == pix_fmt)
-            return &vaapi_format_map[i];
+    const VAAPIFormatDescriptor *end = &vaapi_format_map[FF_ARRAY_ELEMS(vaapi_format_map)];
+    if (!prev)
+        prev = vaapi_format_map;
+    else
+        prev++;
+
+    for (; prev < end; prev++)
+        if (prev->pix_fmt == pix_fmt)
+            return prev;
     return NULL;
 }
 
@@ -214,31 +219,39 @@ static enum AVPixelFormat vaapi_pix_fmt_from_fourcc(unsigned int fourcc)
         return AV_PIX_FMT_NONE;
 }
 
-static int vaapi_get_image_format(AVHWDeviceContext *hwdev,
+static int vaapi_get_img_desc_and_format(AVHWDeviceContext *hwdev,
                                   enum AVPixelFormat pix_fmt,
+                                  const VAAPIFormatDescriptor **_desc,
                                   VAImageFormat **image_format)
 {
     VAAPIDeviceContext *ctx = hwdev->hwctx;
-    const VAAPIFormatDescriptor *desc;
+    const VAAPIFormatDescriptor *desc = NULL;
     int i;
 
-    if (!image_format)
-        return AVERROR(EINVAL);
-
-    desc = vaapi_format_from_pix_fmt(pix_fmt);
-    if (!desc)
-        return AVERROR(ENOSYS);
-
-    for (i = 0; i < ctx->nb_formats; i++) {
-        if (ctx->formats[i].fourcc == desc->fourcc) {
-            *image_format = &ctx->formats[i].image_format;
-            return 0;
+    while ((desc = vaapi_format_from_pix_fmt(pix_fmt, desc))) {
+        for (i = 0; i < ctx->nb_formats; i++) {
+            if (ctx->formats[i].fourcc == desc->fourcc) {
+                if (_desc)
+                    *_desc = desc;
+                if (image_format)
+                    *image_format = &ctx->formats[i].image_format;
+                return 0;
+            }
         }
     }
 
     return AVERROR(ENOSYS);
 }
 
+static int vaapi_get_image_format(AVHWDeviceContext *hwdev,
+                                  enum AVPixelFormat pix_fmt,
+                                  VAImageFormat **image_format)
+{
+    if (!image_format)
+        return AVERROR(EINVAL);
+    return vaapi_get_img_desc_and_format(hwdev, pix_fmt, NULL, image_format);
+}
+
 static int vaapi_frames_get_constraints(AVHWDeviceContext *hwdev,
                                         const void *hwconfig,
                                         AVHWFramesConstraints *constraints)
@@ -564,19 +577,23 @@ static int vaapi_frames_init(AVHWFramesContext *hwfc)
     VAAPIFramesContext     *ctx = hwfc->hwctx;
     AVVAAPIFramesContext  *avfc = &ctx->p;
     AVVAAPIDeviceContext *hwctx = hwfc->device_ctx->hwctx;
-    const VAAPIFormatDescriptor *desc;
-    VAImageFormat *expected_format;
+    const VAAPIFormatDescriptor *desc = NULL;
+    VAImageFormat *expected_format = NULL;
     AVBufferRef *test_surface = NULL;
     VASurfaceID test_surface_id;
     VAImage test_image;
     VAStatus vas;
     int err, i;
 
-    desc = vaapi_format_from_pix_fmt(hwfc->sw_format);
-    if (!desc) {
-        av_log(hwfc, AV_LOG_ERROR, "Unsupported format: %s.\n",
-               av_get_pix_fmt_name(hwfc->sw_format));
-        return AVERROR(EINVAL);
+    err = vaapi_get_img_desc_and_format(hwfc->device_ctx, hwfc->sw_format,
+                                        &desc, &expected_format);
+    if (err < 0) {
+        // Use a relaxed check when pool exist. It can be an external pool.
+        if (!hwfc->pool || !vaapi_format_from_pix_fmt(hwfc->sw_format, NULL)) {
+            av_log(hwfc, AV_LOG_ERROR, "Unsupported format: %s.\n",
+                   av_get_pix_fmt_name(hwfc->sw_format));
+            return AVERROR(EINVAL);
+        }
     }
 
     if (!hwfc->pool) {
@@ -675,10 +692,7 @@ static int vaapi_frames_init(AVHWFramesContext *hwfc)
     test_surface_id = (VASurfaceID)(uintptr_t)test_surface->data;
 
     ctx->derive_works = 0;
-
-    err = vaapi_get_image_format(hwfc->device_ctx,
-                                 hwfc->sw_format, &expected_format);
-    if (err == 0) {
+    if (expected_format) {
         vas = vaDeriveImage(hwctx->display, test_surface_id, &test_image);
         if (vas == VA_STATUS_SUCCESS) {
             if (expected_format->fourcc == test_image.format.fourcc) {
-- 
2.49.1

_______________________________________________
ffmpeg-devel mailing list -- ffmpeg-devel@ffmpeg.org
To unsubscribe send an email to ffmpeg-devel-leave@ffmpeg.org