From 09c11ebba68076c5c17254ea42183e6031a9a44f Mon Sep 17 00:00:00 2001
From: Andik <andik@localhost>
Date: Sat, 10 Sep 2022 18:22:03 +0200
Subject: [PATCH] improve VAAPI error handling

---
 libavutil/hwcontext_vaapi.c | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/libavutil/hwcontext_vaapi.c b/libavutil/hwcontext_vaapi.c
index 9ba5225ad2..82218c4c9c 100644
--- a/libavutil/hwcontext_vaapi.c
+++ b/libavutil/hwcontext_vaapi.c
@@ -1655,6 +1655,7 @@ static int vaapi_device_create(AVHWDeviceContext *ctx, const char *device,
     VADisplay display = NULL;
     const AVDictionaryEntry *ent;
     int try_drm, try_x11, try_all;
+    int dev_errno = ENOENT;
 
     priv = av_mallocz(sizeof(*priv));
     if (!priv)
@@ -1692,6 +1693,7 @@ static int vaapi_device_create(AVHWDeviceContext *ctx, const char *device,
         if (device) {
             priv->drm_fd = open(device, O_RDWR);
             if (priv->drm_fd < 0) {
+                dev_errno = errno;
                 av_log(ctx, loglevel, "Failed to open %s as "
                        "DRM device node.\n", device);
                 break;
@@ -1708,15 +1710,18 @@ static int vaapi_device_create(AVHWDeviceContext *ctx, const char *device,
                          "/dev/dri/renderD%d", 128 + n);
                 priv->drm_fd = open(path, O_RDWR);
                 if (priv->drm_fd < 0) {
+                    if (errno != ENOENT)
+                        dev_errno = errno;
                     av_log(ctx, AV_LOG_VERBOSE, "Cannot open "
                            "DRM render node for device %d.\n", n);
-                    break;
+                    continue;
                 }
 #if CONFIG_LIBDRM
                 if (kernel_driver) {
                     drmVersion *info;
                     info = drmGetVersion(priv->drm_fd);
                     if (strcmp(kernel_driver->value, info->name)) {
+                        dev_errno = EINVAL;
                         av_log(ctx, AV_LOG_VERBOSE, "Ignoring device %d "
                                "with non-matching kernel driver (%s).\n",
                                n, info->name);
@@ -1744,9 +1749,9 @@ static int vaapi_device_create(AVHWDeviceContext *ctx, const char *device,
 
         display = vaGetDisplayDRM(priv->drm_fd);
         if (!display) {
+            dev_errno = ENODEV;
             av_log(ctx, AV_LOG_VERBOSE, "Cannot open a VA display "
                    "from DRM device %s.\n", device);
-            return AVERROR_EXTERNAL;
         }
         break;
     }
@@ -1757,6 +1762,8 @@ static int vaapi_device_create(AVHWDeviceContext *ctx, const char *device,
         // Try to open the device as an X11 display.
         priv->x11_display = XOpenDisplay(device);
         if (!priv->x11_display) {
+            if (errno != ENOENT)
+                dev_errno = errno;
             av_log(ctx, AV_LOG_VERBOSE, "Cannot open X11 display "
                    "%s.\n", XDisplayName(device));
         } else {
@@ -1780,7 +1787,7 @@ static int vaapi_device_create(AVHWDeviceContext *ctx, const char *device,
         else
             av_log(ctx, AV_LOG_ERROR, "No VA display found for "
                    "any default device.\n");
-        return AVERROR(EINVAL);
+        return AVERROR(dev_errno);
     }
 
     ent = av_dict_get(opts, "driver", NULL, 0);
-- 
2.34.1