[FFmpeg-devel] [PATCH] avdevice/gdigrab: fix -show_region 1 overlay window crashing FFmpeg when clicked (PR #20148)

[Hunter Kvalevog <code@ffmpeg.org>]

PR #20148 opened by Hunter Kvalevog (k-vog)
URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/20148
Patch URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/20148.patch

Tested on Windows 7, 10, 11, and Debian (Xorg + Wine)

Fixes: #11539


From b1012d3796f98a8f1eb6543fcd62be80ca963fa1 Mon Sep 17 00:00:00 2001
From: Hunter Kvalevog <hunter@kvog.sh>
Date: Mon, 4 Aug 2025 22:09:47 -0500
Subject: [PATCH 1/2] avdevice/gdigrab: process window in a separate thread

Move window creation and event processing to a dedicated thread.

GetMessage only processes events from the calling thread's message
queue. Because gdigrab_read_header and gdigrab_read_packet don't run on
the same thread, the message queue was not being drained.

Fixes: #11539
---
 libavdevice/gdigrab.c | 105 ++++++++++++++++++++++++++----------------
 1 file changed, 65 insertions(+), 40 deletions(-)

diff --git a/libavdevice/gdigrab.c b/libavdevice/gdigrab.c
index 08a41c304b..66d849035f 100644
--- a/libavdevice/gdigrab.c
+++ b/libavdevice/gdigrab.c
@@ -64,7 +64,10 @@ struct gdigrab {
     void      *buffer;      /**< The buffer containing the bitmap image data */
     RECT       clip_rect;   /**< The subarea of the screen or window to clip */
 
-    HWND       region_hwnd; /**< Handle of the region border window */
+    HANDLE        region_thread;    /**< Region window thread */
+    DWORD         region_thread_id; /**< Region window thread ID */
+    HANDLE        region_init;      /**< Region window ready signal */
+    volatile LONG region_result;    /**< Region window status, 0 success */
 
     int cursor_error_printed;
 };
@@ -114,19 +117,21 @@ gdigrab_region_wnd_proc(HWND hwnd, UINT msg, WPARAM wparam, LPARAM lparam)
 }
 
 /**
- * Initialize the region outline window.
+ * Run the region outline window loop.
  *
- * @param s1 The format context.
- * @param gdigrab gdigrab context.
- * @return 0 success, !0 failure
+ * @param opaque gdigrab context.
+ * @return 0
  */
-static int
-gdigrab_region_wnd_init(AVFormatContext *s1, struct gdigrab *gdigrab)
+static DWORD WINAPI
+gdigrab_region_wnd_task(LPVOID opaque)
 {
+    struct gdigrab* gdigrab = (struct gdigrab*)opaque;
+
     HWND hwnd;
     RECT rect = gdigrab->clip_rect;
     HRGN region = NULL;
     HRGN region_interior = NULL;
+    MSG msg;
 
     DWORD style = WS_POPUP | WS_VISIBLE;
     DWORD ex = WS_EX_TOOLWINDOW | WS_EX_TOPMOST | WS_EX_TRANSPARENT;
@@ -142,7 +147,8 @@ gdigrab_region_wnd_init(AVFormatContext *s1, struct gdigrab *gdigrab)
             rect.right - rect.left, rect.bottom - rect.top,
             NULL, NULL, NULL, NULL);
     if (!hwnd) {
-        WIN32_API_ERROR("Could not create region display window");
+        av_log(NULL, AV_LOG_ERROR, "Could not create region display window "
+            "(error %li)\n", GetLastError());
         goto error;
     }
 
@@ -155,7 +161,8 @@ gdigrab_region_wnd_init(AVFormatContext *s1, struct gdigrab *gdigrab)
             rect.bottom - rect.top - REGION_WND_BORDER);
     CombineRgn(region, region, region_interior, RGN_DIFF);
     if (!SetWindowRgn(hwnd, region, FALSE)) {
-        WIN32_API_ERROR("Could not set window region");
+        av_log(NULL, AV_LOG_ERROR, "Could not set window region "
+            "(error %li)\n", GetLastError());
         goto error;
     }
     // The "region" memory is now owned by the window
@@ -166,8 +173,14 @@ gdigrab_region_wnd_init(AVFormatContext *s1, struct gdigrab *gdigrab)
 
     ShowWindow(hwnd, SW_SHOW);
 
-    gdigrab->region_hwnd = hwnd;
+    InterlockedExchange(&gdigrab->region_result, 0);
+    SetEvent(gdigrab->region_init);
 
+    while (GetMessage(&msg, NULL, 0, 0)) {
+        DispatchMessage(&msg);
+    }
+
+    DestroyWindow(hwnd);
     return 0;
 
 error:
@@ -177,7 +190,44 @@ error:
         DeleteObject(region_interior);
     if (hwnd)
         DestroyWindow(hwnd);
-    return 1;
+    InterlockedExchange(&gdigrab->region_result, AVERROR(EIO));
+    SetEvent(gdigrab->region_init);
+    return 0;
+}
+
+/**
+ * Start the region outline window thread.
+ *
+ * @param gdigrab gdigrab context.
+ * @return 0 success, !0 failure
+ */
+static int
+gdigrab_region_wnd_init(struct gdigrab *gdigrab)
+{
+    int result = AVERROR(EIO);
+
+    /* Start the thread and wait for it to try creating a window */
+    gdigrab->region_result = 0;
+    gdigrab->region_init = CreateEvent(NULL, TRUE, FALSE, NULL);
+    if (!gdigrab->region_init)
+        goto end;
+    gdigrab->region_thread = CreateThread(NULL, 0, gdigrab_region_wnd_task,
+        gdigrab, 0, &gdigrab->region_thread_id);
+    if (!gdigrab->region_thread)
+        goto end;
+
+    WaitForSingleObject(gdigrab->region_init, INFINITE);
+    result = gdigrab->region_result;
+
+ end:
+    if (gdigrab->region_thread && result < 0) {
+        WaitForSingleObject(gdigrab->region_thread, INFINITE);
+        CloseHandle(gdigrab->region_thread);
+    }
+    if (gdigrab->region_init) {
+        CloseHandle(gdigrab->region_init);
+    }
+    return result;
 }
 
 /**
@@ -189,30 +239,9 @@ error:
 static void
 gdigrab_region_wnd_destroy(AVFormatContext *s1, struct gdigrab *gdigrab)
 {
-    if (gdigrab->region_hwnd)
-        DestroyWindow(gdigrab->region_hwnd);
-    gdigrab->region_hwnd = NULL;
-}
-
-/**
- * Process the Windows message queue.
- *
- * This is important to prevent Windows from thinking the window has become
- * unresponsive. As well, things like WM_PAINT (to actually draw the window
- * contents) are handled from the message queue context.
- *
- * @param s1 The format context.
- * @param gdigrab gdigrab context.
- */
-static void
-gdigrab_region_wnd_update(AVFormatContext *s1, struct gdigrab *gdigrab)
-{
-    HWND hwnd = gdigrab->region_hwnd;
-    MSG msg;
-
-    while (PeekMessage(&msg, hwnd, 0, 0, PM_REMOVE)) {
-        DispatchMessage(&msg);
-    }
+    PostThreadMessage(gdigrab->region_thread_id, WM_QUIT, 0, 0);
+    WaitForSingleObject(gdigrab->region_thread, INFINITE);
+    CloseHandle(gdigrab->region_thread);
 }
 
 /**
@@ -435,7 +464,7 @@ gdigrab_read_header(AVFormatContext *s1)
     gdigrab->cursor_error_printed = 0;
 
     if (gdigrab->show_region) {
-        if (gdigrab_region_wnd_init(s1, gdigrab)) {
+        if (gdigrab_region_wnd_init(gdigrab)) {
             ret = AVERROR(EIO);
             goto error;
         }
@@ -573,10 +602,6 @@ static int gdigrab_read_packet(AVFormatContext *s1, AVPacket *pkt)
     /* Calculate the time of the next frame */
     time_frame += INT64_C(1000000);
 
-    /* Run Window message processing queue */
-    if (gdigrab->show_region)
-        gdigrab_region_wnd_update(s1, gdigrab);
-
     /* wait based on the frame rate */
     for (;;) {
         curtime = av_gettime_relative();
-- 
2.49.1


From 14a78b0686b7060d1654db9dd2574c354a21caeb Mon Sep 17 00:00:00 2001
From: Hunter Kvalevog <hunter@kvog.sh>
Date: Tue, 5 Aug 2025 20:57:30 -0500
Subject: [PATCH 2/2] avdevice/gdigrab: make overlay window layered

WS_EX_LAYERED allows input events to pass through to windows beneath.

WS_EX_NOACTIVATE prevents the window from stealing focus when created.
---
 libavdevice/gdigrab.c | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/libavdevice/gdigrab.c b/libavdevice/gdigrab.c
index 66d849035f..eb9021d74d 100644
--- a/libavdevice/gdigrab.c
+++ b/libavdevice/gdigrab.c
@@ -134,7 +134,8 @@ gdigrab_region_wnd_task(LPVOID opaque)
     MSG msg;
 
     DWORD style = WS_POPUP | WS_VISIBLE;
-    DWORD ex = WS_EX_TOOLWINDOW | WS_EX_TOPMOST | WS_EX_TRANSPARENT;
+    DWORD ex = WS_EX_LAYERED | WS_EX_NOACTIVATE | WS_EX_TOOLWINDOW |
+        WS_EX_TOPMOST | WS_EX_TRANSPARENT;
 
     rect.left -= REGION_WND_BORDER; rect.top -= REGION_WND_BORDER;
     rect.right += REGION_WND_BORDER; rect.bottom += REGION_WND_BORDER;
@@ -170,6 +171,7 @@ gdigrab_region_wnd_task(LPVOID opaque)
     DeleteObject(region_interior);
 
     SetWindowLongPtr(hwnd, GWLP_WNDPROC, (LONG_PTR) gdigrab_region_wnd_proc);
+    SetLayeredWindowAttributes(hwnd, 0, 0xFF, LWA_ALPHA);
 
     ShowWindow(hwnd, SW_SHOW);
 
-- 
2.49.1

_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".