[FFmpeg-devel] [PATCH] avcodec/amfenc: fix async_depth deadlock with lookahead (PR #21266)

[ArazIusubov via ffmpeg-devel <ffmpeg-devel@ffmpeg.org>]

PR #21266 opened by ArazIusubov
URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/21266
Patch URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/21266.patch

AMF encoders may deadlock when lookahead > async_depth.
Automatically adjust async_depth to lookahead + 1 to prevent hangs.


>From 699104934bf3da0e98011915374d9e5717acca60 Mon Sep 17 00:00:00 2001
From: Araz Iusubov <Primeadvice@gmail.com>
Date: Mon, 22 Dec 2025 17:09:16 +0100
Subject: [PATCH] avcodec/amfenc: fix async_depth deadlock with lookahead

AMF encoders may deadlock when lookahead > async_depth.
Automatically adjust async_depth to lookahead + 1 to prevent hangs.
---
 libavcodec/amfenc.c      | 9 +++++++++
 libavcodec/amfenc_av1.c  | 2 +-
 libavcodec/amfenc_h264.c | 2 +-
 libavcodec/amfenc_hevc.c | 2 +-
 4 files changed, 12 insertions(+), 3 deletions(-)

diff --git a/libavcodec/amfenc.c b/libavcodec/amfenc.c
index b07da236c7..329ce29005 100644
--- a/libavcodec/amfenc.c
+++ b/libavcodec/amfenc.c
@@ -351,6 +351,15 @@ int ff_amf_encode_init(AVCodecContext *avctx)
         AMF_RETURN_IF_FALSE(ctx, ret == 0, ret, "Failed to create  hardware device context (AMF) : %s\n", av_err2str(ret));
     }
 
+    if (ctx->pa_lookahead_buffer_depth >= ctx->hwsurfaces_in_queue_max) {
+        av_log(avctx, AV_LOG_WARNING,
+               "async_depth (%d) too small for lookahead (%d), increasing to (%d)\n",
+                ctx->hwsurfaces_in_queue_max,
+                ctx->pa_lookahead_buffer_depth,
+                ctx->pa_lookahead_buffer_depth + 1);
+        ctx->hwsurfaces_in_queue_max = ctx->pa_lookahead_buffer_depth + 1;
+    }
+
     if ((ret = amf_init_encoder(avctx)) == 0) {
         return 0;
     }
diff --git a/libavcodec/amfenc_av1.c b/libavcodec/amfenc_av1.c
index 7202e0fcd5..70642a6b54 100644
--- a/libavcodec/amfenc_av1.c
+++ b/libavcodec/amfenc_av1.c
@@ -101,7 +101,7 @@ static const AVOption options[] = {
     { "gop",                    "", 0, AV_OPT_TYPE_CONST, {.i64 = AMF_VIDEO_ENCODER_AV1_HEADER_INSERTION_MODE_GOP_ALIGNED       }, 0, 0, VE, .unit = "hdrmode" },
     { "frame",                  "", 0, AV_OPT_TYPE_CONST, {.i64 = AMF_VIDEO_ENCODER_AV1_HEADER_INSERTION_MODE_KEY_FRAME_ALIGNED }, 0, 0, VE, .unit = "hdrmode" },
 
-    { "async_depth",            "Set maximum encoding parallelism. Higher values increase output latency.", OFFSET(hwsurfaces_in_queue_max), AV_OPT_TYPE_INT, {.i64 = 16 }, 1, 16, VE },
+    { "async_depth",            "Set maximum encoding parallelism. Higher values increase output latency.", OFFSET(hwsurfaces_in_queue_max), AV_OPT_TYPE_INT, {.i64 = 16 }, 1, MAX_LOOKAHEAD_DEPTH + 1, VE },
 
     { "preencode",              "Enable preencode",     OFFSET(preencode),      AV_OPT_TYPE_BOOL, {.i64 = -1  }, -1, 1, VE},
     { "enforce_hrd",            "Enforce HRD",          OFFSET(enforce_hrd),    AV_OPT_TYPE_BOOL, {.i64 = -1  }, -1, 1, VE},
diff --git a/libavcodec/amfenc_h264.c b/libavcodec/amfenc_h264.c
index 131ba871f5..c39a65aead 100644
--- a/libavcodec/amfenc_h264.c
+++ b/libavcodec/amfenc_h264.c
@@ -111,7 +111,7 @@ static const AVOption options[] = {
     { "header_spacing", "Header Insertion Spacing",             OFFSET(header_spacing),     AV_OPT_TYPE_INT, { .i64 = -1 }, -1, 1000, VE },
 
     /// Maximum queued frames
-    { "async_depth",    "Set maximum encoding parallelism. Higher values increase output latency.", OFFSET(hwsurfaces_in_queue_max), AV_OPT_TYPE_INT, {.i64 = 16 }, 1, 16, VE },
+    { "async_depth",    "Set maximum encoding parallelism. Higher values increase output latency.", OFFSET(hwsurfaces_in_queue_max), AV_OPT_TYPE_INT, {.i64 = 16 }, 1, MAX_LOOKAHEAD_DEPTH + 1, VE },
 
     /// B-Frames
     // BPicturesPattern=bf
diff --git a/libavcodec/amfenc_hevc.c b/libavcodec/amfenc_hevc.c
index 73c555db38..fe1d948c92 100644
--- a/libavcodec/amfenc_hevc.c
+++ b/libavcodec/amfenc_hevc.c
@@ -87,7 +87,7 @@ static const AVOption options[] = {
     { "gop",            "", 0, AV_OPT_TYPE_CONST, { .i64 = AMF_VIDEO_ENCODER_HEVC_HEADER_INSERTION_MODE_GOP_ALIGNED }, 0, 0, VE, .unit = "hdrmode" },
     { "idr",            "", 0, AV_OPT_TYPE_CONST, { .i64 = AMF_VIDEO_ENCODER_HEVC_HEADER_INSERTION_MODE_IDR_ALIGNED }, 0, 0, VE, .unit = "hdrmode" },
 
-    { "async_depth",    "Set maximum encoding parallelism. Higher values increase output latency.",             OFFSET(hwsurfaces_in_queue_max), AV_OPT_TYPE_INT, {.i64 = 16 }, 1, 16, VE },
+    { "async_depth",    "Set maximum encoding parallelism. Higher values increase output latency.",             OFFSET(hwsurfaces_in_queue_max), AV_OPT_TYPE_INT, {.i64 = 16 }, 1, MAX_LOOKAHEAD_DEPTH + 1, VE },
 
     { "high_motion_quality_boost_enable",   "Enable High motion quality boost mode",  OFFSET(hw_high_motion_quality_boost), AV_OPT_TYPE_BOOL,   {.i64 = -1 }, -1, 1, VE },
     { "gops_per_idr",   "GOPs per IDR 0-no IDR will be inserted",   OFFSET(gops_per_idr),  AV_OPT_TYPE_INT,  { .i64 = 1  },  0, INT_MAX, VE },
-- 
2.49.1

_______________________________________________
ffmpeg-devel mailing list -- ffmpeg-devel@ffmpeg.org
To unsubscribe send an email to ffmpeg-devel-leave@ffmpeg.org