[FFmpeg-devel] [PATCH v4] checkasm/h264dsp: Fix stack overflow in check_idct_dequant

[FFmpeg-devel] [PATCH v4] checkasm/h264dsp: Fix stack overflow in check_idct_dequant

[Zhao Zhili]

From: Zhao Zhili <zhilizhao@tencent.com>

---
 tests/checkasm/h264dsp.c | 19 ++++++++++++++-----
 1 file changed, 14 insertions(+), 5 deletions(-)

diff --git a/tests/checkasm/h264dsp.c b/tests/checkasm/h264dsp.c
index f5f9650224..cdcb1f3aad 100644
--- a/tests/checkasm/h264dsp.c
+++ b/tests/checkasm/h264dsp.c
@@ -328,8 +328,9 @@ static void check_idct_multiple(void)
 static void check_idct_dequant(void)
 {
     static const int depths[5] = { 8, 9, 10, 12, 14 };
-    LOCAL_ALIGNED_16(int16_t, src, [16]);
-    /* Ensure dst buffers are large enough to hold dctcoefs of all bit-depths. */
+    /* Ensure buffers are large enough to hold dctcoefs of all bit-depths. */
+    LOCAL_ALIGNED_16(int16_t, src_buf0, [16]);
+    LOCAL_ALIGNED_16(int32_t, src_buf1, [16]);
     LOCAL_ALIGNED_16(uint8_t, dst0, [16 * 16 * sizeof(int32_t)]);
     LOCAL_ALIGNED_16(uint8_t, dst1, [16 * 16 * sizeof(int32_t)]);
     int16_t *dst_ref = (int16_t *)dst0;
@@ -338,15 +339,23 @@ static void check_idct_dequant(void)
     int bit_depth, i, qmul;
     declare_func_emms(AV_CPU_FLAG_MMX | AV_CPU_FLAG_SSE2, void, int16_t *output, int16_t *input, int qmul);
 
-    for (int j = 0; j < 16; j++)
-        src[j] = (rnd() % 512) - 256;
-
     qmul = rnd() % 4096;
 
     for (i = 0; i < FF_ARRAY_ELEMS(depths); i++) {
+        int16_t *src;
         bit_depth = depths[i];
         ff_h264dsp_init(&h, bit_depth, 1);
 
+        if (bit_depth == 8) {
+            src = src_buf0;
+            for (size_t j = 0; j < 16; j++)
+                src_buf0[0] = (rnd() % 512) - 256;
+        } else {
+            src = (int16_t *)src_buf1;
+            for (size_t j = 0; j < 16; j++)
+                src_buf1[1] = (rnd() % (1 << (bit_depth + 1))) - (1 << bit_depth);
+        }
+
         memset(dst0, 0, 16 * 16 * SIZEOF_COEF);
         memset(dst1, 0, 16 * 16 * SIZEOF_COEF);
 
-- 
2.25.1

_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".

Re: [FFmpeg-devel] [PATCH v4] checkasm/h264dsp: Fix stack overflow in check_idct_dequant

[Zhao Zhili]

Please ignore this version.

> On Jun 16, 2025, at 18:43, Zhao Zhili <quinkblack-at-foxmail.com@ffmpeg.org> wrote:
> 
> From: Zhao Zhili <zhilizhao@tencent.com>
> 
> ---
> tests/checkasm/h264dsp.c | 19 ++++++++++++++-----
> 1 file changed, 14 insertions(+), 5 deletions(-)
> 
> diff --git a/tests/checkasm/h264dsp.c b/tests/checkasm/h264dsp.c
> index f5f9650224..cdcb1f3aad 100644
> --- a/tests/checkasm/h264dsp.c
> +++ b/tests/checkasm/h264dsp.c
> @@ -328,8 +328,9 @@ static void check_idct_multiple(void)
> static void check_idct_dequant(void)
> {
>     static const int depths[5] = { 8, 9, 10, 12, 14 };
> -    LOCAL_ALIGNED_16(int16_t, src, [16]);
> -    /* Ensure dst buffers are large enough to hold dctcoefs of all bit-depths. */
> +    /* Ensure buffers are large enough to hold dctcoefs of all bit-depths. */
> +    LOCAL_ALIGNED_16(int16_t, src_buf0, [16]);
> +    LOCAL_ALIGNED_16(int32_t, src_buf1, [16]);
>     LOCAL_ALIGNED_16(uint8_t, dst0, [16 * 16 * sizeof(int32_t)]);
>     LOCAL_ALIGNED_16(uint8_t, dst1, [16 * 16 * sizeof(int32_t)]);
>     int16_t *dst_ref = (int16_t *)dst0;
> @@ -338,15 +339,23 @@ static void check_idct_dequant(void)
>     int bit_depth, i, qmul;
>     declare_func_emms(AV_CPU_FLAG_MMX | AV_CPU_FLAG_SSE2, void, int16_t *output, int16_t *input, int qmul);
> 
> -    for (int j = 0; j < 16; j++)
> -        src[j] = (rnd() % 512) - 256;
> -
>     qmul = rnd() % 4096;
> 
>     for (i = 0; i < FF_ARRAY_ELEMS(depths); i++) {
> +        int16_t *src;
>         bit_depth = depths[i];
>         ff_h264dsp_init(&h, bit_depth, 1);
> 
> +        if (bit_depth == 8) {
> +            src = src_buf0;
> +            for (size_t j = 0; j < 16; j++)
> +                src_buf0[0] = (rnd() % 512) - 256;
> +        } else {
> +            src = (int16_t *)src_buf1;
> +            for (size_t j = 0; j < 16; j++)
> +                src_buf1[1] = (rnd() % (1 << (bit_depth + 1))) - (1 << bit_depth);
> +        }

> +
>         memset(dst0, 0, 16 * 16 * SIZEOF_COEF);
>         memset(dst1, 0, 16 * 16 * SIZEOF_COEF);
> 
> -- 
> 2.25.1
> 
> _______________________________________________
> ffmpeg-devel mailing list
> ffmpeg-devel@ffmpeg.org
> https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
> 
> To unsubscribe, visit link above, or email
> ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".

_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".