[FFmpeg-devel] [PR] libswscale/utils: avoid sizeFactor related overflows (PR #21536)

[FFmpeg-devel] [PR] libswscale/utils: avoid sizeFactor related overflows (PR #21536)

[michaelni via ffmpeg-devel]

PR #21536 opened by michaelni
URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/21536
Patch URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/21536.patch

Fixes: multiple integer overflows
Fixes: out of array access

Found-by: Zhenpeng (Leo) Lin from depthfirst


>From fe2c61e3046a485a898741a152518d6018b7d0e1 Mon Sep 17 00:00:00 2001
From: Michael Niedermayer <michael@niedermayer.cc>
Date: Wed, 21 Jan 2026 01:38:42 +0100
Subject: [PATCH 1/2] swscale/utils: Avoid FF_ALLOC_TYPED_ARRAY() and use
 av_malloc_array() directly

Fixes: multiple integer overflows
Fixes: out of array access

Regression since: a408d03ee6eeda98e77301dcdea3bdf40c0d4afc

The PoC modifies filter parameters generally inaccessable to an attacker

Found-by: Zhenpeng (Leo) Lin from depthfirst
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
---
 libswscale/utils.c | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

diff --git a/libswscale/utils.c b/libswscale/utils.c
index 8f8789b24d..846ee21207 100644
--- a/libswscale/utils.c
+++ b/libswscale/utils.c
@@ -103,7 +103,8 @@ int ff_shuffle_filter_coefficients(SwsInternal *c, int *filterPos,
         if ((c->srcBpc == 8) && (c->dstBpc <= 14)) {
            int16_t *filterCopy = NULL;
            if (filterSize > 4) {
-               if (!FF_ALLOC_TYPED_ARRAY(filterCopy, dstW * filterSize))
+               filterCopy = av_malloc_array(dstW, filterSize * sizeof(*filterCopy));
+               if (!filterCopy)
                    return AVERROR(ENOMEM);
                memcpy(filterCopy, filter, dstW * filterSize * sizeof(int16_t));
            }
@@ -282,7 +283,8 @@ static av_cold int initFilter(int16_t **outFilter, int32_t **filterPos,
         filterSize = FFMIN(filterSize, srcW - 2);
         filterSize = FFMAX(filterSize, 1);
 
-        if (!FF_ALLOC_TYPED_ARRAY(filter, dstW * filterSize))
+        filter = av_malloc_array(dstW, filterSize * sizeof(*filter));
+        if (!filter)
             goto nomem;
         xDstInSrc = ((dstPos*(int64_t)xInc)>>7) - ((srcPos*0x10000LL)>>7);
         for (i = 0; i < dstW; i++) {
@@ -381,7 +383,8 @@ static av_cold int initFilter(int16_t **outFilter, int32_t **filterPos,
     if (dstFilter)
         filter2Size += dstFilter->length - 1;
     av_assert0(filter2Size > 0);
-    if (!FF_ALLOCZ_TYPED_ARRAY(filter2, dstW * filter2Size))
+    filter2 = av_malloc_array(dstW, filter2Size * sizeof(*filter2));
+    if (!filter2)
         goto nomem;
     for (i = 0; i < dstW; i++) {
         int j, k;
@@ -549,7 +552,8 @@ static av_cold int initFilter(int16_t **outFilter, int32_t **filterPos,
 
     // Note the +1 is for the MMX scaler which reads over the end
     /* align at 16 for AltiVec (needed by hScale_altivec_real) */
-    if (!FF_ALLOCZ_TYPED_ARRAY(*outFilter, *outFilterSize * (dstW + 3)))
+    *outFilter = av_malloc_array(dstW + 3, *outFilterSize * sizeof(**outFilter));
+    if (!outFilter)
         goto nomem;
 
     /* normalize & store in outFilter */
@@ -1716,8 +1720,9 @@ av_cold int ff_sws_init_single_context(SwsContext *sws, SwsFilter *srcFilter,
             goto fail;
 
 #if HAVE_ALTIVEC
-        if (!FF_ALLOC_TYPED_ARRAY(c->vYCoeffsBank, c->vLumFilterSize * sws->dst_h) ||
-            !FF_ALLOC_TYPED_ARRAY(c->vCCoeffsBank, c->vChrFilterSize * c->chrDstH))
+        c->vYCoeffsBank = av_malloc_array(sws->dst_h, c->vLumFilterSize * sizeof(*c->vYCoeffsBank));
+        c->vCCoeffsBank = av_malloc_array(c->chrDstH, c->vChrFilterSize * sizeof(*c->vCCoeffsBank));
+        if (c->vYCoeffsBank == NULL || c->vCCoeffsBank == NULL)
             goto nomem;
 
         for (i = 0; i < c->vLumFilterSize * sws->dst_h; i++) {
-- 
2.52.0


>From 60bcef52ca42c11b5a51ba1c20f9db26fd0cf910 Mon Sep 17 00:00:00 2001
From: Michael Niedermayer <michael@niedermayer.cc>
Date: Wed, 21 Jan 2026 01:38:42 +0100
Subject: [PATCH 2/2] swscale/utils: Sanity check sizeFactor

Fixes: multiple integer overflows
Fixes: out of array access

The PoC modifies filter parameters generally inaccessable to an attacker

Found-by: Zhenpeng (Leo) Lin from depthfirst
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
---
 libswscale/utils.c | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/libswscale/utils.c b/libswscale/utils.c
index 846ee21207..ee4342884a 100644
--- a/libswscale/utils.c
+++ b/libswscale/utils.c
@@ -275,6 +275,11 @@ static av_cold int initFilter(int16_t **outFilter, int32_t **filterPos,
             sizeFactor = param[0] != SWS_PARAM_DEFAULT ? ceil(2 * param[0]) : 6;
         av_assert0(sizeFactor > 0);
 
+        if (sizeFactor > 50) {
+            ret = AVERROR(EINVAL);
+            goto fail;
+        }
+
         if (xInc <= 1 << 16)
             filterSize = 1 + sizeFactor;    // upscale
         else
-- 
2.52.0

_______________________________________________
ffmpeg-devel mailing list -- ffmpeg-devel@ffmpeg.org
To unsubscribe send an email to ffmpeg-devel-leave@ffmpeg.org