[FFmpeg-devel] [PR] avfilter/vf_convolution: Handle corner cases with small frames (PR #22281)

[FFmpeg-devel] [PR] avfilter/vf_convolution: Handle corner cases with small frames (PR #22281)

[michaelni via ffmpeg-devel]

PR #22281 opened by michaelni
URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/22281
Patch URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/22281.patch

Fixes: out of array read
Fixes: #YWH-PGM40646-35

Found-by: jpraveenrao
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>


>From 774e03931cda201589b0280348ec16123297315e Mon Sep 17 00:00:00 2001
From: Michael Niedermayer <michael@niedermayer.cc>
Date: Tue, 24 Feb 2026 23:37:21 +0100
Subject: [PATCH 1/2] avfilter/vf_convolution: Use avpriv_mirror

Fixes: out of array read
Fixes: #YWH-PGM40646-35

Found-by: jpraveenrao
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
---
 libavfilter/convolution.h    |  8 +++-----
 libavfilter/vf_convolution.c | 22 ++++++----------------
 2 files changed, 9 insertions(+), 21 deletions(-)

diff --git a/libavfilter/convolution.h b/libavfilter/convolution.h
index f88b708fab..71ec3b6804 100644
--- a/libavfilter/convolution.h
+++ b/libavfilter/convolution.h
@@ -21,6 +21,7 @@
 #ifndef AVFILTER_CONVOLUTION_H
 #define AVFILTER_CONVOLUTION_H
 #include "avfilter.h"
+#include "libavutil/internal.h"
 #include "libavutil/intreadwrite.h"
 
 enum MatrixMode {
@@ -71,11 +72,8 @@ static void setup_3x3(int radius, const uint8_t *c[], const uint8_t *src, int st
     int i;
 
     for (i = 0; i < 9; i++) {
-        int xoff = FFABS(x + ((i % 3) - 1));
-        int yoff = FFABS(y + (i / 3) - 1);
-
-        xoff = xoff >= w ? 2 * w - 1 - xoff : xoff;
-        yoff = yoff >= h ? 2 * h - 1 - yoff : yoff;
+        int xoff = avpriv_mirror(x + (i % 3) - 1, w - 1);
+        int yoff = avpriv_mirror(y + (i / 3) - 1, h - 1);
 
         c[i] = src + xoff * bpc + yoff * stride;
     }
diff --git a/libavfilter/vf_convolution.c b/libavfilter/vf_convolution.c
index ce42df2cde..f0d2cec78e 100644
--- a/libavfilter/vf_convolution.c
+++ b/libavfilter/vf_convolution.c
@@ -520,11 +520,8 @@ static void setup_5x5(int radius, const uint8_t *c[], const uint8_t *src, int st
     int i;
 
     for (i = 0; i < 25; i++) {
-        int xoff = FFABS(x + ((i % 5) - 2));
-        int yoff = FFABS(y + (i / 5) - 2);
-
-        xoff = xoff >= w ? 2 * w - 1 - xoff : xoff;
-        yoff = yoff >= h ? 2 * h - 1 - yoff : yoff;
+        int xoff = avpriv_mirror(x + (i % 5) - 2, w - 1);
+        int yoff = avpriv_mirror(y + (i / 5) - 2, h - 1);
 
         c[i] = src + xoff * bpc + yoff * stride;
     }
@@ -536,11 +533,8 @@ static void setup_7x7(int radius, const uint8_t *c[], const uint8_t *src, int st
     int i;
 
     for (i = 0; i < 49; i++) {
-        int xoff = FFABS(x + ((i % 7) - 3));
-        int yoff = FFABS(y + (i / 7) - 3);
-
-        xoff = xoff >= w ? 2 * w - 1 - xoff : xoff;
-        yoff = yoff >= h ? 2 * h - 1 - yoff : yoff;
+        int xoff = avpriv_mirror(x + (i % 7) - 3, w - 1);
+        int yoff = avpriv_mirror(y + (i / 7) - 3, h - 1);
 
         c[i] = src + xoff * bpc + yoff * stride;
     }
@@ -552,9 +546,7 @@ static void setup_row(int radius, const uint8_t *c[], const uint8_t *src, int st
     int i;
 
     for (i = 0; i < radius * 2 + 1; i++) {
-        int xoff = FFABS(x + i - radius);
-
-        xoff = xoff >= w ? 2 * w - 1 - xoff : xoff;
+        int xoff = avpriv_mirror(x + i - radius, w - 1);
 
         c[i] = src + xoff * bpc + y * stride;
     }
@@ -566,9 +558,7 @@ static void setup_column(int radius, const uint8_t *c[], const uint8_t *src, int
     int i;
 
     for (i = 0; i < radius * 2 + 1; i++) {
-        int xoff = FFABS(x + i - radius);
-
-        xoff = xoff >= h ? 2 * h - 1 - xoff : xoff;
+        int xoff = avpriv_mirror(x + i - radius, h - 1);
 
         c[i] = src + y * bpc + xoff * stride;
     }
-- 
2.52.0


>From 1a7468b9a240da0c7d5ec0adcf0f637a247f91f4 Mon Sep 17 00:00:00 2001
From: Michael Niedermayer <michael@niedermayer.cc>
Date: Wed, 25 Feb 2026 00:44:41 +0100
Subject: [PATCH 2/2] avfilter/vf_convolution: Handle corner cases with small
 frames

Fixes: out of array read
Fixes: #YWH-PGM40646-35

Found-by: jpraveenrao
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
---
 libavfilter/vf_convolution.c | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/libavfilter/vf_convolution.c b/libavfilter/vf_convolution.c
index f0d2cec78e..7a27bdba20 100644
--- a/libavfilter/vf_convolution.c
+++ b/libavfilter/vf_convolution.c
@@ -604,10 +604,12 @@ static int filter_slice(AVFilterContext *ctx, void *arg, int jobnr, int nb_jobs)
             continue;
         }
         for (y = slice_start; y < slice_end; y += step) {
-            const int xoff = mode == MATRIX_COLUMN ? (y - slice_start) * bpc : radius * bpc;
-            const int yoff = mode == MATRIX_COLUMN ? radius * dstride : 0;
+            const int left = FFMIN(radius, sizew);
+            const int right = FFMAX(left, sizew - radius);
+            const int xoff = mode == MATRIX_COLUMN ? (y - slice_start) * bpc : left * bpc;
+            const int yoff = mode == MATRIX_COLUMN ? left * dstride : 0;
 
-            for (x = 0; x < radius; x++) {
+            for (x = 0; x < left; x++) {
                 const int xoff = mode == MATRIX_COLUMN ? (y - slice_start) * bpc : x * bpc;
                 const int yoff = mode == MATRIX_COLUMN ? x * dstride : 0;
 
@@ -616,11 +618,11 @@ static int filter_slice(AVFilterContext *ctx, void *arg, int jobnr, int nb_jobs)
                                  bias, matrix, c, s->max, radius,
                                  dstride, stride, slice_end - step);
             }
-            s->setup[plane](radius, c, src, stride, radius, width, y, height, bpc);
-            s->filter[plane](dst + yoff + xoff, sizew - 2 * radius,
+            s->setup[plane](radius, c, src, stride, left, width, y, height, bpc);
+            s->filter[plane](dst + yoff + xoff, right - left,
                              rdiv, bias, matrix, c, s->max, radius,
                              dstride, stride, slice_end - step);
-            for (x = sizew - radius; x < sizew; x++) {
+            for (x = right; x < sizew; x++) {
                 const int xoff = mode == MATRIX_COLUMN ? (y - slice_start) * bpc : x * bpc;
                 const int yoff = mode == MATRIX_COLUMN ? x * dstride : 0;
 
-- 
2.52.0

_______________________________________________
ffmpeg-devel mailing list -- ffmpeg-devel@ffmpeg.org
To unsubscribe send an email to ffmpeg-devel-leave@ffmpeg.org