[FFmpeg-devel] [PR] avcodec/vp9: Reallocate on resolution change which does not change tile_cols (PR #21550)

[michaelni via ffmpeg-devel <ffmpeg-devel@ffmpeg.org>]

PR #21550 opened by michaelni
URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/21550
Patch URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/21550.patch

Fixes: out of array access on resolution change with slices threads
Fixes: VULN-10/poc.ivf

Found-by: Zhenpeng (Leo) Lin from depthfirst
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>


>From bb74d752a1ba32a76157f13d2ddac119084f863f Mon Sep 17 00:00:00 2001
From: Michael Niedermayer <michael@niedermayer.cc>
Date: Fri, 23 Jan 2026 00:06:23 +0100
Subject: [PATCH] avcodec/vp9: Reallocate on resolution change which does not
 change tile_cols

Fixes: out of array access on resolution change with slices threads
Fixes: VULN-10/poc.ivf

Found-by: Zhenpeng (Leo) Lin from depthfirst
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
---
 libavcodec/vp9.c | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/libavcodec/vp9.c b/libavcodec/vp9.c
index 715d3b7563..454346532c 100644
--- a/libavcodec/vp9.c
+++ b/libavcodec/vp9.c
@@ -177,10 +177,12 @@ static int update_size(AVCodecContext *avctx, int w, int h)
     uint8_t *p;
     int bytesperpixel = s->bytesperpixel, ret, cols, rows;
     int lflvl_len, i;
+    int changed = 0;
 
     av_assert0(w > 0 && h > 0);
 
     if (!(s->pix_fmt == s->gf_fmt && w == s->w && h == s->h)) {
+        changed = 1;
         if ((ret = ff_set_dimensions(avctx, w, h)) < 0)
             return ret;
 
@@ -266,7 +268,7 @@ static int update_size(AVCodecContext *avctx, int w, int h)
     rows = (h + 7) >> 3;
 
     if (s->intra_pred_data[0] && cols == s->cols && rows == s->rows && s->pix_fmt == s->last_fmt)
-        return 0;
+        return changed;
 
     s->last_fmt  = s->pix_fmt;
     s->sb_cols   = (w + 63) >> 6;
@@ -311,9 +313,10 @@ static int update_size(AVCodecContext *avctx, int w, int h)
         ff_vp9dsp_init(&s->dsp, s->s.h.bpp, avctx->flags & AV_CODEC_FLAG_BITEXACT);
         ff_videodsp_init(&s->vdsp, s->s.h.bpp);
         s->last_bpp = s->s.h.bpp;
+        changed = 1;
     }
 
-    return 0;
+    return changed;
 }
 
 static int update_block_buffers(AVCodecContext *avctx)
@@ -520,6 +523,7 @@ static int decode_frame_header(AVCodecContext *avctx,
     int c, i, j, k, l, m, n, w, h, max, size2, ret, sharp;
     int last_invisible;
     const uint8_t *data2;
+    int changed;
 
     /* general header */
     if ((ret = init_get_bits8(&s->gb, data, size)) < 0) {
@@ -789,10 +793,10 @@ FF_ENABLE_DEPRECATION_WARNINGS
     }
 
     /* tiling info */
-    if ((ret = update_size(avctx, w, h)) < 0) {
+    if ((changed = update_size(avctx, w, h)) < 0) {
         av_log(avctx, AV_LOG_ERROR, "Failed to initialize decoder for %dx%d @ %d\n",
                w, h, s->pix_fmt);
-        return ret;
+        return changed;
     }
     for (s->s.h.tiling.log2_tile_cols = 0;
          s->sb_cols > (64 << s->s.h.tiling.log2_tile_cols);
@@ -807,7 +811,7 @@ FF_ENABLE_DEPRECATION_WARNINGS
     }
     s->s.h.tiling.log2_tile_rows = decode012(&s->gb);
     s->s.h.tiling.tile_rows = 1 << s->s.h.tiling.log2_tile_rows;
-    if (s->s.h.tiling.tile_cols != (1 << s->s.h.tiling.log2_tile_cols)) {
+    if (s->s.h.tiling.tile_cols != (1 << s->s.h.tiling.log2_tile_cols) || changed) {
         int n_range_coders;
         VPXRangeCoder *rc;
 
-- 
2.52.0

_______________________________________________
ffmpeg-devel mailing list -- ffmpeg-devel@ffmpeg.org
To unsubscribe send an email to ffmpeg-devel-leave@ffmpeg.org