[FFmpeg-devel] [PATCH 3/3] avcodec/jpeg2000htdec: Check m

From: Michael Niedermayer <michael@niedermayer.cc>
To: FFmpeg development discussions and patches <ffmpeg-devel@ffmpeg.org>
Subject: [FFmpeg-devel] [PATCH 3/3] avcodec/jpeg2000htdec: Check m
Date: Wed,  2 Aug 2023 02:01:35 +0200
Message-ID: <20230802000135.26482-3-michael@niedermayer.cc> (raw)
In-Reply-To: <20230802000135.26482-1-michael@niedermayer.cc>

This also fixes assertion failures

Fixes: shift exponent 95 is too large for 64-bit type 'unsigned long long'
Fixes: 58299/clusterfuzz-testcase-minimized-ffmpeg_AV_CODEC_ID_JPEG2000_fuzzer-5828618092937216

Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
---
 libavcodec/jpeg2000htdec.c | 22 ++++++++++++++++++++--
 1 file changed, 20 insertions(+), 2 deletions(-)

diff --git a/libavcodec/jpeg2000htdec.c b/libavcodec/jpeg2000htdec.c
index 3985783f3a..ae2ee6d6ee 100644
--- a/libavcodec/jpeg2000htdec.c
+++ b/libavcodec/jpeg2000htdec.c
@@ -689,6 +689,10 @@ static int jpeg2000_decode_ht_cleanup_segment(const Jpeg2000DecoderContext *s,
         for (int i = 0; i < 4; i++) {
             m[J2K_Q1][i] = sigma_n[4 * q1 + i] * U[J2K_Q1] - ((emb_pat_k[J2K_Q1] >> i) & 1);
             m[J2K_Q2][i] = sigma_n[4 * q2 + i] * U[J2K_Q2] - ((emb_pat_k[J2K_Q2] >> i) & 1);
+            if (m[J2K_Q1][i] > 63 || m[J2K_Q2][i] > 63) {
+                ret = AVERROR_INVALIDDATA;
+                goto free;
+            }
         }
 
         recover_mag_sgn(mag_sgn_stream, J2K_Q1, q1, m_n, known_1, emb_pat_1, v, m,
@@ -723,8 +727,13 @@ static int jpeg2000_decode_ht_cleanup_segment(const Jpeg2000DecoderContext *s,
 
         U[J2K_Q1] = kappa[J2K_Q1] + u[J2K_Q1];
 
-        for (int i = 0; i < 4; i++)
+        for (int i = 0; i < 4; i++) {
             m[J2K_Q1][i] = sigma_n[4 * q1 + i] * U[J2K_Q1] - ((emb_pat_k[J2K_Q1] >> i) & 1);
+            if (m[J2K_Q1][i] > 63) {
+                ret = AVERROR_INVALIDDATA;
+                goto free;
+            }
+        }
 
         recover_mag_sgn(mag_sgn_stream, J2K_Q1, q1, m_n, known_1, emb_pat_1, v, m,
                         E, mu_n, Dcup, Pcup, pLSB);
@@ -855,6 +864,10 @@ static int jpeg2000_decode_ht_cleanup_segment(const Jpeg2000DecoderContext *s,
             for (int i = 0; i < 4; i++) {
                 m[J2K_Q1][i] = sigma_n[4 * q1 + i] * U[J2K_Q1] - ((emb_pat_k[J2K_Q1] >> i) & 1);
                 m[J2K_Q2][i] = sigma_n[4 * q2 + i] * U[J2K_Q2] - ((emb_pat_k[J2K_Q2] >> i) & 1);
+                if (m[J2K_Q1][i] > 63 || m[J2K_Q2][i] > 63) {
+                    ret = AVERROR_INVALIDDATA;
+                    goto free;
+                }
             }
             recover_mag_sgn(mag_sgn_stream, J2K_Q1, q1, m_n, known_1, emb_pat_1, v, m,
                             E, mu_n, Dcup, Pcup, pLSB);
@@ -920,8 +933,13 @@ static int jpeg2000_decode_ht_cleanup_segment(const Jpeg2000DecoderContext *s,
 
             U[J2K_Q1] = kappa[J2K_Q1] + u[J2K_Q1];
 
-            for (int i = 0; i < 4; i++)
+            for (int i = 0; i < 4; i++) {
                 m[J2K_Q1][i] = sigma_n[4 * q1 + i] * U[J2K_Q1] - ((emb_pat_k[J2K_Q1] >> i) & 1);
+                if (m[J2K_Q1][i] > 63) {
+                    ret = AVERROR_INVALIDDATA;
+                    goto free;
+                }
+            }
 
             recover_mag_sgn(mag_sgn_stream, J2K_Q1, q1, m_n, known_1, emb_pat_1, v, m,
                             E, mu_n, Dcup, Pcup, pLSB);
-- 
2.17.1

_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
