From c150cc5f4ae4961a56d035b1fdad7c2eb0eccfad Mon Sep 17 00:00:00 2001
From: Andreas Rheinhardt <andreas.rheinhardt@outlook.com>
Date: Wed, 16 Apr 2025 11:03:53 +0200
Subject: [PATCH 2/2] avcodec/dvbsubenc: Check nb_colors before using it
Avoids a potential overflow when multiplying nb_colors by 6.
Also make the nb_colors check a bit more strict.
Signed-off-by: Andreas Rheinhardt <andreas.rheinhardt@outlook.com>
---
libavcodec/dvbsubenc.c | 11 +++++------
1 file changed, 5 insertions(+), 6 deletions(-)
diff --git a/libavcodec/dvbsubenc.c b/libavcodec/dvbsubenc.c
index 8eeac76855..3659e3f9ed 100644
--- a/libavcodec/dvbsubenc.c
+++ b/libavcodec/dvbsubenc.c
@@ -329,24 +329,23 @@ static int dvbsub_encode(AVCodecContext *avctx, uint8_t *outbuf, int buf_size,
if (h->num_rects) {
for (clut_id = 0; clut_id < h->num_rects; clut_id++) {
- if (buf_size < 6 + h->rects[clut_id]->nb_colors * 6)
- return AVERROR_BUFFER_TOO_SMALL;
-
/* CLUT segment */
- if (h->rects[clut_id]->nb_colors <= 4) {
+ if (h->rects[clut_id]->nb_colors <= 4U) {
/* 2 bpp, some decoders do not support it correctly */
bpp_index = 0;
- } else if (h->rects[clut_id]->nb_colors <= 16) {
+ } else if (h->rects[clut_id]->nb_colors <= 16U) {
/* 4 bpp, standard encoding */
bpp_index = 1;
- } else if (h->rects[clut_id]->nb_colors <= 256) {
+ } else if (h->rects[clut_id]->nb_colors <= 256U) {
/* 8 bpp, standard encoding */
bpp_index = 2;
} else {
return AVERROR(EINVAL);
}
+ if (buf_size < 6 + h->rects[clut_id]->nb_colors * 6)
+ return AVERROR_BUFFER_TOO_SMALL;
/* CLUT segment */
*q++ = 0x0f; /* sync byte */
--
2.45.2