[FFmpeg-devel] [PATCH] av_format/whip: fix potential overflow (PR #20357)

[Jack Lau via ffmpeg-devel <ffmpeg-devel@ffmpeg.org>]

PR #20357 opened by Jack Lau (JackLau)
URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/20357
Patch URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/20357.patch


>From 7c533ae7404fb20a4513ef766e368300dfe3cc54 Mon Sep 17 00:00:00 2001
From: Jack Lau <jacklau1222@qq.com>
Date: Thu, 28 Aug 2025 07:22:04 +0800
Subject: [PATCH 1/2] avformat/whip: fix 8-bit overflow and map constraint_set
 bits for H264

profile_iop is an 8-bit field. Previous code copied
AVCodecParameters::profile (which can contain bits
beyond 8 bits) into profile_iop, producing overflow
and wrong values.

This patch maps the constrained flags into the proper
profile_iop bits (constraint_set1/3)

Signed-off-by: Jack Lau <jacklau1222@qq.com>
---
 libavformat/whip.c | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/libavformat/whip.c b/libavformat/whip.c
index 1fcf19aaa3..de8513aadf 100644
--- a/libavformat/whip.c
+++ b/libavformat/whip.c
@@ -562,7 +562,7 @@ static int parse_codec(AVFormatContext *s)
  */
 static int generate_sdp_offer(AVFormatContext *s)
 {
-    int ret = 0, profile, level, profile_iop;
+    int ret = 0, profile, level, profile_iop = 0;
     const char *acodec_name = NULL, *vcodec_name = NULL;
     AVBPrint bp;
     WHIPContext *whip = s->priv_data;
@@ -630,11 +630,12 @@ static int generate_sdp_offer(AVFormatContext *s)
     }
 
     if (whip->video_par) {
-        profile_iop = profile = whip->video_par->profile;
+        profile = whip->video_par->profile;
         level = whip->video_par->level;
         if (whip->video_par->codec_id == AV_CODEC_ID_H264) {
             vcodec_name = "H264";
-            profile_iop &= AV_PROFILE_H264_CONSTRAINED;
+            profile_iop |= profile & AV_PROFILE_H264_CONSTRAINED ? 1 << 6 : 0;
+            profile_iop |= profile & AV_PROFILE_H264_INTRA ? 1 << 4 : 0;
             profile &= (~AV_PROFILE_H264_CONSTRAINED);
         }
 
-- 
2.49.1


>From df611cb6101d136a300f6ee17a57d7a966f40b80 Mon Sep 17 00:00:00 2001
From: Jack Lau <jacklau1222@qq.com>
Date: Thu, 28 Aug 2025 07:40:40 +0800
Subject: [PATCH 2/2] avformat/whip: fix potential 8bit overflow for
 profile_idc

The profile contains profile_idc and constraint_set*_flag,
throws away high 8 bit flags and then we get profile_idc.

Signed-off-by: Jack Lau <jacklau1222@qq.com>
---
 libavformat/whip.c | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/libavformat/whip.c b/libavformat/whip.c
index de8513aadf..0089c17b9a 100644
--- a/libavformat/whip.c
+++ b/libavformat/whip.c
@@ -562,7 +562,7 @@ static int parse_codec(AVFormatContext *s)
  */
 static int generate_sdp_offer(AVFormatContext *s)
 {
-    int ret = 0, profile, level, profile_iop = 0;
+    int ret = 0, profile_idc = 0, level, profile_iop = 0;
     const char *acodec_name = NULL, *vcodec_name = NULL;
     AVBPrint bp;
     WHIPContext *whip = s->priv_data;
@@ -630,13 +630,12 @@ static int generate_sdp_offer(AVFormatContext *s)
     }
 
     if (whip->video_par) {
-        profile = whip->video_par->profile;
         level = whip->video_par->level;
         if (whip->video_par->codec_id == AV_CODEC_ID_H264) {
             vcodec_name = "H264";
-            profile_iop |= profile & AV_PROFILE_H264_CONSTRAINED ? 1 << 6 : 0;
-            profile_iop |= profile & AV_PROFILE_H264_INTRA ? 1 << 4 : 0;
-            profile &= (~AV_PROFILE_H264_CONSTRAINED);
+            profile_iop |= whip->video_par->profile & AV_PROFILE_H264_CONSTRAINED ? 1 << 6 : 0;
+            profile_iop |= whip->video_par->profile & AV_PROFILE_H264_INTRA ? 1 << 4 : 0;
+            profile_idc = whip->video_par->profile & 0x00ff;
         }
 
         av_bprintf(&bp, ""
@@ -662,7 +661,7 @@ static int generate_sdp_offer(AVFormatContext *s)
             whip->video_payload_type,
             vcodec_name,
             whip->video_payload_type,
-            profile,
+            profile_idc,
             profile_iop,
             level,
             whip->video_ssrc,
-- 
2.49.1

_______________________________________________
ffmpeg-devel mailing list -- ffmpeg-devel@ffmpeg.org
To unsubscribe send an email to ffmpeg-devel-leave@ffmpeg.org