From: Zhao Zhili via ffmpeg-devel <ffmpeg-devel@ffmpeg.org>
To: ffmpeg-devel@ffmpeg.org
Cc: Zhao Zhili <code@ffmpeg.org>
Subject: [FFmpeg-devel] [PATCH] avcodec/videotoolboxenc: fix crash with negative linesize (PR #20952)
Date: Tue, 18 Nov 2025 08:54:07 -0000
Message-ID: <176345604788.25.3035538795809242875@2cb04c0e5124> (raw)
PR #20952 opened by Zhao Zhili (quink)
URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/20952
Patch URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/20952.patch
>From 3d02b4ed961e928a6f103e3a5c39238866df70bb Mon Sep 17 00:00:00 2001
From: Zhao Zhili <zhilizhao@tencent.com>
Date: Tue, 18 Nov 2025 12:46:13 +0800
Subject: [PATCH 1/3] avcodec/videotoolboxenc: fix continue after
LockBaseAddress failed
---
libavcodec/videotoolboxenc.c | 10 ++++------
1 file changed, 4 insertions(+), 6 deletions(-)
diff --git a/libavcodec/videotoolboxenc.c b/libavcodec/videotoolboxenc.c
index 729072c0b9..b587383051 100644
--- a/libavcodec/videotoolboxenc.c
+++ b/libavcodec/videotoolboxenc.c
@@ -2414,12 +2414,10 @@ static int copy_avframe_to_pixel_buffer(AVCodecContext *avctx,
status = CVPixelBufferLockBaseAddress(cv_img, 0);
if (status) {
- av_log(
- avctx,
- AV_LOG_ERROR,
- "Error: Could not lock base address of CVPixelBuffer: %d.\n",
- status
- );
+ av_log(avctx, AV_LOG_ERROR,
+ "Error: Could not lock base address of CVPixelBuffer: %d.\n",
+ status);
+ return AVERROR_EXTERNAL;
}
if (CVPixelBufferIsPlanar(cv_img)) {
--
2.49.1
>From 6b42f372b86ce90da4a493d0cc05f91c416c5d40 Mon Sep 17 00:00:00 2001
From: Zhao Zhili <zhilizhao@tencent.com>
Date: Tue, 18 Nov 2025 11:02:59 +0800
Subject: [PATCH 2/3] avcodec/videotoolboxenc: fix crash with negative linesize
---
libavcodec/videotoolboxenc.c | 195 ++++++-----------------------------
1 file changed, 31 insertions(+), 164 deletions(-)
diff --git a/libavcodec/videotoolboxenc.c b/libavcodec/videotoolboxenc.c
index b587383051..aa48ab05f3 100644
--- a/libavcodec/videotoolboxenc.c
+++ b/libavcodec/videotoolboxenc.c
@@ -28,6 +28,7 @@
#include "libavutil/opt.h"
#include "libavutil/avassert.h"
#include "libavutil/avstring.h"
+#include "libavutil/imgutils.h"
#include "libavcodec/avcodec.h"
#include "libavutil/pixdesc.h"
#include "libavutil/hwcontext_videotoolbox.h"
@@ -2328,89 +2329,21 @@ static int vtenc_cm_to_avpacket(
return 0;
}
-/*
- * contiguous_buf_size is 0 if not contiguous, and the size of the buffer
- * containing all planes if so.
- */
-static int get_cv_pixel_info(
- AVCodecContext *avctx,
- const AVFrame *frame,
- int *color,
- int *plane_count,
- size_t *widths,
- size_t *heights,
- size_t *strides,
- size_t *contiguous_buf_size)
-{
- const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(avctx->pix_fmt);
- VTEncContext *vtctx = avctx->priv_data;
- int av_format = frame->format;
- int av_color_range = avctx->color_range;
- int i;
- int range_guessed;
- int status;
-
- if (!desc)
- return AVERROR(EINVAL);
-
- status = get_cv_pixel_format(avctx, av_format, av_color_range, color, &range_guessed);
- if (status)
- return status;
-
- if (range_guessed) {
- if (!vtctx->warned_color_range) {
- vtctx->warned_color_range = true;
- av_log(avctx,
- AV_LOG_WARNING,
- "Color range not set for %s. Using MPEG range.\n",
- av_get_pix_fmt_name(av_format));
- }
- }
-
- *plane_count = av_pix_fmt_count_planes(avctx->pix_fmt);
-
- for (i = 0; i < desc->nb_components; i++) {
- int p = desc->comp[i].plane;
- bool hasAlpha = (desc->flags & AV_PIX_FMT_FLAG_ALPHA);
- bool isAlpha = hasAlpha && (p + 1 == *plane_count);
- bool isChroma = (p != 0) && !isAlpha;
- int shiftw = isChroma ? desc->log2_chroma_w : 0;
- int shifth = isChroma ? desc->log2_chroma_h : 0;
- widths[p] = (avctx->width + ((1 << shiftw) >> 1)) >> shiftw;
- heights[p] = (avctx->height + ((1 << shifth) >> 1)) >> shifth;
- strides[p] = frame->linesize[p];
- }
-
- *contiguous_buf_size = 0;
- for (i = 0; i < *plane_count; i++) {
- if (i < *plane_count - 1 &&
- frame->data[i] + strides[i] * heights[i] != frame->data[i + 1]) {
- *contiguous_buf_size = 0;
- break;
- }
-
- *contiguous_buf_size += strides[i] * heights[i];
- }
-
- return 0;
-}
-
//Not used on OSX - frame is never copied.
static int copy_avframe_to_pixel_buffer(AVCodecContext *avctx,
const AVFrame *frame,
- CVPixelBufferRef cv_img,
- const size_t *plane_strides,
- const size_t *plane_rows)
+ CVPixelBufferRef cv_img)
{
- int i, j;
- size_t plane_count;
int status;
- int rows;
- int src_stride;
- int dst_stride;
- uint8_t *src_addr;
- uint8_t *dst_addr;
- size_t copy_bytes;
+
+ int plane = av_pix_fmt_count_planes(frame->format);
+ size_t cv_plane = CVPixelBufferIsPlanar(cv_img)
+ ? CVPixelBufferGetPlaneCount(cv_img) : 1;
+ if (plane != cv_plane) {
+ av_log(avctx, AV_LOG_ERROR,
+ "Error: different number of planes in AVFrame and CVPixelBuffer.\n");
+ return AVERROR_EXTERNAL;
+ }
status = CVPixelBufferLockBaseAddress(cv_img, 0);
if (status) {
@@ -2420,62 +2353,14 @@ static int copy_avframe_to_pixel_buffer(AVCodecContext *avctx,
return AVERROR_EXTERNAL;
}
- if (CVPixelBufferIsPlanar(cv_img)) {
- plane_count = CVPixelBufferGetPlaneCount(cv_img);
- for (i = 0; frame->data[i]; i++) {
- if (i == plane_count) {
- CVPixelBufferUnlockBaseAddress(cv_img, 0);
- av_log(avctx,
- AV_LOG_ERROR,
- "Error: different number of planes in AVFrame and CVPixelBuffer.\n"
- );
-
- return AVERROR_EXTERNAL;
- }
-
- dst_addr = (uint8_t*)CVPixelBufferGetBaseAddressOfPlane(cv_img, i);
- src_addr = (uint8_t*)frame->data[i];
- dst_stride = CVPixelBufferGetBytesPerRowOfPlane(cv_img, i);
- src_stride = plane_strides[i];
- rows = plane_rows[i];
-
- if (dst_stride == src_stride) {
- memcpy(dst_addr, src_addr, src_stride * rows);
- } else {
- copy_bytes = dst_stride < src_stride ? dst_stride : src_stride;
-
- for (j = 0; j < rows; j++) {
- memcpy(dst_addr + j * dst_stride, src_addr + j * src_stride, copy_bytes);
- }
- }
- }
- } else {
- if (frame->data[1]) {
- CVPixelBufferUnlockBaseAddress(cv_img, 0);
- av_log(avctx,
- AV_LOG_ERROR,
- "Error: different number of planes in AVFrame and non-planar CVPixelBuffer.\n"
- );
-
- return AVERROR_EXTERNAL;
- }
-
- dst_addr = (uint8_t*)CVPixelBufferGetBaseAddress(cv_img);
- src_addr = (uint8_t*)frame->data[0];
- dst_stride = CVPixelBufferGetBytesPerRow(cv_img);
- src_stride = plane_strides[0];
- rows = plane_rows[0];
-
- if (dst_stride == src_stride) {
- memcpy(dst_addr, src_addr, src_stride * rows);
- } else {
- copy_bytes = dst_stride < src_stride ? dst_stride : src_stride;
-
- for (j = 0; j < rows; j++) {
- memcpy(dst_addr + j * dst_stride, src_addr + j * src_stride, copy_bytes);
- }
- }
+ int dst_stride[4] = {0};
+ uint8_t *dst_addr[4] = {0};
+ for (int i = 0; i < plane; i++) {
+ dst_addr[i] = (uint8_t*)CVPixelBufferGetBaseAddressOfPlane(cv_img, i);
+ dst_stride[i] = CVPixelBufferGetBytesPerRowOfPlane(cv_img, i);
}
+ av_image_copy2(dst_addr, dst_stride, frame->data, frame->linesize,
+ frame->format, frame->width, frame->height);
status = CVPixelBufferUnlockBaseAddress(cv_img, 0);
if (status) {
@@ -2491,13 +2376,7 @@ static int create_cv_pixel_buffer(AVCodecContext *avctx,
CVPixelBufferRef *cv_img,
BufNode *node)
{
- int plane_count;
- int color;
- size_t widths [AV_NUM_DATA_POINTERS];
- size_t heights[AV_NUM_DATA_POINTERS];
- size_t strides[AV_NUM_DATA_POINTERS];
int status;
- size_t contiguous_buf_size;
CVPixelBufferPoolRef pix_buf_pool;
VTEncContext* vtctx = avctx->priv_data;
@@ -2517,33 +2396,21 @@ static int create_cv_pixel_buffer(AVCodecContext *avctx,
return 0;
}
- memset(widths, 0, sizeof(widths));
- memset(heights, 0, sizeof(heights));
- memset(strides, 0, sizeof(strides));
-
- status = get_cv_pixel_info(
- avctx,
- frame,
- &color,
- &plane_count,
- widths,
- heights,
- strides,
- &contiguous_buf_size
- );
-
+ int range_guessed;
+ status = get_cv_pixel_format(avctx, frame->format, avctx->color_range,
+ &(int) {0}, &range_guessed);
if (status) {
- av_log(
- avctx,
- AV_LOG_ERROR,
- "Error: Cannot convert format %d color_range %d: %d\n",
- frame->format,
- frame->color_range,
- status
- );
-
+ av_log(avctx, AV_LOG_ERROR, "Error: Cannot convert format %d color_range %d: %d\n",
+ frame->format, frame->color_range, status);
return status;
}
+ if (range_guessed) {
+ if (!vtctx->warned_color_range) {
+ vtctx->warned_color_range = true;
+ av_log(avctx, AV_LOG_WARNING, "Color range not set for %s. Using MPEG range.\n",
+ av_get_pix_fmt_name(frame->format));
+ }
+ }
pix_buf_pool = VTCompressionSessionGetPixelBufferPool(vtctx->session);
if (!pix_buf_pool) {
@@ -2580,7 +2447,7 @@ static int create_cv_pixel_buffer(AVCodecContext *avctx,
return AVERROR_EXTERNAL;
}
- status = copy_avframe_to_pixel_buffer(avctx, frame, *cv_img, strides, heights);
+ status = copy_avframe_to_pixel_buffer(avctx, frame, *cv_img);
if (status) {
CFRelease(*cv_img);
*cv_img = NULL;
--
2.49.1
>From 341730a84e70bb187811e2edf2ae466569a1d306 Mon Sep 17 00:00:00 2001
From: Zhao Zhili <zhilizhao@tencent.com>
Date: Tue, 18 Nov 2025 15:23:16 +0800
Subject: [PATCH 3/3] avcodec/videotoolboxenc: reorder and cleanup headers
---
libavcodec/videotoolboxenc.c | 25 ++++++++++++-------------
1 file changed, 12 insertions(+), 13 deletions(-)
diff --git a/libavcodec/videotoolboxenc.c b/libavcodec/videotoolboxenc.c
index aa48ab05f3..e0684b21a2 100644
--- a/libavcodec/videotoolboxenc.c
+++ b/libavcodec/videotoolboxenc.c
@@ -18,29 +18,28 @@
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
-#include <VideoToolbox/VideoToolbox.h>
-#include <CoreVideo/CoreVideo.h>
-#include <CoreMedia/CoreMedia.h>
-#include <TargetConditionals.h>
#include <Availability.h>
-#include "avcodec.h"
+#include <CoreMedia/CoreMedia.h>
+#include <CoreVideo/CoreVideo.h>
+#include <dlfcn.h>
+#include <pthread.h>
+#include <TargetConditionals.h>
+#include <VideoToolbox/VideoToolbox.h>
+
+#include "libavutil/avassert.h"
+#include "libavutil/imgutils.h"
#include "libavutil/mem.h"
#include "libavutil/opt.h"
-#include "libavutil/avassert.h"
-#include "libavutil/avstring.h"
-#include "libavutil/imgutils.h"
-#include "libavcodec/avcodec.h"
#include "libavutil/pixdesc.h"
#include "libavutil/hwcontext_videotoolbox.h"
-#include "codec_internal.h"
-#include "internal.h"
-#include <pthread.h>
+
#include "atsc_a53.h"
+#include "codec_internal.h"
#include "encode.h"
#include "h264.h"
#include "h264_sei.h"
#include "hwconfig.h"
-#include <dlfcn.h>
+#include "internal.h"
#if !HAVE_KCMVIDEOCODECTYPE_HEVC
enum { kCMVideoCodecType_HEVC = 'hvc1' };
--
2.49.1
_______________________________________________
ffmpeg-devel mailing list -- ffmpeg-devel@ffmpeg.org
To unsubscribe send an email to ffmpeg-devel-leave@ffmpeg.org
reply other threads:[~2025-11-18 8:54 UTC|newest]
Thread overview: [no followups] expand[flat|nested] mbox.gz Atom feed
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=176345604788.25.3035538795809242875@2cb04c0e5124 \
--to=ffmpeg-devel@ffmpeg.org \
--cc=code@ffmpeg.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Git Inbox Mirror of the ffmpeg-devel mailing list - see https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
This inbox may be cloned and mirrored by anyone:
git clone --mirror https://master.gitmailbox.com/ffmpegdev/0 ffmpegdev/git/0.git
# If you have public-inbox 1.1+ installed, you may
# initialize and index your mirror using the following commands:
public-inbox-init -V2 ffmpegdev ffmpegdev/ https://master.gitmailbox.com/ffmpegdev \
ffmpegdev@gitmailbox.com
public-inbox-index ffmpegdev
Example config snippet for mirrors.
AGPL code for this site: git clone https://public-inbox.org/public-inbox.git