[FFmpeg-devel] [PATCH v3] avcodec/libvpxenc: fix sRGB colorspace for non-RGB pixel formats

[Guangyu Sun via ffmpeg-devel <ffmpeg-devel@ffmpeg.org>]

When encoding VP9 with a YUV pixel format (e.g. yuv420p) and
AVCOL_SPC_RGB colorspace metadata, libvpxenc unconditionally set
VPX_CS_SRGB. This produced a spec-violating bitstream: Profile 0
(4:2:0) with sRGB colorspace, which is only valid for Profile 1/3
(4:4:4). The resulting file is undecodable.

Fix this by setting ctx->vpx_cs to VPX_CS_SRGB in set_pix_fmt()
for 4:4:4 YUV formats when AVCOL_SPC_RGB is set, matching the
existing GBRP path. This covers the legitimate case of RGB data in
YUV444 containers (e.g. H.264 High 4:4:4 with identity matrix).

With this change, any AVCOL_SPC_RGB that reaches the switch in
set_colorspace() is guaranteed to be a subsampled format where
sRGB is invalid. Return an error so the user can fix their
pipeline rather than silently producing incorrect output.

To reproduce:

  ffmpeg -f lavfi -i testsrc=s=64x64:d=1:r=1 \
    -c:v libvpx-vp9 -pix_fmt yuv420p -colorspace rgb bad.webm
  ffprobe bad.webm
  # -> "vp9 (Profile 0), none(pc, gbr/...), 64x64"
  ffmpeg -i bad.webm -f null -
  # -> 0 frames decoded, error

Signed-off-by: Guangyu Sun <gsun@roblox.com>
---
v2:
  - Only fall back to BT.709 if the pixel format is not 4:4:4,
    addressing feedback regarding Profile 1/3 support.
v3:
  - Hard error via AVERROR(EINVAL) - set_colorspace() now returns
    int, and the call site in vpx_init() propagates the error

 libavcodec/libvpxenc.c | 19 +++++++++++++++----
 1 file changed, 15 insertions(+), 4 deletions(-)

diff --git a/libavcodec/libvpxenc.c b/libavcodec/libvpxenc.c
index 88c058c403..c181927a1d 100644
--- a/libavcodec/libvpxenc.c
+++ b/libavcodec/libvpxenc.c
@@ -838,6 +838,8 @@ static int set_pix_fmt(AVCodecContext *avctx, vpx_codec_caps_t codec_caps,
         ctx->vpx_cs = VPX_CS_SRGB;
     case AV_PIX_FMT_YUV444P:
     case AV_PIX_FMT_YUVA444P:
+        if (avctx->colorspace == AVCOL_SPC_RGB)
+            ctx->vpx_cs = VPX_CS_SRGB;
         enccfg->g_profile = 1;
         *img_fmt = VPX_IMG_FMT_I444;
         return 0;
@@ -879,6 +881,8 @@ static int set_pix_fmt(AVCodecContext *avctx, vpx_codec_caps_t codec_caps,
     case AV_PIX_FMT_YUVA444P10:
     case AV_PIX_FMT_YUV444P12:
     case AV_PIX_FMT_YUVA444P12:
+        if (avctx->colorspace == AVCOL_SPC_RGB)
+            ctx->vpx_cs = VPX_CS_SRGB;
         if (codec_caps & VPX_CODEC_CAP_HIGHBITDEPTH) {
             enccfg->g_profile = 3;
             *img_fmt = VPX_IMG_FMT_I44416;
@@ -893,7 +897,7 @@ static int set_pix_fmt(AVCodecContext *avctx, vpx_codec_caps_t codec_caps,
     return AVERROR_INVALIDDATA;
 }
 
-static void set_colorspace(AVCodecContext *avctx)
+static int set_colorspace(AVCodecContext *avctx)
 {
     enum vpx_color_space vpx_cs;
     VPxContext *ctx = avctx->priv_data;
@@ -902,7 +906,11 @@ static void set_colorspace(AVCodecContext *avctx)
         vpx_cs = ctx->vpx_cs;
     } else {
         switch (avctx->colorspace) {
-        case AVCOL_SPC_RGB:         vpx_cs = VPX_CS_SRGB;      break;
+        case AVCOL_SPC_RGB:
+            av_log(avctx, AV_LOG_ERROR,
+                   "RGB colorspace is not compatible with pixel format %s.\n",
+                   av_get_pix_fmt_name(avctx->pix_fmt));
+            return AVERROR(EINVAL);
         case AVCOL_SPC_BT709:       vpx_cs = VPX_CS_BT_709;    break;
         case AVCOL_SPC_UNSPECIFIED: vpx_cs = VPX_CS_UNKNOWN;   break;
         case AVCOL_SPC_RESERVED:    vpx_cs = VPX_CS_RESERVED;  break;
@@ -913,10 +921,11 @@ static void set_colorspace(AVCodecContext *avctx)
         default:
             av_log(avctx, AV_LOG_WARNING, "Unsupported colorspace (%d)\n",
                    avctx->colorspace);
-            return;
+            return 0;
         }
     }
     codecctl_int(avctx, VP9E_SET_COLOR_SPACE, vpx_cs);
+    return 0;
 }
 
 #if VPX_ENCODER_ABI_VERSION >= 11
@@ -1277,7 +1286,9 @@ static av_cold int vpx_init(AVCodecContext *avctx,
             codecctl_int(avctx, VP9E_SET_FRAME_PARALLEL_DECODING, ctx->frame_parallel);
         if (ctx->aq_mode >= 0)
             codecctl_int(avctx, VP9E_SET_AQ_MODE, ctx->aq_mode);
-        set_colorspace(avctx);
+        res = set_colorspace(avctx);
+        if (res < 0)
+            return res;
 #if VPX_ENCODER_ABI_VERSION >= 11
         set_color_range(avctx);
 #endif
-- 
2.52.0

_______________________________________________
ffmpeg-devel mailing list -- ffmpeg-devel@ffmpeg.org
To unsubscribe send an email to ffmpeg-devel-leave@ffmpeg.org