Re: [FFmpeg-devel] [PATCH v2 03/15] avfilter: add negotiation API for color space/range

["Xiang, Haihao" <haihao.xiang-at-intel.com@ffmpeg.org>]

On Wo, 2023-12-13 at 14:12 +0100, Niklas Haas wrote:
> From: Niklas Haas <git@haasn.dev>
> 
> Motivated by YUVJ removal. This change will allow full negotiation
> between color ranges and matrices as needed. By default, all ranges and
> matrices are marked as supported.
> 
> Because grayscale formats are currently handled very inconsistently (and
> in particular, assumed as forced full-range by swscale), we exclude them
> from negotiation altogether for the time being, to get this API merged.
> 
> After filter negotiation is available, we can relax the
> grayscale-is-forced-jpeg restriction again, when it will be more
> feasible to do so without breaking a million test cases.
> 
> Note that this commit updates one FATE test as a consequence of the
> sanity fallback for non-YUV formats. In particular, the test case now
> writes rgb24(pc, gbr/unspecified/unspecified) to the matroska file,
> instead of rgb24(unspecified/unspecified/unspecified) as before.
> ---
>  doc/APIchanges              |   3 +
>  libavfilter/avfilter.c      |  17 +++-
>  libavfilter/avfilter.h      |  28 ++++++
>  libavfilter/avfiltergraph.c | 173 +++++++++++++++++++++++++++++++++++-
>  libavfilter/formats.c       | 122 ++++++++++++++++++++++++-
>  libavfilter/formats.h       |  54 +++++++++++
>  libavfilter/internal.h      |   6 ++
>  libavfilter/vaapi_vpp.c     |   4 +
>  libavfilter/video.c         |   2 +
>  tests/ref/fate/shortest-sub |   4 +-
>  10 files changed, 404 insertions(+), 9 deletions(-)

It caused segfault when using hw accelerations, such as vulkan, vaapi, qsv.

$ffmpeg -init_hw_device qsv -f lavfi -i yuvtestsrc=duration=1,format=nv12 -vf 'hwupload=extra_hw_frames=8' -f null -
$ffmpeg -init_hw_device vaapi -f lavfi -i yuvtestsrc=duration=1,format=nv12 -vf 'hwupload=extra_hw_frames=8' -f null -
$ffmpeg -init_hw_device vulkan -f lavfi -i yuvtestsrc=duration=1,format=nv12 -vf 'hwupload=extra_hw_frames=8' -f null -

Thread 36 "vf#0:0" received signal SIGSEGV, Segmentation fault.
[Switching to Thread 0x7fffc4fe1640 (LWP 3795664)]
pick_format (link=0x7fffb8004100, ref=0x0) at libavfilter/avfiltergraph.c:671
671                 swfmt = ((AVHWFramesContext *) link->hw_frames_ctx->data)->sw_format;
(gdb) bt
#0  pick_format (link=0x7fffb8004100, ref=0x0) at libavfilter/avfiltergraph.c:671
#1  0x00007ffff75d5b57 in pick_formats (graph=0x7fffb8000ff0) at libavfilter/avfiltergraph.c:1213
#2  0x00007ffff75d5ded in graph_config_formats (graph=0x7fffb8000ff0, log_ctx=0x0) at libavfilter/avfiltergraph.c:1273
#3  0x00007ffff75d60c3 in avfilter_graph_config (graphctx=0x7fffb8000ff0, log_ctx=0x0) at libavfilter/avfiltergraph.c:1326
#4  0x00005555555786c8 in configure_filtergraph (fg=0x555555666e60, fgt=0x7fffc4fe0b30) at fftools/ffmpeg_filter.c:1758
#5  0x000055555557b4f5 in send_frame (fg=0x555555666e60, fgt=0x7fffc4fe0b30, ifilter=0x555555663b80, frame=0x7fffb8000b70) at fftools/ffmpeg_filter.c:2652
#6  0x000055555557bdce in filter_thread (arg=0x555555666e60) at fftools/ffmpeg_filter.c:2813
#7  0x000055555559d8cd in task_wrapper (arg=0x555555663df8) at fftools/ffmpeg_sched.c:2200
#8  0x00007ffff4094b43 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#9  0x00007ffff4126a00 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81
(gdb) p link->hw_frames_ctx
$1 = (AVBufferRef *) 0x0

link->hw_frames_ctx is set after graph_config_formats()

Thanks
Haihao


> 
> diff --git a/doc/APIchanges b/doc/APIchanges
> index 4a2dc1c44f..2f6ea50f63 100644
> --- a/doc/APIchanges
> +++ b/doc/APIchanges
> @@ -2,6 +2,9 @@ The last version increases of all libraries were on 2023-02-09
>  
>  API changes, most recent first:
>  
> +2023-11-xx - xxxxxxxxxx - lavf 58.14.100 - avfilter.h
> +  Add AVFilterLink.colorspace, AVFilterLink.color_range
> +
>  2023-11-08 - b82957a66a7 - lavu 58.32.100 - channel_layout.h
>    Add AV_CH_LAYOUT_7POINT2POINT3 and AV_CHANNEL_LAYOUT_7POINT2POINT3.
>    Add AV_CH_LAYOUT_9POINT1POINT4_BACK and
> AV_CHANNEL_LAYOUT_9POINT1POINT4_BACK.
> diff --git a/libavfilter/avfilter.c b/libavfilter/avfilter.c
> index bde1c33d07..31300bb515 100644
> --- a/libavfilter/avfilter.c
> +++ b/libavfilter/avfilter.c
> @@ -185,6 +185,7 @@ int avfilter_link(AVFilterContext *src, unsigned srcpad,
>      link->type    = src->output_pads[srcpad].type;
>      av_assert0(AV_PIX_FMT_NONE == -1 && AV_SAMPLE_FMT_NONE == -1);
>      link->format  = -1;
> +    link->colorspace = AVCOL_SPC_UNSPECIFIED;
>      ff_framequeue_init(&link->fifo, &src->graph->internal->frame_queues);
>  
>      return 0;
> @@ -286,6 +287,12 @@ int avfilter_insert_filter(AVFilterLink *link,
> AVFilterContext *filt,
>      if (link->outcfg.formats)
>          ff_formats_changeref(&link->outcfg.formats,
>                               &filt->outputs[filt_dstpad_idx]-
> >outcfg.formats);
> +    if (link->outcfg.color_spaces)
> +        ff_formats_changeref(&link->outcfg.color_spaces,
> +                             &filt->outputs[filt_dstpad_idx]-
> >outcfg.color_spaces);
> +    if (link->outcfg.color_ranges)
> +        ff_formats_changeref(&link->outcfg.color_ranges,
> +                             &filt->outputs[filt_dstpad_idx]-
> >outcfg.color_ranges);
>      if (link->outcfg.samplerates)
>          ff_formats_changeref(&link->outcfg.samplerates,
>                               &filt->outputs[filt_dstpad_idx]-
> >outcfg.samplerates);
> @@ -730,6 +737,10 @@ static void free_link(AVFilterLink *link)
>  
>      ff_formats_unref(&link->incfg.formats);
>      ff_formats_unref(&link->outcfg.formats);
> +    ff_formats_unref(&link->incfg.color_spaces);
> +    ff_formats_unref(&link->outcfg.color_spaces);
> +    ff_formats_unref(&link->incfg.color_ranges);
> +    ff_formats_unref(&link->outcfg.color_ranges);
>      ff_formats_unref(&link->incfg.samplerates);
>      ff_formats_unref(&link->outcfg.samplerates);
>      ff_channel_layouts_unref(&link->incfg.channel_layouts);
> @@ -987,9 +998,9 @@ int ff_filter_frame(AVFilterLink *link, AVFrame *frame)
>              strcmp(link->dst->filter->name, "idet") &&
>              strcmp(link->dst->filter->name, "null") &&
>              strcmp(link->dst->filter->name, "scale")) {
> -            av_assert1(frame->format                 == link->format);
> -            av_assert1(frame->width               == link->w);
> -            av_assert1(frame->height               == link->h);
> +            av_assert1(frame->format        == link->format);
> +            av_assert1(frame->width         == link->w);
> +            av_assert1(frame->height        == link->h);
>          }
>  
>          frame->sample_aspect_ratio = link->sample_aspect_ratio;
> diff --git a/libavfilter/avfilter.h b/libavfilter/avfilter.h
> index d69381aed4..246d000251 100644
> --- a/libavfilter/avfilter.h
> +++ b/libavfilter/avfilter.h
> @@ -301,6 +301,14 @@ typedef struct AVFilter {
>           * @ref AVFilterFormatsConfig.formats "incfg.formats"
>           * on every output link to a list of pixel/sample formats that the
> filter
>           * supports on that link.
> +         * For video links, this filter may also set
> +         * @ref AVFilterFormatsConfig.color_spaces "incfg.color_spaces"
> +         *  /
> +         * @ref AVFilterFormatsConfig.color_spaces "outcfg.color_spaces"
> +         * and @ref AVFilterFormatsConfig.color_ranges "incfg.color_ranges"
> +         *  /
> +         * @ref AVFilterFormatsConfig.color_ranges "outcfg.color_ranges"
> +         * analogously.
>           * For audio links, this filter must also set
>           * @ref AVFilterFormatsConfig.samplerates "incfg.samplerates"
>           *  /
> @@ -322,6 +330,10 @@ typedef struct AVFilter {
>           * to indicate that this filter supports each of these pixel formats,
>           * provided that all inputs and outputs use the same pixel format.
>           *
> +         * In addition to that the generic code will mark all inputs
> +         * and all outputs as supporting all color spaces and ranges, as
> +         * long as all inputs and outputs use the same color space/range.
> +         *
>           * This list must never be NULL if the union is in this state.
>           * The type of all inputs and outputs of filters using this must
>           * be AVMEDIA_TYPE_VIDEO.
> @@ -514,6 +526,12 @@ typedef struct AVFilterFormatsConfig {
>       */
>      AVFilterChannelLayouts  *channel_layouts;
>  
> +    /**
> +     * Lists of supported YUV color metadata, only for YUV video.
> +     */
> +    AVFilterFormats *color_spaces;  ///< AVColorSpace
> +    AVFilterFormats *color_ranges;  ///< AVColorRange
> +
>  } AVFilterFormatsConfig;
>  
>  /**
> @@ -565,6 +583,16 @@ struct AVFilterLink {
>  
>      AVChannelLayout ch_layout;  ///< channel layout of current buffer (see
> libavutil/channel_layout.h)
>  
> +    /**
> +     * For non-YUV links, these are respectively set to fallback values (as
> +     * appropriate for that colorspace).
> +     *
> +     * Note: This includes grayscale formats, as these are currently treated
> +     * as forced full range always.
> +     */
> +    enum AVColorSpace colorspace;   ///< agreed upon YUV color space
> +    enum AVColorRange color_range;  ///< agreed upon YUV color range
> +
>      /*****************************************************************
>       * All fields below this line are not part of the public API. They
>       * may not be used outside of libavfilter and can be changed and
> diff --git a/libavfilter/avfiltergraph.c b/libavfilter/avfiltergraph.c
> index 625cbc022e..ef739735bd 100644
> --- a/libavfilter/avfiltergraph.c
> +++ b/libavfilter/avfiltergraph.c
> @@ -27,6 +27,7 @@
>  #include "libavutil/avassert.h"
>  #include "libavutil/bprint.h"
>  #include "libavutil/channel_layout.h"
> +#include "libavutil/hwcontext.h"
>  #include "libavutil/imgutils.h"
>  #include "libavutil/opt.h"
>  #include "libavutil/pixdesc.h"
> @@ -298,7 +299,9 @@ static int filter_link_check_formats(void *log,
> AVFilterLink *link, AVFilterForm
>      switch (link->type) {
>  
>      case AVMEDIA_TYPE_VIDEO:
> -        if ((ret = ff_formats_check_pixel_formats(log, cfg->formats)) < 0)
> +        if ((ret = ff_formats_check_pixel_formats(log, cfg->formats)) < 0 ||
> +            (ret = ff_formats_check_color_spaces(log, cfg->color_spaces)) < 0
> ||
> +            (ret = ff_formats_check_color_ranges(log, cfg->color_ranges)) <
> 0)
>              return ret;
>          break;
>  
> @@ -365,6 +368,10 @@ static int formats_declared(AVFilterContext *f)
>      for (i = 0; i < f->nb_inputs; i++) {
>          if (!f->inputs[i]->outcfg.formats)
>              return 0;
> +        if (f->inputs[i]->type == AVMEDIA_TYPE_VIDEO &&
> +            !(f->inputs[i]->outcfg.color_ranges &&
> +              f->inputs[i]->outcfg.color_spaces))
> +            return 0;
>          if (f->inputs[i]->type == AVMEDIA_TYPE_AUDIO &&
>              !(f->inputs[i]->outcfg.samplerates &&
>                f->inputs[i]->outcfg.channel_layouts))
> @@ -373,6 +380,10 @@ static int formats_declared(AVFilterContext *f)
>      for (i = 0; i < f->nb_outputs; i++) {
>          if (!f->outputs[i]->incfg.formats)
>              return 0;
> +        if (f->outputs[i]->type == AVMEDIA_TYPE_VIDEO &&
> +            !(f->outputs[i]->incfg.color_ranges &&
> +              f->outputs[i]->incfg.color_spaces))
> +            return 0;
>          if (f->outputs[i]->type == AVMEDIA_TYPE_AUDIO &&
>              !(f->outputs[i]->incfg.samplerates &&
>                f->outputs[i]->incfg.channel_layouts))
> @@ -493,7 +504,16 @@ static int query_formats(AVFilterGraph *graph, void
> *log_ctx)
>                  av_assert0( inlink->outcfg.formats->refcount > 0);
>                  av_assert0(outlink->incfg.formats->refcount > 0);
>                  av_assert0(outlink->outcfg.formats->refcount > 0);
> -                if (outlink->type == AVMEDIA_TYPE_AUDIO) {
> +                if (outlink->type == AVMEDIA_TYPE_VIDEO) {
> +                    av_assert0( inlink-> incfg.color_spaces->refcount > 0);
> +                    av_assert0( inlink->outcfg.color_spaces->refcount > 0);
> +                    av_assert0(outlink-> incfg.color_spaces->refcount > 0);
> +                    av_assert0(outlink->outcfg.color_spaces->refcount > 0);
> +                    av_assert0( inlink-> incfg.color_ranges->refcount > 0);
> +                    av_assert0( inlink->outcfg.color_ranges->refcount > 0);
> +                    av_assert0(outlink-> incfg.color_ranges->refcount > 0);
> +                    av_assert0(outlink->outcfg.color_ranges->refcount > 0);
> +                } else if (outlink->type == AVMEDIA_TYPE_AUDIO) {
>                      av_assert0( inlink-> incfg.samplerates->refcount > 0);
>                      av_assert0( inlink->outcfg.samplerates->refcount > 0);
>                      av_assert0(outlink-> incfg.samplerates->refcount > 0);
> @@ -583,6 +603,30 @@ static enum AVSampleFormat find_best_sample_fmt_of_2(enum
> AVSampleFormat dst_fmt
>      return score1 < score2 ? dst_fmt1 : dst_fmt2;
>  }
>  
> +int ff_fmt_is_regular_yuv(enum AVPixelFormat fmt)
> +{
> +    const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(fmt);
> +    if (!desc)
> +        return 0;
> +    if (desc->nb_components < 3)
> +        return 0; /* Grayscale is explicitly full-range in swscale */
> +    av_assert1(!(desc->flags & AV_PIX_FMT_FLAG_HWACCEL));
> +    if (desc->flags & (AV_PIX_FMT_FLAG_RGB | AV_PIX_FMT_FLAG_PAL |
> +                       AV_PIX_FMT_FLAG_XYZ | AV_PIX_FMT_FLAG_FLOAT))
> +        return 0;
> +
> +    switch (fmt) {
> +    case AV_PIX_FMT_YUVJ420P:
> +    case AV_PIX_FMT_YUVJ422P:
> +    case AV_PIX_FMT_YUVJ444P:
> +    case AV_PIX_FMT_YUVJ440P:
> +    case AV_PIX_FMT_YUVJ411P:
> +        return 0;
> +    default:
> +        return 1;
> +    }
> +}
> +
>  static int pick_format(AVFilterLink *link, AVFilterLink *ref)
>  {
>      if (!link || !link->incfg.formats)
> @@ -621,7 +665,46 @@ static int pick_format(AVFilterLink *link, AVFilterLink
> *ref)
>      link->incfg.formats->nb_formats = 1;
>      link->format = link->incfg.formats->formats[0];
>  
> -    if (link->type == AVMEDIA_TYPE_AUDIO) {
> +    if (link->type == AVMEDIA_TYPE_VIDEO) {
> +        enum AVPixelFormat swfmt = link->format;
> +        if (av_pix_fmt_desc_get(swfmt)->flags & AV_PIX_FMT_FLAG_HWACCEL) {
> +            av_assert1(link->hw_frames_ctx);
> +            swfmt = ((AVHWFramesContext *) link->hw_frames_ctx->data)-
> >sw_format;
> +        }
> +
> +        if (!ff_fmt_is_regular_yuv(swfmt)) {
> +            const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(swfmt);
> +            /* These fields are explicitly documented as affecting YUV only,
> +             * so set them to sane values for other formats. */
> +            if (desc->flags & AV_PIX_FMT_FLAG_FLOAT)
> +                link->color_range = AVCOL_RANGE_UNSPECIFIED;
> +            else
> +                link->color_range = AVCOL_RANGE_JPEG;
> +            if (desc->flags & (AV_PIX_FMT_FLAG_RGB | AV_PIX_FMT_FLAG_XYZ)) {
> +                link->colorspace = AVCOL_SPC_RGB;
> +            } else {
> +                link->colorspace = AVCOL_SPC_UNSPECIFIED;
> +            }
> +        } else {
> +            if (!link->incfg.color_spaces->nb_formats) {
> +                av_log(link->src, AV_LOG_ERROR, "Cannot select color space
> for"
> +                       " the link between filters %s and %s.\n", link->src-
> >name,
> +                       link->dst->name);
> +                return AVERROR(EINVAL);
> +            }
> +            link->incfg.color_spaces->nb_formats = 1;
> +            link->colorspace = link->incfg.color_spaces->formats[0];
> +
> +            if (!link->incfg.color_ranges->nb_formats) {
> +                av_log(link->src, AV_LOG_ERROR, "Cannot select color range
> for"
> +                       " the link between filters %s and %s.\n", link->src-
> >name,
> +                       link->dst->name);
> +                return AVERROR(EINVAL);
> +            }
> +            link->incfg.color_ranges->nb_formats = 1;
> +            link->color_range = link->incfg.color_ranges->formats[0];
> +        }
> +    } else if (link->type == AVMEDIA_TYPE_AUDIO) {
>          int ret;
>  
>          if (!link->incfg.samplerates->nb_formats) {
> @@ -661,6 +744,10 @@ FF_ENABLE_DEPRECATION_WARNINGS
>      ff_formats_unref(&link->outcfg.samplerates);
>      ff_channel_layouts_unref(&link->incfg.channel_layouts);
>      ff_channel_layouts_unref(&link->outcfg.channel_layouts);
> +    ff_formats_unref(&link->incfg.color_spaces);
> +    ff_formats_unref(&link->outcfg.color_spaces);
> +    ff_formats_unref(&link->incfg.color_ranges);
> +    ff_formats_unref(&link->outcfg.color_ranges);
>  
>      return 0;
>  }
> @@ -822,6 +909,82 @@ static void swap_samplerates(AVFilterGraph *graph)
>          swap_samplerates_on_filter(graph->filters[i]);
>  }
>  
> +static void swap_color_spaces_on_filter(AVFilterContext *filter)
> +{
> +    AVFilterLink *link = NULL;
> +    enum AVColorSpace csp;
> +    int i;
> +
> +    for (i = 0; i < filter->nb_inputs; i++) {
> +        link = filter->inputs[i];
> +        if (link->type == AVMEDIA_TYPE_VIDEO &&
> +            link->outcfg.color_spaces->nb_formats == 1)
> +            break;
> +    }
> +    if (i == filter->nb_inputs)
> +        return;
> +
> +    csp = link->outcfg.color_spaces->formats[0];
> +
> +    for (i = 0; i < filter->nb_outputs; i++) {
> +        AVFilterLink *outlink = filter->outputs[i];
> +        if (outlink->type != AVMEDIA_TYPE_VIDEO)
> +            continue;
> +        /* there is no meaningful 'score' between different yuv matrices,
> +         * so just prioritize an exact match if it exists */
> +        for (int j = 0; j < outlink->incfg.color_spaces->nb_formats; j++) {
> +            if (csp == outlink->incfg.color_spaces->formats[j]) {
> +                FFSWAP(int, outlink->incfg.color_spaces->formats[0],
> +                       outlink->incfg.color_spaces->formats[j]);
> +                break;
> +            }
> +        }
> +    }
> +}
> +
> +static void swap_color_spaces(AVFilterGraph *graph)
> +{
> +    for (int i = 0; i < graph->nb_filters; i++)
> +        swap_color_spaces_on_filter(graph->filters[i]);
> +}
> +
> +static void swap_color_ranges_on_filter(AVFilterContext *filter)
> +{
> +    AVFilterLink *link = NULL;
> +    enum AVColorRange range;
> +    int i;
> +
> +    for (i = 0; i < filter->nb_inputs; i++) {
> +        link = filter->inputs[i];
> +        if (link->type == AVMEDIA_TYPE_VIDEO &&
> +            link->outcfg.color_ranges->nb_formats == 1)
> +            break;
> +    }
> +    if (i == filter->nb_inputs)
> +        return;
> +
> +    range = link->outcfg.color_ranges->formats[0];
> +
> +    for (i = 0; i < filter->nb_outputs; i++) {
> +        AVFilterLink *outlink = filter->outputs[i];
> +        if (outlink->type != AVMEDIA_TYPE_VIDEO)
> +            continue;
> +        for (int j = 0; j < outlink->incfg.color_ranges->nb_formats; j++) {
> +            if (range == outlink->incfg.color_ranges->formats[j]) {
> +                FFSWAP(int, outlink->incfg.color_ranges->formats[0],
> +                       outlink->incfg.color_ranges->formats[j]);
> +                break;
> +            }
> +        }
> +    }
> +}
> +
> +static void swap_color_ranges(AVFilterGraph *graph)
> +{
> +    for (int i = 0; i < graph->nb_filters; i++)
> +        swap_color_ranges_on_filter(graph->filters[i]);
> +}
> +
>  #define CH_CENTER_PAIR (AV_CH_FRONT_LEFT_OF_CENTER |
> AV_CH_FRONT_RIGHT_OF_CENTER)
>  #define CH_FRONT_PAIR  (AV_CH_FRONT_LEFT           | AV_CH_FRONT_RIGHT)
>  #define CH_STEREO_PAIR (AV_CH_STEREO_LEFT          | AV_CH_STEREO_RIGHT)
> @@ -1098,6 +1261,10 @@ static int graph_config_formats(AVFilterGraph *graph,
> void *log_ctx)
>      if ((ret = reduce_formats(graph)) < 0)
>          return ret;
>  
> +    /* for video filters, ensure that the best colorspace metadata is
> selected */
> +    swap_color_spaces(graph);
> +    swap_color_ranges(graph);
> +
>      /* for audio filters, ensure the best format, sample rate and channel
> layout
>       * is selected */
>      swap_sample_fmts(graph);
> diff --git a/libavfilter/formats.c b/libavfilter/formats.c
> index d1c97daf64..20f00d2db4 100644
> --- a/libavfilter/formats.c
> +++ b/libavfilter/formats.c
> @@ -321,12 +321,46 @@ static int merge_channel_layouts(void *a, void *b)
>      return merge_channel_layouts_internal(a, b, 0);
>  }
>  
> +static int merge_generic_internal(AVFilterFormats *a,
> +                                  AVFilterFormats *b, int check)
> +{
> +    av_assert2(check || (a->refcount && b->refcount));
> +
> +    if (a == b)
> +        return 1;
> +
> +    MERGE_FORMATS(a, b, formats, nb_formats, AVFilterFormats, check, 0);
> +
> +    return 1;
> +}
> +
> +static int can_merge_generic(const void *a, const void *b)
> +{
> +    return merge_generic_internal((AVFilterFormats *)a,
> +                                  (AVFilterFormats *)b, 1);
> +}
> +
> +static int merge_generic(void *a, void *b)
> +{
> +    return merge_generic_internal(a, b, 0);
> +}
> +
>  static const AVFilterFormatsMerger mergers_video[] = {
>      {
>          .offset     = offsetof(AVFilterFormatsConfig, formats),
>          .merge      = merge_pix_fmts,
>          .can_merge  = can_merge_pix_fmts,
>      },
> +    {
> +        .offset     = offsetof(AVFilterFormatsConfig, color_spaces),
> +        .merge      = merge_generic,
> +        .can_merge  = can_merge_generic,
> +    },
> +    {
> +        .offset     = offsetof(AVFilterFormatsConfig, color_ranges),
> +        .merge      = merge_generic,
> +        .can_merge  = can_merge_generic,
> +    },
>  };
>  
>  static const AVFilterFormatsMerger mergers_audio[] = {
> @@ -594,6 +628,33 @@ AVFilterChannelLayouts *ff_all_channel_counts(void)
>      return ret;
>  }
>  
> +AVFilterFormats *ff_all_color_spaces(void)
> +{
> +    AVFilterFormats *ret = NULL;
> +    if (ff_add_format(&ret, AVCOL_SPC_UNSPECIFIED) < 0)
> +        return NULL;
> +    for (int csp = 0; csp < AVCOL_SPC_NB; csp++) {
> +        if (csp == AVCOL_SPC_RESERVED ||
> +            csp == AVCOL_SPC_UNSPECIFIED)
> +            continue;
> +        if (ff_add_format(&ret, csp) < 0)
> +            return NULL;
> +    }
> +
> +    return ret;
> +}
> +
> +AVFilterFormats *ff_all_color_ranges(void)
> +{
> +    AVFilterFormats *ret = NULL;
> +    for (int range = 0; range < AVCOL_RANGE_NB; range++) {
> +        if (ff_add_format(&ret, range) < 0)
> +            return NULL;
> +    }
> +
> +    return ret;
> +}
> +
>  #define FORMATS_REF(f, ref,
> unref_fn)                                           \
>      void
> *tmp;                                                                  \
>                                                                               
>    \
> @@ -763,6 +824,42 @@ int ff_set_common_all_samplerates(AVFilterContext *ctx)
>      return ff_set_common_samplerates(ctx, ff_all_samplerates());
>  }
>  
> +int ff_set_common_color_spaces(AVFilterContext *ctx,
> +                               AVFilterFormats *color_spaces)
> +{
> +    SET_COMMON_FORMATS(ctx, color_spaces, AVMEDIA_TYPE_VIDEO,
> +                       ff_formats_ref, ff_formats_unref);
> +}
> +
> +int ff_set_common_color_spaces_from_list(AVFilterContext *ctx,
> +                                         const int *color_ranges)
> +{
> +    return ff_set_common_color_spaces(ctx,
> ff_make_format_list(color_ranges));
> +}
> +
> +int ff_set_common_all_color_spaces(AVFilterContext *ctx)
> +{
> +    return ff_set_common_color_spaces(ctx, ff_all_color_spaces());
> +}
> +
> +int ff_set_common_color_ranges(AVFilterContext *ctx,
> +                               AVFilterFormats *color_ranges)
> +{
> +    SET_COMMON_FORMATS(ctx, color_ranges, AVMEDIA_TYPE_VIDEO,
> +                       ff_formats_ref, ff_formats_unref);
> +}
> +
> +int ff_set_common_color_ranges_from_list(AVFilterContext *ctx,
> +                                         const int *color_ranges)
> +{
> +    return ff_set_common_color_ranges(ctx,
> ff_make_format_list(color_ranges));
> +}
> +
> +int ff_set_common_all_color_ranges(AVFilterContext *ctx)
> +{
> +    return ff_set_common_color_ranges(ctx, ff_all_color_ranges());
> +}
> +
>  /**
>   * A helper for query_formats() which sets all links to the same list of
>   * formats. If there are no links hooked to this filter, the list of formats
> is
> @@ -817,7 +914,14 @@ int ff_default_query_formats(AVFilterContext *ctx)
>      ret = ff_set_common_formats(ctx, formats);
>      if (ret < 0)
>          return ret;
> -    if (type == AVMEDIA_TYPE_AUDIO) {
> +    if (type == AVMEDIA_TYPE_VIDEO) {
> +        ret = ff_set_common_all_color_spaces(ctx);
> +        if (ret < 0)
> +            return ret;
> +        ret = ff_set_common_all_color_ranges(ctx);
> +        if (ret < 0)
> +            return ret;
> +    } else if (type == AVMEDIA_TYPE_AUDIO) {
>          ret = ff_set_common_all_channel_counts(ctx);
>          if (ret < 0)
>              return ret;
> @@ -935,6 +1039,22 @@ int ff_formats_check_sample_rates(void *log, const
> AVFilterFormats *fmts)
>      return check_list(log, "sample rate", fmts);
>  }
>  
> +int ff_formats_check_color_spaces(void *log, const AVFilterFormats *fmts)
> +{
> +    for (int i = 0; fmts && i < fmts->nb_formats; i++) {
> +        if (fmts->formats[i] == AVCOL_SPC_RESERVED) {
> +            av_log(log, AV_LOG_ERROR, "Invalid color range\n");
> +            return AVERROR(EINVAL);
> +        }
> +    }
> +    return check_list(log, "color space", fmts);
> +}
> +
> +int ff_formats_check_color_ranges(void *log, const AVFilterFormats *fmts)
> +{
> +    return check_list(log, "color range", fmts);
> +}
> +
>  static int layouts_compatible(const AVChannelLayout *a, const AVChannelLayout
> *b)
>  {
>      return !av_channel_layout_compare(a, b) ||
> diff --git a/libavfilter/formats.h b/libavfilter/formats.h
> index d44890109e..82b3af4be1 100644
> --- a/libavfilter/formats.h
> +++ b/libavfilter/formats.h
> @@ -130,6 +130,20 @@ AVFilterChannelLayouts *ff_all_channel_counts(void);
>  av_warn_unused_result
>  AVFilterChannelLayouts *ff_make_channel_layout_list(const AVChannelLayout
> *fmts);
>  
> +/**
> + * Construct an AVFilterFormats representing all possible color spaces.
> + *
> + * Note: This list does not include AVCOL_SPC_RESERVED.
> + */
> +av_warn_unused_result
> +AVFilterFormats *ff_all_color_spaces(void);
> +
> +/**
> + * Construct an AVFilterFormats representing all possible color ranges.
> + */
> +av_warn_unused_result
> +AVFilterFormats *ff_all_color_ranges(void);
> +
>  /**
>   * Helpers for query_formats() which set all free audio links to the same
> list
>   * of channel layouts/sample rates. If there are no links hooked to this
> list,
> @@ -165,6 +179,38 @@ int ff_set_common_samplerates_from_list(AVFilterContext
> *ctx,
>  av_warn_unused_result
>  int ff_set_common_all_samplerates(AVFilterContext *ctx);
>  
> +av_warn_unused_result
> +int ff_set_common_color_spaces(AVFilterContext *ctx,
> +                               AVFilterFormats *color_spaces);
> +/**
> + * Equivalent to ff_set_common_color_spaces(ctx,
> ff_make_format_list(color_spaces))
> + */
> +av_warn_unused_result
> +int ff_set_common_color_spaces_from_list(AVFilterContext *ctx,
> +                                         const int *color_spaces);
> +
> +/**
> + * Equivalent to ff_set_common_color_spaces(ctx, ff_all_color_spaces())
> + */
> +av_warn_unused_result
> +int ff_set_common_all_color_spaces(AVFilterContext *ctx);
> +
> +av_warn_unused_result
> +int ff_set_common_color_ranges(AVFilterContext *ctx,
> +                               AVFilterFormats *color_ranges);
> +/**
> + * Equivalent to ff_set_common_color_ranges(ctx,
> ff_make_format_list(color_ranges))
> + */
> +av_warn_unused_result
> +int ff_set_common_color_ranges_from_list(AVFilterContext *ctx,
> +                                         const int *color_ranges);
> +
> +/**
> + * Equivalent to ff_set_common_color_ranges(ctx, ff_all_color_ranges())
> + */
> +av_warn_unused_result
> +int ff_set_common_all_color_ranges(AVFilterContext *ctx);
> +
>  /**
>   * A helper for query_formats() which sets all links to the same list of
>   * formats. If there are no links hooked to this filter, the list of formats
> is
> @@ -328,6 +374,14 @@ int ff_formats_check_sample_rates(void *log, const
> AVFilterFormats *fmts);
>   */
>  int ff_formats_check_channel_layouts(void *log, const AVFilterChannelLayouts
> *fmts);
>  
> +/**
> + * Check that fmts is a valid formats list for YUV colorspace metadata.
> + *
> + * In particular, check for duplicates.
> + */
> +int ff_formats_check_color_spaces(void *log, const AVFilterFormats *fmts);
> +int ff_formats_check_color_ranges(void *log, const AVFilterFormats *fmts);
> +
>  typedef struct AVFilterFormatMerger {
>      unsigned offset;
>      int (*merge)(void *a, void *b);
> diff --git a/libavfilter/internal.h b/libavfilter/internal.h
> index 2dbc5def0a..a6cdf9994c 100644
> --- a/libavfilter/internal.h
> +++ b/libavfilter/internal.h
> @@ -203,6 +203,12 @@ enum FilterFormatsState {
>   */
>  int ff_fmt_is_in(int fmt, const int *fmts);
>  
> +/**
> + * Returns true if a pixel format is "regular YUV", which includes all pixel
> + * formats that are affected by YUV colorspace negotiation.
> + */
> +int ff_fmt_is_regular_yuv(enum AVPixelFormat fmt);
> +
>  /* Functions to parse audio format arguments */
>  
>  /**
> diff --git a/libavfilter/vaapi_vpp.c b/libavfilter/vaapi_vpp.c
> index cf2592e068..59961bfa4a 100644
> --- a/libavfilter/vaapi_vpp.c
> +++ b/libavfilter/vaapi_vpp.c
> @@ -38,6 +38,10 @@ int ff_vaapi_vpp_query_formats(AVFilterContext *avctx)
>                                &avctx->outputs[0]->incfg.formats)) < 0)
>          return err;
>  
> +    if ((err = ff_set_common_all_color_spaces(avctx)) < 0 ||
> +        (err = ff_set_common_all_color_ranges(avctx)) < 0)
> +        return err;
> +
>      return 0;
>  }
>  
> diff --git a/libavfilter/video.c b/libavfilter/video.c
> index 42eeb98c28..243762c8fd 100644
> --- a/libavfilter/video.c
> +++ b/libavfilter/video.c
> @@ -96,6 +96,8 @@ AVFrame *ff_default_get_video_buffer2(AVFilterLink *link,
> int w, int h, int alig
>          return NULL;
>  
>      frame->sample_aspect_ratio = link->sample_aspect_ratio;
> +    frame->colorspace  = link->colorspace;
> +    frame->color_range = link->color_range;
>  
>      return frame;
>  }
> diff --git a/tests/ref/fate/shortest-sub b/tests/ref/fate/shortest-sub
> index 9caee587ce..16b1324e83 100644
> --- a/tests/ref/fate/shortest-sub
> +++ b/tests/ref/fate/shortest-sub
> @@ -1,5 +1,5 @@
> -73d142a80965f9e0884a5863abde0dab *tests/data/fate/shortest-sub.matroska
> -139249 tests/data/fate/shortest-sub.matroska
> +d6608277c93097383e62388196dc62f0 *tests/data/fate/shortest-sub.matroska
> +139260 tests/data/fate/shortest-sub.matroska
>  #extradata 1:      167, 0xf7272d5f
>  #tb 0: 1/1000
>  #media_type 0: video

_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".