Re: [FFmpeg-devel] [PATCH] avfilter/xstack: Add support for fixed size grid

[Vignesh Venkatasubramanian <vigneshv-at-google.com@ffmpeg.org>]

On Wed, Jun 29, 2022 at 10:15 AM Vignesh Venkatasubramanian
<vigneshv@google.com> wrote:
>
> On Wed, Jun 29, 2022 at 12:24 AM Paul B Mahol <onemda@gmail.com> wrote:
> >
> >
> >
> > On Wed, Jun 29, 2022 at 9:21 AM Paul B Mahol <onemda@gmail.com> wrote:
> >>
> >>
> >>
> >> On Tue, Jun 28, 2022 at 9:01 PM Vignesh Venkatasubramanian <vigneshv-at-google.com@ffmpeg.org> wrote:
> >>>
> >>> Add a short hand parameter for making a fixed size grid. The existing
> >>> xstack layout parameter syntax gets tedious if all one wants is a
> >>> matrix like grid of the input streams. Add a grid option to the xstack
> >>> filter that simplifies this use case by simply specifying the number of
> >>> rows and columns instead of specific x/y co-ordinate for each stream.
> >>
> >>
> >> Use SIZE AVOption for grid size option.
> >> As already done in tile filter.
>
> Ah, i did not know about this option. Updated.
>
> >
> >
> > Also there is no need to always force same size of frames in grid, can either force same width or same height.
> >
>
> I have relaxed the condition to "images within the same row must have
> same height (similar to hstack) and all rows must have the same width
> (similar to vstack)".
>
> > By default it can use grid option, (just set grid default to 2x1, and remove default layout setup thing)
>
> Done.
>
> >>
> >>
> >>
> >>
> >>>
> >>>
> >>> Also updating the filter documentation to explain the new option.
> >>>
> >>> Signed-off-by: Vignesh Venkatasubramanian <vigneshv@google.com>
> >>> ---
> >>>  doc/filters.texi       | 19 +++++++++--
> >>>  libavfilter/vf_stack.c | 73 ++++++++++++++++++++++++++++++++++++------
> >>>  2 files changed, 81 insertions(+), 11 deletions(-)
> >>>
> >>> diff --git a/doc/filters.texi b/doc/filters.texi
> >>> index e525e87b3c..9d800a0fd6 100644
> >>> --- a/doc/filters.texi
> >>> +++ b/doc/filters.texi
> >>> @@ -24381,8 +24381,23 @@ the output video frame will be filled. Similarly, videos can overlap each
> >>>  other if their position doesn't leave enough space for the full frame of
> >>>  adjoining videos.
> >>>
> >>> -For 2 inputs, a default layout of @code{0_0|w0_0} is set. In all other cases,
> >>> -a layout must be set by the user.
> >>> +For 2 inputs, a default layout of @code{0_0|w0_0} is set. In all other cases, a
> >>> +layout or a grid must be set by the user. Either grid or layout option can be
> >>> +specified at a time. Specifying both will result in an error.
> >>> +
> >>> +@item grid
> >>> +Specify a fixed size grid of inputs.
> >>> +This option is used to create a fixed size grid of the input streams.  The
> >>> +option is of the form <rows>x<columns> (e.g. 2x4). There must be <rows> *
> >>> +<columns> input streams and they will be arranged as a grid with <rows> rows and
> >>> +<columns> columns. When using this option, all the input streams must have the
> >>> +same width and height.
> >>> +
> >>> +Either grid or layout option can be specified at a time. Specifying both will
> >>> +result in an error.
> >>> +
> >>> +If grid is set, then inputs option is ignored and is implicitly set to
> >>> +<rows>*<columns>.
> >>>
> >>>  @item shortest
> >>>  If set to 1, force the output to terminate when the shortest input
> >>> diff --git a/libavfilter/vf_stack.c b/libavfilter/vf_stack.c
> >>> index aa32a1bf5e..b38a193355 100644
> >>> --- a/libavfilter/vf_stack.c
> >>> +++ b/libavfilter/vf_stack.c
> >>> @@ -48,6 +48,9 @@ typedef struct StackContext {
> >>>      int is_vertical;
> >>>      int is_horizontal;
> >>>      int nb_planes;
> >>> +    char *grid;
> >>> +    int nb_grid_rows;
> >>> +    int nb_grid_columns;
> >>>      uint8_t fillcolor[4];
> >>>      char *fillcolor_str;
> >>>      int fillcolor_enable;
> >>> @@ -85,14 +88,6 @@ static av_cold int init(AVFilterContext *ctx)
> >>>      if (!strcmp(ctx->filter->name, "hstack"))
> >>>          s->is_horizontal = 1;
> >>>
> >>> -    s->frames = av_calloc(s->nb_inputs, sizeof(*s->frames));
> >>> -    if (!s->frames)
> >>> -        return AVERROR(ENOMEM);
> >>> -
> >>> -    s->items = av_calloc(s->nb_inputs, sizeof(*s->items));
> >>> -    if (!s->items)
> >>> -        return AVERROR(ENOMEM);
> >>> -
> >>>      if (!strcmp(ctx->filter->name, "xstack")) {
> >>>          if (strcmp(s->fillcolor_str, "none") &&
> >>>              av_parse_color(s->fillcolor, s->fillcolor_str, -1, ctx) >= 0) {
> >>> @@ -100,7 +95,21 @@ static av_cold int init(AVFilterContext *ctx)
> >>>          } else {
> >>>              s->fillcolor_enable = 0;
> >>>          }
> >>> -        if (!s->layout) {
> >>> +        if (s->grid && s->layout) {
> >>> +            av_log(ctx, AV_LOG_ERROR, "Both layout and grid were specified. Only one is allowed.\n");
> >>> +            return AVERROR(EINVAL);
> >>> +        }
> >>> +        if (s->grid) {
> >>> +            if (sscanf(s->grid, "%dx%d", &s->nb_grid_rows, &s->nb_grid_columns) != 2) {
> >>> +                av_log(ctx, AV_LOG_ERROR, "grid string is not of the form rowsxcolumns.\n");
> >>> +                return AVERROR(EINVAL);
> >>> +            }
> >>> +            s->nb_inputs = s->nb_grid_rows * s->nb_grid_columns;
> >>> +        } else {
> >>> +            s->nb_grid_rows = 0;
> >>> +            s->nb_grid_columns = 0;
> >>> +        }
> >>> +        if (!s->layout && !s->grid) {
> >>>              if (s->nb_inputs == 2) {
> >>>                  s->layout = av_strdup("0_0|w0_0");
> >>>                  if (!s->layout)
> >>> @@ -112,6 +121,15 @@ static av_cold int init(AVFilterContext *ctx)
> >>>          }
> >>>      }
> >>>
> >>> +    s->frames = av_calloc(s->nb_inputs, sizeof(*s->frames));
> >>> +    if (!s->frames)
> >>> +        return AVERROR(ENOMEM);
> >>> +
> >>> +    s->items = av_calloc(s->nb_inputs, sizeof(*s->items));
> >>> +    if (!s->items)
> >>> +        return AVERROR(ENOMEM);
> >>> +
> >>> +
> >>>      for (i = 0; i < s->nb_inputs; i++) {
> >>>          AVFilterPad pad = { 0 };
> >>>
> >>> @@ -244,6 +262,42 @@ static int config_output(AVFilterLink *outlink)
> >>>                  width += ctx->inputs[i]->w;
> >>>              }
> >>>          }
> >>> +    } else if (s->nb_grid_rows > 0 && s->nb_grid_columns > 0) {
> >>> +        int inw = 0, inh = 0;
> >>> +        int k = 0;
> >>> +        for (i = 0; i < s->nb_grid_rows; i++, inh += height) {
> >>> +            inw = 0;
> >>> +            for (int j = 0; j < s->nb_grid_columns; j++, k++, inw += width) {
> >>> +                AVFilterLink *inlink = ctx->inputs[k];
> >>> +                StackItem *item = &s->items[k];
> >>> +
> >>> +                if (ctx->inputs[k]->w != width) {
> >>> +                    av_log(ctx, AV_LOG_ERROR, "Input %d width %d does not match input %d width %d.\n", k, ctx->inputs[k]->w, 0, width);
> >>> +                    return AVERROR(EINVAL);
> >>> +                }
> >>> +
> >>> +                if (ctx->inputs[k]->h != height) {
> >>> +                    av_log(ctx, AV_LOG_ERROR, "Input %d height %d does not match input %d height %d.\n", k, ctx->inputs[k]->h, 0, height);
> >>> +                    return AVERROR(EINVAL);
> >>> +                }
> >>> +
> >>> +                if ((ret = av_image_fill_linesizes(item->linesize, inlink->format, inlink->w)) < 0) {
> >>> +                    return ret;
> >>> +                }
> >>> +
> >>> +                item->height[1] = item->height[2] = AV_CEIL_RSHIFT(inlink->h, s->desc->log2_chroma_h);
> >>> +                item->height[0] = item->height[3] = inlink->h;
> >>> +
> >>> +                if ((ret = av_image_fill_linesizes(item->x, inlink->format, inw)) < 0) {
> >>> +                    return ret;
> >>> +                }
> >>> +
> >>> +                item->y[1] = item->y[2] = AV_CEIL_RSHIFT(inh, s->desc->log2_chroma_h);
> >>> +                item->y[0] = item->y[3] = inh;
> >>> +            }
> >>> +        }
> >>> +        width *= s->nb_grid_columns;
> >>> +        height *= s->nb_grid_rows;
> >>>      } else {
> >>>          char *arg, *p = s->layout, *saveptr = NULL;
> >>>          char *arg2, *p2, *saveptr2 = NULL;
> >>> @@ -436,6 +490,7 @@ const AVFilter ff_vf_vstack = {
> >>>  static const AVOption xstack_options[] = {
> >>>      { "inputs", "set number of inputs", OFFSET(nb_inputs), AV_OPT_TYPE_INT, {.i64=2}, 2, INT_MAX, .flags = FLAGS },
> >>>      { "layout", "set custom layout", OFFSET(layout), AV_OPT_TYPE_STRING, {.str=NULL}, 0, 0, .flags = FLAGS },
> >>> +    { "grid", "set fixed size grid layout", OFFSET(grid), AV_OPT_TYPE_STRING, {.str=NULL}, 0, 0, .flags = FLAGS },
> >>>      { "shortest", "force termination when the shortest input terminates", OFFSET(shortest), AV_OPT_TYPE_BOOL, {.i64=0}, 0, 1, .flags = FLAGS },
> >>>      { "fill",  "set the color for unused pixels", OFFSET(fillcolor_str), AV_OPT_TYPE_STRING, {.str = "none"}, .flags = FLAGS },
> >>>      { NULL },
> >>> --
> >>> 2.37.0.rc0.161.g10f37bed90-goog
> >>>
> >>> _______________________________________________
> >>> ffmpeg-devel mailing list
> >>> ffmpeg-devel@ffmpeg.org
> >>> https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
> >>>
> >>> To unsubscribe, visit link above, or email
> >>> ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
>
>
>
> --
> Vignesh

Any further comments on this? If not, can this be merged please?

-- 
Vignesh
_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".