Re: [FFmpeg-devel] [PATCH] avfilter: add sdlvsink for video display

[Zhao Zhili <quinkblack@foxmail.com>]

> On Jun 7, 2024, at 10:12, Hiccup Zhu <hiccupzhu@gmail.com> wrote:
> 
> On Thu, 6 Jun 2024 at 20:20, Zhao Zhili <quinkblack@foxmail.com> wrote:
>> 
>> 
>> On Jun 6, 2024, at 19:51, Shiqi Zhu <hiccupzhu@gmail.com> wrote:
>> 
>> 
>> I’m afraid this has the same issue as libavdevice/sdl[1].
>> 
>> [1] https://patchwork.ffmpeg.org/project/ffmpeg/patch/20230918063728.198377-1-haihao.xiang@intel.com/
>> 
> 
> Thank you for the reminder. This patch does not have that issue. All
> SDL initialization is within the filter_frame() function, so there's
> no problem with different threads being created.

Initialization and render in the same thread is one of the issues. Another
issue is SDL render should be run in main thread. Maybe it’s not the
duty of this filter to control which thread to be called, but this wouldn’t
work with ffmpeg cmd.

> 
> Furthermore, my expertise lies primarily in multimedia framework
> development. I've noticed that the libavfilter now offers a
> comprehensive basic framework, providing various types of filters such
> as src, trans, and sink within the filter-graph.

libavdevice/sdl should work for this case, although it’s deprecated,
and it support more pixel formats than current patch.

TBH, a videosink in libavfilter make sense than libavdevice. But with
the reasons mentioned earlier, I don’t know how to make progress.

> 
> There are already many src and trans type filters in the existing
> codebase, with rich functionalities. However, there are only two sink
> filters: buffersink and nullsink, which is not quite sufficient for
> the development of filter-graphs.
> 
> Thus, I submitted this patch with the intention of enriching the types
> of sink filters available.
> 
> Many thanks for your valuable comments.
> 
>> 
>> Signed-off-by: Shiqi Zhu <hiccupzhu@gmail.com>
>> ---
>> configure                    |   1 +
>> libavfilter/Makefile         |   1 +
>> libavfilter/allfilters.c     |   1 +
>> libavfilter/vsink_sdlvsink.c | 142 +++++++++++++++++++++++++++++++++++
>> 4 files changed, 145 insertions(+)
>> create mode 100644 libavfilter/vsink_sdlvsink.c
>> 
>> diff --git a/configure b/configure
>> index 6c5b8aab9a..968b5c8912 100755
>> --- a/configure
>> +++ b/configure
>> @@ -3977,6 +3977,7 @@ xstack_qsv_filter_deps="libmfx"
>> xstack_qsv_filter_select="qsvvpp"
>> pad_vaapi_filter_deps="vaapi_1"
>> drawbox_vaapi_filter_deps="vaapi_1"
>> +sdlvsink_filter_deps="sdl2"
>> 
>> # examples
>> avio_http_serve_files_deps="avformat avutil fork"
>> diff --git a/libavfilter/Makefile b/libavfilter/Makefile
>> index 5992fd161f..feac6b464d 100644
>> --- a/libavfilter/Makefile
>> +++ b/libavfilter/Makefile
>> @@ -616,6 +616,7 @@ OBJS-$(CONFIG_YUVTESTSRC_FILTER)             += vsrc_testsrc.o
>> OBJS-$(CONFIG_ZONEPLATE_FILTER)              += vsrc_testsrc.o
>> 
>> OBJS-$(CONFIG_NULLSINK_FILTER)               += vsink_nullsink.o
>> +OBJS-$(CONFIG_SDLVSINK_FILTER)               += vsink_sdlvsink.o
>> 
>> # multimedia filters
>> OBJS-$(CONFIG_A3DSCOPE_FILTER)               += avf_a3dscope.o
>> diff --git a/libavfilter/allfilters.c b/libavfilter/allfilters.c
>> index c532682fc2..d5f942ffb5 100644
>> --- a/libavfilter/allfilters.c
>> +++ b/libavfilter/allfilters.c
>> @@ -579,6 +579,7 @@ extern const AVFilter ff_vsrc_yuvtestsrc;
>> extern const AVFilter ff_vsrc_zoneplate;
>> 
>> extern const AVFilter ff_vsink_nullsink;
>> +extern const AVFilter ff_vsink_sdlvsink;
>> 
>> /* multimedia filters */
>> extern const AVFilter ff_avf_a3dscope;
>> diff --git a/libavfilter/vsink_sdlvsink.c b/libavfilter/vsink_sdlvsink.c
>> new file mode 100644
>> index 0000000000..630f719c7e
>> --- /dev/null
>> +++ b/libavfilter/vsink_sdlvsink.c
>> @@ -0,0 +1,142 @@
>> +/*
>> + * This file is part of FFmpeg.
>> + *
>> + * FFmpeg is free software; you can redistribute it and/or
>> + * modify it under the terms of the GNU Lesser General Public
>> + * License as published by the Free Software Foundation; either
>> + * version 2.1 of the License, or (at your option) any later version.
>> + *
>> + * FFmpeg is distributed in the hope that it will be useful,
>> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
>> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
>> + * Lesser General Public License for more details.
>> + *
>> + * You should have received a copy of the GNU Lesser General Public
>> + * License along with FFmpeg; if not, write to the Free Software
>> + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
>> + */
>> +
>> +#include "avfilter.h"
>> +#include "internal.h"
>> +#include "libavutil/internal.h"
>> +#include <SDL2/SDL.h>
>> +#include <unistd.h>
>> +
>> +typedef struct {
>> +    SDL_Window   *window;
>> +    SDL_Renderer *render;
>> +    SDL_Texture  *texture;
>> +} SDLVideoContext;
>> +
>> +static int init(AVFilterContext *ctx)
>> +{
>> +    if (SDL_Init(SDL_INIT_VIDEO) < 0)
>> +    {
>> +        av_log(ctx, AV_LOG_ERROR, "SDL2 could not initialize! %s\n", SDL_GetError());
>> +        return -ENOMEM;
>> +    }
>> +
>> +    SDL_PollEvent(NULL);
>> +    SDL_SetHint(SDL_HINT_VIDEO_X11_NET_WM_BYPASS_COMPOSITOR, "0");
>> +
>> +    return 0;
>> +}
>> +
>> +static void uninit(AVFilterContext *ctx)
>> +{
>> +    SDLVideoContext *sink = ctx->priv;
>> +
>> +    if (sink->texture) {
>> +        SDL_DestroyTexture(sink->texture);
>> +        sink->texture = NULL;
>> +    }
>> +
>> +    if (sink->render) {
>> +        SDL_DestroyRenderer(sink->render);
>> +        sink->render = NULL;
>> +    }
>> +
>> +    if (sink->window) {
>> +        SDL_DestroyWindow(sink->window);
>> +        sink->window = NULL;
>> +    }
>> +
>> +    SDL_Quit();
>> +}
>> +
>> +static int filter_frame(AVFilterLink *link, AVFrame *frame)
>> +{
>> +    AVFilterContext *ctx = link->dst;
>> +    SDLVideoContext *sink = ctx->priv;
>> +    SDL_RendererInfo renderer_info;
>> +
>> +    if (!sink->window) {
>> +        sink->window = SDL_CreateWindow("YUV Player",
>> +                                         SDL_WINDOWPOS_UNDEFINED, SDL_WINDOWPOS_UNDEFINED,
>> +                                         frame->width, frame->height, SDL_WINDOW_SHOWN | SDL_WINDOW_RESIZABLE);
>> +        if (!sink->window) {
>> +            av_log(ctx, AV_LOG_ERROR, "SDL2 could not create window %s\n", SDL_GetError());
>> +            return -ENOMEM;
>> +        }
>> +    }
>> +
>> +    if (!sink->render) {
>> +        sink->render = SDL_CreateRenderer(sink->window, -1, 0);
>> +        if (!sink->render) {
>> +            av_log(ctx, AV_LOG_ERROR, "SDL2 could not create render %s\n", SDL_GetError());
>> +            return -ENOMEM;
>> +        }
>> +        SDL_SetHint(SDL_HINT_RENDER_SCALE_QUALITY, "linear");
>> +
>> +        SDL_GetRendererInfo(sink->render, &renderer_info);
>> +        av_log(ctx, AV_LOG_INFO, "sdl2 renderer name:%s\n", renderer_info.name);
>> +    }
>> +
>> +    if (!sink->texture) {
>> +        sink->texture = SDL_CreateTexture(sink->render, SDL_PIXELFORMAT_IYUV,
>> +                                           SDL_TEXTUREACCESS_STREAMING, frame->width, frame->height);
>> +        if (!sink->texture) {
>> +            av_log(ctx, AV_LOG_ERROR, "create texture failed %s\n", SDL_GetError());
>> +            return -EINVAL;
>> +        }
>> +    }
>> +
>> +    SDL_UpdateYUVTexture(sink->texture, NULL,
>> +                         frame->data[0], frame->linesize[0],
>> +                         frame->data[1], frame->linesize[1],
>> +                         frame->data[2], frame->linesize[2]);
>> +
>> +    SDL_RenderClear(sink->render);
>> +
>> +    SDL_RenderCopy(sink->render, sink->texture, NULL, NULL);
>> +    SDL_RenderPresent(sink->render);
>> +
>> +    av_frame_free(&frame);
>> +
>> +    return 0;
>> +}
>> +
>> +static const AVFilterPad avfilter_vsink_sdlvsink_inputs[] = {
>> +    {
>> +        .name        = "default",
>> +        .type        = AVMEDIA_TYPE_VIDEO,
>> +        .filter_frame = filter_frame,
>> +    },
>> +};
>> +
>> +static const AVFilterPad avfilter_vsink_sdlvsink_outputs[] = {
>> +    {
>> +        .name = NULL,
>> +        .type = AVMEDIA_TYPE_VIDEO,
>> +    },
>> +};
>> +
>> +const AVFilter ff_vsink_sdlvsink = {
>> +    .name        = "sdlvsink",
>> +    .description = NULL_IF_CONFIG_SMALL("Do absolutely nothing with the input video."),
>> +    .priv_size   = sizeof(SDLVideoContext),
>> +    .init        = init,
>> +    .uninit      = uninit,
>> +    FILTER_INPUTS(avfilter_vsink_sdlvsink_inputs),
>> +    FILTER_OUTPUTS(avfilter_vsink_sdlvsink_outputs),
>> +};
>> --
>> 2.34.1
>> 
>> _______________________________________________
>> ffmpeg-devel mailing list
>> ffmpeg-devel@ffmpeg.org
>> https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
>> 
>> To unsubscribe, visit link above, or email
>> ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
>> 
>> 

_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".