[FFmpeg-devel] [PATCH] avfilter: add sdlvsink for video display

[FFmpeg-devel] [PATCH] avfilter: add sdlvsink for video display

[Shiqi Zhu]

Signed-off-by: Shiqi Zhu <hiccupzhu@gmail.com>
---
 configure                    |   1 +
 libavfilter/Makefile         |   1 +
 libavfilter/allfilters.c     |   1 +
 libavfilter/vsink_sdlvsink.c | 142 +++++++++++++++++++++++++++++++++++
 4 files changed, 145 insertions(+)
 create mode 100644 libavfilter/vsink_sdlvsink.c

diff --git a/configure b/configure
index 6c5b8aab9a..968b5c8912 100755
--- a/configure
+++ b/configure
@@ -3977,6 +3977,7 @@ xstack_qsv_filter_deps="libmfx"
 xstack_qsv_filter_select="qsvvpp"
 pad_vaapi_filter_deps="vaapi_1"
 drawbox_vaapi_filter_deps="vaapi_1"
+sdlvsink_filter_deps="sdl2"
 
 # examples
 avio_http_serve_files_deps="avformat avutil fork"
diff --git a/libavfilter/Makefile b/libavfilter/Makefile
index 5992fd161f..feac6b464d 100644
--- a/libavfilter/Makefile
+++ b/libavfilter/Makefile
@@ -616,6 +616,7 @@ OBJS-$(CONFIG_YUVTESTSRC_FILTER)             += vsrc_testsrc.o
 OBJS-$(CONFIG_ZONEPLATE_FILTER)              += vsrc_testsrc.o
 
 OBJS-$(CONFIG_NULLSINK_FILTER)               += vsink_nullsink.o
+OBJS-$(CONFIG_SDLVSINK_FILTER)               += vsink_sdlvsink.o
 
 # multimedia filters
 OBJS-$(CONFIG_A3DSCOPE_FILTER)               += avf_a3dscope.o
diff --git a/libavfilter/allfilters.c b/libavfilter/allfilters.c
index c532682fc2..d5f942ffb5 100644
--- a/libavfilter/allfilters.c
+++ b/libavfilter/allfilters.c
@@ -579,6 +579,7 @@ extern const AVFilter ff_vsrc_yuvtestsrc;
 extern const AVFilter ff_vsrc_zoneplate;
 
 extern const AVFilter ff_vsink_nullsink;
+extern const AVFilter ff_vsink_sdlvsink;
 
 /* multimedia filters */
 extern const AVFilter ff_avf_a3dscope;
diff --git a/libavfilter/vsink_sdlvsink.c b/libavfilter/vsink_sdlvsink.c
new file mode 100644
index 0000000000..630f719c7e
--- /dev/null
+++ b/libavfilter/vsink_sdlvsink.c
@@ -0,0 +1,142 @@
+/*
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include "avfilter.h"
+#include "internal.h"
+#include "libavutil/internal.h"
+#include <SDL2/SDL.h>
+#include <unistd.h>
+
+typedef struct {
+    SDL_Window   *window;
+    SDL_Renderer *render;
+    SDL_Texture  *texture;
+} SDLVideoContext;
+
+static int init(AVFilterContext *ctx)
+{
+    if (SDL_Init(SDL_INIT_VIDEO) < 0)
+    {
+        av_log(ctx, AV_LOG_ERROR, "SDL2 could not initialize! %s\n", SDL_GetError());
+        return -ENOMEM;
+    }
+
+    SDL_PollEvent(NULL);
+    SDL_SetHint(SDL_HINT_VIDEO_X11_NET_WM_BYPASS_COMPOSITOR, "0");
+
+    return 0;
+}
+
+static void uninit(AVFilterContext *ctx)
+{
+    SDLVideoContext *sink = ctx->priv;
+    
+    if (sink->texture) {
+        SDL_DestroyTexture(sink->texture);
+        sink->texture = NULL;
+    }
+
+    if (sink->render) {
+        SDL_DestroyRenderer(sink->render);
+        sink->render = NULL;
+    }
+
+    if (sink->window) {
+        SDL_DestroyWindow(sink->window);
+        sink->window = NULL;
+    }
+
+    SDL_Quit();
+}
+
+static int filter_frame(AVFilterLink *link, AVFrame *frame)
+{
+    AVFilterContext *ctx = link->dst;
+    SDLVideoContext *sink = ctx->priv;
+    SDL_RendererInfo renderer_info;
+
+    if (!sink->window) {
+        sink->window = SDL_CreateWindow("YUV Player",
+                                         SDL_WINDOWPOS_UNDEFINED, SDL_WINDOWPOS_UNDEFINED,
+                                         frame->width, frame->height, SDL_WINDOW_SHOWN | SDL_WINDOW_RESIZABLE);
+        if (!sink->window) {
+            av_log(ctx, AV_LOG_ERROR, "SDL2 could not create window %s\n", SDL_GetError());
+            return -ENOMEM;
+        }
+    }
+
+    if (!sink->render) {
+        sink->render = SDL_CreateRenderer(sink->window, -1, 0);
+        if (!sink->render) {
+            av_log(ctx, AV_LOG_ERROR, "SDL2 could not create render %s\n", SDL_GetError());
+            return -ENOMEM;
+        }
+        SDL_SetHint(SDL_HINT_RENDER_SCALE_QUALITY, "linear");
+
+        SDL_GetRendererInfo(sink->render, &renderer_info);
+        av_log(ctx, AV_LOG_INFO, "sdl2 renderer name:%s\n", renderer_info.name);
+    }
+    
+    if (!sink->texture) {
+        sink->texture = SDL_CreateTexture(sink->render, SDL_PIXELFORMAT_IYUV,
+                                           SDL_TEXTUREACCESS_STREAMING, frame->width, frame->height);
+        if (!sink->texture) {
+            av_log(ctx, AV_LOG_ERROR, "create texture failed %s\n", SDL_GetError());
+            return -EINVAL;
+        }
+    }
+
+    SDL_UpdateYUVTexture(sink->texture, NULL,
+                         frame->data[0], frame->linesize[0],
+                         frame->data[1], frame->linesize[1],
+                         frame->data[2], frame->linesize[2]);
+
+    SDL_RenderClear(sink->render);
+
+    SDL_RenderCopy(sink->render, sink->texture, NULL, NULL);
+    SDL_RenderPresent(sink->render);
+
+    av_frame_free(&frame);
+
+    return 0;
+}
+
+static const AVFilterPad avfilter_vsink_sdlvsink_inputs[] = {
+    {
+        .name        = "default",
+        .type        = AVMEDIA_TYPE_VIDEO,
+        .filter_frame = filter_frame,
+    },
+};
+
+static const AVFilterPad avfilter_vsink_sdlvsink_outputs[] = {
+    {
+        .name = NULL,
+        .type = AVMEDIA_TYPE_VIDEO,
+    },
+};
+
+const AVFilter ff_vsink_sdlvsink = {
+    .name        = "sdlvsink",
+    .description = NULL_IF_CONFIG_SMALL("Do absolutely nothing with the input video."),
+    .priv_size   = sizeof(SDLVideoContext),
+    .init        = init,
+    .uninit      = uninit,
+    FILTER_INPUTS(avfilter_vsink_sdlvsink_inputs),
+    FILTER_OUTPUTS(avfilter_vsink_sdlvsink_outputs),
+};
-- 
2.34.1

_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".

Re: [FFmpeg-devel] [PATCH] avfilter: add sdlvsink for video display

[Zhao Zhili]

> On Jun 6, 2024, at 19:51, Shiqi Zhu <hiccupzhu@gmail.com> wrote:

I’m afraid this has the same issue as libavdevice/sdl[1].

[1] https://patchwork.ffmpeg.org/project/ffmpeg/patch/20230918063728.198377-1-haihao.xiang@intel.com/
 
> 
> Signed-off-by: Shiqi Zhu <hiccupzhu@gmail.com>
> ---
> configure                    |   1 +
> libavfilter/Makefile         |   1 +
> libavfilter/allfilters.c     |   1 +
> libavfilter/vsink_sdlvsink.c | 142 +++++++++++++++++++++++++++++++++++
> 4 files changed, 145 insertions(+)
> create mode 100644 libavfilter/vsink_sdlvsink.c
> 
> diff --git a/configure b/configure
> index 6c5b8aab9a..968b5c8912 100755
> --- a/configure
> +++ b/configure
> @@ -3977,6 +3977,7 @@ xstack_qsv_filter_deps="libmfx"
> xstack_qsv_filter_select="qsvvpp"
> pad_vaapi_filter_deps="vaapi_1"
> drawbox_vaapi_filter_deps="vaapi_1"
> +sdlvsink_filter_deps="sdl2"
> 
> # examples
> avio_http_serve_files_deps="avformat avutil fork"
> diff --git a/libavfilter/Makefile b/libavfilter/Makefile
> index 5992fd161f..feac6b464d 100644
> --- a/libavfilter/Makefile
> +++ b/libavfilter/Makefile
> @@ -616,6 +616,7 @@ OBJS-$(CONFIG_YUVTESTSRC_FILTER)             += vsrc_testsrc.o
> OBJS-$(CONFIG_ZONEPLATE_FILTER)              += vsrc_testsrc.o
> 
> OBJS-$(CONFIG_NULLSINK_FILTER)               += vsink_nullsink.o
> +OBJS-$(CONFIG_SDLVSINK_FILTER)               += vsink_sdlvsink.o
> 
> # multimedia filters
> OBJS-$(CONFIG_A3DSCOPE_FILTER)               += avf_a3dscope.o
> diff --git a/libavfilter/allfilters.c b/libavfilter/allfilters.c
> index c532682fc2..d5f942ffb5 100644
> --- a/libavfilter/allfilters.c
> +++ b/libavfilter/allfilters.c
> @@ -579,6 +579,7 @@ extern const AVFilter ff_vsrc_yuvtestsrc;
> extern const AVFilter ff_vsrc_zoneplate;
> 
> extern const AVFilter ff_vsink_nullsink;
> +extern const AVFilter ff_vsink_sdlvsink;
> 
> /* multimedia filters */
> extern const AVFilter ff_avf_a3dscope;
> diff --git a/libavfilter/vsink_sdlvsink.c b/libavfilter/vsink_sdlvsink.c
> new file mode 100644
> index 0000000000..630f719c7e
> --- /dev/null
> +++ b/libavfilter/vsink_sdlvsink.c
> @@ -0,0 +1,142 @@
> +/*
> + * This file is part of FFmpeg.
> + *
> + * FFmpeg is free software; you can redistribute it and/or
> + * modify it under the terms of the GNU Lesser General Public
> + * License as published by the Free Software Foundation; either
> + * version 2.1 of the License, or (at your option) any later version.
> + *
> + * FFmpeg is distributed in the hope that it will be useful,
> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
> + * Lesser General Public License for more details.
> + *
> + * You should have received a copy of the GNU Lesser General Public
> + * License along with FFmpeg; if not, write to the Free Software
> + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
> + */
> +
> +#include "avfilter.h"
> +#include "internal.h"
> +#include "libavutil/internal.h"
> +#include <SDL2/SDL.h>
> +#include <unistd.h>
> +
> +typedef struct {
> +    SDL_Window   *window;
> +    SDL_Renderer *render;
> +    SDL_Texture  *texture;
> +} SDLVideoContext;
> +
> +static int init(AVFilterContext *ctx)
> +{
> +    if (SDL_Init(SDL_INIT_VIDEO) < 0)
> +    {
> +        av_log(ctx, AV_LOG_ERROR, "SDL2 could not initialize! %s\n", SDL_GetError());
> +        return -ENOMEM;
> +    }
> +
> +    SDL_PollEvent(NULL);
> +    SDL_SetHint(SDL_HINT_VIDEO_X11_NET_WM_BYPASS_COMPOSITOR, "0");
> +
> +    return 0;
> +}
> +
> +static void uninit(AVFilterContext *ctx)
> +{
> +    SDLVideoContext *sink = ctx->priv;
> +    
> +    if (sink->texture) {
> +        SDL_DestroyTexture(sink->texture);
> +        sink->texture = NULL;
> +    }
> +
> +    if (sink->render) {
> +        SDL_DestroyRenderer(sink->render);
> +        sink->render = NULL;
> +    }
> +
> +    if (sink->window) {
> +        SDL_DestroyWindow(sink->window);
> +        sink->window = NULL;
> +    }
> +
> +    SDL_Quit();
> +}
> +
> +static int filter_frame(AVFilterLink *link, AVFrame *frame)
> +{
> +    AVFilterContext *ctx = link->dst;
> +    SDLVideoContext *sink = ctx->priv;
> +    SDL_RendererInfo renderer_info;
> +
> +    if (!sink->window) {
> +        sink->window = SDL_CreateWindow("YUV Player",
> +                                         SDL_WINDOWPOS_UNDEFINED, SDL_WINDOWPOS_UNDEFINED,
> +                                         frame->width, frame->height, SDL_WINDOW_SHOWN | SDL_WINDOW_RESIZABLE);
> +        if (!sink->window) {
> +            av_log(ctx, AV_LOG_ERROR, "SDL2 could not create window %s\n", SDL_GetError());
> +            return -ENOMEM;
> +        }
> +    }
> +
> +    if (!sink->render) {
> +        sink->render = SDL_CreateRenderer(sink->window, -1, 0);
> +        if (!sink->render) {
> +            av_log(ctx, AV_LOG_ERROR, "SDL2 could not create render %s\n", SDL_GetError());
> +            return -ENOMEM;
> +        }
> +        SDL_SetHint(SDL_HINT_RENDER_SCALE_QUALITY, "linear");
> +
> +        SDL_GetRendererInfo(sink->render, &renderer_info);
> +        av_log(ctx, AV_LOG_INFO, "sdl2 renderer name:%s\n", renderer_info.name);
> +    }
> +    
> +    if (!sink->texture) {
> +        sink->texture = SDL_CreateTexture(sink->render, SDL_PIXELFORMAT_IYUV,
> +                                           SDL_TEXTUREACCESS_STREAMING, frame->width, frame->height);
> +        if (!sink->texture) {
> +            av_log(ctx, AV_LOG_ERROR, "create texture failed %s\n", SDL_GetError());
> +            return -EINVAL;
> +        }
> +    }
> +
> +    SDL_UpdateYUVTexture(sink->texture, NULL,
> +                         frame->data[0], frame->linesize[0],
> +                         frame->data[1], frame->linesize[1],
> +                         frame->data[2], frame->linesize[2]);
> +
> +    SDL_RenderClear(sink->render);
> +
> +    SDL_RenderCopy(sink->render, sink->texture, NULL, NULL);
> +    SDL_RenderPresent(sink->render);
> +
> +    av_frame_free(&frame);
> +
> +    return 0;
> +}
> +
> +static const AVFilterPad avfilter_vsink_sdlvsink_inputs[] = {
> +    {
> +        .name        = "default",
> +        .type        = AVMEDIA_TYPE_VIDEO,
> +        .filter_frame = filter_frame,
> +    },
> +};
> +
> +static const AVFilterPad avfilter_vsink_sdlvsink_outputs[] = {
> +    {
> +        .name = NULL,
> +        .type = AVMEDIA_TYPE_VIDEO,
> +    },
> +};
> +
> +const AVFilter ff_vsink_sdlvsink = {
> +    .name        = "sdlvsink",
> +    .description = NULL_IF_CONFIG_SMALL("Do absolutely nothing with the input video."),
> +    .priv_size   = sizeof(SDLVideoContext),
> +    .init        = init,
> +    .uninit      = uninit,
> +    FILTER_INPUTS(avfilter_vsink_sdlvsink_inputs),
> +    FILTER_OUTPUTS(avfilter_vsink_sdlvsink_outputs),
> +};
> -- 
> 2.34.1
> 
> _______________________________________________
> ffmpeg-devel mailing list
> ffmpeg-devel@ffmpeg.org
> https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
> 
> To unsubscribe, visit link above, or email
> ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".

_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".

Re: [FFmpeg-devel] [PATCH] avfilter: add sdlvsink for video display

[Hiccup Zhu]

On Thu, 6 Jun 2024 at 20:20, Zhao Zhili <quinkblack@foxmail.com> wrote:
>
>
> On Jun 6, 2024, at 19:51, Shiqi Zhu <hiccupzhu@gmail.com> wrote:
>
>
> I’m afraid this has the same issue as libavdevice/sdl[1].
>
> [1] https://patchwork.ffmpeg.org/project/ffmpeg/patch/20230918063728.198377-1-haihao.xiang@intel.com/
>

Thank you for the reminder. This patch does not have that issue. All
SDL initialization is within the filter_frame() function, so there's
no problem with different threads being created.

Furthermore, my expertise lies primarily in multimedia framework
development. I've noticed that the libavfilter now offers a
comprehensive basic framework, providing various types of filters such
as src, trans, and sink within the filter-graph.

There are already many src and trans type filters in the existing
codebase, with rich functionalities. However, there are only two sink
filters: buffersink and nullsink, which is not quite sufficient for
the development of filter-graphs.

Thus, I submitted this patch with the intention of enriching the types
of sink filters available.

Many thanks for your valuable comments.

>
> Signed-off-by: Shiqi Zhu <hiccupzhu@gmail.com>
> ---
> configure                    |   1 +
> libavfilter/Makefile         |   1 +
> libavfilter/allfilters.c     |   1 +
> libavfilter/vsink_sdlvsink.c | 142 +++++++++++++++++++++++++++++++++++
> 4 files changed, 145 insertions(+)
> create mode 100644 libavfilter/vsink_sdlvsink.c
>
> diff --git a/configure b/configure
> index 6c5b8aab9a..968b5c8912 100755
> --- a/configure
> +++ b/configure
> @@ -3977,6 +3977,7 @@ xstack_qsv_filter_deps="libmfx"
> xstack_qsv_filter_select="qsvvpp"
> pad_vaapi_filter_deps="vaapi_1"
> drawbox_vaapi_filter_deps="vaapi_1"
> +sdlvsink_filter_deps="sdl2"
>
> # examples
> avio_http_serve_files_deps="avformat avutil fork"
> diff --git a/libavfilter/Makefile b/libavfilter/Makefile
> index 5992fd161f..feac6b464d 100644
> --- a/libavfilter/Makefile
> +++ b/libavfilter/Makefile
> @@ -616,6 +616,7 @@ OBJS-$(CONFIG_YUVTESTSRC_FILTER)             += vsrc_testsrc.o
> OBJS-$(CONFIG_ZONEPLATE_FILTER)              += vsrc_testsrc.o
>
> OBJS-$(CONFIG_NULLSINK_FILTER)               += vsink_nullsink.o
> +OBJS-$(CONFIG_SDLVSINK_FILTER)               += vsink_sdlvsink.o
>
> # multimedia filters
> OBJS-$(CONFIG_A3DSCOPE_FILTER)               += avf_a3dscope.o
> diff --git a/libavfilter/allfilters.c b/libavfilter/allfilters.c
> index c532682fc2..d5f942ffb5 100644
> --- a/libavfilter/allfilters.c
> +++ b/libavfilter/allfilters.c
> @@ -579,6 +579,7 @@ extern const AVFilter ff_vsrc_yuvtestsrc;
> extern const AVFilter ff_vsrc_zoneplate;
>
> extern const AVFilter ff_vsink_nullsink;
> +extern const AVFilter ff_vsink_sdlvsink;
>
> /* multimedia filters */
> extern const AVFilter ff_avf_a3dscope;
> diff --git a/libavfilter/vsink_sdlvsink.c b/libavfilter/vsink_sdlvsink.c
> new file mode 100644
> index 0000000000..630f719c7e
> --- /dev/null
> +++ b/libavfilter/vsink_sdlvsink.c
> @@ -0,0 +1,142 @@
> +/*
> + * This file is part of FFmpeg.
> + *
> + * FFmpeg is free software; you can redistribute it and/or
> + * modify it under the terms of the GNU Lesser General Public
> + * License as published by the Free Software Foundation; either
> + * version 2.1 of the License, or (at your option) any later version.
> + *
> + * FFmpeg is distributed in the hope that it will be useful,
> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
> + * Lesser General Public License for more details.
> + *
> + * You should have received a copy of the GNU Lesser General Public
> + * License along with FFmpeg; if not, write to the Free Software
> + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
> + */
> +
> +#include "avfilter.h"
> +#include "internal.h"
> +#include "libavutil/internal.h"
> +#include <SDL2/SDL.h>
> +#include <unistd.h>
> +
> +typedef struct {
> +    SDL_Window   *window;
> +    SDL_Renderer *render;
> +    SDL_Texture  *texture;
> +} SDLVideoContext;
> +
> +static int init(AVFilterContext *ctx)
> +{
> +    if (SDL_Init(SDL_INIT_VIDEO) < 0)
> +    {
> +        av_log(ctx, AV_LOG_ERROR, "SDL2 could not initialize! %s\n", SDL_GetError());
> +        return -ENOMEM;
> +    }
> +
> +    SDL_PollEvent(NULL);
> +    SDL_SetHint(SDL_HINT_VIDEO_X11_NET_WM_BYPASS_COMPOSITOR, "0");
> +
> +    return 0;
> +}
> +
> +static void uninit(AVFilterContext *ctx)
> +{
> +    SDLVideoContext *sink = ctx->priv;
> +
> +    if (sink->texture) {
> +        SDL_DestroyTexture(sink->texture);
> +        sink->texture = NULL;
> +    }
> +
> +    if (sink->render) {
> +        SDL_DestroyRenderer(sink->render);
> +        sink->render = NULL;
> +    }
> +
> +    if (sink->window) {
> +        SDL_DestroyWindow(sink->window);
> +        sink->window = NULL;
> +    }
> +
> +    SDL_Quit();
> +}
> +
> +static int filter_frame(AVFilterLink *link, AVFrame *frame)
> +{
> +    AVFilterContext *ctx = link->dst;
> +    SDLVideoContext *sink = ctx->priv;
> +    SDL_RendererInfo renderer_info;
> +
> +    if (!sink->window) {
> +        sink->window = SDL_CreateWindow("YUV Player",
> +                                         SDL_WINDOWPOS_UNDEFINED, SDL_WINDOWPOS_UNDEFINED,
> +                                         frame->width, frame->height, SDL_WINDOW_SHOWN | SDL_WINDOW_RESIZABLE);
> +        if (!sink->window) {
> +            av_log(ctx, AV_LOG_ERROR, "SDL2 could not create window %s\n", SDL_GetError());
> +            return -ENOMEM;
> +        }
> +    }
> +
> +    if (!sink->render) {
> +        sink->render = SDL_CreateRenderer(sink->window, -1, 0);
> +        if (!sink->render) {
> +            av_log(ctx, AV_LOG_ERROR, "SDL2 could not create render %s\n", SDL_GetError());
> +            return -ENOMEM;
> +        }
> +        SDL_SetHint(SDL_HINT_RENDER_SCALE_QUALITY, "linear");
> +
> +        SDL_GetRendererInfo(sink->render, &renderer_info);
> +        av_log(ctx, AV_LOG_INFO, "sdl2 renderer name:%s\n", renderer_info.name);
> +    }
> +
> +    if (!sink->texture) {
> +        sink->texture = SDL_CreateTexture(sink->render, SDL_PIXELFORMAT_IYUV,
> +                                           SDL_TEXTUREACCESS_STREAMING, frame->width, frame->height);
> +        if (!sink->texture) {
> +            av_log(ctx, AV_LOG_ERROR, "create texture failed %s\n", SDL_GetError());
> +            return -EINVAL;
> +        }
> +    }
> +
> +    SDL_UpdateYUVTexture(sink->texture, NULL,
> +                         frame->data[0], frame->linesize[0],
> +                         frame->data[1], frame->linesize[1],
> +                         frame->data[2], frame->linesize[2]);
> +
> +    SDL_RenderClear(sink->render);
> +
> +    SDL_RenderCopy(sink->render, sink->texture, NULL, NULL);
> +    SDL_RenderPresent(sink->render);
> +
> +    av_frame_free(&frame);
> +
> +    return 0;
> +}
> +
> +static const AVFilterPad avfilter_vsink_sdlvsink_inputs[] = {
> +    {
> +        .name        = "default",
> +        .type        = AVMEDIA_TYPE_VIDEO,
> +        .filter_frame = filter_frame,
> +    },
> +};
> +
> +static const AVFilterPad avfilter_vsink_sdlvsink_outputs[] = {
> +    {
> +        .name = NULL,
> +        .type = AVMEDIA_TYPE_VIDEO,
> +    },
> +};
> +
> +const AVFilter ff_vsink_sdlvsink = {
> +    .name        = "sdlvsink",
> +    .description = NULL_IF_CONFIG_SMALL("Do absolutely nothing with the input video."),
> +    .priv_size   = sizeof(SDLVideoContext),
> +    .init        = init,
> +    .uninit      = uninit,
> +    FILTER_INPUTS(avfilter_vsink_sdlvsink_inputs),
> +    FILTER_OUTPUTS(avfilter_vsink_sdlvsink_outputs),
> +};
> --
> 2.34.1
>
> _______________________________________________
> ffmpeg-devel mailing list
> ffmpeg-devel@ffmpeg.org
> https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
>
> To unsubscribe, visit link above, or email
> ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
>
>
_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".

Re: [FFmpeg-devel] [PATCH] avfilter: add sdlvsink for video display

[Zhao Zhili]

> On Jun 7, 2024, at 10:12, Hiccup Zhu <hiccupzhu@gmail.com> wrote:
> 
> On Thu, 6 Jun 2024 at 20:20, Zhao Zhili <quinkblack@foxmail.com> wrote:
>> 
>> 
>> On Jun 6, 2024, at 19:51, Shiqi Zhu <hiccupzhu@gmail.com> wrote:
>> 
>> 
>> I’m afraid this has the same issue as libavdevice/sdl[1].
>> 
>> [1] https://patchwork.ffmpeg.org/project/ffmpeg/patch/20230918063728.198377-1-haihao.xiang@intel.com/
>> 
> 
> Thank you for the reminder. This patch does not have that issue. All
> SDL initialization is within the filter_frame() function, so there's
> no problem with different threads being created.

Initialization and render in the same thread is one of the issues. Another
issue is SDL render should be run in main thread. Maybe it’s not the
duty of this filter to control which thread to be called, but this wouldn’t
work with ffmpeg cmd.

> 
> Furthermore, my expertise lies primarily in multimedia framework
> development. I've noticed that the libavfilter now offers a
> comprehensive basic framework, providing various types of filters such
> as src, trans, and sink within the filter-graph.

libavdevice/sdl should work for this case, although it’s deprecated,
and it support more pixel formats than current patch.

TBH, a videosink in libavfilter make sense than libavdevice. But with
the reasons mentioned earlier, I don’t know how to make progress.

> 
> There are already many src and trans type filters in the existing
> codebase, with rich functionalities. However, there are only two sink
> filters: buffersink and nullsink, which is not quite sufficient for
> the development of filter-graphs.
> 
> Thus, I submitted this patch with the intention of enriching the types
> of sink filters available.
> 
> Many thanks for your valuable comments.
> 
>> 
>> Signed-off-by: Shiqi Zhu <hiccupzhu@gmail.com>
>> ---
>> configure                    |   1 +
>> libavfilter/Makefile         |   1 +
>> libavfilter/allfilters.c     |   1 +
>> libavfilter/vsink_sdlvsink.c | 142 +++++++++++++++++++++++++++++++++++
>> 4 files changed, 145 insertions(+)
>> create mode 100644 libavfilter/vsink_sdlvsink.c
>> 
>> diff --git a/configure b/configure
>> index 6c5b8aab9a..968b5c8912 100755
>> --- a/configure
>> +++ b/configure
>> @@ -3977,6 +3977,7 @@ xstack_qsv_filter_deps="libmfx"
>> xstack_qsv_filter_select="qsvvpp"
>> pad_vaapi_filter_deps="vaapi_1"
>> drawbox_vaapi_filter_deps="vaapi_1"
>> +sdlvsink_filter_deps="sdl2"
>> 
>> # examples
>> avio_http_serve_files_deps="avformat avutil fork"
>> diff --git a/libavfilter/Makefile b/libavfilter/Makefile
>> index 5992fd161f..feac6b464d 100644
>> --- a/libavfilter/Makefile
>> +++ b/libavfilter/Makefile
>> @@ -616,6 +616,7 @@ OBJS-$(CONFIG_YUVTESTSRC_FILTER)             += vsrc_testsrc.o
>> OBJS-$(CONFIG_ZONEPLATE_FILTER)              += vsrc_testsrc.o
>> 
>> OBJS-$(CONFIG_NULLSINK_FILTER)               += vsink_nullsink.o
>> +OBJS-$(CONFIG_SDLVSINK_FILTER)               += vsink_sdlvsink.o
>> 
>> # multimedia filters
>> OBJS-$(CONFIG_A3DSCOPE_FILTER)               += avf_a3dscope.o
>> diff --git a/libavfilter/allfilters.c b/libavfilter/allfilters.c
>> index c532682fc2..d5f942ffb5 100644
>> --- a/libavfilter/allfilters.c
>> +++ b/libavfilter/allfilters.c
>> @@ -579,6 +579,7 @@ extern const AVFilter ff_vsrc_yuvtestsrc;
>> extern const AVFilter ff_vsrc_zoneplate;
>> 
>> extern const AVFilter ff_vsink_nullsink;
>> +extern const AVFilter ff_vsink_sdlvsink;
>> 
>> /* multimedia filters */
>> extern const AVFilter ff_avf_a3dscope;
>> diff --git a/libavfilter/vsink_sdlvsink.c b/libavfilter/vsink_sdlvsink.c
>> new file mode 100644
>> index 0000000000..630f719c7e
>> --- /dev/null
>> +++ b/libavfilter/vsink_sdlvsink.c
>> @@ -0,0 +1,142 @@
>> +/*
>> + * This file is part of FFmpeg.
>> + *
>> + * FFmpeg is free software; you can redistribute it and/or
>> + * modify it under the terms of the GNU Lesser General Public
>> + * License as published by the Free Software Foundation; either
>> + * version 2.1 of the License, or (at your option) any later version.
>> + *
>> + * FFmpeg is distributed in the hope that it will be useful,
>> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
>> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
>> + * Lesser General Public License for more details.
>> + *
>> + * You should have received a copy of the GNU Lesser General Public
>> + * License along with FFmpeg; if not, write to the Free Software
>> + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
>> + */
>> +
>> +#include "avfilter.h"
>> +#include "internal.h"
>> +#include "libavutil/internal.h"
>> +#include <SDL2/SDL.h>
>> +#include <unistd.h>
>> +
>> +typedef struct {
>> +    SDL_Window   *window;
>> +    SDL_Renderer *render;
>> +    SDL_Texture  *texture;
>> +} SDLVideoContext;
>> +
>> +static int init(AVFilterContext *ctx)
>> +{
>> +    if (SDL_Init(SDL_INIT_VIDEO) < 0)
>> +    {
>> +        av_log(ctx, AV_LOG_ERROR, "SDL2 could not initialize! %s\n", SDL_GetError());
>> +        return -ENOMEM;
>> +    }
>> +
>> +    SDL_PollEvent(NULL);
>> +    SDL_SetHint(SDL_HINT_VIDEO_X11_NET_WM_BYPASS_COMPOSITOR, "0");
>> +
>> +    return 0;
>> +}
>> +
>> +static void uninit(AVFilterContext *ctx)
>> +{
>> +    SDLVideoContext *sink = ctx->priv;
>> +
>> +    if (sink->texture) {
>> +        SDL_DestroyTexture(sink->texture);
>> +        sink->texture = NULL;
>> +    }
>> +
>> +    if (sink->render) {
>> +        SDL_DestroyRenderer(sink->render);
>> +        sink->render = NULL;
>> +    }
>> +
>> +    if (sink->window) {
>> +        SDL_DestroyWindow(sink->window);
>> +        sink->window = NULL;
>> +    }
>> +
>> +    SDL_Quit();
>> +}
>> +
>> +static int filter_frame(AVFilterLink *link, AVFrame *frame)
>> +{
>> +    AVFilterContext *ctx = link->dst;
>> +    SDLVideoContext *sink = ctx->priv;
>> +    SDL_RendererInfo renderer_info;
>> +
>> +    if (!sink->window) {
>> +        sink->window = SDL_CreateWindow("YUV Player",
>> +                                         SDL_WINDOWPOS_UNDEFINED, SDL_WINDOWPOS_UNDEFINED,
>> +                                         frame->width, frame->height, SDL_WINDOW_SHOWN | SDL_WINDOW_RESIZABLE);
>> +        if (!sink->window) {
>> +            av_log(ctx, AV_LOG_ERROR, "SDL2 could not create window %s\n", SDL_GetError());
>> +            return -ENOMEM;
>> +        }
>> +    }
>> +
>> +    if (!sink->render) {
>> +        sink->render = SDL_CreateRenderer(sink->window, -1, 0);
>> +        if (!sink->render) {
>> +            av_log(ctx, AV_LOG_ERROR, "SDL2 could not create render %s\n", SDL_GetError());
>> +            return -ENOMEM;
>> +        }
>> +        SDL_SetHint(SDL_HINT_RENDER_SCALE_QUALITY, "linear");
>> +
>> +        SDL_GetRendererInfo(sink->render, &renderer_info);
>> +        av_log(ctx, AV_LOG_INFO, "sdl2 renderer name:%s\n", renderer_info.name);
>> +    }
>> +
>> +    if (!sink->texture) {
>> +        sink->texture = SDL_CreateTexture(sink->render, SDL_PIXELFORMAT_IYUV,
>> +                                           SDL_TEXTUREACCESS_STREAMING, frame->width, frame->height);
>> +        if (!sink->texture) {
>> +            av_log(ctx, AV_LOG_ERROR, "create texture failed %s\n", SDL_GetError());
>> +            return -EINVAL;
>> +        }
>> +    }
>> +
>> +    SDL_UpdateYUVTexture(sink->texture, NULL,
>> +                         frame->data[0], frame->linesize[0],
>> +                         frame->data[1], frame->linesize[1],
>> +                         frame->data[2], frame->linesize[2]);
>> +
>> +    SDL_RenderClear(sink->render);
>> +
>> +    SDL_RenderCopy(sink->render, sink->texture, NULL, NULL);
>> +    SDL_RenderPresent(sink->render);
>> +
>> +    av_frame_free(&frame);
>> +
>> +    return 0;
>> +}
>> +
>> +static const AVFilterPad avfilter_vsink_sdlvsink_inputs[] = {
>> +    {
>> +        .name        = "default",
>> +        .type        = AVMEDIA_TYPE_VIDEO,
>> +        .filter_frame = filter_frame,
>> +    },
>> +};
>> +
>> +static const AVFilterPad avfilter_vsink_sdlvsink_outputs[] = {
>> +    {
>> +        .name = NULL,
>> +        .type = AVMEDIA_TYPE_VIDEO,
>> +    },
>> +};
>> +
>> +const AVFilter ff_vsink_sdlvsink = {
>> +    .name        = "sdlvsink",
>> +    .description = NULL_IF_CONFIG_SMALL("Do absolutely nothing with the input video."),
>> +    .priv_size   = sizeof(SDLVideoContext),
>> +    .init        = init,
>> +    .uninit      = uninit,
>> +    FILTER_INPUTS(avfilter_vsink_sdlvsink_inputs),
>> +    FILTER_OUTPUTS(avfilter_vsink_sdlvsink_outputs),
>> +};
>> --
>> 2.34.1
>> 
>> _______________________________________________
>> ffmpeg-devel mailing list
>> ffmpeg-devel@ffmpeg.org
>> https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
>> 
>> To unsubscribe, visit link above, or email
>> ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
>> 
>> 

_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".

Re: [FFmpeg-devel] [PATCH] avfilter: add sdlvsink for video display

[Shiqi Zhu]

On Fri, 7 Jun 2024 at 10:32, Zhao Zhili <quinkblack@foxmail.com> wrote:
>
>
>
> > On Jun 7, 2024, at 10:12, Hiccup Zhu <hiccupzhu@gmail.com> wrote:
> >
> > On Thu, 6 Jun 2024 at 20:20, Zhao Zhili <quinkblack@foxmail.com> wrote:
> >>
> >>
> >> On Jun 6, 2024, at 19:51, Shiqi Zhu <hiccupzhu@gmail.com> wrote:
> >>
> >>
> >> I’m afraid this has the same issue as libavdevice/sdl[1].
> >>
> >> [1] https://patchwork.ffmpeg.org/project/ffmpeg/patch/20230918063728.198377-1-haihao.xiang@intel.com/
> >>
> >
> > Thank you for the reminder. This patch does not have that issue. All
> > SDL initialization is within the filter_frame() function, so there's
> > no problem with different threads being created.
>
> Initialization and render in the same thread is one of the issues. Another
> issue is SDL render should be run in main thread. Maybe it’s not the
> duty of this filter to control which thread to be called, but this wouldn’t
> work with ffmpeg cmd.
>
Yes, I agree with your opinion. The control of the filter should be
left to the application, and the ffmpeg command should only serve the
purpose of demonstrating functionality.

> >
> > Furthermore, my expertise lies primarily in multimedia framework
> > development. I've noticed that the libavfilter now offers a
> > comprehensive basic framework, providing various types of filters such
> > as src, trans, and sink within the filter-graph.
>
> libavdevice/sdl should work for this case, although it’s deprecated,
> and it support more pixel formats than current patch.
>
Of course, in the next patch, I will expand the pixel format, provided
that this sink-filter is meaningful for the entire project and can be
merged into the master branch.

> TBH, a videosink in libavfilter make sense than libavdevice. But with
> the reasons mentioned earlier, I don’t know how to make progress.
>
Thanks you for your comments. I believe there are many experienced
engineers in the community with whom we can collectively discuss this
issue and then push the patch forward together.

> >
> > There are already many src and trans type filters in the existing
> > codebase, with rich functionalities. However, there are only two sink
> > filters: buffersink and nullsink, which is not quite sufficient for
> > the development of filter-graphs.
> >
> > Thus, I submitted this patch with the intention of enriching the types
> > of sink filters available.
> >
> > Many thanks for your valuable comments.
> >
> >>
> >> Signed-off-by: Shiqi Zhu <hiccupzhu@gmail.com>
> >> ---
> >> configure                    |   1 +
> >> libavfilter/Makefile         |   1 +
> >> libavfilter/allfilters.c     |   1 +
> >> libavfilter/vsink_sdlvsink.c | 142 +++++++++++++++++++++++++++++++++++
> >> 4 files changed, 145 insertions(+)
> >> create mode 100644 libavfilter/vsink_sdlvsink.c
> >>
> >> diff --git a/configure b/configure
> >> index 6c5b8aab9a..968b5c8912 100755
> >> --- a/configure
> >> +++ b/configure
> >> @@ -3977,6 +3977,7 @@ xstack_qsv_filter_deps="libmfx"
> >> xstack_qsv_filter_select="qsvvpp"
> >> pad_vaapi_filter_deps="vaapi_1"
> >> drawbox_vaapi_filter_deps="vaapi_1"
> >> +sdlvsink_filter_deps="sdl2"
> >>
> >> # examples
> >> avio_http_serve_files_deps="avformat avutil fork"
> >> diff --git a/libavfilter/Makefile b/libavfilter/Makefile
> >> index 5992fd161f..feac6b464d 100644
> >> --- a/libavfilter/Makefile
> >> +++ b/libavfilter/Makefile
> >> @@ -616,6 +616,7 @@ OBJS-$(CONFIG_YUVTESTSRC_FILTER)             += vsrc_testsrc.o
> >> OBJS-$(CONFIG_ZONEPLATE_FILTER)              += vsrc_testsrc.o
> >>
> >> OBJS-$(CONFIG_NULLSINK_FILTER)               += vsink_nullsink.o
> >> +OBJS-$(CONFIG_SDLVSINK_FILTER)               += vsink_sdlvsink.o
> >>
> >> # multimedia filters
> >> OBJS-$(CONFIG_A3DSCOPE_FILTER)               += avf_a3dscope.o
> >> diff --git a/libavfilter/allfilters.c b/libavfilter/allfilters.c
> >> index c532682fc2..d5f942ffb5 100644
> >> --- a/libavfilter/allfilters.c
> >> +++ b/libavfilter/allfilters.c
> >> @@ -579,6 +579,7 @@ extern const AVFilter ff_vsrc_yuvtestsrc;
> >> extern const AVFilter ff_vsrc_zoneplate;
> >>
> >> extern const AVFilter ff_vsink_nullsink;
> >> +extern const AVFilter ff_vsink_sdlvsink;
> >>
> >> /* multimedia filters */
> >> extern const AVFilter ff_avf_a3dscope;
> >> diff --git a/libavfilter/vsink_sdlvsink.c b/libavfilter/vsink_sdlvsink.c
> >> new file mode 100644
> >> index 0000000000..630f719c7e
> >> --- /dev/null
> >> +++ b/libavfilter/vsink_sdlvsink.c
> >> @@ -0,0 +1,142 @@
> >> +/*
> >> + * This file is part of FFmpeg.
> >> + *
> >> + * FFmpeg is free software; you can redistribute it and/or
> >> + * modify it under the terms of the GNU Lesser General Public
> >> + * License as published by the Free Software Foundation; either
> >> + * version 2.1 of the License, or (at your option) any later version.
> >> + *
> >> + * FFmpeg is distributed in the hope that it will be useful,
> >> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> >> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
> >> + * Lesser General Public License for more details.
> >> + *
> >> + * You should have received a copy of the GNU Lesser General Public
> >> + * License along with FFmpeg; if not, write to the Free Software
> >> + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
> >> + */
> >> +
> >> +#include "avfilter.h"
> >> +#include "internal.h"
> >> +#include "libavutil/internal.h"
> >> +#include <SDL2/SDL.h>
> >> +#include <unistd.h>
> >> +
> >> +typedef struct {
> >> +    SDL_Window   *window;
> >> +    SDL_Renderer *render;
> >> +    SDL_Texture  *texture;
> >> +} SDLVideoContext;
> >> +
> >> +static int init(AVFilterContext *ctx)
> >> +{
> >> +    if (SDL_Init(SDL_INIT_VIDEO) < 0)
> >> +    {
> >> +        av_log(ctx, AV_LOG_ERROR, "SDL2 could not initialize! %s\n", SDL_GetError());
> >> +        return -ENOMEM;
> >> +    }
> >> +
> >> +    SDL_PollEvent(NULL);
> >> +    SDL_SetHint(SDL_HINT_VIDEO_X11_NET_WM_BYPASS_COMPOSITOR, "0");
> >> +
> >> +    return 0;
> >> +}
> >> +
> >> +static void uninit(AVFilterContext *ctx)
> >> +{
> >> +    SDLVideoContext *sink = ctx->priv;
> >> +
> >> +    if (sink->texture) {
> >> +        SDL_DestroyTexture(sink->texture);
> >> +        sink->texture = NULL;
> >> +    }
> >> +
> >> +    if (sink->render) {
> >> +        SDL_DestroyRenderer(sink->render);
> >> +        sink->render = NULL;
> >> +    }
> >> +
> >> +    if (sink->window) {
> >> +        SDL_DestroyWindow(sink->window);
> >> +        sink->window = NULL;
> >> +    }
> >> +
> >> +    SDL_Quit();
> >> +}
> >> +
> >> +static int filter_frame(AVFilterLink *link, AVFrame *frame)
> >> +{
> >> +    AVFilterContext *ctx = link->dst;
> >> +    SDLVideoContext *sink = ctx->priv;
> >> +    SDL_RendererInfo renderer_info;
> >> +
> >> +    if (!sink->window) {
> >> +        sink->window = SDL_CreateWindow("YUV Player",
> >> +                                         SDL_WINDOWPOS_UNDEFINED, SDL_WINDOWPOS_UNDEFINED,
> >> +                                         frame->width, frame->height, SDL_WINDOW_SHOWN | SDL_WINDOW_RESIZABLE);
> >> +        if (!sink->window) {
> >> +            av_log(ctx, AV_LOG_ERROR, "SDL2 could not create window %s\n", SDL_GetError());
> >> +            return -ENOMEM;
> >> +        }
> >> +    }
> >> +
> >> +    if (!sink->render) {
> >> +        sink->render = SDL_CreateRenderer(sink->window, -1, 0);
> >> +        if (!sink->render) {
> >> +            av_log(ctx, AV_LOG_ERROR, "SDL2 could not create render %s\n", SDL_GetError());
> >> +            return -ENOMEM;
> >> +        }
> >> +        SDL_SetHint(SDL_HINT_RENDER_SCALE_QUALITY, "linear");
> >> +
> >> +        SDL_GetRendererInfo(sink->render, &renderer_info);
> >> +        av_log(ctx, AV_LOG_INFO, "sdl2 renderer name:%s\n", renderer_info.name);
> >> +    }
> >> +
> >> +    if (!sink->texture) {
> >> +        sink->texture = SDL_CreateTexture(sink->render, SDL_PIXELFORMAT_IYUV,
> >> +                                           SDL_TEXTUREACCESS_STREAMING, frame->width, frame->height);
> >> +        if (!sink->texture) {
> >> +            av_log(ctx, AV_LOG_ERROR, "create texture failed %s\n", SDL_GetError());
> >> +            return -EINVAL;
> >> +        }
> >> +    }
> >> +
> >> +    SDL_UpdateYUVTexture(sink->texture, NULL,
> >> +                         frame->data[0], frame->linesize[0],
> >> +                         frame->data[1], frame->linesize[1],
> >> +                         frame->data[2], frame->linesize[2]);
> >> +
> >> +    SDL_RenderClear(sink->render);
> >> +
> >> +    SDL_RenderCopy(sink->render, sink->texture, NULL, NULL);
> >> +    SDL_RenderPresent(sink->render);
> >> +
> >> +    av_frame_free(&frame);
> >> +
> >> +    return 0;
> >> +}
> >> +
> >> +static const AVFilterPad avfilter_vsink_sdlvsink_inputs[] = {
> >> +    {
> >> +        .name        = "default",
> >> +        .type        = AVMEDIA_TYPE_VIDEO,
> >> +        .filter_frame = filter_frame,
> >> +    },
> >> +};
> >> +
> >> +static const AVFilterPad avfilter_vsink_sdlvsink_outputs[] = {
> >> +    {
> >> +        .name = NULL,
> >> +        .type = AVMEDIA_TYPE_VIDEO,
> >> +    },
> >> +};
> >> +
> >> +const AVFilter ff_vsink_sdlvsink = {
> >> +    .name        = "sdlvsink",
> >> +    .description = NULL_IF_CONFIG_SMALL("Do absolutely nothing with the input video."),
> >> +    .priv_size   = sizeof(SDLVideoContext),
> >> +    .init        = init,
> >> +    .uninit      = uninit,
> >> +    FILTER_INPUTS(avfilter_vsink_sdlvsink_inputs),
> >> +    FILTER_OUTPUTS(avfilter_vsink_sdlvsink_outputs),
> >> +};
> >> --
> >> 2.34.1
> >>
> >> _______________________________________________
> >> ffmpeg-devel mailing list
> >> ffmpeg-devel@ffmpeg.org
> >> https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
> >>
> >> To unsubscribe, visit link above, or email
> >> ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
> >>
> >>
>
_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".

Re: [FFmpeg-devel] [PATCH] avfilter: add sdlvsink for video display

[Rémi Denis-Courmont]

Hi,

Le 7 juin 2024 06:33:20 GMT+03:00, Shiqi Zhu <hiccupzhu@gmail.com> a écrit :
>> Initialization and render in the same thread is one of the issues. Another
>> issue is SDL render should be run in main thread. Maybe it’s not the
>> duty of this filter to control which thread to be called, but this wouldn’t
>> work with ffmpeg cmd.
>>
>Yes, I agree with your opinion. The control of the filter should be
>left to the application, and the ffmpeg command should only serve the
>purpose of demonstrating functionality.

You can't require running on the main thread. There are no ifs or buts here; libavfilter simply can't require that from the calling application or upper-layer framework and therefore it also can't warrant it to SDL.

SDL is meant to be used by programs, notably games, not middleware thread-agnostic libraries like FFmpeg's libav*. I don't think that there is a point arguing *again*.
_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".

Re: [FFmpeg-devel] [PATCH] avfilter: add sdlvsink for video display

[Michael Niedermayer]

[-- Attachment #1.1: Type: text/plain, Size: 1304 bytes --]

On Fri, Jun 07, 2024 at 09:45:09AM +0300, Rémi Denis-Courmont wrote:
> Hi,
> 
> Le 7 juin 2024 06:33:20 GMT+03:00, Shiqi Zhu <hiccupzhu@gmail.com> a écrit :
> >> Initialization and render in the same thread is one of the issues. Another
> >> issue is SDL render should be run in main thread. Maybe it’s not the
> >> duty of this filter to control which thread to be called, but this wouldn’t
> >> work with ffmpeg cmd.
> >>
> >Yes, I agree with your opinion. The control of the filter should be
> >left to the application, and the ffmpeg command should only serve the
> >purpose of demonstrating functionality.
> 
> You can't require running on the main thread. There are no ifs or buts here; libavfilter simply can't require that from the calling application or upper-layer framework and therefore it also can't warrant it to SDL.

We can require anything from an API that we are able to change and extend
Of course we can decide not to allow such requirment even if optional
but we surely _could_ add such a feature if we choose to do so

thx

[...]
-- 
Michael     GnuPG fingerprint: 9FF2128B147EF6730BADF133611EC787040B0FAB

Its not that you shouldnt use gotos but rather that you should write
readable code and code with gotos often but not always is less readable

[-- Attachment #1.2: signature.asc --]
[-- Type: application/pgp-signature, Size: 195 bytes --]

[-- Attachment #2: Type: text/plain, Size: 251 bytes --]

_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".

Re: [FFmpeg-devel] [PATCH] avfilter: add sdlvsink for video display

[Anton Khirnov]

Quoting Michael Niedermayer (2024-06-07 11:53:51)
> On Fri, Jun 07, 2024 at 09:45:09AM +0300, Rémi Denis-Courmont wrote:
> > Hi,
> > 
> > Le 7 juin 2024 06:33:20 GMT+03:00, Shiqi Zhu <hiccupzhu@gmail.com> a écrit :
> > >> Initialization and render in the same thread is one of the issues. Another
> > >> issue is SDL render should be run in main thread. Maybe it’s not the
> > >> duty of this filter to control which thread to be called, but this wouldn’t
> > >> work with ffmpeg cmd.
> > >>
> > >Yes, I agree with your opinion. The control of the filter should be
> > >left to the application, and the ffmpeg command should only serve the
> > >purpose of demonstrating functionality.
> > 
> > You can't require running on the main thread. There are no ifs or buts here; libavfilter simply can't require that from the calling application or upper-layer framework and therefore it also can't warrant it to SDL.
> 
> We can require anything from an API that we are able to change and extend
> Of course we can decide not to allow such requirment even if optional
> but we surely _could_ add such a feature if we choose to do so

We cannot do it in a backwards compatible way.

-- 
Anton Khirnov
_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".

Re: [FFmpeg-devel] [PATCH] avfilter: add sdlvsink for video display

[Rémi Denis-Courmont]

Le 7 juin 2024 12:53:51 GMT+03:00, Michael Niedermayer <michael@niedermayer.cc> a écrit :
>We can require anything from an API that we are able to change and extend
>Of course we can decide not to allow such requirment even if optional
>but we surely _could_ add such a feature if we choose to do so

Sure. You can also require infinite memory or an oracle be provided. That's just not going to happen though. And having libraries depend on the main thread is a well-documented malpractice.

I don't think we should add maintenance burden with code that can't be used safely.
_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".

Re: [FFmpeg-devel] [PATCH] avfilter: add sdlvsink for video display

[Shiqi Zhu]

On Fri, 7 Jun 2024 at 19:55, Rémi Denis-Courmont <remi@remlab.net> wrote:
>
>
>
> Le 7 juin 2024 12:53:51 GMT+03:00, Michael Niedermayer <michael@niedermayer.cc> a écrit :
> >We can require anything from an API that we are able to change and extend
> >Of course we can decide not to allow such requirment even if optional
> >but we surely _could_ add such a feature if we choose to do so
>
> Sure. You can also require infinite memory or an oracle be provided. That's just not going to happen though. And having libraries depend on the main thread is a well-documented malpractice.
>
> I don't think we should add maintenance burden with code that can't be used safely.

Thank you all for your attention to this patch; I greatly appreciate it.

I'd like to provide a brief recap of the issue we've been discussing,
with the following points:

1. Addition of sink type in the filter:
This enhancement is primarily based on the existing avfilter mechanism
and serves as a strengthening module. Using SDL as the sink doesn't
seem to be a good fit, as I'll attempt to rectify in the following
patch.
https://patchwork.ffmpeg.org/project/ffmpeg/patch/20240611130310.1131755-1-hiccupzhu@gmail.com/

2. Utilizing SDL as an implementation for the sink:
Before submitting the patch, I hadn't considered many aspects. During
the intense discussions, I retested the patch on different operating
systems, with the following results, hoping it may assist those
interested in this issue:

Command: ./ffmpeg -lavfi
"testsrc2=size=300x200:rate=25:duration=500,format=yuv420p,sdlvsink"
-f null /dev/null

Test1 Ubuntu 22.04 32GB test result: SUCCESS
Test2 Win11 16GB test result: SUCCESS
Test3 MacOS 14.4 16GB test result: FAILED

Additional notes:
In Test2, slight block occurred when resizing the window, possibly due
to the lack of Windows message handling mechanism, solely compensated
by SDL_PollEvent. I haven't delved deeper into this; if anyone knows
the exact reason, please feel free to add.
In Test3, direct crash occurred, indicating a segmentation fault.

In conclusion:
Due to differences in how graphical interfaces are implemented on
different platforms, the usage of SDL varies (such as init and render
must be on the main thread). Handling such differences at the
application level might be more flexible and less suitable for
inclusion in general-purpose libraries like ffmpeg.

Finally, I'd like to extend my gratitude to everyone for their
valuable contributions and insights.

> _______________________________________________
> ffmpeg-devel mailing list
> ffmpeg-devel@ffmpeg.org
> https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
>
> To unsubscribe, visit link above, or email
> ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".

Re: [FFmpeg-devel] [PATCH] avfilter: add sdlvsink for video display

[Stefano Sabatini]

[-- Attachment #1: Type: text/plain, Size: 2512 bytes --]

On date Tuesday 2024-06-11 21:13:48 +0800, Shiqi Zhu wrote:
> On Fri, 7 Jun 2024 at 19:55, Rémi Denis-Courmont <remi@remlab.net> wrote:
> > Le 7 juin 2024 12:53:51 GMT+03:00, Michael Niedermayer <michael@niedermayer.cc> a écrit :
> > >We can require anything from an API that we are able to change and extend
> > >Of course we can decide not to allow such requirment even if optional
> > >but we surely _could_ add such a feature if we choose to do so
> >
> > Sure. You can also require infinite memory or an oracle be provided. That's just not going to happen though. And having libraries depend on the main thread is a well-documented malpractice.
> >
> > I don't think we should add maintenance burden with code that can't be used safely.
> 
> Thank you all for your attention to this patch; I greatly appreciate it.
> 
> I'd like to provide a brief recap of the issue we've been discussing,
> with the following points:
> 
> 1. Addition of sink type in the filter:
> This enhancement is primarily based on the existing avfilter mechanism
> and serves as a strengthening module. Using SDL as the sink doesn't
> seem to be a good fit, as I'll attempt to rectify in the following
> patch.
> https://patchwork.ffmpeg.org/project/ffmpeg/patch/20240611130310.1131755-1-hiccupzhu@gmail.com/
> 
> 2. Utilizing SDL as an implementation for the sink:
> Before submitting the patch, I hadn't considered many aspects. During
> the intense discussions, I retested the patch on different operating
> systems, with the following results, hoping it may assist those
> interested in this issue:
> 
> Command: ./ffmpeg -lavfi
> "testsrc2=size=300x200:rate=25:duration=500,format=yuv420p,sdlvsink"
> -f null /dev/null

In addition to this, I wonder if adding a vsink for each different
output device is the correct way.

We have a movie source which can be used to read from
libavformat/libavdevice, probablhy we should have a movie sink to be
used to write to libavformat/libavdevice, meaning that a single sink
would enable access to all the supported libavformat/libavdevice
outputs.

I started having a look in that direction a looot of time ago. This
was never finalized because I was not sure about ways to pass options
to encoders and muxers, and about dealing with a variable number of
outputs, I'm attaching this very old proof-of-concept patch for
reference.

This approach would be possibly much more complex, but should provide
a single bridge in place of having a different sink for every output
device or muxer.


[-- Attachment #2: 0001-lavfi-add-moviesink-and-amoviesink-sinks.patch --]
[-- Type: text/x-diff, Size: 15228 bytes --]

From dc88e3f19ad0e481e0adc5e192ad3e2b4b249f55 Mon Sep 17 00:00:00 2001
From: Stefano Sabatini <stefasab@gmail.com>
Date: Wed, 11 Apr 2012 15:10:14 +0200
Subject: [PATCH] lavfi: add moviesink and amoviesink sinks

---
 configure                    |   2 +
 libavfilter/Makefile         |   4 +
 libavfilter/allfilters.c     |   2 +
 libavfilter/sink_moviesink.c | 365 +++++++++++++++++++++++++++++++++++
 4 files changed, 373 insertions(+)
 create mode 100644 libavfilter/sink_moviesink.c

diff --git a/configure b/configure
index 81c7cc18a2..5e08fbbf6c 100755
--- a/configure
+++ b/configure
@@ -1674,6 +1674,7 @@ udp_protocol_deps="network"
 # filters
 aconvert_filter_deps="swresample"
 amovie_filter_deps="avcodec avformat"
+amoviesink_filter_deps="avcodec avformat"
 aresample_filter_deps="swresample"
 ass_filter_deps="libass"
 asyncts_filter_deps="avresample"
@@ -1690,6 +1691,7 @@ frei0r_src_filter_deps="frei0r dlopen"
 frei0r_src_filter_extralibs='$ldl'
 hqdn3d_filter_deps="gpl"
 movie_filter_deps="avcodec avformat"
+moviesink_filter_deps="avcodec avformat"
 mp_filter_deps="gpl avcodec swscale postproc"
 mptestsrc_filter_deps="gpl"
 negate_filter_deps="lut_filter"
diff --git a/libavfilter/Makefile b/libavfilter/Makefile
index 29345fc15e..a2bd404304 100644
--- a/libavfilter/Makefile
+++ b/libavfilter/Makefile
@@ -7,8 +7,10 @@ FFLIBS-$(CONFIG_RESAMPLE_FILTER) += avresample
 
 FFLIBS-$(CONFIG_ACONVERT_FILTER)             += swresample
 FFLIBS-$(CONFIG_AMOVIE_FILTER)               += avformat avcodec
+FFLIBS-$(CONFIG_AMOVIESINK_FILTER)           += avformat avcodec
 FFLIBS-$(CONFIG_ARESAMPLE_FILTER)            += swresample
 FFLIBS-$(CONFIG_MOVIE_FILTER)                += avformat avcodec
+FFLIBS-$(CONFIG_MOVIESINK_FILTER)            += avformat avcodec
 FFLIBS-$(CONFIG_PAN_FILTER)                  += swresample
 FFLIBS-$(CONFIG_REMOVELOGO_FILTER)           += avformat avcodec
 FFLIBS-$(CONFIG_MP_FILTER)                   += avcodec postproc
@@ -65,6 +67,7 @@ OBJS-$(CONFIG_AMOVIE_FILTER)                 += src_movie.o
 OBJS-$(CONFIG_ANULLSRC_FILTER)               += asrc_anullsrc.o
 
 OBJS-$(CONFIG_ABUFFERSINK_FILTER)            += sink_buffer.o
+OBJS-$(CONFIG_AMOVIESINK_FILTER)             += sink_moviesink.o
 OBJS-$(CONFIG_ANULLSINK_FILTER)              += asink_anullsink.o
 
 OBJS-$(CONFIG_ASS_FILTER)                    += vf_ass.o
@@ -133,6 +136,7 @@ OBJS-$(CONFIG_RGBTESTSRC_FILTER)             += vsrc_testsrc.o
 OBJS-$(CONFIG_TESTSRC_FILTER)                += vsrc_testsrc.o
 
 OBJS-$(CONFIG_BUFFERSINK_FILTER)             += sink_buffer.o
+OBJS-$(CONFIG_MOVIESINK_FILTER)              += sink_moviesink.o
 OBJS-$(CONFIG_NULLSINK_FILTER)               += vsink_nullsink.o
 
 OBJS-$(CONFIG_MP_FILTER) += libmpcodecs/mp_image.o
diff --git a/libavfilter/allfilters.c b/libavfilter/allfilters.c
index b9d44f2fdf..a7cc9ad3de 100644
--- a/libavfilter/allfilters.c
+++ b/libavfilter/allfilters.c
@@ -56,6 +56,7 @@ void avfilter_register_all(void)
 
     REGISTER_FILTER (ABUFFERSINK, abuffersink, asink);
     REGISTER_FILTER (ANULLSINK,   anullsink,   asink);
+    REGISTER_FILTER (AMOVIESINK,  amoviesink,  asink);
 
     REGISTER_FILTER (ASS,         ass,         vf);
     REGISTER_FILTER (BBOX,        bbox,        vf);
@@ -123,6 +124,7 @@ void avfilter_register_all(void)
     REGISTER_FILTER (TESTSRC,     testsrc,     vsrc);
 
     REGISTER_FILTER (BUFFERSINK,  buffersink,  vsink);
+    REGISTER_FILTER (MOVIESINK,   moviesink,   vsink);
     REGISTER_FILTER (NULLSINK,    nullsink,    vsink);
 
     /* those filters are part of public or internal API => registered
diff --git a/libavfilter/sink_moviesink.c b/libavfilter/sink_moviesink.c
new file mode 100644
index 0000000000..27c8cca078
--- /dev/null
+++ b/libavfilter/sink_moviesink.c
@@ -0,0 +1,365 @@
+/*
+ * Copyright (c) 2012 Stefano Sabatini
+ *
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+/**
+ * @file
+ * Movie video sink filter
+ */
+
+#include "libavutil/avstring.h"
+#include "libavutil/opt.h"
+#include "libavformat/avformat.h"
+#include "avcodec.h"
+#include "avfilter.h"
+
+typedef struct {
+    const AVClass *class;
+    char *codec_name;
+    char *file_name;
+    char *format_name;
+
+    AVOutputFormat   *oformat;
+    AVFormatContext  *oformat_ctx;
+    AVCodec          *codec;
+
+    int w, h;
+    AVFilterBufferRef *bufref;
+} MovieSinkContext;
+
+#define OFFSET(x) offsetof(MovieSinkContext, x)
+
+static const AVOption moviesink_options[] = {
+{"codec_name",  "set codec name",  OFFSET(codec_name),  AV_OPT_TYPE_STRING, {.str =  0},  CHAR_MIN, CHAR_MAX },
+{"c",           "set codec name" , OFFSET(codec_name),  AV_OPT_TYPE_STRING, {.str =  0},  CHAR_MIN, CHAR_MAX },
+{"format_name", "set format name", OFFSET(format_name),  AV_OPT_TYPE_STRING, {.str =  0},  CHAR_MIN, CHAR_MAX },
+{"f",           "set format name", OFFSET(format_name),  AV_OPT_TYPE_STRING, {.str =  0},  CHAR_MIN, CHAR_MAX },
+{NULL},
+};
+
+static const char *moviesink_get_name(void *ctx)
+{
+    return "moviesink";
+}
+
+static const AVClass moviesink_class = {
+    "MovieSinkContext",
+    moviesink_get_name,
+    moviesink_options
+};
+
+static av_cold int moviesink_common_init(AVFilterContext *ctx, const char *args, void *opaque,
+                                         enum AVMediaType type)
+{
+    MovieSinkContext *movie = ctx->priv;
+    AVOutputFormat *oformat = NULL;
+    AVStream *st;
+    enum CodecID codec_id;
+    int ret;
+
+    movie->class = &moviesink_class;
+    av_opt_set_defaults(movie);
+
+    if (args)
+        movie->file_name = av_get_token(&args, ":");
+    if (!movie->file_name || !*movie->file_name) {
+        av_log(ctx, AV_LOG_ERROR, "No filename provided!\n");
+        return AVERROR(EINVAL);
+    }
+
+    if (*args++ == ':' && (ret = av_set_options_string(movie, args, "=", ":")) < 0) {
+        av_log(ctx, AV_LOG_ERROR, "Error parsing options string: '%s'\n", args);
+        return ret;
+    }
+
+    av_register_all();
+    movie->oformat_ctx = avformat_alloc_context();
+    av_strlcpy(movie->oformat_ctx->filename, movie->file_name,
+               sizeof(movie->oformat_ctx->filename));
+
+    if (!(oformat = av_guess_format(movie->format_name, movie->file_name, NULL))) {
+        av_log(ctx, AV_LOG_ERROR,
+               "Unable to find a suitable output format for '%s'\n", movie->file_name);
+        return AVERROR(EINVAL);
+    }
+    movie->oformat_ctx->oformat = movie->oformat = oformat;
+
+    /* add a stream */
+    st = avformat_new_stream(movie->oformat_ctx, NULL);
+    if (!st) {
+        av_log(ctx, AV_LOG_ERROR, "Could not allocate stream\n");
+        return AVERROR(ENOMEM);
+    }
+
+    avcodec_get_context_defaults3(st->codec, NULL);
+
+    if (movie->codec_name) {
+        movie->codec = avcodec_find_decoder_by_name(movie->codec_name);
+    } else {
+        codec_id = av_guess_codec(movie->oformat_ctx->oformat, NULL,
+                                  movie->oformat_ctx->filename, NULL, type);
+        movie->codec = avcodec_find_encoder(codec_id);
+    }
+
+    if (!movie->codec) {
+        av_log(ctx, AV_LOG_ERROR, "Failed to find any codec\n");
+        return AVERROR(EINVAL);
+    }
+
+    return 0;
+}
+
+static av_cold void moviesink_common_uninit(AVFilterContext *ctx)
+{
+    MovieSinkContext *movie = ctx->priv;
+
+    av_freep(&movie->codec_name);
+    av_freep(&movie->file_name);
+    av_freep(&movie->format_name);
+
+    if (movie->oformat_ctx)
+        avformat_close_input(&movie->oformat_ctx);
+
+    avfilter_unref_bufferp(&movie->bufref);
+}
+
+static int make_filter_formats(AVFilterFormats **ffmts, const int *fmts)
+{
+    int i, len;
+    const int *p = fmts;
+    for (len = 0; *p != -1; p++, len++)
+        ;
+
+    if (!(*ffmts = av_mallocz(sizeof(AVFilterFormats))))
+        return AVERROR(ENOMEM);
+    if ((!((*ffmts)->formats = av_malloc(sizeof((*ffmts)->formats) * len)))) {
+        av_free(*ffmts);
+        return AVERROR(ENOMEM);
+    }
+
+    (*ffmts)->format_count = len;
+    p = fmts;
+    for (i = 0; i < len; i++, p++)
+        (*ffmts)->formats[i] = *p;
+
+    return 0;
+}
+
+static int moviesink_common_config_props(AVFilterLink *inlink, enum AVMediaType type)
+{
+    AVFilterContext *ctx = inlink->dst;
+    MovieSinkContext *movie = ctx->priv;
+    AVCodecContext *encctx;
+    AVStream *st = movie->oformat_ctx->streams[0];
+    int ret;
+
+    switch (type) {
+    case AVMEDIA_TYPE_VIDEO:
+        st->codec->pix_fmt   = inlink->format;
+        st->codec->width     = inlink->w;
+        st->codec->height    = inlink->h;
+        st->codec->time_base = inlink->time_base;
+        break;
+    case AVMEDIA_TYPE_AUDIO:
+        st->codec->sample_fmt     = inlink->format;
+        st->codec->sample_rate    = inlink->sample_rate;
+        st->codec->time_base      = inlink->time_base;
+        st->codec->channel_layout = inlink->channel_layout;
+        st->codec->channels       =
+            av_get_channel_layout_nb_channels(inlink->channel_layout);
+        break;
+    }
+    encctx = avcodec_alloc_context3(movie->codec);
+    if (!encctx) {
+        av_log(ctx, AV_LOG_ERROR, "Error while allocating the codec context\n");
+        return AVERROR(ENOMEM);
+    }
+
+    if ((ret = avcodec_open2(st->codec, movie->codec, NULL)) < 0) {
+        av_log(ctx, AV_LOG_ERROR, "Error while opening the codec\n");
+        return ret;
+    }
+
+    av_dump_format(movie->oformat_ctx, 0, movie->file_name, 1);
+
+    /* open the output file, if needed */
+    if (!(movie->oformat->flags & AVFMT_NOFILE)) {
+        if ((ret = avio_open(&movie->oformat_ctx->pb,
+                             movie->file_name, AVIO_FLAG_WRITE)) < 0) {
+            av_log(ctx, AV_LOG_ERROR, "Could not open '%s'\n", movie->file_name);
+            return ret;
+        }
+    }
+
+    if ((ret = avformat_write_header(movie->oformat_ctx, NULL)) < 0) {
+        av_log(ctx, AV_LOG_ERROR, "Could not write header for '%s'\n", movie->file_name);
+        return ret;
+    }
+
+    return 0;
+}
+
+#if CONFIG_MOVIESINK_FILTER
+
+static av_cold int moviesink_init(AVFilterContext *ctx, const char *args, void *opaque)
+{
+    return moviesink_common_init(ctx, args, opaque, AVMEDIA_TYPE_VIDEO);
+}
+
+static int moviesink_query_formats(AVFilterContext *ctx)
+{
+    MovieSinkContext *movie = ctx->priv;
+    AVFilterFormats *ffmts;
+    int ret;
+
+    if ((ret = make_filter_formats(&ffmts, movie->codec->pix_fmts)))
+        return ret;
+
+    avfilter_set_common_pixel_formats(ctx, ffmts);
+    return 0;
+}
+
+static int moviesink_config_props(AVFilterLink *inlink)
+{
+    return moviesink_common_config_props(inlink, AVMEDIA_TYPE_VIDEO);
+}
+
+static void moviesink_start_frame(AVFilterLink *inlink, AVFilterBufferRef *bufref) { }
+
+static void moviesink_end_frame(AVFilterLink *inlink)
+{
+    MovieSinkContext *movie = inlink->dst->priv;
+    AVFilterBufferRef *picref = inlink->cur_buf;
+    AVFrame frame;
+    int ret, got_packet;
+    AVPacket packet;
+
+    ret = avfilter_fill_frame_from_video_buffer_ref(&frame, picref);
+    if (ret < 0) {
+        av_log(inlink->dst, AV_LOG_ERROR, "Cannot convert picref to frame\n");
+        return;
+    }
+
+    av_init_packet(&packet);
+    packet.data = NULL;
+    packet.size = 0;
+    got_packet = 0;
+    ret = avcodec_encode_video2(movie->oformat_ctx->streams[0]->codec,
+                                &packet, &frame, &got_packet);
+    if (ret < 0) {
+        av_log(inlink->dst, AV_LOG_ERROR, "Video encoding failed\n");
+        return;
+    }
+    packet.stream_index = 0;
+
+    av_interleaved_write_frame(movie->oformat_ctx, &packet);
+}
+
+AVFilter avfilter_vsink_moviesink = {
+    .name          = "moviesink",
+    .description   = NULL_IF_CONFIG_SMALL("Output the video to a media file."),
+    .priv_size     = sizeof(MovieSinkContext),
+    .query_formats = moviesink_query_formats,
+    .init          = moviesink_init,
+    .uninit        = moviesink_common_uninit,
+
+    .inputs = (const AVFilterPad[]) {
+        { .name            = "default",
+          .type            = AVMEDIA_TYPE_VIDEO,
+          .start_frame     = moviesink_start_frame,
+          .end_frame       = moviesink_end_frame,
+          .config_props    = moviesink_config_props, },
+        { .name = NULL }
+    },
+    .outputs = (const AVFilterPad[]) {
+        { .name = NULL }
+    },
+};
+
+#endif /* CONFIG_MOVIESINK_FILTER */
+
+#ifdef CONFIG_AMOVIESINK_FILTER
+
+static av_cold int amoviesink_init(AVFilterContext *ctx, const char *args, void *opaque)
+{
+    return moviesink_common_init(ctx, args, opaque, AVMEDIA_TYPE_AUDIO);
+}
+
+static int amoviesink_query_formats(AVFilterContext *ctx)
+{
+    MovieSinkContext *movie = ctx->priv;
+    AVFilterFormats *ffmts;
+    int ret;
+
+    if ((ret = make_filter_formats(&ffmts, movie->codec->sample_fmts)))
+        return ret;
+
+    avfilter_set_common_sample_formats(ctx, ffmts);
+    return 0;
+}
+
+static int amoviesink_config_props(AVFilterLink *inlink)
+{
+    return moviesink_common_config_props(inlink, AVMEDIA_TYPE_AUDIO);
+}
+
+static void amoviesink_filter_samples(AVFilterLink *inlink, AVFilterBufferRef *insamples)
+{
+    MovieSinkContext *movie = inlink->dst->priv;
+    AVFrame frame;
+    AVPacket packet;
+    int ret, got_packet;
+
+    av_init_packet(&packet);
+    packet.data = NULL;
+    packet.size = 0;
+    got_packet = 0;
+    ret = avcodec_encode_audio2(movie->oformat_ctx->streams[0]->codec,
+                                &packet, &frame, &got_packet);
+    if (ret < 0) {
+        av_log(inlink->dst, AV_LOG_ERROR, "Audio encoding failed\n");
+        return;
+    }
+    packet.stream_index = 0;
+
+    av_interleaved_write_frame(movie->oformat_ctx, &packet);
+}
+
+AVFilter avfilter_asink_amoviesink = {
+    .name          = "amoviesink",
+    .description   = NULL_IF_CONFIG_SMALL("Output audio to a movie file."),
+    .priv_size     = sizeof(MovieSinkContext),
+    .init          = amoviesink_init,
+    .uninit        = moviesink_common_uninit,
+    .query_formats = amoviesink_query_formats,
+
+    .inputs = (const AVFilterPad[]) {
+        { .name            = "default",
+          .type            = AVMEDIA_TYPE_AUDIO,
+          .config_props    = amoviesink_config_props,
+          .filter_samples  = amoviesink_filter_samples,
+        },
+        { .name = NULL }
+    },
+    .outputs = (const AVFilterPad[]) {
+        { .name = NULL }
+    },
+};
+
+#endif /* CONFIG_AMOVIESINK_FILTER */
-- 
2.34.1


[-- Attachment #3: Type: text/plain, Size: 251 bytes --]

_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".

Re: [FFmpeg-devel] [PATCH] avfilter: add sdlvsink for video display

[Paul B Mahol]

On Wed, Jun 12, 2024 at 9:52 PM Stefano Sabatini <stefasab@gmail.com> wrote:

> On date Tuesday 2024-06-11 21:13:48 +0800, Shiqi Zhu wrote:
> > On Fri, 7 Jun 2024 at 19:55, Rémi Denis-Courmont <remi@remlab.net>
> wrote:
> > > Le 7 juin 2024 12:53:51 GMT+03:00, Michael Niedermayer <
> michael@niedermayer.cc> a écrit :
> > > >We can require anything from an API that we are able to change and
> extend
> > > >Of course we can decide not to allow such requirment even if optional
> > > >but we surely _could_ add such a feature if we choose to do so
> > >
> > > Sure. You can also require infinite memory or an oracle be provided.
> That's just not going to happen though. And having libraries depend on the
> main thread is a well-documented malpractice.
> > >
> > > I don't think we should add maintenance burden with code that can't be
> used safely.
> >
> > Thank you all for your attention to this patch; I greatly appreciate it.
> >
> > I'd like to provide a brief recap of the issue we've been discussing,
> > with the following points:
> >
> > 1. Addition of sink type in the filter:
> > This enhancement is primarily based on the existing avfilter mechanism
> > and serves as a strengthening module. Using SDL as the sink doesn't
> > seem to be a good fit, as I'll attempt to rectify in the following
> > patch.
> >
> https://patchwork.ffmpeg.org/project/ffmpeg/patch/20240611130310.1131755-1-hiccupzhu@gmail.com/
> >
> > 2. Utilizing SDL as an implementation for the sink:
> > Before submitting the patch, I hadn't considered many aspects. During
> > the intense discussions, I retested the patch on different operating
> > systems, with the following results, hoping it may assist those
> > interested in this issue:
> >
> > Command: ./ffmpeg -lavfi
> > "testsrc2=size=300x200:rate=25:duration=500,format=yuv420p,sdlvsink"
> > -f null /dev/null
>
> In addition to this, I wonder if adding a vsink for each different
> output device is the correct way.
>
> We have a movie source which can be used to read from
> libavformat/libavdevice, probablhy we should have a movie sink to be
> used to write to libavformat/libavdevice, meaning that a single sink
> would enable access to all the supported libavformat/libavdevice
> outputs.
>
> I started having a look in that direction a looot of time ago. This
> was never finalized because I was not sure about ways to pass options
> to encoders and muxers, and about dealing with a variable number of
> outputs, I'm attaching this very old proof-of-concept patch for
> reference.
>
> This approach would be possibly much more complex, but should provide
> a single bridge in place of having a different sink for every output
> device or muxer.
>


Pointless as real plan is to get rid of immature libavdevice API.


>
> _______________________________________________
> ffmpeg-devel mailing list
> ffmpeg-devel@ffmpeg.org
> https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
>
> To unsubscribe, visit link above, or email
> ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
>
_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".

Re: [FFmpeg-devel] [PATCH] avfilter: add sdlvsink for video display

[Shiqi Zhu]

On Thu, 13 Jun 2024 at 04:14, Paul B Mahol <onemda@gmail.com> wrote:
>
> On Wed, Jun 12, 2024 at 9:52 PM Stefano Sabatini <stefasab@gmail.com> wrote:
>
> > On date Tuesday 2024-06-11 21:13:48 +0800, Shiqi Zhu wrote:
> > > On Fri, 7 Jun 2024 at 19:55, Rémi Denis-Courmont <remi@remlab.net>
> > wrote:
> > > > Le 7 juin 2024 12:53:51 GMT+03:00, Michael Niedermayer <
> > michael@niedermayer.cc> a écrit :
> > > > >We can require anything from an API that we are able to change and
> > extend
> > > > >Of course we can decide not to allow such requirment even if optional
> > > > >but we surely _could_ add such a feature if we choose to do so
> > > >
> > > > Sure. You can also require infinite memory or an oracle be provided.
> > That's just not going to happen though. And having libraries depend on the
> > main thread is a well-documented malpractice.
> > > >
> > > > I don't think we should add maintenance burden with code that can't be
> > used safely.
> > >
> > > Thank you all for your attention to this patch; I greatly appreciate it.
> > >
> > > I'd like to provide a brief recap of the issue we've been discussing,
> > > with the following points:
> > >
> > > 1. Addition of sink type in the filter:
> > > This enhancement is primarily based on the existing avfilter mechanism
> > > and serves as a strengthening module. Using SDL as the sink doesn't
> > > seem to be a good fit, as I'll attempt to rectify in the following
> > > patch.
> > >
> > https://patchwork.ffmpeg.org/project/ffmpeg/patch/20240611130310.1131755-1-hiccupzhu@gmail.com/
> > >
> > > 2. Utilizing SDL as an implementation for the sink:
> > > Before submitting the patch, I hadn't considered many aspects. During
> > > the intense discussions, I retested the patch on different operating
> > > systems, with the following results, hoping it may assist those
> > > interested in this issue:
> > >
> > > Command: ./ffmpeg -lavfi
> > > "testsrc2=size=300x200:rate=25:duration=500,format=yuv420p,sdlvsink"
> > > -f null /dev/null
> >
> > In addition to this, I wonder if adding a vsink for each different
> > output device is the correct way.
> >
> > We have a movie source which can be used to read from
> > libavformat/libavdevice, probablhy we should have a movie sink to be
> > used to write to libavformat/libavdevice, meaning that a single sink
> > would enable access to all the supported libavformat/libavdevice
> > outputs.
> >
> > I started having a look in that direction a looot of time ago. This
> > was never finalized because I was not sure about ways to pass options
> > to encoders and muxers, and about dealing with a variable number of
> > outputs, I'm attaching this very old proof-of-concept patch for
> > reference.
> >
> > This approach would be possibly much more complex, but should provide
> > a single bridge in place of having a different sink for every output
> > device or muxer.
> >
>
>
> Pointless as real plan is to get rid of immature libavdevice API.
>
>
Using the libavdevice API can be somewhat confusing, especially
regarding concepts like mux and demux.

> >
> > _______________________________________________
> > ffmpeg-devel mailing list
> > ffmpeg-devel@ffmpeg.org
> > https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
> >
> > To unsubscribe, visit link above, or email
> > ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
> >
> _______________________________________________
> ffmpeg-devel mailing list
> ffmpeg-devel@ffmpeg.org
> https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
>
> To unsubscribe, visit link above, or email
> ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".

Re: [FFmpeg-devel] [PATCH] avfilter: add sdlvsink for video display

[Shiqi Zhu]

On Thu, 13 Jun 2024 at 03:52, Stefano Sabatini <stefasab@gmail.com> wrote:
>
> On date Tuesday 2024-06-11 21:13:48 +0800, Shiqi Zhu wrote:
> > On Fri, 7 Jun 2024 at 19:55, Rémi Denis-Courmont <remi@remlab.net> wrote:
> > > Le 7 juin 2024 12:53:51 GMT+03:00, Michael Niedermayer <michael@niedermayer.cc> a écrit :
> > > >We can require anything from an API that we are able to change and extend
> > > >Of course we can decide not to allow such requirment even if optional
> > > >but we surely _could_ add such a feature if we choose to do so
> > >
> > > Sure. You can also require infinite memory or an oracle be provided. That's just not going to happen though. And having libraries depend on the main thread is a well-documented malpractice.
> > >
> > > I don't think we should add maintenance burden with code that can't be used safely.
> >
> > Thank you all for your attention to this patch; I greatly appreciate it.
> >
> > I'd like to provide a brief recap of the issue we've been discussing,
> > with the following points:
> >
> > 1. Addition of sink type in the filter:
> > This enhancement is primarily based on the existing avfilter mechanism
> > and serves as a strengthening module. Using SDL as the sink doesn't
> > seem to be a good fit, as I'll attempt to rectify in the following
> > patch.
> > https://patchwork.ffmpeg.org/project/ffmpeg/patch/20240611130310.1131755-1-hiccupzhu@gmail.com/
> >
> > 2. Utilizing SDL as an implementation for the sink:
> > Before submitting the patch, I hadn't considered many aspects. During
> > the intense discussions, I retested the patch on different operating
> > systems, with the following results, hoping it may assist those
> > interested in this issue:
> >
> > Command: ./ffmpeg -lavfi
> > "testsrc2=size=300x200:rate=25:duration=500,format=yuv420p,sdlvsink"
> > -f null /dev/null
>
> In addition to this, I wonder if adding a vsink for each different
> output device is the correct way.
>
Indeed, the mechanism of libavfilter is more robust, providing
essential mechanisms like format_query. These are crucial for
implementing media graph mechanisms (equivalent to DirectShow's graph
and GStreamer's pipeline concepts).

Therefore, gradually replacing libavdevice with sink-filter should be
a relatively gentle approach.

> We have a movie source which can be used to read from
> libavformat/libavdevice, probablhy we should have a movie sink to be
> used to write to libavformat/libavdevice, meaning that a single sink
> would enable access to all the supported libavformat/libavdevice
> outputs.
>
> I started having a look in that direction a looot of time ago. This
> was never finalized because I was not sure about ways to pass options
> to encoders and muxers, and about dealing with a variable number of
> outputs, I'm attaching this very old proof-of-concept patch for
> reference.
>
> This approach would be possibly much more complex, but should provide
> a single bridge in place of having a different sink for every output
> device or muxer.
>
I carefully reviewed your patch, and I currently have similar
functionality in my code. Here are my thoughts:

Using a filter as a bridge to connect to the avdevice allows for
setting the device type and basic information through parameters and
options.

However, the formats supported by devices are often variable. During
format negotiation, it's necessary to extend AV_APP_TO_DEV_xxx for
each device to implement queries for format, sampling rate, channels,
etc. This can be quite labor-intensive.

Using a filter as a sink can better address this issue.


> _______________________________________________
> ffmpeg-devel mailing list
> ffmpeg-devel@ffmpeg.org
> https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
>
> To unsubscribe, visit link above, or email
> ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".