* [FFmpeg-devel] [PATCH v2 FFmpeg 11/20] libavfilter/dnn_filter_common: dnn execute functions for CLIP and CLAP
@ 2025-03-10 19:54 m.kaindl0208
2025-05-12 10:08 ` Guo, Yejun
2025-05-12 10:11 ` Guo, Yejun
0 siblings, 2 replies; 3+ messages in thread
From: m.kaindl0208 @ 2025-03-10 19:54 UTC (permalink / raw)
To: ffmpeg-devel
Signed-off-by: MaximilianKaindl <m.kaindl0208@gmail.com>
---
libavfilter/dnn_filter_common.c | 81 +++++++++++++++++++++++----------
libavfilter/dnn_filter_common.h | 34 +++++++-------
2 files changed, 76 insertions(+), 39 deletions(-)
diff --git a/libavfilter/dnn_filter_common.c b/libavfilter/dnn_filter_common.c
index 899e869cca..f3117748ae 100644
--- a/libavfilter/dnn_filter_common.c
+++ b/libavfilter/dnn_filter_common.c
@@ -1,20 +1,20 @@
/*
- * This file is part of FFmpeg.
- *
- * FFmpeg is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * FFmpeg is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with FFmpeg; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
- */
+* This file is part of FFmpeg.
+*
+* FFmpeg is free software; you can redistribute it and/or
+* modify it under the terms of the GNU Lesser General Public
+* License as published by the Free Software Foundation; either
+* version 2.1 of the License, or (at your option) any later version.
+*
+* FFmpeg is distributed in the hope that it will be useful,
+* but WITHOUT ANY WARRANTY; without even the implied warranty of
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+* Lesser General Public License for more details.
+*
+* You should have received a copy of the GNU Lesser General Public
+* License along with FFmpeg; if not, write to the Free Software
+* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+*/
#include "dnn_filter_common.h"
#include "libavutil/avstring.h"
@@ -87,10 +87,10 @@ static int ff_dnn_init_priv(DnnContext *ctx, DNNFunctionType func_type, AVFilter
if (backend == DNN_TH) {
if (ctx->model_inputname)
av_log(filter_ctx, AV_LOG_WARNING, "LibTorch backend do not require inputname, "\
- "inputname will be ignored.\n");
+ "inputname will be ignored.\n");
if (ctx->model_outputnames)
av_log(filter_ctx, AV_LOG_WARNING, "LibTorch backend do not require outputname(s), "\
- "all outputname(s) will be ignored.\n");
+ "all outputname(s) will be ignored.\n");
#if (CONFIG_LIBTOKENIZERS == 0)
if ((func_type == DFT_ANALYTICS_CLIP || func_type == DFT_ANALYTICS_CLAP)) {
@@ -126,14 +126,14 @@ static int ff_dnn_init_priv(DnnContext *ctx, DNNFunctionType func_type, AVFilter
void *child = NULL;
av_log(filter_ctx, AV_LOG_WARNING,
- "backend_configs is deprecated, please set backend options directly\n");
+ "backend_configs is deprecated, please set backend options directly\n");
while (child = ff_dnn_child_next(ctx, child)) {
if (*(const AVClass **)child == &ctx->dnn_module->clazz) {
int ret = av_opt_set_from_string(child, ctx->backend_options,
- NULL, "=", "&");
+ NULL, "=", "&");
if (ret < 0) {
av_log(filter_ctx, AV_LOG_ERROR, "failed to parse options \"%s\"\n",
- ctx->backend_options);
+ ctx->backend_options);
return ret;
}
}
@@ -199,9 +199,9 @@ int ff_dnn_get_input(DnnContext *ctx, DNNData *input)
int ff_dnn_get_output(DnnContext *ctx, int input_width, int input_height, int *output_width, int *output_height)
{
char * output_name = ctx->model_outputnames && ctx->backend_type != DNN_TH ?
- ctx->model_outputnames[0] : NULL;
+ ctx->model_outputnames[0] : NULL;
return ctx->model->get_output(ctx->model, ctx->model_inputname, input_width, input_height,
- (const char *)output_name, output_width, output_height);
+ (const char *)output_name, output_width, output_height);
}
int ff_dnn_execute_model(DnnContext *ctx, AVFrame *in_frame, AVFrame *out_frame)
@@ -231,6 +231,41 @@ int ff_dnn_execute_model_classification(DnnContext *ctx, AVFrame *in_frame, AVFr
return (ctx->dnn_module->execute_model)(ctx->model, &class_params.base);
}
+int ff_dnn_execute_model_clip(DnnContext *ctx, AVFrame *in_frame, AVFrame *out_frame, const char **labels, int label_count, const char* tokenizer_path, char *target)
+{
+ DNNExecZeroShotClassificationParams class_params = {
+ {
+ .input_name = ctx->model_inputname,
+ .output_names = (const char **)ctx->model_outputnames,
+ .nb_output = ctx->nb_outputs,
+ .in_frame = in_frame,
+ .out_frame = out_frame,
+ },
+ .labels = labels,
+ .label_count = label_count,
+ .tokenizer_path = tokenizer_path,
+ .target = target,
+ };
+ return (ctx->dnn_module->execute_model)(ctx->model, &class_params.base);
+}
+
+int ff_dnn_execute_model_clap(DnnContext *ctx, AVFrame *in_frame, AVFrame *out_frame, const char **labels, int label_count, const char* tokenizer_path)
+{
+ DNNExecZeroShotClassificationParams class_params = {
+ {
+ .input_name = ctx->model_inputname,
+ .output_names = (const char **)ctx->model_outputnames,
+ .nb_output = ctx->nb_outputs,
+ .in_frame = in_frame,
+ .out_frame = out_frame,
+ },
+ .labels = labels,
+ .label_count = label_count,
+ .tokenizer_path = tokenizer_path,
+ };
+ return (ctx->dnn_module->execute_model)(ctx->model, &class_params.base);
+}
+
DNNAsyncStatusType ff_dnn_get_result(DnnContext *ctx, AVFrame **in_frame, AVFrame **out_frame)
{
return (ctx->dnn_module->get_result)(ctx->model, in_frame, out_frame);
diff --git a/libavfilter/dnn_filter_common.h b/libavfilter/dnn_filter_common.h
index 301441a37a..a782f64a1f 100644
--- a/libavfilter/dnn_filter_common.h
+++ b/libavfilter/dnn_filter_common.h
@@ -1,20 +1,20 @@
/*
- * This file is part of FFmpeg.
- *
- * FFmpeg is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * FFmpeg is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with FFmpeg; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
- */
+* This file is part of FFmpeg.
+*
+* FFmpeg is free software; you can redistribute it and/or
+* modify it under the terms of the GNU Lesser General Public
+* License as published by the Free Software Foundation; either
+* version 2.1 of the License, or (at your option) any later version.
+*
+* FFmpeg is distributed in the hope that it will be useful,
+* but WITHOUT ANY WARRANTY; without even the implied warranty of
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+* Lesser General Public License for more details.
+*
+* You should have received a copy of the GNU Lesser General Public
+* License along with FFmpeg; if not, write to the Free Software
+* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+*/
/**
* @file
@@ -63,6 +63,8 @@ int ff_dnn_get_input(DnnContext *ctx, DNNData *input);
int ff_dnn_get_output(DnnContext *ctx, int input_width, int input_height, int *output_width, int *output_height);
int ff_dnn_execute_model(DnnContext *ctx, AVFrame *in_frame, AVFrame *out_frame);
int ff_dnn_execute_model_classification(DnnContext *ctx, AVFrame *in_frame, AVFrame *out_frame, const char *target);
+int ff_dnn_execute_model_clip(DnnContext *ctx, AVFrame *in_frame, AVFrame *out_frame, const char **labels, int label_count, const char* tokenizer_path, char *target);
+int ff_dnn_execute_model_clap(DnnContext *ctx, AVFrame *in_frame, AVFrame *out_frame, const char **labels, int label_count, const char* tokenizer_path);
DNNAsyncStatusType ff_dnn_get_result(DnnContext *ctx, AVFrame **in_frame, AVFrame **out_frame);
int ff_dnn_flush(DnnContext *ctx);
void ff_dnn_uninit(DnnContext *ctx);
--
2.34.1
_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
^ permalink raw reply [flat|nested] 3+ messages in thread
* Re: [FFmpeg-devel] [PATCH v2 FFmpeg 11/20] libavfilter/dnn_filter_common: dnn execute functions for CLIP and CLAP
2025-03-10 19:54 [FFmpeg-devel] [PATCH v2 FFmpeg 11/20] libavfilter/dnn_filter_common: dnn execute functions for CLIP and CLAP m.kaindl0208
@ 2025-05-12 10:08 ` Guo, Yejun
2025-05-12 10:11 ` Guo, Yejun
1 sibling, 0 replies; 3+ messages in thread
From: Guo, Yejun @ 2025-05-12 10:08 UTC (permalink / raw)
To: FFmpeg development discussions and patches
> -----Original Message-----
> From: ffmpeg-devel <ffmpeg-devel-bounces@ffmpeg.org> On Behalf Of
> m.kaindl0208@gmail.com
> Sent: Tuesday, March 11, 2025 3:54 AM
> To: ffmpeg-devel@ffmpeg.org
> Subject: [FFmpeg-devel] [PATCH v2 FFmpeg 11/20]
> libavfilter/dnn_filter_common: dnn execute functions for CLIP and CLAP
>
> Signed-off-by: MaximilianKaindl <m.kaindl0208@gmail.com>
> ---
> libavfilter/dnn_filter_common.c | 81 +++++++++++++++++++++++----------
> libavfilter/dnn_filter_common.h | 34 +++++++-------
> 2 files changed, 76 insertions(+), 39 deletions(-)
>
> diff --git a/libavfilter/dnn_filter_common.c b/libavfilter/dnn_filter_common.c
> index 899e869cca..f3117748ae 100644
> --- a/libavfilter/dnn_filter_common.c
> +++ b/libavfilter/dnn_filter_common.c
> @@ -1,20 +1,20 @@
> /*
> - * This file is part of FFmpeg.
> - *
> - * FFmpeg is free software; you can redistribute it and/or
> - * modify it under the terms of the GNU Lesser General Public
> - * License as published by the Free Software Foundation; either
> - * version 2.1 of the License, or (at your option) any later version.
> - *
> - * FFmpeg is distributed in the hope that it will be useful,
> - * but WITHOUT ANY WARRANTY; without even the implied warranty of
> - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
> - * Lesser General Public License for more details.
> - *
> - * You should have received a copy of the GNU Lesser General Public
> - * License along with FFmpeg; if not, write to the Free Software
> - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301
> USA
> - */
> +* This file is part of FFmpeg.
> +*
> +* FFmpeg is free software; you can redistribute it and/or
> +* modify it under the terms of the GNU Lesser General Public
> +* License as published by the Free Software Foundation; either
> +* version 2.1 of the License, or (at your option) any later version.
> +*
> +* FFmpeg is distributed in the hope that it will be useful,
> +* but WITHOUT ANY WARRANTY; without even the implied warranty of
> +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
> +* Lesser General Public License for more details.
> +*
> +* You should have received a copy of the GNU Lesser General Public
> +* License along with FFmpeg; if not, write to the Free Software
> +* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301
> USA
> +*/
No actual change above? And same comment in the other file below.
> #include "dnn_filter_common.h"
> #include "libavutil/avstring.h"
> @@ -87,10 +87,10 @@ static int ff_dnn_init_priv(DnnContext *ctx,
> DNNFunctionType func_type, AVFilter
> if (backend == DNN_TH) {
> if (ctx->model_inputname)
> av_log(filter_ctx, AV_LOG_WARNING, "LibTorch backend do not require
> inputname, "\
> - "inputname will be ignored.\n");
> + "inputname will be ignored.\n");
We don't need to add such format change in this commit.
And same comment for other similar changes.
_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
^ permalink raw reply [flat|nested] 3+ messages in thread
* Re: [FFmpeg-devel] [PATCH v2 FFmpeg 11/20] libavfilter/dnn_filter_common: dnn execute functions for CLIP and CLAP
2025-03-10 19:54 [FFmpeg-devel] [PATCH v2 FFmpeg 11/20] libavfilter/dnn_filter_common: dnn execute functions for CLIP and CLAP m.kaindl0208
2025-05-12 10:08 ` Guo, Yejun
@ 2025-05-12 10:11 ` Guo, Yejun
1 sibling, 0 replies; 3+ messages in thread
From: Guo, Yejun @ 2025-05-12 10:11 UTC (permalink / raw)
To: FFmpeg development discussions and patches
> -----Original Message-----
> From: ffmpeg-devel <ffmpeg-devel-bounces@ffmpeg.org> On Behalf Of
> m.kaindl0208@gmail.com
> Sent: Tuesday, March 11, 2025 3:54 AM
> To: ffmpeg-devel@ffmpeg.org
> Subject: [FFmpeg-devel] [PATCH v2 FFmpeg 11/20]
> libavfilter/dnn_filter_common: dnn execute functions for CLIP and CLAP
>
> Signed-off-by: MaximilianKaindl <m.kaindl0208@gmail.com>
> ---
> libavfilter/dnn_filter_common.c | 81 +++++++++++++++++++++++----------
> libavfilter/dnn_filter_common.h | 34 +++++++-------
> 2 files changed, 76 insertions(+), 39 deletions(-)
>
> diff --git a/libavfilter/dnn_filter_common.c b/libavfilter/dnn_filter_common.c
> index 899e869cca..f3117748ae 100644
> --- a/libavfilter/dnn_filter_common.c
> +++ b/libavfilter/dnn_filter_common.c
> +int ff_dnn_execute_model_clip(DnnContext *ctx, AVFrame *in_frame,
> AVFrame *out_frame, const char **labels, int label_count, const char*
> tokenizer_path, char *target)
> +{
> + DNNExecZeroShotClassificationParams class_params = {
> + {
> + .input_name = ctx->model_inputname,
> + .output_names = (const char **)ctx->model_outputnames,
> + .nb_output = ctx->nb_outputs,
> + .in_frame = in_frame,
> + .out_frame = out_frame,
> + },
> + .labels = labels,
> + .label_count = label_count,
> + .tokenizer_path = tokenizer_path,
Why we need tokenizer_path to execute model since it is already initialized in load_model_with_tokenizer from patch 10.
> + .target = target,
> + };
> + return (ctx->dnn_module->execute_model)(ctx->model,
> &class_params.base);
> +}
> +
_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
^ permalink raw reply [flat|nested] 3+ messages in thread
end of thread, other threads:[~2025-05-12 10:11 UTC | newest]
Thread overview: 3+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2025-03-10 19:54 [FFmpeg-devel] [PATCH v2 FFmpeg 11/20] libavfilter/dnn_filter_common: dnn execute functions for CLIP and CLAP m.kaindl0208
2025-05-12 10:08 ` Guo, Yejun
2025-05-12 10:11 ` Guo, Yejun
Git Inbox Mirror of the ffmpeg-devel mailing list - see https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
This inbox may be cloned and mirrored by anyone:
git clone --mirror https://master.gitmailbox.com/ffmpegdev/0 ffmpegdev/git/0.git
# If you have public-inbox 1.1+ installed, you may
# initialize and index your mirror using the following commands:
public-inbox-init -V2 ffmpegdev ffmpegdev/ https://master.gitmailbox.com/ffmpegdev \
ffmpegdev@gitmailbox.com
public-inbox-index ffmpegdev
Example config snippet for mirrors.
AGPL code for this site: git clone https://public-inbox.org/public-inbox.git