[FFmpeg-devel] [PATCH v2 FFmpeg 11/20] libavfilter/dnn_filter_common: dnn execute functions for CLIP and CLAP

* [FFmpeg-devel] [PATCH v2 FFmpeg 11/20] libavfilter/dnn_filter_common: dnn execute functions for CLIP and CLAP
@ 2025-03-10 19:54 m.kaindl0208
  2025-05-12 10:08 ` Guo, Yejun
  2025-05-12 10:11 ` Guo, Yejun
  0 siblings, 2 replies; 3+ messages in thread
From: m.kaindl0208 @ 2025-03-10 19:54 UTC (permalink / raw)
  To: ffmpeg-devel

Signed-off-by: MaximilianKaindl <m.kaindl0208@gmail.com>
---
 libavfilter/dnn_filter_common.c | 81 +++++++++++++++++++++++----------
 libavfilter/dnn_filter_common.h | 34 +++++++-------
 2 files changed, 76 insertions(+), 39 deletions(-)

diff --git a/libavfilter/dnn_filter_common.c b/libavfilter/dnn_filter_common.c
index 899e869cca..f3117748ae 100644
--- a/libavfilter/dnn_filter_common.c
+++ b/libavfilter/dnn_filter_common.c
@@ -1,20 +1,20 @@
 /*
- * This file is part of FFmpeg.
- *
- * FFmpeg is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * FFmpeg is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with FFmpeg; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
- */
+* This file is part of FFmpeg.
+*
+* FFmpeg is free software; you can redistribute it and/or
+* modify it under the terms of the GNU Lesser General Public
+* License as published by the Free Software Foundation; either
+* version 2.1 of the License, or (at your option) any later version.
+*
+* FFmpeg is distributed in the hope that it will be useful,
+* but WITHOUT ANY WARRANTY; without even the implied warranty of
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+* Lesser General Public License for more details.
+*
+* You should have received a copy of the GNU Lesser General Public
+* License along with FFmpeg; if not, write to the Free Software
+* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+*/

 #include "dnn_filter_common.h"
 #include "libavutil/avstring.h"
@@ -87,10 +87,10 @@ static int ff_dnn_init_priv(DnnContext *ctx, DNNFunctionType func_type, AVFilter
     if (backend == DNN_TH) {
         if (ctx->model_inputname)
             av_log(filter_ctx, AV_LOG_WARNING, "LibTorch backend do not require inputname, "\
-                                               "inputname will be ignored.\n");
+                                                "inputname will be ignored.\n");
         if (ctx->model_outputnames)
             av_log(filter_ctx, AV_LOG_WARNING, "LibTorch backend do not require outputname(s), "\
-                                               "all outputname(s) will be ignored.\n");
+                                            "all outputname(s) will be ignored.\n");

 #if (CONFIG_LIBTOKENIZERS == 0)
         if ((func_type == DFT_ANALYTICS_CLIP || func_type == DFT_ANALYTICS_CLAP)) {
@@ -126,14 +126,14 @@ static int ff_dnn_init_priv(DnnContext *ctx, DNNFunctionType func_type, AVFilter
         void *child = NULL;

         av_log(filter_ctx, AV_LOG_WARNING,
-               "backend_configs is deprecated, please set backend options directly\n");
+                "backend_configs is deprecated, please set backend options directly\n");
         while (child = ff_dnn_child_next(ctx, child)) {
             if (*(const AVClass **)child == &ctx->dnn_module->clazz) {
                 int ret = av_opt_set_from_string(child, ctx->backend_options,
-                                                 NULL, "=", "&");
+                                                NULL, "=", "&");
                 if (ret < 0) {
                     av_log(filter_ctx, AV_LOG_ERROR, "failed to parse options \"%s\"\n",
-                           ctx->backend_options);
+                            ctx->backend_options);
                     return ret;
                 }
             }
@@ -199,9 +199,9 @@ int ff_dnn_get_input(DnnContext *ctx, DNNData *input)
 int ff_dnn_get_output(DnnContext *ctx, int input_width, int input_height, int *output_width, int *output_height)
 {
     char * output_name = ctx->model_outputnames && ctx->backend_type != DNN_TH ?
-                         ctx->model_outputnames[0] : NULL;
+                        ctx->model_outputnames[0] : NULL;
     return ctx->model->get_output(ctx->model, ctx->model_inputname, input_width, input_height,
-                                  (const char *)output_name, output_width, output_height);
+                                (const char *)output_name, output_width, output_height);
 }

 int ff_dnn_execute_model(DnnContext *ctx, AVFrame *in_frame, AVFrame *out_frame)
@@ -231,6 +231,41 @@ int ff_dnn_execute_model_classification(DnnContext *ctx, AVFrame *in_frame, AVFr
     return (ctx->dnn_module->execute_model)(ctx->model, &class_params.base);
 }

+int ff_dnn_execute_model_clip(DnnContext *ctx, AVFrame *in_frame, AVFrame *out_frame, const char **labels, int label_count, const char* tokenizer_path, char *target)
+{
+    DNNExecZeroShotClassificationParams class_params = {
+        {
+            .input_name     = ctx->model_inputname,
+            .output_names   = (const char **)ctx->model_outputnames,
+            .nb_output      = ctx->nb_outputs,
+            .in_frame       = in_frame,
+            .out_frame      = out_frame,
+        },
+        .labels = labels,
+        .label_count = label_count,
+        .tokenizer_path = tokenizer_path,
+        .target = target,
+    };
+    return (ctx->dnn_module->execute_model)(ctx->model, &class_params.base);
+}
+
+int ff_dnn_execute_model_clap(DnnContext *ctx, AVFrame *in_frame, AVFrame *out_frame, const char **labels, int label_count, const char* tokenizer_path)
+{
+    DNNExecZeroShotClassificationParams class_params = {
+        {
+            .input_name     = ctx->model_inputname,
+            .output_names   = (const char **)ctx->model_outputnames,
+            .nb_output      = ctx->nb_outputs,
+            .in_frame       = in_frame,
+            .out_frame      = out_frame,
+        },
+        .labels = labels,
+        .label_count = label_count,
+        .tokenizer_path = tokenizer_path,
+    };
+    return (ctx->dnn_module->execute_model)(ctx->model, &class_params.base);
+}
+
 DNNAsyncStatusType ff_dnn_get_result(DnnContext *ctx, AVFrame **in_frame, AVFrame **out_frame)
 {
     return (ctx->dnn_module->get_result)(ctx->model, in_frame, out_frame);
diff --git a/libavfilter/dnn_filter_common.h b/libavfilter/dnn_filter_common.h
index 301441a37a..a782f64a1f 100644
--- a/libavfilter/dnn_filter_common.h
+++ b/libavfilter/dnn_filter_common.h
@@ -1,20 +1,20 @@
 /*
- * This file is part of FFmpeg.
- *
- * FFmpeg is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * FFmpeg is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with FFmpeg; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
- */
+* This file is part of FFmpeg.
+*
+* FFmpeg is free software; you can redistribute it and/or
+* modify it under the terms of the GNU Lesser General Public
+* License as published by the Free Software Foundation; either
+* version 2.1 of the License, or (at your option) any later version.
+*
+* FFmpeg is distributed in the hope that it will be useful,
+* but WITHOUT ANY WARRANTY; without even the implied warranty of
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+* Lesser General Public License for more details.
+*
+* You should have received a copy of the GNU Lesser General Public
+* License along with FFmpeg; if not, write to the Free Software
+* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+*/

 /**
  * @file
@@ -63,6 +63,8 @@ int ff_dnn_get_input(DnnContext *ctx, DNNData *input);
 int ff_dnn_get_output(DnnContext *ctx, int input_width, int input_height, int *output_width, int *output_height);
 int ff_dnn_execute_model(DnnContext *ctx, AVFrame *in_frame, AVFrame *out_frame);
 int ff_dnn_execute_model_classification(DnnContext *ctx, AVFrame *in_frame, AVFrame *out_frame, const char *target);
+int ff_dnn_execute_model_clip(DnnContext *ctx, AVFrame *in_frame, AVFrame *out_frame, const char **labels, int label_count, const char* tokenizer_path, char *target);
+int ff_dnn_execute_model_clap(DnnContext *ctx, AVFrame *in_frame, AVFrame *out_frame, const char **labels, int label_count, const char* tokenizer_path);
 DNNAsyncStatusType ff_dnn_get_result(DnnContext *ctx, AVFrame **in_frame, AVFrame **out_frame);
 int ff_dnn_flush(DnnContext *ctx);
 void ff_dnn_uninit(DnnContext *ctx);
--
2.34.1


_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".

^ permalink raw reply	[flat|nested] 3+ messages in thread