[FFmpeg-devel] [PATCH v2 FFmpeg 10/20] libavfilter/dnn_filter_common: add support for new loading function with ff_dnn_init_with_tokenizer

[FFmpeg-devel] [PATCH v2 FFmpeg 10/20] libavfilter/dnn_filter_common: add support for new loading function with ff_dnn_init_with_tokenizer

[m.kaindl0208]

Signed-off-by: MaximilianKaindl <m.kaindl0208@gmail.com>
---
 libavfilter/dnn_filter_common.c | 33 ++++++++++++++++++++++++++++++++-
 libavfilter/dnn_filter_common.h |  1 +
 2 files changed, 33 insertions(+), 1 deletion(-)

diff --git a/libavfilter/dnn_filter_common.c b/libavfilter/dnn_filter_common.c
index c4ad000409..899e869cca 100644
--- a/libavfilter/dnn_filter_common.c
+++ b/libavfilter/dnn_filter_common.c
@@ -75,7 +75,7 @@ void *ff_dnn_filter_child_next(void *obj, void *prev)
     return ff_dnn_child_next(&base->dnnctx, prev);
 }

-int ff_dnn_init(DnnContext *ctx, DNNFunctionType func_type, AVFilterContext *filter_ctx)
+static int ff_dnn_init_priv(DnnContext *ctx, DNNFunctionType func_type, AVFilterContext *filter_ctx)
 {
     DNNBackendType backend = ctx->backend_type;

@@ -91,6 +91,14 @@ int ff_dnn_init(DnnContext *ctx, DNNFunctionType func_type, AVFilterContext *fil
         if (ctx->model_outputnames)
             av_log(filter_ctx, AV_LOG_WARNING, "LibTorch backend do not require outputname(s), "\
                                                "all outputname(s) will be ignored.\n");
+
+#if (CONFIG_LIBTOKENIZERS == 0)
+        if ((func_type == DFT_ANALYTICS_CLIP || func_type == DFT_ANALYTICS_CLAP)) {
+            av_log(ctx, AV_LOG_ERROR,
+                "tokenizers-cpp is not included. CLIP/CLAP Classification requires tokenizers-cpp library. Include it with configure.\n");
+            return AVERROR(EINVAL);
+        }
+#endif
         ctx->nb_outputs = 1;
     } else if (backend == DNN_TF) {
         if (!ctx->model_inputname) {
@@ -131,13 +139,36 @@ int ff_dnn_init(DnnContext *ctx, DNNFunctionType func_type, AVFilterContext *fil
             }
         }
     }
+    return 0;
+}

+int ff_dnn_init(DnnContext *ctx, DNNFunctionType func_type, AVFilterContext *filter_ctx)
+{
+    int ret = ff_dnn_init_priv(ctx, func_type, filter_ctx);
+    if (ret < 0) {
+        return ret;
+    }
     ctx->model = (ctx->dnn_module->load_model)(ctx, func_type, filter_ctx);
     if (!ctx->model) {
         av_log(filter_ctx, AV_LOG_ERROR, "could not load DNN model\n");
         return AVERROR(EINVAL);
     }
+    return 0;
+}

+int ff_dnn_init_with_tokenizer(DnnContext *ctx, DNNFunctionType func_type, char **labels, int label_count,
+                            int *softmax_units, int softmax_units_count, char *tokenizer_path, AVFilterContext *filter_ctx)
+{
+    int ret = ff_dnn_init_priv(ctx, func_type, filter_ctx);
+    if (ret < 0) {
+        return ret;
+    }
+    ctx->model = (ctx->dnn_module->load_model_with_tokenizer)(ctx, func_type, labels, label_count, softmax_units,
+                                                            softmax_units_count, tokenizer_path, filter_ctx);
+    if (!ctx->model) {
+        av_log(filter_ctx, AV_LOG_ERROR, "could not load DNN model\n");
+        return AVERROR(EINVAL);
+    }
     return 0;
 }

diff --git a/libavfilter/dnn_filter_common.h b/libavfilter/dnn_filter_common.h
index fffa676a9e..301441a37a 100644
--- a/libavfilter/dnn_filter_common.h
+++ b/libavfilter/dnn_filter_common.h
@@ -55,6 +55,7 @@ void *ff_dnn_filter_child_next(void *obj, void *prev);
 int ff_dnn_filter_init_child_class(AVFilterContext *filter);

 int ff_dnn_init(DnnContext *ctx, DNNFunctionType func_type, AVFilterContext *filter_ctx);
+int ff_dnn_init_with_tokenizer(DnnContext *ctx, DNNFunctionType func_type, char** labels, int label_count, int* softmax_units, int softmax_units_count, char* tokenizer_path, AVFilterContext *filter_ctx);
 int ff_dnn_set_frame_proc(DnnContext *ctx, FramePrePostProc pre_proc, FramePrePostProc post_proc);
 int ff_dnn_set_detect_post_proc(DnnContext *ctx, DetectPostProc post_proc);
 int ff_dnn_set_classify_post_proc(DnnContext *ctx, ClassifyPostProc post_proc);
--
2.34.1


_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".