* [FFmpeg-devel] [PATCH 1/2] libavfilter/dnn_backend_openvino: Add dynamic output support
@ 2023-12-27 4:16 wenbin.chen-at-intel.com
2023-12-27 4:16 ` [FFmpeg-devel] [PATCH 2/2] libavfilter/vf_dnn_detect: Add two outputs ssd support wenbin.chen-at-intel.com
2023-12-29 10:37 ` [FFmpeg-devel] [PATCH 1/2] libavfilter/dnn_backend_openvino: Add dynamic output support Guo, Yejun
0 siblings, 2 replies; 3+ messages in thread
From: wenbin.chen-at-intel.com @ 2023-12-27 4:16 UTC (permalink / raw)
To: ffmpeg-devel
From: Wenbin Chen <wenbin.chen@intel.com>
Add dynamic outputs support. Some models don't have fixed output size.
Its size changes according to result. Now openvino can run these kinds of
models.
Signed-off-by: Wenbin Chen <wenbin.chen@intel.com>
---
libavfilter/dnn/dnn_backend_openvino.c | 134 +++++++++++--------------
1 file changed, 59 insertions(+), 75 deletions(-)
diff --git a/libavfilter/dnn/dnn_backend_openvino.c b/libavfilter/dnn/dnn_backend_openvino.c
index 671a995c70..e207d44584 100644
--- a/libavfilter/dnn/dnn_backend_openvino.c
+++ b/libavfilter/dnn/dnn_backend_openvino.c
@@ -219,31 +219,26 @@ static int fill_model_input_ov(OVModel *ov_model, OVRequestItem *request)
task = lltask->task;
#if HAVE_OPENVINO2
- if (!ov_model_is_dynamic(ov_model->ov_model)) {
- if (ov_model->input_port) {
- ov_output_const_port_free(ov_model->input_port);
- ov_model->input_port = NULL;
- }
- status = ov_model_const_input_by_name(ov_model->ov_model, task->input_name, &ov_model->input_port);
- if (status != OK) {
- av_log(ctx, AV_LOG_ERROR, "Failed to get input port shape.\n");
- return ov2_map_error(status, NULL);
- }
- status = ov_const_port_get_shape(ov_model->input_port, &input_shape);
- if (status != OK) {
- av_log(ctx, AV_LOG_ERROR, "Failed to get input port shape.\n");
- return ov2_map_error(status, NULL);
- }
- dims = input_shape.dims;
- status = ov_port_get_element_type(ov_model->input_port, &precision);
- if (status != OK) {
- av_log(ctx, AV_LOG_ERROR, "Failed to get input port data type.\n");
- ov_shape_free(&input_shape);
- return ov2_map_error(status, NULL);
- }
- } else {
- avpriv_report_missing_feature(ctx, "Do not support dynamic model.");
- return AVERROR(ENOSYS);
+ if (ov_model->input_port) {
+ ov_output_const_port_free(ov_model->input_port);
+ ov_model->input_port = NULL;
+ }
+ status = ov_model_const_input_by_name(ov_model->ov_model, task->input_name, &ov_model->input_port);
+ if (status != OK) {
+ av_log(ctx, AV_LOG_ERROR, "Failed to get input port shape.\n");
+ return ov2_map_error(status, NULL);
+ }
+ status = ov_const_port_get_shape(ov_model->input_port, &input_shape);
+ if (status != OK) {
+ av_log(ctx, AV_LOG_ERROR, "Failed to get input port shape.\n");
+ return ov2_map_error(status, NULL);
+ }
+ dims = input_shape.dims;
+ status = ov_port_get_element_type(ov_model->input_port, &precision);
+ if (status != OK) {
+ av_log(ctx, AV_LOG_ERROR, "Failed to get input port data type.\n");
+ ov_shape_free(&input_shape);
+ return ov2_map_error(status, NULL);
}
input.height = dims[1];
input.width = dims[2];
@@ -1049,30 +1044,22 @@ static int get_input_ov(void *model, DNNData *input, const char *input_name)
ov_element_type_e precision;
int64_t* dims;
ov_status_e status;
- if (!ov_model_is_dynamic(ov_model->ov_model)) {
- status = ov_model_const_input_by_name(ov_model->ov_model, input_name, &ov_model->input_port);
- if (status != OK) {
- av_log(ctx, AV_LOG_ERROR, "Failed to get input port shape.\n");
- return ov2_map_error(status, NULL);
- }
-
- status = ov_const_port_get_shape(ov_model->input_port, &input_shape);
- if (status != OK) {
- av_log(ctx, AV_LOG_ERROR, "Failed to get input port shape.\n");
- return ov2_map_error(status, NULL);
- }
- dims = input_shape.dims;
-
- status = ov_port_get_element_type(ov_model->input_port, &precision);
- if (status != OK) {
- av_log(ctx, AV_LOG_ERROR, "Failed to get input port data type.\n");
- return ov2_map_error(status, NULL);
- }
- } else {
- avpriv_report_missing_feature(ctx, "Do not support dynamic model now.");
- return AVERROR(ENOSYS);
+ status = ov_model_const_input_by_name(ov_model->ov_model, input_name, &ov_model->input_port);
+ if (status != OK) {
+ av_log(ctx, AV_LOG_ERROR, "Failed to get input port shape.\n");
+ return ov2_map_error(status, NULL);
}
-
+ status = ov_port_get_element_type(ov_model->input_port, &precision);
+ if (status != OK) {
+ av_log(ctx, AV_LOG_ERROR, "Failed to get input port data type.\n");
+ return ov2_map_error(status, NULL);
+ }
+ status = ov_const_port_get_shape(ov_model->input_port, &input_shape);
+ if (status != OK) {
+ av_log(ctx, AV_LOG_ERROR, "Failed to get input port shape.\n");
+ return ov2_map_error(status, NULL);
+ }
+ dims = input_shape.dims;
if (dims[1] <= 3) { // NCHW
input->channels = dims[1];
input->height = input_resizable ? -1 : dims[2];
@@ -1083,7 +1070,7 @@ static int get_input_ov(void *model, DNNData *input, const char *input_name)
input->channels = dims[3];
}
input->dt = precision_to_datatype(precision);
-
+ ov_shape_free(&input_shape);
return 0;
#else
char *model_input_name = NULL;
@@ -1267,34 +1254,31 @@ static int get_output_ov(void *model, const char *input_name, int input_width, i
#if HAVE_OPENVINO2
if (ctx->options.input_resizable) {
- if (!ov_model_is_dynamic(ov_model->ov_model)) {
- status = ov_partial_shape_create(4, dims, &partial_shape);
- if (status != OK) {
- av_log(ctx, AV_LOG_ERROR, "Failed create partial shape.\n");
- return ov2_map_error(status, NULL);
- }
- status = ov_const_port_get_shape(ov_model->input_port, &input_shape);
- input_shape.dims[2] = input_height;
- input_shape.dims[3] = input_width;
- if (status != OK) {
- av_log(ctx, AV_LOG_ERROR, "Failed create shape for model input resize.\n");
- return ov2_map_error(status, NULL);
- }
+ status = ov_partial_shape_create(4, dims, &partial_shape);
+ if (status != OK) {
+ av_log(ctx, AV_LOG_ERROR, "Failed to create partial shape.\n");
+ return ov2_map_error(status, NULL);
+ }
+ status = ov_const_port_get_shape(ov_model->input_port, &input_shape);
+ if (status != OK) {
+ av_log(ctx, AV_LOG_ERROR, "Failed to create shape for model input resize.\n");
+ return ov2_map_error(status, NULL);
+ }
+ input_shape.dims[2] = input_height;
+ input_shape.dims[3] = input_width;
- status = ov_shape_to_partial_shape(input_shape, &partial_shape);
- if (status != OK) {
- av_log(ctx, AV_LOG_ERROR, "Failed create partial shape for model input resize.\n");
- return ov2_map_error(status, NULL);
- }
+ status = ov_shape_to_partial_shape(input_shape, &partial_shape);
+ ov_shape_free(&input_shape);
+ if (status != OK) {
+ av_log(ctx, AV_LOG_ERROR, "Failed to create partial shape for model input resize.\n");
+ return ov2_map_error(status, NULL);
+ }
- status = ov_model_reshape_single_input(ov_model->ov_model, partial_shape);
- if (status != OK) {
- av_log(ctx, AV_LOG_ERROR, "Failed to reszie model input.\n");
- return ov2_map_error(status, NULL);
- }
- } else {
- avpriv_report_missing_feature(ctx, "Do not support dynamic model.");
- return AVERROR(ENOTSUP);
+ status = ov_model_reshape_single_input(ov_model->ov_model, partial_shape);
+ ov_partial_shape_free(&partial_shape);
+ if (status != OK) {
+ av_log(ctx, AV_LOG_ERROR, "Failed to reszie model input.\n");
+ return ov2_map_error(status, NULL);
}
}
--
2.34.1
_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
^ permalink raw reply [flat|nested] 3+ messages in thread
* [FFmpeg-devel] [PATCH 2/2] libavfilter/vf_dnn_detect: Add two outputs ssd support
2023-12-27 4:16 [FFmpeg-devel] [PATCH 1/2] libavfilter/dnn_backend_openvino: Add dynamic output support wenbin.chen-at-intel.com
@ 2023-12-27 4:16 ` wenbin.chen-at-intel.com
2023-12-29 10:37 ` [FFmpeg-devel] [PATCH 1/2] libavfilter/dnn_backend_openvino: Add dynamic output support Guo, Yejun
1 sibling, 0 replies; 3+ messages in thread
From: wenbin.chen-at-intel.com @ 2023-12-27 4:16 UTC (permalink / raw)
To: ffmpeg-devel
From: Wenbin Chen <wenbin.chen@intel.com>
For this kind of model, we can directly use its output as final result
just like ssd model. The difference is that it splits output into two
tensors. [x_min, y_min, x_max, y_max, confidence] and [lable_id].
Model example refer to: https://github.com/openvinotoolkit/open_model_zoo/tree/master/models/intel/person-detection-0106
Signed-off-by: Wenbin Chen <wenbin.chen@intel.com>
---
libavfilter/vf_dnn_detect.c | 64 +++++++++++++++++++++++++++++--------
1 file changed, 50 insertions(+), 14 deletions(-)
diff --git a/libavfilter/vf_dnn_detect.c b/libavfilter/vf_dnn_detect.c
index 88865c8a8e..249cbba0f7 100644
--- a/libavfilter/vf_dnn_detect.c
+++ b/libavfilter/vf_dnn_detect.c
@@ -359,24 +359,48 @@ static int dnn_detect_post_proc_yolov3(AVFrame *frame, DNNData *output,
return 0;
}
-static int dnn_detect_post_proc_ssd(AVFrame *frame, DNNData *output, AVFilterContext *filter_ctx)
+static int dnn_detect_post_proc_ssd(AVFrame *frame, DNNData *output, int nb_outputs,
+ AVFilterContext *filter_ctx)
{
DnnDetectContext *ctx = filter_ctx->priv;
float conf_threshold = ctx->confidence;
- int proposal_count = output->height;
- int detect_size = output->width;
- float *detections = output->data;
+ int proposal_count = 0;
+ int detect_size = 0;
+ float *detections = NULL, *labels = NULL;
int nb_bboxes = 0;
AVDetectionBBoxHeader *header;
AVDetectionBBox *bbox;
-
- if (output->width != 7) {
+ int scale_w = ctx->scale_width;
+ int scale_h = ctx->scale_height;
+
+ if (nb_outputs == 1 && output->width == 7) {
+ proposal_count = output->height;
+ detect_size = output->width;
+ detections = output->data;
+ } else if (nb_outputs == 2 && output[0].width == 5) {
+ proposal_count = output[0].height;
+ detect_size = output[0].width;
+ detections = output[0].data;
+ labels = output[1].data;
+ } else if (nb_outputs == 2 && output[1].width == 5) {
+ proposal_count = output[1].height;
+ detect_size = output[1].width;
+ detections = output[1].data;
+ labels = output[0].data;
+ } else {
av_log(filter_ctx, AV_LOG_ERROR, "Model output shape doesn't match ssd requirement.\n");
return AVERROR(EINVAL);
}
+ if (proposal_count == 0)
+ return 0;
+
for (int i = 0; i < proposal_count; ++i) {
- float conf = detections[i * detect_size + 2];
+ float conf;
+ if (nb_outputs == 1)
+ conf = detections[i * detect_size + 2];
+ else
+ conf = detections[i * detect_size + 4];
if (conf < conf_threshold) {
continue;
}
@@ -398,12 +422,24 @@ static int dnn_detect_post_proc_ssd(AVFrame *frame, DNNData *output, AVFilterCon
for (int i = 0; i < proposal_count; ++i) {
int av_unused image_id = (int)detections[i * detect_size + 0];
- int label_id = (int)detections[i * detect_size + 1];
- float conf = detections[i * detect_size + 2];
- float x0 = detections[i * detect_size + 3];
- float y0 = detections[i * detect_size + 4];
- float x1 = detections[i * detect_size + 5];
- float y1 = detections[i * detect_size + 6];
+ int label_id;
+ float conf, x0, y0, x1, y1;
+
+ if (nb_outputs == 1) {
+ label_id = (int)detections[i * detect_size + 1];
+ conf = detections[i * detect_size + 2];
+ x0 = detections[i * detect_size + 3];
+ y0 = detections[i * detect_size + 4];
+ x1 = detections[i * detect_size + 5];
+ y1 = detections[i * detect_size + 6];
+ } else {
+ label_id = (int)labels[i];
+ x0 = detections[i * detect_size] / scale_w;
+ y0 = detections[i * detect_size + 1] / scale_h;
+ x1 = detections[i * detect_size + 2] / scale_w;
+ y1 = detections[i * detect_size + 3] / scale_h;
+ conf = detections[i * detect_size + 4];
+ }
if (conf < conf_threshold) {
continue;
@@ -447,7 +483,7 @@ static int dnn_detect_post_proc_ov(AVFrame *frame, DNNData *output, int nb_outpu
switch (ctx->model_type) {
case DDMT_SSD:
- ret = dnn_detect_post_proc_ssd(frame, output, filter_ctx);
+ ret = dnn_detect_post_proc_ssd(frame, output, nb_outputs, filter_ctx);
if (ret < 0)
return ret;
break;
--
2.34.1
_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
^ permalink raw reply [flat|nested] 3+ messages in thread
* Re: [FFmpeg-devel] [PATCH 1/2] libavfilter/dnn_backend_openvino: Add dynamic output support
2023-12-27 4:16 [FFmpeg-devel] [PATCH 1/2] libavfilter/dnn_backend_openvino: Add dynamic output support wenbin.chen-at-intel.com
2023-12-27 4:16 ` [FFmpeg-devel] [PATCH 2/2] libavfilter/vf_dnn_detect: Add two outputs ssd support wenbin.chen-at-intel.com
@ 2023-12-29 10:37 ` Guo, Yejun
1 sibling, 0 replies; 3+ messages in thread
From: Guo, Yejun @ 2023-12-29 10:37 UTC (permalink / raw)
To: FFmpeg development discussions and patches
> -----Original Message-----
> From: ffmpeg-devel <ffmpeg-devel-bounces@ffmpeg.org> On Behalf Of
> wenbin.chen-at-intel.com@ffmpeg.org
> Sent: Wednesday, December 27, 2023 12:17 PM
> To: ffmpeg-devel@ffmpeg.org
> Subject: [FFmpeg-devel] [PATCH 1/2] libavfilter/dnn_backend_openvino: Add
> dynamic output support
>
> From: Wenbin Chen <wenbin.chen@intel.com>
>
> Add dynamic outputs support. Some models don't have fixed output size.
> Its size changes according to result. Now openvino can run these kinds of
> models.
>
> Signed-off-by: Wenbin Chen <wenbin.chen@intel.com>
> ---
> libavfilter/dnn/dnn_backend_openvino.c | 134 +++++++++++--------------
> 1 file changed, 59 insertions(+), 75 deletions(-)
>
> diff --git a/libavfilter/dnn/dnn_backend_openvino.c
> b/libavfilter/dnn/dnn_backend_openvino.c
> index 671a995c70..e207d44584 100644
> --- a/libavfilter/dnn/dnn_backend_openvino.c
> +++ b/libavfilter/dnn/dnn_backend_openvino.c
> @@ -219,31 +219,26 @@ static int fill_model_input_ov(OVModel
> *ov_model, OVRequestItem *request)
> task = lltask->task;
>
LGTM, will push tomorrow, thanks.
_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
^ permalink raw reply [flat|nested] 3+ messages in thread
end of thread, other threads:[~2023-12-29 10:37 UTC | newest]
Thread overview: 3+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2023-12-27 4:16 [FFmpeg-devel] [PATCH 1/2] libavfilter/dnn_backend_openvino: Add dynamic output support wenbin.chen-at-intel.com
2023-12-27 4:16 ` [FFmpeg-devel] [PATCH 2/2] libavfilter/vf_dnn_detect: Add two outputs ssd support wenbin.chen-at-intel.com
2023-12-29 10:37 ` [FFmpeg-devel] [PATCH 1/2] libavfilter/dnn_backend_openvino: Add dynamic output support Guo, Yejun
Git Inbox Mirror of the ffmpeg-devel mailing list - see https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
This inbox may be cloned and mirrored by anyone:
git clone --mirror https://master.gitmailbox.com/ffmpegdev/0 ffmpegdev/git/0.git
# If you have public-inbox 1.1+ installed, you may
# initialize and index your mirror using the following commands:
public-inbox-init -V2 ffmpegdev ffmpegdev/ https://master.gitmailbox.com/ffmpegdev \
ffmpegdev@gitmailbox.com
public-inbox-index ffmpegdev
Example config snippet for mirrors.
AGPL code for this site: git clone https://public-inbox.org/public-inbox.git