From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from ffbox0-bg.mplayerhq.hu (ffbox0-bg.ffmpeg.org [79.124.17.100]) by master.gitmailbox.com (Postfix) with ESMTP id BBB4D46250 for ; Wed, 10 May 2023 13:55:27 +0000 (UTC) Received: from [127.0.1.1] (localhost [127.0.0.1]) by ffbox0-bg.mplayerhq.hu (Postfix) with ESMTP id 0BA7D68C0BC; Wed, 10 May 2023 16:55:24 +0300 (EEST) Received: from haasn.dev (haasn.dev [78.46.187.166]) by ffbox0-bg.mplayerhq.hu (Postfix) with ESMTP id 1C665680BEB for ; Wed, 10 May 2023 16:55:17 +0300 (EEST) Received: from localhost (217-74-0-168.hsi.r-kom.net [217.74.0.168]) by haasn.dev (Postfix) with ESMTPSA id 8D35D4994A; Wed, 10 May 2023 15:55:16 +0200 (CEST) DKIM-Signature: v=1; a=rsa-sha256; c=simple/simple; d=haasn.xyz; s=mail; t=1683726916; bh=s4wbMZ7MbpanTczp2cz7snSv8fvSZlB5HMwrOzuraO4=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=jqyePTkVLcrmnfwPs1bKBj49USSPlQJHfPPbi7X+jnPgrq380pa6SjbCtG5D+uGIJ urECctyl9akl61CdgE/k8sFyWHnO8qPDIzopV+NKiEQdLdqjt8p28RewFsa9b7Ne0F oCGBxRFEQdKDJsPZwS7WRvOp/niHfT5OHm3LMfP4= From: Niklas Haas To: ffmpeg-devel@ffmpeg.org Date: Wed, 10 May 2023 15:55:06 +0200 Message-Id: <20230510135509.4074-3-ffmpeg@haasn.xyz> X-Mailer: git-send-email 2.40.1 In-Reply-To: <20230510135509.4074-1-ffmpeg@haasn.xyz> References: <20230510135509.4074-1-ffmpeg@haasn.xyz> MIME-Version: 1.0 Subject: [FFmpeg-devel] [PATCH 3/6] lavfi/vf_libplacebo: switch to pl_queue-based design X-BeenThere: ffmpeg-devel@ffmpeg.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: FFmpeg development discussions and patches List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Reply-To: FFmpeg development discussions and patches Cc: Niklas Haas Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit Errors-To: ffmpeg-devel-bounces@ffmpeg.org Sender: "ffmpeg-devel" Archived-At: List-Archive: List-Post: From: Niklas Haas This does not leverage any immediate benefits, but refactors and prepares the codebase for upcoming changes, which will include the ability to do deinterlacing and resampling (frame mixing). --- libavfilter/vf_libplacebo.c | 214 ++++++++++++++++++++++++------------ 1 file changed, 144 insertions(+), 70 deletions(-) diff --git a/libavfilter/vf_libplacebo.c b/libavfilter/vf_libplacebo.c index e84f81c143..afb4e6a914 100644 --- a/libavfilter/vf_libplacebo.c +++ b/libavfilter/vf_libplacebo.c @@ -16,6 +16,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA */ +#include "libavutil/avassert.h" #include "libavutil/eval.h" #include "libavutil/file.h" #include "libavutil/opt.h" @@ -26,6 +27,7 @@ #include #include +#include #include /* Backwards compatibility with older libplacebo */ @@ -122,7 +124,8 @@ typedef struct LibplaceboContext { pl_vulkan vulkan; pl_gpu gpu; pl_renderer renderer; - pl_tex tex[8]; + pl_queue queue; + pl_tex tex[4]; /* settings */ char *out_format_string; @@ -392,6 +395,7 @@ static int update_settings(AVFilterContext *ctx) .num_hooks = s->num_hooks, .skip_anti_aliasing = s->skip_aa, + .skip_caching_single_frame = true, .polar_cutoff = s->polar_cutoff, .disable_linear_scaling = s->disable_linear, .disable_builtin_scalers = s->disable_builtin, @@ -518,6 +522,7 @@ static int init_vulkan(AVFilterContext *avctx) /* Create the renderer */ s->gpu = s->vulkan->gpu; s->renderer = pl_renderer_create(s->log, s->gpu); + s->queue = pl_queue_create(s->gpu); /* Parse the user shaders, if requested */ if (s->shader_bin_len) @@ -544,6 +549,7 @@ static void libplacebo_uninit(AVFilterContext *avctx) for (int i = 0; i < s->num_hooks; i++) pl_mpv_user_shader_destroy(&s->hooks[i]); pl_renderer_destroy(&s->renderer); + pl_queue_destroy(&s->queue); pl_vulkan_destroy(&s->vulkan); pl_log_destroy(&s->log); ff_vk_uninit(&s->vkctx); @@ -573,69 +579,92 @@ fail: } static void update_crops(AVFilterContext *ctx, - struct pl_frame *image, - struct pl_frame *target, - const double target_pts) + struct pl_frame_mix *mix, struct pl_frame *target, + uint64_t ref_sig, double base_pts) { LibplaceboContext *s = ctx->priv; - const AVFrame *in = pl_get_mapped_avframe(image); - const AVFilterLink *inlink = ctx->inputs[0]; - const double in_pts = in->pts * av_q2d(inlink->time_base); - - s->var_values[VAR_IN_T] = s->var_values[VAR_T] = in_pts; - s->var_values[VAR_OUT_T] = s->var_values[VAR_OT] = target_pts; - s->var_values[VAR_N] = inlink->frame_count_out; - - /* Clear these explicitly to avoid leaking previous frames' state */ - s->var_values[VAR_CROP_W] = s->var_values[VAR_CW] = NAN; - s->var_values[VAR_CROP_H] = s->var_values[VAR_CH] = NAN; - s->var_values[VAR_POS_W] = s->var_values[VAR_PW] = NAN; - s->var_values[VAR_POS_H] = s->var_values[VAR_PH] = NAN; - - /* Evaluate crop/pos dimensions first, and placement second */ - s->var_values[VAR_CROP_W] = s->var_values[VAR_CW] = - av_expr_eval(s->crop_w_pexpr, s->var_values, NULL); - s->var_values[VAR_CROP_H] = s->var_values[VAR_CH] = - av_expr_eval(s->crop_h_pexpr, s->var_values, NULL); - s->var_values[VAR_POS_W] = s->var_values[VAR_PW] = - av_expr_eval(s->pos_w_pexpr, s->var_values, NULL); - s->var_values[VAR_POS_H] = s->var_values[VAR_PH] = - av_expr_eval(s->pos_h_pexpr, s->var_values, NULL); - - image->crop.x0 = av_expr_eval(s->crop_x_pexpr, s->var_values, NULL); - image->crop.y0 = av_expr_eval(s->crop_y_pexpr, s->var_values, NULL); - image->crop.x1 = image->crop.x0 + s->var_values[VAR_CROP_W]; - image->crop.y1 = image->crop.y0 + s->var_values[VAR_CROP_H]; - - target->crop.x0 = av_expr_eval(s->pos_x_pexpr, s->var_values, NULL); - target->crop.y0 = av_expr_eval(s->pos_y_pexpr, s->var_values, NULL); - target->crop.x1 = target->crop.x0 + s->var_values[VAR_POS_W]; - target->crop.y1 = target->crop.y0 + s->var_values[VAR_POS_H]; - - if (s->target_sar.num) { - float aspect = pl_rect2df_aspect(&target->crop) * av_q2d(s->target_sar); - pl_rect2df_aspect_set(&target->crop, aspect, s->pad_crop_ratio); + + for (int i = 0; i < mix->num_frames; i++) { + // Mutate the `pl_frame.crop` fields in-place. This is fine because we + // own the entire pl_queue, and hence, the pointed-at frames. + struct pl_frame *image = (struct pl_frame *) mix->frames[i]; + double image_pts = base_pts + mix->timestamps[i]; + + /* Update dynamic variables */ + s->var_values[VAR_IN_T] = s->var_values[VAR_T] = image_pts; + s->var_values[VAR_OUT_T] = s->var_values[VAR_OT] = base_pts; + s->var_values[VAR_N] = ctx->outputs[0]->frame_count_out; + + /* Clear these explicitly to avoid leaking previous frames' state */ + s->var_values[VAR_CROP_W] = s->var_values[VAR_CW] = NAN; + s->var_values[VAR_CROP_H] = s->var_values[VAR_CH] = NAN; + s->var_values[VAR_POS_W] = s->var_values[VAR_PW] = NAN; + s->var_values[VAR_POS_H] = s->var_values[VAR_PH] = NAN; + + /* Compute dimensions first and placement second */ + s->var_values[VAR_CROP_W] = s->var_values[VAR_CW] = + av_expr_eval(s->crop_w_pexpr, s->var_values, NULL); + s->var_values[VAR_CROP_H] = s->var_values[VAR_CH] = + av_expr_eval(s->crop_h_pexpr, s->var_values, NULL); + s->var_values[VAR_POS_W] = s->var_values[VAR_PW] = + av_expr_eval(s->pos_w_pexpr, s->var_values, NULL); + s->var_values[VAR_POS_H] = s->var_values[VAR_PH] = + av_expr_eval(s->pos_h_pexpr, s->var_values, NULL); + + image->crop.x0 = av_expr_eval(s->crop_x_pexpr, s->var_values, NULL); + image->crop.y0 = av_expr_eval(s->crop_y_pexpr, s->var_values, NULL); + image->crop.x1 = image->crop.x0 + s->var_values[VAR_CROP_W]; + image->crop.y1 = image->crop.y0 + s->var_values[VAR_CROP_H]; + + if (mix->signatures[i] == ref_sig) { + /* Only update the target crop once, for the 'reference' frame */ + target->crop.x0 = av_expr_eval(s->pos_x_pexpr, s->var_values, NULL); + target->crop.y0 = av_expr_eval(s->pos_y_pexpr, s->var_values, NULL); + target->crop.x1 = target->crop.x0 + s->var_values[VAR_POS_W]; + target->crop.y1 = target->crop.y0 + s->var_values[VAR_POS_H]; + + if (s->target_sar.num) { + float aspect = pl_rect2df_aspect(&target->crop) * av_q2d(s->target_sar); + pl_rect2df_aspect_set(&target->crop, aspect, s->pad_crop_ratio); + } + } } } -/* Construct and emit an output frame for `image` */ -static int output_frame(AVFilterContext *ctx, struct pl_frame *image) +/* Construct and emit an output frame for a given frame mix */ +static int output_frame_mix(AVFilterContext *ctx, + struct pl_frame_mix *mix, + int64_t pts) { int err = 0, ok, changed_csp; LibplaceboContext *s = ctx->priv; AVFilterLink *outlink = ctx->outputs[0]; const AVPixFmtDescriptor *outdesc = av_pix_fmt_desc_get(outlink->format); - AVFrame *out = ff_get_video_buffer(outlink, outlink->w, outlink->h); - const AVFrame *in = pl_get_mapped_avframe(image); struct pl_frame target; + const AVFrame *ref; + AVFrame *out; + uint64_t ref_sig; + if (!mix->num_frames) + return 0; + + out = ff_get_video_buffer(outlink, outlink->w, outlink->h); if (!out) return AVERROR(ENOMEM); - RET(av_frame_copy_props(out, in)); + /* Use the last frame before current PTS value as reference */ + for (int i = 0; i < mix->num_frames; i++) { + if (i && mix->timestamps[i] > 0.0f) + break; + ref = pl_get_mapped_avframe(mix->frames[i]); + ref_sig = mix->signatures[i]; + } + + RET(av_frame_copy_props(out, ref)); + out->pts = pts; out->width = outlink->w; out->height = outlink->h; - if (s->apply_dovi && av_frame_get_side_data(in, AV_FRAME_DATA_DOVI_METADATA)) { + if (s->apply_dovi && av_frame_get_side_data(ref, AV_FRAME_DATA_DOVI_METADATA)) { /* Output of dovi reshaping is always BT.2020+PQ, so infer the correct * output colorspace defaults */ out->colorspace = AVCOL_SPC_BT2020_NCL; @@ -652,10 +681,10 @@ static int output_frame(AVFilterContext *ctx, struct pl_frame *image) if (s->color_primaries >= 0) out->color_primaries = s->color_primaries; - changed_csp = in->colorspace != out->colorspace || - in->color_range != out->color_range || - in->color_trc != out->color_trc || - in->color_primaries != out->color_primaries; + changed_csp = ref->colorspace != out->colorspace || + ref->color_range != out->color_range || + ref->color_trc != out->color_trc || + ref->color_primaries != out->color_primaries; /* Strip side data if no longer relevant */ if (changed_csp) { @@ -677,15 +706,15 @@ static int output_frame(AVFilterContext *ctx, struct pl_frame *image) .map_dovi = false, )); } else { - ok = pl_frame_recreate_from_avframe(s->gpu, &target, s->tex + 4, out); + ok = pl_frame_recreate_from_avframe(s->gpu, &target, s->tex, out); } if (!ok) { err = AVERROR_EXTERNAL; goto fail; } - update_crops(ctx, image, &target, out->pts * av_q2d(outlink->time_base)); - pl_render_image(s->renderer, image, &target, &s->params); + update_crops(ctx, mix, &target, ref_sig, out->pts * av_q2d(outlink->time_base)); + pl_render_image_mix(s->renderer, mix, &target, &s->params); if (outdesc->flags & AV_PIX_FMT_FLAG_HWACCEL) { pl_unmap_avframe(s->gpu, &target); @@ -700,33 +729,78 @@ fail: return err; } +static bool map_frame(pl_gpu gpu, pl_tex *tex, + const struct pl_source_frame *src, + struct pl_frame *out) +{ + AVFrame *avframe = src->frame_data; + LibplaceboContext *s = avframe->opaque; + bool ok = pl_map_avframe_ex(gpu, out, pl_avframe_params( + .frame = avframe, + .tex = tex, + .map_dovi = s->apply_dovi, + )); + + if (!s->apply_filmgrain) + out->film_grain.type = PL_FILM_GRAIN_NONE; + + av_frame_free(&avframe); + return ok; +} + +static void unmap_frame(pl_gpu gpu, struct pl_frame *frame, + const struct pl_source_frame *src) +{ + pl_unmap_avframe(gpu, frame); +} + +static void discard_frame(const struct pl_source_frame *src) +{ + AVFrame *avframe = src->frame_data; + av_frame_free(&avframe); +} + static int filter_frame(AVFilterLink *link, AVFrame *in) { - int ret, ok; AVFilterContext *ctx = link->dst; LibplaceboContext *s = ctx->priv; - struct pl_frame image; + AVFilterLink *outlink = ctx->outputs[0]; + enum pl_queue_status status; + struct pl_frame_mix mix; pl_log_level_update(s->log, get_log_level()); - /* Map input frame */ - ok = pl_map_avframe_ex(s->gpu, &image, pl_avframe_params( - .frame = in, - .tex = s->tex, - .map_dovi = s->apply_dovi, + /* Push input frame */ + in->opaque = s; + pl_queue_push(s->queue, &(struct pl_source_frame) { + .pts = in->pts * av_q2d(link->time_base), + .duration = in->duration * av_q2d(link->time_base), + .first_field = pl_field_from_avframe(in), + .frame_data = in, + .map = map_frame, + .unmap = unmap_frame, + .discard = discard_frame, + }); + + /* Immediately return an output frame for the same PTS */ + av_assert1(!av_cmp_q(link->time_base, outlink->time_base)); + status = pl_queue_update(s->queue, &mix, pl_queue_params( + .pts = in->pts * av_q2d(outlink->time_base), + .radius = pl_frame_mix_radius(&s->params), + .vsync_duration = av_q2d(av_inv_q(outlink->frame_rate)), )); - av_frame_free(&in); - - if (!s->apply_filmgrain) - image.film_grain.type = PL_FILM_GRAIN_NONE; - if (!ok) + switch (status) { + case PL_QUEUE_MORE: // TODO: switch to activate() and handle properly + case PL_QUEUE_OK: + return output_frame_mix(ctx, &mix, in->pts); + case PL_QUEUE_EOF: + return 0; + case PL_QUEUE_ERR: return AVERROR_EXTERNAL; + } - ret = output_frame(ctx, &image); - - pl_unmap_avframe(s->gpu, &image); - return ret; + return AVERROR_BUG; } static int libplacebo_query_format(AVFilterContext *ctx) -- 2.40.1 _______________________________________________ ffmpeg-devel mailing list ffmpeg-devel@ffmpeg.org https://ffmpeg.org/mailman/listinfo/ffmpeg-devel To unsubscribe, visit link above, or email ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".