Re: [FFmpeg-devel] [PATCH 12/18] avcodec/vvcdec: inter, wait reference with a diffrent resolution

[Nuo Mi <nuomi2021@gmail.com>]

On Sun, May 19, 2024 at 11:21 PM Jean-Baptiste Kempf <jb@videolan.org>
wrote:

> Careful about the typo on "different" on the title of the patch.
>
Changed locally.
Thank you, jb

>
> On Sun, 19 May 2024, at 13:27, Nuo Mi wrote:
> > For RPR, the current frame may reference a frame with a different
> resolution.
> > Therefore, we need to consider frame scaling when we wait for reference
> pixels.
> > ---
> >  libavcodec/vvc/dec.c    |  5 +++++
> >  libavcodec/vvc/dec.h    | 17 +++++++++++++++++
> >  libavcodec/vvc/refs.c   | 39 +++++++++++++++++++++++++++++++++++++++
> >  libavcodec/vvc/thread.c | 10 +++++++---
> >  4 files changed, 68 insertions(+), 3 deletions(-)
> >
> > diff --git a/libavcodec/vvc/dec.c b/libavcodec/vvc/dec.c
> > index 3325133efb..584c7b219f 100644
> > --- a/libavcodec/vvc/dec.c
> > +++ b/libavcodec/vvc/dec.c
> > @@ -573,6 +573,11 @@ static int ref_frame(VVCFrame *dst, const VVCFrame
> *src)
> >
> >      dst->poc = src->poc;
> >      dst->ctb_count = src->ctb_count;
> > +
> > +    dst->scaling_win = src->scaling_win;
> > +    dst->ref_width   = src->ref_width;
> > +    dst->ref_height  = src->ref_height;
> > +
> >      dst->flags = src->flags;
> >      dst->sequence = src->sequence;
> >
> > diff --git a/libavcodec/vvc/dec.h b/libavcodec/vvc/dec.h
> > index 6f14cc1860..1e0b76f283 100644
> > --- a/libavcodec/vvc/dec.h
> > +++ b/libavcodec/vvc/dec.h
> > @@ -46,6 +46,10 @@ typedef struct VVCRefPic {
> >      struct VVCFrame *ref;
> >      int poc;
> >      int is_lt;                  // is long term reference
> > +
> > +    // for RPR
> > +    int is_scaled;              ///< RprConstraintsActiveFlag
> > +    int scale[2];               ///< RefPicScale[]
> >  } VVCRefPic;
> >
> >  typedef struct RefPicList {
> > @@ -57,6 +61,13 @@ typedef struct RefPicListTab {
> >      RefPicList refPicList[2];
> >  } RefPicListTab;
> >
> > +typedef struct VVCWindow {
> > +    int16_t left_offset;
> > +    int16_t right_offset;
> > +    int16_t top_offset;
> > +    int16_t bottom_offset;
> > +} VVCWindow;
> > +
> >  typedef struct VVCFrame {
> >      struct AVFrame *frame;
> >
> > @@ -71,6 +82,12 @@ typedef struct VVCFrame {
> >
> >      int poc;
> >
> > +    //for RPR
> > +    VVCWindow scaling_win;                      ///<
> > pps_scaling_win_left_offset * SubWithC,  pps_scaling_win_right_offset
> > * SubWithC,
> > +                                                ///<
> > pps_scaling_win_top_offset  * SubHeigtC, pps_scaling_win_bottom_offset
> > * SubHiehgtC
> > +    int ref_width;                              ///<
> > CurrPicScalWinWidthL
> > +    int ref_height;                             ///<
> > CurrPicScalWinHeightL
> > +
> >      struct VVCFrame *collocated_ref;
> >
> >      struct FrameProgress *progress;             ///< RefStruct reference
> > diff --git a/libavcodec/vvc/refs.c b/libavcodec/vvc/refs.c
> > index 954db4a8c8..fb42963034 100644
> > --- a/libavcodec/vvc/refs.c
> > +++ b/libavcodec/vvc/refs.c
> > @@ -114,10 +114,12 @@ static FrameProgress *alloc_progress(void)
> >
> >  static VVCFrame *alloc_frame(VVCContext *s, VVCFrameContext *fc)
> >  {
> > +    const VVCSPS *sps = fc->ps.sps;
> >      const VVCPPS *pps = fc->ps.pps;
> >      for (int i = 0; i < FF_ARRAY_ELEMS(fc->DPB); i++) {
> >          int ret;
> >          VVCFrame *frame = &fc->DPB[i];
> > +        VVCWindow *win = &frame->scaling_win;
> >          if (frame->frame->buf[0])
> >              continue;
> >
> > @@ -144,6 +146,13 @@ static VVCFrame *alloc_frame(VVCContext *s,
> > VVCFrameContext *fc)
> >          for (int j = 0; j < frame->ctb_count; j++)
> >              frame->rpl_tab[j] = frame->rpl;
> >
> > +        win->left_offset   = pps->r->pps_scaling_win_left_offset   <<
> > sps->hshift[CHROMA];
> > +        win->right_offset  = pps->r->pps_scaling_win_right_offset  <<
> > sps->hshift[CHROMA];
> > +        win->top_offset    = pps->r->pps_scaling_win_top_offset    <<
> > sps->vshift[CHROMA];
> > +        win->bottom_offset = pps->r->pps_scaling_win_bottom_offset <<
> > sps->vshift[CHROMA];
> > +        frame->ref_width   = pps->r->pps_pic_width_in_luma_samples  -
> > win->left_offset   - win->right_offset;
> > +        frame->ref_height  = pps->r->pps_pic_height_in_luma_samples -
> > win->bottom_offset - win->top_offset;
> > +
> >          frame->progress = alloc_progress();
> >          if (!frame->progress)
> >              goto fail;
> > @@ -353,6 +362,24 @@ static VVCFrame *generate_missing_ref(VVCContext
> > *s, VVCFrameContext *fc, int po
> >      return frame;
> >  }
> >
> > +#define CHECK_MAX(d) (frame->ref_##d *
> > frame->sps->r->sps_pic_##d##_max_in_luma_samples >= ref->ref_##d *
> > (frame->pps->r->pps_pic_##d##_in_luma_samples - max))
> > +#define CHECK_SAMPLES(d) (frame->pps->r->pps_pic_##d##_in_luma_samples
> > == ref->pps->r->pps_pic_##d##_in_luma_samples)
> > +static int check_candidate_ref(const VVCFrame *frame, const VVCRefPic
> > *refp)
> > +{
> > +    const VVCFrame *ref = refp->ref;
> > +
> > +    if (refp->is_scaled) {
> > +        const int max = FFMAX(8, frame->sps->min_cb_size_y);
> > +        return frame->ref_width * 2 >= ref->ref_width &&
> > +            frame->ref_height * 2 >= ref->ref_height &&
> > +            frame->ref_width <= ref->ref_width * 8 &&
> > +            frame->ref_height <= ref->ref_height * 8 &&
> > +            CHECK_MAX(width) && CHECK_MAX(height);
> > +    }
> > +    return CHECK_SAMPLES(width) && CHECK_SAMPLES(height);
> > +}
> > +
> > +#define RPR_SCALE(f) (((ref->f << 14) + (fc->ref->f >> 1)) /
> > fc->ref->f)
> >  /* add a reference with the given poc to the list and mark it as used
> > in DPB */
> >  static int add_candidate_ref(VVCContext *s, VVCFrameContext *fc,
> > RefPicList *list,
> >                               int poc, int ref_flag, uint8_t use_msb)
> > @@ -372,6 +399,18 @@ static int add_candidate_ref(VVCContext *s,
> > VVCFrameContext *fc, RefPicList *lis
> >      refp->poc = poc;
> >      refp->ref = ref;
> >      refp->is_lt = ref_flag & VVC_FRAME_FLAG_LONG_REF;
> > +    refp->is_scaled = ref->sps->r->sps_num_subpics_minus1 !=
> > fc->ref->sps->r->sps_num_subpics_minus1||
> > +        memcmp(&ref->scaling_win, &fc->ref->scaling_win,
> > sizeof(ref->scaling_win)) ||
> > +        ref->pps->r->pps_pic_width_in_luma_samples !=
> > fc->ref->pps->r->pps_pic_width_in_luma_samples ||
> > +        ref->pps->r->pps_pic_height_in_luma_samples !=
> > fc->ref->pps->r->pps_pic_height_in_luma_samples;
> > +
> > +    if (!check_candidate_ref(fc->ref, refp))
> > +        return AVERROR_INVALIDDATA;
> > +
> > +    if (refp->is_scaled) {
> > +        refp->scale[0] = RPR_SCALE(ref_width);
> > +        refp->scale[1] = RPR_SCALE(ref_height);
> > +    }
> >      list->nb_refs++;
> >
> >      mark_ref(ref, ref_flag);
> > diff --git a/libavcodec/vvc/thread.c b/libavcodec/vvc/thread.c
> > index 2654b40058..8777d380bf 100644
> > --- a/libavcodec/vvc/thread.c
> > +++ b/libavcodec/vvc/thread.c
> > @@ -293,10 +293,14 @@ static void schedule_inter(VVCContext *s,
> > VVCFrameContext *fc, const SliceContex
> >          CTU *ctu = fc->tab.ctus + rs;
> >          for (int lx = 0; lx < 2; lx++) {
> >              for (int i = 0; i < sh->r->num_ref_idx_active[lx]; i++) {
> > -                const int y = ctu->max_y[lx][i];
> > -                VVCFrame *ref = sc->rpl[lx].refs[i].ref;
> > -                if (ref && y >= 0)
> > +                int y = ctu->max_y[lx][i];
> > +                VVCRefPic *refp = sc->rpl[lx].refs + i;
> > +                VVCFrame *ref   = refp->ref;
> > +                if (ref && y >= 0) {
> > +                    if (refp->is_scaled)
> > +                        y = y * refp->scale[1] >> 14;
> >                      add_progress_listener(ref, &t->listener[lx][i], t,
> > s, VVC_PROGRESS_PIXEL, y + LUMA_EXTRA_AFTER);
> > +                }
> >              }
> >          }
> >      }
> > --
> > 2.34.1
> >
> > _______________________________________________
> > ffmpeg-devel mailing list
> > ffmpeg-devel@ffmpeg.org
> > https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
> >
> > To unsubscribe, visit link above, or email
> > ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
>
> --
> Jean-Baptiste Kempf -  President
> +33 672 704 734
> https://jbkempf.com/
> _______________________________________________
> ffmpeg-devel mailing list
> ffmpeg-devel@ffmpeg.org
> https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
>
> To unsubscribe, visit link above, or email
> ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
>
_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".