Re: [FFmpeg-devel] [PATCH v3 01/14] vvcdec: add vvc decoder stub

[Andreas Rheinhardt <andreas.rheinhardt@outlook.com>]

Nuo Mi:
> ---
>  configure               |   1 +
>  libavcodec/Makefile     |   1 +
>  libavcodec/allcodecs.c  |   1 +
>  libavcodec/vvc/Makefile |   4 +
>  libavcodec/vvc/vvcdec.c |  84 +++++++++++
>  libavcodec/vvc/vvcdec.h | 307 ++++++++++++++++++++++++++++++++++++++++
>  6 files changed, 398 insertions(+)
>  create mode 100644 libavcodec/vvc/Makefile
>  create mode 100644 libavcodec/vvc/vvcdec.c
>  create mode 100644 libavcodec/vvc/vvcdec.h
> 
> diff --git a/configure b/configure
> index 8a1902810a..442c439c3e 100755
> --- a/configure
> +++ b/configure
> @@ -3025,6 +3025,7 @@ vp6f_decoder_select="vp6_decoder"
>  vp7_decoder_select="h264pred videodsp vp8dsp"
>  vp8_decoder_select="h264pred videodsp vp8dsp"
>  vp9_decoder_select="videodsp vp9_parser vp9_superframe_split_bsf"
> +vvc_decoder_select="cabac golomb videodsp"
>  wcmv_decoder_select="inflate_wrapper"
>  webp_decoder_select="vp8_decoder exif"
>  wmalossless_decoder_select="llauddsp"
> diff --git a/libavcodec/Makefile b/libavcodec/Makefile
> index 08fd151619..fb2738d8eb 100644
> --- a/libavcodec/Makefile
> +++ b/libavcodec/Makefile
> @@ -62,6 +62,7 @@ OBJS = ac3_parser.o                                                     \
>         xiph.o                                                           \
>  
>  # subsystems
> +include $(SRC_PATH)/libavcodec/vvc/Makefile

I don't see why vvc should have a subfolder of its own.

>  OBJS-$(CONFIG_AANDCTTABLES)            += aandcttab.o
>  OBJS-$(CONFIG_AC3DSP)                  += ac3dsp.o ac3.o ac3tab.o
>  OBJS-$(CONFIG_ADTS_HEADER)             += adts_header.o mpeg4audio_sample_rates.o
> diff --git a/libavcodec/allcodecs.c b/libavcodec/allcodecs.c
> index 6e95ca5636..c16d80cb85 100644
> --- a/libavcodec/allcodecs.c
> +++ b/libavcodec/allcodecs.c
> @@ -388,6 +388,7 @@ extern const FFCodec ff_vp9_rkmpp_decoder;
>  extern const FFCodec ff_vp9_v4l2m2m_decoder;
>  extern const FFCodec ff_vqa_decoder;
>  extern const FFCodec ff_vqc_decoder;
> +extern const FFCodec ff_vvc_decoder;
>  extern const FFCodec ff_wbmp_decoder;
>  extern const FFCodec ff_wbmp_encoder;
>  extern const FFCodec ff_webp_decoder;
> diff --git a/libavcodec/vvc/Makefile b/libavcodec/vvc/Makefile
> new file mode 100644
> index 0000000000..bd14dbc1df
> --- /dev/null
> +++ b/libavcodec/vvc/Makefile
> @@ -0,0 +1,4 @@
> +clean::
> +	$(RM) $(CLEANSUFFIXES:%=libavcodec/vvc/%)
> +
> +OBJS-$(CONFIG_VVC_DECODER)          +=  vvc/vvcdec.o            \
> diff --git a/libavcodec/vvc/vvcdec.c b/libavcodec/vvc/vvcdec.c
> new file mode 100644
> index 0000000000..8d027af0b9
> --- /dev/null
> +++ b/libavcodec/vvc/vvcdec.c
> @@ -0,0 +1,84 @@
> +/*
> + * VVC video decoder
> + *
> + * Copyright (C) 2021 Nuo Mi
> + * Copyright (C) 2022 Xu Mu
> + *
> + * This file is part of FFmpeg.
> + *
> + * FFmpeg is free software; you can redistribute it and/or
> + * modify it under the terms of the GNU Lesser General Public
> + * License as published by the Free Software Foundation; either
> + * version 2.1 of the License, or (at your option) any later version.
> + *
> + * FFmpeg is distributed in the hope that it will be useful,
> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
> + * Lesser General Public License for more details.
> + *
> + * You should have received a copy of the GNU Lesser General Public
> + * License along with FFmpeg; if not, write to the Free Software
> + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
> + */
> +#include "config_components.h"

What do you need this for?

> +
> +#include "libavcodec/codec_internal.h"
> +#include "libavcodec/decode.h"
> +#include "libavcodec/golomb.h"
> +#include "libavcodec/profiles.h"
> +#include "libavcodec/vvc.h"
> +
> +#include "libavutil/cpu.h"

Why would you ever need cpu.h in the decoder?
Even if a later patch needs it, it should be added in the later patch.

> +
> +#include "vvcdec.h"
> +
> +static int vvc_decode_frame(AVCodecContext *avctx, AVFrame *output,
> +    int *got_output, AVPacket *avpkt)
> +{
> +    return avpkt->size;
> +}
> +
> +static void vvc_decode_flush(AVCodecContext *avctx)
> +{
> +}
> +
> +static av_cold int vvc_decode_free(AVCodecContext *avctx)
> +{
> +    return 0;
> +}
> +
> +static av_cold int vvc_decode_init(AVCodecContext *avctx)
> +{
> +    return 0;
> +}
> +
> +#define OFFSET(x) offsetof(VVCContext, x)
> +#define PAR (AV_OPT_FLAG_DECODING_PARAM | AV_OPT_FLAG_VIDEO_PARAM)
> +
> +static const AVOption options[] = {
> +    { NULL },
> +};
> +
> +static const AVClass vvc_decoder_class = {
> +    .class_name = "vvc decoder",
> +    .item_name  = av_default_item_name,
> +    .option     = options,
> +    .version    = LIBAVUTIL_VERSION_INT,
> +};

This seems to be based upon H.264, but the H.264 decoder has options,
whereas this decoder doesn't, so this AVClass and the options are
unnecessary.

> +
> +const FFCodec ff_vvc_decoder = {
> +    .p.name                  = "vvc",
> +    .p.long_name             = NULL_IF_CONFIG_SMALL("VVC (Versatile Video Coding)"),
> +    .p.type                  = AVMEDIA_TYPE_VIDEO,
> +    .p.id                    = AV_CODEC_ID_VVC,
> +    .priv_data_size          = sizeof(VVCContext),
> +    .p.priv_class            = &vvc_decoder_class,
> +    .init                    = vvc_decode_init,
> +    .close                   = vvc_decode_free,
> +    FF_CODEC_DECODE_CB(vvc_decode_frame),
> +    .flush                   = vvc_decode_flush,
> +    .p.capabilities          = AV_CODEC_CAP_DR1 | AV_CODEC_CAP_DELAY | AV_CODEC_CAP_OTHER_THREADS,
> +    .caps_internal           = FF_CODEC_CAP_EXPORTS_CROPPING | FF_CODEC_CAP_INIT_CLEANUP |
> +                               FF_CODEC_CAP_AUTO_THREADS,
> +    .p.profiles              = NULL_IF_CONFIG_SMALL(ff_vvc_profiles),
> +};
> diff --git a/libavcodec/vvc/vvcdec.h b/libavcodec/vvc/vvcdec.h
> new file mode 100644
> index 0000000000..698ab89b9d
> --- /dev/null
> +++ b/libavcodec/vvc/vvcdec.h
> @@ -0,0 +1,307 @@
> +/*
> + * VVC video decoder
> + *
> + * Copyright (C) 2021 Nuo Mi
> + * Copyright (C) 2022 Xu Mu
> + *
> + * This file is part of FFmpeg.
> + *
> + * FFmpeg is free software; you can redistribute it and/or
> + * modify it under the terms of the GNU Lesser General Public
> + * License as published by the Free Software Foundation; either
> + * version 2.1 of the License, or (at your option) any later version.
> + *
> + * FFmpeg is distributed in the hope that it will be useful,
> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
> + * Lesser General Public License for more details.
> + *
> + * You should have received a copy of the GNU Lesser General Public
> + * License along with FFmpeg; if not, write to the Free Software
> + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
> + */
> +
> +#ifndef AVCODEC_VVCDEC_H
> +#define AVCODEC_VVCDEC_H
> +
> +#include "libavcodec/cbs.h"
> +#include "libavcodec/cbs_h266.h"
> +#include "libavcodec/h2645_parse.h"
> +#include "libavcodec/threadframe.h"
> +#include "libavcodec/videodsp.h"
> +#include "libavcodec/vvc.h"
> +#include "libavutil/executor.h"
> +
> +#define LUMA                    0
> +#define CHROMA                  1
> +#define CB                      1
> +#define CR                      2
> +#define JCBCR                   3
> +
> +#define MAX_CTU_SIZE            128
> +
> +#define MAX_CU_SIZE             MAX_CTU_SIZE
> +#define MIN_CU_SIZE             4
> +#define MIN_CU_LOG2             2
> +#define MAX_CU_DEPTH            7
> +
> +#define MIN_PU_SIZE             4
> +#define MIN_PU_LOG2             2
> +
> +#define MAX_TB_SIZE             64
> +#define MIN_TU_LOG2             2                       ///< MinTbLog2SizeY
> +#define MIN_TU_SIZE             4
> +#define MAX_TUS_IN_CU           64
> +
> +#define MAX_PARTS_IN_CTU        ((MAX_CTU_SIZE >> MIN_CU_LOG2) * (MAX_CTU_SIZE >> MIN_CU_LOG2))
> +
> +#define MAX_CONTROL_POINTS      3
> +
> +#define MRG_MAX_NUM_CANDS       6
> +#define MAX_NUM_HMVP_CANDS      5
> +
> +#define L0                      0
> +#define L1                      1
> +
> +#define CHROMA_EXTRA_BEFORE     1
> +#define CHROMA_EXTRA_AFTER      2
> +#define CHROMA_EXTRA            3
> +#define LUMA_EXTRA_BEFORE       3
> +#define LUMA_EXTRA_AFTER        4
> +#define LUMA_EXTRA              7
> +#define BILINEAR_EXTRA_BEFORE   0
> +#define BILINEAR_EXTRA_AFTER    1
> +#define BILINEAR_EXTRA          1
> +
> +#define MAX_QP                  63
> +#define DEFAULT_INTRA_TC_OFFSET 2
> +
> +#define SAO_PADDING_SIZE        1
> +
> +#define ALF_PADDING_SIZE        8
> +#define ALF_BLOCK_SIZE          4
> +
> +#define ALF_BORDER_LUMA         3
> +#define ALF_BORDER_CHROMA       2
> +
> +#define ALF_VB_POS_ABOVE_LUMA   4
> +#define ALF_VB_POS_ABOVE_CHROMA 2
> +
> +#define ALF_GRADIENT_STEP       2
> +#define ALF_GRADIENT_BORDER     2
> +#define ALF_GRADIENT_SIZE       ((MAX_CU_SIZE + ALF_GRADIENT_BORDER * 2) / ALF_GRADIENT_STEP)
> +#define ALF_NUM_DIR             4
> +
> +#define MAX_PB_SIZE             128
> +#define EDGE_EMU_BUFFER_STRIDE  (MAX_PB_SIZE + 32)
> +
> +#define AFFINE_MIN_BLOCK_SIZE   4
> +#define PROF_BORDER_EXT         1
> +#define PROF_BLOCK_SIZE         (AFFINE_MIN_BLOCK_SIZE + PROF_BORDER_EXT * 2)
> +#define BDOF_BORDER_EXT         1
> +
> +#define BDOF_PADDED_SIZE        (16 + BDOF_BORDER_EXT * 2)
> +#define BDOF_BLOCK_SIZE         4
> +#define BDOF_GRADIENT_SIZE      (BDOF_BLOCK_SIZE + BDOF_BORDER_EXT * 2)
> +
> +/**
> + * Value of the luma sample at position (x, y) in the 2D array tab.
> + */
> +#define SAMPLE(tab, x, y) ((tab)[(y) * s->sps->width + (x)])
> +#define SAMPLE_CTB(tab, x, y) ((tab)[(y) * min_cb_width + (x)])
> +#define CTB(tab, x, y) ((tab)[(y) * fc->ps.pps->ctb_width + (x)])
> +
> +typedef struct VVCLocalContext VVCLocalContext;
> +typedef struct SliceContext SliceContext;
> +typedef struct VVCFrameContext  VVCFrameContext;
> +typedef struct VVCFrameThread VVCFrameThread;
> +typedef struct EntryPoint EntryPoint;
> +typedef struct VVCTask VVCTask;
> +typedef struct Mv Mv;
> +typedef struct MvField MvField;
> +typedef struct DMVRInfo DMVRInfo;
> +typedef struct CTU CTU;
> +typedef struct SAOParams SAOParams;
> +typedef struct ALFParams ALFParams;
> +
> +typedef struct RefPicList {
> +    struct VVCFrame *ref[VVC_MAX_REF_ENTRIES];
> +    int list[VVC_MAX_REF_ENTRIES];
> +    int isLongTerm[VVC_MAX_REF_ENTRIES];
> +    int nb_refs;
> +} RefPicList;
> +
> +typedef struct RefPicListTab {
> +    RefPicList refPicList[2];
> +} RefPicListTab;
> +
> +typedef struct VVCFrame {
> +    AVFrame *frame;
> +    ThreadFrame tf;

You are not using the ThreadFrame API at all; you can simply remove this
by switching to ff_thread_get_buffer() instead of
ff_thread_get_ext_buffer().

> +
> +    MvField  *tab_dmvr_mvf;
> +    RefPicListTab **rpl_tab;
> +
> +    int ctb_count;
> +
> +    int poc;
> +
> +    struct VVCFrame *collocated_ref;
> +
> +    AVBufferRef *tab_dmvr_mvf_buf;
> +    AVBufferRef *rpl_tab_buf;
> +    AVBufferRef *rpl_buf;
> +    AVBufferRef *progress_buf;
> +
> +    /**
> +     * A sequence counter, so that old frames are output first
> +     * after a POC reset
> +     */
> +    uint16_t sequence;
> +    /**
> +     * A combination of VVC_FRAME_FLAG_*
> +     */
> +    uint8_t flags;
> +} VVCFrame;
> +
> +struct SliceContext {
> +    int slice_idx;
> +    EntryPoint *eps;
> +    int nb_eps;
> +    RefPicList *rpl;
> +};
> +
> +struct VVCFrameContext {
> +    AVCodecContext *avctx;

Lots of files of the H.264 and HEVC decoder actually don't need
AVCodecContext as complete type at all, they can work with a forward
declaration. It is probably the same for this one, so could you use
struct AVCodecContext* in the header?

> +
> +    // +1 for the current frame
> +    VVCFrame DPB[VVC_MAX_DPB_SIZE + 1];
> +
> +    AVFrame *frame;
> +    AVFrame *output_frame;
> +
> +    SliceContext  **slices;
> +    int nb_slices;
> +    int nb_slices_allocated;
> +
> +    VVCFrame *ref;
> +
> +    VideoDSPContext vdsp;
> +
> +    VVCFrameThread *frame_thread;
> +
> +    uint64_t decode_order;
> +
> +    AVBufferPool *tab_dmvr_mvf_pool;
> +    AVBufferPool *rpl_tab_pool;
> +
> +    AVBufferPool *cu_pool;
> +    AVBufferPool *tu_pool;
> +
> +    struct {
> +        int16_t *slice_idx;
> +
> +        DMVRInfo  *dmvr;
> +
> +        int     *cb_pos_x[2];                           ///< CbPosX[][][]
> +        int     *cb_pos_y[2];                           ///< CbPosY[][][]
> +        uint8_t *cb_width[2];                           ///< CbWidth[][][]
> +        uint8_t *cb_height[2];                          ///< CbHeight[][][]
> +        uint8_t *cqt_depth[2];                          ///< CqtDepth[][][]
> +        int8_t  *qp[VVC_MAX_SAMPLE_ARRAYS];
> +
> +        uint8_t *skip;                                  ///< CuSkipFlag[][]
> +        uint8_t *ispmf;                                 ///< intra_sub_partitions_mode_flag
> +        uint8_t *msm[2];                                ///< MttSplitMode[][][] in 32 pixels
> +        uint8_t *imf;                                   ///< IntraMipFlag[][]
> +        uint8_t *imtf;                                  ///< intra_mip_transposed_flag[][]
> +        uint8_t *imm;                                   ///< intra_mip_mode[][]
> +        uint8_t *ipm;                                   ///< IntraPredModeY[][]
> +        uint8_t *cpm[2];                                ///< CuPredMode[][][]
> +        uint8_t *msf;                                   ///< MergeSubblockFlag[][]
> +        uint8_t *iaf;                                   ///< InterAffineFlag[][]
> +        uint8_t *mmi;                                   ///< MotionModelIdc[][]
> +        Mv      *cp_mv[2];                              ///< CpMvLX[][][][MAX_CONTROL_POINTS];
> +        MvField *mvf;                                   ///< MvDmvrL0, MvDmvrL1
> +
> +        uint8_t *tu_coded_flag[VVC_MAX_SAMPLE_ARRAYS];  ///< tu_y_coded_flag[][],  tu_cb_coded_flag[][],  tu_cr_coded_flag[][]
> +        uint8_t *tu_joint_cbcr_residual_flag;           ///< tu_joint_cbcr_residual_flag[][]
> +        int     *tb_pos_x0[2];
> +        int     *tb_pos_y0[2];
> +        uint8_t *tb_width[2];
> +        uint8_t *tb_height[2];
> +        uint8_t *pcmf[2];
> +
> +        uint8_t *horizontal_bs[VVC_MAX_SAMPLE_ARRAYS];
> +        uint8_t *vertical_bs[VVC_MAX_SAMPLE_ARRAYS];
> +        uint8_t *horizontal_p;                          ///< horizontal maxFilterLengthPs for luma
> +        uint8_t *horizontal_q;                          ///< horizontal maxFilterLengthPs for luma
> +        uint8_t *vertical_p;                            ///< vertical   maxFilterLengthQs for luma
> +        uint8_t *vertical_q;                            ///< vertical   maxFilterLengthQs for luma
> +
> +        uint8_t *sao_pixel_buffer_h[VVC_MAX_SAMPLE_ARRAYS];
> +        uint8_t *sao_pixel_buffer_v[VVC_MAX_SAMPLE_ARRAYS];
> +        uint8_t *alf_pixel_buffer_h[VVC_MAX_SAMPLE_ARRAYS][2];
> +        uint8_t *alf_pixel_buffer_v[VVC_MAX_SAMPLE_ARRAYS][2];
> +
> +        int     *coeffs;
> +        CTU     *ctus;
> +
> +        //used in arrays_init only
> +        int ctu_count;
> +        int ctu_size;
> +        int pic_size_in_min_cb;
> +        int pic_size_in_min_pu;
> +        int pic_size_in_min_tu;
> +        int ctu_width;
> +        int ctu_height;
> +        int width;
> +        int height;
> +        int chroma_format_idc;
> +        int pixel_shift;
> +        int bs_width;
> +        int bs_height;
> +    } tab;
> +} ;
> +
> +typedef struct VVCContext {
> +    const AVClass *c;       // needed by private avoptions
> +    AVCodecContext *avctx;
> +
> +    CodedBitstreamContext *cbc;
> +    CodedBitstreamFragment current_frame;
> +
> +    int temporal_id;        ///< temporal_id_plus1 - 1
> +    int poc_tid0;
> +
> +    int eos;                ///< current packet contains an EOS/EOB NAL
> +    int last_eos;           ///< last packet contains an EOS/EOB NAL
> +
> +
> +    enum VVCNALUnitType vcl_unit_type;
> +    int no_output_before_recovery_flag; ///< NoOutputBeforeRecoveryFlag
> +    int gdr_recovery_point_poc;         ///< recoveryPointPocVal
> +
> +    /**
> +     * Sequence counters for decoded and output frames, so that old
> +     * frames are output first after a POC reset
> +     */
> +    uint16_t seq_decode;
> +    uint16_t seq_output;
> +
> +    int is_nalff;           ///< this flag is != 0 if bitstream is encapsulated
> +                            ///< as a format defined in 14496-15
> +
> +    int apply_defdispwin;
> +    int nal_length_size;    ///< Number of bytes used for nal length (1, 2 or 4)
> +
> +    AVExecutor *executor;

Same as above with AVCodecContext: It seems only vvc_thread.c will ever
need to have executor.h included, so by using struct AVExecutor you can
avoid the inclusion.

> +
> +    VVCFrameContext *fcs;
> +    int nb_fcs;
> +
> +    uint64_t nb_frames;     ///< processed frames
> +    int nb_delayed;         ///< delayed frames
> +}  VVCContext ;
> +
> +#endif /* AVCODEC_VVCDEC_H */

_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".