diff --git a/libavcodec/h264_mb.c b/libavcodec/h264_mb.c index 0d6562b583..400d769fce 100644 --- a/libavcodec/h264_mb.c +++ b/libavcodec/h264_mb.c @@ -37,6 +37,68 @@ #include "rectangle.h" #include "threadframe.h" +/** + * Collects detailed mode, reference, and motion vector information for the + * current macroblock and stores it in the picture's mb_info buffer. This allows + * the information to be passed to filters via frame side data. + */ +static void ff_h264_collect_mb_info(const H264Context *h, H264SliceContext *sl) +{ + // Check for NULL pointers at the very beginning. + if (!h->cur_pic_ptr) { + /* av_log(h->avctx, AV_LOG_ERROR, "collect_mb_info: h->cur_pic_ptr is NULL! mb_xy=%d\n", sl->mb_xy); */ + return; + } + + if (!h->cur_pic_ptr->mb_info_ref) { + return; + } + + // Check for out-of-bounds access. + if (sl->mb_xy >= h->mb_num) { + /* av_log(h->avctx, AV_LOG_ERROR, "collect_mb_info: mb_xy out of bounds! mb_xy=%d, mb_num=%d\n", sl->mb_xy, h->mb_num); */ + return; + } + + // Get the data pointer from the buffer + H264MBInfo *mb_info = (H264MBInfo*)h->cur_pic_ptr->mb_info_ref->data; + H264MBInfo *info = &mb_info[sl->mb_xy]; + int mb_type = h->cur_pic.mb_type[sl->mb_xy]; + int i, list; + + // Clear previous info to avoid stale data + memset(info, 0, sizeof(H264MBInfo)); + + info->mb_type = mb_type; + + if (IS_INTRA(mb_type)) { + if (IS_INTRA4x4(mb_type)) { + for (i = 0; i < 16; i++) + info->intra.intra4x4_pred_mode[i] = sl->intra4x4_pred_mode_cache[scan8[i]]; + } else { + info->intra.intra16x16_pred_mode = sl->intra16x16_pred_mode; + } + info->intra.chroma_pred_mode = sl->chroma_pred_mode; + } else { // Inter modes + if (IS_8X8(mb_type)) { + for (i = 0; i < 4; i++) + info->inter.sub_mb_type[i] = sl->sub_mb_type[i]; + } + + for (list = 0; list < 2; list++) { + // Check if the list is used by the macroblock partition or any sub-partition + if (USES_LIST(mb_type, list) || (IS_8X8(mb_type) && USES_LIST(info->inter.sub_mb_type[0]|info->inter.sub_mb_type[1]|info->inter.sub_mb_type[2]|info->inter.sub_mb_type[3], list))) { + // Store ref_idx and MVs for all 16 4x4 blocks + for (i = 0; i < 16; i++) { + info->inter.ref_idx[list][i] = sl->ref_cache[list][scan8[i]]; + info->inter.mv[list][i][0] = sl->mv_cache[list][scan8[i]][0]; + info->inter.mv[list][i][1] = sl->mv_cache[list][scan8[i]][1]; + } + } + } + } +} + static inline int get_lowest_part_list_y(H264SliceContext *sl, int n, int height, int y_offset, int list) { diff --git a/libavcodec/h264_mb_info.h b/libavcodec/h264_mb_info.h new file mode 100644 index 0000000000..104d6f5662 --- /dev/null +++ b/libavcodec/h264_mb_info.h @@ -0,0 +1,27 @@ +#ifndef AVCODEC_H264_MB_INFO_H +#define AVCODEC_H264_MB_INFO_H + +#include + +typedef struct H264MBInfo { + uint32_t mb_type; // The base macroblock type from H.264 specs + + union { + // Information for Intra-coded macroblocks + struct { + int8_t intra4x4_pred_mode[16]; + uint8_t intra16x16_pred_mode; + uint8_t chroma_pred_mode; + } intra; + + // Information for Inter-coded macroblocks + struct { + uint8_t sub_mb_type[4]; // Type for each 8x8 partition + // For each of the 16 4x4 blocks, store ref_idx and MV for L0 and L1 + int8_t ref_idx[2][16]; + int16_t mv[2][16][2]; + } inter; + }; +} H264MBInfo; + +#endif /* AVCODEC_H264_MB_INFO_H */ diff --git a/libavcodec/h264_mb_template.c b/libavcodec/h264_mb_template.c index d5ea26a6e3..5c5ea2ae4c 100644 --- a/libavcodec/h264_mb_template.c +++ b/libavcodec/h264_mb_template.c @@ -53,6 +53,9 @@ static av_noinline void FUNC(hl_decode_mb)(const H264Context *h, H264SliceContex const int block_h = 16 >> h->chroma_y_shift; const int chroma422 = CHROMA422(h); + // Collect macroblock information after decoding + ff_h264_collect_mb_info(h, sl); + dest_y = h->cur_pic.f->data[0] + ((mb_x << PIXEL_SHIFT) + mb_y * sl->linesize) * 16; dest_cb = h->cur_pic.f->data[1] + (mb_x << PIXEL_SHIFT) * 8 + mb_y * sl->uvlinesize * block_h; dest_cr = h->cur_pic.f->data[2] + (mb_x << PIXEL_SHIFT) * 8 + mb_y * sl->uvlinesize * block_h; diff --git a/libavcodec/h264_picture.c b/libavcodec/h264_picture.c index f5d2b31cd6..767e17f83e 100644 --- a/libavcodec/h264_picture.c +++ b/libavcodec/h264_picture.c @@ -35,6 +35,7 @@ #include "libavutil/refstruct.h" #include "thread.h" #include "threadframe.h" +#include "libavutil/mem.h" void ff_h264_unref_picture(H264Picture *pic) { @@ -56,6 +57,7 @@ void ff_h264_unref_picture(H264Picture *pic) av_refstruct_unref(&pic->ref_index[i]); } av_refstruct_unref(&pic->decode_error_flags); + av_buffer_unref(&pic->mb_info_ref); memset((uint8_t*)pic + off, 0, sizeof(*pic) - off); } @@ -103,6 +105,7 @@ static void h264_copy_picture_params(H264Picture *dst, const H264Picture *src) dst->mb_height = src->mb_height; dst->mb_stride = src->mb_stride; dst->needs_fg = src->needs_fg; + dst->mb_info_ref = av_buffer_ref(src->mb_info_ref); } int ff_h264_ref_picture(H264Picture *dst, const H264Picture *src) diff --git a/libavcodec/h264_slice.c b/libavcodec/h264_slice.c index 7e53e38cca..f441361d6f 100644 --- a/libavcodec/h264_slice.c +++ b/libavcodec/h264_slice.c @@ -266,6 +266,13 @@ static int alloc_picture(H264Context *h, H264Picture *pic) pic->mb_height = h->mb_height; pic->mb_stride = h->mb_stride; + // Allocate the mb_info buffer for this picture. + pic->mb_info_ref = av_buffer_allocz(h->mb_num * sizeof(H264MBInfo)); + av_log(h->avctx, AV_LOG_DEBUG, "Allocated mb_info buffer for pic %p (size: %zu)\n", pic, (size_t)h->mb_num * sizeof(H264MBInfo)); + + if (!pic->mb_info_ref) + goto fail; + return 0; fail: ff_h264_unref_picture(pic); diff --git a/libavcodec/h264dec.c b/libavcodec/h264dec.c index 82b85b3387..1560ab1b33 100644 --- a/libavcodec/h264dec.c +++ b/libavcodec/h264dec.c @@ -887,6 +887,29 @@ static int output_frame(H264Context *h, AVFrame *dst, H264Picture *srcp) goto fail; } + av_log(h->avctx, AV_LOG_ERROR, "Will try to attach the macroblock info inside as side data\n"); + + // Attach the macroblock info from the source picture (srcp). + if (srcp->mb_info_ref) { + AVFrameSideData *side_data; + size_t mb_info_size = srcp->mb_info_ref->size; + + av_log(h->avctx, AV_LOG_DEBUG, "Attaching mb_info from pic %p to frame %"PRId64"\n", srcp, dst->pts); + + // Create a new side data entry and copy the data into it. + side_data = av_frame_new_side_data(dst, AV_FRAME_DATA_H264_MB_INFO, mb_info_size); + if (!side_data) { + av_log(h->avctx, AV_LOG_ERROR, "Failed to allocate side data for MB info.\n"); + } else { + av_log(h->avctx, AV_LOG_ERROR, "Copying side data for MB info.\n"); + memcpy(side_data->data, srcp->mb_info_ref->data, mb_info_size); + } + } else { + av_log(h->avctx, AV_LOG_WARNING, "output_frame: srcp->mb_info_ref was NULL for pic %p. No side data to attach.\n", srcp); + } + + av_log(h->avctx, AV_LOG_ERROR, "End of block attach the macroblock info inside as side data\n"); + if (!(h->avctx->export_side_data & AV_CODEC_EXPORT_DATA_FILM_GRAIN)) av_frame_remove_side_data(dst, AV_FRAME_DATA_FILM_GRAIN_PARAMS); diff --git a/libavcodec/h264dec.h b/libavcodec/h264dec.h index c28d278240..fe15075c64 100644 --- a/libavcodec/h264dec.h +++ b/libavcodec/h264dec.h @@ -45,6 +45,7 @@ #include "mpegutils.h" #include "threadframe.h" #include "videodsp.h" +#include "h264_mb_info.h" #define H264_MAX_PICTURE_COUNT 36 @@ -164,6 +165,9 @@ typedef struct H264Picture { atomic_int *decode_error_flags; int gray; + + // Buffer to store macroblock mode information for this picture. + AVBufferRef *mb_info_ref; } H264Picture; typedef struct H264Ref { diff --git a/libavfilter/vf_codecview.c b/libavfilter/vf_codecview.c index a4a701b00c..9635f6420d 100644 --- a/libavfilter/vf_codecview.c +++ b/libavfilter/vf_codecview.c @@ -39,6 +39,11 @@ #include "qp_table.h" #include "video.h" +#include "libavcodec/h264.h" +#include "libavcodec/h264pred.h" +#include "libavcodec/h264_mb_info.h" +#include "libavcodec/mpegutils.h" + #define MV_P_FOR (1<<0) #define MV_B_FOR (1<<1) #define MV_B_BACK (1<<2) @@ -56,6 +61,8 @@ typedef struct CodecViewContext { int hsub, vsub; int qp; int block; + int show_modes; + int frame_count; } CodecViewContext; #define OFFSET(x) offsetof(CodecViewContext, x) @@ -78,9 +85,55 @@ static const AVOption codecview_options[] = { CONST("pf", "P-frames", FRAME_TYPE_P, "frame_type"), CONST("bf", "B-frames", FRAME_TYPE_B, "frame_type"), { "block", "set block partitioning structure to visualize", OFFSET(block), AV_OPT_TYPE_BOOL, {.i64=0}, 0, 1, FLAGS }, + { "show_modes", "Visualize macroblock modes", OFFSET(show_modes), AV_OPT_TYPE_BOOL, {.i64=0}, 0, 1, FLAGS }, { NULL } }; +static const char *get_intra_4x4_mode_name(int mode) { + switch (mode) { + case VERT_PRED: return "V"; + case HOR_PRED: return "H"; + case DC_PRED: return "DC"; + case DIAG_DOWN_LEFT_PRED: return "DL"; + case DIAG_DOWN_RIGHT_PRED: return "DR"; + case VERT_RIGHT_PRED: return "VR"; + case HOR_DOWN_PRED: return "HD"; + case VERT_LEFT_PRED: return "VL"; + case HOR_UP_PRED: return "HU"; + default: return "?"; + } +} + +static const char *get_intra_16x16_mode_name(int mode) { + switch (mode) { + case VERT_PRED8x8: return "Vertical"; + case HOR_PRED8x8: return "Horizontal"; + case DC_PRED8x8: return "DC"; + case PLANE_PRED8x8: return "Plane"; + default: return "Unknown"; + } +} + +static const char *get_chroma_mode_name(int mode) { + switch (mode) { + case DC_PRED8x8: return "DC"; + case HOR_PRED8x8: return "H"; + case VERT_PRED8x8: return "V"; + case PLANE_PRED8x8: return "Plane"; + default: return "Unknown"; + } +} + +static const char *get_inter_sub_mb_type_name(uint8_t type) { + switch(type){ + case 0: return "D"; // Direct + case 1: return "L0"; + case 2: return "L1"; + case 3: return "BI"; + default: return "?"; + } +} + AVFILTER_DEFINE_CLASS(codecview); static int clip_line(int *sx, int *sy, int *ex, int *ey, int maxx) @@ -219,12 +272,131 @@ static void draw_block_rectangle(uint8_t *buf, int sx, int sy, int w, int h, ptr } } +static void log_mb_info(AVFilterContext *ctx, AVFrame *frame, int64_t frame_num) +{ + AVFrameSideData *sd = av_frame_get_side_data(frame, AV_FRAME_DATA_H264_MB_INFO); + if (!sd) + return; + + const H264MBInfo *mb_info = (const H264MBInfo *)sd->data; + int nb_mb = sd->size / sizeof(H264MBInfo); + int mb_w = (frame->width + 15) / 16; + + // Allocate a large buffer to build the log string. + size_t buf_size = 32768; + char *log_buf = av_malloc(buf_size); + if (!log_buf) + return; + + char *p = log_buf; + size_t remaining = buf_size; + int ret; + + // Write the main header for the frame into the buffer + ret = snprintf(p, remaining, "H.264 Modes for frame_num %"PRId64" (pts: %"PRId64", type: %c):\n", + frame_num, frame->pts, av_get_picture_type_char(frame->pict_type)); + if (ret > 0 && ret < remaining) { + p += ret; + remaining -= ret; + } + + for (int i = 0; i < nb_mb; i++) { + const H264MBInfo *info = &mb_info[i]; + int mb_x = i % mb_w; + int mb_y = i / mb_w; + + if (remaining < 256) // Safety check, break if buffer is almost full + break; + + if (IS_INTRA(info->mb_type)) { + if (IS_INTRA4x4(info->mb_type)) { + ret = snprintf(p, remaining, "MB(%2d,%2d): I_4x4 C:%-5s P:[%s,%s,%s,%s|%s,%s,%s,%s|%s,%s,%s,%s|%s,%s,%s,%s]\n", + mb_x, mb_y, get_chroma_mode_name(info->intra.chroma_pred_mode), + get_intra_4x4_mode_name(info->intra.intra4x4_pred_mode[0]), + get_intra_4x4_mode_name(info->intra.intra4x4_pred_mode[1]), + get_intra_4x4_mode_name(info->intra.intra4x4_pred_mode[2]), + get_intra_4x4_mode_name(info->intra.intra4x4_pred_mode[3]), + get_intra_4x4_mode_name(info->intra.intra4x4_pred_mode[4]), + get_intra_4x4_mode_name(info->intra.intra4x4_pred_mode[5]), + get_intra_4x4_mode_name(info->intra.intra4x4_pred_mode[6]), + get_intra_4x4_mode_name(info->intra.intra4x4_pred_mode[7]), + get_intra_4x4_mode_name(info->intra.intra4x4_pred_mode[8]), + get_intra_4x4_mode_name(info->intra.intra4x4_pred_mode[9]), + get_intra_4x4_mode_name(info->intra.intra4x4_pred_mode[10]), + get_intra_4x4_mode_name(info->intra.intra4x4_pred_mode[11]), + get_intra_4x4_mode_name(info->intra.intra4x4_pred_mode[12]), + get_intra_4x4_mode_name(info->intra.intra4x4_pred_mode[13]), + get_intra_4x4_mode_name(info->intra.intra4x4_pred_mode[14]), + get_intra_4x4_mode_name(info->intra.intra4x4_pred_mode[15])); + } else if (IS_INTRA16x16(info->mb_type)) { + ret = snprintf(p, remaining, "MB(%2d,%2d): I_16x16 Mode:%-10s ChromaMode:%s\n", + mb_x, mb_y, get_intra_16x16_mode_name(info->intra.intra16x16_pred_mode), + get_chroma_mode_name(info->intra.chroma_pred_mode)); + } else if (IS_INTRA_PCM(info->mb_type)) { + ret = snprintf(p, remaining, "MB(%2d,%2d): I_PCM\n", mb_x, mb_y); + } + } else { // Inter + if (IS_SKIP(info->mb_type)) { + ret = snprintf(p, remaining, "MB(%2d,%2d): Skip\n", mb_x, mb_y); + } else if (IS_16X16(info->mb_type)) { + ret = snprintf(p, remaining, "MB(%2d,%2d): P_16x16 L0:[%d %4d,%4d] L1:[%d %4d,%4d]\n", mb_x, mb_y, + info->inter.ref_idx[0][0], info->inter.mv[0][0][0], info->inter.mv[0][0][1], + info->inter.ref_idx[1][0], info->inter.mv[1][0][0], info->inter.mv[1][0][1]); + } else if (IS_16X8(info->mb_type)) { + ret = snprintf(p, remaining, "MB(%2d,%2d): P_16x8 T: L0:[%d %4d,%4d] L1:[%d %4d,%4d] B: L0:[%d %4d,%4d] L1:[%d %4d,%4d]\n", mb_x, mb_y, + info->inter.ref_idx[0][0], info->inter.mv[0][0][0], info->inter.mv[0][0][1], + info->inter.ref_idx[1][0], info->inter.mv[1][0][0], info->inter.mv[1][0][1], + info->inter.ref_idx[0][8], info->inter.mv[0][8][0], info->inter.mv[0][8][1], + info->inter.ref_idx[1][8], info->inter.mv[1][8][0], info->inter.mv[1][8][1]); + } else if (IS_8X16(info->mb_type)) { + ret = snprintf(p, remaining, "MB(%2d,%2d): P_8x16 L: L0:[%d %4d,%4d] L1:[%d %4d,%4d] R: L0:[%d %4d,%4d] L1:[%d %4d,%4d]\n", mb_x, mb_y, + info->inter.ref_idx[0][0], info->inter.mv[0][0][0], info->inter.mv[0][0][1], + info->inter.ref_idx[1][0], info->inter.mv[1][0][0], info->inter.mv[1][0][1], + info->inter.ref_idx[0][4], info->inter.mv[0][4][0], info->inter.mv[0][4][1], + info->inter.ref_idx[1][4], info->inter.mv[1][4][0], info->inter.mv[1][4][1]); + } else if (IS_8X8(info->mb_type)) { + ret = snprintf(p, remaining, "MB(%2d,%2d): P_8x8\n", mb_x, mb_y); + if (ret > 0 && ret < remaining) { + p += ret; + remaining -= ret; + } + for (int j = 0; j < 4; j++) { + if (remaining < 128) break; + ret = snprintf(p, remaining, "\tBlk %d: %-2s L0:[%d %4d,%4d] L1:[%d %4d,%4d]\n", j, + get_inter_sub_mb_type_name(info->inter.sub_mb_type[j]), + info->inter.ref_idx[0][j*4], info->inter.mv[0][j*4][0], info->inter.mv[0][j*4][1], + info->inter.ref_idx[1][j*4], info->inter.mv[1][j*4][0], info->inter.mv[1][j*4][1]); + if (ret > 0 && ret < remaining) { + p += ret; + remaining -= ret; + } + } + ret = 0; + } + } + + if (ret > 0 && ret < remaining) { + p += ret; + remaining -= ret; + } + } + + // Print the entire buffer in one go. + av_log(ctx, AV_LOG_INFO, "%s", log_buf); + av_free(log_buf); +} static int filter_frame(AVFilterLink *inlink, AVFrame *frame) { AVFilterContext *ctx = inlink->dst; CodecViewContext *s = ctx->priv; AVFilterLink *outlink = ctx->outputs[0]; + if (s->show_modes) { + log_mb_info(ctx, frame, s->frame_count); + } + + s->frame_count++; + if (s->qp) { enum AVVideoEncParamsType qp_type; int qstride, ret; diff --git a/libavutil/frame.h b/libavutil/frame.h index c50cd263d9..8a54ca7989 100644 --- a/libavutil/frame.h +++ b/libavutil/frame.h @@ -254,6 +254,11 @@ enum AVFrameSideDataType { * libavutil/tdrdi.h. */ AV_FRAME_DATA_3D_REFERENCE_DISPLAYS, + + /** + * H.264 Macroblock Info, the data is an array of H264MBInfo structures. + */ + AV_FRAME_DATA_H264_MB_INFO, }; enum AVActiveFormatDescription {