From 44c347f5a2b6160abd8fc09a0255ee21a7ba6ee4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomas=20H=C3=A4rdin?= Date: Wed, 4 Jan 2023 16:27:26 +0100 Subject: [PATCH 2/2] lavc/mediacodecenc: Probe actually supported color formats using JNI Fall back on trying to open the encoder if JNI doesn't work. This patch has been released by Epic Games' legal department. --- libavcodec/mediacodec_wrapper.c | 93 +++++++++++++++++++++++++++++++++ libavcodec/mediacodec_wrapper.h | 15 ++++++ libavcodec/mediacodecenc.c | 27 +++++++++- 3 files changed, 134 insertions(+), 1 deletion(-) diff --git a/libavcodec/mediacodec_wrapper.c b/libavcodec/mediacodec_wrapper.c index 4d6e9487b8..ac77672013 100644 --- a/libavcodec/mediacodec_wrapper.c +++ b/libavcodec/mediacodec_wrapper.c @@ -2,6 +2,7 @@ * Android MediaCodec Wrapper * * Copyright (c) 2015-2016 Matthieu Bouron + * Modifications by Epic Games, Inc., 2023. * * This file is part of FFmpeg. * @@ -176,6 +177,7 @@ struct JNIAMediaCodecFields { jmethodID release_id; jmethodID get_output_format_id; + jmethodID get_codec_info_id; jmethodID dequeue_input_buffer_id; jmethodID queue_input_buffer_id; @@ -228,6 +230,7 @@ static const struct FFJniField jni_amediacodec_mapping[] = { { "android/media/MediaCodec", "release", "()V", FF_JNI_METHOD, offsetof(struct JNIAMediaCodecFields, release_id), 1 }, { "android/media/MediaCodec", "getOutputFormat", "()Landroid/media/MediaFormat;", FF_JNI_METHOD, offsetof(struct JNIAMediaCodecFields, get_output_format_id), 1 }, + { "android/media/MediaCodec", "getCodecInfo", "()Landroid/media/MediaCodecInfo;", FF_JNI_METHOD, offsetof(struct JNIAMediaCodecFields, get_codec_info_id), 1 }, { "android/media/MediaCodec", "dequeueInputBuffer", "(J)I", FF_JNI_METHOD, offsetof(struct JNIAMediaCodecFields, dequeue_input_buffer_id), 1 }, { "android/media/MediaCodec", "queueInputBuffer", "(IIIJI)V", FF_JNI_METHOD, offsetof(struct JNIAMediaCodecFields, queue_input_buffer_id), 1 }, @@ -604,6 +607,96 @@ done: return name; } +int ff_AMediaCodec_color_formats_intersect(FFAMediaCodec *codec, const char *mime, + const int *in_formats, int nin_formats, + int *out_formats, void *log_ctx) +{ + FFAMediaCodecJni *jni_codec = (FFAMediaCodecJni*)codec; + jobject info = NULL; + jobject capabilities = NULL; + JNIEnv *env = NULL; + jintArray color_formats = NULL; + int color_count, ret, *elems; + jboolean is_copy; + jstring jmime = NULL; + struct JNIAMediaCodecFields jfields = { 0 }; + struct JNIAMediaCodecListFields jfields2 = { 0 }; + + // make sure we have a JVM + JNI_GET_ENV_OR_RETURN(env, log_ctx, -1); + + // grab mappings + if ((ret = ff_jni_init_jfields(env, &jfields, jni_amediacodec_mapping, 0, log_ctx)) < 0) { + goto done; + } + + if ((ret = ff_jni_init_jfields(env, &jfields2, jni_amediacodeclist_mapping, 0, log_ctx)) < 0) { + goto done; + } + + // codec.getCodecInfo().getCapabilitiesForType(mime).colorFormats + ret = -1; + info = (*env)->CallObjectMethod(env, jni_codec->object, jfields.get_codec_info_id); + if (ff_jni_exception_check(env, 1, log_ctx) < 0) { + goto done; + } + + // convert const char* to java.lang.String + jmime = ff_jni_utf_chars_to_jstring(env, mime, log_ctx); + if (!jmime) { + goto done; + } + + capabilities = (*env)->CallObjectMethod(env, info, jfields2.get_codec_capabilities_id, jmime); + if (ff_jni_exception_check(env, 1, log_ctx) < 0) { + goto done; + } + + color_formats = (*env)->GetObjectField(env, capabilities, jfields2.color_formats_id); + if (ff_jni_exception_check(env, 1, log_ctx) < 0) { + goto done; + } + + color_count = (*env)->GetArrayLength(env, color_formats); + elems = (*env)->GetIntArrayElements(env, color_formats, &is_copy); + ret = 0; + + // out_formats = intersect(in_formats, elems) + for (int i = 0; i < nin_formats; i++) { + for (int j = 0; j < color_count; j++) { + if (elems[j] == in_formats[i]) { + out_formats[ret++] = in_formats[i]; + break; + } + } + } + + (*env)->ReleaseIntArrayElements(env, color_formats, elems, JNI_ABORT); + +done: + // clean up + if (jmime) { + (*env)->DeleteLocalRef(env, jmime); + } + + if (color_formats) { + (*env)->DeleteLocalRef(env, color_formats); + } + + if (capabilities) { + (*env)->DeleteLocalRef(env, capabilities); + } + + if (info) { + (*env)->DeleteLocalRef(env, info); + } + + ff_jni_reset_jfields(env, &jfields, jni_amediacodec_mapping, 0, log_ctx); + ff_jni_reset_jfields(env, &jfields2, jni_amediacodeclist_mapping, 0, log_ctx); + + return ret; +} + static FFAMediaFormat *mediaformat_jni_new(void) { JNIEnv *env = NULL; diff --git a/libavcodec/mediacodec_wrapper.h b/libavcodec/mediacodec_wrapper.h index 1b81e6db84..f0427f3287 100644 --- a/libavcodec/mediacodec_wrapper.h +++ b/libavcodec/mediacodec_wrapper.h @@ -2,6 +2,7 @@ * Android MediaCodec Wrapper * * Copyright (c) 2015-2016 Matthieu Bouron + * Modifications by Epic Games, Inc., 2023. * * This file is part of FFmpeg. * @@ -230,6 +231,20 @@ FFAMediaCodec* ff_AMediaCodec_createCodecByName(const char *name, int ndk); FFAMediaCodec* ff_AMediaCodec_createDecoderByType(const char *mime_type, int ndk); FFAMediaCodec* ff_AMediaCodec_createEncoderByType(const char *mime_type, int ndk); +/** + * Intersects the given list of color formats with the formats actually supported by the device. + * @param[in] codec The codec to query + * @param[in] mime MIME format. Typically "video/avc" or "video/hevc" + * @param[in] in_formats Array of input color formats + * @param[in] nin_formats Number of elements in in_formats + * @param[out] out_formats Computed intersection of in_formats and supported formats + * @param[in] log_ctx Logging context + * @return Size of out_formats or negative in case of error + */ +int ff_AMediaCodec_color_formats_intersect(FFAMediaCodec *codec, const char *mime, + const int *in_formats, int nin_formats, + int *out_formats, void *log_ctx); + static inline int ff_AMediaCodec_configure(FFAMediaCodec *codec, const FFAMediaFormat *format, FFANativeWindow *surface, diff --git a/libavcodec/mediacodecenc.c b/libavcodec/mediacodecenc.c index fd90d41625..2185e0b5ab 100644 --- a/libavcodec/mediacodecenc.c +++ b/libavcodec/mediacodecenc.c @@ -90,6 +90,12 @@ static const struct { { COLOR_FormatSurface, AV_PIX_FMT_MEDIACODEC }, }; +static const int in_formats[] = { + COLOR_FormatYUV420Planar, + COLOR_FormatYUV420SemiPlanar, + COLOR_FormatSurface, +}; + // filled in by mediacodec_init_static_data() static enum AVPixelFormat probed_pix_fmts[FF_ARRAY_ELEMS(color_formats)+1]; @@ -535,7 +541,7 @@ static av_cold void mediacodec_init_static_data(FFCodec *ffcodec) { const char *codec_mime = ffcodec->p.id == AV_CODEC_ID_H264 ? "video/avc" : "video/hevc"; FFAMediaCodec *codec; - int num_pix_fmts = 0; + int num_pix_fmts, out_formats[FF_ARRAY_ELEMS(in_formats)]; int use_ndk_codec = !av_jni_get_java_vm(NULL); if (!(codec = ff_AMediaCodec_createEncoderByType(codec_mime, use_ndk_codec))) { @@ -543,6 +549,24 @@ static av_cold void mediacodec_init_static_data(FFCodec *ffcodec) return; } + // attempt to query color formats using JNI + // fall back on NDK-compatible method below if this fails + if ((num_pix_fmts = ff_AMediaCodec_color_formats_intersect( + codec, codec_mime, in_formats, FF_ARRAY_ELEMS(in_formats), + out_formats, NULL)) >= 0) { + // translate color formats to pixel formats + for (int i = 0; i < num_pix_fmts; i++) { + for (int j = 0; j < FF_ARRAY_ELEMS(color_formats); j++) { + if (out_formats[i] == color_formats[j].color_format) { + probed_pix_fmts[i] = color_formats[j].pix_fmt; + break; + } + } + } + goto done; + } + + num_pix_fmts = 0; for (int i = 0; i < FF_ARRAY_ELEMS(color_formats); i++) { if (color_formats[i].pix_fmt == AV_PIX_FMT_MEDIACODEC) { // assumme AV_PIX_FMT_MEDIACODEC always works @@ -589,6 +613,7 @@ static av_cold void mediacodec_init_static_data(FFCodec *ffcodec) } } +done: probed_pix_fmts[num_pix_fmts] = AV_PIX_FMT_NONE; ffcodec->p.pix_fmts = probed_pix_fmts; ff_AMediaCodec_delete(codec); -- 2.30.2