[FFmpeg-devel] [PATCH 2/2] lavc/mediacodecenc: Probe actually supported color formats using JNI

["Tomas Härdin" <git@haerdin.se>]

[-- Attachment #1: Type: text/plain, Size: 70 bytes --]

This should be faster than opening the encoder several times.

/Tomas

[-- Attachment #2: 0002-lavc-mediacodecenc-Probe-actually-supported-color-fo.patch --]
[-- Type: text/x-patch, Size: 8990 bytes --]

From 44c347f5a2b6160abd8fc09a0255ee21a7ba6ee4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tomas=20H=C3=A4rdin?= <git@haerdin.se>
Date: Wed, 4 Jan 2023 16:27:26 +0100
Subject: [PATCH 2/2] lavc/mediacodecenc: Probe actually supported color
 formats using JNI

Fall back on trying to open the encoder if JNI doesn't work.

This patch has been released by Epic Games' legal department.
---
 libavcodec/mediacodec_wrapper.c | 93 +++++++++++++++++++++++++++++++++
 libavcodec/mediacodec_wrapper.h | 15 ++++++
 libavcodec/mediacodecenc.c      | 27 +++++++++-
 3 files changed, 134 insertions(+), 1 deletion(-)

diff --git a/libavcodec/mediacodec_wrapper.c b/libavcodec/mediacodec_wrapper.c
index 4d6e9487b8..ac77672013 100644
--- a/libavcodec/mediacodec_wrapper.c
+++ b/libavcodec/mediacodec_wrapper.c
@@ -2,6 +2,7 @@
  * Android MediaCodec Wrapper
  *
  * Copyright (c) 2015-2016 Matthieu Bouron <matthieu.bouron stupeflix.com>
+ * Modifications by Epic Games, Inc., 2023.
  *
  * This file is part of FFmpeg.
  *
@@ -176,6 +177,7 @@ struct JNIAMediaCodecFields {
     jmethodID release_id;
 
     jmethodID get_output_format_id;
+    jmethodID get_codec_info_id;
 
     jmethodID dequeue_input_buffer_id;
     jmethodID queue_input_buffer_id;
@@ -228,6 +230,7 @@ static const struct FFJniField jni_amediacodec_mapping[] = {
         { "android/media/MediaCodec", "release", "()V", FF_JNI_METHOD, offsetof(struct JNIAMediaCodecFields, release_id), 1 },
 
         { "android/media/MediaCodec", "getOutputFormat", "()Landroid/media/MediaFormat;", FF_JNI_METHOD, offsetof(struct JNIAMediaCodecFields, get_output_format_id), 1 },
+        { "android/media/MediaCodec", "getCodecInfo", "()Landroid/media/MediaCodecInfo;", FF_JNI_METHOD, offsetof(struct JNIAMediaCodecFields, get_codec_info_id), 1 },
 
         { "android/media/MediaCodec", "dequeueInputBuffer", "(J)I", FF_JNI_METHOD, offsetof(struct JNIAMediaCodecFields, dequeue_input_buffer_id), 1 },
         { "android/media/MediaCodec", "queueInputBuffer", "(IIIJI)V", FF_JNI_METHOD, offsetof(struct JNIAMediaCodecFields, queue_input_buffer_id), 1 },
@@ -604,6 +607,96 @@ done:
     return name;
 }
 
+int ff_AMediaCodec_color_formats_intersect(FFAMediaCodec *codec, const char *mime,
+                                           const int *in_formats, int nin_formats,
+                                           int *out_formats, void *log_ctx)
+{
+    FFAMediaCodecJni *jni_codec = (FFAMediaCodecJni*)codec;
+    jobject info = NULL;
+    jobject capabilities = NULL;
+    JNIEnv *env = NULL;
+    jintArray color_formats = NULL;
+    int color_count, ret, *elems;
+    jboolean is_copy;
+    jstring jmime = NULL;
+    struct JNIAMediaCodecFields jfields = { 0 };
+    struct JNIAMediaCodecListFields jfields2 = { 0 };
+
+    // make sure we have a JVM
+    JNI_GET_ENV_OR_RETURN(env, log_ctx, -1);
+
+    // grab mappings
+    if ((ret = ff_jni_init_jfields(env, &jfields, jni_amediacodec_mapping, 0, log_ctx)) < 0) {
+        goto done;
+    }
+
+    if ((ret = ff_jni_init_jfields(env, &jfields2, jni_amediacodeclist_mapping, 0, log_ctx)) < 0) {
+        goto done;
+    }
+
+    // codec.getCodecInfo().getCapabilitiesForType(mime).colorFormats
+    ret = -1;
+    info = (*env)->CallObjectMethod(env, jni_codec->object, jfields.get_codec_info_id);
+    if (ff_jni_exception_check(env, 1, log_ctx) < 0) {
+        goto done;
+    }
+
+    // convert const char* to java.lang.String
+    jmime = ff_jni_utf_chars_to_jstring(env, mime, log_ctx);
+    if (!jmime) {
+        goto done;
+    }
+
+    capabilities = (*env)->CallObjectMethod(env, info, jfields2.get_codec_capabilities_id, jmime);
+    if (ff_jni_exception_check(env, 1, log_ctx) < 0) {
+        goto done;
+    }
+
+    color_formats = (*env)->GetObjectField(env, capabilities, jfields2.color_formats_id);
+    if (ff_jni_exception_check(env, 1, log_ctx) < 0) {
+        goto done;
+    }
+
+    color_count = (*env)->GetArrayLength(env, color_formats);
+    elems = (*env)->GetIntArrayElements(env, color_formats, &is_copy);
+    ret = 0;
+
+    // out_formats = intersect(in_formats, elems)
+    for (int i = 0; i < nin_formats; i++) {
+        for (int j = 0; j < color_count; j++) {
+            if (elems[j] == in_formats[i]) {
+                out_formats[ret++] = in_formats[i];
+                break;
+            }
+        }
+    }
+
+    (*env)->ReleaseIntArrayElements(env, color_formats, elems, JNI_ABORT);
+
+done:
+    // clean up
+    if (jmime) {
+        (*env)->DeleteLocalRef(env, jmime);
+    }
+
+    if (color_formats) {
+        (*env)->DeleteLocalRef(env, color_formats);
+    }
+
+    if (capabilities) {
+        (*env)->DeleteLocalRef(env, capabilities);
+    }
+
+    if (info) {
+        (*env)->DeleteLocalRef(env, info);
+    }
+
+    ff_jni_reset_jfields(env, &jfields, jni_amediacodec_mapping, 0, log_ctx);
+    ff_jni_reset_jfields(env, &jfields2, jni_amediacodeclist_mapping, 0, log_ctx);
+
+    return ret;
+}
+
 static FFAMediaFormat *mediaformat_jni_new(void)
 {
     JNIEnv *env = NULL;
diff --git a/libavcodec/mediacodec_wrapper.h b/libavcodec/mediacodec_wrapper.h
index 1b81e6db84..f0427f3287 100644
--- a/libavcodec/mediacodec_wrapper.h
+++ b/libavcodec/mediacodec_wrapper.h
@@ -2,6 +2,7 @@
  * Android MediaCodec Wrapper
  *
  * Copyright (c) 2015-2016 Matthieu Bouron <matthieu.bouron stupeflix.com>
+ * Modifications by Epic Games, Inc., 2023.
  *
  * This file is part of FFmpeg.
  *
@@ -230,6 +231,20 @@ FFAMediaCodec* ff_AMediaCodec_createCodecByName(const char *name, int ndk);
 FFAMediaCodec* ff_AMediaCodec_createDecoderByType(const char *mime_type, int ndk);
 FFAMediaCodec* ff_AMediaCodec_createEncoderByType(const char *mime_type, int ndk);
 
+/**
+ * Intersects the given list of color formats with the formats actually supported by the device.
+ * @param[in]  codec        The codec to query
+ * @param[in]  mime         MIME format. Typically "video/avc" or "video/hevc"
+ * @param[in]  in_formats   Array of input color formats
+ * @param[in]  nin_formats  Number of elements in in_formats
+ * @param[out] out_formats  Computed intersection of in_formats and supported formats
+ * @param[in]  log_ctx      Logging context
+ * @return Size of out_formats or negative in case of error
+ */
+int ff_AMediaCodec_color_formats_intersect(FFAMediaCodec *codec, const char *mime,
+                                           const int *in_formats, int nin_formats,
+                                           int *out_formats, void *log_ctx);
+
 static inline int ff_AMediaCodec_configure(FFAMediaCodec *codec,
                                            const FFAMediaFormat *format,
                                            FFANativeWindow *surface,
diff --git a/libavcodec/mediacodecenc.c b/libavcodec/mediacodecenc.c
index fd90d41625..2185e0b5ab 100644
--- a/libavcodec/mediacodecenc.c
+++ b/libavcodec/mediacodecenc.c
@@ -90,6 +90,12 @@ static const struct {
     { COLOR_FormatSurface,              AV_PIX_FMT_MEDIACODEC },
 };
 
+static const int in_formats[] = {
+    COLOR_FormatYUV420Planar,
+    COLOR_FormatYUV420SemiPlanar,
+    COLOR_FormatSurface,
+};
+
 // filled in by mediacodec_init_static_data()
 static enum AVPixelFormat probed_pix_fmts[FF_ARRAY_ELEMS(color_formats)+1];
 
@@ -535,7 +541,7 @@ static av_cold void mediacodec_init_static_data(FFCodec *ffcodec)
 {
     const char *codec_mime = ffcodec->p.id == AV_CODEC_ID_H264 ? "video/avc" : "video/hevc";
     FFAMediaCodec *codec;
-    int num_pix_fmts = 0;
+    int num_pix_fmts, out_formats[FF_ARRAY_ELEMS(in_formats)];
     int use_ndk_codec = !av_jni_get_java_vm(NULL);
 
     if (!(codec = ff_AMediaCodec_createEncoderByType(codec_mime, use_ndk_codec))) {
@@ -543,6 +549,24 @@ static av_cold void mediacodec_init_static_data(FFCodec *ffcodec)
         return;
     }
 
+    // attempt to query color formats using JNI
+    // fall back on NDK-compatible method below if this fails
+    if ((num_pix_fmts = ff_AMediaCodec_color_formats_intersect(
+            codec, codec_mime, in_formats, FF_ARRAY_ELEMS(in_formats),
+            out_formats, NULL)) >= 0) {
+        // translate color formats to pixel formats
+        for (int i = 0; i < num_pix_fmts; i++) {
+            for (int j = 0; j < FF_ARRAY_ELEMS(color_formats); j++) {
+                if (out_formats[i] == color_formats[j].color_format) {
+                    probed_pix_fmts[i] = color_formats[j].pix_fmt;
+                    break;
+                }
+            }
+        }
+        goto done;
+    }
+
+    num_pix_fmts = 0;
     for (int i = 0; i < FF_ARRAY_ELEMS(color_formats); i++) {
         if (color_formats[i].pix_fmt == AV_PIX_FMT_MEDIACODEC) {
             // assumme AV_PIX_FMT_MEDIACODEC always works
@@ -589,6 +613,7 @@ static av_cold void mediacodec_init_static_data(FFCodec *ffcodec)
         }
     }
 
+done:
     probed_pix_fmts[num_pix_fmts] = AV_PIX_FMT_NONE;
     ffcodec->p.pix_fmts = probed_pix_fmts;
     ff_AMediaCodec_delete(codec);
-- 
2.30.2


[-- Attachment #3: Type: text/plain, Size: 251 bytes --]

_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".