[FFmpeg-devel] [PR] avformat/tls_mbedtls: add dtls support (PR #21695)

[FFmpeg-devel] [PR] avformat/tls_mbedtls: add dtls support (PR #21695)

[Jack Lau via ffmpeg-devel]

PR #21695 opened by Jack Lau (JackLau)
URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/21695
Patch URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/21695.patch


>From 5bd1b0b77f1d199d08290e09449c539dea953884 Mon Sep 17 00:00:00 2001
From: Jack Lau <jacklau1222gm@gmail.com>
Date: Mon, 9 Feb 2026 12:53:16 +0800
Subject: [PATCH 1/3] avformat/tls_mbdetls: add dtls support

Signed-off-by: Jack Lau <jacklau1222gm@gmail.com>
---
 configure                 |   4 +-
 libavformat/tls_mbedtls.c | 411 ++++++++++++++++++++++++++++++++++++--
 2 files changed, 394 insertions(+), 21 deletions(-)

diff --git a/configure b/configure
index 6d6e6673dc..267c56a7d9 100755
--- a/configure
+++ b/configure
@@ -4002,8 +4002,8 @@ srtp_protocol_select="rtp_protocol srtp"
 tcp_protocol_select="network"
 tls_protocol_deps_any="gnutls openssl schannel securetransport libtls mbedtls"
 tls_protocol_select="tcp_protocol"
-# TODO: Support libtls, mbedtls.
-dtls_protocol_deps_any="openssl schannel gnutls"
+# TODO: Support libtls.
+dtls_protocol_deps_any="openssl schannel gnutls mbedtls"
 dtls_protocol_select="udp_protocol"
 udp_protocol_select="network"
 udplite_protocol_select="network"
diff --git a/libavformat/tls_mbedtls.c b/libavformat/tls_mbedtls.c
index 8aa142b9d2..dd08bd6f15 100644
--- a/libavformat/tls_mbedtls.c
+++ b/libavformat/tls_mbedtls.c
@@ -27,6 +27,7 @@
 #include <mbedtls/ssl.h>
 #include <mbedtls/x509_crt.h>
 #include <mbedtls/debug.h>
+#include <mbedtls/timing.h>
 #ifdef MBEDTLS_PSA_CRYPTO_C
 #include <psa/crypto.h>
 #endif
@@ -38,6 +39,212 @@
 #include "libavutil/mem.h"
 #include "libavutil/parseutils.h"
 #include "libavutil/avstring.h"
+#include "libavutil/random_seed.h"
+
+static int mbedtls_x509_fingerprint(char *cert_buf, size_t cert_sz, char **fingerprint)
+{
+    unsigned char md[32];
+    size_t n = sizeof(md);
+    AVBPrint buf;
+    int ret;
+    mbedtls_x509_crt crt;
+
+    mbedtls_x509_crt_init(&crt);
+
+    if ((ret = mbedtls_x509_crt_parse(&crt, cert_buf, cert_sz)) != 0) {
+        mbedtls_x509_crt_free(&crt);
+        return AVERROR(EINVAL);
+    }
+
+    if ((ret = mbedtls_sha256(crt.raw.p, crt.raw.len, md, 0)) != 0) {
+        mbedtls_x509_crt_free(&crt);
+        return AVERROR(EINVAL);
+    }
+
+    av_bprint_init(&buf, n*3, n*3);
+
+    for (int i = 0; i < n - 1; i++)
+        av_bprintf(&buf, "%02X:", md[i]);
+    av_bprintf(&buf, "%02X", md[n - 1]);
+
+    return av_bprint_finalize(&buf, fingerprint);
+}
+
+int ff_ssl_read_key_cert(char *key_url, char *cert_url, char *key_buf, size_t key_sz, char *cert_buf, size_t cert_sz, char **fingerprint)
+{
+    int ret = 0;
+    AVBPrint key_bp, cert_bp;
+    av_bprint_init(&key_bp, 1, MAX_CERTIFICATE_SIZE);
+    av_bprint_init(&cert_bp, 1, MAX_CERTIFICATE_SIZE);
+
+    ret = ff_url_read_all(key_url, &key_bp);
+    if (ret < 0) {
+        av_log(NULL, AV_LOG_ERROR, "TLS: Failed to open key file %s\n", key_url);
+        goto end;
+    }
+
+    ret = ff_url_read_all(cert_url, &cert_bp);
+    if (ret < 0) {
+        av_log(NULL, AV_LOG_ERROR, "TLS: Failed to open cert file %s\n", cert_url);
+        goto end;
+    }
+
+    if (key_sz < key_bp.size || cert_sz < cert_bp.size) {
+        av_log(NULL, AV_LOG_ERROR, "TLS: Key or Cert buffer is too samall\n");
+        ret = AVERROR_BUFFER_TOO_SMALL;
+        goto end;
+    }
+
+    key_buf = key_bp.str;
+    cert_buf = cert_bp.str;
+
+    ret = mbedtls_x509_fingerprint(cert_buf, cert_sz, fingerprint);
+    if (ret < 0)
+        av_log(NULL, AV_LOG_ERROR, "TLS: Failed to generate fingerprint\n");
+end:
+    av_bprint_finalize(&key_bp, NULL);
+    av_bprint_finalize(&cert_bp, NULL);
+    return ret;
+}
+
+static int mbedtls_gen_pkey(mbedtls_pk_context *key)
+{
+    int ret = 0;
+    mbedtls_entropy_context entropy;
+    mbedtls_ctr_drbg_context ctr_drbg;
+
+    mbedtls_entropy_init(&entropy);
+    mbedtls_ctr_drbg_init(&ctr_drbg);
+
+    if ((ret = mbedtls_ctr_drbg_seed(&ctr_drbg, mbedtls_entropy_func,
+                                     &entropy, NULL, 0)) != 0) {
+        av_log(NULL, AV_LOG_ERROR, "mbedtls_ctr_drbg_seed returned %d\n", ret);
+        goto end;
+    }
+
+    if ((ret = mbedtls_pk_setup(key,
+                            mbedtls_pk_info_from_type(MBEDTLS_PK_ECKEY))) != 0) {
+        av_log(NULL, AV_LOG_ERROR, "mbedtls_pk_setup returned %d\n", ret);
+        goto end;
+    }
+    /**
+     * See RFC 8827 section 6.5,
+     * All implementations MUST support DTLS 1.2 with the
+     * TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 cipher suite
+     * and the P-256 curve.
+     */
+    if ((ret = mbedtls_ecp_gen_key(MBEDTLS_ECP_DP_SECP256R1,
+                                mbedtls_pk_ec(*key),
+                                mbedtls_ctr_drbg_random, &ctr_drbg)) != 0) {
+        av_log(NULL, AV_LOG_ERROR, "mbedtls_ecp_gen_key returned %d\n", ret);
+        goto end;
+    }
+end:
+    mbedtls_entropy_free(&entropy);
+    mbedtls_ctr_drbg_free(&ctr_drbg);
+    return ret;
+}
+
+static int mbedtls_gen_x509_cert(mbedtls_pk_context *key, char *cert_buf, size_t cert_sz)
+{
+    int ret = 0;
+    const char *name = "CN=lavf";
+    time_t now;
+    struct tm tm;
+    char not_before[16], not_after[16];
+    unsigned char serial[20];
+    mbedtls_entropy_context entropy;
+    mbedtls_ctr_drbg_context ctr_drbg;
+    mbedtls_x509write_cert crt;
+
+    mbedtls_entropy_init(&entropy);
+    mbedtls_ctr_drbg_init(&ctr_drbg);
+    mbedtls_x509write_crt_init(&crt);
+
+    if ((ret = mbedtls_ctr_drbg_seed(&ctr_drbg, mbedtls_entropy_func, &entropy, NULL, 0)) != 0) {
+        av_log(NULL, AV_LOG_ERROR, "mbedtls_ctr_drbg_seed returned %d\n", ret);
+        goto end;
+    }
+
+    mbedtls_x509write_crt_set_subject_key(&crt, key);
+    mbedtls_x509write_crt_set_issuer_key(&crt, key);
+    if ((ret = mbedtls_x509write_crt_set_subject_name(&crt, name)) != 0) {
+        av_log(NULL, AV_LOG_ERROR, "mbedtls_x509write_crt_set_subject_name returned %d\n", ret);
+        goto end;
+    }
+
+    if ((ret = mbedtls_x509write_crt_set_issuer_name(&crt, name)) != 0) {
+        av_log(NULL, AV_LOG_ERROR, "mbedtls_x509write_crt_set_issuer_name returned %d\n", ret);
+        goto end;
+    }
+    mbedtls_x509write_crt_set_version(&crt, MBEDTLS_X509_CRT_VERSION_3);
+    mbedtls_x509write_crt_set_md_alg(&crt, MBEDTLS_MD_SHA256);
+
+    ret = av_random_bytes((uint8_t *)serial, sizeof(serial));
+    if (ret < 0) {
+        av_log(NULL, AV_LOG_ERROR, "Failed to generate random serial number!\n");
+        return ret;
+    }
+
+    if ((ret = mbedtls_x509write_crt_set_serial_raw(&crt, serial, sizeof(serial))) != 0) {
+        av_log(NULL, AV_LOG_ERROR, "mbedtls_x509write_crt_set_serial_raw returned %d\n", ret);
+        goto end;
+    }
+
+    time(&now);
+    gmtime_r(&now, &tm);
+    strftime(not_before, sizeof(not_before), "%Y%m%d%H%M%S", &tm);
+    tm.tm_year += 1;
+    strftime(not_after, sizeof(not_after), "%Y%m%d%H%M%S", &tm);
+
+    if ((ret = mbedtls_x509write_crt_set_validity(&crt, not_before, not_after)) != 0) {
+        av_log(NULL, AV_LOG_ERROR, "mbedtls_x509write_crt_set_validity returned %d\n", ret);
+        goto end;
+    }
+
+    if ((ret = mbedtls_x509write_crt_pem(&crt, cert_buf, cert_sz,
+                                        mbedtls_ctr_drbg_random, &ctr_drbg)) != 0) {
+        av_log(NULL, AV_LOG_ERROR, "mbedtls_x509write_crt_pem returned %d\n", ret);
+        return ret;
+    }
+
+end:
+    mbedtls_entropy_free(&entropy);
+    mbedtls_ctr_drbg_free(&ctr_drbg);
+    mbedtls_x509write_crt_free(&crt);
+    return ret;
+}
+
+int ff_ssl_gen_key_cert(char *key_buf, size_t key_sz, char *cert_buf, size_t cert_sz, char **fingerprint)
+{
+    int ret = 0;
+    mbedtls_pk_context key;
+
+    mbedtls_pk_init(&key);
+
+    if ((ret = mbedtls_gen_pkey(&key)) != 0)
+        goto end;
+
+    if ((ret = mbedtls_pk_write_key_pem(&key, key_buf, key_sz)) != 0)
+        goto end;
+
+    if ((ret = mbedtls_gen_x509_cert(&key, cert_buf, cert_sz)) != 0)
+        goto end;
+
+    ret = mbedtls_x509_fingerprint(cert_buf, cert_sz, fingerprint);
+    if (ret < 0)
+        av_log(NULL, AV_LOG_ERROR, "TLS: Failed to generate fingerprint\n");
+
+end:
+    mbedtls_pk_free(&key);
+    return ret;
+}
+
+typedef struct dtls_srtp_keys {
+    unsigned char master_secret[48];
+    unsigned char randbytes[64];
+    mbedtls_tls_prf_types tls_prf_type;
+} dtls_srtp_keys;
 
 typedef struct TLSContext {
     TLSShared tls_shared;
@@ -45,17 +252,78 @@ typedef struct TLSContext {
     mbedtls_ssl_config ssl_config;
     mbedtls_entropy_context entropy_context;
     mbedtls_ctr_drbg_context ctr_drbg_context;
+    mbedtls_timing_delay_context timer;
     mbedtls_x509_crt ca_cert;
     mbedtls_x509_crt own_cert;
     mbedtls_pk_context priv_key;
     char *priv_key_pw;
+    dtls_srtp_keys srtp_key;
 } TLSContext;
 
+int ff_tls_set_external_socket(URLContext *h, URLContext *sock)
+{
+    TLSContext *tls_ctx = h->priv_data;
+    TLSShared *shr = &tls_ctx->tls_shared;
+
+    if (shr->is_dtls)
+        shr->udp = sock;
+    else
+        shr->tcp = sock;
+
+    return 0;
+}
+
+#if defined(MBEDTLS_SSL_DTLS_SRTP)
+static void dtls_srtp_key_derivation(void *p_expkey,
+                                     mbedtls_ssl_key_export_type secret_type,
+                                     const unsigned char *secret,
+                                     size_t secret_len,
+                                     const unsigned char client_random[32],
+                                     const unsigned char server_random[32],
+                                     mbedtls_tls_prf_types tls_prf_type)
+{
+    dtls_srtp_keys *keys = (dtls_srtp_keys *) p_expkey;
+
+    memcpy(keys->master_secret, secret, sizeof(keys->master_secret));
+    memcpy(keys->randbytes, client_random, 32);
+    memcpy(keys->randbytes + 32, server_random, 32);
+    keys->tls_prf_type = tls_prf_type;
+}
+#endif
+
+int ff_dtls_export_materials(URLContext *h, char *dtls_srtp_materials, size_t materials_sz)
+{
+    int ret = 0;
+    TLSContext *tls_ctx = h->priv_data;
+#if defined(MBEDTLS_SSL_DTLS_SRTP)
+    const char* dst = "EXTRACTOR-dtls_srtp";
+    mbedtls_dtls_srtp_info dtls_srtp_negotiation_result;
+    mbedtls_ssl_get_dtls_srtp_negotiation_result(&tls_ctx->ssl_context, &dtls_srtp_negotiation_result);
+
+    if ((ret = mbedtls_ssl_tls_prf(tls_ctx->srtp_key.tls_prf_type,
+                                    tls_ctx->srtp_key.master_secret,
+                                    sizeof(tls_ctx->srtp_key.master_secret),
+                                    dst,
+                                    tls_ctx->srtp_key.randbytes,
+                                    sizeof(tls_ctx->srtp_key.randbytes),
+                                    dtls_srtp_materials,
+                                    materials_sz)) != 0) {
+        av_log(h, AV_LOG_ERROR,"mbedtls_ssl_tls_prf returned %d\n", ret);
+        ret = AVERROR(EINVAL);
+    }
+#else
+    av_log(h, AV_LOG_ERROR, "DTLS-SRTP is not supported in this mbedtls build\n");
+    ret = AVERROR(ENOSYS);
+#endif
+    return ret;
+}
+
 #define OFFSET(x) offsetof(TLSContext, x)
 
 static int tls_close(URLContext *h)
 {
     TLSContext *tls_ctx = h->priv_data;
+    TLSShared *shr = &tls_ctx->tls_shared;
 
     mbedtls_ssl_close_notify(&tls_ctx->ssl_context);
     mbedtls_pk_free(&tls_ctx->priv_key);
@@ -65,8 +333,8 @@ static int tls_close(URLContext *h)
     mbedtls_ssl_config_free(&tls_ctx->ssl_config);
     mbedtls_ctr_drbg_free(&tls_ctx->ctr_drbg_context);
     mbedtls_entropy_free(&tls_ctx->entropy_context);
-
-    ffurl_closep(&tls_ctx->tls_shared.tcp);
+    if (!shr->external_sock)
+        ffurl_closep(shr->is_dtls ? &shr->udp : &shr->tcp);
     return 0;
 }
 
@@ -89,7 +357,9 @@ static int handle_transport_error(URLContext *h, const char* func_name, int reac
 
 static int mbedtls_send(void *ctx, const unsigned char *buf, size_t len)
 {
-    URLContext *h = (URLContext*) ctx;
+    TLSContext *tls_ctx = (TLSContext*) ctx;
+    TLSShared *shr = &tls_ctx->tls_shared;
+    URLContext *h = shr->is_dtls ? shr->udp : shr->tcp;
     int ret = ffurl_write(h, buf, len);
     if (ret >= 0)
         return ret;
@@ -102,11 +372,12 @@ static int mbedtls_send(void *ctx, const unsigned char *buf, size_t len)
 
 static int mbedtls_recv(void *ctx, unsigned char *buf, size_t len)
 {
-    URLContext *h = (URLContext*) ctx;
+    TLSContext *tls_ctx = (TLSContext*) ctx;
+    TLSShared *shr = &tls_ctx->tls_shared;
+    URLContext *h = shr->is_dtls ? shr->udp : shr->tcp;
     int ret = ffurl_read(h, buf, len);
     if (ret >= 0)
         return ret;
-
     if (h->max_packet_size && len > h->max_packet_size)
         return MBEDTLS_ERR_SSL_BUFFER_TOO_SMALL;
 
@@ -174,15 +445,46 @@ static void handle_handshake_error(URLContext *h, int ret)
     }
 }
 
+static int tls_handshake(URLContext *h)
+{
+    TLSContext *tls_ctx = h->priv_data;
+    TLSShared *shr = &tls_ctx->tls_shared;
+    URLContext *uc = shr->is_dtls ? shr->udp : shr->tcp;
+    int ret;
+
+    uc->flags &= ~AVIO_FLAG_NONBLOCK;
+
+    while (1) {
+        ret = mbedtls_ssl_handshake(&tls_ctx->ssl_context);
+
+        if (!ret)
+            break;
+        if (ret != MBEDTLS_ERR_SSL_WANT_READ && ret != MBEDTLS_ERR_SSL_WANT_WRITE) {
+            handle_handshake_error(h, ret);
+            return ret;
+        }
+    }
+
+    return ret;
+}
+
 static int tls_open(URLContext *h, const char *uri, int flags, AVDictionary **options)
 {
     TLSContext *tls_ctx = h->priv_data;
     TLSShared *shr = &tls_ctx->tls_shared;
     uint32_t verify_res_flags;
     int ret;
+#if defined(MBEDTLS_SSL_DTLS_SRTP)
+    const mbedtls_ssl_srtp_profile profiles[] = {
+        MBEDTLS_TLS_SRTP_AES128_CM_HMAC_SHA1_80,
+        MBEDTLS_TLS_SRTP_UNSET
+    };
+#endif
 
-    if ((ret = ff_tls_open_underlying(shr, h, uri, options)) < 0)
-        goto fail;
+    if (!shr->external_sock) {
+        if ((ret = ff_tls_open_underlying(shr, h, uri, options)) < 0)
+            goto fail;
+    }
 
 #ifdef MBEDTLS_PSA_CRYPTO_C
     if ((ret = psa_crypto_init()) != PSA_SUCCESS) {
@@ -199,7 +501,7 @@ static int tls_open(URLContext *h, const char *uri, int flags, AVDictionary **op
     mbedtls_pk_init(&tls_ctx->priv_key);
 
     if (av_log_get_level() >= AV_LOG_DEBUG) {
-        mbedtls_ssl_conf_dbg(&tls_ctx->ssl_config, mbedtls_debug, shr->tcp);
+        mbedtls_ssl_conf_dbg(&tls_ctx->ssl_config, mbedtls_debug, shr->is_dtls ? shr->udp : shr->tcp);
         /*
          * Note: we can't call mbedtls_debug_set_threshold() here because
          * it's global state. The user is thus expected to manage this.
@@ -220,6 +522,11 @@ static int tls_open(URLContext *h, const char *uri, int flags, AVDictionary **op
             av_log(h, AV_LOG_ERROR, "mbedtls_x509_crt_parse_file for own cert returned %d\n", ret);
             goto fail;
         }
+    } else if (shr->cert_buf) {
+        if ((ret = mbedtls_x509_crt_parse(&tls_ctx->own_cert, shr->cert_buf, strlen(shr->cert_buf) + 1)) != 0) {
+            av_log(h, AV_LOG_ERROR, "mbedtls_x509_crt_parse for own cert returned %d\n", ret);
+            goto fail;
+        }
     }
 
     // seed the random number generator
@@ -244,11 +551,25 @@ static int tls_open(URLContext *h, const char *uri, int flags, AVDictionary **op
             handle_pk_parse_error(h, ret);
             goto fail;
         }
+    } else if (shr->key_buf) {
+        if ((ret = mbedtls_pk_parse_key(&tls_ctx->priv_key,
+                                        shr->key_buf,
+                                        strlen(shr->key_buf) + 1,
+                                        NULL,
+                                        0
+#if MBEDTLS_VERSION_MAJOR >= 3
+                                        , mbedtls_ctr_drbg_random,
+                                        &tls_ctx->ctr_drbg_context
+#endif
+                                        )) != 0) {
+            handle_pk_parse_error(h, ret);
+            goto fail;
+        }
     }
 
     if ((ret = mbedtls_ssl_config_defaults(&tls_ctx->ssl_config,
                                            shr->listen ? MBEDTLS_SSL_IS_SERVER : MBEDTLS_SSL_IS_CLIENT,
-                                           MBEDTLS_SSL_TRANSPORT_STREAM,
+                                           shr->is_dtls ? MBEDTLS_SSL_TRANSPORT_DATAGRAM : MBEDTLS_SSL_TRANSPORT_STREAM,
                                            MBEDTLS_SSL_PRESET_DEFAULT)) != 0) {
         av_log(h, AV_LOG_ERROR, "mbedtls_ssl_config_defaults returned %d\n", ret);
         goto fail;
@@ -273,7 +594,23 @@ static int tls_open(URLContext *h, const char *uri, int flags, AVDictionary **op
         av_log(h, AV_LOG_ERROR, "mbedtls_ssl_conf_own_cert returned %d\n", ret);
         goto fail;
     }
+    if (shr->is_dtls) {
+        mbedtls_ssl_conf_dtls_cookies(&tls_ctx->ssl_config, NULL, NULL, NULL);
+        if (shr->use_srtp) {
+#if defined(MBEDTLS_SSL_DTLS_SRTP)
+            if ((ret = mbedtls_ssl_conf_dtls_srtp_protection_profiles(&tls_ctx->ssl_config, profiles)) != 0) {
+                av_log(h, AV_LOG_ERROR, "mbedtls_ssl_conf_dtls_srtp_protection_profiles returned %d\n", ret);
+                goto fail;
+            }
+            mbedtls_ssl_set_export_keys_cb(&tls_ctx->ssl_context, dtls_srtp_key_derivation, &tls_ctx->srtp_key);
+#else
+            av_log(h, AV_LOG_ERROR, "DTLS-SRTP is not supported in this mbedtls build\n");
+            ret = AVERROR(ENOSYS);
+            goto fail;
+#endif
+        }
 
+    }
     if ((ret = mbedtls_ssl_setup(&tls_ctx->ssl_context, &tls_ctx->ssl_config)) != 0) {
         av_log(h, AV_LOG_ERROR, "mbedtls_ssl_setup returned %d\n", ret);
         goto fail;
@@ -287,14 +624,17 @@ static int tls_open(URLContext *h, const char *uri, int flags, AVDictionary **op
     }
 
     // set I/O functions to use FFmpeg internal code for transport layer
-    mbedtls_ssl_set_bio(&tls_ctx->ssl_context, shr->tcp, mbedtls_send, mbedtls_recv, NULL);
+    mbedtls_ssl_set_bio(&tls_ctx->ssl_context, tls_ctx, mbedtls_send, mbedtls_recv, NULL);
 
-    // ssl handshake
-    while ((ret = mbedtls_ssl_handshake(&tls_ctx->ssl_context)) != 0) {
-        if (ret != MBEDTLS_ERR_SSL_WANT_READ && ret != MBEDTLS_ERR_SSL_WANT_WRITE) {
-            handle_handshake_error(h, ret);
+    if (shr->is_dtls) {
+        mbedtls_ssl_set_timer_cb(&tls_ctx->ssl_context, &tls_ctx->timer, mbedtls_timing_set_delay, mbedtls_timing_get_delay);
+        if (shr->mtu)
+            mbedtls_ssl_set_mtu(&tls_ctx->ssl_context, shr->mtu);
+    }
+    if (!shr->external_sock) {
+        ret = tls_handshake(h);
+        if (ret < 0)
             goto fail;
-        }
     }
 
     if (shr->verify) {
@@ -316,6 +656,14 @@ fail:
     return AVERROR(EIO);
 }
 
+static int dtls_open(URLContext *h, const char *uri, int flags, AVDictionary **options)
+{
+    TLSContext *tls_ctx = h->priv_data;
+    TLSShared *shr = &tls_ctx->tls_shared;
+    shr->is_dtls = 1;
+    return tls_open(h, uri, flags, options);
+}
+
 static int handle_tls_error(URLContext *h, const char* func_name, int ret)
 {
     switch (ret) {
@@ -341,10 +689,12 @@ static int handle_tls_error(URLContext *h, const char* func_name, int ret)
 static int tls_read(URLContext *h, uint8_t *buf, int size)
 {
     TLSContext *tls_ctx = h->priv_data;
+    TLSShared *shr = &tls_ctx->tls_shared;
+    URLContext *uc = shr->is_dtls ? shr->udp : shr->tcp;
     int ret;
 
-    tls_ctx->tls_shared.tcp->flags &= ~AVIO_FLAG_NONBLOCK;
-    tls_ctx->tls_shared.tcp->flags |= h->flags & AVIO_FLAG_NONBLOCK;
+    uc->flags &= ~AVIO_FLAG_NONBLOCK;
+    uc->flags |= h->flags & AVIO_FLAG_NONBLOCK;
     if ((ret = mbedtls_ssl_read(&tls_ctx->ssl_context, buf, size)) > 0) {
         // return read length
         return ret;
@@ -356,10 +706,12 @@ static int tls_read(URLContext *h, uint8_t *buf, int size)
 static int tls_write(URLContext *h, const uint8_t *buf, int size)
 {
     TLSContext *tls_ctx = h->priv_data;
+    TLSShared *shr = &tls_ctx->tls_shared;
+    URLContext *uc = shr->is_dtls ? shr->udp : shr->tcp;
     int ret;
 
-    tls_ctx->tls_shared.tcp->flags &= ~AVIO_FLAG_NONBLOCK;
-    tls_ctx->tls_shared.tcp->flags |= h->flags & AVIO_FLAG_NONBLOCK;
+    uc->flags &= ~AVIO_FLAG_NONBLOCK;
+    uc->flags |= h->flags & AVIO_FLAG_NONBLOCK;
     if ((ret = mbedtls_ssl_write(&tls_ctx->ssl_context, buf, size)) > 0) {
         // return written length
         return ret;
@@ -405,3 +757,24 @@ const URLProtocol ff_tls_protocol = {
     .flags          = URL_PROTOCOL_FLAG_NETWORK,
     .priv_data_class = &tls_class,
 };
+
+static const AVClass dtls_class = {
+    .class_name = "dtls",
+    .item_name  = av_default_item_name,
+    .option     = options,
+    .version    = LIBAVUTIL_VERSION_INT,
+};
+
+const URLProtocol ff_dtls_protocol = {
+    .name           = "dtls",
+    .url_open2      = dtls_open,
+    .url_handshake  = tls_handshake,
+    .url_read       = tls_read,
+    .url_write      = tls_write,
+    .url_close      = tls_close,
+    .url_get_file_handle = tls_get_file_handle,
+    .url_get_short_seek  = tls_get_short_seek,
+    .priv_data_size = sizeof(TLSContext),
+    .flags          = URL_PROTOCOL_FLAG_NETWORK,
+    .priv_data_class = &dtls_class,
+};
-- 
2.52.0


>From 8f8288bae91982f2930b806584f9b78e82916535 Mon Sep 17 00:00:00 2001
From: Jack Lau <jacklau1222gm@gmail.com>
Date: Mon, 9 Feb 2026 15:32:59 +0800
Subject: [PATCH 2/3] avformat/tls_mbdetls: set dtls remote addr in listen mode

Signed-off-by: Jack Lau <jacklau1222gm@gmail.com>
---
 libavformat/tls_mbedtls.c | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/libavformat/tls_mbedtls.c b/libavformat/tls_mbedtls.c
index dd08bd6f15..1762bd2b91 100644
--- a/libavformat/tls_mbedtls.c
+++ b/libavformat/tls_mbedtls.c
@@ -34,6 +34,7 @@
 
 #include "avformat.h"
 #include "internal.h"
+#include "network.h"
 #include "url.h"
 #include "tls.h"
 #include "libavutil/mem.h"
@@ -258,6 +259,8 @@ typedef struct TLSContext {
     mbedtls_pk_context priv_key;
     char *priv_key_pw;
     dtls_srtp_keys srtp_key;
+    struct sockaddr_storage dest_addr;
+    socklen_t dest_addr_len;
 } TLSContext;
 
 int ff_tls_set_external_socket(URLContext *h, URLContext *sock)
@@ -376,8 +379,20 @@ static int mbedtls_recv(void *ctx, unsigned char *buf, size_t len)
     TLSShared *shr = &tls_ctx->tls_shared;
     URLContext *h = shr->is_dtls ? shr->udp : shr->tcp;
     int ret = ffurl_read(h, buf, len);
-    if (ret >= 0)
+    if (ret >= 0) {
+        if (shr->is_dtls && shr->listen && !tls_ctx->dest_addr_len) {
+            int err_ret;
+
+            ff_udp_get_last_recv_addr(shr->udp, &tls_ctx->dest_addr, &tls_ctx->dest_addr_len);
+            err_ret = ff_udp_set_remote_addr(shr->udp, (struct sockaddr *)&tls_ctx->dest_addr, tls_ctx->dest_addr_len, 1);
+            if (err_ret < 0) {
+                av_log(tls_ctx, AV_LOG_ERROR, "Failed connecting udp context\n");
+                return err_ret;
+            }
+            av_log(tls_ctx, AV_LOG_TRACE, "Set UDP remote addr on UDP socket, now 'connected'\n");
+        }
         return ret;
+    }
     if (h->max_packet_size && len > h->max_packet_size)
         return MBEDTLS_ERR_SSL_BUFFER_TOO_SMALL;
 
-- 
2.52.0


>From 267711a6a07629f9a39b33a723dadfbbb3d8d264 Mon Sep 17 00:00:00 2001
From: Jack Lau <jacklau1222gm@gmail.com>
Date: Mon, 9 Feb 2026 15:57:23 +0800
Subject: [PATCH 3/3] avformat/tls_mbdetls: generate self-signed cert and key
 when none is provided in listen mode

Signed-off-by: Jack Lau <jacklau1222gm@gmail.com>
---
 libavformat/tls_mbedtls.c | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/libavformat/tls_mbedtls.c b/libavformat/tls_mbedtls.c
index 1762bd2b91..97f2530b1c 100644
--- a/libavformat/tls_mbedtls.c
+++ b/libavformat/tls_mbedtls.c
@@ -582,6 +582,22 @@ static int tls_open(URLContext *h, const char *uri, int flags, AVDictionary **op
         }
     }
 
+    if (shr->listen && !shr->cert_file && !shr->cert_buf && !shr->key_file && !shr->key_buf) {
+        char buf[4096];
+        if ((ret = mbedtls_gen_pkey(&tls_ctx->priv_key)) != 0) {
+            av_log(h, AV_LOG_ERROR, "failed to generate priv_key, returned %d\n", ret);
+            goto fail;
+        }
+        if ((ret = mbedtls_gen_x509_cert(&tls_ctx->priv_key, buf, sizeof(buf))) != 0) {
+            av_log(h, AV_LOG_ERROR, "failed to generate cert, returned %d\n", ret);
+            goto fail;
+        }
+        if ((ret = mbedtls_x509_crt_parse(&tls_ctx->own_cert, buf, sizeof(buf))) != 0) {
+            av_log(h, AV_LOG_ERROR, "failed to parse generated cert, returned %d\n", ret);
+            goto fail;
+        }
+    }
+
     if ((ret = mbedtls_ssl_config_defaults(&tls_ctx->ssl_config,
                                            shr->listen ? MBEDTLS_SSL_IS_SERVER : MBEDTLS_SSL_IS_CLIENT,
                                            shr->is_dtls ? MBEDTLS_SSL_TRANSPORT_DATAGRAM : MBEDTLS_SSL_TRANSPORT_STREAM,
-- 
2.52.0

_______________________________________________
ffmpeg-devel mailing list -- ffmpeg-devel@ffmpeg.org
To unsubscribe send an email to ffmpeg-devel-leave@ffmpeg.org