[FFmpeg-devel] [PATCH] avformat/tls_gnutls: add initial dtls support (PR #20464)

[Jack Lau via ffmpeg-devel <ffmpeg-devel@ffmpeg.org>]

PR #20464 opened by Jack Lau (JackLau)
URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/20464
Patch URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/20464.patch


>From 042808e328443b002bd64bd6f3bcf1749e8be4a9 Mon Sep 17 00:00:00 2001
From: Jack Lau <jacklau1222@qq.com>
Date: Mon, 8 Sep 2025 16:08:22 +0800
Subject: [PATCH 1/2] avformat/tls_gnutls: add initial dtls support

Set GNUTLS_DATAGRAM flag when is_dtls is true.
Set mtu when it's specified.
Modify the read/write function could use udp socket.

There are more patches to make dtls really work.

Signed-off-by: Jack Lau <jacklau1222@qq.com>
---
 libavformat/tls_gnutls.c | 55 ++++++++++++++++++++++++++++++++++++----
 1 file changed, 50 insertions(+), 5 deletions(-)

diff --git a/libavformat/tls_gnutls.c b/libavformat/tls_gnutls.c
index e894765d7c..d4751139a9 100644
--- a/libavformat/tls_gnutls.c
+++ b/libavformat/tls_gnutls.c
@@ -22,6 +22,7 @@
 #include <errno.h>
 
 #include <gnutls/gnutls.h>
+#include <gnutls/dtls.h>
 #include <gnutls/x509.h>
 
 #include "avformat.h"
@@ -151,6 +152,7 @@ static int tls_open(URLContext *h, const char *uri, int flags, AVDictionary **op
 {
     TLSContext *c = h->priv_data;
     TLSShared *s = &c->tls_shared;
+    uint16_t gnutls_flags = 0;
     int ret;
 
     ff_gnutls_init();
@@ -158,7 +160,14 @@ static int tls_open(URLContext *h, const char *uri, int flags, AVDictionary **op
     if ((ret = ff_tls_open_underlying(s, h, uri, options)) < 0)
         goto fail;
 
-    gnutls_init(&c->session, s->listen ? GNUTLS_SERVER : GNUTLS_CLIENT);
+    if (s->is_dtls)
+        gnutls_flags |= GNUTLS_DATAGRAM;
+
+    if (s->listen)
+        gnutls_flags |= GNUTLS_SERVER
+    else
+        gnutls_flags |= GNUTLS_CLIENT;
+    gnutls_init(&c->session, gnutls_flags);
     if (!s->listen && !s->numerichost)
         gnutls_server_name_set(c->session, GNUTLS_NAME_DNS, s->host, strlen(s->host));
     gnutls_certificate_allocate_credentials(&c->cred);
@@ -190,6 +199,10 @@ static int tls_open(URLContext *h, const char *uri, int flags, AVDictionary **op
     gnutls_transport_set_pull_function(c->session, gnutls_url_pull);
     gnutls_transport_set_push_function(c->session, gnutls_url_push);
     gnutls_transport_set_ptr(c->session, c);
+    if (s->is_dtls){
+        if (s->mtu)
+            gnutls_dtls_set_mtu(c->session, s->mtu);
+    }
     gnutls_set_default_priority(c->session);
     do {
         if (ff_check_interrupt(&h->interrupt_callback)) {
@@ -243,13 +256,23 @@ fail:
     return ret;
 }
 
+static int dtls_open(URLContext *h, const char *uri, int flags, AVDictionary **options)
+{
+    TLSContext *c = h->priv_data;
+    TLSShared *s = &c->tls_shared;
+    s->is_dtls = 1;
+    return tls_open(h, uri, flags, options);
+}
+
 static int tls_read(URLContext *h, uint8_t *buf, int size)
 {
     TLSContext *c = h->priv_data;
+    TLSShared *s = &c->tls_shared;
+    URLContext *uc = s->is_dtls ? s->udp : s->tcp;
     int ret;
     // Set or clear the AVIO_FLAG_NONBLOCK on c->tls_shared.tcp
-    c->tls_shared.tcp->flags &= ~AVIO_FLAG_NONBLOCK;
-    c->tls_shared.tcp->flags |= h->flags & AVIO_FLAG_NONBLOCK;
+    uc->flags &= ~AVIO_FLAG_NONBLOCK;
+    uc->flags |= h->flags & AVIO_FLAG_NONBLOCK;
     ret = gnutls_record_recv(c->session, buf, size);
     if (ret > 0)
         return ret;
@@ -261,10 +284,12 @@ static int tls_read(URLContext *h, uint8_t *buf, int size)
 static int tls_write(URLContext *h, const uint8_t *buf, int size)
 {
     TLSContext *c = h->priv_data;
+    TLSShared *s = &c->tls_shared;
+    URLContext *uc = s->is_dtls ? s->udp : s->tcp;
     int ret;
     // Set or clear the AVIO_FLAG_NONBLOCK on c->tls_shared.tcp
-    c->tls_shared.tcp->flags &= ~AVIO_FLAG_NONBLOCK;
-    c->tls_shared.tcp->flags |= h->flags & AVIO_FLAG_NONBLOCK;
+    uc->flags &= ~AVIO_FLAG_NONBLOCK;
+    uc->flags |= h->flags & AVIO_FLAG_NONBLOCK;
     ret = gnutls_record_send(c->session, buf, size);
     if (ret > 0)
         return ret;
@@ -309,3 +334,23 @@ const URLProtocol ff_tls_protocol = {
     .flags          = URL_PROTOCOL_FLAG_NETWORK,
     .priv_data_class = &tls_class,
 };
+
+static const AVClass dtls_class = {
+    .class_name = "dtls",
+    .item_name  = av_default_item_name,
+    .option     = options,
+    .version    = LIBAVUTIL_VERSION_INT,
+};
+
+const URLProtocol ff_dtls_protocol = {
+    .name           = "dtls",
+    .url_open2      = dtls_open,
+    .url_read       = tls_read,
+    .url_write      = tls_write,
+    .url_close      = tls_close,
+    .url_get_file_handle = tls_get_file_handle,
+    .url_get_short_seek  = tls_get_short_seek,
+    .priv_data_size = sizeof(TLSContext),
+    .flags          = URL_PROTOCOL_FLAG_NETWORK,
+    .priv_data_class = &dtls_class,
+};
-- 
2.49.1


>From 75d6a40a91ae6d9a25f4c32e3dba5fac97e4f74a Mon Sep 17 00:00:00 2001
From: Jack Lau <jacklau1222@qq.com>
Date: Mon, 8 Sep 2025 16:31:10 +0800
Subject: [PATCH 2/2] avformat/tls_gnutls: add av_assert0() for tls_shared

Signed-off-by: Jack Lau <jacklau1222@qq.com>
---
 libavformat/tls_gnutls.c | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/libavformat/tls_gnutls.c b/libavformat/tls_gnutls.c
index d4751139a9..61c5d79c9b 100644
--- a/libavformat/tls_gnutls.c
+++ b/libavformat/tls_gnutls.c
@@ -30,6 +30,7 @@
 #include "os_support.h"
 #include "url.h"
 #include "tls.h"
+#include "libavutil/avassert.h"
 #include "libavutil/opt.h"
 #include "libavutil/thread.h"
 
@@ -154,6 +155,7 @@ static int tls_open(URLContext *h, const char *uri, int flags, AVDictionary **op
     TLSShared *s = &c->tls_shared;
     uint16_t gnutls_flags = 0;
     int ret;
+    av_assert0(s);
 
     ff_gnutls_init();
 
@@ -260,6 +262,7 @@ static int dtls_open(URLContext *h, const char *uri, int flags, AVDictionary **o
 {
     TLSContext *c = h->priv_data;
     TLSShared *s = &c->tls_shared;
+    av_assert0(s);
     s->is_dtls = 1;
     return tls_open(h, uri, flags, options);
 }
-- 
2.49.1

_______________________________________________
ffmpeg-devel mailing list -- ffmpeg-devel@ffmpeg.org
To unsubscribe send an email to ffmpeg-devel-leave@ffmpeg.org