* [FFmpeg-devel] [PR] avformat/tls_openssl: fix DTLS handshake retransmission in non-blocking mode (PR #21777)
@ 2026-02-17 22:50 Nariman-Sayed via ffmpeg-devel
2026-02-20 20:06 ` [FFmpeg-devel] " Nariman Sayed via ffmpeg-devel
0 siblings, 1 reply; 2+ messages in thread
From: Nariman-Sayed via ffmpeg-devel @ 2026-02-17 22:50 UTC (permalink / raw)
To: ffmpeg-devel; +Cc: Nariman-Sayed
PR #21777 opened by Nariman-Sayed
URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/21777
Patch URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/21777.patch
Switch from blocking to non-blocking mode to properly handle DTLS
handshake retransmissions when packets are lost.
Key changes:
- Enable AVIO_FLAG_NONBLOCK on UDP socket for DTLS connections
- Implement retry loop with poll() to wait for socket readiness
- Use DTLSv1_get_timeout() to get proper timeout from OpenSSL
- Call DTLSv1_handle_timeout() to trigger retransmissions on timeout
- Handle SSL_ERROR_WANT_READ/WRITE correctly in non-blocking mode
>From 9144a430e6ea89ad5199b679faf0b8ef1155145f Mon Sep 17 00:00:00 2001
From: Nariman-Sayed <narimansayed28@gmail.com>
Date: Wed, 11 Feb 2026 23:20:08 +0000
Subject: [PATCH 1/3] avformat/tls_openssl: Improve DTLS handshake retry logic
This PR implements a proper retry loop for the DTLS handshake using `poll()` to handle timeouts and retransmissions correctly.
Key changes:
* Added a `while (1)` loop around `SSL_do_handshake` for DTLS connections.
* Integrated `poll()` with a 100ms timeout to wait for socket readiness before retrying.
* Correctly handles `SSL_ERROR_WANT_READ` and `SSL_ERROR_WANT_WRITE`.
* Preserved all original file formatting, headers, and labels (such as `end:`) to avoid unnecessary changes.
This version supersedes my previous PR and addresses all formatting feedback provided by @JackLau.
Signed-off-by: Nariman-Sayed <narimansayed28@gmail.com>
---
libavformat/tls_openssl.c | 58 ++++++++++++++++++++++++++++++++++-----
1 file changed, 51 insertions(+), 7 deletions(-)
diff --git a/libavformat/tls_openssl.c b/libavformat/tls_openssl.c
index c7d6ff6cf4..aa7a7a3f95 100644
--- a/libavformat/tls_openssl.c
+++ b/libavformat/tls_openssl.c
@@ -33,6 +33,13 @@
#include <openssl/err.h>
#include <openssl/x509v3.h>
+#include "libavformat/network.h"
+#include "libavformat/url.h"
+#include "libavutil/time.h"
+#include <sys/time.h>
+#if HAVE_POLL_H
+#include <poll.h>
+#endif
/**
* Convert an EVP_PKEY to a PEM string.
*/
@@ -630,24 +637,61 @@ static int dtls_handshake(URLContext *h)
c->tls_shared.udp->flags &= ~AVIO_FLAG_NONBLOCK;
- r0 = SSL_do_handshake(c->ssl);
- if (r0 <= 0) {
+ while (1) {
+ r0 = SSL_do_handshake(c->ssl);
+ if (r0 > 0) {
+ av_log(c, AV_LOG_TRACE, "Handshake success, r0=%d\n", r0);
+ break;
+ }
+
r1 = SSL_get_error(c->ssl, r0);
- if (r1 != SSL_ERROR_WANT_READ && r1 != SSL_ERROR_WANT_WRITE && r1 != SSL_ERROR_ZERO_RETURN) {
+ if (r1 == SSL_ERROR_WANT_READ || r1 == SSL_ERROR_WANT_WRITE) {
+#if HAVE_POLL_H
+ struct pollfd p;
+ struct timeval tv;
+ int timeout_ms = 100;
+ int fd = ffurl_get_file_handle(c->tls_shared.udp);
+ int r;
+
+ if (DTLSv1_get_timeout(c->ssl, &tv))
+ timeout_ms = tv.tv_sec * 1000 + tv.tv_usec / 1000;
+
+ if (timeout_ms <= 0)
+ timeout_ms = 100;
+
+ p.fd = fd;
+ p.events = (r1 == SSL_ERROR_WANT_READ) ? POLLIN : POLLOUT;
+
+ r = poll(&p, 1, timeout_ms);
+ if (r == 0) {
+ DTLSv1_handle_timeout(c->ssl);
+ continue;
+ }
+ else if (r < 0) {
+ if (errno == EINTR || errno == EAGAIN)
+ continue;
+ ret = AVERROR(errno);
+ goto end;
+ }
+ continue;
+#else
+ goto end;
+#endif
+ }
+
+ if (r1 != SSL_ERROR_ZERO_RETURN) {
av_log(c, AV_LOG_ERROR, "Handshake failed, r0=%d, r1=%d\n", r0, r1);
ret = print_ssl_error(h, r0);
- goto end;
}
- } else {
- av_log(c, AV_LOG_TRACE, "Handshake success, r0=%d\n", r0);
+ goto end;
}
- /* Check whether the handshake is completed. */
if (SSL_is_init_finished(c->ssl) != TLS_ST_OK)
goto end;
ret = 0;
+
end:
return ret;
}
--
2.52.0
>From ff8702405d2133639737d187b4691493471f6616 Mon Sep 17 00:00:00 2001
From: Nariman-Sayed <narimansayed28@gmail.com>
Date: Thu, 12 Feb 2026 03:41:04 +0000
Subject: [PATCH 2/3] avformat/tls_openssl: fix DTLS handshake timeout handling
The DTLS handshake could fail or hang indefinitely when packets are lost or delayed, as there was no proper retry mechanism with timeout handling.
This commit implements a proper retry loop using poll() to handle timeouts and retransmissions correctly.
Key changes:
- Added a while(1) loop around SSL_do_handshake for DTLS connections
- Integrated poll() with timeout to wait for socket readiness
- Properly handles SSL_ERROR_WANT_READ and SSL_ERROR_WANT_WRITE
- Calls DTLSv1_handle_timeout() when poll() times out
Addresses review feedback from @JackLau.
Signed-off-by: Nariman-Sayed <narimansayed28@gmail.com>
---
libavformat/tls_openssl.c | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/libavformat/tls_openssl.c b/libavformat/tls_openssl.c
index aa7a7a3f95..ee84e87f33 100644
--- a/libavformat/tls_openssl.c
+++ b/libavformat/tls_openssl.c
@@ -643,7 +643,7 @@ static int dtls_handshake(URLContext *h)
av_log(c, AV_LOG_TRACE, "Handshake success, r0=%d\n", r0);
break;
}
-
+ /* Check whether the handshake is completed. */
r1 = SSL_get_error(c->ssl, r0);
if (r1 == SSL_ERROR_WANT_READ || r1 == SSL_ERROR_WANT_WRITE) {
@@ -691,7 +691,6 @@ static int dtls_handshake(URLContext *h)
goto end;
ret = 0;
-
end:
return ret;
}
--
2.52.0
>From ba29586e0e72f7f64ed3605ae1e1eb783b65eed3 Mon Sep 17 00:00:00 2001
From: Nariman-Sayed <narimansayed28@gmail.com>
Date: Sun, 15 Feb 2026 12:53:56 +0000
Subject: [PATCH 3/3] avformat/tls_openssl: Use non-blocking mode for DTLS
handshake retry
Switch from blocking to non-blocking mode to properly handle DTLS
handshake retransmissions when packets are lost.
Key changes:
- Enable AVIO_FLAG_NONBLOCK on UDP socket for DTLS connections
- Implement retry loop with poll() to wait for socket readiness
- Use DTLSv1_get_timeout() to get proper timeout from OpenSSL
- Call DTLSv1_handle_timeout() to trigger retransmissions on timeout
- Handle SSL_ERROR_WANT_READ/WRITE correctly in non-blocking mode
Signed-off-by: Nariman-Sayed <narimansayed28@gmail.com>
---
libavformat/tls_openssl.c | 77 +++++++++++++++------------------------
1 file changed, 29 insertions(+), 48 deletions(-)
diff --git a/libavformat/tls_openssl.c b/libavformat/tls_openssl.c
index ee84e87f33..bbc01c7d75 100644
--- a/libavformat/tls_openssl.c
+++ b/libavformat/tls_openssl.c
@@ -632,67 +632,48 @@ static void openssl_info_callback(const SSL *ssl, int where, int ret) {
static int dtls_handshake(URLContext *h)
{
- int ret = 1, r0, r1;
TLSContext *c = h->priv_data;
+ int ret, err;
- c->tls_shared.udp->flags &= ~AVIO_FLAG_NONBLOCK;
+ struct pollfd pfd;
+ pfd.fd = ffurl_get_file_handle(c->tls_shared.udp);
+ pfd.events = POLLIN;
+ pfd.revents = 0;
- while (1) {
- r0 = SSL_do_handshake(c->ssl);
- if (r0 > 0) {
- av_log(c, AV_LOG_TRACE, "Handshake success, r0=%d\n", r0);
- break;
- }
- /* Check whether the handshake is completed. */
- r1 = SSL_get_error(c->ssl, r0);
+ /* use nonblocking mode for DTLS retransmission */
+ c->tls_shared.udp->flags |= AVIO_FLAG_NONBLOCK;
- if (r1 == SSL_ERROR_WANT_READ || r1 == SSL_ERROR_WANT_WRITE) {
-#if HAVE_POLL_H
- struct pollfd p;
- struct timeval tv;
- int timeout_ms = 100;
- int fd = ffurl_get_file_handle(c->tls_shared.udp);
- int r;
+ for (;;) {
+ ret = SSL_do_handshake(c->ssl);
+ if (ret == 1)
+ return 0;
- if (DTLSv1_get_timeout(c->ssl, &tv))
- timeout_ms = tv.tv_sec * 1000 + tv.tv_usec / 1000;
+ err = SSL_get_error(c->ssl, ret);
- if (timeout_ms <= 0)
- timeout_ms = 100;
+ if (err == SSL_ERROR_WANT_READ || err == SSL_ERROR_WANT_WRITE) {
+ struct timeval timeout;
+ int timeout_ms = 1000;
- p.fd = fd;
- p.events = (r1 == SSL_ERROR_WANT_READ) ? POLLIN : POLLOUT;
+ if (DTLSv1_get_timeout(c->ssl, &timeout))
+ timeout_ms = timeout.tv_sec * 1000 +
+ timeout.tv_usec / 1000;
- r = poll(&p, 1, timeout_ms);
- if (r == 0) {
- DTLSv1_handle_timeout(c->ssl);
+ ret = poll(&pfd, 1, timeout_ms);
+ if (ret > 0) {
+ if (pfd.revents & (POLLERR | POLLHUP))
+ return AVERROR(EIO);
continue;
+ } else if (ret == 0) {
+ if (DTLSv1_handle_timeout(c->ssl) < 0)
+ return AVERROR(EIO);
+ continue;
+ } else {
+ return AVERROR(errno);
}
- else if (r < 0) {
- if (errno == EINTR || errno == EAGAIN)
- continue;
- ret = AVERROR(errno);
- goto end;
- }
- continue;
-#else
- goto end;
-#endif
}
- if (r1 != SSL_ERROR_ZERO_RETURN) {
- av_log(c, AV_LOG_ERROR, "Handshake failed, r0=%d, r1=%d\n", r0, r1);
- ret = print_ssl_error(h, r0);
- }
- goto end;
+ return AVERROR(EIO);
}
-
- if (SSL_is_init_finished(c->ssl) != TLS_ST_OK)
- goto end;
-
- ret = 0;
-end:
- return ret;
}
static av_cold int openssl_init_ca_key_cert(URLContext *h)
--
2.52.0
_______________________________________________
ffmpeg-devel mailing list -- ffmpeg-devel@ffmpeg.org
To unsubscribe send an email to ffmpeg-devel-leave@ffmpeg.org
^ permalink raw reply [flat|nested] 2+ messages in thread
end of thread, other threads:[~2026-02-20 20:17 UTC | newest]
Thread overview: 2+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2026-02-17 22:50 [FFmpeg-devel] [PR] avformat/tls_openssl: fix DTLS handshake retransmission in non-blocking mode (PR #21777) Nariman-Sayed via ffmpeg-devel
2026-02-20 20:06 ` [FFmpeg-devel] " Nariman Sayed via ffmpeg-devel
Git Inbox Mirror of the ffmpeg-devel mailing list - see https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
This inbox may be cloned and mirrored by anyone:
git clone --mirror https://master.gitmailbox.com/ffmpegdev/0 ffmpegdev/git/0.git
# If you have public-inbox 1.1+ installed, you may
# initialize and index your mirror using the following commands:
public-inbox-init -V2 ffmpegdev ffmpegdev/ https://master.gitmailbox.com/ffmpegdev \
ffmpegdev@gitmailbox.com
public-inbox-index ffmpegdev
Example config snippet for mirrors.
AGPL code for this site: git clone https://public-inbox.org/public-inbox.git