[FFmpeg-devel] [PATCH] avformat/whip: remove DTLSState enum (PR #20231)

[FFmpeg-devel] [PATCH] avformat/whip: remove DTLSState enum (PR #20231)

[Jack Lau]

PR #20231 opened by Jack Lau (JackLau)
URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/20231
Patch URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/20231.patch

This patch aims to simplify the dtls handshake process
since dtls handshake use force block mode

We can just use the return code instead of DTLSState enum

Signed-off-by: Jack Lau <jacklau1222@qq.com>


This patch was split from #20030 to make the patchset smaller and easier to review.


From cdf8678bc18debac8d9bc1ebf8aa2563ff6be878 Mon Sep 17 00:00:00 2001
From: Jack Lau <jacklau1222@qq.com>
Date: Mon, 21 Jul 2025 15:47:37 +0800
Subject: [PATCH] avformat/whip: remove DTLSState enum

This patch aims to simplify the dtls handshake process
since dtls handshake use force block mode

We can just use the return code instead of DTLSState enum

Signed-off-by: Jack Lau <jacklau1222@qq.com>
---
 libavformat/tls.h          | 15 ----------
 libavformat/tls_openssl.c  |  7 -----
 libavformat/tls_schannel.c |  7 -----
 libavformat/whip.c         | 61 ++++++++------------------------------
 4 files changed, 13 insertions(+), 77 deletions(-)

diff --git a/libavformat/tls.h b/libavformat/tls.h
index 4d4999aa7c..2afa248f4c 100644
--- a/libavformat/tls.h
+++ b/libavformat/tls.h
@@ -34,17 +34,6 @@
  */
 #define MAX_CERTIFICATE_SIZE 8192
 
-enum DTLSState {
-    DTLS_STATE_NONE,
-
-    /* Whether DTLS handshake is finished. */
-    DTLS_STATE_FINISHED,
-    /* Whether DTLS session is closed. */
-    DTLS_STATE_CLOSED,
-    /* Whether DTLS handshake is failed. */
-    DTLS_STATE_FAILED,
-};
-
 typedef struct TLSShared {
     char *ca_file;
     int verify;
@@ -65,8 +54,6 @@ typedef struct TLSShared {
     int is_dtls;
     int use_srtp;
 
-    enum DTLSState state;
-
     /* The certificate and private key content used for DTLS handshake */
     char* cert_buf;
     char* key_buf;
@@ -112,8 +99,6 @@ int ff_tls_set_external_socket(URLContext *h, URLContext *sock);
 
 int ff_dtls_export_materials(URLContext *h, char *dtls_srtp_materials, size_t materials_sz);
 
-int ff_dtls_state(URLContext *h);
-
 int ff_ssl_read_key_cert(char *key_url, char *cert_url, char *key_buf, size_t key_sz, char *cert_buf, size_t cert_sz, char **fingerprint);
 
 int ff_ssl_gen_key_cert(char *key_buf, size_t key_sz, char *cert_buf, size_t cert_sz, char **fingerprint);
diff --git a/libavformat/tls_openssl.c b/libavformat/tls_openssl.c
index 65b2966688..ca7251dcf1 100644
--- a/libavformat/tls_openssl.c
+++ b/libavformat/tls_openssl.c
@@ -541,12 +541,6 @@ int ff_dtls_export_materials(URLContext *h, char *dtls_srtp_materials, size_t ma
     return 0;
 }
 
-int ff_dtls_state(URLContext *h)
-{
-    TLSContext *c = h->priv_data;
-    return c->tls_shared.state;
-}
-
 static int print_ssl_error(URLContext *h, int ret)
 {
     TLSContext *c = h->priv_data;
@@ -721,7 +715,6 @@ static int dtls_handshake(URLContext *h)
         goto end;
 
     ret = 0;
-    c->tls_shared.state = DTLS_STATE_FINISHED;
 end:
     return ret;
 }
diff --git a/libavformat/tls_schannel.c b/libavformat/tls_schannel.c
index b60e3100be..b854f484fa 100644
--- a/libavformat/tls_schannel.c
+++ b/libavformat/tls_schannel.c
@@ -681,12 +681,6 @@ int ff_dtls_export_materials(URLContext *h, char *dtls_srtp_materials, size_t ma
 #endif
 }
 
-int ff_dtls_state(URLContext *h)
-{
-    TLSContext *c = h->priv_data;
-    return c->tls_shared.state;
-}
-
 static void init_sec_buffer(SecBuffer *buffer, unsigned long type,
                             void *data, unsigned long size)
 {
@@ -1111,7 +1105,6 @@ static int tls_handshake(URLContext *h)
 #endif
 
     c->connected = 1;
-    s->state = DTLS_STATE_FINISHED;
 
 fail:
     return ret;
diff --git a/libavformat/whip.c b/libavformat/whip.c
index 65fd3b39b2..e2fa774c6f 100644
--- a/libavformat/whip.c
+++ b/libavformat/whip.c
@@ -198,9 +198,6 @@ typedef struct WHIPContext {
 
     /* The state of the RTC connection. */
     enum WHIPState state;
-    /* The callback return value for DTLS. */
-    int dtls_ret;
-    int dtls_closed;
 
     /* Parameters for the input audio and video codecs. */
     AVCodecParameters *audio_par;
@@ -348,41 +345,6 @@ static av_cold int certificate_key_init(AVFormatContext *s)
     return ret;
 }
 
-/**
- * When DTLS state change.
- */
-static int dtls_context_on_state(AVFormatContext *s, const char* type, const char* desc)
-{
-    int ret = 0;
-    WHIPContext *whip = s->priv_data;
-    int state = ff_dtls_state(whip->dtls_uc);
-
-    if (state == DTLS_STATE_CLOSED) {
-        whip->dtls_closed = 1;
-        av_log(whip, AV_LOG_VERBOSE, "DTLS session closed, type=%s, desc=%s, elapsed=%dms\n",
-            type ? type : "", desc ? desc : "", ELAPSED(whip->whip_starttime, av_gettime()));
-        goto error;
-    }
-
-    if (state == DTLS_STATE_FAILED) {
-        whip->state = WHIP_STATE_FAILED;
-        av_log(whip, AV_LOG_ERROR, "DTLS session failed, type=%s, desc=%s\n",
-            type ? type : "", desc ? desc : "");
-        whip->dtls_ret = AVERROR(EIO);
-        goto error;
-    }
-
-    if (state == DTLS_STATE_FINISHED && whip->state < WHIP_STATE_DTLS_FINISHED) {
-        whip->state = WHIP_STATE_DTLS_FINISHED;
-        whip->whip_dtls_time = av_gettime();
-        av_log(whip, AV_LOG_VERBOSE, "DTLS handshake is done, elapsed=%dms\n",
-            ELAPSED(whip->whip_starttime, av_gettime()));
-        return ret;
-    }
-error:
-    return -1;
-}
-
 static av_cold int dtls_initialize(AVFormatContext *s)
 {
     WHIPContext *whip = s->priv_data;
@@ -1326,9 +1288,18 @@ next_packet:
         /* If got any DTLS messages, handle it. */
         if (is_dtls_packet(whip->buf, ret) && whip->state >= WHIP_STATE_ICE_CONNECTED || whip->state == WHIP_STATE_DTLS_CONNECTING) {
             whip->state = WHIP_STATE_DTLS_CONNECTING;
-            if ((ret = ffurl_handshake(whip->dtls_uc)) < 0)
+            ret = ffurl_handshake(whip->dtls_uc);
+            if (ret < 0) {
+                whip->state = WHIP_STATE_FAILED;
+                av_log(whip, AV_LOG_VERBOSE, "DTLS session failed\n");
                 goto end;
-            dtls_context_on_state(s, NULL, NULL);
+            }
+            if (!ret) {
+                whip->state = WHIP_STATE_DTLS_FINISHED;
+                whip->whip_dtls_time = av_gettime();
+                av_log(whip, AV_LOG_VERBOSE, "DTLS handshake is done, elapsed=%dms\n",
+                    ELAPSED(whip->whip_starttime, whip->whip_dtls_time));
+            }
             goto next_packet;
         }
     }
@@ -1771,10 +1742,8 @@ static av_cold int whip_init(AVFormatContext *s)
         goto end;
 
 end:
-    if (ret < 0 && whip->state < WHIP_STATE_FAILED)
+    if (ret < 0)
         whip->state = WHIP_STATE_FAILED;
-    if (ret >= 0 && whip->state >= WHIP_STATE_FAILED && whip->dtls_ret < 0)
-        ret = whip->dtls_ret;
     return ret;
 }
 
@@ -1822,12 +1791,8 @@ static int whip_write_packet(AVFormatContext *s, AVPacket *pkt)
     }
 
 end:
-    if (ret < 0 && whip->state < WHIP_STATE_FAILED)
+    if (ret < 0)
         whip->state = WHIP_STATE_FAILED;
-    if (ret >= 0 && whip->state >= WHIP_STATE_FAILED && whip->dtls_ret < 0)
-        ret = whip->dtls_ret;
-    if (ret >= 0 && whip->dtls_closed)
-        ret = AVERROR(EIO);
     return ret;
 }
 
-- 
2.49.1

_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".