Implement flushing for TLSv1.3 handshakes.

When we finish sending a flight of records, flush the record layer output.
This effectively means calling BIO_flush() on the wbio.

Some things (such as apache2) have custom BIOs that perform buffering and
do not actually send on BIO_write(). Without BIO_flush() the server thinks
it has sent data and starts receiving records, however the client never
sends records since it never received those that the server should have
sent.

Joint work with tb@

ok tb@

5 files changed, 69 insertions, 7 deletions

diff --git a/src/lib/libssl/tls13_handshake.c b/src/lib/libssl/tls13_handshake.c
index 310a2116b8..cca8560fc2 100644
--- a/src/lib/libssl/tls13_handshake.c
+++ b/src/lib/libssl/tls13_handshake.c
@@ -1,4 +1,4 @@
-/*      $OpenBSD: tls13_handshake.c,v 1.69 2021/07/01 17:53:39 jsing Exp $      */
+/*      $OpenBSD: tls13_handshake.c,v 1.70 2021/09/16 19:25:30 jsing Exp $      */
 /*
  * Copyright (c) 2018-2021 Theo Buehler <tb@openbsd.org>
  * Copyright (c) 2019 Joel Sing <jsing@openbsd.org>
@@ -331,6 +331,18 @@ tls13_handshake_advance_state_machine(struct tls13_ctx *ctx)
         return 1;
 }
 
+static int
+tls13_handshake_end_of_flight(struct tls13_ctx *ctx,
+    const struct tls13_handshake_action *previous)
+{
+        const struct tls13_handshake_action *current;
+
+        if ((current = tls13_handshake_active_action(ctx)) == NULL)
+                return 1;
+
+        return current->sender != previous->sender;
+}
+
 int
 tls13_handshake_msg_record(struct tls13_ctx *ctx)
 {
@@ -344,6 +356,7 @@ int
 tls13_handshake_perform(struct tls13_ctx *ctx)
 {
         const struct tls13_handshake_action *action;
+        int sending;
         int ret;
 
         if (!ctx->handshake_started) {
@@ -367,6 +380,13 @@ tls13_handshake_perform(struct tls13_ctx *ctx)
                 if ((action = tls13_handshake_active_action(ctx)) == NULL)
                         return TLS13_IO_FAILURE;
 
+                if (ctx->need_flush) {
+                        if ((ret = tls13_record_layer_flush(ctx->rl)) !=
+                            TLS13_IO_SUCCESS)
+                                return ret;
+                        ctx->need_flush = 0;
+                }
+
                 if (action->handshake_complete) {
                         ctx->handshake_completed = 1;
                         tls13_record_layer_handshake_completed(ctx->rl);
@@ -379,14 +399,16 @@ tls13_handshake_perform(struct tls13_ctx *ctx)
                         return TLS13_IO_SUCCESS;
                 }
 
+                sending = action->sender == ctx->mode;
+
                 DEBUGF("%s %s %s\n", tls13_handshake_mode_name(ctx->mode),
-                    (action->sender == ctx->mode) ? "sending" : "receiving",
+                    sending ? "sending" : "receiving",
                     tls13_handshake_message_name(action->handshake_type));
 
                 if (ctx->alert != 0)
                         return tls13_send_alert(ctx->rl, ctx->alert);
 
-                if (action->sender == ctx->mode)
+                if (sending)
                         ret = tls13_handshake_send_action(ctx, action);
                 else
                         ret = tls13_handshake_recv_action(ctx, action);
@@ -408,6 +430,10 @@ tls13_handshake_perform(struct tls13_ctx *ctx)
                 if (!tls13_handshake_advance_state_machine(ctx))
                         return TLS13_IO_FAILURE;
 
+                if (sending)
+                        ctx->need_flush = tls13_handshake_end_of_flight(ctx,
+                            action);
+
                 if (!tls13_handshake_set_legacy_state(ctx))
                         return TLS13_IO_FAILURE;
         }
diff --git a/src/lib/libssl/tls13_internal.h b/src/lib/libssl/tls13_internal.h
index cb59634edc..20cb52ebdd 100644
--- a/src/lib/libssl/tls13_internal.h
+++ b/src/lib/libssl/tls13_internal.h
@@ -1,4 +1,4 @@
-/* $OpenBSD: tls13_internal.h,v 1.93 2021/09/14 14:35:09 tb Exp $ */
+/* $OpenBSD: tls13_internal.h,v 1.94 2021/09/16 19:25:30 jsing Exp $ */
 /*
  * Copyright (c) 2018 Bob Beck <beck@openbsd.org>
  * Copyright (c) 2018 Theo Buehler <tb@openbsd.org>
@@ -92,6 +92,7 @@ typedef void (*tls13_phh_sent_cb)(void *_cb_arg);
 typedef ssize_t (*tls13_read_cb)(void *_buf, size_t _buflen, void *_cb_arg);
 typedef ssize_t (*tls13_write_cb)(const void *_buf, size_t _buflen,
     void *_cb_arg);
+typedef ssize_t (*tls13_flush_cb)(void *_cb_arg);
 typedef void (*tls13_handshake_message_cb)(void *_cb_arg);
 typedef void (*tls13_info_cb)(void *_cb_arg, int _state, int _ret);
 typedef int (*tls13_ocsp_status_cb)(void *_cb_arg);
@@ -200,6 +201,7 @@ struct tls13_record_layer;
 struct tls13_record_layer_callbacks {
         tls13_read_cb wire_read;
         tls13_write_cb wire_write;
+        tls13_flush_cb wire_flush;
         tls13_alert_cb alert_recv;
         tls13_alert_cb alert_sent;
         tls13_phh_recv_cb phh_recv;
@@ -226,6 +228,7 @@ int tls13_record_layer_set_write_traffic_key(struct tls13_record_layer *rl,
     struct tls13_secret *write_key);
 ssize_t tls13_record_layer_send_pending(struct tls13_record_layer *rl);
 ssize_t tls13_record_layer_phh(struct tls13_record_layer *rl, CBS *cbs);
+ssize_t tls13_record_layer_flush(struct tls13_record_layer *rl);
 
 ssize_t tls13_read_handshake_data(struct tls13_record_layer *rl, uint8_t *buf, size_t n);
 ssize_t tls13_write_handshake_data(struct tls13_record_layer *rl, const uint8_t *buf,
@@ -283,6 +286,7 @@ struct tls13_ctx {
         struct tls13_handshake_stage handshake_stage;
         int handshake_started;
         int handshake_completed;
+        int need_flush;
         int middlebox_compat;
         int send_dummy_ccs;
         int send_dummy_ccs_after;
@@ -328,6 +332,7 @@ int tls13_legacy_connect(SSL *ssl);
 int tls13_legacy_return_code(SSL *ssl, ssize_t ret);
 ssize_t tls13_legacy_wire_read_cb(void *buf, size_t n, void *arg);
 ssize_t tls13_legacy_wire_write_cb(const void *buf, size_t n, void *arg);
+ssize_t tls13_legacy_wire_flush_cb(void *arg);
 int tls13_legacy_pending(const SSL *ssl);
 int tls13_legacy_read_bytes(SSL *ssl, int type, unsigned char *buf, int len,
     int peek);
diff --git a/src/lib/libssl/tls13_legacy.c b/src/lib/libssl/tls13_legacy.c
index 3368600c60..f668dd4ea3 100644
--- a/src/lib/libssl/tls13_legacy.c
+++ b/src/lib/libssl/tls13_legacy.c
@@ -1,4 +1,4 @@
-/*      $OpenBSD: tls13_legacy.c,v 1.30 2021/09/14 14:31:21 tb Exp $ */
+/*      $OpenBSD: tls13_legacy.c,v 1.31 2021/09/16 19:25:30 jsing Exp $ */
 /*
  * Copyright (c) 2018, 2019 Joel Sing <jsing@openbsd.org>
  *
@@ -96,6 +96,30 @@ tls13_legacy_wire_write_cb(const void *buf, size_t n, void *arg)
         return tls13_legacy_wire_write(ctx->ssl, buf, n);
 }
 
+static ssize_t
+tls13_legacy_wire_flush(SSL *ssl)
+{
+        if (BIO_flush(ssl->wbio) <= 0) {
+                if (BIO_should_write(ssl->wbio))
+                        return TLS13_IO_WANT_POLLOUT;
+
+                if (ERR_peek_error() == 0 && errno != 0)
+                        SYSerror(errno);
+
+                return TLS13_IO_FAILURE;
+        }
+
+        return TLS13_IO_SUCCESS;
+}
+
+ssize_t
+tls13_legacy_wire_flush_cb(void *arg)
+{
+        struct tls13_ctx *ctx = arg;
+
+        return tls13_legacy_wire_flush(ctx->ssl);
+}
+
 static void
 tls13_legacy_error(SSL *ssl)
 {
diff --git a/src/lib/libssl/tls13_lib.c b/src/lib/libssl/tls13_lib.c
index 6615efc4d2..1a9596adca 100644
--- a/src/lib/libssl/tls13_lib.c
+++ b/src/lib/libssl/tls13_lib.c
@@ -1,4 +1,4 @@
-/*      $OpenBSD: tls13_lib.c,v 1.61 2021/09/02 11:58:30 beck Exp $ */
+/*      $OpenBSD: tls13_lib.c,v 1.62 2021/09/16 19:25:30 jsing Exp $ */
 /*
  * Copyright (c) 2018, 2019 Joel Sing <jsing@openbsd.org>
  * Copyright (c) 2019 Bob Beck <beck@openbsd.org>
@@ -374,6 +374,7 @@ tls13_phh_received_cb(void *cb_arg, CBS *cbs)
 static const struct tls13_record_layer_callbacks rl_callbacks = {
         .wire_read = tls13_legacy_wire_read_cb,
         .wire_write = tls13_legacy_wire_write_cb,
+        .wire_flush = tls13_legacy_wire_flush_cb,
         .alert_recv = tls13_alert_received_cb,
         .alert_sent = tls13_alert_sent_cb,
         .phh_recv = tls13_phh_received_cb,
diff --git a/src/lib/libssl/tls13_record_layer.c b/src/lib/libssl/tls13_record_layer.c
index 2e32cb8a37..6b9f5d1419 100644
--- a/src/lib/libssl/tls13_record_layer.c
+++ b/src/lib/libssl/tls13_record_layer.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: tls13_record_layer.c,v 1.63 2021/09/04 16:26:12 jsing Exp $ */
+/* $OpenBSD: tls13_record_layer.c,v 1.64 2021/09/16 19:25:30 jsing Exp $ */
 /*
  * Copyright (c) 2018, 2019 Joel Sing <jsing@openbsd.org>
  *
@@ -1096,6 +1096,12 @@ tls13_record_layer_write(struct tls13_record_layer *rl, uint8_t content_type,
         return ret;
 }
 
+ssize_t
+tls13_record_layer_flush(struct tls13_record_layer *rl)
+{
+        return rl->cb.wire_flush(rl->cb_arg);
+}
+
 static const uint8_t tls13_dummy_ccs[] = { 0x01 };
 
 ssize_t