Tweak return value handling in the TLSv1.3 handshake code.

The I/O paths are from the tls13_handshake_send_action() and
tls13_handshake_recv_action() functions - both of these need to propagate
I/O conditions (EOF, failure, want poll in, want poll out) up the stack,
so we need to capture and return values <= 0. Use an I/O condition to
indicate successful handshake completion.

Also, the various send/recv functions are currently unimplemented, so
return 0 (failure) rather than 1 (success).

ok tb@

2 files changed, 47 insertions, 44 deletions

diff --git a/src/lib/libssl/tls13_handshake.c b/src/lib/libssl/tls13_handshake.c
index 77e59f1930..b566ed2298 100644
--- a/src/lib/libssl/tls13_handshake.c
+++ b/src/lib/libssl/tls13_handshake.c
@@ -1,4 +1,4 @@
-/*      $OpenBSD: tls13_handshake.c,v 1.8 2019/01/18 06:51:29 tb Exp $  */
+/*      $OpenBSD: tls13_handshake.c,v 1.9 2019/01/19 03:32:03 jsing Exp $       */
 /*
  * Copyright (c) 2018-2019 Theo Buehler <tb@openbsd.org>
  * Copyright (c) 2019 Joel Sing <jsing@openbsd.org>
@@ -279,26 +279,27 @@ int
 tls13_connect(struct tls13_ctx *ctx)
 {
         struct tls13_handshake_action *action;
+        int ret;
 
         ctx->mode = TLS13_HS_CLIENT;
 
         for (;;) {
                 if ((action = tls13_handshake_active_action(ctx)) == NULL)
-                        return -1;
+                        return TLS13_IO_FAILURE;
 
                 if (action->sender == TLS13_HS_BOTH)
-                        return 1;
+                        return TLS13_IO_SUCCESS;
 
                 if (action->sender == TLS13_HS_CLIENT) {
-                        if (!tls13_handshake_send_action(ctx, action))
-                                return 0;
+                        if ((ret = tls13_handshake_send_action(ctx, action)) <= 0)
+                                return ret;
                 } else {
-                        if (!tls13_handshake_recv_action(ctx, action))
-                                return 0;
+                        if ((ret = tls13_handshake_recv_action(ctx, action)) <= 0)
+                                return ret;
                 }
 
                 if (!tls13_handshake_advance_state_machine(ctx))
-                        return 0;
+                        return TLS13_IO_FAILURE;
         }
 }
 
@@ -306,26 +307,27 @@ int
 tls13_accept(struct tls13_ctx *ctx)
 {
         struct tls13_handshake_action *action;
+        int ret;
 
         ctx->mode = TLS13_HS_SERVER;
 
         for (;;) {
                 if ((action = tls13_handshake_active_action(ctx)) == NULL)
-                        return -1;
+                        return TLS13_IO_FAILURE;
 
                 if (action->sender == TLS13_HS_BOTH)
-                        return 1;
+                        return TLS13_IO_SUCCESS;
 
                 if (action->sender == TLS13_HS_SERVER) {
-                        if (!tls13_handshake_send_action(ctx, action))
-                                return 0;
+                        if ((ret = tls13_handshake_send_action(ctx, action)) <= 0)
+                                return ret;
                 } else {
-                        if (!tls13_handshake_recv_action(ctx, action))
-                                return 0;
+                        if ((ret = tls13_handshake_recv_action(ctx, action)) <= 0)
+                                return ret;
                 }
 
                 if (!tls13_handshake_advance_state_machine(ctx))
-                        return 0;
+                        return TLS13_IO_FAILURE;
         }
 
         return 1;
@@ -335,7 +337,7 @@ int
 tls13_handshake_advance_state_machine(struct tls13_ctx *ctx)
 {
         ctx->handshake.message_number++;
-        return 1;
+        return 0;
 }
 
 int
@@ -355,86 +357,86 @@ tls13_handshake_recv_action(struct tls13_ctx *ctx,
 int
 tls13_client_hello_send(struct tls13_ctx *ctx)
 {
-        return 1;
+        return 0;
 }
 
 int
 tls13_client_hello_recv(struct tls13_ctx *ctx)
 {
-        return 1;
+        return 0;
 }
 
 int
 tls13_client_hello_retry_send(struct tls13_ctx *ctx)
 {
-        return 1;
+        return 0;
 }
 
 int
 tls13_client_hello_retry_recv(struct tls13_ctx *ctx)
 {
-        return 1;
+        return 0;
 }
 
 
 int
 tls13_client_end_of_early_data_send(struct tls13_ctx *ctx)
 {
-        return 1;
+        return 0;
 }
 
 int
 tls13_client_end_of_early_data_recv(struct tls13_ctx *ctx)
 {
-        return 1;
+        return 0;
 }
 
 int
 tls13_client_certificate_send(struct tls13_ctx *ctx)
 {
-        return 1;
+        return 0;
 }
 
 int
 tls13_client_certificate_recv(struct tls13_ctx *ctx)
 {
-        return 1;
+        return 0;
 }
 
 int
 tls13_client_certificate_verify_send(struct tls13_ctx *ctx)
 {
-        return 1;
+        return 0;
 }
 
 int
 tls13_client_certificate_verify_recv(struct tls13_ctx *ctx)
 {
-        return 1;
+        return 0;
 }
 
 int
 tls13_client_finished_recv(struct tls13_ctx *ctx)
 {
-        return 1;
+        return 0;
 }
 
 int
 tls13_client_finished_send(struct tls13_ctx *ctx)
 {
-        return 1;
+        return 0;
 }
 
 int
 tls13_client_key_update_send(struct tls13_ctx *ctx)
 {
-        return 1;
+        return 0;
 }
 
 int
 tls13_client_key_update_recv(struct tls13_ctx *ctx)
 {
-        return 1;
+        return 0;
 }
 
 int
@@ -442,7 +444,7 @@ tls13_server_hello_recv(struct tls13_ctx *ctx)
 {
         ctx->handshake.hs_type |= NEGOTIATED;
 
-        return 1;
+        return 0;
 }
 
 int
@@ -450,65 +452,65 @@ tls13_server_hello_send(struct tls13_ctx *ctx)
 {
         ctx->handshake.hs_type |= NEGOTIATED;
 
-        return 1;
+        return 0;
 }
 
 int
 tls13_server_encrypted_extensions_recv(struct tls13_ctx *ctx)
 {
-        return 1;
+        return 0;
 }
 
 int
 tls13_server_encrypted_extensions_send(struct tls13_ctx *ctx)
 {
-        return 1;
+        return 0;
 }
 
 int
 tls13_server_certificate_recv(struct tls13_ctx *ctx)
 {
-        return 1;
+        return 0;
 }
 
 int
 tls13_server_certificate_send(struct tls13_ctx *ctx)
 {
-        return 1;
+        return 0;
 }
 
 int
 tls13_server_certificate_request_recv(struct tls13_ctx *ctx)
 {
-        return 1;
+        return 0;
 }
 
 int
 tls13_server_certificate_request_send(struct tls13_ctx *ctx)
 {
-        return 1;
+        return 0;
 }
 
 int
 tls13_server_certificate_verify_send(struct tls13_ctx *ctx)
 {
-        return 1;
+        return 0;
 }
 
 int
 tls13_server_certificate_verify_recv(struct tls13_ctx *ctx)
 {
-        return 1;
+        return 0;
 }
 
 int
 tls13_server_finished_recv(struct tls13_ctx *ctx)
 {
-        return 1;
+        return 0;
 }
 
 int
 tls13_server_finished_send(struct tls13_ctx *ctx)
 {
-        return 1;
+        return 0;
 }
diff --git a/src/lib/libssl/tls13_internal.h b/src/lib/libssl/tls13_internal.h
index e672df37e3..876f339c80 100644
--- a/src/lib/libssl/tls13_internal.h
+++ b/src/lib/libssl/tls13_internal.h
@@ -1,4 +1,4 @@
-/* $OpenBSD: tls13_internal.h,v 1.7 2019/01/18 06:51:29 tb Exp $ */
+/* $OpenBSD: tls13_internal.h,v 1.8 2019/01/19 03:32:03 jsing Exp $ */
 /*
  * Copyright (c) 2018 Bob Beck <beck@openbsd.org>
  * Copyright (c) 2018 Theo Buehler <tb@openbsd.org>
@@ -25,7 +25,8 @@
 
 __BEGIN_HIDDEN_DECLS
 
-#define TLS13_IO_EOF            0
+#define TLS13_IO_SUCCESS         1
+#define TLS13_IO_EOF             0
 #define TLS13_IO_FAILURE        -1
 #define TLS13_IO_WANT_POLLIN    -2
 #define TLS13_IO_WANT_POLLOUT   -3