Provide functions to determine if TLSv1.2 record protection is engaged.

Call these functions from code that needs to know if we've changed cipher
state and enabled record protection, rather than inconsistently checking
various pointers from other places in the code base. This also fixes a
minor bug where the wrong pointers are checked if we're operating with
AEAD.

ok inoguchi@ tb@

5 files changed, 37 insertions, 18 deletions

diff --git a/src/lib/libssl/d1_pkt.c b/src/lib/libssl/d1_pkt.c
index 4f15015145..14ff8221be 100644
--- a/src/lib/libssl/d1_pkt.c
+++ b/src/lib/libssl/d1_pkt.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: d1_pkt.c,v 1.88 2021/01/13 18:38:34 jsing Exp $ */
+/* $OpenBSD: d1_pkt.c,v 1.89 2021/01/19 18:57:09 jsing Exp $ */
 /*
  * DTLS implementation written by Nagendra Modadugu
  * (nagendra@cs.stanford.edu) for the OpenSSL project 2005.
@@ -642,13 +642,12 @@ dtls1_read_bytes(SSL *s, int type, unsigned char *buf, int len, int peek)
                 return (0);
         }
 
-
-        if (type == rr->type) /* SSL3_RT_APPLICATION_DATA or SSL3_RT_HANDSHAKE */
-        {
+        /* SSL3_RT_APPLICATION_DATA or SSL3_RT_HANDSHAKE */
+        if (type == rr->type) {
                 /* make sure that we are not getting application data when we
                  * are doing a handshake for the first time */
-                if (SSL_in_init(s) && (type == SSL3_RT_APPLICATION_DATA) &&
-                        (s->enc_read_ctx == NULL)) {
+                if (SSL_in_init(s) && type == SSL3_RT_APPLICATION_DATA &&
+                    !tls12_record_layer_read_protected(s->internal->rl)) {
                         al = SSL_AD_UNEXPECTED_MESSAGE;
                         SSLerror(s, SSL_R_APP_DATA_IN_HANDSHAKE);
                         goto f_err;
diff --git a/src/lib/libssl/ssl_locl.h b/src/lib/libssl/ssl_locl.h
index e09f668121..e0a4c49ccb 100644
--- a/src/lib/libssl/ssl_locl.h
+++ b/src/lib/libssl/ssl_locl.h
@@ -1,4 +1,4 @@
-/* $OpenBSD: ssl_locl.h,v 1.313 2021/01/19 18:51:08 jsing Exp $ */
+/* $OpenBSD: ssl_locl.h,v 1.314 2021/01/19 18:57:09 jsing Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -479,6 +479,8 @@ void tls12_record_layer_alert(struct tls12_record_layer *rl,
     uint8_t *alert_desc);
 int tls12_record_layer_write_overhead(struct tls12_record_layer *rl,
     size_t *overhead);
+int tls12_record_layer_read_protected(struct tls12_record_layer *rl);
+int tls12_record_layer_write_protected(struct tls12_record_layer *rl);
 void tls12_record_layer_set_version(struct tls12_record_layer *rl,
     uint16_t version);
 void tls12_record_layer_set_write_epoch(struct tls12_record_layer *rl,
diff --git a/src/lib/libssl/ssl_pkt.c b/src/lib/libssl/ssl_pkt.c
index 4cc1914ecd..31a66753bf 100644
--- a/src/lib/libssl/ssl_pkt.c
+++ b/src/lib/libssl/ssl_pkt.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: ssl_pkt.c,v 1.33 2020/10/14 16:57:33 jsing Exp $ */
+/* $OpenBSD: ssl_pkt.c,v 1.34 2021/01/19 18:57:09 jsing Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -370,11 +370,12 @@ ssl3_get_record(SSL *s)
 
                 /* Lets check version */
                 if (!s->internal->first_packet && ssl_version != s->version) {
-                        SSLerror(s, SSL_R_WRONG_VERSION_NUMBER);
                         if ((s->version & 0xFF00) == (ssl_version & 0xFF00) &&
-                            !s->internal->enc_write_ctx && !s->internal->write_hash)
+                            !tls12_record_layer_write_protected(s->internal->rl)) {
                                 /* Send back error using their minor version number :-) */
                                 s->version = ssl_version;
+                        }
+                        SSLerror(s, SSL_R_WRONG_VERSION_NUMBER);
                         al = SSL_AD_PROTOCOL_VERSION;
                         goto f_err;
                 }
@@ -569,8 +570,7 @@ do_ssl3_write(SSL *s, int type, const unsigned char *buf, unsigned int len)
          * (see http://www.openssl.org/~bodo/tls-cbc.txt). Note that this
          * is unnecessary for AEAD.
          */
-        if (sess != NULL && s->internal->enc_write_ctx != NULL &&
-            EVP_MD_CTX_md(s->internal->write_hash) != NULL) {
+        if (sess != NULL && tls12_record_layer_write_protected(s->internal->rl)) {
                 if (S3I(s)->need_empty_fragments &&
                     !S3I(s)->empty_fragment_done &&
                     type == SSL3_RT_APPLICATION_DATA)
@@ -814,8 +814,8 @@ start:
         if (type == rr->type) {
                 /* make sure that we are not getting application data when we
                  * are doing a handshake for the first time */
-                if (SSL_in_init(s) && (type == SSL3_RT_APPLICATION_DATA) &&
-                        (s->enc_read_ctx == NULL)) {
+                if (SSL_in_init(s) && type == SSL3_RT_APPLICATION_DATA &&
+                    !tls12_record_layer_read_protected(s->internal->rl)) {
                         al = SSL_AD_UNEXPECTED_MESSAGE;
                         SSLerror(s, SSL_R_APP_DATA_IN_HANDSHAKE);
                         goto f_err;
diff --git a/src/lib/libssl/ssl_srvr.c b/src/lib/libssl/ssl_srvr.c
index ac3669550c..000cac6785 100644
--- a/src/lib/libssl/ssl_srvr.c
+++ b/src/lib/libssl/ssl_srvr.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: ssl_srvr.c,v 1.88 2020/10/14 16:57:33 jsing Exp $ */
+/* $OpenBSD: ssl_srvr.c,v 1.89 2021/01/19 18:57:09 jsing Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -853,15 +853,15 @@ ssl3_get_client_hello(SSL *s)
         if (!ssl_downgrade_max_version(s, &max_version))
                 goto err;
         if (ssl_max_shared_version(s, client_version, &shared_version) != 1) {
-                SSLerror(s, SSL_R_WRONG_VERSION_NUMBER);
                 if ((s->client_version >> 8) == SSL3_VERSION_MAJOR &&
-                    !s->internal->enc_write_ctx && !s->internal->write_hash) {
+                    !tls12_record_layer_write_protected(s->internal->rl)) {
                         /*
                          * Similar to ssl3_get_record, send alert using remote
                          * version number.
                          */
                         s->version = s->client_version;
                 }
+                SSLerror(s, SSL_R_WRONG_VERSION_NUMBER);
                 al = SSL_AD_PROTOCOL_VERSION;
                 goto f_err;
         }
diff --git a/src/lib/libssl/tls12_record_layer.c b/src/lib/libssl/tls12_record_layer.c
index 7fa31707d3..affc5375a2 100644
--- a/src/lib/libssl/tls12_record_layer.c
+++ b/src/lib/libssl/tls12_record_layer.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: tls12_record_layer.c,v 1.11 2021/01/19 18:51:08 jsing Exp $ */
+/* $OpenBSD: tls12_record_layer.c,v 1.12 2021/01/19 18:57:09 jsing Exp $ */
 /*
  * Copyright (c) 2020 Joel Sing <jsing@openbsd.org>
  *
@@ -59,6 +59,12 @@ tls12_record_protection_free(struct tls12_record_protection *rp)
 }
 
 static int
+tls12_record_protection_engaged(struct tls12_record_protection *rp)
+{
+        return rp->aead_ctx != NULL || rp->cipher_ctx != NULL;
+}
+
+static int
 tls12_record_protection_eiv_len(struct tls12_record_protection *rp,
     size_t *out_eiv_len)
 {
@@ -195,6 +201,18 @@ tls12_record_layer_write_overhead(struct tls12_record_layer *rl,
         return 1;
 }
 
+int
+tls12_record_layer_read_protected(struct tls12_record_layer *rl)
+{
+        return tls12_record_protection_engaged(rl->read);
+}
+
+int
+tls12_record_layer_write_protected(struct tls12_record_layer *rl)
+{
+        return tls12_record_protection_engaged(rl->write);
+}
+
 void
 tls12_record_layer_set_version(struct tls12_record_layer *rl, uint16_t version)
 {