Move the RSA-PSS check for TLSv1.3 to ssl_sigalg_pkey_ok().

Also, rather than passing in a check_curve flag, pass in the SSL * and
handle version checks internally to ssl_sigalg_pkey_ok(), simplifying
the callers.

ok inoguchi@ tb@

6 files changed, 29 insertions, 34 deletions

diff --git a/src/lib/libssl/ssl_clnt.c b/src/lib/libssl/ssl_clnt.c
index 261bf426cc..25a3321324 100644
--- a/src/lib/libssl/ssl_clnt.c
+++ b/src/lib/libssl/ssl_clnt.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: ssl_clnt.c,v 1.102 2021/06/27 19:16:59 jsing Exp $ */
+/* $OpenBSD: ssl_clnt.c,v 1.103 2021/06/29 19:10:08 jsing Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -1562,7 +1562,7 @@ ssl3_get_server_key_exchange(SSL *s)
                                 al = SSL_AD_DECODE_ERROR;
                                 goto fatal_err;
                         }
-                        if (!ssl_sigalg_pkey_ok(sigalg, pkey, 0)) {
+                        if (!ssl_sigalg_pkey_ok(s, sigalg, pkey)) {
                                 SSLerror(s, SSL_R_WRONG_SIGNATURE_TYPE);
                                 al = SSL_AD_DECODE_ERROR;
                                 goto fatal_err;
diff --git a/src/lib/libssl/ssl_sigalgs.c b/src/lib/libssl/ssl_sigalgs.c
index 456332e7cf..bd896c829b 100644
--- a/src/lib/libssl/ssl_sigalgs.c
+++ b/src/lib/libssl/ssl_sigalgs.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: ssl_sigalgs.c,v 1.31 2021/06/29 18:59:25 jsing Exp $ */
+/* $OpenBSD: ssl_sigalgs.c,v 1.32 2021/06/29 19:10:08 jsing Exp $ */
 /*
  * Copyright (c) 2018-2020 Bob Beck <beck@openbsd.org>
  *
@@ -260,32 +260,37 @@ ssl_sigalg_for_legacy(SSL *s, EVP_PKEY *pkey)
 }
 
 int
-ssl_sigalg_pkey_ok(const struct ssl_sigalg *sigalg, EVP_PKEY *pkey,
-    int check_curve)
+ssl_sigalg_pkey_ok(SSL *s, const struct ssl_sigalg *sigalg, EVP_PKEY *pkey)
 {
         if (sigalg == NULL || pkey == NULL)
                 return 0;
         if (sigalg->key_type != pkey->type)
                 return 0;
 
+        /*
+         * RSA PSS must have an RSA key that needs to be at
+         * least as big as twice the size of the hash + 2
+         */
         if ((sigalg->flags & SIGALG_FLAG_RSA_PSS)) {
-                /*
-                 * RSA PSS Must have an RSA key that needs to be at
-                 * least as big as twice the size of the hash + 2
-                 */
                 if (pkey->type != EVP_PKEY_RSA ||
                     EVP_PKEY_size(pkey) < (2 * EVP_MD_size(sigalg->md()) + 2))
                         return 0;
         }
 
-        if (pkey->type == EVP_PKEY_EC && check_curve) {
-                /* Curve must match for EC keys. */
+        /* RSA cannot be used without PSS in TLSv1.3. */
+        if (S3I(s)->hs.negotiated_tls_version >= TLS1_3_VERSION &&
+            sigalg->key_type == EVP_PKEY_RSA &&
+            (sigalg->flags & SIGALG_FLAG_RSA_PSS) == 0)
+                return 0;
+
+        /* Ensure that curve matches for EC keys. */
+        if (S3I(s)->hs.negotiated_tls_version >= TLS1_3_VERSION &&
+            pkey->type == EVP_PKEY_EC) {
                 if (sigalg->curve_nid == 0)
                         return 0;
-                if (EC_GROUP_get_curve_name(EC_KEY_get0_group
-                    (EVP_PKEY_get0_EC_KEY(pkey))) != sigalg->curve_nid) {
+                if (EC_GROUP_get_curve_name(EC_KEY_get0_group(
+                    EVP_PKEY_get0_EC_KEY(pkey))) != sigalg->curve_nid)
                         return 0;
-                }
         }
 
         return 1;
@@ -294,12 +299,8 @@ ssl_sigalg_pkey_ok(const struct ssl_sigalg *sigalg, EVP_PKEY *pkey,
 const struct ssl_sigalg *
 ssl_sigalg_select(SSL *s, EVP_PKEY *pkey)
 {
-        int check_curve = 0;
         CBS cbs;
 
-        if (S3I(s)->hs.negotiated_tls_version >= TLS1_3_VERSION)
-                check_curve = 1;
-
         if (!SSL_USE_SIGALGS(s))
                 return ssl_sigalg_for_legacy(s, pkey);
 
@@ -326,13 +327,7 @@ ssl_sigalg_select(SSL *s, EVP_PKEY *pkey)
                     S3I(s)->hs.negotiated_tls_version, sigalg_value)) == NULL)
                         continue;
 
-                /* RSA cannot be used without PSS in TLSv1.3. */
-                if (S3I(s)->hs.negotiated_tls_version >= TLS1_3_VERSION &&
-                    sigalg->key_type == EVP_PKEY_RSA &&
-                    (sigalg->flags & SIGALG_FLAG_RSA_PSS) == 0)
-                        continue;
-
-                if (ssl_sigalg_pkey_ok(sigalg, pkey, check_curve))
+                if (ssl_sigalg_pkey_ok(s, sigalg, pkey))
                         return sigalg;
         }
 
diff --git a/src/lib/libssl/ssl_sigalgs.h b/src/lib/libssl/ssl_sigalgs.h
index c91e66a5a9..6905bba060 100644
--- a/src/lib/libssl/ssl_sigalgs.h
+++ b/src/lib/libssl/ssl_sigalgs.h
@@ -1,4 +1,4 @@
-/* $OpenBSD: ssl_sigalgs.h,v 1.20 2021/06/27 18:15:35 jsing Exp $ */
+/* $OpenBSD: ssl_sigalgs.h,v 1.21 2021/06/29 19:10:08 jsing Exp $ */
 /*
  * Copyright (c) 2018-2019 Bob Beck <beck@openbsd.org>
  *
@@ -72,8 +72,8 @@ const struct ssl_sigalg *ssl_sigalg_lookup(uint16_t sigalg);
 const struct ssl_sigalg *ssl_sigalg_from_value(uint16_t tls_version,
     uint16_t value);
 int ssl_sigalgs_build(uint16_t tls_version, CBB *cbb);
-int ssl_sigalg_pkey_ok(const struct ssl_sigalg *sigalg, EVP_PKEY *pkey,
-    int check_curve);
+int ssl_sigalg_pkey_ok(SSL *s, const struct ssl_sigalg *sigalg,
+    EVP_PKEY *pkey);
 const struct ssl_sigalg *ssl_sigalg_select(SSL *s, EVP_PKEY *pkey);
 
 __END_HIDDEN_DECLS
diff --git a/src/lib/libssl/ssl_srvr.c b/src/lib/libssl/ssl_srvr.c
index 259c6679f2..04e81a5d76 100644
--- a/src/lib/libssl/ssl_srvr.c
+++ b/src/lib/libssl/ssl_srvr.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: ssl_srvr.c,v 1.114 2021/06/27 18:15:35 jsing Exp $ */
+/* $OpenBSD: ssl_srvr.c,v 1.115 2021/06/29 19:10:08 jsing Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -2199,7 +2199,7 @@ ssl3_get_cert_verify(SSL *s)
                         al = SSL_AD_DECODE_ERROR;
                         goto fatal_err;
                 }
-                if (!ssl_sigalg_pkey_ok(sigalg, pkey, 0)) {
+                if (!ssl_sigalg_pkey_ok(s, sigalg, pkey)) {
                         SSLerror(s, SSL_R_WRONG_SIGNATURE_TYPE);
                         al = SSL_AD_DECODE_ERROR;
                         goto fatal_err;
diff --git a/src/lib/libssl/tls13_client.c b/src/lib/libssl/tls13_client.c
index 0a237567fd..dd9a5b1606 100644
--- a/src/lib/libssl/tls13_client.c
+++ b/src/lib/libssl/tls13_client.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: tls13_client.c,v 1.84 2021/06/29 18:47:15 jsing Exp $ */
+/* $OpenBSD: tls13_client.c,v 1.85 2021/06/29 19:10:08 jsing Exp $ */
 /*
  * Copyright (c) 2018, 2019 Joel Sing <jsing@openbsd.org>
  *
@@ -704,7 +704,7 @@ tls13_server_certificate_verify_recv(struct tls13_ctx *ctx, CBS *cbs)
                 goto err;
         if ((pkey = X509_get0_pubkey(cert)) == NULL)
                 goto err;
-        if (!ssl_sigalg_pkey_ok(sigalg, pkey, 1))
+        if (!ssl_sigalg_pkey_ok(ctx->ssl, sigalg, pkey))
                 goto err;
         ctx->hs->peer_sigalg = sigalg;
 
diff --git a/src/lib/libssl/tls13_server.c b/src/lib/libssl/tls13_server.c
index 18cb056755..c3d4ca9bd8 100644
--- a/src/lib/libssl/tls13_server.c
+++ b/src/lib/libssl/tls13_server.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: tls13_server.c,v 1.81 2021/06/27 19:23:51 jsing Exp $ */
+/* $OpenBSD: tls13_server.c,v 1.82 2021/06/29 19:10:08 jsing Exp $ */
 /*
  * Copyright (c) 2019, 2020 Joel Sing <jsing@openbsd.org>
  * Copyright (c) 2020 Bob Beck <beck@openbsd.org>
@@ -994,7 +994,7 @@ tls13_client_certificate_verify_recv(struct tls13_ctx *ctx, CBS *cbs)
                 goto err;
         if ((pkey = X509_get0_pubkey(cert)) == NULL)
                 goto err;
-        if (!ssl_sigalg_pkey_ok(sigalg, pkey, 1))
+        if (!ssl_sigalg_pkey_ok(ctx->ssl, sigalg, pkey))
                 goto err;
         ctx->hs->peer_sigalg = sigalg;