Add check function to verify that pkey is usable with a sigalg.

Include check for appropriate RSA key size when used with PSS.
ok tb@

4 files changed, 24 insertions, 7 deletions

diff --git a/src/lib/libssl/ssl_clnt.c b/src/lib/libssl/ssl_clnt.c
index 9f8d999ff1..2094417994 100644
--- a/src/lib/libssl/ssl_clnt.c
+++ b/src/lib/libssl/ssl_clnt.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: ssl_clnt.c,v 1.43 2018/11/11 02:22:34 beck Exp $ */
+/* $OpenBSD: ssl_clnt.c,v 1.44 2018/11/11 21:54:47 beck Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -1527,7 +1527,7 @@ ssl3_get_server_key_exchange(SSL *s)
                                 al = SSL_AD_DECODE_ERROR;
                                 goto f_err;
                         }
-                        if (sigalg->key_type != pkey->type) {
+                        if (!ssl_sigalg_pkey_ok(sigalg, pkey)) {
                                 SSLerror(s, SSL_R_WRONG_SIGNATURE_TYPE);
                                 al = SSL_AD_DECODE_ERROR;
                                 goto f_err;
diff --git a/src/lib/libssl/ssl_sigalgs.c b/src/lib/libssl/ssl_sigalgs.c
index 5dc261810b..a6c5a4e9d8 100644
--- a/src/lib/libssl/ssl_sigalgs.c
+++ b/src/lib/libssl/ssl_sigalgs.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: ssl_sigalgs.c,v 1.6 2018/11/11 02:03:23 beck Exp $ */
+/* $OpenBSD: ssl_sigalgs.c,v 1.7 2018/11/11 21:54:47 beck Exp $ */
 /*
  * Copyright (c) 2018, Bob Beck <beck@openbsd.org>
  *
@@ -225,3 +225,19 @@ ssl_sigalgs_build(CBB *cbb, uint16_t *values, size_t len)
         }
         return 1;
 }
+
+int
+ssl_sigalg_pkey_ok(const struct ssl_sigalg *sigalg, EVP_PKEY *pkey)
+{
+        if (sigalg->key_type == pkey->type) {
+                if (!(sigalg->flags & SIGALG_FLAG_RSA_PSS))
+                        return 1;
+                /*
+                 * RSA keys for PSS need to be at least
+                 * as big as twice the size of the hash + 2
+                 */
+                if (EVP_PKEY_size(pkey) < (2 * EVP_MD_size(sigalg->md()) + 2))
+                        return 1;
+        }
+        return 0;
+}
diff --git a/src/lib/libssl/ssl_sigalgs.h b/src/lib/libssl/ssl_sigalgs.h
index f6fb5e2ad6..1bce6e8ee3 100644
--- a/src/lib/libssl/ssl_sigalgs.h
+++ b/src/lib/libssl/ssl_sigalgs.h
@@ -1,4 +1,4 @@
-/* $OpenBSD: ssl_sigalgs.h,v 1.6 2018/11/10 08:42:39 beck Exp $ */
+/* $OpenBSD: ssl_sigalgs.h,v 1.7 2018/11/11 21:54:47 beck Exp $ */
 /*
  * Copyright (c) 2018, Bob Beck <beck@openbsd.org>
  *
@@ -73,6 +73,7 @@ const struct ssl_sigalg *ssl_sigalg_lookup(uint16_t sigalg);
 const struct ssl_sigalg *ssl_sigalg(uint16_t sigalg, uint16_t *values, size_t len);
 int ssl_sigalgs_build(CBB *cbb, uint16_t *values, size_t len);
 int ssl_sigalg_pkey_check(uint16_t sigalg, EVP_PKEY *pk);
+int ssl_sigalg_pkey_ok(const struct ssl_sigalg *sigalg, EVP_PKEY *pkey);
 
 __END_HIDDEN_DECLS
 
diff --git a/src/lib/libssl/ssl_srvr.c b/src/lib/libssl/ssl_srvr.c
index e09817e2d0..27024be856 100644
--- a/src/lib/libssl/ssl_srvr.c
+++ b/src/lib/libssl/ssl_srvr.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: ssl_srvr.c,v 1.59 2018/11/11 07:57:44 bcook Exp $ */
+/* $OpenBSD: ssl_srvr.c,v 1.60 2018/11/11 21:54:47 beck Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -2179,7 +2179,7 @@ ssl3_get_cert_verify(SSL *s)
                                 al = SSL_AD_DECODE_ERROR;
                                 goto f_err;
                         }
-                        if (sigalg->key_type != pkey->type) {
+                        if (!ssl_sigalg_pkey_ok(sigalg, pkey)) {
                                 SSLerror(s, SSL_R_WRONG_SIGNATURE_TYPE);
                                 al = SSL_AD_DECODE_ERROR;
                                 goto f_err;
@@ -2216,7 +2216,7 @@ ssl3_get_cert_verify(SSL *s)
                     (pctx, RSA_PKCS1_PSS_PADDING) ||
                     !EVP_PKEY_CTX_set_rsa_pss_saltlen(pctx, -1))) {
                         al = SSL_AD_INTERNAL_ERROR;
-                        goto err;
+                        goto f_err;
                 }
                 if (!EVP_DigestVerifyUpdate(&mctx, hdata, hdatalen)) {
                         SSLerror(s, ERR_R_EVP_LIB);