Add support for RSA PSS algorithims being used in sigalgs.

lightly tested, but will need sanity checks and regress test changes
before being added to any sigalgs list for real
ok jsing@ tb@

2 files changed, 29 insertions, 2 deletions

diff --git a/src/lib/libssl/ssl_clnt.c b/src/lib/libssl/ssl_clnt.c
index 298e4b7ff8..9f8d999ff1 100644
--- a/src/lib/libssl/ssl_clnt.c
+++ b/src/lib/libssl/ssl_clnt.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: ssl_clnt.c,v 1.42 2018/11/11 02:03:23 beck Exp $ */
+/* $OpenBSD: ssl_clnt.c,v 1.43 2018/11/11 02:22:34 beck Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -1556,6 +1556,11 @@ ssl3_get_server_key_exchange(SSL *s)
                 if (!EVP_DigestVerifyUpdate(&md_ctx, s->s3->client_random,
                     SSL3_RANDOM_SIZE))
                         goto err;
+                if ((sigalg->flags & SIGALG_FLAG_RSA_PSS) &&
+                    (!EVP_PKEY_CTX_set_rsa_padding(pctx,
+                    RSA_PKCS1_PSS_PADDING) ||
+                    !EVP_PKEY_CTX_set_rsa_pss_saltlen(pctx, -1)))
+                        goto err;
                 if (!EVP_DigestVerifyUpdate(&md_ctx, s->s3->server_random,
                     SSL3_RANDOM_SIZE))
                         goto err;
@@ -2427,6 +2432,14 @@ ssl3_send_client_verify(SSL *s)
                                 SSLerror(s, ERR_R_EVP_LIB);
                                 goto err;
                         }
+                        if ((s->cert->key->sigalg->flags &
+                            SIGALG_FLAG_RSA_PSS) &&
+                            (!EVP_PKEY_CTX_set_rsa_padding(pctx,
+                            RSA_PKCS1_PSS_PADDING) ||
+                            !EVP_PKEY_CTX_set_rsa_pss_saltlen(pctx, -1))) {
+                                SSLerror(s, ERR_R_EVP_LIB);
+                                goto err;
+                        }
                         if (!EVP_DigestSignUpdate(&mctx, hdata, hdatalen)) {
                                 SSLerror(s, ERR_R_EVP_LIB);
                                 goto err;
diff --git a/src/lib/libssl/ssl_srvr.c b/src/lib/libssl/ssl_srvr.c
index f1b8a49468..03ae29a278 100644
--- a/src/lib/libssl/ssl_srvr.c
+++ b/src/lib/libssl/ssl_srvr.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: ssl_srvr.c,v 1.56 2018/11/11 02:03:23 beck Exp $ */
+/* $OpenBSD: ssl_srvr.c,v 1.57 2018/11/11 02:22:34 beck Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -1549,6 +1549,13 @@ ssl3_send_server_key_exchange(SSL *s)
                                 SSLerror(s, ERR_R_EVP_LIB);
                                 goto err;
                         }
+                        if ((sigalg->flags & SIGALG_FLAG_RSA_PSS) &&
+                            (!EVP_PKEY_CTX_set_rsa_padding(pctx,
+                            RSA_PKCS1_PSS_PADDING) ||
+                            !EVP_PKEY_CTX_set_rsa_pss_saltlen(pctx, -1))) {
+                                SSLerror(s, ERR_R_EVP_LIB);
+                                goto err;
+                        }
                         if (!EVP_DigestSignUpdate(&md_ctx, s->s3->client_random,
                             SSL3_RANDOM_SIZE)) {
                                 SSLerror(s, ERR_R_EVP_LIB);
@@ -2203,6 +2210,13 @@ ssl3_get_cert_verify(SSL *s)
                         al = SSL_AD_INTERNAL_ERROR;
                         goto f_err;
                 }
+                if ((sigalg->flags & SIGALG_FLAG_RSA_PSS) &&
+                    (!EVP_PKEY_CTX_set_rsa_padding
+                    (pctx, RSA_PKCS1_PSS_PADDING) ||
+                    !EVP_PKEY_CTX_set_rsa_pss_saltlen(pctx, -1))) {
+                        al = SSL_AD_INTERNAL_ERROR;
+                        goto err;
+                }
                 if (!EVP_DigestVerifyUpdate(&mctx, hdata, hdatalen)) {
                         SSLerror(s, ERR_R_EVP_LIB);
                         al = SSL_AD_INTERNAL_ERROR;