Replace SSL_PKEY_RSA_ENC/SSL_PKEY_RSA_SIGN with SSL_PKEY_RSA.

Some time prior to SSLeay 0.8.1b, SSL_PKEY_RSA_SIGN got added with the
intention of handling RSA sign only certificates... this incomplete code
had the following comment:

  /* check to see if this is a signing only certificate */
  /* EAY EAY EAY EAY */

And while the comment was removed in 2005, the incomplete RSA sign-only
handling has remained ever since.

Remove SSL_PKEY_RSA_SIGN and rename SSL_PKEY_RSA_ENC to SSL_PKEY_RSA. While
here also remove the unused SSL_PKEY_DH_RSA.

ok tb@

1 files changed, 8 insertions, 16 deletions

diff --git a/src/lib/libssl/ssl_lib.c b/src/lib/libssl/ssl_lib.c
index 1c4ab636a1..fa1d209c8c 100644
--- a/src/lib/libssl/ssl_lib.c
+++ b/src/lib/libssl/ssl_lib.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: ssl_lib.c,v 1.213 2020/05/10 14:17:47 jsing Exp $ */
+/* $OpenBSD: ssl_lib.c,v 1.214 2020/05/19 16:35:20 jsing Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -1968,7 +1968,7 @@ SSL_CTX_set_verify_depth(SSL_CTX *ctx, int depth)
 void
 ssl_set_cert_masks(CERT *c, const SSL_CIPHER *cipher)
 {
-        int              rsa_enc, rsa_sign, dh_tmp;
+        int              rsa, dh_tmp;
         int              have_ecc_cert;
         unsigned long    mask_k, mask_a;
         X509            *x = NULL;
@@ -1980,10 +1980,8 @@ ssl_set_cert_masks(CERT *c, const SSL_CIPHER *cipher)
         dh_tmp = (c->dh_tmp != NULL || c->dh_tmp_cb != NULL ||
             c->dh_tmp_auto != 0);
 
-        cpk = &(c->pkeys[SSL_PKEY_RSA_ENC]);
-        rsa_enc = (cpk->x509 != NULL && cpk->privatekey != NULL);
-        cpk = &(c->pkeys[SSL_PKEY_RSA_SIGN]);
-        rsa_sign = (cpk->x509 != NULL && cpk->privatekey != NULL);
+        cpk = &(c->pkeys[SSL_PKEY_RSA]);
+        rsa = (cpk->x509 != NULL && cpk->privatekey != NULL);
         cpk = &(c->pkeys[SSL_PKEY_ECC]);
         have_ecc_cert = (cpk->x509 != NULL && cpk->privatekey != NULL);
 
@@ -1996,13 +1994,13 @@ ssl_set_cert_masks(CERT *c, const SSL_CIPHER *cipher)
                 mask_a |= SSL_aGOST01;
         }
 
-        if (rsa_enc)
+        if (rsa)
                 mask_k |= SSL_kRSA;
 
         if (dh_tmp)
                 mask_k |= SSL_kDHE;
 
-        if (rsa_enc || rsa_sign)
+        if (rsa)
                 mask_a |= SSL_aRSA;
 
         mask_a |= SSL_aNULL;
@@ -2085,10 +2083,7 @@ ssl_get_server_send_pkey(const SSL *s)
         if (alg_a & SSL_aECDSA) {
                 i = SSL_PKEY_ECC;
         } else if (alg_a & SSL_aRSA) {
-                if (c->pkeys[SSL_PKEY_RSA_ENC].x509 == NULL)
-                        i = SSL_PKEY_RSA_SIGN;
-                else
-                        i = SSL_PKEY_RSA_ENC;
+                i = SSL_PKEY_RSA;
         } else if (alg_a & SSL_aGOST01) {
                 i = SSL_PKEY_GOST01;
         } else { /* if (alg_a & SSL_aNULL) */
@@ -2113,10 +2108,7 @@ ssl_get_sign_pkey(SSL *s, const SSL_CIPHER *cipher, const EVP_MD **pmd,
         c = s->cert;
 
         if (alg_a & SSL_aRSA) {
-                if (c->pkeys[SSL_PKEY_RSA_SIGN].privatekey != NULL)
-                        idx = SSL_PKEY_RSA_SIGN;
-                else if (c->pkeys[SSL_PKEY_RSA_ENC].privatekey != NULL)
-                        idx = SSL_PKEY_RSA_ENC;
+                idx = SSL_PKEY_RSA;
         } else if ((alg_a & SSL_aECDSA) &&
             (c->pkeys[SSL_PKEY_ECC].privatekey != NULL))
                 idx = SSL_PKEY_ECC;