Replace SSL_PKEY_RSA_ENC/SSL_PKEY_RSA_SIGN with SSL_PKEY_RSA.

Some time prior to SSLeay 0.8.1b, SSL_PKEY_RSA_SIGN got added with the
intention of handling RSA sign only certificates... this incomplete code
had the following comment:

  /* check to see if this is a signing only certificate */
  /* EAY EAY EAY EAY */

And while the comment was removed in 2005, the incomplete RSA sign-only
handling has remained ever since.

Remove SSL_PKEY_RSA_SIGN and rename SSL_PKEY_RSA_ENC to SSL_PKEY_RSA. While
here also remove the unused SSL_PKEY_DH_RSA.

ok tb@

8 files changed, 31 insertions, 46 deletions

diff --git a/src/lib/libssl/ssl_both.c b/src/lib/libssl/ssl_both.c
index b8929d8f84..488a5ff7c9 100644
--- a/src/lib/libssl/ssl_both.c
+++ b/src/lib/libssl/ssl_both.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: ssl_both.c,v 1.17 2020/03/12 17:15:33 jsing Exp $ */
+/* $OpenBSD: ssl_both.c,v 1.18 2020/05/19 16:35:20 jsing Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -591,7 +591,7 @@ ssl_cert_type(X509 *x, EVP_PKEY *pkey)
 
         i = pk->type;
         if (i == EVP_PKEY_RSA) {
-                ret = SSL_PKEY_RSA_ENC;
+                ret = SSL_PKEY_RSA;
         } else if (i == EVP_PKEY_EC) {
                 ret = SSL_PKEY_ECC;
         } else if (i == NID_id_GostR3410_2001 ||
diff --git a/src/lib/libssl/ssl_cert.c b/src/lib/libssl/ssl_cert.c
index af8ef329b4..3567b7b426 100644
--- a/src/lib/libssl/ssl_cert.c
+++ b/src/lib/libssl/ssl_cert.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: ssl_cert.c,v 1.76 2019/05/15 09:13:16 bcook Exp $ */
+/* $OpenBSD: ssl_cert.c,v 1.77 2020/05/19 16:35:20 jsing Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -168,7 +168,7 @@ ssl_cert_new(void)
                 SSLerrorx(ERR_R_MALLOC_FAILURE);
                 return (NULL);
         }
-        ret->key = &(ret->pkeys[SSL_PKEY_RSA_ENC]);
+        ret->key = &(ret->pkeys[SSL_PKEY_RSA]);
         ret->references = 1;
         return (ret);
 }
@@ -240,15 +240,10 @@ ssl_cert_dup(CERT *cert)
                                  * (Nothing at the moment, I think.)
                                  */
 
-                        case SSL_PKEY_RSA_ENC:
-                        case SSL_PKEY_RSA_SIGN:
+                        case SSL_PKEY_RSA:
                                 /* We have an RSA key. */
                                 break;
 
-                        case SSL_PKEY_DH_RSA:
-                                /* We have a DH key. */
-                                break;
-
                         case SSL_PKEY_ECC:
                                 /* We have an ECC key */
                                 break;
@@ -377,7 +372,7 @@ ssl_sess_cert_new(void)
                 SSLerrorx(ERR_R_MALLOC_FAILURE);
                 return NULL;
         }
-        ret->peer_key = &(ret->peer_pkeys[SSL_PKEY_RSA_ENC]);
+        ret->peer_key = &(ret->peer_pkeys[SSL_PKEY_RSA]);
         ret->references = 1;
 
         return ret;
diff --git a/src/lib/libssl/ssl_clnt.c b/src/lib/libssl/ssl_clnt.c
index 6b457569a3..fb29e4f5f6 100644
--- a/src/lib/libssl/ssl_clnt.c
+++ b/src/lib/libssl/ssl_clnt.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: ssl_clnt.c,v 1.66 2020/05/10 14:17:47 jsing Exp $ */
+/* $OpenBSD: ssl_clnt.c,v 1.67 2020/05/19 16:35:20 jsing Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -1264,7 +1264,7 @@ ssl3_get_server_kex_dhe(SSL *s, EVP_PKEY **pkey, CBS *cbs)
         }
 
         if (alg_a & SSL_aRSA)
-                *pkey = X509_get_pubkey(sc->peer_pkeys[SSL_PKEY_RSA_ENC].x509);
+                *pkey = X509_get_pubkey(sc->peer_pkeys[SSL_PKEY_RSA].x509);
         else
                 /* XXX - Anonymous DH, so no certificate or pkey. */
                 *pkey = NULL;
@@ -1397,7 +1397,7 @@ ssl3_get_server_kex_ecdhe(SSL *s, EVP_PKEY **pkey, CBS *cbs)
          * and ECDSA.
          */
         if (alg_a & SSL_aRSA)
-                *pkey = X509_get_pubkey(sc->peer_pkeys[SSL_PKEY_RSA_ENC].x509);
+                *pkey = X509_get_pubkey(sc->peer_pkeys[SSL_PKEY_RSA].x509);
         else if (alg_a & SSL_aECDSA)
                 *pkey = X509_get_pubkey(sc->peer_pkeys[SSL_PKEY_ECC].x509);
         else
@@ -1933,7 +1933,7 @@ ssl3_send_client_kex_rsa(SSL *s, SESS_CERT *sess_cert, CBB *cbb)
          * RSA-Encrypted Premaster Secret Message - RFC 5246 section 7.4.7.1.
          */
 
-        pkey = X509_get_pubkey(sess_cert->peer_pkeys[SSL_PKEY_RSA_ENC].x509);
+        pkey = X509_get_pubkey(sess_cert->peer_pkeys[SSL_PKEY_RSA].x509);
         if (pkey == NULL || pkey->type != EVP_PKEY_RSA ||
             pkey->pkey.rsa == NULL) {
                 SSLerror(s, ERR_R_INTERNAL_ERROR);
diff --git a/src/lib/libssl/ssl_lib.c b/src/lib/libssl/ssl_lib.c
index 1c4ab636a1..fa1d209c8c 100644
--- a/src/lib/libssl/ssl_lib.c
+++ b/src/lib/libssl/ssl_lib.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: ssl_lib.c,v 1.213 2020/05/10 14:17:47 jsing Exp $ */
+/* $OpenBSD: ssl_lib.c,v 1.214 2020/05/19 16:35:20 jsing Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -1968,7 +1968,7 @@ SSL_CTX_set_verify_depth(SSL_CTX *ctx, int depth)
 void
 ssl_set_cert_masks(CERT *c, const SSL_CIPHER *cipher)
 {
-        int              rsa_enc, rsa_sign, dh_tmp;
+        int              rsa, dh_tmp;
         int              have_ecc_cert;
         unsigned long    mask_k, mask_a;
         X509            *x = NULL;
@@ -1980,10 +1980,8 @@ ssl_set_cert_masks(CERT *c, const SSL_CIPHER *cipher)
         dh_tmp = (c->dh_tmp != NULL || c->dh_tmp_cb != NULL ||
             c->dh_tmp_auto != 0);
 
-        cpk = &(c->pkeys[SSL_PKEY_RSA_ENC]);
-        rsa_enc = (cpk->x509 != NULL && cpk->privatekey != NULL);
-        cpk = &(c->pkeys[SSL_PKEY_RSA_SIGN]);
-        rsa_sign = (cpk->x509 != NULL && cpk->privatekey != NULL);
+        cpk = &(c->pkeys[SSL_PKEY_RSA]);
+        rsa = (cpk->x509 != NULL && cpk->privatekey != NULL);
         cpk = &(c->pkeys[SSL_PKEY_ECC]);
         have_ecc_cert = (cpk->x509 != NULL && cpk->privatekey != NULL);
 
@@ -1996,13 +1994,13 @@ ssl_set_cert_masks(CERT *c, const SSL_CIPHER *cipher)
                 mask_a |= SSL_aGOST01;
         }
 
-        if (rsa_enc)
+        if (rsa)
                 mask_k |= SSL_kRSA;
 
         if (dh_tmp)
                 mask_k |= SSL_kDHE;
 
-        if (rsa_enc || rsa_sign)
+        if (rsa)
                 mask_a |= SSL_aRSA;
 
         mask_a |= SSL_aNULL;
@@ -2085,10 +2083,7 @@ ssl_get_server_send_pkey(const SSL *s)
         if (alg_a & SSL_aECDSA) {
                 i = SSL_PKEY_ECC;
         } else if (alg_a & SSL_aRSA) {
-                if (c->pkeys[SSL_PKEY_RSA_ENC].x509 == NULL)
-                        i = SSL_PKEY_RSA_SIGN;
-                else
-                        i = SSL_PKEY_RSA_ENC;
+                i = SSL_PKEY_RSA;
         } else if (alg_a & SSL_aGOST01) {
                 i = SSL_PKEY_GOST01;
         } else { /* if (alg_a & SSL_aNULL) */
@@ -2113,10 +2108,7 @@ ssl_get_sign_pkey(SSL *s, const SSL_CIPHER *cipher, const EVP_MD **pmd,
         c = s->cert;
 
         if (alg_a & SSL_aRSA) {
-                if (c->pkeys[SSL_PKEY_RSA_SIGN].privatekey != NULL)
-                        idx = SSL_PKEY_RSA_SIGN;
-                else if (c->pkeys[SSL_PKEY_RSA_ENC].privatekey != NULL)
-                        idx = SSL_PKEY_RSA_ENC;
+                idx = SSL_PKEY_RSA;
         } else if ((alg_a & SSL_aECDSA) &&
             (c->pkeys[SSL_PKEY_ECC].privatekey != NULL))
                 idx = SSL_PKEY_ECC;
diff --git a/src/lib/libssl/ssl_locl.h b/src/lib/libssl/ssl_locl.h
index 41ac3ca35a..736005b5c9 100644
--- a/src/lib/libssl/ssl_locl.h
+++ b/src/lib/libssl/ssl_locl.h
@@ -1,4 +1,4 @@
-/* $OpenBSD: ssl_locl.h,v 1.274 2020/05/11 18:19:19 jsing Exp $ */
+/* $OpenBSD: ssl_locl.h,v 1.275 2020/05/19 16:35:20 jsing Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -333,12 +333,10 @@ __BEGIN_HIDDEN_DECLS
 #define SSL_USE_TLS1_3_CIPHERS(s) \
         (s->method->internal->ssl3_enc->enc_flags & SSL_ENC_FLAG_TLS1_3_CIPHERS)
 
-#define SSL_PKEY_RSA_ENC        0
-#define SSL_PKEY_RSA_SIGN       1
-#define SSL_PKEY_DH_RSA         2
-#define SSL_PKEY_ECC            3
-#define SSL_PKEY_GOST01         4
-#define SSL_PKEY_NUM            5
+#define SSL_PKEY_RSA            0
+#define SSL_PKEY_ECC            1
+#define SSL_PKEY_GOST01         2
+#define SSL_PKEY_NUM            3
 
 #define SSL_MAX_EMPTY_RECORDS   32
 
diff --git a/src/lib/libssl/ssl_srvr.c b/src/lib/libssl/ssl_srvr.c
index e78099cdad..6a90ad17eb 100644
--- a/src/lib/libssl/ssl_srvr.c
+++ b/src/lib/libssl/ssl_srvr.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: ssl_srvr.c,v 1.75 2020/05/10 14:17:48 jsing Exp $ */
+/* $OpenBSD: ssl_srvr.c,v 1.76 2020/05/19 16:35:20 jsing Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -1693,7 +1693,7 @@ ssl3_get_client_kex_rsa(SSL *s, CBS *cbs)
         fakekey[0] = s->client_version >> 8;
         fakekey[1] = s->client_version & 0xff;
 
-        pkey = s->cert->pkeys[SSL_PKEY_RSA_ENC].privatekey;
+        pkey = s->cert->pkeys[SSL_PKEY_RSA].privatekey;
         if ((pkey == NULL) || (pkey->type != EVP_PKEY_RSA) ||
             (pkey->pkey.rsa == NULL)) {
                 al = SSL_AD_HANDSHAKE_FAILURE;
diff --git a/src/lib/libssl/tls13_client.c b/src/lib/libssl/tls13_client.c
index a17b2bd47f..cef49c496e 100644
--- a/src/lib/libssl/tls13_client.c
+++ b/src/lib/libssl/tls13_client.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: tls13_client.c,v 1.62 2020/05/19 01:30:34 beck Exp $ */
+/* $OpenBSD: tls13_client.c,v 1.63 2020/05/19 16:35:21 jsing Exp $ */
 /*
  * Copyright (c) 2018, 2019 Joel Sing <jsing@openbsd.org>
  *
@@ -834,7 +834,7 @@ tls13_client_certificate_send(struct tls13_ctx *ctx, CBB *cbb)
         int i, ret = 0;
 
         /* XXX - Need to revisit certificate selection. */
-        cpk = &s->cert->pkeys[SSL_PKEY_RSA_ENC];
+        cpk = &s->cert->pkeys[SSL_PKEY_RSA];
 
         if ((chain = cpk->chain) == NULL)
                 chain = s->ctx->extra_certs;
@@ -884,7 +884,7 @@ tls13_client_certificate_verify_send(struct tls13_ctx *ctx, CBB *cbb)
         memset(&sig_cbb, 0, sizeof(sig_cbb));
 
         /* XXX - Need to revisit certificate selection. */
-        cpk = &s->cert->pkeys[SSL_PKEY_RSA_ENC];
+        cpk = &s->cert->pkeys[SSL_PKEY_RSA];
         pkey = cpk->privatekey;
 
         if ((sigalg = ssl_sigalg_select(s, pkey)) == NULL) {
diff --git a/src/lib/libssl/tls13_server.c b/src/lib/libssl/tls13_server.c
index ea14cfa683..03d0e488ba 100644
--- a/src/lib/libssl/tls13_server.c
+++ b/src/lib/libssl/tls13_server.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: tls13_server.c,v 1.48 2020/05/19 01:30:34 beck Exp $ */
+/* $OpenBSD: tls13_server.c,v 1.49 2020/05/19 16:35:21 jsing Exp $ */
 /*
  * Copyright (c) 2019, 2020 Joel Sing <jsing@openbsd.org>
  * Copyright (c) 2020 Bob Beck <beck@openbsd.org>
@@ -437,7 +437,7 @@ tls13_server_certificate_send(struct tls13_ctx *ctx, CBB *cbb)
         int i, ret = 0;
 
         /* XXX - Need to revisit certificate selection. */
-        cpk = &s->cert->pkeys[SSL_PKEY_RSA_ENC];
+        cpk = &s->cert->pkeys[SSL_PKEY_RSA];
         if (cpk->x509 == NULL) {
                 /* A server must always provide a certificate. */
                 ctx->alert = TLS13_ALERT_HANDSHAKE_FAILURE;
@@ -489,7 +489,7 @@ tls13_server_certificate_verify_send(struct tls13_ctx *ctx, CBB *cbb)
         memset(&sig_cbb, 0, sizeof(sig_cbb));
 
         /* XXX - Need to revisit certificate selection. */
-        cpk = &s->cert->pkeys[SSL_PKEY_RSA_ENC];
+        cpk = &s->cert->pkeys[SSL_PKEY_RSA];
         pkey = cpk->privatekey;
 
         if ((sigalg = ssl_sigalg_select(s, pkey)) == NULL) {