Remove the pkey_{,public_,param_}check() handlers

This disables the EVP_PKEY_*check() API and makes it fail (more precisely
indicate lack of support) on all key types.

This is an intermediate step to full removal.
Removal is ok beck jsing

5 files changed, 15 insertions, 140 deletions

diff --git a/src/lib/libcrypto/dh/dh_ameth.c b/src/lib/libcrypto/dh/dh_ameth.c
index d51a5bea4c..13141761d4 100644
--- a/src/lib/libcrypto/dh/dh_ameth.c
+++ b/src/lib/libcrypto/dh/dh_ameth.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: dh_ameth.c,v 1.40 2024/01/04 17:01:26 tb Exp $ */
+/* $OpenBSD: dh_ameth.c,v 1.41 2024/08/29 16:58:19 tb Exp $ */
 /* Written by Dr Stephen N Henson (steve@openssl.org) for the OpenSSL
  * project 2006.
  */
@@ -496,32 +496,6 @@ DHparams_print_fp(FILE *fp, const DH *x)
 }
 LCRYPTO_ALIAS(DHparams_print_fp);
 
-static int
-dh_pkey_public_check(const EVP_PKEY *pkey)
-{
-        DH *dh = pkey->pkey.dh;
-
-        if (dh->pub_key == NULL) {
-                DHerror(DH_R_MISSING_PUBKEY);
-                return 0;
-        }
-
-        return DH_check_pub_key_ex(dh, dh->pub_key);
-}
-
-static int
-dh_pkey_param_check(const EVP_PKEY *pkey)
-{
-        DH *dh = pkey->pkey.dh;
-
-        /*
-         * It would have made more sense to support EVP_PKEY_check() for DH
-         * keys and call DH_check_ex() there and keeping this as a wrapper
-         * for DH_param_check_ex(). We follow OpenSSL's choice.
-         */
-        return DH_check_ex(dh);
-}
-
 const EVP_PKEY_ASN1_METHOD dh_asn1_meth = {
         .base_method = &dh_asn1_meth,
         .pkey_id = EVP_PKEY_DH,
@@ -550,8 +524,4 @@ const EVP_PKEY_ASN1_METHOD dh_asn1_meth = {
         .param_print = dh_param_print,
 
         .pkey_free = dh_free,
-
-        .pkey_check = NULL,
-        .pkey_public_check = dh_pkey_public_check,
-        .pkey_param_check = dh_pkey_param_check,
 };
diff --git a/src/lib/libcrypto/ec/ec_ameth.c b/src/lib/libcrypto/ec/ec_ameth.c
index 378261b946..f42093c3db 100644
--- a/src/lib/libcrypto/ec/ec_ameth.c
+++ b/src/lib/libcrypto/ec/ec_ameth.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: ec_ameth.c,v 1.68 2024/05/10 05:12:03 tb Exp $ */
+/* $OpenBSD: ec_ameth.c,v 1.69 2024/08/29 16:58:19 tb Exp $ */
 /* Written by Dr Stephen N Henson (steve@openssl.org) for the OpenSSL
  * project 2006.
  */
@@ -689,41 +689,6 @@ ec_pkey_ctrl(EVP_PKEY *pkey, int op, long arg1, void *arg2)
 
 }
 
-static int
-ec_pkey_check(const EVP_PKEY *pkey)
-{
-        EC_KEY *eckey = pkey->pkey.ec;
-
-        if (eckey->priv_key == NULL) {
-                ECerror(EC_R_MISSING_PRIVATE_KEY);
-                return 0;
-        }
-
-        return EC_KEY_check_key(eckey);
-}
-
-static int
-ec_pkey_public_check(const EVP_PKEY *pkey)
-{
-        EC_KEY *eckey = pkey->pkey.ec;
-
-        /* This also checks the private key, but oh, well... */
-        return EC_KEY_check_key(eckey);
-}
-
-static int
-ec_pkey_param_check(const EVP_PKEY *pkey)
-{
-        EC_KEY *eckey = pkey->pkey.ec;
-
-        if (eckey->group == NULL) {
-                ECerror(EC_R_MISSING_PARAMETERS);
-                return 0;
-        }
-
-        return EC_GROUP_check(eckey->group, NULL);
-}
-
 #ifndef OPENSSL_NO_CMS
 
 static int
@@ -1092,8 +1057,4 @@ const EVP_PKEY_ASN1_METHOD eckey_asn1_meth = {
         .pkey_ctrl = ec_pkey_ctrl,
         .old_priv_decode = old_ec_priv_decode,
         .old_priv_encode = old_ec_priv_encode,
-
-        .pkey_check = ec_pkey_check,
-        .pkey_public_check = ec_pkey_public_check,
-        .pkey_param_check = ec_pkey_param_check,
 };
diff --git a/src/lib/libcrypto/evp/evp_local.h b/src/lib/libcrypto/evp/evp_local.h
index 5d541ffec4..54cd65d0af 100644
--- a/src/lib/libcrypto/evp/evp_local.h
+++ b/src/lib/libcrypto/evp/evp_local.h
@@ -1,4 +1,4 @@
-/* $OpenBSD: evp_local.h,v 1.24 2024/08/28 07:15:04 tb Exp $ */
+/* $OpenBSD: evp_local.h,v 1.25 2024/08/29 16:58:19 tb Exp $ */
 /* Written by Dr Stephen N Henson (steve@openssl.org) for the OpenSSL
  * project 2000.
  */
@@ -140,10 +140,6 @@ struct evp_pkey_asn1_method_st {
         int (*item_sign)(EVP_MD_CTX *ctx, const ASN1_ITEM *it, void *asn,
             X509_ALGOR *alg1, X509_ALGOR *alg2, ASN1_BIT_STRING *sig);
 
-        int (*pkey_check)(const EVP_PKEY *pk);
-        int (*pkey_public_check)(const EVP_PKEY *pk);
-        int (*pkey_param_check)(const EVP_PKEY *pk);
-
         int (*set_priv_key)(EVP_PKEY *pk, const unsigned char *private_key,
             size_t len);
         int (*set_pub_key)(EVP_PKEY *pk, const unsigned char *public_key,
@@ -322,10 +318,6 @@ struct evp_pkey_method_st {
             const unsigned char *tbs, size_t tbslen);
         int (*digestverify) (EVP_MD_CTX *ctx, const unsigned char *sig,
             size_t siglen, const unsigned char *tbs, size_t tbslen);
-
-        int (*check)(EVP_PKEY *pkey);
-        int (*public_check)(EVP_PKEY *pkey);
-        int (*param_check)(EVP_PKEY *pkey);
 } /* EVP_PKEY_METHOD */;
 
 void evp_pkey_set_cb_translate(BN_GENCB *cb, EVP_PKEY_CTX *ctx);
diff --git a/src/lib/libcrypto/evp/pmeth_gn.c b/src/lib/libcrypto/evp/pmeth_gn.c
index 1c355e594a..415690cd0e 100644
--- a/src/lib/libcrypto/evp/pmeth_gn.c
+++ b/src/lib/libcrypto/evp/pmeth_gn.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: pmeth_gn.c,v 1.19 2024/04/17 08:24:11 tb Exp $ */
+/* $OpenBSD: pmeth_gn.c,v 1.20 2024/08/29 16:58:19 tb Exp $ */
 /* Written by Dr Stephen N Henson (steve@openssl.org) for the OpenSSL
  * project 2006.
  */
@@ -226,68 +226,30 @@ merr:
 }
 LCRYPTO_ALIAS(EVP_PKEY_new_mac_key);
 
+/*
+ * XXX - remove the API below in the next bump.
+ */
+
 int
 EVP_PKEY_check(EVP_PKEY_CTX *ctx)
 {
-        EVP_PKEY *pkey;
-
-        if ((pkey = ctx->pkey) == NULL) {
-                EVPerror(EVP_R_NO_KEY_SET);
-                return 0;
-        }
-
-        if (ctx->pmeth->check != NULL)
-                return ctx->pmeth->check(pkey);
-
-        if (pkey->ameth == NULL || pkey->ameth->pkey_check == NULL) {
-                EVPerror(EVP_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE);
-                return -2;
-        }
-
-        return pkey->ameth->pkey_check(pkey);
+        EVPerror(ERR_R_DISABLED);
+        return -2;
 }
 LCRYPTO_ALIAS(EVP_PKEY_check);
 
 int
 EVP_PKEY_public_check(EVP_PKEY_CTX *ctx)
 {
-        EVP_PKEY *pkey;
-
-        if ((pkey = ctx->pkey) == NULL) {
-                EVPerror(EVP_R_NO_KEY_SET);
-                return 0;
-        }
-
-        if (ctx->pmeth->public_check != NULL)
-                return ctx->pmeth->public_check(pkey);
-
-        if (pkey->ameth == NULL || pkey->ameth->pkey_public_check == NULL) {
-                EVPerror(EVP_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE);
-                return -2;
-        }
-
-        return pkey->ameth->pkey_public_check(pkey);
+        EVPerror(ERR_R_DISABLED);
+        return -2;
 }
 LCRYPTO_ALIAS(EVP_PKEY_public_check);
 
 int
 EVP_PKEY_param_check(EVP_PKEY_CTX *ctx)
 {
-        EVP_PKEY *pkey;
-
-        if ((pkey = ctx->pkey) == NULL) {
-                EVPerror(EVP_R_NO_KEY_SET);
-                return 0;
-        }
-
-        if (ctx->pmeth->param_check != NULL)
-                return ctx->pmeth->param_check(pkey);
-
-        if (pkey->ameth == NULL || pkey->ameth->pkey_param_check == NULL) {
-                EVPerror(EVP_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE);
-                return -2;
-        }
-
-        return pkey->ameth->pkey_param_check(pkey);
+        EVPerror(ERR_R_DISABLED);
+        return -2;
 }
 LCRYPTO_ALIAS(EVP_PKEY_param_check);
diff --git a/src/lib/libcrypto/rsa/rsa_ameth.c b/src/lib/libcrypto/rsa/rsa_ameth.c
index d7ce931733..eb5d599bf0 100644
--- a/src/lib/libcrypto/rsa/rsa_ameth.c
+++ b/src/lib/libcrypto/rsa/rsa_ameth.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: rsa_ameth.c,v 1.59 2024/08/28 07:15:04 tb Exp $ */
+/* $OpenBSD: rsa_ameth.c,v 1.60 2024/08/29 16:58:19 tb Exp $ */
 /* Written by Dr Stephen N Henson (steve@openssl.org) for the OpenSSL
  * project 2006.
  */
@@ -1082,12 +1082,6 @@ rsa_item_sign(EVP_MD_CTX *ctx, const ASN1_ITEM *it, void *asn,
         return 2;
 }
 
-static int
-rsa_pkey_check(const EVP_PKEY *pkey)
-{
-        return RSA_check_key(pkey->pkey.rsa);
-}
-
 #ifndef OPENSSL_NO_CMS
 static RSA_OAEP_PARAMS *
 rsa_oaep_decode(const X509_ALGOR *alg)
@@ -1235,16 +1229,12 @@ const EVP_PKEY_ASN1_METHOD rsa_asn1_meth = {
         .old_priv_encode = old_rsa_priv_encode,
         .item_verify = rsa_item_verify,
         .item_sign = rsa_item_sign,
-
-        .pkey_check = rsa_pkey_check,
 };
 
 const EVP_PKEY_ASN1_METHOD rsa2_asn1_meth = {
         .base_method = &rsa_asn1_meth,
         .pkey_id = EVP_PKEY_RSA2,
         .pkey_flags = ASN1_PKEY_ALIAS,
-
-        .pkey_check = rsa_pkey_check,
 };
 
 const EVP_PKEY_ASN1_METHOD rsa_pss_asn1_meth = {