Remove EVP_PKEY_*check again

This API turned out to be a really bad idea. OpenSSL 3 extended it, with
the result that basically every key type had its own DoS issues fixed in
a recent security release. We eschewed these by having some upper bounds
that kick in when keys get insanely large.

Initially added on tobhe's request who fortunately never used it in iked,
this was picked up only by ruby/openssl (one of the rare projects doing
proper configure checks rather than branching on VERSION defines) and of
course xca, since it uses everything it can. So it was easy to get rid of
this again.

ok beck jsing

4 files changed, 3 insertions, 40 deletions

diff --git a/src/lib/libcrypto/Symbols.list b/src/lib/libcrypto/Symbols.list
index dfe7ce4923..05c39eddbb 100644
--- a/src/lib/libcrypto/Symbols.list
+++ b/src/lib/libcrypto/Symbols.list
@@ -1245,7 +1245,6 @@ EVP_PKEY_asn1_get_count
 EVP_PKEY_assign
 EVP_PKEY_base_id
 EVP_PKEY_bits
-EVP_PKEY_check
 EVP_PKEY_cmp
 EVP_PKEY_cmp_parameters
 EVP_PKEY_copy_parameters
@@ -1287,13 +1286,11 @@ EVP_PKEY_new_CMAC_key
 EVP_PKEY_new_mac_key
 EVP_PKEY_new_raw_private_key
 EVP_PKEY_new_raw_public_key
-EVP_PKEY_param_check
 EVP_PKEY_paramgen
 EVP_PKEY_paramgen_init
 EVP_PKEY_print_params
 EVP_PKEY_print_private
 EVP_PKEY_print_public
-EVP_PKEY_public_check
 EVP_PKEY_save_parameters
 EVP_PKEY_security_bits
 EVP_PKEY_set1_DH
diff --git a/src/lib/libcrypto/evp/evp.h b/src/lib/libcrypto/evp/evp.h
index 2db8acfa01..7c767758da 100644
--- a/src/lib/libcrypto/evp/evp.h
+++ b/src/lib/libcrypto/evp/evp.h
@@ -1,4 +1,4 @@
-/* $OpenBSD: evp.h,v 1.135 2024/07/09 16:15:37 tb Exp $ */
+/* $OpenBSD: evp.h,v 1.136 2024/08/31 09:14:21 tb Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -985,9 +985,6 @@ int EVP_PKEY_paramgen_init(EVP_PKEY_CTX *ctx);
 int EVP_PKEY_paramgen(EVP_PKEY_CTX *ctx, EVP_PKEY **ppkey);
 int EVP_PKEY_keygen_init(EVP_PKEY_CTX *ctx);
 int EVP_PKEY_keygen(EVP_PKEY_CTX *ctx, EVP_PKEY **ppkey);
-int EVP_PKEY_check(EVP_PKEY_CTX *ctx);
-int EVP_PKEY_public_check(EVP_PKEY_CTX *ctx);
-int EVP_PKEY_param_check(EVP_PKEY_CTX *ctx);
 
 void EVP_PKEY_CTX_set_cb(EVP_PKEY_CTX *ctx, EVP_PKEY_gen_cb *cb);
 EVP_PKEY_gen_cb *EVP_PKEY_CTX_get_cb(EVP_PKEY_CTX *ctx);
diff --git a/src/lib/libcrypto/evp/pmeth_gn.c b/src/lib/libcrypto/evp/pmeth_gn.c
index 415690cd0e..bc1c5bd7d2 100644
--- a/src/lib/libcrypto/evp/pmeth_gn.c
+++ b/src/lib/libcrypto/evp/pmeth_gn.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: pmeth_gn.c,v 1.20 2024/08/29 16:58:19 tb Exp $ */
+/* $OpenBSD: pmeth_gn.c,v 1.21 2024/08/31 09:14:21 tb Exp $ */
 /* Written by Dr Stephen N Henson (steve@openssl.org) for the OpenSSL
  * project 2006.
  */
@@ -225,31 +225,3 @@ merr:
         return mac_key;
 }
 LCRYPTO_ALIAS(EVP_PKEY_new_mac_key);
-
-/*
- * XXX - remove the API below in the next bump.
- */
-
-int
-EVP_PKEY_check(EVP_PKEY_CTX *ctx)
-{
-        EVPerror(ERR_R_DISABLED);
-        return -2;
-}
-LCRYPTO_ALIAS(EVP_PKEY_check);
-
-int
-EVP_PKEY_public_check(EVP_PKEY_CTX *ctx)
-{
-        EVPerror(ERR_R_DISABLED);
-        return -2;
-}
-LCRYPTO_ALIAS(EVP_PKEY_public_check);
-
-int
-EVP_PKEY_param_check(EVP_PKEY_CTX *ctx)
-{
-        EVPerror(ERR_R_DISABLED);
-        return -2;
-}
-LCRYPTO_ALIAS(EVP_PKEY_param_check);
diff --git a/src/lib/libcrypto/hidden/openssl/evp.h b/src/lib/libcrypto/hidden/openssl/evp.h
index 7721a2f412..fea609933e 100644
--- a/src/lib/libcrypto/hidden/openssl/evp.h
+++ b/src/lib/libcrypto/hidden/openssl/evp.h
@@ -1,4 +1,4 @@
-/* $OpenBSD: evp.h,v 1.4 2024/04/10 15:00:38 beck Exp $ */
+/* $OpenBSD: evp.h,v 1.5 2024/08/31 09:14:21 tb Exp $ */
 /*
  * Copyright (c) 2024 Bob Beck <beck@openbsd.org>
  *
@@ -353,9 +353,6 @@ LCRYPTO_USED(EVP_PKEY_paramgen_init);
 LCRYPTO_USED(EVP_PKEY_paramgen);
 LCRYPTO_USED(EVP_PKEY_keygen_init);
 LCRYPTO_USED(EVP_PKEY_keygen);
-LCRYPTO_USED(EVP_PKEY_check);
-LCRYPTO_USED(EVP_PKEY_public_check);
-LCRYPTO_USED(EVP_PKEY_param_check);
 LCRYPTO_USED(EVP_PKEY_CTX_set_cb);
 LCRYPTO_USED(EVP_PKEY_CTX_get_cb);
 LCRYPTO_USED(EVP_PKEY_CTX_get_keygen_info);