Garbage collect the DH_check*_ex() API

This was only needed by the EVP_PKEY_*check() API, which was defanged. So
this silly garbage can now go: it translated flags to errors on the error
stack so that openssl *check could print ugly errors while DoS-ing the
user.

ok beck

2 files changed, 4 insertions, 75 deletions

diff --git a/src/lib/libcrypto/dh/dh_check.c b/src/lib/libcrypto/dh/dh_check.c
index be79c2a04b..57330b2068 100644
--- a/src/lib/libcrypto/dh/dh_check.c
+++ b/src/lib/libcrypto/dh/dh_check.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: dh_check.c,v 1.28 2023/07/24 16:25:02 tb Exp $ */
+/* $OpenBSD: dh_check.c,v 1.29 2024/08/30 17:44:56 tb Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -68,27 +68,10 @@
 #define DH_NUMBER_ITERATIONS_FOR_PRIME 64
 
 /*
- * Check that p is odd and 1 < g < p - 1. The _ex version removes the need of
- * inspecting flags and pushes errors on the stack instead.
+ * Check that p is odd and 1 < g < p - 1.
  */
 
-int
-DH_check_params_ex(const DH *dh)
-{
-        int flags = 0;
-
-        if (!DH_check_params(dh, &flags))
-                return 0;
-
-        if ((flags & DH_CHECK_P_NOT_PRIME) != 0)
-                DHerror(DH_R_CHECK_P_NOT_PRIME);
-        if ((flags & DH_NOT_SUITABLE_GENERATOR) != 0)
-                DHerror(DH_R_NOT_SUITABLE_GENERATOR);
-
-        return flags == 0;
-}
-
-int
+static int
 DH_check_params(const DH *dh, int *flags)
 {
         BIGNUM *max_g = NULL;
@@ -124,36 +107,9 @@ DH_check_params(const DH *dh, int *flags)
 
 /*
  * Check that p is a safe prime and that g is a suitable generator.
- * The _ex version puts errors on the stack instead of returning flags.
  */
 
 int
-DH_check_ex(const DH *dh)
-{
-        int flags = 0;
-
-        if (!DH_check(dh, &flags))
-                return 0;
-
-        if ((flags & DH_NOT_SUITABLE_GENERATOR) != 0)
-                DHerror(DH_R_NOT_SUITABLE_GENERATOR);
-        if ((flags & DH_CHECK_Q_NOT_PRIME) != 0)
-                DHerror(DH_R_CHECK_Q_NOT_PRIME);
-        if ((flags & DH_CHECK_INVALID_Q_VALUE) != 0)
-                DHerror(DH_R_CHECK_INVALID_Q_VALUE);
-        if ((flags & DH_CHECK_INVALID_J_VALUE) != 0)
-                DHerror(DH_R_CHECK_INVALID_J_VALUE);
-        if ((flags & DH_UNABLE_TO_CHECK_GENERATOR) != 0)
-                DHerror(DH_R_UNABLE_TO_CHECK_GENERATOR);
-        if ((flags & DH_CHECK_P_NOT_PRIME) != 0)
-                DHerror(DH_R_CHECK_P_NOT_PRIME);
-        if ((flags & DH_CHECK_P_NOT_SAFE_PRIME) != 0)
-                DHerror(DH_R_CHECK_P_NOT_SAFE_PRIME);
-
-        return flags == 0;
-}
-
-int
 DH_check(const DH *dh, int *flags)
 {
         BN_CTX *ctx = NULL;
@@ -230,24 +186,6 @@ DH_check(const DH *dh, int *flags)
 LCRYPTO_ALIAS(DH_check);
 
 int
-DH_check_pub_key_ex(const DH *dh, const BIGNUM *pub_key)
-{
-        int flags = 0;
-
-        if (!DH_check_pub_key(dh, pub_key, &flags))
-                return 0;
-
-        if ((flags & DH_CHECK_PUBKEY_TOO_SMALL) != 0)
-                DHerror(DH_R_CHECK_PUBKEY_TOO_SMALL);
-        if ((flags & DH_CHECK_PUBKEY_TOO_LARGE) != 0)
-                DHerror(DH_R_CHECK_PUBKEY_TOO_LARGE);
-        if ((flags & DH_CHECK_PUBKEY_INVALID) != 0)
-                DHerror(DH_R_CHECK_PUBKEY_INVALID);
-
-        return flags == 0;
-}
-
-int
 DH_check_pub_key(const DH *dh, const BIGNUM *pub_key, int *flags)
 {
         BN_CTX *ctx = NULL;
diff --git a/src/lib/libcrypto/dh/dh_local.h b/src/lib/libcrypto/dh/dh_local.h
index 22e2256906..fe7c12bb05 100644
--- a/src/lib/libcrypto/dh/dh_local.h
+++ b/src/lib/libcrypto/dh/dh_local.h
@@ -1,4 +1,4 @@
-/* $OpenBSD: dh_local.h,v 1.4 2023/11/29 21:35:57 tb Exp $ */
+/* $OpenBSD: dh_local.h,v 1.5 2024/08/30 17:44:56 tb Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -102,15 +102,6 @@ struct dh_st {
         const DH_METHOD *meth;
 };
 
-/*
- * Public API in OpenSSL that we only want to use internally.
- */
-
-int DH_check_params_ex(const DH *dh);
-int DH_check_params(const DH *dh, int *flags);
-int DH_check_ex(const DH *dh);
-int DH_check_pub_key_ex(const DH *dh, const BIGNUM *pub_key);
-
 __END_HIDDEN_DECLS
 
 #endif /* !HEADER_DH_LOCAL_H */