Provide DH_check*_ex and many error codes

DH_check{,_pub_key}_ex() wrap their non-ex versions to translate
the flags argument of the original functions into OpenSSL errors.
For this almost a dozen new error codes need to be added.

DH_params_check{,_ex}() is a new version of DH_check that only
performs a cheap subset of the checks.

They are needed to implement EVP_PKEY_{public,param}_check()
(observe the consistent naming) although the actual implementation
of EVP_PKEY_param_check() chose to use DH_check_ex().

As far as I can tell, the only raison d'être of the _ex functions
and error codes is to spew them to stderr in a couple of openssl(1)
commands. This couldn't have been solved differently...

These functions will not be exposed publicly.

ok inoguchi jsing

1 files changed, 68 insertions, 1 deletions

diff --git a/src/lib/libcrypto/dh/dh_check.c b/src/lib/libcrypto/dh/dh_check.c
index 7203936611..1d20952e26 100644
--- a/src/lib/libcrypto/dh/dh_check.c
+++ b/src/lib/libcrypto/dh/dh_check.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: dh_check.c,v 1.23 2022/01/07 09:27:13 tb Exp $ */
+/* $OpenBSD: dh_check.c,v 1.24 2022/01/10 12:00:52 tb Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -60,12 +60,34 @@
 
 #include <openssl/bn.h>
 #include <openssl/dh.h>
+#include <openssl/err.h>
 
 #include "bn_lcl.h"
 #include "dh_local.h"
 
 #define DH_NUMBER_ITERATIONS_FOR_PRIME 64
 
+/*
+ * Check that p is odd and 1 < g < p - 1. The _ex version removes the need of
+ * inspecting flags and pushes errors on the stack instead.
+ */
+
+int
+DH_check_params_ex(const DH *dh)
+{
+        int flags = 0;
+
+        if (!DH_check_params(dh, &flags))
+                return 0;
+
+        if ((flags & DH_CHECK_P_NOT_PRIME) != 0)
+                DHerror(DH_R_CHECK_P_NOT_PRIME);
+        if ((flags & DH_NOT_SUITABLE_GENERATOR) != 0)
+                DHerror(DH_R_NOT_SUITABLE_GENERATOR);
+
+        return flags == 0;
+}
+
 int
 DH_check_params(const DH *dh, int *flags)
 {
@@ -102,9 +124,36 @@ DH_check_params(const DH *dh, int *flags)
 
 /*
  * Check that p is a safe prime and that g is a suitable generator.
+ * The _ex version puts errors on the stack instead of returning flags.
  */
 
 int
+DH_check_ex(const DH *dh)
+{
+        int flags = 0;
+
+        if (!DH_check(dh, &flags))
+                return 0;
+
+        if ((flags & DH_NOT_SUITABLE_GENERATOR) != 0)
+                DHerror(DH_R_NOT_SUITABLE_GENERATOR);
+        if ((flags & DH_CHECK_Q_NOT_PRIME) != 0)
+                DHerror(DH_R_CHECK_Q_NOT_PRIME);
+        if ((flags & DH_CHECK_INVALID_Q_VALUE) != 0)
+                DHerror(DH_R_CHECK_INVALID_Q_VALUE);
+        if ((flags & DH_CHECK_INVALID_J_VALUE) != 0)
+                DHerror(DH_R_CHECK_INVALID_J_VALUE);
+        if ((flags & DH_UNABLE_TO_CHECK_GENERATOR) != 0)
+                DHerror(DH_R_UNABLE_TO_CHECK_GENERATOR);
+        if ((flags & DH_CHECK_P_NOT_PRIME) != 0)
+                DHerror(DH_R_CHECK_P_NOT_PRIME);
+        if ((flags & DH_CHECK_P_NOT_SAFE_PRIME) != 0)
+                DHerror(DH_R_CHECK_P_NOT_SAFE_PRIME);
+
+        return flags == 0;
+}
+
+int
 DH_check(const DH *dh, int *flags)
 {
         BN_CTX *ctx = NULL;
@@ -180,6 +229,24 @@ DH_check(const DH *dh, int *flags)
 }
 
 int
+DH_check_pub_key_ex(const DH *dh, const BIGNUM *pub_key)
+{
+        int flags = 0;
+
+        if (!DH_check_pub_key(dh, pub_key, &flags))
+                return 0;
+
+        if ((flags & DH_CHECK_PUBKEY_TOO_SMALL) != 0)
+                DHerror(DH_R_CHECK_PUBKEY_TOO_SMALL);
+        if ((flags & DH_CHECK_PUBKEY_TOO_LARGE) != 0)
+                DHerror(DH_R_CHECK_PUBKEY_TOO_LARGE);
+        if ((flags & DH_CHECK_PUBKEY_INVALID) != 0)
+                DHerror(DH_R_CHECK_PUBKEY_INVALID);
+
+        return flags == 0;
+}
+
+int
 DH_check_pub_key(const DH *dh, const BIGNUM *pub_key, int *flags)
 {
         BN_CTX *ctx = NULL;