Provide DH_check*_ex and many error codes

DH_check{,_pub_key}_ex() wrap their non-ex versions to translate
the flags argument of the original functions into OpenSSL errors.
For this almost a dozen new error codes need to be added.

DH_params_check{,_ex}() is a new version of DH_check that only
performs a cheap subset of the checks.

They are needed to implement EVP_PKEY_{public,param}_check()
(observe the consistent naming) although the actual implementation
of EVP_PKEY_param_check() chose to use DH_check_ex().

As far as I can tell, the only raison d'être of the _ex functions
and error codes is to spew them to stderr in a couple of openssl(1)
commands. This couldn't have been solved differently...

These functions will not be exposed publicly.

ok inoguchi jsing

4 files changed, 102 insertions, 4 deletions

diff --git a/src/lib/libcrypto/dh/dh.h b/src/lib/libcrypto/dh/dh.h
index 3059b291cf..21e840efc4 100644
--- a/src/lib/libcrypto/dh/dh.h
+++ b/src/lib/libcrypto/dh/dh.h
@@ -1,4 +1,4 @@
-/* $OpenBSD: dh.h,v 1.29 2022/01/07 09:21:21 tb Exp $ */
+/* $OpenBSD: dh.h,v 1.30 2022/01/10 12:00:52 tb Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -288,6 +288,17 @@ void ERR_load_DH_strings(void);
 #define DH_R_NO_PARAMETERS_SET                           107
 #define DH_R_NO_PRIVATE_VALUE                            100
 #define DH_R_PARAMETER_ENCODING_ERROR                    105
+#define DH_R_CHECK_INVALID_J_VALUE                       115
+#define DH_R_CHECK_INVALID_Q_VALUE                       116
+#define DH_R_CHECK_PUBKEY_INVALID                        122
+#define DH_R_CHECK_PUBKEY_TOO_LARGE                      123
+#define DH_R_CHECK_PUBKEY_TOO_SMALL                      124
+#define DH_R_CHECK_P_NOT_PRIME                           117
+#define DH_R_CHECK_P_NOT_SAFE_PRIME                      118
+#define DH_R_CHECK_Q_NOT_PRIME                           119
+#define DH_R_MISSING_PUBKEY                              125
+#define DH_R_NOT_SUITABLE_GENERATOR                      120
+#define DH_R_UNABLE_TO_CHECK_GENERATOR                   121
 
 #ifdef  __cplusplus
 }
diff --git a/src/lib/libcrypto/dh/dh_check.c b/src/lib/libcrypto/dh/dh_check.c
index 7203936611..1d20952e26 100644
--- a/src/lib/libcrypto/dh/dh_check.c
+++ b/src/lib/libcrypto/dh/dh_check.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: dh_check.c,v 1.23 2022/01/07 09:27:13 tb Exp $ */
+/* $OpenBSD: dh_check.c,v 1.24 2022/01/10 12:00:52 tb Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -60,12 +60,34 @@
 
 #include <openssl/bn.h>
 #include <openssl/dh.h>
+#include <openssl/err.h>
 
 #include "bn_lcl.h"
 #include "dh_local.h"
 
 #define DH_NUMBER_ITERATIONS_FOR_PRIME 64
 
+/*
+ * Check that p is odd and 1 < g < p - 1. The _ex version removes the need of
+ * inspecting flags and pushes errors on the stack instead.
+ */
+
+int
+DH_check_params_ex(const DH *dh)
+{
+        int flags = 0;
+
+        if (!DH_check_params(dh, &flags))
+                return 0;
+
+        if ((flags & DH_CHECK_P_NOT_PRIME) != 0)
+                DHerror(DH_R_CHECK_P_NOT_PRIME);
+        if ((flags & DH_NOT_SUITABLE_GENERATOR) != 0)
+                DHerror(DH_R_NOT_SUITABLE_GENERATOR);
+
+        return flags == 0;
+}
+
 int
 DH_check_params(const DH *dh, int *flags)
 {
@@ -102,9 +124,36 @@ DH_check_params(const DH *dh, int *flags)
 
 /*
  * Check that p is a safe prime and that g is a suitable generator.
+ * The _ex version puts errors on the stack instead of returning flags.
  */
 
 int
+DH_check_ex(const DH *dh)
+{
+        int flags = 0;
+
+        if (!DH_check(dh, &flags))
+                return 0;
+
+        if ((flags & DH_NOT_SUITABLE_GENERATOR) != 0)
+                DHerror(DH_R_NOT_SUITABLE_GENERATOR);
+        if ((flags & DH_CHECK_Q_NOT_PRIME) != 0)
+                DHerror(DH_R_CHECK_Q_NOT_PRIME);
+        if ((flags & DH_CHECK_INVALID_Q_VALUE) != 0)
+                DHerror(DH_R_CHECK_INVALID_Q_VALUE);
+        if ((flags & DH_CHECK_INVALID_J_VALUE) != 0)
+                DHerror(DH_R_CHECK_INVALID_J_VALUE);
+        if ((flags & DH_UNABLE_TO_CHECK_GENERATOR) != 0)
+                DHerror(DH_R_UNABLE_TO_CHECK_GENERATOR);
+        if ((flags & DH_CHECK_P_NOT_PRIME) != 0)
+                DHerror(DH_R_CHECK_P_NOT_PRIME);
+        if ((flags & DH_CHECK_P_NOT_SAFE_PRIME) != 0)
+                DHerror(DH_R_CHECK_P_NOT_SAFE_PRIME);
+
+        return flags == 0;
+}
+
+int
 DH_check(const DH *dh, int *flags)
 {
         BN_CTX *ctx = NULL;
@@ -180,6 +229,24 @@ DH_check(const DH *dh, int *flags)
 }
 
 int
+DH_check_pub_key_ex(const DH *dh, const BIGNUM *pub_key)
+{
+        int flags = 0;
+
+        if (!DH_check_pub_key(dh, pub_key, &flags))
+                return 0;
+
+        if ((flags & DH_CHECK_PUBKEY_TOO_SMALL) != 0)
+                DHerror(DH_R_CHECK_PUBKEY_TOO_SMALL);
+        if ((flags & DH_CHECK_PUBKEY_TOO_LARGE) != 0)
+                DHerror(DH_R_CHECK_PUBKEY_TOO_LARGE);
+        if ((flags & DH_CHECK_PUBKEY_INVALID) != 0)
+                DHerror(DH_R_CHECK_PUBKEY_INVALID);
+
+        return flags == 0;
+}
+
+int
 DH_check_pub_key(const DH *dh, const BIGNUM *pub_key, int *flags)
 {
         BN_CTX *ctx = NULL;
diff --git a/src/lib/libcrypto/dh/dh_err.c b/src/lib/libcrypto/dh/dh_err.c
index 497f88436e..a387c37cca 100644
--- a/src/lib/libcrypto/dh/dh_err.c
+++ b/src/lib/libcrypto/dh/dh_err.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: dh_err.c,v 1.16 2017/01/29 17:49:22 beck Exp $ */
+/* $OpenBSD: dh_err.c,v 1.17 2022/01/10 12:00:52 tb Exp $ */
 /* ====================================================================
  * Copyright (c) 1999-2011 The OpenSSL Project.  All rights reserved.
  *
@@ -90,6 +90,17 @@ static ERR_STRING_DATA DH_str_reasons[]=
 {ERR_REASON(DH_R_NO_PARAMETERS_SET)      ,"no parameters set"},
 {ERR_REASON(DH_R_NO_PRIVATE_VALUE)       ,"no private value"},
 {ERR_REASON(DH_R_PARAMETER_ENCODING_ERROR),"parameter encoding error"},
+{ERR_REASON(DH_R_CHECK_INVALID_J_VALUE)   ,"check invalid j value"},
+{ERR_REASON(DH_R_CHECK_INVALID_Q_VALUE)   ,"check invalid q value"},
+{ERR_REASON(DH_R_CHECK_PUBKEY_INVALID)    ,"check pubkey invalid"},
+{ERR_REASON(DH_R_CHECK_PUBKEY_TOO_LARGE)  ,"check pubkey too large"},
+{ERR_REASON(DH_R_CHECK_PUBKEY_TOO_SMALL)  ,"check pubkey too small"},
+{ERR_REASON(DH_R_CHECK_P_NOT_PRIME)       ,"check p not prime"},
+{ERR_REASON(DH_R_CHECK_P_NOT_SAFE_PRIME)  ,"check p not safe prime"},
+{ERR_REASON(DH_R_CHECK_Q_NOT_PRIME)       ,"check q not prime"},
+{ERR_REASON(DH_R_MISSING_PUBKEY)          ,"missing pubkey"},
+{ERR_REASON(DH_R_NOT_SUITABLE_GENERATOR)  ,"not suitable generator"},
+{ERR_REASON(DH_R_UNABLE_TO_CHECK_GENERATOR),"unable to check generator"},
 {0,NULL}
         };
 
diff --git a/src/lib/libcrypto/dh/dh_local.h b/src/lib/libcrypto/dh/dh_local.h
index 82054af757..21bc266a9c 100644
--- a/src/lib/libcrypto/dh/dh_local.h
+++ b/src/lib/libcrypto/dh/dh_local.h
@@ -1,4 +1,4 @@
-/* $OpenBSD: dh_local.h,v 1.1 2022/01/07 09:27:13 tb Exp $ */
+/* $OpenBSD: dh_local.h,v 1.2 2022/01/10 12:00:52 tb Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -61,6 +61,15 @@
 
 __BEGIN_HIDDEN_DECLS
 
+/*
+ * Public API in OpenSSL that we only want to use internally.
+ */
+
+int DH_check_params_ex(const DH *dh);
+int DH_check_params(const DH *dh, int *flags);
+int DH_check_ex(const DH *dh);
+int DH_check_pub_key_ex(const DH *dh, const BIGNUM *pub_key);
+
 __END_HIDDEN_DECLS
 
 #endif /* !HEADER_DH_LOCAL_H */