Prepare to provide EVP_PKEY_{public,param}_check

This implements checking of a public key and of key generation
parameters for DH and EC keys. With the same logic and setters
and const quirks as for EVP_PKEY_check().

There are a couple of quirks: For DH no default EVP_PKEY_check()
is implemented, instead EVP_PKEY_param_check() calls DH_check_ex()
even though DH_param_check_ex() was added for this purpose.
EVP_PKEY_public_check() for EC curves also checks the private key
if present.

ok inoguchi jsing

4 files changed, 72 insertions, 4 deletions

diff --git a/src/lib/libcrypto/evp/evp.h b/src/lib/libcrypto/evp/evp.h
index e122a6b329..a3a55caf88 100644
--- a/src/lib/libcrypto/evp/evp.h
+++ b/src/lib/libcrypto/evp/evp.h
@@ -1,4 +1,4 @@
-/* $OpenBSD: evp.h,v 1.93 2022/01/10 11:52:43 tb Exp $ */
+/* $OpenBSD: evp.h,v 1.94 2022/01/10 12:10:26 tb Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -1090,6 +1090,10 @@ void EVP_PKEY_asn1_set_ctrl(EVP_PKEY_ASN1_METHOD *ameth,
 #if defined(LIBRESSL_CRYPTO_INTERNAL) || defined(LIBRESSL_NEXT_API)
 void EVP_PKEY_asn1_set_check(EVP_PKEY_ASN1_METHOD *ameth,
     int (*pkey_check)(const EVP_PKEY *pk));
+void EVP_PKEY_asn1_set_public_check(EVP_PKEY_ASN1_METHOD *ameth,
+    int (*pkey_public_check)(const EVP_PKEY *pk));
+void EVP_PKEY_asn1_set_param_check(EVP_PKEY_ASN1_METHOD *ameth,
+    int (*pkey_check)(const EVP_PKEY *pk));
 #endif
 
 #define EVP_PKEY_OP_UNDEFINED           0
@@ -1220,6 +1224,8 @@ int EVP_PKEY_keygen_init(EVP_PKEY_CTX *ctx);
 int EVP_PKEY_keygen(EVP_PKEY_CTX *ctx, EVP_PKEY **ppkey);
 #if defined(LIBRESSL_CRYPTO_INTERNAL) || defined(LIBRESSL_NEXT_API)
 int EVP_PKEY_check(EVP_PKEY_CTX *ctx);
+int EVP_PKEY_public_check(EVP_PKEY_CTX *ctx);
+int EVP_PKEY_param_check(EVP_PKEY_CTX *ctx);
 #endif
 
 void EVP_PKEY_CTX_set_cb(EVP_PKEY_CTX *ctx, EVP_PKEY_gen_cb *cb);
@@ -1290,6 +1296,10 @@ void EVP_PKEY_meth_set_ctrl(EVP_PKEY_METHOD *pmeth,
 #if defined(LIBRESSL_CRYPTO_INTERNAL) || defined(LIBRESSL_NEXT_API)
 void EVP_PKEY_meth_set_check(EVP_PKEY_METHOD *pmeth,
     int (*check)(EVP_PKEY *pkey));
+void EVP_PKEY_meth_set_public_check(EVP_PKEY_METHOD *pmeth,
+    int (*public_check)(EVP_PKEY *pkey));
+void EVP_PKEY_meth_set_param_check(EVP_PKEY_METHOD *pmeth,
+    int (*param_check)(EVP_PKEY *pkey));
 #endif
 
 /* Authenticated Encryption with Additional Data.
diff --git a/src/lib/libcrypto/evp/evp_locl.h b/src/lib/libcrypto/evp/evp_locl.h
index 3ff8e8ad99..44e2d5cadb 100644
--- a/src/lib/libcrypto/evp/evp_locl.h
+++ b/src/lib/libcrypto/evp/evp_locl.h
@@ -1,4 +1,4 @@
-/* $OpenBSD: evp_locl.h,v 1.19 2022/01/10 11:52:43 tb Exp $ */
+/* $OpenBSD: evp_locl.h,v 1.20 2022/01/10 12:10:26 tb Exp $ */
 /* Written by Dr Stephen N Henson (steve@openssl.org) for the OpenSSL
  * project 2000.
  */
@@ -349,6 +349,8 @@ struct evp_pkey_method_st {
         int (*ctrl_str)(EVP_PKEY_CTX *ctx, const char *type, const char *value);
 
         int (*check)(EVP_PKEY *pkey);
+        int (*public_check)(EVP_PKEY *pkey);
+        int (*param_check)(EVP_PKEY *pkey);
 } /* EVP_PKEY_METHOD */;
 
 void evp_pkey_set_cb_translate(BN_GENCB *cb, EVP_PKEY_CTX *ctx);
diff --git a/src/lib/libcrypto/evp/pmeth_gn.c b/src/lib/libcrypto/evp/pmeth_gn.c
index a8a4cc97db..7d921d23b4 100644
--- a/src/lib/libcrypto/evp/pmeth_gn.c
+++ b/src/lib/libcrypto/evp/pmeth_gn.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: pmeth_gn.c,v 1.9 2022/01/10 11:52:43 tb Exp $ */
+/* $OpenBSD: pmeth_gn.c,v 1.10 2022/01/10 12:10:26 tb Exp $ */
 /* Written by Dr Stephen N Henson (steve@openssl.org) for the OpenSSL
  * project 2006.
  */
@@ -244,3 +244,45 @@ EVP_PKEY_check(EVP_PKEY_CTX *ctx)
 
         return pkey->ameth->pkey_check(pkey);
 }
+
+int
+EVP_PKEY_public_check(EVP_PKEY_CTX *ctx)
+{
+        EVP_PKEY *pkey;
+
+        if ((pkey = ctx->pkey) == NULL) {
+                EVPerror(EVP_R_NO_KEY_SET);
+                return 0;
+        }
+
+        if (ctx->pmeth->public_check != NULL)
+                return ctx->pmeth->public_check(pkey);
+
+        if (pkey->ameth == NULL || pkey->ameth->pkey_public_check == NULL) {
+                EVPerror(EVP_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE);
+                return -2;
+        }
+
+        return pkey->ameth->pkey_public_check(pkey);
+}
+
+int
+EVP_PKEY_param_check(EVP_PKEY_CTX *ctx)
+{
+        EVP_PKEY *pkey;
+
+        if ((pkey = ctx->pkey) == NULL) {
+                EVPerror(EVP_R_NO_KEY_SET);
+                return 0;
+        }
+
+        if (ctx->pmeth->param_check != NULL)
+                return ctx->pmeth->param_check(pkey);
+
+        if (pkey->ameth == NULL || pkey->ameth->pkey_param_check == NULL) {
+                EVPerror(EVP_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE);
+                return -2;
+        }
+
+        return pkey->ameth->pkey_param_check(pkey);
+}
diff --git a/src/lib/libcrypto/evp/pmeth_lib.c b/src/lib/libcrypto/evp/pmeth_lib.c
index 92328dd246..d265e2aced 100644
--- a/src/lib/libcrypto/evp/pmeth_lib.c
+++ b/src/lib/libcrypto/evp/pmeth_lib.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: pmeth_lib.c,v 1.19 2022/01/10 11:52:43 tb Exp $ */
+/* $OpenBSD: pmeth_lib.c,v 1.20 2022/01/10 12:10:26 tb Exp $ */
 /* Written by Dr Stephen N Henson (steve@openssl.org) for the OpenSSL
  * project 2006.
  */
@@ -588,3 +588,17 @@ EVP_PKEY_meth_set_check(EVP_PKEY_METHOD *pmeth, int (*check)(EVP_PKEY *pkey))
 {
         pmeth->check = check;
 }
+
+void
+EVP_PKEY_meth_set_public_check(EVP_PKEY_METHOD *pmeth,
+    int (*public_check)(EVP_PKEY *pkey))
+{
+        pmeth->public_check = public_check;
+}
+
+void
+EVP_PKEY_meth_set_param_check(EVP_PKEY_METHOD *pmeth,
+    int (*param_check)(EVP_PKEY *pkey))
+{
+        pmeth->param_check = param_check;
+}