Prepare to provide EVP_PKEY_{public,param}_check

This implements checking of a public key and of key generation
parameters for DH and EC keys. With the same logic and setters
and const quirks as for EVP_PKEY_check().

There are a couple of quirks: For DH no default EVP_PKEY_check()
is implemented, instead EVP_PKEY_param_check() calls DH_check_ex()
even though DH_param_check_ex() was added for this purpose.
EVP_PKEY_public_check() for EC curves also checks the private key
if present.

ok inoguchi jsing

8 files changed, 146 insertions, 8 deletions

diff --git a/src/lib/libcrypto/asn1/ameth_lib.c b/src/lib/libcrypto/asn1/ameth_lib.c
index 96669bbd2f..8ff5a35d78 100644
--- a/src/lib/libcrypto/asn1/ameth_lib.c
+++ b/src/lib/libcrypto/asn1/ameth_lib.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: ameth_lib.c,v 1.24 2022/01/10 11:52:43 tb Exp $ */
+/* $OpenBSD: ameth_lib.c,v 1.25 2022/01/10 12:10:26 tb Exp $ */
 /* Written by Dr Stephen N Henson (steve@openssl.org) for the OpenSSL
  * project 2006.
  */
@@ -436,3 +436,17 @@ EVP_PKEY_asn1_set_check(EVP_PKEY_ASN1_METHOD *ameth,
 {
         ameth->pkey_check = pkey_check;
 }
+
+void
+EVP_PKEY_asn1_set_public_check(EVP_PKEY_ASN1_METHOD *ameth,
+    int (*pkey_public_check)(const EVP_PKEY *pk))
+{
+        ameth->pkey_public_check = pkey_public_check;
+}
+
+void
+EVP_PKEY_asn1_set_param_check(EVP_PKEY_ASN1_METHOD *ameth,
+    int (*pkey_param_check)(const EVP_PKEY *pk))
+{
+        ameth->pkey_param_check = pkey_param_check;
+}
diff --git a/src/lib/libcrypto/asn1/asn1_locl.h b/src/lib/libcrypto/asn1/asn1_locl.h
index 31fcbef20d..76b165e77f 100644
--- a/src/lib/libcrypto/asn1/asn1_locl.h
+++ b/src/lib/libcrypto/asn1/asn1_locl.h
@@ -1,4 +1,4 @@
-/* $OpenBSD: asn1_locl.h,v 1.16 2022/01/10 11:52:43 tb Exp $ */
+/* $OpenBSD: asn1_locl.h,v 1.17 2022/01/10 12:10:26 tb Exp $ */
 /* Written by Dr Stephen N Henson (steve@openssl.org) for the OpenSSL
  * project 2006.
  */
@@ -125,6 +125,8 @@ struct evp_pkey_asn1_method_st {
             X509_ALGOR *alg1, X509_ALGOR *alg2, ASN1_BIT_STRING *sig);
 
         int (*pkey_check)(const EVP_PKEY *pk);
+        int (*pkey_public_check)(const EVP_PKEY *pk);
+        int (*pkey_param_check)(const EVP_PKEY *pk);
 } /* EVP_PKEY_ASN1_METHOD */;
 
 /* Method to handle CRL access.
diff --git a/src/lib/libcrypto/dh/dh_ameth.c b/src/lib/libcrypto/dh/dh_ameth.c
index bbb687da8b..eaca890a50 100644
--- a/src/lib/libcrypto/dh/dh_ameth.c
+++ b/src/lib/libcrypto/dh/dh_ameth.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: dh_ameth.c,v 1.21 2022/01/10 00:09:06 tb Exp $ */
+/* $OpenBSD: dh_ameth.c,v 1.22 2022/01/10 12:10:26 tb Exp $ */
 /* Written by Dr Stephen N Henson (steve@openssl.org) for the OpenSSL
  * project 2006.
  */
@@ -466,6 +466,32 @@ DHparams_print(BIO *bp, const DH *x)
         return do_dh_print(bp, x, 4, NULL, 0);
 }
 
+static int
+dh_pkey_public_check(const EVP_PKEY *pkey)
+{
+        DH *dh = pkey->pkey.dh;
+
+        if (dh->pub_key == NULL) {
+                DHerror(DH_R_MISSING_PUBKEY);
+                return 0;
+        }
+
+        return DH_check_pub_key_ex(dh, dh->pub_key);
+}
+
+static int
+dh_pkey_param_check(const EVP_PKEY *pkey)
+{
+        DH *dh = pkey->pkey.dh;
+
+        /*
+         * It would have made more sense to support EVP_PKEY_check() for DH
+         * keys and call DH_check_ex() there and keeping this as a wrapper
+         * for DH_param_check_ex(). We follow OpenSSL's choice.
+         */
+        return DH_check_ex(dh);
+}
+
 const EVP_PKEY_ASN1_METHOD dh_asn1_meth = {
         .pkey_id = EVP_PKEY_DH,
         .pkey_base_id = EVP_PKEY_DH,
@@ -493,4 +519,8 @@ const EVP_PKEY_ASN1_METHOD dh_asn1_meth = {
         .param_print = dh_param_print,
 
         .pkey_free = int_dh_free,
+
+        .pkey_check = NULL,
+        .pkey_public_check = dh_pkey_public_check,
+        .pkey_param_check = dh_pkey_param_check,
 };
diff --git a/src/lib/libcrypto/ec/ec_ameth.c b/src/lib/libcrypto/ec/ec_ameth.c
index 8316683f8f..86f509b736 100644
--- a/src/lib/libcrypto/ec/ec_ameth.c
+++ b/src/lib/libcrypto/ec/ec_ameth.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: ec_ameth.c,v 1.30 2022/01/10 11:52:43 tb Exp $ */
+/* $OpenBSD: ec_ameth.c,v 1.31 2022/01/10 12:10:26 tb Exp $ */
 /* Written by Dr Stephen N Henson (steve@openssl.org) for the OpenSSL
  * project 2006.
  */
@@ -634,6 +634,28 @@ ec_pkey_check(const EVP_PKEY *pkey)
         return EC_KEY_check_key(eckey);
 }
 
+static int
+ec_pkey_public_check(const EVP_PKEY *pkey)
+{
+        EC_KEY *eckey = pkey->pkey.ec;
+
+        /* This also checks the private key, but oh, well... */
+        return EC_KEY_check_key(eckey);
+}
+
+static int
+ec_pkey_param_check(const EVP_PKEY *pkey)
+{
+        EC_KEY *eckey = pkey->pkey.ec;
+
+        if (eckey->group == NULL) {
+                ECerror(EC_R_MISSING_PARAMETERS);
+                return 0;
+        }
+
+        return EC_GROUP_check(eckey->group, NULL);
+}
+
 #ifndef OPENSSL_NO_CMS
 
 static int
@@ -998,4 +1020,6 @@ const EVP_PKEY_ASN1_METHOD eckey_asn1_meth = {
         .old_priv_encode = old_ec_priv_encode,
 
         .pkey_check = ec_pkey_check,
+        .pkey_public_check = ec_pkey_public_check,
+        .pkey_param_check = ec_pkey_param_check,
 };
diff --git a/src/lib/libcrypto/evp/evp.h b/src/lib/libcrypto/evp/evp.h
index e122a6b329..a3a55caf88 100644
--- a/src/lib/libcrypto/evp/evp.h
+++ b/src/lib/libcrypto/evp/evp.h
@@ -1,4 +1,4 @@
-/* $OpenBSD: evp.h,v 1.93 2022/01/10 11:52:43 tb Exp $ */
+/* $OpenBSD: evp.h,v 1.94 2022/01/10 12:10:26 tb Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -1090,6 +1090,10 @@ void EVP_PKEY_asn1_set_ctrl(EVP_PKEY_ASN1_METHOD *ameth,
 #if defined(LIBRESSL_CRYPTO_INTERNAL) || defined(LIBRESSL_NEXT_API)
 void EVP_PKEY_asn1_set_check(EVP_PKEY_ASN1_METHOD *ameth,
     int (*pkey_check)(const EVP_PKEY *pk));
+void EVP_PKEY_asn1_set_public_check(EVP_PKEY_ASN1_METHOD *ameth,
+    int (*pkey_public_check)(const EVP_PKEY *pk));
+void EVP_PKEY_asn1_set_param_check(EVP_PKEY_ASN1_METHOD *ameth,
+    int (*pkey_check)(const EVP_PKEY *pk));
 #endif
 
 #define EVP_PKEY_OP_UNDEFINED           0
@@ -1220,6 +1224,8 @@ int EVP_PKEY_keygen_init(EVP_PKEY_CTX *ctx);
 int EVP_PKEY_keygen(EVP_PKEY_CTX *ctx, EVP_PKEY **ppkey);
 #if defined(LIBRESSL_CRYPTO_INTERNAL) || defined(LIBRESSL_NEXT_API)
 int EVP_PKEY_check(EVP_PKEY_CTX *ctx);
+int EVP_PKEY_public_check(EVP_PKEY_CTX *ctx);
+int EVP_PKEY_param_check(EVP_PKEY_CTX *ctx);
 #endif
 
 void EVP_PKEY_CTX_set_cb(EVP_PKEY_CTX *ctx, EVP_PKEY_gen_cb *cb);
@@ -1290,6 +1296,10 @@ void EVP_PKEY_meth_set_ctrl(EVP_PKEY_METHOD *pmeth,
 #if defined(LIBRESSL_CRYPTO_INTERNAL) || defined(LIBRESSL_NEXT_API)
 void EVP_PKEY_meth_set_check(EVP_PKEY_METHOD *pmeth,
     int (*check)(EVP_PKEY *pkey));
+void EVP_PKEY_meth_set_public_check(EVP_PKEY_METHOD *pmeth,
+    int (*public_check)(EVP_PKEY *pkey));
+void EVP_PKEY_meth_set_param_check(EVP_PKEY_METHOD *pmeth,
+    int (*param_check)(EVP_PKEY *pkey));
 #endif
 
 /* Authenticated Encryption with Additional Data.
diff --git a/src/lib/libcrypto/evp/evp_locl.h b/src/lib/libcrypto/evp/evp_locl.h
index 3ff8e8ad99..44e2d5cadb 100644
--- a/src/lib/libcrypto/evp/evp_locl.h
+++ b/src/lib/libcrypto/evp/evp_locl.h
@@ -1,4 +1,4 @@
-/* $OpenBSD: evp_locl.h,v 1.19 2022/01/10 11:52:43 tb Exp $ */
+/* $OpenBSD: evp_locl.h,v 1.20 2022/01/10 12:10:26 tb Exp $ */
 /* Written by Dr Stephen N Henson (steve@openssl.org) for the OpenSSL
  * project 2000.
  */
@@ -349,6 +349,8 @@ struct evp_pkey_method_st {
         int (*ctrl_str)(EVP_PKEY_CTX *ctx, const char *type, const char *value);
 
         int (*check)(EVP_PKEY *pkey);
+        int (*public_check)(EVP_PKEY *pkey);
+        int (*param_check)(EVP_PKEY *pkey);
 } /* EVP_PKEY_METHOD */;
 
 void evp_pkey_set_cb_translate(BN_GENCB *cb, EVP_PKEY_CTX *ctx);
diff --git a/src/lib/libcrypto/evp/pmeth_gn.c b/src/lib/libcrypto/evp/pmeth_gn.c
index a8a4cc97db..7d921d23b4 100644
--- a/src/lib/libcrypto/evp/pmeth_gn.c
+++ b/src/lib/libcrypto/evp/pmeth_gn.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: pmeth_gn.c,v 1.9 2022/01/10 11:52:43 tb Exp $ */
+/* $OpenBSD: pmeth_gn.c,v 1.10 2022/01/10 12:10:26 tb Exp $ */
 /* Written by Dr Stephen N Henson (steve@openssl.org) for the OpenSSL
  * project 2006.
  */
@@ -244,3 +244,45 @@ EVP_PKEY_check(EVP_PKEY_CTX *ctx)
 
         return pkey->ameth->pkey_check(pkey);
 }
+
+int
+EVP_PKEY_public_check(EVP_PKEY_CTX *ctx)
+{
+        EVP_PKEY *pkey;
+
+        if ((pkey = ctx->pkey) == NULL) {
+                EVPerror(EVP_R_NO_KEY_SET);
+                return 0;
+        }
+
+        if (ctx->pmeth->public_check != NULL)
+                return ctx->pmeth->public_check(pkey);
+
+        if (pkey->ameth == NULL || pkey->ameth->pkey_public_check == NULL) {
+                EVPerror(EVP_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE);
+                return -2;
+        }
+
+        return pkey->ameth->pkey_public_check(pkey);
+}
+
+int
+EVP_PKEY_param_check(EVP_PKEY_CTX *ctx)
+{
+        EVP_PKEY *pkey;
+
+        if ((pkey = ctx->pkey) == NULL) {
+                EVPerror(EVP_R_NO_KEY_SET);
+                return 0;
+        }
+
+        if (ctx->pmeth->param_check != NULL)
+                return ctx->pmeth->param_check(pkey);
+
+        if (pkey->ameth == NULL || pkey->ameth->pkey_param_check == NULL) {
+                EVPerror(EVP_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE);
+                return -2;
+        }
+
+        return pkey->ameth->pkey_param_check(pkey);
+}
diff --git a/src/lib/libcrypto/evp/pmeth_lib.c b/src/lib/libcrypto/evp/pmeth_lib.c
index 92328dd246..d265e2aced 100644
--- a/src/lib/libcrypto/evp/pmeth_lib.c
+++ b/src/lib/libcrypto/evp/pmeth_lib.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: pmeth_lib.c,v 1.19 2022/01/10 11:52:43 tb Exp $ */
+/* $OpenBSD: pmeth_lib.c,v 1.20 2022/01/10 12:10:26 tb Exp $ */
 /* Written by Dr Stephen N Henson (steve@openssl.org) for the OpenSSL
  * project 2006.
  */
@@ -588,3 +588,17 @@ EVP_PKEY_meth_set_check(EVP_PKEY_METHOD *pmeth, int (*check)(EVP_PKEY *pkey))
 {
         pmeth->check = check;
 }
+
+void
+EVP_PKEY_meth_set_public_check(EVP_PKEY_METHOD *pmeth,
+    int (*public_check)(EVP_PKEY *pkey))
+{
+        pmeth->public_check = public_check;
+}
+
+void
+EVP_PKEY_meth_set_param_check(EVP_PKEY_METHOD *pmeth,
+    int (*param_check)(EVP_PKEY *pkey))
+{
+        pmeth->param_check = param_check;
+}