Prepare to provide EVP_PKEY_check()

This allows checking the validity of an EVP_PKEY. Only RSA and EC keys
are supported. If a check function is set the EVP_PKEY_METHOD, it will
be used, otherwise the check function on the EVP_PKEY_ASN1_METHOD is
used.  The default ASN.1 methods wrap RSA_check_key() and
EC_KEY_check_key(), respectively.

The corresponding setters are EVP_PKEY_{asn1,meth}_set_check().

It is unclear why the PKEY method has no const while the ASN.1 method
has const.

Requested by tobhe and used by PHP 8.1.
Based on OpenSSL commit 2aee35d3

ok inoguchi jsing

4 files changed, 47 insertions, 4 deletions

diff --git a/src/lib/libcrypto/evp/evp.h b/src/lib/libcrypto/evp/evp.h
index aa5b35f67c..e122a6b329 100644
--- a/src/lib/libcrypto/evp/evp.h
+++ b/src/lib/libcrypto/evp/evp.h
@@ -1,4 +1,4 @@
-/* $OpenBSD: evp.h,v 1.92 2022/01/09 15:15:25 tb Exp $ */
+/* $OpenBSD: evp.h,v 1.93 2022/01/10 11:52:43 tb Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -1087,6 +1087,11 @@ void EVP_PKEY_asn1_set_free(EVP_PKEY_ASN1_METHOD *ameth,
 void EVP_PKEY_asn1_set_ctrl(EVP_PKEY_ASN1_METHOD *ameth,
     int (*pkey_ctrl)(EVP_PKEY *pkey, int op, long arg1, void *arg2));
 
+#if defined(LIBRESSL_CRYPTO_INTERNAL) || defined(LIBRESSL_NEXT_API)
+void EVP_PKEY_asn1_set_check(EVP_PKEY_ASN1_METHOD *ameth,
+    int (*pkey_check)(const EVP_PKEY *pk));
+#endif
+
 #define EVP_PKEY_OP_UNDEFINED           0
 #define EVP_PKEY_OP_PARAMGEN            (1<<1)
 #define EVP_PKEY_OP_KEYGEN              (1<<2)
@@ -1213,6 +1218,9 @@ int EVP_PKEY_paramgen_init(EVP_PKEY_CTX *ctx);
 int EVP_PKEY_paramgen(EVP_PKEY_CTX *ctx, EVP_PKEY **ppkey);
 int EVP_PKEY_keygen_init(EVP_PKEY_CTX *ctx);
 int EVP_PKEY_keygen(EVP_PKEY_CTX *ctx, EVP_PKEY **ppkey);
+#if defined(LIBRESSL_CRYPTO_INTERNAL) || defined(LIBRESSL_NEXT_API)
+int EVP_PKEY_check(EVP_PKEY_CTX *ctx);
+#endif
 
 void EVP_PKEY_CTX_set_cb(EVP_PKEY_CTX *ctx, EVP_PKEY_gen_cb *cb);
 EVP_PKEY_gen_cb *EVP_PKEY_CTX_get_cb(EVP_PKEY_CTX *ctx);
@@ -1279,6 +1287,11 @@ void EVP_PKEY_meth_set_ctrl(EVP_PKEY_METHOD *pmeth,
     int (*ctrl)(EVP_PKEY_CTX *ctx, int type, int p1, void *p2),
     int (*ctrl_str)(EVP_PKEY_CTX *ctx, const char *type, const char *value));
 
+#if defined(LIBRESSL_CRYPTO_INTERNAL) || defined(LIBRESSL_NEXT_API)
+void EVP_PKEY_meth_set_check(EVP_PKEY_METHOD *pmeth,
+    int (*check)(EVP_PKEY *pkey));
+#endif
+
 /* Authenticated Encryption with Additional Data.
  *
  * AEAD couples confidentiality and integrity in a single primtive. AEAD
diff --git a/src/lib/libcrypto/evp/evp_locl.h b/src/lib/libcrypto/evp/evp_locl.h
index 5eef0b244f..3ff8e8ad99 100644
--- a/src/lib/libcrypto/evp/evp_locl.h
+++ b/src/lib/libcrypto/evp/evp_locl.h
@@ -1,4 +1,4 @@
-/* $OpenBSD: evp_locl.h,v 1.18 2022/01/09 15:15:25 tb Exp $ */
+/* $OpenBSD: evp_locl.h,v 1.19 2022/01/10 11:52:43 tb Exp $ */
 /* Written by Dr Stephen N Henson (steve@openssl.org) for the OpenSSL
  * project 2000.
  */
@@ -347,6 +347,8 @@ struct evp_pkey_method_st {
 
         int (*ctrl)(EVP_PKEY_CTX *ctx, int type, int p1, void *p2);
         int (*ctrl_str)(EVP_PKEY_CTX *ctx, const char *type, const char *value);
+
+        int (*check)(EVP_PKEY *pkey);
 } /* EVP_PKEY_METHOD */;
 
 void evp_pkey_set_cb_translate(BN_GENCB *cb, EVP_PKEY_CTX *ctx);
diff --git a/src/lib/libcrypto/evp/pmeth_gn.c b/src/lib/libcrypto/evp/pmeth_gn.c
index 066291b800..a8a4cc97db 100644
--- a/src/lib/libcrypto/evp/pmeth_gn.c
+++ b/src/lib/libcrypto/evp/pmeth_gn.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: pmeth_gn.c,v 1.8 2021/12/04 16:08:32 tb Exp $ */
+/* $OpenBSD: pmeth_gn.c,v 1.9 2022/01/10 11:52:43 tb Exp $ */
 /* Written by Dr Stephen N Henson (steve@openssl.org) for the OpenSSL
  * project 2006.
  */
@@ -64,6 +64,7 @@
 #include <openssl/evp.h>
 #include <openssl/objects.h>
 
+#include "asn1_locl.h"
 #include "bn_lcl.h"
 #include "evp_locl.h"
 
@@ -222,3 +223,24 @@ merr:
         EVP_PKEY_CTX_free(mac_ctx);
         return mac_key;
 }
+
+int
+EVP_PKEY_check(EVP_PKEY_CTX *ctx)
+{
+        EVP_PKEY *pkey;
+
+        if ((pkey = ctx->pkey) == NULL) {
+                EVPerror(EVP_R_NO_KEY_SET);
+                return 0;
+        }
+
+        if (ctx->pmeth->check != NULL)
+                return ctx->pmeth->check(pkey);
+
+        if (pkey->ameth == NULL || pkey->ameth->pkey_check == NULL) {
+                EVPerror(EVP_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE);
+                return -2;
+        }
+
+        return pkey->ameth->pkey_check(pkey);
+}
diff --git a/src/lib/libcrypto/evp/pmeth_lib.c b/src/lib/libcrypto/evp/pmeth_lib.c
index 33924dbd66..92328dd246 100644
--- a/src/lib/libcrypto/evp/pmeth_lib.c
+++ b/src/lib/libcrypto/evp/pmeth_lib.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: pmeth_lib.c,v 1.18 2021/12/03 14:19:57 tb Exp $ */
+/* $OpenBSD: pmeth_lib.c,v 1.19 2022/01/10 11:52:43 tb Exp $ */
 /* Written by Dr Stephen N Henson (steve@openssl.org) for the OpenSSL
  * project 2006.
  */
@@ -582,3 +582,9 @@ EVP_PKEY_meth_set_ctrl(EVP_PKEY_METHOD *pmeth,
         pmeth->ctrl = ctrl;
         pmeth->ctrl_str = ctrl_str;
 }
+
+void
+EVP_PKEY_meth_set_check(EVP_PKEY_METHOD *pmeth, int (*check)(EVP_PKEY *pkey))
+{
+        pmeth->check = check;
+}