Make RSA, RSA_PSS_PARAMS and RSA_METHOD opaque

Move the struct internals to rsa_locl.h and provide a missing
typedef in ossl_typ.h.

ok inoguchi jsing

3 files changed, 80 insertions, 83 deletions

diff --git a/src/lib/libcrypto/ossl_typ.h b/src/lib/libcrypto/ossl_typ.h
index 161bf86a3e..4f94c4032e 100644
--- a/src/lib/libcrypto/ossl_typ.h
+++ b/src/lib/libcrypto/ossl_typ.h
@@ -1,4 +1,4 @@
-/* $OpenBSD: ossl_typ.h,v 1.19 2022/01/14 08:21:12 tb Exp $ */
+/* $OpenBSD: ossl_typ.h,v 1.20 2022/01/14 08:34:39 tb Exp $ */
 /* ====================================================================
  * Copyright (c) 1998-2001 The OpenSSL Project.  All rights reserved.
  *
@@ -121,6 +121,7 @@ typedef struct dsa_method DSA_METHOD;
 
 typedef struct rsa_st RSA;
 typedef struct rsa_meth_st RSA_METHOD;
+typedef struct rsa_pss_params_st RSA_PSS_PARAMS;
 
 typedef struct rand_meth_st RAND_METHOD;
 
diff --git a/src/lib/libcrypto/rsa/rsa.h b/src/lib/libcrypto/rsa/rsa.h
index e01e6ba553..d59fd03f76 100644
--- a/src/lib/libcrypto/rsa/rsa.h
+++ b/src/lib/libcrypto/rsa/rsa.h
@@ -1,4 +1,4 @@
-/* $OpenBSD: rsa.h,v 1.54 2022/01/14 08:12:31 tb Exp $ */
+/* $OpenBSD: rsa.h,v 1.55 2022/01/14 08:34:39 tb Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -80,11 +80,7 @@
 extern "C" {
 #endif
 
-/* Declared already in ossl_typ.h */
-/* typedef struct rsa_st RSA; */
-/* typedef struct rsa_meth_st RSA_METHOD; */
-
-typedef struct rsa_pss_params_st {
+struct rsa_pss_params_st {
         X509_ALGOR *hashAlgorithm;
         X509_ALGOR *maskGenAlgorithm;
         ASN1_INTEGER *saltLength;
@@ -92,7 +88,7 @@ typedef struct rsa_pss_params_st {
 
         /* Hash algorithm decoded from maskGenAlgorithm. */
         X509_ALGOR *maskHash;
-} RSA_PSS_PARAMS;
+} /* RSA_PSS_PARAMS */;
 
 typedef struct rsa_oaep_params_st {
         X509_ALGOR *hashFunc;
@@ -103,80 +99,6 @@ typedef struct rsa_oaep_params_st {
         X509_ALGOR *maskHash;
 } RSA_OAEP_PARAMS;
 
-struct rsa_meth_st {
-        const char *name;
-        int (*rsa_pub_enc)(int flen, const unsigned char *from,
-            unsigned char *to, RSA *rsa, int padding);
-        int (*rsa_pub_dec)(int flen, const unsigned char *from,
-            unsigned char *to, RSA *rsa, int padding);
-        int (*rsa_priv_enc)(int flen, const unsigned char *from,
-            unsigned char *to, RSA *rsa, int padding);
-        int (*rsa_priv_dec)(int flen, const unsigned char *from,
-            unsigned char *to, RSA *rsa, int padding);
-        int (*rsa_mod_exp)(BIGNUM *r0, const BIGNUM *I, RSA *rsa,
-            BN_CTX *ctx); /* Can be null */
-        int (*bn_mod_exp)(BIGNUM *r, const BIGNUM *a, const BIGNUM *p,
-            const BIGNUM *m, BN_CTX *ctx, BN_MONT_CTX *m_ctx); /* Can be null */
-        int (*init)(RSA *rsa);          /* called at new */
-        int (*finish)(RSA *rsa);        /* called at free */
-        int flags;                      /* RSA_METHOD_FLAG_* things */
-        char *app_data;                 /* may be needed! */
-/* New sign and verify functions: some libraries don't allow arbitrary data
- * to be signed/verified: this allows them to be used. Note: for this to work
- * the RSA_public_decrypt() and RSA_private_encrypt() should *NOT* be used
- * RSA_sign(), RSA_verify() should be used instead. Note: for backwards
- * compatibility this functionality is only enabled if the RSA_FLAG_SIGN_VER
- * option is set in 'flags'.
- */
-        int (*rsa_sign)(int type, const unsigned char *m, unsigned int m_length,
-            unsigned char *sigret, unsigned int *siglen, const RSA *rsa);
-        int (*rsa_verify)(int dtype, const unsigned char *m,
-            unsigned int m_length, const unsigned char *sigbuf,
-            unsigned int siglen, const RSA *rsa);
-/* If this callback is NULL, the builtin software RSA key-gen will be used. This
- * is for behavioural compatibility whilst the code gets rewired, but one day
- * it would be nice to assume there are no such things as "builtin software"
- * implementations. */
-        int (*rsa_keygen)(RSA *rsa, int bits, BIGNUM *e, BN_GENCB *cb);
-};
-
-struct rsa_st {
-        /* The first parameter is used to pickup errors where
-         * this is passed instead of aEVP_PKEY, it is set to 0 */
-        int pad;
-        long version;
-        const RSA_METHOD *meth;
-
-        /* functional reference if 'meth' is ENGINE-provided */
-        ENGINE *engine;
-        BIGNUM *n;
-        BIGNUM *e;
-        BIGNUM *d;
-        BIGNUM *p;
-        BIGNUM *q;
-        BIGNUM *dmp1;
-        BIGNUM *dmq1;
-        BIGNUM *iqmp;
-
-        /* Parameter restrictions for PSS only keys. */
-        RSA_PSS_PARAMS *pss;
-
-        /* be careful using this if the RSA structure is shared */
-        CRYPTO_EX_DATA ex_data;
-        int references;
-        int flags;
-
-        /* Used to cache montgomery values */
-        BN_MONT_CTX *_method_mod_n;
-        BN_MONT_CTX *_method_mod_p;
-        BN_MONT_CTX *_method_mod_q;
-
-        /* all BIGNUM values are actually in the following data, if it is not
-         * NULL */
-        BN_BLINDING *blinding;
-        BN_BLINDING *mt_blinding;
-};
-
 #ifndef OPENSSL_RSA_MAX_MODULUS_BITS
 # define OPENSSL_RSA_MAX_MODULUS_BITS   16384
 #endif
diff --git a/src/lib/libcrypto/rsa/rsa_locl.h b/src/lib/libcrypto/rsa/rsa_locl.h
index 7036449c36..9eae2b3a24 100644
--- a/src/lib/libcrypto/rsa/rsa_locl.h
+++ b/src/lib/libcrypto/rsa/rsa_locl.h
@@ -1,4 +1,4 @@
-/* $OpenBSD: rsa_locl.h,v 1.11 2019/11/02 13:47:41 jsing Exp $ */
+/* $OpenBSD: rsa_locl.h,v 1.12 2022/01/14 08:34:39 tb Exp $ */
 
 __BEGIN_HIDDEN_DECLS
 
@@ -8,6 +8,80 @@ __BEGIN_HIDDEN_DECLS
 #define pkey_is_pss(pkey) (pkey->ameth->pkey_id == EVP_PKEY_RSA_PSS)
 #define pkey_ctx_is_pss(ctx) (ctx->pmeth->pkey_id == EVP_PKEY_RSA_PSS)
 
+struct rsa_meth_st {
+        const char *name;
+        int (*rsa_pub_enc)(int flen, const unsigned char *from,
+            unsigned char *to, RSA *rsa, int padding);
+        int (*rsa_pub_dec)(int flen, const unsigned char *from,
+            unsigned char *to, RSA *rsa, int padding);
+        int (*rsa_priv_enc)(int flen, const unsigned char *from,
+            unsigned char *to, RSA *rsa, int padding);
+        int (*rsa_priv_dec)(int flen, const unsigned char *from,
+            unsigned char *to, RSA *rsa, int padding);
+        int (*rsa_mod_exp)(BIGNUM *r0, const BIGNUM *I, RSA *rsa,
+            BN_CTX *ctx); /* Can be null */
+        int (*bn_mod_exp)(BIGNUM *r, const BIGNUM *a, const BIGNUM *p,
+            const BIGNUM *m, BN_CTX *ctx, BN_MONT_CTX *m_ctx); /* Can be null */
+        int (*init)(RSA *rsa);          /* called at new */
+        int (*finish)(RSA *rsa);        /* called at free */
+        int flags;                      /* RSA_METHOD_FLAG_* things */
+        char *app_data;                 /* may be needed! */
+/* New sign and verify functions: some libraries don't allow arbitrary data
+ * to be signed/verified: this allows them to be used. Note: for this to work
+ * the RSA_public_decrypt() and RSA_private_encrypt() should *NOT* be used
+ * RSA_sign(), RSA_verify() should be used instead. Note: for backwards
+ * compatibility this functionality is only enabled if the RSA_FLAG_SIGN_VER
+ * option is set in 'flags'.
+ */
+        int (*rsa_sign)(int type, const unsigned char *m, unsigned int m_length,
+            unsigned char *sigret, unsigned int *siglen, const RSA *rsa);
+        int (*rsa_verify)(int dtype, const unsigned char *m,
+            unsigned int m_length, const unsigned char *sigbuf,
+            unsigned int siglen, const RSA *rsa);
+/* If this callback is NULL, the builtin software RSA key-gen will be used. This
+ * is for behavioural compatibility whilst the code gets rewired, but one day
+ * it would be nice to assume there are no such things as "builtin software"
+ * implementations. */
+        int (*rsa_keygen)(RSA *rsa, int bits, BIGNUM *e, BN_GENCB *cb);
+};
+
+struct rsa_st {
+        /* The first parameter is used to pickup errors where
+         * this is passed instead of aEVP_PKEY, it is set to 0 */
+        int pad;
+        long version;
+        const RSA_METHOD *meth;
+
+        /* functional reference if 'meth' is ENGINE-provided */
+        ENGINE *engine;
+        BIGNUM *n;
+        BIGNUM *e;
+        BIGNUM *d;
+        BIGNUM *p;
+        BIGNUM *q;
+        BIGNUM *dmp1;
+        BIGNUM *dmq1;
+        BIGNUM *iqmp;
+
+        /* Parameter restrictions for PSS only keys. */
+        RSA_PSS_PARAMS *pss;
+
+        /* be careful using this if the RSA structure is shared */
+        CRYPTO_EX_DATA ex_data;
+        int references;
+        int flags;
+
+        /* Used to cache montgomery values */
+        BN_MONT_CTX *_method_mod_n;
+        BN_MONT_CTX *_method_mod_p;
+        BN_MONT_CTX *_method_mod_q;
+
+        /* all BIGNUM values are actually in the following data, if it is not
+         * NULL */
+        BN_BLINDING *blinding;
+        BN_BLINDING *mt_blinding;
+};
+
 RSA_PSS_PARAMS *rsa_pss_params_create(const EVP_MD *sigmd, const EVP_MD *mgf1md,
     int saltlen);
 int rsa_pss_get_param(const RSA_PSS_PARAMS *pss, const EVP_MD **pmd,