openssl security fixes, diff from markus@, ok & "commit it" djm@

http://www.openssl.org/news/secadv_20060928.txt for more

3 files changed, 51 insertions, 0 deletions

diff --git a/src/lib/libcrypto/rsa/rsa.h b/src/lib/libcrypto/rsa/rsa.h
index 0b639cd37f..dbed701e89 100644
--- a/src/lib/libcrypto/rsa/rsa.h
+++ b/src/lib/libcrypto/rsa/rsa.h
@@ -154,6 +154,11 @@ struct rsa_st
         BN_BLINDING *blinding;
         };
 
+#define OPENSSL_RSA_MAX_MODULUS_BITS    16384
+
+#define OPENSSL_RSA_SMALL_MODULUS_BITS  3072
+#define OPENSSL_RSA_MAX_PUBEXP_BITS     64 /* exponent limit enforced for "small" modulus only */
+
 #define RSA_3   0x3L
 #define RSA_F4  0x10001L
 
@@ -386,6 +391,7 @@ void ERR_load_RSA_strings(void);
 #define RSA_R_IQMP_NOT_INVERSE_OF_Q                      126
 #define RSA_R_KEY_SIZE_TOO_SMALL                         120
 #define RSA_R_LAST_OCTET_INVALID                         134
+#define RSA_R_MODULUS_TOO_LARGE                          105
 #define RSA_R_NULL_BEFORE_BLOCK_MISSING                  113
 #define RSA_R_N_DOES_NOT_EQUAL_P_Q                       127
 #define RSA_R_OAEP_DECODING_ERROR                        121
diff --git a/src/lib/libcrypto/rsa/rsa_eay.c b/src/lib/libcrypto/rsa/rsa_eay.c
index be4ac96ce3..610889dc80 100644
--- a/src/lib/libcrypto/rsa/rsa_eay.c
+++ b/src/lib/libcrypto/rsa/rsa_eay.c
@@ -295,6 +295,28 @@ static int RSA_eay_private_encrypt(int flen, const unsigned char *from,
         BN_init(&f);
         BN_init(&ret);
 
+        if (BN_num_bits(rsa->n) > OPENSSL_RSA_MAX_MODULUS_BITS)
+                {
+                RSAerr(RSA_F_RSA_EAY_PUBLIC_ENCRYPT, RSA_R_MODULUS_TOO_LARGE);
+                return -1;
+                }
+
+        if (BN_ucmp(rsa->n, rsa->e) <= 0)
+                {
+                RSAerr(RSA_F_RSA_EAY_PUBLIC_ENCRYPT, RSA_R_BAD_E_VALUE);
+                return -1;
+                }
+
+        /* for large moduli, enforce exponent limit */
+        if (BN_num_bits(rsa->n) > OPENSSL_RSA_SMALL_MODULUS_BITS)
+                {
+                if (BN_num_bits(rsa->e) > OPENSSL_RSA_MAX_PUBEXP_BITS)
+                        {
+                        RSAerr(RSA_F_RSA_EAY_PUBLIC_ENCRYPT, RSA_R_BAD_E_VALUE);
+                        return -1;
+                        }
+                }
+
         if ((ctx=BN_CTX_new()) == NULL) goto err;
         num=BN_num_bytes(rsa->n);
         if ((buf=(unsigned char *)OPENSSL_malloc(num)) == NULL)
@@ -576,6 +598,28 @@ static int RSA_eay_public_decrypt(int flen, const unsigned char *from,
         unsigned char *buf=NULL;
         BN_CTX *ctx=NULL;
 
+        if (BN_num_bits(rsa->n) > OPENSSL_RSA_MAX_MODULUS_BITS)
+                {
+                RSAerr(RSA_F_RSA_EAY_PUBLIC_DECRYPT, RSA_R_MODULUS_TOO_LARGE);
+                return -1;
+                }
+
+        if (BN_ucmp(rsa->n, rsa->e) <= 0)
+                {
+                RSAerr(RSA_F_RSA_EAY_PUBLIC_DECRYPT, RSA_R_BAD_E_VALUE);
+                return -1;
+                }
+
+        /* for large moduli, enforce exponent limit */
+        if (BN_num_bits(rsa->n) > OPENSSL_RSA_SMALL_MODULUS_BITS)
+                {
+                if (BN_num_bits(rsa->e) > OPENSSL_RSA_MAX_PUBEXP_BITS)
+                        {
+                        RSAerr(RSA_F_RSA_EAY_PUBLIC_DECRYPT, RSA_R_BAD_E_VALUE);
+                        return -1;
+                        }
+                }
+
         BN_init(&f);
         BN_init(&ret);
         ctx=BN_CTX_new();
diff --git a/src/lib/libcrypto/rsa/rsa_err.c b/src/lib/libcrypto/rsa/rsa_err.c
index 2ec4b30ff7..ddcb28e663 100644
--- a/src/lib/libcrypto/rsa/rsa_err.c
+++ b/src/lib/libcrypto/rsa/rsa_err.c
@@ -129,6 +129,7 @@ static ERR_STRING_DATA RSA_str_reasons[]=
 {ERR_REASON(RSA_R_IQMP_NOT_INVERSE_OF_Q) ,"iqmp not inverse of q"},
 {ERR_REASON(RSA_R_KEY_SIZE_TOO_SMALL)    ,"key size too small"},
 {ERR_REASON(RSA_R_LAST_OCTET_INVALID)    ,"last octet invalid"},
+{ERR_REASON(RSA_R_MODULUS_TOO_LARGE)     ,"modulus too large"},
 {ERR_REASON(RSA_R_NULL_BEFORE_BLOCK_MISSING),"null before block missing"},
 {ERR_REASON(RSA_R_N_DOES_NOT_EQUAL_P_Q)  ,"n does not equal p q"},
 {ERR_REASON(RSA_R_OAEP_DECODING_ERROR)   ,"oaep decoding error"},