backport checks for degenerate Diffie-Hellman public exponents from

OpenSSL-0.9.8a, where they were added without a corresponding patch to
0.9.7 or an advisory!  ok theo@ markus@

9 files changed, 73 insertions, 1 deletions

diff --git a/src/lib/libcrypto/dh/dh.h b/src/lib/libcrypto/dh/dh.h
index d51dc130f4..0aff7fe21f 100644
--- a/src/lib/libcrypto/dh/dh.h
+++ b/src/lib/libcrypto/dh/dh.h
@@ -130,6 +130,10 @@ struct dh_st
 #define DH_UNABLE_TO_CHECK_GENERATOR    0x04
 #define DH_NOT_SUITABLE_GENERATOR       0x08
 
+/* DH_check_pub_key error codes */
+#define DH_CHECK_PUBKEY_TOO_SMALL       0x01
+#define DH_CHECK_PUBKEY_TOO_LARGE       0x02
+
 /* primes p where (p-1)/2 is prime too are called "safe"; we define
    this for backward compatibility: */
 #define DH_CHECK_P_NOT_STRONG_PRIME     DH_CHECK_P_NOT_SAFE_PRIME
@@ -168,6 +172,7 @@ void *DH_get_ex_data(DH *d, int idx);
 DH *    DH_generate_parameters(int prime_len,int generator,
                 void (*callback)(int,int,void *),void *cb_arg);
 int     DH_check(const DH *dh,int *codes);
+int     DH_check_pub_key(const DH *dh,const BIGNUM *pub_key, int *codes);
 int     DH_generate_key(DH *dh);
 int     DH_compute_key(unsigned char *key,const BIGNUM *pub_key,DH *dh);
 DH *    d2i_DHparams(DH **a,const unsigned char **pp, long length);
@@ -200,6 +205,7 @@ void ERR_load_DH_strings(void);
 /* Reason codes. */
 #define DH_R_BAD_GENERATOR                               101
 #define DH_R_NO_PRIVATE_VALUE                            100
+#define DH_R_INVALID_PUBKEY                              102
 
 #ifdef  __cplusplus
 }
diff --git a/src/lib/libcrypto/dh/dh_check.c b/src/lib/libcrypto/dh/dh_check.c
index a7e9920efb..17debff62d 100644
--- a/src/lib/libcrypto/dh/dh_check.c
+++ b/src/lib/libcrypto/dh/dh_check.c
@@ -121,4 +121,26 @@ err:
         return(ok);
         }
 
+int DH_check_pub_key(const DH *dh, const BIGNUM *pub_key, int *ret)
+        {
+        int ok=0;
+        BIGNUM *q=NULL;
+
+        *ret=0;
+        q=BN_new();
+        if (q == NULL) goto err;
+        BN_set_word(q,1);
+        if (BN_cmp(pub_key,q) <= 0)
+                *ret|=DH_CHECK_PUBKEY_TOO_SMALL;
+        BN_copy(q,dh->p);
+        BN_sub_word(q,1);
+        if (BN_cmp(pub_key,q) >= 0)
+                *ret|=DH_CHECK_PUBKEY_TOO_LARGE;
+
+        ok = 1;
+err:
+        if (q != NULL) BN_free(q);
+        return(ok);
+        }
+
 #endif
diff --git a/src/lib/libcrypto/dh/dh_err.c b/src/lib/libcrypto/dh/dh_err.c
index c2715044c9..914b8a9c53 100644
--- a/src/lib/libcrypto/dh/dh_err.c
+++ b/src/lib/libcrypto/dh/dh_err.c
@@ -79,6 +79,7 @@ static ERR_STRING_DATA DH_str_reasons[]=
         {
 {DH_R_BAD_GENERATOR                      ,"bad generator"},
 {DH_R_NO_PRIVATE_VALUE                   ,"no private value"},
+{DH_R_INVALID_PUBKEY                     ,"invalid public key"},
 {0,NULL}
         };
 
diff --git a/src/lib/libcrypto/dh/dh_key.c b/src/lib/libcrypto/dh/dh_key.c
index ff125c2296..648766a6ec 100644
--- a/src/lib/libcrypto/dh/dh_key.c
+++ b/src/lib/libcrypto/dh/dh_key.c
@@ -163,6 +163,7 @@ static int compute_key(unsigned char *key, const BIGNUM *pub_key, DH *dh)
         BN_MONT_CTX *mont;
         BIGNUM *tmp;
         int ret= -1;
+        int check_result;
 
         ctx = BN_CTX_new();
         if (ctx == NULL) goto err;
@@ -182,6 +183,12 @@ static int compute_key(unsigned char *key, const BIGNUM *pub_key, DH *dh)
                 }
 
         mont=(BN_MONT_CTX *)dh->method_mont_p;
+
+        if (!DH_check_pub_key(dh, pub_key, &check_result) || check_result)
+                {
+                DHerr(DH_F_DH_COMPUTE_KEY,DH_R_INVALID_PUBKEY);
+                goto err;
+                }
         if (!dh->meth->bn_mod_exp(dh, tmp, pub_key, dh->priv_key,dh->p,ctx,mont))
                 {
                 DHerr(DH_F_DH_COMPUTE_KEY,ERR_R_BN_LIB);
diff --git a/src/lib/libssl/crypto/shlib_version b/src/lib/libssl/crypto/shlib_version
index 56246d02b2..eb2c603aec 100644
--- a/src/lib/libssl/crypto/shlib_version
+++ b/src/lib/libssl/crypto/shlib_version
@@ -1,2 +1,2 @@
 major=12
-minor=0
+minor=1
diff --git a/src/lib/libssl/src/crypto/dh/dh.h b/src/lib/libssl/src/crypto/dh/dh.h
index d51dc130f4..0aff7fe21f 100644
--- a/src/lib/libssl/src/crypto/dh/dh.h
+++ b/src/lib/libssl/src/crypto/dh/dh.h
@@ -130,6 +130,10 @@ struct dh_st
 #define DH_UNABLE_TO_CHECK_GENERATOR    0x04
 #define DH_NOT_SUITABLE_GENERATOR       0x08
 
+/* DH_check_pub_key error codes */
+#define DH_CHECK_PUBKEY_TOO_SMALL       0x01
+#define DH_CHECK_PUBKEY_TOO_LARGE       0x02
+
 /* primes p where (p-1)/2 is prime too are called "safe"; we define
    this for backward compatibility: */
 #define DH_CHECK_P_NOT_STRONG_PRIME     DH_CHECK_P_NOT_SAFE_PRIME
@@ -168,6 +172,7 @@ void *DH_get_ex_data(DH *d, int idx);
 DH *    DH_generate_parameters(int prime_len,int generator,
                 void (*callback)(int,int,void *),void *cb_arg);
 int     DH_check(const DH *dh,int *codes);
+int     DH_check_pub_key(const DH *dh,const BIGNUM *pub_key, int *codes);
 int     DH_generate_key(DH *dh);
 int     DH_compute_key(unsigned char *key,const BIGNUM *pub_key,DH *dh);
 DH *    d2i_DHparams(DH **a,const unsigned char **pp, long length);
@@ -200,6 +205,7 @@ void ERR_load_DH_strings(void);
 /* Reason codes. */
 #define DH_R_BAD_GENERATOR                               101
 #define DH_R_NO_PRIVATE_VALUE                            100
+#define DH_R_INVALID_PUBKEY                              102
 
 #ifdef  __cplusplus
 }
diff --git a/src/lib/libssl/src/crypto/dh/dh_check.c b/src/lib/libssl/src/crypto/dh/dh_check.c
index a7e9920efb..17debff62d 100644
--- a/src/lib/libssl/src/crypto/dh/dh_check.c
+++ b/src/lib/libssl/src/crypto/dh/dh_check.c
@@ -121,4 +121,26 @@ err:
         return(ok);
         }
 
+int DH_check_pub_key(const DH *dh, const BIGNUM *pub_key, int *ret)
+        {
+        int ok=0;
+        BIGNUM *q=NULL;
+
+        *ret=0;
+        q=BN_new();
+        if (q == NULL) goto err;
+        BN_set_word(q,1);
+        if (BN_cmp(pub_key,q) <= 0)
+                *ret|=DH_CHECK_PUBKEY_TOO_SMALL;
+        BN_copy(q,dh->p);
+        BN_sub_word(q,1);
+        if (BN_cmp(pub_key,q) >= 0)
+                *ret|=DH_CHECK_PUBKEY_TOO_LARGE;
+
+        ok = 1;
+err:
+        if (q != NULL) BN_free(q);
+        return(ok);
+        }
+
 #endif
diff --git a/src/lib/libssl/src/crypto/dh/dh_err.c b/src/lib/libssl/src/crypto/dh/dh_err.c
index c2715044c9..914b8a9c53 100644
--- a/src/lib/libssl/src/crypto/dh/dh_err.c
+++ b/src/lib/libssl/src/crypto/dh/dh_err.c
@@ -79,6 +79,7 @@ static ERR_STRING_DATA DH_str_reasons[]=
         {
 {DH_R_BAD_GENERATOR                      ,"bad generator"},
 {DH_R_NO_PRIVATE_VALUE                   ,"no private value"},
+{DH_R_INVALID_PUBKEY                     ,"invalid public key"},
 {0,NULL}
         };
 
diff --git a/src/lib/libssl/src/crypto/dh/dh_key.c b/src/lib/libssl/src/crypto/dh/dh_key.c
index ff125c2296..648766a6ec 100644
--- a/src/lib/libssl/src/crypto/dh/dh_key.c
+++ b/src/lib/libssl/src/crypto/dh/dh_key.c
@@ -163,6 +163,7 @@ static int compute_key(unsigned char *key, const BIGNUM *pub_key, DH *dh)
         BN_MONT_CTX *mont;
         BIGNUM *tmp;
         int ret= -1;
+        int check_result;
 
         ctx = BN_CTX_new();
         if (ctx == NULL) goto err;
@@ -182,6 +183,12 @@ static int compute_key(unsigned char *key, const BIGNUM *pub_key, DH *dh)
                 }
 
         mont=(BN_MONT_CTX *)dh->method_mont_p;
+
+        if (!DH_check_pub_key(dh, pub_key, &check_result) || check_result)
+                {
+                DHerr(DH_F_DH_COMPUTE_KEY,DH_R_INVALID_PUBKEY);
+                goto err;
+                }
         if (!dh->meth->bn_mod_exp(dh, tmp, pub_key, dh->priv_key,dh->p,ctx,mont))
                 {
                 DHerr(DH_F_DH_COMPUTE_KEY,ERR_R_BN_LIB);