Add bounds checking for BN_hex2bn/BN_dec2bn.

Need to make sure i * 4 won't overflow.  Based on OpenSSL:

commit 99ba9fd02fd481eb971023a3a0a251a37eb87e4c

input + ok bcook@
ok beck@

4 files changed, 52 insertions, 20 deletions

diff --git a/src/lib/libcrypto/bn/bn.h b/src/lib/libcrypto/bn/bn.h
index 2c648ba2ee..5efccd180b 100644
--- a/src/lib/libcrypto/bn/bn.h
+++ b/src/lib/libcrypto/bn/bn.h
@@ -1,4 +1,4 @@
-/* $OpenBSD: bn.h,v 1.28 2015/10/21 19:02:22 miod Exp $ */
+/* $OpenBSD: bn.h,v 1.29 2016/03/02 06:16:11 doug Exp $ */
 /* Copyright (C) 1995-1997 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -125,6 +125,7 @@
 #ifndef HEADER_BN_H
 #define HEADER_BN_H
 
+#include <limits.h>
 #include <stdio.h>
 #include <stdlib.h>
 
@@ -619,10 +620,20 @@ const BIGNUM *BN_get0_nist_prime_521(void);
 
 /* library internal functions */
 
-#define bn_expand(a,bits) ((((((bits+BN_BITS2-1))/BN_BITS2)) <= (a)->dmax)?\
-        (a):bn_expand2((a),(bits+BN_BITS2-1)/BN_BITS2))
 #define bn_wexpand(a,words) (((words) <= (a)->dmax)?(a):bn_expand2((a),(words)))
 BIGNUM *bn_expand2(BIGNUM *a, int words);
+
+static inline BIGNUM *bn_expand(BIGNUM *a, int bits)
+{
+        if (bits > (INT_MAX - BN_BITS2 + 1))
+                return (NULL);
+
+        if (((bits + BN_BITS2 - 1) / BN_BITS2) <= a->dmax)
+                return (a);
+
+        return bn_expand2(a, (bits + BN_BITS2 - 1) / BN_BITS2);
+}
+
 #ifndef OPENSSL_NO_DEPRECATED
 BIGNUM *bn_dup_expand(const BIGNUM *a, int words); /* unused */
 #endif
diff --git a/src/lib/libcrypto/bn/bn_print.c b/src/lib/libcrypto/bn/bn_print.c
index 1614a11449..2c1681a2c0 100644
--- a/src/lib/libcrypto/bn/bn_print.c
+++ b/src/lib/libcrypto/bn/bn_print.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: bn_print.c,v 1.28 2015/09/28 18:58:33 deraadt Exp $ */
+/* $OpenBSD: bn_print.c,v 1.29 2016/03/02 06:16:11 doug Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -57,6 +57,7 @@
  */
 
 #include <ctype.h>
+#include <limits.h>
 #include <stdio.h>
 
 #include <openssl/opensslconf.h>
@@ -119,7 +120,7 @@ BN_bn2dec(const BIGNUM *a)
                 if (buf == NULL) {
                         BNerr(BN_F_BN_BN2DEC, ERR_R_MALLOC_FAILURE);
                         goto err;
-                }               
+                }
                 p = buf;
                 if (BN_is_negative(a))
                         *p++ = '-';
@@ -127,7 +128,7 @@ BN_bn2dec(const BIGNUM *a)
                 *p++ = '\0';
                 return (buf);
         }
-                
+
         /* get an upper bound for the length of the decimal integer
          * num <= (BN_num_bits(a) + 1) * log(2)
          *     <= 3 * BN_num_bits(a) * 0.1001 + log(2) + 1     (rounding error)
@@ -197,8 +198,10 @@ BN_hex2bn(BIGNUM **bn, const char *a)
                 a++;
         }
 
-        for (i = 0; isxdigit((unsigned char)a[i]); i++)
+        for (i = 0; i <= (INT_MAX / 4) && isxdigit((unsigned char)a[i]); i++)
                 ;
+        if (i > INT_MAX / 4)
+                goto err;
 
         num = i + neg;
         if (bn == NULL)
@@ -213,7 +216,7 @@ BN_hex2bn(BIGNUM **bn, const char *a)
                 BN_zero(ret);
         }
 
-        /* i is the number of hex digests; */
+        /* i is the number of hex digits */
         if (bn_expand(ret, i * 4) == NULL)
                 goto err;
 
@@ -271,8 +274,10 @@ BN_dec2bn(BIGNUM **bn, const char *a)
                 a++;
         }
 
-        for (i = 0; isdigit((unsigned char)a[i]); i++)
+        for (i = 0; i <= (INT_MAX / 4) && isdigit((unsigned char)a[i]); i++)
                 ;
+        if (i > INT_MAX / 4)
+                goto err;
 
         num = i + neg;
         if (bn == NULL)
@@ -288,7 +293,7 @@ BN_dec2bn(BIGNUM **bn, const char *a)
                 BN_zero(ret);
         }
 
-        /* i is the number of digests, a bit of an over expand; */
+        /* i is the number of digits, a bit of an over expand */
         if (bn_expand(ret, i * 4) == NULL)
                 goto err;
 
diff --git a/src/lib/libssl/src/crypto/bn/bn.h b/src/lib/libssl/src/crypto/bn/bn.h
index 2c648ba2ee..5efccd180b 100644
--- a/src/lib/libssl/src/crypto/bn/bn.h
+++ b/src/lib/libssl/src/crypto/bn/bn.h
@@ -1,4 +1,4 @@
-/* $OpenBSD: bn.h,v 1.28 2015/10/21 19:02:22 miod Exp $ */
+/* $OpenBSD: bn.h,v 1.29 2016/03/02 06:16:11 doug Exp $ */
 /* Copyright (C) 1995-1997 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -125,6 +125,7 @@
 #ifndef HEADER_BN_H
 #define HEADER_BN_H
 
+#include <limits.h>
 #include <stdio.h>
 #include <stdlib.h>
 
@@ -619,10 +620,20 @@ const BIGNUM *BN_get0_nist_prime_521(void);
 
 /* library internal functions */
 
-#define bn_expand(a,bits) ((((((bits+BN_BITS2-1))/BN_BITS2)) <= (a)->dmax)?\
-        (a):bn_expand2((a),(bits+BN_BITS2-1)/BN_BITS2))
 #define bn_wexpand(a,words) (((words) <= (a)->dmax)?(a):bn_expand2((a),(words)))
 BIGNUM *bn_expand2(BIGNUM *a, int words);
+
+static inline BIGNUM *bn_expand(BIGNUM *a, int bits)
+{
+        if (bits > (INT_MAX - BN_BITS2 + 1))
+                return (NULL);
+
+        if (((bits + BN_BITS2 - 1) / BN_BITS2) <= a->dmax)
+                return (a);
+
+        return bn_expand2(a, (bits + BN_BITS2 - 1) / BN_BITS2);
+}
+
 #ifndef OPENSSL_NO_DEPRECATED
 BIGNUM *bn_dup_expand(const BIGNUM *a, int words); /* unused */
 #endif
diff --git a/src/lib/libssl/src/crypto/bn/bn_print.c b/src/lib/libssl/src/crypto/bn/bn_print.c
index 1614a11449..2c1681a2c0 100644
--- a/src/lib/libssl/src/crypto/bn/bn_print.c
+++ b/src/lib/libssl/src/crypto/bn/bn_print.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: bn_print.c,v 1.28 2015/09/28 18:58:33 deraadt Exp $ */
+/* $OpenBSD: bn_print.c,v 1.29 2016/03/02 06:16:11 doug Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -57,6 +57,7 @@
  */
 
 #include <ctype.h>
+#include <limits.h>
 #include <stdio.h>
 
 #include <openssl/opensslconf.h>
@@ -119,7 +120,7 @@ BN_bn2dec(const BIGNUM *a)
                 if (buf == NULL) {
                         BNerr(BN_F_BN_BN2DEC, ERR_R_MALLOC_FAILURE);
                         goto err;
-                }               
+                }
                 p = buf;
                 if (BN_is_negative(a))
                         *p++ = '-';
@@ -127,7 +128,7 @@ BN_bn2dec(const BIGNUM *a)
                 *p++ = '\0';
                 return (buf);
         }
-                
+
         /* get an upper bound for the length of the decimal integer
          * num <= (BN_num_bits(a) + 1) * log(2)
          *     <= 3 * BN_num_bits(a) * 0.1001 + log(2) + 1     (rounding error)
@@ -197,8 +198,10 @@ BN_hex2bn(BIGNUM **bn, const char *a)
                 a++;
         }
 
-        for (i = 0; isxdigit((unsigned char)a[i]); i++)
+        for (i = 0; i <= (INT_MAX / 4) && isxdigit((unsigned char)a[i]); i++)
                 ;
+        if (i > INT_MAX / 4)
+                goto err;
 
         num = i + neg;
         if (bn == NULL)
@@ -213,7 +216,7 @@ BN_hex2bn(BIGNUM **bn, const char *a)
                 BN_zero(ret);
         }
 
-        /* i is the number of hex digests; */
+        /* i is the number of hex digits */
         if (bn_expand(ret, i * 4) == NULL)
                 goto err;
 
@@ -271,8 +274,10 @@ BN_dec2bn(BIGNUM **bn, const char *a)
                 a++;
         }
 
-        for (i = 0; isdigit((unsigned char)a[i]); i++)
+        for (i = 0; i <= (INT_MAX / 4) && isdigit((unsigned char)a[i]); i++)
                 ;
+        if (i > INT_MAX / 4)
+                goto err;
 
         num = i + neg;
         if (bn == NULL)
@@ -288,7 +293,7 @@ BN_dec2bn(BIGNUM **bn, const char *a)
                 BN_zero(ret);
         }
 
-        /* i is the number of digests, a bit of an over expand; */
+        /* i is the number of digits, a bit of an over expand */
         if (bn_expand(ret, i * 4) == NULL)
                 goto err;