Add bounds checking for BN_hex2bn/BN_dec2bn.

Need to make sure i * 4 won't overflow.  Based on OpenSSL:

commit 99ba9fd02fd481eb971023a3a0a251a37eb87e4c

input + ok bcook@
ok beck@

1 files changed, 12 insertions, 7 deletions

diff --git a/src/lib/libcrypto/bn/bn_print.c b/src/lib/libcrypto/bn/bn_print.c
index 1614a11449..2c1681a2c0 100644
--- a/src/lib/libcrypto/bn/bn_print.c
+++ b/src/lib/libcrypto/bn/bn_print.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: bn_print.c,v 1.28 2015/09/28 18:58:33 deraadt Exp $ */
+/* $OpenBSD: bn_print.c,v 1.29 2016/03/02 06:16:11 doug Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -57,6 +57,7 @@
  */
 
 #include <ctype.h>
+#include <limits.h>
 #include <stdio.h>
 
 #include <openssl/opensslconf.h>
@@ -119,7 +120,7 @@ BN_bn2dec(const BIGNUM *a)
                 if (buf == NULL) {
                         BNerr(BN_F_BN_BN2DEC, ERR_R_MALLOC_FAILURE);
                         goto err;
-                }               
+                }
                 p = buf;
                 if (BN_is_negative(a))
                         *p++ = '-';
@@ -127,7 +128,7 @@ BN_bn2dec(const BIGNUM *a)
                 *p++ = '\0';
                 return (buf);
         }
-                
+
         /* get an upper bound for the length of the decimal integer
          * num <= (BN_num_bits(a) + 1) * log(2)
          *     <= 3 * BN_num_bits(a) * 0.1001 + log(2) + 1     (rounding error)
@@ -197,8 +198,10 @@ BN_hex2bn(BIGNUM **bn, const char *a)
                 a++;
         }
 
-        for (i = 0; isxdigit((unsigned char)a[i]); i++)
+        for (i = 0; i <= (INT_MAX / 4) && isxdigit((unsigned char)a[i]); i++)
                 ;
+        if (i > INT_MAX / 4)
+                goto err;
 
         num = i + neg;
         if (bn == NULL)
@@ -213,7 +216,7 @@ BN_hex2bn(BIGNUM **bn, const char *a)
                 BN_zero(ret);
         }
 
-        /* i is the number of hex digests; */
+        /* i is the number of hex digits */
         if (bn_expand(ret, i * 4) == NULL)
                 goto err;
 
@@ -271,8 +274,10 @@ BN_dec2bn(BIGNUM **bn, const char *a)
                 a++;
         }
 
-        for (i = 0; isdigit((unsigned char)a[i]); i++)
+        for (i = 0; i <= (INT_MAX / 4) && isdigit((unsigned char)a[i]); i++)
                 ;
+        if (i > INT_MAX / 4)
+                goto err;
 
         num = i + neg;
         if (bn == NULL)
@@ -288,7 +293,7 @@ BN_dec2bn(BIGNUM **bn, const char *a)
                 BN_zero(ret);
         }
 
-        /* i is the number of digests, a bit of an over expand; */
+        /* i is the number of digits, a bit of an over expand */
         if (bn_expand(ret, i * 4) == NULL)
                 goto err;