Add bounds checking for BN_hex2bn/BN_dec2bn.

Need to make sure i * 4 won't overflow.  Based on OpenSSL:

commit 99ba9fd02fd481eb971023a3a0a251a37eb87e4c

input + ok bcook@
ok beck@

1 files changed, 14 insertions, 3 deletions

diff --git a/src/lib/libcrypto/bn/bn.h b/src/lib/libcrypto/bn/bn.h
index 2c648ba2ee..5efccd180b 100644
--- a/src/lib/libcrypto/bn/bn.h
+++ b/src/lib/libcrypto/bn/bn.h
@@ -1,4 +1,4 @@
-/* $OpenBSD: bn.h,v 1.28 2015/10/21 19:02:22 miod Exp $ */
+/* $OpenBSD: bn.h,v 1.29 2016/03/02 06:16:11 doug Exp $ */
 /* Copyright (C) 1995-1997 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -125,6 +125,7 @@
 #ifndef HEADER_BN_H
 #define HEADER_BN_H
 
+#include <limits.h>
 #include <stdio.h>
 #include <stdlib.h>
 
@@ -619,10 +620,20 @@ const BIGNUM *BN_get0_nist_prime_521(void);
 
 /* library internal functions */
 
-#define bn_expand(a,bits) ((((((bits+BN_BITS2-1))/BN_BITS2)) <= (a)->dmax)?\
-        (a):bn_expand2((a),(bits+BN_BITS2-1)/BN_BITS2))
 #define bn_wexpand(a,words) (((words) <= (a)->dmax)?(a):bn_expand2((a),(words)))
 BIGNUM *bn_expand2(BIGNUM *a, int words);
+
+static inline BIGNUM *bn_expand(BIGNUM *a, int bits)
+{
+        if (bits > (INT_MAX - BN_BITS2 + 1))
+                return (NULL);
+
+        if (((bits + BN_BITS2 - 1) / BN_BITS2) <= a->dmax)
+                return (a);
+
+        return bn_expand2(a, (bits + BN_BITS2 - 1) / BN_BITS2);
+}
+
 #ifndef OPENSSL_NO_DEPRECATED
 BIGNUM *bn_dup_expand(const BIGNUM *a, int words); /* unused */
 #endif