Add bounds checking for BN_hex2bn/BN_dec2bn.

Need to make sure i * 4 won't overflow.  Based on OpenSSL:

commit 99ba9fd02fd481eb971023a3a0a251a37eb87e4c

input + ok bcook@
ok beck@

2 files changed, 26 insertions, 10 deletions

diff --git a/lib/libssl/src/crypto/bn/bn.h b/lib/libssl/src/crypto/bn/bn.h
index 2c648ba2ee3..5efccd180b7 100644
--- a/lib/libssl/src/crypto/bn/bn.h
+++ b/lib/libssl/src/crypto/bn/bn.h
@@ -1,4 +1,4 @@
-/* $OpenBSD: bn.h,v 1.28 2015/10/21 19:02:22 miod Exp $ */
+/* $OpenBSD: bn.h,v 1.29 2016/03/02 06:16:11 doug Exp $ */
 /* Copyright (C) 1995-1997 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -125,6 +125,7 @@
 #ifndef HEADER_BN_H
 #define HEADER_BN_H
 
+#include <limits.h>
 #include <stdio.h>
 #include <stdlib.h>
 
@@ -619,10 +620,20 @@ const BIGNUM *BN_get0_nist_prime_521(void);
 
 /* library internal functions */
 
-#define bn_expand(a,bits) ((((((bits+BN_BITS2-1))/BN_BITS2)) <= (a)->dmax)?\
-	(a):bn_expand2((a),(bits+BN_BITS2-1)/BN_BITS2))
 #define bn_wexpand(a,words) (((words) <= (a)->dmax)?(a):bn_expand2((a),(words)))
 BIGNUM *bn_expand2(BIGNUM *a, int words);
+
+static inline BIGNUM *bn_expand(BIGNUM *a, int bits)
+{
+	if (bits > (INT_MAX - BN_BITS2 + 1))
+		return (NULL);
+
+	if (((bits + BN_BITS2 - 1) / BN_BITS2) <= a->dmax)
+		return (a);
+
+	return bn_expand2(a, (bits + BN_BITS2 - 1) / BN_BITS2);
+}
+
 #ifndef OPENSSL_NO_DEPRECATED
 BIGNUM *bn_dup_expand(const BIGNUM *a, int words); /* unused */
 #endif
diff --git a/lib/libssl/src/crypto/bn/bn_print.c b/lib/libssl/src/crypto/bn/bn_print.c
index 1614a114491..2c1681a2c04 100644
--- a/lib/libssl/src/crypto/bn/bn_print.c
+++ b/lib/libssl/src/crypto/bn/bn_print.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: bn_print.c,v 1.28 2015/09/28 18:58:33 deraadt Exp $ */
+/* $OpenBSD: bn_print.c,v 1.29 2016/03/02 06:16:11 doug Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -57,6 +57,7 @@
  */
 
 #include <ctype.h>
+#include <limits.h>
 #include <stdio.h>
 
 #include <openssl/opensslconf.h>
@@ -119,7 +120,7 @@ BN_bn2dec(const BIGNUM *a)
 		if (buf == NULL) {
 			BNerr(BN_F_BN_BN2DEC, ERR_R_MALLOC_FAILURE);
 			goto err;
-		}		
+		}
 		p = buf;
 		if (BN_is_negative(a))
 			*p++ = '-';
@@ -127,7 +128,7 @@ BN_bn2dec(const BIGNUM *a)
 		*p++ = '\0';
 		return (buf);
 	}
-		
+
 	/* get an upper bound for the length of the decimal integer
 	 * num <= (BN_num_bits(a) + 1) * log(2)
 	 *     <= 3 * BN_num_bits(a) * 0.1001 + log(2) + 1     (rounding error)
@@ -197,8 +198,10 @@ BN_hex2bn(BIGNUM **bn, const char *a)
 		a++;
 	}
 
-	for (i = 0; isxdigit((unsigned char)a[i]); i++)
+	for (i = 0; i <= (INT_MAX / 4) && isxdigit((unsigned char)a[i]); i++)
 		;
+	if (i > INT_MAX / 4)
+		goto err;
 
 	num = i + neg;
 	if (bn == NULL)
@@ -213,7 +216,7 @@ BN_hex2bn(BIGNUM **bn, const char *a)
 		BN_zero(ret);
 	}
 
-	/* i is the number of hex digests; */
+	/* i is the number of hex digits */
 	if (bn_expand(ret, i * 4) == NULL)
 		goto err;
 
@@ -271,8 +274,10 @@ BN_dec2bn(BIGNUM **bn, const char *a)
 		a++;
 	}
 
-	for (i = 0; isdigit((unsigned char)a[i]); i++)
+	for (i = 0; i <= (INT_MAX / 4) && isdigit((unsigned char)a[i]); i++)
 		;
+	if (i > INT_MAX / 4)
+		goto err;
 
 	num = i + neg;
 	if (bn == NULL)
@@ -288,7 +293,7 @@ BN_dec2bn(BIGNUM **bn, const char *a)
 		BN_zero(ret);
 	}
 
-	/* i is the number of digests, a bit of an over expand; */
+	/* i is the number of digits, a bit of an over expand */
 	if (bn_expand(ret, i * 4) == NULL)
 		goto err;