MFC: Reject excessively large primes in DH key generation. Problem reportedlibressl-v2.7.4

by Guido Vranken to OpenSSL (https://github.com/openssl/openssl/pull/6457)
and based on his diff.  suggestions from tb@, ok tb@ jsing@

Original commit by sthen@

1 files changed, 6 insertions, 1 deletions

diff --git a/src/lib/libcrypto/dh/dh_key.c b/src/lib/libcrypto/dh/dh_key.c
index 63d38771c3..d79e98e785 100644
--- a/src/lib/libcrypto/dh/dh_key.c
+++ b/src/lib/libcrypto/dh/dh_key.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: dh_key.c,v 1.27 2017/01/29 17:49:22 beck Exp $ */
+/* $OpenBSD: dh_key.c,v 1.27.2.1 2018/06/13 15:13:30 jsing Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -108,6 +108,11 @@ generate_key(DH *dh)
         BN_MONT_CTX *mont = NULL;
         BIGNUM *pub_key = NULL, *priv_key = NULL;
 
+        if (BN_num_bits(dh->p) > OPENSSL_DH_MAX_MODULUS_BITS) {
+                DHerror(DH_R_MODULUS_TOO_LARGE);
+                return 0;
+        }
+
         ctx = BN_CTX_new();
         if (ctx == NULL)
                 goto err;