Reject too small bits value in BN_generate_prime_ex(), so that it does not risk

becoming negative in probable_prime_dh_safe(). Reported by Franck Denis who
noticed `openssl gendh 0' would segfault.
Fix adapted from OpenSSL RT#2701.

ok beck@ jsing@

6 files changed, 44 insertions, 8 deletions

diff --git a/src/lib/libcrypto/bn/bn.h b/src/lib/libcrypto/bn/bn.h
index 33c6162874..2c648ba2ee 100644
--- a/src/lib/libcrypto/bn/bn.h
+++ b/src/lib/libcrypto/bn/bn.h
@@ -1,4 +1,4 @@
-/* $OpenBSD: bn.h,v 1.27 2015/10/16 12:41:29 beck Exp $ */
+/* $OpenBSD: bn.h,v 1.28 2015/10/21 19:02:22 miod Exp $ */
 /* Copyright (C) 1995-1997 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -768,6 +768,7 @@ void ERR_load_BN_strings(void);
 #define BN_F_BN_DIV_RECP                                 130
 #define BN_F_BN_EXP                                      123
 #define BN_F_BN_EXPAND2                                  108
+#define BN_F_BN_GENERATE_PRIME_EX                        140
 #define BN_F_BN_EXPAND_INTERNAL                          120
 #define BN_F_BN_GF2M_MOD                                 131
 #define BN_F_BN_GF2M_MOD_EXP                             132
@@ -797,6 +798,7 @@ void ERR_load_BN_strings(void);
 #define BN_R_ARG2_LT_ARG3                                100
 #define BN_R_BAD_RECIPROCAL                              101
 #define BN_R_BIGNUM_TOO_LONG                             114
+#define BN_R_BITS_TOO_SMALL                              117
 #define BN_R_CALLED_WITH_EVEN_MODULUS                    102
 #define BN_R_DIV_BY_ZERO                                 103
 #define BN_R_ENCODING_ERROR                              104
diff --git a/src/lib/libcrypto/bn/bn_err.c b/src/lib/libcrypto/bn/bn_err.c
index 5a0f359d86..149e58eafc 100644
--- a/src/lib/libcrypto/bn/bn_err.c
+++ b/src/lib/libcrypto/bn/bn_err.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: bn_err.c,v 1.12 2014/07/10 22:45:56 jsing Exp $ */
+/* $OpenBSD: bn_err.c,v 1.13 2015/10/21 19:02:22 miod Exp $ */
 /* ====================================================================
  * Copyright (c) 1999-2007 The OpenSSL Project.  All rights reserved.
  *
@@ -89,6 +89,7 @@ static ERR_STRING_DATA BN_str_functs[]= {
         {ERR_FUNC(BN_F_BN_EXP), "BN_exp"},
         {ERR_FUNC(BN_F_BN_EXPAND2),     "bn_expand2"},
         {ERR_FUNC(BN_F_BN_EXPAND_INTERNAL),     "BN_EXPAND_INTERNAL"},
+        {ERR_FUNC(BN_F_BN_GENERATE_PRIME_EX),   "BN_generate_prime_ex"},
         {ERR_FUNC(BN_F_BN_GF2M_MOD),    "BN_GF2m_mod"},
         {ERR_FUNC(BN_F_BN_GF2M_MOD_EXP),        "BN_GF2m_mod_exp"},
         {ERR_FUNC(BN_F_BN_GF2M_MOD_MUL),        "BN_GF2m_mod_mul"},
@@ -119,6 +120,7 @@ static ERR_STRING_DATA BN_str_reasons[]= {
         {ERR_REASON(BN_R_ARG2_LT_ARG3)           , "arg2 lt arg3"},
         {ERR_REASON(BN_R_BAD_RECIPROCAL)         , "bad reciprocal"},
         {ERR_REASON(BN_R_BIGNUM_TOO_LONG)        , "bignum too long"},
+        {ERR_REASON(BN_R_BITS_TOO_SMALL)         , "bits too small"},
         {ERR_REASON(BN_R_CALLED_WITH_EVEN_MODULUS), "called with even modulus"},
         {ERR_REASON(BN_R_DIV_BY_ZERO)            , "div by zero"},
         {ERR_REASON(BN_R_ENCODING_ERROR)         , "encoding error"},
diff --git a/src/lib/libcrypto/bn/bn_prime.c b/src/lib/libcrypto/bn/bn_prime.c
index 02780d32e6..b1aba663df 100644
--- a/src/lib/libcrypto/bn/bn_prime.c
+++ b/src/lib/libcrypto/bn/bn_prime.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: bn_prime.c,v 1.13 2015/02/09 15:49:22 jsing Exp $ */
+/* $OpenBSD: bn_prime.c,v 1.14 2015/10/21 19:02:22 miod Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -112,6 +112,8 @@
 #include <stdio.h>
 #include <time.h>
 
+#include <openssl/err.h>
+
 #include "bn_lcl.h"
 
 /* NB: these functions have been "upgraded", the deprecated versions (which are
@@ -164,7 +166,16 @@ BN_generate_prime_ex(BIGNUM *ret, int bits, int safe, const BIGNUM *add,
         int found = 0;
         int i, j, c1 = 0;
         BN_CTX *ctx;
-        int checks = BN_prime_checks_for_size(bits);
+        int checks;
+
+        if (bits < 2 || (bits == 2 && safe)) {
+                /*
+                 * There are no prime numbers smaller than 2, and the smallest
+                 * safe prime (7) spans three bits.
+                 */
+                BNerr(BN_F_BN_GENERATE_PRIME_EX, BN_R_BITS_TOO_SMALL);
+                return 0;
+        }
 
         ctx = BN_CTX_new();
         if (ctx == NULL)
@@ -172,6 +183,9 @@ BN_generate_prime_ex(BIGNUM *ret, int bits, int safe, const BIGNUM *add,
         BN_CTX_start(ctx);
         if ((t = BN_CTX_get(ctx)) == NULL)
                 goto err;
+
+        checks = BN_prime_checks_for_size(bits);
+
 loop:
         /* make a random number and set the top and bottom bits */
         if (add == NULL) {
diff --git a/src/lib/libssl/src/crypto/bn/bn.h b/src/lib/libssl/src/crypto/bn/bn.h
index 33c6162874..2c648ba2ee 100644
--- a/src/lib/libssl/src/crypto/bn/bn.h
+++ b/src/lib/libssl/src/crypto/bn/bn.h
@@ -1,4 +1,4 @@
-/* $OpenBSD: bn.h,v 1.27 2015/10/16 12:41:29 beck Exp $ */
+/* $OpenBSD: bn.h,v 1.28 2015/10/21 19:02:22 miod Exp $ */
 /* Copyright (C) 1995-1997 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -768,6 +768,7 @@ void ERR_load_BN_strings(void);
 #define BN_F_BN_DIV_RECP                                 130
 #define BN_F_BN_EXP                                      123
 #define BN_F_BN_EXPAND2                                  108
+#define BN_F_BN_GENERATE_PRIME_EX                        140
 #define BN_F_BN_EXPAND_INTERNAL                          120
 #define BN_F_BN_GF2M_MOD                                 131
 #define BN_F_BN_GF2M_MOD_EXP                             132
@@ -797,6 +798,7 @@ void ERR_load_BN_strings(void);
 #define BN_R_ARG2_LT_ARG3                                100
 #define BN_R_BAD_RECIPROCAL                              101
 #define BN_R_BIGNUM_TOO_LONG                             114
+#define BN_R_BITS_TOO_SMALL                              117
 #define BN_R_CALLED_WITH_EVEN_MODULUS                    102
 #define BN_R_DIV_BY_ZERO                                 103
 #define BN_R_ENCODING_ERROR                              104
diff --git a/src/lib/libssl/src/crypto/bn/bn_err.c b/src/lib/libssl/src/crypto/bn/bn_err.c
index 5a0f359d86..149e58eafc 100644
--- a/src/lib/libssl/src/crypto/bn/bn_err.c
+++ b/src/lib/libssl/src/crypto/bn/bn_err.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: bn_err.c,v 1.12 2014/07/10 22:45:56 jsing Exp $ */
+/* $OpenBSD: bn_err.c,v 1.13 2015/10/21 19:02:22 miod Exp $ */
 /* ====================================================================
  * Copyright (c) 1999-2007 The OpenSSL Project.  All rights reserved.
  *
@@ -89,6 +89,7 @@ static ERR_STRING_DATA BN_str_functs[]= {
         {ERR_FUNC(BN_F_BN_EXP), "BN_exp"},
         {ERR_FUNC(BN_F_BN_EXPAND2),     "bn_expand2"},
         {ERR_FUNC(BN_F_BN_EXPAND_INTERNAL),     "BN_EXPAND_INTERNAL"},
+        {ERR_FUNC(BN_F_BN_GENERATE_PRIME_EX),   "BN_generate_prime_ex"},
         {ERR_FUNC(BN_F_BN_GF2M_MOD),    "BN_GF2m_mod"},
         {ERR_FUNC(BN_F_BN_GF2M_MOD_EXP),        "BN_GF2m_mod_exp"},
         {ERR_FUNC(BN_F_BN_GF2M_MOD_MUL),        "BN_GF2m_mod_mul"},
@@ -119,6 +120,7 @@ static ERR_STRING_DATA BN_str_reasons[]= {
         {ERR_REASON(BN_R_ARG2_LT_ARG3)           , "arg2 lt arg3"},
         {ERR_REASON(BN_R_BAD_RECIPROCAL)         , "bad reciprocal"},
         {ERR_REASON(BN_R_BIGNUM_TOO_LONG)        , "bignum too long"},
+        {ERR_REASON(BN_R_BITS_TOO_SMALL)         , "bits too small"},
         {ERR_REASON(BN_R_CALLED_WITH_EVEN_MODULUS), "called with even modulus"},
         {ERR_REASON(BN_R_DIV_BY_ZERO)            , "div by zero"},
         {ERR_REASON(BN_R_ENCODING_ERROR)         , "encoding error"},
diff --git a/src/lib/libssl/src/crypto/bn/bn_prime.c b/src/lib/libssl/src/crypto/bn/bn_prime.c
index 02780d32e6..b1aba663df 100644
--- a/src/lib/libssl/src/crypto/bn/bn_prime.c
+++ b/src/lib/libssl/src/crypto/bn/bn_prime.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: bn_prime.c,v 1.13 2015/02/09 15:49:22 jsing Exp $ */
+/* $OpenBSD: bn_prime.c,v 1.14 2015/10/21 19:02:22 miod Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -112,6 +112,8 @@
 #include <stdio.h>
 #include <time.h>
 
+#include <openssl/err.h>
+
 #include "bn_lcl.h"
 
 /* NB: these functions have been "upgraded", the deprecated versions (which are
@@ -164,7 +166,16 @@ BN_generate_prime_ex(BIGNUM *ret, int bits, int safe, const BIGNUM *add,
         int found = 0;
         int i, j, c1 = 0;
         BN_CTX *ctx;
-        int checks = BN_prime_checks_for_size(bits);
+        int checks;
+
+        if (bits < 2 || (bits == 2 && safe)) {
+                /*
+                 * There are no prime numbers smaller than 2, and the smallest
+                 * safe prime (7) spans three bits.
+                 */
+                BNerr(BN_F_BN_GENERATE_PRIME_EX, BN_R_BITS_TOO_SMALL);
+                return 0;
+        }
 
         ctx = BN_CTX_new();
         if (ctx == NULL)
@@ -172,6 +183,9 @@ BN_generate_prime_ex(BIGNUM *ret, int bits, int safe, const BIGNUM *add,
         BN_CTX_start(ctx);
         if ((t = BN_CTX_get(ctx)) == NULL)
                 goto err;
+
+        checks = BN_prime_checks_for_size(bits);
+
 loop:
         /* make a random number and set the top and bottom bits */
         if (add == NULL) {