Fix BN_is_prime_* calls in libcrypto, the API returns -1 on error.

From BoringSSL's commit 53409ee3d7595ed37da472bc73b010cd2c8a5ffd
by David Benjamin.

ok djm, jsing

1 files changed, 12 insertions, 5 deletions

diff --git a/src/lib/libcrypto/dh/dh_check.c b/src/lib/libcrypto/dh/dh_check.c
index a6010f0a6d..a8227d31ca 100644
--- a/src/lib/libcrypto/dh/dh_check.c
+++ b/src/lib/libcrypto/dh/dh_check.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: dh_check.c,v 1.16 2016/07/05 02:54:35 bcook Exp $ */
+/* $OpenBSD: dh_check.c,v 1.17 2019/01/20 01:56:59 tb Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -74,7 +74,7 @@
 int
 DH_check(const DH *dh, int *ret)
 {
-        int ok = 0;
+        int is_prime, ok = 0;
         BN_CTX *ctx = NULL;
         BN_ULONG l;
         BIGNUM *q = NULL;
@@ -102,16 +102,23 @@ DH_check(const DH *dh, int *ret)
         } else
                 *ret |= DH_UNABLE_TO_CHECK_GENERATOR;
 
-        if (!BN_is_prime_ex(dh->p, BN_prime_checks, ctx, NULL))
+        is_prime = BN_is_prime_ex(dh->p, BN_prime_checks, ctx, NULL);
+        if (is_prime < 0)
+                goto err;
+        if (is_prime == 0)
                 *ret |= DH_CHECK_P_NOT_PRIME;
         else {
                 if (!BN_rshift1(q, dh->p))
                         goto err;
-                if (!BN_is_prime_ex(q, BN_prime_checks, ctx, NULL))
+                is_prime = BN_is_prime_ex(q, BN_prime_checks, ctx, NULL);
+                if (is_prime < 0)
+                        goto err;
+                if (is_prime == 0)
                         *ret |= DH_CHECK_P_NOT_SAFE_PRIME;
         }
         ok = 1;
-err:
+
+ err:
         BN_CTX_free(ctx);
         BN_free(q);
         return ok;