Avoid an infinite loop that can be triggered by parsing an ASN.1

ECParameters structure that has a specially malformed binary polynomial
field.

Issue reported by Joseph Barr-Pixton and fix based on OpenSSL.

Fixes CVE-2015-1788.

ok doug@ miod@

2 files changed, 16 insertions, 6 deletions

diff --git a/src/lib/libcrypto/bn/bn_gf2m.c b/src/lib/libcrypto/bn/bn_gf2m.c
index e1537d5379..40c1a94220 100644
--- a/src/lib/libcrypto/bn/bn_gf2m.c
+++ b/src/lib/libcrypto/bn/bn_gf2m.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: bn_gf2m.c,v 1.19 2015/04/29 00:11:12 doug Exp $ */
+/* $OpenBSD: bn_gf2m.c,v 1.20 2015/06/11 15:55:28 jsing Exp $ */
 /* ====================================================================
  * Copyright 2002 Sun Microsystems, Inc. ALL RIGHTS RESERVED.
  *
@@ -745,8 +745,13 @@ BN_GF2m_mod_inv(BIGNUM *r, const BIGNUM *a, const BIGNUM *p, BN_CTX *ctx)
                                 ubits--;
                         }
 
-                        if (ubits <= BN_BITS2 && udp[0] == 1)
-                                break;
+                        if (ubits <= BN_BITS2) {
+                                /* See if poly was reducible. */
+                                if (udp[0] == 0)
+                                        goto err;
+                                if (udp[0] == 1)
+                                        break;
+                        }
 
                         if (ubits < vbits) {
                                 i = ubits;
diff --git a/src/lib/libssl/src/crypto/bn/bn_gf2m.c b/src/lib/libssl/src/crypto/bn/bn_gf2m.c
index e1537d5379..40c1a94220 100644
--- a/src/lib/libssl/src/crypto/bn/bn_gf2m.c
+++ b/src/lib/libssl/src/crypto/bn/bn_gf2m.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: bn_gf2m.c,v 1.19 2015/04/29 00:11:12 doug Exp $ */
+/* $OpenBSD: bn_gf2m.c,v 1.20 2015/06/11 15:55:28 jsing Exp $ */
 /* ====================================================================
  * Copyright 2002 Sun Microsystems, Inc. ALL RIGHTS RESERVED.
  *
@@ -745,8 +745,13 @@ BN_GF2m_mod_inv(BIGNUM *r, const BIGNUM *a, const BIGNUM *p, BN_CTX *ctx)
                                 ubits--;
                         }
 
-                        if (ubits <= BN_BITS2 && udp[0] == 1)
-                                break;
+                        if (ubits <= BN_BITS2) {
+                                /* See if poly was reducible. */
+                                if (udp[0] == 0)
+                                        goto err;
+                                if (udp[0] == 1)
+                                        break;
+                        }
 
                         if (ubits < vbits) {
                                 i = ubits;