Limit ASN.1 constructed types recursive definition depth

Fixes for CVE-2018-0739.
Copied from commit below, and modified for adaption to our code.
https://github.com/openssl/openssl/commit/9310d45087ae546e27e61ddf8f6367f29848220d

ok bcook@ beck@ jsing@

3 files changed, 45 insertions, 22 deletions

diff --git a/src/lib/libcrypto/asn1/asn1.h b/src/lib/libcrypto/asn1/asn1.h
index 6fc4cd7527..3a2da6463e 100644
--- a/src/lib/libcrypto/asn1/asn1.h
+++ b/src/lib/libcrypto/asn1/asn1.h
@@ -1,4 +1,4 @@
-/* $OpenBSD: asn1.h,v 1.44 2018/02/14 16:46:04 jsing Exp $ */
+/* $OpenBSD: asn1.h,v 1.45 2018/03/29 02:29:24 inoguchi Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -1405,6 +1405,7 @@ void ERR_load_ASN1_strings(void);
 #define ASN1_R_MSTRING_NOT_UNIVERSAL                     139
 #define ASN1_R_MSTRING_WRONG_TAG                         140
 #define ASN1_R_NESTED_ASN1_STRING                        197
+#define ASN1_R_NESTED_TOO_DEEP                           219
 #define ASN1_R_NON_HEX_CHARACTERS                        141
 #define ASN1_R_NOT_ASCII_FORMAT                          190
 #define ASN1_R_NOT_ENOUGH_DATA                           142
diff --git a/src/lib/libcrypto/asn1/asn1_err.c b/src/lib/libcrypto/asn1/asn1_err.c
index 0c827a9236..5cc355084f 100644
--- a/src/lib/libcrypto/asn1/asn1_err.c
+++ b/src/lib/libcrypto/asn1/asn1_err.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: asn1_err.c,v 1.20 2017/01/29 17:49:22 beck Exp $ */
+/* $OpenBSD: asn1_err.c,v 1.21 2018/03/29 02:29:24 inoguchi Exp $ */
 /* ====================================================================
  * Copyright (c) 1999-2011 The OpenSSL Project.  All rights reserved.
  *
@@ -150,6 +150,7 @@ static ERR_STRING_DATA ASN1_str_reasons[] = {
         {ERR_REASON(ASN1_R_MSTRING_NOT_UNIVERSAL), "mstring not universal"},
         {ERR_REASON(ASN1_R_MSTRING_WRONG_TAG)    , "mstring wrong tag"},
         {ERR_REASON(ASN1_R_NESTED_ASN1_STRING)   , "nested asn1 string"},
+        {ERR_REASON(ASN1_R_NESTED_TOO_DEEP)      , "nested too deep"},
         {ERR_REASON(ASN1_R_NON_HEX_CHARACTERS)   , "non hex characters"},
         {ERR_REASON(ASN1_R_NOT_ASCII_FORMAT)     , "not ascii format"},
         {ERR_REASON(ASN1_R_NOT_ENOUGH_DATA)      , "not enough data"},
diff --git a/src/lib/libcrypto/asn1/tasn_dec.c b/src/lib/libcrypto/asn1/tasn_dec.c
index 3f680c60fd..bf536cb667 100644
--- a/src/lib/libcrypto/asn1/tasn_dec.c
+++ b/src/lib/libcrypto/asn1/tasn_dec.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: tasn_dec.c,v 1.34 2017/01/29 17:49:22 beck Exp $ */
+/* $OpenBSD: tasn_dec.c,v 1.35 2018/03/29 02:29:24 inoguchi Exp $ */
 /* Written by Dr Stephen N Henson (steve@openssl.org) for the OpenSSL
  * project 2000.
  */
@@ -65,6 +65,12 @@
 #include <openssl/buffer.h>
 #include <openssl/err.h>
 
+/* Constructed types with a recursive definition (such as can be found in PKCS7)
+ * could eventually exceed the stack given malicious input with excessive
+ * recursion. Therefore we limit the stack depth.
+ */
+#define ASN1_MAX_CONSTRUCTED_NEST 30
+
 static int asn1_check_eoc(const unsigned char **in, long len);
 static int asn1_find_end(const unsigned char **in, long len, char inf);
 
@@ -78,9 +84,9 @@ static int asn1_check_tlen(long *olen, int *otag, unsigned char *oclass,
     int expclass, char opt, ASN1_TLC *ctx);
 
 static int asn1_template_ex_d2i(ASN1_VALUE **pval, const unsigned char **in,
-    long len, const ASN1_TEMPLATE *tt, char opt, ASN1_TLC *ctx);
+    long len, const ASN1_TEMPLATE *tt, char opt, ASN1_TLC *ctx, int depth);
 static int asn1_template_noexp_d2i(ASN1_VALUE **val, const unsigned char **in,
-    long len, const ASN1_TEMPLATE *tt, char opt, ASN1_TLC *ctx);
+    long len, const ASN1_TEMPLATE *tt, char opt, ASN1_TLC *ctx, int depth);
 static int asn1_d2i_ex_primitive(ASN1_VALUE **pval, const unsigned char **in,
     long len, const ASN1_ITEM *it, int tag, int aclass, char opt,
     ASN1_TLC *ctx);
@@ -142,7 +148,7 @@ ASN1_template_d2i(ASN1_VALUE **pval, const unsigned char **in, long len,
         ASN1_TLC c;
 
         asn1_tlc_clear_nc(&c);
-        return asn1_template_ex_d2i(pval, in, len, tt, 0, &c);
+        return asn1_template_ex_d2i(pval, in, len, tt, 0, &c, 0);
 }
 
 
@@ -150,9 +156,10 @@ ASN1_template_d2i(ASN1_VALUE **pval, const unsigned char **in, long len,
  * If 'opt' set and tag mismatch return -1 to handle OPTIONAL
  */
 
-int
-ASN1_item_ex_d2i(ASN1_VALUE **pval, const unsigned char **in, long len,
-    const ASN1_ITEM *it, int tag, int aclass, char opt, ASN1_TLC *ctx)
+static int
+asn1_item_ex_d2i(ASN1_VALUE **pval, const unsigned char **in, long len,
+    const ASN1_ITEM *it, int tag, int aclass, char opt, ASN1_TLC *ctx,
+    int depth)
 {
         const ASN1_TEMPLATE *tt, *errtt = NULL;
         const ASN1_EXTERN_FUNCS *ef;
@@ -179,6 +186,11 @@ ASN1_item_ex_d2i(ASN1_VALUE **pval, const unsigned char **in, long len,
         else
                 asn1_cb = 0;
 
+        if (++depth > ASN1_MAX_CONSTRUCTED_NEST) {
+                ASN1error(ASN1_R_NESTED_TOO_DEEP);
+                goto err;
+        }
+
         switch (it->itype) {
         case ASN1_ITYPE_PRIMITIVE:
                 if (it->templates) {
@@ -193,7 +205,7 @@ ASN1_item_ex_d2i(ASN1_VALUE **pval, const unsigned char **in, long len,
                                 goto err;
                         }
                         return asn1_template_ex_d2i(pval, in, len,
-                            it->templates, opt, ctx);
+                            it->templates, opt, ctx, depth);
                 }
                 return asn1_d2i_ex_primitive(pval, in, len, it,
                     tag, aclass, opt, ctx);
@@ -258,7 +270,8 @@ ASN1_item_ex_d2i(ASN1_VALUE **pval, const unsigned char **in, long len,
                         /* We mark field as OPTIONAL so its absence
                          * can be recognised.
                          */
-                        ret = asn1_template_ex_d2i(pchptr, &p, len, tt, 1, ctx);
+                        ret = asn1_template_ex_d2i(pchptr, &p, len, tt, 1, ctx,
+                                depth);
                         /* If field not present, try the next one */
                         if (ret == -1)
                                 continue;
@@ -376,7 +389,7 @@ ASN1_item_ex_d2i(ASN1_VALUE **pval, const unsigned char **in, long len,
                          * OPTIONAL */
 
                         ret = asn1_template_ex_d2i(pseqval, &p, len,
-                            seqtt, isopt, ctx);
+                            seqtt, isopt, ctx, depth);
                         if (!ret) {
                                 errtt = seqtt;
                                 goto err;
@@ -448,13 +461,20 @@ err:
         return 0;
 }
 
+int
+ASN1_item_ex_d2i(ASN1_VALUE **pval, const unsigned char **in, long len,
+    const ASN1_ITEM *it, int tag, int aclass, char opt, ASN1_TLC *ctx)
+{
+        return asn1_item_ex_d2i(pval, in, len, it, tag, aclass, opt, ctx, 0);
+}
+
 /* Templates are handled with two separate functions.
  * One handles any EXPLICIT tag and the other handles the rest.
  */
 
 static int
 asn1_template_ex_d2i(ASN1_VALUE **val, const unsigned char **in, long inlen,
-    const ASN1_TEMPLATE *tt, char opt, ASN1_TLC *ctx)
+    const ASN1_TEMPLATE *tt, char opt, ASN1_TLC *ctx, int depth)
 {
         int flags, aclass;
         int ret;
@@ -489,7 +509,7 @@ asn1_template_ex_d2i(ASN1_VALUE **val, const unsigned char **in, long inlen,
                         return 0;
                 }
                 /* We've found the field so it can't be OPTIONAL now */
-                ret = asn1_template_noexp_d2i(val, &p, len, tt, 0, ctx);
+                ret = asn1_template_noexp_d2i(val, &p, len, tt, 0, ctx, depth);
                 if (!ret) {
                         ASN1error(ERR_R_NESTED_ASN1_ERROR);
                         return 0;
@@ -511,7 +531,8 @@ asn1_template_ex_d2i(ASN1_VALUE **val, const unsigned char **in, long inlen,
                         }
                 }
         } else
-                return asn1_template_noexp_d2i(val, in, inlen, tt, opt, ctx);
+                return asn1_template_noexp_d2i(val, in, inlen, tt, opt, ctx,
+                        depth);
 
         *in = p;
         return 1;
@@ -523,7 +544,7 @@ err:
 
 static int
 asn1_template_noexp_d2i(ASN1_VALUE **val, const unsigned char **in, long len,
-    const ASN1_TEMPLATE *tt, char opt, ASN1_TLC *ctx)
+    const ASN1_TEMPLATE *tt, char opt, ASN1_TLC *ctx, int depth)
 {
         int flags, aclass;
         int ret;
@@ -594,8 +615,8 @@ asn1_template_noexp_d2i(ASN1_VALUE **val, const unsigned char **in, long len,
                                 break;
                         }
                         skfield = NULL;
-                        if (!ASN1_item_ex_d2i(&skfield, &p, len,
-                            tt->item, -1, 0, 0, ctx)) {
+                        if (!asn1_item_ex_d2i(&skfield, &p, len,
+                            tt->item, -1, 0, 0, ctx, depth)) {
                                 ASN1error(ERR_R_NESTED_ASN1_ERROR);
                                 goto err;
                         }
@@ -612,8 +633,8 @@ asn1_template_noexp_d2i(ASN1_VALUE **val, const unsigned char **in, long len,
                 }
         } else if (flags & ASN1_TFLG_IMPTAG) {
                 /* IMPLICIT tagging */
-                ret = ASN1_item_ex_d2i(val, &p, len,
-                    tt->item, tt->tag, aclass, opt, ctx);
+                ret = asn1_item_ex_d2i(val, &p, len,
+                    tt->item, tt->tag, aclass, opt, ctx, depth);
                 if (!ret) {
                         ASN1error(ERR_R_NESTED_ASN1_ERROR);
                         goto err;
@@ -621,8 +642,8 @@ asn1_template_noexp_d2i(ASN1_VALUE **val, const unsigned char **in, long len,
                         return -1;
         } else {
                 /* Nothing special */
-                ret = ASN1_item_ex_d2i(val, &p, len, tt->item,
-                    -1, tt->flags & ASN1_TFLG_COMBINE, opt, ctx);
+                ret = asn1_item_ex_d2i(val, &p, len, tt->item,
+                    -1, tt->flags & ASN1_TFLG_COMBINE, opt, ctx, depth);
                 if (!ret) {
                         ASN1error(ERR_R_NESTED_ASN1_ERROR);
                         goto err;