Avoid stack overflow when handling nested captures

The C code uses recursion to handle nested captures, so a too deep
nesting could create a stack overflow. The fix limits the handling
of nested captures to 'MAXRECLEVEL' (default is 200 levels).

5 files changed, 50 insertions, 20 deletions

diff --git a/lpcap.c b/lpcap.c
index 26058e2..1c2af0b 100644
--- a/lpcap.c
+++ b/lpcap.c
@@ -441,70 +441,88 @@ static int addonestring (luaL_Buffer *b, CapState *cs, const char *what) {
 }
 
 
+#if !defined(MAXRECLEVEL)
+#define MAXRECLEVEL     200
+#endif
+
+
 /*
 ** Push all values of the current capture into the stack; returns
 ** number of values pushed
 */
 static int pushcapture (CapState *cs) {
   lua_State *L = cs->L;
+  int res;
   luaL_checkstack(L, 4, "too many captures");
+  if (cs->reclevel++ > MAXRECLEVEL)
+    return luaL_error(L, "subcapture nesting too deep");
   switch (captype(cs->cap)) {
     case Cposition: {
       lua_pushinteger(L, cs->cap->s - cs->s + 1);
       cs->cap++;
-      return 1;
+      res = 1;
+      break;
     }
     case Cconst: {
       pushluaval(cs);
       cs->cap++;
-      return 1;
+      res = 1;
+      break;
     }
     case Carg: {
       int arg = (cs->cap++)->idx;
       if (arg + FIXEDARGS > cs->ptop)
         return luaL_error(L, "reference to absent extra argument #%d", arg);
       lua_pushvalue(L, arg + FIXEDARGS);
-      return 1;
+      res = 1;
+      break;
     }
     case Csimple: {
       int k = pushnestedvalues(cs, 1);
       lua_insert(L, -k);  /* make whole match be first result */
-      return k;
+      res = k;
+      break;
     }
     case Cruntime: {
       lua_pushvalue(L, (cs->cap++)->idx);  /* value is in the stack */
-      return 1;
+      res = 1;
+      break;
     }
     case Cstring: {
       luaL_Buffer b;
       luaL_buffinit(L, &b);
       stringcap(&b, cs);
       luaL_pushresult(&b);
-      return 1;
+      res = 1;
+      break;
     }
     case Csubst: {
       luaL_Buffer b;
       luaL_buffinit(L, &b);
       substcap(&b, cs);
       luaL_pushresult(&b);
-      return 1;
+      res = 1;
+      break;
     }
     case Cgroup: {
       if (cs->cap->idx == 0)  /* anonymous group? */
-        return pushnestedvalues(cs, 0);  /* add all nested values */
+        res = pushnestedvalues(cs, 0);  /* add all nested values */
       else {  /* named group: add no values */
         nextcap(cs);  /* skip capture */
-        return 0;
+        res = 0;
       }
+      break;
     }
-    case Cbackref: return backrefcap(cs);
-    case Ctable: return tablecap(cs);
-    case Cfunction: return functioncap(cs);
-    case Cnum: return numcap(cs);
-    case Cquery: return querycap(cs);
-    case Cfold: return foldcap(cs);
-    default: assert(0); return 0;
+    case Cbackref: res = backrefcap(cs); break;
+    case Ctable: res = tablecap(cs); break;
+    case Cfunction: res = functioncap(cs); break;
+    case Cnum: res = numcap(cs); break;
+    case Cquery: res = querycap(cs); break;
+    case Cfold: res = foldcap(cs); break;
+    default: assert(0); res = 0;
   }
+  cs->reclevel--;
+  return res;
 }
 
 
@@ -521,7 +539,7 @@ int getcaptures (lua_State *L, const char *s, const char *r, int ptop) {
   int n = 0;
   if (!isclosecap(capture)) {  /* is there any capture? */
     CapState cs;
-    cs.ocap = cs.cap = capture; cs.L = L;
+    cs.ocap = cs.cap = capture; cs.L = L; cs.reclevel = 0;
     cs.s = s; cs.valuecached = 0; cs.ptop = ptop;
     do {  /* collect their values */
       n += pushcapture(&cs);
diff --git a/lpcap.h b/lpcap.h
index bf52b67..dc10d69 100644
--- a/lpcap.h
+++ b/lpcap.h
@@ -44,6 +44,7 @@ typedef struct CapState {
   int ptop;  /* index of last argument to 'match' */
   const char *s;  /* original string */
   int valuecached;  /* value stored in cache slot */
+  int reclevel;  /* recursion level */
 } CapState;
 
 
diff --git a/lpvm.c b/lpvm.c
index 52c14ae..813d3fb 100644
--- a/lpvm.c
+++ b/lpvm.c
@@ -296,7 +296,8 @@ const char *match (lua_State *L, const char *o, const char *s, const char *e,
         CapState cs;
         int rem, res, n;
         int fr = lua_gettop(L) + 1;  /* stack index of first result */
-        cs.s = o; cs.L = L; cs.ocap = capture; cs.ptop = ptop;
+        cs.reclevel = 0; cs.L = L;
+        cs.s = o; cs.ocap = capture; cs.ptop = ptop;
         n = runtimecap(&cs, capture + captop, s, &rem);  /* call function */
         captop -= n;  /* remove nested captures */
         ndyncap -= rem;  /* update number of dynamic captures */
diff --git a/makefile b/makefile
index d803c12..9518034 100644
--- a/makefile
+++ b/makefile
@@ -29,11 +29,11 @@ FILES = lpvm.o lpcap.o lptree.o lpcode.o lpprint.o
 
 # For Linux
 linux:
-        make lpeg.so "DLLFLAGS = -shared -fPIC"
+        $(MAKE) lpeg.so "DLLFLAGS = -shared -fPIC"
 
 # For Mac OS
 macosx:
-        make lpeg.so "DLLFLAGS = -bundle -undefined dynamic_lookup"
+        $(MAKE) lpeg.so "DLLFLAGS = -bundle -undefined dynamic_lookup"
 
 lpeg.so: $(FILES)
         env $(CC) $(DLLFLAGS) $(FILES) -o lpeg.so
diff --git a/test.lua b/test.lua
index 476da0a..8f9f574 100755
--- a/test.lua
+++ b/test.lua
@@ -424,6 +424,16 @@ do
 end
 
 
+do
+  -- nesting of captures too deep
+  local p = m.C(1)
+  for i = 1, 300 do
+    p = m.Ct(p)
+  end
+  checkerr("too deep", p.match, p, "x")
+end
+
+
 -- tests for non-pattern as arguments to pattern functions
 
 p = { ('a' * m.V(1))^-1 } * m.P'b' * { 'a' * m.V(2); m.V(1)^-1 }