Changes in the control of C-stack overflow

  * unification of the 'nny' and 'nCcalls' counters;
  * external C functions ('lua_CFunction') count more "slots" in
    the C stack (to allow for their possible use of buffers)
  * added a new test script specific for C-stack overflows. (Most
    of those tests were already present, but concentrating them
    in a single script easies the task of checking whether
    'LUAI_MAXCCALLS' is adequate in a system.)

12 files changed, 170 insertions, 74 deletions

diff --git a/lapi.c b/lapi.c
index 9cabe7ca..2d10bb73 100644
--- a/lapi.c
+++ b/lapi.c
@@ -956,7 +956,7 @@ LUA_API void lua_callk (lua_State *L, int nargs, int nresults,
   api_check(L, L->status == LUA_OK, "cannot do calls on non-normal thread");
   checkresults(L, nargs, nresults);
   func = L->top - (nargs+1);
-  if (k != NULL && L->nny == 0) {  /* need to prepare continuation? */
+  if (k != NULL && yieldable(L)) {  /* need to prepare continuation? */
     L->ci->u.c.k = k;  /* save continuation */
     L->ci->u.c.ctx = ctx;  /* save context */
     luaD_call(L, func, nresults);  /* do the call */
@@ -1004,7 +1004,7 @@ LUA_API int lua_pcallk (lua_State *L, int nargs, int nresults, int errfunc,
     func = savestack(L, o);
   }
   c.func = L->top - (nargs+1);  /* function to be called */
-  if (k == NULL || L->nny > 0) {  /* no continuation or no yieldable? */
+  if (k == NULL || !yieldable(L)) {  /* no continuation or no yieldable? */
     c.nresults = nresults;  /* do a 'conventional' protected call */
     status = luaD_pcall(L, f_call, &c, savestack(L, c.func), func);
   }
diff --git a/ldo.c b/ldo.c
index 056fef0c..f8d8f11c 100644
--- a/ldo.c
+++ b/ldo.c
@@ -138,7 +138,7 @@ l_noret luaD_throw (lua_State *L, int errcode) {
 
 
 int luaD_rawrunprotected (lua_State *L, Pfunc f, void *ud) {
-  unsigned short oldnCcalls = L->nCcalls - L->nci;
+  l_uint32 oldnCcalls = L->nCcalls - L->nci;
   struct lua_longjmp lj;
   lua_assert(L->nCcalls >= L->nci);
   lj.status = LUA_OK;
@@ -513,12 +513,17 @@ void luaD_call (lua_State *L, StkId func, int nresults) {
 
 
 /*
-** Similar to 'luaD_call', but does not allow yields during the call
+** Similar to 'luaD_call', but does not allow yields during the call.
+** If there is a stack overflow, freeing all CI structures will
+** force the subsequent call to invoke 'luaE_extendCI', which then
+** will raise any errors.
 */
 void luaD_callnoyield (lua_State *L, StkId func, int nResults) {
-  L->nny++;
+  incXCcalls(L);
+  if (getCcalls(L) >= LUAI_MAXCCALLS)  /* possible stack overflow? */
+    luaE_freeCI(L);
   luaD_call(L, func, nResults);
-  L->nny--;
+  decXCcalls(L);
 }
 
 
@@ -530,7 +535,7 @@ static void finishCcall (lua_State *L, int status) {
   CallInfo *ci = L->ci;
   int n;
   /* must have a continuation and must be able to call it */
-  lua_assert(ci->u.c.k != NULL && L->nny == 0);
+  lua_assert(ci->u.c.k != NULL && yieldable(L));
   /* error status can only happen in a protected call */
   lua_assert((ci->callstatus & CIST_YPCALL) || status == LUA_YIELD);
   if (ci->callstatus & CIST_YPCALL) {  /* was inside a pcall? */
@@ -601,7 +606,6 @@ static int recover (lua_State *L, int status) {
   luaD_seterrorobj(L, status, oldtop);
   L->ci = ci;
   L->allowhook = getoah(ci->callstatus);  /* restore original 'allowhook' */
-  L->nny = 0;  /* should be zero to be yieldable */
   luaD_shrinkstack(L);
   L->errfunc = ci->u.c.old_errfunc;
   return 1;  /* continue running the coroutine */
@@ -623,13 +627,6 @@ static int resume_error (lua_State *L, const char *msg, int narg) {
 
 
 /*
-** "Cost" in the C stack for a coroutine invocation.
-*/
-#if !defined(LUAL_COROCSTK)
-#define LUAL_COROCSTK   3
-#endif
-
-/*
 ** Do the work for 'lua_resume' in protected mode. Most of the work
 ** depends on the status of the coroutine: initial state, suspended
 ** inside a hook, or regularly suspended (optionally with a continuation
@@ -664,7 +661,6 @@ static void resume (lua_State *L, void *ud) {
 LUA_API int lua_resume (lua_State *L, lua_State *from, int nargs,
                                       int *nresults) {
   int status;
-  unsigned short oldnny = L->nny;  /* save "number of non-yieldable" calls */
   lua_lock(L);
   if (L->status == LUA_OK) {  /* may be starting a coroutine */
     if (L->ci != &L->base_ci)  /* not in base level? */
@@ -675,11 +671,10 @@ LUA_API int lua_resume (lua_State *L, lua_State *from, int nargs,
   if (from == NULL)
     L->nCcalls = 1;
   else  /* correct 'nCcalls' for this thread */
-    L->nCcalls = from->nCcalls - from->nci + L->nci + LUAL_COROCSTK;
+    L->nCcalls = getCcalls(from) - from->nci + L->nci + CSTACKCF;
   if (L->nCcalls >= LUAI_MAXCCALLS)
     return resume_error(L, "C stack overflow", nargs);
   luai_userstateresume(L, nargs);
-  L->nny = 0;  /* allow yields */
   api_checknelems(L, (L->status == LUA_OK) ? nargs + 1 : nargs);
   status = luaD_rawrunprotected(L, resume, &nargs);
    /* continue running after recoverable errors */
@@ -698,14 +693,13 @@ LUA_API int lua_resume (lua_State *L, lua_State *from, int nargs,
   }
   *nresults = (status == LUA_YIELD) ? L->ci->u2.nyield
                                     : cast_int(L->top - (L->ci->func + 1));
-  L->nny = oldnny;  /* restore 'nny' */
   lua_unlock(L);
   return status;
 }
 
 
 LUA_API int lua_isyieldable (lua_State *L) {
-  return (L->nny == 0);
+  return yieldable(L);
 }
 
 
@@ -715,7 +709,7 @@ LUA_API int lua_yieldk (lua_State *L, int nresults, lua_KContext ctx,
   luai_userstateyield(L, nresults);
   lua_lock(L);
   api_checknelems(L, nresults);
-  if (unlikely(L->nny > 0)) {
+  if (unlikely(!yieldable(L))) {
     if (L != G(L)->mainthread)
       luaG_runerror(L, "attempt to yield across a C-call boundary");
     else
@@ -741,7 +735,7 @@ LUA_API int lua_yieldk (lua_State *L, int nresults, lua_KContext ctx,
 
 /*
 ** Call the C function 'func' in protected mode, restoring basic
-** thread information ('allowhook', 'nny', etc.) and in particular
+** thread information ('allowhook', etc.) and in particular
 ** its stack level in case of errors.
 */
 int luaD_pcall (lua_State *L, Pfunc func, void *u,
@@ -749,7 +743,6 @@ int luaD_pcall (lua_State *L, Pfunc func, void *u,
   int status;
   CallInfo *old_ci = L->ci;
   lu_byte old_allowhooks = L->allowhook;
-  unsigned short old_nny = L->nny;
   ptrdiff_t old_errfunc = L->errfunc;
   L->errfunc = ef;
   status = luaD_rawrunprotected(L, func, u);
@@ -757,7 +750,6 @@ int luaD_pcall (lua_State *L, Pfunc func, void *u,
     StkId oldtop = restorestack(L, old_top);
     L->ci = old_ci;
     L->allowhook = old_allowhooks;
-    L->nny = old_nny;
     status = luaF_close(L, oldtop, status);
     oldtop = restorestack(L, old_top);  /* previous call may change stack */
     luaD_seterrorobj(L, status, oldtop);
@@ -811,7 +803,7 @@ int luaD_protectedparser (lua_State *L, ZIO *z, const char *name,
                                         const char *mode) {
   struct SParser p;
   int status;
-  L->nny++;  /* cannot yield during parsing */
+  incnny(L);  /* cannot yield during parsing */
   p.z = z; p.name = name; p.mode = mode;
   p.dyd.actvar.arr = NULL; p.dyd.actvar.size = 0;
   p.dyd.gt.arr = NULL; p.dyd.gt.size = 0;
@@ -822,7 +814,7 @@ int luaD_protectedparser (lua_State *L, ZIO *z, const char *name,
   luaM_freearray(L, p.dyd.actvar.arr, p.dyd.actvar.size);
   luaM_freearray(L, p.dyd.gt.arr, p.dyd.gt.size);
   luaM_freearray(L, p.dyd.label.arr, p.dyd.label.size);
-  L->nny--;
+  decnny(L);
   return status;
 }
 
diff --git a/llimits.h b/llimits.h
index 8ab58b5c..9d35d1c7 100644
--- a/llimits.h
+++ b/llimits.h
@@ -184,11 +184,13 @@ typedef LUAI_UACINT l_uacInt;
 ** must be an unsigned with (at least) 4 bytes (see details in lopcodes.h)
 */
 #if LUAI_BITSINT >= 32
-typedef unsigned int Instruction;
+typedef unsigned int l_uint32;
 #else
-typedef unsigned long Instruction;
+typedef unsigned long l_uint32;
 #endif
 
+typedef l_uint32 Instruction;
+
 
 
 /*
diff --git a/lparser.c b/lparser.c
index eed8bffd..3887958e 100644
--- a/lparser.c
+++ b/lparser.c
@@ -367,10 +367,12 @@ static void adjust_assign (LexState *ls, int nvars, int nexps, expdesc *e) {
 }
 
 
-#define enterlevel(ls)  luaE_incCcalls((ls)->L)
-
+/*
+** Macros to limit the maximum recursion depth while parsing
+*/
+#define enterlevel(ls)  luaE_enterCcall((ls)->L)
 
-#define leavelevel(ls)  ((ls)->L->nCcalls--)
+#define leavelevel(ls)  luaE_exitCcall((ls)->L)
 
 
 /*
diff --git a/lstate.c b/lstate.c
index 5ee024fc..04a9e064 100644
--- a/lstate.c
+++ b/lstate.c
@@ -99,24 +99,42 @@ void luaE_setdebt (global_State *g, l_mem debt) {
 /*
 ** Increment count of "C calls" and check for overflows. In case of
 ** a stack overflow, check appropriate error ("regular" overflow or
-** overflow while handling stack overflow). If 'nCalls' is larger than
-** LUAI_MAXCCALLS (which means it is handling a "regular" overflow) but
-** smaller than 9/8 of LUAI_MAXCCALLS, does not report an error (to
-** allow overflow handling to work)
+** overflow while handling stack overflow).
+** If 'nCcalls' is larger than LUAI_MAXCCALLS but smaller than
+** LUAI_MAXCCALLS + CSTACKCF (plus 2 to avoid by-one errors), it means
+** it has just entered the "overflow zone", so the function raises an
+** overflow error.
+** If 'nCcalls' is larger than LUAI_MAXCCALLS + CSTACKCF + 2
+** (which means it is already handling an overflow) but smaller than
+** 9/8 of LUAI_MAXCCALLS, does not report an error (to allow message
+** handling to work).
+** Otherwise, report a stack overflow while handling a stack overflow
+** (probably caused by a repeating error in the message handling
+** function).
 */
-void luaE_incCcalls (lua_State *L) {
-  if (++L->nCcalls >= LUAI_MAXCCALLS) {
-    if (L->nCcalls == LUAI_MAXCCALLS)
-      luaG_runerror(L, "C stack overflow");
-    else if (L->nCcalls >= (LUAI_MAXCCALLS + (LUAI_MAXCCALLS>>3)))
-      luaD_throw(L, LUA_ERRERR);  /* error while handing stack error */
+void luaE_enterCcall (lua_State *L) {
+  int ncalls = getCcalls(L);
+  L->nCcalls++;
+  if (ncalls >= LUAI_MAXCCALLS) {  /* possible overflow? */
+    luaE_freeCI(L);  /* release unused CIs */
+    ncalls = getCcalls(L);  /* update call count */
+    if (ncalls >= LUAI_MAXCCALLS) {  /* still overflow? */
+      if (ncalls <= LUAI_MAXCCALLS + CSTACKCF + 2) {
+        /* no error before increments; raise the error now */
+        L->nCcalls += (CSTACKCF + 4);  /* avoid raising it again */
+        luaG_runerror(L, "C stack overflow");
+      }
+      else if (ncalls >= (LUAI_MAXCCALLS + (LUAI_MAXCCALLS >> 3)))
+        luaD_throw(L, LUA_ERRERR);  /* error while handling stack error */
+    }
   }
 }
 
 
 CallInfo *luaE_extendCI (lua_State *L) {
   CallInfo *ci;
-  luaE_incCcalls(L);
+  lua_assert(L->ci->next == NULL);
+  luaE_enterCcall(L);
   ci = luaM_new(L, CallInfo);
   lua_assert(L->ci->next == NULL);
   L->ci->next = ci;
@@ -135,13 +153,13 @@ void luaE_freeCI (lua_State *L) {
   CallInfo *ci = L->ci;
   CallInfo *next = ci->next;
   ci->next = NULL;
-  L->nCcalls -= L->nci;  /* to subtract removed elements from 'nCcalls' */
+  L->nCcalls -= L->nci;  /* subtract removed elements from 'nCcalls' */
   while ((ci = next) != NULL) {
     next = ci->next;
     luaM_free(L, ci);
     L->nci--;
   }
-  L->nCcalls += L->nci;  /* to subtract removed elements from 'nCcalls' */
+  L->nCcalls += L->nci;  /* adjust result */
 }
 
 
@@ -151,7 +169,7 @@ void luaE_freeCI (lua_State *L) {
 void luaE_shrinkCI (lua_State *L) {
   CallInfo *ci = L->ci;
   CallInfo *next2;  /* next's next */
-  L->nCcalls -= L->nci;  /* to subtract removed elements from 'nCcalls' */
+  L->nCcalls -= L->nci;  /* subtract removed elements from 'nCcalls' */
   /* while there are two nexts */
   while (ci->next != NULL && (next2 = ci->next->next) != NULL) {
     luaM_free(L, ci->next);  /* free next */
@@ -160,7 +178,7 @@ void luaE_shrinkCI (lua_State *L) {
     next2->previous = ci;
     ci = next2;  /* keep next's next */
   }
-  L->nCcalls += L->nci;  /* to subtract removed elements from 'nCcalls' */
+  L->nCcalls += L->nci;  /* adjust result */
 }
 
 
@@ -250,7 +268,6 @@ static void preinit_thread (lua_State *L, global_State *g) {
   L->allowhook = 1;
   resethookcount(L);
   L->openupval = NULL;
-  L->nny = 1;
   L->status = LUA_OK;
   L->errfunc = 0;
 }
diff --git a/lstate.h b/lstate.h
index ce337707..b069b390 100644
--- a/lstate.h
+++ b/lstate.h
@@ -15,7 +15,6 @@
 
 
 /*
-
 ** Some notes about garbage-collected objects: All objects in Lua must
 ** be kept somehow accessible until being freed, so all objects always
 ** belong to one (and only one) of these lists, using field 'next' of
@@ -43,26 +42,58 @@
 ** 'weak': tables with weak values to be cleared;
 ** 'ephemeron': ephemeron tables with white->white entries;
 ** 'allweak': tables with weak keys and/or weak values to be cleared.
-
 */
 
 
-/*
 
+/*
 ** About 'nCcalls': each thread in Lua (a lua_State) keeps a count of
-** how many "C calls" it has in the C stack, to avoid C-stack overflow.
+** how many "C calls" it can do in the C stack, to avoid C-stack overflow.
 ** This count is very rough approximation; it considers only recursive
 ** functions inside the interpreter, as non-recursive calls can be
 ** considered using a fixed (although unknown) amount of stack space.
 **
+** The count itself has two parts: the lower part is the count itself;
+** the higher part counts the number of non-yieldable calls in the stack.
+**
+** Because calls to external C functions can use of unkown amount
+** of space (e.g., functions using an auxiliary buffer), calls
+** to these functions add more than one to the count.
+**
 ** The proper count also includes the number of CallInfo structures
 ** allocated by Lua, as a kind of "potential" calls. So, when Lua
 ** calls a function (and "consumes" one CallInfo), it needs neither to
 ** increment nor to check 'nCcalls', as its use of C stack is already
 ** accounted for.
-
 */
 
+/* number of "C stack slots" used by an external C function */
+#define CSTACKCF        10
+
+/* true if this thread does not have non-yieldable calls in the stack */
+#define yieldable(L)            (((L)->nCcalls & 0xffff0000) == 0)
+
+/* real number of C calls */
+#define getCcalls(L)    ((L)->nCcalls & 0xffff)
+
+
+/* Increment the number of non-yieldable calls */
+#define incnny(L)       ((L)->nCcalls += 0x10000)
+
+/* Decrement the number of non-yieldable calls */
+#define decnny(L)       ((L)->nCcalls -= 0x10000)
+
+/* Increment the number of non-yieldable calls and nCcalls */
+#define incXCcalls(L)   ((L)->nCcalls += 0x10000 + CSTACKCF)
+
+/* Decrement the number of non-yieldable calls and nCcalls */
+#define decXCcalls(L)   ((L)->nCcalls -= 0x10000 + CSTACKCF)
+
+
+
+
+
+
 struct lua_longjmp;  /* defined in ldo.c */
 
 
@@ -208,8 +239,9 @@ typedef struct global_State {
 */
 struct lua_State {
   CommonHeader;
-  unsigned short nci;  /* number of items in 'ci' list */
   lu_byte status;
+  lu_byte allowhook;
+  unsigned short nci;  /* number of items in 'ci' list */
   StkId top;  /* first free slot in the stack */
   global_State *l_G;
   CallInfo *ci;  /* call info for current function */
@@ -223,13 +255,11 @@ struct lua_State {
   CallInfo base_ci;  /* CallInfo for first level (C calling Lua) */
   volatile lua_Hook hook;
   ptrdiff_t errfunc;  /* current error handling function (stack index) */
+  l_uint32 nCcalls;  /* number of allowed nested C calls - 'nci' */
   int stacksize;
   int basehookcount;
   int hookcount;
-  unsigned short nny;  /* number of non-yieldable calls in stack */
-  unsigned short nCcalls;  /* number of nested C calls + 'nny' */
   l_signalT hookmask;
-  lu_byte allowhook;
 };
 
 
@@ -283,8 +313,10 @@ LUAI_FUNC void luaE_freethread (lua_State *L, lua_State *L1);
 LUAI_FUNC CallInfo *luaE_extendCI (lua_State *L);
 LUAI_FUNC void luaE_freeCI (lua_State *L);
 LUAI_FUNC void luaE_shrinkCI (lua_State *L);
-LUAI_FUNC void luaE_incCcalls (lua_State *L);
+LUAI_FUNC void luaE_enterCcall (lua_State *L);
+
 
+#define luaE_exitCcall(L)       ((L)->nCcalls--)
 
 #endif
 
diff --git a/ltests.h b/ltests.h
index 9d409c8d..997e1c4b 100644
--- a/ltests.h
+++ b/ltests.h
@@ -31,7 +31,7 @@
 
 /* compiled with -O0, Lua uses a lot of C stack space... */
 #undef LUAI_MAXCCALLS
-#define LUAI_MAXCCALLS  200
+#define LUAI_MAXCCALLS  400
 
 /* to avoid warnings, and to make sure value is really unused */
 #define UNUSED(x)       (x=0, (void)(x))
diff --git a/luaconf.h b/luaconf.h
index ff708513..0fc161a4 100644
--- a/luaconf.h
+++ b/luaconf.h
@@ -695,14 +695,14 @@
 /*
 @@ LUAL_BUFFERSIZE is the buffer size used by the lauxlib buffer system.
 ** CHANGE it if it uses too much C-stack space. (For long double,
-** 'string.format("%.99f", -1e4932)' needs 5034 bytes, so a
+** 'string.format("%.99f", -1e4932)' needs 5052 bytes, so a
 ** smaller buffer would force a memory allocation for each call to
 ** 'string.format'.)
 */
 #if LUA_FLOAT_TYPE == LUA_FLOAT_LONGDOUBLE
 #define LUAL_BUFFERSIZE         8192
 #else
-#define LUAL_BUFFERSIZE   ((int)(0x80 * sizeof(void*) * sizeof(lua_Integer)))
+#define LUAL_BUFFERSIZE   ((int)(16 * sizeof(void*) * sizeof(lua_Number)))
 #endif
 
 /*
diff --git a/testes/all.lua b/testes/all.lua
index 26d24976..84ba80a6 100644
--- a/testes/all.lua
+++ b/testes/all.lua
@@ -172,6 +172,7 @@ if not _G._soft then
   assert(f() == 'b')
   assert(f() == 'a')
 end
+dofile('cstack.lua')
 dofile('nextvar.lua')
 dofile('pm.lua')
 dofile('utf8.lua')
diff --git a/testes/coroutine.lua b/testes/coroutine.lua
index 5674a4dd..ca30011f 100644
--- a/testes/coroutine.lua
+++ b/testes/coroutine.lua
@@ -107,7 +107,7 @@ function filter (p, g)
   end)
 end
 
-local x = gen(100)
+local x = gen(80)
 local a = {}
 while 1 do
   local n = x()
@@ -116,7 +116,7 @@ while 1 do
   x = filter(n, x)
 end
 
-assert(#a == 25 and a[#a] == 97)
+assert(#a == 22 and a[#a] == 79)
 x, a = nil
 
 
diff --git a/testes/cstack.lua b/testes/cstack.lua
new file mode 100644
index 00000000..9e5bbaef
--- /dev/null
+++ b/testes/cstack.lua
@@ -0,0 +1,62 @@
+-- $Id: testes/cstack.lua $
+-- See Copyright Notice in file all.lua
+
+print"testing C-stack overflow detection"
+
+-- Segmentation faults in these tests probably result from a C-stack
+-- overflow. To avoid these errors, recompile Lua with a smaller
+-- value for the constant 'LUAI_MAXCCALLS' or else ensure a larger
+-- stack for the program.
+
+local function checkerror (msg, f, ...)
+  local s, err = pcall(f, ...)
+  assert(not s and string.find(err, msg))
+end
+
+
+do    -- simple recursion
+  local count = 0
+  local function foo ()
+    count = count + 1
+    foo()
+  end
+  checkerror("stack overflow", foo)
+  print("  maximum recursion: " .. count)
+end
+
+
+-- bug since 2.5 (C-stack overflow in recursion inside pattern matching)
+do
+  local function f (size)
+    local s = string.rep("a", size)
+    local p = string.rep(".?", size)
+    return string.match(s, p)
+  end
+  local m = f(80)
+  assert(#m == 80)
+  checkerror("too complex", f, 200000)
+end
+
+
+-- testing stack-overflow in recursive 'gsub'
+do
+  local count = 0
+  local function foo ()
+    count = count + 1
+    string.gsub("a", ".", foo)
+  end
+  checkerror("stack overflow", foo)
+  print("  maximum 'gsub' nest (calls): " .. count)
+
+  -- can be done with metamethods, too
+  count = 0
+  local t = setmetatable({}, {__index = foo})
+  foo = function ()
+    count = count + 1
+    string.gsub("a", ".", t)
+  end
+  checkerror("stack overflow", foo)
+  print("  maximum 'gsub' nest (metamethods): " .. count)
+end
+
+print'OK'
diff --git a/testes/pm.lua b/testes/pm.lua
index cdcf3bec..1afaccf6 100644
--- a/testes/pm.lua
+++ b/testes/pm.lua
@@ -237,18 +237,6 @@ checkerror("invalid capture index %%0", string.gsub, "alo", "(%0)", "a")
 checkerror("invalid capture index %%1", string.gsub, "alo", "(%1)", "a")
 checkerror("invalid use of '%%'", string.gsub, "alo", ".", "%x")
 
--- bug since 2.5 (C-stack overflow)
-do
-  local function f (size)
-    local s = string.rep("a", size)
-    local p = string.rep(".?", size)
-    return pcall(string.match, s, p)
-  end
-  local r, m = f(80)
-  assert(r and #m == 80)
-  r, m = f(200000)
-  assert(not r and string.find(m, "too complex"))
-end
 
 if not _soft then
   print("big strings")