'lua_checkstack' doesn't need to check stack overflow

'luaD_growstack' already checks that. This commit also fixes an
internal bug in 'luaD_growstack': a large 'n' could cause an arithmetic
overflow when computing 'needed'.

5 files changed, 17 insertions, 26 deletions

diff --git a/all b/all
index 039f6095..86f38ac1 100755
--- a/all
+++ b/all
@@ -1,7 +1,7 @@
 make -s -j
 cd testes/libs; make -s
 cd ..     # back to directory 'testes'
-ulimit -S -s 1000
+ulimit -S -s 1100
 if { ../lua -W all.lua; } then
   echo -e "\n\n    final OK!!!!\n\n"
 else
diff --git a/lapi.c b/lapi.c
index 5ee65792..352a385a 100644
--- a/lapi.c
+++ b/lapi.c
@@ -114,13 +114,8 @@ LUA_API int lua_checkstack (lua_State *L, int n) {
   api_check(L, n >= 0, "negative 'n'");
   if (L->stack_last - L->top > n)  /* stack large enough? */
     res = 1;  /* yes; check is OK */
-  else {  /* no; need to grow stack */
-    int inuse = cast_int(L->top - L->stack) + EXTRA_STACK;
-    if (inuse > LUAI_MAXSTACK - n)  /* can grow without overflow? */
-      res = 0;  /* no */
-    else  /* try to grow stack */
-      res = luaD_growstack(L, n, 0);
-  }
+  else  /* need to grow stack */
+    res = luaD_growstack(L, n, 0);
   if (res && ci->top < L->top + n)
     ci->top = L->top + n;  /* adjust frame top */
   lua_unlock(L);
diff --git a/ldo.c b/ldo.c
index 8e4faf02..5aa6d59d 100644
--- a/ldo.c
+++ b/ldo.c
@@ -227,7 +227,7 @@ int luaD_growstack (lua_State *L, int n, int raiseerror) {
       luaD_throw(L, LUA_ERRERR);  /* error inside message handler */
     return 0;  /* if not 'raiseerror', just signal it */
   }
-  else {
+  else if (n < LUAI_MAXSTACK) {  /* avoids arithmetic overflows */
     int newsize = 2 * size;  /* tentative new size */
     int needed = cast_int(L->top - L->stack) + n;
     if (newsize > LUAI_MAXSTACK)  /* cannot cross the limit */
@@ -236,14 +236,13 @@ int luaD_growstack (lua_State *L, int n, int raiseerror) {
       newsize = needed;
     if (l_likely(newsize <= LUAI_MAXSTACK))
       return luaD_reallocstack(L, newsize, raiseerror);
-    else {  /* stack overflow */
-      /* add extra size to be able to handle the error message */
-      luaD_reallocstack(L, ERRORSTACKSIZE, raiseerror);
-      if (raiseerror)
-        luaG_runerror(L, "stack overflow");
-      return 0;
-    }
   }
+  /* else stack overflow */
+  /* add extra size to be able to handle the error message */
+  luaD_reallocstack(L, ERRORSTACKSIZE, raiseerror);
+  if (raiseerror)
+    luaG_runerror(L, "stack overflow");
+  return 0;
 }
 
 
diff --git a/luaconf.h b/luaconf.h
index d42d14b7..fcc0018b 100644
--- a/luaconf.h
+++ b/luaconf.h
@@ -728,7 +728,7 @@
 ** CHANGE it if you need a different limit. This limit is arbitrary;
 ** its only purpose is to stop Lua from consuming unlimited stack
 ** space (and to reserve some numbers for pseudo-indices).
-** (It must fit into max(size_t)/32.)
+** (It must fit into max(size_t)/32 and max(int)/2.)
 */
 #if LUAI_IS32INT
 #define LUAI_MAXSTACK           1000000
diff --git a/testes/coroutine.lua b/testes/coroutine.lua
index 76c9d6e6..15fccc30 100644
--- a/testes/coroutine.lua
+++ b/testes/coroutine.lua
@@ -741,20 +741,17 @@ _X()
 
 if not _soft then
   -- bug (stack overflow)
-  local j = 2^9
-  local lim = 1000000    -- (C stack limit; assume 32-bit machine)
-  local t = {lim - 10, lim - 5, lim - 1, lim, lim + 1}
+  local lim = 1000000    -- stack limit; assume 32-bit machine
+  local t = {lim - 10, lim - 5, lim - 1, lim, lim + 1, lim + 5}
   for i = 1, #t do
     local j = t[i]
-    co = coroutine.create(function()
-           local t = {}
-           for i = 1, j do t[i] = i end
-           return table.unpack(t)
+    local co = coroutine.create(function()
+           return table.unpack({}, 1, j)
          end)
     local r, msg = coroutine.resume(co)
-    assert(not r)
+    -- must fail for unpacking larger than stack limit
+    assert(j < lim or not r)
   end
-  co = nil
 end