GC checks stack space before running finalizerHEADmaster

If the stack does not have some minimum available space, the GC defers
calling a finalizer until the next cycle. That avoids errors while
running a finalizer that the programmer cannot control.

3 files changed, 16 insertions, 3 deletions

diff --git a/ldo.c b/ldo.c
index 75ce1488..6d0184ec 100644
--- a/ldo.c
+++ b/ldo.c
@@ -221,6 +221,17 @@ l_noret luaD_errerr (lua_State *L) {
 
 
 /*
+** Check whether stack has enough space to run a simple function (such
+** as a finalizer): At least BASIC_STACK_SIZE in the Lua stack and
+** 2 slots in the C stack.
+*/
+int luaD_checkminstack (lua_State *L) {
+  return ((stacksize(L) < MAXSTACK - BASIC_STACK_SIZE) &&
+          (getCcalls(L) < LUAI_MAXCCALLS - 2));
+}
+
+
+/*
 ** In ISO C, any pointer use after the pointer has been deallocated is
 ** undefined behavior. So, before a stack reallocation, all pointers
 ** should be changed to offsets, and after the reallocation they should
diff --git a/ldo.h b/ldo.h
index 2d4ca8be..b6472954 100644
--- a/ldo.h
+++ b/ldo.h
@@ -89,6 +89,7 @@ LUAI_FUNC int luaD_reallocstack (lua_State *L, int newsize, int raiseerror);
 LUAI_FUNC int luaD_growstack (lua_State *L, int n, int raiseerror);
 LUAI_FUNC void luaD_shrinkstack (lua_State *L);
 LUAI_FUNC void luaD_inctop (lua_State *L);
+LUAI_FUNC int luaD_checkminstack (lua_State *L);
 
 LUAI_FUNC l_noret luaD_throw (lua_State *L, TStatus errcode);
 LUAI_FUNC l_noret luaD_throwbaselevel (lua_State *L, TStatus errcode);
diff --git a/lgc.c b/lgc.c
index 60f042c7..c64d74b8 100644
--- a/lgc.c
+++ b/lgc.c
@@ -1293,7 +1293,7 @@ static void finishgencycle (lua_State *L, global_State *g) {
   correctgraylists(g);
   checkSizes(L, g);
   g->gcstate = GCSpropagate;  /* skip restart */
-  if (!g->gcemergency)
+  if (!g->gcemergency && luaD_checkminstack(L))
     callallpendingfinalizers(L);
 }
 
@@ -1667,12 +1667,13 @@ static l_mem singlestep (lua_State *L, int fast) {
       break;
     }
     case GCScallfin: {  /* call finalizers */
-      if (g->tobefnz && !g->gcemergency) {
+      if (g->tobefnz && !g->gcemergency && luaD_checkminstack(L)) {
         g->gcstopem = 0;  /* ok collections during finalizers */
         GCTM(L);  /* call one finalizer */
         stepresult = CWUFIN;
       }
-      else {  /* emergency mode or no more finalizers */
+      else {  /* no more finalizers or emergency mode or no enough stack
+                 to run finalizers */
         g->gcstate = GCSpause;  /* finish collection */
         stepresult = step2pause;
       }