Handle OOM error on stack resize in coroutine.resume and lua_checkstack.

Thanks to Peter Cawley. #1066

1 files changed, 6 insertions, 1 deletions

diff --git a/src/lj_api.c b/src/lj_api.c
index 386bcada..d4048d79 100644
--- a/src/lj_api.c
+++ b/src/lj_api.c
@@ -104,7 +104,12 @@ LUA_API int lua_checkstack(lua_State *L, int size)
   if (size > LUAI_MAXCSTACK || (L->top - L->base + size) > LUAI_MAXCSTACK) {
     return 0;  /* Stack overflow. */
   } else if (size > 0) {
-    lj_state_checkstack(L, (MSize)size);
+    int avail = (int)(mref(L->maxstack, TValue) - L->top);
+    if (size > avail &&
+        lj_state_cpgrowstack(L, (MSize)(size - avail)) != LUA_OK) {
+      L->top--;
+      return 0;  /* Out of memory. */
+    }
   }
   return 1;
 }