BUG: Arithmetic overflow in 'collectgarbage"step"'

The computation of a new debt could overflow when we give a too large
step to 'collectgarbage"step"' and the current debt was already
negative. This is only an issue if your platform cares for it or if you
compile Lua with an option like '-fsanitize=undefined'.

1 files changed, 7 insertions, 2 deletions

diff --git a/lapi.c b/lapi.c
index 42c8fcdd..9b6ca1ec 100644
--- a/lapi.c
+++ b/lapi.c
@@ -1201,11 +1201,16 @@ LUA_API int lua_gc (lua_State *L, int what, ...) {
     case LUA_GCSTEP: {
       lu_byte oldstp = g->gcstp;
       l_mem n = cast(l_mem, va_arg(argp, size_t));
+      l_mem newdebt;
       int work = 0;  /* true if GC did some work */
       g->gcstp = 0;  /* allow GC to run (other bits must be zero here) */
       if (n <= 0)
-        n = g->GCdebt;  /* force to run one basic step */
-      luaE_setdebt(g, g->GCdebt - n);
+        newdebt = 0;  /* force to run one basic step */
+      else if (g->GCdebt >= n - MAX_LMEM)  /* no overflow? */
+        newdebt = g->GCdebt - n;
+      else  /* overflow */
+        newdebt = -MAX_LMEM;  /* set debt to miminum value */
+      luaE_setdebt(g, newdebt);
       luaC_condGC(L, (void)0, work = 1);
       if (work && g->gcstate == GCSpause)  /* end of cycle? */
         res = 1;  /* signal it */