LJ_FR2: Fix stack checks in vararg calls.

Thanks to Peter Cawley. #1048

4 files changed, 4 insertions, 2 deletions

diff --git a/src/lj_def.h b/src/lj_def.h
index 1461d3d7..0d6c346b 100644
--- a/src/lj_def.h
+++ b/src/lj_def.h
@@ -69,7 +69,7 @@ typedef unsigned int uintptr_t;
 #define LJ_MAX_UPVAL    60              /* Max. # of upvalues. */
 
 #define LJ_MAX_IDXCHAIN 100             /* __index/__newindex chain limit. */
-#define LJ_STACK_EXTRA  (5+2*LJ_FR2)    /* Extra stack space (metamethods). */
+#define LJ_STACK_EXTRA  (5+3*LJ_FR2)    /* Extra stack space (metamethods). */
 
 #define LJ_NUM_CBPAGE   1               /* Number of FFI callback pages. */
 
diff --git a/src/lj_dispatch.c b/src/lj_dispatch.c
index 57809e62..b9748bba 100644
--- a/src/lj_dispatch.c
+++ b/src/lj_dispatch.c
@@ -453,7 +453,7 @@ static int call_init(lua_State *L, GCfunc *fn)
     int numparams = pt->numparams;
     int gotparams = (int)(L->top - L->base);
     int need = pt->framesize;
-    if ((pt->flags & PROTO_VARARG)) need += 1+gotparams;
+    if ((pt->flags & PROTO_VARARG)) need += 1+LJ_FR2+gotparams;
     lj_state_checkstack(L, (MSize)need);
     numparams -= gotparams;
     return numparams >= 0 ? numparams : 0;
diff --git a/src/vm_arm64.dasc b/src/vm_arm64.dasc
index 61a3ba6d..3044a8ac 100644
--- a/src/vm_arm64.dasc
+++ b/src/vm_arm64.dasc
@@ -3916,6 +3916,7 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
     |   add TMP2, BASE, RC
     |   add LFUNC:CARG3, CARG3, TMP0, lsl #47
     |  add RA, RA, RC
+    |  sub CARG1, CARG1, #8
     |   add TMP0, RC, #16+FRAME_VARG
     |   str LFUNC:CARG3, [TMP2], #8     // Store (tagged) copy of LFUNC.
     |    ldr KBASE, [PC, #-4+PC2PROTO(k)]
diff --git a/src/vm_mips64.dasc b/src/vm_mips64.dasc
index 6c215f2b..ef0d901d 100644
--- a/src/vm_mips64.dasc
+++ b/src/vm_mips64.dasc
@@ -5396,6 +5396,7 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
     |   settp LFUNC:RB, TMP0
     |  daddu TMP0, RA, RC
     |   sd LFUNC:RB, 0(TMP1)            // Store (tagged) copy of LFUNC.
+    |  daddiu TMP2, TMP2, -8
     |   daddiu TMP3, RC, 16+FRAME_VARG
     |  sltu AT, TMP0, TMP2
     |    ld KBASE, -4+PC2PROTO(k)(PC)