Use g->jit_base for on/off-trace detection.

9 files changed, 31 insertions, 24 deletions

diff --git a/src/lj_ccallback.c b/src/lj_ccallback.c
index c975c88f..37f45bf5 100644
--- a/src/lj_ccallback.c
+++ b/src/lj_ccallback.c
@@ -529,7 +529,7 @@ lua_State * LJ_FASTCALL lj_ccallback_enter(CTState *cts, void *cf)
   lua_State *L = cts->L;
   global_State *g = cts->g;
   lua_assert(L != NULL);
-  if (gcref(g->jit_L)) {
+  if (tvref(g->jit_base)) {
     setstrV(L, L->top++, lj_err_str(L, LJ_ERR_FFI_BADCBACK));
     if (g->panic) g->panic(L);
     exit(EXIT_FAILURE);
diff --git a/src/lj_err.c b/src/lj_err.c
index 7ae87a82..3b44c984 100644
--- a/src/lj_err.c
+++ b/src/lj_err.c
@@ -453,7 +453,7 @@ LJ_NOINLINE void LJ_FASTCALL lj_err_throw(lua_State *L, int errcode)
 {
   global_State *g = G(L);
   lj_trace_abort(g);
-  setgcrefnull(g->jit_L);
+  setmref(g->jit_base, NULL);
   L->status = 0;
 #if LJ_UNWIND_EXT
   err_raise_ext(errcode);
diff --git a/src/lj_gc.c b/src/lj_gc.c
index c9eaf21a..8dbf2090 100644
--- a/src/lj_gc.c
+++ b/src/lj_gc.c
@@ -475,7 +475,7 @@ static void gc_finalize(lua_State *L)
   global_State *g = G(L);
   GCobj *o = gcnext(gcref(g->gc.mmudata));
   cTValue *mo;
-  lua_assert(gcref(g->jit_L) == NULL);  /* Must not be called on trace. */
+  lua_assert(tvref(g->jit_base) == NULL);  /* Must not be called on trace. */
   /* Unchain from list of userdata to be finalized. */
   if (o == gcref(g->gc.mmudata))
     setgcrefnull(g->gc.mmudata);
@@ -606,7 +606,7 @@ static size_t gc_onestep(lua_State *L)
     g->gc.state = GCSatomic;  /* End of mark phase. */
     return 0;
   case GCSatomic:
-    if (gcref(g->jit_L))  /* Don't run atomic phase on trace. */
+    if (tvref(g->jit_base))  /* Don't run atomic phase on trace. */
       return LJ_MAX_MEM;
     atomic(g, L);
     g->gc.state = GCSsweepstring;  /* Start of sweep phase. */
@@ -640,7 +640,7 @@ static size_t gc_onestep(lua_State *L)
     }
   case GCSfinalize:
     if (gcref(g->gc.mmudata) != NULL) {
-      if (gcref(g->jit_L))  /* Don't call finalizers on trace. */
+      if (tvref(g->jit_base))  /* Don't call finalizers on trace. */
         return LJ_MAX_MEM;
       gc_finalize(L);  /* Finalize one userdata object. */
       if (g->gc.estimate > GCFINALIZECOST)
@@ -697,7 +697,7 @@ void LJ_FASTCALL lj_gc_step_fixtop(lua_State *L)
 int LJ_FASTCALL lj_gc_step_jit(global_State *g, MSize steps)
 {
   lua_State *L = gco2th(gcref(g->jit_L));
-  L->base = mref(G(L)->jit_base, TValue);
+  L->base = tvref(G(L)->jit_base);
   L->top = curr_topL(L);
   while (steps-- > 0 && lj_gc_step(L) == 0)
     ;
diff --git a/src/lj_obj.h b/src/lj_obj.h
index 5a05f38d..e97acef0 100644
--- a/src/lj_obj.h
+++ b/src/lj_obj.h
@@ -536,8 +536,8 @@ typedef struct global_State {
   lua_CFunction panic;  /* Called as a last resort for errors. */
   BCIns bc_cfunc_int;   /* Bytecode for internal C function calls. */
   BCIns bc_cfunc_ext;   /* Bytecode for external C function calls. */
-  GCRef jit_L;          /* Current JIT code lua_State or NULL. */
-  MRef jit_base;        /* Current JIT code L->base. */
+  GCRef jit_L;          /* Current JIT code lua_State. */
+  MRef jit_base;        /* Current JIT code L->base or NULL. */
   MRef ctype_state;     /* Pointer to C type state. */
   GCRef gcroot[GCROOT_MAX];  /* GC roots. */
 } global_State;
diff --git a/src/lj_state.c b/src/lj_state.c
index 604ff886..486e66e3 100644
--- a/src/lj_state.c
+++ b/src/lj_state.c
@@ -60,7 +60,7 @@ static void resizestack(lua_State *L, MSize n)
   GCobj *up;
   lua_assert((MSize)(tvref(L->maxstack)-oldst)==L->stacksize-LJ_STACK_EXTRA-1);
   st = (TValue *)lj_mem_realloc(L, tvref(L->stack),
-                                (MSize)(L->stacksize*sizeof(TValue)),
+                                (MSize)(oldsize*sizeof(TValue)),
                                 (MSize)(realsize*sizeof(TValue)));
   setmref(L->stack, st);
   delta = (char *)st - (char *)oldst;
@@ -68,12 +68,12 @@ static void resizestack(lua_State *L, MSize n)
   while (oldsize < realsize)  /* Clear new slots. */
     setnilV(st + oldsize++);
   L->stacksize = realsize;
+  if ((size_t)(mref(G(L)->jit_base, char) - (char *)oldst) < oldsize)
+    setmref(G(L)->jit_base, mref(G(L)->jit_base, char) + delta);
   L->base = (TValue *)((char *)L->base + delta);
   L->top = (TValue *)((char *)L->top + delta);
   for (up = gcref(L->openupval); up != NULL; up = gcnext(up))
     setmref(gco2uv(up)->v, (TValue *)((char *)uvval(gco2uv(up)) + delta));
-  if (obj2gco(L) == gcref(G(L)->jit_L))
-    setmref(G(L)->jit_base, mref(G(L)->jit_base, char) + delta);
 }
 
 /* Relimit stack after error, in case the limit was overdrawn. */
@@ -90,7 +90,8 @@ void lj_state_shrinkstack(lua_State *L, MSize used)
     return;  /* Avoid stack shrinking while handling stack overflow. */
   if (4*used < L->stacksize &&
       2*(LJ_STACK_START+LJ_STACK_EXTRA) < L->stacksize &&
-      obj2gco(L) != gcref(G(L)->jit_L))  /* Don't shrink stack of live trace. */
+      /* Don't shrink stack of live trace. */
+      (tvref(G(L)->jit_base) == NULL || obj2gco(L) != gcref(G(L)->jit_L)))
     resizestack(L, L->stacksize >> 1);
 }
 
diff --git a/src/vm_arm.dasc b/src/vm_arm.dasc
index 83341e41..b235967f 100644
--- a/src/vm_arm.dasc
+++ b/src/vm_arm.dasc
@@ -2107,9 +2107,9 @@ static void build_subroutines(BuildCtx *ctx)
   |    ldr BASE, [DISPATCH, #DISPATCH_GL(jit_base)]
   |   str CARG1, [DISPATCH, #DISPATCH_J(exitno)]
   |   mov CARG4, #0
-  |  str L, [DISPATCH, #DISPATCH_J(L)]
   |    str BASE, L->base
-  |   str CARG4, [DISPATCH, #DISPATCH_GL(jit_L)]
+  |  str L, [DISPATCH, #DISPATCH_J(L)]
+  |   str CARG4, [DISPATCH, #DISPATCH_GL(jit_base)]
   |  sub CARG1, DISPATCH, #-GG_DISP2J
   |  mov CARG2, sp
   |  bl extern lj_trace_exit            // (jit_State *J, ExitState *ex)
@@ -2133,8 +2133,9 @@ static void build_subroutines(BuildCtx *ctx)
   |  ldr LFUNC:CARG2, [BASE, FRAME_FUNC]
   |   str RC, SAVE_MULTRES
   |   mov CARG3, #0
+  |   str BASE, L->base
   |  ldr CARG2, LFUNC:CARG2->field_pc
-  |   str CARG3, [DISPATCH, #DISPATCH_GL(jit_L)]
+  |   str CARG3, [DISPATCH, #DISPATCH_GL(jit_base)]
   |    mv_vmstate CARG4, INTERP
   |  ldr KBASE, [CARG2, #PC2PROTO(k)]
   |  // Modified copy of ins_next which handles function header dispatch, too.
diff --git a/src/vm_mips.dasc b/src/vm_mips.dasc
index 41ed303d..aad4fccf 100644
--- a/src/vm_mips.dasc
+++ b/src/vm_mips.dasc
@@ -2046,13 +2046,14 @@ static void build_subroutines(BuildCtx *ctx)
   |  st_vmstate
   |   sw TMP2, 16+32*8+29*4(sp)         // Store sp in RID_SP.
   |  lw L, DISPATCH_GL(jit_L)(DISPATCH)
-  |  lw BASE, DISPATCH_GL(jit_base)(DISPATCH)
+  |   lw BASE, DISPATCH_GL(jit_base)(DISPATCH)
   |  load_got lj_trace_exit
   |  sw L, DISPATCH_J(L)(DISPATCH)
   |  sw ra, DISPATCH_J(parent)(DISPATCH)  // Store trace number.
+  |   sw BASE, L->base
   |  sw TMP1, DISPATCH_J(exitno)(DISPATCH)  // Store exit number.
   |  addiu CARG1, DISPATCH, GG_DISP2J
-  |  sw BASE, L->base
+  |   sw r0, DISPATCH_GL(jit_base)(DISPATCH)
   |  call_intern lj_trace_exit          // (jit_State *J, ExitState *ex)
   |.  addiu CARG2, sp, 16
   |  // Returns MULTRES (unscaled) or negated error code.
@@ -2068,7 +2069,8 @@ static void build_subroutines(BuildCtx *ctx)
   |.if JIT
   |  // CRET1 = MULTRES or negated error code, BASE, PC and JGL set.
   |  lw L, SAVE_L
-  |  addiu DISPATCH, JGL, -GG_DISP2G-32768
+  |   addiu DISPATCH, JGL, -GG_DISP2G-32768
+  |  sw BASE, L->base
   |1:
   |  bltz CRET1, >3                     // Check for error from exit.
   |.  lw LFUNC:TMP1, FRAME_FUNC(BASE)
@@ -2078,7 +2080,7 @@ static void build_subroutines(BuildCtx *ctx)
   |  sw MULTRES, SAVE_MULTRES
   |    mtc1 TMP3, TOBIT
   |  lw TMP1, LFUNC:TMP1->pc
-  |   sw r0, DISPATCH_GL(jit_L)(DISPATCH)
+  |   sw r0, DISPATCH_GL(jit_base)(DISPATCH)
   |  lw KBASE, PC2PROTO(k)(TMP1)
   |    cvt.d.s TOBIT, TOBIT
   |  // Modified copy of ins_next which handles function header dispatch, too.
diff --git a/src/vm_ppc.dasc b/src/vm_ppc.dasc
index a7707ccc..d009d005 100644
--- a/src/vm_ppc.dasc
+++ b/src/vm_ppc.dasc
@@ -2542,9 +2542,9 @@ static void build_subroutines(BuildCtx *ctx)
   |   srwi CARG3, CARG3, 2
   |  stw L, DISPATCH_J(L)(DISPATCH)
   |   subi CARG3, CARG3, 2
-  |  stw TMP1, DISPATCH_GL(jit_L)(DISPATCH)
-  |   stw CARG4, DISPATCH_J(parent)(DISPATCH)
   |  stp BASE, L->base
+  |   stw CARG4, DISPATCH_J(parent)(DISPATCH)
+  |  stw TMP1, DISPATCH_GL(jit_base)(DISPATCH)
   |  addi CARG1, DISPATCH, GG_DISP2J
   |   stw CARG3, DISPATCH_J(exitno)(DISPATCH)
   |  addi CARG2, sp, 16
@@ -2568,6 +2568,7 @@ static void build_subroutines(BuildCtx *ctx)
   |  // CARG1 = MULTRES or negated error code, BASE, PC and JGL set.
   |  lwz L, SAVE_L
   |  addi DISPATCH, JGL, -GG_DISP2G-32768
+  |  stp BASE, L->base
   |1:
   |  cmpwi CARG1, 0
   |  blt >3                             // Check for error from exit.
@@ -2576,7 +2577,7 @@ static void build_subroutines(BuildCtx *ctx)
   |    li TMP2, 0
   |   stw MULTRES, SAVE_MULTRES
   |  lwz TMP1, LFUNC:TMP1->pc
-  |    stw TMP2, DISPATCH_GL(jit_L)(DISPATCH)
+  |    stw TMP2, DISPATCH_GL(jit_base)(DISPATCH)
   |  lwz KBASE, PC2PROTO(k)(TMP1)
   |  // Setup type comparison constants.
   |  li TISNUM, LJ_TISNUM
diff --git a/src/vm_x86.dasc b/src/vm_x86.dasc
index 62a5e139..b8f1c449 100644
--- a/src/vm_x86.dasc
+++ b/src/vm_x86.dasc
@@ -2708,7 +2708,6 @@ static void build_subroutines(BuildCtx *ctx)
   |  mov L:RB, [DISPATCH+DISPATCH_GL(jit_L)]
   |  mov BASE, [DISPATCH+DISPATCH_GL(jit_base)]
   |  mov aword [DISPATCH+DISPATCH_J(L)], L:RBa
-  |  mov dword [DISPATCH+DISPATCH_GL(jit_L)], 0
   |  mov L:RB->base, BASE
   |.if X64WIN
   |  lea CARG2, [rsp+4*8]
@@ -2718,6 +2717,7 @@ static void build_subroutines(BuildCtx *ctx)
   |  lea FCARG2, [esp+16]
   |.endif
   |  lea FCARG1, [DISPATCH+GG_DISP2J]
+  |  mov dword [DISPATCH+DISPATCH_GL(jit_base)], 0
   |  call extern lj_trace_exit@8        // (jit_State *J, ExitState *ex)
   |  // MULTRES or negated error code returned in eax (RD).
   |  mov RAa, L:RB->cframe
@@ -2765,11 +2765,13 @@ static void build_subroutines(BuildCtx *ctx)
   |  mov r12, TMPQ
   |.endif
   |  test RD, RD; js >3                 // Check for error from exit.
+  |  mov L:RB, SAVE_L
   |  mov MULTRES, RD
   |  mov LFUNC:KBASE, [BASE-8]
   |  mov KBASE, LFUNC:KBASE->pc
   |  mov KBASE, [KBASE+PC2PROTO(k)]
-  |  mov dword [DISPATCH+DISPATCH_GL(jit_L)], 0
+  |  mov L:RB->base, BASE
+  |  mov dword [DISPATCH+DISPATCH_GL(jit_base)], 0
   |  set_vmstate INTERP
   |  // Modified copy of ins_next which handles function header dispatch, too.
   |  mov RC, [PC]