1 files changed, 67 insertions, 100 deletions

diff --git a/src/lj_trace.c b/src/lj_trace.c
index 5571293e..4c9c4669 100644
--- a/src/lj_trace.c
+++ b/src/lj_trace.c
@@ -52,7 +52,7 @@ void lj_trace_err_info(jit_State *J, TraceError e)
 /* The current trace is first assembled in J->cur. The variable length
 ** arrays point to shared, growable buffers (J->irbuf etc.). The trace is
 ** kept in this state until a new trace needs to be created. Then the current
-** trace and its data structures are copied to a new (compact) Trace object.
+** trace and its data structures are copied to a new (compact) GCtrace object.
 */
 
 /* Find a free trace number. */
@@ -62,7 +62,7 @@ static TraceNo trace_findfree(jit_State *J)
   if (J->freetrace == 0)
     J->freetrace = 1;
   for (; J->freetrace < J->sizetrace; J->freetrace++)
-    if (J->trace[J->freetrace] == NULL)
+    if (traceref(J, J->freetrace) == NULL)
       return J->freetrace++;
   /* Need to grow trace array. */
   lim = (MSize)J->param[JIT_P_maxtrace] + 1;
@@ -70,9 +70,9 @@ static TraceNo trace_findfree(jit_State *J)
   osz = J->sizetrace;
   if (osz >= lim)
     return 0;  /* Too many traces. */
-  lj_mem_growvec(J->L, J->trace, J->sizetrace, lim, Trace *);
-  while (osz < J->sizetrace)
-    J->trace[osz++] = NULL;
+  lj_mem_growvec(J->L, J->trace, J->sizetrace, lim, GCRef);
+  for (; osz < J->sizetrace; osz++)
+    setgcrefnull(J->trace[osz]);
   return J->freetrace;
 }
 
@@ -82,64 +82,40 @@ static TraceNo trace_findfree(jit_State *J)
   p += T->szfield*sizeof(tp);
 
 /* Save a trace by copying and compacting it. */
-static Trace *trace_save(jit_State *J, Trace *T)
+static GCtrace *trace_save(jit_State *J, GCtrace *T)
 {
-  size_t sztr = ((sizeof(Trace)+7)&~7);
+  size_t sztr = ((sizeof(GCtrace)+7)&~7);
   size_t szins = (T->nins-T->nk)*sizeof(IRIns);
   size_t sz = sztr + szins +
               T->nsnap*sizeof(SnapShot) +
               T->nsnapmap*sizeof(SnapEntry);
-  Trace *T2 = lj_mem_newt(J->L, (MSize)sz, Trace);
+  GCtrace *T2 = lj_mem_newt(J->L, (MSize)sz, GCtrace);
   char *p = (char *)T2 + sztr;
-  memcpy(T2, T, sizeof(Trace));
+  memcpy(T2, T, sizeof(GCtrace));
+  setgcrefr(T2->nextgc, J2G(J)->gc.root);
+  setgcrefp(J2G(J)->gc.root, T2);
+  newwhite(J2G(J), T2);
+  T2->gct = ~LJ_TTRACE;
   T2->ir = (IRIns *)p - T->nk;
   memcpy(p, T->ir+T->nk, szins);
   p += szins;
   TRACE_COPYELEM(snap, nsnap, SnapShot)
   TRACE_COPYELEM(snapmap, nsnapmap, SnapEntry)
-  lj_gc_barriertrace(J2G(J), T);
   return T2;
 }
 
-/* Free a trace. */
-static void trace_free(jit_State *J, TraceNo traceno)
-{
-  lua_assert(traceno != 0);
-  if (traceno < J->freetrace)
-    J->freetrace = traceno;
-  lj_gdbjit_deltrace(J, J->trace[traceno]);
-  if (traceno == J->curtrace) {
-    lua_assert(J->trace[traceno] == &J->cur);
-    J->trace[traceno] = NULL;
-    J->curtrace = 0;
-  } else {
-    Trace *T = J->trace[traceno];
-    lua_assert(T != NULL && T != &J->cur);
-    J->trace[traceno] = NULL;
-    lj_mem_free(J2G(J), T,
-      ((sizeof(Trace)+7)&~7) + (T->nins-T->nk)*sizeof(IRIns) +
-      T->nsnap*sizeof(SnapShot) + T->nsnapmap*sizeof(SnapEntry));
-  }
-}
-
-/* Free all traces associated with a prototype. No unpatching needed. */
-void lj_trace_freeproto(global_State *g, GCproto *pt)
+void LJ_FASTCALL lj_trace_free(global_State *g, GCtrace *T)
 {
   jit_State *J = G2J(g);
-  TraceNo traceno;
-  /* Free all root traces. */
-  for (traceno = pt->trace; traceno != 0; ) {
-    TraceNo side, nextroot = J->trace[traceno]->nextroot;
-    /* Free all side traces. */
-    for (side = J->trace[traceno]->nextside; side != 0; ) {
-      TraceNo next = J->trace[side]->nextside;
-      trace_free(J, side);
-      side = next;
-    }
-    /* Now free the trace itself. */
-    trace_free(J, traceno);
-    traceno = nextroot;
+  if (T->traceno) {
+    lj_gdbjit_deltrace(J, T);
+    if (T->traceno < J->freetrace)
+      J->freetrace = T->traceno;
+    setgcrefnull(J->trace[T->traceno]);
   }
+  lj_mem_free(g, T,
+    ((sizeof(GCtrace)+7)&~7) + (T->nins-T->nk)*sizeof(IRIns) +
+    T->nsnap*sizeof(SnapShot) + T->nsnapmap*sizeof(SnapEntry));
 }
 
 /* Re-enable compiling a prototype by unpatching any modified bytecode. */
@@ -160,7 +136,7 @@ void lj_trace_reenableproto(GCproto *pt)
 }
 
 /* Unpatch the bytecode modified by a root trace. */
-static void trace_unpatch(jit_State *J, Trace *T)
+static void trace_unpatch(jit_State *J, GCtrace *T)
 {
   BCOp op = bc_op(T->startins);
   MSize pcofs = T->snap[0].mapofs + T->snap[0].nent;
@@ -171,21 +147,21 @@ static void trace_unpatch(jit_State *J, Trace *T)
     lua_assert(bc_op(*pc) == BC_JFORI);
     setbc_op(pc, BC_FORI);  /* Unpatch JFORI, too. */
     pc += bc_j(*pc);
-    lua_assert(bc_op(*pc) == BC_JFORL && J->trace[bc_d(*pc)] == T);
+    lua_assert(bc_op(*pc) == BC_JFORL && traceref(J, bc_d(*pc)) == T);
     *pc = T->startins;
     break;
   case BC_LOOP:
-    lua_assert(bc_op(*pc) == BC_JLOOP && J->trace[bc_d(*pc)] == T);
+    lua_assert(bc_op(*pc) == BC_JLOOP && traceref(J, bc_d(*pc)) == T);
     *pc = T->startins;
     break;
   case BC_ITERL:
     lua_assert(bc_op(*pc) == BC_JMP);
     pc += bc_j(*pc)+2;
-    lua_assert(bc_op(*pc) == BC_JITERL && J->trace[bc_d(*pc)] == T);
+    lua_assert(bc_op(*pc) == BC_JITERL && traceref(J, bc_d(*pc)) == T);
     *pc = T->startins;
     break;
   case BC_FUNCF:
-    lua_assert(bc_op(*pc) == BC_JFUNCF && J->trace[bc_d(*pc)] == T);
+    lua_assert(bc_op(*pc) == BC_JFUNCF && traceref(J, bc_d(*pc)) == T);
     *pc = T->startins;
     break;
   case BC_JMP:  /* No need to unpatch branches in parent traces (yet). */
@@ -195,58 +171,41 @@ static void trace_unpatch(jit_State *J, Trace *T)
   }
 }
 
-/* Free a root trace and any attached side traces. */
-static void trace_freeroot(jit_State *J, Trace *T, TraceNo traceno)
+/* Flush a root trace. */
+static void trace_flushroot(jit_State *J, GCtrace *T)
 {
   GCproto *pt = &gcref(T->startpt)->pt;
-  TraceNo side;
   lua_assert(T->root == 0 && pt != NULL);
   /* First unpatch any modified bytecode. */
   trace_unpatch(J, T);
   /* Unlink root trace from chain anchored in prototype. */
-  if (pt->trace == traceno) {  /* Trace is first in chain. Easy. */
+  if (pt->trace == T->traceno) {  /* Trace is first in chain. Easy. */
     pt->trace = T->nextroot;
   } else {  /* Otherwise search in chain of root traces. */
-    Trace *T2 = J->trace[pt->trace];
-    while (T2->nextroot != traceno) {
+    GCtrace *T2 = traceref(J, pt->trace);
+    while (T2->nextroot != T->traceno) {
       lua_assert(T2->nextroot != 0);
-      T2 = J->trace[T2->nextroot];
+      T2 = traceref(J, T2->nextroot);
     }
     T2->nextroot = T->nextroot;  /* Unlink from chain. */
   }
-  /* Free all side traces. */
-  for (side = T->nextside; side != 0; ) {
-    TraceNo next = J->trace[side]->nextside;
-    trace_free(J, side);
-    side = next;
-  }
-  /* Now free the trace itself. */
-  trace_free(J, traceno);
 }
 
-/* Flush a root trace + side traces, if there are no links to it. */
-int lj_trace_flush(jit_State *J, TraceNo traceno)
+/* Flush a trace. Only root traces are considered. */
+void lj_trace_flush(jit_State *J, TraceNo traceno)
 {
   if (traceno > 0 && traceno < J->sizetrace) {
-    Trace *T = J->trace[traceno];
-    if (T && T->root == 0) {
-      ptrdiff_t i;
-      for (i = (ptrdiff_t)J->sizetrace-1; i > 0; i--)
-        if (i != (ptrdiff_t)traceno && J->trace[i] &&
-            J->trace[i]->root != traceno && J->trace[i]->link == traceno)
-          return 0;  /* Failed: existing link to trace. */
-      trace_freeroot(J, T, traceno);
-      return 1;  /* Ok. */
-    }
+    GCtrace *T = traceref(J, traceno);
+    if (T && T->root == 0)
+      trace_flushroot(J, T);
   }
-  return 0;  /* Failed. */
 }
 
 /* Flush all traces associated with a prototype. */
 void lj_trace_flushproto(global_State *g, GCproto *pt)
 {
   while (pt->trace != 0)
-    trace_freeroot(G2J(g), G2J(g)->trace[pt->trace], pt->trace);
+    trace_flushroot(G2J(g), traceref(G2J(g), pt->trace));
 }
 
 /* Flush all traces. */
@@ -257,14 +216,16 @@ int lj_trace_flushall(lua_State *L)
   if ((J2G(J)->hookmask & HOOK_GC))
     return 1;
   for (i = (ptrdiff_t)J->sizetrace-1; i > 0; i--) {
-    Trace *T = J->trace[i];
-    if (T && T->root == 0)
-      trace_freeroot(J, T, (TraceNo)i);
+    GCtrace *T = traceref(J, i);
+    if (T) {
+      if (T->root == 0)
+        trace_flushroot(J, T);
+      lj_gdbjit_deltrace(J, T);
+      T->traceno = 0;
+      setgcrefnull(J->trace[i]);
+    }
   }
-#ifdef LUA_USE_ASSERT
-  for (i = 0; i < (ptrdiff_t)J->sizetrace; i++)
-    lua_assert(J->trace[i] == NULL);
-#endif
+  J->curtrace = 0;
   J->freetrace = 0;
   /* Free the whole machine code and invalidate all exit stub groups. */
   lj_mcode_free(J);
@@ -293,11 +254,13 @@ void lj_trace_initstate(global_State *g)
 void lj_trace_freestate(global_State *g)
 {
   jit_State *J = G2J(g);
+  if (J->curtrace)
+    lj_gdbjit_deltrace(J, &J->cur);
 #ifdef LUA_USE_ASSERT
   {  /* This assumes all traces have already been freed. */
     ptrdiff_t i;
-    for (i = 0; i < (ptrdiff_t)J->sizetrace; i++)
-      lua_assert(J->trace[i] == NULL);
+    for (i = 1; i < (ptrdiff_t)J->sizetrace; i++)
+      lua_assert(i == (ptrdiff_t)J->curtrace || traceref(J, i) == NULL);
   }
 #endif
   lj_mcode_free(J);
@@ -305,7 +268,7 @@ void lj_trace_freestate(global_State *g)
   lj_mem_freevec(g, J->snapmapbuf, J->sizesnapmap, SnapEntry);
   lj_mem_freevec(g, J->snapbuf, J->sizesnap, SnapShot);
   lj_mem_freevec(g, J->irbuf + J->irbotlim, J->irtoplim - J->irbotlim, IRIns);
-  lj_mem_freevec(g, J->trace, J->sizetrace, Trace *);
+  lj_mem_freevec(g, J->trace, J->sizetrace, GCRef);
 }
 
 /* -- Penalties and blacklisting ------------------------------------------ */
@@ -349,9 +312,11 @@ static void trace_start(jit_State *J)
 {
   lua_State *L;
 
-  if (J->curtrace != 0 && J->trace[J->curtrace] == &J->cur) {
-    J->trace[J->curtrace] = trace_save(J, &J->cur);  /* Save current trace. */
+  if (J->curtrace != 0 && traceref(J, J->curtrace) == &J->cur) {
+    TraceNo tr = J->curtrace;  /* Save current trace. */
+    setgcrefp(J->trace[tr], trace_save(J, &J->cur));
     J->curtrace = 0;
+    lj_gc_barriertrace(J2G(J), tr);
   }
 
   if ((J->pt->flags & PROTO_NO_JIT)) {  /* JIT disabled for this proto? */
@@ -374,10 +339,11 @@ static void trace_start(jit_State *J)
     J->state = LJ_TRACE_IDLE;  /* Silently ignored. */
     return;
   }
-  J->trace[J->curtrace] = &J->cur;
+  setgcrefp(J->trace[J->curtrace], &J->cur);
 
   /* Setup enough of the current trace to be able to send the vmevent. */
-  memset(&J->cur, 0, sizeof(Trace));
+  memset(&J->cur, 0, sizeof(GCtrace));
+  J->cur.traceno = J->curtrace;
   J->cur.nins = J->cur.nk = REF_BASE;
   J->cur.ir = J->irbuf;
   J->cur.snap = J->snapbuf;
@@ -385,6 +351,7 @@ static void trace_start(jit_State *J)
   J->mergesnap = 0;
   J->needsnap = 0;
   J->guardemit.irt = 0;
+  setgcref(J->cur.startpt, obj2gco(J->pt));
 
   L = J->L;
   lj_vmevent_send(L, TRACE,
@@ -431,12 +398,12 @@ static void trace_stop(jit_State *J)
   case BC_JMP:
     /* Patch exit branch in parent to side trace entry. */
     lua_assert(J->parent != 0 && J->cur.root != 0);
-    lj_asm_patchexit(J, J->trace[J->parent], J->exitno, J->cur.mcode);
+    lj_asm_patchexit(J, traceref(J, J->parent), J->exitno, J->cur.mcode);
     /* Avoid compiling a side trace twice (stack resizing uses parent exit). */
-    J->trace[J->parent]->snap[J->exitno].count = SNAPCOUNT_DONE;
+    traceref(J, J->parent)->snap[J->exitno].count = SNAPCOUNT_DONE;
     /* Add to side trace chain in root trace. */
     {
-      Trace *root = J->trace[J->cur.root];
+      GCtrace *root = traceref(J, J->cur.root);
       root->nchild++;
       J->cur.nextside = root->nextside;
       root->nextside = (TraceNo1)J->curtrace;
@@ -510,7 +477,7 @@ static int trace_abort(jit_State *J)
       copyTV(L, L->top++, &J->errinfo);
     );
     /* Drop aborted trace after the vmevent (which may still access it). */
-    J->trace[J->curtrace] = NULL;
+    setgcrefnull(J->trace[J->curtrace]);
     if (J->curtrace < J->freetrace)
       J->freetrace = J->curtrace;
     J->curtrace = 0;
@@ -631,7 +598,7 @@ void LJ_FASTCALL lj_trace_hot(jit_State *J, const BCIns *pc)
 /* Check for a hot side exit. If yes, start recording a side trace. */
 static void trace_hotside(jit_State *J, const BCIns *pc)
 {
-  SnapShot *snap = &J->trace[J->parent]->snap[J->exitno];
+  SnapShot *snap = &traceref(J, J->parent)->snap[J->exitno];
   if (!(J2G(J)->hookmask & (HOOK_GC|HOOK_VMEVENT)) &&
       snap->count != SNAPCOUNT_DONE &&
       ++snap->count >= J->param[JIT_P_hotexit]) {
@@ -699,7 +666,7 @@ int LJ_FASTCALL lj_trace_exit(jit_State *J, void *exptr)
   else
     trace_hotside(J, pc);
   if (bc_op(*pc) == BC_JLOOP) {
-    BCIns *retpc = &J->trace[bc_d(*pc)]->startins;
+    BCIns *retpc = &traceref(J, bc_d(*pc))->startins;
     if (bc_isret(bc_op(*retpc))) {
       if (J->state == LJ_TRACE_RECORD) {
         J->patchins = *pc;