Add explicit IR_GCSTEP instruction.

6 files changed, 27 insertions, 7 deletions

diff --git a/src/lj_asm.c b/src/lj_asm.c
index e8e823c3..dccc4eab 100644
--- a/src/lj_asm.c
+++ b/src/lj_asm.c
@@ -71,7 +71,7 @@ typedef struct ASMState {
   IRRef loopref;        /* Reference of LOOP instruction (or 0). */
 
   BCReg topslot;        /* Number of slots for stack check (unless 0). */
-  MSize gcsteps;        /* Accumulated number of GC steps (per section). */
+  int32_t gcsteps;      /* Accumulated number of GC steps (per section). */
 
   GCtrace *T;           /* Trace to assemble. */
   GCtrace *parent;      /* Parent trace (or NULL). */
@@ -972,6 +972,22 @@ static void asm_tdup(ASMState *as, IRIns *ir)
   asm_gencall(as, ci, args);
 }
 
+static void asm_gc_check(ASMState *as);
+
+/* Explicit GC step. */
+static void asm_gcstep(ASMState *as, IRIns *ir)
+{
+  IRIns *ira;
+  for (ira = IR(as->stopins+1); ira < ir; ira++)
+    if ((ira->o == IR_TNEW || ira->o == IR_TDUP ||
+         (LJ_HASFFI && (ira->o == IR_CNEW || ira->o == IR_CNEWI))) &&
+        ra_used(ira))
+      as->gcsteps++;
+  if (as->gcsteps)
+    asm_gc_check(as);
+  as->gcsteps = 0x80000000;  /* Prevent implicit GC check further up. */
+}
+
 /* -- PHI and loop handling ----------------------------------------------- */
 
 /* Break a PHI cycle by renaming to a free register (evict if needed). */
@@ -1191,7 +1207,6 @@ static void asm_phi(ASMState *as, IRIns *ir)
   }
 }
 
-static void asm_gc_check(ASMState *as);
 static void asm_loop_fixup(ASMState *as);
 
 /* Middle part of a loop. */
@@ -1757,7 +1772,7 @@ void lj_asm_trace(jit_State *J, GCtrace *T)
   /* Emit head of trace. */
   RA_DBG_REF();
   checkmclim(as);
-  if (as->gcsteps) {
+  if (as->gcsteps > 0) {
     as->curins = as->T->snap[0].ref;
     asm_snap_prep(as);  /* The GC check is a guard. */
     asm_gc_check(as);
diff --git a/src/lj_asm_arm.h b/src/lj_asm_arm.h
index c08b6196..e6ab3573 100644
--- a/src/lj_asm_arm.h
+++ b/src/lj_asm_arm.h
@@ -1532,7 +1532,7 @@ static void asm_gc_check(ASMState *as)
   asm_gencall(as, ci, args);
   tmp1 = ra_releasetmp(as, ASMREF_TMP1);
   tmp2 = ra_releasetmp(as, ASMREF_TMP2);
-  emit_loadi(as, tmp2, (int32_t)as->gcsteps);
+  emit_loadi(as, tmp2, as->gcsteps);
   /* Jump around GC step if GC total < GC threshold. */
   emit_branch(as, ARMF_CC(ARMI_B, CC_LS), l_end);
   emit_nm(as, ARMI_CMP, RID_TMP, tmp2);
@@ -1646,6 +1646,7 @@ static void asm_ir(ASMState *as, IRIns *ir)
   case IR_USE: ra_alloc1(as, ir->op1, RSET_GPR); break;
   case IR_PHI: asm_phi(as, ir); break;
   case IR_HIOP: asm_hiop(as, ir); break;
+  case IR_GCSTEP: asm_gcstep(as, ir); break;
 
   /* Guarded assertions. */
   case IR_EQ: case IR_NE:
diff --git a/src/lj_asm_mips.h b/src/lj_asm_mips.h
index a3a4da6c..b42f9f9a 100644
--- a/src/lj_asm_mips.h
+++ b/src/lj_asm_mips.h
@@ -1673,7 +1673,7 @@ static void asm_gc_check(ASMState *as)
   asm_gencall(as, ci, args);
   emit_tsi(as, MIPSI_ADDIU, ra_releasetmp(as, ASMREF_TMP1), RID_JGL, -32768);
   tmp = ra_releasetmp(as, ASMREF_TMP2);
-  emit_loadi(as, tmp, (int32_t)as->gcsteps);
+  emit_loadi(as, tmp, as->gcsteps);
   /* Jump around GC step if GC total < GC threshold. */
   emit_branch(as, MIPSI_BNE, RID_TMP, RID_ZERO, l_end);
   emit_dst(as, MIPSI_SLTU, RID_TMP, RID_TMP, tmp);
@@ -1770,6 +1770,7 @@ static void asm_ir(ASMState *as, IRIns *ir)
     ra_alloc1(as, ir->op1, irt_isfp(ir->t) ? RSET_FPR : RSET_GPR); break;
   case IR_PHI: asm_phi(as, ir); break;
   case IR_HIOP: asm_hiop(as, ir); break;
+  case IR_GCSTEP: asm_gcstep(as, ir); break;
 
   /* Guarded assertions. */
   case IR_EQ: case IR_NE: asm_compeq(as, ir); break;
diff --git a/src/lj_asm_ppc.h b/src/lj_asm_ppc.h
index ec22e260..5d538fc8 100644
--- a/src/lj_asm_ppc.h
+++ b/src/lj_asm_ppc.h
@@ -1856,7 +1856,7 @@ static void asm_gc_check(ASMState *as)
   asm_gencall(as, ci, args);
   emit_tai(as, PPCI_ADDI, ra_releasetmp(as, ASMREF_TMP1), RID_JGL, -32768);
   tmp = ra_releasetmp(as, ASMREF_TMP2);
-  emit_loadi(as, tmp, (int32_t)as->gcsteps);
+  emit_loadi(as, tmp, as->gcsteps);
   /* Jump around GC step if GC total < GC threshold. */
   emit_condbranch(as, PPCI_BC|PPCF_Y, CC_LT, l_end);
   emit_ab(as, PPCI_CMPLW, RID_TMP, tmp);
@@ -1966,6 +1966,7 @@ static void asm_ir(ASMState *as, IRIns *ir)
     ra_alloc1(as, ir->op1, irt_isfp(ir->t) ? RSET_FPR : RSET_GPR); break;
   case IR_PHI: asm_phi(as, ir); break;
   case IR_HIOP: asm_hiop(as, ir); break;
+  case IR_GCSTEP: asm_gcstep(as, ir); break;
 
   /* Guarded assertions. */
   case IR_EQ: case IR_NE:
diff --git a/src/lj_asm_x86.h b/src/lj_asm_x86.h
index 7647b03f..4537e1d5 100644
--- a/src/lj_asm_x86.h
+++ b/src/lj_asm_x86.h
@@ -2379,7 +2379,7 @@ static void asm_gc_check(ASMState *as)
   asm_gencall(as, ci, args);
   tmp = ra_releasetmp(as, ASMREF_TMP1);
   emit_loada(as, tmp, J2G(as->J));
-  emit_loadi(as, ra_releasetmp(as, ASMREF_TMP2), (int32_t)as->gcsteps);
+  emit_loadi(as, ra_releasetmp(as, ASMREF_TMP2), as->gcsteps);
   /* Jump around GC step if GC total < GC threshold. */
   emit_sjcc(as, CC_B, l_end);
   emit_opgl(as, XO_ARITH(XOg_CMP), tmp, gc.threshold);
@@ -2556,6 +2556,7 @@ static void asm_ir(ASMState *as, IRIns *ir)
     ra_alloc1(as, ir->op1, irt_isfp(ir->t) ? RSET_FPR : RSET_GPR); break;
   case IR_PHI: asm_phi(as, ir); break;
   case IR_HIOP: asm_hiop(as, ir); break;
+  case IR_GCSTEP: asm_gcstep(as, ir); break;
 
   /* Guarded assertions. */
   case IR_LT: case IR_GE: case IR_LE: case IR_GT:
diff --git a/src/lj_ir.h b/src/lj_ir.h
index a8f48ffb..b10c65df 100644
--- a/src/lj_ir.h
+++ b/src/lj_ir.h
@@ -33,6 +33,7 @@
   /* Miscellaneous ops. */ \
   _(NOP,        N , ___, ___) \
   _(BASE,       N , lit, lit) \
+  _(GCSTEP,     S , ___, ___) \
   _(HIOP,       S , ref, ref) \
   _(LOOP,       S , ___, ___) \
   _(USE,        S , ref, ___) \