Eliminate IR_FRAME. Replace with KGC and TRef/SnapEntry flags.

4 files changed, 81 insertions, 105 deletions

diff --git a/src/lj_asm.c b/src/lj_asm.c
index 3912bbeb..48b6ec5a 100644
--- a/src/lj_asm.c
+++ b/src/lj_asm.c
@@ -931,7 +931,7 @@ static void asm_snap_alloc(ASMState *as)
     IRRef ref = snap_ref(map[n]);
     if (!irref_isk(ref)) {
       IRIns *ir = IR(ref);
-      if (!ra_used(ir) && ir->o != IR_FRAME) {
+      if (!ra_used(ir)) {
         RegSet allow = irt_isnum(ir->t) ? RSET_FPR : RSET_GPR;
         /* Not a var-to-invar ref and got a free register (or a remat)? */
         if ((!iscrossref(as, ref) || irt_isphi(ir->t)) &&
@@ -2831,27 +2831,25 @@ static void asm_head_side(ASMState *as)
   /* Scan all parent SLOADs and collect register dependencies. */
   for (i = as->curins; i > REF_BASE; i--) {
     IRIns *ir = IR(i);
-    lua_assert((ir->o == IR_SLOAD && (ir->op2 & IRSLOAD_PARENT)) ||
-               ir->o == IR_FRAME);
-    if (ir->o == IR_SLOAD) {
-      RegSP rs = as->parentmap[ir->op1];
-      if (ra_hasreg(ir->r)) {
-        rset_clear(allow, ir->r);
-        if (ra_hasspill(ir->s))
-          ra_save(as, ir, ir->r);
-      } else if (ra_hasspill(ir->s)) {
-        irt_setmark(ir->t);
-        pass2 = 1;
-      }
-      if (ir->r == rs) {  /* Coalesce matching registers right now. */
-        ra_free(as, ir->r);
-      } else if (ra_hasspill(regsp_spill(rs))) {
-        if (ra_hasreg(ir->r))
-          pass3 = 1;
-      } else if (ra_used(ir)) {
-        sloadins[rs] = (IRRef1)i;
-        rset_set(live, rs);  /* Block live parent register. */
-      }
+    RegSP rs;
+    lua_assert(ir->o == IR_SLOAD && (ir->op2 & IRSLOAD_PARENT));
+    rs = as->parentmap[ir->op1];
+    if (ra_hasreg(ir->r)) {
+      rset_clear(allow, ir->r);
+      if (ra_hasspill(ir->s))
+        ra_save(as, ir, ir->r);
+    } else if (ra_hasspill(ir->s)) {
+      irt_setmark(ir->t);
+      pass2 = 1;
+    }
+    if (ir->r == rs) {  /* Coalesce matching registers right now. */
+      ra_free(as, ir->r);
+    } else if (ra_hasspill(regsp_spill(rs))) {
+      if (ra_hasreg(ir->r))
+        pass3 = 1;
+    } else if (ra_used(ir)) {
+      sloadins[rs] = (IRRef1)i;
+      rset_set(live, rs);  /* Block live parent register. */
     }
   }
 
@@ -2979,8 +2977,7 @@ static void asm_tail_sync(ASMState *as)
     SnapEntry sn = map[n];
     if ((sn & SNAP_FRAME)) {
       IRIns *ir = IR(snap_ref(sn));
-      GCfunc *fn = ir_kfunc(IR(ir->op2));
-      lua_assert(ir->o == IR_FRAME && irt_isfunc(ir->t));
+      GCfunc *fn = ir_kfunc(ir);
       if (isluafunc(fn)) {
         BCReg s = snap_slot(sn);
         BCReg fs = s + funcproto(fn)->framesize;
@@ -3019,9 +3016,10 @@ static void asm_tail_sync(ASMState *as)
 
   /* Store the value of all modified slots to the Lua stack. */
   for (n = 0; n < nent; n++) {
-    BCReg s = snap_slot(map[n]);
+    SnapEntry sn = map[n];
+    BCReg s = snap_slot(sn);
     int32_t ofs = 8*((int32_t)s-1);
-    IRRef ref = snap_ref(map[n]);
+    IRRef ref = snap_ref(sn);
     IRIns *ir = IR(ref);
     /* No need to restore readonly slots and unmodified non-parent slots. */
     if (ir->o == IR_SLOAD && ir->op1 == s &&
@@ -3030,10 +3028,6 @@ static void asm_tail_sync(ASMState *as)
     if (irt_isnum(ir->t)) {
       Reg src = ra_alloc1(as, ref, RSET_FPR);
       emit_rmro(as, XO_MOVSDto, src, RID_BASE, ofs);
-    } else if (ir->o == IR_FRAME) {
-      emit_movmroi(as, RID_BASE, ofs, ptr2addr(ir_kgc(IR(ir->op2))));
-      if (s != 0)  /* Do not overwrite link to previous frame. */
-        emit_movmroi(as, RID_BASE, ofs+4, (int32_t)(*--flinks));
     } else {
       lua_assert(irt_ispri(ir->t) || irt_isaddr(ir->t));
       if (!irref_isk(ref)) {
@@ -3042,7 +3036,10 @@ static void asm_tail_sync(ASMState *as)
       } else if (!irt_ispri(ir->t)) {
         emit_movmroi(as, RID_BASE, ofs, ir->i);
       }
-      emit_movmroi(as, RID_BASE, ofs+4, irt_toitype(ir->t));
+      if (!(sn & (SNAP_CONT|SNAP_FRAME)))
+        emit_movmroi(as, RID_BASE, ofs+4, irt_toitype(ir->t));
+      else if (s != 0)  /* Do not overwrite link to previous frame. */
+        emit_movmroi(as, RID_BASE, ofs+4, (int32_t)(*--flinks));
     }
     checkmclim(as);
   }
@@ -3110,10 +3107,6 @@ static void asm_ir(ASMState *as, IRIns *ir)
   case IR_ULE: asm_comp(as, ir, CC_A,  CC_A,  VCC_U); break;
   case IR_ABC:
   case IR_UGT: asm_comp(as, ir, CC_BE, CC_BE, VCC_U|VCC_PS); break;
-
-  case IR_FRAME:
-    if (ir->op1 == ir->op2) break;  /* No check needed for placeholder. */
-    /* fallthrough */
   case IR_EQ:  asm_comp(as, ir, CC_NE, CC_NE, VCC_P); break;
   case IR_NE:  asm_comp(as, ir, CC_E,  CC_E,  VCC_U|VCC_P); break;
 
@@ -3272,10 +3265,6 @@ static void asm_setup_regsp(ASMState *as, Trace *T)
         }
       }
       break;
-    case IR_FRAME:
-      if (i == as->stopins+1 && ir->op1 == ir->op2)
-        as->stopins++;
-      break;
     case IR_CALLN: case IR_CALLL: case IR_CALLS: {
       const CCallInfo *ci = &lj_ir_callinfo[ir->op2];
       /* NYI: not fastcall-aware, but doesn't matter (yet). */
diff --git a/src/lj_ir.h b/src/lj_ir.h
index 672aca4a..34c14519 100644
--- a/src/lj_ir.h
+++ b/src/lj_ir.h
@@ -34,7 +34,7 @@
   _(NE,         GC, ref, ref) \
   \
   _(ABC,        G , ref, ref) \
-  _(FRAME,      G , ref, ref) \
+  _(UNUSED,     G , ref, ref) /* Placeholder. */ \
   \
   _(LT,         G , ref, ref) \
   _(GE,         G , ref, ref) \
@@ -511,11 +511,11 @@ typedef union IRIns {
   MRef ptr;             /* Pointer constant (overlaps op12). */
 } IRIns;
 
-#define ir_kgc(ir)      (gcref((ir)->gcr))
+#define ir_kgc(ir)      check_exp((ir)->o == IR_KGC, gcref((ir)->gcr))
 #define ir_kstr(ir)     (gco2str(ir_kgc((ir))))
 #define ir_ktab(ir)     (gco2tab(ir_kgc((ir))))
 #define ir_kfunc(ir)    (gco2func(ir_kgc((ir))))
-#define ir_knum(ir)     (mref((ir)->ptr, cTValue))
-#define ir_kptr(ir)     (mref((ir)->ptr, void))
+#define ir_knum(ir)     check_exp((ir)->o == IR_KNUM, mref((ir)->ptr, cTValue))
+#define ir_kptr(ir)     check_exp((ir)->o == IR_KPTR, mref((ir)->ptr, void))
 
 #endif
diff --git a/src/lj_record.c b/src/lj_record.c
index 94ea42ed..7b2e977e 100644
--- a/src/lj_record.c
+++ b/src/lj_record.c
@@ -148,15 +148,10 @@ static TRef sload(jit_State *J, int32_t slot)
 /* Get TRef for current function. */
 static TRef getcurrf(jit_State *J)
 {
-  if (J->base[-1]) {
-    IRIns *ir = IR(tref_ref(J->base[-1]));
-    if (ir->o == IR_FRAME)  /* Shortcut if already specialized. */
-      return TREF(ir->op2, IRT_FUNC);  /* Return TRef of KFUNC. */
+  if (J->base[-1])
     return J->base[-1];
-  } else {
-    lua_assert(J->baseslot == 1);
-    return sloadt(J, -1, IRT_FUNC, IRSLOAD_READONLY);
-  }
+  lua_assert(J->baseslot == 1);
+  return sloadt(J, -1, IRT_FUNC, IRSLOAD_READONLY);
 }
 
 /* Compare for raw object equality.
@@ -424,7 +419,7 @@ static BCReg rec_mm_prep(jit_State *J, ASMFunction cont)
 #else
   trcont = lj_ir_kptr(J, (void *)cont);
 #endif
-  J->base[top] = emitir(IRTG(IR_FRAME, IRT_PTR), trcont, trcont) | TREF_CONT;
+  J->base[top] = trcont | TREF_CONT;
   for (s = J->maxslot; s < top; s++)
     J->base[s] = TREF_NIL;
   return top+1;
@@ -1586,7 +1581,7 @@ static void check_call_unroll(jit_State *J, GCfunc *fn)
 static int rec_call(jit_State *J, BCReg func, int cres, int nargs)
 {
   RecordFFData rd;
-  TRef *res = &J->base[func];
+  TRef trfunc, *res = &J->base[func];
   TValue *tv = &J->L->base[func];
 
   if (tref_isfunc(res[0])) {  /* Regular function call. */
@@ -1608,7 +1603,9 @@ static int rec_call(jit_State *J, BCReg func, int cres, int nargs)
   }
 
   /* Specialize to the runtime value of the called function. */
-  res[0] = emitir(IRTG(IR_FRAME, IRT_FUNC), res[0], lj_ir_kfunc(J, rd.fn)) | TREF_FRAME;
+  trfunc = lj_ir_kfunc(J, rd.fn);
+  emitir(IRTG(IR_EQ, IRT_FUNC), res[0], trfunc);
+  res[0] = trfunc | TREF_FRAME;
 
   if (isluafunc(rd.fn)) {  /* Record call to Lua function. */
     GCproto *pt = funcproto(rd.fn);
@@ -2175,12 +2172,7 @@ static void rec_setup_side(jit_State *J, Trace *T)
       for (j = 0; j < n; j++)
         if (snap_ref(map[j]) == ref) {
           tr = J->slot[snap_slot(map[j])];
-          if (ir->o == IR_FRAME && irt_isfunc(ir->t)) {
-            lua_assert(s != 0);
-            J->baseslot = s+1;
-            J->framedepth++;
-          }
-          goto dupslot;
+          goto setslot;
         }
     }
     bloomset(seen, ref);
@@ -2190,30 +2182,24 @@ static void rec_setup_side(jit_State *J, Trace *T)
     case IR_KINT: tr = lj_ir_kint(J, ir->i); break;
     case IR_KGC:  tr = lj_ir_kgc(J, ir_kgc(ir), irt_t(ir->t)); break;
     case IR_KNUM: tr = lj_ir_knum_addr(J, ir_knum(ir)); break;
-    case IR_FRAME:  /* Placeholder FRAMEs don't need a guard. */
-      if (irt_isfunc(ir->t)) {
-        if (s != 0) {
-          J->baseslot = s+1;
-          J->framedepth++;
-        }
-        tr = lj_ir_kfunc(J, ir_kfunc(&T->ir[ir->op2]));
-        tr = emitir_raw(IRT(IR_FRAME, IRT_FUNC), tr, tr) | TREF_FRAME;
-      } else {
-        tr = lj_ir_kptr(J, mref(T->ir[ir->op2].ptr, void));
-        tr = emitir_raw(IRT(IR_FRAME, IRT_PTR), tr, tr) | TREF_CONT;
-      }
-      break;
-    case IR_SLOAD:  /* Inherited SLOADs don't need a guard or type check. */
+    case IR_KPTR:  tr = lj_ir_kptr(J, ir_kptr(ir)); break;  /* Continuation. */
+    /* Inherited SLOADs don't need a guard or type check. */
+    case IR_SLOAD:
       tr = emitir_raw(ir->ot & ~IRT_GUARD, s,
              (ir->op2&IRSLOAD_READONLY) | IRSLOAD_INHERIT|IRSLOAD_PARENT);
       break;
-    default:  /* Parent refs are already typed and don't need a guard. */
+    /* Parent refs are already typed and don't need a guard. */
+    default:
       tr = emitir_raw(IRT(IR_SLOAD, irt_type(ir->t)), s,
                       IRSLOAD_INHERIT|IRSLOAD_PARENT);
       break;
     }
-  dupslot:
-    J->slot[s] = tr;
+  setslot:
+    J->slot[s] = tr | (sn&(SNAP_CONT|SNAP_FRAME));  /* Same as TREF_* flags. */
+    if ((sn & SNAP_FRAME) && s != 0) {
+      J->baseslot = s+1;
+      J->framedepth++;
+    }
   }
   J->base = J->slot + J->baseslot;
   J->maxslot = snap->nslots - J->baseslot;
diff --git a/src/lj_snap.c b/src/lj_snap.c
index 731b8f92..8a53e3f6 100644
--- a/src/lj_snap.c
+++ b/src/lj_snap.c
@@ -211,6 +211,7 @@ void lj_snap_restore(jit_State *J, void *exptr)
   MSize n, nent = snap->nent;
   SnapEntry *map = &T->snapmap[snap->mapofs];
   SnapEntry *flinks = map + nent + snap->nframelinks;
+  int32_t ftsz0;
   BCReg nslots = snap->nslots;
   TValue *frame;
   BloomFilter rfilt = snap_renamefilter(T, snapno);
@@ -224,6 +225,7 @@ void lj_snap_restore(jit_State *J, void *exptr)
 
   /* Fill stack slots with data from the registers and spill slots. */
   frame = L->base-1;
+  ftsz0 = frame_ftsz(frame);  /* Preserve link to previous frame in slot #0. */
   for (n = 0; n < nent; n++) {
     SnapEntry sn = map[n];
     IRRef ref = snap_ref(sn);
@@ -232,9 +234,34 @@ void lj_snap_restore(jit_State *J, void *exptr)
     IRIns *ir = &T->ir[ref];
     if (irref_isk(ref)) {  /* Restore constant slot. */
       lj_ir_kvalue(L, o, ir);
+      if ((sn & (SNAP_CONT|SNAP_FRAME))) {
+        /* Overwrite tag with frame link. */
+        o->fr.tp.ftsz = s != 0 ? (int32_t)*--flinks : ftsz0;
+        if ((sn & SNAP_FRAME)) {
+          GCfunc *fn = ir_kfunc(ir);
+          if (isluafunc(fn)) {
+            MSize framesize = funcproto(fn)->framesize;
+            TValue *fs;
+            L->base = ++o;
+            if (LJ_UNLIKELY(o + framesize > L->maxstack)) {  /* Grow again? */
+              ptrdiff_t fsave = savestack(L, frame);
+              L->top = o;
+              lj_state_growstack(L, framesize);
+              frame = restorestack(L, fsave);
+              o = L->top;
+            }
+            fs = o + framesize;
+            if (s == 0)  /* Only partially clear tail call frame at #0. */
+              o = &frame[nslots];
+            while (o < fs)  /* Clear slots of newly added frames. */
+              setnilV(o++);
+          }
+        }
+      }
     } else {
       IRType1 t = ir->t;
       RegSP rs = ir->prev;
+      lua_assert(!(sn & (SNAP_CONT|SNAP_FRAME)));
       if (LJ_UNLIKELY(bloomtest(rfilt, ref)))
         rs = snap_renameref(T, snapno, ref, rs);
       if (ra_hasspill(regsp_spill(rs))) {  /* Restore from spill slot. */
@@ -248,8 +275,9 @@ void lj_snap_restore(jit_State *J, void *exptr)
           setgcrefi(o->gcr, *sps);
           setitype(o, irt_toitype(t));
         }
-      } else if (ra_hasreg(regsp_reg(rs))) {  /* Restore from register. */
+      } else {  /* Restore from register. */
         Reg r = regsp_reg(rs);
+        lua_assert(ra_hasreg(r));
         if (irt_isinteger(t)) {
           setintV(o, ex->gpr[r-RID_MIN_GPR]);
         } else if (irt_isnum(t)) {
@@ -259,33 +287,6 @@ void lj_snap_restore(jit_State *J, void *exptr)
             setgcrefi(o->gcr, ex->gpr[r-RID_MIN_GPR]);
           setitype(o, irt_toitype(t));
         }
-      } else {  /* Restore frame slot. */
-        lua_assert((sn & (SNAP_CONT|SNAP_FRAME)));
-        lua_assert(ir->o == IR_FRAME);
-        /* This works for both PTR and FUNC IR_FRAME. */
-        setgcrefp(o->fr.func, mref(T->ir[ir->op2].ptr, void));
-        if (s != 0)  /* Do not overwrite link to previous frame. */
-          o->fr.tp.ftsz = (int32_t)*--flinks;
-        if (irt_isfunc(ir->t)) {
-          GCfunc *fn = gco2func(gcref(T->ir[ir->op2].gcr));
-          if (isluafunc(fn)) {
-            MSize framesize = funcproto(fn)->framesize;
-            TValue *fs;
-            L->base = ++o;
-            if (LJ_UNLIKELY(o + framesize > L->maxstack)) {  /* Grow again? */
-              ptrdiff_t fsave = savestack(L, frame);
-              L->top = o;
-              lj_state_growstack(L, framesize);
-              frame = restorestack(L, fsave);
-              o = L->top;
-            }
-            fs = o + framesize;
-            if (s == 0)  /* Only partially clear tail call frame at #0. */
-              o = &frame[nslots];
-            while (o < fs)  /* Clear slots of newly added frames. */
-              setnilV(o++);
-          }
-        }
       }
     }
   }