1 files changed, 131 insertions, 116 deletions

diff --git a/src/lj_snap.c b/src/lj_snap.c
index f262e1c9..d22c90a4 100644
--- a/src/lj_snap.c
+++ b/src/lj_snap.c
@@ -23,28 +23,50 @@
 /* Some local macros to save typing. Undef'd at the end. */
 #define IR(ref)         (&J->cur.ir[(ref)])
 
+/* -- Snapshot buffer allocation ------------------------------------------ */
+
+/* Grow snapshot buffer. */
+void lj_snap_grow_buf_(jit_State *J, MSize need)
+{
+  MSize maxsnap = (MSize)J->param[JIT_P_maxsnap];
+  if (need > maxsnap)
+    lj_trace_err(J, LJ_TRERR_SNAPOV);
+  lj_mem_growvec(J->L, J->snapbuf, J->sizesnap, maxsnap, SnapShot);
+  J->cur.snap = J->snapbuf;
+}
+
+/* Grow snapshot map buffer. */
+void lj_snap_grow_map_(jit_State *J, MSize need)
+{
+  if (need < 2*J->sizesnapmap)
+    need = 2*J->sizesnapmap;
+  else if (need < 64)
+    need = 64;
+  J->snapmapbuf = (SnapEntry *)lj_mem_realloc(J->L, J->snapmapbuf,
+                    J->sizesnapmap*sizeof(SnapEntry), need*sizeof(SnapEntry));
+  J->cur.snapmap = J->snapmapbuf;
+  J->sizesnapmap = need;
+}
+
 /* -- Snapshot generation ------------------------------------------------- */
 
-/* NYI: Snapshots are in need of a redesign. The current storage model for
-** snapshot maps is too wasteful. They could be compressed (1D or 2D) and
-** made more flexible at the same time. Iterators should no longer need to
-** skip unmodified slots. IR_FRAME should be eliminated, too.
-*/
+/* NYI: IR_FRAME should be eliminated, too. */
 
 /* Add all modified slots to the snapshot. */
 static MSize snapshot_slots(jit_State *J, SnapEntry *map, BCReg nslots)
 {
   BCReg s;
+  MSize n = 0;
   for (s = 0; s < nslots; s++) {
     IRRef ref = tref_ref(J->slot[s]);
     if (ref) {
       IRIns *ir = IR(ref);
-      if (ir->o == IR_SLOAD && ir->op1 == s && !(ir->op2 & IRSLOAD_INHERIT))
-        ref = 0;
+      if (!(ir->o == IR_SLOAD && ir->op1 == s &&
+            !(ir->op2 & IRSLOAD_INHERIT)))
+        map[n++] = SNAP(s, ir->o == IR_FRAME ? SNAP_FRAME : 0, ref);
     }
-    map[s] = (SnapEntry)ref;
   }
-  return nslots;
+  return n;
 }
 
 /* Add frame links at the end of the snapshot. */
@@ -53,17 +75,17 @@ static MSize snapshot_framelinks(jit_State *J, SnapEntry *map)
   cTValue *frame = J->L->base - 1;
   cTValue *lim = J->L->base - J->baseslot;
   MSize f = 0;
-  map[f++] = u32ptr(J->pc);
-  while (frame > lim) {
+  map[f++] = SNAP_MKPC(J->pc);  /* The current PC is always the first entry. */
+  while (frame > lim) {  /* Backwards traversal of all frames above base. */
     if (frame_islua(frame)) {
-      map[f++] = u32ptr(frame_pc(frame));
+      map[f++] = SNAP_MKPC(frame_pc(frame));
       frame = frame_prevl(frame);
     } else if (frame_ispcall(frame)) {
-      map[f++] = (uint32_t)frame_ftsz(frame);
+      map[f++] = SNAP_MKFTSZ(frame_ftsz(frame));
       frame = frame_prevd(frame);
     } else if (frame_iscont(frame)) {
-      map[f++] = (uint32_t)frame_ftsz(frame);
-      map[f++] = u32ptr(frame_contpc(frame));
+      map[f++] = SNAP_MKFTSZ(frame_ftsz(frame));
+      map[f++] = SNAP_MKPC(frame_contpc(frame));
       frame = frame_prevd(frame);
     } else {
       lua_assert(0);
@@ -76,28 +98,19 @@ static MSize snapshot_framelinks(jit_State *J, SnapEntry *map)
 static void snapshot_stack(jit_State *J, SnapShot *snap, MSize nsnapmap)
 {
   BCReg nslots = J->baseslot + J->maxslot;
-  MSize nsm, nframelinks;
+  MSize nent, nframelinks;
   SnapEntry *p;
   /* Conservative estimate. Continuation frames need 2 slots. */
-  nsm = nsnapmap + nslots + (uint32_t)J->framedepth*2+1;
-  if (LJ_UNLIKELY(nsm > J->sizesnapmap)) {  /* Need to grow snapshot map? */
-    if (nsm < 2*J->sizesnapmap)
-      nsm = 2*J->sizesnapmap;
-    else if (nsm < 64)
-      nsm = 64;
-    J->snapmapbuf = (SnapEntry *)lj_mem_realloc(J->L, J->snapmapbuf,
-                      J->sizesnapmap*sizeof(SnapEntry), nsm*sizeof(SnapEntry));
-    J->cur.snapmap = J->snapmapbuf;
-    J->sizesnapmap = nsm;
-  }
+  lj_snap_grow_map(J, nsnapmap + nslots + (MSize)J->framedepth*2+1);
   p = &J->cur.snapmap[nsnapmap];
-  nslots = snapshot_slots(J, p, nslots);
-  nframelinks = snapshot_framelinks(J, p + nslots);
-  J->cur.nsnapmap = (uint16_t)(nsnapmap + nslots + nframelinks);
+  nent = snapshot_slots(J, p, nslots);
+  nframelinks = snapshot_framelinks(J, p + nent);
+  J->cur.nsnapmap = (uint16_t)(nsnapmap + nent + nframelinks);
   snap->mapofs = (uint16_t)nsnapmap;
   snap->ref = (IRRef1)J->cur.nins;
-  snap->nslots = (uint8_t)nslots;
+  snap->nent = (uint8_t)nent;
   snap->nframelinks = (uint8_t)nframelinks;
+  snap->nslots = (uint8_t)nslots;
   snap->count = 0;
 }
 
@@ -111,14 +124,7 @@ void lj_snap_add(jit_State *J)
       (nsnap > 0 && J->cur.snap[nsnap-1].ref == J->cur.nins)) {
     nsnapmap = J->cur.snap[--nsnap].mapofs;
   } else {
-    /* Need to grow snapshot buffer? */
-    if (LJ_UNLIKELY(nsnap >= J->sizesnap)) {
-      MSize maxsnap = (MSize)J->param[JIT_P_maxsnap];
-      if (nsnap >= maxsnap)
-        lj_trace_err(J, LJ_TRERR_SNAPOV);
-      lj_mem_growvec(J->L, J->snapbuf, J->sizesnap, maxsnap, SnapShot);
-      J->cur.snap = J->snapbuf;
-    }
+    lj_snap_grow_buf(J, nsnap+1);
     J->cur.nsnap = (uint16_t)(nsnap+1);
   }
   J->mergesnap = 0;
@@ -131,14 +137,21 @@ void lj_snap_shrink(jit_State *J)
 {
   BCReg nslots = J->baseslot + J->maxslot;
   SnapShot *snap = &J->cur.snap[J->cur.nsnap-1];
-  SnapEntry *oflinks = &J->cur.snapmap[snap->mapofs + snap->nslots];
-  SnapEntry *nflinks = &J->cur.snapmap[snap->mapofs + nslots];
-  uint32_t s, nframelinks = snap->nframelinks;
+  SnapEntry *map = &J->cur.snapmap[snap->mapofs];
+  MSize nent = snap->nent;
   lua_assert(nslots < snap->nslots);
   snap->nslots = (uint8_t)nslots;
-  J->cur.nsnapmap = (uint16_t)(snap->mapofs + nslots + nframelinks);
-  for (s = 0; s < nframelinks; s++)  /* Move frame links down. */
-    nflinks[s] = oflinks[s];
+  if (nent > 0 && snap_slot(map[nent-1]) >= nslots) {
+    MSize s, delta, nframelinks = snap->nframelinks;
+    for (nent--; nent > 0 && snap_slot(map[nent-1]) >= nslots; nent--)
+      ;
+    delta = snap->nent - nent;
+    snap->nent = (uint8_t)nent;
+    J->cur.nsnapmap = (uint16_t)(snap->mapofs + nent + nframelinks);
+    map += nent;
+    for (s = 0; s < nframelinks; s++)  /* Move frame links down. */
+      map[s] = map[s+delta];
+  }
 }
 
 /* -- Snapshot access ----------------------------------------------------- */
@@ -167,21 +180,24 @@ static RegSP snap_renameref(Trace *T, SnapNo lim, IRRef ref, RegSP rs)
   return rs;
 }
 
-/* Convert a snapshot into a linear slot -> RegSP map. */
+/* Convert a snapshot into a linear slot -> RegSP map.
+** Note: unused slots are not initialized!
+*/
 void lj_snap_regspmap(uint16_t *rsmap, Trace *T, SnapNo snapno)
 {
   SnapShot *snap = &T->snap[snapno];
-  BCReg s, nslots = snap->nslots;
+  MSize n, nent = snap->nent;
   SnapEntry *map = &T->snapmap[snap->mapofs];
   BloomFilter rfilt = snap_renamefilter(T, snapno);
-  for (s = 0; s < nslots; s++) {
-    IRRef ref = snap_ref(map[s]);
+  for (n = 0; n < nent; n++) {
+    SnapEntry sn = map[n];
+    IRRef ref = snap_ref(sn);
     if (!irref_isk(ref)) {
       IRIns *ir = &T->ir[ref];
       uint32_t rs = ir->prev;
       if (bloomtest(rfilt, ref))
         rs = snap_renameref(T, snapno, ref, rs);
-      rsmap[s] = (uint16_t)rs;
+      rsmap[snap_slot(sn)] = (uint16_t)rs;
     }
   }
 }
@@ -193,89 +209,88 @@ void lj_snap_restore(jit_State *J, void *exptr)
   SnapNo snapno = J->exitno;  /* For now, snapno == exitno. */
   Trace *T = J->trace[J->parent];
   SnapShot *snap = &T->snap[snapno];
-  BCReg s, nslots = snap->nslots;
+  MSize n, nent = snap->nent;
   SnapEntry *map = &T->snapmap[snap->mapofs];
-  SnapEntry *flinks = map + nslots + snap->nframelinks;
-  TValue *o, *newbase, *ntop;
+  SnapEntry *flinks = map + nent + snap->nframelinks;
+  BCReg nslots = snap->nslots;
+  TValue *frame;
   BloomFilter rfilt = snap_renamefilter(T, snapno);
   lua_State *L = J->L;
 
   /* Make sure the stack is big enough for the slots from the snapshot. */
-  if (L->base + nslots >= L->maxstack) {
+  if (LJ_UNLIKELY(L->base + nslots > L->maxstack)) {
     L->top = curr_topL(L);
     lj_state_growstack(L, nslots - curr_proto(L)->framesize);
   }
 
   /* Fill stack slots with data from the registers and spill slots. */
-  newbase = NULL;
-  ntop = L->base;
-  for (s = 0, o = L->base-1; s < nslots; s++, o++) {
-    IRRef ref = snap_ref(map[s]);
-    if (ref) {
-      IRIns *ir = &T->ir[ref];
-      if (irref_isk(ref)) {  /* Restore constant slot. */
-        lj_ir_kvalue(L, o, ir);
-      } else {
-        IRType1 t = ir->t;
-        RegSP rs = ir->prev;
-        if (LJ_UNLIKELY(bloomtest(rfilt, ref)))
-          rs = snap_renameref(T, snapno, ref, rs);
-        if (ra_hasspill(regsp_spill(rs))) {  /* Restore from spill slot. */
-          int32_t *sps = &ex->spill[regsp_spill(rs)];
-          if (irt_isinteger(t)) {
-            setintV(o, *sps);
-          } else if (irt_isnum(t)) {
-            o->u64 = *(uint64_t *)sps;
-          } else {
-            lua_assert(!irt_ispri(t));  /* PRI refs never have a spill slot. */
-            setgcrefi(o->gcr, *sps);
-            setitype(o, irt_toitype(t));
-          }
-        } else if (ra_hasreg(regsp_reg(rs))) {  /* Restore from register. */
-          Reg r = regsp_reg(rs);
-          if (irt_isinteger(t)) {
-            setintV(o, ex->gpr[r-RID_MIN_GPR]);
-          } else if (irt_isnum(t)) {
-            setnumV(o, ex->fpr[r-RID_MIN_FPR]);
-          } else {
-            if (!irt_ispri(t))
-              setgcrefi(o->gcr, ex->gpr[r-RID_MIN_GPR]);
-            setitype(o, irt_toitype(t));
-          }
-        } else {  /* Restore frame slot. */
-          lua_assert(ir->o == IR_FRAME);
-          /* This works for both PTR and FUNC IR_FRAME. */
-          setgcrefp(o->fr.func, mref(T->ir[ir->op2].ptr, void));
-          if (s != 0)  /* Do not overwrite link to previous frame. */
-            o->fr.tp.ftsz = (int32_t)*--flinks;
-          if (irt_isfunc(ir->t)) {
-            GCfunc *fn = gco2func(gcref(T->ir[ir->op2].gcr));
-            if (isluafunc(fn)) {
-              TValue *fs;
-              fs = o+1 + funcproto(fn)->framesize;
-              if (fs > ntop) ntop = fs; /* Update top for newly added frames. */
-              if (s != 0) newbase = o+1;
+  frame = L->base-1;
+  for (n = 0; n < nent; n++) {
+    IRRef ref = snap_ref(map[n]);
+    BCReg s = snap_slot(map[n]);
+    TValue *o = &frame[s];  /* Stack slots are relative to start frame. */
+    IRIns *ir = &T->ir[ref];
+    if (irref_isk(ref)) {  /* Restore constant slot. */
+      lj_ir_kvalue(L, o, ir);
+    } else {
+      IRType1 t = ir->t;
+      RegSP rs = ir->prev;
+      if (LJ_UNLIKELY(bloomtest(rfilt, ref)))
+        rs = snap_renameref(T, snapno, ref, rs);
+      if (ra_hasspill(regsp_spill(rs))) {  /* Restore from spill slot. */
+        int32_t *sps = &ex->spill[regsp_spill(rs)];
+        if (irt_isinteger(t)) {
+          setintV(o, *sps);
+        } else if (irt_isnum(t)) {
+          o->u64 = *(uint64_t *)sps;
+        } else {
+          lua_assert(!irt_ispri(t));  /* PRI refs never have a spill slot. */
+          setgcrefi(o->gcr, *sps);
+          setitype(o, irt_toitype(t));
+        }
+      } else if (ra_hasreg(regsp_reg(rs))) {  /* Restore from register. */
+        Reg r = regsp_reg(rs);
+        if (irt_isinteger(t)) {
+          setintV(o, ex->gpr[r-RID_MIN_GPR]);
+        } else if (irt_isnum(t)) {
+          setnumV(o, ex->fpr[r-RID_MIN_FPR]);
+        } else {
+          if (!irt_ispri(t))
+            setgcrefi(o->gcr, ex->gpr[r-RID_MIN_GPR]);
+          setitype(o, irt_toitype(t));
+        }
+      } else {  /* Restore frame slot. */
+        lua_assert(ir->o == IR_FRAME);
+        /* This works for both PTR and FUNC IR_FRAME. */
+        setgcrefp(o->fr.func, mref(T->ir[ir->op2].ptr, void));
+        if (s != 0)  /* Do not overwrite link to previous frame. */
+          o->fr.tp.ftsz = (int32_t)*--flinks;
+        if (irt_isfunc(ir->t)) {
+          GCfunc *fn = gco2func(gcref(T->ir[ir->op2].gcr));
+          if (isluafunc(fn)) {
+            MSize framesize = funcproto(fn)->framesize;
+            TValue *fs;
+            L->base = ++o;
+            if (LJ_UNLIKELY(o + framesize > L->maxstack)) {  /* Grow again? */
+              ptrdiff_t fsave = savestack(L, frame);
+              L->top = o;
+              lj_state_growstack(L, framesize);
+              frame = restorestack(L, fsave);
+              o = L->top;
             }
+            fs = o + framesize;
+            if (s == 0)  /* Only partially clear tail call frame at #0. */
+              o = &frame[nslots];
+            while (o < fs)  /* Clear slots of newly added frames. */
+              setnilV(o++);
           }
         }
       }
-    } else {
-      lua_assert(!newbase);
     }
   }
-  if (newbase) L->base = newbase;
-  if (ntop >= L->maxstack) {  /* Need to grow the stack again. */
-    MSize need = (MSize)(ntop - o);
-    L->top = o;
-    lj_state_growstack(L, need);
-    o = L->top;
-    ntop = o + need;
-  }
   L->top = curr_topL(L);
-  for (; o < ntop; o++)  /* Clear remainder of newly added frames. */
-    setnilV(o);
-  lua_assert(map + nslots == flinks-1);
-  J->pc = (const BCIns *)(uintptr_t)(*--flinks);
+  J->pc = snap_pc(*--flinks);
+  lua_assert(map + nent == flinks);
 }
 
 #undef IR