Split CALL/FUNC recording.

Record __call resolving and specialization for CALL* bytecodes.
Record argument adjustment and fast functions for FUNC* bytecodes.
Avoids all pending/immediate decisions for chained fast functions.
Cleaner semantics for pcall(), xpcall() and __tostring metamethod.
Prerequisite to drop the shadow frame link stack again.

2 files changed, 455 insertions, 447 deletions

diff --git a/src/lj_dispatch.c b/src/lj_dispatch.c
index 29b06438..4629fb7e 100644
--- a/src/lj_dispatch.c
+++ b/src/lj_dispatch.c
@@ -71,7 +71,8 @@ void lj_dispatch_update(global_State *g)
   uint8_t mode = 0;
 #if LJ_HASJIT
   mode |= (G2J(g)->flags & JIT_F_ON) ? DISPMODE_JIT : 0;
-  mode |= G2J(g)->state != LJ_TRACE_IDLE ? (DISPMODE_REC|DISPMODE_INS) : 0;
+  mode |= G2J(g)->state != LJ_TRACE_IDLE ?
+            (DISPMODE_REC|DISPMODE_INS|DISPMODE_CALL) : 0;
 #endif
   mode |= (g->hookmask & (LUA_MASKLINE|LUA_MASKCOUNT)) ? DISPMODE_INS : 0;
   mode |= (g->hookmask & LUA_MASKCALL) ? DISPMODE_CALL : 0;
@@ -398,7 +399,7 @@ static void call_init(lua_State *L, GCfunc *fn)
   }
 }
 
-/* Call dispatch. Used by call hooks and hot calls. */
+/* Call dispatch. Used by call hooks, hot calls or when recording. */
 ASMFunction LJ_FASTCALL lj_dispatch_call(lua_State *L, const BCIns *pc)
 {
   GCfunc *fn = curr_func(L);
@@ -409,10 +410,14 @@ ASMFunction LJ_FASTCALL lj_dispatch_call(lua_State *L, const BCIns *pc)
 #endif
   call_init(L, fn);
 #if LJ_HASJIT
+  J->L = L;
   if ((uintptr_t)pc & 1) {  /* Marker for hot call. */
-    J->L = L;
     lj_trace_hot(J, (const BCIns *)((uintptr_t)pc & ~(uintptr_t)1));
     goto out;
+  } else if (J->state != LJ_TRACE_IDLE &&
+             !(g->hookmask & (HOOK_GC|HOOK_VMEVENT))) {
+    /* Record the FUNC* bytecodes, too. */
+    lj_trace_ins(J, pc-1);  /* The interpreter bytecode PC is offset by 1. */
   }
 #endif
   if ((g->hookmask & LUA_MASKCALL))
diff --git a/src/lj_record.c b/src/lj_record.c
index ed882f1e..5d99f3a4 100644
--- a/src/lj_record.c
+++ b/src/lj_record.c
@@ -53,18 +53,9 @@ typedef struct RecordIndex {
   int idxchain;         /* Index indirections left or 0 for raw lookup. */
 } RecordIndex;
 
-/* Requested results from rec_call(). */
-enum {
-  /* Non-negative numbers are number of requested results. */
-  CALLRES_MULTI = -1,           /* Return multiple results. */
-  CALLRES_TAILCALL = -2,        /* Tail call. */
-  CALLRES_PENDING = -3,         /* Call is pending, no results yet. */
-  CALLRES_CONT = -4             /* Continuation call. */
-};
-
 /* Forward declarations. */
+static int rec_mm_lookup(jit_State *J, RecordIndex *ix, MMS mm);
 static TRef rec_idx(jit_State *J, RecordIndex *ix);
-static int rec_call(jit_State *J, BCReg func, ptrdiff_t cres, ptrdiff_t nargs);
 
 /* -- Sanity checks ------------------------------------------------------- */
 
@@ -474,6 +465,126 @@ static void rec_loop_jit(jit_State *J, TraceNo lnk, LoopEvent ev)
   }  /* Side trace continues across a loop that's left or not entered. */
 }
 
+/* -- Record calls and returns -------------------------------------------- */
+
+/* Record call. */
+static void rec_call(jit_State *J, BCReg func, ptrdiff_t nargs)
+{
+  RecordIndex ix;
+  TValue *functv = &J->L->base[func];
+  TRef trfunc, *fbase = &J->base[func];
+
+  if (!tref_isfunc(fbase[0])) {  /* Resolve __call metamethod. */
+    ptrdiff_t i;
+    ix.tab = fbase[0];
+    copyTV(J->L, &ix.tabv, functv);
+    if (!rec_mm_lookup(J, &ix, MM_call) || !tref_isfunc(ix.mobj))
+      lj_trace_err(J, LJ_TRERR_NOMM);
+    for (i = ++nargs; i > 0; i--)  /* Shift arguments up. */
+      fbase[i] = fbase[i-1];
+    fbase[0] = ix.mobj;  /* Replace function. */
+    functv = &ix.mobjv;
+  }
+
+  /* Specialize to the runtime value of the called function. */
+  trfunc = lj_ir_kfunc(J, funcV(functv));
+  emitir(IRTG(IR_EQ, IRT_FUNC), fbase[0], trfunc);
+  fbase[0] = trfunc | TREF_FRAME;
+
+  /* Bump frame. */
+  J->frame[J->framedepth++] = SNAP_MKPC(J->pc+1);
+  if (J->framedepth > LJ_MAX_JFRAME)
+    lj_trace_err(J, LJ_TRERR_STACKOV);
+  J->base += func+1;
+  J->baseslot += func+1;
+  J->maxslot = nargs;
+}
+
+/* Record tail call. */
+static void rec_tailcall(jit_State *J, BCReg func, ptrdiff_t nargs)
+{
+  rec_call(J, func, nargs);
+  /* Move func + args down. */
+  J->framedepth--;
+  J->base -= func+1;
+  J->baseslot -= func+1;
+  memmove(&J->base[-1], &J->base[func], sizeof(TRef)*(J->maxslot+1));
+  /* Note: the new TREF_FRAME is now at J->base[-1] (even for slot #0). */
+}
+
+/* Record return. */
+static void rec_ret(jit_State *J, BCReg rbase, ptrdiff_t gotresults)
+{
+  TValue *frame = J->L->base - 1;
+  ptrdiff_t i;
+  for (i = 0; i < gotresults; i++)
+    getslot(J, rbase+i);  /* Ensure all results have a reference. */
+  while (frame_ispcall(frame)) {  /* Immediately resolve pcall() returns. */
+    BCReg cbase = (BCReg)frame_delta(frame);
+    if (--J->framedepth < 0)
+      lj_trace_err(J, LJ_TRERR_NYIRETL);
+    lua_assert(J->baseslot > 1);
+    J->base[--rbase] = TREF_TRUE;  /* Prepend true to results. */
+    gotresults++;
+    rbase += cbase;
+    J->baseslot -= (BCReg)cbase;
+    J->base -= cbase;
+    frame = frame_prevd(frame);
+  }
+  if (frame_islua(frame)) {  /* Return to Lua frame. */
+    BCIns callins = *(frame_pc(frame)-1);
+    ptrdiff_t nresults = bc_b(callins) ? (ptrdiff_t)bc_b(callins)-1 :gotresults;
+    BCReg cbase = bc_a(callins);
+    for (i = 0; i < nresults; i++)  /* Adjust results. */
+      J->base[i-1] = i < gotresults ? J->base[rbase+i] : TREF_NIL;
+    J->maxslot = cbase+(BCReg)nresults;
+    if (J->framedepth > 0) {  /* Return to a frame that is part of the trace. */
+      J->framedepth--;
+      lua_assert(J->baseslot > cbase+1);
+      J->baseslot -= cbase+1;
+      J->base -= cbase+1;
+    } else if (J->parent == 0) {
+      /* Return to lower frame would leave the loop in a root trace. */
+      lj_trace_err(J, LJ_TRERR_LLEAVE);
+    } else {  /* Return to lower frame. Guard for the target we return to. */
+      GCproto *pt = funcproto(frame_func(frame - (cbase+1)));
+      TRef trpt = lj_ir_kgc(J, obj2gco(pt), IRT_PROTO);
+      TRef trpc = lj_ir_kptr(J, (void *)frame_pc(frame));
+      emitir(IRTG(IR_RETF, IRT_PTR), trpt, trpc);
+      J->retdepth++;
+      J->needsnap = 1;
+      lua_assert(J->baseslot == 1);
+      /* Shift result slots up and clear the slots of the new frame below. */
+      memmove(J->base + cbase, J->base-1, sizeof(TRef)*nresults);
+      memset(J->base-1, 0, sizeof(TRef)*(cbase+1));
+    }
+  } else if (frame_iscont(frame)) {  /* Return to continuation frame. */
+    ASMFunction cont = frame_contf(frame);
+    BCReg cbase = (BCReg)frame_delta(frame);
+    if ((J->framedepth -= 2) < 0)
+      lj_trace_err(J, LJ_TRERR_NYIRETL);
+    J->baseslot -= (BCReg)cbase;
+    J->base -= cbase;
+    J->maxslot = cbase-2;
+    if (cont == lj_cont_ra) {
+      /* Copy result to destination slot. */
+      BCReg dst = bc_a(*(frame_contpc(frame)-1));
+      J->base[dst] = gotresults ? J->base[cbase+rbase] : TREF_NIL;
+      if (dst > J->maxslot) J->maxslot = dst+1;
+    } else if (cont == lj_cont_nop) {
+      /* Nothing to do here. */
+    } else if (cont == lj_cont_cat) {
+      lua_assert(0);
+    } else {
+      /* Result type already specialized. */
+      lua_assert(cont == lj_cont_condf || cont == lj_cont_condt);
+    }
+  } else {
+    lj_trace_err(J, LJ_TRERR_NYIRETL);  /* NYI: handle return to C frame. */
+  }
+  lua_assert(J->baseslot >= 1);
+}
+
 /* -- Metamethod handling ------------------------------------------------- */
 
 /* Prepare to record call to metamethod. */
@@ -556,7 +667,9 @@ static TRef rec_mm_arith(jit_State *J, RecordIndex *ix, MMS mm)
 ok:
   base[0] = ix->mobj;
   copyTV(J->L, basev+0, &ix->mobjv);
-  return rec_call(J, func, CALLRES_CONT, 2) ? J->base[func] : 0;
+  rec_call(J, func, 2);
+  J->frame[J->framedepth++] = SNAP_MKFTSZ((func+1)*sizeof(TValue)+FRAME_CONT);
+  return 0;  /* No result yet. */
 }
 
 /* Call a comparison metamethod. */
@@ -569,11 +682,8 @@ static void rec_mm_callcomp(jit_State *J, RecordIndex *ix, int op)
   copyTV(J->L, tv+0, &ix->mobjv);
   copyTV(J->L, tv+1, &ix->valv);
   copyTV(J->L, tv+2, &ix->keyv);
-  rec_call(J, func, CALLRES_CONT, 2);
-  /* It doesn't matter whether this is immediately resolved or not.
-  ** Type specialization of the return type suffices to specialize
-  ** the control flow.
-  */
+  rec_call(J, func, 2);
+  J->frame[J->framedepth++] = SNAP_MKFTSZ((func+1)*sizeof(TValue)+FRAME_CONT);
 }
 
 /* Record call to equality comparison metamethod (for tab and udata only). */
@@ -752,11 +862,13 @@ static TRef rec_idx(jit_State *J, RecordIndex *ix)
       if (ix->val) {
         base[3] = ix->val;
         copyTV(J->L, tv+3, &ix->valv);
-        rec_call(J, func, CALLRES_CONT, 3);  /* mobj(tab, key, val) */
+        rec_call(J, func, 3);  /* mobj(tab, key, val) */
+        J->frame[J->framedepth++] = SNAP_MKFTSZ((func+1)*sizeof(TValue)+FRAME_CONT);
         return 0;
       } else {
-        /* res = mobj(tab, key) */
-        return rec_call(J, func, CALLRES_CONT, 2) ? J->base[func] : 0;
+        rec_call(J, func, 2);  /* res = mobj(tab, key) */
+        J->frame[J->framedepth++] = SNAP_MKFTSZ((func+1)*sizeof(TValue)+FRAME_CONT);
+        return 0;  /* No result yet. */
       }
     }
     /* Otherwise retry lookup with metaobject. */
@@ -915,19 +1027,12 @@ static TRef rec_upvalue(jit_State *J, uint32_t uv, TRef val)
 /* Data used by handlers to record a fast function. */
 typedef struct RecordFFData {
   TValue *argv;         /* Runtime argument values. */
-  GCfunc *fn;           /* The currently recorded function. */
-  ptrdiff_t nargs;      /* Number of passed arguments. */
   ptrdiff_t nres;       /* Number of returned results (defaults to 1). */
-  ptrdiff_t cres;       /* Wanted number of call results. */
   uint32_t data;        /* Per-ffid auxiliary data (opcode, literal etc.). */
-  int metacall;         /* True if function was resolved via __call. */
 } RecordFFData;
 
 /* Type of handler to record a fast function. */
-typedef void (*RecordFunc)(jit_State *J, TRef *res, RecordFFData *rd);
-
-/* Avoid carrying two pointers around. */
-#define arg     (res+1)
+typedef void (LJ_FASTCALL *RecordFunc)(jit_State *J, RecordFFData *rd);
 
 /* Get runtime value of int argument. */
 static int32_t argv2int(jit_State *J, TValue *o)
@@ -951,65 +1056,73 @@ static GCstr *argv2str(jit_State *J, TValue *o)
   }
 }
 
-/* Fallback handler for all fast functions that are not recorded (yet). */
-static void recff_nyi(jit_State *J, TRef *res, RecordFFData *rd)
+/* Return number of results wanted by caller. */
+static ptrdiff_t results_wanted(jit_State *J)
 {
-  UNUSED(res);
-  setfuncV(J->L, &J->errinfo, rd->fn);
-  lj_trace_err_info(J, LJ_TRERR_NYIFF);
+  TValue *frame = J->L->base-1;
+  if (frame_islua(frame))
+    return (ptrdiff_t)bc_b(frame_pc(frame)[-1]) - 1;
+  else
+    return -1;
 }
 
-LJ_NORET static void recff_err_nyi(jit_State *J, RecordFFData *rd)
+/* Throw error for unsupported variant of fast function. */
+LJ_NORET static void recff_nyiu(jit_State *J)
 {
-  setfuncV(J->L, &J->errinfo, rd->fn);
+  setfuncV(J->L, &J->errinfo, J->fn);
   lj_trace_err_info(J, LJ_TRERR_NYIFFU);
 }
 
+/* Fallback handler for all fast functions that are not recorded (yet). */
+static void LJ_FASTCALL recff_nyi(jit_State *J, RecordFFData *rd)
+{
+  setfuncV(J->L, &J->errinfo, J->fn);
+  lj_trace_err_info(J, LJ_TRERR_NYIFF);
+  UNUSED(rd);
+}
+
 /* C functions can have arbitrary side-effects and are not recorded (yet). */
-static void recff_c(jit_State *J, TRef *res, RecordFFData *rd)
+static void LJ_FASTCALL recff_c(jit_State *J, RecordFFData *rd)
 {
-  UNUSED(res);
-  setlightudV(&J->errinfo, (void *)rd->fn->c.f);
+  setfuncV(J->L, &J->errinfo, J->fn);
   lj_trace_err_info(J, LJ_TRERR_NYICF);
+  UNUSED(rd);
 }
 
 /* -- Base library fast functions ----------------------------------------- */
 
-static void recff_assert(jit_State *J, TRef *res, RecordFFData *rd)
+static void LJ_FASTCALL recff_assert(jit_State *J, RecordFFData *rd)
 {
   /* Arguments already specialized. The interpreter throws for nil/false. */
-  ptrdiff_t i;
-  for (i = 0; arg[i]; i++)  /* Need to pass through all arguments. */
-    res[i] = arg[i];
-  rd->nres = i;
-  UNUSED(J);
+  rd->nres = J->maxslot;  /* Pass through all arguments. */
 }
 
-static void recff_type(jit_State *J, TRef *res, RecordFFData *rd)
+static void LJ_FASTCALL recff_type(jit_State *J, RecordFFData *rd)
 {
   /* Arguments already specialized. Result is a constant string. Neat, huh? */
-  IRType t = tref_isinteger(arg[0]) ? IRT_NUM : tref_type(arg[0]);
-  res[0] = lj_ir_kstr(J, strV(&rd->fn->c.upvalue[t]));
+  IRType t = tref_isinteger(J->base[0]) ? IRT_NUM : tref_type(J->base[0]);
+  J->base[0] = lj_ir_kstr(J, strV(&J->fn->c.upvalue[t]));
+  UNUSED(rd);
 }
 
-static void recff_getmetatable(jit_State *J, TRef *res, RecordFFData *rd)
+static void LJ_FASTCALL recff_getmetatable(jit_State *J, RecordFFData *rd)
 {
-  TRef tr = arg[0];
+  TRef tr = J->base[0];
   if (tref_istab(tr)) {
     RecordIndex ix;
     ix.tab = tr;
     copyTV(J->L, &ix.tabv, &rd->argv[0]);
     if (rec_mm_lookup(J, &ix, MM_metatable))
-      res[0] = ix.mobj;
+      J->base[0] = ix.mobj;
     else
-      res[0] = ix.mt;
+      J->base[0] = ix.mt;
   }  /* else: Interpreter will throw. */
 }
 
-static void recff_setmetatable(jit_State *J, TRef *res, RecordFFData *rd)
+static void LJ_FASTCALL recff_setmetatable(jit_State *J, RecordFFData *rd)
 {
-  TRef tr = arg[0];
-  TRef mt = arg[1];
+  TRef tr = J->base[0];
+  TRef mt = J->base[1];
   if (tref_istab(tr) && (tref_istab(mt) || (mt && tref_isnil(mt)))) {
     TRef fref, mtref;
     RecordIndex ix;
@@ -1021,227 +1134,213 @@ static void recff_setmetatable(jit_State *J, TRef *res, RecordFFData *rd)
     emitir(IRT(IR_FSTORE, IRT_TAB), fref, mtref);
     if (!tref_isnil(mt))
       emitir(IRT(IR_TBAR, IRT_TAB), tr, 0);
-    res[0] = tr;
+    J->base[0] = tr;
     J->needsnap = 1;
   }  /* else: Interpreter will throw. */
 }
 
-static void recff_rawget(jit_State *J, TRef *res, RecordFFData *rd)
+static void LJ_FASTCALL recff_rawget(jit_State *J, RecordFFData *rd)
 {
-  if (tref_istab(arg[0]) && arg[1]) {
-    RecordIndex ix;
-    ix.tab = arg[0]; ix.key = arg[1]; ix.val = 0; ix.idxchain = 0;
+  RecordIndex ix;
+  ix.tab = J->base[0]; ix.key = J->base[1];
+  if (tref_istab(ix.tab) && ix.key) {
+    ix.val = 0; ix.idxchain = 0;
     settabV(J->L, &ix.tabv, tabV(&rd->argv[0]));
     copyTV(J->L, &ix.keyv, &rd->argv[1]);
-    res[0] = rec_idx(J, &ix);
+    J->base[0] = rec_idx(J, &ix);
   }  /* else: Interpreter will throw. */
 }
 
-static void recff_rawset(jit_State *J, TRef *res, RecordFFData *rd)
+static void LJ_FASTCALL recff_rawset(jit_State *J, RecordFFData *rd)
 {
-  if (tref_istab(arg[0]) && arg[1] && arg[2]) {
-    RecordIndex ix;
-    ix.tab = arg[0]; ix.key = arg[1]; ix.val = arg[2]; ix.idxchain = 0;
+  RecordIndex ix;
+  ix.tab = J->base[0]; ix.key = J->base[1]; ix.val = J->base[2];
+  if (tref_istab(ix.tab) && ix.key && ix.val) {
+    ix.idxchain = 0;
     settabV(J->L, &ix.tabv, tabV(&rd->argv[0]));
     copyTV(J->L, &ix.keyv, &rd->argv[1]);
     copyTV(J->L, &ix.valv, &rd->argv[2]);
     rec_idx(J, &ix);
-    res[0] = arg[0];  /* Returns table. */
+    /* Pass through table at J->base[0] as result. */
   }  /* else: Interpreter will throw. */
 }
 
-static void recff_rawequal(jit_State *J, TRef *res, RecordFFData *rd)
+static void LJ_FASTCALL recff_rawequal(jit_State *J, RecordFFData *rd)
 {
-  if (arg[0] && arg[1]) {
-    int diff = rec_objcmp(J, arg[0], arg[1], &rd->argv[0], &rd->argv[1]);
-    res[0] = diff ? TREF_FALSE : TREF_TRUE;
+  TRef tra = J->base[0];
+  TRef trb = J->base[1];
+  if (tra && trb) {
+    int diff = rec_objcmp(J, tra, trb, &rd->argv[0], &rd->argv[1]);
+    J->base[0] = diff ? TREF_FALSE : TREF_TRUE;
   }  /* else: Interpreter will throw. */
 }
 
-static void recff_tonumber(jit_State *J, TRef *res, RecordFFData *rd)
+static void LJ_FASTCALL recff_tonumber(jit_State *J, RecordFFData *rd)
 {
-  TRef tr = arg[0];
+  TRef tr = J->base[0];
   if (tref_isnumber_str(tr)) {
-    if (arg[1]) {
-      TRef base = lj_ir_toint(J, arg[1]);
+    TRef base = J->base[1];
+    if (base) {
+      base = lj_ir_toint(J, base);
       if (!tref_isk(base) || IR(tref_ref(base))->i != 10)
-        recff_err_nyi(J, rd);
+        recff_nyiu(J);
     }
     if (tref_isstr(tr))
       tr = emitir(IRTG(IR_STRTO, IRT_NUM), tr, 0);
   } else {
     tr = TREF_NIL;
   }
-  res[0] = tr;
+  J->base[0] = tr;
   UNUSED(rd);
 }
 
-static void recff_tostring(jit_State *J, TRef *res, RecordFFData *rd)
+static TValue *recff_tostring_cp(lua_State *L, lua_CFunction dummy, void *ud)
+{
+  jit_State *J = (jit_State *)ud;
+  rec_tailcall(J, 0, 1);
+  UNUSED(L); UNUSED(dummy);
+  return NULL;
+}
+
+static void LJ_FASTCALL recff_tostring(jit_State *J, RecordFFData *rd)
 {
-  TRef tr = arg[0];
+  TRef tr = J->base[0];
   if (tref_isstr(tr)) {
     /* Ignore __tostring in the string base metatable. */
-    res[0] = tr;
+    /* Pass on result in J->base[0]. */
   } else {
     RecordIndex ix;
     ix.tab = tr;
     copyTV(J->L, &ix.tabv, &rd->argv[0]);
     if (rec_mm_lookup(J, &ix, MM_tostring)) {  /* Has __tostring metamethod? */
-      if (rd->metacall)  /* Must not use kludge. */
-        recff_err_nyi(J, rd);
-      res[0] = ix.mobj;
-      copyTV(J->L, rd->argv - 1, &ix.mobjv);  /* Kludge. */
-      J->framedepth--;
-      if (rec_call(J, (BCReg)(res - J->base), 1, 1))
-        J->framedepth++;
-      else
-        rd->cres = CALLRES_PENDING;
-      /* Otherwise res[0] already contains the result. */
+      int errcode;
+      /* Temporarily insert metamethod below object. */
+      J->base[1] = tr;
+      J->base[0] = ix.mobj;
+      copyTV(J->L, &rd->argv[1], &rd->argv[0]);
+      copyTV(J->L, &rd->argv[0], &ix.mobjv);
+      /* Need to protect rec_tailcall because it may throw. */
+      errcode = lj_vm_cpcall(J->L, NULL, J, recff_tostring_cp);
+      /* Always undo Lua stack changes to avoid confusing the interpreter. */
+      copyTV(J->L, &rd->argv[0], &rd->argv[1]);
+      if (errcode)
+        lj_err_throw(J->L, errcode);  /* Propagate errors. */
+      rd->nres = -1;  /* Pending call. */
     } else if (tref_isnumber(tr)) {
-      res[0] = emitir(IRT(IR_TOSTR, IRT_STR), tr, 0);
+      J->base[0] = emitir(IRT(IR_TOSTR, IRT_STR), tr, 0);
     } else if (tref_ispri(tr)) {
-      res[0] = lj_ir_kstr(J, strV(&rd->fn->c.upvalue[tref_type(tr)]));
+      J->base[0] = lj_ir_kstr(J, strV(&J->fn->c.upvalue[tref_type(tr)]));
     } else {
-      recff_err_nyi(J, rd);
+      recff_nyiu(J);
     }
   }
 }
 
-static void recff_ipairs_aux(jit_State *J, TRef *res, RecordFFData *rd)
+static void LJ_FASTCALL recff_ipairs_aux(jit_State *J, RecordFFData *rd)
 {
   RecordIndex ix;
-  ix.tab = arg[0];
+  ix.tab = J->base[0];
   if (tref_istab(ix.tab)) {
     if (!tvisnum(&rd->argv[1]))  /* No support for string coercion. */
       lj_trace_err(J, LJ_TRERR_BADTYPE);
     setnumV(&ix.keyv, numV(&rd->argv[1])+(lua_Number)1);
     settabV(J->L, &ix.tabv, tabV(&rd->argv[0]));
     ix.val = 0; ix.idxchain = 0;
-    ix.key = lj_ir_toint(J, arg[1]);
-    res[0] = ix.key = emitir(IRTI(IR_ADD), ix.key, lj_ir_kint(J, 1));
-    res[1] = rec_idx(J, &ix);
-    rd->nres = tref_isnil(res[1]) ? 0 : 2;
+    ix.key = lj_ir_toint(J, J->base[1]);
+    J->base[0] = ix.key = emitir(IRTI(IR_ADD), ix.key, lj_ir_kint(J, 1));
+    J->base[1] = rec_idx(J, &ix);
+    rd->nres = tref_isnil(J->base[1]) ? 0 : 2;
   }  /* else: Interpreter will throw. */
 }
 
-static void recff_ipairs(jit_State *J, TRef *res, RecordFFData *rd)
+static void LJ_FASTCALL recff_ipairs(jit_State *J, RecordFFData *rd)
 {
-  TRef tab = arg[0];
+  TRef tab = J->base[0];
   if (tref_istab(tab)) {
-    res[0] = lj_ir_kfunc(J, funcV(&rd->fn->c.upvalue[0]));
-    res[1] = tab;
-    res[2] = lj_ir_kint(J, 0);
+    J->base[0] = lj_ir_kfunc(J, funcV(&J->fn->c.upvalue[0]));
+    J->base[1] = tab;
+    J->base[2] = lj_ir_kint(J, 0);
     rd->nres = 3;
   }  /* else: Interpreter will throw. */
 }
 
-static void recff_pcall(jit_State *J, TRef *res, RecordFFData *rd)
+static void LJ_FASTCALL recff_pcall(jit_State *J, RecordFFData *rd)
 {
-  if (rd->nargs >= 1) {
-    BCReg parg = (BCReg)(arg - J->base);
+  if (J->maxslot >= 1) {
     J->pc = (const BCIns *)(sizeof(TValue) - 4 +
                             (hook_active(J2G(J)) ? FRAME_PCALLH : FRAME_PCALL));
-    if (rec_call(J, parg, CALLRES_MULTI, rd->nargs - 1)) {  /* Resolved call. */
-      res[0] = TREF_TRUE;  /* Prepend true result. No need to move results. */
-      rd->nres = (ptrdiff_t)J->maxslot - (ptrdiff_t)parg + 1;
-    } else {  /* Propagate pending call. */
-      rd->cres = CALLRES_PENDING;
-    }
+    rec_call(J, 0, J->maxslot - 1);
+    rd->nres = -1;  /* Pending call. */
   }  /* else: Interpreter will throw. */
 }
 
-/* Struct to pass context across lj_vm_cpcall. */
-typedef struct RecordXpcall {
-  ptrdiff_t nargs;
-  BCReg parg;
-  int resolved;
-} RecordXpcall;
-
 static TValue *recff_xpcall_cp(lua_State *L, lua_CFunction dummy, void *ud)
 {
-  jit_State *J = L2J(L);
-  RecordXpcall *rx = (RecordXpcall *)ud;
-  UNUSED(dummy);
-  rx->resolved = rec_call(J, rx->parg, CALLRES_MULTI, rx->nargs);
+  jit_State *J = (jit_State *)ud;
+  rec_call(J, 1, J->maxslot - 2);
+  UNUSED(L); UNUSED(dummy);
   return NULL;
 }
 
-static void recff_xpcall(jit_State *J, TRef *res, RecordFFData *rd)
+static void LJ_FASTCALL recff_xpcall(jit_State *J, RecordFFData *rd)
 {
-  if (rd->nargs >= 2) {
-    RecordXpcall rx;
-    BCReg parg = (BCReg)(arg - J->base) + 1;
-    TRef tmp;
+  if (J->maxslot >= 2) {
     TValue argv0, argv1;
-    ptrdiff_t oargv;
+    TRef tmp;
     int errcode;
     /* Swap function and traceback. */
-    tmp = arg[0]; arg[0] = arg[1]; arg[1] = tmp;
+    tmp = J->base[0]; J->base[0] = J->base[1]; J->base[1] = tmp;
     copyTV(J->L, &argv0, &rd->argv[0]);
     copyTV(J->L, &argv1, &rd->argv[1]);
     copyTV(J->L, &rd->argv[0], &argv1);
     copyTV(J->L, &rd->argv[1], &argv0);
-    oargv = savestack(J->L, rd->argv);
     J->pc = (const BCIns *)(2*sizeof(TValue) - 4 +
                             (hook_active(J2G(J)) ? FRAME_PCALLH : FRAME_PCALL));
-    /* Need to protect rec_call because the recorder may throw. */
-    rx.parg = parg;
-    rx.nargs = rd->nargs - 2;
-    errcode = lj_vm_cpcall(J->L, NULL, &rx, recff_xpcall_cp);
+    /* Need to protect rec_call because it may throw. */
+    errcode = lj_vm_cpcall(J->L, NULL, J, recff_xpcall_cp);
     /* Always undo Lua stack swap to avoid confusing the interpreter. */
-    rd->argv = restorestack(J->L, oargv);  /* Stack may have been resized. */
     copyTV(J->L, &rd->argv[0], &argv0);
     copyTV(J->L, &rd->argv[1], &argv1);
     if (errcode)
       lj_err_throw(J->L, errcode);  /* Propagate errors. */
-    if (rx.resolved) {  /* Resolved call. */
-      ptrdiff_t i, nres = (ptrdiff_t)J->maxslot - (ptrdiff_t)parg;
-      rd->nres = nres + 1;
-      res[0] = TREF_TRUE;  /* Prepend true result. */
-      for (i = 1; i <= nres; i++)  /* Move results down. */
-        res[i] = res[i+1];
-    } else {  /* Propagate pending call. */
-      rd->cres = CALLRES_PENDING;
-    }
+    rd->nres = -1;  /* Pending call. */
   }  /* else: Interpreter will throw. */
 }
 
 /* -- Math library fast functions ----------------------------------------- */
 
-static void recff_math_abs(jit_State *J, TRef *res, RecordFFData *rd)
+static void LJ_FASTCALL recff_math_abs(jit_State *J, RecordFFData *rd)
 {
-  TRef tr = lj_ir_tonum(J, arg[0]);
-  res[0] = emitir(IRTN(IR_ABS), tr, lj_ir_knum_abs(J));
+  TRef tr = lj_ir_tonum(J, J->base[0]);
+  J->base[0] = emitir(IRTN(IR_ABS), tr, lj_ir_knum_abs(J));
   UNUSED(rd);
 }
 
 /* Record rounding functions math.floor and math.ceil. */
-static void recff_math_round(jit_State *J, TRef *res, RecordFFData *rd)
+static void LJ_FASTCALL recff_math_round(jit_State *J, RecordFFData *rd)
 {
-  if (tref_isinteger(arg[0]))
-    res[0] = arg[0];
-  else
-    res[0] = emitir(IRTN(IR_FPMATH), lj_ir_tonum(J, arg[0]), rd->data);
+  if (!tref_isinteger(J->base[0]))  /* Pass through integers unmodified. */
+    J->base[0] = emitir(IRTN(IR_FPMATH), lj_ir_tonum(J, J->base[0]), rd->data);
   /* Note: result is integral (or NaN/Inf), but may not fit into an integer. */
 }
 
 /* Record unary math.* functions, mapped to IR_FPMATH opcode. */
-static void recff_math_unary(jit_State *J, TRef *res, RecordFFData *rd)
+static void LJ_FASTCALL recff_math_unary(jit_State *J, RecordFFData *rd)
 {
-  res[0] = emitir(IRTN(IR_FPMATH), lj_ir_tonum(J, arg[0]), rd->data);
+  J->base[0] = emitir(IRTN(IR_FPMATH), lj_ir_tonum(J, J->base[0]), rd->data);
 }
 
 /* Record binary math.* functions math.atan2 and math.ldexp. */
-static void recff_math_binary(jit_State *J, TRef *res, RecordFFData *rd)
+static void LJ_FASTCALL recff_math_binary(jit_State *J, RecordFFData *rd)
 {
-  TRef tr = lj_ir_tonum(J, arg[0]);
-  res[0] = emitir(IRTN(rd->data), tr, lj_ir_tonum(J, arg[1]));
+  TRef tr = lj_ir_tonum(J, J->base[0]);
+  J->base[0] = emitir(IRTN(rd->data), tr, lj_ir_tonum(J, J->base[1]));
 }
 
 /* Record math.asin, math.acos, math.atan. */
-static void recff_math_atrig(jit_State *J, TRef *res, RecordFFData *rd)
+static void LJ_FASTCALL recff_math_atrig(jit_State *J, RecordFFData *rd)
 {
-  TRef y = lj_ir_tonum(J, arg[0]);
+  TRef y = lj_ir_tonum(J, J->base[0]);
   TRef x = lj_ir_knum_one(J);
   uint32_t ffid = rd->data;
   if (ffid != FF_math_atan) {
@@ -1250,66 +1349,70 @@ static void recff_math_atrig(jit_State *J, TRef *res, RecordFFData *rd)
     tmp = emitir(IRTN(IR_FPMATH), tmp, IRFPM_SQRT);
     if (ffid == FF_math_asin) { x = tmp; } else { x = y; y = tmp; }
   }
-  res[0] = emitir(IRTN(IR_ATAN2), y, x);
+  J->base[0] = emitir(IRTN(IR_ATAN2), y, x);
 }
 
-static void recff_math_htrig(jit_State *J, TRef *res, RecordFFData *rd)
+static void LJ_FASTCALL recff_math_htrig(jit_State *J, RecordFFData *rd)
 {
-  TRef tr = lj_ir_tonum(J, arg[0]);
-  res[0] = lj_ir_call(J, rd->data, tr);
+  TRef tr = lj_ir_tonum(J, J->base[0]);
+  J->base[0] = lj_ir_call(J, rd->data, tr);
 }
 
-static void recff_math_modf(jit_State *J, TRef *res, RecordFFData *rd)
+static void LJ_FASTCALL recff_math_modf(jit_State *J, RecordFFData *rd)
 {
-  TRef tr = arg[0];
+  TRef tr = J->base[0];
   if (tref_isinteger(tr)) {
-    res[0] = tr;
-    res[1] = lj_ir_kint(J, 0);
+    J->base[0] = tr;
+    J->base[1] = lj_ir_kint(J, 0);
   } else {
+    TRef trt;
     tr = lj_ir_tonum(J, tr);
-    res[0] = emitir(IRTN(IR_FPMATH), tr, IRFPM_TRUNC);
-    res[1] = emitir(IRTN(IR_SUB), tr, res[0]);
+    trt = emitir(IRTN(IR_FPMATH), tr, IRFPM_TRUNC);
+    J->base[0] = trt;
+    J->base[1] = emitir(IRTN(IR_SUB), tr, trt);
   }
   rd->nres = 2;
 }
 
-static void recff_math_degrad(jit_State *J, TRef *res, RecordFFData *rd)
+static void LJ_FASTCALL recff_math_degrad(jit_State *J, RecordFFData *rd)
 {
-  TRef tr = lj_ir_tonum(J, arg[0]);
-  res[0] = emitir(IRTN(IR_MUL), tr, lj_ir_knum(J, numV(&rd->fn->c.upvalue[0])));
+  TRef tr = lj_ir_tonum(J, J->base[0]);
+  TRef trm = lj_ir_knum(J, numV(&J->fn->c.upvalue[0]));
+  J->base[0] = emitir(IRTN(IR_MUL), tr, trm);
+  UNUSED(rd);
 }
 
-static void recff_math_pow(jit_State *J, TRef *res, RecordFFData *rd)
+static void LJ_FASTCALL recff_math_pow(jit_State *J, RecordFFData *rd)
 {
-  TRef tr = lj_ir_tonum(J, arg[0]);
-  if (!tref_isnumber_str(arg[1]))
+  TRef tr = lj_ir_tonum(J, J->base[0]);
+  if (!tref_isnumber_str(J->base[1]))
     lj_trace_err(J, LJ_TRERR_BADTYPE);
-  res[0] = lj_opt_narrow_pow(J, tr, arg[1], &rd->argv[1]);
+  J->base[0] = lj_opt_narrow_pow(J, tr, J->base[1], &rd->argv[1]);
   UNUSED(rd);
 }
 
-static void recff_math_minmax(jit_State *J, TRef *res, RecordFFData *rd)
+static void LJ_FASTCALL recff_math_minmax(jit_State *J, RecordFFData *rd)
 {
-  TRef tr = lj_ir_tonum(J, arg[0]);
+  TRef tr = lj_ir_tonum(J, J->base[0]);
   uint32_t op = rd->data;
   BCReg i;
-  for (i = 1; arg[i]; i++)
-    tr = emitir(IRTN(op), tr, lj_ir_tonum(J, arg[i]));
-  res[0] = tr;
+  for (i = 1; J->base[i] != 0; i++)
+    tr = emitir(IRTN(op), tr, lj_ir_tonum(J, J->base[i]));
+  J->base[0] = tr;
 }
 
-static void recff_math_random(jit_State *J, TRef *res, RecordFFData *rd)
+static void LJ_FASTCALL recff_math_random(jit_State *J, RecordFFData *rd)
 {
-  GCudata *ud = udataV(&rd->fn->c.upvalue[0]);
+  GCudata *ud = udataV(&J->fn->c.upvalue[0]);
   TRef tr, one;
   lj_ir_kgc(J, obj2gco(ud), IRT_UDATA);  /* Prevent collection. */
   tr = lj_ir_call(J, IRCALL_lj_math_random_step, lj_ir_kptr(J, uddata(ud)));
   one = lj_ir_knum_one(J);
   tr = emitir(IRTN(IR_SUB), tr, one);
-  if (arg[0]) {
-    TRef tr1 = lj_ir_tonum(J, arg[0]);
-    if (arg[1]) {  /* d = floor(d*(r2-r1+1.0)) + r1 */
-      TRef tr2 = lj_ir_tonum(J, arg[1]);
+  if (J->base[0]) {
+    TRef tr1 = lj_ir_tonum(J, J->base[0]);
+    if (J->base[1]) {  /* d = floor(d*(r2-r1+1.0)) + r1 */
+      TRef tr2 = lj_ir_tonum(J, J->base[1]);
       tr2 = emitir(IRTN(IR_SUB), tr2, tr1);
       tr2 = emitir(IRTN(IR_ADD), tr2, one);
       tr = emitir(IRTN(IR_MUL), tr, tr2);
@@ -1321,69 +1424,82 @@ static void recff_math_random(jit_State *J, TRef *res, RecordFFData *rd)
       tr = emitir(IRTN(IR_ADD), tr, one);
     }
   }
-  res[0] = tr;
+  J->base[0] = tr;
+  UNUSED(rd);
 }
 
 /* -- Bit library fast functions ------------------------------------------ */
 
 /* Record unary bit.tobit, bit.bnot, bit.bswap. */
-static void recff_bit_unary(jit_State *J, TRef *res, RecordFFData *rd)
+static void LJ_FASTCALL recff_bit_unary(jit_State *J, RecordFFData *rd)
 {
-  TRef tr = lj_ir_tobit(J, arg[0]);
-  res[0] = (rd->data == IR_TOBIT) ? tr : emitir(IRTI(rd->data), tr, 0);
+  TRef tr = lj_ir_tobit(J, J->base[0]);
+  J->base[0] = (rd->data == IR_TOBIT) ? tr : emitir(IRTI(rd->data), tr, 0);
 }
 
 /* Record N-ary bit.band, bit.bor, bit.bxor. */
-static void recff_bit_nary(jit_State *J, TRef *res, RecordFFData *rd)
+static void LJ_FASTCALL recff_bit_nary(jit_State *J, RecordFFData *rd)
 {
-  TRef tr = lj_ir_tobit(J, arg[0]);
+  TRef tr = lj_ir_tobit(J, J->base[0]);
   uint32_t op = rd->data;
   BCReg i;
-  for (i = 1; arg[i]; i++)
-    tr = emitir(IRTI(op), tr, lj_ir_tobit(J, arg[i]));
-  res[0] = tr;
+  for (i = 1; J->base[i] != 0; i++)
+    tr = emitir(IRTI(op), tr, lj_ir_tobit(J, J->base[i]));
+  J->base[0] = tr;
 }
 
 /* Record bit shifts. */
-static void recff_bit_shift(jit_State *J, TRef *res, RecordFFData *rd)
+static void LJ_FASTCALL recff_bit_shift(jit_State *J, RecordFFData *rd)
 {
-  TRef tr = lj_ir_tobit(J, arg[0]);
-  TRef tsh = lj_ir_tobit(J, arg[1]);
+  TRef tr = lj_ir_tobit(J, J->base[0]);
+  TRef tsh = lj_ir_tobit(J, J->base[1]);
 #if !LJ_TARGET_MASKEDSHIFT
   if (!tref_isk(tsh))
     tsh = emitir(IRTI(IR_BAND), tsh, lj_ir_kint(J, 31));
 #endif
-  res[0] = emitir(IRTI(rd->data), tr, tsh);
+  J->base[0] = emitir(IRTI(rd->data), tr, tsh);
 }
 
 /* -- String library fast functions --------------------------------------- */
 
-static void recff_string_len(jit_State *J, TRef *res, RecordFFData *rd)
+static void LJ_FASTCALL recff_string_len(jit_State *J, RecordFFData *rd)
 {
-  res[0] = emitir(IRTI(IR_FLOAD), lj_ir_tostr(J, arg[0]), IRFL_STR_LEN);
+  J->base[0] = emitir(IRTI(IR_FLOAD), lj_ir_tostr(J, J->base[0]), IRFL_STR_LEN);
   UNUSED(rd);
 }
 
 /* Handle string.byte (rd->data = 0) and string.sub (rd->data = 1). */
-static void recff_string_range(jit_State *J, TRef *res, RecordFFData *rd)
+static void LJ_FASTCALL recff_string_range(jit_State *J, RecordFFData *rd)
 {
-  TRef trstr = lj_ir_tostr(J, arg[0]);
+  TRef trstr = lj_ir_tostr(J, J->base[0]);
   TRef trlen = emitir(IRTI(IR_FLOAD), trstr, IRFL_STR_LEN);
   TRef tr0 = lj_ir_kint(J, 0);
   TRef trstart, trend;
   GCstr *str = argv2str(J, &rd->argv[0]);
   int32_t start, end;
   if (rd->data) {  /* string.sub(str, start [,end]) */
-    trstart = lj_ir_toint(J, arg[1]);
-    trend = tref_isnil(arg[2]) ? lj_ir_kint(J, -1) : lj_ir_toint(J, arg[2]);
     start = argv2int(J, &rd->argv[1]);
-    end = tref_isnil(arg[2]) ? -1 : argv2int(J, &rd->argv[2]);
+    trstart = lj_ir_toint(J, J->base[1]);
+    trend = J->base[2];
+    if (tref_isnil(trend)) {
+      trend = lj_ir_kint(J, -1);
+      end = -1;
+    } else {
+      trend = lj_ir_toint(J, trend);
+      end = argv2int(J, &rd->argv[2]);
+    }
   } else {  /* string.byte(str, [,start [,end]]) */
-    if (arg[1]) {
-      trstart = lj_ir_toint(J, arg[1]);
-      trend = tref_isnil(arg[2]) ? trstart : lj_ir_toint(J, arg[2]);
+    if (J->base[1]) {
       start = argv2int(J, &rd->argv[1]);
-      end = tref_isnil(arg[2]) ? start : argv2int(J, &rd->argv[2]);
+      trstart = lj_ir_toint(J, J->base[1]);
+      trend = J->base[2];
+      if (tref_isnil(trend)) {
+        trend = trstart;
+        end = start;
+      } else {
+        trend = lj_ir_toint(J, trend);
+        end = argv2int(J, &rd->argv[2]);
+      }
     } else {
       trend = trstart = lj_ir_kint(J, 1);
       end = start = 1;
@@ -1426,23 +1542,23 @@ static void recff_string_range(jit_State *J, TRef *res, RecordFFData *rd)
       TRef trptr, trslen = emitir(IRTI(IR_SUB), trend, trstart);
       emitir(IRTGI(IR_GE), trslen, tr0);
       trptr = emitir(IRT(IR_STRREF, IRT_PTR), trstr, trstart);
-      res[0] = emitir(IRT(IR_SNEW, IRT_STR), trptr, trslen);
+      J->base[0] = emitir(IRT(IR_SNEW, IRT_STR), trptr, trslen);
     } else {  /* Range underflow: return empty string. */
       emitir(IRTGI(IR_LT), trend, trstart);
-      res[0] = lj_ir_kstr(J, lj_str_new(J->L, strdata(str), 0));
+      J->base[0] = lj_ir_kstr(J, lj_str_new(J->L, strdata(str), 0));
     }
   } else {  /* Return string.byte result(s). */
     ptrdiff_t i, len = end - start;
     if (len > 0) {
       TRef trslen = emitir(IRTI(IR_SUB), trend, trstart);
       emitir(IRTGI(IR_EQ), trslen, lj_ir_kint(J, len));
-      if (res + len > J->slot + LJ_MAX_JSLOTS)
-        lj_trace_err(J, LJ_TRERR_STACKOV);
+      if (J->baseslot + len > LJ_MAX_JSLOTS)
+        lj_trace_err_info(J, LJ_TRERR_STACKOV);
       rd->nres = len;
       for (i = 0; i < len; i++) {
         TRef tmp = emitir(IRTI(IR_ADD), trstart, lj_ir_kint(J, i));
         tmp = emitir(IRT(IR_STRREF, IRT_PTR), trstr, tmp);
-        res[i] = emitir(IRT(IR_XLOAD, IRT_U8), tmp, IRXLOAD_READONLY);
+        J->base[i] = emitir(IRT(IR_XLOAD, IRT_U8), tmp, IRXLOAD_READONLY);
       }
     } else {  /* Empty range or range underflow: return no results. */
       emitir(IRTGI(IR_LE), trend, trstart);
@@ -1453,62 +1569,63 @@ static void recff_string_range(jit_State *J, TRef *res, RecordFFData *rd)
 
 /* -- Table library fast functions ---------------------------------------- */
 
-static void recff_table_getn(jit_State *J, TRef *res, RecordFFData *rd)
+static void LJ_FASTCALL recff_table_getn(jit_State *J, RecordFFData *rd)
 {
-  if (tref_istab(arg[0])) {
-    res[0] = lj_ir_call(J, IRCALL_lj_tab_len, arg[0]);
-  }  /* else: Interpreter will throw. */
+  if (tref_istab(J->base[0]))
+    J->base[0] = lj_ir_call(J, IRCALL_lj_tab_len, J->base[0]);
+  /* else: Interpreter will throw. */
   UNUSED(rd);
 }
 
-static void recff_table_remove(jit_State *J, TRef *res, RecordFFData *rd)
+static void LJ_FASTCALL recff_table_remove(jit_State *J, RecordFFData *rd)
 {
-  if (tref_istab(arg[0])) {
-    if (!arg[1] || tref_isnil(arg[1])) {  /* Simple pop: t[#t] = nil */
-      TRef trlen = lj_ir_call(J, IRCALL_lj_tab_len, arg[0]);
+  TRef tab = J->base[0];
+  rd->nres = 0;
+  if (tref_istab(tab)) {
+    if (!J->base[1] || tref_isnil(J->base[1])) {  /* Simple pop: t[#t] = nil */
+      TRef trlen = lj_ir_call(J, IRCALL_lj_tab_len, tab);
       GCtab *t = tabV(&rd->argv[0]);
       MSize len = lj_tab_len(t);
       emitir(IRTGI(len ? IR_NE : IR_EQ), trlen, lj_ir_kint(J, 0));
       if (len) {
         RecordIndex ix;
-        ix.tab = arg[0];
+        ix.tab = tab;
         ix.key = trlen;
         settabV(J->L, &ix.tabv, t);
         setintV(&ix.keyv, len);
         ix.idxchain = 0;
-        if (rd->cres != 0) {  /* Specialize load only if result needed. */
+        if (results_wanted(J) != 0) {  /* Specialize load only if needed. */
           ix.val = 0;
-          res[0] = rec_idx(J, &ix);  /* Load previous value. */
+          J->base[0] = rec_idx(J, &ix);  /* Load previous value. */
+          rd->nres = 1;
           /* Assumes ix.key/ix.tab is not modified for raw rec_idx(). */
         }
         ix.val = TREF_NIL;
         rec_idx(J, &ix);  /* Remove value. */
-      } else {
-        rd->nres = 0;
       }
     } else {  /* Complex case: remove in the middle. */
-      recff_err_nyi(J, rd);
+      recff_nyiu(J);
     }
   }  /* else: Interpreter will throw. */
 }
 
-static void recff_table_insert(jit_State *J, TRef *res, RecordFFData *rd)
+static void LJ_FASTCALL recff_table_insert(jit_State *J, RecordFFData *rd)
 {
+  RecordIndex ix;
+  ix.tab = J->base[0];
+  ix.val = J->base[1];
   rd->nres = 0;
-  if (tref_istab(arg[0]) && arg[1]) {
-    if (!arg[2]) {  /* Simple push: t[#t+1] = v */
-      TRef trlen = lj_ir_call(J, IRCALL_lj_tab_len, arg[0]);
+  if (tref_istab(ix.tab) && ix.val) {
+    if (!J->base[2]) {  /* Simple push: t[#t+1] = v */
+      TRef trlen = lj_ir_call(J, IRCALL_lj_tab_len, ix.tab);
       GCtab *t = tabV(&rd->argv[0]);
-      RecordIndex ix;
-      ix.tab = arg[0];
-      ix.val = arg[1];
       ix.key = emitir(IRTI(IR_ADD), trlen, lj_ir_kint(J, 1));
       settabV(J->L, &ix.tabv, t);
       setintV(&ix.keyv, lj_tab_len(t) + 1);
       ix.idxchain = 0;
       rec_idx(J, &ix);  /* Set new value. */
     } else {  /* Complex case: insert in the middle. */
-      recff_err_nyi(J, rd);
+      recff_nyiu(J);
     }
   }  /* else: Interpreter will throw. */
 }
@@ -1518,14 +1635,14 @@ static void recff_table_insert(jit_State *J, TRef *res, RecordFFData *rd)
 /* Get FILE* for I/O function. Any I/O error aborts recording, so there's
 ** no need to encode the alternate cases for any of the guards.
 */
-static TRef recff_io_fp(jit_State *J, TRef *res, uint32_t id)
+static TRef recff_io_fp(jit_State *J, uint32_t id)
 {
   TRef tr, ud, fp;
   if (id) {  /* io.func() */
     tr = lj_ir_kptr(J, &J2G(J)->gcroot[id]);
     ud = emitir(IRT(IR_XLOAD, IRT_UDATA), tr, 0);
   } else {  /* fp:method() */
-    ud = arg[0];
+    ud = J->base[0];
     if (!tref_isudata(ud))
       lj_trace_err(J, LJ_TRERR_BADTYPE);
     tr = emitir(IRT(IR_FLOAD, IRT_U8), ud, IRFL_UDATA_UDTYPE);
@@ -1536,130 +1653,75 @@ static TRef recff_io_fp(jit_State *J, TRef *res, uint32_t id)
   return fp;
 }
 
-static void recff_io_write(jit_State *J, TRef *res, RecordFFData *rd)
+static void LJ_FASTCALL recff_io_write(jit_State *J, RecordFFData *rd)
 {
-  TRef fp = recff_io_fp(J, res, rd->data);
+  TRef fp = recff_io_fp(J, rd->data);
   TRef zero = lj_ir_kint(J, 0);
   TRef one = lj_ir_kint(J, 1);
   ptrdiff_t i = rd->data == 0 ? 1 : 0;
-  for (; arg[i]; i++) {
-    TRef str = lj_ir_tostr(J, arg[i]);
+  for (; J->base[i]; i++) {
+    TRef str = lj_ir_tostr(J, J->base[i]);
     TRef buf = emitir(IRT(IR_STRREF, IRT_PTR), str, zero);
     TRef len = emitir(IRTI(IR_FLOAD), str, IRFL_STR_LEN);
     if (tref_isk(len) && IR(tref_ref(len))->i == 1) {
       TRef tr = emitir(IRT(IR_XLOAD, IRT_U8), buf, IRXLOAD_READONLY);
       tr = lj_ir_call(J, IRCALL_fputc, tr, fp);
-      if (rd->cres != 0)  /* Check result only if requested. */
+      if (results_wanted(J) != 0)  /* Check result only if not ignored. */
         emitir(IRTGI(IR_NE), tr, lj_ir_kint(J, -1));
     } else {
       TRef tr = lj_ir_call(J, IRCALL_fwrite, buf, one, len, fp);
-      if (rd->cres != 0)  /* Check result only if requested. */
+      if (results_wanted(J) != 0)  /* Check result only if not ignored. */
         emitir(IRTGI(IR_EQ), tr, len);
     }
   }
-  res[0] = TREF_TRUE;
+  J->base[0] = TREF_TRUE;
 }
 
-static void recff_io_flush(jit_State *J, TRef *res, RecordFFData *rd)
+static void LJ_FASTCALL recff_io_flush(jit_State *J, RecordFFData *rd)
 {
-  TRef fp = recff_io_fp(J, res, rd->data);
+  TRef fp = recff_io_fp(J, rd->data);
   TRef tr = lj_ir_call(J, IRCALL_fflush, fp);
-  if (rd->cres != 0)  /* Check result only if requested. */
+  if (results_wanted(J) != 0)  /* Check result only if not ignored. */
     emitir(IRTGI(IR_EQ), tr, lj_ir_kint(J, 0));
-  res[0] = TREF_TRUE;
+  J->base[0] = TREF_TRUE;
 }
 
-/* -- Record calls and returns -------------------------------------------- */
-
-#undef arg
-
 #include "lj_recdef.h"
 
-/* Record return. */
-static void rec_ret(jit_State *J, BCReg rbase, ptrdiff_t gotresults)
+static uint32_t recdef_lookup(GCfunc *fn)
 {
-  TValue *frame = J->L->base - 1;
-  ptrdiff_t i;
-  for (i = 0; i < gotresults; i++)
-    getslot(J, rbase+i);  /* Ensure all results have a reference. */
-  J->tailcalled = 0;
-  while (frame_ispcall(frame)) {  /* Immediately resolve pcall() returns. */
-    BCReg cbase = (BCReg)frame_delta(frame);
-    if (J->framedepth-- <= 0)
-      lj_trace_err(J, LJ_TRERR_NYIRETL);
-    lua_assert(J->baseslot > 1);
-    J->base[--rbase] = TREF_TRUE;  /* Prepend true to results. */
-    gotresults++;
-    rbase += cbase;
-    J->baseslot -= (BCReg)cbase;
-    J->base -= cbase;
-    frame = frame_prevd(frame);
-  }
-  if (frame_islua(frame)) {  /* Return to Lua frame. */
-    BCIns callins = *(frame_pc(frame)-1);
-    ptrdiff_t nresults = bc_b(callins) ? (ptrdiff_t)bc_b(callins)-1 :gotresults;
-    BCReg cbase = bc_a(callins);
-    for (i = 0; i < nresults; i++)  /* Adjust results. */
-      J->base[i-1] = i < gotresults ? J->base[rbase+i] : TREF_NIL;
-    J->maxslot = cbase+(BCReg)nresults;
-    if (J->framedepth > 0) {  /* Return to a frame that is part of the trace. */
-      J->framedepth--;
-      lua_assert(J->baseslot > cbase+1);
-      J->baseslot -= cbase+1;
-      J->base -= cbase+1;
-    } else if (J->parent == 0) {
-      /* Return to lower frame would leave the loop in a root trace. */
-      lj_trace_err(J, LJ_TRERR_LLEAVE);
-    } else {  /* Return to lower frame. Guard for the target we return to. */
-      GCproto *pt = funcproto(frame_func(frame - (cbase+1)));
-      TRef trpt = lj_ir_kgc(J, obj2gco(pt), IRT_PROTO);
-      TRef trpc = lj_ir_kptr(J, (void *)frame_pc(frame));
-      emitir(IRTG(IR_RETF, IRT_PTR), trpt, trpc);
-      J->retdepth++;
-      J->needsnap = 1;
-      lua_assert(J->baseslot == 1);
-      /* Shift result slots up and clear the slots of the new frame below. */
-      memmove(J->base + cbase, J->base-1, sizeof(TRef)*nresults);
-      memset(J->base-1, 0, sizeof(TRef)*(cbase+1));
-    }
-  } else if (frame_iscont(frame)) {  /* Return to continuation frame. */
-    ASMFunction cont = frame_contf(frame);
-    BCReg cbase = (BCReg)frame_delta(frame);
-    if ((J->framedepth -= 2) <= 0)
-      lj_trace_err(J, LJ_TRERR_NYIRETL);
-    J->baseslot -= (BCReg)cbase;
-    J->base -= cbase;
-    J->maxslot = cbase-2;
-    if (cont == lj_cont_ra) {
-      /* Copy result to destination slot. */
-      BCReg dst = bc_a(*(frame_contpc(frame)-1));
-      J->base[dst] = gotresults ? J->base[cbase+rbase] : TREF_NIL;
-      if (dst > J->maxslot) J->maxslot = dst+1;
-    } else if (cont == lj_cont_nop) {
-      /* Nothing to do here. */
-    } else if (cont == lj_cont_cat) {
-      lua_assert(0);
-    } else {
-      /* Result type already specialized. */
-      lua_assert(cont == lj_cont_condf || cont == lj_cont_condt);
-    }
-  } else {
-    lj_trace_err(J, LJ_TRERR_NYIRETL);  /* NYI: handle return to C frame. */
-  }
-  lua_assert(J->baseslot >= 1);
+  if (fn->c.ffid < sizeof(recff_idmap)/sizeof(recff_idmap[0]))
+    return recff_idmap[fn->c.ffid];
+  else
+    return 0;
+}
+
+/* Record call to fast function or C function. */
+static void rec_func_ff(jit_State *J)
+{
+  RecordFFData rd;
+  uint32_t m = recdef_lookup(J->fn);
+  rd.data = m & 0xff;
+  rd.nres = 1;  /* Default is one result. */
+  rd.argv = J->L->base;
+  J->base[J->maxslot] = 0;  /* Mark end of arguments. */
+  (recff_func[m >> 8])(J, &rd);  /* Call recff_* handler. */
+  if (rd.nres >= 0)
+    rec_ret(J, 0, rd.nres);
 }
 
+/* -- Record calls to Lua functions --------------------------------------- */
+
 /* Check unroll limits for calls. */
-static void check_call_unroll(jit_State *J, GCfunc *fn)
+static void check_call_unroll(jit_State *J)
 {
   IRRef fref = tref_ref(J->base[-1]);
   int32_t count = 0;
-  ptrdiff_t s;
-  for (s = (ptrdiff_t)J->baseslot - 1; s > 0; s--)
+  BCReg s;
+  for (s = J->baseslot - 1; s > 0; s--)
     if ((J->slot[s] & TREF_FRAME) && tref_ref(J->slot[s]) == fref)
       count++;
-  if (bc_op(J->cur.startins) == BC_CALL &&
-      funcproto(fn) == &gcref(J->cur.startpt)->pt) {
+  if (J->pc == J->startpc) {
     if (count + J->tailcalled > J->param[JIT_P_recunroll])
       lj_trace_err(J, LJ_TRERR_NYIRECU);
   } else {
@@ -1668,110 +1730,20 @@ static void check_call_unroll(jit_State *J, GCfunc *fn)
   }
 }
 
-/* Record call. Returns 0 for pending calls and 1 for resolved calls. */
-static int rec_call(jit_State *J, BCReg func, ptrdiff_t cres, ptrdiff_t nargs)
+static void rec_func_lua(jit_State *J)
 {
-  RecordFFData rd;
-  TRef trfunc, *res = &J->base[func];
-  TValue *tv = &J->L->base[func];
-
-  if (tref_isfunc(res[0])) {  /* Regular function call. */
-    rd.fn = funcV(tv);
-    rd.argv = tv+1;
-    rd.metacall = 0;
-  } else {  /* Otherwise resolve __call metamethod for called object. */
-    RecordIndex ix;
-    ptrdiff_t i;
-    ix.tab = res[0];
-    copyTV(J->L, &ix.tabv, tv);
-    if (!rec_mm_lookup(J, &ix, MM_call) || !tref_isfunc(ix.mobj))
-      lj_trace_err(J, LJ_TRERR_NOMM);
-    /* Update the recorder state, but not the Lua stack. */
-    for (i = ++nargs; i > 0; i--)
-      res[i] = res[i-1];
-    res[0] = ix.mobj;
-    rd.fn = funcV(&ix.mobjv);
-    rd.argv = tv;  /* The called object is the 1st arg. */
-    rd.metacall = 1;
-  }
-
-  /* Specialize to the runtime value of the called function. */
-  trfunc = lj_ir_kfunc(J, rd.fn);
-  emitir(IRTG(IR_EQ, IRT_FUNC), res[0], trfunc);
-  res[0] = trfunc | TREF_FRAME;
-
-  /* Add frame links. */
-  J->frame[J->framedepth++] = SNAP_MKPC(J->pc+1);
-  if (cres == CALLRES_CONT)  /* Continuations need an extra frame stack slot. */
-    J->frame[J->framedepth++] = SNAP_MKFTSZ((func+1)*sizeof(TValue)+FRAME_CONT);
-    /* NYI: func is wrong if any fast function ever sets up a continuation. */
-  if (J->framedepth > LJ_MAX_JFRAME)
+  GCproto *pt = J->pt;
+  if ((pt->flags & PROTO_NO_JIT))
+    lj_trace_err(J, LJ_TRERR_CJITOFF);
+  lua_assert(!(pt->flags & PROTO_IS_VARARG));
+  if (J->baseslot + pt->framesize >= LJ_MAX_JSLOTS)
     lj_trace_err(J, LJ_TRERR_STACKOV);
-
-  if (isluafunc(rd.fn)) {  /* Record call to Lua function. */
-    GCproto *pt = funcproto(rd.fn);
-    if ((pt->flags & PROTO_NO_JIT))
-      lj_trace_err(J, LJ_TRERR_CJITOFF);
-    if ((pt->flags & PROTO_IS_VARARG))
-      lj_trace_err(J, LJ_TRERR_NYIVF);
-    if (cres == CALLRES_TAILCALL) {
-      ptrdiff_t i;
-      /* Tailcalls can form a loop, so count towards the loop unroll limit. */
-      if (++J->tailcalled > J->loopunroll)
-        lj_trace_err(J, LJ_TRERR_LUNROLL);
-      for (i = 0; i <= nargs; i++)  /* Move func + args down. */
-        J->base[i-1] = res[i];
-      J->framedepth--;
-      /* Note: the new FRAME is now at J->base[-1] (even for slot #0). */
-    } else {  /* Regular call. */
-      J->base += func+1;
-      J->baseslot += func+1;
-    }
-    if (J->baseslot + pt->framesize >= LJ_MAX_JSLOTS)
-      lj_trace_err(J, LJ_TRERR_STACKOV);
-    /* Fill up missing args with nil. */
-    while (nargs < pt->numparams)
-      J->base[nargs++] = TREF_NIL;
-    /* The remaining slots should never be read before they are written. */
-    J->maxslot = pt->numparams;
-    check_call_unroll(J, rd.fn);
-    return 0;  /* No result yet. */
-  } else {  /* Record call to C function or fast function. */
-    uint32_t m = 0;
-    BCReg oldmaxslot = J->maxslot;
-    res[1+nargs] = 0;
-    rd.nargs = nargs;
-    if (rd.fn->c.ffid < sizeof(recff_idmap)/sizeof(recff_idmap[0]))
-      m = recff_idmap[rd.fn->c.ffid];
-    rd.data = m & 0xff;
-    rd.cres = cres;
-    rd.nres = 1;  /* Default is one result. */
-    (recff_func[m >> 8])(J, res, &rd);  /* Call recff_* handler. */
-    J->framedepth--;
-    cres = rd.cres;
-    if (cres >= 0) {
-      /* Caller takes fixed number of results: local a,b = f() */
-      J->maxslot = func + (BCReg)cres;
-      while (rd.nres < cres)  /* Fill up missing results with nil. */
-        res[rd.nres++] = TREF_NIL;
-    } else if (cres == CALLRES_MULTI) {
-      /* Caller takes any number of results: return 1,f() */
-      J->maxslot = func + (BCReg)rd.nres;
-    } else if (cres == CALLRES_TAILCALL) {
-      /* Tail call: return f() */
-      rec_ret(J, func, rd.nres);
-    } else if (cres == CALLRES_CONT) {
-      /* Note: immediately resolved continuations must not change J->maxslot. */
-      J->maxslot = oldmaxslot;
-      J->framedepth--;
-      res[rd.nres] = TREF_NIL;  /* Turn 0 results into nil result. */
-    } else {
-      lua_assert(cres == CALLRES_PENDING);
-      J->framedepth++;
-      return 0;  /* Pending call, no result yet. */
-    }
-    return 1;  /* Result resolved immediately. */
-  }
+  /* Fill up missing args with nil. */
+  while (J->maxslot < pt->numparams)
+    J->base[J->maxslot++] = TREF_NIL;
+  /* The remaining slots should never be read before they are written. */
+  J->maxslot = pt->numparams;
+  check_call_unroll(J);
 }
 
 /* -- Record allocations -------------------------------------------------- */
@@ -2068,22 +2040,25 @@ void lj_record_ins(jit_State *J)
       copyTV(J->L, b+1, b-2);
       copyTV(J->L, b+2, b-1);
     }
-    goto callop;
+    rec_call(J, ra, (ptrdiff_t)rc-1);
+    break;
 
-  case BC_CALLMT:
-    rb = (TRef)(CALLRES_TAILCALL+1);
-    /* fallthrough */
+  /* L->top is set to L->base+ra+rc+NARGS-1+1. See lj_dispatch_ins(). */
   case BC_CALLM:
-    /* L->top is set to L->base+ra+rc+NRESULTS-1+1, see lj_dispatch_ins(). */
     rc = (BCReg)(J->L->top - J->L->base) - ra;
-    goto callop;
-
-  case BC_CALLT:
-    rb = (TRef)(CALLRES_TAILCALL+1);
     /* fallthrough */
   case BC_CALL:
-  callop:
-    rec_call(J, ra, (ptrdiff_t)rb-1, (ptrdiff_t)rc-1);
+    rec_call(J, ra, (ptrdiff_t)rc-1);
+    break;
+
+  case BC_CALLMT:
+    rc = (BCReg)(J->L->top - J->L->base) - ra;
+    /* fallthrough */
+  case BC_CALLT:
+    rec_tailcall(J, ra, (ptrdiff_t)rc-1);
+    /* Tailcalls can form a loop, so count towards the loop unroll limit. */
+    if (++J->tailcalled > J->loopunroll)
+      lj_trace_err(J, LJ_TRERR_LUNROLL);
     break;
 
   /* -- Returns ----------------------------------------------------------- */
@@ -2094,6 +2069,7 @@ void lj_record_ins(jit_State *J)
     /* fallthrough */
   case BC_RET: case BC_RET0: case BC_RET1:
     rec_ret(J, ra, (ptrdiff_t)rc-1);
+    J->tailcalled = 0;  /* NYI: logic is broken, need a better check. */
     break;
 
   /* -- Loops and branches ------------------------------------------------ */
@@ -2132,6 +2108,8 @@ void lj_record_ins(jit_State *J)
   case BC_IFORL:
   case BC_IITERL:
   case BC_ILOOP:
+  case BC_IFUNCF:
+  case BC_IFUNCV:
     lj_trace_err(J, LJ_TRERR_LBLACKL);
     break;
 
@@ -2140,12 +2118,37 @@ void lj_record_ins(jit_State *J)
       J->maxslot = ra;  /* Shrink used slots. */
     break;
 
+  /* -- Function headers -------------------------------------------------- */
+
+  case BC_FUNCF:
+    rec_func_lua(J);
+    break;
+
+  case BC_JFUNCF:
+    lj_trace_err(J, LJ_TRERR_NYILNKF);
+    break;
+
+  case BC_FUNCV:
+  case BC_JFUNCV:
+    lj_trace_err(J, LJ_TRERR_NYIVF);
+    break;
+
+  case BC_FUNCC:
+  case BC_FUNCCW:
+    rec_func_ff(J);
+    break;
+
+  default:
+    if (op >= BC__MAX) {
+      rec_func_ff(J);
+      break;
+    }
+    /* fallthrough */
   case BC_CAT:
   case BC_UCLO:
   case BC_FNEW:
   case BC_TSETM:
   case BC_VARG:
-  default:
     setintV(&J->errinfo, (int32_t)op);
     lj_trace_err_info(J, LJ_TRERR_NYIBC);
     break;