summaryrefslogtreecommitdiff
path: root/src/lj_record.c
diff options
context:
space:
mode:
authorMike Pall <mike>2009-12-08 19:46:35 +0100
committerMike Pall <mike>2009-12-08 19:46:35 +0100
commit55b16959717084884fd4a0cbae6d19e3786c20c7 (patch)
treec8a07a43c13679751ed25a9d06796e9e7b2134a6 /src/lj_record.c
downloadluajit-2.0.0-beta1.tar.gz
luajit-2.0.0-beta1.tar.bz2
luajit-2.0.0-beta1.zip
RELEASE LuaJIT-2.0.0-beta1v2.0.0-beta1
Diffstat (limited to 'src/lj_record.c')
-rw-r--r--src/lj_record.c2136
1 files changed, 2136 insertions, 0 deletions
diff --git a/src/lj_record.c b/src/lj_record.c
new file mode 100644
index 00000000..e101ba23
--- /dev/null
+++ b/src/lj_record.c
@@ -0,0 +1,2136 @@
1/*
2** Trace recorder (bytecode -> SSA IR).
3** Copyright (C) 2005-2009 Mike Pall. See Copyright Notice in luajit.h
4*/
5
6#define lj_record_c
7#define LUA_CORE
8
9#include "lj_obj.h"
10
11#if LJ_HASJIT
12
13#include "lj_gc.h"
14#include "lj_err.h"
15#include "lj_str.h"
16#include "lj_tab.h"
17#include "lj_state.h"
18#include "lj_frame.h"
19#include "lj_bc.h"
20#include "lj_ff.h"
21#include "lj_ir.h"
22#include "lj_jit.h"
23#include "lj_iropt.h"
24#include "lj_trace.h"
25#include "lj_record.h"
26#include "lj_snap.h"
27#include "lj_asm.h"
28#include "lj_dispatch.h"
29#include "lj_vm.h"
30
31/* Some local macros to save typing. Undef'd at the end. */
32#define IR(ref) (&J->cur.ir[(ref)])
33
34/* Pass IR on to next optimization in chain (FOLD). */
35#define emitir(ot, a, b) (lj_ir_set(J, (ot), (a), (b)), lj_opt_fold(J))
36
37/* Emit raw IR without passing through optimizations. */
38#define emitir_raw(ot, a, b) (lj_ir_set(J, (ot), (a), (b)), lj_ir_emit(J))
39
40/* Context for recording an indexed load/store. */
41typedef struct RecordIndex {
42 TValue tabv; /* Runtime value of table (or indexed object). */
43 TValue keyv; /* Runtime value of key. */
44 TValue valv; /* Runtime value of stored value. */
45 TValue mobjv; /* Runtime value of metamethod object. */
46 GCtab *mtv; /* Runtime value of metatable object. */
47 cTValue *oldv; /* Runtime value of previously stored value. */
48 TRef tab; /* Table (or indexed object) reference. */
49 TRef key; /* Key reference. */
50 TRef val; /* Value reference for a store or 0 for a load. */
51 TRef mt; /* Metatable reference. */
52 TRef mobj; /* Metamethod object reference. */
53 int idxchain; /* Index indirections left or 0 for raw lookup. */
54} RecordIndex;
55
56/* Requested results from rec_call(). */
57enum {
58 /* Non-negative numbers are number of requested results. */
59 CALLRES_MULTI = -1, /* Return multiple results. */
60 CALLRES_TAILCALL = -2, /* Tail call. */
61 CALLRES_PENDING = -3, /* Call is pending, no results yet. */
62 CALLRES_CONT = -4 /* Continuation call. */
63};
64
65/* Forward declarations. */
66static TRef rec_idx(jit_State *J, RecordIndex *ix);
67static int rec_call(jit_State *J, BCReg func, int cres, int nargs);
68
69/* -- Sanity checks ------------------------------------------------------- */
70
71#ifdef LUA_USE_ASSERT
72/* Sanity check the whole IR -- sloooow. */
73static void rec_check_ir(jit_State *J)
74{
75 IRRef i, nins = J->cur.nins, nk = J->cur.nk;
76 lua_assert(nk <= REF_BIAS && nins >= REF_BIAS && nins < 65536);
77 for (i = nins-1; i >= nk; i--) {
78 IRIns *ir = IR(i);
79 uint32_t mode = lj_ir_mode[ir->o];
80 IRRef op1 = ir->op1;
81 IRRef op2 = ir->op2;
82 switch (irm_op1(mode)) {
83 case IRMnone: lua_assert(op1 == 0); break;
84 case IRMref: lua_assert(op1 >= nk);
85 lua_assert(i >= REF_BIAS ? op1 < i : op1 > i); break;
86 case IRMlit: break;
87 case IRMcst: lua_assert(i < REF_BIAS); continue;
88 }
89 switch (irm_op2(mode)) {
90 case IRMnone: lua_assert(op2 == 0); break;
91 case IRMref: lua_assert(op2 >= nk);
92 lua_assert(i >= REF_BIAS ? op2 < i : op2 > i); break;
93 case IRMlit: break;
94 case IRMcst: lua_assert(0); break;
95 }
96 if (ir->prev) {
97 lua_assert(ir->prev >= nk);
98 lua_assert(i >= REF_BIAS ? ir->prev < i : ir->prev > i);
99 lua_assert(IR(ir->prev)->o == ir->o);
100 }
101 }
102}
103
104/* Sanity check the slots. */
105static void rec_check_slots(jit_State *J)
106{
107 BCReg s, nslots = J->baseslot + J->maxslot;
108 lua_assert(J->baseslot >= 1 && J->baseslot < LJ_MAX_JSLOTS);
109 lua_assert(nslots < LJ_MAX_JSLOTS);
110 for (s = 0; s < nslots; s++) {
111 TRef tr = J->slot[s];
112 if (tr) {
113 IRRef ref = tref_ref(tr);
114 lua_assert(ref >= J->cur.nk && ref < J->cur.nins);
115 lua_assert(irt_t(IR(ref)->t) == tref_t(tr));
116 }
117 }
118}
119#endif
120
121/* -- Type handling and specialization ------------------------------------ */
122
123/* Note: these functions return tagged references (TRef). */
124
125/* Specialize a slot to a specific type. Note: slot can be negative! */
126static TRef sloadt(jit_State *J, int32_t slot, IRType t, int mode)
127{
128 /* No guard, since none of the callers need a type-checking SLOAD. */
129 TRef ref = emitir_raw(IRT(IR_SLOAD, t), (int32_t)J->baseslot+slot, mode);
130 J->base[slot] = ref;
131 return ref;
132}
133
134/* Specialize a slot to the runtime type. Note: slot can be negative! */
135static TRef sload(jit_State *J, int32_t slot)
136{
137 IRType t = itype2irt(&J->L->base[slot]);
138 TRef ref = emitir_raw(IRTG(IR_SLOAD, t), (int32_t)J->baseslot+slot, 0);
139 if (irtype_ispri(t)) ref = TREF_PRI(t); /* Canonicalize primitive refs. */
140 J->base[slot] = ref;
141 return ref;
142}
143
144/* Get TRef from slot. Load slot and specialize if not done already. */
145#define getslot(J, s) (J->base[(s)] ? J->base[(s)] : sload(J, (int32_t)(s)))
146
147/* Get TRef for current function. */
148static TRef getcurrf(jit_State *J)
149{
150 if (J->base[-1]) {
151 IRIns *ir = IR(tref_ref(J->base[-1]));
152 if (ir->o == IR_FRAME) /* Shortcut if already specialized. */
153 return TREF(ir->op2, IRT_FUNC); /* Return TRef of KFUNC. */
154 return J->base[-1];
155 } else {
156 lua_assert(J->baseslot == 1);
157 return sloadt(J, -1, IRT_FUNC, IRSLOAD_READONLY);
158 }
159}
160
161/* Compare for raw object equality.
162** Returns 0 if the objects are the same.
163** Returns 1 if they are different, but the same type.
164** Returns 2 for two different types.
165** Comparisons between primitives always return 1 -- no caller cares about it.
166*/
167static int rec_objcmp(jit_State *J, TRef a, TRef b, cTValue *av, cTValue *bv)
168{
169 int diff = !lj_obj_equal(av, bv);
170 if (!tref_isk2(a, b)) { /* Shortcut, also handles primitives. */
171 IRType ta = tref_type(a);
172 IRType tb = tref_type(b);
173 if (ta != tb) {
174 /* Widen mixed number/int comparisons to number/number comparison. */
175 if (ta == IRT_INT && tb == IRT_NUM) {
176 a = emitir(IRTN(IR_TONUM), a, 0);
177 ta = IRT_NUM;
178 } else if (ta == IRT_NUM && tb == IRT_INT) {
179 b = emitir(IRTN(IR_TONUM), b, 0);
180 } else {
181 return 2; /* Two different types are never equal. */
182 }
183 }
184 emitir(IRTG(diff ? IR_NE : IR_EQ, ta), a, b);
185 }
186 return diff;
187}
188
189/* -- Record loop ops ----------------------------------------------------- */
190
191/* Loop event. */
192typedef enum {
193 LOOPEV_LEAVE, /* Loop is left or not entered. */
194 LOOPEV_ENTER /* Loop is entered. */
195} LoopEvent;
196
197/* Canonicalize slots: convert integers to numbers. */
198static void canonicalize_slots(jit_State *J)
199{
200 BCReg s;
201 for (s = J->baseslot+J->maxslot-1; s >= 1; s--) {
202 TRef tr = J->slot[s];
203 if (tref_isinteger(tr)) {
204 IRIns *ir = IR(tref_ref(tr));
205 if (!(ir->o == IR_SLOAD && (ir->op2 & IRSLOAD_READONLY)))
206 J->slot[s] = emitir(IRTN(IR_TONUM), tr, 0);
207 }
208 }
209}
210
211/* Stop recording. */
212static void rec_stop(jit_State *J, TraceNo lnk)
213{
214 lj_trace_end(J);
215 J->cur.link = (uint16_t)lnk;
216 if (lnk == J->curtrace) { /* Looping back? */
217 if ((J->flags & JIT_F_OPT_LOOP)) /* Shall we try to create a loop? */
218 goto nocanon; /* Do not canonicalize or we lose the narrowing. */
219 if (J->cur.root) /* Otherwise ensure we always link to the root trace. */
220 J->cur.link = J->cur.root;
221 }
222 canonicalize_slots(J);
223nocanon:
224 /* Note: all loop ops must set J->pc to the following instruction! */
225 lj_snap_add(J); /* Add loop snapshot. */
226 J->needsnap = 0;
227 J->mergesnap = 1; /* In case recording continues. */
228}
229
230/* Peek before FORI to find a const initializer, otherwise load from slot. */
231static TRef fori_arg(jit_State *J, const BCIns *pc, BCReg slot, IRType t)
232{
233 /* A store to slot-1 means there's no conditional assignment for slot. */
234 if (bc_a(pc[-1]) == slot-1 && bcmode_a(bc_op(pc[-1])) == BCMdst) {
235 BCIns ins = pc[0];
236 if (bc_a(ins) == slot) {
237 if (bc_op(ins) == BC_KSHORT) {
238 int32_t k = (int32_t)(int16_t)bc_d(ins);
239 if (t == IRT_INT)
240 return lj_ir_kint(J, k);
241 else
242 return lj_ir_knum(J, cast_num(k));
243 } else if (bc_op(ins) == BC_KNUM) {
244 lua_Number n = J->pt->k.n[bc_d(ins)];
245 if (t == IRT_INT)
246 return lj_ir_kint(J, lj_num2int(n));
247 else
248 return lj_ir_knum(J, n);
249 }
250 }
251 }
252 if (J->base[slot])
253 return J->base[slot];
254 else
255 return sloadt(J, (int32_t)slot, t, IRSLOAD_READONLY|IRSLOAD_INHERIT);
256}
257
258/* Simulate the runtime behavior of the FOR loop iterator.
259** It's important to exactly reproduce the semantics of the interpreter.
260*/
261static LoopEvent for_iter(jit_State *J, IROp *op, BCReg ra, int isforl)
262{
263 cTValue *forbase = &J->L->base[ra];
264 lua_Number stopv = numV(&forbase[FORL_STOP]);
265 lua_Number idxv = numV(&forbase[FORL_IDX]);
266 if (isforl)
267 idxv += numV(&forbase[FORL_STEP]);
268 if ((int32_t)forbase[FORL_STEP].u32.hi >= 0) {
269 if (idxv <= stopv) { *op = IR_LE; return LOOPEV_ENTER; }
270 *op = IR_GT; return LOOPEV_LEAVE;
271 } else {
272 if (stopv <= idxv) { *op = IR_GE; return LOOPEV_ENTER; }
273 *op = IR_LT; return LOOPEV_LEAVE;
274 }
275}
276
277/* Record FORL/JFORL or FORI/JFORI. */
278static LoopEvent rec_for(jit_State *J, const BCIns *fori, int isforl)
279{
280 BCReg ra = bc_a(*fori);
281 IROp op;
282 LoopEvent ev = for_iter(J, &op, ra, isforl);
283 TRef *tr = &J->base[ra];
284 TRef idx, stop;
285 IRType t;
286 if (isforl) { /* Handle FORL/JFORL opcodes. */
287 TRef step;
288 idx = tr[FORL_IDX];
289 if (!idx) idx = sloadt(J, (int32_t)(ra+FORL_IDX), IRT_NUM, 0);
290 t = tref_type(idx);
291 stop = fori_arg(J, fori-2, ra+FORL_STOP, t);
292 step = fori_arg(J, fori-1, ra+FORL_STEP, t);
293 tr[FORL_IDX] = idx = emitir(IRT(IR_ADD, t), idx, step);
294 } else { /* Handle FORI/JFORI opcodes. */
295 BCReg i;
296 t = IRT_NUM;
297 for (i = FORL_IDX; i <= FORL_STEP; i++) {
298 lua_assert(J->base[ra+i] != 0); /* Assumes the slots are already set. */
299 tr[i] = lj_ir_tonum(J, J->base[ra+i]);
300 }
301 idx = tr[FORL_IDX];
302 stop = tr[FORL_STOP];
303 if (!tref_isk(tr[FORL_STEP])) /* Non-const step: need direction guard. */
304 emitir(IRTG(((op-IR_LT)>>1)+IR_LT, IRT_NUM),
305 tr[FORL_STEP], lj_ir_knum_zero(J));
306 }
307
308 tr[FORL_EXT] = idx;
309 if (ev == LOOPEV_LEAVE) {
310 J->maxslot = ra+FORL_EXT+1;
311 J->pc = fori+1;
312 } else {
313 J->maxslot = ra;
314 J->pc = fori+bc_j(*fori)+1;
315 }
316 lj_snap_add(J);
317
318 emitir(IRTG(op, t), idx, stop);
319
320 if (ev == LOOPEV_LEAVE) {
321 J->maxslot = ra;
322 J->pc = fori+bc_j(*fori)+1;
323 } else {
324 J->maxslot = ra+FORL_EXT+1;
325 J->pc = fori+1;
326 }
327 J->needsnap = 1;
328 return ev;
329}
330
331/* Record ITERL/JITERL. */
332static LoopEvent rec_iterl(jit_State *J, const BCIns iterins)
333{
334 BCReg ra = bc_a(iterins);
335 lua_assert(J->base[ra] != 0);
336 if (!tref_isnil(J->base[ra])) { /* Looping back? */
337 J->base[ra-1] = J->base[ra]; /* Copy result of ITERC to control var. */
338 J->maxslot = ra-1+bc_b(J->pc[-1]);
339 J->pc += bc_j(iterins)+1;
340 return LOOPEV_ENTER;
341 } else {
342 J->maxslot = ra-3;
343 J->pc++;
344 return LOOPEV_LEAVE;
345 }
346}
347
348/* Record LOOP/JLOOP. Now, that was easy. */
349static LoopEvent rec_loop(jit_State *J, BCReg ra)
350{
351 J->maxslot = ra;
352 J->pc++;
353 return LOOPEV_ENTER;
354}
355
356/* Check if a loop repeatedly failed to trace because it didn't loop back. */
357static int innerloopleft(jit_State *J, const BCIns *pc)
358{
359 ptrdiff_t i;
360 for (i = 0; i < PENALTY_SLOTS; i++)
361 if (J->penalty[i].pc == pc) {
362 if (J->penalty[i].reason == LJ_TRERR_LLEAVE &&
363 J->penalty[i].val >= 2*HOTCOUNT_MIN_PENALTY)
364 return 1;
365 break;
366 }
367 return 0;
368}
369
370/* Handle the case when an interpreted loop op is hit. */
371static void rec_loop_interp(jit_State *J, const BCIns *pc, LoopEvent ev)
372{
373 if (J->parent == 0) {
374 if (pc == J->startpc && J->framedepth == 0) { /* Same loop? */
375 if (ev == LOOPEV_LEAVE) /* Must loop back to form a root trace. */
376 lj_trace_err(J, LJ_TRERR_LLEAVE);
377 rec_stop(J, J->curtrace); /* Root trace forms a loop. */
378 } else if (ev != LOOPEV_LEAVE) { /* Entering inner loop? */
379 /* It's usually better to abort here and wait until the inner loop
380 ** is traced. But if the inner loop repeatedly didn't loop back,
381 ** this indicates a low trip count. In this case try unrolling
382 ** an inner loop even in a root trace. But it's better to be a bit
383 ** more conservative here and only do it for very short loops.
384 */
385 if (!innerloopleft(J, pc))
386 lj_trace_err(J, LJ_TRERR_LINNER); /* Root trace hit an inner loop. */
387 if ((J->loopref && J->cur.nins - J->loopref > 8) || --J->loopunroll < 0)
388 lj_trace_err(J, LJ_TRERR_LUNROLL); /* Limit loop unrolling. */
389 J->loopref = J->cur.nins;
390 }
391 } else if (ev != LOOPEV_LEAVE) { /* Side trace enters an inner loop. */
392 J->loopref = J->cur.nins;
393 if (--J->loopunroll < 0)
394 lj_trace_err(J, LJ_TRERR_LUNROLL); /* Limit loop unrolling. */
395 } /* Side trace continues across a loop that's left or not entered. */
396}
397
398/* Handle the case when an already compiled loop op is hit. */
399static void rec_loop_jit(jit_State *J, TraceNo lnk, LoopEvent ev)
400{
401 if (J->parent == 0) { /* Root trace hit an inner loop. */
402 /* Better let the inner loop spawn a side trace back here. */
403 lj_trace_err(J, LJ_TRERR_LINNER);
404 } else if (ev != LOOPEV_LEAVE) { /* Side trace enters a compiled loop. */
405 J->instunroll = 0; /* Cannot continue across a compiled loop op. */
406 if (J->pc == J->startpc && J->framedepth == 0)
407 lnk = J->curtrace; /* Can form an extra loop. */
408 rec_stop(J, lnk); /* Link to the loop. */
409 } /* Side trace continues across a loop that's left or not entered. */
410}
411
412/* -- Metamethod handling ------------------------------------------------- */
413
414/* Prepare to record call to metamethod. */
415static BCReg rec_mm_prep(jit_State *J, ASMFunction cont)
416{
417 BCReg s, top = curr_proto(J->L)->framesize;
418 TRef trcont;
419 setcont(&J->L->base[top], cont);
420#if LJ_64
421 trcont = lj_ir_kptr(J, (void *)((int64_t)cont - (int64_t)lj_vm_asm_begin));
422#else
423 trcont = lj_ir_kptr(J, (void *)cont);
424#endif
425 J->base[top] = emitir(IRTG(IR_FRAME, IRT_PTR), trcont, trcont);
426 for (s = J->maxslot; s < top; s++)
427 J->base[s] = 0;
428 return top+1;
429}
430
431/* Record metamethod lookup. */
432static int rec_mm_lookup(jit_State *J, RecordIndex *ix, MMS mm)
433{
434 RecordIndex mix;
435 GCtab *mt;
436 if (tref_istab(ix->tab)) {
437 mt = tabref(tabV(&ix->tabv)->metatable);
438 mix.tab = emitir(IRT(IR_FLOAD, IRT_TAB), ix->tab, IRFL_TAB_META);
439 } else if (tref_isudata(ix->tab)) {
440 mt = tabref(udataV(&ix->tabv)->metatable);
441 mix.tab = emitir(IRT(IR_FLOAD, IRT_TAB), ix->tab, IRFL_UDATA_META);
442 } else {
443 /* Specialize to base metatable. Must flush mcode in lua_setmetatable(). */
444 mt = tabref(J2G(J)->basemt[itypemap(&ix->tabv)]);
445 if (mt == NULL)
446 return 0; /* No metamethod. */
447 mix.tab = lj_ir_ktab(J, mt);
448 goto nocheck;
449 }
450 ix->mt = mix.tab;
451 emitir(IRTG(mt ? IR_NE : IR_EQ, IRT_TAB), mix.tab, lj_ir_knull(J, IRT_TAB));
452nocheck:
453 if (mt) {
454 GCstr *mmstr = strref(J2G(J)->mmname[mm]);
455 cTValue *mo = lj_tab_getstr(mt, mmstr);
456 if (mo && !tvisnil(mo))
457 copyTV(J->L, &ix->mobjv, mo);
458 ix->mtv = mt;
459 settabV(J->L, &mix.tabv, mt);
460 setstrV(J->L, &mix.keyv, mmstr);
461 mix.key = lj_ir_kstr(J, mmstr);
462 mix.val = 0;
463 mix.idxchain = 0;
464 ix->mobj = rec_idx(J, &mix);
465 return !tref_isnil(ix->mobj); /* 1 if metamethod found, 0 if not. */
466 }
467 return 0; /* No metamethod. */
468}
469
470/* Record call to arithmetic metamethod (and MM_len). */
471static TRef rec_mm_arith(jit_State *J, RecordIndex *ix, MMS mm)
472{
473 /* Set up metamethod call first to save ix->tab and ix->tabv. */
474 BCReg func = rec_mm_prep(J, lj_cont_ra);
475 TRef *base = J->base + func;
476 TValue *basev = J->L->base + func;
477 base[1] = ix->tab; base[2] = ix->key;
478 copyTV(J->L, basev+1, &ix->tabv);
479 copyTV(J->L, basev+2, &ix->keyv);
480 if (!rec_mm_lookup(J, ix, mm)) { /* Lookup metamethod on 1st operand. */
481 if (mm != MM_len) {
482 ix->tab = ix->key;
483 copyTV(J->L, &ix->tabv, &ix->keyv);
484 if (rec_mm_lookup(J, ix, mm)) /* Lookup metamethod on 2nd operand. */
485 goto ok;
486 }
487 lj_trace_err(J, LJ_TRERR_NOMM);
488 }
489ok:
490 base[0] = ix->mobj;
491 copyTV(J->L, basev+0, &ix->mobjv);
492 return rec_call(J, func, CALLRES_CONT, 2) ? J->base[func] : 0;
493}
494
495/* Call a comparison metamethod. */
496static void rec_mm_callcomp(jit_State *J, RecordIndex *ix, int op)
497{
498 BCReg func = rec_mm_prep(J, (op&1) ? lj_cont_condf : lj_cont_condt);
499 TRef *base = J->base + func;
500 TValue *tv = J->L->base + func;
501 base[0] = ix->mobj; base[1] = ix->val; base[2] = ix->key;
502 copyTV(J->L, tv+0, &ix->mobjv);
503 copyTV(J->L, tv+1, &ix->valv);
504 copyTV(J->L, tv+2, &ix->keyv);
505 rec_call(J, func, CALLRES_CONT, 2);
506 /* It doesn't matter whether this is immediately resolved or not.
507 ** Type specialization of the return type suffices to specialize
508 ** the control flow.
509 */
510}
511
512/* Record call to equality comparison metamethod (for tab and udata only). */
513static void rec_mm_equal(jit_State *J, RecordIndex *ix, int op)
514{
515 ix->tab = ix->val;
516 copyTV(J->L, &ix->tabv, &ix->valv);
517 if (rec_mm_lookup(J, ix, MM_eq)) { /* Lookup metamethod on 1st operand. */
518 cTValue *bv;
519 TRef mo1 = ix->mobj;
520 TValue mo1v;
521 copyTV(J->L, &mo1v, &ix->mobjv);
522 /* Avoid the 2nd lookup and the objcmp if the metatables are equal. */
523 bv = &ix->keyv;
524 if (tvistab(bv) && tabref(tabV(bv)->metatable) == ix->mtv) {
525 TRef mt2 = emitir(IRT(IR_FLOAD, IRT_TAB), ix->key, IRFL_TAB_META);
526 emitir(IRTG(IR_EQ, IRT_TAB), mt2, ix->mt);
527 } else if (tvisudata(bv) && tabref(udataV(bv)->metatable) == ix->mtv) {
528 TRef mt2 = emitir(IRT(IR_FLOAD, IRT_TAB), ix->key, IRFL_UDATA_META);
529 emitir(IRTG(IR_EQ, IRT_TAB), mt2, ix->mt);
530 } else { /* Lookup metamethod on 2nd operand and compare both. */
531 ix->tab = ix->key;
532 copyTV(J->L, &ix->tabv, bv);
533 if (!rec_mm_lookup(J, ix, MM_eq) ||
534 rec_objcmp(J, mo1, ix->mobj, &mo1v, &ix->mobjv))
535 return;
536 }
537 rec_mm_callcomp(J, ix, op);
538 }
539}
540
541/* Record call to ordered comparison metamethods (for arbitrary objects). */
542static void rec_mm_comp(jit_State *J, RecordIndex *ix, int op)
543{
544 ix->tab = ix->val;
545 copyTV(J->L, &ix->tabv, &ix->valv);
546 while (1) {
547 MMS mm = (op & 2) ? MM_le : MM_lt; /* Try __le + __lt or only __lt. */
548 if (rec_mm_lookup(J, ix, mm)) { /* Lookup metamethod on 1st operand. */
549 cTValue *bv;
550 TRef mo1 = ix->mobj;
551 TValue mo1v;
552 copyTV(J->L, &mo1v, &ix->mobjv);
553 /* Avoid the 2nd lookup and the objcmp if the metatables are equal. */
554 bv = &ix->keyv;
555 if (tvistab(bv) && tabref(tabV(bv)->metatable) == ix->mtv) {
556 TRef mt2 = emitir(IRT(IR_FLOAD, IRT_TAB), ix->key, IRFL_TAB_META);
557 emitir(IRTG(IR_EQ, IRT_TAB), mt2, ix->mt);
558 } else if (tvisudata(bv) && tabref(udataV(bv)->metatable) == ix->mtv) {
559 TRef mt2 = emitir(IRT(IR_FLOAD, IRT_TAB), ix->key, IRFL_UDATA_META);
560 emitir(IRTG(IR_EQ, IRT_TAB), mt2, ix->mt);
561 } else { /* Lookup metamethod on 2nd operand and compare both. */
562 ix->tab = ix->key;
563 copyTV(J->L, &ix->tabv, bv);
564 if (!rec_mm_lookup(J, ix, mm) ||
565 rec_objcmp(J, mo1, ix->mobj, &mo1v, &ix->mobjv))
566 goto nomatch;
567 }
568 rec_mm_callcomp(J, ix, op);
569 return;
570 }
571 nomatch:
572 /* First lookup failed. Retry with __lt and swapped operands. */
573 if (!(op & 2)) break; /* Already at __lt. Interpreter will throw. */
574 ix->tab = ix->key; ix->key = ix->val; ix->val = ix->tab;
575 copyTV(J->L, &ix->tabv, &ix->keyv);
576 copyTV(J->L, &ix->keyv, &ix->valv);
577 copyTV(J->L, &ix->valv, &ix->tabv);
578 op ^= 3;
579 }
580}
581
582/* -- Indexed access ------------------------------------------------------ */
583
584/* Record indexed key lookup. */
585static TRef rec_idx_key(jit_State *J, RecordIndex *ix)
586{
587 TRef key;
588 GCtab *t = tabV(&ix->tabv);
589 ix->oldv = lj_tab_get(J->L, t, &ix->keyv); /* Lookup previous value. */
590
591 /* Integer keys are looked up in the array part first. */
592 key = ix->key;
593 if (tref_isnumber(key)) {
594 lua_Number n = numV(&ix->keyv);
595 int32_t k = lj_num2int(n);
596 lua_assert(tvisnum(&ix->keyv));
597 /* Potential array key? */
598 if ((MSize)k < LJ_MAX_ASIZE && n == cast_num(k)) {
599 TRef asizeref, ikey = key;
600 if (!tref_isinteger(ikey))
601 ikey = emitir(IRTGI(IR_TOINT), ikey, IRTOINT_INDEX);
602 asizeref = emitir(IRTI(IR_FLOAD), ix->tab, IRFL_TAB_ASIZE);
603 if ((MSize)k < t->asize) { /* Currently an array key? */
604 TRef arrayref;
605 emitir(IRTGI(IR_ABC), asizeref, ikey); /* Bounds check. */
606 arrayref = emitir(IRT(IR_FLOAD, IRT_PTR), ix->tab, IRFL_TAB_ARRAY);
607 return emitir(IRT(IR_AREF, IRT_PTR), arrayref, ikey);
608 } else { /* Currently not in array (may be an array extension)? */
609 emitir(IRTGI(IR_ULE), asizeref, ikey); /* Inv. bounds check. */
610 if (k == 0 && tref_isk(key))
611 key = lj_ir_knum_zero(J); /* Canonicalize 0 or +-0.0 to +0.0. */
612 /* And continue with the hash lookup. */
613 }
614 } else if (!tref_isk(key)) {
615 /* We can rule out const numbers which failed the integerness test
616 ** above. But all other numbers are potential array keys.
617 */
618 if (t->asize == 0) { /* True sparse tables have an empty array part. */
619 /* Guard that the array part stays empty. */
620 TRef tmp = emitir(IRTI(IR_FLOAD), ix->tab, IRFL_TAB_ASIZE);
621 emitir(IRTGI(IR_EQ), tmp, lj_ir_kint(J, 0));
622 } else {
623 lj_trace_err(J, LJ_TRERR_NYITMIX);
624 }
625 }
626 }
627
628 /* Otherwise the key is located in the hash part. */
629 if (tref_isinteger(key)) /* Hash keys are based on numbers, not ints. */
630 ix->key = key = emitir(IRTN(IR_TONUM), key, 0);
631 if (tref_isk(key)) {
632 /* Optimize lookup of constant hash keys. */
633 MSize hslot = (MSize)((char *)ix->oldv - (char *)&noderef(t->node)[0].val);
634 if (t->hmask > 0 && hslot <= t->hmask*(MSize)sizeof(Node) &&
635 hslot <= 65535*(MSize)sizeof(Node)) {
636 TRef node, kslot;
637 TRef hm = emitir(IRTI(IR_FLOAD), ix->tab, IRFL_TAB_HMASK);
638 emitir(IRTGI(IR_EQ), hm, lj_ir_kint(J, (int32_t)t->hmask));
639 node = emitir(IRT(IR_FLOAD, IRT_PTR), ix->tab, IRFL_TAB_NODE);
640 kslot = lj_ir_kslot(J, key, hslot / sizeof(Node));
641 return emitir(IRTG(IR_HREFK, IRT_PTR), node, kslot);
642 }
643 }
644 /* Fall back to a regular hash lookup. */
645 return emitir(IRT(IR_HREF, IRT_PTR), ix->tab, key);
646}
647
648/* Determine whether a key is NOT one of the fast metamethod names. */
649static int nommstr(jit_State *J, TRef key)
650{
651 if (tref_isstr(key)) {
652 if (tref_isk(key)) {
653 GCstr *str = ir_kstr(IR(tref_ref(key)));
654 uint32_t i;
655 for (i = 0; i <= MM_FAST; i++)
656 if (strref(J2G(J)->mmname[i]) == str)
657 return 0; /* MUST be one the fast metamethod names. */
658 } else {
659 return 0; /* Variable string key MAY be a metamethod name. */
660 }
661 }
662 return 1; /* CANNOT be a metamethod name. */
663}
664
665/* Record indexed load/store. */
666static TRef rec_idx(jit_State *J, RecordIndex *ix)
667{
668 TRef xref;
669 IROp xrefop, loadop;
670 cTValue *oldv;
671
672 while (!tref_istab(ix->tab)) { /* Handle non-table lookup. */
673 lua_assert(ix->idxchain != 0); /* Never call raw rec_idx() on non-table. */
674 if (!rec_mm_lookup(J, ix, ix->val ? MM_newindex : MM_index))
675 lj_trace_err(J, LJ_TRERR_NOMM);
676 handlemm:
677 if (tref_isfunc(ix->mobj)) { /* Handle metamethod call. */
678 BCReg func = rec_mm_prep(J, ix->val ? lj_cont_nop : lj_cont_ra);
679 TRef *base = J->base + func;
680 TValue *tv = J->L->base + func;
681 base[0] = ix->mobj; base[1] = ix->tab; base[2] = ix->key;
682 setfuncV(J->L, tv+0, funcV(&ix->mobjv));
683 copyTV(J->L, tv+1, &ix->tabv);
684 copyTV(J->L, tv+2, &ix->keyv);
685 if (ix->val) {
686 base[3] = ix->val;
687 copyTV(J->L, tv+3, &ix->valv);
688 rec_call(J, func, CALLRES_CONT, 3); /* mobj(tab, key, val) */
689 return 0;
690 } else {
691 /* res = mobj(tab, key) */
692 return rec_call(J, func, CALLRES_CONT, 2) ? J->base[func] : 0;
693 }
694 }
695 /* Otherwise retry lookup with metaobject. */
696 ix->tab = ix->mobj;
697 copyTV(J->L, &ix->tabv, &ix->mobjv);
698 if (--ix->idxchain == 0)
699 lj_trace_err(J, LJ_TRERR_IDXLOOP);
700 }
701
702 /* First catch nil and NaN keys for tables. */
703 if (tvisnil(&ix->keyv) || (tvisnum(&ix->keyv) && tvisnan(&ix->keyv))) {
704 if (ix->val) /* Better fail early. */
705 lj_trace_err(J, LJ_TRERR_STORENN);
706 if (tref_isk(ix->key)) {
707 if (ix->idxchain && rec_mm_lookup(J, ix, MM_index))
708 goto handlemm;
709 return TREF_NIL;
710 }
711 }
712
713 /* Record the key lookup. */
714 xref = rec_idx_key(J, ix);
715 xrefop = IR(tref_ref(xref))->o;
716 loadop = xrefop == IR_AREF ? IR_ALOAD : IR_HLOAD;
717 oldv = ix->oldv;
718
719 if (ix->val == 0) { /* Indexed load */
720 IRType t = itype2irt(oldv);
721 TRef res = emitir(IRTG(loadop, t), xref, 0);
722 if (t == IRT_NIL && ix->idxchain && rec_mm_lookup(J, ix, MM_index))
723 goto handlemm;
724 if (irtype_ispri(t)) res = TREF_PRI(t); /* Canonicalize primitives. */
725 return res;
726 } else { /* Indexed store. */
727 GCtab *mt = tabref(tabV(&ix->tabv)->metatable);
728 if (tvisnil(oldv)) { /* Previous value was nil? */
729 /* Need to duplicate the hasmm check for the early guards. */
730 int hasmm = 0;
731 if (ix->idxchain && mt) {
732 cTValue *mo = lj_tab_getstr(mt, strref(J2G(J)->mmname[MM_newindex]));
733 hasmm = mo && !tvisnil(mo);
734 }
735 if (hasmm || oldv == niltvg(J2G(J)))
736 emitir(IRTG(loadop, IRT_NIL), xref, 0); /* Guard for nil value. */
737 else if (xrefop == IR_HREF)
738 emitir(IRTG(IR_NE, IRT_PTR), xref, lj_ir_kptr(J, niltvg(J2G(J))));
739 if (ix->idxchain && rec_mm_lookup(J, ix, MM_newindex)) { /* Metamethod? */
740 lua_assert(hasmm);
741 goto handlemm;
742 }
743 lua_assert(!hasmm);
744 if (oldv == niltvg(J2G(J))) { /* Need to insert a new key. */
745 TRef key = ix->key;
746 if (tref_isinteger(key)) /* NEWREF needs a TValue as a key. */
747 key = emitir(IRTN(IR_TONUM), key, 0);
748 xref = emitir(IRT(IR_NEWREF, IRT_PTR), ix->tab, key);
749 }
750 } else if (!lj_opt_fwd_wasnonnil(J, loadop, tref_ref(xref))) {
751 /* Cannot derive that the previous value was non-nil, must do checks. */
752 if (xrefop == IR_HREF) /* Guard against store to niltv. */
753 emitir(IRTG(IR_NE, IRT_PTR), xref, lj_ir_kptr(J, niltvg(J2G(J))));
754 if (ix->idxchain) { /* Metamethod lookup required? */
755 /* A check for NULL metatable is cheaper (hoistable) than a load. */
756 if (!mt) {
757 TRef mtref = emitir(IRT(IR_FLOAD, IRT_TAB), ix->tab, IRFL_TAB_META);
758 emitir(IRTG(IR_EQ, IRT_TAB), mtref, lj_ir_knull(J, IRT_TAB));
759 } else {
760 IRType t = itype2irt(oldv);
761 emitir(IRTG(loadop, t), xref, 0); /* Guard for non-nil value. */
762 }
763 }
764 }
765 if (tref_isinteger(ix->val)) /* Convert int to number before storing. */
766 ix->val = emitir(IRTN(IR_TONUM), ix->val, 0);
767 emitir(IRT(loadop+IRDELTA_L2S, tref_type(ix->val)), xref, ix->val);
768 if (tref_isgcv(ix->val))
769 emitir(IRT(IR_TBAR, IRT_NIL), ix->tab, 0);
770 /* Invalidate neg. metamethod cache for stores with certain string keys. */
771 if (!nommstr(J, ix->key)) {
772 TRef fref = emitir(IRT(IR_FREF, IRT_PTR), ix->tab, IRFL_TAB_NOMM);
773 emitir(IRT(IR_FSTORE, IRT_U8), fref, lj_ir_kint(J, 0));
774 }
775 J->needsnap = 1;
776 return 0;
777 }
778}
779
780/* -- Upvalue access ------------------------------------------------------ */
781
782/* Record upvalue load/store. */
783static TRef rec_upvalue(jit_State *J, uint32_t uv, TRef val)
784{
785 GCupval *uvp = &gcref(J->fn->l.uvptr[uv])->uv;
786 TRef fn = getcurrf(J);
787 IRRef uref;
788 int needbarrier = 0;
789 if (!uvp->closed) {
790 /* In current stack? */
791 if (uvp->v >= J->L->stack && uvp->v < J->L->maxstack) {
792 int32_t slot = (int32_t)(uvp->v - (J->L->base - J->baseslot));
793 if (slot >= 0) { /* Aliases an SSA slot? */
794 slot -= (int32_t)J->baseslot; /* Note: slot number may be negative! */
795 /* NYI: add IR to guard that it's still aliasing the same slot. */
796 if (val == 0) {
797 return getslot(J, slot);
798 } else {
799 J->base[slot] = val;
800 if (slot >= (int32_t)J->maxslot) J->maxslot = (BCReg)(slot+1);
801 return 0;
802 }
803 }
804 }
805 uref = tref_ref(emitir(IRTG(IR_UREFO, IRT_PTR), fn, uv));
806 } else {
807 needbarrier = 1;
808 uref = tref_ref(emitir(IRTG(IR_UREFC, IRT_PTR), fn, uv));
809 }
810 if (val == 0) { /* Upvalue load */
811 IRType t = itype2irt(uvp->v);
812 TRef res = emitir(IRTG(IR_ULOAD, t), uref, 0);
813 if (irtype_ispri(t)) res = TREF_PRI(t); /* Canonicalize primitive refs. */
814 return res;
815 } else { /* Upvalue store. */
816 if (tref_isinteger(val)) /* Convert int to number before storing. */
817 val = emitir(IRTN(IR_TONUM), val, 0);
818 emitir(IRT(IR_USTORE, tref_type(val)), uref, val);
819 if (needbarrier && tref_isgcv(val))
820 emitir(IRT(IR_OBAR, IRT_NIL), uref, val);
821 J->needsnap = 1;
822 return 0;
823 }
824}
825
826/* -- Record calls to fast functions -------------------------------------- */
827
828/* Note: The function and the arguments for the bytecode CALL instructions
829** always occupy _new_ stack slots (above the highest active variable).
830** This means they must have been stored there by previous instructions
831** (MOV, K*, ADD etc.) which must be part of the same trace. This in turn
832** means their reference slots are already valid and their types have
833** already been specialized (i.e. getslot() would be redundant).
834** The 1st slot beyond the arguments is set to 0 before calling recff_*.
835*/
836
837/* Data used by handlers to record a fast function. */
838typedef struct RecordFFData {
839 TValue *argv; /* Runtime argument values. */
840 GCfunc *fn; /* The currently recorded function. */
841 int nargs; /* Number of passed arguments. */
842 int nres; /* Number of returned results (defaults to 1). */
843 int cres; /* Wanted number of call results. */
844 uint32_t data; /* Per-ffid auxiliary data (opcode, literal etc.). */
845} RecordFFData;
846
847/* Type of handler to record a fast function. */
848typedef void (*RecordFunc)(jit_State *J, TRef *res, RecordFFData *rd);
849
850/* Avoid carrying two pointers around. */
851#define arg (res+1)
852
853/* Get runtime value of int argument. */
854static int32_t argv2int(jit_State *J, TValue *o)
855{
856 if (tvisstr(o) && !lj_str_numconv(strVdata(o), o))
857 lj_trace_err(J, LJ_TRERR_BADTYPE);
858 return lj_num2bit(numV(o));
859}
860
861/* Get runtime value of string argument. */
862static GCstr *argv2str(jit_State *J, TValue *o)
863{
864 if (LJ_LIKELY(tvisstr(o))) {
865 return strV(o);
866 } else {
867 GCstr *s;
868 lua_assert(tvisnum(o));
869 s = lj_str_fromnum(J->L, &o->n);
870 setstrV(J->L, o, s);
871 return s;
872 }
873}
874
875/* Fallback handler for all fast functions that are not recorded (yet). */
876static void recff_nyi(jit_State *J, TRef *res, RecordFFData *rd)
877{
878 UNUSED(res);
879 setfuncV(J->L, &J->errinfo, rd->fn);
880 lj_trace_err_info(J, LJ_TRERR_NYIFF);
881}
882
883LJ_NORET static void recff_err_ffu(jit_State *J, RecordFFData *rd)
884{
885 setfuncV(J->L, &J->errinfo, rd->fn);
886 lj_trace_err_info(J, LJ_TRERR_NYIFFU);
887}
888
889/* C functions can have arbitrary side-effects and are not recorded (yet). */
890static void recff_c(jit_State *J, TRef *res, RecordFFData *rd)
891{
892 UNUSED(res);
893 setlightudV(&J->errinfo, (void *)rd->fn->c.f);
894 lj_trace_err_info(J, LJ_TRERR_NYICF);
895}
896
897/* -- Base library fast functions ----------------------------------------- */
898
899static void recff_assert(jit_State *J, TRef *res, RecordFFData *rd)
900{
901 /* Arguments already specialized. The interpreter throws for nil/false. */
902 BCReg i;
903 for (i = 0; arg[i]; i++) /* Need to pass through all arguments. */
904 res[i] = arg[i];
905 rd->nres = (int)i;
906 UNUSED(J);
907}
908
909static void recff_type(jit_State *J, TRef *res, RecordFFData *rd)
910{
911 /* Arguments already specialized. Result is a constant string. Neat, huh? */
912 IRType t = tref_isinteger(arg[0]) ? IRT_NUM : tref_type(arg[0]);
913 res[0] = lj_ir_kstr(J, strV(&rd->fn->c.upvalue[t]));
914}
915
916static void recff_getmetatable(jit_State *J, TRef *res, RecordFFData *rd)
917{
918 TRef tr = arg[0];
919 if (tref_istab(tr)) {
920 RecordIndex ix;
921 ix.tab = tr;
922 copyTV(J->L, &ix.tabv, &rd->argv[0]);
923 if (rec_mm_lookup(J, &ix, MM_metatable))
924 res[0] = ix.mobj;
925 else
926 res[0] = ix.mt;
927 } /* else: Interpreter will throw. */
928}
929
930static void recff_setmetatable(jit_State *J, TRef *res, RecordFFData *rd)
931{
932 TRef tr = arg[0];
933 TRef mt = arg[1];
934 if (tref_istab(tr) && (tref_istab(mt) || (mt && tref_isnil(mt)))) {
935 TRef fref, mtref;
936 RecordIndex ix;
937 ix.tab = tr;
938 copyTV(J->L, &ix.tabv, &rd->argv[0]);
939 rec_mm_lookup(J, &ix, MM_metatable); /* Guard for no __metatable field. */
940 fref = emitir(IRT(IR_FREF, IRT_PTR), tr, IRFL_TAB_META);
941 mtref = tref_isnil(mt) ? lj_ir_knull(J, IRT_TAB) : mt;
942 emitir(IRT(IR_FSTORE, IRT_TAB), fref, mtref);
943 if (!tref_isnil(mt))
944 emitir(IRT(IR_TBAR, IRT_TAB), tr, 0);
945 res[0] = tr;
946 J->needsnap = 1;
947 } /* else: Interpreter will throw. */
948}
949
950static void recff_rawget(jit_State *J, TRef *res, RecordFFData *rd)
951{
952 if (tref_istab(arg[0]) && arg[1]) {
953 RecordIndex ix;
954 ix.tab = arg[0]; ix.key = arg[1]; ix.val = 0; ix.idxchain = 0;
955 settabV(J->L, &ix.tabv, tabV(&rd->argv[0]));
956 copyTV(J->L, &ix.keyv, &rd->argv[1]);
957 res[0] = rec_idx(J, &ix);
958 } /* else: Interpreter will throw. */
959}
960
961static void recff_rawset(jit_State *J, TRef *res, RecordFFData *rd)
962{
963 if (tref_istab(arg[0]) && arg[1] && arg[2]) {
964 RecordIndex ix;
965 ix.tab = arg[0]; ix.key = arg[1]; ix.val = arg[2]; ix.idxchain = 0;
966 settabV(J->L, &ix.tabv, tabV(&rd->argv[0]));
967 copyTV(J->L, &ix.keyv, &rd->argv[1]);
968 copyTV(J->L, &ix.valv, &rd->argv[2]);
969 rec_idx(J, &ix);
970 res[0] = arg[0]; /* Returns table. */
971 } /* else: Interpreter will throw. */
972}
973
974static void recff_rawequal(jit_State *J, TRef *res, RecordFFData *rd)
975{
976 if (arg[0] && arg[1]) {
977 int diff = rec_objcmp(J, arg[0], arg[1], &rd->argv[0], &rd->argv[1]);
978 res[0] = diff ? TREF_FALSE : TREF_TRUE;
979 } /* else: Interpreter will throw. */
980}
981
982static void recff_tonumber(jit_State *J, TRef *res, RecordFFData *rd)
983{
984 TRef tr = arg[0];
985 if (tref_isnumber_str(tr)) {
986 if (arg[1]) {
987 TRef base = lj_ir_toint(J, arg[1]);
988 if (!tref_isk(base) || IR(tref_ref(base))->i != 10)
989 recff_err_ffu(J, rd);
990 }
991 if (tref_isstr(tr))
992 tr = emitir(IRTG(IR_STRTO, IRT_NUM), tr, 0);
993 } else {
994 tr = TREF_NIL;
995 }
996 res[0] = tr;
997 UNUSED(rd);
998}
999
1000static void recff_tostring(jit_State *J, TRef *res, RecordFFData *rd)
1001{
1002 TRef tr = arg[0];
1003 if (tref_isstr(tr)) {
1004 /* Ignore __tostring in the string base metatable. */
1005 res[0] = tr;
1006 } else {
1007 RecordIndex ix;
1008 ix.tab = tr;
1009 copyTV(J->L, &ix.tabv, &rd->argv[0]);
1010 if (rec_mm_lookup(J, &ix, MM_tostring)) { /* Has __tostring metamethod? */
1011 res[0] = ix.mobj;
1012 copyTV(J->L, rd->argv - 1, &ix.mobjv);
1013 if (!rec_call(J, (BCReg)(res - J->base), 1, 1)) /* Pending call? */
1014 rd->cres = CALLRES_PENDING;
1015 /* Otherwise res[0] already contains the result. */
1016 } else if (tref_isnumber(tr)) {
1017 res[0] = emitir(IRT(IR_TOSTR, IRT_STR), tr, 0);
1018 } else {
1019 recff_err_ffu(J, rd);
1020 }
1021 }
1022}
1023
1024static void recff_ipairs_aux(jit_State *J, TRef *res, RecordFFData *rd)
1025{
1026 RecordIndex ix;
1027 ix.tab = arg[0];
1028 if (tref_istab(ix.tab)) {
1029 if (!tvisnum(&rd->argv[1])) /* No support for string coercion. */
1030 lj_trace_err(J, LJ_TRERR_BADTYPE);
1031 setnumV(&ix.keyv, numV(&rd->argv[1])+(lua_Number)1);
1032 settabV(J->L, &ix.tabv, tabV(&rd->argv[0]));
1033 ix.val = 0; ix.idxchain = 0;
1034 ix.key = lj_ir_toint(J, arg[1]);
1035 res[0] = ix.key = emitir(IRTI(IR_ADD), ix.key, lj_ir_kint(J, 1));
1036 res[1] = rec_idx(J, &ix);
1037 rd->nres = tref_isnil(res[1]) ? 0 : 2;
1038 } /* else: Interpreter will throw. */
1039}
1040
1041static void recff_ipairs(jit_State *J, TRef *res, RecordFFData *rd)
1042{
1043 TRef tab = arg[0];
1044 if (tref_istab(tab)) {
1045 res[0] = lj_ir_kfunc(J, funcV(&rd->fn->c.upvalue[0]));
1046 res[1] = tab;
1047 res[2] = lj_ir_kint(J, 0);
1048 rd->nres = 3;
1049 } /* else: Interpreter will throw. */
1050}
1051
1052static void recff_pcall(jit_State *J, TRef *res, RecordFFData *rd)
1053{
1054 if (rd->nargs >= 1) {
1055 BCReg parg = (BCReg)(arg - J->base);
1056 if (rec_call(J, parg, CALLRES_MULTI, rd->nargs - 1)) { /* Resolved call. */
1057 res[0] = TREF_TRUE; /* Prepend true result. No need to move results. */
1058 rd->nres = (int)((J->maxslot - parg) + 1);
1059 } else { /* Propagate pending call. */
1060 rd->cres = CALLRES_PENDING;
1061 }
1062 } /* else: Interpreter will throw. */
1063}
1064
1065/* Struct to pass context across lj_vm_cpcall. */
1066typedef struct RecordXpcall {
1067 BCReg parg;
1068 int nargs;
1069 int resolved;
1070} RecordXpcall;
1071
1072static TValue *recff_xpcall_cp(lua_State *L, lua_CFunction dummy, void *ud)
1073{
1074 jit_State *J = L2J(L);
1075 RecordXpcall *rx = (RecordXpcall *)ud;
1076 UNUSED(dummy);
1077 rx->resolved = rec_call(J, rx->parg, CALLRES_MULTI, rx->nargs);
1078 return NULL;
1079}
1080
1081static void recff_xpcall(jit_State *J, TRef *res, RecordFFData *rd)
1082{
1083 if (rd->nargs >= 2) {
1084 RecordXpcall rx;
1085 BCReg parg = (BCReg)(arg - J->base) + 1;
1086 TRef tmp;
1087 TValue argv0, argv1;
1088 ptrdiff_t oargv;
1089 int errcode;
1090 /* Swap function and traceback. */
1091 tmp = arg[0]; arg[0] = arg[1]; arg[1] = tmp;
1092 copyTV(J->L, &argv0, &rd->argv[0]);
1093 copyTV(J->L, &argv1, &rd->argv[1]);
1094 copyTV(J->L, &rd->argv[0], &argv1);
1095 copyTV(J->L, &rd->argv[1], &argv0);
1096 oargv = savestack(J->L, rd->argv);
1097 /* Need to protect rec_call because the recorder may throw. */
1098 rx.parg = parg;
1099 rx.nargs = rd->nargs - 2;
1100 errcode = lj_vm_cpcall(J->L, recff_xpcall_cp, NULL, &rx);
1101 /* Always undo Lua stack swap to avoid confusing the interpreter. */
1102 rd->argv = restorestack(J->L, oargv); /* Stack may have been resized. */
1103 copyTV(J->L, &rd->argv[0], &argv0);
1104 copyTV(J->L, &rd->argv[1], &argv1);
1105 if (errcode)
1106 lj_err_throw(J->L, errcode); /* Propagate errors. */
1107 if (rx.resolved) { /* Resolved call. */
1108 int i, nres = (int)(J->maxslot - parg);
1109 rd->nres = nres + 1;
1110 res[0] = TREF_TRUE; /* Prepend true result. */
1111 for (i = 1; i <= nres; i++) /* Move results down. */
1112 res[i] = res[i+1];
1113 } else { /* Propagate pending call. */
1114 rd->cres = CALLRES_PENDING;
1115 }
1116 } /* else: Interpreter will throw. */
1117}
1118
1119/* -- Math library fast functions ----------------------------------------- */
1120
1121static void recff_math_abs(jit_State *J, TRef *res, RecordFFData *rd)
1122{
1123 TRef tr = lj_ir_tonum(J, arg[0]);
1124 res[0] = emitir(IRTN(IR_ABS), tr, lj_ir_knum_abs(J));
1125 UNUSED(rd);
1126}
1127
1128/* Record rounding functions math.floor and math.ceil. */
1129static void recff_math_round(jit_State *J, TRef *res, RecordFFData *rd)
1130{
1131 if (tref_isinteger(arg[0]))
1132 res[0] = arg[0];
1133 else
1134 res[0] = emitir(IRTN(IR_FPMATH), lj_ir_tonum(J, arg[0]), rd->data);
1135 /* Note: result is integral (or NaN/Inf), but may not fit into an integer. */
1136}
1137
1138/* Record unary math.* functions, mapped to IR_FPMATH opcode. */
1139static void recff_math_unary(jit_State *J, TRef *res, RecordFFData *rd)
1140{
1141 res[0] = emitir(IRTN(IR_FPMATH), lj_ir_tonum(J, arg[0]), rd->data);
1142}
1143
1144/* Record binary math.* functions math.atan2 and math.ldexp. */
1145static void recff_math_binary(jit_State *J, TRef *res, RecordFFData *rd)
1146{
1147 TRef tr = lj_ir_tonum(J, arg[0]);
1148 res[0] = emitir(IRTN(rd->data), tr, lj_ir_tonum(J, arg[1]));
1149}
1150
1151/* Record math.asin, math.acos, math.atan. */
1152static void recff_math_atrig(jit_State *J, TRef *res, RecordFFData *rd)
1153{
1154 TRef y = lj_ir_tonum(J, arg[0]);
1155 TRef x = lj_ir_knum_one(J);
1156 uint32_t ffid = rd->data;
1157 if (ffid != FF_math_atan) {
1158 TRef tmp = emitir(IRTN(IR_MUL), y, y);
1159 tmp = emitir(IRTN(IR_SUB), x, tmp);
1160 tmp = emitir(IRTN(IR_FPMATH), tmp, IRFPM_SQRT);
1161 if (ffid == FF_math_asin) { x = tmp; } else { x = y; y = tmp; }
1162 }
1163 res[0] = emitir(IRTN(IR_ATAN2), y, x);
1164}
1165
1166static void recff_math_modf(jit_State *J, TRef *res, RecordFFData *rd)
1167{
1168 TRef tr = arg[0];
1169 if (tref_isinteger(arg[0])) {
1170 res[0] = tr;
1171 res[1] = lj_ir_kint(J, 0);
1172 } else {
1173 tr = lj_ir_tonum(J, tr);
1174 res[0] = emitir(IRTN(IR_FPMATH), tr, IRFPM_TRUNC);
1175 res[1] = emitir(IRTN(IR_SUB), tr, res[0]);
1176 }
1177 rd->nres = 2;
1178}
1179
1180static void recff_math_degrad(jit_State *J, TRef *res, RecordFFData *rd)
1181{
1182 TRef tr = lj_ir_tonum(J, arg[0]);
1183 res[0] = emitir(IRTN(IR_MUL), tr, lj_ir_knum(J, numV(&rd->fn->c.upvalue[0])));
1184}
1185
1186static void recff_math_pow(jit_State *J, TRef *res, RecordFFData *rd)
1187{
1188 if (!tref_isnumber_str(arg[1]))
1189 lj_trace_err(J, LJ_TRERR_BADTYPE);
1190 res[0] = lj_opt_narrow_pow(J, lj_ir_tonum(J, arg[0]), arg[1], &rd->argv[1]);
1191 UNUSED(rd);
1192}
1193
1194static void recff_math_minmax(jit_State *J, TRef *res, RecordFFData *rd)
1195{
1196 TRef tr = lj_ir_tonum(J, arg[0]);
1197 uint32_t op = rd->data;
1198 BCReg i;
1199 for (i = 1; arg[i]; i++)
1200 tr = emitir(IRTN(op), tr, lj_ir_tonum(J, arg[i]));
1201 res[0] = tr;
1202}
1203
1204/* -- Bit library fast functions ------------------------------------------ */
1205
1206/* Record unary bit.tobit, bit.bnot, bit.bswap. */
1207static void recff_bit_unary(jit_State *J, TRef *res, RecordFFData *rd)
1208{
1209 TRef tr = lj_ir_tobit(J, arg[0]);
1210 res[0] = (rd->data == IR_TOBIT) ? tr : emitir(IRTI(rd->data), tr, 0);
1211}
1212
1213/* Record N-ary bit.band, bit.bor, bit.bxor. */
1214static void recff_bit_nary(jit_State *J, TRef *res, RecordFFData *rd)
1215{
1216 TRef tr = lj_ir_tobit(J, arg[0]);
1217 uint32_t op = rd->data;
1218 BCReg i;
1219 for (i = 1; arg[i]; i++)
1220 tr = emitir(IRTI(op), tr, lj_ir_tobit(J, arg[i]));
1221 res[0] = tr;
1222}
1223
1224/* Record bit shifts. */
1225static void recff_bit_shift(jit_State *J, TRef *res, RecordFFData *rd)
1226{
1227 TRef tr = lj_ir_tobit(J, arg[0]);
1228 TRef tsh = lj_ir_tobit(J, arg[1]);
1229#if !LJ_TARGET_MASKEDSHIFT
1230 if (!tref_isk(tsh))
1231 tsh = emitir(IRTI(IR_BAND), tsh, lj_ir_kint(J, 31));
1232#endif
1233 res[0] = emitir(IRTI(rd->data), tr, tsh);
1234}
1235
1236/* -- String library fast functions --------------------------------------- */
1237
1238static void recff_string_len(jit_State *J, TRef *res, RecordFFData *rd)
1239{
1240 res[0] = emitir(IRTI(IR_FLOAD), lj_ir_tostr(J, arg[0]), IRFL_STR_LEN);
1241 UNUSED(rd);
1242}
1243
1244/* Handle string.byte (rd->data = 0) and string.sub (rd->data = 1). */
1245static void recff_string_range(jit_State *J, TRef *res, RecordFFData *rd)
1246{
1247 TRef trstr = lj_ir_tostr(J, arg[0]);
1248 TRef trlen = emitir(IRTI(IR_FLOAD), trstr, IRFL_STR_LEN);
1249 TRef tr0 = lj_ir_kint(J, 0);
1250 TRef trstart, trend;
1251 GCstr *str = argv2str(J, &rd->argv[0]);
1252 int32_t start, end;
1253 if (rd->data) { /* string.sub(str, start [,end]) */
1254 trstart = lj_ir_toint(J, arg[1]);
1255 trend = tref_isnil(arg[2]) ? lj_ir_kint(J, -1) : lj_ir_toint(J, arg[2]);
1256 start = argv2int(J, &rd->argv[1]);
1257 end = tref_isnil(arg[2]) ? -1 : argv2int(J, &rd->argv[2]);
1258 } else { /* string.byte(str, [,start [,end]]) */
1259 if (arg[1]) {
1260 trstart = lj_ir_toint(J, arg[1]);
1261 trend = tref_isnil(arg[2]) ? trstart : lj_ir_toint(J, arg[2]);
1262 start = argv2int(J, &rd->argv[1]);
1263 end = tref_isnil(arg[2]) ? start : argv2int(J, &rd->argv[2]);
1264 } else {
1265 trend = trstart = lj_ir_kint(J, 1);
1266 end = start = 1;
1267 }
1268 }
1269 if (end < 0) {
1270 emitir(IRTGI(IR_LT), trend, tr0);
1271 trend = emitir(IRTI(IR_ADD), emitir(IRTI(IR_ADD), trlen, trend),
1272 lj_ir_kint(J, 1));
1273 end = end+(int32_t)str->len+1;
1274 } else if ((MSize)end <= str->len) {
1275 emitir(IRTGI(IR_ULE), trend, trlen);
1276 } else {
1277 emitir(IRTGI(IR_GT), trend, trlen);
1278 end = (int32_t)str->len;
1279 trend = trlen;
1280 }
1281 if (start < 0) {
1282 emitir(IRTGI(IR_LT), trstart, tr0);
1283 trstart = emitir(IRTI(IR_ADD), trlen, trstart);
1284 start = start+(int32_t)str->len;
1285 emitir(start < 0 ? IRTGI(IR_LT) : IRTGI(IR_GE), trstart, tr0);
1286 if (start < 0) {
1287 trstart = tr0;
1288 start = 0;
1289 }
1290 } else {
1291 if (start == 0) {
1292 emitir(IRTGI(IR_EQ), trstart, tr0);
1293 trstart = tr0;
1294 } else {
1295 trstart = emitir(IRTI(IR_ADD), trstart, lj_ir_kint(J, -1));
1296 emitir(IRTGI(IR_GE), trstart, tr0);
1297 start--;
1298 }
1299 }
1300 if (rd->data) { /* Return string.sub result. */
1301 if (end - start >= 0) {
1302 /* Also handle empty range here, to avoid extra traces. */
1303 TRef trptr, trslen = emitir(IRTI(IR_SUB), trend, trstart);
1304 emitir(IRTGI(IR_GE), trslen, tr0);
1305 trptr = emitir(IRT(IR_STRREF, IRT_PTR), trstr, trstart);
1306 res[0] = emitir(IRT(IR_SNEW, IRT_STR), trptr, trslen);
1307 } else { /* Range underflow: return empty string. */
1308 emitir(IRTGI(IR_LT), trend, trstart);
1309 res[0] = lj_ir_kstr(J, lj_str_new(J->L, strdata(str), 0));
1310 }
1311 } else { /* Return string.byte result(s). */
1312 int32_t i, len = end - start;
1313 if (len > 0) {
1314 TRef trslen = emitir(IRTI(IR_SUB), trend, trstart);
1315 emitir(IRTGI(IR_EQ), trslen, lj_ir_kint(J, len));
1316 if (res + len > J->slot + LJ_MAX_JSLOTS)
1317 lj_trace_err(J, LJ_TRERR_STACKOV);
1318 rd->nres = len;
1319 for (i = 0; i < len; i++) {
1320 TRef tmp = emitir(IRTI(IR_ADD), trstart, lj_ir_kint(J, i));
1321 tmp = emitir(IRT(IR_STRREF, IRT_PTR), trstr, tmp);
1322 res[i] = emitir(IRT(IR_XLOAD, IRT_U8), tmp, 0);
1323 }
1324 } else { /* Empty range or range underflow: return no results. */
1325 emitir(IRTGI(IR_LE), trend, trstart);
1326 rd->nres = 0;
1327 }
1328 }
1329}
1330
1331/* -- Table library fast functions ---------------------------------------- */
1332
1333static void recff_table_getn(jit_State *J, TRef *res, RecordFFData *rd)
1334{
1335 if (tref_istab(arg[0])) {
1336 res[0] = emitir(IRTI(IR_TLEN), arg[0], 0);
1337 } /* else: Interpreter will throw. */
1338 UNUSED(rd);
1339}
1340
1341/* -- Record calls and returns -------------------------------------------- */
1342
1343#undef arg
1344
1345#include "lj_recdef.h"
1346
1347/* Record return. */
1348static void rec_ret(jit_State *J, BCReg rbase, int gotresults)
1349{
1350 TValue *frame = J->L->base - 1;
1351 TRef *res = J->base + rbase;
1352 J->tailcalled = 0;
1353 while (frame_ispcall(frame)) {
1354 BCReg cbase = (BCReg)frame_delta(frame);
1355 lua_assert(J->baseslot > 1);
1356 J->baseslot -= (BCReg)cbase;
1357 J->base -= cbase;
1358 *--res = TREF_TRUE; /* Prepend true to results. */
1359 gotresults++;
1360 J->framedepth--;
1361 frame = frame_prevd(frame);
1362 }
1363 if (J->framedepth-- <= 0)
1364 lj_trace_err(J, LJ_TRERR_NYIRETL);
1365 lua_assert(J->baseslot > 1);
1366 if (frame_islua(frame)) {
1367 BCIns callins = *(J->pc = frame_pc(frame)-1);
1368 ptrdiff_t nresults = bc_b(callins) ? (int)bc_b(callins)-1 : gotresults;
1369 BCReg cbase = bc_a(callins);
1370 int i;
1371 for (i = 0; i < nresults; i++)
1372 J->base[i-1] = i < gotresults ? res[i] : TREF_NIL;
1373 J->maxslot = cbase+(BCReg)nresults;
1374 J->baseslot -= cbase+1;
1375 J->base -= cbase+1;
1376 } else if (frame_iscont(frame)) {
1377 ASMFunction cont = frame_contf(frame);
1378 BCReg i, cbase = (BCReg)frame_delta(frame);
1379 J->pc = frame_contpc(frame)-1;
1380 J->baseslot -= (BCReg)cbase;
1381 J->base -= cbase;
1382 /* Shrink maxslot as much as possible after return from continuation. */
1383 for (i = cbase-2; i > 0 && J->base[i] == 0; i--) ;
1384 J->maxslot = i;
1385 if (cont == lj_cont_ra) {
1386 /* Copy result to destination slot. */
1387 BCReg dst = bc_a(*J->pc);
1388 J->base[dst] = res[0];
1389 if (dst > J->maxslot) J->maxslot = dst+1;
1390 } else if (cont == lj_cont_nop) {
1391 /* Nothing to do here. */
1392 } else if (cont == lj_cont_cat) {
1393 lua_assert(0);
1394 } else {
1395 /* Result type already specialized. */
1396 lua_assert(cont == lj_cont_condf || cont == lj_cont_condt);
1397 }
1398 } else {
1399 lua_assert(0);
1400 }
1401 lua_assert(J->baseslot >= 1);
1402}
1403
1404/* Check unroll limits for calls. */
1405static void check_call_unroll(jit_State *J, GCfunc *fn)
1406{
1407 TValue *first = J->L->base - J->baseslot;
1408 TValue *frame = J->L->base - 1;
1409 int count = 0;
1410 while (frame > first) {
1411 if (frame_func(frame) == fn)
1412 count++;
1413 if (frame_isvarg(frame))
1414 frame = frame_prevd(frame);
1415 frame = frame_prev(frame);
1416 }
1417 if (frame_func(first) == fn && bc_op(J->cur.startins) == BC_CALL) {
1418 if (count >= J->param[JIT_P_recunroll])
1419 lj_trace_err(J, LJ_TRERR_NYIRECU);
1420 } else {
1421 if (count >= J->param[JIT_P_callunroll])
1422 lj_trace_err(J, LJ_TRERR_CUNROLL);
1423 }
1424}
1425
1426/* Record call. Returns 0 for pending calls and 1 for resolved calls. */
1427static int rec_call(jit_State *J, BCReg func, int cres, int nargs)
1428{
1429 RecordFFData rd;
1430 TRef *res = &J->base[func];
1431 TValue *tv = &J->L->base[func];
1432
1433 if (tref_isfunc(res[0])) { /* Regular function call. */
1434 rd.fn = funcV(tv);
1435 rd.argv = tv+1;
1436 } else { /* Otherwise resolve __call metamethod for called object. */
1437 RecordIndex ix;
1438 int i;
1439 ix.tab = res[0];
1440 copyTV(J->L, &ix.tabv, tv);
1441 if (!rec_mm_lookup(J, &ix, MM_call) || !tref_isfunc(ix.mobj))
1442 lj_trace_err(J, LJ_TRERR_NOMM);
1443 /* Update the recorder state, but not the Lua stack. */
1444 for (i = ++nargs; i > 0; i--)
1445 res[i] = res[i-1];
1446 res[0] = ix.mobj;
1447 rd.fn = funcV(&ix.mobjv);
1448 rd.argv = tv; /* The called object is the 1st arg. */
1449 }
1450
1451 /* Specialize to the runtime value of the called function. */
1452 res[0] = emitir(IRTG(IR_FRAME, IRT_FUNC), res[0], lj_ir_kfunc(J, rd.fn));
1453
1454 if (isluafunc(rd.fn)) { /* Record call to Lua function. */
1455 GCproto *pt = funcproto(rd.fn);
1456 if ((pt->flags & PROTO_NO_JIT))
1457 lj_trace_err(J, LJ_TRERR_CJITOFF);
1458 if ((pt->flags & PROTO_IS_VARARG)) {
1459 if (rd.fn->l.gate != lj_gate_lv)
1460 lj_trace_err(J, LJ_TRERR_NYILNKF);
1461 lj_trace_err(J, LJ_TRERR_NYIVF);
1462 } else {
1463 if (rd.fn->l.gate != lj_gate_lf)
1464 lj_trace_err(J, LJ_TRERR_NYILNKF);
1465 }
1466 check_call_unroll(J, rd.fn);
1467 if (cres == CALLRES_TAILCALL) {
1468 int i;
1469 /* Tailcalls can form a loop, so count towards the loop unroll limit. */
1470 if (++J->tailcalled > J->loopunroll)
1471 lj_trace_err(J, LJ_TRERR_LUNROLL);
1472 for (i = 0; i <= nargs; i++) /* Move func + args down. */
1473 J->base[i-1] = res[i];
1474 /* Note: the new FRAME is now at J->base[-1] (even for slot #0). */
1475 } else { /* Regular call. */
1476 J->base += func+1;
1477 J->baseslot += func+1;
1478 J->framedepth++;
1479 }
1480 if (J->baseslot + pt->framesize >= LJ_MAX_JSLOTS)
1481 lj_trace_err(J, LJ_TRERR_STACKOV);
1482 /* Fill up missing args with nil. */
1483 while (nargs < pt->numparams)
1484 J->base[nargs++] = TREF_NIL;
1485 /* The remaining slots should never be read before they are written. */
1486 J->maxslot = pt->numparams;
1487 return 0; /* No result yet. */
1488 } else { /* Record call to C function or fast function. */
1489 uint32_t m = 0;
1490 res[1+nargs] = 0;
1491 rd.nargs = nargs;
1492 if (rd.fn->c.ffid < sizeof(recff_idmap)/sizeof(recff_idmap[0]))
1493 m = recff_idmap[rd.fn->c.ffid];
1494 rd.data = m & 0xff;
1495 rd.cres = cres;
1496 rd.nres = 1; /* Default is one result. */
1497 (recff_func[m >> 8])(J, res, &rd); /* Call recff_* handler. */
1498 cres = rd.cres;
1499 if (cres >= 0) {
1500 /* Caller takes fixed number of results: local a,b = f() */
1501 J->maxslot = func + (BCReg)cres;
1502 while (rd.nres < cres) /* Fill up missing results with nil. */
1503 res[rd.nres++] = TREF_NIL;
1504 } else if (cres == CALLRES_MULTI) {
1505 /* Caller takes any number of results: return 1,f() */
1506 J->maxslot = func + (BCReg)rd.nres;
1507 } else if (cres == CALLRES_TAILCALL) {
1508 /* Tail call: return f() */
1509 rec_ret(J, func, rd.nres);
1510 } else if (cres == CALLRES_CONT) {
1511 /* Note: immediately resolved continuations must not change J->maxslot. */
1512 res[rd.nres] = TREF_NIL; /* Turn 0 results into nil result. */
1513 } else {
1514 J->framedepth++;
1515 lua_assert(cres == CALLRES_PENDING);
1516 return 0; /* Pending call, no result yet. */
1517 }
1518 return 1; /* Result resolved immediately. */
1519 }
1520}
1521
1522/* -- Record allocations -------------------------------------------------- */
1523
1524static TRef rec_tnew(jit_State *J, uint32_t ah)
1525{
1526 uint32_t asize = ah & 0x7ff;
1527 uint32_t hbits = ah >> 11;
1528 if (asize == 0x7ff) asize = 0x801;
1529 return emitir(IRT(IR_TNEW, IRT_TAB), asize, hbits);
1530}
1531
1532/* -- Record bytecode ops ------------------------------------------------- */
1533
1534/* Optimize state after comparison. */
1535static void optstate_comp(jit_State *J, int cond)
1536{
1537 BCIns jmpins = J->pc[1];
1538 const BCIns *npc = J->pc + 2 + (cond ? bc_j(jmpins) : 0);
1539 SnapShot *snap = &J->cur.snap[J->cur.nsnap-1];
1540 /* Avoid re-recording the comparison in side traces. */
1541 J->cur.snapmap[snap->mapofs + snap->nslots] = u32ptr(npc);
1542 J->needsnap = 1;
1543 /* Shrink last snapshot if possible. */
1544 if (bc_a(jmpins) < J->maxslot) {
1545 J->maxslot = bc_a(jmpins);
1546 lj_snap_shrink(J);
1547 }
1548}
1549
1550/* Record the next bytecode instruction (_before_ it's executed). */
1551void lj_record_ins(jit_State *J)
1552{
1553 cTValue *lbase;
1554 RecordIndex ix;
1555 const BCIns *pc;
1556 BCIns ins;
1557 BCOp op;
1558 TRef ra, rb, rc;
1559
1560 /* Need snapshot before recording next bytecode (e.g. after a store). */
1561 if (J->needsnap) {
1562 J->needsnap = 0;
1563 lj_snap_add(J);
1564 J->mergesnap = 1;
1565 }
1566
1567 /* Record only closed loops for root traces. */
1568 pc = J->pc;
1569 if (J->framedepth == 0 &&
1570 (MSize)((char *)pc - (char *)J->bc_min) >= J->bc_extent)
1571 lj_trace_err(J, LJ_TRERR_LLEAVE);
1572
1573#ifdef LUA_USE_ASSERT
1574 rec_check_slots(J);
1575 rec_check_ir(J);
1576#endif
1577
1578 /* Keep a copy of the runtime values of var/num/str operands. */
1579#define rav (&ix.valv)
1580#define rbv (&ix.tabv)
1581#define rcv (&ix.keyv)
1582
1583 lbase = J->L->base;
1584 ins = *pc;
1585 op = bc_op(ins);
1586 ra = bc_a(ins);
1587 ix.val = 0;
1588 switch (bcmode_a(op)) {
1589 case BCMvar:
1590 copyTV(J->L, rav, &lbase[ra]); ix.val = ra = getslot(J, ra); break;
1591 default: break; /* Handled later. */
1592 }
1593 rb = bc_b(ins);
1594 rc = bc_c(ins);
1595 switch (bcmode_b(op)) {
1596 case BCMnone: rb = 0; rc = bc_d(ins); break; /* Upgrade rc to 'rd'. */
1597 case BCMvar:
1598 copyTV(J->L, rbv, &lbase[rb]); ix.tab = rb = getslot(J, rb); break;
1599 case BCMnum: { lua_Number n = J->pt->k.n[rb];
1600 setnumV(rbv, n); ix.tab = rb = lj_ir_knumint(J, n); } break;
1601 default: break; /* Handled later. */
1602 }
1603 switch (bcmode_c(op)) {
1604 case BCMvar:
1605 copyTV(J->L, rcv, &lbase[rc]); ix.key = rc = getslot(J, rc); break;
1606 case BCMpri: setitype(rcv, (int32_t)~rc); rc = TREF_PRI(IRT_NIL+rc); break;
1607 case BCMnum: { lua_Number n = J->pt->k.n[rc];
1608 setnumV(rcv, n); ix.key = rc = lj_ir_knumint(J, n); } break;
1609 case BCMstr: { GCstr *s = strref(J->pt->k.gc[~rc]);
1610 setstrV(J->L, rcv, s); ix.key = rc = lj_ir_kstr(J, s); } break;
1611 default: break; /* Handled later. */
1612 }
1613
1614 switch (op) {
1615
1616 /* -- Comparison ops ---------------------------------------------------- */
1617
1618 case BC_ISLT: case BC_ISGE: case BC_ISLE: case BC_ISGT:
1619 /* Emit nothing for two numeric or string consts. */
1620 if (!(tref_isk2(ra,rc) && tref_isnumber_str(ra) && tref_isnumber_str(rc))) {
1621 IRType ta = tref_type(ra);
1622 IRType tc = tref_type(rc);
1623 int irop;
1624 if (ta != tc) {
1625 /* Widen mixed number/int comparisons to number/number comparison. */
1626 if (ta == IRT_INT && tc == IRT_NUM) {
1627 ra = emitir(IRTN(IR_TONUM), ra, 0);
1628 ta = IRT_NUM;
1629 } else if (ta == IRT_NUM && tc == IRT_INT) {
1630 rc = emitir(IRTN(IR_TONUM), rc, 0);
1631 } else if (!((ta == IRT_FALSE || ta == IRT_TRUE) &&
1632 (tc == IRT_FALSE || tc == IRT_TRUE))) {
1633 break; /* Interpreter will throw for two different types. */
1634 }
1635 }
1636 lj_snap_add(J);
1637 irop = (int)op - (int)BC_ISLT + (int)IR_LT;
1638 if (ta == IRT_NUM) {
1639 if ((irop & 1)) irop ^= 4; /* ISGE/ISGT are unordered. */
1640 if (!lj_ir_numcmp(numV(rav), numV(rcv), (IROp)irop)) irop ^= 5;
1641 } else if (ta == IRT_INT) {
1642 if (!lj_ir_numcmp(numV(rav), numV(rcv), (IROp)irop)) irop ^= 1;
1643 } else if (ta == IRT_STR) {
1644 if (!lj_ir_strcmp(strV(rav), strV(rcv), (IROp)irop)) irop ^= 1;
1645 } else {
1646 rec_mm_comp(J, &ix, (int)op);
1647 break;
1648 }
1649 emitir(IRTG(irop, ta), ra, rc);
1650 optstate_comp(J, ((int)op ^ irop) & 1);
1651 }
1652 break;
1653
1654 case BC_ISEQV: case BC_ISNEV:
1655 case BC_ISEQS: case BC_ISNES:
1656 case BC_ISEQN: case BC_ISNEN:
1657 case BC_ISEQP: case BC_ISNEP:
1658 /* Emit nothing for two non-table, non-udata consts. */
1659 if (!(tref_isk2(ra, rc) && !(tref_istab(ra) || tref_isudata(ra)))) {
1660 int diff;
1661 lj_snap_add(J);
1662 diff = rec_objcmp(J, ra, rc, rav, rcv);
1663 if (diff == 1 && (tref_istab(ra) || tref_isudata(ra))) {
1664 /* Only check __eq if different, but the same type (table or udata). */
1665 rec_mm_equal(J, &ix, (int)op);
1666 break;
1667 }
1668 optstate_comp(J, ((int)op & 1) == !diff);
1669 }
1670 break;
1671
1672 /* -- Unary test and copy ops ------------------------------------------- */
1673
1674 case BC_ISTC: case BC_ISFC:
1675 if ((op & 1) == tref_istruecond(rc))
1676 rc = 0; /* Don't store if condition is not true. */
1677 /* fallthrough */
1678 case BC_IST: case BC_ISF: /* Type specialization suffices. */
1679 if (bc_a(pc[1]) < J->maxslot)
1680 J->maxslot = bc_a(pc[1]); /* Shrink used slots. */
1681 break;
1682
1683 /* -- Unary ops --------------------------------------------------------- */
1684
1685 case BC_NOT:
1686 /* Type specialization already forces const result. */
1687 rc = tref_istruecond(rc) ? TREF_FALSE : TREF_TRUE;
1688 break;
1689
1690 case BC_LEN:
1691 if (tref_isstr(rc)) {
1692 rc = emitir(IRTI(IR_FLOAD), rc, IRFL_STR_LEN);
1693 } else if (tref_istab(rc)) {
1694 rc = emitir(IRTI(IR_TLEN), rc, 0);
1695 } else {
1696 ix.tab = rc;
1697 copyTV(J->L, &ix.tabv, &ix.keyv);
1698 ix.key = IRT_NIL;
1699 setnilV(&ix.keyv);
1700 rc = rec_mm_arith(J, &ix, MM_len);
1701 }
1702 break;
1703
1704 /* -- Arithmetic ops ---------------------------------------------------- */
1705
1706 case BC_UNM:
1707 if (tref_isnumber_str(rc)) {
1708 rc = lj_ir_tonum(J, rc);
1709 rc = emitir(IRTN(IR_NEG), rc, lj_ir_knum_neg(J));
1710 } else {
1711 ix.tab = rc;
1712 copyTV(J->L, &ix.tabv, &ix.keyv);
1713 rc = rec_mm_arith(J, &ix, MM_unm);
1714 }
1715 break;
1716
1717 case BC_ADDNV: case BC_SUBNV: case BC_MULNV: case BC_DIVNV: case BC_MODNV:
1718 ix.tab = rc; ix.key = rc = rb; rb = ix.tab;
1719 copyTV(J->L, &ix.valv, &ix.tabv);
1720 copyTV(J->L, &ix.tabv, &ix.keyv);
1721 copyTV(J->L, &ix.keyv, &ix.valv);
1722 if (op == BC_MODNV)
1723 goto recmod;
1724 /* fallthrough */
1725 case BC_ADDVN: case BC_SUBVN: case BC_MULVN: case BC_DIVVN:
1726 case BC_ADDVV: case BC_SUBVV: case BC_MULVV: case BC_DIVVV: {
1727 MMS mm = bcmode_mm(op);
1728 if (tref_isnumber_str(rb) && tref_isnumber_str(rc)) {
1729 rb = lj_ir_tonum(J, rb);
1730 rc = lj_ir_tonum(J, rc);
1731 rc = emitir(IRTN((int)mm - (int)MM_add + (int)IR_ADD), rb, rc);
1732 } else {
1733 rc = rec_mm_arith(J, &ix, mm);
1734 }
1735 break;
1736 }
1737
1738 case BC_MODVN: case BC_MODVV:
1739 recmod:
1740 if (tref_isnumber_str(rb) && tref_isnumber_str(rc))
1741 rc = lj_opt_narrow_mod(J, rb, rc);
1742 else
1743 rc = rec_mm_arith(J, &ix, MM_mod);
1744 break;
1745
1746 case BC_POW:
1747 if (tref_isnumber_str(rb) && tref_isnumber_str(rc))
1748 rc = lj_opt_narrow_pow(J, lj_ir_tonum(J, rb), rc, rcv);
1749 else
1750 rc = rec_mm_arith(J, &ix, MM_pow);
1751 break;
1752
1753 /* -- Constant and move ops --------------------------------------------- */
1754
1755 case BC_KSTR: case BC_KNUM: case BC_KPRI: case BC_MOV:
1756 break;
1757 case BC_KSHORT:
1758 rc = lj_ir_kint(J, (int32_t)(int16_t)rc);
1759 break;
1760 case BC_KNIL:
1761 while (ra <= rc)
1762 J->base[ra++] = TREF_NIL;
1763 if (rc >= J->maxslot) J->maxslot = rc+1;
1764 break;
1765
1766 /* -- Upvalue and function ops ------------------------------------------ */
1767
1768 case BC_UGET:
1769 rc = rec_upvalue(J, rc, 0);
1770 break;
1771 case BC_USETV: case BC_USETS: case BC_USETN: case BC_USETP:
1772 rec_upvalue(J, ra, rc);
1773 break;
1774
1775 /* -- Table ops --------------------------------------------------------- */
1776
1777 case BC_GGET: case BC_GSET:
1778 settabV(J->L, &ix.tabv, tabref(J->fn->l.env));
1779 ix.tab = emitir(IRT(IR_FLOAD, IRT_TAB), getcurrf(J), IRFL_FUNC_ENV);
1780 ix.idxchain = LJ_MAX_IDXCHAIN;
1781 rc = rec_idx(J, &ix);
1782 break;
1783
1784 case BC_TGETB: case BC_TSETB:
1785 setintV(&ix.keyv, (int32_t)rc);
1786 ix.key = lj_ir_kint(J, (int32_t)rc);
1787 /* fallthrough */
1788 case BC_TGETV: case BC_TGETS: case BC_TSETV: case BC_TSETS:
1789 ix.idxchain = LJ_MAX_IDXCHAIN;
1790 rc = rec_idx(J, &ix);
1791 break;
1792
1793 case BC_TNEW:
1794 rc = rec_tnew(J, rc);
1795 break;
1796 case BC_TDUP:
1797 rc = emitir(IRT(IR_TDUP, IRT_TAB),
1798 lj_ir_ktab(J, tabref(J->pt->k.gc[~rc])), 0);
1799 break;
1800
1801 /* -- Calls and vararg handling ----------------------------------------- */
1802
1803 case BC_ITERC:
1804 J->base[ra] = getslot(J, ra-3);
1805 J->base[ra+1] = getslot(J, ra-2);
1806 J->base[ra+2] = getslot(J, ra-1);
1807 { /* Have to do the actual copy now because rec_call needs the values. */
1808 TValue *b = &J->L->base[ra];
1809 copyTV(J->L, b, b-3);
1810 copyTV(J->L, b+1, b-2);
1811 copyTV(J->L, b+2, b-1);
1812 }
1813 goto callop;
1814
1815 case BC_CALLMT:
1816 rb = (TRef)(CALLRES_TAILCALL+1);
1817 /* fallthrough */
1818 case BC_CALLM:
1819 /* L->top is set to L->base+ra+rc+NRESULTS-1+1, see lj_dispatch_ins(). */
1820 rc = (BCReg)(J->L->top - J->L->base) - ra;
1821 goto callop;
1822
1823 case BC_CALLT:
1824 rb = (TRef)(CALLRES_TAILCALL+1);
1825 /* fallthrough */
1826 case BC_CALL:
1827 callop:
1828 if (rb == (TRef)(CALLRES_TAILCALL+1)) { /* Tail call. */
1829 }
1830 rec_call(J, ra, (int)(rb-1), (int)(rc-1));
1831 break;
1832
1833 /* -- Returns ----------------------------------------------------------- */
1834
1835 case BC_RETM:
1836 /* L->top is set to L->base+ra+rc+NRESULTS-1, see lj_dispatch_ins(). */
1837 rc = (BCReg)(J->L->top - J->L->base) - ra + 1;
1838 /* fallthrough */
1839 case BC_RET: case BC_RET0: case BC_RET1:
1840 rec_ret(J, ra, (int)(rc-1));
1841 break;
1842
1843 /* -- Loops and branches ------------------------------------------------ */
1844
1845 case BC_FORI:
1846 if (rec_for(J, pc, 0) != LOOPEV_LEAVE)
1847 J->loopref = J->cur.nins;
1848 break;
1849 case BC_JFORI:
1850 lua_assert(bc_op(pc[(ptrdiff_t)rc-BCBIAS_J]) == BC_JFORL);
1851 if (rec_for(J, pc, 0) != LOOPEV_LEAVE) /* Link to existing loop. */
1852 rec_stop(J, bc_d(pc[(ptrdiff_t)rc-BCBIAS_J]));
1853 /* Continue tracing if the loop is not entered. */
1854 break;
1855
1856 case BC_FORL:
1857 rec_loop_interp(J, pc, rec_for(J, pc+((ptrdiff_t)rc-BCBIAS_J), 1));
1858 break;
1859 case BC_ITERL:
1860 rec_loop_interp(J, pc, rec_iterl(J, *pc));
1861 break;
1862 case BC_LOOP:
1863 rec_loop_interp(J, pc, rec_loop(J, ra));
1864 break;
1865
1866 case BC_JFORL:
1867 rec_loop_jit(J, rc, rec_for(J, pc+bc_j(J->trace[rc]->startins), 1));
1868 break;
1869 case BC_JITERL:
1870 rec_loop_jit(J, rc, rec_iterl(J, J->trace[rc]->startins));
1871 break;
1872 case BC_JLOOP:
1873 rec_loop_jit(J, rc, rec_loop(J, ra));
1874 break;
1875
1876 case BC_IFORL:
1877 case BC_IITERL:
1878 case BC_ILOOP:
1879 lj_trace_err_info(J, LJ_TRERR_LBLACKL);
1880 break;
1881
1882 case BC_JMP:
1883 if (ra < J->maxslot)
1884 J->maxslot = ra; /* Shrink used slots. */
1885 break;
1886
1887 case BC_CAT:
1888 case BC_UCLO:
1889 case BC_FNEW:
1890 case BC_TSETM:
1891 case BC_VARG:
1892 default:
1893 setintV(&J->errinfo, (int32_t)op);
1894 lj_trace_err_info(J, LJ_TRERR_NYIBC);
1895 break;
1896 }
1897
1898 /* rc == 0 if we have no result yet, e.g. pending __index metamethod call. */
1899 if (bcmode_a(op) == BCMdst && rc) {
1900 J->base[ra] = rc;
1901 if (ra >= J->maxslot) J->maxslot = ra+1;
1902 }
1903
1904#undef rav
1905#undef rbv
1906#undef rcv
1907
1908 /* Limit the number of recorded IR instructions. */
1909 if (J->cur.nins > REF_FIRST+(IRRef)J->param[JIT_P_maxrecord])
1910 lj_trace_err(J, LJ_TRERR_TRACEOV);
1911}
1912
1913/* -- Recording setup ----------------------------------------------------- */
1914
1915/* Setup recording for a FORL loop. */
1916static void rec_setup_forl(jit_State *J, const BCIns *fori)
1917{
1918 BCReg ra = bc_a(*fori);
1919 cTValue *forbase = &J->L->base[ra];
1920 IRType t = (J->flags & JIT_F_OPT_NARROW) ? lj_opt_narrow_forl(forbase)
1921 : IRT_NUM;
1922 TRef stop = fori_arg(J, fori-2, ra+FORL_STOP, t);
1923 TRef step = fori_arg(J, fori-1, ra+FORL_STEP, t);
1924 int dir = (0 <= numV(&forbase[FORL_STEP]));
1925 lua_assert(bc_op(*fori) == BC_FORI || bc_op(*fori) == BC_JFORI);
1926 if (!tref_isk(step)) {
1927 /* Non-constant step: need a guard for the direction. */
1928 TRef zero = (t == IRT_INT) ? lj_ir_kint(J, 0) : lj_ir_knum_zero(J);
1929 emitir(IRTG(dir ? IR_GE : IR_LT, t), step, zero);
1930 /* Add hoistable overflow checks for a narrowed FORL index. */
1931 if (t == IRT_INT) {
1932 if (tref_isk(stop)) {
1933 /* Constant stop: optimize check away or to a range check for step. */
1934 int32_t k = IR(tref_ref(stop))->i;
1935 if (dir) {
1936 if (k > 0)
1937 emitir(IRTGI(IR_LE), step, lj_ir_kint(J, (int32_t)0x7fffffff-k));
1938 } else {
1939 if (k < 0)
1940 emitir(IRTGI(IR_GE), step, lj_ir_kint(J, (int32_t)0x80000000-k));
1941 }
1942 } else {
1943 /* Stop+step variable: need full overflow check (with dead result). */
1944 emitir(IRTGI(IR_ADDOV), step, stop);
1945 }
1946 }
1947 } else if (t == IRT_INT && !tref_isk(stop)) {
1948 /* Constant step: optimize overflow check to a range check for stop. */
1949 int32_t k = IR(tref_ref(step))->i;
1950 k = (int32_t)(dir ? 0x7fffffff : 0x80000000) - k;
1951 emitir(IRTGI(dir ? IR_LE : IR_GE), stop, lj_ir_kint(J, k));
1952 }
1953 J->base[ra+FORL_EXT] = sloadt(J, (int32_t)(ra+FORL_IDX), t, IRSLOAD_INHERIT);
1954 J->maxslot = ra+FORL_EXT+1;
1955}
1956
1957/* Setup recording for a root trace started by a hot loop. */
1958static const BCIns *rec_setup_root(jit_State *J)
1959{
1960 /* Determine the next PC and the bytecode range for the loop. */
1961 const BCIns *pcj, *pc = J->pc;
1962 BCIns ins = *pc;
1963 BCReg ra = bc_a(ins);
1964 switch (bc_op(ins)) {
1965 case BC_FORL:
1966 J->bc_extent = (MSize)(-bc_j(ins))*sizeof(BCIns);
1967 pc += 1+bc_j(ins);
1968 J->bc_min = pc;
1969 break;
1970 case BC_ITERL:
1971 lua_assert(bc_op(pc[-1]) == BC_ITERC);
1972 J->maxslot = ra + bc_b(pc[-1]) - 1;
1973 J->bc_extent = (MSize)(-bc_j(ins))*sizeof(BCIns);
1974 pc += 1+bc_j(ins);
1975 lua_assert(bc_op(pc[-1]) == BC_JMP);
1976 J->bc_min = pc;
1977 break;
1978 case BC_LOOP:
1979 /* Only check BC range for real loops, but not for "repeat until true". */
1980 pcj = pc + bc_j(ins);
1981 ins = *pcj;
1982 if (bc_op(ins) == BC_JMP && bc_j(ins) < 0) {
1983 J->bc_min = pcj+1 + bc_j(ins);
1984 J->bc_extent = (MSize)(-bc_j(ins))*sizeof(BCIns);
1985 }
1986 J->maxslot = ra;
1987 pc++;
1988 break;
1989 default:
1990 lua_assert(0);
1991 break;
1992 }
1993 return pc;
1994}
1995
1996/* Setup recording for a side trace. */
1997static void rec_setup_side(jit_State *J, Trace *T)
1998{
1999 SnapShot *snap = &T->snap[J->exitno];
2000 IRRef2 *map = &T->snapmap[snap->mapofs];
2001 BCReg s, nslots = snap->nslots;
2002 BloomFilter seen = 0;
2003 for (s = 0; s < nslots; s++) {
2004 IRRef ref = snap_ref(map[s]);
2005 if (ref) {
2006 IRIns *ir = &T->ir[ref];
2007 TRef tr = 0;
2008 /* The bloom filter avoids O(nslots^2) overhead for de-duping slots. */
2009 if (bloomtest(seen, ref)) {
2010 BCReg j;
2011 for (j = 0; j < s; j++)
2012 if (snap_ref(map[j]) == ref) {
2013 if (ir->o == IR_FRAME && irt_isfunc(ir->t))
2014 J->baseslot = s+1;
2015 tr = J->slot[j];
2016 goto dupslot;
2017 }
2018 }
2019 bloomset(seen, ref);
2020 switch ((IROp)ir->o) {
2021 case IR_KPRI: tr = TREF_PRI(irt_type(ir->t)); break;
2022 case IR_KINT: tr = lj_ir_kint(J, ir->i); break;
2023 case IR_KGC: tr = lj_ir_kgc(J, ir_kgc(ir), irt_t(ir->t)); break;
2024 case IR_KNUM: tr = lj_ir_knum_addr(J, ir_knum(ir)); break;
2025 case IR_FRAME: /* Placeholder FRAMEs don't need a guard. */
2026 if (irt_isfunc(ir->t)) {
2027 J->baseslot = s+1;
2028 J->framedepth++;
2029 tr = lj_ir_kfunc(J, ir_kfunc(&T->ir[ir->op2]));
2030 tr = emitir_raw(IRT(IR_FRAME, IRT_FUNC), tr, tr);
2031 } else {
2032 tr = lj_ir_kptr(J, mref(T->ir[ir->op2].ptr, void));
2033 tr = emitir_raw(IRT(IR_FRAME, IRT_PTR), tr, tr);
2034 }
2035 break;
2036 case IR_SLOAD: /* Inherited SLOADs don't need a guard. */
2037 tr = emitir_raw(ir->ot & ~IRT_GUARD, s,
2038 (ir->op2&IRSLOAD_READONLY) | IRSLOAD_INHERIT|IRSLOAD_PARENT);
2039 break;
2040 default: /* Parent refs are already typed and don't need a guard. */
2041 tr = emitir_raw(IRT(IR_SLOAD, irt_type(ir->t)), s,
2042 IRSLOAD_INHERIT|IRSLOAD_PARENT);
2043 break;
2044 }
2045 dupslot:
2046 J->slot[s] = tr;
2047 }
2048 }
2049 J->base = J->slot + J->baseslot;
2050 J->maxslot = nslots - J->baseslot;
2051 lj_snap_add(J);
2052}
2053
2054/* Setup for recording a new trace. */
2055void lj_record_setup(jit_State *J)
2056{
2057 uint32_t i;
2058
2059 /* Initialize state related to current trace. */
2060 memset(J->slot, 0, sizeof(J->slot));
2061 memset(J->chain, 0, sizeof(J->chain));
2062 memset(J->bpropcache, 0, sizeof(J->bpropcache));
2063
2064 J->baseslot = 1; /* Invoking function is at base[-1]. */
2065 J->base = J->slot + J->baseslot;
2066 J->maxslot = 0;
2067 J->framedepth = 0;
2068
2069 J->instunroll = J->param[JIT_P_instunroll];
2070 J->loopunroll = J->param[JIT_P_loopunroll];
2071 J->tailcalled = 0;
2072 J->loopref = 0;
2073
2074 J->bc_min = NULL; /* Means no limit. */
2075 J->bc_extent = ~(MSize)0;
2076
2077 /* Emit instructions for fixed references. Also triggers initial IR alloc. */
2078 emitir_raw(IRT(IR_BASE, IRT_PTR), J->parent, J->exitno);
2079 for (i = 0; i <= 2; i++) {
2080 IRIns *ir = IR(REF_NIL-i);
2081 ir->i = 0;
2082 ir->t.irt = (uint8_t)(IRT_NIL+i);
2083 ir->o = IR_KPRI;
2084 ir->prev = 0;
2085 }
2086 J->cur.nk = REF_TRUE;
2087
2088 setgcref(J->cur.startpt, obj2gco(J->pt));
2089 J->startpc = J->pc;
2090 if (J->parent) { /* Side trace. */
2091 Trace *T = J->trace[J->parent];
2092 TraceNo root = T->root ? T->root : J->parent;
2093 J->cur.root = (uint16_t)root;
2094 J->cur.startins = BCINS_AD(BC_JMP, 0, 0);
2095 /* Check whether we could at least potentially form an extra loop. */
2096 if (J->exitno == 0 && T->snap[0].nslots == 1 && T->snapmap[0] == 0) {
2097 /* We can narrow a FORL for some side traces, too. */
2098 if (J->pc > J->pt->bc && bc_op(J->pc[-1]) == BC_JFORI &&
2099 bc_d(J->pc[bc_j(J->pc[-1])-1]) == root) {
2100 lj_snap_add(J);
2101 rec_setup_forl(J, J->pc-1);
2102 goto sidecheck;
2103 }
2104 } else {
2105 J->startpc = NULL; /* Prevent forming an extra loop. */
2106 }
2107 rec_setup_side(J, T);
2108 sidecheck:
2109 if (J->trace[J->cur.root]->nchild >= J->param[JIT_P_maxside] ||
2110 T->snap[J->exitno].count >= J->param[JIT_P_hotexit] +
2111 J->param[JIT_P_tryside])
2112 rec_stop(J, TRACE_INTERP);
2113 } else { /* Root trace. */
2114 J->cur.root = 0;
2115 if (J->pc >= J->pt->bc) { /* Not a hot CALL? */
2116 J->cur.startins = *J->pc;
2117 J->pc = rec_setup_root(J);
2118 /* Note: the loop instruction itself is recorded at the end and not
2119 ** at the start! So snapshot #0 needs to point to the *next* instruction.
2120 */
2121 } else {
2122 J->cur.startins = BCINS_ABC(BC_CALL, 0, 0, 0);
2123 }
2124 lj_snap_add(J);
2125 if (bc_op(J->cur.startins) == BC_FORL)
2126 rec_setup_forl(J, J->pc-1);
2127 if (1 + J->pt->framesize >= LJ_MAX_JSLOTS)
2128 lj_trace_err(J, LJ_TRERR_STACKOV);
2129 }
2130}
2131
2132#undef IR
2133#undef emitir_raw
2134#undef emitir
2135
2136#endif