diff options
Diffstat (limited to 'src/lj_record.c')
-rw-r--r-- | src/lj_record.c | 2136 |
1 files changed, 2136 insertions, 0 deletions
diff --git a/src/lj_record.c b/src/lj_record.c new file mode 100644 index 00000000..e101ba23 --- /dev/null +++ b/src/lj_record.c | |||
@@ -0,0 +1,2136 @@ | |||
1 | /* | ||
2 | ** Trace recorder (bytecode -> SSA IR). | ||
3 | ** Copyright (C) 2005-2009 Mike Pall. See Copyright Notice in luajit.h | ||
4 | */ | ||
5 | |||
6 | #define lj_record_c | ||
7 | #define LUA_CORE | ||
8 | |||
9 | #include "lj_obj.h" | ||
10 | |||
11 | #if LJ_HASJIT | ||
12 | |||
13 | #include "lj_gc.h" | ||
14 | #include "lj_err.h" | ||
15 | #include "lj_str.h" | ||
16 | #include "lj_tab.h" | ||
17 | #include "lj_state.h" | ||
18 | #include "lj_frame.h" | ||
19 | #include "lj_bc.h" | ||
20 | #include "lj_ff.h" | ||
21 | #include "lj_ir.h" | ||
22 | #include "lj_jit.h" | ||
23 | #include "lj_iropt.h" | ||
24 | #include "lj_trace.h" | ||
25 | #include "lj_record.h" | ||
26 | #include "lj_snap.h" | ||
27 | #include "lj_asm.h" | ||
28 | #include "lj_dispatch.h" | ||
29 | #include "lj_vm.h" | ||
30 | |||
31 | /* Some local macros to save typing. Undef'd at the end. */ | ||
32 | #define IR(ref) (&J->cur.ir[(ref)]) | ||
33 | |||
34 | /* Pass IR on to next optimization in chain (FOLD). */ | ||
35 | #define emitir(ot, a, b) (lj_ir_set(J, (ot), (a), (b)), lj_opt_fold(J)) | ||
36 | |||
37 | /* Emit raw IR without passing through optimizations. */ | ||
38 | #define emitir_raw(ot, a, b) (lj_ir_set(J, (ot), (a), (b)), lj_ir_emit(J)) | ||
39 | |||
40 | /* Context for recording an indexed load/store. */ | ||
41 | typedef struct RecordIndex { | ||
42 | TValue tabv; /* Runtime value of table (or indexed object). */ | ||
43 | TValue keyv; /* Runtime value of key. */ | ||
44 | TValue valv; /* Runtime value of stored value. */ | ||
45 | TValue mobjv; /* Runtime value of metamethod object. */ | ||
46 | GCtab *mtv; /* Runtime value of metatable object. */ | ||
47 | cTValue *oldv; /* Runtime value of previously stored value. */ | ||
48 | TRef tab; /* Table (or indexed object) reference. */ | ||
49 | TRef key; /* Key reference. */ | ||
50 | TRef val; /* Value reference for a store or 0 for a load. */ | ||
51 | TRef mt; /* Metatable reference. */ | ||
52 | TRef mobj; /* Metamethod object reference. */ | ||
53 | int idxchain; /* Index indirections left or 0 for raw lookup. */ | ||
54 | } RecordIndex; | ||
55 | |||
56 | /* Requested results from rec_call(). */ | ||
57 | enum { | ||
58 | /* Non-negative numbers are number of requested results. */ | ||
59 | CALLRES_MULTI = -1, /* Return multiple results. */ | ||
60 | CALLRES_TAILCALL = -2, /* Tail call. */ | ||
61 | CALLRES_PENDING = -3, /* Call is pending, no results yet. */ | ||
62 | CALLRES_CONT = -4 /* Continuation call. */ | ||
63 | }; | ||
64 | |||
65 | /* Forward declarations. */ | ||
66 | static TRef rec_idx(jit_State *J, RecordIndex *ix); | ||
67 | static int rec_call(jit_State *J, BCReg func, int cres, int nargs); | ||
68 | |||
69 | /* -- Sanity checks ------------------------------------------------------- */ | ||
70 | |||
71 | #ifdef LUA_USE_ASSERT | ||
72 | /* Sanity check the whole IR -- sloooow. */ | ||
73 | static void rec_check_ir(jit_State *J) | ||
74 | { | ||
75 | IRRef i, nins = J->cur.nins, nk = J->cur.nk; | ||
76 | lua_assert(nk <= REF_BIAS && nins >= REF_BIAS && nins < 65536); | ||
77 | for (i = nins-1; i >= nk; i--) { | ||
78 | IRIns *ir = IR(i); | ||
79 | uint32_t mode = lj_ir_mode[ir->o]; | ||
80 | IRRef op1 = ir->op1; | ||
81 | IRRef op2 = ir->op2; | ||
82 | switch (irm_op1(mode)) { | ||
83 | case IRMnone: lua_assert(op1 == 0); break; | ||
84 | case IRMref: lua_assert(op1 >= nk); | ||
85 | lua_assert(i >= REF_BIAS ? op1 < i : op1 > i); break; | ||
86 | case IRMlit: break; | ||
87 | case IRMcst: lua_assert(i < REF_BIAS); continue; | ||
88 | } | ||
89 | switch (irm_op2(mode)) { | ||
90 | case IRMnone: lua_assert(op2 == 0); break; | ||
91 | case IRMref: lua_assert(op2 >= nk); | ||
92 | lua_assert(i >= REF_BIAS ? op2 < i : op2 > i); break; | ||
93 | case IRMlit: break; | ||
94 | case IRMcst: lua_assert(0); break; | ||
95 | } | ||
96 | if (ir->prev) { | ||
97 | lua_assert(ir->prev >= nk); | ||
98 | lua_assert(i >= REF_BIAS ? ir->prev < i : ir->prev > i); | ||
99 | lua_assert(IR(ir->prev)->o == ir->o); | ||
100 | } | ||
101 | } | ||
102 | } | ||
103 | |||
104 | /* Sanity check the slots. */ | ||
105 | static void rec_check_slots(jit_State *J) | ||
106 | { | ||
107 | BCReg s, nslots = J->baseslot + J->maxslot; | ||
108 | lua_assert(J->baseslot >= 1 && J->baseslot < LJ_MAX_JSLOTS); | ||
109 | lua_assert(nslots < LJ_MAX_JSLOTS); | ||
110 | for (s = 0; s < nslots; s++) { | ||
111 | TRef tr = J->slot[s]; | ||
112 | if (tr) { | ||
113 | IRRef ref = tref_ref(tr); | ||
114 | lua_assert(ref >= J->cur.nk && ref < J->cur.nins); | ||
115 | lua_assert(irt_t(IR(ref)->t) == tref_t(tr)); | ||
116 | } | ||
117 | } | ||
118 | } | ||
119 | #endif | ||
120 | |||
121 | /* -- Type handling and specialization ------------------------------------ */ | ||
122 | |||
123 | /* Note: these functions return tagged references (TRef). */ | ||
124 | |||
125 | /* Specialize a slot to a specific type. Note: slot can be negative! */ | ||
126 | static TRef sloadt(jit_State *J, int32_t slot, IRType t, int mode) | ||
127 | { | ||
128 | /* No guard, since none of the callers need a type-checking SLOAD. */ | ||
129 | TRef ref = emitir_raw(IRT(IR_SLOAD, t), (int32_t)J->baseslot+slot, mode); | ||
130 | J->base[slot] = ref; | ||
131 | return ref; | ||
132 | } | ||
133 | |||
134 | /* Specialize a slot to the runtime type. Note: slot can be negative! */ | ||
135 | static TRef sload(jit_State *J, int32_t slot) | ||
136 | { | ||
137 | IRType t = itype2irt(&J->L->base[slot]); | ||
138 | TRef ref = emitir_raw(IRTG(IR_SLOAD, t), (int32_t)J->baseslot+slot, 0); | ||
139 | if (irtype_ispri(t)) ref = TREF_PRI(t); /* Canonicalize primitive refs. */ | ||
140 | J->base[slot] = ref; | ||
141 | return ref; | ||
142 | } | ||
143 | |||
144 | /* Get TRef from slot. Load slot and specialize if not done already. */ | ||
145 | #define getslot(J, s) (J->base[(s)] ? J->base[(s)] : sload(J, (int32_t)(s))) | ||
146 | |||
147 | /* Get TRef for current function. */ | ||
148 | static TRef getcurrf(jit_State *J) | ||
149 | { | ||
150 | if (J->base[-1]) { | ||
151 | IRIns *ir = IR(tref_ref(J->base[-1])); | ||
152 | if (ir->o == IR_FRAME) /* Shortcut if already specialized. */ | ||
153 | return TREF(ir->op2, IRT_FUNC); /* Return TRef of KFUNC. */ | ||
154 | return J->base[-1]; | ||
155 | } else { | ||
156 | lua_assert(J->baseslot == 1); | ||
157 | return sloadt(J, -1, IRT_FUNC, IRSLOAD_READONLY); | ||
158 | } | ||
159 | } | ||
160 | |||
161 | /* Compare for raw object equality. | ||
162 | ** Returns 0 if the objects are the same. | ||
163 | ** Returns 1 if they are different, but the same type. | ||
164 | ** Returns 2 for two different types. | ||
165 | ** Comparisons between primitives always return 1 -- no caller cares about it. | ||
166 | */ | ||
167 | static int rec_objcmp(jit_State *J, TRef a, TRef b, cTValue *av, cTValue *bv) | ||
168 | { | ||
169 | int diff = !lj_obj_equal(av, bv); | ||
170 | if (!tref_isk2(a, b)) { /* Shortcut, also handles primitives. */ | ||
171 | IRType ta = tref_type(a); | ||
172 | IRType tb = tref_type(b); | ||
173 | if (ta != tb) { | ||
174 | /* Widen mixed number/int comparisons to number/number comparison. */ | ||
175 | if (ta == IRT_INT && tb == IRT_NUM) { | ||
176 | a = emitir(IRTN(IR_TONUM), a, 0); | ||
177 | ta = IRT_NUM; | ||
178 | } else if (ta == IRT_NUM && tb == IRT_INT) { | ||
179 | b = emitir(IRTN(IR_TONUM), b, 0); | ||
180 | } else { | ||
181 | return 2; /* Two different types are never equal. */ | ||
182 | } | ||
183 | } | ||
184 | emitir(IRTG(diff ? IR_NE : IR_EQ, ta), a, b); | ||
185 | } | ||
186 | return diff; | ||
187 | } | ||
188 | |||
189 | /* -- Record loop ops ----------------------------------------------------- */ | ||
190 | |||
191 | /* Loop event. */ | ||
192 | typedef enum { | ||
193 | LOOPEV_LEAVE, /* Loop is left or not entered. */ | ||
194 | LOOPEV_ENTER /* Loop is entered. */ | ||
195 | } LoopEvent; | ||
196 | |||
197 | /* Canonicalize slots: convert integers to numbers. */ | ||
198 | static void canonicalize_slots(jit_State *J) | ||
199 | { | ||
200 | BCReg s; | ||
201 | for (s = J->baseslot+J->maxslot-1; s >= 1; s--) { | ||
202 | TRef tr = J->slot[s]; | ||
203 | if (tref_isinteger(tr)) { | ||
204 | IRIns *ir = IR(tref_ref(tr)); | ||
205 | if (!(ir->o == IR_SLOAD && (ir->op2 & IRSLOAD_READONLY))) | ||
206 | J->slot[s] = emitir(IRTN(IR_TONUM), tr, 0); | ||
207 | } | ||
208 | } | ||
209 | } | ||
210 | |||
211 | /* Stop recording. */ | ||
212 | static void rec_stop(jit_State *J, TraceNo lnk) | ||
213 | { | ||
214 | lj_trace_end(J); | ||
215 | J->cur.link = (uint16_t)lnk; | ||
216 | if (lnk == J->curtrace) { /* Looping back? */ | ||
217 | if ((J->flags & JIT_F_OPT_LOOP)) /* Shall we try to create a loop? */ | ||
218 | goto nocanon; /* Do not canonicalize or we lose the narrowing. */ | ||
219 | if (J->cur.root) /* Otherwise ensure we always link to the root trace. */ | ||
220 | J->cur.link = J->cur.root; | ||
221 | } | ||
222 | canonicalize_slots(J); | ||
223 | nocanon: | ||
224 | /* Note: all loop ops must set J->pc to the following instruction! */ | ||
225 | lj_snap_add(J); /* Add loop snapshot. */ | ||
226 | J->needsnap = 0; | ||
227 | J->mergesnap = 1; /* In case recording continues. */ | ||
228 | } | ||
229 | |||
230 | /* Peek before FORI to find a const initializer, otherwise load from slot. */ | ||
231 | static TRef fori_arg(jit_State *J, const BCIns *pc, BCReg slot, IRType t) | ||
232 | { | ||
233 | /* A store to slot-1 means there's no conditional assignment for slot. */ | ||
234 | if (bc_a(pc[-1]) == slot-1 && bcmode_a(bc_op(pc[-1])) == BCMdst) { | ||
235 | BCIns ins = pc[0]; | ||
236 | if (bc_a(ins) == slot) { | ||
237 | if (bc_op(ins) == BC_KSHORT) { | ||
238 | int32_t k = (int32_t)(int16_t)bc_d(ins); | ||
239 | if (t == IRT_INT) | ||
240 | return lj_ir_kint(J, k); | ||
241 | else | ||
242 | return lj_ir_knum(J, cast_num(k)); | ||
243 | } else if (bc_op(ins) == BC_KNUM) { | ||
244 | lua_Number n = J->pt->k.n[bc_d(ins)]; | ||
245 | if (t == IRT_INT) | ||
246 | return lj_ir_kint(J, lj_num2int(n)); | ||
247 | else | ||
248 | return lj_ir_knum(J, n); | ||
249 | } | ||
250 | } | ||
251 | } | ||
252 | if (J->base[slot]) | ||
253 | return J->base[slot]; | ||
254 | else | ||
255 | return sloadt(J, (int32_t)slot, t, IRSLOAD_READONLY|IRSLOAD_INHERIT); | ||
256 | } | ||
257 | |||
258 | /* Simulate the runtime behavior of the FOR loop iterator. | ||
259 | ** It's important to exactly reproduce the semantics of the interpreter. | ||
260 | */ | ||
261 | static LoopEvent for_iter(jit_State *J, IROp *op, BCReg ra, int isforl) | ||
262 | { | ||
263 | cTValue *forbase = &J->L->base[ra]; | ||
264 | lua_Number stopv = numV(&forbase[FORL_STOP]); | ||
265 | lua_Number idxv = numV(&forbase[FORL_IDX]); | ||
266 | if (isforl) | ||
267 | idxv += numV(&forbase[FORL_STEP]); | ||
268 | if ((int32_t)forbase[FORL_STEP].u32.hi >= 0) { | ||
269 | if (idxv <= stopv) { *op = IR_LE; return LOOPEV_ENTER; } | ||
270 | *op = IR_GT; return LOOPEV_LEAVE; | ||
271 | } else { | ||
272 | if (stopv <= idxv) { *op = IR_GE; return LOOPEV_ENTER; } | ||
273 | *op = IR_LT; return LOOPEV_LEAVE; | ||
274 | } | ||
275 | } | ||
276 | |||
277 | /* Record FORL/JFORL or FORI/JFORI. */ | ||
278 | static LoopEvent rec_for(jit_State *J, const BCIns *fori, int isforl) | ||
279 | { | ||
280 | BCReg ra = bc_a(*fori); | ||
281 | IROp op; | ||
282 | LoopEvent ev = for_iter(J, &op, ra, isforl); | ||
283 | TRef *tr = &J->base[ra]; | ||
284 | TRef idx, stop; | ||
285 | IRType t; | ||
286 | if (isforl) { /* Handle FORL/JFORL opcodes. */ | ||
287 | TRef step; | ||
288 | idx = tr[FORL_IDX]; | ||
289 | if (!idx) idx = sloadt(J, (int32_t)(ra+FORL_IDX), IRT_NUM, 0); | ||
290 | t = tref_type(idx); | ||
291 | stop = fori_arg(J, fori-2, ra+FORL_STOP, t); | ||
292 | step = fori_arg(J, fori-1, ra+FORL_STEP, t); | ||
293 | tr[FORL_IDX] = idx = emitir(IRT(IR_ADD, t), idx, step); | ||
294 | } else { /* Handle FORI/JFORI opcodes. */ | ||
295 | BCReg i; | ||
296 | t = IRT_NUM; | ||
297 | for (i = FORL_IDX; i <= FORL_STEP; i++) { | ||
298 | lua_assert(J->base[ra+i] != 0); /* Assumes the slots are already set. */ | ||
299 | tr[i] = lj_ir_tonum(J, J->base[ra+i]); | ||
300 | } | ||
301 | idx = tr[FORL_IDX]; | ||
302 | stop = tr[FORL_STOP]; | ||
303 | if (!tref_isk(tr[FORL_STEP])) /* Non-const step: need direction guard. */ | ||
304 | emitir(IRTG(((op-IR_LT)>>1)+IR_LT, IRT_NUM), | ||
305 | tr[FORL_STEP], lj_ir_knum_zero(J)); | ||
306 | } | ||
307 | |||
308 | tr[FORL_EXT] = idx; | ||
309 | if (ev == LOOPEV_LEAVE) { | ||
310 | J->maxslot = ra+FORL_EXT+1; | ||
311 | J->pc = fori+1; | ||
312 | } else { | ||
313 | J->maxslot = ra; | ||
314 | J->pc = fori+bc_j(*fori)+1; | ||
315 | } | ||
316 | lj_snap_add(J); | ||
317 | |||
318 | emitir(IRTG(op, t), idx, stop); | ||
319 | |||
320 | if (ev == LOOPEV_LEAVE) { | ||
321 | J->maxslot = ra; | ||
322 | J->pc = fori+bc_j(*fori)+1; | ||
323 | } else { | ||
324 | J->maxslot = ra+FORL_EXT+1; | ||
325 | J->pc = fori+1; | ||
326 | } | ||
327 | J->needsnap = 1; | ||
328 | return ev; | ||
329 | } | ||
330 | |||
331 | /* Record ITERL/JITERL. */ | ||
332 | static LoopEvent rec_iterl(jit_State *J, const BCIns iterins) | ||
333 | { | ||
334 | BCReg ra = bc_a(iterins); | ||
335 | lua_assert(J->base[ra] != 0); | ||
336 | if (!tref_isnil(J->base[ra])) { /* Looping back? */ | ||
337 | J->base[ra-1] = J->base[ra]; /* Copy result of ITERC to control var. */ | ||
338 | J->maxslot = ra-1+bc_b(J->pc[-1]); | ||
339 | J->pc += bc_j(iterins)+1; | ||
340 | return LOOPEV_ENTER; | ||
341 | } else { | ||
342 | J->maxslot = ra-3; | ||
343 | J->pc++; | ||
344 | return LOOPEV_LEAVE; | ||
345 | } | ||
346 | } | ||
347 | |||
348 | /* Record LOOP/JLOOP. Now, that was easy. */ | ||
349 | static LoopEvent rec_loop(jit_State *J, BCReg ra) | ||
350 | { | ||
351 | J->maxslot = ra; | ||
352 | J->pc++; | ||
353 | return LOOPEV_ENTER; | ||
354 | } | ||
355 | |||
356 | /* Check if a loop repeatedly failed to trace because it didn't loop back. */ | ||
357 | static int innerloopleft(jit_State *J, const BCIns *pc) | ||
358 | { | ||
359 | ptrdiff_t i; | ||
360 | for (i = 0; i < PENALTY_SLOTS; i++) | ||
361 | if (J->penalty[i].pc == pc) { | ||
362 | if (J->penalty[i].reason == LJ_TRERR_LLEAVE && | ||
363 | J->penalty[i].val >= 2*HOTCOUNT_MIN_PENALTY) | ||
364 | return 1; | ||
365 | break; | ||
366 | } | ||
367 | return 0; | ||
368 | } | ||
369 | |||
370 | /* Handle the case when an interpreted loop op is hit. */ | ||
371 | static void rec_loop_interp(jit_State *J, const BCIns *pc, LoopEvent ev) | ||
372 | { | ||
373 | if (J->parent == 0) { | ||
374 | if (pc == J->startpc && J->framedepth == 0) { /* Same loop? */ | ||
375 | if (ev == LOOPEV_LEAVE) /* Must loop back to form a root trace. */ | ||
376 | lj_trace_err(J, LJ_TRERR_LLEAVE); | ||
377 | rec_stop(J, J->curtrace); /* Root trace forms a loop. */ | ||
378 | } else if (ev != LOOPEV_LEAVE) { /* Entering inner loop? */ | ||
379 | /* It's usually better to abort here and wait until the inner loop | ||
380 | ** is traced. But if the inner loop repeatedly didn't loop back, | ||
381 | ** this indicates a low trip count. In this case try unrolling | ||
382 | ** an inner loop even in a root trace. But it's better to be a bit | ||
383 | ** more conservative here and only do it for very short loops. | ||
384 | */ | ||
385 | if (!innerloopleft(J, pc)) | ||
386 | lj_trace_err(J, LJ_TRERR_LINNER); /* Root trace hit an inner loop. */ | ||
387 | if ((J->loopref && J->cur.nins - J->loopref > 8) || --J->loopunroll < 0) | ||
388 | lj_trace_err(J, LJ_TRERR_LUNROLL); /* Limit loop unrolling. */ | ||
389 | J->loopref = J->cur.nins; | ||
390 | } | ||
391 | } else if (ev != LOOPEV_LEAVE) { /* Side trace enters an inner loop. */ | ||
392 | J->loopref = J->cur.nins; | ||
393 | if (--J->loopunroll < 0) | ||
394 | lj_trace_err(J, LJ_TRERR_LUNROLL); /* Limit loop unrolling. */ | ||
395 | } /* Side trace continues across a loop that's left or not entered. */ | ||
396 | } | ||
397 | |||
398 | /* Handle the case when an already compiled loop op is hit. */ | ||
399 | static void rec_loop_jit(jit_State *J, TraceNo lnk, LoopEvent ev) | ||
400 | { | ||
401 | if (J->parent == 0) { /* Root trace hit an inner loop. */ | ||
402 | /* Better let the inner loop spawn a side trace back here. */ | ||
403 | lj_trace_err(J, LJ_TRERR_LINNER); | ||
404 | } else if (ev != LOOPEV_LEAVE) { /* Side trace enters a compiled loop. */ | ||
405 | J->instunroll = 0; /* Cannot continue across a compiled loop op. */ | ||
406 | if (J->pc == J->startpc && J->framedepth == 0) | ||
407 | lnk = J->curtrace; /* Can form an extra loop. */ | ||
408 | rec_stop(J, lnk); /* Link to the loop. */ | ||
409 | } /* Side trace continues across a loop that's left or not entered. */ | ||
410 | } | ||
411 | |||
412 | /* -- Metamethod handling ------------------------------------------------- */ | ||
413 | |||
414 | /* Prepare to record call to metamethod. */ | ||
415 | static BCReg rec_mm_prep(jit_State *J, ASMFunction cont) | ||
416 | { | ||
417 | BCReg s, top = curr_proto(J->L)->framesize; | ||
418 | TRef trcont; | ||
419 | setcont(&J->L->base[top], cont); | ||
420 | #if LJ_64 | ||
421 | trcont = lj_ir_kptr(J, (void *)((int64_t)cont - (int64_t)lj_vm_asm_begin)); | ||
422 | #else | ||
423 | trcont = lj_ir_kptr(J, (void *)cont); | ||
424 | #endif | ||
425 | J->base[top] = emitir(IRTG(IR_FRAME, IRT_PTR), trcont, trcont); | ||
426 | for (s = J->maxslot; s < top; s++) | ||
427 | J->base[s] = 0; | ||
428 | return top+1; | ||
429 | } | ||
430 | |||
431 | /* Record metamethod lookup. */ | ||
432 | static int rec_mm_lookup(jit_State *J, RecordIndex *ix, MMS mm) | ||
433 | { | ||
434 | RecordIndex mix; | ||
435 | GCtab *mt; | ||
436 | if (tref_istab(ix->tab)) { | ||
437 | mt = tabref(tabV(&ix->tabv)->metatable); | ||
438 | mix.tab = emitir(IRT(IR_FLOAD, IRT_TAB), ix->tab, IRFL_TAB_META); | ||
439 | } else if (tref_isudata(ix->tab)) { | ||
440 | mt = tabref(udataV(&ix->tabv)->metatable); | ||
441 | mix.tab = emitir(IRT(IR_FLOAD, IRT_TAB), ix->tab, IRFL_UDATA_META); | ||
442 | } else { | ||
443 | /* Specialize to base metatable. Must flush mcode in lua_setmetatable(). */ | ||
444 | mt = tabref(J2G(J)->basemt[itypemap(&ix->tabv)]); | ||
445 | if (mt == NULL) | ||
446 | return 0; /* No metamethod. */ | ||
447 | mix.tab = lj_ir_ktab(J, mt); | ||
448 | goto nocheck; | ||
449 | } | ||
450 | ix->mt = mix.tab; | ||
451 | emitir(IRTG(mt ? IR_NE : IR_EQ, IRT_TAB), mix.tab, lj_ir_knull(J, IRT_TAB)); | ||
452 | nocheck: | ||
453 | if (mt) { | ||
454 | GCstr *mmstr = strref(J2G(J)->mmname[mm]); | ||
455 | cTValue *mo = lj_tab_getstr(mt, mmstr); | ||
456 | if (mo && !tvisnil(mo)) | ||
457 | copyTV(J->L, &ix->mobjv, mo); | ||
458 | ix->mtv = mt; | ||
459 | settabV(J->L, &mix.tabv, mt); | ||
460 | setstrV(J->L, &mix.keyv, mmstr); | ||
461 | mix.key = lj_ir_kstr(J, mmstr); | ||
462 | mix.val = 0; | ||
463 | mix.idxchain = 0; | ||
464 | ix->mobj = rec_idx(J, &mix); | ||
465 | return !tref_isnil(ix->mobj); /* 1 if metamethod found, 0 if not. */ | ||
466 | } | ||
467 | return 0; /* No metamethod. */ | ||
468 | } | ||
469 | |||
470 | /* Record call to arithmetic metamethod (and MM_len). */ | ||
471 | static TRef rec_mm_arith(jit_State *J, RecordIndex *ix, MMS mm) | ||
472 | { | ||
473 | /* Set up metamethod call first to save ix->tab and ix->tabv. */ | ||
474 | BCReg func = rec_mm_prep(J, lj_cont_ra); | ||
475 | TRef *base = J->base + func; | ||
476 | TValue *basev = J->L->base + func; | ||
477 | base[1] = ix->tab; base[2] = ix->key; | ||
478 | copyTV(J->L, basev+1, &ix->tabv); | ||
479 | copyTV(J->L, basev+2, &ix->keyv); | ||
480 | if (!rec_mm_lookup(J, ix, mm)) { /* Lookup metamethod on 1st operand. */ | ||
481 | if (mm != MM_len) { | ||
482 | ix->tab = ix->key; | ||
483 | copyTV(J->L, &ix->tabv, &ix->keyv); | ||
484 | if (rec_mm_lookup(J, ix, mm)) /* Lookup metamethod on 2nd operand. */ | ||
485 | goto ok; | ||
486 | } | ||
487 | lj_trace_err(J, LJ_TRERR_NOMM); | ||
488 | } | ||
489 | ok: | ||
490 | base[0] = ix->mobj; | ||
491 | copyTV(J->L, basev+0, &ix->mobjv); | ||
492 | return rec_call(J, func, CALLRES_CONT, 2) ? J->base[func] : 0; | ||
493 | } | ||
494 | |||
495 | /* Call a comparison metamethod. */ | ||
496 | static void rec_mm_callcomp(jit_State *J, RecordIndex *ix, int op) | ||
497 | { | ||
498 | BCReg func = rec_mm_prep(J, (op&1) ? lj_cont_condf : lj_cont_condt); | ||
499 | TRef *base = J->base + func; | ||
500 | TValue *tv = J->L->base + func; | ||
501 | base[0] = ix->mobj; base[1] = ix->val; base[2] = ix->key; | ||
502 | copyTV(J->L, tv+0, &ix->mobjv); | ||
503 | copyTV(J->L, tv+1, &ix->valv); | ||
504 | copyTV(J->L, tv+2, &ix->keyv); | ||
505 | rec_call(J, func, CALLRES_CONT, 2); | ||
506 | /* It doesn't matter whether this is immediately resolved or not. | ||
507 | ** Type specialization of the return type suffices to specialize | ||
508 | ** the control flow. | ||
509 | */ | ||
510 | } | ||
511 | |||
512 | /* Record call to equality comparison metamethod (for tab and udata only). */ | ||
513 | static void rec_mm_equal(jit_State *J, RecordIndex *ix, int op) | ||
514 | { | ||
515 | ix->tab = ix->val; | ||
516 | copyTV(J->L, &ix->tabv, &ix->valv); | ||
517 | if (rec_mm_lookup(J, ix, MM_eq)) { /* Lookup metamethod on 1st operand. */ | ||
518 | cTValue *bv; | ||
519 | TRef mo1 = ix->mobj; | ||
520 | TValue mo1v; | ||
521 | copyTV(J->L, &mo1v, &ix->mobjv); | ||
522 | /* Avoid the 2nd lookup and the objcmp if the metatables are equal. */ | ||
523 | bv = &ix->keyv; | ||
524 | if (tvistab(bv) && tabref(tabV(bv)->metatable) == ix->mtv) { | ||
525 | TRef mt2 = emitir(IRT(IR_FLOAD, IRT_TAB), ix->key, IRFL_TAB_META); | ||
526 | emitir(IRTG(IR_EQ, IRT_TAB), mt2, ix->mt); | ||
527 | } else if (tvisudata(bv) && tabref(udataV(bv)->metatable) == ix->mtv) { | ||
528 | TRef mt2 = emitir(IRT(IR_FLOAD, IRT_TAB), ix->key, IRFL_UDATA_META); | ||
529 | emitir(IRTG(IR_EQ, IRT_TAB), mt2, ix->mt); | ||
530 | } else { /* Lookup metamethod on 2nd operand and compare both. */ | ||
531 | ix->tab = ix->key; | ||
532 | copyTV(J->L, &ix->tabv, bv); | ||
533 | if (!rec_mm_lookup(J, ix, MM_eq) || | ||
534 | rec_objcmp(J, mo1, ix->mobj, &mo1v, &ix->mobjv)) | ||
535 | return; | ||
536 | } | ||
537 | rec_mm_callcomp(J, ix, op); | ||
538 | } | ||
539 | } | ||
540 | |||
541 | /* Record call to ordered comparison metamethods (for arbitrary objects). */ | ||
542 | static void rec_mm_comp(jit_State *J, RecordIndex *ix, int op) | ||
543 | { | ||
544 | ix->tab = ix->val; | ||
545 | copyTV(J->L, &ix->tabv, &ix->valv); | ||
546 | while (1) { | ||
547 | MMS mm = (op & 2) ? MM_le : MM_lt; /* Try __le + __lt or only __lt. */ | ||
548 | if (rec_mm_lookup(J, ix, mm)) { /* Lookup metamethod on 1st operand. */ | ||
549 | cTValue *bv; | ||
550 | TRef mo1 = ix->mobj; | ||
551 | TValue mo1v; | ||
552 | copyTV(J->L, &mo1v, &ix->mobjv); | ||
553 | /* Avoid the 2nd lookup and the objcmp if the metatables are equal. */ | ||
554 | bv = &ix->keyv; | ||
555 | if (tvistab(bv) && tabref(tabV(bv)->metatable) == ix->mtv) { | ||
556 | TRef mt2 = emitir(IRT(IR_FLOAD, IRT_TAB), ix->key, IRFL_TAB_META); | ||
557 | emitir(IRTG(IR_EQ, IRT_TAB), mt2, ix->mt); | ||
558 | } else if (tvisudata(bv) && tabref(udataV(bv)->metatable) == ix->mtv) { | ||
559 | TRef mt2 = emitir(IRT(IR_FLOAD, IRT_TAB), ix->key, IRFL_UDATA_META); | ||
560 | emitir(IRTG(IR_EQ, IRT_TAB), mt2, ix->mt); | ||
561 | } else { /* Lookup metamethod on 2nd operand and compare both. */ | ||
562 | ix->tab = ix->key; | ||
563 | copyTV(J->L, &ix->tabv, bv); | ||
564 | if (!rec_mm_lookup(J, ix, mm) || | ||
565 | rec_objcmp(J, mo1, ix->mobj, &mo1v, &ix->mobjv)) | ||
566 | goto nomatch; | ||
567 | } | ||
568 | rec_mm_callcomp(J, ix, op); | ||
569 | return; | ||
570 | } | ||
571 | nomatch: | ||
572 | /* First lookup failed. Retry with __lt and swapped operands. */ | ||
573 | if (!(op & 2)) break; /* Already at __lt. Interpreter will throw. */ | ||
574 | ix->tab = ix->key; ix->key = ix->val; ix->val = ix->tab; | ||
575 | copyTV(J->L, &ix->tabv, &ix->keyv); | ||
576 | copyTV(J->L, &ix->keyv, &ix->valv); | ||
577 | copyTV(J->L, &ix->valv, &ix->tabv); | ||
578 | op ^= 3; | ||
579 | } | ||
580 | } | ||
581 | |||
582 | /* -- Indexed access ------------------------------------------------------ */ | ||
583 | |||
584 | /* Record indexed key lookup. */ | ||
585 | static TRef rec_idx_key(jit_State *J, RecordIndex *ix) | ||
586 | { | ||
587 | TRef key; | ||
588 | GCtab *t = tabV(&ix->tabv); | ||
589 | ix->oldv = lj_tab_get(J->L, t, &ix->keyv); /* Lookup previous value. */ | ||
590 | |||
591 | /* Integer keys are looked up in the array part first. */ | ||
592 | key = ix->key; | ||
593 | if (tref_isnumber(key)) { | ||
594 | lua_Number n = numV(&ix->keyv); | ||
595 | int32_t k = lj_num2int(n); | ||
596 | lua_assert(tvisnum(&ix->keyv)); | ||
597 | /* Potential array key? */ | ||
598 | if ((MSize)k < LJ_MAX_ASIZE && n == cast_num(k)) { | ||
599 | TRef asizeref, ikey = key; | ||
600 | if (!tref_isinteger(ikey)) | ||
601 | ikey = emitir(IRTGI(IR_TOINT), ikey, IRTOINT_INDEX); | ||
602 | asizeref = emitir(IRTI(IR_FLOAD), ix->tab, IRFL_TAB_ASIZE); | ||
603 | if ((MSize)k < t->asize) { /* Currently an array key? */ | ||
604 | TRef arrayref; | ||
605 | emitir(IRTGI(IR_ABC), asizeref, ikey); /* Bounds check. */ | ||
606 | arrayref = emitir(IRT(IR_FLOAD, IRT_PTR), ix->tab, IRFL_TAB_ARRAY); | ||
607 | return emitir(IRT(IR_AREF, IRT_PTR), arrayref, ikey); | ||
608 | } else { /* Currently not in array (may be an array extension)? */ | ||
609 | emitir(IRTGI(IR_ULE), asizeref, ikey); /* Inv. bounds check. */ | ||
610 | if (k == 0 && tref_isk(key)) | ||
611 | key = lj_ir_knum_zero(J); /* Canonicalize 0 or +-0.0 to +0.0. */ | ||
612 | /* And continue with the hash lookup. */ | ||
613 | } | ||
614 | } else if (!tref_isk(key)) { | ||
615 | /* We can rule out const numbers which failed the integerness test | ||
616 | ** above. But all other numbers are potential array keys. | ||
617 | */ | ||
618 | if (t->asize == 0) { /* True sparse tables have an empty array part. */ | ||
619 | /* Guard that the array part stays empty. */ | ||
620 | TRef tmp = emitir(IRTI(IR_FLOAD), ix->tab, IRFL_TAB_ASIZE); | ||
621 | emitir(IRTGI(IR_EQ), tmp, lj_ir_kint(J, 0)); | ||
622 | } else { | ||
623 | lj_trace_err(J, LJ_TRERR_NYITMIX); | ||
624 | } | ||
625 | } | ||
626 | } | ||
627 | |||
628 | /* Otherwise the key is located in the hash part. */ | ||
629 | if (tref_isinteger(key)) /* Hash keys are based on numbers, not ints. */ | ||
630 | ix->key = key = emitir(IRTN(IR_TONUM), key, 0); | ||
631 | if (tref_isk(key)) { | ||
632 | /* Optimize lookup of constant hash keys. */ | ||
633 | MSize hslot = (MSize)((char *)ix->oldv - (char *)&noderef(t->node)[0].val); | ||
634 | if (t->hmask > 0 && hslot <= t->hmask*(MSize)sizeof(Node) && | ||
635 | hslot <= 65535*(MSize)sizeof(Node)) { | ||
636 | TRef node, kslot; | ||
637 | TRef hm = emitir(IRTI(IR_FLOAD), ix->tab, IRFL_TAB_HMASK); | ||
638 | emitir(IRTGI(IR_EQ), hm, lj_ir_kint(J, (int32_t)t->hmask)); | ||
639 | node = emitir(IRT(IR_FLOAD, IRT_PTR), ix->tab, IRFL_TAB_NODE); | ||
640 | kslot = lj_ir_kslot(J, key, hslot / sizeof(Node)); | ||
641 | return emitir(IRTG(IR_HREFK, IRT_PTR), node, kslot); | ||
642 | } | ||
643 | } | ||
644 | /* Fall back to a regular hash lookup. */ | ||
645 | return emitir(IRT(IR_HREF, IRT_PTR), ix->tab, key); | ||
646 | } | ||
647 | |||
648 | /* Determine whether a key is NOT one of the fast metamethod names. */ | ||
649 | static int nommstr(jit_State *J, TRef key) | ||
650 | { | ||
651 | if (tref_isstr(key)) { | ||
652 | if (tref_isk(key)) { | ||
653 | GCstr *str = ir_kstr(IR(tref_ref(key))); | ||
654 | uint32_t i; | ||
655 | for (i = 0; i <= MM_FAST; i++) | ||
656 | if (strref(J2G(J)->mmname[i]) == str) | ||
657 | return 0; /* MUST be one the fast metamethod names. */ | ||
658 | } else { | ||
659 | return 0; /* Variable string key MAY be a metamethod name. */ | ||
660 | } | ||
661 | } | ||
662 | return 1; /* CANNOT be a metamethod name. */ | ||
663 | } | ||
664 | |||
665 | /* Record indexed load/store. */ | ||
666 | static TRef rec_idx(jit_State *J, RecordIndex *ix) | ||
667 | { | ||
668 | TRef xref; | ||
669 | IROp xrefop, loadop; | ||
670 | cTValue *oldv; | ||
671 | |||
672 | while (!tref_istab(ix->tab)) { /* Handle non-table lookup. */ | ||
673 | lua_assert(ix->idxchain != 0); /* Never call raw rec_idx() on non-table. */ | ||
674 | if (!rec_mm_lookup(J, ix, ix->val ? MM_newindex : MM_index)) | ||
675 | lj_trace_err(J, LJ_TRERR_NOMM); | ||
676 | handlemm: | ||
677 | if (tref_isfunc(ix->mobj)) { /* Handle metamethod call. */ | ||
678 | BCReg func = rec_mm_prep(J, ix->val ? lj_cont_nop : lj_cont_ra); | ||
679 | TRef *base = J->base + func; | ||
680 | TValue *tv = J->L->base + func; | ||
681 | base[0] = ix->mobj; base[1] = ix->tab; base[2] = ix->key; | ||
682 | setfuncV(J->L, tv+0, funcV(&ix->mobjv)); | ||
683 | copyTV(J->L, tv+1, &ix->tabv); | ||
684 | copyTV(J->L, tv+2, &ix->keyv); | ||
685 | if (ix->val) { | ||
686 | base[3] = ix->val; | ||
687 | copyTV(J->L, tv+3, &ix->valv); | ||
688 | rec_call(J, func, CALLRES_CONT, 3); /* mobj(tab, key, val) */ | ||
689 | return 0; | ||
690 | } else { | ||
691 | /* res = mobj(tab, key) */ | ||
692 | return rec_call(J, func, CALLRES_CONT, 2) ? J->base[func] : 0; | ||
693 | } | ||
694 | } | ||
695 | /* Otherwise retry lookup with metaobject. */ | ||
696 | ix->tab = ix->mobj; | ||
697 | copyTV(J->L, &ix->tabv, &ix->mobjv); | ||
698 | if (--ix->idxchain == 0) | ||
699 | lj_trace_err(J, LJ_TRERR_IDXLOOP); | ||
700 | } | ||
701 | |||
702 | /* First catch nil and NaN keys for tables. */ | ||
703 | if (tvisnil(&ix->keyv) || (tvisnum(&ix->keyv) && tvisnan(&ix->keyv))) { | ||
704 | if (ix->val) /* Better fail early. */ | ||
705 | lj_trace_err(J, LJ_TRERR_STORENN); | ||
706 | if (tref_isk(ix->key)) { | ||
707 | if (ix->idxchain && rec_mm_lookup(J, ix, MM_index)) | ||
708 | goto handlemm; | ||
709 | return TREF_NIL; | ||
710 | } | ||
711 | } | ||
712 | |||
713 | /* Record the key lookup. */ | ||
714 | xref = rec_idx_key(J, ix); | ||
715 | xrefop = IR(tref_ref(xref))->o; | ||
716 | loadop = xrefop == IR_AREF ? IR_ALOAD : IR_HLOAD; | ||
717 | oldv = ix->oldv; | ||
718 | |||
719 | if (ix->val == 0) { /* Indexed load */ | ||
720 | IRType t = itype2irt(oldv); | ||
721 | TRef res = emitir(IRTG(loadop, t), xref, 0); | ||
722 | if (t == IRT_NIL && ix->idxchain && rec_mm_lookup(J, ix, MM_index)) | ||
723 | goto handlemm; | ||
724 | if (irtype_ispri(t)) res = TREF_PRI(t); /* Canonicalize primitives. */ | ||
725 | return res; | ||
726 | } else { /* Indexed store. */ | ||
727 | GCtab *mt = tabref(tabV(&ix->tabv)->metatable); | ||
728 | if (tvisnil(oldv)) { /* Previous value was nil? */ | ||
729 | /* Need to duplicate the hasmm check for the early guards. */ | ||
730 | int hasmm = 0; | ||
731 | if (ix->idxchain && mt) { | ||
732 | cTValue *mo = lj_tab_getstr(mt, strref(J2G(J)->mmname[MM_newindex])); | ||
733 | hasmm = mo && !tvisnil(mo); | ||
734 | } | ||
735 | if (hasmm || oldv == niltvg(J2G(J))) | ||
736 | emitir(IRTG(loadop, IRT_NIL), xref, 0); /* Guard for nil value. */ | ||
737 | else if (xrefop == IR_HREF) | ||
738 | emitir(IRTG(IR_NE, IRT_PTR), xref, lj_ir_kptr(J, niltvg(J2G(J)))); | ||
739 | if (ix->idxchain && rec_mm_lookup(J, ix, MM_newindex)) { /* Metamethod? */ | ||
740 | lua_assert(hasmm); | ||
741 | goto handlemm; | ||
742 | } | ||
743 | lua_assert(!hasmm); | ||
744 | if (oldv == niltvg(J2G(J))) { /* Need to insert a new key. */ | ||
745 | TRef key = ix->key; | ||
746 | if (tref_isinteger(key)) /* NEWREF needs a TValue as a key. */ | ||
747 | key = emitir(IRTN(IR_TONUM), key, 0); | ||
748 | xref = emitir(IRT(IR_NEWREF, IRT_PTR), ix->tab, key); | ||
749 | } | ||
750 | } else if (!lj_opt_fwd_wasnonnil(J, loadop, tref_ref(xref))) { | ||
751 | /* Cannot derive that the previous value was non-nil, must do checks. */ | ||
752 | if (xrefop == IR_HREF) /* Guard against store to niltv. */ | ||
753 | emitir(IRTG(IR_NE, IRT_PTR), xref, lj_ir_kptr(J, niltvg(J2G(J)))); | ||
754 | if (ix->idxchain) { /* Metamethod lookup required? */ | ||
755 | /* A check for NULL metatable is cheaper (hoistable) than a load. */ | ||
756 | if (!mt) { | ||
757 | TRef mtref = emitir(IRT(IR_FLOAD, IRT_TAB), ix->tab, IRFL_TAB_META); | ||
758 | emitir(IRTG(IR_EQ, IRT_TAB), mtref, lj_ir_knull(J, IRT_TAB)); | ||
759 | } else { | ||
760 | IRType t = itype2irt(oldv); | ||
761 | emitir(IRTG(loadop, t), xref, 0); /* Guard for non-nil value. */ | ||
762 | } | ||
763 | } | ||
764 | } | ||
765 | if (tref_isinteger(ix->val)) /* Convert int to number before storing. */ | ||
766 | ix->val = emitir(IRTN(IR_TONUM), ix->val, 0); | ||
767 | emitir(IRT(loadop+IRDELTA_L2S, tref_type(ix->val)), xref, ix->val); | ||
768 | if (tref_isgcv(ix->val)) | ||
769 | emitir(IRT(IR_TBAR, IRT_NIL), ix->tab, 0); | ||
770 | /* Invalidate neg. metamethod cache for stores with certain string keys. */ | ||
771 | if (!nommstr(J, ix->key)) { | ||
772 | TRef fref = emitir(IRT(IR_FREF, IRT_PTR), ix->tab, IRFL_TAB_NOMM); | ||
773 | emitir(IRT(IR_FSTORE, IRT_U8), fref, lj_ir_kint(J, 0)); | ||
774 | } | ||
775 | J->needsnap = 1; | ||
776 | return 0; | ||
777 | } | ||
778 | } | ||
779 | |||
780 | /* -- Upvalue access ------------------------------------------------------ */ | ||
781 | |||
782 | /* Record upvalue load/store. */ | ||
783 | static TRef rec_upvalue(jit_State *J, uint32_t uv, TRef val) | ||
784 | { | ||
785 | GCupval *uvp = &gcref(J->fn->l.uvptr[uv])->uv; | ||
786 | TRef fn = getcurrf(J); | ||
787 | IRRef uref; | ||
788 | int needbarrier = 0; | ||
789 | if (!uvp->closed) { | ||
790 | /* In current stack? */ | ||
791 | if (uvp->v >= J->L->stack && uvp->v < J->L->maxstack) { | ||
792 | int32_t slot = (int32_t)(uvp->v - (J->L->base - J->baseslot)); | ||
793 | if (slot >= 0) { /* Aliases an SSA slot? */ | ||
794 | slot -= (int32_t)J->baseslot; /* Note: slot number may be negative! */ | ||
795 | /* NYI: add IR to guard that it's still aliasing the same slot. */ | ||
796 | if (val == 0) { | ||
797 | return getslot(J, slot); | ||
798 | } else { | ||
799 | J->base[slot] = val; | ||
800 | if (slot >= (int32_t)J->maxslot) J->maxslot = (BCReg)(slot+1); | ||
801 | return 0; | ||
802 | } | ||
803 | } | ||
804 | } | ||
805 | uref = tref_ref(emitir(IRTG(IR_UREFO, IRT_PTR), fn, uv)); | ||
806 | } else { | ||
807 | needbarrier = 1; | ||
808 | uref = tref_ref(emitir(IRTG(IR_UREFC, IRT_PTR), fn, uv)); | ||
809 | } | ||
810 | if (val == 0) { /* Upvalue load */ | ||
811 | IRType t = itype2irt(uvp->v); | ||
812 | TRef res = emitir(IRTG(IR_ULOAD, t), uref, 0); | ||
813 | if (irtype_ispri(t)) res = TREF_PRI(t); /* Canonicalize primitive refs. */ | ||
814 | return res; | ||
815 | } else { /* Upvalue store. */ | ||
816 | if (tref_isinteger(val)) /* Convert int to number before storing. */ | ||
817 | val = emitir(IRTN(IR_TONUM), val, 0); | ||
818 | emitir(IRT(IR_USTORE, tref_type(val)), uref, val); | ||
819 | if (needbarrier && tref_isgcv(val)) | ||
820 | emitir(IRT(IR_OBAR, IRT_NIL), uref, val); | ||
821 | J->needsnap = 1; | ||
822 | return 0; | ||
823 | } | ||
824 | } | ||
825 | |||
826 | /* -- Record calls to fast functions -------------------------------------- */ | ||
827 | |||
828 | /* Note: The function and the arguments for the bytecode CALL instructions | ||
829 | ** always occupy _new_ stack slots (above the highest active variable). | ||
830 | ** This means they must have been stored there by previous instructions | ||
831 | ** (MOV, K*, ADD etc.) which must be part of the same trace. This in turn | ||
832 | ** means their reference slots are already valid and their types have | ||
833 | ** already been specialized (i.e. getslot() would be redundant). | ||
834 | ** The 1st slot beyond the arguments is set to 0 before calling recff_*. | ||
835 | */ | ||
836 | |||
837 | /* Data used by handlers to record a fast function. */ | ||
838 | typedef struct RecordFFData { | ||
839 | TValue *argv; /* Runtime argument values. */ | ||
840 | GCfunc *fn; /* The currently recorded function. */ | ||
841 | int nargs; /* Number of passed arguments. */ | ||
842 | int nres; /* Number of returned results (defaults to 1). */ | ||
843 | int cres; /* Wanted number of call results. */ | ||
844 | uint32_t data; /* Per-ffid auxiliary data (opcode, literal etc.). */ | ||
845 | } RecordFFData; | ||
846 | |||
847 | /* Type of handler to record a fast function. */ | ||
848 | typedef void (*RecordFunc)(jit_State *J, TRef *res, RecordFFData *rd); | ||
849 | |||
850 | /* Avoid carrying two pointers around. */ | ||
851 | #define arg (res+1) | ||
852 | |||
853 | /* Get runtime value of int argument. */ | ||
854 | static int32_t argv2int(jit_State *J, TValue *o) | ||
855 | { | ||
856 | if (tvisstr(o) && !lj_str_numconv(strVdata(o), o)) | ||
857 | lj_trace_err(J, LJ_TRERR_BADTYPE); | ||
858 | return lj_num2bit(numV(o)); | ||
859 | } | ||
860 | |||
861 | /* Get runtime value of string argument. */ | ||
862 | static GCstr *argv2str(jit_State *J, TValue *o) | ||
863 | { | ||
864 | if (LJ_LIKELY(tvisstr(o))) { | ||
865 | return strV(o); | ||
866 | } else { | ||
867 | GCstr *s; | ||
868 | lua_assert(tvisnum(o)); | ||
869 | s = lj_str_fromnum(J->L, &o->n); | ||
870 | setstrV(J->L, o, s); | ||
871 | return s; | ||
872 | } | ||
873 | } | ||
874 | |||
875 | /* Fallback handler for all fast functions that are not recorded (yet). */ | ||
876 | static void recff_nyi(jit_State *J, TRef *res, RecordFFData *rd) | ||
877 | { | ||
878 | UNUSED(res); | ||
879 | setfuncV(J->L, &J->errinfo, rd->fn); | ||
880 | lj_trace_err_info(J, LJ_TRERR_NYIFF); | ||
881 | } | ||
882 | |||
883 | LJ_NORET static void recff_err_ffu(jit_State *J, RecordFFData *rd) | ||
884 | { | ||
885 | setfuncV(J->L, &J->errinfo, rd->fn); | ||
886 | lj_trace_err_info(J, LJ_TRERR_NYIFFU); | ||
887 | } | ||
888 | |||
889 | /* C functions can have arbitrary side-effects and are not recorded (yet). */ | ||
890 | static void recff_c(jit_State *J, TRef *res, RecordFFData *rd) | ||
891 | { | ||
892 | UNUSED(res); | ||
893 | setlightudV(&J->errinfo, (void *)rd->fn->c.f); | ||
894 | lj_trace_err_info(J, LJ_TRERR_NYICF); | ||
895 | } | ||
896 | |||
897 | /* -- Base library fast functions ----------------------------------------- */ | ||
898 | |||
899 | static void recff_assert(jit_State *J, TRef *res, RecordFFData *rd) | ||
900 | { | ||
901 | /* Arguments already specialized. The interpreter throws for nil/false. */ | ||
902 | BCReg i; | ||
903 | for (i = 0; arg[i]; i++) /* Need to pass through all arguments. */ | ||
904 | res[i] = arg[i]; | ||
905 | rd->nres = (int)i; | ||
906 | UNUSED(J); | ||
907 | } | ||
908 | |||
909 | static void recff_type(jit_State *J, TRef *res, RecordFFData *rd) | ||
910 | { | ||
911 | /* Arguments already specialized. Result is a constant string. Neat, huh? */ | ||
912 | IRType t = tref_isinteger(arg[0]) ? IRT_NUM : tref_type(arg[0]); | ||
913 | res[0] = lj_ir_kstr(J, strV(&rd->fn->c.upvalue[t])); | ||
914 | } | ||
915 | |||
916 | static void recff_getmetatable(jit_State *J, TRef *res, RecordFFData *rd) | ||
917 | { | ||
918 | TRef tr = arg[0]; | ||
919 | if (tref_istab(tr)) { | ||
920 | RecordIndex ix; | ||
921 | ix.tab = tr; | ||
922 | copyTV(J->L, &ix.tabv, &rd->argv[0]); | ||
923 | if (rec_mm_lookup(J, &ix, MM_metatable)) | ||
924 | res[0] = ix.mobj; | ||
925 | else | ||
926 | res[0] = ix.mt; | ||
927 | } /* else: Interpreter will throw. */ | ||
928 | } | ||
929 | |||
930 | static void recff_setmetatable(jit_State *J, TRef *res, RecordFFData *rd) | ||
931 | { | ||
932 | TRef tr = arg[0]; | ||
933 | TRef mt = arg[1]; | ||
934 | if (tref_istab(tr) && (tref_istab(mt) || (mt && tref_isnil(mt)))) { | ||
935 | TRef fref, mtref; | ||
936 | RecordIndex ix; | ||
937 | ix.tab = tr; | ||
938 | copyTV(J->L, &ix.tabv, &rd->argv[0]); | ||
939 | rec_mm_lookup(J, &ix, MM_metatable); /* Guard for no __metatable field. */ | ||
940 | fref = emitir(IRT(IR_FREF, IRT_PTR), tr, IRFL_TAB_META); | ||
941 | mtref = tref_isnil(mt) ? lj_ir_knull(J, IRT_TAB) : mt; | ||
942 | emitir(IRT(IR_FSTORE, IRT_TAB), fref, mtref); | ||
943 | if (!tref_isnil(mt)) | ||
944 | emitir(IRT(IR_TBAR, IRT_TAB), tr, 0); | ||
945 | res[0] = tr; | ||
946 | J->needsnap = 1; | ||
947 | } /* else: Interpreter will throw. */ | ||
948 | } | ||
949 | |||
950 | static void recff_rawget(jit_State *J, TRef *res, RecordFFData *rd) | ||
951 | { | ||
952 | if (tref_istab(arg[0]) && arg[1]) { | ||
953 | RecordIndex ix; | ||
954 | ix.tab = arg[0]; ix.key = arg[1]; ix.val = 0; ix.idxchain = 0; | ||
955 | settabV(J->L, &ix.tabv, tabV(&rd->argv[0])); | ||
956 | copyTV(J->L, &ix.keyv, &rd->argv[1]); | ||
957 | res[0] = rec_idx(J, &ix); | ||
958 | } /* else: Interpreter will throw. */ | ||
959 | } | ||
960 | |||
961 | static void recff_rawset(jit_State *J, TRef *res, RecordFFData *rd) | ||
962 | { | ||
963 | if (tref_istab(arg[0]) && arg[1] && arg[2]) { | ||
964 | RecordIndex ix; | ||
965 | ix.tab = arg[0]; ix.key = arg[1]; ix.val = arg[2]; ix.idxchain = 0; | ||
966 | settabV(J->L, &ix.tabv, tabV(&rd->argv[0])); | ||
967 | copyTV(J->L, &ix.keyv, &rd->argv[1]); | ||
968 | copyTV(J->L, &ix.valv, &rd->argv[2]); | ||
969 | rec_idx(J, &ix); | ||
970 | res[0] = arg[0]; /* Returns table. */ | ||
971 | } /* else: Interpreter will throw. */ | ||
972 | } | ||
973 | |||
974 | static void recff_rawequal(jit_State *J, TRef *res, RecordFFData *rd) | ||
975 | { | ||
976 | if (arg[0] && arg[1]) { | ||
977 | int diff = rec_objcmp(J, arg[0], arg[1], &rd->argv[0], &rd->argv[1]); | ||
978 | res[0] = diff ? TREF_FALSE : TREF_TRUE; | ||
979 | } /* else: Interpreter will throw. */ | ||
980 | } | ||
981 | |||
982 | static void recff_tonumber(jit_State *J, TRef *res, RecordFFData *rd) | ||
983 | { | ||
984 | TRef tr = arg[0]; | ||
985 | if (tref_isnumber_str(tr)) { | ||
986 | if (arg[1]) { | ||
987 | TRef base = lj_ir_toint(J, arg[1]); | ||
988 | if (!tref_isk(base) || IR(tref_ref(base))->i != 10) | ||
989 | recff_err_ffu(J, rd); | ||
990 | } | ||
991 | if (tref_isstr(tr)) | ||
992 | tr = emitir(IRTG(IR_STRTO, IRT_NUM), tr, 0); | ||
993 | } else { | ||
994 | tr = TREF_NIL; | ||
995 | } | ||
996 | res[0] = tr; | ||
997 | UNUSED(rd); | ||
998 | } | ||
999 | |||
1000 | static void recff_tostring(jit_State *J, TRef *res, RecordFFData *rd) | ||
1001 | { | ||
1002 | TRef tr = arg[0]; | ||
1003 | if (tref_isstr(tr)) { | ||
1004 | /* Ignore __tostring in the string base metatable. */ | ||
1005 | res[0] = tr; | ||
1006 | } else { | ||
1007 | RecordIndex ix; | ||
1008 | ix.tab = tr; | ||
1009 | copyTV(J->L, &ix.tabv, &rd->argv[0]); | ||
1010 | if (rec_mm_lookup(J, &ix, MM_tostring)) { /* Has __tostring metamethod? */ | ||
1011 | res[0] = ix.mobj; | ||
1012 | copyTV(J->L, rd->argv - 1, &ix.mobjv); | ||
1013 | if (!rec_call(J, (BCReg)(res - J->base), 1, 1)) /* Pending call? */ | ||
1014 | rd->cres = CALLRES_PENDING; | ||
1015 | /* Otherwise res[0] already contains the result. */ | ||
1016 | } else if (tref_isnumber(tr)) { | ||
1017 | res[0] = emitir(IRT(IR_TOSTR, IRT_STR), tr, 0); | ||
1018 | } else { | ||
1019 | recff_err_ffu(J, rd); | ||
1020 | } | ||
1021 | } | ||
1022 | } | ||
1023 | |||
1024 | static void recff_ipairs_aux(jit_State *J, TRef *res, RecordFFData *rd) | ||
1025 | { | ||
1026 | RecordIndex ix; | ||
1027 | ix.tab = arg[0]; | ||
1028 | if (tref_istab(ix.tab)) { | ||
1029 | if (!tvisnum(&rd->argv[1])) /* No support for string coercion. */ | ||
1030 | lj_trace_err(J, LJ_TRERR_BADTYPE); | ||
1031 | setnumV(&ix.keyv, numV(&rd->argv[1])+(lua_Number)1); | ||
1032 | settabV(J->L, &ix.tabv, tabV(&rd->argv[0])); | ||
1033 | ix.val = 0; ix.idxchain = 0; | ||
1034 | ix.key = lj_ir_toint(J, arg[1]); | ||
1035 | res[0] = ix.key = emitir(IRTI(IR_ADD), ix.key, lj_ir_kint(J, 1)); | ||
1036 | res[1] = rec_idx(J, &ix); | ||
1037 | rd->nres = tref_isnil(res[1]) ? 0 : 2; | ||
1038 | } /* else: Interpreter will throw. */ | ||
1039 | } | ||
1040 | |||
1041 | static void recff_ipairs(jit_State *J, TRef *res, RecordFFData *rd) | ||
1042 | { | ||
1043 | TRef tab = arg[0]; | ||
1044 | if (tref_istab(tab)) { | ||
1045 | res[0] = lj_ir_kfunc(J, funcV(&rd->fn->c.upvalue[0])); | ||
1046 | res[1] = tab; | ||
1047 | res[2] = lj_ir_kint(J, 0); | ||
1048 | rd->nres = 3; | ||
1049 | } /* else: Interpreter will throw. */ | ||
1050 | } | ||
1051 | |||
1052 | static void recff_pcall(jit_State *J, TRef *res, RecordFFData *rd) | ||
1053 | { | ||
1054 | if (rd->nargs >= 1) { | ||
1055 | BCReg parg = (BCReg)(arg - J->base); | ||
1056 | if (rec_call(J, parg, CALLRES_MULTI, rd->nargs - 1)) { /* Resolved call. */ | ||
1057 | res[0] = TREF_TRUE; /* Prepend true result. No need to move results. */ | ||
1058 | rd->nres = (int)((J->maxslot - parg) + 1); | ||
1059 | } else { /* Propagate pending call. */ | ||
1060 | rd->cres = CALLRES_PENDING; | ||
1061 | } | ||
1062 | } /* else: Interpreter will throw. */ | ||
1063 | } | ||
1064 | |||
1065 | /* Struct to pass context across lj_vm_cpcall. */ | ||
1066 | typedef struct RecordXpcall { | ||
1067 | BCReg parg; | ||
1068 | int nargs; | ||
1069 | int resolved; | ||
1070 | } RecordXpcall; | ||
1071 | |||
1072 | static TValue *recff_xpcall_cp(lua_State *L, lua_CFunction dummy, void *ud) | ||
1073 | { | ||
1074 | jit_State *J = L2J(L); | ||
1075 | RecordXpcall *rx = (RecordXpcall *)ud; | ||
1076 | UNUSED(dummy); | ||
1077 | rx->resolved = rec_call(J, rx->parg, CALLRES_MULTI, rx->nargs); | ||
1078 | return NULL; | ||
1079 | } | ||
1080 | |||
1081 | static void recff_xpcall(jit_State *J, TRef *res, RecordFFData *rd) | ||
1082 | { | ||
1083 | if (rd->nargs >= 2) { | ||
1084 | RecordXpcall rx; | ||
1085 | BCReg parg = (BCReg)(arg - J->base) + 1; | ||
1086 | TRef tmp; | ||
1087 | TValue argv0, argv1; | ||
1088 | ptrdiff_t oargv; | ||
1089 | int errcode; | ||
1090 | /* Swap function and traceback. */ | ||
1091 | tmp = arg[0]; arg[0] = arg[1]; arg[1] = tmp; | ||
1092 | copyTV(J->L, &argv0, &rd->argv[0]); | ||
1093 | copyTV(J->L, &argv1, &rd->argv[1]); | ||
1094 | copyTV(J->L, &rd->argv[0], &argv1); | ||
1095 | copyTV(J->L, &rd->argv[1], &argv0); | ||
1096 | oargv = savestack(J->L, rd->argv); | ||
1097 | /* Need to protect rec_call because the recorder may throw. */ | ||
1098 | rx.parg = parg; | ||
1099 | rx.nargs = rd->nargs - 2; | ||
1100 | errcode = lj_vm_cpcall(J->L, recff_xpcall_cp, NULL, &rx); | ||
1101 | /* Always undo Lua stack swap to avoid confusing the interpreter. */ | ||
1102 | rd->argv = restorestack(J->L, oargv); /* Stack may have been resized. */ | ||
1103 | copyTV(J->L, &rd->argv[0], &argv0); | ||
1104 | copyTV(J->L, &rd->argv[1], &argv1); | ||
1105 | if (errcode) | ||
1106 | lj_err_throw(J->L, errcode); /* Propagate errors. */ | ||
1107 | if (rx.resolved) { /* Resolved call. */ | ||
1108 | int i, nres = (int)(J->maxslot - parg); | ||
1109 | rd->nres = nres + 1; | ||
1110 | res[0] = TREF_TRUE; /* Prepend true result. */ | ||
1111 | for (i = 1; i <= nres; i++) /* Move results down. */ | ||
1112 | res[i] = res[i+1]; | ||
1113 | } else { /* Propagate pending call. */ | ||
1114 | rd->cres = CALLRES_PENDING; | ||
1115 | } | ||
1116 | } /* else: Interpreter will throw. */ | ||
1117 | } | ||
1118 | |||
1119 | /* -- Math library fast functions ----------------------------------------- */ | ||
1120 | |||
1121 | static void recff_math_abs(jit_State *J, TRef *res, RecordFFData *rd) | ||
1122 | { | ||
1123 | TRef tr = lj_ir_tonum(J, arg[0]); | ||
1124 | res[0] = emitir(IRTN(IR_ABS), tr, lj_ir_knum_abs(J)); | ||
1125 | UNUSED(rd); | ||
1126 | } | ||
1127 | |||
1128 | /* Record rounding functions math.floor and math.ceil. */ | ||
1129 | static void recff_math_round(jit_State *J, TRef *res, RecordFFData *rd) | ||
1130 | { | ||
1131 | if (tref_isinteger(arg[0])) | ||
1132 | res[0] = arg[0]; | ||
1133 | else | ||
1134 | res[0] = emitir(IRTN(IR_FPMATH), lj_ir_tonum(J, arg[0]), rd->data); | ||
1135 | /* Note: result is integral (or NaN/Inf), but may not fit into an integer. */ | ||
1136 | } | ||
1137 | |||
1138 | /* Record unary math.* functions, mapped to IR_FPMATH opcode. */ | ||
1139 | static void recff_math_unary(jit_State *J, TRef *res, RecordFFData *rd) | ||
1140 | { | ||
1141 | res[0] = emitir(IRTN(IR_FPMATH), lj_ir_tonum(J, arg[0]), rd->data); | ||
1142 | } | ||
1143 | |||
1144 | /* Record binary math.* functions math.atan2 and math.ldexp. */ | ||
1145 | static void recff_math_binary(jit_State *J, TRef *res, RecordFFData *rd) | ||
1146 | { | ||
1147 | TRef tr = lj_ir_tonum(J, arg[0]); | ||
1148 | res[0] = emitir(IRTN(rd->data), tr, lj_ir_tonum(J, arg[1])); | ||
1149 | } | ||
1150 | |||
1151 | /* Record math.asin, math.acos, math.atan. */ | ||
1152 | static void recff_math_atrig(jit_State *J, TRef *res, RecordFFData *rd) | ||
1153 | { | ||
1154 | TRef y = lj_ir_tonum(J, arg[0]); | ||
1155 | TRef x = lj_ir_knum_one(J); | ||
1156 | uint32_t ffid = rd->data; | ||
1157 | if (ffid != FF_math_atan) { | ||
1158 | TRef tmp = emitir(IRTN(IR_MUL), y, y); | ||
1159 | tmp = emitir(IRTN(IR_SUB), x, tmp); | ||
1160 | tmp = emitir(IRTN(IR_FPMATH), tmp, IRFPM_SQRT); | ||
1161 | if (ffid == FF_math_asin) { x = tmp; } else { x = y; y = tmp; } | ||
1162 | } | ||
1163 | res[0] = emitir(IRTN(IR_ATAN2), y, x); | ||
1164 | } | ||
1165 | |||
1166 | static void recff_math_modf(jit_State *J, TRef *res, RecordFFData *rd) | ||
1167 | { | ||
1168 | TRef tr = arg[0]; | ||
1169 | if (tref_isinteger(arg[0])) { | ||
1170 | res[0] = tr; | ||
1171 | res[1] = lj_ir_kint(J, 0); | ||
1172 | } else { | ||
1173 | tr = lj_ir_tonum(J, tr); | ||
1174 | res[0] = emitir(IRTN(IR_FPMATH), tr, IRFPM_TRUNC); | ||
1175 | res[1] = emitir(IRTN(IR_SUB), tr, res[0]); | ||
1176 | } | ||
1177 | rd->nres = 2; | ||
1178 | } | ||
1179 | |||
1180 | static void recff_math_degrad(jit_State *J, TRef *res, RecordFFData *rd) | ||
1181 | { | ||
1182 | TRef tr = lj_ir_tonum(J, arg[0]); | ||
1183 | res[0] = emitir(IRTN(IR_MUL), tr, lj_ir_knum(J, numV(&rd->fn->c.upvalue[0]))); | ||
1184 | } | ||
1185 | |||
1186 | static void recff_math_pow(jit_State *J, TRef *res, RecordFFData *rd) | ||
1187 | { | ||
1188 | if (!tref_isnumber_str(arg[1])) | ||
1189 | lj_trace_err(J, LJ_TRERR_BADTYPE); | ||
1190 | res[0] = lj_opt_narrow_pow(J, lj_ir_tonum(J, arg[0]), arg[1], &rd->argv[1]); | ||
1191 | UNUSED(rd); | ||
1192 | } | ||
1193 | |||
1194 | static void recff_math_minmax(jit_State *J, TRef *res, RecordFFData *rd) | ||
1195 | { | ||
1196 | TRef tr = lj_ir_tonum(J, arg[0]); | ||
1197 | uint32_t op = rd->data; | ||
1198 | BCReg i; | ||
1199 | for (i = 1; arg[i]; i++) | ||
1200 | tr = emitir(IRTN(op), tr, lj_ir_tonum(J, arg[i])); | ||
1201 | res[0] = tr; | ||
1202 | } | ||
1203 | |||
1204 | /* -- Bit library fast functions ------------------------------------------ */ | ||
1205 | |||
1206 | /* Record unary bit.tobit, bit.bnot, bit.bswap. */ | ||
1207 | static void recff_bit_unary(jit_State *J, TRef *res, RecordFFData *rd) | ||
1208 | { | ||
1209 | TRef tr = lj_ir_tobit(J, arg[0]); | ||
1210 | res[0] = (rd->data == IR_TOBIT) ? tr : emitir(IRTI(rd->data), tr, 0); | ||
1211 | } | ||
1212 | |||
1213 | /* Record N-ary bit.band, bit.bor, bit.bxor. */ | ||
1214 | static void recff_bit_nary(jit_State *J, TRef *res, RecordFFData *rd) | ||
1215 | { | ||
1216 | TRef tr = lj_ir_tobit(J, arg[0]); | ||
1217 | uint32_t op = rd->data; | ||
1218 | BCReg i; | ||
1219 | for (i = 1; arg[i]; i++) | ||
1220 | tr = emitir(IRTI(op), tr, lj_ir_tobit(J, arg[i])); | ||
1221 | res[0] = tr; | ||
1222 | } | ||
1223 | |||
1224 | /* Record bit shifts. */ | ||
1225 | static void recff_bit_shift(jit_State *J, TRef *res, RecordFFData *rd) | ||
1226 | { | ||
1227 | TRef tr = lj_ir_tobit(J, arg[0]); | ||
1228 | TRef tsh = lj_ir_tobit(J, arg[1]); | ||
1229 | #if !LJ_TARGET_MASKEDSHIFT | ||
1230 | if (!tref_isk(tsh)) | ||
1231 | tsh = emitir(IRTI(IR_BAND), tsh, lj_ir_kint(J, 31)); | ||
1232 | #endif | ||
1233 | res[0] = emitir(IRTI(rd->data), tr, tsh); | ||
1234 | } | ||
1235 | |||
1236 | /* -- String library fast functions --------------------------------------- */ | ||
1237 | |||
1238 | static void recff_string_len(jit_State *J, TRef *res, RecordFFData *rd) | ||
1239 | { | ||
1240 | res[0] = emitir(IRTI(IR_FLOAD), lj_ir_tostr(J, arg[0]), IRFL_STR_LEN); | ||
1241 | UNUSED(rd); | ||
1242 | } | ||
1243 | |||
1244 | /* Handle string.byte (rd->data = 0) and string.sub (rd->data = 1). */ | ||
1245 | static void recff_string_range(jit_State *J, TRef *res, RecordFFData *rd) | ||
1246 | { | ||
1247 | TRef trstr = lj_ir_tostr(J, arg[0]); | ||
1248 | TRef trlen = emitir(IRTI(IR_FLOAD), trstr, IRFL_STR_LEN); | ||
1249 | TRef tr0 = lj_ir_kint(J, 0); | ||
1250 | TRef trstart, trend; | ||
1251 | GCstr *str = argv2str(J, &rd->argv[0]); | ||
1252 | int32_t start, end; | ||
1253 | if (rd->data) { /* string.sub(str, start [,end]) */ | ||
1254 | trstart = lj_ir_toint(J, arg[1]); | ||
1255 | trend = tref_isnil(arg[2]) ? lj_ir_kint(J, -1) : lj_ir_toint(J, arg[2]); | ||
1256 | start = argv2int(J, &rd->argv[1]); | ||
1257 | end = tref_isnil(arg[2]) ? -1 : argv2int(J, &rd->argv[2]); | ||
1258 | } else { /* string.byte(str, [,start [,end]]) */ | ||
1259 | if (arg[1]) { | ||
1260 | trstart = lj_ir_toint(J, arg[1]); | ||
1261 | trend = tref_isnil(arg[2]) ? trstart : lj_ir_toint(J, arg[2]); | ||
1262 | start = argv2int(J, &rd->argv[1]); | ||
1263 | end = tref_isnil(arg[2]) ? start : argv2int(J, &rd->argv[2]); | ||
1264 | } else { | ||
1265 | trend = trstart = lj_ir_kint(J, 1); | ||
1266 | end = start = 1; | ||
1267 | } | ||
1268 | } | ||
1269 | if (end < 0) { | ||
1270 | emitir(IRTGI(IR_LT), trend, tr0); | ||
1271 | trend = emitir(IRTI(IR_ADD), emitir(IRTI(IR_ADD), trlen, trend), | ||
1272 | lj_ir_kint(J, 1)); | ||
1273 | end = end+(int32_t)str->len+1; | ||
1274 | } else if ((MSize)end <= str->len) { | ||
1275 | emitir(IRTGI(IR_ULE), trend, trlen); | ||
1276 | } else { | ||
1277 | emitir(IRTGI(IR_GT), trend, trlen); | ||
1278 | end = (int32_t)str->len; | ||
1279 | trend = trlen; | ||
1280 | } | ||
1281 | if (start < 0) { | ||
1282 | emitir(IRTGI(IR_LT), trstart, tr0); | ||
1283 | trstart = emitir(IRTI(IR_ADD), trlen, trstart); | ||
1284 | start = start+(int32_t)str->len; | ||
1285 | emitir(start < 0 ? IRTGI(IR_LT) : IRTGI(IR_GE), trstart, tr0); | ||
1286 | if (start < 0) { | ||
1287 | trstart = tr0; | ||
1288 | start = 0; | ||
1289 | } | ||
1290 | } else { | ||
1291 | if (start == 0) { | ||
1292 | emitir(IRTGI(IR_EQ), trstart, tr0); | ||
1293 | trstart = tr0; | ||
1294 | } else { | ||
1295 | trstart = emitir(IRTI(IR_ADD), trstart, lj_ir_kint(J, -1)); | ||
1296 | emitir(IRTGI(IR_GE), trstart, tr0); | ||
1297 | start--; | ||
1298 | } | ||
1299 | } | ||
1300 | if (rd->data) { /* Return string.sub result. */ | ||
1301 | if (end - start >= 0) { | ||
1302 | /* Also handle empty range here, to avoid extra traces. */ | ||
1303 | TRef trptr, trslen = emitir(IRTI(IR_SUB), trend, trstart); | ||
1304 | emitir(IRTGI(IR_GE), trslen, tr0); | ||
1305 | trptr = emitir(IRT(IR_STRREF, IRT_PTR), trstr, trstart); | ||
1306 | res[0] = emitir(IRT(IR_SNEW, IRT_STR), trptr, trslen); | ||
1307 | } else { /* Range underflow: return empty string. */ | ||
1308 | emitir(IRTGI(IR_LT), trend, trstart); | ||
1309 | res[0] = lj_ir_kstr(J, lj_str_new(J->L, strdata(str), 0)); | ||
1310 | } | ||
1311 | } else { /* Return string.byte result(s). */ | ||
1312 | int32_t i, len = end - start; | ||
1313 | if (len > 0) { | ||
1314 | TRef trslen = emitir(IRTI(IR_SUB), trend, trstart); | ||
1315 | emitir(IRTGI(IR_EQ), trslen, lj_ir_kint(J, len)); | ||
1316 | if (res + len > J->slot + LJ_MAX_JSLOTS) | ||
1317 | lj_trace_err(J, LJ_TRERR_STACKOV); | ||
1318 | rd->nres = len; | ||
1319 | for (i = 0; i < len; i++) { | ||
1320 | TRef tmp = emitir(IRTI(IR_ADD), trstart, lj_ir_kint(J, i)); | ||
1321 | tmp = emitir(IRT(IR_STRREF, IRT_PTR), trstr, tmp); | ||
1322 | res[i] = emitir(IRT(IR_XLOAD, IRT_U8), tmp, 0); | ||
1323 | } | ||
1324 | } else { /* Empty range or range underflow: return no results. */ | ||
1325 | emitir(IRTGI(IR_LE), trend, trstart); | ||
1326 | rd->nres = 0; | ||
1327 | } | ||
1328 | } | ||
1329 | } | ||
1330 | |||
1331 | /* -- Table library fast functions ---------------------------------------- */ | ||
1332 | |||
1333 | static void recff_table_getn(jit_State *J, TRef *res, RecordFFData *rd) | ||
1334 | { | ||
1335 | if (tref_istab(arg[0])) { | ||
1336 | res[0] = emitir(IRTI(IR_TLEN), arg[0], 0); | ||
1337 | } /* else: Interpreter will throw. */ | ||
1338 | UNUSED(rd); | ||
1339 | } | ||
1340 | |||
1341 | /* -- Record calls and returns -------------------------------------------- */ | ||
1342 | |||
1343 | #undef arg | ||
1344 | |||
1345 | #include "lj_recdef.h" | ||
1346 | |||
1347 | /* Record return. */ | ||
1348 | static void rec_ret(jit_State *J, BCReg rbase, int gotresults) | ||
1349 | { | ||
1350 | TValue *frame = J->L->base - 1; | ||
1351 | TRef *res = J->base + rbase; | ||
1352 | J->tailcalled = 0; | ||
1353 | while (frame_ispcall(frame)) { | ||
1354 | BCReg cbase = (BCReg)frame_delta(frame); | ||
1355 | lua_assert(J->baseslot > 1); | ||
1356 | J->baseslot -= (BCReg)cbase; | ||
1357 | J->base -= cbase; | ||
1358 | *--res = TREF_TRUE; /* Prepend true to results. */ | ||
1359 | gotresults++; | ||
1360 | J->framedepth--; | ||
1361 | frame = frame_prevd(frame); | ||
1362 | } | ||
1363 | if (J->framedepth-- <= 0) | ||
1364 | lj_trace_err(J, LJ_TRERR_NYIRETL); | ||
1365 | lua_assert(J->baseslot > 1); | ||
1366 | if (frame_islua(frame)) { | ||
1367 | BCIns callins = *(J->pc = frame_pc(frame)-1); | ||
1368 | ptrdiff_t nresults = bc_b(callins) ? (int)bc_b(callins)-1 : gotresults; | ||
1369 | BCReg cbase = bc_a(callins); | ||
1370 | int i; | ||
1371 | for (i = 0; i < nresults; i++) | ||
1372 | J->base[i-1] = i < gotresults ? res[i] : TREF_NIL; | ||
1373 | J->maxslot = cbase+(BCReg)nresults; | ||
1374 | J->baseslot -= cbase+1; | ||
1375 | J->base -= cbase+1; | ||
1376 | } else if (frame_iscont(frame)) { | ||
1377 | ASMFunction cont = frame_contf(frame); | ||
1378 | BCReg i, cbase = (BCReg)frame_delta(frame); | ||
1379 | J->pc = frame_contpc(frame)-1; | ||
1380 | J->baseslot -= (BCReg)cbase; | ||
1381 | J->base -= cbase; | ||
1382 | /* Shrink maxslot as much as possible after return from continuation. */ | ||
1383 | for (i = cbase-2; i > 0 && J->base[i] == 0; i--) ; | ||
1384 | J->maxslot = i; | ||
1385 | if (cont == lj_cont_ra) { | ||
1386 | /* Copy result to destination slot. */ | ||
1387 | BCReg dst = bc_a(*J->pc); | ||
1388 | J->base[dst] = res[0]; | ||
1389 | if (dst > J->maxslot) J->maxslot = dst+1; | ||
1390 | } else if (cont == lj_cont_nop) { | ||
1391 | /* Nothing to do here. */ | ||
1392 | } else if (cont == lj_cont_cat) { | ||
1393 | lua_assert(0); | ||
1394 | } else { | ||
1395 | /* Result type already specialized. */ | ||
1396 | lua_assert(cont == lj_cont_condf || cont == lj_cont_condt); | ||
1397 | } | ||
1398 | } else { | ||
1399 | lua_assert(0); | ||
1400 | } | ||
1401 | lua_assert(J->baseslot >= 1); | ||
1402 | } | ||
1403 | |||
1404 | /* Check unroll limits for calls. */ | ||
1405 | static void check_call_unroll(jit_State *J, GCfunc *fn) | ||
1406 | { | ||
1407 | TValue *first = J->L->base - J->baseslot; | ||
1408 | TValue *frame = J->L->base - 1; | ||
1409 | int count = 0; | ||
1410 | while (frame > first) { | ||
1411 | if (frame_func(frame) == fn) | ||
1412 | count++; | ||
1413 | if (frame_isvarg(frame)) | ||
1414 | frame = frame_prevd(frame); | ||
1415 | frame = frame_prev(frame); | ||
1416 | } | ||
1417 | if (frame_func(first) == fn && bc_op(J->cur.startins) == BC_CALL) { | ||
1418 | if (count >= J->param[JIT_P_recunroll]) | ||
1419 | lj_trace_err(J, LJ_TRERR_NYIRECU); | ||
1420 | } else { | ||
1421 | if (count >= J->param[JIT_P_callunroll]) | ||
1422 | lj_trace_err(J, LJ_TRERR_CUNROLL); | ||
1423 | } | ||
1424 | } | ||
1425 | |||
1426 | /* Record call. Returns 0 for pending calls and 1 for resolved calls. */ | ||
1427 | static int rec_call(jit_State *J, BCReg func, int cres, int nargs) | ||
1428 | { | ||
1429 | RecordFFData rd; | ||
1430 | TRef *res = &J->base[func]; | ||
1431 | TValue *tv = &J->L->base[func]; | ||
1432 | |||
1433 | if (tref_isfunc(res[0])) { /* Regular function call. */ | ||
1434 | rd.fn = funcV(tv); | ||
1435 | rd.argv = tv+1; | ||
1436 | } else { /* Otherwise resolve __call metamethod for called object. */ | ||
1437 | RecordIndex ix; | ||
1438 | int i; | ||
1439 | ix.tab = res[0]; | ||
1440 | copyTV(J->L, &ix.tabv, tv); | ||
1441 | if (!rec_mm_lookup(J, &ix, MM_call) || !tref_isfunc(ix.mobj)) | ||
1442 | lj_trace_err(J, LJ_TRERR_NOMM); | ||
1443 | /* Update the recorder state, but not the Lua stack. */ | ||
1444 | for (i = ++nargs; i > 0; i--) | ||
1445 | res[i] = res[i-1]; | ||
1446 | res[0] = ix.mobj; | ||
1447 | rd.fn = funcV(&ix.mobjv); | ||
1448 | rd.argv = tv; /* The called object is the 1st arg. */ | ||
1449 | } | ||
1450 | |||
1451 | /* Specialize to the runtime value of the called function. */ | ||
1452 | res[0] = emitir(IRTG(IR_FRAME, IRT_FUNC), res[0], lj_ir_kfunc(J, rd.fn)); | ||
1453 | |||
1454 | if (isluafunc(rd.fn)) { /* Record call to Lua function. */ | ||
1455 | GCproto *pt = funcproto(rd.fn); | ||
1456 | if ((pt->flags & PROTO_NO_JIT)) | ||
1457 | lj_trace_err(J, LJ_TRERR_CJITOFF); | ||
1458 | if ((pt->flags & PROTO_IS_VARARG)) { | ||
1459 | if (rd.fn->l.gate != lj_gate_lv) | ||
1460 | lj_trace_err(J, LJ_TRERR_NYILNKF); | ||
1461 | lj_trace_err(J, LJ_TRERR_NYIVF); | ||
1462 | } else { | ||
1463 | if (rd.fn->l.gate != lj_gate_lf) | ||
1464 | lj_trace_err(J, LJ_TRERR_NYILNKF); | ||
1465 | } | ||
1466 | check_call_unroll(J, rd.fn); | ||
1467 | if (cres == CALLRES_TAILCALL) { | ||
1468 | int i; | ||
1469 | /* Tailcalls can form a loop, so count towards the loop unroll limit. */ | ||
1470 | if (++J->tailcalled > J->loopunroll) | ||
1471 | lj_trace_err(J, LJ_TRERR_LUNROLL); | ||
1472 | for (i = 0; i <= nargs; i++) /* Move func + args down. */ | ||
1473 | J->base[i-1] = res[i]; | ||
1474 | /* Note: the new FRAME is now at J->base[-1] (even for slot #0). */ | ||
1475 | } else { /* Regular call. */ | ||
1476 | J->base += func+1; | ||
1477 | J->baseslot += func+1; | ||
1478 | J->framedepth++; | ||
1479 | } | ||
1480 | if (J->baseslot + pt->framesize >= LJ_MAX_JSLOTS) | ||
1481 | lj_trace_err(J, LJ_TRERR_STACKOV); | ||
1482 | /* Fill up missing args with nil. */ | ||
1483 | while (nargs < pt->numparams) | ||
1484 | J->base[nargs++] = TREF_NIL; | ||
1485 | /* The remaining slots should never be read before they are written. */ | ||
1486 | J->maxslot = pt->numparams; | ||
1487 | return 0; /* No result yet. */ | ||
1488 | } else { /* Record call to C function or fast function. */ | ||
1489 | uint32_t m = 0; | ||
1490 | res[1+nargs] = 0; | ||
1491 | rd.nargs = nargs; | ||
1492 | if (rd.fn->c.ffid < sizeof(recff_idmap)/sizeof(recff_idmap[0])) | ||
1493 | m = recff_idmap[rd.fn->c.ffid]; | ||
1494 | rd.data = m & 0xff; | ||
1495 | rd.cres = cres; | ||
1496 | rd.nres = 1; /* Default is one result. */ | ||
1497 | (recff_func[m >> 8])(J, res, &rd); /* Call recff_* handler. */ | ||
1498 | cres = rd.cres; | ||
1499 | if (cres >= 0) { | ||
1500 | /* Caller takes fixed number of results: local a,b = f() */ | ||
1501 | J->maxslot = func + (BCReg)cres; | ||
1502 | while (rd.nres < cres) /* Fill up missing results with nil. */ | ||
1503 | res[rd.nres++] = TREF_NIL; | ||
1504 | } else if (cres == CALLRES_MULTI) { | ||
1505 | /* Caller takes any number of results: return 1,f() */ | ||
1506 | J->maxslot = func + (BCReg)rd.nres; | ||
1507 | } else if (cres == CALLRES_TAILCALL) { | ||
1508 | /* Tail call: return f() */ | ||
1509 | rec_ret(J, func, rd.nres); | ||
1510 | } else if (cres == CALLRES_CONT) { | ||
1511 | /* Note: immediately resolved continuations must not change J->maxslot. */ | ||
1512 | res[rd.nres] = TREF_NIL; /* Turn 0 results into nil result. */ | ||
1513 | } else { | ||
1514 | J->framedepth++; | ||
1515 | lua_assert(cres == CALLRES_PENDING); | ||
1516 | return 0; /* Pending call, no result yet. */ | ||
1517 | } | ||
1518 | return 1; /* Result resolved immediately. */ | ||
1519 | } | ||
1520 | } | ||
1521 | |||
1522 | /* -- Record allocations -------------------------------------------------- */ | ||
1523 | |||
1524 | static TRef rec_tnew(jit_State *J, uint32_t ah) | ||
1525 | { | ||
1526 | uint32_t asize = ah & 0x7ff; | ||
1527 | uint32_t hbits = ah >> 11; | ||
1528 | if (asize == 0x7ff) asize = 0x801; | ||
1529 | return emitir(IRT(IR_TNEW, IRT_TAB), asize, hbits); | ||
1530 | } | ||
1531 | |||
1532 | /* -- Record bytecode ops ------------------------------------------------- */ | ||
1533 | |||
1534 | /* Optimize state after comparison. */ | ||
1535 | static void optstate_comp(jit_State *J, int cond) | ||
1536 | { | ||
1537 | BCIns jmpins = J->pc[1]; | ||
1538 | const BCIns *npc = J->pc + 2 + (cond ? bc_j(jmpins) : 0); | ||
1539 | SnapShot *snap = &J->cur.snap[J->cur.nsnap-1]; | ||
1540 | /* Avoid re-recording the comparison in side traces. */ | ||
1541 | J->cur.snapmap[snap->mapofs + snap->nslots] = u32ptr(npc); | ||
1542 | J->needsnap = 1; | ||
1543 | /* Shrink last snapshot if possible. */ | ||
1544 | if (bc_a(jmpins) < J->maxslot) { | ||
1545 | J->maxslot = bc_a(jmpins); | ||
1546 | lj_snap_shrink(J); | ||
1547 | } | ||
1548 | } | ||
1549 | |||
1550 | /* Record the next bytecode instruction (_before_ it's executed). */ | ||
1551 | void lj_record_ins(jit_State *J) | ||
1552 | { | ||
1553 | cTValue *lbase; | ||
1554 | RecordIndex ix; | ||
1555 | const BCIns *pc; | ||
1556 | BCIns ins; | ||
1557 | BCOp op; | ||
1558 | TRef ra, rb, rc; | ||
1559 | |||
1560 | /* Need snapshot before recording next bytecode (e.g. after a store). */ | ||
1561 | if (J->needsnap) { | ||
1562 | J->needsnap = 0; | ||
1563 | lj_snap_add(J); | ||
1564 | J->mergesnap = 1; | ||
1565 | } | ||
1566 | |||
1567 | /* Record only closed loops for root traces. */ | ||
1568 | pc = J->pc; | ||
1569 | if (J->framedepth == 0 && | ||
1570 | (MSize)((char *)pc - (char *)J->bc_min) >= J->bc_extent) | ||
1571 | lj_trace_err(J, LJ_TRERR_LLEAVE); | ||
1572 | |||
1573 | #ifdef LUA_USE_ASSERT | ||
1574 | rec_check_slots(J); | ||
1575 | rec_check_ir(J); | ||
1576 | #endif | ||
1577 | |||
1578 | /* Keep a copy of the runtime values of var/num/str operands. */ | ||
1579 | #define rav (&ix.valv) | ||
1580 | #define rbv (&ix.tabv) | ||
1581 | #define rcv (&ix.keyv) | ||
1582 | |||
1583 | lbase = J->L->base; | ||
1584 | ins = *pc; | ||
1585 | op = bc_op(ins); | ||
1586 | ra = bc_a(ins); | ||
1587 | ix.val = 0; | ||
1588 | switch (bcmode_a(op)) { | ||
1589 | case BCMvar: | ||
1590 | copyTV(J->L, rav, &lbase[ra]); ix.val = ra = getslot(J, ra); break; | ||
1591 | default: break; /* Handled later. */ | ||
1592 | } | ||
1593 | rb = bc_b(ins); | ||
1594 | rc = bc_c(ins); | ||
1595 | switch (bcmode_b(op)) { | ||
1596 | case BCMnone: rb = 0; rc = bc_d(ins); break; /* Upgrade rc to 'rd'. */ | ||
1597 | case BCMvar: | ||
1598 | copyTV(J->L, rbv, &lbase[rb]); ix.tab = rb = getslot(J, rb); break; | ||
1599 | case BCMnum: { lua_Number n = J->pt->k.n[rb]; | ||
1600 | setnumV(rbv, n); ix.tab = rb = lj_ir_knumint(J, n); } break; | ||
1601 | default: break; /* Handled later. */ | ||
1602 | } | ||
1603 | switch (bcmode_c(op)) { | ||
1604 | case BCMvar: | ||
1605 | copyTV(J->L, rcv, &lbase[rc]); ix.key = rc = getslot(J, rc); break; | ||
1606 | case BCMpri: setitype(rcv, (int32_t)~rc); rc = TREF_PRI(IRT_NIL+rc); break; | ||
1607 | case BCMnum: { lua_Number n = J->pt->k.n[rc]; | ||
1608 | setnumV(rcv, n); ix.key = rc = lj_ir_knumint(J, n); } break; | ||
1609 | case BCMstr: { GCstr *s = strref(J->pt->k.gc[~rc]); | ||
1610 | setstrV(J->L, rcv, s); ix.key = rc = lj_ir_kstr(J, s); } break; | ||
1611 | default: break; /* Handled later. */ | ||
1612 | } | ||
1613 | |||
1614 | switch (op) { | ||
1615 | |||
1616 | /* -- Comparison ops ---------------------------------------------------- */ | ||
1617 | |||
1618 | case BC_ISLT: case BC_ISGE: case BC_ISLE: case BC_ISGT: | ||
1619 | /* Emit nothing for two numeric or string consts. */ | ||
1620 | if (!(tref_isk2(ra,rc) && tref_isnumber_str(ra) && tref_isnumber_str(rc))) { | ||
1621 | IRType ta = tref_type(ra); | ||
1622 | IRType tc = tref_type(rc); | ||
1623 | int irop; | ||
1624 | if (ta != tc) { | ||
1625 | /* Widen mixed number/int comparisons to number/number comparison. */ | ||
1626 | if (ta == IRT_INT && tc == IRT_NUM) { | ||
1627 | ra = emitir(IRTN(IR_TONUM), ra, 0); | ||
1628 | ta = IRT_NUM; | ||
1629 | } else if (ta == IRT_NUM && tc == IRT_INT) { | ||
1630 | rc = emitir(IRTN(IR_TONUM), rc, 0); | ||
1631 | } else if (!((ta == IRT_FALSE || ta == IRT_TRUE) && | ||
1632 | (tc == IRT_FALSE || tc == IRT_TRUE))) { | ||
1633 | break; /* Interpreter will throw for two different types. */ | ||
1634 | } | ||
1635 | } | ||
1636 | lj_snap_add(J); | ||
1637 | irop = (int)op - (int)BC_ISLT + (int)IR_LT; | ||
1638 | if (ta == IRT_NUM) { | ||
1639 | if ((irop & 1)) irop ^= 4; /* ISGE/ISGT are unordered. */ | ||
1640 | if (!lj_ir_numcmp(numV(rav), numV(rcv), (IROp)irop)) irop ^= 5; | ||
1641 | } else if (ta == IRT_INT) { | ||
1642 | if (!lj_ir_numcmp(numV(rav), numV(rcv), (IROp)irop)) irop ^= 1; | ||
1643 | } else if (ta == IRT_STR) { | ||
1644 | if (!lj_ir_strcmp(strV(rav), strV(rcv), (IROp)irop)) irop ^= 1; | ||
1645 | } else { | ||
1646 | rec_mm_comp(J, &ix, (int)op); | ||
1647 | break; | ||
1648 | } | ||
1649 | emitir(IRTG(irop, ta), ra, rc); | ||
1650 | optstate_comp(J, ((int)op ^ irop) & 1); | ||
1651 | } | ||
1652 | break; | ||
1653 | |||
1654 | case BC_ISEQV: case BC_ISNEV: | ||
1655 | case BC_ISEQS: case BC_ISNES: | ||
1656 | case BC_ISEQN: case BC_ISNEN: | ||
1657 | case BC_ISEQP: case BC_ISNEP: | ||
1658 | /* Emit nothing for two non-table, non-udata consts. */ | ||
1659 | if (!(tref_isk2(ra, rc) && !(tref_istab(ra) || tref_isudata(ra)))) { | ||
1660 | int diff; | ||
1661 | lj_snap_add(J); | ||
1662 | diff = rec_objcmp(J, ra, rc, rav, rcv); | ||
1663 | if (diff == 1 && (tref_istab(ra) || tref_isudata(ra))) { | ||
1664 | /* Only check __eq if different, but the same type (table or udata). */ | ||
1665 | rec_mm_equal(J, &ix, (int)op); | ||
1666 | break; | ||
1667 | } | ||
1668 | optstate_comp(J, ((int)op & 1) == !diff); | ||
1669 | } | ||
1670 | break; | ||
1671 | |||
1672 | /* -- Unary test and copy ops ------------------------------------------- */ | ||
1673 | |||
1674 | case BC_ISTC: case BC_ISFC: | ||
1675 | if ((op & 1) == tref_istruecond(rc)) | ||
1676 | rc = 0; /* Don't store if condition is not true. */ | ||
1677 | /* fallthrough */ | ||
1678 | case BC_IST: case BC_ISF: /* Type specialization suffices. */ | ||
1679 | if (bc_a(pc[1]) < J->maxslot) | ||
1680 | J->maxslot = bc_a(pc[1]); /* Shrink used slots. */ | ||
1681 | break; | ||
1682 | |||
1683 | /* -- Unary ops --------------------------------------------------------- */ | ||
1684 | |||
1685 | case BC_NOT: | ||
1686 | /* Type specialization already forces const result. */ | ||
1687 | rc = tref_istruecond(rc) ? TREF_FALSE : TREF_TRUE; | ||
1688 | break; | ||
1689 | |||
1690 | case BC_LEN: | ||
1691 | if (tref_isstr(rc)) { | ||
1692 | rc = emitir(IRTI(IR_FLOAD), rc, IRFL_STR_LEN); | ||
1693 | } else if (tref_istab(rc)) { | ||
1694 | rc = emitir(IRTI(IR_TLEN), rc, 0); | ||
1695 | } else { | ||
1696 | ix.tab = rc; | ||
1697 | copyTV(J->L, &ix.tabv, &ix.keyv); | ||
1698 | ix.key = IRT_NIL; | ||
1699 | setnilV(&ix.keyv); | ||
1700 | rc = rec_mm_arith(J, &ix, MM_len); | ||
1701 | } | ||
1702 | break; | ||
1703 | |||
1704 | /* -- Arithmetic ops ---------------------------------------------------- */ | ||
1705 | |||
1706 | case BC_UNM: | ||
1707 | if (tref_isnumber_str(rc)) { | ||
1708 | rc = lj_ir_tonum(J, rc); | ||
1709 | rc = emitir(IRTN(IR_NEG), rc, lj_ir_knum_neg(J)); | ||
1710 | } else { | ||
1711 | ix.tab = rc; | ||
1712 | copyTV(J->L, &ix.tabv, &ix.keyv); | ||
1713 | rc = rec_mm_arith(J, &ix, MM_unm); | ||
1714 | } | ||
1715 | break; | ||
1716 | |||
1717 | case BC_ADDNV: case BC_SUBNV: case BC_MULNV: case BC_DIVNV: case BC_MODNV: | ||
1718 | ix.tab = rc; ix.key = rc = rb; rb = ix.tab; | ||
1719 | copyTV(J->L, &ix.valv, &ix.tabv); | ||
1720 | copyTV(J->L, &ix.tabv, &ix.keyv); | ||
1721 | copyTV(J->L, &ix.keyv, &ix.valv); | ||
1722 | if (op == BC_MODNV) | ||
1723 | goto recmod; | ||
1724 | /* fallthrough */ | ||
1725 | case BC_ADDVN: case BC_SUBVN: case BC_MULVN: case BC_DIVVN: | ||
1726 | case BC_ADDVV: case BC_SUBVV: case BC_MULVV: case BC_DIVVV: { | ||
1727 | MMS mm = bcmode_mm(op); | ||
1728 | if (tref_isnumber_str(rb) && tref_isnumber_str(rc)) { | ||
1729 | rb = lj_ir_tonum(J, rb); | ||
1730 | rc = lj_ir_tonum(J, rc); | ||
1731 | rc = emitir(IRTN((int)mm - (int)MM_add + (int)IR_ADD), rb, rc); | ||
1732 | } else { | ||
1733 | rc = rec_mm_arith(J, &ix, mm); | ||
1734 | } | ||
1735 | break; | ||
1736 | } | ||
1737 | |||
1738 | case BC_MODVN: case BC_MODVV: | ||
1739 | recmod: | ||
1740 | if (tref_isnumber_str(rb) && tref_isnumber_str(rc)) | ||
1741 | rc = lj_opt_narrow_mod(J, rb, rc); | ||
1742 | else | ||
1743 | rc = rec_mm_arith(J, &ix, MM_mod); | ||
1744 | break; | ||
1745 | |||
1746 | case BC_POW: | ||
1747 | if (tref_isnumber_str(rb) && tref_isnumber_str(rc)) | ||
1748 | rc = lj_opt_narrow_pow(J, lj_ir_tonum(J, rb), rc, rcv); | ||
1749 | else | ||
1750 | rc = rec_mm_arith(J, &ix, MM_pow); | ||
1751 | break; | ||
1752 | |||
1753 | /* -- Constant and move ops --------------------------------------------- */ | ||
1754 | |||
1755 | case BC_KSTR: case BC_KNUM: case BC_KPRI: case BC_MOV: | ||
1756 | break; | ||
1757 | case BC_KSHORT: | ||
1758 | rc = lj_ir_kint(J, (int32_t)(int16_t)rc); | ||
1759 | break; | ||
1760 | case BC_KNIL: | ||
1761 | while (ra <= rc) | ||
1762 | J->base[ra++] = TREF_NIL; | ||
1763 | if (rc >= J->maxslot) J->maxslot = rc+1; | ||
1764 | break; | ||
1765 | |||
1766 | /* -- Upvalue and function ops ------------------------------------------ */ | ||
1767 | |||
1768 | case BC_UGET: | ||
1769 | rc = rec_upvalue(J, rc, 0); | ||
1770 | break; | ||
1771 | case BC_USETV: case BC_USETS: case BC_USETN: case BC_USETP: | ||
1772 | rec_upvalue(J, ra, rc); | ||
1773 | break; | ||
1774 | |||
1775 | /* -- Table ops --------------------------------------------------------- */ | ||
1776 | |||
1777 | case BC_GGET: case BC_GSET: | ||
1778 | settabV(J->L, &ix.tabv, tabref(J->fn->l.env)); | ||
1779 | ix.tab = emitir(IRT(IR_FLOAD, IRT_TAB), getcurrf(J), IRFL_FUNC_ENV); | ||
1780 | ix.idxchain = LJ_MAX_IDXCHAIN; | ||
1781 | rc = rec_idx(J, &ix); | ||
1782 | break; | ||
1783 | |||
1784 | case BC_TGETB: case BC_TSETB: | ||
1785 | setintV(&ix.keyv, (int32_t)rc); | ||
1786 | ix.key = lj_ir_kint(J, (int32_t)rc); | ||
1787 | /* fallthrough */ | ||
1788 | case BC_TGETV: case BC_TGETS: case BC_TSETV: case BC_TSETS: | ||
1789 | ix.idxchain = LJ_MAX_IDXCHAIN; | ||
1790 | rc = rec_idx(J, &ix); | ||
1791 | break; | ||
1792 | |||
1793 | case BC_TNEW: | ||
1794 | rc = rec_tnew(J, rc); | ||
1795 | break; | ||
1796 | case BC_TDUP: | ||
1797 | rc = emitir(IRT(IR_TDUP, IRT_TAB), | ||
1798 | lj_ir_ktab(J, tabref(J->pt->k.gc[~rc])), 0); | ||
1799 | break; | ||
1800 | |||
1801 | /* -- Calls and vararg handling ----------------------------------------- */ | ||
1802 | |||
1803 | case BC_ITERC: | ||
1804 | J->base[ra] = getslot(J, ra-3); | ||
1805 | J->base[ra+1] = getslot(J, ra-2); | ||
1806 | J->base[ra+2] = getslot(J, ra-1); | ||
1807 | { /* Have to do the actual copy now because rec_call needs the values. */ | ||
1808 | TValue *b = &J->L->base[ra]; | ||
1809 | copyTV(J->L, b, b-3); | ||
1810 | copyTV(J->L, b+1, b-2); | ||
1811 | copyTV(J->L, b+2, b-1); | ||
1812 | } | ||
1813 | goto callop; | ||
1814 | |||
1815 | case BC_CALLMT: | ||
1816 | rb = (TRef)(CALLRES_TAILCALL+1); | ||
1817 | /* fallthrough */ | ||
1818 | case BC_CALLM: | ||
1819 | /* L->top is set to L->base+ra+rc+NRESULTS-1+1, see lj_dispatch_ins(). */ | ||
1820 | rc = (BCReg)(J->L->top - J->L->base) - ra; | ||
1821 | goto callop; | ||
1822 | |||
1823 | case BC_CALLT: | ||
1824 | rb = (TRef)(CALLRES_TAILCALL+1); | ||
1825 | /* fallthrough */ | ||
1826 | case BC_CALL: | ||
1827 | callop: | ||
1828 | if (rb == (TRef)(CALLRES_TAILCALL+1)) { /* Tail call. */ | ||
1829 | } | ||
1830 | rec_call(J, ra, (int)(rb-1), (int)(rc-1)); | ||
1831 | break; | ||
1832 | |||
1833 | /* -- Returns ----------------------------------------------------------- */ | ||
1834 | |||
1835 | case BC_RETM: | ||
1836 | /* L->top is set to L->base+ra+rc+NRESULTS-1, see lj_dispatch_ins(). */ | ||
1837 | rc = (BCReg)(J->L->top - J->L->base) - ra + 1; | ||
1838 | /* fallthrough */ | ||
1839 | case BC_RET: case BC_RET0: case BC_RET1: | ||
1840 | rec_ret(J, ra, (int)(rc-1)); | ||
1841 | break; | ||
1842 | |||
1843 | /* -- Loops and branches ------------------------------------------------ */ | ||
1844 | |||
1845 | case BC_FORI: | ||
1846 | if (rec_for(J, pc, 0) != LOOPEV_LEAVE) | ||
1847 | J->loopref = J->cur.nins; | ||
1848 | break; | ||
1849 | case BC_JFORI: | ||
1850 | lua_assert(bc_op(pc[(ptrdiff_t)rc-BCBIAS_J]) == BC_JFORL); | ||
1851 | if (rec_for(J, pc, 0) != LOOPEV_LEAVE) /* Link to existing loop. */ | ||
1852 | rec_stop(J, bc_d(pc[(ptrdiff_t)rc-BCBIAS_J])); | ||
1853 | /* Continue tracing if the loop is not entered. */ | ||
1854 | break; | ||
1855 | |||
1856 | case BC_FORL: | ||
1857 | rec_loop_interp(J, pc, rec_for(J, pc+((ptrdiff_t)rc-BCBIAS_J), 1)); | ||
1858 | break; | ||
1859 | case BC_ITERL: | ||
1860 | rec_loop_interp(J, pc, rec_iterl(J, *pc)); | ||
1861 | break; | ||
1862 | case BC_LOOP: | ||
1863 | rec_loop_interp(J, pc, rec_loop(J, ra)); | ||
1864 | break; | ||
1865 | |||
1866 | case BC_JFORL: | ||
1867 | rec_loop_jit(J, rc, rec_for(J, pc+bc_j(J->trace[rc]->startins), 1)); | ||
1868 | break; | ||
1869 | case BC_JITERL: | ||
1870 | rec_loop_jit(J, rc, rec_iterl(J, J->trace[rc]->startins)); | ||
1871 | break; | ||
1872 | case BC_JLOOP: | ||
1873 | rec_loop_jit(J, rc, rec_loop(J, ra)); | ||
1874 | break; | ||
1875 | |||
1876 | case BC_IFORL: | ||
1877 | case BC_IITERL: | ||
1878 | case BC_ILOOP: | ||
1879 | lj_trace_err_info(J, LJ_TRERR_LBLACKL); | ||
1880 | break; | ||
1881 | |||
1882 | case BC_JMP: | ||
1883 | if (ra < J->maxslot) | ||
1884 | J->maxslot = ra; /* Shrink used slots. */ | ||
1885 | break; | ||
1886 | |||
1887 | case BC_CAT: | ||
1888 | case BC_UCLO: | ||
1889 | case BC_FNEW: | ||
1890 | case BC_TSETM: | ||
1891 | case BC_VARG: | ||
1892 | default: | ||
1893 | setintV(&J->errinfo, (int32_t)op); | ||
1894 | lj_trace_err_info(J, LJ_TRERR_NYIBC); | ||
1895 | break; | ||
1896 | } | ||
1897 | |||
1898 | /* rc == 0 if we have no result yet, e.g. pending __index metamethod call. */ | ||
1899 | if (bcmode_a(op) == BCMdst && rc) { | ||
1900 | J->base[ra] = rc; | ||
1901 | if (ra >= J->maxslot) J->maxslot = ra+1; | ||
1902 | } | ||
1903 | |||
1904 | #undef rav | ||
1905 | #undef rbv | ||
1906 | #undef rcv | ||
1907 | |||
1908 | /* Limit the number of recorded IR instructions. */ | ||
1909 | if (J->cur.nins > REF_FIRST+(IRRef)J->param[JIT_P_maxrecord]) | ||
1910 | lj_trace_err(J, LJ_TRERR_TRACEOV); | ||
1911 | } | ||
1912 | |||
1913 | /* -- Recording setup ----------------------------------------------------- */ | ||
1914 | |||
1915 | /* Setup recording for a FORL loop. */ | ||
1916 | static void rec_setup_forl(jit_State *J, const BCIns *fori) | ||
1917 | { | ||
1918 | BCReg ra = bc_a(*fori); | ||
1919 | cTValue *forbase = &J->L->base[ra]; | ||
1920 | IRType t = (J->flags & JIT_F_OPT_NARROW) ? lj_opt_narrow_forl(forbase) | ||
1921 | : IRT_NUM; | ||
1922 | TRef stop = fori_arg(J, fori-2, ra+FORL_STOP, t); | ||
1923 | TRef step = fori_arg(J, fori-1, ra+FORL_STEP, t); | ||
1924 | int dir = (0 <= numV(&forbase[FORL_STEP])); | ||
1925 | lua_assert(bc_op(*fori) == BC_FORI || bc_op(*fori) == BC_JFORI); | ||
1926 | if (!tref_isk(step)) { | ||
1927 | /* Non-constant step: need a guard for the direction. */ | ||
1928 | TRef zero = (t == IRT_INT) ? lj_ir_kint(J, 0) : lj_ir_knum_zero(J); | ||
1929 | emitir(IRTG(dir ? IR_GE : IR_LT, t), step, zero); | ||
1930 | /* Add hoistable overflow checks for a narrowed FORL index. */ | ||
1931 | if (t == IRT_INT) { | ||
1932 | if (tref_isk(stop)) { | ||
1933 | /* Constant stop: optimize check away or to a range check for step. */ | ||
1934 | int32_t k = IR(tref_ref(stop))->i; | ||
1935 | if (dir) { | ||
1936 | if (k > 0) | ||
1937 | emitir(IRTGI(IR_LE), step, lj_ir_kint(J, (int32_t)0x7fffffff-k)); | ||
1938 | } else { | ||
1939 | if (k < 0) | ||
1940 | emitir(IRTGI(IR_GE), step, lj_ir_kint(J, (int32_t)0x80000000-k)); | ||
1941 | } | ||
1942 | } else { | ||
1943 | /* Stop+step variable: need full overflow check (with dead result). */ | ||
1944 | emitir(IRTGI(IR_ADDOV), step, stop); | ||
1945 | } | ||
1946 | } | ||
1947 | } else if (t == IRT_INT && !tref_isk(stop)) { | ||
1948 | /* Constant step: optimize overflow check to a range check for stop. */ | ||
1949 | int32_t k = IR(tref_ref(step))->i; | ||
1950 | k = (int32_t)(dir ? 0x7fffffff : 0x80000000) - k; | ||
1951 | emitir(IRTGI(dir ? IR_LE : IR_GE), stop, lj_ir_kint(J, k)); | ||
1952 | } | ||
1953 | J->base[ra+FORL_EXT] = sloadt(J, (int32_t)(ra+FORL_IDX), t, IRSLOAD_INHERIT); | ||
1954 | J->maxslot = ra+FORL_EXT+1; | ||
1955 | } | ||
1956 | |||
1957 | /* Setup recording for a root trace started by a hot loop. */ | ||
1958 | static const BCIns *rec_setup_root(jit_State *J) | ||
1959 | { | ||
1960 | /* Determine the next PC and the bytecode range for the loop. */ | ||
1961 | const BCIns *pcj, *pc = J->pc; | ||
1962 | BCIns ins = *pc; | ||
1963 | BCReg ra = bc_a(ins); | ||
1964 | switch (bc_op(ins)) { | ||
1965 | case BC_FORL: | ||
1966 | J->bc_extent = (MSize)(-bc_j(ins))*sizeof(BCIns); | ||
1967 | pc += 1+bc_j(ins); | ||
1968 | J->bc_min = pc; | ||
1969 | break; | ||
1970 | case BC_ITERL: | ||
1971 | lua_assert(bc_op(pc[-1]) == BC_ITERC); | ||
1972 | J->maxslot = ra + bc_b(pc[-1]) - 1; | ||
1973 | J->bc_extent = (MSize)(-bc_j(ins))*sizeof(BCIns); | ||
1974 | pc += 1+bc_j(ins); | ||
1975 | lua_assert(bc_op(pc[-1]) == BC_JMP); | ||
1976 | J->bc_min = pc; | ||
1977 | break; | ||
1978 | case BC_LOOP: | ||
1979 | /* Only check BC range for real loops, but not for "repeat until true". */ | ||
1980 | pcj = pc + bc_j(ins); | ||
1981 | ins = *pcj; | ||
1982 | if (bc_op(ins) == BC_JMP && bc_j(ins) < 0) { | ||
1983 | J->bc_min = pcj+1 + bc_j(ins); | ||
1984 | J->bc_extent = (MSize)(-bc_j(ins))*sizeof(BCIns); | ||
1985 | } | ||
1986 | J->maxslot = ra; | ||
1987 | pc++; | ||
1988 | break; | ||
1989 | default: | ||
1990 | lua_assert(0); | ||
1991 | break; | ||
1992 | } | ||
1993 | return pc; | ||
1994 | } | ||
1995 | |||
1996 | /* Setup recording for a side trace. */ | ||
1997 | static void rec_setup_side(jit_State *J, Trace *T) | ||
1998 | { | ||
1999 | SnapShot *snap = &T->snap[J->exitno]; | ||
2000 | IRRef2 *map = &T->snapmap[snap->mapofs]; | ||
2001 | BCReg s, nslots = snap->nslots; | ||
2002 | BloomFilter seen = 0; | ||
2003 | for (s = 0; s < nslots; s++) { | ||
2004 | IRRef ref = snap_ref(map[s]); | ||
2005 | if (ref) { | ||
2006 | IRIns *ir = &T->ir[ref]; | ||
2007 | TRef tr = 0; | ||
2008 | /* The bloom filter avoids O(nslots^2) overhead for de-duping slots. */ | ||
2009 | if (bloomtest(seen, ref)) { | ||
2010 | BCReg j; | ||
2011 | for (j = 0; j < s; j++) | ||
2012 | if (snap_ref(map[j]) == ref) { | ||
2013 | if (ir->o == IR_FRAME && irt_isfunc(ir->t)) | ||
2014 | J->baseslot = s+1; | ||
2015 | tr = J->slot[j]; | ||
2016 | goto dupslot; | ||
2017 | } | ||
2018 | } | ||
2019 | bloomset(seen, ref); | ||
2020 | switch ((IROp)ir->o) { | ||
2021 | case IR_KPRI: tr = TREF_PRI(irt_type(ir->t)); break; | ||
2022 | case IR_KINT: tr = lj_ir_kint(J, ir->i); break; | ||
2023 | case IR_KGC: tr = lj_ir_kgc(J, ir_kgc(ir), irt_t(ir->t)); break; | ||
2024 | case IR_KNUM: tr = lj_ir_knum_addr(J, ir_knum(ir)); break; | ||
2025 | case IR_FRAME: /* Placeholder FRAMEs don't need a guard. */ | ||
2026 | if (irt_isfunc(ir->t)) { | ||
2027 | J->baseslot = s+1; | ||
2028 | J->framedepth++; | ||
2029 | tr = lj_ir_kfunc(J, ir_kfunc(&T->ir[ir->op2])); | ||
2030 | tr = emitir_raw(IRT(IR_FRAME, IRT_FUNC), tr, tr); | ||
2031 | } else { | ||
2032 | tr = lj_ir_kptr(J, mref(T->ir[ir->op2].ptr, void)); | ||
2033 | tr = emitir_raw(IRT(IR_FRAME, IRT_PTR), tr, tr); | ||
2034 | } | ||
2035 | break; | ||
2036 | case IR_SLOAD: /* Inherited SLOADs don't need a guard. */ | ||
2037 | tr = emitir_raw(ir->ot & ~IRT_GUARD, s, | ||
2038 | (ir->op2&IRSLOAD_READONLY) | IRSLOAD_INHERIT|IRSLOAD_PARENT); | ||
2039 | break; | ||
2040 | default: /* Parent refs are already typed and don't need a guard. */ | ||
2041 | tr = emitir_raw(IRT(IR_SLOAD, irt_type(ir->t)), s, | ||
2042 | IRSLOAD_INHERIT|IRSLOAD_PARENT); | ||
2043 | break; | ||
2044 | } | ||
2045 | dupslot: | ||
2046 | J->slot[s] = tr; | ||
2047 | } | ||
2048 | } | ||
2049 | J->base = J->slot + J->baseslot; | ||
2050 | J->maxslot = nslots - J->baseslot; | ||
2051 | lj_snap_add(J); | ||
2052 | } | ||
2053 | |||
2054 | /* Setup for recording a new trace. */ | ||
2055 | void lj_record_setup(jit_State *J) | ||
2056 | { | ||
2057 | uint32_t i; | ||
2058 | |||
2059 | /* Initialize state related to current trace. */ | ||
2060 | memset(J->slot, 0, sizeof(J->slot)); | ||
2061 | memset(J->chain, 0, sizeof(J->chain)); | ||
2062 | memset(J->bpropcache, 0, sizeof(J->bpropcache)); | ||
2063 | |||
2064 | J->baseslot = 1; /* Invoking function is at base[-1]. */ | ||
2065 | J->base = J->slot + J->baseslot; | ||
2066 | J->maxslot = 0; | ||
2067 | J->framedepth = 0; | ||
2068 | |||
2069 | J->instunroll = J->param[JIT_P_instunroll]; | ||
2070 | J->loopunroll = J->param[JIT_P_loopunroll]; | ||
2071 | J->tailcalled = 0; | ||
2072 | J->loopref = 0; | ||
2073 | |||
2074 | J->bc_min = NULL; /* Means no limit. */ | ||
2075 | J->bc_extent = ~(MSize)0; | ||
2076 | |||
2077 | /* Emit instructions for fixed references. Also triggers initial IR alloc. */ | ||
2078 | emitir_raw(IRT(IR_BASE, IRT_PTR), J->parent, J->exitno); | ||
2079 | for (i = 0; i <= 2; i++) { | ||
2080 | IRIns *ir = IR(REF_NIL-i); | ||
2081 | ir->i = 0; | ||
2082 | ir->t.irt = (uint8_t)(IRT_NIL+i); | ||
2083 | ir->o = IR_KPRI; | ||
2084 | ir->prev = 0; | ||
2085 | } | ||
2086 | J->cur.nk = REF_TRUE; | ||
2087 | |||
2088 | setgcref(J->cur.startpt, obj2gco(J->pt)); | ||
2089 | J->startpc = J->pc; | ||
2090 | if (J->parent) { /* Side trace. */ | ||
2091 | Trace *T = J->trace[J->parent]; | ||
2092 | TraceNo root = T->root ? T->root : J->parent; | ||
2093 | J->cur.root = (uint16_t)root; | ||
2094 | J->cur.startins = BCINS_AD(BC_JMP, 0, 0); | ||
2095 | /* Check whether we could at least potentially form an extra loop. */ | ||
2096 | if (J->exitno == 0 && T->snap[0].nslots == 1 && T->snapmap[0] == 0) { | ||
2097 | /* We can narrow a FORL for some side traces, too. */ | ||
2098 | if (J->pc > J->pt->bc && bc_op(J->pc[-1]) == BC_JFORI && | ||
2099 | bc_d(J->pc[bc_j(J->pc[-1])-1]) == root) { | ||
2100 | lj_snap_add(J); | ||
2101 | rec_setup_forl(J, J->pc-1); | ||
2102 | goto sidecheck; | ||
2103 | } | ||
2104 | } else { | ||
2105 | J->startpc = NULL; /* Prevent forming an extra loop. */ | ||
2106 | } | ||
2107 | rec_setup_side(J, T); | ||
2108 | sidecheck: | ||
2109 | if (J->trace[J->cur.root]->nchild >= J->param[JIT_P_maxside] || | ||
2110 | T->snap[J->exitno].count >= J->param[JIT_P_hotexit] + | ||
2111 | J->param[JIT_P_tryside]) | ||
2112 | rec_stop(J, TRACE_INTERP); | ||
2113 | } else { /* Root trace. */ | ||
2114 | J->cur.root = 0; | ||
2115 | if (J->pc >= J->pt->bc) { /* Not a hot CALL? */ | ||
2116 | J->cur.startins = *J->pc; | ||
2117 | J->pc = rec_setup_root(J); | ||
2118 | /* Note: the loop instruction itself is recorded at the end and not | ||
2119 | ** at the start! So snapshot #0 needs to point to the *next* instruction. | ||
2120 | */ | ||
2121 | } else { | ||
2122 | J->cur.startins = BCINS_ABC(BC_CALL, 0, 0, 0); | ||
2123 | } | ||
2124 | lj_snap_add(J); | ||
2125 | if (bc_op(J->cur.startins) == BC_FORL) | ||
2126 | rec_setup_forl(J, J->pc-1); | ||
2127 | if (1 + J->pt->framesize >= LJ_MAX_JSLOTS) | ||
2128 | lj_trace_err(J, LJ_TRERR_STACKOV); | ||
2129 | } | ||
2130 | } | ||
2131 | |||
2132 | #undef IR | ||
2133 | #undef emitir_raw | ||
2134 | #undef emitir | ||
2135 | |||
2136 | #endif | ||