summaryrefslogtreecommitdiff
path: root/src/lj_trace.c
diff options
context:
space:
mode:
Diffstat (limited to 'src/lj_trace.c')
-rw-r--r--src/lj_trace.c291
1 files changed, 225 insertions, 66 deletions
diff --git a/src/lj_trace.c b/src/lj_trace.c
index a72e73a3..f311d54b 100644
--- a/src/lj_trace.c
+++ b/src/lj_trace.c
@@ -30,6 +30,7 @@
30#include "lj_vm.h" 30#include "lj_vm.h"
31#include "lj_vmevent.h" 31#include "lj_vmevent.h"
32#include "lj_target.h" 32#include "lj_target.h"
33#include "lj_prng.h"
33 34
34/* -- Error handling ------------------------------------------------------ */ 35/* -- Error handling ------------------------------------------------------ */
35 36
@@ -104,7 +105,8 @@ static void perftools_addtrace(GCtrace *T)
104 name++; 105 name++;
105 else 106 else
106 name = "(string)"; 107 name = "(string)";
107 lua_assert(startpc >= proto_bc(pt) && startpc < proto_bc(pt) + pt->sizebc); 108 lj_assertX(startpc >= proto_bc(pt) && startpc < proto_bc(pt) + pt->sizebc,
109 "trace PC out of range");
108 lineno = lj_debug_line(pt, proto_bcpos(pt, startpc)); 110 lineno = lj_debug_line(pt, proto_bcpos(pt, startpc));
109 if (!fp) { 111 if (!fp) {
110 char fname[40]; 112 char fname[40];
@@ -117,15 +119,26 @@ static void perftools_addtrace(GCtrace *T)
117} 119}
118#endif 120#endif
119 121
120/* Allocate space for copy of trace. */ 122/* Allocate space for copy of T. */
121static GCtrace *trace_save_alloc(jit_State *J) 123GCtrace * LJ_FASTCALL lj_trace_alloc(lua_State *L, GCtrace *T)
122{ 124{
123 size_t sztr = ((sizeof(GCtrace)+7)&~7); 125 size_t sztr = ((sizeof(GCtrace)+7)&~7);
124 size_t szins = (J->cur.nins-J->cur.nk)*sizeof(IRIns); 126 size_t szins = (T->nins-T->nk)*sizeof(IRIns);
125 size_t sz = sztr + szins + 127 size_t sz = sztr + szins +
126 J->cur.nsnap*sizeof(SnapShot) + 128 T->nsnap*sizeof(SnapShot) +
127 J->cur.nsnapmap*sizeof(SnapEntry); 129 T->nsnapmap*sizeof(SnapEntry);
128 return lj_mem_newt(J->L, (MSize)sz, GCtrace); 130 GCtrace *T2 = lj_mem_newt(L, (MSize)sz, GCtrace);
131 char *p = (char *)T2 + sztr;
132 T2->gct = ~LJ_TTRACE;
133 T2->marked = 0;
134 T2->traceno = 0;
135 T2->ir = (IRIns *)p - T->nk;
136 T2->nins = T->nins;
137 T2->nk = T->nk;
138 T2->nsnap = T->nsnap;
139 T2->nsnapmap = T->nsnapmap;
140 memcpy(p, T->ir + T->nk, szins);
141 return T2;
129} 142}
130 143
131/* Save current trace by copying and compacting it. */ 144/* Save current trace by copying and compacting it. */
@@ -139,12 +152,15 @@ static void trace_save(jit_State *J, GCtrace *T)
139 setgcrefp(J2G(J)->gc.root, T); 152 setgcrefp(J2G(J)->gc.root, T);
140 newwhite(J2G(J), T); 153 newwhite(J2G(J), T);
141 T->gct = ~LJ_TTRACE; 154 T->gct = ~LJ_TTRACE;
142 T->ir = (IRIns *)p - J->cur.nk; 155 T->ir = (IRIns *)p - J->cur.nk; /* The IR has already been copied above. */
143 memcpy(p, J->cur.ir+J->cur.nk, szins); 156#if LJ_ABI_PAUTH
157 T->mcauth = lj_ptr_sign((ASMFunction)T->mcode, T);
158#endif
144 p += szins; 159 p += szins;
145 TRACE_APPENDVEC(snap, nsnap, SnapShot) 160 TRACE_APPENDVEC(snap, nsnap, SnapShot)
146 TRACE_APPENDVEC(snapmap, nsnapmap, SnapEntry) 161 TRACE_APPENDVEC(snapmap, nsnapmap, SnapEntry)
147 J->cur.traceno = 0; 162 J->cur.traceno = 0;
163 J->curfinal = NULL;
148 setgcrefp(J->trace[T->traceno], T); 164 setgcrefp(J->trace[T->traceno], T);
149 lj_gc_barriertrace(J2G(J), T->traceno); 165 lj_gc_barriertrace(J2G(J), T->traceno);
150 lj_gdbjit_addtrace(J, T); 166 lj_gdbjit_addtrace(J, T);
@@ -172,7 +188,7 @@ void lj_trace_reenableproto(GCproto *pt)
172{ 188{
173 if ((pt->flags & PROTO_ILOOP)) { 189 if ((pt->flags & PROTO_ILOOP)) {
174 BCIns *bc = proto_bc(pt); 190 BCIns *bc = proto_bc(pt);
175 BCPos i, sizebc = pt->sizebc;; 191 BCPos i, sizebc = pt->sizebc;
176 pt->flags &= ~PROTO_ILOOP; 192 pt->flags &= ~PROTO_ILOOP;
177 if (bc_op(bc[0]) == BC_IFUNCF) 193 if (bc_op(bc[0]) == BC_IFUNCF)
178 setbc_op(&bc[0], BC_FUNCF); 194 setbc_op(&bc[0], BC_FUNCF);
@@ -194,27 +210,28 @@ static void trace_unpatch(jit_State *J, GCtrace *T)
194 return; /* No need to unpatch branches in parent traces (yet). */ 210 return; /* No need to unpatch branches in parent traces (yet). */
195 switch (bc_op(*pc)) { 211 switch (bc_op(*pc)) {
196 case BC_JFORL: 212 case BC_JFORL:
197 lua_assert(traceref(J, bc_d(*pc)) == T); 213 lj_assertJ(traceref(J, bc_d(*pc)) == T, "JFORL references other trace");
198 *pc = T->startins; 214 *pc = T->startins;
199 pc += bc_j(T->startins); 215 pc += bc_j(T->startins);
200 lua_assert(bc_op(*pc) == BC_JFORI); 216 lj_assertJ(bc_op(*pc) == BC_JFORI, "FORL does not point to JFORI");
201 setbc_op(pc, BC_FORI); 217 setbc_op(pc, BC_FORI);
202 break; 218 break;
203 case BC_JITERL: 219 case BC_JITERL:
204 case BC_JLOOP: 220 case BC_JLOOP:
205 lua_assert(op == BC_ITERL || op == BC_LOOP || bc_isret(op)); 221 lj_assertJ(op == BC_ITERL || op == BC_ITERN || op == BC_LOOP ||
222 bc_isret(op), "bad original bytecode %d", op);
206 *pc = T->startins; 223 *pc = T->startins;
207 break; 224 break;
208 case BC_JMP: 225 case BC_JMP:
209 lua_assert(op == BC_ITERL); 226 lj_assertJ(op == BC_ITERL, "bad original bytecode %d", op);
210 pc += bc_j(*pc)+2; 227 pc += bc_j(*pc)+2;
211 if (bc_op(*pc) == BC_JITERL) { 228 if (bc_op(*pc) == BC_JITERL) {
212 lua_assert(traceref(J, bc_d(*pc)) == T); 229 lj_assertJ(traceref(J, bc_d(*pc)) == T, "JITERL references other trace");
213 *pc = T->startins; 230 *pc = T->startins;
214 } 231 }
215 break; 232 break;
216 case BC_JFUNCF: 233 case BC_JFUNCF:
217 lua_assert(op == BC_FUNCF); 234 lj_assertJ(op == BC_FUNCF, "bad original bytecode %d", op);
218 *pc = T->startins; 235 *pc = T->startins;
219 break; 236 break;
220 default: /* Already unpatched. */ 237 default: /* Already unpatched. */
@@ -226,7 +243,8 @@ static void trace_unpatch(jit_State *J, GCtrace *T)
226static void trace_flushroot(jit_State *J, GCtrace *T) 243static void trace_flushroot(jit_State *J, GCtrace *T)
227{ 244{
228 GCproto *pt = &gcref(T->startpt)->pt; 245 GCproto *pt = &gcref(T->startpt)->pt;
229 lua_assert(T->root == 0 && pt != NULL); 246 lj_assertJ(T->root == 0, "not a root trace");
247 lj_assertJ(pt != NULL, "trace has no prototype");
230 /* First unpatch any modified bytecode. */ 248 /* First unpatch any modified bytecode. */
231 trace_unpatch(J, T); 249 trace_unpatch(J, T);
232 /* Unlink root trace from chain anchored in prototype. */ 250 /* Unlink root trace from chain anchored in prototype. */
@@ -274,7 +292,7 @@ int lj_trace_flushall(lua_State *L)
274 if (T->root == 0) 292 if (T->root == 0)
275 trace_flushroot(J, T); 293 trace_flushroot(J, T);
276 lj_gdbjit_deltrace(J, T); 294 lj_gdbjit_deltrace(J, T);
277 T->traceno = 0; 295 T->traceno = T->link = 0; /* Blacklist the link for cont_stitch. */
278 setgcrefnull(J->trace[i]); 296 setgcrefnull(J->trace[i]);
279 } 297 }
280 } 298 }
@@ -296,13 +314,42 @@ void lj_trace_initstate(global_State *g)
296{ 314{
297 jit_State *J = G2J(g); 315 jit_State *J = G2J(g);
298 TValue *tv; 316 TValue *tv;
299 /* Initialize SIMD constants. */ 317
318 /* Initialize aligned SIMD constants. */
300 tv = LJ_KSIMD(J, LJ_KSIMD_ABS); 319 tv = LJ_KSIMD(J, LJ_KSIMD_ABS);
301 tv[0].u64 = U64x(7fffffff,ffffffff); 320 tv[0].u64 = U64x(7fffffff,ffffffff);
302 tv[1].u64 = U64x(7fffffff,ffffffff); 321 tv[1].u64 = U64x(7fffffff,ffffffff);
303 tv = LJ_KSIMD(J, LJ_KSIMD_NEG); 322 tv = LJ_KSIMD(J, LJ_KSIMD_NEG);
304 tv[0].u64 = U64x(80000000,00000000); 323 tv[0].u64 = U64x(80000000,00000000);
305 tv[1].u64 = U64x(80000000,00000000); 324 tv[1].u64 = U64x(80000000,00000000);
325
326 /* Initialize 32/64 bit constants. */
327#if LJ_TARGET_X86ORX64
328 J->k64[LJ_K64_TOBIT].u64 = U64x(43380000,00000000);
329#if LJ_32
330 J->k64[LJ_K64_M2P64_31].u64 = U64x(c1e00000,00000000);
331#endif
332 J->k64[LJ_K64_2P64].u64 = U64x(43f00000,00000000);
333 J->k32[LJ_K32_M2P64_31] = LJ_64 ? 0xdf800000 : 0xcf000000;
334#endif
335#if LJ_TARGET_X86ORX64 || LJ_TARGET_MIPS64
336 J->k64[LJ_K64_M2P64].u64 = U64x(c3f00000,00000000);
337#endif
338#if LJ_TARGET_PPC
339 J->k32[LJ_K32_2P52_2P31] = 0x59800004;
340 J->k32[LJ_K32_2P52] = 0x59800000;
341#endif
342#if LJ_TARGET_PPC || LJ_TARGET_MIPS
343 J->k32[LJ_K32_2P31] = 0x4f000000;
344#endif
345#if LJ_TARGET_MIPS
346 J->k64[LJ_K64_2P31].u64 = U64x(41e00000,00000000);
347#if LJ_64
348 J->k64[LJ_K64_2P63].u64 = U64x(43e00000,00000000);
349 J->k32[LJ_K32_2P63] = 0x5f000000;
350 J->k32[LJ_K32_M2P64] = 0xdf800000;
351#endif
352#endif
306} 353}
307 354
308/* Free everything associated with the JIT compiler state. */ 355/* Free everything associated with the JIT compiler state. */
@@ -313,11 +360,11 @@ void lj_trace_freestate(global_State *g)
313 { /* This assumes all traces have already been freed. */ 360 { /* This assumes all traces have already been freed. */
314 ptrdiff_t i; 361 ptrdiff_t i;
315 for (i = 1; i < (ptrdiff_t)J->sizetrace; i++) 362 for (i = 1; i < (ptrdiff_t)J->sizetrace; i++)
316 lua_assert(i == (ptrdiff_t)J->cur.traceno || traceref(J, i) == NULL); 363 lj_assertG(i == (ptrdiff_t)J->cur.traceno || traceref(J, i) == NULL,
364 "trace still allocated");
317 } 365 }
318#endif 366#endif
319 lj_mcode_free(J); 367 lj_mcode_free(J);
320 lj_ir_k64_freeall(J);
321 lj_mem_freevec(g, J->snapmapbuf, J->sizesnapmap, SnapEntry); 368 lj_mem_freevec(g, J->snapmapbuf, J->sizesnapmap, SnapEntry);
322 lj_mem_freevec(g, J->snapbuf, J->sizesnap, SnapShot); 369 lj_mem_freevec(g, J->snapbuf, J->sizesnap, SnapShot);
323 lj_mem_freevec(g, J->irbuf + J->irbotlim, J->irtoplim - J->irbotlim, IRIns); 370 lj_mem_freevec(g, J->irbuf + J->irbotlim, J->irtoplim - J->irbotlim, IRIns);
@@ -329,8 +376,13 @@ void lj_trace_freestate(global_State *g)
329/* Blacklist a bytecode instruction. */ 376/* Blacklist a bytecode instruction. */
330static void blacklist_pc(GCproto *pt, BCIns *pc) 377static void blacklist_pc(GCproto *pt, BCIns *pc)
331{ 378{
332 setbc_op(pc, (int)bc_op(*pc)+(int)BC_ILOOP-(int)BC_LOOP); 379 if (bc_op(*pc) == BC_ITERN) {
333 pt->flags |= PROTO_ILOOP; 380 setbc_op(pc, BC_ITERC);
381 setbc_op(pc+1+bc_j(pc[1]), BC_JMP);
382 } else {
383 setbc_op(pc, (int)bc_op(*pc)+(int)BC_ILOOP-(int)BC_LOOP);
384 pt->flags |= PROTO_ILOOP;
385 }
334} 386}
335 387
336/* Penalize a bytecode instruction. */ 388/* Penalize a bytecode instruction. */
@@ -341,7 +393,7 @@ static void penalty_pc(jit_State *J, GCproto *pt, BCIns *pc, TraceError e)
341 if (mref(J->penalty[i].pc, const BCIns) == pc) { /* Cache slot found? */ 393 if (mref(J->penalty[i].pc, const BCIns) == pc) { /* Cache slot found? */
342 /* First try to bump its hotcount several times. */ 394 /* First try to bump its hotcount several times. */
343 val = ((uint32_t)J->penalty[i].val << 1) + 395 val = ((uint32_t)J->penalty[i].val << 1) +
344 LJ_PRNG_BITS(J, PENALTY_RNDBITS); 396 (lj_prng_u64(&J2G(J)->prng) & ((1u<<PENALTY_RNDBITS)-1));
345 if (val > PENALTY_MAX) { 397 if (val > PENALTY_MAX) {
346 blacklist_pc(pt, pc); /* Blacklist it, if that didn't help. */ 398 blacklist_pc(pt, pc); /* Blacklist it, if that didn't help. */
347 return; 399 return;
@@ -367,10 +419,11 @@ static void trace_start(jit_State *J)
367 TraceNo traceno; 419 TraceNo traceno;
368 420
369 if ((J->pt->flags & PROTO_NOJIT)) { /* JIT disabled for this proto? */ 421 if ((J->pt->flags & PROTO_NOJIT)) { /* JIT disabled for this proto? */
370 if (J->parent == 0) { 422 if (J->parent == 0 && J->exitno == 0 && bc_op(*J->pc) != BC_ITERN) {
371 /* Lazy bytecode patching to disable hotcount events. */ 423 /* Lazy bytecode patching to disable hotcount events. */
372 lua_assert(bc_op(*J->pc) == BC_FORL || bc_op(*J->pc) == BC_ITERL || 424 lj_assertJ(bc_op(*J->pc) == BC_FORL || bc_op(*J->pc) == BC_ITERL ||
373 bc_op(*J->pc) == BC_LOOP || bc_op(*J->pc) == BC_FUNCF); 425 bc_op(*J->pc) == BC_LOOP || bc_op(*J->pc) == BC_FUNCF,
426 "bad hot bytecode %d", bc_op(*J->pc));
374 setbc_op(J->pc, (int)bc_op(*J->pc)+(int)BC_ILOOP-(int)BC_LOOP); 427 setbc_op(J->pc, (int)bc_op(*J->pc)+(int)BC_ILOOP-(int)BC_LOOP);
375 J->pt->flags |= PROTO_ILOOP; 428 J->pt->flags |= PROTO_ILOOP;
376 } 429 }
@@ -378,10 +431,17 @@ static void trace_start(jit_State *J)
378 return; 431 return;
379 } 432 }
380 433
434 /* Ensuring forward progress for BC_ITERN can trigger hotcount again. */
435 if (!J->parent && bc_op(*J->pc) == BC_JLOOP) { /* Already compiled. */
436 J->state = LJ_TRACE_IDLE; /* Silently ignored. */
437 return;
438 }
439
381 /* Get a new trace number. */ 440 /* Get a new trace number. */
382 traceno = trace_findfree(J); 441 traceno = trace_findfree(J);
383 if (LJ_UNLIKELY(traceno == 0)) { /* No free trace? */ 442 if (LJ_UNLIKELY(traceno == 0)) { /* No free trace? */
384 lua_assert((J2G(J)->hookmask & HOOK_GC) == 0); 443 lj_assertJ((J2G(J)->hookmask & HOOK_GC) == 0,
444 "recorder called from GC hook");
385 lj_trace_flushall(J->L); 445 lj_trace_flushall(J->L);
386 J->state = LJ_TRACE_IDLE; /* Silently ignored. */ 446 J->state = LJ_TRACE_IDLE; /* Silently ignored. */
387 return; 447 return;
@@ -401,6 +461,8 @@ static void trace_start(jit_State *J)
401 J->guardemit.irt = 0; 461 J->guardemit.irt = 0;
402 J->postproc = LJ_POST_NONE; 462 J->postproc = LJ_POST_NONE;
403 lj_resetsplit(J); 463 lj_resetsplit(J);
464 J->retryrec = 0;
465 J->ktrace = 0;
404 setgcref(J->cur.startpt, obj2gco(J->pt)); 466 setgcref(J->cur.startpt, obj2gco(J->pt));
405 467
406 L = J->L; 468 L = J->L;
@@ -412,6 +474,12 @@ static void trace_start(jit_State *J)
412 if (J->parent) { 474 if (J->parent) {
413 setintV(L->top++, J->parent); 475 setintV(L->top++, J->parent);
414 setintV(L->top++, J->exitno); 476 setintV(L->top++, J->exitno);
477 } else {
478 BCOp op = bc_op(*J->pc);
479 if (op == BC_CALLM || op == BC_CALL || op == BC_ITERC) {
480 setintV(L->top++, J->exitno); /* Parent of stitched trace. */
481 setintV(L->top++, -1);
482 }
415 } 483 }
416 ); 484 );
417 lj_record_setup(J); 485 lj_record_setup(J);
@@ -424,7 +492,7 @@ static void trace_stop(jit_State *J)
424 BCOp op = bc_op(J->cur.startins); 492 BCOp op = bc_op(J->cur.startins);
425 GCproto *pt = &gcref(J->cur.startpt)->pt; 493 GCproto *pt = &gcref(J->cur.startpt)->pt;
426 TraceNo traceno = J->cur.traceno; 494 TraceNo traceno = J->cur.traceno;
427 GCtrace *T = trace_save_alloc(J); /* Do this first. May throw OOM. */ 495 GCtrace *T = J->curfinal;
428 lua_State *L; 496 lua_State *L;
429 497
430 switch (op) { 498 switch (op) {
@@ -442,6 +510,7 @@ static void trace_stop(jit_State *J)
442 J->cur.nextroot = pt->trace; 510 J->cur.nextroot = pt->trace;
443 pt->trace = (TraceNo1)traceno; 511 pt->trace = (TraceNo1)traceno;
444 break; 512 break;
513 case BC_ITERN:
445 case BC_RET: 514 case BC_RET:
446 case BC_RET0: 515 case BC_RET0:
447 case BC_RET1: 516 case BC_RET1:
@@ -449,7 +518,7 @@ static void trace_stop(jit_State *J)
449 goto addroot; 518 goto addroot;
450 case BC_JMP: 519 case BC_JMP:
451 /* Patch exit branch in parent to side trace entry. */ 520 /* Patch exit branch in parent to side trace entry. */
452 lua_assert(J->parent != 0 && J->cur.root != 0); 521 lj_assertJ(J->parent != 0 && J->cur.root != 0, "not a side trace");
453 lj_asm_patchexit(J, traceref(J, J->parent), J->exitno, J->cur.mcode); 522 lj_asm_patchexit(J, traceref(J, J->parent), J->exitno, J->cur.mcode);
454 /* Avoid compiling a side trace twice (stack resizing uses parent exit). */ 523 /* Avoid compiling a side trace twice (stack resizing uses parent exit). */
455 { 524 {
@@ -465,8 +534,14 @@ static void trace_stop(jit_State *J)
465 root->nextside = (TraceNo1)traceno; 534 root->nextside = (TraceNo1)traceno;
466 } 535 }
467 break; 536 break;
537 case BC_CALLM:
538 case BC_CALL:
539 case BC_ITERC:
540 /* Trace stitching: patch link of previous trace. */
541 traceref(J, J->exitno)->link = traceno;
542 break;
468 default: 543 default:
469 lua_assert(0); 544 lj_assertJ(0, "bad stop bytecode %d", op);
470 break; 545 break;
471 } 546 }
472 547
@@ -479,6 +554,7 @@ static void trace_stop(jit_State *J)
479 lj_vmevent_send(L, TRACE, 554 lj_vmevent_send(L, TRACE,
480 setstrV(L, L->top++, lj_str_newlit(L, "stop")); 555 setstrV(L, L->top++, lj_str_newlit(L, "stop"));
481 setintV(L->top++, traceno); 556 setintV(L->top++, traceno);
557 setfuncV(L, L->top++, J->fn);
482 ); 558 );
483} 559}
484 560
@@ -486,8 +562,8 @@ static void trace_stop(jit_State *J)
486static int trace_downrec(jit_State *J) 562static int trace_downrec(jit_State *J)
487{ 563{
488 /* Restart recording at the return instruction. */ 564 /* Restart recording at the return instruction. */
489 lua_assert(J->pt != NULL); 565 lj_assertJ(J->pt != NULL, "no active prototype");
490 lua_assert(bc_isret(bc_op(*J->pc))); 566 lj_assertJ(bc_isret(bc_op(*J->pc)), "not at a return bytecode");
491 if (bc_op(*J->pc) == BC_RETM) 567 if (bc_op(*J->pc) == BC_RETM)
492 return 0; /* NYI: down-recursion with RETM. */ 568 return 0; /* NYI: down-recursion with RETM. */
493 J->parent = 0; 569 J->parent = 0;
@@ -506,6 +582,10 @@ static int trace_abort(jit_State *J)
506 582
507 J->postproc = LJ_POST_NONE; 583 J->postproc = LJ_POST_NONE;
508 lj_mcode_abort(J); 584 lj_mcode_abort(J);
585 if (J->curfinal) {
586 lj_trace_free(J2G(J), J->curfinal);
587 J->curfinal = NULL;
588 }
509 if (tvisnumber(L->top-1)) 589 if (tvisnumber(L->top-1))
510 e = (TraceError)numberVint(L->top-1); 590 e = (TraceError)numberVint(L->top-1);
511 if (e == LJ_TRERR_MCODELM) { 591 if (e == LJ_TRERR_MCODELM) {
@@ -514,8 +594,17 @@ static int trace_abort(jit_State *J)
514 return 1; /* Retry ASM with new MCode area. */ 594 return 1; /* Retry ASM with new MCode area. */
515 } 595 }
516 /* Penalize or blacklist starting bytecode instruction. */ 596 /* Penalize or blacklist starting bytecode instruction. */
517 if (J->parent == 0 && !bc_isret(bc_op(J->cur.startins))) 597 if (J->parent == 0 && !bc_isret(bc_op(J->cur.startins))) {
518 penalty_pc(J, &gcref(J->cur.startpt)->pt, mref(J->cur.startpc, BCIns), e); 598 if (J->exitno == 0) {
599 BCIns *startpc = mref(J->cur.startpc, BCIns);
600 if (e == LJ_TRERR_RETRY)
601 hotcount_set(J2GG(J), startpc+1, 1); /* Immediate retry. */
602 else
603 penalty_pc(J, &gcref(J->cur.startpt)->pt, startpc, e);
604 } else {
605 traceref(J, J->exitno)->link = J->exitno; /* Self-link is blacklisted. */
606 }
607 }
519 608
520 /* Is there anything to abort? */ 609 /* Is there anything to abort? */
521 traceno = J->cur.traceno; 610 traceno = J->cur.traceno;
@@ -581,8 +670,13 @@ static TValue *trace_state(lua_State *L, lua_CFunction dummy, void *ud)
581 J->state = LJ_TRACE_RECORD; /* trace_start() may change state. */ 670 J->state = LJ_TRACE_RECORD; /* trace_start() may change state. */
582 trace_start(J); 671 trace_start(J);
583 lj_dispatch_update(J2G(J)); 672 lj_dispatch_update(J2G(J));
584 break; 673 if (J->state != LJ_TRACE_RECORD_1ST)
674 break;
675 /* fallthrough */
585 676
677 case LJ_TRACE_RECORD_1ST:
678 J->state = LJ_TRACE_RECORD;
679 /* fallthrough */
586 case LJ_TRACE_RECORD: 680 case LJ_TRACE_RECORD:
587 trace_pendpatch(J, 0); 681 trace_pendpatch(J, 0);
588 setvmstate(J2G(J), RECORD); 682 setvmstate(J2G(J), RECORD);
@@ -688,15 +782,30 @@ static void trace_hotside(jit_State *J, const BCIns *pc)
688{ 782{
689 SnapShot *snap = &traceref(J, J->parent)->snap[J->exitno]; 783 SnapShot *snap = &traceref(J, J->parent)->snap[J->exitno];
690 if (!(J2G(J)->hookmask & (HOOK_GC|HOOK_VMEVENT)) && 784 if (!(J2G(J)->hookmask & (HOOK_GC|HOOK_VMEVENT)) &&
785 isluafunc(curr_func(J->L)) &&
691 snap->count != SNAPCOUNT_DONE && 786 snap->count != SNAPCOUNT_DONE &&
692 ++snap->count >= J->param[JIT_P_hotexit]) { 787 ++snap->count >= J->param[JIT_P_hotexit]) {
693 lua_assert(J->state == LJ_TRACE_IDLE); 788 lj_assertJ(J->state == LJ_TRACE_IDLE, "hot side exit while recording");
694 /* J->parent is non-zero for a side trace. */ 789 /* J->parent is non-zero for a side trace. */
695 J->state = LJ_TRACE_START; 790 J->state = LJ_TRACE_START;
696 lj_trace_ins(J, pc); 791 lj_trace_ins(J, pc);
697 } 792 }
698} 793}
699 794
795/* Stitch a new trace to the previous trace. */
796void LJ_FASTCALL lj_trace_stitch(jit_State *J, const BCIns *pc)
797{
798 /* Only start a new trace if not recording or inside __gc call or vmevent. */
799 if (J->state == LJ_TRACE_IDLE &&
800 !(J2G(J)->hookmask & (HOOK_GC|HOOK_VMEVENT))) {
801 J->parent = 0; /* Have to treat it like a root trace. */
802 /* J->exitno is set to the invoking trace. */
803 J->state = LJ_TRACE_START;
804 lj_trace_ins(J, pc);
805 }
806}
807
808
700/* Tiny struct to pass data to protected call. */ 809/* Tiny struct to pass data to protected call. */
701typedef struct ExitDataCP { 810typedef struct ExitDataCP {
702 jit_State *J; 811 jit_State *J;
@@ -740,7 +849,7 @@ static void trace_exit_regs(lua_State *L, ExitState *ex)
740} 849}
741#endif 850#endif
742 851
743#ifdef EXITSTATE_PCREG 852#if defined(EXITSTATE_PCREG) || (LJ_UNWIND_JIT && !EXITTRACE_VMSTATE)
744/* Determine trace number from pc of exit instruction. */ 853/* Determine trace number from pc of exit instruction. */
745static TraceNo trace_exit_find(jit_State *J, MCode *pc) 854static TraceNo trace_exit_find(jit_State *J, MCode *pc)
746{ 855{
@@ -750,7 +859,7 @@ static TraceNo trace_exit_find(jit_State *J, MCode *pc)
750 if (T && pc >= T->mcode && pc < (MCode *)((char *)T->mcode + T->szmcode)) 859 if (T && pc >= T->mcode && pc < (MCode *)((char *)T->mcode + T->szmcode))
751 return traceno; 860 return traceno;
752 } 861 }
753 lua_assert(0); 862 lj_assertJ(0, "bad exit pc");
754 return 0; 863 return 0;
755} 864}
756#endif 865#endif
@@ -762,68 +871,81 @@ int LJ_FASTCALL lj_trace_exit(jit_State *J, void *exptr)
762 lua_State *L = J->L; 871 lua_State *L = J->L;
763 ExitState *ex = (ExitState *)exptr; 872 ExitState *ex = (ExitState *)exptr;
764 ExitDataCP exd; 873 ExitDataCP exd;
765 int errcode; 874 int errcode, exitcode = J->exitcode;
766 const BCIns *pc; 875 TValue exiterr;
876 const BCIns *pc, *retpc;
767 void *cf; 877 void *cf;
768 GCtrace *T; 878 GCtrace *T;
879
880 setnilV(&exiterr);
881 if (exitcode) { /* Trace unwound with error code. */
882 J->exitcode = 0;
883 copyTV(L, &exiterr, L->top-1);
884 }
885
769#ifdef EXITSTATE_PCREG 886#ifdef EXITSTATE_PCREG
770 J->parent = trace_exit_find(J, (MCode *)(intptr_t)ex->gpr[EXITSTATE_PCREG]); 887 J->parent = trace_exit_find(J, (MCode *)(intptr_t)ex->gpr[EXITSTATE_PCREG]);
771#endif 888#endif
772 T = traceref(J, J->parent); UNUSED(T); 889 T = traceref(J, J->parent); UNUSED(T);
773#ifdef EXITSTATE_CHECKEXIT 890#ifdef EXITSTATE_CHECKEXIT
774 if (J->exitno == T->nsnap) { /* Treat stack check like a parent exit. */ 891 if (J->exitno == T->nsnap) { /* Treat stack check like a parent exit. */
775 lua_assert(T->root != 0); 892 lj_assertJ(T->root != 0, "stack check in root trace");
776 J->exitno = T->ir[REF_BASE].op2; 893 J->exitno = T->ir[REF_BASE].op2;
777 J->parent = T->ir[REF_BASE].op1; 894 J->parent = T->ir[REF_BASE].op1;
778 T = traceref(J, J->parent); 895 T = traceref(J, J->parent);
779 } 896 }
780#endif 897#endif
781 lua_assert(T != NULL && J->exitno < T->nsnap); 898 lj_assertJ(T != NULL && J->exitno < T->nsnap, "bad trace or exit number");
782 exd.J = J; 899 exd.J = J;
783 exd.exptr = exptr; 900 exd.exptr = exptr;
784 errcode = lj_vm_cpcall(L, NULL, &exd, trace_exit_cp); 901 errcode = lj_vm_cpcall(L, NULL, &exd, trace_exit_cp);
785 if (errcode) 902 if (errcode)
786 return -errcode; /* Return negated error code. */ 903 return -errcode; /* Return negated error code. */
787 904
788 lj_vmevent_send(L, TEXIT, 905 if (exitcode) copyTV(L, L->top++, &exiterr); /* Anchor the error object. */
789 lj_state_checkstack(L, 4+RID_NUM_GPR+RID_NUM_FPR+LUA_MINSTACK); 906
790 setintV(L->top++, J->parent); 907 if (!(LJ_HASPROFILE && (G(L)->hookmask & HOOK_PROFILE)))
791 setintV(L->top++, J->exitno); 908 lj_vmevent_send(L, TEXIT,
792 trace_exit_regs(L, ex); 909 lj_state_checkstack(L, 4+RID_NUM_GPR+RID_NUM_FPR+LUA_MINSTACK);
793 ); 910 setintV(L->top++, J->parent);
911 setintV(L->top++, J->exitno);
912 trace_exit_regs(L, ex);
913 );
794 914
795 pc = exd.pc; 915 pc = exd.pc;
796 cf = cframe_raw(L->cframe); 916 cf = cframe_raw(L->cframe);
797 setcframe_pc(cf, pc); 917 setcframe_pc(cf, pc);
798 if (G(L)->gc.state == GCSatomic || G(L)->gc.state == GCSfinalize) { 918 if (exitcode) {
919 return -exitcode;
920 } else if (LJ_HASPROFILE && (G(L)->hookmask & HOOK_PROFILE)) {
921 /* Just exit to interpreter. */
922 } else if (G(L)->gc.state == GCSatomic || G(L)->gc.state == GCSfinalize) {
799 if (!(G(L)->hookmask & HOOK_GC)) 923 if (!(G(L)->hookmask & HOOK_GC))
800 lj_gc_step(L); /* Exited because of GC: drive GC forward. */ 924 lj_gc_step(L); /* Exited because of GC: drive GC forward. */
801 } else { 925 } else {
802 trace_hotside(J, pc); 926 trace_hotside(J, pc);
803 } 927 }
804 if (bc_op(*pc) == BC_JLOOP) { 928 /* Return MULTRES or 0 or -17. */
805 BCIns *retpc = &traceref(J, bc_d(*pc))->startins;
806 if (bc_isret(bc_op(*retpc))) {
807 if (J->state == LJ_TRACE_RECORD) {
808 J->patchins = *pc;
809 J->patchpc = (BCIns *)pc;
810 *J->patchpc = *retpc;
811 J->bcskip = 1;
812 } else {
813 pc = retpc;
814 setcframe_pc(cf, pc);
815 }
816 }
817 }
818 /* Return MULTRES or 0. */
819 ERRNO_RESTORE 929 ERRNO_RESTORE
820 switch (bc_op(*pc)) { 930 switch (bc_op(*pc)) {
821 case BC_CALLM: case BC_CALLMT: 931 case BC_CALLM: case BC_CALLMT:
822 return (int)((BCReg)(L->top - L->base) - bc_a(*pc) - bc_c(*pc)); 932 return (int)((BCReg)(L->top - L->base) - bc_a(*pc) - bc_c(*pc) - LJ_FR2);
823 case BC_RETM: 933 case BC_RETM:
824 return (int)((BCReg)(L->top - L->base) + 1 - bc_a(*pc) - bc_d(*pc)); 934 return (int)((BCReg)(L->top - L->base) + 1 - bc_a(*pc) - bc_d(*pc));
825 case BC_TSETM: 935 case BC_TSETM:
826 return (int)((BCReg)(L->top - L->base) + 1 - bc_a(*pc)); 936 return (int)((BCReg)(L->top - L->base) + 1 - bc_a(*pc));
937 case BC_JLOOP:
938 retpc = &traceref(J, bc_d(*pc))->startins;
939 if (bc_isret(bc_op(*retpc)) || bc_op(*retpc) == BC_ITERN) {
940 /* Dispatch to original ins to ensure forward progress. */
941 if (J->state != LJ_TRACE_RECORD) return -17;
942 /* Unpatch bytecode when recording. */
943 J->patchins = *pc;
944 J->patchpc = (BCIns *)pc;
945 *J->patchpc = *retpc;
946 J->bcskip = 1;
947 }
948 return 0;
827 default: 949 default:
828 if (bc_op(*pc) >= BC_FUNCF) 950 if (bc_op(*pc) >= BC_FUNCF)
829 return (int)((BCReg)(L->top - L->base) + 1); 951 return (int)((BCReg)(L->top - L->base) + 1);
@@ -831,4 +953,41 @@ int LJ_FASTCALL lj_trace_exit(jit_State *J, void *exptr)
831 } 953 }
832} 954}
833 955
956#if LJ_UNWIND_JIT
957/* Given an mcode address determine trace exit address for unwinding. */
958uintptr_t LJ_FASTCALL lj_trace_unwind(jit_State *J, uintptr_t addr, ExitNo *ep)
959{
960#if EXITTRACE_VMSTATE
961 TraceNo traceno = J2G(J)->vmstate;
962#else
963 TraceNo traceno = trace_exit_find(J, (MCode *)addr);
964#endif
965 GCtrace *T = traceref(J, traceno);
966 if (T
967#if EXITTRACE_VMSTATE
968 && addr >= (uintptr_t)T->mcode && addr < (uintptr_t)T->mcode + T->szmcode
969#endif
970 ) {
971 SnapShot *snap = T->snap;
972 SnapNo lo = 0, exitno = T->nsnap;
973 uintptr_t ofs = (uintptr_t)((MCode *)addr - T->mcode); /* MCode units! */
974 /* Rightmost binary search for mcode offset to determine exit number. */
975 do {
976 SnapNo mid = (lo+exitno) >> 1;
977 if (ofs < snap[mid].mcofs) exitno = mid; else lo = mid + 1;
978 } while (lo < exitno);
979 exitno--;
980 *ep = exitno;
981#ifdef EXITSTUBS_PER_GROUP
982 return (uintptr_t)exitstub_addr(J, exitno);
983#else
984 return (uintptr_t)exitstub_trace_addr(T, exitno);
985#endif
986 }
987 /* Cannot correlate addr with trace/exit. This will be fatal. */
988 lj_assertJ(0, "bad exit pc");
989 return 0;
990}
991#endif
992
834#endif 993#endif