diff options
Diffstat (limited to 'src/lj_opt_loop.c')
| -rw-r--r-- | src/lj_opt_loop.c | 45 |
1 files changed, 31 insertions, 14 deletions
diff --git a/src/lj_opt_loop.c b/src/lj_opt_loop.c index 22b5458c..4cb89c6f 100644 --- a/src/lj_opt_loop.c +++ b/src/lj_opt_loop.c | |||
| @@ -11,7 +11,7 @@ | |||
| 11 | #if LJ_HASJIT | 11 | #if LJ_HASJIT |
| 12 | 12 | ||
| 13 | #include "lj_err.h" | 13 | #include "lj_err.h" |
| 14 | #include "lj_str.h" | 14 | #include "lj_buf.h" |
| 15 | #include "lj_ir.h" | 15 | #include "lj_ir.h" |
| 16 | #include "lj_jit.h" | 16 | #include "lj_jit.h" |
| 17 | #include "lj_iropt.h" | 17 | #include "lj_iropt.h" |
| @@ -225,6 +225,7 @@ static void loop_subst_snap(jit_State *J, SnapShot *osnap, | |||
| 225 | /* Setup new snapshot. */ | 225 | /* Setup new snapshot. */ |
| 226 | snap->mapofs = (uint32_t)nmapofs; | 226 | snap->mapofs = (uint32_t)nmapofs; |
| 227 | snap->ref = (IRRef1)J->cur.nins; | 227 | snap->ref = (IRRef1)J->cur.nins; |
| 228 | snap->mcofs = 0; | ||
| 228 | snap->nslots = nslots; | 229 | snap->nslots = nslots; |
| 229 | snap->topslot = osnap->topslot; | 230 | snap->topslot = osnap->topslot; |
| 230 | snap->count = 0; | 231 | snap->count = 0; |
| @@ -254,9 +255,16 @@ static void loop_subst_snap(jit_State *J, SnapShot *osnap, | |||
| 254 | J->cur.nsnapmap = (uint32_t)(nmap - J->cur.snapmap); | 255 | J->cur.nsnapmap = (uint32_t)(nmap - J->cur.snapmap); |
| 255 | } | 256 | } |
| 256 | 257 | ||
| 258 | typedef struct LoopState { | ||
| 259 | jit_State *J; | ||
| 260 | IRRef1 *subst; | ||
| 261 | MSize sizesubst; | ||
| 262 | } LoopState; | ||
| 263 | |||
| 257 | /* Unroll loop. */ | 264 | /* Unroll loop. */ |
| 258 | static void loop_unroll(jit_State *J) | 265 | static void loop_unroll(LoopState *lps) |
| 259 | { | 266 | { |
| 267 | jit_State *J = lps->J; | ||
| 260 | IRRef1 phi[LJ_MAX_PHI]; | 268 | IRRef1 phi[LJ_MAX_PHI]; |
| 261 | uint32_t nphi = 0; | 269 | uint32_t nphi = 0; |
| 262 | IRRef1 *subst; | 270 | IRRef1 *subst; |
| @@ -265,13 +273,13 @@ static void loop_unroll(jit_State *J) | |||
| 265 | SnapEntry *loopmap, *psentinel; | 273 | SnapEntry *loopmap, *psentinel; |
| 266 | IRRef ins, invar; | 274 | IRRef ins, invar; |
| 267 | 275 | ||
| 268 | /* Use temp buffer for substitution table. | 276 | /* Allocate substitution table. |
| 269 | ** Only non-constant refs in [REF_BIAS,invar) are valid indexes. | 277 | ** Only non-constant refs in [REF_BIAS,invar) are valid indexes. |
| 270 | ** Caveat: don't call into the VM or run the GC or the buffer may be gone. | ||
| 271 | */ | 278 | */ |
| 272 | invar = J->cur.nins; | 279 | invar = J->cur.nins; |
| 273 | subst = (IRRef1 *)lj_str_needbuf(J->L, &G(J->L)->tmpbuf, | 280 | lps->sizesubst = invar - REF_BIAS; |
| 274 | (invar-REF_BIAS)*sizeof(IRRef1)) - REF_BIAS; | 281 | lps->subst = lj_mem_newvec(J->L, lps->sizesubst, IRRef1); |
| 282 | subst = lps->subst - REF_BIAS; | ||
| 275 | subst[REF_BASE] = REF_BASE; | 283 | subst[REF_BASE] = REF_BASE; |
| 276 | 284 | ||
| 277 | /* LOOP separates the pre-roll from the loop body. */ | 285 | /* LOOP separates the pre-roll from the loop body. */ |
| @@ -292,7 +300,8 @@ static void loop_unroll(jit_State *J) | |||
| 292 | loopmap = &J->cur.snapmap[loopsnap->mapofs]; | 300 | loopmap = &J->cur.snapmap[loopsnap->mapofs]; |
| 293 | /* The PC of snapshot #0 and the loop snapshot must match. */ | 301 | /* The PC of snapshot #0 and the loop snapshot must match. */ |
| 294 | psentinel = &loopmap[loopsnap->nent]; | 302 | psentinel = &loopmap[loopsnap->nent]; |
| 295 | lua_assert(*psentinel == J->cur.snapmap[J->cur.snap[0].nent]); | 303 | lj_assertJ(*psentinel == J->cur.snapmap[J->cur.snap[0].nent], |
| 304 | "mismatched PC for loop snapshot"); | ||
| 296 | *psentinel = SNAP(255, 0, 0); /* Replace PC with temporary sentinel. */ | 305 | *psentinel = SNAP(255, 0, 0); /* Replace PC with temporary sentinel. */ |
| 297 | 306 | ||
| 298 | /* Start substitution with snapshot #1 (#0 is empty for root traces). */ | 307 | /* Start substitution with snapshot #1 (#0 is empty for root traces). */ |
| @@ -345,10 +354,12 @@ static void loop_unroll(jit_State *J) | |||
| 345 | irr = IR(ref); | 354 | irr = IR(ref); |
| 346 | goto phiconv; | 355 | goto phiconv; |
| 347 | } | 356 | } |
| 348 | } else if (ref != REF_DROP && irr->o == IR_CONV && | 357 | } else if (ref != REF_DROP && ref > invar && |
| 349 | ref > invar && irr->op1 < invar) { | 358 | ((irr->o == IR_CONV && irr->op1 < invar) || |
| 350 | /* May need an extra PHI for a CONV. */ | 359 | (irr->o == IR_ALEN && irr->op2 < invar && |
| 351 | ref = irr->op1; | 360 | irr->op2 != REF_NIL))) { |
| 361 | /* May need an extra PHI for a CONV or ALEN hint. */ | ||
| 362 | ref = irr->o == IR_CONV ? irr->op1 : irr->op2; | ||
| 352 | irr = IR(ref); | 363 | irr = IR(ref); |
| 353 | phiconv: | 364 | phiconv: |
| 354 | if (ref < invar && !irref_isk(ref) && !irt_isphi(irr->t)) { | 365 | if (ref < invar && !irref_isk(ref) && !irt_isphi(irr->t)) { |
| @@ -363,7 +374,7 @@ static void loop_unroll(jit_State *J) | |||
| 363 | } | 374 | } |
| 364 | if (!irt_isguard(J->guardemit)) /* Drop redundant snapshot. */ | 375 | if (!irt_isguard(J->guardemit)) /* Drop redundant snapshot. */ |
| 365 | J->cur.nsnapmap = (uint32_t)J->cur.snap[--J->cur.nsnap].mapofs; | 376 | J->cur.nsnapmap = (uint32_t)J->cur.snap[--J->cur.nsnap].mapofs; |
| 366 | lua_assert(J->cur.nsnapmap <= J->sizesnapmap); | 377 | lj_assertJ(J->cur.nsnapmap <= J->sizesnapmap, "bad snapshot map index"); |
| 367 | *psentinel = J->cur.snapmap[J->cur.snap[0].nent]; /* Restore PC. */ | 378 | *psentinel = J->cur.snapmap[J->cur.snap[0].nent]; /* Restore PC. */ |
| 368 | 379 | ||
| 369 | loop_emit_phi(J, subst, phi, nphi, onsnap); | 380 | loop_emit_phi(J, subst, phi, nphi, onsnap); |
| @@ -396,7 +407,7 @@ static void loop_undo(jit_State *J, IRRef ins, SnapNo nsnap, MSize nsnapmap) | |||
| 396 | static TValue *cploop_opt(lua_State *L, lua_CFunction dummy, void *ud) | 407 | static TValue *cploop_opt(lua_State *L, lua_CFunction dummy, void *ud) |
| 397 | { | 408 | { |
| 398 | UNUSED(L); UNUSED(dummy); | 409 | UNUSED(L); UNUSED(dummy); |
| 399 | loop_unroll((jit_State *)ud); | 410 | loop_unroll((LoopState *)ud); |
| 400 | return NULL; | 411 | return NULL; |
| 401 | } | 412 | } |
| 402 | 413 | ||
| @@ -406,7 +417,13 @@ int lj_opt_loop(jit_State *J) | |||
| 406 | IRRef nins = J->cur.nins; | 417 | IRRef nins = J->cur.nins; |
| 407 | SnapNo nsnap = J->cur.nsnap; | 418 | SnapNo nsnap = J->cur.nsnap; |
| 408 | MSize nsnapmap = J->cur.nsnapmap; | 419 | MSize nsnapmap = J->cur.nsnapmap; |
| 409 | int errcode = lj_vm_cpcall(J->L, NULL, J, cploop_opt); | 420 | LoopState lps; |
| 421 | int errcode; | ||
| 422 | lps.J = J; | ||
| 423 | lps.subst = NULL; | ||
| 424 | lps.sizesubst = 0; | ||
| 425 | errcode = lj_vm_cpcall(J->L, NULL, &lps, cploop_opt); | ||
| 426 | lj_mem_freevec(J2G(J), lps.subst, lps.sizesubst, IRRef1); | ||
| 410 | if (LJ_UNLIKELY(errcode)) { | 427 | if (LJ_UNLIKELY(errcode)) { |
| 411 | lua_State *L = J->L; | 428 | lua_State *L = J->L; |
| 412 | if (errcode == LUA_ERRRUN && tvisnumber(L->top-1)) { /* Trace error? */ | 429 | if (errcode == LUA_ERRRUN && tvisnumber(L->top-1)) { /* Trace error? */ |
