diff options
Diffstat (limited to 'src/lj_asm_x86.h')
-rw-r--r-- | src/lj_asm_x86.h | 161 |
1 files changed, 93 insertions, 68 deletions
diff --git a/src/lj_asm_x86.h b/src/lj_asm_x86.h index 7356a5f0..a3adee14 100644 --- a/src/lj_asm_x86.h +++ b/src/lj_asm_x86.h | |||
@@ -31,7 +31,7 @@ static MCode *asm_exitstub_gen(ASMState *as, ExitNo group) | |||
31 | #endif | 31 | #endif |
32 | /* Jump to exit handler which fills in the ExitState. */ | 32 | /* Jump to exit handler which fills in the ExitState. */ |
33 | *mxp++ = XI_JMP; mxp += 4; | 33 | *mxp++ = XI_JMP; mxp += 4; |
34 | *((int32_t *)(mxp-4)) = jmprel(mxp, (MCode *)(void *)lj_vm_exit_handler); | 34 | *((int32_t *)(mxp-4)) = jmprel(as->J, mxp, (MCode *)(void *)lj_vm_exit_handler); |
35 | /* Commit the code for this group (even if assembly fails later on). */ | 35 | /* Commit the code for this group (even if assembly fails later on). */ |
36 | lj_mcode_commitbot(as->J, mxp); | 36 | lj_mcode_commitbot(as->J, mxp); |
37 | as->mcbot = mxp; | 37 | as->mcbot = mxp; |
@@ -60,7 +60,7 @@ static void asm_guardcc(ASMState *as, int cc) | |||
60 | MCode *p = as->mcp; | 60 | MCode *p = as->mcp; |
61 | if (LJ_UNLIKELY(p == as->invmcp)) { | 61 | if (LJ_UNLIKELY(p == as->invmcp)) { |
62 | as->loopinv = 1; | 62 | as->loopinv = 1; |
63 | *(int32_t *)(p+1) = jmprel(p+5, target); | 63 | *(int32_t *)(p+1) = jmprel(as->J, p+5, target); |
64 | target = p; | 64 | target = p; |
65 | cc ^= 1; | 65 | cc ^= 1; |
66 | if (as->realign) { | 66 | if (as->realign) { |
@@ -131,7 +131,7 @@ static IRRef asm_fuseabase(ASMState *as, IRRef ref) | |||
131 | as->mrm.ofs = 0; | 131 | as->mrm.ofs = 0; |
132 | if (irb->o == IR_FLOAD) { | 132 | if (irb->o == IR_FLOAD) { |
133 | IRIns *ira = IR(irb->op1); | 133 | IRIns *ira = IR(irb->op1); |
134 | lua_assert(irb->op2 == IRFL_TAB_ARRAY); | 134 | lj_assertA(irb->op2 == IRFL_TAB_ARRAY, "expected FLOAD TAB_ARRAY"); |
135 | /* We can avoid the FLOAD of t->array for colocated arrays. */ | 135 | /* We can avoid the FLOAD of t->array for colocated arrays. */ |
136 | if (ira->o == IR_TNEW && ira->op1 <= LJ_MAX_COLOSIZE && | 136 | if (ira->o == IR_TNEW && ira->op1 <= LJ_MAX_COLOSIZE && |
137 | !neverfuse(as) && noconflict(as, irb->op1, IR_NEWREF, 1)) { | 137 | !neverfuse(as) && noconflict(as, irb->op1, IR_NEWREF, 1)) { |
@@ -150,7 +150,7 @@ static IRRef asm_fuseabase(ASMState *as, IRRef ref) | |||
150 | static void asm_fusearef(ASMState *as, IRIns *ir, RegSet allow) | 150 | static void asm_fusearef(ASMState *as, IRIns *ir, RegSet allow) |
151 | { | 151 | { |
152 | IRIns *irx; | 152 | IRIns *irx; |
153 | lua_assert(ir->o == IR_AREF); | 153 | lj_assertA(ir->o == IR_AREF, "expected AREF"); |
154 | as->mrm.base = (uint8_t)ra_alloc1(as, asm_fuseabase(as, ir->op1), allow); | 154 | as->mrm.base = (uint8_t)ra_alloc1(as, asm_fuseabase(as, ir->op1), allow); |
155 | irx = IR(ir->op2); | 155 | irx = IR(ir->op2); |
156 | if (irref_isk(ir->op2)) { | 156 | if (irref_isk(ir->op2)) { |
@@ -217,8 +217,9 @@ static void asm_fuseahuref(ASMState *as, IRRef ref, RegSet allow) | |||
217 | } | 217 | } |
218 | break; | 218 | break; |
219 | default: | 219 | default: |
220 | lua_assert(ir->o == IR_HREF || ir->o == IR_NEWREF || ir->o == IR_UREFO || | 220 | lj_assertA(ir->o == IR_HREF || ir->o == IR_NEWREF || ir->o == IR_UREFO || |
221 | ir->o == IR_KKPTR); | 221 | ir->o == IR_KKPTR, |
222 | "bad IR op %d", ir->o); | ||
222 | break; | 223 | break; |
223 | } | 224 | } |
224 | } | 225 | } |
@@ -230,9 +231,10 @@ static void asm_fuseahuref(ASMState *as, IRRef ref, RegSet allow) | |||
230 | /* Fuse FLOAD/FREF reference into memory operand. */ | 231 | /* Fuse FLOAD/FREF reference into memory operand. */ |
231 | static void asm_fusefref(ASMState *as, IRIns *ir, RegSet allow) | 232 | static void asm_fusefref(ASMState *as, IRIns *ir, RegSet allow) |
232 | { | 233 | { |
233 | lua_assert(ir->o == IR_FLOAD || ir->o == IR_FREF); | 234 | lj_assertA(ir->o == IR_FLOAD || ir->o == IR_FREF, |
235 | "bad IR op %d", ir->o); | ||
234 | as->mrm.idx = RID_NONE; | 236 | as->mrm.idx = RID_NONE; |
235 | if (ir->op1 == REF_NIL) { | 237 | if (ir->op1 == REF_NIL) { /* FLOAD from GG_State with offset. */ |
236 | #if LJ_GC64 | 238 | #if LJ_GC64 |
237 | as->mrm.ofs = (int32_t)(ir->op2 << 2) - GG_OFS(dispatch); | 239 | as->mrm.ofs = (int32_t)(ir->op2 << 2) - GG_OFS(dispatch); |
238 | as->mrm.base = RID_DISPATCH; | 240 | as->mrm.base = RID_DISPATCH; |
@@ -271,7 +273,7 @@ static void asm_fusefref(ASMState *as, IRIns *ir, RegSet allow) | |||
271 | static void asm_fusestrref(ASMState *as, IRIns *ir, RegSet allow) | 273 | static void asm_fusestrref(ASMState *as, IRIns *ir, RegSet allow) |
272 | { | 274 | { |
273 | IRIns *irr; | 275 | IRIns *irr; |
274 | lua_assert(ir->o == IR_STRREF); | 276 | lj_assertA(ir->o == IR_STRREF, "bad IR op %d", ir->o); |
275 | as->mrm.base = as->mrm.idx = RID_NONE; | 277 | as->mrm.base = as->mrm.idx = RID_NONE; |
276 | as->mrm.scale = XM_SCALE1; | 278 | as->mrm.scale = XM_SCALE1; |
277 | as->mrm.ofs = sizeof(GCstr); | 279 | as->mrm.ofs = sizeof(GCstr); |
@@ -378,9 +380,10 @@ static Reg asm_fuseloadk64(ASMState *as, IRIns *ir) | |||
378 | checki32(mctopofs(as, k)) && checki32(mctopofs(as, k+1))) { | 380 | checki32(mctopofs(as, k)) && checki32(mctopofs(as, k+1))) { |
379 | as->mrm.ofs = (int32_t)mcpofs(as, k); | 381 | as->mrm.ofs = (int32_t)mcpofs(as, k); |
380 | as->mrm.base = RID_RIP; | 382 | as->mrm.base = RID_RIP; |
381 | } else { | 383 | } else { /* Intern 64 bit constant at bottom of mcode. */ |
382 | if (ir->i) { | 384 | if (ir->i) { |
383 | lua_assert(*k == *(uint64_t*)(as->mctop - ir->i)); | 385 | lj_assertA(*k == *(uint64_t*)(as->mctop - ir->i), |
386 | "bad interned 64 bit constant"); | ||
384 | } else { | 387 | } else { |
385 | while ((uintptr_t)as->mcbot & 7) *as->mcbot++ = XI_INT3; | 388 | while ((uintptr_t)as->mcbot & 7) *as->mcbot++ = XI_INT3; |
386 | *(uint64_t*)as->mcbot = *k; | 389 | *(uint64_t*)as->mcbot = *k; |
@@ -420,12 +423,12 @@ static Reg asm_fuseload(ASMState *as, IRRef ref, RegSet allow) | |||
420 | } | 423 | } |
421 | if (ir->o == IR_KNUM) { | 424 | if (ir->o == IR_KNUM) { |
422 | RegSet avail = as->freeset & ~as->modset & RSET_FPR; | 425 | RegSet avail = as->freeset & ~as->modset & RSET_FPR; |
423 | lua_assert(allow != RSET_EMPTY); | 426 | lj_assertA(allow != RSET_EMPTY, "no register allowed"); |
424 | if (!(avail & (avail-1))) /* Fuse if less than two regs available. */ | 427 | if (!(avail & (avail-1))) /* Fuse if less than two regs available. */ |
425 | return asm_fuseloadk64(as, ir); | 428 | return asm_fuseloadk64(as, ir); |
426 | } else if (ref == REF_BASE || ir->o == IR_KINT64) { | 429 | } else if (ref == REF_BASE || ir->o == IR_KINT64) { |
427 | RegSet avail = as->freeset & ~as->modset & RSET_GPR; | 430 | RegSet avail = as->freeset & ~as->modset & RSET_GPR; |
428 | lua_assert(allow != RSET_EMPTY); | 431 | lj_assertA(allow != RSET_EMPTY, "no register allowed"); |
429 | if (!(avail & (avail-1))) { /* Fuse if less than two regs available. */ | 432 | if (!(avail & (avail-1))) { /* Fuse if less than two regs available. */ |
430 | if (ref == REF_BASE) { | 433 | if (ref == REF_BASE) { |
431 | #if LJ_GC64 | 434 | #if LJ_GC64 |
@@ -606,7 +609,8 @@ static void asm_gencall(ASMState *as, const CCallInfo *ci, IRRef *args) | |||
606 | #endif | 609 | #endif |
607 | emit_loadi(as, r, ir->i); | 610 | emit_loadi(as, r, ir->i); |
608 | } else { | 611 | } else { |
609 | lua_assert(rset_test(as->freeset, r)); /* Must have been evicted. */ | 612 | /* Must have been evicted. */ |
613 | lj_assertA(rset_test(as->freeset, r), "reg %d not free", r); | ||
610 | if (ra_hasreg(ir->r)) { | 614 | if (ra_hasreg(ir->r)) { |
611 | ra_noweak(as, ir->r); | 615 | ra_noweak(as, ir->r); |
612 | emit_movrr(as, ir, r, ir->r); | 616 | emit_movrr(as, ir, r, ir->r); |
@@ -615,7 +619,8 @@ static void asm_gencall(ASMState *as, const CCallInfo *ci, IRRef *args) | |||
615 | } | 619 | } |
616 | } | 620 | } |
617 | } else if (irt_isfp(ir->t)) { /* FP argument is on stack. */ | 621 | } else if (irt_isfp(ir->t)) { /* FP argument is on stack. */ |
618 | lua_assert(!(irt_isfloat(ir->t) && irref_isk(ref))); /* No float k. */ | 622 | lj_assertA(!(irt_isfloat(ir->t) && irref_isk(ref)), |
623 | "unexpected float constant"); | ||
619 | if (LJ_32 && (ofs & 4) && irref_isk(ref)) { | 624 | if (LJ_32 && (ofs & 4) && irref_isk(ref)) { |
620 | /* Split stores for unaligned FP consts. */ | 625 | /* Split stores for unaligned FP consts. */ |
621 | emit_movmroi(as, RID_ESP, ofs, (int32_t)ir_knum(ir)->u32.lo); | 626 | emit_movmroi(as, RID_ESP, ofs, (int32_t)ir_knum(ir)->u32.lo); |
@@ -691,7 +696,7 @@ static void asm_setupresult(ASMState *as, IRIns *ir, const CCallInfo *ci) | |||
691 | ra_destpair(as, ir); | 696 | ra_destpair(as, ir); |
692 | #endif | 697 | #endif |
693 | } else { | 698 | } else { |
694 | lua_assert(!irt_ispri(ir->t)); | 699 | lj_assertA(!irt_ispri(ir->t), "PRI dest"); |
695 | ra_destreg(as, ir, RID_RET); | 700 | ra_destreg(as, ir, RID_RET); |
696 | } | 701 | } |
697 | } else if (LJ_32 && irt_isfp(ir->t) && !(ci->flags & CCI_CASTU64)) { | 702 | } else if (LJ_32 && irt_isfp(ir->t) && !(ci->flags & CCI_CASTU64)) { |
@@ -810,8 +815,10 @@ static void asm_conv(ASMState *as, IRIns *ir) | |||
810 | int st64 = (st == IRT_I64 || st == IRT_U64 || (LJ_64 && st == IRT_P64)); | 815 | int st64 = (st == IRT_I64 || st == IRT_U64 || (LJ_64 && st == IRT_P64)); |
811 | int stfp = (st == IRT_NUM || st == IRT_FLOAT); | 816 | int stfp = (st == IRT_NUM || st == IRT_FLOAT); |
812 | IRRef lref = ir->op1; | 817 | IRRef lref = ir->op1; |
813 | lua_assert(irt_type(ir->t) != st); | 818 | lj_assertA(irt_type(ir->t) != st, "inconsistent types for CONV"); |
814 | lua_assert(!(LJ_32 && (irt_isint64(ir->t) || st64))); /* Handled by SPLIT. */ | 819 | lj_assertA(!(LJ_32 && (irt_isint64(ir->t) || st64)), |
820 | "IR %04d has unsplit 64 bit type", | ||
821 | (int)(ir - as->ir) - REF_BIAS); | ||
815 | if (irt_isfp(ir->t)) { | 822 | if (irt_isfp(ir->t)) { |
816 | Reg dest = ra_dest(as, ir, RSET_FPR); | 823 | Reg dest = ra_dest(as, ir, RSET_FPR); |
817 | if (stfp) { /* FP to FP conversion. */ | 824 | if (stfp) { /* FP to FP conversion. */ |
@@ -847,7 +854,8 @@ static void asm_conv(ASMState *as, IRIns *ir) | |||
847 | } else if (stfp) { /* FP to integer conversion. */ | 854 | } else if (stfp) { /* FP to integer conversion. */ |
848 | if (irt_isguard(ir->t)) { | 855 | if (irt_isguard(ir->t)) { |
849 | /* Checked conversions are only supported from number to int. */ | 856 | /* Checked conversions are only supported from number to int. */ |
850 | lua_assert(irt_isint(ir->t) && st == IRT_NUM); | 857 | lj_assertA(irt_isint(ir->t) && st == IRT_NUM, |
858 | "bad type for checked CONV"); | ||
851 | asm_tointg(as, ir, ra_alloc1(as, lref, RSET_FPR)); | 859 | asm_tointg(as, ir, ra_alloc1(as, lref, RSET_FPR)); |
852 | } else { | 860 | } else { |
853 | Reg dest = ra_dest(as, ir, RSET_GPR); | 861 | Reg dest = ra_dest(as, ir, RSET_GPR); |
@@ -882,7 +890,7 @@ static void asm_conv(ASMState *as, IRIns *ir) | |||
882 | Reg left, dest = ra_dest(as, ir, RSET_GPR); | 890 | Reg left, dest = ra_dest(as, ir, RSET_GPR); |
883 | RegSet allow = RSET_GPR; | 891 | RegSet allow = RSET_GPR; |
884 | x86Op op; | 892 | x86Op op; |
885 | lua_assert(irt_isint(ir->t) || irt_isu32(ir->t)); | 893 | lj_assertA(irt_isint(ir->t) || irt_isu32(ir->t), "bad type for CONV EXT"); |
886 | if (st == IRT_I8) { | 894 | if (st == IRT_I8) { |
887 | op = XO_MOVSXb; allow = RSET_GPR8; dest |= FORCE_REX; | 895 | op = XO_MOVSXb; allow = RSET_GPR8; dest |= FORCE_REX; |
888 | } else if (st == IRT_U8) { | 896 | } else if (st == IRT_U8) { |
@@ -953,7 +961,7 @@ static void asm_conv_fp_int64(ASMState *as, IRIns *ir) | |||
953 | emit_sjcc(as, CC_NS, l_end); | 961 | emit_sjcc(as, CC_NS, l_end); |
954 | emit_rr(as, XO_TEST, hi, hi); /* Check if u64 >= 2^63. */ | 962 | emit_rr(as, XO_TEST, hi, hi); /* Check if u64 >= 2^63. */ |
955 | } else { | 963 | } else { |
956 | lua_assert(((ir-1)->op2 & IRCONV_SRCMASK) == IRT_I64); | 964 | lj_assertA(((ir-1)->op2 & IRCONV_SRCMASK) == IRT_I64, "bad type for CONV"); |
957 | } | 965 | } |
958 | emit_rmro(as, XO_FILDq, XOg_FILDq, RID_ESP, 0); | 966 | emit_rmro(as, XO_FILDq, XOg_FILDq, RID_ESP, 0); |
959 | /* NYI: Avoid narrow-to-wide store-to-load forwarding stall. */ | 967 | /* NYI: Avoid narrow-to-wide store-to-load forwarding stall. */ |
@@ -967,8 +975,8 @@ static void asm_conv_int64_fp(ASMState *as, IRIns *ir) | |||
967 | IRType st = (IRType)((ir-1)->op2 & IRCONV_SRCMASK); | 975 | IRType st = (IRType)((ir-1)->op2 & IRCONV_SRCMASK); |
968 | IRType dt = (((ir-1)->op2 & IRCONV_DSTMASK) >> IRCONV_DSH); | 976 | IRType dt = (((ir-1)->op2 & IRCONV_DSTMASK) >> IRCONV_DSH); |
969 | Reg lo, hi; | 977 | Reg lo, hi; |
970 | lua_assert(st == IRT_NUM || st == IRT_FLOAT); | 978 | lj_assertA(st == IRT_NUM || st == IRT_FLOAT, "bad type for CONV"); |
971 | lua_assert(dt == IRT_I64 || dt == IRT_U64); | 979 | lj_assertA(dt == IRT_I64 || dt == IRT_U64, "bad type for CONV"); |
972 | hi = ra_dest(as, ir, RSET_GPR); | 980 | hi = ra_dest(as, ir, RSET_GPR); |
973 | lo = ra_dest(as, ir-1, rset_exclude(RSET_GPR, hi)); | 981 | lo = ra_dest(as, ir-1, rset_exclude(RSET_GPR, hi)); |
974 | if (ra_used(ir-1)) emit_rmro(as, XO_MOV, lo, RID_ESP, 0); | 982 | if (ra_used(ir-1)) emit_rmro(as, XO_MOV, lo, RID_ESP, 0); |
@@ -1180,13 +1188,13 @@ static void asm_href(ASMState *as, IRIns *ir, IROp merge) | |||
1180 | emit_rmro(as, XO_CMP, tmp|REX_64, dest, offsetof(Node, key.u64)); | 1188 | emit_rmro(as, XO_CMP, tmp|REX_64, dest, offsetof(Node, key.u64)); |
1181 | } | 1189 | } |
1182 | } else { | 1190 | } else { |
1183 | lua_assert(irt_ispri(kt) && !irt_isnil(kt)); | 1191 | lj_assertA(irt_ispri(kt) && !irt_isnil(kt), "bad HREF key type"); |
1184 | emit_u32(as, (irt_toitype(kt)<<15)|0x7fff); | 1192 | emit_u32(as, (irt_toitype(kt)<<15)|0x7fff); |
1185 | emit_rmro(as, XO_ARITHi, XOg_CMP, dest, offsetof(Node, key.it)); | 1193 | emit_rmro(as, XO_ARITHi, XOg_CMP, dest, offsetof(Node, key.it)); |
1186 | #else | 1194 | #else |
1187 | } else { | 1195 | } else { |
1188 | if (!irt_ispri(kt)) { | 1196 | if (!irt_ispri(kt)) { |
1189 | lua_assert(irt_isaddr(kt)); | 1197 | lj_assertA(irt_isaddr(kt), "bad HREF key type"); |
1190 | if (isk) | 1198 | if (isk) |
1191 | emit_gmroi(as, XG_ARITHi(XOg_CMP), dest, offsetof(Node, key.gcr), | 1199 | emit_gmroi(as, XG_ARITHi(XOg_CMP), dest, offsetof(Node, key.gcr), |
1192 | ptr2addr(ir_kgc(irkey))); | 1200 | ptr2addr(ir_kgc(irkey))); |
@@ -1194,7 +1202,7 @@ static void asm_href(ASMState *as, IRIns *ir, IROp merge) | |||
1194 | emit_rmro(as, XO_CMP, key, dest, offsetof(Node, key.gcr)); | 1202 | emit_rmro(as, XO_CMP, key, dest, offsetof(Node, key.gcr)); |
1195 | emit_sjcc(as, CC_NE, l_next); | 1203 | emit_sjcc(as, CC_NE, l_next); |
1196 | } | 1204 | } |
1197 | lua_assert(!irt_isnil(kt)); | 1205 | lj_assertA(!irt_isnil(kt), "bad HREF key type"); |
1198 | emit_i8(as, irt_toitype(kt)); | 1206 | emit_i8(as, irt_toitype(kt)); |
1199 | emit_rmro(as, XO_ARITHi8, XOg_CMP, dest, offsetof(Node, key.it)); | 1207 | emit_rmro(as, XO_ARITHi8, XOg_CMP, dest, offsetof(Node, key.it)); |
1200 | #endif | 1208 | #endif |
@@ -1209,7 +1217,7 @@ static void asm_href(ASMState *as, IRIns *ir, IROp merge) | |||
1209 | #endif | 1217 | #endif |
1210 | 1218 | ||
1211 | /* Load main position relative to tab->node into dest. */ | 1219 | /* Load main position relative to tab->node into dest. */ |
1212 | khash = isk ? ir_khash(irkey) : 1; | 1220 | khash = isk ? ir_khash(as, irkey) : 1; |
1213 | if (khash == 0) { | 1221 | if (khash == 0) { |
1214 | emit_rmro(as, XO_MOV, dest|REX_GC64, tab, offsetof(GCtab, node)); | 1222 | emit_rmro(as, XO_MOV, dest|REX_GC64, tab, offsetof(GCtab, node)); |
1215 | } else { | 1223 | } else { |
@@ -1271,7 +1279,7 @@ static void asm_hrefk(ASMState *as, IRIns *ir) | |||
1271 | #if !LJ_64 | 1279 | #if !LJ_64 |
1272 | MCLabel l_exit; | 1280 | MCLabel l_exit; |
1273 | #endif | 1281 | #endif |
1274 | lua_assert(ofs % sizeof(Node) == 0); | 1282 | lj_assertA(ofs % sizeof(Node) == 0, "unaligned HREFK slot"); |
1275 | if (ra_hasreg(dest)) { | 1283 | if (ra_hasreg(dest)) { |
1276 | if (ofs != 0) { | 1284 | if (ofs != 0) { |
1277 | if (dest == node) | 1285 | if (dest == node) |
@@ -1288,7 +1296,8 @@ static void asm_hrefk(ASMState *as, IRIns *ir) | |||
1288 | Reg key = ra_scratch(as, rset_exclude(RSET_GPR, node)); | 1296 | Reg key = ra_scratch(as, rset_exclude(RSET_GPR, node)); |
1289 | emit_rmro(as, XO_CMP, key|REX_64, node, | 1297 | emit_rmro(as, XO_CMP, key|REX_64, node, |
1290 | ofs + (int32_t)offsetof(Node, key.u64)); | 1298 | ofs + (int32_t)offsetof(Node, key.u64)); |
1291 | lua_assert(irt_isnum(irkey->t) || irt_isgcv(irkey->t)); | 1299 | lj_assertA(irt_isnum(irkey->t) || irt_isgcv(irkey->t), |
1300 | "bad HREFK key type"); | ||
1292 | /* Assumes -0.0 is already canonicalized to +0.0. */ | 1301 | /* Assumes -0.0 is already canonicalized to +0.0. */ |
1293 | emit_loadu64(as, key, irt_isnum(irkey->t) ? ir_knum(irkey)->u64 : | 1302 | emit_loadu64(as, key, irt_isnum(irkey->t) ? ir_knum(irkey)->u64 : |
1294 | #if LJ_GC64 | 1303 | #if LJ_GC64 |
@@ -1299,7 +1308,7 @@ static void asm_hrefk(ASMState *as, IRIns *ir) | |||
1299 | (uint64_t)(uint32_t)ptr2addr(ir_kgc(irkey))); | 1308 | (uint64_t)(uint32_t)ptr2addr(ir_kgc(irkey))); |
1300 | #endif | 1309 | #endif |
1301 | } else { | 1310 | } else { |
1302 | lua_assert(!irt_isnil(irkey->t)); | 1311 | lj_assertA(!irt_isnil(irkey->t), "bad HREFK key type"); |
1303 | #if LJ_GC64 | 1312 | #if LJ_GC64 |
1304 | emit_i32(as, (irt_toitype(irkey->t)<<15)|0x7fff); | 1313 | emit_i32(as, (irt_toitype(irkey->t)<<15)|0x7fff); |
1305 | emit_rmro(as, XO_ARITHi, XOg_CMP, node, | 1314 | emit_rmro(as, XO_ARITHi, XOg_CMP, node, |
@@ -1323,13 +1332,13 @@ static void asm_hrefk(ASMState *as, IRIns *ir) | |||
1323 | (int32_t)ir_knum(irkey)->u32.hi); | 1332 | (int32_t)ir_knum(irkey)->u32.hi); |
1324 | } else { | 1333 | } else { |
1325 | if (!irt_ispri(irkey->t)) { | 1334 | if (!irt_ispri(irkey->t)) { |
1326 | lua_assert(irt_isgcv(irkey->t)); | 1335 | lj_assertA(irt_isgcv(irkey->t), "bad HREFK key type"); |
1327 | emit_gmroi(as, XG_ARITHi(XOg_CMP), node, | 1336 | emit_gmroi(as, XG_ARITHi(XOg_CMP), node, |
1328 | ofs + (int32_t)offsetof(Node, key.gcr), | 1337 | ofs + (int32_t)offsetof(Node, key.gcr), |
1329 | ptr2addr(ir_kgc(irkey))); | 1338 | ptr2addr(ir_kgc(irkey))); |
1330 | emit_sjcc(as, CC_NE, l_exit); | 1339 | emit_sjcc(as, CC_NE, l_exit); |
1331 | } | 1340 | } |
1332 | lua_assert(!irt_isnil(irkey->t)); | 1341 | lj_assertA(!irt_isnil(irkey->t), "bad HREFK key type"); |
1333 | emit_i8(as, irt_toitype(irkey->t)); | 1342 | emit_i8(as, irt_toitype(irkey->t)); |
1334 | emit_rmro(as, XO_ARITHi8, XOg_CMP, node, | 1343 | emit_rmro(as, XO_ARITHi8, XOg_CMP, node, |
1335 | ofs + (int32_t)offsetof(Node, key.it)); | 1344 | ofs + (int32_t)offsetof(Node, key.it)); |
@@ -1402,7 +1411,8 @@ static void asm_fxload(ASMState *as, IRIns *ir) | |||
1402 | if (LJ_64 && irt_is64(ir->t)) | 1411 | if (LJ_64 && irt_is64(ir->t)) |
1403 | dest |= REX_64; | 1412 | dest |= REX_64; |
1404 | else | 1413 | else |
1405 | lua_assert(irt_isint(ir->t) || irt_isu32(ir->t) || irt_isaddr(ir->t)); | 1414 | lj_assertA(irt_isint(ir->t) || irt_isu32(ir->t) || irt_isaddr(ir->t), |
1415 | "unsplit 64 bit load"); | ||
1406 | xo = XO_MOV; | 1416 | xo = XO_MOV; |
1407 | break; | 1417 | break; |
1408 | } | 1418 | } |
@@ -1447,13 +1457,16 @@ static void asm_fxstore(ASMState *as, IRIns *ir) | |||
1447 | case IRT_NUM: xo = XO_MOVSDto; break; | 1457 | case IRT_NUM: xo = XO_MOVSDto; break; |
1448 | case IRT_FLOAT: xo = XO_MOVSSto; break; | 1458 | case IRT_FLOAT: xo = XO_MOVSSto; break; |
1449 | #if LJ_64 && !LJ_GC64 | 1459 | #if LJ_64 && !LJ_GC64 |
1450 | case IRT_LIGHTUD: lua_assert(0); /* NYI: mask 64 bit lightuserdata. */ | 1460 | case IRT_LIGHTUD: |
1461 | /* NYI: mask 64 bit lightuserdata. */ | ||
1462 | lj_assertA(0, "store of lightuserdata"); | ||
1451 | #endif | 1463 | #endif |
1452 | default: | 1464 | default: |
1453 | if (LJ_64 && irt_is64(ir->t)) | 1465 | if (LJ_64 && irt_is64(ir->t)) |
1454 | src |= REX_64; | 1466 | src |= REX_64; |
1455 | else | 1467 | else |
1456 | lua_assert(irt_isint(ir->t) || irt_isu32(ir->t) || irt_isaddr(ir->t)); | 1468 | lj_assertA(irt_isint(ir->t) || irt_isu32(ir->t) || irt_isaddr(ir->t), |
1469 | "unsplit 64 bit store"); | ||
1457 | xo = XO_MOVto; | 1470 | xo = XO_MOVto; |
1458 | break; | 1471 | break; |
1459 | } | 1472 | } |
@@ -1467,8 +1480,8 @@ static void asm_fxstore(ASMState *as, IRIns *ir) | |||
1467 | emit_i8(as, k); | 1480 | emit_i8(as, k); |
1468 | emit_mrm(as, XO_MOVmib, 0, RID_MRM); | 1481 | emit_mrm(as, XO_MOVmib, 0, RID_MRM); |
1469 | } else { | 1482 | } else { |
1470 | lua_assert(irt_is64(ir->t) || irt_isint(ir->t) || irt_isu32(ir->t) || | 1483 | lj_assertA(irt_is64(ir->t) || irt_isint(ir->t) || irt_isu32(ir->t) || |
1471 | irt_isaddr(ir->t)); | 1484 | irt_isaddr(ir->t), "bad store type"); |
1472 | emit_i32(as, k); | 1485 | emit_i32(as, k); |
1473 | emit_mrm(as, XO_MOVmi, REX_64IR(ir, 0), RID_MRM); | 1486 | emit_mrm(as, XO_MOVmi, REX_64IR(ir, 0), RID_MRM); |
1474 | } | 1487 | } |
@@ -1503,8 +1516,9 @@ static void asm_ahuvload(ASMState *as, IRIns *ir) | |||
1503 | #if LJ_GC64 | 1516 | #if LJ_GC64 |
1504 | Reg tmp = RID_NONE; | 1517 | Reg tmp = RID_NONE; |
1505 | #endif | 1518 | #endif |
1506 | lua_assert(irt_isnum(ir->t) || irt_ispri(ir->t) || irt_isaddr(ir->t) || | 1519 | lj_assertA(irt_isnum(ir->t) || irt_ispri(ir->t) || irt_isaddr(ir->t) || |
1507 | (LJ_DUALNUM && irt_isint(ir->t))); | 1520 | (LJ_DUALNUM && irt_isint(ir->t)), |
1521 | "bad load type %d", irt_type(ir->t)); | ||
1508 | #if LJ_64 && !LJ_GC64 | 1522 | #if LJ_64 && !LJ_GC64 |
1509 | if (irt_islightud(ir->t)) { | 1523 | if (irt_islightud(ir->t)) { |
1510 | Reg dest = asm_load_lightud64(as, ir, 1); | 1524 | Reg dest = asm_load_lightud64(as, ir, 1); |
@@ -1551,7 +1565,8 @@ static void asm_ahuvload(ASMState *as, IRIns *ir) | |||
1551 | as->mrm.ofs += 4; | 1565 | as->mrm.ofs += 4; |
1552 | asm_guardcc(as, irt_isnum(ir->t) ? CC_AE : CC_NE); | 1566 | asm_guardcc(as, irt_isnum(ir->t) ? CC_AE : CC_NE); |
1553 | if (LJ_64 && irt_type(ir->t) >= IRT_NUM) { | 1567 | if (LJ_64 && irt_type(ir->t) >= IRT_NUM) { |
1554 | lua_assert(irt_isinteger(ir->t) || irt_isnum(ir->t)); | 1568 | lj_assertA(irt_isinteger(ir->t) || irt_isnum(ir->t), |
1569 | "bad load type %d", irt_type(ir->t)); | ||
1555 | #if LJ_GC64 | 1570 | #if LJ_GC64 |
1556 | emit_u32(as, LJ_TISNUM << 15); | 1571 | emit_u32(as, LJ_TISNUM << 15); |
1557 | #else | 1572 | #else |
@@ -1633,13 +1648,14 @@ static void asm_ahustore(ASMState *as, IRIns *ir) | |||
1633 | #endif | 1648 | #endif |
1634 | emit_mrm(as, XO_MOVto, src, RID_MRM); | 1649 | emit_mrm(as, XO_MOVto, src, RID_MRM); |
1635 | } else if (!irt_ispri(irr->t)) { | 1650 | } else if (!irt_ispri(irr->t)) { |
1636 | lua_assert(irt_isaddr(ir->t) || (LJ_DUALNUM && irt_isinteger(ir->t))); | 1651 | lj_assertA(irt_isaddr(ir->t) || (LJ_DUALNUM && irt_isinteger(ir->t)), |
1652 | "bad store type"); | ||
1637 | emit_i32(as, irr->i); | 1653 | emit_i32(as, irr->i); |
1638 | emit_mrm(as, XO_MOVmi, 0, RID_MRM); | 1654 | emit_mrm(as, XO_MOVmi, 0, RID_MRM); |
1639 | } | 1655 | } |
1640 | as->mrm.ofs += 4; | 1656 | as->mrm.ofs += 4; |
1641 | #if LJ_GC64 | 1657 | #if LJ_GC64 |
1642 | lua_assert(LJ_DUALNUM && irt_isinteger(ir->t)); | 1658 | lj_assertA(LJ_DUALNUM && irt_isinteger(ir->t), "bad store type"); |
1643 | emit_i32(as, LJ_TNUMX << 15); | 1659 | emit_i32(as, LJ_TNUMX << 15); |
1644 | #else | 1660 | #else |
1645 | emit_i32(as, (int32_t)irt_toitype(ir->t)); | 1661 | emit_i32(as, (int32_t)irt_toitype(ir->t)); |
@@ -1654,10 +1670,13 @@ static void asm_sload(ASMState *as, IRIns *ir) | |||
1654 | (!LJ_FR2 && (ir->op2 & IRSLOAD_FRAME) ? 4 : 0); | 1670 | (!LJ_FR2 && (ir->op2 & IRSLOAD_FRAME) ? 4 : 0); |
1655 | IRType1 t = ir->t; | 1671 | IRType1 t = ir->t; |
1656 | Reg base; | 1672 | Reg base; |
1657 | lua_assert(!(ir->op2 & IRSLOAD_PARENT)); /* Handled by asm_head_side(). */ | 1673 | lj_assertA(!(ir->op2 & IRSLOAD_PARENT), |
1658 | lua_assert(irt_isguard(t) || !(ir->op2 & IRSLOAD_TYPECHECK)); | 1674 | "bad parent SLOAD"); /* Handled by asm_head_side(). */ |
1659 | lua_assert(LJ_DUALNUM || | 1675 | lj_assertA(irt_isguard(t) || !(ir->op2 & IRSLOAD_TYPECHECK), |
1660 | !irt_isint(t) || (ir->op2 & (IRSLOAD_CONVERT|IRSLOAD_FRAME))); | 1676 | "inconsistent SLOAD variant"); |
1677 | lj_assertA(LJ_DUALNUM || | ||
1678 | !irt_isint(t) || (ir->op2 & (IRSLOAD_CONVERT|IRSLOAD_FRAME)), | ||
1679 | "bad SLOAD type"); | ||
1661 | if ((ir->op2 & IRSLOAD_CONVERT) && irt_isguard(t) && irt_isint(t)) { | 1680 | if ((ir->op2 & IRSLOAD_CONVERT) && irt_isguard(t) && irt_isint(t)) { |
1662 | Reg left = ra_scratch(as, RSET_FPR); | 1681 | Reg left = ra_scratch(as, RSET_FPR); |
1663 | asm_tointg(as, ir, left); /* Frees dest reg. Do this before base alloc. */ | 1682 | asm_tointg(as, ir, left); /* Frees dest reg. Do this before base alloc. */ |
@@ -1677,7 +1696,8 @@ static void asm_sload(ASMState *as, IRIns *ir) | |||
1677 | RegSet allow = irt_isnum(t) ? RSET_FPR : RSET_GPR; | 1696 | RegSet allow = irt_isnum(t) ? RSET_FPR : RSET_GPR; |
1678 | Reg dest = ra_dest(as, ir, allow); | 1697 | Reg dest = ra_dest(as, ir, allow); |
1679 | base = ra_alloc1(as, REF_BASE, RSET_GPR); | 1698 | base = ra_alloc1(as, REF_BASE, RSET_GPR); |
1680 | lua_assert(irt_isnum(t) || irt_isint(t) || irt_isaddr(t)); | 1699 | lj_assertA(irt_isnum(t) || irt_isint(t) || irt_isaddr(t), |
1700 | "bad SLOAD type %d", irt_type(t)); | ||
1681 | if ((ir->op2 & IRSLOAD_CONVERT)) { | 1701 | if ((ir->op2 & IRSLOAD_CONVERT)) { |
1682 | t.irt = irt_isint(t) ? IRT_NUM : IRT_INT; /* Check for original type. */ | 1702 | t.irt = irt_isint(t) ? IRT_NUM : IRT_INT; /* Check for original type. */ |
1683 | emit_rmro(as, irt_isint(t) ? XO_CVTSI2SD : XO_CVTTSD2SI, dest, base, ofs); | 1703 | emit_rmro(as, irt_isint(t) ? XO_CVTSI2SD : XO_CVTTSD2SI, dest, base, ofs); |
@@ -1723,7 +1743,8 @@ static void asm_sload(ASMState *as, IRIns *ir) | |||
1723 | /* Need type check, even if the load result is unused. */ | 1743 | /* Need type check, even if the load result is unused. */ |
1724 | asm_guardcc(as, irt_isnum(t) ? CC_AE : CC_NE); | 1744 | asm_guardcc(as, irt_isnum(t) ? CC_AE : CC_NE); |
1725 | if (LJ_64 && irt_type(t) >= IRT_NUM) { | 1745 | if (LJ_64 && irt_type(t) >= IRT_NUM) { |
1726 | lua_assert(irt_isinteger(t) || irt_isnum(t)); | 1746 | lj_assertA(irt_isinteger(t) || irt_isnum(t), |
1747 | "bad SLOAD type %d", irt_type(t)); | ||
1727 | #if LJ_GC64 | 1748 | #if LJ_GC64 |
1728 | emit_u32(as, LJ_TISNUM << 15); | 1749 | emit_u32(as, LJ_TISNUM << 15); |
1729 | #else | 1750 | #else |
@@ -1775,7 +1796,8 @@ static void asm_cnew(ASMState *as, IRIns *ir) | |||
1775 | CTInfo info = lj_ctype_info(cts, id, &sz); | 1796 | CTInfo info = lj_ctype_info(cts, id, &sz); |
1776 | const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_mem_newgco]; | 1797 | const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_mem_newgco]; |
1777 | IRRef args[4]; | 1798 | IRRef args[4]; |
1778 | lua_assert(sz != CTSIZE_INVALID || (ir->o == IR_CNEW && ir->op2 != REF_NIL)); | 1799 | lj_assertA(sz != CTSIZE_INVALID || (ir->o == IR_CNEW && ir->op2 != REF_NIL), |
1800 | "bad CNEW/CNEWI operands"); | ||
1779 | 1801 | ||
1780 | as->gcsteps++; | 1802 | as->gcsteps++; |
1781 | asm_setupresult(as, ir, ci); /* GCcdata * */ | 1803 | asm_setupresult(as, ir, ci); /* GCcdata * */ |
@@ -1805,7 +1827,7 @@ static void asm_cnew(ASMState *as, IRIns *ir) | |||
1805 | int32_t ofs = sizeof(GCcdata); | 1827 | int32_t ofs = sizeof(GCcdata); |
1806 | if (sz == 8) { | 1828 | if (sz == 8) { |
1807 | ofs += 4; ir++; | 1829 | ofs += 4; ir++; |
1808 | lua_assert(ir->o == IR_HIOP); | 1830 | lj_assertA(ir->o == IR_HIOP, "missing CNEWI HIOP"); |
1809 | } | 1831 | } |
1810 | do { | 1832 | do { |
1811 | if (irref_isk(ir->op2)) { | 1833 | if (irref_isk(ir->op2)) { |
@@ -1819,7 +1841,7 @@ static void asm_cnew(ASMState *as, IRIns *ir) | |||
1819 | ofs -= 4; ir--; | 1841 | ofs -= 4; ir--; |
1820 | } while (1); | 1842 | } while (1); |
1821 | #endif | 1843 | #endif |
1822 | lua_assert(sz == 4 || sz == 8); | 1844 | lj_assertA(sz == 4 || sz == 8, "bad CNEWI size %d", sz); |
1823 | } else if (ir->op2 != REF_NIL) { /* Create VLA/VLS/aligned cdata. */ | 1845 | } else if (ir->op2 != REF_NIL) { /* Create VLA/VLS/aligned cdata. */ |
1824 | ci = &lj_ir_callinfo[IRCALL_lj_cdata_newv]; | 1846 | ci = &lj_ir_callinfo[IRCALL_lj_cdata_newv]; |
1825 | args[0] = ASMREF_L; /* lua_State *L */ | 1847 | args[0] = ASMREF_L; /* lua_State *L */ |
@@ -1869,7 +1891,7 @@ static void asm_obar(ASMState *as, IRIns *ir) | |||
1869 | MCLabel l_end; | 1891 | MCLabel l_end; |
1870 | Reg obj; | 1892 | Reg obj; |
1871 | /* No need for other object barriers (yet). */ | 1893 | /* No need for other object barriers (yet). */ |
1872 | lua_assert(IR(ir->op1)->o == IR_UREFC); | 1894 | lj_assertA(IR(ir->op1)->o == IR_UREFC, "bad OBAR type"); |
1873 | ra_evictset(as, RSET_SCRATCH); | 1895 | ra_evictset(as, RSET_SCRATCH); |
1874 | l_end = emit_label(as); | 1896 | l_end = emit_label(as); |
1875 | args[0] = ASMREF_TMP1; /* global_State *g */ | 1897 | args[0] = ASMREF_TMP1; /* global_State *g */ |
@@ -1986,7 +2008,7 @@ static int asm_swapops(ASMState *as, IRIns *ir) | |||
1986 | { | 2008 | { |
1987 | IRIns *irl = IR(ir->op1); | 2009 | IRIns *irl = IR(ir->op1); |
1988 | IRIns *irr = IR(ir->op2); | 2010 | IRIns *irr = IR(ir->op2); |
1989 | lua_assert(ra_noreg(irr->r)); | 2011 | lj_assertA(ra_noreg(irr->r), "bad usage"); |
1990 | if (!irm_iscomm(lj_ir_mode[ir->o])) | 2012 | if (!irm_iscomm(lj_ir_mode[ir->o])) |
1991 | return 0; /* Can't swap non-commutative operations. */ | 2013 | return 0; /* Can't swap non-commutative operations. */ |
1992 | if (irref_isk(ir->op2)) | 2014 | if (irref_isk(ir->op2)) |
@@ -2376,8 +2398,9 @@ static void asm_comp(ASMState *as, IRIns *ir) | |||
2376 | IROp leftop = (IROp)(IR(lref)->o); | 2398 | IROp leftop = (IROp)(IR(lref)->o); |
2377 | Reg r64 = REX_64IR(ir, 0); | 2399 | Reg r64 = REX_64IR(ir, 0); |
2378 | int32_t imm = 0; | 2400 | int32_t imm = 0; |
2379 | lua_assert(irt_is64(ir->t) || irt_isint(ir->t) || | 2401 | lj_assertA(irt_is64(ir->t) || irt_isint(ir->t) || |
2380 | irt_isu32(ir->t) || irt_isaddr(ir->t) || irt_isu8(ir->t)); | 2402 | irt_isu32(ir->t) || irt_isaddr(ir->t) || irt_isu8(ir->t), |
2403 | "bad comparison data type %d", irt_type(ir->t)); | ||
2381 | /* Swap constants (only for ABC) and fusable loads to the right. */ | 2404 | /* Swap constants (only for ABC) and fusable loads to the right. */ |
2382 | if (irref_isk(lref) || (!irref_isk(rref) && opisfusableload(leftop))) { | 2405 | if (irref_isk(lref) || (!irref_isk(rref) && opisfusableload(leftop))) { |
2383 | if ((cc & 0xc) == 0xc) cc ^= 0x53; /* L <-> G, LE <-> GE */ | 2406 | if ((cc & 0xc) == 0xc) cc ^= 0x53; /* L <-> G, LE <-> GE */ |
@@ -2459,7 +2482,7 @@ static void asm_comp(ASMState *as, IRIns *ir) | |||
2459 | /* Use test r,r instead of cmp r,0. */ | 2482 | /* Use test r,r instead of cmp r,0. */ |
2460 | x86Op xo = XO_TEST; | 2483 | x86Op xo = XO_TEST; |
2461 | if (irt_isu8(ir->t)) { | 2484 | if (irt_isu8(ir->t)) { |
2462 | lua_assert(ir->o == IR_EQ || ir->o == IR_NE); | 2485 | lj_assertA(ir->o == IR_EQ || ir->o == IR_NE, "bad usage"); |
2463 | xo = XO_TESTb; | 2486 | xo = XO_TESTb; |
2464 | if (!rset_test(RSET_RANGE(RID_EAX, RID_EBX+1), left)) { | 2487 | if (!rset_test(RSET_RANGE(RID_EAX, RID_EBX+1), left)) { |
2465 | if (LJ_64) { | 2488 | if (LJ_64) { |
@@ -2615,10 +2638,11 @@ static void asm_hiop(ASMState *as, IRIns *ir) | |||
2615 | case IR_CNEWI: | 2638 | case IR_CNEWI: |
2616 | /* Nothing to do here. Handled by CNEWI itself. */ | 2639 | /* Nothing to do here. Handled by CNEWI itself. */ |
2617 | break; | 2640 | break; |
2618 | default: lua_assert(0); break; | 2641 | default: lj_assertA(0, "bad HIOP for op %d", (ir-1)->o); break; |
2619 | } | 2642 | } |
2620 | #else | 2643 | #else |
2621 | UNUSED(as); UNUSED(ir); lua_assert(0); /* Unused on x64 or without FFI. */ | 2644 | /* Unused on x64 or without FFI. */ |
2645 | UNUSED(as); UNUSED(ir); lj_assertA(0, "unexpected HIOP"); | ||
2622 | #endif | 2646 | #endif |
2623 | } | 2647 | } |
2624 | 2648 | ||
@@ -2684,8 +2708,9 @@ static void asm_stack_restore(ASMState *as, SnapShot *snap) | |||
2684 | Reg src = ra_alloc1(as, ref, RSET_FPR); | 2708 | Reg src = ra_alloc1(as, ref, RSET_FPR); |
2685 | emit_rmro(as, XO_MOVSDto, src, RID_BASE, ofs); | 2709 | emit_rmro(as, XO_MOVSDto, src, RID_BASE, ofs); |
2686 | } else { | 2710 | } else { |
2687 | lua_assert(irt_ispri(ir->t) || irt_isaddr(ir->t) || | 2711 | lj_assertA(irt_ispri(ir->t) || irt_isaddr(ir->t) || |
2688 | (LJ_DUALNUM && irt_isinteger(ir->t))); | 2712 | (LJ_DUALNUM && irt_isinteger(ir->t)), |
2713 | "restore of IR type %d", irt_type(ir->t)); | ||
2689 | if (!irref_isk(ref)) { | 2714 | if (!irref_isk(ref)) { |
2690 | Reg src = ra_alloc1(as, ref, rset_exclude(RSET_GPR, RID_BASE)); | 2715 | Reg src = ra_alloc1(as, ref, rset_exclude(RSET_GPR, RID_BASE)); |
2691 | #if LJ_GC64 | 2716 | #if LJ_GC64 |
@@ -2730,7 +2755,7 @@ static void asm_stack_restore(ASMState *as, SnapShot *snap) | |||
2730 | } | 2755 | } |
2731 | checkmclim(as); | 2756 | checkmclim(as); |
2732 | } | 2757 | } |
2733 | lua_assert(map + nent == flinks); | 2758 | lj_assertA(map + nent == flinks, "inconsistent frames in snapshot"); |
2734 | } | 2759 | } |
2735 | 2760 | ||
2736 | /* -- GC handling --------------------------------------------------------- */ | 2761 | /* -- GC handling --------------------------------------------------------- */ |
@@ -2774,16 +2799,16 @@ static void asm_loop_fixup(ASMState *as) | |||
2774 | MCode *target = as->mcp; | 2799 | MCode *target = as->mcp; |
2775 | if (as->realign) { /* Realigned loops use short jumps. */ | 2800 | if (as->realign) { /* Realigned loops use short jumps. */ |
2776 | as->realign = NULL; /* Stop another retry. */ | 2801 | as->realign = NULL; /* Stop another retry. */ |
2777 | lua_assert(((intptr_t)target & 15) == 0); | 2802 | lj_assertA(((intptr_t)target & 15) == 0, "loop realign failed"); |
2778 | if (as->loopinv) { /* Inverted loop branch? */ | 2803 | if (as->loopinv) { /* Inverted loop branch? */ |
2779 | p -= 5; | 2804 | p -= 5; |
2780 | p[0] = XI_JMP; | 2805 | p[0] = XI_JMP; |
2781 | lua_assert(target - p >= -128); | 2806 | lj_assertA(target - p >= -128, "loop realign failed"); |
2782 | p[-1] = (MCode)(target - p); /* Patch sjcc. */ | 2807 | p[-1] = (MCode)(target - p); /* Patch sjcc. */ |
2783 | if (as->loopinv == 2) | 2808 | if (as->loopinv == 2) |
2784 | p[-3] = (MCode)(target - p + 2); /* Patch opt. short jp. */ | 2809 | p[-3] = (MCode)(target - p + 2); /* Patch opt. short jp. */ |
2785 | } else { | 2810 | } else { |
2786 | lua_assert(target - p >= -128); | 2811 | lj_assertA(target - p >= -128, "loop realign failed"); |
2787 | p[-1] = (MCode)(int8_t)(target - p); /* Patch short jmp. */ | 2812 | p[-1] = (MCode)(int8_t)(target - p); /* Patch short jmp. */ |
2788 | p[-2] = XI_JMPs; | 2813 | p[-2] = XI_JMPs; |
2789 | } | 2814 | } |
@@ -2880,7 +2905,7 @@ static void asm_tail_fixup(ASMState *as, TraceNo lnk) | |||
2880 | } | 2905 | } |
2881 | /* Patch exit branch. */ | 2906 | /* Patch exit branch. */ |
2882 | target = lnk ? traceref(as->J, lnk)->mcode : (MCode *)lj_vm_exit_interp; | 2907 | target = lnk ? traceref(as->J, lnk)->mcode : (MCode *)lj_vm_exit_interp; |
2883 | *(int32_t *)(p-4) = jmprel(p, target); | 2908 | *(int32_t *)(p-4) = jmprel(as->J, p, target); |
2884 | p[-5] = XI_JMP; | 2909 | p[-5] = XI_JMP; |
2885 | /* Drop unused mcode tail. Fill with NOPs to make the prefetcher happy. */ | 2910 | /* Drop unused mcode tail. Fill with NOPs to make the prefetcher happy. */ |
2886 | for (q = as->mctop-1; q >= p; q--) | 2911 | for (q = as->mctop-1; q >= p; q--) |
@@ -3053,17 +3078,17 @@ void lj_asm_patchexit(jit_State *J, GCtrace *T, ExitNo exitno, MCode *target) | |||
3053 | uint32_t statei = u32ptr(&J2G(J)->vmstate); | 3078 | uint32_t statei = u32ptr(&J2G(J)->vmstate); |
3054 | #endif | 3079 | #endif |
3055 | if (len > 5 && p[len-5] == XI_JMP && p+len-6 + *(int32_t *)(p+len-4) == px) | 3080 | if (len > 5 && p[len-5] == XI_JMP && p+len-6 + *(int32_t *)(p+len-4) == px) |
3056 | *(int32_t *)(p+len-4) = jmprel(p+len, target); | 3081 | *(int32_t *)(p+len-4) = jmprel(J, p+len, target); |
3057 | /* Do not patch parent exit for a stack check. Skip beyond vmstate update. */ | 3082 | /* Do not patch parent exit for a stack check. Skip beyond vmstate update. */ |
3058 | for (; p < pe; p += asm_x86_inslen(p)) { | 3083 | for (; p < pe; p += asm_x86_inslen(p)) { |
3059 | intptr_t ofs = LJ_GC64 ? (p[0] & 0xf0) == 0x40 : LJ_64; | 3084 | intptr_t ofs = LJ_GC64 ? (p[0] & 0xf0) == 0x40 : LJ_64; |
3060 | if (*(uint32_t *)(p+2+ofs) == statei && p[ofs+LJ_GC64-LJ_64] == XI_MOVmi) | 3085 | if (*(uint32_t *)(p+2+ofs) == statei && p[ofs+LJ_GC64-LJ_64] == XI_MOVmi) |
3061 | break; | 3086 | break; |
3062 | } | 3087 | } |
3063 | lua_assert(p < pe); | 3088 | lj_assertJ(p < pe, "instruction length decoder failed"); |
3064 | for (; p < pe; p += asm_x86_inslen(p)) | 3089 | for (; p < pe; p += asm_x86_inslen(p)) |
3065 | if ((*(uint16_t *)p & 0xf0ff) == 0x800f && p + *(int32_t *)(p+2) == px) | 3090 | if ((*(uint16_t *)p & 0xf0ff) == 0x800f && p + *(int32_t *)(p+2) == px) |
3066 | *(int32_t *)(p+2) = jmprel(p+6, target); | 3091 | *(int32_t *)(p+2) = jmprel(J, p+6, target); |
3067 | lj_mcode_sync(T->mcode, T->mcode + T->szmcode); | 3092 | lj_mcode_sync(T->mcode, T->mcode + T->szmcode); |
3068 | lj_mcode_patch(J, mcarea, 1); | 3093 | lj_mcode_patch(J, mcarea, 1); |
3069 | } | 3094 | } |