aboutsummaryrefslogtreecommitdiff
path: root/src/lj_asm_x86.h
diff options
context:
space:
mode:
Diffstat (limited to 'src/lj_asm_x86.h')
-rw-r--r--src/lj_asm_x86.h161
1 files changed, 93 insertions, 68 deletions
diff --git a/src/lj_asm_x86.h b/src/lj_asm_x86.h
index 7356a5f0..a3adee14 100644
--- a/src/lj_asm_x86.h
+++ b/src/lj_asm_x86.h
@@ -31,7 +31,7 @@ static MCode *asm_exitstub_gen(ASMState *as, ExitNo group)
31#endif 31#endif
32 /* Jump to exit handler which fills in the ExitState. */ 32 /* Jump to exit handler which fills in the ExitState. */
33 *mxp++ = XI_JMP; mxp += 4; 33 *mxp++ = XI_JMP; mxp += 4;
34 *((int32_t *)(mxp-4)) = jmprel(mxp, (MCode *)(void *)lj_vm_exit_handler); 34 *((int32_t *)(mxp-4)) = jmprel(as->J, mxp, (MCode *)(void *)lj_vm_exit_handler);
35 /* Commit the code for this group (even if assembly fails later on). */ 35 /* Commit the code for this group (even if assembly fails later on). */
36 lj_mcode_commitbot(as->J, mxp); 36 lj_mcode_commitbot(as->J, mxp);
37 as->mcbot = mxp; 37 as->mcbot = mxp;
@@ -60,7 +60,7 @@ static void asm_guardcc(ASMState *as, int cc)
60 MCode *p = as->mcp; 60 MCode *p = as->mcp;
61 if (LJ_UNLIKELY(p == as->invmcp)) { 61 if (LJ_UNLIKELY(p == as->invmcp)) {
62 as->loopinv = 1; 62 as->loopinv = 1;
63 *(int32_t *)(p+1) = jmprel(p+5, target); 63 *(int32_t *)(p+1) = jmprel(as->J, p+5, target);
64 target = p; 64 target = p;
65 cc ^= 1; 65 cc ^= 1;
66 if (as->realign) { 66 if (as->realign) {
@@ -131,7 +131,7 @@ static IRRef asm_fuseabase(ASMState *as, IRRef ref)
131 as->mrm.ofs = 0; 131 as->mrm.ofs = 0;
132 if (irb->o == IR_FLOAD) { 132 if (irb->o == IR_FLOAD) {
133 IRIns *ira = IR(irb->op1); 133 IRIns *ira = IR(irb->op1);
134 lua_assert(irb->op2 == IRFL_TAB_ARRAY); 134 lj_assertA(irb->op2 == IRFL_TAB_ARRAY, "expected FLOAD TAB_ARRAY");
135 /* We can avoid the FLOAD of t->array for colocated arrays. */ 135 /* We can avoid the FLOAD of t->array for colocated arrays. */
136 if (ira->o == IR_TNEW && ira->op1 <= LJ_MAX_COLOSIZE && 136 if (ira->o == IR_TNEW && ira->op1 <= LJ_MAX_COLOSIZE &&
137 !neverfuse(as) && noconflict(as, irb->op1, IR_NEWREF, 1)) { 137 !neverfuse(as) && noconflict(as, irb->op1, IR_NEWREF, 1)) {
@@ -150,7 +150,7 @@ static IRRef asm_fuseabase(ASMState *as, IRRef ref)
150static void asm_fusearef(ASMState *as, IRIns *ir, RegSet allow) 150static void asm_fusearef(ASMState *as, IRIns *ir, RegSet allow)
151{ 151{
152 IRIns *irx; 152 IRIns *irx;
153 lua_assert(ir->o == IR_AREF); 153 lj_assertA(ir->o == IR_AREF, "expected AREF");
154 as->mrm.base = (uint8_t)ra_alloc1(as, asm_fuseabase(as, ir->op1), allow); 154 as->mrm.base = (uint8_t)ra_alloc1(as, asm_fuseabase(as, ir->op1), allow);
155 irx = IR(ir->op2); 155 irx = IR(ir->op2);
156 if (irref_isk(ir->op2)) { 156 if (irref_isk(ir->op2)) {
@@ -217,8 +217,9 @@ static void asm_fuseahuref(ASMState *as, IRRef ref, RegSet allow)
217 } 217 }
218 break; 218 break;
219 default: 219 default:
220 lua_assert(ir->o == IR_HREF || ir->o == IR_NEWREF || ir->o == IR_UREFO || 220 lj_assertA(ir->o == IR_HREF || ir->o == IR_NEWREF || ir->o == IR_UREFO ||
221 ir->o == IR_KKPTR); 221 ir->o == IR_KKPTR,
222 "bad IR op %d", ir->o);
222 break; 223 break;
223 } 224 }
224 } 225 }
@@ -230,9 +231,10 @@ static void asm_fuseahuref(ASMState *as, IRRef ref, RegSet allow)
230/* Fuse FLOAD/FREF reference into memory operand. */ 231/* Fuse FLOAD/FREF reference into memory operand. */
231static void asm_fusefref(ASMState *as, IRIns *ir, RegSet allow) 232static void asm_fusefref(ASMState *as, IRIns *ir, RegSet allow)
232{ 233{
233 lua_assert(ir->o == IR_FLOAD || ir->o == IR_FREF); 234 lj_assertA(ir->o == IR_FLOAD || ir->o == IR_FREF,
235 "bad IR op %d", ir->o);
234 as->mrm.idx = RID_NONE; 236 as->mrm.idx = RID_NONE;
235 if (ir->op1 == REF_NIL) { 237 if (ir->op1 == REF_NIL) { /* FLOAD from GG_State with offset. */
236#if LJ_GC64 238#if LJ_GC64
237 as->mrm.ofs = (int32_t)(ir->op2 << 2) - GG_OFS(dispatch); 239 as->mrm.ofs = (int32_t)(ir->op2 << 2) - GG_OFS(dispatch);
238 as->mrm.base = RID_DISPATCH; 240 as->mrm.base = RID_DISPATCH;
@@ -271,7 +273,7 @@ static void asm_fusefref(ASMState *as, IRIns *ir, RegSet allow)
271static void asm_fusestrref(ASMState *as, IRIns *ir, RegSet allow) 273static void asm_fusestrref(ASMState *as, IRIns *ir, RegSet allow)
272{ 274{
273 IRIns *irr; 275 IRIns *irr;
274 lua_assert(ir->o == IR_STRREF); 276 lj_assertA(ir->o == IR_STRREF, "bad IR op %d", ir->o);
275 as->mrm.base = as->mrm.idx = RID_NONE; 277 as->mrm.base = as->mrm.idx = RID_NONE;
276 as->mrm.scale = XM_SCALE1; 278 as->mrm.scale = XM_SCALE1;
277 as->mrm.ofs = sizeof(GCstr); 279 as->mrm.ofs = sizeof(GCstr);
@@ -378,9 +380,10 @@ static Reg asm_fuseloadk64(ASMState *as, IRIns *ir)
378 checki32(mctopofs(as, k)) && checki32(mctopofs(as, k+1))) { 380 checki32(mctopofs(as, k)) && checki32(mctopofs(as, k+1))) {
379 as->mrm.ofs = (int32_t)mcpofs(as, k); 381 as->mrm.ofs = (int32_t)mcpofs(as, k);
380 as->mrm.base = RID_RIP; 382 as->mrm.base = RID_RIP;
381 } else { 383 } else { /* Intern 64 bit constant at bottom of mcode. */
382 if (ir->i) { 384 if (ir->i) {
383 lua_assert(*k == *(uint64_t*)(as->mctop - ir->i)); 385 lj_assertA(*k == *(uint64_t*)(as->mctop - ir->i),
386 "bad interned 64 bit constant");
384 } else { 387 } else {
385 while ((uintptr_t)as->mcbot & 7) *as->mcbot++ = XI_INT3; 388 while ((uintptr_t)as->mcbot & 7) *as->mcbot++ = XI_INT3;
386 *(uint64_t*)as->mcbot = *k; 389 *(uint64_t*)as->mcbot = *k;
@@ -420,12 +423,12 @@ static Reg asm_fuseload(ASMState *as, IRRef ref, RegSet allow)
420 } 423 }
421 if (ir->o == IR_KNUM) { 424 if (ir->o == IR_KNUM) {
422 RegSet avail = as->freeset & ~as->modset & RSET_FPR; 425 RegSet avail = as->freeset & ~as->modset & RSET_FPR;
423 lua_assert(allow != RSET_EMPTY); 426 lj_assertA(allow != RSET_EMPTY, "no register allowed");
424 if (!(avail & (avail-1))) /* Fuse if less than two regs available. */ 427 if (!(avail & (avail-1))) /* Fuse if less than two regs available. */
425 return asm_fuseloadk64(as, ir); 428 return asm_fuseloadk64(as, ir);
426 } else if (ref == REF_BASE || ir->o == IR_KINT64) { 429 } else if (ref == REF_BASE || ir->o == IR_KINT64) {
427 RegSet avail = as->freeset & ~as->modset & RSET_GPR; 430 RegSet avail = as->freeset & ~as->modset & RSET_GPR;
428 lua_assert(allow != RSET_EMPTY); 431 lj_assertA(allow != RSET_EMPTY, "no register allowed");
429 if (!(avail & (avail-1))) { /* Fuse if less than two regs available. */ 432 if (!(avail & (avail-1))) { /* Fuse if less than two regs available. */
430 if (ref == REF_BASE) { 433 if (ref == REF_BASE) {
431#if LJ_GC64 434#if LJ_GC64
@@ -606,7 +609,8 @@ static void asm_gencall(ASMState *as, const CCallInfo *ci, IRRef *args)
606#endif 609#endif
607 emit_loadi(as, r, ir->i); 610 emit_loadi(as, r, ir->i);
608 } else { 611 } else {
609 lua_assert(rset_test(as->freeset, r)); /* Must have been evicted. */ 612 /* Must have been evicted. */
613 lj_assertA(rset_test(as->freeset, r), "reg %d not free", r);
610 if (ra_hasreg(ir->r)) { 614 if (ra_hasreg(ir->r)) {
611 ra_noweak(as, ir->r); 615 ra_noweak(as, ir->r);
612 emit_movrr(as, ir, r, ir->r); 616 emit_movrr(as, ir, r, ir->r);
@@ -615,7 +619,8 @@ static void asm_gencall(ASMState *as, const CCallInfo *ci, IRRef *args)
615 } 619 }
616 } 620 }
617 } else if (irt_isfp(ir->t)) { /* FP argument is on stack. */ 621 } else if (irt_isfp(ir->t)) { /* FP argument is on stack. */
618 lua_assert(!(irt_isfloat(ir->t) && irref_isk(ref))); /* No float k. */ 622 lj_assertA(!(irt_isfloat(ir->t) && irref_isk(ref)),
623 "unexpected float constant");
619 if (LJ_32 && (ofs & 4) && irref_isk(ref)) { 624 if (LJ_32 && (ofs & 4) && irref_isk(ref)) {
620 /* Split stores for unaligned FP consts. */ 625 /* Split stores for unaligned FP consts. */
621 emit_movmroi(as, RID_ESP, ofs, (int32_t)ir_knum(ir)->u32.lo); 626 emit_movmroi(as, RID_ESP, ofs, (int32_t)ir_knum(ir)->u32.lo);
@@ -691,7 +696,7 @@ static void asm_setupresult(ASMState *as, IRIns *ir, const CCallInfo *ci)
691 ra_destpair(as, ir); 696 ra_destpair(as, ir);
692#endif 697#endif
693 } else { 698 } else {
694 lua_assert(!irt_ispri(ir->t)); 699 lj_assertA(!irt_ispri(ir->t), "PRI dest");
695 ra_destreg(as, ir, RID_RET); 700 ra_destreg(as, ir, RID_RET);
696 } 701 }
697 } else if (LJ_32 && irt_isfp(ir->t) && !(ci->flags & CCI_CASTU64)) { 702 } else if (LJ_32 && irt_isfp(ir->t) && !(ci->flags & CCI_CASTU64)) {
@@ -810,8 +815,10 @@ static void asm_conv(ASMState *as, IRIns *ir)
810 int st64 = (st == IRT_I64 || st == IRT_U64 || (LJ_64 && st == IRT_P64)); 815 int st64 = (st == IRT_I64 || st == IRT_U64 || (LJ_64 && st == IRT_P64));
811 int stfp = (st == IRT_NUM || st == IRT_FLOAT); 816 int stfp = (st == IRT_NUM || st == IRT_FLOAT);
812 IRRef lref = ir->op1; 817 IRRef lref = ir->op1;
813 lua_assert(irt_type(ir->t) != st); 818 lj_assertA(irt_type(ir->t) != st, "inconsistent types for CONV");
814 lua_assert(!(LJ_32 && (irt_isint64(ir->t) || st64))); /* Handled by SPLIT. */ 819 lj_assertA(!(LJ_32 && (irt_isint64(ir->t) || st64)),
820 "IR %04d has unsplit 64 bit type",
821 (int)(ir - as->ir) - REF_BIAS);
815 if (irt_isfp(ir->t)) { 822 if (irt_isfp(ir->t)) {
816 Reg dest = ra_dest(as, ir, RSET_FPR); 823 Reg dest = ra_dest(as, ir, RSET_FPR);
817 if (stfp) { /* FP to FP conversion. */ 824 if (stfp) { /* FP to FP conversion. */
@@ -847,7 +854,8 @@ static void asm_conv(ASMState *as, IRIns *ir)
847 } else if (stfp) { /* FP to integer conversion. */ 854 } else if (stfp) { /* FP to integer conversion. */
848 if (irt_isguard(ir->t)) { 855 if (irt_isguard(ir->t)) {
849 /* Checked conversions are only supported from number to int. */ 856 /* Checked conversions are only supported from number to int. */
850 lua_assert(irt_isint(ir->t) && st == IRT_NUM); 857 lj_assertA(irt_isint(ir->t) && st == IRT_NUM,
858 "bad type for checked CONV");
851 asm_tointg(as, ir, ra_alloc1(as, lref, RSET_FPR)); 859 asm_tointg(as, ir, ra_alloc1(as, lref, RSET_FPR));
852 } else { 860 } else {
853 Reg dest = ra_dest(as, ir, RSET_GPR); 861 Reg dest = ra_dest(as, ir, RSET_GPR);
@@ -882,7 +890,7 @@ static void asm_conv(ASMState *as, IRIns *ir)
882 Reg left, dest = ra_dest(as, ir, RSET_GPR); 890 Reg left, dest = ra_dest(as, ir, RSET_GPR);
883 RegSet allow = RSET_GPR; 891 RegSet allow = RSET_GPR;
884 x86Op op; 892 x86Op op;
885 lua_assert(irt_isint(ir->t) || irt_isu32(ir->t)); 893 lj_assertA(irt_isint(ir->t) || irt_isu32(ir->t), "bad type for CONV EXT");
886 if (st == IRT_I8) { 894 if (st == IRT_I8) {
887 op = XO_MOVSXb; allow = RSET_GPR8; dest |= FORCE_REX; 895 op = XO_MOVSXb; allow = RSET_GPR8; dest |= FORCE_REX;
888 } else if (st == IRT_U8) { 896 } else if (st == IRT_U8) {
@@ -953,7 +961,7 @@ static void asm_conv_fp_int64(ASMState *as, IRIns *ir)
953 emit_sjcc(as, CC_NS, l_end); 961 emit_sjcc(as, CC_NS, l_end);
954 emit_rr(as, XO_TEST, hi, hi); /* Check if u64 >= 2^63. */ 962 emit_rr(as, XO_TEST, hi, hi); /* Check if u64 >= 2^63. */
955 } else { 963 } else {
956 lua_assert(((ir-1)->op2 & IRCONV_SRCMASK) == IRT_I64); 964 lj_assertA(((ir-1)->op2 & IRCONV_SRCMASK) == IRT_I64, "bad type for CONV");
957 } 965 }
958 emit_rmro(as, XO_FILDq, XOg_FILDq, RID_ESP, 0); 966 emit_rmro(as, XO_FILDq, XOg_FILDq, RID_ESP, 0);
959 /* NYI: Avoid narrow-to-wide store-to-load forwarding stall. */ 967 /* NYI: Avoid narrow-to-wide store-to-load forwarding stall. */
@@ -967,8 +975,8 @@ static void asm_conv_int64_fp(ASMState *as, IRIns *ir)
967 IRType st = (IRType)((ir-1)->op2 & IRCONV_SRCMASK); 975 IRType st = (IRType)((ir-1)->op2 & IRCONV_SRCMASK);
968 IRType dt = (((ir-1)->op2 & IRCONV_DSTMASK) >> IRCONV_DSH); 976 IRType dt = (((ir-1)->op2 & IRCONV_DSTMASK) >> IRCONV_DSH);
969 Reg lo, hi; 977 Reg lo, hi;
970 lua_assert(st == IRT_NUM || st == IRT_FLOAT); 978 lj_assertA(st == IRT_NUM || st == IRT_FLOAT, "bad type for CONV");
971 lua_assert(dt == IRT_I64 || dt == IRT_U64); 979 lj_assertA(dt == IRT_I64 || dt == IRT_U64, "bad type for CONV");
972 hi = ra_dest(as, ir, RSET_GPR); 980 hi = ra_dest(as, ir, RSET_GPR);
973 lo = ra_dest(as, ir-1, rset_exclude(RSET_GPR, hi)); 981 lo = ra_dest(as, ir-1, rset_exclude(RSET_GPR, hi));
974 if (ra_used(ir-1)) emit_rmro(as, XO_MOV, lo, RID_ESP, 0); 982 if (ra_used(ir-1)) emit_rmro(as, XO_MOV, lo, RID_ESP, 0);
@@ -1180,13 +1188,13 @@ static void asm_href(ASMState *as, IRIns *ir, IROp merge)
1180 emit_rmro(as, XO_CMP, tmp|REX_64, dest, offsetof(Node, key.u64)); 1188 emit_rmro(as, XO_CMP, tmp|REX_64, dest, offsetof(Node, key.u64));
1181 } 1189 }
1182 } else { 1190 } else {
1183 lua_assert(irt_ispri(kt) && !irt_isnil(kt)); 1191 lj_assertA(irt_ispri(kt) && !irt_isnil(kt), "bad HREF key type");
1184 emit_u32(as, (irt_toitype(kt)<<15)|0x7fff); 1192 emit_u32(as, (irt_toitype(kt)<<15)|0x7fff);
1185 emit_rmro(as, XO_ARITHi, XOg_CMP, dest, offsetof(Node, key.it)); 1193 emit_rmro(as, XO_ARITHi, XOg_CMP, dest, offsetof(Node, key.it));
1186#else 1194#else
1187 } else { 1195 } else {
1188 if (!irt_ispri(kt)) { 1196 if (!irt_ispri(kt)) {
1189 lua_assert(irt_isaddr(kt)); 1197 lj_assertA(irt_isaddr(kt), "bad HREF key type");
1190 if (isk) 1198 if (isk)
1191 emit_gmroi(as, XG_ARITHi(XOg_CMP), dest, offsetof(Node, key.gcr), 1199 emit_gmroi(as, XG_ARITHi(XOg_CMP), dest, offsetof(Node, key.gcr),
1192 ptr2addr(ir_kgc(irkey))); 1200 ptr2addr(ir_kgc(irkey)));
@@ -1194,7 +1202,7 @@ static void asm_href(ASMState *as, IRIns *ir, IROp merge)
1194 emit_rmro(as, XO_CMP, key, dest, offsetof(Node, key.gcr)); 1202 emit_rmro(as, XO_CMP, key, dest, offsetof(Node, key.gcr));
1195 emit_sjcc(as, CC_NE, l_next); 1203 emit_sjcc(as, CC_NE, l_next);
1196 } 1204 }
1197 lua_assert(!irt_isnil(kt)); 1205 lj_assertA(!irt_isnil(kt), "bad HREF key type");
1198 emit_i8(as, irt_toitype(kt)); 1206 emit_i8(as, irt_toitype(kt));
1199 emit_rmro(as, XO_ARITHi8, XOg_CMP, dest, offsetof(Node, key.it)); 1207 emit_rmro(as, XO_ARITHi8, XOg_CMP, dest, offsetof(Node, key.it));
1200#endif 1208#endif
@@ -1209,7 +1217,7 @@ static void asm_href(ASMState *as, IRIns *ir, IROp merge)
1209#endif 1217#endif
1210 1218
1211 /* Load main position relative to tab->node into dest. */ 1219 /* Load main position relative to tab->node into dest. */
1212 khash = isk ? ir_khash(irkey) : 1; 1220 khash = isk ? ir_khash(as, irkey) : 1;
1213 if (khash == 0) { 1221 if (khash == 0) {
1214 emit_rmro(as, XO_MOV, dest|REX_GC64, tab, offsetof(GCtab, node)); 1222 emit_rmro(as, XO_MOV, dest|REX_GC64, tab, offsetof(GCtab, node));
1215 } else { 1223 } else {
@@ -1271,7 +1279,7 @@ static void asm_hrefk(ASMState *as, IRIns *ir)
1271#if !LJ_64 1279#if !LJ_64
1272 MCLabel l_exit; 1280 MCLabel l_exit;
1273#endif 1281#endif
1274 lua_assert(ofs % sizeof(Node) == 0); 1282 lj_assertA(ofs % sizeof(Node) == 0, "unaligned HREFK slot");
1275 if (ra_hasreg(dest)) { 1283 if (ra_hasreg(dest)) {
1276 if (ofs != 0) { 1284 if (ofs != 0) {
1277 if (dest == node) 1285 if (dest == node)
@@ -1288,7 +1296,8 @@ static void asm_hrefk(ASMState *as, IRIns *ir)
1288 Reg key = ra_scratch(as, rset_exclude(RSET_GPR, node)); 1296 Reg key = ra_scratch(as, rset_exclude(RSET_GPR, node));
1289 emit_rmro(as, XO_CMP, key|REX_64, node, 1297 emit_rmro(as, XO_CMP, key|REX_64, node,
1290 ofs + (int32_t)offsetof(Node, key.u64)); 1298 ofs + (int32_t)offsetof(Node, key.u64));
1291 lua_assert(irt_isnum(irkey->t) || irt_isgcv(irkey->t)); 1299 lj_assertA(irt_isnum(irkey->t) || irt_isgcv(irkey->t),
1300 "bad HREFK key type");
1292 /* Assumes -0.0 is already canonicalized to +0.0. */ 1301 /* Assumes -0.0 is already canonicalized to +0.0. */
1293 emit_loadu64(as, key, irt_isnum(irkey->t) ? ir_knum(irkey)->u64 : 1302 emit_loadu64(as, key, irt_isnum(irkey->t) ? ir_knum(irkey)->u64 :
1294#if LJ_GC64 1303#if LJ_GC64
@@ -1299,7 +1308,7 @@ static void asm_hrefk(ASMState *as, IRIns *ir)
1299 (uint64_t)(uint32_t)ptr2addr(ir_kgc(irkey))); 1308 (uint64_t)(uint32_t)ptr2addr(ir_kgc(irkey)));
1300#endif 1309#endif
1301 } else { 1310 } else {
1302 lua_assert(!irt_isnil(irkey->t)); 1311 lj_assertA(!irt_isnil(irkey->t), "bad HREFK key type");
1303#if LJ_GC64 1312#if LJ_GC64
1304 emit_i32(as, (irt_toitype(irkey->t)<<15)|0x7fff); 1313 emit_i32(as, (irt_toitype(irkey->t)<<15)|0x7fff);
1305 emit_rmro(as, XO_ARITHi, XOg_CMP, node, 1314 emit_rmro(as, XO_ARITHi, XOg_CMP, node,
@@ -1323,13 +1332,13 @@ static void asm_hrefk(ASMState *as, IRIns *ir)
1323 (int32_t)ir_knum(irkey)->u32.hi); 1332 (int32_t)ir_knum(irkey)->u32.hi);
1324 } else { 1333 } else {
1325 if (!irt_ispri(irkey->t)) { 1334 if (!irt_ispri(irkey->t)) {
1326 lua_assert(irt_isgcv(irkey->t)); 1335 lj_assertA(irt_isgcv(irkey->t), "bad HREFK key type");
1327 emit_gmroi(as, XG_ARITHi(XOg_CMP), node, 1336 emit_gmroi(as, XG_ARITHi(XOg_CMP), node,
1328 ofs + (int32_t)offsetof(Node, key.gcr), 1337 ofs + (int32_t)offsetof(Node, key.gcr),
1329 ptr2addr(ir_kgc(irkey))); 1338 ptr2addr(ir_kgc(irkey)));
1330 emit_sjcc(as, CC_NE, l_exit); 1339 emit_sjcc(as, CC_NE, l_exit);
1331 } 1340 }
1332 lua_assert(!irt_isnil(irkey->t)); 1341 lj_assertA(!irt_isnil(irkey->t), "bad HREFK key type");
1333 emit_i8(as, irt_toitype(irkey->t)); 1342 emit_i8(as, irt_toitype(irkey->t));
1334 emit_rmro(as, XO_ARITHi8, XOg_CMP, node, 1343 emit_rmro(as, XO_ARITHi8, XOg_CMP, node,
1335 ofs + (int32_t)offsetof(Node, key.it)); 1344 ofs + (int32_t)offsetof(Node, key.it));
@@ -1402,7 +1411,8 @@ static void asm_fxload(ASMState *as, IRIns *ir)
1402 if (LJ_64 && irt_is64(ir->t)) 1411 if (LJ_64 && irt_is64(ir->t))
1403 dest |= REX_64; 1412 dest |= REX_64;
1404 else 1413 else
1405 lua_assert(irt_isint(ir->t) || irt_isu32(ir->t) || irt_isaddr(ir->t)); 1414 lj_assertA(irt_isint(ir->t) || irt_isu32(ir->t) || irt_isaddr(ir->t),
1415 "unsplit 64 bit load");
1406 xo = XO_MOV; 1416 xo = XO_MOV;
1407 break; 1417 break;
1408 } 1418 }
@@ -1447,13 +1457,16 @@ static void asm_fxstore(ASMState *as, IRIns *ir)
1447 case IRT_NUM: xo = XO_MOVSDto; break; 1457 case IRT_NUM: xo = XO_MOVSDto; break;
1448 case IRT_FLOAT: xo = XO_MOVSSto; break; 1458 case IRT_FLOAT: xo = XO_MOVSSto; break;
1449#if LJ_64 && !LJ_GC64 1459#if LJ_64 && !LJ_GC64
1450 case IRT_LIGHTUD: lua_assert(0); /* NYI: mask 64 bit lightuserdata. */ 1460 case IRT_LIGHTUD:
1461 /* NYI: mask 64 bit lightuserdata. */
1462 lj_assertA(0, "store of lightuserdata");
1451#endif 1463#endif
1452 default: 1464 default:
1453 if (LJ_64 && irt_is64(ir->t)) 1465 if (LJ_64 && irt_is64(ir->t))
1454 src |= REX_64; 1466 src |= REX_64;
1455 else 1467 else
1456 lua_assert(irt_isint(ir->t) || irt_isu32(ir->t) || irt_isaddr(ir->t)); 1468 lj_assertA(irt_isint(ir->t) || irt_isu32(ir->t) || irt_isaddr(ir->t),
1469 "unsplit 64 bit store");
1457 xo = XO_MOVto; 1470 xo = XO_MOVto;
1458 break; 1471 break;
1459 } 1472 }
@@ -1467,8 +1480,8 @@ static void asm_fxstore(ASMState *as, IRIns *ir)
1467 emit_i8(as, k); 1480 emit_i8(as, k);
1468 emit_mrm(as, XO_MOVmib, 0, RID_MRM); 1481 emit_mrm(as, XO_MOVmib, 0, RID_MRM);
1469 } else { 1482 } else {
1470 lua_assert(irt_is64(ir->t) || irt_isint(ir->t) || irt_isu32(ir->t) || 1483 lj_assertA(irt_is64(ir->t) || irt_isint(ir->t) || irt_isu32(ir->t) ||
1471 irt_isaddr(ir->t)); 1484 irt_isaddr(ir->t), "bad store type");
1472 emit_i32(as, k); 1485 emit_i32(as, k);
1473 emit_mrm(as, XO_MOVmi, REX_64IR(ir, 0), RID_MRM); 1486 emit_mrm(as, XO_MOVmi, REX_64IR(ir, 0), RID_MRM);
1474 } 1487 }
@@ -1503,8 +1516,9 @@ static void asm_ahuvload(ASMState *as, IRIns *ir)
1503#if LJ_GC64 1516#if LJ_GC64
1504 Reg tmp = RID_NONE; 1517 Reg tmp = RID_NONE;
1505#endif 1518#endif
1506 lua_assert(irt_isnum(ir->t) || irt_ispri(ir->t) || irt_isaddr(ir->t) || 1519 lj_assertA(irt_isnum(ir->t) || irt_ispri(ir->t) || irt_isaddr(ir->t) ||
1507 (LJ_DUALNUM && irt_isint(ir->t))); 1520 (LJ_DUALNUM && irt_isint(ir->t)),
1521 "bad load type %d", irt_type(ir->t));
1508#if LJ_64 && !LJ_GC64 1522#if LJ_64 && !LJ_GC64
1509 if (irt_islightud(ir->t)) { 1523 if (irt_islightud(ir->t)) {
1510 Reg dest = asm_load_lightud64(as, ir, 1); 1524 Reg dest = asm_load_lightud64(as, ir, 1);
@@ -1551,7 +1565,8 @@ static void asm_ahuvload(ASMState *as, IRIns *ir)
1551 as->mrm.ofs += 4; 1565 as->mrm.ofs += 4;
1552 asm_guardcc(as, irt_isnum(ir->t) ? CC_AE : CC_NE); 1566 asm_guardcc(as, irt_isnum(ir->t) ? CC_AE : CC_NE);
1553 if (LJ_64 && irt_type(ir->t) >= IRT_NUM) { 1567 if (LJ_64 && irt_type(ir->t) >= IRT_NUM) {
1554 lua_assert(irt_isinteger(ir->t) || irt_isnum(ir->t)); 1568 lj_assertA(irt_isinteger(ir->t) || irt_isnum(ir->t),
1569 "bad load type %d", irt_type(ir->t));
1555#if LJ_GC64 1570#if LJ_GC64
1556 emit_u32(as, LJ_TISNUM << 15); 1571 emit_u32(as, LJ_TISNUM << 15);
1557#else 1572#else
@@ -1633,13 +1648,14 @@ static void asm_ahustore(ASMState *as, IRIns *ir)
1633#endif 1648#endif
1634 emit_mrm(as, XO_MOVto, src, RID_MRM); 1649 emit_mrm(as, XO_MOVto, src, RID_MRM);
1635 } else if (!irt_ispri(irr->t)) { 1650 } else if (!irt_ispri(irr->t)) {
1636 lua_assert(irt_isaddr(ir->t) || (LJ_DUALNUM && irt_isinteger(ir->t))); 1651 lj_assertA(irt_isaddr(ir->t) || (LJ_DUALNUM && irt_isinteger(ir->t)),
1652 "bad store type");
1637 emit_i32(as, irr->i); 1653 emit_i32(as, irr->i);
1638 emit_mrm(as, XO_MOVmi, 0, RID_MRM); 1654 emit_mrm(as, XO_MOVmi, 0, RID_MRM);
1639 } 1655 }
1640 as->mrm.ofs += 4; 1656 as->mrm.ofs += 4;
1641#if LJ_GC64 1657#if LJ_GC64
1642 lua_assert(LJ_DUALNUM && irt_isinteger(ir->t)); 1658 lj_assertA(LJ_DUALNUM && irt_isinteger(ir->t), "bad store type");
1643 emit_i32(as, LJ_TNUMX << 15); 1659 emit_i32(as, LJ_TNUMX << 15);
1644#else 1660#else
1645 emit_i32(as, (int32_t)irt_toitype(ir->t)); 1661 emit_i32(as, (int32_t)irt_toitype(ir->t));
@@ -1654,10 +1670,13 @@ static void asm_sload(ASMState *as, IRIns *ir)
1654 (!LJ_FR2 && (ir->op2 & IRSLOAD_FRAME) ? 4 : 0); 1670 (!LJ_FR2 && (ir->op2 & IRSLOAD_FRAME) ? 4 : 0);
1655 IRType1 t = ir->t; 1671 IRType1 t = ir->t;
1656 Reg base; 1672 Reg base;
1657 lua_assert(!(ir->op2 & IRSLOAD_PARENT)); /* Handled by asm_head_side(). */ 1673 lj_assertA(!(ir->op2 & IRSLOAD_PARENT),
1658 lua_assert(irt_isguard(t) || !(ir->op2 & IRSLOAD_TYPECHECK)); 1674 "bad parent SLOAD"); /* Handled by asm_head_side(). */
1659 lua_assert(LJ_DUALNUM || 1675 lj_assertA(irt_isguard(t) || !(ir->op2 & IRSLOAD_TYPECHECK),
1660 !irt_isint(t) || (ir->op2 & (IRSLOAD_CONVERT|IRSLOAD_FRAME))); 1676 "inconsistent SLOAD variant");
1677 lj_assertA(LJ_DUALNUM ||
1678 !irt_isint(t) || (ir->op2 & (IRSLOAD_CONVERT|IRSLOAD_FRAME)),
1679 "bad SLOAD type");
1661 if ((ir->op2 & IRSLOAD_CONVERT) && irt_isguard(t) && irt_isint(t)) { 1680 if ((ir->op2 & IRSLOAD_CONVERT) && irt_isguard(t) && irt_isint(t)) {
1662 Reg left = ra_scratch(as, RSET_FPR); 1681 Reg left = ra_scratch(as, RSET_FPR);
1663 asm_tointg(as, ir, left); /* Frees dest reg. Do this before base alloc. */ 1682 asm_tointg(as, ir, left); /* Frees dest reg. Do this before base alloc. */
@@ -1677,7 +1696,8 @@ static void asm_sload(ASMState *as, IRIns *ir)
1677 RegSet allow = irt_isnum(t) ? RSET_FPR : RSET_GPR; 1696 RegSet allow = irt_isnum(t) ? RSET_FPR : RSET_GPR;
1678 Reg dest = ra_dest(as, ir, allow); 1697 Reg dest = ra_dest(as, ir, allow);
1679 base = ra_alloc1(as, REF_BASE, RSET_GPR); 1698 base = ra_alloc1(as, REF_BASE, RSET_GPR);
1680 lua_assert(irt_isnum(t) || irt_isint(t) || irt_isaddr(t)); 1699 lj_assertA(irt_isnum(t) || irt_isint(t) || irt_isaddr(t),
1700 "bad SLOAD type %d", irt_type(t));
1681 if ((ir->op2 & IRSLOAD_CONVERT)) { 1701 if ((ir->op2 & IRSLOAD_CONVERT)) {
1682 t.irt = irt_isint(t) ? IRT_NUM : IRT_INT; /* Check for original type. */ 1702 t.irt = irt_isint(t) ? IRT_NUM : IRT_INT; /* Check for original type. */
1683 emit_rmro(as, irt_isint(t) ? XO_CVTSI2SD : XO_CVTTSD2SI, dest, base, ofs); 1703 emit_rmro(as, irt_isint(t) ? XO_CVTSI2SD : XO_CVTTSD2SI, dest, base, ofs);
@@ -1723,7 +1743,8 @@ static void asm_sload(ASMState *as, IRIns *ir)
1723 /* Need type check, even if the load result is unused. */ 1743 /* Need type check, even if the load result is unused. */
1724 asm_guardcc(as, irt_isnum(t) ? CC_AE : CC_NE); 1744 asm_guardcc(as, irt_isnum(t) ? CC_AE : CC_NE);
1725 if (LJ_64 && irt_type(t) >= IRT_NUM) { 1745 if (LJ_64 && irt_type(t) >= IRT_NUM) {
1726 lua_assert(irt_isinteger(t) || irt_isnum(t)); 1746 lj_assertA(irt_isinteger(t) || irt_isnum(t),
1747 "bad SLOAD type %d", irt_type(t));
1727#if LJ_GC64 1748#if LJ_GC64
1728 emit_u32(as, LJ_TISNUM << 15); 1749 emit_u32(as, LJ_TISNUM << 15);
1729#else 1750#else
@@ -1775,7 +1796,8 @@ static void asm_cnew(ASMState *as, IRIns *ir)
1775 CTInfo info = lj_ctype_info(cts, id, &sz); 1796 CTInfo info = lj_ctype_info(cts, id, &sz);
1776 const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_mem_newgco]; 1797 const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_mem_newgco];
1777 IRRef args[4]; 1798 IRRef args[4];
1778 lua_assert(sz != CTSIZE_INVALID || (ir->o == IR_CNEW && ir->op2 != REF_NIL)); 1799 lj_assertA(sz != CTSIZE_INVALID || (ir->o == IR_CNEW && ir->op2 != REF_NIL),
1800 "bad CNEW/CNEWI operands");
1779 1801
1780 as->gcsteps++; 1802 as->gcsteps++;
1781 asm_setupresult(as, ir, ci); /* GCcdata * */ 1803 asm_setupresult(as, ir, ci); /* GCcdata * */
@@ -1805,7 +1827,7 @@ static void asm_cnew(ASMState *as, IRIns *ir)
1805 int32_t ofs = sizeof(GCcdata); 1827 int32_t ofs = sizeof(GCcdata);
1806 if (sz == 8) { 1828 if (sz == 8) {
1807 ofs += 4; ir++; 1829 ofs += 4; ir++;
1808 lua_assert(ir->o == IR_HIOP); 1830 lj_assertA(ir->o == IR_HIOP, "missing CNEWI HIOP");
1809 } 1831 }
1810 do { 1832 do {
1811 if (irref_isk(ir->op2)) { 1833 if (irref_isk(ir->op2)) {
@@ -1819,7 +1841,7 @@ static void asm_cnew(ASMState *as, IRIns *ir)
1819 ofs -= 4; ir--; 1841 ofs -= 4; ir--;
1820 } while (1); 1842 } while (1);
1821#endif 1843#endif
1822 lua_assert(sz == 4 || sz == 8); 1844 lj_assertA(sz == 4 || sz == 8, "bad CNEWI size %d", sz);
1823 } else if (ir->op2 != REF_NIL) { /* Create VLA/VLS/aligned cdata. */ 1845 } else if (ir->op2 != REF_NIL) { /* Create VLA/VLS/aligned cdata. */
1824 ci = &lj_ir_callinfo[IRCALL_lj_cdata_newv]; 1846 ci = &lj_ir_callinfo[IRCALL_lj_cdata_newv];
1825 args[0] = ASMREF_L; /* lua_State *L */ 1847 args[0] = ASMREF_L; /* lua_State *L */
@@ -1869,7 +1891,7 @@ static void asm_obar(ASMState *as, IRIns *ir)
1869 MCLabel l_end; 1891 MCLabel l_end;
1870 Reg obj; 1892 Reg obj;
1871 /* No need for other object barriers (yet). */ 1893 /* No need for other object barriers (yet). */
1872 lua_assert(IR(ir->op1)->o == IR_UREFC); 1894 lj_assertA(IR(ir->op1)->o == IR_UREFC, "bad OBAR type");
1873 ra_evictset(as, RSET_SCRATCH); 1895 ra_evictset(as, RSET_SCRATCH);
1874 l_end = emit_label(as); 1896 l_end = emit_label(as);
1875 args[0] = ASMREF_TMP1; /* global_State *g */ 1897 args[0] = ASMREF_TMP1; /* global_State *g */
@@ -1986,7 +2008,7 @@ static int asm_swapops(ASMState *as, IRIns *ir)
1986{ 2008{
1987 IRIns *irl = IR(ir->op1); 2009 IRIns *irl = IR(ir->op1);
1988 IRIns *irr = IR(ir->op2); 2010 IRIns *irr = IR(ir->op2);
1989 lua_assert(ra_noreg(irr->r)); 2011 lj_assertA(ra_noreg(irr->r), "bad usage");
1990 if (!irm_iscomm(lj_ir_mode[ir->o])) 2012 if (!irm_iscomm(lj_ir_mode[ir->o]))
1991 return 0; /* Can't swap non-commutative operations. */ 2013 return 0; /* Can't swap non-commutative operations. */
1992 if (irref_isk(ir->op2)) 2014 if (irref_isk(ir->op2))
@@ -2376,8 +2398,9 @@ static void asm_comp(ASMState *as, IRIns *ir)
2376 IROp leftop = (IROp)(IR(lref)->o); 2398 IROp leftop = (IROp)(IR(lref)->o);
2377 Reg r64 = REX_64IR(ir, 0); 2399 Reg r64 = REX_64IR(ir, 0);
2378 int32_t imm = 0; 2400 int32_t imm = 0;
2379 lua_assert(irt_is64(ir->t) || irt_isint(ir->t) || 2401 lj_assertA(irt_is64(ir->t) || irt_isint(ir->t) ||
2380 irt_isu32(ir->t) || irt_isaddr(ir->t) || irt_isu8(ir->t)); 2402 irt_isu32(ir->t) || irt_isaddr(ir->t) || irt_isu8(ir->t),
2403 "bad comparison data type %d", irt_type(ir->t));
2381 /* Swap constants (only for ABC) and fusable loads to the right. */ 2404 /* Swap constants (only for ABC) and fusable loads to the right. */
2382 if (irref_isk(lref) || (!irref_isk(rref) && opisfusableload(leftop))) { 2405 if (irref_isk(lref) || (!irref_isk(rref) && opisfusableload(leftop))) {
2383 if ((cc & 0xc) == 0xc) cc ^= 0x53; /* L <-> G, LE <-> GE */ 2406 if ((cc & 0xc) == 0xc) cc ^= 0x53; /* L <-> G, LE <-> GE */
@@ -2459,7 +2482,7 @@ static void asm_comp(ASMState *as, IRIns *ir)
2459 /* Use test r,r instead of cmp r,0. */ 2482 /* Use test r,r instead of cmp r,0. */
2460 x86Op xo = XO_TEST; 2483 x86Op xo = XO_TEST;
2461 if (irt_isu8(ir->t)) { 2484 if (irt_isu8(ir->t)) {
2462 lua_assert(ir->o == IR_EQ || ir->o == IR_NE); 2485 lj_assertA(ir->o == IR_EQ || ir->o == IR_NE, "bad usage");
2463 xo = XO_TESTb; 2486 xo = XO_TESTb;
2464 if (!rset_test(RSET_RANGE(RID_EAX, RID_EBX+1), left)) { 2487 if (!rset_test(RSET_RANGE(RID_EAX, RID_EBX+1), left)) {
2465 if (LJ_64) { 2488 if (LJ_64) {
@@ -2615,10 +2638,11 @@ static void asm_hiop(ASMState *as, IRIns *ir)
2615 case IR_CNEWI: 2638 case IR_CNEWI:
2616 /* Nothing to do here. Handled by CNEWI itself. */ 2639 /* Nothing to do here. Handled by CNEWI itself. */
2617 break; 2640 break;
2618 default: lua_assert(0); break; 2641 default: lj_assertA(0, "bad HIOP for op %d", (ir-1)->o); break;
2619 } 2642 }
2620#else 2643#else
2621 UNUSED(as); UNUSED(ir); lua_assert(0); /* Unused on x64 or without FFI. */ 2644 /* Unused on x64 or without FFI. */
2645 UNUSED(as); UNUSED(ir); lj_assertA(0, "unexpected HIOP");
2622#endif 2646#endif
2623} 2647}
2624 2648
@@ -2684,8 +2708,9 @@ static void asm_stack_restore(ASMState *as, SnapShot *snap)
2684 Reg src = ra_alloc1(as, ref, RSET_FPR); 2708 Reg src = ra_alloc1(as, ref, RSET_FPR);
2685 emit_rmro(as, XO_MOVSDto, src, RID_BASE, ofs); 2709 emit_rmro(as, XO_MOVSDto, src, RID_BASE, ofs);
2686 } else { 2710 } else {
2687 lua_assert(irt_ispri(ir->t) || irt_isaddr(ir->t) || 2711 lj_assertA(irt_ispri(ir->t) || irt_isaddr(ir->t) ||
2688 (LJ_DUALNUM && irt_isinteger(ir->t))); 2712 (LJ_DUALNUM && irt_isinteger(ir->t)),
2713 "restore of IR type %d", irt_type(ir->t));
2689 if (!irref_isk(ref)) { 2714 if (!irref_isk(ref)) {
2690 Reg src = ra_alloc1(as, ref, rset_exclude(RSET_GPR, RID_BASE)); 2715 Reg src = ra_alloc1(as, ref, rset_exclude(RSET_GPR, RID_BASE));
2691#if LJ_GC64 2716#if LJ_GC64
@@ -2730,7 +2755,7 @@ static void asm_stack_restore(ASMState *as, SnapShot *snap)
2730 } 2755 }
2731 checkmclim(as); 2756 checkmclim(as);
2732 } 2757 }
2733 lua_assert(map + nent == flinks); 2758 lj_assertA(map + nent == flinks, "inconsistent frames in snapshot");
2734} 2759}
2735 2760
2736/* -- GC handling --------------------------------------------------------- */ 2761/* -- GC handling --------------------------------------------------------- */
@@ -2774,16 +2799,16 @@ static void asm_loop_fixup(ASMState *as)
2774 MCode *target = as->mcp; 2799 MCode *target = as->mcp;
2775 if (as->realign) { /* Realigned loops use short jumps. */ 2800 if (as->realign) { /* Realigned loops use short jumps. */
2776 as->realign = NULL; /* Stop another retry. */ 2801 as->realign = NULL; /* Stop another retry. */
2777 lua_assert(((intptr_t)target & 15) == 0); 2802 lj_assertA(((intptr_t)target & 15) == 0, "loop realign failed");
2778 if (as->loopinv) { /* Inverted loop branch? */ 2803 if (as->loopinv) { /* Inverted loop branch? */
2779 p -= 5; 2804 p -= 5;
2780 p[0] = XI_JMP; 2805 p[0] = XI_JMP;
2781 lua_assert(target - p >= -128); 2806 lj_assertA(target - p >= -128, "loop realign failed");
2782 p[-1] = (MCode)(target - p); /* Patch sjcc. */ 2807 p[-1] = (MCode)(target - p); /* Patch sjcc. */
2783 if (as->loopinv == 2) 2808 if (as->loopinv == 2)
2784 p[-3] = (MCode)(target - p + 2); /* Patch opt. short jp. */ 2809 p[-3] = (MCode)(target - p + 2); /* Patch opt. short jp. */
2785 } else { 2810 } else {
2786 lua_assert(target - p >= -128); 2811 lj_assertA(target - p >= -128, "loop realign failed");
2787 p[-1] = (MCode)(int8_t)(target - p); /* Patch short jmp. */ 2812 p[-1] = (MCode)(int8_t)(target - p); /* Patch short jmp. */
2788 p[-2] = XI_JMPs; 2813 p[-2] = XI_JMPs;
2789 } 2814 }
@@ -2880,7 +2905,7 @@ static void asm_tail_fixup(ASMState *as, TraceNo lnk)
2880 } 2905 }
2881 /* Patch exit branch. */ 2906 /* Patch exit branch. */
2882 target = lnk ? traceref(as->J, lnk)->mcode : (MCode *)lj_vm_exit_interp; 2907 target = lnk ? traceref(as->J, lnk)->mcode : (MCode *)lj_vm_exit_interp;
2883 *(int32_t *)(p-4) = jmprel(p, target); 2908 *(int32_t *)(p-4) = jmprel(as->J, p, target);
2884 p[-5] = XI_JMP; 2909 p[-5] = XI_JMP;
2885 /* Drop unused mcode tail. Fill with NOPs to make the prefetcher happy. */ 2910 /* Drop unused mcode tail. Fill with NOPs to make the prefetcher happy. */
2886 for (q = as->mctop-1; q >= p; q--) 2911 for (q = as->mctop-1; q >= p; q--)
@@ -3053,17 +3078,17 @@ void lj_asm_patchexit(jit_State *J, GCtrace *T, ExitNo exitno, MCode *target)
3053 uint32_t statei = u32ptr(&J2G(J)->vmstate); 3078 uint32_t statei = u32ptr(&J2G(J)->vmstate);
3054#endif 3079#endif
3055 if (len > 5 && p[len-5] == XI_JMP && p+len-6 + *(int32_t *)(p+len-4) == px) 3080 if (len > 5 && p[len-5] == XI_JMP && p+len-6 + *(int32_t *)(p+len-4) == px)
3056 *(int32_t *)(p+len-4) = jmprel(p+len, target); 3081 *(int32_t *)(p+len-4) = jmprel(J, p+len, target);
3057 /* Do not patch parent exit for a stack check. Skip beyond vmstate update. */ 3082 /* Do not patch parent exit for a stack check. Skip beyond vmstate update. */
3058 for (; p < pe; p += asm_x86_inslen(p)) { 3083 for (; p < pe; p += asm_x86_inslen(p)) {
3059 intptr_t ofs = LJ_GC64 ? (p[0] & 0xf0) == 0x40 : LJ_64; 3084 intptr_t ofs = LJ_GC64 ? (p[0] & 0xf0) == 0x40 : LJ_64;
3060 if (*(uint32_t *)(p+2+ofs) == statei && p[ofs+LJ_GC64-LJ_64] == XI_MOVmi) 3085 if (*(uint32_t *)(p+2+ofs) == statei && p[ofs+LJ_GC64-LJ_64] == XI_MOVmi)
3061 break; 3086 break;
3062 } 3087 }
3063 lua_assert(p < pe); 3088 lj_assertJ(p < pe, "instruction length decoder failed");
3064 for (; p < pe; p += asm_x86_inslen(p)) 3089 for (; p < pe; p += asm_x86_inslen(p))
3065 if ((*(uint16_t *)p & 0xf0ff) == 0x800f && p + *(int32_t *)(p+2) == px) 3090 if ((*(uint16_t *)p & 0xf0ff) == 0x800f && p + *(int32_t *)(p+2) == px)
3066 *(int32_t *)(p+2) = jmprel(p+6, target); 3091 *(int32_t *)(p+2) = jmprel(J, p+6, target);
3067 lj_mcode_sync(T->mcode, T->mcode + T->szmcode); 3092 lj_mcode_sync(T->mcode, T->mcode + T->szmcode);
3068 lj_mcode_patch(J, mcarea, 1); 3093 lj_mcode_patch(J, mcarea, 1);
3069} 3094}