diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/lj_opt_narrow.c | 63 |
1 files changed, 35 insertions, 28 deletions
diff --git a/src/lj_opt_narrow.c b/src/lj_opt_narrow.c index 50dd97e6..1938766c 100644 --- a/src/lj_opt_narrow.c +++ b/src/lj_opt_narrow.c | |||
| @@ -166,16 +166,6 @@ | |||
| 166 | #define NARROW_MAX_BACKPROP 100 | 166 | #define NARROW_MAX_BACKPROP 100 |
| 167 | #define NARROW_MAX_STACK 256 | 167 | #define NARROW_MAX_STACK 256 |
| 168 | 168 | ||
| 169 | /* Context used for narrowing of type conversions. */ | ||
| 170 | typedef struct NarrowConv { | ||
| 171 | jit_State *J; /* JIT compiler state. */ | ||
| 172 | IRRef2 *sp; /* Current stack pointer. */ | ||
| 173 | IRRef2 *maxsp; /* Maximum stack pointer minus redzone. */ | ||
| 174 | int lim; /* Limit on the number of emitted conversions. */ | ||
| 175 | IRRef mode; /* Conversion mode (IRTOINT_*). */ | ||
| 176 | IRRef2 stack[NARROW_MAX_STACK]; /* Stack holding the stack-machine code. */ | ||
| 177 | } NarrowConv; | ||
| 178 | |||
| 179 | /* The stack machine has a 32 bit instruction format: [IROpT | IRRef1] | 169 | /* The stack machine has a 32 bit instruction format: [IROpT | IRRef1] |
| 180 | ** The lower 16 bits hold a reference (or 0). The upper 16 bits hold | 170 | ** The lower 16 bits hold a reference (or 0). The upper 16 bits hold |
| 181 | ** the IR opcode + type or one of the following special opcodes: | 171 | ** the IR opcode + type or one of the following special opcodes: |
| @@ -186,6 +176,22 @@ enum { | |||
| 186 | NARROW_INT /* Push KINT ref. The next code holds an int32_t. */ | 176 | NARROW_INT /* Push KINT ref. The next code holds an int32_t. */ |
| 187 | }; | 177 | }; |
| 188 | 178 | ||
| 179 | typedef uint32_t NarrowIns; | ||
| 180 | |||
| 181 | #define NARROWINS(op, ref) (((op) << 16) + (ref)) | ||
| 182 | #define narrow_op(ins) ((IROpT)((ins) >> 16)) | ||
| 183 | #define narrow_ref(ins) ((IRRef1)(ins)) | ||
| 184 | |||
| 185 | /* Context used for narrowing of type conversions. */ | ||
| 186 | typedef struct NarrowConv { | ||
| 187 | jit_State *J; /* JIT compiler state. */ | ||
| 188 | NarrowIns *sp; /* Current stack pointer. */ | ||
| 189 | NarrowIns *maxsp; /* Maximum stack pointer minus redzone. */ | ||
| 190 | int lim; /* Limit on the number of emitted conversions. */ | ||
| 191 | IRRef mode; /* Conversion mode (IRTOINT_*). */ | ||
| 192 | NarrowIns stack[NARROW_MAX_STACK]; /* Stack holding stack-machine code. */ | ||
| 193 | } NarrowConv; | ||
| 194 | |||
| 189 | /* Lookup a reference in the backpropagation cache. */ | 195 | /* Lookup a reference in the backpropagation cache. */ |
| 190 | static IRRef narrow_bpc_get(jit_State *J, IRRef1 key, IRRef mode) | 196 | static IRRef narrow_bpc_get(jit_State *J, IRRef1 key, IRRef mode) |
| 191 | { | 197 | { |
| @@ -218,22 +224,22 @@ static int narrow_conv_backprop(NarrowConv *nc, IRRef ref, int depth) | |||
| 218 | 224 | ||
| 219 | /* Check the easy cases first. */ | 225 | /* Check the easy cases first. */ |
| 220 | if (ir->o == IR_TONUM) { /* Undo inverse conversion. */ | 226 | if (ir->o == IR_TONUM) { /* Undo inverse conversion. */ |
| 221 | *nc->sp++ = IRREF2(ir->op1, NARROW_REF); | 227 | *nc->sp++ = NARROWINS(NARROW_REF, ir->op1); |
| 222 | return 0; | 228 | return 0; |
| 223 | } else if (ir->o == IR_KNUM) { /* Narrow FP constant. */ | 229 | } else if (ir->o == IR_KNUM) { /* Narrow FP constant. */ |
| 224 | lua_Number n = ir_knum(ir)->n; | 230 | lua_Number n = ir_knum(ir)->n; |
| 225 | if (nc->mode == IRTOINT_TOBIT) { /* Allows a wider range of constants. */ | 231 | if (nc->mode == IRTOINT_TOBIT) { /* Allows a wider range of constants. */ |
| 226 | int64_t k64 = (int64_t)n; | 232 | int64_t k64 = (int64_t)n; |
| 227 | if (n == cast_num(k64)) { /* Only if constant doesn't lose precision. */ | 233 | if (n == cast_num(k64)) { /* Only if constant doesn't lose precision. */ |
| 228 | *nc->sp++ = IRREF2(0, NARROW_INT); | 234 | *nc->sp++ = NARROWINS(NARROW_INT, 0); |
| 229 | *nc->sp++ = (IRRef2)k64; /* But always truncate to 32 bits. */ | 235 | *nc->sp++ = (NarrowIns)k64; /* But always truncate to 32 bits. */ |
| 230 | return 0; | 236 | return 0; |
| 231 | } | 237 | } |
| 232 | } else { | 238 | } else { |
| 233 | int32_t k = lj_num2int(n); | 239 | int32_t k = lj_num2int(n); |
| 234 | if (n == cast_num(k)) { /* Only if constant is really an integer. */ | 240 | if (n == cast_num(k)) { /* Only if constant is really an integer. */ |
| 235 | *nc->sp++ = IRREF2(0, NARROW_INT); | 241 | *nc->sp++ = NARROWINS(NARROW_INT, 0); |
| 236 | *nc->sp++ = (IRRef2)k; | 242 | *nc->sp++ = (NarrowIns)k; |
| 237 | return 0; | 243 | return 0; |
| 238 | } | 244 | } |
| 239 | } | 245 | } |
| @@ -244,7 +250,7 @@ static int narrow_conv_backprop(NarrowConv *nc, IRRef ref, int depth) | |||
| 244 | for (cref = J->chain[fins->o]; cref > ref; cref = IR(cref)->prev) | 250 | for (cref = J->chain[fins->o]; cref > ref; cref = IR(cref)->prev) |
| 245 | if (IR(cref)->op1 == ref && | 251 | if (IR(cref)->op1 == ref && |
| 246 | irt_isguard(IR(cref)->t) >= irt_isguard(fins->t)) { | 252 | irt_isguard(IR(cref)->t) >= irt_isguard(fins->t)) { |
| 247 | *nc->sp++ = IRREF2(cref, NARROW_REF); | 253 | *nc->sp++ = NARROWINS(NARROW_REF, cref); |
| 248 | return 0; /* Already there, no additional conversion needed. */ | 254 | return 0; /* Already there, no additional conversion needed. */ |
| 249 | } | 255 | } |
| 250 | 256 | ||
| @@ -256,15 +262,15 @@ static int narrow_conv_backprop(NarrowConv *nc, IRRef ref, int depth) | |||
| 256 | mode = IRTOINT_CHECK; /* Inner conversions need a stronger check. */ | 262 | mode = IRTOINT_CHECK; /* Inner conversions need a stronger check. */ |
| 257 | bpref = narrow_bpc_get(nc->J, (IRRef1)ref, mode); | 263 | bpref = narrow_bpc_get(nc->J, (IRRef1)ref, mode); |
| 258 | if (bpref) { | 264 | if (bpref) { |
| 259 | *nc->sp++ = IRREF2(bpref, NARROW_REF); | 265 | *nc->sp++ = NARROWINS(NARROW_REF, bpref); |
| 260 | return 0; | 266 | return 0; |
| 261 | } | 267 | } |
| 262 | if (++depth < NARROW_MAX_BACKPROP && nc->sp < nc->maxsp) { | 268 | if (++depth < NARROW_MAX_BACKPROP && nc->sp < nc->maxsp) { |
| 263 | IRRef2 *savesp = nc->sp; | 269 | NarrowIns *savesp = nc->sp; |
| 264 | int count = narrow_conv_backprop(nc, ir->op1, depth); | 270 | int count = narrow_conv_backprop(nc, ir->op1, depth); |
| 265 | count += narrow_conv_backprop(nc, ir->op2, depth); | 271 | count += narrow_conv_backprop(nc, ir->op2, depth); |
| 266 | if (count <= nc->lim) { /* Limit total number of conversions. */ | 272 | if (count <= nc->lim) { /* Limit total number of conversions. */ |
| 267 | *nc->sp++ = IRREF2(ref, IRTI(ir->o)); | 273 | *nc->sp++ = NARROWINS(IRTI(ir->o), ref); |
| 268 | return count; | 274 | return count; |
| 269 | } | 275 | } |
| 270 | nc->sp = savesp; /* Too many conversions, need to backtrack. */ | 276 | nc->sp = savesp; /* Too many conversions, need to backtrack. */ |
| @@ -272,7 +278,7 @@ static int narrow_conv_backprop(NarrowConv *nc, IRRef ref, int depth) | |||
| 272 | } | 278 | } |
| 273 | 279 | ||
| 274 | /* Otherwise add a conversion. */ | 280 | /* Otherwise add a conversion. */ |
| 275 | *nc->sp++ = IRREF2(ref, NARROW_CONV); | 281 | *nc->sp++ = NARROWINS(NARROW_CONV, ref); |
| 276 | return 1; | 282 | return 1; |
| 277 | } | 283 | } |
| 278 | 284 | ||
| @@ -283,12 +289,12 @@ static IRRef narrow_conv_emit(jit_State *J, NarrowConv *nc) | |||
| 283 | IROpT guardot = irt_isguard(fins->t) ? IRTG(IR_ADDOV-IR_ADD, 0) : 0; | 289 | IROpT guardot = irt_isguard(fins->t) ? IRTG(IR_ADDOV-IR_ADD, 0) : 0; |
| 284 | IROpT convot = fins->ot; | 290 | IROpT convot = fins->ot; |
| 285 | IRRef1 convop2 = fins->op2; | 291 | IRRef1 convop2 = fins->op2; |
| 286 | IRRef2 *next = nc->stack; /* List of instructions from backpropagation. */ | 292 | NarrowIns *next = nc->stack; /* List of instructions from backpropagation. */ |
| 287 | IRRef2 *last = nc->sp; | 293 | NarrowIns *last = nc->sp; |
| 288 | IRRef2 *sp = nc->stack; /* Recycle the stack to store operands. */ | 294 | NarrowIns *sp = nc->stack; /* Recycle the stack to store operands. */ |
| 289 | while (next < last) { /* Simple stack machine to process the ins. list. */ | 295 | while (next < last) { /* Simple stack machine to process the ins. list. */ |
| 290 | IRRef2 ref = *next++; | 296 | NarrowIns ref = *next++; |
| 291 | IROpT op = ref >> 16; | 297 | IROpT op = narrow_op(ref); |
| 292 | if (op == NARROW_REF) { | 298 | if (op == NARROW_REF) { |
| 293 | *sp++ = ref; | 299 | *sp++ = ref; |
| 294 | } else if (op == NARROW_CONV) { | 300 | } else if (op == NARROW_CONV) { |
| @@ -302,14 +308,15 @@ static IRRef narrow_conv_emit(jit_State *J, NarrowConv *nc) | |||
| 302 | sp--; | 308 | sp--; |
| 303 | /* Omit some overflow checks for array indexing. See comments above. */ | 309 | /* Omit some overflow checks for array indexing. See comments above. */ |
| 304 | if (mode == IRTOINT_INDEX) { | 310 | if (mode == IRTOINT_INDEX) { |
| 305 | if (next == last && irref_isk((IRRef1)sp[0]) && | 311 | if (next == last && irref_isk(narrow_ref(sp[0])) && |
| 306 | (uint32_t)IR((IRRef1)sp[0])->i + 0x40000000 < 0x80000000) | 312 | (uint32_t)IR(narrow_ref(sp[0]))->i + 0x40000000 < 0x80000000) |
| 307 | guardot = 0; | 313 | guardot = 0; |
| 308 | else | 314 | else |
| 309 | mode = IRTOINT_CHECK; /* Otherwise cache a stronger check. */ | 315 | mode = IRTOINT_CHECK; /* Otherwise cache a stronger check. */ |
| 310 | } | 316 | } |
| 311 | sp[-1] = emitir(op+guardot, sp[-1], sp[0]); | 317 | sp[-1] = emitir(op+guardot, sp[-1], sp[0]); |
| 312 | narrow_bpc_set(J, (IRRef1)ref, (IRRef1)sp[-1], mode); /* Add to cache. */ | 318 | /* Add to cache. */ |
| 319 | narrow_bpc_set(J, narrow_ref(ref), narrow_ref(sp[-1]), mode); | ||
| 313 | } | 320 | } |
| 314 | } | 321 | } |
| 315 | lua_assert(sp == nc->stack+1); | 322 | lua_assert(sp == nc->stack+1); |
