summaryrefslogtreecommitdiff
path: root/src/lib_jit.c
diff options
context:
space:
mode:
Diffstat (limited to '')
-rw-r--r--src/lib_jit.c236
1 files changed, 167 insertions, 69 deletions
diff --git a/src/lib_jit.c b/src/lib_jit.c
index 04a564c7..c0294927 100644
--- a/src/lib_jit.c
+++ b/src/lib_jit.c
@@ -10,13 +10,17 @@
10#include "lauxlib.h" 10#include "lauxlib.h"
11#include "lualib.h" 11#include "lualib.h"
12 12
13#include "lj_arch.h"
14#include "lj_obj.h" 13#include "lj_obj.h"
14#include "lj_gc.h"
15#include "lj_err.h" 15#include "lj_err.h"
16#include "lj_debug.h" 16#include "lj_debug.h"
17#include "lj_str.h" 17#include "lj_str.h"
18#include "lj_tab.h" 18#include "lj_tab.h"
19#include "lj_state.h"
19#include "lj_bc.h" 20#include "lj_bc.h"
21#if LJ_HASFFI
22#include "lj_ctype.h"
23#endif
20#if LJ_HASJIT 24#if LJ_HASJIT
21#include "lj_ir.h" 25#include "lj_ir.h"
22#include "lj_jit.h" 26#include "lj_jit.h"
@@ -24,6 +28,7 @@
24#include "lj_iropt.h" 28#include "lj_iropt.h"
25#include "lj_target.h" 29#include "lj_target.h"
26#endif 30#endif
31#include "lj_trace.h"
27#include "lj_dispatch.h" 32#include "lj_dispatch.h"
28#include "lj_vm.h" 33#include "lj_vm.h"
29#include "lj_vmevent.h" 34#include "lj_vmevent.h"
@@ -99,8 +104,8 @@ LJLIB_CF(jit_status)
99 jit_State *J = L2J(L); 104 jit_State *J = L2J(L);
100 L->top = L->base; 105 L->top = L->base;
101 setboolV(L->top++, (J->flags & JIT_F_ON) ? 1 : 0); 106 setboolV(L->top++, (J->flags & JIT_F_ON) ? 1 : 0);
102 flagbits_to_strings(L, J->flags, JIT_F_CPU_FIRST, JIT_F_CPUSTRING); 107 flagbits_to_strings(L, J->flags, JIT_F_CPU, JIT_F_CPUSTRING);
103 flagbits_to_strings(L, J->flags, JIT_F_OPT_FIRST, JIT_F_OPTSTRING); 108 flagbits_to_strings(L, J->flags, JIT_F_OPT, JIT_F_OPTSTRING);
104 return (int)(L->top - L->base); 109 return (int)(L->top - L->base);
105#else 110#else
106 setboolV(L->top++, 0); 111 setboolV(L->top++, 0);
@@ -108,6 +113,13 @@ LJLIB_CF(jit_status)
108#endif 113#endif
109} 114}
110 115
116LJLIB_CF(jit_security)
117{
118 int idx = lj_lib_checkopt(L, 1, -1, LJ_SECURITY_MODESTRING);
119 setintV(L->top++, ((LJ_SECURITY_MODE >> (2*idx)) & 3));
120 return 1;
121}
122
111LJLIB_CF(jit_attach) 123LJLIB_CF(jit_attach)
112{ 124{
113#ifdef LUAJIT_DISABLE_VMEVENT 125#ifdef LUAJIT_DISABLE_VMEVENT
@@ -222,7 +234,7 @@ LJLIB_CF(jit_util_funcbc)
222 if (pc < pt->sizebc) { 234 if (pc < pt->sizebc) {
223 BCIns ins = proto_bc(pt)[pc]; 235 BCIns ins = proto_bc(pt)[pc];
224 BCOp op = bc_op(ins); 236 BCOp op = bc_op(ins);
225 lua_assert(op < BC__MAX); 237 lj_assertL(op < BC__MAX, "bad bytecode op %d", op);
226 setintV(L->top, ins); 238 setintV(L->top, ins);
227 setintV(L->top+1, lj_bc_mode[op]); 239 setintV(L->top+1, lj_bc_mode[op]);
228 L->top += 2; 240 L->top += 2;
@@ -280,7 +292,7 @@ static GCtrace *jit_checktrace(lua_State *L)
280/* Names of link types. ORDER LJ_TRLINK */ 292/* Names of link types. ORDER LJ_TRLINK */
281static const char *const jit_trlinkname[] = { 293static const char *const jit_trlinkname[] = {
282 "none", "root", "loop", "tail-recursion", "up-recursion", "down-recursion", 294 "none", "root", "loop", "tail-recursion", "up-recursion", "down-recursion",
283 "interpreter", "return" 295 "interpreter", "return", "stitch"
284}; 296};
285 297
286/* local info = jit.util.traceinfo(tr) */ 298/* local info = jit.util.traceinfo(tr) */
@@ -333,6 +345,9 @@ LJLIB_CF(jit_util_tracek)
333 slot = ir->op2; 345 slot = ir->op2;
334 ir = &T->ir[ir->op1]; 346 ir = &T->ir[ir->op1];
335 } 347 }
348#if LJ_HASFFI
349 if (ir->o == IR_KINT64) ctype_loadffi(L);
350#endif
336 lj_ir_kvalue(L, L->top-2, ir); 351 lj_ir_kvalue(L, L->top-2, ir);
337 setintV(L->top-1, (int32_t)irt_type(ir->t)); 352 setintV(L->top-1, (int32_t)irt_type(ir->t));
338 if (slot == -1) 353 if (slot == -1)
@@ -407,7 +422,8 @@ LJLIB_CF(jit_util_ircalladdr)
407{ 422{
408 uint32_t idx = (uint32_t)lj_lib_checkint(L, 1); 423 uint32_t idx = (uint32_t)lj_lib_checkint(L, 1);
409 if (idx < IRCALL__MAX) { 424 if (idx < IRCALL__MAX) {
410 setintptrV(L->top-1, (intptr_t)(void *)lj_ir_callinfo[idx].func); 425 ASMFunction func = lj_ir_callinfo[idx].func;
426 setintptrV(L->top-1, (intptr_t)(void *)lj_ptr_strip(func));
411 return 1; 427 return 1;
412 } 428 }
413 return 0; 429 return 0;
@@ -417,6 +433,12 @@ LJLIB_CF(jit_util_ircalladdr)
417 433
418#include "lj_libdef.h" 434#include "lj_libdef.h"
419 435
436static int luaopen_jit_util(lua_State *L)
437{
438 LJ_LIB_REG(L, NULL, jit_util);
439 return 1;
440}
441
420/* -- jit.opt module ------------------------------------------------------ */ 442/* -- jit.opt module ------------------------------------------------------ */
421 443
422#if LJ_HASJIT 444#if LJ_HASJIT
@@ -453,7 +475,7 @@ static int jitopt_flag(jit_State *J, const char *str)
453 str += str[2] == '-' ? 3 : 2; 475 str += str[2] == '-' ? 3 : 2;
454 set = 0; 476 set = 0;
455 } 477 }
456 for (opt = JIT_F_OPT_FIRST; ; opt <<= 1) { 478 for (opt = JIT_F_OPT; ; opt <<= 1) {
457 size_t len = *(const uint8_t *)lst; 479 size_t len = *(const uint8_t *)lst;
458 if (len == 0) 480 if (len == 0)
459 break; 481 break;
@@ -473,7 +495,7 @@ static int jitopt_param(jit_State *J, const char *str)
473 int i; 495 int i;
474 for (i = 0; i < JIT_P__MAX; i++) { 496 for (i = 0; i < JIT_P__MAX; i++) {
475 size_t len = *(const uint8_t *)lst; 497 size_t len = *(const uint8_t *)lst;
476 lua_assert(len != 0); 498 lj_assertJ(len != 0, "bad JIT_P_STRING");
477 if (strncmp(str, lst+1, len) == 0 && str[len] == '=') { 499 if (strncmp(str, lst+1, len) == 0 && str[len] == '=') {
478 int32_t n = 0; 500 int32_t n = 0;
479 const char *p = &str[len+1]; 501 const char *p = &str[len+1];
@@ -514,6 +536,104 @@ LJLIB_CF(jit_opt_start)
514 536
515#endif 537#endif
516 538
539/* -- jit.profile module -------------------------------------------------- */
540
541#if LJ_HASPROFILE
542
543#define LJLIB_MODULE_jit_profile
544
545/* Not loaded by default, use: local profile = require("jit.profile") */
546
547#define KEY_PROFILE_THREAD (U64x(80000000,00000000)|'t')
548#define KEY_PROFILE_FUNC (U64x(80000000,00000000)|'f')
549
550static void jit_profile_callback(lua_State *L2, lua_State *L, int samples,
551 int vmstate)
552{
553 TValue key;
554 cTValue *tv;
555 key.u64 = KEY_PROFILE_FUNC;
556 tv = lj_tab_get(L, tabV(registry(L)), &key);
557 if (tvisfunc(tv)) {
558 char vmst = (char)vmstate;
559 int status;
560 setfuncV(L2, L2->top++, funcV(tv));
561 setthreadV(L2, L2->top++, L);
562 setintV(L2->top++, samples);
563 setstrV(L2, L2->top++, lj_str_new(L2, &vmst, 1));
564 status = lua_pcall(L2, 3, 0, 0); /* callback(thread, samples, vmstate) */
565 if (status) {
566 if (G(L2)->panic) G(L2)->panic(L2);
567 exit(EXIT_FAILURE);
568 }
569 lj_trace_abort(G(L2));
570 }
571}
572
573/* profile.start(mode, cb) */
574LJLIB_CF(jit_profile_start)
575{
576 GCtab *registry = tabV(registry(L));
577 GCstr *mode = lj_lib_optstr(L, 1);
578 GCfunc *func = lj_lib_checkfunc(L, 2);
579 lua_State *L2 = lua_newthread(L); /* Thread that runs profiler callback. */
580 TValue key;
581 /* Anchor thread and function in registry. */
582 key.u64 = KEY_PROFILE_THREAD;
583 setthreadV(L, lj_tab_set(L, registry, &key), L2);
584 key.u64 = KEY_PROFILE_FUNC;
585 setfuncV(L, lj_tab_set(L, registry, &key), func);
586 lj_gc_anybarriert(L, registry);
587 luaJIT_profile_start(L, mode ? strdata(mode) : "",
588 (luaJIT_profile_callback)jit_profile_callback, L2);
589 return 0;
590}
591
592/* profile.stop() */
593LJLIB_CF(jit_profile_stop)
594{
595 GCtab *registry;
596 TValue key;
597 luaJIT_profile_stop(L);
598 registry = tabV(registry(L));
599 key.u64 = KEY_PROFILE_THREAD;
600 setnilV(lj_tab_set(L, registry, &key));
601 key.u64 = KEY_PROFILE_FUNC;
602 setnilV(lj_tab_set(L, registry, &key));
603 lj_gc_anybarriert(L, registry);
604 return 0;
605}
606
607/* dump = profile.dumpstack([thread,] fmt, depth) */
608LJLIB_CF(jit_profile_dumpstack)
609{
610 lua_State *L2 = L;
611 int arg = 0;
612 size_t len;
613 int depth;
614 GCstr *fmt;
615 const char *p;
616 if (L->top > L->base && tvisthread(L->base)) {
617 L2 = threadV(L->base);
618 arg = 1;
619 }
620 fmt = lj_lib_checkstr(L, arg+1);
621 depth = lj_lib_checkint(L, arg+2);
622 p = luaJIT_profile_dumpstack(L2, strdata(fmt), depth, &len);
623 lua_pushlstring(L, p, len);
624 return 1;
625}
626
627#include "lj_libdef.h"
628
629static int luaopen_jit_profile(lua_State *L)
630{
631 LJ_LIB_REG(L, NULL, jit_profile);
632 return 1;
633}
634
635#endif
636
517/* -- JIT compiler initialization ----------------------------------------- */ 637/* -- JIT compiler initialization ----------------------------------------- */
518 638
519#if LJ_HASJIT 639#if LJ_HASJIT
@@ -524,66 +644,41 @@ JIT_PARAMDEF(JIT_PARAMINIT)
524#undef JIT_PARAMINIT 644#undef JIT_PARAMINIT
525 0 645 0
526}; 646};
527#endif
528 647
529#if LJ_TARGET_ARM && LJ_TARGET_LINUX 648#if LJ_TARGET_ARM && LJ_TARGET_LINUX
530#include <sys/utsname.h> 649#include <sys/utsname.h>
531#endif 650#endif
532 651
533/* Arch-dependent CPU detection. */ 652/* Arch-dependent CPU feature detection. */
534static uint32_t jit_cpudetect(lua_State *L) 653static uint32_t jit_cpudetect(void)
535{ 654{
536 uint32_t flags = 0; 655 uint32_t flags = 0;
537#if LJ_TARGET_X86ORX64 656#if LJ_TARGET_X86ORX64
657
538 uint32_t vendor[4]; 658 uint32_t vendor[4];
539 uint32_t features[4]; 659 uint32_t features[4];
540 if (lj_vm_cpuid(0, vendor) && lj_vm_cpuid(1, features)) { 660 if (lj_vm_cpuid(0, vendor) && lj_vm_cpuid(1, features)) {
541#if !LJ_HASJIT
542#define JIT_F_CMOV 1
543#define JIT_F_SSE2 2
544#endif
545 flags |= ((features[3] >> 15)&1) * JIT_F_CMOV;
546 flags |= ((features[3] >> 26)&1) * JIT_F_SSE2;
547#if LJ_HASJIT
548 flags |= ((features[2] >> 0)&1) * JIT_F_SSE3; 661 flags |= ((features[2] >> 0)&1) * JIT_F_SSE3;
549 flags |= ((features[2] >> 19)&1) * JIT_F_SSE4_1; 662 flags |= ((features[2] >> 19)&1) * JIT_F_SSE4_1;
550 if (vendor[2] == 0x6c65746e) { /* Intel. */ 663 if (vendor[0] >= 7) {
551 if ((features[0] & 0x0ff00f00) == 0x00000f00) /* P4. */ 664 uint32_t xfeatures[4];
552 flags |= JIT_F_P4; /* Currently unused. */ 665 lj_vm_cpuid(7, xfeatures);
553 else if ((features[0] & 0x0fff0ff0) == 0x000106c0) /* Atom. */ 666 flags |= ((xfeatures[1] >> 8)&1) * JIT_F_BMI2;
554 flags |= JIT_F_LEA_AGU;
555 } else if (vendor[2] == 0x444d4163) { /* AMD. */
556 uint32_t fam = (features[0] & 0x0ff00f00);
557 if (fam == 0x00000f00) /* K8. */
558 flags |= JIT_F_SPLIT_XMM;
559 if (fam >= 0x00000f00) /* K8, K10. */
560 flags |= JIT_F_PREFER_IMUL;
561 } 667 }
562#endif
563 } 668 }
564 /* Check for required instruction set support on x86 (unnecessary on x64). */ 669 /* Don't bother checking for SSE2 -- the VM will crash before getting here. */
565#if LJ_TARGET_X86 670
566#if !defined(LUAJIT_CPU_NOCMOV)
567 if (!(flags & JIT_F_CMOV))
568 luaL_error(L, "CPU not supported");
569#endif
570#if defined(LUAJIT_CPU_SSE2)
571 if (!(flags & JIT_F_SSE2))
572 luaL_error(L, "CPU does not support SSE2 (recompile without -DLUAJIT_CPU_SSE2)");
573#endif
574#endif
575#elif LJ_TARGET_ARM 671#elif LJ_TARGET_ARM
576#if LJ_HASJIT 672
577 int ver = LJ_ARCH_VERSION; /* Compile-time ARM CPU detection. */ 673 int ver = LJ_ARCH_VERSION; /* Compile-time ARM CPU detection. */
578#if LJ_TARGET_LINUX 674#if LJ_TARGET_LINUX
579 if (ver < 70) { /* Runtime ARM CPU detection. */ 675 if (ver < 70) { /* Runtime ARM CPU detection. */
580 struct utsname ut; 676 struct utsname ut;
581 uname(&ut); 677 uname(&ut);
582 if (strncmp(ut.machine, "armv", 4) == 0) { 678 if (strncmp(ut.machine, "armv", 4) == 0) {
583 if (ut.machine[4] >= '7') 679 if (ut.machine[4] >= '8') ver = 80;
584 ver = 70; 680 else if (ut.machine[4] == '7') ver = 70;
585 else if (ut.machine[4] == '6') 681 else if (ut.machine[4] == '6') ver = 60;
586 ver = 60;
587 } 682 }
588 } 683 }
589#endif 684#endif
@@ -591,74 +686,77 @@ static uint32_t jit_cpudetect(lua_State *L)
591 ver >= 61 ? JIT_F_ARMV6T2_ : 686 ver >= 61 ? JIT_F_ARMV6T2_ :
592 ver >= 60 ? JIT_F_ARMV6_ : 0; 687 ver >= 60 ? JIT_F_ARMV6_ : 0;
593 flags |= LJ_ARCH_HASFPU == 0 ? 0 : ver >= 70 ? JIT_F_VFPV3 : JIT_F_VFPV2; 688 flags |= LJ_ARCH_HASFPU == 0 ? 0 : ver >= 70 ? JIT_F_VFPV3 : JIT_F_VFPV2;
594#endif 689
690#elif LJ_TARGET_ARM64
691
692 /* No optional CPU features to detect (for now). */
693
595#elif LJ_TARGET_PPC 694#elif LJ_TARGET_PPC
596#if LJ_HASJIT 695
597#if LJ_ARCH_SQRT 696#if LJ_ARCH_SQRT
598 flags |= JIT_F_SQRT; 697 flags |= JIT_F_SQRT;
599#endif 698#endif
600#if LJ_ARCH_ROUND 699#if LJ_ARCH_ROUND
601 flags |= JIT_F_ROUND; 700 flags |= JIT_F_ROUND;
602#endif 701#endif
603#endif 702
604#elif LJ_TARGET_PPCSPE
605 /* Nothing to do. */
606#elif LJ_TARGET_MIPS 703#elif LJ_TARGET_MIPS
607#if LJ_HASJIT 704
608 /* Compile-time MIPS CPU detection. */ 705 /* Compile-time MIPS CPU detection. */
609#if LJ_ARCH_VERSION >= 20 706#if LJ_ARCH_VERSION >= 20
610 flags |= JIT_F_MIPS32R2; 707 flags |= JIT_F_MIPSXXR2;
611#endif 708#endif
612 /* Runtime MIPS CPU detection. */ 709 /* Runtime MIPS CPU detection. */
613#if defined(__GNUC__) 710#if defined(__GNUC__)
614 if (!(flags & JIT_F_MIPS32R2)) { 711 if (!(flags & JIT_F_MIPSXXR2)) {
615 int x; 712 int x;
713#ifdef __mips16
714 x = 0; /* Runtime detection is difficult. Ensure optimal -march flags. */
715#else
616 /* On MIPS32R1 rotr is treated as srl. rotr r2,r2,1 -> srl r2,r2,1. */ 716 /* On MIPS32R1 rotr is treated as srl. rotr r2,r2,1 -> srl r2,r2,1. */
617 __asm__("li $2, 1\n\t.long 0x00221042\n\tmove %0, $2" : "=r"(x) : : "$2"); 717 __asm__("li $2, 1\n\t.long 0x00221042\n\tmove %0, $2" : "=r"(x) : : "$2");
618 if (x) flags |= JIT_F_MIPS32R2; /* Either 0x80000000 (R2) or 0 (R1). */
619 }
620#endif 718#endif
719 if (x) flags |= JIT_F_MIPSXXR2; /* Either 0x80000000 (R2) or 0 (R1). */
720 }
621#endif 721#endif
722
622#else 723#else
623#error "Missing CPU detection for this architecture" 724#error "Missing CPU detection for this architecture"
624#endif 725#endif
625 UNUSED(L);
626 return flags; 726 return flags;
627} 727}
628 728
629/* Initialize JIT compiler. */ 729/* Initialize JIT compiler. */
630static void jit_init(lua_State *L) 730static void jit_init(lua_State *L)
631{ 731{
632 uint32_t flags = jit_cpudetect(L);
633#if LJ_HASJIT
634 jit_State *J = L2J(L); 732 jit_State *J = L2J(L);
635#if LJ_TARGET_X86 733 J->flags = jit_cpudetect() | JIT_F_ON | JIT_F_OPT_DEFAULT;
636 /* Silently turn off the JIT compiler on CPUs without SSE2. */
637 if ((flags & JIT_F_SSE2))
638#endif
639 J->flags = flags | JIT_F_ON | JIT_F_OPT_DEFAULT;
640 memcpy(J->param, jit_param_default, sizeof(J->param)); 734 memcpy(J->param, jit_param_default, sizeof(J->param));
641 lj_dispatch_update(G(L)); 735 lj_dispatch_update(G(L));
642#else
643 UNUSED(flags);
644#endif
645} 736}
737#endif
646 738
647LUALIB_API int luaopen_jit(lua_State *L) 739LUALIB_API int luaopen_jit(lua_State *L)
648{ 740{
741#if LJ_HASJIT
742 jit_init(L);
743#endif
649 lua_pushliteral(L, LJ_OS_NAME); 744 lua_pushliteral(L, LJ_OS_NAME);
650 lua_pushliteral(L, LJ_ARCH_NAME); 745 lua_pushliteral(L, LJ_ARCH_NAME);
651 lua_pushinteger(L, LUAJIT_VERSION_NUM); /* Deprecated. */ 746 lua_pushinteger(L, LUAJIT_VERSION_NUM); /* Deprecated. */
652 lua_pushliteral(L, LUAJIT_VERSION); 747 lua_pushliteral(L, LUAJIT_VERSION);
653 LJ_LIB_REG(L, LUA_JITLIBNAME, jit); 748 LJ_LIB_REG(L, LUA_JITLIBNAME, jit);
749#if LJ_HASPROFILE
750 lj_lib_prereg(L, LUA_JITLIBNAME ".profile", luaopen_jit_profile,
751 tabref(L->env));
752#endif
654#ifndef LUAJIT_DISABLE_JITUTIL 753#ifndef LUAJIT_DISABLE_JITUTIL
655 LJ_LIB_REG(L, "jit.util", jit_util); 754 lj_lib_prereg(L, LUA_JITLIBNAME ".util", luaopen_jit_util, tabref(L->env));
656#endif 755#endif
657#if LJ_HASJIT 756#if LJ_HASJIT
658 LJ_LIB_REG(L, "jit.opt", jit_opt); 757 LJ_LIB_REG(L, "jit.opt", jit_opt);
659#endif 758#endif
660 L->top -= 2; 759 L->top -= 2;
661 jit_init(L);
662 return 1; 760 return 1;
663} 761}
664 762