diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/Makefile | 1 | ||||
| -rw-r--r-- | src/lj.supp | 41 | ||||
| -rw-r--r-- | src/lj_arch.h | 17 | ||||
| -rw-r--r-- | src/lj_asm.c | 2 | ||||
| -rw-r--r-- | src/lj_asm_arm.h | 4 | ||||
| -rw-r--r-- | src/lj_asm_arm64.h | 4 | ||||
| -rw-r--r-- | src/lj_asm_mips.h | 2 | ||||
| -rw-r--r-- | src/lj_asm_ppc.h | 2 | ||||
| -rw-r--r-- | src/lj_asm_x86.h | 2 | ||||
| -rw-r--r-- | src/lj_gc.c | 38 | ||||
| -rw-r--r-- | src/lj_obj.h | 37 | ||||
| -rw-r--r-- | src/lj_state.c | 8 | ||||
| -rw-r--r-- | src/lj_str.c | 358 | ||||
| -rw-r--r-- | src/lj_str.h | 4 | ||||
| -rw-r--r-- | src/lj_tab.c | 4 | ||||
| -rw-r--r-- | src/vm_arm.dasc | 12 | ||||
| -rw-r--r-- | src/vm_arm64.dasc | 12 | ||||
| -rw-r--r-- | src/vm_mips.dasc | 12 | ||||
| -rw-r--r-- | src/vm_mips64.dasc | 12 | ||||
| -rw-r--r-- | src/vm_ppc.dasc | 12 | ||||
| -rw-r--r-- | src/vm_x64.dasc | 6 | ||||
| -rw-r--r-- | src/vm_x86.dasc | 6 |
22 files changed, 394 insertions, 202 deletions
diff --git a/src/Makefile b/src/Makefile index 6a9de5db..178a5acd 100644 --- a/src/Makefile +++ b/src/Makefile | |||
| @@ -132,7 +132,6 @@ XCFLAGS= | |||
| 132 | # | 132 | # |
| 133 | # This define is required to run LuaJIT under Valgrind. The Valgrind | 133 | # This define is required to run LuaJIT under Valgrind. The Valgrind |
| 134 | # header files must be installed. You should enable debug information, too. | 134 | # header files must be installed. You should enable debug information, too. |
| 135 | # Use --suppressions=lj.supp to avoid some false positives. | ||
| 136 | #XCFLAGS+= -DLUAJIT_USE_VALGRIND | 135 | #XCFLAGS+= -DLUAJIT_USE_VALGRIND |
| 137 | # | 136 | # |
| 138 | # This is the client for the GDB JIT API. GDB 7.0 or higher is required | 137 | # This is the client for the GDB JIT API. GDB 7.0 or higher is required |
diff --git a/src/lj.supp b/src/lj.supp deleted file mode 100644 index 217f7c89..00000000 --- a/src/lj.supp +++ /dev/null | |||
| @@ -1,41 +0,0 @@ | |||
| 1 | # Valgrind suppression file for LuaJIT 2.0. | ||
| 2 | { | ||
| 3 | Optimized string compare | ||
| 4 | Memcheck:Addr4 | ||
| 5 | fun:lj_str_cmp | ||
| 6 | } | ||
| 7 | { | ||
| 8 | Optimized string compare | ||
| 9 | Memcheck:Addr1 | ||
| 10 | fun:lj_str_cmp | ||
| 11 | } | ||
| 12 | { | ||
| 13 | Optimized string compare | ||
| 14 | Memcheck:Addr4 | ||
| 15 | fun:lj_str_new | ||
| 16 | } | ||
| 17 | { | ||
| 18 | Optimized string compare | ||
| 19 | Memcheck:Addr1 | ||
| 20 | fun:lj_str_new | ||
| 21 | } | ||
| 22 | { | ||
| 23 | Optimized string compare | ||
| 24 | Memcheck:Cond | ||
| 25 | fun:lj_str_new | ||
| 26 | } | ||
| 27 | { | ||
| 28 | Optimized string compare | ||
| 29 | Memcheck:Addr4 | ||
| 30 | fun:str_fastcmp | ||
| 31 | } | ||
| 32 | { | ||
| 33 | Optimized string compare | ||
| 34 | Memcheck:Addr1 | ||
| 35 | fun:str_fastcmp | ||
| 36 | } | ||
| 37 | { | ||
| 38 | Optimized string compare | ||
| 39 | Memcheck:Cond | ||
| 40 | fun:str_fastcmp | ||
| 41 | } | ||
diff --git a/src/lj_arch.h b/src/lj_arch.h index 626f6c13..f148b3f7 100644 --- a/src/lj_arch.h +++ b/src/lj_arch.h | |||
| @@ -637,14 +637,31 @@ extern void *LJ_WIN_LOADLIBA(const char *path); | |||
| 637 | 637 | ||
| 638 | /* Don't make any changes here. Instead build with: | 638 | /* Don't make any changes here. Instead build with: |
| 639 | ** make "XCFLAGS=-DLUAJIT_SECURITY_flag=value" | 639 | ** make "XCFLAGS=-DLUAJIT_SECURITY_flag=value" |
| 640 | ** | ||
| 641 | ** Important note to distro maintainers: DO NOT change the defaults for a | ||
| 642 | ** regular distro build -- neither upwards, nor downwards! | ||
| 643 | ** These build-time configurable security flags are intended for embedders | ||
| 644 | ** who may have specific needs wrt. security vs. performance. | ||
| 640 | */ | 645 | */ |
| 641 | 646 | ||
| 642 | /* Security defaults. */ | 647 | /* Security defaults. */ |
| 643 | #ifndef LUAJIT_SECURITY_PRNG | 648 | #ifndef LUAJIT_SECURITY_PRNG |
| 649 | /* PRNG init: 0 = fixed/insecure, 1 = secure from OS. */ | ||
| 644 | #define LUAJIT_SECURITY_PRNG 1 | 650 | #define LUAJIT_SECURITY_PRNG 1 |
| 645 | #endif | 651 | #endif |
| 646 | 652 | ||
| 653 | #ifndef LUAJIT_SECURITY_STRHASH | ||
| 654 | /* String hash: 0 = sparse only, 1 = sparse + dense. */ | ||
| 655 | #define LUAJIT_SECURITY_STRHASH 1 | ||
| 656 | #endif | ||
| 657 | |||
| 658 | #ifndef LUAJIT_SECURITY_STRID | ||
| 659 | /* String IDs: 0 = linear, 1 = reseed < 255, 2 = reseed < 15, 3 = random. */ | ||
| 660 | #define LUAJIT_SECURITY_STRID 1 | ||
| 661 | #endif | ||
| 662 | |||
| 647 | #ifndef LUAJIT_SECURITY_MCODE | 663 | #ifndef LUAJIT_SECURITY_MCODE |
| 664 | /* Machine code page protection: 0 = insecure RWX, 1 = secure RW^X. */ | ||
| 648 | #define LUAJIT_SECURITY_MCODE 1 | 665 | #define LUAJIT_SECURITY_MCODE 1 |
| 649 | #endif | 666 | #endif |
| 650 | 667 | ||
diff --git a/src/lj_asm.c b/src/lj_asm.c index 2659c8a2..cc7841c0 100644 --- a/src/lj_asm.c +++ b/src/lj_asm.c | |||
| @@ -1029,7 +1029,7 @@ static uint32_t ir_khash(ASMState *as, IRIns *ir) | |||
| 1029 | uint32_t lo, hi; | 1029 | uint32_t lo, hi; |
| 1030 | UNUSED(as); | 1030 | UNUSED(as); |
| 1031 | if (irt_isstr(ir->t)) { | 1031 | if (irt_isstr(ir->t)) { |
| 1032 | return ir_kstr(ir)->hash; | 1032 | return ir_kstr(ir)->sid; |
| 1033 | } else if (irt_isnum(ir->t)) { | 1033 | } else if (irt_isnum(ir->t)) { |
| 1034 | lo = ir_knum(ir)->u32.lo; | 1034 | lo = ir_knum(ir)->u32.lo; |
| 1035 | hi = ir_knum(ir)->u32.hi << 1; | 1035 | hi = ir_knum(ir)->u32.hi << 1; |
diff --git a/src/lj_asm_arm.h b/src/lj_asm_arm.h index d2fad141..e7d2bf17 100644 --- a/src/lj_asm_arm.h +++ b/src/lj_asm_arm.h | |||
| @@ -825,10 +825,10 @@ static void asm_href(ASMState *as, IRIns *ir, IROp merge) | |||
| 825 | } else { | 825 | } else { |
| 826 | emit_dnm(as, ARMI_ADD|ARMF_SH(ARMSH_LSL, 3), dest, dest, tmp); | 826 | emit_dnm(as, ARMI_ADD|ARMF_SH(ARMSH_LSL, 3), dest, dest, tmp); |
| 827 | emit_dnm(as, ARMI_ADD|ARMF_SH(ARMSH_LSL, 1), tmp, tmp, tmp); | 827 | emit_dnm(as, ARMI_ADD|ARMF_SH(ARMSH_LSL, 1), tmp, tmp, tmp); |
| 828 | if (irt_isstr(kt)) { /* Fetch of str->hash is cheaper than ra_allock. */ | 828 | if (irt_isstr(kt)) { /* Fetch of str->sid is cheaper than ra_allock. */ |
| 829 | emit_dnm(as, ARMI_AND, tmp, tmp+1, RID_TMP); | 829 | emit_dnm(as, ARMI_AND, tmp, tmp+1, RID_TMP); |
| 830 | emit_lso(as, ARMI_LDR, dest, tab, (int32_t)offsetof(GCtab, node)); | 830 | emit_lso(as, ARMI_LDR, dest, tab, (int32_t)offsetof(GCtab, node)); |
| 831 | emit_lso(as, ARMI_LDR, tmp+1, key, (int32_t)offsetof(GCstr, hash)); | 831 | emit_lso(as, ARMI_LDR, tmp+1, key, (int32_t)offsetof(GCstr, sid)); |
| 832 | emit_lso(as, ARMI_LDR, RID_TMP, tab, (int32_t)offsetof(GCtab, hmask)); | 832 | emit_lso(as, ARMI_LDR, RID_TMP, tab, (int32_t)offsetof(GCtab, hmask)); |
| 833 | } else if (irref_isk(refkey)) { | 833 | } else if (irref_isk(refkey)) { |
| 834 | emit_opk(as, ARMI_AND, tmp, RID_TMP, (int32_t)khash, | 834 | emit_opk(as, ARMI_AND, tmp, RID_TMP, (int32_t)khash, |
diff --git a/src/lj_asm_arm64.h b/src/lj_asm_arm64.h index 0729a3a5..b1fd3acc 100644 --- a/src/lj_asm_arm64.h +++ b/src/lj_asm_arm64.h | |||
| @@ -847,9 +847,9 @@ static void asm_href(ASMState *as, IRIns *ir, IROp merge) | |||
| 847 | emit_dnm(as, A64I_ANDw, dest, dest, tmphash); | 847 | emit_dnm(as, A64I_ANDw, dest, dest, tmphash); |
| 848 | emit_lso(as, A64I_LDRw, dest, tab, offsetof(GCtab, hmask)); | 848 | emit_lso(as, A64I_LDRw, dest, tab, offsetof(GCtab, hmask)); |
| 849 | } else if (irt_isstr(kt)) { | 849 | } else if (irt_isstr(kt)) { |
| 850 | /* Fetch of str->hash is cheaper than ra_allock. */ | 850 | /* Fetch of str->sid is cheaper than ra_allock. */ |
| 851 | emit_dnm(as, A64I_ANDw, dest, dest, tmp); | 851 | emit_dnm(as, A64I_ANDw, dest, dest, tmp); |
| 852 | emit_lso(as, A64I_LDRw, tmp, key, offsetof(GCstr, hash)); | 852 | emit_lso(as, A64I_LDRw, tmp, key, offsetof(GCstr, sid)); |
| 853 | emit_lso(as, A64I_LDRw, dest, tab, offsetof(GCtab, hmask)); | 853 | emit_lso(as, A64I_LDRw, dest, tab, offsetof(GCtab, hmask)); |
| 854 | } else { /* Must match with hash*() in lj_tab.c. */ | 854 | } else { /* Must match with hash*() in lj_tab.c. */ |
| 855 | emit_dnm(as, A64I_ANDw, dest, dest, tmp); | 855 | emit_dnm(as, A64I_ANDw, dest, dest, tmp); |
diff --git a/src/lj_asm_mips.h b/src/lj_asm_mips.h index a2b8d8e0..513bd5ca 100644 --- a/src/lj_asm_mips.h +++ b/src/lj_asm_mips.h | |||
| @@ -1041,7 +1041,7 @@ static void asm_href(ASMState *as, IRIns *ir, IROp merge) | |||
| 1041 | if (isk) { | 1041 | if (isk) { |
| 1042 | /* Nothing to do. */ | 1042 | /* Nothing to do. */ |
| 1043 | } else if (irt_isstr(kt)) { | 1043 | } else if (irt_isstr(kt)) { |
| 1044 | emit_tsi(as, MIPSI_LW, tmp1, key, (int32_t)offsetof(GCstr, hash)); | 1044 | emit_tsi(as, MIPSI_LW, tmp1, key, (int32_t)offsetof(GCstr, sid)); |
| 1045 | } else { /* Must match with hash*() in lj_tab.c. */ | 1045 | } else { /* Must match with hash*() in lj_tab.c. */ |
| 1046 | emit_dst(as, MIPSI_SUBU, tmp1, tmp1, tmp2); | 1046 | emit_dst(as, MIPSI_SUBU, tmp1, tmp1, tmp2); |
| 1047 | emit_rotr(as, tmp2, tmp2, dest, (-HASH_ROT3)&31); | 1047 | emit_rotr(as, tmp2, tmp2, dest, (-HASH_ROT3)&31); |
diff --git a/src/lj_asm_ppc.h b/src/lj_asm_ppc.h index 498fdac3..77ab09d6 100644 --- a/src/lj_asm_ppc.h +++ b/src/lj_asm_ppc.h | |||
| @@ -721,7 +721,7 @@ static void asm_href(ASMState *as, IRIns *ir, IROp merge) | |||
| 721 | if (isk) { | 721 | if (isk) { |
| 722 | /* Nothing to do. */ | 722 | /* Nothing to do. */ |
| 723 | } else if (irt_isstr(kt)) { | 723 | } else if (irt_isstr(kt)) { |
| 724 | emit_tai(as, PPCI_LWZ, tmp1, key, (int32_t)offsetof(GCstr, hash)); | 724 | emit_tai(as, PPCI_LWZ, tmp1, key, (int32_t)offsetof(GCstr, sid)); |
| 725 | } else { /* Must match with hash*() in lj_tab.c. */ | 725 | } else { /* Must match with hash*() in lj_tab.c. */ |
| 726 | emit_tab(as, PPCI_SUBF, tmp1, tmp2, tmp1); | 726 | emit_tab(as, PPCI_SUBF, tmp1, tmp2, tmp1); |
| 727 | emit_rotlwi(as, tmp2, tmp2, HASH_ROT3); | 727 | emit_rotlwi(as, tmp2, tmp2, HASH_ROT3); |
diff --git a/src/lj_asm_x86.h b/src/lj_asm_x86.h index a3adee14..e40b5e54 100644 --- a/src/lj_asm_x86.h +++ b/src/lj_asm_x86.h | |||
| @@ -1228,7 +1228,7 @@ static void asm_href(ASMState *as, IRIns *ir, IROp merge) | |||
| 1228 | emit_gri(as, XG_ARITHi(XOg_AND), dest, (int32_t)khash); | 1228 | emit_gri(as, XG_ARITHi(XOg_AND), dest, (int32_t)khash); |
| 1229 | emit_rmro(as, XO_MOV, dest, tab, offsetof(GCtab, hmask)); | 1229 | emit_rmro(as, XO_MOV, dest, tab, offsetof(GCtab, hmask)); |
| 1230 | } else if (irt_isstr(kt)) { | 1230 | } else if (irt_isstr(kt)) { |
| 1231 | emit_rmro(as, XO_ARITH(XOg_AND), dest, key, offsetof(GCstr, hash)); | 1231 | emit_rmro(as, XO_ARITH(XOg_AND), dest, key, offsetof(GCstr, sid)); |
| 1232 | emit_rmro(as, XO_MOV, dest, tab, offsetof(GCtab, hmask)); | 1232 | emit_rmro(as, XO_MOV, dest, tab, offsetof(GCtab, hmask)); |
| 1233 | } else { /* Must match with hashrot() in lj_tab.c. */ | 1233 | } else { /* Must match with hashrot() in lj_tab.c. */ |
| 1234 | emit_rmro(as, XO_ARITH(XOg_AND), dest, tab, offsetof(GCtab, hmask)); | 1234 | emit_rmro(as, XO_ARITH(XOg_AND), dest, tab, offsetof(GCtab, hmask)); |
diff --git a/src/lj_gc.c b/src/lj_gc.c index 671b5983..cc4232a6 100644 --- a/src/lj_gc.c +++ b/src/lj_gc.c | |||
| @@ -417,6 +417,32 @@ static GCRef *gc_sweep(global_State *g, GCRef *p, uint32_t lim) | |||
| 417 | return p; | 417 | return p; |
| 418 | } | 418 | } |
| 419 | 419 | ||
| 420 | /* Sweep one string interning table chain. Preserves hashalg bit. */ | ||
| 421 | static void gc_sweepstr(global_State *g, GCRef *chain) | ||
| 422 | { | ||
| 423 | /* Mask with other white and LJ_GC_FIXED. Or LJ_GC_SFIXED on shutdown. */ | ||
| 424 | int ow = otherwhite(g); | ||
| 425 | uintptr_t u = gcrefu(*chain); | ||
| 426 | GCRef q; | ||
| 427 | GCRef *p = &q; | ||
| 428 | GCobj *o; | ||
| 429 | setgcrefp(q, (u & ~(uintptr_t)1)); | ||
| 430 | while ((o = gcref(*p)) != NULL) { | ||
| 431 | if (((o->gch.marked ^ LJ_GC_WHITES) & ow)) { /* Black or current white? */ | ||
| 432 | lj_assertG(!isdead(g, o) || (o->gch.marked & LJ_GC_FIXED), | ||
| 433 | "sweep of undead string"); | ||
| 434 | makewhite(g, o); /* String is alive, change to the current white. */ | ||
| 435 | p = &o->gch.nextgc; | ||
| 436 | } else { /* Otherwise string is dead, free it. */ | ||
| 437 | lj_assertG(isdead(g, o) || ow == LJ_GC_SFIXED, | ||
| 438 | "sweep of unlive string"); | ||
| 439 | setgcrefr(*p, o->gch.nextgc); | ||
| 440 | lj_str_free(g, gco2str(o)); | ||
| 441 | } | ||
| 442 | } | ||
| 443 | setgcrefp(*chain, (gcrefu(q) | (u & 1))); | ||
| 444 | } | ||
| 445 | |||
| 420 | /* Check whether we can clear a key or a value slot from a table. */ | 446 | /* Check whether we can clear a key or a value slot from a table. */ |
| 421 | static int gc_mayclear(cTValue *o, int val) | 447 | static int gc_mayclear(cTValue *o, int val) |
| 422 | { | 448 | { |
| @@ -571,9 +597,9 @@ void lj_gc_freeall(global_State *g) | |||
| 571 | /* Free everything, except super-fixed objects (the main thread). */ | 597 | /* Free everything, except super-fixed objects (the main thread). */ |
| 572 | g->gc.currentwhite = LJ_GC_WHITES | LJ_GC_SFIXED; | 598 | g->gc.currentwhite = LJ_GC_WHITES | LJ_GC_SFIXED; |
| 573 | gc_fullsweep(g, &g->gc.root); | 599 | gc_fullsweep(g, &g->gc.root); |
| 574 | strmask = g->strmask; | 600 | strmask = g->str.mask; |
| 575 | for (i = 0; i <= strmask; i++) /* Free all string hash chains. */ | 601 | for (i = 0; i <= strmask; i++) /* Free all string hash chains. */ |
| 576 | gc_fullsweep(g, &g->strhash[i]); | 602 | gc_sweepstr(g, &g->str.tab[i]); |
| 577 | } | 603 | } |
| 578 | 604 | ||
| 579 | /* -- Collector ----------------------------------------------------------- */ | 605 | /* -- Collector ----------------------------------------------------------- */ |
| @@ -636,8 +662,8 @@ static size_t gc_onestep(lua_State *L) | |||
| 636 | return 0; | 662 | return 0; |
| 637 | case GCSsweepstring: { | 663 | case GCSsweepstring: { |
| 638 | GCSize old = g->gc.total; | 664 | GCSize old = g->gc.total; |
| 639 | gc_fullsweep(g, &g->strhash[g->gc.sweepstr++]); /* Sweep one chain. */ | 665 | gc_sweepstr(g, &g->str.tab[g->gc.sweepstr++]); /* Sweep one chain. */ |
| 640 | if (g->gc.sweepstr > g->strmask) | 666 | if (g->gc.sweepstr > g->str.mask) |
| 641 | g->gc.state = GCSsweep; /* All string hash chains sweeped. */ | 667 | g->gc.state = GCSsweep; /* All string hash chains sweeped. */ |
| 642 | lj_assertG(old >= g->gc.total, "sweep increased memory"); | 668 | lj_assertG(old >= g->gc.total, "sweep increased memory"); |
| 643 | g->gc.estimate -= old - g->gc.total; | 669 | g->gc.estimate -= old - g->gc.total; |
| @@ -649,8 +675,8 @@ static size_t gc_onestep(lua_State *L) | |||
| 649 | lj_assertG(old >= g->gc.total, "sweep increased memory"); | 675 | lj_assertG(old >= g->gc.total, "sweep increased memory"); |
| 650 | g->gc.estimate -= old - g->gc.total; | 676 | g->gc.estimate -= old - g->gc.total; |
| 651 | if (gcref(*mref(g->gc.sweep, GCRef)) == NULL) { | 677 | if (gcref(*mref(g->gc.sweep, GCRef)) == NULL) { |
| 652 | if (g->strnum <= (g->strmask >> 2) && g->strmask > LJ_MIN_STRTAB*2-1) | 678 | if (g->str.num <= (g->str.mask >> 2) && g->str.mask > LJ_MIN_STRTAB*2-1) |
| 653 | lj_str_resize(L, g->strmask >> 1); /* Shrink string table. */ | 679 | lj_str_resize(L, g->str.mask >> 1); /* Shrink string table. */ |
| 654 | if (gcref(g->gc.mmudata)) { /* Need any finalizations? */ | 680 | if (gcref(g->gc.mmudata)) { /* Need any finalizations? */ |
| 655 | g->gc.state = GCSfinalize; | 681 | g->gc.state = GCSfinalize; |
| 656 | #if LJ_HASFFI | 682 | #if LJ_HASFFI |
diff --git a/src/lj_obj.h b/src/lj_obj.h index 6c974812..9d4bec08 100644 --- a/src/lj_obj.h +++ b/src/lj_obj.h | |||
| @@ -13,7 +13,7 @@ | |||
| 13 | #include "lj_def.h" | 13 | #include "lj_def.h" |
| 14 | #include "lj_arch.h" | 14 | #include "lj_arch.h" |
| 15 | 15 | ||
| 16 | /* -- Memory references (32 bit address space) ---------------------------- */ | 16 | /* -- Memory references --------------------------------------------------- */ |
| 17 | 17 | ||
| 18 | /* Memory and GC object sizes. */ | 18 | /* Memory and GC object sizes. */ |
| 19 | typedef uint32_t MSize; | 19 | typedef uint32_t MSize; |
| @@ -44,7 +44,7 @@ typedef struct MRef { | |||
| 44 | #define setmrefr(r, v) ((r).ptr32 = (v).ptr32) | 44 | #define setmrefr(r, v) ((r).ptr32 = (v).ptr32) |
| 45 | #endif | 45 | #endif |
| 46 | 46 | ||
| 47 | /* -- GC object references (32 bit address space) ------------------------- */ | 47 | /* -- GC object references ------------------------------------------------ */ |
| 48 | 48 | ||
| 49 | /* GCobj reference */ | 49 | /* GCobj reference */ |
| 50 | typedef struct GCRef { | 50 | typedef struct GCRef { |
| @@ -287,12 +287,16 @@ typedef const TValue cTValue; | |||
| 287 | 287 | ||
| 288 | /* -- String object ------------------------------------------------------- */ | 288 | /* -- String object ------------------------------------------------------- */ |
| 289 | 289 | ||
| 290 | typedef uint32_t StrHash; /* String hash value. */ | ||
| 291 | typedef uint32_t StrID; /* String ID. */ | ||
| 292 | |||
| 290 | /* String object header. String payload follows. */ | 293 | /* String object header. String payload follows. */ |
| 291 | typedef struct GCstr { | 294 | typedef struct GCstr { |
| 292 | GCHeader; | 295 | GCHeader; |
| 293 | uint8_t reserved; /* Used by lexer for fast lookup of reserved words. */ | 296 | uint8_t reserved; /* Used by lexer for fast lookup of reserved words. */ |
| 294 | uint8_t unused; | 297 | uint8_t hashalg; /* Hash algorithm. */ |
| 295 | MSize hash; /* Hash of string. */ | 298 | StrID sid; /* Interned string ID. */ |
| 299 | StrHash hash; /* Hash of string. */ | ||
| 296 | MSize len; /* Size of string. */ | 300 | MSize len; /* Size of string. */ |
| 297 | } GCstr; | 301 | } GCstr; |
| 298 | 302 | ||
| @@ -300,7 +304,6 @@ typedef struct GCstr { | |||
| 300 | #define strdata(s) ((const char *)((s)+1)) | 304 | #define strdata(s) ((const char *)((s)+1)) |
| 301 | #define strdatawr(s) ((char *)((s)+1)) | 305 | #define strdatawr(s) ((char *)((s)+1)) |
| 302 | #define strVdata(o) strdata(strV(o)) | 306 | #define strVdata(o) strdata(strV(o)) |
| 303 | #define sizestring(s) (sizeof(struct GCstr)+(s)->len+1) | ||
| 304 | 307 | ||
| 305 | /* -- Userdata object ----------------------------------------------------- */ | 308 | /* -- Userdata object ----------------------------------------------------- */ |
| 306 | 309 | ||
| @@ -570,6 +573,7 @@ typedef enum { | |||
| 570 | #define basemt_obj(g, o) ((g)->gcroot[GCROOT_BASEMT+itypemap(o)]) | 573 | #define basemt_obj(g, o) ((g)->gcroot[GCROOT_BASEMT+itypemap(o)]) |
| 571 | #define mmname_str(g, mm) (strref((g)->gcroot[GCROOT_MMNAME+(mm)])) | 574 | #define mmname_str(g, mm) (strref((g)->gcroot[GCROOT_MMNAME+(mm)])) |
| 572 | 575 | ||
| 576 | /* Garbage collector state. */ | ||
| 573 | typedef struct GCState { | 577 | typedef struct GCState { |
| 574 | GCSize total; /* Memory currently allocated. */ | 578 | GCSize total; /* Memory currently allocated. */ |
| 575 | GCSize threshold; /* Memory threshold. */ | 579 | GCSize threshold; /* Memory threshold. */ |
| @@ -590,25 +594,36 @@ typedef struct GCState { | |||
| 590 | MSize pause; /* Pause between successive GC cycles. */ | 594 | MSize pause; /* Pause between successive GC cycles. */ |
| 591 | } GCState; | 595 | } GCState; |
| 592 | 596 | ||
| 597 | /* String interning state. */ | ||
| 598 | typedef struct StrInternState { | ||
| 599 | GCRef *tab; /* String hash table anchors. */ | ||
| 600 | MSize mask; /* String hash mask (size of hash table - 1). */ | ||
| 601 | MSize num; /* Number of strings in hash table. */ | ||
| 602 | StrID id; /* Next string ID. */ | ||
| 603 | uint8_t idreseed; /* String ID reseed counter. */ | ||
| 604 | uint8_t second; /* String interning table uses secondary hashing. */ | ||
| 605 | uint8_t unused1; | ||
| 606 | uint8_t unused2; | ||
| 607 | LJ_ALIGN(8) uint64_t seed; /* Random string seed. */ | ||
| 608 | } StrInternState; | ||
| 609 | |||
| 593 | /* Global state, shared by all threads of a Lua universe. */ | 610 | /* Global state, shared by all threads of a Lua universe. */ |
| 594 | typedef struct global_State { | 611 | typedef struct global_State { |
| 595 | GCRef *strhash; /* String hash table (hash chain anchors). */ | ||
| 596 | MSize strmask; /* String hash mask (size of hash table - 1). */ | ||
| 597 | MSize strnum; /* Number of strings in hash table. */ | ||
| 598 | lua_Alloc allocf; /* Memory allocator. */ | 612 | lua_Alloc allocf; /* Memory allocator. */ |
| 599 | void *allocd; /* Memory allocator data. */ | 613 | void *allocd; /* Memory allocator data. */ |
| 600 | GCState gc; /* Garbage collector. */ | 614 | GCState gc; /* Garbage collector. */ |
| 601 | volatile int32_t vmstate; /* VM state or current JIT code trace number. */ | ||
| 602 | SBuf tmpbuf; /* Temporary string buffer. */ | ||
| 603 | GCstr strempty; /* Empty string. */ | 615 | GCstr strempty; /* Empty string. */ |
| 604 | uint8_t stremptyz; /* Zero terminator of empty string. */ | 616 | uint8_t stremptyz; /* Zero terminator of empty string. */ |
| 605 | uint8_t hookmask; /* Hook mask. */ | 617 | uint8_t hookmask; /* Hook mask. */ |
| 606 | uint8_t dispatchmode; /* Dispatch mode. */ | 618 | uint8_t dispatchmode; /* Dispatch mode. */ |
| 607 | uint8_t vmevmask; /* VM event mask. */ | 619 | uint8_t vmevmask; /* VM event mask. */ |
| 620 | StrInternState str; /* String interning. */ | ||
| 621 | volatile int32_t vmstate; /* VM state or current JIT code trace number. */ | ||
| 608 | GCRef mainthref; /* Link to main thread. */ | 622 | GCRef mainthref; /* Link to main thread. */ |
| 609 | TValue registrytv; /* Anchor for registry. */ | 623 | SBuf tmpbuf; /* Temporary string buffer. */ |
| 610 | TValue tmptv, tmptv2; /* Temporary TValues. */ | 624 | TValue tmptv, tmptv2; /* Temporary TValues. */ |
| 611 | Node nilnode; /* Fallback 1-element hash part (nil key and value). */ | 625 | Node nilnode; /* Fallback 1-element hash part (nil key and value). */ |
| 626 | TValue registrytv; /* Anchor for registry. */ | ||
| 612 | GCupval uvhead; /* Head of double-linked list of all open upvalues. */ | 627 | GCupval uvhead; /* Head of double-linked list of all open upvalues. */ |
| 613 | int32_t hookcount; /* Instruction hook countdown. */ | 628 | int32_t hookcount; /* Instruction hook countdown. */ |
| 614 | int32_t hookcstart; /* Start count for instruction hook counter. */ | 629 | int32_t hookcstart; /* Start count for instruction hook counter. */ |
diff --git a/src/lj_state.c b/src/lj_state.c index a4d072be..4f77e71f 100644 --- a/src/lj_state.c +++ b/src/lj_state.c | |||
| @@ -150,7 +150,7 @@ static TValue *cpluaopen(lua_State *L, lua_CFunction dummy, void *ud) | |||
| 150 | /* NOBARRIER: State initialization, all objects are white. */ | 150 | /* NOBARRIER: State initialization, all objects are white. */ |
| 151 | setgcref(L->env, obj2gco(lj_tab_new(L, 0, LJ_MIN_GLOBAL))); | 151 | setgcref(L->env, obj2gco(lj_tab_new(L, 0, LJ_MIN_GLOBAL))); |
| 152 | settabV(L, registry(L), lj_tab_new(L, 0, LJ_MIN_REGISTRY)); | 152 | settabV(L, registry(L), lj_tab_new(L, 0, LJ_MIN_REGISTRY)); |
| 153 | lj_str_resize(L, LJ_MIN_STRTAB-1); | 153 | lj_str_init(L); |
| 154 | lj_meta_init(L); | 154 | lj_meta_init(L); |
| 155 | lj_lex_init(L); | 155 | lj_lex_init(L); |
| 156 | fixstring(lj_err_str(L, LJ_ERR_ERRMEM)); /* Preallocate memory error msg. */ | 156 | fixstring(lj_err_str(L, LJ_ERR_ERRMEM)); /* Preallocate memory error msg. */ |
| @@ -166,12 +166,12 @@ static void close_state(lua_State *L) | |||
| 166 | lj_gc_freeall(g); | 166 | lj_gc_freeall(g); |
| 167 | lj_assertG(gcref(g->gc.root) == obj2gco(L), | 167 | lj_assertG(gcref(g->gc.root) == obj2gco(L), |
| 168 | "main thread is not first GC object"); | 168 | "main thread is not first GC object"); |
| 169 | lj_assertG(g->strnum == 0, "leaked %d strings", g->strnum); | 169 | lj_assertG(g->str.num == 0, "leaked %d strings", g->str.num); |
| 170 | lj_trace_freestate(g); | 170 | lj_trace_freestate(g); |
| 171 | #if LJ_HASFFI | 171 | #if LJ_HASFFI |
| 172 | lj_ctype_freestate(g); | 172 | lj_ctype_freestate(g); |
| 173 | #endif | 173 | #endif |
| 174 | lj_mem_freevec(g, g->strhash, g->strmask+1, GCRef); | 174 | lj_str_freetab(g); |
| 175 | lj_buf_free(g, &g->tmpbuf); | 175 | lj_buf_free(g, &g->tmpbuf); |
| 176 | lj_mem_freevec(g, tvref(L->stack), L->stacksize, TValue); | 176 | lj_mem_freevec(g, tvref(L->stack), L->stacksize, TValue); |
| 177 | lj_assertG(g->gc.total == sizeof(GG_State), | 177 | lj_assertG(g->gc.total == sizeof(GG_State), |
| @@ -231,7 +231,7 @@ LUA_API lua_State *lua_newstate(lua_Alloc allocf, void *allocd) | |||
| 231 | setgcref(g->mainthref, obj2gco(L)); | 231 | setgcref(g->mainthref, obj2gco(L)); |
| 232 | setgcref(g->uvhead.prev, obj2gco(&g->uvhead)); | 232 | setgcref(g->uvhead.prev, obj2gco(&g->uvhead)); |
| 233 | setgcref(g->uvhead.next, obj2gco(&g->uvhead)); | 233 | setgcref(g->uvhead.next, obj2gco(&g->uvhead)); |
| 234 | g->strmask = ~(MSize)0; | 234 | g->str.mask = ~(MSize)0; |
| 235 | setnilV(registry(L)); | 235 | setnilV(registry(L)); |
| 236 | setnilV(&g->nilnode.val); | 236 | setnilV(&g->nilnode.val); |
| 237 | setnilV(&g->nilnode.key); | 237 | setnilV(&g->nilnode.key); |
diff --git a/src/lj_str.c b/src/lj_str.c index 0253c15e..5bf8426c 100644 --- a/src/lj_str.c +++ b/src/lj_str.c | |||
| @@ -11,6 +11,7 @@ | |||
| 11 | #include "lj_err.h" | 11 | #include "lj_err.h" |
| 12 | #include "lj_str.h" | 12 | #include "lj_str.h" |
| 13 | #include "lj_char.h" | 13 | #include "lj_char.h" |
| 14 | #include "lj_prng.h" | ||
| 14 | 15 | ||
| 15 | /* -- String helpers ------------------------------------------------------ */ | 16 | /* -- String helpers ------------------------------------------------------ */ |
| 16 | 17 | ||
| @@ -37,28 +38,6 @@ int32_t LJ_FASTCALL lj_str_cmp(GCstr *a, GCstr *b) | |||
| 37 | return (int32_t)(a->len - b->len); | 38 | return (int32_t)(a->len - b->len); |
| 38 | } | 39 | } |
| 39 | 40 | ||
| 40 | /* Fast string data comparison. Caveat: unaligned access to 1st string! */ | ||
| 41 | static LJ_AINLINE int str_fastcmp(const char *a, const char *b, MSize len) | ||
| 42 | { | ||
| 43 | MSize i = 0; | ||
| 44 | lj_assertX(len > 0, "fast string compare with zero length"); | ||
| 45 | lj_assertX((((uintptr_t)a+len-1) & (LJ_PAGESIZE-1)) <= LJ_PAGESIZE-4, | ||
| 46 | "fast string compare crossing page boundary"); | ||
| 47 | do { /* Note: innocuous access up to end of string + 3. */ | ||
| 48 | uint32_t v = lj_getu32(a+i) ^ *(const uint32_t *)(b+i); | ||
| 49 | if (v) { | ||
| 50 | i -= len; | ||
| 51 | #if LJ_LE | ||
| 52 | return (int32_t)i >= -3 ? (v << (32+(i<<3))) : 1; | ||
| 53 | #else | ||
| 54 | return (int32_t)i >= -3 ? (v >> (32+(i<<3))) : 1; | ||
| 55 | #endif | ||
| 56 | } | ||
| 57 | i += 4; | ||
| 58 | } while (i < len); | ||
| 59 | return 0; | ||
| 60 | } | ||
| 61 | |||
| 62 | /* Find fixed string p inside string s. */ | 41 | /* Find fixed string p inside string s. */ |
| 63 | const char *lj_str_find(const char *s, const char *p, MSize slen, MSize plen) | 42 | const char *lj_str_find(const char *s, const char *p, MSize slen, MSize plen) |
| 64 | { | 43 | { |
| @@ -91,108 +70,301 @@ int lj_str_haspattern(GCstr *s) | |||
| 91 | return 0; /* No pattern matching chars found. */ | 70 | return 0; /* No pattern matching chars found. */ |
| 92 | } | 71 | } |
| 93 | 72 | ||
| 94 | /* -- String interning ---------------------------------------------------- */ | 73 | /* -- String hashing ------------------------------------------------------ */ |
| 95 | |||
| 96 | /* Resize the string hash table (grow and shrink). */ | ||
| 97 | void lj_str_resize(lua_State *L, MSize newmask) | ||
| 98 | { | ||
| 99 | global_State *g = G(L); | ||
| 100 | GCRef *newhash; | ||
| 101 | MSize i; | ||
| 102 | if (g->gc.state == GCSsweepstring || newmask >= LJ_MAX_STRTAB-1) | ||
| 103 | return; /* No resizing during GC traversal or if already too big. */ | ||
| 104 | newhash = lj_mem_newvec(L, newmask+1, GCRef); | ||
| 105 | memset(newhash, 0, (newmask+1)*sizeof(GCRef)); | ||
| 106 | for (i = g->strmask; i != ~(MSize)0; i--) { /* Rehash old table. */ | ||
| 107 | GCobj *p = gcref(g->strhash[i]); | ||
| 108 | while (p) { /* Follow each hash chain and reinsert all strings. */ | ||
| 109 | MSize h = gco2str(p)->hash & newmask; | ||
| 110 | GCobj *next = gcnext(p); | ||
| 111 | /* NOBARRIER: The string table is a GC root. */ | ||
| 112 | setgcrefr(p->gch.nextgc, newhash[h]); | ||
| 113 | setgcref(newhash[h], p); | ||
| 114 | p = next; | ||
| 115 | } | ||
| 116 | } | ||
| 117 | lj_mem_freevec(g, g->strhash, g->strmask+1, GCRef); | ||
| 118 | g->strmask = newmask; | ||
| 119 | g->strhash = newhash; | ||
| 120 | } | ||
| 121 | 74 | ||
| 122 | /* Intern a string and return string object. */ | 75 | /* Keyed sparse ARX string hash. Constant time. */ |
| 123 | GCstr *lj_str_new(lua_State *L, const char *str, size_t lenx) | 76 | static StrHash hash_sparse(uint64_t seed, const char *str, MSize len) |
| 124 | { | 77 | { |
| 125 | global_State *g; | 78 | /* Constants taken from lookup3 hash by Bob Jenkins. */ |
| 126 | GCstr *s; | 79 | StrHash a, b, h = len ^ (StrHash)seed; |
| 127 | GCobj *o; | ||
| 128 | MSize len = (MSize)lenx; | ||
| 129 | MSize a, b, h = len; | ||
| 130 | if (lenx >= LJ_MAX_STR) | ||
| 131 | lj_err_msg(L, LJ_ERR_STROV); | ||
| 132 | g = G(L); | ||
| 133 | /* Compute string hash. Constants taken from lookup3 hash by Bob Jenkins. */ | ||
| 134 | if (len >= 4) { /* Caveat: unaligned access! */ | 80 | if (len >= 4) { /* Caveat: unaligned access! */ |
| 135 | a = lj_getu32(str); | 81 | a = lj_getu32(str); |
| 136 | h ^= lj_getu32(str+len-4); | 82 | h ^= lj_getu32(str+len-4); |
| 137 | b = lj_getu32(str+(len>>1)-2); | 83 | b = lj_getu32(str+(len>>1)-2); |
| 138 | h ^= b; h -= lj_rol(b, 14); | 84 | h ^= b; h -= lj_rol(b, 14); |
| 139 | b += lj_getu32(str+(len>>2)-1); | 85 | b += lj_getu32(str+(len>>2)-1); |
| 140 | } else if (len > 0) { | 86 | } else { |
| 141 | a = *(const uint8_t *)str; | 87 | a = *(const uint8_t *)str; |
| 142 | h ^= *(const uint8_t *)(str+len-1); | 88 | h ^= *(const uint8_t *)(str+len-1); |
| 143 | b = *(const uint8_t *)(str+(len>>1)); | 89 | b = *(const uint8_t *)(str+(len>>1)); |
| 144 | h ^= b; h -= lj_rol(b, 14); | 90 | h ^= b; h -= lj_rol(b, 14); |
| 145 | } else { | ||
| 146 | return &g->strempty; | ||
| 147 | } | 91 | } |
| 148 | a ^= h; a -= lj_rol(h, 11); | 92 | a ^= h; a -= lj_rol(h, 11); |
| 149 | b ^= a; b -= lj_rol(a, 25); | 93 | b ^= a; b -= lj_rol(a, 25); |
| 150 | h ^= b; h -= lj_rol(b, 16); | 94 | h ^= b; h -= lj_rol(b, 16); |
| 151 | /* Check if the string has already been interned. */ | 95 | return h; |
| 152 | o = gcref(g->strhash[h & g->strmask]); | 96 | } |
| 153 | if (LJ_LIKELY((((uintptr_t)str+len-1) & (LJ_PAGESIZE-1)) <= LJ_PAGESIZE-4)) { | 97 | |
| 154 | while (o != NULL) { | 98 | #if LUAJIT_SECURITY_STRHASH |
| 155 | GCstr *sx = gco2str(o); | 99 | /* Keyed dense ARX string hash. Linear time. */ |
| 156 | if (sx->len == len && str_fastcmp(str, strdata(sx), len) == 0) { | 100 | static LJ_NOINLINE StrHash hash_dense(uint64_t seed, StrHash h, |
| 157 | /* Resurrect if dead. Can only happen with fixstring() (keywords). */ | 101 | const char *str, MSize len) |
| 158 | if (isdead(g, o)) flipwhite(o); | 102 | { |
| 159 | return sx; /* Return existing string. */ | 103 | StrHash b = lj_bswap(lj_rol(h ^ (StrHash)(seed >> 32), 4)); |
| 104 | if (len > 12) { | ||
| 105 | StrHash a = (StrHash)seed; | ||
| 106 | const char *pe = str+len-12, *p = pe, *q = str; | ||
| 107 | do { | ||
| 108 | a += lj_getu32(p); | ||
| 109 | b += lj_getu32(p+4); | ||
| 110 | h += lj_getu32(p+8); | ||
| 111 | p = q; q += 12; | ||
| 112 | h ^= b; h -= lj_rol(b, 14); | ||
| 113 | a ^= h; a -= lj_rol(h, 11); | ||
| 114 | b ^= a; b -= lj_rol(a, 25); | ||
| 115 | } while (p < pe); | ||
| 116 | h ^= b; h -= lj_rol(b, 16); | ||
| 117 | a ^= h; a -= lj_rol(h, 4); | ||
| 118 | b ^= a; b -= lj_rol(a, 14); | ||
| 119 | } | ||
| 120 | return b; | ||
| 121 | } | ||
| 122 | #endif | ||
| 123 | |||
| 124 | /* -- String interning ---------------------------------------------------- */ | ||
| 125 | |||
| 126 | #define LJ_STR_MAXCOLL 32 | ||
| 127 | |||
| 128 | /* Resize the string interning hash table (grow and shrink). */ | ||
| 129 | void lj_str_resize(lua_State *L, MSize newmask) | ||
| 130 | { | ||
| 131 | global_State *g = G(L); | ||
| 132 | GCRef *newtab, *oldtab = g->str.tab; | ||
| 133 | MSize i; | ||
| 134 | |||
| 135 | /* No resizing during GC traversal or if already too big. */ | ||
| 136 | if (g->gc.state == GCSsweepstring || newmask >= LJ_MAX_STRTAB-1) | ||
| 137 | return; | ||
| 138 | |||
| 139 | newtab = lj_mem_newvec(L, newmask+1, GCRef); | ||
| 140 | memset(newtab, 0, (newmask+1)*sizeof(GCRef)); | ||
| 141 | |||
| 142 | #if LUAJIT_SECURITY_STRHASH | ||
| 143 | /* Check which chains need secondary hashes. */ | ||
| 144 | if (g->str.second) { | ||
| 145 | int newsecond = 0; | ||
| 146 | /* Compute primary chain lengths. */ | ||
| 147 | for (i = g->str.mask; i != ~(MSize)0; i--) { | ||
| 148 | GCobj *o = (GCobj *)(gcrefu(oldtab[i]) & ~(uintptr_t)1); | ||
| 149 | while (o) { | ||
| 150 | GCstr *s = gco2str(o); | ||
| 151 | MSize hash = s->hashalg ? hash_sparse(g->str.seed, strdata(s), s->len) : | ||
| 152 | s->hash; | ||
| 153 | hash &= newmask; | ||
| 154 | setgcrefp(newtab[hash], gcrefu(newtab[hash]) + 1); | ||
| 155 | o = gcnext(o); | ||
| 160 | } | 156 | } |
| 161 | o = gcnext(o); | ||
| 162 | } | 157 | } |
| 163 | } else { /* Slow path: end of string is too close to a page boundary. */ | 158 | /* Mark secondary chains. */ |
| 164 | while (o != NULL) { | 159 | for (i = newmask; i != ~(MSize)0; i--) { |
| 165 | GCstr *sx = gco2str(o); | 160 | int secondary = gcrefu(newtab[i]) > LJ_STR_MAXCOLL; |
| 166 | if (sx->len == len && memcmp(str, strdata(sx), len) == 0) { | 161 | newsecond |= secondary; |
| 167 | /* Resurrect if dead. Can only happen with fixstring() (keywords). */ | 162 | setgcrefp(newtab[i], secondary); |
| 168 | if (isdead(g, o)) flipwhite(o); | 163 | } |
| 169 | return sx; /* Return existing string. */ | 164 | g->str.second = newsecond; |
| 165 | } | ||
| 166 | #endif | ||
| 167 | |||
| 168 | /* Reinsert all strings from the old table into the new table. */ | ||
| 169 | for (i = g->str.mask; i != ~(MSize)0; i--) { | ||
| 170 | GCobj *o = (GCobj *)(gcrefu(oldtab[i]) & ~(uintptr_t)1); | ||
| 171 | while (o) { | ||
| 172 | GCobj *next = gcnext(o); | ||
| 173 | GCstr *s = gco2str(o); | ||
| 174 | MSize hash = s->hash; | ||
| 175 | #if LUAJIT_SECURITY_STRHASH | ||
| 176 | uintptr_t u; | ||
| 177 | if (LJ_LIKELY(!s->hashalg)) { /* String hashed with primary hash. */ | ||
| 178 | hash &= newmask; | ||
| 179 | u = gcrefu(newtab[hash]); | ||
| 180 | if (LJ_UNLIKELY(u & 1)) { /* Switch string to secondary hash. */ | ||
| 181 | s->hash = hash = hash_dense(g->str.seed, s->hash, strdata(s), s->len); | ||
| 182 | s->hashalg = 1; | ||
| 183 | hash &= newmask; | ||
| 184 | u = gcrefu(newtab[hash]); | ||
| 185 | } | ||
| 186 | } else { /* String hashed with secondary hash. */ | ||
| 187 | MSize shash = hash_sparse(g->str.seed, strdata(s), s->len); | ||
| 188 | u = gcrefu(newtab[shash & newmask]); | ||
| 189 | if (u & 1) { | ||
| 190 | hash &= newmask; | ||
| 191 | u = gcrefu(newtab[hash]); | ||
| 192 | } else { /* Revert string back to primary hash. */ | ||
| 193 | s->hash = shash; | ||
| 194 | s->hashalg = 0; | ||
| 195 | hash = (shash & newmask); | ||
| 196 | } | ||
| 197 | } | ||
| 198 | /* NOBARRIER: The string table is a GC root. */ | ||
| 199 | setgcrefp(o->gch.nextgc, (u & ~(uintptr_t)1)); | ||
| 200 | setgcrefp(newtab[hash], ((uintptr_t)o | (u & 1))); | ||
| 201 | #else | ||
| 202 | hash &= newmask; | ||
| 203 | /* NOBARRIER: The string table is a GC root. */ | ||
| 204 | setgcrefr(o->gch.nextgc, newtab[hash]); | ||
| 205 | setgcref(newtab[hash], o); | ||
| 206 | #endif | ||
| 207 | o = next; | ||
| 208 | } | ||
| 209 | } | ||
| 210 | |||
| 211 | /* Free old table and replace with new table. */ | ||
| 212 | lj_str_freetab(g); | ||
| 213 | g->str.tab = newtab; | ||
| 214 | g->str.mask = newmask; | ||
| 215 | } | ||
| 216 | |||
| 217 | #if LUAJIT_SECURITY_STRHASH | ||
| 218 | /* Rehash and rechain all strings in a chain. */ | ||
| 219 | static LJ_NOINLINE GCstr *lj_str_rehash_chain(lua_State *L, StrHash hashc, | ||
| 220 | const char *str, MSize len) | ||
| 221 | { | ||
| 222 | global_State *g = G(L); | ||
| 223 | int ow = g->gc.state == GCSsweepstring ? otherwhite(g) : 0; /* Sweeping? */ | ||
| 224 | GCRef *strtab = g->str.tab; | ||
| 225 | MSize strmask = g->str.mask; | ||
| 226 | GCobj *o = gcref(strtab[hashc & strmask]); | ||
| 227 | setgcrefp(strtab[hashc & strmask], (void *)((uintptr_t)1)); | ||
| 228 | g->str.second = 1; | ||
| 229 | while (o) { | ||
| 230 | uintptr_t u; | ||
| 231 | GCobj *next = gcnext(o); | ||
| 232 | GCstr *s = gco2str(o); | ||
| 233 | StrHash hash; | ||
| 234 | if (ow) { /* Must sweep while rechaining. */ | ||
| 235 | if (((o->gch.marked ^ LJ_GC_WHITES) & ow)) { /* String alive? */ | ||
| 236 | lj_assertG(!isdead(g, o) || (o->gch.marked & LJ_GC_FIXED), | ||
| 237 | "sweep of undead string"); | ||
| 238 | makewhite(g, o); | ||
| 239 | } else { /* Free dead string. */ | ||
| 240 | lj_assertG(isdead(g, o) || ow == LJ_GC_SFIXED, | ||
| 241 | "sweep of unlive string"); | ||
| 242 | lj_str_free(g, s); | ||
| 243 | o = next; | ||
| 244 | continue; | ||
| 170 | } | 245 | } |
| 171 | o = gcnext(o); | ||
| 172 | } | 246 | } |
| 247 | hash = s->hash; | ||
| 248 | if (!s->hashalg) { /* Rehash with secondary hash. */ | ||
| 249 | hash = hash_dense(g->str.seed, hash, strdata(s), s->len); | ||
| 250 | s->hash = hash; | ||
| 251 | s->hashalg = 1; | ||
| 252 | } | ||
| 253 | /* Rechain. */ | ||
| 254 | hash &= strmask; | ||
| 255 | u = gcrefu(strtab[hash]); | ||
| 256 | setgcrefp(o->gch.nextgc, (u & ~(uintptr_t)1)); | ||
| 257 | setgcrefp(strtab[hash], ((uintptr_t)o | (u & 1))); | ||
| 258 | o = next; | ||
| 173 | } | 259 | } |
| 174 | /* Nope, create a new string. */ | 260 | /* Try to insert the pending string again. */ |
| 175 | s = lj_mem_newt(L, sizeof(GCstr)+len+1, GCstr); | 261 | return lj_str_new(L, str, len); |
| 262 | } | ||
| 263 | #endif | ||
| 264 | |||
| 265 | /* Reseed String ID from PRNG after random interval < 2^bits. */ | ||
| 266 | #if LUAJIT_SECURITY_STRID == 1 | ||
| 267 | #define STRID_RESEED_INTERVAL 8 | ||
| 268 | #elif LUAJIT_SECURITY_STRID == 2 | ||
| 269 | #define STRID_RESEED_INTERVAL 4 | ||
| 270 | #elif LUAJIT_SECURITY_STRID >= 3 | ||
| 271 | #define STRID_RESEED_INTERVAL 0 | ||
| 272 | #endif | ||
| 273 | |||
| 274 | /* Allocate a new string and add to string interning table. */ | ||
| 275 | static GCstr *lj_str_alloc(lua_State *L, const char *str, MSize len, | ||
| 276 | StrHash hash, int hashalg) | ||
| 277 | { | ||
| 278 | GCstr *s = lj_mem_newt(L, lj_str_size(len), GCstr); | ||
| 279 | global_State *g = G(L); | ||
| 280 | uintptr_t u; | ||
| 176 | newwhite(g, s); | 281 | newwhite(g, s); |
| 177 | s->gct = ~LJ_TSTR; | 282 | s->gct = ~LJ_TSTR; |
| 178 | s->len = len; | 283 | s->len = len; |
| 179 | s->hash = h; | 284 | s->hash = hash; |
| 285 | #ifndef STRID_RESEED_INTERVAL | ||
| 286 | s->sid = g->str.id++; | ||
| 287 | #elif STRID_RESEED_INTERVAL | ||
| 288 | if (!g->str.idreseed--) { | ||
| 289 | uint64_t r = lj_prng_u64(&g->prng); | ||
| 290 | g->str.id = (StrID)r; | ||
| 291 | g->str.idreseed = (uint8_t)(r >> (64 - STRID_RESEED_INTERVAL)); | ||
| 292 | } | ||
| 293 | s->sid = g->str.id++; | ||
| 294 | #else | ||
| 295 | s->sid = (StrID)lj_prng_u64(&g->prng); | ||
| 296 | #endif | ||
| 180 | s->reserved = 0; | 297 | s->reserved = 0; |
| 298 | s->hashalg = (uint8_t)hashalg; | ||
| 299 | /* Clear last 4 bytes of allocated memory. Implies zero-termination, too. */ | ||
| 300 | *(uint32_t *)(strdatawr(s)+(len & ~(MSize)3)) = 0; | ||
| 181 | memcpy(strdatawr(s), str, len); | 301 | memcpy(strdatawr(s), str, len); |
| 182 | strdatawr(s)[len] = '\0'; /* Zero-terminate string. */ | 302 | /* Add to string hash table. */ |
| 183 | /* Add it to string hash table. */ | 303 | hash &= g->str.mask; |
| 184 | h &= g->strmask; | 304 | u = gcrefu(g->str.tab[hash]); |
| 185 | s->nextgc = g->strhash[h]; | 305 | setgcrefp(s->nextgc, (u & ~(uintptr_t)1)); |
| 186 | /* NOBARRIER: The string table is a GC root. */ | 306 | /* NOBARRIER: The string table is a GC root. */ |
| 187 | setgcref(g->strhash[h], obj2gco(s)); | 307 | setgcrefp(g->str.tab[hash], ((uintptr_t)s | (u & 1))); |
| 188 | if (g->strnum++ > g->strmask) /* Allow a 100% load factor. */ | 308 | if (g->str.num++ > g->str.mask) /* Allow a 100% load factor. */ |
| 189 | lj_str_resize(L, (g->strmask<<1)+1); /* Grow string table. */ | 309 | lj_str_resize(L, (g->str.mask<<1)+1); /* Grow string table. */ |
| 190 | return s; /* Return newly interned string. */ | 310 | return s; /* Return newly interned string. */ |
| 191 | } | 311 | } |
| 192 | 312 | ||
| 313 | /* Intern a string and return string object. */ | ||
| 314 | GCstr *lj_str_new(lua_State *L, const char *str, size_t lenx) | ||
| 315 | { | ||
| 316 | global_State *g = G(L); | ||
| 317 | if (lenx-1 < LJ_MAX_STR-1) { | ||
| 318 | MSize len = (MSize)lenx; | ||
| 319 | StrHash hash = hash_sparse(g->str.seed, str, len); | ||
| 320 | MSize coll = 0; | ||
| 321 | int hashalg = 0; | ||
| 322 | /* Check if the string has already been interned. */ | ||
| 323 | GCobj *o = gcref(g->str.tab[hash & g->str.mask]); | ||
| 324 | #if LUAJIT_SECURITY_STRHASH | ||
| 325 | if (LJ_UNLIKELY((uintptr_t)o & 1)) { /* Secondary hash for this chain? */ | ||
| 326 | hashalg = 1; | ||
| 327 | hash = hash_dense(g->str.seed, hash, str, len); | ||
| 328 | o = (GCobj *)(gcrefu(g->str.tab[hash & g->str.mask]) & ~(uintptr_t)1); | ||
| 329 | } | ||
| 330 | #endif | ||
| 331 | while (o != NULL) { | ||
| 332 | GCstr *sx = gco2str(o); | ||
| 333 | if (sx->hash == hash && sx->len == len) { | ||
| 334 | if (memcmp(str, strdata(sx), len) == 0) { | ||
| 335 | if (isdead(g, o)) flipwhite(o); /* Resurrect if dead. */ | ||
| 336 | return sx; /* Return existing string. */ | ||
| 337 | } | ||
| 338 | coll++; | ||
| 339 | } | ||
| 340 | coll++; | ||
| 341 | o = gcnext(o); | ||
| 342 | } | ||
| 343 | #if LUAJIT_SECURITY_STRHASH | ||
| 344 | /* Rehash chain if there are too many collisions. */ | ||
| 345 | if (LJ_UNLIKELY(coll > LJ_STR_MAXCOLL) && !hashalg) { | ||
| 346 | return lj_str_rehash_chain(L, hash, str, len); | ||
| 347 | } | ||
| 348 | #endif | ||
| 349 | /* Otherwise allocate a new string. */ | ||
| 350 | return lj_str_alloc(L, str, len, hash, hashalg); | ||
| 351 | } else { | ||
| 352 | if (lenx) | ||
| 353 | lj_err_msg(L, LJ_ERR_STROV); | ||
| 354 | return &g->strempty; | ||
| 355 | } | ||
| 356 | } | ||
| 357 | |||
| 193 | void LJ_FASTCALL lj_str_free(global_State *g, GCstr *s) | 358 | void LJ_FASTCALL lj_str_free(global_State *g, GCstr *s) |
| 194 | { | 359 | { |
| 195 | g->strnum--; | 360 | g->str.num--; |
| 196 | lj_mem_free(g, s, sizestring(s)); | 361 | lj_mem_free(g, s, lj_str_size(s->len)); |
| 362 | } | ||
| 363 | |||
| 364 | void LJ_FASTCALL lj_str_init(lua_State *L) | ||
| 365 | { | ||
| 366 | global_State *g = G(L); | ||
| 367 | g->str.seed = lj_prng_u64(&g->prng); | ||
| 368 | lj_str_resize(L, LJ_MIN_STRTAB-1); | ||
| 197 | } | 369 | } |
| 198 | 370 | ||
diff --git a/src/lj_str.h b/src/lj_str.h index 2e9bfc1d..01c6ba6b 100644 --- a/src/lj_str.h +++ b/src/lj_str.h | |||
| @@ -20,8 +20,12 @@ LJ_FUNC int lj_str_haspattern(GCstr *s); | |||
| 20 | LJ_FUNC void lj_str_resize(lua_State *L, MSize newmask); | 20 | LJ_FUNC void lj_str_resize(lua_State *L, MSize newmask); |
| 21 | LJ_FUNCA GCstr *lj_str_new(lua_State *L, const char *str, size_t len); | 21 | LJ_FUNCA GCstr *lj_str_new(lua_State *L, const char *str, size_t len); |
| 22 | LJ_FUNC void LJ_FASTCALL lj_str_free(global_State *g, GCstr *s); | 22 | LJ_FUNC void LJ_FASTCALL lj_str_free(global_State *g, GCstr *s); |
| 23 | LJ_FUNC void LJ_FASTCALL lj_str_init(lua_State *L); | ||
| 24 | #define lj_str_freetab(g) \ | ||
| 25 | (lj_mem_freevec(g, g->str.tab, g->str.mask+1, GCRef)) | ||
| 23 | 26 | ||
| 24 | #define lj_str_newz(L, s) (lj_str_new(L, s, strlen(s))) | 27 | #define lj_str_newz(L, s) (lj_str_new(L, s, strlen(s))) |
| 25 | #define lj_str_newlit(L, s) (lj_str_new(L, "" s, sizeof(s)-1)) | 28 | #define lj_str_newlit(L, s) (lj_str_new(L, "" s, sizeof(s)-1)) |
| 29 | #define lj_str_size(len) (sizeof(GCstr) + (((len)+4) & ~(MSize)3)) | ||
| 26 | 30 | ||
| 27 | #endif | 31 | #endif |
diff --git a/src/lj_tab.c b/src/lj_tab.c index efc423cb..982b0763 100644 --- a/src/lj_tab.c +++ b/src/lj_tab.c | |||
| @@ -23,8 +23,8 @@ static LJ_AINLINE Node *hashmask(const GCtab *t, uint32_t hash) | |||
| 23 | return &n[hash & t->hmask]; | 23 | return &n[hash & t->hmask]; |
| 24 | } | 24 | } |
| 25 | 25 | ||
| 26 | /* String hashes are precomputed when they are interned. */ | 26 | /* String IDs are generated when a string is interned. */ |
| 27 | #define hashstr(t, s) hashmask(t, (s)->hash) | 27 | #define hashstr(t, s) hashmask(t, (s)->sid) |
| 28 | 28 | ||
| 29 | #define hashlohi(t, lo, hi) hashmask((t), hashrot((lo), (hi))) | 29 | #define hashlohi(t, lo, hi) hashmask((t), hashrot((lo), (hi))) |
| 30 | #define hashnum(t, o) hashlohi((t), (o)->u32.lo, ((o)->u32.hi << 1)) | 30 | #define hashnum(t, o) hashlohi((t), (o)->u32.lo, ((o)->u32.hi << 1)) |
diff --git a/src/vm_arm.dasc b/src/vm_arm.dasc index 013688fb..770a8e21 100644 --- a/src/vm_arm.dasc +++ b/src/vm_arm.dasc | |||
| @@ -1012,9 +1012,9 @@ static void build_subroutines(BuildCtx *ctx) | |||
| 1012 | | cmp TAB:RB, #0 | 1012 | | cmp TAB:RB, #0 |
| 1013 | | beq ->fff_restv | 1013 | | beq ->fff_restv |
| 1014 | | ldr CARG3, TAB:RB->hmask | 1014 | | ldr CARG3, TAB:RB->hmask |
| 1015 | | ldr CARG4, STR:RC->hash | 1015 | | ldr CARG4, STR:RC->sid |
| 1016 | | ldr NODE:INS, TAB:RB->node | 1016 | | ldr NODE:INS, TAB:RB->node |
| 1017 | | and CARG3, CARG3, CARG4 // idx = str->hash & tab->hmask | 1017 | | and CARG3, CARG3, CARG4 // idx = str->sid & tab->hmask |
| 1018 | | add CARG3, CARG3, CARG3, lsl #1 | 1018 | | add CARG3, CARG3, CARG3, lsl #1 |
| 1019 | | add NODE:INS, NODE:INS, CARG3, lsl #3 // node = tab->node + idx*3*8 | 1019 | | add NODE:INS, NODE:INS, CARG3, lsl #3 // node = tab->node + idx*3*8 |
| 1020 | |3: // Rearranged logic, because we expect _not_ to find the key. | 1020 | |3: // Rearranged logic, because we expect _not_ to find the key. |
| @@ -3500,10 +3500,10 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop) | |||
| 3500 | |->BC_TGETS_Z: | 3500 | |->BC_TGETS_Z: |
| 3501 | | // (TAB:RB =) TAB:CARG1 = GCtab *, STR:RC = GCstr *, RA = dst*8 | 3501 | | // (TAB:RB =) TAB:CARG1 = GCtab *, STR:RC = GCstr *, RA = dst*8 |
| 3502 | | ldr CARG3, TAB:CARG1->hmask | 3502 | | ldr CARG3, TAB:CARG1->hmask |
| 3503 | | ldr CARG4, STR:RC->hash | 3503 | | ldr CARG4, STR:RC->sid |
| 3504 | | ldr NODE:INS, TAB:CARG1->node | 3504 | | ldr NODE:INS, TAB:CARG1->node |
| 3505 | | mov TAB:RB, TAB:CARG1 | 3505 | | mov TAB:RB, TAB:CARG1 |
| 3506 | | and CARG3, CARG3, CARG4 // idx = str->hash & tab->hmask | 3506 | | and CARG3, CARG3, CARG4 // idx = str->sid & tab->hmask |
| 3507 | | add CARG3, CARG3, CARG3, lsl #1 | 3507 | | add CARG3, CARG3, CARG3, lsl #1 |
| 3508 | | add NODE:INS, NODE:INS, CARG3, lsl #3 // node = tab->node + idx*3*8 | 3508 | | add NODE:INS, NODE:INS, CARG3, lsl #3 // node = tab->node + idx*3*8 |
| 3509 | |1: | 3509 | |1: |
| @@ -3647,10 +3647,10 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop) | |||
| 3647 | |->BC_TSETS_Z: | 3647 | |->BC_TSETS_Z: |
| 3648 | | // (TAB:RB =) TAB:CARG1 = GCtab *, STR:RC = GCstr *, RA = dst*8 | 3648 | | // (TAB:RB =) TAB:CARG1 = GCtab *, STR:RC = GCstr *, RA = dst*8 |
| 3649 | | ldr CARG3, TAB:CARG1->hmask | 3649 | | ldr CARG3, TAB:CARG1->hmask |
| 3650 | | ldr CARG4, STR:RC->hash | 3650 | | ldr CARG4, STR:RC->sid |
| 3651 | | ldr NODE:INS, TAB:CARG1->node | 3651 | | ldr NODE:INS, TAB:CARG1->node |
| 3652 | | mov TAB:RB, TAB:CARG1 | 3652 | | mov TAB:RB, TAB:CARG1 |
| 3653 | | and CARG3, CARG3, CARG4 // idx = str->hash & tab->hmask | 3653 | | and CARG3, CARG3, CARG4 // idx = str->sid & tab->hmask |
| 3654 | | add CARG3, CARG3, CARG3, lsl #1 | 3654 | | add CARG3, CARG3, CARG3, lsl #1 |
| 3655 | | mov CARG4, #0 | 3655 | | mov CARG4, #0 |
| 3656 | | add NODE:INS, NODE:INS, CARG3, lsl #3 // node = tab->node + idx*3*8 | 3656 | | add NODE:INS, NODE:INS, CARG3, lsl #3 // node = tab->node + idx*3*8 |
diff --git a/src/vm_arm64.dasc b/src/vm_arm64.dasc index c157696c..4a729f65 100644 --- a/src/vm_arm64.dasc +++ b/src/vm_arm64.dasc | |||
| @@ -993,9 +993,9 @@ static void build_subroutines(BuildCtx *ctx) | |||
| 993 | | ldr STR:RC, GL->gcroot[GCROOT_MMNAME+MM_metatable] | 993 | | ldr STR:RC, GL->gcroot[GCROOT_MMNAME+MM_metatable] |
| 994 | | cbz TAB:RB, ->fff_restv | 994 | | cbz TAB:RB, ->fff_restv |
| 995 | | ldr TMP1w, TAB:RB->hmask | 995 | | ldr TMP1w, TAB:RB->hmask |
| 996 | | ldr TMP2w, STR:RC->hash | 996 | | ldr TMP2w, STR:RC->sid |
| 997 | | ldr NODE:CARG3, TAB:RB->node | 997 | | ldr NODE:CARG3, TAB:RB->node |
| 998 | | and TMP1w, TMP1w, TMP2w // idx = str->hash & tab->hmask | 998 | | and TMP1w, TMP1w, TMP2w // idx = str->sid & tab->hmask |
| 999 | | add TMP1, TMP1, TMP1, lsl #1 | 999 | | add TMP1, TMP1, TMP1, lsl #1 |
| 1000 | | movn CARG4, #~LJ_TSTR | 1000 | | movn CARG4, #~LJ_TSTR |
| 1001 | | add NODE:CARG3, NODE:CARG3, TMP1, lsl #3 // node = tab->node + idx*3*8 | 1001 | | add NODE:CARG3, NODE:CARG3, TMP1, lsl #3 // node = tab->node + idx*3*8 |
| @@ -2943,9 +2943,9 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop) | |||
| 2943 | |->BC_TGETS_Z: | 2943 | |->BC_TGETS_Z: |
| 2944 | | // TAB:CARG2 = GCtab *, STR:RC = GCstr *, RA = dst | 2944 | | // TAB:CARG2 = GCtab *, STR:RC = GCstr *, RA = dst |
| 2945 | | ldr TMP1w, TAB:CARG2->hmask | 2945 | | ldr TMP1w, TAB:CARG2->hmask |
| 2946 | | ldr TMP2w, STR:RC->hash | 2946 | | ldr TMP2w, STR:RC->sid |
| 2947 | | ldr NODE:CARG3, TAB:CARG2->node | 2947 | | ldr NODE:CARG3, TAB:CARG2->node |
| 2948 | | and TMP1w, TMP1w, TMP2w // idx = str->hash & tab->hmask | 2948 | | and TMP1w, TMP1w, TMP2w // idx = str->sid & tab->hmask |
| 2949 | | add TMP1, TMP1, TMP1, lsl #1 | 2949 | | add TMP1, TMP1, TMP1, lsl #1 |
| 2950 | | movn CARG4, #~LJ_TSTR | 2950 | | movn CARG4, #~LJ_TSTR |
| 2951 | | add NODE:CARG3, NODE:CARG3, TMP1, lsl #3 // node = tab->node + idx*3*8 | 2951 | | add NODE:CARG3, NODE:CARG3, TMP1, lsl #3 // node = tab->node + idx*3*8 |
| @@ -3069,9 +3069,9 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop) | |||
| 3069 | |->BC_TSETS_Z: | 3069 | |->BC_TSETS_Z: |
| 3070 | | // TAB:CARG2 = GCtab *, STR:RC = GCstr *, RA = src | 3070 | | // TAB:CARG2 = GCtab *, STR:RC = GCstr *, RA = src |
| 3071 | | ldr TMP1w, TAB:CARG2->hmask | 3071 | | ldr TMP1w, TAB:CARG2->hmask |
| 3072 | | ldr TMP2w, STR:RC->hash | 3072 | | ldr TMP2w, STR:RC->sid |
| 3073 | | ldr NODE:CARG3, TAB:CARG2->node | 3073 | | ldr NODE:CARG3, TAB:CARG2->node |
| 3074 | | and TMP1w, TMP1w, TMP2w // idx = str->hash & tab->hmask | 3074 | | and TMP1w, TMP1w, TMP2w // idx = str->sid & tab->hmask |
| 3075 | | add TMP1, TMP1, TMP1, lsl #1 | 3075 | | add TMP1, TMP1, TMP1, lsl #1 |
| 3076 | | movn CARG4, #~LJ_TSTR | 3076 | | movn CARG4, #~LJ_TSTR |
| 3077 | | add NODE:CARG3, NODE:CARG3, TMP1, lsl #3 // node = tab->node + idx*3*8 | 3077 | | add NODE:CARG3, NODE:CARG3, TMP1, lsl #3 // node = tab->node + idx*3*8 |
diff --git a/src/vm_mips.dasc b/src/vm_mips.dasc index 0c84c13b..91de4b5c 100644 --- a/src/vm_mips.dasc +++ b/src/vm_mips.dasc | |||
| @@ -1152,9 +1152,9 @@ static void build_subroutines(BuildCtx *ctx) | |||
| 1152 | |. li SFARG1HI, LJ_TNIL | 1152 | |. li SFARG1HI, LJ_TNIL |
| 1153 | | lw TMP0, TAB:SFARG1LO->hmask | 1153 | | lw TMP0, TAB:SFARG1LO->hmask |
| 1154 | | li SFARG1HI, LJ_TTAB // Use metatable as default result. | 1154 | | li SFARG1HI, LJ_TTAB // Use metatable as default result. |
| 1155 | | lw TMP1, STR:RC->hash | 1155 | | lw TMP1, STR:RC->sid |
| 1156 | | lw NODE:TMP2, TAB:SFARG1LO->node | 1156 | | lw NODE:TMP2, TAB:SFARG1LO->node |
| 1157 | | and TMP1, TMP1, TMP0 // idx = str->hash & tab->hmask | 1157 | | and TMP1, TMP1, TMP0 // idx = str->sid & tab->hmask |
| 1158 | | sll TMP0, TMP1, 5 | 1158 | | sll TMP0, TMP1, 5 |
| 1159 | | sll TMP1, TMP1, 3 | 1159 | | sll TMP1, TMP1, 3 |
| 1160 | | subu TMP1, TMP0, TMP1 | 1160 | | subu TMP1, TMP0, TMP1 |
| @@ -4029,9 +4029,9 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop) | |||
| 4029 | |->BC_TGETS_Z: | 4029 | |->BC_TGETS_Z: |
| 4030 | | // TAB:RB = GCtab *, STR:RC = GCstr *, RA = dst*8 | 4030 | | // TAB:RB = GCtab *, STR:RC = GCstr *, RA = dst*8 |
| 4031 | | lw TMP0, TAB:RB->hmask | 4031 | | lw TMP0, TAB:RB->hmask |
| 4032 | | lw TMP1, STR:RC->hash | 4032 | | lw TMP1, STR:RC->sid |
| 4033 | | lw NODE:TMP2, TAB:RB->node | 4033 | | lw NODE:TMP2, TAB:RB->node |
| 4034 | | and TMP1, TMP1, TMP0 // idx = str->hash & tab->hmask | 4034 | | and TMP1, TMP1, TMP0 // idx = str->sid & tab->hmask |
| 4035 | | sll TMP0, TMP1, 5 | 4035 | | sll TMP0, TMP1, 5 |
| 4036 | | sll TMP1, TMP1, 3 | 4036 | | sll TMP1, TMP1, 3 |
| 4037 | | subu TMP1, TMP0, TMP1 | 4037 | | subu TMP1, TMP0, TMP1 |
| @@ -4203,10 +4203,10 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop) | |||
| 4203 | |->BC_TSETS_Z: | 4203 | |->BC_TSETS_Z: |
| 4204 | | // TAB:RB = GCtab *, STR:RC = GCstr *, RA = BASE+src*8 | 4204 | | // TAB:RB = GCtab *, STR:RC = GCstr *, RA = BASE+src*8 |
| 4205 | | lw TMP0, TAB:RB->hmask | 4205 | | lw TMP0, TAB:RB->hmask |
| 4206 | | lw TMP1, STR:RC->hash | 4206 | | lw TMP1, STR:RC->sid |
| 4207 | | lw NODE:TMP2, TAB:RB->node | 4207 | | lw NODE:TMP2, TAB:RB->node |
| 4208 | | sb r0, TAB:RB->nomm // Clear metamethod cache. | 4208 | | sb r0, TAB:RB->nomm // Clear metamethod cache. |
| 4209 | | and TMP1, TMP1, TMP0 // idx = str->hash & tab->hmask | 4209 | | and TMP1, TMP1, TMP0 // idx = str->sid & tab->hmask |
| 4210 | | sll TMP0, TMP1, 5 | 4210 | | sll TMP0, TMP1, 5 |
| 4211 | | sll TMP1, TMP1, 3 | 4211 | | sll TMP1, TMP1, 3 |
| 4212 | | subu TMP1, TMP0, TMP1 | 4212 | | subu TMP1, TMP0, TMP1 |
diff --git a/src/vm_mips64.dasc b/src/vm_mips64.dasc index dac143a4..71acf9ed 100644 --- a/src/vm_mips64.dasc +++ b/src/vm_mips64.dasc | |||
| @@ -1201,9 +1201,9 @@ static void build_subroutines(BuildCtx *ctx) | |||
| 1201 | | beqz TAB:RB, ->fff_restv | 1201 | | beqz TAB:RB, ->fff_restv |
| 1202 | |. li CARG1, LJ_TNIL | 1202 | |. li CARG1, LJ_TNIL |
| 1203 | | lw TMP0, TAB:RB->hmask | 1203 | | lw TMP0, TAB:RB->hmask |
| 1204 | | lw TMP1, STR:RC->hash | 1204 | | lw TMP1, STR:RC->sid |
| 1205 | | ld NODE:TMP2, TAB:RB->node | 1205 | | ld NODE:TMP2, TAB:RB->node |
| 1206 | | and TMP1, TMP1, TMP0 // idx = str->hash & tab->hmask | 1206 | | and TMP1, TMP1, TMP0 // idx = str->sid & tab->hmask |
| 1207 | | dsll TMP0, TMP1, 5 | 1207 | | dsll TMP0, TMP1, 5 |
| 1208 | | dsll TMP1, TMP1, 3 | 1208 | | dsll TMP1, TMP1, 3 |
| 1209 | | dsubu TMP1, TMP0, TMP1 | 1209 | | dsubu TMP1, TMP0, TMP1 |
| @@ -4239,9 +4239,9 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop) | |||
| 4239 | |->BC_TGETS_Z: | 4239 | |->BC_TGETS_Z: |
| 4240 | | // TAB:RB = GCtab *, STR:RC = GCstr *, RA = dst*8 | 4240 | | // TAB:RB = GCtab *, STR:RC = GCstr *, RA = dst*8 |
| 4241 | | lw TMP0, TAB:RB->hmask | 4241 | | lw TMP0, TAB:RB->hmask |
| 4242 | | lw TMP1, STR:RC->hash | 4242 | | lw TMP1, STR:RC->sid |
| 4243 | | ld NODE:TMP2, TAB:RB->node | 4243 | | ld NODE:TMP2, TAB:RB->node |
| 4244 | | and TMP1, TMP1, TMP0 // idx = str->hash & tab->hmask | 4244 | | and TMP1, TMP1, TMP0 // idx = str->sid & tab->hmask |
| 4245 | | sll TMP0, TMP1, 5 | 4245 | | sll TMP0, TMP1, 5 |
| 4246 | | sll TMP1, TMP1, 3 | 4246 | | sll TMP1, TMP1, 3 |
| 4247 | | subu TMP1, TMP0, TMP1 | 4247 | | subu TMP1, TMP0, TMP1 |
| @@ -4402,10 +4402,10 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop) | |||
| 4402 | |->BC_TSETS_Z: | 4402 | |->BC_TSETS_Z: |
| 4403 | | // TAB:RB = GCtab *, STR:RC = GCstr *, RA = BASE+src*8 | 4403 | | // TAB:RB = GCtab *, STR:RC = GCstr *, RA = BASE+src*8 |
| 4404 | | lw TMP0, TAB:RB->hmask | 4404 | | lw TMP0, TAB:RB->hmask |
| 4405 | | lw TMP1, STR:RC->hash | 4405 | | lw TMP1, STR:RC->sid |
| 4406 | | ld NODE:TMP2, TAB:RB->node | 4406 | | ld NODE:TMP2, TAB:RB->node |
| 4407 | | sb r0, TAB:RB->nomm // Clear metamethod cache. | 4407 | | sb r0, TAB:RB->nomm // Clear metamethod cache. |
| 4408 | | and TMP1, TMP1, TMP0 // idx = str->hash & tab->hmask | 4408 | | and TMP1, TMP1, TMP0 // idx = str->sid & tab->hmask |
| 4409 | | sll TMP0, TMP1, 5 | 4409 | | sll TMP0, TMP1, 5 |
| 4410 | | sll TMP1, TMP1, 3 | 4410 | | sll TMP1, TMP1, 3 |
| 4411 | | subu TMP1, TMP0, TMP1 | 4411 | | subu TMP1, TMP0, TMP1 |
diff --git a/src/vm_ppc.dasc b/src/vm_ppc.dasc index 7a2d321e..18fc6f93 100644 --- a/src/vm_ppc.dasc +++ b/src/vm_ppc.dasc | |||
| @@ -1447,9 +1447,9 @@ static void build_subroutines(BuildCtx *ctx) | |||
| 1447 | | beq ->fff_restv | 1447 | | beq ->fff_restv |
| 1448 | | lwz TMP0, TAB:CARG1->hmask | 1448 | | lwz TMP0, TAB:CARG1->hmask |
| 1449 | | li CARG3, LJ_TTAB // Use metatable as default result. | 1449 | | li CARG3, LJ_TTAB // Use metatable as default result. |
| 1450 | | lwz TMP1, STR:RC->hash | 1450 | | lwz TMP1, STR:RC->sid |
| 1451 | | lwz NODE:TMP2, TAB:CARG1->node | 1451 | | lwz NODE:TMP2, TAB:CARG1->node |
| 1452 | | and TMP1, TMP1, TMP0 // idx = str->hash & tab->hmask | 1452 | | and TMP1, TMP1, TMP0 // idx = str->sid & tab->hmask |
| 1453 | | slwi TMP0, TMP1, 5 | 1453 | | slwi TMP0, TMP1, 5 |
| 1454 | | slwi TMP1, TMP1, 3 | 1454 | | slwi TMP1, TMP1, 3 |
| 1455 | | sub TMP1, TMP0, TMP1 | 1455 | | sub TMP1, TMP0, TMP1 |
| @@ -4588,9 +4588,9 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop) | |||
| 4588 | |->BC_TGETS_Z: | 4588 | |->BC_TGETS_Z: |
| 4589 | | // TAB:RB = GCtab *, STR:RC = GCstr *, RA = dst*8 | 4589 | | // TAB:RB = GCtab *, STR:RC = GCstr *, RA = dst*8 |
| 4590 | | lwz TMP0, TAB:RB->hmask | 4590 | | lwz TMP0, TAB:RB->hmask |
| 4591 | | lwz TMP1, STR:RC->hash | 4591 | | lwz TMP1, STR:RC->sid |
| 4592 | | lwz NODE:TMP2, TAB:RB->node | 4592 | | lwz NODE:TMP2, TAB:RB->node |
| 4593 | | and TMP1, TMP1, TMP0 // idx = str->hash & tab->hmask | 4593 | | and TMP1, TMP1, TMP0 // idx = str->sid & tab->hmask |
| 4594 | | slwi TMP0, TMP1, 5 | 4594 | | slwi TMP0, TMP1, 5 |
| 4595 | | slwi TMP1, TMP1, 3 | 4595 | | slwi TMP1, TMP1, 3 |
| 4596 | | sub TMP1, TMP0, TMP1 | 4596 | | sub TMP1, TMP0, TMP1 |
| @@ -4784,10 +4784,10 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop) | |||
| 4784 | |->BC_TSETS_Z: | 4784 | |->BC_TSETS_Z: |
| 4785 | | // TAB:RB = GCtab *, STR:RC = GCstr *, RA = src*8 | 4785 | | // TAB:RB = GCtab *, STR:RC = GCstr *, RA = src*8 |
| 4786 | | lwz TMP0, TAB:RB->hmask | 4786 | | lwz TMP0, TAB:RB->hmask |
| 4787 | | lwz TMP1, STR:RC->hash | 4787 | | lwz TMP1, STR:RC->sid |
| 4788 | | lwz NODE:TMP2, TAB:RB->node | 4788 | | lwz NODE:TMP2, TAB:RB->node |
| 4789 | | stb ZERO, TAB:RB->nomm // Clear metamethod cache. | 4789 | | stb ZERO, TAB:RB->nomm // Clear metamethod cache. |
| 4790 | | and TMP1, TMP1, TMP0 // idx = str->hash & tab->hmask | 4790 | | and TMP1, TMP1, TMP0 // idx = str->sid & tab->hmask |
| 4791 | |.if FPU | 4791 | |.if FPU |
| 4792 | | lfdx f14, BASE, RA | 4792 | | lfdx f14, BASE, RA |
| 4793 | |.else | 4793 | |.else |
diff --git a/src/vm_x64.dasc b/src/vm_x64.dasc index 77a579d5..14a54e34 100644 --- a/src/vm_x64.dasc +++ b/src/vm_x64.dasc | |||
| @@ -1230,7 +1230,7 @@ static void build_subroutines(BuildCtx *ctx) | |||
| 1230 | | mov [BASE-16], TAB:RC // Store metatable as default result. | 1230 | | mov [BASE-16], TAB:RC // Store metatable as default result. |
| 1231 | | mov STR:RC, [DISPATCH+DISPATCH_GL(gcroot)+8*(GCROOT_MMNAME+MM_metatable)] | 1231 | | mov STR:RC, [DISPATCH+DISPATCH_GL(gcroot)+8*(GCROOT_MMNAME+MM_metatable)] |
| 1232 | | mov RAd, TAB:RB->hmask | 1232 | | mov RAd, TAB:RB->hmask |
| 1233 | | and RAd, STR:RC->hash | 1233 | | and RAd, STR:RC->sid |
| 1234 | | settp STR:RC, LJ_TSTR | 1234 | | settp STR:RC, LJ_TSTR |
| 1235 | | imul RAd, #NODE | 1235 | | imul RAd, #NODE |
| 1236 | | add NODE:RA, TAB:RB->node | 1236 | | add NODE:RA, TAB:RB->node |
| @@ -3674,7 +3674,7 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop) | |||
| 3674 | | checktab TAB:RB, ->vmeta_tgets | 3674 | | checktab TAB:RB, ->vmeta_tgets |
| 3675 | |->BC_TGETS_Z: // RB = GCtab *, RC = GCstr * | 3675 | |->BC_TGETS_Z: // RB = GCtab *, RC = GCstr * |
| 3676 | | mov TMPRd, TAB:RB->hmask | 3676 | | mov TMPRd, TAB:RB->hmask |
| 3677 | | and TMPRd, STR:RC->hash | 3677 | | and TMPRd, STR:RC->sid |
| 3678 | | imul TMPRd, #NODE | 3678 | | imul TMPRd, #NODE |
| 3679 | | add NODE:TMPR, TAB:RB->node | 3679 | | add NODE:TMPR, TAB:RB->node |
| 3680 | | settp ITYPE, STR:RC, LJ_TSTR | 3680 | | settp ITYPE, STR:RC, LJ_TSTR |
| @@ -3806,7 +3806,7 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop) | |||
| 3806 | | checktab TAB:RB, ->vmeta_tsets | 3806 | | checktab TAB:RB, ->vmeta_tsets |
| 3807 | |->BC_TSETS_Z: // RB = GCtab *, RC = GCstr * | 3807 | |->BC_TSETS_Z: // RB = GCtab *, RC = GCstr * |
| 3808 | | mov TMPRd, TAB:RB->hmask | 3808 | | mov TMPRd, TAB:RB->hmask |
| 3809 | | and TMPRd, STR:RC->hash | 3809 | | and TMPRd, STR:RC->sid |
| 3810 | | imul TMPRd, #NODE | 3810 | | imul TMPRd, #NODE |
| 3811 | | mov byte TAB:RB->nomm, 0 // Clear metamethod cache. | 3811 | | mov byte TAB:RB->nomm, 0 // Clear metamethod cache. |
| 3812 | | add NODE:TMPR, TAB:RB->node | 3812 | | add NODE:TMPR, TAB:RB->node |
diff --git a/src/vm_x86.dasc b/src/vm_x86.dasc index 57c8e4fc..f9bea426 100644 --- a/src/vm_x86.dasc +++ b/src/vm_x86.dasc | |||
| @@ -1522,7 +1522,7 @@ static void build_subroutines(BuildCtx *ctx) | |||
| 1522 | | mov dword [BASE-4], LJ_TTAB // Store metatable as default result. | 1522 | | mov dword [BASE-4], LJ_TTAB // Store metatable as default result. |
| 1523 | | mov [BASE-8], TAB:RB | 1523 | | mov [BASE-8], TAB:RB |
| 1524 | | mov RA, TAB:RB->hmask | 1524 | | mov RA, TAB:RB->hmask |
| 1525 | | and RA, STR:RC->hash | 1525 | | and RA, STR:RC->sid |
| 1526 | | imul RA, #NODE | 1526 | | imul RA, #NODE |
| 1527 | | add NODE:RA, TAB:RB->node | 1527 | | add NODE:RA, TAB:RB->node |
| 1528 | |3: // Rearranged logic, because we expect _not_ to find the key. | 1528 | |3: // Rearranged logic, because we expect _not_ to find the key. |
| @@ -4286,7 +4286,7 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop) | |||
| 4286 | | mov TAB:RB, [BASE+RB*8] | 4286 | | mov TAB:RB, [BASE+RB*8] |
| 4287 | |->BC_TGETS_Z: // RB = GCtab *, RC = GCstr *, refetches PC_RA. | 4287 | |->BC_TGETS_Z: // RB = GCtab *, RC = GCstr *, refetches PC_RA. |
| 4288 | | mov RA, TAB:RB->hmask | 4288 | | mov RA, TAB:RB->hmask |
| 4289 | | and RA, STR:RC->hash | 4289 | | and RA, STR:RC->sid |
| 4290 | | imul RA, #NODE | 4290 | | imul RA, #NODE |
| 4291 | | add NODE:RA, TAB:RB->node | 4291 | | add NODE:RA, TAB:RB->node |
| 4292 | |1: | 4292 | |1: |
| @@ -4457,7 +4457,7 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop) | |||
| 4457 | | mov TAB:RB, [BASE+RB*8] | 4457 | | mov TAB:RB, [BASE+RB*8] |
| 4458 | |->BC_TSETS_Z: // RB = GCtab *, RC = GCstr *, refetches PC_RA. | 4458 | |->BC_TSETS_Z: // RB = GCtab *, RC = GCstr *, refetches PC_RA. |
| 4459 | | mov RA, TAB:RB->hmask | 4459 | | mov RA, TAB:RB->hmask |
| 4460 | | and RA, STR:RC->hash | 4460 | | and RA, STR:RC->sid |
| 4461 | | imul RA, #NODE | 4461 | | imul RA, #NODE |
| 4462 | | mov byte TAB:RB->nomm, 0 // Clear metamethod cache. | 4462 | | mov byte TAB:RB->nomm, 0 // Clear metamethod cache. |
| 4463 | | add NODE:RA, TAB:RB->node | 4463 | | add NODE:RA, TAB:RB->node |
