Add special bytecodes for builtins.

BC_ISTYPE, BC_ISNUM: fast type checks/coercions.
BC_TGETR, BC_TSETR: fast rawgeti/rawseti, no type checks for table/key.

16 files changed, 614 insertions, 38 deletions

diff --git a/src/Makefile.dep b/src/Makefile.dep
index 54004f7c..10118c5e 100644
--- a/src/Makefile.dep
+++ b/src/Makefile.dep
@@ -134,7 +134,7 @@ lj_mcode.o: lj_mcode.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \
  lj_traceerr.h lj_vm.h
 lj_meta.o: lj_meta.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h lj_gc.h \
  lj_err.h lj_errmsg.h lj_str.h lj_tab.h lj_meta.h lj_frame.h lj_bc.h \
- lj_vm.h lj_strscan.h
+ lj_vm.h lj_strscan.h lj_lib.h
 lj_obj.o: lj_obj.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h
 lj_opt_dce.o: lj_opt_dce.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \
  lj_ir.h lj_jit.h lj_iropt.h
@@ -195,13 +195,13 @@ ljamalg.o: ljamalg.c lua.h luaconf.h lauxlib.h lj_gc.c lj_obj.h lj_def.h \
  lj_trace.h lj_jit.h lj_ir.h lj_dispatch.h lj_traceerr.h lj_vm.h lj_err.c \
  lj_debug.h lj_ff.h lj_ffdef.h lj_char.c lj_char.h lj_bc.c lj_bcdef.h \
  lj_obj.c lj_str.c lj_tab.c lj_func.c lj_udata.c lj_meta.c lj_strscan.h \
- lj_debug.c lj_state.c lj_lex.h lj_alloc.h lj_dispatch.c lj_ccallback.h \
- luajit.h lj_vmevent.c lj_vmevent.h lj_vmmath.c lj_strscan.c lj_api.c \
- lj_lex.c lualib.h lj_parse.h lj_parse.c lj_bcread.c lj_bcdump.h \
- lj_bcwrite.c lj_load.c lj_ctype.c lj_cdata.c lj_cconv.h lj_cconv.c \
- lj_ccall.c lj_ccall.h lj_ccallback.c lj_target.h lj_target_*.h \
- lj_mcode.h lj_carith.c lj_carith.h lj_clib.c lj_clib.h lj_cparse.c \
- lj_cparse.h lj_lib.c lj_lib.h lj_ir.c lj_ircall.h lj_iropt.h \
+ lj_lib.h lj_debug.c lj_state.c lj_lex.h lj_alloc.h lj_dispatch.c \
+ lj_ccallback.h luajit.h lj_vmevent.c lj_vmevent.h lj_vmmath.c \
+ lj_strscan.c lj_api.c lj_lex.c lualib.h lj_parse.h lj_parse.c \
+ lj_bcread.c lj_bcdump.h lj_bcwrite.c lj_load.c lj_ctype.c lj_cdata.c \
+ lj_cconv.h lj_cconv.c lj_ccall.c lj_ccall.h lj_ccallback.c lj_target.h \
+ lj_target_*.h lj_mcode.h lj_carith.c lj_carith.h lj_clib.c lj_clib.h \
+ lj_cparse.c lj_cparse.h lj_lib.c lj_ir.c lj_ircall.h lj_iropt.h \
  lj_opt_mem.c lj_opt_fold.c lj_folddef.h lj_opt_narrow.c lj_opt_dce.c \
  lj_opt_loop.c lj_snap.h lj_opt_split.c lj_opt_sink.c lj_mcode.c \
  lj_snap.c lj_record.c lj_record.h lj_ffrecord.h lj_crecord.c \
@@ -221,7 +221,7 @@ host/buildvm_asm.o: host/buildvm_asm.c host/buildvm.h lj_def.h lua.h luaconf.h \
 host/buildvm_fold.o: host/buildvm_fold.c host/buildvm.h lj_def.h lua.h \
  luaconf.h lj_arch.h lj_obj.h lj_def.h lj_arch.h lj_ir.h lj_obj.h
 host/buildvm_lib.o: host/buildvm_lib.c host/buildvm.h lj_def.h lua.h luaconf.h \
- lj_arch.h lj_obj.h lj_def.h lj_arch.h lj_lib.h lj_obj.h \
+ lj_arch.h lj_obj.h lj_def.h lj_arch.h lj_bc.h lj_lib.h lj_obj.h \
  host/buildvm_libbc.h
 host/buildvm_peobj.o: host/buildvm_peobj.c host/buildvm.h lj_def.h lua.h \
  luaconf.h lj_arch.h lj_bc.h lj_def.h lj_arch.h
diff --git a/src/host/buildvm_lib.c b/src/host/buildvm_lib.c
index 182ab90f..dcd3ca41 100644
--- a/src/host/buildvm_lib.c
+++ b/src/host/buildvm_lib.c
@@ -5,6 +5,7 @@
 
 #include "buildvm.h"
 #include "lj_obj.h"
+#include "lj_bc.h"
 #include "lj_lib.h"
 #include "buildvm_libbc.h"
 
@@ -152,28 +153,36 @@ static void libdef_func(BuildCtx *ctx, char *p, int arg)
   regfunc = REGFUNC_OK;
 }
 
-static uint32_t libdef_uleb128(uint8_t **pp)
+static uint8_t *libdef_uleb128(uint8_t *p, uint32_t *vv)
 {
-  uint8_t *p = *pp;
   uint32_t v = *p++;
   if (v >= 0x80) {
     int sh = 0; v &= 0x7f;
     do { v |= ((*p & 0x7f) << (sh += 7)); } while (*p++ >= 0x80);
   }
-  *pp = p;
-  return v;
+  *vv = v;
+  return p;
 }
 
-static void libdef_swapbc(uint8_t *p)
+static void libdef_fixupbc(uint8_t *p)
 {
   uint32_t i, sizebc;
   p += 4;
-  libdef_uleb128(&p);
-  libdef_uleb128(&p);
-  sizebc = libdef_uleb128(&p);
+  p = libdef_uleb128(p, &sizebc);
+  p = libdef_uleb128(p, &sizebc);
+  p = libdef_uleb128(p, &sizebc);
   for (i = 0; i < sizebc; i++, p += 4) {
-    uint8_t t = p[0]; p[0] = p[3]; p[3] = t;
-    t = p[1]; p[1] = p[2]; p[2] = t;
+    uint8_t op = p[libbc_endian ? 3 : 0];
+    uint8_t ra = p[libbc_endian ? 2 : 1];
+    uint8_t rc = p[libbc_endian ? 1 : 2];
+    uint8_t rb = p[libbc_endian ? 0 : 3];
+    if (!LJ_DUALNUM && op == BC_ISTYPE && rc == ~LJ_TNUMX+1) {
+      op = BC_ISNUM; rc++;
+    }
+    p[LJ_ENDIAN_SELECT(0, 3)] = op;
+    p[LJ_ENDIAN_SELECT(1, 2)] = ra;
+    p[LJ_ENDIAN_SELECT(2, 1)] = rc;
+    p[LJ_ENDIAN_SELECT(3, 0)] = rb;
   }
 }
 
@@ -190,8 +199,7 @@ static void libdef_lua(BuildCtx *ctx, char *p, int arg)
         *optr++ = LIBINIT_LUA;
         libdef_name(p, 0);
         memcpy(optr, libbc_code + ofs, len);
-        if (libbc_endian != LJ_BE)
-          libdef_swapbc(optr);
+        libdef_fixupbc(optr);
         optr += len;
         return;
       }
diff --git a/src/host/buildvm_libbc.h b/src/host/buildvm_libbc.h
index ee97836a..6b0e1d03 100644
--- a/src/host/buildvm_libbc.h
+++ b/src/host/buildvm_libbc.h
@@ -3,8 +3,8 @@
 static const int libbc_endian = 0;
 
 static const uint8_t libbc_code[] = {
-0,1,2,0,0,1,2,22,1,0,0,72,1,2,0,241,135,158,166,3,220,203,178,130,4,0,1,2,0,
-0,1,2,22,1,0,0,72,1,2,0,243,244,148,165,20,198,190,199,252,3,0
+0,1,2,0,0,1,2,24,1,0,0,76,1,2,0,241,135,158,166,3,220,203,178,130,4,0,1,2,0,
+0,1,2,24,1,0,0,76,1,2,0,243,244,148,165,20,198,190,199,252,3,0
 };
 
 static const struct { const char *name; int ofs; } libbc_map[] = {
diff --git a/src/host/genlibbc.lua b/src/host/genlibbc.lua
index bdcfb588..72c55d73 100644
--- a/src/host/genlibbc.lua
+++ b/src/host/genlibbc.lua
@@ -6,6 +6,15 @@
 -- Released under the MIT license. See Copyright Notice in luajit.h
 ----------------------------------------------------------------------------
 
+local ffi = require("ffi")
+local bit = require("bit")
+local vmdef = require("jit.vmdef")
+local bcnames = vmdef.bcnames
+
+local format = string.format
+
+local isbe = (string.byte(string.dump(function() end), 5) % 2 == 1)
+
 local function usage(arg)
   io.stderr:write("Usage: ", arg and arg[0] or "genlibbc",
                   " [-o buildvm_libbc.h] lib_*.c\n")
@@ -36,15 +45,100 @@ local function read_files(names)
   return src
 end
 
+local function transform_lua(code)
+  local fixup = {}
+  local n = -30000
+  code = string.gsub(code, "CHECK_(%w*)%((.-)%)", function(tp, var)
+    n = n + 1
+    fixup[n] = { "CHECK", tp }
+    return format("%s=%d", var, n)
+  end)
+  code = string.gsub(code, "PAIRS%((.-)%)", function(var)
+    fixup.PAIRS = true
+    return format("nil, %s, 0", var)
+  end)
+  return "return "..code, fixup
+end
+
+local function read_uleb128(p)
+  local v = p[0]; p = p + 1
+  if v >= 128 then
+    local sh = 7; v = v - 128
+    repeat
+      local r = p[0]
+      v = v + bit.lshift(bit.band(r, 128), sh)
+      sh = sh + 7
+      p = p + 1
+    until r < 128
+  end
+  return p, v
+end
+
+-- ORDER LJ_T
+local name2itype = {
+  str = 5, func = 9, tab = 12, int = 14, num = 15
+}
+
+local BC = {}
+for i=0,#bcnames/6-1 do
+  BC[string.gsub(string.sub(bcnames, i*6+1, i*6+6), " ", "")] = i
+end
+local xop, xra = isbe and 3 or 0, isbe and 2 or 1
+local xrc, xrb = isbe and 1 or 2, isbe and 0 or 3
+
+local function fixup_dump(dump, fixup)
+  local buf = ffi.new("uint8_t[?]", #dump+1, dump)
+  local p = buf+5
+  local n, sizebc
+  p, n = read_uleb128(p)
+  local start = p
+  p = p + 4
+  p = read_uleb128(p)
+  p = read_uleb128(p)
+  p, sizebc = read_uleb128(p)
+  local rawtab = {}
+  for i=0,sizebc-1 do
+    local op = p[xop]
+    if op == BC.KSHORT then
+      local rd = p[xrc] + 256*p[xrb]
+      rd = bit.arshift(bit.lshift(rd, 16), 16)
+      local f = fixup[rd]
+      if f then
+        if f[1] == "CHECK" then
+          local tp = f[2]
+          if tp == "tab" then rawtab[p[xra]] = true end
+          p[xop] = tp == "num" and BC.ISNUM or BC.ISTYPE
+          p[xrb] = 0
+          p[xrc] = name2itype[tp]
+        else
+          error("unhandled fixup type: "..f[1])
+        end
+      end
+    elseif op == BC.TGETV then
+      if rawtab[p[xrb]] then
+        p[xop] = BC.TGETR
+      end
+    elseif op == BC.TSETV then
+      if rawtab[p[xrb]] then
+        p[xop] = BC.TSETR
+      end
+    elseif op == BC.ITERC then
+      if fixup.PAIRS then
+        p[xop] = BC.ITERN
+      end
+    end
+    p = p + 4
+  end
+  return ffi.string(start, n)
+end
+
 local function find_defs(src)
   local defs = {}
   for name, code in string.gmatch(src, "LJLIB_LUA%(([^)]*)%)%s*/%*(.-)%*/") do
     local env = {}
-    local func = assert(load("return "..code, "", nil, env))()
-    local d = string.dump(func, true)
-    local ofs = 6
-    while string.byte(d, ofs) > 127 do ofs = ofs + 1 end
-    defs[name] = string.sub(d, ofs+1, -2)
+    local tcode, fixup = transform_lua(code)
+    local func = assert(load(tcode, "", nil, env))()
+    defs[name] = fixup_dump(string.dump(func, true), fixup)
     defs[#defs+1] = name
   end
   return defs
@@ -54,9 +148,7 @@ local function gen_header(defs)
   local t = {}
   local function w(x) t[#t+1] = x end
   w("/* This is a generated file. DO NOT EDIT! */\n\n")
-  w("static const int libbc_endian = ")
-  w(string.byte(string.dump(function() end), 5) % 2)
-  w(";\n\n")
+  w("static const int libbc_endian = ") w(isbe and 1 or 0) w(";\n\n")
   local s = ""
   for _,name in ipairs(defs) do
     s = s .. defs[name]
diff --git a/src/lj_arch.h b/src/lj_arch.h
index 9ea10d0f..c5f2fb3d 100644
--- a/src/lj_arch.h
+++ b/src/lj_arch.h
@@ -227,6 +227,7 @@
 
 #elif LUAJIT_TARGET == LUAJIT_ARCH_PPCSPE
 
+#error "The PPC/e500 port is broken and will be abandoned with LuaJIT 2.1"
 #define LJ_ARCH_NAME            "ppcspe"
 #define LJ_ARCH_BITS            32
 #define LJ_ARCH_ENDIAN          LUAJIT_BE
diff --git a/src/lj_bc.h b/src/lj_bc.h
index 56e71dd9..ac9cc5e1 100644
--- a/src/lj_bc.h
+++ b/src/lj_bc.h
@@ -89,6 +89,8 @@
   _(ISFC,       dst,    ___,    var,    ___) \
   _(IST,        ___,    ___,    var,    ___) \
   _(ISF,        ___,    ___,    var,    ___) \
+  _(ISTYPE,     var,    ___,    lit,    ___) \
+  _(ISNUM,      var,    ___,    lit,    ___) \
   \
   /* Unary ops. */ \
   _(MOV,        dst,    ___,    var,    ___) \
@@ -143,10 +145,12 @@
   _(TGETV,      dst,    var,    var,    index) \
   _(TGETS,      dst,    var,    str,    index) \
   _(TGETB,      dst,    var,    lit,    index) \
+  _(TGETR,      dst,    var,    var,    index) \
   _(TSETV,      var,    var,    var,    newindex) \
   _(TSETS,      var,    var,    str,    newindex) \
   _(TSETB,      var,    var,    lit,    newindex) \
   _(TSETM,      base,   ___,    num,    newindex) \
+  _(TSETR,      var,    var,    var,    newindex) \
   \
   /* Calls and vararg handling. T = tail call. */ \
   _(CALLM,      base,   lit,    lit,    call) \
diff --git a/src/lj_bcdump.h b/src/lj_bcdump.h
index c1ed54e7..22a8b823 100644
--- a/src/lj_bcdump.h
+++ b/src/lj_bcdump.h
@@ -36,7 +36,7 @@
 /* If you perform *any* kind of private modifications to the bytecode itself
 ** or to the dump format, you *must* set BCDUMP_VERSION to 0x80 or higher.
 */
-#define BCDUMP_VERSION          1
+#define BCDUMP_VERSION          2
 
 /* Compatibility flags. */
 #define BCDUMP_F_BE             0x01
diff --git a/src/lj_dispatch.h b/src/lj_dispatch.h
index 57614d99..a662439b 100644
--- a/src/lj_dispatch.h
+++ b/src/lj_dispatch.h
@@ -33,11 +33,11 @@
   _(lj_ffh_coroutine_wrap_err) _(lj_func_closeuv) _(lj_func_newL_gc) \
   _(lj_gc_barrieruv) _(lj_gc_step) _(lj_gc_step_fixtop) _(lj_meta_arith) \
   _(lj_meta_call) _(lj_meta_cat) _(lj_meta_comp) _(lj_meta_equal) \
-  _(lj_meta_for) _(lj_meta_len) _(lj_meta_tget) _(lj_meta_tset) \
-  _(lj_state_growstack) _(lj_str_fromnum) _(lj_str_fromnumber) _(lj_str_new) \
-  _(lj_tab_dup) _(lj_tab_get) _(lj_tab_getinth) _(lj_tab_len) _(lj_tab_new) \
-  _(lj_tab_newkey) _(lj_tab_next) _(lj_tab_reasize) \
-  JITGOTDEF(_) FFIGOTDEF(_)
+  _(lj_meta_for) _(lj_meta_istype) _(lj_meta_len) _(lj_meta_tget) \
+  _(lj_meta_tset) _(lj_state_growstack) _(lj_str_fromnum) _(lj_str_fromnumber) \
+  _(lj_str_new) _(lj_tab_dup) _(lj_tab_get) _(lj_tab_getinth) _(lj_tab_len) \
+  _(lj_tab_new) _(lj_tab_newkey) _(lj_tab_next) _(lj_tab_reasize) \
+  _(lj_tab_setinth) JITGOTDEF(_) FFIGOTDEF(_)
 
 enum {
 #define GOTENUM(name) LJ_GOT_##name,
diff --git a/src/lj_meta.c b/src/lj_meta.c
index 441d571a..e11f1b75 100644
--- a/src/lj_meta.c
+++ b/src/lj_meta.c
@@ -19,6 +19,7 @@
 #include "lj_bc.h"
 #include "lj_vm.h"
 #include "lj_strscan.h"
+#include "lj_lib.h"
 
 /* -- Metamethod handling ------------------------------------------------- */
 
@@ -423,6 +424,18 @@ TValue *lj_meta_comp(lua_State *L, cTValue *o1, cTValue *o2, int op)
   }
 }
 
+/* Helper for ISTYPE and ISNUM. Implicit coercion or error. */
+void lj_meta_istype(lua_State *L, BCReg ra, BCReg tp)
+{
+  L->top = curr_topL(L);
+  ra++; tp--;
+  lua_assert(LJ_DUALNUM || tp != ~LJ_TNUMX);  /* ISTYPE -> ISNUM broken. */
+  if (LJ_DUALNUM && tp == ~LJ_TNUMX) lj_lib_checkint(L, ra);
+  else if (tp == ~LJ_TNUMX+1) lj_lib_checknum(L, ra);
+  else if (tp == ~LJ_TSTR) lj_lib_checkstr(L, ra);
+  else lj_err_argtype(L, ra, lj_obj_itypename[tp]);
+}
+
 /* Helper for calls. __call metamethod. */
 void lj_meta_call(lua_State *L, TValue *func, TValue *top)
 {
diff --git a/src/lj_meta.h b/src/lj_meta.h
index 6af5e514..970398ec 100644
--- a/src/lj_meta.h
+++ b/src/lj_meta.h
@@ -31,6 +31,7 @@ LJ_FUNCA TValue * LJ_FASTCALL lj_meta_len(lua_State *L, cTValue *o);
 LJ_FUNCA TValue *lj_meta_equal(lua_State *L, GCobj *o1, GCobj *o2, int ne);
 LJ_FUNCA TValue * LJ_FASTCALL lj_meta_equal_cd(lua_State *L, BCIns ins);
 LJ_FUNCA TValue *lj_meta_comp(lua_State *L, cTValue *o1, cTValue *o2, int op);
+LJ_FUNCA void lj_meta_istype(lua_State *L, BCReg ra, BCReg tp);
 LJ_FUNCA void lj_meta_call(lua_State *L, TValue *func, TValue *top);
 LJ_FUNCA void LJ_FASTCALL lj_meta_for(lua_State *L, TValue *o);
 
diff --git a/src/lj_record.c b/src/lj_record.c
index 7336e0ac..003910a9 100644
--- a/src/lj_record.c
+++ b/src/lj_record.c
@@ -1826,6 +1826,18 @@ void lj_record_ins(jit_State *J)
       J->maxslot = bc_a(pc[1]);  /* Shrink used slots. */
     break;
 
+  case BC_ISTYPE: case BC_ISNUM:
+    /* These coercions need to correspond with lj_meta_istype(). */
+    if (LJ_DUALNUM && rc == ~LJ_TNUMX+1)
+      ra = lj_opt_narrow_toint(J, ra);
+    else if (rc == ~LJ_TNUMX+2)
+      ra = lj_ir_tonum(J, ra);
+    else if (rc == ~LJ_TSTR+1)
+      ra = lj_ir_tostr(J, ra);
+    /* else: type specialization suffices. */
+    J->base[bc_a(ins)] = ra;
+    break;
+
   /* -- Unary ops --------------------------------------------------------- */
 
   case BC_NOT:
@@ -1937,6 +1949,10 @@ void lj_record_ins(jit_State *J)
     ix.idxchain = LJ_MAX_IDXCHAIN;
     rc = lj_record_idx(J, &ix);
     break;
+  case BC_TGETR: case BC_TSETR:
+    ix.idxchain = 0;
+    rc = lj_record_idx(J, &ix);
+    break;
 
   case BC_TNEW:
     rc = rec_tnew(J, rc);
diff --git a/src/lj_tab.h b/src/lj_tab.h
index 2787caa0..d361137c 100644
--- a/src/lj_tab.h
+++ b/src/lj_tab.h
@@ -50,7 +50,7 @@ LJ_FUNCA cTValue *lj_tab_get(lua_State *L, GCtab *t, cTValue *key);
 /* Caveat: all setters require a write barrier for the stored value. */
 
 LJ_FUNCA TValue *lj_tab_newkey(lua_State *L, GCtab *t, cTValue *key);
-LJ_FUNC TValue *lj_tab_setinth(lua_State *L, GCtab *t, int32_t key);
+LJ_FUNCA TValue *lj_tab_setinth(lua_State *L, GCtab *t, int32_t key);
 LJ_FUNC TValue *lj_tab_setstr(lua_State *L, GCtab *t, GCstr *key);
 LJ_FUNC TValue *lj_tab_set(lua_State *L, GCtab *t, cTValue *key);
 
diff --git a/src/vm_arm.dasc b/src/vm_arm.dasc
index c46d9243..d999d5ff 100644
--- a/src/vm_arm.dasc
+++ b/src/vm_arm.dasc
@@ -615,6 +615,16 @@ static void build_subroutines(BuildCtx *ctx)
   |  ldr LFUNC:CARG3, [BASE, FRAME_FUNC]  // Guaranteed to be a function here.
   |  b ->vm_call_dispatch_f
   |
+  |->vmeta_tgetr:
+  |  .IOS mov RC, BASE
+  |  bl extern lj_tab_getinth           // (GCtab *t, int32_t key)
+  |  // Returns cTValue * or NULL.
+  |  .IOS mov BASE, RC
+  |  cmp CRET1, #0
+  |  ldrdne CARG12, [CRET1]
+  |  mvneq CARG2, #~LJ_TNIL
+  |  b ->BC_TGETR_Z
+  |
   |//-----------------------------------------------------------------------
   |
   |->vmeta_tsets1:
@@ -672,6 +682,15 @@ static void build_subroutines(BuildCtx *ctx)
   |  ldr LFUNC:CARG3, [BASE, FRAME_FUNC]  // Guaranteed to be a function here.
   |  b ->vm_call_dispatch_f
   |
+  |->vmeta_tsetr:
+  |  str BASE, L->base
+  |  .IOS mov RC, BASE
+  |  str PC, SAVE_PC
+  |  bl extern lj_tab_setinth  // (lua_State *L, GCtab *t, int32_t key)
+  |  // Returns TValue *.
+  |  .IOS mov BASE, RC
+  |  b ->BC_TSETR_Z
+  |
   |//-- Comparison metamethods ---------------------------------------------
   |
   |->vmeta_comp:
@@ -736,6 +755,17 @@ static void build_subroutines(BuildCtx *ctx)
   |  b <3
   |.endif
   |
+  |->vmeta_istype:
+  |  sub PC, PC, #4
+  |   str BASE, L->base
+  |   mov CARG1, L
+  |   lsr CARG2, RA, #3
+  |   mov CARG3, RC
+  |  str PC, SAVE_PC
+  |  bl extern lj_meta_istype  // (lua_State *L, BCReg ra, BCReg tp)
+  |  .IOS ldr BASE, L->base
+  |  b ->cont_nop
+  |
   |//-- Arithmetic metamethods ---------------------------------------------
   |
   |->vmeta_arith_vn:
@@ -2821,6 +2851,25 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
     |  ins_next
     break;
 
+  case BC_ISTYPE:
+    |  // RA = src*8, RC = -type
+    |  ldrd CARG12, [BASE, RA]
+    |   ins_next1
+    |  cmn CARG2, RC
+    |   ins_next2
+    |  bne ->vmeta_istype
+    |   ins_next3
+    break;
+  case BC_ISNUM:
+    |  // RA = src*8, RC = -(TISNUM-1)
+    |  ldrd CARG12, [BASE, RA]
+    |   ins_next1
+    |  checktp CARG2, LJ_TISNUM
+    |   ins_next2
+    |  bhs ->vmeta_istype
+    |   ins_next3
+    break;
+
   /* -- Unary ops --------------------------------------------------------- */
 
   case BC_MOV:
@@ -3491,6 +3540,24 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
     |  bne <1                           // 'no __index' flag set: done.
     |  b ->vmeta_tgetb
     break;
+  case BC_TGETR:
+    |  decode_RB8 RB, INS
+    |   decode_RC8 RC, INS
+    |  // RA = dst*8, RB = table*8, RC = key*8
+    |  ldr TAB:CARG1, [BASE, RB]
+    |   ldr CARG2, [BASE, RC]
+    |  ldr CARG4, TAB:CARG1->array
+    |    ldr CARG3, TAB:CARG1->asize
+    |  add CARG4, CARG4, CARG2, lsl #3
+    |    cmp CARG2, CARG3               // In array part?
+    |    bhs ->vmeta_tgetr
+    |  ldrd CARG12, [CARG4]
+    |->BC_TGETR_Z:
+    |   ins_next1
+    |   ins_next2
+    |  strd CARG12, [BASE, RA]
+    |   ins_next3
+    break;
 
   case BC_TSETV:
     |  decode_RB8 RB, INS
@@ -3661,6 +3728,32 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
     |  barrierback TAB:CARG1, INS, CARG3
     |  b <2
     break;
+  case BC_TSETR:
+    |  decode_RB8 RB, INS
+    |   decode_RC8 RC, INS
+    |  // RA = dst*8, RB = table*8, RC = key*8
+    |  ldr TAB:CARG2, [BASE, RB]
+    |   ldr CARG3, [BASE, RC]
+    |     ldrb INS, TAB:CARG2->marked
+    |  ldr CARG1, TAB:CARG2->array
+    |    ldr CARG4, TAB:CARG2->asize
+    |     tst INS, #LJ_GC_BLACK         // isblack(table)
+    |  add CARG1, CARG1, CARG3, lsl #3
+    |     bne >7
+    |2:
+    |    cmp CARG3, CARG4               // In array part?
+    |    bhs ->vmeta_tsetr
+    |->BC_TSETR_Z:
+    |  ldrd CARG34, [BASE, RA]
+    |   ins_next1
+    |   ins_next2
+    |  strd CARG34, [CARG1]
+    |   ins_next3
+    |
+    |7:  // Possible table write barrier for the value. Skip valiswhite check.
+    |  barrierback TAB:CARG2, INS, RB
+    |  b <2
+    break;
 
   case BC_TSETM:
     |  // RA = base*8 (table at base-1), RC = num_const (start index)
diff --git a/src/vm_mips.dasc b/src/vm_mips.dasc
index f37cd931..6db5801f 100644
--- a/src/vm_mips.dasc
+++ b/src/vm_mips.dasc
@@ -688,6 +688,16 @@ static void build_subroutines(BuildCtx *ctx)
   |  b ->vm_call_dispatch_f
   |.  li NARGS8:RC, 16                  // 2 args for func(t, k).
   |
+  |->vmeta_tgetr:
+  |  load_got lj_tab_getinth
+  |  call_intern lj_tab_getinth         // (GCtab *t, int32_t key)
+  |.  nop
+  |  // Returns cTValue * or NULL.
+  |  beqz CRET1, >1
+  |.  nop
+  |  b ->BC_TGETR_Z
+  |.  ldc1 f0, 0(CRET1)
+  |
   |//-----------------------------------------------------------------------
   |
   |->vmeta_tsets1:
@@ -740,6 +750,16 @@ static void build_subroutines(BuildCtx *ctx)
   |  b ->vm_call_dispatch_f
   |.  li NARGS8:RC, 24                  // 3 args for func(t, k, v)
   |
+  |->vmeta_tsetr:
+  |  load_got lj_tab_setinth
+  |  sw BASE, L->base
+  |  sw PC, SAVE_PC
+  |  call_intern lj_tab_setinth  // (lua_State *L, GCtab *t, int32_t key)
+  |.  move CARG1, L
+  |  // Returns TValue *.
+  |  b ->BC_TSETR_Z
+  |.  nop
+  |
   |//-- Comparison metamethods ---------------------------------------------
   |
   |->vmeta_comp:
@@ -813,6 +833,18 @@ static void build_subroutines(BuildCtx *ctx)
   |.  nop
   |.endif
   |
+  |->vmeta_istype:
+  |  load_got lj_meta_istype
+  |  addiu PC, PC, -4
+  |   sw BASE, L->base
+  |   srl CARG2, RA, 3
+  |   srl CARG3, RD, 3
+  |  sw PC, SAVE_PC
+  |  call_intern lj_meta_istype  // (lua_State *L, BCReg ra, BCReg tp)
+  |.  move CARG1, L
+  |  b ->cont_nop
+  |.  nop
+  |
   |//-- Arithmetic metamethods ---------------------------------------------
   |
   |->vmeta_unm:
@@ -2566,6 +2598,26 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
     |  ins_next
     break;
 
+  case BC_ISTYPE:
+    |  // RA = src*8, RD = -type*8
+    |  addu TMP2, BASE, RA
+    |  srl TMP1, RD, 3
+    |  lw TMP0, HI(TMP2)
+    |  ins_next1
+    |  addu AT, TMP0, TMP1
+    |  bnez AT, ->vmeta_istype
+    |.  ins_next2
+    break;
+  case BC_ISNUM:
+    |  // RA = src*8, RD = -(TISNUM-1)*8
+    |  addu TMP2, BASE, RA
+    |  lw TMP0, HI(TMP2)
+    |  ins_next1
+    |  sltiu AT, TMP0, LJ_TISNUM
+    |  beqz AT, ->vmeta_istype
+    |.  ins_next2
+    break;
+
   /* -- Unary ops --------------------------------------------------------- */
 
   case BC_MOV:
@@ -3204,6 +3256,30 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
     |  b ->vmeta_tgetb                  // Caveat: preserve TMP0!
     |.  nop
     break;
+  case BC_TGETR:
+    |  // RA = dst*8, RB = table*8, RC = key*8
+    |  decode_RB8a RB, INS
+    |  decode_RB8b RB
+    |   decode_RDtoRC8 RC, RD
+    |  addu CARG2, BASE, RB
+    |   addu CARG3, BASE, RC
+    |    lw TAB:CARG1, LO(CARG2)
+    |   ldc1 f0, 0(CARG3)
+    |  trunc.w.d f2, f0
+    |   lw TMP0, TAB:CARG1->asize
+    |  mfc1 CARG2, f2
+    |   lw TMP1, TAB:CARG1->array
+    |  sltu AT, CARG2, TMP0
+    |   sll TMP2, CARG2, 3
+    |  beqz AT, ->vmeta_tgetr           // In array part?
+    |.  addu TMP2, TMP1, TMP2
+    |   ldc1 f0, 0(TMP2)
+    |->BC_TGETR_Z:
+    |   addu RA, BASE, RA
+    |  ins_next1
+    |   sdc1 f0, 0(RA)
+    |  ins_next2
+    break;
 
   case BC_TSETV:
     |  // RA = src*8, RB = table*8, RC = key*8
@@ -3392,6 +3468,38 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
     |7:  // Possible table write barrier for the value. Skip valiswhite check.
     |  barrierback TAB:RB, TMP3, TMP0, <2
     break;
+  case BC_TSETR:
+    |  // RA = dst*8, RB = table*8, RC = key*8
+    |  decode_RB8a RB, INS
+    |  decode_RB8b RB
+    |   decode_RDtoRC8 RC, RD
+    |  addu CARG1, BASE, RB
+    |   addu CARG3, BASE, RC
+    |    lw TAB:CARG2, LO(CARG1)
+    |   ldc1 f0, 0(CARG3)
+    |  trunc.w.d f2, f0
+    |    lbu TMP3, TAB:CARG2->marked
+    |   lw TMP0, TAB:CARG2->asize
+    |  mfc1 CARG3, f2
+    |   lw TMP1, TAB:CARG2->array
+    |  andi AT, TMP3, LJ_GC_BLACK       // isblack(table)
+    |  bnez AT, >7
+    |.  addu RA, BASE, RA
+    |2:
+    |  sltu AT, CARG3, TMP0
+    |   sll TMP2, CARG3, 3
+    |  beqz AT, ->vmeta_tsetr           // In array part?
+    |.  ldc1 f20, 0(RA)
+    |   addu CRET1, TMP1, TMP2
+    |->BC_TSETR_Z:
+    |  ins_next1
+    |   sdc1 f20, 0(CRET1)
+    |  ins_next2
+    |
+    |7:  // Possible table write barrier for the value. Skip valiswhite check.
+    |  barrierback TAB:RB, TMP3, TMP0, <2
+    break;
+
 
   case BC_TSETM:
     |  // RA = base*8 (table at base-1), RD = num_const*8 (start index)
diff --git a/src/vm_ppc.dasc b/src/vm_ppc.dasc
index e6fd977b..d76e3a7b 100644
--- a/src/vm_ppc.dasc
+++ b/src/vm_ppc.dasc
@@ -895,6 +895,17 @@ static void build_subroutines(BuildCtx *ctx)
   |   li NARGS8:RC, 16                  // 2 args for func(t, k).
   |  b ->vm_call_dispatch_f
   |
+  |->vmeta_tgetr:
+  |  bl extern lj_tab_getinth           // (GCtab *t, int32_t key)
+  |  // Returns cTValue * or NULL.
+  |  cmplwi CRET1, 0
+  |  beq >1
+  |  lfd f14, 0(CRET1)
+  |  b ->BC_TGETR_Z
+  |1:
+  |  stwx TISNIL, BASE, RA
+  |  b ->cont_nop
+  |
   |//-----------------------------------------------------------------------
   |
   |->vmeta_tsets1:
@@ -962,6 +973,14 @@ static void build_subroutines(BuildCtx *ctx)
   |  stfd f0, 16(BASE)                  // Copy value to third argument.
   |  b ->vm_call_dispatch_f
   |
+  |->vmeta_tsetr:
+  |  stp BASE, L->base
+  |  stw PC, SAVE_PC
+  |  bl extern lj_tab_setinth  // (lua_State *L, GCtab *t, int32_t key)
+  |  // Returns TValue *.
+  |  stfd f14, 0(CRET1)
+  |  b ->cont_nop
+  |
   |//-- Comparison metamethods ---------------------------------------------
   |
   |->vmeta_comp:
@@ -1040,6 +1059,16 @@ static void build_subroutines(BuildCtx *ctx)
   |  b <3
   |.endif
   |
+  |->vmeta_istype:
+  |  subi PC, PC, 4
+  |   stp BASE, L->base
+  |   srwi CARG2, RA, 3
+  |   mr CARG1, L
+  |   srwi CARG3, RD, 3
+  |  stw PC, SAVE_PC
+  |  bl extern lj_meta_istype  // (lua_State *L, BCReg ra, BCReg tp)
+  |  b ->cont_nop
+  |
   |//-- Arithmetic metamethods ---------------------------------------------
   |
   |->vmeta_arith_nv:
@@ -3259,6 +3288,29 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
     |  ins_next
     break;
 
+  case BC_ISTYPE:
+    |  // RA = src*8, RD = -type*8
+    |  lwzx TMP0, BASE, RA
+    |  srwi TMP1, RD, 3
+    |  ins_next1
+    |.if not PPE and not GPR64
+    |  add. TMP0, TMP0, TMP1
+    |.else
+    |  neg TMP1
+    |  cmpw TMP0, TMP1
+    |.endif
+    |  bne ->vmeta_istype
+    |  ins_next2
+    break;
+  case BC_ISNUM:
+    |  // RA = src*8, RD = -(TISNUM-1)*8
+    |  lwzx TMP0, BASE, RA
+    |  ins_next1
+    |  checknum TMP0
+    |  bge ->vmeta_istype
+    |  ins_next2
+    break;
+
   /* -- Unary ops --------------------------------------------------------- */
 
   case BC_MOV:
@@ -4010,6 +4062,30 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
     |  bne <1                           // 'no __index' flag set: done.
     |  b ->vmeta_tgetb                  // Caveat: preserve TMP0!
     break;
+  case BC_TGETR:
+    |  // RA = dst*8, RB = table*8, RC = key*8
+    |  add RB, BASE, RB
+    |  lwz TAB:CARG1, 4(RB)
+    |.if DUALNUM
+    |  add RC, BASE, RC
+    |  lwz TMP0, TAB:CARG1->asize
+    |  lwz CARG2, 4(RC)
+    |   lwz TMP1, TAB:CARG1->array
+    |.else
+    |  lfdx f0, BASE, RC
+    |  lwz TMP0, TAB:CARG1->asize
+    |  toint CARG2, f0
+    |   lwz TMP1, TAB:CARG1->array
+    |.endif
+    |  cmplw TMP0, CARG2
+    |   slwi TMP2, CARG2, 3
+    |  ble ->vmeta_tgetr                // In array part?
+    |   lfdx f14, TMP1, TMP2
+    |->BC_TGETR_Z:
+    |  ins_next1
+    |   stfdx f14, BASE, RA
+    |  ins_next2
+    break;
 
   case BC_TSETV:
     |  // RA = src*8, RB = table*8, RC = key*8
@@ -4189,6 +4265,39 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
     |  barrierback TAB:RB, TMP3, TMP0
     |  b <2
     break;
+  case BC_TSETR:
+    |  // RA = dst*8, RB = table*8, RC = key*8
+    |  add RB, BASE, RB
+    |  lwz TAB:CARG2, 4(RB)
+    |.if DUALNUM
+    |  add RC, BASE, RC
+    |    lbz TMP3, TAB:RB->marked
+    |  lwz TMP0, TAB:CARG2->asize
+    |  lwz CARG3, 4(RC)
+    |   lwz TMP1, TAB:CARG2->array
+    |.else
+    |  lfdx f0, BASE, RC
+    |    lbz TMP3, TAB:RB->marked
+    |  lwz TMP0, TAB:CARG2->asize
+    |  toint CARG3, f0
+    |   lwz TMP1, TAB:CARG2->array
+    |.endif
+    |  andix. TMP2, TMP3, LJ_GC_BLACK   // isblack(table)
+    |  bne >7
+    |2:
+    |  cmplw TMP0, CARG3
+    |   slwi TMP2, CARG3, 3
+    |   lfdx f14, BASE, RA
+    |  ble ->vmeta_tsetr                // In array part?
+    |  ins_next1
+    |   stfdx f14, TMP1, TMP2
+    |  ins_next2
+    |
+    |7:  // Possible table write barrier for the value. Skip valiswhite check.
+    |  barrierback TAB:CARG2, TMP3, TMP2
+    |  b <2
+    break;
+
 
   case BC_TSETM:
     |  // RA = base*8 (table at base-1), RD = num_const*8 (start index)
diff --git a/src/vm_x86.dasc b/src/vm_x86.dasc
index c8095db2..0c4b0ce6 100644
--- a/src/vm_x86.dasc
+++ b/src/vm_x86.dasc
@@ -911,6 +911,19 @@ static void build_subroutines(BuildCtx *ctx)
   |  mov NARGS:RD, 2+1                  // 2 args for func(t, k).
   |  jmp ->vm_call_dispatch_f
   |
+  |->vmeta_tgetr:
+  |  mov FCARG1, TAB:RB
+  |  mov RB, BASE                       // Save BASE.
+  |  mov FCARG2, RC                     // Caveat: FCARG2 == BASE
+  |  call extern lj_tab_getinth@8       // (GCtab *t, int32_t key)
+  |  // cTValue * or NULL returned in eax (RC).
+  |  movzx RA, PC_RA
+  |  mov BASE, RB                       // Restore BASE.
+  |  test RC, RC
+  |  jnz ->BC_TGETR_Z
+  |  mov dword [BASE+RA*8+4], LJ_TNIL
+  |  jmp ->BC_TGETR2_Z
+  |
   |//-----------------------------------------------------------------------
   |
   |->vmeta_tsets:
@@ -998,6 +1011,33 @@ static void build_subroutines(BuildCtx *ctx)
   |  mov NARGS:RD, 3+1                  // 3 args for func(t, k, v).
   |  jmp ->vm_call_dispatch_f
   |
+  |->vmeta_tsetr:
+  |.if X64WIN
+  |  mov L:CARG1d, SAVE_L
+  |  mov CARG3d, RC
+  |  mov L:CARG1d->base, BASE
+  |  xchg CARG2d, TAB:RB                // Caveat: CARG2d == BASE.
+  |.elif X64
+  |  mov L:CARG1d, SAVE_L
+  |  mov CARG2d, TAB:RB
+  |  mov L:CARG1d->base, BASE
+  |  mov RB, BASE                       // Save BASE.
+  |  mov CARG3d, RC                     // Caveat: CARG3d == BASE.
+  |.else
+  |  mov L:RA, SAVE_L
+  |  mov ARG2, TAB:RB
+  |  mov RB, BASE                       // Save BASE.
+  |  mov ARG3, RC
+  |  mov ARG1, L:RA
+  |  mov L:RA->base, BASE
+  |.endif
+  |  mov SAVE_PC, PC
+  |  call extern lj_tab_setinth  // (lua_State *L, GCtab *t, int32_t key)
+  |  // TValue * returned in eax (RC).
+  |  movzx RA, PC_RA
+  |  mov BASE, RB                       // Restore BASE.
+  |  jmp ->BC_TSETR_Z
+  |
   |//-- Comparison metamethods ---------------------------------------------
   |
   |->vmeta_comp:
@@ -1092,6 +1132,26 @@ static void build_subroutines(BuildCtx *ctx)
   |  jmp <3
   |.endif
   |
+  |->vmeta_istype:
+  |.if X64
+  |  mov L:CARG1d, SAVE_L
+  |  mov L:CARG1d->base, BASE           // Caveat: CARG2d/CARG3d may be BASE.
+  |  mov CARG2d, RA
+  |  movzx CARG3d, PC_RD
+  |  mov L:RB, L:CARG1d
+  |.else
+  |  movzx RD, PC_RD
+  |  mov ARG2, RA
+  |  mov L:RB, SAVE_L
+  |  mov ARG3, RD
+  |  mov ARG1, L:RB
+  |  mov L:RB->base, BASE
+  |.endif
+  |  mov SAVE_PC, PC
+  |  call extern lj_meta_istype  // (lua_State *L, BCReg ra, BCReg tp)
+  |  mov BASE, L:RB->base
+  |  jmp <6
+  |
   |//-- Arithmetic metamethods ---------------------------------------------
   |
   |->vmeta_arith_vno:
@@ -3827,6 +3887,18 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
     |  ins_next
     break;
 
+  case BC_ISTYPE:
+    |  ins_AD   // RA = src, RD = -type
+    |  add RD, [BASE+RA*8+4]
+    |  jne ->vmeta_istype
+    |  ins_next
+    break;
+  case BC_ISNUM:
+    |  ins_AD   // RA = src, RD = -(TISNUM-1)
+    |  checknum RA, ->vmeta_istype
+    |  ins_next
+    break;
+
   /* -- Unary ops --------------------------------------------------------- */
 
   case BC_MOV:
@@ -4502,6 +4574,32 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
     |  mov dword [BASE+RA*8+4], LJ_TNIL
     |  jmp <1
     break;
+  case BC_TGETR:
+    |  ins_ABC  // RA = dst, RB = table, RC = key
+    |  mov TAB:RB, [BASE+RB*8]
+    |.if DUALNUM
+    |  mov RC, dword [BASE+RC*8]
+    |.else
+    |  cvttsd2si RC, qword [BASE+RC*8]
+    |.endif
+    |  cmp RC, TAB:RB->asize
+    |  jae ->vmeta_tgetr                // Not in array part? Use fallback.
+    |  shl RC, 3
+    |  add RC, TAB:RB->array
+    |  // Get array slot.
+    |->BC_TGETR_Z:
+    |.if X64
+    |  mov RBa, [RC]
+    |  mov [BASE+RA*8], RBa
+    |.else
+    |  mov RB, [RC]
+    |  mov RC, [RC+4]
+    |  mov [BASE+RA*8], RB
+    |  mov [BASE+RA*8+4], RC
+    |.endif
+    |->BC_TGETR2_Z:
+    |  ins_next
+    break;
 
   case BC_TSETV:
     |  ins_ABC  // RA = src, RB = table, RC = key
@@ -4688,6 +4786,39 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
     |  movzx RA, PC_RA                  // Restore RA.
     |  jmp <2
     break;
+  case BC_TSETR:
+    |  ins_ABC  // RA = src, RB = table, RC = key
+    |  mov TAB:RB, [BASE+RB*8]
+    |.if DUALNUM
+    |  mov RC, dword [BASE+RC*8]
+    |.else
+    |  cvttsd2si RC, qword [BASE+RC*8]
+    |.endif
+    |  test byte TAB:RB->marked, LJ_GC_BLACK    // isblack(table)
+    |  jnz >7
+    |2:
+    |  cmp RC, TAB:RB->asize
+    |  jae ->vmeta_tsetr
+    |  shl RC, 3
+    |  add RC, TAB:RB->array
+    |  // Set array slot.
+    |->BC_TSETR_Z:
+    |.if X64
+    |  mov RBa, [BASE+RA*8]
+    |  mov [RC], RBa
+    |.else
+    |  mov RB, [BASE+RA*8+4]
+    |  mov RA, [BASE+RA*8]
+    |  mov [RC+4], RB
+    |  mov [RC], RA
+    |.endif
+    |  ins_next
+    |
+    |7:  // Possible table write barrier for the value. Skip valiswhite check.
+    |  barrierback TAB:RB, RA
+    |  movzx RA, PC_RA                  // Restore RA.
+    |  jmp <2
+    break;
 
   case BC_TSETM:
     |  ins_AD   // RA = base (table at base-1), RD = num const (start index)