100 files changed, 5230 insertions, 4170 deletions
diff --git a/src/Makefile b/src/Makefile
index 84b3355d..fa5aed1c 100644
--- a/src/Makefile
+++ b/src/Makefile
@@ -42,13 +42,10 @@ CCOPT= -O2 -fomit-frame-pointer
 #
 # Target-specific compiler options:
 #
-# x86 only: it's recommended to compile at least for i686. Better yet,
-# compile for an architecture that has SSE2, too (-msse -msse2).
-#
 # x86/x64 only: For GCC 4.2 or higher and if you don't intend to distribute
 # the binaries to a different machine you could also use: -march=native
 #
-CCOPT_x86= -march=i686
+CCOPT_x86= -march=i686 -msse -msse2 -mfpmath=sse
 CCOPT_x64=
 CCOPT_arm=
 CCOPT_ppc=
@@ -394,11 +391,6 @@ DASM_AFLAGS+= -D VER=$(subst LJ_ARCH_VERSION_,,$(filter LJ_ARCH_VERSION_%,$(subs
 ifeq (Windows,$(TARGET_SYS))
  DASM_AFLAGS+= -D WIN
 endif
-ifeq (x86,$(TARGET_LJARCH))
-  ifneq (,$(findstring __SSE2__ 1,$(TARGET_TESTARCH)))
-    DASM_AFLAGS+= -D SSE
-  endif
-else
 ifeq (x64,$(TARGET_LJARCH))
  DASM_ARCH= x86
 else
@@ -423,7 +415,6 @@ ifeq (ppc,$(TARGET_LJARCH))
 endif
 endif
 endif
-endif
 DASM_FLAGS= $(DASM_XFLAGS) $(DASM_AFLAGS)
 DASM_DASC= vm_$(DASM_ARCH).dasc
@@ -445,10 +436,11 @@ LJLIB_O= lib_base.o lib_math.o lib_bit.o lib_string.o lib_table.o \
         lib_io.o lib_os.o lib_package.o lib_debug.o lib_jit.o lib_ffi.o
 LJLIB_C= $(LJLIB_O:.o=.c)
-LJCORE_O= lj_gc.o lj_err.o lj_char.o lj_bc.o lj_obj.o \
+LJCORE_O= lj_gc.o lj_err.o lj_char.o lj_bc.o lj_obj.o lj_buf.o \
          lj_str.o lj_tab.o lj_func.o lj_udata.o lj_meta.o lj_debug.o \
          lj_state.o lj_dispatch.o lj_vmevent.o lj_vmmath.o lj_strscan.o \
-          lj_api.o lj_lex.o lj_parse.o lj_bcread.o lj_bcwrite.o lj_load.o \
+          lj_strfmt.o lj_api.o \
+          lj_lex.o lj_parse.o lj_bcread.o lj_bcwrite.o lj_load.o \
          lj_ir.o lj_opt_mem.o lj_opt_fold.o lj_opt_narrow.o \
          lj_opt_dce.o lj_opt_loop.o lj_opt_split.o lj_opt_sink.o \
          lj_mcode.o lj_snap.o lj_record.o lj_crecord.o lj_ffrecord.o \
@@ -568,6 +560,10 @@ amalg:
 clean:
        $(HOST_RM) $(ALL_RM)
+libbc:
+        ./$(LUAJIT_T) host/genlibbc.lua -o host/buildvm_libbc.h $(LJLIB_C)
+        $(MAKE) all
 depend:
        @for file in $(ALL_HDRGEN); do \
          test -f $$file || touch $$file; \
@@ -582,7 +578,7 @@ depend:
          test -s $$file || $(HOST_RM) $$file; \
          done
-.PHONY: default all amalg clean depend
+.PHONY: default all amalg clean libbc depend
 ##############################################################################
 # Rules for generated files.
diff --git a/src/Makefile.dep b/src/Makefile.dep
index 5d91723a..0ea0d98e 100644
--- a/src/Makefile.dep
+++ b/src/Makefile.dep
@@ -5,43 +5,47 @@ lib_base.o: lib_base.c lua.h luaconf.h lauxlib.h lualib.h lj_obj.h \
 lj_def.h lj_arch.h lj_gc.h lj_err.h lj_errmsg.h lj_debug.h lj_str.h \
 lj_tab.h lj_meta.h lj_state.h lj_ctype.h lj_cconv.h lj_bc.h lj_ff.h \
 lj_ffdef.h lj_dispatch.h lj_jit.h lj_ir.h lj_char.h lj_strscan.h \
- lj_lib.h lj_libdef.h
+ lj_strfmt.h lj_lib.h lj_libdef.h
 lib_bit.o: lib_bit.c lua.h luaconf.h lauxlib.h lualib.h lj_obj.h lj_def.h \
- lj_arch.h lj_err.h lj_errmsg.h lj_str.h lj_lib.h lj_libdef.h
+ lj_arch.h lj_err.h lj_errmsg.h lj_buf.h lj_gc.h lj_str.h lj_strscan.h \
+ lj_strfmt.h lj_ctype.h lj_cdata.h lj_cconv.h lj_carith.h lj_ff.h \
+ lj_ffdef.h lj_lib.h lj_libdef.h
 lib_debug.o: lib_debug.c lua.h luaconf.h lauxlib.h lualib.h lj_obj.h \
 lj_def.h lj_arch.h lj_gc.h lj_err.h lj_errmsg.h lj_debug.h lj_lib.h \
 lj_libdef.h
 lib_ffi.o: lib_ffi.c lua.h luaconf.h lauxlib.h lualib.h lj_obj.h lj_def.h \
 lj_arch.h lj_gc.h lj_err.h lj_errmsg.h lj_str.h lj_tab.h lj_meta.h \
 lj_ctype.h lj_cparse.h lj_cdata.h lj_cconv.h lj_carith.h lj_ccall.h \
- lj_ccallback.h lj_clib.h lj_ff.h lj_ffdef.h lj_lib.h lj_libdef.h
+ lj_ccallback.h lj_clib.h lj_strfmt.h lj_ff.h lj_ffdef.h lj_lib.h \
+ lj_libdef.h
 lib_init.o: lib_init.c lua.h luaconf.h lauxlib.h lualib.h lj_arch.h
 lib_io.o: lib_io.c lua.h luaconf.h lauxlib.h lualib.h lj_obj.h lj_def.h \
- lj_arch.h lj_err.h lj_errmsg.h lj_str.h lj_state.h lj_ff.h lj_ffdef.h \
+ lj_arch.h lj_gc.h lj_err.h lj_errmsg.h lj_buf.h lj_str.h lj_state.h \
- lj_lib.h lj_libdef.h
+ lj_strfmt.h lj_ff.h lj_ffdef.h lj_lib.h lj_libdef.h
 lib_jit.o: lib_jit.c lua.h luaconf.h lauxlib.h lualib.h lj_arch.h \
 lj_obj.h lj_def.h lj_err.h lj_errmsg.h lj_debug.h lj_str.h lj_tab.h \
- lj_bc.h lj_ir.h lj_jit.h lj_ircall.h lj_iropt.h lj_target.h \
+ lj_state.h lj_bc.h lj_ctype.h lj_gc.h lj_ir.h lj_jit.h lj_ircall.h \
- lj_target_*.h lj_dispatch.h lj_vm.h lj_vmevent.h lj_lib.h luajit.h \
+ lj_iropt.h lj_target.h lj_target_*.h lj_dispatch.h lj_vm.h \
- lj_libdef.h
+ lj_vmevent.h lj_lib.h luajit.h lj_libdef.h
 lib_math.o: lib_math.c lua.h luaconf.h lauxlib.h lualib.h lj_obj.h \
 lj_def.h lj_arch.h lj_lib.h lj_vm.h lj_libdef.h
 lib_os.o: lib_os.c lua.h luaconf.h lauxlib.h lualib.h lj_obj.h lj_def.h \
- lj_arch.h lj_err.h lj_errmsg.h lj_lib.h lj_libdef.h
+ lj_arch.h lj_gc.h lj_err.h lj_errmsg.h lj_buf.h lj_str.h lj_lib.h \
+ lj_libdef.h
 lib_package.o: lib_package.c lua.h luaconf.h lauxlib.h lualib.h lj_obj.h \
 lj_def.h lj_arch.h lj_err.h lj_errmsg.h lj_lib.h
 lib_string.o: lib_string.c lua.h luaconf.h lauxlib.h lualib.h lj_obj.h \
- lj_def.h lj_arch.h lj_gc.h lj_err.h lj_errmsg.h lj_str.h lj_tab.h \
+ lj_def.h lj_arch.h lj_gc.h lj_err.h lj_errmsg.h lj_buf.h lj_str.h \
- lj_meta.h lj_state.h lj_ff.h lj_ffdef.h lj_bcdump.h lj_lex.h lj_char.h \
+ lj_tab.h lj_meta.h lj_state.h lj_ff.h lj_ffdef.h lj_bcdump.h lj_lex.h \
- lj_lib.h lj_libdef.h
+ lj_char.h lj_strfmt.h lj_lib.h lj_libdef.h
 lib_table.o: lib_table.c lua.h luaconf.h lauxlib.h lualib.h lj_obj.h \
- lj_def.h lj_arch.h lj_gc.h lj_err.h lj_errmsg.h lj_tab.h lj_lib.h \
+ lj_def.h lj_arch.h lj_gc.h lj_err.h lj_errmsg.h lj_buf.h lj_str.h \
- lj_libdef.h
+ lj_tab.h lj_lib.h lj_libdef.h
 lj_alloc.o: lj_alloc.c lj_def.h lua.h luaconf.h lj_arch.h lj_alloc.h
 lj_api.o: lj_api.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h lj_gc.h \
 lj_err.h lj_errmsg.h lj_debug.h lj_str.h lj_tab.h lj_func.h lj_udata.h \
 lj_meta.h lj_state.h lj_bc.h lj_frame.h lj_trace.h lj_jit.h lj_ir.h \
- lj_dispatch.h lj_traceerr.h lj_vm.h lj_strscan.h
+ lj_dispatch.h lj_traceerr.h lj_vm.h lj_strscan.h lj_strfmt.h
 lj_asm.o: lj_asm.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h lj_gc.h \
 lj_str.h lj_tab.h lj_frame.h lj_bc.h lj_ctype.h lj_ir.h lj_jit.h \
 lj_ircall.h lj_iropt.h lj_mcode.h lj_trace.h lj_dispatch.h lj_traceerr.h \
@@ -50,17 +54,20 @@ lj_asm.o: lj_asm.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h lj_gc.h \
 lj_bc.o: lj_bc.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h lj_bc.h \
 lj_bcdef.h
 lj_bcread.o: lj_bcread.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \
- lj_gc.h lj_err.h lj_errmsg.h lj_str.h lj_tab.h lj_bc.h lj_ctype.h \
+ lj_gc.h lj_err.h lj_errmsg.h lj_buf.h lj_str.h lj_tab.h lj_bc.h \
- lj_cdata.h lualib.h lj_lex.h lj_bcdump.h lj_state.h
+ lj_ctype.h lj_cdata.h lualib.h lj_lex.h lj_bcdump.h lj_state.h \
+ lj_strfmt.h
 lj_bcwrite.o: lj_bcwrite.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \
- lj_gc.h lj_str.h lj_bc.h lj_ctype.h lj_dispatch.h lj_jit.h lj_ir.h \
+ lj_gc.h lj_buf.h lj_str.h lj_bc.h lj_ctype.h lj_dispatch.h lj_jit.h \
- lj_bcdump.h lj_lex.h lj_err.h lj_errmsg.h lj_vm.h
+ lj_ir.h lj_strfmt.h lj_bcdump.h lj_lex.h lj_err.h lj_errmsg.h lj_vm.h
+lj_buf.o: lj_buf.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h lj_gc.h \
+ lj_err.h lj_errmsg.h lj_buf.h lj_str.h lj_tab.h lj_strfmt.h
 lj_carith.o: lj_carith.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \
- lj_gc.h lj_err.h lj_errmsg.h lj_tab.h lj_meta.h lj_ctype.h lj_cconv.h \
+ lj_gc.h lj_err.h lj_errmsg.h lj_tab.h lj_meta.h lj_ir.h lj_ctype.h \
- lj_cdata.h lj_carith.h
+ lj_cconv.h lj_cdata.h lj_carith.h lj_strscan.h
 lj_ccall.o: lj_ccall.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \
- lj_gc.h lj_err.h lj_errmsg.h lj_str.h lj_tab.h lj_ctype.h lj_cconv.h \
+ lj_gc.h lj_err.h lj_errmsg.h lj_tab.h lj_ctype.h lj_cconv.h lj_cdata.h \
- lj_cdata.h lj_ccall.h lj_trace.h lj_jit.h lj_ir.h lj_dispatch.h lj_bc.h \
+ lj_ccall.h lj_trace.h lj_jit.h lj_ir.h lj_dispatch.h lj_bc.h \
 lj_traceerr.h
 lj_ccallback.o: lj_ccallback.c lj_obj.h lua.h luaconf.h lj_def.h \
 lj_arch.h lj_gc.h lj_err.h lj_errmsg.h lj_tab.h lj_state.h lj_frame.h \
@@ -71,79 +78,82 @@ lj_cconv.o: lj_cconv.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \
 lj_err.h lj_errmsg.h lj_tab.h lj_ctype.h lj_gc.h lj_cdata.h lj_cconv.h \
 lj_ccallback.h
 lj_cdata.o: lj_cdata.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \
- lj_gc.h lj_err.h lj_errmsg.h lj_str.h lj_tab.h lj_ctype.h lj_cconv.h \
+ lj_gc.h lj_err.h lj_errmsg.h lj_tab.h lj_ctype.h lj_cconv.h lj_cdata.h
- lj_cdata.h
 lj_char.o: lj_char.c lj_char.h lj_def.h lua.h luaconf.h
 lj_clib.o: lj_clib.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h lj_gc.h \
 lj_err.h lj_errmsg.h lj_tab.h lj_str.h lj_udata.h lj_ctype.h lj_cconv.h \
- lj_cdata.h lj_clib.h
+ lj_cdata.h lj_clib.h lj_strfmt.h
 lj_cparse.o: lj_cparse.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \
- lj_gc.h lj_err.h lj_errmsg.h lj_str.h lj_ctype.h lj_cparse.h lj_frame.h \
+ lj_gc.h lj_err.h lj_errmsg.h lj_buf.h lj_str.h lj_ctype.h lj_cparse.h \
- lj_bc.h lj_vm.h lj_char.h lj_strscan.h
+ lj_frame.h lj_bc.h lj_vm.h lj_char.h lj_strscan.h lj_strfmt.h
 lj_crecord.o: lj_crecord.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \
- lj_err.h lj_errmsg.h lj_str.h lj_tab.h lj_frame.h lj_bc.h lj_ctype.h \
+ lj_err.h lj_errmsg.h lj_tab.h lj_frame.h lj_bc.h lj_ctype.h lj_gc.h \
- lj_gc.h lj_cdata.h lj_cparse.h lj_cconv.h lj_clib.h lj_ccall.h lj_ff.h \
+ lj_cdata.h lj_cparse.h lj_cconv.h lj_carith.h lj_clib.h lj_ccall.h \
- lj_ffdef.h lj_ir.h lj_jit.h lj_ircall.h lj_iropt.h lj_trace.h \
+ lj_ff.h lj_ffdef.h lj_ir.h lj_jit.h lj_ircall.h lj_iropt.h lj_trace.h \
 lj_dispatch.h lj_traceerr.h lj_record.h lj_ffrecord.h lj_snap.h \
- lj_crecord.h
+ lj_crecord.h lj_strfmt.h
 lj_ctype.o: lj_ctype.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \
- lj_gc.h lj_err.h lj_errmsg.h lj_str.h lj_tab.h lj_ctype.h lj_ccallback.h
+ lj_gc.h lj_err.h lj_errmsg.h lj_str.h lj_tab.h lj_strfmt.h lj_ctype.h \
+ lj_ccallback.h
 lj_debug.o: lj_debug.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \
- lj_err.h lj_errmsg.h lj_debug.h lj_str.h lj_tab.h lj_state.h lj_frame.h \
+ lj_err.h lj_errmsg.h lj_debug.h lj_buf.h lj_gc.h lj_str.h lj_tab.h \
- lj_bc.h lj_jit.h lj_ir.h
+ lj_state.h lj_frame.h lj_bc.h lj_strfmt.h lj_jit.h lj_ir.h
 lj_dispatch.o: lj_dispatch.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \
- lj_err.h lj_errmsg.h lj_func.h lj_str.h lj_tab.h lj_meta.h lj_debug.h \
+ lj_err.h lj_errmsg.h lj_buf.h lj_gc.h lj_str.h lj_func.h lj_tab.h \
- lj_state.h lj_frame.h lj_bc.h lj_ff.h lj_ffdef.h lj_jit.h lj_ir.h \
+ lj_meta.h lj_debug.h lj_state.h lj_frame.h lj_bc.h lj_ff.h lj_ffdef.h \
- lj_ccallback.h lj_ctype.h lj_gc.h lj_trace.h lj_dispatch.h lj_traceerr.h \
+ lj_strfmt.h lj_jit.h lj_ir.h lj_ccallback.h lj_ctype.h lj_trace.h \
- lj_vm.h luajit.h
+ lj_dispatch.h lj_traceerr.h lj_vm.h luajit.h
 lj_err.o: lj_err.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h lj_err.h \
 lj_errmsg.h lj_debug.h lj_str.h lj_func.h lj_state.h lj_frame.h lj_bc.h \
 lj_ff.h lj_ffdef.h lj_trace.h lj_jit.h lj_ir.h lj_dispatch.h \
- lj_traceerr.h lj_vm.h
+ lj_traceerr.h lj_vm.h lj_strfmt.h
 lj_ffrecord.o: lj_ffrecord.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \
 lj_err.h lj_errmsg.h lj_str.h lj_tab.h lj_frame.h lj_bc.h lj_ff.h \
 lj_ffdef.h lj_ir.h lj_jit.h lj_ircall.h lj_iropt.h lj_trace.h \
 lj_dispatch.h lj_traceerr.h lj_record.h lj_ffrecord.h lj_crecord.h \
- lj_vm.h lj_strscan.h lj_recdef.h
+ lj_vm.h lj_strscan.h lj_strfmt.h lj_recdef.h
 lj_func.o: lj_func.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h lj_gc.h \
 lj_func.h lj_trace.h lj_jit.h lj_ir.h lj_dispatch.h lj_bc.h \
 lj_traceerr.h lj_vm.h
 lj_gc.o: lj_gc.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h lj_gc.h \
- lj_err.h lj_errmsg.h lj_str.h lj_tab.h lj_func.h lj_udata.h lj_meta.h \
+ lj_err.h lj_errmsg.h lj_buf.h lj_str.h lj_tab.h lj_func.h lj_udata.h \
- lj_state.h lj_frame.h lj_bc.h lj_ctype.h lj_cdata.h lj_trace.h lj_jit.h \
+ lj_meta.h lj_state.h lj_frame.h lj_bc.h lj_ctype.h lj_cdata.h lj_trace.h \
- lj_ir.h lj_dispatch.h lj_traceerr.h lj_vm.h
+ lj_jit.h lj_ir.h lj_dispatch.h lj_traceerr.h lj_vm.h
 lj_gdbjit.o: lj_gdbjit.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \
- lj_gc.h lj_err.h lj_errmsg.h lj_debug.h lj_frame.h lj_bc.h lj_jit.h \
+ lj_gc.h lj_err.h lj_errmsg.h lj_debug.h lj_frame.h lj_bc.h lj_buf.h \
- lj_ir.h lj_dispatch.h
+ lj_str.h lj_strfmt.h lj_jit.h lj_ir.h lj_dispatch.h
 lj_ir.o: lj_ir.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h lj_gc.h \
- lj_str.h lj_tab.h lj_ir.h lj_jit.h lj_ircall.h lj_iropt.h lj_trace.h \
+ lj_buf.h lj_str.h lj_tab.h lj_ir.h lj_jit.h lj_ircall.h lj_iropt.h \
- lj_dispatch.h lj_bc.h lj_traceerr.h lj_ctype.h lj_cdata.h lj_carith.h \
+ lj_trace.h lj_dispatch.h lj_bc.h lj_traceerr.h lj_ctype.h lj_cdata.h \
- lj_vm.h lj_strscan.h lj_lib.h
+ lj_carith.h lj_vm.h lj_strscan.h lj_strfmt.h lj_lib.h
 lj_lex.o: lj_lex.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h lj_gc.h \
- lj_err.h lj_errmsg.h lj_str.h lj_tab.h lj_ctype.h lj_cdata.h lualib.h \
+ lj_err.h lj_errmsg.h lj_buf.h lj_str.h lj_tab.h lj_ctype.h lj_cdata.h \
- lj_state.h lj_lex.h lj_parse.h lj_char.h lj_strscan.h
+ lualib.h lj_state.h lj_lex.h lj_parse.h lj_char.h lj_strscan.h \
+ lj_strfmt.h
 lj_lib.o: lj_lib.c lauxlib.h lua.h luaconf.h lj_obj.h lj_def.h lj_arch.h \
 lj_gc.h lj_err.h lj_errmsg.h lj_str.h lj_tab.h lj_func.h lj_bc.h \
- lj_dispatch.h lj_jit.h lj_ir.h lj_vm.h lj_strscan.h lj_lib.h
+ lj_dispatch.h lj_jit.h lj_ir.h lj_vm.h lj_strscan.h lj_strfmt.h lj_lex.h \
+ lj_bcdump.h lj_lib.h
 lj_load.o: lj_load.c lua.h luaconf.h lauxlib.h lj_obj.h lj_def.h \
- lj_arch.h lj_gc.h lj_err.h lj_errmsg.h lj_str.h lj_func.h lj_frame.h \
+ lj_arch.h lj_gc.h lj_err.h lj_errmsg.h lj_buf.h lj_str.h lj_func.h \
- lj_bc.h lj_vm.h lj_lex.h lj_bcdump.h lj_parse.h
+ lj_frame.h lj_bc.h lj_vm.h lj_lex.h lj_bcdump.h lj_parse.h
 lj_mcode.o: lj_mcode.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \
 lj_gc.h lj_jit.h lj_ir.h lj_mcode.h lj_trace.h lj_dispatch.h lj_bc.h \
 lj_traceerr.h lj_vm.h
 lj_meta.o: lj_meta.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h lj_gc.h \
- lj_err.h lj_errmsg.h lj_str.h lj_tab.h lj_meta.h lj_frame.h lj_bc.h \
+ lj_err.h lj_errmsg.h lj_buf.h lj_str.h lj_tab.h lj_meta.h lj_frame.h \
- lj_vm.h lj_strscan.h
+ lj_bc.h lj_vm.h lj_strscan.h lj_strfmt.h lj_lib.h
 lj_obj.o: lj_obj.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h
 lj_opt_dce.o: lj_opt_dce.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \
 lj_ir.h lj_jit.h lj_iropt.h
 lj_opt_fold.o: lj_opt_fold.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \
- lj_str.h lj_tab.h lj_ir.h lj_jit.h lj_iropt.h lj_trace.h lj_dispatch.h \
+ lj_buf.h lj_gc.h lj_str.h lj_tab.h lj_ir.h lj_jit.h lj_ircall.h \
- lj_bc.h lj_traceerr.h lj_ctype.h lj_gc.h lj_carith.h lj_vm.h \
+ lj_iropt.h lj_trace.h lj_dispatch.h lj_bc.h lj_traceerr.h lj_ctype.h \
- lj_strscan.h lj_folddef.h
+ lj_carith.h lj_vm.h lj_strscan.h lj_strfmt.h lj_folddef.h
 lj_opt_loop.o: lj_opt_loop.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \
- lj_err.h lj_errmsg.h lj_str.h lj_ir.h lj_jit.h lj_iropt.h lj_trace.h \
+ lj_err.h lj_errmsg.h lj_buf.h lj_gc.h lj_str.h lj_ir.h lj_jit.h \
- lj_dispatch.h lj_bc.h lj_traceerr.h lj_snap.h lj_vm.h
+ lj_iropt.h lj_trace.h lj_dispatch.h lj_bc.h lj_traceerr.h lj_snap.h \
+ lj_vm.h
 lj_opt_mem.o: lj_opt_mem.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \
 lj_tab.h lj_ir.h lj_jit.h lj_iropt.h
 lj_opt_narrow.o: lj_opt_narrow.c lj_obj.h lua.h luaconf.h lj_def.h \
@@ -152,11 +162,12 @@ lj_opt_narrow.o: lj_opt_narrow.c lj_obj.h lua.h luaconf.h lj_def.h \
 lj_opt_sink.o: lj_opt_sink.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \
 lj_ir.h lj_jit.h lj_iropt.h lj_target.h lj_target_*.h
 lj_opt_split.o: lj_opt_split.c lj_obj.h lua.h luaconf.h lj_def.h \
- lj_arch.h lj_err.h lj_errmsg.h lj_str.h lj_ir.h lj_jit.h lj_ircall.h \
+ lj_arch.h lj_err.h lj_errmsg.h lj_buf.h lj_gc.h lj_str.h lj_ir.h \
- lj_iropt.h lj_vm.h
+ lj_jit.h lj_ircall.h lj_iropt.h lj_vm.h
 lj_parse.o: lj_parse.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \
- lj_gc.h lj_err.h lj_errmsg.h lj_debug.h lj_str.h lj_tab.h lj_func.h \
+ lj_gc.h lj_err.h lj_errmsg.h lj_debug.h lj_buf.h lj_str.h lj_tab.h \
- lj_state.h lj_bc.h lj_ctype.h lj_lex.h lj_parse.h lj_vm.h lj_vmevent.h
+ lj_func.h lj_state.h lj_bc.h lj_ctype.h lj_strfmt.h lj_lex.h lj_parse.h \
+ lj_vm.h lj_vmevent.h
 lj_record.o: lj_record.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \
 lj_err.h lj_errmsg.h lj_str.h lj_tab.h lj_meta.h lj_frame.h lj_bc.h \
 lj_ctype.h lj_gc.h lj_ff.h lj_ffdef.h lj_ir.h lj_jit.h lj_ircall.h \
@@ -167,11 +178,13 @@ lj_snap.o: lj_snap.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h lj_gc.h \
 lj_trace.h lj_dispatch.h lj_traceerr.h lj_snap.h lj_target.h \
 lj_target_*.h lj_ctype.h lj_cdata.h
 lj_state.o: lj_state.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \
- lj_gc.h lj_err.h lj_errmsg.h lj_str.h lj_tab.h lj_func.h lj_meta.h \
+ lj_gc.h lj_err.h lj_errmsg.h lj_buf.h lj_str.h lj_tab.h lj_func.h \
- lj_state.h lj_frame.h lj_bc.h lj_ctype.h lj_trace.h lj_jit.h lj_ir.h \
+ lj_meta.h lj_state.h lj_frame.h lj_bc.h lj_ctype.h lj_trace.h lj_jit.h \
- lj_dispatch.h lj_traceerr.h lj_vm.h lj_lex.h lj_alloc.h
+ lj_ir.h lj_dispatch.h lj_traceerr.h lj_vm.h lj_lex.h lj_alloc.h
 lj_str.o: lj_str.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h lj_gc.h \
- lj_err.h lj_errmsg.h lj_str.h lj_state.h lj_char.h
+ lj_err.h lj_errmsg.h lj_str.h lj_char.h
+lj_strfmt.o: lj_strfmt.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \
+ lj_buf.h lj_gc.h lj_str.h lj_state.h lj_char.h lj_strfmt.h
 lj_strscan.o: lj_strscan.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \
 lj_char.h lj_strscan.h
 lj_tab.o: lj_tab.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h lj_gc.h \
@@ -189,26 +202,26 @@ lj_vmevent.o: lj_vmevent.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \
 lj_vmmath.o: lj_vmmath.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \
 lj_ir.h lj_vm.h
 ljamalg.o: ljamalg.c lua.h luaconf.h lauxlib.h lj_gc.c lj_obj.h lj_def.h \
- lj_arch.h lj_gc.h lj_err.h lj_errmsg.h lj_str.h lj_tab.h lj_func.h \
+ lj_arch.h lj_gc.h lj_err.h lj_errmsg.h lj_buf.h lj_str.h lj_tab.h \
- lj_udata.h lj_meta.h lj_state.h lj_frame.h lj_bc.h lj_ctype.h lj_cdata.h \
+ lj_func.h lj_udata.h lj_meta.h lj_state.h lj_frame.h lj_bc.h lj_ctype.h \
- lj_trace.h lj_jit.h lj_ir.h lj_dispatch.h lj_traceerr.h lj_vm.h lj_err.c \
+ lj_cdata.h lj_trace.h lj_jit.h lj_ir.h lj_dispatch.h lj_traceerr.h \
- lj_debug.h lj_ff.h lj_ffdef.h lj_char.c lj_char.h lj_bc.c lj_bcdef.h \
+ lj_vm.h lj_err.c lj_debug.h lj_ff.h lj_ffdef.h lj_strfmt.h lj_char.c \
- lj_obj.c lj_str.c lj_tab.c lj_func.c lj_udata.c lj_meta.c lj_strscan.h \
+ lj_char.h lj_bc.c lj_bcdef.h lj_obj.c lj_buf.c lj_str.c lj_tab.c \
- lj_debug.c lj_state.c lj_lex.h lj_alloc.h lj_dispatch.c lj_ccallback.h \
+ lj_func.c lj_udata.c lj_meta.c lj_strscan.h lj_lib.h lj_debug.c \
- luajit.h lj_vmevent.c lj_vmevent.h lj_vmmath.c lj_strscan.c lj_api.c \
+ lj_state.c lj_lex.h lj_alloc.h lj_dispatch.c lj_ccallback.h luajit.h \
+ lj_vmevent.c lj_vmevent.h lj_vmmath.c lj_strscan.c lj_strfmt.c lj_api.c \
 lj_lex.c lualib.h lj_parse.h lj_parse.c lj_bcread.c lj_bcdump.h \
 lj_bcwrite.c lj_load.c lj_ctype.c lj_cdata.c lj_cconv.h lj_cconv.c \
 lj_ccall.c lj_ccall.h lj_ccallback.c lj_target.h lj_target_*.h \
 lj_mcode.h lj_carith.c lj_carith.h lj_clib.c lj_clib.h lj_cparse.c \
- lj_cparse.h lj_lib.c lj_lib.h lj_ir.c lj_ircall.h lj_iropt.h \
+ lj_cparse.h lj_lib.c lj_ir.c lj_ircall.h lj_iropt.h lj_opt_mem.c \
- lj_opt_mem.c lj_opt_fold.c lj_folddef.h lj_opt_narrow.c lj_opt_dce.c \
+ lj_opt_fold.c lj_folddef.h lj_opt_narrow.c lj_opt_dce.c lj_opt_loop.c \
- lj_opt_loop.c lj_snap.h lj_opt_split.c lj_opt_sink.c lj_mcode.c \
+ lj_snap.h lj_opt_split.c lj_opt_sink.c lj_mcode.c lj_snap.c lj_record.c \
- lj_snap.c lj_record.c lj_record.h lj_ffrecord.h lj_crecord.c \
+ lj_record.h lj_ffrecord.h lj_crecord.c lj_crecord.h lj_ffrecord.c \
- lj_crecord.h lj_ffrecord.c lj_recdef.h lj_asm.c lj_asm.h lj_emit_*.h \
+ lj_recdef.h lj_asm.c lj_asm.h lj_emit_*.h lj_asm_*.h lj_trace.c \
- lj_asm_*.h lj_trace.c lj_gdbjit.h lj_gdbjit.c lj_alloc.c lib_aux.c \
+ lj_gdbjit.h lj_gdbjit.c lj_alloc.c lib_aux.c lib_base.c lj_libdef.h \
- lib_base.c lj_libdef.h lib_math.c lib_string.c lib_table.c lib_io.c \
+ lib_math.c lib_string.c lib_table.c lib_io.c lib_os.c lib_package.c \
- lib_os.c lib_package.c lib_debug.c lib_bit.c lib_jit.c lib_ffi.c \
+ lib_debug.c lib_bit.c lib_jit.c lib_ffi.c lib_init.c
- lib_init.c
 luajit.o: luajit.c lua.h luaconf.h lauxlib.h lualib.h luajit.h lj_arch.h
 host/buildvm.o: host/buildvm.c host/buildvm.h lj_def.h lua.h luaconf.h \
 lj_arch.h lj_obj.h lj_def.h lj_arch.h lj_gc.h lj_obj.h lj_bc.h lj_ir.h \
@@ -220,7 +233,8 @@ host/buildvm_asm.o: host/buildvm_asm.c host/buildvm.h lj_def.h lua.h luaconf.h \
 host/buildvm_fold.o: host/buildvm_fold.c host/buildvm.h lj_def.h lua.h \
 luaconf.h lj_arch.h lj_obj.h lj_def.h lj_arch.h lj_ir.h lj_obj.h
 host/buildvm_lib.o: host/buildvm_lib.c host/buildvm.h lj_def.h lua.h luaconf.h \
- lj_arch.h lj_obj.h lj_def.h lj_arch.h lj_lib.h lj_obj.h
+ lj_arch.h lj_obj.h lj_def.h lj_arch.h lj_bc.h lj_lib.h lj_obj.h \
+ host/buildvm_libbc.h
 host/buildvm_peobj.o: host/buildvm_peobj.c host/buildvm.h lj_def.h lua.h \
 luaconf.h lj_arch.h lj_bc.h lj_def.h lj_arch.h
 host/minilua.o: host/minilua.c
diff --git a/src/host/buildvm.c b/src/host/buildvm.c
index ba8fbcba..ef5f7fb9 100644
--- a/src/host/buildvm.c
+++ b/src/host/buildvm.c
@@ -314,20 +314,20 @@ static void emit_vmdef(BuildCtx *ctx)
  char buf[80];
  int i;
  fprintf(ctx->fp, "-- This is a generated file. DO NOT EDIT!\n\n");
-  fprintf(ctx->fp, "module(...)\n\n");
+  fprintf(ctx->fp, "return {\n\n");
  fprintf(ctx->fp, "bcnames = \"");
  for (i = 0; bc_names[i]; i++) fprintf(ctx->fp, "%-6s", bc_names[i]);
-  fprintf(ctx->fp, "\"\n\n");
+  fprintf(ctx->fp, "\",\n\n");
  fprintf(ctx->fp, "irnames = \"");
  for (i = 0; ir_names[i]; i++) fprintf(ctx->fp, "%-6s", ir_names[i]);
-  fprintf(ctx->fp, "\"\n\n");
+  fprintf(ctx->fp, "\",\n\n");
  fprintf(ctx->fp, "irfpm = { [0]=");
  for (i = 0; irfpm_names[i]; i++)
    fprintf(ctx->fp, "\"%s\", ", lower(buf, irfpm_names[i]));
-  fprintf(ctx->fp, "}\n\n");
+  fprintf(ctx->fp, "},\n\n");
  fprintf(ctx->fp, "irfield = { [0]=");
  for (i = 0; irfield_names[i]; i++) {
@@ -337,17 +337,17 @@ static void emit_vmdef(BuildCtx *ctx)
    if (p) *p = '.';
    fprintf(ctx->fp, "\"%s\", ", buf);
  }
-  fprintf(ctx->fp, "}\n\n");
+  fprintf(ctx->fp, "},\n\n");
  fprintf(ctx->fp, "ircall = {\n[0]=");
  for (i = 0; ircall_names[i]; i++)
    fprintf(ctx->fp, "\"%s\",\n", ircall_names[i]);
-  fprintf(ctx->fp, "}\n\n");
+  fprintf(ctx->fp, "},\n\n");
  fprintf(ctx->fp, "traceerr = {\n[0]=");
  for (i = 0; trace_errors[i]; i++)
    fprintf(ctx->fp, "\"%s\",\n", trace_errors[i]);
-  fprintf(ctx->fp, "}\n\n");
+  fprintf(ctx->fp, "},\n\n");
 }
 /* -- Argument parsing ---------------------------------------------------- */
@@ -484,6 +484,7 @@ int main(int argc, char **argv)
  case BUILD_vmdef:
    emit_vmdef(ctx);
    emit_lib(ctx);
+    fprintf(ctx->fp, "}\n\n");
    break;
  case BUILD_ffdef:
  case BUILD_libdef:
diff --git a/src/host/buildvm_lib.c b/src/host/buildvm_lib.c
index 40141dfb..f578bbc9 100644
--- a/src/host/buildvm_lib.c
+++ b/src/host/buildvm_lib.c
@@ -5,7 +5,9 @@
 #include "buildvm.h"
 #include "lj_obj.h"
+#include "lj_bc.h"
 #include "lj_lib.h"
+#include "buildvm_libbc.h"
 /* Context for library definitions. */
 static uint8_t obuf[8192];
@@ -151,6 +153,62 @@ static void libdef_func(BuildCtx *ctx, char *p, int arg)
  regfunc = REGFUNC_OK;
 }
+static uint8_t *libdef_uleb128(uint8_t *p, uint32_t *vv)
+{
+  uint32_t v = *p++;
+  if (v >= 0x80) {
+    int sh = 0; v &= 0x7f;
+    do { v |= ((*p & 0x7f) << (sh += 7)); } while (*p++ >= 0x80);
+  }
+  *vv = v;
+  return p;
+}
+static void libdef_fixupbc(uint8_t *p)
+{
+  uint32_t i, sizebc;
+  p += 4;
+  p = libdef_uleb128(p, &sizebc);
+  p = libdef_uleb128(p, &sizebc);
+  p = libdef_uleb128(p, &sizebc);
+  for (i = 0; i < sizebc; i++, p += 4) {
+    uint8_t op = p[libbc_endian ? 3 : 0];
+    uint8_t ra = p[libbc_endian ? 2 : 1];
+    uint8_t rc = p[libbc_endian ? 1 : 2];
+    uint8_t rb = p[libbc_endian ? 0 : 3];
+    if (!LJ_DUALNUM && op == BC_ISTYPE && rc == ~LJ_TNUMX+1) {
+      op = BC_ISNUM; rc++;
+    }
+    p[LJ_ENDIAN_SELECT(0, 3)] = op;
+    p[LJ_ENDIAN_SELECT(1, 2)] = ra;
+    p[LJ_ENDIAN_SELECT(2, 1)] = rc;
+    p[LJ_ENDIAN_SELECT(3, 0)] = rb;
+  }
+}
+static void libdef_lua(BuildCtx *ctx, char *p, int arg)
+{
+  UNUSED(arg);
+  if (ctx->mode == BUILD_libdef) {
+    int i;
+    for (i = 0; libbc_map[i].name != NULL; i++) {
+      if (!strcmp(libbc_map[i].name, p)) {
+        int ofs = libbc_map[i].ofs;
+        int len = libbc_map[i+1].ofs - ofs;
+        obuf[2]++;  /* Bump hash table size. */
+        *optr++ = LIBINIT_LUA;
+        libdef_name(p, 0);
+        memcpy(optr, libbc_code + ofs, len);
+        libdef_fixupbc(optr);
+        optr += len;
+        return;
+      }
+    }
+    fprintf(stderr, "Error: missing libbc definition for %s\n", p);
+    exit(1);
+  }
+}
 static uint32_t find_rec(char *name)
 {
  char *p = (char *)obuf;
@@ -277,6 +335,7 @@ static const LibDefHandler libdef_handlers[] = {
  { "CF(",      ")",            libdef_func,            LIBINIT_CF },
  { "ASM(",     ")",            libdef_func,            LIBINIT_ASM },
  { "ASM_(",    ")",            libdef_func,            LIBINIT_ASM_ },
+  { "LUA(",     ")",            libdef_lua,             0 },
  { "REC(",     ")",            libdef_rec,             0 },
  { "PUSH(",    ")",            libdef_push,            0 },
  { "SET(",     ")",            libdef_set,             0 },
@@ -373,7 +432,7 @@ void emit_lib(BuildCtx *ctx)
      "#ifndef FF_NUM_ASMFUNC\n#define FF_NUM_ASMFUNC %d\n#endif\n\n",
      ffasmfunc);
  } else if (ctx->mode == BUILD_vmdef) {
-    fprintf(ctx->fp, "}\n\n");
+    fprintf(ctx->fp, "},\n\n");
  } else if (ctx->mode == BUILD_bcdef) {
    int i;
    fprintf(ctx->fp, "\n};\n\n");
diff --git a/src/host/buildvm_libbc.h b/src/host/buildvm_libbc.h
new file mode 100644
index 00000000..e96c8a53
--- /dev/null
+++ b/src/host/buildvm_libbc.h
@@ -0,0 +1,30 @@
+/* This is a generated file. DO NOT EDIT! */
+static const int libbc_endian = 0;
+static const uint8_t libbc_code[] = {
+0,1,2,0,0,1,2,24,1,0,0,76,1,2,0,241,135,158,166,3,220,203,178,130,4,0,1,2,0,
+0,1,2,24,1,0,0,76,1,2,0,243,244,148,165,20,198,190,199,252,3,0,1,2,0,0,0,3,
+16,0,5,0,21,1,0,0,76,1,2,0,0,2,9,0,0,0,15,16,0,12,0,16,1,9,0,41,2,1,0,21,3,
+0,0,41,4,1,0,77,2,8,128,18,6,1,0,18,7,5,0,59,8,5,0,66,6,3,2,10,6,0,0,88,7,1,
+128,76,6,2,0,79,2,248,127,75,0,1,0,0,2,10,0,0,0,16,16,0,12,0,16,1,9,0,43,2,
+0,0,18,3,0,0,41,4,0,0,88,5,7,128,18,7,1,0,18,8,5,0,18,9,6,0,66,7,3,2,10,7,0,
+0,88,8,1,128,76,7,2,0,70,5,3,3,82,5,247,127,75,0,1,0,0,1,2,0,0,0,3,16,0,12,
+0,21,1,0,0,76,1,2,0,0,2,10,0,0,2,30,16,0,12,0,21,2,0,0,11,1,0,0,88,3,7,128,
+8,2,0,0,88,3,23,128,59,3,2,0,43,4,0,0,64,4,2,0,76,3,2,0,88,3,18,128,16,1,14,
+0,41,3,1,0,3,3,1,0,88,3,14,128,3,1,2,0,88,3,12,128,59,3,1,0,22,4,1,1,18,5,2,
+0,41,6,1,0,77,4,4,128,23,8,1,7,59,9,7,0,64,9,8,0,79,4,252,127,43,4,0,0,64,4,
+2,0,76,3,2,0,75,0,1,0,0,2,0
+};
+static const struct { const char *name; int ofs; } libbc_map[] = {
+{"math_deg",0},
+{"math_rad",25},
+{"string_len",50},
+{"table_foreachi",69},
+{"table_foreach",136},
+{"table_getn",207},
+{"table_remove",226},
+{NULL,355}
+};
diff --git a/src/host/genlibbc.lua b/src/host/genlibbc.lua
new file mode 100644
index 00000000..16f0a0b6
--- /dev/null
+++ b/src/host/genlibbc.lua
@@ -0,0 +1,197 @@
+----------------------------------------------------------------------------
+-- Lua script to dump the bytecode of the library functions written in Lua.
+-- The resulting 'buildvm_libbc.h' is used for the build process of LuaJIT.
+----------------------------------------------------------------------------
+-- Copyright (C) 2005-2013 Mike Pall. All rights reserved.
+-- Released under the MIT license. See Copyright Notice in luajit.h
+----------------------------------------------------------------------------
+local ffi = require("ffi")
+local bit = require("bit")
+local vmdef = require("jit.vmdef")
+local bcnames = vmdef.bcnames
+local format = string.format
+local isbe = (string.byte(string.dump(function() end), 5) % 2 == 1)
+local function usage(arg)
+  io.stderr:write("Usage: ", arg and arg[0] or "genlibbc",
+                  " [-o buildvm_libbc.h] lib_*.c\n")
+  os.exit(1)
+end
+local function parse_arg(arg)
+  local outfile = "-"
+  if not (arg and arg[1]) then
+    usage(arg)
+  end
+  if arg[1] == "-o" then
+    outfile = arg[2]
+    if not outfile then usage(arg) end
+    table.remove(arg, 1)
+    table.remove(arg, 1)
+  end
+  return outfile
+end
+local function read_files(names)
+  local src = ""
+  for _,name in ipairs(names) do
+    local fp = assert(io.open(name))
+    src = src .. fp:read("*a")
+    fp:close()
+  end
+  return src
+end
+local function transform_lua(code)
+  local fixup = {}
+  local n = -30000
+  code = string.gsub(code, "CHECK_(%w*)%((.-)%)", function(tp, var)
+    n = n + 1
+    fixup[n] = { "CHECK", tp }
+    return format("%s=%d", var, n)
+  end)
+  code = string.gsub(code, "PAIRS%((.-)%)", function(var)
+    fixup.PAIRS = true
+    return format("nil, %s, 0", var)
+  end)
+  return "return "..code, fixup
+end
+local function read_uleb128(p)
+  local v = p[0]; p = p + 1
+  if v >= 128 then
+    local sh = 7; v = v - 128
+    repeat
+      local r = p[0]
+      v = v + bit.lshift(bit.band(r, 127), sh)
+      sh = sh + 7
+      p = p + 1
+    until r < 128
+  end
+  return p, v
+end
+-- ORDER LJ_T
+local name2itype = {
+  str = 5, func = 9, tab = 12, int = 14, num = 15
+}
+local BC = {}
+for i=0,#bcnames/6-1 do
+  BC[string.gsub(string.sub(bcnames, i*6+1, i*6+6), " ", "")] = i
+end
+local xop, xra = isbe and 3 or 0, isbe and 2 or 1
+local xrc, xrb = isbe and 1 or 2, isbe and 0 or 3
+local function fixup_dump(dump, fixup)
+  local buf = ffi.new("uint8_t[?]", #dump+1, dump)
+  local p = buf+5
+  local n, sizebc
+  p, n = read_uleb128(p)
+  local start = p
+  p = p + 4
+  p = read_uleb128(p)
+  p = read_uleb128(p)
+  p, sizebc = read_uleb128(p)
+  local rawtab = {}
+  for i=0,sizebc-1 do
+    local op = p[xop]
+    if op == BC.KSHORT then
+      local rd = p[xrc] + 256*p[xrb]
+      rd = bit.arshift(bit.lshift(rd, 16), 16)
+      local f = fixup[rd]
+      if f then
+        if f[1] == "CHECK" then
+          local tp = f[2]
+          if tp == "tab" then rawtab[p[xra]] = true end
+          p[xop] = tp == "num" and BC.ISNUM or BC.ISTYPE
+          p[xrb] = 0
+          p[xrc] = name2itype[tp]
+        else
+          error("unhandled fixup type: "..f[1])
+        end
+      end
+    elseif op == BC.TGETV then
+      if rawtab[p[xrb]] then
+        p[xop] = BC.TGETR
+      end
+    elseif op == BC.TSETV then
+      if rawtab[p[xrb]] then
+        p[xop] = BC.TSETR
+      end
+    elseif op == BC.ITERC then
+      if fixup.PAIRS then
+        p[xop] = BC.ITERN
+      end
+    end
+    p = p + 4
+  end
+  return ffi.string(start, n)
+end
+local function find_defs(src)
+  local defs = {}
+  for name, code in string.gmatch(src, "LJLIB_LUA%(([^)]*)%)%s*/%*(.-)%*/") do
+    local env = {}
+    local tcode, fixup = transform_lua(code)
+    local func = assert(load(tcode, "", nil, env))()
+    defs[name] = fixup_dump(string.dump(func, true), fixup)
+    defs[#defs+1] = name
+  end
+  return defs
+end
+local function gen_header(defs)
+  local t = {}
+  local function w(x) t[#t+1] = x end
+  w("/* This is a generated file. DO NOT EDIT! */\n\n")
+  w("static const int libbc_endian = ") w(isbe and 1 or 0) w(";\n\n")
+  local s = ""
+  for _,name in ipairs(defs) do
+    s = s .. defs[name]
+  end
+  w("static const uint8_t libbc_code[] = {\n")
+  local n = 0
+  for i=1,#s do
+    local x = string.byte(s, i)
+    w(x); w(",")
+    n = n + (x < 10 and 2 or (x < 100 and 3 or 4))
+    if n >= 75 then n = 0; w("\n") end
+  end
+  w("0\n};\n\n")
+  w("static const struct { const char *name; int ofs; } libbc_map[] = {\n")
+  local m = 0
+  for _,name in ipairs(defs) do
+    w('{"'); w(name); w('",'); w(m) w('},\n')
+    m = m + #defs[name]
+  end
+  w("{NULL,"); w(m); w("}\n};\n\n")
+  return table.concat(t)
+end
+local function write_file(name, data)
+  if name == "-" then
+    assert(io.write(data))
+    assert(io.flush())
+  else
+    local fp = io.open(name)
+    if fp then
+      local old = fp:read("*a")
+      fp:close()
+      if data == old then return end
+    end
+    fp = assert(io.open(name, "w"))
+    assert(fp:write(data))
+    assert(fp:close())
+  end
+end
+local outfile = parse_arg(arg)
+local src = read_files(arg)
+local defs = find_defs(src)
+local hdr = gen_header(defs)
+write_file(outfile, hdr)
diff --git a/src/jit/bc.lua b/src/jit/bc.lua
index 5c00ebe3..566b09c3 100644
--- a/src/jit/bc.lua
+++ b/src/jit/bc.lua
@@ -41,7 +41,7 @@
 -- Cache some library functions and objects.
 local jit = require("jit")
-assert(jit.version_num == 20001, "LuaJIT core/library version mismatch")
+assert(jit.version_num == 20100, "LuaJIT core/library version mismatch")
 local jutil = require("jit.util")
 local vmdef = require("jit.vmdef")
 local bit = require("bit")
@@ -179,13 +179,12 @@ local function bcliston(outfile)
 end
 -- Public module functions.
-module(...)
+return {
+  line = bcline,
-line = bcline
+  dump = bcdump,
-dump = bcdump
+  targets = bctargets,
-targets = bctargets
+  on = bcliston,
+  off = bclistoff,
-on = bcliston
+  start = bcliston -- For -j command line option.
-off = bclistoff
+}
-start = bcliston -- For -j command line option.
diff --git a/src/jit/bcsave.lua b/src/jit/bcsave.lua
index 25bd6042..8aad7596 100644
--- a/src/jit/bcsave.lua
+++ b/src/jit/bcsave.lua
@@ -11,7 +11,7 @@
 ------------------------------------------------------------------------------
 local jit = require("jit")
-assert(jit.version_num == 20001, "LuaJIT core/library version mismatch")
+assert(jit.version_num == 20100, "LuaJIT core/library version mismatch")
 local bit = require("bit")
 -- Symbol name prefix for LuaJIT bytecode.
@@ -653,7 +653,7 @@ end
 ------------------------------------------------------------------------------
 -- Public module functions.
-module(...)
+return {
+  start = docmd -- Process -b command line option.
-start = docmd -- Process -b command line option.
+}
diff --git a/src/jit/dis_arm.lua b/src/jit/dis_arm.lua
index dc7ca71f..87a84e93 100644
--- a/src/jit/dis_arm.lua
+++ b/src/jit/dis_arm.lua
@@ -658,7 +658,7 @@ local function disass_block(ctx, ofs, len)
 end
 -- Extended API: create a disassembler context. Then call ctx:disass(ofs, len).
-local function create_(code, addr, out)
+local function create(code, addr, out)
  local ctx = {}
  ctx.code = code
  ctx.addr = addr or 0
@@ -670,20 +670,20 @@ local function create_(code, addr, out)
 end
 -- Simple API: disassemble code (a string) at address and output via out.
-local function disass_(code, addr, out)
+local function disass(code, addr, out)
-  create_(code, addr, out):disass()
+  create(code, addr, out):disass()
 end
 -- Return register name for RID.
-local function regname_(r)
+local function regname(r)
  if r < 16 then return map_gpr[r] end
  return "d"..(r-16)
 end
 -- Public module functions.
-module(...)
+return {
+  create = create,
-create = create_
+  disass = disass,
-disass = disass_
+  regname = regname
-regname = regname_
+}
diff --git a/src/jit/dis_mips.lua b/src/jit/dis_mips.lua
index 830db409..9ba0e019 100644
--- a/src/jit/dis_mips.lua
+++ b/src/jit/dis_mips.lua
@@ -384,7 +384,7 @@ local function disass_block(ctx, ofs, len)
 end
 -- Extended API: create a disassembler context. Then call ctx:disass(ofs, len).
-local function create_(code, addr, out)
+local function create(code, addr, out)
  local ctx = {}
  ctx.code = code
  ctx.addr = addr or 0
@@ -396,33 +396,33 @@ local function create_(code, addr, out)
  return ctx
 end
-local function create_el_(code, addr, out)
+local function create_el(code, addr, out)
-  local ctx = create_(code, addr, out)
+  local ctx = create(code, addr, out)
  ctx.get = get_le
  return ctx
 end
 -- Simple API: disassemble code (a string) at address and output via out.
-local function disass_(code, addr, out)
+local function disass(code, addr, out)
-  create_(code, addr, out):disass()
+  create(code, addr, out):disass()
 end
-local function disass_el_(code, addr, out)
+local function disass_el(code, addr, out)
-  create_el_(code, addr, out):disass()
+  create_el(code, addr, out):disass()
 end
 -- Return register name for RID.
-local function regname_(r)
+local function regname(r)
  if r < 32 then return map_gpr[r] end
  return "f"..(r-32)
 end
 -- Public module functions.
-module(...)
+return {
+  create = create,
-create = create_
+  create_el = create_el,
-create_el = create_el_
+  disass = disass,
-disass = disass_
+  disass_el = disass_el,
-disass_el = disass_el_
+  regname = regname
-regname = regname_
+}
diff --git a/src/jit/dis_mipsel.lua b/src/jit/dis_mipsel.lua
index 8a10c462..60e0233b 100644
--- a/src/jit/dis_mipsel.lua
+++ b/src/jit/dis_mipsel.lua
@@ -8,13 +8,10 @@
 -- MIPS disassembler module. All the interesting stuff is there.
 ------------------------------------------------------------------------------
-local require = require
+local dis_mips = require((string.match(..., ".*%.") or "").."dis_mips")
+return {
-module(...)
+  create = dis_mips.create_el,
+  disass = dis_mips.disass_el,
-local dis_mips = require(_PACKAGE.."dis_mips")
+  regname = dis_mips.regname
+}
-create = dis_mips.create_el
-disass = dis_mips.disass_el
-regname = dis_mips.regname
diff --git a/src/jit/dis_ppc.lua b/src/jit/dis_ppc.lua
index 169a534c..5143d47a 100644
--- a/src/jit/dis_ppc.lua
+++ b/src/jit/dis_ppc.lua
@@ -560,7 +560,7 @@ local function disass_block(ctx, ofs, len)
 end
 -- Extended API: create a disassembler context. Then call ctx:disass(ofs, len).
-local function create_(code, addr, out)
+local function create(code, addr, out)
  local ctx = {}
  ctx.code = code
  ctx.addr = addr or 0
@@ -572,20 +572,20 @@ local function create_(code, addr, out)
 end
 -- Simple API: disassemble code (a string) at address and output via out.
-local function disass_(code, addr, out)
+local function disass(code, addr, out)
-  create_(code, addr, out):disass()
+  create(code, addr, out):disass()
 end
 -- Return register name for RID.
-local function regname_(r)
+local function regname(r)
  if r < 32 then return map_gpr[r] end
  return "f"..(r-32)
 end
 -- Public module functions.
-module(...)
+return {
+  create = create,
-create = create_
+  disass = disass,
-disass = disass_
+  regname = regname
-regname = regname_
+}
diff --git a/src/jit/dis_x64.lua b/src/jit/dis_x64.lua
index 4a1894ac..2f4f6e7c 100644
--- a/src/jit/dis_x64.lua
+++ b/src/jit/dis_x64.lua
@@ -8,13 +8,10 @@
 -- x86/x64 disassembler module. All the interesting stuff is there.
 ------------------------------------------------------------------------------
-local require = require
+local dis_x86 = require((string.match(..., ".*%.") or "").."dis_x86")
+return {
-module(...)
+  create = dis_x86.create64,
+  disass = dis_x86.disass64,
-local dis_x86 = require(_PACKAGE.."dis_x86")
+  regname = dis_x86.regname64
+}
-create = dis_x86.create64
-disass = dis_x86.disass64
-regname = dis_x86.regname64
diff --git a/src/jit/dis_x86.lua b/src/jit/dis_x86.lua
index c442a176..14b0fd61 100644
--- a/src/jit/dis_x86.lua
+++ b/src/jit/dis_x86.lua
@@ -28,6 +28,8 @@ local type = type
 local sub, byte, format = string.sub, string.byte, string.format
 local match, gmatch, gsub = string.match, string.gmatch, string.gsub
 local lower, rep = string.lower, string.rep
+local bit = require("bit")
+local tohex = bit.tohex
 -- Map for 1st opcode byte in 32 bit mode. Ugly? Well ... read on.
 local map_opc1_32 = {
@@ -532,7 +534,7 @@ local function putpat(ctx, name, pat)
        local lo = imm % 0x1000000
        x = format("0x%02x%06x", (imm-lo) / 0x1000000, lo)
      else
-        x = format("0x%08x", imm)
+        x = "0x"..tohex(imm)
      end
    elseif p == "R" then
      local r = byte(code, pos-1, pos-1)%8
@@ -782,7 +784,7 @@ local function disass_block(ctx, ofs, len)
 end
 -- Extended API: create a disassembler context. Then call ctx:disass(ofs, len).
-local function create_(code, addr, out)
+local function create(code, addr, out)
  local ctx = {}
  ctx.code = code
  ctx.addr = (addr or 0) - 1
@@ -796,8 +798,8 @@ local function create_(code, addr, out)
  return ctx
 end
-local function create64_(code, addr, out)
+local function create64(code, addr, out)
-  local ctx = create_(code, addr, out)
+  local ctx = create(code, addr, out)
  ctx.x64 = true
  ctx.map1 = map_opc1_64
  ctx.aregs = map_regs.Q
@@ -805,32 +807,32 @@ local function create64_(code, addr, out)
 end
 -- Simple API: disassemble code (a string) at address and output via out.
-local function disass_(code, addr, out)
+local function disass(code, addr, out)
-  create_(code, addr, out):disass()
+  create(code, addr, out):disass()
 end
-local function disass64_(code, addr, out)
+local function disass64(code, addr, out)
-  create64_(code, addr, out):disass()
+  create64(code, addr, out):disass()
 end
 -- Return register name for RID.
-local function regname_(r)
+local function regname(r)
  if r < 8 then return map_regs.D[r+1] end
  return map_regs.X[r-7]
 end
-local function regname64_(r)
+local function regname64(r)
  if r < 16 then return map_regs.Q[r+1] end
  return map_regs.X[r-15]
 end
 -- Public module functions.
-module(...)
+return {
+  create = create,
-create = create_
+  create64 = create64,
-create64 = create64_
+  disass = disass,
-disass = disass_
+  disass64 = disass64,
-disass64 = disass64_
+  regname = regname,
-regname = regname_
+  regname64 = regname64
-regname64 = regname64_
+}
diff --git a/src/jit/dump.lua b/src/jit/dump.lua
index 70a59280..18a4d260 100644
--- a/src/jit/dump.lua
+++ b/src/jit/dump.lua
@@ -54,7 +54,7 @@
 -- Cache some library functions and objects.
 local jit = require("jit")
-assert(jit.version_num == 20001, "LuaJIT core/library version mismatch")
+assert(jit.version_num == 20100, "LuaJIT core/library version mismatch")
 local jutil = require("jit.util")
 local vmdef = require("jit.vmdef")
 local funcinfo, funcbc = jutil.funcinfo, jutil.funcbc
@@ -62,7 +62,7 @@ local traceinfo, traceir, tracek = jutil.traceinfo, jutil.traceir, jutil.tracek
 local tracemc, tracesnap = jutil.tracemc, jutil.tracesnap
 local traceexitstub, ircalladdr = jutil.traceexitstub, jutil.ircalladdr
 local bit = require("bit")
-local band, shl, shr = bit.band, bit.lshift, bit.rshift
+local band, shl, shr, tohex = bit.band, bit.lshift, bit.rshift, bit.tohex
 local sub, gsub, format = string.sub, string.gsub, string.format
 local byte, char, rep = string.byte, string.char, string.rep
 local type, tostring = type, tostring
@@ -135,6 +135,7 @@ local function dump_mcode(tr)
  local mcode, addr, loop = tracemc(tr)
  if not mcode then return end
  if not disass then disass = require("jit.dis_"..jit.arch) end
+  if addr < 0 then addr = addr + 2^32 end
  out:write("---- TRACE ", tr, " mcode ", #mcode, "\n")
  local ctx = disass.create(mcode, addr, dumpwrite)
  ctx.hexdump = 0
@@ -269,8 +270,7 @@ local litname = {
  ["CONV  "] = setmetatable({}, { __index = function(t, mode)
    local s = irtype[band(mode, 31)]
    s = irtype[band(shr(mode, 5), 31)].."."..s
-    if band(mode, 0x400) ~= 0 then s = s.." trunc"
+    if band(mode, 0x800) ~= 0 then s = s.." sext" end
-    elseif band(mode, 0x800) ~= 0 then s = s.." sext" end
    local c = shr(mode, 14)
    if c == 2 then s = s.." index" elseif c == 3 then s = s.." check" end
    t[mode] = s
@@ -279,6 +279,8 @@ local litname = {
  ["FLOAD "] = vmdef.irfield,
  ["FREF  "] = vmdef.irfield,
  ["FPMATH"] = vmdef.irfpm,
+  ["BUFHDR"] = { [0] = "RESET", "APPEND" },
+  ["TOSTR "] = { [0] = "INT", "NUM", "CHAR" },
 }
 local function ctlsub(c)
@@ -608,7 +610,7 @@ local function dump_texit(tr, ex, ngpr, nfpr, ...)
      end
    else
      for i=1,ngpr do
-        out:write(format(" %08x", regs[i]))
+        out:write(" ", tohex(regs[i]))
        if i % 8 == 0 then out:write("\n") end
      end
    end
@@ -692,9 +694,9 @@ local function dumpon(opt, outfile)
 end
 -- Public module functions.
-module(...)
+return {
+  on = dumpon,
-on = dumpon
+  off = dumpoff,
-off = dumpoff
+  start = dumpon -- For -j command line option.
-start = dumpon -- For -j command line option.
+}
diff --git a/src/jit/v.lua b/src/jit/v.lua
index f4a9b054..22bee3ff 100644
--- a/src/jit/v.lua
+++ b/src/jit/v.lua
@@ -59,7 +59,7 @@
 -- Cache some library functions and objects.
 local jit = require("jit")
-assert(jit.version_num == 20001, "LuaJIT core/library version mismatch")
+assert(jit.version_num == 20100, "LuaJIT core/library version mismatch")
 local jutil = require("jit.util")
 local vmdef = require("jit.vmdef")
 local funcinfo, traceinfo = jutil.funcinfo, jutil.traceinfo
@@ -159,9 +159,9 @@ local function dumpon(outfile)
 end
 -- Public module functions.
-module(...)
+return {
+  on = dumpon,
-on = dumpon
+  off = dumpoff,
-off = dumpoff
+  start = dumpon -- For -j command line option.
-start = dumpon -- For -j command line option.
+}
diff --git a/src/lib_base.c b/src/lib_base.c
index 070970ed..44817187 100644
--- a/src/lib_base.c
+++ b/src/lib_base.c
@@ -32,6 +32,7 @@
 #include "lj_dispatch.h"
 #include "lj_char.h"
 #include "lj_strscan.h"
+#include "lj_strfmt.h"
 #include "lj_lib.h"
 /* -- Base library: checks ------------------------------------------------ */
@@ -301,9 +302,6 @@ LJLIB_ASM(tonumber)		LJLIB_REC(.)
  return FFH_RES(1);
 }
-LJLIB_PUSH("nil")
-LJLIB_PUSH("false")
-LJLIB_PUSH("true")
 LJLIB_ASM(tostring)             LJLIB_REC(.)
 {
  TValue *o = lj_lib_checkany(L, 1);
@@ -312,23 +310,10 @@ LJLIB_ASM(tostring)		LJLIB_REC(.)
  if (!tvisnil(mo = lj_meta_lookup(L, o, MM_tostring))) {
    copyTV(L, L->base-1, mo);  /* Replace callable. */
    return FFH_TAILCALL;
-  } else {
-    GCstr *s;
-    if (tvisnumber(o)) {
-      s = lj_str_fromnumber(L, o);
-    } else if (tvispri(o)) {
-      s = strV(lj_lib_upvalue(L, -(int32_t)itype(o)));
-    } else {
-      if (tvisfunc(o) && isffunc(funcV(o)))
-        lua_pushfstring(L, "function: builtin#%d", funcV(o)->c.ffid);
-      else
-        lua_pushfstring(L, "%s: %p", lj_typename(o), lua_topointer(L, 1));
-      /* Note: lua_pushfstring calls the GC which may invalidate o. */
-      s = strV(L->top-1);
-    }
-    setstrV(L, L->base-1, s);
-    return FFH_RES(1);
  }
+  lj_gc_check(L);
+  setstrV(L, L->base-1, lj_strfmt_obj(L, L->base));
+  return FFH_RES(1);
 }
 /* -- Base library: throw and catch errors -------------------------------- */
@@ -506,21 +491,13 @@ LJLIB_CF(print)
  }
  shortcut = (tvisfunc(tv) && funcV(tv)->c.ffid == FF_tostring);
  for (i = 0; i < nargs; i++) {
+    cTValue *o = &L->base[i];
+    char buf[STRFMT_MAXBUF_NUM];
    const char *str;
    size_t size;
-    cTValue *o = &L->base[i];
+    MSize len;
-    if (shortcut && tvisstr(o)) {
+    if (shortcut && (str = lj_strfmt_wstrnum(buf, o, &len)) != NULL) {
-      str = strVdata(o);
+      size = len;
-      size = strV(o)->len;
-    } else if (shortcut && tvisint(o)) {
-      char buf[LJ_STR_INTBUF];
-      char *p = lj_str_bufint(buf, intV(o));
-      size = (size_t)(buf+LJ_STR_INTBUF-p);
-      str = p;
-    } else if (shortcut && tvisnum(o)) {
-      char buf[LJ_STR_NUMBUF];
-      size = lj_str_bufnum(buf, o);
-      str = buf;
    } else {
      copyTV(L, L->top+1, o);
      copyTV(L, L->top, L->top-1);
diff --git a/src/lib_bit.c b/src/lib_bit.c
index 93fead92..a3f7c1ac 100644
--- a/src/lib_bit.c
+++ b/src/lib_bit.c
@@ -12,26 +12,99 @@
 #include "lj_obj.h"
 #include "lj_err.h"
-#include "lj_str.h"
+#include "lj_buf.h"
+#include "lj_strscan.h"
+#include "lj_strfmt.h"
+#if LJ_HASFFI
+#include "lj_ctype.h"
+#include "lj_cdata.h"
+#include "lj_cconv.h"
+#include "lj_carith.h"
+#endif
+#include "lj_ff.h"
 #include "lj_lib.h"
 /* ------------------------------------------------------------------------ */
 #define LJLIB_MODULE_bit
-LJLIB_ASM(bit_tobit)            LJLIB_REC(bit_unary IR_TOBIT)
+#if LJ_HASFFI
+static int bit_result64(lua_State *L, CTypeID id, uint64_t x)
 {
+  GCcdata *cd = lj_cdata_new_(L, id, 8);
+  *(uint64_t *)cdataptr(cd) = x;
+  setcdataV(L, L->base-1, cd);
+  return FFH_RES(1);
+}
+#else
+static int32_t bit_checkbit(lua_State *L, int narg)
+{
+  TValue *o = L->base + narg-1;
+  if (!(o < L->top && lj_strscan_numberobj(o)))
+    lj_err_argt(L, narg, LUA_TNUMBER);
+  if (LJ_LIKELY(tvisint(o))) {
+    return intV(o);
+  } else {
+    int32_t i = lj_num2bit(numV(o));
+    if (LJ_DUALNUM) setintV(o, i);
+    return i;
+  }
+}
+#endif
+LJLIB_ASM(bit_tobit)            LJLIB_REC(bit_tobit)
+{
+#if LJ_HASFFI
+  CTypeID id = 0;
+  setintV(L->base-1, (int32_t)lj_carith_check64(L, 1, &id));
+  return FFH_RES(1);
+#else
+  lj_lib_checknumber(L, 1);
+  return FFH_RETRY;
+#endif
+}
+LJLIB_ASM(bit_bnot)             LJLIB_REC(bit_unary IR_BNOT)
+{
+#if LJ_HASFFI
+  CTypeID id = 0;
+  uint64_t x = lj_carith_check64(L, 1, &id);
+  return id ? bit_result64(L, id, ~x) : FFH_RETRY;
+#else
  lj_lib_checknumber(L, 1);
  return FFH_RETRY;
+#endif
+}
+LJLIB_ASM(bit_bswap)            LJLIB_REC(bit_unary IR_BSWAP)
+{
+#if LJ_HASFFI
+  CTypeID id = 0;
+  uint64_t x = lj_carith_check64(L, 1, &id);
+  return id ? bit_result64(L, id, lj_bswap64(x)) : FFH_RETRY;
+#else
+  lj_lib_checknumber(L, 1);
+  return FFH_RETRY;
+#endif
 }
-LJLIB_ASM_(bit_bnot)            LJLIB_REC(bit_unary IR_BNOT)
-LJLIB_ASM_(bit_bswap)           LJLIB_REC(bit_unary IR_BSWAP)
 LJLIB_ASM(bit_lshift)           LJLIB_REC(bit_shift IR_BSHL)
 {
+#if LJ_HASFFI
+  CTypeID id = 0, id2 = 0;
+  uint64_t x = lj_carith_check64(L, 1, &id);
+  int32_t sh = (int32_t)lj_carith_check64(L, 2, &id2);
+  if (id) {
+    x = lj_carith_shift64(x, sh, curr_func(L)->c.ffid - (int)FF_bit_lshift);
+    return bit_result64(L, id, x);
+  }
+  if (id2) setintV(L->base+1, sh);
+  return FFH_RETRY;
+#else
  lj_lib_checknumber(L, 1);
-  lj_lib_checkbit(L, 2);
+  bit_checkbit(L, 2);
  return FFH_RETRY;
+#endif
 }
 LJLIB_ASM_(bit_rshift)          LJLIB_REC(bit_shift IR_BSHR)
 LJLIB_ASM_(bit_arshift)         LJLIB_REC(bit_shift IR_BSAR)
@@ -40,25 +113,58 @@ LJLIB_ASM_(bit_ror)		LJLIB_REC(bit_shift IR_BROR)
 LJLIB_ASM(bit_band)             LJLIB_REC(bit_nary IR_BAND)
 {
+#if LJ_HASFFI
+  CTypeID id = 0;
+  TValue *o = L->base, *top = L->top;
+  int i = 0;
+  do { lj_carith_check64(L, ++i, &id); } while (++o < top);
+  if (id) {
+    CTState *cts = ctype_cts(L);
+    CType *ct = ctype_get(cts, id);
+    int op = curr_func(L)->c.ffid - (int)FF_bit_bor;
+    uint64_t x, y = op >= 0 ? 0 : ~(uint64_t)0;
+    o = L->base;
+    do {
+      lj_cconv_ct_tv(cts, ct, (uint8_t *)&x, o, 0);
+      if (op < 0) y &= x; else if (op == 0) y |= x; else y ^= x;
+    } while (++o < top);
+    return bit_result64(L, id, y);
+  }
+  return FFH_RETRY;
+#else
  int i = 0;
  do { lj_lib_checknumber(L, ++i); } while (L->base+i < L->top);
  return FFH_RETRY;
+#endif
 }
 LJLIB_ASM_(bit_bor)             LJLIB_REC(bit_nary IR_BOR)
 LJLIB_ASM_(bit_bxor)            LJLIB_REC(bit_nary IR_BXOR)
 /* ------------------------------------------------------------------------ */
-LJLIB_CF(bit_tohex)
+LJLIB_CF(bit_tohex)             LJLIB_REC(.)
 {
-  uint32_t b = (uint32_t)lj_lib_checkbit(L, 1);
+#if LJ_HASFFI
-  int32_t i, n = L->base+1 >= L->top ? 8 : lj_lib_checkbit(L, 2);
+  CTypeID id = 0, id2 = 0;
-  const char *hexdigits = "0123456789abcdef";
+  uint64_t b = lj_carith_check64(L, 1, &id);
-  char buf[8];
+  int32_t n = L->base+1>=L->top ? (id ? 16 : 8) :
-  if (n < 0) { n = -n; hexdigits = "0123456789ABCDEF"; }
+                                  (int32_t)lj_carith_check64(L, 2, &id2);
-  if (n > 8) n = 8;
+#else
-  for (i = n; --i >= 0; ) { buf[i] = hexdigits[b & 15]; b >>= 4; }
+  uint32_t b = (uint32_t)bit_checkbit(L, 1);
-  lua_pushlstring(L, buf, (size_t)n);
+  int32_t n = L->base+1>=L->top ? 8 : bit_checkbit(L, 2);
+#endif
+  SBuf *sb = lj_buf_tmp_(L);
+  SFormat sf = (STRFMT_UINT|STRFMT_T_HEX);
+  if (n < 0) { n = -n; sf |= STRFMT_F_UPPER; }
+  sf |= ((SFormat)((n+1)&255) << STRFMT_SH_PREC);
+#if LJ_HASFFI
+  if (n < 16) b &= ((uint64_t)1 << 4*n)-1;
+#else
+  if (n < 8) b &= (1u << 4*n)-1;
+#endif
+  sb = lj_strfmt_putfxint(sb, sf, b);
+  setstrV(L, L->top-1, lj_buf_str(L, sb));
+  lj_gc_check(L);
  return 1;
 }
diff --git a/src/lib_ffi.c b/src/lib_ffi.c
index f61fabc0..3310b205 100644
--- a/src/lib_ffi.c
+++ b/src/lib_ffi.c
@@ -29,6 +29,7 @@
 #include "lj_ccall.h"
 #include "lj_ccallback.h"
 #include "lj_clib.h"
+#include "lj_strfmt.h"
 #include "lj_ff.h"
 #include "lj_lib.h"
@@ -317,7 +318,7 @@ LJLIB_CF(ffi_meta___tostring)
      }
    }
  }
-  lj_str_pushf(L, msg, strdata(lj_ctype_repr(L, id, NULL)), p);
+  lj_strfmt_pushf(L, msg, strdata(lj_ctype_repr(L, id, NULL)), p);
 checkgc:
  lj_gc_check(L);
  return 1;
@@ -506,7 +507,7 @@ LJLIB_CF(ffi_new)	LJLIB_REC(.)
  if (!(info & CTF_VLA) && ctype_align(info) <= CT_MEMALIGN)
    cd = lj_cdata_new(cts, id, sz);
  else
-    cd = lj_cdata_newv(cts, id, sz, ctype_align(info));
+    cd = lj_cdata_newv(L, id, sz, ctype_align(info));
  setcdataV(L, o-1, cd);  /* Anchor the uninitialized cdata. */
  lj_cconv_ct_init(cts, ct, sz, cdataptr(cd),
                   o, (MSize)(L->top - o));  /* Initialize cdata. */
@@ -767,19 +768,11 @@ LJLIB_CF(ffi_gc)	LJLIB_REC(.)
  GCcdata *cd = ffi_checkcdata(L, 1);
  TValue *fin = lj_lib_checkany(L, 2);
  CTState *cts = ctype_cts(L);
-  GCtab *t = cts->finalizer;
  CType *ct = ctype_raw(cts, cd->ctypeid);
  if (!(ctype_isptr(ct->info) || ctype_isstruct(ct->info) ||
        ctype_isrefarray(ct->info)))
    lj_err_arg(L, 1, LJ_ERR_FFI_INVTYPE);
-  if (gcref(t->metatable)) {  /* Update finalizer table, if still enabled. */
+  lj_cdata_setfin(L, cd, gcval(fin), itype(fin));
-    copyTV(L, lj_tab_set(L, t, L->base), fin);
-    lj_gc_anybarriert(L, t);
-    if (!tvisnil(fin))
-      cd->marked |= LJ_GC_CDATA_FIN;
-    else
-      cd->marked &= ~LJ_GC_CDATA_FIN;
-  }
  L->top = L->base+1;  /* Pass through the cdata object. */
  return 1;
 }
diff --git a/src/lib_io.c b/src/lib_io.c
index e0c6908f..ca87ec4d 100644
--- a/src/lib_io.c
+++ b/src/lib_io.c
@@ -19,8 +19,10 @@
 #include "lj_obj.h"
 #include "lj_gc.h"
 #include "lj_err.h"
+#include "lj_buf.h"
 #include "lj_str.h"
 #include "lj_state.h"
+#include "lj_strfmt.h"
 #include "lj_ff.h"
 #include "lj_lib.h"
@@ -84,7 +86,7 @@ static IOFileUD *io_file_open(lua_State *L, const char *mode)
  IOFileUD *iof = io_file_new(L);
  iof->fp = fopen(fname, mode);
  if (iof->fp == NULL)
-    luaL_argerror(L, 1, lj_str_pushf(L, "%s: %s", fname, strerror(errno)));
+    luaL_argerror(L, 1, lj_strfmt_pushf(L, "%s: %s", fname, strerror(errno)));
  return iof;
 }
@@ -145,7 +147,7 @@ static int io_file_readline(lua_State *L, FILE *fp, MSize chop)
  MSize m = LUAL_BUFFERSIZE, n = 0, ok = 0;
  char *buf;
  for (;;) {
-    buf = lj_str_needbuf(L, &G(L)->tmpbuf, m);
+    buf = lj_buf_tmp(L, m);
    if (fgets(buf+n, m-n, fp) == NULL) break;
    n += (MSize)strlen(buf+n);
    ok |= n;
@@ -161,7 +163,7 @@ static void io_file_readall(lua_State *L, FILE *fp)
 {
  MSize m, n;
  for (m = LUAL_BUFFERSIZE, n = 0; ; m += m) {
-    char *buf = lj_str_needbuf(L, &G(L)->tmpbuf, m);
+    char *buf = lj_buf_tmp(L, m);
    n += (MSize)fread(buf+n, 1, m-n, fp);
    if (n != m) {
      setstrV(L, L->top++, lj_str_new(L, buf, (size_t)n));
@@ -174,7 +176,7 @@ static void io_file_readall(lua_State *L, FILE *fp)
 static int io_file_readlen(lua_State *L, FILE *fp, MSize m)
 {
  if (m) {
-    char *buf = lj_str_needbuf(L, &G(L)->tmpbuf, m);
+    char *buf = lj_buf_tmp(L, m);
    MSize n = (MSize)fread(buf, 1, m, fp);
    setstrV(L, L->top++, lj_str_new(L, buf, (size_t)n));
    lj_gc_check(L);
@@ -230,19 +232,12 @@ static int io_file_write(lua_State *L, FILE *fp, int start)
  cTValue *tv;
  int status = 1;
  for (tv = L->base+start; tv < L->top; tv++) {
-    if (tvisstr(tv)) {
+    char buf[STRFMT_MAXBUF_NUM];
-      MSize len = strV(tv)->len;
+    MSize len;
-      status = status && (fwrite(strVdata(tv), 1, len, fp) == len);
+    const char *p = lj_strfmt_wstrnum(buf, tv, &len);
-    } else if (tvisint(tv)) {
+    if (!p)
-      char buf[LJ_STR_INTBUF];
-      char *p = lj_str_bufint(buf, intV(tv));
-      size_t len = (size_t)(buf+LJ_STR_INTBUF-p);
-      status = status && (fwrite(p, 1, len, fp) == len);
-    } else if (tvisnum(tv)) {
-      status = status && (fprintf(fp, LUA_NUMBER_FMT, numV(tv)) > 0);
-    } else {
      lj_err_argt(L, (int)(tv - L->base) + 1, LUA_TSTRING);
-    }
+    status = status && (fwrite(p, 1, len, fp) == len);
  }
  if (LJ_52 && status) {
    L->top = L->base+1;
diff --git a/src/lib_jit.c b/src/lib_jit.c
index 82e68258..555e581c 100644
--- a/src/lib_jit.c
+++ b/src/lib_jit.c
@@ -16,7 +16,11 @@
 #include "lj_debug.h"
 #include "lj_str.h"
 #include "lj_tab.h"
+#include "lj_state.h"
 #include "lj_bc.h"
+#if LJ_HASFFI
+#include "lj_ctype.h"
+#endif
 #if LJ_HASJIT
 #include "lj_ir.h"
 #include "lj_jit.h"
@@ -332,6 +336,13 @@ LJLIB_CF(jit_util_tracek)
      slot = ir->op2;
      ir = &T->ir[ir->op1];
    }
+#if LJ_HASFFI
+    if (ir->o == IR_KINT64 && !ctype_ctsG(G(L))) {
+      ptrdiff_t oldtop = savestack(L, L->top);
+      luaopen_ffi(L);  /* Load FFI library on-demand. */
+      L->top = restorestack(L, oldtop);
+    }
+#endif
    lj_ir_kvalue(L, L->top-2, ir);
    setintV(L->top-1, (int32_t)irt_type(ir->t));
    if (slot == -1)
@@ -538,23 +549,17 @@ static uint32_t jit_cpudetect(lua_State *L)
  uint32_t features[4];
  if (lj_vm_cpuid(0, vendor) && lj_vm_cpuid(1, features)) {
 #if !LJ_HASJIT
-#define JIT_F_CMOV      1
 #define JIT_F_SSE2      2
 #endif
-    flags |= ((features[3] >> 15)&1) * JIT_F_CMOV;
    flags |= ((features[3] >> 26)&1) * JIT_F_SSE2;
 #if LJ_HASJIT
    flags |= ((features[2] >> 0)&1) * JIT_F_SSE3;
    flags |= ((features[2] >> 19)&1) * JIT_F_SSE4_1;
    if (vendor[2] == 0x6c65746e) {  /* Intel. */
-      if ((features[0] & 0x0ff00f00) == 0x00000f00)  /* P4. */
+      if ((features[0] & 0x0fff0ff0) == 0x000106c0)  /* Atom. */
-        flags |= JIT_F_P4;  /* Currently unused. */
-      else if ((features[0] & 0x0fff0ff0) == 0x000106c0)  /* Atom. */
        flags |= JIT_F_LEA_AGU;
    } else if (vendor[2] == 0x444d4163) {  /* AMD. */
      uint32_t fam = (features[0] & 0x0ff00f00);
-      if (fam == 0x00000f00)  /* K8. */
-        flags |= JIT_F_SPLIT_XMM;
      if (fam >= 0x00000f00)  /* K8, K10. */
        flags |= JIT_F_PREFER_IMUL;
    }
@@ -562,14 +567,8 @@ static uint32_t jit_cpudetect(lua_State *L)
  }
  /* Check for required instruction set support on x86 (unnecessary on x64). */
 #if LJ_TARGET_X86
-#if !defined(LUAJIT_CPU_NOCMOV)
-  if (!(flags & JIT_F_CMOV))
-    luaL_error(L, "CPU not supported");
-#endif
-#if defined(LUAJIT_CPU_SSE2)
  if (!(flags & JIT_F_SSE2))
-    luaL_error(L, "CPU does not support SSE2 (recompile without -DLUAJIT_CPU_SSE2)");
+    luaL_error(L, "CPU with SSE2 required");
-#endif
 #endif
 #elif LJ_TARGET_ARM
 #if LJ_HASJIT
@@ -631,11 +630,7 @@ static void jit_init(lua_State *L)
  uint32_t flags = jit_cpudetect(L);
 #if LJ_HASJIT
  jit_State *J = L2J(L);
-#if LJ_TARGET_X86
+  J->flags = flags | JIT_F_ON | JIT_F_OPT_DEFAULT;
-  /* Silently turn off the JIT compiler on CPUs without SSE2. */
-  if ((flags & JIT_F_SSE2))
-#endif
-    J->flags = flags | JIT_F_ON | JIT_F_OPT_DEFAULT;
  memcpy(J->param, jit_param_default, sizeof(J->param));
  lj_dispatch_update(G(L));
 #else
@@ -645,6 +640,7 @@ static void jit_init(lua_State *L)
 LUALIB_API int luaopen_jit(lua_State *L)
 {
+  jit_init(L);
  lua_pushliteral(L, LJ_OS_NAME);
  lua_pushliteral(L, LJ_ARCH_NAME);
  lua_pushinteger(L, LUAJIT_VERSION_NUM);
@@ -657,7 +653,6 @@ LUALIB_API int luaopen_jit(lua_State *L)
  LJ_LIB_REG(L, "jit.opt", jit_opt);
 #endif
  L->top -= 2;
-  jit_init(L);
  return 1;
 }
diff --git a/src/lib_math.c b/src/lib_math.c
index b23d9a2d..e474f980 100644
--- a/src/lib_math.c
+++ b/src/lib_math.c
@@ -63,11 +63,8 @@ LJLIB_ASM(math_log)		LJLIB_REC(math_log)
  return FFH_RETRY;
 }
-LJLIB_PUSH(57.29577951308232)
+LJLIB_LUA(math_deg) /* function(x) return x * 57.29577951308232 end */
-LJLIB_ASM_(math_deg)            LJLIB_REC(math_degrad)
+LJLIB_LUA(math_rad) /* function(x) return x * 0.017453292519943295 end */
-LJLIB_PUSH(0.017453292519943295)
-LJLIB_ASM_(math_rad)            LJLIB_REC(math_degrad)
 LJLIB_ASM(math_atan2)           LJLIB_REC(.)
 {
diff --git a/src/lib_os.c b/src/lib_os.c
index 0a784129..de2bc623 100644
--- a/src/lib_os.c
+++ b/src/lib_os.c
@@ -18,7 +18,10 @@
 #include "lualib.h"
 #include "lj_obj.h"
+#include "lj_gc.h"
 #include "lj_err.h"
+#include "lj_buf.h"
+#include "lj_str.h"
 #include "lj_lib.h"
 #if LJ_TARGET_POSIX
@@ -185,7 +188,7 @@ LJLIB_CF(os_date)
 #endif
  }
  if (stm == NULL) {  /* Invalid date? */
-    setnilV(L->top-1);
+    setnilV(L->top++);
  } else if (strcmp(s, "*t") == 0) {
    lua_createtable(L, 0, 9);  /* 9 = number of fields */
    setfield(L, "sec", stm->tm_sec);
@@ -197,23 +200,25 @@ LJLIB_CF(os_date)
    setfield(L, "wday", stm->tm_wday+1);
    setfield(L, "yday", stm->tm_yday+1);
    setboolfield(L, "isdst", stm->tm_isdst);
-  } else {
+  } else if (*s) {
-    char cc[3];
+    SBuf *sb = &G(L)->tmpbuf;
-    luaL_Buffer b;
+    MSize sz = 0;
-    cc[0] = '%'; cc[2] = '\0';
+    const char *q;
-    luaL_buffinit(L, &b);
+    for (q = s; *q; q++)
-    for (; *s; s++) {
+      sz += (*q == '%') ? 30 : 1;  /* Overflow doesn't matter. */
-      if (*s != '%' || *(s + 1) == '\0') {  /* No conversion specifier? */
+    setsbufL(sb, L);
-        luaL_addchar(&b, *s);
+    for (;;) {
-      } else {
+      char *buf = lj_buf_need(sb, sz);
-        size_t reslen;
+      size_t len = strftime(buf, sbufsz(sb), s, stm);
-        char buff[200];  /* Should be big enough for any conversion result. */
+      if (len) {
-        cc[1] = *(++s);
+        setstrV(L, L->top++, lj_str_new(L, buf, len));
-        reslen = strftime(buff, sizeof(buff), cc, stm);
+        lj_gc_check(L);
-        luaL_addlstring(&b, buff, reslen);
+        break;
      }
+      sz += (sz|1);
    }
-    luaL_pushresult(&b);
+  } else {
+    setstrV(L, L->top++, &G(L)->strempty);
  }
  return 1;
 }
diff --git a/src/lib_string.c b/src/lib_string.c
index 9e8ab900..204f6975 100644
--- a/src/lib_string.c
+++ b/src/lib_string.c
@@ -6,8 +6,6 @@
 ** Copyright (C) 1994-2008 Lua.org, PUC-Rio. See Copyright Notice in lua.h
 */
-#include <stdio.h>
 #define lib_string_c
 #define LUA_LIB
@@ -18,6 +16,7 @@
 #include "lj_obj.h"
 #include "lj_gc.h"
 #include "lj_err.h"
+#include "lj_buf.h"
 #include "lj_str.h"
 #include "lj_tab.h"
 #include "lj_meta.h"
@@ -25,17 +24,19 @@
 #include "lj_ff.h"
 #include "lj_bcdump.h"
 #include "lj_char.h"
+#include "lj_strfmt.h"
 #include "lj_lib.h"
 /* ------------------------------------------------------------------------ */
 #define LJLIB_MODULE_string
-LJLIB_ASM(string_len)           LJLIB_REC(.)
+LJLIB_LUA(string_len) /*
-{
+  function(s)
-  lj_lib_checkstr(L, 1);
+    CHECK_str(s)
-  return FFH_RETRY;
+    return #s
-}
+  end
+*/
 LJLIB_ASM(string_byte)          LJLIB_REC(string_range 0)
 {
@@ -61,10 +62,10 @@ LJLIB_ASM(string_byte)		LJLIB_REC(string_range 0)
  return FFH_RES(n);
 }
-LJLIB_ASM(string_char)
+LJLIB_ASM(string_char)          LJLIB_REC(.)
 {
  int i, nargs = (int)(L->top - L->base);
-  char *buf = lj_str_needbuf(L, &G(L)->tmpbuf, (size_t)nargs);
+  char *buf = lj_buf_tmp(L, (size_t)nargs);
  for (i = 1; i <= nargs; i++) {
    int32_t k = lj_lib_checkint(L, i);
    if (!checku8(k))
@@ -83,68 +84,38 @@ LJLIB_ASM(string_sub)		LJLIB_REC(string_range 1)
  return FFH_RETRY;
 }
-LJLIB_ASM(string_rep)
+LJLIB_CF(string_rep)            LJLIB_REC(.)
 {
  GCstr *s = lj_lib_checkstr(L, 1);
-  int32_t k = lj_lib_checkint(L, 2);
+  int32_t rep = lj_lib_checkint(L, 2);
  GCstr *sep = lj_lib_optstr(L, 3);
-  int32_t len = (int32_t)s->len;
+  SBuf *sb = lj_buf_tmp_(L);
-  global_State *g = G(L);
+  if (sep && rep > 1) {
-  int64_t tlen;
+    GCstr *s2 = lj_buf_cat2str(L, sep, s);
-  const char *src;
+    lj_buf_reset(sb);
-  char *buf;
+    lj_buf_putstr(sb, s);
-  if (k <= 0) {
+    s = s2;
-  empty:
+    rep--;
-    setstrV(L, L->base-1, &g->strempty);
-    return FFH_RES(1);
  }
-  if (sep) {
+  sb = lj_buf_putstr_rep(sb, s, rep);
-    tlen = (int64_t)len + sep->len;
+  setstrV(L, L->top-1, lj_buf_str(L, sb));
-    if (tlen > LJ_MAX_STR)
+  lj_gc_check(L);
-      lj_err_caller(L, LJ_ERR_STROV);
+  return 1;
-    tlen *= k;
-    if (tlen > LJ_MAX_STR)
-      lj_err_caller(L, LJ_ERR_STROV);
-  } else {
-    tlen = (int64_t)k * len;
-    if (tlen > LJ_MAX_STR)
-      lj_err_caller(L, LJ_ERR_STROV);
-  }
-  if (tlen == 0) goto empty;
-  buf = lj_str_needbuf(L, &g->tmpbuf, (MSize)tlen);
-  src = strdata(s);
-  if (sep) {
-    tlen -= sep->len;  /* Ignore trailing separator. */
-    if (k > 1) {  /* Paste one string and one separator. */
-      int32_t i;
-      i = 0; while (i < len) *buf++ = src[i++];
-      src = strdata(sep); len = sep->len;
-      i = 0; while (i < len) *buf++ = src[i++];
-      src = g->tmpbuf.buf; len += s->len; k--;  /* Now copy that k-1 times. */
-    }
-  }
-  do {
-    int32_t i = 0;
-    do { *buf++ = src[i++]; } while (i < len);
-  } while (--k > 0);
-  setstrV(L, L->base-1, lj_str_new(L, g->tmpbuf.buf, (size_t)tlen));
-  return FFH_RES(1);
 }
-LJLIB_ASM(string_reverse)
+LJLIB_ASM(string_reverse)  LJLIB_REC(string_op IRCALL_lj_buf_putstr_reverse)
 {
-  GCstr *s = lj_lib_checkstr(L, 1);
+  lj_lib_checkstr(L, 1);
-  lj_str_needbuf(L, &G(L)->tmpbuf, s->len);
  return FFH_RETRY;
 }
-LJLIB_ASM_(string_lower)
+LJLIB_ASM_(string_lower)  LJLIB_REC(string_op IRCALL_lj_buf_putstr_lower)
-LJLIB_ASM_(string_upper)
+LJLIB_ASM_(string_upper)  LJLIB_REC(string_op IRCALL_lj_buf_putstr_upper)
 /* ------------------------------------------------------------------------ */
-static int writer_buf(lua_State *L, const void *p, size_t size, void *b)
+static int writer_buf(lua_State *L, const void *p, size_t size, void *sb)
 {
-  luaL_addlstring((luaL_Buffer *)b, (const char *)p, size);
+  lj_buf_putmem((SBuf *)sb, p, (MSize)size);
  UNUSED(L);
  return 0;
 }
@@ -153,12 +124,12 @@ LJLIB_CF(string_dump)
 {
  GCfunc *fn = lj_lib_checkfunc(L, 1);
  int strip = L->base+1 < L->top && tvistruecond(L->base+1);
-  luaL_Buffer b;
+  SBuf *sb = lj_buf_tmp_(L);  /* Assumes lj_bcwrite() doesn't use tmpbuf. */
  L->top = L->base+1;
-  luaL_buffinit(L, &b);
+  if (!isluafunc(fn) || lj_bcwrite(L, funcproto(fn), writer_buf, sb, strip))
-  if (!isluafunc(fn) || lj_bcwrite(L, funcproto(fn), writer_buf, &b, strip))
    lj_err_caller(L, LJ_ERR_STRDUMP);
-  luaL_pushresult(&b);
+  setstrV(L, L->top-1, lj_buf_str(L, sb));
+  lj_gc_check(L);
  return 1;
 }
@@ -183,7 +154,6 @@ typedef struct MatchState {
 } MatchState;
 #define L_ESC           '%'
-#define SPECIALS        "^$*+?.([%-"
 static int check_capture(MatchState *ms, int l)
 {
@@ -450,30 +420,6 @@ static const char *match(MatchState *ms, const char *s, const char *p)
  return s;
 }
-static const char *lmemfind(const char *s1, size_t l1,
-                            const char *s2, size_t l2)
-{
-  if (l2 == 0) {
-    return s1;  /* empty strings are everywhere */
-  } else if (l2 > l1) {
-    return NULL;  /* avoids a negative `l1' */
-  } else {
-    const char *init;  /* to search for a `*s2' inside `s1' */
-    l2--;  /* 1st char will be checked by `memchr' */
-    l1 = l1-l2;  /* `s2' cannot be found after that */
-    while (l1 > 0 && (init = (const char *)memchr(s1, *s2, l1)) != NULL) {
-      init++;   /* 1st char is already checked */
-      if (memcmp(init, s2+1, l2) == 0) {
-        return init-1;
-      } else {  /* correct `l1' and `s1' to try again */
-        l1 -= (size_t)(init-s1);
-        s1 = init;
-      }
-    }
-    return NULL;  /* not found */
-  }
-}
 static void push_onecapture(MatchState *ms, int i, const char *s, const char *e)
 {
  if (i >= ms->level) {
@@ -501,64 +447,60 @@ static int push_captures(MatchState *ms, const char *s, const char *e)
  return nlevels;  /* number of strings pushed */
 }
-static ptrdiff_t posrelat(ptrdiff_t pos, size_t len)
-{
-  /* relative string position: negative means back from end */
-  if (pos < 0) pos += (ptrdiff_t)len + 1;
-  return (pos >= 0) ? pos : 0;
-}
 static int str_find_aux(lua_State *L, int find)
 {
-  size_t l1, l2;
+  GCstr *s = lj_lib_checkstr(L, 1);
-  const char *s = luaL_checklstring(L, 1, &l1);
+  GCstr *p = lj_lib_checkstr(L, 2);
-  const char *p = luaL_checklstring(L, 2, &l2);
+  int32_t start = lj_lib_optint(L, 3, 1);
-  ptrdiff_t init = posrelat(luaL_optinteger(L, 3, 1), l1) - 1;
+  MSize st;
-  if (init < 0) {
+  if (start < 0) start += (int32_t)s->len; else start--;
-    init = 0;
+  if (start < 0) start = 0;
-  } else if ((size_t)(init) > l1) {
+  st = (MSize)start;
+  if (st > s->len) {
 #if LJ_52
    setnilV(L->top-1);
    return 1;
 #else
-    init = (ptrdiff_t)l1;
+    st = s->len;
 #endif
  }
-  if (find && (lua_toboolean(L, 4) ||  /* explicit request? */
+  if (find && ((L->base+3 < L->top && tvistruecond(L->base+3)) ||
-      strpbrk(p, SPECIALS) == NULL)) {  /* or no special characters? */
+               !lj_str_haspattern(p))) {  /* Search for fixed string. */
-    /* do a plain search */
+    const char *q = lj_str_find(strdata(s)+st, strdata(p), s->len-st, p->len);
-    const char *s2 = lmemfind(s+init, l1-(size_t)init, p, l2);
+    if (q) {
-    if (s2) {
+      setintV(L->top-2, (int32_t)(q-strdata(s)) + 1);
-      lua_pushinteger(L, s2-s+1);
+      setintV(L->top-1, (int32_t)(q-strdata(s)) + (int32_t)p->len);
-      lua_pushinteger(L, s2-s+(ptrdiff_t)l2);
      return 2;
    }
-  } else {
+  } else {  /* Search for pattern. */
    MatchState ms;
-    int anchor = (*p == '^') ? (p++, 1) : 0;
+    const char *pstr = strdata(p);
-    const char *s1=s+init;
+    const char *sstr = strdata(s) + st;
+    int anchor = 0;
+    if (*pstr == '^') { pstr++; anchor = 1; }
    ms.L = L;
-    ms.src_init = s;
+    ms.src_init = strdata(s);
-    ms.src_end = s+l1;
+    ms.src_end = strdata(s) + s->len;
-    do {
+    do {  /* Loop through string and try to match the pattern. */
-      const char *res;
+      const char *q;
      ms.level = ms.depth = 0;
-      if ((res=match(&ms, s1, p)) != NULL) {
+      q = match(&ms, sstr, pstr);
+      if (q) {
        if (find) {
-          lua_pushinteger(L, s1-s+1);  /* start */
+          setintV(L->top++, (int32_t)(sstr-(strdata(s)-1)));
-          lua_pushinteger(L, res-s);   /* end */
+          setintV(L->top++, (int32_t)(q-strdata(s)));
-          return push_captures(&ms, NULL, 0) + 2;
+          return push_captures(&ms, NULL, NULL) + 2;
        } else {
-          return push_captures(&ms, s1, res);
+          return push_captures(&ms, sstr, q);
        }
      }
-    } while (s1++ < ms.src_end && !anchor);
+    } while (sstr++ < ms.src_end && !anchor);
  }
-  lua_pushnil(L);  /* not found */
+  setnilV(L->top-1);  /* Not found. */
  return 1;
 }
-LJLIB_CF(string_find)
+LJLIB_CF(string_find)           LJLIB_REC(.)
 {
  return str_find_aux(L, 1);
 }
@@ -698,221 +640,92 @@ LJLIB_CF(string_gsub)
 /* ------------------------------------------------------------------------ */
-/* maximum size of each formatted item (> len(format('%99.99f', -1e308))) */
+/* Emulate tostring() inline. */
-#define MAX_FMTITEM     512
+static GCstr *string_fmt_tostring(lua_State *L, int arg, int retry)
-/* valid flags in a format specification */
-#define FMT_FLAGS       "-+ #0"
-/*
-** maximum size of each format specification (such as '%-099.99d')
-** (+10 accounts for %99.99x plus margin of error)
-*/
-#define MAX_FMTSPEC     (sizeof(FMT_FLAGS) + sizeof(LUA_INTFRMLEN) + 10)
-static void addquoted(lua_State *L, luaL_Buffer *b, int arg)
-{
-  GCstr *str = lj_lib_checkstr(L, arg);
-  int32_t len = (int32_t)str->len;
-  const char *s = strdata(str);
-  luaL_addchar(b, '"');
-  while (len--) {
-    uint32_t c = uchar(*s);
-    if (c == '"' || c == '\\' || c == '\n') {
-      luaL_addchar(b, '\\');
-    } else if (lj_char_iscntrl(c)) {  /* This can only be 0-31 or 127. */
-      uint32_t d;
-      luaL_addchar(b, '\\');
-      if (c >= 100 || lj_char_isdigit(uchar(s[1]))) {
-        luaL_addchar(b, '0'+(c >= 100)); if (c >= 100) c -= 100;
-        goto tens;
-      } else if (c >= 10) {
-      tens:
-        d = (c * 205) >> 11; c -= d * 10; luaL_addchar(b, '0'+d);
-      }
-      c += '0';
-    }
-    luaL_addchar(b, c);
-    s++;
-  }
-  luaL_addchar(b, '"');
-}
-static const char *scanformat(lua_State *L, const char *strfrmt, char *form)
-{
-  const char *p = strfrmt;
-  while (*p != '\0' && strchr(FMT_FLAGS, *p) != NULL) p++;  /* skip flags */
-  if ((size_t)(p - strfrmt) >= sizeof(FMT_FLAGS))
-    lj_err_caller(L, LJ_ERR_STRFMTR);
-  if (lj_char_isdigit(uchar(*p))) p++;  /* skip width */
-  if (lj_char_isdigit(uchar(*p))) p++;  /* (2 digits at most) */
-  if (*p == '.') {
-    p++;
-    if (lj_char_isdigit(uchar(*p))) p++;  /* skip precision */
-    if (lj_char_isdigit(uchar(*p))) p++;  /* (2 digits at most) */
-  }
-  if (lj_char_isdigit(uchar(*p)))
-    lj_err_caller(L, LJ_ERR_STRFMTW);
-  *(form++) = '%';
-  strncpy(form, strfrmt, (size_t)(p - strfrmt + 1));
-  form += p - strfrmt + 1;
-  *form = '\0';
-  return p;
-}
-static void addintlen(char *form)
-{
-  size_t l = strlen(form);
-  char spec = form[l - 1];
-  strcpy(form + l - 1, LUA_INTFRMLEN);
-  form[l + sizeof(LUA_INTFRMLEN) - 2] = spec;
-  form[l + sizeof(LUA_INTFRMLEN) - 1] = '\0';
-}
-static unsigned LUA_INTFRM_T num2intfrm(lua_State *L, int arg)
-{
-  if (sizeof(LUA_INTFRM_T) == 4) {
-    return (LUA_INTFRM_T)lj_lib_checkbit(L, arg);
-  } else {
-    cTValue *o;
-    lj_lib_checknumber(L, arg);
-    o = L->base+arg-1;
-    if (tvisint(o))
-      return (LUA_INTFRM_T)intV(o);
-    else
-      return (LUA_INTFRM_T)numV(o);
-  }
-}
-static unsigned LUA_INTFRM_T num2uintfrm(lua_State *L, int arg)
-{
-  if (sizeof(LUA_INTFRM_T) == 4) {
-    return (unsigned LUA_INTFRM_T)lj_lib_checkbit(L, arg);
-  } else {
-    cTValue *o;
-    lj_lib_checknumber(L, arg);
-    o = L->base+arg-1;
-    if (tvisint(o))
-      return (unsigned LUA_INTFRM_T)intV(o);
-    else if ((int32_t)o->u32.hi < 0)
-      return (unsigned LUA_INTFRM_T)(LUA_INTFRM_T)numV(o);
-    else
-      return (unsigned LUA_INTFRM_T)numV(o);
-  }
-}
-static GCstr *meta_tostring(lua_State *L, int arg)
 {
  TValue *o = L->base+arg-1;
  cTValue *mo;
  lua_assert(o < L->top);  /* Caller already checks for existence. */
  if (LJ_LIKELY(tvisstr(o)))
    return strV(o);
-  if (!tvisnil(mo = lj_meta_lookup(L, o, MM_tostring))) {
+  if (retry != 2 && !tvisnil(mo = lj_meta_lookup(L, o, MM_tostring))) {
    copyTV(L, L->top++, mo);
    copyTV(L, L->top++, o);
    lua_call(L, 1, 1);
-    L->top--;
+    copyTV(L, L->base+arg-1, --L->top);
-    if (tvisstr(L->top))
+    return NULL;  /* Buffer may be overwritten, retry. */
-      return strV(L->top);
-    o = L->base+arg-1;
-    copyTV(L, o, L->top);
-  }
-  if (tvisnumber(o)) {
-    return lj_str_fromnumber(L, o);
-  } else if (tvisnil(o)) {
-    return lj_str_newlit(L, "nil");
-  } else if (tvisfalse(o)) {
-    return lj_str_newlit(L, "false");
-  } else if (tvistrue(o)) {
-    return lj_str_newlit(L, "true");
-  } else {
-    if (tvisfunc(o) && isffunc(funcV(o)))
-      lj_str_pushf(L, "function: builtin#%d", funcV(o)->c.ffid);
-    else
-      lj_str_pushf(L, "%s: %p", lj_typename(o), lua_topointer(L, arg));
-    L->top--;
-    return strV(L->top);
  }
-}
+  return lj_strfmt_obj(L, o);
+}
-LJLIB_CF(string_format)
-{
+LJLIB_CF(string_format)         LJLIB_REC(.)
-  int arg = 1, top = (int)(L->top - L->base);
+{
-  GCstr *fmt = lj_lib_checkstr(L, arg);
+  int arg, top = (int)(L->top - L->base);
-  const char *strfrmt = strdata(fmt);
+  GCstr *fmt;
-  const char *strfrmt_end = strfrmt + fmt->len;
+  SBuf *sb;
-  luaL_Buffer b;
+  FormatState fs;
-  luaL_buffinit(L, &b);
+  SFormat sf;
-  while (strfrmt < strfrmt_end) {
+  int retry = 0;
-    if (*strfrmt != L_ESC) {
+again:
-      luaL_addchar(&b, *strfrmt++);
+  arg = 1;
-    } else if (*++strfrmt == L_ESC) {
+  sb = lj_buf_tmp_(L);
-      luaL_addchar(&b, *strfrmt++);  /* %% */
+  fmt = lj_lib_checkstr(L, arg);
-    } else { /* format item */
+  lj_strfmt_init(&fs, strdata(fmt), fmt->len);
-      char form[MAX_FMTSPEC];  /* to store the format (`%...') */
+  while ((sf = lj_strfmt_parse(&fs)) != STRFMT_EOF) {
-      char buff[MAX_FMTITEM];  /* to store the formatted item */
+    if (sf == STRFMT_LIT) {
+      lj_buf_putmem(sb, fs.str, fs.len);
+    } else if (sf == STRFMT_ERR) {
+      lj_err_callerv(L, LJ_ERR_STRFMT, strdata(lj_str_new(L, fs.str, fs.len)));
+    } else {
      if (++arg > top)
        luaL_argerror(L, arg, lj_obj_typename[0]);
-      strfrmt = scanformat(L, strfrmt, form);
+      switch (STRFMT_TYPE(sf)) {
-      switch (*strfrmt++) {
+      case STRFMT_INT:
-      case 'c':
+        if (tvisint(L->base+arg-1)) {
-        sprintf(buff, form, lj_lib_checkint(L, arg));
+          int32_t k = intV(L->base+arg-1);
+          if (sf == STRFMT_INT)
+            lj_strfmt_putint(sb, k);  /* Shortcut for plain %d. */
+          else
+            lj_strfmt_putfxint(sb, sf, k);
+        } else {
+          lj_strfmt_putfnum_int(sb, sf, lj_lib_checknum(L, arg));
+        }
        break;
-      case 'd':  case 'i':
+      case STRFMT_UINT:
-        addintlen(form);
+        if (tvisint(L->base+arg-1))
-        sprintf(buff, form, num2intfrm(L, arg));
+          lj_strfmt_putfxint(sb, sf, intV(L->base+arg-1));
+        else
+          lj_strfmt_putfnum_uint(sb, sf, lj_lib_checknum(L, arg));
        break;
-      case 'o':  case 'u':  case 'x':  case 'X':
+      case STRFMT_NUM:
-        addintlen(form);
+        lj_strfmt_putfnum(sb, sf, lj_lib_checknum(L, arg));
-        sprintf(buff, form, num2uintfrm(L, arg));
        break;
-      case 'e':  case 'E': case 'f': case 'g': case 'G': case 'a': case 'A': {
+      case STRFMT_STR: {
-        TValue tv;
+        GCstr *str = string_fmt_tostring(L, arg, retry);
-        tv.n = lj_lib_checknum(L, arg);
+        if (str == NULL)
-        if (LJ_UNLIKELY((tv.u32.hi << 1) >= 0xffe00000)) {
+          retry = 1;
-          /* Canonicalize output of non-finite values. */
+        else if ((sf & STRFMT_T_QUOTED))
-          char *p, nbuf[LJ_STR_NUMBUF];
+          lj_strfmt_putquoted(sb, str);  /* No formatting. */
-          size_t len = lj_str_bufnum(nbuf, &tv);
+        else
-          if (strfrmt[-1] < 'a') {
+          lj_strfmt_putfstr(sb, sf, str);
-            nbuf[len-3] = nbuf[len-3] - 0x20;
-            nbuf[len-2] = nbuf[len-2] - 0x20;
-            nbuf[len-1] = nbuf[len-1] - 0x20;
-          }
-          nbuf[len] = '\0';
-          for (p = form; *p < 'A' && *p != '.'; p++) ;
-          *p++ = 's'; *p = '\0';
-          sprintf(buff, form, nbuf);
-          break;
-        }
-        sprintf(buff, form, (double)tv.n);
        break;
        }
-      case 'q':
+      case STRFMT_CHAR:
-        addquoted(L, &b, arg);
+        lj_strfmt_putfchar(sb, sf, lj_lib_checkint(L, arg));
-        continue;
+        break;
-      case 'p':
+      case STRFMT_PTR:  /* No formatting. */
-        lj_str_pushf(L, "%p", lua_topointer(L, arg));
+        setsbufP(sb, lj_strfmt_wptr(lj_buf_more(sb, STRFMT_MAXBUF_PTR),
-        luaL_addvalue(&b);
+                                    lj_obj_ptr(L->base+arg-1)));
-        continue;
-      case 's': {
-        GCstr *str = meta_tostring(L, arg);
-        if (!strchr(form, '.') && str->len >= 100) {
-          /* no precision and string is too long to be formatted;
-             keep original string */
-          setstrV(L, L->top++, str);
-          luaL_addvalue(&b);
-          continue;
-        }
-        sprintf(buff, form, strdata(str));
        break;
-        }
      default:
-        lj_err_callerv(L, LJ_ERR_STRFMTO, *(strfrmt -1));
+        lua_assert(0);
        break;
      }
-      luaL_addlstring(&b, buff, strlen(buff));
    }
  }
-  luaL_pushresult(&b);
+  if (retry++ == 1) goto again;
+  setstrV(L, L->top-1, lj_buf_str(L, sb));
+  lj_gc_check(L);
  return 1;
 }
diff --git a/src/lib_table.c b/src/lib_table.c
index 542ed1f8..d7df8399 100644
--- a/src/lib_table.c
+++ b/src/lib_table.c
@@ -16,6 +16,7 @@
 #include "lj_obj.h"
 #include "lj_gc.h"
 #include "lj_err.h"
+#include "lj_buf.h"
 #include "lj_tab.h"
 #include "lj_lib.h"
@@ -23,50 +24,34 @@
 #define LJLIB_MODULE_table
-LJLIB_CF(table_foreachi)
+LJLIB_LUA(table_foreachi) /*
-{
+  function(t, f)
-  GCtab *t = lj_lib_checktab(L, 1);
+    CHECK_tab(t)
-  GCfunc *func = lj_lib_checkfunc(L, 2);
+    CHECK_func(f)
-  MSize i, n = lj_tab_len(t);
+    for i=1,#t do
-  for (i = 1; i <= n; i++) {
+      local r = f(i, t[i])
-    cTValue *val;
+      if r ~= nil then return r end
-    setfuncV(L, L->top, func);
+    end
-    setintV(L->top+1, i);
+  end
-    val = lj_tab_getint(t, (int32_t)i);
+*/
-    if (val) { copyTV(L, L->top+2, val); } else { setnilV(L->top+2); }
-    L->top += 3;
-    lua_call(L, 2, 1);
-    if (!tvisnil(L->top-1))
-      return 1;
-    L->top--;
-  }
-  return 0;
-}
-LJLIB_CF(table_foreach)
+LJLIB_LUA(table_foreach) /*
-{
+  function(t, f)
-  GCtab *t = lj_lib_checktab(L, 1);
+    CHECK_tab(t)
-  GCfunc *func = lj_lib_checkfunc(L, 2);
+    CHECK_func(f)
-  L->top = L->base+3;
+    for k, v in PAIRS(t) do
-  setnilV(L->top-1);
+      local r = f(k, v)
-  while (lj_tab_next(L, t, L->top-1)) {
+      if r ~= nil then return r end
-    copyTV(L, L->top+2, L->top);
+    end
-    copyTV(L, L->top+1, L->top-1);
+  end
-    setfuncV(L, L->top, func);
+*/
-    L->top += 3;
-    lua_call(L, 2, 1);
-    if (!tvisnil(L->top-1))
-      return 1;
-    L->top--;
-  }
-  return 0;
-}
-LJLIB_ASM(table_getn)           LJLIB_REC(.)
+LJLIB_LUA(table_getn) /*
-{
+  function(t)
-  lj_lib_checktab(L, 1);
+    CHECK_tab(t)
-  return FFH_UNREACHABLE;
+    return #t
-}
+  end
+*/
 LJLIB_CF(table_maxn)
 {
@@ -119,52 +104,47 @@ LJLIB_CF(table_insert)		LJLIB_REC(.)
  return 0;
 }
-LJLIB_CF(table_remove)          LJLIB_REC(.)
+LJLIB_LUA(table_remove) /*
-{
+  function(t, pos)
-  GCtab *t = lj_lib_checktab(L, 1);
+    CHECK_tab(t)
-  int32_t e = (int32_t)lj_tab_len(t);
+    local len = #t
-  int32_t pos = lj_lib_optint(L, 2, e);
+    if pos == nil then
-  if (!(1 <= pos && pos <= e))  /* Nothing to remove? */
+      if len ~= 0 then
-    return 0;
+        local old = t[len]
-  lua_rawgeti(L, 1, pos);  /* Get previous value. */
+        t[len] = nil
-  /* NOBARRIER: This just moves existing elements around. */
+        return old
-  for (; pos < e; pos++) {
+      end
-    cTValue *src = lj_tab_getint(t, pos+1);
+    else
-    TValue *dst = lj_tab_setint(L, t, pos);
+      CHECK_int(pos)
-    if (src) {
+      if pos >= 1 and pos <= len then
-      copyTV(L, dst, src);
+        local old = t[pos]
-    } else {
+        for i=pos+1,len do
-      setnilV(dst);
+          t[i-1] = t[i]
-    }
+        end
-  }
+        t[len] = nil
-  setnilV(lj_tab_setint(L, t, e));  /* Remove (last) value. */
+        return old
-  return 1;  /* Return previous value. */
+      end
-}
+    end
+  end
+*/
-LJLIB_CF(table_concat)
+LJLIB_CF(table_concat)          LJLIB_REC(.)
 {
-  luaL_Buffer b;
  GCtab *t = lj_lib_checktab(L, 1);
  GCstr *sep = lj_lib_optstr(L, 2);
-  MSize seplen = sep ? sep->len : 0;
  int32_t i = lj_lib_optint(L, 3, 1);
  int32_t e = (L->base+3 < L->top && !tvisnil(L->base+3)) ?
              lj_lib_checkint(L, 4) : (int32_t)lj_tab_len(t);
-  luaL_buffinit(L, &b);
+  SBuf *sb = lj_buf_tmp_(L);
-  if (i <= e) {
+  SBuf *sbx = lj_buf_puttab(sb, t, sep, i, e);
-    for (;;) {
+  if (LJ_UNLIKELY(!sbx)) {  /* Error: bad element type. */
-      cTValue *o;
+    int32_t idx = (int32_t)(intptr_t)sbufP(sb);
-      lua_rawgeti(L, 1, i);
+    cTValue *o = lj_tab_getint(t, idx);
-      o = L->top-1;
+    lj_err_callerv(L, LJ_ERR_TABCAT,
-      if (!(tvisstr(o) || tvisnumber(o)))
+                   lj_obj_itypename[o ? itypemap(o) : ~LJ_TNIL], idx);
-        lj_err_callerv(L, LJ_ERR_TABCAT, lj_typename(o), i);
-      luaL_addvalue(&b);
-      if (i++ == e) break;
-      if (seplen)
-        luaL_addlstring(&b, strdata(sep), seplen);
-    }
  }
-  luaL_pushresult(&b);
+  setstrV(L, L->top-1, lj_buf_str(L, sbx));
+  lj_gc_check(L);
  return 1;
 }
diff --git a/src/lj_api.c b/src/lj_api.c
index edb2d620..8aaafdc5 100644
--- a/src/lj_api.c
+++ b/src/lj_api.c
@@ -24,6 +24,7 @@
 #include "lj_trace.h"
 #include "lj_vm.h"
 #include "lj_strscan.h"
+#include "lj_strfmt.h"
 /* -- Common helper functions --------------------------------------------- */
@@ -434,7 +435,7 @@ LUA_API const char *lua_tolstring(lua_State *L, int idx, size_t *len)
  } else if (tvisnumber(o)) {
    lj_gc_check(L);
    o = index2adr(L, idx);  /* GC may move the stack. */
-    s = lj_str_fromnumber(L, o);
+    s = lj_strfmt_number(L, o);
    setstrV(L, o, s);
  } else {
    if (len != NULL) *len = 0;
@@ -453,7 +454,7 @@ LUALIB_API const char *luaL_checklstring(lua_State *L, int idx, size_t *len)
  } else if (tvisnumber(o)) {
    lj_gc_check(L);
    o = index2adr(L, idx);  /* GC may move the stack. */
-    s = lj_str_fromnumber(L, o);
+    s = lj_strfmt_number(L, o);
    setstrV(L, o, s);
  } else {
    lj_err_argt(L, idx, LUA_TSTRING);
@@ -475,7 +476,7 @@ LUALIB_API const char *luaL_optlstring(lua_State *L, int idx,
  } else if (tvisnumber(o)) {
    lj_gc_check(L);
    o = index2adr(L, idx);  /* GC may move the stack. */
-    s = lj_str_fromnumber(L, o);
+    s = lj_strfmt_number(L, o);
    setstrV(L, o, s);
  } else {
    lj_err_argt(L, idx, LUA_TSTRING);
@@ -507,7 +508,7 @@ LUA_API size_t lua_objlen(lua_State *L, int idx)
  } else if (tvisudata(o)) {
    return udataV(o)->len;
  } else if (tvisnumber(o)) {
-    GCstr *s = lj_str_fromnumber(L, o);
+    GCstr *s = lj_strfmt_number(L, o);
    setstrV(L, o, s);
    return s->len;
  } else {
@@ -545,17 +546,7 @@ LUA_API lua_State *lua_tothread(lua_State *L, int idx)
 LUA_API const void *lua_topointer(lua_State *L, int idx)
 {
-  cTValue *o = index2adr(L, idx);
+  return lj_obj_ptr(index2adr(L, idx));
-  if (tvisudata(o))
-    return uddata(udataV(o));
-  else if (tvislightud(o))
-    return lightudV(o);
-  else if (tviscdata(o))
-    return cdataptr(cdataV(o));
-  else if (tvisgcv(o))
-    return gcV(o);
-  else
-    return NULL;
 }
 /* -- Stack setters (object creation) ------------------------------------- */
@@ -606,7 +597,7 @@ LUA_API const char *lua_pushvfstring(lua_State *L, const char *fmt,
                                     va_list argp)
 {
  lj_gc_check(L);
-  return lj_str_pushvf(L, fmt, argp);
+  return lj_strfmt_pushvf(L, fmt, argp);
 }
 LUA_API const char *lua_pushfstring(lua_State *L, const char *fmt, ...)
@@ -615,7 +606,7 @@ LUA_API const char *lua_pushfstring(lua_State *L, const char *fmt, ...)
  va_list argp;
  lj_gc_check(L);
  va_start(argp, fmt);
-  ret = lj_str_pushvf(L, fmt, argp);
+  ret = lj_strfmt_pushvf(L, fmt, argp);
  va_end(argp);
  return ret;
 }
diff --git a/src/lj_arch.h b/src/lj_arch.h
index 9ea10d0f..c5f2fb3d 100644
--- a/src/lj_arch.h
+++ b/src/lj_arch.h
@@ -227,6 +227,7 @@
 #elif LUAJIT_TARGET == LUAJIT_ARCH_PPCSPE
+#error "The PPC/e500 port is broken and will be abandoned with LuaJIT 2.1"
 #define LJ_ARCH_NAME            "ppcspe"
 #define LJ_ARCH_BITS            32
 #define LJ_ARCH_ENDIAN          LUAJIT_BE
diff --git a/src/lj_asm.c b/src/lj_asm.c
index 316e81d6..a80d6adf 100644
--- a/src/lj_asm.c
+++ b/src/lj_asm.c
@@ -179,6 +179,12 @@ IRFLDEF(FLOFS)
 #error "Missing instruction emitter for target CPU"
 #endif
+/* Generic load/store of register from/to stack slot. */
+#define emit_spload(as, ir, r, ofs) \
+  emit_loadofs(as, ir, (r), RID_SP, (ofs))
+#define emit_spstore(as, ir, r, ofs) \
+  emit_storeofs(as, ir, (r), RID_SP, (ofs))
 /* -- Register allocator debugging ---------------------------------------- */
 /* #define LUAJIT_DEBUG_RA */
@@ -943,44 +949,6 @@ static void asm_snap_prep(ASMState *as)
 /* -- Miscellaneous helpers ----------------------------------------------- */
-/* Collect arguments from CALL* and CARG instructions. */
-static void asm_collectargs(ASMState *as, IRIns *ir,
-                            const CCallInfo *ci, IRRef *args)
-{
-  uint32_t n = CCI_NARGS(ci);
-  lua_assert(n <= CCI_NARGS_MAX*2);  /* Account for split args. */
-  if ((ci->flags & CCI_L)) { *args++ = ASMREF_L; n--; }
-  while (n-- > 1) {
-    ir = IR(ir->op1);
-    lua_assert(ir->o == IR_CARG);
-    args[n] = ir->op2 == REF_NIL ? 0 : ir->op2;
-  }
-  args[0] = ir->op1 == REF_NIL ? 0 : ir->op1;
-  lua_assert(IR(ir->op1)->o != IR_CARG);
-}
-/* Reconstruct CCallInfo flags for CALLX*. */
-static uint32_t asm_callx_flags(ASMState *as, IRIns *ir)
-{
-  uint32_t nargs = 0;
-  if (ir->op1 != REF_NIL) {  /* Count number of arguments first. */
-    IRIns *ira = IR(ir->op1);
-    nargs++;
-    while (ira->o == IR_CARG) { nargs++; ira = IR(ira->op1); }
-  }
-#if LJ_HASFFI
-  if (IR(ir->op2)->o == IR_CARG) {  /* Copy calling convention info. */
-    CTypeID id = (CTypeID)IR(IR(ir->op2)->op2)->i;
-    CType *ct = ctype_get(ctype_ctsG(J2G(as->J)), id);
-    nargs |= ((ct->info & CTF_VARARG) ? CCI_VARARG : 0);
-#if LJ_TARGET_X86
-    nargs |= (ctype_cconv(ct->info) << CCI_CC_SHIFT);
-#endif
-  }
-#endif
-  return (nargs | (ir->t.irt << CCI_OTSHIFT));
-}
 /* Calculate stack adjustment. */
 static int32_t asm_stack_adjust(ASMState *as)
 {
@@ -1065,6 +1033,253 @@ static void asm_gcstep(ASMState *as, IRIns *ir)
  as->gcsteps = 0x80000000;  /* Prevent implicit GC check further up. */
 }
+/* -- Buffer operations --------------------------------------------------- */
+static void asm_tvptr(ASMState *as, Reg dest, IRRef ref);
+static void asm_bufhdr(ASMState *as, IRIns *ir)
+{
+  Reg sb = ra_dest(as, ir, RSET_GPR);
+  if ((ir->op2 & IRBUFHDR_APPEND)) {
+    /* Rematerialize const buffer pointer instead of likely spill. */
+    IRIns *irp = IR(ir->op1);
+    if (!(ra_hasreg(irp->r) || irp == ir-1 ||
+          (irp == ir-2 && !ra_used(ir-1)))) {
+      while (!(irp->o == IR_BUFHDR && !(irp->op2 & IRBUFHDR_APPEND)))
+        irp = IR(irp->op1);
+      if (irref_isk(irp->op1)) {
+        ra_weak(as, ra_allocref(as, ir->op1, RSET_GPR));
+        ir = irp;
+      }
+    }
+  } else {
+    Reg tmp = ra_scratch(as, rset_exclude(RSET_GPR, sb));
+    /* Passing ir isn't strictly correct, but it's an IRT_P32, too. */
+    emit_storeofs(as, ir, tmp, sb, offsetof(SBuf, p));
+    emit_loadofs(as, ir, tmp, sb, offsetof(SBuf, b));
+  }
+#if LJ_TARGET_X86ORX64
+  ra_left(as, sb, ir->op1);
+#else
+  ra_leftov(as, sb, ir->op1);
+#endif
+}
+static void asm_bufput(ASMState *as, IRIns *ir)
+{
+  const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_buf_putstr];
+  IRRef args[3];
+  IRIns *irs;
+  int kchar = -1;
+  args[0] = ir->op1;  /* SBuf * */
+  args[1] = ir->op2;  /* GCstr * */
+  irs = IR(ir->op2);
+  lua_assert(irt_isstr(irs->t));
+  if (irs->o == IR_KGC) {
+    GCstr *s = ir_kstr(irs);
+    if (s->len == 1) {  /* Optimize put of single-char string constant. */
+      kchar = strdata(s)[0];
+      args[1] = ASMREF_TMP1;  /* int, truncated to char */
+      ci = &lj_ir_callinfo[IRCALL_lj_buf_putchar];
+    }
+  } else if (mayfuse(as, ir->op2) && ra_noreg(irs->r)) {
+    if (irs->o == IR_TOSTR) {  /* Fuse number to string conversions. */
+      if (irs->op2 == IRTOSTR_NUM) {
+        args[1] = ASMREF_TMP1;  /* TValue * */
+        ci = &lj_ir_callinfo[IRCALL_lj_strfmt_putnum];
+      } else {
+        lua_assert(irt_isinteger(IR(irs->op1)->t));
+        args[1] = irs->op1;  /* int */
+        if (irs->op2 == IRTOSTR_INT)
+          ci = &lj_ir_callinfo[IRCALL_lj_strfmt_putint];
+        else
+          ci = &lj_ir_callinfo[IRCALL_lj_buf_putchar];
+      }
+    } else if (irs->o == IR_SNEW) {  /* Fuse string allocation. */
+      args[1] = irs->op1;  /* const void * */
+      args[2] = irs->op2;  /* MSize */
+      ci = &lj_ir_callinfo[IRCALL_lj_buf_putmem];
+    }
+  }
+  asm_setupresult(as, ir, ci);  /* SBuf * */
+  asm_gencall(as, ci, args);
+  if (args[1] == ASMREF_TMP1) {
+    Reg tmp = ra_releasetmp(as, ASMREF_TMP1);
+    if (kchar == -1)
+      asm_tvptr(as, tmp, irs->op1);
+    else
+      ra_allockreg(as, kchar, tmp);
+  }
+}
+static void asm_bufstr(ASMState *as, IRIns *ir)
+{
+  const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_buf_tostr];
+  IRRef args[1];
+  args[0] = ir->op1;  /* SBuf *sb */
+  as->gcsteps++;
+  asm_setupresult(as, ir, ci);  /* GCstr * */
+  asm_gencall(as, ci, args);
+}
+/* -- Type conversions ---------------------------------------------------- */
+static void asm_tostr(ASMState *as, IRIns *ir)
+{
+  const CCallInfo *ci;
+  IRRef args[2];
+  args[0] = ASMREF_L;
+  as->gcsteps++;
+  if (ir->op2 == IRTOSTR_NUM) {
+    args[1] = ASMREF_TMP1;  /* cTValue * */
+    ci = &lj_ir_callinfo[IRCALL_lj_strfmt_num];
+  } else {
+    args[1] = ir->op1;  /* int32_t k */
+    if (ir->op2 == IRTOSTR_INT)
+      ci = &lj_ir_callinfo[IRCALL_lj_strfmt_int];
+    else
+      ci = &lj_ir_callinfo[IRCALL_lj_strfmt_char];
+  }
+  asm_setupresult(as, ir, ci);  /* GCstr * */
+  asm_gencall(as, ci, args);
+  if (ir->op2 == IRTOSTR_NUM)
+    asm_tvptr(as, ra_releasetmp(as, ASMREF_TMP1), ir->op1);
+}
+#if LJ_32 && LJ_HASFFI && !LJ_SOFTFP && !LJ_TARGET_X86
+static void asm_conv64(ASMState *as, IRIns *ir)
+{
+  IRType st = (IRType)((ir-1)->op2 & IRCONV_SRCMASK);
+  IRType dt = (((ir-1)->op2 & IRCONV_DSTMASK) >> IRCONV_DSH);
+  IRCallID id;
+  IRRef args[2];
+  lua_assert((ir-1)->o == IR_CONV && ir->o == IR_HIOP);
+  args[LJ_BE] = (ir-1)->op1;
+  args[LJ_LE] = ir->op1;
+  if (st == IRT_NUM || st == IRT_FLOAT) {
+    id = IRCALL_fp64_d2l + ((st == IRT_FLOAT) ? 2 : 0) + (dt - IRT_I64);
+    ir--;
+  } else {
+    id = IRCALL_fp64_l2d + ((dt == IRT_FLOAT) ? 2 : 0) + (st - IRT_I64);
+  }
+  {
+#if LJ_TARGET_ARM && !LJ_ABI_SOFTFP
+    CCallInfo cim = lj_ir_callinfo[id], *ci = &cim;
+    cim.flags |= CCI_VARARG;  /* These calls don't use the hard-float ABI! */
+#else
+    const CCallInfo *ci = &lj_ir_callinfo[id];
+#endif
+    asm_setupresult(as, ir, ci);
+    asm_gencall(as, ci, args);
+  }
+}
+#endif
+/* -- Memory references --------------------------------------------------- */
+static void asm_newref(ASMState *as, IRIns *ir)
+{
+  const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_tab_newkey];
+  IRRef args[3];
+  if (ir->r == RID_SINK)
+    return;
+  args[0] = ASMREF_L;     /* lua_State *L */
+  args[1] = ir->op1;      /* GCtab *t     */
+  args[2] = ASMREF_TMP1;  /* cTValue *key */
+  asm_setupresult(as, ir, ci);  /* TValue * */
+  asm_gencall(as, ci, args);
+  asm_tvptr(as, ra_releasetmp(as, ASMREF_TMP1), ir->op2);
+}
+/* -- Calls --------------------------------------------------------------- */
+/* Collect arguments from CALL* and CARG instructions. */
+static void asm_collectargs(ASMState *as, IRIns *ir,
+                            const CCallInfo *ci, IRRef *args)
+{
+  uint32_t n = CCI_XNARGS(ci);
+  lua_assert(n <= CCI_NARGS_MAX*2);  /* Account for split args. */
+  if ((ci->flags & CCI_L)) { *args++ = ASMREF_L; n--; }
+  while (n-- > 1) {
+    ir = IR(ir->op1);
+    lua_assert(ir->o == IR_CARG);
+    args[n] = ir->op2 == REF_NIL ? 0 : ir->op2;
+  }
+  args[0] = ir->op1 == REF_NIL ? 0 : ir->op1;
+  lua_assert(IR(ir->op1)->o != IR_CARG);
+}
+/* Reconstruct CCallInfo flags for CALLX*. */
+static uint32_t asm_callx_flags(ASMState *as, IRIns *ir)
+{
+  uint32_t nargs = 0;
+  if (ir->op1 != REF_NIL) {  /* Count number of arguments first. */
+    IRIns *ira = IR(ir->op1);
+    nargs++;
+    while (ira->o == IR_CARG) { nargs++; ira = IR(ira->op1); }
+  }
+#if LJ_HASFFI
+  if (IR(ir->op2)->o == IR_CARG) {  /* Copy calling convention info. */
+    CTypeID id = (CTypeID)IR(IR(ir->op2)->op2)->i;
+    CType *ct = ctype_get(ctype_ctsG(J2G(as->J)), id);
+    nargs |= ((ct->info & CTF_VARARG) ? CCI_VARARG : 0);
+#if LJ_TARGET_X86
+    nargs |= (ctype_cconv(ct->info) << CCI_CC_SHIFT);
+#endif
+  }
+#endif
+  return (nargs | (ir->t.irt << CCI_OTSHIFT));
+}
+static void asm_callid(ASMState *as, IRIns *ir, IRCallID id)
+{
+  const CCallInfo *ci = &lj_ir_callinfo[id];
+  IRRef args[2];
+  args[0] = ir->op1;
+  args[1] = ir->op2;
+  asm_setupresult(as, ir, ci);
+  asm_gencall(as, ci, args);
+}
+static void asm_call(ASMState *as, IRIns *ir)
+{
+  IRRef args[CCI_NARGS_MAX];
+  const CCallInfo *ci = &lj_ir_callinfo[ir->op2];
+  asm_collectargs(as, ir, ci, args);
+  asm_setupresult(as, ir, ci);
+  asm_gencall(as, ci, args);
+}
+#if !LJ_SOFTFP
+static void asm_fppow(ASMState *as, IRIns *ir, IRRef lref, IRRef rref);
+#if !LJ_TARGET_X86ORX64
+static void asm_fppow(ASMState *as, IRIns *ir, IRRef lref, IRRef rref)
+{
+  const CCallInfo *ci = &lj_ir_callinfo[IRCALL_pow];
+  IRRef args[2];
+  args[0] = lref;
+  args[1] = rref;
+  asm_setupresult(as, ir, ci);
+  asm_gencall(as, ci, args);
+}
+#endif
+static int asm_fpjoin_pow(ASMState *as, IRIns *ir)
+{
+  IRIns *irp = IR(ir->op1);
+  if (irp == ir-1 && irp->o == IR_MUL && !ra_used(irp)) {
+    IRIns *irpp = IR(irp->op1);
+    if (irpp == ir-2 && irpp->o == IR_FPMATH &&
+        irpp->op2 == IRFPM_LOG2 && !ra_used(irpp)) {
+      asm_fppow(as, ir, irpp->op1, irp->op2);
+      return 1;
+    }
+  }
+  return 0;
+}
+#endif
 /* -- PHI and loop handling ----------------------------------------------- */
 /* Break a PHI cycle by renaming to a free register (evict if needed). */
@@ -1336,6 +1551,124 @@ static void asm_loop(ASMState *as)
 #error "Missing assembler for target CPU"
 #endif
+/* -- Instruction dispatch ------------------------------------------------ */
+/* Assemble a single instruction. */
+static void asm_ir(ASMState *as, IRIns *ir)
+{
+  switch ((IROp)ir->o) {
+  /* Miscellaneous ops. */
+  case IR_LOOP: asm_loop(as); break;
+  case IR_NOP: case IR_XBAR: lua_assert(!ra_used(ir)); break;
+  case IR_USE:
+    ra_alloc1(as, ir->op1, irt_isfp(ir->t) ? RSET_FPR : RSET_GPR); break;
+  case IR_PHI: asm_phi(as, ir); break;
+  case IR_HIOP: asm_hiop(as, ir); break;
+  case IR_GCSTEP: asm_gcstep(as, ir); break;
+  /* Guarded assertions. */
+  case IR_LT: case IR_GE: case IR_LE: case IR_GT:
+  case IR_ULT: case IR_UGE: case IR_ULE: case IR_UGT:
+  case IR_ABC:
+    asm_comp(as, ir);
+    break;
+  case IR_EQ: case IR_NE:
+    if ((ir-1)->o == IR_HREF && ir->op1 == as->curins-1) {
+      as->curins--;
+      asm_href(as, ir-1, (IROp)ir->o);
+    } else {
+      asm_equal(as, ir);
+    }
+    break;
+  case IR_RETF: asm_retf(as, ir); break;
+  /* Bit ops. */
+  case IR_BNOT: asm_bnot(as, ir); break;
+  case IR_BSWAP: asm_bswap(as, ir); break;
+  case IR_BAND: asm_band(as, ir); break;
+  case IR_BOR: asm_bor(as, ir); break;
+  case IR_BXOR: asm_bxor(as, ir); break;
+  case IR_BSHL: asm_bshl(as, ir); break;
+  case IR_BSHR: asm_bshr(as, ir); break;
+  case IR_BSAR: asm_bsar(as, ir); break;
+  case IR_BROL: asm_brol(as, ir); break;
+  case IR_BROR: asm_bror(as, ir); break;
+  /* Arithmetic ops. */
+  case IR_ADD: asm_add(as, ir); break;
+  case IR_SUB: asm_sub(as, ir); break;
+  case IR_MUL: asm_mul(as, ir); break;
+  case IR_DIV: asm_div(as, ir); break;
+  case IR_MOD: asm_mod(as, ir); break;
+  case IR_POW: asm_pow(as, ir); break;
+  case IR_NEG: asm_neg(as, ir); break;
+  case IR_ABS: asm_abs(as, ir); break;
+  case IR_ATAN2: asm_atan2(as, ir); break;
+  case IR_LDEXP: asm_ldexp(as, ir); break;
+  case IR_MIN: asm_min(as, ir); break;
+  case IR_MAX: asm_max(as, ir); break;
+  case IR_FPMATH: asm_fpmath(as, ir); break;
+  /* Overflow-checking arithmetic ops. */
+  case IR_ADDOV: asm_addov(as, ir); break;
+  case IR_SUBOV: asm_subov(as, ir); break;
+  case IR_MULOV: asm_mulov(as, ir); break;
+  /* Memory references. */
+  case IR_AREF: asm_aref(as, ir); break;
+  case IR_HREF: asm_href(as, ir, 0); break;
+  case IR_HREFK: asm_hrefk(as, ir); break;
+  case IR_NEWREF: asm_newref(as, ir); break;
+  case IR_UREFO: case IR_UREFC: asm_uref(as, ir); break;
+  case IR_FREF: asm_fref(as, ir); break;
+  case IR_STRREF: asm_strref(as, ir); break;
+  /* Loads and stores. */
+  case IR_ALOAD: case IR_HLOAD: case IR_ULOAD: case IR_VLOAD:
+    asm_ahuvload(as, ir);
+    break;
+  case IR_FLOAD: asm_fload(as, ir); break;
+  case IR_XLOAD: asm_xload(as, ir); break;
+  case IR_SLOAD: asm_sload(as, ir); break;
+  case IR_ASTORE: case IR_HSTORE: case IR_USTORE: asm_ahustore(as, ir); break;
+  case IR_FSTORE: asm_fstore(as, ir); break;
+  case IR_XSTORE: asm_xstore(as, ir); break;
+  /* Allocations. */
+  case IR_SNEW: case IR_XSNEW: asm_snew(as, ir); break;
+  case IR_TNEW: asm_tnew(as, ir); break;
+  case IR_TDUP: asm_tdup(as, ir); break;
+  case IR_CNEW: case IR_CNEWI: asm_cnew(as, ir); break;
+  /* Buffer operations. */
+  case IR_BUFHDR: asm_bufhdr(as, ir); break;
+  case IR_BUFPUT: asm_bufput(as, ir); break;
+  case IR_BUFSTR: asm_bufstr(as, ir); break;
+  /* Write barriers. */
+  case IR_TBAR: asm_tbar(as, ir); break;
+  case IR_OBAR: asm_obar(as, ir); break;
+  /* Type conversions. */
+  case IR_TOBIT: asm_tobit(as, ir); break;
+  case IR_CONV: asm_conv(as, ir); break;
+  case IR_TOSTR: asm_tostr(as, ir); break;
+  case IR_STRTO: asm_strto(as, ir); break;
+  /* Calls. */
+  case IR_CALLN: case IR_CALLL: case IR_CALLS: asm_call(as, ir); break;
+  case IR_CALLXS: asm_callx(as, ir); break;
+  case IR_CARG: break;
+  default:
+    setintV(&as->J->errinfo, ir->o);
+    lj_trace_err_info(as->J, LJ_TRERR_NYIIR);
+    break;
+  }
+}
 /* -- Head of trace ------------------------------------------------------- */
 /* Head of a root trace. */
@@ -1714,10 +2047,20 @@ static void asm_setup_regsp(ASMState *as)
      /* fallthrough */
 #endif
    /* C calls evict all scratch regs and return results in RID_RET. */
-    case IR_SNEW: case IR_XSNEW: case IR_NEWREF:
+    case IR_SNEW: case IR_XSNEW: case IR_NEWREF: case IR_BUFPUT:
      if (REGARG_NUMGPR < 3 && as->evenspill < 3)
        as->evenspill = 3;  /* lj_str_new and lj_tab_newkey need 3 args. */
-    case IR_TNEW: case IR_TDUP: case IR_CNEW: case IR_CNEWI: case IR_TOSTR:
+#if LJ_TARGET_X86 && LJ_HASFFI
+      if (0) {
+    case IR_CNEW:
+        if (ir->op2 != REF_NIL && as->evenspill < 4)
+          as->evenspill = 4;  /* lj_cdata_newv needs 4 args. */
+      }
+#else
+    case IR_CNEW:
+#endif
+    case IR_TNEW: case IR_TDUP: case IR_CNEWI: case IR_TOSTR:
+    case IR_BUFSTR:
      ir->prev = REGSP_HINT(RID_RET);
      if (inloop)
        as->modset = RSET_SCRATCH;
@@ -1753,7 +2096,7 @@ static void asm_setup_regsp(ASMState *as)
      break;
    case IR_FPMATH:
 #if LJ_TARGET_X86ORX64
-      if (ir->op2 == IRFPM_EXP2) {  /* May be joined to lj_vm_pow_sse. */
+      if (ir->op2 == IRFPM_EXP2) {  /* May be joined to pow. */
        ir->prev = REGSP_HINT(RID_XMM0);
 #if !LJ_64
        if (as->evenspill < 4)  /* Leave room for 16 byte scratch area. */
diff --git a/src/lj_asm_arm.h b/src/lj_asm_arm.h
index a66573c0..9b661eb7 100644
--- a/src/lj_asm_arm.h
+++ b/src/lj_asm_arm.h
@@ -338,7 +338,7 @@ static int asm_fusemadd(ASMState *as, IRIns *ir, ARMIns ai, ARMIns air)
 /* Generate a call to a C function. */
 static void asm_gencall(ASMState *as, const CCallInfo *ci, IRRef *args)
 {
-  uint32_t n, nargs = CCI_NARGS(ci);
+  uint32_t n, nargs = CCI_XNARGS(ci);
  int32_t ofs = 0;
 #if LJ_SOFTFP
  Reg gpr = REGARG_FIRSTGPR;
@@ -453,15 +453,6 @@ static void asm_setupresult(ASMState *as, IRIns *ir, const CCallInfo *ci)
  UNUSED(ci);
 }
-static void asm_call(ASMState *as, IRIns *ir)
-{
-  IRRef args[CCI_NARGS_MAX];
-  const CCallInfo *ci = &lj_ir_callinfo[ir->op2];
-  asm_collectargs(as, ir, ci, args);
-  asm_setupresult(as, ir, ci);
-  asm_gencall(as, ci, args);
-}
 static void asm_callx(ASMState *as, IRIns *ir)
 {
  IRRef args[CCI_NARGS_MAX*2];
@@ -528,6 +519,8 @@ static void asm_tobit(ASMState *as, IRIns *ir)
  emit_dn(as, ARMI_VMOV_R_S, dest, (tmp & 15));
  emit_dnm(as, ARMI_VADD_D, (tmp & 15), (left & 15), (right & 15));
 }
+#else
+#define asm_tobit(as, ir)       lua_assert(0)
 #endif
 static void asm_conv(ASMState *as, IRIns *ir)
@@ -600,31 +593,6 @@ static void asm_conv(ASMState *as, IRIns *ir)
  }
 }
-#if !LJ_SOFTFP && LJ_HASFFI
-static void asm_conv64(ASMState *as, IRIns *ir)
-{
-  IRType st = (IRType)((ir-1)->op2 & IRCONV_SRCMASK);
-  IRType dt = (((ir-1)->op2 & IRCONV_DSTMASK) >> IRCONV_DSH);
-  IRCallID id;
-  CCallInfo ci;
-  IRRef args[2];
-  args[0] = (ir-1)->op1;
-  args[1] = ir->op1;
-  if (st == IRT_NUM || st == IRT_FLOAT) {
-    id = IRCALL_fp64_d2l + ((st == IRT_FLOAT) ? 2 : 0) + (dt - IRT_I64);
-    ir--;
-  } else {
-    id = IRCALL_fp64_l2d + ((dt == IRT_FLOAT) ? 2 : 0) + (st - IRT_I64);
-  }
-  ci = lj_ir_callinfo[id];
-#if !LJ_ABI_SOFTFP
-  ci.flags |= CCI_VARARG;  /* These calls don't use the hard-float ABI! */
-#endif
-  asm_setupresult(as, ir, &ci);
-  asm_gencall(as, &ci, args);
-}
-#endif
 static void asm_strto(ASMState *as, IRIns *ir)
 {
  const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_strscan_num];
@@ -688,6 +656,8 @@ static void asm_strto(ASMState *as, IRIns *ir)
    emit_opk(as, ARMI_ADD, tmp, RID_SP, ofs, RSET_GPR);
 }
+/* -- Memory references --------------------------------------------------- */
 /* Get pointer to TValue. */
 static void asm_tvptr(ASMState *as, Reg dest, IRRef ref)
 {
@@ -713,7 +683,7 @@ static void asm_tvptr(ASMState *as, Reg dest, IRRef ref)
      Reg src = ra_alloc1(as, ref, allow);
      emit_lso(as, ARMI_STR, src, RID_SP, 0);
    }
-    if ((ir+1)->o == IR_HIOP)
+    if (LJ_SOFTFP && (ir+1)->o == IR_HIOP)
      type = ra_alloc1(as, ref+1, allow);
    else
      type = ra_allock(as, irt_toitype(ir->t), allow);
@@ -721,27 +691,6 @@ static void asm_tvptr(ASMState *as, Reg dest, IRRef ref)
  }
 }
-static void asm_tostr(ASMState *as, IRIns *ir)
-{
-  IRRef args[2];
-  args[0] = ASMREF_L;
-  as->gcsteps++;
-  if (irt_isnum(IR(ir->op1)->t) || (ir+1)->o == IR_HIOP) {
-    const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_str_fromnum];
-    args[1] = ASMREF_TMP1;  /* const lua_Number * */
-    asm_setupresult(as, ir, ci);  /* GCstr * */
-    asm_gencall(as, ci, args);
-    asm_tvptr(as, ra_releasetmp(as, ASMREF_TMP1), ir->op1);
-  } else {
-    const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_str_fromint];
-    args[1] = ir->op1;  /* int32_t k */
-    asm_setupresult(as, ir, ci);  /* GCstr * */
-    asm_gencall(as, ci, args);
-  }
-}
-/* -- Memory references --------------------------------------------------- */
 static void asm_aref(ASMState *as, IRIns *ir)
 {
  Reg dest = ra_dest(as, ir, RSET_GPR);
@@ -959,20 +908,6 @@ static void asm_hrefk(ASMState *as, IRIns *ir)
    emit_opk(as, ARMI_ADD, dest, node, ofs, RSET_GPR);
 }
-static void asm_newref(ASMState *as, IRIns *ir)
-{
-  const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_tab_newkey];
-  IRRef args[3];
-  if (ir->r == RID_SINK)
-    return;
-  args[0] = ASMREF_L;     /* lua_State *L */
-  args[1] = ir->op1;      /* GCtab *t     */
-  args[2] = ASMREF_TMP1;  /* cTValue *key */
-  asm_setupresult(as, ir, ci);  /* TValue * */
-  asm_gencall(as, ci, args);
-  asm_tvptr(as, ra_releasetmp(as, ASMREF_TMP1), ir->op2);
-}
 static void asm_uref(ASMState *as, IRIns *ir)
 {
  /* NYI: Check that UREFO is still open and not aliasing a slot. */
@@ -1105,7 +1040,7 @@ static void asm_xload(ASMState *as, IRIns *ir)
  asm_fusexref(as, asm_fxloadins(ir), dest, ir->op1, RSET_GPR, 0);
 }
-static void asm_xstore(ASMState *as, IRIns *ir, int32_t ofs)
+static void asm_xstore_(ASMState *as, IRIns *ir, int32_t ofs)
 {
  if (ir->r != RID_SINK) {
    Reg src = ra_alloc1(as, ir->op2,
@@ -1115,6 +1050,8 @@ static void asm_xstore(ASMState *as, IRIns *ir, int32_t ofs)
  }
 }
+#define asm_xstore(as, ir)      asm_xstore_(as, ir, 0)
 static void asm_ahuvload(ASMState *as, IRIns *ir)
 {
  int hiop = (LJ_SOFTFP && (ir+1)->o == IR_HIOP);
@@ -1272,19 +1209,16 @@ dotypecheck:
 static void asm_cnew(ASMState *as, IRIns *ir)
 {
  CTState *cts = ctype_ctsG(J2G(as->J));
-  CTypeID ctypeid = (CTypeID)IR(ir->op1)->i;
+  CTypeID id = (CTypeID)IR(ir->op1)->i;
-  CTSize sz = (ir->o == IR_CNEWI || ir->op2 == REF_NIL) ?
+  CTSize sz;
-              lj_ctype_size(cts, ctypeid) : (CTSize)IR(ir->op2)->i;
+  CTInfo info = lj_ctype_info(cts, id, &sz);
  const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_mem_newgco];
-  IRRef args[2];
+  IRRef args[4];
  RegSet allow = (RSET_GPR & ~RSET_SCRATCH);
  RegSet drop = RSET_SCRATCH;
-  lua_assert(sz != CTSIZE_INVALID);
+  lua_assert(sz != CTSIZE_INVALID || (ir->o == IR_CNEW && ir->op2 != REF_NIL));
-  args[0] = ASMREF_L;     /* lua_State *L */
-  args[1] = ASMREF_TMP1;  /* MSize size   */
  as->gcsteps++;
  if (ra_hasreg(ir->r))
    rset_clear(drop, ir->r);  /* Dest reg handled below. */
  ra_evictset(as, drop);
@@ -1306,16 +1240,28 @@ static void asm_cnew(ASMState *as, IRIns *ir)
      if (ofs == sizeof(GCcdata)) break;
      ofs -= 4; ir--;
    }
+  } else if (ir->op2 != REF_NIL) {  /* Create VLA/VLS/aligned cdata. */
+    ci = &lj_ir_callinfo[IRCALL_lj_cdata_newv];
+    args[0] = ASMREF_L;     /* lua_State *L */
+    args[1] = ir->op1;      /* CTypeID id   */
+    args[2] = ir->op2;      /* CTSize sz    */
+    args[3] = ASMREF_TMP1;  /* CTSize align */
+    asm_gencall(as, ci, args);
+    emit_loadi(as, ra_releasetmp(as, ASMREF_TMP1), (int32_t)ctype_align(info));
+    return;
  }
  /* Initialize gct and ctypeid. lj_mem_newgco() already sets marked. */
  {
-    uint32_t k = emit_isk12(ARMI_MOV, ctypeid);
+    uint32_t k = emit_isk12(ARMI_MOV, id);
-    Reg r = k ? RID_R1 : ra_allock(as, ctypeid, allow);
+    Reg r = k ? RID_R1 : ra_allock(as, id, allow);
    emit_lso(as, ARMI_STRB, RID_TMP, RID_RET, offsetof(GCcdata, gct));
    emit_lsox(as, ARMI_STRH, r, RID_RET, offsetof(GCcdata, ctypeid));
    emit_d(as, ARMI_MOV|ARMI_K12|~LJ_TCDATA, RID_TMP);
    if (k) emit_d(as, ARMI_MOV^k, RID_R1);
  }
+  args[0] = ASMREF_L;     /* lua_State *L */
+  args[1] = ASMREF_TMP1;  /* MSize size   */
  asm_gencall(as, ci, args);
  ra_allockreg(as, (int32_t)(sz+sizeof(GCcdata)),
               ra_releasetmp(as, ASMREF_TMP1));
@@ -1392,24 +1338,41 @@ static void asm_fpunary(ASMState *as, IRIns *ir, ARMIns ai)
  emit_dm(as, ai, (dest & 15), (left & 15));
 }
-static int asm_fpjoin_pow(ASMState *as, IRIns *ir)
+static void asm_callround(ASMState *as, IRIns *ir, int id)
-{
+{
-  IRIns *irp = IR(ir->op1);
+  /* The modified regs must match with the *.dasc implementation. */
-  if (irp == ir-1 && irp->o == IR_MUL && !ra_used(irp)) {
+  RegSet drop = RID2RSET(RID_R0)|RID2RSET(RID_R1)|RID2RSET(RID_R2)|
-    IRIns *irpp = IR(irp->op1);
+                RID2RSET(RID_R3)|RID2RSET(RID_R12);
-    if (irpp == ir-2 && irpp->o == IR_FPMATH &&
+  RegSet of;
-        irpp->op2 == IRFPM_LOG2 && !ra_used(irpp)) {
+  Reg dest, src;
-      const CCallInfo *ci = &lj_ir_callinfo[IRCALL_pow];
+  ra_evictset(as, drop);
-      IRRef args[2];
+  dest = ra_dest(as, ir, RSET_FPR);
-      args[0] = irpp->op1;
+  emit_dnm(as, ARMI_VMOV_D_RR, RID_RETLO, RID_RETHI, (dest & 15));
-      args[1] = irp->op2;
+  emit_call(as, id == IRFPM_FLOOR ? (void *)lj_vm_floor_sf :
-      asm_setupresult(as, ir, ci);
+                id == IRFPM_CEIL ? (void *)lj_vm_ceil_sf :
-      asm_gencall(as, ci, args);
+                                   (void *)lj_vm_trunc_sf);
-      return 1;
+  /* Workaround to protect argument GPRs from being used for remat. */
-    }
+  of = as->freeset;
-  }
+  as->freeset &= ~RSET_RANGE(RID_R0, RID_R1+1);
-  return 0;
+  as->cost[RID_R0] = as->cost[RID_R1] = REGCOST(~0u, ASMREF_L);
+  src = ra_alloc1(as, ir->op1, RSET_FPR);  /* May alloc GPR to remat FPR. */
+  as->freeset |= (of & RSET_RANGE(RID_R0, RID_R1+1));
+  emit_dnm(as, ARMI_VMOV_RR_D, RID_R0, RID_R1, (src & 15));
+}
+static void asm_fpmath(ASMState *as, IRIns *ir)
+{
+  if (ir->op2 == IRFPM_EXP2 && asm_fpjoin_pow(as, ir))
+    return;
+  if (ir->op2 <= IRFPM_TRUNC)
+    asm_callround(as, ir, ir->op2);
+  else if (ir->op2 == IRFPM_SQRT)
+    asm_fpunary(as, ir, ARMI_VSQRT_D);
+  else
+    asm_callid(as, ir, IRCALL_lj_vm_floor + ir->op2);
 }
+#else
+#define asm_fpmath(as, ir)      lua_assert(0)
 #endif
 static int asm_swapops(ASMState *as, IRRef lref, IRRef rref)
@@ -1459,32 +1422,6 @@ static void asm_intop_s(ASMState *as, IRIns *ir, ARMIns ai)
  asm_intop(as, ir, ai);
 }
-static void asm_bitop(ASMState *as, IRIns *ir, ARMIns ai)
-{
-  if (as->flagmcp == as->mcp) {  /* Try to drop cmp r, #0. */
-    uint32_t cc = (as->mcp[1] >> 28);
-    as->flagmcp = NULL;
-    if (cc <= CC_NE) {
-      as->mcp++;
-      ai |= ARMI_S;
-    } else if (cc == CC_GE) {
-      *++as->mcp ^= ((CC_GE^CC_PL) << 28);
-      ai |= ARMI_S;
-    } else if (cc == CC_LT) {
-      *++as->mcp ^= ((CC_LT^CC_MI) << 28);
-      ai |= ARMI_S;
-    }  /* else: other conds don't work with bit ops. */
-  }
-  if (ir->op2 == 0) {
-    Reg dest = ra_dest(as, ir, RSET_GPR);
-    uint32_t m = asm_fuseopm(as, ai, ir->op1, RSET_GPR);
-    emit_d(as, ai^m, dest);
-  } else {
-    /* NYI: Turn BAND !k12 into uxtb, uxth or bfc or shl+shr. */
-    asm_intop(as, ir, ai);
-  }
-}
 static void asm_intneg(ASMState *as, IRIns *ir, ARMIns ai)
 {
  Reg dest = ra_dest(as, ir, RSET_GPR);
@@ -1550,6 +1487,26 @@ static void asm_mul(ASMState *as, IRIns *ir)
  asm_intmul(as, ir);
 }
+#define asm_addov(as, ir)       asm_add(as, ir)
+#define asm_subov(as, ir)       asm_sub(as, ir)
+#define asm_mulov(as, ir)       asm_mul(as, ir)
+#if LJ_SOFTFP
+#define asm_div(as, ir)         lua_assert(0)
+#define asm_pow(as, ir)         lua_assert(0)
+#define asm_abs(as, ir)         lua_assert(0)
+#define asm_atan2(as, ir)       lua_assert(0)
+#define asm_ldexp(as, ir)       lua_assert(0)
+#else
+#define asm_div(as, ir)         asm_fparith(as, ir, ARMI_VDIV_D)
+#define asm_pow(as, ir)         asm_callid(as, ir, IRCALL_lj_vm_powi)
+#define asm_abs(as, ir)         asm_fpunary(as, ir, ARMI_VABS_D)
+#define asm_atan2(as, ir)       asm_callid(as, ir, IRCALL_atan2)
+#define asm_ldexp(as, ir)       asm_callid(as, ir, IRCALL_ldexp)
+#endif
+#define asm_mod(as, ir)         asm_callid(as, ir, IRCALL_lj_vm_modi)
 static void asm_neg(ASMState *as, IRIns *ir)
 {
 #if !LJ_SOFTFP
@@ -1561,41 +1518,35 @@ static void asm_neg(ASMState *as, IRIns *ir)
  asm_intneg(as, ir, ARMI_RSB);
 }
-static void asm_callid(ASMState *as, IRIns *ir, IRCallID id)
+static void asm_bitop(ASMState *as, IRIns *ir, ARMIns ai)
 {
-  const CCallInfo *ci = &lj_ir_callinfo[id];
+  if (as->flagmcp == as->mcp) {  /* Try to drop cmp r, #0. */
-  IRRef args[2];
+    uint32_t cc = (as->mcp[1] >> 28);
-  args[0] = ir->op1;
+    as->flagmcp = NULL;
-  args[1] = ir->op2;
+    if (cc <= CC_NE) {
-  asm_setupresult(as, ir, ci);
+      as->mcp++;
-  asm_gencall(as, ci, args);
+      ai |= ARMI_S;
+    } else if (cc == CC_GE) {
+      *++as->mcp ^= ((CC_GE^CC_PL) << 28);
+      ai |= ARMI_S;
+    } else if (cc == CC_LT) {
+      *++as->mcp ^= ((CC_LT^CC_MI) << 28);
+      ai |= ARMI_S;
+    }  /* else: other conds don't work with bit ops. */
+  }
+  if (ir->op2 == 0) {
+    Reg dest = ra_dest(as, ir, RSET_GPR);
+    uint32_t m = asm_fuseopm(as, ai, ir->op1, RSET_GPR);
+    emit_d(as, ai^m, dest);
+  } else {
+    /* NYI: Turn BAND !k12 into uxtb, uxth or bfc or shl+shr. */
+    asm_intop(as, ir, ai);
+  }
 }
-#if !LJ_SOFTFP
+#define asm_bnot(as, ir)        asm_bitop(as, ir, ARMI_MVN)
-static void asm_callround(ASMState *as, IRIns *ir, int id)
-{
-  /* The modified regs must match with the *.dasc implementation. */
-  RegSet drop = RID2RSET(RID_R0)|RID2RSET(RID_R1)|RID2RSET(RID_R2)|
-                RID2RSET(RID_R3)|RID2RSET(RID_R12);
-  RegSet of;
-  Reg dest, src;
-  ra_evictset(as, drop);
-  dest = ra_dest(as, ir, RSET_FPR);
-  emit_dnm(as, ARMI_VMOV_D_RR, RID_RETLO, RID_RETHI, (dest & 15));
-  emit_call(as, id == IRFPM_FLOOR ? (void *)lj_vm_floor_sf :
-                id == IRFPM_CEIL ? (void *)lj_vm_ceil_sf :
-                                   (void *)lj_vm_trunc_sf);
-  /* Workaround to protect argument GPRs from being used for remat. */
-  of = as->freeset;
-  as->freeset &= ~RSET_RANGE(RID_R0, RID_R1+1);
-  as->cost[RID_R0] = as->cost[RID_R1] = REGCOST(~0u, ASMREF_L);
-  src = ra_alloc1(as, ir->op1, RSET_FPR);  /* May alloc GPR to remat FPR. */
-  as->freeset |= (of & RSET_RANGE(RID_R0, RID_R1+1));
-  emit_dnm(as, ARMI_VMOV_RR_D, RID_R0, RID_R1, (src & 15));
-}
-#endif
-static void asm_bitswap(ASMState *as, IRIns *ir)
+static void asm_bswap(ASMState *as, IRIns *ir)
 {
  Reg dest = ra_dest(as, ir, RSET_GPR);
  Reg left = ra_alloc1(as, ir->op1, RSET_GPR);
@@ -1612,6 +1563,10 @@ static void asm_bitswap(ASMState *as, IRIns *ir)
  }
 }
+#define asm_band(as, ir)        asm_bitop(as, ir, ARMI_AND)
+#define asm_bor(as, ir)         asm_bitop(as, ir, ARMI_ORR)
+#define asm_bxor(as, ir)        asm_bitop(as, ir, ARMI_EOR)
 static void asm_bitshift(ASMState *as, IRIns *ir, ARMShift sh)
 {
  if (irref_isk(ir->op2)) {  /* Constant shifts. */
@@ -1629,6 +1584,12 @@ static void asm_bitshift(ASMState *as, IRIns *ir, ARMShift sh)
  }
 }
+#define asm_bshl(as, ir)        asm_bitshift(as, ir, ARMSH_LSL)
+#define asm_bshr(as, ir)        asm_bitshift(as, ir, ARMSH_LSR)
+#define asm_bsar(as, ir)        asm_bitshift(as, ir, ARMSH_ASR)
+#define asm_bror(as, ir)        asm_bitshift(as, ir, ARMSH_ROR)
+#define asm_brol(as, ir)        lua_assert(0)
 static void asm_intmin_max(ASMState *as, IRIns *ir, int cc)
 {
  uint32_t kcmp = 0, kmov = 0;
@@ -1702,6 +1663,9 @@ static void asm_min_max(ASMState *as, IRIns *ir, int cc, int fcc)
    asm_intmin_max(as, ir, cc);
 }
+#define asm_min(as, ir)         asm_min_max(as, ir, CC_GT, CC_HI)
+#define asm_max(as, ir)         asm_min_max(as, ir, CC_LT, CC_LO)
 /* -- Comparisons --------------------------------------------------------- */
 /* Map of comparisons to flags. ORDER IR. */
@@ -1817,6 +1781,18 @@ notst:
    as->flagmcp = as->mcp;  /* Allow elimination of the compare. */
 }
+static void asm_comp(ASMState *as, IRIns *ir)
+{
+#if !LJ_SOFTFP
+  if (irt_isnum(ir->t))
+    asm_fpcomp(as, ir);
+  else
+#endif
+    asm_intcomp(as, ir);
+}
+#define asm_equal(as, ir)       asm_comp(as, ir)
 #if LJ_HASFFI
 /* 64 bit integer comparisons. */
 static void asm_int64comp(ASMState *as, IRIns *ir)
@@ -1891,7 +1867,7 @@ static void asm_hiop(ASMState *as, IRIns *ir)
 #endif
  } else if ((ir-1)->o == IR_XSTORE) {
    if ((ir-1)->r != RID_SINK)
-      asm_xstore(as, ir, 4);
+      asm_xstore_(as, ir, 4);
    return;
  }
  if (!usehi) return;  /* Skip unused hiword op for all remaining ops. */
@@ -2160,143 +2136,13 @@ static void asm_tail_prep(ASMState *as)
  *p = 0;  /* Prevent load/store merging. */
 }
-/* -- Instruction dispatch ------------------------------------------------ */
-/* Assemble a single instruction. */
-static void asm_ir(ASMState *as, IRIns *ir)
-{
-  switch ((IROp)ir->o) {
-  /* Miscellaneous ops. */
-  case IR_LOOP: asm_loop(as); break;
-  case IR_NOP: case IR_XBAR: lua_assert(!ra_used(ir)); break;
-  case IR_USE:
-    ra_alloc1(as, ir->op1, irt_isfp(ir->t) ? RSET_FPR : RSET_GPR); break;
-  case IR_PHI: asm_phi(as, ir); break;
-  case IR_HIOP: asm_hiop(as, ir); break;
-  case IR_GCSTEP: asm_gcstep(as, ir); break;
-  /* Guarded assertions. */
-  case IR_EQ: case IR_NE:
-    if ((ir-1)->o == IR_HREF && ir->op1 == as->curins-1) {
-      as->curins--;
-      asm_href(as, ir-1, (IROp)ir->o);
-      break;
-    }
-    /* fallthrough */
-  case IR_LT: case IR_GE: case IR_LE: case IR_GT:
-  case IR_ULT: case IR_UGE: case IR_ULE: case IR_UGT:
-  case IR_ABC:
-#if !LJ_SOFTFP
-    if (irt_isnum(ir->t)) { asm_fpcomp(as, ir); break; }
-#endif
-    asm_intcomp(as, ir);
-    break;
-  case IR_RETF: asm_retf(as, ir); break;
-  /* Bit ops. */
-  case IR_BNOT: asm_bitop(as, ir, ARMI_MVN); break;
-  case IR_BSWAP: asm_bitswap(as, ir); break;
-  case IR_BAND: asm_bitop(as, ir, ARMI_AND); break;
-  case IR_BOR:  asm_bitop(as, ir, ARMI_ORR); break;
-  case IR_BXOR: asm_bitop(as, ir, ARMI_EOR); break;
-  case IR_BSHL: asm_bitshift(as, ir, ARMSH_LSL); break;
-  case IR_BSHR: asm_bitshift(as, ir, ARMSH_LSR); break;
-  case IR_BSAR: asm_bitshift(as, ir, ARMSH_ASR); break;
-  case IR_BROR: asm_bitshift(as, ir, ARMSH_ROR); break;
-  case IR_BROL: lua_assert(0); break;
-  /* Arithmetic ops. */
-  case IR_ADD: case IR_ADDOV: asm_add(as, ir); break;
-  case IR_SUB: case IR_SUBOV: asm_sub(as, ir); break;
-  case IR_MUL: case IR_MULOV: asm_mul(as, ir); break;
-  case IR_MOD: asm_callid(as, ir, IRCALL_lj_vm_modi); break;
-  case IR_NEG: asm_neg(as, ir); break;
-#if LJ_SOFTFP
-  case IR_DIV: case IR_POW: case IR_ABS:
-  case IR_ATAN2: case IR_LDEXP: case IR_FPMATH: case IR_TOBIT:
-    lua_assert(0);  /* Unused for LJ_SOFTFP. */
-    break;
-#else
-  case IR_DIV: asm_fparith(as, ir, ARMI_VDIV_D); break;
-  case IR_POW: asm_callid(as, ir, IRCALL_lj_vm_powi); break;
-  case IR_ABS: asm_fpunary(as, ir, ARMI_VABS_D); break;
-  case IR_ATAN2: asm_callid(as, ir, IRCALL_atan2); break;
-  case IR_LDEXP: asm_callid(as, ir, IRCALL_ldexp); break;
-  case IR_FPMATH:
-    if (ir->op2 == IRFPM_EXP2 && asm_fpjoin_pow(as, ir))
-      break;
-    if (ir->op2 <= IRFPM_TRUNC)
-      asm_callround(as, ir, ir->op2);
-    else if (ir->op2 == IRFPM_SQRT)
-      asm_fpunary(as, ir, ARMI_VSQRT_D);
-    else
-      asm_callid(as, ir, IRCALL_lj_vm_floor + ir->op2);
-    break;
-  case IR_TOBIT: asm_tobit(as, ir); break;
-#endif
-  case IR_MIN: asm_min_max(as, ir, CC_GT, CC_HI); break;
-  case IR_MAX: asm_min_max(as, ir, CC_LT, CC_LO); break;
-  /* Memory references. */
-  case IR_AREF: asm_aref(as, ir); break;
-  case IR_HREF: asm_href(as, ir, 0); break;
-  case IR_HREFK: asm_hrefk(as, ir); break;
-  case IR_NEWREF: asm_newref(as, ir); break;
-  case IR_UREFO: case IR_UREFC: asm_uref(as, ir); break;
-  case IR_FREF: asm_fref(as, ir); break;
-  case IR_STRREF: asm_strref(as, ir); break;
-  /* Loads and stores. */
-  case IR_ALOAD: case IR_HLOAD: case IR_ULOAD: case IR_VLOAD:
-    asm_ahuvload(as, ir);
-    break;
-  case IR_FLOAD: asm_fload(as, ir); break;
-  case IR_XLOAD: asm_xload(as, ir); break;
-  case IR_SLOAD: asm_sload(as, ir); break;
-  case IR_ASTORE: case IR_HSTORE: case IR_USTORE: asm_ahustore(as, ir); break;
-  case IR_FSTORE: asm_fstore(as, ir); break;
-  case IR_XSTORE: asm_xstore(as, ir, 0); break;
-  /* Allocations. */
-  case IR_SNEW: case IR_XSNEW: asm_snew(as, ir); break;
-  case IR_TNEW: asm_tnew(as, ir); break;
-  case IR_TDUP: asm_tdup(as, ir); break;
-  case IR_CNEW: case IR_CNEWI: asm_cnew(as, ir); break;
-  /* Write barriers. */
-  case IR_TBAR: asm_tbar(as, ir); break;
-  case IR_OBAR: asm_obar(as, ir); break;
-  /* Type conversions. */
-  case IR_CONV: asm_conv(as, ir); break;
-  case IR_TOSTR: asm_tostr(as, ir); break;
-  case IR_STRTO: asm_strto(as, ir); break;
-  /* Calls. */
-  case IR_CALLN: case IR_CALLL: case IR_CALLS: asm_call(as, ir); break;
-  case IR_CALLXS: asm_callx(as, ir); break;
-  case IR_CARG: break;
-  default:
-    setintV(&as->J->errinfo, ir->o);
-    lj_trace_err_info(as->J, LJ_TRERR_NYIIR);
-    break;
-  }
-}
 /* -- Trace setup --------------------------------------------------------- */
 /* Ensure there are enough stack slots for call arguments. */
 static Reg asm_setup_call_slots(ASMState *as, IRIns *ir, const CCallInfo *ci)
 {
  IRRef args[CCI_NARGS_MAX*2];
-  uint32_t i, nargs = (int)CCI_NARGS(ci);
+  uint32_t i, nargs = CCI_XNARGS(ci);
  int nslots = 0, ngpr = REGARG_NUMGPR, nfpr = REGARG_NUMFPR, fprodd = 0;
  asm_collectargs(as, ir, ci, args);
  for (i = 0; i < nargs; i++) {
diff --git a/src/lj_asm_mips.h b/src/lj_asm_mips.h
index 9fe7c9c3..3d061eb4 100644
--- a/src/lj_asm_mips.h
+++ b/src/lj_asm_mips.h
@@ -226,7 +226,7 @@ static void asm_fusexref(ASMState *as, MIPSIns mi, Reg rt, IRRef ref,
 /* Generate a call to a C function. */
 static void asm_gencall(ASMState *as, const CCallInfo *ci, IRRef *args)
 {
-  uint32_t n, nargs = CCI_NARGS(ci);
+  uint32_t n, nargs = CCI_XNARGS(ci);
  int32_t ofs = 16;
  Reg gpr, fpr = REGARG_FIRSTFPR;
  if ((void *)ci->func)
@@ -326,15 +326,6 @@ static void asm_setupresult(ASMState *as, IRIns *ir, const CCallInfo *ci)
  }
 }
-static void asm_call(ASMState *as, IRIns *ir)
-{
-  IRRef args[CCI_NARGS_MAX];
-  const CCallInfo *ci = &lj_ir_callinfo[ir->op2];
-  asm_collectargs(as, ir, ci, args);
-  asm_setupresult(as, ir, ci);
-  asm_gencall(as, ci, args);
-}
 static void asm_callx(ASMState *as, IRIns *ir)
 {
  IRRef args[CCI_NARGS_MAX*2];
@@ -362,16 +353,6 @@ static void asm_callx(ASMState *as, IRIns *ir)
  asm_gencall(as, &ci, args);
 }
-static void asm_callid(ASMState *as, IRIns *ir, IRCallID id)
-{
-  const CCallInfo *ci = &lj_ir_callinfo[id];
-  IRRef args[2];
-  args[0] = ir->op1;
-  args[1] = ir->op2;
-  asm_setupresult(as, ir, ci);
-  asm_gencall(as, ci, args);
-}
 static void asm_callround(ASMState *as, IRIns *ir, IRCallID id)
 {
  /* The modified regs must match with the *.dasc implementation. */
@@ -519,28 +500,6 @@ static void asm_conv(ASMState *as, IRIns *ir)
  }
 }
-#if LJ_HASFFI
-static void asm_conv64(ASMState *as, IRIns *ir)
-{
-  IRType st = (IRType)((ir-1)->op2 & IRCONV_SRCMASK);
-  IRType dt = (((ir-1)->op2 & IRCONV_DSTMASK) >> IRCONV_DSH);
-  IRCallID id;
-  const CCallInfo *ci;
-  IRRef args[2];
-  args[LJ_BE?0:1] = ir->op1;
-  args[LJ_BE?1:0] = (ir-1)->op1;
-  if (st == IRT_NUM || st == IRT_FLOAT) {
-    id = IRCALL_fp64_d2l + ((st == IRT_FLOAT) ? 2 : 0) + (dt - IRT_I64);
-    ir--;
-  } else {
-    id = IRCALL_fp64_l2d + ((dt == IRT_FLOAT) ? 2 : 0) + (st - IRT_I64);
-  }
-  ci = &lj_ir_callinfo[id];
-  asm_setupresult(as, ir, ci);
-  asm_gencall(as, ci, args);
-}
-#endif
 static void asm_strto(ASMState *as, IRIns *ir)
 {
  const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_strscan_num];
@@ -557,6 +516,8 @@ static void asm_strto(ASMState *as, IRIns *ir)
           RID_SP, sps_scale(ir->s));
 }
+/* -- Memory references --------------------------------------------------- */
 /* Get pointer to TValue. */
 static void asm_tvptr(ASMState *as, Reg dest, IRRef ref)
 {
@@ -580,27 +541,6 @@ static void asm_tvptr(ASMState *as, Reg dest, IRRef ref)
  }
 }
-static void asm_tostr(ASMState *as, IRIns *ir)
-{
-  IRRef args[2];
-  args[0] = ASMREF_L;
-  as->gcsteps++;
-  if (irt_isnum(IR(ir->op1)->t) || (ir+1)->o == IR_HIOP) {
-    const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_str_fromnum];
-    args[1] = ASMREF_TMP1;  /* const lua_Number * */
-    asm_setupresult(as, ir, ci);  /* GCstr * */
-    asm_gencall(as, ci, args);
-    asm_tvptr(as, ra_releasetmp(as, ASMREF_TMP1), ir->op1);
-  } else {
-    const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_str_fromint];
-    args[1] = ir->op1;  /* int32_t k */
-    asm_setupresult(as, ir, ci);  /* GCstr * */
-    asm_gencall(as, ci, args);
-  }
-}
-/* -- Memory references --------------------------------------------------- */
 static void asm_aref(ASMState *as, IRIns *ir)
 {
  Reg dest = ra_dest(as, ir, RSET_GPR);
@@ -630,7 +570,7 @@ static void asm_aref(ASMState *as, IRIns *ir)
 **   } while ((n = nextnode(n)));
 **   return niltv(L);
 */
-static void asm_href(ASMState *as, IRIns *ir)
+static void asm_href(ASMState *as, IRIns *ir, IROp merge)
 {
  RegSet allow = RSET_GPR;
  int destused = ra_used(ir);
@@ -656,37 +596,42 @@ static void asm_href(ASMState *as, IRIns *ir)
  tmp2 = ra_scratch(as, allow);
  rset_clear(allow, tmp2);
-  /* Key not found in chain: load niltv. */
+  /* Key not found in chain: jump to exit (if merged) or load niltv. */
  l_end = emit_label(as);
-  if (destused)
+  as->invmcp = NULL;
+  if (merge == IR_NE)
+    asm_guard(as, MIPSI_B, RID_ZERO, RID_ZERO);
+  else if (destused)
    emit_loada(as, dest, niltvg(J2G(as->J)));
-  else
-    *--as->mcp = MIPSI_NOP;
  /* Follow hash chain until the end. */
-  emit_move(as, dest, tmp1);
+  emit_move(as, dest, tmp2);
  l_loop = --as->mcp;
-  emit_tsi(as, MIPSI_LW, tmp1, dest, (int32_t)offsetof(Node, next));
+  emit_tsi(as, MIPSI_LW, tmp2, dest, (int32_t)offsetof(Node, next));
  l_next = emit_label(as);
  /* Type and value comparison. */
+  if (merge == IR_EQ) {  /* Must match asm_guard(). */
+    emit_ti(as, MIPSI_LI, RID_TMP, as->snapno);
+    l_end = asm_exitstub_addr(as);
+  }
  if (irt_isnum(kt)) {
    emit_branch(as, MIPSI_BC1T, 0, 0, l_end);
    emit_fgh(as, MIPSI_C_EQ_D, 0, tmpnum, key);
-        emit_tg(as, MIPSI_MFC1, tmp1, key+1);
+    *--as->mcp = MIPSI_NOP;  /* Avoid NaN comparison overhead. */
-    emit_branch(as, MIPSI_BEQ, tmp1, RID_ZERO, l_next);
+    emit_branch(as, MIPSI_BEQ, tmp2, RID_ZERO, l_next);
-    emit_tsi(as, MIPSI_SLTIU, tmp1, tmp1, (int32_t)LJ_TISNUM);
+    emit_tsi(as, MIPSI_SLTIU, tmp2, tmp2, (int32_t)LJ_TISNUM);
    emit_hsi(as, MIPSI_LDC1, tmpnum, dest, (int32_t)offsetof(Node, key.n));
  } else {
    if (irt_ispri(kt)) {
-      emit_branch(as, MIPSI_BEQ, tmp1, type, l_end);
+      emit_branch(as, MIPSI_BEQ, tmp2, type, l_end);
    } else {
-      emit_branch(as, MIPSI_BEQ, tmp2, key, l_end);
+      emit_branch(as, MIPSI_BEQ, tmp1, key, l_end);
-      emit_tsi(as, MIPSI_LW, tmp2, dest, (int32_t)offsetof(Node, key.gcr));
+      emit_tsi(as, MIPSI_LW, tmp1, dest, (int32_t)offsetof(Node, key.gcr));
-      emit_branch(as, MIPSI_BNE, tmp1, type, l_next);
+      emit_branch(as, MIPSI_BNE, tmp2, type, l_next);
    }
  }
-  emit_tsi(as, MIPSI_LW, tmp1, dest, (int32_t)offsetof(Node, key.it));
+  emit_tsi(as, MIPSI_LW, tmp2, dest, (int32_t)offsetof(Node, key.it));
-  *l_loop = MIPSI_BNE | MIPSF_S(tmp1) | ((as->mcp-l_loop-1) & 0xffffu);
+  *l_loop = MIPSI_BNE | MIPSF_S(tmp2) | ((as->mcp-l_loop-1) & 0xffffu);
  /* Load main position relative to tab->node into dest. */
  khash = irref_isk(refkey) ? ir_khash(irkey) : 1;
@@ -776,20 +721,6 @@ nolo:
    emit_tsi(as, MIPSI_ADDU, dest, node, ra_allock(as, ofs, allow));
 }
-static void asm_newref(ASMState *as, IRIns *ir)
-{
-  if (ir->r != RID_SINK) {
-    const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_tab_newkey];
-    IRRef args[3];
-    args[0] = ASMREF_L;     /* lua_State *L */
-    args[1] = ir->op1;      /* GCtab *t     */
-    args[2] = ASMREF_TMP1;  /* cTValue *key */
-    asm_setupresult(as, ir, ci);  /* TValue * */
-    asm_gencall(as, ci, args);
-    asm_tvptr(as, ra_releasetmp(as, ASMREF_TMP1), ir->op2);
-  }
-}
 static void asm_uref(ASMState *as, IRIns *ir)
 {
  /* NYI: Check that UREFO is still open and not aliasing a slot. */
@@ -918,7 +849,7 @@ static void asm_xload(ASMState *as, IRIns *ir)
  asm_fusexref(as, asm_fxloadins(ir), dest, ir->op1, RSET_GPR, 0);
 }
-static void asm_xstore(ASMState *as, IRIns *ir, int32_t ofs)
+static void asm_xstore_(ASMState *as, IRIns *ir, int32_t ofs)
 {
  if (ir->r != RID_SINK) {
    Reg src = ra_alloc1z(as, ir->op2, irt_isfp(ir->t) ? RSET_FPR : RSET_GPR);
@@ -927,6 +858,8 @@ static void asm_xstore(ASMState *as, IRIns *ir, int32_t ofs)
  }
 }
+#define asm_xstore(as, ir)      asm_xstore_(as, ir, 0)
 static void asm_ahuvload(ASMState *as, IRIns *ir)
 {
  IRType1 t = ir->t;
@@ -1002,7 +935,7 @@ static void asm_sload(ASMState *as, IRIns *ir)
      if (irt_isint(t)) {
        Reg tmp = ra_scratch(as, RSET_FPR);
        emit_tg(as, MIPSI_MFC1, dest, tmp);
-        emit_fg(as, MIPSI_CVT_W_D, tmp, tmp);
+        emit_fg(as, MIPSI_TRUNC_W_D, tmp, tmp);
        dest = tmp;
        t.irt = IRT_NUM;  /* Check for original type. */
      } else {
@@ -1042,19 +975,15 @@ dotypecheck:
 static void asm_cnew(ASMState *as, IRIns *ir)
 {
  CTState *cts = ctype_ctsG(J2G(as->J));
-  CTypeID ctypeid = (CTypeID)IR(ir->op1)->i;
+  CTypeID id = (CTypeID)IR(ir->op1)->i;
-  CTSize sz = (ir->o == IR_CNEWI || ir->op2 == REF_NIL) ?
+  CTSize sz;
-              lj_ctype_size(cts, ctypeid) : (CTSize)IR(ir->op2)->i;
+  CTInfo info = lj_ctype_info(cts, id, &sz);
  const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_mem_newgco];
-  IRRef args[2];
+  IRRef args[4];
-  RegSet allow = (RSET_GPR & ~RSET_SCRATCH);
  RegSet drop = RSET_SCRATCH;
-  lua_assert(sz != CTSIZE_INVALID);
+  lua_assert(sz != CTSIZE_INVALID || (ir->o == IR_CNEW && ir->op2 != REF_NIL));
-  args[0] = ASMREF_L;     /* lua_State *L */
-  args[1] = ASMREF_TMP1;  /* MSize size   */
  as->gcsteps++;
  if (ra_hasreg(ir->r))
    rset_clear(drop, ir->r);  /* Dest reg handled below. */
  ra_evictset(as, drop);
@@ -1063,6 +992,7 @@ static void asm_cnew(ASMState *as, IRIns *ir)
  /* Initialize immutable cdata object. */
  if (ir->o == IR_CNEWI) {
+    RegSet allow = (RSET_GPR & ~RSET_SCRATCH);
    int32_t ofs = sizeof(GCcdata);
    lua_assert(sz == 4 || sz == 8);
    if (sz == 8) {
@@ -1077,12 +1007,24 @@ static void asm_cnew(ASMState *as, IRIns *ir)
      if (ofs == sizeof(GCcdata)) break;
      ofs -= 4; if (LJ_BE) ir++; else ir--;
    }
+  } else if (ir->op2 != REF_NIL) {  /* Create VLA/VLS/aligned cdata. */
+    ci = &lj_ir_callinfo[IRCALL_lj_cdata_newv];
+    args[0] = ASMREF_L;     /* lua_State *L */
+    args[1] = ir->op1;      /* CTypeID id   */
+    args[2] = ir->op2;      /* CTSize sz    */
+    args[3] = ASMREF_TMP1;  /* CTSize align */
+    asm_gencall(as, ci, args);
+    emit_loadi(as, ra_releasetmp(as, ASMREF_TMP1), (int32_t)ctype_align(info));
+    return;
  }
  /* Initialize gct and ctypeid. lj_mem_newgco() already sets marked. */
  emit_tsi(as, MIPSI_SB, RID_RET+1, RID_RET, offsetof(GCcdata, gct));
  emit_tsi(as, MIPSI_SH, RID_TMP, RID_RET, offsetof(GCcdata, ctypeid));
  emit_ti(as, MIPSI_LI, RID_RET+1, ~LJ_TCDATA);
-  emit_ti(as, MIPSI_LI, RID_TMP, ctypeid); /* Lower 16 bit used. Sign-ext ok. */
+  emit_ti(as, MIPSI_LI, RID_TMP, id); /* Lower 16 bit used. Sign-ext ok. */
+  args[0] = ASMREF_L;     /* lua_State *L */
+  args[1] = ASMREF_TMP1;  /* MSize size   */
  asm_gencall(as, ci, args);
  ra_allockreg(as, (int32_t)(sz+sizeof(GCcdata)),
               ra_releasetmp(as, ASMREF_TMP1));
@@ -1152,23 +1094,16 @@ static void asm_fpunary(ASMState *as, IRIns *ir, MIPSIns mi)
  emit_fg(as, mi, dest, left);
 }
-static int asm_fpjoin_pow(ASMState *as, IRIns *ir)
+static void asm_fpmath(ASMState *as, IRIns *ir)
-{
+{
-  IRIns *irp = IR(ir->op1);
+  if (ir->op2 == IRFPM_EXP2 && asm_fpjoin_pow(as, ir))
-  if (irp == ir-1 && irp->o == IR_MUL && !ra_used(irp)) {
+    return;
-    IRIns *irpp = IR(irp->op1);
+  if (ir->op2 <= IRFPM_TRUNC)
-    if (irpp == ir-2 && irpp->o == IR_FPMATH &&
+    asm_callround(as, ir, IRCALL_lj_vm_floor + ir->op2);
-        irpp->op2 == IRFPM_LOG2 && !ra_used(irpp)) {
+  else if (ir->op2 == IRFPM_SQRT)
-      const CCallInfo *ci = &lj_ir_callinfo[IRCALL_pow];
+    asm_fpunary(as, ir, MIPSI_SQRT_D);
-      IRRef args[2];
+  else
-      args[0] = irpp->op1;
+    asm_callid(as, ir, IRCALL_lj_vm_floor + ir->op2);
-      args[1] = irp->op2;
-      asm_setupresult(as, ir, ci);
-      asm_gencall(as, ci, args);
-      return 1;
-    }
-  }
-  return 0;
 }
 static void asm_add(ASMState *as, IRIns *ir)
@@ -1214,6 +1149,10 @@ static void asm_mul(ASMState *as, IRIns *ir)
  }
 }
+#define asm_div(as, ir)         asm_fparith(as, ir, MIPSI_DIV_D)
+#define asm_mod(as, ir)         asm_callid(as, ir, IRCALL_lj_vm_modi)
+#define asm_pow(as, ir)         asm_callid(as, ir, IRCALL_lj_vm_powi)
 static void asm_neg(ASMState *as, IRIns *ir)
 {
  if (irt_isnum(ir->t)) {
@@ -1225,6 +1164,10 @@ static void asm_neg(ASMState *as, IRIns *ir)
  }
 }
+#define asm_abs(as, ir)         asm_fpunary(as, ir, MIPSI_ABS_D)
+#define asm_atan2(as, ir)       asm_callid(as, ir, IRCALL_atan2)
+#define asm_ldexp(as, ir)       asm_callid(as, ir, IRCALL_ldexp)
 static void asm_arithov(ASMState *as, IRIns *ir)
 {
  Reg right, left, tmp, dest = ra_dest(as, ir, RSET_GPR);
@@ -1258,13 +1201,21 @@ static void asm_arithov(ASMState *as, IRIns *ir)
    emit_move(as, RID_TMP, dest == left ? left : right);
 }
+#define asm_addov(as, ir)       asm_arithov(as, ir)
+#define asm_subov(as, ir)       asm_arithov(as, ir)
 static void asm_mulov(ASMState *as, IRIns *ir)
 {
-#if LJ_DUALNUM
+  Reg dest = ra_dest(as, ir, RSET_GPR);
-#error "NYI: MULOV"
+  Reg tmp, right, left = ra_alloc2(as, ir, RSET_GPR);
-#else
+  right = (left >> 8); left &= 255;
-  UNUSED(as); UNUSED(ir); lua_assert(0);  /* Unused in single-number mode. */
+  tmp = ra_scratch(as, rset_exclude(rset_exclude(rset_exclude(RSET_GPR, left),
-#endif
+                                                 right), dest));
+  asm_guard(as, MIPSI_BNE, RID_TMP, tmp);
+  emit_dta(as, MIPSI_SRA, RID_TMP, dest, 31);
+  emit_dst(as, MIPSI_MFHI, tmp, 0, 0);
+  emit_dst(as, MIPSI_MFLO, dest, 0, 0);
+  emit_dst(as, MIPSI_MULT, 0, left, right);
 }
 #if LJ_HASFFI
@@ -1351,7 +1302,7 @@ static void asm_neg64(ASMState *as, IRIns *ir)
 }
 #endif
-static void asm_bitnot(ASMState *as, IRIns *ir)
+static void asm_bnot(ASMState *as, IRIns *ir)
 {
  Reg left, right, dest = ra_dest(as, ir, RSET_GPR);
  IRIns *irl = IR(ir->op1);
@@ -1365,7 +1316,7 @@ static void asm_bitnot(ASMState *as, IRIns *ir)
  emit_dst(as, MIPSI_NOR, dest, left, right);
 }
-static void asm_bitswap(ASMState *as, IRIns *ir)
+static void asm_bswap(ASMState *as, IRIns *ir)
 {
  Reg dest = ra_dest(as, ir, RSET_GPR);
  Reg left = ra_alloc1(as, ir->op1, RSET_GPR);
@@ -1401,6 +1352,10 @@ static void asm_bitop(ASMState *as, IRIns *ir, MIPSIns mi, MIPSIns mik)
  emit_dst(as, mi, dest, left, right);
 }
+#define asm_band(as, ir)        asm_bitop(as, ir, MIPSI_AND, MIPSI_ANDI)
+#define asm_bor(as, ir)         asm_bitop(as, ir, MIPSI_OR, MIPSI_ORI)
+#define asm_bxor(as, ir)        asm_bitop(as, ir, MIPSI_XOR, MIPSI_XORI)
 static void asm_bitshift(ASMState *as, IRIns *ir, MIPSIns mi, MIPSIns mik)
 {
  Reg dest = ra_dest(as, ir, RSET_GPR);
@@ -1414,7 +1369,12 @@ static void asm_bitshift(ASMState *as, IRIns *ir, MIPSIns mi, MIPSIns mik)
  }
 }
-static void asm_bitror(ASMState *as, IRIns *ir)
+#define asm_bshl(as, ir)        asm_bitshift(as, ir, MIPSI_SLLV, MIPSI_SLL)
+#define asm_bshr(as, ir)        asm_bitshift(as, ir, MIPSI_SRLV, MIPSI_SRL)
+#define asm_bsar(as, ir)        asm_bitshift(as, ir, MIPSI_SRAV, MIPSI_SRA)
+#define asm_brol(as, ir)        lua_assert(0)
+static void asm_bror(ASMState *as, IRIns *ir)
 {
  if ((as->flags & JIT_F_MIPS32R2)) {
    asm_bitshift(as, ir, MIPSI_ROTRV, MIPSI_ROTR);
@@ -1463,6 +1423,9 @@ static void asm_min_max(ASMState *as, IRIns *ir, int ismax)
  }
 }
+#define asm_min(as, ir)         asm_min_max(as, ir, 0)
+#define asm_max(as, ir)         asm_min_max(as, ir, 1)
 /* -- Comparisons --------------------------------------------------------- */
 static void asm_comp(ASMState *as, IRIns *ir)
@@ -1500,7 +1463,7 @@ static void asm_comp(ASMState *as, IRIns *ir)
  }
 }
-static void asm_compeq(ASMState *as, IRIns *ir)
+static void asm_equal(ASMState *as, IRIns *ir)
 {
  Reg right, left = ra_alloc2(as, ir, irt_isnum(ir->t) ? RSET_FPR : RSET_GPR);
  right = (left >> 8); left &= 255;
@@ -1574,8 +1537,8 @@ static void asm_hiop(ASMState *as, IRIns *ir)
  } else if ((ir-1)->o == IR_XSTORE) {
    as->curins--;  /* Handle both stores here. */
    if ((ir-1)->r != RID_SINK) {
-      asm_xstore(as, ir, LJ_LE ? 4 : 0);
+      asm_xstore_(as, ir, LJ_LE ? 4 : 0);
-      asm_xstore(as, ir-1, LJ_LE ? 0 : 4);
+      asm_xstore_(as, ir-1, LJ_LE ? 0 : 4);
    }
    return;
  }
@@ -1771,131 +1734,13 @@ static void asm_tail_prep(ASMState *as)
  as->invmcp = as->loopref ? as->mcp : NULL;
 }
-/* -- Instruction dispatch ------------------------------------------------ */
-/* Assemble a single instruction. */
-static void asm_ir(ASMState *as, IRIns *ir)
-{
-  switch ((IROp)ir->o) {
-  /* Miscellaneous ops. */
-  case IR_LOOP: asm_loop(as); break;
-  case IR_NOP: case IR_XBAR: lua_assert(!ra_used(ir)); break;
-  case IR_USE:
-    ra_alloc1(as, ir->op1, irt_isfp(ir->t) ? RSET_FPR : RSET_GPR); break;
-  case IR_PHI: asm_phi(as, ir); break;
-  case IR_HIOP: asm_hiop(as, ir); break;
-  case IR_GCSTEP: asm_gcstep(as, ir); break;
-  /* Guarded assertions. */
-  case IR_EQ: case IR_NE: asm_compeq(as, ir); break;
-  case IR_LT: case IR_GE: case IR_LE: case IR_GT:
-  case IR_ULT: case IR_UGE: case IR_ULE: case IR_UGT:
-  case IR_ABC:
-    asm_comp(as, ir);
-    break;
-  case IR_RETF: asm_retf(as, ir); break;
-  /* Bit ops. */
-  case IR_BNOT: asm_bitnot(as, ir); break;
-  case IR_BSWAP: asm_bitswap(as, ir); break;
-  case IR_BAND: asm_bitop(as, ir, MIPSI_AND, MIPSI_ANDI); break;
-  case IR_BOR:  asm_bitop(as, ir, MIPSI_OR, MIPSI_ORI); break;
-  case IR_BXOR: asm_bitop(as, ir, MIPSI_XOR, MIPSI_XORI); break;
-  case IR_BSHL: asm_bitshift(as, ir, MIPSI_SLLV, MIPSI_SLL); break;
-  case IR_BSHR: asm_bitshift(as, ir, MIPSI_SRLV, MIPSI_SRL); break;
-  case IR_BSAR: asm_bitshift(as, ir, MIPSI_SRAV, MIPSI_SRA); break;
-  case IR_BROL: lua_assert(0); break;
-  case IR_BROR: asm_bitror(as, ir); break;
-  /* Arithmetic ops. */
-  case IR_ADD: asm_add(as, ir); break;
-  case IR_SUB: asm_sub(as, ir); break;
-  case IR_MUL: asm_mul(as, ir); break;
-  case IR_DIV: asm_fparith(as, ir, MIPSI_DIV_D); break;
-  case IR_MOD: asm_callid(as, ir, IRCALL_lj_vm_modi); break;
-  case IR_POW: asm_callid(as, ir, IRCALL_lj_vm_powi); break;
-  case IR_NEG: asm_neg(as, ir); break;
-  case IR_ABS: asm_fpunary(as, ir, MIPSI_ABS_D); break;
-  case IR_ATAN2: asm_callid(as, ir, IRCALL_atan2); break;
-  case IR_LDEXP: asm_callid(as, ir, IRCALL_ldexp); break;
-  case IR_MIN: asm_min_max(as, ir, 0); break;
-  case IR_MAX: asm_min_max(as, ir, 1); break;
-  case IR_FPMATH:
-    if (ir->op2 == IRFPM_EXP2 && asm_fpjoin_pow(as, ir))
-      break;
-    if (ir->op2 <= IRFPM_TRUNC)
-      asm_callround(as, ir, IRCALL_lj_vm_floor + ir->op2);
-    else if (ir->op2 == IRFPM_SQRT)
-      asm_fpunary(as, ir, MIPSI_SQRT_D);
-    else
-      asm_callid(as, ir, IRCALL_lj_vm_floor + ir->op2);
-    break;
-  /* Overflow-checking arithmetic ops. */
-  case IR_ADDOV: asm_arithov(as, ir); break;
-  case IR_SUBOV: asm_arithov(as, ir); break;
-  case IR_MULOV: asm_mulov(as, ir); break;
-  /* Memory references. */
-  case IR_AREF: asm_aref(as, ir); break;
-  case IR_HREF: asm_href(as, ir); break;
-  case IR_HREFK: asm_hrefk(as, ir); break;
-  case IR_NEWREF: asm_newref(as, ir); break;
-  case IR_UREFO: case IR_UREFC: asm_uref(as, ir); break;
-  case IR_FREF: asm_fref(as, ir); break;
-  case IR_STRREF: asm_strref(as, ir); break;
-  /* Loads and stores. */
-  case IR_ALOAD: case IR_HLOAD: case IR_ULOAD: case IR_VLOAD:
-    asm_ahuvload(as, ir);
-    break;
-  case IR_FLOAD: asm_fload(as, ir); break;
-  case IR_XLOAD: asm_xload(as, ir); break;
-  case IR_SLOAD: asm_sload(as, ir); break;
-  case IR_ASTORE: case IR_HSTORE: case IR_USTORE: asm_ahustore(as, ir); break;
-  case IR_FSTORE: asm_fstore(as, ir); break;
-  case IR_XSTORE: asm_xstore(as, ir, 0); break;
-  /* Allocations. */
-  case IR_SNEW: case IR_XSNEW: asm_snew(as, ir); break;
-  case IR_TNEW: asm_tnew(as, ir); break;
-  case IR_TDUP: asm_tdup(as, ir); break;
-  case IR_CNEW: case IR_CNEWI: asm_cnew(as, ir); break;
-  /* Write barriers. */
-  case IR_TBAR: asm_tbar(as, ir); break;
-  case IR_OBAR: asm_obar(as, ir); break;
-  /* Type conversions. */
-  case IR_CONV: asm_conv(as, ir); break;
-  case IR_TOBIT: asm_tobit(as, ir); break;
-  case IR_TOSTR: asm_tostr(as, ir); break;
-  case IR_STRTO: asm_strto(as, ir); break;
-  /* Calls. */
-  case IR_CALLN: case IR_CALLL: case IR_CALLS: asm_call(as, ir); break;
-  case IR_CALLXS: asm_callx(as, ir); break;
-  case IR_CARG: break;
-  default:
-    setintV(&as->J->errinfo, ir->o);
-    lj_trace_err_info(as->J, LJ_TRERR_NYIIR);
-    break;
-  }
-}
 /* -- Trace setup --------------------------------------------------------- */
 /* Ensure there are enough stack slots for call arguments. */
 static Reg asm_setup_call_slots(ASMState *as, IRIns *ir, const CCallInfo *ci)
 {
  IRRef args[CCI_NARGS_MAX*2];
-  uint32_t i, nargs = (int)CCI_NARGS(ci);
+  uint32_t i, nargs = CCI_XNARGS(ci);
  int nslots = 4, ngpr = REGARG_NUMGPR, nfpr = REGARG_NUMFPR;
  asm_collectargs(as, ir, ci, args);
  for (i = 0; i < nargs; i++) {
diff --git a/src/lj_asm_ppc.h b/src/lj_asm_ppc.h
index 651fa318..10cd79dd 100644
--- a/src/lj_asm_ppc.h
+++ b/src/lj_asm_ppc.h
@@ -249,7 +249,7 @@ static int asm_fusemadd(ASMState *as, IRIns *ir, PPCIns pi, PPCIns pir)
 /* Generate a call to a C function. */
 static void asm_gencall(ASMState *as, const CCallInfo *ci, IRRef *args)
 {
-  uint32_t n, nargs = CCI_NARGS(ci);
+  uint32_t n, nargs = CCI_XNARGS(ci);
  int32_t ofs = 8;
  Reg gpr = REGARG_FIRSTGPR, fpr = REGARG_FIRSTFPR;
  if ((void *)ci->func)
@@ -329,15 +329,6 @@ static void asm_setupresult(ASMState *as, IRIns *ir, const CCallInfo *ci)
  }
 }
-static void asm_call(ASMState *as, IRIns *ir)
-{
-  IRRef args[CCI_NARGS_MAX];
-  const CCallInfo *ci = &lj_ir_callinfo[ir->op2];
-  asm_collectargs(as, ir, ci, args);
-  asm_setupresult(as, ir, ci);
-  asm_gencall(as, ci, args);
-}
 static void asm_callx(ASMState *as, IRIns *ir)
 {
  IRRef args[CCI_NARGS_MAX*2];
@@ -361,16 +352,6 @@ static void asm_callx(ASMState *as, IRIns *ir)
  asm_gencall(as, &ci, args);
 }
-static void asm_callid(ASMState *as, IRIns *ir, IRCallID id)
-{
-  const CCallInfo *ci = &lj_ir_callinfo[id];
-  IRRef args[2];
-  args[0] = ir->op1;
-  args[1] = ir->op2;
-  asm_setupresult(as, ir, ci);
-  asm_gencall(as, ci, args);
-}
 /* -- Returns ------------------------------------------------------------- */
 /* Return to lower frame. Guard that it goes to the right spot. */
@@ -510,28 +491,6 @@ static void asm_conv(ASMState *as, IRIns *ir)
  }
 }
-#if LJ_HASFFI
-static void asm_conv64(ASMState *as, IRIns *ir)
-{
-  IRType st = (IRType)((ir-1)->op2 & IRCONV_SRCMASK);
-  IRType dt = (((ir-1)->op2 & IRCONV_DSTMASK) >> IRCONV_DSH);
-  IRCallID id;
-  const CCallInfo *ci;
-  IRRef args[2];
-  args[0] = ir->op1;
-  args[1] = (ir-1)->op1;
-  if (st == IRT_NUM || st == IRT_FLOAT) {
-    id = IRCALL_fp64_d2l + ((st == IRT_FLOAT) ? 2 : 0) + (dt - IRT_I64);
-    ir--;
-  } else {
-    id = IRCALL_fp64_l2d + ((dt == IRT_FLOAT) ? 2 : 0) + (st - IRT_I64);
-  }
-  ci = &lj_ir_callinfo[id];
-  asm_setupresult(as, ir, ci);
-  asm_gencall(as, ci, args);
-}
-#endif
 static void asm_strto(ASMState *as, IRIns *ir)
 {
  const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_strscan_num];
@@ -550,6 +509,8 @@ static void asm_strto(ASMState *as, IRIns *ir)
  emit_tai(as, PPCI_ADDI, ra_releasetmp(as, ASMREF_TMP1), RID_SP, ofs);
 }
+/* -- Memory references --------------------------------------------------- */
 /* Get pointer to TValue. */
 static void asm_tvptr(ASMState *as, Reg dest, IRRef ref)
 {
@@ -573,27 +534,6 @@ static void asm_tvptr(ASMState *as, Reg dest, IRRef ref)
  }
 }
-static void asm_tostr(ASMState *as, IRIns *ir)
-{
-  IRRef args[2];
-  args[0] = ASMREF_L;
-  as->gcsteps++;
-  if (irt_isnum(IR(ir->op1)->t) || (ir+1)->o == IR_HIOP) {
-    const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_str_fromnum];
-    args[1] = ASMREF_TMP1;  /* const lua_Number * */
-    asm_setupresult(as, ir, ci);  /* GCstr * */
-    asm_gencall(as, ci, args);
-    asm_tvptr(as, ra_releasetmp(as, ASMREF_TMP1), ir->op1);
-  } else {
-    const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_str_fromint];
-    args[1] = ir->op1;  /* int32_t k */
-    asm_setupresult(as, ir, ci);  /* GCstr * */
-    asm_gencall(as, ci, args);
-  }
-}
-/* -- Memory references --------------------------------------------------- */
 static void asm_aref(ASMState *as, IRIns *ir)
 {
  Reg dest = ra_dest(as, ir, RSET_GPR);
@@ -770,20 +710,6 @@ static void asm_hrefk(ASMState *as, IRIns *ir)
  }
 }
-static void asm_newref(ASMState *as, IRIns *ir)
-{
-  const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_tab_newkey];
-  IRRef args[3];
-  if (ir->r == RID_SINK)
-    return;
-  args[0] = ASMREF_L;     /* lua_State *L */
-  args[1] = ir->op1;      /* GCtab *t     */
-  args[2] = ASMREF_TMP1;  /* cTValue *key */
-  asm_setupresult(as, ir, ci);  /* TValue * */
-  asm_gencall(as, ci, args);
-  asm_tvptr(as, ra_releasetmp(as, ASMREF_TMP1), ir->op2);
-}
 static void asm_uref(ASMState *as, IRIns *ir)
 {
  /* NYI: Check that UREFO is still open and not aliasing a slot. */
@@ -914,7 +840,7 @@ static void asm_xload(ASMState *as, IRIns *ir)
  asm_fusexref(as, asm_fxloadins(ir), dest, ir->op1, RSET_GPR, 0);
 }
-static void asm_xstore(ASMState *as, IRIns *ir, int32_t ofs)
+static void asm_xstore_(ASMState *as, IRIns *ir, int32_t ofs)
 {
  IRIns *irb;
  if (ir->r == RID_SINK)
@@ -931,6 +857,8 @@ static void asm_xstore(ASMState *as, IRIns *ir, int32_t ofs)
  }
 }
+#define asm_xstore(as, ir)      asm_xstore_(as, ir, 0)
 static void asm_ahuvload(ASMState *as, IRIns *ir)
 {
  IRType1 t = ir->t;
@@ -1081,19 +1009,15 @@ dotypecheck:
 static void asm_cnew(ASMState *as, IRIns *ir)
 {
  CTState *cts = ctype_ctsG(J2G(as->J));
-  CTypeID ctypeid = (CTypeID)IR(ir->op1)->i;
+  CTypeID id = (CTypeID)IR(ir->op1)->i;
-  CTSize sz = (ir->o == IR_CNEWI || ir->op2 == REF_NIL) ?
+  CTSize sz;
-              lj_ctype_size(cts, ctypeid) : (CTSize)IR(ir->op2)->i;
+  CTInfo info = lj_ctype_info(cts, id, &sz);
  const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_mem_newgco];
-  IRRef args[2];
+  IRRef args[4];
-  RegSet allow = (RSET_GPR & ~RSET_SCRATCH);
  RegSet drop = RSET_SCRATCH;
-  lua_assert(sz != CTSIZE_INVALID);
+  lua_assert(sz != CTSIZE_INVALID || (ir->o == IR_CNEW && ir->op2 != REF_NIL));
-  args[0] = ASMREF_L;     /* lua_State *L */
-  args[1] = ASMREF_TMP1;  /* MSize size   */
  as->gcsteps++;
  if (ra_hasreg(ir->r))
    rset_clear(drop, ir->r);  /* Dest reg handled below. */
  ra_evictset(as, drop);
@@ -1102,6 +1026,7 @@ static void asm_cnew(ASMState *as, IRIns *ir)
  /* Initialize immutable cdata object. */
  if (ir->o == IR_CNEWI) {
+    RegSet allow = (RSET_GPR & ~RSET_SCRATCH);
    int32_t ofs = sizeof(GCcdata);
    lua_assert(sz == 4 || sz == 8);
    if (sz == 8) {
@@ -1115,12 +1040,24 @@ static void asm_cnew(ASMState *as, IRIns *ir)
      if (ofs == sizeof(GCcdata)) break;
      ofs -= 4; ir++;
    }
+  } else if (ir->op2 != REF_NIL) {  /* Create VLA/VLS/aligned cdata. */
+    ci = &lj_ir_callinfo[IRCALL_lj_cdata_newv];
+    args[0] = ASMREF_L;     /* lua_State *L */
+    args[1] = ir->op1;      /* CTypeID id   */
+    args[2] = ir->op2;      /* CTSize sz    */
+    args[3] = ASMREF_TMP1;  /* CTSize align */
+    asm_gencall(as, ci, args);
+    emit_loadi(as, ra_releasetmp(as, ASMREF_TMP1), (int32_t)ctype_align(info));
+    return;
  }
  /* Initialize gct and ctypeid. lj_mem_newgco() already sets marked. */
  emit_tai(as, PPCI_STB, RID_RET+1, RID_RET, offsetof(GCcdata, gct));
  emit_tai(as, PPCI_STH, RID_TMP, RID_RET, offsetof(GCcdata, ctypeid));
  emit_ti(as, PPCI_LI, RID_RET+1, ~LJ_TCDATA);
-  emit_ti(as, PPCI_LI, RID_TMP, ctypeid);  /* Lower 16 bit used. Sign-ext ok. */
+  emit_ti(as, PPCI_LI, RID_TMP, id);  /* Lower 16 bit used. Sign-ext ok. */
+  args[0] = ASMREF_L;     /* lua_State *L */
+  args[1] = ASMREF_TMP1;  /* MSize size   */
  asm_gencall(as, ci, args);
  ra_allockreg(as, (int32_t)(sz+sizeof(GCcdata)),
               ra_releasetmp(as, ASMREF_TMP1));
@@ -1194,23 +1131,14 @@ static void asm_fpunary(ASMState *as, IRIns *ir, PPCIns pi)
  emit_fb(as, pi, dest, left);
 }
-static int asm_fpjoin_pow(ASMState *as, IRIns *ir)
+static void asm_fpmath(ASMState *as, IRIns *ir)
-{
+{
-  IRIns *irp = IR(ir->op1);
+  if (ir->op2 == IRFPM_EXP2 && asm_fpjoin_pow(as, ir))
-  if (irp == ir-1 && irp->o == IR_MUL && !ra_used(irp)) {
+    return;
-    IRIns *irpp = IR(irp->op1);
+  if (ir->op2 == IRFPM_SQRT && (as->flags & JIT_F_SQRT))
-    if (irpp == ir-2 && irpp->o == IR_FPMATH &&
+    asm_fpunary(as, ir, PPCI_FSQRT);
-        irpp->op2 == IRFPM_LOG2 && !ra_used(irpp)) {
+  else
-      const CCallInfo *ci = &lj_ir_callinfo[IRCALL_pow];
+    asm_callid(as, ir, IRCALL_lj_vm_floor + ir->op2);
-      IRRef args[2];
-      args[0] = irpp->op1;
-      args[1] = irp->op2;
-      asm_setupresult(as, ir, ci);
-      asm_gencall(as, ci, args);
-      return 1;
-    }
-  }
-  return 0;
 }
 static void asm_add(ASMState *as, IRIns *ir)
@@ -1310,6 +1238,10 @@ static void asm_mul(ASMState *as, IRIns *ir)
  }
 }
+#define asm_div(as, ir)         asm_fparith(as, ir, PPCI_FDIV)
+#define asm_mod(as, ir)         asm_callid(as, ir, IRCALL_lj_vm_modi)
+#define asm_pow(as, ir)         asm_callid(as, ir, IRCALL_lj_vm_powi)
 static void asm_neg(ASMState *as, IRIns *ir)
 {
  if (irt_isnum(ir->t)) {
@@ -1328,6 +1260,10 @@ static void asm_neg(ASMState *as, IRIns *ir)
  }
 }
+#define asm_abs(as, ir)         asm_fpunary(as, ir, PPCI_FABS)
+#define asm_atan2(as, ir)       asm_callid(as, ir, IRCALL_atan2)
+#define asm_ldexp(as, ir)       asm_callid(as, ir, IRCALL_ldexp)
 static void asm_arithov(ASMState *as, IRIns *ir, PPCIns pi)
 {
  Reg dest, left, right;
@@ -1343,6 +1279,10 @@ static void asm_arithov(ASMState *as, IRIns *ir, PPCIns pi)
  emit_tab(as, pi|PPCF_DOT, dest, left, right);
 }
+#define asm_addov(as, ir)       asm_arithov(as, ir, PPCI_ADDO)
+#define asm_subov(as, ir)       asm_arithov(as, ir, PPCI_SUBFO)
+#define asm_mulov(as, ir)       asm_arithov(as, ir, PPCI_MULLWO)
 #if LJ_HASFFI
 static void asm_add64(ASMState *as, IRIns *ir)
 {
@@ -1422,7 +1362,7 @@ static void asm_neg64(ASMState *as, IRIns *ir)
 }
 #endif
-static void asm_bitnot(ASMState *as, IRIns *ir)
+static void asm_bnot(ASMState *as, IRIns *ir)
 {
  Reg dest, left, right;
  PPCIns pi = PPCI_NOR;
@@ -1449,7 +1389,7 @@ nofuse:
  emit_asb(as, pi, dest, left, right);
 }
-static void asm_bitswap(ASMState *as, IRIns *ir)
+static void asm_bswap(ASMState *as, IRIns *ir)
 {
  Reg dest = ra_dest(as, ir, RSET_GPR);
  IRIns *irx;
@@ -1470,32 +1410,6 @@ static void asm_bitswap(ASMState *as, IRIns *ir)
  }
 }
-static void asm_bitop(ASMState *as, IRIns *ir, PPCIns pi, PPCIns pik)
-{
-  Reg dest = ra_dest(as, ir, RSET_GPR);
-  Reg right, left = ra_hintalloc(as, ir->op1, dest, RSET_GPR);
-  if (irref_isk(ir->op2)) {
-    int32_t k = IR(ir->op2)->i;
-    Reg tmp = left;
-    if ((checku16(k) || (k & 0xffff) == 0) || (tmp = dest, !as->sectref)) {
-      if (!checku16(k)) {
-        emit_asi(as, pik ^ (PPCI_ORI ^ PPCI_ORIS), dest, tmp, (k >> 16));
-        if ((k & 0xffff) == 0) return;
-      }
-      emit_asi(as, pik, dest, left, k);
-      return;
-    }
-  }
-  /* May fail due to spills/restores above, but simplifies the logic. */
-  if (as->flagmcp == as->mcp) {
-    as->flagmcp = NULL;
-    as->mcp++;
-    pi |= PPCF_DOT;
-  }
-  right = ra_alloc1(as, ir->op2, rset_exclude(RSET_GPR, left));
-  emit_asb(as, pi, dest, left, right);
-}
 /* Fuse BAND with contiguous bitmask and a shift to rlwinm. */
 static void asm_fuseandsh(ASMState *as, PPCIns pi, int32_t mask, IRRef ref)
 {
@@ -1526,7 +1440,7 @@ nofuse:
  *--as->mcp = pi | PPCF_T(left);
 }
-static void asm_bitand(ASMState *as, IRIns *ir)
+static void asm_band(ASMState *as, IRIns *ir)
 {
  Reg dest, left, right;
  IRRef lref = ir->op1;
@@ -1581,6 +1495,35 @@ static void asm_bitand(ASMState *as, IRIns *ir)
  emit_asb(as, PPCI_AND ^ dot, dest, left, right);
 }
+static void asm_bitop(ASMState *as, IRIns *ir, PPCIns pi, PPCIns pik)
+{
+  Reg dest = ra_dest(as, ir, RSET_GPR);
+  Reg right, left = ra_hintalloc(as, ir->op1, dest, RSET_GPR);
+  if (irref_isk(ir->op2)) {
+    int32_t k = IR(ir->op2)->i;
+    Reg tmp = left;
+    if ((checku16(k) || (k & 0xffff) == 0) || (tmp = dest, !as->sectref)) {
+      if (!checku16(k)) {
+        emit_asi(as, pik ^ (PPCI_ORI ^ PPCI_ORIS), dest, tmp, (k >> 16));
+        if ((k & 0xffff) == 0) return;
+      }
+      emit_asi(as, pik, dest, left, k);
+      return;
+    }
+  }
+  /* May fail due to spills/restores above, but simplifies the logic. */
+  if (as->flagmcp == as->mcp) {
+    as->flagmcp = NULL;
+    as->mcp++;
+    pi |= PPCF_DOT;
+  }
+  right = ra_alloc1(as, ir->op2, rset_exclude(RSET_GPR, left));
+  emit_asb(as, pi, dest, left, right);
+}
+#define asm_bor(as, ir)         asm_bitop(as, ir, PPCI_OR, PPCI_ORI)
+#define asm_bxor(as, ir)        asm_bitop(as, ir, PPCI_XOR, PPCI_XORI)
 static void asm_bitshift(ASMState *as, IRIns *ir, PPCIns pi, PPCIns pik)
 {
  Reg dest, left;
@@ -1606,6 +1549,14 @@ static void asm_bitshift(ASMState *as, IRIns *ir, PPCIns pi, PPCIns pik)
  }
 }
+#define asm_bshl(as, ir)        asm_bitshift(as, ir, PPCI_SLW, 0)
+#define asm_bshr(as, ir)        asm_bitshift(as, ir, PPCI_SRW, 1)
+#define asm_bsar(as, ir)        asm_bitshift(as, ir, PPCI_SRAW, PPCI_SRAWI)
+#define asm_brol(as, ir) \
+  asm_bitshift(as, ir, PPCI_RLWNM|PPCF_MB(0)|PPCF_ME(31), \
+                       PPCI_RLWINM|PPCF_MB(0)|PPCF_ME(31))
+#define asm_bror(as, ir)        lua_assert(0)
 static void asm_min_max(ASMState *as, IRIns *ir, int ismax)
 {
  if (irt_isnum(ir->t)) {
@@ -1636,6 +1587,9 @@ static void asm_min_max(ASMState *as, IRIns *ir, int ismax)
  }
 }
+#define asm_min(as, ir)         asm_min_max(as, ir, 0)
+#define asm_max(as, ir)         asm_min_max(as, ir, 1)
 /* -- Comparisons --------------------------------------------------------- */
 #define CC_UNSIGNED     0x08    /* Unsigned integer comparison. */
@@ -1712,6 +1666,8 @@ static void asm_comp(ASMState *as, IRIns *ir)
  }
 }
+#define asm_equal(as, ir)       asm_comp(as, ir)
 #if LJ_HASFFI
 /* 64 bit integer comparisons. */
 static void asm_comp64(ASMState *as, IRIns *ir)
@@ -1757,8 +1713,8 @@ static void asm_hiop(ASMState *as, IRIns *ir)
  } else if ((ir-1)->o == IR_XSTORE) {
    as->curins--;  /* Handle both stores here. */
    if ((ir-1)->r != RID_SINK) {
-      asm_xstore(as, ir, 0);
+      asm_xstore_(as, ir, 0);
-      asm_xstore(as, ir-1, 4);
+      asm_xstore_(as, ir-1, 4);
    }
    return;
  }
@@ -1964,136 +1920,13 @@ static void asm_tail_prep(ASMState *as)
  }
 }
-/* -- Instruction dispatch ------------------------------------------------ */
-/* Assemble a single instruction. */
-static void asm_ir(ASMState *as, IRIns *ir)
-{
-  switch ((IROp)ir->o) {
-  /* Miscellaneous ops. */
-  case IR_LOOP: asm_loop(as); break;
-  case IR_NOP: case IR_XBAR: lua_assert(!ra_used(ir)); break;
-  case IR_USE:
-    ra_alloc1(as, ir->op1, irt_isfp(ir->t) ? RSET_FPR : RSET_GPR); break;
-  case IR_PHI: asm_phi(as, ir); break;
-  case IR_HIOP: asm_hiop(as, ir); break;
-  case IR_GCSTEP: asm_gcstep(as, ir); break;
-  /* Guarded assertions. */
-  case IR_EQ: case IR_NE:
-    if ((ir-1)->o == IR_HREF && ir->op1 == as->curins-1) {
-      as->curins--;
-      asm_href(as, ir-1, (IROp)ir->o);
-      break;
-    }
-    /* fallthrough */
-  case IR_LT: case IR_GE: case IR_LE: case IR_GT:
-  case IR_ULT: case IR_UGE: case IR_ULE: case IR_UGT:
-  case IR_ABC:
-    asm_comp(as, ir);
-    break;
-  case IR_RETF: asm_retf(as, ir); break;
-  /* Bit ops. */
-  case IR_BNOT: asm_bitnot(as, ir); break;
-  case IR_BSWAP: asm_bitswap(as, ir); break;
-  case IR_BAND: asm_bitand(as, ir); break;
-  case IR_BOR:  asm_bitop(as, ir, PPCI_OR, PPCI_ORI); break;
-  case IR_BXOR: asm_bitop(as, ir, PPCI_XOR, PPCI_XORI); break;
-  case IR_BSHL: asm_bitshift(as, ir, PPCI_SLW, 0); break;
-  case IR_BSHR: asm_bitshift(as, ir, PPCI_SRW, 1); break;
-  case IR_BSAR: asm_bitshift(as, ir, PPCI_SRAW, PPCI_SRAWI); break;
-  case IR_BROL: asm_bitshift(as, ir, PPCI_RLWNM|PPCF_MB(0)|PPCF_ME(31),
-                             PPCI_RLWINM|PPCF_MB(0)|PPCF_ME(31)); break;
-  case IR_BROR: lua_assert(0); break;
-  /* Arithmetic ops. */
-  case IR_ADD: asm_add(as, ir); break;
-  case IR_SUB: asm_sub(as, ir); break;
-  case IR_MUL: asm_mul(as, ir); break;
-  case IR_DIV: asm_fparith(as, ir, PPCI_FDIV); break;
-  case IR_MOD: asm_callid(as, ir, IRCALL_lj_vm_modi); break;
-  case IR_POW: asm_callid(as, ir, IRCALL_lj_vm_powi); break;
-  case IR_NEG: asm_neg(as, ir); break;
-  case IR_ABS: asm_fpunary(as, ir, PPCI_FABS); break;
-  case IR_ATAN2: asm_callid(as, ir, IRCALL_atan2); break;
-  case IR_LDEXP: asm_callid(as, ir, IRCALL_ldexp); break;
-  case IR_MIN: asm_min_max(as, ir, 0); break;
-  case IR_MAX: asm_min_max(as, ir, 1); break;
-  case IR_FPMATH:
-    if (ir->op2 == IRFPM_EXP2 && asm_fpjoin_pow(as, ir))
-      break;
-    if (ir->op2 == IRFPM_SQRT && (as->flags & JIT_F_SQRT))
-      asm_fpunary(as, ir, PPCI_FSQRT);
-    else
-      asm_callid(as, ir, IRCALL_lj_vm_floor + ir->op2);
-    break;
-  /* Overflow-checking arithmetic ops. */
-  case IR_ADDOV: asm_arithov(as, ir, PPCI_ADDO); break;
-  case IR_SUBOV: asm_arithov(as, ir, PPCI_SUBFO); break;
-  case IR_MULOV: asm_arithov(as, ir, PPCI_MULLWO); break;
-  /* Memory references. */
-  case IR_AREF: asm_aref(as, ir); break;
-  case IR_HREF: asm_href(as, ir, 0); break;
-  case IR_HREFK: asm_hrefk(as, ir); break;
-  case IR_NEWREF: asm_newref(as, ir); break;
-  case IR_UREFO: case IR_UREFC: asm_uref(as, ir); break;
-  case IR_FREF: asm_fref(as, ir); break;
-  case IR_STRREF: asm_strref(as, ir); break;
-  /* Loads and stores. */
-  case IR_ALOAD: case IR_HLOAD: case IR_ULOAD: case IR_VLOAD:
-    asm_ahuvload(as, ir);
-    break;
-  case IR_FLOAD: asm_fload(as, ir); break;
-  case IR_XLOAD: asm_xload(as, ir); break;
-  case IR_SLOAD: asm_sload(as, ir); break;
-  case IR_ASTORE: case IR_HSTORE: case IR_USTORE: asm_ahustore(as, ir); break;
-  case IR_FSTORE: asm_fstore(as, ir); break;
-  case IR_XSTORE: asm_xstore(as, ir, 0); break;
-  /* Allocations. */
-  case IR_SNEW: case IR_XSNEW: asm_snew(as, ir); break;
-  case IR_TNEW: asm_tnew(as, ir); break;
-  case IR_TDUP: asm_tdup(as, ir); break;
-  case IR_CNEW: case IR_CNEWI: asm_cnew(as, ir); break;
-  /* Write barriers. */
-  case IR_TBAR: asm_tbar(as, ir); break;
-  case IR_OBAR: asm_obar(as, ir); break;
-  /* Type conversions. */
-  case IR_CONV: asm_conv(as, ir); break;
-  case IR_TOBIT: asm_tobit(as, ir); break;
-  case IR_TOSTR: asm_tostr(as, ir); break;
-  case IR_STRTO: asm_strto(as, ir); break;
-  /* Calls. */
-  case IR_CALLN: case IR_CALLL: case IR_CALLS: asm_call(as, ir); break;
-  case IR_CALLXS: asm_callx(as, ir); break;
-  case IR_CARG: break;
-  default:
-    setintV(&as->J->errinfo, ir->o);
-    lj_trace_err_info(as->J, LJ_TRERR_NYIIR);
-    break;
-  }
-}
 /* -- Trace setup --------------------------------------------------------- */
 /* Ensure there are enough stack slots for call arguments. */
 static Reg asm_setup_call_slots(ASMState *as, IRIns *ir, const CCallInfo *ci)
 {
  IRRef args[CCI_NARGS_MAX*2];
-  uint32_t i, nargs = (int)CCI_NARGS(ci);
+  uint32_t i, nargs = CCI_XNARGS(ci);
  int nslots = 2, ngpr = REGARG_NUMGPR, nfpr = REGARG_NUMFPR;
  asm_collectargs(as, ir, ci, args);
  for (i = 0; i < nargs; i++)
diff --git a/src/lj_asm_x86.h b/src/lj_asm_x86.h
index 64441ccb..f2f8157d 100644
--- a/src/lj_asm_x86.h
+++ b/src/lj_asm_x86.h
@@ -384,7 +384,7 @@ static Reg asm_fuseloadm(ASMState *as, IRRef ref, RegSet allow, int is64)
 /* Count the required number of stack slots for a call. */
 static int asm_count_call_slots(ASMState *as, const CCallInfo *ci, IRRef *args)
 {
-  uint32_t i, nargs = CCI_NARGS(ci);
+  uint32_t i, nargs = CCI_XNARGS(ci);
  int nslots = 0;
 #if LJ_64
  if (LJ_ABI_WIN) {
@@ -417,7 +417,7 @@ static int asm_count_call_slots(ASMState *as, const CCallInfo *ci, IRRef *args)
 /* Generate a call to a C function. */
 static void asm_gencall(ASMState *as, const CCallInfo *ci, IRRef *args)
 {
-  uint32_t n, nargs = CCI_NARGS(ci);
+  uint32_t n, nargs = CCI_XNARGS(ci);
  int32_t ofs = STACKARG_OFS;
 #if LJ_64
  uint32_t gprs = REGARG_GPRS;
@@ -552,7 +552,7 @@ static void asm_setupresult(ASMState *as, IRIns *ir, const CCallInfo *ci)
      if (ra_hasreg(dest)) {
        ra_free(as, dest);
        ra_modified(as, dest);
-        emit_rmro(as, irt_isnum(ir->t) ? XMM_MOVRM(as) : XO_MOVSS,
+        emit_rmro(as, irt_isnum(ir->t) ? XO_MOVSD : XO_MOVSS,
                  dest, RID_ESP, ofs);
      }
      if ((ci->flags & CCI_CASTU64)) {
@@ -576,15 +576,6 @@ static void asm_setupresult(ASMState *as, IRIns *ir, const CCallInfo *ci)
  }
 }
-static void asm_call(ASMState *as, IRIns *ir)
-{
-  IRRef args[CCI_NARGS_MAX];
-  const CCallInfo *ci = &lj_ir_callinfo[ir->op2];
-  asm_collectargs(as, ir, ci, args);
-  asm_setupresult(as, ir, ci);
-  asm_gencall(as, ci, args);
-}
 /* Return a constant function pointer or NULL for indirect calls. */
 static void *asm_callx_func(ASMState *as, IRIns *irf, IRRef func)
 {
@@ -663,8 +654,7 @@ static void asm_tointg(ASMState *as, IRIns *ir, Reg left)
  asm_guardcc(as, CC_NE);
  emit_rr(as, XO_UCOMISD, left, tmp);
  emit_rr(as, XO_CVTSI2SD, tmp, dest);
-  if (!(as->flags & JIT_F_SPLIT_XMM))
+  emit_rr(as, XO_XORPS, tmp, tmp);  /* Avoid partial register stall. */
-    emit_rr(as, XO_XORPS, tmp, tmp);  /* Avoid partial register stall. */
  emit_rr(as, XO_CVTTSD2SI, dest, left);
  /* Can't fuse since left is needed twice. */
 }
@@ -720,8 +710,7 @@ static void asm_conv(ASMState *as, IRIns *ir)
      emit_mrm(as, irt_isnum(ir->t) ? XO_CVTSI2SD : XO_CVTSI2SS,
               dest|((LJ_64 && (st64 || st == IRT_U32)) ? REX_64 : 0), left);
    }
-    if (!(as->flags & JIT_F_SPLIT_XMM))
+    emit_rr(as, XO_XORPS, dest, dest);  /* Avoid partial register stall. */
-      emit_rr(as, XO_XORPS, dest, dest);  /* Avoid partial register stall. */
  } else if (stfp) {  /* FP to integer conversion. */
    if (irt_isguard(ir->t)) {
      /* Checked conversions are only supported from number to int. */
@@ -729,9 +718,7 @@ static void asm_conv(ASMState *as, IRIns *ir)
      asm_tointg(as, ir, ra_alloc1(as, lref, RSET_FPR));
    } else {
      Reg dest = ra_dest(as, ir, RSET_GPR);
-      x86Op op = st == IRT_NUM ?
+      x86Op op = st == IRT_NUM ? XO_CVTTSD2SI : XO_CVTTSS2SI;
-                 ((ir->op2 & IRCONV_TRUNC) ? XO_CVTTSD2SI : XO_CVTSD2SI) :
-                 ((ir->op2 & IRCONV_TRUNC) ? XO_CVTTSS2SI : XO_CVTSS2SI);
      if (LJ_64 ? irt_isu64(ir->t) : irt_isu32(ir->t)) {
        /* LJ_64: For inputs >= 2^63 add -2^64, convert again. */
        /* LJ_32: For inputs >= 2^31 add -2^31, convert again and add 2^31. */
@@ -825,8 +812,7 @@ static void asm_conv_fp_int64(ASMState *as, IRIns *ir)
  if (ra_hasreg(dest)) {
    ra_free(as, dest);
    ra_modified(as, dest);
-    emit_rmro(as, irt_isnum(ir->t) ? XMM_MOVRM(as) : XO_MOVSS,
+    emit_rmro(as, irt_isnum(ir->t) ? XO_MOVSD : XO_MOVSS, dest, RID_ESP, ofs);
-              dest, RID_ESP, ofs);
  }
  emit_rmro(as, irt_isnum(ir->t) ? XO_FSTPq : XO_FSTPd,
            irt_isnum(ir->t) ? XOg_FSTPq : XOg_FSTPd, RID_ESP, ofs);
@@ -854,7 +840,6 @@ static void asm_conv_int64_fp(ASMState *as, IRIns *ir)
  Reg lo, hi;
  lua_assert(st == IRT_NUM || st == IRT_FLOAT);
  lua_assert(dt == IRT_I64 || dt == IRT_U64);
-  lua_assert(((ir-1)->op2 & IRCONV_TRUNC));
  hi = ra_dest(as, ir, RSET_GPR);
  lo = ra_dest(as, ir-1, rset_exclude(RSET_GPR, hi));
  if (ra_used(ir-1)) emit_rmro(as, XO_MOV, lo, RID_ESP, 0);
@@ -897,6 +882,14 @@ static void asm_conv_int64_fp(ASMState *as, IRIns *ir)
           st == IRT_NUM ? XOg_FLDq: XOg_FLDd,
           asm_fuseload(as, ir->op1, RSET_EMPTY));
 }
+static void asm_conv64(ASMState *as, IRIns *ir)
+{
+  if (irt_isfp(ir->t))
+    asm_conv_fp_int64(as, ir);
+  else
+    asm_conv_int64_fp(as, ir);
+}
 #endif
 static void asm_strto(ASMState *as, IRIns *ir)
@@ -918,29 +911,32 @@ static void asm_strto(ASMState *as, IRIns *ir)
            RID_ESP, sps_scale(ir->s));
 }
-static void asm_tostr(ASMState *as, IRIns *ir)
+/* -- Memory references --------------------------------------------------- */
+/* Get pointer to TValue. */
+static void asm_tvptr(ASMState *as, Reg dest, IRRef ref)
 {
-  IRIns *irl = IR(ir->op1);
+  IRIns *ir = IR(ref);
-  IRRef args[2];
+  if (irt_isnum(ir->t)) {
-  args[0] = ASMREF_L;
+    /* For numbers use the constant itself or a spill slot as a TValue. */
-  as->gcsteps++;
+    if (irref_isk(ref))
-  if (irt_isnum(irl->t)) {
+      emit_loada(as, dest, ir_knum(ir));
-    const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_str_fromnum];
+    else
-    args[1] = ASMREF_TMP1;  /* const lua_Number * */
+      emit_rmro(as, XO_LEA, dest|REX_64, RID_ESP, ra_spill(as, ir));
-    asm_setupresult(as, ir, ci);  /* GCstr * */
-    asm_gencall(as, ci, args);
-    emit_rmro(as, XO_LEA, ra_releasetmp(as, ASMREF_TMP1)|REX_64,
-              RID_ESP, ra_spill(as, irl));
  } else {
-    const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_str_fromint];
+    /* Otherwise use g->tmptv to hold the TValue. */
-    args[1] = ir->op1;  /* int32_t k */
+    if (!irref_isk(ref)) {
-    asm_setupresult(as, ir, ci);  /* GCstr * */
+      Reg src = ra_alloc1(as, ref, rset_exclude(RSET_GPR, dest));
-    asm_gencall(as, ci, args);
+      emit_movtomro(as, REX_64IR(ir, src), dest, 0);
+    } else if (!irt_ispri(ir->t)) {
+      emit_movmroi(as, dest, 0, ir->i);
+    }
+    if (!(LJ_64 && irt_islightud(ir->t)))
+      emit_movmroi(as, dest, 4, irt_toitype(ir->t));
+    emit_loada(as, dest, &J2G(as->J)->tmptv);
  }
 }
-/* -- Memory references --------------------------------------------------- */
 static void asm_aref(ASMState *as, IRIns *ir)
 {
  Reg dest = ra_dest(as, ir, RSET_GPR);
@@ -951,23 +947,6 @@ static void asm_aref(ASMState *as, IRIns *ir)
    emit_rr(as, XO_MOV, dest, as->mrm.base);
 }
-/* Merge NE(HREF, niltv) check. */
-static MCode *merge_href_niltv(ASMState *as, IRIns *ir)
-{
-  /* Assumes nothing else generates NE of HREF. */
-  if ((ir[1].o == IR_NE || ir[1].o == IR_EQ) && ir[1].op1 == as->curins &&
-      ra_hasreg(ir->r)) {
-    MCode *p = as->mcp;
-    p += (LJ_64 && *p != XI_ARITHi) ? 7+6 : 6+6;
-    /* Ensure no loop branch inversion happened. */
-    if (p[-6] == 0x0f && p[-5] == XI_JCCn+(CC_NE^(ir[1].o & 1))) {
-      as->mcp = p;  /* Kill cmp reg, imm32 + jz exit. */
-      return p + *(int32_t *)(p-4);  /* Return exit address. */
-    }
-  }
-  return NULL;
-}
 /* Inlined hash lookup. Specialized for key type and for const keys.
 ** The equivalent C code is:
 **   Node *n = hashkey(t, key);
@@ -976,10 +955,10 @@ static MCode *merge_href_niltv(ASMState *as, IRIns *ir)
 **   } while ((n = nextnode(n)));
 **   return niltv(L);
 */
-static void asm_href(ASMState *as, IRIns *ir)
+static void asm_href(ASMState *as, IRIns *ir, IROp merge)
 {
-  MCode *nilexit = merge_href_niltv(as, ir);  /* Do this before any restores. */
  RegSet allow = RSET_GPR;
+  int destused = ra_used(ir);
  Reg dest = ra_dest(as, ir, allow);
  Reg tab = ra_alloc1(as, ir->op1, rset_clear(allow, dest));
  Reg key = RID_NONE, tmp = RID_NONE;
@@ -996,14 +975,12 @@ static void asm_href(ASMState *as, IRIns *ir)
      tmp = ra_scratch(as, rset_exclude(allow, key));
  }
-  /* Key not found in chain: jump to exit (if merged with NE) or load niltv. */
+  /* Key not found in chain: jump to exit (if merged) or load niltv. */
  l_end = emit_label(as);
-  if (nilexit && ir[1].o == IR_NE) {
+  if (merge == IR_NE)
-    emit_jcc(as, CC_E, nilexit);  /* XI_JMP is not found by lj_asm_patchexit. */
+    asm_guardcc(as, CC_E);  /* XI_JMP is not found by lj_asm_patchexit. */
-    nilexit = NULL;
+  else if (destused)
-  } else {
    emit_loada(as, dest, niltvg(J2G(as->J)));
-  }
  /* Follow hash chain until the end. */
  l_loop = emit_sjcc_label(as, CC_NZ);
@@ -1012,8 +989,8 @@ static void asm_href(ASMState *as, IRIns *ir)
  l_next = emit_label(as);
  /* Type and value comparison. */
-  if (nilexit)
+  if (merge == IR_EQ)
-    emit_jcc(as, CC_E, nilexit);
+    asm_guardcc(as, CC_E);
  else
    emit_sjcc(as, CC_E, l_end);
  if (irt_isnum(kt)) {
@@ -1169,41 +1146,6 @@ static void asm_hrefk(ASMState *as, IRIns *ir)
 #endif
 }
-static void asm_newref(ASMState *as, IRIns *ir)
-{
-  const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_tab_newkey];
-  IRRef args[3];
-  IRIns *irkey;
-  Reg tmp;
-  if (ir->r == RID_SINK)
-    return;
-  args[0] = ASMREF_L;     /* lua_State *L */
-  args[1] = ir->op1;      /* GCtab *t     */
-  args[2] = ASMREF_TMP1;  /* cTValue *key */
-  asm_setupresult(as, ir, ci);  /* TValue * */
-  asm_gencall(as, ci, args);
-  tmp = ra_releasetmp(as, ASMREF_TMP1);
-  irkey = IR(ir->op2);
-  if (irt_isnum(irkey->t)) {
-    /* For numbers use the constant itself or a spill slot as a TValue. */
-    if (irref_isk(ir->op2))
-      emit_loada(as, tmp, ir_knum(irkey));
-    else
-      emit_rmro(as, XO_LEA, tmp|REX_64, RID_ESP, ra_spill(as, irkey));
-  } else {
-    /* Otherwise use g->tmptv to hold the TValue. */
-    if (!irref_isk(ir->op2)) {
-      Reg src = ra_alloc1(as, ir->op2, rset_exclude(RSET_GPR, tmp));
-      emit_movtomro(as, REX_64IR(irkey, src), tmp, 0);
-    } else if (!irt_ispri(irkey->t)) {
-      emit_movmroi(as, tmp, 0, irkey->i);
-    }
-    if (!(LJ_64 && irt_islightud(irkey->t)))
-      emit_movmroi(as, tmp, 4, irt_toitype(irkey->t));
-    emit_loada(as, tmp, &J2G(as->J)->tmptv);
-  }
-}
 static void asm_uref(ASMState *as, IRIns *ir)
 {
  /* NYI: Check that UREFO is still open and not aliasing a slot. */
@@ -1263,7 +1205,7 @@ static void asm_fxload(ASMState *as, IRIns *ir)
  case IRT_U8: xo = XO_MOVZXb; break;
  case IRT_I16: xo = XO_MOVSXw; break;
  case IRT_U16: xo = XO_MOVZXw; break;
-  case IRT_NUM: xo = XMM_MOVRM(as); break;
+  case IRT_NUM: xo = XO_MOVSD; break;
  case IRT_FLOAT: xo = XO_MOVSS; break;
  default:
    if (LJ_64 && irt_is64(ir->t))
@@ -1276,6 +1218,9 @@ static void asm_fxload(ASMState *as, IRIns *ir)
  emit_mrm(as, xo, dest, RID_MRM);
 }
+#define asm_fload(as, ir)       asm_fxload(as, ir)
+#define asm_xload(as, ir)       asm_fxload(as, ir)
 static void asm_fxstore(ASMState *as, IRIns *ir)
 {
  RegSet allow = RSET_GPR;
@@ -1339,6 +1284,9 @@ static void asm_fxstore(ASMState *as, IRIns *ir)
  }
 }
+#define asm_fstore(as, ir)      asm_fxstore(as, ir)
+#define asm_xstore(as, ir)      asm_fxstore(as, ir)
 #if LJ_64
 static Reg asm_load_lightud64(ASMState *as, IRIns *ir, int typecheck)
 {
@@ -1377,7 +1325,7 @@ static void asm_ahuvload(ASMState *as, IRIns *ir)
    RegSet allow = irt_isnum(ir->t) ? RSET_FPR : RSET_GPR;
    Reg dest = ra_dest(as, ir, allow);
    asm_fuseahuref(as, ir->op1, RSET_GPR);
-    emit_mrm(as, dest < RID_MAX_GPR ? XO_MOV : XMM_MOVRM(as), dest, RID_MRM);
+    emit_mrm(as, dest < RID_MAX_GPR ? XO_MOV : XO_MOVSD, dest, RID_MRM);
  } else {
    asm_fuseahuref(as, ir->op1, RSET_GPR);
  }
@@ -1443,7 +1391,7 @@ static void asm_sload(ASMState *as, IRIns *ir)
    Reg left = ra_scratch(as, RSET_FPR);
    asm_tointg(as, ir, left);  /* Frees dest reg. Do this before base alloc. */
    base = ra_alloc1(as, REF_BASE, RSET_GPR);
-    emit_rmro(as, XMM_MOVRM(as), left, base, ofs);
+    emit_rmro(as, XO_MOVSD, left, base, ofs);
    t.irt = IRT_NUM;  /* Continue with a regular number type check. */
 #if LJ_64
  } else if (irt_islightud(t)) {
@@ -1461,11 +1409,9 @@ static void asm_sload(ASMState *as, IRIns *ir)
    lua_assert(irt_isnum(t) || irt_isint(t) || irt_isaddr(t));
    if ((ir->op2 & IRSLOAD_CONVERT)) {
      t.irt = irt_isint(t) ? IRT_NUM : IRT_INT;  /* Check for original type. */
-      emit_rmro(as, irt_isint(t) ? XO_CVTSI2SD : XO_CVTSD2SI, dest, base, ofs);
+      emit_rmro(as, irt_isint(t) ? XO_CVTSI2SD : XO_CVTTSD2SI, dest, base, ofs);
-    } else if (irt_isnum(t)) {
-      emit_rmro(as, XMM_MOVRM(as), dest, base, ofs);
    } else {
-      emit_rmro(as, XO_MOV, dest, base, ofs);
+      emit_rmro(as, irt_isnum(t) ? XO_MOVSD : XO_MOV, dest, base, ofs);
    }
  } else {
    if (!(ir->op2 & IRSLOAD_TYPECHECK))
@@ -1492,15 +1438,13 @@ static void asm_sload(ASMState *as, IRIns *ir)
 static void asm_cnew(ASMState *as, IRIns *ir)
 {
  CTState *cts = ctype_ctsG(J2G(as->J));
-  CTypeID ctypeid = (CTypeID)IR(ir->op1)->i;
+  CTypeID id = (CTypeID)IR(ir->op1)->i;
-  CTSize sz = (ir->o == IR_CNEWI || ir->op2 == REF_NIL) ?
+  CTSize sz;
-              lj_ctype_size(cts, ctypeid) : (CTSize)IR(ir->op2)->i;
+  CTInfo info = lj_ctype_info(cts, id, &sz);
  const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_mem_newgco];
-  IRRef args[2];
+  IRRef args[4];
-  lua_assert(sz != CTSIZE_INVALID);
+  lua_assert(sz != CTSIZE_INVALID || (ir->o == IR_CNEW && ir->op2 != REF_NIL));
-  args[0] = ASMREF_L;     /* lua_State *L */
-  args[1] = ASMREF_TMP1;  /* MSize size   */
  as->gcsteps++;
  asm_setupresult(as, ir, ci);  /* GCcdata * */
@@ -1543,15 +1487,26 @@ static void asm_cnew(ASMState *as, IRIns *ir)
    } while (1);
 #endif
    lua_assert(sz == 4 || sz == 8);
+  } else if (ir->op2 != REF_NIL) {  /* Create VLA/VLS/aligned cdata. */
+    ci = &lj_ir_callinfo[IRCALL_lj_cdata_newv];
+    args[0] = ASMREF_L;     /* lua_State *L */
+    args[1] = ir->op1;      /* CTypeID id   */
+    args[2] = ir->op2;      /* CTSize sz    */
+    args[3] = ASMREF_TMP1;  /* CTSize align */
+    asm_gencall(as, ci, args);
+    emit_loadi(as, ra_releasetmp(as, ASMREF_TMP1), (int32_t)ctype_align(info));
+    return;
  }
  /* Combine initialization of marked, gct and ctypeid. */
  emit_movtomro(as, RID_ECX, RID_RET, offsetof(GCcdata, marked));
  emit_gri(as, XG_ARITHi(XOg_OR), RID_ECX,
-           (int32_t)((~LJ_TCDATA<<8)+(ctypeid<<16)));
+           (int32_t)((~LJ_TCDATA<<8)+(id<<16)));
  emit_gri(as, XG_ARITHi(XOg_AND), RID_ECX, LJ_GC_WHITES);
  emit_opgl(as, XO_MOVZXb, RID_ECX, gc.currentwhite);
+  args[0] = ASMREF_L;     /* lua_State *L */
+  args[1] = ASMREF_TMP1;  /* MSize size   */
  asm_gencall(as, ci, args);
  emit_loadi(as, ra_releasetmp(as, ASMREF_TMP1), (int32_t)(sz+sizeof(GCcdata)));
 }
@@ -1629,31 +1584,21 @@ static void asm_x87load(ASMState *as, IRRef ref)
  }
 }
-/* Try to rejoin pow from EXP2, MUL and LOG2 (if still unsplit). */
+static void asm_fppow(ASMState *as, IRIns *ir, IRRef lref, IRRef rref)
-static int fpmjoin_pow(ASMState *as, IRIns *ir)
 {
-  IRIns *irp = IR(ir->op1);
+  /* The modified regs must match with the *.dasc implementation. */
-  if (irp == ir-1 && irp->o == IR_MUL && !ra_used(irp)) {
+  RegSet drop = RSET_RANGE(RID_XMM0, RID_XMM2+1)|RID2RSET(RID_EAX);
-    IRIns *irpp = IR(irp->op1);
+  IRIns *irx;
-    if (irpp == ir-2 && irpp->o == IR_FPMATH &&
+  if (ra_hasreg(ir->r))
-        irpp->op2 == IRFPM_LOG2 && !ra_used(irpp)) {
+    rset_clear(drop, ir->r);  /* Dest reg handled below. */
-      /* The modified regs must match with the *.dasc implementation. */
+  ra_evictset(as, drop);
-      RegSet drop = RSET_RANGE(RID_XMM0, RID_XMM2+1)|RID2RSET(RID_EAX);
+  ra_destreg(as, ir, RID_XMM0);
-      IRIns *irx;
+  emit_call(as, lj_vm_pow_sse);
-      if (ra_hasreg(ir->r))
+  irx = IR(lref);
-        rset_clear(drop, ir->r);  /* Dest reg handled below. */
+  if (ra_noreg(irx->r) && ra_gethint(irx->r) == RID_XMM1)
-      ra_evictset(as, drop);
+    irx->r = RID_INIT;  /* Avoid allocating xmm1 for x. */
-      ra_destreg(as, ir, RID_XMM0);
+  ra_left(as, RID_XMM0, lref);
-      emit_call(as, lj_vm_pow_sse);
+  ra_left(as, RID_XMM1, rref);
-      irx = IR(irpp->op1);
-      if (ra_noreg(irx->r) && ra_gethint(irx->r) == RID_XMM1)
-        irx->r = RID_INIT;  /* Avoid allocating xmm1 for x. */
-      ra_left(as, RID_XMM0, irpp->op1);
-      ra_left(as, RID_XMM1, irp->op2);
-      return 1;
-    }
-  }
-  return 0;
 }
 static void asm_fpmath(ASMState *as, IRIns *ir)
@@ -1689,7 +1634,7 @@ static void asm_fpmath(ASMState *as, IRIns *ir)
                    fpm == IRFPM_CEIL ? lj_vm_ceil_sse : lj_vm_trunc_sse);
      ra_left(as, RID_XMM0, ir->op1);
    }
-  } else if (fpm == IRFPM_EXP2 && fpmjoin_pow(as, ir)) {
+  } else if (fpm == IRFPM_EXP2 && asm_fpjoin_pow(as, ir)) {
    /* Rejoined to pow(). */
  } else {  /* Handle x87 ops. */
    int32_t ofs = sps_scale(ir->s);  /* Use spill slot or temp slots. */
@@ -1697,7 +1642,7 @@ static void asm_fpmath(ASMState *as, IRIns *ir)
    if (ra_hasreg(dest)) {
      ra_free(as, dest);
      ra_modified(as, dest);
-      emit_rmro(as, XMM_MOVRM(as), dest, RID_ESP, ofs);
+      emit_rmro(as, XO_MOVSD, dest, RID_ESP, ofs);
    }
    emit_rmro(as, XO_FSTPq, XOg_FSTPq, RID_ESP, ofs);
    switch (fpm) {  /* st0 = lj_vm_*(st0) */
@@ -1736,6 +1681,9 @@ static void asm_fpmath(ASMState *as, IRIns *ir)
  }
 }
+#define asm_atan2(as, ir)       asm_fpmath(as, ir)
+#define asm_ldexp(as, ir)       asm_fpmath(as, ir)
 static void asm_fppowi(ASMState *as, IRIns *ir)
 {
  /* The modified regs must match with the *.dasc implementation. */
@@ -1749,26 +1697,15 @@ static void asm_fppowi(ASMState *as, IRIns *ir)
  ra_left(as, RID_EAX, ir->op2);
 }
-#if LJ_64 && LJ_HASFFI
+static void asm_pow(ASMState *as, IRIns *ir)
-static void asm_arith64(ASMState *as, IRIns *ir, IRCallID id)
 {
-  const CCallInfo *ci = &lj_ir_callinfo[id];
+#if LJ_64 && LJ_HASFFI
-  IRRef args[2];
+  if (!irt_isnum(ir->t))
-  args[0] = ir->op1;
+    asm_callid(as, ir, irt_isi64(ir->t) ? IRCALL_lj_carith_powi64 :
-  args[1] = ir->op2;
+                                          IRCALL_lj_carith_powu64);
-  asm_setupresult(as, ir, ci);
+  else
-  asm_gencall(as, ci, args);
-}
 #endif
+    asm_fppowi(as, ir);
-static void asm_intmod(ASMState *as, IRIns *ir)
-{
-  const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_vm_modi];
-  IRRef args[2];
-  args[0] = ir->op1;
-  args[1] = ir->op2;
-  asm_setupresult(as, ir, ci);
-  asm_gencall(as, ci, args);
 }
 static int asm_swapops(ASMState *as, IRIns *ir)
@@ -1947,6 +1884,44 @@ static void asm_add(ASMState *as, IRIns *ir)
    asm_intarith(as, ir, XOg_ADD);
 }
+static void asm_sub(ASMState *as, IRIns *ir)
+{
+  if (irt_isnum(ir->t))
+    asm_fparith(as, ir, XO_SUBSD);
+  else  /* Note: no need for LEA trick here. i-k is encoded as i+(-k). */
+    asm_intarith(as, ir, XOg_SUB);
+}
+static void asm_mul(ASMState *as, IRIns *ir)
+{
+  if (irt_isnum(ir->t))
+    asm_fparith(as, ir, XO_MULSD);
+  else
+    asm_intarith(as, ir, XOg_X_IMUL);
+}
+static void asm_div(ASMState *as, IRIns *ir)
+{
+#if LJ_64 && LJ_HASFFI
+  if (!irt_isnum(ir->t))
+    asm_callid(as, ir, irt_isi64(ir->t) ? IRCALL_lj_carith_divi64 :
+                                          IRCALL_lj_carith_divu64);
+  else
+#endif
+    asm_fparith(as, ir, XO_DIVSD);
+}
+static void asm_mod(ASMState *as, IRIns *ir)
+{
+#if LJ_64 && LJ_HASFFI
+  if (!irt_isint(ir->t))
+    asm_callid(as, ir, irt_isi64(ir->t) ? IRCALL_lj_carith_modi64 :
+                                          IRCALL_lj_carith_modu64);
+  else
+#endif
+    asm_callid(as, ir, IRCALL_lj_vm_modi);
+}
 static void asm_neg_not(ASMState *as, IRIns *ir, x86Group3 xg)
 {
  Reg dest = ra_dest(as, ir, RSET_GPR);
@@ -1954,7 +1929,17 @@ static void asm_neg_not(ASMState *as, IRIns *ir, x86Group3 xg)
  ra_left(as, dest, ir->op1);
 }
-static void asm_min_max(ASMState *as, IRIns *ir, int cc)
+static void asm_neg(ASMState *as, IRIns *ir)
+{
+  if (irt_isnum(ir->t))
+    asm_fparith(as, ir, XO_XORPS);
+  else
+    asm_neg_not(as, ir, XOg_NEG);
+}
+#define asm_abs(as, ir)         asm_fparith(as, ir, XO_ANDPS)
+static void asm_intmin_max(ASMState *as, IRIns *ir, int cc)
 {
  Reg right, dest = ra_dest(as, ir, RSET_GPR);
  IRRef lref = ir->op1, rref = ir->op2;
@@ -1965,7 +1950,30 @@ static void asm_min_max(ASMState *as, IRIns *ir, int cc)
  ra_left(as, dest, lref);
 }
-static void asm_bitswap(ASMState *as, IRIns *ir)
+static void asm_min(ASMState *as, IRIns *ir)
+{
+  if (irt_isnum(ir->t))
+    asm_fparith(as, ir, XO_MINSD);
+  else
+    asm_intmin_max(as, ir, CC_G);
+}
+static void asm_max(ASMState *as, IRIns *ir)
+{
+  if (irt_isnum(ir->t))
+    asm_fparith(as, ir, XO_MAXSD);
+  else
+    asm_intmin_max(as, ir, CC_L);
+}
+/* Note: don't use LEA for overflow-checking arithmetic! */
+#define asm_addov(as, ir)       asm_intarith(as, ir, XOg_ADD)
+#define asm_subov(as, ir)       asm_intarith(as, ir, XOg_SUB)
+#define asm_mulov(as, ir)       asm_intarith(as, ir, XOg_X_IMUL)
+#define asm_bnot(as, ir)        asm_neg_not(as, ir, XOg_NOT)
+static void asm_bswap(ASMState *as, IRIns *ir)
 {
  Reg dest = ra_dest(as, ir, RSET_GPR);
  as->mcp = emit_op(XO_BSWAP + ((dest&7) << 24),
@@ -1973,6 +1981,10 @@ static void asm_bitswap(ASMState *as, IRIns *ir)
  ra_left(as, dest, ir->op1);
 }
+#define asm_band(as, ir)        asm_intarith(as, ir, XOg_AND)
+#define asm_bor(as, ir)         asm_intarith(as, ir, XOg_OR)
+#define asm_bxor(as, ir)        asm_intarith(as, ir, XOg_XOR)
 static void asm_bitshift(ASMState *as, IRIns *ir, x86Shift xs)
 {
  IRRef rref = ir->op2;
@@ -2012,6 +2024,12 @@ static void asm_bitshift(ASMState *as, IRIns *ir, x86Shift xs)
  */
 }
+#define asm_bshl(as, ir)        asm_bitshift(as, ir, XOg_SHL)
+#define asm_bshr(as, ir)        asm_bitshift(as, ir, XOg_SHR)
+#define asm_bsar(as, ir)        asm_bitshift(as, ir, XOg_SAR)
+#define asm_brol(as, ir)        asm_bitshift(as, ir, XOg_ROL)
+#define asm_bror(as, ir)        asm_bitshift(as, ir, XOg_ROR)
 /* -- Comparisons --------------------------------------------------------- */
 /* Virtual flags for unordered FP comparisons. */
@@ -2038,8 +2056,9 @@ static const uint16_t asm_compmap[IR_ABC+1] = {
 };
 /* FP and integer comparisons. */
-static void asm_comp(ASMState *as, IRIns *ir, uint32_t cc)
+static void asm_comp(ASMState *as, IRIns *ir)
 {
+  uint32_t cc = asm_compmap[ir->o];
  if (irt_isnum(ir->t)) {
    IRRef lref = ir->op1;
    IRRef rref = ir->op2;
@@ -2194,6 +2213,8 @@ static void asm_comp(ASMState *as, IRIns *ir, uint32_t cc)
  }
 }
+#define asm_equal(as, ir)       asm_comp(as, ir)
 #if LJ_32 && LJ_HASFFI
 /* 64 bit integer comparisons in 32 bit mode. */
 static void asm_comp_int64(ASMState *as, IRIns *ir)
@@ -2276,13 +2297,9 @@ static void asm_hiop(ASMState *as, IRIns *ir)
  int uselo = ra_used(ir-1), usehi = ra_used(ir);  /* Loword/hiword used? */
  if (LJ_UNLIKELY(!(as->flags & JIT_F_OPT_DCE))) uselo = usehi = 1;
  if ((ir-1)->o == IR_CONV) {  /* Conversions to/from 64 bit. */
-    if (usehi || uselo) {
-      if (irt_isfp(ir->t))
-        asm_conv_fp_int64(as, ir);
-      else
-        asm_conv_int64_fp(as, ir);
-    }
    as->curins--;  /* Always skip the CONV. */
+    if (usehi || uselo)
+      asm_conv64(as, ir);
    return;
  } else if ((ir-1)->o <= IR_NE) {  /* 64 bit integer comparisons. ORDER IR. */
    asm_comp_int64(as, ir);
@@ -2580,163 +2597,6 @@ static void asm_tail_prep(ASMState *as)
  }
 }
-/* -- Instruction dispatch ------------------------------------------------ */
-/* Assemble a single instruction. */
-static void asm_ir(ASMState *as, IRIns *ir)
-{
-  switch ((IROp)ir->o) {
-  /* Miscellaneous ops. */
-  case IR_LOOP: asm_loop(as); break;
-  case IR_NOP: case IR_XBAR: lua_assert(!ra_used(ir)); break;
-  case IR_USE:
-    ra_alloc1(as, ir->op1, irt_isfp(ir->t) ? RSET_FPR : RSET_GPR); break;
-  case IR_PHI: asm_phi(as, ir); break;
-  case IR_HIOP: asm_hiop(as, ir); break;
-  case IR_GCSTEP: asm_gcstep(as, ir); break;
-  /* Guarded assertions. */
-  case IR_LT: case IR_GE: case IR_LE: case IR_GT:
-  case IR_ULT: case IR_UGE: case IR_ULE: case IR_UGT:
-  case IR_EQ: case IR_NE: case IR_ABC:
-    asm_comp(as, ir, asm_compmap[ir->o]);
-    break;
-  case IR_RETF: asm_retf(as, ir); break;
-  /* Bit ops. */
-  case IR_BNOT: asm_neg_not(as, ir, XOg_NOT); break;
-  case IR_BSWAP: asm_bitswap(as, ir); break;
-  case IR_BAND: asm_intarith(as, ir, XOg_AND); break;
-  case IR_BOR:  asm_intarith(as, ir, XOg_OR); break;
-  case IR_BXOR: asm_intarith(as, ir, XOg_XOR); break;
-  case IR_BSHL: asm_bitshift(as, ir, XOg_SHL); break;
-  case IR_BSHR: asm_bitshift(as, ir, XOg_SHR); break;
-  case IR_BSAR: asm_bitshift(as, ir, XOg_SAR); break;
-  case IR_BROL: asm_bitshift(as, ir, XOg_ROL); break;
-  case IR_BROR: asm_bitshift(as, ir, XOg_ROR); break;
-  /* Arithmetic ops. */
-  case IR_ADD: asm_add(as, ir); break;
-  case IR_SUB:
-    if (irt_isnum(ir->t))
-      asm_fparith(as, ir, XO_SUBSD);
-    else  /* Note: no need for LEA trick here. i-k is encoded as i+(-k). */
-      asm_intarith(as, ir, XOg_SUB);
-    break;
-  case IR_MUL:
-    if (irt_isnum(ir->t))
-      asm_fparith(as, ir, XO_MULSD);
-    else
-      asm_intarith(as, ir, XOg_X_IMUL);
-    break;
-  case IR_DIV:
-#if LJ_64 && LJ_HASFFI
-    if (!irt_isnum(ir->t))
-      asm_arith64(as, ir, irt_isi64(ir->t) ? IRCALL_lj_carith_divi64 :
-                                             IRCALL_lj_carith_divu64);
-    else
-#endif
-      asm_fparith(as, ir, XO_DIVSD);
-    break;
-  case IR_MOD:
-#if LJ_64 && LJ_HASFFI
-    if (!irt_isint(ir->t))
-      asm_arith64(as, ir, irt_isi64(ir->t) ? IRCALL_lj_carith_modi64 :
-                                             IRCALL_lj_carith_modu64);
-    else
-#endif
-      asm_intmod(as, ir);
-    break;
-  case IR_NEG:
-    if (irt_isnum(ir->t))
-      asm_fparith(as, ir, XO_XORPS);
-    else
-      asm_neg_not(as, ir, XOg_NEG);
-    break;
-  case IR_ABS: asm_fparith(as, ir, XO_ANDPS); break;
-  case IR_MIN:
-    if (irt_isnum(ir->t))
-      asm_fparith(as, ir, XO_MINSD);
-    else
-      asm_min_max(as, ir, CC_G);
-    break;
-  case IR_MAX:
-    if (irt_isnum(ir->t))
-      asm_fparith(as, ir, XO_MAXSD);
-    else
-      asm_min_max(as, ir, CC_L);
-    break;
-  case IR_FPMATH: case IR_ATAN2: case IR_LDEXP:
-    asm_fpmath(as, ir);
-    break;
-  case IR_POW:
-#if LJ_64 && LJ_HASFFI
-    if (!irt_isnum(ir->t))
-      asm_arith64(as, ir, irt_isi64(ir->t) ? IRCALL_lj_carith_powi64 :
-                                             IRCALL_lj_carith_powu64);
-    else
-#endif
-      asm_fppowi(as, ir);
-    break;
-  /* Overflow-checking arithmetic ops. Note: don't use LEA here! */
-  case IR_ADDOV: asm_intarith(as, ir, XOg_ADD); break;
-  case IR_SUBOV: asm_intarith(as, ir, XOg_SUB); break;
-  case IR_MULOV: asm_intarith(as, ir, XOg_X_IMUL); break;
-  /* Memory references. */
-  case IR_AREF: asm_aref(as, ir); break;
-  case IR_HREF: asm_href(as, ir); break;
-  case IR_HREFK: asm_hrefk(as, ir); break;
-  case IR_NEWREF: asm_newref(as, ir); break;
-  case IR_UREFO: case IR_UREFC: asm_uref(as, ir); break;
-  case IR_FREF: asm_fref(as, ir); break;
-  case IR_STRREF: asm_strref(as, ir); break;
-  /* Loads and stores. */
-  case IR_ALOAD: case IR_HLOAD: case IR_ULOAD: case IR_VLOAD:
-    asm_ahuvload(as, ir);
-    break;
-  case IR_FLOAD: case IR_XLOAD: asm_fxload(as, ir); break;
-  case IR_SLOAD: asm_sload(as, ir); break;
-  case IR_ASTORE: case IR_HSTORE: case IR_USTORE: asm_ahustore(as, ir); break;
-  case IR_FSTORE: case IR_XSTORE: asm_fxstore(as, ir); break;
-  /* Allocations. */
-  case IR_SNEW: case IR_XSNEW: asm_snew(as, ir); break;
-  case IR_TNEW: asm_tnew(as, ir); break;
-  case IR_TDUP: asm_tdup(as, ir); break;
-  case IR_CNEW: case IR_CNEWI: asm_cnew(as, ir); break;
-  /* Write barriers. */
-  case IR_TBAR: asm_tbar(as, ir); break;
-  case IR_OBAR: asm_obar(as, ir); break;
-  /* Type conversions. */
-  case IR_TOBIT: asm_tobit(as, ir); break;
-  case IR_CONV: asm_conv(as, ir); break;
-  case IR_TOSTR: asm_tostr(as, ir); break;
-  case IR_STRTO: asm_strto(as, ir); break;
-  /* Calls. */
-  case IR_CALLN: case IR_CALLL: case IR_CALLS: asm_call(as, ir); break;
-  case IR_CALLXS: asm_callx(as, ir); break;
-  case IR_CARG: break;
-  default:
-    setintV(&as->J->errinfo, ir->o);
-    lj_trace_err_info(as->J, LJ_TRERR_NYIIR);
-    break;
-  }
-}
 /* -- Trace setup --------------------------------------------------------- */
 /* Ensure there are enough stack slots for call arguments. */
diff --git a/src/lj_bc.h b/src/lj_bc.h
index 56e71dd9..ac9cc5e1 100644
--- a/src/lj_bc.h
+++ b/src/lj_bc.h
@@ -89,6 +89,8 @@
  _(ISFC,       dst,    ___,    var,    ___) \
  _(IST,        ___,    ___,    var,    ___) \
  _(ISF,        ___,    ___,    var,    ___) \
+  _(ISTYPE,     var,    ___,    lit,    ___) \
+  _(ISNUM,      var,    ___,    lit,    ___) \
  \
  /* Unary ops. */ \
  _(MOV,        dst,    ___,    var,    ___) \
@@ -143,10 +145,12 @@
  _(TGETV,      dst,    var,    var,    index) \
  _(TGETS,      dst,    var,    str,    index) \
  _(TGETB,      dst,    var,    lit,    index) \
+  _(TGETR,      dst,    var,    var,    index) \
  _(TSETV,      var,    var,    var,    newindex) \
  _(TSETS,      var,    var,    str,    newindex) \
  _(TSETB,      var,    var,    lit,    newindex) \
  _(TSETM,      base,   ___,    num,    newindex) \
+  _(TSETR,      var,    var,    var,    newindex) \
  \
  /* Calls and vararg handling. T = tail call. */ \
  _(CALLM,      base,   lit,    lit,    call) \
diff --git a/src/lj_bcdump.h b/src/lj_bcdump.h
index e660156d..22a8b823 100644
--- a/src/lj_bcdump.h
+++ b/src/lj_bcdump.h
@@ -36,7 +36,7 @@
 /* If you perform *any* kind of private modifications to the bytecode itself
 ** or to the dump format, you *must* set BCDUMP_VERSION to 0x80 or higher.
 */
-#define BCDUMP_VERSION          1
+#define BCDUMP_VERSION          2
 /* Compatibility flags. */
 #define BCDUMP_F_BE             0x01
@@ -61,6 +61,7 @@ enum {
 LJ_FUNC int lj_bcwrite(lua_State *L, GCproto *pt, lua_Writer writer,
                       void *data, int strip);
+LJ_FUNC GCproto *lj_bcread_proto(LexState *ls);
 LJ_FUNC GCproto *lj_bcread(LexState *ls);
 #endif
diff --git a/src/lj_bcread.c b/src/lj_bcread.c
index 2b5ba855..9f025500 100644
--- a/src/lj_bcread.c
+++ b/src/lj_bcread.c
@@ -9,6 +9,7 @@
 #include "lj_obj.h"
 #include "lj_gc.h"
 #include "lj_err.h"
+#include "lj_buf.h"
 #include "lj_str.h"
 #include "lj_tab.h"
 #include "lj_bc.h"
@@ -20,6 +21,7 @@
 #include "lj_lex.h"
 #include "lj_bcdump.h"
 #include "lj_state.h"
+#include "lj_strfmt.h"
 /* Reuse some lexer fields for our own purposes. */
 #define bcread_flags(ls)        ls->level
@@ -38,84 +40,73 @@ static LJ_NOINLINE void bcread_error(LexState *ls, ErrMsg em)
  const char *name = ls->chunkarg;
  if (*name == BCDUMP_HEAD1) name = "(binary)";
  else if (*name == '@' || *name == '=') name++;
-  lj_str_pushf(L, "%s: %s", name, err2msg(em));
+  lj_strfmt_pushf(L, "%s: %s", name, err2msg(em));
  lj_err_throw(L, LUA_ERRSYNTAX);
 }
-/* Resize input buffer. */
+/* Refill buffer. */
-static void bcread_resize(LexState *ls, MSize len)
-{
-  if (ls->sb.sz < len) {
-    MSize sz = ls->sb.sz * 2;
-    while (len > sz) sz = sz * 2;
-    lj_str_resizebuf(ls->L, &ls->sb, sz);
-    /* Caveat: this may change ls->sb.buf which may affect ls->p. */
-  }
-}
-/* Refill buffer if needed. */
 static LJ_NOINLINE void bcread_fill(LexState *ls, MSize len, int need)
 {
  lua_assert(len != 0);
-  if (len > LJ_MAX_MEM || ls->current < 0)
+  if (len > LJ_MAX_MEM || ls->c < 0)
    bcread_error(ls, LJ_ERR_BCBAD);
  do {
    const char *buf;
-    size_t size;
+    size_t sz;
-    if (ls->n) {  /* Copy remainder to buffer. */
+    char *p = sbufB(&ls->sb);
-      if (ls->sb.n) {  /* Move down in buffer. */
+    MSize n = (MSize)(ls->pe - ls->p);
-        lua_assert(ls->p + ls->n == ls->sb.buf + ls->sb.n);
+    if (n) {  /* Copy remainder to buffer. */
-        if (ls->n != ls->sb.n)
+      if (sbuflen(&ls->sb)) {  /* Move down in buffer. */
-          memmove(ls->sb.buf, ls->p, ls->n);
+        lua_assert(ls->pe == sbufP(&ls->sb));
+        if (ls->p != p) memmove(p, ls->p, n);
      } else {  /* Copy from buffer provided by reader. */
-        bcread_resize(ls, len);
+        p = lj_buf_need(&ls->sb, len);
-        memcpy(ls->sb.buf, ls->p, ls->n);
+        memcpy(p, ls->p, n);
      }
-      ls->p = ls->sb.buf;
+      ls->p = p;
+      ls->pe = p + n;
    }
-    ls->sb.n = ls->n;
+    setsbufP(&ls->sb, p + n);
-    buf = ls->rfunc(ls->L, ls->rdata, &size);  /* Get more data from reader. */
+    buf = ls->rfunc(ls->L, ls->rdata, &sz);  /* Get more data from reader. */
-    if (buf == NULL || size == 0) {  /* EOF? */
+    if (buf == NULL || sz == 0) {  /* EOF? */
      if (need) bcread_error(ls, LJ_ERR_BCBAD);
-      ls->current = -1;  /* Only bad if we get called again. */
+      ls->c = -1;  /* Only bad if we get called again. */
      break;
    }
-    if (ls->sb.n) {  /* Append to buffer. */
+    if (n) {  /* Append to buffer. */
-      MSize n = ls->sb.n + (MSize)size;
+      n += (MSize)sz;
-      bcread_resize(ls, n < len ? len : n);
+      p = lj_buf_need(&ls->sb, n < len ? len : n);
-      memcpy(ls->sb.buf + ls->sb.n, buf, size);
+      memcpy(sbufP(&ls->sb), buf, sz);
-      ls->n = ls->sb.n = n;
+      setsbufP(&ls->sb, p + n);
-      ls->p = ls->sb.buf;
+      ls->p = p;
+      ls->pe = p + n;
    } else {  /* Return buffer provided by reader. */
-      ls->n = (MSize)size;
      ls->p = buf;
+      ls->pe = buf + sz;
    }
-  } while (ls->n < len);
+  } while (ls->p + len > ls->pe);
 }
 /* Need a certain number of bytes. */
 static LJ_AINLINE void bcread_need(LexState *ls, MSize len)
 {
-  if (LJ_UNLIKELY(ls->n < len))
+  if (LJ_UNLIKELY(ls->p + len > ls->pe))
    bcread_fill(ls, len, 1);
 }
 /* Want to read up to a certain number of bytes, but may need less. */
 static LJ_AINLINE void bcread_want(LexState *ls, MSize len)
 {
-  if (LJ_UNLIKELY(ls->n < len))
+  if (LJ_UNLIKELY(ls->p + len > ls->pe))
    bcread_fill(ls, len, 0);
 }
-#define bcread_dec(ls)          check_exp(ls->n > 0, ls->n--)
-#define bcread_consume(ls, len) check_exp(ls->n >= (len), ls->n -= (len))
 /* Return memory block from buffer. */
-static uint8_t *bcread_mem(LexState *ls, MSize len)
+static LJ_AINLINE uint8_t *bcread_mem(LexState *ls, MSize len)
 {
  uint8_t *p = (uint8_t *)ls->p;
-  bcread_consume(ls, len);
+  ls->p += len;
-  ls->p = (char *)p + len;
+  lua_assert(ls->p <= ls->pe);
  return p;
 }
@@ -128,25 +119,15 @@ static void bcread_block(LexState *ls, void *q, MSize len)
 /* Read byte from buffer. */
 static LJ_AINLINE uint32_t bcread_byte(LexState *ls)
 {
-  bcread_dec(ls);
+  lua_assert(ls->p < ls->pe);
  return (uint32_t)(uint8_t)*ls->p++;
 }
 /* Read ULEB128 value from buffer. */
-static uint32_t bcread_uleb128(LexState *ls)
+static LJ_AINLINE uint32_t bcread_uleb128(LexState *ls)
 {
-  const uint8_t *p = (const uint8_t *)ls->p;
+  uint32_t v = lj_buf_ruleb128(&ls->p);
-  uint32_t v = *p++;
+  lua_assert(ls->p <= ls->pe);
-  if (LJ_UNLIKELY(v >= 0x80)) {
-    int sh = 0;
-    v &= 0x7f;
-    do {
-     v |= ((*p & 0x7f) << (sh += 7));
-     bcread_dec(ls);
-   } while (*p++ >= 0x80);
-  }
-  bcread_dec(ls);
-  ls->p = (char *)p;
  return v;
 }
@@ -160,11 +141,10 @@ static uint32_t bcread_uleb128_33(LexState *ls)
    v &= 0x3f;
    do {
     v |= ((*p & 0x7f) << (sh += 7));
-     bcread_dec(ls);
   } while (*p++ >= 0x80);
  }
-  bcread_dec(ls);
  ls->p = (char *)p;
+  lua_assert(ls->p <= ls->pe);
  return v;
 }
@@ -326,25 +306,13 @@ static void bcread_uv(LexState *ls, GCproto *pt, MSize sizeuv)
 }
 /* Read a prototype. */
-static GCproto *bcread_proto(LexState *ls)
+GCproto *lj_bcread_proto(LexState *ls)
 {
  GCproto *pt;
  MSize framesize, numparams, flags, sizeuv, sizekgc, sizekn, sizebc, sizept;
  MSize ofsk, ofsuv, ofsdbg;
  MSize sizedbg = 0;
  BCLine firstline = 0, numline = 0;
-  MSize len, startn;
-  /* Read length. */
-  if (ls->n > 0 && ls->p[0] == 0) {  /* Shortcut EOF. */
-    ls->n--; ls->p++;
-    return NULL;
-  }
-  bcread_want(ls, 5);
-  len = bcread_uleb128(ls);
-  if (!len) return NULL;  /* EOF */
-  bcread_need(ls, len);
-  startn = ls->n;
  /* Read prototype header. */
  flags = bcread_byte(ls);
@@ -413,9 +381,6 @@ static GCproto *bcread_proto(LexState *ls)
    setmref(pt->uvinfo, NULL);
    setmref(pt->varinfo, NULL);
  }
-  if (len != startn - ls->n)
-    bcread_error(ls, LJ_ERR_BCBAD);
  return pt;
 }
@@ -455,19 +420,33 @@ static int bcread_header(LexState *ls)
 GCproto *lj_bcread(LexState *ls)
 {
  lua_State *L = ls->L;
-  lua_assert(ls->current == BCDUMP_HEAD1);
+  lua_assert(ls->c == BCDUMP_HEAD1);
  bcread_savetop(L, ls, L->top);
-  lj_str_resetbuf(&ls->sb);
+  lj_buf_reset(&ls->sb);
  /* Check for a valid bytecode dump header. */
  if (!bcread_header(ls))
    bcread_error(ls, LJ_ERR_BCFMT);
  for (;;) {  /* Process all prototypes in the bytecode dump. */
-    GCproto *pt = bcread_proto(ls);
+    GCproto *pt;
-    if (!pt) break;
+    MSize len;
+    const char *startp;
+    /* Read length. */
+    if (ls->p < ls->pe && ls->p[0] == 0) {  /* Shortcut EOF. */
+      ls->p++;
+      break;
+    }
+    bcread_want(ls, 5);
+    len = bcread_uleb128(ls);
+    if (!len) break;  /* EOF */
+    bcread_need(ls, len);
+    startp = ls->p;
+    pt = lj_bcread_proto(ls);
+    if (ls->p != startp + len)
+      bcread_error(ls, LJ_ERR_BCBAD);
    setprotoV(L, L->top, pt);
    incr_top(L);
  }
-  if ((int32_t)ls->n > 0 || L->top-1 != bcread_oldtop(L, ls))
+  if (ls->p < ls->pe || L->top-1 != bcread_oldtop(L, ls))
    bcread_error(ls, LJ_ERR_BCBAD);
  /* Pop off last prototype. */
  L->top--;
diff --git a/src/lj_bcwrite.c b/src/lj_bcwrite.c
index 4805d515..c95086c7 100644
--- a/src/lj_bcwrite.c
+++ b/src/lj_bcwrite.c
@@ -8,7 +8,7 @@
 #include "lj_obj.h"
 #include "lj_gc.h"
-#include "lj_str.h"
+#include "lj_buf.h"
 #include "lj_bc.h"
 #if LJ_HASFFI
 #include "lj_ctype.h"
@@ -17,13 +17,13 @@
 #include "lj_dispatch.h"
 #include "lj_jit.h"
 #endif
+#include "lj_strfmt.h"
 #include "lj_bcdump.h"
 #include "lj_vm.h"
 /* Context for bytecode writer. */
 typedef struct BCWriteCtx {
  SBuf sb;                      /* Output buffer. */
-  lua_State *L;                 /* Lua state. */
  GCproto *pt;                  /* Root prototype. */
  lua_Writer wfunc;             /* Writer callback. */
  void *wdata;                  /* Writer callback data. */
@@ -31,85 +31,44 @@ typedef struct BCWriteCtx {
  int status;                   /* Status from writer callback. */
 } BCWriteCtx;
-/* -- Output buffer handling ---------------------------------------------- */
-/* Resize buffer if needed. */
-static LJ_NOINLINE void bcwrite_resize(BCWriteCtx *ctx, MSize len)
-{
-  MSize sz = ctx->sb.sz * 2;
-  while (ctx->sb.n + len > sz) sz = sz * 2;
-  lj_str_resizebuf(ctx->L, &ctx->sb, sz);
-}
-/* Need a certain amount of buffer space. */
-static LJ_AINLINE void bcwrite_need(BCWriteCtx *ctx, MSize len)
-{
-  if (LJ_UNLIKELY(ctx->sb.n + len > ctx->sb.sz))
-    bcwrite_resize(ctx, len);
-}
-/* Add memory block to buffer. */
-static void bcwrite_block(BCWriteCtx *ctx, const void *p, MSize len)
-{
-  uint8_t *q = (uint8_t *)(ctx->sb.buf + ctx->sb.n);
-  MSize i;
-  ctx->sb.n += len;
-  for (i = 0; i < len; i++) q[i] = ((uint8_t *)p)[i];
-}
-/* Add byte to buffer. */
-static LJ_AINLINE void bcwrite_byte(BCWriteCtx *ctx, uint8_t b)
-{
-  ctx->sb.buf[ctx->sb.n++] = b;
-}
-/* Add ULEB128 value to buffer. */
-static void bcwrite_uleb128(BCWriteCtx *ctx, uint32_t v)
-{
-  MSize n = ctx->sb.n;
-  uint8_t *p = (uint8_t *)ctx->sb.buf;
-  for (; v >= 0x80; v >>= 7)
-    p[n++] = (uint8_t)((v & 0x7f) | 0x80);
-  p[n++] = (uint8_t)v;
-  ctx->sb.n = n;
-}
 /* -- Bytecode writer ----------------------------------------------------- */
 /* Write a single constant key/value of a template table. */
 static void bcwrite_ktabk(BCWriteCtx *ctx, cTValue *o, int narrow)
 {
-  bcwrite_need(ctx, 1+10);
+  char *p = lj_buf_more(&ctx->sb, 1+10);
  if (tvisstr(o)) {
    const GCstr *str = strV(o);
    MSize len = str->len;
-    bcwrite_need(ctx, 5+len);
+    p = lj_buf_more(&ctx->sb, 5+len);
-    bcwrite_uleb128(ctx, BCDUMP_KTAB_STR+len);
+    p = lj_strfmt_wuleb128(p, BCDUMP_KTAB_STR+len);
-    bcwrite_block(ctx, strdata(str), len);
+    p = lj_buf_wmem(p, strdata(str), len);
  } else if (tvisint(o)) {
-    bcwrite_byte(ctx, BCDUMP_KTAB_INT);
+    *p++ = BCDUMP_KTAB_INT;
-    bcwrite_uleb128(ctx, intV(o));
+    p = lj_strfmt_wuleb128(p, intV(o));
  } else if (tvisnum(o)) {
    if (!LJ_DUALNUM && narrow) {  /* Narrow number constants to integers. */
      lua_Number num = numV(o);
      int32_t k = lj_num2int(num);
      if (num == (lua_Number)k) {  /* -0 is never a constant. */
-        bcwrite_byte(ctx, BCDUMP_KTAB_INT);
+        *p++ = BCDUMP_KTAB_INT;
-        bcwrite_uleb128(ctx, k);
+        p = lj_strfmt_wuleb128(p, k);
+        setsbufP(&ctx->sb, p);
        return;
      }
    }
-    bcwrite_byte(ctx, BCDUMP_KTAB_NUM);
+    *p++ = BCDUMP_KTAB_NUM;
-    bcwrite_uleb128(ctx, o->u32.lo);
+    p = lj_strfmt_wuleb128(p, o->u32.lo);
-    bcwrite_uleb128(ctx, o->u32.hi);
+    p = lj_strfmt_wuleb128(p, o->u32.hi);
  } else {
    lua_assert(tvispri(o));
-    bcwrite_byte(ctx, BCDUMP_KTAB_NIL+~itype(o));
+    *p++ = BCDUMP_KTAB_NIL+~itype(o);
  }
+  setsbufP(&ctx->sb, p);
 }
 /* Write a template table. */
-static void bcwrite_ktab(BCWriteCtx *ctx, const GCtab *t)
+static void bcwrite_ktab(BCWriteCtx *ctx, char *p, const GCtab *t)
 {
  MSize narray = 0, nhash = 0;
  if (t->asize > 0) {  /* Determine max. length of array part. */
@@ -127,8 +86,9 @@ static void bcwrite_ktab(BCWriteCtx *ctx, const GCtab *t)
      nhash += !tvisnil(&node[i].val);
  }
  /* Write number of array slots and hash slots. */
-  bcwrite_uleb128(ctx, narray);
+  p = lj_strfmt_wuleb128(p, narray);
-  bcwrite_uleb128(ctx, nhash);
+  p = lj_strfmt_wuleb128(p, nhash);
+  setsbufP(&ctx->sb, p);
  if (narray) {  /* Write array entries (may contain nil). */
    MSize i;
    TValue *o = tvref(t->array);
@@ -155,6 +115,7 @@ static void bcwrite_kgc(BCWriteCtx *ctx, GCproto *pt)
  for (i = 0; i < sizekgc; i++, kr++) {
    GCobj *o = gcref(*kr);
    MSize tp, need = 1;
+    char *p;
    /* Determine constant type and needed size. */
    if (o->gch.gct == ~LJ_TSTR) {
      tp = BCDUMP_KGC_STR + gco2str(o)->len;
@@ -181,24 +142,26 @@ static void bcwrite_kgc(BCWriteCtx *ctx, GCproto *pt)
      need = 1+2*5;
    }
    /* Write constant type. */
-    bcwrite_need(ctx, need);
+    p = lj_buf_more(&ctx->sb, need);
-    bcwrite_uleb128(ctx, tp);
+    p = lj_strfmt_wuleb128(p, tp);
    /* Write constant data (if any). */
    if (tp >= BCDUMP_KGC_STR) {
-      bcwrite_block(ctx, strdata(gco2str(o)), gco2str(o)->len);
+      p = lj_buf_wmem(p, strdata(gco2str(o)), gco2str(o)->len);
    } else if (tp == BCDUMP_KGC_TAB) {
-      bcwrite_ktab(ctx, gco2tab(o));
+      bcwrite_ktab(ctx, p, gco2tab(o));
+      continue;
 #if LJ_HASFFI
    } else if (tp != BCDUMP_KGC_CHILD) {
-      cTValue *p = (TValue *)cdataptr(gco2cd(o));
+      cTValue *q = (TValue *)cdataptr(gco2cd(o));
-      bcwrite_uleb128(ctx, p[0].u32.lo);
+      p = lj_strfmt_wuleb128(p, q[0].u32.lo);
-      bcwrite_uleb128(ctx, p[0].u32.hi);
+      p = lj_strfmt_wuleb128(p, q[0].u32.hi);
      if (tp == BCDUMP_KGC_COMPLEX) {
-        bcwrite_uleb128(ctx, p[1].u32.lo);
+        p = lj_strfmt_wuleb128(p, q[1].u32.lo);
-        bcwrite_uleb128(ctx, p[1].u32.hi);
+        p = lj_strfmt_wuleb128(p, q[1].u32.hi);
      }
 #endif
    }
+    setsbufP(&ctx->sb, p);
  }
 }
@@ -207,7 +170,7 @@ static void bcwrite_knum(BCWriteCtx *ctx, GCproto *pt)
 {
  MSize i, sizekn = pt->sizekn;
  cTValue *o = mref(pt->k, TValue);
-  bcwrite_need(ctx, 10*sizekn);
+  char *p = lj_buf_more(&ctx->sb, 10*sizekn);
  for (i = 0; i < sizekn; i++, o++) {
    int32_t k;
    if (tvisint(o)) {
@@ -220,58 +183,58 @@ static void bcwrite_knum(BCWriteCtx *ctx, GCproto *pt)
        k = lj_num2int(num);
        if (num == (lua_Number)k) {  /* -0 is never a constant. */
        save_int:
-          bcwrite_uleb128(ctx, 2*(uint32_t)k | ((uint32_t)k & 0x80000000u));
+          p = lj_strfmt_wuleb128(p, 2*(uint32_t)k | ((uint32_t)k&0x80000000u));
-          if (k < 0) {
+          if (k < 0)
-            char *p = &ctx->sb.buf[ctx->sb.n-1];
+            p[-1] = (p[-1] & 7) | ((k>>27) & 0x18);
-            *p = (*p & 7) | ((k>>27) & 0x18);
-          }
          continue;
        }
      }
-      bcwrite_uleb128(ctx, 1+(2*o->u32.lo | (o->u32.lo & 0x80000000u)));
+      p = lj_strfmt_wuleb128(p, 1+(2*o->u32.lo | (o->u32.lo & 0x80000000u)));
-      if (o->u32.lo >= 0x80000000u) {
+      if (o->u32.lo >= 0x80000000u)
-        char *p = &ctx->sb.buf[ctx->sb.n-1];
+        p[-1] = (p[-1] & 7) | ((o->u32.lo>>27) & 0x18);
-        *p = (*p & 7) | ((o->u32.lo>>27) & 0x18);
+      p = lj_strfmt_wuleb128(p, o->u32.hi);
-      }
-      bcwrite_uleb128(ctx, o->u32.hi);
    }
  }
+  setsbufP(&ctx->sb, p);
 }
 /* Write bytecode instructions. */
-static void bcwrite_bytecode(BCWriteCtx *ctx, GCproto *pt)
+static char *bcwrite_bytecode(BCWriteCtx *ctx, char *p, GCproto *pt)
 {
  MSize nbc = pt->sizebc-1;  /* Omit the [JI]FUNC* header. */
 #if LJ_HASJIT
-  uint8_t *p = (uint8_t *)&ctx->sb.buf[ctx->sb.n];
+  uint8_t *q = (uint8_t *)p;
 #endif
-  bcwrite_block(ctx, proto_bc(pt)+1, nbc*(MSize)sizeof(BCIns));
+  p = lj_buf_wmem(p, proto_bc(pt)+1, nbc*(MSize)sizeof(BCIns));
+  UNUSED(ctx);
 #if LJ_HASJIT
  /* Unpatch modified bytecode containing ILOOP/JLOOP etc. */
  if ((pt->flags & PROTO_ILOOP) || pt->trace) {
-    jit_State *J = L2J(ctx->L);
+    jit_State *J = L2J(sbufL(&ctx->sb));
    MSize i;
-    for (i = 0; i < nbc; i++, p += sizeof(BCIns)) {
+    for (i = 0; i < nbc; i++, q += sizeof(BCIns)) {
-      BCOp op = (BCOp)p[LJ_ENDIAN_SELECT(0, 3)];
+      BCOp op = (BCOp)q[LJ_ENDIAN_SELECT(0, 3)];
      if (op == BC_IFORL || op == BC_IITERL || op == BC_ILOOP ||
          op == BC_JFORI) {
-        p[LJ_ENDIAN_SELECT(0, 3)] = (uint8_t)(op-BC_IFORL+BC_FORL);
+        q[LJ_ENDIAN_SELECT(0, 3)] = (uint8_t)(op-BC_IFORL+BC_FORL);
      } else if (op == BC_JFORL || op == BC_JITERL || op == BC_JLOOP) {
-        BCReg rd = p[LJ_ENDIAN_SELECT(2, 1)] + (p[LJ_ENDIAN_SELECT(3, 0)] << 8);
+        BCReg rd = q[LJ_ENDIAN_SELECT(2, 1)] + (q[LJ_ENDIAN_SELECT(3, 0)] << 8);
        BCIns ins = traceref(J, rd)->startins;
-        p[LJ_ENDIAN_SELECT(0, 3)] = (uint8_t)(op-BC_JFORL+BC_FORL);
+        q[LJ_ENDIAN_SELECT(0, 3)] = (uint8_t)(op-BC_JFORL+BC_FORL);
-        p[LJ_ENDIAN_SELECT(2, 1)] = bc_c(ins);
+        q[LJ_ENDIAN_SELECT(2, 1)] = bc_c(ins);
-        p[LJ_ENDIAN_SELECT(3, 0)] = bc_b(ins);
+        q[LJ_ENDIAN_SELECT(3, 0)] = bc_b(ins);
      }
    }
  }
 #endif
+  return p;
 }
 /* Write prototype. */
 static void bcwrite_proto(BCWriteCtx *ctx, GCproto *pt)
 {
  MSize sizedbg = 0;
+  char *p;
  /* Recursively write children of prototype. */
  if ((pt->flags & PROTO_CHILD)) {
@@ -285,31 +248,32 @@ static void bcwrite_proto(BCWriteCtx *ctx, GCproto *pt)
  }
  /* Start writing the prototype info to a buffer. */
-  lj_str_resetbuf(&ctx->sb);
+  p = lj_buf_need(&ctx->sb,
-  ctx->sb.n = 5;  /* Leave room for final size. */
+                  5+4+6*5+(pt->sizebc-1)*(MSize)sizeof(BCIns)+pt->sizeuv*2);
-  bcwrite_need(ctx, 4+6*5+(pt->sizebc-1)*(MSize)sizeof(BCIns)+pt->sizeuv*2);
+  p += 5;  /* Leave room for final size. */
  /* Write prototype header. */
-  bcwrite_byte(ctx, (pt->flags & (PROTO_CHILD|PROTO_VARARG|PROTO_FFI)));
+  *p++ = (pt->flags & (PROTO_CHILD|PROTO_VARARG|PROTO_FFI));
-  bcwrite_byte(ctx, pt->numparams);
+  *p++ = pt->numparams;
-  bcwrite_byte(ctx, pt->framesize);
+  *p++ = pt->framesize;
-  bcwrite_byte(ctx, pt->sizeuv);
+  *p++ = pt->sizeuv;
-  bcwrite_uleb128(ctx, pt->sizekgc);
+  p = lj_strfmt_wuleb128(p, pt->sizekgc);
-  bcwrite_uleb128(ctx, pt->sizekn);
+  p = lj_strfmt_wuleb128(p, pt->sizekn);
-  bcwrite_uleb128(ctx, pt->sizebc-1);
+  p = lj_strfmt_wuleb128(p, pt->sizebc-1);
  if (!ctx->strip) {
    if (proto_lineinfo(pt))
      sizedbg = pt->sizept - (MSize)((char *)proto_lineinfo(pt) - (char *)pt);
-    bcwrite_uleb128(ctx, sizedbg);
+    p = lj_strfmt_wuleb128(p, sizedbg);
    if (sizedbg) {
-      bcwrite_uleb128(ctx, pt->firstline);
+      p = lj_strfmt_wuleb128(p, pt->firstline);
-      bcwrite_uleb128(ctx, pt->numline);
+      p = lj_strfmt_wuleb128(p, pt->numline);
    }
  }
  /* Write bytecode instructions and upvalue refs. */
-  bcwrite_bytecode(ctx, pt);
+  p = bcwrite_bytecode(ctx, p, pt);
-  bcwrite_block(ctx, proto_uv(pt), pt->sizeuv*2);
+  p = lj_buf_wmem(p, proto_uv(pt), pt->sizeuv*2);
+  setsbufP(&ctx->sb, p);
  /* Write constants. */
  bcwrite_kgc(ctx, pt);
@@ -317,18 +281,19 @@ static void bcwrite_proto(BCWriteCtx *ctx, GCproto *pt)
  /* Write debug info, if not stripped. */
  if (sizedbg) {
-    bcwrite_need(ctx, sizedbg);
+    p = lj_buf_more(&ctx->sb, sizedbg);
-    bcwrite_block(ctx, proto_lineinfo(pt), sizedbg);
+    p = lj_buf_wmem(p, proto_lineinfo(pt), sizedbg);
+    setsbufP(&ctx->sb, p);
  }
  /* Pass buffer to writer function. */
  if (ctx->status == 0) {
-    MSize n = ctx->sb.n - 5;
+    MSize n = sbuflen(&ctx->sb) - 5;
    MSize nn = (lj_fls(n)+8)*9 >> 6;
-    ctx->sb.n = 5 - nn;
+    char *q = sbufB(&ctx->sb) + (5 - nn);
-    bcwrite_uleb128(ctx, n);  /* Fill in final size. */
+    p = lj_strfmt_wuleb128(q, n);  /* Fill in final size. */
-    lua_assert(ctx->sb.n == 5);
+    lua_assert(p == sbufB(&ctx->sb) + 5);
-    ctx->status = ctx->wfunc(ctx->L, ctx->sb.buf+5-nn, nn+n, ctx->wdata);
+    ctx->status = ctx->wfunc(sbufL(&ctx->sb), q, nn+n, ctx->wdata);
  }
 }
@@ -338,20 +303,20 @@ static void bcwrite_header(BCWriteCtx *ctx)
  GCstr *chunkname = proto_chunkname(ctx->pt);
  const char *name = strdata(chunkname);
  MSize len = chunkname->len;
-  lj_str_resetbuf(&ctx->sb);
+  char *p = lj_buf_need(&ctx->sb, 5+5+len);
-  bcwrite_need(ctx, 5+5+len);
+  *p++ = BCDUMP_HEAD1;
-  bcwrite_byte(ctx, BCDUMP_HEAD1);
+  *p++ = BCDUMP_HEAD2;
-  bcwrite_byte(ctx, BCDUMP_HEAD2);
+  *p++ = BCDUMP_HEAD3;
-  bcwrite_byte(ctx, BCDUMP_HEAD3);
+  *p++ = BCDUMP_VERSION;
-  bcwrite_byte(ctx, BCDUMP_VERSION);
+  *p++ = (ctx->strip ? BCDUMP_F_STRIP : 0) +
-  bcwrite_byte(ctx, (ctx->strip ? BCDUMP_F_STRIP : 0) +
+                       (LJ_BE ? BCDUMP_F_BE : 0) +
-                   (LJ_BE ? BCDUMP_F_BE : 0) +
+                       ((ctx->pt->flags & PROTO_FFI) ? BCDUMP_F_FFI : 0);
-                   ((ctx->pt->flags & PROTO_FFI) ? BCDUMP_F_FFI : 0));
  if (!ctx->strip) {
-    bcwrite_uleb128(ctx, len);
+    p = lj_strfmt_wuleb128(p, len);
-    bcwrite_block(ctx, name, len);
+    p = lj_buf_wmem(p, name, len);
  }
-  ctx->status = ctx->wfunc(ctx->L, ctx->sb.buf, ctx->sb.n, ctx->wdata);
+  ctx->status = ctx->wfunc(sbufL(&ctx->sb), sbufB(&ctx->sb),
+                           (MSize)(p - sbufB(&ctx->sb)), ctx->wdata);
 }
 /* Write footer of bytecode dump. */
@@ -359,7 +324,7 @@ static void bcwrite_footer(BCWriteCtx *ctx)
 {
  if (ctx->status == 0) {
    uint8_t zero = 0;
-    ctx->status = ctx->wfunc(ctx->L, &zero, 1, ctx->wdata);
+    ctx->status = ctx->wfunc(sbufL(&ctx->sb), &zero, 1, ctx->wdata);
  }
 }
@@ -367,8 +332,8 @@ static void bcwrite_footer(BCWriteCtx *ctx)
 static TValue *cpwriter(lua_State *L, lua_CFunction dummy, void *ud)
 {
  BCWriteCtx *ctx = (BCWriteCtx *)ud;
-  UNUSED(dummy);
+  UNUSED(L); UNUSED(dummy);
-  lj_str_resizebuf(L, &ctx->sb, 1024);  /* Avoids resize for most prototypes. */
+  lj_buf_need(&ctx->sb, 1024);  /* Avoids resize for most prototypes. */
  bcwrite_header(ctx);
  bcwrite_proto(ctx, ctx->pt);
  bcwrite_footer(ctx);
@@ -381,16 +346,15 @@ int lj_bcwrite(lua_State *L, GCproto *pt, lua_Writer writer, void *data,
 {
  BCWriteCtx ctx;
  int status;
-  ctx.L = L;
  ctx.pt = pt;
  ctx.wfunc = writer;
  ctx.wdata = data;
  ctx.strip = strip;
  ctx.status = 0;
-  lj_str_initbuf(&ctx.sb);
+  lj_buf_init(L, &ctx.sb);
  status = lj_vm_cpcall(L, NULL, &ctx, cpwriter);
  if (status == 0) status = ctx.status;
-  lj_str_freebuf(G(ctx.L), &ctx.sb);
+  lj_buf_free(G(sbufL(&ctx.sb)), &ctx.sb);
  return status;
 }
diff --git a/src/lj_buf.c b/src/lj_buf.c
new file mode 100644
index 00000000..1786c10d
--- /dev/null
+++ b/src/lj_buf.c
@@ -0,0 +1,222 @@
+/*
+** Buffer handling.
+** Copyright (C) 2005-2013 Mike Pall. See Copyright Notice in luajit.h
+*/
+#include <stdio.h>
+#define lj_buf_c
+#define LUA_CORE
+#include "lj_obj.h"
+#include "lj_gc.h"
+#include "lj_err.h"
+#include "lj_buf.h"
+#include "lj_str.h"
+#include "lj_tab.h"
+#include "lj_strfmt.h"
+/* -- Buffer management --------------------------------------------------- */
+LJ_NOINLINE void LJ_FASTCALL lj_buf_grow(SBuf *sb, char *en)
+{
+  lua_State *L = sbufL(sb);
+  char *b = sbufB(sb);
+  MSize sz = (MSize)(en - b);
+  MSize osz = (MSize)(sbufE(sb) - b), nsz = osz;
+  MSize n = (MSize)(sbufP(sb) - b);
+  if (LJ_UNLIKELY(sz > LJ_MAX_MEM))
+    lj_err_mem(L);
+  if (nsz < LJ_MIN_SBUF) nsz = LJ_MIN_SBUF;
+  while (nsz < sz) nsz += nsz;
+  b = (char *)lj_mem_realloc(L, b, osz, nsz);
+  setmref(sb->b, b);
+  setmref(sb->p, b + n);
+  setmref(sb->e, b + nsz);
+}
+void LJ_FASTCALL lj_buf_shrink(lua_State *L, SBuf *sb)
+{
+  char *b = sbufB(sb);
+  MSize osz = (MSize)(sbufE(sb) - b);
+  if (osz > 2*LJ_MIN_SBUF) {
+    MSize n = (MSize)(sbufP(sb) - b);
+    b = lj_mem_realloc(L, b, osz, (osz >> 1));
+    setmref(sb->b, b);
+    setmref(sb->p, b + n);
+    setmref(sb->e, b + (osz >> 1));
+  }
+}
+char * LJ_FASTCALL lj_buf_tmp(lua_State *L, MSize sz)
+{
+  SBuf *sb = &G(L)->tmpbuf;
+  setsbufL(sb, L);
+  return lj_buf_need(sb, sz);
+}
+/* -- Low-level buffer put operations ------------------------------------- */
+SBuf *lj_buf_putmem(SBuf *sb, const void *q, MSize len)
+{
+  char *p = lj_buf_more(sb, len);
+  p = lj_buf_wmem(p, q, len);
+  setsbufP(sb, p);
+  return sb;
+}
+#if LJ_HASJIT
+SBuf * LJ_FASTCALL lj_buf_putchar(SBuf *sb, int c)
+{
+  char *p = lj_buf_more(sb, 1);
+  *p++ = (char)c;
+  setsbufP(sb, p);
+  return sb;
+}
+#endif
+SBuf * LJ_FASTCALL lj_buf_putstr(SBuf *sb, GCstr *s)
+{
+  MSize len = s->len;
+  char *p = lj_buf_more(sb, len);
+  p = lj_buf_wmem(p, strdata(s), len);
+  setsbufP(sb, p);
+  return sb;
+}
+/* -- High-level buffer put operations ------------------------------------ */
+SBuf * LJ_FASTCALL lj_buf_putstr_reverse(SBuf *sb, GCstr *s)
+{
+  MSize len = s->len;
+  char *p = lj_buf_more(sb, len), *e = p+len;
+  const char *q = strdata(s)+len-1;
+  while (p < e)
+    *p++ = *q--;
+  setsbufP(sb, p);
+  return sb;
+}
+SBuf * LJ_FASTCALL lj_buf_putstr_lower(SBuf *sb, GCstr *s)
+{
+  MSize len = s->len;
+  char *p = lj_buf_more(sb, len), *e = p+len;
+  const char *q = strdata(s);
+  for (; p < e; p++, q++) {
+    uint32_t c = *(unsigned char *)q;
+#if LJ_TARGET_PPC
+    *p = c + ((c >= 'A' && c <= 'Z') << 5);
+#else
+    if (c >= 'A' && c <= 'Z') c += 0x20;
+    *p = c;
+#endif
+  }
+  setsbufP(sb, p);
+  return sb;
+}
+SBuf * LJ_FASTCALL lj_buf_putstr_upper(SBuf *sb, GCstr *s)
+{
+  MSize len = s->len;
+  char *p = lj_buf_more(sb, len), *e = p+len;
+  const char *q = strdata(s);
+  for (; p < e; p++, q++) {
+    uint32_t c = *(unsigned char *)q;
+#if LJ_TARGET_PPC
+    *p = c - ((c >= 'a' && c <= 'z') << 5);
+#else
+    if (c >= 'a' && c <= 'z') c -= 0x20;
+    *p = c;
+#endif
+  }
+  setsbufP(sb, p);
+  return sb;
+}
+SBuf *lj_buf_putstr_rep(SBuf *sb, GCstr *s, int32_t rep)
+{
+  MSize len = s->len;
+  if (rep > 0 && len) {
+    uint64_t tlen = (uint64_t)rep * len;
+    char *p;
+    if (LJ_UNLIKELY(tlen > LJ_MAX_STR))
+      lj_err_mem(sbufL(sb));
+    p = lj_buf_more(sb, (MSize)tlen);
+    if (len == 1) {  /* Optimize a common case. */
+      uint32_t c = strdata(s)[0];
+      do { *p++ = c; } while (--rep > 0);
+    } else {
+      const char *e = strdata(s) + len;
+      do {
+        const char *q = strdata(s);
+        do { *p++ = *q++; } while (q < e);
+      } while (--rep > 0);
+    }
+    setsbufP(sb, p);
+  }
+  return sb;
+}
+SBuf *lj_buf_puttab(SBuf *sb, GCtab *t, GCstr *sep, int32_t i, int32_t e)
+{
+  MSize seplen = sep ? sep->len : 0;
+  if (i <= e) {
+    for (;;) {
+      cTValue *o = lj_tab_getint(t, i);
+      char *p;
+      if (!o) {
+      badtype:  /* Error: bad element type. */
+        setsbufP(sb, (intptr_t)i);  /* Store failing index. */
+        return NULL;
+      } else if (tvisstr(o)) {
+        MSize len = strV(o)->len;
+        p = lj_buf_wmem(lj_buf_more(sb, len + seplen), strVdata(o), len);
+      } else if (tvisint(o)) {
+        p = lj_strfmt_wint(lj_buf_more(sb, STRFMT_MAXBUF_INT+seplen), intV(o));
+      } else if (tvisnum(o)) {
+        p = lj_strfmt_wnum(lj_buf_more(sb, STRFMT_MAXBUF_NUM+seplen), o);
+      } else {
+        goto badtype;
+      }
+      if (i++ == e) {
+        setsbufP(sb, p);
+        break;
+      }
+      if (seplen) p = lj_buf_wmem(p, strdata(sep), seplen);
+      setsbufP(sb, p);
+    }
+  }
+  return sb;
+}
+/* -- Miscellaneous buffer operations ------------------------------------- */
+GCstr * LJ_FASTCALL lj_buf_tostr(SBuf *sb)
+{
+  return lj_str_new(sbufL(sb), sbufB(sb), sbuflen(sb));
+}
+/* Concatenate two strings. */
+GCstr *lj_buf_cat2str(lua_State *L, GCstr *s1, GCstr *s2)
+{
+  MSize len1 = s1->len, len2 = s2->len;
+  char *buf = lj_buf_tmp(L, len1 + len2);
+  memcpy(buf, strdata(s1), len1);
+  memcpy(buf+len1, strdata(s2), len2);
+  return lj_str_new(L, buf, len1 + len2);
+}
+/* Read ULEB128 from buffer. */
+uint32_t LJ_FASTCALL lj_buf_ruleb128(const char **pp)
+{
+  const uint8_t *p = (const uint8_t *)*pp;
+  uint32_t v = *p++;
+  if (LJ_UNLIKELY(v >= 0x80)) {
+    int sh = 0;
+    v &= 0x7f;
+    do { v |= ((*p & 0x7f) << (sh += 7)); } while (*p++ >= 0x80);
+  }
+  *pp = (const char *)p;
+  return v;
+}
diff --git a/src/lj_buf.h b/src/lj_buf.h
new file mode 100644
index 00000000..5f78c4a9
--- /dev/null
+++ b/src/lj_buf.h
@@ -0,0 +1,105 @@
+/*
+** Buffer handling.
+** Copyright (C) 2005-2013 Mike Pall. See Copyright Notice in luajit.h
+*/
+#ifndef _LJ_BUF_H
+#define _LJ_BUF_H
+#include "lj_obj.h"
+#include "lj_gc.h"
+#include "lj_str.h"
+/* Resizable string buffers. Struct definition in lj_obj.h. */
+#define sbufB(sb)       (mref((sb)->b, char))
+#define sbufP(sb)       (mref((sb)->p, char))
+#define sbufE(sb)       (mref((sb)->e, char))
+#define sbufL(sb)       (mref((sb)->L, lua_State))
+#define sbufsz(sb)      ((MSize)(sbufE((sb)) - sbufB((sb))))
+#define sbuflen(sb)     ((MSize)(sbufP((sb)) - sbufB((sb))))
+#define setsbufP(sb, q) (setmref((sb)->p, (q)))
+#define setsbufL(sb, l) (setmref((sb)->L, (l)))
+/* Buffer management */
+LJ_FUNC void LJ_FASTCALL lj_buf_grow(SBuf *sb, char *en);
+LJ_FUNC void LJ_FASTCALL lj_buf_shrink(lua_State *L, SBuf *sb);
+LJ_FUNC char * LJ_FASTCALL lj_buf_tmp(lua_State *L, MSize sz);
+static LJ_AINLINE void lj_buf_init(lua_State *L, SBuf *sb)
+{
+  setsbufL(sb, L);
+  setmref(sb->p, NULL); setmref(sb->e, NULL); setmref(sb->b, NULL);
+}
+static LJ_AINLINE void lj_buf_reset(SBuf *sb)
+{
+  setmrefr(sb->p, sb->b);
+}
+static LJ_AINLINE SBuf *lj_buf_tmp_(lua_State *L)
+{
+  SBuf *sb = &G(L)->tmpbuf;
+  setsbufL(sb, L);
+  lj_buf_reset(sb);
+  return sb;
+}
+static LJ_AINLINE void lj_buf_free(global_State *g, SBuf *sb)
+{
+  lj_mem_free(g, sbufB(sb), sbufsz(sb));
+}
+static LJ_AINLINE char *lj_buf_need(SBuf *sb, MSize sz)
+{
+  char *en = sbufB(sb) + sz;
+  if (LJ_UNLIKELY(en > sbufE(sb)))
+    lj_buf_grow(sb, en);
+  return sbufB(sb);
+}
+static LJ_AINLINE char *lj_buf_more(SBuf *sb, MSize sz)
+{
+  char *en = sbufP(sb) + sz;
+  if (LJ_UNLIKELY(en > sbufE(sb)))
+    lj_buf_grow(sb, en);
+  return sbufP(sb);
+}
+/* Low-level buffer put operations */
+LJ_FUNC SBuf *lj_buf_putmem(SBuf *sb, const void *q, MSize len);
+#if LJ_HASJIT
+LJ_FUNC SBuf * LJ_FASTCALL lj_buf_putchar(SBuf *sb, int c);
+#endif
+LJ_FUNC SBuf * LJ_FASTCALL lj_buf_putstr(SBuf *sb, GCstr *s);
+static LJ_AINLINE char *lj_buf_wmem(char *p, const void *q, MSize len)
+{
+  return (char *)memcpy(p, q, len) + len;
+}
+static LJ_AINLINE void lj_buf_putb(SBuf *sb, int c)
+{
+  char *p = lj_buf_more(sb, 1);
+  *p++ = (char)c;
+  setsbufP(sb, p);
+}
+/* High-level buffer put operations */
+LJ_FUNCA SBuf * LJ_FASTCALL lj_buf_putstr_reverse(SBuf *sb, GCstr *s);
+LJ_FUNCA SBuf * LJ_FASTCALL lj_buf_putstr_lower(SBuf *sb, GCstr *s);
+LJ_FUNCA SBuf * LJ_FASTCALL lj_buf_putstr_upper(SBuf *sb, GCstr *s);
+LJ_FUNC SBuf *lj_buf_putstr_rep(SBuf *sb, GCstr *s, int32_t rep);
+LJ_FUNC SBuf *lj_buf_puttab(SBuf *sb, GCtab *t, GCstr *sep,
+                            int32_t i, int32_t e);
+/* Miscellaneous buffer operations */
+LJ_FUNCA GCstr * LJ_FASTCALL lj_buf_tostr(SBuf *sb);
+LJ_FUNC GCstr *lj_buf_cat2str(lua_State *L, GCstr *s1, GCstr *s2);
+LJ_FUNC uint32_t LJ_FASTCALL lj_buf_ruleb128(const char **pp);
+static LJ_AINLINE GCstr *lj_buf_str(lua_State *L, SBuf *sb)
+{
+  return lj_str_new(L, sbufB(sb), sbuflen(sb));
+}
+#endif
diff --git a/src/lj_carith.c b/src/lj_carith.c
index afe7e682..bb810af8 100644
--- a/src/lj_carith.c
+++ b/src/lj_carith.c
@@ -11,10 +11,12 @@
 #include "lj_err.h"
 #include "lj_tab.h"
 #include "lj_meta.h"
+#include "lj_ir.h"
 #include "lj_ctype.h"
 #include "lj_cconv.h"
 #include "lj_cdata.h"
 #include "lj_carith.h"
+#include "lj_strscan.h"
 /* -- C data arithmetic --------------------------------------------------- */
@@ -270,6 +272,80 @@ int lj_carith_op(lua_State *L, MMS mm)
  return lj_carith_meta(L, cts, &ca, mm);
 }
+/* -- 64 bit bit operations helpers --------------------------------------- */
+#if LJ_64
+#define B64DEF(name) \
+  static LJ_AINLINE uint64_t lj_carith_##name(uint64_t x, int32_t sh)
+#else
+/* Not inlined on 32 bit archs, since some of these are quite lengthy. */
+#define B64DEF(name) \
+  uint64_t LJ_NOINLINE lj_carith_##name(uint64_t x, int32_t sh)
+#endif
+B64DEF(shl64) { return x << (sh&63); }
+B64DEF(shr64) { return x >> (sh&63); }
+B64DEF(sar64) { return (uint64_t)((int64_t)x >> (sh&63)); }
+B64DEF(rol64) { return lj_rol(x, (sh&63)); }
+B64DEF(ror64) { return lj_ror(x, (sh&63)); }
+#undef B64DEF
+uint64_t lj_carith_shift64(uint64_t x, int32_t sh, int op)
+{
+  switch (op) {
+  case IR_BSHL-IR_BSHL: x = lj_carith_shl64(x, sh); break;
+  case IR_BSHR-IR_BSHL: x = lj_carith_shr64(x, sh); break;
+  case IR_BSAR-IR_BSHL: x = lj_carith_sar64(x, sh); break;
+  case IR_BROL-IR_BSHL: x = lj_carith_rol64(x, sh); break;
+  case IR_BROR-IR_BSHL: x = lj_carith_ror64(x, sh); break;
+  default: lua_assert(0); break;
+  }
+  return x;
+}
+/* Equivalent to lj_lib_checkbit(), but handles cdata. */
+uint64_t lj_carith_check64(lua_State *L, int narg, CTypeID *id)
+{
+  TValue *o = L->base + narg-1;
+  if (o >= L->top) {
+  err:
+    lj_err_argt(L, narg, LUA_TNUMBER);
+  } else if (LJ_LIKELY(tvisnumber(o))) {
+    /* Handled below. */
+  } else if (tviscdata(o)) {
+    CTState *cts = ctype_cts(L);
+    uint8_t *sp = (uint8_t *)cdataptr(cdataV(o));
+    CTypeID sid = cdataV(o)->ctypeid;
+    CType *s = ctype_get(cts, sid);
+    uint64_t x;
+    if (ctype_isref(s->info)) {
+      sp = *(void **)sp;
+      sid = ctype_cid(s->info);
+    }
+    s = ctype_raw(cts, sid);
+    if (ctype_isenum(s->info)) s = ctype_child(cts, s);
+    if ((s->info & (CTMASK_NUM|CTF_BOOL|CTF_FP|CTF_UNSIGNED)) ==
+        CTINFO(CT_NUM, CTF_UNSIGNED) && s->size == 8)
+      *id = CTID_UINT64;  /* Use uint64_t, since it has the highest rank. */
+    else if (!*id)
+      *id = CTID_INT64;  /* Use int64_t, unless already set. */
+    lj_cconv_ct_ct(cts, ctype_get(cts, *id), s,
+                   (uint8_t *)&x, sp, CCF_ARG(narg));
+    return x;
+  } else if (!(tvisstr(o) && lj_strscan_number(strV(o), o))) {
+    goto err;
+  }
+  if (LJ_LIKELY(tvisint(o))) {
+    return (uint32_t)intV(o);
+  } else {
+    int32_t i = lj_num2bit(numV(o));
+    if (LJ_DUALNUM) setintV(o, i);
+    return (uint32_t)i;
+  }
+}
 /* -- 64 bit integer arithmetic helpers ----------------------------------- */
 #if LJ_32 && LJ_HASJIT
diff --git a/src/lj_carith.h b/src/lj_carith.h
index ae17df00..b1a65d35 100644
--- a/src/lj_carith.h
+++ b/src/lj_carith.h
@@ -12,6 +12,16 @@
 LJ_FUNC int lj_carith_op(lua_State *L, MMS mm);
+#if LJ_32
+LJ_FUNC uint64_t lj_carith_shl64(uint64_t x, int32_t sh);
+LJ_FUNC uint64_t lj_carith_shr64(uint64_t x, int32_t sh);
+LJ_FUNC uint64_t lj_carith_sar64(uint64_t x, int32_t sh);
+LJ_FUNC uint64_t lj_carith_rol64(uint64_t x, int32_t sh);
+LJ_FUNC uint64_t lj_carith_ror64(uint64_t x, int32_t sh);
+#endif
+LJ_FUNC uint64_t lj_carith_shift64(uint64_t x, int32_t sh, int op);
+LJ_FUNC uint64_t lj_carith_check64(lua_State *L, int narg, CTypeID *id);
 #if LJ_32 && LJ_HASJIT
 LJ_FUNC int64_t lj_carith_mul64(int64_t x, int64_t k);
 #endif
diff --git a/src/lj_ccall.c b/src/lj_ccall.c
index eb73604f..995729b0 100644
--- a/src/lj_ccall.c
+++ b/src/lj_ccall.c
@@ -9,7 +9,6 @@
 #include "lj_gc.h"
 #include "lj_err.h"
-#include "lj_str.h"
 #include "lj_tab.h"
 #include "lj_ctype.h"
 #include "lj_cconv.h"
diff --git a/src/lj_cdata.c b/src/lj_cdata.c
index 10f4809c..aa3bd1a5 100644
--- a/src/lj_cdata.c
+++ b/src/lj_cdata.c
@@ -9,7 +9,6 @@
 #include "lj_gc.h"
 #include "lj_err.h"
-#include "lj_str.h"
 #include "lj_tab.h"
 #include "lj_ctype.h"
 #include "lj_cconv.h"
@@ -27,12 +26,12 @@ GCcdata *lj_cdata_newref(CTState *cts, const void *p, CTypeID id)
 }
 /* Allocate variable-sized or specially aligned C data object. */
-GCcdata *lj_cdata_newv(CTState *cts, CTypeID id, CTSize sz, CTSize align)
+GCcdata *lj_cdata_newv(lua_State *L, CTypeID id, CTSize sz, CTSize align)
 {
  global_State *g;
  MSize extra = sizeof(GCcdataVar) + sizeof(GCcdata) +
                (align > CT_MEMALIGN ? (1u<<align) - (1u<<CT_MEMALIGN) : 0);
-  char *p = lj_mem_newt(cts->L, extra + sz, char);
+  char *p = lj_mem_newt(L, extra + sz, char);
  uintptr_t adata = (uintptr_t)p + sizeof(GCcdataVar) + sizeof(GCcdata);
  uintptr_t almask = (1u << align) - 1u;
  GCcdata *cd = (GCcdata *)(((adata + almask) & ~almask) - sizeof(GCcdata));
@@ -40,7 +39,7 @@ GCcdata *lj_cdata_newv(CTState *cts, CTypeID id, CTSize sz, CTSize align)
  cdatav(cd)->offset = (uint16_t)((char *)cd - p);
  cdatav(cd)->extra = extra;
  cdatav(cd)->len = sz;
-  g = cts->g;
+  g = G(L);
  setgcrefr(cd->nextgc, g->gc.root);
  setgcref(g->gc.root, obj2gco(cd));
  newwhite(g, obj2gco(cd));
@@ -76,21 +75,20 @@ void LJ_FASTCALL lj_cdata_free(global_State *g, GCcdata *cd)
  }
 }
-TValue * LJ_FASTCALL lj_cdata_setfin(lua_State *L, GCcdata *cd)
+void lj_cdata_setfin(lua_State *L, GCcdata *cd, GCobj *obj, uint32_t it)
 {
-  global_State *g = G(L);
+  GCtab *t = ctype_ctsG(G(L))->finalizer;
-  GCtab *t = ctype_ctsG(g)->finalizer;
  if (gcref(t->metatable)) {
    /* Add cdata to finalizer table, if still enabled. */
    TValue *tv, tmp;
    setcdataV(L, &tmp, cd);
    lj_gc_anybarriert(L, t);
    tv = lj_tab_set(L, t, &tmp);
-    cd->marked |= LJ_GC_CDATA_FIN;
+    setgcV(L, tv, obj, it);
-    return tv;
+    if (!tvisnil(tv))
-  } else {
+      cd->marked |= LJ_GC_CDATA_FIN;
-    /* Otherwise return dummy TValue. */
+    else
-    return &g->tmptv;
+      cd->marked &= ~LJ_GC_CDATA_FIN;
  }
 }
diff --git a/src/lj_cdata.h b/src/lj_cdata.h
index 0c81b02b..7db1ca1e 100644
--- a/src/lj_cdata.h
+++ b/src/lj_cdata.h
@@ -58,11 +58,12 @@ static LJ_AINLINE GCcdata *lj_cdata_new_(lua_State *L, CTypeID id, CTSize sz)
 }
 LJ_FUNC GCcdata *lj_cdata_newref(CTState *cts, const void *pp, CTypeID id);
-LJ_FUNC GCcdata *lj_cdata_newv(CTState *cts, CTypeID id, CTSize sz,
+LJ_FUNC GCcdata *lj_cdata_newv(lua_State *L, CTypeID id, CTSize sz,
                               CTSize align);
 LJ_FUNC void LJ_FASTCALL lj_cdata_free(global_State *g, GCcdata *cd);
-LJ_FUNCA TValue * LJ_FASTCALL lj_cdata_setfin(lua_State *L, GCcdata *cd);
+LJ_FUNC void lj_cdata_setfin(lua_State *L, GCcdata *cd, GCobj *obj,
+                             uint32_t it);
 LJ_FUNC CType *lj_cdata_index(CTState *cts, GCcdata *cd, cTValue *key,
                              uint8_t **pp, CTInfo *qual);
diff --git a/src/lj_clib.c b/src/lj_clib.c
index 23d1f182..263028e4 100644
--- a/src/lj_clib.c
+++ b/src/lj_clib.c
@@ -16,6 +16,7 @@
 #include "lj_cconv.h"
 #include "lj_cdata.h"
 #include "lj_clib.h"
+#include "lj_strfmt.h"
 /* -- OS-specific functions ----------------------------------------------- */
@@ -61,7 +62,7 @@ static const char *clib_extname(lua_State *L, const char *name)
 #endif
     ) {
    if (!strchr(name, '.')) {
-      name = lj_str_pushf(L, CLIB_SOEXT, name);
+      name = lj_strfmt_pushf(L, CLIB_SOEXT, name);
      L->top--;
 #ifdef __CYGWIN__
    } else {
@@ -70,7 +71,7 @@ static const char *clib_extname(lua_State *L, const char *name)
    }
    if (!(name[0] == CLIB_SOPREFIX[0] && name[1] == CLIB_SOPREFIX[1] &&
          name[2] == CLIB_SOPREFIX[2])) {
-      name = lj_str_pushf(L, CLIB_SOPREFIX "%s", name);
+      name = lj_strfmt_pushf(L, CLIB_SOPREFIX "%s", name);
      L->top--;
    }
  }
@@ -178,7 +179,7 @@ LJ_NORET LJ_NOINLINE static void clib_error(lua_State *L, const char *fmt,
  if (!FormatMessageA(FORMAT_MESSAGE_IGNORE_INSERTS|FORMAT_MESSAGE_FROM_SYSTEM,
                      NULL, err, 0, buf, sizeof(buf), NULL))
    buf[0] = '\0';
-  lj_err_callermsg(L, lj_str_pushf(L, fmt, name, buf));
+  lj_err_callermsg(L, lj_strfmt_pushf(L, fmt, name, buf));
 }
 static int clib_needext(const char *s)
@@ -193,7 +194,7 @@ static int clib_needext(const char *s)
 static const char *clib_extname(lua_State *L, const char *name)
 {
  if (clib_needext(name)) {
-    name = lj_str_pushf(L, "%s.dll", name);
+    name = lj_strfmt_pushf(L, "%s.dll", name);
    L->top--;
  }
  return name;
@@ -266,7 +267,7 @@ static void *clib_getsym(CLibrary *cl, const char *name)
 LJ_NORET LJ_NOINLINE static void clib_error(lua_State *L, const char *fmt,
                                            const char *name)
 {
-  lj_err_callermsg(L, lj_str_pushf(L, fmt, name, "no support for this OS"));
+  lj_err_callermsg(L, lj_strfmt_pushf(L, fmt, name, "no support for this OS"));
 }
 static void *clib_loadlib(lua_State *L, const char *name, int global)
@@ -350,7 +351,7 @@ TValue *lj_clib_index(lua_State *L, CLibrary *cl, GCstr *name)
        CTInfo cconv = ctype_cconv(ct->info);
        if (cconv == CTCC_FASTCALL || cconv == CTCC_STDCALL) {
          CTSize sz = clib_func_argsize(cts, ct);
-          const char *symd = lj_str_pushf(L,
+          const char *symd = lj_strfmt_pushf(L,
                               cconv == CTCC_FASTCALL ? "@%s@%d" : "_%s@%d",
                               sym, sz);
          L->top--;
diff --git a/src/lj_cparse.c b/src/lj_cparse.c
index 107c0381..4392b7c4 100644
--- a/src/lj_cparse.c
+++ b/src/lj_cparse.c
@@ -9,13 +9,14 @@
 #include "lj_gc.h"
 #include "lj_err.h"
-#include "lj_str.h"
+#include "lj_buf.h"
 #include "lj_ctype.h"
 #include "lj_cparse.h"
 #include "lj_frame.h"
 #include "lj_vm.h"
 #include "lj_char.h"
 #include "lj_strscan.h"
+#include "lj_strfmt.h"
 /*
 ** Important note: this is NOT a validating C parser! This is a minimal
@@ -46,9 +47,9 @@ static const char *cp_tok2str(CPState *cp, CPToken tok)
  if (tok > CTOK_OFS)
    return ctoknames[tok-CTOK_OFS-1];
  else if (!lj_char_iscntrl(tok))
-    return lj_str_pushf(cp->L, "%c", tok);
+    return lj_strfmt_pushf(cp->L, "%c", tok);
  else
-    return lj_str_pushf(cp->L, "char(%d)", tok);
+    return lj_strfmt_pushf(cp->L, "char(%d)", tok);
 }
 /* End-of-line? */
@@ -85,24 +86,10 @@ static LJ_AINLINE CPChar cp_get(CPState *cp)
  return cp_get_bs(cp);
 }
-/* Grow save buffer. */
-static LJ_NOINLINE void cp_save_grow(CPState *cp, CPChar c)
-{
-  MSize newsize;
-  if (cp->sb.sz >= CPARSE_MAX_BUF/2)
-    cp_err(cp, LJ_ERR_XELEM);
-  newsize = cp->sb.sz * 2;
-  lj_str_resizebuf(cp->L, &cp->sb, newsize);
-  cp->sb.buf[cp->sb.n++] = (char)c;
-}
 /* Save character in buffer. */
 static LJ_AINLINE void cp_save(CPState *cp, CPChar c)
 {
-  if (LJ_UNLIKELY(cp->sb.n + 1 > cp->sb.sz))
+  lj_buf_putb(&cp->sb, c);
-    cp_save_grow(cp, c);
-  else
-    cp->sb.buf[cp->sb.n++] = (char)c;
 }
 /* Skip line break. Handles "\n", "\r", "\r\n" or "\n\r". */
@@ -122,20 +109,20 @@ LJ_NORET static void cp_errmsg(CPState *cp, CPToken tok, ErrMsg em, ...)
    tokstr = NULL;
  } else if (tok == CTOK_IDENT || tok == CTOK_INTEGER || tok == CTOK_STRING ||
             tok >= CTOK_FIRSTDECL) {
-    if (cp->sb.n == 0) cp_save(cp, '$');
+    if (sbufP(&cp->sb) == sbufB(&cp->sb)) cp_save(cp, '$');
    cp_save(cp, '\0');
-    tokstr = cp->sb.buf;
+    tokstr = sbufB(&cp->sb);
  } else {
    tokstr = cp_tok2str(cp, tok);
  }
  L = cp->L;
  va_start(argp, em);
-  msg = lj_str_pushvf(L, err2msg(em), argp);
+  msg = lj_strfmt_pushvf(L, err2msg(em), argp);
  va_end(argp);
  if (tokstr)
-    msg = lj_str_pushf(L, err2msg(LJ_ERR_XNEAR), msg, tokstr);
+    msg = lj_strfmt_pushf(L, err2msg(LJ_ERR_XNEAR), msg, tokstr);
  if (cp->linenumber > 1)
-    msg = lj_str_pushf(L, "%s at line %d", msg, cp->linenumber);
+    msg = lj_strfmt_pushf(L, "%s at line %d", msg, cp->linenumber);
  lj_err_callermsg(L, msg);
 }
@@ -164,7 +151,7 @@ static CPToken cp_number(CPState *cp)
  TValue o;
  do { cp_save(cp, cp->c); } while (lj_char_isident(cp_get(cp)));
  cp_save(cp, '\0');
-  fmt = lj_strscan_scan((const uint8_t *)cp->sb.buf, &o, STRSCAN_OPT_C);
+  fmt = lj_strscan_scan((const uint8_t *)sbufB(&cp->sb), &o, STRSCAN_OPT_C);
  if (fmt == STRSCAN_INT) cp->val.id = CTID_INT32;
  else if (fmt == STRSCAN_U32) cp->val.id = CTID_UINT32;
  else if (!(cp->mode & CPARSE_MODE_SKIP))
@@ -177,7 +164,7 @@ static CPToken cp_number(CPState *cp)
 static CPToken cp_ident(CPState *cp)
 {
  do { cp_save(cp, cp->c); } while (lj_char_isident(cp_get(cp)));
-  cp->str = lj_str_new(cp->L, cp->sb.buf, cp->sb.n);
+  cp->str = lj_buf_str(cp->L, &cp->sb);
  cp->val.id = lj_ctype_getname(cp->cts, &cp->ct, cp->str, cp->tmask);
  if (ctype_type(cp->ct->info) == CT_KW)
    return ctype_cid(cp->ct->info);
@@ -263,11 +250,11 @@ static CPToken cp_string(CPState *cp)
  }
  cp_get(cp);
  if (delim == '"') {
-    cp->str = lj_str_new(cp->L, cp->sb.buf, cp->sb.n);
+    cp->str = lj_buf_str(cp->L, &cp->sb);
    return CTOK_STRING;
  } else {
-    if (cp->sb.n != 1) cp_err_token(cp, '\'');
+    if (sbuflen(&cp->sb) != 1) cp_err_token(cp, '\'');
-    cp->val.i32 = (int32_t)(char)cp->sb.buf[0];
+    cp->val.i32 = (int32_t)(char)*sbufB(&cp->sb);
    cp->val.id = CTID_INT32;
    return CTOK_INTEGER;
  }
@@ -296,7 +283,7 @@ static void cp_comment_cpp(CPState *cp)
 /* Lexical scanner for C. Only a minimal subset is implemented. */
 static CPToken cp_next_(CPState *cp)
 {
-  lj_str_resetbuf(&cp->sb);
+  lj_buf_reset(&cp->sb);
  for (;;) {
    if (lj_char_isident(cp->c))
      return lj_char_isdigit(cp->c) ? cp_number(cp) : cp_ident(cp);
@@ -380,8 +367,7 @@ static void cp_init(CPState *cp)
  cp->depth = 0;
  cp->curpack = 0;
  cp->packstack[0] = 255;
-  lj_str_initbuf(&cp->sb);
+  lj_buf_init(cp->L, &cp->sb);
-  lj_str_resizebuf(cp->L, &cp->sb, LJ_MIN_SBUF);
  lua_assert(cp->p != NULL);
  cp_get(cp);  /* Read-ahead first char. */
  cp->tok = 0;
@@ -393,7 +379,7 @@ static void cp_init(CPState *cp)
 static void cp_cleanup(CPState *cp)
 {
  global_State *g = G(cp->L);
-  lj_str_freebuf(g, &cp->sb);
+  lj_buf_free(g, &cp->sb);
 }
 /* Check and consume optional token. */
@@ -1012,7 +998,7 @@ static void cp_decl_asm(CPState *cp, CPDecl *decl)
  if (cp->tok == CTOK_STRING) {
    GCstr *str = cp->str;
    while (cp_next(cp) == CTOK_STRING) {
-      lj_str_pushf(cp->L, "%s%s", strdata(str), strdata(cp->str));
+      lj_strfmt_pushf(cp->L, "%s%s", strdata(str), strdata(cp->str));
      cp->L->top--;
      str = strV(cp->L->top);
    }
diff --git a/src/lj_crecord.c b/src/lj_crecord.c
index 8577de2c..c3b01f63 100644
--- a/src/lj_crecord.c
+++ b/src/lj_crecord.c
@@ -11,13 +11,13 @@
 #if LJ_HASJIT && LJ_HASFFI
 #include "lj_err.h"
-#include "lj_str.h"
 #include "lj_tab.h"
 #include "lj_frame.h"
 #include "lj_ctype.h"
 #include "lj_cdata.h"
 #include "lj_cparse.h"
 #include "lj_cconv.h"
+#include "lj_carith.h"
 #include "lj_clib.h"
 #include "lj_ccall.h"
 #include "lj_ff.h"
@@ -31,6 +31,7 @@
 #include "lj_snap.h"
 #include "lj_crecord.h"
 #include "lj_dispatch.h"
+#include "lj_strfmt.h"
 /* Some local macros to save typing. Undef'd at the end. */
 #define IR(ref)                 (&J->cur.ir[(ref)])
@@ -441,7 +442,7 @@ static TRef crec_ct_ct(jit_State *J, CType *d, CType *s, TRef dp, TRef sp,
    /* fallthrough */
  case CCX(I, F):
    if (dt == IRT_CDATA || st == IRT_CDATA) goto err_nyi;
-    sp = emitconv(sp, dsize < 4 ? IRT_INT : dt, st, IRCONV_TRUNC|IRCONV_ANY);
+    sp = emitconv(sp, dsize < 4 ? IRT_INT : dt, st, IRCONV_ANY);
    goto xstore;
  case CCX(I, P):
  case CCX(I, A):
@@ -521,7 +522,7 @@ static TRef crec_ct_ct(jit_State *J, CType *d, CType *s, TRef dp, TRef sp,
    if (st == IRT_CDATA) goto err_nyi;
    /* The signed conversion is cheaper. x64 really has 47 bit pointers. */
    sp = emitconv(sp, (LJ_64 && dsize == 8) ? IRT_I64 : IRT_U32,
-                  st, IRCONV_TRUNC|IRCONV_ANY);
+                  st, IRCONV_ANY);
    goto xstore;
  /* Destination is an array. */
@@ -862,21 +863,17 @@ again:
 }
 /* Record setting a finalizer. */
-static void crec_finalizer(jit_State *J, TRef trcd, cTValue *fin)
+static void crec_finalizer(jit_State *J, TRef trcd, TRef trfin, cTValue *fin)
 {
-  TRef trlo = lj_ir_call(J, IRCALL_lj_cdata_setfin, trcd);
+  if (tvisgcv(fin)) {
-  TRef trhi = emitir(IRT(IR_ADD, IRT_P32), trlo, lj_ir_kint(J, 4));
+    if (!trfin) trfin = lj_ir_kptr(J, gcval(fin));
-  if (LJ_BE) { TRef tmp = trlo; trlo = trhi; trhi = tmp; }
+  } else if (tvisnil(fin)) {
-  if (tvisfunc(fin)) {
+    trfin = lj_ir_kptr(J, NULL);
-    emitir(IRT(IR_XSTORE, IRT_P32), trlo, lj_ir_kfunc(J, funcV(fin)));
-    emitir(IRTI(IR_XSTORE), trhi, lj_ir_kint(J, LJ_TFUNC));
-  } else if (tviscdata(fin)) {
-    emitir(IRT(IR_XSTORE, IRT_P32), trlo,
-           lj_ir_kgc(J, obj2gco(cdataV(fin)), IRT_CDATA));
-    emitir(IRTI(IR_XSTORE), trhi, lj_ir_kint(J, LJ_TCDATA));
  } else {
    lj_trace_err(J, LJ_TRERR_BADTYPE);
  }
+  lj_ir_call(J, IRCALL_lj_cdata_setfin, trcd,
+             trfin, lj_ir_kint(J, (int32_t)itype(fin)));
  J->needsnap = 1;
 }
@@ -887,10 +884,8 @@ static void crec_alloc(jit_State *J, RecordFFData *rd, CTypeID id)
  CTSize sz;
  CTInfo info = lj_ctype_info(cts, id, &sz);
  CType *d = ctype_raw(cts, id);
-  TRef trid;
+  TRef trcd, trid = lj_ir_kint(J, id);
-  if (!sz || sz > 128 || (info & CTF_VLA) || ctype_align(info) > CT_MEMALIGN)
+  cTValue *fin;
-    lj_trace_err(J, LJ_TRERR_NYICONV);  /* NYI: large/special allocations. */
-  trid = lj_ir_kint(J, id);
  /* Use special instruction to box pointer or 32/64 bit integer. */
  if (ctype_isptr(info) || (ctype_isinteger(info) && (sz == 4 || sz == 8))) {
    TRef sp = J->base[1] ? crec_ct_tv(J, d, 0, J->base[1], &rd->argv[1]) :
@@ -898,11 +893,36 @@ static void crec_alloc(jit_State *J, RecordFFData *rd, CTypeID id)
              sz == 4 ? lj_ir_kint(J, 0) :
              (lj_needsplit(J), lj_ir_kint64(J, 0));
    J->base[0] = emitir(IRTG(IR_CNEWI, IRT_CDATA), trid, sp);
+    return;
  } else {
-    TRef trcd = emitir(IRTG(IR_CNEW, IRT_CDATA), trid, TREF_NIL);
+    TRef trsz = TREF_NIL;
-    cTValue *fin;
+    if ((info & CTF_VLA)) {  /* Calculate VLA/VLS size at runtime. */
-    J->base[0] = trcd;
+      CTSize sz0, sz1;
-    if (J->base[1] && !J->base[2] &&
+      if (!J->base[1] || J->base[2])
+        lj_trace_err(J, LJ_TRERR_NYICONV);  /* NYI: init VLA/VLS. */
+      trsz = crec_ct_tv(J, ctype_get(cts, CTID_INT32), 0,
+                        J->base[1], &rd->argv[1]);
+      sz0 = lj_ctype_vlsize(cts, d, 0);
+      sz1 = lj_ctype_vlsize(cts, d, 1);
+      trsz = emitir(IRTGI(IR_MULOV), trsz, lj_ir_kint(J, (int32_t)(sz1-sz0)));
+      trsz = emitir(IRTGI(IR_ADDOV), trsz, lj_ir_kint(J, (int32_t)sz0));
+      J->base[1] = 0;  /* Simplify logic below. */
+    } else if (ctype_align(info) > CT_MEMALIGN) {
+      trsz = lj_ir_kint(J, sz);
+    }
+    trcd = emitir(IRTG(IR_CNEW, IRT_CDATA), trid, trsz);
+    if (sz > 128 || (info & CTF_VLA)) {
+      TRef dp;
+      CTSize align;
+    special:  /* Only handle bulk zero-fill for large/VLA/VLS types. */
+      if (J->base[1])
+        lj_trace_err(J, LJ_TRERR_NYICONV);  /* NYI: init large/VLA/VLS types. */
+      dp = emitir(IRT(IR_ADD, IRT_PTR), trcd, lj_ir_kintp(J, sizeof(GCcdata)));
+      if (trsz == TREF_NIL) trsz = lj_ir_kint(J, sz);
+      align = ctype_align(info);
+      if (align < CT_MEMALIGN) align = CT_MEMALIGN;
+      crec_fill(J, dp, trsz, lj_ir_kint(J, 0), (1u << align));
+    } else if (J->base[1] && !J->base[2] &&
        !lj_cconv_multi_init(cts, d, &rd->argv[1])) {
      goto single_init;
    } else if (ctype_isarray(d->info)) {
@@ -913,8 +933,9 @@ static void crec_alloc(jit_State *J, RecordFFData *rd, CTypeID id)
      TValue *sval = &tv;
      MSize i;
      tv.u64 = 0;
-      if (!(ctype_isnum(dc->info) || ctype_isptr(dc->info)))
+      if (!(ctype_isnum(dc->info) || ctype_isptr(dc->info)) ||
-        lj_trace_err(J, LJ_TRERR_NYICONV);  /* NYI: init array of aggregates. */
+          esize * CREC_FILL_MAXUNROLL < sz)
+        goto special;
      for (i = 1, ofs = 0; ofs < sz; ofs += esize) {
        TRef dp = emitir(IRT(IR_ADD, IRT_PTR), trcd,
                         lj_ir_kintp(J, ofs + sizeof(GCcdata)));
@@ -971,11 +992,12 @@ static void crec_alloc(jit_State *J, RecordFFData *rd, CTypeID id)
        crec_ct_tv(J, d, dp, lj_ir_kint(J, 0), &tv);
      }
    }
-    /* Handle __gc metamethod. */
-    fin = lj_ctype_meta(cts, id, MM_gc);
-    if (fin)
-      crec_finalizer(J, trcd, fin);
  }
+  J->base[0] = trcd;
+  /* Handle __gc metamethod. */
+  fin = lj_ctype_meta(cts, id, MM_gc);
+  if (fin)
+    crec_finalizer(J, trcd, 0, fin);
 }
 /* Record argument conversions. */
@@ -1228,7 +1250,7 @@ static TRef crec_arith_int64(jit_State *J, TRef *sp, CType **s, MMS mm)
    for (i = 0; i < 2; i++) {
      IRType st = tref_type(sp[i]);
      if (st == IRT_NUM || st == IRT_FLOAT)
-        sp[i] = emitconv(sp[i], dt, st, IRCONV_TRUNC|IRCONV_ANY);
+        sp[i] = emitconv(sp[i], dt, st, IRCONV_ANY);
      else if (!(st == IRT_I64 || st == IRT_U64))
        sp[i] = emitconv(sp[i], dt, IRT_INT,
                         (s[i]->info & CTF_UNSIGNED) ? 0 : IRCONV_SEXT);
@@ -1296,15 +1318,14 @@ static TRef crec_arith_ptr(jit_State *J, TRef *sp, CType **s, MMS mm)
    CTypeID id;
 #if LJ_64
    if (t == IRT_NUM || t == IRT_FLOAT)
-      tr = emitconv(tr, IRT_INTP, t, IRCONV_TRUNC|IRCONV_ANY);
+      tr = emitconv(tr, IRT_INTP, t, IRCONV_ANY);
    else if (!(t == IRT_I64 || t == IRT_U64))
      tr = emitconv(tr, IRT_INTP, IRT_INT,
                    ((t - IRT_I8) & 1) ? 0 : IRCONV_SEXT);
 #else
    if (!tref_typerange(sp[1], IRT_I8, IRT_U32)) {
      tr = emitconv(tr, IRT_INTP, t,
-                    (t == IRT_NUM || t == IRT_FLOAT) ?
+                    (t == IRT_NUM || t == IRT_FLOAT) ? IRCONV_ANY : 0);
-                    IRCONV_TRUNC|IRCONV_ANY : 0);
    }
 #endif
    tr = emitir(IRT(IR_MUL, IRT_INTP), tr, lj_ir_kintp(J, sz));
@@ -1623,7 +1644,139 @@ void LJ_FASTCALL recff_ffi_xof(jit_State *J, RecordFFData *rd)
 void LJ_FASTCALL recff_ffi_gc(jit_State *J, RecordFFData *rd)
 {
  argv2cdata(J, J->base[0], &rd->argv[0]);
-  crec_finalizer(J, J->base[0], &rd->argv[1]);
+  if (!J->base[1])
+    lj_trace_err(J, LJ_TRERR_BADTYPE);
+  crec_finalizer(J, J->base[0], J->base[1], &rd->argv[1]);
+}
+/* -- 64 bit bit.* library functions -------------------------------------- */
+/* Determine bit operation type from argument type. */
+static CTypeID crec_bit64_type(CTState *cts, cTValue *tv)
+{
+  if (tviscdata(tv)) {
+    CType *ct = lj_ctype_rawref(cts, cdataV(tv)->ctypeid);
+    if (ctype_isenum(ct->info)) ct = ctype_child(cts, ct);
+    if ((ct->info & (CTMASK_NUM|CTF_BOOL|CTF_FP|CTF_UNSIGNED)) ==
+        CTINFO(CT_NUM, CTF_UNSIGNED) && ct->size == 8)
+      return CTID_UINT64;  /* Use uint64_t, since it has the highest rank. */
+    return CTID_INT64;  /* Otherwise use int64_t. */
+  }
+  return 0;  /* Use regular 32 bit ops. */
+}
+void LJ_FASTCALL recff_bit64_tobit(jit_State *J, RecordFFData *rd)
+{
+  CTState *cts = ctype_ctsG(J2G(J));
+  TRef tr = crec_ct_tv(J, ctype_get(cts, CTID_INT64), 0,
+                       J->base[0], &rd->argv[0]);
+  if (!tref_isinteger(tr))
+    tr = emitconv(tr, IRT_INT, tref_type(tr), 0);
+  J->base[0] = tr;
+}
+int LJ_FASTCALL recff_bit64_unary(jit_State *J, RecordFFData *rd)
+{
+  CTState *cts = ctype_ctsG(J2G(J));
+  CTypeID id = crec_bit64_type(cts, &rd->argv[0]);
+  if (id) {
+    TRef tr = crec_ct_tv(J, ctype_get(cts, id), 0, J->base[0], &rd->argv[0]);
+    tr = emitir(IRT(rd->data, id-CTID_INT64+IRT_I64), tr, 0);
+    J->base[0] = emitir(IRTG(IR_CNEWI, IRT_CDATA), lj_ir_kint(J, id), tr);
+    return 1;
+  }
+  return 0;
+}
+int LJ_FASTCALL recff_bit64_nary(jit_State *J, RecordFFData *rd)
+{
+  CTState *cts = ctype_ctsG(J2G(J));
+  CTypeID id = 0;
+  MSize i;
+  for (i = 0; J->base[i] != 0; i++) {
+    CTypeID aid = crec_bit64_type(cts, &rd->argv[i]);
+    if (id < aid) id = aid;  /* Determine highest type rank of all arguments. */
+  }
+  if (id) {
+    CType *ct = ctype_get(cts, id);
+    uint32_t ot = IRT(rd->data, id-CTID_INT64+IRT_I64);
+    TRef tr = crec_ct_tv(J, ct, 0, J->base[0], &rd->argv[0]);
+    for (i = 1; J->base[i] != 0; i++) {
+      TRef tr2 = crec_ct_tv(J, ct, 0, J->base[i], &rd->argv[i]);
+      tr = emitir(ot, tr, tr2);
+    }
+    J->base[0] = emitir(IRTG(IR_CNEWI, IRT_CDATA), lj_ir_kint(J, id), tr);
+    return 1;
+  }
+  return 0;
+}
+int LJ_FASTCALL recff_bit64_shift(jit_State *J, RecordFFData *rd)
+{
+  CTState *cts = ctype_ctsG(J2G(J));
+  CTypeID id;
+  TRef tsh = 0;
+  if (J->base[0] && tref_iscdata(J->base[1])) {
+    tsh = crec_ct_tv(J, ctype_get(cts, CTID_INT64), 0,
+                     J->base[1], &rd->argv[1]);
+    if (!tref_isinteger(tsh))
+      tsh = emitconv(tsh, IRT_INT, tref_type(tsh), 0);
+    J->base[1] = tsh;
+  }
+  id = crec_bit64_type(cts, &rd->argv[0]);
+  if (id) {
+    TRef tr = crec_ct_tv(J, ctype_get(cts, id), 0, J->base[0], &rd->argv[0]);
+    uint32_t op = rd->data;
+    if (!tsh) tsh = lj_opt_narrow_tobit(J, J->base[1]);
+    if (!(op < IR_BROL ? LJ_TARGET_MASKSHIFT : LJ_TARGET_MASKROT) &&
+        !tref_isk(tsh))
+      tsh = emitir(IRTI(IR_BAND), tsh, lj_ir_kint(J, 63));
+#ifdef LJ_TARGET_UNIFYROT
+      if (op == (LJ_TARGET_UNIFYROT == 1 ? IR_BROR : IR_BROL)) {
+        op = LJ_TARGET_UNIFYROT == 1 ? IR_BROL : IR_BROR;
+        tsh = emitir(IRTI(IR_NEG), tsh, tsh);
+      }
+#endif
+    tr = emitir(IRT(op, id-CTID_INT64+IRT_I64), tr, tsh);
+    J->base[0] = emitir(IRTG(IR_CNEWI, IRT_CDATA), lj_ir_kint(J, id), tr);
+    return 1;
+  }
+  return 0;
+}
+TRef recff_bit64_tohex(jit_State *J, RecordFFData *rd, TRef hdr)
+{
+  CTState *cts = ctype_ctsG(J2G(J));
+  CTypeID id = crec_bit64_type(cts, &rd->argv[0]);
+  TRef tr, trsf = J->base[1];
+  SFormat sf = (STRFMT_UINT|STRFMT_T_HEX);
+  int32_t n;
+  if (trsf) {
+    CTypeID id2 = 0;
+    n = (int32_t)lj_carith_check64(J->L, 2, &id2);
+    if (id2)
+      trsf = crec_ct_tv(J, ctype_get(cts, CTID_INT32), 0, trsf, &rd->argv[1]);
+    else
+      trsf = lj_opt_narrow_tobit(J, trsf);
+    emitir(IRTGI(IR_EQ), trsf, lj_ir_kint(J, n));  /* Specialize to n. */
+  } else {
+    n = id ? 16 : 8;
+  }
+  if (n < 0) { n = -n; sf |= STRFMT_F_UPPER; }
+  sf |= ((SFormat)((n+1)&255) << STRFMT_SH_PREC);
+  if (id) {
+    tr = crec_ct_tv(J, ctype_get(cts, id), 0, J->base[0], &rd->argv[0]);
+    if (n < 16)
+      tr = emitir(IRT(IR_BAND, IRT_U64), tr,
+                  lj_ir_kint64(J, ((uint64_t)1 << 4*n)-1));
+  } else {
+    tr = lj_opt_narrow_tobit(J, J->base[0]);
+    if (n < 8)
+      tr = emitir(IRTI(IR_BAND), tr, lj_ir_kint(J, (int32_t)((1u << 4*n)-1)));
+    tr = emitconv(tr, IRT_U64, IRT_INT, 0);  /* No sign-extension. */
+    lj_needsplit(J);
+  }
+  return lj_ir_call(J, IRCALL_lj_strfmt_putfxint, hdr, lj_ir_kint(J, sf), tr);
 }
 /* -- Miscellaneous library functions ------------------------------------- */
diff --git a/src/lj_crecord.h b/src/lj_crecord.h
index dea05f78..47c0a69d 100644
--- a/src/lj_crecord.h
+++ b/src/lj_crecord.h
@@ -25,6 +25,13 @@ LJ_FUNC void LJ_FASTCALL recff_ffi_istype(jit_State *J, RecordFFData *rd);
 LJ_FUNC void LJ_FASTCALL recff_ffi_abi(jit_State *J, RecordFFData *rd);
 LJ_FUNC void LJ_FASTCALL recff_ffi_xof(jit_State *J, RecordFFData *rd);
 LJ_FUNC void LJ_FASTCALL recff_ffi_gc(jit_State *J, RecordFFData *rd);
+LJ_FUNC void LJ_FASTCALL recff_bit64_tobit(jit_State *J, RecordFFData *rd);
+LJ_FUNC int LJ_FASTCALL recff_bit64_unary(jit_State *J, RecordFFData *rd);
+LJ_FUNC int LJ_FASTCALL recff_bit64_nary(jit_State *J, RecordFFData *rd);
+LJ_FUNC int LJ_FASTCALL recff_bit64_shift(jit_State *J, RecordFFData *rd);
+LJ_FUNC TRef recff_bit64_tohex(jit_State *J, RecordFFData *rd, TRef hdr);
 LJ_FUNC void LJ_FASTCALL lj_crecord_tonumber(jit_State *J, RecordFFData *rd);
 #endif
diff --git a/src/lj_ctype.c b/src/lj_ctype.c
index 57a0d7cc..19a79c45 100644
--- a/src/lj_ctype.c
+++ b/src/lj_ctype.c
@@ -11,6 +11,7 @@
 #include "lj_err.h"
 #include "lj_str.h"
 #include "lj_tab.h"
+#include "lj_strfmt.h"
 #include "lj_ctype.h"
 #include "lj_ccallback.h"
@@ -568,19 +569,19 @@ GCstr *lj_ctype_repr_int64(lua_State *L, uint64_t n, int isunsigned)
 /* Convert complex to string with 'i' or 'I' suffix. */
 GCstr *lj_ctype_repr_complex(lua_State *L, void *sp, CTSize size)
 {
-  char buf[2*LJ_STR_NUMBUF+2+1];
+  char buf[2*STRFMT_MAXBUF_NUM+2+1], *p = buf;
  TValue re, im;
-  size_t len;
  if (size == 2*sizeof(double)) {
    re.n = *(double *)sp; im.n = ((double *)sp)[1];
  } else {
    re.n = (double)*(float *)sp; im.n = (double)((float *)sp)[1];
  }
-  len = lj_str_bufnum(buf, &re);
+  p = lj_strfmt_wnum(p, &re);
-  if (!(im.u32.hi & 0x80000000u) || im.n != im.n) buf[len++] = '+';
+  if (!(im.u32.hi & 0x80000000u) || im.n != im.n) *p++ = '+';
-  len += lj_str_bufnum(buf+len, &im);
+  p = lj_strfmt_wnum(p, &im);
-  buf[len] = buf[len-1] >= 'a' ? 'I' : 'i';
+  *p = *(p-1) >= 'a' ? 'I' : 'i';
-  return lj_str_new(L, buf, len+1);
+  p++;
+  return lj_str_new(L, buf, p-buf);
 }
 /* -- C type state -------------------------------------------------------- */
diff --git a/src/lj_debug.c b/src/lj_debug.c
index be7fb2b1..54f7db74 100644
--- a/src/lj_debug.c
+++ b/src/lj_debug.c
@@ -9,11 +9,12 @@
 #include "lj_obj.h"
 #include "lj_err.h"
 #include "lj_debug.h"
-#include "lj_str.h"
+#include "lj_buf.h"
 #include "lj_tab.h"
 #include "lj_state.h"
 #include "lj_frame.h"
 #include "lj_bc.h"
+#include "lj_strfmt.h"
 #if LJ_HASJIT
 #include "lj_jit.h"
 #endif
@@ -133,20 +134,6 @@ static BCLine debug_frameline(lua_State *L, GCfunc *fn, cTValue *nextframe)
 /* -- Variable names ------------------------------------------------------ */
-/* Read ULEB128 value. */
-static uint32_t debug_read_uleb128(const uint8_t **pp)
-{
-  const uint8_t *p = *pp;
-  uint32_t v = *p++;
-  if (LJ_UNLIKELY(v >= 0x80)) {
-    int sh = 0;
-    v &= 0x7f;
-    do { v |= ((*p & 0x7f) << (sh += 7)); } while (*p++ >= 0x80);
-  }
-  *pp = p;
-  return v;
-}
 /* Get name of a local variable from slot number and PC. */
 static const char *debug_varname(const GCproto *pt, BCPos pc, BCReg slot)
 {
@@ -162,9 +149,9 @@ static const char *debug_varname(const GCproto *pt, BCPos pc, BCReg slot)
      } else {
        while (*p++) ;  /* Skip over variable name string. */
      }
-      lastpc = startpc = lastpc + debug_read_uleb128(&p);
+      lastpc = startpc = lastpc + lj_buf_ruleb128((const char **)&p);
      if (startpc > pc) break;
-      endpc = startpc + debug_read_uleb128(&p);
+      endpc = startpc + lj_buf_ruleb128((const char **)&p);
      if (pc < endpc && slot-- == 0) {
        if (vn < VARNAME__MAX) {
 #define VARNAMESTR(name, str)   str "\0"
@@ -321,7 +308,7 @@ const char *lj_debug_funcname(lua_State *L, TValue *frame, const char **name)
 /* -- Source code locations ----------------------------------------------- */
 /* Generate shortened source name. */
-void lj_debug_shortname(char *out, GCstr *str)
+void lj_debug_shortname(char *out, GCstr *str, BCLine line)
 {
  const char *src = strdata(str);
  if (*src == '=') {
@@ -335,11 +322,11 @@ void lj_debug_shortname(char *out, GCstr *str)
      *out++ = '.'; *out++ = '.'; *out++ = '.';
    }
    strcpy(out, src);
-  } else {  /* Output [string "string"]. */
+  } else {  /* Output [string "string"] or [builtin:name]. */
    size_t len;  /* Length, up to first control char. */
    for (len = 0; len < LUA_IDSIZE-12; len++)
      if (((const unsigned char *)src)[len] < ' ') break;
-    strcpy(out, "[string \""); out += 9;
+    strcpy(out, line == ~(BCLine)0 ? "[builtin:" : "[string \""); out += 9;
    if (src[len] != '\0') {  /* Must truncate? */
      if (len > LUA_IDSIZE-15) len = LUA_IDSIZE-15;
      strncpy(out, src, len); out += len;
@@ -347,7 +334,7 @@ void lj_debug_shortname(char *out, GCstr *str)
    } else {
      strcpy(out, src); out += len;
    }
-    strcpy(out, "\"]");
+    strcpy(out, line == ~(BCLine)0 ? "]" : "\"]");
  }
 }
@@ -360,14 +347,15 @@ void lj_debug_addloc(lua_State *L, const char *msg,
    if (isluafunc(fn)) {
      BCLine line = debug_frameline(L, fn, nextframe);
      if (line >= 0) {
+        GCproto *pt = funcproto(fn);
        char buf[LUA_IDSIZE];
-        lj_debug_shortname(buf, proto_chunkname(funcproto(fn)));
+        lj_debug_shortname(buf, proto_chunkname(pt), pt->firstline);
-        lj_str_pushf(L, "%s:%d: %s", buf, line, msg);
+        lj_strfmt_pushf(L, "%s:%d: %s", buf, line, msg);
        return;
      }
    }
  }
-  lj_str_pushf(L, "%s", msg);
+  lj_strfmt_pushf(L, "%s", msg);
 }
 /* Push location string for a bytecode position to Lua stack. */
@@ -377,20 +365,22 @@ void lj_debug_pushloc(lua_State *L, GCproto *pt, BCPos pc)
  const char *s = strdata(name);
  MSize i, len = name->len;
  BCLine line = lj_debug_line(pt, pc);
-  if (*s == '@') {
+  if (pt->firstline == ~(BCLine)0) {
+    lj_strfmt_pushf(L, "builtin:%s", s);
+  } else if (*s == '@') {
    s++; len--;
    for (i = len; i > 0; i--)
      if (s[i] == '/' || s[i] == '\\') {
        s += i+1;
        break;
      }
-    lj_str_pushf(L, "%s:%d", s, line);
+    lj_strfmt_pushf(L, "%s:%d", s, line);
  } else if (len > 40) {
-    lj_str_pushf(L, "%p:%d", pt, line);
+    lj_strfmt_pushf(L, "%p:%d", pt, line);
  } else if (*s == '=') {
-    lj_str_pushf(L, "%s:%d", s+1, line);
+    lj_strfmt_pushf(L, "%s:%d", s+1, line);
  } else {
-    lj_str_pushf(L, "\"%s\":%d", s, line);
+    lj_strfmt_pushf(L, "\"%s\":%d", s, line);
  }
 }
@@ -453,7 +443,7 @@ int lj_debug_getinfo(lua_State *L, const char *what, lj_Debug *ar, int ext)
        BCLine firstline = pt->firstline;
        GCstr *name = proto_chunkname(pt);
        ar->source = strdata(name);
-        lj_debug_shortname(ar->short_src, name);
+        lj_debug_shortname(ar->short_src, name, pt->firstline);
        ar->linedefined = (int)firstline;
        ar->lastlinedefined = (int)(firstline + pt->numline);
        ar->what = firstline ? "Lua" : "main";
diff --git a/src/lj_debug.h b/src/lj_debug.h
index 7cf57de7..4144b47e 100644
--- a/src/lj_debug.h
+++ b/src/lj_debug.h
@@ -34,7 +34,7 @@ LJ_FUNC const char *lj_debug_slotname(GCproto *pt, const BCIns *pc,
                                      BCReg slot, const char **name);
 LJ_FUNC const char *lj_debug_funcname(lua_State *L, TValue *frame,
                                      const char **name);
-LJ_FUNC void lj_debug_shortname(char *out, GCstr *str);
+LJ_FUNC void lj_debug_shortname(char *out, GCstr *str, BCLine line);
 LJ_FUNC void lj_debug_addloc(lua_State *L, const char *msg,
                             cTValue *frame, cTValue *nextframe);
 LJ_FUNC void lj_debug_pushloc(lua_State *L, GCproto *pt, BCPos pc);
diff --git a/src/lj_dispatch.c b/src/lj_dispatch.c
index d57f1a6f..a78075a7 100644
--- a/src/lj_dispatch.c
+++ b/src/lj_dispatch.c
@@ -8,6 +8,7 @@
 #include "lj_obj.h"
 #include "lj_err.h"
+#include "lj_buf.h"
 #include "lj_func.h"
 #include "lj_str.h"
 #include "lj_tab.h"
@@ -17,6 +18,7 @@
 #include "lj_frame.h"
 #include "lj_bc.h"
 #include "lj_ff.h"
+#include "lj_strfmt.h"
 #if LJ_HASJIT
 #include "lj_jit.h"
 #endif
diff --git a/src/lj_dispatch.h b/src/lj_dispatch.h
index a56b6260..326297cd 100644
--- a/src/lj_dispatch.h
+++ b/src/lj_dispatch.h
@@ -33,11 +33,12 @@
  _(lj_ffh_coroutine_wrap_err) _(lj_func_closeuv) _(lj_func_newL_gc) \
  _(lj_gc_barrieruv) _(lj_gc_step) _(lj_gc_step_fixtop) _(lj_meta_arith) \
  _(lj_meta_call) _(lj_meta_cat) _(lj_meta_comp) _(lj_meta_equal) \
-  _(lj_meta_for) _(lj_meta_len) _(lj_meta_tget) _(lj_meta_tset) \
+  _(lj_meta_for) _(lj_meta_istype) _(lj_meta_len) _(lj_meta_tget) \
-  _(lj_state_growstack) _(lj_str_fromnum) _(lj_str_fromnumber) _(lj_str_new) \
+  _(lj_meta_tset) _(lj_state_growstack) _(lj_strfmt_num) \
-  _(lj_tab_dup) _(lj_tab_get) _(lj_tab_getinth) _(lj_tab_len) _(lj_tab_new) \
+  _(lj_str_new) _(lj_tab_dup) _(lj_tab_get) _(lj_tab_getinth) _(lj_tab_len) \
-  _(lj_tab_newkey) _(lj_tab_next) _(lj_tab_reasize) \
+  _(lj_tab_new) _(lj_tab_newkey) _(lj_tab_next) _(lj_tab_reasize) \
-  JITGOTDEF(_) FFIGOTDEF(_)
+  _(lj_tab_setinth) _(lj_buf_putstr_reverse) _(lj_buf_putstr_lower) \
+  _(lj_buf_putstr_upper) _(lj_buf_tostr) JITGOTDEF(_) FFIGOTDEF(_)
 enum {
 #define GOTENUM(name) LJ_GOT_##name,
@@ -60,7 +61,7 @@ typedef uint16_t HotCount;
 #define HOTCOUNT_CALL           1
 /* This solves a circular dependency problem -- bump as needed. Sigh. */
-#define GG_NUM_ASMFF    62
+#define GG_NUM_ASMFF    57
 #define GG_LEN_DDISP    (BC__MAX + GG_NUM_ASMFF)
 #define GG_LEN_SDISP    BC_FUNCF
diff --git a/src/lj_emit_arm.h b/src/lj_emit_arm.h
index b76a9a45..1a9a6852 100644
--- a/src/lj_emit_arm.h
+++ b/src/lj_emit_arm.h
@@ -308,30 +308,30 @@ static void emit_movrr(ASMState *as, IRIns *ir, Reg dst, Reg src)
  emit_dm(as, ARMI_MOV, dst, src);
 }
-/* Generic load of register from stack slot. */
+/* Generic load of register with base and (small) offset address. */
-static void emit_spload(ASMState *as, IRIns *ir, Reg r, int32_t ofs)
+static void emit_loadofs(ASMState *as, IRIns *ir, Reg r, Reg base, int32_t ofs)
 {
 #if LJ_SOFTFP
  lua_assert(!irt_isnum(ir->t)); UNUSED(ir);
 #else
  if (r >= RID_MAX_GPR)
-    emit_vlso(as, irt_isnum(ir->t) ? ARMI_VLDR_D : ARMI_VLDR_S, r, RID_SP, ofs);
+    emit_vlso(as, irt_isnum(ir->t) ? ARMI_VLDR_D : ARMI_VLDR_S, r, base, ofs);
  else
 #endif
-    emit_lso(as, ARMI_LDR, r, RID_SP, ofs);
+    emit_lso(as, ARMI_LDR, r, base, ofs);
 }
-/* Generic store of register to stack slot. */
+/* Generic store of register with base and (small) offset address. */
-static void emit_spstore(ASMState *as, IRIns *ir, Reg r, int32_t ofs)
+static void emit_storeofs(ASMState *as, IRIns *ir, Reg r, Reg base, int32_t ofs)
 {
 #if LJ_SOFTFP
  lua_assert(!irt_isnum(ir->t)); UNUSED(ir);
 #else
  if (r >= RID_MAX_GPR)
-    emit_vlso(as, irt_isnum(ir->t) ? ARMI_VSTR_D : ARMI_VSTR_S, r, RID_SP, ofs);
+    emit_vlso(as, irt_isnum(ir->t) ? ARMI_VSTR_D : ARMI_VSTR_S, r, base, ofs);
  else
 #endif
-    emit_lso(as, ARMI_STR, r, RID_SP, ofs);
+    emit_lso(as, ARMI_STR, r, base, ofs);
 }
 /* Emit an arithmetic/logic operation with a constant operand. */
diff --git a/src/lj_emit_mips.h b/src/lj_emit_mips.h
index 74821b8b..d6ea1d52 100644
--- a/src/lj_emit_mips.h
+++ b/src/lj_emit_mips.h
@@ -178,24 +178,24 @@ static void emit_movrr(ASMState *as, IRIns *ir, Reg dst, Reg src)
    emit_fg(as, irt_isnum(ir->t) ? MIPSI_MOV_D : MIPSI_MOV_S, dst, src);
 }
-/* Generic load of register from stack slot. */
+/* Generic load of register with base and (small) offset address. */
-static void emit_spload(ASMState *as, IRIns *ir, Reg r, int32_t ofs)
+static void emit_loadofs(ASMState *as, IRIns *ir, Reg r, Reg base, int32_t ofs)
 {
  if (r < RID_MAX_GPR)
-    emit_tsi(as, MIPSI_LW, r, RID_SP, ofs);
+    emit_tsi(as, MIPSI_LW, r, base, ofs);
  else
    emit_tsi(as, irt_isnum(ir->t) ? MIPSI_LDC1 : MIPSI_LWC1,
-             (r & 31), RID_SP, ofs);
+             (r & 31), base, ofs);
 }
-/* Generic store of register to stack slot. */
+/* Generic store of register with base and (small) offset address. */
-static void emit_spstore(ASMState *as, IRIns *ir, Reg r, int32_t ofs)
+static void emit_storeofs(ASMState *as, IRIns *ir, Reg r, Reg base, int32_t ofs)
 {
  if (r < RID_MAX_GPR)
-    emit_tsi(as, MIPSI_SW, r, RID_SP, ofs);
+    emit_tsi(as, MIPSI_SW, r, base, ofs);
  else
    emit_tsi(as, irt_isnum(ir->t) ? MIPSI_SDC1 : MIPSI_SWC1,
-             (r&31), RID_SP, ofs);
+             (r&31), base, ofs);
 }
 /* Add offset to pointer. */
diff --git a/src/lj_emit_ppc.h b/src/lj_emit_ppc.h
index a589f3a6..6938c446 100644
--- a/src/lj_emit_ppc.h
+++ b/src/lj_emit_ppc.h
@@ -186,22 +186,22 @@ static void emit_movrr(ASMState *as, IRIns *ir, Reg dst, Reg src)
    emit_fb(as, PPCI_FMR, dst, src);
 }
-/* Generic load of register from stack slot. */
+/* Generic load of register with base and (small) offset address. */
-static void emit_spload(ASMState *as, IRIns *ir, Reg r, int32_t ofs)
+static void emit_loadofs(ASMState *as, IRIns *ir, Reg r, Reg base, int32_t ofs)
 {
  if (r < RID_MAX_GPR)
-    emit_tai(as, PPCI_LWZ, r, RID_SP, ofs);
+    emit_tai(as, PPCI_LWZ, r, base, ofs);
  else
-    emit_fai(as, irt_isnum(ir->t) ? PPCI_LFD : PPCI_LFS, r, RID_SP, ofs);
+    emit_fai(as, irt_isnum(ir->t) ? PPCI_LFD : PPCI_LFS, r, base, ofs);
 }
-/* Generic store of register to stack slot. */
+/* Generic store of register with base and (small) offset address. */
-static void emit_spstore(ASMState *as, IRIns *ir, Reg r, int32_t ofs)
+static void emit_storeofs(ASMState *as, IRIns *ir, Reg r, Reg base, int32_t ofs)
 {
  if (r < RID_MAX_GPR)
-    emit_tai(as, PPCI_STW, r, RID_SP, ofs);
+    emit_tai(as, PPCI_STW, r, base, ofs);
  else
-    emit_fai(as, irt_isnum(ir->t) ? PPCI_STFD : PPCI_STFS, r, RID_SP, ofs);
+    emit_fai(as, irt_isnum(ir->t) ? PPCI_STFD : PPCI_STFS, r, base, ofs);
 }
 /* Emit a compare (for equality) with a constant operand. */
diff --git a/src/lj_emit_x86.h b/src/lj_emit_x86.h
index bd184a30..ca63f59c 100644
--- a/src/lj_emit_x86.h
+++ b/src/lj_emit_x86.h
@@ -241,10 +241,6 @@ static void emit_gmrmi(ASMState *as, x86Group xg, Reg rb, int32_t i)
 /* -- Emit loads/stores --------------------------------------------------- */
-/* Instruction selection for XMM moves. */
-#define XMM_MOVRR(as)   ((as->flags & JIT_F_SPLIT_XMM) ? XO_MOVSD : XO_MOVAPS)
-#define XMM_MOVRM(as)   ((as->flags & JIT_F_SPLIT_XMM) ? XO_MOVLPD : XO_MOVSD)
 /* mov [base+ofs], i */
 static void emit_movmroi(ASMState *as, Reg base, int32_t ofs, int32_t i)
 {
@@ -314,7 +310,7 @@ static void emit_loadn(ASMState *as, Reg r, cTValue *tv)
  if (tvispzero(tv))  /* Use xor only for +0. */
    emit_rr(as, XO_XORPS, r, r);
  else
-    emit_rma(as, XMM_MOVRM(as), r, &tv->n);
+    emit_rma(as, XO_MOVSD, r, &tv->n);
 }
 /* -- Emit control-flow instructions -------------------------------------- */
@@ -427,25 +423,25 @@ static void emit_movrr(ASMState *as, IRIns *ir, Reg dst, Reg src)
  if (dst < RID_MAX_GPR)
    emit_rr(as, XO_MOV, REX_64IR(ir, dst), src);
  else
-    emit_rr(as, XMM_MOVRR(as), dst, src);
+    emit_rr(as, XO_MOVAPS, dst, src);
 }
-/* Generic load of register from stack slot. */
+/* Generic load of register with base and (small) offset address. */
-static void emit_spload(ASMState *as, IRIns *ir, Reg r, int32_t ofs)
+static void emit_loadofs(ASMState *as, IRIns *ir, Reg r, Reg base, int32_t ofs)
 {
  if (r < RID_MAX_GPR)
-    emit_rmro(as, XO_MOV, REX_64IR(ir, r), RID_ESP, ofs);
+    emit_rmro(as, XO_MOV, REX_64IR(ir, r), base, ofs);
  else
-    emit_rmro(as, irt_isnum(ir->t) ? XMM_MOVRM(as) : XO_MOVSS, r, RID_ESP, ofs);
+    emit_rmro(as, irt_isnum(ir->t) ? XO_MOVSD : XO_MOVSS, r, base, ofs);
 }
-/* Generic store of register to stack slot. */
+/* Generic store of register with base and (small) offset address. */
-static void emit_spstore(ASMState *as, IRIns *ir, Reg r, int32_t ofs)
+static void emit_storeofs(ASMState *as, IRIns *ir, Reg r, Reg base, int32_t ofs)
 {
  if (r < RID_MAX_GPR)
-    emit_rmro(as, XO_MOVto, REX_64IR(ir, r), RID_ESP, ofs);
+    emit_rmro(as, XO_MOVto, REX_64IR(ir, r), base, ofs);
  else
-    emit_rmro(as, irt_isnum(ir->t) ? XO_MOVSDto : XO_MOVSSto, r, RID_ESP, ofs);
+    emit_rmro(as, irt_isnum(ir->t) ? XO_MOVSDto : XO_MOVSSto, r, base, ofs);
 }
 /* Add offset to pointer. */
diff --git a/src/lj_err.c b/src/lj_err.c
index 42cd12b4..7ae87a82 100644
--- a/src/lj_err.c
+++ b/src/lj_err.c
@@ -16,6 +16,7 @@
 #include "lj_ff.h"
 #include "lj_trace.h"
 #include "lj_vm.h"
+#include "lj_strfmt.h"
 /*
 ** LuaJIT can either use internal or external frame unwinding:
@@ -573,7 +574,7 @@ LJ_NORET LJ_NOINLINE static void err_msgv(lua_State *L, ErrMsg em, ...)
  va_list argp;
  va_start(argp, em);
  if (curr_funcisL(L)) L->top = curr_topL(L);
-  msg = lj_str_pushvf(L, err2msg(em), argp);
+  msg = lj_strfmt_pushvf(L, err2msg(em), argp);
  va_end(argp);
  lj_debug_addloc(L, msg, L->base-1, NULL);
  lj_err_run(L);
@@ -591,11 +592,11 @@ LJ_NOINLINE void lj_err_lex(lua_State *L, GCstr *src, const char *tok,
 {
  char buff[LUA_IDSIZE];
  const char *msg;
-  lj_debug_shortname(buff, src);
+  lj_debug_shortname(buff, src, line);
-  msg = lj_str_pushvf(L, err2msg(em), argp);
+  msg = lj_strfmt_pushvf(L, err2msg(em), argp);
-  msg = lj_str_pushf(L, "%s:%d: %s", buff, line, msg);
+  msg = lj_strfmt_pushf(L, "%s:%d: %s", buff, line, msg);
  if (tok)
-    lj_str_pushf(L, err2msg(LJ_ERR_XNEAR), msg, tok);
+    lj_strfmt_pushf(L, err2msg(LJ_ERR_XNEAR), msg, tok);
  lj_err_throw(L, LUA_ERRSYNTAX);
 }
@@ -679,7 +680,7 @@ LJ_NOINLINE void lj_err_callerv(lua_State *L, ErrMsg em, ...)
  const char *msg;
  va_list argp;
  va_start(argp, em);
-  msg = lj_str_pushvf(L, err2msg(em), argp);
+  msg = lj_strfmt_pushvf(L, err2msg(em), argp);
  va_end(argp);
  lj_err_callermsg(L, msg);
 }
@@ -699,9 +700,9 @@ LJ_NORET LJ_NOINLINE static void err_argmsg(lua_State *L, int narg,
  if (narg < 0 && narg > LUA_REGISTRYINDEX)
    narg = (int)(L->top - L->base) + narg + 1;
  if (ftype && ftype[3] == 'h' && --narg == 0)  /* Check for "method". */
-    msg = lj_str_pushf(L, err2msg(LJ_ERR_BADSELF), fname, msg);
+    msg = lj_strfmt_pushf(L, err2msg(LJ_ERR_BADSELF), fname, msg);
  else
-    msg = lj_str_pushf(L, err2msg(LJ_ERR_BADARG), narg, fname, msg);
+    msg = lj_strfmt_pushf(L, err2msg(LJ_ERR_BADARG), narg, fname, msg);
  lj_err_callermsg(L, msg);
 }
@@ -711,7 +712,7 @@ LJ_NOINLINE void lj_err_argv(lua_State *L, int narg, ErrMsg em, ...)
  const char *msg;
  va_list argp;
  va_start(argp, em);
-  msg = lj_str_pushvf(L, err2msg(em), argp);
+  msg = lj_strfmt_pushvf(L, err2msg(em), argp);
  va_end(argp);
  err_argmsg(L, narg, msg);
 }
@@ -727,7 +728,7 @@ LJ_NOINLINE void lj_err_argtype(lua_State *L, int narg, const char *xname)
 {
  TValue *o = narg < 0 ? L->top + narg : L->base + narg-1;
  const char *tname = o < L->top ? lj_typename(o) : lj_obj_typename[0];
-  const char *msg = lj_str_pushf(L, err2msg(LJ_ERR_BADTYPE), xname, tname);
+  const char *msg = lj_strfmt_pushf(L, err2msg(LJ_ERR_BADTYPE), xname, tname);
  err_argmsg(L, narg, msg);
 }
@@ -777,7 +778,7 @@ LUALIB_API int luaL_error(lua_State *L, const char *fmt, ...)
  const char *msg;
  va_list argp;
  va_start(argp, fmt);
-  msg = lj_str_pushvf(L, fmt, argp);
+  msg = lj_strfmt_pushvf(L, fmt, argp);
  va_end(argp);
  lj_err_callermsg(L, msg);
  return 0;  /* unreachable */
diff --git a/src/lj_errmsg.h b/src/lj_errmsg.h
index fd46acd4..e62dc237 100644
--- a/src/lj_errmsg.h
+++ b/src/lj_errmsg.h
@@ -96,9 +96,7 @@ ERRDEF(STRPATX,	"pattern too complex")
 ERRDEF(STRCAPI, "invalid capture index")
 ERRDEF(STRCAPN, "too many captures")
 ERRDEF(STRCAPU, "unfinished capture")
-ERRDEF(STRFMTO, "invalid option " LUA_QL("%%%c") " to " LUA_QL("format"))
+ERRDEF(STRFMT,  "invalid option " LUA_QS " to " LUA_QL("format"))
-ERRDEF(STRFMTR, "invalid format (repeated flags)")
-ERRDEF(STRFMTW, "invalid format (width or precision too long)")
 ERRDEF(STRGSRV, "invalid replacement value (a %s)")
 ERRDEF(BADMODN, "name conflict for module " LUA_QS)
 #if LJ_HASJIT
@@ -117,7 +115,6 @@ ERRDEF(JITOPT,	"unknown or malformed optimization flag " LUA_QS)
 /* Lexer/parser errors. */
 ERRDEF(XMODE,   "attempt to load chunk with wrong mode")
 ERRDEF(XNEAR,   "%s near " LUA_QS)
-ERRDEF(XELEM,   "lexical element too long")
 ERRDEF(XLINES,  "chunk has too many lines")
 ERRDEF(XLEVELS, "chunk has too many syntax levels")
 ERRDEF(XNUMBER, "malformed number")
diff --git a/src/lj_ffrecord.c b/src/lj_ffrecord.c
index 35e2e88e..fcc46319 100644
--- a/src/lj_ffrecord.c
+++ b/src/lj_ffrecord.c
@@ -27,6 +27,7 @@
 #include "lj_dispatch.h"
 #include "lj_vm.h"
 #include "lj_strscan.h"
+#include "lj_strfmt.h"
 /* Some local macros to save typing. Undef'd at the end. */
 #define IR(ref)                 (&J->cur.ir[(ref)])
@@ -79,10 +80,7 @@ static GCstr *argv2str(jit_State *J, TValue *o)
    GCstr *s;
    if (!tvisnumber(o))
      lj_trace_err(J, LJ_TRERR_BADTYPE);
-    if (tvisint(o))
+    s = lj_strfmt_number(J->L, o);
-      s = lj_str_fromint(J->L, intV(o));
-    else
-      s = lj_str_fromnum(J->L, &o->n);
    setstrV(J->L, o, s);
    return s;
  }
@@ -121,6 +119,13 @@ static void LJ_FASTCALL recff_c(jit_State *J, RecordFFData *rd)
  UNUSED(rd);
 }
+/* Emit BUFHDR for the global temporary buffer. */
+static TRef recff_bufhdr(jit_State *J)
+{
+  return emitir(IRT(IR_BUFHDR, IRT_P32),
+                lj_ir_kptr(J, &J2G(J)->tmpbuf), IRBUFHDR_RESET);
+}
 /* -- Base library fast functions ----------------------------------------- */
 static void LJ_FASTCALL recff_assert(jit_State *J, RecordFFData *rd)
@@ -336,11 +341,12 @@ static void LJ_FASTCALL recff_tostring(jit_State *J, RecordFFData *rd)
  if (tref_isstr(tr)) {
    /* Ignore __tostring in the string base metatable. */
    /* Pass on result in J->base[0]. */
-  } else if (!recff_metacall(J, rd, MM_tostring)) {
+  } else if (tr && !recff_metacall(J, rd, MM_tostring)) {
    if (tref_isnumber(tr)) {
-      J->base[0] = emitir(IRT(IR_TOSTR, IRT_STR), tr, 0);
+      J->base[0] = emitir(IRT(IR_TOSTR, IRT_STR), tr,
+                          tref_isnum(tr) ? IRTOSTR_NUM : IRTOSTR_INT);
    } else if (tref_ispri(tr)) {
-      J->base[0] = lj_ir_kstr(J, strV(&J->fn->c.upvalue[tref_type(tr)]));
+      J->base[0] = lj_ir_kstr(J, lj_strfmt_obj(J->L, &rd->argv[0]));
    } else {
      recff_nyiu(J);
    }
@@ -528,14 +534,6 @@ static void LJ_FASTCALL recff_math_modf(jit_State *J, RecordFFData *rd)
  rd->nres = 2;
 }
-static void LJ_FASTCALL recff_math_degrad(jit_State *J, RecordFFData *rd)
-{
-  TRef tr = lj_ir_tonum(J, J->base[0]);
-  TRef trm = lj_ir_knum(J, numV(&J->fn->c.upvalue[0]));
-  J->base[0] = emitir(IRTN(IR_MUL), tr, trm);
-  UNUSED(rd);
-}
 static void LJ_FASTCALL recff_math_pow(jit_State *J, RecordFFData *rd)
 {
  TRef tr = lj_ir_tonum(J, J->base[0]);
@@ -592,48 +590,105 @@ static void LJ_FASTCALL recff_math_random(jit_State *J, RecordFFData *rd)
 /* -- Bit library fast functions ------------------------------------------ */
-/* Record unary bit.tobit, bit.bnot, bit.bswap. */
+/* Record bit.tobit. */
+static void LJ_FASTCALL recff_bit_tobit(jit_State *J, RecordFFData *rd)
+{
+  TRef tr = J->base[0];
+#if LJ_HASFFI
+  if (tref_iscdata(tr)) { recff_bit64_tobit(J, rd); return; }
+#endif
+  J->base[0] = lj_opt_narrow_tobit(J, tr);
+  UNUSED(rd);
+}
+/* Record unary bit.bnot, bit.bswap. */
 static void LJ_FASTCALL recff_bit_unary(jit_State *J, RecordFFData *rd)
 {
-  TRef tr = lj_opt_narrow_tobit(J, J->base[0]);
+#if LJ_HASFFI
-  J->base[0] = (rd->data == IR_TOBIT) ? tr : emitir(IRTI(rd->data), tr, 0);
+  if (recff_bit64_unary(J, rd))
+    return;
+#endif
+  J->base[0] = emitir(IRTI(rd->data), lj_opt_narrow_tobit(J, J->base[0]), 0);
 }
 /* Record N-ary bit.band, bit.bor, bit.bxor. */
 static void LJ_FASTCALL recff_bit_nary(jit_State *J, RecordFFData *rd)
 {
-  TRef tr = lj_opt_narrow_tobit(J, J->base[0]);
+#if LJ_HASFFI
-  uint32_t op = rd->data;
+  if (recff_bit64_nary(J, rd))
-  BCReg i;
+    return;
-  for (i = 1; J->base[i] != 0; i++)
+#endif
-    tr = emitir(IRTI(op), tr, lj_opt_narrow_tobit(J, J->base[i]));
+  {
-  J->base[0] = tr;
+    TRef tr = lj_opt_narrow_tobit(J, J->base[0]);
+    uint32_t ot = IRTI(rd->data);
+    BCReg i;
+    for (i = 1; J->base[i] != 0; i++)
+      tr = emitir(ot, tr, lj_opt_narrow_tobit(J, J->base[i]));
+    J->base[0] = tr;
+  }
 }
 /* Record bit shifts. */
 static void LJ_FASTCALL recff_bit_shift(jit_State *J, RecordFFData *rd)
 {
-  TRef tr = lj_opt_narrow_tobit(J, J->base[0]);
+#if LJ_HASFFI
-  TRef tsh = lj_opt_narrow_tobit(J, J->base[1]);
+  if (recff_bit64_shift(J, rd))
-  IROp op = (IROp)rd->data;
+    return;
-  if (!(op < IR_BROL ? LJ_TARGET_MASKSHIFT : LJ_TARGET_MASKROT) &&
+#endif
-      !tref_isk(tsh))
+  {
-    tsh = emitir(IRTI(IR_BAND), tsh, lj_ir_kint(J, 31));
+    TRef tr = lj_opt_narrow_tobit(J, J->base[0]);
+    TRef tsh = lj_opt_narrow_tobit(J, J->base[1]);
+    IROp op = (IROp)rd->data;
+    if (!(op < IR_BROL ? LJ_TARGET_MASKSHIFT : LJ_TARGET_MASKROT) &&
+        !tref_isk(tsh))
+      tsh = emitir(IRTI(IR_BAND), tsh, lj_ir_kint(J, 31));
 #ifdef LJ_TARGET_UNIFYROT
-  if (op == (LJ_TARGET_UNIFYROT == 1 ? IR_BROR : IR_BROL)) {
+    if (op == (LJ_TARGET_UNIFYROT == 1 ? IR_BROR : IR_BROL)) {
-    op = LJ_TARGET_UNIFYROT == 1 ? IR_BROL : IR_BROR;
+      op = LJ_TARGET_UNIFYROT == 1 ? IR_BROL : IR_BROR;
-    tsh = emitir(IRTI(IR_NEG), tsh, tsh);
+      tsh = emitir(IRTI(IR_NEG), tsh, tsh);
+    }
+#endif
+    J->base[0] = emitir(IRTI(op), tr, tsh);
  }
+}
+static void LJ_FASTCALL recff_bit_tohex(jit_State *J, RecordFFData *rd)
+{
+#if LJ_HASFFI
+  TRef hdr = recff_bufhdr(J);
+  TRef tr = recff_bit64_tohex(J, rd, hdr);
+  J->base[0] = emitir(IRT(IR_BUFSTR, IRT_STR), tr, hdr);
+#else
+  recff_nyiu(J);  /* Don't bother working around this NYI. */
 #endif
-  J->base[0] = emitir(IRTI(op), tr, tsh);
 }
 /* -- String library fast functions --------------------------------------- */
-static void LJ_FASTCALL recff_string_len(jit_State *J, RecordFFData *rd)
+/* Specialize to relative starting position for string. */
+static TRef recff_string_start(jit_State *J, GCstr *s, int32_t *st, TRef tr,
+                               TRef trlen, TRef tr0)
 {
-  J->base[0] = emitir(IRTI(IR_FLOAD), lj_ir_tostr(J, J->base[0]), IRFL_STR_LEN);
+  int32_t start = *st;
-  UNUSED(rd);
+  if (start < 0) {
+    emitir(IRTGI(IR_LT), tr, tr0);
+    tr = emitir(IRTI(IR_ADD), trlen, tr);
+    start = start + (int32_t)s->len;
+    emitir(start < 0 ? IRTGI(IR_LT) : IRTGI(IR_GE), tr, tr0);
+    if (start < 0) {
+      tr = tr0;
+      start = 0;
+    }
+  } else if (start == 0) {
+    emitir(IRTGI(IR_EQ), tr, tr0);
+    tr = tr0;
+  } else {
+    tr = emitir(IRTI(IR_ADD), tr, lj_ir_kint(J, -1));
+    emitir(IRTGI(IR_GE), tr, tr0);
+    start--;
+  }
+  *st = start;
+  return tr;
 }
 /* Handle string.byte (rd->data = 0) and string.sub (rd->data = 1). */
@@ -681,29 +736,11 @@ static void LJ_FASTCALL recff_string_range(jit_State *J, RecordFFData *rd)
  } else if ((MSize)end <= str->len) {
    emitir(IRTGI(IR_ULE), trend, trlen);
  } else {
-    emitir(IRTGI(IR_GT), trend, trlen);
+    emitir(IRTGI(IR_UGT), trend, trlen);
    end = (int32_t)str->len;
    trend = trlen;
  }
-  if (start < 0) {
+  trstart = recff_string_start(J, str, &start, trstart, trlen, tr0);
-    emitir(IRTGI(IR_LT), trstart, tr0);
-    trstart = emitir(IRTI(IR_ADD), trlen, trstart);
-    start = start+(int32_t)str->len;
-    emitir(start < 0 ? IRTGI(IR_LT) : IRTGI(IR_GE), trstart, tr0);
-    if (start < 0) {
-      trstart = tr0;
-      start = 0;
-    }
-  } else {
-    if (start == 0) {
-      emitir(IRTGI(IR_EQ), trstart, tr0);
-      trstart = tr0;
-    } else {
-      trstart = emitir(IRTI(IR_ADD), trstart, lj_ir_kint(J, -1));
-      emitir(IRTGI(IR_GE), trstart, tr0);
-      start--;
-    }
-  }
  if (rd->data) {  /* Return string.sub result. */
    if (end - start >= 0) {
      /* Also handle empty range here, to avoid extra traces. */
@@ -713,7 +750,7 @@ static void LJ_FASTCALL recff_string_range(jit_State *J, RecordFFData *rd)
      J->base[0] = emitir(IRT(IR_SNEW, IRT_STR), trptr, trslen);
    } else {  /* Range underflow: return empty string. */
      emitir(IRTGI(IR_LT), trend, trstart);
-      J->base[0] = lj_ir_kstr(J, lj_str_new(J->L, strdata(str), 0));
+      J->base[0] = lj_ir_kstr(J, &J2G(J)->strempty);
    }
  } else {  /* Return string.byte result(s). */
    ptrdiff_t i, len = end - start;
@@ -735,48 +772,196 @@ static void LJ_FASTCALL recff_string_range(jit_State *J, RecordFFData *rd)
  }
 }
-/* -- Table library fast functions ---------------------------------------- */
+static void LJ_FASTCALL recff_string_char(jit_State *J, RecordFFData *rd)
-static void LJ_FASTCALL recff_table_getn(jit_State *J, RecordFFData *rd)
 {
-  if (tref_istab(J->base[0]))
+  TRef k255 = lj_ir_kint(J, 255);
-    J->base[0] = lj_ir_call(J, IRCALL_lj_tab_len, J->base[0]);
+  BCReg i;
-  /* else: Interpreter will throw. */
+  for (i = 0; J->base[i] != 0; i++) {  /* Convert char values to strings. */
+    TRef tr = lj_opt_narrow_toint(J, J->base[i]);
+    emitir(IRTGI(IR_ULE), tr, k255);
+    J->base[i] = emitir(IRT(IR_TOSTR, IRT_STR), tr, IRTOSTR_CHAR);
+  }
+  if (i > 1) {  /* Concatenate the strings, if there's more than one. */
+    TRef hdr = recff_bufhdr(J), tr = hdr;
+    for (i = 0; J->base[i] != 0; i++)
+      tr = emitir(IRT(IR_BUFPUT, IRT_P32), tr, J->base[i]);
+    J->base[0] = emitir(IRT(IR_BUFSTR, IRT_STR), tr, hdr);
+  }
  UNUSED(rd);
 }
-static void LJ_FASTCALL recff_table_remove(jit_State *J, RecordFFData *rd)
+static void LJ_FASTCALL recff_string_rep(jit_State *J, RecordFFData *rd)
 {
-  TRef tab = J->base[0];
+  TRef str = lj_ir_tostr(J, J->base[0]);
-  rd->nres = 0;
+  TRef rep = lj_opt_narrow_toint(J, J->base[1]);
-  if (tref_istab(tab)) {
+  TRef hdr, tr, str2 = 0;
-    if (tref_isnil(J->base[1])) {  /* Simple pop: t[#t] = nil */
+  if (!tref_isnil(J->base[2])) {
-      TRef trlen = lj_ir_call(J, IRCALL_lj_tab_len, tab);
+    TRef sep = lj_ir_tostr(J, J->base[2]);
-      GCtab *t = tabV(&rd->argv[0]);
+    int32_t vrep = argv2int(J, &rd->argv[1]);
-      MSize len = lj_tab_len(t);
+    emitir(IRTGI(vrep > 1 ? IR_GT : IR_LE), rep, lj_ir_kint(J, 1));
-      emitir(IRTGI(len ? IR_NE : IR_EQ), trlen, lj_ir_kint(J, 0));
+    if (vrep > 1) {
-      if (len) {
+      TRef hdr2 = recff_bufhdr(J);
-        RecordIndex ix;
+      TRef tr2 = emitir(IRT(IR_BUFPUT, IRT_P32), hdr2, sep);
-        ix.tab = tab;
+      tr2 = emitir(IRT(IR_BUFPUT, IRT_P32), tr2, str);
-        ix.key = trlen;
+      str2 = emitir(IRT(IR_BUFSTR, IRT_STR), tr2, hdr2);
-        settabV(J->L, &ix.tabv, t);
+    }
-        setintV(&ix.keyv, len);
+  }
-        ix.idxchain = 0;
+  tr = hdr = recff_bufhdr(J);
-        if (results_wanted(J) != 0) {  /* Specialize load only if needed. */
+  if (str2) {
-          ix.val = 0;
+    tr = emitir(IRT(IR_BUFPUT, IRT_P32), tr, str);
-          J->base[0] = lj_record_idx(J, &ix);  /* Load previous value. */
+    str = str2;
-          rd->nres = 1;
+    rep = emitir(IRTI(IR_ADD), rep, lj_ir_kint(J, -1));
-          /* Assumes ix.key/ix.tab is not modified for raw lj_record_idx(). */
+  }
-        }
+  tr = lj_ir_call(J, IRCALL_lj_buf_putstr_rep, tr, str, rep);
-        ix.val = TREF_NIL;
+  J->base[0] = emitir(IRT(IR_BUFSTR, IRT_STR), tr, hdr);
-        lj_record_idx(J, &ix);  /* Remove value. */
+}
+static void LJ_FASTCALL recff_string_op(jit_State *J, RecordFFData *rd)
+{
+  TRef str = lj_ir_tostr(J, J->base[0]);
+  TRef hdr = recff_bufhdr(J);
+  TRef tr = lj_ir_call(J, rd->data, hdr, str);
+  J->base[0] = emitir(IRT(IR_BUFSTR, IRT_STR), tr, hdr);
+}
+static void LJ_FASTCALL recff_string_find(jit_State *J, RecordFFData *rd)
+{
+  TRef trstr = lj_ir_tostr(J, J->base[0]);
+  TRef trpat = lj_ir_tostr(J, J->base[1]);
+  TRef trlen = emitir(IRTI(IR_FLOAD), trstr, IRFL_STR_LEN);
+  TRef tr0 = lj_ir_kint(J, 0);
+  TRef trstart;
+  GCstr *str = argv2str(J, &rd->argv[0]);
+  GCstr *pat = argv2str(J, &rd->argv[1]);
+  int32_t start;
+  J->needsnap = 1;
+  if (tref_isnil(J->base[2])) {
+    trstart = lj_ir_kint(J, 1);
+    start = 1;
+  } else {
+    trstart = lj_opt_narrow_toint(J, J->base[2]);
+    start = argv2int(J, &rd->argv[2]);
+  }
+  trstart = recff_string_start(J, str, &start, trstart, trlen, tr0);
+  if ((MSize)start <= str->len) {
+    emitir(IRTGI(IR_ULE), trstart, trlen);
+  } else {
+    emitir(IRTGI(IR_UGT), trstart, trlen);
+#if LJ_52
+    J->base[0] = TREF_NIL;
+    return;
+#else
+    trstart = trlen;
+    start = str->len;
+#endif
+  }
+  /* Fixed arg or no pattern matching chars? (Specialized to pattern string.) */
+  if ((J->base[2] && tref_istruecond(J->base[3])) ||
+      (emitir(IRTG(IR_EQ, IRT_STR), trpat, lj_ir_kstr(J, pat)),
+       !lj_str_haspattern(pat))) {  /* Search for fixed string. */
+    TRef trsptr = emitir(IRT(IR_STRREF, IRT_P32), trstr, trstart);
+    TRef trpptr = emitir(IRT(IR_STRREF, IRT_P32), trpat, tr0);
+    TRef trslen = emitir(IRTI(IR_SUB), trlen, trstart);
+    TRef trplen = emitir(IRTI(IR_FLOAD), trpat, IRFL_STR_LEN);
+    TRef tr = lj_ir_call(J, IRCALL_lj_str_find, trsptr, trpptr, trslen, trplen);
+    TRef trp0 = lj_ir_kkptr(J, NULL);
+    if (lj_str_find(strdata(str)+(MSize)start, strdata(pat),
+                    str->len-(MSize)start, pat->len)) {
+      TRef pos;
+      emitir(IRTG(IR_NE, IRT_P32), tr, trp0);
+      pos = emitir(IRTI(IR_SUB), tr, emitir(IRT(IR_STRREF, IRT_P32), trstr, tr0));
+      J->base[0] = emitir(IRTI(IR_ADD), pos, lj_ir_kint(J, 1));
+      J->base[1] = emitir(IRTI(IR_ADD), pos, trplen);
+      rd->nres = 2;
+    } else {
+      emitir(IRTG(IR_EQ, IRT_P32), tr, trp0);
+      J->base[0] = TREF_NIL;
+    }
+  } else {  /* Search for pattern. */
+    recff_nyiu(J);
+  }
+}
+static void LJ_FASTCALL recff_string_format(jit_State *J, RecordFFData *rd)
+{
+  TRef trfmt = lj_ir_tostr(J, J->base[0]);
+  GCstr *fmt = argv2str(J, &rd->argv[0]);
+  int arg = 1;
+  TRef hdr, tr;
+  FormatState fs;
+  SFormat sf;
+  /* Specialize to the format string. */
+  emitir(IRTG(IR_EQ, IRT_STR), trfmt, lj_ir_kstr(J, fmt));
+  tr = hdr = recff_bufhdr(J);
+  lj_strfmt_init(&fs, strdata(fmt), fmt->len);
+  while ((sf = lj_strfmt_parse(&fs)) != STRFMT_EOF) {  /* Parse format. */
+    TRef tra = sf == STRFMT_LIT ? 0 : J->base[arg++];
+    TRef trsf = lj_ir_kint(J, (int32_t)sf);
+    IRCallID id;
+    switch (STRFMT_TYPE(sf)) {
+    case STRFMT_LIT:
+      tr = emitir(IRT(IR_BUFPUT, IRT_P32), tr,
+                  lj_ir_kstr(J, lj_str_new(J->L, fs.str, fs.len)));
+      break;
+    case STRFMT_INT:
+      id = IRCALL_lj_strfmt_putfnum_int;
+    handle_int:
+      if (!tref_isinteger(tra))
+        goto handle_num;
+      if (sf == STRFMT_INT) { /* Shortcut for plain %d. */
+        tr = emitir(IRT(IR_BUFPUT, IRT_P32), tr,
+                    emitir(IRT(IR_TOSTR, IRT_STR), tra, IRTOSTR_INT));
+      } else {
+#if LJ_HASFFI
+        tra = emitir(IRT(IR_CONV, IRT_U64), tra,
+                     (IRT_INT|(IRT_U64<<5)|IRCONV_SEXT));
+        tr = lj_ir_call(J, IRCALL_lj_strfmt_putfxint, tr, trsf, tra);
+        lj_needsplit(J);
+#else
+        recff_nyiu(J);  /* Don't bother working around this NYI. */
+#endif
      }
-    } else {  /* Complex case: remove in the middle. */
+      break;
+    case STRFMT_UINT:
+      id = IRCALL_lj_strfmt_putfnum_uint;
+      goto handle_int;
+    case STRFMT_NUM:
+      id = IRCALL_lj_strfmt_putfnum;
+    handle_num:
+      tra = lj_ir_tonum(J, tra);
+      tr = lj_ir_call(J, id, tr, trsf, tra);
+      if (LJ_SOFTFP) lj_needsplit(J);
+      break;
+    case STRFMT_STR:
+      if (!tref_isstr(tra))
+        recff_nyiu(J);  /* NYI: __tostring and non-string types for %s. */
+      if (sf == STRFMT_STR)  /* Shortcut for plain %s. */
+        tr = emitir(IRT(IR_BUFPUT, IRT_P32), tr, tra);
+      else if ((sf & STRFMT_T_QUOTED))
+        tr = lj_ir_call(J, IRCALL_lj_strfmt_putquoted, tr, tra);
+      else
+        tr = lj_ir_call(J, IRCALL_lj_strfmt_putfstr, tr, trsf, tra);
+      break;
+    case STRFMT_CHAR:
+      tra = lj_opt_narrow_toint(J, tra);
+      if (sf == STRFMT_CHAR)  /* Shortcut for plain %c. */
+        tr = emitir(IRT(IR_BUFPUT, IRT_P32), tr,
+                    emitir(IRT(IR_TOSTR, IRT_STR), tra, IRTOSTR_CHAR));
+      else
+        tr = lj_ir_call(J, IRCALL_lj_strfmt_putfchar, tr, trsf, tra);
+      break;
+    case STRFMT_PTR:  /* NYI */
+    case STRFMT_ERR:
+    default:
      recff_nyiu(J);
+      break;
    }
-  }  /* else: Interpreter will throw. */
+  }
+  J->base[0] = emitir(IRT(IR_BUFSTR, IRT_STR), tr, hdr);
 }
+/* -- Table library fast functions ---------------------------------------- */
 static void LJ_FASTCALL recff_table_insert(jit_State *J, RecordFFData *rd)
 {
  RecordIndex ix;
@@ -798,6 +983,25 @@ static void LJ_FASTCALL recff_table_insert(jit_State *J, RecordFFData *rd)
  }  /* else: Interpreter will throw. */
 }
+static void LJ_FASTCALL recff_table_concat(jit_State *J, RecordFFData *rd)
+{
+  TRef tab = J->base[0];
+  if (tref_istab(tab)) {
+    TRef sep = !tref_isnil(J->base[1]) ?
+               lj_ir_tostr(J, J->base[1]) : lj_ir_knull(J, IRT_STR);
+    TRef tri = (J->base[1] && !tref_isnil(J->base[2])) ?
+               lj_opt_narrow_toint(J, J->base[2]) : lj_ir_kint(J, 1);
+    TRef tre = (J->base[1] && J->base[2] && !tref_isnil(J->base[3])) ?
+               lj_opt_narrow_toint(J, J->base[3]) :
+               lj_ir_call(J, IRCALL_lj_tab_len, tab);
+    TRef hdr = recff_bufhdr(J);
+    TRef tr = lj_ir_call(J, IRCALL_lj_buf_puttab, hdr, tab, sep, tri, tre);
+    emitir(IRTG(IR_NE, IRT_PTR), tr, lj_ir_kptr(J, NULL));
+    J->base[0] = emitir(IRT(IR_BUFSTR, IRT_STR), tr, hdr);
+  }  /* else: Interpreter will throw. */
+  UNUSED(rd);
+}
 /* -- I/O library fast functions ------------------------------------------ */
 /* Get FILE* for I/O function. Any I/O error aborts recording, so there's
@@ -833,7 +1037,10 @@ static void LJ_FASTCALL recff_io_write(jit_State *J, RecordFFData *rd)
    TRef buf = emitir(IRT(IR_STRREF, IRT_P32), str, zero);
    TRef len = emitir(IRTI(IR_FLOAD), str, IRFL_STR_LEN);
    if (tref_isk(len) && IR(tref_ref(len))->i == 1) {
-      TRef tr = emitir(IRT(IR_XLOAD, IRT_U8), buf, IRXLOAD_READONLY);
+      IRIns *irs = IR(tref_ref(str));
+      TRef tr = (irs->o == IR_TOSTR && irs->op2 == IRTOSTR_CHAR) ?
+                irs->op1 :
+                emitir(IRT(IR_XLOAD, IRT_U8), buf, IRXLOAD_READONLY);
      tr = lj_ir_call(J, IRCALL_fputc, tr, fp);
      if (results_wanted(J) != 0)  /* Check result only if not ignored. */
        emitir(IRTGI(IR_NE), tr, lj_ir_kint(J, -1));
diff --git a/src/lj_gc.c b/src/lj_gc.c
index 79f8b720..c9eaf21a 100644
--- a/src/lj_gc.c
+++ b/src/lj_gc.c
@@ -12,6 +12,7 @@
 #include "lj_obj.h"
 #include "lj_gc.h"
 #include "lj_err.h"
+#include "lj_buf.h"
 #include "lj_str.h"
 #include "lj_tab.h"
 #include "lj_func.h"
@@ -348,15 +349,6 @@ static size_t gc_propagate_gray(global_State *g)
 /* -- Sweep phase --------------------------------------------------------- */
-/* Try to shrink some common data structures. */
-static void gc_shrink(global_State *g, lua_State *L)
-{
-  if (g->strnum <= (g->strmask >> 2) && g->strmask > LJ_MIN_STRTAB*2-1)
-    lj_str_resize(L, g->strmask >> 1);  /* Shrink string table. */
-  if (g->tmpbuf.sz > LJ_MIN_SBUF*2)
-    lj_str_resizebuf(L, &g->tmpbuf, g->tmpbuf.sz >> 1);  /* Shrink temp buf. */
-}
 /* Type of GC free functions. */
 typedef void (LJ_FASTCALL *GCFreeFunc)(global_State *g, GCobj *o);
@@ -591,6 +583,8 @@ static void atomic(global_State *g, lua_State *L)
  /* All marking done, clear weak tables. */
  gc_clearweak(gcref(g->gc.weak));
+  lj_buf_shrink(L, &g->tmpbuf);  /* Shrink temp buffer. */
  /* Prepare for sweep phase. */
  g->gc.currentwhite = (uint8_t)otherwhite(g);  /* Flip current white. */
  g->strempty.marked = g->gc.currentwhite;
@@ -631,7 +625,8 @@ static size_t gc_onestep(lua_State *L)
    MSize old = g->gc.total;
    setmref(g->gc.sweep, gc_sweep(g, mref(g->gc.sweep, GCRef), GCSWEEPMAX));
    if (gcref(*mref(g->gc.sweep, GCRef)) == NULL) {
-      gc_shrink(g, L);
+      if (g->strnum <= (g->strmask >> 2) && g->strmask > LJ_MIN_STRTAB*2-1)
+        lj_str_resize(L, g->strmask >> 1);  /* Shrink string table. */
      if (gcref(g->gc.mmudata)) {  /* Need any finalizations? */
        g->gc.state = GCSfinalize;
      } else {  /* Otherwise skip this phase to help the JIT. */
diff --git a/src/lj_gdbjit.c b/src/lj_gdbjit.c
index 284195a1..4dbc9593 100644
--- a/src/lj_gdbjit.c
+++ b/src/lj_gdbjit.c
@@ -14,6 +14,8 @@
 #include "lj_err.h"
 #include "lj_debug.h"
 #include "lj_frame.h"
+#include "lj_buf.h"
+#include "lj_strfmt.h"
 #include "lj_jit.h"
 #include "lj_dispatch.h"
@@ -426,16 +428,6 @@ static void gdbjit_catnum(GDBJITctx *ctx, uint32_t n)
  *ctx->p++ = '0' + n;
 }
-/* Add a ULEB128 value. */
-static void gdbjit_uleb128(GDBJITctx *ctx, uint32_t v)
-{
-  uint8_t *p = ctx->p;
-  for (; v >= 0x80; v >>= 7)
-    *p++ = (uint8_t)((v & 0x7f) | 0x80);
-  *p++ = (uint8_t)v;
-  ctx->p = p;
-}
 /* Add a SLEB128 value. */
 static void gdbjit_sleb128(GDBJITctx *ctx, int32_t v)
 {
@@ -452,7 +444,7 @@ static void gdbjit_sleb128(GDBJITctx *ctx, int32_t v)
 #define DU16(x)         (*(uint16_t *)p = (x), p += 2)
 #define DU32(x)         (*(uint32_t *)p = (x), p += 4)
 #define DADDR(x)        (*(uintptr_t *)p = (x), p += sizeof(uintptr_t))
-#define DUV(x)          (ctx->p = p, gdbjit_uleb128(ctx, (x)), p = ctx->p)
+#define DUV(x)          (p = (uint8_t *)lj_strfmt_wuleb128((char *)p, (x)))
 #define DSV(x)          (ctx->p = p, gdbjit_sleb128(ctx, (x)), p = ctx->p)
 #define DSTR(str)       (ctx->p = p, gdbjit_strz(ctx, (str)), p = ctx->p)
 #define DALIGNNOP(s)    while ((uintptr_t)p & ((s)-1)) *p++ = DW_CFA_nop
diff --git a/src/lj_ir.c b/src/lj_ir.c
index e1a59105..fc2fc2ce 100644
--- a/src/lj_ir.c
+++ b/src/lj_ir.c
@@ -15,6 +15,7 @@
 #if LJ_HASJIT
 #include "lj_gc.h"
+#include "lj_buf.h"
 #include "lj_str.h"
 #include "lj_tab.h"
 #include "lj_ir.h"
@@ -29,6 +30,7 @@
 #endif
 #include "lj_vm.h"
 #include "lj_strscan.h"
+#include "lj_strfmt.h"
 #include "lj_lib.h"
 /* Some local macros to save typing. Undef'd at the end. */
@@ -443,7 +445,8 @@ TRef LJ_FASTCALL lj_ir_tostr(jit_State *J, TRef tr)
  if (!tref_isstr(tr)) {
    if (!tref_isnumber(tr))
      lj_trace_err(J, LJ_TRERR_BADTYPE);
-    tr = emitir(IRT(IR_TOSTR, IRT_STR), tr, 0);
+    tr = emitir(IRT(IR_TOSTR, IRT_STR), tr,
+                tref_isnum(tr) ? IRTOSTR_NUM : IRTOSTR_INT);
  }
  return tr;
 }
diff --git a/src/lj_ir.h b/src/lj_ir.h
index a9824325..841153d8 100644
--- a/src/lj_ir.h
+++ b/src/lj_ir.h
@@ -120,6 +120,11 @@
  _(CNEW,       AW, ref, ref) \
  _(CNEWI,      NW, ref, ref)  /* CSE is ok, not marked as A. */ \
  \
+  /* Buffer operations. */ \
+  _(BUFHDR,     L , ref, lit) \
+  _(BUFPUT,     L , ref, ref) \
+  _(BUFSTR,     A , ref, ref) \
+  \
  /* Barriers. */ \
  _(TBAR,       S , ref, ___) \
  _(OBAR,       S , ref, ref) \
@@ -128,7 +133,7 @@
  /* Type conversions. */ \
  _(CONV,       NW, ref, lit) \
  _(TOBIT,      N , ref, ref) \
-  _(TOSTR,      N , ref, ___) \
+  _(TOSTR,      N , ref, lit) \
  _(STRTO,      N , ref, ___) \
  \
  /* Calls. */ \
@@ -221,13 +226,16 @@ IRFLDEF(FLENUM)
 #define IRXLOAD_VOLATILE        2       /* Load from volatile data. */
 #define IRXLOAD_UNALIGNED       4       /* Unaligned load. */
+/* BUFHDR mode, stored in op2. */
+#define IRBUFHDR_RESET          0       /* Reset buffer. */
+#define IRBUFHDR_APPEND         1       /* Append to buffer. */
 /* CONV mode, stored in op2. */
 #define IRCONV_SRCMASK          0x001f  /* Source IRType. */
 #define IRCONV_DSTMASK          0x03e0  /* Dest. IRType (also in ir->t). */
 #define IRCONV_DSH              5
 #define IRCONV_NUM_INT          ((IRT_NUM<<IRCONV_DSH)|IRT_INT)
 #define IRCONV_INT_NUM          ((IRT_INT<<IRCONV_DSH)|IRT_NUM)
-#define IRCONV_TRUNC            0x0400  /* Truncate number to integer. */
 #define IRCONV_SEXT             0x0800  /* Sign-extend integer to integer. */
 #define IRCONV_MODEMASK         0x0fff
 #define IRCONV_CONVMASK         0xf000
@@ -238,6 +246,11 @@ IRFLDEF(FLENUM)
 #define IRCONV_INDEX  (2<<IRCONV_CSH)   /* Check + special backprop rules. */
 #define IRCONV_CHECK  (3<<IRCONV_CSH)   /* Number checked for integerness. */
+/* TOSTR mode, stored in op2. */
+#define IRTOSTR_INT             0       /* Convert integer to string. */
+#define IRTOSTR_NUM             1       /* Convert number to string. */
+#define IRTOSTR_CHAR            2       /* Convert char value to string. */
 /* -- IR operands --------------------------------------------------------- */
 /* IR operand mode (2 bit). */
diff --git a/src/lj_ircall.h b/src/lj_ircall.h
index 7fcc532e..9e1fb367 100644
--- a/src/lj_ircall.h
+++ b/src/lj_ircall.h
@@ -16,7 +16,7 @@ typedef struct CCallInfo {
  uint32_t flags;               /* Number of arguments and flags. */
 } CCallInfo;
-#define CCI_NARGS(ci)           ((ci)->flags & 0xff)    /* Extract # of args. */
+#define CCI_NARGS(ci)           ((ci)->flags & 0xff)    /* # of args. */
 #define CCI_NARGS_MAX           32                      /* Max. # of args. */
 #define CCI_OTSHIFT             16
@@ -45,6 +45,17 @@ typedef struct CCallInfo {
 #define CCI_CC_FASTCALL         0x2000  /* Fastcall calling convention. */
 #define CCI_CC_STDCALL          0x3000  /* Stdcall calling convention. */
+/* Extra args for SOFTFP, SPLIT 64 bit. */
+#define CCI_XARGS_SHIFT         14
+#define CCI_XARGS(ci)           (((ci)->flags >> CCI_XARGS_SHIFT) & 3)
+#define CCI_XA                  (1u << CCI_XARGS_SHIFT)
+#if LJ_SOFTFP || (LJ_32 && LJ_HASFFI)
+#define CCI_XNARGS(ci)          (CCI_NARGS((ci)) + CCI_XARGS((ci)))
+#else
+#define CCI_XNARGS(ci)          CCI_NARGS((ci))
+#endif
 /* Helpers for conditional function definitions. */
 #define IRCALLCOND_ANY(x)               x
@@ -87,24 +98,48 @@ typedef struct CCallInfo {
 #endif
 #if LJ_SOFTFP
-#define ARG1_FP         2       /* Treat as 2 32 bit arguments. */
+#define XA_FP           CCI_XA
+#define XA2_FP          (CCI_XA+CCI_XA)
 #else
-#define ARG1_FP         1
+#define XA_FP           0
+#define XA2_FP          0
 #endif
 #if LJ_32
-#define ARG2_64         4       /* Treat as 4 32 bit arguments. */
+#define XA_64           CCI_XA
+#define XA2_64          (CCI_XA+CCI_XA)
 #else
-#define ARG2_64         2
+#define XA_64           0
+#define XA2_64          0
 #endif
 /* Function definitions for CALL* instructions. */
 #define IRCALLDEF(_) \
  _(ANY,        lj_str_cmp,             2,  FN, INT, CCI_NOFPRCLOBBER) \
+  _(ANY,        lj_str_find,            4,   N, P32, 0) \
  _(ANY,        lj_str_new,             3,   S, STR, CCI_L) \
  _(ANY,        lj_strscan_num,         2,  FN, INT, 0) \
-  _(ANY,        lj_str_fromint,         2,  FN, STR, CCI_L) \
+  _(ANY,        lj_strfmt_int,          2,  FN, STR, CCI_L) \
-  _(ANY,        lj_str_fromnum,         2,  FN, STR, CCI_L) \
+  _(ANY,        lj_strfmt_num,          2,  FN, STR, CCI_L) \
+  _(ANY,        lj_strfmt_char,         2,  FN, STR, CCI_L) \
+  _(ANY,        lj_strfmt_putint,       2,  FL, P32, 0) \
+  _(ANY,        lj_strfmt_putnum,       2,  FL, P32, 0) \
+  _(ANY,        lj_strfmt_putquoted,    2,  FL, P32, 0) \
+  _(ANY,        lj_strfmt_putfxint,     3,   L, P32, XA_64) \
+  _(ANY,        lj_strfmt_putfnum_int,  3,   L, P32, XA_FP) \
+  _(ANY,        lj_strfmt_putfnum_uint, 3,   L, P32, XA_FP) \
+  _(ANY,        lj_strfmt_putfnum,      3,   L, P32, XA_FP) \
+  _(ANY,        lj_strfmt_putfstr,      3,   L, P32, 0) \
+  _(ANY,        lj_strfmt_putfchar,     3,   L, P32, 0) \
+  _(ANY,        lj_buf_putmem,          3,   S, P32, 0) \
+  _(ANY,        lj_buf_putstr,          2,  FL, P32, 0) \
+  _(ANY,        lj_buf_putchar,         2,  FL, P32, 0) \
+  _(ANY,        lj_buf_putstr_reverse,  2,  FL, P32, 0) \
+  _(ANY,        lj_buf_putstr_lower,    2,  FL, P32, 0) \
+  _(ANY,        lj_buf_putstr_upper,    2,  FL, P32, 0) \
+  _(ANY,        lj_buf_putstr_rep,      3,   L, P32, 0) \
+  _(ANY,        lj_buf_puttab,          5,   L, P32, 0) \
+  _(ANY,        lj_buf_tostr,           1,  FL, STR, 0) \
  _(ANY,        lj_tab_new1,            2,  FS, TAB, CCI_L) \
  _(ANY,        lj_tab_dup,             2,  FS, TAB, CCI_L) \
  _(ANY,        lj_tab_newkey,          3,   S, P32, CCI_L) \
@@ -114,29 +149,29 @@ typedef struct CCallInfo {
  _(ANY,        lj_mem_newgco,          2,  FS, P32, CCI_L) \
  _(ANY,        lj_math_random_step, 1, FS, NUM, CCI_CASTU64|CCI_NOFPRCLOBBER) \
  _(ANY,        lj_vm_modi,             2,  FN, INT, 0) \
-  _(ANY,        sinh,                   ARG1_FP,  N, NUM, 0) \
+  _(ANY,        sinh,                   1,   N, NUM, XA_FP) \
-  _(ANY,        cosh,                   ARG1_FP,  N, NUM, 0) \
+  _(ANY,        cosh,                   1,   N, NUM, XA_FP) \
-  _(ANY,        tanh,                   ARG1_FP,  N, NUM, 0) \
+  _(ANY,        tanh,                   1,   N, NUM, XA_FP) \
-  _(ANY,        fputc,                  2,  S, INT, 0) \
+  _(ANY,        fputc,                  2,   S, INT, 0) \
-  _(ANY,        fwrite,                 4,  S, INT, 0) \
+  _(ANY,        fwrite,                 4,   S, INT, 0) \
-  _(ANY,        fflush,                 1,  S, INT, 0) \
+  _(ANY,        fflush,                 1,   S, INT, 0) \
  /* ORDER FPM */ \
-  _(FPMATH,     lj_vm_floor,            ARG1_FP,   N, NUM, 0) \
+  _(FPMATH,     lj_vm_floor,            1,   N, NUM, XA_FP) \
-  _(FPMATH,     lj_vm_ceil,             ARG1_FP,   N, NUM, 0) \
+  _(FPMATH,     lj_vm_ceil,             1,   N, NUM, XA_FP) \
-  _(FPMATH,     lj_vm_trunc,            ARG1_FP,   N, NUM, 0) \
+  _(FPMATH,     lj_vm_trunc,            1,   N, NUM, XA_FP) \
-  _(FPMATH,     sqrt,                   ARG1_FP,   N, NUM, 0) \
+  _(FPMATH,     sqrt,                   1,   N, NUM, XA_FP) \
-  _(FPMATH,     exp,                    ARG1_FP,   N, NUM, 0) \
+  _(FPMATH,     exp,                    1,   N, NUM, XA_FP) \
-  _(FPMATH,     lj_vm_exp2,             ARG1_FP,   N, NUM, 0) \
+  _(FPMATH,     lj_vm_exp2,             1,   N, NUM, XA_FP) \
-  _(FPMATH,     log,                    ARG1_FP,   N, NUM, 0) \
+  _(FPMATH,     log,                    1,   N, NUM, XA_FP) \
-  _(FPMATH,     lj_vm_log2,             ARG1_FP,   N, NUM, 0) \
+  _(FPMATH,     lj_vm_log2,             1,   N, NUM, XA_FP) \
-  _(FPMATH,     log10,                  ARG1_FP,   N, NUM, 0) \
+  _(FPMATH,     log10,                  1,   N, NUM, XA_FP) \
-  _(FPMATH,     sin,                    ARG1_FP,   N, NUM, 0) \
+  _(FPMATH,     sin,                    1,   N, NUM, XA_FP) \
-  _(FPMATH,     cos,                    ARG1_FP,   N, NUM, 0) \
+  _(FPMATH,     cos,                    1,   N, NUM, XA_FP) \
-  _(FPMATH,     tan,                    ARG1_FP,   N, NUM, 0) \
+  _(FPMATH,     tan,                    1,   N, NUM, XA_FP) \
-  _(FPMATH,     lj_vm_powi,             ARG1_FP+1, N, NUM, 0) \
+  _(FPMATH,     lj_vm_powi,             2,   N, NUM, XA_FP) \
-  _(FPMATH,     pow,                    ARG1_FP*2, N, NUM, 0) \
+  _(FPMATH,     pow,                    2,   N, NUM, XA2_FP) \
-  _(FPMATH,     atan2,                  ARG1_FP*2, N, NUM, 0) \
+  _(FPMATH,     atan2,                  2,   N, NUM, XA2_FP) \
-  _(FPMATH,     ldexp,                  ARG1_FP+1, N, NUM, 0) \
+  _(FPMATH,     ldexp,                  2,   N, NUM, XA_FP) \
  _(SOFTFP,     lj_vm_tobit,            2,   N, INT, 0) \
  _(SOFTFP,     softfp_add,             4,   N, NUM, 0) \
  _(SOFTFP,     softfp_sub,             4,   N, NUM, 0) \
@@ -153,26 +188,32 @@ typedef struct CCallInfo {
  _(SOFTFP_FFI, softfp_ui2f,            1,   N, FLOAT, 0) \
  _(SOFTFP_FFI, softfp_f2i,             1,   N, INT, 0) \
  _(SOFTFP_FFI, softfp_f2ui,            1,   N, INT, 0) \
-  _(FP64_FFI,   fp64_l2d,               2,   N, NUM, 0) \
+  _(FP64_FFI,   fp64_l2d,               1,   N, NUM, XA_64) \
-  _(FP64_FFI,   fp64_ul2d,              2,   N, NUM, 0) \
+  _(FP64_FFI,   fp64_ul2d,              1,   N, NUM, XA_64) \
-  _(FP64_FFI,   fp64_l2f,               2,   N, FLOAT, 0) \
+  _(FP64_FFI,   fp64_l2f,               1,   N, FLOAT, XA_64) \
-  _(FP64_FFI,   fp64_ul2f,              2,   N, FLOAT, 0) \
+  _(FP64_FFI,   fp64_ul2f,              1,   N, FLOAT, XA_64) \
-  _(FP64_FFI,   fp64_d2l,               ARG1_FP,   N, I64, 0) \
+  _(FP64_FFI,   fp64_d2l,               1,   N, I64, XA_FP) \
-  _(FP64_FFI,   fp64_d2ul,              ARG1_FP,   N, U64, 0) \
+  _(FP64_FFI,   fp64_d2ul,              1,   N, U64, XA_FP) \
  _(FP64_FFI,   fp64_f2l,               1,   N, I64, 0) \
  _(FP64_FFI,   fp64_f2ul,              1,   N, U64, 0) \
-  _(FFI,        lj_carith_divi64,       ARG2_64,   N, I64, CCI_NOFPRCLOBBER) \
+  _(FFI,        lj_carith_divi64,       2,   N, I64, XA2_64|CCI_NOFPRCLOBBER) \
-  _(FFI,        lj_carith_divu64,       ARG2_64,   N, U64, CCI_NOFPRCLOBBER) \
+  _(FFI,        lj_carith_divu64,       2,   N, U64, XA2_64|CCI_NOFPRCLOBBER) \
-  _(FFI,        lj_carith_modi64,       ARG2_64,   N, I64, CCI_NOFPRCLOBBER) \
+  _(FFI,        lj_carith_modi64,       2,   N, I64, XA2_64|CCI_NOFPRCLOBBER) \
-  _(FFI,        lj_carith_modu64,       ARG2_64,   N, U64, CCI_NOFPRCLOBBER) \
+  _(FFI,        lj_carith_modu64,       2,   N, U64, XA2_64|CCI_NOFPRCLOBBER) \
-  _(FFI,        lj_carith_powi64,       ARG2_64,   N, I64, CCI_NOFPRCLOBBER) \
+  _(FFI,        lj_carith_powi64,       2,   N, I64, XA2_64|CCI_NOFPRCLOBBER) \
-  _(FFI,        lj_carith_powu64,       ARG2_64,   N, U64, CCI_NOFPRCLOBBER) \
+  _(FFI,        lj_carith_powu64,       2,   N, U64, XA2_64|CCI_NOFPRCLOBBER) \
-  _(FFI,        lj_cdata_setfin,        2,        FN, P32, CCI_L) \
+  _(FFI,        lj_cdata_newv,          4,   S, CDATA, CCI_L) \
-  _(FFI,        strlen,                 1,         L, INTP, 0) \
+  _(FFI,        lj_cdata_setfin,        4,   S, NIL, CCI_L) \
-  _(FFI,        memcpy,                 3,         S, PTR, 0) \
+  _(FFI,        strlen,                 1,   L, INTP, 0) \
-  _(FFI,        memset,                 3,         S, PTR, 0) \
+  _(FFI,        memcpy,                 3,   S, PTR, 0) \
-  _(FFI,        lj_vm_errno,            0,         S, INT, CCI_NOFPRCLOBBER) \
+  _(FFI,        memset,                 3,   S, PTR, 0) \
-  _(FFI32,      lj_carith_mul64,        ARG2_64,   N, I64, CCI_NOFPRCLOBBER)
+  _(FFI,        lj_vm_errno,            0,   S, INT, CCI_NOFPRCLOBBER) \
+  _(FFI32,      lj_carith_mul64,        2,   N, I64, XA2_64|CCI_NOFPRCLOBBER) \
+  _(FFI32,      lj_carith_shl64,        2,   N, U64, XA_64|CCI_NOFPRCLOBBER) \
+  _(FFI32,      lj_carith_shr64,        2,   N, U64, XA_64|CCI_NOFPRCLOBBER) \
+  _(FFI32,      lj_carith_sar64,        2,   N, U64, XA_64|CCI_NOFPRCLOBBER) \
+  _(FFI32,      lj_carith_rol64,        2,   N, U64, XA_64|CCI_NOFPRCLOBBER) \
+  _(FFI32,      lj_carith_ror64,        2,   N, U64, XA_64|CCI_NOFPRCLOBBER) \
  \
  /* End of list. */
diff --git a/src/lj_jit.h b/src/lj_jit.h
index c0b1c41e..2683b462 100644
--- a/src/lj_jit.h
+++ b/src/lj_jit.h
@@ -14,18 +14,15 @@
 /* CPU-specific JIT engine flags. */
 #if LJ_TARGET_X86ORX64
-#define JIT_F_CMOV              0x00000010
+#define JIT_F_SSE2              0x00000010
-#define JIT_F_SSE2              0x00000020
+#define JIT_F_SSE3              0x00000020
-#define JIT_F_SSE3              0x00000040
+#define JIT_F_SSE4_1            0x00000040
-#define JIT_F_SSE4_1            0x00000080
+#define JIT_F_PREFER_IMUL       0x00000080
-#define JIT_F_P4                0x00000100
+#define JIT_F_LEA_AGU           0x00000100
-#define JIT_F_PREFER_IMUL       0x00000200
-#define JIT_F_SPLIT_XMM         0x00000400
-#define JIT_F_LEA_AGU           0x00000800
 /* Names for the CPU-specific flags. Must match the order above. */
-#define JIT_F_CPU_FIRST         JIT_F_CMOV
+#define JIT_F_CPU_FIRST         JIT_F_SSE2
-#define JIT_F_CPUSTRING         "\4CMOV\4SSE2\4SSE3\6SSE4.1\2P4\3AMD\2K8\4ATOM"
+#define JIT_F_CPUSTRING         "\4SSE2\4SSE3\6SSE4.1\3AMD\4ATOM"
 #elif LJ_TARGET_ARM
 #define JIT_F_ARMV6_            0x00000010
 #define JIT_F_ARMV6T2_          0x00000020
diff --git a/src/lj_lex.c b/src/lj_lex.c
index 9f2b06f8..7c2c6677 100644
--- a/src/lj_lex.c
+++ b/src/lj_lex.c
@@ -12,6 +12,7 @@
 #include "lj_obj.h"
 #include "lj_gc.h"
 #include "lj_err.h"
+#include "lj_buf.h"
 #include "lj_str.h"
 #if LJ_HASFFI
 #include "lj_tab.h"
@@ -24,6 +25,7 @@
 #include "lj_parse.h"
 #include "lj_char.h"
 #include "lj_strscan.h"
+#include "lj_strfmt.h"
 /* Lua lexer token names. */
 static const char *const tokennames[] = {
@@ -37,50 +39,48 @@ TKDEF(TKSTR1, TKSTR2)
 /* -- Buffer handling ----------------------------------------------------- */
-#define char2int(c)             ((int)(uint8_t)(c))
+#define LEX_EOF                 (-1)
-#define next(ls) \
+#define lex_iseol(ls)           (ls->c == '\n' || ls->c == '\r')
-  (ls->current = (ls->n--) > 0 ? char2int(*ls->p++) : fillbuf(ls))
-#define save_and_next(ls)       (save(ls, ls->current), next(ls))
-#define currIsNewline(ls)       (ls->current == '\n' || ls->current == '\r')
-#define END_OF_STREAM           (-1)
-static int fillbuf(LexState *ls)
+/* Get more input from reader. */
+static LJ_NOINLINE LexChar lex_more(LexState *ls)
 {
  size_t sz;
-  const char *buf = ls->rfunc(ls->L, ls->rdata, &sz);
+  const char *p = ls->rfunc(ls->L, ls->rdata, &sz);
-  if (buf == NULL || sz == 0) return END_OF_STREAM;
+  if (p == NULL || sz == 0) return LEX_EOF;
-  ls->n = (MSize)sz - 1;
+  ls->pe = p + sz;
-  ls->p = buf;
+  ls->p = p + 1;
-  return char2int(*(ls->p++));
+  return (LexChar)(uint8_t)p[0];
 }
-static LJ_NOINLINE void save_grow(LexState *ls, int c)
+/* Get next character. */
+static LJ_AINLINE LexChar lex_next(LexState *ls)
 {
-  MSize newsize;
+  return (ls->c = ls->p < ls->pe ? (LexChar)(uint8_t)*ls->p++ : lex_more(ls));
-  if (ls->sb.sz >= LJ_MAX_STR/2)
-    lj_lex_error(ls, 0, LJ_ERR_XELEM);
-  newsize = ls->sb.sz * 2;
-  lj_str_resizebuf(ls->L, &ls->sb, newsize);
-  ls->sb.buf[ls->sb.n++] = (char)c;
 }
-static LJ_AINLINE void save(LexState *ls, int c)
+/* Save character. */
+static LJ_AINLINE void lex_save(LexState *ls, LexChar c)
 {
-  if (LJ_UNLIKELY(ls->sb.n + 1 > ls->sb.sz))
+  lj_buf_putb(&ls->sb, c);
-    save_grow(ls, c);
+}
-  else
-    ls->sb.buf[ls->sb.n++] = (char)c;
+/* Save previous character and get next character. */
+static LJ_AINLINE LexChar lex_savenext(LexState *ls)
+{
+  lex_save(ls, ls->c);
+  return lex_next(ls);
 }
-static void inclinenumber(LexState *ls)
+/* Skip line break. Handles "\n", "\r", "\r\n" or "\n\r". */
+static void lex_newline(LexState *ls)
 {
-  int old = ls->current;
+  LexChar old = ls->c;
-  lua_assert(currIsNewline(ls));
+  lua_assert(lex_iseol(ls));
-  next(ls);  /* skip `\n' or `\r' */
+  lex_next(ls);  /* Skip "\n" or "\r". */
-  if (currIsNewline(ls) && ls->current != old)
+  if (lex_iseol(ls) && ls->c != old) lex_next(ls);  /* Skip "\n\r" or "\r\n". */
-    next(ls);  /* skip `\n\r' or `\r\n' */
  if (++ls->linenumber >= LJ_MAX_LINE)
-    lj_lex_error(ls, ls->token, LJ_ERR_XLINES);
+    lj_lex_error(ls, ls->tok, LJ_ERR_XLINES);
 }
 /* -- Scanner for terminals ----------------------------------------------- */
@@ -89,19 +89,17 @@ static void inclinenumber(LexState *ls)
 static void lex_number(LexState *ls, TValue *tv)
 {
  StrScanFmt fmt;
-  int c, xp = 'e';
+  LexChar c, xp = 'e';
-  lua_assert(lj_char_isdigit(ls->current));
+  lua_assert(lj_char_isdigit(ls->c));
-  if ((c = ls->current) == '0') {
+  if ((c = ls->c) == '0' && (lex_savenext(ls) | 0x20) == 'x')
-    save_and_next(ls);
+    xp = 'p';
-    if ((ls->current | 0x20) == 'x') xp = 'p';
+  while (lj_char_isident(ls->c) || ls->c == '.' ||
-  }
+         ((ls->c == '-' || ls->c == '+') && (c | 0x20) == xp)) {
-  while (lj_char_isident(ls->current) || ls->current == '.' ||
+    c = ls->c;
-         ((ls->current == '-' || ls->current == '+') && (c | 0x20) == xp)) {
+    lex_savenext(ls);
-    c = ls->current;
-    save_and_next(ls);
  }
-  save(ls, '\0');
+  lex_save(ls, '\0');
-  fmt = lj_strscan_scan((const uint8_t *)ls->sb.buf, tv,
+  fmt = lj_strscan_scan((const uint8_t *)sbufB(&ls->sb), tv,
          (LJ_DUALNUM ? STRSCAN_OPT_TOINT : STRSCAN_OPT_TONUM) |
          (LJ_HASFFI ? (STRSCAN_OPT_LL|STRSCAN_OPT_IMAG) : 0));
  if (LJ_DUALNUM && fmt == STRSCAN_INT) {
@@ -134,60 +132,60 @@ static void lex_number(LexState *ls, TValue *tv)
  }
 }
-static int skip_sep(LexState *ls)
+/* Skip equal signs for "[=...=[" and "]=...=]" and return their count. */
+static int lex_skipeq(LexState *ls)
 {
  int count = 0;
-  int s = ls->current;
+  LexChar s = ls->c;
  lua_assert(s == '[' || s == ']');
-  save_and_next(ls);
+  while (lex_savenext(ls) == '=')
-  while (ls->current == '=') {
-    save_and_next(ls);
    count++;
-  }
+  return (ls->c == s) ? count : (-count) - 1;
-  return (ls->current == s) ? count : (-count) - 1;
 }
-static void read_long_string(LexState *ls, TValue *tv, int sep)
+/* Parse a long string or long comment (tv set to NULL). */
+static void lex_longstring(LexState *ls, TValue *tv, int sep)
 {
-  save_and_next(ls);  /* skip 2nd `[' */
+  lex_savenext(ls);  /* Skip second '['. */
-  if (currIsNewline(ls))  /* string starts with a newline? */
+  if (lex_iseol(ls))  /* Skip initial newline. */
-    inclinenumber(ls);  /* skip it */
+    lex_newline(ls);
  for (;;) {
-    switch (ls->current) {
+    switch (ls->c) {
-    case END_OF_STREAM:
+    case LEX_EOF:
      lj_lex_error(ls, TK_eof, tv ? LJ_ERR_XLSTR : LJ_ERR_XLCOM);
      break;
    case ']':
-      if (skip_sep(ls) == sep) {
+      if (lex_skipeq(ls) == sep) {
-        save_and_next(ls);  /* skip 2nd `]' */
+        lex_savenext(ls);  /* Skip second ']'. */
        goto endloop;
      }
      break;
    case '\n':
    case '\r':
-      save(ls, '\n');
+      lex_save(ls, '\n');
-      inclinenumber(ls);
+      lex_newline(ls);
-      if (!tv) lj_str_resetbuf(&ls->sb);  /* avoid wasting space */
+      if (!tv) lj_buf_reset(&ls->sb);  /* Don't waste space for comments. */
      break;
    default:
-      if (tv) save_and_next(ls);
+      lex_savenext(ls);
-      else next(ls);
      break;
    }
  } endloop:
  if (tv) {
-    GCstr *str = lj_parse_keepstr(ls, ls->sb.buf + (2 + (MSize)sep),
+    GCstr *str = lj_parse_keepstr(ls, sbufB(&ls->sb) + (2 + (MSize)sep),
-                                      ls->sb.n - 2*(2 + (MSize)sep));
+                                      sbuflen(&ls->sb) - 2*(2 + (MSize)sep));
    setstrV(ls->L, tv, str);
  }
 }
-static void read_string(LexState *ls, int delim, TValue *tv)
+/* Parse a string. */
+static void lex_string(LexState *ls, TValue *tv)
 {
-  save_and_next(ls);
+  LexChar delim = ls->c;  /* Delimiter is '\'' or '"'. */
-  while (ls->current != delim) {
+  lex_savenext(ls);
-    switch (ls->current) {
+  while (ls->c != delim) {
-    case END_OF_STREAM:
+    switch (ls->c) {
+    case LEX_EOF:
      lj_lex_error(ls, TK_eof, LJ_ERR_XSTR);
      continue;
    case '\n':
@@ -195,7 +193,7 @@ static void read_string(LexState *ls, int delim, TValue *tv)
      lj_lex_error(ls, TK_string, LJ_ERR_XSTR);
      continue;
    case '\\': {
-      int c = next(ls);  /* Skip the '\\'. */
+      LexChar c = lex_next(ls);  /* Skip the '\\'. */
      switch (c) {
      case 'a': c = '\a'; break;
      case 'b': c = '\b'; break;
@@ -205,111 +203,112 @@ static void read_string(LexState *ls, int delim, TValue *tv)
      case 't': c = '\t'; break;
      case 'v': c = '\v'; break;
      case 'x':  /* Hexadecimal escape '\xXX'. */
-        c = (next(ls) & 15u) << 4;
+        c = (lex_next(ls) & 15u) << 4;
-        if (!lj_char_isdigit(ls->current)) {
+        if (!lj_char_isdigit(ls->c)) {
-          if (!lj_char_isxdigit(ls->current)) goto err_xesc;
+          if (!lj_char_isxdigit(ls->c)) goto err_xesc;
          c += 9 << 4;
        }
-        c += (next(ls) & 15u);
+        c += (lex_next(ls) & 15u);
-        if (!lj_char_isdigit(ls->current)) {
+        if (!lj_char_isdigit(ls->c)) {
-          if (!lj_char_isxdigit(ls->current)) goto err_xesc;
+          if (!lj_char_isxdigit(ls->c)) goto err_xesc;
          c += 9;
        }
        break;
      case 'z':  /* Skip whitespace. */
-        next(ls);
+        lex_next(ls);
-        while (lj_char_isspace(ls->current))
+        while (lj_char_isspace(ls->c))
-          if (currIsNewline(ls)) inclinenumber(ls); else next(ls);
+          if (lex_iseol(ls)) lex_newline(ls); else lex_next(ls);
        continue;
-      case '\n': case '\r': save(ls, '\n'); inclinenumber(ls); continue;
+      case '\n': case '\r': lex_save(ls, '\n'); lex_newline(ls); continue;
      case '\\': case '\"': case '\'': break;
-      case END_OF_STREAM: continue;
+      case LEX_EOF: continue;
      default:
        if (!lj_char_isdigit(c))
          goto err_xesc;
        c -= '0';  /* Decimal escape '\ddd'. */
-        if (lj_char_isdigit(next(ls))) {
+        if (lj_char_isdigit(lex_next(ls))) {
-          c = c*10 + (ls->current - '0');
+          c = c*10 + (ls->c - '0');
-          if (lj_char_isdigit(next(ls))) {
+          if (lj_char_isdigit(lex_next(ls))) {
-            c = c*10 + (ls->current - '0');
+            c = c*10 + (ls->c - '0');
            if (c > 255) {
            err_xesc:
              lj_lex_error(ls, TK_string, LJ_ERR_XESC);
            }
-            next(ls);
+            lex_next(ls);
          }
        }
-        save(ls, c);
+        lex_save(ls, c);
        continue;
      }
-      save(ls, c);
+      lex_save(ls, c);
-      next(ls);
+      lex_next(ls);
      continue;
      }
    default:
-      save_and_next(ls);
+      lex_savenext(ls);
      break;
    }
  }
-  save_and_next(ls);  /* skip delimiter */
+  lex_savenext(ls);  /* Skip trailing delimiter. */
-  setstrV(ls->L, tv, lj_parse_keepstr(ls, ls->sb.buf + 1, ls->sb.n - 2));
+  setstrV(ls->L, tv,
+          lj_parse_keepstr(ls, sbufB(&ls->sb)+1, sbuflen(&ls->sb)-2));
 }
 /* -- Main lexical scanner ------------------------------------------------ */
-static int llex(LexState *ls, TValue *tv)
+/* Get next lexical token. */
+static LexToken lex_scan(LexState *ls, TValue *tv)
 {
-  lj_str_resetbuf(&ls->sb);
+  lj_buf_reset(&ls->sb);
  for (;;) {
-    if (lj_char_isident(ls->current)) {
+    if (lj_char_isident(ls->c)) {
      GCstr *s;
-      if (lj_char_isdigit(ls->current)) {  /* Numeric literal. */
+      if (lj_char_isdigit(ls->c)) {  /* Numeric literal. */
        lex_number(ls, tv);
        return TK_number;
      }
      /* Identifier or reserved word. */
      do {
-        save_and_next(ls);
+        lex_savenext(ls);
-      } while (lj_char_isident(ls->current));
+      } while (lj_char_isident(ls->c));
-      s = lj_parse_keepstr(ls, ls->sb.buf, ls->sb.n);
+      s = lj_parse_keepstr(ls, sbufB(&ls->sb), sbuflen(&ls->sb));
      setstrV(ls->L, tv, s);
      if (s->reserved > 0)  /* Reserved word? */
        return TK_OFS + s->reserved;
      return TK_name;
    }
-    switch (ls->current) {
+    switch (ls->c) {
    case '\n':
    case '\r':
-      inclinenumber(ls);
+      lex_newline(ls);
      continue;
    case ' ':
    case '\t':
    case '\v':
    case '\f':
-      next(ls);
+      lex_next(ls);
      continue;
    case '-':
-      next(ls);
+      lex_next(ls);
-      if (ls->current != '-') return '-';
+      if (ls->c != '-') return '-';
-      /* else is a comment */
+      lex_next(ls);
-      next(ls);
+      if (ls->c == '[') {  /* Long comment "--[=*[...]=*]". */
-      if (ls->current == '[') {
+        int sep = lex_skipeq(ls);
-        int sep = skip_sep(ls);
+        lj_buf_reset(&ls->sb);  /* `lex_skipeq' may dirty the buffer */
-        lj_str_resetbuf(&ls->sb);  /* `skip_sep' may dirty the buffer */
        if (sep >= 0) {
-          read_long_string(ls, NULL, sep);  /* long comment */
+          lex_longstring(ls, NULL, sep);
-          lj_str_resetbuf(&ls->sb);
+          lj_buf_reset(&ls->sb);
          continue;
        }
      }
-      /* else short comment */
+      /* Short comment "--.*\n". */
-      while (!currIsNewline(ls) && ls->current != END_OF_STREAM)
+      while (!lex_iseol(ls) && ls->c != LEX_EOF)
-        next(ls);
+        lex_next(ls);
      continue;
    case '[': {
-      int sep = skip_sep(ls);
+      int sep = lex_skipeq(ls);
      if (sep >= 0) {
-        read_long_string(ls, tv, sep);
+        lex_longstring(ls, tv, sep);
        return TK_string;
      } else if (sep == -1) {
        return '[';
@@ -319,44 +318,43 @@ static int llex(LexState *ls, TValue *tv)
      }
      }
    case '=':
-      next(ls);
+      lex_next(ls);
-      if (ls->current != '=') return '='; else { next(ls); return TK_eq; }
+      if (ls->c != '=') return '='; else { lex_next(ls); return TK_eq; }
    case '<':
-      next(ls);
+      lex_next(ls);
-      if (ls->current != '=') return '<'; else { next(ls); return TK_le; }
+      if (ls->c != '=') return '<'; else { lex_next(ls); return TK_le; }
    case '>':
-      next(ls);
+      lex_next(ls);
-      if (ls->current != '=') return '>'; else { next(ls); return TK_ge; }
+      if (ls->c != '=') return '>'; else { lex_next(ls); return TK_ge; }
    case '~':
-      next(ls);
+      lex_next(ls);
-      if (ls->current != '=') return '~'; else { next(ls); return TK_ne; }
+      if (ls->c != '=') return '~'; else { lex_next(ls); return TK_ne; }
    case ':':
-      next(ls);
+      lex_next(ls);
-      if (ls->current != ':') return ':'; else { next(ls); return TK_label; }
+      if (ls->c != ':') return ':'; else { lex_next(ls); return TK_label; }
    case '"':
    case '\'':
-      read_string(ls, ls->current, tv);
+      lex_string(ls, tv);
      return TK_string;
    case '.':
-      save_and_next(ls);
+      if (lex_savenext(ls) == '.') {
-      if (ls->current == '.') {
+        lex_next(ls);
-        next(ls);
+        if (ls->c == '.') {
-        if (ls->current == '.') {
+          lex_next(ls);
-          next(ls);
          return TK_dots;   /* ... */
        }
        return TK_concat;   /* .. */
-      } else if (!lj_char_isdigit(ls->current)) {
+      } else if (!lj_char_isdigit(ls->c)) {
        return '.';
      } else {
        lex_number(ls, tv);
        return TK_number;
      }
-    case END_OF_STREAM:
+    case LEX_EOF:
      return TK_eof;
    default: {
-      int c = ls->current;
+      LexChar c = ls->c;
-      next(ls);
+      lex_next(ls);
      return c;  /* Single-char tokens (+ - / ...). */
    }
    }
@@ -371,8 +369,7 @@ int lj_lex_setup(lua_State *L, LexState *ls)
  int header = 0;
  ls->L = L;
  ls->fs = NULL;
-  ls->n = 0;
+  ls->pe = ls->p = NULL;
-  ls->p = NULL;
  ls->vstack = NULL;
  ls->sizevstack = 0;
  ls->vtop = 0;
@@ -381,24 +378,22 @@ int lj_lex_setup(lua_State *L, LexState *ls)
  ls->lookahead = TK_eof;  /* No look-ahead token. */
  ls->linenumber = 1;
  ls->lastline = 1;
-  lj_str_resizebuf(ls->L, &ls->sb, LJ_MIN_SBUF);
+  lex_next(ls);  /* Read-ahead first char. */
-  next(ls);  /* Read-ahead first char. */
+  if (ls->c == 0xef && ls->p + 2 <= ls->pe && (uint8_t)ls->p[0] == 0xbb &&
-  if (ls->current == 0xef && ls->n >= 2 && char2int(ls->p[0]) == 0xbb &&
+      (uint8_t)ls->p[1] == 0xbf) {  /* Skip UTF-8 BOM (if buffered). */
-      char2int(ls->p[1]) == 0xbf) {  /* Skip UTF-8 BOM (if buffered). */
-    ls->n -= 2;
    ls->p += 2;
-    next(ls);
+    lex_next(ls);
    header = 1;
  }
-  if (ls->current == '#') {  /* Skip POSIX #! header line. */
+  if (ls->c == '#') {  /* Skip POSIX #! header line. */
    do {
-      next(ls);
+      lex_next(ls);
-      if (ls->current == END_OF_STREAM) return 0;
+      if (ls->c == LEX_EOF) return 0;
-    } while (!currIsNewline(ls));
+    } while (!lex_iseol(ls));
-    inclinenumber(ls);
+    lex_newline(ls);
    header = 1;
  }
-  if (ls->current == LUA_SIGNATURE[0]) {  /* Bytecode dump. */
+  if (ls->c == LUA_SIGNATURE[0]) {  /* Bytecode dump. */
    if (header) {
      /*
      ** Loading bytecode with an extra header is disabled for security
@@ -420,55 +415,60 @@ void lj_lex_cleanup(lua_State *L, LexState *ls)
  global_State *g = G(L);
  lj_mem_freevec(g, ls->bcstack, ls->sizebcstack, BCInsLine);
  lj_mem_freevec(g, ls->vstack, ls->sizevstack, VarInfo);
-  lj_str_freebuf(g, &ls->sb);
+  lj_buf_free(g, &ls->sb);
 }
+/* Return next lexical token. */
 void lj_lex_next(LexState *ls)
 {
  ls->lastline = ls->linenumber;
  if (LJ_LIKELY(ls->lookahead == TK_eof)) {  /* No lookahead token? */
-    ls->token = llex(ls, &ls->tokenval);  /* Get next token. */
+    ls->tok = lex_scan(ls, &ls->tokval);  /* Get next token. */
  } else {  /* Otherwise return lookahead token. */
-    ls->token = ls->lookahead;
+    ls->tok = ls->lookahead;
    ls->lookahead = TK_eof;
-    ls->tokenval = ls->lookaheadval;
+    ls->tokval = ls->lookaheadval;
  }
 }
+/* Look ahead for the next token. */
 LexToken lj_lex_lookahead(LexState *ls)
 {
  lua_assert(ls->lookahead == TK_eof);
-  ls->lookahead = llex(ls, &ls->lookaheadval);
+  ls->lookahead = lex_scan(ls, &ls->lookaheadval);
  return ls->lookahead;
 }
-const char *lj_lex_token2str(LexState *ls, LexToken token)
+/* Convert token to string. */
+const char *lj_lex_token2str(LexState *ls, LexToken tok)
 {
-  if (token > TK_OFS)
+  if (tok > TK_OFS)
-    return tokennames[token-TK_OFS-1];
+    return tokennames[tok-TK_OFS-1];
-  else if (!lj_char_iscntrl(token))
+  else if (!lj_char_iscntrl(tok))
-    return lj_str_pushf(ls->L, "%c", token);
+    return lj_strfmt_pushf(ls->L, "%c", tok);
  else
-    return lj_str_pushf(ls->L, "char(%d)", token);
+    return lj_strfmt_pushf(ls->L, "char(%d)", tok);
 }
-void lj_lex_error(LexState *ls, LexToken token, ErrMsg em, ...)
+/* Lexer error. */
+void lj_lex_error(LexState *ls, LexToken tok, ErrMsg em, ...)
 {
-  const char *tok;
+  const char *tokstr;
  va_list argp;
-  if (token == 0) {
+  if (tok == 0) {
-    tok = NULL;
+    tokstr = NULL;
-  } else if (token == TK_name || token == TK_string || token == TK_number) {
+  } else if (tok == TK_name || tok == TK_string || tok == TK_number) {
-    save(ls, '\0');
+    lex_save(ls, '\0');
-    tok = ls->sb.buf;
+    tokstr = sbufB(&ls->sb);
  } else {
-    tok = lj_lex_token2str(ls, token);
+    tokstr = lj_lex_token2str(ls, tok);
  }
  va_start(argp, em);
-  lj_err_lex(ls->L, ls->chunkname, tok, ls->linenumber, em, argp);
+  lj_err_lex(ls->L, ls->chunkname, tokstr, ls->linenumber, em, argp);
  va_end(argp);
 }
+/* Initialize strings for reserved words. */
 void lj_lex_init(lua_State *L)
 {
  uint32_t i;
diff --git a/src/lj_lex.h b/src/lj_lex.h
index 6e18e4b0..3e76e72a 100644
--- a/src/lj_lex.h
+++ b/src/lj_lex.h
@@ -30,7 +30,8 @@ TKDEF(TKENUM1, TKENUM2)
  TK_RESERVED = TK_while - TK_OFS
 };
-typedef int LexToken;
+typedef int LexChar;    /* Lexical character. Unsigned ext. from char. */
+typedef int LexToken;   /* Lexical token. */
 /* Combined bytecode ins/line. Only used during bytecode generation. */
 typedef struct BCInsLine {
@@ -51,13 +52,13 @@ typedef struct VarInfo {
 typedef struct LexState {
  struct FuncState *fs; /* Current FuncState. Defined in lj_parse.c. */
  struct lua_State *L;  /* Lua state. */
-  TValue tokenval;      /* Current token value. */
+  TValue tokval;        /* Current token value. */
  TValue lookaheadval;  /* Lookahead token value. */
-  int current;          /* Current character (charint). */
-  LexToken token;       /* Current token. */
-  LexToken lookahead;   /* Lookahead token. */
-  MSize n;              /* Bytes left in input buffer. */
  const char *p;        /* Current position in input buffer. */
+  const char *pe;       /* End of input buffer. */
+  LexChar c;            /* Current character. */
+  LexToken tok;         /* Current token. */
+  LexToken lookahead;   /* Lookahead token. */
  SBuf sb;              /* String buffer for tokens. */
  lua_Reader rfunc;     /* Reader callback. */
  void *rdata;          /* Reader callback data. */
@@ -78,8 +79,8 @@ LJ_FUNC int lj_lex_setup(lua_State *L, LexState *ls);
 LJ_FUNC void lj_lex_cleanup(lua_State *L, LexState *ls);
 LJ_FUNC void lj_lex_next(LexState *ls);
 LJ_FUNC LexToken lj_lex_lookahead(LexState *ls);
-LJ_FUNC const char *lj_lex_token2str(LexState *ls, LexToken token);
+LJ_FUNC const char *lj_lex_token2str(LexState *ls, LexToken tok);
-LJ_FUNC_NORET void lj_lex_error(LexState *ls, LexToken token, ErrMsg em, ...);
+LJ_FUNC_NORET void lj_lex_error(LexState *ls, LexToken tok, ErrMsg em, ...);
 LJ_FUNC void lj_lex_init(lua_State *L);
 #endif
diff --git a/src/lj_lib.c b/src/lj_lib.c
index 331eaa6a..bdbaba1d 100644
--- a/src/lj_lib.c
+++ b/src/lj_lib.c
@@ -18,6 +18,9 @@
 #include "lj_dispatch.h"
 #include "lj_vm.h"
 #include "lj_strscan.h"
+#include "lj_strfmt.h"
+#include "lj_lex.h"
+#include "lj_bcdump.h"
 #include "lj_lib.h"
 /* -- Library initialization ---------------------------------------------- */
@@ -43,6 +46,28 @@ static GCtab *lib_create_table(lua_State *L, const char *libname, int hsize)
  return tabV(L->top-1);
 }
+static const uint8_t *lib_read_lfunc(lua_State *L, const uint8_t *p, GCtab *tab)
+{
+  int len = *p++;
+  GCstr *name = lj_str_new(L, (const char *)p, len);
+  LexState ls;
+  GCproto *pt;
+  GCfunc *fn;
+  memset(&ls, 0, sizeof(ls));
+  ls.L = L;
+  ls.p = (const char *)(p+len);
+  ls.pe = (const char *)~(uintptr_t)0;
+  ls.c = -1;
+  ls.level = (BCDUMP_F_STRIP|(LJ_BE*BCDUMP_F_BE));
+  ls.chunkname = name;
+  pt = lj_bcread_proto(&ls);
+  pt->firstline = ~(BCLine)0;
+  fn = lj_func_newL_empty(L, pt, tabref(L->env));
+  /* NOBARRIER: See below for common barrier. */
+  setfuncV(L, lj_tab_setstr(L, tab, name), fn);
+  return (const uint8_t *)ls.p;
+}
 void lj_lib_register(lua_State *L, const char *libname,
                     const uint8_t *p, const lua_CFunction *cf)
 {
@@ -87,6 +112,9 @@ void lj_lib_register(lua_State *L, const char *libname,
      ofn = fn;
    } else {
      switch (tag | len) {
+      case LIBINIT_LUA:
+        p = lib_read_lfunc(L, p, tab);
+        break;
      case LIBINIT_SET:
        L->top -= 2;
        if (tvisstr(L->top+1) && strV(L->top+1)->len == 0)
@@ -137,7 +165,7 @@ GCstr *lj_lib_checkstr(lua_State *L, int narg)
    if (LJ_LIKELY(tvisstr(o))) {
      return strV(o);
    } else if (tvisnumber(o)) {
-      GCstr *s = lj_str_fromnumber(L, o);
+      GCstr *s = lj_strfmt_number(L, o);
      setstrV(L, o, s);
      return s;
    }
@@ -196,20 +224,6 @@ int32_t lj_lib_optint(lua_State *L, int narg, int32_t def)
  return (o < L->top && !tvisnil(o)) ? lj_lib_checkint(L, narg) : def;
 }
-int32_t lj_lib_checkbit(lua_State *L, int narg)
-{
-  TValue *o = L->base + narg-1;
-  if (!(o < L->top && lj_strscan_numberobj(o)))
-    lj_err_argt(L, narg, LUA_TNUMBER);
-  if (LJ_LIKELY(tvisint(o))) {
-    return intV(o);
-  } else {
-    int32_t i = lj_num2bit(numV(o));
-    if (LJ_DUALNUM) setintV(o, i);
-    return i;
-  }
-}
 GCfunc *lj_lib_checkfunc(lua_State *L, int narg)
 {
  TValue *o = L->base + narg-1;
diff --git a/src/lj_lib.h b/src/lj_lib.h
index 2fe6d2a8..2dd45adb 100644
--- a/src/lj_lib.h
+++ b/src/lj_lib.h
@@ -41,7 +41,6 @@ LJ_FUNC void lj_lib_checknumber(lua_State *L, int narg);
 LJ_FUNC lua_Number lj_lib_checknum(lua_State *L, int narg);
 LJ_FUNC int32_t lj_lib_checkint(lua_State *L, int narg);
 LJ_FUNC int32_t lj_lib_optint(lua_State *L, int narg, int32_t def);
-LJ_FUNC int32_t lj_lib_checkbit(lua_State *L, int narg);
 LJ_FUNC GCfunc *lj_lib_checkfunc(lua_State *L, int narg);
 LJ_FUNC GCtab *lj_lib_checktab(lua_State *L, int narg);
 LJ_FUNC GCtab *lj_lib_checktabornil(lua_State *L, int narg);
@@ -77,6 +76,7 @@ static LJ_AINLINE void lj_lib_pushcc(lua_State *L, lua_CFunction f,
 #define LJLIB_CF(name)          static int lj_cf_##name(lua_State *L)
 #define LJLIB_ASM(name)         static int lj_ffh_##name(lua_State *L)
 #define LJLIB_ASM_(name)
+#define LJLIB_LUA(name)
 #define LJLIB_SET(name)
 #define LJLIB_PUSH(arg)
 #define LJLIB_REC(handler)
@@ -96,7 +96,8 @@ LJ_FUNC void lj_lib_register(lua_State *L, const char *libname,
 #define LIBINIT_ASM     0x40
 #define LIBINIT_ASM_    0x80
 #define LIBINIT_STRING  0xc0
-#define LIBINIT_MAXSTR  0x39
+#define LIBINIT_MAXSTR  0x38
+#define LIBINIT_LUA     0xf9
 #define LIBINIT_SET     0xfa
 #define LIBINIT_NUMBER  0xfb
 #define LIBINIT_COPY    0xfc
diff --git a/src/lj_load.c b/src/lj_load.c
index 9d892678..37dde79f 100644
--- a/src/lj_load.c
+++ b/src/lj_load.c
@@ -15,7 +15,7 @@
 #include "lj_obj.h"
 #include "lj_gc.h"
 #include "lj_err.h"
-#include "lj_str.h"
+#include "lj_buf.h"
 #include "lj_func.h"
 #include "lj_frame.h"
 #include "lj_vm.h"
@@ -54,7 +54,7 @@ LUA_API int lua_loadx(lua_State *L, lua_Reader reader, void *data,
  ls.rdata = data;
  ls.chunkarg = chunkname ? chunkname : "?";
  ls.mode = mode;
-  lj_str_initbuf(&ls.sb);
+  lj_buf_init(L, &ls.sb);
  status = lj_vm_cpcall(L, NULL, &ls, cpparser);
  lj_lex_cleanup(L, &ls);
  lj_gc_check(L);
diff --git a/src/lj_meta.c b/src/lj_meta.c
index 441d571a..6cbfd927 100644
--- a/src/lj_meta.c
+++ b/src/lj_meta.c
@@ -12,6 +12,7 @@
 #include "lj_obj.h"
 #include "lj_gc.h"
 #include "lj_err.h"
+#include "lj_buf.h"
 #include "lj_str.h"
 #include "lj_tab.h"
 #include "lj_meta.h"
@@ -19,6 +20,8 @@
 #include "lj_bc.h"
 #include "lj_vm.h"
 #include "lj_strscan.h"
+#include "lj_strfmt.h"
+#include "lj_lib.h"
 /* -- Metamethod handling ------------------------------------------------- */
@@ -225,27 +228,14 @@ TValue *lj_meta_arith(lua_State *L, TValue *ra, cTValue *rb, cTValue *rc,
  }
 }
-/* In-place coercion of a number to a string. */
-static LJ_AINLINE int tostring(lua_State *L, TValue *o)
-{
-  if (tvisstr(o)) {
-    return 1;
-  } else if (tvisnumber(o)) {
-    setstrV(L, o, lj_str_fromnumber(L, o));
-    return 1;
-  } else {
-    return 0;
-  }
-}
 /* Helper for CAT. Coercion, iterative concat, __concat metamethod. */
 TValue *lj_meta_cat(lua_State *L, TValue *top, int left)
 {
  int fromc = 0;
  if (left < 0) { left = -left; fromc = 1; }
  do {
-    int n = 1;
+    if (!(tvisstr(top) || tvisnumber(top)) ||
-    if (!(tvisstr(top-1) || tvisnumber(top-1)) || !tostring(L, top)) {
+        !(tvisstr(top-1) || tvisnumber(top-1))) {
      cTValue *mo = lj_meta_lookup(L, top-1, MM_concat);
      if (tvisnil(mo)) {
        mo = lj_meta_lookup(L, top, MM_concat);
@@ -271,8 +261,6 @@ TValue *lj_meta_cat(lua_State *L, TValue *top, int left)
      copyTV(L, top, mo);
      setcont(top-1, lj_cont_cat);
      return top+1;  /* Trigger metamethod call. */
-    } else if (strV(top)->len == 0) {  /* Shortcut. */
-      (void)tostring(L, top-1);
    } else {
      /* Pick as many strings as possible from the top and concatenate them:
      **
@@ -281,27 +269,28 @@ TValue *lj_meta_cat(lua_State *L, TValue *top, int left)
      ** concat:    [...][CAT stack ...] [result]
      ** next step: [...][CAT stack ............]
      */
-      MSize tlen = strV(top)->len;
+      TValue *e, *o = top;
-      char *buffer;
+      uint64_t tlen = tvisstr(o) ? strV(o)->len : STRFMT_MAXBUF_NUM;
-      int i;
+      char *p, *buf;
-      for (n = 1; n <= left && tostring(L, top-n); n++) {
+      do {
-        MSize len = strV(top-n)->len;
+        o--; tlen += tvisstr(o) ? strV(o)->len : STRFMT_MAXBUF_NUM;
-        if (len >= LJ_MAX_STR - tlen)
+      } while (--left > 0 && (tvisstr(o-1) || tvisnumber(o-1)));
-          lj_err_msg(L, LJ_ERR_STROV);
+      if (tlen >= LJ_MAX_STR) lj_err_msg(L, LJ_ERR_STROV);
-        tlen += len;
+      p = buf = lj_buf_tmp(L, (MSize)tlen);
-      }
+      for (e = top, top = o; o <= e; o++) {
-      buffer = lj_str_needbuf(L, &G(L)->tmpbuf, tlen);
+        if (tvisstr(o)) {
-      n--;
+          GCstr *s = strV(o);
-      tlen = 0;
+          MSize len = s->len;
-      for (i = n; i >= 0; i--) {
+          p = lj_buf_wmem(p, strdata(s), len);
-        MSize len = strV(top-i)->len;
+        } else if (tvisint(o)) {
-        memcpy(buffer + tlen, strVdata(top-i), len);
+          p = lj_strfmt_wint(p, intV(o));
-        tlen += len;
+        } else {
+          lua_assert(tvisnum(o));
+          p = lj_strfmt_wnum(p, o);
+        }
      }
-      setstrV(L, top-n, lj_str_new(L, buffer, tlen));
+      setstrV(L, top, lj_str_new(L, buf, (size_t)(p-buf)));
    }
-    left -= n;
-    top -= n;
  } while (left >= 1);
  if (LJ_UNLIKELY(G(L)->gc.total >= G(L)->gc.threshold)) {
    if (!fromc) L->top = curr_topL(L);
@@ -423,6 +412,18 @@ TValue *lj_meta_comp(lua_State *L, cTValue *o1, cTValue *o2, int op)
  }
 }
+/* Helper for ISTYPE and ISNUM. Implicit coercion or error. */
+void lj_meta_istype(lua_State *L, BCReg ra, BCReg tp)
+{
+  L->top = curr_topL(L);
+  ra++; tp--;
+  lua_assert(LJ_DUALNUM || tp != ~LJ_TNUMX);  /* ISTYPE -> ISNUM broken. */
+  if (LJ_DUALNUM && tp == ~LJ_TNUMX) lj_lib_checkint(L, ra);
+  else if (tp == ~LJ_TNUMX+1) lj_lib_checknum(L, ra);
+  else if (tp == ~LJ_TSTR) lj_lib_checkstr(L, ra);
+  else lj_err_argtype(L, ra, lj_obj_itypename[tp]);
+}
 /* Helper for calls. __call metamethod. */
 void lj_meta_call(lua_State *L, TValue *func, TValue *top)
 {
diff --git a/src/lj_meta.h b/src/lj_meta.h
index 6af5e514..970398ec 100644
--- a/src/lj_meta.h
+++ b/src/lj_meta.h
@@ -31,6 +31,7 @@ LJ_FUNCA TValue * LJ_FASTCALL lj_meta_len(lua_State *L, cTValue *o);
 LJ_FUNCA TValue *lj_meta_equal(lua_State *L, GCobj *o1, GCobj *o2, int ne);
 LJ_FUNCA TValue * LJ_FASTCALL lj_meta_equal_cd(lua_State *L, BCIns ins);
 LJ_FUNCA TValue *lj_meta_comp(lua_State *L, cTValue *o1, cTValue *o2, int op);
+LJ_FUNCA void lj_meta_istype(lua_State *L, BCReg ra, BCReg tp);
 LJ_FUNCA void lj_meta_call(lua_State *L, TValue *func, TValue *top);
 LJ_FUNCA void LJ_FASTCALL lj_meta_for(lua_State *L, TValue *o);
diff --git a/src/lj_obj.c b/src/lj_obj.c
index 322b7bec..208e4955 100644
--- a/src/lj_obj.c
+++ b/src/lj_obj.c
@@ -20,7 +20,7 @@ LJ_DATADEF const char *const lj_obj_itypename[] = {  /* ORDER LJ_T */
 };
 /* Compare two objects without calling metamethods. */
-int lj_obj_equal(cTValue *o1, cTValue *o2)
+int LJ_FASTCALL lj_obj_equal(cTValue *o1, cTValue *o2)
 {
  if (itype(o1) == itype(o2)) {
    if (tvispri(o1))
@@ -33,3 +33,18 @@ int lj_obj_equal(cTValue *o1, cTValue *o2)
  return numberVnum(o1) == numberVnum(o2);
 }
+/* Return pointer to object or its object data. */
+const void * LJ_FASTCALL lj_obj_ptr(cTValue *o)
+{
+  if (tvisudata(o))
+    return uddata(udataV(o));
+  else if (tvislightud(o))
+    return lightudV(o);
+  else if (LJ_HASFFI && tviscdata(o))
+    return cdataptr(cdataV(o));
+  else if (tvisgcv(o))
+    return gcV(o);
+  else
+    return NULL;
+}
diff --git a/src/lj_obj.h b/src/lj_obj.h
index b967819d..5a05f38d 100644
--- a/src/lj_obj.h
+++ b/src/lj_obj.h
@@ -119,11 +119,12 @@ typedef int32_t BCLine;  /* Bytecode line number. */
 /* Internal assembler functions. Never call these directly from C. */
 typedef void (*ASMFunction)(void);
-/* Resizable string buffer. Need this here, details in lj_str.h. */
+/* Resizable string buffer. Need this here, details in lj_buf.h. */
 typedef struct SBuf {
-  char *buf;            /* String buffer base. */
+  MRef p;               /* String buffer pointer. */
-  MSize n;              /* String buffer length. */
+  MRef e;               /* String buffer end pointer. */
-  MSize sz;             /* String buffer size. */
+  MRef b;               /* String buffer base. */
+  MRef L;               /* lua_State, used for buffer resizing. */
 } SBuf;
 /* -- Tags and values ----------------------------------------------------- */
@@ -516,8 +517,8 @@ typedef struct global_State {
  lua_Alloc allocf;     /* Memory allocator. */
  void *allocd;         /* Memory allocator data. */
  GCState gc;           /* Garbage collector. */
-  SBuf tmpbuf;          /* Temporary buffer for string concatenation. */
+  volatile int32_t vmstate;  /* VM state or current JIT code trace number. */
-  Node nilnode;         /* Fallback 1-element hash part (nil key and value). */
+  SBuf tmpbuf;          /* Temporary string buffer. */
  GCstr strempty;       /* Empty string. */
  uint8_t stremptyz;    /* Zero terminator of empty string. */
  uint8_t hookmask;     /* Hook mask. */
@@ -526,13 +527,13 @@ typedef struct global_State {
  GCRef mainthref;      /* Link to main thread. */
  TValue registrytv;    /* Anchor for registry. */
  TValue tmptv, tmptv2; /* Temporary TValues. */
+  Node nilnode;         /* Fallback 1-element hash part (nil key and value). */
  GCupval uvhead;       /* Head of double-linked list of all open upvalues. */
  int32_t hookcount;    /* Instruction hook countdown. */
  int32_t hookcstart;   /* Start count for instruction hook counter. */
  lua_Hook hookf;       /* Hook function. */
  lua_CFunction wrapf;  /* Wrapper for C function calls. */
  lua_CFunction panic;  /* Called as a last resort for errors. */
-  volatile int32_t vmstate;  /* VM state or current JIT code trace number. */
  BCIns bc_cfunc_int;   /* Bytecode for internal C function calls. */
  BCIns bc_cfunc_ext;   /* Bytecode for external C function calls. */
  GCRef jit_L;          /* Current JIT code lua_State or NULL. */
@@ -810,11 +811,7 @@ static LJ_AINLINE int32_t lj_num2bit(lua_Number n)
 #endif
 }
-#if LJ_TARGET_X86 && !defined(__SSE2__)
-#define lj_num2int(n)   lj_num2bit((n))
-#else
 #define lj_num2int(n)   ((int32_t)(n))
-#endif
 static LJ_AINLINE uint64_t lj_num2u64(lua_Number n)
 {
@@ -851,6 +848,7 @@ LJ_DATA const char *const lj_obj_itypename[~LJ_TNUMX+1];
 #define lj_typename(o)  (lj_obj_itypename[itypemap(o)])
 /* Compare two objects without calling metamethods. */
-LJ_FUNC int lj_obj_equal(cTValue *o1, cTValue *o2);
+LJ_FUNC int LJ_FASTCALL lj_obj_equal(cTValue *o1, cTValue *o2);
+LJ_FUNC const void * LJ_FASTCALL lj_obj_ptr(cTValue *o);
 #endif
diff --git a/src/lj_opt_fold.c b/src/lj_opt_fold.c
index fe37b98a..1d37a7fd 100644
--- a/src/lj_opt_fold.c
+++ b/src/lj_opt_fold.c
@@ -14,18 +14,21 @@
 #if LJ_HASJIT
+#include "lj_buf.h"
 #include "lj_str.h"
 #include "lj_tab.h"
 #include "lj_ir.h"
 #include "lj_jit.h"
+#include "lj_ircall.h"
 #include "lj_iropt.h"
 #include "lj_trace.h"
 #if LJ_HASFFI
 #include "lj_ctype.h"
-#endif
 #include "lj_carith.h"
+#endif
 #include "lj_vm.h"
 #include "lj_strscan.h"
+#include "lj_strfmt.h"
 /* Here's a short description how the FOLD engine processes instructions:
 **
@@ -155,13 +158,14 @@ typedef IRRef (LJ_FASTCALL *FoldFunc)(jit_State *J);
 /* Barrier to prevent folding across a GC step.
 ** GC steps can only happen at the head of a trace and at LOOP.
-** And the GC is only driven forward if there is at least one allocation.
+** And the GC is only driven forward if there's at least one allocation.
 */
 #define gcstep_barrier(J, ref) \
  ((ref) < J->chain[IR_LOOP] && \
   (J->chain[IR_SNEW] || J->chain[IR_XSNEW] || \
    J->chain[IR_TNEW] || J->chain[IR_TDUP] || \
-    J->chain[IR_CNEW] || J->chain[IR_CNEWI] || J->chain[IR_TOSTR]))
+    J->chain[IR_CNEW] || J->chain[IR_CNEWI] || \
+    J->chain[IR_BUFSTR] || J->chain[IR_TOSTR]))
 /* -- Constant folding for FP numbers ------------------------------------- */
@@ -336,11 +340,9 @@ LJFOLDF(kfold_intcomp0)
 static uint64_t kfold_int64arith(uint64_t k1, uint64_t k2, IROp op)
 {
  switch (op) {
-#if LJ_64 || LJ_HASFFI
+#if LJ_HASFFI
  case IR_ADD: k1 += k2; break;
  case IR_SUB: k1 -= k2; break;
-#endif
-#if LJ_HASFFI
  case IR_MUL: k1 *= k2; break;
  case IR_BAND: k1 &= k2; break;
  case IR_BOR: k1 |= k2; break;
@@ -392,20 +394,10 @@ LJFOLD(BROL KINT64 KINT)
 LJFOLD(BROR KINT64 KINT)
 LJFOLDF(kfold_int64shift)
 {
-#if LJ_HASFFI || LJ_64
+#if LJ_HASFFI
  uint64_t k = ir_k64(fleft)->u64;
  int32_t sh = (fright->i & 63);
-  switch ((IROp)fins->o) {
+  return INT64FOLD(lj_carith_shift64(k, sh, fins->o - IR_BSHL));
-  case IR_BSHL: k <<= sh; break;
-#if LJ_HASFFI
-  case IR_BSHR: k >>= sh; break;
-  case IR_BSAR: k = (uint64_t)((int64_t)k >> sh); break;
-  case IR_BROL: k = lj_rol(k, sh); break;
-  case IR_BROR: k = lj_ror(k, sh); break;
-#endif
-  default: lua_assert(0); break;
-  }
-  return INT64FOLD(k);
 #else
  UNUSED(J); lua_assert(0); return FAILFOLD;
 #endif
@@ -527,6 +519,179 @@ LJFOLDF(kfold_strcmp)
  return NEXTFOLD;
 }
+/* -- Constant folding and forwarding for buffers ------------------------- */
+/*
+** Buffer ops perform stores, but their effect is limited to the buffer
+** itself. Also, buffer ops are chained: a use of an op implies a use of
+** all other ops up the chain. Conversely, if an op is unused, all ops
+** up the chain can go unsed. This largely eliminates the need to treat
+** them as stores.
+**
+** Alas, treating them as normal (IRM_N) ops doesn't work, because they
+** cannot be CSEd in isolation. CSE for IRM_N is implicitly done in LOOP
+** or if FOLD is disabled.
+**
+** The compromise is to declare them as loads, emit them like stores and
+** CSE whole chains manually when the BUFSTR is to be emitted. Any chain
+** fragments left over from CSE are eliminated by DCE.
+*/
+/* BUFHDR is emitted like a store, see below. */
+LJFOLD(BUFPUT BUFHDR BUFSTR)
+LJFOLDF(bufput_append)
+{
+  /* New buffer, no other buffer op inbetween and same buffer? */
+  if ((J->flags & JIT_F_OPT_FWD) &&
+      !(fleft->op2 & IRBUFHDR_APPEND) &&
+      fleft->prev == fright->op2 &&
+      fleft->op1 == IR(fright->op2)->op1) {
+    IRRef ref = fins->op1;
+    IR(ref)->op2 = (fleft->op2 | IRBUFHDR_APPEND);  /* Modify BUFHDR. */
+    IR(ref)->op1 = fright->op1;
+    return ref;
+  }
+  return EMITFOLD;  /* Always emit, CSE later. */
+}
+LJFOLD(BUFPUT any any)
+LJFOLDF(bufput_kgc)
+{
+  if (LJ_LIKELY(J->flags & JIT_F_OPT_FOLD) && fright->o == IR_KGC) {
+    GCstr *s2 = ir_kstr(fright);
+    if (s2->len == 0) {  /* Empty string? */
+      return LEFTFOLD;
+    } else {
+      if (fleft->o == IR_BUFPUT && irref_isk(fleft->op2) &&
+          !irt_isphi(fleft->t)) {  /* Join two constant string puts in a row. */
+        GCstr *s1 = ir_kstr(IR(fleft->op2));
+        IRRef kref = lj_ir_kstr(J, lj_buf_cat2str(J->L, s1, s2));
+        /* lj_ir_kstr() may realloc the IR and invalidates any IRIns *. */
+        IR(fins->op1)->op2 = kref;  /* Modify previous BUFPUT. */
+        return fins->op1;
+      }
+    }
+  }
+  return EMITFOLD;  /* Always emit, CSE later. */
+}
+LJFOLD(BUFSTR any any)
+LJFOLDF(bufstr_kfold_cse)
+{
+  lua_assert(fleft->o == IR_BUFHDR || fleft->o == IR_BUFPUT ||
+             fleft->o == IR_CALLL);
+  if (LJ_LIKELY(J->flags & JIT_F_OPT_FOLD)) {
+    if (fleft->o == IR_BUFHDR) {  /* No put operations? */
+      if (!(fleft->op2 & IRBUFHDR_APPEND))  /* Empty buffer? */
+        return lj_ir_kstr(J, &J2G(J)->strempty);
+      fins->op1 = fleft->prev;  /* Relies on checks in bufput_append. */
+      return CSEFOLD;
+    } else if (fleft->o == IR_BUFPUT) {
+      IRIns *irb = IR(fleft->op1);
+      if (irb->o == IR_BUFHDR && !(irb->op2 & IRBUFHDR_APPEND))
+        return fleft->op2;  /* Shortcut for a single put operation. */
+    }
+  }
+  /* Try to CSE the whole chain. */
+  if (LJ_LIKELY(J->flags & JIT_F_OPT_CSE)) {
+    IRRef ref = J->chain[IR_BUFSTR];
+    while (ref) {
+      IRIns *irs = IR(ref), *ira = fleft, *irb = IR(irs->op1);
+      while (ira->o == irb->o && ira->op2 == irb->op2) {
+        lua_assert(ira->o == IR_BUFHDR || ira->o == IR_BUFPUT ||
+                   ira->o == IR_CALLL || ira->o == IR_CARG);
+        if (ira->o == IR_BUFHDR && !(ira->op2 & IRBUFHDR_APPEND))
+          return ref;  /* CSE succeeded. */
+        if (ira->o == IR_CALLL && ira->op2 == IRCALL_lj_buf_puttab)
+          break;
+        ira = IR(ira->op1);
+        irb = IR(irb->op1);
+      }
+      ref = irs->prev;
+    }
+  }
+  return EMITFOLD;  /* No CSE possible. */
+}
+LJFOLD(CALLL CARG IRCALL_lj_buf_putstr_reverse)
+LJFOLD(CALLL CARG IRCALL_lj_buf_putstr_upper)
+LJFOLD(CALLL CARG IRCALL_lj_buf_putstr_lower)
+LJFOLD(CALLL CARG IRCALL_lj_strfmt_putquoted)
+LJFOLDF(bufput_kfold_op)
+{
+  if (irref_isk(fleft->op2)) {
+    const CCallInfo *ci = &lj_ir_callinfo[fins->op2];
+    SBuf *sb = lj_buf_tmp_(J->L);
+    sb = ((SBuf * (LJ_FASTCALL *)(SBuf *, GCstr *))ci->func)(sb,
+                                                       ir_kstr(IR(fleft->op2)));
+    fins->o = IR_BUFPUT;
+    fins->op1 = fleft->op1;
+    fins->op2 = lj_ir_kstr(J, lj_buf_tostr(sb));
+    return RETRYFOLD;
+  }
+  return EMITFOLD;  /* Always emit, CSE later. */
+}
+LJFOLD(CALLL CARG IRCALL_lj_buf_putstr_rep)
+LJFOLDF(bufput_kfold_rep)
+{
+  if (irref_isk(fleft->op2)) {
+    IRIns *irc = IR(fleft->op1);
+    if (irref_isk(irc->op2)) {
+      SBuf *sb = lj_buf_tmp_(J->L);
+      sb = lj_buf_putstr_rep(sb, ir_kstr(IR(irc->op2)), IR(fleft->op2)->i);
+      fins->o = IR_BUFPUT;
+      fins->op1 = irc->op1;
+      fins->op2 = lj_ir_kstr(J, lj_buf_tostr(sb));
+      return RETRYFOLD;
+    }
+  }
+  return EMITFOLD;  /* Always emit, CSE later. */
+}
+LJFOLD(CALLL CARG IRCALL_lj_strfmt_putfxint)
+LJFOLD(CALLL CARG IRCALL_lj_strfmt_putfnum_int)
+LJFOLD(CALLL CARG IRCALL_lj_strfmt_putfnum_uint)
+LJFOLD(CALLL CARG IRCALL_lj_strfmt_putfnum)
+LJFOLD(CALLL CARG IRCALL_lj_strfmt_putfstr)
+LJFOLD(CALLL CARG IRCALL_lj_strfmt_putfchar)
+LJFOLDF(bufput_kfold_fmt)
+{
+  IRIns *irc = IR(fleft->op1);
+  lua_assert(irref_isk(irc->op2));  /* SFormat must be const. */
+  if (irref_isk(fleft->op2)) {
+    SFormat sf = (SFormat)IR(irc->op2)->i;
+    IRIns *ira = IR(fleft->op2);
+    SBuf *sb = lj_buf_tmp_(J->L);
+    switch (fins->op2) {
+    case IRCALL_lj_strfmt_putfxint:
+      sb = lj_strfmt_putfxint(sb, sf, ir_k64(ira)->u64);
+      break;
+    case IRCALL_lj_strfmt_putfstr:
+      sb = lj_strfmt_putfstr(sb, sf, ir_kstr(ira));
+      break;
+    case IRCALL_lj_strfmt_putfchar:
+      sb = lj_strfmt_putfchar(sb, sf, ira->i);
+      break;
+    case IRCALL_lj_strfmt_putfnum_int:
+    case IRCALL_lj_strfmt_putfnum_uint:
+    case IRCALL_lj_strfmt_putfnum:
+    default: {
+      const CCallInfo *ci = &lj_ir_callinfo[fins->op2];
+      sb = ((SBuf * (*)(SBuf *, SFormat, lua_Number))ci->func)(sb, sf,
+                                                         ir_knum(ira)->n);
+      break;
+      }
+    }
+    fins->o = IR_BUFPUT;
+    fins->op1 = irc->op1;
+    fins->op2 = lj_ir_kstr(J, lj_buf_tostr(sb));
+    return RETRYFOLD;
+  }
+  return EMITFOLD;  /* Always emit, CSE later. */
+}
 /* -- Constant folding of pointer arithmetic ------------------------------ */
 LJFOLD(ADD KGC KINT)
@@ -647,27 +812,22 @@ LJFOLD(CONV KNUM IRCONV_INT_NUM)
 LJFOLDF(kfold_conv_knum_int_num)
 {
  lua_Number n = knumleft;
-  if (!(fins->op2 & IRCONV_TRUNC)) {
+  int32_t k = lj_num2int(n);
-    int32_t k = lj_num2int(n);
+  if (irt_isguard(fins->t) && n != (lua_Number)k) {
-    if (irt_isguard(fins->t) && n != (lua_Number)k) {
+    /* We're about to create a guard which always fails, like CONV +1.5.
-      /* We're about to create a guard which always fails, like CONV +1.5.
+    ** Some pathological loops cause this during LICM, e.g.:
-      ** Some pathological loops cause this during LICM, e.g.:
+    **   local x,k,t = 0,1.5,{1,[1.5]=2}
-      **   local x,k,t = 0,1.5,{1,[1.5]=2}
+    **   for i=1,200 do x = x+ t[k]; k = k == 1 and 1.5 or 1 end
-      **   for i=1,200 do x = x+ t[k]; k = k == 1 and 1.5 or 1 end
+    **   assert(x == 300)
-      **   assert(x == 300)
+    */
-      */
+    return FAILFOLD;
-      return FAILFOLD;
-    }
-    return INTFOLD(k);
-  } else {
-    return INTFOLD((int32_t)n);
  }
+  return INTFOLD(k);
 }
 LJFOLD(CONV KNUM IRCONV_U32_NUM)
 LJFOLDF(kfold_conv_knum_u32_num)
 {
-  lua_assert((fins->op2 & IRCONV_TRUNC));
 #ifdef _MSC_VER
  {  /* Workaround for MSVC bug. */
    volatile uint32_t u = (uint32_t)knumleft;
@@ -681,27 +841,27 @@ LJFOLDF(kfold_conv_knum_u32_num)
 LJFOLD(CONV KNUM IRCONV_I64_NUM)
 LJFOLDF(kfold_conv_knum_i64_num)
 {
-  lua_assert((fins->op2 & IRCONV_TRUNC));
  return INT64FOLD((uint64_t)(int64_t)knumleft);
 }
 LJFOLD(CONV KNUM IRCONV_U64_NUM)
 LJFOLDF(kfold_conv_knum_u64_num)
 {
-  lua_assert((fins->op2 & IRCONV_TRUNC));
  return INT64FOLD(lj_num2u64(knumleft));
 }
-LJFOLD(TOSTR KNUM)
+LJFOLD(TOSTR KNUM any)
 LJFOLDF(kfold_tostr_knum)
 {
-  return lj_ir_kstr(J, lj_str_fromnum(J->L, &knumleft));
+  return lj_ir_kstr(J, lj_strfmt_num(J->L, ir_knum(fleft)));
 }
-LJFOLD(TOSTR KINT)
+LJFOLD(TOSTR KINT any)
 LJFOLDF(kfold_tostr_kint)
 {
-  return lj_ir_kstr(J, lj_str_fromint(J->L, fleft->i));
+  return lj_ir_kstr(J, fins->op2 == IRTOSTR_INT ?
+                       lj_strfmt_int(J->L, fleft->i) :
+                       lj_strfmt_char(J->L, fleft->i));
 }
 LJFOLD(STRTO KGC)
@@ -1199,7 +1359,9 @@ static TRef simplify_intmul_k(jit_State *J, int32_t k)
  ** But this is mainly intended for simple address arithmetic.
  ** Also it's easier for the backend to optimize the original multiplies.
  */
-  if (k == 1) {  /* i * 1 ==> i */
+  if (k == 0) {  /* i * 0 ==> 0 */
+    return RIGHTFOLD;
+  } else if (k == 1) {  /* i * 1 ==> i */
    return LEFTFOLD;
  } else if ((k & (k-1)) == 0) {  /* i * 2^k ==> i << k */
    fins->o = IR_BSHL;
@@ -1212,9 +1374,7 @@ static TRef simplify_intmul_k(jit_State *J, int32_t k)
 LJFOLD(MUL any KINT)
 LJFOLDF(simplify_intmul_k32)
 {
-  if (fright->i == 0)  /* i * 0 ==> 0 */
+  if (fright->i >= 0)
-    return INTFOLD(0);
-  else if (fright->i > 0)
    return simplify_intmul_k(J, fright->i);
  return NEXTFOLD;
 }
@@ -1222,14 +1382,13 @@ LJFOLDF(simplify_intmul_k32)
 LJFOLD(MUL any KINT64)
 LJFOLDF(simplify_intmul_k64)
 {
-  if (ir_kint64(fright)->u64 == 0)  /* i * 0 ==> 0 */
+#if LJ_HASFFI
-    return INT64FOLD(0);
+  if (ir_kint64(fright)->u64 < 0x80000000u)
-#if LJ_64
-  /* NYI: SPLIT for BSHL and 32 bit backend support. */
-  else if (ir_kint64(fright)->u64 < 0x80000000u)
    return simplify_intmul_k(J, (int32_t)ir_kint64(fright)->u64);
-#endif
  return NEXTFOLD;
+#else
+  UNUSED(J); lua_assert(0); return FAILFOLD;
+#endif
 }
 LJFOLD(MOD any KINT)
@@ -1529,7 +1688,7 @@ LJFOLD(BOR BOR KINT64)
 LJFOLD(BXOR BXOR KINT64)
 LJFOLDF(reassoc_intarith_k64)
 {
-#if LJ_HASFFI || LJ_64
+#if LJ_HASFFI
  IRIns *irk = IR(fleft->op2);
  if (irk->o == IR_KINT64) {
    uint64_t k = kfold_int64arith(ir_k64(irk)->u64,
@@ -2007,6 +2166,14 @@ LJFOLDF(fload_str_len_snew)
  return NEXTFOLD;
 }
+LJFOLD(FLOAD TOSTR IRFL_STR_LEN)
+LJFOLDF(fload_str_len_tostr)
+{
+  if (LJ_LIKELY(J->flags & JIT_F_OPT_FOLD) && fleft->op2 == IRTOSTR_CHAR)
+    return INTFOLD(1);
+  return NEXTFOLD;
+}
 /* The C type ID of cdata objects is immutable. */
 LJFOLD(FLOAD KGC IRFL_CDATA_CTYPEID)
 LJFOLDF(fload_cdata_typeid_kgc)
@@ -2149,6 +2316,7 @@ LJFOLD(TNEW any any)
 LJFOLD(TDUP any)
 LJFOLD(CNEW any any)
 LJFOLD(XSNEW any any)
+LJFOLD(BUFHDR any any)
 LJFOLDX(lj_ir_emit)
 /* ------------------------------------------------------------------------ */
diff --git a/src/lj_opt_loop.c b/src/lj_opt_loop.c
index 3a119f47..9637263c 100644
--- a/src/lj_opt_loop.c
+++ b/src/lj_opt_loop.c
@@ -11,7 +11,7 @@
 #if LJ_HASJIT
 #include "lj_err.h"
-#include "lj_str.h"
+#include "lj_buf.h"
 #include "lj_ir.h"
 #include "lj_jit.h"
 #include "lj_iropt.h"
@@ -271,8 +271,7 @@ static void loop_unroll(jit_State *J)
  ** Caveat: don't call into the VM or run the GC or the buffer may be gone.
  */
  invar = J->cur.nins;
-  subst = (IRRef1 *)lj_str_needbuf(J->L, &G(J->L)->tmpbuf,
+  subst = (IRRef1 *)lj_buf_tmp(J->L, (invar-REF_BIAS)*sizeof(IRRef1))-REF_BIAS;
-                                   (invar-REF_BIAS)*sizeof(IRRef1)) - REF_BIAS;
  subst[REF_BASE] = REF_BASE;
  /* LOOP separates the pre-roll from the loop body. */
diff --git a/src/lj_opt_narrow.c b/src/lj_opt_narrow.c
index caf2a8df..5d0ea9cb 100644
--- a/src/lj_opt_narrow.c
+++ b/src/lj_opt_narrow.c
@@ -496,8 +496,7 @@ TRef LJ_FASTCALL lj_opt_narrow_cindex(jit_State *J, TRef tr)
 {
  lua_assert(tref_isnumber(tr));
  if (tref_isnum(tr))
-    return emitir(IRT(IR_CONV, IRT_INTP), tr,
+    return emitir(IRT(IR_CONV, IRT_INTP), tr, (IRT_INTP<<5)|IRT_NUM|IRCONV_ANY);
-                  (IRT_INTP<<5)|IRT_NUM|IRCONV_TRUNC|IRCONV_ANY);
  /* Undefined overflow semantics allow stripping of ADDOV, SUBOV and MULOV. */
  return narrow_stripov(J, tr, IR_MULOV,
                        LJ_64 ? ((IRT_INTP<<5)|IRT_INT|IRCONV_SEXT) :
diff --git a/src/lj_opt_split.c b/src/lj_opt_split.c
index 5a8c33b9..89db0e92 100644
--- a/src/lj_opt_split.c
+++ b/src/lj_opt_split.c
@@ -11,7 +11,7 @@
 #if LJ_HASJIT && (LJ_SOFTFP || (LJ_32 && LJ_HASFFI))
 #include "lj_err.h"
-#include "lj_str.h"
+#include "lj_buf.h"
 #include "lj_ir.h"
 #include "lj_jit.h"
 #include "lj_ircall.h"
@@ -139,6 +139,7 @@ static IRRef split_call_l(jit_State *J, IRRef1 *hisubst, IRIns *oir,
  ir->prev = tmp = split_emit(J, IRTI(IR_CALLN), tmp, id);
  return split_emit(J, IRT(IR_HIOP, IRT_SOFTFP), tmp, tmp);
 }
+#endif
 /* Emit a CALLN with one split 64 bit argument and a 32 bit argument. */
 static IRRef split_call_li(jit_State *J, IRRef1 *hisubst, IRIns *oir,
@@ -155,7 +156,6 @@ static IRRef split_call_li(jit_State *J, IRRef1 *hisubst, IRIns *oir,
  ir->prev = tmp = split_emit(J, IRTI(IR_CALLN), tmp, id);
  return split_emit(J, IRT(IR_HIOP, IRT_SOFTFP), tmp, tmp);
 }
-#endif
 /* Emit a CALLN with two split 64 bit arguments. */
 static IRRef split_call_ll(jit_State *J, IRRef1 *hisubst, IRIns *oir,
@@ -195,6 +195,118 @@ static IRRef split_ptr(jit_State *J, IRIns *oir, IRRef ref)
  return split_emit(J, IRTI(IR_ADD), nref, lj_ir_kint(J, ofs));
 }
+#if LJ_HASFFI
+static IRRef split_bitshift(jit_State *J, IRRef1 *hisubst,
+                            IRIns *oir, IRIns *nir, IRIns *ir)
+{
+  IROp op = ir->o;
+  IRRef kref = nir->op2;
+  if (irref_isk(kref)) {  /* Optimize constant shifts. */
+    int32_t k = (IR(kref)->i & 63);
+    IRRef lo = nir->op1, hi = hisubst[ir->op1];
+    if (op == IR_BROL || op == IR_BROR) {
+      if (op == IR_BROR) k = (-k & 63);
+      if (k >= 32) { IRRef t = lo; lo = hi; hi = t; k -= 32; }
+      if (k == 0) {
+      passthrough:
+        J->cur.nins--;
+        ir->prev = lo;
+        return hi;
+      } else {
+        TRef k1, k2;
+        IRRef t1, t2, t3, t4;
+        J->cur.nins--;
+        k1 = lj_ir_kint(J, k);
+        k2 = lj_ir_kint(J, (-k & 31));
+        t1 = split_emit(J, IRTI(IR_BSHL), lo, k1);
+        t2 = split_emit(J, IRTI(IR_BSHL), hi, k1);
+        t3 = split_emit(J, IRTI(IR_BSHR), lo, k2);
+        t4 = split_emit(J, IRTI(IR_BSHR), hi, k2);
+        ir->prev = split_emit(J, IRTI(IR_BOR), t1, t4);
+        return split_emit(J, IRTI(IR_BOR), t2, t3);
+      }
+    } else if (k == 0) {
+      goto passthrough;
+    } else if (k < 32) {
+      if (op == IR_BSHL) {
+        IRRef t1 = split_emit(J, IRTI(IR_BSHL), hi, kref);
+        IRRef t2 = split_emit(J, IRTI(IR_BSHR), lo, lj_ir_kint(J, (-k&31)));
+        return split_emit(J, IRTI(IR_BOR), t1, t2);
+      } else {
+        IRRef t1 = ir->prev, t2;
+        lua_assert(op == IR_BSHR || op == IR_BSAR);
+        nir->o = IR_BSHR;
+        t2 = split_emit(J, IRTI(IR_BSHL), hi, lj_ir_kint(J, (-k&31)));
+        ir->prev = split_emit(J, IRTI(IR_BOR), t1, t2);
+        return split_emit(J, IRTI(op), hi, kref);
+      }
+    } else {
+      if (op == IR_BSHL) {
+        if (k == 32)
+          J->cur.nins--;
+        else
+          lo = ir->prev;
+        ir->prev = lj_ir_kint(J, 0);
+        return lo;
+      } else {
+        lua_assert(op == IR_BSHR || op == IR_BSAR);
+        if (k == 32) {
+          J->cur.nins--;
+          ir->prev = hi;
+        } else {
+          nir->op1 = hi;
+        }
+        if (op == IR_BSHR)
+          return lj_ir_kint(J, 0);
+        else
+          return split_emit(J, IRTI(IR_BSAR), hi, lj_ir_kint(J, 31));
+      }
+    }
+  }
+  return split_call_li(J, hisubst, oir, ir,
+                       op - IR_BSHL + IRCALL_lj_carith_shl64);
+}
+static IRRef split_bitop(jit_State *J, IRRef1 *hisubst,
+                         IRIns *nir, IRIns *ir)
+{
+  IROp op = ir->o;
+  IRRef hi, kref = nir->op2;
+  if (irref_isk(kref)) {  /* Optimize bit operations with lo constant. */
+    int32_t k = IR(kref)->i;
+    if (k == 0 || k == -1) {
+      if (op == IR_BAND) k = ~k;
+      if (k == 0) {
+        J->cur.nins--;
+        ir->prev = nir->op1;
+      } else if (op == IR_BXOR) {
+        nir->o = IR_BNOT;
+        nir->op2 = 0;
+      } else {
+        J->cur.nins--;
+        ir->prev = kref;
+      }
+    }
+  }
+  hi = hisubst[ir->op1];
+  kref = hisubst[ir->op2];
+  if (irref_isk(kref)) {  /* Optimize bit operations with hi constant. */
+    int32_t k = IR(kref)->i;
+    if (k == 0 || k == -1) {
+      if (op == IR_BAND) k = ~k;
+      if (k == 0) {
+        return hi;
+      } else if (op == IR_BXOR) {
+        return split_emit(J, IRTI(IR_BNOT), hi, 0);
+      } else {
+        return kref;
+      }
+    }
+  }
+  return split_emit(J, IRTI(op), hi, kref);
+}
+#endif
 /* Substitute references of a snapshot. */
 static void split_subst_snap(jit_State *J, SnapShot *snap, IRIns *oir)
 {
@@ -214,7 +326,7 @@ static void split_ir(jit_State *J)
  IRRef nins = J->cur.nins, nk = J->cur.nk;
  MSize irlen = nins - nk;
  MSize need = (irlen+1)*(sizeof(IRIns) + sizeof(IRRef1));
-  IRIns *oir = (IRIns *)lj_str_needbuf(J->L, &G(J->L)->tmpbuf, need);
+  IRIns *oir = (IRIns *)lj_buf_tmp(J->L, need);
  IRRef1 *hisubst;
  IRRef ref, snref;
  SnapShot *snap;
@@ -438,6 +550,19 @@ static void split_ir(jit_State *J)
                           irt_isi64(ir->t) ? IRCALL_lj_carith_powi64 :
                                              IRCALL_lj_carith_powu64);
        break;
+      case IR_BNOT:
+        hi = split_emit(J, IRTI(IR_BNOT), hiref, 0);
+        break;
+      case IR_BSWAP:
+        ir->prev = split_emit(J, IRTI(IR_BSWAP), hiref, 0);
+        hi = nref;
+        break;
+      case IR_BAND: case IR_BOR: case IR_BXOR:
+        hi = split_bitop(J, hisubst, nir, ir);
+        break;
+      case IR_BSHL: case IR_BSHR: case IR_BSAR: case IR_BROL: case IR_BROR:
+        hi = split_bitshift(J, hisubst, oir, nir, ir);
+        break;
      case IR_FLOAD:
        lua_assert(ir->op2 == IRFL_CDATA_INT64);
        hi = split_emit(J, IRTI(IR_FLOAD), nir->op1, IRFL_CDATA_INT64_4);
diff --git a/src/lj_parse.c b/src/lj_parse.c
index 7ff7d728..cdb89baf 100644
--- a/src/lj_parse.c
+++ b/src/lj_parse.c
@@ -13,6 +13,7 @@
 #include "lj_gc.h"
 #include "lj_err.h"
 #include "lj_debug.h"
+#include "lj_buf.h"
 #include "lj_str.h"
 #include "lj_tab.h"
 #include "lj_func.h"
@@ -21,6 +22,7 @@
 #if LJ_HASFFI
 #include "lj_ctype.h"
 #endif
+#include "lj_strfmt.h"
 #include "lj_lex.h"
 #include "lj_parse.h"
 #include "lj_vm.h"
@@ -165,12 +167,12 @@ LJ_STATIC_ASSERT((int)BC_MODVV-(int)BC_ADDVV == (int)OPR_MOD-(int)OPR_ADD);
 LJ_NORET LJ_NOINLINE static void err_syntax(LexState *ls, ErrMsg em)
 {
-  lj_lex_error(ls, ls->token, em);
+  lj_lex_error(ls, ls->tok, em);
 }
-LJ_NORET LJ_NOINLINE static void err_token(LexState *ls, LexToken token)
+LJ_NORET LJ_NOINLINE static void err_token(LexState *ls, LexToken tok)
 {
-  lj_lex_error(ls, ls->token, LJ_ERR_XTOKEN, lj_lex_token2str(ls, token));
+  lj_lex_error(ls, ls->tok, LJ_ERR_XTOKEN, lj_lex_token2str(ls, tok));
 }
 LJ_NORET static void err_limit(FuncState *fs, uint32_t limit, const char *what)
@@ -981,7 +983,7 @@ static void bcemit_unop(FuncState *fs, BCOp op, ExpDesc *e)
 /* Check and consume optional token. */
 static int lex_opt(LexState *ls, LexToken tok)
 {
-  if (ls->token == tok) {
+  if (ls->tok == tok) {
    lj_lex_next(ls);
    return 1;
  }
@@ -991,7 +993,7 @@ static int lex_opt(LexState *ls, LexToken tok)
 /* Check and consume token. */
 static void lex_check(LexState *ls, LexToken tok)
 {
-  if (ls->token != tok)
+  if (ls->tok != tok)
    err_token(ls, tok);
  lj_lex_next(ls);
 }
@@ -1005,7 +1007,7 @@ static void lex_match(LexState *ls, LexToken what, LexToken who, BCLine line)
    } else {
      const char *swhat = lj_lex_token2str(ls, what);
      const char *swho = lj_lex_token2str(ls, who);
-      lj_lex_error(ls, ls->token, LJ_ERR_XMATCH, swhat, swho, line);
+      lj_lex_error(ls, ls->tok, LJ_ERR_XMATCH, swhat, swho, line);
    }
  }
 }
@@ -1014,9 +1016,9 @@ static void lex_match(LexState *ls, LexToken what, LexToken who, BCLine line)
 static GCstr *lex_str(LexState *ls)
 {
  GCstr *s;
-  if (ls->token != TK_name && (LJ_52 || ls->token != TK_goto))
+  if (ls->tok != TK_name && (LJ_52 || ls->tok != TK_goto))
    err_token(ls, TK_name);
-  s = strV(&ls->tokenval);
+  s = strV(&ls->tokval);
  lj_lex_next(ls);
  return s;
 }
@@ -1429,78 +1431,46 @@ static void fs_fixup_line(FuncState *fs, GCproto *pt,
  }
 }
-/* Resize buffer if needed. */
-static LJ_NOINLINE void fs_buf_resize(LexState *ls, MSize len)
-{
-  MSize sz = ls->sb.sz * 2;
-  while (ls->sb.n + len > sz) sz = sz * 2;
-  lj_str_resizebuf(ls->L, &ls->sb, sz);
-}
-static LJ_AINLINE void fs_buf_need(LexState *ls, MSize len)
-{
-  if (LJ_UNLIKELY(ls->sb.n + len > ls->sb.sz))
-    fs_buf_resize(ls, len);
-}
-/* Add string to buffer. */
-static void fs_buf_str(LexState *ls, const char *str, MSize len)
-{
-  char *p = ls->sb.buf + ls->sb.n;
-  MSize i;
-  ls->sb.n += len;
-  for (i = 0; i < len; i++) p[i] = str[i];
-}
-/* Add ULEB128 value to buffer. */
-static void fs_buf_uleb128(LexState *ls, uint32_t v)
-{
-  MSize n = ls->sb.n;
-  uint8_t *p = (uint8_t *)ls->sb.buf;
-  for (; v >= 0x80; v >>= 7)
-    p[n++] = (uint8_t)((v & 0x7f) | 0x80);
-  p[n++] = (uint8_t)v;
-  ls->sb.n = n;
-}
 /* Prepare variable info for prototype. */
 static size_t fs_prep_var(LexState *ls, FuncState *fs, size_t *ofsvar)
 {
  VarInfo *vs =ls->vstack, *ve;
  MSize i, n;
  BCPos lastpc;
-  lj_str_resetbuf(&ls->sb);  /* Copy to temp. string buffer. */
+  lj_buf_reset(&ls->sb);  /* Copy to temp. string buffer. */
  /* Store upvalue names. */
  for (i = 0, n = fs->nuv; i < n; i++) {
    GCstr *s = strref(vs[fs->uvmap[i]].name);
    MSize len = s->len+1;
-    fs_buf_need(ls, len);
+    char *p = lj_buf_more(&ls->sb, len);
-    fs_buf_str(ls, strdata(s), len);
+    p = lj_buf_wmem(p, strdata(s), len);
+    setsbufP(&ls->sb, p);
  }
-  *ofsvar = ls->sb.n;
+  *ofsvar = sbuflen(&ls->sb);
  lastpc = 0;
  /* Store local variable names and compressed ranges. */
  for (ve = vs + ls->vtop, vs += fs->vbase; vs < ve; vs++) {
    if (!gola_isgotolabel(vs)) {
      GCstr *s = strref(vs->name);
      BCPos startpc;
+      char *p;
      if ((uintptr_t)s < VARNAME__MAX) {
-        fs_buf_need(ls, 1 + 2*5);
+        p = lj_buf_more(&ls->sb, 1 + 2*5);
-        ls->sb.buf[ls->sb.n++] = (uint8_t)(uintptr_t)s;
+        *p++ = (char)(uintptr_t)s;
      } else {
        MSize len = s->len+1;
-        fs_buf_need(ls, len + 2*5);
+        p = lj_buf_more(&ls->sb, len + 2*5);
-        fs_buf_str(ls, strdata(s), len);
+        p = lj_buf_wmem(p, strdata(s), len);
      }
      startpc = vs->startpc;
-      fs_buf_uleb128(ls, startpc-lastpc);
+      p = lj_strfmt_wuleb128(p, startpc-lastpc);
-      fs_buf_uleb128(ls, vs->endpc-startpc);
+      p = lj_strfmt_wuleb128(p, vs->endpc-startpc);
+      setsbufP(&ls->sb, p);
      lastpc = startpc;
    }
  }
-  fs_buf_need(ls, 1);
+  lj_buf_putb(&ls->sb, '\0');  /* Terminator for varinfo. */
-  ls->sb.buf[ls->sb.n++] = '\0';  /* Terminator for varinfo. */
+  return sbuflen(&ls->sb);
-  return ls->sb.n;
 }
 /* Fixup variable info for prototype. */
@@ -1508,7 +1478,7 @@ static void fs_fixup_var(LexState *ls, GCproto *pt, uint8_t *p, size_t ofsvar)
 {
  setmref(pt->uvinfo, p);
  setmref(pt->varinfo, (char *)p + ofsvar);
-  memcpy(p, ls->sb.buf, ls->sb.n);  /* Copy from temp. string buffer. */
+  memcpy(p, sbufB(&ls->sb), sbuflen(&ls->sb));  /* Copy from temp. buffer. */
 }
 #else
@@ -1615,7 +1585,7 @@ static GCproto *fs_finish(LexState *ls, BCLine line)
  L->top--;  /* Pop table of constants. */
  ls->vtop = fs->vbase;  /* Reset variable stack. */
  ls->fs = fs->prev;
-  lua_assert(ls->fs != NULL || ls->token == TK_eof);
+  lua_assert(ls->fs != NULL || ls->tok == TK_eof);
  return pt;
 }
@@ -1737,15 +1707,15 @@ static void expr_table(LexState *ls, ExpDesc *e)
  bcreg_reserve(fs, 1);
  freg++;
  lex_check(ls, '{');
-  while (ls->token != '}') {
+  while (ls->tok != '}') {
    ExpDesc key, val;
    vcall = 0;
-    if (ls->token == '[') {
+    if (ls->tok == '[') {
      expr_bracket(ls, &key);  /* Already calls expr_toval. */
      if (!expr_isk(&key)) expr_index(fs, e, &key);
      if (expr_isnumk(&key) && expr_numiszero(&key)) needarr = 1; else nhash++;
      lex_check(ls, '=');
-    } else if ((ls->token == TK_name || (!LJ_52 && ls->token == TK_goto)) &&
+    } else if ((ls->tok == TK_name || (!LJ_52 && ls->tok == TK_goto)) &&
               lj_lex_lookahead(ls) == '=') {
      expr_str(ls, &key);
      lex_check(ls, '=');
@@ -1838,11 +1808,11 @@ static BCReg parse_params(LexState *ls, int needself)
  lex_check(ls, '(');
  if (needself)
    var_new_lit(ls, nparams++, "self");
-  if (ls->token != ')') {
+  if (ls->tok != ')') {
    do {
-      if (ls->token == TK_name || (!LJ_52 && ls->token == TK_goto)) {
+      if (ls->tok == TK_name || (!LJ_52 && ls->tok == TK_goto)) {
        var_new(ls, nparams++, lex_str(ls));
-      } else if (ls->token == TK_dots) {
+      } else if (ls->tok == TK_dots) {
        lj_lex_next(ls);
        fs->flags |= PROTO_VARARG;
        break;
@@ -1876,7 +1846,7 @@ static void parse_body(LexState *ls, ExpDesc *e, int needself, BCLine line)
  fs.bclim = pfs->bclim - pfs->pc;
  bcemit_AD(&fs, BC_FUNCF, 0, 0);  /* Placeholder. */
  parse_chunk(ls);
-  if (ls->token != TK_end) lex_match(ls, TK_end, TK_function, line);
+  if (ls->tok != TK_end) lex_match(ls, TK_end, TK_function, line);
  pt = fs_finish(ls, (ls->lastline = ls->linenumber));
  pfs->bcbase = ls->bcstack + oldbase;  /* May have been reallocated. */
  pfs->bclim = (BCPos)(ls->sizebcstack - oldbase);
@@ -1915,13 +1885,13 @@ static void parse_args(LexState *ls, ExpDesc *e)
  BCIns ins;
  BCReg base;
  BCLine line = ls->linenumber;
-  if (ls->token == '(') {
+  if (ls->tok == '(') {
 #if !LJ_52
    if (line != ls->lastline)
      err_syntax(ls, LJ_ERR_XAMBIG);
 #endif
    lj_lex_next(ls);
-    if (ls->token == ')') {  /* f(). */
+    if (ls->tok == ')') {  /* f(). */
      args.k = VVOID;
    } else {
      expr_list(ls, &args);
@@ -1929,11 +1899,11 @@ static void parse_args(LexState *ls, ExpDesc *e)
        setbc_b(bcptr(fs, &args), 0);  /* Pass on multiple results. */
    }
    lex_match(ls, ')', '(', line);
-  } else if (ls->token == '{') {
+  } else if (ls->tok == '{') {
    expr_table(ls, &args);
-  } else if (ls->token == TK_string) {
+  } else if (ls->tok == TK_string) {
    expr_init(&args, VKSTR, 0);
-    args.u.sval = strV(&ls->tokenval);
+    args.u.sval = strV(&ls->tokval);
    lj_lex_next(ls);
  } else {
    err_syntax(ls, LJ_ERR_XFUNARG);
@@ -1959,32 +1929,32 @@ static void expr_primary(LexState *ls, ExpDesc *v)
 {
  FuncState *fs = ls->fs;
  /* Parse prefix expression. */
-  if (ls->token == '(') {
+  if (ls->tok == '(') {
    BCLine line = ls->linenumber;
    lj_lex_next(ls);
    expr(ls, v);
    lex_match(ls, ')', '(', line);
    expr_discharge(ls->fs, v);
-  } else if (ls->token == TK_name || (!LJ_52 && ls->token == TK_goto)) {
+  } else if (ls->tok == TK_name || (!LJ_52 && ls->tok == TK_goto)) {
    var_lookup(ls, v);
  } else {
    err_syntax(ls, LJ_ERR_XSYMBOL);
  }
  for (;;) {  /* Parse multiple expression suffixes. */
-    if (ls->token == '.') {
+    if (ls->tok == '.') {
      expr_field(ls, v);
-    } else if (ls->token == '[') {
+    } else if (ls->tok == '[') {
      ExpDesc key;
      expr_toanyreg(fs, v);
      expr_bracket(ls, &key);
      expr_index(fs, v, &key);
-    } else if (ls->token == ':') {
+    } else if (ls->tok == ':') {
      ExpDesc key;
      lj_lex_next(ls);
      expr_str(ls, &key);
      bcemit_method(fs, v, &key);
      parse_args(ls, v);
-    } else if (ls->token == '(' || ls->token == TK_string || ls->token == '{') {
+    } else if (ls->tok == '(' || ls->tok == TK_string || ls->tok == '{') {
      expr_tonextreg(fs, v);
      parse_args(ls, v);
    } else {
@@ -1996,14 +1966,14 @@ static void expr_primary(LexState *ls, ExpDesc *v)
 /* Parse simple expression. */
 static void expr_simple(LexState *ls, ExpDesc *v)
 {
-  switch (ls->token) {
+  switch (ls->tok) {
  case TK_number:
-    expr_init(v, (LJ_HASFFI && tviscdata(&ls->tokenval)) ? VKCDATA : VKNUM, 0);
+    expr_init(v, (LJ_HASFFI && tviscdata(&ls->tokval)) ? VKCDATA : VKNUM, 0);
-    copyTV(ls->L, &v->u.nval, &ls->tokenval);
+    copyTV(ls->L, &v->u.nval, &ls->tokval);
    break;
  case TK_string:
    expr_init(v, VKSTR, 0);
-    v->u.sval = strV(&ls->tokenval);
+    v->u.sval = strV(&ls->tokval);
    break;
  case TK_nil:
    expr_init(v, VKNIL, 0);
@@ -2091,11 +2061,11 @@ static BinOpr expr_binop(LexState *ls, ExpDesc *v, uint32_t limit);
 static void expr_unop(LexState *ls, ExpDesc *v)
 {
  BCOp op;
-  if (ls->token == TK_not) {
+  if (ls->tok == TK_not) {
    op = BC_NOT;
-  } else if (ls->token == '-') {
+  } else if (ls->tok == '-') {
    op = BC_UNM;
-  } else if (ls->token == '#') {
+  } else if (ls->tok == '#') {
    op = BC_LEN;
  } else {
    expr_simple(ls, v);
@@ -2112,7 +2082,7 @@ static BinOpr expr_binop(LexState *ls, ExpDesc *v, uint32_t limit)
  BinOpr op;
  synlevel_begin(ls);
  expr_unop(ls, v);
-  op = token2binop(ls->token);
+  op = token2binop(ls->tok);
  while (op != OPR_NOBINOPR && priority[op].left > limit) {
    ExpDesc v2;
    BinOpr nextop;
@@ -2301,9 +2271,9 @@ static void parse_func(LexState *ls, BCLine line)
  lj_lex_next(ls);  /* Skip 'function'. */
  /* Parse function name. */
  var_lookup(ls, &v);
-  while (ls->token == '.')  /* Multiple dot-separated fields. */
+  while (ls->tok == '.')  /* Multiple dot-separated fields. */
    expr_field(ls, &v);
-  if (ls->token == ':') {  /* Optional colon to signify method call. */
+  if (ls->tok == ':') {  /* Optional colon to signify method call. */
    needself = 1;
    expr_field(ls, &v);
  }
@@ -2316,9 +2286,9 @@ static void parse_func(LexState *ls, BCLine line)
 /* -- Control transfer statements ----------------------------------------- */
 /* Check for end of block. */
-static int endofblock(LexToken token)
+static int parse_isend(LexToken tok)
 {
-  switch (token) {
+  switch (tok) {
  case TK_else: case TK_elseif: case TK_end: case TK_until: case TK_eof:
    return 1;
  default:
@@ -2333,7 +2303,7 @@ static void parse_return(LexState *ls)
  FuncState *fs = ls->fs;
  lj_lex_next(ls);  /* Skip 'return'. */
  fs->flags |= PROTO_HAS_RETURN;
-  if (endofblock(ls->token) || ls->token == ';') {  /* Bare return. */
+  if (parse_isend(ls->tok) || ls->tok == ';') {  /* Bare return. */
    ins = BCINS_AD(BC_RET0, 0, 1);
  } else {  /* Return with one or more values. */
    ExpDesc e;  /* Receives the _last_ expression in the list. */
@@ -2399,18 +2369,18 @@ static void parse_label(LexState *ls)
  lex_check(ls, TK_label);
  /* Recursively parse trailing statements: labels and ';' (Lua 5.2 only). */
  for (;;) {
-    if (ls->token == TK_label) {
+    if (ls->tok == TK_label) {
      synlevel_begin(ls);
      parse_label(ls);
      synlevel_end(ls);
-    } else if (LJ_52 && ls->token == ';') {
+    } else if (LJ_52 && ls->tok == ';') {
      lj_lex_next(ls);
    } else {
      break;
    }
  }
  /* Trailing label is considered to be outside of scope. */
-  if (endofblock(ls->token) && ls->token != TK_until)
+  if (parse_isend(ls->tok) && ls->tok != TK_until)
    ls->vstack[idx].slot = fs->bl->nactvar;
  gola_resolve(ls, fs->bl, idx);
 }
@@ -2594,9 +2564,9 @@ static void parse_for(LexState *ls, BCLine line)
  fscope_begin(fs, &bl, FSCOPE_LOOP);
  lj_lex_next(ls);  /* Skip 'for'. */
  varname = lex_str(ls);  /* Get first variable name. */
-  if (ls->token == '=')
+  if (ls->tok == '=')
    parse_for_num(ls, varname, line);
-  else if (ls->token == ',' || ls->token == TK_in)
+  else if (ls->tok == ',' || ls->tok == TK_in)
    parse_for_iter(ls, varname);
  else
    err_syntax(ls, LJ_ERR_XFOR);
@@ -2622,12 +2592,12 @@ static void parse_if(LexState *ls, BCLine line)
  BCPos flist;
  BCPos escapelist = NO_JMP;
  flist = parse_then(ls);
-  while (ls->token == TK_elseif) {  /* Parse multiple 'elseif' blocks. */
+  while (ls->tok == TK_elseif) {  /* Parse multiple 'elseif' blocks. */
    jmp_append(fs, &escapelist, bcemit_jmp(fs));
    jmp_tohere(fs, flist);
    flist = parse_then(ls);
  }
-  if (ls->token == TK_else) {  /* Parse optional 'else' block. */
+  if (ls->tok == TK_else) {  /* Parse optional 'else' block. */
    jmp_append(fs, &escapelist, bcemit_jmp(fs));
    jmp_tohere(fs, flist);
    lj_lex_next(ls);  /* Skip 'else'. */
@@ -2645,7 +2615,7 @@ static void parse_if(LexState *ls, BCLine line)
 static int parse_stmt(LexState *ls)
 {
  BCLine line = ls->linenumber;
-  switch (ls->token) {
+  switch (ls->tok) {
  case TK_if:
    parse_if(ls, line);
    break;
@@ -2703,7 +2673,7 @@ static void parse_chunk(LexState *ls)
 {
  int islast = 0;
  synlevel_begin(ls);
-  while (!islast && !endofblock(ls->token)) {
+  while (!islast && !parse_isend(ls->tok)) {
    islast = parse_stmt(ls);
    lex_opt(ls, ';');
    lua_assert(ls->fs->framesize >= ls->fs->freereg &&
@@ -2738,7 +2708,7 @@ GCproto *lj_parse(LexState *ls)
  bcemit_AD(&fs, BC_FUNCV, 0, 0);  /* Placeholder. */
  lj_lex_next(ls);  /* Read-ahead first token. */
  parse_chunk(ls);
-  if (ls->token != TK_eof)
+  if (ls->tok != TK_eof)
    err_token(ls, TK_eof);
  pt = fs_finish(ls, ls->linenumber);
  L->top--;  /* Drop chunkname. */
diff --git a/src/lj_record.c b/src/lj_record.c
index 7336e0ac..bdf0212a 100644
--- a/src/lj_record.c
+++ b/src/lj_record.c
@@ -677,6 +677,8 @@ static int check_downrec_unroll(jit_State *J, GCproto *pt)
  return 0;
 }
+static TRef rec_cat(jit_State *J, BCReg baseslot, BCReg topslot);
 /* Record return. */
 void lj_record_ret(jit_State *J, BCReg rbase, ptrdiff_t gotresults)
 {
@@ -769,7 +771,24 @@ void lj_record_ret(jit_State *J, BCReg rbase, ptrdiff_t gotresults)
    } else if (cont == lj_cont_nop) {
      /* Nothing to do here. */
    } else if (cont == lj_cont_cat) {
-      lua_assert(0);
+      BCReg bslot = bc_b(*(frame_contpc(frame)-1));
+      TRef tr = gotresults ? J->base[cbase+rbase] : TREF_NIL;
+      if (bslot != cbase-2) {  /* Concatenate the remainder. */
+        TValue *b = J->L->base, save;  /* Simulate lower frame and result. */
+        J->base[cbase-2] = tr;
+        copyTV(J->L, &save, b-2);
+        if (gotresults) copyTV(J->L, b-2, b+rbase); else setnilV(b-2);
+        J->L->base = b - cbase;
+        tr = rec_cat(J, bslot, cbase-2);
+        b = J->L->base + cbase;  /* Undo. */
+        J->L->base = b;
+        copyTV(J->L, b-2, &save);
+      }
+      if (tr) {  /* Store final result. */
+        BCReg dst = bc_a(*(frame_contpc(frame)-1));
+        J->base[dst] = tr;
+        if (dst >= J->maxslot) J->maxslot = dst+1;
+      }  /* Otherwise continue with another __concat call. */
    } else {
      /* Result type already specialized. */
      lua_assert(cont == lj_cont_condf || cont == lj_cont_condt);
@@ -785,13 +804,11 @@ void lj_record_ret(jit_State *J, BCReg rbase, ptrdiff_t gotresults)
 /* Prepare to record call to metamethod. */
 static BCReg rec_mm_prep(jit_State *J, ASMFunction cont)
 {
-  BCReg s, top = curr_proto(J->L)->framesize;
+  BCReg s, top = cont == lj_cont_cat ? J->maxslot : curr_proto(J->L)->framesize;
-  TRef trcont;
-  setcont(&J->L->base[top], cont);
 #if LJ_64
-  trcont = lj_ir_kptr(J, (void *)((int64_t)cont - (int64_t)lj_vm_asm_begin));
+  TRef trcont = lj_ir_kptr(J, (void *)((int64_t)cont-(int64_t)lj_vm_asm_begin));
 #else
-  trcont = lj_ir_kptr(J, (void *)cont);
+  TRef trcont = lj_ir_kptr(J, (void *)cont);
 #endif
  J->base[top] = trcont | TREF_CONT;
  J->framedepth++;
@@ -872,7 +889,7 @@ nocheck:
 static TRef rec_mm_arith(jit_State *J, RecordIndex *ix, MMS mm)
 {
  /* Set up metamethod call first to save ix->tab and ix->tabv. */
-  BCReg func = rec_mm_prep(J, lj_cont_ra);
+  BCReg func = rec_mm_prep(J, mm == MM_concat ? lj_cont_cat : lj_cont_ra);
  TRef *base = J->base + func;
  TValue *basev = J->L->base + func;
  base[1] = ix->tab; base[2] = ix->key;
@@ -1599,6 +1616,54 @@ static TRef rec_tnew(jit_State *J, uint32_t ah)
  return emitir(IRTG(IR_TNEW, IRT_TAB), asize, hbits);
 }
+/* -- Concatenation ------------------------------------------------------- */
+static TRef rec_cat(jit_State *J, BCReg baseslot, BCReg topslot)
+{
+  TRef *top = &J->base[topslot];
+  TValue savetv[5];
+  BCReg s;
+  RecordIndex ix;
+  lua_assert(baseslot < topslot);
+  for (s = baseslot; s <= topslot; s++)
+    (void)getslot(J, s);  /* Ensure all arguments have a reference. */
+  if (tref_isnumber_str(top[0]) && tref_isnumber_str(top[-1])) {
+    TRef tr, hdr, *trp, *xbase, *base = &J->base[baseslot];
+    /* First convert numbers to strings. */
+    for (trp = top; trp >= base; trp--) {
+      if (tref_isnumber(*trp))
+        *trp = emitir(IRT(IR_TOSTR, IRT_STR), *trp,
+                      tref_isnum(*trp) ? IRTOSTR_NUM : IRTOSTR_INT);
+      else if (!tref_isstr(*trp))
+        break;
+    }
+    xbase = ++trp;
+    tr = hdr = emitir(IRT(IR_BUFHDR, IRT_P32),
+                      lj_ir_kptr(J, &J2G(J)->tmpbuf), IRBUFHDR_RESET);
+    do {
+      tr = emitir(IRT(IR_BUFPUT, IRT_P32), tr, *trp++);
+    } while (trp <= top);
+    tr = emitir(IRT(IR_BUFSTR, IRT_STR), tr, hdr);
+    J->maxslot = (BCReg)(xbase - J->base);
+    if (xbase == base) return tr;  /* Return simple concatenation result. */
+    /* Pass partial result. */
+    topslot = J->maxslot--;
+    *xbase = tr;
+    top = xbase;
+    setstrV(J->L, &ix.keyv, &J2G(J)->strempty);  /* Simulate string result. */
+  } else {
+    J->maxslot = topslot-1;
+    copyTV(J->L, &ix.keyv, &J->L->base[topslot]);
+  }
+  copyTV(J->L, &ix.tabv, &J->L->base[topslot-1]);
+  ix.tab = top[-1];
+  ix.key = top[0];
+  memcpy(savetv, &J->L->base[topslot-1], sizeof(savetv));  /* Save slots. */
+  rec_mm_arith(J, &ix, MM_concat);  /* Call __concat metamethod. */
+  memcpy(&J->L->base[topslot-1], savetv, sizeof(savetv));  /* Restore slots. */
+  return 0;  /* No result yet. */
+}
 /* -- Record bytecode ops ------------------------------------------------- */
 /* Prepare for comparison. */
@@ -1826,6 +1891,18 @@ void lj_record_ins(jit_State *J)
      J->maxslot = bc_a(pc[1]);  /* Shrink used slots. */
    break;
+  case BC_ISTYPE: case BC_ISNUM:
+    /* These coercions need to correspond with lj_meta_istype(). */
+    if (LJ_DUALNUM && rc == ~LJ_TNUMX+1)
+      ra = lj_opt_narrow_toint(J, ra);
+    else if (rc == ~LJ_TNUMX+2)
+      ra = lj_ir_tonum(J, ra);
+    else if (rc == ~LJ_TSTR+1)
+      ra = lj_ir_tostr(J, ra);
+    /* else: type specialization suffices. */
+    J->base[bc_a(ins)] = ra;
+    break;
  /* -- Unary ops --------------------------------------------------------- */
  case BC_NOT:
@@ -1889,6 +1966,12 @@ void lj_record_ins(jit_State *J)
      rc = rec_mm_arith(J, &ix, MM_pow);
    break;
+  /* -- Miscellaneous ops ------------------------------------------------- */
+  case BC_CAT:
+    rc = rec_cat(J, rb, rc);
+    break;
  /* -- Constant and move ops --------------------------------------------- */
  case BC_MOV:
@@ -1937,6 +2020,10 @@ void lj_record_ins(jit_State *J)
    ix.idxchain = LJ_MAX_IDXCHAIN;
    rc = lj_record_idx(J, &ix);
    break;
+  case BC_TGETR: case BC_TSETR:
+    ix.idxchain = 0;
+    rc = lj_record_idx(J, &ix);
+    break;
  case BC_TNEW:
    rc = rec_tnew(J, rc);
@@ -2066,7 +2153,6 @@ void lj_record_ins(jit_State *J)
    /* fallthrough */
  case BC_ITERN:
  case BC_ISNEXT:
-  case BC_CAT:
  case BC_UCLO:
  case BC_FNEW:
  case BC_TSETM:
diff --git a/src/lj_state.c b/src/lj_state.c
index 8c53d37f..604ff886 100644
--- a/src/lj_state.c
+++ b/src/lj_state.c
@@ -12,6 +12,7 @@
 #include "lj_obj.h"
 #include "lj_gc.h"
 #include "lj_err.h"
+#include "lj_buf.h"
 #include "lj_str.h"
 #include "lj_tab.h"
 #include "lj_func.h"
@@ -164,7 +165,7 @@ static void close_state(lua_State *L)
  lj_ctype_freestate(g);
 #endif
  lj_mem_freevec(g, g->strhash, g->strmask+1, GCRef);
-  lj_str_freebuf(g, &g->tmpbuf);
+  lj_buf_free(g, &g->tmpbuf);
  lj_mem_freevec(g, tvref(L->stack), L->stacksize, TValue);
  lua_assert(g->gc.total == sizeof(GG_State));
 #ifndef LUAJIT_USE_SYSMALLOC
@@ -203,7 +204,7 @@ LUA_API lua_State *lua_newstate(lua_Alloc f, void *ud)
  setnilV(&g->nilnode.val);
  setnilV(&g->nilnode.key);
  setmref(g->nilnode.freetop, &g->nilnode);
-  lj_str_initbuf(&g->tmpbuf);
+  lj_buf_init(NULL, &g->tmpbuf);
  g->gc.state = GCSpause;
  setgcref(g->gc.root, obj2gco(L));
  setmref(g->gc.sweep, &g->gc.root);
diff --git a/src/lj_str.c b/src/lj_str.c
index 6548ee4d..24d96067 100644
--- a/src/lj_str.c
+++ b/src/lj_str.c
@@ -1,9 +1,6 @@
 /*
 ** String handling.
 ** Copyright (C) 2005-2013 Mike Pall. See Copyright Notice in luajit.h
-**
-** Portions taken verbatim or adapted from the Lua interpreter.
-** Copyright (C) 1994-2008 Lua.org, PUC-Rio. See Copyright Notice in lua.h
 */
 #include <stdio.h>
@@ -15,10 +12,9 @@
 #include "lj_gc.h"
 #include "lj_err.h"
 #include "lj_str.h"
-#include "lj_state.h"
 #include "lj_char.h"
-/* -- String interning ---------------------------------------------------- */
+/* -- String helpers ------------------------------------------------------ */
 /* Ordered compare of strings. Assumes string data is 4-byte aligned. */
 int32_t LJ_FASTCALL lj_str_cmp(GCstr *a, GCstr *b)
@@ -64,6 +60,40 @@ static LJ_AINLINE int str_fastcmp(const char *a, const char *b, MSize len)
  return 0;
 }
+/* Find fixed string p inside string s. */
+const char *lj_str_find(const char *s, const char *p, MSize slen, MSize plen)
+{
+  if (plen <= slen) {
+    if (plen == 0) {
+      return s;
+    } else {
+      int c = *(const uint8_t *)p++;
+      plen--; slen -= plen;
+      while (slen) {
+        const char *q = (const char *)memchr(s, c, slen);
+        if (!q) break;
+        if (memcmp(q+1, p, plen) == 0) return q;
+        q++; slen -= (MSize)(q-s); s = q;
+      }
+    }
+  }
+  return NULL;
+}
+/* Check whether a string has a pattern matching character. */
+int lj_str_haspattern(GCstr *s)
+{
+  const char *p = strdata(s), *q = p + s->len;
+  while (p < q) {
+    int c = *(const uint8_t *)p++;
+    if (lj_char_ispunct(c) && strchr("^$*+?.([%-", c))
+      return 1;  /* Found a pattern matching char. */
+  }
+  return 0;  /* No pattern matching chars found. */
+}
+/* -- String interning ---------------------------------------------------- */
 /* Resize the string hash table (grow and shrink). */
 void lj_str_resize(lua_State *L, MSize newmask)
 {
@@ -167,173 +197,3 @@ void LJ_FASTCALL lj_str_free(global_State *g, GCstr *s)
  lj_mem_free(g, s, sizestring(s));
 }
-/* -- Type conversions ---------------------------------------------------- */
-/* Print number to buffer. Canonicalizes non-finite values. */
-size_t LJ_FASTCALL lj_str_bufnum(char *s, cTValue *o)
-{
-  if (LJ_LIKELY((o->u32.hi << 1) < 0xffe00000)) {  /* Finite? */
-    lua_Number n = o->n;
-#if __BIONIC__
-    if (tvismzero(o)) { s[0] = '-'; s[1] = '0'; return 2; }
-#endif
-    return (size_t)lua_number2str(s, n);
-  } else if (((o->u32.hi & 0x000fffff) | o->u32.lo) != 0) {
-    s[0] = 'n'; s[1] = 'a'; s[2] = 'n'; return 3;
-  } else if ((o->u32.hi & 0x80000000) == 0) {
-    s[0] = 'i'; s[1] = 'n'; s[2] = 'f'; return 3;
-  } else {
-    s[0] = '-'; s[1] = 'i'; s[2] = 'n'; s[3] = 'f'; return 4;
-  }
-}
-/* Print integer to buffer. Returns pointer to start. */
-char * LJ_FASTCALL lj_str_bufint(char *p, int32_t k)
-{
-  uint32_t u = (uint32_t)(k < 0 ? -k : k);
-  p += 1+10;
-  do { *--p = (char)('0' + u % 10); } while (u /= 10);
-  if (k < 0) *--p = '-';
-  return p;
-}
-/* Convert number to string. */
-GCstr * LJ_FASTCALL lj_str_fromnum(lua_State *L, const lua_Number *np)
-{
-  char buf[LJ_STR_NUMBUF];
-  size_t len = lj_str_bufnum(buf, (TValue *)np);
-  return lj_str_new(L, buf, len);
-}
-/* Convert integer to string. */
-GCstr * LJ_FASTCALL lj_str_fromint(lua_State *L, int32_t k)
-{
-  char s[1+10];
-  char *p = lj_str_bufint(s, k);
-  return lj_str_new(L, p, (size_t)(s+sizeof(s)-p));
-}
-GCstr * LJ_FASTCALL lj_str_fromnumber(lua_State *L, cTValue *o)
-{
-  return tvisint(o) ? lj_str_fromint(L, intV(o)) : lj_str_fromnum(L, &o->n);
-}
-/* -- String formatting --------------------------------------------------- */
-static void addstr(lua_State *L, SBuf *sb, const char *str, MSize len)
-{
-  char *p;
-  MSize i;
-  if (sb->n + len > sb->sz) {
-    MSize sz = sb->sz * 2;
-    while (sb->n + len > sz) sz = sz * 2;
-    lj_str_resizebuf(L, sb, sz);
-  }
-  p = sb->buf + sb->n;
-  sb->n += len;
-  for (i = 0; i < len; i++) p[i] = str[i];
-}
-static void addchar(lua_State *L, SBuf *sb, int c)
-{
-  if (sb->n + 1 > sb->sz) {
-    MSize sz = sb->sz * 2;
-    lj_str_resizebuf(L, sb, sz);
-  }
-  sb->buf[sb->n++] = (char)c;
-}
-/* Push formatted message as a string object to Lua stack. va_list variant. */
-const char *lj_str_pushvf(lua_State *L, const char *fmt, va_list argp)
-{
-  SBuf *sb = &G(L)->tmpbuf;
-  lj_str_needbuf(L, sb, (MSize)strlen(fmt));
-  lj_str_resetbuf(sb);
-  for (;;) {
-    const char *e = strchr(fmt, '%');
-    if (e == NULL) break;
-    addstr(L, sb, fmt, (MSize)(e-fmt));
-    /* This function only handles %s, %c, %d, %f and %p formats. */
-    switch (e[1]) {
-    case 's': {
-      const char *s = va_arg(argp, char *);
-      if (s == NULL) s = "(null)";
-      addstr(L, sb, s, (MSize)strlen(s));
-      break;
-      }
-    case 'c':
-      addchar(L, sb, va_arg(argp, int));
-      break;
-    case 'd': {
-      char buf[LJ_STR_INTBUF];
-      char *p = lj_str_bufint(buf, va_arg(argp, int32_t));
-      addstr(L, sb, p, (MSize)(buf+LJ_STR_INTBUF-p));
-      break;
-      }
-    case 'f': {
-      char buf[LJ_STR_NUMBUF];
-      TValue tv;
-      MSize len;
-      tv.n = (lua_Number)(va_arg(argp, LUAI_UACNUMBER));
-      len = (MSize)lj_str_bufnum(buf, &tv);
-      addstr(L, sb, buf, len);
-      break;
-      }
-    case 'p': {
-#define FMTP_CHARS      (2*sizeof(ptrdiff_t))
-      char buf[2+FMTP_CHARS];
-      ptrdiff_t p = (ptrdiff_t)(va_arg(argp, void *));
-      ptrdiff_t i, lasti = 2+FMTP_CHARS;
-      if (p == 0) {
-        addstr(L, sb, "NULL", 4);
-        break;
-      }
-#if LJ_64
-      /* Shorten output for 64 bit pointers. */
-      lasti = 2+2*4+((p >> 32) ? 2+2*(lj_fls((uint32_t)(p >> 32))>>3) : 0);
-#endif
-      buf[0] = '0';
-      buf[1] = 'x';
-      for (i = lasti-1; i >= 2; i--, p >>= 4)
-        buf[i] = "0123456789abcdef"[(p & 15)];
-      addstr(L, sb, buf, (MSize)lasti);
-      break;
-      }
-    case '%':
-      addchar(L, sb, '%');
-      break;
-    default:
-      addchar(L, sb, '%');
-      addchar(L, sb, e[1]);
-      break;
-    }
-    fmt = e+2;
-  }
-  addstr(L, sb, fmt, (MSize)strlen(fmt));
-  setstrV(L, L->top, lj_str_new(L, sb->buf, sb->n));
-  incr_top(L);
-  return strVdata(L->top - 1);
-}
-/* Push formatted message as a string object to Lua stack. Vararg variant. */
-const char *lj_str_pushf(lua_State *L, const char *fmt, ...)
-{
-  const char *msg;
-  va_list argp;
-  va_start(argp, fmt);
-  msg = lj_str_pushvf(L, fmt, argp);
-  va_end(argp);
-  return msg;
-}
-/* -- Buffer handling ----------------------------------------------------- */
-char *lj_str_needbuf(lua_State *L, SBuf *sb, MSize sz)
-{
-  if (sz > sb->sz) {
-    if (sz < LJ_MIN_SBUF) sz = LJ_MIN_SBUF;
-    lj_str_resizebuf(L, sb, sz);
-  }
-  return sb->buf;
-}
diff --git a/src/lj_str.h b/src/lj_str.h
index 3aa03662..f156ff6f 100644
--- a/src/lj_str.h
+++ b/src/lj_str.h
@@ -10,8 +10,13 @@
 #include "lj_obj.h"
-/* String interning. */
+/* String helpers. */
 LJ_FUNC int32_t LJ_FASTCALL lj_str_cmp(GCstr *a, GCstr *b);
+LJ_FUNC const char *lj_str_find(const char *s, const char *f,
+                                MSize slen, MSize flen);
+LJ_FUNC int lj_str_haspattern(GCstr *s);
+/* String interning. */
 LJ_FUNC void lj_str_resize(lua_State *L, MSize newmask);
 LJ_FUNCA GCstr *lj_str_new(lua_State *L, const char *str, size_t len);
 LJ_FUNC void LJ_FASTCALL lj_str_free(global_State *g, GCstr *s);
@@ -19,32 +24,4 @@ LJ_FUNC void LJ_FASTCALL lj_str_free(global_State *g, GCstr *s);
 #define lj_str_newz(L, s)       (lj_str_new(L, s, strlen(s)))
 #define lj_str_newlit(L, s)     (lj_str_new(L, "" s, sizeof(s)-1))
-/* Type conversions. */
-LJ_FUNC size_t LJ_FASTCALL lj_str_bufnum(char *s, cTValue *o);
-LJ_FUNC char * LJ_FASTCALL lj_str_bufint(char *p, int32_t k);
-LJ_FUNCA GCstr * LJ_FASTCALL lj_str_fromnum(lua_State *L, const lua_Number *np);
-LJ_FUNC GCstr * LJ_FASTCALL lj_str_fromint(lua_State *L, int32_t k);
-LJ_FUNCA GCstr * LJ_FASTCALL lj_str_fromnumber(lua_State *L, cTValue *o);
-#define LJ_STR_INTBUF           (1+10)
-#define LJ_STR_NUMBUF           LUAI_MAXNUMBER2STR
-/* String formatting. */
-LJ_FUNC const char *lj_str_pushvf(lua_State *L, const char *fmt, va_list argp);
-LJ_FUNC const char *lj_str_pushf(lua_State *L, const char *fmt, ...)
-#if defined(__GNUC__)
-  __attribute__ ((format (printf, 2, 3)))
-#endif
-  ;
-/* Resizable string buffers. Struct definition in lj_obj.h. */
-LJ_FUNC char *lj_str_needbuf(lua_State *L, SBuf *sb, MSize sz);
-#define lj_str_initbuf(sb)      ((sb)->buf = NULL, (sb)->sz = 0)
-#define lj_str_resetbuf(sb)     ((sb)->n = 0)
-#define lj_str_resizebuf(L, sb, size) \
-  ((sb)->buf = (char *)lj_mem_realloc(L, (sb)->buf, (sb)->sz, (size)), \
-   (sb)->sz = (size))
-#define lj_str_freebuf(g, sb)   lj_mem_free(g, (void *)(sb)->buf, (sb)->sz)
 #endif
diff --git a/src/lj_strfmt.c b/src/lj_strfmt.c
new file mode 100644
index 00000000..0003de70
--- /dev/null
+++ b/src/lj_strfmt.c
@@ -0,0 +1,549 @@
+/*
+** String formatting.
+** Copyright (C) 2005-2013 Mike Pall. See Copyright Notice in luajit.h
+*/
+#include <stdio.h>
+#define lj_strfmt_c
+#define LUA_CORE
+#include "lj_obj.h"
+#include "lj_buf.h"
+#include "lj_str.h"
+#include "lj_state.h"
+#include "lj_char.h"
+#include "lj_strfmt.h"
+/* -- Format parser ------------------------------------------------------- */
+static const uint8_t strfmt_map[('x'-'A')+1] = {
+  STRFMT_A,0,0,0,STRFMT_E,0,STRFMT_G,0,0,0,0,0,0,
+  0,0,0,0,0,0,0,0,0,0,STRFMT_X,0,0,
+  0,0,0,0,0,0,
+  STRFMT_A,0,STRFMT_C,STRFMT_D,STRFMT_E,STRFMT_F,STRFMT_G,0,STRFMT_I,0,0,0,0,
+  0,STRFMT_O,STRFMT_P,STRFMT_Q,0,STRFMT_S,0,STRFMT_U,0,0,STRFMT_X
+};
+SFormat LJ_FASTCALL lj_strfmt_parse(FormatState *fs)
+{
+  const uint8_t *p = fs->p, *e = fs->e;
+  fs->str = (const char *)p;
+  for (; p < e; p++) {
+    if (*p == '%') {  /* Escape char? */
+      if (p[1] == '%') {  /* '%%'? */
+        fs->p = ++p+1;
+        goto retlit;
+      } else {
+        SFormat sf = 0;
+        uint32_t c;
+        if (p != (const uint8_t *)fs->str)
+          break;
+        for (p++; (uint32_t)*p - ' ' <= (uint32_t)('0' - ' '); p++) {
+          /* Parse flags. */
+          if (*p == '-') sf |= STRFMT_F_LEFT;
+          else if (*p == '+') sf |= STRFMT_F_PLUS;
+          else if (*p == '0') sf |= STRFMT_F_ZERO;
+          else if (*p == ' ') sf |= STRFMT_F_SPACE;
+          else if (*p == '#') sf |= STRFMT_F_ALT;
+          else break;
+        }
+        if ((uint32_t)*p - '0' < 10) {  /* Parse width. */
+          uint32_t width = (uint32_t)*p++ - '0';
+          if ((uint32_t)*p - '0' < 10)
+            width = (uint32_t)*p++ - '0' + width*10;
+          sf |= (width << STRFMT_SH_WIDTH);
+        }
+        if (*p == '.') {  /* Parse precision. */
+          uint32_t prec = 0;
+          p++;
+          if ((uint32_t)*p - '0' < 10) {
+            prec = (uint32_t)*p++ - '0';
+            if ((uint32_t)*p - '0' < 10)
+              prec = (uint32_t)*p++ - '0' + prec*10;
+          }
+          sf |= ((prec+1) << STRFMT_SH_PREC);
+        }
+        /* Parse conversion. */
+        c = (uint32_t)*p - 'A';
+        if (LJ_LIKELY(c <= (uint32_t)('x' - 'A'))) {
+          uint32_t sx = strfmt_map[c];
+          if (sx) {
+            fs->p = p+1;
+            return (sf | sx | ((c & 0x20) ? 0 : STRFMT_F_UPPER));
+          }
+        }
+        /* Return error location. */
+        if (*p >= 32) p++;
+        fs->len = (MSize)(p - (const uint8_t *)fs->str);
+        fs->p = fs->e;
+        return STRFMT_ERR;
+      }
+    }
+  }
+  fs->p = p;
+retlit:
+  fs->len = (MSize)(p - (const uint8_t *)fs->str);
+  return fs->len ? STRFMT_LIT : STRFMT_EOF;
+}
+/* -- Raw conversions ----------------------------------------------------- */
+/* Write number to bufer. */
+char * LJ_FASTCALL lj_strfmt_wnum(char *p, cTValue *o)
+{
+  if (LJ_LIKELY((o->u32.hi << 1) < 0xffe00000)) {  /* Finite? */
+#if __BIONIC__
+    if (tvismzero(o)) { *p++ = '-'; *p++ = '0'; return p; }
+#endif
+    return p + lua_number2str(p, o->n);
+  } else if (((o->u32.hi & 0x000fffff) | o->u32.lo) != 0) {
+    *p++ = 'n'; *p++ = 'a'; *p++ = 'n';
+  } else if ((o->u32.hi & 0x80000000) == 0) {
+    *p++ = 'i'; *p++ = 'n'; *p++ = 'f';
+  } else {
+    *p++ = '-'; *p++ = 'i'; *p++ = 'n'; *p++ = 'f';
+  }
+  return p;
+}
+#define WINT_R(x, sh, sc) \
+  { uint32_t d = (x*(((1<<sh)+sc-1)/sc))>>sh; x -= d*sc; *p++ = (char)('0'+d); }
+/* Write integer to buffer. */
+char * LJ_FASTCALL lj_strfmt_wint(char *p, int32_t k)
+{
+  uint32_t u = (uint32_t)k;
+  if (k < 0) { u = (uint32_t)-k; *p++ = '-'; }
+  if (u < 10000) {
+    if (u < 10) goto dig1; if (u < 100) goto dig2; if (u < 1000) goto dig3;
+  } else {
+    uint32_t v = u / 10000; u -= v * 10000;
+    if (v < 10000) {
+      if (v < 10) goto dig5; if (v < 100) goto dig6; if (v < 1000) goto dig7;
+    } else {
+      uint32_t w = v / 10000; v -= w * 10000;
+      if (w >= 10) WINT_R(w, 10, 10)
+      *p++ = (char)('0'+w);
+    }
+    WINT_R(v, 23, 1000)
+    dig7: WINT_R(v, 12, 100)
+    dig6: WINT_R(v, 10, 10)
+    dig5: *p++ = (char)('0'+v);
+  }
+  WINT_R(u, 23, 1000)
+  dig3: WINT_R(u, 12, 100)
+  dig2: WINT_R(u, 10, 10)
+  dig1: *p++ = (char)('0'+u);
+  return p;
+}
+#undef WINT_R
+/* Write pointer to buffer. */
+char * LJ_FASTCALL lj_strfmt_wptr(char *p, const void *v)
+{
+  ptrdiff_t x = (ptrdiff_t)v;
+  MSize i, n = STRFMT_MAXBUF_PTR;
+  if (x == 0) {
+    *p++ = 'N'; *p++ = 'U'; *p++ = 'L'; *p++ = 'L';
+    return p;
+  }
+#if LJ_64
+  /* Shorten output for 64 bit pointers. */
+  n = 2+2*4+((x >> 32) ? 2+2*(lj_fls((uint32_t)(x >> 32))>>3) : 0);
+#endif
+  p[0] = '0';
+  p[1] = 'x';
+  for (i = n-1; i >= 2; i--, x >>= 4)
+    p[i] = "0123456789abcdef"[(x & 15)];
+  return p+n;
+}
+/* Write ULEB128 to buffer. */
+char * LJ_FASTCALL lj_strfmt_wuleb128(char *p, uint32_t v)
+{
+  for (; v >= 0x80; v >>= 7)
+    *p++ = (char)((v & 0x7f) | 0x80);
+  *p++ = (char)v;
+  return p;
+}
+/* Return string or write number to buffer and return pointer to start. */
+const char *lj_strfmt_wstrnum(char *buf, cTValue *o, MSize *lenp)
+{
+  if (tvisstr(o)) {
+    *lenp = strV(o)->len;
+    return strVdata(o);
+  } else if (tvisint(o)) {
+    *lenp = (MSize)(lj_strfmt_wint(buf, intV(o)) - buf);
+    return buf;
+  } else if (tvisnum(o)) {
+    *lenp = (MSize)(lj_strfmt_wnum(buf, o) - buf);
+    return buf;
+  } else {
+    return NULL;
+  }
+}
+/* -- Unformatted conversions to buffer ----------------------------------- */
+/* Add integer to buffer. */
+SBuf * LJ_FASTCALL lj_strfmt_putint(SBuf *sb, int32_t k)
+{
+  setsbufP(sb, lj_strfmt_wint(lj_buf_more(sb, STRFMT_MAXBUF_INT), k));
+  return sb;
+}
+#if LJ_HASJIT
+/* Add number to buffer. */
+SBuf * LJ_FASTCALL lj_strfmt_putnum(SBuf *sb, cTValue *o)
+{
+  setsbufP(sb, lj_strfmt_wnum(lj_buf_more(sb, STRFMT_MAXBUF_NUM), o));
+  return sb;
+}
+#endif
+/* Add quoted string to buffer. */
+SBuf * LJ_FASTCALL lj_strfmt_putquoted(SBuf *sb, GCstr *str)
+{
+  const char *s = strdata(str);
+  MSize len = str->len;
+  lj_buf_putb(sb, '"');
+  while (len--) {
+    uint32_t c = (uint32_t)(uint8_t)*s++;
+    char *p = lj_buf_more(sb, 4);
+    if (c == '"' || c == '\\' || c == '\n') {
+      *p++ = '\\';
+    } else if (lj_char_iscntrl(c)) {  /* This can only be 0-31 or 127. */
+      uint32_t d;
+      *p++ = '\\';
+      if (c >= 100 || lj_char_isdigit((uint8_t)*s)) {
+        *p++ = (char)('0'+(c >= 100)); if (c >= 100) c -= 100;
+        goto tens;
+      } else if (c >= 10) {
+      tens:
+        d = (c * 205) >> 11; c -= d * 10; *p++ = (char)('0'+d);
+      }
+      c += '0';
+    }
+    *p++ = (char)c;
+    setsbufP(sb, p);
+  }
+  lj_buf_putb(sb, '"');
+  return sb;
+}
+/* -- Formatted conversions to buffer ------------------------------------- */
+/* Add formatted char to buffer. */
+SBuf *lj_strfmt_putfchar(SBuf *sb, SFormat sf, int32_t c)
+{
+  MSize width = STRFMT_WIDTH(sf);
+  char *p = lj_buf_more(sb, width > 1 ? width : 1);
+  if ((sf & STRFMT_F_LEFT)) *p++ = (char)c;
+  while (width-- > 1) *p++ = ' ';
+  if (!(sf & STRFMT_F_LEFT)) *p++ = (char)c;
+  setsbufP(sb, p);
+  return sb;
+}
+/* Add formatted string to buffer. */
+SBuf *lj_strfmt_putfstr(SBuf *sb, SFormat sf, GCstr *str)
+{
+  MSize len = str->len <= STRFMT_PREC(sf) ? str->len : STRFMT_PREC(sf);
+  MSize width = STRFMT_WIDTH(sf);
+  char *p = lj_buf_more(sb, width > len ? width : len);
+  if ((sf & STRFMT_F_LEFT)) p = lj_buf_wmem(p, strdata(str), len);
+  while (width-- > len) *p++ = ' ';
+  if (!(sf & STRFMT_F_LEFT)) p = lj_buf_wmem(p, strdata(str), len);
+  setsbufP(sb, p);
+  return sb;
+}
+/* Add formatted signed/unsigned integer to buffer. */
+SBuf *lj_strfmt_putfxint(SBuf *sb, SFormat sf, uint64_t k)
+{
+  char buf[STRFMT_MAXBUF_XINT], *q = buf + sizeof(buf), *p;
+#ifdef LUA_USE_ASSERT
+  char *ps;
+#endif
+  MSize prefix = 0, len, prec, pprec, width, need;
+  /* Figure out signed prefixes. */
+  if (STRFMT_TYPE(sf) == STRFMT_INT) {
+    if ((int64_t)k < 0) {
+      k = (uint64_t)-(int64_t)k;
+      prefix = 256 + '-';
+    } else if ((sf & STRFMT_F_PLUS)) {
+      prefix = 256 + '+';
+    } else if ((sf & STRFMT_F_SPACE)) {
+      prefix = 256 + ' ';
+    }
+  }
+  /* Convert number and store to fixed-size buffer in reverse order. */
+  prec = STRFMT_PREC(sf);
+  if ((int32_t)prec >= 0) sf &= ~STRFMT_F_ZERO;
+  if (k == 0) {  /* Special-case zero argument. */
+    if (prec != 0 ||
+        (sf & (STRFMT_T_OCT|STRFMT_F_ALT)) == (STRFMT_T_OCT|STRFMT_F_ALT))
+      *--q = '0';
+  } else if (!(sf & (STRFMT_T_HEX|STRFMT_T_OCT))) {  /* Decimal. */
+    uint32_t k2;
+    while ((k >> 32)) { *--q = (char)('0' + k % 10); k /= 10; }
+    k2 = (uint32_t)k;
+    do { *--q = (char)('0' + k2 % 10); k2 /= 10; } while (k2);
+  } else if ((sf & STRFMT_T_HEX)) {  /* Hex. */
+    const char *hexdig = (sf & STRFMT_F_UPPER) ? "0123456789ABCDEF" :
+                                                 "0123456789abcdef";
+    do { *--q = hexdig[(k & 15)]; k >>= 4; } while (k);
+    if ((sf & STRFMT_F_ALT)) prefix = 512 + 'x';
+  } else {  /* Octal. */
+    do { *--q = (char)('0' + (uint32_t)(k & 7)); k >>= 3; } while (k);
+    if ((sf & STRFMT_F_ALT)) *--q = '0';
+  }
+  /* Calculate sizes. */
+  len = (MSize)(buf + sizeof(buf) - q);
+  if ((int32_t)len >= (int32_t)prec) prec = len;
+  width = STRFMT_WIDTH(sf);
+  pprec = prec + (prefix >> 8);
+  need = width > pprec ? width : pprec;
+  p = lj_buf_more(sb, need);
+#ifdef LUA_USE_ASSERT
+  ps = p;
+#endif
+  /* Format number with leading/trailing whitespace and zeros. */
+  if ((sf & (STRFMT_F_LEFT|STRFMT_F_ZERO)) == 0)
+    while (width-- > pprec) *p++ = ' ';
+  if (prefix) {
+    if ((char)prefix == 'x') *p++ = '0';
+    *p++ = (char)prefix;
+  }
+  if ((sf & (STRFMT_F_LEFT|STRFMT_F_ZERO)) == STRFMT_F_ZERO)
+    while (width-- > pprec) *p++ = '0';
+  while (prec-- > len) *p++ = '0';
+  while (q < buf + sizeof(buf)) *p++ = *q++;  /* Add number itself. */
+  if ((sf & STRFMT_F_LEFT))
+    while (width-- > pprec) *p++ = ' ';
+  lua_assert(need == (MSize)(p - ps));
+  setsbufP(sb, p);
+  return sb;
+}
+/* Add number formatted as signed integer to buffer. */
+SBuf *lj_strfmt_putfnum_int(SBuf *sb, SFormat sf, lua_Number n)
+{
+  int64_t k = (int64_t)n;
+  if (checki32(k) && sf == STRFMT_INT)
+    return lj_strfmt_putint(sb, (int32_t)k);  /* Shortcut for plain %d. */
+  else
+    return lj_strfmt_putfxint(sb, sf, (uint64_t)k);
+}
+/* Add number formatted as unsigned integer to buffer. */
+SBuf *lj_strfmt_putfnum_uint(SBuf *sb, SFormat sf, lua_Number n)
+{
+  int64_t k;
+  if (n >= 9223372036854775808.0)
+    k = (int64_t)(n - 18446744073709551616.0);
+  else
+    k = (int64_t)n;
+  return lj_strfmt_putfxint(sb, sf, (uint64_t)k);
+}
+/* Max. sprintf buffer size needed. At least #string.format("%.99f", -1e308). */
+#define STRFMT_FMTNUMBUF        512
+/* Add formatted floating-point number to buffer. */
+SBuf *lj_strfmt_putfnum(SBuf *sb, SFormat sf, lua_Number n)
+{
+  TValue tv;
+  tv.n = n;
+  if (LJ_UNLIKELY((tv.u32.hi << 1) >= 0xffe00000)) {
+    /* Canonicalize output of non-finite values. */
+    MSize width = STRFMT_WIDTH(sf), len = 3;
+    int prefix = 0, ch = (sf & STRFMT_F_UPPER) ? 0x202020 : 0;
+    char *p;
+    if (((tv.u32.hi & 0x000fffff) | tv.u32.lo) != 0) {
+      ch ^= ('n' << 16) | ('a' << 8) | 'n';
+      if ((sf & STRFMT_F_SPACE)) prefix = ' ';
+    } else {
+      ch ^= ('i' << 16) | ('n' << 8) | 'f';
+      if ((tv.u32.hi & 0x80000000)) prefix = '-';
+      else if ((sf & STRFMT_F_PLUS)) prefix = '+';
+      else if ((sf & STRFMT_F_SPACE)) prefix = ' ';
+    }
+    if (prefix) len = 4;
+    p = lj_buf_more(sb, width > len ? width : len);
+    if (!(sf & STRFMT_F_LEFT)) while (width-- > len) *p++ = ' ';
+    if (prefix) *p++ = prefix;
+    *p++ = (char)(ch >> 16); *p++ = (char)(ch >> 8); *p++ = (char)ch;
+    if ((sf & STRFMT_F_LEFT)) while (width-- > len) *p++ = ' ';
+    setsbufP(sb, p);
+  } else {  /* Delegate to sprintf() for now. */
+    uint8_t width = (uint8_t)STRFMT_WIDTH(sf), prec = (uint8_t)STRFMT_PREC(sf);
+    char fmt[1+5+2+3+1+1], *p = fmt;
+    *p++ = '%';
+    if ((sf & STRFMT_F_LEFT)) *p++ = '-';
+    if ((sf & STRFMT_F_PLUS)) *p++ = '+';
+    if ((sf & STRFMT_F_ZERO)) *p++ = '0';
+    if ((sf & STRFMT_F_SPACE)) *p++ = ' ';
+    if ((sf & STRFMT_F_ALT)) *p++ = '#';
+    if (width) {
+      uint8_t x = width / 10, y = width % 10;
+      if (x) *p++ = '0' + x;
+      *p++ = '0' + y;
+    }
+    if (prec != 255) {
+      uint8_t x = prec / 10, y = prec % 10;
+      *p++ = '.';
+      if (x) *p++ = '0' + x;
+      *p++ = '0' + y;
+    }
+    *p++ = (0x67666561 >> (STRFMT_FP(sf)<<3)) ^ ((sf & STRFMT_F_UPPER)?0x20:0);
+    *p = '\0';
+    p = lj_buf_more(sb, STRFMT_FMTNUMBUF);
+    setsbufP(sb, p + sprintf(p, fmt, n));
+  }
+  return sb;
+}
+/* -- Conversions to strings ---------------------------------------------- */
+/* Convert integer to string. */
+GCstr * LJ_FASTCALL lj_strfmt_int(lua_State *L, int32_t k)
+{
+  char buf[STRFMT_MAXBUF_INT];
+  MSize len = (MSize)(lj_strfmt_wint(buf, k) - buf);
+  return lj_str_new(L, buf, len);
+}
+/* Convert number to string. */
+GCstr * LJ_FASTCALL lj_strfmt_num(lua_State *L, cTValue *o)
+{
+  char buf[STRFMT_MAXBUF_NUM];
+  MSize len = (MSize)(lj_strfmt_wnum(buf, o) - buf);
+  return lj_str_new(L, buf, len);
+}
+/* Convert integer or number to string. */
+GCstr * LJ_FASTCALL lj_strfmt_number(lua_State *L, cTValue *o)
+{
+  return tvisint(o) ? lj_strfmt_int(L, intV(o)) : lj_strfmt_num(L, o);
+}
+#if LJ_HASJIT
+/* Convert char value to string. */
+GCstr * LJ_FASTCALL lj_strfmt_char(lua_State *L, int c)
+{
+  char buf[1];
+  buf[0] = c;
+  return lj_str_new(L, buf, 1);
+}
+#endif
+/* Raw conversion of object to string. */
+GCstr * LJ_FASTCALL lj_strfmt_obj(lua_State *L, cTValue *o)
+{
+  if (tvisstr(o)) {
+    return strV(o);
+  } else if (tvisnumber(o)) {
+    return lj_strfmt_number(L, o);
+  } else if (tvisnil(o)) {
+    return lj_str_newlit(L, "nil");
+  } else if (tvisfalse(o)) {
+    return lj_str_newlit(L, "false");
+  } else if (tvistrue(o)) {
+    return lj_str_newlit(L, "true");
+  } else {
+    char buf[8+2+2+16], *p = buf;
+    p = lj_buf_wmem(p, lj_typename(o), (MSize)strlen(lj_typename(o)));
+    *p++ = ':'; *p++ = ' ';
+    if (tvisfunc(o) && isffunc(funcV(o))) {
+      p = lj_buf_wmem(p, "builtin#", 8);
+      p = lj_strfmt_wint(p, funcV(o)->c.ffid);
+    } else {
+      p = lj_strfmt_wptr(p, lj_obj_ptr(o));
+    }
+    return lj_str_new(L, buf, (size_t)(p - buf));
+  }
+}
+/* -- Internal string formatting ------------------------------------------ */
+/*
+** These functions are only used for lua_pushfstring(), lua_pushvfstring()
+** and for internal string formatting (e.g. error messages). Caveat: unlike
+** string.format(), only a limited subset of formats and flags are supported!
+**
+** LuaJIT has support for a couple more formats than Lua 5.1/5.2:
+** - %d %u %o %x with full formatting, 32 bit integers only.
+** - %f and other FP formats are really %.14g.
+** - %s %c %p without formatting.
+*/
+/* Push formatted message as a string object to Lua stack. va_list variant. */
+const char *lj_strfmt_pushvf(lua_State *L, const char *fmt, va_list argp)
+{
+  SBuf *sb = lj_buf_tmp_(L);
+  FormatState fs;
+  SFormat sf;
+  GCstr *str;
+  lj_strfmt_init(&fs, fmt, (MSize)strlen(fmt));
+  while ((sf = lj_strfmt_parse(&fs)) != STRFMT_EOF) {
+    switch (STRFMT_TYPE(sf)) {
+    case STRFMT_LIT:
+      lj_buf_putmem(sb, fs.str, fs.len);
+      break;
+    case STRFMT_INT:
+      lj_strfmt_putfxint(sb, sf, va_arg(argp, int32_t));
+      break;
+    case STRFMT_UINT:
+      lj_strfmt_putfxint(sb, sf, va_arg(argp, uint32_t));
+      break;
+    case STRFMT_NUM: {
+      TValue tv;
+      tv.n = va_arg(argp, lua_Number);
+      setsbufP(sb, lj_strfmt_wnum(lj_buf_more(sb, STRFMT_MAXBUF_NUM), &tv));
+      break;
+      }
+    case STRFMT_STR: {
+      const char *s = va_arg(argp, char *);
+      if (s == NULL) s = "(null)";
+      lj_buf_putmem(sb, s, (MSize)strlen(s));
+      break;
+      }
+    case STRFMT_CHAR:
+      lj_buf_putb(sb, va_arg(argp, int));
+      break;
+    case STRFMT_PTR:
+      setsbufP(sb, lj_strfmt_wptr(lj_buf_more(sb, STRFMT_MAXBUF_PTR),
+                                  va_arg(argp, void *)));
+      break;
+    case STRFMT_ERR:
+    default:
+      lj_buf_putb(sb, '?');
+      lua_assert(0);
+      break;
+    }
+  }
+  str = lj_buf_str(L, sb);
+  setstrV(L, L->top, str);
+  incr_top(L);
+  return strdata(str);
+}
+/* Push formatted message as a string object to Lua stack. Vararg variant. */
+const char *lj_strfmt_pushf(lua_State *L, const char *fmt, ...)
+{
+  const char *msg;
+  va_list argp;
+  va_start(argp, fmt);
+  msg = lj_strfmt_pushvf(L, fmt, argp);
+  va_end(argp);
+  return msg;
+}
diff --git a/src/lj_strfmt.h b/src/lj_strfmt.h
new file mode 100644
index 00000000..5454336f
--- /dev/null
+++ b/src/lj_strfmt.h
@@ -0,0 +1,124 @@
+/*
+** String formatting.
+** Copyright (C) 2005-2013 Mike Pall. See Copyright Notice in luajit.h
+*/
+#ifndef _LJ_STRFMT_H
+#define _LJ_STRFMT_H
+#include "lj_obj.h"
+typedef uint32_t SFormat;  /* Format indicator. */
+/* Format parser state. */
+typedef struct FormatState {
+  const uint8_t *p;     /* Current format string pointer. */
+  const uint8_t *e;     /* End of format string. */
+  const char *str;      /* Returned literal string. */
+  MSize len;            /* Size of literal string. */
+} FormatState;
+/* Format types (max. 16). */
+typedef enum FormatType {
+  STRFMT_EOF, STRFMT_ERR, STRFMT_LIT,
+  STRFMT_INT, STRFMT_UINT, STRFMT_NUM, STRFMT_STR, STRFMT_CHAR, STRFMT_PTR
+} FormatType;
+/* Format subtypes (bits are reused). */
+#define STRFMT_T_HEX    0x0010  /* STRFMT_UINT */
+#define STRFMT_T_OCT    0x0020  /* STRFMT_UINT */
+#define STRFMT_T_FP_A   0x0000  /* STRFMT_NUM */
+#define STRFMT_T_FP_E   0x0010  /* STRFMT_NUM */
+#define STRFMT_T_FP_F   0x0020  /* STRFMT_NUM */
+#define STRFMT_T_FP_G   0x0030  /* STRFMT_NUM */
+#define STRFMT_T_QUOTED 0x0010  /* STRFMT_STR */
+/* Format flags. */
+#define STRFMT_F_LEFT   0x0100
+#define STRFMT_F_PLUS   0x0200
+#define STRFMT_F_ZERO   0x0400
+#define STRFMT_F_SPACE  0x0800
+#define STRFMT_F_ALT    0x1000
+#define STRFMT_F_UPPER  0x2000
+/* Format indicator fields. */
+#define STRFMT_SH_WIDTH 16
+#define STRFMT_SH_PREC  24
+#define STRFMT_TYPE(sf)         ((FormatType)((sf) & 15))
+#define STRFMT_WIDTH(sf)        (((sf) >> STRFMT_SH_WIDTH) & 255u)
+#define STRFMT_PREC(sf)         ((((sf) >> STRFMT_SH_PREC) & 255u) - 1u)
+#define STRFMT_FP(sf)           (((sf) >> 4) & 3)
+/* Formats for conversion characters. */
+#define STRFMT_A        (STRFMT_NUM|STRFMT_T_FP_A)
+#define STRFMT_C        (STRFMT_CHAR)
+#define STRFMT_D        (STRFMT_INT)
+#define STRFMT_E        (STRFMT_NUM|STRFMT_T_FP_E)
+#define STRFMT_F        (STRFMT_NUM|STRFMT_T_FP_F)
+#define STRFMT_G        (STRFMT_NUM|STRFMT_T_FP_G)
+#define STRFMT_I        STRFMT_D
+#define STRFMT_O        (STRFMT_UINT|STRFMT_T_OCT)
+#define STRFMT_P        (STRFMT_PTR)
+#define STRFMT_Q        (STRFMT_STR|STRFMT_T_QUOTED)
+#define STRFMT_S        (STRFMT_STR)
+#define STRFMT_U        (STRFMT_UINT)
+#define STRFMT_X        (STRFMT_UINT|STRFMT_T_HEX)
+/* Maximum buffer sizes for conversions. */
+#define STRFMT_MAXBUF_XINT      (1+22)  /* '0' prefix + uint64_t in octal. */
+#define STRFMT_MAXBUF_INT       (1+10)  /* Sign + int32_t in decimal. */
+#define STRFMT_MAXBUF_NUM       LUAI_MAXNUMBER2STR
+#define STRFMT_MAXBUF_PTR       (2+2*sizeof(ptrdiff_t))  /* "0x" + hex ptr. */
+/* Format parser. */
+LJ_FUNC SFormat LJ_FASTCALL lj_strfmt_parse(FormatState *fs);
+static LJ_AINLINE void lj_strfmt_init(FormatState *fs, const char *p, MSize len)
+{
+  fs->p = (const uint8_t *)p;
+  fs->e = (const uint8_t *)p + len;
+  lua_assert(*fs->e == 0);  /* Must be NUL-terminated (may have NULs inside). */
+}
+/* Raw conversions. */
+LJ_FUNC char * LJ_FASTCALL lj_strfmt_wint(char *p, int32_t k);
+LJ_FUNC char * LJ_FASTCALL lj_strfmt_wnum(char *p, cTValue *o);
+LJ_FUNC char * LJ_FASTCALL lj_strfmt_wptr(char *p, const void *v);
+LJ_FUNC char * LJ_FASTCALL lj_strfmt_wuleb128(char *p, uint32_t v);
+LJ_FUNC const char *lj_strfmt_wstrnum(char *buf, cTValue *o, MSize *lenp);
+/* Unformatted conversions to buffer. */
+LJ_FUNC SBuf * LJ_FASTCALL lj_strfmt_putint(SBuf *sb, int32_t k);
+#if LJ_HASJIT
+LJ_FUNC SBuf * LJ_FASTCALL lj_strfmt_putnum(SBuf *sb, cTValue *o);
+#endif
+LJ_FUNC SBuf * LJ_FASTCALL lj_strfmt_putquoted(SBuf *sb, GCstr *str);
+/* Formatted conversions to buffer. */
+LJ_FUNC SBuf *lj_strfmt_putfxint(SBuf *sb, SFormat sf, uint64_t k);
+LJ_FUNC SBuf *lj_strfmt_putfnum_int(SBuf *sb, SFormat sf, lua_Number n);
+LJ_FUNC SBuf *lj_strfmt_putfnum_uint(SBuf *sb, SFormat sf, lua_Number n);
+LJ_FUNC SBuf *lj_strfmt_putfnum(SBuf *sb, SFormat, lua_Number n);
+LJ_FUNC SBuf *lj_strfmt_putfchar(SBuf *sb, SFormat, int32_t c);
+LJ_FUNC SBuf *lj_strfmt_putfstr(SBuf *sb, SFormat, GCstr *str);
+/* Conversions to strings. */
+LJ_FUNC GCstr * LJ_FASTCALL lj_strfmt_int(lua_State *L, int32_t k);
+LJ_FUNCA GCstr * LJ_FASTCALL lj_strfmt_num(lua_State *L, cTValue *o);
+LJ_FUNCA GCstr * LJ_FASTCALL lj_strfmt_number(lua_State *L, cTValue *o);
+#if LJ_HASJIT
+LJ_FUNC GCstr * LJ_FASTCALL lj_strfmt_char(lua_State *L, int c);
+#endif
+LJ_FUNC GCstr * LJ_FASTCALL lj_strfmt_obj(lua_State *L, cTValue *o);
+/* Internal string formatting. */
+LJ_FUNC const char *lj_strfmt_pushvf(lua_State *L, const char *fmt,
+                                     va_list argp);
+LJ_FUNC const char *lj_strfmt_pushf(lua_State *L, const char *fmt, ...)
+#ifdef __GNUC__
+  __attribute__ ((format (printf, 2, 3)))
+#endif
+  ;
+#endif
diff --git a/src/lj_tab.h b/src/lj_tab.h
index 2787caa0..d361137c 100644
--- a/src/lj_tab.h
+++ b/src/lj_tab.h
@@ -50,7 +50,7 @@ LJ_FUNCA cTValue *lj_tab_get(lua_State *L, GCtab *t, cTValue *key);
 /* Caveat: all setters require a write barrier for the stored value. */
 LJ_FUNCA TValue *lj_tab_newkey(lua_State *L, GCtab *t, cTValue *key);
-LJ_FUNC TValue *lj_tab_setinth(lua_State *L, GCtab *t, int32_t key);
+LJ_FUNCA TValue *lj_tab_setinth(lua_State *L, GCtab *t, int32_t key);
 LJ_FUNC TValue *lj_tab_setstr(lua_State *L, GCtab *t, GCstr *key);
 LJ_FUNC TValue *lj_tab_set(lua_State *L, GCtab *t, cTValue *key);
diff --git a/src/lj_target_arm.h b/src/lj_target_arm.h
index bec55772..f1aedff0 100644
--- a/src/lj_target_arm.h
+++ b/src/lj_target_arm.h
@@ -243,10 +243,6 @@ typedef enum ARMIns {
  ARMI_VCVT_S32_F64 = 0xeebd0bc0,
  ARMI_VCVT_U32_F32 = 0xeebc0ac0,
  ARMI_VCVT_U32_F64 = 0xeebc0bc0,
-  ARMI_VCVTR_S32_F32 = 0xeebd0a40,
-  ARMI_VCVTR_S32_F64 = 0xeebd0b40,
-  ARMI_VCVTR_U32_F32 = 0xeebc0a40,
-  ARMI_VCVTR_U32_F64 = 0xeebc0b40,
  ARMI_VCVT_F32_S32 = 0xeeb80ac0,
  ARMI_VCVT_F64_S32 = 0xeeb80bc0,
  ARMI_VCVT_F32_U32 = 0xeeb80a40,
diff --git a/src/lj_target_mips.h b/src/lj_target_mips.h
index 36f46c83..9bf1d2cb 100644
--- a/src/lj_target_mips.h
+++ b/src/lj_target_mips.h
@@ -169,6 +169,9 @@ typedef enum MIPSIns {
  MIPSI_SLTU = 0x0000002b,
  MIPSI_MOVZ = 0x0000000a,
  MIPSI_MOVN = 0x0000000b,
+  MIPSI_MFHI = 0x00000010,
+  MIPSI_MFLO = 0x00000012,
+  MIPSI_MULT = 0x00000018,
  MIPSI_SLL = 0x00000000,
  MIPSI_SRL = 0x00000002,
diff --git a/src/lj_target_x86.h b/src/lj_target_x86.h
index 84b0871d..38f464fc 100644
--- a/src/lj_target_x86.h
+++ b/src/lj_target_x86.h
@@ -33,6 +33,7 @@ enum {
  RID_MRM = RID_MAX,            /* Pseudo-id for ModRM operand. */
  /* Calling conventions. */
+  RID_SP = RID_ESP,
  RID_RET = RID_EAX,
 #if LJ_64
  RID_FPRET = RID_XMM0,
@@ -277,10 +278,8 @@ typedef enum {
  XO_ROUNDSD =  0x0b3a0ffc,  /* Really 66 0f 3a 0b. See asm_fpmath. */
  XO_UCOMISD =  XO_660f(2e),
  XO_CVTSI2SD = XO_f20f(2a),
-  XO_CVTSD2SI = XO_f20f(2d),
  XO_CVTTSD2SI= XO_f20f(2c),
  XO_CVTSI2SS = XO_f30f(2a),
-  XO_CVTSS2SI = XO_f30f(2d),
  XO_CVTTSS2SI= XO_f30f(2c),
  XO_CVTSS2SD = XO_f30f(5a),
  XO_CVTSD2SS = XO_f20f(5a),
diff --git a/src/lj_vm.h b/src/lj_vm.h
index c5d05de4..948d63c2 100644
--- a/src/lj_vm.h
+++ b/src/lj_vm.h
@@ -49,12 +49,14 @@ LJ_ASMF void lj_vm_exit_handler(void);
 LJ_ASMF void lj_vm_exit_interp(void);
 /* Internal math helper functions. */
-#if LJ_TARGET_X86ORX64 || LJ_TARGET_PPC
+#if LJ_TARGET_PPC
 #define lj_vm_floor     floor
 #define lj_vm_ceil      ceil
 #else
 LJ_ASMF double lj_vm_floor(double);
+#if !LJ_TARGET_X86ORX64
 LJ_ASMF double lj_vm_ceil(double);
+#endif
 #if LJ_TARGET_ARM
 LJ_ASMF double lj_vm_floor_sf(double);
 LJ_ASMF double lj_vm_ceil_sf(double);
diff --git a/src/ljamalg.c b/src/ljamalg.c
index 962b3134..7198a09f 100644
--- a/src/ljamalg.c
+++ b/src/ljamalg.c
@@ -33,6 +33,7 @@
 #include "lj_char.c"
 #include "lj_bc.c"
 #include "lj_obj.c"
+#include "lj_buf.c"
 #include "lj_str.c"
 #include "lj_tab.c"
 #include "lj_func.c"
@@ -44,6 +45,7 @@
 #include "lj_vmevent.c"
 #include "lj_vmmath.c"
 #include "lj_strscan.c"
+#include "lj_strfmt.c"
 #include "lj_api.c"
 #include "lj_lex.c"
 #include "lj_parse.c"
diff --git a/src/luaconf.h b/src/luaconf.h
index 8e3a7aaa..d283233d 100644
--- a/src/luaconf.h
+++ b/src/luaconf.h
@@ -30,12 +30,12 @@
 #define LUA_LDIR        LUA_ROOT "share/lua/5.1/"
 #define LUA_CDIR        LUA_ROOT "lib/lua/5.1/"
 #ifdef LUA_XROOT
-#define LUA_JDIR        LUA_XROOT "share/luajit-2.0.1/"
+#define LUA_JDIR        LUA_XROOT "share/luajit-2.1.0-alpha/"
 #define LUA_XPATH \
  ";" LUA_XROOT "share/lua/5.1/?.lua;" LUA_XROOT "share/lua/5.1/?/init.lua"
 #define LUA_XCPATH      LUA_XROOT "lib/lua/5.1/?.so;"
 #else
-#define LUA_JDIR        LUA_ROOT "share/luajit-2.0.1/"
+#define LUA_JDIR        LUA_ROOT "share/luajit-2.1.0-alpha/"
 #define LUA_XPATH
 #define LUA_XCPATH
 #endif
diff --git a/src/luajit.c b/src/luajit.c
index e0eacc42..680984da 100644
--- a/src/luajit.c
+++ b/src/luajit.c
@@ -61,8 +61,9 @@ static void laction(int i)
 static void print_usage(void)
 {
-  fprintf(stderr,
+  fputs("usage: ", stderr);
-  "usage: %s [options]... [script [args]...].\n"
+  fputs(progname, stderr);
+  fputs(" [options]... [script [args]...].\n"
  "Available options are:\n"
  "  -e chunk  Execute string " LUA_QL("chunk") ".\n"
  "  -l name   Require library " LUA_QL("name") ".\n"
@@ -73,16 +74,14 @@ static void print_usage(void)
  "  -v        Show version information.\n"
  "  -E        Ignore environment variables.\n"
  "  --        Stop handling options.\n"
-  "  -         Execute stdin and stop handling options.\n"
+  "  -         Execute stdin and stop handling options.\n", stderr);
-  ,
-  progname);
  fflush(stderr);
 }
 static void l_message(const char *pname, const char *msg)
 {
-  if (pname) fprintf(stderr, "%s: ", pname);
+  if (pname) { fputs(pname, stderr); fputc(':', stderr); fputc(' ', stderr); }
-  fprintf(stderr, "%s\n", msg);
+  fputs(msg, stderr); fputc('\n', stderr);
  fflush(stderr);
 }
diff --git a/src/luajit.h b/src/luajit.h
index ed39d014..a4c939bf 100644
--- a/src/luajit.h
+++ b/src/luajit.h
@@ -30,9 +30,9 @@
 #include "lua.h"
-#define LUAJIT_VERSION          "LuaJIT 2.0.1"
+#define LUAJIT_VERSION          "LuaJIT 2.1.0-alpha"
-#define LUAJIT_VERSION_NUM      20001  /* Version 2.0.1 = 02.00.01. */
+#define LUAJIT_VERSION_NUM      20100  /* Version 2.1.0 = 02.01.00. */
-#define LUAJIT_VERSION_SYM      luaJIT_version_2_0_1
+#define LUAJIT_VERSION_SYM      luaJIT_version_2_1_0_alpha
 #define LUAJIT_COPYRIGHT        "Copyright (C) 2005-2013 Mike Pall"
 #define LUAJIT_URL              "http://luajit.org/"
diff --git a/src/msvcbuild.bat b/src/msvcbuild.bat
index cdb42a8d..4a3c8e84 100644
--- a/src/msvcbuild.bat
+++ b/src/msvcbuild.bat
@@ -37,6 +37,7 @@ if exist minilua.exe.manifest^
 @if errorlevel 8 goto :X64
 @set DASMFLAGS=-D WIN -D JIT -D FFI
 @set LJARCH=x86
+@set LJCOMPILE=%LJCOMPILE% /arch:SSE2
 :X64
 minilua %DASM% -LN %DASMFLAGS% -o host\buildvm_arch.h vm_x86.dasc
 @if errorlevel 1 goto :BAD
diff --git a/src/vm_arm.dasc b/src/vm_arm.dasc
index 114416a4..83341e41 100644
--- a/src/vm_arm.dasc
+++ b/src/vm_arm.dasc
@@ -99,6 +99,7 @@
 |.type NODE,            Node
 |.type NARGS8,          int
 |.type TRACE,           GCtrace
+|.type SBUF,            SBuf
 |
 |//-----------------------------------------------------------------------
 |
@@ -615,6 +616,16 @@ static void build_subroutines(BuildCtx *ctx)
  |  ldr LFUNC:CARG3, [BASE, FRAME_FUNC]  // Guaranteed to be a function here.
  |  b ->vm_call_dispatch_f
  |
+  |->vmeta_tgetr:
+  |  .IOS mov RC, BASE
+  |  bl extern lj_tab_getinth           // (GCtab *t, int32_t key)
+  |  // Returns cTValue * or NULL.
+  |  .IOS mov BASE, RC
+  |  cmp CRET1, #0
+  |  ldrdne CARG12, [CRET1]
+  |  mvneq CARG2, #~LJ_TNIL
+  |  b ->BC_TGETR_Z
+  |
  |//-----------------------------------------------------------------------
  |
  |->vmeta_tsets1:
@@ -672,6 +683,15 @@ static void build_subroutines(BuildCtx *ctx)
  |  ldr LFUNC:CARG3, [BASE, FRAME_FUNC]  // Guaranteed to be a function here.
  |  b ->vm_call_dispatch_f
  |
+  |->vmeta_tsetr:
+  |  str BASE, L->base
+  |  .IOS mov RC, BASE
+  |  str PC, SAVE_PC
+  |  bl extern lj_tab_setinth  // (lua_State *L, GCtab *t, int32_t key)
+  |  // Returns TValue *.
+  |  .IOS mov BASE, RC
+  |  b ->BC_TSETR_Z
+  |
  |//-- Comparison metamethods ---------------------------------------------
  |
  |->vmeta_comp:
@@ -736,6 +756,17 @@ static void build_subroutines(BuildCtx *ctx)
  |  b <3
  |.endif
  |
+  |->vmeta_istype:
+  |  sub PC, PC, #4
+  |   str BASE, L->base
+  |   mov CARG1, L
+  |   lsr CARG2, RA, #3
+  |   mov CARG3, RC
+  |  str PC, SAVE_PC
+  |  bl extern lj_meta_istype  // (lua_State *L, BCReg ra, BCReg tp)
+  |  .IOS ldr BASE, L->base
+  |  b ->cont_nop
+  |
  |//-- Arithmetic metamethods ---------------------------------------------
  |
  |->vmeta_arith_vn:
@@ -1053,7 +1084,7 @@ static void build_subroutines(BuildCtx *ctx)
  |  ffgccheck
  |  mov CARG1, L
  |  mov CARG2, BASE
-  |  bl extern lj_str_fromnumber        // (lua_State *L, cTValue *o)
+  |  bl extern lj_strfmt_number         // (lua_State *L, cTValue *o)
  |  // Returns GCstr *.
  |  ldr BASE, L->base
  |  mvn CARG2, #~LJ_TSTR
@@ -1501,19 +1532,6 @@ static void build_subroutines(BuildCtx *ctx)
  |  math_extern2 atan2
  |  math_extern2 fmod
  |
-  |->ff_math_deg:
-  |.if FPU
-  |  .ffunc_d math_rad
-  |  vldr d1, CFUNC:CARG3->upvalue[0]
-  |  vmul.f64 d0, d0, d1
-  |  b ->fff_resd
-  |.else
-  |  .ffunc_n math_rad
-  |  ldrd CARG34, CFUNC:CARG3->upvalue[0]
-  |  bl extern __aeabi_dmul
-  |  b ->fff_restv
-  |.endif
-  |
  |.if HFABI
  |  .ffunc math_ldexp
  |  ldr CARG4, [BASE, #4]
@@ -1688,12 +1706,6 @@ static void build_subroutines(BuildCtx *ctx)
  |
  |//-- String library -----------------------------------------------------
  |
-  |.ffunc_1 string_len
-  |  checkstr CARG2, ->fff_fallback
-  |  ldr CARG1, STR:CARG1->len
-  |  mvn CARG2, #~LJ_TISNUM
-  |  b ->fff_restv
-  |
  |.ffunc string_byte                   // Only handle the 1-arg case here.
  |  ldrd CARG12, [BASE]
  |    ldr PC, [BASE, FRAME_PC]
@@ -1726,6 +1738,7 @@ static void build_subroutines(BuildCtx *ctx)
  |  mov CARG1, L
  |   str PC, SAVE_PC
  |  bl extern lj_str_new               // (lua_State *L, char *str, size_t l)
+  |->fff_resstr:
  |  // Returns GCstr *.
  |  ldr BASE, L->base
  |   mvn CARG2, #~LJ_TSTR
@@ -1769,91 +1782,28 @@ static void build_subroutines(BuildCtx *ctx)
  |  mvn CARG2, #~LJ_TSTR
  |  b ->fff_restv
  |
-  |.ffunc string_rep                    // Only handle the 1-char case inline.
+  |.macro ffstring_op, name
-  |  ffgccheck
+  |  .ffunc string_ .. name
-  |  ldrd CARG12, [BASE]
-  |   ldrd CARG34, [BASE, #8]
-  |    cmp NARGS8:RC, #16
-  |    bne ->fff_fallback               // Exactly 2 arguments
-  |  checktp CARG2, LJ_TSTR
-  |   checktpeq CARG4, LJ_TISNUM
-  |   bne ->fff_fallback
-  |  subs CARG4, CARG3, #1
-  |   ldr CARG2, STR:CARG1->len
-  |  blt ->fff_emptystr                 // Count <= 0?
-  |   cmp CARG2, #1
-  |   blo ->fff_emptystr                // Zero-length string?
-  |   bne ->fff_fallback                // Fallback for > 1-char strings.
-  |  ldr RB, [DISPATCH, #DISPATCH_GL(tmpbuf.sz)]
-  |   ldr CARG2, [DISPATCH, #DISPATCH_GL(tmpbuf.buf)]
-  |   ldr CARG1, STR:CARG1[1]
-  |  cmp RB, CARG3
-  |  blo ->fff_fallback
-  |1:  // Fill buffer with char.
-  |   strb CARG1, [CARG2, CARG4]
-  |  subs CARG4, CARG4, #1
-  |  bge <1
-  |  b ->fff_newstr
-  |
-  |.ffunc string_reverse
-  |  ffgccheck
-  |  ldrd CARG12, [BASE]
-  |   cmp NARGS8:RC, #8
-  |   blo ->fff_fallback
-  |  checkstr CARG2, ->fff_fallback
-  |  ldr CARG3, STR:CARG1->len
-  |   ldr RB, [DISPATCH, #DISPATCH_GL(tmpbuf.sz)]
-  |    ldr CARG2, [DISPATCH, #DISPATCH_GL(tmpbuf.buf)]
-  |  mov CARG4, CARG3
-  |  add CARG1, STR:CARG1, #sizeof(GCstr)
-  |   cmp RB, CARG3
-  |   blo ->fff_fallback
-  |1:  // Reverse string copy.
-  |  ldrb RB, [CARG1], #1
-  |   subs CARG4, CARG4, #1
-  |   blt ->fff_newstr
-  |  strb RB, [CARG2, CARG4]
-  |  b <1
-  |
-  |.macro ffstring_case, name, lo
-  |  .ffunc name
  |  ffgccheck
-  |  ldrd CARG12, [BASE]
+  |  ldr CARG3, [BASE, #4]
  |   cmp NARGS8:RC, #8
+  |  ldr STR:CARG2, [BASE]
  |   blo ->fff_fallback
-  |  checkstr CARG2, ->fff_fallback
+  |  sub SBUF:CARG1, DISPATCH, #-DISPATCH_GL(tmpbuf)
-  |  ldr CARG3, STR:CARG1->len
+  |  checkstr CARG3, ->fff_fallback
-  |   ldr RB, [DISPATCH, #DISPATCH_GL(tmpbuf.sz)]
+  |  ldr CARG4, SBUF:CARG1->b
-  |    ldr CARG2, [DISPATCH, #DISPATCH_GL(tmpbuf.buf)]
+  |   str BASE, L->base
-  |  mov CARG4, #0
+  |   str PC, SAVE_PC
-  |  add CARG1, STR:CARG1, #sizeof(GCstr)
+  |   str L, SBUF:CARG1->L
-  |   cmp RB, CARG3
+  |  str CARG4, SBUF:CARG1->p
-  |   blo ->fff_fallback
+  |  bl extern lj_buf_putstr_ .. name
-  |1:  // ASCII case conversion.
+  |  bl extern lj_buf_tostr
-  |  ldrb RB, [CARG1, CARG4]
+  |  b ->fff_resstr
-  |   cmp CARG4, CARG3
-  |   bhs ->fff_newstr
-  |  sub RC, RB, #lo
-  |  cmp RC, #26
-  |  eorlo RB, RB, #0x20
-  |  strb RB, [CARG2, CARG4]
-  |   add CARG4, CARG4, #1
-  |  b <1
  |.endmacro
  |
-  |ffstring_case string_lower, 65
+  |ffstring_op reverse
-  |ffstring_case string_upper, 97
+  |ffstring_op lower
-  |
+  |ffstring_op upper
-  |//-- Table library ------------------------------------------------------
-  |
-  |.ffunc_1 table_getn
-  |  checktab CARG2, ->fff_fallback
-  |  .IOS mov RA, BASE
-  |  bl extern lj_tab_len               // (GCtab *t)
-  |  // Returns uint32_t (but less than 2^31).
-  |  .IOS mov BASE, RA
-  |  mvn CARG2, #~LJ_TISNUM
-  |  b ->fff_restv
  |
  |//-- Bit library --------------------------------------------------------
  |
@@ -2834,6 +2784,25 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
    |  ins_next
    break;
+  case BC_ISTYPE:
+    |  // RA = src*8, RC = -type
+    |  ldrd CARG12, [BASE, RA]
+    |   ins_next1
+    |  cmn CARG2, RC
+    |   ins_next2
+    |  bne ->vmeta_istype
+    |   ins_next3
+    break;
+  case BC_ISNUM:
+    |  // RA = src*8, RC = -(TISNUM-1)
+    |  ldrd CARG12, [BASE, RA]
+    |   ins_next1
+    |  checktp CARG2, LJ_TISNUM
+    |   ins_next2
+    |  bhs ->vmeta_istype
+    |   ins_next3
+    break;
  /* -- Unary ops --------------------------------------------------------- */
  case BC_MOV:
@@ -3504,6 +3473,24 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
    |  bne <1                           // 'no __index' flag set: done.
    |  b ->vmeta_tgetb
    break;
+  case BC_TGETR:
+    |  decode_RB8 RB, INS
+    |   decode_RC8 RC, INS
+    |  // RA = dst*8, RB = table*8, RC = key*8
+    |  ldr TAB:CARG1, [BASE, RB]
+    |   ldr CARG2, [BASE, RC]
+    |  ldr CARG4, TAB:CARG1->array
+    |    ldr CARG3, TAB:CARG1->asize
+    |  add CARG4, CARG4, CARG2, lsl #3
+    |    cmp CARG2, CARG3               // In array part?
+    |    bhs ->vmeta_tgetr
+    |  ldrd CARG12, [CARG4]
+    |->BC_TGETR_Z:
+    |   ins_next1
+    |   ins_next2
+    |  strd CARG12, [BASE, RA]
+    |   ins_next3
+    break;
  case BC_TSETV:
    |  decode_RB8 RB, INS
@@ -3674,6 +3661,32 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
    |  barrierback TAB:CARG1, INS, CARG3
    |  b <2
    break;
+  case BC_TSETR:
+    |  decode_RB8 RB, INS
+    |   decode_RC8 RC, INS
+    |  // RA = dst*8, RB = table*8, RC = key*8
+    |  ldr TAB:CARG2, [BASE, RB]
+    |   ldr CARG3, [BASE, RC]
+    |     ldrb INS, TAB:CARG2->marked
+    |  ldr CARG1, TAB:CARG2->array
+    |    ldr CARG4, TAB:CARG2->asize
+    |     tst INS, #LJ_GC_BLACK         // isblack(table)
+    |  add CARG1, CARG1, CARG3, lsl #3
+    |     bne >7
+    |2:
+    |    cmp CARG3, CARG4               // In array part?
+    |    bhs ->vmeta_tsetr
+    |->BC_TSETR_Z:
+    |  ldrd CARG34, [BASE, RA]
+    |   ins_next1
+    |   ins_next2
+    |  strd CARG34, [CARG1]
+    |   ins_next3
+    |
+    |7:  // Possible table write barrier for the value. Skip valiswhite check.
+    |  barrierback TAB:CARG2, INS, RB
+    |  b <2
+    break;
  case BC_TSETM:
    |  // RA = base*8 (table at base-1), RC = num_const (start index)
@@ -4272,6 +4285,7 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
    |  ldr RA, TRACE:RC->mcode
    |   str BASE, [DISPATCH, #DISPATCH_GL(jit_base)]
    |   str L, [DISPATCH, #DISPATCH_GL(jit_L)]
+    |   str L, [DISPATCH, #DISPATCH_GL(tmpbuf.L)]
    |  bx RA
    |.endif
    break;
diff --git a/src/vm_mips.dasc b/src/vm_mips.dasc
index a81dbeeb..41ed303d 100644
--- a/src/vm_mips.dasc
+++ b/src/vm_mips.dasc
@@ -138,6 +138,7 @@
 |.type NODE,            Node
 |.type NARGS8,          int
 |.type TRACE,           GCtrace
+|.type SBUF,            SBuf
 |
 |//-----------------------------------------------------------------------
 |
@@ -688,6 +689,16 @@ static void build_subroutines(BuildCtx *ctx)
  |  b ->vm_call_dispatch_f
  |.  li NARGS8:RC, 16                  // 2 args for func(t, k).
  |
+  |->vmeta_tgetr:
+  |  load_got lj_tab_getinth
+  |  call_intern lj_tab_getinth         // (GCtab *t, int32_t key)
+  |.  nop
+  |  // Returns cTValue * or NULL.
+  |  beqz CRET1, >1
+  |.  nop
+  |  b ->BC_TGETR_Z
+  |.  ldc1 f0, 0(CRET1)
+  |
  |//-----------------------------------------------------------------------
  |
  |->vmeta_tsets1:
@@ -740,6 +751,16 @@ static void build_subroutines(BuildCtx *ctx)
  |  b ->vm_call_dispatch_f
  |.  li NARGS8:RC, 24                  // 3 args for func(t, k, v)
  |
+  |->vmeta_tsetr:
+  |  load_got lj_tab_setinth
+  |  sw BASE, L->base
+  |  sw PC, SAVE_PC
+  |  call_intern lj_tab_setinth  // (lua_State *L, GCtab *t, int32_t key)
+  |.  move CARG1, L
+  |  // Returns TValue *.
+  |  b ->BC_TSETR_Z
+  |.  nop
+  |
  |//-- Comparison metamethods ---------------------------------------------
  |
  |->vmeta_comp:
@@ -813,6 +834,18 @@ static void build_subroutines(BuildCtx *ctx)
  |.  nop
  |.endif
  |
+  |->vmeta_istype:
+  |  load_got lj_meta_istype
+  |  addiu PC, PC, -4
+  |   sw BASE, L->base
+  |   srl CARG2, RA, 3
+  |   srl CARG3, RD, 3
+  |  sw PC, SAVE_PC
+  |  call_intern lj_meta_istype  // (lua_State *L, BCReg ra, BCReg tp)
+  |.  move CARG1, L
+  |  b ->cont_nop
+  |.  nop
+  |
  |//-- Arithmetic metamethods ---------------------------------------------
  |
  |->vmeta_unm:
@@ -1119,9 +1152,9 @@ static void build_subroutines(BuildCtx *ctx)
  |.  sw BASE, L->base                  // Add frame since C call can throw.
  |  ffgccheck
  |.  sw PC, SAVE_PC                    // Redundant (but a defined value).
-  |  load_got lj_str_fromnum
+  |  load_got lj_strfmt_num
  |  move CARG1, L
-  |  call_intern lj_str_fromnum         // (lua_State *L, lua_Number *np)
+  |  call_intern lj_strfmt_num          // (lua_State *L, lua_Number *np)
  |.  move CARG2, BASE
  |  // Returns GCstr *.
  |  li CARG3, LJ_TSTR
@@ -1188,7 +1221,7 @@ static void build_subroutines(BuildCtx *ctx)
  |   mtc1 TMP0, FARG1
  |  beqz AT, ->fff_fallback
  |.  lw PC, FRAME_PC(BASE)
-  |   cvt.w.d FRET1, FARG2
+  |   trunc.w.d FRET1, FARG2
  |  cvt.d.w FARG1, FARG1
  |   lw TMP0, TAB:CARG1->asize
  |   lw TMP1, TAB:CARG1->array
@@ -1521,14 +1554,8 @@ static void build_subroutines(BuildCtx *ctx)
  |  b ->fff_resn
  |.  nop
  |
-  |->ff_math_deg:
-  |.ffunc_n math_rad
-  |.  ldc1 FARG2, CFUNC:RB->upvalue[0]
-  |  b ->fff_resn
-  |.  mul.d FRET1, FARG1, FARG2
-  |
  |.ffunc_nn math_ldexp
-  |  cvt.w.d FARG2, FARG2
+  |  trunc.w.d FARG2, FARG2
  |  load_got ldexp
  |  mfc1 CARG3, FARG2
  |  call_extern
@@ -1592,13 +1619,6 @@ static void build_subroutines(BuildCtx *ctx)
  |
  |//-- String library -----------------------------------------------------
  |
-  |.ffunc_1 string_len
-  |  li AT, LJ_TSTR
-  |  bne CARG3, AT, ->fff_fallback
-  |.  nop
-  |  b ->fff_resi
-  |.  lw CRET1, STR:CARG1->len
-  |
  |.ffunc string_byte                   // Only handle the 1-arg case here.
  |  lw CARG3, HI(BASE)
  |   lw STR:CARG1, LO(BASE)
@@ -1628,7 +1648,7 @@ static void build_subroutines(BuildCtx *ctx)
  |.  sltiu AT, CARG3, LJ_TISNUM
  |  beqz AT, ->fff_fallback
  |.  li CARG3, 1
-  |   cvt.w.d FARG1, FARG1
+  |   trunc.w.d FARG1, FARG1
  |  addiu CARG2, sp, ARG5_OFS
  |  sltiu AT, TMP0, 256
  |   mfc1 TMP0, FARG1
@@ -1642,6 +1662,7 @@ static void build_subroutines(BuildCtx *ctx)
  |.  move CARG1, L
  |  // Returns GCstr *.
  |  lw BASE, L->base
+  |->fff_resstr:
  |  move CARG1, CRET1
  |  b ->fff_restv
  |.  li CARG3, LJ_TSTR
@@ -1658,7 +1679,7 @@ static void build_subroutines(BuildCtx *ctx)
  |    ldc1 f2, 8(BASE)
  |  beqz AT, >1
  |.  li CARG4, -1
-  |   cvt.w.d f0, f0
+  |   trunc.w.d f0, f0
  |  sltiu AT, CARG3, LJ_TISNUM
  |  beqz AT, ->fff_fallback
  |.  mfc1 CARG4, f0
@@ -1666,7 +1687,7 @@ static void build_subroutines(BuildCtx *ctx)
  |  sltiu AT, CARG2, LJ_TISNUM
  |  beqz AT, ->fff_fallback
  |.  li AT, LJ_TSTR
-  |  cvt.w.d f2, f2
+  |  trunc.w.d f2, f2
  |  bne TMP0, AT, ->fff_fallback
  |.  lw CARG2, STR:CARG1->len
  |  mfc1 CARG3, f2
@@ -1695,108 +1716,32 @@ static void build_subroutines(BuildCtx *ctx)
  |  b ->fff_restv
  |.  li CARG3, LJ_TSTR
  |
-  |.ffunc string_rep                    // Only handle the 1-char case inline.
+  |.macro ffstring_op, name
-  |  ffgccheck
+  |  .ffunc string_ .. name
-  |  lw TMP0, HI(BASE)
-  |   addiu AT, NARGS8:RC, -16          // Exactly 2 arguments.
-  |  lw CARG4, 8+HI(BASE)
-  |   lw STR:CARG1, LO(BASE)
-  |  addiu TMP0, TMP0, -LJ_TSTR
-  |   ldc1 f0, 8(BASE)
-  |  or AT, AT, TMP0
-  |  bnez AT, ->fff_fallback
-  |.  sltiu AT, CARG4, LJ_TISNUM
-  |   cvt.w.d f0, f0
-  |  beqz AT, ->fff_fallback
-  |.  lw TMP0, STR:CARG1->len
-  |   mfc1 CARG3, f0
-  |  lw TMP1, DISPATCH_GL(tmpbuf.sz)(DISPATCH)
-  |  li AT, 1
-  |   blez CARG3, ->fff_emptystr        // Count <= 0?
-  |.   sltu AT, AT, TMP0
-  |  beqz TMP0, ->fff_emptystr          // Zero length string?
-  |.  sltu TMP0, TMP1, CARG3
-  |  or AT, AT, TMP0
-  |   lw CARG2, DISPATCH_GL(tmpbuf.buf)(DISPATCH)
-  |  bnez AT, ->fff_fallback            // Fallback for > 1-char strings.
-  |.  lbu TMP0, STR:CARG1[1]
-  |  addu TMP2, CARG2, CARG3
-  |1:  // Fill buffer with char. Yes, this is suboptimal code (do you care?).
-  |  addiu TMP2, TMP2, -1
-  |  sltu AT, CARG2, TMP2
-  |  bnez AT, <1
-  |.  sb TMP0, 0(TMP2)
-  |  b ->fff_newstr
-  |.  nop
-  |
-  |.ffunc string_reverse
-  |  ffgccheck
-  |  lw CARG3, HI(BASE)
-  |   lw STR:CARG1, LO(BASE)
-  |  beqz NARGS8:RC, ->fff_fallback
-  |.  li AT, LJ_TSTR
-  |  bne CARG3, AT, ->fff_fallback
-  |.  lw TMP1, DISPATCH_GL(tmpbuf.sz)(DISPATCH)
-  |  lw CARG3, STR:CARG1->len
-  |   addiu CARG1, STR:CARG1, #STR
-  |   lw CARG2, DISPATCH_GL(tmpbuf.buf)(DISPATCH)
-  |  sltu AT, TMP1, CARG3
-  |  bnez AT, ->fff_fallback
-  |.  addu TMP3, CARG1, CARG3
-  |  addu CARG4, CARG2, CARG3
-  |1:  // Reverse string copy.
-  |   lbu TMP1, 0(CARG1)
-  |  sltu AT, CARG1, TMP3
-  |  beqz AT, ->fff_newstr
-  |.  addiu CARG1, CARG1, 1
-  |  addiu CARG4, CARG4, -1
-  |  b <1
-  |   sb TMP1, 0(CARG4)
-  |
-  |.macro ffstring_case, name, lo
-  |  .ffunc name
  |  ffgccheck
  |  lw CARG3, HI(BASE)
-  |   lw STR:CARG1, LO(BASE)
+  |   lw STR:CARG2, LO(BASE)
  |  beqz NARGS8:RC, ->fff_fallback
  |.  li AT, LJ_TSTR
  |  bne CARG3, AT, ->fff_fallback
-  |.  lw TMP1, DISPATCH_GL(tmpbuf.sz)(DISPATCH)
+  |.  addiu SBUF:CARG1, DISPATCH, DISPATCH_GL(tmpbuf)
-  |  lw CARG3, STR:CARG1->len
+  |  load_got lj_buf_putstr_ .. name
-  |   addiu CARG1, STR:CARG1, #STR
+  |  lw TMP0, SBUF:CARG1->b
-  |   lw CARG2, DISPATCH_GL(tmpbuf.buf)(DISPATCH)
+  |   sw L, SBUF:CARG1->L
-  |  sltu AT, TMP1, CARG3
+  |   sw BASE, L->base
-  |  bnez AT, ->fff_fallback
+  |  sw TMP0, SBUF:CARG1->p
-  |.  addu TMP3, CARG1, CARG3
+  |  call_intern extern lj_buf_putstr_ .. name
-  |  move CARG4, CARG2
+  |.  sw PC, SAVE_PC
-  |1:  // ASCII case conversion.
+  |  load_got lj_buf_tostr
-  |   lbu TMP1, 0(CARG1)
+  |  call_intern lj_buf_tostr
-  |  sltu AT, CARG1, TMP3
+  |.  move SBUF:CARG1, SBUF:CRET1
-  |  beqz AT, ->fff_newstr
+  |  b ->fff_resstr
-  |.  addiu TMP0, TMP1, -lo
+  |.  lw BASE, L->base
-  |   xori TMP2, TMP1, 0x20
-  |   sltiu AT, TMP0, 26
-  |   movn TMP1, TMP2, AT
-  |  addiu CARG1, CARG1, 1
-  |   sb TMP1, 0(CARG4)
-  |  b <1
-  |.  addiu CARG4, CARG4, 1
  |.endmacro
  |
-  |ffstring_case string_lower, 65
+  |ffstring_op reverse
-  |ffstring_case string_upper, 97
+  |ffstring_op lower
-  |
+  |ffstring_op upper
-  |//-- Table library ------------------------------------------------------
-  |
-  |.ffunc_1 table_getn
-  |  li AT, LJ_TTAB
-  |  bne CARG3, AT, ->fff_fallback
-  |.  load_got lj_tab_len
-  |  call_intern lj_tab_len             // (GCtab *t)
-  |.  nop
-  |  // Returns uint32_t (but less than 2^31).
-  |  b ->fff_resi
-  |.  nop
  |
  |//-- Bit library --------------------------------------------------------
  |
@@ -2572,6 +2517,26 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
    |  ins_next
    break;
+  case BC_ISTYPE:
+    |  // RA = src*8, RD = -type*8
+    |  addu TMP2, BASE, RA
+    |  srl TMP1, RD, 3
+    |  lw TMP0, HI(TMP2)
+    |  ins_next1
+    |  addu AT, TMP0, TMP1
+    |  bnez AT, ->vmeta_istype
+    |.  ins_next2
+    break;
+  case BC_ISNUM:
+    |  // RA = src*8, RD = -(TISNUM-1)*8
+    |  addu TMP2, BASE, RA
+    |  lw TMP0, HI(TMP2)
+    |  ins_next1
+    |  sltiu AT, TMP0, LJ_TISNUM
+    |  beqz AT, ->vmeta_istype
+    |.  ins_next2
+    break;
  /* -- Unary ops --------------------------------------------------------- */
  case BC_MOV:
@@ -3210,6 +3175,30 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
    |  b ->vmeta_tgetb                  // Caveat: preserve TMP0!
    |.  nop
    break;
+  case BC_TGETR:
+    |  // RA = dst*8, RB = table*8, RC = key*8
+    |  decode_RB8a RB, INS
+    |  decode_RB8b RB
+    |   decode_RDtoRC8 RC, RD
+    |  addu CARG2, BASE, RB
+    |   addu CARG3, BASE, RC
+    |    lw TAB:CARG1, LO(CARG2)
+    |   ldc1 f0, 0(CARG3)
+    |  trunc.w.d f2, f0
+    |   lw TMP0, TAB:CARG1->asize
+    |  mfc1 CARG2, f2
+    |   lw TMP1, TAB:CARG1->array
+    |  sltu AT, CARG2, TMP0
+    |   sll TMP2, CARG2, 3
+    |  beqz AT, ->vmeta_tgetr           // In array part?
+    |.  addu TMP2, TMP1, TMP2
+    |   ldc1 f0, 0(TMP2)
+    |->BC_TGETR_Z:
+    |   addu RA, BASE, RA
+    |  ins_next1
+    |   sdc1 f0, 0(RA)
+    |  ins_next2
+    break;
  case BC_TSETV:
    |  // RA = src*8, RB = table*8, RC = key*8
@@ -3398,6 +3387,38 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
    |7:  // Possible table write barrier for the value. Skip valiswhite check.
    |  barrierback TAB:RB, TMP3, TMP0, <2
    break;
+  case BC_TSETR:
+    |  // RA = dst*8, RB = table*8, RC = key*8
+    |  decode_RB8a RB, INS
+    |  decode_RB8b RB
+    |   decode_RDtoRC8 RC, RD
+    |  addu CARG1, BASE, RB
+    |   addu CARG3, BASE, RC
+    |    lw TAB:CARG2, LO(CARG1)
+    |   ldc1 f0, 0(CARG3)
+    |  trunc.w.d f2, f0
+    |    lbu TMP3, TAB:CARG2->marked
+    |   lw TMP0, TAB:CARG2->asize
+    |  mfc1 CARG3, f2
+    |   lw TMP1, TAB:CARG2->array
+    |  andi AT, TMP3, LJ_GC_BLACK       // isblack(table)
+    |  bnez AT, >7
+    |.  addu RA, BASE, RA
+    |2:
+    |  sltu AT, CARG3, TMP0
+    |   sll TMP2, CARG3, 3
+    |  beqz AT, ->vmeta_tsetr           // In array part?
+    |.  ldc1 f20, 0(RA)
+    |   addu CRET1, TMP1, TMP2
+    |->BC_TSETR_Z:
+    |  ins_next1
+    |   sdc1 f20, 0(CRET1)
+    |  ins_next2
+    |
+    |7:  // Possible table write barrier for the value. Skip valiswhite check.
+    |  barrierback TAB:RB, TMP3, TMP0, <2
+    break;
  case BC_TSETM:
    |  // RA = base*8 (table at base-1), RD = num_const*8 (start index)
@@ -3959,6 +3980,7 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
    |   sw BASE, DISPATCH_GL(jit_base)(DISPATCH)
    |   sw L, DISPATCH_GL(jit_L)(DISPATCH)
    |  lw TMP2, TRACE:TMP2->mcode
+    |   sw L, DISPATCH_GL(tmpbuf.L)(DISPATCH)
    |  jr TMP2
    |.  addiu JGL, DISPATCH, GG_DISP2G+32768
    |.endif
diff --git a/src/vm_ppc.dasc b/src/vm_ppc.dasc
index 685ea518..6bceff9c 100644
--- a/src/vm_ppc.dasc
+++ b/src/vm_ppc.dasc
@@ -293,6 +293,7 @@
 |.type NODE,            Node
 |.type NARGS8,          int
 |.type TRACE,           GCtrace
+|.type SBUF,            SBuf
 |
 |//-----------------------------------------------------------------------
 |
@@ -895,6 +896,17 @@ static void build_subroutines(BuildCtx *ctx)
  |   li NARGS8:RC, 16                  // 2 args for func(t, k).
  |  b ->vm_call_dispatch_f
  |
+  |->vmeta_tgetr:
+  |  bl extern lj_tab_getinth           // (GCtab *t, int32_t key)
+  |  // Returns cTValue * or NULL.
+  |  cmplwi CRET1, 0
+  |  beq >1
+  |  lfd f14, 0(CRET1)
+  |  b ->BC_TGETR_Z
+  |1:
+  |  stwx TISNIL, BASE, RA
+  |  b ->cont_nop
+  |
  |//-----------------------------------------------------------------------
  |
  |->vmeta_tsets1:
@@ -962,6 +974,14 @@ static void build_subroutines(BuildCtx *ctx)
  |  stfd f0, 16(BASE)                  // Copy value to third argument.
  |  b ->vm_call_dispatch_f
  |
+  |->vmeta_tsetr:
+  |  stp BASE, L->base
+  |  stw PC, SAVE_PC
+  |  bl extern lj_tab_setinth  // (lua_State *L, GCtab *t, int32_t key)
+  |  // Returns TValue *.
+  |  stfd f14, 0(CRET1)
+  |  b ->cont_nop
+  |
  |//-- Comparison metamethods ---------------------------------------------
  |
  |->vmeta_comp:
@@ -1040,6 +1060,16 @@ static void build_subroutines(BuildCtx *ctx)
  |  b <3
  |.endif
  |
+  |->vmeta_istype:
+  |  subi PC, PC, 4
+  |   stp BASE, L->base
+  |   srwi CARG2, RA, 3
+  |   mr CARG1, L
+  |   srwi CARG3, RD, 3
+  |  stw PC, SAVE_PC
+  |  bl extern lj_meta_istype  // (lua_State *L, BCReg ra, BCReg tp)
+  |  b ->cont_nop
+  |
  |//-- Arithmetic metamethods ---------------------------------------------
  |
  |->vmeta_arith_nv:
@@ -1364,9 +1394,9 @@ static void build_subroutines(BuildCtx *ctx)
  |  mr CARG1, L
  |  mr CARG2, BASE
  |.if DUALNUM
-  |  bl extern lj_str_fromnumber        // (lua_State *L, cTValue *o)
+  |  bl extern lj_strfmt_number         // (lua_State *L, cTValue *o)
  |.else
-  |  bl extern lj_str_fromnum           // (lua_State *L, lua_Number *np)
+  |  bl extern lj_strfmt_num            // (lua_State *L, lua_Number *np)
  |.endif
  |  // Returns GCstr *.
  |  li CARG3, LJ_TSTR
@@ -1870,12 +1900,6 @@ static void build_subroutines(BuildCtx *ctx)
  |  math_extern2 atan2
  |  math_extern2 fmod
  |
-  |->ff_math_deg:
-  |.ffunc_n math_rad
-  |  lfd FARG2, CFUNC:RB->upvalue[0]
-  |  fmul FARG1, FARG1, FARG2
-  |  b ->fff_resn
-  |
  |.if DUALNUM
  |.ffunc math_ldexp
  |  cmplwi NARGS8:RC, 16
@@ -2021,11 +2045,6 @@ static void build_subroutines(BuildCtx *ctx)
  |
  |//-- String library -----------------------------------------------------
  |
-  |.ffunc_1 string_len
-  |  checkstr CARG3; bne ->fff_fallback
-  |  lwz CRET1, STR:CARG1->len
-  |  b ->fff_resi
-  |
  |.ffunc string_byte                   // Only handle the 1-arg case here.
  |  cmplwi NARGS8:RC, 8
  |   lwz CARG3, 0(BASE)
@@ -2080,6 +2099,7 @@ static void build_subroutines(BuildCtx *ctx)
  |  stp BASE, L->base
  |  stw PC, SAVE_PC
  |  bl extern lj_str_new               // (lua_State *L, char *str, size_t l)
+  |->fff_resstr:
  |  // Returns GCstr *.
  |  lp BASE, L->base
  |  li CARG3, LJ_TSTR
@@ -2157,114 +2177,29 @@ static void build_subroutines(BuildCtx *ctx)
  |  addi TMP1, TMP1, 1                 // start = 1 + (start ? start+len : 0)
  |  b <3
  |
-  |.ffunc string_rep                    // Only handle the 1-char case inline.
+  |.macro ffstring_op, name
-  |  ffgccheck
+  |  .ffunc string_ .. name
-  |  cmplwi NARGS8:RC, 16
-  |   lwz TMP0, 0(BASE)
-  |    lwz STR:CARG1, 4(BASE)
-  |   lwz CARG4, 8(BASE)
-  |.if DUALNUM
-  |    lwz CARG3, 12(BASE)
-  |.else
-  |    lfd FARG2, 8(BASE)
-  |.endif
-  |  bne ->fff_fallback                 // Exactly 2 arguments.
-  |  checkstr TMP0; bne ->fff_fallback
-  |.if DUALNUM
-  |  checknum CARG4; bne ->fff_fallback
-  |.else
-  |  checknum CARG4; bge ->fff_fallback
-  |    toint CARG3, FARG2
-  |.endif
-  |   lwz TMP0, STR:CARG1->len
-  |  cmpwi CARG3, 0
-  |   lwz TMP1, DISPATCH_GL(tmpbuf.sz)(DISPATCH)
-  |  ble >2                             // Count <= 0? (or non-int)
-  |   cmplwi TMP0, 1
-  |  subi TMP2, CARG3, 1
-  |   blt >2                            // Zero length string?
-  |  cmplw cr1, TMP1, CARG3
-  |   bne ->fff_fallback                // Fallback for > 1-char strings.
-  |   lbz TMP0, STR:CARG1[1]
-  |   lp CARG2, DISPATCH_GL(tmpbuf.buf)(DISPATCH)
-  |  blt cr1, ->fff_fallback
-  |1:  // Fill buffer with char. Yes, this is suboptimal code (do you care?).
-  |  cmplwi TMP2, 0
-  |   stbx TMP0, CARG2, TMP2
-  |   subi TMP2, TMP2, 1
-  |  bne <1
-  |  b ->fff_newstr
-  |2:  // Return empty string.
-  |  la STR:CARG1, DISPATCH_GL(strempty)(DISPATCH)
-  |  li CARG3, LJ_TSTR
-  |  b ->fff_restv
-  |
-  |.ffunc string_reverse
-  |  ffgccheck
-  |  cmplwi NARGS8:RC, 8
-  |   lwz CARG3, 0(BASE)
-  |    lwz STR:CARG1, 4(BASE)
-  |  blt ->fff_fallback
-  |  checkstr CARG3
-  |   lwz TMP1, DISPATCH_GL(tmpbuf.sz)(DISPATCH)
-  |  bne ->fff_fallback
-  |  lwz CARG3, STR:CARG1->len
-  |   la CARG1, #STR(STR:CARG1)
-  |   lp CARG2, DISPATCH_GL(tmpbuf.buf)(DISPATCH)
-  |   li TMP2, 0
-  |  cmplw TMP1, CARG3
-  |   subi TMP3, CARG3, 1
-  |  blt ->fff_fallback
-  |1:  // Reverse string copy.
-  |  cmpwi TMP3, 0
-  |   lbzx TMP1, CARG1, TMP2
-  |  blty ->fff_newstr
-  |   stbx TMP1, CARG2, TMP3
-  |  subi TMP3, TMP3, 1
-  |  addi TMP2, TMP2, 1
-  |  b <1
-  |
-  |.macro ffstring_case, name, lo
-  |  .ffunc name
  |  ffgccheck
  |  cmplwi NARGS8:RC, 8
  |   lwz CARG3, 0(BASE)
-  |    lwz STR:CARG1, 4(BASE)
+  |    lwz STR:CARG2, 4(BASE)
  |  blt ->fff_fallback
  |  checkstr CARG3
-  |   lwz TMP1, DISPATCH_GL(tmpbuf.sz)(DISPATCH)
+  |   la SBUF:CARG1, DISPATCH_GL(tmpbuf)(DISPATCH)
  |  bne ->fff_fallback
-  |  lwz CARG3, STR:CARG1->len
+  |   lwz TMP0, SBUF:CARG1->b
-  |   la CARG1, #STR(STR:CARG1)
+  |  stw L, SBUF:CARG1->L
-  |   lp CARG2, DISPATCH_GL(tmpbuf.buf)(DISPATCH)
+  |  stp BASE, L->base
-  |  cmplw TMP1, CARG3
+  |  stw PC, SAVE_PC
-  |   li TMP2, 0
+  |   stw TMP0, SBUF:CARG1->p
-  |  blt ->fff_fallback
+  |  bl extern lj_buf_putstr_ .. name
-  |1:  // ASCII case conversion.
+  |  bl extern lj_buf_tostr
-  |  cmplw TMP2, CARG3
+  |  b ->fff_resstr
-  |   lbzx TMP1, CARG1, TMP2
-  |  bgey ->fff_newstr
-  |   subi TMP0, TMP1, lo
-  |    xori TMP3, TMP1, 0x20
-  |   addic TMP0, TMP0, -26
-  |   subfe TMP3, TMP3, TMP3
-  |   rlwinm TMP3, TMP3, 0, 26, 26      // x &= 0x20.
-  |   xor TMP1, TMP1, TMP3
-  |   stbx TMP1, CARG2, TMP2
-  |  addi TMP2, TMP2, 1
-  |  b <1
  |.endmacro
  |
-  |ffstring_case string_lower, 65
+  |ffstring_op reverse
-  |ffstring_case string_upper, 97
+  |ffstring_op lower
-  |
+  |ffstring_op upper
-  |//-- Table library ------------------------------------------------------
-  |
-  |.ffunc_1 table_getn
-  |  checktab CARG3; bne ->fff_fallback
-  |  bl extern lj_tab_len               // (GCtab *t)
-  |  // Returns uint32_t (but less than 2^31).
-  |  b ->fff_resi
  |
  |//-- Bit library --------------------------------------------------------
  |
@@ -3265,6 +3200,29 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
    |  ins_next
    break;
+  case BC_ISTYPE:
+    |  // RA = src*8, RD = -type*8
+    |  lwzx TMP0, BASE, RA
+    |  srwi TMP1, RD, 3
+    |  ins_next1
+    |.if not PPE and not GPR64
+    |  add. TMP0, TMP0, TMP1
+    |.else
+    |  neg TMP1
+    |  cmpw TMP0, TMP1
+    |.endif
+    |  bne ->vmeta_istype
+    |  ins_next2
+    break;
+  case BC_ISNUM:
+    |  // RA = src*8, RD = -(TISNUM-1)*8
+    |  lwzx TMP0, BASE, RA
+    |  ins_next1
+    |  checknum TMP0
+    |  bge ->vmeta_istype
+    |  ins_next2
+    break;
  /* -- Unary ops --------------------------------------------------------- */
  case BC_MOV:
@@ -4016,6 +3974,30 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
    |  bne <1                           // 'no __index' flag set: done.
    |  b ->vmeta_tgetb                  // Caveat: preserve TMP0!
    break;
+  case BC_TGETR:
+    |  // RA = dst*8, RB = table*8, RC = key*8
+    |  add RB, BASE, RB
+    |  lwz TAB:CARG1, 4(RB)
+    |.if DUALNUM
+    |  add RC, BASE, RC
+    |  lwz TMP0, TAB:CARG1->asize
+    |  lwz CARG2, 4(RC)
+    |   lwz TMP1, TAB:CARG1->array
+    |.else
+    |  lfdx f0, BASE, RC
+    |  lwz TMP0, TAB:CARG1->asize
+    |  toint CARG2, f0
+    |   lwz TMP1, TAB:CARG1->array
+    |.endif
+    |  cmplw TMP0, CARG2
+    |   slwi TMP2, CARG2, 3
+    |  ble ->vmeta_tgetr                // In array part?
+    |   lfdx f14, TMP1, TMP2
+    |->BC_TGETR_Z:
+    |  ins_next1
+    |   stfdx f14, BASE, RA
+    |  ins_next2
+    break;
  case BC_TSETV:
    |  // RA = src*8, RB = table*8, RC = key*8
@@ -4195,6 +4177,39 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
    |  barrierback TAB:RB, TMP3, TMP0
    |  b <2
    break;
+  case BC_TSETR:
+    |  // RA = dst*8, RB = table*8, RC = key*8
+    |  add RB, BASE, RB
+    |  lwz TAB:CARG2, 4(RB)
+    |.if DUALNUM
+    |  add RC, BASE, RC
+    |    lbz TMP3, TAB:RB->marked
+    |  lwz TMP0, TAB:CARG2->asize
+    |  lwz CARG3, 4(RC)
+    |   lwz TMP1, TAB:CARG2->array
+    |.else
+    |  lfdx f0, BASE, RC
+    |    lbz TMP3, TAB:RB->marked
+    |  lwz TMP0, TAB:CARG2->asize
+    |  toint CARG3, f0
+    |   lwz TMP1, TAB:CARG2->array
+    |.endif
+    |  andix. TMP2, TMP3, LJ_GC_BLACK   // isblack(table)
+    |  bne >7
+    |2:
+    |  cmplw TMP0, CARG3
+    |   slwi TMP2, CARG3, 3
+    |   lfdx f14, BASE, RA
+    |  ble ->vmeta_tsetr                // In array part?
+    |  ins_next1
+    |   stfdx f14, TMP1, TMP2
+    |  ins_next2
+    |
+    |7:  // Possible table write barrier for the value. Skip valiswhite check.
+    |  barrierback TAB:CARG2, TMP3, TMP2
+    |  b <2
+    break;
  case BC_TSETM:
    |  // RA = base*8 (table at base-1), RD = num_const*8 (start index)
@@ -4838,6 +4853,7 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
    |  mtctr TMP2
    |   stw L, DISPATCH_GL(jit_L)(DISPATCH)
    |   addi JGL, DISPATCH, GG_DISP2G+32768
+    |   stw L, DISPATCH_GL(tmpbuf.L)(DISPATCH)
    |  bctr
    |.endif
    break;
diff --git a/src/vm_ppcspe.dasc b/src/vm_ppcspe.dasc
index 4fabc02f..b443f1b3 100644
--- a/src/vm_ppcspe.dasc
+++ b/src/vm_ppcspe.dasc
@@ -1456,12 +1456,6 @@ static void build_subroutines(BuildCtx *ctx)
  |  math_extern2 atan2
  |  math_extern2 fmod
  |
-  |->ff_math_deg:
-  |.ffunc_n math_rad
-  |  evldd CARG2, CFUNC:RB->upvalue[0]
-  |  efdmul CRET1, CARG1, CARG2
-  |  b ->fff_restv
-  |
  |.ffunc math_ldexp
  |  cmplwi NARGS8:RC, 16
  |   evldd CARG2, 0(BASE)
diff --git a/src/vm_x86.dasc b/src/vm_x86.dasc
index f25dfd30..62a5e139 100644
--- a/src/vm_x86.dasc
+++ b/src/vm_x86.dasc
@@ -18,7 +18,6 @@
 |
 |.if P64
 |.define X64, 1
-|.define SSE, 1
 |.if WIN
 |.define X64WIN, 1
 |.endif
@@ -116,6 +115,7 @@
 |.type NODE,            Node
 |.type NARGS,           int
 |.type TRACE,           GCtrace
+|.type SBUF,            SBuf
 |
 |// Stack layout while in interpreter. Must match with lj_frame.h.
 |//-----------------------------------------------------------------------
@@ -856,13 +856,9 @@ static void build_subroutines(BuildCtx *ctx)
  |.if DUALNUM
  |  mov TMP2, LJ_TISNUM
  |  mov TMP1, RC
-  |.elif SSE
+  |.else
  |  cvtsi2sd xmm0, RC
  |  movsd TMPQ, xmm0
-  |.else
-  |  mov ARG4, RC
-  |  fild ARG4
-  |  fstp TMPQ
  |.endif
  |  lea RCa, TMPQ                      // Store temp. TValue in TMPQ.
  |  jmp >1
@@ -916,6 +912,19 @@ static void build_subroutines(BuildCtx *ctx)
  |  mov NARGS:RD, 2+1                  // 2 args for func(t, k).
  |  jmp ->vm_call_dispatch_f
  |
+  |->vmeta_tgetr:
+  |  mov FCARG1, TAB:RB
+  |  mov RB, BASE                       // Save BASE.
+  |  mov FCARG2, RC                     // Caveat: FCARG2 == BASE
+  |  call extern lj_tab_getinth@8       // (GCtab *t, int32_t key)
+  |  // cTValue * or NULL returned in eax (RC).
+  |  movzx RA, PC_RA
+  |  mov BASE, RB                       // Restore BASE.
+  |  test RC, RC
+  |  jnz ->BC_TGETR_Z
+  |  mov dword [BASE+RA*8+4], LJ_TNIL
+  |  jmp ->BC_TGETR2_Z
+  |
  |//-----------------------------------------------------------------------
  |
  |->vmeta_tsets:
@@ -935,13 +944,9 @@ static void build_subroutines(BuildCtx *ctx)
  |.if DUALNUM
  |  mov TMP2, LJ_TISNUM
  |  mov TMP1, RC
-  |.elif SSE
+  |.else
  |  cvtsi2sd xmm0, RC
  |  movsd TMPQ, xmm0
-  |.else
-  |  mov ARG4, RC
-  |  fild ARG4
-  |  fstp TMPQ
  |.endif
  |  lea RCa, TMPQ                      // Store temp. TValue in TMPQ.
  |  jmp >1
@@ -1007,6 +1012,33 @@ static void build_subroutines(BuildCtx *ctx)
  |  mov NARGS:RD, 3+1                  // 3 args for func(t, k, v).
  |  jmp ->vm_call_dispatch_f
  |
+  |->vmeta_tsetr:
+  |.if X64WIN
+  |  mov L:CARG1d, SAVE_L
+  |  mov CARG3d, RC
+  |  mov L:CARG1d->base, BASE
+  |  xchg CARG2d, TAB:RB                // Caveat: CARG2d == BASE.
+  |.elif X64
+  |  mov L:CARG1d, SAVE_L
+  |  mov CARG2d, TAB:RB
+  |  mov L:CARG1d->base, BASE
+  |  mov RB, BASE                       // Save BASE.
+  |  mov CARG3d, RC                     // Caveat: CARG3d == BASE.
+  |.else
+  |  mov L:RA, SAVE_L
+  |  mov ARG2, TAB:RB
+  |  mov RB, BASE                       // Save BASE.
+  |  mov ARG3, RC
+  |  mov ARG1, L:RA
+  |  mov L:RA->base, BASE
+  |.endif
+  |  mov SAVE_PC, PC
+  |  call extern lj_tab_setinth  // (lua_State *L, GCtab *t, int32_t key)
+  |  // TValue * returned in eax (RC).
+  |  movzx RA, PC_RA
+  |  mov BASE, RB                       // Restore BASE.
+  |  jmp ->BC_TSETR_Z
+  |
  |//-- Comparison metamethods ---------------------------------------------
  |
  |->vmeta_comp:
@@ -1101,6 +1133,26 @@ static void build_subroutines(BuildCtx *ctx)
  |  jmp <3
  |.endif
  |
+  |->vmeta_istype:
+  |.if X64
+  |  mov L:RB, SAVE_L
+  |  mov L:RB->base, BASE               // Caveat: CARG2d/CARG3d may be BASE.
+  |  mov CARG2d, RA
+  |  movzx CARG3d, PC_RD
+  |  mov L:CARG1d, L:RB
+  |.else
+  |  movzx RD, PC_RD
+  |  mov ARG2, RA
+  |  mov L:RB, SAVE_L
+  |  mov ARG3, RD
+  |  mov ARG1, L:RB
+  |  mov L:RB->base, BASE
+  |.endif
+  |  mov SAVE_PC, PC
+  |  call extern lj_meta_istype  // (lua_State *L, BCReg ra, BCReg tp)
+  |  mov BASE, L:RB->base
+  |  jmp <6
+  |
  |//-- Arithmetic metamethods ---------------------------------------------
  |
  |->vmeta_arith_vno:
@@ -1509,11 +1561,7 @@ static void build_subroutines(BuildCtx *ctx)
  |.else
  |  jae ->fff_fallback
  |.endif
-  |.if SSE
  |  movsd xmm0, qword [BASE]; jmp ->fff_resxmm0
-  |.else
-  |  fld qword [BASE]; jmp ->fff_resn
-  |.endif
  |
  |.ffunc_1 tostring
  |  // Only handles the string or number case inline.
@@ -1538,9 +1586,9 @@ static void build_subroutines(BuildCtx *ctx)
  |.endif
  |  mov L:FCARG1, L:RB
  |.if DUALNUM
-  |  call extern lj_str_fromnumber@8    // (lua_State *L, cTValue *o)
+  |  call extern lj_strfmt_number@8     // (lua_State *L, cTValue *o)
  |.else
-  |  call extern lj_str_fromnum@8       // (lua_State *L, lua_Number *np)
+  |  call extern lj_strfmt_num@8        // (lua_State *L, lua_Number *np)
  |.endif
  |  // GCstr returned in eax (RD).
  |  mov BASE, L:RB->base
@@ -1631,19 +1679,12 @@ static void build_subroutines(BuildCtx *ctx)
  |  add RD, 1
  |  mov dword [BASE-4], LJ_TISNUM
  |  mov dword [BASE-8], RD
-  |.elif SSE
+  |.else
  |  movsd xmm0, qword [BASE+8]
  |  sseconst_1 xmm1, RBa
  |  addsd xmm0, xmm1
-  |  cvtsd2si RD, xmm0
+  |  cvttsd2si RD, xmm0
  |  movsd qword [BASE-8], xmm0
-  |.else
-  |  fld qword [BASE+8]
-  |  fld1
-  |  faddp st1
-  |  fist ARG1
-  |  fstp qword [BASE-8]
-  |  mov RD, ARG1
  |.endif
  |  mov TAB:RB, [BASE]
  |  cmp RD, TAB:RB->asize;  jae >2     // Not in array part?
@@ -1690,12 +1731,9 @@ static void build_subroutines(BuildCtx *ctx)
  |.if DUALNUM
  |  mov dword [BASE+12], LJ_TISNUM
  |  mov dword [BASE+8], 0
-  |.elif SSE
+  |.else
  |  xorps xmm0, xmm0
  |  movsd qword [BASE+8], xmm0
-  |.else
-  |  fldz
-  |  fstp qword [BASE+8]
  |.endif
  |  mov RD, 1+3
  |  jmp ->fff_res
@@ -1925,12 +1963,10 @@ static void build_subroutines(BuildCtx *ctx)
  |->fff_resi:  // Dummy.
  |.endif
  |
-  |.if SSE
  |->fff_resn:
  |  mov PC, [BASE-4]
  |  fstp qword [BASE-8]
  |  jmp ->fff_res1
-  |.endif
  |
  |  .ffunc_1 math_abs
  |.if DUALNUM
@@ -1954,8 +1990,6 @@ static void build_subroutines(BuildCtx *ctx)
  |.else
  |  cmp dword [BASE+4], LJ_TISNUM; jae ->fff_fallback
  |.endif
-  |
-  |.if SSE
  |  movsd xmm0, qword [BASE]
  |  sseconst_abs xmm1, RDa
  |  andps xmm0, xmm1
@@ -1963,15 +1997,6 @@ static void build_subroutines(BuildCtx *ctx)
  |  mov PC, [BASE-4]
  |  movsd qword [BASE-8], xmm0
  |  // fallthrough
-  |.else
-  |  fld qword [BASE]
-  |  fabs
-  |  // fallthrough
-  |->fff_resxmm0:  // Dummy.
-  |->fff_resn:
-  |  mov PC, [BASE-4]
-  |  fstp qword [BASE-8]
-  |.endif
  |
  |->fff_res1:
  |  mov RD, 1+1
@@ -2008,48 +2033,24 @@ static void build_subroutines(BuildCtx *ctx)
  |.else
  |  cmp dword [BASE+4], LJ_TISNUM; jae ->fff_fallback
  |.endif
-  |.if SSE
  |  movsd xmm0, qword [BASE]
-  |  call ->vm_ .. func
+  |  call ->vm_ .. func .. _sse
-  |  .if DUALNUM
+  |.if DUALNUM
-  |    cvtsd2si RB, xmm0
+  |  cvttsd2si RB, xmm0
-  |    cmp RB, 0x80000000
+  |  cmp RB, 0x80000000
-  |    jne ->fff_resi
+  |  jne ->fff_resi
-  |    cvtsi2sd xmm1, RB
+  |  cvtsi2sd xmm1, RB
-  |    ucomisd xmm0, xmm1
+  |  ucomisd xmm0, xmm1
-  |    jp ->fff_resxmm0
+  |  jp ->fff_resxmm0
-  |    je ->fff_resi
+  |  je ->fff_resi
-  |  .endif
-  |  jmp ->fff_resxmm0
-  |.else
-  |  fld qword [BASE]
-  |  call ->vm_ .. func
-  |  .if DUALNUM
-  |    fist ARG1
-  |    mov RB, ARG1
-  |    cmp RB, 0x80000000; jne >2
-  |    fdup
-  |    fild ARG1
-  |    fcomparepp
-  |    jp ->fff_resn
-  |    jne ->fff_resn
-  |2:
-  |    fpop
-  |    jmp ->fff_resi
-  | .else
-  |    jmp ->fff_resn
-  | .endif
  |.endif
+  |  jmp ->fff_resxmm0
  |.endmacro
  |
  |  math_round floor
  |  math_round ceil
  |
-  |.if SSE
  |.ffunc_nsse math_sqrt, sqrtsd; jmp ->fff_resxmm0
-  |.else
-  |.ffunc_n math_sqrt; fsqrt; jmp ->fff_resn
-  |.endif
  |
  |.ffunc math_log
  |  cmp NARGS:RD, 1+1; jne ->fff_fallback      // Exactly one argument.
@@ -2072,42 +2073,24 @@ static void build_subroutines(BuildCtx *ctx)
  |.ffunc_n math_atan;  fld1; fpatan;           jmp ->fff_resn
  |
  |.macro math_extern, func
-  |.if SSE
  |  .ffunc_nsse math_ .. func
-  |  .if not X64
+  |.if not X64
-  |    movsd FPARG1, xmm0
+  |  movsd FPARG1, xmm0
-  |  .endif
-  |.else
-  |  .ffunc_n math_ .. func
-  |  fstp FPARG1
  |.endif
  |  mov RB, BASE
  |  call extern lj_vm_ .. func
  |  mov BASE, RB
-  |  .if X64
+  |.if X64
-  |    jmp ->fff_resxmm0
+  |  jmp ->fff_resxmm0
-  |  .else
+  |.else
-  |    jmp ->fff_resn
+  |  jmp ->fff_resn
-  |  .endif
+  |.endif
  |.endmacro
  |
  |  math_extern sinh
  |  math_extern cosh
  |  math_extern tanh
  |
-  |->ff_math_deg:
-  |.if SSE
-  |.ffunc_nsse math_rad
-  |  mov CFUNC:RB, [BASE-8]
-  |  mulsd xmm0, qword CFUNC:RB->upvalue[0]
-  |  jmp ->fff_resxmm0
-  |.else
-  |.ffunc_n math_rad
-  |  mov CFUNC:RB, [BASE-8]
-  |  fmul qword CFUNC:RB->upvalue[0]
-  |  jmp ->fff_resn
-  |.endif
-  |
  |.ffunc_nn math_atan2;        fpatan;         jmp ->fff_resn
  |.ffunc_nnr math_ldexp;       fscale; fpop1;  jmp ->fff_resn
  |
@@ -2123,65 +2106,34 @@ static void build_subroutines(BuildCtx *ctx)
  |  cmp RB, 0x00200000; jb >4
  |1:
  |  shr RB, 21; sub RB, RC             // Extract and unbias exponent.
-  |.if SSE
  |  cvtsi2sd xmm0, RB
-  |.else
-  |  mov TMP1, RB; fild TMP1
-  |.endif
  |  mov RB, [BASE-4]
  |  and RB, 0x800fffff                 // Mask off exponent.
  |  or RB, 0x3fe00000                  // Put mantissa in range [0.5,1) or 0.
  |  mov [BASE-4], RB
  |2:
-  |.if SSE
  |  movsd qword [BASE], xmm0
-  |.else
-  |  fstp qword [BASE]
-  |.endif
  |  mov RD, 1+2
  |  jmp ->fff_res
  |3:  // Return +-0, +-Inf, NaN unmodified and an exponent of 0.
-  |.if SSE
  |  xorps xmm0, xmm0; jmp <2
-  |.else
-  |  fldz; jmp <2
-  |.endif
  |4:  // Handle denormals by multiplying with 2^54 and adjusting the bias.
-  |.if SSE
  |  movsd xmm0, qword [BASE]
  |  sseconst_hi xmm1, RBa, 43500000  // 2^54.
  |  mulsd xmm0, xmm1
  |  movsd qword [BASE-8], xmm0
-  |.else
-  |  fld qword [BASE]
-  |  mov TMP1, 0x5a800000; fmul TMP1    // x = x*2^54
-  |  fstp qword [BASE-8]
-  |.endif
  |  mov RB, [BASE-4]; mov RC, 1076; shl RB, 1; jmp <1
  |
-  |.if SSE
  |.ffunc_nsse math_modf
-  |.else
-  |.ffunc_n math_modf
-  |.endif
  |  mov RB, [BASE+4]
  |  mov PC, [BASE-4]
  |  shl RB, 1; cmp RB, 0xffe00000; je >4       // +-Inf?
-  |.if SSE
  |  movaps xmm4, xmm0
-  |  call ->vm_trunc
+  |  call ->vm_trunc_sse
  |  subsd xmm4, xmm0
  |1:
  |  movsd qword [BASE-8], xmm0
  |  movsd qword [BASE], xmm4
-  |.else
-  |  fdup
-  |  call ->vm_trunc
-  |  fsub st1, st0
-  |1:
-  |  fstp qword [BASE-8]
-  |  fstp qword [BASE]
-  |.endif
  |  mov RC, [BASE-4]; mov RB, [BASE+4]
  |  xor RC, RB; js >3                          // Need to adjust sign?
  |2:
@@ -2191,24 +2143,16 @@ static void build_subroutines(BuildCtx *ctx)
  |  xor RB, 0x80000000; mov [BASE+4], RB       // Flip sign of fraction.
  |  jmp <2
  |4:
-  |.if SSE
  |  xorps xmm4, xmm4; jmp <1                   // Return +-Inf and +-0.
-  |.else
-  |  fldz; fxch; jmp <1                         // Return +-Inf and +-0.
-  |.endif
  |
  |.ffunc_nnr math_fmod
  |1: ; fprem; fnstsw ax; sahf; jp <1
  |  fpop1
  |  jmp ->fff_resn
  |
-  |.if SSE
+  |.ffunc_nnsse math_pow;       call ->vm_pow_sse;      jmp ->fff_resxmm0
-  |.ffunc_nnsse math_pow;       call ->vm_pow;  jmp ->fff_resxmm0
-  |.else
-  |.ffunc_nn math_pow;          call ->vm_pow;  jmp ->fff_resn
-  |.endif
  |
-  |.macro math_minmax, name, cmovop, fcmovop, sseop
+  |.macro math_minmax, name, cmovop, sseop
  |  .ffunc name
  |  mov RA, 2
  |  cmp dword [BASE+4], LJ_TISNUM
@@ -2225,12 +2169,7 @@ static void build_subroutines(BuildCtx *ctx)
  |3:
  |  ja ->fff_fallback
  |  // Convert intermediate result to number and continue below.
-  |.if SSE
  |  cvtsi2sd xmm0, RB
-  |.else
-  |  mov TMP1, RB
-  |  fild TMP1
-  |.endif
  |  jmp >6
  |4:
  |  ja ->fff_fallback
@@ -2238,7 +2177,6 @@ static void build_subroutines(BuildCtx *ctx)
  |  jae ->fff_fallback
  |.endif
  |
-  |.if SSE
  |  movsd xmm0, qword [BASE]
  |5:  // Handle numbers or integers.
  |  cmp RA, RD; jae ->fff_resxmm0
@@ -2257,48 +2195,13 @@ static void build_subroutines(BuildCtx *ctx)
  |  sseop xmm0, xmm1
  |  add RA, 1
  |  jmp <5
-  |.else
-  |  fld qword [BASE]
-  |5:  // Handle numbers or integers.
-  |  cmp RA, RD; jae ->fff_resn
-  |  cmp dword [BASE+RA*8-4], LJ_TISNUM
-  |.if DUALNUM
-  |  jb >6
-  |  ja >9
-  |  fild dword [BASE+RA*8-8]
-  |  jmp >7
-  |.else
-  |  jae >9
-  |.endif
-  |6:
-  |  fld qword [BASE+RA*8-8]
-  |7:
-  |  fucomi st1; fcmovop st1; fpop1
-  |  add RA, 1
-  |  jmp <5
-  |.endif
  |.endmacro
  |
-  |  math_minmax math_min, cmovg, fcmovnbe, minsd
+  |  math_minmax math_min, cmovg, minsd
-  |  math_minmax math_max, cmovl, fcmovbe, maxsd
+  |  math_minmax math_max, cmovl, maxsd
-  |.if not SSE
-  |9:
-  |  fpop; jmp ->fff_fallback
-  |.endif
  |
  |//-- String library -----------------------------------------------------
  |
-  |.ffunc_1 string_len
-  |  cmp dword [BASE+4], LJ_TSTR;  jne ->fff_fallback
-  |  mov STR:RB, [BASE]
-  |.if DUALNUM
-  |  mov RB, dword STR:RB->len; jmp ->fff_resi
-  |.elif SSE
-  |  cvtsi2sd xmm0, dword STR:RB->len; jmp ->fff_resxmm0
-  |.else
-  |  fild dword STR:RB->len; jmp ->fff_resn
-  |.endif
-  |
  |.ffunc string_byte                   // Only handle the 1-arg case here.
  |  cmp NARGS:RD, 1+1;  jne ->fff_fallback
  |  cmp dword [BASE+4], LJ_TSTR;  jne ->fff_fallback
@@ -2309,10 +2212,8 @@ static void build_subroutines(BuildCtx *ctx)
  |  movzx RB, byte STR:RB[1]
  |.if DUALNUM
  |  jmp ->fff_resi
-  |.elif SSE
-  |  cvtsi2sd xmm0, RB; jmp ->fff_resxmm0
  |.else
-  |  mov TMP1, RB; fild TMP1; jmp ->fff_resn
+  |  cvtsi2sd xmm0, RB; jmp ->fff_resxmm0
  |.endif
  |
  |.ffunc string_char                   // Only handle the 1-arg case here.
@@ -2324,16 +2225,11 @@ static void build_subroutines(BuildCtx *ctx)
  |  mov RB, dword [BASE]
  |  cmp RB, 255;  ja ->fff_fallback
  |  mov TMP2, RB
-  |.elif SSE
+  |.else
  |  jae ->fff_fallback
  |  cvttsd2si RB, qword [BASE]
  |  cmp RB, 255;  ja ->fff_fallback
  |  mov TMP2, RB
-  |.else
-  |  jae ->fff_fallback
-  |  fld qword [BASE]
-  |  fistp TMP2
-  |  cmp TMP2, 255;  ja ->fff_fallback
  |.endif
  |.if X64
  |  mov TMP3, 1
@@ -2354,6 +2250,7 @@ static void build_subroutines(BuildCtx *ctx)
  |.endif
  |  mov SAVE_PC, PC
  |  call extern lj_str_new             // (lua_State *L, char *str, size_t l)
+  |->fff_resstr:
  |  // GCstr * returned in eax (RD).
  |  mov BASE, L:RB->base
  |  mov PC, [BASE-4]
@@ -2371,14 +2268,10 @@ static void build_subroutines(BuildCtx *ctx)
  |  jne ->fff_fallback
  |  mov RB, dword [BASE+16]
  |  mov TMP2, RB
-  |.elif SSE
+  |.else
  |  jae ->fff_fallback
  |  cvttsd2si RB, qword [BASE+16]
  |  mov TMP2, RB
-  |.else
-  |  jae ->fff_fallback
-  |  fld qword [BASE+16]
-  |  fistp TMP2
  |.endif
  |1:
  |  cmp dword [BASE+4], LJ_TSTR;  jne ->fff_fallback
@@ -2393,12 +2286,8 @@ static void build_subroutines(BuildCtx *ctx)
  |  mov RB, STR:RB->len
  |.if DUALNUM
  |  mov RA, dword [BASE+8]
-  |.elif SSE
-  |  cvttsd2si RA, qword [BASE+8]
  |.else
-  |  fld qword [BASE+8]
+  |  cvttsd2si RA, qword [BASE+8]
-  |  fistp ARG3
-  |  mov RA, ARG3
  |.endif
  |  mov RC, TMP2
  |  cmp RB, RC                         // len < end? (unsigned compare)
@@ -2442,123 +2331,27 @@ static void build_subroutines(BuildCtx *ctx)
  |  xor RC, RC                         // Zero length. Any ptr in RB is ok.
  |  jmp <4
  |
-  |.ffunc string_rep                    // Only handle the 1-char case inline.
+  |.macro ffstring_op, name
+  |  .ffunc_1 string_ .. name
  |  ffgccheck
-  |  cmp NARGS:RD, 2+1; jne ->fff_fallback      // Exactly 2 arguments.
  |  cmp dword [BASE+4], LJ_TSTR;  jne ->fff_fallback
-  |  cmp dword [BASE+12], LJ_TISNUM
+  |  mov L:RB, SAVE_L
-  |  mov STR:RB, [BASE]
+  |   lea SBUF:FCARG1, [DISPATCH+DISPATCH_GL(tmpbuf)]
-  |.if DUALNUM
+  |  mov L:RB->base, BASE
-  |  jne ->fff_fallback
+  |  mov STR:FCARG2, [BASE]             // Caveat: FCARG2 == BASE
-  |  mov RC, dword [BASE+8]
+  |   mov RC, SBUF:FCARG1->b
-  |.elif SSE
+  |   mov SBUF:FCARG1->L, L:RB
-  |  jae ->fff_fallback
+  |   mov SBUF:FCARG1->p, RC
-  |  cvttsd2si RC, qword [BASE+8]
+  |  mov SAVE_PC, PC
-  |.else
+  |  call extern lj_buf_putstr_ .. name .. @8
-  |  jae ->fff_fallback
+  |  mov FCARG1, eax
-  |  fld qword [BASE+8]
+  |  call extern lj_buf_tostr@4
-  |  fistp TMP2
+  |  jmp ->fff_resstr
-  |  mov RC, TMP2
-  |.endif
-  |  test RC, RC
-  |  jle ->fff_emptystr                 // Count <= 0? (or non-int)
-  |  cmp dword STR:RB->len, 1
-  |  jb ->fff_emptystr                  // Zero length string?
-  |  jne ->fff_fallback_2               // Fallback for > 1-char strings.
-  |  cmp [DISPATCH+DISPATCH_GL(tmpbuf.sz)], RC;  jb ->fff_fallback_2
-  |  movzx RA, byte STR:RB[1]
-  |  mov RB, [DISPATCH+DISPATCH_GL(tmpbuf.buf)]
-  |.if X64
-  |  mov TMP3, RC
-  |.else
-  |  mov ARG3, RC
-  |.endif
-  |1:  // Fill buffer with char. Yes, this is suboptimal code (do you care?).
-  |  mov [RB], RAL
-  |  add RB, 1
-  |  sub RC, 1
-  |  jnz <1
-  |  mov RD, [DISPATCH+DISPATCH_GL(tmpbuf.buf)]
-  |  jmp ->fff_newstr
-  |
-  |.ffunc_1 string_reverse
-  |  ffgccheck
-  |  cmp dword [BASE+4], LJ_TSTR;  jne ->fff_fallback
-  |  mov STR:RB, [BASE]
-  |  mov RC, STR:RB->len
-  |  test RC, RC
-  |  jz ->fff_emptystr                  // Zero length string?
-  |  cmp [DISPATCH+DISPATCH_GL(tmpbuf.sz)], RC;  jb ->fff_fallback_1
-  |  add RB, #STR
-  |  mov TMP2, PC                       // Need another temp register.
-  |.if X64
-  |  mov TMP3, RC
-  |.else
-  |  mov ARG3, RC
-  |.endif
-  |  mov PC, [DISPATCH+DISPATCH_GL(tmpbuf.buf)]
-  |1:
-  |  movzx RA, byte [RB]
-  |  add RB, 1
-  |  sub RC, 1
-  |  mov [PC+RC], RAL
-  |  jnz <1
-  |  mov RD, PC
-  |  mov PC, TMP2
-  |  jmp ->fff_newstr
-  |
-  |.macro ffstring_case, name, lo, hi
-  |  .ffunc_1 name
-  |  ffgccheck
-  |  cmp dword [BASE+4], LJ_TSTR;  jne ->fff_fallback
-  |  mov STR:RB, [BASE]
-  |  mov RC, STR:RB->len
-  |  cmp [DISPATCH+DISPATCH_GL(tmpbuf.sz)], RC;  jb ->fff_fallback_1
-  |  add RB, #STR
-  |  mov TMP2, PC                       // Need another temp register.
-  |.if X64
-  |  mov TMP3, RC
-  |.else
-  |  mov ARG3, RC
-  |.endif
-  |  mov PC, [DISPATCH+DISPATCH_GL(tmpbuf.buf)]
-  |  jmp >3
-  |1:  // ASCII case conversion. Yes, this is suboptimal code (do you care?).
-  |  movzx RA, byte [RB+RC]
-  |  cmp RA, lo
-  |  jb >2
-  |  cmp RA, hi
-  |  ja >2
-  |  xor RA, 0x20
-  |2:
-  |  mov [PC+RC], RAL
-  |3:
-  |  sub RC, 1
-  |  jns <1
-  |  mov RD, PC
-  |  mov PC, TMP2
-  |  jmp ->fff_newstr
  |.endmacro
  |
-  |ffstring_case string_lower, 0x41, 0x5a
+  |ffstring_op reverse
-  |ffstring_case string_upper, 0x61, 0x7a
+  |ffstring_op lower
-  |
+  |ffstring_op upper
-  |//-- Table library ------------------------------------------------------
-  |
-  |.ffunc_1 table_getn
-  |  cmp dword [BASE+4], LJ_TTAB;  jne ->fff_fallback
-  |  mov RB, BASE                       // Save BASE.
-  |  mov TAB:FCARG1, [BASE]
-  |  call extern lj_tab_len@4           // LJ_FASTCALL (GCtab *t)
-  |  // Length of table returned in eax (RD).
-  |  mov BASE, RB                       // Restore BASE.
-  |.if DUALNUM
-  |  mov RB, RD; jmp ->fff_resi
-  |.elif SSE
-  |  cvtsi2sd xmm0, RD; jmp ->fff_resxmm0
-  |.else
-  |  mov ARG1, RD; fild ARG1; jmp ->fff_resn
-  |.endif
  |
  |//-- Bit library --------------------------------------------------------
  |
@@ -2567,11 +2360,7 @@ static void build_subroutines(BuildCtx *ctx)
  |.macro .ffunc_bit, name, kind
  |  .ffunc_1 name
  |.if kind == 2
-  |.if SSE
  |  sseconst_tobit xmm1, RBa
-  |.else
-  |  mov TMP1, TOBIT_BIAS
-  |.endif
  |.endif
  |  cmp dword [BASE+4], LJ_TISNUM
  |.if DUALNUM
@@ -2587,37 +2376,17 @@ static void build_subroutines(BuildCtx *ctx)
  |.else
  |  jae ->fff_fallback
  |.endif
-  |.if SSE
  |  movsd xmm0, qword [BASE]
  |.if kind < 2
  |  sseconst_tobit xmm1, RBa
  |.endif
  |  addsd xmm0, xmm1
  |  movd RB, xmm0
-  |.else
-  |  fld qword [BASE]
-  |.if kind < 2
-  |  mov TMP1, TOBIT_BIAS
-  |.endif
-  |  fadd TMP1
-  |  fstp FPARG1
-  |.if kind > 0
-  |  mov RB, ARG1
-  |.endif
-  |.endif
  |2:
  |.endmacro
  |
  |.ffunc_bit bit_tobit, 0
-  |.if DUALNUM or SSE
-  |.if not SSE
-  |  mov RB, ARG1
-  |.endif
  |  jmp ->fff_resbit
-  |.else
-  |  fild ARG1
-  |  jmp ->fff_resn
-  |.endif
  |
  |.macro .ffunc_bit_op, name, ins
  |  .ffunc_bit name, 2
@@ -2637,17 +2406,10 @@ static void build_subroutines(BuildCtx *ctx)
  |.else
  |  jae ->fff_fallback_bit_op
  |.endif
-  |.if SSE
  |  movsd xmm0, qword [RD]
  |  addsd xmm0, xmm1
  |  movd RA, xmm0
  |  ins RB, RA
-  |.else
-  |  fld qword [RD]
-  |  fadd TMP1
-  |  fstp FPARG1
-  |  ins RB, ARG1
-  |.endif
  |  sub RD, 8
  |  jmp <1
  |.endmacro
@@ -2664,15 +2426,10 @@ static void build_subroutines(BuildCtx *ctx)
  |  not RB
  |.if DUALNUM
  |  jmp ->fff_resbit
-  |.elif SSE
+  |.else
  |->fff_resbit:
  |  cvtsi2sd xmm0, RB
  |  jmp ->fff_resxmm0
-  |.else
-  |->fff_resbit:
-  |  mov ARG1, RB
-  |  fild ARG1
-  |  jmp ->fff_resn
  |.endif
  |
  |->fff_fallback_bit_op:
@@ -2685,22 +2442,13 @@ static void build_subroutines(BuildCtx *ctx)
  |  // Note: no inline conversion from number for 2nd argument!
  |  cmp dword [BASE+12], LJ_TISNUM; jne ->fff_fallback
  |  mov RA, dword [BASE+8]
-  |.elif SSE
+  |.else
  |  .ffunc_nnsse name
  |  sseconst_tobit xmm2, RBa
  |  addsd xmm0, xmm2
  |  addsd xmm1, xmm2
  |  movd RB, xmm0
  |  movd RA, xmm1
-  |.else
-  |  .ffunc_nn name
-  |  mov TMP1, TOBIT_BIAS
-  |  fadd TMP1
-  |  fstp FPARG3
-  |  fadd TMP1
-  |  fstp FPARG1
-  |  mov RA, ARG3
-  |  mov RB, ARG1
  |.endif
  |  ins RB, cl                         // Assumes RA is ecx.
  |  jmp ->fff_resbit
@@ -3051,27 +2799,9 @@ static void build_subroutines(BuildCtx *ctx)
  |//-----------------------------------------------------------------------
  |
  |// FP value rounding. Called by math.floor/math.ceil fast functions
-  |// and from JIT code.
+  |// and from JIT code. arg/ret is xmm0. xmm0-xmm3 and RD (eax) modified.
-  |
+  |.macro vm_round, name, mode
-  |// x87 variant: Arg/ret on x87 stack. No int/xmm registers modified.
+  |->name .. _sse:
-  |.macro vm_round_x87, mode1, mode2
-  |  fnstcw word [esp+4]                // Caveat: overwrites ARG1 and ARG2.
-  |  mov [esp+8], eax
-  |  mov ax, mode1
-  |  or ax, [esp+4]
-  |.if mode2 ~= 0xffff
-  |  and ax, mode2
-  |.endif
-  |  mov [esp+6], ax
-  |  fldcw word [esp+6]
-  |  frndint
-  |  fldcw word [esp+4]
-  |  mov eax, [esp+8]
-  |  ret
-  |.endmacro
-  |
-  |// SSE variant: arg/ret is xmm0. xmm0-xmm3 and RD (eax) modified.
-  |.macro vm_round_sse, mode
  |  sseconst_abs xmm2, RDa
  |  sseconst_2p52 xmm3, RDa
  |  movaps xmm1, xmm0
@@ -3107,22 +2837,21 @@ static void build_subroutines(BuildCtx *ctx)
  |  ret
  |.endmacro
  |
-  |.macro vm_round, name, ssemode, mode1, mode2
+  |->vm_floor:
-  |->name:
+  |.if not X64
-  |.if not SSE
+  |  movsd xmm0, qword [esp+4]
-  |  vm_round_x87 mode1, mode2
+  |  call ->vm_floor_sse
+  |  movsd qword [esp+4], xmm0  // Overwrite callee-owned arg.
+  |  fld qword [esp+4]
+  |  ret
  |.endif
-  |->name .. _sse:
-  |  vm_round_sse ssemode
-  |.endmacro
  |
-  |  vm_round vm_floor, 0, 0x0400, 0xf7ff
+  |  vm_round vm_floor, 0
-  |  vm_round vm_ceil,  1, 0x0800, 0xfbff
+  |  vm_round vm_ceil,  1
-  |  vm_round vm_trunc, 2, 0x0c00, 0xffff
+  |  vm_round vm_trunc, 2
  |
  |// FP modulo x%y. Called by BC_MOD* and vm_arith.
  |->vm_mod:
-  |.if SSE
  |// Args in xmm0/xmm1, return value in xmm0.
  |// Caveat: xmm0-xmm5 and RC (eax) modified!
  |  movaps xmm5, xmm0
@@ -3150,23 +2879,6 @@ static void build_subroutines(BuildCtx *ctx)
  |  movaps xmm0, xmm5
  |  subsd xmm0, xmm1
  |  ret
-  |.else
-  |// Args/ret on x87 stack (y on top). No xmm registers modified.
-  |// Caveat: needs 3 slots on x87 stack! RC (eax) modified!
-  |  fld st1
-  |  fdiv st1
-  |  fnstcw word [esp+4]
-  |  mov ax, 0x0400
-  |  or ax, [esp+4]
-  |  and ax, 0xf7ff
-  |  mov [esp+6], ax
-  |  fldcw word [esp+6]
-  |  frndint
-  |  fldcw word [esp+4]
-  |  fmulp st1
-  |  fsubp st1
-  |  ret
-  |.endif
  |
  |// FP log2(x). Called by math.log(x, base).
  |->vm_log2:
@@ -3217,105 +2929,15 @@ static void build_subroutines(BuildCtx *ctx)
  |
  |// Generic power function x^y. Called by BC_POW, math.pow fast function,
  |// and vm_arith.
-  |// Args/ret on x87 stack (y on top). RC (eax) modified.
-  |// Caveat: needs 3 slots on x87 stack!
-  |->vm_pow:
-  |.if not SSE
-  |  fist dword [esp+4]                 // Store/reload int before comparison.
-  |  fild dword [esp+4]                 // Integral exponent used in vm_powi.
-  |  fucomip st1
-  |  jnz >8                             // Branch for FP exponents.
-  |  jp >9                              // Branch for NaN exponent.
-  |  fpop                               // Pop y and fallthrough to vm_powi.
-  |
-  |// FP/int power function x^i. Arg1/ret on x87 stack.
-  |// Arg2 (int) on C stack. RC (eax) modified.
-  |// Caveat: needs 2 slots on x87 stack!
-  |  mov eax, [esp+4]
-  |  cmp eax, 1; jle >6                 // i<=1?
-  |  // Now 1 < (unsigned)i <= 0x80000000.
-  |1:  // Handle leading zeros.
-  |  test eax, 1; jnz >2
-  |  fmul st0
-  |  shr eax, 1
-  |  jmp <1
-  |2:
-  |  shr eax, 1; jz >5
-  |  fdup
-  |3:  // Handle trailing bits.
-  |  fmul st0
-  |  shr eax, 1; jz >4
-  |  jnc <3
-  |  fmul st1, st0
-  |  jmp <3
-  |4:
-  |  fmulp st1
-  |5:
-  |  ret
-  |6:
-  |  je <5                              // x^1 ==> x
-  |  jb >7
-  |  fld1; fdivrp st1
-  |  neg eax
-  |  cmp eax, 1; je <5                  // x^-1 ==> 1/x
-  |  jmp <1                             // x^-i ==> (1/x)^i
-  |7:
-  |  fpop; fld1                         // x^0 ==> 1
-  |  ret
-  |
-  |8:  // FP/FP power function x^y.
-  |  fst dword [esp+4]
-  |  fxch
-  |  fst dword [esp+8]
-  |  mov eax, [esp+4]; shl eax, 1
-  |  cmp eax, 0xff000000; je >2                 // x^+-Inf?
-  |  mov eax, [esp+8]; shl eax, 1; je >4        // +-0^y?
-  |  cmp eax, 0xff000000; je >4                 // +-Inf^y?
-  |  fyl2x
-  |  jmp ->vm_exp2raw
-  |
-  |9:  // Handle x^NaN.
-  |  fld1
-  |  fucomip st2
-  |  je >1                              // 1^NaN ==> 1
-  |  fxch                               // x^NaN ==> NaN
-  |1:
-  |  fpop
-  |  ret
-  |
-  |2:  // Handle x^+-Inf.
-  |  fabs
-  |  fld1
-  |  fucomip st1
-  |  je >3                                      // +-1^+-Inf ==> 1
-  |  fpop; fabs; fldz; mov eax, 0; setc al
-  |  ror eax, 1; xor eax, [esp+4]; jns >3       // |x|<>1, x^+-Inf ==> +Inf/0
-  |  fxch
-  |3:
-  |  fpop1; fabs
-  |  ret
-  |
-  |4:  // Handle +-0^y or +-Inf^y.
-  |  cmp dword [esp+4], 0; jge <3               // y >= 0, x^y ==> |x|
-  |  fpop; fpop
-  |  test eax, eax; jz >5                       // y < 0, +-0^y ==> +Inf
-  |  fldz                                       // y < 0, +-Inf^y ==> 0
-  |  ret
-  |5:
-  |  mov dword [esp+4], 0x7f800000              // Return +Inf.
-  |  fld dword [esp+4]
-  |  ret
-  |.endif
-  |
  |// Args in xmm0/xmm1. Ret in xmm0. xmm0-xmm2 and RC (eax) modified.
  |// Needs 16 byte scratch area for x86. Also called from JIT code.
  |->vm_pow_sse:
-  |  cvtsd2si eax, xmm1
+  |  cvttsd2si eax, xmm1
  |  cvtsi2sd xmm2, eax
  |  ucomisd xmm1, xmm2
  |  jnz >8                             // Branch for FP exponents.
  |  jp >9                              // Branch for NaN exponent.
-  |  // Fallthrough to vm_powi_sse.
+  |  // Fallthrough.
  |
  |// Args in xmm0/eax. Ret in xmm0. xmm0-xmm1 and eax modified.
  |->vm_powi_sse:
@@ -3437,8 +3059,8 @@ static void build_subroutines(BuildCtx *ctx)
  |  .else
  |    .define fpmop, CARG1d
  |  .endif
-  |  cmp fpmop, 1; jb ->vm_floor; je ->vm_ceil
+  |  cmp fpmop, 1; jb ->vm_floor_sse; je ->vm_ceil_sse
-  |  cmp fpmop, 3; jb ->vm_trunc; ja >2
+  |  cmp fpmop, 3; jb ->vm_trunc_sse; ja >2
  |  sqrtsd xmm0, xmm0; ret
  |2:
  |  .if X64WIN
@@ -3478,14 +3100,13 @@ static void build_subroutines(BuildCtx *ctx)
  |  ret
  |.else  // x86 calling convention.
  |  .define fpmop, eax
-  |.if SSE
  |  mov fpmop, [esp+12]
  |  movsd xmm0, qword [esp+4]
  |  cmp fpmop, 1; je >1; ja >2
-  |  call ->vm_floor; jmp >7
+  |  call ->vm_floor_sse; jmp >7
-  |1: ; call ->vm_ceil; jmp >7
+  |1: ; call ->vm_ceil_sse; jmp >7
  |2: ; cmp fpmop, 3; je >1; ja >2
-  |  call ->vm_trunc; jmp >7
+  |  call ->vm_trunc_sse; jmp >7
  |1:
  |  sqrtsd xmm0, xmm0
  |7:
@@ -3503,23 +3124,6 @@ static void build_subroutines(BuildCtx *ctx)
  |2: ; cmp fpmop, 11; je >1; ja >9
  |   fcos; ret
  |1: ; fptan; fpop; ret
-  |.else
-  |  mov fpmop, [esp+12]
-  |  fld qword [esp+4]
-  |  cmp fpmop, 1; jb ->vm_floor; je ->vm_ceil
-  |  cmp fpmop, 3; jb ->vm_trunc; ja >2
-  |  fsqrt; ret
-  |2: ; cmp fpmop, 5; jb ->vm_exp_x87; je ->vm_exp2_x87
-  |  cmp fpmop, 7; je >1; ja >2
-  |  fldln2; fxch; fyl2x; ret
-  |1: ; fld1; fxch; fyl2x; ret
-  |2: ; cmp fpmop, 9; je >1; ja >2
-  |  fldlg2; fxch; fyl2x; ret
-  |1: ; fsin; ret
-  |2: ; cmp fpmop, 11; je >1; ja >9
-  |   fcos; ret
-  |1: ; fptan; fpop; ret
-  |.endif
  |.endif
  |9: ; int3                                    // Bad fpm.
  |.endif
@@ -3541,7 +3145,7 @@ static void build_subroutines(BuildCtx *ctx)
  |2: ; cmp foldop, 3; je >1; ja >2
  |  mulsd xmm0, xmm1; ret
  |1: ; divsd xmm0, xmm1; ret
-  |2: ; cmp foldop, 5; jb ->vm_mod; je ->vm_pow
+  |2: ; cmp foldop, 5; jb ->vm_mod; je ->vm_pow_sse
  |  cmp foldop, 7; je >1; ja >2
  |  sseconst_sign xmm1, RDa; xorps xmm0, xmm1; ret
  |1: ; sseconst_abs xmm1, RDa; andps xmm0, xmm1; ret
@@ -3574,7 +3178,7 @@ static void build_subroutines(BuildCtx *ctx)
  |1: ; maxsd xmm0, xmm1; ret
  |9: ; int3                            // Bad op.
  |
-  |.elif SSE  // x86 calling convention with SSE ops.
+  |.else  // x86 calling convention.
  |
  |  .define foldop, eax
  |  mov foldop, [esp+20]
@@ -3593,7 +3197,7 @@ static void build_subroutines(BuildCtx *ctx)
  |2: ; cmp foldop, 5
  |  je >1; ja >2
  |  call ->vm_mod; jmp <7
-  |1: ; pop edx; call ->vm_pow; push edx; jmp <7  // Writes to scratch area.
+  |1: ; pop edx; call ->vm_pow_sse; push edx; jmp <7  // Writes to scratch area.
  |2: ; cmp foldop, 7; je >1; ja >2
  |  sseconst_sign xmm1, RDa; xorps xmm0, xmm1; jmp <7
  |1: ; sseconst_abs xmm1, RDa; andps xmm0, xmm1; jmp <7
@@ -3608,29 +3212,6 @@ static void build_subroutines(BuildCtx *ctx)
  |1: ; maxsd xmm0, xmm1; jmp <7
  |9: ; int3                            // Bad op.
  |
-  |.else  // x86 calling convention with x87 ops.
-  |
-  |  mov eax, [esp+20]
-  |  fld qword [esp+4]
-  |  fld qword [esp+12]
-  |  cmp eax, 1; je >1; ja >2
-  |  faddp st1; ret
-  |1: ; fsubp st1; ret
-  |2: ; cmp eax, 3; je >1; ja >2
-  |  fmulp st1; ret
-  |1: ; fdivp st1; ret
-  |2: ; cmp eax, 5; jb ->vm_mod; je ->vm_pow
-  |  cmp eax, 7; je >1; ja >2
-  |  fpop; fchs; ret
-  |1: ; fpop; fabs; ret
-  |2: ; cmp eax, 9; je >1; ja >2
-  |  fpatan; ret
-  |1: ; fxch; fscale; fpop1; ret
-  |2: ; cmp eax, 11; je >1; ja >9
-  |  fucomi st1; fcmovnbe st1; fpop1; ret
-  |1: ; fucomi st1; fcmovbe st1; fpop1; ret
-  |9: ; int3                            // Bad op.
-  |
  |.endif
  |
  |//-----------------------------------------------------------------------
@@ -3943,19 +3524,12 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
    |  // RA is a number.
    |  cmp dword [BASE+RD*8+4], LJ_TISNUM; jb >1; jne ->vmeta_comp
    |  // RA is a number, RD is an integer.
-    |.if SSE
    |  cvtsi2sd xmm0, dword [BASE+RD*8]
    |  jmp >2
-    |.else
-    |  fld qword [BASE+RA*8]
-    |  fild dword [BASE+RD*8]
-    |  jmp >3
-    |.endif
    |
    |8:  // RA is an integer, RD is not an integer.
    |  ja ->vmeta_comp
    |  // RA is an integer, RD is a number.
-    |.if SSE
    |  cvtsi2sd xmm1, dword [BASE+RA*8]
    |  movsd xmm0, qword [BASE+RD*8]
    |  add PC, 4
@@ -3963,29 +3537,15 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
    |  jmp_comp jbe, ja, jb, jae, <9
    |  jmp <6
    |.else
-    |  fild dword [BASE+RA*8]
-    |  jmp >2
-    |.endif
-    |.else
    |  checknum RA, ->vmeta_comp
    |  checknum RD, ->vmeta_comp
    |.endif
-    |.if SSE
    |1:
    |  movsd xmm0, qword [BASE+RD*8]
    |2:
    |  add PC, 4
    |  ucomisd xmm0, qword [BASE+RA*8]
    |3:
-    |.else
-    |1:
-    |  fld qword [BASE+RA*8]            // Reverse order, i.e like cmp D, A.
-    |2:
-    |  fld qword [BASE+RD*8]
-    |3:
-    |  add PC, 4
-    |  fcomparepp
-    |.endif
    |  // Unordered: all of ZF CF PF set, ordered: PF clear.
    |  // To preserve NaN semantics GE/GT branch on unordered, but LT/LE don't.
    |.if DUALNUM
@@ -4025,43 +3585,25 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
    |  // RD is a number.
    |  cmp dword [BASE+RA*8+4], LJ_TISNUM; jb >1; jne >5
    |  // RD is a number, RA is an integer.
-    |.if SSE
    |  cvtsi2sd xmm0, dword [BASE+RA*8]
-    |.else
-    |  fild dword [BASE+RA*8]
-    |.endif
    |  jmp >2
    |
    |8:  // RD is an integer, RA is not an integer.
    |  ja >5
    |  // RD is an integer, RA is a number.
-    |.if SSE
    |  cvtsi2sd xmm0, dword [BASE+RD*8]
    |  ucomisd xmm0, qword [BASE+RA*8]
-    |.else
-    |  fild dword [BASE+RD*8]
-    |  fld qword [BASE+RA*8]
-    |.endif
    |  jmp >4
    |
    |.else
    |  cmp RB, LJ_TISNUM; jae >5
    |  checknum RA, >5
    |.endif
-    |.if SSE
    |1:
    |  movsd xmm0, qword [BASE+RA*8]
    |2:
    |  ucomisd xmm0, qword [BASE+RD*8]
    |4:
-    |.else
-    |1:
-    |  fld qword [BASE+RA*8]
-    |2:
-    |  fld qword [BASE+RD*8]
-    |4:
-    |  fcomparepp
-    |.endif
  iseqne_fp:
    if (vk) {
      |  jp >2                          // Unordered means not equal.
@@ -4184,39 +3726,21 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
    |  // RA is a number.
    |  cmp dword [KBASE+RD*8+4], LJ_TISNUM; jb >1
    |  // RA is a number, RD is an integer.
-    |.if SSE
    |  cvtsi2sd xmm0, dword [KBASE+RD*8]
-    |.else
-    |  fild dword [KBASE+RD*8]
-    |.endif
    |  jmp >2
    |
    |8:  // RA is an integer, RD is a number.
-    |.if SSE
    |  cvtsi2sd xmm0, dword [BASE+RA*8]
    |  ucomisd xmm0, qword [KBASE+RD*8]
-    |.else
-    |  fild dword [BASE+RA*8]
-    |  fld qword [KBASE+RD*8]
-    |.endif
    |  jmp >4
    |.else
    |  cmp RB, LJ_TISNUM; jae >3
    |.endif
-    |.if SSE
    |1:
    |  movsd xmm0, qword [KBASE+RD*8]
    |2:
    |  ucomisd xmm0, qword [BASE+RA*8]
    |4:
-    |.else
-    |1:
-    |  fld qword [KBASE+RD*8]
-    |2:
-    |  fld qword [BASE+RA*8]
-    |4:
-    |  fcomparepp
-    |.endif
    goto iseqne_fp;
  case BC_ISEQP: case BC_ISNEP:
    vk = op == BC_ISEQP;
@@ -4267,6 +3791,18 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
    |  ins_next
    break;
+  case BC_ISTYPE:
+    |  ins_AD   // RA = src, RD = -type
+    |  add RD, [BASE+RA*8+4]
+    |  jne ->vmeta_istype
+    |  ins_next
+    break;
+  case BC_ISNUM:
+    |  ins_AD   // RA = src, RD = -(TISNUM-1)
+    |  checknum RA, ->vmeta_istype
+    |  ins_next
+    break;
  /* -- Unary ops --------------------------------------------------------- */
  case BC_MOV:
@@ -4310,16 +3846,10 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
    |.else
    |  checknum RD, ->vmeta_unm
    |.endif
-    |.if SSE
    |  movsd xmm0, qword [BASE+RD*8]
    |  sseconst_sign xmm1, RDa
    |  xorps xmm0, xmm1
    |  movsd qword [BASE+RA*8], xmm0
-    |.else
-    |  fld qword [BASE+RD*8]
-    |  fchs
-    |  fstp qword [BASE+RA*8]
-    |.endif
    |.if DUALNUM
    |  jmp <9
    |.else
@@ -4335,15 +3865,11 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
    |1:
    |  mov dword [BASE+RA*8+4], LJ_TISNUM
    |  mov dword [BASE+RA*8], RD
-    |.elif SSE
+    |.else
    |  xorps xmm0, xmm0
    |  cvtsi2sd xmm0, dword STR:RD->len
    |1:
    |  movsd qword [BASE+RA*8], xmm0
-    |.else
-    |  fild dword STR:RD->len
-    |1:
-    |  fstp qword [BASE+RA*8]
    |.endif
    |  ins_next
    |2:
@@ -4361,11 +3887,8 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
    |  // Length of table returned in eax (RD).
    |.if DUALNUM
    |  // Nothing to do.
-    |.elif SSE
-    |  cvtsi2sd xmm0, RD
    |.else
-    |  mov ARG1, RD
+    |  cvtsi2sd xmm0, RD
-    |  fild ARG1
    |.endif
    |  mov BASE, RB                     // Restore BASE.
    |  movzx RA, PC_RA
@@ -4380,7 +3903,7 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
  /* -- Binary ops -------------------------------------------------------- */
-    |.macro ins_arithpre, x87ins, sseins, ssereg
+    |.macro ins_arithpre, sseins, ssereg
    |  ins_ABC
    ||vk = ((int)op - BC_ADDVN) / (BC_ADDNV-BC_ADDVN);
    ||switch (vk) {
@@ -4389,37 +3912,22 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
    |   .if DUALNUM
    |     cmp dword [KBASE+RC*8+4], LJ_TISNUM; jae ->vmeta_arith_vn
    |   .endif
-    |   .if SSE
+    |   movsd xmm0, qword [BASE+RB*8]
-    |     movsd xmm0, qword [BASE+RB*8]
+    |   sseins ssereg, qword [KBASE+RC*8]
-    |     sseins ssereg, qword [KBASE+RC*8]
-    |   .else
-    |     fld qword [BASE+RB*8]
-    |     x87ins qword [KBASE+RC*8]
-    |   .endif
    ||  break;
    ||case 1:
    |   checknum RB, ->vmeta_arith_nv
    |   .if DUALNUM
    |     cmp dword [KBASE+RC*8+4], LJ_TISNUM; jae ->vmeta_arith_nv
    |   .endif
-    |   .if SSE
+    |   movsd xmm0, qword [KBASE+RC*8]
-    |     movsd xmm0, qword [KBASE+RC*8]
+    |   sseins ssereg, qword [BASE+RB*8]
-    |     sseins ssereg, qword [BASE+RB*8]
-    |   .else
-    |     fld qword [KBASE+RC*8]
-    |     x87ins qword [BASE+RB*8]
-    |   .endif
    ||  break;
    ||default:
    |   checknum RB, ->vmeta_arith_vv
    |   checknum RC, ->vmeta_arith_vv
-    |   .if SSE
+    |   movsd xmm0, qword [BASE+RB*8]
-    |     movsd xmm0, qword [BASE+RB*8]
+    |   sseins ssereg, qword [BASE+RC*8]
-    |     sseins ssereg, qword [BASE+RC*8]
-    |   .else
-    |     fld qword [BASE+RB*8]
-    |     x87ins qword [BASE+RC*8]
-    |   .endif
    ||  break;
    ||}
    |.endmacro
@@ -4457,54 +3965,50 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
    |.endmacro
    |
    |.macro ins_arithpost
-    |.if SSE
    |  movsd qword [BASE+RA*8], xmm0
-    |.else
-    |  fstp qword [BASE+RA*8]
-    |.endif
    |.endmacro
    |
-    |.macro ins_arith, x87ins, sseins
+    |.macro ins_arith, sseins
-    |  ins_arithpre x87ins, sseins, xmm0
+    |  ins_arithpre sseins, xmm0
    |  ins_arithpost
    |  ins_next
    |.endmacro
    |
-    |.macro ins_arith, intins, x87ins, sseins
+    |.macro ins_arith, intins, sseins
    |.if DUALNUM
    |  ins_arithdn intins
    |.else
-    |  ins_arith, x87ins, sseins
+    |  ins_arith, sseins
    |.endif
    |.endmacro
    |  // RA = dst, RB = src1 or num const, RC = src2 or num const
  case BC_ADDVN: case BC_ADDNV: case BC_ADDVV:
-    |  ins_arith add, fadd, addsd
+    |  ins_arith add, addsd
    break;
  case BC_SUBVN: case BC_SUBNV: case BC_SUBVV:
-    |  ins_arith sub, fsub, subsd
+    |  ins_arith sub, subsd
    break;
  case BC_MULVN: case BC_MULNV: case BC_MULVV:
-    |  ins_arith imul, fmul, mulsd
+    |  ins_arith imul, mulsd
    break;
  case BC_DIVVN: case BC_DIVNV: case BC_DIVVV:
-    |  ins_arith fdiv, divsd
+    |  ins_arith divsd
    break;
  case BC_MODVN:
-    |  ins_arithpre fld, movsd, xmm1
+    |  ins_arithpre movsd, xmm1
    |->BC_MODVN_Z:
    |  call ->vm_mod
    |  ins_arithpost
    |  ins_next
    break;
  case BC_MODNV: case BC_MODVV:
-    |  ins_arithpre fld, movsd, xmm1
+    |  ins_arithpre movsd, xmm1
    |  jmp ->BC_MODVN_Z                 // Avoid 3 copies. It's slow anyway.
    break;
  case BC_POW:
-    |  ins_arithpre fld, movsd, xmm1
+    |  ins_arithpre movsd, xmm1
-    |  call ->vm_pow
+    |  call ->vm_pow_sse
    |  ins_arithpost
    |  ins_next
    break;
@@ -4573,25 +4077,17 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
    |  movsx RD, RDW
    |  mov dword [BASE+RA*8+4], LJ_TISNUM
    |  mov dword [BASE+RA*8], RD
-    |.elif SSE
+    |.else
    |  movsx RD, RDW                    // Sign-extend literal.
    |  cvtsi2sd xmm0, RD
    |  movsd qword [BASE+RA*8], xmm0
-    |.else
-    |  fild PC_RD                       // Refetch signed RD from instruction.
-    |  fstp qword [BASE+RA*8]
    |.endif
    |  ins_next
    break;
  case BC_KNUM:
    |  ins_AD   // RA = dst, RD = num const
-    |.if SSE
    |  movsd xmm0, qword [KBASE+RD*8]
    |  movsd qword [BASE+RA*8], xmm0
-    |.else
-    |  fld qword [KBASE+RD*8]
-    |  fstp qword [BASE+RA*8]
-    |.endif
    |  ins_next
    break;
  case BC_KPRI:
@@ -4698,18 +4194,10 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
  case BC_USETN:
    |  ins_AD   // RA = upvalue #, RD = num const
    |  mov LFUNC:RB, [BASE-8]
-    |.if SSE
    |  movsd xmm0, qword [KBASE+RD*8]
-    |.else
-    |  fld qword [KBASE+RD*8]
-    |.endif
    |  mov UPVAL:RB, [LFUNC:RB+RA*4+offsetof(GCfuncL, uvptr)]
    |  mov RA, UPVAL:RB->v
-    |.if SSE
    |  movsd qword [RA], xmm0
-    |.else
-    |  fstp qword [RA]
-    |.endif
    |  ins_next
    break;
  case BC_USETP:
@@ -4863,18 +4351,10 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
    |.else
    |  // Convert number to int and back and compare.
    |  checknum RC, >5
-    |.if SSE
    |  movsd xmm0, qword [BASE+RC*8]
-    |  cvtsd2si RC, xmm0
+    |  cvttsd2si RC, xmm0
    |  cvtsi2sd xmm1, RC
    |  ucomisd xmm0, xmm1
-    |.else
-    |  fld qword [BASE+RC*8]
-    |  fist ARG1
-    |  fild ARG1
-    |  fcomparepp
-    |  mov RC, ARG1
-    |.endif
    |  jne ->vmeta_tgetv                // Generic numeric key? Use fallback.
    |.endif
    |  cmp RC, TAB:RB->asize    // Takes care of unordered, too.
@@ -4998,6 +4478,32 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
    |  mov dword [BASE+RA*8+4], LJ_TNIL
    |  jmp <1
    break;
+  case BC_TGETR:
+    |  ins_ABC  // RA = dst, RB = table, RC = key
+    |  mov TAB:RB, [BASE+RB*8]
+    |.if DUALNUM
+    |  mov RC, dword [BASE+RC*8]
+    |.else
+    |  cvttsd2si RC, qword [BASE+RC*8]
+    |.endif
+    |  cmp RC, TAB:RB->asize
+    |  jae ->vmeta_tgetr                // Not in array part? Use fallback.
+    |  shl RC, 3
+    |  add RC, TAB:RB->array
+    |  // Get array slot.
+    |->BC_TGETR_Z:
+    |.if X64
+    |  mov RBa, [RC]
+    |  mov [BASE+RA*8], RBa
+    |.else
+    |  mov RB, [RC]
+    |  mov RC, [RC+4]
+    |  mov [BASE+RA*8], RB
+    |  mov [BASE+RA*8+4], RC
+    |.endif
+    |->BC_TGETR2_Z:
+    |  ins_next
+    break;
  case BC_TSETV:
    |  ins_ABC  // RA = src, RB = table, RC = key
@@ -5011,18 +4517,10 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
    |.else
    |  // Convert number to int and back and compare.
    |  checknum RC, >5
-    |.if SSE
    |  movsd xmm0, qword [BASE+RC*8]
-    |  cvtsd2si RC, xmm0
+    |  cvttsd2si RC, xmm0
    |  cvtsi2sd xmm1, RC
    |  ucomisd xmm0, xmm1
-    |.else
-    |  fld qword [BASE+RC*8]
-    |  fist ARG1
-    |  fild ARG1
-    |  fcomparepp
-    |  mov RC, ARG1
-    |.endif
    |  jne ->vmeta_tsetv                // Generic numeric key? Use fallback.
    |.endif
    |  cmp RC, TAB:RB->asize            // Takes care of unordered, too.
@@ -5192,6 +4690,39 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
    |  movzx RA, PC_RA                  // Restore RA.
    |  jmp <2
    break;
+  case BC_TSETR:
+    |  ins_ABC  // RA = src, RB = table, RC = key
+    |  mov TAB:RB, [BASE+RB*8]
+    |.if DUALNUM
+    |  mov RC, dword [BASE+RC*8]
+    |.else
+    |  cvttsd2si RC, qword [BASE+RC*8]
+    |.endif
+    |  test byte TAB:RB->marked, LJ_GC_BLACK    // isblack(table)
+    |  jnz >7
+    |2:
+    |  cmp RC, TAB:RB->asize
+    |  jae ->vmeta_tsetr
+    |  shl RC, 3
+    |  add RC, TAB:RB->array
+    |  // Set array slot.
+    |->BC_TSETR_Z:
+    |.if X64
+    |  mov RBa, [BASE+RA*8]
+    |  mov [RC], RBa
+    |.else
+    |  mov RB, [BASE+RA*8+4]
+    |  mov RA, [BASE+RA*8]
+    |  mov [RC+4], RB
+    |  mov [RC], RA
+    |.endif
+    |  ins_next
+    |
+    |7:  // Possible table write barrier for the value. Skip valiswhite check.
+    |  barrierback TAB:RB, RA
+    |  movzx RA, PC_RA                  // Restore RA.
+    |  jmp <2
+    break;
  case BC_TSETM:
    |  ins_AD   // RA = base (table at base-1), RD = num const (start index)
@@ -5386,10 +4917,8 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
    |.if DUALNUM
    |  mov dword [BASE+RA*8+4], LJ_TISNUM
    |  mov dword [BASE+RA*8], RC
-    |.elif SSE
-    |  cvtsi2sd xmm0, RC
    |.else
-    |  fild dword [BASE+RA*8-8]
+    |  cvtsi2sd xmm0, RC
    |.endif
    |  // Copy array slot to returned value.
    |.if X64
@@ -5405,10 +4934,8 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
    |  // Return array index as a numeric key.
    |.if DUALNUM
    |  // See above.
-    |.elif SSE
-    |  movsd qword [BASE+RA*8], xmm0
    |.else
-    |  fstp qword [BASE+RA*8]
+    |  movsd qword [BASE+RA*8], xmm0
    |.endif
    |  mov [BASE+RA*8-8], RC            // Update control var.
    |2:
@@ -5421,9 +4948,6 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
    |
    |4:  // Skip holes in array part.
    |  add RC, 1
-    |.if not (DUALNUM or SSE)
-    |  mov [BASE+RA*8-8], RC
-    |.endif
    |  jmp <1
    |
    |5:  // Traverse hash part.
@@ -5757,7 +5281,6 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
    if (!vk) {
      |  cmp RB, LJ_TISNUM; jae ->vmeta_for
    }
-    |.if SSE
    |  movsd xmm0, qword FOR_IDX
    |  movsd xmm1, qword FOR_STOP
    if (vk) {
@@ -5770,22 +5293,6 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
    |  ucomisd xmm1, xmm0
    |1:
    |  movsd qword FOR_EXT, xmm0
-    |.else
-    |  fld qword FOR_STOP
-    |  fld qword FOR_IDX
-    if (vk) {
-      |  fadd qword FOR_STEP            // nidx = idx + step
-      |  fst qword FOR_IDX
-      |  fst qword FOR_EXT
-      |  test RB, RB; js >1
-    } else {
-      |  fst qword FOR_EXT
-      |  jl >1
-    }
-    |  fxch                             // Swap lim/(n)idx if step non-negative.
-    |1:
-    |  fcomparepp
-    |.endif
    if (op == BC_FORI) {
      |.if DUALNUM
      |  jnb <7
@@ -5813,11 +5320,10 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
    |2:
    |  ins_next
    |.endif
-    |.if SSE
+    |
    |3:  // Invert comparison if step is negative.
    |  ucomisd xmm0, xmm1
    |  jmp <1
-    |.endif
    break;
  case BC_ITERL:
@@ -5875,6 +5381,7 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
    |  mov L:RB, SAVE_L
    |  mov [DISPATCH+DISPATCH_GL(jit_base)], BASE
    |  mov [DISPATCH+DISPATCH_GL(jit_L)], L:RB
+    |  mov [DISPATCH+DISPATCH_GL(tmpbuf.L)], L:RB
    |  // Save additional callee-save registers only used in compiled code.
    |.if X64WIN
    |  mov TMPQ, r12