14 files changed, 112 insertions, 5 deletions

diff --git a/src/jit/dis_arm64.lua b/src/jit/dis_arm64.lua
index ea7ca828..ad909fbd 100644
--- a/src/jit/dis_arm64.lua
+++ b/src/jit/dis_arm64.lua
@@ -1089,7 +1089,7 @@ local function disass_ins(ctx)
           last = "#"..(sf+32 - immr)
           operands[#operands] = last
           x = x + 1
-        elseif x >= immr then
+        else
           name = a2
           x = x - immr + 1
         end
diff --git a/src/jit/dump.lua b/src/jit/dump.lua
index 4806f4c4..03140196 100644
--- a/src/jit/dump.lua
+++ b/src/jit/dump.lua
@@ -288,7 +288,7 @@ local litname = {
   ["FREF  "] = vmdef.irfield,
   ["FPMATH"] = vmdef.irfpm,
   ["TMPREF"] = { [0] = "", "IN", "OUT", "INOUT", "", "", "OUT2", "INOUT2" },
-  ["BUFHDR"] = { [0] = "RESET", "APPEND" },
+  ["BUFHDR"] = { [0] = "RESET", "APPEND", "WRITE" },
   ["TOSTR "] = { [0] = "INT", "NUM", "CHAR" },
 }
 
diff --git a/src/lj_asm.c b/src/lj_asm.c
index 9dae2707..fae5b241 100644
--- a/src/lj_asm.c
+++ b/src/lj_asm.c
@@ -1145,6 +1145,9 @@ static void asm_gcstep(ASMState *as, IRIns *ir)
 /* -- Buffer operations --------------------------------------------------- */
 
 static void asm_tvptr(ASMState *as, Reg dest, IRRef ref, MSize mode);
+#if LJ_HASBUFFER
+static void asm_bufhdr_write(ASMState *as, Reg sb);
+#endif
 
 static void asm_bufhdr(ASMState *as, IRIns *ir)
 {
@@ -1172,6 +1175,11 @@ static void asm_bufhdr(ASMState *as, IRIns *ir)
     }
     break;
     }
+#if LJ_HASBUFFER
+  case IRBUFHDR_WRITE:
+    asm_bufhdr_write(as, sb);
+    break;
+#endif
   default: lj_assertA(0, "bad BUFHDR op2 %d", ir->op2); break;
   }
 #if LJ_TARGET_X86ORX64
diff --git a/src/lj_asm_arm.h b/src/lj_asm_arm.h
index 208c3203..96703d7d 100644
--- a/src/lj_asm_arm.h
+++ b/src/lj_asm_arm.h
@@ -501,6 +501,30 @@ static void asm_retf(ASMState *as, IRIns *ir)
   emit_lso(as, ARMI_LDR, RID_TMP, base, -4);
 }
 
+/* -- Buffer operations --------------------------------------------------- */
+
+#if LJ_HASBUFFER
+static void asm_bufhdr_write(ASMState *as, Reg sb)
+{
+  Reg tmp = ra_scratch(as, rset_exclude(RSET_GPR, sb));
+  IRIns irgc;
+  int32_t addr = i32ptr((void *)&J2G(as->J)->cur_L);
+  irgc.ot = IRT(0, IRT_PGC);  /* GC type. */
+  emit_storeofs(as, &irgc, RID_TMP, sb, offsetof(SBuf, L));
+  if ((as->flags & JIT_F_ARMV6T2)) {
+    emit_dnm(as, ARMI_BFI, RID_TMP, lj_fls(SBUF_MASK_FLAG), tmp);
+  } else {
+    emit_dnm(as, ARMI_ORR, RID_TMP, RID_TMP, tmp);
+    emit_dn(as, ARMI_AND|ARMI_K12|SBUF_MASK_FLAG, tmp, tmp);
+  }
+  emit_lso(as, ARMI_LDR, RID_TMP,
+           ra_allock(as, (addr & ~4095),
+                     rset_exclude(rset_exclude(RSET_GPR, sb), tmp)),
+           (addr & 4095));
+  emit_loadofs(as, &irgc, tmp, sb, offsetof(SBuf, L));
+}
+#endif
+
 /* -- Type conversions ---------------------------------------------------- */
 
 #if !LJ_SOFTFP
diff --git a/src/lj_asm_arm64.h b/src/lj_asm_arm64.h
index 5bd4e0d5..dde52347 100644
--- a/src/lj_asm_arm64.h
+++ b/src/lj_asm_arm64.h
@@ -522,6 +522,21 @@ static void asm_retf(ASMState *as, IRIns *ir)
   emit_lso(as, A64I_LDRx, RID_TMP, base, -8);
 }
 
+/* -- Buffer operations --------------------------------------------------- */
+
+#if LJ_HASBUFFER
+static void asm_bufhdr_write(ASMState *as, Reg sb)
+{
+  Reg tmp = ra_scratch(as, rset_exclude(RSET_GPR, sb));
+  IRIns irgc;
+  irgc.ot = IRT(0, IRT_PGC);  /* GC type. */
+  emit_storeofs(as, &irgc, RID_TMP, sb, offsetof(SBuf, L));
+  emit_dn(as, A64I_BFMx | A64F_IMMS(lj_fls(SBUF_MASK_FLAG)) | A64F_IMMR(0), RID_TMP, tmp);
+  emit_getgl(as, RID_TMP, cur_L);
+  emit_loadofs(as, &irgc, tmp, sb, offsetof(SBuf, L));
+}
+#endif
+
 /* -- Type conversions ---------------------------------------------------- */
 
 static void asm_tointg(ASMState *as, IRIns *ir, Reg left)
diff --git a/src/lj_asm_mips.h b/src/lj_asm_mips.h
index 058df8be..22aa88bf 100644
--- a/src/lj_asm_mips.h
+++ b/src/lj_asm_mips.h
@@ -465,6 +465,27 @@ static void asm_retf(ASMState *as, IRIns *ir)
   emit_tsi(as, MIPSI_AL, RID_TMP, base, -8);
 }
 
+/* -- Buffer operations --------------------------------------------------- */
+
+#if LJ_HASBUFFER
+static void asm_bufhdr_write(ASMState *as, Reg sb)
+{
+  Reg tmp = ra_scratch(as, rset_exclude(RSET_GPR, sb));
+  IRIns irgc;
+  irgc.ot = IRT(0, IRT_PGC);  /* GC type. */
+  emit_storeofs(as, &irgc, RID_TMP, sb, offsetof(SBuf, L));
+  if ((as->flags & JIT_F_MIPSXXR2)) {
+    emit_tsml(as, LJ_64 ? MIPSI_DINS : MIPSI_INS, RID_TMP, tmp,
+              lj_fls(SBUF_MASK_FLAG), 0);
+  } else {
+    emit_dst(as, MIPSI_OR, RID_TMP, RID_TMP, tmp);
+    emit_tsi(as, MIPSI_ANDI, tmp, tmp, SBUF_MASK_FLAG);
+  }
+  emit_getgl(as, RID_TMP, cur_L);
+  emit_loadofs(as, &irgc, tmp, sb, offsetof(SBuf, L));
+}
+#endif
+
 /* -- Type conversions ---------------------------------------------------- */
 
 #if !LJ_SOFTFP
diff --git a/src/lj_asm_ppc.h b/src/lj_asm_ppc.h
index caef8fd7..c27ee71c 100644
--- a/src/lj_asm_ppc.h
+++ b/src/lj_asm_ppc.h
@@ -392,6 +392,21 @@ static void asm_retf(ASMState *as, IRIns *ir)
   emit_tai(as, PPCI_LWZ, RID_TMP, base, -8);
 }
 
+/* -- Buffer operations --------------------------------------------------- */
+
+#if LJ_HASBUFFER
+static void asm_bufhdr_write(ASMState *as, Reg sb)
+{
+  Reg tmp = ra_scratch(as, rset_exclude(RSET_GPR, sb));
+  IRIns irgc;
+  irgc.ot = IRT(0, IRT_PGC);  /* GC type. */
+  emit_storeofs(as, &irgc, RID_TMP, sb, offsetof(SBuf, L));
+  emit_rot(as, PPCI_RLWIMI, RID_TMP, tmp, 0, 31-lj_fls(SBUF_MASK_FLAG), 31);
+  emit_getgl(as, RID_TMP, cur_L);
+  emit_loadofs(as, &irgc, tmp, sb, offsetof(SBuf, L));
+}
+#endif
+
 /* -- Type conversions ---------------------------------------------------- */
 
 #if !LJ_SOFTFP
diff --git a/src/lj_asm_x86.h b/src/lj_asm_x86.h
index b8abf9d6..c8ed46d2 100644
--- a/src/lj_asm_x86.h
+++ b/src/lj_asm_x86.h
@@ -791,6 +791,21 @@ static void asm_retf(ASMState *as, IRIns *ir)
 #endif
 }
 
+/* -- Buffer operations --------------------------------------------------- */
+
+#if LJ_HASBUFFER
+static void asm_bufhdr_write(ASMState *as, Reg sb)
+{
+  Reg tmp = ra_scratch(as, rset_exclude(RSET_GPR, sb));
+  IRIns irgc;
+  irgc.ot = IRT(0, IRT_PGC);  /* GC type. */
+  emit_storeofs(as, &irgc, tmp, sb, offsetof(SBuf, L));
+  emit_opgl(as, XO_ARITH(XOg_OR), tmp|REX_GC64, cur_L);
+  emit_gri(as, XG_ARITHi(XOg_AND), tmp, SBUF_MASK_FLAG);
+  emit_loadofs(as, &irgc, tmp, sb, offsetof(SBuf, L));
+}
+#endif
+
 /* -- Type conversions ---------------------------------------------------- */
 
 static void asm_tointg(ASMState *as, IRIns *ir, Reg left)
diff --git a/src/lj_emit_mips.h b/src/lj_emit_mips.h
index 7825eabb..c13615dd 100644
--- a/src/lj_emit_mips.h
+++ b/src/lj_emit_mips.h
@@ -70,7 +70,7 @@ static void emit_rotr(ASMState *as, Reg dest, Reg src, Reg tmp, uint32_t shift)
   }
 }
 
-#if LJ_64
+#if LJ_64 || LJ_HASBUFFER
 static void emit_tsml(ASMState *as, MIPSIns mi, Reg rt, Reg rs, uint32_t msb,
                       uint32_t lsb)
 {
diff --git a/src/lj_ir.h b/src/lj_ir.h
index af10cc4f..fa10f4bc 100644
--- a/src/lj_ir.h
+++ b/src/lj_ir.h
@@ -240,6 +240,7 @@ IRFLDEF(FLENUM)
 /* BUFHDR mode, stored in op2. */
 #define IRBUFHDR_RESET          0       /* Reset buffer. */
 #define IRBUFHDR_APPEND         1       /* Append to buffer. */
+#define IRBUFHDR_WRITE          2       /* Write to string buffer. */
 
 /* CONV mode, stored in op2. */
 #define IRCONV_SRCMASK          0x001f  /* Source IRType. */
diff --git a/src/lj_opt_fold.c b/src/lj_opt_fold.c
index 06731c7a..4d179f3f 100644
--- a/src/lj_opt_fold.c
+++ b/src/lj_opt_fold.c
@@ -579,7 +579,11 @@ LJFOLDF(kfold_strcmp)
 ** fragments left over from CSE are eliminated by DCE.
 */
 
-/* BUFHDR is emitted like a store, see below. */
+LJFOLD(BUFHDR any any)
+LJFOLDF(bufhdr_merge)
+{
+  return fins->op2 == IRBUFHDR_WRITE ? CSEFOLD : EMITFOLD;
+}
 
 LJFOLD(BUFPUT BUFHDR BUFSTR)
 LJFOLDF(bufput_append)
@@ -2432,7 +2436,6 @@ LJFOLD(TNEW any any)
 LJFOLD(TDUP any)
 LJFOLD(CNEW any any)
 LJFOLD(XSNEW any any)
-LJFOLD(BUFHDR any any)
 LJFOLDX(lj_ir_emit)
 
 /* ------------------------------------------------------------------------ */
diff --git a/src/lj_target_arm.h b/src/lj_target_arm.h
index f41f53e9..72516bc2 100644
--- a/src/lj_target_arm.h
+++ b/src/lj_target_arm.h
@@ -211,6 +211,7 @@ typedef enum ARMIns {
   /* ARMv6T2 */
   ARMI_MOVW = 0xe3000000,
   ARMI_MOVT = 0xe3400000,
+  ARMI_BFI = 0xe7c00010,
 
   /* VFP */
   ARMI_VMOV_D = 0xeeb00b40,
diff --git a/src/lj_target_arm64.h b/src/lj_target_arm64.h
index c2f71153..bf568a8d 100644
--- a/src/lj_target_arm64.h
+++ b/src/lj_target_arm64.h
@@ -210,6 +210,8 @@ typedef enum A64Ins {
 
   A64I_EXTRw = 0x13800000,
   A64I_EXTRx = 0x93c00000,
+  A64I_BFMw = 0x33000000,
+  A64I_BFMx = 0xb3400000,
   A64I_SBFMw = 0x13000000,
   A64I_SBFMx = 0x93400000,
   A64I_SXTBw = 0x13001c00,
diff --git a/src/lj_target_mips.h b/src/lj_target_mips.h
index 8d8f68ec..5da94605 100644
--- a/src/lj_target_mips.h
+++ b/src/lj_target_mips.h
@@ -256,6 +256,8 @@ typedef enum MIPSIns {
   MIPSI_ROTRV = 0x00000046,     /* MIPSXXR2 */
   MIPSI_DROTRV = 0x00000056,
 
+  MIPSI_INS = 0x7c000004,       /* MIPSXXR2 */
+
   MIPSI_SEB = 0x7c000420,       /* MIPSXXR2 */
   MIPSI_SEH = 0x7c000620,       /* MIPSXXR2 */
   MIPSI_WSBH = 0x7c0000a0,      /* MIPSXXR2 */