1 files changed, 162 insertions, 51 deletions

diff --git a/src/lj_opt_split.c b/src/lj_opt_split.c
index c8439cb8..32160c9a 100644
--- a/src/lj_opt_split.c
+++ b/src/lj_opt_split.c
@@ -8,14 +8,15 @@
 
 #include "lj_obj.h"
 
-#if LJ_HASJIT && (LJ_SOFTFP || (LJ_32 && LJ_HASFFI))
+#if LJ_HASJIT && (LJ_SOFTFP32 || (LJ_32 && LJ_HASFFI))
 
 #include "lj_err.h"
-#include "lj_str.h"
+#include "lj_buf.h"
 #include "lj_ir.h"
 #include "lj_jit.h"
 #include "lj_ircall.h"
 #include "lj_iropt.h"
+#include "lj_dispatch.h"
 #include "lj_vm.h"
 
 /* SPLIT pass:
@@ -139,6 +140,7 @@ static IRRef split_call_l(jit_State *J, IRRef1 *hisubst, IRIns *oir,
   ir->prev = tmp = split_emit(J, IRTI(IR_CALLN), tmp, id);
   return split_emit(J, IRT(IR_HIOP, IRT_SOFTFP), tmp, tmp);
 }
+#endif
 
 /* Emit a CALLN with one split 64 bit argument and a 32 bit argument. */
 static IRRef split_call_li(jit_State *J, IRRef1 *hisubst, IRIns *oir,
@@ -155,7 +157,6 @@ static IRRef split_call_li(jit_State *J, IRRef1 *hisubst, IRIns *oir,
   ir->prev = tmp = split_emit(J, IRTI(IR_CALLN), tmp, id);
   return split_emit(J, IRT(IR_HIOP, IRT_SOFTFP), tmp, tmp);
 }
-#endif
 
 /* Emit a CALLN with two split 64 bit arguments. */
 static IRRef split_call_ll(jit_State *J, IRRef1 *hisubst, IRIns *oir,
@@ -192,9 +193,121 @@ static IRRef split_ptr(jit_State *J, IRIns *oir, IRRef ref)
     nref = ir->op1;
     if (ofs == 0) return nref;
   }
-  return split_emit(J, IRTI(IR_ADD), nref, lj_ir_kint(J, ofs));
+  return split_emit(J, IRT(IR_ADD, IRT_PTR), nref, lj_ir_kint(J, ofs));
+}
+
+#if LJ_HASFFI
+static IRRef split_bitshift(jit_State *J, IRRef1 *hisubst,
+                            IRIns *oir, IRIns *nir, IRIns *ir)
+{
+  IROp op = ir->o;
+  IRRef kref = nir->op2;
+  if (irref_isk(kref)) {  /* Optimize constant shifts. */
+    int32_t k = (IR(kref)->i & 63);
+    IRRef lo = nir->op1, hi = hisubst[ir->op1];
+    if (op == IR_BROL || op == IR_BROR) {
+      if (op == IR_BROR) k = (-k & 63);
+      if (k >= 32) { IRRef t = lo; lo = hi; hi = t; k -= 32; }
+      if (k == 0) {
+      passthrough:
+        J->cur.nins--;
+        ir->prev = lo;
+        return hi;
+      } else {
+        TRef k1, k2;
+        IRRef t1, t2, t3, t4;
+        J->cur.nins--;
+        k1 = lj_ir_kint(J, k);
+        k2 = lj_ir_kint(J, (-k & 31));
+        t1 = split_emit(J, IRTI(IR_BSHL), lo, k1);
+        t2 = split_emit(J, IRTI(IR_BSHL), hi, k1);
+        t3 = split_emit(J, IRTI(IR_BSHR), lo, k2);
+        t4 = split_emit(J, IRTI(IR_BSHR), hi, k2);
+        ir->prev = split_emit(J, IRTI(IR_BOR), t1, t4);
+        return split_emit(J, IRTI(IR_BOR), t2, t3);
+      }
+    } else if (k == 0) {
+      goto passthrough;
+    } else if (k < 32) {
+      if (op == IR_BSHL) {
+        IRRef t1 = split_emit(J, IRTI(IR_BSHL), hi, kref);
+        IRRef t2 = split_emit(J, IRTI(IR_BSHR), lo, lj_ir_kint(J, (-k&31)));
+        return split_emit(J, IRTI(IR_BOR), t1, t2);
+      } else {
+        IRRef t1 = ir->prev, t2;
+        lj_assertJ(op == IR_BSHR || op == IR_BSAR, "bad usage");
+        nir->o = IR_BSHR;
+        t2 = split_emit(J, IRTI(IR_BSHL), hi, lj_ir_kint(J, (-k&31)));
+        ir->prev = split_emit(J, IRTI(IR_BOR), t1, t2);
+        return split_emit(J, IRTI(op), hi, kref);
+      }
+    } else {
+      if (op == IR_BSHL) {
+        if (k == 32)
+          J->cur.nins--;
+        else
+          lo = ir->prev;
+        ir->prev = lj_ir_kint(J, 0);
+        return lo;
+      } else {
+        lj_assertJ(op == IR_BSHR || op == IR_BSAR, "bad usage");
+        if (k == 32) {
+          J->cur.nins--;
+          ir->prev = hi;
+        } else {
+          nir->op1 = hi;
+        }
+        if (op == IR_BSHR)
+          return lj_ir_kint(J, 0);
+        else
+          return split_emit(J, IRTI(IR_BSAR), hi, lj_ir_kint(J, 31));
+      }
+    }
+  }
+  return split_call_li(J, hisubst, oir, ir,
+                       op - IR_BSHL + IRCALL_lj_carith_shl64);
 }
 
+static IRRef split_bitop(jit_State *J, IRRef1 *hisubst,
+                         IRIns *nir, IRIns *ir)
+{
+  IROp op = ir->o;
+  IRRef hi, kref = nir->op2;
+  if (irref_isk(kref)) {  /* Optimize bit operations with lo constant. */
+    int32_t k = IR(kref)->i;
+    if (k == 0 || k == -1) {
+      if (op == IR_BAND) k = ~k;
+      if (k == 0) {
+        J->cur.nins--;
+        ir->prev = nir->op1;
+      } else if (op == IR_BXOR) {
+        nir->o = IR_BNOT;
+        nir->op2 = 0;
+      } else {
+        J->cur.nins--;
+        ir->prev = kref;
+      }
+    }
+  }
+  hi = hisubst[ir->op1];
+  kref = hisubst[ir->op2];
+  if (irref_isk(kref)) {  /* Optimize bit operations with hi constant. */
+    int32_t k = IR(kref)->i;
+    if (k == 0 || k == -1) {
+      if (op == IR_BAND) k = ~k;
+      if (k == 0) {
+        return hi;
+      } else if (op == IR_BXOR) {
+        return split_emit(J, IRTI(IR_BNOT), hi, 0);
+      } else {
+        return kref;
+      }
+    }
+  }
+  return split_emit(J, IRTI(op), hi, kref);
+}
+#endif
+
 /* Substitute references of a snapshot. */
 static void split_subst_snap(jit_State *J, SnapShot *snap, IRIns *oir)
 {
@@ -214,7 +327,7 @@ static void split_ir(jit_State *J)
   IRRef nins = J->cur.nins, nk = J->cur.nk;
   MSize irlen = nins - nk;
   MSize need = (irlen+1)*(sizeof(IRIns) + sizeof(IRRef1));
-  IRIns *oir = (IRIns *)lj_str_needbuf(J->L, &G(J->L)->tmpbuf, need);
+  IRIns *oir = (IRIns *)lj_buf_tmp(J->L, need);
   IRRef1 *hisubst;
   IRRef ref, snref;
   SnapShot *snap;
@@ -241,6 +354,8 @@ static void split_ir(jit_State *J)
       ir->prev = ref;  /* Identity substitution for loword. */
       hisubst[ref] = 0;
     }
+    if (irt_is64(ir->t) && ir->o != IR_KNULL)
+      ref++;
   }
 
   /* Process old IR instructions. */
@@ -285,35 +400,11 @@ static void split_ir(jit_State *J)
         hi = split_call_ll(J, hisubst, oir, ir, IRCALL_softfp_div);
         break;
       case IR_POW:
-        hi = split_call_li(J, hisubst, oir, ir, IRCALL_lj_vm_powi);
+        hi = split_call_ll(J, hisubst, oir, ir, IRCALL_pow);
         break;
       case IR_FPMATH:
-        /* Try to rejoin pow from EXP2, MUL and LOG2. */
-        if (nir->op2 == IRFPM_EXP2 && nir->op1 > J->loopref) {
-          IRIns *irp = IR(nir->op1);
-          if (irp->o == IR_CALLN && irp->op2 == IRCALL_softfp_mul) {
-            IRIns *irm4 = IR(irp->op1);
-            IRIns *irm3 = IR(irm4->op1);
-            IRIns *irm12 = IR(irm3->op1);
-            IRIns *irl1 = IR(irm12->op1);
-            if (irm12->op1 > J->loopref && irl1->o == IR_CALLN &&
-                irl1->op2 == IRCALL_lj_vm_log2) {
-              IRRef tmp = irl1->op1;  /* Recycle first two args from LOG2. */
-              IRRef arg3 = irm3->op2, arg4 = irm4->op2;
-              J->cur.nins--;
-              tmp = split_emit(J, IRT(IR_CARG, IRT_NIL), tmp, arg3);
-              tmp = split_emit(J, IRT(IR_CARG, IRT_NIL), tmp, arg4);
-              ir->prev = tmp = split_emit(J, IRTI(IR_CALLN), tmp, IRCALL_pow);
-              hi = split_emit(J, IRT(IR_HIOP, IRT_SOFTFP), tmp, tmp);
-              break;
-            }
-          }
-        }
         hi = split_call_l(J, hisubst, oir, ir, IRCALL_lj_vm_floor + ir->op2);
         break;
-      case IR_ATAN2:
-        hi = split_call_ll(J, hisubst, oir, ir, IRCALL_atan2);
-        break;
       case IR_LDEXP:
         hi = split_call_li(J, hisubst, oir, ir, IRCALL_ldexp);
         break;
@@ -321,7 +412,8 @@ static void split_ir(jit_State *J)
         nir->o = IR_CONV;  /* Pass through loword. */
         nir->op2 = (IRT_INT << 5) | IRT_INT;
         hi = split_emit(J, IRT(ir->o == IR_NEG ? IR_BXOR : IR_BAND, IRT_SOFTFP),
-                        hisubst[ir->op1], hisubst[ir->op2]);
+               hisubst[ir->op1],
+               lj_ir_kint(J, (int32_t)(0x7fffffffu + (ir->o == IR_NEG))));
         break;
       case IR_SLOAD:
         if ((nir->op2 & IRSLOAD_CONVERT)) {  /* Convert from int to number. */
@@ -336,15 +428,24 @@ static void split_ir(jit_State *J)
       case IR_STRTO:
         hi = split_emit(J, IRT(IR_HIOP, IRT_SOFTFP), nref, nref);
         break;
+      case IR_FLOAD:
+        lj_assertJ(ir->op1 == REF_NIL, "expected FLOAD from GG_State");
+        hi = lj_ir_kint(J, *(int32_t*)((char*)J2GG(J) + ir->op2 + LJ_LE*4));
+        nir->op2 += LJ_BE*4;
+        break;
       case IR_XLOAD: {
         IRIns inslo = *nir;  /* Save/undo the emit of the lo XLOAD. */
         J->cur.nins--;
         hi = split_ptr(J, oir, ir->op1);  /* Insert the hiref ADD. */
+#if LJ_BE
+        hi = split_emit(J, IRT(IR_XLOAD, IRT_INT), hi, ir->op2);
+        inslo.t.irt = IRT_SOFTFP | (inslo.t.irt & IRT_GUARD);
+#endif
         nref = lj_ir_nextins(J);
         nir = IR(nref);
-        *nir = inslo;  /* Re-emit lo XLOAD immediately before hi XLOAD. */
-        hi = split_emit(J, IRT(IR_XLOAD, IRT_SOFTFP), hi, ir->op2);
+        *nir = inslo;  /* Re-emit lo XLOAD. */
 #if LJ_LE
+        hi = split_emit(J, IRT(IR_XLOAD, IRT_SOFTFP), hi, ir->op2);
         ir->prev = nref;
 #else
         ir->prev = hi; hi = nref;
@@ -364,8 +465,9 @@ static void split_ir(jit_State *J)
           break;
         }
 #endif
-        lua_assert(st == IRT_INT ||
-                   (LJ_32 && LJ_HASFFI && (st == IRT_U32 || st == IRT_FLOAT)));
+        lj_assertJ(st == IRT_INT ||
+                   (LJ_32 && LJ_HASFFI && (st == IRT_U32 || st == IRT_FLOAT)),
+                   "bad source type for CONV");
         nir->o = IR_CALLN;
 #if LJ_32 && LJ_HASFFI
         nir->op2 = st == IRT_INT ? IRCALL_softfp_i2d :
@@ -395,7 +497,8 @@ static void split_ir(jit_State *J)
         hi = nir->op2;
         break;
       default:
-        lua_assert(ir->o <= IR_NE || ir->o == IR_MIN || ir->o == IR_MAX);
+        lj_assertJ(ir->o <= IR_NE || ir->o == IR_MIN || ir->o == IR_MAX,
+                   "bad IR op %d", ir->o);
         hi = split_emit(J, IRTG(IR_HIOP, IRT_SOFTFP),
                         hisubst[ir->op1], hisubst[ir->op2]);
         break;
@@ -438,8 +541,21 @@ static void split_ir(jit_State *J)
                            irt_isi64(ir->t) ? IRCALL_lj_carith_powi64 :
                                               IRCALL_lj_carith_powu64);
         break;
+      case IR_BNOT:
+        hi = split_emit(J, IRTI(IR_BNOT), hiref, 0);
+        break;
+      case IR_BSWAP:
+        ir->prev = split_emit(J, IRTI(IR_BSWAP), hiref, 0);
+        hi = nref;
+        break;
+      case IR_BAND: case IR_BOR: case IR_BXOR:
+        hi = split_bitop(J, hisubst, nir, ir);
+        break;
+      case IR_BSHL: case IR_BSHR: case IR_BSAR: case IR_BROL: case IR_BROR:
+        hi = split_bitshift(J, hisubst, oir, nir, ir);
+        break;
       case IR_FLOAD:
-        lua_assert(ir->op2 == IRFL_CDATA_INT64);
+        lj_assertJ(ir->op2 == IRFL_CDATA_INT64, "only INT64 supported");
         hi = split_emit(J, IRTI(IR_FLOAD), nir->op1, IRFL_CDATA_INT64_4);
 #if LJ_BE
         ir->prev = hi; hi = nref;
@@ -457,13 +573,9 @@ static void split_ir(jit_State *J)
       case IR_CONV: {  /* Conversion to 64 bit integer. Others handled below. */
         IRType st = (IRType)(ir->op2 & IRCONV_SRCMASK);
 #if LJ_SOFTFP
+        lj_assertJ(st != IRT_FLOAT, "bad CONV *64.float emitted");
         if (st == IRT_NUM) {  /* NUM to 64 bit int conv. */
-          hi = split_call_l(J, hisubst, oir, ir,
-                 irt_isi64(ir->t) ? IRCALL_fp64_d2l : IRCALL_fp64_d2ul);
-        } else if (st == IRT_FLOAT) {  /* FLOAT to 64 bit int conv. */
-          nir->o = IR_CALLN;
-          nir->op2 = irt_isi64(ir->t) ? IRCALL_fp64_f2l : IRCALL_fp64_f2ul;
-          hi = split_emit(J, IRTI(IR_HIOP), nref, nref);
+          hi = split_call_l(J, hisubst, oir, ir, IRCALL_lj_vm_num2u64);
         }
 #else
         if (st == IRT_NUM || st == IRT_FLOAT) {  /* FP to 64 bit int conv. */
@@ -505,7 +617,7 @@ static void split_ir(jit_State *J)
         hi = nir->op2;
         break;
       default:
-        lua_assert(ir->o <= IR_NE);  /* Comparisons. */
+        lj_assertJ(ir->o <= IR_NE, "bad IR op %d", ir->o);  /* Comparisons. */
         split_emit(J, IRTGI(IR_HIOP), hiref, hisubst[ir->op2]);
         break;
       }
@@ -529,7 +641,7 @@ static void split_ir(jit_State *J)
       tmp = split_emit(J, IRT(IR_CARG, IRT_NIL), hisubst[op1], oir[op1].prev);
 #endif
       ir->prev = split_emit(J, IRTI(IR_CALLN), tmp, IRCALL_lj_vm_tobit);
-    } else if (ir->o == IR_TOSTR) {
+    } else if (ir->o == IR_TOSTR || ir->o == IR_TMPREF) {
       if (hisubst[ir->op1]) {
         if (irref_isk(ir->op1))
           nir->op1 = ir->op1;
@@ -576,22 +688,21 @@ static void split_ir(jit_State *J)
           nir->op2 = st == IRT_INT ? IRCALL_softfp_i2f : IRCALL_softfp_ui2f;
         }
       } else if (st == IRT_FLOAT) {
+        lj_assertJ(!irt_isu32(ir->t), "bad CONV u32.fp emitted");
         nir->o = IR_CALLN;
-        nir->op2 = irt_isint(ir->t) ? IRCALL_softfp_f2i : IRCALL_softfp_f2ui;
+        nir->op2 = IRCALL_softfp_f2i;
       } else
 #endif
 #if LJ_SOFTFP
       if (st == IRT_NUM || (LJ_32 && LJ_HASFFI && st == IRT_FLOAT)) {
         if (irt_isguard(ir->t)) {
-          lua_assert(st == IRT_NUM && irt_isint(ir->t));
+          lj_assertJ(st == IRT_NUM && irt_isint(ir->t), "bad CONV types");
           J->cur.nins--;
           ir->prev = split_num2int(J, nir->op1, hisubst[ir->op1], 1);
         } else {
           split_call_l(J, hisubst, oir, ir,
 #if LJ_32 && LJ_HASFFI
-            st == IRT_NUM ?
-              (irt_isint(ir->t) ? IRCALL_softfp_d2i : IRCALL_softfp_d2ui) :
-              (irt_isint(ir->t) ? IRCALL_softfp_f2i : IRCALL_softfp_f2ui)
+            st == IRT_NUM ? IRCALL_softfp_d2i : IRCALL_softfp_f2i
 #else
             IRCALL_softfp_d2i
 #endif
@@ -714,7 +825,7 @@ void lj_opt_split(jit_State *J)
   if (!J->needsplit)
     J->needsplit = split_needsplit(J);
 #else
-  lua_assert(J->needsplit >= split_needsplit(J));  /* Verify flag. */
+  lj_assertJ(J->needsplit >= split_needsplit(J), "bad SPLIT state");
 #endif
   if (J->needsplit) {
     int errcode = lj_vm_cpcall(J->L, NULL, J, cpsplit);