FFI: Record 64 bit integer divide and modulo.

6 files changed, 86 insertions, 34 deletions

diff --git a/src/lj_asm.c b/src/lj_asm.c
index 441700d4..27c9ac31 100644
--- a/src/lj_asm.c
+++ b/src/lj_asm.c
@@ -3988,7 +3988,23 @@ static void asm_ir(ASMState *as, IRIns *ir)
     else
       asm_intarith(as, ir, XOg_X_IMUL);
     break;
-  case IR_DIV: asm_fparith(as, ir, XO_DIVSD); break;
+  case IR_DIV:
+#if LJ_64 && LJ_HASFFI
+    if (!irt_isnum(ir->t))
+      asm_arith64(as, ir, irt_isi64(ir->t) ? IRCALL_lj_carith_divi64 :
+                                             IRCALL_lj_carith_divu64);
+    else
+#endif
+      asm_fparith(as, ir, XO_DIVSD);
+    break;
+  case IR_MOD:
+#if LJ_64 && LJ_HASFFI
+    asm_arith64(as, ir, irt_isi64(ir->t) ? IRCALL_lj_carith_modi64 :
+                                           IRCALL_lj_carith_modu64);
+#else
+    lua_assert(0);
+#endif
+    break;
 
   case IR_NEG:
     if (irt_isnum(ir->t))
@@ -4168,6 +4184,14 @@ static void asm_setup_regsp(ASMState *as, GCtrace *T)
         as->modset = RSET_SCRATCH;
       break;
     case IR_POWI:
+      if (irt_isnum(ir->t)) {
+        ir->prev = REGSP_HINT(RID_XMM0);
+        if (inloop)
+          as->modset |= RSET_RANGE(RID_XMM0, RID_XMM1+1)|RID2RSET(RID_EAX);
+        continue;
+      }
+      /* fallthrough */
+    case IR_DIV: case IR_MOD:
 #if LJ_64 && LJ_HASFFI
       if (!irt_isnum(ir->t)) {
         ir->prev = REGSP_HINT(RID_RET);
@@ -4176,10 +4200,7 @@ static void asm_setup_regsp(ASMState *as, GCtrace *T)
         continue;
       }
 #endif
-      ir->prev = REGSP_HINT(RID_XMM0);
-      if (inloop)
-        as->modset |= RSET_RANGE(RID_XMM0, RID_XMM1+1)|RID2RSET(RID_EAX);
-      continue;
+      break;
     case IR_FPMATH:
       if (ir->op2 == IRFPM_EXP2) {  /* May be joined to lj_vm_pow_sse. */
         ir->prev = REGSP_HINT(RID_XMM0);
diff --git a/src/lj_carith.c b/src/lj_carith.c
index 134a61fb..bba9f6b9 100644
--- a/src/lj_carith.c
+++ b/src/lj_carith.c
@@ -148,21 +148,6 @@ static int carith_int64(lua_State *L, CTState *cts, CDArith *ca, MMS mm)
       setboolV(L->top-1,
                id == CTID_INT64 ? ((int64_t)u0 <= (int64_t)u1) : (u0 <= u1));
       return 1;
-    case MM_div: case MM_mod:
-      if (u1 == 0) {  /* Division by zero. */
-        if (u0 == 0)
-          setnanV(L->top-1);
-        else if (id == CTID_INT64 && (int64_t)u0 < 0)
-          setminfV(L->top-1);
-        else
-          setpinfV(L->top-1);
-        return 1;
-      } else if (id == CTID_INT64 && (int64_t)u1 == -1 &&
-                 u0 == U64x(80000000,00000000)) {  /* MIN64 / -1. */
-        if (mm == MM_div) id = CTID_UINT64; else u0 = 0;
-        mm = MM_unm;  /* Result is 0x8000000000000000ULL or 0LL. */
-      }
-      break;
     default: break;
     }
     cd = lj_cdata_new(cts, id, 8);
@@ -174,15 +159,15 @@ static int carith_int64(lua_State *L, CTState *cts, CDArith *ca, MMS mm)
     case MM_mul: *up = u0 * u1; break;
     case MM_div:
       if (id == CTID_INT64)
-        *up = (uint64_t)((int64_t)u0 / (int64_t)u1);
+        *up = (uint64_t)lj_carith_divi64((int64_t)u0, (int64_t)u1);
       else
-        *up = u0 / u1;
+        *up = lj_carith_divu64(u0, u1);
       break;
     case MM_mod:
       if (id == CTID_INT64)
-        *up = (uint64_t)((int64_t)u0 % (int64_t)u1);
+        *up = (uint64_t)lj_carith_modi64((int64_t)u0, (int64_t)u1);
       else
-        *up = u0 % u1;
+        *up = lj_carith_modu64(u0, u1);
       break;
     case MM_pow:
       if (id == CTID_INT64)
@@ -231,13 +216,43 @@ int lj_carith_op(lua_State *L, MMS mm)
 /* -- 64 bit integer arithmetic helpers ----------------------------------- */
 
 #if LJ_32
-/* Signed/unsigned 64 bit multiply. */
+/* Signed/unsigned 64 bit multiplication. */
 int64_t lj_carith_mul64(int64_t a, int64_t b)
 {
   return a * b;
 }
 #endif
 
+/* Unsigned 64 bit division. */
+uint64_t lj_carith_divu64(uint64_t a, uint64_t b)
+{
+  if (b == 0) return U64x(80000000,00000000);
+  return a / b;
+}
+
+/* Signed 64 bit division. */
+int64_t lj_carith_divi64(int64_t a, int64_t b)
+{
+  if (b == 0 || (a == (int64_t)U64x(80000000,00000000) && b == -1))
+    return U64x(80000000,00000000);
+  return a / b;
+}
+
+/* Unsigned 64 bit modulo. */
+uint64_t lj_carith_modu64(uint64_t a, uint64_t b)
+{
+  if (b == 0) return U64x(80000000,00000000);
+  return a % b;
+}
+
+/* Signed 64 bit modulo. */
+int64_t lj_carith_modi64(int64_t a, int64_t b)
+{
+  if (b == 0) return U64x(80000000,00000000);
+  if (a == (int64_t)U64x(80000000,00000000) && b == -1) return 0;
+  return a % b;
+}
+
 /* Unsigned 64 bit x^k. */
 uint64_t lj_carith_powu64(uint64_t x, uint64_t k)
 {
diff --git a/src/lj_carith.h b/src/lj_carith.h
index 14073603..faef7a07 100644
--- a/src/lj_carith.h
+++ b/src/lj_carith.h
@@ -15,6 +15,10 @@ LJ_FUNC int lj_carith_op(lua_State *L, MMS mm);
 #if LJ_32
 LJ_FUNC int64_t lj_carith_mul64(int64_t x, int64_t k);
 #endif
+LJ_FUNC uint64_t lj_carith_divu64(uint64_t a, uint64_t b);
+LJ_FUNC int64_t lj_carith_divi64(int64_t a, int64_t b);
+LJ_FUNC uint64_t lj_carith_modu64(uint64_t a, uint64_t b);
+LJ_FUNC int64_t lj_carith_modi64(int64_t a, int64_t b);
 LJ_FUNC uint64_t lj_carith_powu64(uint64_t x, uint64_t k);
 LJ_FUNC int64_t lj_carith_powi64(int64_t x, int64_t k);
 
diff --git a/src/lj_crecord.c b/src/lj_crecord.c
index 5eafa3a7..cd5c7d49 100644
--- a/src/lj_crecord.c
+++ b/src/lj_crecord.c
@@ -700,8 +700,6 @@ static TRef crec_arith_int64(jit_State *J, TRef *sp, CType **s, MMS mm)
       J->postproc = LJ_POST_FIXGUARD;
       return TREF_TRUE;
     } else {
-      if (mm == MM_div || mm == MM_mod)
-        return 0;  /* NYI: integer div, mod. */
       tr = emitir(IRT(mm+(int)IR_ADD-(int)MM_add, dt), sp[0], sp[1]);
     }
     dp = emitir(IRTG(IR_CNEW, IRT_CDATA), lj_ir_kint(J, id), TREF_NIL);
diff --git a/src/lj_ir.h b/src/lj_ir.h
index 286eb219..d2ad87f3 100644
--- a/src/lj_ir.h
+++ b/src/lj_ir.h
@@ -10,7 +10,7 @@
 
 /* -- IR instructions ----------------------------------------------------- */
 
-/* IR instruction definition. Order matters, see below. */
+/* IR instruction definition. Order matters, see below. ORDER IR */
 #define IRDEF(_) \
   /* Guarded assertions. */ \
   /* Must be properly aligned to flip opposites (^1) and (un)ordered (^4). */ \
@@ -61,21 +61,21 @@
   _(BROL,       N , ref, ref) \
   _(BROR,       N , ref, ref) \
   \
-  /* Arithmetic ops. ORDER ARITH (FPMATH/POWI take the space for MOD/POW). */ \
+  /* Arithmetic ops. ORDER ARITH */ \
   _(ADD,        C , ref, ref) \
   _(SUB,        N , ref, ref) \
   _(MUL,        C , ref, ref) \
   _(DIV,        N , ref, ref) \
-  \
-  _(FPMATH,     N , ref, lit) \
+  _(MOD,        N , ref, ref) \
   _(POWI,       N , ref, ref) \
-  \
   _(NEG,        N , ref, ref) \
+  \
   _(ABS,        N , ref, ref) \
   _(ATAN2,      N , ref, ref) \
   _(LDEXP,      N , ref, ref) \
   _(MIN,        C , ref, ref) \
   _(MAX,        C , ref, ref) \
+  _(FPMATH,     N , ref, lit) \
   \
   /* Overflow-checking arithmetic ops. */ \
   _(ADDOV,      C , ref, ref) \
@@ -266,6 +266,10 @@ typedef struct CCallInfo {
 #endif
 #define IRCALLDEF_FFI(_) \
   IRCALLDEF_FFI32(_) \
+  _(lj_carith_divi64,   ARG2_64,   N, I64, CCI_NOFPRCLOBBER) \
+  _(lj_carith_divu64,   ARG2_64,   N, U64, CCI_NOFPRCLOBBER) \
+  _(lj_carith_modi64,   ARG2_64,   N, I64, CCI_NOFPRCLOBBER) \
+  _(lj_carith_modu64,   ARG2_64,   N, U64, CCI_NOFPRCLOBBER) \
   _(lj_carith_powi64,   ARG2_64,   N, I64, CCI_NOFPRCLOBBER) \
   _(lj_carith_powu64,   ARG2_64,   N, U64, CCI_NOFPRCLOBBER)
 #else
@@ -584,12 +588,12 @@ typedef union IRIns {
 #define ir_kptr(ir) \
   check_exp((ir)->o == IR_KPTR || (ir)->o == IR_KKPTR, mref((ir)->ptr, void))
 
-LJ_STATIC_ASSERT((int)IRT_GUARD == (int)IRM_W);
-
 /* A store or any other op with a non-weak guard has a side-effect. */
 static LJ_AINLINE int ir_sideeff(IRIns *ir)
 {
   return (((ir->t.irt | ~IRT_GUARD) & lj_ir_mode[ir->o]) >= IRM_S);
 }
 
+LJ_STATIC_ASSERT((int)IRT_GUARD == (int)IRM_W);
+
 #endif
diff --git a/src/lj_opt_split.c b/src/lj_opt_split.c
index 3cb30514..f4339e85 100644
--- a/src/lj_opt_split.c
+++ b/src/lj_opt_split.c
@@ -197,6 +197,16 @@ static void split_ir(jit_State *J)
       case IR_MUL:
         hi = split_call64(J, hisubst, oir, ir, IRCALL_lj_carith_mul64);
         break;
+      case IR_DIV:
+        hi = split_call64(J, hisubst, oir, ir,
+                          irt_isi64(ir->t) ? IRCALL_lj_carith_divi64 :
+                                             IRCALL_lj_carith_divu64);
+        break;
+      case IR_MOD:
+        hi = split_call64(J, hisubst, oir, ir,
+                          irt_isi64(ir->t) ? IRCALL_lj_carith_modi64 :
+                                             IRCALL_lj_carith_modu64);
+        break;
       case IR_POWI:
         hi = split_call64(J, hisubst, oir, ir,
                           irt_isi64(ir->t) ? IRCALL_lj_carith_powi64 :