Merge branch 'master' into v2.1

author: Mike Pall <mike> 2020-09-27 17:20:37 +0200
committer: Mike Pall <mike> 2020-09-27 17:20:37 +0200
commit: 2e55a42c07f97fe535123675f24a4f5315a66d7c (patch)
tree: 9a71145e6619b422b9e16447a5096161b67164ce
parent: 03a7ebca4f6819658cdaa12ba3af54a17b8035e9 (diff)
parent: e8ec6fe996cf48ef23755581dafe372eb71ad75c (diff)
download: luajit-2e55a42c07f97fe535123675f24a4f5315a66d7c.tar.gz
luajit-2e55a42c07f97fe535123675f24a4f5315a66d7c.tar.bz2
luajit-2e55a42c07f97fe535123675f24a4f5315a66d7c.zip
5 files changed, 41 insertions, 9 deletions
diff --git a/src/lj_asm_arm.h b/src/lj_asm_arm.h
index e7d2bf17..44dc02e4 100644
--- a/src/lj_asm_arm.h
+++ b/src/lj_asm_arm.h
@@ -2033,6 +2033,9 @@ static void asm_stack_restore(ASMState *as, SnapShot *snap)
 /* -- GC handling --------------------------------------------------------- */
+/* Marker to prevent patching the GC check exit. */
+#define ARM_NOPATCH_GC_CHECK    (ARMI_BIC|ARMI_K12)
 /* Check GC threshold and do one or more GC steps. */
 static void asm_gc_check(ASMState *as)
 {
@@ -2044,6 +2047,7 @@ static void asm_gc_check(ASMState *as)
  l_end = emit_label(as);
  /* Exit trace if in GCSatomic or GCSfinalize. Avoids syncing GC objects. */
  asm_guardcc(as, CC_NE);  /* Assumes asm_snap_prep() already done. */
+  *--as->mcp = ARM_NOPATCH_GC_CHECK;
  emit_n(as, ARMI_CMP|ARMI_K12|0, RID_RET);
  args[0] = ASMREF_TMP1;  /* global_State *g */
  args[1] = ASMREF_TMP2;  /* MSize steps     */
@@ -2212,7 +2216,8 @@ void lj_asm_patchexit(jit_State *J, GCtrace *T, ExitNo exitno, MCode *target)
    /* Look for bl_cc exitstub, replace with b_cc target. */
    uint32_t ins = *p;
    if ((ins & 0x0f000000u) == 0x0b000000u && ins < 0xf0000000u &&
-        ((ins ^ (px-p)) & 0x00ffffffu) == 0) {
+        ((ins ^ (px-p)) & 0x00ffffffu) == 0 &&
+        p[-1] != ARM_NOPATCH_GC_CHECK) {
      *p = (ins & 0xfe000000u) | (((target-p)-2) & 0x00ffffffu);
      cend = p+1;
      if (!cstart) cstart = p;
diff --git a/src/lj_asm_arm64.h b/src/lj_asm_arm64.h
index b1fd3acc..37b79cde 100644
--- a/src/lj_asm_arm64.h
+++ b/src/lj_asm_arm64.h
@@ -1794,6 +1794,10 @@ static void asm_stack_restore(ASMState *as, SnapShot *snap)
 /* -- GC handling --------------------------------------------------------- */
+/* Marker to prevent patching the GC check exit. */
+#define ARM64_NOPATCH_GC_CHECK \
+  (A64I_ORRx|A64F_D(RID_TMP)|A64F_M(RID_TMP)|A64F_N(RID_TMP))
 /* Check GC threshold and do one or more GC steps. */
 static void asm_gc_check(ASMState *as)
 {
@@ -1805,6 +1809,7 @@ static void asm_gc_check(ASMState *as)
  l_end = emit_label(as);
  /* Exit trace if in GCSatomic or GCSfinalize. Avoids syncing GC objects. */
  asm_guardcnb(as, A64I_CBNZ, RID_RET); /* Assumes asm_snap_prep() is done. */
+  *--as->mcp = ARM64_NOPATCH_GC_CHECK;
  args[0] = ASMREF_TMP1;  /* global_State *g */
  args[1] = ASMREF_TMP2;  /* MSize steps     */
  asm_gencall(as, ci, args);
@@ -1972,6 +1977,7 @@ void lj_asm_patchexit(jit_State *J, GCtrace *T, ExitNo exitno, MCode *target)
  MCode *cstart = NULL;
  MCode *mcarea = lj_mcode_patch(J, p, 0);
  MCode *px = exitstub_trace_addr(T, exitno);
+  int patchlong = 1;
  /* Note: this assumes a trace exit is only ever patched once. */
  for (; p < pe; p++) {
    /* Look for exitstub branch, replace with branch to target. */
@@ -1993,7 +1999,9 @@ void lj_asm_patchexit(jit_State *J, GCtrace *T, ExitNo exitno, MCode *target)
    } else if ((ins & 0x7e000000u) == 0x34000000u &&
               ((ins ^ ((px-p)<<5)) & 0x00ffffe0u) == 0) {
      /* Patch cbz/cbnz, if within range. */
-      if (A64F_S_OK(delta, 19)) {
+      if (p[-1] == ARM64_NOPATCH_GC_CHECK) {
+        patchlong = 0;
+      } else if (A64F_S_OK(delta, 19)) {
        *p = A64I_LE((ins & 0xff00001fu) | A64F_S19(delta));
        if (!cstart) cstart = p;
      }
@@ -2006,7 +2014,8 @@ void lj_asm_patchexit(jit_State *J, GCtrace *T, ExitNo exitno, MCode *target)
      }
    }
  }
-  {  /* Always patch long-range branch in exit stub itself. */
+  /* Always patch long-range branch in exit stub itself. Except, if we can't. */
+  if (patchlong) {
    ptrdiff_t delta = target - px;
    lj_assertJ(A64F_S_OK(delta, 26), "branch target out of range");
    *px = A64I_B | A64F_S26(delta);
diff --git a/src/lj_asm_mips.h b/src/lj_asm_mips.h
index 513bd5ca..8abb2544 100644
--- a/src/lj_asm_mips.h
+++ b/src/lj_asm_mips.h
@@ -2528,6 +2528,9 @@ static void asm_stack_restore(ASMState *as, SnapShot *snap)
 /* -- GC handling --------------------------------------------------------- */
+/* Marker to prevent patching the GC check exit. */
+#define MIPS_NOPATCH_GC_CHECK   MIPSI_OR
 /* Check GC threshold and do one or more GC steps. */
 static void asm_gc_check(ASMState *as)
 {
@@ -2543,6 +2546,7 @@ static void asm_gc_check(ASMState *as)
  args[0] = ASMREF_TMP1;  /* global_State *g */
  args[1] = ASMREF_TMP2;  /* MSize steps     */
  asm_gencall(as, ci, args);
+  l_end[-3] = MIPS_NOPATCH_GC_CHECK;  /* Replace the nop after the call. */
  emit_tsi(as, MIPSI_AADDIU, ra_releasetmp(as, ASMREF_TMP1), RID_JGL, -32768);
  tmp = ra_releasetmp(as, ASMREF_TMP2);
  emit_loadi(as, tmp, as->gcsteps);
@@ -2694,7 +2698,7 @@ void lj_asm_patchexit(jit_State *J, GCtrace *T, ExitNo exitno, MCode *target)
 #else
           (p[-1] & 0xff600000u) == MIPSI_BC1EQZ
 #endif
-          )) {
+          ) && p[-2] != MIPS_NOPATCH_GC_CHECK) {
        ptrdiff_t delta = target - p;
        if (((delta + 0x8000) >> 16) == 0) {  /* Patch in-range branch. */
        patchbranch:
diff --git a/src/lj_asm_ppc.h b/src/lj_asm_ppc.h
index 77ab09d6..0952ca26 100644
--- a/src/lj_asm_ppc.h
+++ b/src/lj_asm_ppc.h
@@ -2067,6 +2067,9 @@ static void asm_stack_restore(ASMState *as, SnapShot *snap)
 /* -- GC handling --------------------------------------------------------- */
+/* Marker to prevent patching the GC check exit. */
+#define PPC_NOPATCH_GC_CHECK    PPCI_ORIS
 /* Check GC threshold and do one or more GC steps. */
 static void asm_gc_check(ASMState *as)
 {
@@ -2078,6 +2081,7 @@ static void asm_gc_check(ASMState *as)
  l_end = emit_label(as);
  /* Exit trace if in GCSatomic or GCSfinalize. Avoids syncing GC objects. */
  asm_guardcc(as, CC_NE);  /* Assumes asm_snap_prep() already done. */
+  *--as->mcp = PPC_NOPATCH_GC_CHECK;
  emit_ai(as, PPCI_CMPWI, RID_RET, 0);
  args[0] = ASMREF_TMP1;  /* global_State *g */
  args[1] = ASMREF_TMP2;  /* MSize steps     */
@@ -2217,7 +2221,7 @@ void lj_asm_patchexit(jit_State *J, GCtrace *T, ExitNo exitno, MCode *target)
  MCode *px = exitstub_trace_addr(T, exitno);
  MCode *cstart = NULL;
  MCode *mcarea = lj_mcode_patch(J, p, 0);
-  int clearso = 0;
+  int clearso = 0, patchlong = 1;
  for (; p < pe; p++) {
    /* Look for exitstub branch, try to replace with branch to target. */
    uint32_t ins = *p;
@@ -2229,7 +2233,9 @@ void lj_asm_patchexit(jit_State *J, GCtrace *T, ExitNo exitno, MCode *target)
        delta -= sizeof(MCode);
      }
      /* Many, but not all short-range branches can be patched directly. */
-      if (((delta + 0x8000) >> 16) == 0) {
+      if (p[-1] == PPC_NOPATCH_GC_CHECK) {
+        patchlong = 0;
+      } else if (((delta + 0x8000) >> 16) == 0) {
        *p = (ins & 0xffdf0000u) | ((uint32_t)delta & 0xffffu) |
             ((delta & 0x8000) * (PPCF_Y/0x8000));
        if (!cstart) cstart = p;
@@ -2243,7 +2249,8 @@ void lj_asm_patchexit(jit_State *J, GCtrace *T, ExitNo exitno, MCode *target)
      if (!cstart) cstart = p;
    }
  }
-  {  /* Always patch long-range branch in exit stub itself. */
+  /* Always patch long-range branch in exit stub itself. Except, if we can't. */
+  if (patchlong) {
    ptrdiff_t delta = (char *)target - (char *)px - clearso;
    lj_assertJ(((delta + 0x02000000) >> 26) == 0,
               "branch target out of range");
diff --git a/src/lj_asm_x86.h b/src/lj_asm_x86.h
index 146371c1..4d1c85b3 100644
--- a/src/lj_asm_x86.h
+++ b/src/lj_asm_x86.h
@@ -3072,6 +3072,7 @@ void lj_asm_patchexit(jit_State *J, GCtrace *T, ExitNo exitno, MCode *target)
  MSize len = T->szmcode;
  MCode *px = exitstub_addr(J, exitno) - 6;
  MCode *pe = p+len-6;
+  MCode *pgc = NULL;
 #if LJ_GC64
  uint32_t statei = (uint32_t)(GG_OFS(g.vmstate) - GG_OFS(dispatch));
 #else
@@ -3086,9 +3087,15 @@ void lj_asm_patchexit(jit_State *J, GCtrace *T, ExitNo exitno, MCode *target)
      break;
  }
  lj_assertJ(p < pe, "instruction length decoder failed");
-  for (; p < pe; p += asm_x86_inslen(p))
+  for (; p < pe; p += asm_x86_inslen(p)) {
-    if ((*(uint16_t *)p & 0xf0ff) == 0x800f && p + *(int32_t *)(p+2) == px)
+    if ((*(uint16_t *)p & 0xf0ff) == 0x800f && p + *(int32_t *)(p+2) == px &&
+        p != pgc) {
      *(int32_t *)(p+2) = jmprel(J, p+6, target);
+    } else if (*p == XI_CALL &&
+              (void *)(p+5+*(int32_t *)(p+1)) == (void *)lj_gc_step_jit) {
+      pgc = p+7;  /* Do not patch GC check exit. */
+    }
+  }
  lj_mcode_sync(T->mcode, T->mcode + T->szmcode);
  lj_mcode_patch(J, mcarea, 1);
 }
author	Mike Pall <mike>	2020-09-27 17:20:37 +0200
committer	Mike Pall <mike>	2020-09-27 17:20:37 +0200
commit	2e55a42c07f97fe535123675f24a4f5315a66d7c (patch)
tree	9a71145e6619b422b9e16447a5096161b67164ce
parent	03a7ebca4f6819658cdaa12ba3af54a17b8035e9 (diff)
parent	e8ec6fe996cf48ef23755581dafe372eb71ad75c (diff)
download	luajit-2e55a42c07f97fe535123675f24a4f5315a66d7c.tar.gz luajit-2e55a42c07f97fe535123675f24a4f5315a66d7c.tar.bz2 luajit-2e55a42c07f97fe535123675f24a4f5315a66d7c.zip