diff --git a/src/lj_asm_arm.h b/src/lj_asm_arm.h index e7d2bf17..44dc02e4 100644 --- a/src/lj_asm_arm.h +++ b/src/lj_asm_arm.h @@ -2033,6 +2033,9 @@ static void asm_stack_restore(ASMState *as, SnapShot *snap) /* -- GC handling --------------------------------------------------------- */ +/* Marker to prevent patching the GC check exit. */ +#define ARM_NOPATCH_GC_CHECK (ARMI_BIC|ARMI_K12) + /* Check GC threshold and do one or more GC steps. */ static void asm_gc_check(ASMState *as) { @@ -2044,6 +2047,7 @@ static void asm_gc_check(ASMState *as) l_end = emit_label(as); /* Exit trace if in GCSatomic or GCSfinalize. Avoids syncing GC objects. */ asm_guardcc(as, CC_NE); /* Assumes asm_snap_prep() already done. */ + *--as->mcp = ARM_NOPATCH_GC_CHECK; emit_n(as, ARMI_CMP|ARMI_K12|0, RID_RET); args[0] = ASMREF_TMP1; /* global_State *g */ args[1] = ASMREF_TMP2; /* MSize steps */ @@ -2212,7 +2216,8 @@ void lj_asm_patchexit(jit_State *J, GCtrace *T, ExitNo exitno, MCode *target) /* Look for bl_cc exitstub, replace with b_cc target. */ uint32_t ins = *p; if ((ins & 0x0f000000u) == 0x0b000000u && ins < 0xf0000000u && - ((ins ^ (px-p)) & 0x00ffffffu) == 0) { + ((ins ^ (px-p)) & 0x00ffffffu) == 0 && + p[-1] != ARM_NOPATCH_GC_CHECK) { *p = (ins & 0xfe000000u) | (((target-p)-2) & 0x00ffffffu); cend = p+1; if (!cstart) cstart = p; diff --git a/src/lj_asm_arm64.h b/src/lj_asm_arm64.h index b1fd3acc..37b79cde 100644 --- a/src/lj_asm_arm64.h +++ b/src/lj_asm_arm64.h @@ -1794,6 +1794,10 @@ static void asm_stack_restore(ASMState *as, SnapShot *snap) /* -- GC handling --------------------------------------------------------- */ +/* Marker to prevent patching the GC check exit. */ +#define ARM64_NOPATCH_GC_CHECK \ + (A64I_ORRx|A64F_D(RID_TMP)|A64F_M(RID_TMP)|A64F_N(RID_TMP)) + /* Check GC threshold and do one or more GC steps. */ static void asm_gc_check(ASMState *as) { @@ -1805,6 +1809,7 @@ static void asm_gc_check(ASMState *as) l_end = emit_label(as); /* Exit trace if in GCSatomic or GCSfinalize. Avoids syncing GC objects. */ asm_guardcnb(as, A64I_CBNZ, RID_RET); /* Assumes asm_snap_prep() is done. */ + *--as->mcp = ARM64_NOPATCH_GC_CHECK; args[0] = ASMREF_TMP1; /* global_State *g */ args[1] = ASMREF_TMP2; /* MSize steps */ asm_gencall(as, ci, args); @@ -1972,6 +1977,7 @@ void lj_asm_patchexit(jit_State *J, GCtrace *T, ExitNo exitno, MCode *target) MCode *cstart = NULL; MCode *mcarea = lj_mcode_patch(J, p, 0); MCode *px = exitstub_trace_addr(T, exitno); + int patchlong = 1; /* Note: this assumes a trace exit is only ever patched once. */ for (; p < pe; p++) { /* Look for exitstub branch, replace with branch to target. */ @@ -1993,7 +1999,9 @@ void lj_asm_patchexit(jit_State *J, GCtrace *T, ExitNo exitno, MCode *target) } else if ((ins & 0x7e000000u) == 0x34000000u && ((ins ^ ((px-p)<<5)) & 0x00ffffe0u) == 0) { /* Patch cbz/cbnz, if within range. */ - if (A64F_S_OK(delta, 19)) { + if (p[-1] == ARM64_NOPATCH_GC_CHECK) { + patchlong = 0; + } else if (A64F_S_OK(delta, 19)) { *p = A64I_LE((ins & 0xff00001fu) | A64F_S19(delta)); if (!cstart) cstart = p; } @@ -2006,7 +2014,8 @@ void lj_asm_patchexit(jit_State *J, GCtrace *T, ExitNo exitno, MCode *target) } } } - { /* Always patch long-range branch in exit stub itself. */ + /* Always patch long-range branch in exit stub itself. Except, if we can't. */ + if (patchlong) { ptrdiff_t delta = target - px; lj_assertJ(A64F_S_OK(delta, 26), "branch target out of range"); *px = A64I_B | A64F_S26(delta); diff --git a/src/lj_asm_mips.h b/src/lj_asm_mips.h index 513bd5ca..8abb2544 100644 --- a/src/lj_asm_mips.h +++ b/src/lj_asm_mips.h @@ -2528,6 +2528,9 @@ static void asm_stack_restore(ASMState *as, SnapShot *snap) /* -- GC handling --------------------------------------------------------- */ +/* Marker to prevent patching the GC check exit. */ +#define MIPS_NOPATCH_GC_CHECK MIPSI_OR + /* Check GC threshold and do one or more GC steps. */ static void asm_gc_check(ASMState *as) { @@ -2543,6 +2546,7 @@ static void asm_gc_check(ASMState *as) args[0] = ASMREF_TMP1; /* global_State *g */ args[1] = ASMREF_TMP2; /* MSize steps */ asm_gencall(as, ci, args); + l_end[-3] = MIPS_NOPATCH_GC_CHECK; /* Replace the nop after the call. */ emit_tsi(as, MIPSI_AADDIU, ra_releasetmp(as, ASMREF_TMP1), RID_JGL, -32768); tmp = ra_releasetmp(as, ASMREF_TMP2); emit_loadi(as, tmp, as->gcsteps); @@ -2694,7 +2698,7 @@ void lj_asm_patchexit(jit_State *J, GCtrace *T, ExitNo exitno, MCode *target) #else (p[-1] & 0xff600000u) == MIPSI_BC1EQZ #endif - )) { + ) && p[-2] != MIPS_NOPATCH_GC_CHECK) { ptrdiff_t delta = target - p; if (((delta + 0x8000) >> 16) == 0) { /* Patch in-range branch. */ patchbranch: diff --git a/src/lj_asm_ppc.h b/src/lj_asm_ppc.h index 77ab09d6..0952ca26 100644 --- a/src/lj_asm_ppc.h +++ b/src/lj_asm_ppc.h @@ -2067,6 +2067,9 @@ static void asm_stack_restore(ASMState *as, SnapShot *snap) /* -- GC handling --------------------------------------------------------- */ +/* Marker to prevent patching the GC check exit. */ +#define PPC_NOPATCH_GC_CHECK PPCI_ORIS + /* Check GC threshold and do one or more GC steps. */ static void asm_gc_check(ASMState *as) { @@ -2078,6 +2081,7 @@ static void asm_gc_check(ASMState *as) l_end = emit_label(as); /* Exit trace if in GCSatomic or GCSfinalize. Avoids syncing GC objects. */ asm_guardcc(as, CC_NE); /* Assumes asm_snap_prep() already done. */ + *--as->mcp = PPC_NOPATCH_GC_CHECK; emit_ai(as, PPCI_CMPWI, RID_RET, 0); args[0] = ASMREF_TMP1; /* global_State *g */ args[1] = ASMREF_TMP2; /* MSize steps */ @@ -2217,7 +2221,7 @@ void lj_asm_patchexit(jit_State *J, GCtrace *T, ExitNo exitno, MCode *target) MCode *px = exitstub_trace_addr(T, exitno); MCode *cstart = NULL; MCode *mcarea = lj_mcode_patch(J, p, 0); - int clearso = 0; + int clearso = 0, patchlong = 1; for (; p < pe; p++) { /* Look for exitstub branch, try to replace with branch to target. */ uint32_t ins = *p; @@ -2229,7 +2233,9 @@ void lj_asm_patchexit(jit_State *J, GCtrace *T, ExitNo exitno, MCode *target) delta -= sizeof(MCode); } /* Many, but not all short-range branches can be patched directly. */ - if (((delta + 0x8000) >> 16) == 0) { + if (p[-1] == PPC_NOPATCH_GC_CHECK) { + patchlong = 0; + } else if (((delta + 0x8000) >> 16) == 0) { *p = (ins & 0xffdf0000u) | ((uint32_t)delta & 0xffffu) | ((delta & 0x8000) * (PPCF_Y/0x8000)); if (!cstart) cstart = p; @@ -2243,7 +2249,8 @@ void lj_asm_patchexit(jit_State *J, GCtrace *T, ExitNo exitno, MCode *target) if (!cstart) cstart = p; } } - { /* Always patch long-range branch in exit stub itself. */ + /* Always patch long-range branch in exit stub itself. Except, if we can't. */ + if (patchlong) { ptrdiff_t delta = (char *)target - (char *)px - clearso; lj_assertJ(((delta + 0x02000000) >> 26) == 0, "branch target out of range"); diff --git a/src/lj_asm_x86.h b/src/lj_asm_x86.h index 146371c1..4d1c85b3 100644 --- a/src/lj_asm_x86.h +++ b/src/lj_asm_x86.h @@ -3072,6 +3072,7 @@ void lj_asm_patchexit(jit_State *J, GCtrace *T, ExitNo exitno, MCode *target) MSize len = T->szmcode; MCode *px = exitstub_addr(J, exitno) - 6; MCode *pe = p+len-6; + MCode *pgc = NULL; #if LJ_GC64 uint32_t statei = (uint32_t)(GG_OFS(g.vmstate) - GG_OFS(dispatch)); #else @@ -3086,9 +3087,15 @@ void lj_asm_patchexit(jit_State *J, GCtrace *T, ExitNo exitno, MCode *target) break; } lj_assertJ(p < pe, "instruction length decoder failed"); - for (; p < pe; p += asm_x86_inslen(p)) - if ((*(uint16_t *)p & 0xf0ff) == 0x800f && p + *(int32_t *)(p+2) == px) + for (; p < pe; p += asm_x86_inslen(p)) { + if ((*(uint16_t *)p & 0xf0ff) == 0x800f && p + *(int32_t *)(p+2) == px && + p != pgc) { *(int32_t *)(p+2) = jmprel(J, p+6, target); + } else if (*p == XI_CALL && + (void *)(p+5+*(int32_t *)(p+1)) == (void *)lj_gc_step_jit) { + pgc = p+7; /* Do not patch GC check exit. */ + } + } lj_mcode_sync(T->mcode, T->mcode + T->szmcode); lj_mcode_patch(J, mcarea, 1); }