gh-mirror/mikepaul-LuaJIT
1
0
Fork 0
mirror of https://github.com/LuaJIT/LuaJIT.git synced 2025-02-07 23:24:09 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge branch 'master' into v2.1

Browse Source
This commit is contained in:
Mike Pall 2020-09-27 17:20:37 +02:00
commit 2e55a42c07
5 changed files with 41 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
src/lj_asm_arm.h
View File

@ -2033,6 +2033,9 @@ static void asm_stack_restore(ASMState *as, SnapShot *snap)
/* -- GC handling --------------------------------------------------------- */ /* -- GC handling --------------------------------------------------------- */
/* Marker to prevent patching the GC check exit. */
#define ARM_NOPATCH_GC_CHECK (ARMI_BIC|ARMI_K12)
/* Check GC threshold and do one or more GC steps. */ /* Check GC threshold and do one or more GC steps. */
static void asm_gc_check(ASMState *as) static void asm_gc_check(ASMState *as)
{ {
@ -2044,6 +2047,7 @@ static void asm_gc_check(ASMState *as)
l_end = emit_label(as); l_end = emit_label(as);
/* Exit trace if in GCSatomic or GCSfinalize. Avoids syncing GC objects. */ /* Exit trace if in GCSatomic or GCSfinalize. Avoids syncing GC objects. */
asm_guardcc(as, CC_NE); /* Assumes asm_snap_prep() already done. */ asm_guardcc(as, CC_NE); /* Assumes asm_snap_prep() already done. */
*--as->mcp = ARM_NOPATCH_GC_CHECK;
emit_n(as, ARMI_CMP|ARMI_K12|0, RID_RET); emit_n(as, ARMI_CMP|ARMI_K12|0, RID_RET);
args[0] = ASMREF_TMP1; /* global_State *g */ args[0] = ASMREF_TMP1; /* global_State *g */
args[1] = ASMREF_TMP2; /* MSize steps */ args[1] = ASMREF_TMP2; /* MSize steps */
@ -2212,7 +2216,8 @@ void lj_asm_patchexit(jit_State *J, GCtrace *T, ExitNo exitno, MCode *target)
/* Look for bl_cc exitstub, replace with b_cc target. */ /* Look for bl_cc exitstub, replace with b_cc target. */
uint32_t ins = *p; uint32_t ins = *p;
if ((ins & 0x0f000000u) == 0x0b000000u && ins < 0xf0000000u && if ((ins & 0x0f000000u) == 0x0b000000u && ins < 0xf0000000u &&
((ins ^ (px-p)) & 0x00ffffffu) == 0) { ((ins ^ (px-p)) & 0x00ffffffu) == 0 &&
p[-1] != ARM_NOPATCH_GC_CHECK) {
*p = (ins & 0xfe000000u) | (((target-p)-2) & 0x00ffffffu); *p = (ins & 0xfe000000u) | (((target-p)-2) & 0x00ffffffu);
cend = p+1; cend = p+1;
if (!cstart) cstart = p; if (!cstart) cstart = p;

13
src/lj_asm_arm64.h
View File

@ -1794,6 +1794,10 @@ static void asm_stack_restore(ASMState *as, SnapShot *snap)
/* -- GC handling --------------------------------------------------------- */ /* -- GC handling --------------------------------------------------------- */
/* Marker to prevent patching the GC check exit. */
#define ARM64_NOPATCH_GC_CHECK \
(A64I_ORRx|A64F_D(RID_TMP)|A64F_M(RID_TMP)|A64F_N(RID_TMP))
/* Check GC threshold and do one or more GC steps. */ /* Check GC threshold and do one or more GC steps. */
static void asm_gc_check(ASMState *as) static void asm_gc_check(ASMState *as)
{ {
@ -1805,6 +1809,7 @@ static void asm_gc_check(ASMState *as)
l_end = emit_label(as); l_end = emit_label(as);
/* Exit trace if in GCSatomic or GCSfinalize. Avoids syncing GC objects. */ /* Exit trace if in GCSatomic or GCSfinalize. Avoids syncing GC objects. */
asm_guardcnb(as, A64I_CBNZ, RID_RET); /* Assumes asm_snap_prep() is done. */ asm_guardcnb(as, A64I_CBNZ, RID_RET); /* Assumes asm_snap_prep() is done. */
*--as->mcp = ARM64_NOPATCH_GC_CHECK;
args[0] = ASMREF_TMP1; /* global_State *g */ args[0] = ASMREF_TMP1; /* global_State *g */
args[1] = ASMREF_TMP2; /* MSize steps */ args[1] = ASMREF_TMP2; /* MSize steps */
asm_gencall(as, ci, args); asm_gencall(as, ci, args);
@ -1972,6 +1977,7 @@ void lj_asm_patchexit(jit_State *J, GCtrace *T, ExitNo exitno, MCode *target)
MCode *cstart = NULL; MCode *cstart = NULL;
MCode *mcarea = lj_mcode_patch(J, p, 0); MCode *mcarea = lj_mcode_patch(J, p, 0);
MCode *px = exitstub_trace_addr(T, exitno); MCode *px = exitstub_trace_addr(T, exitno);
int patchlong = 1;
/* Note: this assumes a trace exit is only ever patched once. */ /* Note: this assumes a trace exit is only ever patched once. */
for (; p < pe; p++) { for (; p < pe; p++) {
/* Look for exitstub branch, replace with branch to target. */ /* Look for exitstub branch, replace with branch to target. */
@ -1993,7 +1999,9 @@ void lj_asm_patchexit(jit_State *J, GCtrace *T, ExitNo exitno, MCode *target)
} else if ((ins & 0x7e000000u) == 0x34000000u && } else if ((ins & 0x7e000000u) == 0x34000000u &&
((ins ^ ((px-p)<<5)) & 0x00ffffe0u) == 0) { ((ins ^ ((px-p)<<5)) & 0x00ffffe0u) == 0) {
/* Patch cbz/cbnz, if within range. */ /* Patch cbz/cbnz, if within range. */
if (A64F_S_OK(delta, 19)) { if (p[-1] == ARM64_NOPATCH_GC_CHECK) {
patchlong = 0;
} else if (A64F_S_OK(delta, 19)) {
*p = A64I_LE((ins & 0xff00001fu) | A64F_S19(delta)); *p = A64I_LE((ins & 0xff00001fu) | A64F_S19(delta));
if (!cstart) cstart = p; if (!cstart) cstart = p;
} }
@ -2006,7 +2014,8 @@ void lj_asm_patchexit(jit_State *J, GCtrace *T, ExitNo exitno, MCode *target)
} }
} }
} }
{ /* Always patch long-range branch in exit stub itself. */ /* Always patch long-range branch in exit stub itself. Except, if we can't. */
if (patchlong) {
ptrdiff_t delta = target - px; ptrdiff_t delta = target - px;
lj_assertJ(A64F_S_OK(delta, 26), "branch target out of range"); lj_assertJ(A64F_S_OK(delta, 26), "branch target out of range");
*px = A64I_B | A64F_S26(delta); *px = A64I_B | A64F_S26(delta);

6
src/lj_asm_mips.h
View File

@ -2528,6 +2528,9 @@ static void asm_stack_restore(ASMState *as, SnapShot *snap)
/* -- GC handling --------------------------------------------------------- */ /* -- GC handling --------------------------------------------------------- */
/* Marker to prevent patching the GC check exit. */
#define MIPS_NOPATCH_GC_CHECK MIPSI_OR
/* Check GC threshold and do one or more GC steps. */ /* Check GC threshold and do one or more GC steps. */
static void asm_gc_check(ASMState *as) static void asm_gc_check(ASMState *as)
{ {
@ -2543,6 +2546,7 @@ static void asm_gc_check(ASMState *as)
args[0] = ASMREF_TMP1; /* global_State *g */ args[0] = ASMREF_TMP1; /* global_State *g */
args[1] = ASMREF_TMP2; /* MSize steps */ args[1] = ASMREF_TMP2; /* MSize steps */
asm_gencall(as, ci, args); asm_gencall(as, ci, args);
l_end[-3] = MIPS_NOPATCH_GC_CHECK; /* Replace the nop after the call. */
emit_tsi(as, MIPSI_AADDIU, ra_releasetmp(as, ASMREF_TMP1), RID_JGL, -32768); emit_tsi(as, MIPSI_AADDIU, ra_releasetmp(as, ASMREF_TMP1), RID_JGL, -32768);
tmp = ra_releasetmp(as, ASMREF_TMP2); tmp = ra_releasetmp(as, ASMREF_TMP2);
emit_loadi(as, tmp, as->gcsteps); emit_loadi(as, tmp, as->gcsteps);
@ -2694,7 +2698,7 @@ void lj_asm_patchexit(jit_State *J, GCtrace *T, ExitNo exitno, MCode *target)
#else #else
(p[-1] & 0xff600000u) == MIPSI_BC1EQZ (p[-1] & 0xff600000u) == MIPSI_BC1EQZ
#endif #endif
)) { ) && p[-2] != MIPS_NOPATCH_GC_CHECK) {
ptrdiff_t delta = target - p; ptrdiff_t delta = target - p;
if (((delta + 0x8000) >> 16) == 0) { /* Patch in-range branch. */ if (((delta + 0x8000) >> 16) == 0) { /* Patch in-range branch. */
patchbranch: patchbranch:

13
src/lj_asm_ppc.h
View File

@ -2067,6 +2067,9 @@ static void asm_stack_restore(ASMState *as, SnapShot *snap)
/* -- GC handling --------------------------------------------------------- */ /* -- GC handling --------------------------------------------------------- */
/* Marker to prevent patching the GC check exit. */
#define PPC_NOPATCH_GC_CHECK PPCI_ORIS
/* Check GC threshold and do one or more GC steps. */ /* Check GC threshold and do one or more GC steps. */
static void asm_gc_check(ASMState *as) static void asm_gc_check(ASMState *as)
{ {
@ -2078,6 +2081,7 @@ static void asm_gc_check(ASMState *as)
l_end = emit_label(as); l_end = emit_label(as);
/* Exit trace if in GCSatomic or GCSfinalize. Avoids syncing GC objects. */ /* Exit trace if in GCSatomic or GCSfinalize. Avoids syncing GC objects. */
asm_guardcc(as, CC_NE); /* Assumes asm_snap_prep() already done. */ asm_guardcc(as, CC_NE); /* Assumes asm_snap_prep() already done. */
*--as->mcp = PPC_NOPATCH_GC_CHECK;
emit_ai(as, PPCI_CMPWI, RID_RET, 0); emit_ai(as, PPCI_CMPWI, RID_RET, 0);
args[0] = ASMREF_TMP1; /* global_State *g */ args[0] = ASMREF_TMP1; /* global_State *g */
args[1] = ASMREF_TMP2; /* MSize steps */ args[1] = ASMREF_TMP2; /* MSize steps */
@ -2217,7 +2221,7 @@ void lj_asm_patchexit(jit_State *J, GCtrace *T, ExitNo exitno, MCode *target)
MCode *px = exitstub_trace_addr(T, exitno); MCode *px = exitstub_trace_addr(T, exitno);
MCode *cstart = NULL; MCode *cstart = NULL;
MCode *mcarea = lj_mcode_patch(J, p, 0); MCode *mcarea = lj_mcode_patch(J, p, 0);
int clearso = 0; int clearso = 0, patchlong = 1;
for (; p < pe; p++) { for (; p < pe; p++) {
/* Look for exitstub branch, try to replace with branch to target. */ /* Look for exitstub branch, try to replace with branch to target. */
uint32_t ins = *p; uint32_t ins = *p;
@ -2229,7 +2233,9 @@ void lj_asm_patchexit(jit_State *J, GCtrace *T, ExitNo exitno, MCode *target)
delta -= sizeof(MCode); delta -= sizeof(MCode);
} }
/* Many, but not all short-range branches can be patched directly. */ /* Many, but not all short-range branches can be patched directly. */
if (((delta + 0x8000) >> 16) == 0) { if (p[-1] == PPC_NOPATCH_GC_CHECK) {
patchlong = 0;
} else if (((delta + 0x8000) >> 16) == 0) {
*p = (ins & 0xffdf0000u) | ((uint32_t)delta & 0xffffu) | *p = (ins & 0xffdf0000u) | ((uint32_t)delta & 0xffffu) |
((delta & 0x8000) * (PPCF_Y/0x8000)); ((delta & 0x8000) * (PPCF_Y/0x8000));
if (!cstart) cstart = p; if (!cstart) cstart = p;
@ -2243,7 +2249,8 @@ void lj_asm_patchexit(jit_State *J, GCtrace *T, ExitNo exitno, MCode *target)
if (!cstart) cstart = p; if (!cstart) cstart = p;
} }
} }
{ /* Always patch long-range branch in exit stub itself. */ /* Always patch long-range branch in exit stub itself. Except, if we can't. */
if (patchlong) {
ptrdiff_t delta = (char *)target - (char *)px - clearso; ptrdiff_t delta = (char *)target - (char *)px - clearso;
lj_assertJ(((delta + 0x02000000) >> 26) == 0, lj_assertJ(((delta + 0x02000000) >> 26) == 0,
"branch target out of range"); "branch target out of range");

11
src/lj_asm_x86.h
View File

@ -3072,6 +3072,7 @@ void lj_asm_patchexit(jit_State *J, GCtrace *T, ExitNo exitno, MCode *target)
MSize len = T->szmcode; MSize len = T->szmcode;
MCode *px = exitstub_addr(J, exitno) - 6; MCode *px = exitstub_addr(J, exitno) - 6;
MCode *pe = p+len-6; MCode *pe = p+len-6;
MCode *pgc = NULL;
#if LJ_GC64 #if LJ_GC64
uint32_t statei = (uint32_t)(GG_OFS(g.vmstate) - GG_OFS(dispatch)); uint32_t statei = (uint32_t)(GG_OFS(g.vmstate) - GG_OFS(dispatch));
#else #else
@ -3086,9 +3087,15 @@ void lj_asm_patchexit(jit_State *J, GCtrace *T, ExitNo exitno, MCode *target)
break; break;
} }
lj_assertJ(p < pe, "instruction length decoder failed"); lj_assertJ(p < pe, "instruction length decoder failed");
for (; p < pe; p += asm_x86_inslen(p)) for (; p < pe; p += asm_x86_inslen(p)) {
if ((*(uint16_t *)p & 0xf0ff) == 0x800f && p + *(int32_t *)(p+2) == px) if ((*(uint16_t *)p & 0xf0ff) == 0x800f && p + *(int32_t *)(p+2) == px &&
p != pgc) {
*(int32_t *)(p+2) = jmprel(J, p+6, target); *(int32_t *)(p+2) = jmprel(J, p+6, target);
} else if (*p == XI_CALL &&
(void *)(p+5+*(int32_t *)(p+1)) == (void *)lj_gc_step_jit) {
pgc = p+7; /* Do not patch GC check exit. */
}
}
lj_mcode_sync(T->mcode, T->mcode + T->szmcode); lj_mcode_sync(T->mcode, T->mcode + T->szmcode);
lj_mcode_patch(J, mcarea, 1); lj_mcode_patch(J, mcarea, 1);
} }