diff --git a/src/lj_ccallback.c b/src/lj_ccallback.c index c975c88f..37f45bf5 100644 --- a/src/lj_ccallback.c +++ b/src/lj_ccallback.c @@ -529,7 +529,7 @@ lua_State * LJ_FASTCALL lj_ccallback_enter(CTState *cts, void *cf) lua_State *L = cts->L; global_State *g = cts->g; lua_assert(L != NULL); - if (gcref(g->jit_L)) { + if (tvref(g->jit_base)) { setstrV(L, L->top++, lj_err_str(L, LJ_ERR_FFI_BADCBACK)); if (g->panic) g->panic(L); exit(EXIT_FAILURE); diff --git a/src/lj_err.c b/src/lj_err.c index 7ae87a82..3b44c984 100644 --- a/src/lj_err.c +++ b/src/lj_err.c @@ -453,7 +453,7 @@ LJ_NOINLINE void LJ_FASTCALL lj_err_throw(lua_State *L, int errcode) { global_State *g = G(L); lj_trace_abort(g); - setgcrefnull(g->jit_L); + setmref(g->jit_base, NULL); L->status = 0; #if LJ_UNWIND_EXT err_raise_ext(errcode); diff --git a/src/lj_gc.c b/src/lj_gc.c index c9eaf21a..8dbf2090 100644 --- a/src/lj_gc.c +++ b/src/lj_gc.c @@ -475,7 +475,7 @@ static void gc_finalize(lua_State *L) global_State *g = G(L); GCobj *o = gcnext(gcref(g->gc.mmudata)); cTValue *mo; - lua_assert(gcref(g->jit_L) == NULL); /* Must not be called on trace. */ + lua_assert(tvref(g->jit_base) == NULL); /* Must not be called on trace. */ /* Unchain from list of userdata to be finalized. */ if (o == gcref(g->gc.mmudata)) setgcrefnull(g->gc.mmudata); @@ -606,7 +606,7 @@ static size_t gc_onestep(lua_State *L) g->gc.state = GCSatomic; /* End of mark phase. */ return 0; case GCSatomic: - if (gcref(g->jit_L)) /* Don't run atomic phase on trace. */ + if (tvref(g->jit_base)) /* Don't run atomic phase on trace. */ return LJ_MAX_MEM; atomic(g, L); g->gc.state = GCSsweepstring; /* Start of sweep phase. */ @@ -640,7 +640,7 @@ static size_t gc_onestep(lua_State *L) } case GCSfinalize: if (gcref(g->gc.mmudata) != NULL) { - if (gcref(g->jit_L)) /* Don't call finalizers on trace. */ + if (tvref(g->jit_base)) /* Don't call finalizers on trace. */ return LJ_MAX_MEM; gc_finalize(L); /* Finalize one userdata object. */ if (g->gc.estimate > GCFINALIZECOST) @@ -697,7 +697,7 @@ void LJ_FASTCALL lj_gc_step_fixtop(lua_State *L) int LJ_FASTCALL lj_gc_step_jit(global_State *g, MSize steps) { lua_State *L = gco2th(gcref(g->jit_L)); - L->base = mref(G(L)->jit_base, TValue); + L->base = tvref(G(L)->jit_base); L->top = curr_topL(L); while (steps-- > 0 && lj_gc_step(L) == 0) ; diff --git a/src/lj_obj.h b/src/lj_obj.h index 5a05f38d..e97acef0 100644 --- a/src/lj_obj.h +++ b/src/lj_obj.h @@ -536,8 +536,8 @@ typedef struct global_State { lua_CFunction panic; /* Called as a last resort for errors. */ BCIns bc_cfunc_int; /* Bytecode for internal C function calls. */ BCIns bc_cfunc_ext; /* Bytecode for external C function calls. */ - GCRef jit_L; /* Current JIT code lua_State or NULL. */ - MRef jit_base; /* Current JIT code L->base. */ + GCRef jit_L; /* Current JIT code lua_State. */ + MRef jit_base; /* Current JIT code L->base or NULL. */ MRef ctype_state; /* Pointer to C type state. */ GCRef gcroot[GCROOT_MAX]; /* GC roots. */ } global_State; diff --git a/src/lj_state.c b/src/lj_state.c index 604ff886..486e66e3 100644 --- a/src/lj_state.c +++ b/src/lj_state.c @@ -60,7 +60,7 @@ static void resizestack(lua_State *L, MSize n) GCobj *up; lua_assert((MSize)(tvref(L->maxstack)-oldst)==L->stacksize-LJ_STACK_EXTRA-1); st = (TValue *)lj_mem_realloc(L, tvref(L->stack), - (MSize)(L->stacksize*sizeof(TValue)), + (MSize)(oldsize*sizeof(TValue)), (MSize)(realsize*sizeof(TValue))); setmref(L->stack, st); delta = (char *)st - (char *)oldst; @@ -68,12 +68,12 @@ static void resizestack(lua_State *L, MSize n) while (oldsize < realsize) /* Clear new slots. */ setnilV(st + oldsize++); L->stacksize = realsize; + if ((size_t)(mref(G(L)->jit_base, char) - (char *)oldst) < oldsize) + setmref(G(L)->jit_base, mref(G(L)->jit_base, char) + delta); L->base = (TValue *)((char *)L->base + delta); L->top = (TValue *)((char *)L->top + delta); for (up = gcref(L->openupval); up != NULL; up = gcnext(up)) setmref(gco2uv(up)->v, (TValue *)((char *)uvval(gco2uv(up)) + delta)); - if (obj2gco(L) == gcref(G(L)->jit_L)) - setmref(G(L)->jit_base, mref(G(L)->jit_base, char) + delta); } /* Relimit stack after error, in case the limit was overdrawn. */ @@ -90,7 +90,8 @@ void lj_state_shrinkstack(lua_State *L, MSize used) return; /* Avoid stack shrinking while handling stack overflow. */ if (4*used < L->stacksize && 2*(LJ_STACK_START+LJ_STACK_EXTRA) < L->stacksize && - obj2gco(L) != gcref(G(L)->jit_L)) /* Don't shrink stack of live trace. */ + /* Don't shrink stack of live trace. */ + (tvref(G(L)->jit_base) == NULL || obj2gco(L) != gcref(G(L)->jit_L))) resizestack(L, L->stacksize >> 1); } diff --git a/src/vm_arm.dasc b/src/vm_arm.dasc index 83341e41..b235967f 100644 --- a/src/vm_arm.dasc +++ b/src/vm_arm.dasc @@ -2107,9 +2107,9 @@ static void build_subroutines(BuildCtx *ctx) | ldr BASE, [DISPATCH, #DISPATCH_GL(jit_base)] | str CARG1, [DISPATCH, #DISPATCH_J(exitno)] | mov CARG4, #0 - | str L, [DISPATCH, #DISPATCH_J(L)] | str BASE, L->base - | str CARG4, [DISPATCH, #DISPATCH_GL(jit_L)] + | str L, [DISPATCH, #DISPATCH_J(L)] + | str CARG4, [DISPATCH, #DISPATCH_GL(jit_base)] | sub CARG1, DISPATCH, #-GG_DISP2J | mov CARG2, sp | bl extern lj_trace_exit // (jit_State *J, ExitState *ex) @@ -2133,8 +2133,9 @@ static void build_subroutines(BuildCtx *ctx) | ldr LFUNC:CARG2, [BASE, FRAME_FUNC] | str RC, SAVE_MULTRES | mov CARG3, #0 + | str BASE, L->base | ldr CARG2, LFUNC:CARG2->field_pc - | str CARG3, [DISPATCH, #DISPATCH_GL(jit_L)] + | str CARG3, [DISPATCH, #DISPATCH_GL(jit_base)] | mv_vmstate CARG4, INTERP | ldr KBASE, [CARG2, #PC2PROTO(k)] | // Modified copy of ins_next which handles function header dispatch, too. diff --git a/src/vm_mips.dasc b/src/vm_mips.dasc index 41ed303d..aad4fccf 100644 --- a/src/vm_mips.dasc +++ b/src/vm_mips.dasc @@ -2046,13 +2046,14 @@ static void build_subroutines(BuildCtx *ctx) | st_vmstate | sw TMP2, 16+32*8+29*4(sp) // Store sp in RID_SP. | lw L, DISPATCH_GL(jit_L)(DISPATCH) - | lw BASE, DISPATCH_GL(jit_base)(DISPATCH) + | lw BASE, DISPATCH_GL(jit_base)(DISPATCH) | load_got lj_trace_exit | sw L, DISPATCH_J(L)(DISPATCH) | sw ra, DISPATCH_J(parent)(DISPATCH) // Store trace number. + | sw BASE, L->base | sw TMP1, DISPATCH_J(exitno)(DISPATCH) // Store exit number. | addiu CARG1, DISPATCH, GG_DISP2J - | sw BASE, L->base + | sw r0, DISPATCH_GL(jit_base)(DISPATCH) | call_intern lj_trace_exit // (jit_State *J, ExitState *ex) |. addiu CARG2, sp, 16 | // Returns MULTRES (unscaled) or negated error code. @@ -2068,7 +2069,8 @@ static void build_subroutines(BuildCtx *ctx) |.if JIT | // CRET1 = MULTRES or negated error code, BASE, PC and JGL set. | lw L, SAVE_L - | addiu DISPATCH, JGL, -GG_DISP2G-32768 + | addiu DISPATCH, JGL, -GG_DISP2G-32768 + | sw BASE, L->base |1: | bltz CRET1, >3 // Check for error from exit. |. lw LFUNC:TMP1, FRAME_FUNC(BASE) @@ -2078,7 +2080,7 @@ static void build_subroutines(BuildCtx *ctx) | sw MULTRES, SAVE_MULTRES | mtc1 TMP3, TOBIT | lw TMP1, LFUNC:TMP1->pc - | sw r0, DISPATCH_GL(jit_L)(DISPATCH) + | sw r0, DISPATCH_GL(jit_base)(DISPATCH) | lw KBASE, PC2PROTO(k)(TMP1) | cvt.d.s TOBIT, TOBIT | // Modified copy of ins_next which handles function header dispatch, too. diff --git a/src/vm_ppc.dasc b/src/vm_ppc.dasc index a7707ccc..d009d005 100644 --- a/src/vm_ppc.dasc +++ b/src/vm_ppc.dasc @@ -2542,9 +2542,9 @@ static void build_subroutines(BuildCtx *ctx) | srwi CARG3, CARG3, 2 | stw L, DISPATCH_J(L)(DISPATCH) | subi CARG3, CARG3, 2 - | stw TMP1, DISPATCH_GL(jit_L)(DISPATCH) - | stw CARG4, DISPATCH_J(parent)(DISPATCH) | stp BASE, L->base + | stw CARG4, DISPATCH_J(parent)(DISPATCH) + | stw TMP1, DISPATCH_GL(jit_base)(DISPATCH) | addi CARG1, DISPATCH, GG_DISP2J | stw CARG3, DISPATCH_J(exitno)(DISPATCH) | addi CARG2, sp, 16 @@ -2568,6 +2568,7 @@ static void build_subroutines(BuildCtx *ctx) | // CARG1 = MULTRES or negated error code, BASE, PC and JGL set. | lwz L, SAVE_L | addi DISPATCH, JGL, -GG_DISP2G-32768 + | stp BASE, L->base |1: | cmpwi CARG1, 0 | blt >3 // Check for error from exit. @@ -2576,7 +2577,7 @@ static void build_subroutines(BuildCtx *ctx) | li TMP2, 0 | stw MULTRES, SAVE_MULTRES | lwz TMP1, LFUNC:TMP1->pc - | stw TMP2, DISPATCH_GL(jit_L)(DISPATCH) + | stw TMP2, DISPATCH_GL(jit_base)(DISPATCH) | lwz KBASE, PC2PROTO(k)(TMP1) | // Setup type comparison constants. | li TISNUM, LJ_TISNUM diff --git a/src/vm_x86.dasc b/src/vm_x86.dasc index 62a5e139..b8f1c449 100644 --- a/src/vm_x86.dasc +++ b/src/vm_x86.dasc @@ -2708,7 +2708,6 @@ static void build_subroutines(BuildCtx *ctx) | mov L:RB, [DISPATCH+DISPATCH_GL(jit_L)] | mov BASE, [DISPATCH+DISPATCH_GL(jit_base)] | mov aword [DISPATCH+DISPATCH_J(L)], L:RBa - | mov dword [DISPATCH+DISPATCH_GL(jit_L)], 0 | mov L:RB->base, BASE |.if X64WIN | lea CARG2, [rsp+4*8] @@ -2718,6 +2717,7 @@ static void build_subroutines(BuildCtx *ctx) | lea FCARG2, [esp+16] |.endif | lea FCARG1, [DISPATCH+GG_DISP2J] + | mov dword [DISPATCH+DISPATCH_GL(jit_base)], 0 | call extern lj_trace_exit@8 // (jit_State *J, ExitState *ex) | // MULTRES or negated error code returned in eax (RD). | mov RAa, L:RB->cframe @@ -2765,11 +2765,13 @@ static void build_subroutines(BuildCtx *ctx) | mov r12, TMPQ |.endif | test RD, RD; js >3 // Check for error from exit. + | mov L:RB, SAVE_L | mov MULTRES, RD | mov LFUNC:KBASE, [BASE-8] | mov KBASE, LFUNC:KBASE->pc | mov KBASE, [KBASE+PC2PROTO(k)] - | mov dword [DISPATCH+DISPATCH_GL(jit_L)], 0 + | mov L:RB->base, BASE + | mov dword [DISPATCH+DISPATCH_GL(jit_base)], 0 | set_vmstate INTERP | // Modified copy of ins_next which handles function header dispatch, too. | mov RC, [PC]