From 2e24770ed3241342a6a5896e05629a382d35262b Mon Sep 17 00:00:00 2001 From: Mike Pall Date: Sun, 25 Apr 2010 13:53:33 +0200 Subject: [PATCH] Simplify management of current trace. Drop lazy save. --- src/lj_asm.c | 4 +- src/lj_gc.c | 23 +++++----- src/lj_gdbjit.c | 4 +- src/lj_gdbjit.h | 4 +- src/lj_jit.h | 1 - src/lj_record.c | 12 +++--- src/lj_trace.c | 111 +++++++++++++++++++++++++----------------------- 7 files changed, 80 insertions(+), 79 deletions(-) diff --git a/src/lj_asm.c b/src/lj_asm.c index 5c564748..985f0cef 100644 --- a/src/lj_asm.c +++ b/src/lj_asm.c @@ -3029,7 +3029,7 @@ static void asm_head_root(ASMState *as) { int32_t spadj; asm_head_root_base(as); - emit_setgli(as, vmstate, (int32_t)as->J->curtrace); + emit_setgli(as, vmstate, (int32_t)as->T->traceno); spadj = asm_stack_adjust(as); as->T->spadjust = (uint16_t)spadj; emit_addptr(as, RID_ESP|REX_64, -spadj); @@ -3145,7 +3145,7 @@ static void asm_head_side(ASMState *as) } /* Store trace number and adjust stack frame relative to the parent. */ - emit_setgli(as, vmstate, (int32_t)as->J->curtrace); + emit_setgli(as, vmstate, (int32_t)as->T->traceno); emit_addptr(as, RID_ESP|REX_64, -spdelta); /* Restore target registers from parent spill slots. */ diff --git a/src/lj_gc.c b/src/lj_gc.c index 7c224759..bd31eabf 100644 --- a/src/lj_gc.c +++ b/src/lj_gc.c @@ -217,13 +217,12 @@ static void gc_traverse_func(global_State *g, GCfunc *fn) /* Mark a trace. */ static void gc_marktrace(global_State *g, TraceNo traceno) { - if (traceno && traceno != G2J(g)->curtrace) { - GCobj *o = obj2gco(traceref(G2J(g), traceno)); - if (iswhite(o)) { - white2gray(o); - setgcrefr(o->gch.gclist, g->gc.gray); - setgcref(g->gc.gray, o); - } + GCobj *o = obj2gco(traceref(G2J(g), traceno)); + lua_assert(traceno != G2J(g)->cur.traceno); + if (iswhite(o)) { + white2gray(o); + setgcrefr(o->gch.gclist, g->gc.gray); + setgcref(g->gc.gray, o); } } @@ -236,15 +235,15 @@ static void gc_traverse_trace(global_State *g, GCtrace *T) if (ir->o == IR_KGC) gc_markobj(g, ir_kgc(ir)); } - gc_marktrace(g, T->link); - gc_marktrace(g, T->nextroot); - gc_marktrace(g, T->nextside); + if (T->link) gc_marktrace(g, T->link); + if (T->nextroot) gc_marktrace(g, T->nextroot); + if (T->nextside) gc_marktrace(g, T->nextside); gc_markobj(g, gcref(T->startpt)); } /* The current trace is a GC root while not anchored in the prototype (yet). */ #define gc_traverse_curtrace(g) \ - { if (G2J(g)->curtrace != 0) gc_traverse_trace(g, &G2J(g)->cur); } + { if (G2J(g)->cur.traceno != 0) gc_traverse_trace(g, &G2J(g)->cur); } #else #define gc_traverse_curtrace(g) UNUSED(g) #endif @@ -261,7 +260,7 @@ static void gc_traverse_proto(global_State *g, GCproto *pt) for (i = 0; i < (ptrdiff_t)pt->sizevarinfo; i++) /* Mark names of locals. */ gc_mark_str(gco2str(gcref(proto_varinfo(pt)[i].name))); #if LJ_HASJIT - gc_marktrace(g, pt->trace); + if (pt->trace) gc_marktrace(g, pt->trace); #endif } diff --git a/src/lj_gdbjit.c b/src/lj_gdbjit.c index d66ef455..dc85c076 100644 --- a/src/lj_gdbjit.c +++ b/src/lj_gdbjit.c @@ -698,7 +698,7 @@ static void gdbjit_newentry(lua_State *L, GDBJITctx *ctx) } /* Add debug info for newly compiled trace and notify GDB. */ -void lj_gdbjit_addtrace(jit_State *J, GCtrace *T, TraceNo traceno) +void lj_gdbjit_addtrace(jit_State *J, GCtrace *T) { GDBJITctx ctx; lua_State *L = J->L; @@ -721,7 +721,7 @@ void lj_gdbjit_addtrace(jit_State *J, GCtrace *T, TraceNo traceno) ctx.filename++; else ctx.filename = "(string)"; - ctx.trname = lj_str_pushf(L, "TRACE_%d", traceno); + ctx.trname = lj_str_pushf(L, "TRACE_%d", T->traceno); L->top--; gdbjit_buildobj(&ctx); gdbjit_newentry(L, &ctx); diff --git a/src/lj_gdbjit.h b/src/lj_gdbjit.h index 037d3344..7824885b 100644 --- a/src/lj_gdbjit.h +++ b/src/lj_gdbjit.h @@ -11,11 +11,11 @@ #if LJ_HASJIT && defined(LUAJIT_USE_GDBJIT) -LJ_FUNC void lj_gdbjit_addtrace(jit_State *J, GCtrace *T, TraceNo traceno); +LJ_FUNC void lj_gdbjit_addtrace(jit_State *J, GCtrace *T); LJ_FUNC void lj_gdbjit_deltrace(jit_State *J, GCtrace *T); #else -#define lj_gdbjit_addtrace(J, T, tn) UNUSED(T) +#define lj_gdbjit_addtrace(J, T) UNUSED(T) #define lj_gdbjit_deltrace(J, T) UNUSED(T) #endif diff --git a/src/lj_jit.h b/src/lj_jit.h index 332596f0..be85ecfe 100644 --- a/src/lj_jit.h +++ b/src/lj_jit.h @@ -284,7 +284,6 @@ typedef struct jit_State { SnapEntry *snapmapbuf; /* Temp. snapshot map buffer. */ MSize sizesnapmap; /* Size of temp. snapshot map buffer. */ - TraceNo curtrace; /* Current trace number (if not 0). Kept in J->cur. */ GCRef *trace; /* Array of traces. */ TraceNo freetrace; /* Start of scan for next free trace. */ MSize sizetrace; /* Size of trace array. */ diff --git a/src/lj_record.c b/src/lj_record.c index a3f5b9c0..c71aa1a4 100644 --- a/src/lj_record.c +++ b/src/lj_record.c @@ -232,7 +232,7 @@ static void rec_stop(jit_State *J, TraceNo lnk) lj_trace_end(J); J->cur.link = (uint16_t)lnk; /* Looping back at the same stack level? */ - if (lnk == J->curtrace && J->framedepth + J->retdepth == 0) { + if (lnk == J->cur.traceno && J->framedepth + J->retdepth == 0) { if ((J->flags & JIT_F_OPT_LOOP)) /* Shall we try to create a loop? */ goto nocanon; /* Do not canonicalize or we lose the narrowing. */ if (J->cur.root) /* Otherwise ensure we always link to the root trace. */ @@ -442,7 +442,7 @@ static void rec_loop_interp(jit_State *J, const BCIns *pc, LoopEvent ev) /* Same loop? */ if (ev == LOOPEV_LEAVE) /* Must loop back to form a root trace. */ lj_trace_err(J, LJ_TRERR_LLEAVE); - rec_stop(J, J->curtrace); /* Root trace forms a loop. */ + rec_stop(J, J->cur.traceno); /* Root trace forms a loop. */ } else if (ev != LOOPEV_LEAVE) { /* Entering inner loop? */ /* It's usually better to abort here and wait until the inner loop ** is traced. But if the inner loop repeatedly didn't loop back, @@ -472,7 +472,7 @@ static void rec_loop_jit(jit_State *J, TraceNo lnk, LoopEvent ev) } else if (ev != LOOPEV_LEAVE) { /* Side trace enters a compiled loop. */ J->instunroll = 0; /* Cannot continue across a compiled loop op. */ if (J->pc == J->startpc && J->framedepth + J->retdepth == 0) - lnk = J->curtrace; /* Can form an extra loop. */ + lnk = J->cur.traceno; /* Can form an extra loop. */ rec_stop(J, lnk); /* Link to the loop. */ } /* Side trace continues across a loop that's left or not entered. */ } @@ -578,7 +578,7 @@ static void rec_ret(jit_State *J, BCReg rbase, ptrdiff_t gotresults) if (J->framedepth == 0 && J->pt && frame == J->L->base - 1) { if (check_downrec_unroll(J, pt)) { J->maxslot = (BCReg)(rbase + nresults); - rec_stop(J, J->curtrace); /* Down-recursion. */ + rec_stop(J, J->cur.traceno); /* Down-recursion. */ return; } lj_snap_add(J); @@ -1836,7 +1836,7 @@ static void check_call_unroll(jit_State *J) if (J->pc == J->startpc) { if (count + J->tailcalled > J->param[JIT_P_recunroll]) { J->pc++; - rec_stop(J, J->curtrace); /* Up-recursion or tail-recursion. */ + rec_stop(J, J->cur.traceno); /* Up-recursion or tail-recursion. */ } } else { if (count > J->param[JIT_P_callunroll]) @@ -1874,7 +1874,7 @@ static void rec_func_jit(jit_State *J, TraceNo lnk) rec_func_setup(J); J->instunroll = 0; /* Cannot continue across a compiled function. */ if (J->pc == J->startpc && J->framedepth + J->retdepth == 0) - lnk = J->curtrace; /* Can form an extra tail-recursive loop. */ + lnk = J->cur.traceno; /* Can form an extra tail-recursive loop. */ rec_stop(J, lnk); /* Link to the function. */ } diff --git a/src/lj_trace.c b/src/lj_trace.c index 4c9c4669..8419c9bb 100644 --- a/src/lj_trace.c +++ b/src/lj_trace.c @@ -50,9 +50,9 @@ void lj_trace_err_info(jit_State *J, TraceError e) /* -- Trace management ---------------------------------------------------- */ /* The current trace is first assembled in J->cur. The variable length -** arrays point to shared, growable buffers (J->irbuf etc.). The trace is -** kept in this state until a new trace needs to be created. Then the current -** trace and its data structures are copied to a new (compact) GCtrace object. +** arrays point to shared, growable buffers (J->irbuf etc.). When trace +** recording ends successfully, the current trace and its data structures +** are copied to a new (compact) GCtrace object. */ /* Find a free trace number. */ @@ -76,32 +76,35 @@ static TraceNo trace_findfree(jit_State *J) return J->freetrace; } -#define TRACE_COPYELEM(field, szfield, tp) \ - T2->field = (tp *)p; \ - memcpy(p, T->field, T->szfield*sizeof(tp)); \ - p += T->szfield*sizeof(tp); +#define TRACE_APPENDVEC(field, szfield, tp) \ + T->field = (tp *)p; \ + memcpy(p, J->cur.field, J->cur.szfield*sizeof(tp)); \ + p += J->cur.szfield*sizeof(tp); -/* Save a trace by copying and compacting it. */ -static GCtrace *trace_save(jit_State *J, GCtrace *T) +/* Save current trace by copying and compacting it. */ +static void trace_save(jit_State *J) { size_t sztr = ((sizeof(GCtrace)+7)&~7); - size_t szins = (T->nins-T->nk)*sizeof(IRIns); + size_t szins = (J->cur.nins-J->cur.nk)*sizeof(IRIns); size_t sz = sztr + szins + - T->nsnap*sizeof(SnapShot) + - T->nsnapmap*sizeof(SnapEntry); - GCtrace *T2 = lj_mem_newt(J->L, (MSize)sz, GCtrace); - char *p = (char *)T2 + sztr; - memcpy(T2, T, sizeof(GCtrace)); - setgcrefr(T2->nextgc, J2G(J)->gc.root); - setgcrefp(J2G(J)->gc.root, T2); - newwhite(J2G(J), T2); - T2->gct = ~LJ_TTRACE; - T2->ir = (IRIns *)p - T->nk; - memcpy(p, T->ir+T->nk, szins); + J->cur.nsnap*sizeof(SnapShot) + + J->cur.nsnapmap*sizeof(SnapEntry); + GCtrace *T = lj_mem_newt(J->L, (MSize)sz, GCtrace); + char *p = (char *)T + sztr; + memcpy(T, &J->cur, sizeof(GCtrace)); + setgcrefr(T->nextgc, J2G(J)->gc.root); + setgcrefp(J2G(J)->gc.root, T); + newwhite(J2G(J), T); + T->gct = ~LJ_TTRACE; + T->ir = (IRIns *)p - J->cur.nk; + memcpy(p, J->cur.ir+J->cur.nk, szins); p += szins; - TRACE_COPYELEM(snap, nsnap, SnapShot) - TRACE_COPYELEM(snapmap, nsnapmap, SnapEntry) - return T2; + TRACE_APPENDVEC(snap, nsnap, SnapShot) + TRACE_APPENDVEC(snapmap, nsnapmap, SnapEntry) + J->cur.traceno = 0; + setgcrefp(J->trace[T->traceno], T); + lj_gc_barriertrace(J2G(J), T->traceno); + lj_gdbjit_addtrace(J, T); } void LJ_FASTCALL lj_trace_free(global_State *g, GCtrace *T) @@ -225,7 +228,7 @@ int lj_trace_flushall(lua_State *L) setgcrefnull(J->trace[i]); } } - J->curtrace = 0; + J->cur.traceno = 0; J->freetrace = 0; /* Free the whole machine code and invalidate all exit stub groups. */ lj_mcode_free(J); @@ -254,13 +257,11 @@ void lj_trace_initstate(global_State *g) void lj_trace_freestate(global_State *g) { jit_State *J = G2J(g); - if (J->curtrace) - lj_gdbjit_deltrace(J, &J->cur); #ifdef LUA_USE_ASSERT { /* This assumes all traces have already been freed. */ ptrdiff_t i; for (i = 1; i < (ptrdiff_t)J->sizetrace; i++) - lua_assert(i == (ptrdiff_t)J->curtrace || traceref(J, i) == NULL); + lua_assert(i == (ptrdiff_t)J->cur.traceno || traceref(J, i) == NULL); } #endif lj_mcode_free(J); @@ -311,13 +312,7 @@ setpenalty: static void trace_start(jit_State *J) { lua_State *L; - - if (J->curtrace != 0 && traceref(J, J->curtrace) == &J->cur) { - TraceNo tr = J->curtrace; /* Save current trace. */ - setgcrefp(J->trace[tr], trace_save(J, &J->cur)); - J->curtrace = 0; - lj_gc_barriertrace(J2G(J), tr); - } + TraceNo traceno; if ((J->pt->flags & PROTO_NO_JIT)) { /* JIT disabled for this proto? */ if (J->parent == 0) { @@ -332,18 +327,18 @@ static void trace_start(jit_State *J) } /* Get a new trace number. */ - J->curtrace = trace_findfree(J); - if (LJ_UNLIKELY(J->curtrace == 0)) { /* No free trace? */ + traceno = trace_findfree(J); + if (LJ_UNLIKELY(traceno == 0)) { /* No free trace? */ lua_assert((J2G(J)->hookmask & HOOK_GC) == 0); lj_trace_flushall(J->L); J->state = LJ_TRACE_IDLE; /* Silently ignored. */ return; } - setgcrefp(J->trace[J->curtrace], &J->cur); + setgcrefp(J->trace[traceno], &J->cur); /* Setup enough of the current trace to be able to send the vmevent. */ memset(&J->cur, 0, sizeof(GCtrace)); - J->cur.traceno = J->curtrace; + J->cur.traceno = traceno; J->cur.nins = J->cur.nk = REF_BASE; J->cur.ir = J->irbuf; J->cur.snap = J->snapbuf; @@ -356,7 +351,7 @@ static void trace_start(jit_State *J) L = J->L; lj_vmevent_send(L, TRACE, setstrV(L, L->top++, lj_str_newlit(L, "start")); - setintV(L->top++, J->curtrace); + setintV(L->top++, traceno); setfuncV(L, L->top++, J->fn); setintV(L->top++, proto_bcpos(J->pt, J->pc)); if (J->parent) { @@ -373,6 +368,7 @@ static void trace_stop(jit_State *J) BCIns *pc = (BCIns *)J->startpc; /* Not const here. */ BCOp op = bc_op(J->cur.startins); GCproto *pt = &gcref(J->cur.startpt)->pt; + TraceNo traceno = J->cur.traceno; lua_State *L; switch (op) { @@ -384,16 +380,16 @@ static void trace_stop(jit_State *J) case BC_FUNCF: /* Patch bytecode of starting instruction in root trace. */ setbc_op(pc, (int)op+(int)BC_JLOOP-(int)BC_LOOP); - setbc_d(pc, J->curtrace); + setbc_d(pc, traceno); addroot: /* Add to root trace chain in prototype. */ J->cur.nextroot = pt->trace; - pt->trace = (TraceNo1)J->curtrace; + pt->trace = (TraceNo1)traceno; break; case BC_RET: case BC_RET0: case BC_RET1: - *pc = BCINS_AD(BC_JLOOP, J->cur.snap[0].nslots, J->curtrace); + *pc = BCINS_AD(BC_JLOOP, J->cur.snap[0].nslots, traceno); goto addroot; case BC_JMP: /* Patch exit branch in parent to side trace entry. */ @@ -406,7 +402,7 @@ static void trace_stop(jit_State *J) GCtrace *root = traceref(J, J->cur.root); root->nchild++; J->cur.nextside = root->nextside; - root->nextside = (TraceNo1)J->curtrace; + root->nextside = (TraceNo1)traceno; } break; default: @@ -416,12 +412,12 @@ static void trace_stop(jit_State *J) /* Commit new mcode only after all patching is done. */ lj_mcode_commit(J, J->cur.mcode); - lj_gdbjit_addtrace(J, &J->cur, J->curtrace); + trace_save(J); L = J->L; lj_vmevent_send(L, TRACE, setstrV(L, L->top++, lj_str_newlit(L, "stop")); - setintV(L->top++, J->curtrace); + setintV(L->top++, traceno); ); } @@ -445,6 +441,8 @@ static int trace_abort(jit_State *J) { lua_State *L = J->L; TraceError e = LJ_TRERR_RECERR; + TraceNo traceno; + lj_mcode_abort(J); if (tvisnum(L->top-1)) e = (TraceError)lj_num2int(numV(L->top-1)); @@ -455,14 +453,18 @@ static int trace_abort(jit_State *J) /* Penalize or blacklist starting bytecode instruction. */ if (J->parent == 0 && !bc_isret(bc_op(J->cur.startins))) penalty_pc(J, &gcref(J->cur.startpt)->pt, (BCIns *)J->startpc, e); - if (J->curtrace) { /* Is there anything to abort? */ + + /* Is there anything to abort? */ + traceno = J->cur.traceno; + if (traceno) { ptrdiff_t errobj = savestack(L, L->top-1); /* Stack may be resized. */ + J->cur.link = 0; lj_vmevent_send(L, TRACE, TValue *frame; const BCIns *pc; GCfunc *fn; setstrV(L, L->top++, lj_str_newlit(L, "abort")); - setintV(L->top++, J->curtrace); + setintV(L->top++, traceno); /* Find original Lua function call to generate a better error message. */ frame = J->L->base-1; pc = J->pc; @@ -477,10 +479,10 @@ static int trace_abort(jit_State *J) copyTV(L, L->top++, &J->errinfo); ); /* Drop aborted trace after the vmevent (which may still access it). */ - setgcrefnull(J->trace[J->curtrace]); - if (J->curtrace < J->freetrace) - J->freetrace = J->curtrace; - J->curtrace = 0; + setgcrefnull(J->trace[traceno]); + if (traceno < J->freetrace) + J->freetrace = traceno; + J->cur.traceno = 0; } L->top--; /* Remove error object */ if (e == LJ_TRERR_DOWNREC) @@ -517,7 +519,7 @@ static TValue *trace_state(lua_State *L, lua_CFunction dummy, void *ud) trace_pendpatch(J, 0); setvmstate(J2G(J), RECORD); lj_vmevent_send(L, RECORD, - setintV(L->top++, J->curtrace); + setintV(L->top++, J->cur.traceno); setfuncV(L, L->top++, J->fn); setintV(L->top++, J->pt ? (int32_t)proto_bcpos(J->pt, J->pc) : -1); setintV(L->top++, J->framedepth); @@ -529,10 +531,11 @@ static TValue *trace_state(lua_State *L, lua_CFunction dummy, void *ud) trace_pendpatch(J, 1); J->loopref = 0; if ((J->flags & JIT_F_OPT_LOOP) && - J->cur.link == J->curtrace && J->framedepth + J->retdepth == 0) { + J->cur.link == J->cur.traceno && J->framedepth + J->retdepth == 0) { setvmstate(J2G(J), OPT); lj_opt_dce(J); if (lj_opt_loop(J)) { /* Loop optimization failed? */ + J->cur.link = 0; J->loopref = J->cur.nins; J->state = LJ_TRACE_RECORD; /* Try to continue recording. */ break;