From b82fc3ddc032dfc3da813cda9715d356975922e7 Mon Sep 17 00:00:00 2001 From: Mike Pall Date: Tue, 19 May 2015 01:59:29 +0200 Subject: [PATCH] Bump table allocations retroactively if they grow later on. --- src/lj_jit.h | 14 ++++++++- src/lj_record.c | 76 +++++++++++++++++++++++++++++++++++++++++++++-- src/lj_tab.c | 6 ++-- src/lj_tab.h | 1 + src/lj_trace.c | 12 ++++++-- src/lj_traceerr.h | 1 + 6 files changed, 101 insertions(+), 9 deletions(-) diff --git a/src/lj_jit.h b/src/lj_jit.h index 4b51baeb..db3d89bb 100644 --- a/src/lj_jit.h +++ b/src/lj_jit.h @@ -290,6 +290,15 @@ typedef struct ScEvEntry { uint8_t dir; /* Direction. 1: +, 0: -. */ } ScEvEntry; +/* Reverse bytecode map (IRRef -> PC). Only for selected instructions. */ +typedef struct RBCHashEntry { + MRef pc; /* Bytecode PC. */ + IRRef ref; /* IR reference. */ +} RBCHashEntry; + +/* Number of slots in the reverse bytecode hash table. Must be a power of 2. */ +#define RBCHASH_SLOTS 8 + /* 128 bit SIMD constants. */ enum { LJ_KSIMD_ABS, @@ -364,8 +373,9 @@ typedef struct jit_State { PostProc postproc; /* Required post-processing after execution. */ #if LJ_SOFTFP || (LJ_32 && LJ_HASFFI) - int needsplit; /* Need SPLIT pass. */ + uint8_t needsplit; /* Need SPLIT pass. */ #endif + uint8_t retryrec; /* Retry recording. */ GCRef *trace; /* Array of traces. */ TraceNo freetrace; /* Start of scan for next free trace. */ @@ -382,6 +392,8 @@ typedef struct jit_State { uint32_t penaltyslot; /* Round-robin index into penalty slots. */ uint32_t prngstate; /* PRNG state. */ + RBCHashEntry rbchash[RBCHASH_SLOTS]; /* Reverse bytecode map. */ + BPropEntry bpropcache[BPROP_SLOTS]; /* Backpropagation cache slots. */ uint32_t bpropslot; /* Round-robin index into bpropcache slots. */ diff --git a/src/lj_record.c b/src/lj_record.c index 56038156..583f80aa 100644 --- a/src/lj_record.c +++ b/src/lj_record.c @@ -235,6 +235,8 @@ static void canonicalize_slots(jit_State *J) /* Stop recording. */ void lj_record_stop(jit_State *J, TraceLink linktype, TraceNo lnk) { + if (J->retryrec) + lj_trace_err(J, LJ_TRERR_RETRY); lj_trace_end(J); J->cur.linktype = (uint8_t)linktype; J->cur.link = (uint16_t)lnk; @@ -1127,6 +1129,60 @@ static void rec_mm_comp_cdata(jit_State *J, RecordIndex *ix, int op, MMS mm) /* -- Indexed access ------------------------------------------------------ */ +/* Bump table allocations in bytecode when they grow during recording. */ +static void rec_idx_bump(jit_State *J, RecordIndex *ix) +{ + RBCHashEntry *rbc = &J->rbchash[(ix->tab & (RBCHASH_SLOTS-1))]; + if (tref_ref(ix->tab) == rbc->ref) { + const BCIns *pc = mref(rbc->pc, const BCIns); + GCtab *tb = tabV(&ix->tabv); + uint32_t nhbits; + IRIns *ir; + if (!tvisnil(&ix->keyv)) + (void)lj_tab_set(J->L, tb, &ix->keyv); /* Grow table right now. */ + nhbits = tb->hmask > 0 ? lj_fls(tb->hmask)+1 : 0; + ir = IR(tref_ref(ix->tab)); + if (ir->o == IR_TNEW) { + uint32_t ah = bc_d(*pc); + uint32_t asize = ah & 0x7ff, hbits = ah >> 11; + if (nhbits > hbits) hbits = nhbits; + if (tb->asize > asize) { + asize = tb->asize <= 0x7ff ? tb->asize : 0x7ff; + } + if ((asize | (hbits<<11)) != ah) { /* Has the size changed? */ + /* Patch bytecode, but continue recording (for more patching). */ + setbc_d(pc, (asize | (hbits<<11))); + /* Patching TNEW operands is only safe if the trace is aborted. */ + ir->op1 = asize; ir->op2 = hbits; + J->retryrec = 1; /* Abort the trace at the end of recording. */ + } + } else if (ir->o == IR_TDUP) { + GCtab *tpl = gco2tab(proto_kgc(J->pt, ~(ptrdiff_t)bc_d(*pc))); + /* Grow template table, but preserve keys with nil values. */ + if (tb->asize > tpl->asize || (1u << nhbits)-1 > tpl->hmask) { + Node *node = noderef(tpl->node); + uint32_t i, hmask = tpl->hmask; + for (i = 0; i <= hmask; i++) { + if (!tvisnil(&node[i].key) && tvisnil(&node[i].val)) + settabV(J->L, &node[i].val, tpl); + } + if (!tvisnil(&ix->keyv) && tref_isk(ix->key)) { + TValue *o = lj_tab_set(J->L, tpl, &ix->keyv); + if (tvisnil(o)) settabV(J->L, o, tpl); + } + lj_tab_resize(J->L, tpl, tb->asize, nhbits); + hmask = tpl->hmask; + for (i = 0; i <= hmask; i++) { + /* This is safe, since template tables only hold immutable values. */ + if (tvistab(&node[i].val)) + setnilV(&node[i].val); + } + J->retryrec = 1; /* Abort the trace at the end of recording. */ + } + } + } +} + /* Record bounds-check. */ static void rec_idx_abc(jit_State *J, TRef asizeref, TRef ikey, uint32_t asize) { @@ -1352,6 +1408,8 @@ TRef lj_record_idx(jit_State *J, RecordIndex *ix) key = emitir(IRTN(IR_CONV), key, IRCONV_NUM_INT); xref = emitir(IRT(IR_NEWREF, IRT_P32), ix->tab, key); keybarrier = 0; /* NEWREF already takes care of the key barrier. */ + if ((J->flags & JIT_F_OPT_SINK)) /* Avoid a separate flag. */ + rec_idx_bump(J, ix); } } else if (!lj_opt_fwd_wasnonnil(J, loadop, tref_ref(xref))) { /* Cannot derive that the previous value was non-nil, must do checks. */ @@ -1390,9 +1448,16 @@ static void rec_tsetm(jit_State *J, BCReg ra, BCReg rn, int32_t i) { RecordIndex ix; cTValue *basev = J->L->base; - copyTV(J->L, &ix.tabv, &basev[ra-1]); + GCtab *t = tabV(&basev[ra-1]); + settabV(J->L, &ix.tabv, t); ix.tab = getslot(J, ra-1); ix.idxchain = 0; + if ((J->flags & JIT_F_OPT_SINK)) { + if (t->asize < i+rn-ra) + lj_tab_reasize(J->L, t, i+rn-ra); + setnilV(&ix.keyv); + rec_idx_bump(J, &ix); + } for (; ra < rn; i++, ra++) { setintV(&ix.keyv, i); ix.key = lj_ir_kint(J, i); @@ -1712,8 +1777,12 @@ static TRef rec_tnew(jit_State *J, uint32_t ah) { uint32_t asize = ah & 0x7ff; uint32_t hbits = ah >> 11; + TRef tr; if (asize == 0x7ff) asize = 0x801; - return emitir(IRTG(IR_TNEW, IRT_TAB), asize, hbits); + tr = emitir(IRTG(IR_TNEW, IRT_TAB), asize, hbits); + J->rbchash[(tr & (RBCHASH_SLOTS-1))].ref = tref_ref(tr); + setmref(J->rbchash[(tr & (RBCHASH_SLOTS-1))].pc, J->pc); + return tr; } /* -- Concatenation ------------------------------------------------------- */ @@ -2139,6 +2208,8 @@ void lj_record_ins(jit_State *J) case BC_TDUP: rc = emitir(IRTG(IR_TDUP, IRT_TAB), lj_ir_ktab(J, gco2tab(proto_kgc(J->pt, ~(ptrdiff_t)rc))), 0); + J->rbchash[(rc & (RBCHASH_SLOTS-1))].ref = tref_ref(rc); + setmref(J->rbchash[(rc & (RBCHASH_SLOTS-1))].pc, pc); break; /* -- Calls and vararg handling ----------------------------------------- */ @@ -2352,6 +2423,7 @@ void lj_record_setup(jit_State *J) /* Initialize state related to current trace. */ memset(J->slot, 0, sizeof(J->slot)); memset(J->chain, 0, sizeof(J->chain)); + memset(J->rbchash, 0, sizeof(J->rbchash)); memset(J->bpropcache, 0, sizeof(J->bpropcache)); J->scev.idx = REF_NIL; setmref(J->scev.pc, NULL); diff --git a/src/lj_tab.c b/src/lj_tab.c index a9f43835..88bf1089 100644 --- a/src/lj_tab.c +++ b/src/lj_tab.c @@ -246,7 +246,7 @@ void LJ_FASTCALL lj_tab_free(global_State *g, GCtab *t) /* -- Table resizing ------------------------------------------------------ */ /* Resize a table to fit the new array/hash part sizes. */ -static void resizetab(lua_State *L, GCtab *t, uint32_t asize, uint32_t hbits) +void lj_tab_resize(lua_State *L, GCtab *t, uint32_t asize, uint32_t hbits) { Node *oldnode = noderef(t->node); uint32_t oldasize = t->asize; @@ -383,7 +383,7 @@ static void rehashtab(lua_State *L, GCtab *t, cTValue *ek) asize += countint(ek, bins); na = bestasize(bins, &asize); total -= na; - resizetab(L, t, asize, hsize2hbits(total)); + lj_tab_resize(L, t, asize, hsize2hbits(total)); } #if LJ_HASFFI @@ -395,7 +395,7 @@ void lj_tab_rehash(lua_State *L, GCtab *t) void lj_tab_reasize(lua_State *L, GCtab *t, uint32_t nasize) { - resizetab(L, t, nasize+1, t->hmask > 0 ? lj_fls(t->hmask)+1 : 0); + lj_tab_resize(L, t, nasize+1, t->hmask > 0 ? lj_fls(t->hmask)+1 : 0); } /* -- Table getters ------------------------------------------------------- */ diff --git a/src/lj_tab.h b/src/lj_tab.h index 1da28bd9..7cf031be 100644 --- a/src/lj_tab.h +++ b/src/lj_tab.h @@ -44,6 +44,7 @@ LJ_FUNC void LJ_FASTCALL lj_tab_free(global_State *g, GCtab *t); #if LJ_HASFFI LJ_FUNC void lj_tab_rehash(lua_State *L, GCtab *t); #endif +LJ_FUNC void lj_tab_resize(lua_State *L, GCtab *t, uint32_t asize, uint32_t hbits); LJ_FUNCA void lj_tab_reasize(lua_State *L, GCtab *t, uint32_t nasize); /* Caveat: all getters except lj_tab_get() can return NULL! */ diff --git a/src/lj_trace.c b/src/lj_trace.c index 39ff0461..79c50b0a 100644 --- a/src/lj_trace.c +++ b/src/lj_trace.c @@ -394,6 +394,7 @@ static void trace_start(jit_State *J) J->guardemit.irt = 0; J->postproc = LJ_POST_NONE; lj_resetsplit(J); + J->retryrec = 0; setgcref(J->cur.startpt, obj2gco(J->pt)); L = J->L; @@ -510,10 +511,15 @@ static int trace_abort(jit_State *J) } /* Penalize or blacklist starting bytecode instruction. */ if (J->parent == 0 && !bc_isret(bc_op(J->cur.startins))) { - if (J->exitno == 0) - penalty_pc(J, &gcref(J->cur.startpt)->pt, mref(J->cur.startpc, BCIns), e); - else + if (J->exitno == 0) { + BCIns *startpc = mref(J->cur.startpc, BCIns); + if (e == LJ_TRERR_RETRY) + hotcount_set(J2GG(J), startpc+1, 1); /* Immediate retry. */ + else + penalty_pc(J, &gcref(J->cur.startpt)->pt, startpc, e); + } else { traceref(J, J->exitno)->link = J->exitno; /* Self-link is blacklisted. */ + } } /* Is there anything to abort? */ diff --git a/src/lj_traceerr.h b/src/lj_traceerr.h index 5a2fe8bb..0f38562b 100644 --- a/src/lj_traceerr.h +++ b/src/lj_traceerr.h @@ -12,6 +12,7 @@ TREDEF(TRACEOV, "trace too long") TREDEF(STACKOV, "trace too deep") TREDEF(SNAPOV, "too many snapshots") TREDEF(BLACKL, "blacklisted") +TREDEF(RETRY, "retry recording") TREDEF(NYIBC, "NYI: bytecode %d") /* Recording loop ops. */