From 7c056488d96434fc9996ba8b2b2a684f03472b62 Mon Sep 17 00:00:00 2001 From: Mike Pall Date: Wed, 4 Jul 2012 21:16:06 +0200 Subject: [PATCH] Drop range limit for sunk stores relative to sunk allocation. --- src/jit/dump.lua | 2 +- src/lj_asm.c | 21 +++++++++++++++++++-- src/lj_opt_sink.c | 10 +++++----- src/lj_snap.c | 29 +++++++++++++++++++++++++---- 4 files changed, 50 insertions(+), 12 deletions(-) diff --git a/src/jit/dump.lua b/src/jit/dump.lua index 98933971..e5871b8d 100644 --- a/src/jit/dump.lua +++ b/src/jit/dump.lua @@ -378,7 +378,7 @@ local function ridsp_name(ridsp, ins) if not disass then disass = require("jit.dis_"..jit.arch) end local rid, slot = band(ridsp, 0xff), shr(ridsp, 8) if rid == 253 or rid == 254 then - return slot == 0 and " {sink" or format(" {%04d", ins-slot) + return (slot == 0 or slot == 255) and " {sink" or format(" {%04d", ins-slot) end if ridsp > 255 then return format("[%x]", slot*4) end if rid < 128 then return disass.regname(rid) end diff --git a/src/lj_asm.c b/src/lj_asm.c index 15685d85..6ea5bc93 100644 --- a/src/lj_asm.c +++ b/src/lj_asm.c @@ -778,6 +778,23 @@ static int asm_snap_canremat(ASMState *as) return 0; } +/* Check whether a sunk store corresponds to an allocation. */ +static int asm_sunk_store(ASMState *as, IRIns *ira, IRIns *irs) +{ + if (irs->s == 255) { + if (irs->o == IR_ASTORE || irs->o == IR_HSTORE || + irs->o == IR_FSTORE || irs->o == IR_XSTORE) { + IRIns *irk = IR(irs->op1); + if (irk->o == IR_AREF || irk->o == IR_HREFK) + irk = IR(irk->op1); + return (IR(irk->op1) == ira); + } + return 0; + } else { + return (ira + irs->s == irs); /* Quick check. */ + } +} + /* Allocate register or spill slot for a ref that escapes to a snapshot. */ static void asm_snap_alloc1(ASMState *as, IRRef ref) { @@ -795,8 +812,8 @@ static void asm_snap_alloc1(ASMState *as, IRRef ref) else { /* Allocate stored values for TNEW, TDUP and CNEW. */ IRIns *irs; lua_assert(ir->o == IR_TNEW || ir->o == IR_TDUP || ir->o == IR_CNEW); - for (irs = IR(as->curins); irs > ir; irs--) - if (irs->r == RID_SINK && ir + irs->s == irs) { + for (irs = IR(as->snapref-1); irs > ir; irs--) + if (irs->r == RID_SINK && asm_sunk_store(as, ir, irs)) { lua_assert(irs->o == IR_ASTORE || irs->o == IR_HSTORE || irs->o == IR_FSTORE || irs->o == IR_XSTORE); asm_snap_alloc1(as, irs->op2); diff --git a/src/lj_opt_sink.c b/src/lj_opt_sink.c index 937cccc7..b7a3af2d 100644 --- a/src/lj_opt_sink.c +++ b/src/lj_opt_sink.c @@ -32,8 +32,6 @@ static IRIns *sink_checkalloc(jit_State *J, IRIns *irs) ir = IR(ir->op1); if (!(ir->o == IR_TNEW || ir->o == IR_TDUP || ir->o == IR_CNEW)) return NULL; /* Not an allocation. */ - if (ir + 255 < irs) - return NULL; /* Out of range. */ return ir; /* Return allocation. */ } @@ -173,10 +171,12 @@ static void sink_sweep_ins(jit_State *J) switch (ir->o) { case IR_ASTORE: case IR_HSTORE: case IR_FSTORE: case IR_XSTORE: { IRIns *ira = sink_checkalloc(J, ir); - if (ira && !irt_ismarked(ira->t)) - ir->prev = REGSP(RID_SINK, (int)(ir - ira)); - else + if (ira && !irt_ismarked(ira->t)) { + int delta = (int)(ir - ira); + ir->prev = REGSP(RID_SINK, delta > 255 ? 255 : delta); + } else { ir->prev = REGSP_INIT; + } break; } case IR_NEWREF: diff --git a/src/lj_snap.c b/src/lj_snap.c index 9fae57d8..b9a82008 100644 --- a/src/lj_snap.c +++ b/src/lj_snap.c @@ -403,6 +403,27 @@ static TRef snap_pref(jit_State *J, GCtrace *T, SnapEntry *map, MSize nmax, return tr; } +/* Check whether a sunk store corresponds to an allocation. Slow path. */ +static int snap_sunk_store2(jit_State *J, IRIns *ira, IRIns *irs) +{ + if (irs->o == IR_ASTORE || irs->o == IR_HSTORE || + irs->o == IR_FSTORE || irs->o == IR_XSTORE) { + IRIns *irk = IR(irs->op1); + if (irk->o == IR_AREF || irk->o == IR_HREFK) + irk = IR(irk->op1); + return (IR(irk->op1) == ira); + } + return 0; +} + +/* Check whether a sunk store corresponds to an allocation. Fast path. */ +static LJ_AINLINE int snap_sunk_store(jit_State *J, IRIns *ira, IRIns *irs) +{ + if (irs->s != 255) + return (ira + irs->s == irs); /* Fast check. */ + return snap_sunk_store2(J, ira, irs); +} + /* Replay snapshot state to setup side trace. */ void lj_snap_replay(jit_State *J, GCtrace *T) { @@ -464,7 +485,7 @@ void lj_snap_replay(jit_State *J, GCtrace *T) } else { IRIns *irs; for (irs = ir+1; irs < irlast; irs++) - if (irs->r == RID_SINK && ir + irs->s == irs) { + if (irs->r == RID_SINK && snap_sunk_store(J, ir, irs)) { if (snap_pref(J, T, map, nent, seen, irs->op2) == 0) snap_pref(J, T, map, nent, seen, T->ir[irs->op2].op1); else if ((LJ_SOFTFP || (LJ_32 && LJ_HASFFI)) && @@ -504,7 +525,7 @@ void lj_snap_replay(jit_State *J, GCtrace *T) TRef tr = emitir(ir->ot, op1, op2); J->slot[snap_slot(sn)] = tr; for (irs = ir+1; irs < irlast; irs++) - if (irs->r == RID_SINK && ir + irs->s == irs) { + if (irs->r == RID_SINK && snap_sunk_store(J, ir, irs)) { IRIns *irr = &T->ir[irs->op1]; TRef val, key = irr->op2, tmp = tr; if (irr->o != IR_FREF) { @@ -700,7 +721,7 @@ static void snap_unsink(jit_State *J, GCtrace *T, ExitState *ex, } else { IRIns *irs, *irlast = &T->ir[T->snap[snapno].ref]; for (irs = ir+1; irs < irlast; irs++) - if (irs->r == RID_SINK && ir + irs->s == irs) { + if (irs->r == RID_SINK && snap_sunk_store(J, ir, irs)) { IRIns *iro = &T->ir[T->ir[irs->op1].op2]; uint8_t *p = (uint8_t *)cd; CTSize szs; @@ -733,7 +754,7 @@ static void snap_unsink(jit_State *J, GCtrace *T, ExitState *ex, settabV(J->L, o, t); irlast = &T->ir[T->snap[snapno].ref]; for (irs = ir+1; irs < irlast; irs++) - if (irs->r == RID_SINK && ir + irs->s == irs) { + if (irs->r == RID_SINK && snap_sunk_store(J, ir, irs)) { IRIns *irk = &T->ir[irs->op1]; TValue tmp, *val; lua_assert(irs->o == IR_ASTORE || irs->o == IR_HSTORE ||