diff --git a/src/lj_asm.c b/src/lj_asm.c index 3912bbeb..48b6ec5a 100644 --- a/src/lj_asm.c +++ b/src/lj_asm.c @@ -931,7 +931,7 @@ static void asm_snap_alloc(ASMState *as) IRRef ref = snap_ref(map[n]); if (!irref_isk(ref)) { IRIns *ir = IR(ref); - if (!ra_used(ir) && ir->o != IR_FRAME) { + if (!ra_used(ir)) { RegSet allow = irt_isnum(ir->t) ? RSET_FPR : RSET_GPR; /* Not a var-to-invar ref and got a free register (or a remat)? */ if ((!iscrossref(as, ref) || irt_isphi(ir->t)) && @@ -2831,27 +2831,25 @@ static void asm_head_side(ASMState *as) /* Scan all parent SLOADs and collect register dependencies. */ for (i = as->curins; i > REF_BASE; i--) { IRIns *ir = IR(i); - lua_assert((ir->o == IR_SLOAD && (ir->op2 & IRSLOAD_PARENT)) || - ir->o == IR_FRAME); - if (ir->o == IR_SLOAD) { - RegSP rs = as->parentmap[ir->op1]; - if (ra_hasreg(ir->r)) { - rset_clear(allow, ir->r); - if (ra_hasspill(ir->s)) - ra_save(as, ir, ir->r); - } else if (ra_hasspill(ir->s)) { - irt_setmark(ir->t); - pass2 = 1; - } - if (ir->r == rs) { /* Coalesce matching registers right now. */ - ra_free(as, ir->r); - } else if (ra_hasspill(regsp_spill(rs))) { - if (ra_hasreg(ir->r)) - pass3 = 1; - } else if (ra_used(ir)) { - sloadins[rs] = (IRRef1)i; - rset_set(live, rs); /* Block live parent register. */ - } + RegSP rs; + lua_assert(ir->o == IR_SLOAD && (ir->op2 & IRSLOAD_PARENT)); + rs = as->parentmap[ir->op1]; + if (ra_hasreg(ir->r)) { + rset_clear(allow, ir->r); + if (ra_hasspill(ir->s)) + ra_save(as, ir, ir->r); + } else if (ra_hasspill(ir->s)) { + irt_setmark(ir->t); + pass2 = 1; + } + if (ir->r == rs) { /* Coalesce matching registers right now. */ + ra_free(as, ir->r); + } else if (ra_hasspill(regsp_spill(rs))) { + if (ra_hasreg(ir->r)) + pass3 = 1; + } else if (ra_used(ir)) { + sloadins[rs] = (IRRef1)i; + rset_set(live, rs); /* Block live parent register. */ } } @@ -2979,8 +2977,7 @@ static void asm_tail_sync(ASMState *as) SnapEntry sn = map[n]; if ((sn & SNAP_FRAME)) { IRIns *ir = IR(snap_ref(sn)); - GCfunc *fn = ir_kfunc(IR(ir->op2)); - lua_assert(ir->o == IR_FRAME && irt_isfunc(ir->t)); + GCfunc *fn = ir_kfunc(ir); if (isluafunc(fn)) { BCReg s = snap_slot(sn); BCReg fs = s + funcproto(fn)->framesize; @@ -3019,9 +3016,10 @@ static void asm_tail_sync(ASMState *as) /* Store the value of all modified slots to the Lua stack. */ for (n = 0; n < nent; n++) { - BCReg s = snap_slot(map[n]); + SnapEntry sn = map[n]; + BCReg s = snap_slot(sn); int32_t ofs = 8*((int32_t)s-1); - IRRef ref = snap_ref(map[n]); + IRRef ref = snap_ref(sn); IRIns *ir = IR(ref); /* No need to restore readonly slots and unmodified non-parent slots. */ if (ir->o == IR_SLOAD && ir->op1 == s && @@ -3030,10 +3028,6 @@ static void asm_tail_sync(ASMState *as) if (irt_isnum(ir->t)) { Reg src = ra_alloc1(as, ref, RSET_FPR); emit_rmro(as, XO_MOVSDto, src, RID_BASE, ofs); - } else if (ir->o == IR_FRAME) { - emit_movmroi(as, RID_BASE, ofs, ptr2addr(ir_kgc(IR(ir->op2)))); - if (s != 0) /* Do not overwrite link to previous frame. */ - emit_movmroi(as, RID_BASE, ofs+4, (int32_t)(*--flinks)); } else { lua_assert(irt_ispri(ir->t) || irt_isaddr(ir->t)); if (!irref_isk(ref)) { @@ -3042,7 +3036,10 @@ static void asm_tail_sync(ASMState *as) } else if (!irt_ispri(ir->t)) { emit_movmroi(as, RID_BASE, ofs, ir->i); } - emit_movmroi(as, RID_BASE, ofs+4, irt_toitype(ir->t)); + if (!(sn & (SNAP_CONT|SNAP_FRAME))) + emit_movmroi(as, RID_BASE, ofs+4, irt_toitype(ir->t)); + else if (s != 0) /* Do not overwrite link to previous frame. */ + emit_movmroi(as, RID_BASE, ofs+4, (int32_t)(*--flinks)); } checkmclim(as); } @@ -3110,10 +3107,6 @@ static void asm_ir(ASMState *as, IRIns *ir) case IR_ULE: asm_comp(as, ir, CC_A, CC_A, VCC_U); break; case IR_ABC: case IR_UGT: asm_comp(as, ir, CC_BE, CC_BE, VCC_U|VCC_PS); break; - - case IR_FRAME: - if (ir->op1 == ir->op2) break; /* No check needed for placeholder. */ - /* fallthrough */ case IR_EQ: asm_comp(as, ir, CC_NE, CC_NE, VCC_P); break; case IR_NE: asm_comp(as, ir, CC_E, CC_E, VCC_U|VCC_P); break; @@ -3272,10 +3265,6 @@ static void asm_setup_regsp(ASMState *as, Trace *T) } } break; - case IR_FRAME: - if (i == as->stopins+1 && ir->op1 == ir->op2) - as->stopins++; - break; case IR_CALLN: case IR_CALLL: case IR_CALLS: { const CCallInfo *ci = &lj_ir_callinfo[ir->op2]; /* NYI: not fastcall-aware, but doesn't matter (yet). */ diff --git a/src/lj_ir.h b/src/lj_ir.h index 672aca4a..34c14519 100644 --- a/src/lj_ir.h +++ b/src/lj_ir.h @@ -34,7 +34,7 @@ _(NE, GC, ref, ref) \ \ _(ABC, G , ref, ref) \ - _(FRAME, G , ref, ref) \ + _(UNUSED, G , ref, ref) /* Placeholder. */ \ \ _(LT, G , ref, ref) \ _(GE, G , ref, ref) \ @@ -511,11 +511,11 @@ typedef union IRIns { MRef ptr; /* Pointer constant (overlaps op12). */ } IRIns; -#define ir_kgc(ir) (gcref((ir)->gcr)) +#define ir_kgc(ir) check_exp((ir)->o == IR_KGC, gcref((ir)->gcr)) #define ir_kstr(ir) (gco2str(ir_kgc((ir)))) #define ir_ktab(ir) (gco2tab(ir_kgc((ir)))) #define ir_kfunc(ir) (gco2func(ir_kgc((ir)))) -#define ir_knum(ir) (mref((ir)->ptr, cTValue)) -#define ir_kptr(ir) (mref((ir)->ptr, void)) +#define ir_knum(ir) check_exp((ir)->o == IR_KNUM, mref((ir)->ptr, cTValue)) +#define ir_kptr(ir) check_exp((ir)->o == IR_KPTR, mref((ir)->ptr, void)) #endif diff --git a/src/lj_record.c b/src/lj_record.c index 94ea42ed..7b2e977e 100644 --- a/src/lj_record.c +++ b/src/lj_record.c @@ -148,15 +148,10 @@ static TRef sload(jit_State *J, int32_t slot) /* Get TRef for current function. */ static TRef getcurrf(jit_State *J) { - if (J->base[-1]) { - IRIns *ir = IR(tref_ref(J->base[-1])); - if (ir->o == IR_FRAME) /* Shortcut if already specialized. */ - return TREF(ir->op2, IRT_FUNC); /* Return TRef of KFUNC. */ + if (J->base[-1]) return J->base[-1]; - } else { - lua_assert(J->baseslot == 1); - return sloadt(J, -1, IRT_FUNC, IRSLOAD_READONLY); - } + lua_assert(J->baseslot == 1); + return sloadt(J, -1, IRT_FUNC, IRSLOAD_READONLY); } /* Compare for raw object equality. @@ -424,7 +419,7 @@ static BCReg rec_mm_prep(jit_State *J, ASMFunction cont) #else trcont = lj_ir_kptr(J, (void *)cont); #endif - J->base[top] = emitir(IRTG(IR_FRAME, IRT_PTR), trcont, trcont) | TREF_CONT; + J->base[top] = trcont | TREF_CONT; for (s = J->maxslot; s < top; s++) J->base[s] = TREF_NIL; return top+1; @@ -1586,7 +1581,7 @@ static void check_call_unroll(jit_State *J, GCfunc *fn) static int rec_call(jit_State *J, BCReg func, int cres, int nargs) { RecordFFData rd; - TRef *res = &J->base[func]; + TRef trfunc, *res = &J->base[func]; TValue *tv = &J->L->base[func]; if (tref_isfunc(res[0])) { /* Regular function call. */ @@ -1608,7 +1603,9 @@ static int rec_call(jit_State *J, BCReg func, int cres, int nargs) } /* Specialize to the runtime value of the called function. */ - res[0] = emitir(IRTG(IR_FRAME, IRT_FUNC), res[0], lj_ir_kfunc(J, rd.fn)) | TREF_FRAME; + trfunc = lj_ir_kfunc(J, rd.fn); + emitir(IRTG(IR_EQ, IRT_FUNC), res[0], trfunc); + res[0] = trfunc | TREF_FRAME; if (isluafunc(rd.fn)) { /* Record call to Lua function. */ GCproto *pt = funcproto(rd.fn); @@ -2175,12 +2172,7 @@ static void rec_setup_side(jit_State *J, Trace *T) for (j = 0; j < n; j++) if (snap_ref(map[j]) == ref) { tr = J->slot[snap_slot(map[j])]; - if (ir->o == IR_FRAME && irt_isfunc(ir->t)) { - lua_assert(s != 0); - J->baseslot = s+1; - J->framedepth++; - } - goto dupslot; + goto setslot; } } bloomset(seen, ref); @@ -2190,30 +2182,24 @@ static void rec_setup_side(jit_State *J, Trace *T) case IR_KINT: tr = lj_ir_kint(J, ir->i); break; case IR_KGC: tr = lj_ir_kgc(J, ir_kgc(ir), irt_t(ir->t)); break; case IR_KNUM: tr = lj_ir_knum_addr(J, ir_knum(ir)); break; - case IR_FRAME: /* Placeholder FRAMEs don't need a guard. */ - if (irt_isfunc(ir->t)) { - if (s != 0) { - J->baseslot = s+1; - J->framedepth++; - } - tr = lj_ir_kfunc(J, ir_kfunc(&T->ir[ir->op2])); - tr = emitir_raw(IRT(IR_FRAME, IRT_FUNC), tr, tr) | TREF_FRAME; - } else { - tr = lj_ir_kptr(J, mref(T->ir[ir->op2].ptr, void)); - tr = emitir_raw(IRT(IR_FRAME, IRT_PTR), tr, tr) | TREF_CONT; - } - break; - case IR_SLOAD: /* Inherited SLOADs don't need a guard or type check. */ + case IR_KPTR: tr = lj_ir_kptr(J, ir_kptr(ir)); break; /* Continuation. */ + /* Inherited SLOADs don't need a guard or type check. */ + case IR_SLOAD: tr = emitir_raw(ir->ot & ~IRT_GUARD, s, (ir->op2&IRSLOAD_READONLY) | IRSLOAD_INHERIT|IRSLOAD_PARENT); break; - default: /* Parent refs are already typed and don't need a guard. */ + /* Parent refs are already typed and don't need a guard. */ + default: tr = emitir_raw(IRT(IR_SLOAD, irt_type(ir->t)), s, IRSLOAD_INHERIT|IRSLOAD_PARENT); break; } - dupslot: - J->slot[s] = tr; + setslot: + J->slot[s] = tr | (sn&(SNAP_CONT|SNAP_FRAME)); /* Same as TREF_* flags. */ + if ((sn & SNAP_FRAME) && s != 0) { + J->baseslot = s+1; + J->framedepth++; + } } J->base = J->slot + J->baseslot; J->maxslot = snap->nslots - J->baseslot; diff --git a/src/lj_snap.c b/src/lj_snap.c index 731b8f92..8a53e3f6 100644 --- a/src/lj_snap.c +++ b/src/lj_snap.c @@ -211,6 +211,7 @@ void lj_snap_restore(jit_State *J, void *exptr) MSize n, nent = snap->nent; SnapEntry *map = &T->snapmap[snap->mapofs]; SnapEntry *flinks = map + nent + snap->nframelinks; + int32_t ftsz0; BCReg nslots = snap->nslots; TValue *frame; BloomFilter rfilt = snap_renamefilter(T, snapno); @@ -224,6 +225,7 @@ void lj_snap_restore(jit_State *J, void *exptr) /* Fill stack slots with data from the registers and spill slots. */ frame = L->base-1; + ftsz0 = frame_ftsz(frame); /* Preserve link to previous frame in slot #0. */ for (n = 0; n < nent; n++) { SnapEntry sn = map[n]; IRRef ref = snap_ref(sn); @@ -232,42 +234,11 @@ void lj_snap_restore(jit_State *J, void *exptr) IRIns *ir = &T->ir[ref]; if (irref_isk(ref)) { /* Restore constant slot. */ lj_ir_kvalue(L, o, ir); - } else { - IRType1 t = ir->t; - RegSP rs = ir->prev; - if (LJ_UNLIKELY(bloomtest(rfilt, ref))) - rs = snap_renameref(T, snapno, ref, rs); - if (ra_hasspill(regsp_spill(rs))) { /* Restore from spill slot. */ - int32_t *sps = &ex->spill[regsp_spill(rs)]; - if (irt_isinteger(t)) { - setintV(o, *sps); - } else if (irt_isnum(t)) { - o->u64 = *(uint64_t *)sps; - } else { - lua_assert(!irt_ispri(t)); /* PRI refs never have a spill slot. */ - setgcrefi(o->gcr, *sps); - setitype(o, irt_toitype(t)); - } - } else if (ra_hasreg(regsp_reg(rs))) { /* Restore from register. */ - Reg r = regsp_reg(rs); - if (irt_isinteger(t)) { - setintV(o, ex->gpr[r-RID_MIN_GPR]); - } else if (irt_isnum(t)) { - setnumV(o, ex->fpr[r-RID_MIN_FPR]); - } else { - if (!irt_ispri(t)) - setgcrefi(o->gcr, ex->gpr[r-RID_MIN_GPR]); - setitype(o, irt_toitype(t)); - } - } else { /* Restore frame slot. */ - lua_assert((sn & (SNAP_CONT|SNAP_FRAME))); - lua_assert(ir->o == IR_FRAME); - /* This works for both PTR and FUNC IR_FRAME. */ - setgcrefp(o->fr.func, mref(T->ir[ir->op2].ptr, void)); - if (s != 0) /* Do not overwrite link to previous frame. */ - o->fr.tp.ftsz = (int32_t)*--flinks; - if (irt_isfunc(ir->t)) { - GCfunc *fn = gco2func(gcref(T->ir[ir->op2].gcr)); + if ((sn & (SNAP_CONT|SNAP_FRAME))) { + /* Overwrite tag with frame link. */ + o->fr.tp.ftsz = s != 0 ? (int32_t)*--flinks : ftsz0; + if ((sn & SNAP_FRAME)) { + GCfunc *fn = ir_kfunc(ir); if (isluafunc(fn)) { MSize framesize = funcproto(fn)->framesize; TValue *fs; @@ -287,6 +258,36 @@ void lj_snap_restore(jit_State *J, void *exptr) } } } + } else { + IRType1 t = ir->t; + RegSP rs = ir->prev; + lua_assert(!(sn & (SNAP_CONT|SNAP_FRAME))); + if (LJ_UNLIKELY(bloomtest(rfilt, ref))) + rs = snap_renameref(T, snapno, ref, rs); + if (ra_hasspill(regsp_spill(rs))) { /* Restore from spill slot. */ + int32_t *sps = &ex->spill[regsp_spill(rs)]; + if (irt_isinteger(t)) { + setintV(o, *sps); + } else if (irt_isnum(t)) { + o->u64 = *(uint64_t *)sps; + } else { + lua_assert(!irt_ispri(t)); /* PRI refs never have a spill slot. */ + setgcrefi(o->gcr, *sps); + setitype(o, irt_toitype(t)); + } + } else { /* Restore from register. */ + Reg r = regsp_reg(rs); + lua_assert(ra_hasreg(r)); + if (irt_isinteger(t)) { + setintV(o, ex->gpr[r-RID_MIN_GPR]); + } else if (irt_isnum(t)) { + setnumV(o, ex->fpr[r-RID_MIN_FPR]); + } else { + if (!irt_ispri(t)) + setgcrefi(o->gcr, ex->gpr[r-RID_MIN_GPR]); + setitype(o, irt_toitype(t)); + } + } } } L->top = curr_topL(L);