diff --git a/lib/dump.lua b/lib/dump.lua index f476f3f0..4e794498 100644 --- a/lib/dump.lua +++ b/lib/dump.lua @@ -418,10 +418,16 @@ local function dump_ir(tr, dumpsnap, dumpreg) band(ot, 64) == 0 and " " or "+", irtype[t], op)) local m1 = band(m, 3) - if sub(op, 1, 4) == "CALL" then + local op4 = sub(op, 1, 4) + if op4 == "CALL" then out:write(format("%-10s (", vmdef.ircall[op2])) if op1 ~= -1 then dumpcallargs(tr, op1) end out:write(")") + elseif op4 == "CNEW" then + out:write(formatk(tr, op2)) + if op1 ~= -1 then + out:write(" ("); dumpcallargs(tr, op1); out:write(")") + end elseif m1 ~= 3 then -- op1 != IRMnone if op1 < 0 then out:write(formatk(tr, op1)) diff --git a/src/lj_asm.c b/src/lj_asm.c index 3d4060b2..215dfb13 100644 --- a/src/lj_asm.c +++ b/src/lj_asm.c @@ -14,6 +14,9 @@ #include "lj_str.h" #include "lj_tab.h" #include "lj_frame.h" +#if LJ_HASFFI +#include "lj_ctype.h" +#endif #include "lj_ir.h" #include "lj_jit.h" #include "lj_iropt.h" @@ -2279,6 +2282,91 @@ static void asm_tdup(ASMState *as, IRIns *ir) asm_gencall(as, ci, args); } +#if LJ_HASFFI +static RegSet asm_cnew_init(ASMState *as, IRRef ref, int32_t ofs, RegSet allow) +{ + IRIns *ir = IR(ref); + if (irref_isk(ref)) { +#if LJ_64 + if (ir->o == IR_KNUM || ir->o == IR_KINT64) { + uint64_t k = ir_k64(ir)->u64; + if (checki32((int64_t)k)) { + emit_i32(as, (int32_t)k); + emit_rmro(as, XO_MOVmi, REX_64, RID_RET, ofs); + } else { + emit_movtomro(as, RID_ECX|REX_64, RID_RET, ofs); + emit_loadu64(as, RID_ECX, k); + } + } else { + emit_movmroi(as, RID_RET, ofs, ir->i); + } +#else + if (ir->o == IR_KNUM) { + emit_rmro(as, XO_MOVSDto, RID_XMM0, RID_RET, ofs); + emit_loadn(as, RID_XMM0, ir_k64(ir)); + } else if (ir->o == IR_KINT64) { + uint64_t k = ir_k64(ir)->u64; + emit_movmroi(as, RID_RET, ofs, (int32_t)k); + emit_movmroi(as, RID_RET, ofs+4, (int32_t)(k >> 32)); + } else { + emit_movmroi(as, RID_RET, ofs, ir->i); + } +#endif + } else { + Reg r; + if (irt_isnum(ir->t)) { + r = ra_alloc1(as, ref, (RSET_FPR & allow)); + emit_rmro(as, XO_MOVSDto, r, RID_RET, ofs); + } else { + r = ra_alloc1(as, ref, (RSET_GPR & allow)); + emit_movtomro(as, REX_64IR(ir, r), RID_RET, ofs); + } + rset_clear(allow, r); + } + return allow; +} + +static void asm_cnew(ASMState *as, IRIns *ir) +{ + CTState *cts = ctype_ctsG(J2G(as->J)); + CTypeID typeid = (CTypeID)IR(ir->op2)->i; + CTSize sz = (ir->o == IR_CNEWI || ir->op1 == REF_NIL) ? + lj_ctype_size(cts, typeid) : (CTSize)IR(ir->op1)->i; + const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_mem_newgco]; + IRRef args[2]; + lua_assert(sz != CTSIZE_INVALID); + + args[0] = ASMREF_L; /* lua_State *L */ + args[1] = ASMREF_TMP1; /* MSize size */ + as->gcsteps++; + asm_setupresult(as, ir, ci); /* GCobj * */ + + /* Initialize immutable cdata object. */ + if (ir->o == IR_CNEWI) { + RegSet allow = ~RSET_SCRATCH; + IRRef ref = ir->op1; + if (IR(ref)->o == IR_CARG) { /* 2nd initializer. */ + IRIns *ira = IR(ref); + allow = asm_cnew_init(as, ira->op2, sizeof(GCcdata) + (sz>>1), allow); + ref = ira->op1; + } + asm_cnew_init(as, ref, sizeof(GCcdata), allow); /* 1st initializer. */ + } + + /* Combine initialization of marked, gct and typeid. */ + emit_movtomro(as, RID_ECX, RID_RET, offsetof(GCcdata, marked)); + emit_gri(as, XG_ARITHi(XOg_OR), RID_ECX, + (int32_t)((~LJ_TCDATA<<8)+(typeid<<16))); + emit_gri(as, XG_ARITHi(XOg_AND), RID_ECX, LJ_GC_WHITES); + emit_opgl(as, XO_MOVZXb, RID_ECX, gc.currentwhite); + + asm_gencall(as, ci, args); + emit_loadi(as, ra_releasetmp(as, ASMREF_TMP1), (int32_t)(sz+sizeof(GCcdata))); +} +#else +#define asm_cnew(as, ir) ((void)0) +#endif + /* -- Write barriers ------------------------------------------------------ */ static void asm_tbar(ASMState *as, IRIns *ir) @@ -3587,6 +3675,7 @@ static void asm_ir(ASMState *as, IRIns *ir) case IR_SNEW: asm_snew(as, ir); break; case IR_TNEW: asm_tnew(as, ir); break; case IR_TDUP: asm_tdup(as, ir); break; + case IR_CNEW: case IR_CNEWI: asm_cnew(as, ir); break; /* Write barriers. */ case IR_TBAR: asm_tbar(as, ir); break; @@ -3704,7 +3793,7 @@ static void asm_setup_regsp(ASMState *as, GCtrace *T) if (as->evenspill < 3) /* lj_str_new and lj_tab_newkey need 3 args. */ as->evenspill = 3; #endif - case IR_TNEW: case IR_TDUP: case IR_TOSTR: + case IR_TNEW: case IR_TDUP: case IR_CNEW: case IR_CNEWI: case IR_TOSTR: ir->prev = REGSP_HINT(RID_RET); if (inloop) as->modset = RSET_SCRATCH; diff --git a/src/lj_crecord.c b/src/lj_crecord.c index a543ffd1..314e0de7 100644 --- a/src/lj_crecord.c +++ b/src/lj_crecord.c @@ -238,6 +238,7 @@ static void crec_ct_ct(jit_State *J, CType *d, CType *s, TRef dp, TRef sp) static TRef crec_tv_ct(jit_State *J, CType *s, CTypeID sid, TRef sp) { + CTState *cts = ctype_ctsG(J2G(J)); CTInfo sinfo = s->info; lua_assert(!ctype_isenum(sinfo)); if (ctype_isnum(sinfo)) { @@ -249,8 +250,8 @@ static TRef crec_tv_ct(jit_State *J, CType *s, CTypeID sid, TRef sp) return emitir(IRT(IR_XLOAD, t), sp, 0); } else if (ctype_isrefarray(sinfo) || ctype_isstruct(sinfo)) { /* Create reference. */ - UNUSED(sid); lj_trace_err(J, LJ_TRERR_NYICONV); - return 0; + CTypeID refid = lj_ctype_intern(cts, CTINFO_REF(sid), CTSIZE_PTR); + return emitir(IRTG(IR_CNEWI, IRT_CDATA), sp, lj_ir_kint(J, refid)); } else { copyval: /* Copy value. */ lj_trace_err(J, LJ_TRERR_NYICONV); @@ -281,7 +282,7 @@ static void crec_ct_tv(jit_State *J, CType *d, TRef dp, TRef sp, TValue *sval) s = ctype_raw(cts, sid); if (ctype_isptr(s->info)) { IRType t = (LJ_64 && s->size == 8) ? IRT_P64 : IRT_P32; - sp = emitir(IRT(IR_FLOAD, t), sp, IRFL_CDATA_DATA); + sp = emitir(IRT(IR_FLOAD, t), sp, IRFL_CDATA_INIT1); if (ctype_isref(s->info)) s = ctype_rawchild(cts, s); else @@ -316,7 +317,7 @@ void LJ_FASTCALL recff_cdata_index(jit_State *J, RecordFFData *rd) if (ctype_isptr(ct->info)) { IRType t = (LJ_64 && ct->size == 8) ? IRT_P64 : IRT_P32; if (ctype_isref(ct->info)) ct = ctype_rawchild(cts, ct); - ptr = emitir(IRT(IR_FLOAD, t), ptr, IRFL_CDATA_DATA); + ptr = emitir(IRT(IR_FLOAD, t), ptr, IRFL_CDATA_INIT1); ofs = 0; } diff --git a/src/lj_ir.h b/src/lj_ir.h index 7814df34..6495a780 100644 --- a/src/lj_ir.h +++ b/src/lj_ir.h @@ -107,9 +107,11 @@ _(XSTORE, S , ref, ref) \ \ /* Allocations. */ \ - _(SNEW, N , ref, ref) /* CSE is ok, so not marked as A. */ \ + _(SNEW, N , ref, ref) /* CSE is ok, not marked as A. */ \ _(TNEW, AW, lit, lit) \ _(TDUP, AW, ref, ___) \ + _(CNEW, AW, ref, ref) \ + _(CNEWI, NW, ref, ref) /* CSE is ok, not marked as A. */ \ \ /* Write barriers. */ \ _(TBAR, S , ref, ___) \ @@ -186,7 +188,9 @@ IRFPMDEF(FPMENUM) _(UDATA_UDTYPE, offsetof(GCudata, udtype)) \ _(UDATA_FILE, sizeof(GCudata)) \ _(CDATA_TYPEID, offsetof(GCcdata, typeid)) \ - _(CDATA_DATA, sizeof(GCcdata)) + _(CDATA_INIT1, sizeof(GCcdata)) \ + _(CDATA_INIT2_4, sizeof(GCcdata)+4) \ + _(CDATA_INIT2_8, sizeof(GCcdata)+8) typedef enum { #define FLENUM(name, ofs) IRFL_##name, @@ -256,6 +260,7 @@ typedef struct CCallInfo { _(lj_tab_len, 1, FL, INT, 0) \ _(lj_gc_step_jit, 2, FS, NIL, CCI_L) \ _(lj_gc_barrieruv, 2, FS, NIL, 0) \ + _(lj_mem_newgco, 2, FS, P32, CCI_L) \ _(lj_math_random_step, 1, FS, NUM, CCI_CASTU64|CCI_NOFPRCLOBBER) \ _(sinh, 1, N, NUM, 0) \ _(cosh, 1, N, NUM, 0) \ @@ -297,6 +302,7 @@ typedef enum { #define IRM_W 0x80 +#define IRM_NW (IRM_N|IRM_W) #define IRM_AW (IRM_A|IRM_W) #define IRM_LW (IRM_L|IRM_W) diff --git a/src/lj_opt_fold.c b/src/lj_opt_fold.c index e8e81a3e..e61a6533 100644 --- a/src/lj_opt_fold.c +++ b/src/lj_opt_fold.c @@ -152,8 +152,8 @@ typedef IRRef (LJ_FASTCALL *FoldFunc)(jit_State *J); */ #define gcstep_barrier(J, ref) \ ((ref) < J->chain[IR_LOOP] && \ - (J->chain[IR_TNEW] || J->chain[IR_TDUP] || \ - J->chain[IR_SNEW] || J->chain[IR_TOSTR])) + (J->chain[IR_SNEW] || J->chain[IR_TNEW] || J->chain[IR_TDUP] || \ + J->chain[IR_CNEW] || J->chain[IR_CNEWI] || J->chain[IR_TOSTR])) /* -- Constant folding ---------------------------------------------------- */ @@ -1477,7 +1477,37 @@ LJFOLDF(fload_str_len_snew) return NEXTFOLD; } +/* The C type ID of cdata objects is immutable. */ +LJFOLD(FLOAD CNEW IRFL_CDATA_TYPEID) +LJFOLD(FLOAD CNEWI IRFL_CDATA_TYPEID) +LJFOLDF(fload_cdata_typeid_cnewi) +{ + if (LJ_LIKELY(J->flags & JIT_F_OPT_FOLD)) + return fleft->op2; /* No PHI barrier needed. CNEW/CNEWI op2 is const. */ + return NEXTFOLD; +} + +/* Fixed initializers in cdata objects are immutable. */ +LJFOLD(FLOAD CNEWI IRFL_CDATA_INIT1) +LJFOLD(FLOAD CNEWI IRFL_CDATA_INIT2_4) +LJFOLD(FLOAD CNEWI IRFL_CDATA_INIT2_8) +LJFOLDF(fload_cdata_init_cnew) +{ + if (LJ_LIKELY(J->flags & JIT_F_OPT_FOLD)) { + IRIns *ir = fleft; + PHIBARRIER(fleft); + lua_assert(ir->op1 != REF_NIL); + if (IR(ir->op1)->o == IR_CARG) ir = IR(ir->op1); + return fins->op2 == IRFL_CDATA_INIT1 ? ir->op1 : ir->op2; + } + return NEXTFOLD; +} + LJFOLD(FLOAD any IRFL_STR_LEN) +LJFOLD(FLOAD any IRFL_CDATA_TYPEID) +LJFOLD(FLOAD any IRFL_CDATA_INIT1) +LJFOLD(FLOAD any IRFL_CDATA_INIT2_4) +LJFOLD(FLOAD any IRFL_CDATA_INIT2_8) LJFOLD(VLOAD any any) /* Vararg loads have no corresponding stores. */ LJFOLDX(lj_opt_cse) @@ -1564,6 +1594,7 @@ LJFOLD(CALLL any any) /* Safeguard fallback. */ LJFOLD(RETF any any) /* Modifies BASE. */ LJFOLD(TNEW any any) LJFOLD(TDUP any) +LJFOLD(CNEW any any) LJFOLDX(lj_ir_emit) /* ------------------------------------------------------------------------ */