FFI: Add IR_CNEW/IR_CNEWI to allocate/init cdata objects.

lib/dump.lua

@@ -418,10 +418,16 @@ local function dump_ir(tr, dumpsnap, dumpreg)
 		       band(ot, 64) == 0 and " " or "+",
 		       irtype[t], op))
       local m1 = band(m, 3)
-      if sub(op, 1, 4) == "CALL" then
+      local op4 = sub(op, 1, 4)
+      if op4 == "CALL" then
 	out:write(format("%-10s  (", vmdef.ircall[op2]))
 	if op1 ~= -1 then dumpcallargs(tr, op1) end
 	out:write(")")
+      elseif op4 == "CNEW" then
+	out:write(formatk(tr, op2))
+	if op1 ~= -1 then
+	  out:write("  ("); dumpcallargs(tr, op1); out:write(")")
+	end
       elseif m1 ~= 3 then -- op1 != IRMnone
 	if op1 < 0 then
 	  out:write(formatk(tr, op1))

src/lj_asm.c

@@ -14,6 +14,9 @@
 #include "lj_str.h"
 #include "lj_tab.h"
 #include "lj_frame.h"
+#if LJ_HASFFI
+#include "lj_ctype.h"
+#endif
 #include "lj_ir.h"
 #include "lj_jit.h"
 #include "lj_iropt.h"
@@ -2279,6 +2282,91 @@ static void asm_tdup(ASMState *as, IRIns *ir)
   asm_gencall(as, ci, args);
 }
 
+#if LJ_HASFFI
+static RegSet asm_cnew_init(ASMState *as, IRRef ref, int32_t ofs, RegSet allow)
+{
+  IRIns *ir = IR(ref);
+  if (irref_isk(ref)) {
+#if LJ_64
+    if (ir->o == IR_KNUM || ir->o == IR_KINT64) {
+      uint64_t k = ir_k64(ir)->u64;
+      if (checki32((int64_t)k)) {
+	emit_i32(as, (int32_t)k);
+	emit_rmro(as, XO_MOVmi, REX_64, RID_RET, ofs);
+      } else {
+	emit_movtomro(as, RID_ECX|REX_64, RID_RET, ofs);
+	emit_loadu64(as, RID_ECX, k);
+      }
+    } else {
+      emit_movmroi(as, RID_RET, ofs, ir->i);
+    }
+#else
+    if (ir->o == IR_KNUM) {
+      emit_rmro(as, XO_MOVSDto, RID_XMM0, RID_RET, ofs);
+      emit_loadn(as, RID_XMM0, ir_k64(ir));
+    } else if (ir->o == IR_KINT64) {
+      uint64_t k = ir_k64(ir)->u64;
+      emit_movmroi(as, RID_RET, ofs, (int32_t)k);
+      emit_movmroi(as, RID_RET, ofs+4, (int32_t)(k >> 32));
+    } else {
+      emit_movmroi(as, RID_RET, ofs, ir->i);
+    }
+#endif
+  } else {
+    Reg r;
+    if (irt_isnum(ir->t)) {
+      r = ra_alloc1(as, ref, (RSET_FPR & allow));
+      emit_rmro(as, XO_MOVSDto, r, RID_RET, ofs);
+    } else {
+      r = ra_alloc1(as, ref, (RSET_GPR & allow));
+      emit_movtomro(as, REX_64IR(ir, r), RID_RET, ofs);
+    }
+    rset_clear(allow, r);
+  }
+  return allow;
+}
+
+static void asm_cnew(ASMState *as, IRIns *ir)
+{
+  CTState *cts = ctype_ctsG(J2G(as->J));
+  CTypeID typeid = (CTypeID)IR(ir->op2)->i;
+  CTSize sz = (ir->o == IR_CNEWI || ir->op1 == REF_NIL) ?
+	      lj_ctype_size(cts, typeid) : (CTSize)IR(ir->op1)->i;
+  const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_mem_newgco];
+  IRRef args[2];
+  lua_assert(sz != CTSIZE_INVALID);
+
+  args[0] = ASMREF_L;     /* lua_State *L */
+  args[1] = ASMREF_TMP1;  /* MSize size   */
+  as->gcsteps++;
+  asm_setupresult(as, ir, ci);  /* GCobj * */
+
+  /* Initialize immutable cdata object. */
+  if (ir->o == IR_CNEWI) {
+    RegSet allow = ~RSET_SCRATCH;
+    IRRef ref = ir->op1;
+    if (IR(ref)->o == IR_CARG) {  /* 2nd initializer. */
+      IRIns *ira = IR(ref);
+      allow = asm_cnew_init(as, ira->op2, sizeof(GCcdata) + (sz>>1), allow);
+      ref = ira->op1;
+    }
+    asm_cnew_init(as, ref, sizeof(GCcdata), allow);  /* 1st initializer. */
+  }
+
+  /* Combine initialization of marked, gct and typeid. */
+  emit_movtomro(as, RID_ECX, RID_RET, offsetof(GCcdata, marked));
+  emit_gri(as, XG_ARITHi(XOg_OR), RID_ECX,
+	   (int32_t)((~LJ_TCDATA<<8)+(typeid<<16)));
+  emit_gri(as, XG_ARITHi(XOg_AND), RID_ECX, LJ_GC_WHITES);
+  emit_opgl(as, XO_MOVZXb, RID_ECX, gc.currentwhite);
+
+  asm_gencall(as, ci, args);
+  emit_loadi(as, ra_releasetmp(as, ASMREF_TMP1), (int32_t)(sz+sizeof(GCcdata)));
+}
+#else
+#define asm_cnew(as, ir)	((void)0)
+#endif
+
 /* -- Write barriers ------------------------------------------------------ */
 
 static void asm_tbar(ASMState *as, IRIns *ir)
@@ -3587,6 +3675,7 @@ static void asm_ir(ASMState *as, IRIns *ir)
   case IR_SNEW: asm_snew(as, ir); break;
   case IR_TNEW: asm_tnew(as, ir); break;
   case IR_TDUP: asm_tdup(as, ir); break;
+  case IR_CNEW: case IR_CNEWI: asm_cnew(as, ir); break;
 
   /* Write barriers. */
   case IR_TBAR: asm_tbar(as, ir); break;
@@ -3704,7 +3793,7 @@ static void asm_setup_regsp(ASMState *as, GCtrace *T)
       if (as->evenspill < 3)  /* lj_str_new and lj_tab_newkey need 3 args. */
 	as->evenspill = 3;
 #endif
-    case IR_TNEW: case IR_TDUP: case IR_TOSTR:
+    case IR_TNEW: case IR_TDUP: case IR_CNEW: case IR_CNEWI: case IR_TOSTR:
       ir->prev = REGSP_HINT(RID_RET);
       if (inloop)
 	as->modset = RSET_SCRATCH;

src/lj_crecord.c

@@ -238,6 +238,7 @@ static void crec_ct_ct(jit_State *J, CType *d, CType *s, TRef dp, TRef sp)
 
 static TRef crec_tv_ct(jit_State *J, CType *s, CTypeID sid, TRef sp)
 {
+  CTState *cts = ctype_ctsG(J2G(J));
   CTInfo sinfo = s->info;
   lua_assert(!ctype_isenum(sinfo));
   if (ctype_isnum(sinfo)) {
@@ -249,8 +250,8 @@ static TRef crec_tv_ct(jit_State *J, CType *s, CTypeID sid, TRef sp)
     return emitir(IRT(IR_XLOAD, t), sp, 0);
   } else if (ctype_isrefarray(sinfo) || ctype_isstruct(sinfo)) {
     /* Create reference. */
-    UNUSED(sid); lj_trace_err(J, LJ_TRERR_NYICONV);
+    CTypeID refid = lj_ctype_intern(cts, CTINFO_REF(sid), CTSIZE_PTR);
-    return 0;
+    return emitir(IRTG(IR_CNEWI, IRT_CDATA), sp, lj_ir_kint(J, refid));
   } else {
   copyval:  /* Copy value. */
     lj_trace_err(J, LJ_TRERR_NYICONV);
@@ -281,7 +282,7 @@ static void crec_ct_tv(jit_State *J, CType *d, TRef dp, TRef sp, TValue *sval)
     s = ctype_raw(cts, sid);
     if (ctype_isptr(s->info)) {
       IRType t = (LJ_64 && s->size == 8) ? IRT_P64 : IRT_P32;
-      sp = emitir(IRT(IR_FLOAD, t), sp, IRFL_CDATA_DATA);
+      sp = emitir(IRT(IR_FLOAD, t), sp, IRFL_CDATA_INIT1);
       if (ctype_isref(s->info))
 	s = ctype_rawchild(cts, s);
       else
@@ -316,7 +317,7 @@ void LJ_FASTCALL recff_cdata_index(jit_State *J, RecordFFData *rd)
   if (ctype_isptr(ct->info)) {
     IRType t = (LJ_64 && ct->size == 8) ? IRT_P64 : IRT_P32;
     if (ctype_isref(ct->info)) ct = ctype_rawchild(cts, ct);
-    ptr = emitir(IRT(IR_FLOAD, t), ptr, IRFL_CDATA_DATA);
+    ptr = emitir(IRT(IR_FLOAD, t), ptr, IRFL_CDATA_INIT1);
     ofs = 0;
   }
 

src/lj_ir.h

@@ -107,9 +107,11 @@
   _(XSTORE,	S , ref, ref) \
   \
   /* Allocations. */ \
-  _(SNEW,	N , ref, ref) /* CSE is ok, so not marked as A. */ \
+  _(SNEW,	N , ref, ref)  /* CSE is ok, not marked as A. */ \
   _(TNEW,	AW, lit, lit) \
   _(TDUP,	AW, ref, ___) \
+  _(CNEW,	AW, ref, ref) \
+  _(CNEWI,	NW, ref, ref)  /* CSE is ok, not marked as A. */ \
   \
   /* Write barriers. */ \
   _(TBAR,	S , ref, ___) \
@@ -186,7 +188,9 @@ IRFPMDEF(FPMENUM)
   _(UDATA_UDTYPE, offsetof(GCudata, udtype)) \
   _(UDATA_FILE,	sizeof(GCudata)) \
   _(CDATA_TYPEID, offsetof(GCcdata, typeid)) \
-  _(CDATA_DATA, sizeof(GCcdata))
+  _(CDATA_INIT1, sizeof(GCcdata)) \
+  _(CDATA_INIT2_4, sizeof(GCcdata)+4) \
+  _(CDATA_INIT2_8, sizeof(GCcdata)+8)
 
 typedef enum {
 #define FLENUM(name, ofs)	IRFL_##name,
@@ -256,6 +260,7 @@ typedef struct CCallInfo {
   _(lj_tab_len,		1,  FL, INT, 0) \
   _(lj_gc_step_jit,	2,  FS, NIL, CCI_L) \
   _(lj_gc_barrieruv,	2,  FS, NIL, 0) \
+  _(lj_mem_newgco,	2,  FS, P32, CCI_L) \
   _(lj_math_random_step, 1, FS, NUM, CCI_CASTU64|CCI_NOFPRCLOBBER) \
   _(sinh,		1,  N, NUM, 0) \
   _(cosh,		1,  N, NUM, 0) \
@@ -297,6 +302,7 @@ typedef enum {
 
 #define IRM_W			0x80
 
+#define IRM_NW			(IRM_N|IRM_W)
 #define IRM_AW			(IRM_A|IRM_W)
 #define IRM_LW			(IRM_L|IRM_W)
 

src/lj_opt_fold.c

@@ -152,8 +152,8 @@ typedef IRRef (LJ_FASTCALL *FoldFunc)(jit_State *J);
 */
 #define gcstep_barrier(J, ref) \
   ((ref) < J->chain[IR_LOOP] && \
-   (J->chain[IR_TNEW] || J->chain[IR_TDUP] || \
+   (J->chain[IR_SNEW] || J->chain[IR_TNEW] || J->chain[IR_TDUP] || \
-    J->chain[IR_SNEW] || J->chain[IR_TOSTR]))
+    J->chain[IR_CNEW] || J->chain[IR_CNEWI] || J->chain[IR_TOSTR]))
 
 /* -- Constant folding ---------------------------------------------------- */
 
@@ -1477,7 +1477,37 @@ LJFOLDF(fload_str_len_snew)
   return NEXTFOLD;
 }
 
+/* The C type ID of cdata objects is immutable. */
+LJFOLD(FLOAD CNEW IRFL_CDATA_TYPEID)
+LJFOLD(FLOAD CNEWI IRFL_CDATA_TYPEID)
+LJFOLDF(fload_cdata_typeid_cnewi)
+{
+  if (LJ_LIKELY(J->flags & JIT_F_OPT_FOLD))
+    return fleft->op2;  /* No PHI barrier needed. CNEW/CNEWI op2 is const. */
+  return NEXTFOLD;
+}
+
+/* Fixed initializers in cdata objects are immutable. */
+LJFOLD(FLOAD CNEWI IRFL_CDATA_INIT1)
+LJFOLD(FLOAD CNEWI IRFL_CDATA_INIT2_4)
+LJFOLD(FLOAD CNEWI IRFL_CDATA_INIT2_8)
+LJFOLDF(fload_cdata_init_cnew)
+{
+  if (LJ_LIKELY(J->flags & JIT_F_OPT_FOLD)) {
+    IRIns *ir = fleft;
+    PHIBARRIER(fleft);
+    lua_assert(ir->op1 != REF_NIL);
+    if (IR(ir->op1)->o == IR_CARG) ir = IR(ir->op1);
+    return fins->op2 == IRFL_CDATA_INIT1 ? ir->op1 : ir->op2;
+  }
+  return NEXTFOLD;
+}
+
 LJFOLD(FLOAD any IRFL_STR_LEN)
+LJFOLD(FLOAD any IRFL_CDATA_TYPEID)
+LJFOLD(FLOAD any IRFL_CDATA_INIT1)
+LJFOLD(FLOAD any IRFL_CDATA_INIT2_4)
+LJFOLD(FLOAD any IRFL_CDATA_INIT2_8)
 LJFOLD(VLOAD any any)  /* Vararg loads have no corresponding stores. */
 LJFOLDX(lj_opt_cse)
 
@@ -1564,6 +1594,7 @@ LJFOLD(CALLL any any)  /* Safeguard fallback. */
 LJFOLD(RETF any any)  /* Modifies BASE. */
 LJFOLD(TNEW any any)
 LJFOLD(TDUP any)
+LJFOLD(CNEW any any)
 LJFOLDX(lj_ir_emit)
 
 /* ------------------------------------------------------------------------ */