Add LJ_GC64 mode: 64 bit GC object references.

Actually NaN tagging with 47 bit pointers and 13+4 bit tags.

src/lj_alloc.c

@@ -77,7 +77,7 @@
 #define WIN32_LEAN_AND_MEAN
 #include <windows.h>
 
-#if LJ_64
+#if LJ_64 && !LJ_GC64
 
 /* Undocumented, but hey, that's what we all love so much about Windows. */
 typedef long (*PNTAVM)(HANDLE handle, void **addr, ULONG zbits,
@@ -174,8 +174,10 @@ static LJ_AINLINE int CALL_MUNMAP(void *ptr, size_t size)
 #endif
 #define MMAP_FLAGS		(MAP_PRIVATE|MAP_ANONYMOUS)
 
-#if LJ_64
+#if LJ_64 && !LJ_GC64
-/* 64 bit mode needs special support for allocating memory in the lower 2GB. */
+/* 64 bit mode with 32 bit pointers needs special support for allocating
+** memory in the lower 2GB.
+*/
 
 #if defined(MAP_32BIT)
 
@@ -258,7 +260,7 @@ static LJ_AINLINE void *CALL_MMAP(size_t size)
 
 #else
 
-/* 32 bit mode is easy. */
+/* 32 bit mode and GC64 mode is easy. */
 static LJ_AINLINE void *CALL_MMAP(size_t size)
 {
   int olderr = errno;
@@ -294,7 +296,7 @@ static LJ_AINLINE void *CALL_MREMAP_(void *ptr, size_t osz, size_t nsz,
 #define CALL_MREMAP(addr, osz, nsz, mv) CALL_MREMAP_((addr), (osz), (nsz), (mv))
 #define CALL_MREMAP_NOMOVE	0
 #define CALL_MREMAP_MAYMOVE	1
-#if LJ_64
+#if LJ_64 && !LJ_GC64
 #define CALL_MREMAP_MV		CALL_MREMAP_NOMOVE
 #else
 #define CALL_MREMAP_MV		CALL_MREMAP_MAYMOVE

src/lj_api.c

@@ -189,7 +189,7 @@ LUA_API int lua_type(lua_State *L, int idx)
   cTValue *o = index2adr(L, idx);
   if (tvisnumber(o)) {
     return LUA_TNUMBER;
-#if LJ_64
+#if LJ_64 && !LJ_GC64
   } else if (tvislightud(o)) {
     return LUA_TLIGHTUSERDATA;
 #endif
@@ -269,7 +269,7 @@ LUA_API int lua_equal(lua_State *L, int idx1, int idx2)
     return 0;
   } else if (tvispri(o1)) {
     return o1 != niltv(L) && o2 != niltv(L);
-#if LJ_64
+#if LJ_64 && !LJ_GC64
   } else if (tvislightud(o1)) {
     return o1->u64 == o2->u64;
 #endif

src/lj_arch.h

@@ -365,11 +365,22 @@
 #endif
 #endif
 
+/* 64 bit GC references. */
+#if LJ_TARGET_GC64
+#define LJ_GC64			1
+#else
+#define LJ_GC64			0
+#endif
+
 /* 2-slot frame info. */
+#if LJ_GC64
+#define LJ_FR2			1
+#else
 #define LJ_FR2			0
+#endif
 
 /* Disable or enable the JIT compiler. */
-#if defined(LUAJIT_DISABLE_JIT) || defined(LJ_ARCH_NOJIT) || defined(LJ_OS_NOJIT) || LJ_FR2
+#if defined(LUAJIT_DISABLE_JIT) || defined(LJ_ARCH_NOJIT) || defined(LJ_OS_NOJIT) || LJ_FR2 || LJ_GC64
 #define LJ_HASJIT		0
 #else
 #define LJ_HASJIT		1

src/lj_def.h

@@ -47,7 +47,9 @@ typedef unsigned int uintptr_t;
 
 /* Various VM limits. */
 #define LJ_MAX_MEM32	0x7fffff00	/* Max. 32 bit memory allocation. */
-#define LJ_MAX_MEM	LJ_MAX_MEM32	/* Max. total memory allocation. */
+#define LJ_MAX_MEM64	((uint64_t)1<<47)  /* Max. 64 bit memory allocation. */
+/* Max. total memory allocation. */
+#define LJ_MAX_MEM	(LJ_GC64 ? LJ_MAX_MEM64 : LJ_MAX_MEM32)
 #define LJ_MAX_ALLOC	LJ_MAX_MEM	/* Max. individual allocation length. */
 #define LJ_MAX_STR	LJ_MAX_MEM32	/* Max. string length. */
 #define LJ_MAX_BUF	LJ_MAX_MEM32	/* Max. buffer length. */
@@ -67,7 +69,7 @@ typedef unsigned int uintptr_t;
 #define LJ_MAX_UPVAL	60		/* Max. # of upvalues. */
 
 #define LJ_MAX_IDXCHAIN	100		/* __index/__newindex chain limit. */
-#define LJ_STACK_EXTRA	5		/* Extra stack space (metamethods). */
+#define LJ_STACK_EXTRA	(5+2*LJ_FR2)	/* Extra stack space (metamethods). */
 
 #define LJ_NUM_CBPAGE	1		/* Number of FFI callback pages. */
 
@@ -101,7 +103,14 @@ typedef unsigned int uintptr_t;
 #define checki32(x)	((x) == (int32_t)(x))
 #define checku32(x)	((x) == (uint32_t)(x))
 #define checkptr32(x)	((uintptr_t)(x) == (uint32_t)(uintptr_t)(x))
-#define checkptrGC(x)	(checkptr32(x))
+#define checkptr47(x)	(((uint64_t)(x) >> 47) == 0)
+#if LJ_GC64
+#define checkptrGC(x)	(checkptr47((x)))
+#elif LJ_64
+#define checkptrGC(x)	(checkptr32((x)))
+#else
+#define checkptrGC(x)	1
+#endif
 
 /* Every half-decent C compiler transforms this into a rotate instruction. */
 #define lj_rol(x, n)	(((x)<<(n)) | ((x)>>(-(int)(n)&(8*sizeof(x)-1))))

src/lj_ffrecord.c

@@ -196,7 +196,7 @@ static void LJ_FASTCALL recff_type(jit_State *J, RecordFFData *rd)
   uint32_t t;
   if (tvisnumber(&rd->argv[0]))
     t = ~LJ_TNUMX;
-  else if (LJ_64 && tvislightud(&rd->argv[0]))
+  else if (LJ_64 && !LJ_GC64 && tvislightud(&rd->argv[0]))
     t = ~LJ_TLIGHTUD;
   else
     t = ~itype(&rd->argv[0]);

src/lj_ir.c

@@ -307,6 +307,7 @@ TRef lj_ir_kgc(jit_State *J, GCobj *o, IRType t)
 {
   IRIns *ir, *cir = J->cur.ir;
   IRRef ref;
+  lua_assert(!LJ_GC64);  /* TODO_GC64: major changes required. */
   lua_assert(!isdead(J2G(J), o));
   for (ref = J->chain[IR_KGC]; ref; ref = cir[ref].prev)
     if (ir_kgc(&cir[ref]) == o)

src/lj_ir.h

@@ -320,6 +320,7 @@ IRTDEF(IRTENUM)
   IRT_PTR = LJ_64 ? IRT_P64 : IRT_P32,
   IRT_INTP = LJ_64 ? IRT_I64 : IRT_INT,
   IRT_UINTP = LJ_64 ? IRT_U64 : IRT_U32,
+  /* TODO_GC64: major changes required for all uses of IRT_P32. */
 
   /* Additional flags. */
   IRT_MARK = 0x20,	/* Marker for misc. purposes. */
@@ -371,7 +372,12 @@ typedef struct IRType1 { uint8_t irt; } IRType1;
 #define irt_isaddr(t)		(irt_typerange((t), IRT_LIGHTUD, IRT_UDATA))
 #define irt_isint64(t)		(irt_typerange((t), IRT_I64, IRT_U64))
 
-#if LJ_64
+#if LJ_GC64
+#define IRT_IS64 \
+  ((1u<<IRT_NUM)|(1u<<IRT_I64)|(1u<<IRT_U64)|(1u<<IRT_P64)|\
+   (1u<<IRT_LIGHTUD)|(1u<<IRT_STR)|(1u<<IRT_THREAD)|(1u<<IRT_PROTO)|\
+   (1u<<IRT_FUNC)|(1u<<IRT_CDATA)|(1u<<IRT_TAB)|(1u<<IRT_UDATA))
+#elif LJ_64
 #define IRT_IS64 \
   ((1u<<IRT_NUM)|(1u<<IRT_I64)|(1u<<IRT_U64)|(1u<<IRT_P64)|(1u<<IRT_LIGHTUD))
 #else
@@ -392,7 +398,7 @@ static LJ_AINLINE IRType itype2irt(const TValue *tv)
     return IRT_INT;
   else if (tvisnum(tv))
     return IRT_NUM;
-#if LJ_64
+#if LJ_64 && !LJ_GC64
   else if (tvislightud(tv))
     return IRT_LIGHTUD;
 #endif
@@ -547,6 +553,7 @@ typedef union IRIns {
   MRef ptr;		/* Pointer constant (overlaps op12). */
 } IRIns;
 
+/* TODO_GC64: major changes required. */
 #define ir_kgc(ir)	check_exp((ir)->o == IR_KGC, gcref((ir)->gcr))
 #define ir_kstr(ir)	(gco2str(ir_kgc((ir))))
 #define ir_ktab(ir)	(gco2tab(ir_kgc((ir))))

src/lj_jit.h

@@ -213,6 +213,9 @@ typedef struct GCtrace {
   uint8_t topslot;	/* Top stack slot already checked to be allocated. */
   uint8_t linktype;	/* Type of link. */
   IRRef nins;		/* Next IR instruction. Biased with REF_BIAS. */
+#if LJ_GC64
+  uint32_t unused_gc64;
+#endif
   GCRef gclist;
   IRIns *ir;		/* IR instructions/constants. Biased with REF_BIAS. */
   IRRef nk;		/* Lowest IR constant. Biased with REF_BIAS. */

src/lj_lib.h

@@ -47,7 +47,10 @@ LJ_FUNC GCtab *lj_lib_checktabornil(lua_State *L, int narg);
 LJ_FUNC int lj_lib_checkopt(lua_State *L, int narg, int def, const char *lst);
 
 /* Avoid including lj_frame.h. */
-#if LJ_FR2
+#if LJ_GC64
+#define lj_lib_upvalue(L, n) \
+  (&gcval(L->base-2)->fn.c.upvalue[(n)-1])
+#elif LJ_FR2
 #define lj_lib_upvalue(L, n) \
   (&gcref((L->base-2)->gcr)->fn.c.upvalue[(n)-1])
 #else

src/lj_obj.h

@@ -17,39 +17,73 @@
 
 /* Memory and GC object sizes. */
 typedef uint32_t MSize;
+#if LJ_GC64
+typedef uint64_t GCSize;
+#else
 typedef uint32_t GCSize;
+#endif
 
 /* Memory reference */
 typedef struct MRef {
+#if LJ_GC64
+  uint64_t ptr64;	/* True 64 bit pointer. */
+#else
   uint32_t ptr32;	/* Pseudo 32 bit pointer. */
+#endif
 } MRef;
 
+#if LJ_GC64
+#define mref(r, t)	((t *)(void *)(r).ptr64)
+
+#define setmref(r, p)	((r).ptr64 = (uint64_t)(void *)(p))
+#define setmrefr(r, v)	((r).ptr64 = (v).ptr64)
+#else
 #define mref(r, t)	((t *)(void *)(uintptr_t)(r).ptr32)
 
 #define setmref(r, p)	((r).ptr32 = (uint32_t)(uintptr_t)(void *)(p))
 #define setmrefr(r, v)	((r).ptr32 = (v).ptr32)
+#endif
 
 /* -- GC object references (32 bit address space) ------------------------- */
 
 /* GCobj reference */
 typedef struct GCRef {
+#if LJ_GC64
+  uint64_t gcptr64;	/* True 64 bit pointer. */
+#else
   uint32_t gcptr32;	/* Pseudo 32 bit pointer. */
+#endif
 } GCRef;
 
 /* Common GC header for all collectable objects. */
 #define GCHeader	GCRef nextgc; uint8_t marked; uint8_t gct
 /* This occupies 6 bytes, so use the next 2 bytes for non-32 bit fields. */
 
+#if LJ_GC64
+#define gcref(r)	((GCobj *)(r).gcptr64)
+#define gcrefp(r, t)	((t *)(void *)(r).gcptr64)
+#define gcrefu(r)	((r).gcptr64)
+#define gcrefeq(r1, r2)	((r1).gcptr64 == (r2).gcptr64)
+
+#define setgcref(r, gc)	((r).gcptr64 = (uint64_t)&(gc)->gch)
+#define setgcreft(r, gc, it) \
+  (r).gcptr64 = (uint64_t)&(gc)->gch | (((uint64_t)(it)) << 47)
+#define setgcrefp(r, p)	((r).gcptr64 = (uint64_t)(p))
+#define setgcrefnull(r)	((r).gcptr64 = 0)
+#define setgcrefr(r, v)	((r).gcptr64 = (v).gcptr64)
+#else
 #define gcref(r)	((GCobj *)(uintptr_t)(r).gcptr32)
 #define gcrefp(r, t)	((t *)(void *)(uintptr_t)(r).gcptr32)
 #define gcrefu(r)	((r).gcptr32)
 #define gcrefeq(r1, r2)	((r1).gcptr32 == (r2).gcptr32)
-#define gcnext(gc)	(gcref((gc)->gch.nextgc))
 
 #define setgcref(r, gc)	((r).gcptr32 = (uint32_t)(uintptr_t)&(gc)->gch)
 #define setgcrefp(r, p)	((r).gcptr32 = (uint32_t)(uintptr_t)(p))
 #define setgcrefnull(r)	((r).gcptr32 = 0)
 #define setgcrefr(r, v)	((r).gcptr32 = (v).gcptr32)
+#endif
+
+#define gcnext(gc)	(gcref((gc)->gch.nextgc))
 
 /* IMPORTANT NOTE:
 **
@@ -138,10 +172,16 @@ typedef union {
 typedef LJ_ALIGN(8) union TValue {
   uint64_t u64;		/* 64 bit pattern overlaps number. */
   lua_Number n;		/* Number object overlaps split tag/value object. */
+#if LJ_GC64
+  GCRef gcr;		/* GCobj reference with tag. */
+  int64_t it64;
+#endif
   struct {
     LJ_ENDIAN_LOHI(
       union {
+#if !LJ_GC64
 	GCRef gcr;	/* GCobj reference (if any). */
+#endif
 	int32_t i;	/* Integer value. */
       };
     , uint32_t it;	/* Internal object tag. Must overlap MSW of number. */
@@ -176,6 +216,8 @@ typedef const TValue cTValue;
 
 /* Internal object tags.
 **
+** Format for 32 bit GC references (!LJ_GC64):
+**
 ** Internal tags overlap the MSW of a number object (must be a double).
 ** Interpreted as a double these are special NaNs. The FPU only generates
 ** one type of NaN (0xfff8_0000_0000_0000). So MSWs > 0xfff80000 are available
@@ -190,6 +232,18 @@ typedef const TValue cTValue;
 ** int (LJ_DUALNUM)|  itype  |   int   |
 ** number           -------double------
 **
+** Format for 64 bit GC references (LJ_GC64):
+**
+** The upper 13 bits must be 1 (0xfff8...) for a special NaN. The next
+** 4 bits hold the internal tag. The lowest 47 bits either hold a pointer,
+** a zero-extended 32 bit integer or all bits set to 1 for primitive types.
+**
+**                     ------MSW------.------LSW------
+** primitive types    |1..1|itype|1..................1|
+** GC objects/lightud |1..1|itype|-------GCRef--------|
+** int (LJ_DUALNUM)   |1..1|itype|0..0|-----int-------|
+** number              ------------double-------------
+**
 ** ORDER LJ_T
 ** Primitive types nil/false/true must be first, lightuserdata next.
 ** GC objects are at the end, table/userdata must be lowest.
@@ -212,7 +266,7 @@ typedef const TValue cTValue;
 #define LJ_TNUMX		(~13u)
 
 /* Integers have itype == LJ_TISNUM doubles have itype < LJ_TISNUM */
-#if LJ_64
+#if LJ_64 && !LJ_GC64
 #define LJ_TISNUM		0xfffeffffu
 #else
 #define LJ_TISNUM		LJ_TNUMX
@@ -222,6 +276,10 @@ typedef const TValue cTValue;
 #define LJ_TISGCV		(LJ_TSTR+1)
 #define LJ_TISTABUD		LJ_TTAB
 
+#if LJ_GC64
+#define LJ_GCVMASK		(((uint64_t)1 << 47) - 1)
+#endif
+
 /* -- String object ------------------------------------------------------- */
 
 /* String object header. String payload follows. */
@@ -295,6 +353,9 @@ typedef struct GCproto {
   uint8_t numparams;	/* Number of parameters. */
   uint8_t framesize;	/* Fixed frame size. */
   MSize sizebc;		/* Number of bytecode instructions. */
+#if LJ_GC64
+  uint32_t unused_gc64;
+#endif
   GCRef gclist;
   MRef k;		/* Split constant array (points to the middle). */
   MRef uv;		/* Upvalue list. local slot|0x8000 or parent uv idx. */
@@ -406,7 +467,9 @@ typedef struct Node {
   TValue val;		/* Value object. Must be first field. */
   TValue key;		/* Key object. */
   MRef next;		/* Hash chain. */
+#if !LJ_GC64
   MRef freetop;		/* Top of free elements (stored in t->node[0]). */
+#endif
 } Node;
 
 LJ_STATIC_ASSERT(offsetof(Node, val) == 0);
@@ -421,12 +484,22 @@ typedef struct GCtab {
   MRef node;		/* Hash part. */
   uint32_t asize;	/* Size of array part (keys [0, asize-1]). */
   uint32_t hmask;	/* Hash part mask (size of hash part - 1). */
+#if LJ_GC64
+  MRef freetop;		/* Top of free elements. */
+#endif
 } GCtab;
 
 #define sizetabcolo(n)	((n)*sizeof(TValue) + sizeof(GCtab))
 #define tabref(r)	(&gcref((r))->tab)
 #define noderef(r)	(mref((r), Node))
 #define nextnode(n)	(mref((n)->next, Node))
+#if LJ_GC64
+#define getfreetop(t, n)	(noderef((t)->freetop))
+#define setfreetop(t, n, v)	(setmref((t)->freetop, (v)))
+#else
+#define getfreetop(t, n)	(noderef((n)->freetop))
+#define setfreetop(t, n, v)	(setmref((n)->freetop, (v)))
+#endif
 
 /* -- State objects ------------------------------------------------------- */
 
@@ -588,7 +661,9 @@ struct lua_State {
 #define registry(L)		(&G(L)->registrytv)
 
 /* Macros to access the currently executing (Lua) function. */
-#if LJ_FR2
+#if LJ_GC64
+#define curr_func(L)		(&gcval(L->base-2)->fn)
+#elif LJ_FR2
 #define curr_func(L)		(&gcref((L->base-2)->gcr)->fn)
 #else
 #define curr_func(L)		(&gcref((L->base-1)->fr.func)->fn)
@@ -656,12 +731,17 @@ typedef union GCobj {
 #endif
 
 /* Macros to test types. */
+#if LJ_GC64
+#define itype(o)	((uint32_t)((o)->it64 >> 47))
+#define tvisnil(o)	((o)->it64 == -1)
+#else
 #define itype(o)	((o)->it)
 #define tvisnil(o)	(itype(o) == LJ_TNIL)
+#endif
 #define tvisfalse(o)	(itype(o) == LJ_TFALSE)
 #define tvistrue(o)	(itype(o) == LJ_TTRUE)
 #define tvisbool(o)	(tvisfalse(o) || tvistrue(o))
-#if LJ_64
+#if LJ_64 && !LJ_GC64
 #define tvislightud(o)	(((int32_t)itype(o) >> 15) == -2)
 #else
 #define tvislightud(o)	(itype(o) == LJ_TLIGHTUD)
@@ -695,7 +775,7 @@ typedef union GCobj {
 #define rawnumequal(o1, o2)	((o1)->u64 == (o2)->u64)
 
 /* Macros to convert type ids. */
-#if LJ_64
+#if LJ_64 && !LJ_GC64
 #define itypemap(o) \
   (tvisnumber(o) ? ~LJ_TNUMX : tvislightud(o) ? ~LJ_TLIGHTUD : ~itype(o))
 #else
@@ -703,8 +783,12 @@ typedef union GCobj {
 #endif
 
 /* Macros to get tagged values. */
+#if LJ_GC64
+#define gcval(o)	((GCobj *)(gcrefu((o)->gcr) & LJ_GCVMASK))
+#else
 #define gcval(o)	(gcref((o)->gcr))
-#define boolV(o)	check_exp(tvisbool(o), (LJ_TFALSE - (o)->it))
+#endif
+#define boolV(o)	check_exp(tvisbool(o), (LJ_TFALSE - itype(o)))
 #if LJ_64
 #define lightudV(o) \
   check_exp(tvislightud(o), (void *)((o)->u64 & U64x(00007fff,ffffffff)))
@@ -723,14 +807,23 @@ typedef union GCobj {
 #define intV(o)		check_exp(tvisint(o), (int32_t)(o)->i)
 
 /* Macros to set tagged values. */
+#if LJ_GC64
+#define setitype(o, i)		((o)->it = ((i) << 15))
+#define setnilV(o)		((o)->it64 = -1)
+#define setpriV(o, x)		((o)->it64 = (int64_t)~((uint64_t)~(x)<<47))
+#define setboolV(o, x)		((o)->it64 = (int64_t)~((uint64_t)((x)+1)<<47))
+#else
 #define setitype(o, i)		((o)->it = (i))
 #define setnilV(o)		((o)->it = LJ_TNIL)
 #define setboolV(o, x)		((o)->it = LJ_TFALSE-(uint32_t)(x))
 #define setpriV(o, i)		(setitype((o), (i)))
+#endif
 
 static LJ_AINLINE void setlightudV(TValue *o, void *p)
 {
-#if LJ_64
+#if LJ_GC64
+  o->u64 = (uint64_t)p | (((uint64_t)LJ_TLIGHTUD) << 47);
+#elif LJ_64
   o->u64 = (uint64_t)p | (((uint64_t)0xffff) << 48);
 #else
   setgcrefp(o->gcr, p); setitype(o, LJ_TLIGHTUD);
@@ -759,7 +852,11 @@ static LJ_AINLINE void setlightudV(TValue *o, void *p)
 
 static LJ_AINLINE void setgcVraw(TValue *o, GCobj *v, uint32_t itype)
 {
+#if LJ_GC64
+  setgcreft(o->gcr, v, itype);
+#else
   setgcref(o->gcr, v); setitype(o, itype);
+#endif
 }
 
 static LJ_AINLINE void setgcV(lua_State *L, TValue *o, GCobj *v, uint32_t it)

src/lj_snap.c

@@ -602,6 +602,7 @@ static void snap_restoreval(jit_State *J, GCtrace *T, ExitState *ex,
   }
   if (LJ_UNLIKELY(bloomtest(rfilt, ref)))
     rs = snap_renameref(T, snapno, ref, rs);
+  lua_assert(!LJ_GC64);  /* TODO_GC64: handle 64 bit references. */
   if (ra_hasspill(regsp_spill(rs))) {  /* Restore from spill slot. */
     int32_t *sps = &ex->spill[regsp_spill(rs)];
     if (irt_isinteger(t)) {

src/lj_state.c

@@ -207,7 +207,9 @@ LUA_API lua_State *lua_newstate(lua_Alloc f, void *ud)
   setnilV(registry(L));
   setnilV(&g->nilnode.val);
   setnilV(&g->nilnode.key);
+#if !LJ_GC64
   setmref(g->nilnode.freetop, &g->nilnode);
+#endif
   lj_buf_init(NULL, &g->tmpbuf);
   g->gc.state = GCSpause;
   setgcref(g->gc.root, obj2gco(L));

src/lj_tab.c

@@ -29,7 +29,12 @@ static LJ_AINLINE Node *hashmask(const GCtab *t, uint32_t hash)
 #define hashlohi(t, lo, hi)	hashmask((t), hashrot((lo), (hi)))
 #define hashnum(t, o)		hashlohi((t), (o)->u32.lo, ((o)->u32.hi << 1))
 #define hashptr(t, p)		hashlohi((t), u32ptr(p), u32ptr(p) + HASH_BIAS)
+#if LJ_GC64
+#define hashgcref(t, r) \
+  hashlohi((t), (uint32_t)gcrefu(r), (uint32_t)(gcrefu(r) >> 32))
+#else
 #define hashgcref(t, r)		hashlohi((t), gcrefu(r), gcrefu(r) + HASH_BIAS)
+#endif
 
 /* Hash an arbitrary key and return its anchor position in the hash table. */
 static Node *hashkey(const GCtab *t, cTValue *key)
@@ -58,8 +63,8 @@ static LJ_AINLINE void newhpart(lua_State *L, GCtab *t, uint32_t hbits)
     lj_err_msg(L, LJ_ERR_TABOV);
   hsize = 1u << hbits;
   node = lj_mem_newvec(L, hsize, Node);
-  setmref(node->freetop, &node[hsize]);
   setmref(t->node, node);
+  setfreetop(t, node, &node[hsize]);
   t->hmask = hsize-1;
 }
 
@@ -98,6 +103,7 @@ static GCtab *newtab(lua_State *L, uint32_t asize, uint32_t hbits)
   GCtab *t;
   /* First try to colocate the array part. */
   if (LJ_MAX_COLOSIZE != 0 && asize > 0 && asize <= LJ_MAX_COLOSIZE) {
+    Node *nilnode;
     lua_assert((sizeof(GCtab) & 7) == 0);
     t = (GCtab *)lj_mem_newgco(L, sizetabcolo(asize));
     t->gct = ~LJ_TTAB;
@@ -107,8 +113,13 @@ static GCtab *newtab(lua_State *L, uint32_t asize, uint32_t hbits)
     setgcrefnull(t->metatable);
     t->asize = asize;
     t->hmask = 0;
-    setmref(t->node, &G(L)->nilnode);
+    nilnode = &G(L)->nilnode;
+    setmref(t->node, nilnode);
+#if LJ_GC64
+    setmref(t->freetop, nilnode);
+#endif
   } else {  /* Otherwise separately allocate the array part. */
+    Node *nilnode;
     t = lj_mem_newobj(L, GCtab);
     t->gct = ~LJ_TTAB;
     t->nomm = (uint8_t)~0;
@@ -117,7 +128,11 @@ static GCtab *newtab(lua_State *L, uint32_t asize, uint32_t hbits)
     setgcrefnull(t->metatable);
     t->asize = 0;  /* In case the array allocation fails. */
     t->hmask = 0;
-    setmref(t->node, &G(L)->nilnode);
+    nilnode = &G(L)->nilnode;
+    setmref(t->node, nilnode);
+#if LJ_GC64
+    setmref(t->freetop, nilnode);
+#endif
     if (asize > 0) {
       if (asize > LJ_MAX_ASIZE)
 	lj_err_msg(L, LJ_ERR_TABOV);
@@ -191,7 +206,7 @@ GCtab * LJ_FASTCALL lj_tab_dup(lua_State *L, const GCtab *kt)
     Node *node = noderef(t->node);
     Node *knode = noderef(kt->node);
     ptrdiff_t d = (char *)node - (char *)knode;
-    setmref(node->freetop, (Node *)((char *)noderef(knode->freetop) + d));
+    setfreetop(t, node, (Node *)((char *)getfreetop(kt, knode) + d));
     for (i = 0; i <= hmask; i++) {
       Node *kn = &knode[i];
       Node *n = &node[i];
@@ -210,7 +225,7 @@ void LJ_FASTCALL lj_tab_clear(GCtab *t)
   clearapart(t);
   if (t->hmask > 0) {
     Node *node = noderef(t->node);
-    setmref(node->freetop, &node[t->hmask+1]);
+    setfreetop(t, node, &node[t->hmask+1]);
     clearhpart(t);
   }
 }
@@ -264,6 +279,9 @@ static void resizetab(lua_State *L, GCtab *t, uint32_t asize, uint32_t hbits)
   } else {
     global_State *g = G(L);
     setmref(t->node, &g->nilnode);
+#if LJ_GC64
+    setmref(t->freetop, &g->nilnode);
+#endif
     t->hmask = 0;
   }
   if (asize < oldasize) {  /* Array part shrinks? */
@@ -445,7 +463,7 @@ TValue *lj_tab_newkey(lua_State *L, GCtab *t, cTValue *key)
   Node *n = hashkey(t, key);
   if (!tvisnil(&n->val) || t->hmask == 0) {
     Node *nodebase = noderef(t->node);
-    Node *collide, *freenode = noderef(nodebase->freetop);
+    Node *collide, *freenode = getfreetop(t, nodebase);
     lua_assert(freenode >= nodebase && freenode <= nodebase+t->hmask+1);
     do {
       if (freenode == nodebase) {  /* No free node found? */
@@ -453,7 +471,7 @@ TValue *lj_tab_newkey(lua_State *L, GCtab *t, cTValue *key)
 	return lj_tab_set(L, t, key);  /* Retry key insertion. */
       }
     } while (!tvisnil(&(--freenode)->key));
-    setmref(nodebase->freetop, freenode);
+    setfreetop(t, nodebase, freenode);
     lua_assert(freenode != &G(L)->nilnode);
     collide = hashkey(t, &n->key);
     if (collide != n) {  /* Colliding node not the main node? */