From cb481ddc8f9d92913ba07d998f4274bbf9711077 Mon Sep 17 00:00:00 2001 From: Mike Pall Date: Sat, 3 Jan 2015 15:23:58 +0100 Subject: [PATCH] Add LJ_GC64 mode: 64 bit GC object references. Actually NaN tagging with 47 bit pointers and 13+4 bit tags. --- src/lj_alloc.c | 12 ++--- src/lj_api.c | 4 +- src/lj_arch.h | 13 +++++- src/lj_def.h | 15 +++++-- src/lj_ffrecord.c | 2 +- src/lj_ir.c | 1 + src/lj_ir.h | 11 ++++- src/lj_jit.h | 3 ++ src/lj_lib.h | 5 ++- src/lj_obj.h | 111 +++++++++++++++++++++++++++++++++++++++++++--- src/lj_snap.c | 1 + src/lj_state.c | 2 + src/lj_tab.c | 32 ++++++++++--- 13 files changed, 183 insertions(+), 29 deletions(-) diff --git a/src/lj_alloc.c b/src/lj_alloc.c index 7c7ec678..0aad826d 100644 --- a/src/lj_alloc.c +++ b/src/lj_alloc.c @@ -77,7 +77,7 @@ #define WIN32_LEAN_AND_MEAN #include -#if LJ_64 +#if LJ_64 && !LJ_GC64 /* Undocumented, but hey, that's what we all love so much about Windows. */ typedef long (*PNTAVM)(HANDLE handle, void **addr, ULONG zbits, @@ -174,8 +174,10 @@ static LJ_AINLINE int CALL_MUNMAP(void *ptr, size_t size) #endif #define MMAP_FLAGS (MAP_PRIVATE|MAP_ANONYMOUS) -#if LJ_64 -/* 64 bit mode needs special support for allocating memory in the lower 2GB. */ +#if LJ_64 && !LJ_GC64 +/* 64 bit mode with 32 bit pointers needs special support for allocating +** memory in the lower 2GB. +*/ #if defined(MAP_32BIT) @@ -258,7 +260,7 @@ static LJ_AINLINE void *CALL_MMAP(size_t size) #else -/* 32 bit mode is easy. */ +/* 32 bit mode and GC64 mode is easy. */ static LJ_AINLINE void *CALL_MMAP(size_t size) { int olderr = errno; @@ -294,7 +296,7 @@ static LJ_AINLINE void *CALL_MREMAP_(void *ptr, size_t osz, size_t nsz, #define CALL_MREMAP(addr, osz, nsz, mv) CALL_MREMAP_((addr), (osz), (nsz), (mv)) #define CALL_MREMAP_NOMOVE 0 #define CALL_MREMAP_MAYMOVE 1 -#if LJ_64 +#if LJ_64 && !LJ_GC64 #define CALL_MREMAP_MV CALL_MREMAP_NOMOVE #else #define CALL_MREMAP_MV CALL_MREMAP_MAYMOVE diff --git a/src/lj_api.c b/src/lj_api.c index 6ad09b9d..f1843723 100644 --- a/src/lj_api.c +++ b/src/lj_api.c @@ -189,7 +189,7 @@ LUA_API int lua_type(lua_State *L, int idx) cTValue *o = index2adr(L, idx); if (tvisnumber(o)) { return LUA_TNUMBER; -#if LJ_64 +#if LJ_64 && !LJ_GC64 } else if (tvislightud(o)) { return LUA_TLIGHTUSERDATA; #endif @@ -269,7 +269,7 @@ LUA_API int lua_equal(lua_State *L, int idx1, int idx2) return 0; } else if (tvispri(o1)) { return o1 != niltv(L) && o2 != niltv(L); -#if LJ_64 +#if LJ_64 && !LJ_GC64 } else if (tvislightud(o1)) { return o1->u64 == o2->u64; #endif diff --git a/src/lj_arch.h b/src/lj_arch.h index 8f3796fc..18555b41 100644 --- a/src/lj_arch.h +++ b/src/lj_arch.h @@ -365,11 +365,22 @@ #endif #endif +/* 64 bit GC references. */ +#if LJ_TARGET_GC64 +#define LJ_GC64 1 +#else +#define LJ_GC64 0 +#endif + /* 2-slot frame info. */ +#if LJ_GC64 +#define LJ_FR2 1 +#else #define LJ_FR2 0 +#endif /* Disable or enable the JIT compiler. */ -#if defined(LUAJIT_DISABLE_JIT) || defined(LJ_ARCH_NOJIT) || defined(LJ_OS_NOJIT) || LJ_FR2 +#if defined(LUAJIT_DISABLE_JIT) || defined(LJ_ARCH_NOJIT) || defined(LJ_OS_NOJIT) || LJ_FR2 || LJ_GC64 #define LJ_HASJIT 0 #else #define LJ_HASJIT 1 diff --git a/src/lj_def.h b/src/lj_def.h index 93420ba5..f4231239 100644 --- a/src/lj_def.h +++ b/src/lj_def.h @@ -47,7 +47,9 @@ typedef unsigned int uintptr_t; /* Various VM limits. */ #define LJ_MAX_MEM32 0x7fffff00 /* Max. 32 bit memory allocation. */ -#define LJ_MAX_MEM LJ_MAX_MEM32 /* Max. total memory allocation. */ +#define LJ_MAX_MEM64 ((uint64_t)1<<47) /* Max. 64 bit memory allocation. */ +/* Max. total memory allocation. */ +#define LJ_MAX_MEM (LJ_GC64 ? LJ_MAX_MEM64 : LJ_MAX_MEM32) #define LJ_MAX_ALLOC LJ_MAX_MEM /* Max. individual allocation length. */ #define LJ_MAX_STR LJ_MAX_MEM32 /* Max. string length. */ #define LJ_MAX_BUF LJ_MAX_MEM32 /* Max. buffer length. */ @@ -67,7 +69,7 @@ typedef unsigned int uintptr_t; #define LJ_MAX_UPVAL 60 /* Max. # of upvalues. */ #define LJ_MAX_IDXCHAIN 100 /* __index/__newindex chain limit. */ -#define LJ_STACK_EXTRA 5 /* Extra stack space (metamethods). */ +#define LJ_STACK_EXTRA (5+2*LJ_FR2) /* Extra stack space (metamethods). */ #define LJ_NUM_CBPAGE 1 /* Number of FFI callback pages. */ @@ -101,7 +103,14 @@ typedef unsigned int uintptr_t; #define checki32(x) ((x) == (int32_t)(x)) #define checku32(x) ((x) == (uint32_t)(x)) #define checkptr32(x) ((uintptr_t)(x) == (uint32_t)(uintptr_t)(x)) -#define checkptrGC(x) (checkptr32(x)) +#define checkptr47(x) (((uint64_t)(x) >> 47) == 0) +#if LJ_GC64 +#define checkptrGC(x) (checkptr47((x))) +#elif LJ_64 +#define checkptrGC(x) (checkptr32((x))) +#else +#define checkptrGC(x) 1 +#endif /* Every half-decent C compiler transforms this into a rotate instruction. */ #define lj_rol(x, n) (((x)<<(n)) | ((x)>>(-(int)(n)&(8*sizeof(x)-1)))) diff --git a/src/lj_ffrecord.c b/src/lj_ffrecord.c index 55cdc63e..6bf26070 100644 --- a/src/lj_ffrecord.c +++ b/src/lj_ffrecord.c @@ -196,7 +196,7 @@ static void LJ_FASTCALL recff_type(jit_State *J, RecordFFData *rd) uint32_t t; if (tvisnumber(&rd->argv[0])) t = ~LJ_TNUMX; - else if (LJ_64 && tvislightud(&rd->argv[0])) + else if (LJ_64 && !LJ_GC64 && tvislightud(&rd->argv[0])) t = ~LJ_TLIGHTUD; else t = ~itype(&rd->argv[0]); diff --git a/src/lj_ir.c b/src/lj_ir.c index 0689bc22..12d38909 100644 --- a/src/lj_ir.c +++ b/src/lj_ir.c @@ -307,6 +307,7 @@ TRef lj_ir_kgc(jit_State *J, GCobj *o, IRType t) { IRIns *ir, *cir = J->cur.ir; IRRef ref; + lua_assert(!LJ_GC64); /* TODO_GC64: major changes required. */ lua_assert(!isdead(J2G(J), o)); for (ref = J->chain[IR_KGC]; ref; ref = cir[ref].prev) if (ir_kgc(&cir[ref]) == o) diff --git a/src/lj_ir.h b/src/lj_ir.h index 14b86165..bc89edf9 100644 --- a/src/lj_ir.h +++ b/src/lj_ir.h @@ -320,6 +320,7 @@ IRTDEF(IRTENUM) IRT_PTR = LJ_64 ? IRT_P64 : IRT_P32, IRT_INTP = LJ_64 ? IRT_I64 : IRT_INT, IRT_UINTP = LJ_64 ? IRT_U64 : IRT_U32, + /* TODO_GC64: major changes required for all uses of IRT_P32. */ /* Additional flags. */ IRT_MARK = 0x20, /* Marker for misc. purposes. */ @@ -371,7 +372,12 @@ typedef struct IRType1 { uint8_t irt; } IRType1; #define irt_isaddr(t) (irt_typerange((t), IRT_LIGHTUD, IRT_UDATA)) #define irt_isint64(t) (irt_typerange((t), IRT_I64, IRT_U64)) -#if LJ_64 +#if LJ_GC64 +#define IRT_IS64 \ + ((1u<o == IR_KGC, gcref((ir)->gcr)) #define ir_kstr(ir) (gco2str(ir_kgc((ir)))) #define ir_ktab(ir) (gco2tab(ir_kgc((ir)))) diff --git a/src/lj_jit.h b/src/lj_jit.h index 4246e9db..ed92f62d 100644 --- a/src/lj_jit.h +++ b/src/lj_jit.h @@ -213,6 +213,9 @@ typedef struct GCtrace { uint8_t topslot; /* Top stack slot already checked to be allocated. */ uint8_t linktype; /* Type of link. */ IRRef nins; /* Next IR instruction. Biased with REF_BIAS. */ +#if LJ_GC64 + uint32_t unused_gc64; +#endif GCRef gclist; IRIns *ir; /* IR instructions/constants. Biased with REF_BIAS. */ IRRef nk; /* Lowest IR constant. Biased with REF_BIAS. */ diff --git a/src/lj_lib.h b/src/lj_lib.h index dff99c09..95e4d4a6 100644 --- a/src/lj_lib.h +++ b/src/lj_lib.h @@ -47,7 +47,10 @@ LJ_FUNC GCtab *lj_lib_checktabornil(lua_State *L, int narg); LJ_FUNC int lj_lib_checkopt(lua_State *L, int narg, int def, const char *lst); /* Avoid including lj_frame.h. */ -#if LJ_FR2 +#if LJ_GC64 +#define lj_lib_upvalue(L, n) \ + (&gcval(L->base-2)->fn.c.upvalue[(n)-1]) +#elif LJ_FR2 #define lj_lib_upvalue(L, n) \ (&gcref((L->base-2)->gcr)->fn.c.upvalue[(n)-1]) #else diff --git a/src/lj_obj.h b/src/lj_obj.h index 438c83d4..d5809229 100644 --- a/src/lj_obj.h +++ b/src/lj_obj.h @@ -17,39 +17,73 @@ /* Memory and GC object sizes. */ typedef uint32_t MSize; +#if LJ_GC64 +typedef uint64_t GCSize; +#else typedef uint32_t GCSize; +#endif /* Memory reference */ typedef struct MRef { +#if LJ_GC64 + uint64_t ptr64; /* True 64 bit pointer. */ +#else uint32_t ptr32; /* Pseudo 32 bit pointer. */ +#endif } MRef; +#if LJ_GC64 +#define mref(r, t) ((t *)(void *)(r).ptr64) + +#define setmref(r, p) ((r).ptr64 = (uint64_t)(void *)(p)) +#define setmrefr(r, v) ((r).ptr64 = (v).ptr64) +#else #define mref(r, t) ((t *)(void *)(uintptr_t)(r).ptr32) #define setmref(r, p) ((r).ptr32 = (uint32_t)(uintptr_t)(void *)(p)) #define setmrefr(r, v) ((r).ptr32 = (v).ptr32) +#endif /* -- GC object references (32 bit address space) ------------------------- */ /* GCobj reference */ typedef struct GCRef { +#if LJ_GC64 + uint64_t gcptr64; /* True 64 bit pointer. */ +#else uint32_t gcptr32; /* Pseudo 32 bit pointer. */ +#endif } GCRef; /* Common GC header for all collectable objects. */ #define GCHeader GCRef nextgc; uint8_t marked; uint8_t gct /* This occupies 6 bytes, so use the next 2 bytes for non-32 bit fields. */ +#if LJ_GC64 +#define gcref(r) ((GCobj *)(r).gcptr64) +#define gcrefp(r, t) ((t *)(void *)(r).gcptr64) +#define gcrefu(r) ((r).gcptr64) +#define gcrefeq(r1, r2) ((r1).gcptr64 == (r2).gcptr64) + +#define setgcref(r, gc) ((r).gcptr64 = (uint64_t)&(gc)->gch) +#define setgcreft(r, gc, it) \ + (r).gcptr64 = (uint64_t)&(gc)->gch | (((uint64_t)(it)) << 47) +#define setgcrefp(r, p) ((r).gcptr64 = (uint64_t)(p)) +#define setgcrefnull(r) ((r).gcptr64 = 0) +#define setgcrefr(r, v) ((r).gcptr64 = (v).gcptr64) +#else #define gcref(r) ((GCobj *)(uintptr_t)(r).gcptr32) #define gcrefp(r, t) ((t *)(void *)(uintptr_t)(r).gcptr32) #define gcrefu(r) ((r).gcptr32) #define gcrefeq(r1, r2) ((r1).gcptr32 == (r2).gcptr32) -#define gcnext(gc) (gcref((gc)->gch.nextgc)) #define setgcref(r, gc) ((r).gcptr32 = (uint32_t)(uintptr_t)&(gc)->gch) #define setgcrefp(r, p) ((r).gcptr32 = (uint32_t)(uintptr_t)(p)) #define setgcrefnull(r) ((r).gcptr32 = 0) #define setgcrefr(r, v) ((r).gcptr32 = (v).gcptr32) +#endif + +#define gcnext(gc) (gcref((gc)->gch.nextgc)) /* IMPORTANT NOTE: ** @@ -138,10 +172,16 @@ typedef union { typedef LJ_ALIGN(8) union TValue { uint64_t u64; /* 64 bit pattern overlaps number. */ lua_Number n; /* Number object overlaps split tag/value object. */ +#if LJ_GC64 + GCRef gcr; /* GCobj reference with tag. */ + int64_t it64; +#endif struct { LJ_ENDIAN_LOHI( union { +#if !LJ_GC64 GCRef gcr; /* GCobj reference (if any). */ +#endif int32_t i; /* Integer value. */ }; , uint32_t it; /* Internal object tag. Must overlap MSW of number. */ @@ -176,6 +216,8 @@ typedef const TValue cTValue; /* Internal object tags. ** +** Format for 32 bit GC references (!LJ_GC64): +** ** Internal tags overlap the MSW of a number object (must be a double). ** Interpreted as a double these are special NaNs. The FPU only generates ** one type of NaN (0xfff8_0000_0000_0000). So MSWs > 0xfff80000 are available @@ -190,6 +232,18 @@ typedef const TValue cTValue; ** int (LJ_DUALNUM)| itype | int | ** number -------double------ ** +** Format for 64 bit GC references (LJ_GC64): +** +** The upper 13 bits must be 1 (0xfff8...) for a special NaN. The next +** 4 bits hold the internal tag. The lowest 47 bits either hold a pointer, +** a zero-extended 32 bit integer or all bits set to 1 for primitive types. +** +** ------MSW------.------LSW------ +** primitive types |1..1|itype|1..................1| +** GC objects/lightud |1..1|itype|-------GCRef--------| +** int (LJ_DUALNUM) |1..1|itype|0..0|-----int-------| +** number ------------double------------- +** ** ORDER LJ_T ** Primitive types nil/false/true must be first, lightuserdata next. ** GC objects are at the end, table/userdata must be lowest. @@ -212,7 +266,7 @@ typedef const TValue cTValue; #define LJ_TNUMX (~13u) /* Integers have itype == LJ_TISNUM doubles have itype < LJ_TISNUM */ -#if LJ_64 +#if LJ_64 && !LJ_GC64 #define LJ_TISNUM 0xfffeffffu #else #define LJ_TISNUM LJ_TNUMX @@ -222,6 +276,10 @@ typedef const TValue cTValue; #define LJ_TISGCV (LJ_TSTR+1) #define LJ_TISTABUD LJ_TTAB +#if LJ_GC64 +#define LJ_GCVMASK (((uint64_t)1 << 47) - 1) +#endif + /* -- String object ------------------------------------------------------- */ /* String object header. String payload follows. */ @@ -295,6 +353,9 @@ typedef struct GCproto { uint8_t numparams; /* Number of parameters. */ uint8_t framesize; /* Fixed frame size. */ MSize sizebc; /* Number of bytecode instructions. */ +#if LJ_GC64 + uint32_t unused_gc64; +#endif GCRef gclist; MRef k; /* Split constant array (points to the middle). */ MRef uv; /* Upvalue list. local slot|0x8000 or parent uv idx. */ @@ -406,7 +467,9 @@ typedef struct Node { TValue val; /* Value object. Must be first field. */ TValue key; /* Key object. */ MRef next; /* Hash chain. */ +#if !LJ_GC64 MRef freetop; /* Top of free elements (stored in t->node[0]). */ +#endif } Node; LJ_STATIC_ASSERT(offsetof(Node, val) == 0); @@ -421,12 +484,22 @@ typedef struct GCtab { MRef node; /* Hash part. */ uint32_t asize; /* Size of array part (keys [0, asize-1]). */ uint32_t hmask; /* Hash part mask (size of hash part - 1). */ +#if LJ_GC64 + MRef freetop; /* Top of free elements. */ +#endif } GCtab; #define sizetabcolo(n) ((n)*sizeof(TValue) + sizeof(GCtab)) #define tabref(r) (&gcref((r))->tab) #define noderef(r) (mref((r), Node)) #define nextnode(n) (mref((n)->next, Node)) +#if LJ_GC64 +#define getfreetop(t, n) (noderef((t)->freetop)) +#define setfreetop(t, n, v) (setmref((t)->freetop, (v))) +#else +#define getfreetop(t, n) (noderef((n)->freetop)) +#define setfreetop(t, n, v) (setmref((n)->freetop, (v))) +#endif /* -- State objects ------------------------------------------------------- */ @@ -588,7 +661,9 @@ struct lua_State { #define registry(L) (&G(L)->registrytv) /* Macros to access the currently executing (Lua) function. */ -#if LJ_FR2 +#if LJ_GC64 +#define curr_func(L) (&gcval(L->base-2)->fn) +#elif LJ_FR2 #define curr_func(L) (&gcref((L->base-2)->gcr)->fn) #else #define curr_func(L) (&gcref((L->base-1)->fr.func)->fn) @@ -656,12 +731,17 @@ typedef union GCobj { #endif /* Macros to test types. */ +#if LJ_GC64 +#define itype(o) ((uint32_t)((o)->it64 >> 47)) +#define tvisnil(o) ((o)->it64 == -1) +#else #define itype(o) ((o)->it) #define tvisnil(o) (itype(o) == LJ_TNIL) +#endif #define tvisfalse(o) (itype(o) == LJ_TFALSE) #define tvistrue(o) (itype(o) == LJ_TTRUE) #define tvisbool(o) (tvisfalse(o) || tvistrue(o)) -#if LJ_64 +#if LJ_64 && !LJ_GC64 #define tvislightud(o) (((int32_t)itype(o) >> 15) == -2) #else #define tvislightud(o) (itype(o) == LJ_TLIGHTUD) @@ -695,7 +775,7 @@ typedef union GCobj { #define rawnumequal(o1, o2) ((o1)->u64 == (o2)->u64) /* Macros to convert type ids. */ -#if LJ_64 +#if LJ_64 && !LJ_GC64 #define itypemap(o) \ (tvisnumber(o) ? ~LJ_TNUMX : tvislightud(o) ? ~LJ_TLIGHTUD : ~itype(o)) #else @@ -703,8 +783,12 @@ typedef union GCobj { #endif /* Macros to get tagged values. */ +#if LJ_GC64 +#define gcval(o) ((GCobj *)(gcrefu((o)->gcr) & LJ_GCVMASK)) +#else #define gcval(o) (gcref((o)->gcr)) -#define boolV(o) check_exp(tvisbool(o), (LJ_TFALSE - (o)->it)) +#endif +#define boolV(o) check_exp(tvisbool(o), (LJ_TFALSE - itype(o))) #if LJ_64 #define lightudV(o) \ check_exp(tvislightud(o), (void *)((o)->u64 & U64x(00007fff,ffffffff))) @@ -723,14 +807,23 @@ typedef union GCobj { #define intV(o) check_exp(tvisint(o), (int32_t)(o)->i) /* Macros to set tagged values. */ +#if LJ_GC64 +#define setitype(o, i) ((o)->it = ((i) << 15)) +#define setnilV(o) ((o)->it64 = -1) +#define setpriV(o, x) ((o)->it64 = (int64_t)~((uint64_t)~(x)<<47)) +#define setboolV(o, x) ((o)->it64 = (int64_t)~((uint64_t)((x)+1)<<47)) +#else #define setitype(o, i) ((o)->it = (i)) #define setnilV(o) ((o)->it = LJ_TNIL) #define setboolV(o, x) ((o)->it = LJ_TFALSE-(uint32_t)(x)) #define setpriV(o, i) (setitype((o), (i))) +#endif static LJ_AINLINE void setlightudV(TValue *o, void *p) { -#if LJ_64 +#if LJ_GC64 + o->u64 = (uint64_t)p | (((uint64_t)LJ_TLIGHTUD) << 47); +#elif LJ_64 o->u64 = (uint64_t)p | (((uint64_t)0xffff) << 48); #else setgcrefp(o->gcr, p); setitype(o, LJ_TLIGHTUD); @@ -759,7 +852,11 @@ static LJ_AINLINE void setlightudV(TValue *o, void *p) static LJ_AINLINE void setgcVraw(TValue *o, GCobj *v, uint32_t itype) { +#if LJ_GC64 + setgcreft(o->gcr, v, itype); +#else setgcref(o->gcr, v); setitype(o, itype); +#endif } static LJ_AINLINE void setgcV(lua_State *L, TValue *o, GCobj *v, uint32_t it) diff --git a/src/lj_snap.c b/src/lj_snap.c index 4bac2d28..abc943b7 100644 --- a/src/lj_snap.c +++ b/src/lj_snap.c @@ -602,6 +602,7 @@ static void snap_restoreval(jit_State *J, GCtrace *T, ExitState *ex, } if (LJ_UNLIKELY(bloomtest(rfilt, ref))) rs = snap_renameref(T, snapno, ref, rs); + lua_assert(!LJ_GC64); /* TODO_GC64: handle 64 bit references. */ if (ra_hasspill(regsp_spill(rs))) { /* Restore from spill slot. */ int32_t *sps = &ex->spill[regsp_spill(rs)]; if (irt_isinteger(t)) { diff --git a/src/lj_state.c b/src/lj_state.c index f7cdb8f3..2b19b747 100644 --- a/src/lj_state.c +++ b/src/lj_state.c @@ -207,7 +207,9 @@ LUA_API lua_State *lua_newstate(lua_Alloc f, void *ud) setnilV(registry(L)); setnilV(&g->nilnode.val); setnilV(&g->nilnode.key); +#if !LJ_GC64 setmref(g->nilnode.freetop, &g->nilnode); +#endif lj_buf_init(NULL, &g->tmpbuf); g->gc.state = GCSpause; setgcref(g->gc.root, obj2gco(L)); diff --git a/src/lj_tab.c b/src/lj_tab.c index ef19ba97..b6bb7805 100644 --- a/src/lj_tab.c +++ b/src/lj_tab.c @@ -29,7 +29,12 @@ static LJ_AINLINE Node *hashmask(const GCtab *t, uint32_t hash) #define hashlohi(t, lo, hi) hashmask((t), hashrot((lo), (hi))) #define hashnum(t, o) hashlohi((t), (o)->u32.lo, ((o)->u32.hi << 1)) #define hashptr(t, p) hashlohi((t), u32ptr(p), u32ptr(p) + HASH_BIAS) +#if LJ_GC64 +#define hashgcref(t, r) \ + hashlohi((t), (uint32_t)gcrefu(r), (uint32_t)(gcrefu(r) >> 32)) +#else #define hashgcref(t, r) hashlohi((t), gcrefu(r), gcrefu(r) + HASH_BIAS) +#endif /* Hash an arbitrary key and return its anchor position in the hash table. */ static Node *hashkey(const GCtab *t, cTValue *key) @@ -58,8 +63,8 @@ static LJ_AINLINE void newhpart(lua_State *L, GCtab *t, uint32_t hbits) lj_err_msg(L, LJ_ERR_TABOV); hsize = 1u << hbits; node = lj_mem_newvec(L, hsize, Node); - setmref(node->freetop, &node[hsize]); setmref(t->node, node); + setfreetop(t, node, &node[hsize]); t->hmask = hsize-1; } @@ -98,6 +103,7 @@ static GCtab *newtab(lua_State *L, uint32_t asize, uint32_t hbits) GCtab *t; /* First try to colocate the array part. */ if (LJ_MAX_COLOSIZE != 0 && asize > 0 && asize <= LJ_MAX_COLOSIZE) { + Node *nilnode; lua_assert((sizeof(GCtab) & 7) == 0); t = (GCtab *)lj_mem_newgco(L, sizetabcolo(asize)); t->gct = ~LJ_TTAB; @@ -107,8 +113,13 @@ static GCtab *newtab(lua_State *L, uint32_t asize, uint32_t hbits) setgcrefnull(t->metatable); t->asize = asize; t->hmask = 0; - setmref(t->node, &G(L)->nilnode); + nilnode = &G(L)->nilnode; + setmref(t->node, nilnode); +#if LJ_GC64 + setmref(t->freetop, nilnode); +#endif } else { /* Otherwise separately allocate the array part. */ + Node *nilnode; t = lj_mem_newobj(L, GCtab); t->gct = ~LJ_TTAB; t->nomm = (uint8_t)~0; @@ -117,7 +128,11 @@ static GCtab *newtab(lua_State *L, uint32_t asize, uint32_t hbits) setgcrefnull(t->metatable); t->asize = 0; /* In case the array allocation fails. */ t->hmask = 0; - setmref(t->node, &G(L)->nilnode); + nilnode = &G(L)->nilnode; + setmref(t->node, nilnode); +#if LJ_GC64 + setmref(t->freetop, nilnode); +#endif if (asize > 0) { if (asize > LJ_MAX_ASIZE) lj_err_msg(L, LJ_ERR_TABOV); @@ -191,7 +206,7 @@ GCtab * LJ_FASTCALL lj_tab_dup(lua_State *L, const GCtab *kt) Node *node = noderef(t->node); Node *knode = noderef(kt->node); ptrdiff_t d = (char *)node - (char *)knode; - setmref(node->freetop, (Node *)((char *)noderef(knode->freetop) + d)); + setfreetop(t, node, (Node *)((char *)getfreetop(kt, knode) + d)); for (i = 0; i <= hmask; i++) { Node *kn = &knode[i]; Node *n = &node[i]; @@ -210,7 +225,7 @@ void LJ_FASTCALL lj_tab_clear(GCtab *t) clearapart(t); if (t->hmask > 0) { Node *node = noderef(t->node); - setmref(node->freetop, &node[t->hmask+1]); + setfreetop(t, node, &node[t->hmask+1]); clearhpart(t); } } @@ -264,6 +279,9 @@ static void resizetab(lua_State *L, GCtab *t, uint32_t asize, uint32_t hbits) } else { global_State *g = G(L); setmref(t->node, &g->nilnode); +#if LJ_GC64 + setmref(t->freetop, &g->nilnode); +#endif t->hmask = 0; } if (asize < oldasize) { /* Array part shrinks? */ @@ -445,7 +463,7 @@ TValue *lj_tab_newkey(lua_State *L, GCtab *t, cTValue *key) Node *n = hashkey(t, key); if (!tvisnil(&n->val) || t->hmask == 0) { Node *nodebase = noderef(t->node); - Node *collide, *freenode = noderef(nodebase->freetop); + Node *collide, *freenode = getfreetop(t, nodebase); lua_assert(freenode >= nodebase && freenode <= nodebase+t->hmask+1); do { if (freenode == nodebase) { /* No free node found? */ @@ -453,7 +471,7 @@ TValue *lj_tab_newkey(lua_State *L, GCtab *t, cTValue *key) return lj_tab_set(L, t, key); /* Retry key insertion. */ } } while (!tvisnil(&(--freenode)->key)); - setmref(nodebase->freetop, freenode); + setfreetop(t, nodebase, freenode); lua_assert(freenode != &G(L)->nilnode); collide = hashkey(t, &n->key); if (collide != n) { /* Colliding node not the main node? */