Add support for full-range 64 bit lightuserdata.

This commit is contained in:
Mike Pall 2020-09-30 01:31:27 +02:00
parent e67e2040be
commit e9af1abec5
17 changed files with 121 additions and 67 deletions

View File

@ -91,17 +91,6 @@ handled correctly. The error may fall through an on-trace
<tt>lua_atpanic</tt> on x64. This issue will be fixed with the new
garbage collector.
</li>
<li>
LuaJIT on 64 bit systems provides a <b>limited range</b> of 47 bits for the
<b>legacy <tt>lightuserdata</tt></b> data type.
This is only relevant on x64 systems which use the negative part of the
virtual address space in user mode, e.g. Solaris/x64, and on ARM64 systems
configured with a 48 bit or 52 bit VA.
Avoid using <tt>lightuserdata</tt> to hold pointers that may point outside
of that range, e.g. variables on the stack. In general, avoid this data
type for new code and replace it with (much more performant) FFI bindings.
FFI cdata pointers can address the full 64 bit range.
</li>
</ul>
<br class="flush">
</div>

View File

@ -315,7 +315,9 @@ local function formatk(tr, idx, sn)
local tn = type(k)
local s
if tn == "number" then
if band(sn or 0, 0x30000) ~= 0 then
if t < 12 then
s = k == 0 and "NULL" or format("[0x%08x]", k)
elseif band(sn or 0, 0x30000) ~= 0 then
s = band(sn, 0x20000) ~= 0 and "contpc" or "ftsz"
elseif k == 2^52+2^51 then
s = "bias"

View File

@ -231,8 +231,8 @@ LJLIB_CF(debug_upvalueid)
int32_t n = lj_lib_checkint(L, 2) - 1;
if ((uint32_t)n >= fn->l.nupvalues)
lj_err_arg(L, 2, LJ_ERR_IDXRNG);
setlightudV(L->top-1, isluafunc(fn) ? (void *)gcref(fn->l.uvptr[n]) :
(void *)&fn->c.upvalue[n]);
lua_pushlightuserdata(L, isluafunc(fn) ? (void *)gcref(fn->l.uvptr[n]) :
(void *)&fn->c.upvalue[n]);
return 1;
}
@ -283,13 +283,13 @@ LJLIB_CF(debug_setuservalue)
/* ------------------------------------------------------------------------ */
#define KEY_HOOK ((void *)0x3004)
#define KEY_HOOK (U64x(80000000,00000000)|'h')
static void hookf(lua_State *L, lua_Debug *ar)
{
static const char *const hooknames[] =
{"call", "return", "line", "count", "tail return"};
lua_pushlightuserdata(L, KEY_HOOK);
(L->top++)->u64 = KEY_HOOK;
lua_rawget(L, LUA_REGISTRYINDEX);
if (lua_isfunction(L, -1)) {
lua_pushstring(L, hooknames[(int)ar->event]);
@ -334,7 +334,7 @@ LJLIB_CF(debug_sethook)
count = luaL_optint(L, arg+3, 0);
func = hookf; mask = makemask(smask, count);
}
lua_pushlightuserdata(L, KEY_HOOK);
(L->top++)->u64 = KEY_HOOK;
lua_pushvalue(L, arg+1);
lua_rawset(L, LUA_REGISTRYINDEX);
lua_sethook(L, func, mask, count);
@ -349,7 +349,7 @@ LJLIB_CF(debug_gethook)
if (hook != NULL && hook != hookf) { /* external hook? */
lua_pushliteral(L, "external hook");
} else {
lua_pushlightuserdata(L, KEY_HOOK);
(L->top++)->u64 = KEY_HOOK;
lua_rawget(L, LUA_REGISTRYINDEX); /* get hook */
}
lua_pushstring(L, unmakemask(mask, buff));

View File

@ -547,15 +547,15 @@ LJLIB_CF(jit_opt_start)
/* Not loaded by default, use: local profile = require("jit.profile") */
static const char KEY_PROFILE_THREAD = 't';
static const char KEY_PROFILE_FUNC = 'f';
#define KEY_PROFILE_THREAD (U64x(80000000,00000000)|'t')
#define KEY_PROFILE_FUNC (U64x(80000000,00000000)|'f')
static void jit_profile_callback(lua_State *L2, lua_State *L, int samples,
int vmstate)
{
TValue key;
cTValue *tv;
setlightudV(&key, (void *)&KEY_PROFILE_FUNC);
key.u64 = KEY_PROFILE_FUNC;
tv = lj_tab_get(L, tabV(registry(L)), &key);
if (tvisfunc(tv)) {
char vmst = (char)vmstate;
@ -582,9 +582,9 @@ LJLIB_CF(jit_profile_start)
lua_State *L2 = lua_newthread(L); /* Thread that runs profiler callback. */
TValue key;
/* Anchor thread and function in registry. */
setlightudV(&key, (void *)&KEY_PROFILE_THREAD);
key.u64 = KEY_PROFILE_THREAD;
setthreadV(L, lj_tab_set(L, registry, &key), L2);
setlightudV(&key, (void *)&KEY_PROFILE_FUNC);
key.u64 = KEY_PROFILE_FUNC;
setfuncV(L, lj_tab_set(L, registry, &key), func);
lj_gc_anybarriert(L, registry);
luaJIT_profile_start(L, mode ? strdata(mode) : "",
@ -599,9 +599,9 @@ LJLIB_CF(jit_profile_stop)
TValue key;
luaJIT_profile_stop(L);
registry = tabV(registry(L));
setlightudV(&key, (void *)&KEY_PROFILE_THREAD);
key.u64 = KEY_PROFILE_THREAD;
setnilV(lj_tab_set(L, registry, &key));
setlightudV(&key, (void *)&KEY_PROFILE_FUNC);
key.u64 = KEY_PROFILE_FUNC;
setnilV(lj_tab_set(L, registry, &key));
lj_gc_anybarriert(L, registry);
return 0;

View File

@ -425,7 +425,7 @@ static int lj_cf_package_loader_preload(lua_State *L)
/* ------------------------------------------------------------------------ */
#define sentinel ((void *)0x4004)
#define KEY_SENTINEL (U64x(80000000,00000000)|'s')
static int lj_cf_package_require(lua_State *L)
{
@ -435,7 +435,7 @@ static int lj_cf_package_require(lua_State *L)
lua_getfield(L, LUA_REGISTRYINDEX, "_LOADED");
lua_getfield(L, 2, name);
if (lua_toboolean(L, -1)) { /* is it there? */
if (lua_touserdata(L, -1) == sentinel) /* check loops */
if ((L->top-1)->u64 == KEY_SENTINEL) /* check loops */
luaL_error(L, "loop or previous error loading module " LUA_QS, name);
return 1; /* package is already loaded */
}
@ -458,14 +458,14 @@ static int lj_cf_package_require(lua_State *L)
else
lua_pop(L, 1);
}
lua_pushlightuserdata(L, sentinel);
(L->top++)->u64 = KEY_SENTINEL;
lua_setfield(L, 2, name); /* _LOADED[name] = sentinel */
lua_pushstring(L, name); /* pass name as argument to module */
lua_call(L, 1, 1); /* run loaded module */
if (!lua_isnil(L, -1)) /* non-nil return? */
lua_setfield(L, 2, name); /* _LOADED[name] = returned value */
lua_getfield(L, 2, name);
if (lua_touserdata(L, -1) == sentinel) { /* module did not set a value? */
if ((L->top-1)->u64 == KEY_SENTINEL) { /* module did not set a value? */
lua_pushboolean(L, 1); /* use true as result */
lua_pushvalue(L, -1); /* extra copy to be returned */
lua_setfield(L, 2, name); /* _LOADED[name] = true */

View File

@ -714,7 +714,7 @@ again:
lj_strfmt_putfchar(sb, sf, lj_lib_checkint(L, arg));
break;
case STRFMT_PTR: /* No formatting. */
lj_strfmt_putptr(sb, lj_obj_ptr(L->base+arg-1));
lj_strfmt_putptr(sb, lj_obj_ptr(G(L), L->base+arg-1));
break;
default:
lj_assertL(0, "bad string format type");

View File

@ -608,7 +608,7 @@ LUA_API void *lua_touserdata(lua_State *L, int idx)
if (tvisudata(o))
return uddata(udataV(o));
else if (tvislightud(o))
return lightudV(o);
return lightudV(G(L), o);
else
return NULL;
}
@ -621,7 +621,7 @@ LUA_API lua_State *lua_tothread(lua_State *L, int idx)
LUA_API const void *lua_topointer(lua_State *L, int idx)
{
return lj_obj_ptr(index2adr(L, idx));
return lj_obj_ptr(G(L), index2adr(L, idx));
}
/* -- Stack setters (object creation) ------------------------------------- */
@ -707,9 +707,38 @@ LUA_API void lua_pushboolean(lua_State *L, int b)
incr_top(L);
}
#if LJ_64
static void *lightud_intern(lua_State *L, void *p)
{
global_State *g = G(L);
uint64_t u = (uint64_t)p;
uint32_t up = lightudup(u);
uint32_t *segmap = mref(g->gc.lightudseg, uint32_t);
MSize segnum = g->gc.lightudnum;
if (segmap) {
MSize seg;
for (seg = 0; seg <= segnum; seg++)
if (segmap[seg] == up) /* Fast path. */
return (void *)(((uint64_t)seg << LJ_LIGHTUD_BITS_LO) | lightudlo(u));
segnum++;
}
if (!((segnum-1) & segnum) && segnum != 1) {
if (segnum >= (1 << LJ_LIGHTUD_BITS_SEG)) lj_err_msg(L, LJ_ERR_BADLU);
lj_mem_reallocvec(L, segmap, segnum, segnum ? 2*segnum : 2u, uint32_t);
setmref(g->gc.lightudseg, segmap);
}
g->gc.lightudnum = segnum;
segmap[segnum] = up;
return (void *)(((uint64_t)segnum << LJ_LIGHTUD_BITS_LO) | lightudlo(u));
}
#endif
LUA_API void lua_pushlightuserdata(lua_State *L, void *p)
{
setlightudV(L->top, checklightudptr(L, p));
#if LJ_64
p = lightud_intern(L, p);
#endif
setrawlightudV(L->top, p);
incr_top(L);
}
@ -1149,7 +1178,10 @@ static TValue *cpcall(lua_State *L, lua_CFunction func, void *ud)
fn->c.f = func;
setfuncV(L, top++, fn);
if (LJ_FR2) setnilV(top++);
setlightudV(top++, checklightudptr(L, ud));
#if LJ_64
ud = lightud_intern(L, ud);
#endif
setrawlightudV(top++, ud);
cframe_nres(L->cframe) = 1+0; /* Zero results. */
L->top = top;
return top-1; /* Now call the newly allocated C function. */

View File

@ -1167,7 +1167,7 @@ int lj_ccall_func(lua_State *L, GCcdata *cd)
lj_vm_ffi_call(&cc);
if (cts->cb.slot != ~0u) { /* Blacklist function that called a callback. */
TValue tv;
setlightudV(&tv, (void *)cc.func);
tv.u64 = ((uintptr_t)(void *)cc.func >> 2) | U64x(800000000, 00000000);
setboolV(lj_tab_set(L, cts->miscmap, &tv), 1);
}
ct = (CType *)((intptr_t)ct+(intptr_t)cts->tab); /* May be reallocated. */

View File

@ -620,7 +620,7 @@ void lj_cconv_ct_tv(CTState *cts, CType *d,
if (ud->udtype == UDTYPE_IO_FILE)
tmpptr = *(void **)tmpptr;
} else if (tvislightud(o)) {
tmpptr = lightudV(o);
tmpptr = lightudV(cts->g, o);
} else if (tvisfunc(o)) {
void *p = lj_ccallback_new(cts, d, funcV(o));
if (p) {

View File

@ -646,8 +646,7 @@ static TRef crec_ct_tv(jit_State *J, CType *d, TRef dp, TRef sp, cTValue *sval)
}
} else if (tref_islightud(sp)) {
#if LJ_64
sp = emitir(IRT(IR_BAND, IRT_P64), sp,
lj_ir_kint64(J, U64x(00007fff,ffffffff)));
lj_trace_err(J, LJ_TRERR_NYICONV);
#endif
} else { /* NYI: tref_istab(sp). */
IRType t;
@ -1212,8 +1211,7 @@ static int crec_call(jit_State *J, RecordFFData *rd, GCcdata *cd)
TRef tr;
TValue tv;
/* Check for blacklisted C functions that might call a callback. */
setlightudV(&tv,
cdata_getptr(cdataptr(cd), (LJ_64 && tp == IRT_P64) ? 8 : 4));
tv.u64 = ((uintptr_t)cdata_getptr(cdataptr(cd), (LJ_64 && tp == IRT_P64) ? 8 : 4) >> 2) | U64x(800000000, 00000000);
if (tvistrue(lj_tab_get(J->L, cts->miscmap, &tv)))
lj_trace_err(J, LJ_TRERR_BLACKL);
if (ctype_isvoid(ctr->info)) {

View File

@ -295,7 +295,7 @@ int luaJIT_setmode(lua_State *L, int idx, int mode)
if (idx != 0) {
cTValue *tv = idx > 0 ? L->base + (idx-1) : L->top + idx;
if (tvislightud(tv))
g->wrapf = (lua_CFunction)lightudV(tv);
g->wrapf = (lua_CFunction)lightudV(g, tv);
else
return 0; /* Failed. */
} else {

View File

@ -389,8 +389,10 @@ void lj_ir_kvalue(lua_State *L, TValue *tv, const IRIns *ir)
case IR_KPRI: setpriV(tv, irt_toitype(ir->t)); break;
case IR_KINT: setintV(tv, ir->i); break;
case IR_KGC: setgcV(L, tv, ir_kgc(ir), irt_toitype(ir->t)); break;
case IR_KPTR: case IR_KKPTR: setlightudV(tv, ir_kptr(ir)); break;
case IR_KNULL: setlightudV(tv, NULL); break;
case IR_KPTR: case IR_KKPTR:
setnumV(tv, (lua_Number)(uintptr_t)ir_kptr(ir));
break;
case IR_KNULL: setintV(tv, 0); break;
case IR_KNUM: setnumV(tv, ir_knum(ir)->n); break;
#if LJ_HASFFI
case IR_KINT64: {

View File

@ -34,12 +34,13 @@ int LJ_FASTCALL lj_obj_equal(cTValue *o1, cTValue *o2)
}
/* Return pointer to object or its object data. */
const void * LJ_FASTCALL lj_obj_ptr(cTValue *o)
const void * LJ_FASTCALL lj_obj_ptr(global_State *g, cTValue *o)
{
UNUSED(g);
if (tvisudata(o))
return uddata(udataV(o));
else if (tvislightud(o))
return lightudV(o);
return lightudV(g, o);
else if (LJ_HASFFI && tviscdata(o))
return cdataptr(cdataV(o));
else if (tvisgcv(o))

View File

@ -232,7 +232,7 @@ typedef const TValue cTValue;
** ---MSW---.---LSW---
** primitive types | itype | |
** lightuserdata | itype | void * | (32 bit platforms)
** lightuserdata |ffff| void * | (64 bit platforms, 47 bit pointers)
** lightuserdata |ffff|seg| ofs | (64 bit platforms)
** GC objects | itype | GCRef |
** int (LJ_DUALNUM)| itype | int |
** number -------double------
@ -245,7 +245,8 @@ typedef const TValue cTValue;
**
** ------MSW------.------LSW------
** primitive types |1..1|itype|1..................1|
** GC objects/lightud |1..1|itype|-------GCRef--------|
** GC objects |1..1|itype|-------GCRef--------|
** lightuserdata |1..1|itype|seg|------ofs-------|
** int (LJ_DUALNUM) |1..1|itype|0..0|-----int-------|
** number ------------double-------------
**
@ -285,6 +286,12 @@ typedef const TValue cTValue;
#define LJ_GCVMASK (((uint64_t)1 << 47) - 1)
#endif
#if LJ_64
/* To stay within 47 bits, lightuserdata is segmented. */
#define LJ_LIGHTUD_BITS_SEG 8
#define LJ_LIGHTUD_BITS_LO (47 - LJ_LIGHTUD_BITS_SEG)
#endif
/* -- String object ------------------------------------------------------- */
typedef uint32_t StrHash; /* String hash value. */
@ -580,7 +587,11 @@ typedef struct GCState {
uint8_t currentwhite; /* Current white color. */
uint8_t state; /* GC state. */
uint8_t nocdatafin; /* No cdata finalizer called. */
uint8_t unused2;
#if LJ_64
uint8_t lightudnum; /* Number of lightuserdata segments - 1. */
#else
uint8_t unused1;
#endif
MSize sweepstr; /* Sweep position in string table. */
GCRef root; /* List of all collectable objects. */
MRef sweep; /* Sweep position in root list. */
@ -592,6 +603,9 @@ typedef struct GCState {
GCSize estimate; /* Estimate of memory actually in use. */
MSize stepmul; /* Incremental GC step granularity. */
MSize pause; /* Pause between successive GC cycles. */
#if LJ_64
MRef lightudseg; /* Upper bits of lightuserdata segments. */
#endif
} GCState;
/* String interning state. */
@ -813,10 +827,23 @@ typedef union GCobj {
#endif
#define boolV(o) check_exp(tvisbool(o), (LJ_TFALSE - itype(o)))
#if LJ_64
#define lightudV(o) \
check_exp(tvislightud(o), (void *)((o)->u64 & U64x(00007fff,ffffffff)))
#define lightudseg(u) \
(((u) >> LJ_LIGHTUD_BITS_LO) & ((1 << LJ_LIGHTUD_BITS_SEG)-1))
#define lightudlo(u) \
((u) & (((uint64_t)1 << LJ_LIGHTUD_BITS_LO) - 1))
#define lightudup(p) \
((uint32_t)(((p) >> LJ_LIGHTUD_BITS_LO) << (LJ_LIGHTUD_BITS_LO-32)))
static LJ_AINLINE void *lightudV(global_State *g, cTValue *o)
{
uint64_t u = o->u64;
uint64_t seg = lightudseg(u);
uint32_t *segmap = mref(g->gc.lightudseg, uint32_t);
lj_assertG(tvislightud(o), "lightuserdata expected");
lj_assertG(seg <= g->gc.lightudnum, "bad lightuserdata segment %d", seg);
return (void *)(((uint64_t)segmap[seg] << 32) | lightudlo(u));
}
#else
#define lightudV(o) check_exp(tvislightud(o), gcrefp((o)->gcr, void))
#define lightudV(g, o) check_exp(tvislightud(o), gcrefp((o)->gcr, void))
#endif
#define gcV(o) check_exp(tvisgcv(o), gcval(o))
#define strV(o) check_exp(tvisstr(o), &gcval(o)->str)
@ -842,7 +869,7 @@ typedef union GCobj {
#define setpriV(o, i) (setitype((o), (i)))
#endif
static LJ_AINLINE void setlightudV(TValue *o, void *p)
static LJ_AINLINE void setrawlightudV(TValue *o, void *p)
{
#if LJ_GC64
o->u64 = (uint64_t)p | (((uint64_t)LJ_TLIGHTUD) << 47);
@ -853,24 +880,14 @@ static LJ_AINLINE void setlightudV(TValue *o, void *p)
#endif
}
#if LJ_64
#define checklightudptr(L, p) \
(((uint64_t)(p) >> 47) ? (lj_err_msg(L, LJ_ERR_BADLU), NULL) : (p))
#else
#define checklightudptr(L, p) (p)
#endif
#if LJ_FR2
#if LJ_FR2 || LJ_32
#define contptr(f) ((void *)(f))
#define setcont(o, f) ((o)->u64 = (uint64_t)(uintptr_t)contptr(f))
#elif LJ_64
#else
#define contptr(f) \
((void *)(uintptr_t)(uint32_t)((intptr_t)(f) - (intptr_t)lj_vm_asm_begin))
#define setcont(o, f) \
((o)->u64 = (uint64_t)(void *)(f) - (uint64_t)lj_vm_asm_begin)
#else
#define contptr(f) ((void *)(f))
#define setcont(o, f) setlightudV((o), contptr(f))
#endif
static LJ_AINLINE void checklivetv(lua_State *L, TValue *o, const char *msg)
@ -1016,6 +1033,6 @@ LJ_DATA const char *const lj_obj_itypename[~LJ_TNUMX+1];
/* Compare two objects without calling metamethods. */
LJ_FUNC int LJ_FASTCALL lj_obj_equal(cTValue *o1, cTValue *o2);
LJ_FUNC const void * LJ_FASTCALL lj_obj_ptr(cTValue *o);
LJ_FUNC const void * LJ_FASTCALL lj_obj_ptr(global_State *g, cTValue *o);
#endif

View File

@ -638,7 +638,14 @@ static void snap_restoreval(jit_State *J, GCtrace *T, ExitState *ex,
IRType1 t = ir->t;
RegSP rs = ir->prev;
if (irref_isk(ref)) { /* Restore constant slot. */
lj_ir_kvalue(J->L, o, ir);
if (ir->o == IR_KPTR) {
o->u64 = (uint64_t)(uintptr_t)ir_kptr(ir);
} else {
lj_assertJ(!(ir->o == IR_KKPTR || ir->o == IR_KNULL),
"restore of const from IR %04d with bad op %d",
ref - REF_BIAS, ir->o);
lj_ir_kvalue(J->L, o, ir);
}
return;
}
if (LJ_UNLIKELY(bloomtest(rfilt, ref)))

View File

@ -174,6 +174,12 @@ static void close_state(lua_State *L)
lj_str_freetab(g);
lj_buf_free(g, &g->tmpbuf);
lj_mem_freevec(g, tvref(L->stack), L->stacksize, TValue);
#if LJ_64
if (mref(g->gc.lightudseg, uint32_t)) {
MSize segnum = g->gc.lightudnum ? (2 << lj_fls(g->gc.lightudnum)) : 2;
lj_mem_freevec(g, mref(g->gc.lightudseg, uint32_t), segnum, uint32_t);
}
#endif
lj_assertG(g->gc.total == sizeof(GG_State),
"memory leak of %lld bytes",
(long long)(g->gc.total - sizeof(GG_State)));

View File

@ -393,7 +393,7 @@ GCstr * LJ_FASTCALL lj_strfmt_obj(lua_State *L, cTValue *o)
p = lj_buf_wmem(p, "builtin#", 8);
p = lj_strfmt_wint(p, funcV(o)->c.ffid);
} else {
p = lj_strfmt_wptr(p, lj_obj_ptr(o));
p = lj_strfmt_wptr(p, lj_obj_ptr(G(L), o));
}
return lj_str_new(L, buf, (size_t)(p - buf));
}