From 4b90f6c4d7420139c135435e1580acb52ea18436 Mon Sep 17 00:00:00 2001 From: Mike Pall Date: Mon, 22 Jan 2024 19:06:36 +0100 Subject: [PATCH] Add cross-32/64 bit and deterministic bytecode generation. Contributed by Peter Cawley. #993 #1008 --- doc/extensions.html | 51 +++++++++++++++----- doc/running.html | 3 ++ src/host/genlibbc.lua | 99 +++++++++++++++++++++----------------- src/jit/bcsave.lua | 31 ++++++++---- src/lib_base.c | 6 ++- src/lib_jit.c | 26 ++-------- src/lib_string.c | 22 +++++++-- src/lj_bcdump.h | 4 +- src/lj_bcread.c | 9 ++-- src/lj_bcwrite.c | 109 ++++++++++++++++++++++++++++++++++++------ src/lj_lex.c | 1 + src/lj_lex.h | 1 + src/lj_lib.c | 18 +++++++ src/lj_lib.h | 1 + src/lj_load.c | 29 ++++++++--- src/lj_parse.c | 28 ++++++----- 16 files changed, 306 insertions(+), 132 deletions(-) diff --git a/doc/extensions.html b/doc/extensions.html index a4f20841..1d28475c 100644 --- a/doc/extensions.html +++ b/doc/extensions.html @@ -160,13 +160,33 @@ passes any arguments after the error function to the function which is called in a protected context.

-

loadfile() etc. handle UTF-8 source code

+

load*() handle UTF-8 source code

Non-ASCII characters are handled transparently by the Lua source code parser. This allows the use of UTF-8 characters in identifiers and strings. A UTF-8 BOM is skipped at the start of the source code.

+

load*() add a mode parameter

+

+As an extension from Lua 5.2, the functions loadstring(), +loadfile() and (new) load() add an optional +mode parameter. +

+

+The default mode string is "bt", which allows loading of both +source code and bytecode. Use "t" to allow only source code +or "b" to allow only bytecode to be loaded. +

+

+By default, the load* functions generate the native bytecode format. +For cross-compilation purposes, add W to the mode string to +force the 32 bit format and X to force the 64 bit format. +Add both to force the opposite format. Note that non-native bytecode +generated by load* cannot be run, but can still be passed +to string.dump. +

+

tostring() etc. canonicalize NaN and ±Inf

All number-to-string conversions consistently convert non-finite numbers @@ -186,26 +206,33 @@ works independently of the current locale and it supports hex floating-point numbers (e.g. 0x1.5p-3).

-

string.dump(f [,strip]) generates portable bytecode

+

string.dump(f [,mode]) generates portable bytecode

An extra argument has been added to string.dump(). If set to -true, 'stripped' bytecode without debug information is -generated. This speeds up later bytecode loading and reduces memory -usage. See also the +true or to a string which contains the character s, +'stripped' bytecode without debug information is generated. This speeds +up later bytecode loading and reduces memory usage. See also the -b command line option.

The generated bytecode is portable and can be loaded on any architecture -that LuaJIT supports, independent of word size or endianess. However, the -bytecode compatibility versions must match. Bytecode stays compatible -for dot releases (x.y.0 → x.y.1), but may change with major or -minor releases (2.0 → 2.1) or between any beta release. Foreign -bytecode (e.g. from Lua 5.1) is incompatible and cannot be loaded. +that LuaJIT supports. However, the bytecode compatibility versions must +match. Bytecode only stays compatible within a major+minor version +(x.y.aaa → x.y.bbb), except for development branches. Foreign bytecode +(e.g. from Lua 5.1) is incompatible and cannot be loaded.

Note: LJ_GC64 mode requires a different frame layout, which implies -a different, incompatible bytecode format for all 64 bit ports. This may be -rectified in the future. +a different, incompatible bytecode format between 32 bit and 64 bit ports. +This may be rectified in the future. In the meantime, use the W +and X modes of the load* functions +for cross-compilation purposes. +

+

+Due to VM hardening, bytecode is not deterministic. Add d to the +mode string to dump it in a deterministic manner: identical source code +always gives a byte-for-byte identical bytecode dump. This feature is +mainly useful for reproducible builds.

table.new(narray, nhash) allocates a pre-sized table

diff --git a/doc/running.html b/doc/running.html index 9dd2b411..142b810f 100644 --- a/doc/running.html +++ b/doc/running.html @@ -106,6 +106,9 @@ are accepted:
  • -l — Only list bytecode.
  • -s — Strip debug info (this is the default).
  • -g — Keep debug info.
    • +
  • -W — Generate 32 bit (non-GC64) bytecode.
    • +
  • -X — Generate 64 bit (GC64) bytecode.
    • +
  • -d — Generate bytecode in deterministic manner.
  • -n name — Set module name (default: auto-detect from input name)
  • -t type — Set output file type (default: auto-detect from output name).
  • -a arch — Override architecture for object files (default: native).
    • diff --git a/src/host/genlibbc.lua b/src/host/genlibbc.lua index 3621c3f5..e697fceb 100644 --- a/src/host/genlibbc.lua +++ b/src/host/genlibbc.lua @@ -138,65 +138,73 @@ local function fixup_dump(dump, fixup) return { dump = ndump, startbc = startbc, sizebc = sizebc } end -local function find_defs(src) +local function find_defs(src, mode) local defs = {} for name, code in string.gmatch(src, "LJLIB_LUA%(([^)]*)%)%s*/%*(.-)%*/") do - local env = {} local tcode, fixup = transform_lua(code) - local func = assert(load(tcode, "", nil, env))() - defs[name] = fixup_dump(string.dump(func, true), fixup) + local func = assert(load(tcode, "", mode)) + defs[name] = fixup_dump(string.dump(func, mode), fixup) defs[#defs+1] = name end return defs end -local function gen_header(defs) +local function gen_header(defs32, defs64) local t = {} local function w(x) t[#t+1] = x end w("/* This is a generated file. DO NOT EDIT! */\n\n") w("static const int libbc_endian = ") w(isbe and 1 or 0) w(";\n\n") - local s, sb = "", "" - for i,name in ipairs(defs) do - local d = defs[name] - s = s .. d.dump - sb = sb .. string.char(i) .. ("\0"):rep(d.startbc - 1) - .. (isbe and "\0\0\0\255" or "\255\0\0\0"):rep(d.sizebc) - .. ("\0"):rep(#d.dump - d.startbc - d.sizebc*4) - end - w("static const uint8_t libbc_code[] = {\n") - local n = 0 - for i=1,#s do - local x = string.byte(s, i) - local xb = string.byte(sb, i) - if xb == 255 then - local name = BCN[x] - local m = #name + 4 - if n + m > 78 then n = 0; w("\n") end - n = n + m - w("BC_"); w(name) - else - local m = x < 10 and 2 or (x < 100 and 3 or 4) - if xb == 0 then - if n + m > 78 then n = 0; w("\n") end - else - local name = defs[xb]:gsub("_", ".") - if n ~= 0 then w("\n") end - w("/* "); w(name); w(" */ ") - n = #name + 7 - end - n = n + m - w(x) + for j,defs in ipairs{defs64, defs32} do + local s, sb = "", "" + for i,name in ipairs(defs) do + local d = defs[name] + s = s .. d.dump + sb = sb .. string.char(i) .. ("\0"):rep(d.startbc - 1) + .. (isbe and "\0\0\0\255" or "\255\0\0\0"):rep(d.sizebc) + .. ("\0"):rep(#d.dump - d.startbc - d.sizebc*4) + end + if j == 1 then + w("static const uint8_t libbc_code[] = {\n#if LJ_FR2\n") + else + w("\n#else\n") + end + local n = 0 + for i=1,#s do + local x = string.byte(s, i) + local xb = string.byte(sb, i) + if xb == 255 then + local name = BCN[x] + local m = #name + 4 + if n + m > 78 then n = 0; w("\n") end + n = n + m + w("BC_"); w(name) + else + local m = x < 10 and 2 or (x < 100 and 3 or 4) + if xb == 0 then + if n + m > 78 then n = 0; w("\n") end + else + local name = defs[xb]:gsub("_", ".") + if n ~= 0 then w("\n") end + w("/* "); w(name); w(" */ ") + n = #name + 7 + end + n = n + m + w(x) + end + w(",") end - w(",") end - w("\n0\n};\n\n") + w("\n#endif\n0\n};\n\n") w("static const struct { const char *name; int ofs; } libbc_map[] = {\n") - local m = 0 - for _,name in ipairs(defs) do - w('{"'); w(name); w('",'); w(m) w('},\n') - m = m + #defs[name].dump + local m32, m64 = 0, 0 + for i,name in ipairs(defs32) do + assert(name == defs64[i]) + w('{"'); w(name); w('",'); w(m32) w('},\n') + m32 = m32 + #defs32[name].dump + m64 = m64 + #defs64[name].dump + assert(m32 == m64) end - w("{NULL,"); w(m); w("}\n};\n\n") + w("{NULL,"); w(m32); w("}\n};\n\n") return table.concat(t) end @@ -219,7 +227,8 @@ end local outfile = parse_arg(arg) local src = read_files(arg) -local defs = find_defs(src) -local hdr = gen_header(defs) +local defs32 = find_defs(src, "Wdts") +local defs64 = find_defs(src, "Xdts") +local hdr = gen_header(defs32, defs64) write_file(outfile, hdr) diff --git a/src/jit/bcsave.lua b/src/jit/bcsave.lua index 390d297c..131bf39b 100644 --- a/src/jit/bcsave.lua +++ b/src/jit/bcsave.lua @@ -29,6 +29,9 @@ Save LuaJIT bytecode: luajit -b[options] input output -l Only list bytecode. -s Strip debug info (default). -g Keep debug info. + -W Generate 32 bit (non-GC64) bytecode. + -X Generate 64 bit (GC64) bytecode. + -d Generate bytecode in deterministic manner. -n name Set module name (default: auto-detect from input name). -t type Set output file type (default: auto-detect from output name). -a arch Override architecture for object files (default: native). @@ -51,8 +54,9 @@ local function check(ok, ...) end local function readfile(ctx, input) - if type(input) == "function" then return input end - if ctx.filename then + if ctx.string then + return check(loadstring(input, nil, ctx.mode)) + elseif ctx.filename then local data if input == "-" then data = io.stdin:read("*a") @@ -61,10 +65,10 @@ local function readfile(ctx, input) data = assert(fp:read("*a")) assert(fp:close()) end - return check(load(data, ctx.filename)) + return check(load(data, ctx.filename, ctx.mode)) else if input == "-" then input = nil end - return check(loadfile(input)) + return check(loadfile(input, ctx.mode)) end end @@ -624,7 +628,7 @@ end local function bcsave(ctx, input, output) local f = readfile(ctx, input) - local s = string.dump(f, ctx.strip) + local s = string.dump(f, ctx.mode) local t = ctx.type if not t then t = detecttype(output) @@ -647,9 +651,11 @@ local function docmd(...) local n = 1 local list = false local ctx = { - strip = true, arch = jit.arch, os = jit.os:lower(), - type = false, modname = false, + mode = "bt", arch = jit.arch, os = jit.os:lower(), + type = false, modname = false, string = false, } + local strip = "s" + local gc64 = "" while n <= #arg do local a = arg[n] if type(a) == "string" and a:sub(1, 1) == "-" and a ~= "-" then @@ -660,14 +666,18 @@ local function docmd(...) if opt == "l" then list = true elseif opt == "s" then - ctx.strip = true + strip = "s" elseif opt == "g" then - ctx.strip = false + strip = "" + elseif opt == "W" or opt == "X" then + gc64 = opt + elseif opt == "d" then + ctx.mode = ctx.mode .. opt else if arg[n] == nil or m ~= #a then usage() end if opt == "e" then if n ~= 1 then usage() end - arg[1] = check(loadstring(arg[1])) + ctx.string = true elseif opt == "n" then ctx.modname = checkmodname(tremove(arg, n)) elseif opt == "t" then @@ -687,6 +697,7 @@ local function docmd(...) n = n + 1 end end + ctx.mode = ctx.mode .. strip .. gc64 if list then if #arg == 0 or #arg > 2 then usage() end bclist(ctx, arg[1], arg[2] or "-") diff --git a/src/lib_base.c b/src/lib_base.c index 4e6f8a30..d644b4f2 100644 --- a/src/lib_base.c +++ b/src/lib_base.c @@ -360,7 +360,11 @@ LJLIB_ASM_(xpcall) LJLIB_REC(.) static int load_aux(lua_State *L, int status, int envarg) { if (status == LUA_OK) { - if (tvistab(L->base+envarg-1)) { + /* + ** Set environment table for top-level function. + ** Don't do this for non-native bytecode, which returns a prototype. + */ + if (tvistab(L->base+envarg-1) && tvisfunc(L->top-1)) { GCfunc *fn = funcV(L->top-1); GCtab *t = tabV(L->base+envarg-1); setgcref(fn->c.env, obj2gco(t)); diff --git a/src/lib_jit.c b/src/lib_jit.c index c0294927..b83c865a 100644 --- a/src/lib_jit.c +++ b/src/lib_jit.c @@ -161,24 +161,6 @@ LJLIB_PUSH(top-2) LJLIB_SET(version) /* -- Reflection API for Lua functions ------------------------------------ */ -/* Return prototype of first argument (Lua function or prototype object) */ -static GCproto *check_Lproto(lua_State *L, int nolua) -{ - TValue *o = L->base; - if (L->top > o) { - if (tvisproto(o)) { - return protoV(o); - } else if (tvisfunc(o)) { - if (isluafunc(funcV(o))) - return funcproto(funcV(o)); - else if (nolua) - return NULL; - } - } - lj_err_argt(L, 1, LUA_TFUNCTION); - return NULL; /* unreachable */ -} - static void setintfield(lua_State *L, GCtab *t, const char *name, int32_t val) { setintV(lj_tab_setstr(L, t, lj_str_newz(L, name)), val); @@ -187,7 +169,7 @@ static void setintfield(lua_State *L, GCtab *t, const char *name, int32_t val) /* local info = jit.util.funcinfo(func [,pc]) */ LJLIB_CF(jit_util_funcinfo) { - GCproto *pt = check_Lproto(L, 1); + GCproto *pt = lj_lib_checkLproto(L, 1, 1); if (pt) { BCPos pc = (BCPos)lj_lib_optint(L, 2, 0); GCtab *t; @@ -229,7 +211,7 @@ LJLIB_CF(jit_util_funcinfo) /* local ins, m = jit.util.funcbc(func, pc) */ LJLIB_CF(jit_util_funcbc) { - GCproto *pt = check_Lproto(L, 0); + GCproto *pt = lj_lib_checkLproto(L, 1, 0); BCPos pc = (BCPos)lj_lib_checkint(L, 2); if (pc < pt->sizebc) { BCIns ins = proto_bc(pt)[pc]; @@ -246,7 +228,7 @@ LJLIB_CF(jit_util_funcbc) /* local k = jit.util.funck(func, idx) */ LJLIB_CF(jit_util_funck) { - GCproto *pt = check_Lproto(L, 0); + GCproto *pt = lj_lib_checkLproto(L, 1, 0); ptrdiff_t idx = (ptrdiff_t)lj_lib_checkint(L, 2); if (idx >= 0) { if (idx < (ptrdiff_t)pt->sizekn) { @@ -266,7 +248,7 @@ LJLIB_CF(jit_util_funck) /* local name = jit.util.funcuvname(func, idx) */ LJLIB_CF(jit_util_funcuvname) { - GCproto *pt = check_Lproto(L, 0); + GCproto *pt = lj_lib_checkLproto(L, 1, 0); uint32_t idx = (uint32_t)lj_lib_checkint(L, 2); if (idx < pt->sizeuv) { setstrV(L, L->top-1, lj_str_newz(L, lj_debug_uvname(pt, idx))); diff --git a/src/lib_string.c b/src/lib_string.c index 29bcb8fe..255689ce 100644 --- a/src/lib_string.c +++ b/src/lib_string.c @@ -122,11 +122,25 @@ static int writer_buf(lua_State *L, const void *p, size_t size, void *sb) LJLIB_CF(string_dump) { - GCfunc *fn = lj_lib_checkfunc(L, 1); - int strip = L->base+1 < L->top && tvistruecond(L->base+1); - SBuf *sb = lj_buf_tmp_(L); /* Assumes lj_bcwrite() doesn't use tmpbuf. */ + GCproto *pt = lj_lib_checkLproto(L, 1, 1); + uint32_t flags = 0; + SBuf *sb; + TValue *o = L->base+1; + if (o < L->top) { + if (tvisstr(o)) { + const char *mode = strVdata(o); + char c; + while ((c = *mode++)) { + if (c == 's') flags |= BCDUMP_F_STRIP; + if (c == 'd') flags |= BCDUMP_F_DETERMINISTIC; + } + } else if (tvistruecond(o)) { + flags |= BCDUMP_F_STRIP; + } + } + sb = lj_buf_tmp_(L); /* Assumes lj_bcwrite() doesn't use tmpbuf. */ L->top = L->base+1; - if (!isluafunc(fn) || lj_bcwrite(L, funcproto(fn), writer_buf, sb, strip)) + if (!pt || lj_bcwrite(L, pt, writer_buf, sb, flags)) lj_err_caller(L, LJ_ERR_STRDUMP); setstrV(L, L->top-1, lj_buf_str(L, sb)); lj_gc_check(L); diff --git a/src/lj_bcdump.h b/src/lj_bcdump.h index 6ba71e25..3e56e39c 100644 --- a/src/lj_bcdump.h +++ b/src/lj_bcdump.h @@ -46,6 +46,8 @@ #define BCDUMP_F_KNOWN (BCDUMP_F_FR2*2-1) +#define BCDUMP_F_DETERMINISTIC 0x80000000 + /* Type codes for the GC constants of a prototype. Plus length for strings. */ enum { BCDUMP_KGC_CHILD, BCDUMP_KGC_TAB, BCDUMP_KGC_I64, BCDUMP_KGC_U64, @@ -61,7 +63,7 @@ enum { /* -- Bytecode reader/writer ---------------------------------------------- */ LJ_FUNC int lj_bcwrite(lua_State *L, GCproto *pt, lua_Writer writer, - void *data, int strip); + void *data, uint32_t flags); LJ_FUNC GCproto *lj_bcread_proto(LexState *ls); LJ_FUNC GCproto *lj_bcread(LexState *ls); diff --git a/src/lj_bcread.c b/src/lj_bcread.c index c98c0d42..637ef067 100644 --- a/src/lj_bcread.c +++ b/src/lj_bcread.c @@ -281,8 +281,11 @@ static void bcread_knum(LexState *ls, GCproto *pt, MSize sizekn) static void bcread_bytecode(LexState *ls, GCproto *pt, MSize sizebc) { BCIns *bc = proto_bc(pt); - bc[0] = BCINS_AD((pt->flags & PROTO_VARARG) ? BC_FUNCV : BC_FUNCF, - pt->framesize, 0); + BCIns op; + if (ls->fr2 != LJ_FR2) op = BC_NOT; /* Mark non-native prototype. */ + else if ((pt->flags & PROTO_VARARG)) op = BC_FUNCV; + else op = BC_FUNCF; + bc[0] = BCINS_AD(op, pt->framesize, 0); bcread_block(ls, bc+1, (sizebc-1)*(MSize)sizeof(BCIns)); /* Swap bytecode instructions if the endianess differs. */ if (bcread_swap(ls)) { @@ -395,7 +398,7 @@ static int bcread_header(LexState *ls) bcread_byte(ls) != BCDUMP_VERSION) return 0; bcread_flags(ls) = flags = bcread_uleb128(ls); if ((flags & ~(BCDUMP_F_KNOWN)) != 0) return 0; - if ((flags & BCDUMP_F_FR2) != LJ_FR2*BCDUMP_F_FR2) return 0; + if ((flags & BCDUMP_F_FR2) != (uint32_t)ls->fr2*BCDUMP_F_FR2) return 0; if ((flags & BCDUMP_F_FFI)) { #if LJ_HASFFI lua_State *L = ls->L; diff --git a/src/lj_bcwrite.c b/src/lj_bcwrite.c index dd969413..c062dc49 100644 --- a/src/lj_bcwrite.c +++ b/src/lj_bcwrite.c @@ -27,7 +27,9 @@ typedef struct BCWriteCtx { GCproto *pt; /* Root prototype. */ lua_Writer wfunc; /* Writer callback. */ void *wdata; /* Writer callback data. */ - int strip; /* Strip debug info. */ + TValue **heap; /* Heap used for deterministic sorting. */ + uint32_t heapsz; /* Size of heap. */ + uint32_t flags; /* BCDUMP_F_* flags. */ int status; /* Status from writer callback. */ #ifdef LUA_USE_ASSERT global_State *g; @@ -76,6 +78,75 @@ static void bcwrite_ktabk(BCWriteCtx *ctx, cTValue *o, int narrow) ctx->sb.w = p; } +/* Compare two template table keys. */ +static LJ_AINLINE int bcwrite_ktabk_lt(TValue *a, TValue *b) +{ + uint32_t at = itype(a), bt = itype(b); + if (at != bt) { /* This also handles false and true keys. */ + return at < bt; + } else if (at == LJ_TSTR) { + return lj_str_cmp(strV(a), strV(b)) < 0; + } else { + return a->u64 < b->u64; /* This works for numbers and integers. */ + } +} + +/* Insert key into a sorted heap. */ +static void bcwrite_ktabk_heap_insert(TValue **heap, MSize idx, MSize end, + TValue *key) +{ + MSize child; + while ((child = idx * 2 + 1) < end) { + /* Find lower of the two children. */ + TValue *c0 = heap[child]; + if (child + 1 < end) { + TValue *c1 = heap[child + 1]; + if (bcwrite_ktabk_lt(c1, c0)) { + c0 = c1; + child++; + } + } + if (bcwrite_ktabk_lt(key, c0)) break; /* Key lower? Found our position. */ + heap[idx] = c0; /* Move lower child up. */ + idx = child; /* Descend. */ + } + heap[idx] = key; /* Insert key here. */ +} + +/* Resize heap, dropping content. */ +static void bcwrite_heap_resize(BCWriteCtx *ctx, uint32_t nsz) +{ + lua_State *L = sbufL(&ctx->sb); + if (ctx->heapsz) { + lj_mem_freevec(G(L), ctx->heap, ctx->heapsz, TValue *); + ctx->heapsz = 0; + } + if (nsz) { + ctx->heap = lj_mem_newvec(L, nsz, TValue *); + ctx->heapsz = nsz; + } +} + +/* Write hash part of template table in sorted order. */ +static void bcwrite_ktab_sorted_hash(BCWriteCtx *ctx, Node *node, MSize nhash) +{ + TValue **heap = ctx->heap; + MSize i = nhash; + for (;; node--) { /* Build heap. */ + if (!tvisnil(&node->val)) { + bcwrite_ktabk_heap_insert(heap, --i, nhash, &node->key); + if (i == 0) break; + } + } + do { /* Drain heap. */ + TValue *key = heap[0]; /* Output lowest key from top. */ + bcwrite_ktabk(ctx, key, 0); + bcwrite_ktabk(ctx, (TValue *)((char *)key - offsetof(Node, key)), 1); + key = heap[--nhash]; /* Remove last key. */ + bcwrite_ktabk_heap_insert(heap, 0, nhash, key); /* Re-insert. */ + } while (nhash); +} + /* Write a template table. */ static void bcwrite_ktab(BCWriteCtx *ctx, char *p, const GCtab *t) { @@ -105,14 +176,20 @@ static void bcwrite_ktab(BCWriteCtx *ctx, char *p, const GCtab *t) bcwrite_ktabk(ctx, o, 1); } if (nhash) { /* Write hash entries. */ - MSize i = nhash; Node *node = noderef(t->node) + t->hmask; - for (;; node--) - if (!tvisnil(&node->val)) { - bcwrite_ktabk(ctx, &node->key, 0); - bcwrite_ktabk(ctx, &node->val, 1); - if (--i == 0) break; - } + if ((ctx->flags & BCDUMP_F_DETERMINISTIC) && nhash > 1) { + if (ctx->heapsz < nhash) + bcwrite_heap_resize(ctx, t->hmask + 1); + bcwrite_ktab_sorted_hash(ctx, node, nhash); + } else { + MSize i = nhash; + for (;; node--) + if (!tvisnil(&node->val)) { + bcwrite_ktabk(ctx, &node->key, 0); + bcwrite_ktabk(ctx, &node->val, 1); + if (--i == 0) break; + } + } } } @@ -269,7 +346,7 @@ static void bcwrite_proto(BCWriteCtx *ctx, GCproto *pt) p = lj_strfmt_wuleb128(p, pt->sizekgc); p = lj_strfmt_wuleb128(p, pt->sizekn); p = lj_strfmt_wuleb128(p, pt->sizebc-1); - if (!ctx->strip) { + if (!(ctx->flags & BCDUMP_F_STRIP)) { if (proto_lineinfo(pt)) sizedbg = pt->sizept - (MSize)((char *)proto_lineinfo(pt) - (char *)pt); p = lj_strfmt_wuleb128(p, sizedbg); @@ -317,11 +394,10 @@ static void bcwrite_header(BCWriteCtx *ctx) *p++ = BCDUMP_HEAD2; *p++ = BCDUMP_HEAD3; *p++ = BCDUMP_VERSION; - *p++ = (ctx->strip ? BCDUMP_F_STRIP : 0) + + *p++ = (ctx->flags & (BCDUMP_F_STRIP | BCDUMP_F_FR2)) + LJ_BE*BCDUMP_F_BE + - ((ctx->pt->flags & PROTO_FFI) ? BCDUMP_F_FFI : 0) + - LJ_FR2*BCDUMP_F_FR2; - if (!ctx->strip) { + ((ctx->pt->flags & PROTO_FFI) ? BCDUMP_F_FFI : 0); + if (!(ctx->flags & BCDUMP_F_STRIP)) { p = lj_strfmt_wuleb128(p, len); p = lj_buf_wmem(p, name, len); } @@ -352,14 +428,16 @@ static TValue *cpwriter(lua_State *L, lua_CFunction dummy, void *ud) /* Write bytecode for a prototype. */ int lj_bcwrite(lua_State *L, GCproto *pt, lua_Writer writer, void *data, - int strip) + uint32_t flags) { BCWriteCtx ctx; int status; ctx.pt = pt; ctx.wfunc = writer; ctx.wdata = data; - ctx.strip = strip; + ctx.heapsz = 0; + if ((bc_op(proto_bc(pt)[0]) != BC_NOT) == LJ_FR2) flags |= BCDUMP_F_FR2; + ctx.flags = flags; ctx.status = 0; #ifdef LUA_USE_ASSERT ctx.g = G(L); @@ -368,6 +446,7 @@ int lj_bcwrite(lua_State *L, GCproto *pt, lua_Writer writer, void *data, status = lj_vm_cpcall(L, NULL, &ctx, cpwriter); if (status == 0) status = ctx.status; lj_buf_free(G(sbufL(&ctx.sb)), &ctx.sb); + bcwrite_heap_resize(&ctx, 0); return status; } diff --git a/src/lj_lex.c b/src/lj_lex.c index 61b04c4b..bd81dc40 100644 --- a/src/lj_lex.c +++ b/src/lj_lex.c @@ -411,6 +411,7 @@ int lj_lex_setup(lua_State *L, LexState *ls) ls->linenumber = 1; ls->lastline = 1; ls->endmark = 0; + ls->fr2 = LJ_FR2; /* Generate native bytecode by default. */ lex_next(ls); /* Read-ahead first char. */ if (ls->c == 0xef && ls->p + 2 <= ls->pe && (uint8_t)ls->p[0] == 0xbb && (uint8_t)ls->p[1] == 0xbf) { /* Skip UTF-8 BOM (if buffered). */ diff --git a/src/lj_lex.h b/src/lj_lex.h index e46fbd89..2ef7fc77 100644 --- a/src/lj_lex.h +++ b/src/lj_lex.h @@ -74,6 +74,7 @@ typedef struct LexState { MSize sizebcstack; /* Size of bytecode stack. */ uint32_t level; /* Syntactical nesting level. */ int endmark; /* Trust bytecode end marker, even if not at EOF. */ + int fr2; /* Generate bytecode for LJ_FR2 mode. */ } LexState; LJ_FUNC int lj_lex_setup(lua_State *L, LexState *ls); diff --git a/src/lj_lib.c b/src/lj_lib.c index ebe0dc78..06ae4fcf 100644 --- a/src/lj_lib.c +++ b/src/lj_lib.c @@ -62,6 +62,7 @@ static const uint8_t *lib_read_lfunc(lua_State *L, const uint8_t *p, GCtab *tab) ls.pe = (const char *)~(uintptr_t)0; ls.c = -1; ls.level = (BCDUMP_F_STRIP|(LJ_BE*BCDUMP_F_BE)); + ls.fr2 = LJ_FR2; ls.chunkname = name; pt = lj_bcread_proto(&ls); pt->firstline = ~(BCLine)0; @@ -266,6 +267,23 @@ GCfunc *lj_lib_checkfunc(lua_State *L, int narg) return funcV(o); } +GCproto *lj_lib_checkLproto(lua_State *L, int narg, int nolua) +{ + TValue *o = L->base + narg-1; + if (L->top > o) { + if (tvisproto(o)) { + return protoV(o); + } else if (tvisfunc(o)) { + if (isluafunc(funcV(o))) + return funcproto(funcV(o)); + else if (nolua) + return NULL; + } + } + lj_err_argt(L, narg, LUA_TFUNCTION); + return NULL; /* unreachable */ +} + GCtab *lj_lib_checktab(lua_State *L, int narg) { TValue *o = L->base + narg-1; diff --git a/src/lj_lib.h b/src/lj_lib.h index 6c3a1c83..a48e3c98 100644 --- a/src/lj_lib.h +++ b/src/lj_lib.h @@ -42,6 +42,7 @@ LJ_FUNC lua_Number lj_lib_checknum(lua_State *L, int narg); LJ_FUNC int32_t lj_lib_checkint(lua_State *L, int narg); LJ_FUNC int32_t lj_lib_optint(lua_State *L, int narg, int32_t def); LJ_FUNC GCfunc *lj_lib_checkfunc(lua_State *L, int narg); +LJ_FUNC GCproto *lj_lib_checkLproto(lua_State *L, int narg, int nolua); LJ_FUNC GCtab *lj_lib_checktab(lua_State *L, int narg); LJ_FUNC GCtab *lj_lib_checktabornil(lua_State *L, int narg); LJ_FUNC int lj_lib_checkopt(lua_State *L, int narg, int def, const char *lst); diff --git a/src/lj_load.c b/src/lj_load.c index 07304487..152ef6da 100644 --- a/src/lj_load.c +++ b/src/lj_load.c @@ -34,14 +34,28 @@ static TValue *cpparser(lua_State *L, lua_CFunction dummy, void *ud) UNUSED(dummy); cframe_errfunc(L->cframe) = -1; /* Inherit error function. */ bc = lj_lex_setup(L, ls); - if (ls->mode && !strchr(ls->mode, bc ? 'b' : 't')) { - setstrV(L, L->top++, lj_err_str(L, LJ_ERR_XMODE)); - lj_err_throw(L, LUA_ERRSYNTAX); + if (ls->mode) { + int xmode = 1; + const char *mode = ls->mode; + char c; + while ((c = *mode++)) { + if (c == (bc ? 'b' : 't')) xmode = 0; + if (c == (LJ_FR2 ? 'W' : 'X')) ls->fr2 = !LJ_FR2; + } + if (xmode) { + setstrV(L, L->top++, lj_err_str(L, LJ_ERR_XMODE)); + lj_err_throw(L, LUA_ERRSYNTAX); + } } pt = bc ? lj_bcread(ls) : lj_parse(ls); - fn = lj_func_newL_empty(L, pt, tabref(L->env)); - /* Don't combine above/below into one statement. */ - setfuncV(L, L->top++, fn); + if (ls->fr2 == LJ_FR2) { + fn = lj_func_newL_empty(L, pt, tabref(L->env)); + /* Don't combine above/below into one statement. */ + setfuncV(L, L->top++, fn); + } else { + /* Non-native generation returns a dumpable, but non-runnable prototype. */ + setprotoV(L, L->top++, pt); + } return NULL; } @@ -159,9 +173,10 @@ LUALIB_API int luaL_loadstring(lua_State *L, const char *s) LUA_API int lua_dump(lua_State *L, lua_Writer writer, void *data) { cTValue *o = L->top-1; + uint32_t flags = LJ_FR2*BCDUMP_F_FR2; /* Default mode for legacy C API. */ lj_checkapi(L->top > L->base, "top slot empty"); if (tvisfunc(o) && isluafunc(funcV(o))) - return lj_bcwrite(L, funcproto(funcV(o)), writer, data, 0); + return lj_bcwrite(L, funcproto(funcV(o)), writer, data, flags); else return 1; } diff --git a/src/lj_parse.c b/src/lj_parse.c index a30921af..5a44f8db 100644 --- a/src/lj_parse.c +++ b/src/lj_parse.c @@ -667,19 +667,20 @@ static void bcemit_store(FuncState *fs, ExpDesc *var, ExpDesc *e) /* Emit method lookup expression. */ static void bcemit_method(FuncState *fs, ExpDesc *e, ExpDesc *key) { - BCReg idx, func, obj = expr_toanyreg(fs, e); + BCReg idx, func, fr2, obj = expr_toanyreg(fs, e); expr_free(fs, e); func = fs->freereg; - bcemit_AD(fs, BC_MOV, func+1+LJ_FR2, obj); /* Copy object to 1st argument. */ + fr2 = fs->ls->fr2; + bcemit_AD(fs, BC_MOV, func+1+fr2, obj); /* Copy object to 1st argument. */ lj_assertFS(expr_isstrk(key), "bad usage"); idx = const_str(fs, key); if (idx <= BCMAX_C) { - bcreg_reserve(fs, 2+LJ_FR2); + bcreg_reserve(fs, 2+fr2); bcemit_ABC(fs, BC_TGETS, func, obj, idx); } else { - bcreg_reserve(fs, 3+LJ_FR2); - bcemit_AD(fs, BC_KSTR, func+2+LJ_FR2, idx); - bcemit_ABC(fs, BC_TGETV, func, obj, func+2+LJ_FR2); + bcreg_reserve(fs, 3+fr2); + bcemit_AD(fs, BC_KSTR, func+2+fr2, idx); + bcemit_ABC(fs, BC_TGETV, func, obj, func+2+fr2); fs->freereg--; } e->u.s.info = func; @@ -1326,9 +1327,12 @@ static void fs_fixup_bc(FuncState *fs, GCproto *pt, BCIns *bc, MSize n) { BCInsLine *base = fs->bcbase; MSize i; + BCIns op; pt->sizebc = n; - bc[0] = BCINS_AD((fs->flags & PROTO_VARARG) ? BC_FUNCV : BC_FUNCF, - fs->framesize, 0); + if (fs->ls->fr2 != LJ_FR2) op = BC_NOT; /* Mark non-native prototype. */ + else if ((fs->flags & PROTO_VARARG)) op = BC_FUNCV; + else op = BC_FUNCF; + bc[0] = BCINS_AD(op, fs->framesize, 0); for (i = 1; i < n; i++) bc[i] = base[i].ins; } @@ -1936,11 +1940,11 @@ static void parse_args(LexState *ls, ExpDesc *e) lj_assertFS(e->k == VNONRELOC, "bad expr type %d", e->k); base = e->u.s.info; /* Base register for call. */ if (args.k == VCALL) { - ins = BCINS_ABC(BC_CALLM, base, 2, args.u.s.aux - base - 1 - LJ_FR2); + ins = BCINS_ABC(BC_CALLM, base, 2, args.u.s.aux - base - 1 - ls->fr2); } else { if (args.k != VVOID) expr_tonextreg(fs, &args); - ins = BCINS_ABC(BC_CALL, base, 2, fs->freereg - base - LJ_FR2); + ins = BCINS_ABC(BC_CALL, base, 2, fs->freereg - base - ls->fr2); } expr_init(e, VCALL, bcemit_INS(fs, ins)); e->u.s.aux = base; @@ -1980,7 +1984,7 @@ static void expr_primary(LexState *ls, ExpDesc *v) parse_args(ls, v); } else if (ls->tok == '(' || ls->tok == TK_string || ls->tok == '{') { expr_tonextreg(fs, v); - if (LJ_FR2) bcreg_reserve(fs, 1); + if (ls->fr2) bcreg_reserve(fs, 1); parse_args(ls, v); } else { break; @@ -2565,7 +2569,7 @@ static void parse_for_iter(LexState *ls, GCstr *indexname) line = ls->linenumber; assign_adjust(ls, 3, expr_list(ls, &e), &e); /* The iterator needs another 3 [4] slots (func [pc] | state ctl). */ - bcreg_bump(fs, 3+LJ_FR2); + bcreg_bump(fs, 3+ls->fr2); isnext = (nvars <= 5 && predict_next(ls, fs, exprpc)); var_add(ls, 3); /* Hidden control variables. */ lex_check(ls, TK_do);