Extend -b to generate c/h/obj/o files with embedded bytecode.

Supported object file formats: ELF or PE/COFF object files.
This commit is contained in:
Mike Pall 2011-08-17 00:13:39 +02:00
parent aad7ea3c02
commit a9dd47b7fc
2 changed files with 448 additions and 47 deletions

View File

@ -118,28 +118,56 @@ are accepted:
<li><tt>-l</tt> &mdash; Only list bytecode.</li>
<li><tt>-s</tt> &mdash; Strip debug info (this is the default).</li>
<li><tt>-g</tt> &mdash; Keep debug info.</li>
<li><tt>-n name</tt> &mdash; Set module name (default: auto-detect from input name)</li>
<li><tt>-t type</tt> &mdash; Set output file type (default: auto-detect from output name).</li>
<li><tt>-a arch</tt> &mdash; Override architecture for object files (default: native).</li>
<li><tt>-o os</tt> &mdash; Override OS for object files (default: native).</li>
<li><tt>-e chunk</tt> &mdash; Use chunk string as input.</li>
<li><tt>-</tt> (a single minus sign) &mdash; Use stdin as input and/or stdout as output.</li>
</ul>
<p>
The output file type is auto-detected from the extension of the output
file name:
</p>
<ul>
<li><tt>c</tt> &mdash; C source file, exported bytecode data.</li>
<li><tt>h</tt> &mdash; C header file, static bytecode data.</li>
<li><tt>obj</tt> or <tt>o</tt> &mdash; Object file, exported bytecode data
(OS- and architecture-specific).</li>
<li><tt>raw</tt> or any other extension &mdash; Raw bytecode file (portable).
</ul>
<p>
Notes:
</p>
<ul>
<li>See also <a href="extensions.html#string_dump">string.dump()</a>
for information on bytecode portability and compatibility.</li>
<li>A file in raw bytecode format is auto-detected and can be loaded like
any Lua source file. E.g. directly from the command line or with
<tt>loadfile()</tt>, <tt>dofile()</tt> etc.</li>
<li>To statically embed the bytecode of a module in your application,
generate an object file and just link it with your application.</li>
<li>On most ELF-based systems (e.g. Linux) you need to explicitly export the
global symbols when linking your application, e.g. with: <tt>-Wl,-E</tt></li>
<li><tt>require()</tt> tries to load embedded bytecode data from exported
symbols (in <tt>*.exe</tt> or <tt>lua51.dll</tt> on Windows) and from
shared libraries in <tt>package.cpath</tt>.</li>
</ul>
<p>
Typical usage examples:
</p>
<pre class="code">
luajit -b test.lua test.out # Save to test.out
luajit -b test.lua test.out # Save bytecode to test.out
luajit -bg test.lua test.out # Keep debug info
luajit -be "print('hello world')" test.out # Save cmdline script
luajit -bl test.lua # List to stdout
luajit -bl test.lua test.txt # List to test.txt
luajit -ble "print('hello world')" # List cmdline script
luajit -b test.lua test.obj # Generate object file
# Link test.obj with your application and load it with require("test")
</pre>
<p>
Note: A file in bytecode format is auto-detected and can be loaded like
any Lua source file. E.g. directly from the command line or with
<tt>loadfile()</tt>, <tt>dofile()</tt> etc. See also
<a href="extensions.html#string_dump">string.dump()</a> for information
on bytecode portability and compatibility.
</p>
<h3 id="opt_j"><tt>-j cmd[=arg[,arg...]]</tt></h3>
<p>

View File

@ -10,10 +10,12 @@
--
------------------------------------------------------------------------------
-- Cache some library functions and objects.
local jit = require("jit")
assert(jit.version_num == 20000, "LuaJIT core/library version mismatch")
-- Symbol name prefix for LuaJIT bytecode.
local LJBC_PREFIX = "luaJIT_BC_"
------------------------------------------------------------------------------
local function usage()
@ -22,41 +24,390 @@ Save LuaJIT bytecode: luajit -b[options] input output
-l Only list bytecode.
-s Strip debug info (default).
-g Keep debug info.
-n name Set module name (default: auto-detect from input name).
-t type Set output file type (default: auto-detect from output name).
-a arch Override architecture for object files (default: native).
-o os Override OS for object files (default: native).
-e chunk Use chunk string as input.
-- Stop handling options.
- Use stdin as input and/or stdout as output.
File types: c h obj o raw (default)
]]
os.exit(1)
end
local function check(ok, ...)
if ok then return ok, ... end
io.stderr:write("luajit: ", ...)
io.stderr:write("\n")
os.exit(1)
end
local function readfile(input)
if type(input) == "function" then return input end
if input == "-" then input = nil end
local f, err = loadfile(input)
if not f then
io.stderr:write("luajit: ", err, "\n")
os.exit(1)
end
return f
end
local function readstring(input)
local f, err = loadstring(input)
if not f then
io.stderr:write("luajit: ", err, "\n")
os.exit(1)
end
return f
return check(loadfile(input))
end
local function savefile(name, mode)
if name == "-" then return io.stdout end
local fp, err = io.open(name, mode)
if not fp then
io.stderr:write("luajit: cannot write ", err, "\n")
os.exit(1)
return check(io.open(name, mode))
end
------------------------------------------------------------------------------
local map_type = {
raw = "raw", c = "c", h = "h", o = "obj", obj = "obj",
}
local map_arch = {
x86 = true, x64 = true, arm = true, ppc = true, ppcspe = true,
}
local map_os = {
linux = true, windows = true, osx = true, freebsd = true, netbsd = true,
openbsd = true, solaris = true,
}
local function checkarg(str, map, err)
str = string.lower(str)
local s = check(map[str], "unknown ", err)
return s == true and str or s
end
local function detecttype(str)
local ext = string.match(string.lower(str), "%.(%a+)$")
return map_type[ext] or "raw"
end
local function checkmodname(str)
check(string.match(str, "^[%w_.%-]+$"), "bad module name")
return string.gsub(str, "[%.%-]", "_")
end
local function detectmodname(str)
if type(str) == "string" then
local tail = string.match(str, "[^/\\]+$")
if tail then str = tail end
local head = string.match(str, "^(.*)%.[^.]*$")
if head then str = head end
str = string.match(str, "^[%w_.%-]+")
else
str = nil
end
check(str, "cannot derive module name, use -n name")
return string.gsub(str, "[%.%-]", "_")
end
------------------------------------------------------------------------------
local function bcsave_tail(fp, output, s)
local ok, err = fp:write(s)
if ok and output ~= "-" then ok, err = fp:close() end
check(ok, "cannot write ", output, ": ", err)
end
local function bcsave_raw(output, s)
local fp = savefile(output, "wb")
bcsave_tail(fp, output, s)
end
local function bcsave_c(ctx, output, s)
local fp = savefile(output, "w")
if ctx.type == "c" then
fp:write(string.format([[
#ifdef _cplusplus
extern "C"
#endif
#ifdef _WIN32
__declspec(dllexport)
#endif
const char %s%s[] = {
]], LJBC_PREFIX, ctx.modname))
else
fp:write(string.format([[
#define %s%s_SIZE %d
static const char %s%s[] = {
]], LJBC_PREFIX, ctx.modname, #s, LJBC_PREFIX, ctx.modname))
end
local t, n, m = {}, 0, 0
for i=1,#s do
local b = tostring(string.byte(s, i))
m = m + #b + 1
if m > 78 then
fp:write(table.concat(t, ",", 1, n), ",\n")
n, m = 0, #b + 1
end
n = n + 1
t[n] = b
end
bcsave_tail(fp, output, table.concat(t, ",", 1, n).."\n};\n")
end
local function bcsave_elfobj(ctx, output, s, ffi)
ffi.cdef[[
typedef struct {
uint8_t emagic[4], eclass, eendian, eversion, eosabi, eabiversion, epad[7];
uint16_t type, machine;
uint32_t version;
uint32_t entry, phofs, shofs;
uint32_t flags;
uint16_t ehsize, phentsize, phnum, shentsize, shnum, shstridx;
} ELF32header;
typedef struct {
uint8_t emagic[4], eclass, eendian, eversion, eosabi, eabiversion, epad[7];
uint16_t type, machine;
uint32_t version;
uint64_t entry, phofs, shofs;
uint32_t flags;
uint16_t ehsize, phentsize, phnum, shentsize, shnum, shstridx;
} ELF64header;
typedef struct {
uint32_t name, type, flags, addr, ofs, size, link, info, align, entsize;
} ELF32sectheader;
typedef struct {
uint32_t name, type;
uint64_t flags, addr, ofs, size;
uint32_t link, info;
uint64_t align, entsize;
} ELF64sectheader;
typedef struct {
uint32_t name, value, size;
uint8_t info, other;
uint16_t sectidx;
} ELF32symbol;
typedef struct {
uint32_t name;
uint8_t info, other;
uint16_t sectidx;
uint64_t value, size;
} ELF64symbol;
typedef struct {
ELF32header hdr;
ELF32sectheader sect[6];
ELF32symbol sym[2];
uint8_t space[4096];
} ELF32obj;
typedef struct {
ELF64header hdr;
ELF64sectheader sect[6];
ELF64symbol sym[2];
uint8_t space[4096];
} ELF64obj;
]]
local symname = LJBC_PREFIX..ctx.modname
local is64, isbe = false, false
if ctx.arch == "x64" then
is64 = true
elseif ctx.arch == "ppc" or ctx.arch == "ppcspe" then
isbe = true
end
-- Handle different host/target endianess.
local function f32(x) return x end
local f16, fofs = f32, f32
if ffi.abi("be") ~= isbe then
f32 = bit.bswap
function f16(x) return bit.rshift(bit.bswap(x), 16) end
if is64 then
function fofs(x) return bit.bswap(x)*(2ll^32) end
else
fofs = f32
end
end
-- Create ELF object and fill in header.
local o = ffi.new(is64 and "ELF64obj" or "ELF32obj")
local hdr = o.hdr
if ctx.os == "bsd" or ctx.os == "other" then -- Determine native hdr.eosabi.
local bf = assert(io.open("/bin/ls", "rb"))
local bs = bf:read(9)
bf:close()
ffi.copy(o, bs, 9)
check(hdr.emagic[0] == 127, "no support for writing native object files")
else
hdr.emagic = "\127ELF"
hdr.eosabi = ({ freebsd=9, netbsd=2, openbsd=12, solaris=6 })[ctx.os] or 0
end
hdr.eclass = is64 and 2 or 1
hdr.eendian = isbe and 2 or 1
hdr.eversion = 1
hdr.type = f16(1)
hdr.machine = f16(({ x86=3, x64=62, arm=40, ppc=20, ppcspe=20 })[ctx.arch])
hdr.version = f32(1)
hdr.shofs = fofs(ffi.offsetof(o, "sect"))
hdr.ehsize = f16(ffi.sizeof(hdr))
hdr.shentsize = f16(ffi.sizeof(o.sect[0]))
hdr.shnum = f16(6)
hdr.shstridx = f16(2)
-- Fill in sections and symbols.
local sofs, ofs = ffi.offsetof(o, "space"), 1
for i,name in ipairs{
".symtab", ".shstrtab", ".strtab", ".rodata", ".note.GNU-stack",
} do
local sect = o.sect[i]
sect.align = fofs(1)
sect.name = f32(ofs)
ffi.copy(o.space+ofs, name)
ofs = ofs + #name+1
end
o.sect[1].type = f32(2) -- .symtab
o.sect[1].link = f32(3)
o.sect[1].info = f32(1)
o.sect[1].align = fofs(8)
o.sect[1].ofs = fofs(ffi.offsetof(o, "sym"))
o.sect[1].entsize = fofs(ffi.sizeof(o.sym[0]))
o.sect[1].size = fofs(ffi.sizeof(o.sym))
o.sym[1].name = f32(1)
o.sym[1].sectidx = f16(4)
o.sym[1].size = fofs(#s)
o.sym[1].info = 17
o.sect[2].type = f32(3) -- .shstrtab
o.sect[2].ofs = fofs(sofs)
o.sect[2].size = fofs(ofs)
o.sect[3].type = f32(3) -- .strtab
o.sect[3].ofs = fofs(sofs + ofs)
o.sect[3].size = fofs(#symname+1)
ffi.copy(o.space+ofs+1, symname)
ofs = ofs + #symname + 2
o.sect[4].type = f32(1) -- .rodata
o.sect[4].flags = fofs(2)
o.sect[4].ofs = fofs(sofs + ofs)
o.sect[4].size = fofs(#s)
o.sect[5].type = f32(1) -- .note.GNU-stack
o.sect[5].ofs = fofs(sofs + ofs + #s)
-- Write ELF object file.
local fp = savefile(output, "wb")
fp:write(ffi.string(o, ffi.sizeof(o)-4096+ofs))
bcsave_tail(fp, output, s)
end
local function bcsave_peobj(ctx, output, s, ffi)
ffi.cdef[[
typedef struct {
uint16_t arch, nsects;
uint32_t time, symtabofs, nsyms;
uint16_t opthdrsz, flags;
} PEheader;
typedef struct {
char name[8];
uint32_t vsize, vaddr, size, ofs, relocofs, lineofs;
uint16_t nreloc, nline;
uint32_t flags;
} PEsection;
typedef struct __attribute((packed)) {
union {
char name[8];
uint32_t nameref[2];
};
uint32_t value;
int16_t sect;
uint16_t type;
uint8_t scl, naux;
} PEsym;
typedef struct __attribute((packed)) {
uint32_t size;
uint16_t nreloc, nline;
uint32_t cksum;
uint16_t assoc;
uint8_t comdatsel, unused[3];
} PEsymaux;
typedef struct {
PEheader hdr;
PEsection sect[2];
// Must be an even number of symbol structs.
PEsym sym0;
PEsymaux sym0aux;
PEsym sym1;
PEsymaux sym1aux;
PEsym sym2;
PEsym sym3;
uint32_t strtabsize;
uint8_t space[4096];
} PEobj;
]]
local symname = LJBC_PREFIX..ctx.modname
local is64, isbe = false, false
if ctx.arch == "x86" then
symname = "_"..symname
elseif ctx.arch == "x64" then
is64 = true
elseif ctx.arch == "ppc" or ctx.arch == "ppcspe" then
isbe = true
end
local symexport = " /EXPORT:"..symname..",DATA "
-- Handle different host/target endianess.
local function f32(x) return x end
local f16 = f32
if ffi.abi("be") ~= isbe then
f32 = bit.bswap
function f16(x) return bit.rshift(bit.bswap(x), 16) end
end
-- Create PE object and fill in header.
local o = ffi.new("PEobj")
local hdr = o.hdr
hdr.arch = f16(({ x86=0x14c, x64=0x8664, arm=0x1c0, ppc=0x1f1 })[ctx.arch])
hdr.nsects = f16(2)
hdr.symtabofs = f32(ffi.offsetof(o, "sym0"))
hdr.nsyms = f32(6)
-- Fill in sections and symbols.
o.sect[0].name = ".drectve"
o.sect[0].size = f32(#symexport)
o.sect[0].flags = f32(0x00100a00)
o.sym0.sect = f16(1)
o.sym0.scl = 3
o.sym0.name = ".drectve"
o.sym0.naux = 1
o.sym0aux.size = f32(#symexport)
o.sect[1].name = ".rdata"
o.sect[1].size = f32(#s)
o.sect[1].flags = f32(0x40300040)
o.sym1.sect = f16(2)
o.sym1.scl = 3
o.sym1.name = ".rdata"
o.sym1.naux = 1
o.sym1aux.size = f32(#s)
o.sym2.sect = f16(2)
o.sym2.scl = 2
o.sym2.nameref[1] = f32(4)
o.sym3.sect = f16(-1)
o.sym3.scl = 2
o.sym3.value = f32(1)
o.sym3.name = "@feat.00" -- Mark as SafeSEH compliant.
ffi.copy(o.space, symname)
local ofs = #symname + 1
o.strtabsize = f32(ofs + 4)
o.sect[0].ofs = f32(ffi.offsetof(o, "space") + ofs)
ffi.copy(o.space + ofs, symexport)
ofs = ofs + #symexport
o.sect[1].ofs = f32(ffi.offsetof(o, "space") + ofs)
-- Write PE object file.
local fp = savefile(output, "wb")
fp:write(ffi.string(o, ffi.sizeof(o)-4096+ofs))
bcsave_tail(fp, output, s)
end
local function bcsave_machobj(ctx, output, s, ffi)
check(false, "NYI: no support for writing OSX object files")
end
local function bcsave_obj(ctx, output, s)
local ok, ffi = pcall(require, "ffi")
check(ok, "FFI library required to write this file type")
if ctx.os == "windows" then
return bcsave_peobj(ctx, output, s, ffi)
elseif ctx.os == "osx" then
return bcsave_machobj(ctx, output, s, ffi)
else
return bcsave_elfobj(ctx, output, s, ffi)
end
return fp
end
------------------------------------------------------------------------------
@ -66,15 +417,23 @@ local function bclist(input, output)
require("jit.bc").dump(f, savefile(output, "w"), true)
end
local function bcsave(input, output, strip)
local function bcsave(ctx, input, output)
local f = readfile(input)
local s = string.dump(f, strip)
local fp = savefile(output, "wb")
local ok, err = fp:write(s)
if ok and output ~= "-" then ok, err = fp:close() end
if not ok then
io.stderr:write("luajit: cannot write ", arg[2], ": ", err, "\n")
os.exit(1)
local s = string.dump(f, ctx.strip)
local t = ctx.type
if not t then
t = detecttype(output)
ctx.type = t
end
if t == "raw" then
bcsave_raw(output, s)
else
if not ctx.modname then ctx.modname = detectmodname(input) end
if t == "obj" then
bcsave_obj(ctx, output, s)
else
bcsave_c(ctx, output, s)
end
end
end
@ -82,27 +441,41 @@ local function docmd(...)
local arg = {...}
local n = 1
local list = false
local strip = true
local ctx = {
strip = true, arch = jit.arch, os = string.lower(jit.os),
type = false, modname = false,
}
while n <= #arg do
local a = arg[n]
if type(a) == "string" and string.sub(a, 1, 1) == "-" and a ~= "-" then
if a == "--" then table.remove(arg, n); break end
table.remove(arg, n)
if a == "--" then break end
for m=2,#a do
local opt = string.sub(a, m, m)
if opt == "l" then
list = true
elseif opt == "s" then
strip = true
ctx.strip = true
elseif opt == "g" then
strip = false
elseif opt == "e" then
if n ~= 1 or #arg < 2 or m ~= #a then usage() end
arg[2] = readstring(arg[2])
ctx.strip = false
else
usage()
if arg[n] == nil or m ~= #a then usage() end
if opt == "e" then
if n ~= 1 then usage() end
arg[1] = check(loadstring(arg[1]))
elseif opt == "n" then
ctx.modname = checkmodname(table.remove(arg, n))
elseif opt == "t" then
ctx.type = checkarg(table.remove(arg, n), map_type, "file type")
elseif opt == "a" then
ctx.arch = checkarg(table.remove(arg, n), map_arch, "architecture")
elseif opt == "o" then
ctx.os = checkarg(table.remove(arg, n), map_os, "OS name")
else
usage()
end
end
end
table.remove(arg, n)
else
n = n + 1
end
@ -112,7 +485,7 @@ local function docmd(...)
bclist(arg[1], arg[2] or "-")
else
if #arg ~= 2 then usage() end
bcsave(arg[1], arg[2], strip)
bcsave(ctx, arg[1], arg[2])
end
end