diff --git a/src/Makefile.dep b/src/Makefile.dep index 68b4f532..8af71749 100644 --- a/src/Makefile.dep +++ b/src/Makefile.dep @@ -80,7 +80,7 @@ lj_lib.o: lj_lib.c lauxlib.h lua.h luaconf.h lj_obj.h lj_def.h lj_arch.h \ lj_dispatch.h lj_jit.h lj_ir.h lj_vm.h lj_lib.h lj_mcode.o: lj_mcode.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \ lj_gc.h lj_jit.h lj_ir.h lj_mcode.h lj_trace.h lj_dispatch.h lj_bc.h \ - lj_traceerr.h + lj_traceerr.h lj_vm.h lj_meta.o: lj_meta.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h lj_gc.h \ lj_err.h lj_errmsg.h lj_str.h lj_tab.h lj_meta.h lj_bc.h lj_vm.h lj_obj.o: lj_obj.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h diff --git a/src/lj_mcode.c b/src/lj_mcode.c index 2ae0c2a9..e9921601 100644 --- a/src/lj_mcode.c +++ b/src/lj_mcode.c @@ -15,6 +15,7 @@ #include "lj_mcode.h" #include "lj_trace.h" #include "lj_dispatch.h" +#include "lj_vm.h" /* -- OS-specific functions ----------------------------------------------- */ @@ -27,21 +28,22 @@ #define MCPROT_RX PAGE_EXECUTE_READ #define MCPROT_RWX PAGE_EXECUTE_READWRITE -static LJ_AINLINE void *mcode_alloc(jit_State *J, size_t sz, DWORD prot) +static void *mcode_alloc_at(jit_State *J, uintptr_t hint, size_t sz, DWORD prot) { - void *p = VirtualAlloc(NULL, sz, MEM_RESERVE|MEM_COMMIT|MEM_TOP_DOWN, prot); + void *p = VirtualAlloc((void *)hint, sz, + MEM_RESERVE|MEM_COMMIT|MEM_TOP_DOWN, prot); if (!p) lj_trace_err(J, LJ_TRERR_MCODEAL); return p; } -static LJ_AINLINE void mcode_free(jit_State *J, void *p, size_t sz) +static void mcode_free(jit_State *J, void *p, size_t sz) { UNUSED(J); UNUSED(sz); VirtualFree(p, 0, MEM_RELEASE); } -static LJ_AINLINE void mcode_setprot(void *p, size_t sz, DWORD prot) +static void mcode_setprot(void *p, size_t sz, DWORD prot) { DWORD oprot; VirtualProtect(p, sz, prot, &oprot); @@ -59,21 +61,21 @@ static LJ_AINLINE void mcode_setprot(void *p, size_t sz, DWORD prot) #define MCPROT_RX (PROT_READ|PROT_EXEC) #define MCPROT_RWX (PROT_READ|PROT_WRITE|PROT_EXEC) -static LJ_AINLINE void *mcode_alloc(jit_State *J, size_t sz, int prot) +static void *mcode_alloc_at(jit_State *J, uintptr_t hint, size_t sz, int prot) { - void *p = mmap(NULL, sz, prot, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0); + void *p = mmap((void *)hint, sz, prot, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0); if (p == MAP_FAILED) lj_trace_err(J, LJ_TRERR_MCODEAL); return p; } -static LJ_AINLINE void mcode_free(jit_State *J, void *p, size_t sz) +static void mcode_free(jit_State *J, void *p, size_t sz) { UNUSED(J); munmap(p, sz); } -static LJ_AINLINE void mcode_setprot(void *p, size_t sz, int prot) +static void mcode_setprot(void *p, size_t sz, int prot) { mprotect(p, sz, prot); } @@ -86,13 +88,13 @@ static LJ_AINLINE void mcode_setprot(void *p, size_t sz, int prot) #define MCPROT_RX 0 #define MCPROT_RWX 0 -static LJ_AINLINE void *mcode_alloc(jit_State *J, size_t sz, int prot) +static void *mcode_alloc_at(jit_State *J, uintptr_t hint, size_t sz, int prot) { - UNUSED(prot); + UNUSED(hint); UNUSED(prot); return lj_mem_new(J->L, sz); } -static LJ_AINLINE void mcode_free(jit_State *J, void *p, size_t sz) +static void mcode_free(jit_State *J, void *p, size_t sz) { lj_mem_free(J2G(J), p, sz); } @@ -101,6 +103,50 @@ static LJ_AINLINE void mcode_free(jit_State *J, void *p, size_t sz) #endif +/* -- MCode area allocation ----------------------------------------------- */ + +#if LJ_64 + +/* Get memory within relative jump distance of our code in 64 bit mode. */ +static void *mcode_alloc(jit_State *J, size_t sz, int prot) +{ + /* Target an address in the static assembler code. + ** Try addresses within a distance of target-1GB+1MB .. target+1GB-1MB. + */ + uintptr_t target = (uintptr_t)(void *)lj_vm_exit_handler; + const uintptr_t range = (1u<<31) - (1u << 21); + int i; + /* First try a contiguous area below the last one. */ + if (J->mcarea && (uintptr_t)J->mcarea - sz < (uintptr_t)1<<47) { + void *p = mcode_alloc_at(J, (uintptr_t)J->mcarea - sz, sz, prot); + if ((uintptr_t)p + sz - target < range || target - (uintptr_t)p < range) + return p; + mcode_free(J, p, sz); /* Free badly placed area. */ + } + /* Next try probing with hinted addresses. */ + for (i = 0; i < 32; i++) { /* 32 attempts ought to be enough ... */ + uintptr_t hint; + void *p; + do { + hint = LJ_PRNG_BITS(J, 15) << 16; + } while (!(hint + sz < range && + target + hint - (range>>1) < (uintptr_t)1<<47)); + p = mcode_alloc_at(J, target + hint - (range>>1), sz, prot); + if ((uintptr_t)p + sz - target < range || target - (uintptr_t)p < range) + return p; + mcode_free(J, p, sz); /* Free badly placed area. */ + } + lj_trace_err(J, LJ_TRERR_MCODEAL); /* Give up. OS probably ignores hints? */ + return NULL; +} + +#else + +/* All 32 bit memory addresses are reachable by relative jumps on x86. */ +#define mcode_alloc(J, sz, prot) mcode_alloc_at((J), 0, (sz), (prot)) + +#endif + /* -- MCode area management ----------------------------------------------- */ /* Define this ONLY if the page protection twiddling becomes a bottleneck. */ diff --git a/src/lj_state.c b/src/lj_state.c index 3305fd18..e690a5d3 100644 --- a/src/lj_state.c +++ b/src/lj_state.c @@ -143,6 +143,7 @@ static void close_state(lua_State *L) global_State *g = G(L); #ifndef LUAJIT_USE_SYSMALLOC if (g->allocf == lj_alloc_f) { + lj_trace_freestate(g); lj_alloc_destroy(g->allocd); } else #endif