Prevent Lua VM re-entry through JIT trace.

JIT recording semantics assumes FFI calls are leaf regarding the LuaJIT
VM: if the execution exited Lua world through FFI machinery it is not
re-entering Lua world again.

However, there is a way to break this assumption via FFI: one can
re-enter LuaJIT VM via Lua C API used within the particular C routine
called via FFI. As a result the following host stack mix is created:
| Lua-FFI -> C routine -> Lua-C API -> Lua VM

This sort of re-entrancy is not supported by LuaJIT tracing compiler.
@mraleph named such kind of the call stack an "FFI sandwich" in the
tarantool/tarantool#4427.

This changeset introduces the mechanism for Lua-C API callbacks similar
to the one implemented for Lua-FFI: trace recording is aborted when the
execution re-enters LuaJIT VM. If re-enter is detected while running the
particular mcode, the runtime finishes its execution with EXIT_FAILURE
code and calls panic routine prior to the exit.

Co-authored-by: Vyacheslav Egorov <vegorov@google.com>
Co-authored-by: Sergey Ostanevich <sergos@tarantool.org>
Signed-off-by: Igor Munkin <imun@cpan.org>

src/lj_api.c

@@ -92,6 +92,18 @@ static GCtab *getcurrenv(lua_State *L)
   return fn->c.gct == ~LJ_TFUNC ? tabref(fn->c.env) : tabref(L->env);
 }
 
+static void check_vm_sandwich(lua_State *L)
+{
+  global_State *g = G(L);
+  /* Forbid Lua world re-entry while running the trace */
+  if (tvref(g->jit_base)) {
+    setstrV(L, L->top++, lj_err_str(L, LJ_ERR_JITREVM));
+    if (g->panic) g->panic(L);
+    exit(EXIT_FAILURE);
+  }
+  lj_trace_abort(g);  /* Never record across Lua VM entrance */
+}
+
 /* -- Miscellaneous API functions ----------------------------------------- */
 
 LUA_API int lua_status(lua_State *L)
@@ -318,6 +330,7 @@ LUA_API int lua_equal(lua_State *L, int idx1, int idx2)
       return (int)(uintptr_t)base;
     } else {
       L->top = base+2;
+      check_vm_sandwich(L);
       lj_vm_call(L, base, 1+1);
       L->top -= 2+LJ_FR2;
       return tvistruecond(L->top+1+LJ_FR2);
@@ -341,6 +354,7 @@ LUA_API int lua_lessthan(lua_State *L, int idx1, int idx2)
       return (int)(uintptr_t)base;
     } else {
       L->top = base+2;
+      check_vm_sandwich(L);
       lj_vm_call(L, base, 1+1);
       L->top -= 2+LJ_FR2;
       return tvistruecond(L->top+1+LJ_FR2);
@@ -786,6 +800,7 @@ LUA_API void lua_concat(lua_State *L, int n)
       }
       n -= (int)(L->top - (top - 2*LJ_FR2));
       L->top = top+2;
+      check_vm_sandwich(L);
       lj_vm_call(L, top, 1+1);
       L->top -= 1+LJ_FR2;
       copyTV(L, L->top-1, L->top+LJ_FR2);
@@ -805,6 +820,7 @@ LUA_API void lua_gettable(lua_State *L, int idx)
   cTValue *v = lj_meta_tget(L, t, L->top-1);
   if (v == NULL) {
     L->top += 2;
+    check_vm_sandwich(L);
     lj_vm_call(L, L->top-2, 1+1);
     L->top -= 2+LJ_FR2;
     v = L->top+1+LJ_FR2;
@@ -820,6 +836,7 @@ LUA_API void lua_getfield(lua_State *L, int idx, const char *k)
   v = lj_meta_tget(L, t, &key);
   if (v == NULL) {
     L->top += 2;
+    check_vm_sandwich(L);
     lj_vm_call(L, L->top-2, 1+1);
     L->top -= 2+LJ_FR2;
     v = L->top+1+LJ_FR2;
@@ -978,6 +995,7 @@ LUA_API void lua_settable(lua_State *L, int idx)
     TValue *base = L->top;
     copyTV(L, base+2, base-3-2*LJ_FR2);
     L->top = base+3;
+    check_vm_sandwich(L);
     lj_vm_call(L, base, 0+1);
     L->top -= 3+LJ_FR2;
   }
@@ -998,6 +1016,7 @@ LUA_API void lua_setfield(lua_State *L, int idx, const char *k)
     TValue *base = L->top;
     copyTV(L, base+2, base-3-2*LJ_FR2);
     L->top = base+3;
+    check_vm_sandwich(L);
     lj_vm_call(L, base, 0+1);
     L->top -= 2+LJ_FR2;
   }
@@ -1129,6 +1148,7 @@ LUA_API void lua_call(lua_State *L, int nargs, int nresults)
   lj_checkapi(L->status == LUA_OK || L->status == LUA_ERRERR,
 	      "thread called in wrong state %d", L->status);
   lj_checkapi_slot(nargs+1);
+  check_vm_sandwich(L);
   lj_vm_call(L, api_call_base(L, nargs), nresults+1);
 }
 
@@ -1147,6 +1167,7 @@ LUA_API int lua_pcall(lua_State *L, int nargs, int nresults, int errfunc)
     cTValue *o = index2adr_stack(L, errfunc);
     ef = savestack(L, o);
   }
+  check_vm_sandwich(L);
   status = lj_vm_pcall(L, api_call_base(L, nargs), nresults+1, ef);
   if (status) hook_restore(g, oldh);
   return status;
@@ -1175,6 +1196,7 @@ LUA_API int lua_cpcall(lua_State *L, lua_CFunction func, void *ud)
   int status;
   lj_checkapi(L->status == LUA_OK || L->status == LUA_ERRERR,
 	      "thread called in wrong state %d", L->status);
+  check_vm_sandwich(L);
   status = lj_vm_cpcall(L, func, ud, cpcall);
   if (status) hook_restore(g, oldh);
   return status;
@@ -1187,6 +1209,7 @@ LUALIB_API int luaL_callmeta(lua_State *L, int idx, const char *field)
     if (LJ_FR2) setnilV(top++);
     copyTV(L, top++, index2adr(L, idx));
     L->top = top;
+    check_vm_sandwich(L);
     lj_vm_call(L, top-1, 1+1);
     return 1;
   }

src/lj_errmsg.h

@@ -109,6 +109,7 @@ ERRDEF(NOJIT,	"no JIT compiler for this architecture (yet)")
 ERRDEF(NOJIT,	"JIT compiler permanently disabled by build option")
 #endif
 ERRDEF(JITOPT,	"unknown or malformed optimization flag " LUA_QS)
+ERRDEF(JITREVM,	"Lua VM re-entry is detected while executing the trace")
 
 /* Lexer/parser errors. */
 ERRDEF(XMODE,	"attempt to load chunk with wrong mode")