Rethrow errors from trace exit handling from the right C frame.

src/buildvm_x86.dasc

@@ -2591,11 +2591,14 @@ static void build_subroutines(BuildCtx *ctx, int cmov, int sse)
   |  lea FCARG2, [esp+16]
   |  lea FCARG1, [DISPATCH+GG_DISP2J]
   |  call extern lj_trace_exit@8	// (jit_State *J, ExitState *ex)
-  |  // Interpreter C frame returned in eax.
-  |  mov esp, eax			// Reposition stack to C frame.
+  |  // Error code returned in eax (RD).
+  |  mov RAa, L:RB->cframe
+  |  and RAa, CFRAME_RAWMASK
+  |  mov esp, RAa			// Reposition stack to C frame.
+  |  mov SAVE_L, L:RB			// Needed for on-trace resume/yield.
+  |  test RD, RD; jnz >1		// Check for error from exit.
   |  mov BASE, L:RB->base
   |  mov PC, SAVE_PC
-  |  mov SAVE_L, L:RB			// Needed for on-trace resume/yield.
   |.endif
 #endif
   |->vm_exit_interp:
@@ -2606,6 +2609,11 @@ static void build_subroutines(BuildCtx *ctx, int cmov, int sse)
   |  mov dword [DISPATCH+DISPATCH_GL(jit_L)], 0
   |  set_vmstate INTERP
   |  ins_next
+  |
+  |1:  // Rethrow error from the right C frame.
+  |  mov FCARG1, L:RB
+  |  mov FCARG2, RD
+  |  call extern lj_err_throw@8		// (lua_State *L, int errcode)
 #endif
   |
   |//-----------------------------------------------------------------------

src/lj_err.c

@@ -695,7 +695,7 @@ static void err_raise_ext(int errcode)
 /* -- Error handling ------------------------------------------------------ */
 
 /* Throw error. Find catch frame, unwind stack and continue. */
-LJ_NOINLINE void lj_err_throw(lua_State *L, int errcode)
+LJ_NOINLINE void LJ_FASTCALL lj_err_throw(lua_State *L, int errcode)
 {
   global_State *g = G(L);
   lj_trace_abort(g);

src/lj_err.h

@@ -18,7 +18,7 @@ typedef enum {
 } ErrMsg;
 
 LJ_FUNC GCstr *lj_err_str(lua_State *L, ErrMsg em);
-LJ_FUNC_NORET void lj_err_throw(lua_State *L, int errcode);
+LJ_FUNC_NORET void LJ_FASTCALL lj_err_throw(lua_State *L, int errcode);
 LJ_FUNC_NORET void lj_err_mem(lua_State *L);
 LJ_FUNC_NORET void lj_err_run(lua_State *L);
 LJ_FUNC_NORET void lj_err_msg(lua_State *L, ErrMsg em);

src/lj_snap.c

@@ -216,8 +216,12 @@ const BCIns *lj_snap_restore(jit_State *J, void *exptr)
   BCReg nslots = snap->nslots;
   TValue *frame;
   BloomFilter rfilt = snap_renamefilter(T, snapno);
+  const BCIns *pc = snap_pc(map[nent]);
   lua_State *L = J->L;
 
+  /* Set interpreter PC to the next PC to get correct error messages. */
+  setcframe_pc(cframe_raw(L->cframe), pc+1);
+
   /* Make sure the stack is big enough for the slots from the snapshot. */
   if (LJ_UNLIKELY(L->base + nslots > L->maxstack)) {
     L->top = curr_topL(L);
@@ -289,7 +293,7 @@ const BCIns *lj_snap_restore(jit_State *J, void *exptr)
   }
   L->top = curr_topL(L);
   lua_assert(map + nent == flinks);
-  return snap_pc(*flinks);
+  return pc;
 }
 
 #undef IR

src/lj_trace.c

@@ -574,13 +574,34 @@ static void trace_hotside(jit_State *J, const BCIns *pc)
   }
 }
 
-/* A trace exited. Restore interpreter state. */
-void * LJ_FASTCALL lj_trace_exit(jit_State *J, void *exptr)
+/* Tiny struct to pass data to protected call. */
+typedef struct ExitDataCP {
+  jit_State *J;
+  void *exptr;		/* Pointer to exit state. */
+  const BCIns *pc;	/* Restart interpreter at this PC. */
+} ExitDataCP;
+
+/* Need to protect lj_snap_restore because it may throw. */
+static TValue *trace_exit_cp(lua_State *L, lua_CFunction dummy, void *ud)
+{
+  ExitDataCP *exd = (ExitDataCP *)ud;
+  cframe_errfunc(L->cframe) = -1;  /* Inherit error function. */
+  exd->pc = lj_snap_restore(exd->J, exd->exptr);
+  UNUSED(dummy);
+  return NULL;
+}
+
+/* A trace exited. Restore interpreter state. */
+int LJ_FASTCALL lj_trace_exit(jit_State *J, void *exptr)
 {
-  const BCIns *pc = lj_snap_restore(J, exptr);
   lua_State *L = J->L;
-  void *cf = cframe_raw(L->cframe);
-  setcframe_pc(cf, pc);  /* Restart interpreter at this PC. */
+  ExitDataCP exd;
+  int errcode;
+  exd.J = J;
+  exd.exptr = exptr;
+  errcode = lj_vm_cpcall(L, NULL, &exd, trace_exit_cp);
+  if (errcode)
+    return errcode;
 
   lj_vmevent_send(L, TEXIT,
     ExitState *ex = (ExitState *)exptr;
@@ -600,8 +621,9 @@ void * LJ_FASTCALL lj_trace_exit(jit_State *J, void *exptr)
     }
   );
 
-  trace_hotside(J, pc);
-  return cf;  /* Return the interpreter C frame. */
+  trace_hotside(J, exd.pc);
+  setcframe_pc(cframe_raw(L->cframe), exd.pc);
+  return 0;
 }
 
 #endif

src/lj_trace.h

@@ -33,7 +33,7 @@ LJ_FUNC void lj_trace_freestate(global_State *g);
 /* Event handling. */
 LJ_FUNC void lj_trace_ins(jit_State *J, const BCIns *pc);
 LJ_FUNCA void LJ_FASTCALL lj_trace_hot(jit_State *J, const BCIns *pc);
-LJ_FUNCA void * LJ_FASTCALL lj_trace_exit(jit_State *J, void *exptr);
+LJ_FUNCA int LJ_FASTCALL lj_trace_exit(jit_State *J, void *exptr);
 
 /* Signal asynchronous abort of trace or end of trace. */
 #define lj_trace_abort(g)	(G2J(g)->state &= ~LJ_TRACE_ACTIVE)