Fix KBASE for Lua functions below stitched fast functions.

src/vm_arm.dasc

@@ -2109,6 +2109,8 @@ static void build_subroutines(BuildCtx *ctx)
   |   bhi >9				// More results wanted?
   |
   |  ldr TRACE:RA, [CARG1, CARG3, lsl #2]
+  |  cmp TRACE:RA, #0
+  |  beq ->cont_nop
   |  ldrh RC, TRACE:RA->link
   |  cmp RC, CARG3
   |  beq ->cont_nop			// Blacklisted.
@@ -2193,7 +2195,7 @@ static void build_subroutines(BuildCtx *ctx)
   |  ldr L, SAVE_L
   |1:
   |  cmp CARG1, #0
-  |  blt >3				// Check for error from exit.
+  |  blt >9				// Check for error from exit.
   |   lsl RC, CARG1, #3
   |  ldr LFUNC:CARG2, [BASE, FRAME_FUNC]
   |   str RC, SAVE_MULTRES
@@ -2209,6 +2211,9 @@ static void build_subroutines(BuildCtx *ctx)
   |   ldr INS, [PC], #4
   |     lsl MASKR8, MASKR8, #3		// MASKR8 = 255*8.
   |    st_vmstate CARG4
+  |  cmp OP, #BC_FUNCC+2		// Fast function?
+  |  bhs >4
+  |2:
   |  cmp OP, #BC_FUNCF			// Function header?
   |  ldr OP, [DISPATCH, OP, lsl #2]
   |   decode_RA8 RA, INS
@@ -2218,7 +2223,20 @@ static void build_subroutines(BuildCtx *ctx)
   |   ldrhs CARG3, [BASE, FRAME_FUNC]
   |  bx OP
   |
-  |3:  // Rethrow error from the right C frame.
+  |4:  // Check frame below fast function.
+  |  ldr CARG1, [BASE, FRAME_PC]
+  |  ands CARG2, CARG1, #FRAME_TYPE
+  |  bne <2			// Trace stitching continuation?
+  |  // Otherwise set KBASE for Lua function below fast function.
+  |  ldr CARG3, [CARG1, #-4]
+  |  decode_RA8 CARG1, CARG3
+  |  sub CARG2, BASE, CARG1
+  |  ldr LFUNC:CARG3, [CARG2, #-16]
+  |  ldr CARG3, LFUNC:CARG3->field_pc
+  |  ldr KBASE, [CARG3, #PC2PROTO(k)]
+  |  b <2
+  |
+  |9:  // Rethrow error from the right C frame.
   |  rsb CARG2, CARG1, #0
   |  mov CARG1, L
   |  bl extern lj_err_throw		// (lua_State *L, int errcode)

src/vm_mips.dasc

@@ -2043,6 +2043,8 @@ static void build_subroutines(BuildCtx *ctx)
   |
   |  addu TMP2, TMP1, TMP2
   |  lw TRACE:TMP2, 0(TMP2)
+  |  beqz TRACE:TMP2, ->cont_nop
+  |.  nop
   |  lhu RD, TRACE:TMP2->link
   |  beq RD, TMP3, ->cont_nop		// Blacklisted.
   |.  load_got lj_dispatch_stitch
@@ -2144,7 +2146,7 @@ static void build_subroutines(BuildCtx *ctx)
   |   addiu DISPATCH, JGL, -GG_DISP2G-32768
   |  sw BASE, L->base
   |1:
-  |  bltz CRET1, >3			// Check for error from exit.
+  |  bltz CRET1, >9			// Check for error from exit.
   |.  lw LFUNC:RB, FRAME_FUNC(BASE)
   |    lui TMP3, 0x59c0			// TOBIT = 2^52 + 2^51 (float).
   |  sll MULTRES, CRET1, 3
@@ -2172,11 +2174,27 @@ static void build_subroutines(BuildCtx *ctx)
   |  jr AT
   |.  decode_RD8b RD
   |2:
+  |  sltiu TMP2, TMP1, (BC_FUNCC+2)*4	// Fast function?
+  |  bnez TMP2, >3
+  |.  lw TMP1, FRAME_PC(BASE)
+  |  // Check frame below fast function.
+  |  andi TMP0, TMP1, FRAME_TYPE
+  |  bnez TMP0, >3			// Trace stitching continuation?
+  |.  nop
+  |  // Otherwise set KBASE for Lua function below fast function.
+  |  lw TMP2, -4(TMP1)
+  |  decode_RA8a TMP0, TMP2
+  |  decode_RA8b TMP0
+  |  subu TMP1, BASE, TMP0
+  |  lw LFUNC:TMP2, -8+FRAME_FUNC(TMP1)
+  |  lw TMP1, LFUNC:TMP2->pc
+  |  lw KBASE, PC2PROTO(k)(TMP1)
+  |3:
   |  addiu RC, MULTRES, -8
   |  jr AT
   |.  addu RA, RA, BASE
   |
-  |3:  // Rethrow error from the right C frame.
+  |9:  // Rethrow error from the right C frame.
   |  load_got lj_err_throw
   |  negu CARG2, CRET1
   |  call_intern lj_err_throw		// (lua_State *L, int errcode)

src/vm_ppc.dasc

@@ -2531,6 +2531,8 @@ static void build_subroutines(BuildCtx *ctx)
   |
   |  slwi TMP2, TMP3, 2
   |  lwzx TRACE:TMP2, TMP1, TMP2
+  |  cmpwi TRACE:TMP2, 0
+  |  beq ->cont_nop
   |  lhz RD, TRACE:TMP2->link
   |  cmpw RD, TMP3
   |   cmpwi cr1, RD, 0
@@ -2637,7 +2639,7 @@ static void build_subroutines(BuildCtx *ctx)
   |  stp BASE, L->base
   |1:
   |  cmpwi CARG1, 0
-  |  blt >3				// Check for error from exit.
+  |  blt >9				// Check for error from exit.
   |  lwz LFUNC:RB, FRAME_FUNC(BASE)
   |   slwi MULTRES, CARG1, 3
   |    li TMP2, 0
@@ -2673,11 +2675,25 @@ static void build_subroutines(BuildCtx *ctx)
   |   decode_RC8 RC, INS
   |  bctr
   |2:
+  |  cmplwi TMP1, (BC_FUNCC+2)*4	// Fast function?
+  |  blt >3
+  |  // Check frame below fast function.
+  |  lwz TMP1, FRAME_PC(BASE)
+  |  andix. TMP0, TMP1, FRAME_TYPE
+  |  bney >3				// Trace stitching continuation?
+  |  // Otherwise set KBASE for Lua function below fast function.
+  |  lwz TMP2, -4(TMP1)
+  |  decode_RA8 TMP0, TMP2
+  |  sub TMP1, BASE, TMP0
+  |  lwz LFUNC:TMP2, -12(TMP1)
+  |  lwz TMP1, LFUNC:TMP2->pc
+  |  lwz KBASE, PC2PROTO(k)(TMP1)
+  |3:
   |   subi RC, MULTRES, 8
   |   add RA, RA, BASE
   |  bctr
   |
-  |3:  // Rethrow error from the right C frame.
+  |9:  // Rethrow error from the right C frame.
   |  neg CARG2, CARG1
   |  mr CARG1, L
   |  bl extern lj_err_throw		// (lua_State *L, int errcode)

src/vm_x86.dasc

@@ -2697,6 +2697,8 @@ static void build_subroutines(BuildCtx *ctx)
   |  mov RB, TMP1			// Get previous trace number.
   |  mov RA, [DISPATCH+DISPATCH_J(trace)]
   |  mov TRACE:RD, [RA+RB*4]
+  |  test TRACE:RD, TRACE:RD
+  |  jz ->cont_nop
   |  movzx RD, word TRACE:RD->link
   |  cmp RD, RB
   |  je ->cont_nop			// Blacklisted.
@@ -2844,7 +2846,7 @@ static void build_subroutines(BuildCtx *ctx)
   |  mov r13, TMPa
   |  mov r12, TMPQ
   |.endif
-  |  test RD, RD; js >3			// Check for error from exit.
+  |  test RD, RD; js >9			// Check for error from exit.
   |  mov L:RB, SAVE_L
   |  mov MULTRES, RD
   |  mov LFUNC:KBASE, [BASE-8]
@@ -2860,16 +2862,31 @@ static void build_subroutines(BuildCtx *ctx)
   |  add PC, 4
   |  shr RC, 16
   |  cmp OP, BC_FUNCF			// Function header?
-  |  jb >2
+  |  jb >3
-  |  mov RC, MULTRES			// RC/RD holds nres+1.
+  |  cmp OP, BC_FUNCC+2			// Fast function?
+  |  jae >4
   |2:
+  |  mov RC, MULTRES			// RC/RD holds nres+1.
+  |3:
   |.if X64
   |  jmp aword [DISPATCH+OP*8]
   |.else
   |  jmp aword [DISPATCH+OP*4]
   |.endif
   |
-  |3:  // Rethrow error from the right C frame.
+  |4:  // Check frame below fast function.
+  |  mov RC, [BASE-4]
+  |  test RC, FRAME_TYPE
+  |  jnz <2				// Trace stitching continuation?
+  |  // Otherwise set KBASE for Lua function below fast function.
+  |  movzx RC, byte [RC-3]
+  |  not RCa
+  |  mov LFUNC:KBASE, [BASE+RC*8-8]
+  |  mov KBASE, LFUNC:KBASE->pc
+  |  mov KBASE, [KBASE+PC2PROTO(k)]
+  |  jmp <2
+  |
+  |9:  // Rethrow error from the right C frame.
   |  neg RD
   |  mov FCARG1, L:RB
   |  mov FCARG2, RD