diff --git a/src/vm_arm.dasc b/src/vm_arm.dasc index 559d20bd..8a48016e 100644 --- a/src/vm_arm.dasc +++ b/src/vm_arm.dasc @@ -2109,6 +2109,8 @@ static void build_subroutines(BuildCtx *ctx) | bhi >9 // More results wanted? | | ldr TRACE:RA, [CARG1, CARG3, lsl #2] + | cmp TRACE:RA, #0 + | beq ->cont_nop | ldrh RC, TRACE:RA->link | cmp RC, CARG3 | beq ->cont_nop // Blacklisted. @@ -2193,7 +2195,7 @@ static void build_subroutines(BuildCtx *ctx) | ldr L, SAVE_L |1: | cmp CARG1, #0 - | blt >3 // Check for error from exit. + | blt >9 // Check for error from exit. | lsl RC, CARG1, #3 | ldr LFUNC:CARG2, [BASE, FRAME_FUNC] | str RC, SAVE_MULTRES @@ -2209,6 +2211,9 @@ static void build_subroutines(BuildCtx *ctx) | ldr INS, [PC], #4 | lsl MASKR8, MASKR8, #3 // MASKR8 = 255*8. | st_vmstate CARG4 + | cmp OP, #BC_FUNCC+2 // Fast function? + | bhs >4 + |2: | cmp OP, #BC_FUNCF // Function header? | ldr OP, [DISPATCH, OP, lsl #2] | decode_RA8 RA, INS @@ -2218,7 +2223,20 @@ static void build_subroutines(BuildCtx *ctx) | ldrhs CARG3, [BASE, FRAME_FUNC] | bx OP | - |3: // Rethrow error from the right C frame. + |4: // Check frame below fast function. + | ldr CARG1, [BASE, FRAME_PC] + | ands CARG2, CARG1, #FRAME_TYPE + | bne <2 // Trace stitching continuation? + | // Otherwise set KBASE for Lua function below fast function. + | ldr CARG3, [CARG1, #-4] + | decode_RA8 CARG1, CARG3 + | sub CARG2, BASE, CARG1 + | ldr LFUNC:CARG3, [CARG2, #-16] + | ldr CARG3, LFUNC:CARG3->field_pc + | ldr KBASE, [CARG3, #PC2PROTO(k)] + | b <2 + | + |9: // Rethrow error from the right C frame. | rsb CARG2, CARG1, #0 | mov CARG1, L | bl extern lj_err_throw // (lua_State *L, int errcode) diff --git a/src/vm_mips.dasc b/src/vm_mips.dasc index 094ffe38..4aaff4b0 100644 --- a/src/vm_mips.dasc +++ b/src/vm_mips.dasc @@ -2043,6 +2043,8 @@ static void build_subroutines(BuildCtx *ctx) | | addu TMP2, TMP1, TMP2 | lw TRACE:TMP2, 0(TMP2) + | beqz TRACE:TMP2, ->cont_nop + |. nop | lhu RD, TRACE:TMP2->link | beq RD, TMP3, ->cont_nop // Blacklisted. |. load_got lj_dispatch_stitch @@ -2144,7 +2146,7 @@ static void build_subroutines(BuildCtx *ctx) | addiu DISPATCH, JGL, -GG_DISP2G-32768 | sw BASE, L->base |1: - | bltz CRET1, >3 // Check for error from exit. + | bltz CRET1, >9 // Check for error from exit. |. lw LFUNC:RB, FRAME_FUNC(BASE) | lui TMP3, 0x59c0 // TOBIT = 2^52 + 2^51 (float). | sll MULTRES, CRET1, 3 @@ -2172,11 +2174,27 @@ static void build_subroutines(BuildCtx *ctx) | jr AT |. decode_RD8b RD |2: + | sltiu TMP2, TMP1, (BC_FUNCC+2)*4 // Fast function? + | bnez TMP2, >3 + |. lw TMP1, FRAME_PC(BASE) + | // Check frame below fast function. + | andi TMP0, TMP1, FRAME_TYPE + | bnez TMP0, >3 // Trace stitching continuation? + |. nop + | // Otherwise set KBASE for Lua function below fast function. + | lw TMP2, -4(TMP1) + | decode_RA8a TMP0, TMP2 + | decode_RA8b TMP0 + | subu TMP1, BASE, TMP0 + | lw LFUNC:TMP2, -8+FRAME_FUNC(TMP1) + | lw TMP1, LFUNC:TMP2->pc + | lw KBASE, PC2PROTO(k)(TMP1) + |3: | addiu RC, MULTRES, -8 | jr AT |. addu RA, RA, BASE | - |3: // Rethrow error from the right C frame. + |9: // Rethrow error from the right C frame. | load_got lj_err_throw | negu CARG2, CRET1 | call_intern lj_err_throw // (lua_State *L, int errcode) diff --git a/src/vm_ppc.dasc b/src/vm_ppc.dasc index c21f5c43..1d5446c2 100644 --- a/src/vm_ppc.dasc +++ b/src/vm_ppc.dasc @@ -2531,6 +2531,8 @@ static void build_subroutines(BuildCtx *ctx) | | slwi TMP2, TMP3, 2 | lwzx TRACE:TMP2, TMP1, TMP2 + | cmpwi TRACE:TMP2, 0 + | beq ->cont_nop | lhz RD, TRACE:TMP2->link | cmpw RD, TMP3 | cmpwi cr1, RD, 0 @@ -2637,7 +2639,7 @@ static void build_subroutines(BuildCtx *ctx) | stp BASE, L->base |1: | cmpwi CARG1, 0 - | blt >3 // Check for error from exit. + | blt >9 // Check for error from exit. | lwz LFUNC:RB, FRAME_FUNC(BASE) | slwi MULTRES, CARG1, 3 | li TMP2, 0 @@ -2673,11 +2675,25 @@ static void build_subroutines(BuildCtx *ctx) | decode_RC8 RC, INS | bctr |2: + | cmplwi TMP1, (BC_FUNCC+2)*4 // Fast function? + | blt >3 + | // Check frame below fast function. + | lwz TMP1, FRAME_PC(BASE) + | andix. TMP0, TMP1, FRAME_TYPE + | bney >3 // Trace stitching continuation? + | // Otherwise set KBASE for Lua function below fast function. + | lwz TMP2, -4(TMP1) + | decode_RA8 TMP0, TMP2 + | sub TMP1, BASE, TMP0 + | lwz LFUNC:TMP2, -12(TMP1) + | lwz TMP1, LFUNC:TMP2->pc + | lwz KBASE, PC2PROTO(k)(TMP1) + |3: | subi RC, MULTRES, 8 | add RA, RA, BASE | bctr | - |3: // Rethrow error from the right C frame. + |9: // Rethrow error from the right C frame. | neg CARG2, CARG1 | mr CARG1, L | bl extern lj_err_throw // (lua_State *L, int errcode) diff --git a/src/vm_x86.dasc b/src/vm_x86.dasc index c2f03d80..451c246b 100644 --- a/src/vm_x86.dasc +++ b/src/vm_x86.dasc @@ -2697,6 +2697,8 @@ static void build_subroutines(BuildCtx *ctx) | mov RB, TMP1 // Get previous trace number. | mov RA, [DISPATCH+DISPATCH_J(trace)] | mov TRACE:RD, [RA+RB*4] + | test TRACE:RD, TRACE:RD + | jz ->cont_nop | movzx RD, word TRACE:RD->link | cmp RD, RB | je ->cont_nop // Blacklisted. @@ -2844,7 +2846,7 @@ static void build_subroutines(BuildCtx *ctx) | mov r13, TMPa | mov r12, TMPQ |.endif - | test RD, RD; js >3 // Check for error from exit. + | test RD, RD; js >9 // Check for error from exit. | mov L:RB, SAVE_L | mov MULTRES, RD | mov LFUNC:KBASE, [BASE-8] @@ -2860,16 +2862,31 @@ static void build_subroutines(BuildCtx *ctx) | add PC, 4 | shr RC, 16 | cmp OP, BC_FUNCF // Function header? - | jb >2 - | mov RC, MULTRES // RC/RD holds nres+1. + | jb >3 + | cmp OP, BC_FUNCC+2 // Fast function? + | jae >4 |2: + | mov RC, MULTRES // RC/RD holds nres+1. + |3: |.if X64 | jmp aword [DISPATCH+OP*8] |.else | jmp aword [DISPATCH+OP*4] |.endif | - |3: // Rethrow error from the right C frame. + |4: // Check frame below fast function. + | mov RC, [BASE-4] + | test RC, FRAME_TYPE + | jnz <2 // Trace stitching continuation? + | // Otherwise set KBASE for Lua function below fast function. + | movzx RC, byte [RC-3] + | not RCa + | mov LFUNC:KBASE, [BASE+RC*8-8] + | mov KBASE, LFUNC:KBASE->pc + | mov KBASE, [KBASE+PC2PROTO(k)] + | jmp <2 + | + |9: // Rethrow error from the right C frame. | neg RD | mov FCARG1, L:RB | mov FCARG2, RD