From 82eca898db87bde10fbbb14a0f35ef75b6c3dcc6 Mon Sep 17 00:00:00 2001
From: Mike Pall <mike>
Date: Wed, 29 Jun 2011 01:51:39 +0200
Subject: [PATCH] Prefer recording loops over calls.

Hotcounts are decremented by 2 for loops and by 1 for calls.
---
 src/buildvm_arm.dasc |  12 ++--
 src/buildvm_arm.h    | 128 ++++++++++++++++++++++---------------------
 src/buildvm_x64.h    |  60 ++++++++++----------
 src/buildvm_x64win.h |  40 +++++++-------
 src/buildvm_x86.dasc |   8 +--
 src/buildvm_x86.h    |  30 +++++-----
 src/lj_dispatch.c    |   3 +-
 src/lj_dispatch.h    |   4 ++
 src/lj_jit.h         |   2 +-
 src/lj_trace.c       |   5 +-
 10 files changed, 151 insertions(+), 141 deletions(-)

diff --git a/src/buildvm_arm.dasc b/src/buildvm_arm.dasc
index fd7c7b81..9b6b5aed 100644
--- a/src/buildvm_arm.dasc
+++ b/src/buildvm_arm.dasc
@@ -185,23 +185,23 @@
 |
 #define PC2PROTO(field)  ((int)offsetof(GCproto, field)-(int)sizeof(GCproto))
 |
-|.macro hotcheck
+|.macro hotcheck, delta
 |  lsr CARG1, PC, #1
 |  and CARG1, CARG1, #126
 |  sub CARG1, CARG1, #-GG_DISP2HOT
 |  ldrh CARG2, [DISPATCH, CARG1]
-|  subs CARG2, CARG2, #1
+|  subs CARG2, CARG2, #delta
 |  strh CARG2, [DISPATCH, CARG1]
 |.endmacro
 |
 |.macro hotloop
-|  hotcheck
-|  beq ->vm_hotloop
+|  hotcheck HOTCOUNT_LOOP
+|  blo ->vm_hotloop
 |.endmacro
 |
 |.macro hotcall
-|  hotcheck
-|  beq ->vm_hotcall
+|  hotcheck HOTCOUNT_CALL
+|  blo ->vm_hotcall
 |.endmacro
 |
 |// Set current VM state.
diff --git a/src/buildvm_arm.h b/src/buildvm_arm.h
index 505048bd..b450f5df 100644
--- a/src/buildvm_arm.h
+++ b/src/buildvm_arm.h
@@ -12,7 +12,7 @@
 #define DASM_SECTION_CODE_OP	0
 #define DASM_SECTION_CODE_SUB	1
 #define DASM_MAXSECTION		2
-static const unsigned int build_actionlist[5702] = {
+static const unsigned int build_actionlist[5706] = {
 0x00010001,
 0x00060014,
 0xe3160000,
@@ -5381,9 +5381,10 @@ static const unsigned int build_actionlist[5702] = {
 0xe2400000,
 0x000a0000,
 0xe19710b0,
-0xe2511001,
+0xe2511000,
+0x000a0000,
 0xe18710b0,
-0x0a000000,
+0x3a000000,
 0x00050091,
 0x00000000,
 0xe1aa00d9,
@@ -5519,9 +5520,10 @@ static const unsigned int build_actionlist[5702] = {
 0xe2400000,
 0x000a0000,
 0xe19710b0,
-0xe2511001,
+0xe2511000,
+0x000a0000,
 0xe18710b0,
-0x0a000000,
+0x3a000000,
 0x00050091,
 0x00000000,
 0xe1aa00d9,
@@ -5550,9 +5552,10 @@ static const unsigned int build_actionlist[5702] = {
 0xe2400000,
 0x000a0000,
 0xe19710b0,
-0xe2511001,
+0xe2511000,
+0x000a0000,
 0xe18710b0,
-0x0a000000,
+0x3a000000,
 0x00050091,
 0x00000000,
 0xe5d6c000,
@@ -5590,9 +5593,10 @@ static const unsigned int build_actionlist[5702] = {
 0xe2400000,
 0x000a0000,
 0xe19710b0,
-0xe2511001,
+0xe2511000,
+0x000a0000,
 0xe18710b0,
-0x0a000000,
+0x3a000000,
 0x00050093,
 0x00000000,
 0xe5180000,
@@ -7138,7 +7142,7 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
 
   case BC_FORL:
 #if LJ_HASJIT
-    dasm_put(Dst, 5363, -GG_DISP2HOT);
+    dasm_put(Dst, 5363, -GG_DISP2HOT, HOTCOUNT_LOOP);
 #endif
     break;
 
@@ -7150,68 +7154,68 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
   case BC_FORI:
   case BC_IFORL:
     vk = (op == BC_IFORL || op == BC_JFORL);
-    dasm_put(Dst, 5373);
+    dasm_put(Dst, 5374);
     if (op != BC_JFORL) {
-      dasm_put(Dst, 5375);
+      dasm_put(Dst, 5376);
     }
     if (!vk) {
-      dasm_put(Dst, 5377, -LJ_TISNUM, -LJ_TISNUM, -LJ_TISNUM);
+      dasm_put(Dst, 5378, -LJ_TISNUM, -LJ_TISNUM, -LJ_TISNUM);
     } else {
-      dasm_put(Dst, 5395, -LJ_TISNUM);
+      dasm_put(Dst, 5396, -LJ_TISNUM);
       if (op == BC_IFORL) {
-	dasm_put(Dst, 5403);
+	dasm_put(Dst, 5404);
       } else {
-	dasm_put(Dst, 5405);
+	dasm_put(Dst, 5406);
       }
-      dasm_put(Dst, 5408);
+      dasm_put(Dst, 5409);
     }
-    dasm_put(Dst, 5413);
+    dasm_put(Dst, 5414);
     if (op == BC_FORI) {
-      dasm_put(Dst, 5415);
+      dasm_put(Dst, 5416);
     } else if (op == BC_JFORI) {
-      dasm_put(Dst, 5417);
+      dasm_put(Dst, 5418);
     } else if (op == BC_IFORL) {
-      dasm_put(Dst, 5420);
+      dasm_put(Dst, 5421);
     }
     if (vk) {
-      dasm_put(Dst, 5422);
+      dasm_put(Dst, 5423);
     }
-    dasm_put(Dst, 5424);
+    dasm_put(Dst, 5425);
     if (op == BC_JFORI || op == BC_JFORL) {
-      dasm_put(Dst, 5429, BC_JLOOP);
+      dasm_put(Dst, 5430, BC_JLOOP);
     }
-    dasm_put(Dst, 5432);
+    dasm_put(Dst, 5433);
     if (!vk) {
-      dasm_put(Dst, 5439);
+      dasm_put(Dst, 5440);
     } else {
-      dasm_put(Dst, 5441);
+      dasm_put(Dst, 5442);
     }
-    dasm_put(Dst, 5443);
+    dasm_put(Dst, 5444);
     if (!vk) {
-      dasm_put(Dst, 5447, -LJ_TISNUM, -LJ_TISNUM);
+      dasm_put(Dst, 5448, -LJ_TISNUM, -LJ_TISNUM);
     } else {
-      dasm_put(Dst, 5459);
+      dasm_put(Dst, 5460);
     }
-    dasm_put(Dst, 5468);
+    dasm_put(Dst, 5469);
     if (op == BC_FORI) {
-      dasm_put(Dst, 5472);
+      dasm_put(Dst, 5473);
     } else if (op == BC_JFORI) {
-      dasm_put(Dst, 5474, BC_JLOOP);
+      dasm_put(Dst, 5475, BC_JLOOP);
     } else if (op == BC_IFORL) {
-      dasm_put(Dst, 5479);
+      dasm_put(Dst, 5480);
     } else {
-      dasm_put(Dst, 5481, BC_JLOOP);
+      dasm_put(Dst, 5482, BC_JLOOP);
     }
-    dasm_put(Dst, 5484);
+    dasm_put(Dst, 5485);
     if (vk) {
-      dasm_put(Dst, 5490);
+      dasm_put(Dst, 5491);
     }
-    dasm_put(Dst, 5495);
+    dasm_put(Dst, 5496);
     break;
 
   case BC_ITERL:
 #if LJ_HASJIT
-    dasm_put(Dst, 5501, -GG_DISP2HOT);
+    dasm_put(Dst, 5502, -GG_DISP2HOT, HOTCOUNT_LOOP);
 #endif
     break;
 
@@ -7220,40 +7224,40 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
     break;
 #endif
   case BC_IITERL:
-    dasm_put(Dst, 5511);
+    dasm_put(Dst, 5513);
     if (op == BC_JITERL) {
-      dasm_put(Dst, 5513, -LJ_TNIL, BC_JLOOP);
+      dasm_put(Dst, 5515, -LJ_TNIL, BC_JLOOP);
     } else {
-      dasm_put(Dst, 5519, -LJ_TNIL);
+      dasm_put(Dst, 5521, -LJ_TNIL);
     }
-    dasm_put(Dst, 5525);
+    dasm_put(Dst, 5527);
     break;
 
   case BC_LOOP:
 #if LJ_HASJIT
-    dasm_put(Dst, 5532, -GG_DISP2HOT);
+    dasm_put(Dst, 5534, -GG_DISP2HOT, HOTCOUNT_LOOP);
 #endif
     break;
 
   case BC_ILOOP:
-    dasm_put(Dst, 5542);
+    dasm_put(Dst, 5545);
     break;
 
   case BC_JLOOP:
 #if LJ_HASJIT
-    dasm_put(Dst, 5549, DISPATCH_J(trace), DISPATCH_GL(vmstate), DtD(->mcode), DISPATCH_GL(jit_base), DISPATCH_GL(jit_L));
+    dasm_put(Dst, 5552, DISPATCH_J(trace), DISPATCH_GL(vmstate), DtD(->mcode), DISPATCH_GL(jit_base), DISPATCH_GL(jit_L));
 #endif
     break;
 
   case BC_JMP:
-    dasm_put(Dst, 5563);
+    dasm_put(Dst, 5566);
     break;
 
   /* -- Function headers -------------------------------------------------- */
 
   case BC_FUNCF:
 #if LJ_HASJIT
-    dasm_put(Dst, 5572, -GG_DISP2HOT);
+    dasm_put(Dst, 5575, -GG_DISP2HOT, HOTCOUNT_CALL);
 #endif
   case BC_FUNCV:  /* NYI: compiled vararg functions. */
     break;
@@ -7263,42 +7267,42 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
     break;
 #endif
   case BC_IFUNCF:
-    dasm_put(Dst, 5582, Dt1(->maxstack), -4+PC2PROTO(numparams), -4+PC2PROTO(k));
+    dasm_put(Dst, 5586, Dt1(->maxstack), -4+PC2PROTO(numparams), -4+PC2PROTO(k));
     if (op != BC_JFUNCF) {
-      dasm_put(Dst, 5592);
+      dasm_put(Dst, 5596);
     }
-    dasm_put(Dst, 5595, ~LJ_TNIL);
+    dasm_put(Dst, 5599, ~LJ_TNIL);
     if (op == BC_JFUNCF) {
-      dasm_put(Dst, 5602, BC_JLOOP);
+      dasm_put(Dst, 5606, BC_JLOOP);
     } else {
-      dasm_put(Dst, 5606);
+      dasm_put(Dst, 5610);
     }
-    dasm_put(Dst, 5611);
+    dasm_put(Dst, 5615);
     break;
 
   case BC_JFUNCV:
 #if !LJ_HASJIT
     break;
 #endif
-    dasm_put(Dst, 5617);
+    dasm_put(Dst, 5621);
     break;  /* NYI: compiled vararg functions. */
 
   case BC_IFUNCV:
-    dasm_put(Dst, 5619, Dt1(->maxstack), 8+FRAME_VARG, -4+PC2PROTO(k), -4+PC2PROTO(numparams), ~LJ_TNIL);
+    dasm_put(Dst, 5623, Dt1(->maxstack), 8+FRAME_VARG, -4+PC2PROTO(k), -4+PC2PROTO(numparams), ~LJ_TNIL);
     break;
 
   case BC_FUNCC:
   case BC_FUNCCW:
     if (op == BC_FUNCC) {
-      dasm_put(Dst, 5660, Dt8(->f));
+      dasm_put(Dst, 5664, Dt8(->f));
     } else {
-      dasm_put(Dst, 5663, DISPATCH_GL(wrapf));
+      dasm_put(Dst, 5667, DISPATCH_GL(wrapf));
     }
-    dasm_put(Dst, 5666, Dt1(->maxstack), Dt1(->base), Dt1(->top));
+    dasm_put(Dst, 5670, Dt1(->maxstack), Dt1(->base), Dt1(->top));
     if (op == BC_FUNCCW) {
-      dasm_put(Dst, 5676, Dt8(->f));
+      dasm_put(Dst, 5680, Dt8(->f));
     }
-    dasm_put(Dst, 5679, LJ_VMST_C, DISPATCH_GL(vmstate), Dt1(->base), LJ_VMST_INTERP, Dt1(->top), DISPATCH_GL(vmstate));
+    dasm_put(Dst, 5683, LJ_VMST_C, DISPATCH_GL(vmstate), Dt1(->base), LJ_VMST_INTERP, Dt1(->top), DISPATCH_GL(vmstate));
     break;
 
   /* ---------------------------------------------------------------------- */
@@ -7318,7 +7322,7 @@ static int build_backend(BuildCtx *ctx)
 
   build_subroutines(ctx);
 
-  dasm_put(Dst, 5701);
+  dasm_put(Dst, 5705);
   for (op = 0; op < BC__MAX; op++)
     build_ins(ctx, (BCOp)op, op);
 
diff --git a/src/buildvm_x64.h b/src/buildvm_x64.h
index f6ed666c..4e4eca54 100644
--- a/src/buildvm_x64.h
+++ b/src/buildvm_x64.h
@@ -751,16 +751,16 @@ static const unsigned char build_actionlist[16156] = {
   182,232,131,195,4,193,232,16,65,252,255,36,252,238,248,6,255,65,199,71,252,
   252,237,65,131,199,8,255,199,68,194,252,244,237,255,131,192,1,252,233,244,
   5,248,7,141,171,233,252,247,197,237,15,133,244,14,41,252,234,255,1,252,233,
-  255,137,221,209,252,237,129,229,239,102,65,131,172,253,46,233,1,15,132,244,
-  148,255,141,12,202,255,129,121,253,4,239,15,133,244,255,255,129,121,253,12,
-  239,15,133,244,60,129,121,253,20,239,15,133,244,60,139,41,131,121,16,0,15,
-  140,244,251,255,129,121,253,12,239,15,133,244,165,129,121,253,20,239,15,133,
-  244,165,255,139,105,16,133,252,237,15,136,244,251,3,41,15,128,244,247,137,
-  41,255,59,105,8,199,65,28,237,137,105,24,255,15,142,244,253,248,1,248,6,141,
-  156,253,131,233,255,141,156,253,131,233,15,183,67,252,254,15,142,245,248,
-  1,248,6,255,15,143,244,253,248,6,141,156,253,131,233,248,1,255,248,7,139,
-  3,15,182,204,15,182,232,131,195,4,193,232,16,65,252,255,36,252,238,248,5,
-  255,3,41,15,128,244,1,137,41,255,15,141,244,7,255,141,156,253,131,233,15,
+  255,137,221,209,252,237,129,229,239,102,65,129,172,253,46,233,238,15,130,
+  244,148,255,141,12,202,255,129,121,253,4,239,15,133,244,255,255,129,121,253,
+  12,239,15,133,244,60,129,121,253,20,239,15,133,244,60,139,41,131,121,16,0,
+  15,140,244,251,255,129,121,253,12,239,15,133,244,165,129,121,253,20,239,15,
+  133,244,165,255,139,105,16,133,252,237,15,136,244,251,3,41,15,128,244,247,
+  137,41,255,59,105,8,199,65,28,237,137,105,24,255,15,142,244,253,248,1,248,
+  6,141,156,253,131,233,255,141,156,253,131,233,15,183,67,252,254,15,142,245,
+  248,1,248,6,255,15,143,244,253,248,6,141,156,253,131,233,248,1,255,248,7,
+  139,3,15,182,204,15,182,232,131,195,4,193,232,16,65,252,255,36,252,238,248,
+  5,255,3,41,15,128,244,1,137,41,255,15,141,244,7,255,141,156,253,131,233,15,
   183,67,252,254,15,141,245,255,15,140,244,7,255,252,233,244,6,248,9,255,129,
   121,253,4,239,255,15,131,244,60,129,121,253,12,239,15,131,244,60,255,129,
   121,253,12,239,15,131,244,165,129,121,253,20,239,15,131,244,165,255,139,105,
@@ -777,22 +777,22 @@ static const unsigned char build_actionlist[16156] = {
   139,4,129,72,139,128,233,139,108,36,24,65,137,150,233,65,137,174,233,76,137,
   36,36,76,137,108,36,8,72,131,252,236,16,252,255,224,255,141,156,253,131,233,
   139,3,15,182,204,15,182,232,131,195,4,193,232,16,65,252,255,36,252,238,255,
-  137,221,209,252,237,129,229,239,102,65,131,172,253,46,233,1,15,132,244,150,
-  255,68,139,187,233,139,108,36,24,141,12,202,59,141,233,15,135,244,24,15,182,
-  139,233,57,200,15,134,244,249,248,2,255,15,183,67,252,254,252,233,245,255,
-  248,3,199,68,194,252,252,237,131,192,1,57,200,15,134,244,3,252,233,244,2,
-  255,141,44,197,237,141,4,194,68,139,122,252,248,137,104,252,252,68,137,120,
-  252,248,139,108,36,24,141,12,200,59,141,233,15,135,244,23,137,209,137,194,
-  15,182,171,233,133,252,237,15,132,244,248,248,1,131,193,8,57,209,15,131,244,
-  249,68,139,121,252,248,68,137,56,68,139,121,252,252,68,137,120,4,131,192,
-  8,199,65,252,252,237,131,252,237,1,15,133,244,1,248,2,255,68,139,187,233,
-  139,3,15,182,204,15,182,232,131,195,4,193,232,16,65,252,255,36,252,238,255,
-  248,3,199,64,4,237,131,192,8,131,252,237,1,15,133,244,3,252,233,244,2,255,
-  139,106,252,248,76,139,189,233,139,108,36,24,141,68,194,252,248,137,149,233,
-  141,136,233,59,141,233,137,133,233,255,76,137,252,254,137,252,239,255,15,
-  135,244,22,65,199,134,233,237,255,65,252,255,215,255,65,252,255,150,233,255,
-  65,199,134,233,237,139,149,233,141,12,194,252,247,217,3,141,233,139,90,252,
-  252,252,233,244,12,255,254,0
+  137,221,209,252,237,129,229,239,102,65,129,172,253,46,233,238,15,130,244,
+  150,255,68,139,187,233,139,108,36,24,141,12,202,59,141,233,15,135,244,24,
+  15,182,139,233,57,200,15,134,244,249,248,2,255,15,183,67,252,254,252,233,
+  245,255,248,3,199,68,194,252,252,237,131,192,1,57,200,15,134,244,3,252,233,
+  244,2,255,141,44,197,237,141,4,194,68,139,122,252,248,137,104,252,252,68,
+  137,120,252,248,139,108,36,24,141,12,200,59,141,233,15,135,244,23,137,209,
+  137,194,15,182,171,233,133,252,237,15,132,244,248,248,1,131,193,8,57,209,
+  15,131,244,249,68,139,121,252,248,68,137,56,68,139,121,252,252,68,137,120,
+  4,131,192,8,199,65,252,252,237,131,252,237,1,15,133,244,1,248,2,255,68,139,
+  187,233,139,3,15,182,204,15,182,232,131,195,4,193,232,16,65,252,255,36,252,
+  238,255,248,3,199,64,4,237,131,192,8,131,252,237,1,15,133,244,3,252,233,244,
+  2,255,139,106,252,248,76,139,189,233,139,108,36,24,141,68,194,252,248,137,
+  149,233,141,136,233,59,141,233,137,133,233,255,76,137,252,254,137,252,239,
+  255,15,135,244,22,65,199,134,233,237,255,65,252,255,215,255,65,252,255,150,
+  233,255,65,199,134,233,237,139,149,233,141,12,194,252,247,217,3,141,233,139,
+  90,252,252,252,233,244,12,255,254,0
 };
 
 enum {
@@ -2777,7 +2777,7 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop, int cmov, int sse)
 
   case BC_FORL:
 #if LJ_HASJIT
-    dasm_put(Dst, 15309, HOTCOUNT_PCMASK, GG_DISP2HOT);
+    dasm_put(Dst, 15309, HOTCOUNT_PCMASK, GG_DISP2HOT, HOTCOUNT_LOOP);
 #endif
     break;
 
@@ -2893,7 +2893,7 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop, int cmov, int sse)
 
   case BC_ITERL:
 #if LJ_HASJIT
-    dasm_put(Dst, 15309, HOTCOUNT_PCMASK, GG_DISP2HOT);
+    dasm_put(Dst, 15309, HOTCOUNT_PCMASK, GG_DISP2HOT, HOTCOUNT_LOOP);
 #endif
     break;
 
@@ -2913,7 +2913,7 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop, int cmov, int sse)
 
   case BC_LOOP:
 #if LJ_HASJIT
-    dasm_put(Dst, 15309, HOTCOUNT_PCMASK, GG_DISP2HOT);
+    dasm_put(Dst, 15309, HOTCOUNT_PCMASK, GG_DISP2HOT, HOTCOUNT_LOOP);
 #endif
     break;
 
@@ -2942,7 +2942,7 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop, int cmov, int sse)
 
   case BC_FUNCF:
 #if LJ_HASJIT
-    dasm_put(Dst, 15842, HOTCOUNT_PCMASK, GG_DISP2HOT);
+    dasm_put(Dst, 15842, HOTCOUNT_PCMASK, GG_DISP2HOT, HOTCOUNT_CALL);
 #endif
   case BC_FUNCV:  /* NYI: compiled vararg functions. */
     break;
diff --git a/src/buildvm_x64win.h b/src/buildvm_x64win.h
index bb813284..0e1b5b46 100644
--- a/src/buildvm_x64win.h
+++ b/src/buildvm_x64win.h
@@ -746,7 +746,7 @@ static const unsigned char build_actionlist[16011] = {
   36,252,235,248,6,255,199,71,252,252,237,131,199,8,255,199,68,194,252,244,
   237,255,131,192,1,252,233,244,5,248,7,141,174,233,252,247,197,237,15,133,
   244,14,41,252,234,255,1,252,233,255,137,252,245,209,252,237,129,229,239,102,
-  131,172,253,43,233,1,15,132,244,148,255,141,12,202,255,129,121,253,4,239,
+  129,172,253,43,233,238,15,130,244,148,255,141,12,202,255,129,121,253,4,239,
   15,133,244,255,255,129,121,253,12,239,15,133,244,60,129,121,253,20,239,15,
   133,244,60,139,41,131,121,16,0,15,140,244,251,255,129,121,253,12,239,15,133,
   244,165,129,121,253,20,239,15,133,244,165,255,139,105,16,133,252,237,15,136,
@@ -775,21 +775,21 @@ static const unsigned char build_actionlist[16011] = {
   15,127,153,233,102,68,15,127,161,233,102,68,15,127,169,233,102,68,15,127,
   177,233,102,68,15,127,185,233,252,255,224,255,141,180,253,134,233,139,6,15,
   182,204,15,182,232,131,198,4,193,232,16,252,255,36,252,235,255,137,252,245,
-  209,252,237,129,229,239,102,131,172,253,43,233,1,15,132,244,150,255,139,190,
-  233,139,108,36,96,141,12,202,59,141,233,15,135,244,24,15,182,142,233,57,200,
-  15,134,244,249,248,2,255,15,183,70,252,254,252,233,245,255,248,3,199,68,194,
-  252,252,237,131,192,1,57,200,15,134,244,3,252,233,244,2,255,141,44,197,237,
-  141,4,194,139,122,252,248,137,104,252,252,137,120,252,248,139,108,36,96,141,
-  12,200,59,141,233,15,135,244,23,137,209,137,194,15,182,174,233,133,252,237,
-  15,132,244,248,248,1,131,193,8,57,209,15,131,244,249,139,121,252,248,137,
-  56,139,121,252,252,137,120,4,131,192,8,199,65,252,252,237,131,252,237,1,15,
-  133,244,1,248,2,255,139,190,233,139,6,15,182,204,15,182,232,131,198,4,193,
-  232,16,252,255,36,252,235,255,248,3,199,64,4,237,131,192,8,131,252,237,1,
-  15,133,244,3,252,233,244,2,255,139,106,252,248,72,139,189,233,139,108,36,
-  96,141,68,194,252,248,137,149,233,141,136,233,59,141,233,137,133,233,255,
-  72,137,252,250,137,252,233,255,15,135,244,22,199,131,233,237,255,252,255,
-  215,255,252,255,147,233,255,199,131,233,237,139,149,233,141,12,194,252,247,
-  217,3,141,233,139,114,252,252,252,233,244,12,255,254,0
+  209,252,237,129,229,239,102,129,172,253,43,233,238,15,130,244,150,255,139,
+  190,233,139,108,36,96,141,12,202,59,141,233,15,135,244,24,15,182,142,233,
+  57,200,15,134,244,249,248,2,255,15,183,70,252,254,252,233,245,255,248,3,199,
+  68,194,252,252,237,131,192,1,57,200,15,134,244,3,252,233,244,2,255,141,44,
+  197,237,141,4,194,139,122,252,248,137,104,252,252,137,120,252,248,139,108,
+  36,96,141,12,200,59,141,233,15,135,244,23,137,209,137,194,15,182,174,233,
+  133,252,237,15,132,244,248,248,1,131,193,8,57,209,15,131,244,249,139,121,
+  252,248,137,56,139,121,252,252,137,120,4,131,192,8,199,65,252,252,237,131,
+  252,237,1,15,133,244,1,248,2,255,139,190,233,139,6,15,182,204,15,182,232,
+  131,198,4,193,232,16,252,255,36,252,235,255,248,3,199,64,4,237,131,192,8,
+  131,252,237,1,15,133,244,3,252,233,244,2,255,139,106,252,248,72,139,189,233,
+  139,108,36,96,141,68,194,252,248,137,149,233,141,136,233,59,141,233,137,133,
+  233,255,72,137,252,250,137,252,233,255,15,135,244,22,199,131,233,237,255,
+  252,255,215,255,252,255,147,233,255,199,131,233,237,139,149,233,141,12,194,
+  252,247,217,3,141,233,139,114,252,252,252,233,244,12,255,254,0
 };
 
 enum {
@@ -2775,7 +2775,7 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop, int cmov, int sse)
 
   case BC_FORL:
 #if LJ_HASJIT
-    dasm_put(Dst, 15111, HOTCOUNT_PCMASK, GG_DISP2HOT);
+    dasm_put(Dst, 15111, HOTCOUNT_PCMASK, GG_DISP2HOT, HOTCOUNT_LOOP);
 #endif
     break;
 
@@ -2891,7 +2891,7 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop, int cmov, int sse)
 
   case BC_ITERL:
 #if LJ_HASJIT
-    dasm_put(Dst, 15111, HOTCOUNT_PCMASK, GG_DISP2HOT);
+    dasm_put(Dst, 15111, HOTCOUNT_PCMASK, GG_DISP2HOT, HOTCOUNT_LOOP);
 #endif
     break;
 
@@ -2911,7 +2911,7 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop, int cmov, int sse)
 
   case BC_LOOP:
 #if LJ_HASJIT
-    dasm_put(Dst, 15111, HOTCOUNT_PCMASK, GG_DISP2HOT);
+    dasm_put(Dst, 15111, HOTCOUNT_PCMASK, GG_DISP2HOT, HOTCOUNT_LOOP);
 #endif
     break;
 
@@ -2940,7 +2940,7 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop, int cmov, int sse)
 
   case BC_FUNCF:
 #if LJ_HASJIT
-    dasm_put(Dst, 15710, HOTCOUNT_PCMASK, GG_DISP2HOT);
+    dasm_put(Dst, 15710, HOTCOUNT_PCMASK, GG_DISP2HOT, HOTCOUNT_CALL);
 #endif
   case BC_FUNCV:  /* NYI: compiled vararg functions. */
     break;
diff --git a/src/buildvm_x86.dasc b/src/buildvm_x86.dasc
index effd0737..21e18ea8 100644
--- a/src/buildvm_x86.dasc
+++ b/src/buildvm_x86.dasc
@@ -333,16 +333,16 @@
 |  mov reg, PC
 |  shr reg, 1
 |  and reg, HOTCOUNT_PCMASK
-|  sub word [DISPATCH+reg+GG_DISP2HOT], 1
-|  jz ->vm_hotloop
+|  sub word [DISPATCH+reg+GG_DISP2HOT], HOTCOUNT_LOOP
+|  jb ->vm_hotloop
 |.endmacro
 |
 |.macro hotcall, reg
 |  mov reg, PC
 |  shr reg, 1
 |  and reg, HOTCOUNT_PCMASK
-|  sub word [DISPATCH+reg+GG_DISP2HOT], 1
-|  jz ->vm_hotcall
+|  sub word [DISPATCH+reg+GG_DISP2HOT], HOTCOUNT_CALL
+|  jb ->vm_hotcall
 |.endmacro
 |
 |// Set current VM state.
diff --git a/src/buildvm_x86.h b/src/buildvm_x86.h
index 7adb2637..a9b74642 100644
--- a/src/buildvm_x86.h
+++ b/src/buildvm_x86.h
@@ -798,12 +798,12 @@ static const unsigned char build_actionlist[17111] = {
   204,15,182,232,131,198,4,193,232,16,252,255,36,171,248,6,255,199,71,252,252,
   237,131,199,8,255,199,68,194,252,244,237,255,131,192,1,252,233,244,5,248,
   7,141,174,233,252,247,197,237,15,133,244,14,41,252,234,255,1,252,233,255,
-  137,252,245,209,252,237,129,229,239,102,131,172,253,43,233,1,15,132,244,148,
-  255,141,12,202,255,129,121,253,4,239,15,133,244,255,255,129,121,253,12,239,
-  15,133,244,60,129,121,253,20,239,15,133,244,60,139,41,131,121,16,0,15,140,
-  244,251,255,129,121,253,12,239,15,133,244,165,129,121,253,20,239,15,133,244,
-  165,255,139,105,16,133,252,237,15,136,244,251,3,41,15,128,244,247,137,41,
-  255,59,105,8,199,65,28,237,137,105,24,255,15,142,244,253,248,1,248,6,141,
+  137,252,245,209,252,237,129,229,239,102,129,172,253,43,233,238,15,130,244,
+  148,255,141,12,202,255,129,121,253,4,239,15,133,244,255,255,129,121,253,12,
+  239,15,133,244,60,129,121,253,20,239,15,133,244,60,139,41,131,121,16,0,15,
+  140,244,251,255,129,121,253,12,239,15,133,244,165,129,121,253,20,239,15,133,
+  244,165,255,139,105,16,133,252,237,15,136,244,251,3,41,15,128,244,247,137,
+  41,255,59,105,8,199,65,28,237,137,105,24,255,15,142,244,253,248,1,248,6,141,
   180,253,134,233,255,141,180,253,134,233,15,183,70,252,254,15,142,245,248,
   1,248,6,255,15,143,244,253,248,6,141,180,253,134,233,248,1,255,248,7,139,
   6,15,182,204,15,182,232,131,198,4,193,232,16,252,255,36,171,248,5,255,3,41,
@@ -823,11 +823,11 @@ static const unsigned char build_actionlist[17111] = {
   134,233,139,1,137,105,252,252,137,65,252,248,255,139,139,233,139,4,129,139,
   128,233,139,108,36,48,137,147,233,137,171,233,252,255,224,255,141,180,253,
   134,233,139,6,15,182,204,15,182,232,131,198,4,193,232,16,252,255,36,171,255,
-  137,252,245,209,252,237,129,229,239,102,131,172,253,43,233,1,15,132,244,150,
-  255,139,190,233,139,108,36,48,141,12,202,59,141,233,15,135,244,24,15,182,
-  142,233,57,200,15,134,244,249,248,2,255,15,183,70,252,254,252,233,245,255,
-  248,3,199,68,194,252,252,237,131,192,1,57,200,15,134,244,3,252,233,244,2,
-  255,141,44,197,237,141,4,194,139,122,252,248,137,104,252,252,137,120,252,
+  137,252,245,209,252,237,129,229,239,102,129,172,253,43,233,238,15,130,244,
+  150,255,139,190,233,139,108,36,48,141,12,202,59,141,233,15,135,244,24,15,
+  182,142,233,57,200,15,134,244,249,248,2,255,15,183,70,252,254,252,233,245,
+  255,248,3,199,68,194,252,252,237,131,192,1,57,200,15,134,244,3,252,233,244,
+  2,255,141,44,197,237,141,4,194,139,122,252,248,137,104,252,252,137,120,252,
   248,139,108,36,48,141,12,200,59,141,233,15,135,244,23,137,209,137,194,15,
   182,174,233,133,252,237,15,132,244,248,248,1,131,193,8,57,209,15,131,244,
   249,139,121,252,248,137,56,139,121,252,252,137,120,4,131,192,8,199,65,252,
@@ -2933,7 +2933,7 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop, int cmov, int sse)
 
   case BC_FORL:
 #if LJ_HASJIT
-    dasm_put(Dst, 16297, HOTCOUNT_PCMASK, GG_DISP2HOT);
+    dasm_put(Dst, 16297, HOTCOUNT_PCMASK, GG_DISP2HOT, HOTCOUNT_LOOP);
 #endif
     break;
 
@@ -3049,7 +3049,7 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop, int cmov, int sse)
 
   case BC_ITERL:
 #if LJ_HASJIT
-    dasm_put(Dst, 16297, HOTCOUNT_PCMASK, GG_DISP2HOT);
+    dasm_put(Dst, 16297, HOTCOUNT_PCMASK, GG_DISP2HOT, HOTCOUNT_LOOP);
 #endif
     break;
 
@@ -3069,7 +3069,7 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop, int cmov, int sse)
 
   case BC_LOOP:
 #if LJ_HASJIT
-    dasm_put(Dst, 16297, HOTCOUNT_PCMASK, GG_DISP2HOT);
+    dasm_put(Dst, 16297, HOTCOUNT_PCMASK, GG_DISP2HOT, HOTCOUNT_LOOP);
 #endif
     break;
 
@@ -3098,7 +3098,7 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop, int cmov, int sse)
 
   case BC_FUNCF:
 #if LJ_HASJIT
-    dasm_put(Dst, 16808, HOTCOUNT_PCMASK, GG_DISP2HOT);
+    dasm_put(Dst, 16808, HOTCOUNT_PCMASK, GG_DISP2HOT, HOTCOUNT_CALL);
 #endif
   case BC_FUNCV:  /* NYI: compiled vararg functions. */
     break;
diff --git a/src/lj_dispatch.c b/src/lj_dispatch.c
index c29cad46..b1d17add 100644
--- a/src/lj_dispatch.c
+++ b/src/lj_dispatch.c
@@ -50,7 +50,8 @@ void lj_dispatch_init(GG_State *GG)
 /* Initialize hotcount table. */
 void lj_dispatch_init_hotcount(global_State *g)
 {
-  HotCount start = (HotCount)G2J(g)->param[JIT_P_hotloop];
+  int32_t hotloop = G2J(g)->param[JIT_P_hotloop];
+  HotCount start = (HotCount)(hotloop*HOTCOUNT_LOOP - 1);
   HotCount *hotcount = G2GG(g)->hotcount;
   uint32_t i;
   for (i = 0; i < HOTCOUNT_SIZE; i++)
diff --git a/src/lj_dispatch.h b/src/lj_dispatch.h
index dd4f68fe..c50d33ac 100644
--- a/src/lj_dispatch.h
+++ b/src/lj_dispatch.h
@@ -20,6 +20,10 @@ typedef uint16_t HotCount;
 #define HOTCOUNT_SIZE		64
 #define HOTCOUNT_PCMASK		((HOTCOUNT_SIZE-1)*sizeof(HotCount))
 
+/* Hotcount decrements. */
+#define HOTCOUNT_LOOP		2
+#define HOTCOUNT_CALL		1
+
 /* This solves a circular dependency problem -- bump as needed. Sigh. */
 #define GG_NUM_ASMFF	62
 
diff --git a/src/lj_jit.h b/src/lj_jit.h
index 7e26aadc..1f7ddc21 100644
--- a/src/lj_jit.h
+++ b/src/lj_jit.h
@@ -232,7 +232,7 @@ typedef struct HotPenalty {
 } HotPenalty;
 
 #define PENALTY_SLOTS	64	/* Penalty cache slot. Must be a power of 2. */
-#define PENALTY_MIN	36	/* Minimum penalty value. */
+#define PENALTY_MIN	(36*2)	/* Minimum penalty value. */
 #define PENALTY_MAX	60000	/* Maximum penalty value. */
 #define PENALTY_RNDBITS	4	/* # of random bits to add to penalty value. */
 
diff --git a/src/lj_trace.c b/src/lj_trace.c
index c65ca9cd..33cc98f7 100644
--- a/src/lj_trace.c
+++ b/src/lj_trace.c
@@ -643,9 +643,10 @@ void lj_trace_ins(jit_State *J, const BCIns *pc)
 /* A hotcount triggered. Start recording a root trace. */
 void LJ_FASTCALL lj_trace_hot(jit_State *J, const BCIns *pc)
 {
-  ERRNO_SAVE
   /* Note: pc is the interpreter bytecode PC here. It's offset by 1. */
-  hotcount_set(J2GG(J), pc, J->param[JIT_P_hotloop]+1);  /* Reset hotcount. */
+  ERRNO_SAVE
+  /* Reset hotcount. */
+  hotcount_set(J2GG(J), pc, J->param[JIT_P_hotloop]*HOTCOUNT_LOOP);
   /* Only start a new trace if not recording or inside __gc call or vmevent. */
   if (J->state == LJ_TRACE_IDLE &&
       !(J2G(J)->hookmask & (HOOK_GC|HOOK_VMEVENT))) {