Turn loads from immutable upvalues into constants.

6 files changed, 72 insertions, 24 deletions

diff --git a/src/lj_crecord.c b/src/lj_crecord.c
index 49b2341a..2a475035 100644
--- a/src/lj_crecord.c
+++ b/src/lj_crecord.c
@@ -483,16 +483,8 @@ static void crec_index_meta(jit_State *J, CTState *cts, CType *ct,
   } else if (rd->data == 0 && tvistab(tv) && tref_isstr(J->base[1])) {
     /* Specialize to result of __index lookup. */
     cTValue *o = lj_tab_get(J->L, tabV(tv), &rd->argv[1]);
-    IRType t = itype2irt(o);
-    if (tvisgcv(o))
-      J->base[0] = lj_ir_kgc(J, gcV(o), t);
-    else if (tvisint(o))
-      J->base[0] = lj_ir_kint(J, intV(o));
-    else if (tvisnum(o))
-      J->base[0] = lj_ir_knumint(J, numV(o));
-    else if (tvisbool(o))
-      J->base[0] = TREF_PRI(t);
-    else
+    J->base[0] = lj_record_constify(J, o);
+    if (!J->base[0])
       lj_trace_err(J, LJ_TRERR_BADTYPE);
     /* Always specialize to the key. */
     emitir(IRTG(IR_EQ, IRT_STR), J->base[1], lj_ir_kstr(J, strV(&rd->argv[1])));
diff --git a/src/lj_func.c b/src/lj_func.c
index 0c0b9014..0af53cc4 100644
--- a/src/lj_func.c
+++ b/src/lj_func.c
@@ -163,8 +163,9 @@ GCfunc *lj_func_newL_gc(lua_State *L, GCproto *pt, GCfuncL *parent)
   for (i = 0; i < nuv; i++) {
     uint32_t v = proto_uv(pt)[i];
     GCupval *uv;
-    if ((v & 0x8000)) {
+    if ((v & PROTO_UV_LOCAL)) {
       uv = func_finduv(L, base + (v & 0xff));
+      uv->immutable = ((v / PROTO_UV_IMMUTABLE) & 1);
       uv->dhash = (uint32_t)(uintptr_t)mref(parent->pc, char) ^ (v << 24);
     } else {
       uv = &gcref(puv[v])->uv;
diff --git a/src/lj_obj.h b/src/lj_obj.h
index 137a04b2..7890e54b 100644
--- a/src/lj_obj.h
+++ b/src/lj_obj.h
@@ -321,6 +321,10 @@ typedef struct GCproto {
 /* Top bits used for counting created closures. */
 #define PROTO_CLCOUNT           0x20    /* Base of saturating 3 bit counter. */
 #define PROTO_CLC_BITS          3
+#define PROTO_CLC_POLY          (3*PROTO_CLCOUNT)  /* Polymorphic threshold. */
+
+#define PROTO_UV_LOCAL          0x8000  /* Upvalue for local slot. */
+#define PROTO_UV_IMMUTABLE      0x4000  /* Immutable upvalue. */
 
 #define proto_kgc(pt, idx) \
   check_exp((uintptr_t)(intptr_t)(idx) >= (uintptr_t)-(intptr_t)(pt)->sizekgc, \
@@ -342,7 +346,7 @@ typedef struct GCproto {
 typedef struct GCupval {
   GCHeader;
   uint8_t closed;       /* Set if closed (i.e. uv->v == &uv->u.value). */
-  uint8_t unused;
+  uint8_t immutable;    /* Immutable value. */
   union {
     TValue tv;          /* If closed: the value itself. */
     struct {            /* If open: double linked list, anchored at thread. */
diff --git a/src/lj_parse.c b/src/lj_parse.c
index 2cbfbe56..c5129ad5 100644
--- a/src/lj_parse.c
+++ b/src/lj_parse.c
@@ -39,8 +39,8 @@ typedef enum {
   VKLAST = VKNUM,
   VKCDATA,      /* nval = cdata value, not treated as a constant expression */
   /* Non-constant expressions follow: */
-  VLOCAL,       /* info = local register */
-  VUPVAL,       /* info = upvalue index */
+  VLOCAL,       /* info = local register, aux = vstack index */
+  VUPVAL,       /* info = upvalue index, aux = vstack index */
   VGLOBAL,      /* sval = string value */
   VINDEXED,     /* info = table register, aux = index reg/byte/string const */
   VJMP,         /* info = instruction PC */
@@ -105,6 +105,8 @@ typedef struct FuncScope {
 typedef uint16_t VarIndex;
 #define LJ_MAX_VSTACK           65536
 
+#define VSTACK_VAR_RW           0x80000000      /* In endpc: R/W variable. */
+
 /* Upvalue map. */
 typedef struct UVMap {
   VarIndex vidx;                /* Varinfo index. */
@@ -608,10 +610,12 @@ static void bcemit_store(FuncState *fs, ExpDesc *var, ExpDesc *e)
 {
   BCIns ins;
   if (var->k == VLOCAL) {
+    fs->ls->vstack[var->u.s.aux].endpc |= VSTACK_VAR_RW;
     expr_free(fs, e);
     expr_toreg(fs, e, var->u.s.info);
     return;
   } else if (var->k == VUPVAL) {
+    fs->ls->vstack[var->u.s.aux].endpc |= VSTACK_VAR_RW;
     expr_toval(fs, e);
     if (e->k <= VKTRUE)
       ins = BCINS_AD(BC_USETP, var->u.s.info, const_pri(e));
@@ -1046,8 +1050,11 @@ static void var_add(LexState *ls, BCReg nvars)
 {
   FuncState *fs = ls->fs;
   fs->nactvar = (uint8_t)(fs->nactvar + nvars);
-  for (; nvars; nvars--)
-    var_get(ls, fs, fs->nactvar - nvars).startpc = fs->pc;
+  for (; nvars; nvars--) {
+    VarInfo *v = &var_get(ls, fs, fs->nactvar - nvars);
+    v->startpc = fs->pc;
+    v->endpc = 0;
+  }
 }
 
 /* Remove local variables. */
@@ -1055,7 +1062,7 @@ static void var_remove(LexState *ls, BCReg tolevel)
 {
   FuncState *fs = ls->fs;
   while (fs->nactvar > tolevel)
-    var_get(ls, fs, --fs->nactvar).endpc = fs->pc;
+    var_get(ls, fs, --fs->nactvar).endpc |= fs->pc;
 }
 
 /* Lookup local variable name. */
@@ -1080,7 +1087,8 @@ static MSize var_lookup_uv(FuncState *fs, MSize vidx, ExpDesc *e)
   checklimit(fs, fs->nuv, LJ_MAX_UPVAL, "upvalues");
   lua_assert(e->k == VLOCAL || e->k == VUPVAL);
   fs->uvloc[n].vidx = (uint16_t)vidx;
-  fs->uvloc[n].slot = (uint16_t)(e->u.s.info | (e->k == VLOCAL ? 0x8000 : 0));
+  fs->uvloc[n].slot = (uint16_t)(e->u.s.info |
+                                 (e->k == VLOCAL ? PROTO_UV_LOCAL : 0));
   fs->nuv = n+1;
   return n;
 }
@@ -1097,7 +1105,7 @@ static MSize var_lookup_(FuncState *fs, GCstr *name, ExpDesc *e, int first)
       expr_init(e, VLOCAL, reg);
       if (!first)
         scope_uvmark(fs, reg);  /* Scope now has an upvalue. */
-      return (MSize)fs->varmap[reg];
+      return (MSize)(e->u.s.aux = (uint32_t)fs->varmap[reg]);
     } else {
       MSize vidx = var_lookup_(fs->prev, name, e, 0);  /* Var in outer func? */
       if ((int32_t)vidx >= 0) {  /* Yes, make it an upvalue here. */
@@ -1185,11 +1193,20 @@ static void fs_fixup_k(FuncState *fs, GCproto *pt, void *kptr)
 /* Fixup upvalues for prototype. */
 static void fs_fixup_uv(FuncState *fs, GCproto *pt, uint16_t *uv)
 {
+  VarInfo *vstack;
+  UVMap *uvloc;
   MSize i, n = fs->nuv;
   setmref(pt->uv, uv);
   pt->sizeuv = n;
-  for (i = 0; i < n; i++)
-    uv[i] = fs->uvloc[i].slot;
+  vstack = fs->ls->vstack;
+  uvloc = fs->uvloc;
+  for (i = 0; i < n; i++) {
+    uint16_t slot = uvloc[i].slot;
+    uint16_t vidx = uvloc[i].vidx;
+    if ((slot & PROTO_UV_LOCAL) && !(vstack[vidx].endpc & VSTACK_VAR_RW))
+      slot |= PROTO_UV_IMMUTABLE;
+    uv[i] = slot;
+  }
 }
 
 #ifndef LUAJIT_DISABLE_DEBUGINFO
@@ -1287,7 +1304,8 @@ static size_t fs_prep_var(LexState *ls, FuncState *fs, size_t *ofsvar)
   /* Store local variable names and compressed ranges. */
   for (i = 0, n = ls->vtop - fs->vbase; i < n; i++) {
     GCstr *s = strref(vstack[i].name);
-    BCPos startpc = vstack[i].startpc, endpc = vstack[i].endpc;
+    BCPos startpc = vstack[i].startpc;
+    BCPos endpc = vstack[i].endpc & ~VSTACK_VAR_RW;
     if ((uintptr_t)s < VARNAME__MAX) {
       fs_buf_need(ls, 1 + 2*5);
       ls->sb.buf[ls->sb.n++] = (uint8_t)(uintptr_t)s;
@@ -2180,6 +2198,7 @@ static void parse_local(LexState *ls)
     FuncState *fs = ls->fs;
     var_new(ls, 0, lex_str(ls));
     expr_init(&v, VLOCAL, fs->freereg);
+    v.u.s.aux = fs->varmap[fs->freereg];
     bcreg_reserve(fs, 1);
     var_add(ls, 1);
     parse_body(ls, &b, 0, ls->linenumber);
diff --git a/src/lj_record.c b/src/lj_record.c
index be5c618f..a593af99 100644
--- a/src/lj_record.c
+++ b/src/lj_record.c
@@ -187,6 +187,21 @@ int lj_record_objcmp(jit_State *J, TRef a, TRef b, cTValue *av, cTValue *bv)
   return diff;
 }
 
+/* Constify a value. Returns 0 for non-representable object types. */
+TRef lj_record_constify(jit_State *J, cTValue *o)
+{
+  if (tvisgcv(o))
+    return lj_ir_kgc(J, gcV(o), itype2irt(o));
+  else if (tvisint(o))
+    return lj_ir_kint(J, intV(o));
+  else if (tvisnum(o))
+    return lj_ir_knumint(J, numV(o));
+  else if (tvisbool(o))
+    return TREF_PRI(itype2irt(o));
+  else
+    return 0;  /* Can't represent lightuserdata (pointless). */
+}
+
 /* -- Record loop ops ----------------------------------------------------- */
 
 /* Loop event. */
@@ -569,8 +584,8 @@ static TRef rec_call_specialize(jit_State *J, GCfunc *fn, TRef tr)
   TRef kfunc;
   if (isluafunc(fn)) {
     GCproto *pt = funcproto(fn);
-    /* 3 or more closures created? Probably not a monomorphic function. */
-    if (pt->flags >= 3*PROTO_CLCOUNT) {  /* Specialize to prototype instead. */
+    /* Too many closures created? Probably not a monomorphic function. */
+    if (pt->flags >= PROTO_CLC_POLY) {  /* Specialize to prototype instead. */
       TRef trpt = emitir(IRT(IR_FLOAD, IRT_P32), tr, IRFL_FUNC_PC);
       emitir(IRTG(IR_EQ, IRT_P32), trpt, lj_ir_kptr(J, proto_bc(pt)));
       (void)lj_ir_kgc(J, obj2gco(pt), IRT_PROTO);  /* Prevent GC of proto. */
@@ -1267,6 +1282,22 @@ static TRef rec_upvalue(jit_State *J, uint32_t uv, TRef val)
   TRef fn = getcurrf(J);
   IRRef uref;
   int needbarrier = 0;
+  if (uvp->immutable) {  /* Try to constify immutable upvalue. */
+    TRef tr, kfunc;
+    lua_assert(val == 0);
+    if (!tref_isk(fn)) {  /* Late specialization of current function. */
+      if (J->pt->flags >= PROTO_CLC_POLY)
+        goto noconstify;
+      kfunc = lj_ir_kfunc(J, J->fn);
+      emitir(IRTG(IR_EQ, IRT_FUNC), fn, kfunc);
+      J->base[-1] = TREF_FRAME | kfunc;
+      fn = kfunc;
+    }
+    tr = lj_record_constify(J, uvval(uvp));
+    if (tr)
+      return tr;
+  }
+noconstify:
   /* Note: this effectively limits LJ_MAX_UPVAL to 127. */
   uv = (uv << 8) | (hashrot(uvp->dhash, uvp->dhash + HASH_BIAS) & 0xff);
   if (!uvp->closed) {
diff --git a/src/lj_record.h b/src/lj_record.h
index 40ffcb97..357392d8 100644
--- a/src/lj_record.h
+++ b/src/lj_record.h
@@ -28,6 +28,7 @@ typedef struct RecordIndex {
 
 LJ_FUNC int lj_record_objcmp(jit_State *J, TRef a, TRef b,
                              cTValue *av, cTValue *bv);
+LJ_FUNC TRef lj_record_constify(jit_State *J, cTValue *o);
 
 LJ_FUNC void lj_record_call(jit_State *J, BCReg func, ptrdiff_t nargs);
 LJ_FUNC void lj_record_tailcall(jit_State *J, BCReg func, ptrdiff_t nargs);