Redesign of prototype generation, part 5: colocation of protoype arrays.

10 files changed, 90 insertions, 129 deletions

diff --git a/src/lib_jit.c b/src/lib_jit.c
index d0b9e833..33571b17 100644
--- a/src/lib_jit.c
+++ b/src/lib_jit.c
@@ -246,8 +246,8 @@ LJLIB_CF(jit_util_funcuvname)
 {
   GCproto *pt = check_Lproto(L, 0);
   uint32_t idx = (uint32_t)lj_lib_checkint(L, 2);
-  if (idx < pt->sizeuvname) {
-    setstrV(L, L->top-1, gco2str(proto_uvname(pt, idx)));
+  if (idx < pt->sizeuv) {
+    setstrV(L, L->top-1, proto_uvname(pt, idx));
     return 1;
   }
   return 0;
diff --git a/src/lj_api.c b/src/lj_api.c
index 48bd605d..ad28bbf2 100644
--- a/src/lj_api.c
+++ b/src/lj_api.c
@@ -811,9 +811,9 @@ static const char *aux_upvalue(cTValue *f, uint32_t idx, TValue **val)
   fn = funcV(f);
   if (isluafunc(fn)) {
     GCproto *pt = funcproto(fn);
-    if (idx < pt->sizeuvname) {
+    if (idx < pt->sizeuv) {
       *val = uvval(&gcref(fn->l.uvptr[idx])->uv);
-      return strdata(gco2str(proto_uvname(pt, idx)));
+      return strdata(proto_uvname(pt, idx));
     }
   } else {
     if (idx < fn->c.nupvalues) {
diff --git a/src/lj_dispatch.c b/src/lj_dispatch.c
index 02fcf6ef..e2605e96 100644
--- a/src/lj_dispatch.c
+++ b/src/lj_dispatch.c
@@ -301,7 +301,7 @@ void LJ_FASTCALL lj_dispatch_ins(lua_State *L, const BCIns *pc)
     g->hookcount = g->hookcstart;
     callhook(L, LUA_HOOKCOUNT, -1);
   }
-  if ((g->hookmask & LUA_MASKLINE) && proto_lineinfo(pt)) {
+  if ((g->hookmask & LUA_MASKLINE)) {
     BCPos npc = proto_bcpos(pt, pc) - 1;
     BCPos opc = proto_bcpos(pt, oldpc) - 1;
     BCLine line = proto_line(pt, npc);
diff --git a/src/lj_err.c b/src/lj_err.c
index b0143c5a..2b021d28 100644
--- a/src/lj_err.c
+++ b/src/lj_err.c
@@ -129,7 +129,7 @@ static BCLine currentline(lua_State *L, GCfunc *fn, cTValue *nextframe)
   if (pc != ~(BCPos)0) {
     GCproto *pt = funcproto(fn);
     lua_assert(pc < pt->sizebc);
-    return proto_lineinfo(pt) ? proto_line(pt, pc) : 0;
+    return proto_line(pt, pc);
   } else {
     return -1;
   }
@@ -138,9 +138,10 @@ static BCLine currentline(lua_State *L, GCfunc *fn, cTValue *nextframe)
 static const char *getvarname(const GCproto *pt, BCPos pc, BCReg slot)
 {
   MSize i;
-  for (i = 0; i < pt->sizevarinfo && proto_varinfo(pt)[i].startpc <= pc; i++)
-    if (pc < proto_varinfo(pt)[i].endpc && slot-- == 0)
-      return strdata(gco2str(gcref(proto_varinfo(pt)[i].name)));
+  VarInfo *vi = proto_varinfo(pt);
+  for (i = 0; i < pt->sizevarinfo && vi[i].startpc <= pc; i++)
+    if (pc < vi[i].endpc && slot-- == 0)
+      return strdata(gco2str(gcref(vi[i].name)));
   return NULL;
 }
 
@@ -176,8 +177,7 @@ restart:
         }
         return "field";
       case BC_UGET:
-        *name = mref(pt->uvname, GCRef) ?
-                strdata(gco2str(proto_uvname(pt,bc_d(ins)))) : "?";
+        *name = strdata(proto_uvname(pt, bc_d(ins)));
         return "upvalue";
       default:
         return NULL;
@@ -224,7 +224,7 @@ void lj_err_pushloc(lua_State *L, GCproto *pt, BCPos pc)
     MSize i, len = name->len;
     BCLine line;
     if (pc)
-      line = proto_lineinfo(pt) ? proto_line(pt, pc-1) : 0;
+      line = proto_line(pt, pc-1);
     else
       line = pt->linedefined;
     if (*s == '@') {
@@ -377,12 +377,11 @@ LUA_API int lua_getinfo(lua_State *L, const char *what, lua_Debug *ar)
     case 'L':
       if (isluafunc(fn)) {
         GCtab *t = lj_tab_new(L, 0, 0);
-        BCLine *lineinfo = proto_lineinfo(funcproto(fn));
-        if (lineinfo) {
-          uint32_t i, szl = funcproto(fn)->sizelineinfo;
-          for (i = 0; i < szl; i++)
-            setboolV(lj_tab_setint(L, t, lineinfo[i]), 1);
-        }
+        GCproto *pt = funcproto(fn);
+        BCLine *lineinfo = proto_lineinfo(pt);
+        MSize i, szl = pt->sizebc;
+        for (i = 0; i < szl; i++)
+          setboolV(lj_tab_setint(L, t, lineinfo[i]), 1);
         settabV(L, L->top, t);
       } else {
         setnilV(L->top);
diff --git a/src/lj_func.c b/src/lj_func.c
index 74aba745..adc8c039 100644
--- a/src/lj_func.c
+++ b/src/lj_func.c
@@ -17,46 +17,10 @@
 
 /* -- Prototypes ---------------------------------------------------------- */
 
-GCproto *lj_func_newproto(lua_State *L)
-{
-  GCproto *pt = lj_mem_newobj(L, GCproto);
-  pt->gct = ~LJ_TPROTO;
-  pt->numparams = 0;
-  pt->framesize = 0;
-  pt->sizeuv = 0;
-  pt->flags = 0;
-  pt->trace = 0;
-  setmref(pt->k, NULL);
-  setmref(pt->bc, NULL);
-  setmref(pt->uv, NULL);
-  pt->sizebc = 0;
-  pt->sizekgc = 0;
-  pt->sizekn = 0;
-  pt->sizelineinfo = 0;
-  pt->sizevarinfo = 0;
-  pt->sizeuvname = 0;
-  pt->linedefined = 0;
-  pt->lastlinedefined = 0;
-  setmref(pt->lineinfo, NULL);
-  setmref(pt->varinfo, NULL);
-  setmref(pt->uvname, NULL);
-  setgcrefnull(pt->chunkname);
-  return pt;
-}
-
 void LJ_FASTCALL lj_func_freeproto(global_State *g, GCproto *pt)
 {
-  MSize nkgc = round_nkgc(pt->sizekgc);
-  MSize sizek = nkgc*(MSize)sizeof(GCRef) +
-                pt->sizekn*(MSize)sizeof(lua_Number);
-  lj_mem_free(g, mref(pt->k, GCRef) - nkgc, sizek);
-  lj_mem_freevec(g, proto_bc(pt), pt->sizebc, BCIns);
-  lj_mem_freevec(g, proto_uv(pt), pt->sizeuv, uint16_t);
-  lj_mem_freevec(g, proto_lineinfo(pt), pt->sizelineinfo, BCLine);
-  lj_mem_freevec(g, proto_varinfo(pt), pt->sizevarinfo, VarInfo);
-  lj_mem_freevec(g, mref(pt->uvname, GCRef), pt->sizeuvname, GCRef);
   lj_trace_freeproto(g, pt);
-  lj_mem_freet(g, pt);
+  lj_mem_free(g, pt, pt->sizept);
 }
 
 /* -- Upvalues ------------------------------------------------------------ */
diff --git a/src/lj_func.h b/src/lj_func.h
index c4bf750f..6ed45065 100644
--- a/src/lj_func.h
+++ b/src/lj_func.h
@@ -9,7 +9,6 @@
 #include "lj_obj.h"
 
 /* Prototypes. */
-LJ_FUNC GCproto *lj_func_newproto(lua_State *L);
 LJ_FUNC void LJ_FASTCALL lj_func_freeproto(global_State *g, GCproto *pt);
 
 /* Upvalues. */
diff --git a/src/lj_gc.c b/src/lj_gc.c
index bcef576b..d8221740 100644
--- a/src/lj_gc.c
+++ b/src/lj_gc.c
@@ -254,8 +254,8 @@ static void gc_traverse_proto(global_State *g, GCproto *pt)
   gc_mark_str(proto_chunkname(pt));
   for (i = -(ptrdiff_t)pt->sizekgc; i < 0; i++)  /* Mark collectable consts. */
     gc_markobj(g, proto_kgc(pt, i));
-  for (i = 0; i < (ptrdiff_t)pt->sizeuvname; i++)  /* Mark upvalue names. */
-    gc_mark_str(gco2str(proto_uvname(pt, i)));
+  for (i = 0; i < (ptrdiff_t)pt->sizeuv; i++)  /* Mark upvalue names. */
+    gc_mark_str(proto_uvname(pt, i));
   for (i = 0; i < (ptrdiff_t)pt->sizevarinfo; i++)  /* Mark names of locals. */
     gc_mark_str(gco2str(gcref(proto_varinfo(pt)[i].name)));
 }
@@ -323,13 +323,7 @@ static size_t propagatemark(global_State *g)
   } else if (LJ_LIKELY(o->gch.gct == ~LJ_TPROTO)) {
     GCproto *pt = gco2pt(o);
     gc_traverse_proto(g, pt);
-    return sizeof(GCproto) + sizeof(BCIns) * pt->sizebc +
-                             sizeof(GCRef) * pt->sizekgc +
-                             sizeof(lua_Number) * pt->sizekn +
-                             sizeof(uint16_t) * pt->sizeuv +
-                             sizeof(BCLine) * pt->sizelineinfo +
-                             sizeof(VarInfo) * pt->sizevarinfo +
-                             sizeof(GCRef) * pt->sizeuvname;
+    return pt->sizept;
   } else {
     lua_State *th = gco2th(o);
     setgcrefr(th->gclist, g->gc.grayagain);
diff --git a/src/lj_gdbjit.c b/src/lj_gdbjit.c
index e60a451b..3a4d5da1 100644
--- a/src/lj_gdbjit.c
+++ b/src/lj_gdbjit.c
@@ -706,7 +706,7 @@ void lj_gdbjit_addtrace(jit_State *J, Trace *T, TraceNo traceno)
   ctx.spadjp = CFRAME_SIZE + (MSize)(parent ? J->trace[parent]->spadjust : 0);
   ctx.spadj = CFRAME_SIZE + T->spadjust;
   if (startpc >= proto_bc(pt))
-    ctx.lineno = proto_lineinfo(pt) ? proto_line(pt,proto_bcpos(pt,startpc)): 0;
+    ctx.lineno = proto_line(pt,proto_bcpos(pt,startpc));
   else
     ctx.lineno = pt->linedefined;
   ctx.filename = strdata(proto_chunkname(pt));
diff --git a/src/lj_obj.h b/src/lj_obj.h
index a0dc689c..d8aab90a 100644
--- a/src/lj_obj.h
+++ b/src/lj_obj.h
@@ -355,19 +355,18 @@ typedef struct GCproto {
   MRef uv;              /* Upvalue list. local slot|0x8000 or parent uv idx. */
   MSize sizekgc;        /* Number of collectable constants. */
   MSize sizekn;         /* Number of lua_Number constants. */
+  MSize sizept;         /* Total size including colocated arrays. */
   uint8_t sizeuv;       /* Number of upvalues. */
   uint8_t flags;        /* Miscellaneous flags (see below). */
   uint16_t trace;       /* Anchor for chain of root traces. */
   /* ------ The following fields are for debugging/tracebacks only ------ */
-  MSize sizelineinfo;   /* Size of lineinfo array (may be 0). */
-  MSize sizevarinfo;    /* Size of local var info array (may be 0). */
-  MSize sizeuvname;     /* Size of upvalue names array (may be 0). */
+  GCRef chunkname;      /* Name of the chunk this function was defined in. */
   BCLine linedefined;   /* First line of the function definition. */
   BCLine lastlinedefined;  /* Last line of the function definition. */
-  MRef lineinfo;        /* Map from bytecode instructions to source lines. */
+  MSize sizevarinfo;    /* Size of local var info array. */
   MRef varinfo;         /* Names and extents of local variables. */
   MRef uvname;          /* Array of upvalue names (GCRef of GCstr). */
-  GCRef chunkname;      /* Name of the chunk this function was defined in. */
+  MRef lineinfo;        /* Map from bytecode instructions to source lines. */
 } GCproto;
 
 #define PROTO_IS_VARARG         0x01
@@ -389,9 +388,10 @@ typedef struct GCproto {
   check_exp((uintptr_t)(pos) < (pt)->sizebc, &proto_bc(pt)[(pos)])
 #define proto_bcpos(pt, pc)     ((BCPos)((pc) - proto_bc(pt)))
 #define proto_uv(pt)            (mref((pt)->uv, uint16_t))
+
 #define proto_uvname(pt, idx) \
-  check_exp((uintptr_t)(idx) < (pt)->sizeuvname, \
-            gcref(mref((pt)->uvname, GCRef)[(idx)]))
+  check_exp((uintptr_t)(idx) < (pt)->sizeuv, \
+            gco2str(gcref(mref((pt)->uvname, GCRef)[(idx)])))
 #define proto_chunkname(pt)     (gco2str(gcref((pt)->chunkname)))
 #define proto_lineinfo(pt)      (mref((pt)->lineinfo, BCLine))
 #define proto_line(pt, pos) \
diff --git a/src/lj_parse.c b/src/lj_parse.c
index ec355403..678dc4fa 100644
--- a/src/lj_parse.c
+++ b/src/lj_parse.c
@@ -853,7 +853,7 @@ static void bcemit_unop(FuncState *fs, BCOp op, ExpDesc *e)
     } else if (expr_isk(e)) {
       e->k = VKFALSE;
       return;
-    } else if (e->k == VJMP) { 
+    } else if (e->k == VJMP) {
       invertcond(fs, e);
       return;
     } else if (e->k == VRELOCABLE) {
@@ -1026,22 +1026,32 @@ static MSize var_lookup_(FuncState *fs, GCstr *name, ExpDesc *e, int first)
 
 /* -- Function state management ------------------------------------------- */
 
+/* NYI: compress debug info. */
+
+/* Fixup bytecode and lineinfo for prototype. */
+static void fs_fixup_bc(FuncState *fs, GCproto *pt, BCIns *bc, BCLine *lineinfo)
+{
+  MSize i, n = fs->pc;
+  BCInsLine *base = fs->bcbase;
+  setmref(pt->bc, bc);
+  setmref(pt->lineinfo, lineinfo);
+  pt->sizebc = n;
+  bc[n] = ~0u;  /* Close potentially uninitialized gap between bc and kgc. */
+  for (i = 0; i < n; i++) {
+    bc[i] = base[i].ins;
+    lineinfo[i] = base[i].line;
+  }
+}
+
 /* Fixup constants for prototype. */
-static void fs_fixup_k(FuncState *fs, GCproto *pt)
+static void fs_fixup_k(FuncState *fs, GCproto *pt, void *kptr)
 {
   GCtab *kt;
   TValue *array;
   Node *node;
-  BCReg nkgc;
-  MSize i, hmask, sizek;
-  GCRef *kptr;
+  MSize i, hmask;
   checklimitgt(fs, fs->nkn, BCMAX_D+1, "constants");
   checklimitgt(fs, fs->nkgc, BCMAX_D+1, "constants");
-  nkgc = round_nkgc(fs->nkgc);
-  sizek = (MSize)(nkgc*sizeof(GCRef) + fs->nkn*sizeof(lua_Number));
-  kptr = lj_mem_newt(fs->L, sizek, GCRef);
-  if (nkgc) setgcrefnull(kptr[0]);  /* May be uninitialized otherwise. */
-  kptr += nkgc;
   setmref(pt->k, kptr);
   pt->sizekn = fs->nkn;
   pt->sizekgc = fs->nkgc;
@@ -1060,7 +1070,7 @@ static void fs_fixup_k(FuncState *fs, GCproto *pt)
         ((lua_Number *)kptr)[kidx] = numV(&n->key);
       } else {
         GCobj *o = gcV(&n->key);
-        setgcref(kptr[~kidx], o);
+        setgcref(((GCRef *)kptr)[~kidx], o);
         lj_gc_objbarrier(fs->L, pt, o);
       }
     }
@@ -1068,16 +1078,29 @@ static void fs_fixup_k(FuncState *fs, GCproto *pt)
 }
 
 /* Fixup upvalues for prototype. */
-static void fs_fixup_uv(FuncState *fs, GCproto *pt)
+static void fs_fixup_uv(FuncState *fs, GCproto *pt, uint16_t *uv)
 {
   MSize i, n = fs->nuv;
-  uint16_t *uv = lj_mem_newvec(fs->L, n, uint16_t);
   setmref(pt->uv, uv);
   pt->sizeuv = n;
   for (i = 0; i < n; i++)
     uv[i] = fs->uvloc[i].slot;
 }
 
+/* Fixup debug info for prototype. */
+static void fs_fixup_dbg(FuncState *fs, GCproto *pt, VarInfo *vi, MSize sizevi)
+{
+  MSize i, n = fs->nuv;
+  GCRef *uvname = (GCRef *)((char *)vi + sizevi*sizeof(VarInfo));
+  VarInfo *vstack = fs->ls->vstack;
+  setmref(pt->varinfo, vi);
+  setmref(pt->uvname, uvname);
+  pt->sizevarinfo = sizevi;
+  memcpy(vi, &vstack[fs->vbase], sizevi*sizeof(VarInfo));
+  for (i = 0; i < n; i++)
+    setgcref(uvname[i], gcref(vstack[fs->uvloc[i].vidx].name));
+}
+
 /* Check if bytecode op returns. */
 static int bcopisret(BCOp op)
 {
@@ -1128,6 +1151,8 @@ static GCproto *fs_finish(LexState *ls, BCLine line)
 {
   lua_State *L = ls->L;
   FuncState *fs = ls->fs;
+  MSize sizevi;
+  size_t sizept, ofsk, ofsuv, ofsdbg, ofsli;
   GCproto *pt;
 
   /* Apply final fixups. */
@@ -1135,61 +1160,41 @@ static GCproto *fs_finish(LexState *ls, BCLine line)
   lua_assert(fs->bl == NULL);
   fs_fixup_ret(fs);
 
+  /* Calculate total size of prototype including all colocated arrays. */
+  sizept = sizeof(GCproto) + fs->pc*sizeof(BCIns) + fs->nkgc*sizeof(GCRef);
+  sizept = (sizept + sizeof(lua_Number)-1) & ~(sizeof(lua_Number)-1);
+  ofsk = sizept;
+  sizept += fs->nkn*sizeof(lua_Number);
+  ofsuv = sizept;
+  sizept += ((fs->nuv+1)&~1)*2;
+  ofsdbg = sizept;
+  sizevi = ls->vtop - fs->vbase;
+  sizept += sizevi*sizeof(VarInfo) + fs->nuv*sizeof(GCRef);
+  ofsli = sizept;
+  sizept += fs->pc*sizeof(BCLine);
+
   /* Allocate prototype and initialize its fields. */
-  pt = lj_func_newproto(L);
+  pt = (GCproto *)lj_mem_newgco(L, (MSize)sizept);
+  pt->gct = ~LJ_TPROTO;
+  pt->sizept = (MSize)sizept;
   setgcref(pt->chunkname, obj2gco(ls->chunkname));
+  pt->trace = 0;
   pt->flags = fs->flags;
   pt->numparams = fs->numparams;
   pt->framesize = fs->framesize;
   pt->linedefined = fs->linedefined;
   pt->lastlinedefined = line;
 
-  /* Anchor prototype since allocation of the arrays may fail. */
-  setprotoV(L, L->top, pt);
-  incr_top(L);
-
-  fs_fixup_k(fs, pt);
-  fs_fixup_uv(fs, pt);
-
-  {
-    MSize i, n = fs->pc;
-    BCInsLine *base = fs->bcbase;
-    BCLine *lineinfo;
-    BCIns *bc = lj_mem_newvec(L, n, BCIns);
-    setmref(pt->bc, bc);
-    pt->sizebc = fs->pc;
-    lineinfo = lj_mem_newvec(L, n, BCLine);
-    setmref(pt->lineinfo, lineinfo);
-    pt->sizelineinfo = n;
-    for (i = 0; i < n; i++) {
-      bc[i] = base[i].ins;
-      lineinfo[i] = base[i].line;
-    }
-  }
-
-  {
-    MSize n = ls->vtop - fs->vbase;
-    VarInfo *vi = lj_mem_newvec(L, n, VarInfo);
-    memcpy(vi, &ls->vstack[fs->vbase], n*sizeof(VarInfo));
-    setmref(pt->varinfo, vi);
-    pt->sizevarinfo = n;
-  }
-
-  {
-    MSize i, n = fs->nuv;
-    GCRef *uvname = lj_mem_newvec(L, n, GCRef);
-    for (i = 0; i < n; i++)
-      setgcref(uvname[i], gcref(ls->vstack[fs->uvloc[i].vidx].name));
-    setmref(pt->uvname, uvname);
-    pt->sizeuvname = n;
-  }
+  fs_fixup_bc(fs, pt, (BCIns *)((char *)pt + sizeof(GCproto)),
+                      (BCLine *)((char *)pt + ofsli));
+  fs_fixup_k(fs, pt, (void *)((char *)pt + ofsk));
+  fs_fixup_uv(fs, pt, (uint16_t *)((char *)pt + ofsuv));
+  fs_fixup_dbg(fs, pt, (VarInfo *)((char *)pt + ofsdbg), sizevi);
 
   lj_vmevent_send(L, BC,
     setprotoV(L, L->top++, pt);
   );
 
-  L->top--;  /* Pop prototype. */
-
   L->top--;  /* Pop table of constants. */
   ls->vtop = fs->vbase;  /* Reset variable stack. */
   ls->fs = fs->prev;
@@ -1427,7 +1432,7 @@ static void parse_body(LexState *ls, ExpDesc *e, int needself, BCLine line)
   lex_match(ls, TK_end, TK_function, line);
   pt = fs_finish(ls, lastline);
   fs->bcbase = ls->bcstack + oldbase;  /* May have been reallocated. */
-  fs->bclim = ls->sizebcstack - oldbase;
+  fs->bclim = (BCPos)(ls->sizebcstack - oldbase);
   /* Store new prototype in the constant array of the parent. */
   kidx = const_gc(fs, obj2gco(pt), LJ_TPROTO);
   expr_init(e, VRELOCABLE, bcemit_AD(fs, BC_FNEW, 0, kidx));