1 files changed, 161 insertions, 33 deletions

diff --git a/src/lj_obj.h b/src/lj_obj.h
index 5c3c88fc..7d582949 100644
--- a/src/lj_obj.h
+++ b/src/lj_obj.h
@@ -15,42 +15,75 @@
 
 /* -- Memory references (32 bit address space) ---------------------------- */
 
-/* Memory size. */
+/* Memory and GC object sizes. */
 typedef uint32_t MSize;
+#if LJ_GC64
+typedef uint64_t GCSize;
+#else
+typedef uint32_t GCSize;
+#endif
 
 /* Memory reference */
 typedef struct MRef {
+#if LJ_GC64
+  uint64_t ptr64;       /* True 64 bit pointer. */
+#else
   uint32_t ptr32;       /* Pseudo 32 bit pointer. */
+#endif
 } MRef;
 
+#if LJ_GC64
+#define mref(r, t)      ((t *)(void *)(r).ptr64)
+
+#define setmref(r, p)   ((r).ptr64 = (uint64_t)(void *)(p))
+#define setmrefr(r, v)  ((r).ptr64 = (v).ptr64)
+#else
 #define mref(r, t)      ((t *)(void *)(uintptr_t)(r).ptr32)
 
 #define setmref(r, p)   ((r).ptr32 = (uint32_t)(uintptr_t)(void *)(p))
 #define setmrefr(r, v)  ((r).ptr32 = (v).ptr32)
+#endif
 
 /* -- GC object references (32 bit address space) ------------------------- */
 
 /* GCobj reference */
 typedef struct GCRef {
+#if LJ_GC64
+  uint64_t gcptr64;     /* True 64 bit pointer. */
+#else
   uint32_t gcptr32;     /* Pseudo 32 bit pointer. */
+#endif
 } GCRef;
 
 /* Common GC header for all collectable objects. */
 #define GCHeader        GCRef nextgc; uint8_t marked; uint8_t gct
 /* This occupies 6 bytes, so use the next 2 bytes for non-32 bit fields. */
 
+#if LJ_GC64
+#define gcref(r)        ((GCobj *)(r).gcptr64)
+#define gcrefp(r, t)    ((t *)(void *)(r).gcptr64)
+#define gcrefu(r)       ((r).gcptr64)
+#define gcrefeq(r1, r2) ((r1).gcptr64 == (r2).gcptr64)
+
+#define setgcref(r, gc) ((r).gcptr64 = (uint64_t)&(gc)->gch)
+#define setgcreft(r, gc, it) \
+  (r).gcptr64 = (uint64_t)&(gc)->gch | (((uint64_t)(it)) << 47)
+#define setgcrefp(r, p) ((r).gcptr64 = (uint64_t)(p))
+#define setgcrefnull(r) ((r).gcptr64 = 0)
+#define setgcrefr(r, v) ((r).gcptr64 = (v).gcptr64)
+#else
 #define gcref(r)        ((GCobj *)(uintptr_t)(r).gcptr32)
 #define gcrefp(r, t)    ((t *)(void *)(uintptr_t)(r).gcptr32)
 #define gcrefu(r)       ((r).gcptr32)
-#define gcrefi(r)       ((int32_t)(r).gcptr32)
 #define gcrefeq(r1, r2) ((r1).gcptr32 == (r2).gcptr32)
-#define gcnext(gc)      (gcref((gc)->gch.nextgc))
 
 #define setgcref(r, gc) ((r).gcptr32 = (uint32_t)(uintptr_t)&(gc)->gch)
-#define setgcrefi(r, i) ((r).gcptr32 = (uint32_t)(i))
 #define setgcrefp(r, p) ((r).gcptr32 = (uint32_t)(uintptr_t)(p))
 #define setgcrefnull(r) ((r).gcptr32 = 0)
 #define setgcrefr(r, v) ((r).gcptr32 = (v).gcptr32)
+#endif
+
+#define gcnext(gc)      (gcref((gc)->gch.nextgc))
 
 /* IMPORTANT NOTE:
 **
@@ -119,11 +152,12 @@ typedef int32_t BCLine;  /* Bytecode line number. */
 /* Internal assembler functions. Never call these directly from C. */
 typedef void (*ASMFunction)(void);
 
-/* Resizable string buffer. Need this here, details in lj_str.h. */
+/* Resizable string buffer. Need this here, details in lj_buf.h. */
 typedef struct SBuf {
-  char *buf;            /* String buffer base. */
-  MSize n;              /* String buffer length. */
-  MSize sz;             /* String buffer size. */
+  MRef p;               /* String buffer pointer. */
+  MRef e;               /* String buffer end pointer. */
+  MRef b;               /* String buffer base. */
+  MRef L;               /* lua_State, used for buffer resizing. */
 } SBuf;
 
 /* -- Tags and values ----------------------------------------------------- */
@@ -131,13 +165,23 @@ typedef struct SBuf {
 /* Frame link. */
 typedef union {
   int32_t ftsz;         /* Frame type and size of previous frame. */
-  MRef pcr;             /* Overlaps PC for Lua frames. */
+  MRef pcr;             /* Or PC for Lua frames. */
 } FrameLink;
 
 /* Tagged value. */
 typedef LJ_ALIGN(8) union TValue {
   uint64_t u64;         /* 64 bit pattern overlaps number. */
   lua_Number n;         /* Number object overlaps split tag/value object. */
+#if LJ_GC64
+  GCRef gcr;            /* GCobj reference with tag. */
+  int64_t it64;
+  struct {
+    LJ_ENDIAN_LOHI(
+      int32_t i;        /* Integer value. */
+    , uint32_t it;      /* Internal object tag. Must overlap MSW of number. */
+    )
+  };
+#else
   struct {
     LJ_ENDIAN_LOHI(
       union {
@@ -147,12 +191,17 @@ typedef LJ_ALIGN(8) union TValue {
     , uint32_t it;      /* Internal object tag. Must overlap MSW of number. */
     )
   };
+#endif
+#if LJ_FR2
+  int64_t ftsz;         /* Frame type and size of previous frame, or PC. */
+#else
   struct {
     LJ_ENDIAN_LOHI(
       GCRef func;       /* Function for next frame (or dummy L). */
     , FrameLink tp;     /* Link to previous frame. */
     )
   } fr;
+#endif
   struct {
     LJ_ENDIAN_LOHI(
       uint32_t lo;      /* Lower 32 bits of number. */
@@ -172,6 +221,8 @@ typedef const TValue cTValue;
 
 /* Internal object tags.
 **
+** Format for 32 bit GC references (!LJ_GC64):
+**
 ** Internal tags overlap the MSW of a number object (must be a double).
 ** Interpreted as a double these are special NaNs. The FPU only generates
 ** one type of NaN (0xfff8_0000_0000_0000). So MSWs > 0xfff80000 are available
@@ -186,6 +237,18 @@ typedef const TValue cTValue;
 ** int (LJ_DUALNUM)|  itype  |   int   |
 ** number           -------double------
 **
+** Format for 64 bit GC references (LJ_GC64):
+**
+** The upper 13 bits must be 1 (0xfff8...) for a special NaN. The next
+** 4 bits hold the internal tag. The lowest 47 bits either hold a pointer,
+** a zero-extended 32 bit integer or all bits set to 1 for primitive types.
+**
+**                     ------MSW------.------LSW------
+** primitive types    |1..1|itype|1..................1|
+** GC objects/lightud |1..1|itype|-------GCRef--------|
+** int (LJ_DUALNUM)   |1..1|itype|0..0|-----int-------|
+** number              ------------double-------------
+**
 ** ORDER LJ_T
 ** Primitive types nil/false/true must be first, lightuserdata next.
 ** GC objects are at the end, table/userdata must be lowest.
@@ -208,7 +271,7 @@ typedef const TValue cTValue;
 #define LJ_TNUMX                (~13u)
 
 /* Integers have itype == LJ_TISNUM doubles have itype < LJ_TISNUM */
-#if LJ_64
+#if LJ_64 && !LJ_GC64
 #define LJ_TISNUM               0xfffeffffu
 #else
 #define LJ_TISNUM               LJ_TNUMX
@@ -218,6 +281,10 @@ typedef const TValue cTValue;
 #define LJ_TISGCV               (LJ_TSTR+1)
 #define LJ_TISTABUD             LJ_TTAB
 
+#if LJ_GC64
+#define LJ_GCVMASK              (((uint64_t)1 << 47) - 1)
+#endif
+
 /* -- String object ------------------------------------------------------- */
 
 /* String object header. String payload follows. */
@@ -291,6 +358,9 @@ typedef struct GCproto {
   uint8_t numparams;    /* Number of parameters. */
   uint8_t framesize;    /* Fixed frame size. */
   MSize sizebc;         /* Number of bytecode instructions. */
+#if LJ_GC64
+  uint32_t unused_gc64;
+#endif
   GCRef gclist;
   MRef k;               /* Split constant array (points to the middle). */
   MRef uv;              /* Upvalue list. local slot|0x8000 or parent uv idx. */
@@ -402,7 +472,9 @@ typedef struct Node {
   TValue val;           /* Value object. Must be first field. */
   TValue key;           /* Key object. */
   MRef next;            /* Hash chain. */
+#if !LJ_GC64
   MRef freetop;         /* Top of free elements (stored in t->node[0]). */
+#endif
 } Node;
 
 LJ_STATIC_ASSERT(offsetof(Node, val) == 0);
@@ -417,12 +489,22 @@ typedef struct GCtab {
   MRef node;            /* Hash part. */
   uint32_t asize;       /* Size of array part (keys [0, asize-1]). */
   uint32_t hmask;       /* Hash part mask (size of hash part - 1). */
+#if LJ_GC64
+  MRef freetop;         /* Top of free elements. */
+#endif
 } GCtab;
 
 #define sizetabcolo(n)  ((n)*sizeof(TValue) + sizeof(GCtab))
 #define tabref(r)       (&gcref((r))->tab)
 #define noderef(r)      (mref((r), Node))
 #define nextnode(n)     (mref((n)->next, Node))
+#if LJ_GC64
+#define getfreetop(t, n)        (noderef((t)->freetop))
+#define setfreetop(t, n, v)     (setmref((t)->freetop, (v)))
+#else
+#define getfreetop(t, n)        (noderef((n)->freetop))
+#define setfreetop(t, n, v)     (setmref((n)->freetop, (v)))
+#endif
 
 /* -- State objects ------------------------------------------------------- */
 
@@ -489,8 +571,8 @@ typedef enum {
 #define mmname_str(g, mm)       (strref((g)->gcroot[GCROOT_MMNAME+(mm)]))
 
 typedef struct GCState {
-  MSize total;          /* Memory currently allocated. */
-  MSize threshold;      /* Memory threshold. */
+  GCSize total;         /* Memory currently allocated. */
+  GCSize threshold;     /* Memory threshold. */
   uint8_t currentwhite; /* Current white color. */
   uint8_t state;        /* GC state. */
   uint8_t nocdatafin;   /* No cdata finalizer called. */
@@ -502,9 +584,9 @@ typedef struct GCState {
   GCRef grayagain;      /* List of objects for atomic traversal. */
   GCRef weak;           /* List of weak tables (to be cleared). */
   GCRef mmudata;        /* List of userdata (to be finalized). */
+  GCSize debt;          /* Debt (how much GC is behind schedule). */
+  GCSize estimate;      /* Estimate of memory actually in use. */
   MSize stepmul;        /* Incremental GC step granularity. */
-  MSize debt;           /* Debt (how much GC is behind schedule). */
-  MSize estimate;       /* Estimate of memory actually in use. */
   MSize pause;          /* Pause between successive GC cycles. */
 } GCState;
 
@@ -516,8 +598,8 @@ typedef struct global_State {
   lua_Alloc allocf;     /* Memory allocator. */
   void *allocd;         /* Memory allocator data. */
   GCState gc;           /* Garbage collector. */
-  SBuf tmpbuf;          /* Temporary buffer for string concatenation. */
-  Node nilnode;         /* Fallback 1-element hash part (nil key and value). */
+  volatile int32_t vmstate;  /* VM state or current JIT code trace number. */
+  SBuf tmpbuf;          /* Temporary string buffer. */
   GCstr strempty;       /* Empty string. */
   uint8_t stremptyz;    /* Zero terminator of empty string. */
   uint8_t hookmask;     /* Hook mask. */
@@ -526,17 +608,17 @@ typedef struct global_State {
   GCRef mainthref;      /* Link to main thread. */
   TValue registrytv;    /* Anchor for registry. */
   TValue tmptv, tmptv2; /* Temporary TValues. */
+  Node nilnode;         /* Fallback 1-element hash part (nil key and value). */
   GCupval uvhead;       /* Head of double-linked list of all open upvalues. */
   int32_t hookcount;    /* Instruction hook countdown. */
   int32_t hookcstart;   /* Start count for instruction hook counter. */
   lua_Hook hookf;       /* Hook function. */
   lua_CFunction wrapf;  /* Wrapper for C function calls. */
   lua_CFunction panic;  /* Called as a last resort for errors. */
-  volatile int32_t vmstate;  /* VM state or current JIT code trace number. */
   BCIns bc_cfunc_int;   /* Bytecode for internal C function calls. */
   BCIns bc_cfunc_ext;   /* Bytecode for external C function calls. */
-  GCRef jit_L;          /* Current JIT code lua_State or NULL. */
-  MRef jit_base;        /* Current JIT code L->base. */
+  GCRef cur_L;          /* Currently executing lua_State. */
+  MRef jit_base;        /* Current JIT code L->base or NULL. */
   MRef ctype_state;     /* Pointer to C type state. */
   GCRef gcroot[GCROOT_MAX];  /* GC roots. */
 } global_State;
@@ -553,9 +635,11 @@ typedef struct global_State {
 #define HOOK_ACTIVE_SHIFT       4
 #define HOOK_VMEVENT            0x20
 #define HOOK_GC                 0x40
+#define HOOK_PROFILE            0x80
 #define hook_active(g)          ((g)->hookmask & HOOK_ACTIVE)
 #define hook_enter(g)           ((g)->hookmask |= HOOK_ACTIVE)
-#define hook_entergc(g)         ((g)->hookmask |= (HOOK_ACTIVE|HOOK_GC))
+#define hook_entergc(g) \
+  ((g)->hookmask = ((g)->hookmask | (HOOK_ACTIVE|HOOK_GC)) & ~HOOK_PROFILE)
 #define hook_vmevent(g)         ((g)->hookmask |= (HOOK_ACTIVE|HOOK_VMEVENT))
 #define hook_leave(g)           ((g)->hookmask &= ~HOOK_ACTIVE)
 #define hook_save(g)            ((g)->hookmask & ~HOOK_EVENTMASK)
@@ -583,7 +667,13 @@ struct lua_State {
 #define registry(L)             (&G(L)->registrytv)
 
 /* Macros to access the currently executing (Lua) function. */
+#if LJ_GC64
+#define curr_func(L)            (&gcval(L->base-2)->fn)
+#elif LJ_FR2
+#define curr_func(L)            (&gcref((L->base-2)->gcr)->fn)
+#else
 #define curr_func(L)            (&gcref((L->base-1)->fr.func)->fn)
+#endif
 #define curr_funcisL(L)         (isluafunc(curr_func(L)))
 #define curr_proto(L)           (funcproto(curr_func(L)))
 #define curr_topL(L)            (L->base + curr_proto(L)->framesize)
@@ -647,12 +737,17 @@ typedef union GCobj {
 #endif
 
 /* Macros to test types. */
+#if LJ_GC64
+#define itype(o)        ((uint32_t)((o)->it64 >> 47))
+#define tvisnil(o)      ((o)->it64 == -1)
+#else
 #define itype(o)        ((o)->it)
 #define tvisnil(o)      (itype(o) == LJ_TNIL)
+#endif
 #define tvisfalse(o)    (itype(o) == LJ_TFALSE)
 #define tvistrue(o)     (itype(o) == LJ_TTRUE)
 #define tvisbool(o)     (tvisfalse(o) || tvistrue(o))
-#if LJ_64
+#if LJ_64 && !LJ_GC64
 #define tvislightud(o)  (((int32_t)itype(o) >> 15) == -2)
 #else
 #define tvislightud(o)  (itype(o) == LJ_TLIGHTUD)
@@ -686,7 +781,7 @@ typedef union GCobj {
 #define rawnumequal(o1, o2)     ((o1)->u64 == (o2)->u64)
 
 /* Macros to convert type ids. */
-#if LJ_64
+#if LJ_64 && !LJ_GC64
 #define itypemap(o) \
   (tvisnumber(o) ? ~LJ_TNUMX : tvislightud(o) ? ~LJ_TLIGHTUD : ~itype(o))
 #else
@@ -694,8 +789,12 @@ typedef union GCobj {
 #endif
 
 /* Macros to get tagged values. */
+#if LJ_GC64
+#define gcval(o)        ((GCobj *)(gcrefu((o)->gcr) & LJ_GCVMASK))
+#else
 #define gcval(o)        (gcref((o)->gcr))
-#define boolV(o)        check_exp(tvisbool(o), (LJ_TFALSE - (o)->it))
+#endif
+#define boolV(o)        check_exp(tvisbool(o), (LJ_TFALSE - itype(o)))
 #if LJ_64
 #define lightudV(o) \
   check_exp(tvislightud(o), (void *)((o)->u64 & U64x(00007fff,ffffffff)))
@@ -714,13 +813,23 @@ typedef union GCobj {
 #define intV(o)         check_exp(tvisint(o), (int32_t)(o)->i)
 
 /* Macros to set tagged values. */
+#if LJ_GC64
+#define setitype(o, i)          ((o)->it = ((i) << 15))
+#define setnilV(o)              ((o)->it64 = -1)
+#define setpriV(o, x)           ((o)->it64 = (int64_t)~((uint64_t)~(x)<<47))
+#define setboolV(o, x)          ((o)->it64 = (int64_t)~((uint64_t)((x)+1)<<47))
+#else
 #define setitype(o, i)          ((o)->it = (i))
 #define setnilV(o)              ((o)->it = LJ_TNIL)
 #define setboolV(o, x)          ((o)->it = LJ_TFALSE-(uint32_t)(x))
+#define setpriV(o, i)           (setitype((o), (i)))
+#endif
 
 static LJ_AINLINE void setlightudV(TValue *o, void *p)
 {
-#if LJ_64
+#if LJ_GC64
+  o->u64 = (uint64_t)p | (((uint64_t)LJ_TLIGHTUD) << 47);
+#elif LJ_64
   o->u64 = (uint64_t)p | (((uint64_t)0xffff) << 48);
 #else
   setgcrefp(o->gcr, p); setitype(o, LJ_TLIGHTUD);
@@ -730,20 +839,39 @@ static LJ_AINLINE void setlightudV(TValue *o, void *p)
 #if LJ_64
 #define checklightudptr(L, p) \
   (((uint64_t)(p) >> 47) ? (lj_err_msg(L, LJ_ERR_BADLU), NULL) : (p))
+#else
+#define checklightudptr(L, p)   (p)
+#endif
+
+#if LJ_FR2
+#define contptr(f)              ((void *)(f))
+#define setcont(o, f)           ((o)->u64 = (uint64_t)(uintptr_t)contptr(f))
+#elif LJ_64
+#define contptr(f) \
+  ((void *)(uintptr_t)(uint32_t)((intptr_t)(f) - (intptr_t)lj_vm_asm_begin))
 #define setcont(o, f) \
   ((o)->u64 = (uint64_t)(void *)(f) - (uint64_t)lj_vm_asm_begin)
 #else
-#define checklightudptr(L, p)   (p)
-#define setcont(o, f)           setlightudV((o), (void *)(f))
+#define contptr(f)              ((void *)(f))
+#define setcont(o, f)           setlightudV((o), contptr(f))
 #endif
 
 #define tvchecklive(L, o) \
   UNUSED(L), lua_assert(!tvisgcv(o) || \
   ((~itype(o) == gcval(o)->gch.gct) && !isdead(G(L), gcval(o))))
 
-static LJ_AINLINE void setgcV(lua_State *L, TValue *o, GCobj *v, uint32_t itype)
+static LJ_AINLINE void setgcVraw(TValue *o, GCobj *v, uint32_t itype)
 {
-  setgcref(o->gcr, v); setitype(o, itype); tvchecklive(L, o);
+#if LJ_GC64
+  setgcreft(o->gcr, v, itype);
+#else
+  setgcref(o->gcr, v); setitype(o, itype);
+#endif
+}
+
+static LJ_AINLINE void setgcV(lua_State *L, TValue *o, GCobj *v, uint32_t it)
+{
+  setgcVraw(o, v, it); tvchecklive(L, o);
 }
 
 #define define_setV(name, type, tag) \
@@ -797,6 +925,9 @@ static LJ_AINLINE void copyTV(lua_State *L, TValue *o1, const TValue *o2)
 
 #if LJ_SOFTFP
 LJ_ASMF int32_t lj_vm_tobit(double x);
+#if LJ_TARGET_MIPS64
+LJ_ASMF int32_t lj_vm_tointg(double x);
+#endif
 #endif
 
 static LJ_AINLINE int32_t lj_num2bit(lua_Number n)
@@ -810,11 +941,7 @@ static LJ_AINLINE int32_t lj_num2bit(lua_Number n)
 #endif
 }
 
-#if LJ_TARGET_X86 && !defined(__SSE2__)
-#define lj_num2int(n)   lj_num2bit((n))
-#else
 #define lj_num2int(n)   ((int32_t)(n))
-#endif
 
 /*
 ** This must match the JIT backend behavior. In particular for archs
@@ -859,6 +986,7 @@ LJ_DATA const char *const lj_obj_itypename[~LJ_TNUMX+1];
 #define lj_typename(o)  (lj_obj_itypename[itypemap(o)])
 
 /* Compare two objects without calling metamethods. */
-LJ_FUNC int lj_obj_equal(cTValue *o1, cTValue *o2);
+LJ_FUNC int LJ_FASTCALL lj_obj_equal(cTValue *o1, cTValue *o2);
+LJ_FUNC const void * LJ_FASTCALL lj_obj_ptr(cTValue *o);
 
 #endif