1 files changed, 241 insertions, 61 deletions

diff --git a/src/lj_obj.h b/src/lj_obj.h
index 3e506bdb..0a63fddf 100644
--- a/src/lj_obj.h
+++ b/src/lj_obj.h
@@ -13,44 +13,81 @@
 #include "lj_def.h"
 #include "lj_arch.h"
 
-/* -- Memory references (32 bit address space) ---------------------------- */
+/* -- Memory references --------------------------------------------------- */
 
-/* Memory size. */
+/* Memory and GC object sizes. */
 typedef uint32_t MSize;
+#if LJ_GC64
+typedef uint64_t GCSize;
+#else
+typedef uint32_t GCSize;
+#endif
 
 /* Memory reference */
 typedef struct MRef {
+#if LJ_GC64
+  uint64_t ptr64;       /* True 64 bit pointer. */
+#else
   uint32_t ptr32;       /* Pseudo 32 bit pointer. */
+#endif
 } MRef;
 
+#if LJ_GC64
+#define mref(r, t)      ((t *)(void *)(r).ptr64)
+#define mrefu(r)        ((r).ptr64)
+
+#define setmref(r, p)   ((r).ptr64 = (uint64_t)(void *)(p))
+#define setmrefu(r, u)  ((r).ptr64 = (uint64_t)(u))
+#define setmrefr(r, v)  ((r).ptr64 = (v).ptr64)
+#else
 #define mref(r, t)      ((t *)(void *)(uintptr_t)(r).ptr32)
+#define mrefu(r)        ((r).ptr32)
 
 #define setmref(r, p)   ((r).ptr32 = (uint32_t)(uintptr_t)(void *)(p))
+#define setmrefu(r, u)  ((r).ptr32 = (uint32_t)(u))
 #define setmrefr(r, v)  ((r).ptr32 = (v).ptr32)
+#endif
 
-/* -- GC object references (32 bit address space) ------------------------- */
+/* -- GC object references ------------------------------------------------ */
 
 /* GCobj reference */
 typedef struct GCRef {
+#if LJ_GC64
+  uint64_t gcptr64;     /* True 64 bit pointer. */
+#else
   uint32_t gcptr32;     /* Pseudo 32 bit pointer. */
+#endif
 } GCRef;
 
 /* Common GC header for all collectable objects. */
 #define GCHeader        GCRef nextgc; uint8_t marked; uint8_t gct
 /* This occupies 6 bytes, so use the next 2 bytes for non-32 bit fields. */
 
+#if LJ_GC64
+#define gcref(r)        ((GCobj *)(r).gcptr64)
+#define gcrefp(r, t)    ((t *)(void *)(r).gcptr64)
+#define gcrefu(r)       ((r).gcptr64)
+#define gcrefeq(r1, r2) ((r1).gcptr64 == (r2).gcptr64)
+
+#define setgcref(r, gc) ((r).gcptr64 = (uint64_t)&(gc)->gch)
+#define setgcreft(r, gc, it) \
+  (r).gcptr64 = (uint64_t)&(gc)->gch | (((uint64_t)(it)) << 47)
+#define setgcrefp(r, p) ((r).gcptr64 = (uint64_t)(p))
+#define setgcrefnull(r) ((r).gcptr64 = 0)
+#define setgcrefr(r, v) ((r).gcptr64 = (v).gcptr64)
+#else
 #define gcref(r)        ((GCobj *)(uintptr_t)(r).gcptr32)
 #define gcrefp(r, t)    ((t *)(void *)(uintptr_t)(r).gcptr32)
 #define gcrefu(r)       ((r).gcptr32)
-#define gcrefi(r)       ((int32_t)(r).gcptr32)
 #define gcrefeq(r1, r2) ((r1).gcptr32 == (r2).gcptr32)
-#define gcnext(gc)      (gcref((gc)->gch.nextgc))
 
 #define setgcref(r, gc) ((r).gcptr32 = (uint32_t)(uintptr_t)&(gc)->gch)
-#define setgcrefi(r, i) ((r).gcptr32 = (uint32_t)(i))
 #define setgcrefp(r, p) ((r).gcptr32 = (uint32_t)(uintptr_t)(p))
 #define setgcrefnull(r) ((r).gcptr32 = 0)
 #define setgcrefr(r, v) ((r).gcptr32 = (v).gcptr32)
+#endif
+
+#define gcnext(gc)      (gcref((gc)->gch.nextgc))
 
 /* IMPORTANT NOTE:
 **
@@ -119,11 +156,10 @@ typedef int32_t BCLine;  /* Bytecode line number. */
 /* Internal assembler functions. Never call these directly from C. */
 typedef void (*ASMFunction)(void);
 
-/* Resizable string buffer. Need this here, details in lj_str.h. */
+/* Resizable string buffer. Need this here, details in lj_buf.h. */
+#define SBufHeader      char *w, *e, *b; MRef L
 typedef struct SBuf {
-  char *buf;            /* String buffer base. */
-  MSize n;              /* String buffer length. */
-  MSize sz;             /* String buffer size. */
+  SBufHeader;
 } SBuf;
 
 /* -- Tags and values ----------------------------------------------------- */
@@ -131,13 +167,23 @@ typedef struct SBuf {
 /* Frame link. */
 typedef union {
   int32_t ftsz;         /* Frame type and size of previous frame. */
-  MRef pcr;             /* Overlaps PC for Lua frames. */
+  MRef pcr;             /* Or PC for Lua frames. */
 } FrameLink;
 
 /* Tagged value. */
 typedef LJ_ALIGN(8) union TValue {
   uint64_t u64;         /* 64 bit pattern overlaps number. */
   lua_Number n;         /* Number object overlaps split tag/value object. */
+#if LJ_GC64
+  GCRef gcr;            /* GCobj reference with tag. */
+  int64_t it64;
+  struct {
+    LJ_ENDIAN_LOHI(
+      int32_t i;        /* Integer value. */
+    , uint32_t it;      /* Internal object tag. Must overlap MSW of number. */
+    )
+  };
+#else
   struct {
     LJ_ENDIAN_LOHI(
       union {
@@ -147,12 +193,17 @@ typedef LJ_ALIGN(8) union TValue {
     , uint32_t it;      /* Internal object tag. Must overlap MSW of number. */
     )
   };
+#endif
+#if LJ_FR2
+  int64_t ftsz;         /* Frame type and size of previous frame, or PC. */
+#else
   struct {
     LJ_ENDIAN_LOHI(
       GCRef func;       /* Function for next frame (or dummy L). */
     , FrameLink tp;     /* Link to previous frame. */
     )
   } fr;
+#endif
   struct {
     LJ_ENDIAN_LOHI(
       uint32_t lo;      /* Lower 32 bits of number. */
@@ -172,6 +223,8 @@ typedef const TValue cTValue;
 
 /* Internal object tags.
 **
+** Format for 32 bit GC references (!LJ_GC64):
+**
 ** Internal tags overlap the MSW of a number object (must be a double).
 ** Interpreted as a double these are special NaNs. The FPU only generates
 ** one type of NaN (0xfff8_0000_0000_0000). So MSWs > 0xfff80000 are available
@@ -181,11 +234,24 @@ typedef const TValue cTValue;
 **                  ---MSW---.---LSW---
 ** primitive types |  itype  |         |
 ** lightuserdata   |  itype  |  void * |  (32 bit platforms)
-** lightuserdata   |ffff|    void *    |  (64 bit platforms, 47 bit pointers)
+** lightuserdata   |ffff|seg|    ofs   |  (64 bit platforms)
 ** GC objects      |  itype  |  GCRef  |
 ** int (LJ_DUALNUM)|  itype  |   int   |
 ** number           -------double------
 **
+** Format for 64 bit GC references (LJ_GC64):
+**
+** The upper 13 bits must be 1 (0xfff8...) for a special NaN. The next
+** 4 bits hold the internal tag. The lowest 47 bits either hold a pointer,
+** a zero-extended 32 bit integer or all bits set to 1 for primitive types.
+**
+**                     ------MSW------.------LSW------
+** primitive types    |1..1|itype|1..................1|
+** GC objects         |1..1|itype|-------GCRef--------|
+** lightuserdata      |1..1|itype|seg|------ofs-------|
+** int (LJ_DUALNUM)   |1..1|itype|0..0|-----int-------|
+** number              ------------double-------------
+**
 ** ORDER LJ_T
 ** Primitive types nil/false/true must be first, lightuserdata next.
 ** GC objects are at the end, table/userdata must be lowest.
@@ -208,7 +274,7 @@ typedef const TValue cTValue;
 #define LJ_TNUMX                (~13u)
 
 /* Integers have itype == LJ_TISNUM doubles have itype < LJ_TISNUM */
-#if LJ_64
+#if LJ_64 && !LJ_GC64
 #define LJ_TISNUM               0xfffeffffu
 #else
 #define LJ_TISNUM               LJ_TNUMX
@@ -218,14 +284,31 @@ typedef const TValue cTValue;
 #define LJ_TISGCV               (LJ_TSTR+1)
 #define LJ_TISTABUD             LJ_TTAB
 
+/* Type marker for slot holding a traversal index. Must be lightuserdata. */
+#define LJ_KEYINDEX             0xfffe7fffu
+
+#if LJ_GC64
+#define LJ_GCVMASK              (((uint64_t)1 << 47) - 1)
+#endif
+
+#if LJ_64
+/* To stay within 47 bits, lightuserdata is segmented. */
+#define LJ_LIGHTUD_BITS_SEG     8
+#define LJ_LIGHTUD_BITS_LO      (47 - LJ_LIGHTUD_BITS_SEG)
+#endif
+
 /* -- String object ------------------------------------------------------- */
 
+typedef uint32_t StrHash;       /* String hash value. */
+typedef uint32_t StrID;         /* String ID. */
+
 /* String object header. String payload follows. */
 typedef struct GCstr {
   GCHeader;
   uint8_t reserved;     /* Used by lexer for fast lookup of reserved words. */
-  uint8_t unused;
-  MSize hash;           /* Hash of string. */
+  uint8_t hashalg;      /* Hash algorithm. */
+  StrID sid;            /* Interned string ID. */
+  StrHash hash;         /* Hash of string. */
   MSize len;            /* Size of string. */
 } GCstr;
 
@@ -233,7 +316,6 @@ typedef struct GCstr {
 #define strdata(s)      ((const char *)((s)+1))
 #define strdatawr(s)    ((char *)((s)+1))
 #define strVdata(o)     strdata(strV(o))
-#define sizestring(s)   (sizeof(struct GCstr)+(s)->len+1)
 
 /* -- Userdata object ----------------------------------------------------- */
 
@@ -253,6 +335,7 @@ enum {
   UDTYPE_USERDATA,      /* Regular userdata. */
   UDTYPE_IO_FILE,       /* I/O library FILE. */
   UDTYPE_FFI_CLIB,      /* FFI C library namespace. */
+  UDTYPE_BUFFER,        /* String buffer. */
   UDTYPE__MAX
 };
 
@@ -291,6 +374,9 @@ typedef struct GCproto {
   uint8_t numparams;    /* Number of parameters. */
   uint8_t framesize;    /* Fixed frame size. */
   MSize sizebc;         /* Number of bytecode instructions. */
+#if LJ_GC64
+  uint32_t unused_gc64;
+#endif
   GCRef gclist;
   MRef k;               /* Split constant array (points to the middle). */
   MRef uv;              /* Upvalue list. local slot|0x8000 or parent uv idx. */
@@ -402,7 +488,9 @@ typedef struct Node {
   TValue val;           /* Value object. Must be first field. */
   TValue key;           /* Key object. */
   MRef next;            /* Hash chain. */
+#if !LJ_GC64
   MRef freetop;         /* Top of free elements (stored in t->node[0]). */
+#endif
 } Node;
 
 LJ_STATIC_ASSERT(offsetof(Node, val) == 0);
@@ -417,12 +505,22 @@ typedef struct GCtab {
   MRef node;            /* Hash part. */
   uint32_t asize;       /* Size of array part (keys [0, asize-1]). */
   uint32_t hmask;       /* Hash part mask (size of hash part - 1). */
+#if LJ_GC64
+  MRef freetop;         /* Top of free elements. */
+#endif
 } GCtab;
 
 #define sizetabcolo(n)  ((n)*sizeof(TValue) + sizeof(GCtab))
 #define tabref(r)       (&gcref((r))->tab)
 #define noderef(r)      (mref((r), Node))
 #define nextnode(n)     (mref((n)->next, Node))
+#if LJ_GC64
+#define getfreetop(t, n)        (noderef((t)->freetop))
+#define setfreetop(t, n, v)     (setmref((t)->freetop, (v)))
+#else
+#define getfreetop(t, n)        (noderef((n)->freetop))
+#define setfreetop(t, n, v)     (setmref((n)->freetop, (v)))
+#endif
 
 /* -- State objects ------------------------------------------------------- */
 
@@ -488,13 +586,18 @@ typedef enum {
 #define basemt_obj(g, o)        ((g)->gcroot[GCROOT_BASEMT+itypemap(o)])
 #define mmname_str(g, mm)       (strref((g)->gcroot[GCROOT_MMNAME+(mm)]))
 
+/* Garbage collector state. */
 typedef struct GCState {
-  MSize total;          /* Memory currently allocated. */
-  MSize threshold;      /* Memory threshold. */
+  GCSize total;         /* Memory currently allocated. */
+  GCSize threshold;     /* Memory threshold. */
   uint8_t currentwhite; /* Current white color. */
   uint8_t state;        /* GC state. */
   uint8_t nocdatafin;   /* No cdata finalizer called. */
-  uint8_t unused2;
+#if LJ_64
+  uint8_t lightudnum;   /* Number of lightuserdata segments - 1. */
+#else
+  uint8_t unused1;
+#endif
   MSize sweepstr;       /* Sweep position in string table. */
   GCRef root;           /* List of all collectable objects. */
   MRef sweep;           /* Sweep position in root list. */
@@ -502,42 +605,57 @@ typedef struct GCState {
   GCRef grayagain;      /* List of objects for atomic traversal. */
   GCRef weak;           /* List of weak tables (to be cleared). */
   GCRef mmudata;        /* List of userdata (to be finalized). */
+  GCSize debt;          /* Debt (how much GC is behind schedule). */
+  GCSize estimate;      /* Estimate of memory actually in use. */
   MSize stepmul;        /* Incremental GC step granularity. */
-  MSize debt;           /* Debt (how much GC is behind schedule). */
-  MSize estimate;       /* Estimate of memory actually in use. */
   MSize pause;          /* Pause between successive GC cycles. */
+#if LJ_64
+  MRef lightudseg;      /* Upper bits of lightuserdata segments. */
+#endif
 } GCState;
 
+/* String interning state. */
+typedef struct StrInternState {
+  GCRef *tab;           /* String hash table anchors. */
+  MSize mask;           /* String hash mask (size of hash table - 1). */
+  MSize num;            /* Number of strings in hash table. */
+  StrID id;             /* Next string ID. */
+  uint8_t idreseed;     /* String ID reseed counter. */
+  uint8_t second;       /* String interning table uses secondary hashing. */
+  uint8_t unused1;
+  uint8_t unused2;
+  LJ_ALIGN(8) uint64_t seed;    /* Random string seed. */
+} StrInternState;
+
 /* Global state, shared by all threads of a Lua universe. */
 typedef struct global_State {
-  GCRef *strhash;       /* String hash table (hash chain anchors). */
-  MSize strmask;        /* String hash mask (size of hash table - 1). */
-  MSize strnum;         /* Number of strings in hash table. */
   lua_Alloc allocf;     /* Memory allocator. */
   void *allocd;         /* Memory allocator data. */
   GCState gc;           /* Garbage collector. */
-  SBuf tmpbuf;          /* Temporary buffer for string concatenation. */
-  Node nilnode;         /* Fallback 1-element hash part (nil key and value). */
   GCstr strempty;       /* Empty string. */
   uint8_t stremptyz;    /* Zero terminator of empty string. */
   uint8_t hookmask;     /* Hook mask. */
   uint8_t dispatchmode; /* Dispatch mode. */
   uint8_t vmevmask;     /* VM event mask. */
+  StrInternState str;   /* String interning. */
+  volatile int32_t vmstate;  /* VM state or current JIT code trace number. */
   GCRef mainthref;      /* Link to main thread. */
-  TValue registrytv;    /* Anchor for registry. */
+  SBuf tmpbuf;          /* Temporary string buffer. */
   TValue tmptv, tmptv2; /* Temporary TValues. */
+  Node nilnode;         /* Fallback 1-element hash part (nil key and value). */
+  TValue registrytv;    /* Anchor for registry. */
   GCupval uvhead;       /* Head of double-linked list of all open upvalues. */
   int32_t hookcount;    /* Instruction hook countdown. */
   int32_t hookcstart;   /* Start count for instruction hook counter. */
   lua_Hook hookf;       /* Hook function. */
   lua_CFunction wrapf;  /* Wrapper for C function calls. */
   lua_CFunction panic;  /* Called as a last resort for errors. */
-  volatile int32_t vmstate;  /* VM state or current JIT code trace number. */
   BCIns bc_cfunc_int;   /* Bytecode for internal C function calls. */
   BCIns bc_cfunc_ext;   /* Bytecode for external C function calls. */
-  GCRef jit_L;          /* Current JIT code lua_State or NULL. */
-  MRef jit_base;        /* Current JIT code L->base. */
+  GCRef cur_L;          /* Currently executing lua_State. */
+  MRef jit_base;        /* Current JIT code L->base or NULL. */
   MRef ctype_state;     /* Pointer to C type state. */
+  PRNGState prng;       /* Global PRNG state. */
   GCRef gcroot[GCROOT_MAX];  /* GC roots. */
 } global_State;
 
@@ -553,9 +671,11 @@ typedef struct global_State {
 #define HOOK_ACTIVE_SHIFT       4
 #define HOOK_VMEVENT            0x20
 #define HOOK_GC                 0x40
+#define HOOK_PROFILE            0x80
 #define hook_active(g)          ((g)->hookmask & HOOK_ACTIVE)
 #define hook_enter(g)           ((g)->hookmask |= HOOK_ACTIVE)
-#define hook_entergc(g)         ((g)->hookmask |= (HOOK_ACTIVE|HOOK_GC))
+#define hook_entergc(g) \
+  ((g)->hookmask = ((g)->hookmask | (HOOK_ACTIVE|HOOK_GC)) & ~HOOK_PROFILE)
 #define hook_vmevent(g)         ((g)->hookmask |= (HOOK_ACTIVE|HOOK_VMEVENT))
 #define hook_leave(g)           ((g)->hookmask &= ~HOOK_ACTIVE)
 #define hook_save(g)            ((g)->hookmask & ~HOOK_EVENTMASK)
@@ -583,12 +703,23 @@ struct lua_State {
 #define registry(L)             (&G(L)->registrytv)
 
 /* Macros to access the currently executing (Lua) function. */
+#if LJ_GC64
+#define curr_func(L)            (&gcval(L->base-2)->fn)
+#elif LJ_FR2
+#define curr_func(L)            (&gcref((L->base-2)->gcr)->fn)
+#else
 #define curr_func(L)            (&gcref((L->base-1)->fr.func)->fn)
+#endif
 #define curr_funcisL(L)         (isluafunc(curr_func(L)))
 #define curr_proto(L)           (funcproto(curr_func(L)))
 #define curr_topL(L)            (L->base + curr_proto(L)->framesize)
 #define curr_top(L)             (curr_funcisL(L) ? curr_topL(L) : L->top)
 
+#if defined(LUA_USE_ASSERT) || defined(LUA_USE_APICHECK)
+LJ_FUNC_NORET void lj_assert_fail(global_State *g, const char *file, int line,
+                                  const char *func, const char *fmt, ...);
+#endif
+
 /* -- GC object definition and conversions -------------------------------- */
 
 /* GC header for generic access to common fields of GC objects. */
@@ -642,17 +773,18 @@ typedef union GCobj {
 
 /* -- TValue getters/setters ---------------------------------------------- */
 
-#ifdef LUA_USE_ASSERT
-#include "lj_gc.h"
-#endif
-
 /* Macros to test types. */
+#if LJ_GC64
+#define itype(o)        ((uint32_t)((o)->it64 >> 47))
+#define tvisnil(o)      ((o)->it64 == -1)
+#else
 #define itype(o)        ((o)->it)
 #define tvisnil(o)      (itype(o) == LJ_TNIL)
+#endif
 #define tvisfalse(o)    (itype(o) == LJ_TFALSE)
 #define tvistrue(o)     (itype(o) == LJ_TTRUE)
 #define tvisbool(o)     (tvisfalse(o) || tvistrue(o))
-#if LJ_64
+#if LJ_64 && !LJ_GC64
 #define tvislightud(o)  (((int32_t)itype(o) >> 15) == -2)
 #else
 #define tvislightud(o)  (itype(o) == LJ_TLIGHTUD)
@@ -686,7 +818,7 @@ typedef union GCobj {
 #define rawnumequal(o1, o2)     ((o1)->u64 == (o2)->u64)
 
 /* Macros to convert type ids. */
-#if LJ_64
+#if LJ_64 && !LJ_GC64
 #define itypemap(o) \
   (tvisnumber(o) ? ~LJ_TNUMX : tvislightud(o) ? ~LJ_TLIGHTUD : ~itype(o))
 #else
@@ -694,13 +826,30 @@ typedef union GCobj {
 #endif
 
 /* Macros to get tagged values. */
+#if LJ_GC64
+#define gcval(o)        ((GCobj *)(gcrefu((o)->gcr) & LJ_GCVMASK))
+#else
 #define gcval(o)        (gcref((o)->gcr))
-#define boolV(o)        check_exp(tvisbool(o), (LJ_TFALSE - (o)->it))
+#endif
+#define boolV(o)        check_exp(tvisbool(o), (LJ_TFALSE - itype(o)))
 #if LJ_64
-#define lightudV(o) \
-  check_exp(tvislightud(o), (void *)((o)->u64 & U64x(00007fff,ffffffff)))
+#define lightudseg(u) \
+  (((u) >> LJ_LIGHTUD_BITS_LO) & ((1 << LJ_LIGHTUD_BITS_SEG)-1))
+#define lightudlo(u) \
+  ((u) & (((uint64_t)1 << LJ_LIGHTUD_BITS_LO) - 1))
+#define lightudup(p) \
+  ((uint32_t)(((p) >> LJ_LIGHTUD_BITS_LO) << (LJ_LIGHTUD_BITS_LO-32)))
+static LJ_AINLINE void *lightudV(global_State *g, cTValue *o)
+{
+  uint64_t u = o->u64;
+  uint64_t seg = lightudseg(u);
+  uint32_t *segmap = mref(g->gc.lightudseg, uint32_t);
+  lj_assertG(tvislightud(o), "lightuserdata expected");
+  lj_assertG(seg <= g->gc.lightudnum, "bad lightuserdata segment %d", seg);
+  return (void *)(((uint64_t)segmap[seg] << 32) | lightudlo(u));
+}
 #else
-#define lightudV(o)     check_exp(tvislightud(o), gcrefp((o)->gcr, void))
+#define lightudV(g, o)  check_exp(tvislightud(o), gcrefp((o)->gcr, void))
 #endif
 #define gcV(o)          check_exp(tvisgcv(o), gcval(o))
 #define strV(o)         check_exp(tvisstr(o), &gcval(o)->str)
@@ -714,40 +863,70 @@ typedef union GCobj {
 #define intV(o)         check_exp(tvisint(o), (int32_t)(o)->i)
 
 /* Macros to set tagged values. */
+#if LJ_GC64
+#define setitype(o, i)          ((o)->it = ((i) << 15))
+#define setnilV(o)              ((o)->it64 = -1)
+#define setpriV(o, x)           ((o)->it64 = (int64_t)~((uint64_t)~(x)<<47))
+#define setboolV(o, x)          ((o)->it64 = (int64_t)~((uint64_t)((x)+1)<<47))
+#else
 #define setitype(o, i)          ((o)->it = (i))
 #define setnilV(o)              ((o)->it = LJ_TNIL)
 #define setboolV(o, x)          ((o)->it = LJ_TFALSE-(uint32_t)(x))
+#define setpriV(o, i)           (setitype((o), (i)))
+#endif
 
-static LJ_AINLINE void setlightudV(TValue *o, void *p)
+static LJ_AINLINE void setrawlightudV(TValue *o, void *p)
 {
-#if LJ_64
+#if LJ_GC64
+  o->u64 = (uint64_t)p | (((uint64_t)LJ_TLIGHTUD) << 47);
+#elif LJ_64
   o->u64 = (uint64_t)p | (((uint64_t)0xffff) << 48);
 #else
   setgcrefp(o->gcr, p); setitype(o, LJ_TLIGHTUD);
 #endif
 }
 
-#if LJ_64
-#define checklightudptr(L, p) \
-  (((uint64_t)(p) >> 47) ? (lj_err_msg(L, LJ_ERR_BADLU), NULL) : (p))
+#if LJ_FR2 || LJ_32
+#define contptr(f)              ((void *)(f))
+#define setcont(o, f)           ((o)->u64 = (uint64_t)(uintptr_t)contptr(f))
+#else
+#define contptr(f) \
+  ((void *)(uintptr_t)(uint32_t)((intptr_t)(f) - (intptr_t)lj_vm_asm_begin))
 #define setcont(o, f) \
   ((o)->u64 = (uint64_t)(void *)(f) - (uint64_t)lj_vm_asm_begin)
-#else
-#define checklightudptr(L, p)   (p)
-#define setcont(o, f)           setlightudV((o), (void *)(f))
 #endif
 
-#define tvchecklive(L, o) \
-  UNUSED(L), lua_assert(!tvisgcv(o) || \
-  ((~itype(o) == gcval(o)->gch.gct) && !isdead(G(L), gcval(o))))
+static LJ_AINLINE void checklivetv(lua_State *L, TValue *o, const char *msg)
+{
+  UNUSED(L); UNUSED(o); UNUSED(msg);
+#if LUA_USE_ASSERT
+  if (tvisgcv(o)) {
+    lj_assertL(~itype(o) == gcval(o)->gch.gct,
+               "mismatch of TValue type %d vs GC type %d",
+               ~itype(o), gcval(o)->gch.gct);
+    /* Copy of isdead check from lj_gc.h to avoid circular include. */
+    lj_assertL(!(gcval(o)->gch.marked & (G(L)->gc.currentwhite ^ 3) & 3), msg);
+  }
+#endif
+}
+
+static LJ_AINLINE void setgcVraw(TValue *o, GCobj *v, uint32_t itype)
+{
+#if LJ_GC64
+  setgcreft(o->gcr, v, itype);
+#else
+  setgcref(o->gcr, v); setitype(o, itype);
+#endif
+}
 
-static LJ_AINLINE void setgcV(lua_State *L, TValue *o, GCobj *v, uint32_t itype)
+static LJ_AINLINE void setgcV(lua_State *L, TValue *o, GCobj *v, uint32_t it)
 {
-  setgcref(o->gcr, v); setitype(o, itype); tvchecklive(L, o);
+  setgcVraw(o, v, it);
+  checklivetv(L, o, "store to dead GC object");
 }
 
 #define define_setV(name, type, tag) \
-static LJ_AINLINE void name(lua_State *L, TValue *o, type *v) \
+static LJ_AINLINE void name(lua_State *L, TValue *o, const type *v) \
 { \
   setgcV(L, o, obj2gco(v), tag); \
 }
@@ -790,13 +969,17 @@ static LJ_AINLINE void setint64V(TValue *o, int64_t i)
 /* Copy tagged values. */
 static LJ_AINLINE void copyTV(lua_State *L, TValue *o1, const TValue *o2)
 {
-  *o1 = *o2; tvchecklive(L, o1);
+  *o1 = *o2;
+  checklivetv(L, o1, "copy of dead GC object");
 }
 
 /* -- Number to integer conversion ---------------------------------------- */
 
 #if LJ_SOFTFP
 LJ_ASMF int32_t lj_vm_tobit(double x);
+#if LJ_TARGET_MIPS64
+LJ_ASMF int32_t lj_vm_tointg(double x);
+#endif
 #endif
 
 static LJ_AINLINE int32_t lj_num2bit(lua_Number n)
@@ -810,11 +993,7 @@ static LJ_AINLINE int32_t lj_num2bit(lua_Number n)
 #endif
 }
 
-#if LJ_TARGET_X86 && !defined(__SSE2__)
-#define lj_num2int(n)   lj_num2bit((n))
-#else
 #define lj_num2int(n)   ((int32_t)(n))
-#endif
 
 /*
 ** This must match the JIT backend behavior. In particular for archs
@@ -859,6 +1038,7 @@ LJ_DATA const char *const lj_obj_itypename[~LJ_TNUMX+1];
 #define lj_typename(o)  (lj_obj_itypename[itypemap(o)])
 
 /* Compare two objects without calling metamethods. */
-LJ_FUNC int lj_obj_equal(cTValue *o1, cTValue *o2);
+LJ_FUNC int LJ_FASTCALL lj_obj_equal(cTValue *o1, cTValue *o2);
+LJ_FUNC const void * LJ_FASTCALL lj_obj_ptr(global_State *g, cTValue *o);
 
 #endif