String buffers, part 2b: Add extended string buffers. Use in serializer.

Sponsored by fmad.io.

6 files changed, 210 insertions, 98 deletions

diff --git a/src/lib_buffer.c b/src/lib_buffer.c
index e21362b8..c9ef9510 100644
--- a/src/lib_buffer.c
+++ b/src/lib_buffer.c
@@ -22,15 +22,13 @@
 
 #define LJLIB_MODULE_buffer
 
-/* Note: this uses interim structs until the SBuf reorg. */
-
 LJLIB_CF(buffer_encode)
 {
   cTValue *o = lj_lib_checkany(L, 1);
-  StrBuf sbuf;
-  sbuf.sb = lj_buf_tmp_(L);
-  lj_serialize_put(&sbuf, o);
-  setstrV(L, L->top++, lj_buf_str(L, sbuf.sb));
+  SBufExt sbx;
+  lj_bufx_init_borrow(L, &sbx, &G(L)->tmpbuf);
+  lj_serialize_put(&sbx, o);
+  setstrV(L, L->top++, lj_buf_str(L, (SBuf *)&sbx));
   lj_gc_check(L);
   return 1;
 }
@@ -38,16 +36,11 @@ LJLIB_CF(buffer_encode)
 LJLIB_CF(buffer_decode)
 {
   GCstr *str = lj_lib_checkstr(L, 1);
-  char *p = (char *)strdata(str);
-  SBuf sb;
-  StrBuf sbuf;
-  setsbufL(&sb, L);
-  sb.b = p;
-  sb.w = sb.e = p + str->len;
-  sbuf.sb = &sb;
-  sbuf.r = p;
+  SBufExt sbx;
+  lj_bufx_init_cow(L, &sbx, strdata(str), str->len);
+  /* No need to set sbx.cowref here. */
   setnilV(L->top++);
-  lj_serialize_get(&sbuf, L->top-1);
+  lj_serialize_get(&sbx, L->top-1);
   lj_gc_check(L);
   return 1;
 }
diff --git a/src/lj_buf.c b/src/lj_buf.c
index 66a096fb..889ccbca 100644
--- a/src/lj_buf.c
+++ b/src/lj_buf.c
@@ -20,12 +20,32 @@ static void buf_grow(SBuf *sb, MSize sz)
 {
   MSize osz = sbufsz(sb), len = sbuflen(sb), nsz = osz;
   char *b;
+  GCSize flag;
   if (nsz < LJ_MIN_SBUF) nsz = LJ_MIN_SBUF;
   while (nsz < sz) nsz += nsz;
-  b = (char *)lj_mem_realloc(sbufL(sb), sb->b, osz, nsz);
+  flag = sbufflag(sb);
+  if ((flag & SBUF_FLAG_COW)) {  /* Copy-on-write semantics. */
+    lj_assertG_(G(sbufL(sb)), sb->w == sb->e, "bad SBuf COW");
+    b = (char *)lj_mem_new(sbufL(sb), nsz);
+    setsbufflag(sb, flag & ~(GCSize)SBUF_FLAG_COW);
+    setgcrefnull(sbufX(sb)->cowref);
+    memcpy(b, sb->b, osz);
+  } else {
+    b = (char *)lj_mem_realloc(sbufL(sb), sb->b, osz, nsz);
+  }
+  if ((flag & SBUF_FLAG_EXT)) {
+    sbufX(sb)->r = sbufX(sb)->r - sb->b + b;  /* Adjust read pointer, too. */
+  }
+  /* Adjust buffer pointers. */
   sb->b = b;
   sb->w = b + len;
   sb->e = b + nsz;
+  if ((flag & SBUF_FLAG_BORROW)) {  /* Adjust borrowed buffer pointers. */
+    SBuf *bsb = mref(sbufX(sb)->bsb, SBuf);
+    bsb->b = b;
+    bsb->w = b + len;
+    bsb->e = b + nsz;
+  }
 }
 
 LJ_NOINLINE char *LJ_FASTCALL lj_buf_need2(SBuf *sb, MSize sz)
@@ -39,11 +59,31 @@ LJ_NOINLINE char *LJ_FASTCALL lj_buf_need2(SBuf *sb, MSize sz)
 
 LJ_NOINLINE char *LJ_FASTCALL lj_buf_more2(SBuf *sb, MSize sz)
 {
-  MSize len = sbuflen(sb);
-  lj_assertG_(G(sbufL(sb)), sz > sbufleft(sb), "SBuf overflow");
-  if (LJ_UNLIKELY(sz > LJ_MAX_BUF || len + sz > LJ_MAX_BUF))
-    lj_err_mem(sbufL(sb));
-  buf_grow(sb, len + sz);
+  if (sbufisext(sb)) {
+    SBufExt *sbx = (SBufExt *)sb;
+    MSize len = sbufxlen(sbx);
+    if (LJ_UNLIKELY(sz > LJ_MAX_BUF || len + sz > LJ_MAX_BUF))
+      lj_err_mem(sbufL(sbx));
+    if (len + sz > sbufsz(sbx)) {  /* Must grow. */
+      buf_grow((SBuf *)sbx, len + sz);
+    } else if (sbufxslack(sbx) < (sbufsz(sbx) >> 3)) {
+      /* Also grow to avoid excessive compactions, if slack < size/8. */
+      buf_grow((SBuf *)sbx, sbuflen(sbx) + sz);  /* Not sbufxlen! */
+      return sbx->w;
+    }
+    if (sbx->r != sbx->b) {  /* Compact by moving down. */
+      memmove(sbx->b, sbx->r, len);
+      sbx->r = sbx->b;
+      sbx->w = sbx->b + len;
+      lj_assertG_(G(sbufL(sbx)), len + sz <= sbufsz(sbx), "bad SBuf compact");
+    }
+  } else {
+    MSize len = sbuflen(sb);
+    lj_assertG_(G(sbufL(sb)), sz > sbufleft(sb), "SBuf overflow");
+    if (LJ_UNLIKELY(sz > LJ_MAX_BUF || len + sz > LJ_MAX_BUF))
+      lj_err_mem(sbufL(sb));
+    buf_grow(sb, len + sz);
+  }
   return sb->w;
 }
 
@@ -58,6 +98,7 @@ void LJ_FASTCALL lj_buf_shrink(lua_State *L, SBuf *sb)
     sb->w = b + n;
     sb->e = b + (osz >> 1);
   }
+  lj_assertG_(G(sbufL(sb)), !sbufisext(sb), "YAGNI shrink SBufExt");
 }
 
 char * LJ_FASTCALL lj_buf_tmp(lua_State *L, MSize sz)
diff --git a/src/lj_buf.h b/src/lj_buf.h
index 268fce5e..1fb70146 100644
--- a/src/lj_buf.h
+++ b/src/lj_buf.h
@@ -10,13 +10,53 @@
 #include "lj_gc.h"
 #include "lj_str.h"
 
-/* Resizable string buffers. SBuf struct definition in lj_obj.h. */
-#define sbufsz(sb)      ((MSize)((sb)->e - (sb)->b))
-#define sbuflen(sb)     ((MSize)((sb)->w - (sb)->b))
-#define sbufleft(sb)    ((MSize)((sb)->e - (sb)->w))
+/* Resizable string buffers. */
 
-#define sbufL(sb)       (mref((sb)->L, lua_State))
-#define setsbufL(sb, l) (setmref((sb)->L, (l)))
+/* The SBuf struct definition is in lj_obj.h:
+**   char *w;   Write pointer.
+**   char *e;   End pointer.
+**   char *b;   Base pointer.
+**   MRef L;    lua_State, used for buffer resizing. Extension bits in 3 LSB.
+*/
+
+/* Extended string buffer. */
+typedef struct SBufExt {
+  SBufHeader;
+  union {
+    GCRef cowref;       /* Copy-on-write object reference. */
+    MRef bsb;           /* Borrowed string buffer. */
+  };
+  char *r;              /* Read pointer. */
+  int depth;            /* Remaining recursion depth. */
+} SBufExt;
+
+#define sbufsz(sb)              ((MSize)((sb)->e - (sb)->b))
+#define sbuflen(sb)             ((MSize)((sb)->w - (sb)->b))
+#define sbufleft(sb)            ((MSize)((sb)->e - (sb)->w))
+#define sbufxlen(sbx)           ((MSize)((sbx)->w - (sbx)->r))
+#define sbufxslack(sbx)         ((MSize)((sbx)->r - (sbx)->b))
+
+#define SBUF_MASK_FLAG          (7)
+#define SBUF_MASK_L             (~(GCSize)SBUF_MASK_FLAG)
+#define SBUF_FLAG_EXT           1       /* Extended string buffer. */
+#define SBUF_FLAG_COW           2       /* Copy-on-write buffer. */
+#define SBUF_FLAG_BORROW        4       /* Borrowed string buffer. */
+
+#define sbufL(sb) \
+  ((lua_State *)(void *)(uintptr_t)(mrefu((sb)->L) & SBUF_MASK_L))
+#define setsbufL(sb, l)         (setmref((sb)->L, (l)))
+#define setsbufXL(sb, l, flag) \
+  (setmrefu((sb)->L, (GCSize)(uintptr_t)(void *)(l) + (flag)))
+#define setsbufXL_(sb, l) \
+  (setmrefu((sb)->L, (GCSize)(uintptr_t)(void *)(l) | (mrefu((sb)->L) & SBUF_MASK_FLAG)))
+
+#define sbufflag(sb)            (mrefu((sb)->L))
+#define sbufisext(sb)           (sbufflag((sb)) & SBUF_FLAG_EXT)
+#define sbufiscow(sb)           (sbufflag((sb)) & SBUF_FLAG_COW)
+#define sbufisborrow(sb)        (sbufflag((sb)) & SBUF_FLAG_BORROW)
+#define sbufX(sb) \
+  (lj_assertG_(G(sbufL(sb)), sbufisext(sb), "not an SBufExt"), (SBufExt *)(sb))
+#define setsbufflag(sb, flag)   (setmrefu((sb)->L, (flag)))
 
 /* Buffer management */
 LJ_FUNC char *LJ_FASTCALL lj_buf_need2(SBuf *sb, MSize sz);
@@ -45,6 +85,7 @@ static LJ_AINLINE SBuf *lj_buf_tmp_(lua_State *L)
 
 static LJ_AINLINE void lj_buf_free(global_State *g, SBuf *sb)
 {
+  lj_assertG(!sbufisext(sb), "bad free of SBufExt");
   lj_mem_free(g, sb->b, sbufsz(sb));
 }
 
@@ -62,6 +103,46 @@ static LJ_AINLINE char *lj_buf_more(SBuf *sb, MSize sz)
   return sb->w;
 }
 
+/* Extended buffer management */
+static LJ_AINLINE void lj_bufx_init(lua_State *L, SBufExt *sbx)
+{
+  memset(sbx, 0, sizeof(SBufExt));
+  setsbufXL(sbx, L, SBUF_FLAG_EXT);
+}
+
+static LJ_AINLINE void lj_bufx_init_borrow(lua_State *L, SBufExt *sbx, SBuf *sb)
+{
+  memset(sbx, 0, sizeof(SBufExt));
+  setsbufXL(sbx, L, SBUF_FLAG_EXT | SBUF_FLAG_BORROW);
+  setmref(sbx->bsb, sb);
+  sbx->r = sbx->w = sbx->b = sb->b;
+  sbx->e = sb->e;
+}
+
+static LJ_AINLINE void lj_bufx_init_cow(lua_State *L, SBufExt *sbx,
+                                        const char *p, MSize len)
+{
+  memset(sbx, 0, sizeof(SBufExt));
+  setsbufXL(sbx, L, SBUF_FLAG_EXT | SBUF_FLAG_COW);
+  sbx->r = sbx->b = (char *)p;
+  sbx->w = sbx->e = (char *)p + len;
+}
+
+static LJ_AINLINE void lj_bufx_reset(SBufExt *sbx)
+{
+  if (sbufiscow(sbx)) {
+    setmrefu(sbx->L, (mrefu(sbx->L) & ~(GCSize)SBUF_FLAG_COW));
+    setgcrefnull(sbx->cowref);
+    sbx->b = sbx->e = NULL;
+  }
+  sbx->r = sbx->w = sbx->b;
+}
+
+static LJ_AINLINE void lj_bufx_free(global_State *g, SBufExt *sbx)
+{
+  if (!sbufiscow(sbx)) lj_mem_free(g, sbx->b, sbufsz(sbx));
+}
+
 /* Low-level buffer put operations */
 LJ_FUNC SBuf *lj_buf_putmem(SBuf *sb, const void *q, MSize len);
 LJ_FUNC SBuf * LJ_FASTCALL lj_buf_putchar(SBuf *sb, int c);
@@ -97,11 +178,4 @@ static LJ_AINLINE GCstr *lj_buf_str(lua_State *L, SBuf *sb)
   return lj_str_new(L, sb->b, sbuflen(sb));
 }
 
-/* Interim user-accessible string buffer. */
-typedef struct StrBuf {
-  SBuf *sb;             /* Pointer to system buffer. */
-  char *r;              /* String buffer read pointer. */
-  int depth;            /* Remaining recursion depth. */
-} StrBuf;
-
 #endif
diff --git a/src/lj_obj.h b/src/lj_obj.h
index 2150e4e2..9b691e49 100644
--- a/src/lj_obj.h
+++ b/src/lj_obj.h
@@ -34,13 +34,17 @@ typedef struct MRef {
 
 #if LJ_GC64
 #define mref(r, t)      ((t *)(void *)(r).ptr64)
+#define mrefu(r)        ((r).ptr64)
 
 #define setmref(r, p)   ((r).ptr64 = (uint64_t)(void *)(p))
+#define setmrefu(r, u)  ((r).ptr64 = (uint64_t)(u))
 #define setmrefr(r, v)  ((r).ptr64 = (v).ptr64)
 #else
 #define mref(r, t)      ((t *)(void *)(uintptr_t)(r).ptr32)
+#define mrefu(r)        ((r).ptr32)
 
 #define setmref(r, p)   ((r).ptr32 = (uint32_t)(uintptr_t)(void *)(p))
+#define setmrefu(r, u)  ((r).ptr32 = (uint32_t)(u))
 #define setmrefr(r, v)  ((r).ptr32 = (v).ptr32)
 #endif
 
diff --git a/src/lj_serialize.c b/src/lj_serialize.c
index 13220c17..4e76502a 100644
--- a/src/lj_serialize.c
+++ b/src/lj_serialize.c
@@ -55,11 +55,11 @@ LJ_STATIC_ASSERT((SER_TAG_TAB & 7) == 0);
 
 /* -- Helper functions ---------------------------------------------------- */
 
-static LJ_AINLINE char *serialize_more(char *w, StrBuf *sbuf, MSize sz)
+static LJ_AINLINE char *serialize_more(char *w, SBufExt *sbx, MSize sz)
 {
-  if (LJ_UNLIKELY(sz > (MSize)(sbuf->sb->e - w))) {
-    sbuf->sb->w = w;
-    w = lj_buf_more2(sbuf->sb, sz);
+  if (LJ_UNLIKELY(sz > (MSize)(sbx->e - w))) {
+    sbx->w = w;
+    w = lj_buf_more2((SBuf *)sbx, sz);
   }
   return w;
 }
@@ -90,14 +90,14 @@ static LJ_AINLINE char *serialize_wu124(char *w, uint32_t v)
   }
 }
 
-static LJ_NOINLINE char *serialize_ru124_(char *r, char *e, uint32_t *pv)
+static LJ_NOINLINE char *serialize_ru124_(char *r, char *w, uint32_t *pv)
 {
   uint32_t v = *pv;
   if (v != 0xff) {
-    if (r >= e) return NULL;
+    if (r >= w) return NULL;
     v = ((v & 0x1f) << 8) + *(uint8_t *)r + 0xe0; r++;
   } else {
-    if (r + 4 > e) return NULL;
+    if (r + 4 > w) return NULL;
     v = lj_getu32(r); r += 4;
 #if LJ_BE
     v = lj_bswap(v);
@@ -107,13 +107,13 @@ static LJ_NOINLINE char *serialize_ru124_(char *r, char *e, uint32_t *pv)
   return r;
 }
 
-static LJ_AINLINE char *serialize_ru124(char *r, char *e, uint32_t *pv)
+static LJ_AINLINE char *serialize_ru124(char *r, char *w, uint32_t *pv)
 {
-  if (LJ_LIKELY(r < e)) {
+  if (LJ_LIKELY(r < w)) {
     uint32_t v = *(uint8_t *)r; r++;
     *pv = v;
     if (LJ_UNLIKELY(v >= 0xe0)) {
-      r = serialize_ru124_(r, e, pv);
+      r = serialize_ru124_(r, w, pv);
     }
     return r;
   }
@@ -123,30 +123,30 @@ static LJ_AINLINE char *serialize_ru124(char *r, char *e, uint32_t *pv)
 /* -- Internal serializer ------------------------------------------------- */
 
 /* Put serialized object into buffer. */
-static char *serialize_put(char *w, StrBuf *sbuf, cTValue *o)
+static char *serialize_put(char *w, SBufExt *sbx, cTValue *o)
 {
   if (LJ_LIKELY(tvisstr(o))) {
     const GCstr *str = strV(o);
     MSize len = str->len;
-    w = serialize_more(w, sbuf, 5+len);
+    w = serialize_more(w, sbx, 5+len);
     w = serialize_wu124(w, SER_TAG_STR + len);
     w = lj_buf_wmem(w, strdata(str), len);
   } else if (tvisint(o)) {
     uint32_t x = LJ_BE ? lj_bswap((uint32_t)intV(o)) : (uint32_t)intV(o);
-    w = serialize_more(w, sbuf, 1+4);
+    w = serialize_more(w, sbx, 1+4);
     *w++ = SER_TAG_INT; memcpy(w, &x, 4); w += 4;
   } else if (tvisnum(o)) {
     uint64_t x = LJ_BE ? lj_bswap64(o->u64) : o->u64;
-    w = serialize_more(w, sbuf, 1+sizeof(lua_Number));
+    w = serialize_more(w, sbx, 1+sizeof(lua_Number));
     *w++ = SER_TAG_NUM; memcpy(w, &x, 8); w += 8;
   } else if (tvispri(o)) {
-    w = serialize_more(w, sbuf, 1);
+    w = serialize_more(w, sbx, 1);
     *w++ = (char)(SER_TAG_NIL + ~itype(o));
   } else if (tvistab(o)) {
     const GCtab *t = tabV(o);
     uint32_t narray = 0, nhash = 0, one = 2;
-    if (sbuf->depth <= 0) lj_err_caller(sbufL(sbuf->sb), LJ_ERR_BUFFER_DEPTH);
-    sbuf->depth--;
+    if (sbx->depth <= 0) lj_err_caller(sbufL(sbx), LJ_ERR_BUFFER_DEPTH);
+    sbx->depth--;
     if (t->asize > 0) {  /* Determine max. length of array part. */
       ptrdiff_t i;
       TValue *array = tvref(t->array);
@@ -163,32 +163,32 @@ static char *serialize_put(char *w, StrBuf *sbuf, cTValue *o)
         nhash += !tvisnil(&node[i].val);
     }
     /* Write number of array slots and hash slots. */
-    w = serialize_more(w, sbuf, 1+2*5);
+    w = serialize_more(w, sbx, 1+2*5);
     *w++ = (char)(SER_TAG_TAB + (nhash ? 1 : 0) + (narray ? one : 0));
     if (narray) w = serialize_wu124(w, narray);
     if (nhash) w = serialize_wu124(w, nhash);
     if (narray) {  /* Write array entries. */
       cTValue *oa = tvref(t->array) + (one >> 2);
       cTValue *oe = tvref(t->array) + narray;
-      while (oa < oe) w = serialize_put(w, sbuf, oa++);
+      while (oa < oe) w = serialize_put(w, sbx, oa++);
     }
     if (nhash) {  /* Write hash entries. */
       const Node *node = noderef(t->node) + t->hmask;
       for (;; node--)
         if (!tvisnil(&node->val)) {
-          w = serialize_put(w, sbuf, &node->key);
-          w = serialize_put(w, sbuf, &node->val);
+          w = serialize_put(w, sbx, &node->key);
+          w = serialize_put(w, sbx, &node->val);
           if (--nhash == 0) break;
         }
     }
-    sbuf->depth++;
+    sbx->depth++;
 #if LJ_HASFFI
   } else if (tviscdata(o)) {
-    CTState *cts = ctype_cts(sbufL(sbuf->sb));
+    CTState *cts = ctype_cts(sbufL(sbx));
     CType *s = ctype_raw(cts, cdataV(o)->ctypeid);
     uint8_t *sp = cdataptr(cdataV(o));
     if (ctype_isinteger(s->info) && s->size == 8) {
-      w = serialize_more(w, sbuf, 1+8);
+      w = serialize_more(w, sbx, 1+8);
       *w++ = (s->info & CTF_UNSIGNED) ? SER_TAG_UINT64 : SER_TAG_INT64;
 #if LJ_BE
       { uint64_t u = lj_bswap64(*(uint64_t *)sp); memcpy(w, &u, 8); }
@@ -197,7 +197,7 @@ static char *serialize_put(char *w, StrBuf *sbuf, cTValue *o)
 #endif
       w += 8;
     } else if (ctype_iscomplex(s->info) && s->size == 16) {
-      w = serialize_more(w, sbuf, 1+16);
+      w = serialize_more(w, sbx, 1+16);
       *w++ = SER_TAG_COMPLEX;
 #if LJ_BE
       {  /* Only swap the doubles. The re/im order stays the same. */
@@ -213,8 +213,8 @@ static char *serialize_put(char *w, StrBuf *sbuf, cTValue *o)
     }
 #endif
   } else if (tvislightud(o)) {
-    uintptr_t ud = (uintptr_t)lightudV(G(sbufL(sbuf->sb)), o);
-    w = serialize_more(w, sbuf, 1+sizeof(ud));
+    uintptr_t ud = (uintptr_t)lightudV(G(sbufL(sbx)), o);
+    w = serialize_more(w, sbx, 1+sizeof(ud));
     if (ud == 0) {
       *w++ = SER_TAG_NULL;
     } else if (LJ_32 || checku32(ud)) {
@@ -237,28 +237,28 @@ static char *serialize_put(char *w, StrBuf *sbuf, cTValue *o)
 #if LJ_HASFFI
   badenc:
 #endif
-    lj_err_callerv(sbufL(sbuf->sb), LJ_ERR_BUFFER_BADENC, lj_typename(o));
+    lj_err_callerv(sbufL(sbx), LJ_ERR_BUFFER_BADENC, lj_typename(o));
   }
   return w;
 }
 
 /* Get serialized object from buffer. */
-static char *serialize_get(char *r, StrBuf *sbuf, TValue *o)
+static char *serialize_get(char *r, SBufExt *sbx, TValue *o)
 {
-  char *e = sbuf->sb->e;
+  char *w = sbx->w;
   uint32_t tp;
-  r = serialize_ru124(r, e, &tp); if (LJ_UNLIKELY(!r)) goto eob;
+  r = serialize_ru124(r, w, &tp); if (LJ_UNLIKELY(!r)) goto eob;
   if (LJ_LIKELY(tp >= SER_TAG_STR)) {
     uint32_t len = tp - SER_TAG_STR;
-    if (LJ_UNLIKELY(len > (uint32_t)(e - r))) goto eob;
-    setstrV(sbufL(sbuf->sb), o, lj_str_new(sbufL(sbuf->sb), r, len));
+    if (LJ_UNLIKELY(len > (uint32_t)(w - r))) goto eob;
+    setstrV(sbufL(sbx), o, lj_str_new(sbufL(sbx), r, len));
     r += len;
   } else if (tp == SER_TAG_INT) {
-    if (LJ_UNLIKELY(r + 4 > e)) goto eob;
+    if (LJ_UNLIKELY(r + 4 > w)) goto eob;
     setintV(o, (int32_t)(LJ_BE ? lj_bswap(lj_getu32(r)) : lj_getu32(r)));
     r += 4;
   } else if (tp == SER_TAG_NUM) {
-    if (LJ_UNLIKELY(r + 8 > e)) goto eob;
+    if (LJ_UNLIKELY(r + 8 > w)) goto eob;
     memcpy(o, r, 8); r += 8;
 #if LJ_BE
     o->u64 = lj_bswap64(o->u64);
@@ -270,34 +270,34 @@ static char *serialize_get(char *r, StrBuf *sbuf, TValue *o)
     uint32_t narray = 0, nhash = 0;
     GCtab *t;
     if (tp >= SER_TAG_TAB+2) {
-      r = serialize_ru124(r, e, &narray); if (LJ_UNLIKELY(!r)) goto eob;
+      r = serialize_ru124(r, w, &narray); if (LJ_UNLIKELY(!r)) goto eob;
     }
     if ((tp & 1)) {
-      r = serialize_ru124(r, e, &nhash); if (LJ_UNLIKELY(!r)) goto eob;
+      r = serialize_ru124(r, w, &nhash); if (LJ_UNLIKELY(!r)) goto eob;
     }
-    t = lj_tab_new(sbufL(sbuf->sb), narray, hsize2hbits(nhash));
-    settabV(sbufL(sbuf->sb), o, t);
+    t = lj_tab_new(sbufL(sbx), narray, hsize2hbits(nhash));
+    settabV(sbufL(sbx), o, t);
     if (narray) {
       TValue *oa = tvref(t->array) + (tp >= SER_TAG_TAB+4);
       TValue *oe = tvref(t->array) + narray;
-      while (oa < oe) r = serialize_get(r, sbuf, oa++);
+      while (oa < oe) r = serialize_get(r, sbx, oa++);
     }
     if (nhash) {
       do {
         TValue k, *v;
-        r = serialize_get(r, sbuf, &k);
-        v = lj_tab_set(sbufL(sbuf->sb), t, &k);
+        r = serialize_get(r, sbx, &k);
+        v = lj_tab_set(sbufL(sbx), t, &k);
         if (LJ_UNLIKELY(!tvisnil(v)))
-          lj_err_caller(sbufL(sbuf->sb), LJ_ERR_BUFFER_DUPKEY);
-        r = serialize_get(r, sbuf, v);
+          lj_err_caller(sbufL(sbx), LJ_ERR_BUFFER_DUPKEY);
+        r = serialize_get(r, sbx, v);
       } while (--nhash);
     }
 #if LJ_HASFFI
   } else if (tp >= SER_TAG_INT64 &&  tp <= SER_TAG_COMPLEX) {
     uint32_t sz = tp == SER_TAG_COMPLEX ? 16 : 8;
     GCcdata *cd;
-    if (LJ_UNLIKELY(r + sz > e)) goto eob;
-    cd = lj_cdata_new_(sbufL(sbuf->sb),
+    if (LJ_UNLIKELY(r + sz > w)) goto eob;
+    cd = lj_cdata_new_(sbufL(sbx),
            tp == SER_TAG_INT64 ? CTID_INT64 :
            tp == SER_TAG_UINT64 ? CTID_UINT64 : CTID_COMPLEX_DOUBLE,
            sz);
@@ -307,50 +307,50 @@ static char *serialize_get(char *r, StrBuf *sbuf, TValue *o)
     if (sz == 16)
       ((uint64_t *)cdataptr(cd))[1] = lj_bswap64(((uint64_t *)cdataptr(cd))[1]);
 #endif
-    setcdataV(sbufL(sbuf->sb), o, cd);
+    setcdataV(sbufL(sbx), o, cd);
 #endif
   } else if (tp <= (LJ_64 ? SER_TAG_LIGHTUD64 : SER_TAG_LIGHTUD32)) {
     uintptr_t ud = 0;
     if (tp == SER_TAG_LIGHTUD32) {
-      if (LJ_UNLIKELY(r + 4 > e)) goto eob;
+      if (LJ_UNLIKELY(r + 4 > w)) goto eob;
       ud = (uintptr_t)(LJ_BE ? lj_bswap(lj_getu32(r)) : lj_getu32(r));
       r += 4;
     }
 #if LJ_64
     else if (tp == SER_TAG_LIGHTUD64) {
-      if (LJ_UNLIKELY(r + 8 > e)) goto eob;
+      if (LJ_UNLIKELY(r + 8 > w)) goto eob;
       memcpy(&ud, r, 8); r += 8;
 #if LJ_BE
       ud = lj_bswap64(ud);
 #endif
     }
-    setrawlightudV(o, lj_lightud_intern(sbufL(sbuf->sb), (void *)ud));
+    setrawlightudV(o, lj_lightud_intern(sbufL(sbx), (void *)ud));
 #else
     setrawlightudV(o, (void *)ud);
 #endif
   } else {
-    lj_err_callerv(sbufL(sbuf->sb), LJ_ERR_BUFFER_BADDEC, tp);
+    lj_err_callerv(sbufL(sbx), LJ_ERR_BUFFER_BADDEC, tp);
   }
   return r;
 eob:
-  lj_err_caller(sbufL(sbuf->sb), LJ_ERR_BUFFER_EOB);
+  lj_err_caller(sbufL(sbx), LJ_ERR_BUFFER_EOB);
   return NULL;
 }
 
-StrBuf * LJ_FASTCALL lj_serialize_put(StrBuf *sbuf, cTValue *o)
+SBufExt * LJ_FASTCALL lj_serialize_put(SBufExt *sbx, cTValue *o)
 {
-  sbuf->depth = LJ_SERIALIZE_DEPTH;
-  sbuf->sb->w = serialize_put(sbuf->sb->w, sbuf, o);
-  return sbuf;
+  sbx->depth = LJ_SERIALIZE_DEPTH;
+  sbx->w = serialize_put(sbx->w, sbx, o);
+  return sbx;
 }
 
-StrBuf * LJ_FASTCALL lj_serialize_get(StrBuf *sbuf, TValue *o)
+SBufExt * LJ_FASTCALL lj_serialize_get(SBufExt *sbx, TValue *o)
 {
-  char *r = serialize_get(sbuf->r, sbuf, o);
-  if (r != sbuf->sb->w)
-    lj_err_caller(sbufL(sbuf->sb), LJ_ERR_BUFFER_LEFTOV);
-  sbuf->r = r;
-  return sbuf;
+  char *r = serialize_get(sbx->r, sbx, o);
+  if (r != sbx->w)
+    lj_err_caller(sbufL(sbx), LJ_ERR_BUFFER_LEFTOV);
+  sbx->r = r;
+  return sbx;
 }
 
 #endif
diff --git a/src/lj_serialize.h b/src/lj_serialize.h
index 95d62f4e..f5617790 100644
--- a/src/lj_serialize.h
+++ b/src/lj_serialize.h
@@ -13,8 +13,8 @@
 
 #define LJ_SERIALIZE_DEPTH      100     /* Default depth. */
 
-LJ_FUNC StrBuf * LJ_FASTCALL lj_serialize_put(StrBuf *sb, cTValue *o);
-LJ_FUNC StrBuf * LJ_FASTCALL lj_serialize_get(StrBuf *sb, TValue *o);
+LJ_FUNC SBufExt * LJ_FASTCALL lj_serialize_put(SBufExt *sbx, cTValue *o);
+LJ_FUNC SBufExt * LJ_FASTCALL lj_serialize_get(SBufExt *sbx, TValue *o);
 
 #endif