new macros 'likely'/'unlikely' with hints for jump predictions

(used only in errors for now)

6 files changed, 78 insertions, 48 deletions

diff --git a/ldo.c b/ldo.c
index 169eb714..fb077211 100644
--- a/ldo.c
+++ b/ldo.c
@@ -1,5 +1,5 @@
 /*
-** $Id: ldo.c,v 2.200 2018/03/16 15:33:34 roberto Exp roberto $
+** $Id: ldo.c,v 2.201 2018/05/22 12:02:36 roberto Exp roberto $
 ** Stack and Call structure of Lua
 ** See Copyright Notice in lua.h
 */
@@ -182,7 +182,7 @@ int luaD_reallocstack (lua_State *L, int newsize, int raiseerror) {
   StkId newstack = luaM_reallocvector(L, L->stack, lim, newsize, StackValue);
   lua_assert(newsize <= LUAI_MAXSTACK || newsize == ERRORSTACKSIZE);
   lua_assert(L->stack_last - L->stack == L->stacksize - EXTRA_STACK);
-  if (newstack == NULL) {  /* reallocation failed? */
+  if (unlikely(newstack == NULL)) {  /* reallocation failed? */
     if (raiseerror)
       luaM_error(L);
     else return 0;  /* do not raise an error */
@@ -204,7 +204,7 @@ int luaD_reallocstack (lua_State *L, int newsize, int raiseerror) {
 int luaD_growstack (lua_State *L, int n, int raiseerror) {
   int size = L->stacksize;
   int newsize = 2 * size;  /* tentative new size */
-  if (size > LUAI_MAXSTACK) {  /* need more space after extra size? */
+  if (unlikely(size > LUAI_MAXSTACK)) {  /* need more space after extra size? */
     if (raiseerror)
       luaD_throw(L, LUA_ERRERR);  /* error inside message handler */
     else return 0;
@@ -215,7 +215,7 @@ int luaD_growstack (lua_State *L, int n, int raiseerror) {
       newsize = LUAI_MAXSTACK;
     if (newsize < needed)  /* but must respect what was asked for */
       newsize = needed;
-    if (newsize > LUAI_MAXSTACK) {  /* stack overflow? */
+    if (unlikely(newsize > LUAI_MAXSTACK)) {  /* stack overflow? */
       /* add extra size to be able to handle the error message */
       luaD_reallocstack(L, ERRORSTACKSIZE, raiseerror);
       if (raiseerror)
@@ -350,7 +350,7 @@ static StkId rethook (lua_State *L, CallInfo *ci, StkId firstres, int nres) {
 void luaD_tryfuncTM (lua_State *L, StkId func) {
   const TValue *tm = luaT_gettmbyobj(L, s2v(func), TM_CALL);
   StkId p;
-  if (!ttisfunction(tm))
+  if (unlikely(!ttisfunction(tm)))
     luaG_typeerror(L, s2v(func), "call");
   for (p = L->top; p > func; p--)
     setobjs2s(L, p, p-1);
@@ -660,14 +660,14 @@ LUA_API int lua_resume (lua_State *L, lua_State *from, int nargs,
   L->nny = 0;  /* allow yields */
   api_checknelems(L, (L->status == LUA_OK) ? nargs + 1 : nargs);
   status = luaD_rawrunprotected(L, resume, &nargs);
-  if (status == -1)  /* error calling 'lua_resume'? */
+  if (unlikely(status == -1))  /* error calling 'lua_resume'? */
     status = LUA_ERRRUN;
   else {  /* continue running after recoverable errors */
     while (errorstatus(status) && recover(L, status)) {
       /* unroll continuation */
       status = luaD_rawrunprotected(L, unroll, &status);
     }
-    if (errorstatus(status)) {  /* unrecoverable error? */
+    if (unlikely(errorstatus(status))) {  /* unrecoverable error? */
       L->status = cast_byte(status);  /* mark thread as 'dead' */
       seterrorobj(L, status, L->top);  /* push error message */
       L->ci->top = L->top;
@@ -694,7 +694,7 @@ LUA_API int lua_yieldk (lua_State *L, int nresults, lua_KContext ctx,
   luai_userstateyield(L, nresults);
   lua_lock(L);
   api_checknelems(L, nresults);
-  if (L->nny > 0) {
+  if (unlikely(L->nny > 0)) {
     if (L != G(L)->mainthread)
       luaG_runerror(L, "attempt to yield across a C-call boundary");
     else
@@ -727,7 +727,7 @@ int luaD_pcall (lua_State *L, Pfunc func, void *u,
   ptrdiff_t old_errfunc = L->errfunc;
   L->errfunc = ef;
   status = luaD_rawrunprotected(L, func, u);
-  if (status != LUA_OK) {  /* an error occurred? */
+  if (unlikely(status != LUA_OK)) {  /* an error occurred? */
     StkId oldtop = restorestack(L, old_top);
     luaF_close(L, oldtop);  /* close possible pending closures */
     seterrorobj(L, status, oldtop);
diff --git a/llimits.h b/llimits.h
index 9a3ae8d0..725d7c8b 100644
--- a/llimits.h
+++ b/llimits.h
@@ -1,5 +1,5 @@
 /*
-** $Id: llimits.h,v 1.148 2017/12/28 11:51:00 roberto Exp roberto $
+** $Id: llimits.h,v 1.149 2018/01/28 15:13:26 roberto Exp roberto $
 ** Limits, basic types, and some other 'installation-dependent' definitions
 ** See Copyright Notice in lua.h
 */
@@ -131,8 +131,26 @@ typedef LUAI_UACINT l_uacInt;
 
 
 /*
+** macros to improve jump prediction (used mainly for error handling)
+*/
+#if !defined(likely)
+
+#if defined(__GNUC__)
+#define likely(x)       (__builtin_expect(((x) != 0), 1))
+#define unlikely(x)     (__builtin_expect(((x) != 0), 0))
+#else
+#define likely(x)       (x)
+#define unlikely(x)     (x)
+#endif
+
+#endif
+
+
+/*
 ** non-return type
 */
+#if !defined(l_noret)
+
 #if defined(__GNUC__)
 #define l_noret         void __attribute__((noreturn))
 #elif defined(_MSC_VER) && _MSC_VER >= 1200
@@ -141,6 +159,7 @@ typedef LUAI_UACINT l_uacInt;
 #define l_noret         void
 #endif
 
+#endif
 
 
 /*
diff --git a/lmem.c b/lmem.c
index 2c1757f5..5c73acd5 100644
--- a/lmem.c
+++ b/lmem.c
@@ -1,5 +1,5 @@
 /*
-** $Id: lmem.c,v 1.95 2017/12/11 12:27:48 roberto Exp roberto $
+** $Id: lmem.c,v 1.96 2018/01/28 15:13:26 roberto Exp roberto $
 ** Interface to Memory Manager
 ** See Copyright Notice in lua.h
 */
@@ -60,7 +60,7 @@ void *luaM_growaux_ (lua_State *L, void *block, int nelems, int *psize,
   if (nelems + 1 <= size)  /* does one extra element still fit? */
     return block;  /* nothing to be done */
   if (size >= limit / 2) {  /* cannot double it? */
-    if (size >= limit)  /* cannot grow even a little? */
+    if (unlikely(size >= limit))  /* cannot grow even a little? */
       luaG_runerror(L, "too many %s (limit is %d)", what, limit);
     size = limit;  /* still have at least one free place */
   }
@@ -73,7 +73,7 @@ void *luaM_growaux_ (lua_State *L, void *block, int nelems, int *psize,
   /* 'limit' ensures that multiplication will not overflow */
   newblock = luaM_realloc_(L, block, cast_sizet(*psize) * size_elems,
                                      cast_sizet(size) * size_elems);
-  if (newblock == NULL)
+  if (unlikely(newblock == NULL))
     luaM_error(L);
   *psize = size;  /* update only when everything else is OK */
   return newblock;
@@ -88,7 +88,7 @@ void *luaM_shrinkvector_ (lua_State *L, void *block, int *size,
   size_t newsize = cast_sizet(final_n * size_elem);
   lua_assert(newsize <= oldsize);
   newblock = (*g->frealloc)(g->ud, block, oldsize, newsize);
-  if (newblock == NULL && final_n > 0)  /* allocation failed? */
+  if (unlikely(newblock == NULL && final_n > 0))  /* allocation failed? */
     luaM_error(L);
   else {
     g->GCdebt += newsize - oldsize;
@@ -114,6 +114,22 @@ void luaM_free_ (lua_State *L, void *block, size_t osize) {
 }
 
 
+/*
+** In case of allocation fail, this function will call the GC to try
+** to free some memory and then try the allocation again.
+** (It should not be called when shrinking a block, because then the
+** interpreter may be in the middle of a collection step.)
+*/
+static void *tryagain (lua_State *L, void *block,
+                       size_t osize, size_t nsize) {
+  global_State *g = G(L);
+  if (g->version) {  /* is state fully build? */
+    luaC_fullgc(L, 1);  /* try to free some memory... */
+    return (*g->frealloc)(g->ud, block, osize, nsize);  /* try again */
+  }
+  else return NULL;  /* cannot free any memory without a full state */
+}
+
 
 /*
 ** generic allocation routine.
@@ -124,13 +140,10 @@ void *luaM_realloc_ (lua_State *L, void *block, size_t osize, size_t nsize) {
   lua_assert((osize == 0) == (block == NULL));
   hardtest(L, osize, nsize);
   newblock = (*g->frealloc)(g->ud, block, osize, nsize);
-  if (newblock == NULL && nsize > 0) {
-    /* Is state fully built? Not shrinking a block? */
-    if (g->version && nsize > osize) {
-      luaC_fullgc(L, 1);  /* try to free some memory... */
-      newblock = (*g->frealloc)(g->ud, block, osize, nsize);  /* try again */
-    }
-    if (newblock == NULL)
+  if (unlikely(newblock == NULL && nsize > 0)) {
+    if (nsize > osize)  /* not shrinking a block? */
+      newblock = tryagain(L, block, osize, nsize);
+    if (newblock == NULL)  /* still no memory? */
       return NULL;
   }
   lua_assert((nsize == 0) == (newblock == NULL));
@@ -142,7 +155,7 @@ void *luaM_realloc_ (lua_State *L, void *block, size_t osize, size_t nsize) {
 void *luaM_saferealloc_ (lua_State *L, void *block, size_t osize,
                                                     size_t nsize) {
   void *newblock = luaM_realloc_(L, block, osize, nsize);
-  if (newblock == NULL && nsize > 0)  /* allocation failed? */
+  if (unlikely(newblock == NULL && nsize > 0))  /* allocation failed? */
     luaM_error(L);
   return newblock;
 }
@@ -155,11 +168,8 @@ void *luaM_malloc_ (lua_State *L, size_t size, int tag) {
   else {
     global_State *g = G(L);
     void *newblock = (*g->frealloc)(g->ud, NULL, tag, size);
-    if (newblock == NULL) {
-      if (g->version) {  /* is state fully built? */
-        luaC_fullgc(L, 1);  /* try to free some memory... */
-        newblock = (*g->frealloc)(g->ud, NULL, tag, size);  /* try again */
-      }
+    if (unlikely(newblock == NULL)) {
+      newblock = tryagain(L, NULL, tag, size);
       if (newblock == NULL)
         luaM_error(L);
     }
diff --git a/lstring.c b/lstring.c
index 29a08212..f1e5d82b 100644
--- a/lstring.c
+++ b/lstring.c
@@ -1,5 +1,5 @@
 /*
-** $Id: lstring.c,v 2.64 2018/02/15 18:06:24 roberto Exp roberto $
+** $Id: lstring.c,v 2.65 2018/02/20 16:52:50 roberto Exp roberto $
 ** String table (keeps all strings handled by Lua)
 ** See Copyright Notice in lua.h
 */
@@ -99,7 +99,7 @@ void luaS_resize (lua_State *L, int nsize) {
   if (nsize < osize)  /* shrinking table? */
     tablerehash(tb->hash, osize, nsize);  /* depopulate shrinking part */
   newvect = luaM_reallocvector(L, tb->hash, osize, nsize, TString*);
-  if (newvect == NULL) {  /* reallocation failed? */
+  if (unlikely(newvect == NULL)) {  /* reallocation failed? */
     if (nsize < osize)  /* was it shrinking table? */
       tablerehash(tb->hash, nsize, osize);  /* restore to original size */
     /* leave table as it was */
@@ -182,7 +182,7 @@ void luaS_remove (lua_State *L, TString *ts) {
 
 
 static void growstrtab (lua_State *L, stringtable *tb) {
-  if (tb->nuse == MAX_INT) {  /* too many strings? */
+  if (unlikely(tb->nuse == MAX_INT)) {  /* too many strings? */
     luaC_fullgc(L, 1);  /* try to free some... */
     if (tb->nuse == MAX_INT)  /* still too many? */
       luaM_error(L);  /* cannot even create a message... */
@@ -233,7 +233,7 @@ TString *luaS_newlstr (lua_State *L, const char *str, size_t l) {
     return internshrstr(L, str, l);
   else {
     TString *ts;
-    if (l >= (MAX_SIZE - sizeof(TString))/sizeof(char))
+    if (unlikely(l >= (MAX_SIZE - sizeof(TString))/sizeof(char)))
       luaM_toobig(L);
     ts = luaS_createlngstrobj(L, l);
     memcpy(getstr(ts), str, l * sizeof(char));
@@ -269,7 +269,7 @@ Udata *luaS_newudata (lua_State *L, size_t s, int nuvalue) {
   Udata *u;
   int i;
   GCObject *o;
-  if (s > MAX_SIZE - udatamemoffset(nuvalue))
+  if (unlikely(s > MAX_SIZE - udatamemoffset(nuvalue)))
     luaM_toobig(L);
   o = luaC_newobj(L, LUA_TUSERDATA, sizeudata(nuvalue, s));
   u = gco2u(o);
diff --git a/ltable.c b/ltable.c
index 7113f0ff..56fe64fd 100644
--- a/ltable.c
+++ b/ltable.c
@@ -1,5 +1,5 @@
 /*
-** $Id: ltable.c,v 2.135 2018/02/26 14:16:05 roberto Exp roberto $
+** $Id: ltable.c,v 2.136 2018/05/29 18:01:50 roberto Exp roberto $
 ** Lua tables (hash)
 ** See Copyright Notice in lua.h
 */
@@ -235,7 +235,7 @@ static unsigned int findindex (lua_State *L, Table *t, TValue *key) {
     return i;  /* yes; that's the index */
   else {
     const TValue *n = getgeneric(t, key);
-    if (n == luaH_emptyobject)
+    if (unlikely(n == luaH_emptyobject))
       luaG_runerror(L, "invalid key to 'next'");  /* key not found */
     i = cast_int(nodefromval(n) - gnode(t, 0));  /* key index in hash table */
     /* hash elements are numbered after array ones */
@@ -467,7 +467,7 @@ void luaH_resize (lua_State *L, Table *t, unsigned int newasize,
   }
   /* allocate new array */
   newarray = luaM_reallocvector(L, t->array, oldasize, newasize, TValue);
-  if (newarray == NULL && newasize > 0) {  /* allocation failed? */
+  if (unlikely(newarray == NULL && newasize > 0)) {  /* allocation failed? */
     freehash(L, &newt);  /* release new hash part */
     luaM_error(L);  /* raise error (with array unchanged) */
   }
@@ -560,7 +560,8 @@ static Node *getfreepos (Table *t) {
 TValue *luaH_newkey (lua_State *L, Table *t, const TValue *key) {
   Node *mp;
   TValue aux;
-  if (ttisnil(key)) luaG_runerror(L, "table index is nil");
+  if (unlikely(ttisnil(key)))
+    luaG_runerror(L, "table index is nil");
   else if (ttisfloat(key)) {
     lua_Number f = fltvalue(key);
     lua_Integer k;
@@ -568,7 +569,7 @@ TValue *luaH_newkey (lua_State *L, Table *t, const TValue *key) {
       setivalue(&aux, k);
       key = &aux;  /* insert it as an integer */
     }
-    else if (luai_numisnan(f))
+    else if (unlikely(luai_numisnan(f)))
       luaG_runerror(L, "table index is NaN");
   }
   mp = mainpositionTV(t, key);
diff --git a/lvm.c b/lvm.c
index 4406afb6..36c7699f 100644
--- a/lvm.c
+++ b/lvm.c
@@ -1,5 +1,5 @@
 /*
-** $Id: lvm.c,v 2.354 2018/05/02 18:17:59 roberto Exp roberto $
+** $Id: lvm.c,v 2.355 2018/05/22 12:02:36 roberto Exp roberto $
 ** Lua virtual machine
 ** See Copyright Notice in lua.h
 */
@@ -196,7 +196,7 @@ void luaV_finishget (lua_State *L, const TValue *t, TValue *key, StkId val,
     if (slot == NULL) {  /* 't' is not a table? */
       lua_assert(!ttistable(t));
       tm = luaT_gettmbyobj(L, t, TM_INDEX);
-      if (notm(tm))
+      if (unlikely(notm(tm)))
         luaG_typeerror(L, t, "index");  /* no metamethod */
       /* else will try the metamethod */
     }
@@ -253,7 +253,7 @@ void luaV_finishset (lua_State *L, const TValue *t, TValue *key,
     }
     else {  /* not a table; check metamethod */
       tm = luaT_gettmbyobj(L, t, TM_NEWINDEX);
-      if (notm(tm))
+      if (unlikely(notm(tm)))
         luaG_typeerror(L, t, "index");
     }
     /* try the metamethod */
@@ -561,7 +561,7 @@ void luaV_concat (lua_State *L, int total) {
       /* collect total length and number of strings */
       for (n = 1; n < total && tostring(L, s2v(top - n - 1)); n++) {
         size_t l = vslen(s2v(top - n - 1));
-        if (l >= (MAX_SIZE/sizeof(char)) - tl)
+        if (unlikely(l >= (MAX_SIZE/sizeof(char)) - tl))
           luaG_runerror(L, "string length overflow");
         tl += l;
       }
@@ -605,7 +605,7 @@ void luaV_objlen (lua_State *L, StkId ra, const TValue *rb) {
     }
     default: {  /* try metamethod */
       tm = luaT_gettmbyobj(L, rb, TM_LEN);
-      if (notm(tm))  /* no metamethod? */
+      if (unlikely(notm(tm)))  /* no metamethod? */
         luaG_typeerror(L, rb, "get length of");
       break;
     }
@@ -622,7 +622,7 @@ void luaV_objlen (lua_State *L, StkId ra, const TValue *rb) {
 */
 lua_Integer luaV_div (lua_State *L, lua_Integer m, lua_Integer n) {
   if (l_castS2U(n) + 1u <= 1u) {  /* special cases: -1 or 0 */
-    if (n == 0)
+    if (unlikely(n == 0))
       luaG_runerror(L, "attempt to divide by zero");
     return intop(-, 0, m);   /* n==-1; avoid overflow with 0x80000...//-1 */
   }
@@ -642,7 +642,7 @@ lua_Integer luaV_div (lua_State *L, lua_Integer m, lua_Integer n) {
 */
 lua_Integer luaV_mod (lua_State *L, lua_Integer m, lua_Integer n) {
   if (l_castS2U(n) + 1u <= 1u) {  /* special cases: -1 or 0 */
-    if (n == 0)
+    if (unlikely(n == 0))
       luaG_runerror(L, "attempt to perform 'n%%0'");
     return 0;   /* m % -1 == 0; avoid overflow with 0x80000...%-1 */
   }
@@ -1665,7 +1665,7 @@ void luaV_execute (lua_State *L, CallInfo *ci) {
         TValue *plimit = s2v(ra + 1);
         lua_Integer ilimit, initv;
         int stopnow;
-        if (!forlimit(plimit, &ilimit, 1, &stopnow)) {
+        if (unlikely(!forlimit(plimit, &ilimit, 1, &stopnow))) {
             savestate(L, ci);  /* for the error message */
             luaG_runerror(L, "'for' limit must be a number");
         }
@@ -1717,13 +1717,13 @@ void luaV_execute (lua_State *L, CallInfo *ci) {
         else {  /* try making all values floats */
           lua_Number ninit; lua_Number nlimit; lua_Number nstep;
           savestate(L, ci);  /* in case of errors */
-          if (!tonumber(plimit, &nlimit))
+          if (unlikely(!tonumber(plimit, &nlimit)))
             luaG_runerror(L, "'for' limit must be a number");
           setfltvalue(plimit, nlimit);
-          if (!tonumber(pstep, &nstep))
+          if (unlikely(!tonumber(pstep, &nstep)))
             luaG_runerror(L, "'for' step must be a number");
           setfltvalue(pstep, nstep);
-          if (!tonumber(init, &ninit))
+          if (unlikely(!tonumber(init, &ninit)))
             luaG_runerror(L, "'for' initial value must be a number");
           setfltvalue(init, luai_numsub(L, ninit, nstep));
         }