From 34aa0c5bd7493b6e01983df28f04af46a3d99967 Mon Sep 17 00:00:00 2001
From: Roberto Ierusalimschy <roberto@inf.puc-rio.br>
Date: Wed, 30 May 2018 11:25:52 -0300
Subject: new macros 'likely'/'unlikely' with hints for jump predictions (used
only in errors for now)
---
ldo.c | 18 +++++++++---------
llimits.h | 21 ++++++++++++++++++++-
lmem.c | 44 +++++++++++++++++++++++++++-----------------
lstring.c | 10 +++++-----
ltable.c | 11 ++++++-----
lvm.c | 22 +++++++++++-----------
6 files changed, 78 insertions(+), 48 deletions(-)
diff --git a/ldo.c b/ldo.c
index 169eb714..fb077211 100644
--- a/ldo.c
+++ b/ldo.c
@@ -1,5 +1,5 @@
/*
-** $Id: ldo.c,v 2.200 2018/03/16 15:33:34 roberto Exp roberto $
+** $Id: ldo.c,v 2.201 2018/05/22 12:02:36 roberto Exp roberto $
** Stack and Call structure of Lua
** See Copyright Notice in lua.h
*/
@@ -182,7 +182,7 @@ int luaD_reallocstack (lua_State *L, int newsize, int raiseerror) {
StkId newstack = luaM_reallocvector(L, L->stack, lim, newsize, StackValue);
lua_assert(newsize <= LUAI_MAXSTACK || newsize == ERRORSTACKSIZE);
lua_assert(L->stack_last - L->stack == L->stacksize - EXTRA_STACK);
- if (newstack == NULL) { /* reallocation failed? */
+ if (unlikely(newstack == NULL)) { /* reallocation failed? */
if (raiseerror)
luaM_error(L);
else return 0; /* do not raise an error */
@@ -204,7 +204,7 @@ int luaD_reallocstack (lua_State *L, int newsize, int raiseerror) {
int luaD_growstack (lua_State *L, int n, int raiseerror) {
int size = L->stacksize;
int newsize = 2 * size; /* tentative new size */
- if (size > LUAI_MAXSTACK) { /* need more space after extra size? */
+ if (unlikely(size > LUAI_MAXSTACK)) { /* need more space after extra size? */
if (raiseerror)
luaD_throw(L, LUA_ERRERR); /* error inside message handler */
else return 0;
@@ -215,7 +215,7 @@ int luaD_growstack (lua_State *L, int n, int raiseerror) {
newsize = LUAI_MAXSTACK;
if (newsize < needed) /* but must respect what was asked for */
newsize = needed;
- if (newsize > LUAI_MAXSTACK) { /* stack overflow? */
+ if (unlikely(newsize > LUAI_MAXSTACK)) { /* stack overflow? */
/* add extra size to be able to handle the error message */
luaD_reallocstack(L, ERRORSTACKSIZE, raiseerror);
if (raiseerror)
@@ -350,7 +350,7 @@ static StkId rethook (lua_State *L, CallInfo *ci, StkId firstres, int nres) {
void luaD_tryfuncTM (lua_State *L, StkId func) {
const TValue *tm = luaT_gettmbyobj(L, s2v(func), TM_CALL);
StkId p;
- if (!ttisfunction(tm))
+ if (unlikely(!ttisfunction(tm)))
luaG_typeerror(L, s2v(func), "call");
for (p = L->top; p > func; p--)
setobjs2s(L, p, p-1);
@@ -660,14 +660,14 @@ LUA_API int lua_resume (lua_State *L, lua_State *from, int nargs,
L->nny = 0; /* allow yields */
api_checknelems(L, (L->status == LUA_OK) ? nargs + 1 : nargs);
status = luaD_rawrunprotected(L, resume, &nargs);
- if (status == -1) /* error calling 'lua_resume'? */
+ if (unlikely(status == -1)) /* error calling 'lua_resume'? */
status = LUA_ERRRUN;
else { /* continue running after recoverable errors */
while (errorstatus(status) && recover(L, status)) {
/* unroll continuation */
status = luaD_rawrunprotected(L, unroll, &status);
}
- if (errorstatus(status)) { /* unrecoverable error? */
+ if (unlikely(errorstatus(status))) { /* unrecoverable error? */
L->status = cast_byte(status); /* mark thread as 'dead' */
seterrorobj(L, status, L->top); /* push error message */
L->ci->top = L->top;
@@ -694,7 +694,7 @@ LUA_API int lua_yieldk (lua_State *L, int nresults, lua_KContext ctx,
luai_userstateyield(L, nresults);
lua_lock(L);
api_checknelems(L, nresults);
- if (L->nny > 0) {
+ if (unlikely(L->nny > 0)) {
if (L != G(L)->mainthread)
luaG_runerror(L, "attempt to yield across a C-call boundary");
else
@@ -727,7 +727,7 @@ int luaD_pcall (lua_State *L, Pfunc func, void *u,
ptrdiff_t old_errfunc = L->errfunc;
L->errfunc = ef;
status = luaD_rawrunprotected(L, func, u);
- if (status != LUA_OK) { /* an error occurred? */
+ if (unlikely(status != LUA_OK)) { /* an error occurred? */
StkId oldtop = restorestack(L, old_top);
luaF_close(L, oldtop); /* close possible pending closures */
seterrorobj(L, status, oldtop);
diff --git a/llimits.h b/llimits.h
index 9a3ae8d0..725d7c8b 100644
--- a/llimits.h
+++ b/llimits.h
@@ -1,5 +1,5 @@
/*
-** $Id: llimits.h,v 1.148 2017/12/28 11:51:00 roberto Exp roberto $
+** $Id: llimits.h,v 1.149 2018/01/28 15:13:26 roberto Exp roberto $
** Limits, basic types, and some other 'installation-dependent' definitions
** See Copyright Notice in lua.h
*/
@@ -130,9 +130,27 @@ typedef LUAI_UACINT l_uacInt;
#endif
+/*
+** macros to improve jump prediction (used mainly for error handling)
+*/
+#if !defined(likely)
+
+#if defined(__GNUC__)
+#define likely(x) (__builtin_expect(((x) != 0), 1))
+#define unlikely(x) (__builtin_expect(((x) != 0), 0))
+#else
+#define likely(x) (x)
+#define unlikely(x) (x)
+#endif
+
+#endif
+
+
/*
** non-return type
*/
+#if !defined(l_noret)
+
#if defined(__GNUC__)
#define l_noret void __attribute__((noreturn))
#elif defined(_MSC_VER) && _MSC_VER >= 1200
@@ -141,6 +159,7 @@ typedef LUAI_UACINT l_uacInt;
#define l_noret void
#endif
+#endif
/*
diff --git a/lmem.c b/lmem.c
index 2c1757f5..5c73acd5 100644
--- a/lmem.c
+++ b/lmem.c
@@ -1,5 +1,5 @@
/*
-** $Id: lmem.c,v 1.95 2017/12/11 12:27:48 roberto Exp roberto $
+** $Id: lmem.c,v 1.96 2018/01/28 15:13:26 roberto Exp roberto $
** Interface to Memory Manager
** See Copyright Notice in lua.h
*/
@@ -60,7 +60,7 @@ void *luaM_growaux_ (lua_State *L, void *block, int nelems, int *psize,
if (nelems + 1 <= size) /* does one extra element still fit? */
return block; /* nothing to be done */
if (size >= limit / 2) { /* cannot double it? */
- if (size >= limit) /* cannot grow even a little? */
+ if (unlikely(size >= limit)) /* cannot grow even a little? */
luaG_runerror(L, "too many %s (limit is %d)", what, limit);
size = limit; /* still have at least one free place */
}
@@ -73,7 +73,7 @@ void *luaM_growaux_ (lua_State *L, void *block, int nelems, int *psize,
/* 'limit' ensures that multiplication will not overflow */
newblock = luaM_realloc_(L, block, cast_sizet(*psize) * size_elems,
cast_sizet(size) * size_elems);
- if (newblock == NULL)
+ if (unlikely(newblock == NULL))
luaM_error(L);
*psize = size; /* update only when everything else is OK */
return newblock;
@@ -88,7 +88,7 @@ void *luaM_shrinkvector_ (lua_State *L, void *block, int *size,
size_t newsize = cast_sizet(final_n * size_elem);
lua_assert(newsize <= oldsize);
newblock = (*g->frealloc)(g->ud, block, oldsize, newsize);
- if (newblock == NULL && final_n > 0) /* allocation failed? */
+ if (unlikely(newblock == NULL && final_n > 0)) /* allocation failed? */
luaM_error(L);
else {
g->GCdebt += newsize - oldsize;
@@ -114,6 +114,22 @@ void luaM_free_ (lua_State *L, void *block, size_t osize) {
}
+/*
+** In case of allocation fail, this function will call the GC to try
+** to free some memory and then try the allocation again.
+** (It should not be called when shrinking a block, because then the
+** interpreter may be in the middle of a collection step.)
+*/
+static void *tryagain (lua_State *L, void *block,
+ size_t osize, size_t nsize) {
+ global_State *g = G(L);
+ if (g->version) { /* is state fully build? */
+ luaC_fullgc(L, 1); /* try to free some memory... */
+ return (*g->frealloc)(g->ud, block, osize, nsize); /* try again */
+ }
+ else return NULL; /* cannot free any memory without a full state */
+}
+
/*
** generic allocation routine.
@@ -124,13 +140,10 @@ void *luaM_realloc_ (lua_State *L, void *block, size_t osize, size_t nsize) {
lua_assert((osize == 0) == (block == NULL));
hardtest(L, osize, nsize);
newblock = (*g->frealloc)(g->ud, block, osize, nsize);
- if (newblock == NULL && nsize > 0) {
- /* Is state fully built? Not shrinking a block? */
- if (g->version && nsize > osize) {
- luaC_fullgc(L, 1); /* try to free some memory... */
- newblock = (*g->frealloc)(g->ud, block, osize, nsize); /* try again */
- }
- if (newblock == NULL)
+ if (unlikely(newblock == NULL && nsize > 0)) {
+ if (nsize > osize) /* not shrinking a block? */
+ newblock = tryagain(L, block, osize, nsize);
+ if (newblock == NULL) /* still no memory? */
return NULL;
}
lua_assert((nsize == 0) == (newblock == NULL));
@@ -142,7 +155,7 @@ void *luaM_realloc_ (lua_State *L, void *block, size_t osize, size_t nsize) {
void *luaM_saferealloc_ (lua_State *L, void *block, size_t osize,
size_t nsize) {
void *newblock = luaM_realloc_(L, block, osize, nsize);
- if (newblock == NULL && nsize > 0) /* allocation failed? */
+ if (unlikely(newblock == NULL && nsize > 0)) /* allocation failed? */
luaM_error(L);
return newblock;
}
@@ -155,11 +168,8 @@ void *luaM_malloc_ (lua_State *L, size_t size, int tag) {
else {
global_State *g = G(L);
void *newblock = (*g->frealloc)(g->ud, NULL, tag, size);
- if (newblock == NULL) {
- if (g->version) { /* is state fully built? */
- luaC_fullgc(L, 1); /* try to free some memory... */
- newblock = (*g->frealloc)(g->ud, NULL, tag, size); /* try again */
- }
+ if (unlikely(newblock == NULL)) {
+ newblock = tryagain(L, NULL, tag, size);
if (newblock == NULL)
luaM_error(L);
}
diff --git a/lstring.c b/lstring.c
index 29a08212..f1e5d82b 100644
--- a/lstring.c
+++ b/lstring.c
@@ -1,5 +1,5 @@
/*
-** $Id: lstring.c,v 2.64 2018/02/15 18:06:24 roberto Exp roberto $
+** $Id: lstring.c,v 2.65 2018/02/20 16:52:50 roberto Exp roberto $
** String table (keeps all strings handled by Lua)
** See Copyright Notice in lua.h
*/
@@ -99,7 +99,7 @@ void luaS_resize (lua_State *L, int nsize) {
if (nsize < osize) /* shrinking table? */
tablerehash(tb->hash, osize, nsize); /* depopulate shrinking part */
newvect = luaM_reallocvector(L, tb->hash, osize, nsize, TString*);
- if (newvect == NULL) { /* reallocation failed? */
+ if (unlikely(newvect == NULL)) { /* reallocation failed? */
if (nsize < osize) /* was it shrinking table? */
tablerehash(tb->hash, nsize, osize); /* restore to original size */
/* leave table as it was */
@@ -182,7 +182,7 @@ void luaS_remove (lua_State *L, TString *ts) {
static void growstrtab (lua_State *L, stringtable *tb) {
- if (tb->nuse == MAX_INT) { /* too many strings? */
+ if (unlikely(tb->nuse == MAX_INT)) { /* too many strings? */
luaC_fullgc(L, 1); /* try to free some... */
if (tb->nuse == MAX_INT) /* still too many? */
luaM_error(L); /* cannot even create a message... */
@@ -233,7 +233,7 @@ TString *luaS_newlstr (lua_State *L, const char *str, size_t l) {
return internshrstr(L, str, l);
else {
TString *ts;
- if (l >= (MAX_SIZE - sizeof(TString))/sizeof(char))
+ if (unlikely(l >= (MAX_SIZE - sizeof(TString))/sizeof(char)))
luaM_toobig(L);
ts = luaS_createlngstrobj(L, l);
memcpy(getstr(ts), str, l * sizeof(char));
@@ -269,7 +269,7 @@ Udata *luaS_newudata (lua_State *L, size_t s, int nuvalue) {
Udata *u;
int i;
GCObject *o;
- if (s > MAX_SIZE - udatamemoffset(nuvalue))
+ if (unlikely(s > MAX_SIZE - udatamemoffset(nuvalue)))
luaM_toobig(L);
o = luaC_newobj(L, LUA_TUSERDATA, sizeudata(nuvalue, s));
u = gco2u(o);
diff --git a/ltable.c b/ltable.c
index 7113f0ff..56fe64fd 100644
--- a/ltable.c
+++ b/ltable.c
@@ -1,5 +1,5 @@
/*
-** $Id: ltable.c,v 2.135 2018/02/26 14:16:05 roberto Exp roberto $
+** $Id: ltable.c,v 2.136 2018/05/29 18:01:50 roberto Exp roberto $
** Lua tables (hash)
** See Copyright Notice in lua.h
*/
@@ -235,7 +235,7 @@ static unsigned int findindex (lua_State *L, Table *t, TValue *key) {
return i; /* yes; that's the index */
else {
const TValue *n = getgeneric(t, key);
- if (n == luaH_emptyobject)
+ if (unlikely(n == luaH_emptyobject))
luaG_runerror(L, "invalid key to 'next'"); /* key not found */
i = cast_int(nodefromval(n) - gnode(t, 0)); /* key index in hash table */
/* hash elements are numbered after array ones */
@@ -467,7 +467,7 @@ void luaH_resize (lua_State *L, Table *t, unsigned int newasize,
}
/* allocate new array */
newarray = luaM_reallocvector(L, t->array, oldasize, newasize, TValue);
- if (newarray == NULL && newasize > 0) { /* allocation failed? */
+ if (unlikely(newarray == NULL && newasize > 0)) { /* allocation failed? */
freehash(L, &newt); /* release new hash part */
luaM_error(L); /* raise error (with array unchanged) */
}
@@ -560,7 +560,8 @@ static Node *getfreepos (Table *t) {
TValue *luaH_newkey (lua_State *L, Table *t, const TValue *key) {
Node *mp;
TValue aux;
- if (ttisnil(key)) luaG_runerror(L, "table index is nil");
+ if (unlikely(ttisnil(key)))
+ luaG_runerror(L, "table index is nil");
else if (ttisfloat(key)) {
lua_Number f = fltvalue(key);
lua_Integer k;
@@ -568,7 +569,7 @@ TValue *luaH_newkey (lua_State *L, Table *t, const TValue *key) {
setivalue(&aux, k);
key = &aux; /* insert it as an integer */
}
- else if (luai_numisnan(f))
+ else if (unlikely(luai_numisnan(f)))
luaG_runerror(L, "table index is NaN");
}
mp = mainpositionTV(t, key);
diff --git a/lvm.c b/lvm.c
index 4406afb6..36c7699f 100644
--- a/lvm.c
+++ b/lvm.c
@@ -1,5 +1,5 @@
/*
-** $Id: lvm.c,v 2.354 2018/05/02 18:17:59 roberto Exp roberto $
+** $Id: lvm.c,v 2.355 2018/05/22 12:02:36 roberto Exp roberto $
** Lua virtual machine
** See Copyright Notice in lua.h
*/
@@ -196,7 +196,7 @@ void luaV_finishget (lua_State *L, const TValue *t, TValue *key, StkId val,
if (slot == NULL) { /* 't' is not a table? */
lua_assert(!ttistable(t));
tm = luaT_gettmbyobj(L, t, TM_INDEX);
- if (notm(tm))
+ if (unlikely(notm(tm)))
luaG_typeerror(L, t, "index"); /* no metamethod */
/* else will try the metamethod */
}
@@ -253,7 +253,7 @@ void luaV_finishset (lua_State *L, const TValue *t, TValue *key,
}
else { /* not a table; check metamethod */
tm = luaT_gettmbyobj(L, t, TM_NEWINDEX);
- if (notm(tm))
+ if (unlikely(notm(tm)))
luaG_typeerror(L, t, "index");
}
/* try the metamethod */
@@ -561,7 +561,7 @@ void luaV_concat (lua_State *L, int total) {
/* collect total length and number of strings */
for (n = 1; n < total && tostring(L, s2v(top - n - 1)); n++) {
size_t l = vslen(s2v(top - n - 1));
- if (l >= (MAX_SIZE/sizeof(char)) - tl)
+ if (unlikely(l >= (MAX_SIZE/sizeof(char)) - tl))
luaG_runerror(L, "string length overflow");
tl += l;
}
@@ -605,7 +605,7 @@ void luaV_objlen (lua_State *L, StkId ra, const TValue *rb) {
}
default: { /* try metamethod */
tm = luaT_gettmbyobj(L, rb, TM_LEN);
- if (notm(tm)) /* no metamethod? */
+ if (unlikely(notm(tm))) /* no metamethod? */
luaG_typeerror(L, rb, "get length of");
break;
}
@@ -622,7 +622,7 @@ void luaV_objlen (lua_State *L, StkId ra, const TValue *rb) {
*/
lua_Integer luaV_div (lua_State *L, lua_Integer m, lua_Integer n) {
if (l_castS2U(n) + 1u <= 1u) { /* special cases: -1 or 0 */
- if (n == 0)
+ if (unlikely(n == 0))
luaG_runerror(L, "attempt to divide by zero");
return intop(-, 0, m); /* n==-1; avoid overflow with 0x80000...//-1 */
}
@@ -642,7 +642,7 @@ lua_Integer luaV_div (lua_State *L, lua_Integer m, lua_Integer n) {
*/
lua_Integer luaV_mod (lua_State *L, lua_Integer m, lua_Integer n) {
if (l_castS2U(n) + 1u <= 1u) { /* special cases: -1 or 0 */
- if (n == 0)
+ if (unlikely(n == 0))
luaG_runerror(L, "attempt to perform 'n%%0'");
return 0; /* m % -1 == 0; avoid overflow with 0x80000...%-1 */
}
@@ -1665,7 +1665,7 @@ void luaV_execute (lua_State *L, CallInfo *ci) {
TValue *plimit = s2v(ra + 1);
lua_Integer ilimit, initv;
int stopnow;
- if (!forlimit(plimit, &ilimit, 1, &stopnow)) {
+ if (unlikely(!forlimit(plimit, &ilimit, 1, &stopnow))) {
savestate(L, ci); /* for the error message */
luaG_runerror(L, "'for' limit must be a number");
}
@@ -1717,13 +1717,13 @@ void luaV_execute (lua_State *L, CallInfo *ci) {
else { /* try making all values floats */
lua_Number ninit; lua_Number nlimit; lua_Number nstep;
savestate(L, ci); /* in case of errors */
- if (!tonumber(plimit, &nlimit))
+ if (unlikely(!tonumber(plimit, &nlimit)))
luaG_runerror(L, "'for' limit must be a number");
setfltvalue(plimit, nlimit);
- if (!tonumber(pstep, &nstep))
+ if (unlikely(!tonumber(pstep, &nstep)))
luaG_runerror(L, "'for' step must be a number");
setfltvalue(pstep, nstep);
- if (!tonumber(init, &ninit))
+ if (unlikely(!tonumber(init, &ninit)))
luaG_runerror(L, "'for' initial value must be a number");
setfltvalue(init, luai_numsub(L, ninit, nstep));
}
--
cgit v1.2.3-55-g6feb