Fix minilua undefined behavior in bit.tohex.v2.0master

Note: this is not a vulnerability! minilua is only used during the LuaJIT
build process. It only runs controlled and static Lua code (DynASM),
which is entirely contained within this repo and does not trigger the
undefined behavior.

This change is solely for the benefit of others, who might possibly use
minilua for purposes other than running DynASM.

Reported by quart27219. #1424

2 files changed, 4 insertions, 4 deletions

diff --git a/src/host/genminilua.lua b/src/host/genminilua.lua
index 2b0008ac..9e129148 100644
--- a/src/host/genminilua.lua
+++ b/src/host/genminilua.lua
@@ -90,11 +90,11 @@ static int bswap(lua_State *L){
 UB b=barg(L,1);b=(b>>24)|((b>>8)&0xff00)|((b&0xff00)<<8)|(b<<24);BRET(b)}
 static int tohex(lua_State *L){
 UB b=barg(L,1);
-int n=lua_isnone(L,2)?8:(int)barg(L,2);
+UB n=lua_isnone(L,2)?8:barg(L,2);
 const char *hexdigits="0123456789abcdef";
 char buf[8];
 int i;
-if(n<0){n=-n;hexdigits="0123456789ABCDEF";}
+if((int)n<0){n=~n+1;hexdigits="0123456789ABCDEF";}
 if(n>8)n=8;
 for(i=(int)n;--i>=0;){buf[i]=hexdigits[b&15];b>>=4;}
 lua_pushlstring(L,buf,(size_t)n);
diff --git a/src/host/minilua.c b/src/host/minilua.c
index 76f32aed..2d59de41 100644
--- a/src/host/minilua.c
+++ b/src/host/minilua.c
@@ -7722,11 +7722,11 @@ static int bswap(lua_State*L){
 UB b=barg(L,1);b=(b>>24)|((b>>8)&0xff00)|((b&0xff00)<<8)|(b<<24);BRET(b)}
 static int tohex(lua_State*L){
 UB b=barg(L,1);
-int n=lua_isnone(L,2)?8:(int)barg(L,2);
+UB n=lua_isnone(L,2)?8:barg(L,2);
 const char*hexdigits="0123456789abcdef";
 char buf[8];
 int i;
-if(n<0){n=-n;hexdigits="0123456789ABCDEF";}
+if((int)n<0){n=~n+1;hexdigits="0123456789ABCDEF";}
 if(n>8)n=8;
 for(i=(int)n;--i>=0;){buf[i]=hexdigits[b&15];b>>=4;}
 lua_pushlstring(L,buf,(size_t)n);