Fix transfer of nil uservalues triggering a Lua API check because of conversion to nil sentinels in keeper states

2 files changed, 24 insertions, 19 deletions

diff --git a/src/compat.cpp b/src/compat.cpp
index 5923d29..3b026b4 100644
--- a/src/compat.cpp
+++ b/src/compat.cpp
@@ -70,7 +70,7 @@ void luaL_requiref(lua_State* L_, const char* modname_, lua_CFunction openf_, in
 // #################################################################################################
 // #################################################################################################
 
-void* lua_newuserdatauv(lua_State* L_, size_t sz_, int nuvalue_)
+void* lua_newuserdatauv(lua_State* L_, size_t sz_, [[maybe_unused]] int nuvalue_)
 {
     LUA_ASSERT(L_, nuvalue_ <= 1);
     return lua_newuserdata(L_, sz_);
@@ -81,26 +81,35 @@ void* lua_newuserdatauv(lua_State* L_, size_t sz_, int nuvalue_)
 // push on stack uservalue #n of full userdata at idx
 int lua_getiuservalue(lua_State* L_, int idx_, int n_)
 {
+    STACK_CHECK_START_REL(L_, 0);
     // full userdata can have only 1 uservalue before 5.4
     if (n_ > 1) {
         lua_pushnil(L_);
         return LUA_TNONE;
     }
-    lua_getuservalue(L_, idx_);
 
 #if LUA_VERSION_NUM == 501
-    /* default environment is not a nil (see lua_getfenv) */
-    lua_getglobal(L_, LUA_LOADLIBNAME);
+    lua_getfenv(L_, idx_);                                                                         // L_: ... {}|nil
+    STACK_CHECK(L_, 1);
+    // default environment is not a nil (see lua_getfenv)
+    lua_getglobal(L_, LUA_LOADLIBNAME);                                                            // L_: ... {}|nil package
     if (lua_rawequal(L_, -2, -1) || lua_rawequal(L_, -2, LUA_GLOBALSINDEX)) {
-        lua_pop(L_, 2);
-        lua_pushnil(L_);
-
+        lua_pop(L_, 2);                                                                            // L_: ...
+        lua_pushnil(L_);                                                                           // L_: ... nil
+        STACK_CHECK(L_, 1);
         return LUA_TNONE;
     }
-    lua_pop(L_, 1); /* remove package */
-#endif
-
-    return lua_type(L_, -1);
+    else {
+        lua_pop(L_, 1);                                                                            // L_: ... nil
+    }
+#else // LUA_VERSION_NUM > 501
+    lua_getuservalue(L_, idx_);                                                                    // L_: {}|nil
+#endif// LUA_VERSION_NUM > 501
+    STACK_CHECK(L_, 1);
+    int const _uvType{ lua_type(L_, -1) };
+    // under Lua 5.2, there is a single uservalue that is either nil or a table.
+    // If nil, don't transfer it, as it can cause issues when copying to a Keeper state because of nil sentinel conversion
+    return (LUA_VERSION_NUM == 502 && _uvType == LUA_TNIL) ? LUA_TNONE : _uvType;
 }
 
 // #################################################################################################
@@ -118,7 +127,11 @@ int lua_setiuservalue(lua_State* L_, int idx_, int n_)
         return 0;
     }
 
+#if LUA_VERSION_NUM == 501
+    lua_setfenv(L_, idx_);
+#else // LUA_VERSION_NUM == 501
     lua_setuservalue(L_, idx_);
+#endif // LUA_VERSION_NUM == 501
     return 1; // I guess anything non-0 is ok
 }
 
diff --git a/src/compat.h b/src/compat.h
index 6307d86..1789a8b 100644
--- a/src/compat.h
+++ b/src/compat.h
@@ -67,14 +67,6 @@ inline void lua_pushglobaltable(lua_State* L_)
     lua_pushvalue(L_, LUA_GLOBALSINDEX);
 }
 #endif // LUAJIT_VERSION_NUM
-inline int lua_setuservalue(lua_State* L_, int idx_)
-{
-    return lua_setfenv(L_, idx_);
-}
-inline void lua_getuservalue(lua_State* L_, int idx_)
-{
-    lua_getfenv(L_, idx_);
-}
 inline size_t lua_rawlen(lua_State* L_, int idx_)
 {
     return lua_objlen(L_, idx_);