From ca7657e24549acb8a2dd45fa81c309b5bf9f61ee Mon Sep 17 00:00:00 2001 From: Benoit Germain Date: Mon, 2 Dec 2024 11:53:09 +0100 Subject: Data transfer supports registered non-deep full userdata --- CHANGES | 1 + docs/index.html | 58 ++++++++++++++-------------- src/intercopycontext.cpp | 98 +++++++++++++++++++++++++++++++++++++++++------- src/intercopycontext.hpp | 2 + src/tools.cpp | 29 ++++++++------ 5 files changed, 134 insertions(+), 54 deletions(-) diff --git a/CHANGES b/CHANGES index a2dbad4..db44b2c 100644 --- a/CHANGES +++ b/CHANGES @@ -23,6 +23,7 @@ CHANGE 2: BGe 27-Nov-24 - strip_functions added. Only useful for Lua 5.3+. - verbose_errors removed. Use lane error_trace_level instead. - with_timers is false by default. + - Non-deep full userdata are processed during module registration just like ordinary module C functions, making them valid transferable (up)values (for example: io.stdin). - Lanes: - Can no longer be "killed" by hard-stopping their thread without any resource cleanup (see lane:cancel()). - lanes.gen() settings: diff --git a/docs/index.html b/docs/index.html index 9ac36b6..7432c53 100644 --- a/docs/index.html +++ b/docs/index.html @@ -436,11 +436,11 @@

- Once Lanes is configured, one should register with Lanes the modules exporting functions that will be transferred either during lane generation or through lindas. + Once Lanes is configured, one should register with Lanes the modules exporting functions/userdata that will be transferred either during lane generation or through lindas.
Use lanes.require() for this purpose. This will call the original require(), then add the result to the lookup databases.
- It is also possible to register a given module a posteriori with lanes.register(). This function will raise an error if the registered module is not a function or table.
+ It is also possible to register a given module a posteriori with lanes.register(). This function will raise an error if the registered module is not a function, table, or full userdata.
Embedders can call the equivalent function lanes_register() from the C side, through lua_call() or similar.

@@ -715,7 +715,7 @@ table - Lists modules that have to be required in order to be able to transfer functions they exposed. Non-Lua functions are searched in lookup tables. + Lists modules that have to be required in order to be able to transfer functions/userdata they exposed. Non-Lua functions are searched in lookup tables. These tables are built from the modules listed here. required must be an array of strings, each one being the name of a module to be required. Each module is required with require() before the lanes function is invoked. So, from the required module's point of view, requiring it manually from inside the lane body or having it required this way doesn't change anything. From the lane body's point of view, the only difference is that a module not creating a global won't be accessible. Therefore, a lane body will also have to require a module manually, but this won't do anything more (see Lua's require documentation).
@@ -785,8 +785,8 @@

- If a lane body pulls a C function imported by a module required before Lanes itself (thus not through a hooked require()), the lane generator creation will raise an error. - The function name it shows is a path where it was found by scanning _G and the registry. As a utility, the name guessing functionality is exposed as such: + If a lane body pulls a C function or userdata exposed by a module required before Lanes itself (thus not through a hooked require()), the lane generator creation will raise an error. + The name in the message is a path where it was found by scanning _G and the registry. As a utility, the name guessing functionality is exposed as such: @@ -1233,10 +1233,10 @@ Characteristics of the Lanes implementation of lindas are:
  • - Non-converted full userdata can be passed only if it is prepared using the deep userdata system, which handles its lifespan management. + Non-converted full userdata can be passed only if it registered, or prepared using the deep userdata system, which handles its lifespan management.
  • Objects (tables with a metatable) are copyable between lanes. Metatables are assumed to be immutable; they are internally indexed and only copied once per each type of objects per lane. @@ -1661,55 +1661,57 @@

    -

    Notes about passing C functions

    +

    Notes about passing C functions and full userdata

    - Functions are transfered as follows (more or less): + C functions and full userdata are transfered as follows (more or less): 

    • -	// expects a C function on top of the source Lua stack
-	copyFunction(lua_State *dest, lua_State* source)
+	// expects a C function or full userdata on top of the source Lua stack
+	copyValue(lua_State *dest, lua_State* source)
 	{
 		// fetch function 'name' from source lookup database
-		char const* funcname = lookup_func_name(source, -1);
-		// lookup a function bound to this name in the destination state, and push it on the stack
-		push_resolved_func(dest, funcname);
+		char const* valuename = lookup_name(source, -1);
+		// lookup a function or userdata bound to this name in the destination state, and push it on the stack
+		push_resolved_value(dest, valuename);
 	}

    - The devil lies in the details: what does "function lookup" mean? + The devil lies in the details: what does "lookup" mean?

    - Since functions are first class values, they don't have a name. All we know for sure is that when a C module registers some functions, they are accessible to the script that required the module through some exposed variables. + Since functions and full userdata are first class values, they don't have a name. All we know for sure is that when a C module registers some functions or full userdata, they are accessible to the script that required the module through some exposed variables.
    For example, loading the string base library creates a table accessible when indexing the global environment with key "string". Indexing this table with "match", "gsub", etc. will give us a function.
    - When a lane generator creates a lane and performs initializations described by the list of base libraries and the list of required modules, it recursively scans the table created by the initialisation of the module, looking for all values that are C functions. + Similarly, loading the io base library creates a table accessible when indexing the global environment with key "io". Indexing this table with "open", will give us a function, and "stdin" will give us a full userdata.
    - Each time a function is encountered, the sequence of keys that reached that function is contatenated in a (hopefully) unique name. The [name, function] and [function, name] pairs are both stored in a lookup table in all involved Lua states (main Lua state and lanes states). + When a lane generator creates a lane and performs initializations described by the list of base libraries and the list of required modules, it recursively scans the table created by the initialisation of the module, looking for all values that are C functions and full userdata.
    - Then when a function is transfered from one state to another, all we have to do is retrieve the name associated to a function in the source Lua state, then with that name retrieve the equivalent function that already exists in the destination state. + Each time a function or full userdata is encountered, the sequence of keys traversed to reach it is contatenated in a (hopefully) unique name. The [name, value] and [value, name] pairs are both stored in a lookup table in all involved Lua states (main Lua state and lanes states).
    - Note that there is no need to transfer upvalues, as they are already bound to the function registered in the destination state. (And in any event, it is not possible to create a closure from a C function pushed on the stack, it can only be created with a lua_CFunction pointer). + Then, when a function or full userdata is transfered from one state to another, all we have to do is retrieve the name associated to this value in the source Lua state, then with that name retrieve the equivalent value that already exists in the destination state. +
    + Note that there is no need to transfer upvalues/uservalues, as they are already bound to the value registered in the destination state. (And in any event, it is not possible to create a closure from a C function pushed on the stack, it can only be created with a lua_CFunction pointer).

    There are several issues here:

    - Another more immediate reason of failed transfer is when the destination state doesn't know about the C function that has to be transferred. This occurs if a function is transferred in a lane before it had a chance to scan the module. - If the C function is sent through a linda, it is enough for the destination lane body to have required the module before the function is sent. - But if the lane body provided to the generator has a C function as upvalue, the transfer itself must succeed, therefore the module that imported that C function must be required in the destination lane before the lane body starts executing. This is where the .required options play their role. + Another more immediate reason of failed transfer is when the destination state doesn't know about the C function or full userdata that has to be transferred. This occurs if a value is transferred in a lane before it had a chance to scan the module that exposed it. + If the C function or full userdata is sent through a linda, it is sufficient for the destination lane body to have required the module before the function is sent. + But if the lane body provided to the generator has a C function or full userdata as upvalue, the transfer itself must succeed, therefore the module that exposed that C function or full userdata must be required in the destination lane before the lane body starts executing. This is where the .required options play their role.

    diff --git a/src/intercopycontext.cpp b/src/intercopycontext.cpp index 568e4cb..1be4c53 100644 --- a/src/intercopycontext.cpp +++ b/src/intercopycontext.cpp @@ -55,7 +55,7 @@ static int buf_writer(lua_State* L_, void const* b_, size_t size_, void* ud_) // function sentinel used to transfer native functions from/to keeper states [[nodiscard]] -static int func_lookup_sentinel(lua_State* L_) +static int func_lookup_sentinel(lua_State* const L_) { raise_luaL_error(L_, "function lookup sentinel for %s, should never be called", lua_tostring(L_, lua_upvalueindex(1))); } @@ -64,7 +64,7 @@ static int func_lookup_sentinel(lua_State* L_) // function sentinel used to transfer native table from/to keeper states [[nodiscard]] -static int table_lookup_sentinel(lua_State* L_) +static int table_lookup_sentinel(lua_State* const L_) { raise_luaL_error(L_, "table lookup sentinel for %s, should never be called", lua_tostring(L_, lua_upvalueindex(1))); } @@ -73,23 +73,32 @@ static int table_lookup_sentinel(lua_State* L_) // function sentinel used to transfer cloned full userdata from/to keeper states [[nodiscard]] -static int userdata_clone_sentinel(lua_State* L_) +static int userdata_clone_sentinel(lua_State* const L_) { raise_luaL_error(L_, "userdata clone sentinel for %s, should never be called", lua_tostring(L_, lua_upvalueindex(1))); } // ################################################################################################# -// retrieve the name of a function/table in the lookup database +// function sentinel used to transfer native table from/to keeper states +[[nodiscard]] +static int userdata_lookup_sentinel(lua_State* const L_) +{ + raise_luaL_error(L_, "userdata lookup sentinel for %s, should never be called", lua_tostring(L_, lua_upvalueindex(1))); +} + +// ################################################################################################# + +// retrieve the name of a function/table/userdata in the lookup database [[nodiscard]] std::string_view InterCopyContext::findLookupName() const { - LUA_ASSERT(L1, lua_isfunction(L1, L1_i) || lua_istable(L1, L1_i)); // L1: ... v ... - STACK_CHECK_START_REL(L1, 0); + LUA_ASSERT(L1, lua_isfunction(L1, L1_i) || lua_istable(L1, L1_i) || luaG_type(L1, L1_i) == LuaType::USERDATA); + STACK_CHECK_START_REL(L1, 0); // L1: ... v ... STACK_GROW(L1, 3); // up to 3 slots are necessary on error if (mode == LookupMode::FromKeeper) { - lua_CFunction const _f{ lua_tocfunction(L1, L1_i) }; // should *always* be one of the function sentinels - if (_f == func_lookup_sentinel || _f == table_lookup_sentinel || _f == userdata_clone_sentinel) { + lua_CFunction const _f{ lua_tocfunction(L1, L1_i) }; // should *always* be one of the sentinel functions + if (_f == func_lookup_sentinel || _f == table_lookup_sentinel || _f == userdata_clone_sentinel || _f == userdata_lookup_sentinel) { lua_getupvalue(L1, L1_i, 1); // L1: ... v ... "f.q.n" } else { // if this is not a sentinel, this is some user-created table we wanted to lookup @@ -110,8 +119,8 @@ std::string_view InterCopyContext::findLookupName() const // popping doesn't invalidate the pointer since this is an interned string gotten from the lookup database lua_pop(L1, (mode == LookupMode::FromKeeper) ? 1 : 2); // L1: ... v ... STACK_CHECK(L1, 0); - if (_fqn.empty() && !lua_istable(L1, L1_i)) { // raise an error if we try to send an unknown function (but not for tables) - // try to discover the name of the function we want to send + if (_fqn.empty() && !lua_istable(L1, L1_i)) { // raise an error if we try to send an unknown function/userdata (but not for tables) + // try to discover the name of the function/userdata we want to send kLaneNameRegKey.pushValue(L1); // L1: ... v ... lane_name std::string_view const _from{ luaG_tostring(L1, kIdxTop) }; lua_pushcfunction(L1, LG_nameof); // L1: ... v ... lane_name LG_nameof @@ -331,7 +340,8 @@ void InterCopyContext::lookupNativeFunction() const lua_rawget(L2, -2); // L1: ... f ... L2: {} f // nil means we don't know how to transfer stuff: user should do something // anything other than function or table should not happen! - if (!lua_isfunction(L2, -1) && !lua_istable(L2, -1)) { + LuaType const _objType{ luaG_type(L2, kIdxTop) }; + if (_objType != LuaType::FUNCTION && _objType != LuaType::TABLE && _objType != LuaType::USERDATA) { kLaneNameRegKey.pushValue(L1); // L1: ... f ... lane_name std::string_view const _from{ luaG_tostring(L1, kIdxTop) }; lua_pop(L1, 1); // L1: ... f ... @@ -340,9 +350,10 @@ void InterCopyContext::lookupNativeFunction() const lua_pop(L2, 1); // L2: {} f raise_luaL_error( getErrL(), - "%s%s: function '%s' not found in %s destination transfer database.", + "%s%s: %s '%s' not found in %s destination transfer database.", lua_isnil(L2, -1) ? "" : "INTERNAL ERROR IN ", _from.empty() ? "main" : _from.data(), + luaG_typename(L2, _objType), _fqn.data(), _to.empty() ? "main" : _to.data()); return; @@ -397,8 +408,8 @@ void InterCopyContext::copyCachedFunction() const LUA_ASSERT(L1, lua_isfunction(L2, -1)); } else { // function is native/LuaJIT: no need to cache lookupNativeFunction(); // L2: ... {cache} ... function - // if the function was in fact a lookup sentinel, we can either get a function or a table here - LUA_ASSERT(L1, lua_isfunction(L2, -1) || lua_istable(L2, -1)); + // if the function was in fact a lookup sentinel, we can either get a function, table or full userdata here + LUA_ASSERT(L1, lua_isfunction(L2, kIdxTop) || lua_istable(L2, kIdxTop) || luaG_type(L2, kIdxTop) == LuaType::USERDATA); } } @@ -678,6 +689,59 @@ bool InterCopyContext::pushCachedTable() const // ################################################################################################# +// Push a looked-up full userdata. +[[nodiscard]] +bool InterCopyContext::lookupUserdata() const +{ + // get the name of the userdata we want to send + std::string_view const _fqn{ findLookupName() }; + // push the equivalent userdata in the destination's stack, retrieved from the lookup table + STACK_CHECK_START_REL(L2, 0); + STACK_GROW(L2, 3); // up to 3 slots are necessary on error + switch (mode) { + default: // shouldn't happen, in theory... + raise_luaL_error(getErrL(), "internal error: unknown lookup mode"); + break; + + case LookupMode::ToKeeper: + // push a sentinel closure that holds the lookup name as upvalue + luaG_pushstring(L2, _fqn); // L1: ... f ... L2: "f.q.n" + lua_pushcclosure(L2, userdata_lookup_sentinel, 1); // L1: ... f ... L2: f + break; + + case LookupMode::LaneBody: + case LookupMode::FromKeeper: + kLookupRegKey.pushValue(L2); // L1: ... f ... L2: {} + STACK_CHECK(L2, 1); + LUA_ASSERT(L1, lua_istable(L2, -1)); + luaG_pushstring(L2, _fqn); // L1: ... f ... L2: {} "f.q.n" + lua_rawget(L2, -2); // L1: ... f ... L2: {} f + // nil means we don't know how to transfer stuff: user should do something + // anything other than function or table should not happen! + if (!lua_isfunction(L2, -1) && !lua_istable(L2, -1)) { + kLaneNameRegKey.pushValue(L1); // L1: ... f ... lane_name + std::string_view const _from{ luaG_tostring(L1, kIdxTop) }; + lua_pop(L1, 1); // L1: ... f ... + kLaneNameRegKey.pushValue(L2); // L1: ... f ... L2: {} f lane_name + std::string_view const _to{ luaG_tostring(L2, kIdxTop) }; + lua_pop(L2, 1); // L2: {} f + raise_luaL_error( + getErrL(), + "%s%s: userdata '%s' not found in %s destination transfer database.", + lua_isnil(L2, -1) ? "" : "INTERNAL ERROR IN ", + _from.empty() ? "main" : _from.data(), + _fqn.data(), + _to.empty() ? "main" : _to.data()); + } + lua_remove(L2, -2); // L2: f + break; + } + STACK_CHECK(L2, 1); + return true; +} + +// ################################################################################################# + [[nodiscard]] bool InterCopyContext::tryCopyClonable() const { @@ -1099,6 +1163,12 @@ bool InterCopyContext::interCopyUserdata() const return true; } + // Last, let's try to see if this userdata is special (aka is it some userdata that we registered in our lookup databases during module registration?) + if (lookupUserdata()) { + LUA_ASSERT(L1, luaG_type(L2, kIdxTop) == LuaType::USERDATA || (lua_tocfunction(L2, kIdxTop) == userdata_lookup_sentinel)); // from lookup data. can also be userdata_lookup_sentinel if this is a userdata we know + return true; + } + raise_luaL_error(getErrL(), "can't copy non-deep full userdata across lanes"); } diff --git a/src/intercopycontext.hpp b/src/intercopycontext.hpp index 84d9e70..7008919 100644 --- a/src/intercopycontext.hpp +++ b/src/intercopycontext.hpp @@ -64,6 +64,8 @@ class InterCopyContext // for use in inter_copy_userdata [[nodiscard]] + bool lookupUserdata() const; + [[nodiscard]] bool tryCopyClonable() const; [[nodiscard]] bool tryCopyDeep() const; diff --git a/src/tools.cpp b/src/tools.cpp index 827c4a4..cd64d13 100644 --- a/src/tools.cpp +++ b/src/tools.cpp @@ -123,7 +123,7 @@ namespace tools { * receives 2 arguments: a name k and an object o * add two entries ["fully.qualified.name"] = o * and [o] = "fully.qualified.name" - * where is either a table or a function + * where is either a table, a function, or a full userdata * if we already had an entry of type [o] = ..., replace the name if the new one is shorter * pops the processed object from the stack */ @@ -191,7 +191,7 @@ static void update_lookup_entry(lua_State* const L_, StackIndex const ctxBase_, // ################################################################################################# -static void populate_func_lookup_table_recur(lua_State* const L_, StackIndex const dbIdx_, StackIndex const i_, int const depth_) +static void populate_lookup_table_recur(lua_State* const L_, StackIndex const dbIdx_, StackIndex const i_, int const depth_) { // slot dbIdx_ contains the lookup database table // slot dbIdx_ + 1 contains a table that, when concatenated, produces the fully qualified name of scanned elements in the table provided at slot i_ @@ -199,7 +199,7 @@ static void populate_func_lookup_table_recur(lua_State* const L_, StackIndex con // slot dbIdx_ + 2 contains a cache that stores all already visited tables to avoid infinite recursion loops StackIndex const _cache{ dbIdx_ + 2 }; DEBUGSPEW_CODE(Universe* const _U{ Universe::Get(L_) }); - DEBUGSPEW_CODE(DebugSpew(_U) << "populate_func_lookup_table_recur()" << std::endl); + DEBUGSPEW_CODE(DebugSpew(_U) << "populate_lookup_table_recur()" << std::endl); DEBUGSPEW_CODE(DebugSpewIndentScope _scope{ _U }); STACK_GROW(L_, 6); @@ -231,14 +231,14 @@ static void populate_func_lookup_table_recur(lua_State* const L_, StackIndex con // we need to remember subtables to process them after functions encountered at the current depth (breadth-first search) lua_newtable(L_); // L_: ... {i_} {bfc} - int const breadthFirstCache{ lua_gettop(L_) }; + StackIndex const _breadthFirstCache{ lua_gettop(L_) }; // iterate over all entries in the processed table lua_pushnil(L_); // L_: ... {i_} {bfc} nil while (lua_next(L_, i_) != 0) { // L_: ... {i_} {bfc} k v // just for debug, not actually needed // std::string_view const _key{ (luaG_type(L_, -2) == LuaType::STRING) ? luaG_tostring(L_, -2) : "not a string" }; // subtable: process it recursively - if (lua_istable(L_, -1)) { // L_: ... {i_} {bfc} k {} + if (lua_istable(L_, kIdxTop)) { // L_: ... {i_} {bfc} k {} // increment visit count to make sure we will actually scan it at this recursive level lua_pushvalue(L_, -1); // L_: ... {i_} {bfc} k {} {} lua_pushvalue(L_, -1); // L_: ... {i_} {bfc} k {} {} {} @@ -250,22 +250,26 @@ static void populate_func_lookup_table_recur(lua_State* const L_, StackIndex con // store the table in the breadth-first cache lua_pushvalue(L_, -2); // L_: ... {i_} {bfc} k {} k lua_pushvalue(L_, -2); // L_: ... {i_} {bfc} k {} k {} - lua_rawset(L_, breadthFirstCache); // L_: ... {i_} {bfc} k {} + lua_rawset(L_, _breadthFirstCache); // L_: ... {i_} {bfc} k {} // generate a name, and if we already had one name, keep whichever is the shorter update_lookup_entry(L_, dbIdx_, depth_); // L_: ... {i_} {bfc} k - } else if (lua_isfunction(L_, -1) && (luaG_getfuncsubtype(L_, kIdxTop) != FuncSubType::Bytecode)) { + } else if (lua_isfunction(L_, kIdxTop) && (luaG_getfuncsubtype(L_, kIdxTop) != FuncSubType::Bytecode)) { // generate a name, and if we already had one name, keep whichever is the shorter // this pops the function from the stack update_lookup_entry(L_, dbIdx_, depth_); // L_: ... {i_} {bfc} k + } else if (luaG_type(L_, kIdxTop) == LuaType::USERDATA) { + // generate a name, and if we already had one name, keep whichever is the shorter + // this pops the userdata from the stack + update_lookup_entry(L_, dbIdx_, depth_); // L_: ... {i_} {bfc} k } else { - lua_pop(L_, 1); // L_: ... {i_} {bfc} k + lua_pop(L_, 1); // L_: ... {i_} {bfc} k } STACK_CHECK(L_, 2); } // now process the tables we encountered at that depth int const _deeper{ depth_ + 1 }; lua_pushnil(L_); // L_: ... {i_} {bfc} nil - while (lua_next(L_, breadthFirstCache) != 0) { // L_: ... {i_} {bfc} k {} + while (lua_next(L_, _breadthFirstCache) != 0) { // L_: ... {i_} {bfc} k {} DEBUGSPEW_CODE(std::string_view const _key{ (luaG_type(L_, -2) == LuaType::STRING) ? luaG_tostring(L_, -2) : std::string_view{ "" } }); DEBUGSPEW_CODE(DebugSpew(_U) << "table '"<< _key <<"'" << std::endl); DEBUGSPEW_CODE(DebugSpewIndentScope _scope2{ _U }); @@ -285,7 +289,7 @@ static void populate_func_lookup_table_recur(lua_State* const L_, StackIndex con // push table name in fqn stack (note that concatenation will crash if name is a not string!) lua_pushvalue(L_, -2); // L_: ... {i_} {bfc} k {} k lua_rawseti(L_, _fqn, _deeper); // L_: ... {i_} {bfc} k {} - populate_func_lookup_table_recur(L_, dbIdx_, StackIndex{ lua_gettop(L_) }, _deeper); + populate_lookup_table_recur(L_, dbIdx_, StackIndex{ lua_gettop(L_) }, _deeper); lua_pop(L_, 1); // L_: ... {i_} {bfc} k STACK_CHECK(L_, 2); } @@ -316,7 +320,8 @@ namespace tools { StackIndex const _dbIdx{ lua_gettop(L_) }; STACK_CHECK(L_, 1); LUA_ASSERT(L_, lua_istable(L_, -1)); - if (luaG_type(L_, _in_base) == LuaType::FUNCTION) { // for example when a module is a simple function + LuaType const _moduleType{ luaG_type(L_, _in_base) }; + if ((_moduleType == LuaType::FUNCTION) || (_moduleType == LuaType::USERDATA)) { // for example when a module is a simple function if (_name.empty()) { _name = "nullptr"; } @@ -343,7 +348,7 @@ namespace tools { // retrieve the cache, create it if we haven't done it yet std::ignore = kLookupCacheRegKey.getSubTable(L_, NArr{ 0 }, NRec{ 0 }); // L_: {} {fqn} {cache} // process everything we find in that table, filling in lookup data for all functions and tables we see there - populate_func_lookup_table_recur(L_, _dbIdx, _in_base, _startDepth); + populate_lookup_table_recur(L_, _dbIdx, _in_base, _startDepth); lua_pop(L_, 3); // L_: } else { lua_pop(L_, 1); // L_: -- cgit v1.2.3-55-g6feb