From f823c6887e28c815234f8b4bd355887b4f554857 Mon Sep 17 00:00:00 2001 From: Benoit Germain Date: Thu, 26 Sep 2013 21:11:24 +0200 Subject: Reduce memory footprint, simplify module order setup in conjuction with Lanes, and send over native functions a bit faster as well * Lanes no longer has to internally require modules inside the keeper states because they no longer need a lookup database. the lookup name is stored as-is and actually converted in the destination state * optimisation: bypass cache when sending native functions over * removed all the KEEPER_MODEL_LUA code, as it can no longer work anyway --- CHANGES | 8 + docs/index.html | 17 +- lanes-3.6.5-1.rockspec | 66 ++++++++ src/keeper.c | 78 +-------- src/keeper.h | 11 -- src/lanes.c | 43 ++--- src/tools.c | 446 +++++++++++++++++++++++++------------------------ src/tools.h | 13 +- 8 files changed, 338 insertions(+), 344 deletions(-) create mode 100644 lanes-3.6.5-1.rockspec diff --git a/CHANGES b/CHANGES index 5fdcd83..a52a404 100644 --- a/CHANGES +++ b/CHANGES @@ -1,5 +1,13 @@ CHANGES: +CHANGE 69: BGe 26-Sept-13 + * version 3.6.5 + * Reduce memory footprint, simplify module order setup in conjuction with Lanes, and send over native functions a bit faster as well + * Lanes no longer has to internally require modules inside the keeper states because they no longer need a lookup database + the lookup name is stored as-is and actually converted in the destination state + * optimisation: bypass cache when sending native functions over + * removed all the KEEPER_MODEL_LUA code, as it can no longer work anyway + CHANGE 68: BGe 24-Sept-13 * version 3.6.4 * Fix possible application hang at shutdown if a keeper state referenced a linda. diff --git a/docs/index.html b/docs/index.html index 1c60aaa..ac4fa6b 100644 --- a/docs/index.html +++ b/docs/index.html @@ -70,7 +70,7 @@

- This document was revised on 24-Sept-13, and applies to version 3.6.4. + This document was revised on 26-Sept-13, and applies to version 3.6.5.

@@ -314,7 +314,7 @@ (Since v3.5.2) If equal to true, Lanes wraps all calls to the state's allocator function inside a mutex. Useful when running Lanes with LuaJIT, whose allocator is not threadsafe. - Default is nil. + Default is true when Lanes detects it is run by LuaJIT, else nil. @@ -603,8 +603,6 @@ Specifying it when libs_str doesn't cause the package library to be loaded will generate an error.
If not specified, the created lane will receive the current values of package. Only path, cpath, preload and loaders (Lua 5.1)/searchers (Lua 5.2) are transfered. -
- IMPORTANT: The contents of whatever package table is actually provided to the lane won't be known to the keeper states: they will stick to whatever was found in the main state's package table when Lane was required. @@ -1251,10 +1249,7 @@ events to a common Linda, but... :).
  • - C functions (lua_CFunction) referring to LUA_ENVIRONINDEX or LUA_REGISTRYINDEX might not work right in the target - + C functions (lua_CFunction) referring to LUA_ENVIRONINDEX or LUA_REGISTRYINDEX might not do what you expect in the target, since they will actually use a different environment.
  • Lua 5.2 functions may have a special _ENV upvalue if they perform 'global namespace' lookups. Unless special care is taken, this upvalue defaults to the table found at LUA_RIDX_GLOBALS. @@ -1319,7 +1314,7 @@ events to a common Linda, but... :).
    When a lane generator creates a lane and performs initializations described by the list of base libraries and the list of required modules, it recursively scans the table created by the initialisation of the module, looking for all values that are C functions.
    - Each time a function is encountered, the sequence of keys that reached that function is contatenated in a (hopefully) unique name. The [name, function] and [function, name] pairs are both stored in a lookup table in all involved Lua states (main Lua state, keeper states, lanes states). + Each time a function is encountered, the sequence of keys that reached that function is contatenated in a (hopefully) unique name. The [name, function] and [function, name] pairs are both stored in a lookup table in all involved Lua states (main Lua state and lanes states).
    Then when a function is transfered from one state to another, all we have to do is retrieve the name associated to a function in the source Lua state, then with that name retrieve the equivalent function that already exists in the destination state.
    @@ -1336,7 +1331,7 @@ events to a common Linda, but... :). string2 = string When iterating over all keys of the global table, Lanes has no guarantee that it will hit "string" before or after "string2". However, the values associated to string.match and string2.match are the same C function. - Lanes doesn't expect a C function value to be encountered more than once. In the event it occurs, when the lookup table is populated with a [function, name] pair, an existing pair won't be updated if it is already found. In other words, the first name that was computed is retained. + Lanes doesn't normally expect a C function value to be encountered more than once. In the event it occurs, the shortest name that was computed is retained. If Lanes processed "string2" first, it means that if the Lua state that contains the "string2" global name sends function string.match, lookup_func_name would return name "string2.match", with the obvious effect that push_resolved_func won't find "string2.match" in the destination lookup database, thus failing the transfer (even though this function exists, but is referenced under name "string.match").
    • @@ -1345,7 +1340,7 @@ events to a common Linda, but... :). When Lua is built with compatibility options (such as LUA_COMPAT_ALL), this causes several base libraries to register functions under multiple names. This, with the randomizer, can cause the first encountered name of a function to be different on different VMs, which breaks function transfer. Even under Lua 5.1, this may cause trouble if some module registers a function under several keys. - To circumvent this, Lanes has to select one name among all candidates, and the rule for this is to keep the 'smaller' one: first in bytes, then in lexical order. + To circumvent this, Lanes has to select one name among all candidates, and the rule for this is to keep the 'smaller' one: first in byte count, then in lexical order. Another more immediate reason of failed transfer is when the destination state doesn't know about the C function that has to be transferred. This occurs if a function is transferred in a lane before it had a chance to scan the module. diff --git a/lanes-3.6.5-1.rockspec b/lanes-3.6.5-1.rockspec new file mode 100644 index 0000000..0f11854 --- /dev/null +++ b/lanes-3.6.5-1.rockspec @@ -0,0 +1,66 @@ +-- +-- Lanes rockspec +-- +-- Ref: +-- +-- + +package = "Lanes" + +version = "3.6.5-1" + +source= { + url= "git://github.com/LuaLanes/lanes.git", + branch= "v3.6.5" +} + +description = { + summary= "Multithreading support for Lua", + detailed= [[ + Lua Lanes is a portable, message passing multithreading library + providing the possibility to run multiple Lua states in parallel. + ]], + license= "MIT/X11", + homepage="https://github.com/LuaLanes/lanes", + maintainer="Benoit Germain " +} + +-- Q: What is the difference of "windows" and "win32"? Seems there is none; +-- so should we list either one or both? +-- +supported_platforms= { "win32", + "macosx", + "linux", + "freebsd", -- TBD: not tested + "msys", -- TBD: not supported by LuaRocks 1.0 (or is it?) +} + +dependencies= { + "lua >= 5.1", -- builds with either 5.1 and 5.2 +} + +build = { + type = "builtin", + platforms = + { + linux = + { + modules = + { + ["lanes.core"] = + { + libraries = "pthread" + }, + } + } + }, + modules = + { + ["lanes.core"] = + { + sources = { "src/lanes.c", "src/keeper.c", "src/tools.c", "src/threading.c"}, + incdirs = { "src"}, + }, + lanes = "src/lanes.lua" + } +} diff --git a/src/keeper.c b/src/keeper.c index 4a5c913..8d9f7ec 100644 --- a/src/keeper.c +++ b/src/keeper.c @@ -51,7 +51,6 @@ #include "tools.h" #include "keeper.h" -#if KEEPER_MODEL == KEEPER_MODEL_C //################################################################################### // Keeper implementation //################################################################################### @@ -207,10 +206,10 @@ int keeper_push_linda_storage( lua_State* L, void* ptr) { keeper_fifo* fifo = prepare_fifo_access( KL, -1); // storage key fifo lua_pushvalue( KL, -2); // storage key fifo key - luaG_inter_move( KL, L, 1); // storage key fifo // out key + luaG_inter_move( KL, L, 1, eLM_FromKeeper); // storage key fifo // out key STACK_MID( L, 2); lua_newtable( L); // out key keyout - luaG_inter_move( KL, L, 1); // storage key // out key keyout fifo + luaG_inter_move( KL, L, 1, eLM_FromKeeper); // storage key // out key keyout fifo lua_pushinteger( L, fifo->first); // out key keyout fifo first STACK_MID( L, 5); lua_setfield( L, -3, "first"); // out key keyout fifo @@ -512,7 +511,6 @@ int keepercall_count( lua_State* L) } return 1; } -#endif // KEEPER_MODEL == KEEPER_MODEL_C //################################################################################### // Keeper API, accessed from linda methods @@ -576,17 +574,11 @@ char const* init_keepers( lua_State* L, int _on_state_create, int const _nbKeepe DEBUGSPEW_CODE( ++ debugspew_indent_depth); // We need to load all base libraries in the keeper states so that the transfer databases are populated properly // - // 'io' for debugging messages, 'package' because we need to require modules exporting idfuncs - // the others because they export functions that we may store in a keeper for transfer between lanes - K = luaG_newstate( L, _on_state_create, "K"); + // we don't need any libs in the keeper states + K = luaG_newstate( L, _on_state_create, NULL); STACK_CHECK( K); - // replace default 'package' contents with stuff gotten from the master state - lua_getglobal( L, "package"); - luaG_inter_copy_package( L, K, -1); - lua_pop( L, 1); - DEBUGSPEW_CODE( fprintf( stderr, INDENT_BEGIN "### init_keepers %d END\n" INDENT_END, i)); DEBUGSPEW_CODE( -- debugspew_indent_depth); @@ -596,32 +588,11 @@ char const* init_keepers( lua_State* L, int _on_state_create, int const _nbKeepe lua_concat( K, 2); lua_setglobal( K, "decoda_name"); -#if KEEPER_MODEL == KEEPER_MODEL_C // create the fifos table in the keeper state lua_pushlightuserdata( K, fifos_key); lua_newtable( K); lua_rawset( K, LUA_REGISTRYINDEX); -#endif // KEEPER_MODEL == KEEPER_MODEL_C - -#if KEEPER_MODEL == KEEPER_MODEL_LUA - // use package.loaders[2] to find keeper microcode (NOTE: this works only if nobody tampered with the loaders table...) - lua_getglobal( K, "package"); // package - lua_getfield( K, -1, "loaders"); // package package.loaders - lua_rawgeti( K, -1, 2); // package package.loaders package.loaders[2] - lua_pushliteral( K, "lanes-keeper"); // package package.loaders package.loaders[2] "lanes-keeper" - STACK_MID( K, 4); - // first pcall loads lanes-keeper.lua, second one runs the chunk - if( lua_pcall( K, 1 /*args*/, 1 /*results*/, 0 /*errfunc*/) || lua_pcall( K, 0 /*args*/, 0 /*results*/, 0 /*errfunc*/)) - { - // LUA_ERRRUN / LUA_ERRMEM / LUA_ERRERR - // - char const* err = lua_tostring( K, -1); - assert( err); - return err; - } // package package.loaders - STACK_MID( K, 2); - lua_pop( K, 2); -#endif // KEEPER_MODEL == KEEPER_MODEL_LUA + STACK_END( K, 0); MUTEX_INIT( &GKeepers[i].lock_); GKeepers[i].L = K; @@ -633,41 +604,6 @@ char const* init_keepers( lua_State* L, int _on_state_create, int const _nbKeepe return NULL; // ok } -// cause each keeper state to populate its database of transferable functions with those from the specified module -// do do this we simply require the module inside the keeper state, then populate the lookup database -void populate_keepers( lua_State* L) -{ - size_t name_len; - char const* name = luaL_checklstring( L, -1, &name_len); - int i; - - DEBUGSPEW_CODE( fprintf( stderr, INDENT_BEGIN "populate_keepers %s BEGIN\n" INDENT_END, name)); - DEBUGSPEW_CODE( ++ debugspew_indent_depth); - - for( i = 0; i < GNbKeepers; ++ i) - { - lua_State* K = GKeepers[i].L; - int res; - MUTEX_LOCK( &GKeepers[i].lock_); - STACK_CHECK( K); - STACK_GROW( K, 2); - lua_getglobal( K, "require"); - lua_pushlstring( K, name, name_len); - res = lua_pcall( K, 1, 1, 0); - if( res != LUA_OK) - { - char const* err = luaL_checkstring( K, -1); - luaL_error( L, "error requiring '%s' in keeper state: %s", name, err); - } - // after requiring the module, register the functions it exported in our name<->function database - populate_func_lookup_table( K, -1, name); - lua_pop( K, 1); - STACK_END( K, 0); - MUTEX_UNLOCK( &GKeepers[i].lock_); - } - DEBUGSPEW_CODE( -- debugspew_indent_depth); -} - struct s_Keeper* keeper_acquire( void const* ptr) { // can be 0 if this happens during main state shutdown (lanes is being GC'ed -> no keepers) @@ -747,12 +683,12 @@ int keeper_call( lua_State *K, keeper_api_t _func, lua_State *L, void *linda, ui lua_pushlightuserdata( K, linda); - if( (args == 0) || luaG_inter_copy( L, K, args) == 0) // L->K + if( (args == 0) || luaG_inter_copy( L, K, args, eLM_ToKeeper) == 0) // L->K { lua_call( K, 1 + args, LUA_MULTRET); retvals = lua_gettop( K) - Ktos; - if( (retvals > 0) && luaG_inter_move( K, L, retvals) != 0) // K->L + if( (retvals > 0) && luaG_inter_move( K, L, retvals, eLM_FromKeeper) != 0) // K->L { retvals = -1; } diff --git a/src/keeper.h b/src/keeper.h index 29a19a9..420eca1 100644 --- a/src/keeper.h +++ b/src/keeper.h @@ -18,21 +18,11 @@ char const* init_keepers( lua_State* L, int _on_state_create, int const _nbKeepe void close_keepers( void); #endif // HAVE_KEEPER_ATEXIT_DESINIT -void populate_keepers( lua_State *L); struct s_Keeper *keeper_acquire( const void *ptr); void keeper_release( struct s_Keeper *K); void keeper_toggle_nil_sentinels( lua_State *L, int _val_i, int _nil_to_sentinel); int keeper_push_linda_storage( lua_State* L, void* ptr); -#define KEEPER_MODEL_LUA 1 -#define KEEPER_MODEL_C 2 -#define KEEPER_MODEL KEEPER_MODEL_C - -#if KEEPER_MODEL == KEEPER_MODEL_LUA -typedef char const* keeper_api_t; -#define KEEPER_API( _op) #_op -#define PUSH_KEEPER_FUNC( K, _api) lua_getglobal( K, _api) -#elif KEEPER_MODEL == KEEPER_MODEL_C typedef lua_CFunction keeper_api_t; #define KEEPER_API( _op) keepercall_ ## _op #define PUSH_KEEPER_FUNC lua_pushcfunction @@ -45,7 +35,6 @@ int keepercall_limit( lua_State* L); int keepercall_get( lua_State* L); int keepercall_set( lua_State* L); int keepercall_count( lua_State* L); -#endif // KEEPER_MODEL int keeper_call( lua_State *K, keeper_api_t _func, lua_State *L, void *linda, uint_t starting_index); diff --git a/src/lanes.c b/src/lanes.c index 7eef2d0..dc1eeed 100644 --- a/src/lanes.c +++ b/src/lanes.c @@ -52,7 +52,7 @@ * ... */ -char const* VERSION = "3.6.4"; +char const* VERSION = "3.6.5"; /* =============================================================================== @@ -1862,23 +1862,23 @@ static THREAD_RETURN_T THREAD_CALLCONV lane_main( void *vs) } // --- If a client wants to transfer stuff of a given module from the current state to another Lane, the module must be required -// with lanes.require, that will call the regular 'require', then populate lookup databases in source and keeper states +// with lanes.require, that will call the regular 'require', then populate the lookup database in the source lane // module = lanes.require( "modname") // upvalue[1]: _G.require LUAG_FUNC( require) { char const* name = lua_tostring( L, 1); + STACK_CHECK( L); DEBUGSPEW_CODE( fprintf( stderr, INDENT_BEGIN "lanes.require %s BEGIN\n" INDENT_END, name)); DEBUGSPEW_CODE( ++ debugspew_indent_depth); lua_pushvalue( L, lua_upvalueindex(1)); // "name" require lua_pushvalue( L, 1); // "name" require "name" lua_call( L, 1, 1); // "name" module populate_func_lookup_table( L, -1, name); - lua_insert( L, -2); // module "name" - populate_keepers( L); - lua_pop( L, 1); // module + lua_remove( L, -2); // module DEBUGSPEW_CODE( fprintf( stderr, INDENT_BEGIN "lanes.require %s END\n" INDENT_END, name)); DEBUGSPEW_CODE( -- debugspew_indent_depth); + STACK_END( L, 1); return 1; } @@ -1937,7 +1937,7 @@ LUAG_FUNC( thread_new) // package if( package) { - luaG_inter_copy_package( L, L2, package); + luaG_inter_copy_package( L, L2, package, eLM_LaneBody); } // modules to require in the target lane *before* the function is transfered! @@ -1986,8 +1986,6 @@ LUAG_FUNC( thread_new) populate_func_lookup_table( L2, -1, name); STACK_MID( L2, 1); lua_pop( L2, 1); - // don't require this module in the keeper states as well, use lanes.require() for that! - //populate_keepers( L); } STACK_END( L2, 0); } @@ -2017,7 +2015,7 @@ LUAG_FUNC( thread_new) lua_pushglobaltable( L2); // Lua 5.2 wants us to push the globals table on the stack while( lua_next( L, glob)) { - luaG_inter_copy( L, L2, 2); // moves the key/value pair to the L2 stack + luaG_inter_copy( L, L2, 2, eLM_LaneBody); // moves the key/value pair to the L2 stack // assign it in L2's globals table lua_rawset( L2, -3); lua_pop( L, 1); @@ -2040,7 +2038,7 @@ LUAG_FUNC( thread_new) DEBUGSPEW_CODE( fprintf( stderr, INDENT_BEGIN "thread_new: transfer lane body\n" INDENT_END)); DEBUGSPEW_CODE( ++ debugspew_indent_depth); lua_pushvalue( L, 1); - res = luaG_inter_move( L, L2, 1); // L->L2 + res = luaG_inter_move( L, L2, 1, eLM_LaneBody); // L->L2 DEBUGSPEW_CODE( -- debugspew_indent_depth); if( res != 0) { @@ -2067,7 +2065,7 @@ LUAG_FUNC( thread_new) int res; DEBUGSPEW_CODE( fprintf( stderr, INDENT_BEGIN "thread_new: transfer lane arguments\n" INDENT_END)); DEBUGSPEW_CODE( ++ debugspew_indent_depth); - res = luaG_inter_copy( L, L2, args); // L->L2 + res = luaG_inter_copy( L, L2, args, eLM_LaneBody); // L->L2 DEBUGSPEW_CODE( -- debugspew_indent_depth); if( res != 0) return luaL_error( L, "tried to copy unsupported types"); @@ -2317,7 +2315,7 @@ LUAG_FUNC( thread_join) case DONE: { uint_t n = lua_gettop( L2); // whole L2 stack - if( (n > 0) && (luaG_inter_move( L2, L, n) != 0)) + if( (n > 0) && (luaG_inter_move( L2, L, n, eLM_LaneBody) != 0)) { return luaL_error( L, "tried to copy unsupported types"); } @@ -2327,7 +2325,7 @@ LUAG_FUNC( thread_join) case ERROR_ST: lua_pushnil( L); - if( luaG_inter_move( L2, L, 2) != 0) // error message at [-2], stack trace at [-1] + if( luaG_inter_move( L2, L, 2, eLM_LaneBody) != 0) // error message at [-2], stack trace at [-1] { return luaL_error( L, "tried to copy unsupported types"); } @@ -2619,25 +2617,6 @@ static const struct luaL_Reg lanes_functions [] = { }; -/* - * minimal function registration for keepers, just so that we can populate the transfer databases with them - * without recursively deadlocking ourselves during one-time inits - */ -void register_core_libfuncs_for_keeper( lua_State* L) -{ - DEBUGSPEW_CODE( fprintf( stderr, INDENT_BEGIN "lanes.register_core_libfuncs_for_keeper()\n" INDENT_END)); - DEBUGSPEW_CODE( ++ debugspew_indent_depth); - STACK_GROW( L, 1); - STACK_CHECK( L); - lua_newtable( L); - luaG_registerlibfuncs( L, lanes_functions); - STACK_MID( L, 1); - populate_func_lookup_table( L, -1, "lanes.core"); - lua_pop( L, 1); - STACK_END( L, 0); - DEBUGSPEW_CODE( -- debugspew_indent_depth); -} - /* ** One-time initializations */ diff --git a/src/tools.c b/src/tools.c index a3cc6b7..a957f41 100644 --- a/src/tools.c +++ b/src/tools.c @@ -515,8 +515,6 @@ void populate_func_lookup_table( lua_State* L, int _i, char const* _name) * Base ("unpack", "print" etc.) is always added, unless 'libs' is NULL. * */ -extern void register_core_libfuncs_for_keeper( lua_State* L); - lua_State* luaG_newstate( lua_State* _from, int const _on_state_create, char const* libs) { // reuse alloc function from the originating state @@ -534,6 +532,7 @@ lua_State* luaG_newstate( lua_State* _from, int const _on_state_create, char con { return L; } + // if we are here, no keeper state is involved (because libs == NULL when we init keepers) DEBUGSPEW_CODE( fprintf( stderr, INDENT_BEGIN "luaG_newstate()\n" INDENT_END)); DEBUGSPEW_CODE( ++ debugspew_indent_depth); @@ -549,9 +548,8 @@ lua_State* luaG_newstate( lua_State* _from, int const _on_state_create, char con if( libs) { // special "*" case (mainly to help with LuaJIT compatibility) - // "K" is used when opening keeper states: almost the same as "*", but for the fact we don't open lanes.core // as we are called from luaopen_lanes_core() already, and that would deadlock - if( (libs[0] == '*' || libs[0] == 'K') && libs[1] == 0) + if( libs[0] == '*' && libs[1] == 0) { DEBUGSPEW_CODE( fprintf( stderr, INDENT_BEGIN "opening ALL standard libraries\n" INDENT_END)); luaL_openlibs( L); @@ -560,12 +558,6 @@ lua_State* luaG_newstate( lua_State* _from, int const _on_state_create, char con // don't forget lanes.core for regular lane states open1lib( L, "lanes.core", 10); } - else - { - // In keeper states however, we only want to register the lanes.core functions to be able to transfer them through lindas - // (we don't care about a full lanes.core init in the keeper states as we won't call anything in there) - register_core_libfuncs_for_keeper( L); - } libs = NULL; // done with libs } else @@ -622,7 +614,7 @@ lua_State* luaG_newstate( lua_State* _from, int const _on_state_create, char con STACK_CHECK( _from); // Lua function: transfer as usual (should work as long as it only uses base libraries) lua_pushvalue( _from, _on_state_create); - luaG_inter_move( _from, L, 1); + luaG_inter_move( _from, L, 1, eLM_LaneBody); STACK_END( _from, 0); } // capture error and forward it to main state @@ -1283,59 +1275,6 @@ static bool_t push_cached_table( lua_State *L2, uint_t L2_cache_i, lua_State *L, } -/* - * Check if we've already copied the same function from 'L', and reuse the old - * copy. - * - * Always pushes a function to 'L2'. - */ -static void inter_copy_func( lua_State* L2, uint_t L2_cache_i, lua_State* L, uint_t i, char const* upName_); - -static void push_cached_func( lua_State* L2, uint_t L2_cache_i, lua_State* L, uint_t i, char const* upName_) -{ - void* const aspointer = (void*)lua_topointer( L, i); - // TBD: Merge this and same code for tables - ASSERT_L( L2_cache_i != 0); - - STACK_GROW( L2, 2); - - // L2_cache[id_str]= function - // - STACK_CHECK( L2); - - // We don't need to use the from state ('L') in ID since the life span - // is only for the duration of a copy (both states are locked). - // - - // push a light userdata uniquely representing the function - lua_pushlightuserdata( L2, aspointer); // ... {cache} ... p - - //fprintf( stderr, "<< ID: %s >>\n", lua_tostring(L2,-1) ); - - lua_pushvalue( L2, -1); // ... {cache} ... p p - lua_rawget( L2, L2_cache_i); // ... {cache} ... p function|nil|true - - if( lua_isnil(L2,-1)) // function is unknown - { - lua_pop( L2, 1); // ... {cache} ... p - - // Set to 'true' for the duration of creation; need to find self-references - // via upvalues - // - // pushes a copy of the func, stores a reference in the cache - inter_copy_func( L2, L2_cache_i, L, i, upName_); // ... {cache} ... function - } - else // found function in the cache - { - lua_remove( L2, -2); // ... {cache} ... function - } - STACK_END( L2, 1); - // - // L2 [-1]: function - - ASSERT_L( lua_isfunction( L2, -1)); -} - /* * Return some name helping to identify an object */ @@ -1509,24 +1448,41 @@ int luaG_nameof( lua_State* L) return 2; } +// function sentinel used to transfer native functions from/to keeper states +static int sentinelfunc( lua_State* L) +{ + return luaL_error( L, "transfer sentinel function for %s, should never be called", lua_tostring( L, lua_upvalueindex( 1))); +} + /* * Push a looked-up native/LuaJIT function. */ -static void lookup_native_func( lua_State* L2, lua_State* L, uint_t i, char const* upName_) +static void lookup_native_func( lua_State* L2, lua_State* L, uint_t i, enum eLookupMode mode_, char const* upName_) { - char const* fqn; // L // L2 + char const* fqn; // L // L2 size_t len; _ASSERT_L( L, lua_isfunction( L, i)); // ... f ... STACK_CHECK( L); - // fetch the name from the source state's lookup table - lua_getfield( L, LUA_REGISTRYINDEX, LOOKUP_KEY); // ... f ... {} - _ASSERT_L( L, lua_istable( L, -1)); - lua_pushvalue( L, i); // ... f ... {} f - lua_rawget( L, -2); // ... f ... {} "f.q.n" - fqn = lua_tolstring( L, -1, &len); + if( mode_ == eLM_FromKeeper) + { + lua_CFunction f = lua_tocfunction( L, i); // should *always* be sentinelfunc! + _ASSERT_L( L, f == sentinelfunc); + lua_getupvalue( L, i, 1); // ... f ... "f.q.n" + fqn = lua_tolstring( L, -1, &len); + } + else + { + // fetch the name from the source state's lookup table + lua_getfield( L, LUA_REGISTRYINDEX, LOOKUP_KEY); // ... f ... {} + _ASSERT_L( L, lua_istable( L, -1)); + lua_pushvalue( L, i); // ... f ... {} f + lua_rawget( L, -2); // ... f ... {} "f.q.n" + fqn = lua_tolstring( L, -1, &len); + } DEBUGSPEW_CODE( fprintf( stderr, INDENT_BEGIN "function [C] %s \n" INDENT_END, fqn)); // popping doesn't invalidate the pointer since this is an interned string gotten from the lookup database - lua_pop( L, 2); // ... f ... + lua_pop( L, (mode_ == eLM_FromKeeper) ? 1 : 2); // ... f ... + STACK_MID( L, 0); if( !fqn) { char const *from, *typewhat, *what, *gotchaA, *gotchaB; @@ -1556,21 +1512,30 @@ static void lookup_native_func( lua_State* L2, lua_State* L, uint_t i, char cons STACK_END( L, 0); // push the equivalent function in the destination's stack, retrieved from the lookup table STACK_CHECK( L2); - lua_getfield( L2, LUA_REGISTRYINDEX, LOOKUP_KEY); // {} - _ASSERT_L( L2, lua_istable( L2, -1)); - lua_pushlstring( L2, fqn, len); // {} "f.q.n" - lua_rawget( L2, -2); // {} f - if( !lua_isfunction( L2, -1)) + if( mode_ == eLM_ToKeeper) { - char const* from, * to; - lua_getglobal( L, "decoda_name"); // // ... f ... decoda_name - from = lua_tostring( L, -1); - lua_getglobal( L2, "decoda_name"); // {} f decoda_name - to = lua_tostring( L2, -1); - (void) luaL_error( L, "%s: function '%s' not found in %s destination transfer database.", from ? from : "main", fqn, to ? to : "main"); - return; + // push a sentinel closure that holds the lookup name as upvalue + lua_pushlstring( L2, fqn, len); // "f.q.n" + lua_pushcclosure( L2, sentinelfunc, 1); // f + } + else + { + lua_getfield( L2, LUA_REGISTRYINDEX, LOOKUP_KEY); // {} + _ASSERT_L( L2, lua_istable( L2, -1)); + lua_pushlstring( L2, fqn, len); // {} "f.q.n" + lua_rawget( L2, -2); // {} f + if( !lua_isfunction( L2, -1)) + { + char const* from, * to; + lua_getglobal( L, "decoda_name"); // ... f ... decoda_name + from = lua_tostring( L, -1); + lua_getglobal( L2, "decoda_name"); // {} f decoda_name + to = lua_tostring( L2, -1); + (void) luaL_error( L, "%s: function '%s' not found in %s destination transfer database.", from ? from : "main", fqn, to ? to : "main"); + return; + } + lua_remove( L2, -2); // f } - lua_remove( L2, -2); // f STACK_END( L2, 1); } @@ -1581,159 +1546,207 @@ static void lookup_native_func( lua_State* L2, lua_State* L, uint_t i, char cons enum e_vt { VT_NORMAL, VT_KEY, VT_METATABLE }; -static bool_t inter_copy_one_( lua_State* L2, uint_t L2_cache_i, lua_State* L, uint_t i, enum e_vt value_type, char const* upName_); +static bool_t inter_copy_one_( lua_State* L2, uint_t L2_cache_i, lua_State* L, uint_t i, enum e_vt value_type, enum eLookupMode mode_, char const* upName_); -static void inter_copy_func( lua_State* L2, uint_t L2_cache_i, lua_State* L, uint_t i, char const* upName_) +static void inter_copy_func( lua_State* L2, uint_t L2_cache_i, lua_State* L, uint_t i, enum eLookupMode mode_, char const* upName_) { - FuncSubType funcSubType; - /*lua_CFunction cfunc =*/ luaG_tocfunction( L, i, &funcSubType); // NULL for LuaJIT-fast && bytecode functions - + int n, needToPush; + luaL_Buffer b; ASSERT_L( L2_cache_i != 0); // ... {cache} ... p STACK_GROW(L,2); STACK_CHECK( L); - if( funcSubType == FST_Bytecode) + // 'lua_dump()' needs the function at top of stack + // if already on top of the stack, no need to push again + needToPush = (i != (uint_t)lua_gettop( L)); + if( needToPush) { - int n; - luaL_Buffer b; - // 'lua_dump()' needs the function at top of stack - // if already on top of the stack, no need to push again - int needToPush = (i != (uint_t)lua_gettop( L)); - if( needToPush) - { - lua_pushvalue( L, i); // ... f - } + lua_pushvalue( L, i); // ... f + } - luaL_buffinit( L, &b); - // - // "value returned is the error code returned by the last call - // to the writer" (and we only return 0) - // not sure this could ever fail but for memory shortage reasons - if( lua_dump( L, buf_writer, &b) != 0) - { - luaL_error( L, "internal error: function dump failed."); - } + luaL_buffinit( L, &b); + // + // "value returned is the error code returned by the last call + // to the writer" (and we only return 0) + // not sure this could ever fail but for memory shortage reasons + if( lua_dump( L, buf_writer, &b) != 0) + { + luaL_error( L, "internal error: function dump failed."); + } - // pushes dumped string on 'L' - luaL_pushresult( &b); // ... f b + // pushes dumped string on 'L' + luaL_pushresult( &b); // ... f b - // if not pushed, no need to pop - if( needToPush) + // if not pushed, no need to pop + if( needToPush) + { + lua_remove( L, -2); // ... b + } + + // transfer the bytecode, then the upvalues, to create a similar closure + { + char const* name = NULL; + + #if LOG_FUNC_INFO + // "To get information about a function you push it onto the + // stack and start the what string with the character '>'." + // { - lua_remove( L, -2); // ... b + lua_Debug ar; + lua_pushvalue( L, i); // ... b f + // fills 'name' 'namewhat' and 'linedefined', pops function + lua_getinfo(L, ">nS", &ar); // ... b + name = ar.namewhat; + fprintf( stderr, INDENT_BEGIN "FNAME: %s @ %d\n", i, s_indent, ar.short_src, ar.linedefined); // just gives NULL } - - // transfer the bytecode, then the upvalues, to create a similar closure + #endif // LOG_FUNC_INFO { - char const* name = NULL; - - #if LOG_FUNC_INFO - // "To get information about a function you push it onto the - // stack and start the what string with the character '>'." + size_t sz; + char const* s = lua_tolstring( L, -1, &sz); // ... b + ASSERT_L( s && sz); + STACK_GROW( L2, 2); + // Note: Line numbers seem to be taken precisely from the + // original function. 'name' is not used since the chunk + // is precompiled (it seems...). // + // TBD: Can we get the function's original name through, as well? + // + if( luaL_loadbuffer( L2, s, sz, name) != 0) // ... {cache} ... p function { - lua_Debug ar; - lua_pushvalue( L, i); // ... b f - // fills 'name' 'namewhat' and 'linedefined', pops function - lua_getinfo(L, ">nS", &ar); // ... b - name = ar.namewhat; - fprintf( stderr, INDENT_BEGIN "FNAME: %s @ %d\n", i, s_indent, ar.short_src, ar.linedefined); // just gives NULL - } - #endif // LOG_FUNC_INFO - { - size_t sz; - char const* s = lua_tolstring( L, -1, &sz); // ... b - ASSERT_L( s && sz); - STACK_GROW( L2, 2); - // Note: Line numbers seem to be taken precisely from the - // original function. 'name' is not used since the chunk - // is precompiled (it seems...). - // - // TBD: Can we get the function's original name through, as well? + // chunk is precompiled so only LUA_ERRMEM can happen + // "Otherwise, it pushes an error message" // - if( luaL_loadbuffer( L2, s, sz, name) != 0) // ... {cache} ... p function - { - // chunk is precompiled so only LUA_ERRMEM can happen - // "Otherwise, it pushes an error message" - // - STACK_GROW( L, 1); - luaL_error( L, "%s", lua_tostring( L2, -1)); - } - // remove the dumped string - lua_pop( L, 1); // ... - // now set the cache as soon as we can. - // this is necessary if one of the function's upvalues references it indirectly - // we need to find it in the cache even if it isn't fully transfered yet - lua_insert( L2, -2); // ... {cache} ... function p - lua_pushvalue( L2, -2); // ... {cache} ... function p function - // cache[p] = function - lua_rawset( L2, L2_cache_i); // ... {cache} ... function + STACK_GROW( L, 1); + luaL_error( L, "%s", lua_tostring( L2, -1)); } - STACK_MID( L, 0); + // remove the dumped string + lua_pop( L, 1); // ... + // now set the cache as soon as we can. + // this is necessary if one of the function's upvalues references it indirectly + // we need to find it in the cache even if it isn't fully transfered yet + lua_insert( L2, -2); // ... {cache} ... function p + lua_pushvalue( L2, -2); // ... {cache} ... function p function + // cache[p] = function + lua_rawset( L2, L2_cache_i); // ... {cache} ... function + } + STACK_MID( L, 0); - /* push over any upvalues; references to this function will come from - * cache so we don't end up in eternal loop. - * Lua5.2: one of the upvalues is _ENV, which we don't want to copy! - * instead, the function shall have LUA_RIDX_GLOBALS taken in the destination state! - */ - { - char const* upname; + /* push over any upvalues; references to this function will come from + * cache so we don't end up in eternal loop. + * Lua5.2: one of the upvalues is _ENV, which we don't want to copy! + * instead, the function shall have LUA_RIDX_GLOBALS taken in the destination state! + */ + { + char const* upname; #if LUA_VERSION_NUM == 502 - // With Lua 5.2, each Lua function gets its environment as one of its upvalues (named LUA_ENV, aka "_ENV" by default) - // Generally this is LUA_RIDX_GLOBALS, which we don't want to copy from the source to the destination state... - // -> if we encounter an upvalue equal to the global table in the source, bind it to the destination's global table - lua_pushglobaltable( L); // ... _G + // With Lua 5.2, each Lua function gets its environment as one of its upvalues (named LUA_ENV, aka "_ENV" by default) + // Generally this is LUA_RIDX_GLOBALS, which we don't want to copy from the source to the destination state... + // -> if we encounter an upvalue equal to the global table in the source, bind it to the destination's global table + lua_pushglobaltable( L); // ... _G #endif // LUA_VERSION_NUM - for( n = 0; (upname = lua_getupvalue( L, i, 1 + n)) != NULL; ++ n) - { // ... _G up[n] - DEBUGSPEW_CODE( fprintf( stderr, INDENT_BEGIN "UPNAME[%d]: %s\n" INDENT_END, n, upname)); + for( n = 0; (upname = lua_getupvalue( L, i, 1 + n)) != NULL; ++ n) + { // ... _G up[n] + DEBUGSPEW_CODE( fprintf( stderr, INDENT_BEGIN "UPNAME[%d]: %s\n" INDENT_END, n, upname)); #if LUA_VERSION_NUM == 502 - if( lua_rawequal( L, -1, -2)) // is the upvalue equal to the global table? - { - lua_pushglobaltable( L2); // ... {cache} ... function - } - else + if( lua_rawequal( L, -1, -2)) // is the upvalue equal to the global table? + { + lua_pushglobaltable( L2); // ... {cache} ... function + } + else #endif // LUA_VERSION_NUM + { + if( !inter_copy_one_( L2, L2_cache_i, L, lua_gettop( L), VT_NORMAL, mode_, upname)) // ... {cache} ... function { - if( !inter_copy_one_( L2, L2_cache_i, L, lua_gettop( L), VT_NORMAL, upname)) // ... {cache} ... function - { - luaL_error( L, "Cannot copy upvalue type '%s'", luaL_typename( L, -1)); - } + luaL_error( L, "Cannot copy upvalue type '%s'", luaL_typename( L, -1)); } - lua_pop( L, 1); // ... _G } + lua_pop( L, 1); // ... _G + } #if LUA_VERSION_NUM == 502 - lua_pop( L, 1); // ... + lua_pop( L, 1); // ... #endif // LUA_VERSION_NUM - } - // L2: function + 'n' upvalues (>=0) + } + // L2: function + 'n' upvalues (>=0) - STACK_MID( L, 0); + STACK_MID( L, 0); - // Set upvalues (originally set to 'nil' by 'lua_load') + // Set upvalues (originally set to 'nil' by 'lua_load') + { + int func_index = lua_gettop( L2) - n; + for( ; n > 0; -- n) { - int func_index = lua_gettop( L2) - n; - for( ; n > 0; -- n) - { - char const* rc = lua_setupvalue( L2, func_index, n); // ... {cache} ... function - // - // "assigns the value at the top of the stack to the upvalue and returns its name. - // It also pops the value from the stack." + char const* rc = lua_setupvalue( L2, func_index, n); // ... {cache} ... function + // + // "assigns the value at the top of the stack to the upvalue and returns its name. + // It also pops the value from the stack." - ASSERT_L( rc); // not having enough slots? - } - // once all upvalues have been set we are left - // with the function at the top of the stack // ... {cache} ... function + ASSERT_L( rc); // not having enough slots? } + // once all upvalues have been set we are left + // with the function at the top of the stack // ... {cache} ... function } } - else // C function OR LuaJIT fast function!!! + STACK_END( L, 0); +} + +/* + * Check if we've already copied the same function from 'L', and reuse the old + * copy. + * + * Always pushes a function to 'L2'. + */ +static void push_cached_func( lua_State* L2, uint_t L2_cache_i, lua_State* L, uint_t i, enum eLookupMode mode_, char const* upName_) +{ + FuncSubType funcSubType; + /*lua_CFunction cfunc =*/ luaG_tocfunction( L, i, &funcSubType); // NULL for LuaJIT-fast && bytecode functions + if( funcSubType == FST_Bytecode) + { + void* const aspointer = (void*)lua_topointer( L, i); + // TBD: Merge this and same code for tables + ASSERT_L( L2_cache_i != 0); + + STACK_GROW( L2, 2); + + // L2_cache[id_str]= function + // + STACK_CHECK( L2); + + // We don't need to use the from state ('L') in ID since the life span + // is only for the duration of a copy (both states are locked). + // + + // push a light userdata uniquely representing the function + lua_pushlightuserdata( L2, aspointer); // ... {cache} ... p + + //fprintf( stderr, "<< ID: %s >>\n", lua_tostring(L2,-1) ); + + lua_pushvalue( L2, -1); // ... {cache} ... p p + lua_rawget( L2, L2_cache_i); // ... {cache} ... p function|nil|true + + if( lua_isnil(L2,-1)) // function is unknown + { + lua_pop( L2, 1); // ... {cache} ... p + + // Set to 'true' for the duration of creation; need to find self-references + // via upvalues + // + // pushes a copy of the func, stores a reference in the cache + inter_copy_func( L2, L2_cache_i, L, i, mode_, upName_); // ... {cache} ... function + } + else // found function in the cache + { + lua_remove( L2, -2); // ... {cache} ... function + } + STACK_END( L2, 1); + } + else // function is native/LuaJIT: no need to cache { - lua_pop( L2, 1); // ... {cache} ... - // No need to transfer upvalues for C/JIT functions since they weren't actually copied, only looked up - lookup_native_func( L2, L, i, upName_); // ... {cache} ... function + lookup_native_func( L2, L, i, mode_, upName_); // ... {cache} ... function } - STACK_END( L, 0); + + // + // L2 [-1]: function + ASSERT_L( lua_isfunction( L2, -1)); } /* @@ -1746,7 +1759,7 @@ static void inter_copy_func( lua_State* L2, uint_t L2_cache_i, lua_State* L, uin * * Returns TRUE if value was pushed, FALSE if its type is non-supported. */ -static bool_t inter_copy_one_( lua_State* L2, uint_t L2_cache_i, lua_State* L, uint_t i, enum e_vt vt, char const* upName_) +static bool_t inter_copy_one_( lua_State* L2, uint_t L2_cache_i, lua_State* L, uint_t i, enum e_vt vt, enum eLookupMode mode_, char const* upName_) { bool_t ret = TRUE; @@ -1764,6 +1777,7 @@ static bool_t inter_copy_one_( lua_State* L2, uint_t L2_cache_i, lua_State* L, u case LUA_TNUMBER: /* LNUM patch support (keeping integer accuracy) */ + // TODO: support for integer in Lua 5.3 #ifdef LUA_LNUM if( lua_isinteger(L,i)) { @@ -1831,7 +1845,7 @@ static bool_t inter_copy_one_( lua_State* L2, uint_t L2_cache_i, lua_State* L, u { DEBUGSPEW_CODE( fprintf( stderr, INDENT_BEGIN "FUNCTION\n" INDENT_END)); STACK_CHECK( L2); - push_cached_func( L2, L2_cache_i, L, i, upName_); + push_cached_func( L2, L2_cache_i, L, i, mode_, upName_); STACK_END( L2, 1); } break; @@ -1873,7 +1887,7 @@ static bool_t inter_copy_one_( lua_State* L2, uint_t L2_cache_i, lua_State* L, u /* Only basic key types are copied over; others ignored */ - if( inter_copy_one_( L2, 0 /*key*/, L, key_i, VT_KEY, upName_)) + if( inter_copy_one_( L2, 0 /*key*/, L, key_i, VT_KEY, mode_, upName_)) { char* valPath = (char*) upName_; if( GVerboseErrors) @@ -1894,7 +1908,7 @@ static bool_t inter_copy_one_( lua_State* L2, uint_t L2_cache_i, lua_State* L, u * Contents of metatables are copied with cache checking; * important to detect loops. */ - if( inter_copy_one_( L2, L2_cache_i, L, val_i, VT_NORMAL, valPath)) + if( inter_copy_one_( L2, L2_cache_i, L, val_i, VT_NORMAL, mode_, valPath)) { ASSERT_L( lua_istable(L2,-3)); lua_rawset( L2, -3); // add to table (pops key & val) @@ -1936,7 +1950,7 @@ static bool_t inter_copy_one_( lua_State* L2, uint_t L2_cache_i, lua_State* L, u lua_pop( L2, 1); STACK_MID( L2, 2); ASSERT_L( lua_istable(L,-1)); - if( inter_copy_one_( L2, L2_cache_i /*for function cacheing*/, L, lua_gettop(L) /*[-1]*/, VT_METATABLE, upName_)) + if( inter_copy_one_( L2, L2_cache_i /*for function cacheing*/, L, lua_gettop(L) /*[-1]*/, VT_METATABLE, mode_, upName_)) { // // L2 ([-3]: copied table) @@ -1981,8 +1995,8 @@ static bool_t inter_copy_one_( lua_State* L2, uint_t L2_cache_i, lua_State* L, u } STACK_END( L2, 1); STACK_END( L, 0); - } - break; + } + break; /* The following types cannot be copied */ @@ -2004,7 +2018,7 @@ static bool_t inter_copy_one_( lua_State* L2, uint_t L2_cache_i, lua_State* L, u * * Note: Parameters are in this order ('L' = from first) to be same as 'lua_xmove'. */ -int luaG_inter_copy( lua_State* L, lua_State* L2, uint_t n) +int luaG_inter_copy( lua_State* L, lua_State* L2, uint_t n, enum eLookupMode mode_) { uint_t top_L = lua_gettop( L); uint_t top_L2 = lua_gettop( L2); @@ -2034,7 +2048,7 @@ int luaG_inter_copy( lua_State* L, lua_State* L2, uint_t n) { sprintf( tmpBuf, "arg_%d", j); } - copyok = inter_copy_one_( L2, top_L2 + 1, L, i, VT_NORMAL, pBuf); + copyok = inter_copy_one_( L2, top_L2 + 1, L, i, VT_NORMAL, mode_, pBuf); if( !copyok) { break; @@ -2062,14 +2076,14 @@ int luaG_inter_copy( lua_State* L, lua_State* L2, uint_t n) } -int luaG_inter_move( lua_State* L, lua_State* L2, uint_t n) +int luaG_inter_move( lua_State* L, lua_State* L2, uint_t n, enum eLookupMode mode_) { - int ret = luaG_inter_copy( L, L2, n); + int ret = luaG_inter_copy( L, L2, n, mode_); lua_pop( L, (int) n); return ret; } -void luaG_inter_copy_package( lua_State* L, lua_State* L2, int _idx) +void luaG_inter_copy_package( lua_State* L, lua_State* L2, int _idx, enum eLookupMode mode_) { DEBUGSPEW_CODE( fprintf( stderr, INDENT_BEGIN "luaG_inter_copy_package()\n" INDENT_END)); DEBUGSPEW_CODE( ++ debugspew_indent_depth); @@ -2100,7 +2114,7 @@ void luaG_inter_copy_package( lua_State* L, lua_State* L2, int _idx) else { DEBUGSPEW_CODE( ++ debugspew_indent_depth); - luaG_inter_move( L, L2, 1); // moves the entry to L2 + luaG_inter_move( L, L2, 1, mode_); // moves the entry to L2 DEBUGSPEW_CODE( -- debugspew_indent_depth); lua_setfield( L2, -2, entries[i]); // set package[entries[i]] } diff --git a/src/tools.h b/src/tools.h index 93ed92c..bf48f1f 100644 --- a/src/tools.h +++ b/src/tools.h @@ -90,10 +90,17 @@ typedef struct { } DEEP_PRELUDE; void luaG_push_proxy( lua_State *L, luaG_IdFunction idfunc, DEEP_PRELUDE *deep_userdata); -void luaG_inter_copy_package( lua_State* L, lua_State* L2, int _idx); +void luaG_inter_copy_package( lua_State* L, lua_State* L2, int _idx, enum eLookupMode mode_); -int luaG_inter_copy( lua_State *L, lua_State *L2, uint_t n); -int luaG_inter_move( lua_State *L, lua_State *L2, uint_t n); +enum eLookupMode +{ + eLM_LaneBody, // send the lane body directly from the source to the destination lane + eLM_ToKeeper, // send a function from a lane to a keeper state + eLM_FromKeeper, // send a function from a keeper state to a lane +}; + +int luaG_inter_copy( lua_State *L, lua_State *L2, uint_t n, enum eLookupMode mode_); +int luaG_inter_move( lua_State *L, lua_State *L2, uint_t n, enum eLookupMode mode_); int luaG_nameof( lua_State* L); int luaG_new_require( lua_State* L); -- cgit v1.2.3-55-g6feb