new .internal_allocator configuration IUNTESTED)

new configuration option .internal_allocator to help LuaJIT users. THIS IS YET UNTESTED, USE AT YOUR OWN RISKS.

13 files changed, 83 insertions, 62 deletions

diff --git a/.vs/Lanes/FileContentIndex/7013d1f4-51ff-4696-9603-47b7bcd2f51f.vsidx b/.vs/Lanes/FileContentIndex/7013d1f4-51ff-4696-9603-47b7bcd2f51f.vsidx
new file mode 100644
index 0000000..7c4c705
--- /dev/null
+++ b/.vs/Lanes/FileContentIndex/7013d1f4-51ff-4696-9603-47b7bcd2f51f.vsidx
diff --git a/CHANGES b/CHANGES
index 3b333b2..3c7f356 100644
--- a/CHANGES
+++ b/CHANGES
@@ -1,5 +1,9 @@
 CHANGES:
 
+CHANGE 156: BGe 9-Aug-23
+   * new configuration option .internal_allocator to help LuaJIT users.
+   * internal version bumped to 3.16.1
+
 CHANGE 155: BGe 28-Jul-23
    * tweaks to linux thread priority management: do nothing if not super-user. if super-user, do nothing if nothing is provided (instead of trying to force a prio when LINUX_SCHED_RR is defined).
 
diff --git a/docs/index.html b/docs/index.html
index 0e1a30b..b1c514d 100644
--- a/docs/index.html
+++ b/docs/index.html
@@ -64,13 +64,13 @@
                 <font size="-1">
                         <p>
                                 <br/>
-                                <i>Copyright &copy; 2007-22 Asko Kauppi, Benoit Germain. All rights reserved.</i>
+                                <i>Copyright &copy; 2007-23 Asko Kauppi, Benoit Germain. All rights reserved.</i>
                                 <br/>
                                 Lua Lanes is published under the same <a href="http://en.wikipedia.org/wiki/MIT_License">MIT license</a> as Lua 5.1, 5.2, 5.3 and 5.4.
                         </p>
 
                         <p>
-                                This document was revised on 8-Feb-22, and applies to version <tt>3.16.0</tt>.
+                                This document was revised on 9-Aug-23, and applies to version <tt>3.16.1</tt>.
                         </p>
                 </font>
         </center>
@@ -338,9 +338,9 @@
                                 <tt>nil</tt>/<tt>"protected"</tt>/function
                         </td>
                         <td>
-                                (Since v3.13.0)<br/>
-                                If <tt>nil</tt>, Lua states are created with <tt>lua_newstate()</tt> and reuse the allocator from the master state.<br/>
-                                If <tt>"protected"</tt>, The default allocator obtained from <tt>lua_getallocf()</tt> in the master state is wrapped inside a critical section and used in all newly created states.<br/>
+                                (Since v3.13.0)<br />
+                                If <tt>nil</tt>, Lua states are created with <tt>lua_newstate()</tt> and reuse the allocator from the master state.<br />
+                                If <tt>"protected"</tt>, The default allocator obtained from <tt>lua_getallocf()</tt> in the master state is wrapped inside a critical section and used in all newly created states.<br />
                                 If a <tt>function</tt>, this function is called prior to creating the state. It should return a full userdata containing the following structure:
                                 <table border="1" bgcolor="#E0E0FF" cellpadding="10" style="width:50%">
                                         <tr>
@@ -356,6 +356,22 @@
                 </tr>
 
                 <tr valign=top>
+                        <td id="internal_allocator">
+                                <code>.internal_allocator</code>
+                        </td>
+                        <td>
+                                <tt>"libc"</tt>/<tt>"allocator"</tt>
+                        </td>
+                        <td>
+                                (Since v3.16.1)<br />
+                                Controls which allocator is used for Lanest internal allocations (for keeper and deep userdata management).
+                                If <tt>"libc"</tt>, Lanes uses <tt>realloc</tt> and <tt>free</tt>.<br />
+                                If <tt>"allocator"</tt>, Lanes uses whatever was obtained from the <tt>"allocator"</tt> setting.<br />
+                                This option is mostly useful for embedders that want control all memory allocations, but have issues when Lanes tries to use the Lua State allocator for internal purposes (especially with LuaJIT).
+                        </td>
+                </tr>
+
+                <tr valign=top>
                         <td id="demote_full_userdata">
                                 <code>.demote_full_userdata</code>
                         </td>
@@ -390,18 +406,18 @@
                         </td>
                         <td>
                                 If provided, will be called in every created Lua state right after initializing the base libraries.
-                                <br/>
+                                <br />
                                 Keeper states will call it as well, but only if it is a C function (keeper states are not able to execute any user Lua code).
-                                <br/>
+                                <br />
                                 Typical usage is twofold:
                                 <ul>
                                         <li>Tweak <tt>package.loaders</tt></li>
                                         <li>Load some additional C functions in the global space (of course only a C function will be able to do this).</li>
                                 </ul>
                                 That way, all changes in the state can be properly taken into account when building the function lookup database. Default is <tt>nil</tt>.
-                                <br/>
+                                <br />
                                 (Since version 3.7.6) If <tt>on_state_create()</tt> is a Lua function, it will be transfered normally before the call.
-                                <br/>
+                                <br />
                                 If it is a C function, a C closure will be reconstructed in the created state from the C pointer. Lanes will raise an error if the function has upvalues.
                         </td>
                 </tr>
diff --git a/src/deep.c b/src/deep.c
index c475dc5..9496477 100644
--- a/src/deep.c
+++ b/src/deep.c
@@ -389,7 +389,7 @@ int luaG_newdeepuserdata( lua_State* L, luaG_IdFunction idfunc, int nuv_)
         DeepPrelude* prelude = idfunc( L, eDO_new);
         if( prelude == NULL)
         {
-            luaL_error( L, "idfunc(eDO_new) failed to create deep userdata (out of memory)");
+            return luaL_error( L, "idfunc(eDO_new) failed to create deep userdata (out of memory)");
         }
         if( prelude->magic.value != DEEP_VERSION.value)
         {
diff --git a/src/keeper.c b/src/keeper.c
index 19b9e1a..f4dde0a 100644
--- a/src/keeper.c
+++ b/src/keeper.c
@@ -580,7 +580,7 @@ int keepercall_count( lua_State* L)
 */
 
 // called as __gc for the keepers array userdata
-void close_keepers( Universe* U, lua_State* L)
+void close_keepers( Universe* U)
 {
     if( U->keepers != NULL)
     {
@@ -611,15 +611,8 @@ void close_keepers( Universe* U, lua_State* L)
         }
         // free the keeper bookkeeping structure
         {
-            // don't hijack the state allocator when running LuaJIT because it looks like LuaJIT does not expect it and might invalidate the memory unexpectedly
-#if USE_LUA_STATE_ALLOCATOR()
-            {
-                AllocatorDefinition* const allocD = &U->protected_allocator.definition;
-                allocD->allocF( allocD->allocUD, U->keepers, sizeof( Keepers) + (nbKeepers - 1) * sizeof( Keeper), 0);
-            }
-#else // USE_LUA_STATE_ALLOCATOR()
-            free(U->keepers);
-#endif // USE_LUA_STATE_ALLOCATOR()
+            AllocatorDefinition* const allocD = &U->internal_allocator;
+            allocD->allocF( allocD->allocUD, U->keepers, sizeof( Keepers) + (nbKeepers - 1) * sizeof( Keeper), 0);
             U->keepers = NULL;
         }
     }
@@ -653,15 +646,10 @@ void init_keepers( Universe* U, lua_State* L)
     // Keepers contains an array of 1 s_Keeper, adjust for the actual number of keeper states
     {
         size_t const bytes = sizeof( Keepers) + (nb_keepers - 1) * sizeof( Keeper);
-        // don't hijack the state allocator when running LuaJIT because it looks like LuaJIT does not expect it and might invalidate the memory unexpectedly
-#if USE_LUA_STATE_ALLOCATOR()
         {
-            AllocatorDefinition* const allocD = &U->protected_allocator.definition;
+            AllocatorDefinition* const allocD = &U->internal_allocator;
             U->keepers = (Keepers*) allocD->allocF( allocD->allocUD, NULL, 0, bytes);
         }
-#else // USE_LUA_STATE_ALLOCATOR()
-        U->keepers = (Keepers*)malloc(bytes);
-#endif // USE_LUA_STATE_ALLOCATOR()
         if( U->keepers == NULL)
         {
             (void) luaL_error( L, "init_keepers() failed while creating keeper array; out of memory");
diff --git a/src/keeper.h b/src/keeper.h
index 8c09322..d30aa36 100644
--- a/src/keeper.h
+++ b/src/keeper.h
@@ -27,7 +27,7 @@ struct s_Keepers
 typedef struct s_Keepers Keepers;
 
 void init_keepers( Universe* U, lua_State* L);
-void close_keepers( Universe* U, lua_State* L);
+void close_keepers( Universe* U);
 
 Keeper* which_keeper( Keepers* keepers_, ptrdiff_t magic_);
 Keeper* keeper_acquire( Keepers* keepers_, ptrdiff_t magic_);
diff --git a/src/lanes.c b/src/lanes.c
index 9f6a4d6..0aab244 100644
--- a/src/lanes.c
+++ b/src/lanes.c
@@ -253,15 +253,10 @@ static void lane_cleanup( Lane* s)
     }
 #endif // HAVE_LANE_TRACKING()
 
-    // don't hijack the state allocator when running LuaJIT because it looks like LuaJIT does not expect it and might invalidate the memory unexpectedly
-#if USE_LUA_STATE_ALLOCATOR()
     {
-        AllocatorDefinition* const allocD = &s->U->protected_allocator.definition;
+        AllocatorDefinition* const allocD = &s->U->internal_allocator;
         allocD->allocF(allocD->allocUD, s, sizeof(Lane), 0);
     }
-#else // USE_LUA_STATE_ALLOCATOR()
-    free(s);
-#endif // USE_LUA_STATE_ALLOCATOR()
 }
 
 /*
@@ -584,7 +579,7 @@ static int selfdestruct_gc( lua_State* L)
         U->timer_deep = NULL;
     }
 
-    close_keepers( U, L);
+    close_keepers( U);
 
     // remove the protected allocator, if any
     cleanup_allocator_function( U, L);
@@ -1231,15 +1226,10 @@ LUAG_FUNC( lane_new)
     //
     // a Lane full userdata needs a single uservalue
     ud = lua_newuserdatauv( L, sizeof( Lane*), 1);           // func libs priority globals package required gc_cb lane
-    // don't hijack the state allocator when running LuaJIT because it looks like LuaJIT does not expect it and might invalidate the memory unexpectedly
-#if USE_LUA_STATE_ALLOCATOR()
     {
-        AllocatorDefinition* const allocD = &U->protected_allocator.definition;
+        AllocatorDefinition* const allocD = &U->internal_allocator;
         s = *ud = (Lane*)allocD->allocF(allocD->allocUD, NULL, 0, sizeof(Lane));
     }
-#else // USE_LUA_STATE_ALLOCATOR()
-    s = *ud = (Lane*) malloc(sizeof(Lane));
-#endif // USE_LUA_STATE_ALLOCATOR()
     if( s == NULL)
     {
         return luaL_error( L, "could not create lane: out of memory");
diff --git a/src/lanes.h b/src/lanes.h
index 20524e6..420b31d 100644
--- a/src/lanes.h
+++ b/src/lanes.h
@@ -12,7 +12,7 @@
 
 #define LANES_VERSION_MAJOR 3
 #define LANES_VERSION_MINOR 16
-#define LANES_VERSION_PATCH 0
+#define LANES_VERSION_PATCH 1
 
 #define LANES_MIN_VERSION_REQUIRED(MAJOR, MINOR, PATCH)     ((LANES_VERSION_MAJOR>MAJOR) || (LANES_VERSION_MAJOR==MAJOR && (LANES_VERSION_MINOR>MINOR || (LANES_VERSION_MINOR==MINOR && LANES_VERSION_PATCH>=PATCH))))
 #define LANES_VERSION_LESS_THAN(MAJOR, MINOR, PATCH)        ((LANES_VERSION_MAJOR<MAJOR) || (LANES_VERSION_MAJOR==MAJOR && (LANES_VERSION_MINOR<MINOR || (LANES_VERSION_MINOR==MINOR && LANES_VERSION_PATCH<PATCH))))
diff --git a/src/lanes.lua b/src/lanes.lua
index 0858ad7..cbcf74f 100644
--- a/src/lanes.lua
+++ b/src/lanes.lua
@@ -77,7 +77,9 @@ lanes.configure = function( settings_)
         demote_full_userdata = nil,
         verbose_errors = false,
         -- LuaJIT provides a thread-unsafe allocator by default, so we need to protect it when used in parallel lanes
-        allocator = (package.loaded.jit and jit.version) and "protected" or nil
+        allocator = (package.loaded.jit and jit.version) and "protected" or nil,
+        -- it looks also like LuaJIT allocator may not appreciate direct use of its allocator for other purposes than the VM operation
+        internal_allocator = (package.loaded.jit and jit.version) and "libc" or "allocator"
     }
     local boolean_param_checker = function( val_)
         -- non-'boolean-false' should be 'boolean-true' or nil
@@ -94,6 +96,10 @@ lanes.configure = function( settings_)
             -- can be nil, "protected", or a function
             return val_ and (type( val_) == "function" or val_ == "protected") or true
         end,
+        internal_allocator = function( val_)
+            -- can be "libc" or "allocator"
+            return val_ == "libc" or val_ == "allocator"
+        end,
         on_state_create = function( val_)
             -- on_state_create may be nil or a function
             return val_ and type( val_) == "function" or true
diff --git a/src/linda.c b/src/linda.c
index 637f909..390816b 100644
--- a/src/linda.c
+++ b/src/linda.c
@@ -794,17 +794,12 @@ static void* linda_id( lua_State* L, DeepOp op_)
             * One can use any memory allocation scheme.
             * just don't use L's allocF because we don't know which state will get the honor of GCing the linda
             */
-            // don't hijack the state allocator when running LuaJIT because it looks like LuaJIT does not expect it and might invalidate the memory unexpectedly
-#if USE_LUA_STATE_ALLOCATOR()
             {
                 Universe* const U = universe_get(L);
-                AllocatorDefinition* const allocD = &U->protected_allocator.definition;
+                AllocatorDefinition* const allocD = &U->internal_allocator;
 
                 s = (struct s_Linda*)allocD->allocF(allocD->allocUD, NULL, 0, sizeof(struct s_Linda) + name_len); // terminating 0 is already included
             }
-#else // USE_LUA_STATE_ALLOCATOR()
-            s = (struct s_Linda*)malloc(sizeof(struct s_Linda) + name_len); // terminating 0 is already included
-#endif // USE_LUA_STATE_ALLOCATOR()
             if( s)
             {
                 s->prelude.magic.value = DEEP_VERSION.value;
@@ -837,17 +832,12 @@ static void* linda_id( lua_State* L, DeepOp op_)
             // There aren't any lanes waiting on these lindas, since all proxies have been gc'ed. Right?
             SIGNAL_FREE( &linda->read_happened);
             SIGNAL_FREE( &linda->write_happened);
-            // don't hijack the state allocator when running LuaJIT because it looks like LuaJIT does not expect it and might invalidate the memory unexpectedly
-#if USE_LUA_STATE_ALLOCATOR()
             {
                 Universe* const U = universe_get(L);
-                AllocatorDefinition* const allocD = &U->protected_allocator.definition;
+                AllocatorDefinition* const allocD = &U->internal_allocator;
 
                 allocD->allocF(allocD->allocUD, linda, sizeof(struct s_Linda) + strlen(linda->name), 0);
             }
-#else // USE_LUA_STATE_ALLOCATOR()
-            free(linda);
-#endif // USE_LUA_STATE_ALLOCATOR()
             return NULL;
         }
 
diff --git a/src/macros_and_utils.h b/src/macros_and_utils.h
index 3ed234a..05a46b5 100644
--- a/src/macros_and_utils.h
+++ b/src/macros_and_utils.h
@@ -99,7 +99,4 @@ extern char const* debugspew_indent;
 
 #define LUAG_FUNC( func_name) int LG_##func_name( lua_State* L)
 
-// after all, it looks like we can use the state allocator for our own usage when running LuaJIT, as long as we mutex-protect it
-#define USE_LUA_STATE_ALLOCATOR() 1 // (LUAJIT_FLAVOR()==0)
-
 #endif // MACROS_AND_UTILS_H
diff --git a/src/tools.c b/src/tools.c
index 626da2b..5a6ae92 100644
--- a/src/tools.c
+++ b/src/tools.c
@@ -155,6 +155,20 @@ void luaG_dump( lua_State* L)
 
 // ################################################################################################
 
+static void* libc_lua_Alloc(void* ud, void* ptr, size_t osize, size_t nsize)
+{
+    (void)ud; (void)osize;  /* not used */
+    if (nsize == 0)
+    {
+        free(ptr);
+        return NULL;
+    }
+    else
+    {
+        return realloc(ptr, nsize);
+    }
+}
+
 static void* protected_lua_Alloc( void *ud, void *ptr, size_t osize, size_t nsize)
 {
     void* p;
@@ -217,6 +231,22 @@ void initialize_allocator_function( Universe* U, lua_State* L)
         U->protected_allocator.definition.allocF = lua_getallocf( L, &U->protected_allocator.definition.allocUD);
     }
     lua_pop( L, 1);                                         // settings
+    STACK_MID(L, 0);
+
+    lua_getfield( L, -1, "internal_allocator");             // settings "libc"|"allocator"
+    {
+        char const* allocator = lua_tostring( L, -1);
+        if (stricmp(allocator, "libc") == 0)
+        {
+            U->internal_allocator.allocF = libc_lua_Alloc;
+            U->internal_allocator.allocUD = NULL;
+        }
+        else
+        {
+            U->internal_allocator = U->protected_allocator.definition;
+        }
+    }
+    lua_pop( L, 1);                                         // settings
     STACK_END( L, 0);
 }
 
@@ -1337,17 +1367,17 @@ static void copy_cached_func( Universe* U, lua_State* L2, uint_t L2_cache_i, lua
 
         if( lua_isnil( L2, -1)) // function is unknown
         {
-            lua_pop( L2, 1);                                            // ... {cache} ... p
+            lua_pop( L2, 1);                                          // ... {cache} ... p
 
             // Set to 'true' for the duration of creation; need to find self-references
             // via upvalues
             //
             // pushes a copy of the func, stores a reference in the cache
-            copy_func( U, L2, L2_cache_i, L, i, mode_, upName_);        // ... {cache} ... function
+            copy_func( U, L2, L2_cache_i, L, i, mode_, upName_);      // ... {cache} ... function
         }
         else // found function in the cache
         {
-            lua_remove( L2, -2);                                        // ... {cache} ... function
+            lua_remove( L2, -2);                                      // ... {cache} ... function
         }
         STACK_END( L2, 1);
         ASSERT_L( lua_isfunction( L2, -1));
@@ -1725,9 +1755,7 @@ static bool_t inter_copy_function( Universe* U, lua_State* L2, uint_t L2_cache_i
     {
         DEBUGSPEW_CODE( fprintf( stderr, "FUNCTION %s\n", upName_));
         DEBUGSPEW_CODE( ++ U->debugspew_indent_depth);
-        STACK_CHECK( L2, 0);
         copy_cached_func( U, L2, L2_cache_i, L, source_i_, mode_, upName_);                                      // ... f
-        STACK_END( L2, 1);
         DEBUGSPEW_CODE( -- U->debugspew_indent_depth);
     }
     STACK_END( L2, 1);
diff --git a/src/universe.h b/src/universe.h
index ba00e87..03c78cf 100644
--- a/src/universe.h
+++ b/src/universe.h
@@ -63,6 +63,8 @@ struct s_Universe
     // contains a mutex and the original allocator definition
     ProtectedAllocator protected_allocator;
 
+    AllocatorDefinition internal_allocator;
+
     Keepers* keepers;
 
     // Initialized by 'init_once_LOCKED()': the deep userdata Linda object