Preparation for lane:close() and correct to-be-closed variables

* lane:join() can no longer be used to read yielded values
* same with lane indexing
* stub for lane:close(), similar to coroutine.close() (not implemented yet)
* preparing tests for to-be-closed variables in yielded coroutine lanes
* yielded lanes unlock and terminate properly at Lanes shutdown

8 files changed, 237 insertions, 106 deletions

diff --git a/unit_tests/UnitTests.vcxproj b/unit_tests/UnitTests.vcxproj
index 79361c5..d6bfd5a 100644
--- a/unit_tests/UnitTests.vcxproj
+++ b/unit_tests/UnitTests.vcxproj
@@ -967,11 +967,13 @@
   </ItemGroup>
   <ItemGroup>
     <None Include="..\.runsettings" />
+    <None Include="scripts\coro\collect_yielded_lane.lua" />
+    <None Include="scripts\coro\regular_function.lua" />
     <None Include="scripts\lane\body_is_a_c_function.lua" />
     <None Include="scripts\lane\tasking_cancelling_with_hook.lua" />
     <None Include="scripts\linda\wake_period.lua" />
     <None Include="UnitTests.makefile" />
-    <None Include="scripts\coro\basics.lua" />
+    <None Include="scripts\coro\yielding_function.lua" />
     <None Include="scripts\coro\error_handling.lua" />
     <None Include="scripts\lane\cooperative_shutdown.lua" />
     <None Include="scripts\lane\stdlib_naming.lua" />
diff --git a/unit_tests/UnitTests.vcxproj.filters b/unit_tests/UnitTests.vcxproj.filters
index dfa642b..c5a2761 100644
--- a/unit_tests/UnitTests.vcxproj.filters
+++ b/unit_tests/UnitTests.vcxproj.filters
@@ -95,7 +95,7 @@
     <None Include="scripts\lane\tasking_join_test.lua">
       <Filter>Scripts\lane</Filter>
     </None>
-    <None Include="scripts\coro\basics.lua">
+    <None Include="scripts\coro\yielding_function.lua">
       <Filter>Scripts\coro</Filter>
     </None>
     <None Include="scripts\coro\error_handling.lua">
@@ -119,8 +119,14 @@
     <None Include="scripts\linda\wake_period.lua">
       <Filter>Scripts\linda</Filter>
     </None>
+    <None Include="scripts\coro\collect_yielded_lane.lua">
+      <Filter>Scripts\coro</Filter>
+    </None>
     <None Include="scripts\lane\body_is_a_c_function.lua">
       <Filter>Scripts\lane</Filter>
     </None>
+    <None Include="scripts\coro\regular_function.lua">
+      <Filter>Scripts\coro</Filter>
+    </None>
   </ItemGroup>
 </Project>
\ No newline at end of file
diff --git a/unit_tests/lane_tests.cpp b/unit_tests/lane_tests.cpp
index 82ca1ad..c4d3c95 100644
--- a/unit_tests/lane_tests.cpp
+++ b/unit_tests/lane_tests.cpp
@@ -421,7 +421,7 @@ TEST_CASE("scripted_tests." #DIR "." #FILE) \
 
 MAKE_TEST_CASE(lane, body_is_a_c_function, AssertNoLuaError)
 MAKE_TEST_CASE(lane, cooperative_shutdown, AssertNoLuaError)
-#if LUA_VERSION_NUM >= 504 // // warnings are a Lua 5.4 feature
+#if LUA_VERSION_NUM >= 504 // warnings are a Lua 5.4 feature
 // NOTE: when this test ends, there are resource leaks and a dangling thread
 MAKE_TEST_CASE(lane, uncooperative_shutdown, AssertWarns)
 #endif // LUA_VERSION_NUM
@@ -434,11 +434,16 @@ MAKE_TEST_CASE(lane, tasking_error, AssertNoLuaError)
 MAKE_TEST_CASE(lane, tasking_join_test, AssertNoLuaError)
 MAKE_TEST_CASE(lane, tasking_send_receive_code, AssertNoLuaError)
 MAKE_TEST_CASE(lane, stdlib_naming, AssertNoLuaError)
-MAKE_TEST_CASE(coro, basics, AssertNoLuaError)
+
+#if LUA_VERSION_NUM >= 504 // this makes use of to-be-closed variables, a Lua 5.4 feature
+MAKE_TEST_CASE(coro, collect_yielded_lane, AssertNoLuaError)
+#endif // LUA_VERSION_NUM
 #if LUAJIT_FLAVOR() == 0
 // TODO: for some reason, the test fails with LuaJIT. To be investigated
 MAKE_TEST_CASE(coro, error_handling, AssertNoLuaError)
 #endif // LUAJIT_FLAVOR()
+MAKE_TEST_CASE(coro, regular_function, AssertNoLuaError)
+MAKE_TEST_CASE(coro, yielding_function, AssertNoLuaError)
 
 /*
 TEST_CASE("lanes.scripted_tests")
diff --git a/unit_tests/scripts/coro/basics.lua b/unit_tests/scripts/coro/basics.lua
deleted file mode 100644
index c0b7a36..0000000
--- a/unit_tests/scripts/coro/basics.lua
+++ /dev/null
@@ -1,99 +0,0 @@
-local lanes = require "lanes"
-
-local fixture = require "fixture"
-lanes.finally(fixture.throwing_finalizer)
-
-local utils = lanes.require "_utils"
-local PRINT = utils.MAKE_PRINT()
-
-if true then
-    -- a lane body that just returns some value
-    local lane = function(msg_)
-        local utils = lanes.require "_utils"
-        local PRINT = utils.MAKE_PRINT()
-        PRINT "In lane"
-        assert(msg_ == "hi")
-        return "bye"
-    end
-
-    -- the generator
-    local g1 = lanes.coro("*", {name = "auto"}, lane)
-
-    -- launch lane
-    local h1 = g1("hi")
-
-    local r = h1[1]
-    assert(r == "bye")
-end
-
--- a lane coroutine that yields back what got in, one element at a time
-local yielder = function(...)
-    local utils = lanes.require "_utils"
-    local PRINT = utils.MAKE_PRINT()
-    PRINT "In lane"
-    for _i  = 1, select('#', ...) do
-        local _val = select(_i, ...)
-        PRINT("yielding #", _i, _val)
-        local _ack = coroutine.yield(_val)
-        assert(_ack == _i)
-    end
-    return "done!"
-end
-
-if true then
-    -- if we start a non-coroutine lane with a yielding function, we should get an error, right?
-    local fun_g = lanes.gen("*", { name = 'auto' }, yielder)
-    local h = fun_g("hello", "world", "!")
-    local err, status, stack = h:join()
-    PRINT(err, status, stack)
-    -- the actual error message is not the same for Lua 5.1
-    -- of course, it also has to be different for LuaJIT as well
-    -- also, LuaJIT prepends a file:line to the actual error message, which Lua5.1 does not.
-    local msgs = {
-        ["Lua 5.1"] = jit and "attempt to yield across C-call boundary" or "attempt to yield across metamethod/C-call boundary",
-        ["Lua 5.2"] = "attempt to yield from outside a coroutine",
-        ["Lua 5.3"] = "attempt to yield from outside a coroutine",
-        ["Lua 5.4"] = "attempt to yield from outside a coroutine"
-    }
-    local expected_msg = msgs[_VERSION]
-    PRINT("expected_msg = " .. expected_msg)
-    assert(err == nil and string.find(status, expected_msg, 1, true) and stack == nil, "status = " .. status)
-end
-
--- the generator
-local coro_g = lanes.coro("*", {name = "auto"}, yielder)
-
-if true then
-    -- launch coroutine lane
-    local h2 = coro_g("hello", "world", "!")
-    -- read the yielded values, sending back the expected index
-    assert(h2:resume(1) == "hello")
-    assert(h2:resume(2) == "world")
-    assert(h2:resume(3) == "!")
-    -- the lane return value is available as usual
-    local r = h2[1]
-    assert(r == "done!")
-end
-
-if true then
-    -- another coroutine lane
-    local h3 = coro_g("hello", "world", "!")
-
-    -- yielded values are available as regular return values
-    assert(h3[1] == "hello" and h3.status == "suspended")
-    -- since we consumed the returned values, they should not be here when we resume
-    assert(h3:resume(1) == nil)
-
-    -- similarly, we can get them with join()
-    local r3, ret3 = h3:join()
-    assert(r3 == true and ret3 == "world" and h3.status == "suspended")
-    -- since we consumed the returned values, they should not be here when we resume
-    assert(h3:resume(2) == nil)
-
-    -- the rest should work as usual
-    assert(h3:resume(3) == "!")
-
-    -- the final return value of the lane body remains to be read
-    local r3, ret3 = h3:join()
-    assert(r3 == true and ret3 == "done!" and h3.status == "done")
-end
diff --git a/unit_tests/scripts/coro/collect_yielded_lane.lua b/unit_tests/scripts/coro/collect_yielded_lane.lua
new file mode 100644
index 0000000..0459698
--- /dev/null
+++ b/unit_tests/scripts/coro/collect_yielded_lane.lua
@@ -0,0 +1,72 @@
+local lanes = require "lanes"
+
+local fixture = require "fixture"
+lanes.finally(fixture.throwing_finalizer)
+
+local utils = lanes.require "_utils"
+local PRINT = utils.MAKE_PRINT()
+
+-- a lane body that yields stuff
+local yielder = function(out_linda_)
+        -- here is a to-be-closed variable that, when closed, sends "Closed!" in the "out" slot of the provided linda
+        local t <close> = setmetatable(
+                { text = "Closed!" }, {
+                        __close = function(self, err)
+                                out_linda_:send("out", self.text)
+                        end
+                }
+        )
+        -- yield forever
+        while true do
+                coroutine.yield("I yield!")
+        end
+end
+
+local out_linda = lanes.linda()
+
+local test_close = function(what_, f_)
+        local c = coroutine.create(f_)
+        for i = 1, 10 do
+                local t, r = coroutine.resume(c, out_linda) -- returns true + <yielded values>
+                assert(t == true and r == "I yield!", "got " .. tostring(t) .. " " .. tostring(r))
+                local s = coroutine.status(c)
+                assert(s == "suspended")
+        end
+        local r, s = coroutine.close(c)
+        assert(r == true and s == nil)
+        -- the local variable inside the yielder body should be closed
+        local s, r = out_linda:receive(0, "out")
+        assert(s == "out" and r == "Closed!", what_ .. " got " .. tostring(s) .. " " .. tostring(r))
+end
+
+-- first, try the close mechanism outside of a lane
+test_close("base", yielder)
+
+-- try again with a function obtained through dump/undump
+-- note this means our yielder implementation can't have upvalues, as they are lost in the process
+test_close("dumped", load(string.dump(yielder)))
+
+------------------------------------------------------------------------------
+-- TEST: to-be-closed variables are properly closed when the lane is collected
+------------------------------------------------------------------------------
+if false then -- NOT IMPLEMENTED YET!
+
+        -- the generator
+        local coro_g = lanes.coro("*", yielder)
+
+        -- start the lane
+        local h = coro_g(out_linda)
+
+        -- join it so that it reaches suspended state
+        local r, v = h:join(0.5)
+        assert(r == nil and v == "timeout", "got " .. tostring(r) .. " " .. tostring(v))
+        assert(h.status == "suspended")
+
+        -- force collection of the lane
+        h = nil
+        collectgarbage()
+
+        -- I want the to-be-closed variable of the coroutine linda to be properly closed
+        local s, r = out_linda:receive(0, "out")
+        assert(s == "out" and r == "Closed!", "coro got " .. tostring(s) .. " " .. tostring(r))
+end
diff --git a/unit_tests/scripts/coro/error_handling.lua b/unit_tests/scripts/coro/error_handling.lua
index ba6cff6..1cfb8c8 100644
--- a/unit_tests/scripts/coro/error_handling.lua
+++ b/unit_tests/scripts/coro/error_handling.lua
@@ -38,15 +38,15 @@ local force_error_test = function(error_trace_level_)
     utils.dump_error_stack(error_trace_level_, c)
 end
 
-if false then
+if true then
     force_error_test("minimal")
 end
 
-if false then
+if true then
     force_error_test("basic")
 end
 
-if false then
+if true then
     force_error_test("extended")
 end
 
diff --git a/unit_tests/scripts/coro/regular_function.lua b/unit_tests/scripts/coro/regular_function.lua
new file mode 100644
index 0000000..09aa3b7
--- /dev/null
+++ b/unit_tests/scripts/coro/regular_function.lua
@@ -0,0 +1,38 @@
+local lanes = require "lanes".configure()
+
+local utils = lanes.require "_utils"
+local PRINT = utils.MAKE_PRINT()
+
+-- a lane body that just returns some value
+local returner = function(msg_)
+    local utils = lanes.require "_utils"
+    local PRINT = utils.MAKE_PRINT()
+    PRINT "In lane"
+    assert(msg_ == "hi")
+    return "bye"
+end
+
+-- a function that returns some value can run in a coroutine
+if true then
+    -- the generator
+    local g = lanes.coro("*", {name = "auto"}, returner)
+
+    -- launch lane
+    local h = g("hi")
+
+    local r = h[1]
+    assert(r == "bye")
+end
+
+-- can't resume a coro after the lane body has returned
+if true then
+    -- the generator
+    local g = lanes.coro("*", {name = "auto"}, returner)
+
+    -- launch lane
+    local h = g("hi")
+
+    -- resuming a lane that terminated execution should raise an error
+    local b, e = pcall(h.resume, h)
+    assert(b == false and type(e) == "string")
+end
diff --git a/unit_tests/scripts/coro/yielding_function.lua b/unit_tests/scripts/coro/yielding_function.lua
new file mode 100644
index 0000000..e7367ea
--- /dev/null
+++ b/unit_tests/scripts/coro/yielding_function.lua
@@ -0,0 +1,107 @@
+local lanes = require "lanes"
+
+local fixture = require "fixture"
+lanes.finally(fixture.throwing_finalizer)
+
+local utils = lanes.require "_utils"
+local PRINT = utils.MAKE_PRINT()
+
+-- a lane coroutine that yields back what got in, one element at a time
+local yielder = function(...)
+    local utils = lanes.require "_utils"
+    local PRINT = utils.MAKE_PRINT()
+    PRINT "In lane"
+    for _i  = 1, select('#', ...) do
+        local _val = select(_i, ...)
+        PRINT("yielding #", _i, _val)
+        local _ack = coroutine.yield(_val)
+        assert(_ack == _i)
+    end
+    return "done!"
+end
+
+--------------------------------------------------------------------------------------------------
+-- TEST: if we start a non-coroutine lane with a yielding function, we should get an error, right?
+--------------------------------------------------------------------------------------------------
+if true then
+    local fun_g = lanes.gen("*", { name = 'auto' }, yielder)
+    local h = fun_g("hello", "world", "!")
+    local err, status, stack = h:join()
+    PRINT(err, status, stack)
+    -- the actual error message is not the same for Lua 5.1
+    -- of course, it also has to be different for LuaJIT as well
+    -- also, LuaJIT prepends a file:line to the actual error message, which Lua5.1 does not.
+    local msgs = {
+        ["Lua 5.1"] = jit and "attempt to yield across C-call boundary" or "attempt to yield across metamethod/C-call boundary",
+        ["Lua 5.2"] = "attempt to yield from outside a coroutine",
+        ["Lua 5.3"] = "attempt to yield from outside a coroutine",
+        ["Lua 5.4"] = "attempt to yield from outside a coroutine"
+    }
+    local expected_msg = msgs[_VERSION]
+    PRINT("expected_msg = " .. expected_msg)
+    assert(err == nil and string.find(status, expected_msg, 1, true) and stack == nil, "status = " .. status)
+end
+
+-- the coroutine generator
+local coro_g = lanes.coro("*", {name = "auto"}, yielder)
+
+-------------------------------------------------------------------------------------------------
+-- TEST: we can resume as many times as the lane yields, then read the returned value on indexing
+-------------------------------------------------------------------------------------------------
+if true then
+    -- launch coroutine lane
+    local h = coro_g("hello", "world", "!")
+    -- read the yielded values, sending back the expected index
+    assert(h:resume(1) == "hello")
+    assert(h:resume(2) == "world")
+    assert(h:resume(3) == "!")
+    -- the lane return value is available as usual
+    local r = h[1]
+    assert(r == "done!")
+end
+
+---------------------------------------------------------------------------------------------
+-- TEST: we can resume as many times as the lane yields, then read the returned value on join
+---------------------------------------------------------------------------------------------
+if true then
+    -- launch coroutine lane
+    local h = coro_g("hello", "world", "!")
+    -- read the yielded values, sending back the expected index
+    assert(h:resume(1) == "hello")
+    assert(h:resume(2) == "world")
+    assert(h:resume(3) == "!")
+    -- the lane return value is available as usual
+    local s, r = h:join()
+    assert(h.status == "done" and s == true and r == "done!")
+end
+
+---------------------------------------------------------------------------------------------------
+-- TEST: if we join a yielded lane, we get a timeout, and we can resume as if we didn't try to join
+---------------------------------------------------------------------------------------------------
+if true then
+    -- launch coroutine lane
+    local h = coro_g("hello", "world", "!")
+    -- read the first yielded value, sending back the expected index
+    assert(h:resume(1) == "hello")
+    -- join the lane. since it will reach a yield point, it remains suspended, and we should get a timeout
+    local b, r = h:join(0.5)
+    local s = h.status
+    assert(s == "suspended" and b == nil and r == "timeout", "got " .. s .. " " .. tostring(b) .. " " .. r)
+    -- trying to resume again should proceed normally, since nothing changed
+    assert(h:resume(2) == "world")
+    assert(h:resume(3) == "!")
+    -- the lane return value is available as usual
+    local s, r = h:join()
+    assert(h.status == "done" and s == true and r == "done!")
+end
+
+---------------------------------------------------------
+-- TEST: if we index yielded lane, we should get an error
+---------------------------------------------------------
+-- TODO: implement this test!
+
+
+-------------------------------------------------------------------------------------
+-- TEST: if we close yielded lane, we can join it and get the last yielded values out
+-------------------------------------------------------------------------------------
+-- TODO: we need to implement lane:close() for that!