New LTN12 test procedures (still short, but growing)

LTN12 avoids coroutines.

5 files changed, 334 insertions, 111 deletions

diff --git a/FIX b/FIX
index a7f244f..5ce3a01 100644
--- a/FIX
+++ b/FIX
@@ -1,3 +1,4 @@
+ltn12 avoids coroutines (so you can go wild on the C side)
 automated tests for ftp now in use
 new compat-5.1 distribution
     instalation should use new directory structure
diff --git a/src/ltn12.lua b/src/ltn12.lua
index 43c2755..5216ff6 100644
--- a/src/ltn12.lua
+++ b/src/ltn12.lua
@@ -35,64 +35,40 @@ function filter.cycle(low, ctx, extra)
     end
 end
 
---[[
-local function chain2(f1, f2)
-    local ff1, ff2 = "", ""
+-- chains a bunch of filters together
+-- (thanks to Wim Couwenberg)
+function filter.chain(...) 
+    local n = table.getn(arg)
+    local top, index = 1, 1
     return function(chunk)
-        local rf1 = chunk and ""
-        local rf2 = ff1 and ""
-        -- if f2 still has pending data, get it and return it
-        if ff2 ~= rf2 then
-            ff2 = f2(rf2)
-            if ff2 ~= "" then return ff2 end
-        end
-        -- here we know f2 needs more data
-        -- we try to get it from f1
-        ff1 = f1(chunk)
-        while 1 do
-            -- if f1 can't produce data, we need more data from the user
-            if ff1 == "" then return "" end
-            -- otherwise we pass new data to f2 until it produces something
-            -- or f1 runs out of data too
-            ff2 = f2(ff1) 
-            if ff2 ~= "" then return ff2 end
-            ff1 = f1(rf1)
-        end
-    end
-end
-]]
-
-local function chain2(f1, f2)
-    local co = coroutine.create(function(chunk)
-        while true do
-            local filtered1 = f1(chunk)
-            local filtered2 = f2(filtered1)
-            local done2 = filtered1 and ""
-            while true do
-                if filtered2 == "" or filtered2 == nil then break end
-                coroutine.yield(filtered2)
-                filtered2 = f2(done2)
+        while true do 
+            if index == top then
+                chunk = arg[index](chunk)
+                if chunk == "" or top == n then 
+                    return chunk
+                elseif chunk then 
+                    index = index + 1
+                else 
+                    top = top+1 
+                    index = top
+                end
+            else
+                local original = chunk
+                chunk = arg[index](original or "")
+                if chunk == "" then
+                    index = index - 1
+                    chunk = original and chunk
+                elseif chunk then
+                    if index == n then return chunk
+                    else index = index + 1 end
+                else 
+                    base.error("filter returned inappropriate nil")
+                end
             end
-            if filtered1 == "" then chunk = coroutine.yield(filtered1)
-            elseif filtered1 == nil then return nil
-            else chunk = chunk and "" end
         end
-    end)
-    return function(chunk)
-        local _, res = coroutine.resume(co, chunk)
-        return res
     end
 end
 
--- chains a bunch of filters together 
-function filter.chain(...)
-     local f = arg[1]
-     for i = 2, table.getn(arg) do
-         f = chain2(f, arg[i])
-     end
-     return f
-end
-
 -----------------------------------------------------------------------------
 -- Source stuff
 -----------------------------------------------------------------------------
@@ -165,23 +141,28 @@ end
 -- chains a source with a filter
 function source.chain(src, f)
     base.assert(src and f)
-    local co = coroutine.create(function()
-        while true do 
-            local chunk, err = src()
-            if err then return nil, err end
-            local filtered = f(chunk)
-            local done = chunk and ""
+    local last_in, last_out = "", ""
+    return function()
+        if last_out == "" then
             while true do
-                coroutine.yield(filtered)
-                if filtered == done then break end
-                filtered = f(done)
+                local err
+                last_in, err = src()
+                if err then return nil, err end
+                last_out = f(last_in)
+                if last_out ~= "" then return last_out end
+                if not last_in then 
+                    error('filter returned inappropriate ""') 
+                end
             end
+        elseif last_out then
+            last_out = f(last_in and "")
+            if last_in and not last_out then
+                error('filter returned inappropriate nil') 
+            end
+            return last_out
+        else
+            base.error("source is empty", 2)
         end
-    end)
-    return function()
-        local ret, a, b  = coroutine.resume(co)
-        if ret then return a, b
-        else return nil, a end
     end
 end
 
@@ -260,14 +241,16 @@ end
 function sink.chain(f, snk)
     base.assert(f and snk)
     return function(chunk, err)
-        local filtered = f(chunk)
-        local done = chunk and ""
-        while true do
-            local ret, snkerr = snk(filtered, err)
-            if not ret then return nil, snkerr end
-            if filtered == done then return 1 end
-            filtered = f(done)
-        end
+        if chunk ~= "" then
+            local filtered = f(chunk)
+            local done = chunk and ""
+            while true do
+                local ret, snkerr = snk(filtered, err)
+                if not ret then return nil, snkerr end
+                if filtered == done then return 1 end
+                filtered = f(done)
+            end
+        else return 1 end
     end
 end
 
diff --git a/src/wsocket.c b/src/wsocket.c
index 0294dce..69fac4d 100644
--- a/src/wsocket.c
+++ b/src/wsocket.c
@@ -193,7 +193,7 @@ int sock_send(p_sock ps, const char *data, size_t count, size_t *sent, p_tm tm)
     *sent = 0;
     for ( ;; ) {
         /* try to send something */
-                int put = send(*ps, data, count, 0);
+                int put = send(*ps, data, (int) count, 0);
         /* if we sent something, we are done */
         if (put > 0) {
             *sent = put;
diff --git a/test/ltn12test.lua b/test/ltn12test.lua
new file mode 100644
index 0000000..7922bf1
--- /dev/null
+++ b/test/ltn12test.lua
@@ -0,0 +1,275 @@
+local ltn12 = require("ltn12")
+
+dofile("testsupport.lua")
+
+local function format(chunk)
+    if chunk then
+        if chunk == "" then return "''"
+        else return string.len(chunk) end
+    else return "nil" end
+end
+
+local function show(name, input, output)
+    local sin = format(input)
+    local sout = format(output)
+    io.write(name, ": ", sin, " -> ", sout, "\n")
+end
+
+local function chunked(length)
+    local tmp
+    return function(chunk)
+        local ret
+        if chunk and chunk ~= "" then
+            tmp = chunk
+        end
+        ret = string.sub(tmp, 1, length)
+        tmp = string.sub(tmp, length+1)
+        if not chunk and ret == "" then ret = nil end
+        return ret
+    end
+end
+
+local function named(f, name)
+    return function(chunk)
+        local ret = f(chunk)
+        show(name, chunk, ret)
+        return ret
+    end
+end
+
+--------------------------------
+local function split(size) 
+    local buffer = ""
+    local last_out = ""
+    local last_in = ""
+    local function output(chunk)
+        local part = string.sub(buffer, 1, size)
+        buffer = string.sub(buffer, size+1)
+        last_out = (part ~= "" or chunk) and part
+        last_in = chunk
+        return last_out
+    end
+    return function(chunk, done)
+        if done then 
+            return not last_in and not last_out 
+        end
+        -- check if argument is consistent with state
+        if not chunk then
+            if last_in and last_in ~= "" and last_out ~= "" then 
+                error("nil chunk following data chunk", 2)
+            end
+            if not last_out then error("extra nil chunk", 2) end
+            return output(chunk)
+        elseif chunk == "" then
+            if last_out == "" then error('extra "" chunk', 2) end
+            if not last_out then error('"" chunk following nil return', 2) end
+            if not last_in then error('"" chunk following nil chunk', 2) end
+            return output(chunk)
+        else
+            if not last_in  then error("data chunk following nil chunk", 2) end
+            if last_in ~= "" and last_out ~= "" then 
+                error("data chunk following data chunk", 2) 
+            end
+            buffer = chunk
+            return output(chunk)
+        end
+    end
+end
+
+--------------------------------
+local function format(chunk)
+    if chunk then
+        if chunk == "" then return "''"
+        else return string.len(chunk) end
+    else return "nil" end
+end
+
+--------------------------------
+local function merge(size) 
+    local buffer = ""
+    local last_out = ""
+    local last_in = ""
+    local function output(chunk)
+        local part
+        if string.len(buffer) >= size or not chunk then
+            part = buffer
+            buffer = ""
+        else
+            part = ""
+        end
+        last_out = (part ~= "" or chunk) and part
+        last_in = chunk
+        return last_out
+    end
+    return function(chunk, done)
+        if done then 
+            return not last_in and not last_out 
+        end
+        -- check if argument is consistent with state
+        if not chunk then
+            if last_in and last_in ~= "" and last_out ~= "" then 
+                error("nil chunk following data chunk", 2)
+            end
+            if not last_out then error("extra nil chunk", 2) end
+            return output(chunk)
+        elseif chunk == "" then
+            if last_out == "" then error('extra "" chunk', 2) end
+            if not last_out then error('"" chunk following nil return', 2) end
+            if not last_in then error('"" chunk following nil chunk', 2) end
+            return output(chunk)
+        else
+            if not last_in  then error("data chunk following nil chunk", 2) end
+            if last_in ~= "" and last_out ~= "" then 
+                error("data chunk following data chunk", 2) 
+            end
+            buffer = buffer .. chunk
+            return output(chunk)
+        end
+    end
+end
+
+--------------------------------
+io.write("testing sink.table: ")
+local sink, t = ltn12.sink.table()
+local s, c = "", ""
+for i = 0, 10 do
+    c = string.rep(string.char(i), i)
+    s = s .. c
+    assert(sink(c), "returned error")
+end
+assert(sink(nil), "returned error")
+assert(table.concat(t) == s, "mismatch")
+print("ok")
+
+--------------------------------
+io.write("testing sink.chain (with split): ")
+sink, t = ltn12.sink.table()
+local filter = split(3)
+sink = ltn12.sink.chain(filter, sink)
+s = "123456789012345678901234567890"
+assert(sink(s), "returned error")
+assert(sink(s), "returned error")
+assert(sink(nil), "returned error")
+assert(table.concat(t) == s .. s, "mismatch")
+assert(filter(nil, 1), "filter not empty")
+print("ok")
+
+--------------------------------
+io.write("testing sink.chain (with merge): ")
+sink, t = ltn12.sink.table()
+filter = merge(10)
+sink = ltn12.sink.chain(filter, sink)
+s = string.rep("123", 30)
+s = s .. string.rep("4321", 30)
+for i = 1, 30 do
+    assert(sink("123"), "returned error")
+end
+for i = 1, 30 do
+    assert(sink("4321"), "returned error")
+end
+assert(sink(nil), "returned error")
+assert(filter(nil, 1), "filter not empty")
+assert(table.concat(t) == s, "mismatch")
+print("ok")
+
+--------------------------------
+io.write("testing source.string and pump.all: ")
+local source = ltn12.source.string(s)
+sink, t = ltn12.sink.table()
+assert(ltn12.pump.all(source, sink), "returned error")
+assert(table.concat(t) == s, "mismatch")
+print("ok")
+
+--------------------------------
+io.write("testing source.chain (with split): ")
+source = ltn12.source.string(s)
+filter = split(5)
+source = ltn12.source.chain(source, filter)
+sink, t = ltn12.sink.table()
+assert(ltn12.pump.all(source, sink), "returned error")
+assert(table.concat(t) == s, "mismatch")
+assert(filter(nil, 1), "filter not empty")
+print("ok")
+
+--------------------------------
+io.write("testing source.chain (with split) and sink.chain (with merge): ")
+source = ltn12.source.string(s)
+filter = split(5)
+source = ltn12.source.chain(source, filter)
+local filter2 = merge(13)
+sink, t = ltn12.sink.table()
+sink = ltn12.sink.chain(filter2, sink)
+assert(ltn12.pump.all(source, sink), "returned error")
+assert(table.concat(t) == s, "mismatch")
+assert(filter(nil, 1), "filter not empty")
+assert(filter2(nil, 1), "filter2 not empty")
+print("ok")
+
+--------------------------------
+io.write("testing filter.chain (and sink.chain, with split, merge): ")
+source = ltn12.source.string(s)
+filter = split(5)
+filter2 = merge(13)
+local chain = ltn12.filter.chain(filter, filter2)
+sink, t = ltn12.sink.table()
+sink = ltn12.sink.chain(chain, sink)
+assert(ltn12.pump.all(source, sink), "returned error")
+assert(table.concat(t) == s, "mismatch")
+assert(filter(nil, 1), "filter not empty")
+assert(filter2(nil, 1), "filter2 not empty")
+print("ok")
+
+--------------------------------
+io.write("testing filter.chain (and sink.chain, a bunch): ")
+source = ltn12.source.string(s)
+filter = split(5)
+filter2 = merge(13)
+local filter3 = split(7)
+local filter4 = merge(11)
+local filter5 = split(10)
+chain = ltn12.filter.chain(filter, filter2, filter3, filter4, filter5)
+sink, t = ltn12.sink.table()
+sink = ltn12.sink.chain(chain, sink)
+assert(ltn12.pump.all(source, sink))
+assert(table.concat(t) == s, "mismatch")
+assert(filter(nil, 1), "filter not empty")
+assert(filter2(nil, 1), "filter2 not empty")
+assert(filter3(nil, 1), "filter3 not empty")
+assert(filter4(nil, 1), "filter4 not empty")
+assert(filter5(nil, 1), "filter5 not empty")
+print("ok")
+
+--------------------------------
+io.write("testing filter.chain (and source.chain, with split, merge): ")
+source = ltn12.source.string(s)
+filter = split(5)
+filter2 = merge(13)
+local chain = ltn12.filter.chain(filter, filter2)
+sink, t = ltn12.sink.table()
+source = ltn12.source.chain(source, chain)
+assert(ltn12.pump.all(source, sink), "returned error")
+assert(table.concat(t) == s, "mismatch")
+assert(filter(nil, 1), "filter not empty")
+assert(filter2(nil, 1), "filter2 not empty")
+print("ok")
+
+--------------------------------
+io.write("testing filter.chain (and source.chain, a bunch): ")
+source = ltn12.source.string(s)
+filter = split(5)
+filter2 = merge(13)
+local filter3 = split(7)
+local filter4 = merge(11)
+local filter5 = split(10)
+chain = ltn12.filter.chain(filter, filter2, filter3, filter4, filter5)
+sink, t = ltn12.sink.table()
+source = ltn12.source.chain(source, chain)
+assert(ltn12.pump.all(source, sink))
+assert(table.concat(t) == s, "mismatch")
+assert(filter(nil, 1), "filter not empty")
+assert(filter2(nil, 1), "filter2 not empty")
+assert(filter3(nil, 1), "filter3 not empty")
+assert(filter4(nil, 1), "filter4 not empty")
+assert(filter5(nil, 1), "filter5 not empty")
+print("ok")
+
diff --git a/test/mimetest.lua b/test/mimetest.lua
index 834d99f..2f6b7a8 100644
--- a/test/mimetest.lua
+++ b/test/mimetest.lua
@@ -47,42 +47,6 @@ local function random(handle, io_err)
     else return ltn12.source.empty(io_err or "unable to open file") end
 end
 
-local function format(chunk)
-    if chunk then
-        if chunk == "" then return "''"
-        else return string.len(chunk) end
-    else return "nil" end
-end
-
-local function show(name, input, output)
-    local sin = format(input)
-    local sout = format(output)
-    io.write(name, ": ", sin, " -> ", sout, "\n")
-end
-
-local function chunked(length)
-    local tmp
-    return function(chunk)
-        local ret
-        if chunk and chunk ~= "" then
-            tmp = chunk
-        end
-        ret = string.sub(tmp, 1, length)
-        tmp = string.sub(tmp, length+1)
-        if not chunk and ret == "" then ret = nil end
-        return ret
-    end
-end
-
---[[
-local function named(f, name)
-    return function(chunk)
-        local ret = f(chunk)
-        show(name, chunk, ret)
-        return ret
-    end
-end
-]]
 
 local function named(f)
     return f