aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorDiego Nehab <diego@tecgraf.puc-rio.br>2007-10-11 21:16:28 +0000
committerDiego Nehab <diego@tecgraf.puc-rio.br>2007-10-11 21:16:28 +0000
commit52ac60af8132ae7e42151d3012a9607d7cadaf95 (patch)
tree95085f9f535d20b6686399af1ba284bb74ac89d3
parente394956cde629e24ecdd285c4c13c206891fcec4 (diff)
downloadluasocket-52ac60af8132ae7e42151d3012a9607d7cadaf95.tar.gz
luasocket-52ac60af8132ae7e42151d3012a9607d7cadaf95.tar.bz2
luasocket-52ac60af8132ae7e42151d3012a9607d7cadaf95.zip
Tested each sample.
-rw-r--r--gem/ex1.lua4
-rw-r--r--gem/ex10.lua17
-rw-r--r--gem/ex11.lua7
-rw-r--r--gem/ex12.lua34
-rw-r--r--gem/ex2.lua11
-rw-r--r--gem/ex3.lua15
-rw-r--r--gem/ex4.lua5
-rw-r--r--gem/ex5.lua15
-rw-r--r--gem/ex6.lua14
-rw-r--r--gem/ex7.lua16
-rw-r--r--gem/ex8.lua5
-rw-r--r--gem/ex9.lua3
-rw-r--r--gem/gem.c54
-rw-r--r--gem/gt.b64206
-rw-r--r--gem/input.binbin0 -> 11732 bytes
-rw-r--r--gem/ltn012.tex355
-rw-r--r--gem/luasocket.pngbin0 -> 11732 bytes
-rw-r--r--gem/makefile9
-rw-r--r--gem/t1.lua25
-rw-r--r--gem/t1lf.txt5
-rw-r--r--gem/t2.lua36
-rw-r--r--gem/t2.txt4
-rw-r--r--gem/t2gt.qp5
-rw-r--r--gem/t3.lua25
-rw-r--r--gem/t4.lua10
-rw-r--r--gem/t5.lua30
-rw-r--r--gem/test.lua46
27 files changed, 791 insertions, 165 deletions
diff --git a/gem/ex1.lua b/gem/ex1.lua
new file mode 100644
index 0000000..327a542
--- /dev/null
+++ b/gem/ex1.lua
@@ -0,0 +1,4 @@
1local CRLF = "\013\010"
2local input = source.chain(source.file(io.stdin), normalize(CRLF))
3local output = sink.file(io.stdout)
4pump.all(input, output)
diff --git a/gem/ex10.lua b/gem/ex10.lua
new file mode 100644
index 0000000..2b1b98f
--- /dev/null
+++ b/gem/ex10.lua
@@ -0,0 +1,17 @@
1function pump.step(src, snk)
2 local chunk, src_err = src()
3 local ret, snk_err = snk(chunk, src_err)
4 if chunk and ret then return 1
5 else return nil, src_err or snk_err end
6end
7
8function pump.all(src, snk, step)
9 step = step or pump.step
10 while true do
11 local ret, err = step(src, snk)
12 if not ret then
13 if err then return nil, err
14 else return 1 end
15 end
16 end
17end
diff --git a/gem/ex11.lua b/gem/ex11.lua
new file mode 100644
index 0000000..1cbf01f
--- /dev/null
+++ b/gem/ex11.lua
@@ -0,0 +1,7 @@
1local input = source.chain(
2 source.file(io.open("input.bin", "rb")),
3 encode("base64"))
4local output = sink.chain(
5 wrap(76),
6 sink.file(io.open("output.b64", "w")))
7pump.all(input, output)
diff --git a/gem/ex12.lua b/gem/ex12.lua
new file mode 100644
index 0000000..de17d76
--- /dev/null
+++ b/gem/ex12.lua
@@ -0,0 +1,34 @@
1local smtp = require"socket.smtp"
2local mime = require"mime"
3local ltn12 = require"ltn12"
4
5CRLF = "\013\010"
6
7local message = smtp.message{
8 headers = {
9 from = "Sicrano <sicrano@example.com>",
10 to = "Fulano <fulano@example.com>",
11 subject = "A message with an attachment"},
12 body = {
13 preamble = "Hope you can see the attachment" .. CRLF,
14 [1] = {
15 body = "Here is our logo" .. CRLF},
16 [2] = {
17 headers = {
18 ["content-type"] = 'image/png; name="luasocket.png"',
19 ["content-disposition"] =
20 'attachment; filename="luasocket.png"',
21 ["content-description"] = 'LuaSocket logo',
22 ["content-transfer-encoding"] = "BASE64"},
23 body = ltn12.source.chain(
24 ltn12.source.file(io.open("luasocket.png", "rb")),
25 ltn12.filter.chain(
26 mime.encode("base64"),
27 mime.wrap()))}}}
28
29assert(smtp.send{
30 rcpt = "<diego@cs.princeton.edu>",
31 from = "<diego@cs.princeton.edu>",
32 server = "localhost",
33 port = 2525,
34 source = message})
diff --git a/gem/ex2.lua b/gem/ex2.lua
new file mode 100644
index 0000000..94bde66
--- /dev/null
+++ b/gem/ex2.lua
@@ -0,0 +1,11 @@
1function filter.cycle(lowlevel, context, extra)
2 return function(chunk)
3 local ret
4 ret, context = lowlevel(context, chunk, extra)
5 return ret
6 end
7end
8
9function normalize(marker)
10 return filter.cycle(eol, 0, marker)
11end
diff --git a/gem/ex3.lua b/gem/ex3.lua
new file mode 100644
index 0000000..a43fefa
--- /dev/null
+++ b/gem/ex3.lua
@@ -0,0 +1,15 @@
1local function chainpair(f1, f2)
2 return function(chunk)
3 local ret = f2(f1(chunk))
4 if chunk then return ret
5 else return (ret or "") .. (f2() or "") end
6 end
7end
8
9function filter.chain(...)
10 local f = select(1, ...)
11 for i = 2, select('#', ...) do
12 f = chainpair(f, select(i, ...))
13 end
14 return f
15end
diff --git a/gem/ex4.lua b/gem/ex4.lua
new file mode 100644
index 0000000..c670e0e
--- /dev/null
+++ b/gem/ex4.lua
@@ -0,0 +1,5 @@
1local qp = filter.chain(normalize(CRLF), encode("quoted-printable"),
2 wrap("quoted-printable"))
3local input = source.chain(source.file(io.stdin), qp)
4local output = sink.file(io.stdout)
5pump.all(input, output)
diff --git a/gem/ex5.lua b/gem/ex5.lua
new file mode 100644
index 0000000..196b30a
--- /dev/null
+++ b/gem/ex5.lua
@@ -0,0 +1,15 @@
1function source.empty(err)
2 return function()
3 return nil, err
4 end
5end
6
7function source.file(handle, io_err)
8 if handle then
9 return function()
10 local chunk = handle:read(20)
11 if not chunk then handle:close() end
12 return chunk
13 end
14 else return source.empty(io_err or "unable to open file") end
15end
diff --git a/gem/ex6.lua b/gem/ex6.lua
new file mode 100644
index 0000000..a3fdca0
--- /dev/null
+++ b/gem/ex6.lua
@@ -0,0 +1,14 @@
1function source.chain(src, f)
2 return function()
3 if not src then
4 return nil
5 end
6 local chunk, err = src()
7 if not chunk then
8 src = nil
9 return f(nil)
10 else
11 return f(chunk)
12 end
13 end
14end
diff --git a/gem/ex7.lua b/gem/ex7.lua
new file mode 100644
index 0000000..c766988
--- /dev/null
+++ b/gem/ex7.lua
@@ -0,0 +1,16 @@
1function sink.table(t)
2 t = t or {}
3 local f = function(chunk, err)
4 if chunk then table.insert(t, chunk) end
5 return 1
6 end
7 return f, t
8end
9
10local function null()
11 return 1
12end
13
14function sink.null()
15 return null
16end
diff --git a/gem/ex8.lua b/gem/ex8.lua
new file mode 100644
index 0000000..81e288c
--- /dev/null
+++ b/gem/ex8.lua
@@ -0,0 +1,5 @@
1local input = source.file(io.stdin)
2local output, t = sink.table()
3output = sink.chain(normalize(CRLF), output)
4pump.all(input, output)
5io.write(table.concat(t))
diff --git a/gem/ex9.lua b/gem/ex9.lua
new file mode 100644
index 0000000..b857698
--- /dev/null
+++ b/gem/ex9.lua
@@ -0,0 +1,3 @@
1for chunk in source.file(io.stdin) do
2 io.write(chunk)
3end
diff --git a/gem/gem.c b/gem/gem.c
new file mode 100644
index 0000000..976f74d
--- /dev/null
+++ b/gem/gem.c
@@ -0,0 +1,54 @@
1#include "lua.h"
2#include "lauxlib.h"
3
4#define CR '\xD'
5#define LF '\xA'
6#define CRLF "\xD\xA"
7
8#define candidate(c) (c == CR || c == LF)
9static int pushchar(int c, int last, const char *marker,
10 luaL_Buffer *buffer) {
11 if (candidate(c)) {
12 if (candidate(last)) {
13 if (c == last)
14 luaL_addstring(buffer, marker);
15 return 0;
16 } else {
17 luaL_addstring(buffer, marker);
18 return c;
19 }
20 } else {
21 luaL_putchar(buffer, c);
22 return 0;
23 }
24}
25
26static int eol(lua_State *L) {
27 int context = luaL_checkint(L, 1);
28 size_t isize = 0;
29 const char *input = luaL_optlstring(L, 2, NULL, &isize);
30 const char *last = input + isize;
31 const char *marker = luaL_optstring(L, 3, CRLF);
32 luaL_Buffer buffer;
33 luaL_buffinit(L, &buffer);
34 if (!input) {
35 lua_pushnil(L);
36 lua_pushnumber(L, 0);
37 return 2;
38 }
39 while (input < last)
40 context = pushchar(*input++, context, marker, &buffer);
41 luaL_pushresult(&buffer);
42 lua_pushnumber(L, context);
43 return 2;
44}
45
46static luaL_reg func[] = {
47 { "eol", eol },
48 { NULL, NULL }
49};
50
51int luaopen_gem(lua_State *L) {
52 luaL_openlib(L, "gem", func, 0);
53 return 0;
54}
diff --git a/gem/gt.b64 b/gem/gt.b64
new file mode 100644
index 0000000..a74c0b3
--- /dev/null
+++ b/gem/gt.b64
@@ -0,0 +1,206 @@
1iVBORw0KGgoAAAANSUhEUgAAAIAAAACACAIAAABMXPacAAAtU0lEQVR42u19eXRURdb4rarXa5LO
2RshKEshC2MLOBIjsCoMLGJhRPnUEcUGZEX7j4Iw6zqd+zjkzzowL6gzKMOoBRHAAPyQKUZQlxLAk
3EIEkQkhCyEoISegs3f1eVf3+qPTj0Z3udEJImN/Pe/rkdF6/V6/q3qp7b92tEOccfoT+A9zfHfj/
4HX4kQD/DjwToZ/iRAP0MPxKgn+FHAvQz/EiAfgapvzvQQ3DfviCE+rtTPYH/AAKouEYIcc4ForUX
5tXeKexhj6k8IIe2DvdUl0SYAcN7RGYQ63oAQ4hx8fBu6BXfC6vBcsHyDeNRi7cYboZQjBIRgl/lB
6KQcAQnyl+q1IAC9YU7/s2bOnsrKSUupwOHQ63cMPP2wymRhjGOOrV6/m5ORYLJbg4OABAwZYLBaD
7waBtQUsD34mqRT0hHc/abEpNjbWlxYEQCgw0RET463QEABjjjHFfyND/LEg737XsQpblhoaGioqK
8CxcunD9/fv78+ampqepgZFk2mUwBAQEYY6PRSAhRG7Tb7cXFxXa73W63W63Wn/zkJ4sXL1YfVHGB
9EFI5VZc0EDcwxjnnkoRbWhw7dxZt316Yn19TW9siyxQADAZddHRAWlrMffeNnDcvUa8nlDKEAGNv
107ffbClCnoYoFFRFiIufn53/88cfBwcERERERERHjxo2LjIz0ZbaqFLXb7ZcuXZIkKSoqShAYY7xn
11z576+vpJkybFxcUZjUZfOJKKfQBACP75z/yXXtpfXX0JAAFIAAQAAXAADsAAZAA0dGjMa6/Nueee
12FEoZQsgLDfqTAFqWIstyRUVFXFycJEniJ6vV2tTUFBUVRQhxkb0q2TTS7xr9tNxG/bdjtAjl5eXl
135ubW1dUhhJKTkzMyMkwmk0p4AMAYq91Tv1DKCMENDW0PPLBj797vEdJjrAfgjF2HP+d8B8YcAMry
145VP//vf5Oh3h3OM66P8V0NTU9N133+Xl5SmKsnr16qCgIBc8MsbE5HXXgjqdU9oRie8YY5c2W1tb
15CwsLS0tLFy5cqEoILWnFI84rHGNUXW29/fYPCwsvSpI/pQLxntYNxxhjDIpinTNn1K5d/2Uy6Zwd
16cNWO+o4A7mjFGOfk5OzcuTMsLGzixInjxo2zWCwqIlSpAL2k47tMc+18FN8vXLgAAHFxce4Cqa1N
17njlzw9GjZZLkryiK6KP3twEgnY7I8tWf/WzCtm33McZVJVV7H3nppZf6BvXaL+rAFEVJSEhYvHjx
184MGDDQaDykxAw1h6S38XLxUcRnRGnXyiM4cOHdqyZUtDQ0N0dLSfn5/4SUz/Z57Zs3PnCZ0uQFEU
19ANQV9jvIwxiTJOPp0xdCQgLS0gZRyjF2Hc5NXwEu866lpUWv1+v1enVBqFsnwWS0dLrZ4K7dlpSU
20ZGZmVlVVpaen33PPPYL1HzlSOXnyewCk+6gSo2OhocaCgl9GR1vEOtCO7qbbglQsY4yPHj366quv
21nj59GjScWtBGq0f2mVHBZbVxzhMSElatWvXzn//cORUAANau/Y5zB8YYoLsUQJxzQqSGhqb1648D
22gFClXO+4eSNUZ9alS5e2b99eXl4+d+7cqVOnCrl361hvOt2LCNWlttY6bNjbTU22Hk9WhBDnjhEj
23IgoKVoqdc1+vAFmW//WvfymK8uyzz86aNUvlP72HPrjBWaR2RkgIoXeJ2ZqbW9nUdBVj0uPGOecA
24ujNn6s+cuQRui6CXd8JaJUedSsJUEBoaqtfrtdd9p4HQ3rTGL9UE1ik2BZ/trmnMRePinAFAQUEt
25AMMYuXMP34EQRKnjzJlLqakRLr3uTQJoJarLzigyMpIxJiStVr/0pTXOQdgAMEaEYACOEPb+tKCU
26UOEVhYq9qKCKTwYyzW0XL169cUaNEAJglZVXwc2Q3msE0GKfEFJYWGg2m+Pj41UtyMeJr8W7olCB
27dFVS2mxKZeXVqqqrFRXN9fVtDQ1tbW2yw0EBQK8nJpNuwABTWJjfoEGB0dEBMTEWk0mHEBYPU8oY
28Y04S+roEbTalt1Bkt1P3i728AjjnhJCjR49u3rw5IyNDEACcvBW8ajgqRhSFCUsvQhghVF/fmptb
29efjwxWPHqs6da6iutlLqAFA86yQIQCJEHxkZkJQUMnFi9JQpg9LSYsLD/THusCtw3mHR7JIMfn66
303sKP2dxJU70sAzDGBw4c2Llz5/333z958mRVqfD+lBb1GCNhxa2oaP788x8++6z4yJFKq9UKQAGI
31+CCkw1jvqVkhPylllZVXKivrv/22EID4+wdMmhS9YEHKggVD4+KCxAqjlHkig9DfASA+PkismO7r
32oNeAMQ6A4+ODwG0K9o4aqtoajx07tnnz5mXLlo0ePVplO12iXhjZMUYYI1mme/aUrF+f/9VXJTZb
33CwAG0GFMhDHLxfjlHQTTF/KTMQogAzCDwW/27ITHHhs/f36SXk+8GO4VhUkSzsoqmTv3XxgbbkQI
34A3BJQmfO/DI5eYAQhL1JAK0l68qVK1euXElMTOyS6av6EqViI4bb2+WNGwveeCO3uLgSAAAMhBCA
35Dh/TjQMhCABRSgHsAJCUFL16ddrDD4/289OrfQDNahBGiKYm2/Dha2tqrAj1YCcMAIAxYsw+aVLs
36kSMr3G2IN7QPcOqFXJ3IISEhCQkJvmBfaIeKIqQifPDBiREj3n3iiW3FxTUYmwgxCWT1FvYBgFJO
37KQVAhJgwNp07V7ty5afDh7+7fn0e50AIVhTGmNZiCIrCgoKMixYNB7D3aCcMTvalPPjgGNEHl597
38vgI8Gd8FL/JkLnaf+IcPV6xatScv7zxCEsYGdQd0k6HDvs2Yg3PH6NFD3npr3vTp8Wqv1D0Hxqik
395MrYse+0tFCn48X3LSTHGDMmJySEnDjxy4AAfa+tAK1yWVpampubqxJDMLhOub9W2BKC29uVX/7y
40i/T09/LyygjxQ0hPKe0T7AMAYoxTShGSCPEvKKiYMWP9E0/sbm11iKXgHAIoCktMDHnxxVkAbTpd
41t9DFnahW/vSneQEBHYzOBS09IYA62THGra2tmzZtOnfunO9PCeF25Ejl+PHr3n13PyE6jI1O1Pex
42dQgxBpRSjA2E6N9//+DYseuysyskCVPKBTsiBDHGn302ffHiCbJs1ekkJ3K7GC5CSKfDlFrXrJm1
43ePFwShnGnYyuJwTQ+vk2bdrk5+e3ZMkS9Scv2GeMU8p1OvLOO0enTn3v7Nk6QvwpFQbRfjTMIcYY
44pZwQ/9LS+mnT3n/99e8kCQtmKNYB53zTpkV33jlGlpslSWzIPZFBhKUQjLksW596auZrr92hYt8d
45Pz1cAQKhmZmZpaWlS5culSRJsKNOJYrWqY0xeuKJz3/1q38DYIz1lIrNYT9gHyFXAxGlFGM9xtIz
46z+xctuwzYUESXnXOQacj//u/S3796zsUxU6pDSGQJEKIsHB0fAhBkkQQ4pS2Ygyvv77o3XfvFNjv
47zagIVZLs27cvMDBwwoQJqpHHE98Xno3WVvlnP9v65ZcFkhSgKKybAu0GgQMgse2iVIQviIFjjDHG
48YnvccZskYUWxzp49cseO+y0Wg+i82DFIEj58uOL55/cdPFgKYHfuDcUoGAAFYISY77572B//OGv4
498DBFYd6jg3pIAE8hCF6w39xsu+uuTdnZZyXJv2+x34F6xhjndgAOoPfzM5nNEqXcarXLsg1AAdBh
50rIcOB5GgQcukSQlffPGL0FCTGIJgSmI65+VV79xZnJNzsby8UQ3MSkgImT49PiNjWHJyqBrC5d3u
511A0CuHstvOv7KufBGFmtjnnzPsrJKZEkP0WhfTnxnV1t0+mMs2YlLVyYMnFiVHS0xWzWUcqammzn
52zl359tuyf/+7sKSkGiEJIT1jFAAkiShK68SJg7OylgYFGcVAAECrqiKEZJm2tysIgdmsc14EWRY2
53FY/q+A0RQG3Re2yIerMsszvv3Pj114WS5N/n2McACufKz38+/uWXZ6SkDHDvs4rH7duLXnjh69LS
54GkLMlHIALmgwbVry3r0PGwwd4T3gNDcJkqiUUC8SgjEWPoyuba6+CmFtAMH+/ftra2s7COjVuim0
55iEcf/axfsI8x5twRGGjYufPhrVsXJyeHUsrEdlf7oZTLMiUE33//yFOnVj7yyBRK2wgBAKQoVJL8
56Dh78YenSHerqV13cOl2HhUr1DmGMdDpSX3/p22/3C1+3FnU3RAC1obNnz+7atau9vd1L007WzwnB
57r756YOPGI/0y9xmTo6IsBw8+vnBhiixT4dIRWNN+CEE6HRF7LoOBbNiw4JVX5lNqwxg5aeC/deux
58F1/cRwimVJV/AM79ppAK6opvb2/ftWtXSUlJl9iHbsUFiXds2rQpOTl52rRpnoydzoAfJkk4M/Ps
59Y4/twNjotIH0ndQFYP7+ur17l40ZEyHLVJKwpy26+q/Q7hWFzZw5uKVFyck5R4gwjQDGhgMHzqam
60Ro8YMVBs472YuYKDg69cuVJQUJCWlubi5nQHn1aAuu5OnDhRU1MzZ84c7/cLda2mpuWJJz4DQJx3
6114Ryo4AxAnC8+ead48dHORxUhIx7R4Rzb48IwYyx116bm56eRGm7sMFxDgDSU0/9b0VFsyRhL/YS
628Yrbb7+9trY2Ly9Pxd4NEUCFc+fOTZgwYeDAgWL6u9+g2kcB4Omnd1dVNRCi57wvN7rC/mWbNWvo
638uXjKWU6He5SErrQQAjb116bCyAJAwnnjBBdXV3jr36122WY7sAYCwsLGz9+vOBCXbzURy3Iydap
64oijafIfr7+kw4UoS3rLl1H/912ZCTJT2tZkBIcS5PTNz6fz5yaIzvicMqWillEsSzsjYsnNnASEm
65oRQRgilt+/DD+x9+eKyzZe6GhA7M2O12Qoga7O3pdb6yIPEXY+w1qodzziUJNzXZXnghC0ByKgJ9
66BxgD546UlIjbb08AAEKuCUwfQTu0hx4aDYDUKcoYB9D9/vdfX77c5oURiZWk1+tFYD14FcVdEECr
67fbq8wH36g9Ph8Ne/ZpeV1fU581HRp8ycOVinI6pVuQftCH1/6tTYoCALY1SIUs45IfrKyvo///mQ
68kx6uyHVHTqc49JUA2na1Ar2zUXHOQZJweXnTO+/kAhj7nvmoMG5c9I08rlpABw70T0oKBVCc4xV+
69JNM//nHk3LkGwdw6fVz7txc2YoyxrVu3lpaWImecs4fbOACsXftdc7OVEOlGwgh6DJwDAImNDdTi
70omcghhMTYwFg2glNCGltbX3jjRzoLNhWizSEUHl5+datW51G307AGwFU/amqqur48eOSJHm9EyQJ
71V1Vd/fDDEwCG/jLxc84BkNEoAXRD8HpoCgDAZNJdP5PEIjBs2lRQXt4kFoEXFi9J0vHjxysrK8GD
72PurTCvj+++9jYmJiY2O9CHQxFz766ERjYxMh0s1OO/AEIoDH4VBUDN4g2GyK20zihEhW69UPPsgD
73z4tACIOYmJiYmBgRkd8pdEEAsXssKioaOnQoeBAj4pokYYeDbtpUAKDrD+eiOmwAoCIKE3ywBHgd
74OwKAqqqrAC68XvBh/ebN37e3y5KEPWOGA0BycnJRURFowgOve0uX/bBarYqiCAJ4gI44hm++KS0q
75qkVI31/TX2AHAPLza26kCTU5oKGhraTkCgBxGRHngLHu/PlLWVkl0FmwiRaGDx8uy3JTU1Onv3at
76hgYEBKxevTo2NhY8y3TRvU8/PQ1ARZbnTcaytw4DSPv3lzHGvMxN39qB3NyLDQ3NGEvubYjYrU8/
77PeOpBRVXMTExq1evDgwM7PQ2bwRQce2Siu4OkoStVntW1vn+5T8AwBhHSHfqVPWBAxfAq5biCdSg
78MQDYvPl7pwrE3V8EoP/669LGxnZP+qgAQojJZPLkG/BIAHXiMK/bWTWO6tixqsrKKwjp+rv2hBgk
79FWqi6Ex3nU6UMknCBQW1//73GQADpZ1MKc4BY6murik3txKgI4PBS8ue3ANdywDkBPDo/AIA2Lev
80FEDpNPSlbwExxhEyff756W3bTksSVhSP4RpuA7mWmgAAzz2XJcs2LxGJgtL79p33gjoXBLpDFwRo
81bGwsLi7W1gXopAmMACAn56K7sOonEGUbpJUrPz93rkGnI7JMVX+Wx2ec2JdlJkn4j3888OWXZwgx
82ednQcM4ByHffXVSR4OEeYIz98MMPjY2N3SCAQHphYeG2bdu8+h0BY9TY2H7mzCUA7+o/BwBJwuKD
838Q1F3HsFYVWWLl+23nXXxoqKZkED1UnrptJ0/KsojFKu15O///3Y73+/F2NTp8zn+gelwsLLly61
84CiO2xw4htHXr1sLCQnBj6dhz0wAADQ0N4eHhXpawuF5aeqW+vsVrKnOHl0pRWsSHMYcz1vWm0IAx
85hrHh7NlLU6a8n51dIXwyAsXOND+uutFlmQonEsbouee+XrlyB8Z6sey9vINzQAg3NbWWlDQAeHMP
86IIQiIyMvXboE18cVgpcMGTHrm5qagoKCwHMqj2iqqOgygEyI5FkjRgA0JMT/oYemMMbNZik7u+Lw
874dKbKbQ7aFBV1Txjxvqnnpry/PO3RUT4u3gyEOpYxAihb74pW7MmKz+/lBATpeCLFw9jRKlcVHR5
88ypRY7wMJCQnpdCvQBQFqampGjRrllQAcAM6fvwLAvOTxYIwYYxER/m++OU+WqU5H/vzn7MOHfyDE
89IIzGN48GCOk452+/vf/DD/MXLhy+cGHK2LER4eH+BgNhjLe0OMrKmg4evLBly+mjR0sBgBA/Sn2N
90GxNDPn/+CnheAeK62WwWDjIXNHZBgGnTpkVFRUFX4ebl5U2+ONc45yIwRKcjvZh54R1FnDPOESF+
91Vqt948bcjRuP6HTmsDA/k0lijDc12RsbW0SQIcZGABBJHD5uZYTtr7y8CTy4SVS8DR8+XPASn1iQ
922sqUKVPUnAsPdwIA1Na2+DhfCMGS1FHWrk8IAKJjlFIATIiZc5BlWl3d6JzjCIBIkr8QBt0NHhDR
93QLW1LeDZ9C2iZuPi4uLj413Q65EAmjypTqrruOAUABobbW4Wq1sN1KhCBIAQujZwkSmlva27LTc2
942gDAwxS9LoPapRwXdOkPgK58GkL/bWlx9GuAfzeQ5RyaWu/gWnC5Om7fmxMsqLXVIaLYfbv/OvDG
95grR830vrjHFZ7gPvu8hX6ZhBIkyhM6q73MY830Mo5ZxTkQ/sXBmYENJVRTJXbMkyY4x7spZ5R6a3
96fUBLS8uWLVvq6+vBqzlFNQfdzG2wCM6hYg9BaZsT+7yz2xTnbe2aeobqDYKjUkVp4dxuNOojI4Ni
97YkIiIgJNJj3nsqK0cE67lRPp3RAkfrpy5cqWLVuam5tdEOUtU16W5ZMnT6alpYWFhXnxhWGMhOHX
98R5NLDwAhxLmSmDhw6dIxisIaG9vffvuou5EAIcS5nJoac999IxWFVVdffe+945p7OIDI226LjBzw
994INjfvrTxKSk0MBAA8ZI5AqUlFzZu/f8Bx/k1dZewdjkm2OVq3GPngiAEGptbT1x4oQIKtQi0xsB
100JEkym83ecSra0uvJTfUBYIwoVZKSQl54YRoAlJc3/f3vx9yttOK21NTw55+/DQAKCmrfe++YBoMI
101IWDM9sQT6X/961x/f9cAJ4vFEBUVMH16/G9/O3X58s+2by/A2OidBsJwrdcTX5Q6s9ks/Oq+pqmK
102ux0Oh1cCdHS9D5wwKsZFioTnLl2z7WgvY4w4t/2f/zNt3bq7jUZJWEnb2uTy8qZz5xpqaqxCkDoc
1031GIxbNt23223JTDmS342t1gMahKcJ7DZbACg07nW6/C2AvR6vUhE7Wq0KDTUBNC9ALQegLrKnUmK
104ncO11S1h7UXG5Li4ga+8MotzTgi6etX+4ovf7thRePlyG6XcYCCDBwc//fRPHntsvMNB9Xry7LO3
105HTpUKp72/C4AYCEhRuiq8Ep7eztCSK/Xd4MAGOPHHntM1PL0nH8KABAdbfEgFW8VEEabO+5I9Pc3
106tLXJZrPu/vs/3bPnBMZ+jImodKWwsOrxxz9ubZVXr04DgPHjowIC/K1WG0Letzg8OtqiosIdBOqS
107kpIef/xx99CeLvwB4eHhQgx42oWJ9e6s6dLfaO4KxoyJBACzWXfgQNmePWckKciZ44gAMCEGAOMn
108n5wUN1ssBn9/PYA3didsQaIOjXcsmUymiIgI9xsk762L8nVqRpj78+JKSkooAOmrKgM9AcY6nPWt
109rQ4AyM4uBxD7gA59X5hFAXBbm+K7QUIUAkpJGQDXMwltipxKg04R6G0jxjVlNzyB2AkPHTqAEEM/
110BoN2CZxzAN2nn5749NPjAICQjhADAEeoo2QQ54xzBaAlPn6okyRdj4UxBmBwEuAa6kGjC6hGuk43
111Yt6iDcUKsFqtfn5+nuISRVNxcUFxcUGlpZcRkm5VixAC4BgbCUGEYIdDobTdyV4wgC4gwBgVFTB9
112+k9efHG6ry0i4JzGxAQPHhwMzrmoTSRV+YdLQrX2YhcEqK+vX7du3YoVK8LDwz3xOEqZwSCNHRtR
113WlqLsa6v7Mw9Ac5BURRZtpnNAWPHJo0eHT506IDBg4NiYizh4f4DBpj1euKJV7iD2HaMGRMhSj6p
114GawIIVGhua2tbefOnQ888IBIquCaepLqsujCHxAYGMg5r62tDQ8PBw9iQEz5GTPit28/0d8Y9oZ8
115hDDnsr+/Yc2a2cuXj42OtrjkPAuk1NW1DhhgliRfeCkC4NOnx6tI4M6ikQcPHszOzo6MjLRarcXF
116xXFxcRaLRSS3MsbKysqioqLE8RHehDDn3Gg0hoWFlZeXjx492jOlOABMnz4Yof7MCegCVQhxLkdF
117WfbsWTpq1EBhvUEItbfLVVXWysqrZWWNZ8827N9fTik7cuQx8MG0RSkD0M+cORg6WLHgchgApkyZ
118Eh8fn5WVxRj7/PPPbTabxWKJiopKTEwMDAz8+OOPn3zySXEgiDcCCGIOGjSouLgYPAgl9YyUUaPC
119x42LyMu7eMP17W4UtPsvFUSm0IYN944aNdBmU4xG6fDhin/841hOzsXKyquybAdQMAbG6MiR8T7y
120H8Yco0ZFjh0bKf510gA45xaLJSgoqLq6OiIiYuTIkefPn7948eKFCxf279/f0NCQkpISGRkJLn6J
121zpArVMyU9vZ2tR5Kp3dSyiUJ3XNPSl5eGUJGgJu7DrwkmwLwyEg/l6uEIErtkycPmTcvyeGgRqP0
122t79995vf7EKIca5T62ASgh0Ouyj02hWIIgjyXXcNxRiJkihOSndwaUrpnDlzBMYSEhKGDBkixHJj
123Y6PZbAY199UL9gVPTEpKSk5O9u6cEZczMob/z/8cuHkZ8S6ntbj/DsABsJiSLmMBoGlpMQCg15Pq
124auvLL2cBSJKkUxQm3DLCNwDABUftCkSahnHx4hHunXGWLcCHDh3Ky8tDCA0aNGjq1KkiwCc0NFSV
12585zzLjxiWsekp4Q/5KzNOXJk+OzZgwEcvgQoIoQAsBqn5eXj3CJdA6NRMplc3B8dWbQDBwbOnDmk
12609GEh/uLb+XlV6xWGWNJRGupN0gSAXAMGxaqGbtHCzyAfcaM+HHjotQCNi5427VrV2ZmZnJycmJi
127Yk5OjsPhOHnypOpcUbUgn6xa2mM/PBn9Bd9/9NEJaje8E4BzGaBFUVrVUC1PH84V56JmAKAoLDzc
128f9y4CACbXt9R+EGSCCEYoPU3v7ltwACzqCbtAlZrh1k3IiJAr8ecc0lSH0eSRByOlvDw0Fdeud05
129duHkwm7hNuI7f/TR8eAWgC12r3V1dceOHVuxYsX8+fMTEhLi4uIGDRqUm5u7bds2uD5+ouvSxej6
1302kyeQDDBBQuGjRoVfepUDcZ6T6JYrI/x4wc98sjtAQEGr1l/YDJJu3efLS6uA5AqKpplmYrH//Sn
131eXPm1FitzSK0i3PKOaxcOXvNmnS1sI8WKQD4++9rAcDhoEOGhDz2WNq77+5jTM8YEtoj5zBpUuLG
132jfeKoiqEYEIwxgqAnXOjtmAlQpgxx9ChkYsWjQC38A6BpbKystDQUBEGcezYsYSEBAC4995733nn
133ncrKypiYGLXUQBcEUGNSDh482NzcfPfdd3dapAA5yyHqdHjVqsmPProNIYO7KBaF6MUsnjVryJw5
134CV62PMLxK0m4vr61uPiiJPn98EPd4cMVM2YMttuVSZOi8/OfWrs2Ny+vRlFYQkLwL34xZt68RADY
135uLHgrruSQ0PN6pZQxPLv23e+pKQhMTFUlunatT/9yU+it207U1fXoteThISQBQtS7rwzyWCQGhvb
136jUbJaEQGg/SrX6W/8UZua6ujudnmHAvHGFOqPP30ZOFUEDWxtKgAAD8/v6tXrzocDs55ZWXlrFmz
137AMBisRiNRhf/iq95wnq9/rvvvrNarWpghadF8NBDY1JTB1HaiStD6KyEYEKQpyqCngBjBMDWrNnb
1380uIwGCRZpoMHB61dOz87+5EjRx7bvHnRHXckAMBf/5rzyiv7goONoIlY5hwwJm1tbatXfymyORnj
139Dz00eteuJUeOPJ6dvfzDDxcuXJhiMEj5+TXp6RvKyhoRQna7smpVWlXVMw8/PAbARggSyg+l9pSU
140qGXLxrlMf62eMmzYMKPRuHXr1ry8vIEDB0ZHRwPA6dOnKaXiu08uSe1948aNy8rKOnny5G233ebJ
141LCoWgV5P/vCHmYsXb3KZzgCorU0+dOiC78YixlhgoLG2tgUAKwrD2HD8+IVZsz745z8XpqaGq3HO
142oj/Nze0vv3zgzTe/iY+PPHSowmzWnTp1SdsUxsbMzNNz5360bt09Q4YEO+cQF1HTly+3vv320ddf
143P9Ta2rxhw4m//W2uWoxAOFydwQ3AOX3xxZkmk+v0V3l1W1ub2Wx+5JFHNm7cKPhPTk5OTU1Nbm5u
144RkaGwWDQchGf4gkFF9q9e/fJkyefe+457dmCbljukEJ33bUxM/MMIWZnpJ/qrunBeQgEAKsBDYzZ
145JUmaPj1xxoy4uLggnY5cvtyan1+7e/cP9fUNGJsZY863IAA1XxyphVSMRuOcOUnp6bExMRaEUG2t
1469ejR6qysksbGKxibADBjjgULRt5zT4rJJFVVWf/1r/yiolqEJIQQY+1z5qR89dVS7cFsKkIF9r/4
1474osFCxbodDpZlk+dOnXq1Kn6+nqz2Zyeni7OI9VObp8IIO6ur6/funXrkiVLhCbrKVZXBBsXFdVP
148nPiP1lbFibsOGvTAaaNWkxT/OQ9BsQOoQZxC2OjV8Gz1LW7hPeJxUT6ROTmw+rhOhOUihDi3qSH1
149AHonq+BGI8rNXTF6dIRaDVQb+EYIaWxsfOutt1asWBEREUEpdT8IE67Hgk8pSuJLaGjok08+6QX7
150HS1ipChs2LCwV16ZA2BzMQyIXU+3Pi7dYYxxDoQYJcmfEDMhJvEFIaI66zXPos4eR86nTNc/TtXH
151CTGpjSMkidgTgPY//GHW6NERatF3AfX19QL7lNLg4OCwsDCRGAwaxb2trU1dKNpJ373kKRfC+MaI
152ThPi52RE/6HACSGUtt1+e0pW1jXmI5Bgs9lef/31gICABQsWDBo0CCH0zTffHDlyJCUlpbGx0Waz
153ORyOpqamMWPGLFy40L3OW/fKVoLGeOuJBiLaUj2BdPLkdRUVTRjr+7tAdM+xL0rQR0YG5OauiI0N
154FEPT8pPa2tq9e/eeOXNm6NChCxcuBIC33norPDw8MDDQZDL5+fkZDIbU1NROmUf3YtmcWZy0tbU1
155ICDAMw2u1e07ePDCnDkbZFn1Cv1n0aDj9BiEWFbWstmzh7gXylLnYmVl5e7du0tLS8ePH19RUbFg
156wYLk5GRtbdtO0dW9mnGilYKCgrffflsEunRKvw5nm4QVhU2bFrdhwyIAu6hZeMvGrXQ6XBHKyLn9
157/ffvnT17iKi+6C5UBURHR69YsWLFihX19fXV1dUHDhxoaWkRKoOQLp1O1m4fZ4sQCgkJOXz4cFNT
1580/Dhw9UW3TNDOOeEYEWhY8dG+vub9+49TYj+epXmVgYOgCQJUdr6xz/euWpVmkjs6TQHpr29/bPP
159PtuxY0dRUdGkSZOmTZsWExNTVFSUlZVlt9tjY2NFPFanWUbdI4DQeXU6XVhY2K5duxISEgRf8xCa
160isQ5RpTy9PRYQvTffHNGkv4jaNCBfUVpfeGFef/93zO0ey4XwwNj7P33329sbExPT9fr9YmJiQI/
161aWlpFoslPz9/xIgRJpMJPOjg3ZYB4NRwPv7448rKymeffRa8pvAh5ylVkoT/9Kfs5577nBAjY7jv
162y8n5PkqEMMac0vaXX57/hz9M91SCXjipjh07lpWVtWbNGrWcoSzLe/bsSU9PDw4OppS6HMbuAt07
163yE3b0J133nnlirfsQO39hICi0N/9Lj0kxLRixQ7OMSG6W1I35RgTzmVK6TvvLF65cqIn7KuGkKqq
164qoiICL1eL8syxlhUNTlx4oSiKPfee2+X7+v5ESYWi2Xw4MEuEqlTd42TBliW6eOPj//yy0eCg42U
165tkuScKrcImJZJPITxtoDAgyff7505cqJskxdsK8OkznPlIuKiqqoqGhtbRWRz4qi6HS66dOni6TU
166Ls9w7DYBtL1Rjy1xiezw9IgkYVmmc+cmHD/+1MSJgxWlhRDo6flcvYx9jDEhoCgtY8bEHj/+5F13
167JQudx9MACSFiso8dO9ZsNn/44YeiUqu48/Lly2qCu/cXd1sLguvLMoovly5dUhTFZDJ5OstE02+s
168KCwkxLRs2Vi7nWRnn+dcIUTv9Oj2PUfqyBdjzME5Xb165iefLB440F/oPNrxav2INpvt8OHDR48e
169tVqt0dHRI0eOzM7OPnjwoF6vlyQpNzf38OHD9913X1BQkJcM347GbySpSDWUbtiwwWq1Pv300ypt
170vItlcWCLOI9lxYrdp0+XI2TEWHKu674hA3dGSimc21JSYtetu+v6s9w6hgiaEAWEUGNj4/r16yml
171AwcOLCsrE5bnkJCQL7/8sqCgQJZlPz+/u+++e8SIEVor6U0hgIrQq1evvvHGG3FxcUuXLgXPSpH2
172EVU1stuVd9459uqr+5uaGvuKDNeh3mIJfP756atWpQkPl/ASg5PBqtNfDeh8//33CSHLly8HgLa2
173to8//ri0tHTVqlXh4eF2u729vT0gIEA1gnYZ5dgTFnQdARFijBmNxmHDhu3Zs+fixYujR4/2/mIt
174OxJG3alTY5ctGwugP3WqzmazAiCMJe8FYHqGdwDkFKoK5+1+fuaVK9O3bFk8b16SKJWrMn2xshlj
175Fy9erK2t9fPz0+v1CKGmpqY9e/YsWrQoKCiIUmowGMaNG1dcXHzmzJlJkyYRQoxGI3Kecuc9lkfA
176jZ4nrHY0PDx8+fLl3377rcPhMBgM4HUdqNNKnISgKCwszO8vf7n9179Oe++9vPXr86qr6wEAQC8E
177XbdOse3sdcI9KU4HdQBARMSARx8dt2LFhOhoC2PcRdcUgyopKdm5c6fVahWCbfHixampqeJXNW1L
178WPx/+tOfrl+/vq6uLjw8XCj+XmoL9DIBtNSOj49ftmyZOgzBSbyXOVBrjgosRET4v/TSjDVrpmRm
179nvvoo5P795e1tVkBAEAHIKk4UvPcPaFbcA6V0XGuUKoAcJMpYNq05IcfHn333UNFlqTgOcLCIxoU
1806M7Pz//kk09mzJiRnp5OCMnMzBTFZgIDA+Pi4r766qvhw4cTQhRFAYCgoCBCiN1uB429wUffU68d
1816KyuXK28cr/i4XEQfFk9XlkMoLraundvyZ49JTk5FysrmwDEKWDCQyk+1zXpNHIw50ds9PRRUUFT
182pgyaNy9x7tzEmJiOoGj1CGn3GOnGxsa//OUv99xzT1pamjYmU8yn+vr6N998MyEh4cEHH9TpdAih
183L7/88uTJk7/97W99n/i9TACVDNfaRSgvLy8iIiI6OrrL7bg7ISnlCF07q6u9Xf7hh4a8vOrvv68r
184LKyvrLx66VKr1eqQZVlzJh4CwDqd5O+vHzjQLybGMmxYWGpq+PjxUcOGDTCZdFoFzNP5aoKlZGdn
185Hzhw4He/+506lxFCLS0ttbW1JpMpOjq6srLygw8+UBRlxIgRjY2NFy9efOSRR4YMGeLLIeIu0Jtn
186yrsYab///vtt27YtW7YsJSVFXQq+tAAA6lmaooSM0SiNGRMxenQ4dIh93txsa262NzfbbDZFVKrQ
1876bDRKAUGGi0WQ1CQ0WVqi7P7xKmFWut8px0wGAytra1NTU2hoaGKopSXlx85cqS4uNhms1FKp0yZ
188snjx4meeeSY3N/f8+fMhISH33nvvwIEDuQ8ZXZ0MuRdXgArq8L744ouvvvrqjjvumD17ttejNzy1
189I8JAROHBDtYv+IYXh6jTRX7tLFRN8lAXJdWdC679jTfeYIwlJiaWl5c3NDRERUVNmDBhyJAhZWVl
19027dv/8UvfjF27NgunS39QwAt98cYnzlzZvPmzUuWLBk1apSWn/asu2pvPVVkVaN3tP92t32EUHV1
191dWZmZnNzc0JCwsSJE0U0lfhp3bp1gYGBS5YsURRF3eX2gPvfLAK406ClpcVgMOh0Og361KolXWvK
192fQlaa4/LF+HVkiTp7bffjo6OzsjIELLtBvvfwyPNvYM6u4Uyqk2yFIYUdffgyX7Xl6BqONq9K3cm
1931MmyzJ1nF0qSdOjQocrKysmTJ4NTON/g7OlNIawFtVtaHU5c+eijjzDGGRkZAwYM8FE43yTQmnVB
194M+XVBVpXV/fBBx/Mnj07NTX16tWr+/bty8vLe+CBByIjIz2dpNZtRPXZ7FOXc2lp6RdffFFRUTF2
1957NhZs2aJBNjr+tQj8dDdzqjTXFWRtdtGZ2CHsmvXrtzcXJPJpChKWFhYRkZGbGyslwOsuwt9vfxV
196Mpw9e3bHjh1JSUmLFi1y2eyoJtxep4SLyFH/LS8vz8zMHD16dHp6urtuc+nSpbq6uuDg4KioKME5
197u9xa3ooEUMejVmJUFEVRFJEuK8Zjs9lUY1ZH/9yQ1bP3goa0Ku7sdntOTk5+fn59fX1CQsIdd9wR
198FxenfbX7svDdyuYj3CwZ4A7qNk0MQARTqmfNAYDNZlu7dq3FYpkwYUJSUpI4ckKrh2hnnIvBw9O7
199tPeD2ykuIm8rMTHxoYceEjsp7SMuEkIVxb27KPtHA3HX9gTDPXv27MmTJ8+fP2+1WtPS0jIyMnqw
200uXdRIgU0NzdXVlYWFhaOHz8+ISFBZXoqu+uyQupNgr5bAVpwd2oCgCRJw4YNGz58uKIo586dcxED
201R44cqampGTRoUGBgoMViCQ4OFhsLLaIZY4qiUEpFjSN1J7hjxw5ZlgkhgYGBqampLj1RVaA+EP6d
202oKJ/dXABWg4LTkah5d0iSe3YsWMOh8Nms8myvHLlyujoaDGR29vb169f39LSIqwI4eHhK1euBKdh
203ubq6uqioaMiQIZGRkULegJvZqh93grcEAQRop7N2q6xlVoyx1tZWq9U6YMAAbSDU8ePHEULiANOg
204oKDY2FithHCRFv0y0z3BLUQAT6C6d7TaIfiAR5c9bZcBA/0C/wEEEKDtZ6duHy1a3Wtk37LwH0OA
205/1fhphjjfgTf4f8C4VLHz/5KLxoAAAA8dEVYdGNvbW1lbnQAIEltYWdlIGdlbmVyYXRlZCBieSBH
206TlUgR2hvc3RzY3JpcHQgKGRldmljZT1wbm1yYXcpCvqLFvMAAAAASUVORK5CYII=
diff --git a/gem/input.bin b/gem/input.bin
new file mode 100644
index 0000000..d24a954
--- /dev/null
+++ b/gem/input.bin
Binary files differ
diff --git a/gem/ltn012.tex b/gem/ltn012.tex
index 0f81b86..8eccd46 100644
--- a/gem/ltn012.tex
+++ b/gem/ltn012.tex
@@ -6,7 +6,10 @@
6\DefineVerbatimEnvironment{mime}{Verbatim}{fontsize=\small,commandchars=\$\#\%} 6\DefineVerbatimEnvironment{mime}{Verbatim}{fontsize=\small,commandchars=\$\#\%}
7\newcommand{\stick}[1]{\vbox{\setlength{\parskip}{0pt}#1}} 7\newcommand{\stick}[1]{\vbox{\setlength{\parskip}{0pt}#1}}
8\newcommand{\bl}{\ensuremath{\mathtt{\backslash}}} 8\newcommand{\bl}{\ensuremath{\mathtt{\backslash}}}
9 9\newcommand{\CR}{\texttt{CR}}
10\newcommand{\LF}{\texttt{LF}}
11\newcommand{\CRLF}{\texttt{CR~LF}}
12\newcommand{\nil}{\texttt{nil}}
10 13
11\title{Filters, sources, sinks, and pumps\\ 14\title{Filters, sources, sinks, and pumps\\
12 {\large or Functional programming for the rest of us}} 15 {\large or Functional programming for the rest of us}}
@@ -17,30 +20,31 @@
17\maketitle 20\maketitle
18 21
19\begin{abstract} 22\begin{abstract}
20Certain data processing operations can be implemented in the 23Certain data processing operations can be implemented in the
21form of filters. A filter is a function that can process data 24form of filters. A filter is a function that can process
22received in consecutive function calls, returning partial 25data received in consecutive invocations, returning partial
23results after each invocation. Examples of operations that can be 26results each time it is called. Examples of operations that
24implemented as filters include the end-of-line normalization 27can be implemented as filters include the end-of-line
25for text, Base64 and Quoted-Printable transfer content 28normalization for text, Base64 and Quoted-Printable transfer
26encodings, the breaking of text into lines, SMTP dot-stuffing, 29content encodings, the breaking of text into lines, SMTP
27and there are many others. Filters become even 30dot-stuffing, and there are many others. Filters become
28more powerful when we allow them to be chained together to 31even more powerful when we allow them to be chained together
29create composite filters. In this context, filters can be seen 32to create composite filters. In this context, filters can be
30as the middle links in a chain of data transformations. Sources an sinks 33seen as the internal links in a chain of data transformations.
31are the corresponding end points of these chains. A source 34Sources and sinks are the corresponding end points in these
32is a function that produces data, chunk by chunk, and a sink 35chains. A source is a function that produces data, chunk by
33is a function that takes data, chunk by chunk. In this 36chunk, and a sink is a function that takes data, chunk by
34article, we describe the design of an elegant interface for filters, 37chunk. Finally, pumps are procedures that actively drive
35sources, sinks, and chaining, and illustrate each step 38data from a source to a sink, and indirectly through all
36with concrete examples. 39intervening filters. In this article, we describe the design of an
40elegant interface for filters, sources, sinks, chains, and
41pumps, and we illustrate each step with concrete examples.
37\end{abstract} 42\end{abstract}
38 43
39
40\section{Introduction} 44\section{Introduction}
41 45
42Within the realm of networking applications, we are often 46Within the realm of networking applications, we are often
43required apply transformations to streams of data. Examples 47required to apply transformations to streams of data. Examples
44include the end-of-line normalization for text, Base64 and 48include the end-of-line normalization for text, Base64 and
45Quoted-Printable transfer content encodings, breaking text 49Quoted-Printable transfer content encodings, breaking text
46into lines with a maximum number of columns, SMTP 50into lines with a maximum number of columns, SMTP
@@ -50,11 +54,10 @@ transfer coding, and the list goes on.
50Many complex tasks require a combination of two or more such 54Many complex tasks require a combination of two or more such
51transformations, and therefore a general mechanism for 55transformations, and therefore a general mechanism for
52promoting reuse is desirable. In the process of designing 56promoting reuse is desirable. In the process of designing
53\texttt{LuaSocket~2.0}, David Burgess and I were forced to deal with 57\texttt{LuaSocket~2.0}, we repeatedly faced this problem.
54this problem. The solution we reached proved to be very 58The solution we reached proved to be very general and
55general and convenient. It is based on the concepts of 59convenient. It is based on the concepts of filters, sources,
56filters, sources, sinks, and pumps, which we introduce 60sinks, and pumps, which we introduce below.
57below.
58 61
59\emph{Filters} are functions that can be repeatedly invoked 62\emph{Filters} are functions that can be repeatedly invoked
60with chunks of input, successively returning processed 63with chunks of input, successively returning processed
@@ -62,34 +65,33 @@ chunks of output. More importantly, the result of
62concatenating all the output chunks must be the same as the 65concatenating all the output chunks must be the same as the
63result of applying the filter to the concatenation of all 66result of applying the filter to the concatenation of all
64input chunks. In fancier language, filters \emph{commute} 67input chunks. In fancier language, filters \emph{commute}
65with the concatenation operator. As a result, chunk 68with the concatenation operator. More importantly, filters
66boundaries are irrelevant: filters correctly handle input 69must handle input data correctly no matter how the stream
67data no matter how it is split. 70has been split into chunks.
68 71
69A \emph{chain} transparently combines the effect of one or 72A \emph{chain} is a function that transparently combines the
70more filters. The interface of a chain is 73effect of one or more filters. The interface of a chain is
71indistinguishable from the interface of its components. 74indistinguishable from the interface of its component
72This allows a chained filter to be used wherever an atomic 75filters. This allows a chained filter to be used wherever
73filter is expected. In particular, chains can be 76an atomic filter is accepted. In particular, chains can be
74themselves chained to create arbitrarily complex operations. 77themselves chained to create arbitrarily complex operations.
75 78
76Filters can be seen as internal nodes in a network through 79Filters can be seen as internal nodes in a network through
77which data will flow, potentially being transformed many 80which data will flow, potentially being transformed many
78times along its way. Chains connect these nodes together. 81times along the way. Chains connect these nodes together.
79To complete the picture, we need \emph{sources} and 82The initial and final nodes of the network are
80\emph{sinks}. These are the initial and final nodes of the 83\emph{sources} and \emph{sinks}, respectively. Less
81network, respectively. Less abstractly, a source is a 84abstractly, a source is a function that produces new data
82function that produces new data every time it is called. 85every time it is invoked. Conversely, sinks are functions
83Conversely, sinks are functions that give a final 86that give a final destination to the data they receive.
84destination to the data they receive. Naturally, sources 87Naturally, sources and sinks can also be chained with
85and sinks can also be chained with filters to produce 88filters to produce filtered sources and sinks.
86filtered sources and sinks.
87 89
88Finally, filters, chains, sources, and sinks are all passive 90Finally, filters, chains, sources, and sinks are all passive
89entities: they must be repeatedly invoked in order for 91entities: they must be repeatedly invoked in order for
90anything to happen. \emph{Pumps} provide the driving force 92anything to happen. \emph{Pumps} provide the driving force
91that pushes data through the network, from a source to a 93that pushes data through the network, from a source to a
92sink. 94sink, and indirectly through all intervening filters.
93 95
94In the following sections, we start with a simplified 96In the following sections, we start with a simplified
95interface, which we later refine. The evolution we present 97interface, which we later refine. The evolution we present
@@ -99,27 +101,28 @@ concepts within our application domain.
99 101
100\subsection{A simple example} 102\subsection{A simple example}
101 103
102Let us use the end-of-line normalization of text as an 104The end-of-line normalization of text is a good
103example to motivate our initial filter interface. 105example to motivate our initial filter interface.
104Assume we are given text in an unknown end-of-line 106Assume we are given text in an unknown end-of-line
105convention (including possibly mixed conventions) out of the 107convention (including possibly mixed conventions) out of the
106commonly found Unix (LF), Mac OS (CR), and DOS (CRLF) 108commonly found Unix (\LF), Mac OS (\CR), and
107conventions. We would like to be able to write code like the 109DOS (\CRLF) conventions. We would like to be able to
108following: 110use the folowing code to normalize the end-of-line markers:
109\begin{quote} 111\begin{quote}
110\begin{lua} 112\begin{lua}
111@stick# 113@stick#
112local in = source.chain(source.file(io.stdin), normalize("\r\n")) 114local CRLF = "\013\010"
113local out = sink.file(io.stdout) 115local input = source.chain(source.file(io.stdin), normalize(CRLF))
114pump.all(in, out) 116local output = sink.file(io.stdout)
117pump.all(input, output)
115% 118%
116\end{lua} 119\end{lua}
117\end{quote} 120\end{quote}
118 121
119This program should read data from the standard input stream 122This program should read data from the standard input stream
120and normalize the end-of-line markers to the canonic CRLF 123and normalize the end-of-line markers to the canonic
121marker, as defined by the MIME standard. Finally, the 124\CRLF\ marker, as defined by the MIME standard.
122normalized text should be sent to the standard output 125Finally, the normalized text should be sent to the standard output
123stream. We use a \emph{file source} that produces data from 126stream. We use a \emph{file source} that produces data from
124standard input, and chain it with a filter that normalizes 127standard input, and chain it with a filter that normalizes
125the data. The pump then repeatedly obtains data from the 128the data. The pump then repeatedly obtains data from the
@@ -127,27 +130,28 @@ source, and passes it to the \emph{file sink}, which sends
127it to the standard output. 130it to the standard output.
128 131
129In the code above, the \texttt{normalize} \emph{factory} is a 132In the code above, the \texttt{normalize} \emph{factory} is a
130function that creates our normalization filter. This filter 133function that creates our normalization filter, which
131will replace any end-of-line marker with the canonic 134replaces any end-of-line marker with the canonic marker.
132`\verb|\r\n|' marker. The initial filter interface is 135The initial filter interface is
133trivial: a filter function receives a chunk of input data, 136trivial: a filter function receives a chunk of input data,
134and returns a chunk of processed data. When there are no 137and returns a chunk of processed data. When there are no
135more input data left, the caller notifies the filter by invoking 138more input data left, the caller notifies the filter by invoking
136it with a \texttt{nil} chunk. The filter responds by returning 139it with a \nil\ chunk. The filter responds by returning
137the final chunk of processed data. 140the final chunk of processed data (which could of course be
141the empty string).
138 142
139Although the interface is extremely simple, the 143Although the interface is extremely simple, the
140implementation is not so obvious. A normalization filter 144implementation is not so obvious. A normalization filter
141respecting this interface needs to keep some kind of context 145respecting this interface needs to keep some kind of context
142between calls. This is because a chunk boundary may lie between 146between calls. This is because a chunk boundary may lie between
143the CR and LF characters marking the end of a line. This 147the \CR\ and \LF\ characters marking the end of a single line. This
144need for contextual storage motivates the use of 148need for contextual storage motivates the use of
145factories: each time the factory is invoked, it returns a 149factories: each time the factory is invoked, it returns a
146filter with its own context so that we can have several 150filter with its own context so that we can have several
147independent filters being used at the same time. For 151independent filters being used at the same time. For
148efficiency reasons, we must avoid the obvious solution of 152efficiency reasons, we must avoid the obvious solution of
149concatenating all the input into the context before 153concatenating all the input into the context before
150producing any output. 154producing any output chunks.
151 155
152To that end, we break the implementation into two parts: 156To that end, we break the implementation into two parts:
153a low-level filter, and a factory of high-level filters. The 157a low-level filter, and a factory of high-level filters. The
@@ -167,10 +171,10 @@ end-of-line normalization filters:
167\begin{quote} 171\begin{quote}
168\begin{lua} 172\begin{lua}
169@stick# 173@stick#
170function filter.cycle(low, ctx, extra) 174function filter.cycle(lowlevel, context, extra)
171 return function(chunk) 175 return function(chunk)
172 local ret 176 local ret
173 ret, ctx = low(ctx, chunk, extra) 177 ret, context = lowlevel(context, chunk, extra)
174 return ret 178 return ret
175 end 179 end
176end 180end
@@ -178,27 +182,30 @@ end
178 182
179@stick# 183@stick#
180function normalize(marker) 184function normalize(marker)
181 return cycle(eol, 0, marker) 185 return filter.cycle(eol, 0, marker)
182end 186end
183% 187%
184\end{lua} 188\end{lua}
185\end{quote} 189\end{quote}
186 190
187The \texttt{normalize} factory simply calls a more generic 191The \texttt{normalize} factory simply calls a more generic
188factory, the \texttt{cycle} factory. This factory receives a 192factory, the \texttt{cycle}~factory, passing the low-level
193filter~\texttt{eol}. The \texttt{cycle}~factory receives a
189low-level filter, an initial context, and an extra 194low-level filter, an initial context, and an extra
190parameter, and returns a new high-level filter. Each time 195parameter, and returns a new high-level filter. Each time
191the high-level filer is passed a new chunk, it invokes the 196the high-level filer is passed a new chunk, it invokes the
192low-level filter with the previous context, the new chunk, 197low-level filter with the previous context, the new chunk,
193and the extra argument. It is the low-level filter that 198and the extra argument. It is the low-level filter that
194does all the work, producing the chunk of processed data and 199does all the work, producing the chunk of processed data and
195a new context. The high-level filter then updates its 200a new context. The high-level filter then replaces its
196internal context, and returns the processed chunk of data to 201internal context, and returns the processed chunk of data to
197the user. Notice that we take advantage of Lua's lexical 202the user. Notice that we take advantage of Lua's lexical
198scoping to store the context in a closure between function 203scoping to store the context in a closure between function
199calls. 204calls.
200 205
201Concerning the low-level filter code, we must first accept 206\subsection{The C part of the filter}
207
208As for the low-level filter, we must first accept
202that there is no perfect solution to the end-of-line marker 209that there is no perfect solution to the end-of-line marker
203normalization problem. The difficulty comes from an 210normalization problem. The difficulty comes from an
204inherent ambiguity in the definition of empty lines within 211inherent ambiguity in the definition of empty lines within
@@ -208,39 +215,39 @@ mixed input. It also does a reasonable job with empty lines
208and serves as a good example of how to implement a low-level 215and serves as a good example of how to implement a low-level
209filter. 216filter.
210 217
211The idea is to consider both CR and~LF as end-of-line 218The idea is to consider both \CR\ and~\LF\ as end-of-line
212\emph{candidates}. We issue a single break if any candidate 219\emph{candidates}. We issue a single break if any candidate
213is seen alone, or followed by a different candidate. In 220is seen alone, or if it is followed by a different
214other words, CR~CR~and LF~LF each issue two end-of-line 221candidate. In other words, \CR~\CR~and \LF~\LF\ each issue
215markers, whereas CR~LF~and LF~CR issue only one marker each. 222two end-of-line markers, whereas \CR~\LF~and \LF~\CR\ issue
216This method correctly handles the Unix, DOS/MIME, VMS, and Mac 223only one marker each. It is easy to see that this method
217OS conventions. 224correctly handles the most common end-of-line conventions.
218 225
219\subsection{The C part of the filter} 226With this in mind, we divide the low-level filter into two
220 227simple functions. The inner function~\texttt{pushchar} performs the
221Our low-level filter is divided into two simple functions. 228normalization itself. It takes each input character in turn,
222The inner function performs the normalization itself. It takes 229deciding what to output and how to modify the context. The
223each input character in turn, deciding what to output and 230context tells if the last processed character was an
224how to modify the context. The context tells if the last 231end-of-line candidate, and if so, which candidate it was.
225processed character was an end-of-line candidate, and if so, 232For efficiency, we use Lua's auxiliary library's buffer
226which candidate it was. For efficiency, it uses 233interface:
227Lua's auxiliary library's buffer interface:
228\begin{quote} 234\begin{quote}
229\begin{C} 235\begin{C}
230@stick# 236@stick#
231@#define candidate(c) (c == CR || c == LF) 237@#define candidate(c) (c == CR || c == LF)
232static int process(int c, int last, const char *marker, 238static int pushchar(int c, int last, const char *marker,
233 luaL_Buffer *buffer) { 239 luaL_Buffer *buffer) {
234 if (candidate(c)) { 240 if (candidate(c)) {
235 if (candidate(last)) { 241 if (candidate(last)) {
236 if (c == last) luaL_addstring(buffer, marker); 242 if (c == last)
243 luaL_addstring(buffer, marker);
237 return 0; 244 return 0;
238 } else { 245 } else {
239 luaL_addstring(buffer, marker); 246 luaL_addstring(buffer, marker);
240 return c; 247 return c;
241 } 248 }
242 } else { 249 } else {
243 luaL_putchar(buffer, c); 250 luaL_pushchar(buffer, c);
244 return 0; 251 return 0;
245 } 252 }
246} 253}
@@ -248,15 +255,20 @@ static int process(int c, int last, const char *marker,
248\end{C} 255\end{C}
249\end{quote} 256\end{quote}
250 257
251The outer function simply interfaces with Lua. It receives the 258The outer function~\texttt{eol} simply interfaces with Lua.
252context and input chunk (as well as an optional 259It receives the context and input chunk (as well as an
253custom end-of-line marker), and returns the transformed 260optional custom end-of-line marker), and returns the
254output chunk and the new context: 261transformed output chunk and the new context.
262Notice that if the input chunk is \nil, the operation
263is considered to be finished. In that case, the loop will
264not execute a single time and the context is reset to the
265initial state. This allows the filter to be reused many
266times:
255\begin{quote} 267\begin{quote}
256\begin{C} 268\begin{C}
257@stick# 269@stick#
258static int eol(lua_State *L) { 270static int eol(lua_State *L) {
259 int ctx = luaL_checkint(L, 1); 271 int context = luaL_checkint(L, 1);
260 size_t isize = 0; 272 size_t isize = 0;
261 const char *input = luaL_optlstring(L, 2, NULL, &isize); 273 const char *input = luaL_optlstring(L, 2, NULL, &isize);
262 const char *last = input + isize; 274 const char *last = input + isize;
@@ -269,24 +281,18 @@ static int eol(lua_State *L) {
269 return 2; 281 return 2;
270 } 282 }
271 while (input < last) 283 while (input < last)
272 ctx = process(*input++, ctx, marker, &buffer); 284 context = pushchar(*input++, context, marker, &buffer);
273 luaL_pushresult(&buffer); 285 luaL_pushresult(&buffer);
274 lua_pushnumber(L, ctx); 286 lua_pushnumber(L, context);
275 return 2; 287 return 2;
276} 288}
277% 289%
278\end{C} 290\end{C}
279\end{quote} 291\end{quote}
280 292
281Notice that if the input chunk is \texttt{nil}, the operation
282is considered to be finished. In that case, the loop will
283not execute a single time and the context is reset to the
284initial state. This allows the filter to be reused many
285times.
286
287When designing your own filters, the challenging part is to 293When designing your own filters, the challenging part is to
288decide what will be in the context. For line breaking, for 294decide what will be in the context. For line breaking, for
289instance, it could be the number of bytes left in the 295instance, it could be the number of bytes that still fit in the
290current line. For Base64 encoding, it could be a string 296current line. For Base64 encoding, it could be a string
291with the bytes that remain after the division of the input 297with the bytes that remain after the division of the input
292into 3-byte atoms. The MIME module in the \texttt{LuaSocket} 298into 3-byte atoms. The MIME module in the \texttt{LuaSocket}
@@ -294,19 +300,22 @@ distribution has many other examples.
294 300
295\section{Filter chains} 301\section{Filter chains}
296 302
297Chains add a lot to the power of filters. For example, 303Chains greatly increase the power of filters. For example,
298according to the standard for Quoted-Printable encoding, 304according to the standard for Quoted-Printable encoding,
299text must be normalized to a canonic end-of-line marker 305text should be normalized to a canonic end-of-line marker
300prior to encoding. To help specifying complex 306prior to encoding. After encoding, the resulting text must
301transformations like this, we define a chain factory that 307be broken into lines of no more than 76 characters, with the
302creates a composite filter from one or more filters. A 308use of soft line breaks (a line terminated by the \texttt{=}
303chained filter passes data through all its components, and 309sign). To help specifying complex transformations like
304can be used wherever a primitive filter is accepted. 310this, we define a chain factory that creates a composite
311filter from one or more filters. A chained filter passes
312data through all its components, and can be used wherever a
313primitive filter is accepted.
305 314
306The chaining factory is very simple. The auxiliary 315The chaining factory is very simple. The auxiliary
307function~\texttt{chainpair} chains two filters together, 316function~\texttt{chainpair} chains two filters together,
308taking special care if the chunk is the last. This is 317taking special care if the chunk is the last. This is
309because the final \texttt{nil} chunk notification has to be 318because the final \nil\ chunk notification has to be
310pushed through both filters in turn: 319pushed through both filters in turn:
311\begin{quote} 320\begin{quote}
312\begin{lua} 321\begin{lua}
@@ -322,9 +331,9 @@ end
322 331
323@stick# 332@stick#
324function filter.chain(...) 333function filter.chain(...)
325 local f = arg[1] 334 local f = select(1, ...)
326 for i = 2, @#arg do 335 for i = 2, select('@#', ...) do
327 f = chainpair(f, arg[i]) 336 f = chainpair(f, select(i, ...))
328 end 337 end
329 return f 338 return f
330end 339end
@@ -337,11 +346,11 @@ define the Quoted-Printable conversion as such:
337\begin{quote} 346\begin{quote}
338\begin{lua} 347\begin{lua}
339@stick# 348@stick#
340local qp = filter.chain(normalize("\r\n"), 349local qp = filter.chain(normalize(CRLF), encode("quoted-printable"),
341 encode("quoted-printable")) 350 wrap("quoted-printable"))
342local in = source.chain(source.file(io.stdin), qp) 351local input = source.chain(source.file(io.stdin), qp)
343local out = sink.file(io.stdout) 352local output = sink.file(io.stdout)
344pump.all(in, out) 353pump.all(input, output)
345% 354%
346\end{lua} 355\end{lua}
347\end{quote} 356\end{quote}
@@ -360,14 +369,14 @@ gives a final destination to the data.
360\subsection{Sources} 369\subsection{Sources}
361 370
362A source returns the next chunk of data each time it is 371A source returns the next chunk of data each time it is
363invoked. When there is no more data, it simply returns 372invoked. When there is no more data, it simply returns~\nil.
364\texttt{nil}. In the event of an error, the source can inform the 373In the event of an error, the source can inform the
365caller by returning \texttt{nil} followed by an error message. 374caller by returning \nil\ followed by the error message.
366 375
367Below are two simple source factories. The \texttt{empty} source 376Below are two simple source factories. The \texttt{empty} source
368returns no data, possibly returning an associated error 377returns no data, possibly returning an associated error
369message. The \texttt{file} source works harder, and 378message. The \texttt{file} source yields the contents of a file
370yields the contents of a file in a chunk by chunk fashion: 379in a chunk by chunk fashion:
371\begin{quote} 380\begin{quote}
372\begin{lua} 381\begin{lua}
373@stick# 382@stick#
@@ -398,7 +407,7 @@ A filtered source passes its data through the
398associated filter before returning it to the caller. 407associated filter before returning it to the caller.
399Filtered sources are useful when working with 408Filtered sources are useful when working with
400functions that get their input data from a source (such as 409functions that get their input data from a source (such as
401the pump in our first example). By chaining a source with one or 410the pumps in our examples). By chaining a source with one or
402more filters, the function can be transparently provided 411more filters, the function can be transparently provided
403with filtered data, with no need to change its interface. 412with filtered data, with no need to change its interface.
404Here is a factory that does the job: 413Here is a factory that does the job:
@@ -406,14 +415,18 @@ Here is a factory that does the job:
406\begin{lua} 415\begin{lua}
407@stick# 416@stick#
408function source.chain(src, f) 417function source.chain(src, f)
409 return source.simplify(function() 418 return function()
410 if not src then return nil end 419 if not src then
420 return nil
421 end
411 local chunk, err = src() 422 local chunk, err = src()
412 if not chunk then 423 if not chunk then
413 src = nil 424 src = nil
414 return f(nil) 425 return f(nil)
415 else return f(chunk) end 426 else
416 end) 427 return f(chunk)
428 end
429 end
417end 430end
418% 431%
419\end{lua} 432\end{lua}
@@ -421,20 +434,20 @@ end
421 434
422\subsection{Sinks} 435\subsection{Sinks}
423 436
424Just as we defined an interface a data source, 437Just as we defined an interface for source of data,
425we can also define an interface for a data destination. 438we can also define an interface for a data destination.
426We call any function respecting this 439We call any function respecting this
427interface a \emph{sink}. In our first example, we used a 440interface a \emph{sink}. In our first example, we used a
428file sink connected to the standard output. 441file sink connected to the standard output.
429 442
430Sinks receive consecutive chunks of data, until the end of 443Sinks receive consecutive chunks of data, until the end of
431data is signaled by a \texttt{nil} chunk. A sink can be 444data is signaled by a \nil\ input chunk. A sink can be
432notified of an error with an optional extra argument that 445notified of an error with an optional extra argument that
433contains the error message, following a \texttt{nil} chunk. 446contains the error message, following a \nil\ chunk.
434If a sink detects an error itself, and 447If a sink detects an error itself, and
435wishes not to be called again, it can return \texttt{nil}, 448wishes not to be called again, it can return \nil,
436followed by an error message. A return value that 449followed by an error message. A return value that
437is not \texttt{nil} means the source will accept more data. 450is not \nil\ means the sink will accept more data.
438 451
439Below are two useful sink factories. 452Below are two useful sink factories.
440The table factory creates a sink that stores 453The table factory creates a sink that stores
@@ -469,7 +482,7 @@ end
469 482
470Naturally, filtered sinks are just as useful as filtered 483Naturally, filtered sinks are just as useful as filtered
471sources. A filtered sink passes each chunk it receives 484sources. A filtered sink passes each chunk it receives
472through the associated filter before handing it to the 485through the associated filter before handing it down to the
473original sink. In the following example, we use a source 486original sink. In the following example, we use a source
474that reads from the standard input. The input chunks are 487that reads from the standard input. The input chunks are
475sent to a table sink, which has been coupled with a 488sent to a table sink, which has been coupled with a
@@ -479,10 +492,10 @@ standard out:
479\begin{quote} 492\begin{quote}
480\begin{lua} 493\begin{lua}
481@stick# 494@stick#
482local in = source.file(io.stdin) 495local input = source.file(io.stdin)
483local out, t = sink.table() 496local output, t = sink.table()
484out = sink.chain(normalize("\r\n"), out) 497output = sink.chain(normalize(CRLF), output)
485pump.all(in, out) 498pump.all(input, output)
486io.write(table.concat(t)) 499io.write(table.concat(t))
487% 500%
488\end{lua} 501\end{lua}
@@ -490,11 +503,11 @@ io.write(table.concat(t))
490 503
491\subsection{Pumps} 504\subsection{Pumps}
492 505
493Adrian Sietsma noticed that, although not on purpose, our 506Although not on purpose, our interface for sources is
494interface for sources is compatible with Lua iterators. 507compatible with Lua iterators. That is, a source can be
495That is, a source can be neatly used in conjunction 508neatly used in conjunction with \texttt{for} loops. Using
496with \texttt{for} loops. Using our file 509our file source as an iterator, we can write the following
497source as an iterator, we can write the following code: 510code:
498\begin{quote} 511\begin{quote}
499\begin{lua} 512\begin{lua}
500@stick# 513@stick#
@@ -539,20 +552,22 @@ end
539The \texttt{pump.step} function moves one chunk of data from 552The \texttt{pump.step} function moves one chunk of data from
540the source to the sink. The \texttt{pump.all} function takes 553the source to the sink. The \texttt{pump.all} function takes
541an optional \texttt{step} function and uses it to pump all the 554an optional \texttt{step} function and uses it to pump all the
542data from the source to the sink. We can now use everything 555data from the source to the sink.
543we have to write a program that reads a binary file from 556Here is an example that uses the Base64 and the
557line wrapping filters from the \texttt{LuaSocket}
558distribution. The program reads a binary file from
544disk and stores it in another file, after encoding it to the 559disk and stores it in another file, after encoding it to the
545Base64 transfer content encoding: 560Base64 transfer content encoding:
546\begin{quote} 561\begin{quote}
547\begin{lua} 562\begin{lua}
548@stick# 563@stick#
549local in = source.chain( 564local input = source.chain(
550 source.file(io.open("input.bin", "rb")), 565 source.file(io.open("input.bin", "rb")),
551 encode("base64")) 566 encode("base64"))
552local out = sink.chain( 567local output = sink.chain(
553 wrap(76), 568 wrap(76),
554 sink.file(io.open("output.b64", "w"))) 569 sink.file(io.open("output.b64", "w")))
555pump.all(in, out) 570pump.all(input, output)
556% 571%
557\end{lua} 572\end{lua}
558\end{quote} 573\end{quote}
@@ -561,19 +576,17 @@ The way we split the filters here is not intuitive, on
561purpose. Alternatively, we could have chained the Base64 576purpose. Alternatively, we could have chained the Base64
562encode filter and the line-wrap filter together, and then 577encode filter and the line-wrap filter together, and then
563chain the resulting filter with either the file source or 578chain the resulting filter with either the file source or
564the file sink. It doesn't really matter. The Base64 and the 579the file sink. It doesn't really matter.
565line wrapping filters are part of the \texttt{LuaSocket}
566distribution.
567 580
568\section{Exploding filters} 581\section{Exploding filters}
569 582
570Our current filter interface has one flagrant shortcoming. 583Our current filter interface has one serious shortcoming.
571When David Burgess was writing his \texttt{gzip} filter, he 584Consider for example a \texttt{gzip} decompression filter.
572noticed that a decompression filter can explode a small 585During decompression, a small input chunk can be exploded
573input chunk into a huge amount of data. To address this 586into a huge amount of data. To address this problem, we
574problem, we decided to change the filter interface and allow 587decided to change the filter interface and allow exploding
575exploding filters to return large quantities of output data 588filters to return large quantities of output data in a chunk
576in a chunk by chunk manner. 589by chunk manner.
577 590
578More specifically, after passing each chunk of input to 591More specifically, after passing each chunk of input to
579a filter, and collecting the first chunk of output, the 592a filter, and collecting the first chunk of output, the
@@ -582,11 +595,11 @@ filtered data is left. Within these secondary calls, the
582caller passes an empty string to the filter. The filter 595caller passes an empty string to the filter. The filter
583responds with an empty string when it is ready for the next 596responds with an empty string when it is ready for the next
584input chunk. In the end, after the user passes a 597input chunk. In the end, after the user passes a
585\texttt{nil} chunk notifying the filter that there is no 598\nil\ chunk notifying the filter that there is no
586more input data, the filter might still have to produce too 599more input data, the filter might still have to produce too
587much output data to return in a single chunk. The user has 600much output data to return in a single chunk. The user has
588to loop again, now passing \texttt{nil} to the filter each time, 601to loop again, now passing \nil\ to the filter each time,
589until the filter itself returns \texttt{nil} to notify the 602until the filter itself returns \nil\ to notify the
590user it is finally done. 603user it is finally done.
591 604
592Fortunately, it is very easy to modify a filter to respect 605Fortunately, it is very easy to modify a filter to respect
@@ -599,13 +612,13 @@ Interestingly, the modifications do not have a measurable
599negative impact in the performance of filters that do 612negative impact in the performance of filters that do
600not need the added flexibility. On the other hand, for a 613not need the added flexibility. On the other hand, for a
601small price in complexity, the changes make exploding 614small price in complexity, the changes make exploding
602filters practical. 615filters practical.
603 616
604\section{A complex example} 617\section{A complex example}
605 618
606The LTN12 module in the \texttt{LuaSocket} distribution 619The LTN12 module in the \texttt{LuaSocket} distribution
607implements the ideas we have described. The MIME 620implements all the ideas we have described. The MIME
608and SMTP modules are especially integrated with LTN12, 621and SMTP modules are tightly integrated with LTN12,
609and can be used to showcase the expressive power of filters, 622and can be used to showcase the expressive power of filters,
610sources, sinks, and pumps. Below is an example 623sources, sinks, and pumps. Below is an example
611of how a user would proceed to define and send a 624of how a user would proceed to define and send a
@@ -622,9 +635,9 @@ local message = smtp.message{
622 to = "Fulano <fulano@example.com>", 635 to = "Fulano <fulano@example.com>",
623 subject = "A message with an attachment"}, 636 subject = "A message with an attachment"},
624 body = { 637 body = {
625 preamble = "Hope you can see the attachment\r\n", 638 preamble = "Hope you can see the attachment" .. CRLF,
626 [1] = { 639 [1] = {
627 body = "Here is our logo\r\n"}, 640 body = "Here is our logo" .. CRLF},
628 [2] = { 641 [2] = {
629 headers = { 642 headers = {
630 ["content-type"] = 'image/png; name="luasocket.png"', 643 ["content-type"] = 'image/png; name="luasocket.png"',
@@ -665,6 +678,18 @@ abstraction for final data destinations. Filters define an
665interface for data transformations. The chaining of 678interface for data transformations. The chaining of
666filters, sources and sinks provides an elegant way to create 679filters, sources and sinks provides an elegant way to create
667arbitrarily complex data transformations from simpler 680arbitrarily complex data transformations from simpler
668components. Pumps simply move the data through. 681components. Pumps simply push the data through.
682
683\section{Acknowledgements}
684
685The concepts described in this text are the result of long
686discussions with David Burgess. A version of this text has
687been released on-line as the Lua Technical Note 012, hence
688the name of the corresponding LuaSocket module,
689\texttt{ltn12}. Wim Couwenberg contributed to the
690implementation of the module, and Adrian Sietsma was the
691first to notice the correspondence between sources and Lua
692iterators.
693
669 694
670\end{document} 695\end{document}
diff --git a/gem/luasocket.png b/gem/luasocket.png
new file mode 100644
index 0000000..d24a954
--- /dev/null
+++ b/gem/luasocket.png
Binary files differ
diff --git a/gem/makefile b/gem/makefile
index a4287c2..d2f0c93 100644
--- a/gem/makefile
+++ b/gem/makefile
@@ -12,3 +12,12 @@ clean:
12 12
13pdf: ltn012.pdf 13pdf: ltn012.pdf
14 open ltn012.pdf 14 open ltn012.pdf
15
16test: gem.so
17
18
19gem.o: gem.c
20 gcc -c -o gem.o -Wall -ansi -W -O2 gem.c
21
22gem.so: gem.o
23 export MACOSX_DEPLOYMENT_TARGET="10.3"; gcc -bundle -undefined dynamic_lookup -o gem.so gem.o
diff --git a/gem/t1.lua b/gem/t1.lua
new file mode 100644
index 0000000..0c054c9
--- /dev/null
+++ b/gem/t1.lua
@@ -0,0 +1,25 @@
1source = {}
2sink = {}
3pump = {}
4filter = {}
5
6-- source.chain
7dofile("ex6.lua")
8
9-- source.file
10dofile("ex5.lua")
11
12-- normalize
13require"gem"
14eol = gem.eol
15dofile("ex2.lua")
16
17-- sink.file
18require"ltn12"
19sink.file = ltn12.sink.file
20
21-- pump.all
22dofile("ex10.lua")
23
24-- run test
25dofile("ex1.lua")
diff --git a/gem/t1lf.txt b/gem/t1lf.txt
new file mode 100644
index 0000000..8cddd1b
--- /dev/null
+++ b/gem/t1lf.txt
@@ -0,0 +1,5 @@
1this is a test file
2it should have been saved as lf eol
3but t1.lua will convert it to crlf eol
4otherwise it is broken!
5
diff --git a/gem/t2.lua b/gem/t2.lua
new file mode 100644
index 0000000..a81ed73
--- /dev/null
+++ b/gem/t2.lua
@@ -0,0 +1,36 @@
1source = {}
2sink = {}
3pump = {}
4filter = {}
5
6-- filter.chain
7dofile("ex3.lua")
8
9-- normalize
10require"gem"
11eol = gem.eol
12dofile("ex2.lua")
13
14-- encode
15require"mime"
16encode = mime.encode
17
18-- wrap
19wrap = mime.wrap
20
21-- source.chain
22dofile("ex6.lua")
23
24-- source.file
25dofile("ex5.lua")
26
27-- sink.file
28require"ltn12"
29sink.file = ltn12.sink.file
30
31-- pump.all
32dofile("ex10.lua")
33
34-- run test
35CRLF = "\013\010"
36dofile("ex4.lua")
diff --git a/gem/t2.txt b/gem/t2.txt
new file mode 100644
index 0000000..f484fe8
--- /dev/null
+++ b/gem/t2.txt
@@ -0,0 +1,4 @@
1esse é um texto com acentos
2quoted-printable tem que quebrar linhas longas, com mais que 76 linhas de texto
3fora que as quebras de linhas têm que ser normalizadas
4vamos ver o que dá isso aqui
diff --git a/gem/t2gt.qp b/gem/t2gt.qp
new file mode 100644
index 0000000..355a845
--- /dev/null
+++ b/gem/t2gt.qp
@@ -0,0 +1,5 @@
1esse =E9 um texto com acentos
2quoted-printable tem que quebrar linhas longas, com mais que 76 linhas de t=
3exto
4fora que as quebras de linhas t=EAm que ser normalizadas
5vamos ver o que d=E1 isso aqui
diff --git a/gem/t3.lua b/gem/t3.lua
new file mode 100644
index 0000000..4bb98ba
--- /dev/null
+++ b/gem/t3.lua
@@ -0,0 +1,25 @@
1source = {}
2sink = {}
3pump = {}
4filter = {}
5
6-- source.file
7dofile("ex5.lua")
8
9-- sink.table
10dofile("ex7.lua")
11
12-- sink.chain
13require"ltn12"
14sink.chain = ltn12.sink.chain
15
16-- normalize
17require"gem"
18eol = gem.eol
19dofile("ex2.lua")
20
21-- pump.all
22dofile("ex10.lua")
23
24-- run test
25dofile("ex8.lua")
diff --git a/gem/t4.lua b/gem/t4.lua
new file mode 100644
index 0000000..8b8071c
--- /dev/null
+++ b/gem/t4.lua
@@ -0,0 +1,10 @@
1source = {}
2sink = {}
3pump = {}
4filter = {}
5
6-- source.file
7dofile("ex5.lua")
8
9-- run test
10dofile("ex9.lua")
diff --git a/gem/t5.lua b/gem/t5.lua
new file mode 100644
index 0000000..7c569ea
--- /dev/null
+++ b/gem/t5.lua
@@ -0,0 +1,30 @@
1source = {}
2sink = {}
3pump = {}
4filter = {}
5
6-- source.chain
7dofile("ex6.lua")
8
9-- source.file
10dofile("ex5.lua")
11
12-- encode
13require"mime"
14encode = mime.encode
15
16-- sink.chain
17require"ltn12"
18sink.chain = ltn12.sink.chain
19
20-- wrap
21wrap = mime.wrap
22
23-- sink.file
24sink.file = ltn12.sink.file
25
26-- pump.all
27dofile("ex10.lua")
28
29-- run test
30dofile("ex11.lua")
diff --git a/gem/test.lua b/gem/test.lua
new file mode 100644
index 0000000..a937b9a
--- /dev/null
+++ b/gem/test.lua
@@ -0,0 +1,46 @@
1function readfile(n)
2 local f = io.open(n, "rb")
3 local s = f:read("*a")
4 f:close()
5 return s
6end
7
8lf = readfile("t1lf.txt")
9os.remove("t1crlf.txt")
10os.execute("lua t1.lua < t1lf.txt > t1crlf.txt")
11crlf = readfile("t1crlf.txt")
12assert(crlf == string.gsub(lf, "\010", "\013\010"), "broken")
13
14gt = readfile("t2gt.qp")
15os.remove("t2.qp")
16os.execute("lua t2.lua < t2.txt > t2.qp")
17t2 = readfile("t2.qp")
18assert(gt == t2, "broken")
19
20os.remove("t1crlf.txt")
21os.execute("lua t3.lua < t1lf.txt > t1crlf.txt")
22crlf = readfile("t1crlf.txt")
23assert(crlf == string.gsub(lf, "\010", "\013\010"), "broken")
24
25t = readfile("test.lua")
26os.execute("lua t4.lua < test.lua > t")
27t2 = readfile("t")
28assert(t == t2, "broken")
29
30os.remove("output.b64")
31gt = readfile("gt.b64")
32os.execute("lua t5.lua")
33t5 = readfile("output.b64")
34assert(gt == t5, "failed")
35
36print("1 2 5 6 10 passed")
37print("2 3 4 5 6 10 passed")
38print("2 5 6 7 8 10 passed")
39print("5 9 passed")
40print("5 6 10 11 passed")
41
42os.remove("t")
43os.remove("t2.qp")
44os.remove("t1crlf.txt")
45os.remove("t11.b64")
46os.remove("output.b64")