Manual is almost done. HTTP is missing.

Implemented new distribution scheme.
Select is now purely C.
HTTP reimplemented seems faster dunno why.
LTN12 functions that coroutines fail gracefully.

1 files changed, 147 insertions, 204 deletions

diff --git a/src/http.lua b/src/http.lua
index ebe6b54..129b562 100644
--- a/src/http.lua
+++ b/src/http.lua
@@ -7,7 +7,7 @@
 -----------------------------------------------------------------------------
 
 -----------------------------------------------------------------------------
--- Load other required modules
+-- Load required modules
 -------------------------------------------------------------------------------
 local socket = require("socket")
 local ltn12 = require("ltn12")
@@ -17,42 +17,68 @@ local url = require("url")
 -----------------------------------------------------------------------------
 -- Setup namespace
 -------------------------------------------------------------------------------
-http = {}
--- make all module globals fall into namespace
-setmetatable(http, { __index = _G })
-setfenv(1, http)
+_LOADED["http"] = getfenv(1)
 
 -----------------------------------------------------------------------------
 -- Program constants
 -----------------------------------------------------------------------------
 -- connection timeout in seconds
-TIMEOUT = 60 
+TIMEOUT = 4 
 -- default port for document retrieval
 PORT = 80
 -- user agent field sent in request
-USERAGENT = socket.version
+USERAGENT = socket.VERSION
 -- block size used in transfers
 BLOCKSIZE = 2048
 
 -----------------------------------------------------------------------------
--- Function return value selectors
+-- Low level HTTP API
 -----------------------------------------------------------------------------
-local function second(a, b)
-    return b
+local metat = { __index = {} }
+
+function open(host, port)
+    local con = socket.try(socket.tcp()) 
+    socket.try(con:settimeout(TIMEOUT))
+    port = port or PORT
+    socket.try(con:connect(host, port))
+    return setmetatable({ con = con }, metat)
+end
+
+function metat.__index:sendrequestline(method, uri)
+    local reqline = string.format("%s %s HTTP/1.1\r\n", method or "GET", uri) 
+    return socket.try(self.con:send(reqline))
 end
 
-local function third(a, b, c)
-    return c
+function metat.__index:sendheaders(headers)
+    for i, v in pairs(headers) do
+        socket.try(self.con:send(i .. ": " .. v .. "\r\n"))
+    end
+    -- mark end of request headers
+    socket.try(self.con:send("\r\n"))
+    return 1
 end
 
-local function receive_headers(reqt, respt, tmp)
-    local sock = tmp.sock
+function metat.__index:sendbody(headers, source, step)
+    source = source or ltn12.source.empty()
+    step = step or ltn12.pump.step
+    -- if we don't know the size in advance, send chunked and hope for the best
+    local mode
+    if headers["content-length"] then mode = "keep-open"
+    else mode = "http-chunked" end
+    return socket.try(ltn12.pump.all(source, socket.sink(mode, self.con), step))
+end
+
+function metat.__index:receivestatusline()
+    local status = socket.try(self.con:receive())
+    local code = socket.skip(2, string.find(status, "HTTP/%d*%.%d* (%d%d%d)"))
+    return socket.try(tonumber(code), status)
+end
+
+function metat.__index:receiveheaders()
     local line, name, value
     local headers = {}
-    -- store results
-    respt.headers = headers
     -- get first line
-    line = socket.try(sock:receive())
+    line = socket.try(self.con:receive())
     -- headers go until a blank line is found
     while line ~= "" do
         -- get field-name and value
@@ -60,189 +86,137 @@ local function receive_headers(reqt, respt, tmp)
         socket.try(name and value, "malformed reponse headers")
         name = string.lower(name)
         -- get next line (value might be folded)
-        line  = socket.try(sock:receive())
+        line  = socket.try(self.con:receive())
         -- unfold any folded values
         while string.find(line, "^%s") do
             value = value .. line
-            line = socket.try(sock:receive())
+            line = socket.try(self.con:receive())
         end
         -- save pair in table
         if headers[name] then headers[name] = headers[name] .. ", " .. value
         else headers[name] = value end
     end
+    return headers
 end
 
-local function receive_body(reqt, respt, tmp)
-    local sink = reqt.sink or ltn12.sink.null()
-    local step = reqt.step or ltn12.pump.step
-    local source
-    local te = respt.headers["transfer-encoding"]
-    if te and te ~= "identity" then 
-        -- get by chunked transfer-coding of message body
-        source = socket.source("http-chunked", tmp.sock)
-    elseif tonumber(respt.headers["content-length"]) then
-        -- get by content-length
-        local length = tonumber(respt.headers["content-length"])
-        source = socket.source("by-length", tmp.sock, length)
-    else 
-        -- get it all until connection closes
-        source = socket.source(tmp.sock)
-    end
-    socket.try(ltn12.pump.all(source, sink, step))
+function metat.__index:receivebody(headers, sink, step)
+    sink = sink or ltn12.sink.null()
+    step = step or ltn12.pump.step
+    local length = tonumber(headers["content-length"])
+    local TE = headers["transfer-encoding"]
+    local mode
+    if TE and TE ~= "identity" then mode = "http-chunked"
+    elseif tonumber(headers["content-length"]) then mode = "by-length"
+    else mode = "default" end
+    return socket.try(ltn12.pump.all(socket.source(mode, self.con, length), 
+        sink, step))
 end
 
-local function send_headers(sock, headers)
-    -- send request headers 
-    for i, v in pairs(headers) do
-        socket.try(sock:send(i .. ": " .. v .. "\r\n"))
-    end
-    -- mark end of request headers
-    socket.try(sock:send("\r\n"))
+function metat.__index:close()
+    return self.con:close()
 end
 
-local function should_receive_body(reqt, respt, tmp)
-    if reqt.method == "HEAD" then return nil end
-    if respt.code == 204 or respt.code == 304 then return nil end
-    if respt.code >= 100 and respt.code < 200 then return nil end
-    return 1
-end
-
-local function receive_status(reqt, respt, tmp)
-    local status = socket.try(tmp.sock:receive())
-    local code = third(string.find(status, "HTTP/%d*%.%d* (%d%d%d)"))
-    -- store results
-    respt.code, respt.status = tonumber(code), status
-end
-
-local function request_uri(reqt, respt, tmp)
-    local u = tmp.parsed
-    if not reqt.proxy then
-        local parsed = tmp.parsed
-        u = { 
-           path = parsed.path, 
-           params = parsed.params, 
-           query = parsed.query, 
-           fragment = parsed.fragment
+-----------------------------------------------------------------------------
+-- High level HTTP API
+-----------------------------------------------------------------------------
+local function uri(reqt)
+    local u = reqt
+    if not reqt.proxy and not PROXY then
+        u = {
+           path = reqt.path,
+           params = reqt.params,
+           query = reqt.query,
+           fragment = reqt.fragment
         }
     end
     return url.build(u)
 end
 
-local function send_request(reqt, respt, tmp)
-    local uri = request_uri(reqt, respt, tmp)
-    local headers = tmp.headers
-    local step = reqt.step or ltn12.pump.step
-    -- send request line
-    socket.try(tmp.sock:send((reqt.method or "GET") 
-        .. " " .. uri .. " HTTP/1.1\r\n"))
-    if reqt.source and not headers["content-length"] then
-        headers["transfer-encoding"] = "chunked"   
-    end
-    send_headers(tmp.sock, headers)
-    -- send request message body, if any
-    if not reqt.source then return end
-    if headers["content-length"] then 
-        socket.try(ltn12.pump.all(reqt.source, 
-            socket.sink(tmp.sock), step))
-    else 
-        socket.try(ltn12.pump.all(reqt.source, 
-            socket.sink("http-chunked", tmp.sock), step))
-    end
-end
-
-local function open(reqt, respt, tmp)
-    local proxy = reqt.proxy or PROXY
-    local host, port
-    if proxy then 
-        local pproxy = url.parse(proxy) 
-        socket.try(pproxy.port and pproxy.host, "invalid proxy")
-        host, port = pproxy.host, pproxy.port
-    else 
-        host, port = tmp.parsed.host, tmp.parsed.port 
-    end
-    -- store results
-    tmp.sock = socket.try(socket.tcp())
-    socket.try(tmp.sock:settimeout(reqt.timeout or TIMEOUT))
-    socket.try(tmp.sock:connect(host, port))
-end
-
-local function adjust_headers(reqt, respt, tmp)
+local function adjustheaders(headers, host)
     local lower = {}
     -- override with user values
-    for i,v in (reqt.headers or lower) do
+    for i,v in (headers or lower) do
         lower[string.lower(i)] = v
     end
     lower["user-agent"] = lower["user-agent"] or USERAGENT
     -- these cannot be overriden
-    lower["host"] = tmp.parsed.host
-    lower["connection"] = "close"
-    -- store results
-    tmp.headers = lower
+    lower["host"] = host
+    return lower
 end
 
-local function parse_url(reqt, respt, tmp)
+local function adjustrequest(reqt)
     -- parse url with default fields
     local parsed = url.parse(reqt.url, {
         host = "",
-        port = PORT, 
+        port = PORT,
         path ="/",
-                scheme = "http"
+        scheme = "http"
     })
-    -- scheme has to be http
-        socket.try(parsed.scheme == "http", 
-        string.format("unknown scheme '%s'", parsed.scheme))
-    -- explicit authentication info overrides that given by the URL
-    parsed.user = reqt.user or parsed.user
-    parsed.password = reqt.password or parsed.password
-    -- store results
-    tmp.parsed = parsed
+    -- explicit info in reqt overrides that given by the URL
+    for i,v in reqt do parsed[i] = v end 
+    -- compute uri if user hasn't overriden
+    parsed.uri = parsed.uri or uri(parsed)
+    -- adjust headers in request
+    parsed.headers = adjustheaders(parsed.headers, parsed.host)
+    return parsed
 end
 
--- forward declaration
-local request_p
+local function shouldredirect(reqt, respt)
+    return (reqt.redirect ~= false) and
+           (respt.code == 301 or respt.code == 302) and
+           (not reqt.method or reqt.method == "GET" or reqt.method == "HEAD")
+           and (not reqt.nredirects or reqt.nredirects < 5)
+end
 
-local function should_authorize(reqt, respt, tmp)
+local function shouldauthorize(reqt, respt)
     -- if there has been an authorization attempt, it must have failed
     if reqt.headers and reqt.headers["authorization"] then return nil end
     -- if last attempt didn't fail due to lack of authentication,
     -- or we don't have authorization information, we can't retry
-    return respt.code == 401 and tmp.parsed.user and tmp.parsed.password 
+    return respt.code == 401 and reqt.user and reqt.password
 end
 
-local function clone(headers)
-    if not headers then return nil end
-    local copy = {}
-    for i,v in pairs(headers) do
-        copy[i] = v
-    end
-    return copy
+local function shouldreceivebody(reqt, respt)
+    if reqt.method == "HEAD" then return nil end
+    local code = respt.code
+    if code == 204 or code == 304 then return nil end
+    if code >= 100 and code < 200 then return nil end
+    return 1
 end
 
-local function authorize(reqt, respt, tmp)
-    local headers = clone(reqt.headers) or {}
-    headers["authorization"] = "Basic " ..
-        (mime.b64(tmp.parsed.user .. ":" .. tmp.parsed.password))
-    local autht = {
-        method = reqt.method,
-        url = reqt.url,
-        source = reqt.source,
-        sink = reqt.sink,
-        headers = headers,
-        timeout = reqt.timeout,
-        proxy = reqt.proxy,
-    }
-    request_p(autht, respt, tmp)
+local requestp, authorizep, redirectp
+
+function requestp(reqt)
+    local reqt = adjustrequest(reqt)
+    local respt = {}
+    local con = open(reqt.host, reqt.port)
+    con:sendrequestline(reqt.method, reqt.uri)
+    con:sendheaders(reqt.headers)
+    con:sendbody(reqt.headers, reqt.source, reqt.step)
+    respt.code, respt.status = con:receivestatusline()
+    respt.headers = con:receiveheaders()
+    if shouldredirect(reqt, respt) then 
+        con:close()
+        return redirectp(reqt, respt)
+    elseif shouldauthorize(reqt, respt) then 
+        con:close()
+        return authorizep(reqt, respt)
+    elseif shouldreceivebody(reqt, respt) then
+        con:receivebody(respt.headers, reqt.sink, reqt.step)
+    end
+    con:close()
+    return respt
 end
 
-local function should_redirect(reqt, respt, tmp)
-    return (reqt.redirect ~= false) and
-           (respt.code == 301 or respt.code == 302) and
-           (not reqt.method or reqt.method == "GET" or reqt.method == "HEAD") 
-           and (not tmp.nredirects or tmp.nredirects < 5)
+function authorizep(reqt, respt)
+    local auth = "Basic " ..  (mime.b64(reqt.user .. ":" .. reqt.password))
+    reqt.headers["authorization"] = auth
+    return requestp(reqt)
 end
 
-local function redirect(reqt, respt, tmp)
-    tmp.nredirects = (tmp.nredirects or 0) + 1
+function redirectp(reqt, respt)
+    -- we create a new table to get rid of anything we don't 
+    -- absolutely need, including authentication info
     local redirt = {
         method = reqt.method,
         -- the RFC says the redirect URL has to be absolute, but some
@@ -251,69 +225,38 @@ local function redirect(reqt, respt, tmp)
         source = reqt.source,
         sink = reqt.sink,
         headers = reqt.headers,
-        timeout = reqt.timeout,
-        proxy = reqt.proxy
+        proxy = reqt.proxy,
+        nredirects = (reqt.nredirects or 0) + 1
     }
-    request_p(redirt, respt, tmp)
+    respt = requestp(redirt)
     -- we pass the location header as a clue we redirected
     if respt.headers then respt.headers.location = redirt.url end
-end
-
-local function skip_continue(reqt, respt, tmp)
-    if respt.code == 100 then
-        receive_status(reqt, respt, tmp)
-    end
-end
-
--- execute a request of through an exception
-function request_p(reqt, respt, tmp)
-    parse_url(reqt, respt, tmp)
-    adjust_headers(reqt, respt, tmp)
-    open(reqt, respt, tmp)
-    send_request(reqt, respt, tmp)
-    receive_status(reqt, respt, tmp)
-    skip_continue(reqt, respt, tmp)
-    receive_headers(reqt, respt, tmp)
-    if should_redirect(reqt, respt, tmp) then
-        tmp.sock:close()
-        redirect(reqt, respt, tmp)
-    elseif should_authorize(reqt, respt, tmp) then
-        tmp.sock:close()
-        authorize(reqt, respt, tmp)
-    elseif should_receive_body(reqt, respt, tmp) then
-        receive_body(reqt, respt, tmp)
-    end
-end
-
-function request(reqt)
-    local respt, tmp = {}, {}
-    local s, e = pcall(request_p, reqt, respt, tmp)
-    if not s then respt.error = e end
-    if tmp.sock then tmp.sock:close() end
     return respt
 end
 
-function get(u)
+request = socket.protect(requestp)
+
+get = socket.protect(function(u)
     local t = {}
-    respt = request { 
-        url = u, 
-        sink = ltn12.sink.table(t) 
+    local respt = requestp {
+        url = u,
+        sink = ltn12.sink.table(t)
     }
-    return (table.getn(t) > 0 or nil) and table.concat(t), respt.headers, 
-        respt.code, respt.error
-end
+    return (table.getn(t) > 0 or nil) and table.concat(t), respt.headers,
+        respt.code
+end)
 
-function post(u, body)
+post = socket.protect(function(u, body)
     local t = {}
-    respt = request { 
-        url = u, 
-        method = "POST", 
+    local respt = requestp {
+        url = u,
+        method = "POST",
         source = ltn12.source.string(body),
         sink = ltn12.sink.table(t),
-        headers = { ["content-length"] = string.len(body) } 
+        headers = { ["content-length"] = string.len(body) }
     }
-    return (table.getn(t) > 0 or nil) and table.concat(t), 
-        respt.headers, respt.code, respt.error
-end
+    return (table.getn(t) > 0 or nil) and table.concat(t),
+        respt.headers, respt.code
+end)
 
 return http