From 73b0780321acb14bab6707006f5136ea9998951d Mon Sep 17 00:00:00 2001 From: Thijs Schreijer Date: Thu, 30 Jul 2026 19:22:17 +0200 Subject: feat(receive): add maxsize argument to bound memory usage client:receive("*l") and receive("*a") are unbounded: a peer that never sends a newline, or never closes, makes LuaSocket buffer until the process runs out of memory (e.g. src/http.lua reading response headers in a loop). Add an optional maxsize argument that caps the payload a single call may accumulate, including prefix. - Hoist all argument validation ahead of timeout_markstart() so bad calls (maxsize < 1, #prefix >= maxsize, numeric pattern > maxsize) raise before any I/O and leave the socket untouched. - recvline/recvall take a budget and return a new internal BUF_OVERSIZED code, surfaced to Lua as the "oversized" error alongside "timeout"/"closed", with the partial held in the 3rd return value. - recvraw is left untouched: argument checks make the cap unreachable for numeric patterns. - Preserve three invariants: a timeout partial is always shorter than maxsize (safe to retry as prefix), completion beats the cap for *a, and no bytes are lost or skipped on overflow. - tcp.c, unixstream.c and serial.c all share this code path unchanged. Adds test coverage (argument errors, *l/*a boundaries, timeout/close at the cap, the drain idiom, numeric patterns, unix-stream mirror) and documents the new argument, error, and recovery idioms in docs/tcp.html. --- CHANGELOG.md | 4 ++ docs/tcp.html | 58 +++++++++++++++++- src/buffer.c | 108 +++++++++++++++++++++++++++------ test/testclnt.lua | 174 +++++++++++++++++++++++++++++++++++++++++++++++++++++ test/utestclnt.lua | 45 ++++++++++++++ 5 files changed, 366 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3a25186..03947c3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## Unreleased + +* Add `maxsize` argument to `receive` to bound the memory a single call may accumulate, returning `"oversized"` instead of growing without limit – @Tieske + ## [v3.1.0](https://github.com/lunarmodules/luasocket/releases/v3.1.0) — 2022-07-27 * Add support for TCP Defer Accept – @Zash diff --git a/docs/tcp.html b/docs/tcp.html index a26228d..4c6c6bc 100644 --- a/docs/tcp.html +++ b/docs/tcp.html @@ -351,7 +351,7 @@ method returns nil followed by an error message.

-client:receive([pattern [, prefix]]) +client:receive([pattern [, prefix [, maxsize]]])

@@ -380,14 +380,23 @@ of bytes from the socket. of any received data before return.

+

+Maxsize is an optional positive integer bounding the number of +payload bytes the call may accumulate, including prefix. +Omitted or nil means unlimited. +

+

If successful, the method returns the received pattern. In case of error, the method returns nil followed by an error message, followed by a (possibly empty) string containing the partial that was received. The error message can be the string 'closed' in case the connection was -closed before the transmission was completed or the string -'timeout' in case there was a timeout during the operation. +closed before the transmission was completed, the string +'timeout' in case there was a timeout during the operation, or, +when maxsize was given, the string 'oversized' in case +the pattern did not complete within maxsize bytes -- in which case +the third return value holds exactly maxsize bytes.

@@ -399,6 +408,49 @@ functions should return nil on error. Thus it was changed too.

+

+Note on maxsize: passing a maxsize that is smaller +than 1, a prefix whose length is greater than or equal to +maxsize, or, for a numeric pattern, a byte count greater +than maxsize, all raise a Lua error rather than returning +nil plus a message -- these are caller logic errors, and +they are detected before any byte is read from the socket. To drain and +discard an oversized line while keeping memory bounded and the stream +aligned: +

+ +
+local data, err, part
+repeat
+    data, err, part = client:receive("*l", "", 4096)
+until err ~= "oversized"
+
+ +

+To instead retry and eventually get the whole thing, carry the partial +forward as prefix and grow maxsize: +

+ +
+local data, err, part = client:receive("*l", nil, 4096)
+if err == "oversized" then
+    data, err, part = client:receive("*l", part, 65536)   -- larger cap, or this raises
+end
+
+ +

+Retrying with prefix set to the previous partial result and an +unchanged maxsize raises the length-check error above by +design -- otherwise it would be a zero-progress spin: no I/O, no timeout, +no error, just CPU. A timeout partial is always strictly shorter +than maxsize, so it is always safe to feed straight back as +prefix with the same maxsize. Finally, note that +maxsize bounds the payload returned, not necessarily the +bytes taken off the wire: for the *l pattern the discarded CR +characters and the line terminator mean more bytes may have been consumed +than the returned length suggests. +

+

diff --git a/src/buffer.c b/src/buffer.c index 7148be3..05d6fb3 100644 --- a/src/buffer.c +++ b/src/buffer.c @@ -9,12 +9,24 @@ * Internal function prototypes \*=========================================================================*/ static int recvraw(p_buffer buf, size_t wanted, luaL_Buffer *b); -static int recvline(p_buffer buf, luaL_Buffer *b); -static int recvall(p_buffer buf, luaL_Buffer *b); +static int recvline(p_buffer buf, luaL_Buffer *b, size_t budget); +static int recvall(p_buffer buf, luaL_Buffer *b, size_t budget); static int buffer_get(p_buffer buf, const char **data, size_t *count); static void buffer_skip(p_buffer buf, size_t count); static int sendraw(p_buffer buf, const char *data, size_t count, size_t *sent); +/* Internal completion code for buffer_meth_receive. err is not confined to + * the IO_* enum: socket_recv/socket_send (usocket.c/wsocket.c) propagate raw + * platform errors (POSIX errno, Windows WSA codes) straight through, and + * those are always positive, so a positive sentinel here could collide with + * a genuine transport error (e.g. errno 1 == EPERM) and get misreported as + * "oversized". Chosen negative and outside {IO_DONE, IO_TIMEOUT, IO_CLOSED, + * IO_UNKNOWN} (0, -1, -2, -3) so it can never collide with anything err + * legitimately takes. + * MUST be handled before buf->io->error() is called -- it is not a transport + * error. */ +#define BUF_OVERSIZED (-1000) + /* min and max macros */ #ifndef MIN #define MIN(x, y) ((x) < (y) ? x : y) @@ -105,8 +117,33 @@ int buffer_meth_send(lua_State *L, p_buffer buf) { int buffer_meth_receive(lua_State *L, p_buffer buf) { int err = IO_DONE, top; luaL_Buffer b; - size_t size; + size_t size, wanted = 0, maxsize = 0; + size_t budget = 0; /* 0 == unlimited */ + int numeric = lua_isnumber(L, 2); const char *part = luaL_optlstring(L, 3, "", &size); + + /* ---- validation: must precede timeout_markstart() and any I/O ---- */ + if (numeric) { + double n = lua_tonumber(L, 2); + luaL_argcheck(L, n >= 0, 2, "invalid receive pattern"); + wanted = (size_t) n; + } else { + const char *p = luaL_optstring(L, 2, "*l"); + luaL_argcheck(L, p[0] == '*' && (p[1] == 'l' || p[1] == 'a'), + 2, "invalid receive pattern"); + } + if (!lua_isnoneornil(L, 4)) { + double m = luaL_checknumber(L, 4); + luaL_argcheck(L, m >= 1, 4, "maxsize must be a positive number"); + maxsize = (size_t) m; + luaL_argcheck(L, size < maxsize, 4, + "prefix length >= maxsize (drain with prefix=\"\" or raise maxsize)"); + if (numeric) + luaL_argcheck(L, wanted <= maxsize, 4, + "maxsize smaller than requested byte count"); + budget = maxsize - size; + } + timeout_markstart(buf->tm); /* make sure we don't confuse buffer stuff with arguments */ lua_settop(L, 3); @@ -116,24 +153,28 @@ int buffer_meth_receive(lua_State *L, p_buffer buf) { luaL_buffinit(L, &b); luaL_addlstring(&b, part, size); /* receive new patterns */ - if (!lua_isnumber(L, 2)) { + if (!numeric) { const char *p= luaL_optstring(L, 2, "*l"); - if (p[0] == '*' && p[1] == 'l') err = recvline(buf, &b); - else if (p[0] == '*' && p[1] == 'a') err = recvall(buf, &b); - else luaL_argcheck(L, 0, 2, "invalid receive pattern"); + if (p[0] == '*' && p[1] == 'l') err = recvline(buf, &b, budget); + else err = recvall(buf, &b, budget); /* get a fixed number of bytes (minus what was already partially * received) */ } else { - double n = lua_tonumber(L, 2); - size_t wanted = (size_t) n; - luaL_argcheck(L, n >= 0, 2, "invalid receive pattern"); if (size == 0 || wanted > size) err = recvraw(buf, wanted-size, &b); } /* check if there was an error */ - if (err != IO_DONE) { - /* we can't push anyting in the stack before pushing the - * contents of the buffer. this is the reason for the complication */ + /* luaL_pushresult(&b) must come first (its accumulator lives on the + * stack), but the partial it produces belongs in slot 3, not 1 -- so + * both error branches push buffer/error/buffer-copy/nil, then + * lua_replace the nil into slot 1. */ + if (err == BUF_OVERSIZED) { + luaL_pushresult(&b); + lua_pushliteral(L, "oversized"); + lua_pushvalue(L, -2); + lua_pushnil(L); + lua_replace(L, -4); + } else if (err != IO_DONE) { luaL_pushresult(&b); lua_pushstring(L, buf->io->error(buf->io->ctx, err)); lua_pushvalue(L, -2); @@ -201,36 +242,61 @@ static int recvraw(p_buffer buf, size_t wanted, luaL_Buffer *b) { /*-------------------------------------------------------------------------*\ * Reads everything until the connection is closed (buffered) +* budget == 0 means unlimited; otherwise the number of payload bytes still +* allowed. Completion (connection closed) beats the cap: filling the cap +* exactly and then seeing EOF means the whole stream was received. \*-------------------------------------------------------------------------*/ -static int recvall(p_buffer buf, luaL_Buffer *b) { +static int recvall(p_buffer buf, luaL_Buffer *b, size_t budget) { int err = IO_DONE; size_t total = 0; while (err == IO_DONE) { const char *data; size_t count; err = buffer_get(buf, &data, &count); + if (budget && count > budget - total) { /* strictly more than fits */ + count = budget - total; + luaL_addlstring(b, data, count); + buffer_skip(buf, count); + return BUF_OVERSIZED; + } total += count; luaL_addlstring(b, data, count); buffer_skip(buf, count); } - if (err == IO_CLOSED) { + if (err == IO_CLOSED) { /* completion beats the cap */ if (total > 0) return IO_DONE; else return IO_CLOSED; - } else return err; + } + if (budget && total == budget) return BUF_OVERSIZED; + return err; } /*-------------------------------------------------------------------------*\ * Reads a line terminated by a CR LF pair or just by a LF. The CR and LF * are not returned by the function and are discarded from the buffer +* budget == 0 means unlimited; otherwise the number of payload bytes still +* allowed. The cap test sits before consuming a byte, so a line of exactly +* budget payload bytes succeeds while budget+1 reports oversized. A timeout +* or close with the payload exactly at the cap and no terminator yet also +* resolves to oversized, never to timeout/closed. \*-------------------------------------------------------------------------*/ -static int recvline(p_buffer buf, luaL_Buffer *b) { +static int recvline(p_buffer buf, luaL_Buffer *b, size_t budget) { int err = IO_DONE; + size_t total = 0; while (err == IO_DONE) { size_t count, pos; const char *data; err = buffer_get(buf, &data, &count); pos = 0; while (pos < count && data[pos] != '\n') { - /* we ignore all \r's */ - if (data[pos] != '\r') luaL_addchar(b, data[pos]); + /* we ignore all \r's -- they are consumed but never counted */ + if (data[pos] != '\r') { + if (budget && total == budget) { + /* leave the offending byte in the buffer for the next call */ + buffer_skip(buf, pos); + return BUF_OVERSIZED; + } + luaL_addchar(b, data[pos]); + total++; + } pos++; } if (pos < count) { /* found '\n' */ @@ -239,7 +305,9 @@ static int recvline(p_buffer buf, luaL_Buffer *b) { } else /* reached the end of the buffer */ buffer_skip(buf, pos); } - return err; + if (err == IO_DONE) return IO_DONE; /* '\n' found: success, regardless of total */ + if (budget && total == budget) return BUF_OVERSIZED; /* stalled/closed exactly at the cap: I1 */ + return err; /* real timeout/closed, below the cap */ } /*-------------------------------------------------------------------------*\ diff --git a/test/testclnt.lua b/test/testclnt.lua index 170e187..3e897c9 100644 --- a/test/testclnt.lua +++ b/test/testclnt.lua @@ -618,6 +618,177 @@ remote([[ pass("ok") end +------------------------------------------------------------------------ +function test_maxsize() + -- Group A: argument errors, raised before any I/O + reconnect() + printf("argument errors: ") + -- bounds pre-implementation calls (where arg 4 is silently dropped and + -- these become real, otherwise-unbounded blocking reads on an idle + -- socket) so a meaningful failure doesn't hang the suite + data:settimeout(0.2) + local ok + ok = pcall(data.receive, data, "*l", nil, 0) + assert(not ok, "A1 failed: maxsize=0 should raise") + ok = pcall(data.receive, data, "*l", nil, -1) + assert(not ok, "A2 failed: maxsize=-1 should raise") + ok = pcall(data.receive, data, "*l", nil, "abc") + assert(not ok, "A3 failed: non-number maxsize should raise") + ok = pcall(data.receive, data, "*l", string.rep("x", 10), 10) + assert(not ok, "A4 failed: #prefix == maxsize should raise") + ok = pcall(data.receive, data, "*l", string.rep("x", 11), 10) + assert(not ok, "A5 failed: #prefix > maxsize should raise") + ok = pcall(data.receive, data, 100, nil, 50) + assert(not ok, "A6 failed: wanted > maxsize should raise") + data:settimeout(0.1) + ok = pcall(data.receive, data, 50, nil, 100) + assert(ok, "A7 failed: wanted <= maxsize should not raise") + data:settimeout(-1) + remote [[ data:send('intact\n') ]] + local line, err = data:receive("*l", nil, 100) + assert(line == "intact", + "A8 failed: socket touched by a failed argcheck (err=" .. + tostring(err) .. ")") + pass("ok") + + -- Group B: *l boundary + reconnect() + printf("*l boundary: ") + remote [[ data:send('hello\n') ]] + local d, e, p = data:receive("*l", nil, 100) + assert(d == "hello" and e == nil, "B1 failed") + + remote(string.format([[data:send(string.rep('a',%d) .. '\n')]], 100)) + d, e = data:receive("*l", nil, 100) + assert(d == string.rep("a", 100) and e == nil, + "B2 failed: exact-budget line should succeed") + + remote(string.format([[data:send(string.rep('a',%d) .. '\n')]], 101)) + d, e, p = data:receive("*l", nil, 100) + assert(d == nil and e == "oversized" and p == string.rep("a", 100), + "B3 failed: one-over-budget should be oversized") + d, e = data:receive("*l", nil, 100) + assert(d == "a" and e == nil, + "B3 failed: leftover byte and terminator should still be there") + + remote(string.format([[data:send(string.rep('\r',%d) .. string.rep('a',%d) .. '\n')]], 50, 100)) + d, e = data:receive("*l", nil, 100) + assert(d == string.rep("a", 100) and e == nil, "B4 failed: CRs must not count") + + remote(string.format([[data:send(string.rep('a',%d) .. '\r\n')]], 100)) + d, e = data:receive("*l", nil, 100) + assert(d == string.rep("a", 100) and e == nil, "B5 failed: CRLF at boundary") + pass("ok") + + -- Group C: *l and I1 (timeout/close at the cap) + reconnect() + printf("I1 (timeout/closed at the cap): ") + remote(string.format([[data:send(string.rep('a',%d))]], 50)) + data:settimeout(0.5) + d, e, p = data:receive("*l", nil, 100) + assert(d == nil and e == "timeout", "C1 failed: expected timeout below cap") + assert(#p < 100, "C1 failed: timeout partial must be strictly < maxsize") + ok = pcall(data.receive, data, "*l", p, 100) + assert(ok, "C1 failed: retry with prefix=partial, same maxsize must not raise") + + reconnect() + remote(string.format([[data:send(string.rep('a',%d))]], 100)) + data:settimeout(0.5) + d, e, p = data:receive("*l", nil, 100) + assert(e == "oversized" and #p == 100, + "C2 failed: exactly-at-cap timeout must be oversized, got " .. tostring(e)) + + reconnect() + remote(string.format([[data:send(string.rep('a',%d)) data:close() data = nil]], 100)) + d, e, p = data:receive("*l", nil, 100) + assert(e == "oversized" and #p == 100, + "C3 failed: exactly-at-cap close must be oversized, got " .. tostring(e)) + pass("ok") + + -- Group D: drain idiom + -- 5030 (not a multiple of the 100 cap) so the boundary of the 50th + -- oversized chunk doesn't land exactly on the '\n': when it does, the + -- terminator check wins over the cap check (by design -- see B2/I1) and + -- the would-be 50th oversized chunk instead succeeds outright. + reconnect() + printf("drain idiom: ") + remote(string.format([[data:send(string.rep('x',%d) .. '\n' .. 'next\n')]], 5030)) + local iterations = 0 + repeat + d, e, p = data:receive("*l", "", 100) + if e == "oversized" then + assert(#p == 100, "D1 failed: oversized partial length " .. #p) + iterations = iterations + 1 + end + until e ~= "oversized" + assert(iterations == 50, + "D1 failed: expected 50 oversized iterations, got " .. iterations) + assert(d == string.rep("x", 30) and e == nil, "D1 failed: final call should succeed") + local nextline = data:receive("*l") + assert(nextline == "next", "D1 failed: stream misaligned, got " .. tostring(nextline)) + pass("ok") + + -- Group E: *a + reconnect() + printf("*a boundary: ") + remote [[ data:send('abc') data:close() data = nil ]] + d, e = data:receive("*a", nil, 100) + assert(d == "abc" and e == nil, "E1 failed") + + reconnect() + remote(string.format([[data:send(string.rep('a',%d)) data:close() data = nil]], 100)) + d, e = data:receive("*a", nil, 100) + assert(d == string.rep("a", 100) and e == nil, + "E2 failed: completion should beat the cap") + + reconnect() + remote(string.format([[data:send(string.rep('a',%d)) data:close() data = nil]], 250)) + d, e, p = data:receive("*a", "", 100) + assert(e == "oversized" and #p == 100, "E3 failed: first chunk") + d, e, p = data:receive("*a", "", 100) + assert(e == "oversized" and #p == 100, "E3 failed: second chunk") + d, e = data:receive("*a", "", 100) + assert(d == string.rep("a", 50) and e == nil, "E3 failed: final chunk") + + reconnect() + remote(string.format([[data:send(string.rep('a',%d))]], 100)) + data:settimeout(0.3) + d, e, p = data:receive("*a", nil, 100) + assert(e == "oversized" and #p == 100, + "E4 failed: expected oversized not timeout, got " .. tostring(e)) + + reconnect() + remote(string.format([[data:send(string.rep('a',%d))]], 50)) + data:settimeout(0.3) + d, e, p = data:receive("*a", nil, 100) + assert(e == "timeout" and #p < 100, "E5 failed") + pass("ok") + + -- Group F: numeric pattern (recvraw untouched) + reconnect() + printf("numeric pattern (recvraw unchanged): ") + remote(string.format([[data:send(string.rep('a',%d))]], 50)) + d, e = data:receive(50, nil, 100) + assert(d == string.rep("a", 50) and e == nil, "F1 failed") + + reconnect() + remote(string.format([[data:send(string.rep('a',%d))]], 25)) + d, e = data:receive(50, string.rep("p", 25), 100) + assert(e == nil and #d == 50, "F2 failed") + pass("ok") + + -- Group G: no-maxsize regression snapshot + reconnect() + printf("no-maxsize regression: ") + remote(string.format([[data:send(string.rep('a',%d) .. '\n')]], 5000)) + d, e = data:receive("*l") + assert(d == string.rep("a", 5000) and e == nil, "G2 failed: plain *l") + remote(string.format([[data:send(string.rep('a',%d) .. '\n')]], 5000)) + d, e = data:receive("*l", "") + assert(d == string.rep("a", 5000) and e == nil, "G2 failed: *l with prefix") + pass("ok") +end + ------------------------------------------------------------------------ test("method registration") @@ -796,6 +967,9 @@ test_blockingtimeoutreceive(800091, 2, 3) test_blockingtimeoutreceive(800091, 3, 2) test_blockingtimeoutreceive(800091, 3, 1) +test("receive maxsize") +test_maxsize() + test("shutting server down") reconnect() remote("os.exit()") diff --git a/test/utestclnt.lua b/test/utestclnt.lua index 7f10643..395260c 100644 --- a/test/utestclnt.lua +++ b/test/utestclnt.lua @@ -510,6 +510,48 @@ remote(string.format([[ print("ok") end +------------------------------------------------------------------------ +function test_maxsize() + -- A4: #prefix == maxsize raises (mirrors testclnt.lua group A) + reconnect() + pass("argument errors") + -- bounds the pre-implementation call (arg 4 silently dropped, so this + -- becomes a real blocking read on an idle socket) so a meaningful + -- failure doesn't hang the suite + data:settimeout(0.2) + local ok = pcall(data.receive, data, "*l", string.rep("x", 10), 10) + assert(not ok, "A4 failed: #prefix == maxsize should raise") + data:settimeout(-1) + + -- B2/B3: *l boundary (mirrors testclnt.lua group B) + pass("*l boundary") + remote(string.format([[data:send(string.rep('a',%d) .. '\n')]], 100)) + local d, e, p = data:receive("*l", nil, 100) + assert(d == string.rep("a", 100) and e == nil, "B2 failed") + + remote(string.format([[data:send(string.rep('a',%d) .. '\n')]], 101)) + d, e, p = data:receive("*l", nil, 100) + assert(d == nil and e == "oversized" and p == string.rep("a", 100), "B3 failed") + d, e = data:receive("*l", nil, 100) + assert(d == "a" and e == nil, "B3 failed: leftover byte") + + -- C2: timeout exactly at the cap is oversized, not timeout (I1) + reconnect() + remote(string.format([[data:send(string.rep('a',%d))]], 100)) + data:settimeout(0.5) + d, e, p = data:receive("*l", nil, 100) + assert(e == "oversized" and #p == 100, + "C2 failed: expected oversized, got " .. tostring(e)) + + -- E2: completion beats the cap (I2) + reconnect() + remote(string.format([[data:send(string.rep('a',%d)) data:close() data = nil]], 100)) + d, e = data:receive("*a", nil, 100) + assert(d == string.rep("a", 100) and e == nil, + "E2 failed: completion should beat the cap") + pass("ok") +end + ------------------------------------------------------------------------ test("method registration") @@ -641,4 +683,7 @@ test_blockingtimeoutreceive(800091, 2, 3) test_blockingtimeoutreceive(800091, 3, 2) test_blockingtimeoutreceive(800091, 3, 1) +test("receive maxsize") +test_maxsize() + test(string.format("done in %.2fs", socket.gettime() - start)) -- cgit v1.2.3-55-g6feb