fix(receive): guard maxsize against size_t overflow on cast

Casting a double larger than SIZE_MAX to size_t is undefined
behavior; bound-check maxsize before the cast.

2 files changed, 6 insertions, 3 deletions

diff --git a/src/buffer.c b/src/buffer.c
index 05d6fb3..5f06fc1 100644
--- a/src/buffer.c
+++ b/src/buffer.c
@@ -134,7 +134,8 @@ int buffer_meth_receive(lua_State *L, p_buffer buf) {
     }
     if (!lua_isnoneornil(L, 4)) {
         double m = luaL_checknumber(L, 4);
-        luaL_argcheck(L, m >= 1, 4, "maxsize must be a positive number");
+        luaL_argcheck(L, m >= 1 && m < (lua_Number) ((size_t) -1), 4,
+            "maxsize must be a positive number");
         maxsize = (size_t) m;
         luaL_argcheck(L, size < maxsize, 4,
             "prefix length >= maxsize (drain with prefix=\"\" or raise maxsize)");
diff --git a/test/testclnt.lua b/test/testclnt.lua
index 3e897c9..ec154ea 100644
--- a/test/testclnt.lua
+++ b/test/testclnt.lua
@@ -640,14 +640,16 @@ function test_maxsize()
     assert(not ok, "A5 failed: #prefix > maxsize should raise")
     ok = pcall(data.receive, data, 100, nil, 50)
     assert(not ok, "A6 failed: wanted > maxsize should raise")
+    ok = pcall(data.receive, data, "*l", nil, math.huge)
+    assert(not ok, "A7 failed: maxsize=math.huge should raise (size_t overflow)")
     data:settimeout(0.1)
     ok = pcall(data.receive, data, 50, nil, 100)
-    assert(ok, "A7 failed: wanted <= maxsize should not raise")
+    assert(ok, "A8 failed: wanted <= maxsize should not raise")
     data:settimeout(-1)
     remote [[ data:send('intact\n') ]]
     local line, err = data:receive("*l", nil, 100)
     assert(line == "intact",
-        "A8 failed: socket touched by a failed argcheck (err=" ..
+        "A9 failed: socket touched by a failed argcheck (err=" ..
         tostring(err) .. ")")
     pass("ok")