Better handling of size limit when resizing a table

Avoid silent conversions from int to unsigned int when calling
'luaH_resize'; avoid silent conversions from lua_Integer to int in
'table.create'; MAXASIZE corrected for the new implementation of arrays;
'luaH_resize' checks explicitly whether new size respects MAXASIZE.
(Even constructors were bypassing that check.)

1 files changed, 13 insertions, 4 deletions

diff --git a/ltable.c b/ltable.c
index 353e567b..b86e2281 100644
--- a/ltable.c
+++ b/ltable.c
@@ -61,18 +61,25 @@ typedef union {
 
 
 /*
-** MAXABITS is the largest integer such that MAXASIZE fits in an
+** MAXABITS is the largest integer such that 2^MAXABITS fits in an
 ** unsigned int.
 */
 #define MAXABITS        cast_int(sizeof(int) * CHAR_BIT - 1)
 
 
 /*
+** MAXASIZEB is the maximum number of elements in the array part such
+** that the size of the array fits in 'size_t'.
+*/
+#define MAXASIZEB       ((MAX_SIZET/sizeof(ArrayCell)) * NM)
+
+
+/*
 ** MAXASIZE is the maximum size of the array part. It is the minimum
-** between 2^MAXABITS and the maximum size that, measured in bytes,
-** fits in a 'size_t'.
+** between 2^MAXABITS and MAXASIZEB.
 */
-#define MAXASIZE        luaM_limitN(1u << MAXABITS, TValue)
+#define MAXASIZE  \
+    (((1u << MAXABITS) < MAXASIZEB) ? (1u << MAXABITS) : cast_uint(MAXASIZEB))
 
 /*
 ** MAXHBITS is the largest integer such that 2^MAXHBITS fits in a
@@ -663,6 +670,8 @@ void luaH_resize (lua_State *L, Table *t, unsigned int newasize,
   Table newt;  /* to keep the new hash part */
   unsigned int oldasize = setlimittosize(t);
   ArrayCell *newarray;
+  if (newasize > MAXASIZE)
+    luaG_runerror(L, "table overflow");
   /* create new hash part with appropriate size into 'newt' */
   newt.flags = 0;
   setnodevector(L, &newt, nhsize);