BUG: shift overflow in utf-8 decode

An initial byte \xFF will ask for 7 continuation bytes, and then the
shift by (count * 5) will try to shift 35 bits.

1 files changed, 4 insertions, 1 deletions

diff --git a/lutf8lib.c b/lutf8lib.c
index b7f3fe1e..73f0e49b 100644
--- a/lutf8lib.c
+++ b/lutf8lib.c
@@ -56,6 +56,8 @@ static const char *utf8_decode (const char *s, l_uint32 *val, int strict) {
   l_uint32 res = 0;  /* final result */
   if (c < 0x80)  /* ASCII? */
     res = c;
+  else if (c >= 0xfe)  /* c >= 1111 1110b ? */
+    return NULL;  /* would need six or more continuation bytes */
   else {
     int count = 0;  /* to count number of continuation bytes */
     for (; c & 0x40; c <<= 1) {  /* while it needs continuation bytes... */
@@ -64,8 +66,9 @@ static const char *utf8_decode (const char *s, l_uint32 *val, int strict) {
         return NULL;  /* invalid byte sequence */
       res = (res << 6) | (cc & 0x3F);  /* add lower 6 bits from cont. byte */
     }
+    lua_assert(count <= 5);
     res |= ((l_uint32)(c & 0x7F) << (count * 5));  /* add first byte */
-    if (count > 5 || res > MAXUTF || res < limits[count])
+    if (res > MAXUTF || res < limits[count])
       return NULL;  /* invalid byte sequence */
     s += count;  /* skip continuation bytes read */
   }