diff options
author | Roberto Ierusalimschy <roberto@inf.puc-rio.br> | 2019-03-15 13:14:17 -0300 |
---|---|---|
committer | Roberto Ierusalimschy <roberto@inf.puc-rio.br> | 2019-03-15 13:14:17 -0300 |
commit | 1e0c73d5b643707335b06abd2546a83d9439d14c (patch) | |
tree | b80b7d5e2cfeeef888ddf98fcc6276832134c1bf /lutf8lib.c | |
parent | 8fa4f1380b9a203bfdf002c2e9e9e13ebb8384c1 (diff) | |
download | lua-1e0c73d5b643707335b06abd2546a83d9439d14c.tar.gz lua-1e0c73d5b643707335b06abd2546a83d9439d14c.tar.bz2 lua-1e0c73d5b643707335b06abd2546a83d9439d14c.zip |
Changes in the validation of UTF-8
All UTF-8 encoding functionality (including the escape
sequence '\u') accepts all values from the original UTF-8
specification (with sequences of up to six bytes).
By default, the decoding functions in the UTF-8 library do not
accept invalid Unicode code points, such as surrogates. A new
parameter 'nonstrict' makes them accept all code points up to
(2^31)-1, as in the original UTF-8 specification.
Diffstat (limited to 'lutf8lib.c')
-rw-r--r-- | lutf8lib.c | 76 |
1 files changed, 49 insertions, 27 deletions
@@ -21,12 +21,14 @@ | |||
21 | #include "lualib.h" | 21 | #include "lualib.h" |
22 | 22 | ||
23 | 23 | ||
24 | #define MAXUNICODE 0x10FFFF | 24 | #define MAXUNICODE 0x10FFFFu |
25 | |||
26 | #define MAXUTF 0x7FFFFFFFu | ||
25 | 27 | ||
26 | /* | 28 | /* |
27 | ** Integer type for decoded UTF-8 values; MAXUNICODE needs 21 bits. | 29 | ** Integer type for decoded UTF-8 values; MAXUTF needs 31 bits. |
28 | */ | 30 | */ |
29 | #if LUAI_BITSINT >= 21 | 31 | #if LUAI_BITSINT >= 31 |
30 | typedef unsigned int utfint; | 32 | typedef unsigned int utfint; |
31 | #else | 33 | #else |
32 | typedef unsigned long utfint; | 34 | typedef unsigned long utfint; |
@@ -46,38 +48,46 @@ static lua_Integer u_posrelat (lua_Integer pos, size_t len) { | |||
46 | 48 | ||
47 | 49 | ||
48 | /* | 50 | /* |
49 | ** Decode one UTF-8 sequence, returning NULL if byte sequence is invalid. | 51 | ** Decode one UTF-8 sequence, returning NULL if byte sequence is |
52 | ** invalid. The array 'limits' stores the minimum value for each | ||
53 | ** sequence length, to check for overlong representations. Its first | ||
54 | ** entry forces an error for non-ascii bytes with no continuation | ||
55 | ** bytes (count == 0). | ||
50 | */ | 56 | */ |
51 | static const char *utf8_decode (const char *o, utfint *val) { | 57 | static const char *utf8_decode (const char *s, utfint *val, int strict) { |
52 | static const unsigned int limits[] = {0xFF, 0x7F, 0x7FF, 0xFFFF}; | 58 | static const utfint limits[] = |
53 | const unsigned char *s = (const unsigned char *)o; | 59 | {~(utfint)0, 0x80, 0x800, 0x10000u, 0x200000u, 0x4000000u}; |
54 | unsigned int c = s[0]; | 60 | unsigned int c = (unsigned char)s[0]; |
55 | utfint res = 0; /* final result */ | 61 | utfint res = 0; /* final result */ |
56 | if (c < 0x80) /* ascii? */ | 62 | if (c < 0x80) /* ascii? */ |
57 | res = c; | 63 | res = c; |
58 | else { | 64 | else { |
59 | int count = 0; /* to count number of continuation bytes */ | 65 | int count = 0; /* to count number of continuation bytes */ |
60 | while (c & 0x40) { /* still have continuation bytes? */ | 66 | for (; c & 0x40; c <<= 1) { /* while it needs continuation bytes... */ |
61 | int cc = s[++count]; /* read next byte */ | 67 | unsigned int cc = (unsigned char)s[++count]; /* read next byte */ |
62 | if ((cc & 0xC0) != 0x80) /* not a continuation byte? */ | 68 | if ((cc & 0xC0) != 0x80) /* not a continuation byte? */ |
63 | return NULL; /* invalid byte sequence */ | 69 | return NULL; /* invalid byte sequence */ |
64 | res = (res << 6) | (cc & 0x3F); /* add lower 6 bits from cont. byte */ | 70 | res = (res << 6) | (cc & 0x3F); /* add lower 6 bits from cont. byte */ |
65 | c <<= 1; /* to test next bit */ | ||
66 | } | 71 | } |
67 | res |= ((utfint)(c & 0x7F) << (count * 5)); /* add first byte */ | 72 | res |= ((utfint)(c & 0x7F) << (count * 5)); /* add first byte */ |
68 | if (count > 3 || res > MAXUNICODE || res <= limits[count]) | 73 | if (count > 5 || res > MAXUTF || res < limits[count]) |
69 | return NULL; /* invalid byte sequence */ | 74 | return NULL; /* invalid byte sequence */ |
70 | s += count; /* skip continuation bytes read */ | 75 | s += count; /* skip continuation bytes read */ |
71 | } | 76 | } |
77 | if (strict) { | ||
78 | /* check for invalid code points; too large or surrogates */ | ||
79 | if (res > MAXUNICODE || (0xD800u <= res && res <= 0xDFFFu)) | ||
80 | return NULL; | ||
81 | } | ||
72 | if (val) *val = res; | 82 | if (val) *val = res; |
73 | return (const char *)s + 1; /* +1 to include first byte */ | 83 | return s + 1; /* +1 to include first byte */ |
74 | } | 84 | } |
75 | 85 | ||
76 | 86 | ||
77 | /* | 87 | /* |
78 | ** utf8len(s [, i [, j]]) --> number of characters that start in the | 88 | ** utf8len(s [, i [, j [, nonstrict]]]) --> number of characters that |
79 | ** range [i,j], or nil + current position if 's' is not well formed in | 89 | ** start in the range [i,j], or nil + current position if 's' is not |
80 | ** that interval | 90 | ** well formed in that interval |
81 | */ | 91 | */ |
82 | static int utflen (lua_State *L) { | 92 | static int utflen (lua_State *L) { |
83 | lua_Integer n = 0; /* counter for the number of characters */ | 93 | lua_Integer n = 0; /* counter for the number of characters */ |
@@ -85,12 +95,13 @@ static int utflen (lua_State *L) { | |||
85 | const char *s = luaL_checklstring(L, 1, &len); | 95 | const char *s = luaL_checklstring(L, 1, &len); |
86 | lua_Integer posi = u_posrelat(luaL_optinteger(L, 2, 1), len); | 96 | lua_Integer posi = u_posrelat(luaL_optinteger(L, 2, 1), len); |
87 | lua_Integer posj = u_posrelat(luaL_optinteger(L, 3, -1), len); | 97 | lua_Integer posj = u_posrelat(luaL_optinteger(L, 3, -1), len); |
98 | int nonstrict = lua_toboolean(L, 4); | ||
88 | luaL_argcheck(L, 1 <= posi && --posi <= (lua_Integer)len, 2, | 99 | luaL_argcheck(L, 1 <= posi && --posi <= (lua_Integer)len, 2, |
89 | "initial position out of string"); | 100 | "initial position out of string"); |
90 | luaL_argcheck(L, --posj < (lua_Integer)len, 3, | 101 | luaL_argcheck(L, --posj < (lua_Integer)len, 3, |
91 | "final position out of string"); | 102 | "final position out of string"); |
92 | while (posi <= posj) { | 103 | while (posi <= posj) { |
93 | const char *s1 = utf8_decode(s + posi, NULL); | 104 | const char *s1 = utf8_decode(s + posi, NULL, !nonstrict); |
94 | if (s1 == NULL) { /* conversion error? */ | 105 | if (s1 == NULL) { /* conversion error? */ |
95 | lua_pushnil(L); /* return nil ... */ | 106 | lua_pushnil(L); /* return nil ... */ |
96 | lua_pushinteger(L, posi + 1); /* ... and current position */ | 107 | lua_pushinteger(L, posi + 1); /* ... and current position */ |
@@ -105,14 +116,15 @@ static int utflen (lua_State *L) { | |||
105 | 116 | ||
106 | 117 | ||
107 | /* | 118 | /* |
108 | ** codepoint(s, [i, [j]]) -> returns codepoints for all characters | 119 | ** codepoint(s, [i, [j [, nonstrict]]]) -> returns codepoints for all |
109 | ** that start in the range [i,j] | 120 | ** characters that start in the range [i,j] |
110 | */ | 121 | */ |
111 | static int codepoint (lua_State *L) { | 122 | static int codepoint (lua_State *L) { |
112 | size_t len; | 123 | size_t len; |
113 | const char *s = luaL_checklstring(L, 1, &len); | 124 | const char *s = luaL_checklstring(L, 1, &len); |
114 | lua_Integer posi = u_posrelat(luaL_optinteger(L, 2, 1), len); | 125 | lua_Integer posi = u_posrelat(luaL_optinteger(L, 2, 1), len); |
115 | lua_Integer pose = u_posrelat(luaL_optinteger(L, 3, posi), len); | 126 | lua_Integer pose = u_posrelat(luaL_optinteger(L, 3, posi), len); |
127 | int nonstrict = lua_toboolean(L, 4); | ||
116 | int n; | 128 | int n; |
117 | const char *se; | 129 | const char *se; |
118 | luaL_argcheck(L, posi >= 1, 2, "out of range"); | 130 | luaL_argcheck(L, posi >= 1, 2, "out of range"); |
@@ -126,7 +138,7 @@ static int codepoint (lua_State *L) { | |||
126 | se = s + pose; /* string end */ | 138 | se = s + pose; /* string end */ |
127 | for (s += posi - 1; s < se;) { | 139 | for (s += posi - 1; s < se;) { |
128 | utfint code; | 140 | utfint code; |
129 | s = utf8_decode(s, &code); | 141 | s = utf8_decode(s, &code, !nonstrict); |
130 | if (s == NULL) | 142 | if (s == NULL) |
131 | return luaL_error(L, "invalid UTF-8 code"); | 143 | return luaL_error(L, "invalid UTF-8 code"); |
132 | lua_pushinteger(L, code); | 144 | lua_pushinteger(L, code); |
@@ -137,8 +149,8 @@ static int codepoint (lua_State *L) { | |||
137 | 149 | ||
138 | 150 | ||
139 | static void pushutfchar (lua_State *L, int arg) { | 151 | static void pushutfchar (lua_State *L, int arg) { |
140 | lua_Integer code = luaL_checkinteger(L, arg); | 152 | lua_Unsigned code = (lua_Unsigned)luaL_checkinteger(L, arg); |
141 | luaL_argcheck(L, 0 <= code && code <= MAXUNICODE, arg, "value out of range"); | 153 | luaL_argcheck(L, code <= MAXUTF, arg, "value out of range"); |
142 | lua_pushfstring(L, "%U", (long)code); | 154 | lua_pushfstring(L, "%U", (long)code); |
143 | } | 155 | } |
144 | 156 | ||
@@ -209,7 +221,7 @@ static int byteoffset (lua_State *L) { | |||
209 | } | 221 | } |
210 | 222 | ||
211 | 223 | ||
212 | static int iter_aux (lua_State *L) { | 224 | static int iter_aux (lua_State *L, int strict) { |
213 | size_t len; | 225 | size_t len; |
214 | const char *s = luaL_checklstring(L, 1, &len); | 226 | const char *s = luaL_checklstring(L, 1, &len); |
215 | lua_Integer n = lua_tointeger(L, 2) - 1; | 227 | lua_Integer n = lua_tointeger(L, 2) - 1; |
@@ -223,8 +235,8 @@ static int iter_aux (lua_State *L) { | |||
223 | return 0; /* no more codepoints */ | 235 | return 0; /* no more codepoints */ |
224 | else { | 236 | else { |
225 | utfint code; | 237 | utfint code; |
226 | const char *next = utf8_decode(s + n, &code); | 238 | const char *next = utf8_decode(s + n, &code, strict); |
227 | if (next == NULL || iscont(next)) | 239 | if (next == NULL) |
228 | return luaL_error(L, "invalid UTF-8 code"); | 240 | return luaL_error(L, "invalid UTF-8 code"); |
229 | lua_pushinteger(L, n + 1); | 241 | lua_pushinteger(L, n + 1); |
230 | lua_pushinteger(L, code); | 242 | lua_pushinteger(L, code); |
@@ -233,9 +245,19 @@ static int iter_aux (lua_State *L) { | |||
233 | } | 245 | } |
234 | 246 | ||
235 | 247 | ||
248 | static int iter_auxstrict (lua_State *L) { | ||
249 | return iter_aux(L, 1); | ||
250 | } | ||
251 | |||
252 | static int iter_auxnostrict (lua_State *L) { | ||
253 | return iter_aux(L, 0); | ||
254 | } | ||
255 | |||
256 | |||
236 | static int iter_codes (lua_State *L) { | 257 | static int iter_codes (lua_State *L) { |
258 | int nonstrict = lua_toboolean(L, 2); | ||
237 | luaL_checkstring(L, 1); | 259 | luaL_checkstring(L, 1); |
238 | lua_pushcfunction(L, iter_aux); | 260 | lua_pushcfunction(L, nonstrict ? iter_auxnostrict : iter_auxstrict); |
239 | lua_pushvalue(L, 1); | 261 | lua_pushvalue(L, 1); |
240 | lua_pushinteger(L, 0); | 262 | lua_pushinteger(L, 0); |
241 | return 3; | 263 | return 3; |
@@ -243,7 +265,7 @@ static int iter_codes (lua_State *L) { | |||
243 | 265 | ||
244 | 266 | ||
245 | /* pattern to match a single UTF-8 character */ | 267 | /* pattern to match a single UTF-8 character */ |
246 | #define UTF8PATT "[\0-\x7F\xC2-\xF4][\x80-\xBF]*" | 268 | #define UTF8PATT "[\0-\x7F\xC2-\xFD][\x80-\xBF]*" |
247 | 269 | ||
248 | 270 | ||
249 | static const luaL_Reg funcs[] = { | 271 | static const luaL_Reg funcs[] = { |