aboutsummaryrefslogtreecommitdiff
path: root/lutf8lib.c
diff options
context:
space:
mode:
authorRoberto Ierusalimschy <roberto@inf.puc-rio.br>2018-07-12 15:56:44 -0300
committerRoberto Ierusalimschy <roberto@inf.puc-rio.br>2018-07-12 15:56:44 -0300
commitfb18346dddcb0400d0396111c56a817a8d4bd8bd (patch)
treeba3c4afcbd9f300cec3b974471e37fd70f2a49dd /lutf8lib.c
parent96f9643f330a4bf0f8cd1973fecdd7161ffbbf68 (diff)
downloadlua-fb18346dddcb0400d0396111c56a817a8d4bd8bd.tar.gz
lua-fb18346dddcb0400d0396111c56a817a8d4bd8bd.tar.bz2
lua-fb18346dddcb0400d0396111c56a817a8d4bd8bd.zip
Avoid using 'int' for UTF-8 values
An 'int' may have only 16 bits, so it may not be big enough for UTF-8 values. The new type 'utfint' (in the utf8 library) ensures at least 21 bits for those values.
Diffstat (limited to 'lutf8lib.c')
-rw-r--r--lutf8lib.c33
1 files changed, 22 insertions, 11 deletions
diff --git a/lutf8lib.c b/lutf8lib.c
index 57bd59e0..dc95b285 100644
--- a/lutf8lib.c
+++ b/lutf8lib.c
@@ -1,5 +1,5 @@
1/* 1/*
2** $Id: lutf8lib.c,v 1.16 2016/12/22 13:08:50 roberto Exp roberto $ 2** $Id: lutf8lib.c $
3** Standard library for UTF-8 manipulation 3** Standard library for UTF-8 manipulation
4** See Copyright Notice in lua.h 4** See Copyright Notice in lua.h
5*/ 5*/
@@ -20,8 +20,19 @@
20#include "lauxlib.h" 20#include "lauxlib.h"
21#include "lualib.h" 21#include "lualib.h"
22 22
23
23#define MAXUNICODE 0x10FFFF 24#define MAXUNICODE 0x10FFFF
24 25
26/*
27** Integer type for decoded UTF-8 values; MAXUNICODE needs 21 bits.
28*/
29#if LUAI_BITSINT >= 21
30typedef unsigned int utfint;
31#else
32typedef unsigned long utfint;
33#endif
34
35
25#define iscont(p) ((*(p) & 0xC0) == 0x80) 36#define iscont(p) ((*(p) & 0xC0) == 0x80)
26 37
27 38
@@ -37,11 +48,11 @@ static lua_Integer u_posrelat (lua_Integer pos, size_t len) {
37/* 48/*
38** Decode one UTF-8 sequence, returning NULL if byte sequence is invalid. 49** Decode one UTF-8 sequence, returning NULL if byte sequence is invalid.
39*/ 50*/
40static const char *utf8_decode (const char *o, int *val) { 51static const char *utf8_decode (const char *o, utfint *val) {
41 static const unsigned int limits[] = {0xFF, 0x7F, 0x7FF, 0xFFFF}; 52 static const unsigned int limits[] = {0xFF, 0x7F, 0x7FF, 0xFFFF};
42 const unsigned char *s = (const unsigned char *)o; 53 const unsigned char *s = (const unsigned char *)o;
43 unsigned int c = s[0]; 54 unsigned int c = s[0];
44 unsigned int res = 0; /* final result */ 55 utfint res = 0; /* final result */
45 if (c < 0x80) /* ascii? */ 56 if (c < 0x80) /* ascii? */
46 res = c; 57 res = c;
47 else { 58 else {
@@ -53,7 +64,7 @@ static const char *utf8_decode (const char *o, int *val) {
53 res = (res << 6) | (cc & 0x3F); /* add lower 6 bits from cont. byte */ 64 res = (res << 6) | (cc & 0x3F); /* add lower 6 bits from cont. byte */
54 c <<= 1; /* to test next bit */ 65 c <<= 1; /* to test next bit */
55 } 66 }
56 res |= ((c & 0x7F) << (count * 5)); /* add first byte */ 67 res |= ((utfint)(c & 0x7F) << (count * 5)); /* add first byte */
57 if (count > 3 || res > MAXUNICODE || res <= limits[count]) 68 if (count > 3 || res > MAXUNICODE || res <= limits[count])
58 return NULL; /* invalid byte sequence */ 69 return NULL; /* invalid byte sequence */
59 s += count; /* skip continuation bytes read */ 70 s += count; /* skip continuation bytes read */
@@ -69,8 +80,8 @@ static const char *utf8_decode (const char *o, int *val) {
69** that interval 80** that interval
70*/ 81*/
71static int utflen (lua_State *L) { 82static int utflen (lua_State *L) {
72 int n = 0; 83 lua_Integer n = 0; /* counter for the number of characters */
73 size_t len; 84 size_t len; /* string length in bytes */
74 const char *s = luaL_checklstring(L, 1, &len); 85 const char *s = luaL_checklstring(L, 1, &len);
75 lua_Integer posi = u_posrelat(luaL_optinteger(L, 2, 1), len); 86 lua_Integer posi = u_posrelat(luaL_optinteger(L, 2, 1), len);
76 lua_Integer posj = u_posrelat(luaL_optinteger(L, 3, -1), len); 87 lua_Integer posj = u_posrelat(luaL_optinteger(L, 3, -1), len);
@@ -109,12 +120,12 @@ static int codepoint (lua_State *L) {
109 if (posi > pose) return 0; /* empty interval; return no values */ 120 if (posi > pose) return 0; /* empty interval; return no values */
110 if (pose - posi >= INT_MAX) /* (lua_Integer -> int) overflow? */ 121 if (pose - posi >= INT_MAX) /* (lua_Integer -> int) overflow? */
111 return luaL_error(L, "string slice too long"); 122 return luaL_error(L, "string slice too long");
112 n = (int)(pose - posi) + 1; 123 n = (int)(pose - posi) + 1; /* upper bound for number of returns */
113 luaL_checkstack(L, n, "string slice too long"); 124 luaL_checkstack(L, n, "string slice too long");
114 n = 0; 125 n = 0; /* count the number of returns */
115 se = s + pose; 126 se = s + pose; /* string end */
116 for (s += posi - 1; s < se;) { 127 for (s += posi - 1; s < se;) {
117 int code; 128 utfint code;
118 s = utf8_decode(s, &code); 129 s = utf8_decode(s, &code);
119 if (s == NULL) 130 if (s == NULL)
120 return luaL_error(L, "invalid UTF-8 code"); 131 return luaL_error(L, "invalid UTF-8 code");
@@ -211,7 +222,7 @@ static int iter_aux (lua_State *L) {
211 if (n >= (lua_Integer)len) 222 if (n >= (lua_Integer)len)
212 return 0; /* no more codepoints */ 223 return 0; /* no more codepoints */
213 else { 224 else {
214 int code; 225 utfint code;
215 const char *next = utf8_decode(s + n, &code); 226 const char *next = utf8_decode(s + n, &code);
216 if (next == NULL || iscont(next)) 227 if (next == NULL || iscont(next))
217 return luaL_error(L, "invalid UTF-8 code"); 228 return luaL_error(L, "invalid UTF-8 code");