diff options
author | Roberto Ierusalimschy <roberto@inf.puc-rio.br> | 2014-02-06 15:32:33 -0200 |
---|---|---|
committer | Roberto Ierusalimschy <roberto@inf.puc-rio.br> | 2014-02-06 15:32:33 -0200 |
commit | 6ccf1500396efffeb38ed54ac78d2f2d41a9b762 (patch) | |
tree | 8985365adae96e4d3c8b9e6f4b02c1c9b94e28c1 /lutf8lib.c | |
parent | 4ea60463f5a5cc5c30bf3f20be0dd5141f48aa3c (diff) | |
download | lua-6ccf1500396efffeb38ed54ac78d2f2d41a9b762.tar.gz lua-6ccf1500396efffeb38ed54ac78d2f2d41a9b762.tar.bz2 lua-6ccf1500396efffeb38ed54ac78d2f2d41a9b762.zip |
new library: utf8
Diffstat (limited to 'lutf8lib.c')
-rw-r--r-- | lutf8lib.c | 233 |
1 files changed, 233 insertions, 0 deletions
diff --git a/lutf8lib.c b/lutf8lib.c new file mode 100644 index 00000000..dfd52832 --- /dev/null +++ b/lutf8lib.c | |||
@@ -0,0 +1,233 @@ | |||
1 | /* gcc -shared -o utf8.so -fpic -O2 -Wall -I.. utf8.c */ | ||
2 | |||
3 | #include <assert.h> | ||
4 | #include <stdlib.h> | ||
5 | #include <string.h> | ||
6 | |||
7 | #include "lua.h" | ||
8 | #include "lauxlib.h" | ||
9 | |||
10 | #define MAXUNICODE 0x10FFFF | ||
11 | |||
12 | #define iscont(p) ((*(p) & 0xC0) == 0x80) | ||
13 | |||
14 | |||
15 | /* from strlib */ | ||
16 | /* translate a relative string position: negative means back from end */ | ||
17 | static lua_Integer posrelat (lua_Integer pos, size_t len) { | ||
18 | if (pos >= 0) return pos; | ||
19 | else if (0u - (size_t)pos > len) return 0; | ||
20 | else return (lua_Integer)len + pos + 1; | ||
21 | } | ||
22 | |||
23 | |||
24 | /* | ||
25 | ** Decode an UTF-8 sequence, returning NULL if byte sequence is invalid. | ||
26 | */ | ||
27 | static const char *utf8_decode (const char *o, int *val) { | ||
28 | static unsigned int limits[] = {0xFF, 0x7F, 0x7FF, 0xFFFF}; | ||
29 | const unsigned char *s = (const unsigned char *)o; | ||
30 | unsigned int c = s[0]; | ||
31 | unsigned int res = 0; /* final result */ | ||
32 | if (c < 0x80) /* ascii? */ | ||
33 | res = c; | ||
34 | else { | ||
35 | int count = 0; /* to count number of continuation bytes */ | ||
36 | while (c & 0x40) { /* still have continuation bytes? */ | ||
37 | int cc = s[++count]; /* read next byte */ | ||
38 | if ((cc & 0xC0) != 0x80) /* not a continuation byte? */ | ||
39 | return NULL; /* invalid byte sequence */ | ||
40 | res = (res << 6) | (cc & 0x3F); /* add lower 6 bits from cont. byte */ | ||
41 | c <<= 1; /* to test next bit */ | ||
42 | } | ||
43 | res |= ((c & 0x7F) << (count * 5)); /* add first byte */ | ||
44 | if (count > 3 || res > MAXUNICODE || res <= limits[count]) | ||
45 | return NULL; /* invalid byte sequence */ | ||
46 | s += count; /* skip continuation bytes read */ | ||
47 | } | ||
48 | if (val) *val = res; | ||
49 | return (const char *)s + 1; /* +1 to include first byte */ | ||
50 | } | ||
51 | |||
52 | |||
53 | /* | ||
54 | ** utf8len(s, [i]) --> number of codepoints in 's' after 'i'; | ||
55 | ** nil if 's' not well formed | ||
56 | */ | ||
57 | static int utflen (lua_State *L) { | ||
58 | int n = 0; | ||
59 | const char *ends; | ||
60 | size_t len; | ||
61 | const char *s = luaL_checklstring(L, 1, &len); | ||
62 | lua_Integer posi = posrelat(luaL_optinteger(L, 2, 1), 1); | ||
63 | luaL_argcheck(L, 1 <= posi && posi <= (lua_Integer)len, 1, | ||
64 | "initial position out of string"); | ||
65 | ends = s + len; | ||
66 | s += posi - 1; | ||
67 | while (s < ends && (s = utf8_decode(s, NULL)) != NULL) | ||
68 | n++; | ||
69 | if (s == ends) | ||
70 | lua_pushinteger(L, n); | ||
71 | else | ||
72 | lua_pushnil(L); | ||
73 | return 1; | ||
74 | } | ||
75 | |||
76 | |||
77 | /* | ||
78 | ** codepoint(s, [i, [j]]) -> returns codepoints for all characters | ||
79 | ** between i and j | ||
80 | */ | ||
81 | static int codepoint (lua_State *L) { | ||
82 | size_t len; | ||
83 | const char *s = luaL_checklstring(L, 1, &len); | ||
84 | lua_Integer posi = posrelat(luaL_optinteger(L, 2, 1), len); | ||
85 | lua_Integer pose = posrelat(luaL_optinteger(L, 3, posi), len); | ||
86 | int n; | ||
87 | const char *se; | ||
88 | luaL_argcheck(L, posi >= 1, 2, "out of range"); | ||
89 | luaL_argcheck(L, pose <= (lua_Integer)len, 3, "out of range"); | ||
90 | if (posi > pose) return 0; /* empty interval; return no values */ | ||
91 | n = (int)(pose - posi + 1); | ||
92 | if (posi + n <= pose) /* (lua_Integer -> int) overflow? */ | ||
93 | return luaL_error(L, "string slice too long"); | ||
94 | luaL_checkstack(L, n, "string slice too long"); | ||
95 | n = 0; | ||
96 | se = s + pose; | ||
97 | for (s += posi - 1; s < se;) { | ||
98 | int code; | ||
99 | s = utf8_decode(s, &code); | ||
100 | if (s == NULL) | ||
101 | luaL_error(L, "invalid UTF-8 code"); | ||
102 | lua_pushinteger(L, code); | ||
103 | n++; | ||
104 | } | ||
105 | return n; | ||
106 | } | ||
107 | |||
108 | |||
109 | static void pushutfchar (lua_State *L, int arg) { | ||
110 | int code = luaL_checkint(L, arg); | ||
111 | luaL_argcheck(L, 0 <= code && code <= MAXUNICODE, arg, "value out of range"); | ||
112 | lua_pushfstring(L, "%U", code); | ||
113 | } | ||
114 | |||
115 | |||
116 | /* | ||
117 | ** utfchar(n1, n2, ...) -> char(n1)..char(n2)... | ||
118 | */ | ||
119 | static int utfchar (lua_State *L) { | ||
120 | int n = lua_gettop(L); /* number of arguments */ | ||
121 | if (n == 1) /* optimize common case of single char */ | ||
122 | pushutfchar(L, 1); | ||
123 | else { | ||
124 | int i; | ||
125 | luaL_Buffer b; | ||
126 | luaL_buffinit(L, &b); | ||
127 | for (i = 1; i <= n; i++) { | ||
128 | pushutfchar(L, i); | ||
129 | luaL_addvalue(&b); | ||
130 | } | ||
131 | luaL_pushresult(&b); | ||
132 | } | ||
133 | return 1; | ||
134 | } | ||
135 | |||
136 | |||
137 | /* | ||
138 | ** offset(s, n, [i]) -> index where n-th character *after* | ||
139 | ** position 'i' starts; 0 means character at 'i'. | ||
140 | */ | ||
141 | static int byteoffset (lua_State *L) { | ||
142 | size_t len; | ||
143 | const char *s = luaL_checklstring(L, 1, &len); | ||
144 | int n = luaL_checkint(L, 2); | ||
145 | lua_Integer posi = posrelat(luaL_optinteger(L, 3, 1), len) - 1; | ||
146 | luaL_argcheck(L, 0 <= posi && posi <= (lua_Integer)len, 3, | ||
147 | "position out of range"); | ||
148 | if (n == 0) { | ||
149 | /* find beginning of current byte sequence */ | ||
150 | while (posi > 0 && iscont(s + posi)) posi--; | ||
151 | } | ||
152 | else if (n < 0) { | ||
153 | while (n < 0 && posi > 0) { /* move back */ | ||
154 | do { /* find beginning of previous character */ | ||
155 | posi--; | ||
156 | } while (posi > 0 && iscont(s + posi)); | ||
157 | n++; | ||
158 | } | ||
159 | } | ||
160 | else { | ||
161 | n--; /* do not move for 1st character */ | ||
162 | while (n > 0 && posi < (lua_Integer)len) { | ||
163 | do { /* find beginning of next character */ | ||
164 | posi++; | ||
165 | } while (iscont(s + posi)); /* ('\0' is not continuation) */ | ||
166 | n--; | ||
167 | } | ||
168 | } | ||
169 | if (n == 0) | ||
170 | lua_pushinteger(L, posi + 1); | ||
171 | else | ||
172 | lua_pushnil(L); /* no such position */ | ||
173 | return 1; | ||
174 | } | ||
175 | |||
176 | |||
177 | static int iter_aux (lua_State *L) { | ||
178 | size_t len; | ||
179 | const char *s = luaL_checklstring(L, 1, &len); | ||
180 | int n = lua_tointeger(L, 2) - 1; | ||
181 | if (n < 0) /* first iteration? */ | ||
182 | n = 0; /* start from here */ | ||
183 | else if (n < (lua_Integer)len) { | ||
184 | n++; /* skip current byte */ | ||
185 | while (iscont(s + n)) n++; /* and its continuations */ | ||
186 | } | ||
187 | if (n >= (lua_Integer)len) | ||
188 | return 0; /* no more codepoints */ | ||
189 | else { | ||
190 | int code; | ||
191 | const char *next = utf8_decode(s + n, &code); | ||
192 | if (next == NULL || iscont(next)) | ||
193 | luaL_error(L, "invalid UTF-8 code"); | ||
194 | lua_pushinteger(L, n + 1); | ||
195 | lua_pushinteger(L, code); | ||
196 | return 2; | ||
197 | } | ||
198 | } | ||
199 | |||
200 | |||
201 | static int iter_codes (lua_State *L) { | ||
202 | luaL_checkstring(L, 1); | ||
203 | lua_pushcfunction(L, iter_aux); | ||
204 | lua_pushvalue(L, 1); | ||
205 | lua_pushinteger(L, 0); | ||
206 | return 3; | ||
207 | } | ||
208 | |||
209 | |||
210 | /* pattern to match a single UTF-8 character */ | ||
211 | #define UTF8PATT "[\0-\x7F\xC2-\xF4][\x80-\xBF]*" | ||
212 | |||
213 | |||
214 | static struct luaL_Reg funcs[] = { | ||
215 | {"offset", byteoffset}, | ||
216 | {"codepoint", codepoint}, | ||
217 | {"char", utfchar}, | ||
218 | {"len", utflen}, | ||
219 | {"codes", iter_codes}, | ||
220 | {NULL, NULL} | ||
221 | }; | ||
222 | |||
223 | |||
224 | |||
225 | int luaopen_utf8 (lua_State *L); | ||
226 | |||
227 | int luaopen_utf8 (lua_State *L) { | ||
228 | luaL_newlib(L, funcs); | ||
229 | lua_pushliteral(L, UTF8PATT); | ||
230 | lua_setfield(L, -2, "charpatt"); | ||
231 | return 1; | ||
232 | } | ||
233 | |||