aboutsummaryrefslogtreecommitdiff
path: root/lutf8lib.c
diff options
context:
space:
mode:
authorRoberto Ierusalimschy <roberto@inf.puc-rio.br>2014-02-06 15:32:33 -0200
committerRoberto Ierusalimschy <roberto@inf.puc-rio.br>2014-02-06 15:32:33 -0200
commit6ccf1500396efffeb38ed54ac78d2f2d41a9b762 (patch)
tree8985365adae96e4d3c8b9e6f4b02c1c9b94e28c1 /lutf8lib.c
parent4ea60463f5a5cc5c30bf3f20be0dd5141f48aa3c (diff)
downloadlua-6ccf1500396efffeb38ed54ac78d2f2d41a9b762.tar.gz
lua-6ccf1500396efffeb38ed54ac78d2f2d41a9b762.tar.bz2
lua-6ccf1500396efffeb38ed54ac78d2f2d41a9b762.zip
new library: utf8
Diffstat (limited to 'lutf8lib.c')
-rw-r--r--lutf8lib.c233
1 files changed, 233 insertions, 0 deletions
diff --git a/lutf8lib.c b/lutf8lib.c
new file mode 100644
index 00000000..dfd52832
--- /dev/null
+++ b/lutf8lib.c
@@ -0,0 +1,233 @@
1/* gcc -shared -o utf8.so -fpic -O2 -Wall -I.. utf8.c */
2
3#include <assert.h>
4#include <stdlib.h>
5#include <string.h>
6
7#include "lua.h"
8#include "lauxlib.h"
9
10#define MAXUNICODE 0x10FFFF
11
12#define iscont(p) ((*(p) & 0xC0) == 0x80)
13
14
15/* from strlib */
16/* translate a relative string position: negative means back from end */
17static lua_Integer posrelat (lua_Integer pos, size_t len) {
18 if (pos >= 0) return pos;
19 else if (0u - (size_t)pos > len) return 0;
20 else return (lua_Integer)len + pos + 1;
21}
22
23
24/*
25** Decode an UTF-8 sequence, returning NULL if byte sequence is invalid.
26*/
27static const char *utf8_decode (const char *o, int *val) {
28 static unsigned int limits[] = {0xFF, 0x7F, 0x7FF, 0xFFFF};
29 const unsigned char *s = (const unsigned char *)o;
30 unsigned int c = s[0];
31 unsigned int res = 0; /* final result */
32 if (c < 0x80) /* ascii? */
33 res = c;
34 else {
35 int count = 0; /* to count number of continuation bytes */
36 while (c & 0x40) { /* still have continuation bytes? */
37 int cc = s[++count]; /* read next byte */
38 if ((cc & 0xC0) != 0x80) /* not a continuation byte? */
39 return NULL; /* invalid byte sequence */
40 res = (res << 6) | (cc & 0x3F); /* add lower 6 bits from cont. byte */
41 c <<= 1; /* to test next bit */
42 }
43 res |= ((c & 0x7F) << (count * 5)); /* add first byte */
44 if (count > 3 || res > MAXUNICODE || res <= limits[count])
45 return NULL; /* invalid byte sequence */
46 s += count; /* skip continuation bytes read */
47 }
48 if (val) *val = res;
49 return (const char *)s + 1; /* +1 to include first byte */
50}
51
52
53/*
54** utf8len(s, [i]) --> number of codepoints in 's' after 'i';
55** nil if 's' not well formed
56*/
57static int utflen (lua_State *L) {
58 int n = 0;
59 const char *ends;
60 size_t len;
61 const char *s = luaL_checklstring(L, 1, &len);
62 lua_Integer posi = posrelat(luaL_optinteger(L, 2, 1), 1);
63 luaL_argcheck(L, 1 <= posi && posi <= (lua_Integer)len, 1,
64 "initial position out of string");
65 ends = s + len;
66 s += posi - 1;
67 while (s < ends && (s = utf8_decode(s, NULL)) != NULL)
68 n++;
69 if (s == ends)
70 lua_pushinteger(L, n);
71 else
72 lua_pushnil(L);
73 return 1;
74}
75
76
77/*
78** codepoint(s, [i, [j]]) -> returns codepoints for all characters
79** between i and j
80*/
81static int codepoint (lua_State *L) {
82 size_t len;
83 const char *s = luaL_checklstring(L, 1, &len);
84 lua_Integer posi = posrelat(luaL_optinteger(L, 2, 1), len);
85 lua_Integer pose = posrelat(luaL_optinteger(L, 3, posi), len);
86 int n;
87 const char *se;
88 luaL_argcheck(L, posi >= 1, 2, "out of range");
89 luaL_argcheck(L, pose <= (lua_Integer)len, 3, "out of range");
90 if (posi > pose) return 0; /* empty interval; return no values */
91 n = (int)(pose - posi + 1);
92 if (posi + n <= pose) /* (lua_Integer -> int) overflow? */
93 return luaL_error(L, "string slice too long");
94 luaL_checkstack(L, n, "string slice too long");
95 n = 0;
96 se = s + pose;
97 for (s += posi - 1; s < se;) {
98 int code;
99 s = utf8_decode(s, &code);
100 if (s == NULL)
101 luaL_error(L, "invalid UTF-8 code");
102 lua_pushinteger(L, code);
103 n++;
104 }
105 return n;
106}
107
108
109static void pushutfchar (lua_State *L, int arg) {
110 int code = luaL_checkint(L, arg);
111 luaL_argcheck(L, 0 <= code && code <= MAXUNICODE, arg, "value out of range");
112 lua_pushfstring(L, "%U", code);
113}
114
115
116/*
117** utfchar(n1, n2, ...) -> char(n1)..char(n2)...
118*/
119static int utfchar (lua_State *L) {
120 int n = lua_gettop(L); /* number of arguments */
121 if (n == 1) /* optimize common case of single char */
122 pushutfchar(L, 1);
123 else {
124 int i;
125 luaL_Buffer b;
126 luaL_buffinit(L, &b);
127 for (i = 1; i <= n; i++) {
128 pushutfchar(L, i);
129 luaL_addvalue(&b);
130 }
131 luaL_pushresult(&b);
132 }
133 return 1;
134}
135
136
137/*
138** offset(s, n, [i]) -> index where n-th character *after*
139** position 'i' starts; 0 means character at 'i'.
140*/
141static int byteoffset (lua_State *L) {
142 size_t len;
143 const char *s = luaL_checklstring(L, 1, &len);
144 int n = luaL_checkint(L, 2);
145 lua_Integer posi = posrelat(luaL_optinteger(L, 3, 1), len) - 1;
146 luaL_argcheck(L, 0 <= posi && posi <= (lua_Integer)len, 3,
147 "position out of range");
148 if (n == 0) {
149 /* find beginning of current byte sequence */
150 while (posi > 0 && iscont(s + posi)) posi--;
151 }
152 else if (n < 0) {
153 while (n < 0 && posi > 0) { /* move back */
154 do { /* find beginning of previous character */
155 posi--;
156 } while (posi > 0 && iscont(s + posi));
157 n++;
158 }
159 }
160 else {
161 n--; /* do not move for 1st character */
162 while (n > 0 && posi < (lua_Integer)len) {
163 do { /* find beginning of next character */
164 posi++;
165 } while (iscont(s + posi)); /* ('\0' is not continuation) */
166 n--;
167 }
168 }
169 if (n == 0)
170 lua_pushinteger(L, posi + 1);
171 else
172 lua_pushnil(L); /* no such position */
173 return 1;
174}
175
176
177static int iter_aux (lua_State *L) {
178 size_t len;
179 const char *s = luaL_checklstring(L, 1, &len);
180 int n = lua_tointeger(L, 2) - 1;
181 if (n < 0) /* first iteration? */
182 n = 0; /* start from here */
183 else if (n < (lua_Integer)len) {
184 n++; /* skip current byte */
185 while (iscont(s + n)) n++; /* and its continuations */
186 }
187 if (n >= (lua_Integer)len)
188 return 0; /* no more codepoints */
189 else {
190 int code;
191 const char *next = utf8_decode(s + n, &code);
192 if (next == NULL || iscont(next))
193 luaL_error(L, "invalid UTF-8 code");
194 lua_pushinteger(L, n + 1);
195 lua_pushinteger(L, code);
196 return 2;
197 }
198}
199
200
201static int iter_codes (lua_State *L) {
202 luaL_checkstring(L, 1);
203 lua_pushcfunction(L, iter_aux);
204 lua_pushvalue(L, 1);
205 lua_pushinteger(L, 0);
206 return 3;
207}
208
209
210/* pattern to match a single UTF-8 character */
211#define UTF8PATT "[\0-\x7F\xC2-\xF4][\x80-\xBF]*"
212
213
214static struct luaL_Reg funcs[] = {
215 {"offset", byteoffset},
216 {"codepoint", codepoint},
217 {"char", utfchar},
218 {"len", utflen},
219 {"codes", iter_codes},
220 {NULL, NULL}
221};
222
223
224
225int luaopen_utf8 (lua_State *L);
226
227int luaopen_utf8 (lua_State *L) {
228 luaL_newlib(L, funcs);
229 lua_pushliteral(L, UTF8PATT);
230 lua_setfield(L, -2, "charpatt");
231 return 1;
232}
233