diff options
Diffstat (limited to 'src/lj_lex.c')
-rw-r--r-- | src/lj_lex.c | 385 |
1 files changed, 208 insertions, 177 deletions
diff --git a/src/lj_lex.c b/src/lj_lex.c index ca942583..61c7ff43 100644 --- a/src/lj_lex.c +++ b/src/lj_lex.c | |||
@@ -12,6 +12,7 @@ | |||
12 | #include "lj_obj.h" | 12 | #include "lj_obj.h" |
13 | #include "lj_gc.h" | 13 | #include "lj_gc.h" |
14 | #include "lj_err.h" | 14 | #include "lj_err.h" |
15 | #include "lj_buf.h" | ||
15 | #include "lj_str.h" | 16 | #include "lj_str.h" |
16 | #if LJ_HASFFI | 17 | #if LJ_HASFFI |
17 | #include "lj_tab.h" | 18 | #include "lj_tab.h" |
@@ -24,6 +25,7 @@ | |||
24 | #include "lj_parse.h" | 25 | #include "lj_parse.h" |
25 | #include "lj_char.h" | 26 | #include "lj_char.h" |
26 | #include "lj_strscan.h" | 27 | #include "lj_strscan.h" |
28 | #include "lj_strfmt.h" | ||
27 | 29 | ||
28 | /* Lua lexer token names. */ | 30 | /* Lua lexer token names. */ |
29 | static const char *const tokennames[] = { | 31 | static const char *const tokennames[] = { |
@@ -37,54 +39,54 @@ TKDEF(TKSTR1, TKSTR2) | |||
37 | 39 | ||
38 | /* -- Buffer handling ----------------------------------------------------- */ | 40 | /* -- Buffer handling ----------------------------------------------------- */ |
39 | 41 | ||
40 | #define char2int(c) ((int)(uint8_t)(c)) | 42 | #define LEX_EOF (-1) |
41 | #define next(ls) \ | 43 | #define lex_iseol(ls) (ls->c == '\n' || ls->c == '\r') |
42 | (ls->current = (ls->n--) > 0 ? char2int(*ls->p++) : fillbuf(ls)) | ||
43 | #define save_and_next(ls) (save(ls, ls->current), next(ls)) | ||
44 | #define currIsNewline(ls) (ls->current == '\n' || ls->current == '\r') | ||
45 | #define END_OF_STREAM (-1) | ||
46 | 44 | ||
47 | static int fillbuf(LexState *ls) | 45 | /* Get more input from reader. */ |
46 | static LJ_NOINLINE LexChar lex_more(LexState *ls) | ||
48 | { | 47 | { |
49 | size_t sz; | 48 | size_t sz; |
50 | const char *buf = ls->rfunc(ls->L, ls->rdata, &sz); | 49 | const char *p = ls->rfunc(ls->L, ls->rdata, &sz); |
51 | if (buf == NULL || sz == 0) return END_OF_STREAM; | 50 | if (p == NULL || sz == 0) return LEX_EOF; |
52 | if (sz >= LJ_MAX_MEM) { | 51 | if (sz >= LJ_MAX_BUF) { |
53 | if (sz != ~(size_t)0) lj_err_mem(ls->L); | 52 | if (sz != ~(size_t)0) lj_err_mem(ls->L); |
53 | sz = ~(uintptr_t)0 - (uintptr_t)p; | ||
54 | if (sz >= LJ_MAX_BUF) sz = LJ_MAX_BUF-1; | ||
54 | ls->endmark = 1; | 55 | ls->endmark = 1; |
55 | } | 56 | } |
56 | ls->n = (MSize)sz - 1; | 57 | ls->pe = p + sz; |
57 | ls->p = buf; | 58 | ls->p = p + 1; |
58 | return char2int(*(ls->p++)); | 59 | return (LexChar)(uint8_t)p[0]; |
59 | } | 60 | } |
60 | 61 | ||
61 | static LJ_NOINLINE void save_grow(LexState *ls, int c) | 62 | /* Get next character. */ |
63 | static LJ_AINLINE LexChar lex_next(LexState *ls) | ||
62 | { | 64 | { |
63 | MSize newsize; | 65 | return (ls->c = ls->p < ls->pe ? (LexChar)(uint8_t)*ls->p++ : lex_more(ls)); |
64 | if (ls->sb.sz >= LJ_MAX_STR/2) | ||
65 | lj_lex_error(ls, 0, LJ_ERR_XELEM); | ||
66 | newsize = ls->sb.sz * 2; | ||
67 | lj_str_resizebuf(ls->L, &ls->sb, newsize); | ||
68 | ls->sb.buf[ls->sb.n++] = (char)c; | ||
69 | } | 66 | } |
70 | 67 | ||
71 | static LJ_AINLINE void save(LexState *ls, int c) | 68 | /* Save character. */ |
69 | static LJ_AINLINE void lex_save(LexState *ls, LexChar c) | ||
72 | { | 70 | { |
73 | if (LJ_UNLIKELY(ls->sb.n + 1 > ls->sb.sz)) | 71 | lj_buf_putb(&ls->sb, c); |
74 | save_grow(ls, c); | ||
75 | else | ||
76 | ls->sb.buf[ls->sb.n++] = (char)c; | ||
77 | } | 72 | } |
78 | 73 | ||
79 | static void inclinenumber(LexState *ls) | 74 | /* Save previous character and get next character. */ |
75 | static LJ_AINLINE LexChar lex_savenext(LexState *ls) | ||
80 | { | 76 | { |
81 | int old = ls->current; | 77 | lex_save(ls, ls->c); |
82 | lua_assert(currIsNewline(ls)); | 78 | return lex_next(ls); |
83 | next(ls); /* skip `\n' or `\r' */ | 79 | } |
84 | if (currIsNewline(ls) && ls->current != old) | 80 | |
85 | next(ls); /* skip `\n\r' or `\r\n' */ | 81 | /* Skip line break. Handles "\n", "\r", "\r\n" or "\n\r". */ |
82 | static void lex_newline(LexState *ls) | ||
83 | { | ||
84 | LexChar old = ls->c; | ||
85 | lj_assertLS(lex_iseol(ls), "bad usage"); | ||
86 | lex_next(ls); /* Skip "\n" or "\r". */ | ||
87 | if (lex_iseol(ls) && ls->c != old) lex_next(ls); /* Skip "\n\r" or "\r\n". */ | ||
86 | if (++ls->linenumber >= LJ_MAX_LINE) | 88 | if (++ls->linenumber >= LJ_MAX_LINE) |
87 | lj_lex_error(ls, ls->token, LJ_ERR_XLINES); | 89 | lj_lex_error(ls, ls->tok, LJ_ERR_XLINES); |
88 | } | 90 | } |
89 | 91 | ||
90 | /* -- Scanner for terminals ----------------------------------------------- */ | 92 | /* -- Scanner for terminals ----------------------------------------------- */ |
@@ -93,19 +95,17 @@ static void inclinenumber(LexState *ls) | |||
93 | static void lex_number(LexState *ls, TValue *tv) | 95 | static void lex_number(LexState *ls, TValue *tv) |
94 | { | 96 | { |
95 | StrScanFmt fmt; | 97 | StrScanFmt fmt; |
96 | int c, xp = 'e'; | 98 | LexChar c, xp = 'e'; |
97 | lua_assert(lj_char_isdigit(ls->current)); | 99 | lj_assertLS(lj_char_isdigit(ls->c), "bad usage"); |
98 | if ((c = ls->current) == '0') { | 100 | if ((c = ls->c) == '0' && (lex_savenext(ls) | 0x20) == 'x') |
99 | save_and_next(ls); | 101 | xp = 'p'; |
100 | if ((ls->current | 0x20) == 'x') xp = 'p'; | 102 | while (lj_char_isident(ls->c) || ls->c == '.' || |
101 | } | 103 | ((ls->c == '-' || ls->c == '+') && (c | 0x20) == xp)) { |
102 | while (lj_char_isident(ls->current) || ls->current == '.' || | 104 | c = ls->c; |
103 | ((ls->current == '-' || ls->current == '+') && (c | 0x20) == xp)) { | 105 | lex_savenext(ls); |
104 | c = ls->current; | ||
105 | save_and_next(ls); | ||
106 | } | 106 | } |
107 | save(ls, '\0'); | 107 | lex_save(ls, '\0'); |
108 | fmt = lj_strscan_scan((const uint8_t *)ls->sb.buf, tv, | 108 | fmt = lj_strscan_scan((const uint8_t *)sbufB(&ls->sb), sbuflen(&ls->sb)-1, tv, |
109 | (LJ_DUALNUM ? STRSCAN_OPT_TOINT : STRSCAN_OPT_TONUM) | | 109 | (LJ_DUALNUM ? STRSCAN_OPT_TOINT : STRSCAN_OPT_TONUM) | |
110 | (LJ_HASFFI ? (STRSCAN_OPT_LL|STRSCAN_OPT_IMAG) : 0)); | 110 | (LJ_HASFFI ? (STRSCAN_OPT_LL|STRSCAN_OPT_IMAG) : 0)); |
111 | if (LJ_DUALNUM && fmt == STRSCAN_INT) { | 111 | if (LJ_DUALNUM && fmt == STRSCAN_INT) { |
@@ -116,7 +116,8 @@ static void lex_number(LexState *ls, TValue *tv) | |||
116 | } else if (fmt != STRSCAN_ERROR) { | 116 | } else if (fmt != STRSCAN_ERROR) { |
117 | lua_State *L = ls->L; | 117 | lua_State *L = ls->L; |
118 | GCcdata *cd; | 118 | GCcdata *cd; |
119 | lua_assert(fmt == STRSCAN_I64 || fmt == STRSCAN_U64 || fmt == STRSCAN_IMAG); | 119 | lj_assertLS(fmt == STRSCAN_I64 || fmt == STRSCAN_U64 || fmt == STRSCAN_IMAG, |
120 | "unexpected number format %d", fmt); | ||
120 | if (!ctype_ctsG(G(L))) { | 121 | if (!ctype_ctsG(G(L))) { |
121 | ptrdiff_t oldtop = savestack(L, L->top); | 122 | ptrdiff_t oldtop = savestack(L, L->top); |
122 | luaopen_ffi(L); /* Load FFI library on-demand. */ | 123 | luaopen_ffi(L); /* Load FFI library on-demand. */ |
@@ -133,65 +134,66 @@ static void lex_number(LexState *ls, TValue *tv) | |||
133 | lj_parse_keepcdata(ls, tv, cd); | 134 | lj_parse_keepcdata(ls, tv, cd); |
134 | #endif | 135 | #endif |
135 | } else { | 136 | } else { |
136 | lua_assert(fmt == STRSCAN_ERROR); | 137 | lj_assertLS(fmt == STRSCAN_ERROR, |
138 | "unexpected number format %d", fmt); | ||
137 | lj_lex_error(ls, TK_number, LJ_ERR_XNUMBER); | 139 | lj_lex_error(ls, TK_number, LJ_ERR_XNUMBER); |
138 | } | 140 | } |
139 | } | 141 | } |
140 | 142 | ||
141 | static int skip_sep(LexState *ls) | 143 | /* Skip equal signs for "[=...=[" and "]=...=]" and return their count. */ |
144 | static int lex_skipeq(LexState *ls) | ||
142 | { | 145 | { |
143 | int count = 0; | 146 | int count = 0; |
144 | int s = ls->current; | 147 | LexChar s = ls->c; |
145 | lua_assert(s == '[' || s == ']'); | 148 | lj_assertLS(s == '[' || s == ']', "bad usage"); |
146 | save_and_next(ls); | 149 | while (lex_savenext(ls) == '=' && count < 0x20000000) |
147 | while (ls->current == '=' && count < 0x20000000) { | ||
148 | save_and_next(ls); | ||
149 | count++; | 150 | count++; |
150 | } | 151 | return (ls->c == s) ? count : (-count) - 1; |
151 | return (ls->current == s) ? count : (-count) - 1; | ||
152 | } | 152 | } |
153 | 153 | ||
154 | static void read_long_string(LexState *ls, TValue *tv, int sep) | 154 | /* Parse a long string or long comment (tv set to NULL). */ |
155 | static void lex_longstring(LexState *ls, TValue *tv, int sep) | ||
155 | { | 156 | { |
156 | save_and_next(ls); /* skip 2nd `[' */ | 157 | lex_savenext(ls); /* Skip second '['. */ |
157 | if (currIsNewline(ls)) /* string starts with a newline? */ | 158 | if (lex_iseol(ls)) /* Skip initial newline. */ |
158 | inclinenumber(ls); /* skip it */ | 159 | lex_newline(ls); |
159 | for (;;) { | 160 | for (;;) { |
160 | switch (ls->current) { | 161 | switch (ls->c) { |
161 | case END_OF_STREAM: | 162 | case LEX_EOF: |
162 | lj_lex_error(ls, TK_eof, tv ? LJ_ERR_XLSTR : LJ_ERR_XLCOM); | 163 | lj_lex_error(ls, TK_eof, tv ? LJ_ERR_XLSTR : LJ_ERR_XLCOM); |
163 | break; | 164 | break; |
164 | case ']': | 165 | case ']': |
165 | if (skip_sep(ls) == sep) { | 166 | if (lex_skipeq(ls) == sep) { |
166 | save_and_next(ls); /* skip 2nd `]' */ | 167 | lex_savenext(ls); /* Skip second ']'. */ |
167 | goto endloop; | 168 | goto endloop; |
168 | } | 169 | } |
169 | break; | 170 | break; |
170 | case '\n': | 171 | case '\n': |
171 | case '\r': | 172 | case '\r': |
172 | save(ls, '\n'); | 173 | lex_save(ls, '\n'); |
173 | inclinenumber(ls); | 174 | lex_newline(ls); |
174 | if (!tv) lj_str_resetbuf(&ls->sb); /* avoid wasting space */ | 175 | if (!tv) lj_buf_reset(&ls->sb); /* Don't waste space for comments. */ |
175 | break; | 176 | break; |
176 | default: | 177 | default: |
177 | if (tv) save_and_next(ls); | 178 | lex_savenext(ls); |
178 | else next(ls); | ||
179 | break; | 179 | break; |
180 | } | 180 | } |
181 | } endloop: | 181 | } endloop: |
182 | if (tv) { | 182 | if (tv) { |
183 | GCstr *str = lj_parse_keepstr(ls, ls->sb.buf + (2 + (MSize)sep), | 183 | GCstr *str = lj_parse_keepstr(ls, sbufB(&ls->sb) + (2 + (MSize)sep), |
184 | ls->sb.n - 2*(2 + (MSize)sep)); | 184 | sbuflen(&ls->sb) - 2*(2 + (MSize)sep)); |
185 | setstrV(ls->L, tv, str); | 185 | setstrV(ls->L, tv, str); |
186 | } | 186 | } |
187 | } | 187 | } |
188 | 188 | ||
189 | static void read_string(LexState *ls, int delim, TValue *tv) | 189 | /* Parse a string. */ |
190 | static void lex_string(LexState *ls, TValue *tv) | ||
190 | { | 191 | { |
191 | save_and_next(ls); | 192 | LexChar delim = ls->c; /* Delimiter is '\'' or '"'. */ |
192 | while (ls->current != delim) { | 193 | lex_savenext(ls); |
193 | switch (ls->current) { | 194 | while (ls->c != delim) { |
194 | case END_OF_STREAM: | 195 | switch (ls->c) { |
196 | case LEX_EOF: | ||
195 | lj_lex_error(ls, TK_eof, LJ_ERR_XSTR); | 197 | lj_lex_error(ls, TK_eof, LJ_ERR_XSTR); |
196 | continue; | 198 | continue; |
197 | case '\n': | 199 | case '\n': |
@@ -199,7 +201,7 @@ static void read_string(LexState *ls, int delim, TValue *tv) | |||
199 | lj_lex_error(ls, TK_string, LJ_ERR_XSTR); | 201 | lj_lex_error(ls, TK_string, LJ_ERR_XSTR); |
200 | continue; | 202 | continue; |
201 | case '\\': { | 203 | case '\\': { |
202 | int c = next(ls); /* Skip the '\\'. */ | 204 | LexChar c = lex_next(ls); /* Skip the '\\'. */ |
203 | switch (c) { | 205 | switch (c) { |
204 | case 'a': c = '\a'; break; | 206 | case 'a': c = '\a'; break; |
205 | case 'b': c = '\b'; break; | 207 | case 'b': c = '\b'; break; |
@@ -209,111 +211,139 @@ static void read_string(LexState *ls, int delim, TValue *tv) | |||
209 | case 't': c = '\t'; break; | 211 | case 't': c = '\t'; break; |
210 | case 'v': c = '\v'; break; | 212 | case 'v': c = '\v'; break; |
211 | case 'x': /* Hexadecimal escape '\xXX'. */ | 213 | case 'x': /* Hexadecimal escape '\xXX'. */ |
212 | c = (next(ls) & 15u) << 4; | 214 | c = (lex_next(ls) & 15u) << 4; |
213 | if (!lj_char_isdigit(ls->current)) { | 215 | if (!lj_char_isdigit(ls->c)) { |
214 | if (!lj_char_isxdigit(ls->current)) goto err_xesc; | 216 | if (!lj_char_isxdigit(ls->c)) goto err_xesc; |
215 | c += 9 << 4; | 217 | c += 9 << 4; |
216 | } | 218 | } |
217 | c += (next(ls) & 15u); | 219 | c += (lex_next(ls) & 15u); |
218 | if (!lj_char_isdigit(ls->current)) { | 220 | if (!lj_char_isdigit(ls->c)) { |
219 | if (!lj_char_isxdigit(ls->current)) goto err_xesc; | 221 | if (!lj_char_isxdigit(ls->c)) goto err_xesc; |
220 | c += 9; | 222 | c += 9; |
221 | } | 223 | } |
222 | break; | 224 | break; |
225 | case 'u': /* Unicode escape '\u{XX...}'. */ | ||
226 | if (lex_next(ls) != '{') goto err_xesc; | ||
227 | lex_next(ls); | ||
228 | c = 0; | ||
229 | do { | ||
230 | c = (c << 4) | (ls->c & 15u); | ||
231 | if (!lj_char_isdigit(ls->c)) { | ||
232 | if (!lj_char_isxdigit(ls->c)) goto err_xesc; | ||
233 | c += 9; | ||
234 | } | ||
235 | if (c >= 0x110000) goto err_xesc; /* Out of Unicode range. */ | ||
236 | } while (lex_next(ls) != '}'); | ||
237 | if (c < 0x800) { | ||
238 | if (c < 0x80) break; | ||
239 | lex_save(ls, 0xc0 | (c >> 6)); | ||
240 | } else { | ||
241 | if (c >= 0x10000) { | ||
242 | lex_save(ls, 0xf0 | (c >> 18)); | ||
243 | lex_save(ls, 0x80 | ((c >> 12) & 0x3f)); | ||
244 | } else { | ||
245 | if (c >= 0xd800 && c < 0xe000) goto err_xesc; /* No surrogates. */ | ||
246 | lex_save(ls, 0xe0 | (c >> 12)); | ||
247 | } | ||
248 | lex_save(ls, 0x80 | ((c >> 6) & 0x3f)); | ||
249 | } | ||
250 | c = 0x80 | (c & 0x3f); | ||
251 | break; | ||
223 | case 'z': /* Skip whitespace. */ | 252 | case 'z': /* Skip whitespace. */ |
224 | next(ls); | 253 | lex_next(ls); |
225 | while (lj_char_isspace(ls->current)) | 254 | while (lj_char_isspace(ls->c)) |
226 | if (currIsNewline(ls)) inclinenumber(ls); else next(ls); | 255 | if (lex_iseol(ls)) lex_newline(ls); else lex_next(ls); |
227 | continue; | 256 | continue; |
228 | case '\n': case '\r': save(ls, '\n'); inclinenumber(ls); continue; | 257 | case '\n': case '\r': lex_save(ls, '\n'); lex_newline(ls); continue; |
229 | case '\\': case '\"': case '\'': break; | 258 | case '\\': case '\"': case '\'': break; |
230 | case END_OF_STREAM: continue; | 259 | case LEX_EOF: continue; |
231 | default: | 260 | default: |
232 | if (!lj_char_isdigit(c)) | 261 | if (!lj_char_isdigit(c)) |
233 | goto err_xesc; | 262 | goto err_xesc; |
234 | c -= '0'; /* Decimal escape '\ddd'. */ | 263 | c -= '0'; /* Decimal escape '\ddd'. */ |
235 | if (lj_char_isdigit(next(ls))) { | 264 | if (lj_char_isdigit(lex_next(ls))) { |
236 | c = c*10 + (ls->current - '0'); | 265 | c = c*10 + (ls->c - '0'); |
237 | if (lj_char_isdigit(next(ls))) { | 266 | if (lj_char_isdigit(lex_next(ls))) { |
238 | c = c*10 + (ls->current - '0'); | 267 | c = c*10 + (ls->c - '0'); |
239 | if (c > 255) { | 268 | if (c > 255) { |
240 | err_xesc: | 269 | err_xesc: |
241 | lj_lex_error(ls, TK_string, LJ_ERR_XESC); | 270 | lj_lex_error(ls, TK_string, LJ_ERR_XESC); |
242 | } | 271 | } |
243 | next(ls); | 272 | lex_next(ls); |
244 | } | 273 | } |
245 | } | 274 | } |
246 | save(ls, c); | 275 | lex_save(ls, c); |
247 | continue; | 276 | continue; |
248 | } | 277 | } |
249 | save(ls, c); | 278 | lex_save(ls, c); |
250 | next(ls); | 279 | lex_next(ls); |
251 | continue; | 280 | continue; |
252 | } | 281 | } |
253 | default: | 282 | default: |
254 | save_and_next(ls); | 283 | lex_savenext(ls); |
255 | break; | 284 | break; |
256 | } | 285 | } |
257 | } | 286 | } |
258 | save_and_next(ls); /* skip delimiter */ | 287 | lex_savenext(ls); /* Skip trailing delimiter. */ |
259 | setstrV(ls->L, tv, lj_parse_keepstr(ls, ls->sb.buf + 1, ls->sb.n - 2)); | 288 | setstrV(ls->L, tv, |
289 | lj_parse_keepstr(ls, sbufB(&ls->sb)+1, sbuflen(&ls->sb)-2)); | ||
260 | } | 290 | } |
261 | 291 | ||
262 | /* -- Main lexical scanner ------------------------------------------------ */ | 292 | /* -- Main lexical scanner ------------------------------------------------ */ |
263 | 293 | ||
264 | static int llex(LexState *ls, TValue *tv) | 294 | /* Get next lexical token. */ |
295 | static LexToken lex_scan(LexState *ls, TValue *tv) | ||
265 | { | 296 | { |
266 | lj_str_resetbuf(&ls->sb); | 297 | lj_buf_reset(&ls->sb); |
267 | for (;;) { | 298 | for (;;) { |
268 | if (lj_char_isident(ls->current)) { | 299 | if (lj_char_isident(ls->c)) { |
269 | GCstr *s; | 300 | GCstr *s; |
270 | if (lj_char_isdigit(ls->current)) { /* Numeric literal. */ | 301 | if (lj_char_isdigit(ls->c)) { /* Numeric literal. */ |
271 | lex_number(ls, tv); | 302 | lex_number(ls, tv); |
272 | return TK_number; | 303 | return TK_number; |
273 | } | 304 | } |
274 | /* Identifier or reserved word. */ | 305 | /* Identifier or reserved word. */ |
275 | do { | 306 | do { |
276 | save_and_next(ls); | 307 | lex_savenext(ls); |
277 | } while (lj_char_isident(ls->current)); | 308 | } while (lj_char_isident(ls->c)); |
278 | s = lj_parse_keepstr(ls, ls->sb.buf, ls->sb.n); | 309 | s = lj_parse_keepstr(ls, sbufB(&ls->sb), sbuflen(&ls->sb)); |
279 | setstrV(ls->L, tv, s); | 310 | setstrV(ls->L, tv, s); |
280 | if (s->reserved > 0) /* Reserved word? */ | 311 | if (s->reserved > 0) /* Reserved word? */ |
281 | return TK_OFS + s->reserved; | 312 | return TK_OFS + s->reserved; |
282 | return TK_name; | 313 | return TK_name; |
283 | } | 314 | } |
284 | switch (ls->current) { | 315 | switch (ls->c) { |
285 | case '\n': | 316 | case '\n': |
286 | case '\r': | 317 | case '\r': |
287 | inclinenumber(ls); | 318 | lex_newline(ls); |
288 | continue; | 319 | continue; |
289 | case ' ': | 320 | case ' ': |
290 | case '\t': | 321 | case '\t': |
291 | case '\v': | 322 | case '\v': |
292 | case '\f': | 323 | case '\f': |
293 | next(ls); | 324 | lex_next(ls); |
294 | continue; | 325 | continue; |
295 | case '-': | 326 | case '-': |
296 | next(ls); | 327 | lex_next(ls); |
297 | if (ls->current != '-') return '-'; | 328 | if (ls->c != '-') return '-'; |
298 | /* else is a comment */ | 329 | lex_next(ls); |
299 | next(ls); | 330 | if (ls->c == '[') { /* Long comment "--[=*[...]=*]". */ |
300 | if (ls->current == '[') { | 331 | int sep = lex_skipeq(ls); |
301 | int sep = skip_sep(ls); | 332 | lj_buf_reset(&ls->sb); /* `lex_skipeq' may dirty the buffer */ |
302 | lj_str_resetbuf(&ls->sb); /* `skip_sep' may dirty the buffer */ | ||
303 | if (sep >= 0) { | 333 | if (sep >= 0) { |
304 | read_long_string(ls, NULL, sep); /* long comment */ | 334 | lex_longstring(ls, NULL, sep); |
305 | lj_str_resetbuf(&ls->sb); | 335 | lj_buf_reset(&ls->sb); |
306 | continue; | 336 | continue; |
307 | } | 337 | } |
308 | } | 338 | } |
309 | /* else short comment */ | 339 | /* Short comment "--.*\n". */ |
310 | while (!currIsNewline(ls) && ls->current != END_OF_STREAM) | 340 | while (!lex_iseol(ls) && ls->c != LEX_EOF) |
311 | next(ls); | 341 | lex_next(ls); |
312 | continue; | 342 | continue; |
313 | case '[': { | 343 | case '[': { |
314 | int sep = skip_sep(ls); | 344 | int sep = lex_skipeq(ls); |
315 | if (sep >= 0) { | 345 | if (sep >= 0) { |
316 | read_long_string(ls, tv, sep); | 346 | lex_longstring(ls, tv, sep); |
317 | return TK_string; | 347 | return TK_string; |
318 | } else if (sep == -1) { | 348 | } else if (sep == -1) { |
319 | return '['; | 349 | return '['; |
@@ -323,44 +353,43 @@ static int llex(LexState *ls, TValue *tv) | |||
323 | } | 353 | } |
324 | } | 354 | } |
325 | case '=': | 355 | case '=': |
326 | next(ls); | 356 | lex_next(ls); |
327 | if (ls->current != '=') return '='; else { next(ls); return TK_eq; } | 357 | if (ls->c != '=') return '='; else { lex_next(ls); return TK_eq; } |
328 | case '<': | 358 | case '<': |
329 | next(ls); | 359 | lex_next(ls); |
330 | if (ls->current != '=') return '<'; else { next(ls); return TK_le; } | 360 | if (ls->c != '=') return '<'; else { lex_next(ls); return TK_le; } |
331 | case '>': | 361 | case '>': |
332 | next(ls); | 362 | lex_next(ls); |
333 | if (ls->current != '=') return '>'; else { next(ls); return TK_ge; } | 363 | if (ls->c != '=') return '>'; else { lex_next(ls); return TK_ge; } |
334 | case '~': | 364 | case '~': |
335 | next(ls); | 365 | lex_next(ls); |
336 | if (ls->current != '=') return '~'; else { next(ls); return TK_ne; } | 366 | if (ls->c != '=') return '~'; else { lex_next(ls); return TK_ne; } |
337 | case ':': | 367 | case ':': |
338 | next(ls); | 368 | lex_next(ls); |
339 | if (ls->current != ':') return ':'; else { next(ls); return TK_label; } | 369 | if (ls->c != ':') return ':'; else { lex_next(ls); return TK_label; } |
340 | case '"': | 370 | case '"': |
341 | case '\'': | 371 | case '\'': |
342 | read_string(ls, ls->current, tv); | 372 | lex_string(ls, tv); |
343 | return TK_string; | 373 | return TK_string; |
344 | case '.': | 374 | case '.': |
345 | save_and_next(ls); | 375 | if (lex_savenext(ls) == '.') { |
346 | if (ls->current == '.') { | 376 | lex_next(ls); |
347 | next(ls); | 377 | if (ls->c == '.') { |
348 | if (ls->current == '.') { | 378 | lex_next(ls); |
349 | next(ls); | ||
350 | return TK_dots; /* ... */ | 379 | return TK_dots; /* ... */ |
351 | } | 380 | } |
352 | return TK_concat; /* .. */ | 381 | return TK_concat; /* .. */ |
353 | } else if (!lj_char_isdigit(ls->current)) { | 382 | } else if (!lj_char_isdigit(ls->c)) { |
354 | return '.'; | 383 | return '.'; |
355 | } else { | 384 | } else { |
356 | lex_number(ls, tv); | 385 | lex_number(ls, tv); |
357 | return TK_number; | 386 | return TK_number; |
358 | } | 387 | } |
359 | case END_OF_STREAM: | 388 | case LEX_EOF: |
360 | return TK_eof; | 389 | return TK_eof; |
361 | default: { | 390 | default: { |
362 | int c = ls->current; | 391 | LexChar c = ls->c; |
363 | next(ls); | 392 | lex_next(ls); |
364 | return c; /* Single-char tokens (+ - / ...). */ | 393 | return c; /* Single-char tokens (+ - / ...). */ |
365 | } | 394 | } |
366 | } | 395 | } |
@@ -375,36 +404,33 @@ int lj_lex_setup(lua_State *L, LexState *ls) | |||
375 | int header = 0; | 404 | int header = 0; |
376 | ls->L = L; | 405 | ls->L = L; |
377 | ls->fs = NULL; | 406 | ls->fs = NULL; |
378 | ls->n = 0; | 407 | ls->pe = ls->p = NULL; |
379 | ls->p = NULL; | ||
380 | ls->vstack = NULL; | 408 | ls->vstack = NULL; |
381 | ls->sizevstack = 0; | 409 | ls->sizevstack = 0; |
382 | ls->vtop = 0; | 410 | ls->vtop = 0; |
383 | ls->bcstack = NULL; | 411 | ls->bcstack = NULL; |
384 | ls->sizebcstack = 0; | 412 | ls->sizebcstack = 0; |
385 | ls->token = 0; | 413 | ls->tok = 0; |
386 | ls->lookahead = TK_eof; /* No look-ahead token. */ | 414 | ls->lookahead = TK_eof; /* No look-ahead token. */ |
387 | ls->linenumber = 1; | 415 | ls->linenumber = 1; |
388 | ls->lastline = 1; | 416 | ls->lastline = 1; |
389 | ls->endmark = 0; | 417 | ls->endmark = 0; |
390 | lj_str_resizebuf(ls->L, &ls->sb, LJ_MIN_SBUF); | 418 | lex_next(ls); /* Read-ahead first char. */ |
391 | next(ls); /* Read-ahead first char. */ | 419 | if (ls->c == 0xef && ls->p + 2 <= ls->pe && (uint8_t)ls->p[0] == 0xbb && |
392 | if (ls->current == 0xef && ls->n >= 2 && char2int(ls->p[0]) == 0xbb && | 420 | (uint8_t)ls->p[1] == 0xbf) { /* Skip UTF-8 BOM (if buffered). */ |
393 | char2int(ls->p[1]) == 0xbf) { /* Skip UTF-8 BOM (if buffered). */ | ||
394 | ls->n -= 2; | ||
395 | ls->p += 2; | 421 | ls->p += 2; |
396 | next(ls); | 422 | lex_next(ls); |
397 | header = 1; | 423 | header = 1; |
398 | } | 424 | } |
399 | if (ls->current == '#') { /* Skip POSIX #! header line. */ | 425 | if (ls->c == '#') { /* Skip POSIX #! header line. */ |
400 | do { | 426 | do { |
401 | next(ls); | 427 | lex_next(ls); |
402 | if (ls->current == END_OF_STREAM) return 0; | 428 | if (ls->c == LEX_EOF) return 0; |
403 | } while (!currIsNewline(ls)); | 429 | } while (!lex_iseol(ls)); |
404 | inclinenumber(ls); | 430 | lex_newline(ls); |
405 | header = 1; | 431 | header = 1; |
406 | } | 432 | } |
407 | if (ls->current == LUA_SIGNATURE[0]) { /* Bytecode dump. */ | 433 | if (ls->c == LUA_SIGNATURE[0]) { /* Bytecode dump. */ |
408 | if (header) { | 434 | if (header) { |
409 | /* | 435 | /* |
410 | ** Loading bytecode with an extra header is disabled for security | 436 | ** Loading bytecode with an extra header is disabled for security |
@@ -426,55 +452,60 @@ void lj_lex_cleanup(lua_State *L, LexState *ls) | |||
426 | global_State *g = G(L); | 452 | global_State *g = G(L); |
427 | lj_mem_freevec(g, ls->bcstack, ls->sizebcstack, BCInsLine); | 453 | lj_mem_freevec(g, ls->bcstack, ls->sizebcstack, BCInsLine); |
428 | lj_mem_freevec(g, ls->vstack, ls->sizevstack, VarInfo); | 454 | lj_mem_freevec(g, ls->vstack, ls->sizevstack, VarInfo); |
429 | lj_str_freebuf(g, &ls->sb); | 455 | lj_buf_free(g, &ls->sb); |
430 | } | 456 | } |
431 | 457 | ||
458 | /* Return next lexical token. */ | ||
432 | void lj_lex_next(LexState *ls) | 459 | void lj_lex_next(LexState *ls) |
433 | { | 460 | { |
434 | ls->lastline = ls->linenumber; | 461 | ls->lastline = ls->linenumber; |
435 | if (LJ_LIKELY(ls->lookahead == TK_eof)) { /* No lookahead token? */ | 462 | if (LJ_LIKELY(ls->lookahead == TK_eof)) { /* No lookahead token? */ |
436 | ls->token = llex(ls, &ls->tokenval); /* Get next token. */ | 463 | ls->tok = lex_scan(ls, &ls->tokval); /* Get next token. */ |
437 | } else { /* Otherwise return lookahead token. */ | 464 | } else { /* Otherwise return lookahead token. */ |
438 | ls->token = ls->lookahead; | 465 | ls->tok = ls->lookahead; |
439 | ls->lookahead = TK_eof; | 466 | ls->lookahead = TK_eof; |
440 | ls->tokenval = ls->lookaheadval; | 467 | ls->tokval = ls->lookaheadval; |
441 | } | 468 | } |
442 | } | 469 | } |
443 | 470 | ||
471 | /* Look ahead for the next token. */ | ||
444 | LexToken lj_lex_lookahead(LexState *ls) | 472 | LexToken lj_lex_lookahead(LexState *ls) |
445 | { | 473 | { |
446 | lua_assert(ls->lookahead == TK_eof); | 474 | lj_assertLS(ls->lookahead == TK_eof, "double lookahead"); |
447 | ls->lookahead = llex(ls, &ls->lookaheadval); | 475 | ls->lookahead = lex_scan(ls, &ls->lookaheadval); |
448 | return ls->lookahead; | 476 | return ls->lookahead; |
449 | } | 477 | } |
450 | 478 | ||
451 | const char *lj_lex_token2str(LexState *ls, LexToken token) | 479 | /* Convert token to string. */ |
480 | const char *lj_lex_token2str(LexState *ls, LexToken tok) | ||
452 | { | 481 | { |
453 | if (token > TK_OFS) | 482 | if (tok > TK_OFS) |
454 | return tokennames[token-TK_OFS-1]; | 483 | return tokennames[tok-TK_OFS-1]; |
455 | else if (!lj_char_iscntrl(token)) | 484 | else if (!lj_char_iscntrl(tok)) |
456 | return lj_str_pushf(ls->L, "%c", token); | 485 | return lj_strfmt_pushf(ls->L, "%c", tok); |
457 | else | 486 | else |
458 | return lj_str_pushf(ls->L, "char(%d)", token); | 487 | return lj_strfmt_pushf(ls->L, "char(%d)", tok); |
459 | } | 488 | } |
460 | 489 | ||
461 | void lj_lex_error(LexState *ls, LexToken token, ErrMsg em, ...) | 490 | /* Lexer error. */ |
491 | void lj_lex_error(LexState *ls, LexToken tok, ErrMsg em, ...) | ||
462 | { | 492 | { |
463 | const char *tok; | 493 | const char *tokstr; |
464 | va_list argp; | 494 | va_list argp; |
465 | if (token == 0) { | 495 | if (tok == 0) { |
466 | tok = NULL; | 496 | tokstr = NULL; |
467 | } else if (token == TK_name || token == TK_string || token == TK_number) { | 497 | } else if (tok == TK_name || tok == TK_string || tok == TK_number) { |
468 | save(ls, '\0'); | 498 | lex_save(ls, '\0'); |
469 | tok = ls->sb.buf; | 499 | tokstr = sbufB(&ls->sb); |
470 | } else { | 500 | } else { |
471 | tok = lj_lex_token2str(ls, token); | 501 | tokstr = lj_lex_token2str(ls, tok); |
472 | } | 502 | } |
473 | va_start(argp, em); | 503 | va_start(argp, em); |
474 | lj_err_lex(ls->L, ls->chunkname, tok, ls->linenumber, em, argp); | 504 | lj_err_lex(ls->L, ls->chunkname, tokstr, ls->linenumber, em, argp); |
475 | va_end(argp); | 505 | va_end(argp); |
476 | } | 506 | } |
477 | 507 | ||
508 | /* Initialize strings for reserved words. */ | ||
478 | void lj_lex_init(lua_State *L) | 509 | void lj_lex_init(lua_State *L) |
479 | { | 510 | { |
480 | uint32_t i; | 511 | uint32_t i; |