aboutsummaryrefslogtreecommitdiff
path: root/src/lj_lex.c
diff options
context:
space:
mode:
Diffstat (limited to 'src/lj_lex.c')
-rw-r--r--src/lj_lex.c385
1 files changed, 208 insertions, 177 deletions
diff --git a/src/lj_lex.c b/src/lj_lex.c
index ca942583..61c7ff43 100644
--- a/src/lj_lex.c
+++ b/src/lj_lex.c
@@ -12,6 +12,7 @@
12#include "lj_obj.h" 12#include "lj_obj.h"
13#include "lj_gc.h" 13#include "lj_gc.h"
14#include "lj_err.h" 14#include "lj_err.h"
15#include "lj_buf.h"
15#include "lj_str.h" 16#include "lj_str.h"
16#if LJ_HASFFI 17#if LJ_HASFFI
17#include "lj_tab.h" 18#include "lj_tab.h"
@@ -24,6 +25,7 @@
24#include "lj_parse.h" 25#include "lj_parse.h"
25#include "lj_char.h" 26#include "lj_char.h"
26#include "lj_strscan.h" 27#include "lj_strscan.h"
28#include "lj_strfmt.h"
27 29
28/* Lua lexer token names. */ 30/* Lua lexer token names. */
29static const char *const tokennames[] = { 31static const char *const tokennames[] = {
@@ -37,54 +39,54 @@ TKDEF(TKSTR1, TKSTR2)
37 39
38/* -- Buffer handling ----------------------------------------------------- */ 40/* -- Buffer handling ----------------------------------------------------- */
39 41
40#define char2int(c) ((int)(uint8_t)(c)) 42#define LEX_EOF (-1)
41#define next(ls) \ 43#define lex_iseol(ls) (ls->c == '\n' || ls->c == '\r')
42 (ls->current = (ls->n--) > 0 ? char2int(*ls->p++) : fillbuf(ls))
43#define save_and_next(ls) (save(ls, ls->current), next(ls))
44#define currIsNewline(ls) (ls->current == '\n' || ls->current == '\r')
45#define END_OF_STREAM (-1)
46 44
47static int fillbuf(LexState *ls) 45/* Get more input from reader. */
46static LJ_NOINLINE LexChar lex_more(LexState *ls)
48{ 47{
49 size_t sz; 48 size_t sz;
50 const char *buf = ls->rfunc(ls->L, ls->rdata, &sz); 49 const char *p = ls->rfunc(ls->L, ls->rdata, &sz);
51 if (buf == NULL || sz == 0) return END_OF_STREAM; 50 if (p == NULL || sz == 0) return LEX_EOF;
52 if (sz >= LJ_MAX_MEM) { 51 if (sz >= LJ_MAX_BUF) {
53 if (sz != ~(size_t)0) lj_err_mem(ls->L); 52 if (sz != ~(size_t)0) lj_err_mem(ls->L);
53 sz = ~(uintptr_t)0 - (uintptr_t)p;
54 if (sz >= LJ_MAX_BUF) sz = LJ_MAX_BUF-1;
54 ls->endmark = 1; 55 ls->endmark = 1;
55 } 56 }
56 ls->n = (MSize)sz - 1; 57 ls->pe = p + sz;
57 ls->p = buf; 58 ls->p = p + 1;
58 return char2int(*(ls->p++)); 59 return (LexChar)(uint8_t)p[0];
59} 60}
60 61
61static LJ_NOINLINE void save_grow(LexState *ls, int c) 62/* Get next character. */
63static LJ_AINLINE LexChar lex_next(LexState *ls)
62{ 64{
63 MSize newsize; 65 return (ls->c = ls->p < ls->pe ? (LexChar)(uint8_t)*ls->p++ : lex_more(ls));
64 if (ls->sb.sz >= LJ_MAX_STR/2)
65 lj_lex_error(ls, 0, LJ_ERR_XELEM);
66 newsize = ls->sb.sz * 2;
67 lj_str_resizebuf(ls->L, &ls->sb, newsize);
68 ls->sb.buf[ls->sb.n++] = (char)c;
69} 66}
70 67
71static LJ_AINLINE void save(LexState *ls, int c) 68/* Save character. */
69static LJ_AINLINE void lex_save(LexState *ls, LexChar c)
72{ 70{
73 if (LJ_UNLIKELY(ls->sb.n + 1 > ls->sb.sz)) 71 lj_buf_putb(&ls->sb, c);
74 save_grow(ls, c);
75 else
76 ls->sb.buf[ls->sb.n++] = (char)c;
77} 72}
78 73
79static void inclinenumber(LexState *ls) 74/* Save previous character and get next character. */
75static LJ_AINLINE LexChar lex_savenext(LexState *ls)
80{ 76{
81 int old = ls->current; 77 lex_save(ls, ls->c);
82 lua_assert(currIsNewline(ls)); 78 return lex_next(ls);
83 next(ls); /* skip `\n' or `\r' */ 79}
84 if (currIsNewline(ls) && ls->current != old) 80
85 next(ls); /* skip `\n\r' or `\r\n' */ 81/* Skip line break. Handles "\n", "\r", "\r\n" or "\n\r". */
82static void lex_newline(LexState *ls)
83{
84 LexChar old = ls->c;
85 lj_assertLS(lex_iseol(ls), "bad usage");
86 lex_next(ls); /* Skip "\n" or "\r". */
87 if (lex_iseol(ls) && ls->c != old) lex_next(ls); /* Skip "\n\r" or "\r\n". */
86 if (++ls->linenumber >= LJ_MAX_LINE) 88 if (++ls->linenumber >= LJ_MAX_LINE)
87 lj_lex_error(ls, ls->token, LJ_ERR_XLINES); 89 lj_lex_error(ls, ls->tok, LJ_ERR_XLINES);
88} 90}
89 91
90/* -- Scanner for terminals ----------------------------------------------- */ 92/* -- Scanner for terminals ----------------------------------------------- */
@@ -93,19 +95,17 @@ static void inclinenumber(LexState *ls)
93static void lex_number(LexState *ls, TValue *tv) 95static void lex_number(LexState *ls, TValue *tv)
94{ 96{
95 StrScanFmt fmt; 97 StrScanFmt fmt;
96 int c, xp = 'e'; 98 LexChar c, xp = 'e';
97 lua_assert(lj_char_isdigit(ls->current)); 99 lj_assertLS(lj_char_isdigit(ls->c), "bad usage");
98 if ((c = ls->current) == '0') { 100 if ((c = ls->c) == '0' && (lex_savenext(ls) | 0x20) == 'x')
99 save_and_next(ls); 101 xp = 'p';
100 if ((ls->current | 0x20) == 'x') xp = 'p'; 102 while (lj_char_isident(ls->c) || ls->c == '.' ||
101 } 103 ((ls->c == '-' || ls->c == '+') && (c | 0x20) == xp)) {
102 while (lj_char_isident(ls->current) || ls->current == '.' || 104 c = ls->c;
103 ((ls->current == '-' || ls->current == '+') && (c | 0x20) == xp)) { 105 lex_savenext(ls);
104 c = ls->current;
105 save_and_next(ls);
106 } 106 }
107 save(ls, '\0'); 107 lex_save(ls, '\0');
108 fmt = lj_strscan_scan((const uint8_t *)ls->sb.buf, tv, 108 fmt = lj_strscan_scan((const uint8_t *)sbufB(&ls->sb), sbuflen(&ls->sb)-1, tv,
109 (LJ_DUALNUM ? STRSCAN_OPT_TOINT : STRSCAN_OPT_TONUM) | 109 (LJ_DUALNUM ? STRSCAN_OPT_TOINT : STRSCAN_OPT_TONUM) |
110 (LJ_HASFFI ? (STRSCAN_OPT_LL|STRSCAN_OPT_IMAG) : 0)); 110 (LJ_HASFFI ? (STRSCAN_OPT_LL|STRSCAN_OPT_IMAG) : 0));
111 if (LJ_DUALNUM && fmt == STRSCAN_INT) { 111 if (LJ_DUALNUM && fmt == STRSCAN_INT) {
@@ -116,7 +116,8 @@ static void lex_number(LexState *ls, TValue *tv)
116 } else if (fmt != STRSCAN_ERROR) { 116 } else if (fmt != STRSCAN_ERROR) {
117 lua_State *L = ls->L; 117 lua_State *L = ls->L;
118 GCcdata *cd; 118 GCcdata *cd;
119 lua_assert(fmt == STRSCAN_I64 || fmt == STRSCAN_U64 || fmt == STRSCAN_IMAG); 119 lj_assertLS(fmt == STRSCAN_I64 || fmt == STRSCAN_U64 || fmt == STRSCAN_IMAG,
120 "unexpected number format %d", fmt);
120 if (!ctype_ctsG(G(L))) { 121 if (!ctype_ctsG(G(L))) {
121 ptrdiff_t oldtop = savestack(L, L->top); 122 ptrdiff_t oldtop = savestack(L, L->top);
122 luaopen_ffi(L); /* Load FFI library on-demand. */ 123 luaopen_ffi(L); /* Load FFI library on-demand. */
@@ -133,65 +134,66 @@ static void lex_number(LexState *ls, TValue *tv)
133 lj_parse_keepcdata(ls, tv, cd); 134 lj_parse_keepcdata(ls, tv, cd);
134#endif 135#endif
135 } else { 136 } else {
136 lua_assert(fmt == STRSCAN_ERROR); 137 lj_assertLS(fmt == STRSCAN_ERROR,
138 "unexpected number format %d", fmt);
137 lj_lex_error(ls, TK_number, LJ_ERR_XNUMBER); 139 lj_lex_error(ls, TK_number, LJ_ERR_XNUMBER);
138 } 140 }
139} 141}
140 142
141static int skip_sep(LexState *ls) 143/* Skip equal signs for "[=...=[" and "]=...=]" and return their count. */
144static int lex_skipeq(LexState *ls)
142{ 145{
143 int count = 0; 146 int count = 0;
144 int s = ls->current; 147 LexChar s = ls->c;
145 lua_assert(s == '[' || s == ']'); 148 lj_assertLS(s == '[' || s == ']', "bad usage");
146 save_and_next(ls); 149 while (lex_savenext(ls) == '=' && count < 0x20000000)
147 while (ls->current == '=' && count < 0x20000000) {
148 save_and_next(ls);
149 count++; 150 count++;
150 } 151 return (ls->c == s) ? count : (-count) - 1;
151 return (ls->current == s) ? count : (-count) - 1;
152} 152}
153 153
154static void read_long_string(LexState *ls, TValue *tv, int sep) 154/* Parse a long string or long comment (tv set to NULL). */
155static void lex_longstring(LexState *ls, TValue *tv, int sep)
155{ 156{
156 save_and_next(ls); /* skip 2nd `[' */ 157 lex_savenext(ls); /* Skip second '['. */
157 if (currIsNewline(ls)) /* string starts with a newline? */ 158 if (lex_iseol(ls)) /* Skip initial newline. */
158 inclinenumber(ls); /* skip it */ 159 lex_newline(ls);
159 for (;;) { 160 for (;;) {
160 switch (ls->current) { 161 switch (ls->c) {
161 case END_OF_STREAM: 162 case LEX_EOF:
162 lj_lex_error(ls, TK_eof, tv ? LJ_ERR_XLSTR : LJ_ERR_XLCOM); 163 lj_lex_error(ls, TK_eof, tv ? LJ_ERR_XLSTR : LJ_ERR_XLCOM);
163 break; 164 break;
164 case ']': 165 case ']':
165 if (skip_sep(ls) == sep) { 166 if (lex_skipeq(ls) == sep) {
166 save_and_next(ls); /* skip 2nd `]' */ 167 lex_savenext(ls); /* Skip second ']'. */
167 goto endloop; 168 goto endloop;
168 } 169 }
169 break; 170 break;
170 case '\n': 171 case '\n':
171 case '\r': 172 case '\r':
172 save(ls, '\n'); 173 lex_save(ls, '\n');
173 inclinenumber(ls); 174 lex_newline(ls);
174 if (!tv) lj_str_resetbuf(&ls->sb); /* avoid wasting space */ 175 if (!tv) lj_buf_reset(&ls->sb); /* Don't waste space for comments. */
175 break; 176 break;
176 default: 177 default:
177 if (tv) save_and_next(ls); 178 lex_savenext(ls);
178 else next(ls);
179 break; 179 break;
180 } 180 }
181 } endloop: 181 } endloop:
182 if (tv) { 182 if (tv) {
183 GCstr *str = lj_parse_keepstr(ls, ls->sb.buf + (2 + (MSize)sep), 183 GCstr *str = lj_parse_keepstr(ls, sbufB(&ls->sb) + (2 + (MSize)sep),
184 ls->sb.n - 2*(2 + (MSize)sep)); 184 sbuflen(&ls->sb) - 2*(2 + (MSize)sep));
185 setstrV(ls->L, tv, str); 185 setstrV(ls->L, tv, str);
186 } 186 }
187} 187}
188 188
189static void read_string(LexState *ls, int delim, TValue *tv) 189/* Parse a string. */
190static void lex_string(LexState *ls, TValue *tv)
190{ 191{
191 save_and_next(ls); 192 LexChar delim = ls->c; /* Delimiter is '\'' or '"'. */
192 while (ls->current != delim) { 193 lex_savenext(ls);
193 switch (ls->current) { 194 while (ls->c != delim) {
194 case END_OF_STREAM: 195 switch (ls->c) {
196 case LEX_EOF:
195 lj_lex_error(ls, TK_eof, LJ_ERR_XSTR); 197 lj_lex_error(ls, TK_eof, LJ_ERR_XSTR);
196 continue; 198 continue;
197 case '\n': 199 case '\n':
@@ -199,7 +201,7 @@ static void read_string(LexState *ls, int delim, TValue *tv)
199 lj_lex_error(ls, TK_string, LJ_ERR_XSTR); 201 lj_lex_error(ls, TK_string, LJ_ERR_XSTR);
200 continue; 202 continue;
201 case '\\': { 203 case '\\': {
202 int c = next(ls); /* Skip the '\\'. */ 204 LexChar c = lex_next(ls); /* Skip the '\\'. */
203 switch (c) { 205 switch (c) {
204 case 'a': c = '\a'; break; 206 case 'a': c = '\a'; break;
205 case 'b': c = '\b'; break; 207 case 'b': c = '\b'; break;
@@ -209,111 +211,139 @@ static void read_string(LexState *ls, int delim, TValue *tv)
209 case 't': c = '\t'; break; 211 case 't': c = '\t'; break;
210 case 'v': c = '\v'; break; 212 case 'v': c = '\v'; break;
211 case 'x': /* Hexadecimal escape '\xXX'. */ 213 case 'x': /* Hexadecimal escape '\xXX'. */
212 c = (next(ls) & 15u) << 4; 214 c = (lex_next(ls) & 15u) << 4;
213 if (!lj_char_isdigit(ls->current)) { 215 if (!lj_char_isdigit(ls->c)) {
214 if (!lj_char_isxdigit(ls->current)) goto err_xesc; 216 if (!lj_char_isxdigit(ls->c)) goto err_xesc;
215 c += 9 << 4; 217 c += 9 << 4;
216 } 218 }
217 c += (next(ls) & 15u); 219 c += (lex_next(ls) & 15u);
218 if (!lj_char_isdigit(ls->current)) { 220 if (!lj_char_isdigit(ls->c)) {
219 if (!lj_char_isxdigit(ls->current)) goto err_xesc; 221 if (!lj_char_isxdigit(ls->c)) goto err_xesc;
220 c += 9; 222 c += 9;
221 } 223 }
222 break; 224 break;
225 case 'u': /* Unicode escape '\u{XX...}'. */
226 if (lex_next(ls) != '{') goto err_xesc;
227 lex_next(ls);
228 c = 0;
229 do {
230 c = (c << 4) | (ls->c & 15u);
231 if (!lj_char_isdigit(ls->c)) {
232 if (!lj_char_isxdigit(ls->c)) goto err_xesc;
233 c += 9;
234 }
235 if (c >= 0x110000) goto err_xesc; /* Out of Unicode range. */
236 } while (lex_next(ls) != '}');
237 if (c < 0x800) {
238 if (c < 0x80) break;
239 lex_save(ls, 0xc0 | (c >> 6));
240 } else {
241 if (c >= 0x10000) {
242 lex_save(ls, 0xf0 | (c >> 18));
243 lex_save(ls, 0x80 | ((c >> 12) & 0x3f));
244 } else {
245 if (c >= 0xd800 && c < 0xe000) goto err_xesc; /* No surrogates. */
246 lex_save(ls, 0xe0 | (c >> 12));
247 }
248 lex_save(ls, 0x80 | ((c >> 6) & 0x3f));
249 }
250 c = 0x80 | (c & 0x3f);
251 break;
223 case 'z': /* Skip whitespace. */ 252 case 'z': /* Skip whitespace. */
224 next(ls); 253 lex_next(ls);
225 while (lj_char_isspace(ls->current)) 254 while (lj_char_isspace(ls->c))
226 if (currIsNewline(ls)) inclinenumber(ls); else next(ls); 255 if (lex_iseol(ls)) lex_newline(ls); else lex_next(ls);
227 continue; 256 continue;
228 case '\n': case '\r': save(ls, '\n'); inclinenumber(ls); continue; 257 case '\n': case '\r': lex_save(ls, '\n'); lex_newline(ls); continue;
229 case '\\': case '\"': case '\'': break; 258 case '\\': case '\"': case '\'': break;
230 case END_OF_STREAM: continue; 259 case LEX_EOF: continue;
231 default: 260 default:
232 if (!lj_char_isdigit(c)) 261 if (!lj_char_isdigit(c))
233 goto err_xesc; 262 goto err_xesc;
234 c -= '0'; /* Decimal escape '\ddd'. */ 263 c -= '0'; /* Decimal escape '\ddd'. */
235 if (lj_char_isdigit(next(ls))) { 264 if (lj_char_isdigit(lex_next(ls))) {
236 c = c*10 + (ls->current - '0'); 265 c = c*10 + (ls->c - '0');
237 if (lj_char_isdigit(next(ls))) { 266 if (lj_char_isdigit(lex_next(ls))) {
238 c = c*10 + (ls->current - '0'); 267 c = c*10 + (ls->c - '0');
239 if (c > 255) { 268 if (c > 255) {
240 err_xesc: 269 err_xesc:
241 lj_lex_error(ls, TK_string, LJ_ERR_XESC); 270 lj_lex_error(ls, TK_string, LJ_ERR_XESC);
242 } 271 }
243 next(ls); 272 lex_next(ls);
244 } 273 }
245 } 274 }
246 save(ls, c); 275 lex_save(ls, c);
247 continue; 276 continue;
248 } 277 }
249 save(ls, c); 278 lex_save(ls, c);
250 next(ls); 279 lex_next(ls);
251 continue; 280 continue;
252 } 281 }
253 default: 282 default:
254 save_and_next(ls); 283 lex_savenext(ls);
255 break; 284 break;
256 } 285 }
257 } 286 }
258 save_and_next(ls); /* skip delimiter */ 287 lex_savenext(ls); /* Skip trailing delimiter. */
259 setstrV(ls->L, tv, lj_parse_keepstr(ls, ls->sb.buf + 1, ls->sb.n - 2)); 288 setstrV(ls->L, tv,
289 lj_parse_keepstr(ls, sbufB(&ls->sb)+1, sbuflen(&ls->sb)-2));
260} 290}
261 291
262/* -- Main lexical scanner ------------------------------------------------ */ 292/* -- Main lexical scanner ------------------------------------------------ */
263 293
264static int llex(LexState *ls, TValue *tv) 294/* Get next lexical token. */
295static LexToken lex_scan(LexState *ls, TValue *tv)
265{ 296{
266 lj_str_resetbuf(&ls->sb); 297 lj_buf_reset(&ls->sb);
267 for (;;) { 298 for (;;) {
268 if (lj_char_isident(ls->current)) { 299 if (lj_char_isident(ls->c)) {
269 GCstr *s; 300 GCstr *s;
270 if (lj_char_isdigit(ls->current)) { /* Numeric literal. */ 301 if (lj_char_isdigit(ls->c)) { /* Numeric literal. */
271 lex_number(ls, tv); 302 lex_number(ls, tv);
272 return TK_number; 303 return TK_number;
273 } 304 }
274 /* Identifier or reserved word. */ 305 /* Identifier or reserved word. */
275 do { 306 do {
276 save_and_next(ls); 307 lex_savenext(ls);
277 } while (lj_char_isident(ls->current)); 308 } while (lj_char_isident(ls->c));
278 s = lj_parse_keepstr(ls, ls->sb.buf, ls->sb.n); 309 s = lj_parse_keepstr(ls, sbufB(&ls->sb), sbuflen(&ls->sb));
279 setstrV(ls->L, tv, s); 310 setstrV(ls->L, tv, s);
280 if (s->reserved > 0) /* Reserved word? */ 311 if (s->reserved > 0) /* Reserved word? */
281 return TK_OFS + s->reserved; 312 return TK_OFS + s->reserved;
282 return TK_name; 313 return TK_name;
283 } 314 }
284 switch (ls->current) { 315 switch (ls->c) {
285 case '\n': 316 case '\n':
286 case '\r': 317 case '\r':
287 inclinenumber(ls); 318 lex_newline(ls);
288 continue; 319 continue;
289 case ' ': 320 case ' ':
290 case '\t': 321 case '\t':
291 case '\v': 322 case '\v':
292 case '\f': 323 case '\f':
293 next(ls); 324 lex_next(ls);
294 continue; 325 continue;
295 case '-': 326 case '-':
296 next(ls); 327 lex_next(ls);
297 if (ls->current != '-') return '-'; 328 if (ls->c != '-') return '-';
298 /* else is a comment */ 329 lex_next(ls);
299 next(ls); 330 if (ls->c == '[') { /* Long comment "--[=*[...]=*]". */
300 if (ls->current == '[') { 331 int sep = lex_skipeq(ls);
301 int sep = skip_sep(ls); 332 lj_buf_reset(&ls->sb); /* `lex_skipeq' may dirty the buffer */
302 lj_str_resetbuf(&ls->sb); /* `skip_sep' may dirty the buffer */
303 if (sep >= 0) { 333 if (sep >= 0) {
304 read_long_string(ls, NULL, sep); /* long comment */ 334 lex_longstring(ls, NULL, sep);
305 lj_str_resetbuf(&ls->sb); 335 lj_buf_reset(&ls->sb);
306 continue; 336 continue;
307 } 337 }
308 } 338 }
309 /* else short comment */ 339 /* Short comment "--.*\n". */
310 while (!currIsNewline(ls) && ls->current != END_OF_STREAM) 340 while (!lex_iseol(ls) && ls->c != LEX_EOF)
311 next(ls); 341 lex_next(ls);
312 continue; 342 continue;
313 case '[': { 343 case '[': {
314 int sep = skip_sep(ls); 344 int sep = lex_skipeq(ls);
315 if (sep >= 0) { 345 if (sep >= 0) {
316 read_long_string(ls, tv, sep); 346 lex_longstring(ls, tv, sep);
317 return TK_string; 347 return TK_string;
318 } else if (sep == -1) { 348 } else if (sep == -1) {
319 return '['; 349 return '[';
@@ -323,44 +353,43 @@ static int llex(LexState *ls, TValue *tv)
323 } 353 }
324 } 354 }
325 case '=': 355 case '=':
326 next(ls); 356 lex_next(ls);
327 if (ls->current != '=') return '='; else { next(ls); return TK_eq; } 357 if (ls->c != '=') return '='; else { lex_next(ls); return TK_eq; }
328 case '<': 358 case '<':
329 next(ls); 359 lex_next(ls);
330 if (ls->current != '=') return '<'; else { next(ls); return TK_le; } 360 if (ls->c != '=') return '<'; else { lex_next(ls); return TK_le; }
331 case '>': 361 case '>':
332 next(ls); 362 lex_next(ls);
333 if (ls->current != '=') return '>'; else { next(ls); return TK_ge; } 363 if (ls->c != '=') return '>'; else { lex_next(ls); return TK_ge; }
334 case '~': 364 case '~':
335 next(ls); 365 lex_next(ls);
336 if (ls->current != '=') return '~'; else { next(ls); return TK_ne; } 366 if (ls->c != '=') return '~'; else { lex_next(ls); return TK_ne; }
337 case ':': 367 case ':':
338 next(ls); 368 lex_next(ls);
339 if (ls->current != ':') return ':'; else { next(ls); return TK_label; } 369 if (ls->c != ':') return ':'; else { lex_next(ls); return TK_label; }
340 case '"': 370 case '"':
341 case '\'': 371 case '\'':
342 read_string(ls, ls->current, tv); 372 lex_string(ls, tv);
343 return TK_string; 373 return TK_string;
344 case '.': 374 case '.':
345 save_and_next(ls); 375 if (lex_savenext(ls) == '.') {
346 if (ls->current == '.') { 376 lex_next(ls);
347 next(ls); 377 if (ls->c == '.') {
348 if (ls->current == '.') { 378 lex_next(ls);
349 next(ls);
350 return TK_dots; /* ... */ 379 return TK_dots; /* ... */
351 } 380 }
352 return TK_concat; /* .. */ 381 return TK_concat; /* .. */
353 } else if (!lj_char_isdigit(ls->current)) { 382 } else if (!lj_char_isdigit(ls->c)) {
354 return '.'; 383 return '.';
355 } else { 384 } else {
356 lex_number(ls, tv); 385 lex_number(ls, tv);
357 return TK_number; 386 return TK_number;
358 } 387 }
359 case END_OF_STREAM: 388 case LEX_EOF:
360 return TK_eof; 389 return TK_eof;
361 default: { 390 default: {
362 int c = ls->current; 391 LexChar c = ls->c;
363 next(ls); 392 lex_next(ls);
364 return c; /* Single-char tokens (+ - / ...). */ 393 return c; /* Single-char tokens (+ - / ...). */
365 } 394 }
366 } 395 }
@@ -375,36 +404,33 @@ int lj_lex_setup(lua_State *L, LexState *ls)
375 int header = 0; 404 int header = 0;
376 ls->L = L; 405 ls->L = L;
377 ls->fs = NULL; 406 ls->fs = NULL;
378 ls->n = 0; 407 ls->pe = ls->p = NULL;
379 ls->p = NULL;
380 ls->vstack = NULL; 408 ls->vstack = NULL;
381 ls->sizevstack = 0; 409 ls->sizevstack = 0;
382 ls->vtop = 0; 410 ls->vtop = 0;
383 ls->bcstack = NULL; 411 ls->bcstack = NULL;
384 ls->sizebcstack = 0; 412 ls->sizebcstack = 0;
385 ls->token = 0; 413 ls->tok = 0;
386 ls->lookahead = TK_eof; /* No look-ahead token. */ 414 ls->lookahead = TK_eof; /* No look-ahead token. */
387 ls->linenumber = 1; 415 ls->linenumber = 1;
388 ls->lastline = 1; 416 ls->lastline = 1;
389 ls->endmark = 0; 417 ls->endmark = 0;
390 lj_str_resizebuf(ls->L, &ls->sb, LJ_MIN_SBUF); 418 lex_next(ls); /* Read-ahead first char. */
391 next(ls); /* Read-ahead first char. */ 419 if (ls->c == 0xef && ls->p + 2 <= ls->pe && (uint8_t)ls->p[0] == 0xbb &&
392 if (ls->current == 0xef && ls->n >= 2 && char2int(ls->p[0]) == 0xbb && 420 (uint8_t)ls->p[1] == 0xbf) { /* Skip UTF-8 BOM (if buffered). */
393 char2int(ls->p[1]) == 0xbf) { /* Skip UTF-8 BOM (if buffered). */
394 ls->n -= 2;
395 ls->p += 2; 421 ls->p += 2;
396 next(ls); 422 lex_next(ls);
397 header = 1; 423 header = 1;
398 } 424 }
399 if (ls->current == '#') { /* Skip POSIX #! header line. */ 425 if (ls->c == '#') { /* Skip POSIX #! header line. */
400 do { 426 do {
401 next(ls); 427 lex_next(ls);
402 if (ls->current == END_OF_STREAM) return 0; 428 if (ls->c == LEX_EOF) return 0;
403 } while (!currIsNewline(ls)); 429 } while (!lex_iseol(ls));
404 inclinenumber(ls); 430 lex_newline(ls);
405 header = 1; 431 header = 1;
406 } 432 }
407 if (ls->current == LUA_SIGNATURE[0]) { /* Bytecode dump. */ 433 if (ls->c == LUA_SIGNATURE[0]) { /* Bytecode dump. */
408 if (header) { 434 if (header) {
409 /* 435 /*
410 ** Loading bytecode with an extra header is disabled for security 436 ** Loading bytecode with an extra header is disabled for security
@@ -426,55 +452,60 @@ void lj_lex_cleanup(lua_State *L, LexState *ls)
426 global_State *g = G(L); 452 global_State *g = G(L);
427 lj_mem_freevec(g, ls->bcstack, ls->sizebcstack, BCInsLine); 453 lj_mem_freevec(g, ls->bcstack, ls->sizebcstack, BCInsLine);
428 lj_mem_freevec(g, ls->vstack, ls->sizevstack, VarInfo); 454 lj_mem_freevec(g, ls->vstack, ls->sizevstack, VarInfo);
429 lj_str_freebuf(g, &ls->sb); 455 lj_buf_free(g, &ls->sb);
430} 456}
431 457
458/* Return next lexical token. */
432void lj_lex_next(LexState *ls) 459void lj_lex_next(LexState *ls)
433{ 460{
434 ls->lastline = ls->linenumber; 461 ls->lastline = ls->linenumber;
435 if (LJ_LIKELY(ls->lookahead == TK_eof)) { /* No lookahead token? */ 462 if (LJ_LIKELY(ls->lookahead == TK_eof)) { /* No lookahead token? */
436 ls->token = llex(ls, &ls->tokenval); /* Get next token. */ 463 ls->tok = lex_scan(ls, &ls->tokval); /* Get next token. */
437 } else { /* Otherwise return lookahead token. */ 464 } else { /* Otherwise return lookahead token. */
438 ls->token = ls->lookahead; 465 ls->tok = ls->lookahead;
439 ls->lookahead = TK_eof; 466 ls->lookahead = TK_eof;
440 ls->tokenval = ls->lookaheadval; 467 ls->tokval = ls->lookaheadval;
441 } 468 }
442} 469}
443 470
471/* Look ahead for the next token. */
444LexToken lj_lex_lookahead(LexState *ls) 472LexToken lj_lex_lookahead(LexState *ls)
445{ 473{
446 lua_assert(ls->lookahead == TK_eof); 474 lj_assertLS(ls->lookahead == TK_eof, "double lookahead");
447 ls->lookahead = llex(ls, &ls->lookaheadval); 475 ls->lookahead = lex_scan(ls, &ls->lookaheadval);
448 return ls->lookahead; 476 return ls->lookahead;
449} 477}
450 478
451const char *lj_lex_token2str(LexState *ls, LexToken token) 479/* Convert token to string. */
480const char *lj_lex_token2str(LexState *ls, LexToken tok)
452{ 481{
453 if (token > TK_OFS) 482 if (tok > TK_OFS)
454 return tokennames[token-TK_OFS-1]; 483 return tokennames[tok-TK_OFS-1];
455 else if (!lj_char_iscntrl(token)) 484 else if (!lj_char_iscntrl(tok))
456 return lj_str_pushf(ls->L, "%c", token); 485 return lj_strfmt_pushf(ls->L, "%c", tok);
457 else 486 else
458 return lj_str_pushf(ls->L, "char(%d)", token); 487 return lj_strfmt_pushf(ls->L, "char(%d)", tok);
459} 488}
460 489
461void lj_lex_error(LexState *ls, LexToken token, ErrMsg em, ...) 490/* Lexer error. */
491void lj_lex_error(LexState *ls, LexToken tok, ErrMsg em, ...)
462{ 492{
463 const char *tok; 493 const char *tokstr;
464 va_list argp; 494 va_list argp;
465 if (token == 0) { 495 if (tok == 0) {
466 tok = NULL; 496 tokstr = NULL;
467 } else if (token == TK_name || token == TK_string || token == TK_number) { 497 } else if (tok == TK_name || tok == TK_string || tok == TK_number) {
468 save(ls, '\0'); 498 lex_save(ls, '\0');
469 tok = ls->sb.buf; 499 tokstr = sbufB(&ls->sb);
470 } else { 500 } else {
471 tok = lj_lex_token2str(ls, token); 501 tokstr = lj_lex_token2str(ls, tok);
472 } 502 }
473 va_start(argp, em); 503 va_start(argp, em);
474 lj_err_lex(ls->L, ls->chunkname, tok, ls->linenumber, em, argp); 504 lj_err_lex(ls->L, ls->chunkname, tokstr, ls->linenumber, em, argp);
475 va_end(argp); 505 va_end(argp);
476} 506}
477 507
508/* Initialize strings for reserved words. */
478void lj_lex_init(lua_State *L) 509void lj_lex_init(lua_State *L)
479{ 510{
480 uint32_t i; 511 uint32_t i;