aboutsummaryrefslogtreecommitdiff
path: root/src/lj_lex.c
diff options
context:
space:
mode:
Diffstat (limited to 'src/lj_lex.c')
-rw-r--r--src/lj_lex.c374
1 files changed, 201 insertions, 173 deletions
diff --git a/src/lj_lex.c b/src/lj_lex.c
index ecfcc015..1c9923cd 100644
--- a/src/lj_lex.c
+++ b/src/lj_lex.c
@@ -12,6 +12,7 @@
12#include "lj_obj.h" 12#include "lj_obj.h"
13#include "lj_gc.h" 13#include "lj_gc.h"
14#include "lj_err.h" 14#include "lj_err.h"
15#include "lj_buf.h"
15#include "lj_str.h" 16#include "lj_str.h"
16#if LJ_HASFFI 17#if LJ_HASFFI
17#include "lj_tab.h" 18#include "lj_tab.h"
@@ -24,6 +25,7 @@
24#include "lj_parse.h" 25#include "lj_parse.h"
25#include "lj_char.h" 26#include "lj_char.h"
26#include "lj_strscan.h" 27#include "lj_strscan.h"
28#include "lj_strfmt.h"
27 29
28/* Lua lexer token names. */ 30/* Lua lexer token names. */
29static const char *const tokennames[] = { 31static const char *const tokennames[] = {
@@ -37,54 +39,53 @@ TKDEF(TKSTR1, TKSTR2)
37 39
38/* -- Buffer handling ----------------------------------------------------- */ 40/* -- Buffer handling ----------------------------------------------------- */
39 41
40#define char2int(c) ((int)(uint8_t)(c)) 42#define LEX_EOF (-1)
41#define next(ls) \ 43#define lex_iseol(ls) (ls->c == '\n' || ls->c == '\r')
42 (ls->current = (ls->n--) > 0 ? char2int(*ls->p++) : fillbuf(ls))
43#define save_and_next(ls) (save(ls, ls->current), next(ls))
44#define currIsNewline(ls) (ls->current == '\n' || ls->current == '\r')
45#define END_OF_STREAM (-1)
46 44
47static int fillbuf(LexState *ls) 45/* Get more input from reader. */
46static LJ_NOINLINE LexChar lex_more(LexState *ls)
48{ 47{
49 size_t sz; 48 size_t sz;
50 const char *buf = ls->rfunc(ls->L, ls->rdata, &sz); 49 const char *p = ls->rfunc(ls->L, ls->rdata, &sz);
51 if (buf == NULL || sz == 0) return END_OF_STREAM; 50 if (p == NULL || sz == 0) return LEX_EOF;
52 if (sz >= LJ_MAX_MEM) { 51 if (sz >= LJ_MAX_BUF) {
53 if (sz != ~(size_t)0) lj_err_mem(ls->L); 52 if (sz != ~(size_t)0) lj_err_mem(ls->L);
53 sz = ~(uintptr_t)0 - (uintptr_t)p;
54 ls->endmark = 1; 54 ls->endmark = 1;
55 } 55 }
56 ls->n = (MSize)sz - 1; 56 ls->pe = p + sz;
57 ls->p = buf; 57 ls->p = p + 1;
58 return char2int(*(ls->p++)); 58 return (LexChar)(uint8_t)p[0];
59} 59}
60 60
61static LJ_NOINLINE void save_grow(LexState *ls, int c) 61/* Get next character. */
62static LJ_AINLINE LexChar lex_next(LexState *ls)
62{ 63{
63 MSize newsize; 64 return (ls->c = ls->p < ls->pe ? (LexChar)(uint8_t)*ls->p++ : lex_more(ls));
64 if (ls->sb.sz >= LJ_MAX_STR/2)
65 lj_lex_error(ls, 0, LJ_ERR_XELEM);
66 newsize = ls->sb.sz * 2;
67 lj_str_resizebuf(ls->L, &ls->sb, newsize);
68 ls->sb.buf[ls->sb.n++] = (char)c;
69} 65}
70 66
71static LJ_AINLINE void save(LexState *ls, int c) 67/* Save character. */
68static LJ_AINLINE void lex_save(LexState *ls, LexChar c)
72{ 69{
73 if (LJ_UNLIKELY(ls->sb.n + 1 > ls->sb.sz)) 70 lj_buf_putb(&ls->sb, c);
74 save_grow(ls, c);
75 else
76 ls->sb.buf[ls->sb.n++] = (char)c;
77} 71}
78 72
79static void inclinenumber(LexState *ls) 73/* Save previous character and get next character. */
74static LJ_AINLINE LexChar lex_savenext(LexState *ls)
80{ 75{
81 int old = ls->current; 76 lex_save(ls, ls->c);
82 lua_assert(currIsNewline(ls)); 77 return lex_next(ls);
83 next(ls); /* skip `\n' or `\r' */ 78}
84 if (currIsNewline(ls) && ls->current != old) 79
85 next(ls); /* skip `\n\r' or `\r\n' */ 80/* Skip line break. Handles "\n", "\r", "\r\n" or "\n\r". */
81static void lex_newline(LexState *ls)
82{
83 LexChar old = ls->c;
84 lua_assert(lex_iseol(ls));
85 lex_next(ls); /* Skip "\n" or "\r". */
86 if (lex_iseol(ls) && ls->c != old) lex_next(ls); /* Skip "\n\r" or "\r\n". */
86 if (++ls->linenumber >= LJ_MAX_LINE) 87 if (++ls->linenumber >= LJ_MAX_LINE)
87 lj_lex_error(ls, ls->token, LJ_ERR_XLINES); 88 lj_lex_error(ls, ls->tok, LJ_ERR_XLINES);
88} 89}
89 90
90/* -- Scanner for terminals ----------------------------------------------- */ 91/* -- Scanner for terminals ----------------------------------------------- */
@@ -93,19 +94,17 @@ static void inclinenumber(LexState *ls)
93static void lex_number(LexState *ls, TValue *tv) 94static void lex_number(LexState *ls, TValue *tv)
94{ 95{
95 StrScanFmt fmt; 96 StrScanFmt fmt;
96 int c, xp = 'e'; 97 LexChar c, xp = 'e';
97 lua_assert(lj_char_isdigit(ls->current)); 98 lua_assert(lj_char_isdigit(ls->c));
98 if ((c = ls->current) == '0') { 99 if ((c = ls->c) == '0' && (lex_savenext(ls) | 0x20) == 'x')
99 save_and_next(ls); 100 xp = 'p';
100 if ((ls->current | 0x20) == 'x') xp = 'p'; 101 while (lj_char_isident(ls->c) || ls->c == '.' ||
101 } 102 ((ls->c == '-' || ls->c == '+') && (c | 0x20) == xp)) {
102 while (lj_char_isident(ls->current) || ls->current == '.' || 103 c = ls->c;
103 ((ls->current == '-' || ls->current == '+') && (c | 0x20) == xp)) { 104 lex_savenext(ls);
104 c = ls->current;
105 save_and_next(ls);
106 } 105 }
107 save(ls, '\0'); 106 lex_save(ls, '\0');
108 fmt = lj_strscan_scan((const uint8_t *)ls->sb.buf, tv, 107 fmt = lj_strscan_scan((const uint8_t *)sbufB(&ls->sb), tv,
109 (LJ_DUALNUM ? STRSCAN_OPT_TOINT : STRSCAN_OPT_TONUM) | 108 (LJ_DUALNUM ? STRSCAN_OPT_TOINT : STRSCAN_OPT_TONUM) |
110 (LJ_HASFFI ? (STRSCAN_OPT_LL|STRSCAN_OPT_IMAG) : 0)); 109 (LJ_HASFFI ? (STRSCAN_OPT_LL|STRSCAN_OPT_IMAG) : 0));
111 if (LJ_DUALNUM && fmt == STRSCAN_INT) { 110 if (LJ_DUALNUM && fmt == STRSCAN_INT) {
@@ -138,60 +137,60 @@ static void lex_number(LexState *ls, TValue *tv)
138 } 137 }
139} 138}
140 139
141static int skip_sep(LexState *ls) 140/* Skip equal signs for "[=...=[" and "]=...=]" and return their count. */
141static int lex_skipeq(LexState *ls)
142{ 142{
143 int count = 0; 143 int count = 0;
144 int s = ls->current; 144 LexChar s = ls->c;
145 lua_assert(s == '[' || s == ']'); 145 lua_assert(s == '[' || s == ']');
146 save_and_next(ls); 146 while (lex_savenext(ls) == '=' && count < 0x20000000)
147 while (ls->current == '=' && count < 0x20000000) {
148 save_and_next(ls);
149 count++; 147 count++;
150 } 148 return (ls->c == s) ? count : (-count) - 1;
151 return (ls->current == s) ? count : (-count) - 1;
152} 149}
153 150
154static void read_long_string(LexState *ls, TValue *tv, int sep) 151/* Parse a long string or long comment (tv set to NULL). */
152static void lex_longstring(LexState *ls, TValue *tv, int sep)
155{ 153{
156 save_and_next(ls); /* skip 2nd `[' */ 154 lex_savenext(ls); /* Skip second '['. */
157 if (currIsNewline(ls)) /* string starts with a newline? */ 155 if (lex_iseol(ls)) /* Skip initial newline. */
158 inclinenumber(ls); /* skip it */ 156 lex_newline(ls);
159 for (;;) { 157 for (;;) {
160 switch (ls->current) { 158 switch (ls->c) {
161 case END_OF_STREAM: 159 case LEX_EOF:
162 lj_lex_error(ls, TK_eof, tv ? LJ_ERR_XLSTR : LJ_ERR_XLCOM); 160 lj_lex_error(ls, TK_eof, tv ? LJ_ERR_XLSTR : LJ_ERR_XLCOM);
163 break; 161 break;
164 case ']': 162 case ']':
165 if (skip_sep(ls) == sep) { 163 if (lex_skipeq(ls) == sep) {
166 save_and_next(ls); /* skip 2nd `]' */ 164 lex_savenext(ls); /* Skip second ']'. */
167 goto endloop; 165 goto endloop;
168 } 166 }
169 break; 167 break;
170 case '\n': 168 case '\n':
171 case '\r': 169 case '\r':
172 save(ls, '\n'); 170 lex_save(ls, '\n');
173 inclinenumber(ls); 171 lex_newline(ls);
174 if (!tv) lj_str_resetbuf(&ls->sb); /* avoid wasting space */ 172 if (!tv) lj_buf_reset(&ls->sb); /* Don't waste space for comments. */
175 break; 173 break;
176 default: 174 default:
177 if (tv) save_and_next(ls); 175 lex_savenext(ls);
178 else next(ls);
179 break; 176 break;
180 } 177 }
181 } endloop: 178 } endloop:
182 if (tv) { 179 if (tv) {
183 GCstr *str = lj_parse_keepstr(ls, ls->sb.buf + (2 + (MSize)sep), 180 GCstr *str = lj_parse_keepstr(ls, sbufB(&ls->sb) + (2 + (MSize)sep),
184 ls->sb.n - 2*(2 + (MSize)sep)); 181 sbuflen(&ls->sb) - 2*(2 + (MSize)sep));
185 setstrV(ls->L, tv, str); 182 setstrV(ls->L, tv, str);
186 } 183 }
187} 184}
188 185
189static void read_string(LexState *ls, int delim, TValue *tv) 186/* Parse a string. */
187static void lex_string(LexState *ls, TValue *tv)
190{ 188{
191 save_and_next(ls); 189 LexChar delim = ls->c; /* Delimiter is '\'' or '"'. */
192 while (ls->current != delim) { 190 lex_savenext(ls);
193 switch (ls->current) { 191 while (ls->c != delim) {
194 case END_OF_STREAM: 192 switch (ls->c) {
193 case LEX_EOF:
195 lj_lex_error(ls, TK_eof, LJ_ERR_XSTR); 194 lj_lex_error(ls, TK_eof, LJ_ERR_XSTR);
196 continue; 195 continue;
197 case '\n': 196 case '\n':
@@ -199,7 +198,7 @@ static void read_string(LexState *ls, int delim, TValue *tv)
199 lj_lex_error(ls, TK_string, LJ_ERR_XSTR); 198 lj_lex_error(ls, TK_string, LJ_ERR_XSTR);
200 continue; 199 continue;
201 case '\\': { 200 case '\\': {
202 int c = next(ls); /* Skip the '\\'. */ 201 LexChar c = lex_next(ls); /* Skip the '\\'. */
203 switch (c) { 202 switch (c) {
204 case 'a': c = '\a'; break; 203 case 'a': c = '\a'; break;
205 case 'b': c = '\b'; break; 204 case 'b': c = '\b'; break;
@@ -209,111 +208,139 @@ static void read_string(LexState *ls, int delim, TValue *tv)
209 case 't': c = '\t'; break; 208 case 't': c = '\t'; break;
210 case 'v': c = '\v'; break; 209 case 'v': c = '\v'; break;
211 case 'x': /* Hexadecimal escape '\xXX'. */ 210 case 'x': /* Hexadecimal escape '\xXX'. */
212 c = (next(ls) & 15u) << 4; 211 c = (lex_next(ls) & 15u) << 4;
213 if (!lj_char_isdigit(ls->current)) { 212 if (!lj_char_isdigit(ls->c)) {
214 if (!lj_char_isxdigit(ls->current)) goto err_xesc; 213 if (!lj_char_isxdigit(ls->c)) goto err_xesc;
215 c += 9 << 4; 214 c += 9 << 4;
216 } 215 }
217 c += (next(ls) & 15u); 216 c += (lex_next(ls) & 15u);
218 if (!lj_char_isdigit(ls->current)) { 217 if (!lj_char_isdigit(ls->c)) {
219 if (!lj_char_isxdigit(ls->current)) goto err_xesc; 218 if (!lj_char_isxdigit(ls->c)) goto err_xesc;
220 c += 9; 219 c += 9;
221 } 220 }
222 break; 221 break;
222 case 'u': /* Unicode escape '\u{XX...}'. */
223 if (lex_next(ls) != '{') goto err_xesc;
224 lex_next(ls);
225 c = 0;
226 do {
227 c = (c << 4) | (ls->c & 15u);
228 if (!lj_char_isdigit(ls->c)) {
229 if (!lj_char_isxdigit(ls->c)) goto err_xesc;
230 c += 9;
231 }
232 if (c >= 0x110000) goto err_xesc; /* Out of Unicode range. */
233 } while (lex_next(ls) != '}');
234 if (c < 0x800) {
235 if (c < 0x80) break;
236 lex_save(ls, 0xc0 | (c >> 6));
237 } else {
238 if (c >= 0x10000) {
239 lex_save(ls, 0xf0 | (c >> 18));
240 lex_save(ls, 0x80 | ((c >> 12) & 0x3f));
241 } else {
242 if (c >= 0xd800 && c < 0xe000) goto err_xesc; /* No surrogates. */
243 lex_save(ls, 0xe0 | (c >> 12));
244 }
245 lex_save(ls, 0x80 | ((c >> 6) & 0x3f));
246 }
247 c = 0x80 | (c & 0x3f);
248 break;
223 case 'z': /* Skip whitespace. */ 249 case 'z': /* Skip whitespace. */
224 next(ls); 250 lex_next(ls);
225 while (lj_char_isspace(ls->current)) 251 while (lj_char_isspace(ls->c))
226 if (currIsNewline(ls)) inclinenumber(ls); else next(ls); 252 if (lex_iseol(ls)) lex_newline(ls); else lex_next(ls);
227 continue; 253 continue;
228 case '\n': case '\r': save(ls, '\n'); inclinenumber(ls); continue; 254 case '\n': case '\r': lex_save(ls, '\n'); lex_newline(ls); continue;
229 case '\\': case '\"': case '\'': break; 255 case '\\': case '\"': case '\'': break;
230 case END_OF_STREAM: continue; 256 case LEX_EOF: continue;
231 default: 257 default:
232 if (!lj_char_isdigit(c)) 258 if (!lj_char_isdigit(c))
233 goto err_xesc; 259 goto err_xesc;
234 c -= '0'; /* Decimal escape '\ddd'. */ 260 c -= '0'; /* Decimal escape '\ddd'. */
235 if (lj_char_isdigit(next(ls))) { 261 if (lj_char_isdigit(lex_next(ls))) {
236 c = c*10 + (ls->current - '0'); 262 c = c*10 + (ls->c - '0');
237 if (lj_char_isdigit(next(ls))) { 263 if (lj_char_isdigit(lex_next(ls))) {
238 c = c*10 + (ls->current - '0'); 264 c = c*10 + (ls->c - '0');
239 if (c > 255) { 265 if (c > 255) {
240 err_xesc: 266 err_xesc:
241 lj_lex_error(ls, TK_string, LJ_ERR_XESC); 267 lj_lex_error(ls, TK_string, LJ_ERR_XESC);
242 } 268 }
243 next(ls); 269 lex_next(ls);
244 } 270 }
245 } 271 }
246 save(ls, c); 272 lex_save(ls, c);
247 continue; 273 continue;
248 } 274 }
249 save(ls, c); 275 lex_save(ls, c);
250 next(ls); 276 lex_next(ls);
251 continue; 277 continue;
252 } 278 }
253 default: 279 default:
254 save_and_next(ls); 280 lex_savenext(ls);
255 break; 281 break;
256 } 282 }
257 } 283 }
258 save_and_next(ls); /* skip delimiter */ 284 lex_savenext(ls); /* Skip trailing delimiter. */
259 setstrV(ls->L, tv, lj_parse_keepstr(ls, ls->sb.buf + 1, ls->sb.n - 2)); 285 setstrV(ls->L, tv,
286 lj_parse_keepstr(ls, sbufB(&ls->sb)+1, sbuflen(&ls->sb)-2));
260} 287}
261 288
262/* -- Main lexical scanner ------------------------------------------------ */ 289/* -- Main lexical scanner ------------------------------------------------ */
263 290
264static int llex(LexState *ls, TValue *tv) 291/* Get next lexical token. */
292static LexToken lex_scan(LexState *ls, TValue *tv)
265{ 293{
266 lj_str_resetbuf(&ls->sb); 294 lj_buf_reset(&ls->sb);
267 for (;;) { 295 for (;;) {
268 if (lj_char_isident(ls->current)) { 296 if (lj_char_isident(ls->c)) {
269 GCstr *s; 297 GCstr *s;
270 if (lj_char_isdigit(ls->current)) { /* Numeric literal. */ 298 if (lj_char_isdigit(ls->c)) { /* Numeric literal. */
271 lex_number(ls, tv); 299 lex_number(ls, tv);
272 return TK_number; 300 return TK_number;
273 } 301 }
274 /* Identifier or reserved word. */ 302 /* Identifier or reserved word. */
275 do { 303 do {
276 save_and_next(ls); 304 lex_savenext(ls);
277 } while (lj_char_isident(ls->current)); 305 } while (lj_char_isident(ls->c));
278 s = lj_parse_keepstr(ls, ls->sb.buf, ls->sb.n); 306 s = lj_parse_keepstr(ls, sbufB(&ls->sb), sbuflen(&ls->sb));
279 setstrV(ls->L, tv, s); 307 setstrV(ls->L, tv, s);
280 if (s->reserved > 0) /* Reserved word? */ 308 if (s->reserved > 0) /* Reserved word? */
281 return TK_OFS + s->reserved; 309 return TK_OFS + s->reserved;
282 return TK_name; 310 return TK_name;
283 } 311 }
284 switch (ls->current) { 312 switch (ls->c) {
285 case '\n': 313 case '\n':
286 case '\r': 314 case '\r':
287 inclinenumber(ls); 315 lex_newline(ls);
288 continue; 316 continue;
289 case ' ': 317 case ' ':
290 case '\t': 318 case '\t':
291 case '\v': 319 case '\v':
292 case '\f': 320 case '\f':
293 next(ls); 321 lex_next(ls);
294 continue; 322 continue;
295 case '-': 323 case '-':
296 next(ls); 324 lex_next(ls);
297 if (ls->current != '-') return '-'; 325 if (ls->c != '-') return '-';
298 /* else is a comment */ 326 lex_next(ls);
299 next(ls); 327 if (ls->c == '[') { /* Long comment "--[=*[...]=*]". */
300 if (ls->current == '[') { 328 int sep = lex_skipeq(ls);
301 int sep = skip_sep(ls); 329 lj_buf_reset(&ls->sb); /* `lex_skipeq' may dirty the buffer */
302 lj_str_resetbuf(&ls->sb); /* `skip_sep' may dirty the buffer */
303 if (sep >= 0) { 330 if (sep >= 0) {
304 read_long_string(ls, NULL, sep); /* long comment */ 331 lex_longstring(ls, NULL, sep);
305 lj_str_resetbuf(&ls->sb); 332 lj_buf_reset(&ls->sb);
306 continue; 333 continue;
307 } 334 }
308 } 335 }
309 /* else short comment */ 336 /* Short comment "--.*\n". */
310 while (!currIsNewline(ls) && ls->current != END_OF_STREAM) 337 while (!lex_iseol(ls) && ls->c != LEX_EOF)
311 next(ls); 338 lex_next(ls);
312 continue; 339 continue;
313 case '[': { 340 case '[': {
314 int sep = skip_sep(ls); 341 int sep = lex_skipeq(ls);
315 if (sep >= 0) { 342 if (sep >= 0) {
316 read_long_string(ls, tv, sep); 343 lex_longstring(ls, tv, sep);
317 return TK_string; 344 return TK_string;
318 } else if (sep == -1) { 345 } else if (sep == -1) {
319 return '['; 346 return '[';
@@ -323,44 +350,43 @@ static int llex(LexState *ls, TValue *tv)
323 } 350 }
324 } 351 }
325 case '=': 352 case '=':
326 next(ls); 353 lex_next(ls);
327 if (ls->current != '=') return '='; else { next(ls); return TK_eq; } 354 if (ls->c != '=') return '='; else { lex_next(ls); return TK_eq; }
328 case '<': 355 case '<':
329 next(ls); 356 lex_next(ls);
330 if (ls->current != '=') return '<'; else { next(ls); return TK_le; } 357 if (ls->c != '=') return '<'; else { lex_next(ls); return TK_le; }
331 case '>': 358 case '>':
332 next(ls); 359 lex_next(ls);
333 if (ls->current != '=') return '>'; else { next(ls); return TK_ge; } 360 if (ls->c != '=') return '>'; else { lex_next(ls); return TK_ge; }
334 case '~': 361 case '~':
335 next(ls); 362 lex_next(ls);
336 if (ls->current != '=') return '~'; else { next(ls); return TK_ne; } 363 if (ls->c != '=') return '~'; else { lex_next(ls); return TK_ne; }
337 case ':': 364 case ':':
338 next(ls); 365 lex_next(ls);
339 if (ls->current != ':') return ':'; else { next(ls); return TK_label; } 366 if (ls->c != ':') return ':'; else { lex_next(ls); return TK_label; }
340 case '"': 367 case '"':
341 case '\'': 368 case '\'':
342 read_string(ls, ls->current, tv); 369 lex_string(ls, tv);
343 return TK_string; 370 return TK_string;
344 case '.': 371 case '.':
345 save_and_next(ls); 372 if (lex_savenext(ls) == '.') {
346 if (ls->current == '.') { 373 lex_next(ls);
347 next(ls); 374 if (ls->c == '.') {
348 if (ls->current == '.') { 375 lex_next(ls);
349 next(ls);
350 return TK_dots; /* ... */ 376 return TK_dots; /* ... */
351 } 377 }
352 return TK_concat; /* .. */ 378 return TK_concat; /* .. */
353 } else if (!lj_char_isdigit(ls->current)) { 379 } else if (!lj_char_isdigit(ls->c)) {
354 return '.'; 380 return '.';
355 } else { 381 } else {
356 lex_number(ls, tv); 382 lex_number(ls, tv);
357 return TK_number; 383 return TK_number;
358 } 384 }
359 case END_OF_STREAM: 385 case LEX_EOF:
360 return TK_eof; 386 return TK_eof;
361 default: { 387 default: {
362 int c = ls->current; 388 LexChar c = ls->c;
363 next(ls); 389 lex_next(ls);
364 return c; /* Single-char tokens (+ - / ...). */ 390 return c; /* Single-char tokens (+ - / ...). */
365 } 391 }
366 } 392 }
@@ -375,36 +401,33 @@ int lj_lex_setup(lua_State *L, LexState *ls)
375 int header = 0; 401 int header = 0;
376 ls->L = L; 402 ls->L = L;
377 ls->fs = NULL; 403 ls->fs = NULL;
378 ls->n = 0; 404 ls->pe = ls->p = NULL;
379 ls->p = NULL;
380 ls->vstack = NULL; 405 ls->vstack = NULL;
381 ls->sizevstack = 0; 406 ls->sizevstack = 0;
382 ls->vtop = 0; 407 ls->vtop = 0;
383 ls->bcstack = NULL; 408 ls->bcstack = NULL;
384 ls->sizebcstack = 0; 409 ls->sizebcstack = 0;
385 ls->token = 0; 410 ls->tok = 0;
386 ls->lookahead = TK_eof; /* No look-ahead token. */ 411 ls->lookahead = TK_eof; /* No look-ahead token. */
387 ls->linenumber = 1; 412 ls->linenumber = 1;
388 ls->lastline = 1; 413 ls->lastline = 1;
389 ls->endmark = 0; 414 ls->endmark = 0;
390 lj_str_resizebuf(ls->L, &ls->sb, LJ_MIN_SBUF); 415 lex_next(ls); /* Read-ahead first char. */
391 next(ls); /* Read-ahead first char. */ 416 if (ls->c == 0xef && ls->p + 2 <= ls->pe && (uint8_t)ls->p[0] == 0xbb &&
392 if (ls->current == 0xef && ls->n >= 2 && char2int(ls->p[0]) == 0xbb && 417 (uint8_t)ls->p[1] == 0xbf) { /* Skip UTF-8 BOM (if buffered). */
393 char2int(ls->p[1]) == 0xbf) { /* Skip UTF-8 BOM (if buffered). */
394 ls->n -= 2;
395 ls->p += 2; 418 ls->p += 2;
396 next(ls); 419 lex_next(ls);
397 header = 1; 420 header = 1;
398 } 421 }
399 if (ls->current == '#') { /* Skip POSIX #! header line. */ 422 if (ls->c == '#') { /* Skip POSIX #! header line. */
400 do { 423 do {
401 next(ls); 424 lex_next(ls);
402 if (ls->current == END_OF_STREAM) return 0; 425 if (ls->c == LEX_EOF) return 0;
403 } while (!currIsNewline(ls)); 426 } while (!lex_iseol(ls));
404 inclinenumber(ls); 427 lex_newline(ls);
405 header = 1; 428 header = 1;
406 } 429 }
407 if (ls->current == LUA_SIGNATURE[0]) { /* Bytecode dump. */ 430 if (ls->c == LUA_SIGNATURE[0]) { /* Bytecode dump. */
408 if (header) { 431 if (header) {
409 /* 432 /*
410 ** Loading bytecode with an extra header is disabled for security 433 ** Loading bytecode with an extra header is disabled for security
@@ -426,55 +449,60 @@ void lj_lex_cleanup(lua_State *L, LexState *ls)
426 global_State *g = G(L); 449 global_State *g = G(L);
427 lj_mem_freevec(g, ls->bcstack, ls->sizebcstack, BCInsLine); 450 lj_mem_freevec(g, ls->bcstack, ls->sizebcstack, BCInsLine);
428 lj_mem_freevec(g, ls->vstack, ls->sizevstack, VarInfo); 451 lj_mem_freevec(g, ls->vstack, ls->sizevstack, VarInfo);
429 lj_str_freebuf(g, &ls->sb); 452 lj_buf_free(g, &ls->sb);
430} 453}
431 454
455/* Return next lexical token. */
432void lj_lex_next(LexState *ls) 456void lj_lex_next(LexState *ls)
433{ 457{
434 ls->lastline = ls->linenumber; 458 ls->lastline = ls->linenumber;
435 if (LJ_LIKELY(ls->lookahead == TK_eof)) { /* No lookahead token? */ 459 if (LJ_LIKELY(ls->lookahead == TK_eof)) { /* No lookahead token? */
436 ls->token = llex(ls, &ls->tokenval); /* Get next token. */ 460 ls->tok = lex_scan(ls, &ls->tokval); /* Get next token. */
437 } else { /* Otherwise return lookahead token. */ 461 } else { /* Otherwise return lookahead token. */
438 ls->token = ls->lookahead; 462 ls->tok = ls->lookahead;
439 ls->lookahead = TK_eof; 463 ls->lookahead = TK_eof;
440 ls->tokenval = ls->lookaheadval; 464 ls->tokval = ls->lookaheadval;
441 } 465 }
442} 466}
443 467
468/* Look ahead for the next token. */
444LexToken lj_lex_lookahead(LexState *ls) 469LexToken lj_lex_lookahead(LexState *ls)
445{ 470{
446 lua_assert(ls->lookahead == TK_eof); 471 lua_assert(ls->lookahead == TK_eof);
447 ls->lookahead = llex(ls, &ls->lookaheadval); 472 ls->lookahead = lex_scan(ls, &ls->lookaheadval);
448 return ls->lookahead; 473 return ls->lookahead;
449} 474}
450 475
451const char *lj_lex_token2str(LexState *ls, LexToken token) 476/* Convert token to string. */
477const char *lj_lex_token2str(LexState *ls, LexToken tok)
452{ 478{
453 if (token > TK_OFS) 479 if (tok > TK_OFS)
454 return tokennames[token-TK_OFS-1]; 480 return tokennames[tok-TK_OFS-1];
455 else if (!lj_char_iscntrl(token)) 481 else if (!lj_char_iscntrl(tok))
456 return lj_str_pushf(ls->L, "%c", token); 482 return lj_strfmt_pushf(ls->L, "%c", tok);
457 else 483 else
458 return lj_str_pushf(ls->L, "char(%d)", token); 484 return lj_strfmt_pushf(ls->L, "char(%d)", tok);
459} 485}
460 486
461void lj_lex_error(LexState *ls, LexToken token, ErrMsg em, ...) 487/* Lexer error. */
488void lj_lex_error(LexState *ls, LexToken tok, ErrMsg em, ...)
462{ 489{
463 const char *tok; 490 const char *tokstr;
464 va_list argp; 491 va_list argp;
465 if (token == 0) { 492 if (tok == 0) {
466 tok = NULL; 493 tokstr = NULL;
467 } else if (token == TK_name || token == TK_string || token == TK_number) { 494 } else if (tok == TK_name || tok == TK_string || tok == TK_number) {
468 save(ls, '\0'); 495 lex_save(ls, '\0');
469 tok = ls->sb.buf; 496 tokstr = sbufB(&ls->sb);
470 } else { 497 } else {
471 tok = lj_lex_token2str(ls, token); 498 tokstr = lj_lex_token2str(ls, tok);
472 } 499 }
473 va_start(argp, em); 500 va_start(argp, em);
474 lj_err_lex(ls->L, ls->chunkname, tok, ls->linenumber, em, argp); 501 lj_err_lex(ls->L, ls->chunkname, tokstr, ls->linenumber, em, argp);
475 va_end(argp); 502 va_end(argp);
476} 503}
477 504
505/* Initialize strings for reserved words. */
478void lj_lex_init(lua_State *L) 506void lj_lex_init(lua_State *L)
479{ 507{
480 uint32_t i; 508 uint32_t i;