From 39b79783297bee79db9853b63d199e120a009a8f Mon Sep 17 00:00:00 2001 From: Roberto Ierusalimschy Date: Fri, 23 Feb 2001 14:17:25 -0300 Subject: first (big) step to support wide chars --- llex.c | 230 +++++++++++++++++++++++++++++++++-------------------------------- 1 file changed, 118 insertions(+), 112 deletions(-) (limited to 'llex.c') diff --git a/llex.c b/llex.c index 386ea316..0bbc43be 100644 --- a/llex.c +++ b/llex.c @@ -1,5 +1,5 @@ /* -** $Id: llex.c,v 1.78 2001/02/22 17:15:18 roberto Exp roberto $ +** $Id: llex.c,v 1.79 2001/02/22 18:59:59 roberto Exp roberto $ ** Lexical Analyzer ** See Copyright Notice in lua.h */ @@ -25,10 +25,13 @@ /* ORDER RESERVED */ -static const char *const token2string [] = { - "and", "break", "do", "else", "elseif", "end", "for", - "function", "if", "local", "nil", "not", "or", "repeat", "return", "then", - "until", "while", "", "..", "...", "==", ">=", "<=", "~=", "", "", ""}; +static const l_char *const token2string [] = { + l_s("and"), l_s("break"), l_s("do"), l_s("else"), l_s("elseif"), + l_s("end"), l_s("for"), l_s("function"), l_s("if"), l_s("local"), + l_s("nil"), l_s("not"), l_s("or"), l_s("repeat"), l_s("return"), + l_s("then"), l_s("until"), l_s("while"), l_s(""), l_s(".."), l_s("..."), + l_s("=="), l_s(">="), l_s("<="), l_s("~="), l_s(""), l_s(""), l_s("") +}; void luaX_init (lua_State *L) { @@ -44,37 +47,38 @@ void luaX_init (lua_State *L) { #define MAXSRC 80 -void luaX_checklimit (LexState *ls, int val, int limit, const char *msg) { +void luaX_checklimit (LexState *ls, int val, int limit, const l_char *msg) { if (val > limit) { - char buff[90]; - sprintf(buff, "too many %.40s (limit=%d)", msg, limit); + l_char buff[90]; + sprintf(buff, l_s("too many %.40s (limit=%d)"), msg, limit); luaX_error(ls, buff, ls->t.token); } } -void luaX_syntaxerror (LexState *ls, const char *s, const char *token) { - char buff[MAXSRC]; +void luaX_syntaxerror (LexState *ls, const l_char *s, const l_char *token) { + l_char buff[MAXSRC]; luaO_chunkid(buff, getstr(ls->source), sizeof(buff)); - luaO_verror(ls->L, "%.99s;\n last token read: `%.30s' at line %d in %.80s", + luaO_verror(ls->L, + l_s("%.99s;\n last token read: `%.30s' at line %d in %.80s"), s, token, ls->linenumber, buff); } -void luaX_error (LexState *ls, const char *s, int token) { - char buff[TOKEN_LEN]; +void luaX_error (LexState *ls, const l_char *s, int token) { + l_char buff[TOKEN_LEN]; luaX_token2str(token, buff); - if (buff[0] == '\0') + if (buff[0] == l_c('\0')) luaX_syntaxerror(ls, s, G(ls->L)->Mbuffer); else luaX_syntaxerror(ls, s, buff); } -void luaX_token2str (int token, char *s) { +void luaX_token2str (int token, l_char *s) { if (token < 256) { - s[0] = (char)token; - s[1] = '\0'; + s[0] = (l_char)token; + s[1] = l_c('\0'); } else strcpy(s, token2string[token-FIRST_RESERVED]); @@ -82,16 +86,16 @@ void luaX_token2str (int token, char *s) { static void luaX_invalidchar (LexState *ls, int c) { - char buff[8]; - sprintf(buff, "0x%02X", c); - luaX_syntaxerror(ls, "invalid control char", buff); + l_char buff[8]; + sprintf(buff, l_s("0x%02X"), c); + luaX_syntaxerror(ls, l_s("invalid control l_char"), buff); } static void inclinenumber (LexState *LS) { next(LS); /* skip `\n' */ ++LS->linenumber; - luaX_checklimit(LS, LS->linenumber, MAX_INT, "lines in a chunk"); + luaX_checklimit(LS, LS->linenumber, MAX_INT, l_s("lines in a chunk")); } @@ -104,10 +108,10 @@ void luaX_setinput (lua_State *L, LexState *LS, ZIO *z, TString *source) { LS->lastline = 1; LS->source = source; next(LS); /* read first char */ - if (LS->current == '#') { + if (LS->current == l_c('#')) { do { /* skip first line */ next(LS); - } while (LS->current != '\n' && LS->current != EOZ); + } while (LS->current != l_c('\n') && LS->current != EOZ); } } @@ -126,7 +130,7 @@ void luaX_setinput (lua_State *L, LexState *LS, ZIO *z, TString *source) { #define checkbuffer(L, n, len) if ((len)+(n) > G(L)->Mbuffsize) \ luaO_openspace(L, (len)+(n)+EXTRABUFF) -#define save(L, c, l) (G(L)->Mbuffer[l++] = (char)c) +#define save(L, c, l) (G(L)->Mbuffer[l++] = (l_char)c) #define save_and_next(L, LS, l) (save(L, LS->current, l), next(LS)) @@ -137,8 +141,8 @@ static size_t readname (LexState *LS) { do { checkbuffer(L, 10, l); save_and_next(L, LS, l); - } while (isalnum(LS->current) || LS->current == '_'); - save(L, '\0', l); + } while (isalnum(LS->current) || LS->current == l_c('_')); + save(L, l_c('\0'), l); return l-1; } @@ -148,36 +152,37 @@ static void read_number (LexState *LS, int comma, SemInfo *seminfo) { lua_State *L = LS->L; size_t l = 0; checkbuffer(L, 10, l); - if (comma) save(L, '.', l); + if (comma) save(L, l_c('.'), l); while (isdigit(LS->current)) { checkbuffer(L, 10, l); save_and_next(L, LS, l); } - if (LS->current == '.') { + if (LS->current == l_c('.')) { save_and_next(L, LS, l); - if (LS->current == '.') { + if (LS->current == l_c('.')) { save_and_next(L, LS, l); - save(L, '\0', l); - luaX_error(LS, "ambiguous syntax" - " (decimal point x string concatenation)", TK_NUMBER); + save(L, l_c('\0'), l); + luaX_error(LS, + l_s("ambiguous syntax (decimal point x string concatenation)"), + TK_NUMBER); } } while (isdigit(LS->current)) { checkbuffer(L, 10, l); save_and_next(L, LS, l); } - if (LS->current == 'e' || LS->current == 'E') { + if (LS->current == l_c('e') || LS->current == l_c('E')) { save_and_next(L, LS, l); /* read `E' */ - if (LS->current == '+' || LS->current == '-') + if (LS->current == l_c('+') || LS->current == l_c('-')) save_and_next(L, LS, l); /* optional exponent sign */ while (isdigit(LS->current)) { checkbuffer(L, 10, l); save_and_next(L, LS, l); } } - save(L, '\0', l); + save(L, l_c('\0'), l); if (!luaO_str2d(G(L)->Mbuffer, &seminfo->r)) - luaX_error(LS, "malformed number", TK_NUMBER); + luaX_error(LS, l_s("malformed number"), TK_NUMBER); } @@ -186,32 +191,32 @@ static void read_long_string (LexState *LS, SemInfo *seminfo) { int cont = 0; size_t l = 0; checkbuffer(L, 10, l); - save(L, '[', l); /* save first `[' */ + save(L, l_c('['), l); /* save first `[' */ save_and_next(L, LS, l); /* pass the second `[' */ for (;;) { checkbuffer(L, 10, l); switch (LS->current) { case EOZ: - save(L, '\0', l); - luaX_error(LS, "unfinished long string", TK_STRING); + save(L, l_c('\0'), l); + luaX_error(LS, l_s("unfinished long string"), TK_STRING); break; /* to avoid warnings */ - case '[': + case l_c('['): save_and_next(L, LS, l); - if (LS->current == '[') { + if (LS->current == l_c('[')) { cont++; save_and_next(L, LS, l); } continue; - case ']': + case l_c(']'): save_and_next(L, LS, l); - if (LS->current == ']') { + if (LS->current == l_c(']')) { if (cont == 0) goto endloop; cont--; save_and_next(L, LS, l); } continue; - case '\n': - save(L, '\n', l); + case l_c('\n'): + save(L, l_c('\n'), l); inclinenumber(LS); continue; default: @@ -219,7 +224,7 @@ static void read_long_string (LexState *LS, SemInfo *seminfo) { } } endloop: save_and_next(L, LS, l); /* skip the second `]' */ - save(L, '\0', l); + save(L, l_c('\0'), l); seminfo->ts = luaS_newlstr(L, G(L)->Mbuffer+2, l-5); } @@ -232,38 +237,38 @@ static void read_string (LexState *LS, int del, SemInfo *seminfo) { while (LS->current != del) { checkbuffer(L, 10, l); switch (LS->current) { - case EOZ: case '\n': - save(L, '\0', l); - luaX_error(LS, "unfinished string", TK_STRING); + case EOZ: case l_c('\n'): + save(L, l_c('\0'), l); + luaX_error(LS, l_s("unfinished string"), TK_STRING); break; /* to avoid warnings */ - case '\\': + case l_c('\\'): next(LS); /* do not save the `\' */ switch (LS->current) { - case 'a': save(L, '\a', l); next(LS); break; - case 'b': save(L, '\b', l); next(LS); break; - case 'f': save(L, '\f', l); next(LS); break; - case 'n': save(L, '\n', l); next(LS); break; - case 'r': save(L, '\r', l); next(LS); break; - case 't': save(L, '\t', l); next(LS); break; - case 'v': save(L, '\v', l); next(LS); break; - case '\n': save(L, '\n', l); inclinenumber(LS); break; - case '0': case '1': case '2': case '3': case '4': - case '5': case '6': case '7': case '8': case '9': { - int c = 0; - int i = 0; - do { - c = 10*c + (LS->current-'0'); - next(LS); - } while (++i<3 && isdigit(LS->current)); - if (c > UCHAR_MAX) { - save(L, '\0', l); - luaX_error(LS, "escape sequence too large", TK_STRING); + case l_c('a'): save(L, l_c('\a'), l); next(LS); break; + case l_c('b'): save(L, l_c('\b'), l); next(LS); break; + case l_c('f'): save(L, l_c('\f'), l); next(LS); break; + case l_c('n'): save(L, l_c('\n'), l); next(LS); break; + case l_c('r'): save(L, l_c('\r'), l); next(LS); break; + case l_c('t'): save(L, l_c('\t'), l); next(LS); break; + case l_c('v'): save(L, l_c('\v'), l); next(LS); break; + case l_c('\n'): save(L, l_c('\n'), l); inclinenumber(LS); break; + default: { + if (!isdigit(LS->current)) + save_and_next(L, LS, l); /* handles \\, \", \', and \? */ + else { /* \xxx */ + int c = 0; + int i = 0; + do { + c = 10*c + (LS->current-l_c('0')); + next(LS); + } while (++i<3 && isdigit(LS->current)); + if (c > UCHAR_MAX) { + save(L, l_c('\0'), l); + luaX_error(LS, l_s("escape sequence too large"), TK_STRING); + } + save(L, c, l); } - save(L, c, l); - break; } - default: /* handles \\, \", \', and \? */ - save_and_next(L, LS, l); } break; default: @@ -271,7 +276,7 @@ static void read_string (LexState *LS, int del, SemInfo *seminfo) { } } save_and_next(L, LS, l); /* skip delimiter */ - save(L, '\0', l); + save(L, l_c('\0'), l); seminfo->ts = luaS_newlstr(L, G(L)->Mbuffer+1, l-3); } @@ -280,92 +285,85 @@ int luaX_lex (LexState *LS, SemInfo *seminfo) { for (;;) { switch (LS->current) { - case ' ': case '\t': case '\r': /* `\r' to avoid problems with DOS */ + case l_c(' '): case l_c('\t'): case l_c('\r'): /* `\r' to avoid problems with DOS */ next(LS); continue; - case '\n': + case l_c('\n'): inclinenumber(LS); continue; - case '$': - luaX_error(LS, "unexpected `$' (pragmas are no longer supported)", '$'); + case l_c('$'): + luaX_error(LS, + l_s("unexpected `$' (pragmas are no longer supported)"), + LS->current); break; - case '-': + case l_c('-'): next(LS); - if (LS->current != '-') return '-'; - do { next(LS); } while (LS->current != '\n' && LS->current != EOZ); + if (LS->current != l_c('-')) return l_c('-'); + do { next(LS); } while (LS->current != l_c('\n') && LS->current != EOZ); continue; - case '[': + case l_c('['): next(LS); - if (LS->current != '[') return '['; + if (LS->current != l_c('[')) return l_c('['); else { read_long_string(LS, seminfo); return TK_STRING; } - case '=': + case l_c('='): next(LS); - if (LS->current != '=') return '='; + if (LS->current != l_c('=')) return l_c('='); else { next(LS); return TK_EQ; } - case '<': + case l_c('<'): next(LS); - if (LS->current != '=') return '<'; + if (LS->current != l_c('=')) return l_c('<'); else { next(LS); return TK_LE; } - case '>': + case l_c('>'): next(LS); - if (LS->current != '=') return '>'; + if (LS->current != l_c('=')) return l_c('>'); else { next(LS); return TK_GE; } - case '~': + case l_c('~'): next(LS); - if (LS->current != '=') return '~'; + if (LS->current != l_c('=')) return l_c('~'); else { next(LS); return TK_NE; } - case '"': - case '\'': + case l_c('"'): + case l_c('\''): read_string(LS, LS->current, seminfo); return TK_STRING; - case '.': + case l_c('.'): next(LS); - if (LS->current == '.') { + if (LS->current == l_c('.')) { next(LS); - if (LS->current == '.') { + if (LS->current == l_c('.')) { next(LS); return TK_DOTS; /* ... */ } else return TK_CONCAT; /* .. */ } - else if (!isdigit(LS->current)) return '.'; + else if (!isdigit(LS->current)) return l_c('.'); else { read_number(LS, 1, seminfo); return TK_NUMBER; } - case '0': case '1': case '2': case '3': case '4': - case '5': case '6': case '7': case '8': case '9': - read_number(LS, 0, seminfo); - return TK_NUMBER; - case EOZ: return TK_EOS; - case '_': goto tname; - - default: - if (!isalpha(LS->current)) { - int c = LS->current; - if (iscntrl(c)) - luaX_invalidchar(LS, c); - next(LS); - return c; + default: { + if (isdigit(LS->current)) { + read_number(LS, 0, seminfo); + return TK_NUMBER; } - tname: { /* identifier or reserved word */ + else if (isalpha(LS->current) || LS->current == l_c('_')) { + /* identifier or reserved word */ size_t l = readname(LS); TString *ts = luaS_newlstr(LS->L, G(LS->L)->Mbuffer, l); if (ts->marked >= RESERVEDMARK) /* reserved word? */ @@ -373,6 +371,14 @@ int luaX_lex (LexState *LS, SemInfo *seminfo) { seminfo->ts = ts; return TK_NAME; } + else { + int c = LS->current; + if (iscntrl(c)) + luaX_invalidchar(LS, c); + next(LS); + return c; /* single-char tokens (+ - / ...) */ + } + } } } } -- cgit v1.2.3-55-g6feb