Cleanup lexer source code.

1 files changed, 157 insertions, 144 deletions

diff --git a/src/lj_lex.c b/src/lj_lex.c
index c4d52da2..2c5128e7 100644
--- a/src/lj_lex.c
+++ b/src/lj_lex.c
@@ -38,37 +38,48 @@ TKDEF(TKSTR1, TKSTR2)
 
 /* -- Buffer handling ----------------------------------------------------- */
 
-#define char2int(c)             ((int)(uint8_t)(c))
-#define next(ls) \
-  (ls->current = (ls->n--) > 0 ? char2int(*ls->p++) : fillbuf(ls))
-#define save_and_next(ls)       (save(ls, ls->current), next(ls))
-#define currIsNewline(ls)       (ls->current == '\n' || ls->current == '\r')
-#define END_OF_STREAM           (-1)
+#define LEX_EOF                 (-1)
+#define lex_iseol(ls)           (ls->c == '\n' || ls->c == '\r')
 
-static int fillbuf(LexState *ls)
+/* Get more input from reader. */
+static LJ_NOINLINE LexChar lex_more(LexState *ls)
 {
   size_t sz;
   const char *buf = ls->rfunc(ls->L, ls->rdata, &sz);
-  if (buf == NULL || sz == 0) return END_OF_STREAM;
+  if (buf == NULL || sz == 0) return LEX_EOF;
   ls->n = (MSize)sz - 1;
   ls->p = buf;
-  return char2int(*(ls->p++));
+  return (LexChar)(uint8_t)*ls->p++;
 }
 
-static LJ_AINLINE void save(LexState *ls, int c)
+/* Get next character. */
+static LJ_AINLINE LexChar lex_next(LexState *ls)
+{
+  return (ls->c = ls->n ? (ls->n--,(LexChar)(uint8_t)*ls->p++) : lex_more(ls));
+}
+
+/* Save character. */
+static LJ_AINLINE void lex_save(LexState *ls, LexChar c)
 {
   lj_buf_putb(ls->L, &ls->sb, c);
 }
 
-static void inclinenumber(LexState *ls)
+/* Save previous character and get next character. */
+static LJ_AINLINE LexChar lex_savenext(LexState *ls)
+{
+  lex_save(ls, ls->c);
+  return lex_next(ls);
+}
+
+/* Skip line break. Handles "\n", "\r", "\r\n" or "\n\r". */
+static void lex_newline(LexState *ls)
 {
-  int old = ls->current;
-  lua_assert(currIsNewline(ls));
-  next(ls);  /* skip `\n' or `\r' */
-  if (currIsNewline(ls) && ls->current != old)
-    next(ls);  /* skip `\n\r' or `\r\n' */
+  LexChar old = ls->c;
+  lua_assert(lex_iseol(ls));
+  lex_next(ls);  /* Skip "\n" or "\r". */
+  if (lex_iseol(ls) && ls->c != old) lex_next(ls);  /* Skip "\n\r" or "\r\n". */
   if (++ls->linenumber >= LJ_MAX_LINE)
-    lj_lex_error(ls, ls->token, LJ_ERR_XLINES);
+    lj_lex_error(ls, ls->tok, LJ_ERR_XLINES);
 }
 
 /* -- Scanner for terminals ----------------------------------------------- */
@@ -77,18 +88,16 @@ static void inclinenumber(LexState *ls)
 static void lex_number(LexState *ls, TValue *tv)
 {
   StrScanFmt fmt;
-  int c, xp = 'e';
-  lua_assert(lj_char_isdigit(ls->current));
-  if ((c = ls->current) == '0') {
-    save_and_next(ls);
-    if ((ls->current | 0x20) == 'x') xp = 'p';
+  LexChar c, xp = 'e';
+  lua_assert(lj_char_isdigit(ls->c));
+  if ((c = ls->c) == '0' && (lex_savenext(ls) | 0x20) == 'x')
+    xp = 'p';
+  while (lj_char_isident(ls->c) || ls->c == '.' ||
+         ((ls->c == '-' || ls->c == '+') && (c | 0x20) == xp)) {
+    c = ls->c;
+    lex_savenext(ls);
   }
-  while (lj_char_isident(ls->current) || ls->current == '.' ||
-         ((ls->current == '-' || ls->current == '+') && (c | 0x20) == xp)) {
-    c = ls->current;
-    save_and_next(ls);
-  }
-  save(ls, '\0');
+  lex_save(ls, '\0');
   fmt = lj_strscan_scan((const uint8_t *)sbufB(&ls->sb), tv,
           (LJ_DUALNUM ? STRSCAN_OPT_TOINT : STRSCAN_OPT_TONUM) |
           (LJ_HASFFI ? (STRSCAN_OPT_LL|STRSCAN_OPT_IMAG) : 0));
@@ -122,44 +131,42 @@ static void lex_number(LexState *ls, TValue *tv)
   }
 }
 
-static int skip_sep(LexState *ls)
+/* Skip equal signs for "[=...=[" and "]=...=]" and return their count. */
+static int lex_skipeq(LexState *ls)
 {
   int count = 0;
-  int s = ls->current;
+  LexChar s = ls->c;
   lua_assert(s == '[' || s == ']');
-  save_and_next(ls);
-  while (ls->current == '=') {
-    save_and_next(ls);
+  while (lex_savenext(ls) == '=')
     count++;
-  }
-  return (ls->current == s) ? count : (-count) - 1;
+  return (ls->c == s) ? count : (-count) - 1;
 }
 
-static void read_long_string(LexState *ls, TValue *tv, int sep)
+/* Parse a long string or long comment (tv set to NULL). */
+static void lex_longstring(LexState *ls, TValue *tv, int sep)
 {
-  save_and_next(ls);  /* skip 2nd `[' */
-  if (currIsNewline(ls))  /* string starts with a newline? */
-    inclinenumber(ls);  /* skip it */
+  lex_savenext(ls);  /* Skip second '['. */
+  if (lex_iseol(ls))  /* Skip initial newline. */
+    lex_newline(ls);
   for (;;) {
-    switch (ls->current) {
-    case END_OF_STREAM:
+    switch (ls->c) {
+    case LEX_EOF:
       lj_lex_error(ls, TK_eof, tv ? LJ_ERR_XLSTR : LJ_ERR_XLCOM);
       break;
     case ']':
-      if (skip_sep(ls) == sep) {
-        save_and_next(ls);  /* skip 2nd `]' */
+      if (lex_skipeq(ls) == sep) {
+        lex_savenext(ls);  /* Skip second ']'. */
         goto endloop;
       }
       break;
     case '\n':
     case '\r':
-      save(ls, '\n');
-      inclinenumber(ls);
+      lex_save(ls, '\n');
+      lex_newline(ls);
       if (!tv) lj_buf_reset(&ls->sb);  /* Don't waste space for comments. */
       break;
     default:
-      if (tv) save_and_next(ls);
-      else next(ls);
+      lex_savenext(ls);
       break;
     }
   } endloop:
@@ -170,12 +177,14 @@ static void read_long_string(LexState *ls, TValue *tv, int sep)
   }
 }
 
-static void read_string(LexState *ls, int delim, TValue *tv)
+/* Parse a string. */
+static void lex_string(LexState *ls, TValue *tv)
 {
-  save_and_next(ls);
-  while (ls->current != delim) {
-    switch (ls->current) {
-    case END_OF_STREAM:
+  LexChar delim = ls->c;  /* Delimiter is '\'' or '"'. */
+  lex_savenext(ls);
+  while (ls->c != delim) {
+    switch (ls->c) {
+    case LEX_EOF:
       lj_lex_error(ls, TK_eof, LJ_ERR_XSTR);
       continue;
     case '\n':
@@ -183,7 +192,7 @@ static void read_string(LexState *ls, int delim, TValue *tv)
       lj_lex_error(ls, TK_string, LJ_ERR_XSTR);
       continue;
     case '\\': {
-      int c = next(ls);  /* Skip the '\\'. */
+      LexChar c = lex_next(ls);  /* Skip the '\\'. */
       switch (c) {
       case 'a': c = '\a'; break;
       case 'b': c = '\b'; break;
@@ -193,112 +202,112 @@ static void read_string(LexState *ls, int delim, TValue *tv)
       case 't': c = '\t'; break;
       case 'v': c = '\v'; break;
       case 'x':  /* Hexadecimal escape '\xXX'. */
-        c = (next(ls) & 15u) << 4;
-        if (!lj_char_isdigit(ls->current)) {
-          if (!lj_char_isxdigit(ls->current)) goto err_xesc;
+        c = (lex_next(ls) & 15u) << 4;
+        if (!lj_char_isdigit(ls->c)) {
+          if (!lj_char_isxdigit(ls->c)) goto err_xesc;
           c += 9 << 4;
         }
-        c += (next(ls) & 15u);
-        if (!lj_char_isdigit(ls->current)) {
-          if (!lj_char_isxdigit(ls->current)) goto err_xesc;
+        c += (lex_next(ls) & 15u);
+        if (!lj_char_isdigit(ls->c)) {
+          if (!lj_char_isxdigit(ls->c)) goto err_xesc;
           c += 9;
         }
         break;
       case 'z':  /* Skip whitespace. */
-        next(ls);
-        while (lj_char_isspace(ls->current))
-          if (currIsNewline(ls)) inclinenumber(ls); else next(ls);
+        lex_next(ls);
+        while (lj_char_isspace(ls->c))
+          if (lex_iseol(ls)) lex_newline(ls); else lex_next(ls);
         continue;
-      case '\n': case '\r': save(ls, '\n'); inclinenumber(ls); continue;
+      case '\n': case '\r': lex_save(ls, '\n'); lex_newline(ls); continue;
       case '\\': case '\"': case '\'': break;
-      case END_OF_STREAM: continue;
+      case LEX_EOF: continue;
       default:
         if (!lj_char_isdigit(c))
           goto err_xesc;
         c -= '0';  /* Decimal escape '\ddd'. */
-        if (lj_char_isdigit(next(ls))) {
-          c = c*10 + (ls->current - '0');
-          if (lj_char_isdigit(next(ls))) {
-            c = c*10 + (ls->current - '0');
+        if (lj_char_isdigit(lex_next(ls))) {
+          c = c*10 + (ls->c - '0');
+          if (lj_char_isdigit(lex_next(ls))) {
+            c = c*10 + (ls->c - '0');
             if (c > 255) {
             err_xesc:
               lj_lex_error(ls, TK_string, LJ_ERR_XESC);
             }
-            next(ls);
+            lex_next(ls);
           }
         }
-        save(ls, c);
+        lex_save(ls, c);
         continue;
       }
-      save(ls, c);
-      next(ls);
+      lex_save(ls, c);
+      lex_next(ls);
       continue;
       }
     default:
-      save_and_next(ls);
+      lex_savenext(ls);
       break;
     }
   }
-  save_and_next(ls);  /* skip delimiter */
+  lex_savenext(ls);  /* Skip trailing delimiter. */
   setstrV(ls->L, tv,
           lj_parse_keepstr(ls, sbufB(&ls->sb)+1, sbuflen(&ls->sb)-2));
 }
 
 /* -- Main lexical scanner ------------------------------------------------ */
 
-static int llex(LexState *ls, TValue *tv)
+/* Get next lexical token. */
+static LexToken lex_scan(LexState *ls, TValue *tv)
 {
   lj_buf_reset(&ls->sb);
   for (;;) {
-    if (lj_char_isident(ls->current)) {
+    if (lj_char_isident(ls->c)) {
       GCstr *s;
-      if (lj_char_isdigit(ls->current)) {  /* Numeric literal. */
+      if (lj_char_isdigit(ls->c)) {  /* Numeric literal. */
         lex_number(ls, tv);
         return TK_number;
       }
       /* Identifier or reserved word. */
       do {
-        save_and_next(ls);
-      } while (lj_char_isident(ls->current));
+        lex_savenext(ls);
+      } while (lj_char_isident(ls->c));
       s = lj_parse_keepstr(ls, sbufB(&ls->sb), sbuflen(&ls->sb));
       setstrV(ls->L, tv, s);
       if (s->reserved > 0)  /* Reserved word? */
         return TK_OFS + s->reserved;
       return TK_name;
     }
-    switch (ls->current) {
+    switch (ls->c) {
     case '\n':
     case '\r':
-      inclinenumber(ls);
+      lex_newline(ls);
       continue;
     case ' ':
     case '\t':
     case '\v':
     case '\f':
-      next(ls);
+      lex_next(ls);
       continue;
     case '-':
-      next(ls);
-      if (ls->current != '-') return '-';
-      /* else is a comment */
-      next(ls);
-      if (ls->current == '[') {
-        int sep = skip_sep(ls);
-        lj_buf_reset(&ls->sb);  /* `skip_sep' may dirty the buffer */
+      lex_next(ls);
+      if (ls->c != '-') return '-';
+      lex_next(ls);
+      if (ls->c == '[') {  /* Long comment "--[=*[...]=*]". */
+        int sep = lex_skipeq(ls);
+        lj_buf_reset(&ls->sb);  /* `lex_skipeq' may dirty the buffer */
         if (sep >= 0) {
-          read_long_string(ls, NULL, sep);  /* long comment */
+          lex_longstring(ls, NULL, sep);
           lj_buf_reset(&ls->sb);
           continue;
         }
       }
-      /* else short comment */
-      while (!currIsNewline(ls) && ls->current != END_OF_STREAM)
-        next(ls);
+      /* Short comment "--.*\n". */
+      while (!lex_iseol(ls) && ls->c != LEX_EOF)
+        lex_next(ls);
       continue;
     case '[': {
-      int sep = skip_sep(ls);
+      int sep = lex_skipeq(ls);
       if (sep >= 0) {
-        read_long_string(ls, tv, sep);
+        lex_longstring(ls, tv, sep);
         return TK_string;
       } else if (sep == -1) {
         return '[';
@@ -308,44 +317,43 @@ static int llex(LexState *ls, TValue *tv)
       }
       }
     case '=':
-      next(ls);
-      if (ls->current != '=') return '='; else { next(ls); return TK_eq; }
+      lex_next(ls);
+      if (ls->c != '=') return '='; else { lex_next(ls); return TK_eq; }
     case '<':
-      next(ls);
-      if (ls->current != '=') return '<'; else { next(ls); return TK_le; }
+      lex_next(ls);
+      if (ls->c != '=') return '<'; else { lex_next(ls); return TK_le; }
     case '>':
-      next(ls);
-      if (ls->current != '=') return '>'; else { next(ls); return TK_ge; }
+      lex_next(ls);
+      if (ls->c != '=') return '>'; else { lex_next(ls); return TK_ge; }
     case '~':
-      next(ls);
-      if (ls->current != '=') return '~'; else { next(ls); return TK_ne; }
+      lex_next(ls);
+      if (ls->c != '=') return '~'; else { lex_next(ls); return TK_ne; }
     case ':':
-      next(ls);
-      if (ls->current != ':') return ':'; else { next(ls); return TK_label; }
+      lex_next(ls);
+      if (ls->c != ':') return ':'; else { lex_next(ls); return TK_label; }
     case '"':
     case '\'':
-      read_string(ls, ls->current, tv);
+      lex_string(ls, tv);
       return TK_string;
     case '.':
-      save_and_next(ls);
-      if (ls->current == '.') {
-        next(ls);
-        if (ls->current == '.') {
-          next(ls);
+      if (lex_savenext(ls) == '.') {
+        lex_next(ls);
+        if (ls->c == '.') {
+          lex_next(ls);
           return TK_dots;   /* ... */
         }
         return TK_concat;   /* .. */
-      } else if (!lj_char_isdigit(ls->current)) {
+      } else if (!lj_char_isdigit(ls->c)) {
         return '.';
       } else {
         lex_number(ls, tv);
         return TK_number;
       }
-    case END_OF_STREAM:
+    case LEX_EOF:
       return TK_eof;
     default: {
-      int c = ls->current;
-      next(ls);
+      LexChar c = ls->c;
+      lex_next(ls);
       return c;  /* Single-char tokens (+ - / ...). */
     }
     }
@@ -370,23 +378,23 @@ int lj_lex_setup(lua_State *L, LexState *ls)
   ls->lookahead = TK_eof;  /* No look-ahead token. */
   ls->linenumber = 1;
   ls->lastline = 1;
-  next(ls);  /* Read-ahead first char. */
-  if (ls->current == 0xef && ls->n >= 2 && char2int(ls->p[0]) == 0xbb &&
-      char2int(ls->p[1]) == 0xbf) {  /* Skip UTF-8 BOM (if buffered). */
+  lex_next(ls);  /* Read-ahead first char. */
+  if (ls->c == 0xef && ls->n >= 2 && (uint8_t)ls->p[0] == 0xbb &&
+      (uint8_t)ls->p[1] == 0xbf) {  /* Skip UTF-8 BOM (if buffered). */
     ls->n -= 2;
     ls->p += 2;
-    next(ls);
+    lex_next(ls);
     header = 1;
   }
-  if (ls->current == '#') {  /* Skip POSIX #! header line. */
+  if (ls->c == '#') {  /* Skip POSIX #! header line. */
     do {
-      next(ls);
-      if (ls->current == END_OF_STREAM) return 0;
-    } while (!currIsNewline(ls));
-    inclinenumber(ls);
+      lex_next(ls);
+      if (ls->c == LEX_EOF) return 0;
+    } while (!lex_iseol(ls));
+    lex_newline(ls);
     header = 1;
   }
-  if (ls->current == LUA_SIGNATURE[0]) {  /* Bytecode dump. */
+  if (ls->c == LUA_SIGNATURE[0]) {  /* Bytecode dump. */
     if (header) {
       /*
       ** Loading bytecode with an extra header is disabled for security
@@ -411,52 +419,57 @@ void lj_lex_cleanup(lua_State *L, LexState *ls)
   lj_buf_free(g, &ls->sb);
 }
 
+/* Return next lexical token. */
 void lj_lex_next(LexState *ls)
 {
   ls->lastline = ls->linenumber;
   if (LJ_LIKELY(ls->lookahead == TK_eof)) {  /* No lookahead token? */
-    ls->token = llex(ls, &ls->tokenval);  /* Get next token. */
+    ls->tok = lex_scan(ls, &ls->tokval);  /* Get next token. */
   } else {  /* Otherwise return lookahead token. */
-    ls->token = ls->lookahead;
+    ls->tok = ls->lookahead;
     ls->lookahead = TK_eof;
-    ls->tokenval = ls->lookaheadval;
+    ls->tokval = ls->lookaheadval;
   }
 }
 
+/* Look ahead for the next token. */
 LexToken lj_lex_lookahead(LexState *ls)
 {
   lua_assert(ls->lookahead == TK_eof);
-  ls->lookahead = llex(ls, &ls->lookaheadval);
+  ls->lookahead = lex_scan(ls, &ls->lookaheadval);
   return ls->lookahead;
 }
 
-const char *lj_lex_token2str(LexState *ls, LexToken token)
+/* Convert token to string. */
+const char *lj_lex_token2str(LexState *ls, LexToken tok)
 {
-  if (token > TK_OFS)
-    return tokennames[token-TK_OFS-1];
-  else if (!lj_char_iscntrl(token))
-    return lj_str_pushf(ls->L, "%c", token);
+  if (tok > TK_OFS)
+    return tokennames[tok-TK_OFS-1];
+  else if (!lj_char_iscntrl(tok))
+    return lj_str_pushf(ls->L, "%c", tok);
   else
-    return lj_str_pushf(ls->L, "char(%d)", token);
+    return lj_str_pushf(ls->L, "char(%d)", tok);
 }
 
-void lj_lex_error(LexState *ls, LexToken token, ErrMsg em, ...)
+/* Lexer error. */
+void lj_lex_error(LexState *ls, LexToken tok, ErrMsg em, ...)
 {
-  const char *tok;
+  const char *tokstr;
   va_list argp;
-  if (token == 0) {
-    tok = NULL;
-  } else if (token == TK_name || token == TK_string || token == TK_number) {
-    save(ls, '\0');
-    tok = sbufB(&ls->sb);
+  if (tok == 0) {
+    tokstr = NULL;
+  } else if (tok == TK_name || tok == TK_string || tok == TK_number) {
+    lex_save(ls, '\0');
+    tokstr = sbufB(&ls->sb);
   } else {
-    tok = lj_lex_token2str(ls, token);
+    tokstr = lj_lex_token2str(ls, tok);
   }
   va_start(argp, em);
-  lj_err_lex(ls->L, ls->chunkname, tok, ls->linenumber, em, argp);
+  lj_err_lex(ls->L, ls->chunkname, tokstr, ls->linenumber, em, argp);
   va_end(argp);
 }
 
+/* Initialize strings for reserved words. */
 void lj_lex_init(lua_State *L)
 {
   uint32_t i;