From cd618f934b0e4a30bd6cd4e98f8c1005bf3d6193 Mon Sep 17 00:00:00 2001 From: Li Jin Date: Sun, 25 Jun 2023 17:46:32 +0800 Subject: fix more ambiguous Lua codes generation cases. --- spec/inputs/ambiguous.yue | 21 ++++++++++++++ spec/outputs/ambiguous.lua | 15 ++++++++++ spec/outputs/attrib.lua | 8 ++--- spec/outputs/macro.lua | 8 ++--- spec/outputs/teal-lang.lua | 10 +++---- spec/outputs/teal-lang.tl | 8 ++--- spec/outputs/using.lua | 2 +- src/yuescript/ast.cpp | 21 ++++++++++++++ src/yuescript/ast.hpp | 10 +++++++ src/yuescript/parser.cpp | 32 ++++++++++++++++++-- src/yuescript/parser.hpp | 17 +++++++++-- src/yuescript/yue_ast.h | 3 +- src/yuescript/yue_compiler.cpp | 66 +++++++++++++++--------------------------- src/yuescript/yue_parser.cpp | 38 ++++++++++++++++++------ src/yuescript/yue_parser.h | 8 ++++- 15 files changed, 189 insertions(+), 78 deletions(-) diff --git a/spec/inputs/ambiguous.yue b/spec/inputs/ambiguous.yue index 3ae0bfc..37857fe 100644 --- a/spec/inputs/ambiguous.yue +++ b/spec/inputs/ambiguous.yue @@ -23,6 +23,27 @@ for i = 1, 10 goto abc (print) 5 +macro lua = (code)-> { + :code + type: "lua" +} + +do + print() + 1 |> b |> (a) + print() + <- (fn) + +do + print() + () <- async_fn() + print() + $lua[==[ + --[[a comment to insert]] + (haha)() + ]==] + nil + macro v = -> 'print 123' do global * diff --git a/spec/outputs/ambiguous.lua b/spec/outputs/ambiguous.lua index f96cee5..3da6f14 100644 --- a/spec/outputs/ambiguous.lua +++ b/spec/outputs/ambiguous.lua @@ -24,6 +24,21 @@ for i = 1, 10 do goto abc; (print)(5) end +do + print(); + (a)(b(1)) + print(); + (fn)(function() end) +end +do + print() + async_fn(function() + print(); +--[[a comment to insert]] + (haha)() + return nil + end) +end do print(123) end diff --git a/spec/outputs/attrib.lua b/spec/outputs/attrib.lua index fbac28e..aaff747 100644 --- a/spec/outputs/attrib.lua +++ b/spec/outputs/attrib.lua @@ -64,12 +64,12 @@ do local _with_0 = io.open("file.txt") _with_0:write("Hello") return _with_0 - end)(); + end)() local _ = setmetatable({ }, { __close = function(self) return print("second") end - }); + }) local _ = setmetatable({ }, { __close = function() return print("first") @@ -86,11 +86,11 @@ do _defers[#_defers + 1] = function() return print(3) end - local _ = _defers; + local _ = _defers _defers[#_defers + 1] = function() return print(2) end - local _ = _defers; + local _ = _defers _defers[#_defers + 1] = function() return print(1) end diff --git a/spec/outputs/macro.lua b/spec/outputs/macro.lua index fbc1d48..f18baed 100644 --- a/spec/outputs/macro.lua +++ b/spec/outputs/macro.lua @@ -1,4 +1,4 @@ -assert(item == nil); +assert(item == nil) local v = (item == nil) if f1() then print("OK") @@ -170,11 +170,11 @@ do end)() print(a) end -local x = 0; +local x = 0 local function f(a) return a + 1 end -x = x + f(3); +x = x + f(3) function tb:func() print(123) end @@ -277,7 +277,7 @@ print((setmetatable({ return 998 end })) -print("current line: " .. tostring(268)); +print("current line: " .. tostring(268)) -- TODO do print(1) diff --git a/spec/outputs/teal-lang.lua b/spec/outputs/teal-lang.lua index 0e627f4..28ba6f8 100644 --- a/spec/outputs/teal-lang.lua +++ b/spec/outputs/teal-lang.lua @@ -1,13 +1,13 @@ local a = { value = 123 -}; -local b = a.value; +} +local b = a.value local add = function(a, b) return a + b end local s = add(a.value, b) -print(s); -local Point = {}; +print(s) +local Point = {} Point.new = function(x, y) local point = setmetatable({ }, { __index = Point @@ -21,7 +21,7 @@ Point.move = function(self, dx, dy) self.y = self.y + dy end local p = Point.new(100, 100) -p:move(50, 50); +p:move(50, 50) local filter = function(tab, handler) local _accum_0 = { } local _len_0 = 1 diff --git a/spec/outputs/teal-lang.tl b/spec/outputs/teal-lang.tl index 8f0bf36..0dc25a1 100644 --- a/spec/outputs/teal-lang.tl +++ b/spec/outputs/teal-lang.tl @@ -1,14 +1,14 @@ local a:{string:number} = { value = 123 -}; -local b:number = a.value; +} +local b:number = a.value local function add(a:number, b:number):number return a + b end local s = add(a.value, b) -print(s); +print(s) local record Point x: number y: number @@ -28,7 +28,7 @@ function Point:move(dx:number, dy:number) end local p:Point = Point.new(100, 100) -p:move(50, 50); +p:move(50, 50) local function filter(tab:{string}, handler:function(item:string):boolean):{string} local _accum_0 = { } diff --git a/spec/outputs/using.lua b/spec/outputs/using.lua index ff23a30..8ecc21e 100644 --- a/spec/outputs/using.lua +++ b/spec/outputs/using.lua @@ -22,7 +22,7 @@ _ = function() local hello = hello or 2 end do - local a = { }; + local a = { } _(function() local x, y = a[1], a[2] end) diff --git a/src/yuescript/ast.cpp b/src/yuescript/ast.cpp index 239aec6..6db2c21 100644 --- a/src/yuescript/ast.cpp +++ b/src/yuescript/ast.cpp @@ -138,4 +138,25 @@ ast_node* parse(input& i, rule& g, error_list& el, void* ud) { return st.front(); } +/** check if the start part of given input matches grammar. + The parse procedures of each rule parsed are executed + before this function returns, if parsing succeeds. + @param i input. + @param g root rule of grammar. + @param ud user data, passed to the parse procedures. + @return true on parsing success, false on failure. +*/ +ast_node* start_with(input& i, rule& g, error_list& el, void* ud) { + ast_stack st; + if (!start_with(i, g, el, &st, ud)) { + for (auto node : st) { + delete node; + } + st.clear(); + return nullptr; + } + assert(st.size() == 1); + return st.front(); +} + } // namespace parserlib diff --git a/src/yuescript/ast.hpp b/src/yuescript/ast.hpp index c6da312..f59b50f 100644 --- a/src/yuescript/ast.hpp +++ b/src/yuescript/ast.hpp @@ -597,4 +597,14 @@ private: */ ast_node* parse(input& i, rule& g, error_list& el, void* ud); +/** check if the start part of given input matches grammar. + The parse procedures of each rule parsed are executed + before this function returns, if parsing succeeds. + @param i input. + @param g root rule of grammar. + @param ud user data, passed to the parse procedures. + @return true on parsing success, false on failure. +*/ +ast_node* start_with(input& i, rule& g, error_list& el, void* ud); + } // namespace parserlib diff --git a/src/yuescript/parser.cpp b/src/yuescript/parser.cpp index 25b735e..04b7ffd 100644 --- a/src/yuescript/parser.cpp +++ b/src/yuescript/parser.cpp @@ -1567,10 +1567,11 @@ expr user(const expr& e, const user_handler& handler) { @param i input. @param g root rule of grammar. @param el list of errors. - @param d user data, passed to the parse procedures. + @param st ast object stack. + @param ud user data, passed to the parse procedures. @return true on parsing success, false on failure. */ -bool parse(input& i, rule& g, error_list& el, void* d, void* ud) { +bool parse(input& i, rule& g, error_list& el, void* st, void* ud) { // prepare context _context con(i, ud); @@ -1591,7 +1592,32 @@ bool parse(input& i, rule& g, error_list& el, void* d, void* ud) { } // success; execute the parse procedures - con.do_parse_procs(d); + con.do_parse_procs(st); + return true; +} + +/** check the start part of given input. + The parse procedures of each rule parsed are executed + before this function returns, if parsing succeeds. + @param i input. + @param g root rule of grammar. + @param el list of errors. + @param st ast object stack. + @param ud user data, passed to the parse procedures. + @return true on parsing success, false on failure. +*/ +bool start_with(input& i, rule& g, error_list& el, void* st, void* ud) { + // prepare context + _context con(i, ud); + + // parse grammar + if (!con.parse_non_term(g)) { + el.push_back(_syntax_error(con)); + return false; + } + + // success; execute the parse procedures + con.do_parse_procs(st); return true; } diff --git a/src/yuescript/parser.hpp b/src/yuescript/parser.hpp index 83682ae..d2612cd 100644 --- a/src/yuescript/parser.hpp +++ b/src/yuescript/parser.hpp @@ -394,10 +394,23 @@ expr user(const expr& e, const user_handler& handler); @param i input. @param g root rule of grammar. @param el list of errors. - @param d user data, passed to the parse procedures. + @param st ast object stack. + @param ud user data, passed to the parse procedures. @return true on parsing success, false on failure. */ -bool parse(input& i, rule& g, error_list& el, void* d, void* ud); +bool parse(input& i, rule& g, error_list& el, void* st, void* ud); + +/** check if the start part of given input matches grammar. + The parse procedures of each rule parsed are executed + before this function returns, if parsing succeeds. + @param i input. + @param g root rule of grammar. + @param el list of errors. + @param st ast object stack. + @param ud user data, passed to the parse procedures. + @return true on parsing success, false on failure. +*/ +bool start_with(input& i, rule& g, error_list& el, void* st, void* ud); /** output the specific input range to the specific stream. @param stream stream. diff --git a/src/yuescript/yue_ast.h b/src/yuescript/yue_ast.h index acb7221..e95e35c 100644 --- a/src/yuescript/yue_ast.h +++ b/src/yuescript/yue_ast.h @@ -875,8 +875,7 @@ AST_NODE(Statement) Backcall_t, LocalAttrib_t, PipeBody_t, ExpListAssign_t, ChainAssign_t > content; ast_ptr appendix; - ast_ptr needSep; - AST_MEMBER(Statement, &sep, &comments, &content, &appendix, &needSep) + AST_MEMBER(Statement, &sep, &comments, &content, &appendix) AST_END(Statement, "statement"sv) AST_NODE(Body) diff --git a/src/yuescript/yue_compiler.cpp b/src/yuescript/yue_compiler.cpp index 98d19a7..8574e6c 100644 --- a/src/yuescript/yue_compiler.cpp +++ b/src/yuescript/yue_compiler.cpp @@ -72,7 +72,7 @@ static std::unordered_set Metamethods = { "close"s // Lua 5.4 }; -const std::string_view version = "0.17.0"sv; +const std::string_view version = "0.17.1"sv; const std::string_view extension = "yue"sv; class CompileError : public std::logic_error { @@ -1497,25 +1497,6 @@ private: case id(): transformChainAssign(static_cast(content), out); break; default: YUEE("AST node mismatch", content); break; } - if (statement->needSep && !out.empty() && !out.back().empty()) { - auto index = std::string::npos; - if (_config.reserveLineNumber) { - index = out.back().rfind(" -- "sv); - } else { - index = out.back().find_last_not_of('\n'); - if (index != std::string::npos) index++; - } - if (index != std::string::npos) { - auto ending = out.back().substr(0, index); - auto ind = ending.find_last_of(" \t\n"sv); - if (ind != std::string::npos) { - ending = ending.substr(ind + 1); - } - if (LuaKeywords.find(ending) == LuaKeywords.end()) { - out.back().insert(index, ";"sv); - } - } - } } str_list getAssignVars(ExpListAssign_t* assignment) { @@ -3543,8 +3524,6 @@ private: } lst->appendix.set(stmt->appendix); stmt->appendix.set(nullptr); - lst->needSep.set(stmt->needSep); - stmt->needSep.set(nullptr); auto exp = lastExpFromStatement(lst); BREAK_IF(!exp); for (auto val : pipeBody->values.objects()) { @@ -3753,22 +3732,6 @@ private: returnNode->valueList.set(expListLow); returnNode->allowBlockMacroReturn = true; last->content.set(returnNode); - last->needSep.set(nullptr); - auto bLast = ++nodes.rbegin(); - if (bLast != nodes.rend()) { - bool isMacro = false; - BLOCK_START - BREAK_IF(expListLow->exprs.size() != 1); - auto exp = static_cast(expListLow->exprs.back()); - BREAK_IF(!exp->opValues.empty()); - auto chainValue = exp->getByPath(); - BREAK_IF(!chainValue); - isMacro = isMacroChain(chainValue); - BLOCK_END - if (!isMacro) { - ast_to(*bLast)->needSep.set(nullptr); - } - } BLOCK_END break; } @@ -3802,11 +3765,6 @@ private: } newAssignment->action.set(assign); last->content.set(newAssignment); - last->needSep.set(nullptr); - auto bLast = ++nodes.rbegin(); - if (bLast != nodes.rend()) { - static_cast(*bLast)->needSep.set(nullptr); - } } else if (!last->content.is()) { throw CompileError("expecting assignable statement or break loop"sv, last); } @@ -3818,6 +3776,28 @@ private: str_list temp; for (auto node : nodes) { transformStatement(static_cast(node), temp); + if (_parser.startWith(temp.back())) { + auto rit = ++temp.rbegin(); + if (rit != temp.rend() && !rit->empty()) { + auto index = std::string::npos; + if (_config.reserveLineNumber) { + index = rit->rfind(" -- "sv); + } else { + index = rit->find_last_not_of('\n'); + if (index != std::string::npos) index++; + } + if (index != std::string::npos) { + auto ending = rit->substr(0, index); + auto ind = ending.find_last_of(" \t\n"sv); + if (ind != std::string::npos) { + ending = ending.substr(ind + 1); + } + if (LuaKeywords.find(ending) == LuaKeywords.end()) { + rit->insert(index, ";"sv); + } + } + } + } } out.push_back(join(temp)); } else { diff --git a/src/yuescript/yue_parser.cpp b/src/yuescript/yue_parser.cpp index f383275..f72407d 100644 --- a/src/yuescript/yue_parser.cpp +++ b/src/yuescript/yue_parser.cpp @@ -853,13 +853,6 @@ YueParser::YueParser() { ChainAssign = Seperator >> Exp >> +(space >> '=' >> space >> Exp >> space >> and_('=')) >> space >> Assign; StatementAppendix = (IfLine | WhileLine | CompInner) >> space; - StatementSep = and_( - *space_break >> check_indent_match >> space >> ( - set("($'\"") | - "[[" | - "[=" - ) - ); Statement = Seperator >> -( @@ -876,8 +869,9 @@ YueParser::YueParser() { StatementAppendix >> empty_block_error ) >> space >> - -StatementAppendix >> - -StatementSep; + -StatementAppendix; + + StatementSep = white >> (set("('\"") | "[[" | "[="); Body = in_block | Statement; @@ -905,6 +899,32 @@ YueParser::YueParser() { } // clang-format on +bool YueParser::startWith(std::string_view codes, rule& r) { + std::unique_ptr converted; + if (codes.substr(0, 3) == "\xEF\xBB\xBF"sv) { + codes = codes.substr(3); + } + try { + if (!codes.empty()) { + converted = std::make_unique(_converter.from_bytes(&codes.front(), &codes.back() + 1)); + } else { + converted = std::make_unique(); + } + } catch (const std::range_error&) { + return false; + } + error_list errors; + try { + State state; + return ::yue::start_with(*converted, r, errors, &state); + } catch (const ParserError&) { + return false; + } catch (const std::logic_error&) { + return false; + } + return true; +} + ParseInfo YueParser::parse(std::string_view codes, rule& r) { ParseInfo res; if (codes.substr(0, 3) == "\xEF\xBB\xBF"sv) { diff --git a/src/yuescript/yue_parser.h b/src/yuescript/yue_parser.h index d4f66eb..d7fbf90 100644 --- a/src/yuescript/yue_parser.h +++ b/src/yuescript/yue_parser.h @@ -84,11 +84,17 @@ public: return parse(codes, rEnd).node; } + template + bool startWith(std::string_view codes) { + return startWith(codes, getRule()); + } + std::string toString(ast_node* node); std::string toString(input::iterator begin, input::iterator end); protected: ParseInfo parse(std::string_view codes, rule& r); + bool startWith(std::string_view codes, rule& r); struct State { State() { @@ -387,8 +393,8 @@ private: AST_RULE(WhileLine) AST_RULE(BreakLoop) AST_RULE(StatementAppendix) - AST_RULE(StatementSep) AST_RULE(Statement) + AST_RULE(StatementSep); AST_RULE(YueLineComment) AST_RULE(MultilineCommentInner) AST_RULE(YueMultilineComment) -- cgit v1.2.3-55-g6feb