From 676da10a45af912d9226dce4cb447bf7bd923da0 Mon Sep 17 00:00:00 2001 From: Li Jin Date: Fri, 24 Jul 2026 17:12:21 +0800 Subject: Optimize compiler parsing and module state --- src/yue.cpp | 132 ++++++++++++--- src/yuescript/ast.cpp | 70 +++++++- src/yuescript/ast.hpp | 95 ++++++++++- src/yuescript/parser.cpp | 80 ++++++++- src/yuescript/parser.hpp | 15 +- src/yuescript/yue_compiler.cpp | 372 ++++++++++++++++++++++++++++++----------- src/yuescript/yue_parser.cpp | 98 ++++++++++- src/yuescript/yue_parser.h | 1 + 8 files changed, 711 insertions(+), 152 deletions(-) (limited to 'src') diff --git a/src/yue.cpp b/src/yue.cpp index 4bb4e70..8fe2aba 100644 --- a/src/yue.cpp +++ b/src/yue.cpp @@ -9,18 +9,25 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI #include "yuescript/yue_compiler.h" #include "yuescript/yue_parser.h" +#include #include +#include #include +#include #include +#include #include #include #include #include #include +#include #include +#include #include #include #include +#include using namespace std::string_view_literals; using namespace std::string_literals; using namespace std::chrono_literals; @@ -32,35 +39,102 @@ using namespace std::chrono_literals; #if __has_include() #include -template -std::future async(const std::function& f) { - using Fn = std::packaged_task; - auto task = new Fn(f); - std::future fut = task->get_future(); +#endif - pthread_attr_t attr; - pthread_attr_init(&attr); - pthread_attr_setstacksize(&attr, 8 * 1024 * 1024); +class AsyncPool { +public: + explicit AsyncPool(size_t workerCount) { + workerCount = std::max(workerCount, 1); +#if __has_include() + pthread_attr_t attr; + pthread_attr_init(&attr); + pthread_attr_setstacksize(&attr, 8 * 1024 * 1024); + _workers.reserve(workerCount); + for (size_t i = 0; i < workerCount; i++) { + pthread_t worker; + const int result = pthread_create(&worker, &attr, [](void* data) -> void* { + static_cast(data)->run(); + return nullptr; + }, this); + if (result != 0) { + { + std::lock_guard lock(_mutex); + _stopping = true; + } + _condition.notify_all(); + for (auto createdWorker : _workers) { + pthread_join(createdWorker, nullptr); + } + pthread_attr_destroy(&attr); + throw std::runtime_error("failed to create compiler worker thread"); + } + _workers.push_back(worker); + } + pthread_attr_destroy(&attr); +#else + _workers.reserve(workerCount); + for (size_t i = 0; i < workerCount; i++) { + _workers.emplace_back([this]() { run(); }); + } +#endif + } - pthread_t th; - pthread_create(&th, &attr, - [](void* p)->void* { - std::unique_ptr fn(static_cast(p)); - (*fn)(); - return nullptr; - }, - task); - pthread_attr_destroy(&attr); - pthread_detach(th); - return fut; -} + ~AsyncPool() { + { + std::lock_guard lock(_mutex); + _stopping = true; + } + _condition.notify_all(); +#if __has_include() + for (auto worker : _workers) { + pthread_join(worker, nullptr); + } #else -template -std::future async(const std::function& f) { - // fallback: ignore stack size - return std::async(std::launch::async, f); -} + for (auto& worker : _workers) { + worker.join(); + } #endif + } + + template + std::future async(const std::function& f) { + auto task = std::make_shared>(f); + auto result = task->get_future(); + { + std::lock_guard lock(_mutex); + _tasks.emplace_back([task]() { (*task)(); }); + } + _condition.notify_one(); + return result; + } + +private: + void run() { + while (true) { + std::function task; + { + std::unique_lock lock(_mutex); + _condition.wait(lock, [this]() { + return _stopping || !_tasks.empty(); + }); + if (_stopping && _tasks.empty()) return; + task = std::move(_tasks.front()); + _tasks.pop_front(); + } + task(); + } + } + + std::mutex _mutex; + std::condition_variable _condition; + std::deque> _tasks; + bool _stopping = false; +#if __has_include() + std::vector _workers; +#else + std::vector _workers; +#endif +}; #if not(defined YUE_NO_MACRO && defined YUE_COMPILER_ONLY) #define _DEFER(code, line) std::shared_ptr _defer_##line(nullptr, [&](auto) { \ @@ -823,6 +897,10 @@ int main(int narg, const char** args) { } } #endif // YUE_COMPILER_ONLY + const size_t workerCount = std::min( + files.size(), + static_cast(std::max(1u, std::thread::hardware_concurrency()))); + AsyncPool pool(workerCount); #ifndef YUE_NO_WATCHER if (watchFiles) { auto fullWorkPath = fs::absolute(fs::path(workPath)).string(); @@ -832,7 +910,7 @@ int main(int narg, const char** args) { } std::list> results; for (const auto& file : files) { - auto task = async([=]() { + auto task = pool.async([=]() { #ifndef YUE_COMPILER_ONLY return compileFile(fs::absolute(file.first), config, fullWorkPath, fullTargetPath, minify, rewrite); #else @@ -870,7 +948,7 @@ int main(int narg, const char** args) { #endif // YUE_NO_WATCHER std::list>> results; for (const auto& file : files) { - auto task = async>([=]() { + auto task = pool.async>([=]() { std::ifstream input(file.first, std::ios::in); if (input) { std::string s( diff --git a/src/yuescript/ast.cpp b/src/yuescript/ast.cpp index 99ab8f5..1de2071 100644 --- a/src/yuescript/ast.cpp +++ b/src/yuescript/ast.cpp @@ -9,12 +9,74 @@ Redistributions in binary form must reproduce the above copyright notice, this l THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.*/ +#include #include +#include #include "yuescript/ast.hpp" namespace parserlib { +namespace { + +thread_local ast_arena* currentAstArena = nullptr; + +struct alignas(std::max_align_t) ast_allocation_header { + ast_arena* arena = nullptr; +}; + +} // namespace + +void* ast_arena::allocate(size_t size, size_t alignment) { + constexpr size_t BlockSize = 1024 * 1024; + auto tryAllocate = [size, alignment](block& item) -> void* { + auto address = reinterpret_cast(item.data.get() + item.used); + auto aligned = (address + alignment - 1) & ~(alignment - 1); + auto offset = static_cast(aligned - reinterpret_cast(item.data.get())); + if (offset + size > item.size) return nullptr; + item.used = offset + size; + return item.data.get() + offset; + }; + if (!_blocks.empty()) { + if (auto ptr = tryAllocate(_blocks.back())) return ptr; + } + block item; + item.size = std::max(BlockSize, size + alignment - 1); + item.data = std::unique_ptr(new std::byte[item.size]); + _blocks.push_back(std::move(item)); + return tryAllocate(_blocks.back()); +} + +ast_arena_scope::ast_arena_scope(ast_arena& arena) + : _previous(currentAstArena) { + currentAstArena = &arena; +} + +ast_arena_scope::~ast_arena_scope() { + currentAstArena = _previous; +} + +void* ast_node::operator new(size_t size) { + const auto allocationSize = sizeof(ast_allocation_header) + size; + ast_allocation_header* header = nullptr; + if (currentAstArena) { + header = static_cast( + currentAstArena->allocate(allocationSize, alignof(ast_allocation_header))); + header->arena = currentAstArena; + } else { + header = static_cast(::operator new(allocationSize)); + } + return header + 1; +} + +void ast_node::operator delete(void* ptr) noexcept { + if (!ptr) return; + auto header = static_cast(ptr) - 1; + if (!header->arena) { + ::operator delete(header); + } +} + traversal ast_node::traverse(const std::function& func) { return func(this); } @@ -125,9 +187,9 @@ bool ast_container::visit_child(const std::function& func) { @return pointer to ast node created, or null if there was an error. The return object must be deleted by the caller. */ -ast_node* parse(input& i, rule& g, error_list& el, void* ud) { +ast_node* parse(input& i, rule& g, error_list& el, void* ud, bool resolveLeftRecursion) { ast_stack st; - if (!parse(i, g, el, &st, ud)) { + if (!parse(i, g, el, &st, ud, resolveLeftRecursion)) { for (auto node : st) { delete node; } @@ -146,9 +208,9 @@ ast_node* parse(input& i, rule& g, error_list& el, void* ud) { @param ud user data, passed to the parse procedures. @return true on parsing success, false on failure. */ -ast_node* start_with(input& i, rule& g, error_list& el, void* ud) { +ast_node* start_with(input& i, rule& g, error_list& el, void* ud, bool resolveLeftRecursion) { ast_stack st; - if (!start_with(i, g, el, &st, ud)) { + if (!start_with(i, g, el, &st, ud, resolveLeftRecursion)) { for (auto node : st) { delete node; } diff --git a/src/yuescript/ast.hpp b/src/yuescript/ast.hpp index b3b0d23..1985b84 100644 --- a/src/yuescript/ast.hpp +++ b/src/yuescript/ast.hpp @@ -11,12 +11,17 @@ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND #pragma once +#include #include -#include +#include #include +#include +#include +#include #include #include #include +#include #include "yuescript/parser.hpp" @@ -25,6 +30,7 @@ namespace parserlib { using namespace std::string_view_literals; class ast_node; +class ast_arena; template class ast_ptr; template @@ -63,10 +69,41 @@ enum class traversal { Stop }; +class ast_arena { +public: + ast_arena() = default; + ast_arena(const ast_arena&) = delete; + ast_arena& operator=(const ast_arena&) = delete; + + void* allocate(size_t size, size_t alignment); + +private: + struct block { + std::unique_ptr data; + size_t size = 0; + size_t used = 0; + }; + std::vector _blocks; +}; + +class ast_arena_scope { +public: + explicit ast_arena_scope(ast_arena& arena); + ~ast_arena_scope(); + ast_arena_scope(const ast_arena_scope&) = delete; + ast_arena_scope& operator=(const ast_arena_scope&) = delete; + +private: + ast_arena* _previous = nullptr; +}; + /** Base class for AST nodes. */ class ast_node : public input_range { public: + static void* operator new(size_t size); + static void operator delete(void* ptr) noexcept; + ast_node() : _ref(0) { } @@ -154,15 +191,59 @@ bool ast_is(ast_node* node) { class ast_member; -/** type of ast member vector. - */ -typedef std::vector ast_member_vector; +class ast_member_vector { +public: + using iterator = ast_member**; + using const_iterator = ast_member* const*; + using reverse_iterator = std::reverse_iterator; + using const_reverse_iterator = std::reverse_iterator; + + void reserve(size_t size) { + if (size <= InlineCapacity) return; + if (_heap) { + _overflow.reserve(size); + } else { + _overflow.reserve(size); + for (size_t i = 0; i < _size; i++) _overflow.push_back(_inline[i]); + _heap = true; + } + } + + void push_back(ast_member* member) { + if (_heap) { + _overflow.push_back(member); + } else if (_size < InlineCapacity) { + _inline[_size] = member; + } else { + reserve(_size + 1); + _overflow.push_back(member); + } + _size++; + } + + iterator begin() { return _heap ? _overflow.data() : _inline.data(); } + iterator end() { return begin() + _size; } + const_iterator begin() const { return _heap ? _overflow.data() : _inline.data(); } + const_iterator end() const { return begin() + _size; } + reverse_iterator rbegin() { return reverse_iterator(end()); } + reverse_iterator rend() { return reverse_iterator(begin()); } + const_reverse_iterator rbegin() const { return const_reverse_iterator(end()); } + const_reverse_iterator rend() const { return const_reverse_iterator(begin()); } + +private: + static constexpr size_t InlineCapacity = 3; + std::array _inline{}; + std::vector _overflow; + size_t _size = 0; + bool _heap = false; +}; /** base class for AST nodes with children. */ class ast_container : public ast_node { public: void add_members(std::initializer_list members) { + m_members.reserve(members.size()); for (auto member : members) { m_members.push_back(member); } @@ -595,10 +676,11 @@ private: @param g root rule of grammar. @param el list of errors. @param ud user data, passed to the parse procedures. + @param resolveLeftRecursion enables the general left-recursion resolver. @return pointer to ast node created, or null if there was an error. The return object must be deleted by the caller. */ -ast_node* parse(input& i, rule& g, error_list& el, void* ud); +ast_node* parse(input& i, rule& g, error_list& el, void* ud, bool resolveLeftRecursion = true); /** check if the start part of given input matches grammar. The parse procedures of each rule parsed are executed @@ -606,8 +688,9 @@ ast_node* parse(input& i, rule& g, error_list& el, void* ud); @param i input. @param g root rule of grammar. @param ud user data, passed to the parse procedures. + @param resolveLeftRecursion enables the general left-recursion resolver. @return true on parsing success, false on failure. */ -ast_node* start_with(input& i, rule& g, error_list& el, void* ud); +ast_node* start_with(input& i, rule& g, error_list& el, void* ud, bool resolveLeftRecursion = true); } // namespace parserlib diff --git a/src/yuescript/parser.cpp b/src/yuescript/parser.cpp index 5910348..ae7b9a2 100644 --- a/src/yuescript/parser.cpp +++ b/src/yuescript/parser.cpp @@ -38,6 +38,20 @@ namespace CodeCvt { namespace parserlib { input utf8_decode(const std::string& str) { + bool ascii = true; + for (unsigned char ch : str) { + if (ch >= 0x80) { + ascii = false; + break; + } + } + if (ascii) { + input result(str.size(), char32_t{}); + for (size_t i = 0; i < str.size(); i++) { + result[i] = static_cast(str[i]); + } + return result; + } return CodeCvt::utf8to32(str); } @@ -131,13 +145,17 @@ public: // matches _match_vector m_matches; + // whether to resolve left-recursive rules + bool m_resolve_left_recursion; + // constructor - _context(input& i, void* ud) + _context(input& i, void* ud, bool resolveLeftRecursion) : m_user_data(ud) , m_pos(i) , m_error_pos(i) , m_begin(i.begin()) - , m_end(i.end()) { + , m_end(i.end()) + , m_resolve_left_recursion(resolveLeftRecursion) { } // check if the end is reached @@ -451,7 +469,7 @@ public: virtual bool parse_non_term(_context& con) const override { pos pos = con.m_pos; if (m_expr->parse_non_term(con)) { - item_t item = {&pos, &con.m_pos, con.m_user_data}; + item_t item = {&pos, &con.m_pos, con.m_user_data, con.m_begin, con.m_end}; return m_handler(item); } return false; @@ -461,7 +479,7 @@ public: virtual bool parse_term(_context& con) const override { pos pos = con.m_pos; if (m_expr->parse_term(con)) { - item_t item = {&pos, &con.m_pos, con.m_user_data}; + item_t item = {&pos, &con.m_pos, con.m_user_data, con.m_begin, con.m_end}; return m_handler(item); } return false; @@ -854,6 +872,40 @@ private: rule& m_rule; }; +// predictive rule dispatch +class _dispatch : public _expr { +public: + _dispatch(const dispatch_handler& handler, std::initializer_list rules) + : m_handler(handler) { + m_rules.reserve(rules.size()); + for (rule* target : rules) { + m_rules.push_back(new _ref(*target)); + } + } + + virtual ~_dispatch() { + for (_expr* target : m_rules) { + delete target; + } + } + + virtual bool parse_non_term(_context& con) const override { + const size_t index = m_handler(con.m_pos.m_it, con.m_end); + if (index >= m_rules.size()) return false; + return m_rules[index]->parse_non_term(con); + } + + virtual bool parse_term(_context& con) const override { + const size_t index = m_handler(con.m_pos.m_it, con.m_end); + if (index >= m_rules.size()) return false; + return m_rules[index]->parse_term(con); + } + +private: + dispatch_handler m_handler; + std::vector<_expr*> m_rules; +}; + // eof class _eof : public _expr { public: @@ -930,6 +982,10 @@ _state::_state(_context& con) // parse non-term rule. bool _context::parse_non_term(rule& r) { + if (!m_resolve_left_recursion) { + return _parse_non_term(r); + } + // save the state of the rule rule::_state old_state = r.m_state; // restore the rule's state @@ -1024,6 +1080,10 @@ bool _context::parse_non_term(rule& r) { // parse term rule. bool _context::parse_term(rule& r) { + if (!m_resolve_left_recursion) { + return _parse_term(r); + } + // save the state of the rule rule::_state old_state = r.m_state; // restore the rule's state @@ -1619,6 +1679,10 @@ expr user(const expr& e, const user_handler& handler) { return _private::construct_expr(new _user(_private::get_expr(e), handler)); } +expr dispatch(const dispatch_handler& handler, std::initializer_list rules) { + return _private::construct_expr(new _dispatch(handler, rules)); +} + /** parses the given input. The parse procedures of each rule parsed are executed before this function returns, if parsing succeeds. @@ -1629,9 +1693,9 @@ expr user(const expr& e, const user_handler& handler) { @param ud user data, passed to the parse procedures. @return true on parsing success, false on failure. */ -bool parse(input& i, rule& g, error_list& el, void* st, void* ud) { +bool parse(input& i, rule& g, error_list& el, void* st, void* ud, bool resolveLeftRecursion) { // prepare context - _context con(i, ud); + _context con(i, ud, resolveLeftRecursion); // parse grammar if (!con.parse_non_term(g)) { @@ -1664,9 +1728,9 @@ bool parse(input& i, rule& g, error_list& el, void* st, void* ud) { @param ud user data, passed to the parse procedures. @return true on parsing success, false on failure. */ -bool start_with(input& i, rule& g, error_list& el, void* st, void* ud) { +bool start_with(input& i, rule& g, error_list& el, void* st, void* ud, bool resolveLeftRecursion) { // prepare context - _context con(i, ud); + _context con(i, ud, resolveLeftRecursion); // parse grammar if (!con.parse_non_term(g)) { diff --git a/src/yuescript/parser.hpp b/src/yuescript/parser.hpp index 4742539..9ce962d 100644 --- a/src/yuescript/parser.hpp +++ b/src/yuescript/parser.hpp @@ -18,6 +18,7 @@ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND #endif #include +#include #include #include #include @@ -64,8 +65,11 @@ struct item_t { pos* begin; pos* end; void* user_data; + input_it input_begin; + input_it input_end; }; typedef std::function user_handler; +typedef std::function dispatch_handler; /** a grammar expression. */ @@ -402,6 +406,11 @@ expr false_(); */ expr user(const expr& e, const user_handler& handler); +/** selects one rule without probing the other alternatives. + @return an expression that parses the selected rule. +*/ +expr dispatch(const dispatch_handler& handler, std::initializer_list rules); + /** parses the given input. The parse procedures of each rule parsed are executed before this function returns, if parsing succeeds. @@ -410,9 +419,10 @@ expr user(const expr& e, const user_handler& handler); @param el list of errors. @param st ast object stack. @param ud user data, passed to the parse procedures. + @param resolveLeftRecursion enables the general left-recursion resolver. @return true on parsing success, false on failure. */ -bool parse(input& i, rule& g, error_list& el, void* st, void* ud); +bool parse(input& i, rule& g, error_list& el, void* st, void* ud, bool resolveLeftRecursion = true); /** check if the start part of given input matches grammar. The parse procedures of each rule parsed are executed @@ -422,9 +432,10 @@ bool parse(input& i, rule& g, error_list& el, void* st, void* ud); @param el list of errors. @param st ast object stack. @param ud user data, passed to the parse procedures. + @param resolveLeftRecursion enables the general left-recursion resolver. @return true on parsing success, false on failure. */ -bool start_with(input& i, rule& g, error_list& el, void* st, void* ud); +bool start_with(input& i, rule& g, error_list& el, void* st, void* ud, bool resolveLeftRecursion = true); /** output the specific input range to the specific stream. @param stream stream. diff --git a/src/yuescript/yue_compiler.cpp b/src/yuescript/yue_compiler.cpp index df260ab..72ae001 100644 --- a/src/yuescript/yue_compiler.cpp +++ b/src/yuescript/yue_compiler.cpp @@ -145,6 +145,8 @@ public: int idx = static_cast(lua_objlen(L, -1)); // idx = #tb, tb BREAK_IF(idx == 0); _useModule = true; + lua_rawgeti(L, -1, idx); // tb current + _moduleScopeBaseline = static_cast(lua_objlen(L, -1)); BLOCK_END } @@ -157,6 +159,8 @@ public: #endif // YUE_NO_MACRO CompileInfo compile(std::string_view codes, const YueConfig& config) { + ast_arena arena; + ast_arena_scope arenaScope(arena); _config = config; #ifndef YUE_NO_MACRO if (L) passOptions(); @@ -337,27 +341,46 @@ public: void clear() { _indentOffset = 0; + _funcLevel = 0; + _gotoScope = 0; _scopes.clear(); + _importedGlobal = nullptr; _codeCache.clear(); _buf.str(""); _buf.clear(); _joinBuf.str(""); _joinBuf.clear(); _globals.clear(); + _rootDefs.clear(); + _labels.clear(); + gotos.clear(); + _exportedKeys.clear(); + _exportedMetaKeys.clear(); _info = {}; _varArgs = {}; _withVars = {}; _continueVars = {}; _funcStates = {}; + _enableBreakLoop = {}; + _gotoScopes = {}; #ifndef YUE_NO_MACRO if (_useModule) { _useModule = false; - if (!_sameModule) { - int top = lua_gettop(L); - DEFER(lua_settop(L, top)); - lua_pushliteral(L, YUE_MODULES); // YUE_MODULES - lua_rawget(L, LUA_REGISTRYINDEX); // reg[YUE_MODULES], tb - int idx = static_cast(lua_objlen(L, -1)); + int top = lua_gettop(L); + DEFER(lua_settop(L, top)); + lua_pushliteral(L, YUE_MODULES); // YUE_MODULES + lua_rawget(L, LUA_REGISTRYINDEX); // reg[YUE_MODULES], tb + int idx = static_cast(lua_objlen(L, -1)); + if (_sameModule) { + if (idx > 0) { + lua_rawgeti(L, -1, idx); // tb current + while (static_cast(lua_objlen(L, -1)) > _moduleScopeBaseline) { + int scopeIndex = static_cast(lua_objlen(L, -1)); + lua_pushnil(L); + lua_rawseti(L, -2, scopeIndex); + } + } + } else { lua_pushnil(L); // tb nil lua_rawseti(L, -2, idx); // tb[idx] = nil, tb } @@ -370,6 +393,7 @@ private: bool _stateOwner = false; bool _useModule = false; bool _sameModule = false; + int _moduleScopeBaseline = 0; lua_State* L = nullptr; std::function _luaOpen; #endif // YUE_NO_MACRO @@ -1366,6 +1390,130 @@ private: return ast_ptr(res.node.template to()); } + template + ast_ptr makeLeaf(std::string_view codes, ast_node* parent) { + auto converted = std::make_unique(utf8_decode(std::string(codes))); + auto node = parent->new_ptr(); + node->m_begin.m_it = converted->begin(); + node->m_end.m_it = converted->end(); + _codeCache.push_back(std::move(converted)); + return node; + } + + ast_ptr makeVariable(std::string_view name, ast_node* parent) { + auto variable = parent->new_ptr(); + if (std::any_of(name.begin(), name.end(), [](unsigned char ch) { return ch >= 0x80; })) { + variable->name.set(makeLeaf(name, parent)); + } else { + variable->name.set(makeLeaf(name, parent)); + } + return variable; + } + + ast_ptr makeVariableCallable(std::string_view name, ast_node* parent) { + auto callable = parent->new_ptr(); + callable->item.set(makeVariable(name, parent)); + return callable; + } + + ast_ptr makeConstValue(std::string_view code, ast_node* parent) { + auto simple = parent->new_ptr(); + simple->value.set(makeLeaf(code, parent)); + auto value = parent->new_ptr(); + value->item.set(simple); + return value; + } + + ast_ptr makeConstExp(std::string_view code, ast_node* parent) { + auto exp = newExp(makeConstValue(code, parent), parent); + exp->sep.set(parent->new_ptr()); + return exp; + } + + ast_ptr makeNumberExp(int value, ast_node* parent) { + auto simple = parent->new_ptr(); + simple->value.set(makeLeaf(std::to_string(value), parent)); + auto exp = newExp(simple, parent); + exp->sep.set(parent->new_ptr()); + return exp; + } + + ast_ptr makeVariableExp(std::string_view name, ast_node* parent) { + auto chain = parent->new_ptr(); + chain->sep.set(parent->new_ptr()); + chain->items.push_back(makeVariableCallable(name, parent)); + auto value = parent->new_ptr(); + value->item.set(chain); + auto unary = parent->new_ptr(); + unary->expos.push_back(value); + auto exp = parent->new_ptr(); + exp->sep.set(parent->new_ptr()); + exp->pipeExprs.push_back(unary); + return exp; + } + + ast_ptr makeVariableExpList(std::string_view name, ast_node* parent) { + auto list = parent->new_ptr(); + list->sep.set(parent->new_ptr()); + list->exprs.push_back(makeVariableExp(name, parent)); + return list; + } + + ast_ptr makeVariableExpList(const str_list& names, ast_node* parent) { + auto list = parent->new_ptr(); + list->sep.set(parent->new_ptr()); + for (const auto& name : names) { + list->exprs.push_back(makeVariableExp(name, parent)); + } + return list; + } + + ast_ptr makeGoto(std::string_view label, ast_node* parent) { + auto node = parent->new_ptr(); + node->label.set(makeLeaf(label, parent)); + return node; + } + + bool startsWithStatementSep(std::string_view codes) const { + size_t index = 0; + while (index < codes.size()) { + while (index < codes.size()) { + switch (codes[index]) { + case ' ': + case '\t': + case '\r': + case '\n': + index++; + continue; + default: + break; + } + break; + } + if (index + 1 >= codes.size() || codes.substr(index, 2) != "--"sv) break; + if (codes.substr(index, 4) == "--[["sv) { + auto close = codes.find("]]"sv, index + 4); + if (close == std::string_view::npos) return false; + index = close + 2; + } else { + auto lineEnd = codes.find_first_of("\r\n"sv, index + 2); + if (lineEnd == std::string_view::npos) return false; + index = lineEnd; + } + } + if (index == codes.size()) return false; + switch (codes[index]) { + case '(': + case '\'': + case '"': + return true; + case '[': + return index + 1 < codes.size() && (codes[index + 1] == '[' || codes[index + 1] == '='); + default: + return false; + } + } + bool isChainValueCall(ChainValue_t* chainValue) const { return ast_is(chainValue->items.back()); } @@ -2297,7 +2445,7 @@ private: for (; i != exprs.end(); ++i) { auto var = getUnusedName("_obj_"sv); addToScope(var); - extraExprs.push_back(toAst(var, *i)); + extraExprs.push_back(makeVariableExp(var, *i)); } popScope(); ast_ptr funcCall = values.back(); @@ -2413,7 +2561,7 @@ private: if (_withVars.empty()) { throw CompileError("short table appending must be called within a with block"sv, x); } else { - tmpChain->items.push_back(toAst(_withVars.top(), chainValue)); + tmpChain->items.push_back(makeVariableCallable(_withVars.top(), chainValue)); } } auto varName = singleVariableFrom(tmpChain, AccessType::Write); @@ -2427,7 +2575,7 @@ private: } auto objVar = getUnusedName("_obj_"sv); auto newAssignment = x->new_ptr(); - newAssignment->expList.set(toAst(objVar, x)); + newAssignment->expList.set(makeVariableExpList(objVar, x)); auto assign = x->new_ptr(); assign->values.push_back(newExp(tmpChain, tmpChain)); newAssignment->action.set(assign); @@ -2481,7 +2629,7 @@ private: break; } leftVar = getUnusedName("_obj_"sv); - auto tmpAsmt = assignmentFrom(toAst(leftVar, tmpLeft), tmpLeft, tmpLeft); + auto tmpAsmt = assignmentFrom(makeVariableExp(leftVar, tmpLeft), tmpLeft, tmpLeft); str_list temp; transformAssignment(tmpAsmt, temp); auto [beforeAssignment, afterAssignment] = splitAssignment(); @@ -2492,7 +2640,7 @@ private: throw CompileError("right value missing"sv, values.front()); } auto newChain = chainValue->new_ptr(); - newChain->items.push_back(toAst(leftVar, newChain)); + newChain->items.push_back(makeVariableCallable(leftVar, newChain)); newChain->items.push_back(chainValue->items.back()); auto newLeft = newExp(newChain, newChain); auto newAsmt = assignmentFrom(newLeft, *vit, newLeft); @@ -2750,7 +2898,7 @@ private: if (pair.targetVar.empty() && pair.defVal) { if (needScope) extraScope = true; auto objVar = getUnusedName("_tmp_"sv); - auto objExp = toAst(objVar, pair.target); + auto objExp = makeVariableExp(objVar, pair.target); leftPairs.push_back({pair.target, objExp.get()}); pair.target.set(objExp); pair.targetVar = objVar; @@ -2804,11 +2952,11 @@ private: pushScope(); } objVar = getUnusedName("_obj_"sv); - auto newAssignment = assignmentFrom(toAst(objVar, x), destruct.value, x); + auto newAssignment = assignmentFrom(makeVariableExp(objVar, x), destruct.value, x); transformAssignment(newAssignment, temp); } auto chain = pair.target->new_ptr(); - chain->items.push_back(toAst(objVar, chain)); + chain->items.push_back(makeVariableCallable(objVar, chain)); chain->items.dup(pair.structure->items); auto valueExp = newExp(chain, pair.target); auto newAssignment = assignmentFrom(pair.target, valueExp, x); @@ -2834,7 +2982,7 @@ private: if (needScope) extraScope = true; auto objVar = getUnusedName("_tmp_"sv); addToScope(objVar); - auto objExp = toAst(objVar, item.target); + auto objExp = makeVariableExp(objVar, item.target); leftPairs.push_back({item.target, objExp.get()}); item.target.set(objExp); item.targetVar = objVar; @@ -2844,7 +2992,7 @@ private: } popScope(); if (_parser.match(destruct.valueVar) && isLocal(destruct.valueVar)) { - auto callable = toAst(destruct.valueVar, destruct.value); + auto callable = makeVariableCallable(destruct.valueVar, destruct.value); for (auto& v : values) { v->items.push_front(callable); } @@ -2873,7 +3021,7 @@ private: pushScope(); } auto valVar = getUnusedName("_obj_"sv); - auto targetVar = toAst(valVar, destruct.value); + auto targetVar = makeVariableExp(valVar, destruct.value); auto newAssignment = assignmentFrom(targetVar, destruct.value, destruct.value); transformAssignment(newAssignment, temp); auto callable = singleValueFrom(targetVar)->item.to()->items.front(); @@ -3061,7 +3209,7 @@ private: int rIndex = count - index; indexItem.set(toAst('#' + (rIndex == 0 ? Empty : "-"s + std::to_string(rIndex)), pair)); } else { - indexItem.set(toAst(std::to_string(index), pair)); + indexItem.set(makeNumberExp(index, pair)); } if (optional && varDefOnly && !assignable) { if (defVal) { @@ -3112,7 +3260,7 @@ private: auto name = _parser.toString(vp->name); auto uname = vp->name->name.as(); auto chain = toAst('.' + name, vp->name); - pairs.push_back({toAst(name, vp).get(), + pairs.push_back({makeVariableExp(name, vp).get(), uname ? variableToString(vp->name) : name, chain, defVal}); @@ -3222,7 +3370,7 @@ private: int rIndex = count - index; indexItem.set(toAst('#' + (rIndex == 0 ? Empty : "-"s + std::to_string(rIndex)), tb)); } else { - indexItem.set(toAst(std::to_string(index), tb)); + indexItem.set(makeNumberExp(index, tb)); } for (auto& p : subPairs) { if (sep) p.structure->items.push_front(sep); @@ -3309,7 +3457,7 @@ private: auto slice = toAst( '[' + (start == 1 ? Empty : std::to_string(start)) + ',' + (stop == -1 ? Empty : std::to_string(stop)) + ']', exp); chain->items.push_back(slice); - auto nil = toAst("nil"sv, slice); + auto nil = makeConstExp("nil"sv, slice); pairs.push_back({exp, varName, chain, @@ -3357,7 +3505,7 @@ private: size_t size = std::max(exprs.size(), values.size()); ast_ptr nil; if (values.size() < size) { - nil = toAst("nil"sv, x); + nil = makeConstExp("nil"sv, x); while (values.size() < size) values.emplace_back(nil); } using iter = node_container::iterator; @@ -3548,7 +3696,7 @@ private: auto objVar = getUnusedName("_obj_"sv); addToScope(objVar); valueItems.pop_back(); - valueItems.push_back(toAst(objVar, *j)); + valueItems.push_back(makeVariableExp(objVar, *j)); auto expList = x->new_ptr(); auto newAssign = x->new_ptr(); newAssign->expList.set(expList); @@ -3676,7 +3824,7 @@ private: auto assign = des.inlineAssignment->action.to(); auto tmpVar = getUnusedName("_tmp_"sv); forceAddToScope(tmpVar); - auto tmpExp = toAst(tmpVar, exp); + auto tmpExp = makeVariableExp(tmpVar, exp); assignList->exprs.push_back(tmpExp); auto vExp = exp->new_ptr(); vExp->pipeExprs.dup(exp->pipeExprs); @@ -3740,13 +3888,13 @@ private: auto exp = newExp(tmpChain, x); auto objVar = getUnusedName("_obj_"sv); auto newAssignment = x->new_ptr(); - newAssignment->expList.set(toAst(objVar, x)); + newAssignment->expList.set(makeVariableExpList(objVar, x)); auto assign = x->new_ptr(); assign->values.push_back(exp); newAssignment->action.set(assign); transformAssignment(newAssignment, temp); chain->items.clear(); - chain->items.push_back(toAst(objVar, x)); + chain->items.push_back(makeVariableCallable(objVar, x)); chain->items.push_back(ptr); } BLOCK_END @@ -3760,12 +3908,12 @@ private: BREAK_IF(!var.empty()); auto upVar = getUnusedName("_update_"sv); auto newAssignment = x->new_ptr(); - newAssignment->expList.set(toAst(upVar, x)); + newAssignment->expList.set(makeVariableExpList(upVar, x)); auto assign = x->new_ptr(); assign->values.push_back(exp); newAssignment->action.set(assign); transformAssignment(newAssignment, temp); - tmpChain->items.push_back(toAst(upVar, x)); + tmpChain->items.push_back(makeVariableExp(upVar, x)); itemAdded = true; BLOCK_END if (!itemAdded) tmpChain->items.push_back(item); @@ -3882,7 +4030,7 @@ private: if (*it != nodes.front() && cond->assignment) { auto x = *it; auto newIf = x->new_ptr(); - newIf->type.set(toAst("if"sv, x)); + newIf->type.set(makeLeaf("if"sv, x)); for (auto j = ns.rbegin(); j != ns.rend(); ++j) { newIf->nodes.push_back(*j); } @@ -3903,7 +4051,7 @@ private: if (nodes.size() != ns.size()) { auto x = ns.back(); auto newIf = x->new_ptr(); - newIf->type.set(toAst("if"sv, x)); + newIf->type.set(makeLeaf("if"sv, x)); for (auto j = ns.rbegin(); j != ns.rend(); ++j) { newIf->nodes.push_back(*j); } @@ -3915,7 +4063,7 @@ private: if (usage == ExpUsage::Closure) { auto x = nodes.front(); auto newIf = x->new_ptr(); - newIf->type.set(toAst(unless ? "unless"sv : "if"sv, x)); + newIf->type.set(makeLeaf(unless ? "unless"sv : "if"sv, x)); for (ast_node* node : nodes) { newIf->nodes.push_back(node); } @@ -3971,7 +4119,7 @@ private: pushScope(); } } - auto expList = toAst(desVar, x); + auto expList = makeVariableExpList(desVar, x); auto assignment = x->new_ptr(); if (asmt->expList) { for (auto expr : asmt->expList->exprs.objects()) { @@ -3986,7 +4134,7 @@ private: auto expList = x->new_ptr(); expList->exprs.push_back(exp); auto assignOne = x->new_ptr(); - auto valExp = toAst(desVar, x); + auto valExp = makeVariableExp(desVar, x); assignOne->values.push_back(valExp); auto assignment = x->new_ptr(); assignment->expList.set(expList); @@ -4211,7 +4359,7 @@ private: addToScope(varName); auto condExp = node->new_ptr(); condExp->pipeExprs.dup(*item.second); - auto varExp = toAst(varName, node); + auto varExp = makeVariableExp(varName, node); auto assignment = assignmentFrom(varExp, condExp, node); preDefine = assignment; stack.push_back(varExp->pipeExprs); @@ -4229,7 +4377,7 @@ private: stack.pop_front(); auto opValue = exp->new_ptr(); const auto& two = std::get(stack.front()); - auto op = toAst(two, exp); + auto op = makeLeaf(two, exp); opValue->op.set(op); stack.pop_front(); const auto& three = std::get>(stack.front()); @@ -4237,7 +4385,7 @@ private: condExp->opValues.push_back(opValue); if (preDefine) { auto ifNode = exp->new_ptr(); - ifNode->type.set(toAst("unless"sv, exp)); + ifNode->type.set(makeLeaf("unless"sv, exp)); auto ifCond = exp->new_ptr(); ifCond->condition.set(condExp); ifNode->nodes.push_back(ifCond); @@ -4247,7 +4395,7 @@ private: if (newCondExp) { if (!nodes) { auto ifNodePrev = exp->new_ptr(); - ifNodePrev->type.set(toAst("unless"sv, exp)); + ifNodePrev->type.set(makeLeaf("unless"sv, exp)); auto ifCondPrev = exp->new_ptr(); ifCondPrev->condition.set(newCondExp); ifNodePrev->nodes.push_back(ifCondPrev); @@ -4309,7 +4457,7 @@ private: nodes->push_back(stmt); } else { auto opValue = exp->new_ptr(); - opValue->op.set(toAst("and"sv, exp)); + opValue->op.set(makeLeaf("and"sv, exp)); opValue->pipeExprs.dup(condExp->pipeExprs); newCondExp->opValues.push_back(opValue); newCondExp->opValues.dup(condExp->opValues); @@ -4432,7 +4580,7 @@ private: codes = YueFormat{}.toString(block); } else { auto withNode = block->new_ptr(); - withNode->valueList.set(toAst(_withVars.top(), x)); + withNode->valueList.set(makeVariableExpList(_withVars.top(), x)); withNode->body.set(block); codes = YueFormat{}.toString(withNode); auto simpleValue = x->new_ptr(); @@ -4519,7 +4667,7 @@ private: auto simpleValue = x->new_ptr(); simpleValue->value.set(funLit); auto funcName = getUnusedName("_anon_func_"sv); - auto assignment = assignmentFrom(toAst(funcName, x), newExp(simpleValue, x), x); + auto assignment = assignmentFrom(makeVariableExp(funcName, x), newExp(simpleValue, x), x); auto scopes = std::move(_scopes); _scopes.push_back(std::move(scopes.front())); scopes.pop_front(); @@ -4620,7 +4768,7 @@ private: } } objVar = getUnusedName("_exp_"sv); - auto expList = toAst(objVar, x); + auto expList = makeVariableExpList(objVar, x); auto assign = x->new_ptr(); assign->values.push_back(left); auto assignment = x->new_ptr(); @@ -4670,7 +4818,7 @@ private: temp.push_back(clearBuf()); pushScope(); assign->values.clear(); - assign->values.push_back(toAst(objVar, x)); + assign->values.push_back(makeVariableExp(objVar, x)); transformAssignment(assignment, temp); popScope(); temp.push_back(indent() + "else"s + nl(x)); @@ -5081,7 +5229,7 @@ private: } auto newAssign = x->new_ptr(); for (const auto& argName : argNames) { - newAssign->values.push_back(toAst(argName, x)); + newAssign->values.push_back(makeVariableExp(argName, x)); } auto newAssignment = x->new_ptr(); newAssignment->expList.set(newExpList); @@ -5480,7 +5628,7 @@ private: _rootDefs.clear(); temp.push_back(std::move(last)); } - if (!temp.empty() && _parser.startWith(temp.back())) { + if (!temp.empty() && startsWithStatementSep(temp.back())) { auto rit = ++temp.rbegin(); if (rit != temp.rend() && !rit->empty()) { auto index = std::string::npos; @@ -5600,12 +5748,34 @@ private: if (_useModule) { lua_pushliteral(L, YUE_MODULES); // YUE_MODULES lua_rawget(L, LUA_REGISTRYINDEX); // reg[YUE_MODULES], mods - int idx = static_cast(lua_objlen(L, -1)); // idx = #mods, mods - lua_rawgeti(L, -1, idx); // mods[idx], mods cur - lua_remove(L, -2); // cur - return; + if (lua_istable(L, -1) != 0) { + int idx = static_cast(lua_objlen(L, -1)); // idx = #mods, mods + if (idx > 0) { + lua_rawgeti(L, -1, idx); // mods[idx], mods cur + lua_remove(L, -2); // cur + return; + } + } + lua_pop(L, 1); + _useModule = false; + } + if (_sameModule) { + lua_pushliteral(L, YUE_MODULES); // YUE_MODULES + lua_rawget(L, LUA_REGISTRYINDEX); // reg[YUE_MODULES], mods + if (lua_istable(L, -1) != 0) { + int idx = static_cast(lua_objlen(L, -1)); // idx = #mods, mods + if (idx > 0) { + lua_rawgeti(L, -1, idx); // mods[idx], mods cur + lua_remove(L, -2); // cur + _useModule = true; + _moduleScopeBaseline = static_cast(lua_objlen(L, -1)); + return; + } + } + lua_pop(L, 1); } _useModule = true; + _moduleScopeBaseline = 0; if (!L) { L = luaL_newstate(); int top = lua_gettop(L); @@ -6026,7 +6196,7 @@ private: arg.name = getUnusedName("_arg_"sv); auto simpleValue = def->new_ptr(); simpleValue->value.set(def->name); - auto asmt = assignmentFrom(newExp(simpleValue, def), toAst(arg.name, def), def); + auto asmt = assignmentFrom(newExp(simpleValue, def), makeVariableExp(arg.name, def), def); arg.assignment = asmt; break; } @@ -6034,7 +6204,7 @@ private: arg.name = getUnusedName("_arg_"sv); auto value = def->new_ptr(); value->item.set(def->name); - auto asmt = assignmentFrom(newExp(value, def), toAst(arg.name, def), def); + auto asmt = assignmentFrom(newExp(value, def), makeVariableExp(arg.name, def), def); arg.assignment = asmt; break; } @@ -6042,7 +6212,7 @@ private: arg.name = getUnusedName("_arg_"sv); auto simpleValue = def->new_ptr(); simpleValue->value.set(def->name); - auto asmt = assignmentFrom(newExp(simpleValue, def), toAst(arg.name, def), def); + auto asmt = assignmentFrom(newExp(simpleValue, def), makeVariableExp(arg.name, def), def); arg.assignment = asmt; break; } @@ -6051,7 +6221,7 @@ private: forceAddToScope(arg.name); if (def->defaultValue) { pushScope(); - auto expList = toAst(arg.name, x); + auto expList = makeVariableExpList(arg.name, x); auto assign = x->new_ptr(); assign->values.push_back(def->defaultValue.get()); auto assignment = x->new_ptr(); @@ -6219,7 +6389,7 @@ private: chainValue->items.pop_back(); auto value = x->new_ptr(); value->item.set(chainValue); - auto exp = newExp(value, toAst("!="sv, x), toAst("nil"sv, x), x); + auto exp = newExp(value, makeLeaf("!="sv, x), makeConstValue("nil"sv, x), x); parens->expr.set(exp); } switch (usage) { @@ -6323,7 +6493,7 @@ private: if (_withVars.empty()) { throw CompileError("short dot/colon syntax must be called within a with block"sv, x); } - chainValue->items.push_back(toAst(_withVars.top(), x)); + chainValue->items.push_back(makeVariableCallable(_withVars.top(), x)); } auto newObj = singleVariableFrom(chainValue, AccessType::Read); if (!newObj.empty()) { @@ -6333,7 +6503,7 @@ private: auto assign = x->new_ptr(); assign->values.push_back(exp); auto expListAssign = x->new_ptr(); - expListAssign->expList.set(toAst(objVar, x)); + expListAssign->expList.set(makeVariableExpList(objVar, x)); expListAssign->action.set(assign); transformAssignment(expListAssign, temp); } @@ -6344,16 +6514,16 @@ private: } dotItem->name.set(name); partOne->items.clear(); - partOne->items.push_back(toAst(objVar, x)); + partOne->items.push_back(makeVariableCallable(objVar, x)); partOne->items.push_back(dotItem); auto it = opIt; ++it; if (it != chainList.end() && ast_is(*it)) { if (auto invoke = ast_cast(*it)) { - invoke->args.push_front(toAst(objVar, x)); + invoke->args.push_front(makeVariableExp(objVar, x)); } else { auto invokeArgs = static_cast(*it); - invokeArgs->args.push_front(toAst(objVar, x)); + invokeArgs->args.push_front(makeVariableExp(objVar, x)); } } objVar = getUnusedName("_obj_"sv); @@ -6362,7 +6532,7 @@ private: auto assign = x->new_ptr(); assign->values.push_back(exp); auto expListAssign = x->new_ptr(); - expListAssign->expList.set(toAst(objVar, x)); + expListAssign->expList.set(makeVariableExpList(objVar, x)); expListAssign->action.set(assign); transformAssignment(expListAssign, temp); } @@ -6378,7 +6548,7 @@ private: temp.push_back(clearBuf()); pushScope(); auto partTwo = x->new_ptr(); - partTwo->items.push_back(toAst(objVar, x)); + partTwo->items.push_back(makeVariableCallable(objVar, x)); for (auto it = ++opIt; it != chainList.end(); ++it) { partTwo->items.push_back(*it); } @@ -6461,7 +6631,7 @@ private: if (_withVars.empty()) { throw CompileError("short dot/colon syntax must be called within a with block"sv, chainList.front()); } else { - baseChain->items.push_back(toAst(_withVars.top(), x)); + baseChain->items.push_back(makeVariableCallable(_withVars.top(), x)); } break; } @@ -6478,7 +6648,7 @@ private: auto assign = x->new_ptr(); assign->values.push_back(exp); auto assignment = x->new_ptr(); - assignment->expList.set(toAst(baseVar, x)); + assignment->expList.set(makeVariableExpList(baseVar, x)); assignment->action.set(assign); transformAssignment(assignment, temp); } @@ -6486,7 +6656,7 @@ private: auto assign = x->new_ptr(); assign->values.push_back(toAst(baseVar + "." + funcName, x)); auto assignment = x->new_ptr(); - assignment->expList.set(toAst(fnVar, x)); + assignment->expList.set(makeVariableExpList(fnVar, x)); assignment->action.set(assign); transformAssignment(assignment, temp); } @@ -6553,7 +6723,7 @@ private: if (_withVars.empty()) { throw CompileError("short dot/colon syntax must be called within a with block"sv, x); } else { - chain->items.push_back(toAst(_withVars.top(), x)); + chain->items.push_back(makeVariableCallable(_withVars.top(), x)); } } for (auto it = chainList.begin(); it != opIt; ++it) { @@ -6600,7 +6770,7 @@ private: } } auto var = getUnusedName("_obj_"sv); - auto target = toAst(var, x); + auto target = makeVariableExp(var, x); { auto assignment = assignmentFrom(target, newExp(chain, x), x); transformAssignment(assignment, temp); @@ -6744,7 +6914,7 @@ private: switch (chainList.front()->get_id()) { case id(): case id(): - chainValue->items.push_back(toAst(_withVars.top(), x)); + chainValue->items.push_back(makeVariableCallable(_withVars.top(), x)); break; } for (auto i = chainList.begin(); i != current; ++i) { @@ -6755,7 +6925,7 @@ private: if (callVar.empty() || !isLocal(callVar)) { callVar = getUnusedName("_call_"s); auto assignment = x->new_ptr(); - assignment->expList.set(toAst(callVar, x)); + assignment->expList.set(makeVariableExpList(callVar, x)); auto assign = x->new_ptr(); assign->values.push_back(exp); assignment->action.set(assign); @@ -6768,21 +6938,21 @@ private: { auto name = _parser.toString(colonItem->name); auto chainValue = x->new_ptr(); - chainValue->items.push_back(toAst(callVar, x)); + chainValue->items.push_back(makeVariableCallable(callVar, x)); if (ast_is(*current)) { chainValue->items.push_back(x->new_ptr()); } chainValue->items.push_back(toAst('\"' + name + '\"', x)); if (auto invoke = ast_cast(followItem)) { auto newInvoke = x->new_ptr(); - newInvoke->args.push_back(toAst(callVar, x)); + newInvoke->args.push_back(makeVariableExp(callVar, x)); newInvoke->args.dup(invoke->args); chainValue->items.push_back(newInvoke); ++next; } else { auto invokeArgs = static_cast(followItem); auto newInvokeArgs = x->new_ptr(); - newInvokeArgs->args.push_back(toAst(callVar, x)); + newInvokeArgs->args.push_back(makeVariableExp(callVar, x)); newInvokeArgs->args.dup(invokeArgs->args); chainValue->items.push_back(newInvokeArgs); ++next; @@ -6867,7 +7037,7 @@ private: auto indexNode = toAst('#' + var, rIndex); if (rIndex->modifier) { auto opValue = rIndex->new_ptr(); - opValue->op.set(toAst("-"sv, rIndex)); + opValue->op.set(makeLeaf("-"sv, rIndex)); opValue->pipeExprs.dup(rIndex->modifier->pipeExprs); indexNode->opValues.push_back(opValue); indexNode->opValues.dup(rIndex->modifier->opValues); @@ -6892,15 +7062,15 @@ private: return; } else { auto itemVar = getUnusedName("_item_"sv); - auto asmt = assignmentFrom(toAst(itemVar, x), newExp(prevChain, x), x); + auto asmt = assignmentFrom(makeVariableExp(itemVar, x), newExp(prevChain, x), x); auto stmt1 = x->new_ptr(); stmt1->content.set(asmt); auto newChain = x->new_ptr(); - newChain->items.push_back(toAst(itemVar, x)); + newChain->items.push_back(makeVariableCallable(itemVar, x)); auto indexNode = toAst('#' + itemVar, rIndex); if (rIndex->modifier) { auto opValue = rIndex->new_ptr(); - opValue->op.set(toAst("-"sv, rIndex)); + opValue->op.set(makeLeaf("-"sv, rIndex)); opValue->pipeExprs.dup(rIndex->modifier->pipeExprs); indexNode->opValues.push_back(opValue); indexNode->opValues.dup(rIndex->modifier->opValues); @@ -7663,7 +7833,7 @@ private: auto block = x->new_ptr(); if (checkVar.empty() || !isLocal(checkVar)) { checkVar = getUnusedName("_check_"sv); - auto assignment = assignmentFrom(toAst(checkVar, inExp), newExp(inExp, inExp), inExp); + auto assignment = assignmentFrom(makeVariableExp(checkVar, inExp), newExp(inExp, inExp), inExp); auto stmt = x->new_ptr(); stmt->content.set(assignment); block->statementOrComments.push_back(stmt); @@ -7674,7 +7844,7 @@ private: newUnaryExp->expos.dup(unary_exp->expos); auto exp = newExp(newUnaryExp, x); varName = getUnusedName("_val_"sv); - auto assignExp = toAst(varName, x); + auto assignExp = makeVariableExp(varName, x); auto assignment = assignmentFrom(assignExp, exp, x); auto stmt = x->new_ptr(); stmt->content.set(assignment); @@ -7759,7 +7929,7 @@ private: } if (checkVar.empty() || !isLocal(checkVar)) { checkVar = getUnusedName("_check_"sv); - auto assignment = assignmentFrom(toAst(checkVar, inExp), newExp(inExp, inExp), inExp); + auto assignment = assignmentFrom(makeVariableExp(checkVar, inExp), newExp(inExp, inExp), inExp); transformAssignment(assignment, temp); } if (varName.empty()) { @@ -7768,7 +7938,7 @@ private: newUnaryExp->expos.dup(unary_exp->expos); auto exp = newExp(newUnaryExp, x); varName = getUnusedName("_val_"sv); - auto assignExp = toAst(varName, x); + auto assignExp = makeVariableExp(varName, x); auto assignment = assignmentFrom(assignExp, exp, x); transformAssignment(assignment, temp); } @@ -7825,7 +7995,7 @@ private: newUnaryExp->expos.dup(unary_exp->expos); auto exp = newExp(newUnaryExp, x); auto newVar = getUnusedName("_val_"sv); - auto assignExp = toAst(newVar, x); + auto assignExp = makeVariableExp(newVar, x); auto assignment = assignmentFrom(assignExp, exp, x); transformAssignment(assignment, temp); @@ -8280,7 +8450,7 @@ private: } case ExpUsage::Assignment: { auto assign = x->new_ptr(); - assign->values.push_back(toAst(tableVar, x)); + assign->values.push_back(makeVariableExp(tableVar, x)); auto assignment = x->new_ptr(); assignment->expList.set(assignList); assignment->action.set(assign); @@ -8722,7 +8892,7 @@ private: case ExpUsage::Assignment: { out.push_back(clearBuf()); auto assign = x->new_ptr(); - assign->values.push_back(toAst(accumVar, x)); + assign->values.push_back(makeVariableExp(accumVar, x)); auto assignment = x->new_ptr(); assignment->expList.set(assignList); assignment->action.set(assign); @@ -8771,7 +8941,7 @@ private: case id(): case id(): { auto desVar = getUnusedName("_des_"sv); - destructPairs.emplace_back(item, toAst(desVar, x)); + destructPairs.emplace_back(item, makeVariableExp(desVar, x)); vars.push_back(desVar); varAfter.push_back(desVar); break; @@ -10227,7 +10397,7 @@ private: str_list tmp; if (usage == ExpUsage::Assignment) { auto assign = x->new_ptr(); - assign->values.push_back(toAst(classVar, x)); + assign->values.push_back(makeVariableExp(classVar, x)); auto assignment = x->new_ptr(); assignment->expList.set(expList); assignment->action.set(assign); @@ -10509,7 +10679,7 @@ private: if (withVar.empty()) { withVar = getUnusedName("_with_"sv); auto assignment = x->new_ptr(); - assignment->expList.set(toAst(withVar, x)); + assignment->expList.set(makeVariableExpList(withVar, x)); auto assign = x->new_ptr(); assign->values.push_back(with->assign->values.objects().front()); assignment->action.set(assign); @@ -10523,7 +10693,7 @@ private: auto assignment = x->new_ptr(); assignment->expList.set(with->valueList); auto assign = x->new_ptr(); - assign->values.push_back(toAst(withVar, x)); + assign->values.push_back(makeVariableExp(withVar, x)); bool skipFirst = true; for (auto value : with->assign->values.objects()) { if (skipFirst) { @@ -10551,7 +10721,7 @@ private: if (withVar.empty() || !isLocal(withVar)) { withVar = getUnusedName("_with_"sv); auto assignment = x->new_ptr(); - assignment->expList.set(toAst(withVar, x)); + assignment->expList.set(makeVariableExpList(withVar, x)); auto assign = x->new_ptr(); assign->values.dup(with->valueList->exprs); assignment->action.set(assign); @@ -10635,7 +10805,7 @@ private: } if (with->eop) { auto ifNode = x->new_ptr(); - ifNode->type.set(toAst("if"sv, x)); + ifNode->type.set(makeLeaf("if"sv, x)); ifNode->nodes.push_back(toAst(withVar + "~=nil"s, x)); ifNode->nodes.push_back(with->body); if (breakWithVar.empty()) { @@ -10717,7 +10887,7 @@ private: auto assignment = x->new_ptr(); assignment->expList.set(assignList); auto assign = x->new_ptr(); - assign->values.push_back(toAst(breakWithVar.empty() ? withVar : breakWithVar, x)); + assign->values.push_back(makeVariableExp(breakWithVar.empty() ? withVar : breakWithVar, x)); assignment->action.set(assign); transformAssignment(assignment, temp); } @@ -10899,7 +11069,7 @@ private: } } auto newChain = x->new_ptr(); - auto callable = toAst(_info.moduleName, x); + auto callable = makeVariableCallable(_info.moduleName, x); newChain->items.push_front(callable); newChain->items.push_back(exportNode->target); auto exp = newExp(newChain, x); @@ -10964,7 +11134,7 @@ private: } else if (_info.exportDefault) { auto exp = exportNode->target.to(); auto assignment = x->new_ptr(); - assignment->expList.set(toAst(_info.moduleName, x)); + assignment->expList.set(makeVariableExpList(_info.moduleName, x)); auto assign = x->new_ptr(); assign->values.push_back(exp); assignment->action.set(assign); @@ -10983,7 +11153,7 @@ private: auto name = variableToString(var); assignment->expList.set(toAst(_info.moduleName + "[\""s + name + "\"]"s, x)); auto assign = x->new_ptr(); - assign->values.push_back(toAst(name, x)); + assign->values.push_back(makeVariableExp(name, x)); assignment->action.set(assign); transformAssignment(assignment, temp); assignment->expList.set(assignList); @@ -11469,7 +11639,7 @@ private: ast_ptr objAssign; if (objVar.empty()) { objVar = getUnusedName("_obj_"sv); - auto expList = toAst(objVar, x); + auto expList = makeVariableExpList(objVar, x); auto assign = x->new_ptr(); if (importNode->item.is()) { assign->values.push_back(importNode->item); @@ -11489,7 +11659,7 @@ private: case id(): { auto var = static_cast(name); { - auto callable = toAst(objVar, x); + auto callable = makeVariableCallable(objVar, x); auto dotChainItem = x->new_ptr(); dotChainItem->name.set(var->name); auto chainValue = x->new_ptr(); @@ -11510,7 +11680,7 @@ private: auto var = static_cast(name)->name.get(); { auto nameNode = var->name.get(); - auto callable = toAst(objVar, x); + auto callable = makeVariableCallable(objVar, x); auto colonChain = x->new_ptr(); colonChain->name.set(nameNode); auto chainValue = x->new_ptr(); @@ -11952,11 +12122,11 @@ private: if (whileNode->assignment) { auto x = whileNode; auto repeat = x->new_ptr(); - repeat->condition.set(toAst("false"sv, x)); + repeat->condition.set(makeConstExp("false"sv, x)); auto ifNode = x->new_ptr(); auto ifCond = x->new_ptr(); bool isUntil = _parser.toString(whileNode->type) == "until"sv; - ifNode->type.set(toAst(isUntil ? "unless"sv : "if"sv, x)); + ifNode->type.set(makeLeaf(isUntil ? "unless"sv : "if"sv, x)); ifCond->condition.set(whileNode->condition); ifCond->assignment.set(whileNode->assignment); ifNode->nodes.push_back(ifCond); @@ -12160,7 +12330,7 @@ private: } } objVar = getUnusedName("_exp_"sv); - auto expList = toAst(objVar, x); + auto expList = makeVariableExpList(objVar, x); auto assign = x->new_ptr(); assign->values.push_back(switchNode->target); auto assignment = x->new_ptr(); @@ -12246,7 +12416,7 @@ private: conds.back().append(vStr); } else { auto varName = getUnusedName("_val_"sv); - auto vExp = toAst(varName, chain); + auto vExp = makeVariableExp(varName, chain); auto asmt = assignmentFrom(vExp, newExp(chain, chain), chain); transformAssignment(asmt, temp); transformExp(item.target, conds, ExpUsage::Closure); @@ -12665,7 +12835,7 @@ private: } if (isBreak) { if (breakLoop->valueList) { - auto expList = toAst(join(breakLoop->vars, ","sv), breakLoop); + auto expList = makeVariableExpList(breakLoop->vars, breakLoop); auto assignment = breakLoop->new_ptr(); assignment->expList.set(expList); auto assign = breakLoop->new_ptr(); @@ -12690,7 +12860,7 @@ private: _buf << indent() << "break"sv << nl(breakLoop); out.push_back(clearBuf()); } else { - transformGoto(toAst("goto "s + item.var, breakLoop), temp); + transformGoto(makeGoto(item.var, breakLoop), temp); out.push_back(join(temp)); } } @@ -12778,7 +12948,7 @@ private: return; } auto valName = getUnusedName("_tmp_"); - auto newValue = toAst(valName, value); + auto newValue = makeVariableExp(valName, value); ast_list assignments; for (auto exp : chainAssign->exprs.objects()) { auto assignment = assignmentFrom(static_cast(exp), newValue, exp); diff --git a/src/yuescript/yue_parser.cpp b/src/yuescript/yue_parser.cpp index 8ab667d..62ce386 100644 --- a/src/yuescript/yue_parser.cpp +++ b/src/yuescript/yue_parser.cpp @@ -769,7 +769,46 @@ YueParser::YueParser() { }); SimpleTable = Seperator >> key_value >> *(space >> ',' >> space >> key_value); - Value = inc_exp_level >> ensure(SimpleValue | SimpleTable | ChainValue, dec_exp_level); + auto chain_value_start = pl::user(true_(), [](const item_t& item) { + auto current = item.begin->m_it; + if (current == item.input_end) return false; + auto ch = *current; + if (ch == '@') return true; + if (ch == '$') { + current++; + return current != item.input_end + && (*current == '_' || *current > 255 + || (*current >= 'a' && *current <= 'z') + || (*current >= 'A' && *current <= 'Z')); + } + if (!(ch == '_' || ch > 255 + || (ch >= 'a' && ch <= 'z') + || (ch >= 'A' && ch <= 'Z'))) { + return false; + } + bool ascii = true; + std::string name; + do { + if (*current > 255) { + ascii = false; + } else { + name += static_cast(*current); + } + current++; + if (current == item.input_end) break; + ch = *current; + } while (ch == '_' || ch > 255 + || (ch >= 'a' && ch <= 'z') + || (ch >= 'A' && ch <= 'Z') + || (ch >= '0' && ch <= '9')); + if (ascii && Keywords.find(name) != Keywords.end()) return false; + while (current != item.input_end && (*current == ' ' || *current == '\t')) current++; + return current == item.input_end || *current != ':'; + }); + Value = inc_exp_level >> ensure( + chain_value_start >> ChainValue | + SimpleValue | SimpleTable | ChainValue, + dec_exp_level); single_string_inner = '\\' >> set("'\\") | not_('\'') >> any_char; SingleString = '\'' >> *single_string_inner >> ('\'' | unclosed_single_string_error); @@ -1114,11 +1153,62 @@ YueParser::YueParser() { ConstValue = (expr("nil") | "true" | "false") >> not_alpha_num; - SimpleValue = + simple_value_fallback = TableLit | ConstValue | If | Switch | Try | With | ClassDecl | For | While | Repeat | Do | UnaryValue | TblComprehension | Comprehension | FunLit | Num | VarArg; + SimpleValue = pl::dispatch([](input_it current, input_it end) -> size_t { + if (current == end) return 14; + auto startsKeyword = [current, end](std::string_view keyword) { + auto it = current; + for (char expected : keyword) { + if (it == end || *it != static_cast(expected)) return false; + ++it; + } + if (it == end) return true; + const auto ch = *it; + return !(ch == '_' || (ch >= 'a' && ch <= 'z') + || (ch >= 'A' && ch <= 'Z') || (ch >= '0' && ch <= '9')); + }; + switch (*current) { + case '{': return 14; + case '-': { + auto next = current + 1; + return next != end && *next == '>' ? 14 : 11; + } + case '#': case '~': return 11; + case '.': { + auto next = current + 1; + if (next != end && *next >= '0' && *next <= '9') return 12; + if (next != end && *next == '.') { + ++next; + if (next != end && *next == '.') return 13; + } + return 14; + } + default: + if (*current >= '0' && *current <= '9') return 12; + break; + } + if (startsKeyword("nil"sv) || startsKeyword("true"sv) || startsKeyword("false"sv)) return 1; + if (startsKeyword("if"sv) || startsKeyword("unless"sv)) return 2; + if (startsKeyword("switch"sv)) return 3; + if (startsKeyword("try"sv)) return 4; + if (startsKeyword("with"sv)) return 5; + if (startsKeyword("class"sv)) return 6; + if (startsKeyword("for"sv)) return 7; + if (startsKeyword("while"sv) || startsKeyword("until"sv)) return 8; + if (startsKeyword("repeat"sv)) return 9; + if (startsKeyword("do"sv)) return 10; + if (startsKeyword("not"sv)) return 11; + return 14; + }, { + TableLit.this_ptr(), ConstValue.this_ptr(), If.this_ptr(), Switch.this_ptr(), + Try.this_ptr(), With.this_ptr(), ClassDecl.this_ptr(), For.this_ptr(), + While.this_ptr(), Repeat.this_ptr(), Do.this_ptr(), UnaryValue.this_ptr(), + Num.this_ptr(), VarArg.this_ptr(), simple_value_fallback.this_ptr() + }); ExpListAssign = ExpList >> -(space >> (Update | Assign | SubBackcall)) >> not_(space >> '='); @@ -1204,7 +1294,7 @@ bool YueParser::startWith(std::string_view codes, rule& r) { error_list errors; try { State state; - return ::yue::start_with(*converted, r, errors, &state); + return ::yue::start_with(*converted, r, errors, &state, false); } catch (const ParserError&) { return false; } catch (const std::logic_error&) { @@ -1232,7 +1322,7 @@ ParseInfo YueParser::parse(std::string_view codes, rule& r, bool lax) { try { State state; state.lax = lax; - res.node.set(::yue::parse(*(res.codes), r, errors, &state)); + res.node.set(::yue::parse(*(res.codes), r, errors, &state, false)); if (state.exportCount > 0) { int index = 0; std::string moduleName; diff --git a/src/yuescript/yue_parser.h b/src/yuescript/yue_parser.h index cfcbb48..f122b59 100644 --- a/src/yuescript/yue_parser.h +++ b/src/yuescript/yue_parser.h @@ -405,6 +405,7 @@ private: AST_RULE(ChainValue); AST_RULE(SimpleTable); AST_RULE(SimpleValue); + NONE_AST_RULE(simple_value_fallback); AST_RULE(Value); AST_RULE(LuaStringOpen); AST_RULE(LuaStringContent); -- cgit v1.2.3-55-g6feb