From c03bf36db11bcd90034b0e67bd1f5c8c0765eb7f Mon Sep 17 00:00:00 2001 From: Li Jin Date: Thu, 17 Jul 2025 18:02:39 +0800 Subject: Added YAML multiline string and macro argument checking. --- src/yuescript/yue_ast.cpp | 24 +++++++ src/yuescript/yue_ast.h | 28 +++++++- src/yuescript/yue_compiler.cpp | 142 +++++++++++++++++++++++++++++++++++++++-- src/yuescript/yue_parser.cpp | 34 ++++++++-- src/yuescript/yue_parser.h | 5 ++ 5 files changed, 221 insertions(+), 12 deletions(-) (limited to 'src') diff --git a/src/yuescript/yue_ast.cpp b/src/yuescript/yue_ast.cpp index da99d07..bacdc01 100644 --- a/src/yuescript/yue_ast.cpp +++ b/src/yuescript/yue_ast.cpp @@ -962,6 +962,30 @@ std::string DoubleString_t::to_string(void* ud) const { } return '"' + join(temp) + '"'; } +std::string YAMLLineInner_t::to_string(void* ud) const { + auto info = reinterpret_cast(ud); + return info->convert(this); +} +std::string YAMLLineContent_t::to_string(void* ud) const { + if (content.is()) { + return "#{"s + content->to_string(ud) + '}'; + } + return content->to_string(ud); +} +std::string YAMLLine_t::to_string(void* ud) const { + str_list temp; + for (auto seg : segments.objects()) { + temp.emplace_back(seg->to_string(ud)); + } + return join(temp); +} +std::string YAMLMultiline_t::to_string(void* ud) const { + str_list temp; + for (auto seg : lines.objects()) { + temp.emplace_back(seg->to_string(ud)); + } + return "|\n" + join(temp, "\n"sv); +} std::string String_t::to_string(void* ud) const { return str->to_string(ud); } diff --git a/src/yuescript/yue_ast.h b/src/yuescript/yue_ast.h index d396d82..c529f34 100644 --- a/src/yuescript/yue_ast.h +++ b/src/yuescript/yue_ast.h @@ -591,8 +591,28 @@ AST_NODE(DoubleString) AST_MEMBER(DoubleString, &sep, &segments) AST_END(DoubleString) +AST_LEAF(YAMLLineInner) +AST_END(YAMLLineInner) + +AST_NODE(YAMLLineContent) + ast_sel content; + AST_MEMBER(YAMLLineContent, &content) +AST_END(YAMLLineContent) + +AST_NODE(YAMLLine) + ast_ptr sep; + ast_list segments; + AST_MEMBER(YAMLLine, &sep, &segments) +AST_END(YAMLLine) + +AST_NODE(YAMLMultiline) + ast_ptr sep; + ast_list lines; + AST_MEMBER(YAMLMultiline, &sep, &lines) +AST_END(YAMLMultiline) + AST_NODE(String) - ast_sel str; + ast_sel str; AST_MEMBER(String, &str) AST_END(String) @@ -752,15 +772,17 @@ AST_END(Export) AST_NODE(FnArgDef) ast_sel name; ast_ptr op; + ast_ptr label; ast_ptr defaultValue; - AST_MEMBER(FnArgDef, &name, &op, &defaultValue) + AST_MEMBER(FnArgDef, &name, &op, &label, &defaultValue) AST_END(FnArgDef) AST_NODE(FnArgDefList) ast_ptr sep; ast_list definitions; ast_ptr varArg; - AST_MEMBER(FnArgDefList, &sep, &definitions, &varArg) + ast_ptr label; + AST_MEMBER(FnArgDefList, &sep, &definitions, &varArg, &label) AST_END(FnArgDefList) AST_NODE(OuterVarShadow) diff --git a/src/yuescript/yue_compiler.cpp b/src/yuescript/yue_compiler.cpp index 9f5a41e..35d99bd 100644 --- a/src/yuescript/yue_compiler.cpp +++ b/src/yuescript/yue_compiler.cpp @@ -78,7 +78,7 @@ static std::unordered_set Metamethods = { "close"s // Lua 5.4 }; -const std::string_view version = "0.29.0"sv; +const std::string_view version = "0.29.1"sv; const std::string_view extension = "yue"sv; class CompileError : public std::logic_error { @@ -5418,18 +5418,29 @@ private: auto macroLit = macro->decl.to(); auto argsDef = macroLit->argsDef.get(); str_list newArgs; + str_list argChecks; + bool hasCheck = false; if (argsDef) { for (auto def_ : argsDef->definitions.objects()) { auto def = static_cast(def_); if (def->name.is()) { throw CompileError("self name is not supported for macro function argument"sv, def->name); } else { + if (def->op) throw CompileError("invalid existence checking"sv, def->op); + if (def->label) { + hasCheck = true; + const auto& astName = argChecks.emplace_back(_parser.toString(def->label)); + if (!_parser.hasAST(astName)) { + throw CompileError("invalid AST name"sv, def->label); + } + } else { + argChecks.emplace_back(); + } std::string defVal; if (def->defaultValue) { defVal = _parser.toString(def->defaultValue); Utils::trim(defVal); - defVal.insert(0, "=[==========["sv); - defVal.append("]==========]"sv); + defVal = '=' + Utils::toLuaString(defVal); } newArgs.emplace_back(_parser.toString(def->name) + defVal); } @@ -5438,6 +5449,14 @@ private: newArgs.emplace_back(_parser.toString(argsDef->varArg)); } } + if (argsDef->label) { + hasCheck = true; + const auto& astName = _parser.toString(argsDef->label); + if (!_parser.hasAST(astName)) { + throw CompileError("invalid AST name"sv, argsDef->label); + } + argChecks.emplace_back("..."s + astName); + } std::string macroCodes = "_ENV=require('yue').macro_env\n("s + join(newArgs, ","sv) + ")->"s + _parser.toString(macroLit->body); auto chunkName = "=(macro "s + macroName + ')'; pushCurrentModule(); // cur @@ -5467,6 +5486,24 @@ private: throw CompileError("failed to generate macro function\n"s + err, macroLit); } // cur true macro lua_remove(L, -2); // cur macro + if (hasCheck) { + lua_createtable(L, 0, 0); // cur macro checks + int i = 1; + for (const auto& check : argChecks) { + if (check.empty()) { + lua_pushboolean(L, 0); + lua_rawseti(L, -2, i); + } else { + lua_pushlstring(L, check.c_str(), check.size()); + lua_rawseti(L, -2, i); + } + i++; + } + lua_createtable(L, 2, 0); // cur macro checks macrotab + lua_insert(L, -3); // cur macrotab macro checks + lua_rawseti(L, -3, 1); // macrotab[1] = checks, cur macrotab macro + lua_rawseti(L, -2, 2); // macrotab[2] = macro, cur macrotab + } // cur macro if (exporting && _config.exporting && !_config.module.empty()) { pushModuleTable(_config.module); // cur macro module lua_pushlstring(L, macroName.c_str(), macroName.size()); // cur macro module name @@ -5624,7 +5661,11 @@ private: auto def = static_cast(_def); auto& arg = argItems.emplace_back(); switch (def->name->get_id()) { - case id(): arg.name = variableToString(static_cast(def->name.get())); break; + case id(): { + if (def->op) throw CompileError("invalid existence checking"sv, def->op); + arg.name = variableToString(static_cast(def->name.get())); + break; + } case id(): { assignSelf = true; if (def->op) { @@ -6748,7 +6789,25 @@ private: break; } } - if (!lua_isfunction(L, -1)) { + str_list checks; + if (lua_istable(L, -1)) { + lua_rawgeti(L, -1, 1); // cur macrotab checks + int len = lua_objlen(L, -1); + for (int i = 1; i <= len; i++) { + lua_rawgeti(L, -1, i); + if (lua_toboolean(L, -1) == 0) { + checks.emplace_back(); + } else { + size_t str_len = 0; + auto str = lua_tolstring(L, -1, &str_len); + checks.emplace_back(std::string{str, str_len}); + } + lua_pop(L, 1); + } + lua_pop(L, 1); + lua_rawgeti(L, -1, 2); // cur macrotab macroFunc + lua_remove(L, -2); // cur macroFunc + } else if (!lua_isfunction(L, -1)) { auto code = expandBuiltinMacro(macroName, x); if (!code.empty()) return code; if (macroName == "is_ast"sv) { @@ -6796,8 +6855,31 @@ private: if (!lua_checkstack(L, argStrs.size())) { throw CompileError("too much macro params"s, x); } + auto checkIt = checks.begin(); + node_container::const_iterator argIt; + if (args) { + argIt = args->begin(); + } for (const auto& arg : argStrs) { + if (checkIt != checks.end()) { + if (checkIt->empty()) { + ++checkIt; + } else { + if ((*checkIt)[0] == '.') { + auto astName = checkIt->substr(3); + if (!_parser.match(astName, arg)) { + throw CompileError("expecting \""s + astName + "\", AST mismatch"s, *argIt); + } + } else { + if (!_parser.match(*checkIt, arg)) { + throw CompileError("expecting \""s + *checkIt + "\", AST mismatch"s, *argIt); + } + ++checkIt; + } + } + } lua_pushlstring(L, arg.c_str(), arg.size()); + ++argIt; } // cur pcall macroFunc args... bool success = lua_pcall(L, static_cast(argStrs.size()), 1, 0) == 0; if (!success) { // cur err @@ -9096,12 +9178,62 @@ private: out.push_back(temp.empty() ? "\"\""s : join(temp, " .. "sv)); } + void transformYAMLMultiline(YAMLMultiline_t* multiline, str_list& out) { + std::optional indent; + str_list temp; + for (auto line_ : multiline->lines.objects()) { + auto line = static_cast(line_); + if (!line->segments.empty()) { + str_list segs; + for (auto seg_ : line->segments.objects()) { + auto content = static_cast(seg_)->content.get(); + switch (content->get_id()) { + case id(): { + auto str = _parser.toString(content); + Utils::replace(str, "\r\n"sv, "\n"sv); + Utils::replace(str, "\n"sv, "\\n"sv); + Utils::replace(str, "\\#"sv, "#"sv); + segs.push_back('\"' + str + '\"'); + break; + } + case id(): { + transformExp(static_cast(content), segs, ExpUsage::Closure); + segs.back() = globalVar("tostring"sv, content, AccessType::Read) + '(' + segs.back() + ')'; + break; + } + default: YUEE("AST node mismatch", content); break; + } + } + auto lineStr = join(segs, " .. "sv); + if (!indent) { + auto pos = lineStr.find_first_not_of("\t "sv, 1); + if (pos == std::string::npos) { + throw CompileError("expecting first line indent"sv, line); + } + indent = std::string_view{lineStr.c_str(), pos}; + } else { + if (std::string_view{lineStr}.substr(0, indent.value().size()) != indent.value()) { + throw CompileError("inconsistent indent"sv, line); + } + } + lineStr = '"' + lineStr.substr(indent.value().size()); + temp.push_back(lineStr); + } + } + auto str = join(temp, " .. '\\n' .. "sv); + Utils::replace(str, "\" .. '\\n' .. \""sv, "\\n"sv); + Utils::replace(str, "\" .. '\\n'"sv, "\\n\""sv); + Utils::replace(str, "'\\n' .. \""sv, "\"\\n"sv); + out.push_back(str); + } + void transformString(String_t* string, str_list& out) { auto str = string->str.get(); switch (str->get_id()) { case id(): transformSingleString(static_cast(str), out); break; case id(): transformDoubleString(static_cast(str), out); break; case id(): transformLuaString(static_cast(str), out); break; + case id(): transformYAMLMultiline(static_cast(str), out); break; default: YUEE("AST node mismatch", str); break; } } diff --git a/src/yuescript/yue_parser.cpp b/src/yuescript/yue_parser.cpp index 1011a49..eebc676 100644 --- a/src/yuescript/yue_parser.cpp +++ b/src/yuescript/yue_parser.cpp @@ -639,7 +639,15 @@ YueParser::YueParser() { DoubleStringInner = +(not_("#{") >> double_string_plain); DoubleStringContent = DoubleStringInner | interp; DoubleString = '"' >> Seperator >> *DoubleStringContent >> '"'; - String = DoubleString | SingleString | LuaString; + + YAMLLineInner = +('\\' >> set("\"\\#") | not_("#{" | stop) >> any_char); + YAMLLineContent = YAMLLineInner | interp; + YAMLLine = check_indent_match >> Seperator >> +YAMLLineContent | + advance_match >> Seperator >> ensure(+YAMLLineContent, pop_indent) | + Seperator >> *set(" \t") >> and_(line_break); + YAMLMultiline = '|' >> Seperator >> +space_break >> advance_match >> ensure(YAMLLine >> *(*set(" \t") >> line_break >> YAMLLine), pop_indent); + + String = DoubleString | SingleString | LuaString | YAMLMultiline; lua_string_open = '[' >> *expr('=') >> '['; lua_string_close = ']' >> *expr('=') >> ']'; @@ -883,11 +891,11 @@ YueParser::YueParser() { fn_arg_def_lit_lines = fn_arg_def_lit_line >> *(-(space >> ',') >> space_break >> fn_arg_def_lit_line); - FnArgDef = (Variable | SelfItem >> -ExistentialOp) >> -(space >> '=' >> space >> Exp); + FnArgDef = (Variable | SelfItem >> -ExistentialOp) >> -(space >> '`' >> space >> Name) >> -(space >> '=' >> space >> Exp); FnArgDefList = Seperator >> ( - fn_arg_def_lit_lines >> -(-(space >> ',') >> white >> VarArg) | - white >> VarArg + fn_arg_def_lit_lines >> -(-(space >> ',') >> white >> VarArg >> -(space >> '`' >> space >> Name)) | + white >> VarArg >> -(space >> '`' >> space >> Name) ); OuterVarShadow = key("using") >> space >> (NameList | key("nil")); @@ -1176,6 +1184,24 @@ void trim(std::string& str) { str.erase(0, str.find_first_not_of(" \t\r\n")); str.erase(str.find_last_not_of(" \t\r\n") + 1); } + +std::string toLuaString(const std::string& input) { + std::string luaStr = "\""; + for (char c : input) { + switch (c) { + case '\"': luaStr += "\\\""; break; + case '\\': luaStr += "\\\\"; break; + case '\n': luaStr += "\\n"; break; + case '\r': luaStr += "\\r"; break; + case '\t': luaStr += "\\t"; break; + default: + luaStr += c; + break; + } + } + luaStr += "\""; + return luaStr; +} } // namespace Utils std::string ParseInfo::errorMessage(std::string_view msg, int errLine, int errCol, int lineOffset) const { diff --git a/src/yuescript/yue_parser.h b/src/yuescript/yue_parser.h index 1057626..15f9277 100644 --- a/src/yuescript/yue_parser.h +++ b/src/yuescript/yue_parser.h @@ -378,6 +378,10 @@ private: AST_RULE(DoubleStringInner); AST_RULE(DoubleStringContent); AST_RULE(DoubleString); + AST_RULE(YAMLLineInner); + AST_RULE(YAMLLineContent); + AST_RULE(YAMLLine); + AST_RULE(YAMLMultiline); AST_RULE(String); AST_RULE(Parens); AST_RULE(DotChainItem); @@ -453,6 +457,7 @@ private: namespace Utils { void replace(std::string& str, std::string_view from, std::string_view to); void trim(std::string& str); +std::string toLuaString(const std::string& input); } // namespace Utils } // namespace yue -- cgit v1.2.3-55-g6feb