From b375f9d930f787c9224b5eaf8940486802098778 Mon Sep 17 00:00:00 2001 From: Hisham Muhammad Date: Fri, 8 Nov 2019 18:08:13 -0300 Subject: refactor dependency scanning for reuse when building lockfile --- spec/install_spec.lua | 2 +- src/luarocks/cmd.lua | 6 ++-- src/luarocks/core/dir.lua | 12 ++++++++ src/luarocks/core/manif.lua | 47 +++++++++++++++++++++++++++++++ src/luarocks/core/path.lua | 38 +++++++++++++++++++++++++ src/luarocks/dir.lua | 13 +-------- src/luarocks/loader.lua | 67 ++++++--------------------------------------- src/luarocks/manif.lua | 2 ++ src/luarocks/path.lua | 39 ++------------------------ 9 files changed, 114 insertions(+), 112 deletions(-) diff --git a/spec/install_spec.lua b/spec/install_spec.lua index 9b082ab8..bf50ced7 100644 --- a/spec/install_spec.lua +++ b/spec/install_spec.lua @@ -184,7 +184,7 @@ describe("luarocks install #integration", function() it("only-deps of luasocket packed rock", function() assert.is_true(run.luarocks_bool("build --pack-binary-rock luasocket 3.0rc1-2")) local output = run.luarocks("install --only-deps " .. "luasocket-3.0rc1-2." .. test_env.platform .. ".rock") - assert.are.same("Successfully installed dependencies for luasocket 3.0rc1-2", output:gsub("\n", "")) + assert.match("Successfully installed dependencies for luasocket 3.0rc1-2", output, 1, true) assert.is_true(os.remove("luasocket-3.0rc1-2." .. test_env.platform .. ".rock")) end) diff --git a/src/luarocks/cmd.lua b/src/luarocks/cmd.lua index 36080eb9..eef043e7 100644 --- a/src/luarocks/cmd.lua +++ b/src/luarocks/cmd.lua @@ -2,7 +2,7 @@ --- Functions for command-line scripts. local cmd = {} -local loader = require("luarocks.loader") +local manif = require("luarocks.manif") local util = require("luarocks.util") local path = require("luarocks.path") local cfg = require("luarocks.core.cfg") @@ -94,7 +94,7 @@ do elseif args.project_tree then local tree = args.project_tree table.insert(cfg.rocks_trees, 1, { name = "project", root = tree } ) - loader.load_rocks_trees() + manif.load_rocks_tree_manifests() path.use_tree(tree) elseif cfg.local_by_default then if cfg.home_tree then @@ -103,7 +103,7 @@ do elseif project_dir then local project_tree = project_dir .. "/lua_modules" table.insert(cfg.rocks_trees, 1, { name = "project", root = project_tree } ) - loader.load_rocks_trees() + manif.load_rocks_tree_manifests() path.use_tree(project_tree) else local trees = cfg.rocks_trees diff --git a/src/luarocks/core/dir.lua b/src/luarocks/core/dir.lua index 70a82bd4..59b7749c 100644 --- a/src/luarocks/core/dir.lua +++ b/src/luarocks/core/dir.lua @@ -37,5 +37,17 @@ function dir.split_url(url) return protocol, pathname end +--- Normalize a url or local path. +-- URLs should be in the "protocol://path" format. System independent +-- forward slashes are used, removing trailing and double slashes +-- @param url string: an URL or a local pathname. +-- @return string: Normalized result. +function dir.normalize(name) + local protocol, pathname = dir.split_url(name) + pathname = pathname:gsub("\\", "/"):gsub("(.)/*$", "%1"):gsub("//", "/") + if protocol ~= "file" then pathname = protocol .."://"..pathname end + return pathname +end + return dir diff --git a/src/luarocks/core/manif.lua b/src/luarocks/core/manif.lua index 7a561f8a..4fd35c6c 100644 --- a/src/luarocks/core/manif.lua +++ b/src/luarocks/core/manif.lua @@ -5,6 +5,9 @@ local manif = {} local persist = require("luarocks.core.persist") local cfg = require("luarocks.core.cfg") local dir = require("luarocks.core.dir") +local util = require("luarocks.core.util") +local vers = require("luarocks.core.vers") +local path = require("luarocks.core.path") local require = nil -------------------------------------------------------------------------------- @@ -64,4 +67,48 @@ function manif.fast_load_local_manifest(repo_url) return manif.manifest_loader(pathname, repo_url, nil, true) end +function manif.load_rocks_tree_manifests(deps_mode) + local trees = {} + path.map_trees(deps_mode, function(tree) + local manifest, err = manif.fast_load_local_manifest(path.rocks_dir(tree)) + if manifest then + table.insert(trees, {tree=tree, manifest=manifest}) + end + end) + return trees +end + +function manif.scan_dependencies(name, version, tree_manifests, dest) + if dest[name] then + return + end + dest[name] = version + + for _, tree in ipairs(tree_manifests) do + local manifest = tree.manifest + + local pkgdeps + if manifest.dependencies and manifest.dependencies[name] then + pkgdeps = manifest.dependencies[name][version] + end + if not pkgdeps then + return nil + end + for _, dep in ipairs(pkgdeps) do + local pkg, constraints = dep.name, dep.constraints + + for _, t in ipairs(tree_manifests) do + local entries = t.manifest.repository[pkg] + if entries then + for ver, _ in util.sortedpairs(entries, vers.compare_versions) do + if (not constraints) or vers.match_constraints(vers.parse_version(ver), constraints) then + manif.scan_dependencies(pkg, version, tree_manifests, dest) + end + end + end + end + end + end +end + return manif diff --git a/src/luarocks/core/path.lua b/src/luarocks/core/path.lua index 179d3aaf..bf6bdc2d 100644 --- a/src/luarocks/core/path.lua +++ b/src/luarocks/core/path.lua @@ -109,4 +109,42 @@ function path.which_i(file_name, name, version, tree, i) return file_name end +function path.rocks_tree_to_string(tree) + if type(tree) == "string" then + return tree + else + assert(type(tree) == "table") + return tree.root + end +end + +--- Apply a given function to the active rocks trees based on chosen dependency mode. +-- @param deps_mode string: Dependency mode: "one" for the current default tree, +-- "all" for all trees, "order" for all trees with priority >= the current default, +-- "none" for no trees (this function becomes a nop). +-- @param fn function: function to be applied, with the tree dir (string) as the first +-- argument and the remaining varargs of map_trees as the following arguments. +-- @return a table with all results of invocations of fn collected. +function path.map_trees(deps_mode, fn, ...) + local result = {} + local current = cfg.root_dir or cfg.rocks_trees[1] + if deps_mode == "one" then + table.insert(result, (fn(current, ...)) or 0) + else + local use = false + if deps_mode == "all" then + use = true + end + for _, tree in ipairs(cfg.rocks_trees or {}) do + if dir.normalize(path.rocks_tree_to_string(tree)) == dir.normalize(path.rocks_tree_to_string(current)) then + use = true + end + if use then + table.insert(result, (fn(tree, ...)) or 0) + end + end + end + return result +end + return path diff --git a/src/luarocks/dir.lua b/src/luarocks/dir.lua index 679e8f0d..74adbea2 100644 --- a/src/luarocks/dir.lua +++ b/src/luarocks/dir.lua @@ -6,6 +6,7 @@ local core = require("luarocks.core.dir") dir.path = core.path dir.split_url = core.split_url +dir.normalize = core.normalize --- Strip the path off a path+filename. -- @param pathname string: A path+name, such as "/a/b/c" @@ -28,18 +29,6 @@ function dir.dir_name(pathname) return (pathname:gsub("/*$", ""):match("(.*)[/]+[^/]*")) or "" end ---- Normalize a url or local path. --- URLs should be in the "protocol://path" format. System independent --- forward slashes are used, removing trailing and double slashes --- @param url string: an URL or a local pathname. --- @return string: Normalized result. -function dir.normalize(name) - local protocol, pathname = dir.split_url(name) - pathname = pathname:gsub("\\", "/"):gsub("(.)/*$", "%1"):gsub("//", "/") - if protocol ~= "file" then pathname = protocol .."://"..pathname end - return pathname -end - --- Returns true if protocol does not require additional tools. -- @param protocol The protocol name function dir.is_basic_protocol(protocol) diff --git a/src/luarocks/loader.lua b/src/luarocks/loader.lua index 780a0371..0cc76027 100644 --- a/src/luarocks/loader.lua +++ b/src/luarocks/loader.lua @@ -24,7 +24,6 @@ end local path = require("luarocks.core.path") local manif = require("luarocks.core.manif") local vers = require("luarocks.core.vers") -local util = require("luarocks.core.util") local require = nil -------------------------------------------------------------------------------- @@ -52,29 +51,6 @@ end loader.context = {} --- Contains a table when rocks trees are loaded, --- or 'false' to indicate rocks trees failed to load. --- 'nil' indicates rocks trees were not attempted to be loaded yet. -loader.rocks_trees = nil - -function loader.load_rocks_trees() - local any_ok = false - local trees = {} - for _, tree in ipairs(cfg.rocks_trees or {}) do - local manifest, err = manif.fast_load_local_manifest(path.rocks_dir(tree)) - if manifest then - any_ok = true - table.insert(trees, {tree=tree, manifest=manifest}) - end - end - if not any_ok then - loader.rocks_trees = false - return false - end - loader.rocks_trees = trees - return true -end - --- Process the dependencies of a package to determine its dependency -- chain for loading modules. -- @param name string: The name of an installed rock. @@ -90,40 +66,12 @@ function loader.add_context(name, version) temporary_global = false end - if loader.context[name] then - return - end - loader.context[name] = version - - if not loader.rocks_trees and not loader.load_rocks_trees() then + local tree_manifests = manif.load_rocks_tree_manifests() + if not tree_manifests then return nil end - for _, tree in ipairs(loader.rocks_trees) do - local manifest = tree.manifest - - local pkgdeps - if manifest.dependencies and manifest.dependencies[name] then - pkgdeps = manifest.dependencies[name][version] - end - if not pkgdeps then - return nil - end - for _, dep in ipairs(pkgdeps) do - local pkg, constraints = dep.name, dep.constraints - - for _, tree in ipairs(loader.rocks_trees) do - local entries = tree.manifest.repository[pkg] - if entries then - for ver, pkgs in util.sortedpairs(entries, vers.compare_versions) do - if (not constraints) or vers.match_constraints(vers.parse_version(ver), constraints) then - loader.add_context(pkg, version) - end - end - end - end - end - end + return manif.scan_dependencies(name, version, tree_manifests, loader.context) end --- Internal sorting function. @@ -186,18 +134,19 @@ end -- * name of the rock containing the module (eg. "luasocket") -- * version of the rock (eg. "2.0.2-1") -- * return value of filter_file_name --- * tree of the module (string or table in `rocks_trees` format) +-- * tree of the module (string or table in `tree_manifests` format) local function select_module(module, filter_file_name) --assert(type(module) == "string") --assert(type(filter_module_name) == "function") - if not loader.rocks_trees and not loader.load_rocks_trees() then + local tree_manifests = manif.load_rocks_tree_manifests() + if not tree_manifests then return nil end local providers = {} local initmodule - for _, tree in ipairs(loader.rocks_trees) do + for _, tree in ipairs(tree_manifests) do local entries = tree.manifest.modules[module] if entries then add_providers(providers, entries, tree, module, filter_file_name) @@ -223,7 +172,7 @@ end -- * name of the rock containing the module (eg. "luasocket") -- * version of the rock (eg. "2.0.2-1") -- * name of the module (eg. "socket.core", or "socket.core_2_0_2" if file is stored versioned). --- * tree of the module (string or table in `rocks_trees` format) +-- * tree of the module (string or table in `tree_manifests` format) local function pick_module(module) return select_module(module, function(file_name, name, version, tree, i) diff --git a/src/luarocks/manif.lua b/src/luarocks/manif.lua index 1337cab3..b9eadc5b 100644 --- a/src/luarocks/manif.lua +++ b/src/luarocks/manif.lua @@ -16,6 +16,8 @@ local queries = require("luarocks.queries") local type_manifest = require("luarocks.type.manifest") manif.cache_manifest = core.cache_manifest +manif.load_rocks_tree_manifests = core.load_rocks_tree_manifests +manif.scan_dependencies = core.scan_dependencies manif.rock_manifest_cache = {} diff --git a/src/luarocks/path.lua b/src/luarocks/path.lua index c7ab9283..b3b01704 100644 --- a/src/luarocks/path.lua +++ b/src/luarocks/path.lua @@ -14,6 +14,8 @@ path.versioned_name = core.versioned_name path.path_to_module = core.path_to_module path.deploy_lua_dir = core.deploy_lua_dir path.deploy_lib_dir = core.deploy_lib_dir +path.map_trees = core.map_trees +path.rocks_tree_to_string = core.rocks_tree_to_string --- Infer rockspec filename from a rock filename. -- @param rock_name string: Pathname of a rock file. @@ -226,43 +228,6 @@ function path.use_tree(tree) -- .. package.cpath end -function path.rocks_tree_to_string(tree) - if type(tree) == "string" then - return tree - else - assert(type(tree) == "table") - return tree.root - end -end - ---- Apply a given function to the active rocks trees based on chosen dependency mode. --- @param deps_mode string: Dependency mode: "one" for the current default tree, --- "all" for all trees, "order" for all trees with priority >= the current default, --- "none" for no trees (this function becomes a nop). --- @param fn function: function to be applied, with the tree dir (string) as the first --- argument and the remaining varargs of map_trees as the following arguments. --- @return a table with all results of invocations of fn collected. -function path.map_trees(deps_mode, fn, ...) - local result = {} - if deps_mode == "one" then - table.insert(result, (fn(cfg.root_dir, ...)) or 0) - elseif deps_mode == "all" or deps_mode == "order" then - local use = false - if deps_mode == "all" then - use = true - end - for _, tree in ipairs(cfg.rocks_trees) do - if dir.normalize(path.rocks_tree_to_string(tree)) == dir.normalize(path.rocks_tree_to_string(cfg.root_dir)) then - use = true - end - if use then - table.insert(result, (fn(tree, ...)) or 0) - end - end - end - return result -end - --- Get the namespace of a locally-installed rock, if any. -- @param name string: The rock name, without a namespace. -- @param version string: The rock version. -- cgit v1.2.3-55-g6feb