From c2fde487834c6d7505d7e803256e8764237d4476 Mon Sep 17 00:00:00 2001 From: Hisham Muhammad Date: Mon, 3 Sep 2018 19:53:18 +0200 Subject: repos: fix upgrade and downgrade of files when module names clash This reworks the logic of upgrading/downgrading multiple versions of the same package (or different packages) when they have different files that resolve to the same module name (foo.lua and foo.so). For a concrete example, this was hitting LuaSec downgrades. --- src/luarocks/cmd/show.lua | 55 +++++-- src/luarocks/repos.lua | 394 ++++++++++++++++++++++++++-------------------- 2 files changed, 262 insertions(+), 187 deletions(-) (limited to 'src') diff --git a/src/luarocks/cmd/show.lua b/src/luarocks/cmd/show.lua index c90a6bea..bb0fdfd7 100644 --- a/src/luarocks/cmd/show.lua +++ b/src/luarocks/cmd/show.lua @@ -5,6 +5,7 @@ local show = {} local queries = require("luarocks.queries") local search = require("luarocks.search") local dir = require("luarocks.core.dir") +local fs = require("luarocks.fs") local cfg = require("luarocks.core.cfg") local util = require("luarocks.util") local path = require("luarocks.path") @@ -145,25 +146,52 @@ local function render(template, data) return table.concat(out, "\n") end -local function files_to_list(name, version, item_set, item_type, repo) - local ret = {} - for item_name in util.sortedpairs(item_set) do - table.insert(ret, { name = item_name, file = repos.which(name, version, item_type, item_name, repo) }) - end - return ret +local function adjust_path(name, version, basedir, pathname, suffix) + pathname = dir.path(basedir, pathname) + local vpathname = path.versioned_name(pathname, basedir, name, version) + return (fs.exists(vpathname) + and vpathname + or pathname) .. (suffix or "") end local function modules_to_list(name, version, repo) local ret = {} local rock_manifest = manif.load_rock_manifest(name, version, repo) - local lua_dir = path.lua_dir(name, version, repo) - local lib_dir = path.lib_dir(name, version, repo) - repos.recurse_rock_manifest_entry(rock_manifest.lua, function(pathname, modname) - table.insert(ret, { name = modname, file = dir.path(lua_dir, pathname) }) + local lua_dir = path.deploy_lua_dir(repo) + local lib_dir = path.deploy_lib_dir(repo) + repos.recurse_rock_manifest_entry(rock_manifest.lua, function(pathname) + table.insert(ret, { + name = path.path_to_module(pathname), + file = adjust_path(name, version, lua_dir, pathname), + }) end) - repos.recurse_rock_manifest_entry(rock_manifest.lib, function(pathname, modname) - table.insert(ret, { name = modname, file = dir.path(lib_dir, pathname) }) + repos.recurse_rock_manifest_entry(rock_manifest.lib, function(pathname) + table.insert(ret, { + name = path.path_to_module(pathname), + file = adjust_path(name, version, lib_dir, pathname), + }) + end) + table.sort(ret, function(a, b) + if a.name == b.name then + return a.file < b.file + end + return a.name < b.name + end) + return ret +end + +local function commands_to_list(name, version, repo) + local ret = {} + local rock_manifest = manif.load_rock_manifest(name, version, repo) + + local bin_dir = path.deploy_bin_dir(repo) + repos.recurse_rock_manifest_entry(rock_manifest.bin, function(pathname) + pathname = adjust_path(name, version, bin_dir, pathname) + table.insert(ret, { + name = pathname, + file = adjust_path(name, version, bin_dir, pathname, cfg.wrapper_suffix), + }) end) table.sort(ret, function(a, b) if a.name == b.name then @@ -209,7 +237,7 @@ local function show_rock(template, namespace, name, version, rockspec, repo, min issues = desc.issues_url, labels = desc.labels and table.concat(desc.labels, ", "), location = path.rocks_tree_to_string(repo), - commands = files_to_list(name, version, minfo.commands, "command", repo), + commands = commands_to_list(name, version, repo), modules = modules_to_list(name, version, repo), bdeps = deps_to_list(rockspec.build_dependencies, tree), tdeps = deps_to_list(rockspec.test_dependencies, tree), @@ -278,5 +306,4 @@ function show.command(flags, name, version) return true end - return show diff --git a/src/luarocks/repos.lua b/src/luarocks/repos.lua index 37a81c20..eaec87b8 100644 --- a/src/luarocks/repos.lua +++ b/src/luarocks/repos.lua @@ -9,6 +9,21 @@ local util = require("luarocks.util") local dir = require("luarocks.dir") local manif = require("luarocks.manif") local vers = require("luarocks.core.vers") +local E = {} + +local unpack = unpack or table.unpack + +--- Get type and name of an item (a module or a command) provided by a file. +-- @param deploy_type string: rock manifest subtree the file comes from ("bin", "lua", or "lib"). +-- @param file_path string: path to the file relatively to deploy_type subdirectory. +-- @return (string, string): item type ("module" or "command") and name. +local function get_provided_item(deploy_type, file_path) + assert(type(deploy_type) == "string") + assert(type(file_path) == "string") + local item_type = deploy_type == "bin" and "command" or "module" + local item_name = item_type == "command" and file_path or path.path_to_module(file_path) + return item_type, item_name +end -- Tree of files installed by a package are stored -- in its rock manifest. Some of these files have to @@ -52,37 +67,35 @@ function repos.is_installed(name, version) end function repos.recurse_rock_manifest_entry(entry, action) - assert(entry == nil or type(entry) == "table") assert(type(action) == "function") if entry == nil then return true end - local function do_recurse_rock_manifest_tree(tree, parent_path, parent_mod) + local function do_recurse_rock_manifest_entry(tree, parent_path) + for file, sub in pairs(tree) do local sub_path = (parent_path and (parent_path .. "/") or "") .. file - local sub_mod = (parent_mod and (parent_mod .. ".") or "") .. file local ok, err if type(sub) == "table" then - ok, err = do_recurse_rock_manifest_tree(sub, sub_path, sub_mod) + ok, err = do_recurse_rock_manifest_entry(sub, sub_path) else - local mod_no_ext = sub_mod:gsub("%.[^.]*$", "") - ok, err = action(sub_path, mod_no_ext) + ok, err = action(sub_path) end if err then return nil, err end end return true end - return do_recurse_rock_manifest_tree(entry) + return do_recurse_rock_manifest_entry(entry) end local function store_package_data(result, rock_manifest, deploy_type) if rock_manifest[deploy_type] then repos.recurse_rock_manifest_entry(rock_manifest[deploy_type], function(file_path) - local _, item_name = manif.get_provided_item(deploy_type, file_path) + local _, item_name = get_provided_item(deploy_type, file_path) result[item_name] = file_path return true end) @@ -206,29 +219,12 @@ local function find_suffixed(file, suffix) return nil, table.concat(filenames, ", ") .. " not found" end -local function move_suffixed(from_file, to_file, suffix) - local suffixed_from_file, err = find_suffixed(from_file, suffix) - if not suffixed_from_file then - return nil, "Could not move " .. from_file .. " to " .. to_file .. ": " .. err +local function check_suffix(filename, suffix) + local suffixed_filename, err = find_suffixed(filename, suffix) + if not suffixed_filename then + return "" end - - suffix = suffixed_from_file:sub(#from_file + 1) - local suffixed_to_file = to_file .. suffix - return fs.move(suffixed_from_file, suffixed_to_file) -end - -local function delete_suffixed(file, suffix) - local suffixed_file, err = find_suffixed(file, suffix) - if not suffixed_file then - return nil, "Could not remove " .. file .. ": " .. err, "not found" - end - - fs.delete(suffixed_file) - if fs.exists(suffixed_file) then - return nil, "Failed deleting " .. suffixed_file .. ": file still exists", "fail" - end - - return true + return suffixed_filename:sub(#filename + 1) end -- Files can be deployed using versioned and non-versioned names. @@ -238,38 +234,92 @@ end -- is deployed using non-versioned name and others use versioned names. local function get_deploy_paths(name, version, deploy_type, file_path, repo) + assert(type(name) == "string") + assert(type(version) == "string") + assert(type(deploy_type) == "string") + assert(type(file_path) == "string") + repo = repo or cfg.root_dir local deploy_dir = path["deploy_" .. deploy_type .. "_dir"](repo) local non_versioned = dir.path(deploy_dir, file_path) local versioned = path.versioned_name(non_versioned, deploy_dir, name, version) - return non_versioned, versioned + return { nv = non_versioned, v = versioned } end -local function prepare_target(name, version, deploy_type, file_path, suffix) - local non_versioned, versioned = get_deploy_paths(name, version, deploy_type, file_path) - local item_type, item_name = manif.get_provided_item(deploy_type, file_path) +local function check_spot_if_available(name, version, deploy_type, file_path) + local item_type, item_name = get_provided_item(deploy_type, file_path) local cur_name, cur_version = manif.get_current_provider(item_type, item_name) + if (not cur_name) + or (name < cur_name) + or (name == cur_name and (version == cur_version + or vers.compare_versions(version, cur_version))) then + return "nv", cur_name, cur_version, item_name + else + -- Existing version has priority, deploy new version using versioned name. + return "v", cur_name, cur_version, item_name + end +end + +local function backup_existing(should_backup, target) + if not should_backup then + return + end + if fs.exists(target) then + local backup = target + repeat + backup = backup.."~" + until not fs.exists(backup) -- Slight race condition here, but shouldn't be a problem. + + util.warning(target.." is not tracked by this installation of LuaRocks. Moving it to "..backup) + local move_ok, move_err = fs.move(target, backup) + if not move_ok then + return nil, move_err + end + end +end + +local function op_install(op) + local ok, err = fs.make_dir(dir.dir_name(op.dst)) + if not ok then + return nil, err + end - if not cur_name then - return non_versioned - elseif name < cur_name or (name == cur_name and vers.compare_versions(version, cur_version)) then - -- New version has priority. Move currently provided version back using versioned name. - local cur_deploy_type, cur_file_path = manif.get_providing_file(cur_name, cur_version, item_type, item_name) - local cur_non_versioned, cur_versioned = get_deploy_paths(cur_name, cur_version, cur_deploy_type, cur_file_path) + ok, err = op.fn(op.src, op.dst, op.backup) + if not ok then + return nil, err + end - local dir_ok, dir_err = fs.make_dir(dir.dir_name(cur_versioned)) - if not dir_ok then return nil, dir_err end + fs.remove_dir_tree_if_empty(dir.dir_name(op.src)) +end - local move_ok, move_err = move_suffixed(cur_non_versioned, cur_versioned, suffix) - if not move_ok then return nil, move_err end +local function op_rename(op) + if op.suffix then + local suffix = check_suffix(op.src, op.suffix) + op.src = op.src .. suffix + op.dst = op.dst .. suffix + end - return non_versioned + if fs.exists(op.src) then + fs.make_dir(dir.dir_name(op.dst)) + local ok, err = fs.move(op.src, op.dst) + fs.remove_dir_tree_if_empty(dir.dir_name(op.src)) + return ok, err else - -- Current version has priority, deploy new version using versioned name. - return versioned + return true end end +local function op_delete(op) + if op.suffix then + local suffix = check_suffix(op.name, op.suffix) + op.name = op.name .. suffix + end + + local ok, err = fs.delete(op.name) + fs.remove_dir_tree_if_empty(dir.dir_name(op.name)) + return ok, err +end + --- Deploy a package from the rocks subdirectory. -- @param name string: name of package -- @param version string: exact package version in string format @@ -284,63 +334,89 @@ function repos.deploy_files(name, version, wrap_bin_scripts, deps_mode) local rock_manifest, load_err = manif.load_rock_manifest(name, version) if not rock_manifest then return nil, load_err end + + local repo = cfg.root_dir + local renames = {} + local installs = {} - local function deploy_file_tree(deploy_type, source_dir, move_fn, suffix) - return repos.recurse_rock_manifest_entry(rock_manifest[deploy_type], function(file_path) - local source = dir.path(source_dir, file_path) + local function install_binary(source, target, should_backup) + if wrap_bin_scripts and fs.is_lua(source) then + backup_existing(should_backup, target .. (cfg.wrapper_suffix or "")) + return fs.wrap_script(source, target, deps_mode, name, version) + else + backup_existing(should_backup, target) + return fs.copy_binary(source, target) + end + end - local target, prepare_err = prepare_target(name, version, deploy_type, file_path, suffix) - if not target then return nil, prepare_err end + local function move_lua(source, target, should_backup) + backup_existing(should_backup, target) + return fs.move(source, target, "read") + end - local dir_ok, dir_err = fs.make_dir(dir.dir_name(target)) - if not dir_ok then return nil, dir_err end + local function move_lib(source, target, should_backup) + backup_existing(should_backup, target) + return fs.move(source, target, "exec") + end - local suffixed_target, mover = move_fn(source, target) - if fs.exists(suffixed_target) then - local backup = suffixed_target - repeat - backup = backup.."~" - until not fs.exists(backup) -- Slight race condition here, but shouldn't be a problem. + if rock_manifest.bin then + local source_dir = path.bin_dir(name, version) + repos.recurse_rock_manifest_entry(rock_manifest.bin, function(file_path) + local source = dir.path(source_dir, file_path) + local paths = get_deploy_paths(name, version, "bin", file_path, repo) + local mode, cur_name, cur_version = check_spot_if_available(name, version, "bin", file_path) - util.warning(suffixed_target.." is not tracked by this installation of LuaRocks. Moving it to "..backup) - local move_ok, move_err = fs.move(suffixed_target, backup) - if not move_ok then return nil, move_err end + if mode == "nv" and cur_name then + local cur_paths = get_deploy_paths(cur_name, cur_version, "bin", file_path, repo) + table.insert(renames, { src = cur_paths.nv, dst = cur_paths.v, suffix = cfg.wrapper_suffix }) end - - local move_ok, move_err = mover() - if not move_ok then return nil, move_err end - - fs.remove_dir_tree_if_empty(dir.dir_name(source)) - return true + local backup = name ~= cur_name or version ~= cur_version + table.insert(installs, { fn = install_binary, src = source, dst = mode == "nv" and paths.nv or paths.v, backup = backup }) end) end - local function install_binary(source, target) - if wrap_bin_scripts and fs.is_lua(source) then - return target .. (cfg.wrapper_suffix or ""), function() - return fs.wrap_script(source, target, deps_mode, name, version) - end - else - return target, function() - return fs.copy_binary(source, target) + if rock_manifest.lua then + local source_dir = path.lua_dir(name, version) + repos.recurse_rock_manifest_entry(rock_manifest.lua, function(file_path) + local source = dir.path(source_dir, file_path) + local paths = get_deploy_paths(name, version, "lua", file_path, repo) + local mode, cur_name, cur_version = check_spot_if_available(name, version, "lua", file_path) + + if mode == "nv" and cur_name then + local cur_paths = get_deploy_paths(cur_name, cur_version, "lua", file_path, repo) + table.insert(renames, { src = cur_paths.nv, dst = cur_paths.v }) + cur_paths = get_deploy_paths(cur_name, cur_version, "lib", file_path:gsub("%.lua$", "." .. cfg.lib_extension), repo) + table.insert(renames, { src = cur_paths.nv, dst = cur_paths.v }) end - end + local backup = name ~= cur_name or version ~= cur_version + table.insert(installs, { fn = move_lua, src = source, dst = mode == "nv" and paths.nv or paths.v, backup = backup }) + end) end - local function make_mover(perms) - return function(source, target) - return target, function() return fs.move(source, target, perms) end - end + if rock_manifest.lib then + local source_dir = path.lib_dir(name, version) + repos.recurse_rock_manifest_entry(rock_manifest.lib, function(file_path) + local source = dir.path(source_dir, file_path) + local paths = get_deploy_paths(name, version, "lib", file_path, repo) + local mode, cur_name, cur_version = check_spot_if_available(name, version, "lib", file_path) + + if mode == "nv" and cur_name then + local cur_paths = get_deploy_paths(cur_name, cur_version, "lua", file_path:gsub("%.[^.]+$", ".lua"), repo) + table.insert(renames, { src = cur_paths.nv, dst = cur_paths.v }) + cur_paths = get_deploy_paths(cur_name, cur_version, "lib", file_path, repo) + table.insert(renames, { src = cur_paths.nv, dst = cur_paths.v }) + end + local backup = name ~= cur_name or version ~= cur_version + table.insert(installs, { fn = move_lib, src = source, dst = mode == "nv" and paths.nv or paths.v, backup = backup }) + end) end - local ok, err = deploy_file_tree("bin", path.bin_dir(name, version), install_binary, cfg.wrapper_suffix) - if not ok then return nil, err end - - ok, err = deploy_file_tree("lua", path.lua_dir(name, version), make_mover("read")) - if not ok then return nil, err end - - ok, err = deploy_file_tree("lib", path.lib_dir(name, version), make_mover("exec")) - if not ok then return nil, err end + for _, op in ipairs(renames) do + op_rename(op) + end + for _, op in ipairs(installs) do + op_install(op) + end local writer = require("luarocks.manif.writer") return writer.add_to_manifest(name, version, nil, deps_mode) @@ -364,60 +440,76 @@ function repos.delete_version(name, version, deps_mode, quick) local rock_manifest, load_err = manif.load_rock_manifest(name, version) if not rock_manifest then return nil, load_err end - local function delete_deployed_file_tree(deploy_type, suffix) - return repos.recurse_rock_manifest_entry(rock_manifest[deploy_type], function(file_path) - local non_versioned, versioned = get_deploy_paths(name, version, deploy_type, file_path) - - -- Figure out if the file is deployed using versioned or non-versioned name. - local target - local item_type, item_name = manif.get_provided_item(deploy_type, file_path) - local cur_name, cur_version = manif.get_current_provider(item_type, item_name) + local repo = cfg.root_dir + local renames = {} + local deletes = {} - if cur_name == name and cur_version == version then - -- This package has highest priority, should be in non-versioned location. - target = non_versioned + if rock_manifest.bin then + repos.recurse_rock_manifest_entry(rock_manifest.bin, function(file_path) + local paths = get_deploy_paths(name, version, "bin", file_path, repo) + local mode, cur_name, cur_version, item_name = check_spot_if_available(name, version, "bin", file_path) + if mode == "v" then + table.insert(deletes, { name = paths.v, suffix = cfg.wrapper_suffix }) else - target = versioned - end + table.insert(deletes, { name = paths.nv, suffix = cfg.wrapper_suffix }) - local ok, err, err_type = delete_suffixed(target, suffix) - if not ok then - if err_type == "not found" then - util.warning(err) - else - return nil, err + local next_name, next_version = manif.get_next_provider("command", item_name) + if next_name then + local next_paths = get_deploy_paths(next_name, next_version, "lua", file_path, repo) + table.insert(renames, { src = next_paths.v, dst = next_paths.nv, suffix = cfg.wrapper_suffix }) end end + end) + end - if not quick and target == non_versioned then - -- If another package provides this file, move its version - -- into non-versioned location instead. - local next_name, next_version = manif.get_next_provider(item_type, item_name) + if rock_manifest.lua then + repos.recurse_rock_manifest_entry(rock_manifest.lua, function(file_path) + local paths = get_deploy_paths(name, version, "lua", file_path, repo) + local mode, cur_name, cur_version, item_name = check_spot_if_available(name, version, "lua", file_path) + if mode == "v" then + table.insert(deletes, { name = paths.v }) + else + table.insert(deletes, { name = paths.nv }) + local next_name, next_version = manif.get_next_provider("module", item_name) if next_name then - local next_deploy_type, next_file_path = manif.get_providing_file(next_name, next_version, item_type, item_name) - local next_non_versioned, next_versioned = get_deploy_paths(next_name, next_version, next_deploy_type, next_file_path) - - local move_ok, move_err = move_suffixed(next_versioned, next_non_versioned, suffix) - if not move_ok then return nil, move_err end - - fs.remove_dir_tree_if_empty(dir.dir_name(next_versioned)) + local next_lua_paths = get_deploy_paths(next_name, next_version, "lua", file_path, repo) + table.insert(renames, { src = next_lua_paths.v, dst = next_lua_paths.nv }) + local next_lib_paths = get_deploy_paths(next_name, next_version, "lib", file_path:gsub("%.[^.]+$", ".lua"), repo) + table.insert(renames, { src = next_lib_paths.v, dst = next_lib_paths.nv }) end end - - fs.remove_dir_tree_if_empty(dir.dir_name(target)) - return true end) end - local ok, err = delete_deployed_file_tree("bin", cfg.wrapper_suffix) - if not ok then return nil, err end + if rock_manifest.lib then + repos.recurse_rock_manifest_entry(rock_manifest.lib, function(file_path) + local paths = get_deploy_paths(name, version, "lib", file_path, repo) + local mode, cur_name, cur_version, item_name = check_spot_if_available(name, version, "lib", file_path) + if mode == "v" then + table.insert(deletes, { name = paths.v }) + else + table.insert(deletes, { name = paths.nv }) - ok, err = delete_deployed_file_tree("lua") - if not ok then return nil, err end + local next_name, next_version = manif.get_next_provider("module", item_name) + if next_name then + local next_lua_paths = get_deploy_paths(next_name, next_version, "lua", file_path:gsub("%.[^.]+$", ".lua"), repo) + table.insert(renames, { src = next_lua_paths.v, dst = next_lua_paths.nv }) + local next_lib_paths = get_deploy_paths(next_name, next_version, "lib", file_path, repo) + table.insert(renames, { src = next_lib_paths.v, dst = next_lib_paths.nv }) + end + end + end) + end - ok, err = delete_deployed_file_tree("lib") - if not ok then return nil, err end + for _, op in ipairs(deletes) do + op_delete(op) + end + if not quick then + for _, op in ipairs(renames) do + op_rename(op) + end + end fs.delete(path.install_dir(name, version)) if not get_installed_versions(name) then @@ -432,48 +524,4 @@ function repos.delete_version(name, version, deps_mode, quick) return writer.remove_from_manifest(name, version, nil, deps_mode) end ---- Find full path to a file providing a module or a command --- in a package. --- @param name string: name of package. --- @param version string: exact package version in string format. --- @param item_type string: "module" or "command". --- @param item_name string: module or command name. --- @param root string or nil: A local root dir for a rocks tree. If not given, the default is used. --- @return string: absolute path to the file providing given module --- or command. -function repos.which(name, version, item_type, item_name, repo) - local deploy_type, file_path = manif.get_providing_file(name, version, item_type, item_name, repo) - local non_versioned, versioned = get_deploy_paths(name, version, deploy_type, file_path, repo) - local cur_name, cur_version = manif.get_current_provider(item_type, item_name) - local deploy_path = (name == cur_name and version == cur_version) and non_versioned or versioned - - if deploy_type == "bin" and cfg.wrapper_suffix and cfg.wrapper_suffix ~= "" then - deploy_path = find_suffixed(deploy_path, cfg.wrapper_suffix) or deploy_path - end - - return deploy_path -end - ---- Find full path to a file providing a module or a command --- in a package. --- @param name string: name of package. --- @param version string: exact package version in string format. --- @param item_type string: "module" or "command". --- @param item_name string: module or command name. --- @param root string or nil: A local root dir for a rocks tree. If not given, the default is used. --- @return string: absolute path to the file providing given module --- or command. -function repos.which(name, version, item_type, item_name, repo) - local deploy_type, file_path = manif.get_providing_file(name, version, item_type, item_name, repo) - local non_versioned, versioned = get_deploy_paths(name, version, deploy_type, file_path, repo) - local cur_name, cur_version = manif.get_current_provider(item_type, item_name) - local deploy_path = (name == cur_name and version == cur_version) and non_versioned or versioned - - if deploy_type == "bin" and cfg.wrapper_suffix and cfg.wrapper_suffix ~= "" then - deploy_path = find_suffixed(deploy_path, cfg.wrapper_suffix) or deploy_path - end - - return deploy_path -end - return repos -- cgit v1.2.3-55-g6feb