From acf8bcb82ae697a5bac94bfbe93c3ad9d7487795 Mon Sep 17 00:00:00 2001 From: Hisham Muhammad Date: Fri, 8 Nov 2019 18:19:17 -0300 Subject: dependency pinning: luarocks.lock file and --pin flag This adds support for pinning dependencies in projects and rocks: * Adds a new flag called `--pin` which creates a `luarocks.lock` when building a rock with `luarocks build` or `luarocks make`. This lock file contains the exact version numbers of every direct or indirect dependency of the rock (in other words, it is the transitive closure of the dependencies.) For `make`, the `luarocks.lock` file is created in the current directory. The lock file is also installed as part of the rock in its metadata directory alongside its rockspec. When using `--pin`, if a lock file already exists, it is ignored and overwritten. * When building a rock with `luarocks make`, if there is a `luarocks.lock` file in the current directory, the exact versions specified there will be used for resolving dependencies. * When building a rock with `luarocks build`, if there is a `luarocks.lock` file in root of its sources, the exact versions specified there will be used for resolving dependencies. * When installing a `.rock` file with `luarocks install`, if the rock contains a `luarocks.lock` file (i.e., if its dependencies were pinned with `--pin` when the rock was built), the exact versions specified there will be used for resolving dependencies. --- spec/build_spec.lua | 41 +++- spec/doc_spec.lua | 2 +- spec/fixtures/a_repo/a_rock-2.0-1.src.rock | Bin 0 -> 541 bytes spec/fixtures/a_repo/manifest | 5 + spec/fixtures/a_repo/manifest-5.1 | 5 + spec/fixtures/a_repo/manifest-5.1.zip | Bin 350 -> 358 bytes spec/fixtures/a_repo/manifest-5.2 | 5 + spec/fixtures/a_repo/manifest-5.2.zip | Bin 350 -> 358 bytes spec/fixtures/a_repo/manifest-5.3 | 5 + spec/fixtures/a_repo/manifest-5.3.zip | Bin 350 -> 358 bytes spec/install_spec.lua | 44 ++++ spec/make_spec.lua | 119 +++++++++++ src/luarocks/build.lua | 25 ++- src/luarocks/cmd/build.lua | 11 +- src/luarocks/cmd/install.lua | 4 +- src/luarocks/cmd/make.lua | 8 + src/luarocks/core/manif.lua | 2 +- src/luarocks/core/persist.lua | 4 +- src/luarocks/deplocks.lua | 106 ++++++++++ src/luarocks/deps.lua | 316 ++++++++++++++++++++--------- src/luarocks/persist.lua | 3 +- src/luarocks/test/busted.lua | 2 +- 22 files changed, 590 insertions(+), 117 deletions(-) create mode 100644 spec/fixtures/a_repo/a_rock-2.0-1.src.rock create mode 100644 src/luarocks/deplocks.lua diff --git a/spec/build_spec.lua b/spec/build_spec.lua index eeca75cf..d64ace4d 100644 --- a/spec/build_spec.lua +++ b/spec/build_spec.lua @@ -116,11 +116,11 @@ describe("LuaRocks build #integration", function() end) it("with --only-sources", function() - assert.is_true(run.luarocks_bool("download --server=" .. testing_paths.fixtures_dir .. "/a_repo --rockspec a_rock")) + assert.is_true(run.luarocks_bool("download --server=" .. testing_paths.fixtures_dir .. "/a_repo --rockspec a_rock 1.0")) assert.is_false(run.luarocks_bool("build --only-sources=\"http://example.com\" a_rock-1.0-1.rockspec")) assert.is.falsy(lfs.attributes(testing_paths.testing_sys_rocks .. "/a_rock/1.0-1/a_rock-1.0-1.rockspec")) - assert.is_true(run.luarocks_bool("download --server=" .. testing_paths.fixtures_dir .. "/a_repo --source a_rock")) + assert.is_true(run.luarocks_bool("download --server=" .. testing_paths.fixtures_dir .. "/a_repo --source a_rock 1.0")) assert.is_true(run.luarocks_bool("build --only-sources=\"http://example.com\" a_rock-1.0-1.src.rock")) assert.is.truthy(lfs.attributes(testing_paths.testing_sys_rocks .. "/a_rock/1.0-1/a_rock-1.0-1.rockspec")) @@ -199,7 +199,42 @@ describe("LuaRocks build #integration", function() assert.is.truthy(lfs.attributes(testing_paths.testing_sys_rocks .. "/test/1.0-1/test-1.0-1.rockspec")) end) end) - + + it("supports --pin #pinning", function() + test_env.run_in_tmp(function(tmpdir) + write_file("test-1.0-1.rockspec", [[ + package = "test" + version = "1.0-1" + source = { + url = "file://]] .. tmpdir:gsub("\\", "/") .. [[/test.lua" + } + dependencies = { + "a_rock >= 0.8" + } + build = { + type = "builtin", + modules = { + test = "test.lua" + } + } + ]], finally) + write_file("test.lua", "return {}", finally) + + assert.is_true(run.luarocks_bool("build --server=" .. testing_paths.fixtures_dir .. "/a_repo test-1.0-1.rockspec --pin --tree=lua_modules")) + assert.is.truthy(lfs.attributes("./lua_modules/lib/luarocks/rocks-" .. test_env.lua_version .. "/test/1.0-1/test-1.0-1.rockspec")) + assert.is.truthy(lfs.attributes("./lua_modules/lib/luarocks/rocks-" .. test_env.lua_version .. "/a_rock/2.0-1/a_rock-2.0-1.rockspec")) + local lockfilename = "./lua_modules/lib/luarocks/rocks-" .. test_env.lua_version .. "/test/1.0-1/luarocks.lock" + assert.is.truthy(lfs.attributes(lockfilename)) + local lockdata = loadfile(lockfilename)() + assert.same({ + dependencies = { + ["a_rock"] = "2.0-1", + ["lua"] = test_env.lua_version .. "-1", + } + }, lockdata) + end) + end) + it("lmathx deps partial match", function() assert.is_true(run.luarocks_bool("build lmathx")) diff --git a/spec/doc_spec.lua b/spec/doc_spec.lua index bf5214d4..2c0f4bce 100644 --- a/spec/doc_spec.lua +++ b/spec/doc_spec.lua @@ -70,7 +70,7 @@ describe("LuaRocks doc tests #integration", function() describe("#namespaces", function() it("retrieves docs for a namespaced package from the command-line", function() assert(run.luarocks_bool("build a_user/a_rock --server=" .. testing_paths.fixtures_dir .. "/a_repo" )) - assert(run.luarocks_bool("build a_rock --keep --server=" .. testing_paths.fixtures_dir .. "/a_repo" )) + assert(run.luarocks_bool("build a_rock 1.0 --keep --server=" .. testing_paths.fixtures_dir .. "/a_repo" )) assert.match("a_rock 2.0", run.luarocks("doc a_user/a_rock")) end) end) diff --git a/spec/fixtures/a_repo/a_rock-2.0-1.src.rock b/spec/fixtures/a_repo/a_rock-2.0-1.src.rock new file mode 100644 index 00000000..5824c767 Binary files /dev/null and b/spec/fixtures/a_repo/a_rock-2.0-1.src.rock differ diff --git a/spec/fixtures/a_repo/manifest b/spec/fixtures/a_repo/manifest index 3b01b427..5ab87d25 100644 --- a/spec/fixtures/a_repo/manifest +++ b/spec/fixtures/a_repo/manifest @@ -19,6 +19,11 @@ repository = { { arch = "rockspec" } + }, + ["2.0-1"] = { + { + arch = "src" + } } }, busted_project = { diff --git a/spec/fixtures/a_repo/manifest-5.1 b/spec/fixtures/a_repo/manifest-5.1 index 3b01b427..5ab87d25 100644 --- a/spec/fixtures/a_repo/manifest-5.1 +++ b/spec/fixtures/a_repo/manifest-5.1 @@ -19,6 +19,11 @@ repository = { { arch = "rockspec" } + }, + ["2.0-1"] = { + { + arch = "src" + } } }, busted_project = { diff --git a/spec/fixtures/a_repo/manifest-5.1.zip b/spec/fixtures/a_repo/manifest-5.1.zip index 7b53aeb0..65e316df 100644 Binary files a/spec/fixtures/a_repo/manifest-5.1.zip and b/spec/fixtures/a_repo/manifest-5.1.zip differ diff --git a/spec/fixtures/a_repo/manifest-5.2 b/spec/fixtures/a_repo/manifest-5.2 index 3b01b427..5ab87d25 100644 --- a/spec/fixtures/a_repo/manifest-5.2 +++ b/spec/fixtures/a_repo/manifest-5.2 @@ -19,6 +19,11 @@ repository = { { arch = "rockspec" } + }, + ["2.0-1"] = { + { + arch = "src" + } } }, busted_project = { diff --git a/spec/fixtures/a_repo/manifest-5.2.zip b/spec/fixtures/a_repo/manifest-5.2.zip index d2eddc40..b4334a65 100644 Binary files a/spec/fixtures/a_repo/manifest-5.2.zip and b/spec/fixtures/a_repo/manifest-5.2.zip differ diff --git a/spec/fixtures/a_repo/manifest-5.3 b/spec/fixtures/a_repo/manifest-5.3 index 3b01b427..5ab87d25 100644 --- a/spec/fixtures/a_repo/manifest-5.3 +++ b/spec/fixtures/a_repo/manifest-5.3 @@ -19,6 +19,11 @@ repository = { { arch = "rockspec" } + }, + ["2.0-1"] = { + { + arch = "src" + } } }, busted_project = { diff --git a/spec/fixtures/a_repo/manifest-5.3.zip b/spec/fixtures/a_repo/manifest-5.3.zip index 686fe232..bab15712 100644 Binary files a/spec/fixtures/a_repo/manifest-5.3.zip and b/spec/fixtures/a_repo/manifest-5.3.zip differ diff --git a/spec/install_spec.lua b/spec/install_spec.lua index bf50ced7..76dc2201 100644 --- a/spec/install_spec.lua +++ b/spec/install_spec.lua @@ -267,6 +267,50 @@ describe("luarocks install #integration", function() end) end) + it("respects luarocks.lock in package #pinning", function() + test_env.run_in_tmp(function(tmpdir) + write_file("test-1.0-1.rockspec", [[ + package = "test" + version = "1.0-1" + source = { + url = "file://]] .. tmpdir:gsub("\\", "/") .. [[/test.lua" + } + dependencies = { + "a_rock >= 0.8" + } + build = { + type = "builtin", + modules = { + test = "test.lua" + } + } + ]], finally) + write_file("test.lua", "return {}", finally) + write_file("luarocks.lock", [[ + return { + dependencies = { + ["a_rock"] = "1.0-1", + } + } + ]], finally) + + assert.is_true(run.luarocks_bool("make --pack-binary-rock --server=" .. testing_paths.fixtures_dir .. "/a_repo test-1.0-1.rockspec")) + assert.is_true(os.remove("luarocks.lock")) + + assert.is.truthy(lfs.attributes("./test-1.0-1.all.rock")) + + assert.is.falsy(lfs.attributes("./lua_modules/lib/luarocks/rocks-" .. test_env.lua_version .. "/test/1.0-1/test-1.0-1.rockspec")) + assert.is.falsy(lfs.attributes("./lua_modules/lib/luarocks/rocks-" .. test_env.lua_version .. "/a_rock/1.0-1/a_rock-1.0-1.rockspec")) + + print(run.luarocks("install ./test-1.0-1.all.rock --tree=lua_modules --server=" .. testing_paths.fixtures_dir .. "/a_repo")) + + assert.is.truthy(lfs.attributes("./lua_modules/lib/luarocks/rocks-" .. test_env.lua_version .. "/test/1.0-1/test-1.0-1.rockspec")) + assert.is.truthy(lfs.attributes("./lua_modules/lib/luarocks/rocks-" .. test_env.lua_version .. "/test/1.0-1/luarocks.lock")) + assert.is.truthy(lfs.attributes("./lua_modules/lib/luarocks/rocks-" .. test_env.lua_version .. "/a_rock/1.0-1/a_rock-1.0-1.rockspec")) + assert.is.falsy(lfs.attributes("./lua_modules/lib/luarocks/rocks-" .. test_env.lua_version .. "/a_rock/2.0-1")) + end) + end) + describe("#unix install runs build from #git", function() local git diff --git a/spec/make_spec.lua b/spec/make_spec.lua index 5ec99fa7..8baa3561 100644 --- a/spec/make_spec.lua +++ b/spec/make_spec.lua @@ -3,6 +3,7 @@ local lfs = require("lfs") local run = test_env.run local testing_paths = test_env.testing_paths local env_variables = test_env.env_variables +local write_file = test_env.write_file test_env.unload_luarocks() @@ -125,6 +126,124 @@ describe("LuaRocks make tests #integration", function() end) end) + it("supports --pin #pinning", function() + test_env.run_in_tmp(function(tmpdir) + write_file("test-1.0-1.rockspec", [[ + package = "test" + version = "1.0-1" + source = { + url = "file://]] .. tmpdir:gsub("\\", "/") .. [[/test.lua" + } + dependencies = { + "a_rock 1.0" + } + build = { + type = "builtin", + modules = { + test = "test.lua" + } + } + ]], finally) + write_file("test.lua", "return {}", finally) + + assert.is_true(run.luarocks_bool("make --server=" .. testing_paths.fixtures_dir .. "/a_repo --pin --tree=lua_modules")) + assert.is.truthy(lfs.attributes("./lua_modules/lib/luarocks/rocks-" .. test_env.lua_version .. "/test/1.0-1/test-1.0-1.rockspec")) + assert.is.truthy(lfs.attributes("./lua_modules/lib/luarocks/rocks-" .. test_env.lua_version .. "/a_rock/1.0-1/a_rock-1.0-1.rockspec")) + local lockfilename = "./lua_modules/lib/luarocks/rocks-" .. test_env.lua_version .. "/test/1.0-1/luarocks.lock" + assert.is.truthy(lfs.attributes(lockfilename)) + local lockdata = loadfile(lockfilename)() + assert.same({ + dependencies = { + ["a_rock"] = "1.0-1", + ["lua"] = test_env.lua_version .. "-1", + } + }, lockdata) + end) + end) + + it("respects luarocks.lock when present #pinning", function() + test_env.run_in_tmp(function(tmpdir) + write_file("test-2.0-1.rockspec", [[ + package = "test" + version = "2.0-1" + source = { + url = "file://]] .. tmpdir:gsub("\\", "/") .. [[/test.lua" + } + dependencies = { + "a_rock >= 0.8" + } + build = { + type = "builtin", + modules = { + test = "test.lua" + } + } + ]], finally) + write_file("test.lua", "return {}", finally) + write_file("luarocks.lock", [[ + return { + dependencies = { + ["a_rock"] = "1.0-1", + } + } + ]], finally) + + print(run.luarocks("make --server=" .. testing_paths.fixtures_dir .. "/a_repo --tree=lua_modules")) + assert.is.truthy(lfs.attributes("./lua_modules/lib/luarocks/rocks-" .. test_env.lua_version .. "/test/2.0-1/test-2.0-1.rockspec")) + assert.is.truthy(lfs.attributes("./lua_modules/lib/luarocks/rocks-" .. test_env.lua_version .. "/a_rock/1.0-1/a_rock-1.0-1.rockspec")) + local lockfilename = "./lua_modules/lib/luarocks/rocks-" .. test_env.lua_version .. "/test/2.0-1/luarocks.lock" + assert.is.truthy(lfs.attributes(lockfilename)) + local lockdata = loadfile(lockfilename)() + assert.same({ + dependencies = { + ["a_rock"] = "1.0-1", + } + }, lockdata) + end) + end) + + it("overrides luarocks.lock with --pin #pinning", function() + test_env.run_in_tmp(function(tmpdir) + write_file("test-2.0-1.rockspec", [[ + package = "test" + version = "2.0-1" + source = { + url = "file://]] .. tmpdir:gsub("\\", "/") .. [[/test.lua" + } + dependencies = { + "a_rock >= 0.8" + } + build = { + type = "builtin", + modules = { + test = "test.lua" + } + } + ]], finally) + write_file("test.lua", "return {}", finally) + write_file("luarocks.lock", [[ + return { + dependencies = { + ["a_rock"] = "1.0-1", + } + } + ]], finally) + + print(run.luarocks("make --server=" .. testing_paths.fixtures_dir .. "/a_repo --tree=lua_modules --pin")) + assert.is.truthy(lfs.attributes("./lua_modules/lib/luarocks/rocks-" .. test_env.lua_version .. "/test/2.0-1/test-2.0-1.rockspec")) + assert.is.truthy(lfs.attributes("./lua_modules/lib/luarocks/rocks-" .. test_env.lua_version .. "/a_rock/2.0-1/a_rock-2.0-1.rockspec")) + local lockfilename = "./lua_modules/lib/luarocks/rocks-" .. test_env.lua_version .. "/test/2.0-1/luarocks.lock" + assert.is.truthy(lfs.attributes(lockfilename)) + local lockdata = loadfile(lockfilename)() + assert.same({ + dependencies = { + ["a_rock"] = "2.0-1", + ["lua"] = test_env.lua_version .. "-1", + } + }, lockdata) + end) + end) + describe("#ddt LuaRocks make upgrading rockspecs with double deploy types", function() local deploy_lib_dir = testing_paths.testing_sys_tree .. "/lib/lua/"..env_variables.LUA_VERSION local deploy_lua_dir = testing_paths.testing_sys_tree .. "/share/lua/"..env_variables.LUA_VERSION diff --git a/src/luarocks/build.lua b/src/luarocks/build.lua index 9aa70345..ca0237b7 100644 --- a/src/luarocks/build.lua +++ b/src/luarocks/build.lua @@ -11,6 +11,7 @@ local deps = require("luarocks.deps") local cfg = require("luarocks.core.cfg") local repos = require("luarocks.repos") local writer = require("luarocks.manif.writer") +local deplocks = require("luarocks.deplocks") build.opts = util.opts_table("build.opts", { need_to_fetch = "boolean", @@ -21,6 +22,7 @@ build.opts = util.opts_table("build.opts", { branch = "string?", verify = "boolean", check_lua_versions = "boolean", + pin = "boolean", }) do @@ -125,6 +127,7 @@ local function process_dependencies(rockspec, opts) if opts.deps_mode == "none" then return true end + if not opts.build_only_deps then if next(rockspec.build_dependencies) then local ok, err, errcode = deps.fulfill_dependencies(rockspec, "build_dependencies", opts.deps_mode, opts.verify) @@ -326,6 +329,11 @@ local function write_rock_dir_files(rockspec, opts) local name, version = rockspec.name, rockspec.version fs.copy(rockspec.local_abs_filename, path.rockspec_file(name, version), "read") + + local deplock_file = deplocks.get_abs_filename(rockspec.name) + if deplock_file then + fs.copy(deplock_file, dir.path(path.install_dir(name, version), "luarocks.lock"), "read") + end local ok, err = writer.make_rock_manifest(name, version) if not ok then return nil, err end @@ -362,7 +370,14 @@ function build.build_rockspec(rockspec, opts) end end - local ok, err = process_dependencies(rockspec, opts) + local ok, err = fetch_and_change_to_source_dir(rockspec, opts) + if not ok then return nil, err end + + if opts.pin then + deplocks.init(rockspec.name, ".") + end + + ok, err = process_dependencies(rockspec, opts) if not ok then return nil, err end local name, version = rockspec.name, rockspec.version @@ -373,10 +388,6 @@ function build.build_rockspec(rockspec, opts) if repos.is_installed(name, version) then repos.delete_version(name, version, opts.deps_mode) end - - ok, err = fetch_and_change_to_source_dir(rockspec, opts) - if not ok then return nil, err end - local dirs, err = prepare_install_dirs(name, version) if not dirs then return nil, err end @@ -406,6 +417,10 @@ function build.build_rockspec(rockspec, opts) fs.pop_dir() end + if opts.pin then + deplocks.write_file() + end + ok, err = write_rock_dir_files(rockspec, opts) if not ok then return nil, err end diff --git a/src/luarocks/cmd/build.lua b/src/luarocks/cmd/build.lua index 6b30666b..3bc1554e 100644 --- a/src/luarocks/cmd/build.lua +++ b/src/luarocks/cmd/build.lua @@ -19,6 +19,8 @@ local cmd = require("luarocks.cmd") function cmd_build.add_to_parser(parser) local cmd = parser:command("build", "Build and install a rock, compiling its C parts if any.\n".. + "If the sources contain a luarocks.lock file, uses it as an authoritative source for ".. + "exact version of dependencies.\n".. "If no arguments are given, behaves as luarocks make.", util.see_also()) :summary("Build/compile a rock.") @@ -33,6 +35,10 @@ function cmd_build.add_to_parser(parser) "rockspec. Allows to specify a different branch to fetch. Particularly ".. 'for "dev" rocks.') :argname("") + parser:flag("--pin", "Create a luarocks.lock file listing the exact ".. + "versions of each dependency found for this rock (recursively), ".. + "and store it in the rock's directory. ".. + "Ignores any existing luarocks.lock file in the rock's sources.") make.cmd_options(cmd) end @@ -124,6 +130,7 @@ function cmd_build.command(args) branch = args.branch, verify = not not args.verify, check_lua_versions = not not args.check_lua_versions, + pin = not not args.pin, }) if args.sign and not args.pack_binary_rock then @@ -167,7 +174,9 @@ function cmd_build.command(args) end end - writer.check_dependencies(nil, deps.get_deps_mode(args)) + if opts.deps_mode ~= "none" then + writer.check_dependencies(nil, deps.get_deps_mode(args)) + end return name, version end diff --git a/src/luarocks/cmd/install.lua b/src/luarocks/cmd/install.lua index cd1df294..ad2a5ea7 100644 --- a/src/luarocks/cmd/install.lua +++ b/src/luarocks/cmd/install.lua @@ -114,7 +114,7 @@ function install.install_binary_rock(rock_file, opts) end if deps_mode ~= "none" then - ok, err, errcode = deps.fulfill_dependencies(rockspec, "dependencies", deps_mode, opts.verify) + ok, err, errcode = deps.fulfill_dependencies(rockspec, "dependencies", deps_mode, opts.verify, install_dir) if err then return nil, err, errcode end end @@ -163,7 +163,7 @@ function install.install_binary_rock_deps(rock_file, opts) return nil, "Failed loading rockspec for installed package: "..err, errcode end - ok, err, errcode = deps.fulfill_dependencies(rockspec, "dependencies", opts.deps_mode, opts.verify) + ok, err, errcode = deps.fulfill_dependencies(rockspec, "dependencies", opts.deps_mode, opts.verify, install_dir) if err then return nil, err, errcode end util.printout() diff --git a/src/luarocks/cmd/make.lua b/src/luarocks/cmd/make.lua index 74de6f26..a2ae4cd0 100644 --- a/src/luarocks/cmd/make.lua +++ b/src/luarocks/cmd/make.lua @@ -13,6 +13,7 @@ local fetch = require("luarocks.fetch") local pack = require("luarocks.pack") local remove = require("luarocks.remove") local deps = require("luarocks.deps") +local deplocks = require("luarocks.deplocks") local writer = require("luarocks.manif.writer") local cmd = require("luarocks.cmd") @@ -37,6 +38,8 @@ function make.cmd_options(parser) "signature file for the generated .rock file.") parser:flag("--check-lua-versions", "If the rock can't be found, check repository ".. "and report if it is available for another Lua version.") + parser:flag("--pin", "Pin the exact dependencies used for the rockspec".. + "being built into a luarocks.lock file in the current directory.") util.deps_mode_option(parser) end @@ -54,6 +57,10 @@ This command is useful as a tool for debugging rockspecs. To install rocks, you'll normally want to use the "install" and "build" commands. See the help on those for details. +If the current directory contains a luarocks.lock file, it is used as the +authoritative source for exact version of dependencies. The --pin flag +overrides and recreates this file scanning dependency based on ranges. + NB: Use `luarocks install` with the `--only-deps` flag if you want to install only dependencies of the rockspec (see `luarocks help install`). ]], util.see_also()) @@ -97,6 +104,7 @@ function make.command(args) branch = args.branch, verify = not not args.verify, check_lua_versions = not not args.check_lua_versions, + pin = not not args.pin }) if args.sign and not args.pack_binary_rock then diff --git a/src/luarocks/core/manif.lua b/src/luarocks/core/manif.lua index 4fd35c6c..1e9da75e 100644 --- a/src/luarocks/core/manif.lua +++ b/src/luarocks/core/manif.lua @@ -102,7 +102,7 @@ function manif.scan_dependencies(name, version, tree_manifests, dest) if entries then for ver, _ in util.sortedpairs(entries, vers.compare_versions) do if (not constraints) or vers.match_constraints(vers.parse_version(ver), constraints) then - manif.scan_dependencies(pkg, version, tree_manifests, dest) + manif.scan_dependencies(pkg, ver, tree_manifests, dest) end end end diff --git a/src/luarocks/core/persist.lua b/src/luarocks/core/persist.lua index 48979184..59089818 100644 --- a/src/luarocks/core/persist.lua +++ b/src/luarocks/core/persist.lua @@ -10,7 +10,7 @@ local require = nil -- @return (true, any) or (nil, string, string): true and the return value -- of the file, or nil, an error message and an error code ("open", "load" -- or "run") in case of errors. -local function run_file(filename, env) +function persist.run_file(filename, env) local fd, err = io.open(filename) if not fd then return nil, err, "open" @@ -67,7 +67,7 @@ function persist.load_into_table(filename, tbl) local save_mt = getmetatable(result) setmetatable(result, globals_mt) - local ok, err, errcode = run_file(filename, result) + local ok, err, errcode = persist.run_file(filename, result) setmetatable(result, save_mt) diff --git a/src/luarocks/deplocks.lua b/src/luarocks/deplocks.lua new file mode 100644 index 00000000..f6449986 --- /dev/null +++ b/src/luarocks/deplocks.lua @@ -0,0 +1,106 @@ +local deplocks = {} + +local fs = require("luarocks.fs") +local dir = require("luarocks.dir") +local util = require("luarocks.util") +local persist = require("luarocks.persist") + +local deptable = {} +local deptable_mode = "start" +local deplock_abs_filename +local deplock_root_rock_name + +function deplocks.init(root_rock_name, dirname) + if deptable_mode ~= "start" then + return + end + deptable_mode = "create" + + local filename = dir.path(dirname, "luarocks.lock") + deplock_abs_filename = fs.absolute_name(filename) + deplock_root_rock_name = root_rock_name + + deptable = {} +end + +function deplocks.get_abs_filename(root_rock_name) + if root_rock_name == deplock_root_rock_name then + return deplock_abs_filename + end +end + +function deplocks.load(root_rock_name, dirname) + if deptable_mode ~= "start" then + return true, nil + end + deptable_mode = "locked" + + local filename = dir.path(dirname, "luarocks.lock") + local ok, result, errcode = persist.run_file(filename, {}) + if errcode == "load" or errcode == "run" then + -- bad config file or depends on env, so error out + return nil, nil, "Could not read existing lockfile " .. filename + end + + if errcode == "open" then + -- could not open, maybe file does not exist + return true, nil + end + + deplock_abs_filename = fs.absolute_name(filename) + deplock_root_rock_name = root_rock_name + + deptable = result + return true, filename +end + +function deplocks.add(depskey, name, version) + if deptable_mode == "locked" then + return + end + + local dk = deptable[depskey] + if not dk then + dk = {} + deptable[depskey] = dk + end + + if not dk[name] then + dk[name] = version + end +end + +function deplocks.get(depskey, name) + local dk = deptable[depskey] + if not dk then + return nil + end + + return deptable[name] +end + +function deplocks.write_file() + if deptable_mode ~= "create" then + return true + end + + return persist.save_as_module(deplock_abs_filename, deptable) +end + +-- a table-like interface to deplocks +function deplocks.proxy(depskey) + return setmetatable({}, { + __index = function(_, k) + return deplocks.get(depskey, k) + end, + __newindex = function(_, k, v) + return deplocks.add(depskey, k, v) + end, + }) +end + +function deplocks.each(depskey) + return util.sortedpairs(deptable[depskey] or {}) +end + +return deplocks diff --git a/src/luarocks/deps.lua b/src/luarocks/deps.lua index 7f695d9c..d54c30de 100644 --- a/src/luarocks/deps.lua +++ b/src/luarocks/deps.lua @@ -11,64 +11,114 @@ local util = require("luarocks.util") local vers = require("luarocks.core.vers") local queries = require("luarocks.queries") local builtin = require("luarocks.build.builtin") +local deplocks = require("luarocks.deplocks") ---- Attempt to match a dependency to an installed rock. --- @param dep table: A dependency parsed in table format. --- @param blacklist table: Versions that can't be accepted. Table where keys --- are program versions and values are 'true'. +--- Generate a function that matches dep queries against the manifest, +-- taking into account rocks_provided, the blacklist and the lockfile. +-- @param deps_mode "one", "none", "all" or "order" +-- @param rocks_provided a one-level table mapping names to versions, +-- listing rocks to consider provided by the VM -- @param rocks_provided table: A table of auto-provided dependencies. -- by this Lua implementation for the given dependency. --- @return string or nil: latest installed version of the rock matching the dependency --- or nil if it could not be matched. -local function match_dep(dep, blacklist, deps_mode, rocks_provided) - assert(type(dep) == "table") +-- @param depskey key to use when matching the lockfile ("dependencies", +-- "build_dependencies", etc.) +-- @param blacklist a two-level table mapping names to versions to boolean, +-- listing rocks to not match +-- @return function(dep): {string}, {string:string}, string, boolean +-- * array of matching versions +-- * map of versions to locations +-- * version matched via lockfile if any +-- * true if rock matched via rocks_provided +local function prepare_get_versions(deps_mode, rocks_provided, depskey, blacklist) + assert(type(deps_mode) == "string") assert(type(rocks_provided) == "table") - - local versions, locations - local provided = rocks_provided[dep.name] - if provided then - -- Provided rocks have higher priority than manifest's rocks. - versions, locations = { provided }, {} - else - versions, locations = manif.get_versions(dep, deps_mode) + assert(type(depskey) == "string") + assert(type(blacklist) == "table" or blacklist == nil) + + return function(dep) + local versions, locations + local provided = rocks_provided[dep.name] + if provided then + -- Provided rocks have higher priority than manifest's rocks. + versions, locations = { provided }, {} + else + if deps_mode == "none" then + deps_mode = "one" + end + versions, locations = manif.get_versions(dep, deps_mode) + end + + if blacklist and blacklist[dep.name] then + local orig_versions = versions + versions = {} + for _, v in ipairs(orig_versions) do + if not blacklist[dep.name][v] then + table.insert(versions, v) + end + end + end + + local lockversion = deplocks.get(depskey, dep.name) + + return versions, locations, lockversion, provided ~= nil end +end +--- Attempt to match a dependency to an installed rock. +-- @param blacklist table: Versions that can't be accepted. Table where keys +-- are program versions and values are 'true'. +-- @param get_versions a getter function obtained via prepare_get_versions +-- @return (string, string, table) or (nil, nil, table): +-- 1. latest installed version of the rock matching the dependency +-- 2. location where the installed version is installed +-- 3. the 'dep' query table +-- 4. true if provided via VM +-- or +-- 1. nil +-- 2. nil +-- 3. either 'dep' or an alternative query to be used +-- 4. false +local function match_dep(dep, get_versions) + assert(type(dep) == "table") + assert(type(get_versions) == "function") + + local versions, locations, lockversion, provided = get_versions(dep) + local latest_version local latest_vstring for _, vstring in ipairs(versions) do - if not blacklist or not blacklist[vstring] then - local version = vers.parse_version(vstring) - if vers.match_constraints(version, dep.constraints) then - if not latest_version or version > latest_version then - latest_version = version - latest_vstring = vstring - end + local version = vers.parse_version(vstring) + if vers.match_constraints(version, dep.constraints) then + if not latest_version or version > latest_version then + latest_version = version + latest_vstring = vstring end end end - return latest_vstring, locations[latest_vstring] + + if lockversion and not locations[lockversion] then + local latest_matching_msg = "" + if latest_vstring and latest_vstring ~= lockversion then + latest_matching_msg = " (latest matching is " .. latest_vstring .. ")" + end + util.printout("Forcing " .. dep.name .. " to pinned version " .. lockversion .. latest_matching_msg) + return nil, nil, queries.new(dep.name, lockversion) + end + + return latest_vstring, locations[latest_vstring], dep, provided end ---- Attempt to match dependencies of a rockspec to installed rocks. --- @param dependencies table: The table of dependencies. --- @param rocks_provided table: The table of auto-provided dependencies. --- @param blacklist table or nil: Program versions to not use as valid matches. --- Table where keys are program names and values are tables where keys --- are program versions and values are 'true'. --- @return table, table, table: A table where keys are dependencies parsed --- in table format and values are tables containing fields 'name' and --- version' representing matches; a table of missing dependencies --- parsed as tables; and a table of "no-upgrade" missing dependencies --- (to be used in plugin modules so that a plugin does not force upgrade of --- its parent application). -function deps.match_deps(dependencies, rocks_provided, blacklist, deps_mode) - assert(type(blacklist) == "table" or not blacklist) +local function match_all_deps(dependencies, get_versions) + assert(type(dependencies) == "table") + assert(type(get_versions) == "function") + local matched, missing, no_upgrade = {}, {}, {} for _, dep in ipairs(dependencies) do - local found = match_dep(dep, blacklist and blacklist[dep.name] or nil, deps_mode, rocks_provided) + local found, _, provided + found, _, dep, provided = match_dep(dep, get_versions) if found then - if not rocks_provided[dep.name] then + if not provided then matched[dep] = {name = dep.name, version = found} end else @@ -82,20 +132,35 @@ function deps.match_deps(dependencies, rocks_provided, blacklist, deps_mode) return matched, missing, no_upgrade end ---- Return a set of values of a table. --- @param tbl table: The input table. --- @return table: The array of keys. -local function values_set(tbl) - local set = {} - for _, v in pairs(tbl) do - set[v] = true - end - return set +--- Attempt to match dependencies of a rockspec to installed rocks. +-- @param dependencies table: The table of dependencies. +-- @param rocks_provided table: The table of auto-provided dependencies. +-- @param blacklist table or nil: Program versions to not use as valid matches. +-- Table where keys are program names and values are tables where keys +-- are program versions and values are 'true'. +-- @param deps_mode string: Which trees to check dependencies for +-- @return table, table, table: A table where keys are dependencies parsed +-- in table format and values are tables containing fields 'name' and +-- version' representing matches; a table of missing dependencies +-- parsed as tables; and a table of "no-upgrade" missing dependencies +-- (to be used in plugin modules so that a plugin does not force upgrade of +-- its parent application). +function deps.match_deps(dependencies, rocks_provided, blacklist, deps_mode) + assert(type(dependencies) == "table") + assert(type(rocks_provided) == "table") + assert(type(blacklist) == "table" or blacklist == nil) + assert(type(deps_mode) == "string") + + local get_versions = prepare_get_versions(deps_mode, rocks_provided, "dependencies", blacklist) + return match_all_deps(dependencies, get_versions) end -local function rock_status(name, deps_mode, rocks_provided) - local installed = match_dep(queries.new(name), nil, deps_mode, rocks_provided) - local installation_type = rocks_provided[name] and "provided by VM" or "installed" +local function rock_status(name, get_versions) + assert(type(name) == "string") + assert(type(get_versions) == "function") + + local installed, _, _, provided = match_dep(queries.new(name), get_versions) + local installation_type = provided and "provided by VM" or "installed" return installed and installed.." "..installation_type or "not installed" end @@ -103,7 +168,7 @@ end -- @param name string: package name. -- @param version string: package version. -- @param dependencies table: array of dependencies. --- @param deps_mode string: Which trees to check dependencies for: +-- @param deps_mode string: Which trees to check dependencies for -- @param rocks_provided table: A table of auto-dependencies provided -- by this Lua implementation for the given dependency. -- "one" for the current default tree, "all" for all trees, @@ -114,58 +179,53 @@ function deps.report_missing_dependencies(name, version, dependencies, deps_mode assert(type(dependencies) == "table") assert(type(deps_mode) == "string") assert(type(rocks_provided) == "table") + + if deps_mode == "none" then + return + end + + local get_versions = prepare_get_versions(deps_mode, rocks_provided, "dependencies") local first_missing_dep = true for _, dep in ipairs(dependencies) do - if not match_dep(dep, nil, deps_mode, rocks_provided) then + local found, _ + found, _, dep = match_dep(dep, get_versions) + if not found then if first_missing_dep then util.printout(("Missing dependencies for %s %s:"):format(name, version)) first_missing_dep = false end - util.printout((" %s (%s)"):format(tostring(dep), rock_status(dep.name, deps_mode, rocks_provided))) + util.printout((" %s (%s)"):format(tostring(dep), rock_status(dep.name, get_versions))) end end end -function deps.fulfill_dependency(dep, deps_mode, name, version, rocks_provided, verify) +function deps.fulfill_dependency(dep, deps_mode, rocks_provided, verify, depskey) assert(dep:type() == "query") assert(type(deps_mode) == "string" or deps_mode == nil) - assert(type(name) == "string" or name == nil) - assert(type(version) == "string" or version == nil) assert(type(rocks_provided) == "table" or rocks_provided == nil) assert(type(verify) == "boolean" or verify == nil) + assert(type(depskey) == "string") + deps_mode = deps_mode or "all" rocks_provided = rocks_provided or {} - local found, where = match_dep(dep, nil, deps_mode, rocks_provided) + local get_versions = prepare_get_versions(deps_mode, rocks_provided, depskey) + + local found, where + found, where, dep = match_dep(dep, get_versions) if found then + local tree_manifests = manif.load_rocks_tree_manifests(deps_mode) + manif.scan_dependencies(dep.name, found, tree_manifests, deplocks.proxy(depskey)) return true, found, where end local search = require("luarocks.search") local install = require("luarocks.cmd.install") - if name and version then - util.printout(("%s %s depends on %s (%s)"):format( - name, version, tostring(dep), rock_status(dep.name, deps_mode, rocks_provided))) - else - util.printout(("Fulfilling dependency on %s (%s)"):format( - tostring(dep), rock_status(dep.name, deps_mode, rocks_provided))) - end - - if dep.constraints[1] and dep.constraints[1].no_upgrade then - util.printerr("This version of "..name.." is designed for use with") - util.printerr(tostring(dep)..", but is configured to avoid upgrading it") - util.printerr("automatically. Please upgrade "..dep.name.." with") - util.printerr(" luarocks install "..dep.name) - util.printerr("or choose an older version of "..name.." with") - util.printerr(" luarocks search "..name) - return nil, "Failed matching dependencies" - end - - local url, search_err = search.find_suitable_rock(dep) + local url, search_err = search.find_suitable_rock(dep, true) if not url then return nil, "Could not satisfy dependency "..tostring(dep)..": "..search_err end @@ -181,27 +241,12 @@ function deps.fulfill_dependency(dep, deps_mode, name, version, rocks_provided, return nil, "Failed installing dependency: "..url.." - "..install_err, errcode end - found, where = match_dep(dep, nil, deps_mode, rocks_provided) + found, where = match_dep(dep, get_versions) assert(found) return true, found, where end ---- Check dependencies of a rock and attempt to install any missing ones. --- Packages are installed using the LuaRocks "install" command. --- Aborts the program if a dependency could not be fulfilled. --- @param rockspec table: A rockspec in table format. --- @param depskey string: Rockspec key to fetch to get dependency table. --- @param deps_mode string --- @param verify boolean --- @return boolean or (nil, string, [string]): True if no errors occurred, or --- nil and an error message if any test failed, followed by an optional --- error code. -function deps.fulfill_dependencies(rockspec, depskey, deps_mode, verify) - assert(type(rockspec) == "table") - assert(type(depskey) == "string") - assert(type(deps_mode) == "string") - assert(type(verify) == "boolean" or verify == nil) - +local function check_supported_platforms(rockspec) if rockspec.supported_platforms and next(rockspec.supported_platforms) then local all_negative = true local supported = false @@ -225,14 +270,82 @@ function deps.fulfill_dependencies(rockspec, depskey, deps_mode, verify) return nil, "This rockspec for "..rockspec.package.." does not support "..plats.." platforms." end end + + return true +end - deps.report_missing_dependencies(rockspec.name, rockspec.version, rockspec[depskey], deps_mode, rockspec.rocks_provided) +--- Check dependencies of a rock and attempt to install any missing ones. +-- Packages are installed using the LuaRocks "install" command. +-- Aborts the program if a dependency could not be fulfilled. +-- @param rockspec table: A rockspec in table format. +-- @param depskey string: Rockspec key to fetch to get dependency table. +-- @param deps_mode string +-- @param verify boolean +-- @param deplock_dir string: dirname of the deplock file +-- @return boolean or (nil, string, [string]): True if no errors occurred, or +-- nil and an error message if any test failed, followed by an optional +-- error code. +function deps.fulfill_dependencies(rockspec, depskey, deps_mode, verify, deplock_dir) + assert(type(rockspec) == "table") + assert(type(depskey) == "string") + assert(type(deps_mode) == "string") + assert(type(verify) == "boolean" or verify == nil) + assert(type(deplock_dir) == "string" or deplock_dir == nil) + + local name = rockspec.name + local version = rockspec.version + local rocks_provided = rockspec.rocks_provided + + local ok, filename, err = deplocks.load(name, deplock_dir or ".") + if filename then + util.printout("Using dependencies pinned in lockfile: " .. filename) + + local get_versions = prepare_get_versions("none", rocks_provided, depskey) + for dname, dversion in deplocks.each(depskey) do + local dep = queries.new(dname, dversion) + + util.printout(("%s %s is pinned to %s (%s)"):format( + name, version, tostring(dep), rock_status(dep.name, get_versions))) + + local ok, err = deps.fulfill_dependency(dep, "none", rocks_provided, verify, depskey) + if not ok then + return nil, err + end + end + util.printout() + return true + elseif err then + util.warning(err) + end + + ok, err = check_supported_platforms(rockspec) + if not ok then + return nil, err + end + + deps.report_missing_dependencies(name, version, rockspec[depskey], deps_mode, rocks_provided) util.printout() + + local get_versions = prepare_get_versions(deps_mode, rocks_provided, depskey) for _, dep in ipairs(rockspec[depskey]) do - local ok, err = deps.fulfill_dependency(dep, deps_mode, rockspec.name, rockspec.version, rockspec.rocks_provided, verify) - if not ok then - return nil, err + + util.printout(("%s %s depends on %s (%s)"):format( + name, version, tostring(dep), rock_status(dep.name, get_versions))) + + local ok, found_or_err, _, no_upgrade = deps.fulfill_dependency(dep, deps_mode, rocks_provided, verify, depskey) + if ok then + deplocks.add(depskey, dep.name, found_or_err) + else + if no_upgrade then + util.printerr("This version of "..name.." is designed for use with") + util.printerr(tostring(dep)..", but is configured to avoid upgrading it") + util.printerr("automatically. Please upgrade "..dep.name.." with") + util.printerr(" luarocks install "..dep.name) + util.printerr("or look for a suitable version of "..name.." with") + util.printerr(" luarocks search "..name) + end + return nil, found_or_err end end @@ -500,7 +613,10 @@ function deps.scan_deps(results, manifest, name, version, deps_mode) else rocks_provided = util.get_rocks_provided() end - local matched = deps.match_deps(dependencies, rocks_provided, nil, deps_mode) + + local get_versions = prepare_get_versions(deps_mode, rocks_provided, "dependencies") + + local matched = match_all_deps(dependencies, get_versions) results[name] = version for _, match in pairs(matched) do deps.scan_deps(results, manifest, match.name, match.version, deps_mode) diff --git a/src/luarocks/persist.lua b/src/luarocks/persist.lua index 1c4b82b5..b21323ce 100644 --- a/src/luarocks/persist.lua +++ b/src/luarocks/persist.lua @@ -8,6 +8,7 @@ local util = require("luarocks.util") local dir = require("luarocks.dir") local fs = require("luarocks.fs") +persist.run_file = core.run_file persist.load_into_table = core.load_into_table local write_table @@ -200,7 +201,7 @@ function persist.save_from_table(filename, tbl, field_order) end --- Save the contents of a table as a module. --- Each element of the table is saved as a global assignment. +-- The module contains a 'return' statement that returns the table. -- Only numbers, strings and tables (containing numbers, strings -- or other recursively processed tables) are supported. -- @param filename string: the output filename diff --git a/src/luarocks/test/busted.lua b/src/luarocks/test/busted.lua index 618054c7..8fa78804 100644 --- a/src/luarocks/test/busted.lua +++ b/src/luarocks/test/busted.lua @@ -21,7 +21,7 @@ function busted.run_tests(test, args) test = {} end - local ok, bustedver, where = deps.fulfill_dependency(queries.new("busted")) + local ok, bustedver, where = deps.fulfill_dependency(queries.new("busted"), nil, nil, nil, "test_dependencies") if not ok then return nil, bustedver end -- cgit v1.2.3-55-g6feb