From c9c83162748bea1f9c1b6574ce3cc6560e5490b4 Mon Sep 17 00:00:00 2001 From: hisham <hisham@9ca3f7c1-7366-0410-b1a3-b5c78f85698c> Date: Sun, 4 Oct 2009 16:12:07 +0000 Subject: Simplify the way things are handled for the new LuaRocks tree by creating rock_manifest files for each rock. git-svn-id: http://luarocks.org/svn/luarocks/trunk@62 9ca3f7c1-7366-0410-b1a3-b5c78f85698c --- src/luarocks/build.lua | 16 +- src/luarocks/install.lua | 25 ++- src/luarocks/make_manifest.lua | 3 +- src/luarocks/manif.lua | 371 ++++++++++++----------------------------- src/luarocks/pack.lua | 9 +- src/luarocks/rep.lua | 148 ++++++++++++---- 6 files changed, 253 insertions(+), 319 deletions(-) (limited to 'src') diff --git a/src/luarocks/build.lua b/src/luarocks/build.lua index a6fe1c45..997929a8 100644 --- a/src/luarocks/build.lua +++ b/src/luarocks/build.lua @@ -211,18 +211,18 @@ function build_rockspec(rockspec_file, need_to_fetch, minimal_mode) fs.pop_dir() end - ok, err = rep.install_bins(name, version) - if not ok then return nil, err end + ok, err = manif.make_rock_manifest(name, version) + if err then return nil, err end + + ok, err = rep.deploy_files(name, version) + if err then return nil, err end ok, err = rep.run_hook(rockspec, "post_install") - if err then - return nil, err - end + if err then return nil, err end ok, err = manif.update_manifest(name, version) - if err then - return nil, err - end + if err then return nil, err end + util.remove_scheduled_function(rollback) return true end diff --git a/src/luarocks/install.lua b/src/luarocks/install.lua index 3ef32c99..15e905e1 100644 --- a/src/luarocks/install.lua +++ b/src/luarocks/install.lua @@ -38,38 +38,35 @@ function install_binary_rock(rock_file) if rep.is_installed(name, version) then rep.delete_version(name, version) end + local rollback = util.schedule_function(function() fs.delete(path.install_dir(name, version)) fs.remove_dir_if_empty(path.versions_dir(name)) end) + local ok, err, errcode = fetch.fetch_and_unpack_rock(rock_file, path.install_dir(name, version)) if not ok then return nil, err, errcode end - ok, err = rep.install_bins(name, version) - + local rockspec, err, errcode = fetch.load_rockspec(path.rockspec_file(name, version)) if err then return nil, "Failed loading rockspec for installed package: "..err, errcode end ok, err, errcode = deps.check_external_deps(rockspec, "install") - if err then - return nil, err, errcode - end + if err then return nil, err, errcode end ok, err, errcode = deps.fulfill_dependencies(rockspec) - if err then - return nil, err, errcode - end + if err then return nil, err, errcode end ok, err = rep.run_hook(rockspec, "post_install") - if err then - return nil, err - end + if err then return nil, err end + + ok, err = rep.deploy_files(name, version) + if err then return nil, err end ok, err = manif.update_manifest(name, version) - if err then - return nil, err - end + if err then return nil, err end + util.remove_scheduled_function(rollback) return true end diff --git a/src/luarocks/make_manifest.lua b/src/luarocks/make_manifest.lua index 622c21a0..bec1b704 100644 --- a/src/luarocks/make_manifest.lua +++ b/src/luarocks/make_manifest.lua @@ -4,6 +4,7 @@ module("luarocks.make_manifest", package.seeall) local manif = require("luarocks.manif") +local index = require("luarocks.index") local cfg = require("luarocks.cfg") help_summary = "Compile a manifest file for a repository." @@ -26,7 +27,7 @@ function run(repo) ok, err = manif.make_manifest(repo) if ok then print("Generating index.html for "..repo) - manif.make_index(repo) + index.make_index(repo) end return ok, err end diff --git a/src/luarocks/manif.lua b/src/luarocks/manif.lua index 2343489b..de885ca5 100644 --- a/src/luarocks/manif.lua +++ b/src/luarocks/manif.lua @@ -1,85 +1,77 @@ ---- Functions for querying and manipulating manifest files. module("luarocks.manif", package.seeall) -local util = require("luarocks.util") +local manif_core = require("luarocks.manif_core") +local persist = require("luarocks.persist") +local fetch = require("luarocks.fetch") +local dir = require("luarocks.dir") local fs = require("luarocks.fs") local search = require("luarocks.search") +local util = require("luarocks.util") +local cfg = require("luarocks.cfg") +local path = require("luarocks.path") local rep = require("luarocks.rep") local deps = require("luarocks.deps") -local cfg = require("luarocks.cfg") -local persist = require("luarocks.persist") -local fetch = require("luarocks.fetch") -local dir = require("luarocks.dir") -local manif_core = require("luarocks.manif_core") -local function find_module_at_file(file, modules) - for module, location in pairs(modules) do - if file == location then - return module - end - end -end +rock_manifest_cache = {} + +--- Commit a table to disk in given local path. +-- @param where string: The directory where the table should be saved. +-- @param name string: The filename. +-- @param tbl table: The table to be saved. +-- @return boolean or (nil, string): true if successful, or nil and a +-- message in case of errors. +local function save_table(where, name, tbl) + assert(type(where) == "string") + assert(type(name) == "string") + assert(type(tbl) == "table") -local function rename_module(file, pkgid) - local path = dir.dir_name(file) - local name = dir.base_name(file) - local pkgid = pkgid:gsub("[/.-]", "_") - return dir.path(path, pkgid.."-"..name) + local filename = dir.path(where, name) + return persist.save_from_table(filename, tbl) end -local function update_global_lib(repo, manifest) - fs.make_dir(cfg.lua_modules_dir) - fs.make_dir(cfg.bin_modules_dir) - for rock, modules in pairs(manifest.modules) do - for module, file in pairs(modules) do - local module_type, modules_dir - - if file:match("%."..cfg.lua_extension.."$") then - module_type = "lua" - modules_dir = cfg.lua_modules_dir - else - module_type = "bin" - modules_dir = cfg.bin_modules_dir - end +function load_rock_manifest(name, version) + assert(type(name) == "string") + assert(type(version) == "string") - if not file:match("^"..modules_dir) then - local path_in_rock = dir.strip_base_dir(file:sub(#dir.path(repo, module)+2)) - local module_dir = dir.dir_name(path_in_rock) - local dest = dir.path(modules_dir, path_in_rock) - if module_dir ~= "" then - fs.make_dir(dir.dir_name(dest)) - end - if not fs.exists(dest) then - fs.move(file, dest) - fs.remove_dir_tree_if_empty(dir.dir_name(file)) - manifest.modules[rock][module] = dest - else - local current = find_module_at_file(dest, modules) - if not current then - util.warning("installed file not tracked by LuaRocks: "..dest) - else - local newname = rename_module(dest, current) - if fs.exists(newname) then - util.warning("conflict when tracking modules: "..newname.." exists.") - else - local ok, err = fs.move(dest, newname) - if ok then - manifest.modules[rock][current] = newname - fs.move(file, dest) - fs.remove_dir_tree_if_empty(dir.dir_name(file)) - manifest.modules[rock][module] = dest - else - util.warning(err) - end - end - end - end - else - -- print("File already in place.") + local name_version = name.."/"..version + if rock_manifest_cache[name_version] then + return rock_manifest_cache[name_version].rock_manifest + end + local install_dir = path.install_dir(name, version) + local pathname = dir.path(install_dir, "rock_manifest") + local rock_manifest = persist.load_into_table(pathname) + if not rock_manifest then return nil end + rock_manifest_cache[name_version] = rock_manifest + return rock_manifest.rock_manifest +end + +function make_rock_manifest(name, version) + local install_dir = path.install_dir(name, version) + local rock_manifest = dir.path(install_dir, "rock_manifest") + local tree = {} + for _, file in ipairs(fs.find(install_dir)) do + local full_path = dir.path(install_dir, file) + local walk = tree + local last + local last_name + for name in file:gmatch("[^/]+") do + local next = walk[name] + if not next then + next = {} + walk[name] = next end + last = walk + last_name = name + walk = next + end + if fs.is_file(full_path) then + last[last_name] = fs.get_md5(full_path) end end + local rock_manifest = { rock_manifest=tree } + rock_manifest_cache[name.."/"..version] = rock_manifest + save_table(install_dir, "rock_manifest", rock_manifest ) end --- Load a local or remote manifest describing a repository. @@ -110,23 +102,6 @@ function load_manifest(repo_url) return manif_core.manifest_loader(pathname, repo_url) end ---- Sort function for ordering rock identifiers in a manifest's --- modules table. Rocks are ordered alphabetically by name, and then --- by version which greater first. --- @param a string: Version to compare. --- @param b string: Version to compare. --- @return boolean: The comparison result, according to the --- rule outlined above. -local function sort_pkgs(a, b) - assert(type(a) == "string") - assert(type(b) == "string") - - local na, va = a:match("(.*)/(.*)$") - local nb, vb = b:match("(.*)/(.*)$") - - return (na == nb) and deps.compare_versions(va, vb) or na < nb -end - --- Output a table listing items of a package. -- @param itemsfn function: a function for obtaining items of a package. -- pkg and version will be passed to it; it should return a table with @@ -143,16 +118,34 @@ local function store_package_items(itemsfn, pkg, version, tbl) local pkg_version = pkg.."/"..version local result = {} + for item, path in pairs(itemsfn(pkg, version)) do result[item] = path if not tbl[item] then tbl[item] = {} end - tbl[item][pkg_version] = path + table.insert(tbl[item], pkg_version) end return result end +--- Sort function for ordering rock identifiers in a manifest's +-- modules table. Rocks are ordered alphabetically by name, and then +-- by version which greater first. +-- @param a string: Version to compare. +-- @param b string: Version to compare. +-- @return boolean: The comparison result, according to the +-- rule outlined above. +local function sort_pkgs(a, b) + assert(type(a) == "string") + assert(type(b) == "string") + + local na, va = a:match("(.*)/(.*)$") + local nb, vb = b:match("(.*)/(.*)$") + + return (na == nb) and deps.compare_versions(va, vb) or na < nb +end + --- Sort items of a package matching table by version number (higher versions first). -- @param tbl table: the package matching table: keys should be strings -- and values arrays of strings with packages names in "name/version" format. @@ -180,19 +173,6 @@ local function sort_package_matching_table(tbl) end end ---- Commit manifest to disk in given local repository. --- @param repo string: The directory of the local repository. --- @param manifest table: The manifest table --- @return boolean or (nil, string): true if successful, or nil and a --- message in case of errors. -local function save_manifest(repo, manifest) - assert(type(repo) == "string") - assert(type(manifest) == "table") - - local filename = dir.path(repo, "manifest") - return persist.save_from_table(filename, manifest) -end - --- Process the dependencies of a package to determine its dependency -- chain for loading modules. -- @param name string: Package name. @@ -252,6 +232,34 @@ local function store_results(results, manifest) sort_package_matching_table(manifest.commands) end +--- Scan a LuaRocks repository and output a manifest file. +-- A file called 'manifest' will be written in the root of the given +-- repository directory. +-- @param repo A local repository directory. +-- @return boolean or (nil, string): True if manifest was generated, +-- or nil and an error message. +function make_manifest(repo) + assert(type(repo) == "string") + + if not fs.is_dir(repo) then + return nil, "Cannot access repository at "..repo + end + + local query = search.make_query("") + query.exact_name = false + query.arch = "any" + local results = search.disk_search(repo, query) + local manifest = { repository = {}, modules = {}, commands = {} } + manif_core.manifest_cache[repo] = manifest + + print(util.show_table(results, "results")) + print(util.show_table(manifest, "manifest")) + + store_results(results, manifest) + + return save_table(repo, "manifest", manifest) +end + --- Load a manifest file from a local repository and add to the repository -- information with regard to the given name and version. -- A file called 'manifest' will be written in the root of the given @@ -284,173 +292,10 @@ function update_manifest(name, version, repo) end local results = {[name] = {[version] = {{arch = "installed", repo = repo}}}} - - store_results(results, manifest) - update_global_lib(repo, manifest) - return save_manifest(repo, manifest) -end - - --- Scan a LuaRocks repository and output a manifest file. --- A file called 'manifest' will be written in the root of the given --- repository directory. --- @param repo A local repository directory. --- @return boolean or (nil, string): True if manifest was generated, --- or nil and an error message. -function make_manifest(repo) - assert(type(repo) == "string") - - if not fs.is_dir(repo) then - return nil, "Cannot access repository at "..repo - end - local query = search.make_query("") - query.exact_name = false - query.arch = "any" - local results = search.disk_search(repo, query) - local manifest = { repository = {}, modules = {}, commands = {} } - manif_core.manifest_cache[repo] = manifest - - --print(util.show_table(results, "results")) - --print(util.show_table(manifest, "manifest")) + print("TODO LR2 update manifest") store_results(results, manifest) - - --print(util.show_table(manifest, "manifest after store")) - - update_global_lib(repo, manifest) - - --print(util.show_table(manifest, "manifest after update")) - - return save_manifest(repo, manifest) -end - -local index_header = [[ -<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"> -<html> -<head> -<title>Available rocks</title> -<meta http-equiv="content-type" content="text/html; charset=iso-8859-1"> -<style> -body { - background-color: white; - font-family: "bitstream vera sans", "verdana", "sans"; - font-size: 14px; -} -a { - color: #0000c0; - text-decoration: none; -} -a:hover { - text-decoration: underline; -} -td.main { - border-style: none; -} -blockquote { - font-size: 12px; -} -td.package { - background-color: #f0f0f0; - vertical-align: top; -} -td.spacer { - height: 5px; -} -td.version { - background-color: #d0d0d0; - vertical-align: top; - text-align: left; - padding: 5px; - width: 100px; -} -p.manifest { - font-size: 8px; -} -</style> -</head> -<body> -<h1>Available rocks</h1> -<p> -Lua modules avaliable from this location for use with <a href="http://www.luarocks.org">LuaRocks</a>: -</p> -<table class="main"> -]] - -local index_package_start = [[ -<td class="package"> -<p><a name="$anchor"></a><b>$package</b> - $summary<br/> -</p><blockquote><p>$detailed<br/> -<font size="-1"><a href="$original">latest sources</a> $homepage | License: $license</font></p> -</blockquote></a></td> -<td class="version"> -]] - -local index_package_end = [[ -</td></tr> -<tr><td colspan="2" class="spacer"></td></tr> -]] - -local index_footer = [[ -</table> -<p class="manifest"> -<a href="manifest">manifest file</a> -</p> -</body> -</html> -]] - -function make_index(repo) - if not fs.is_dir(repo) then - return nil, "Cannot access repository at "..repo - end - local manifest = load_manifest(repo) - local out = io.open(dir.path(repo, "index.html"), "w") - out:write(index_header) - for package, version_list in util.sortedpairs(manifest.repository) do - local latest_rockspec = nil - local output = index_package_start - for version, data in util.sortedpairs(version_list, deps.compare_versions) do - local out_versions = {} - local arches = 0 - output = output..version - local sep = ': ' - for _, item in ipairs(data) do - output = output .. sep .. '<a href="$url">'..item.arch..'</a>' - sep = ', ' - if item.arch == 'rockspec' then - local rs = ("%s-%s.rockspec"):format(package, version) - if not latest_rockspec then latest_rockspec = rs end - output = output:gsub("$url", rs) - else - output = output:gsub("$url", ("%s-%s.%s.rock"):format(package, version, item.arch)) - end - end - output = output .. '<br/>' - output = output:gsub("$na", arches) - end - output = output .. index_package_end - if latest_rockspec then - local rockspec = persist.load_into_table(dir.path(repo, latest_rockspec)) - local vars = { - anchor = package, - package = rockspec.package, - original = rockspec.source.url, - summary = rockspec.description.summary or "", - detailed = rockspec.description.detailed or "", - license = rockspec.description.license or "N/A", - homepage = rockspec.description.homepage and ("| <a href="..rockspec.description.homepage..">project homepage</a>") or "" - } - vars.detailed = vars.detailed:gsub("\n\n", "</p><p>"):gsub("%s+", " ") - output = output:gsub("$(%w+)", vars) - else - output = output:gsub("$anchor", package) - output = output:gsub("$package", package) - output = output:gsub("$(%w+)", "") - end - out:write(output) - end - out:write(index_footer) - out:close() + return save_table(repo, "manifest", manifest) end - diff --git a/src/luarocks/pack.lua b/src/luarocks/pack.lua index 4bfdb320..e4dc1794 100644 --- a/src/luarocks/pack.lua +++ b/src/luarocks/pack.lua @@ -95,11 +95,12 @@ local function pack_binary_rock(name, version) for package, file in pairs(module_data) do if package == name.."/"..version then local dest - if file:match("^"..cfg.lua_modules_dir) then - local pathname = file:sub(#cfg.lua_modules_dir + 1) + print("TODO LR2 do this based on rock_manifest") + if file:match("^"..cfg.deploy_lua_dir) then + local pathname = file:sub(#cfg.deploy_lua_dir + 1) dest = dir.path(temp_dir, "lua", dir.dir_name(pathname)) - elseif file:match("^"..cfg.bin_modules_dir) then - local pathname = file:sub(#cfg.bin_modules_dir + 1) + elseif file:match("^"..cfg.deploy_lib_dir) then + local pathname = file:sub(#cfg.deploy_lib_dir + 1) dest = dir.path(temp_dir, "lib", dir.dir_name(pathname)) is_binary = true end diff --git a/src/luarocks/rep.lua b/src/luarocks/rep.lua index 2c7659c1..0a2e9cbb 100644 --- a/src/luarocks/rep.lua +++ b/src/luarocks/rep.lua @@ -7,6 +7,7 @@ local path = require("luarocks.path") local cfg = require("luarocks.cfg") local util = require("luarocks.util") local dir = require("luarocks.dir") +local manif = require("luarocks.manif") --- Get all installed versions of a package. -- @param name string: a package name. @@ -41,19 +42,13 @@ function delete_version(name, version) assert(type(version) == "string") fs.delete(path.install_dir(name, version)) + print("TODO LR2 remove deployed files based on rock_manifest") if not get_versions(name) then fs.delete(dir.path(cfg.rocks_dir, name)) end end ---- Delete a command-line item from the bin directory. --- @param command string: name of script -function delete_bin(command) - assert(type(command) == "string") - - fs.delete(dir.path(cfg.scripts_dir, command)) -end - +--[[ --- Install bin entries in the repository bin dir. -- @param name string: name of package -- @param version string: package version in string format @@ -68,9 +63,9 @@ function install_bins(name, version, single_file) local bindir = path.bin_dir(name, version) if fs.exists(bindir) then - local ok, err = fs.make_dir(cfg.scripts_dir) + local ok, err = fs.make_dir(cfg.deploy_bin_dir) if not ok then - return nil, "Could not create "..cfg.scripts_dir + return nil, "Could not create "..cfg.deploy_bin_dir end local files = single_file and {single_file} or fs.list_dir(bindir) for _, file in pairs(files) do @@ -81,9 +76,9 @@ function install_bins(name, version, single_file) file = io.open(fullname) end if match or (file and file:read():match("#!.*lua.*")) then - ok, err = fs.wrap_script(fullname, cfg.scripts_dir) + ok, err = fs.wrap_script(fullname, cfg.deploy_bin_dir) else - ok, err = fs.copy_binary(fullname, cfg.scripts_dir) + ok, err = fs.copy_binary(fullname, cfg.deploy_bin_dir) end if file then file:close() end if not ok then @@ -93,6 +88,23 @@ function install_bins(name, version, single_file) end return true end +]] + +local function store_package_data(result, name, sub, prefix) + assert(type(result) == "table") + assert(type(name) == "string") + assert(type(sub) == "table" or type(sub) == "string") + assert(type(prefix) == "string") + + if type(sub) == "table" then + for sname, ssub in pairs(sub) do + store_package_data(result, sname, ssub, prefix..name.."/") + end + elseif type(sub) == "string" then + local pathname = prefix..name + result[path.path_to_module(pathname)] = pathname + end +end --- Obtain a list of modules within an installed package. -- @param package string: The package name; for example "luasocket" @@ -107,14 +119,18 @@ function package_modules(package, version) assert(type(version) == "string") local result = {} - for _, pathdir in pairs{ path.lua_dir, path.lib_dir } do - local basedir = pathdir(package, version) - local files = fs.find(basedir) - for _, file in ipairs(files) do - local name = path.path_to_module(file) - if name then - result[name] = dir.path(basedir, file) - end + local rock_manifest = manif.load_rock_manifest(package, version) + + print(pkg, version, util.show_table(rock_manifest)) + + if rock_manifest.lib then + for name,sub in pairs(rock_manifest.lib) do + store_package_data(result, name, sub, "", "") + end + end + if rock_manifest.lua then + for name,sub in pairs(rock_manifest.lua) do + store_package_data(result, name, sub, "", "") end end return result @@ -133,26 +149,30 @@ function package_commands(package, version) assert(type(version) == "string") local result = {} - local bindir = path.bin_dir(package, version) - local bins = fs.find(bindir) - for _, file in ipairs(bins) do - if file then - result[file] = dir.path(bindir, file) + local rock_manifest = manif.load_rock_manifest(package, version) + if rock_manifest.bin then + for name,sub in pairs(rock_manifest.bin) do + store_package_data(result, name, sub, "", "") end end return result end + --- Check if a rock contains binary executables. -- @param name string: name of an installed rock -- @param version string: version of an installed rock -- @return boolean: returns true if rock contains platform-specific -- binary executables, or false if it is a pure-Lua rock. function has_binaries(name, version) - local bin_dir = path.bin_dir(name, version) - if fs.exists(bin_dir) then - for _, name in pairs(fs.find(bin_dir)) do - if fs.is_actual_binary(dir.path(bin_dir, name)) then + assert(type(name) == "string") + assert(type(version) == "string") + + local rock_manifest = manif.load_rock_manifest(name, version) + if rock_manifest.bin then + for name, md5 in pairs(rock_manifest.bin) do + -- TODO verify that it is the same file. If it isn't, find the actual command. + if fs.is_actual_binary(dir.path(cfg.deploy_bin_dir, name)) then return true end end @@ -161,6 +181,9 @@ function has_binaries(name, version) end function run_hook(rockspec, hook_name) + assert(type(rockspec) == "table") + assert(type(hook_name) == "string") + local hooks = rockspec.hooks if not hooks then return true @@ -178,3 +201,70 @@ function run_hook(rockspec, hook_name) end return true end + +local function deploy_file_tree(file_tree, source_dir, deploy_dir, move_fn) + assert(type(file_tree) == "table") + assert(type(source_dir) == "string") + assert(type(deploy_dir) == "string") + assert(type(move_fn) == "function" or not move_fn) + + print("TODO LR2 actually fs.move") + if not move_fn then move_fn = fs.copy end + + local ok, err = fs.make_dir(deploy_dir) + if not ok then + return nil, "Could not create "..deploy_dir + end + for file, sub in pairs(file_tree) do + if type(sub) == "table" then + ok, err = deploy_file_tree(sub, dir.path(source_dir, file), dir.path(deploy_dir, file)) + if not ok then return nil, err end + else + local target = dir.path(deploy_dir, file) + if fs.exists(target) then + print("TODO LR2 make_way_for_new_version(target)") + end + local source = dir.path(source_dir, file) + ok, err = move_fn(source, deploy_dir) + if not ok then return nil, err end + end + end +end + +local function install_binary(source, target) + assert(type(source) == "string") + assert(type(target) == "string") + + local match = source:match("%.lua$") + local file, ok, err + if not match then + file = io.open(source) + end + if match or (file and file:read():match("^#!.*lua.*")) then + ok, err = fs.wrap_script(source, target) + else + ok, err = fs.copy_binary(source, target) + end + if file then file:close() end + return ok, err +end + +function deploy_files(name, version) + assert(type(name) == "string") + assert(type(version) == "string") + + local rock_manifest = manif.load_rock_manifest(name, version) + + if rock_manifest.bin then + local ok, err = deploy_file_tree(rock_manifest.bin, path_bin_dir(name, version), cfg.deploy_bin_dir, install_binary) + if err then return nil, err end + end + if rock_manifest.lua then + local ok, err = deploy_file_tree(rock_manifest.lua, path.lua_dir(name, version), cfg.deploy_lua_dir) + if err then return nil, err end + end + if rock_manifest.lib then + local ok, err = deploy_file_tree(rock_manifest.lib, path.lib_dir(name, version), cfg.deploy_lib_dir) + if err then return nil, err end + end +end -- cgit v1.2.3-55-g6feb