From a1bc8ec547edd8466390cc984fca91bc93eb6e60 Mon Sep 17 00:00:00 2001 From: Hisham Muhammad Date: Sun, 20 Oct 2013 02:52:24 -0200 Subject: Cache remote manifest files locally and hit server with a HEAD request to check timestamp. --- src/luarocks/cfg.lua | 2 ++ src/luarocks/fetch.lua | 8 +++--- src/luarocks/fs/lua.lua | 58 ++++++++++++++++++++++++++++++++--------- src/luarocks/fs/unix/tools.lua | 30 +++++++++++++++------ src/luarocks/fs/win32/tools.lua | 31 +++++++++++++++------- src/luarocks/manif.lua | 7 ++++- src/luarocks/repos.lua | 46 ++++++++++++++++++++------------ 7 files changed, 132 insertions(+), 50 deletions(-) diff --git a/src/luarocks/cfg.lua b/src/luarocks/cfg.lua index 94afe0a0..019d9293 100644 --- a/src/luarocks/cfg.lua +++ b/src/luarocks/cfg.lua @@ -315,6 +315,7 @@ if detected.windows then defaults.export_path_separator = ";" defaults.export_lua_path = "SET LUA_PATH=%s" defaults.export_lua_cpath = "SET LUA_CPATH=%s" + defaults.wrapper_suffix = ".bat" defaults.local_cache = home.."/cache/luarocks" end @@ -372,6 +373,7 @@ if detected.unix then defaults.export_path_separator = ":" defaults.export_lua_path = "export LUA_PATH='%s'" defaults.export_lua_cpath = "export LUA_CPATH='%s'" + defaults.wrapper_suffix = "" defaults.local_cache = home.."/.cache/luarocks" if not defaults.variables.CFLAGS:match("-fPIC") then defaults.variables.CFLAGS = defaults.variables.CFLAGS.." -fPIC" diff --git a/src/luarocks/fetch.lua b/src/luarocks/fetch.lua index 3ba90647..e98d9cc2 100644 --- a/src/luarocks/fetch.lua +++ b/src/luarocks/fetch.lua @@ -27,7 +27,7 @@ end -- @return string or (nil, string, [string]): the absolute local pathname for the -- fetched file, or nil and a message in case of errors, followed by -- an optional error code. -function fetch_url(url, filename) +function fetch_url(url, filename, cache) assert(type(url) == "string") assert(type(filename) == "string" or not filename) @@ -35,11 +35,11 @@ function fetch_url(url, filename) if protocol == "file" then return fs.absolute_name(pathname) elseif is_basic_protocol(protocol, true) then - local ok, err = fs.download(url, filename) + local ok, filename = fs.download(url, filename, cache) if not ok then - return nil, "Failed downloading "..url..(err and " - "..err or ""), "network" + return nil, "Failed downloading "..url..(filename and " - "..filename or ""), "network" end - return dir.path(fs.current_dir(), filename or dir.base_name(url)) + return filename else return nil, "Unsupported protocol "..protocol end diff --git a/src/luarocks/fs/lua.lua b/src/luarocks/fs/lua.lua index 707b82bd..e637f6be 100644 --- a/src/luarocks/fs/lua.lua +++ b/src/luarocks/fs/lua.lua @@ -190,7 +190,7 @@ function pop_dir() end --- Create a directory if it does not already exist. --- If any of the higher levels in the path name does not exist +-- If any of the higher levels in the path name do not exist -- too, they are created as well. -- @param directory string: pathname of directory to create. -- @return boolean or (boolean, string): true on success or (false, error message) on failure. @@ -506,7 +506,7 @@ local redirect_protocols = { https = luasec_ok and https, } -local function http_request(url, http, loop_control) +local function request(url, method, http, loop_control) local result = {} local proxy = cfg.proxy @@ -516,14 +516,17 @@ local function http_request(url, http, loop_control) proxy = "http://" .. proxy end - io.write("Downloading "..url.." ...\n") + if cfg.show_downloads then + io.write(method.." "..url.." ...\n") + end local dots = 0 local res, status, headers, err = http.request { url = url, proxy = proxy, + method = method, redirect = false, sink = ltn12.sink.table(result), - step = function(...) + step = cfg.show_downloads and function(...) io.write(".") io.flush() dots = dots + 1 @@ -537,7 +540,9 @@ local function http_request(url, http, loop_control) ["user-agent"] = cfg.user_agent.." via LuaSocket" }, } - io.write("\n") + if cfg.show_downloads then + io.write("\n") + end if not res then return nil, status elseif status == 301 or status == 302 then @@ -551,7 +556,7 @@ local function http_request(url, http, loop_control) return nil, "Redirection loop -- broken URL?" end loop_control[url] = true - return http_request(location, redirect_protocols[protocol], loop_control) + return request(location, method, redirect_protocols[protocol], loop_control) else return nil, "URL redirected to unsupported protocol - install luasec to get HTTPS support." end @@ -560,7 +565,32 @@ local function http_request(url, http, loop_control) elseif status ~= 200 then return nil, err else + return result, status, headers, err + end +end + +local function http_request(url, http, cached) + if cached then + local tsfd = io.open(cached..".timestamp", "r") + if tsfd then + local timestamp = tsfd:read("*a") + tsfd:close() + local result, status, headers, err = request(url, "HEAD", http) + if status == 200 and headers["last-modified"] == timestamp then + return true + end + end + end + local result, status, headers, err = request(url, "GET", http) + if result then + if cached and headers["last-modified"] then + local tsfd = io.open(cached..".timestamp", "w") + tsfd:write(headers["last-modified"]) + tsfd:close() + end return table.concat(result) + else + return nil, status end end @@ -570,27 +600,31 @@ end -- resulting local filename of the remote file as the basename of the URL; -- if that is not correct (due to a redirection, for example), the local -- filename can be given explicitly as this second argument. --- @return boolean: true on success, false on failure. -function download(url, filename) +-- @return (boolean, string): true and the filename on success, +-- false and the error message on failure. +function download(url, filename, cache) assert(type(url) == "string") assert(type(filename) == "string" or not filename) - filename = dir.path(fs.current_dir(), filename or dir.base_name(url)) + filename = fs.absolute_name(filename or dir.base_name(url)) local content, err if util.starts_with(url, "http:") then - content, err = http_request(url, http) + content, err = http_request(url, http, cache and filename) elseif util.starts_with(url, "ftp:") then content, err = ftp.get(url) elseif util.starts_with(url, "https:") then if luasec_ok then - content, err = http_request(url, https) + content, err = http_request(url, https, cache and filename) else err = "Unsupported protocol - install luasec to get HTTPS support." end else err = "Unsupported protocol" end + if cache and content == true then + return true, filename + end if not content then return false, tostring(err) end @@ -598,7 +632,7 @@ function download(url, filename) if not file then return false end file:write(content) file:close() - return true + return true, filename end end diff --git a/src/luarocks/fs/unix/tools.lua b/src/luarocks/fs/unix/tools.lua index 1b2931e9..2dfca715 100644 --- a/src/luarocks/fs/unix/tools.lua +++ b/src/luarocks/fs/unix/tools.lua @@ -239,21 +239,35 @@ end -- resulting local filename of the remote file as the basename of the URL; -- if that is not correct (due to a redirection, for example), the local -- filename can be given explicitly as this second argument. --- @return boolean: true on success, false on failure. -function download(url, filename) +-- @return (boolean, string): true and the filename on success, +-- false and the error message on failure. +function download(url, filename, cache) assert(type(url) == "string") assert(type(filename) == "string" or not filename) + filename = fs.absolute_name(filename or dir.base_name(url)) + + local ok if cfg.downloader == "wget" then - local wget_cmd = vars.WGET.." --no-check-certificate --no-cache --user-agent='"..cfg.user_agent.." via wget' --quiet --continue " - if filename then - return fs.execute(wget_cmd.." --output-document ", filename, url) + local wget_cmd = vars.WGET.." --no-check-certificate --no-cache --user-agent='"..cfg.user_agent.." via wget' --quiet " + if cache then + -- --timestamping is incompatible with --output-document, + -- but that's not a problem for our use cases. + fs.change_dir(dir.dir_name(filename)) + ok = fs.execute(wget_cmd.." --timestamping ", url) + fs.pop_dir() + elseif filename then + ok = fs.execute(wget_cmd.." --output-document "..fs.Q(filename), url) else - return fs.execute(wget_cmd, url) + ok = fs.execute(wget_cmd, url) end elseif cfg.downloader == "curl" then - filename = filename or dir.base_name(url) - return fs.execute_string(vars.CURL.." -L --user-agent '"..cfg.user_agent.." via curl' "..fs.Q(url).." 2> /dev/null 1> "..fs.Q(filename)) + ok = fs.execute_string(vars.CURL.." -L --user-agent '"..cfg.user_agent.." via curl' "..fs.Q(url).." 2> /dev/null 1> "..fs.Q(filename)) + end + if ok then + return true, filename + else + return false end end diff --git a/src/luarocks/fs/win32/tools.lua b/src/luarocks/fs/win32/tools.lua index 2e125923..a79421b6 100644 --- a/src/luarocks/fs/win32/tools.lua +++ b/src/luarocks/fs/win32/tools.lua @@ -262,22 +262,35 @@ end -- resulting local filename of the remote file as the basename of the URL; -- if that is not correct (due to a redirection, for example), the local -- filename can be given explicitly as this second argument. --- @return boolean: true on success, false on failure. -function download(url, filename) +-- @return (boolean, string): true and the filename on success, +-- false and the error message on failure. +function download(url, filename, cache) assert(type(url) == "string") assert(type(filename) == "string" or not filename) + filename = fs.absolute_name(filename or dir.base_name(url)) + + local ok if cfg.downloader == "wget" then - local wget_cmd = vars.WGET.." --no-check-certificate --no-cache --user-agent=\""..cfg.user_agent.." via wget\" --quiet --continue " - if filename then - wget_cmd = wget_cmd.." --output-document "..fs.Q(filename).." "..fs.Q(url) + local wget_cmd = vars.WGET.." --no-check-certificate --no-cache --user-agent=\""..cfg.user_agent.." via wget\" --quiet " + if cache then + -- --timestamping is incompatible with --output-document, + -- but that's not a problem for our use cases. + fs.change_dir(dir.dir_name(filename)) + ok = fs.execute(wget_cmd.." --timestamping "..fs.Q(url).." 2> NUL 1> NUL") + fs.pop_dir() + elseif filename then + ok = fs.execute(wget_cmd.." --output-document "..fs.Q(filename)..fs.Q(url).." 2> NUL 1> NUL") else - wget_cmd = wget_cmd.." "..fs.Q(url) + ok = fs.execute(wget_cmd..fs.Q(url).." 2> NUL 1> NUL") end - return fs.execute(wget_cmd.." 2> NUL 1> NUL") elseif cfg.downloader == "curl" then - filename = filename or dir.base_name(url) - return fs.execute_string(vars.CURL.." -L --user-agent \""..cfg.user_agent.." via curl\" "..fs.Q(url).." 2> NUL 1> "..fs.Q(filename)) + ok = fs.execute_string(vars.CURL.." -L --user-agent \""..cfg.user_agent.." via curl\" "..fs.Q(url).." 2> NUL 1> "..fs.Q(filename)) + end + if ok then + return true, filename + else + return false end end diff --git a/src/luarocks/manif.lua b/src/luarocks/manif.lua index 03377d52..9feb35ce 100644 --- a/src/luarocks/manif.lua +++ b/src/luarocks/manif.lua @@ -84,7 +84,12 @@ end local function fetch_manifest_from(repo_url, filename) local url = dir.path(repo_url, filename) local name = repo_url:gsub("[/:]","_") - local file, err, errcode = fetch.fetch_url_at_temp_dir(url, "luarocks-manifest-"..name) + local cache_dir = dir.path(cfg.local_cache, name) + local ok = fs.make_dir(cache_dir) + if not ok then + return nil, "Failed creating temporary cache directory "..cache_dir + end + local file, err, errcode = fetch.fetch_url(url, dir.path(cache_dir, filename), true) if not file then return nil, "Failed fetching manifest for "..repo_url..(err and " - "..err or ""), errcode end diff --git a/src/luarocks/repos.lua b/src/luarocks/repos.lua index 08810015..86e82ec9 100644 --- a/src/luarocks/repos.lua +++ b/src/luarocks/repos.lua @@ -252,6 +252,20 @@ function deploy_files(name, version, wrap_bin_scripts) return ok, err end +local function delete_suffixed(filename, suffix) + local filenames = { filename } + if suffix and suffix ~= "" then filenames = { filename..suffix, filename } end + for _, name in ipairs(filenames) do + local ok, err = fs.delete(name) + if ok then + return true, name + elseif fs.exists(name) then + return nil, "Failed deleting "..name, "fail" + end + end + return false, "File not found", "not found" +end + --- Delete a package from the local repository. -- Version numbers are compared as exact string comparison. -- @param name string: name of package @@ -264,28 +278,28 @@ function delete_version(name, version, quick) assert(type(name) == "string") assert(type(version) == "string") - local function delete_deployed_file_tree(file_tree, deploy_dir) + local function delete_deployed_file_tree(file_tree, deploy_dir, suffix) return recurse_rock_manifest_tree(file_tree, function(parent_path, parent_module, file) local target = dir.path(deploy_dir, parent_path, file) local versioned = path.versioned_name(target, deploy_dir, name, version) - if fs.exists(versioned) then - local ok = fs.delete(versioned) + local ok, name, err = delete_suffixed(versioned, suffix) + if ok then fs.remove_dir_tree_if_empty(dir.dir_name(versioned)) - if not ok then return nil, "Failed deleting "..versioned end - else - local ok = fs.delete(target) - if not quick then - local next_name, next_version = manif.find_next_provider(target) - if next_name then - local versioned = path.versioned_name(target, deploy_dir, next_name, next_version) - fs.move(versioned, target) - fs.remove_dir_tree_if_empty(dir.dir_name(versioned)) - end + return true + end + if err == "fail" then return nil, name end + ok, name, err = delete_suffixed(target, suffix) + if err == "fail" then return nil, name end + if not quick then + local next_name, next_version = manif.find_next_provider(target) + if next_name then + local versioned = path.versioned_name(name, deploy_dir, next_name, next_version) + fs.move(versioned, name) + fs.remove_dir_tree_if_empty(dir.dir_name(versioned)) end - fs.remove_dir_tree_if_empty(dir.dir_name(target)) - if not ok then return nil, "Failed deleting "..target end end + fs.remove_dir_tree_if_empty(dir.dir_name(target)) return true end ) @@ -298,7 +312,7 @@ function delete_version(name, version, quick) local ok, err = true if rock_manifest.bin then - ok, err = delete_deployed_file_tree(rock_manifest.bin, cfg.deploy_bin_dir) + ok, err = delete_deployed_file_tree(rock_manifest.bin, cfg.deploy_bin_dir, cfg.wrapper_suffix) end if ok and rock_manifest.lua then ok, err = delete_deployed_file_tree(rock_manifest.lua, cfg.deploy_lua_dir) -- cgit v1.2.3-55-g6feb