From d4990de9dcfead8cbef9aec79c0f088655595f11 Mon Sep 17 00:00:00 2001 From: Hisham Muhammad Date: Tue, 12 Nov 2019 19:26:38 -0300 Subject: cache rockspecs and src.rocks and check timestamp --- src/luarocks/fetch.lua | 50 +++++++++++++++++++++++++++++++++++++++++++---- src/luarocks/fs/tools.lua | 7 +++++-- src/luarocks/manif.lua | 23 +--------------------- 3 files changed, 52 insertions(+), 28 deletions(-) diff --git a/src/luarocks/fetch.lua b/src/luarocks/fetch.lua index 6835462a..0fc3f506 100644 --- a/src/luarocks/fetch.lua +++ b/src/luarocks/fetch.lua @@ -10,6 +10,33 @@ local persist = require("luarocks.persist") local util = require("luarocks.util") local cfg = require("luarocks.core.cfg") +function fetch.fetch_caching(url) + local repo_url, filename = url:match("^(.*)/([^/]+)$") + local name = repo_url:gsub("[/:]","_") + local cache_dir = dir.path(cfg.local_cache, name) + local ok = fs.make_dir(cache_dir) + if not ok then + cfg.local_cache = fs.make_temp_dir("local_cache") + cache_dir = dir.path(cfg.local_cache, name) + ok = fs.make_dir(cache_dir) + if not ok then + return nil, "Failed creating temporary cache directory "..cache_dir + end + end + + local cachefile = dir.path(cache_dir, filename) + if cfg.aggressive_cache and (not name:match("^manifest")) and fs.exists(cachefile) then + print("FAST TRACK!", cfg.aggressive_cache) + return cachefile, nil, nil, true + end + + local file, err, errcode, from_cache = fetch.fetch_url(url, cachefile, true) + if not file then + return nil, "Failed downloading "..repo_url..(err and " - "..err or ""), errcode + end + return file, nil, nil, from_cache +end + --- Fetch a local or remote file. -- Make a remote or local URL/pathname local, fetching the file if necessary. -- Other "fetch" and "load" functions use this function to obtain files. @@ -60,7 +87,7 @@ end -- @return (string, string) or (nil, string, [string]): absolute local pathname of -- the fetched file and temporary directory name; or nil and an error message -- followed by an optional error code -function fetch.fetch_url_at_temp_dir(url, tmpname, filename) +function fetch.fetch_url_at_temp_dir(url, tmpname, filename, cache) assert(type(url) == "string") assert(type(tmpname) == "string") assert(type(filename) == "string" or not filename) @@ -81,11 +108,26 @@ function fetch.fetch_url_at_temp_dir(url, tmpname, filename) util.schedule_function(fs.delete, temp_dir) local ok, err = fs.change_dir(temp_dir) if not ok then return nil, err end - local file, err, errcode = fetch.fetch_url(url, filename) + + local file, err, errcode + + if cache then + local cachefile + cachefile, err, errcode = fetch.fetch_caching(url) + + if cachefile then + file = dir.path(temp_dir, filename) + fs.copy(cachefile, file) + end + else + file, err, errcode = fetch.fetch_url(url, filename, cache) + end + fs.pop_dir() if not file then return nil, "Error fetching file: "..err, errcode end + return file, temp_dir end end @@ -159,7 +201,7 @@ function fetch.fetch_and_unpack_rock(url, dest, verify) local name = dir.base_name(url):match("(.*)%.[^.]*%.rock") local tmpname = "luarocks-rock-" .. name - local rock_file, err, errcode = fetch.fetch_url_at_temp_dir(url, tmpname) + local rock_file, err, errcode = fetch.fetch_url_at_temp_dir(url, tmpname, nil, true) if not rock_file then return nil, "Could not fetch rock file: " .. err, errcode end @@ -274,7 +316,7 @@ function fetch.load_rockspec(url, location, verify) filename, err = fetch.fetch_url(url) fs.pop_dir() else - filename, err, errcode = fetch.fetch_url_at_temp_dir(url, tmpname) + filename, err, errcode = fetch.fetch_url_at_temp_dir(url, tmpname, nil, true) end if not filename then return nil, err, errcode diff --git a/src/luarocks/fs/tools.lua b/src/luarocks/fs/tools.lua index ef5abe63..541352c2 100644 --- a/src/luarocks/fs/tools.lua +++ b/src/luarocks/fs/tools.lua @@ -15,8 +15,8 @@ do local tool_options = { downloader = { - { var = "CURL", name = "curl" }, { var = "WGET", name = "wget" }, + { var = "CURL", name = "curl" }, }, md5checker = { { var = "MD5SUM", name = "md5sum" }, @@ -168,7 +168,10 @@ function tools.use_downloader(url, filename, cache) if cfg.connection_timeout and cfg.connection_timeout > 0 then curl_cmd = curl_cmd .. "--connect-timeout "..tostring(cfg.connection_timeout).." " end - ok = fs.execute_string(fs.quiet_stderr(curl_cmd..fs.Q(url).." > "..fs.Q(filename))) + if cache then + curl_cmd = curl_cmd .. " -R -z \"" .. filename .. "\" " + end + ok = fs.execute_string(fs.quiet_stderr(curl_cmd..fs.Q(url).." --output "..fs.Q(filename))) else return false, "No downloader tool available -- please install 'wget' or 'curl' in your system" end diff --git a/src/luarocks/manif.lua b/src/luarocks/manif.lua index b32df971..1337cab3 100644 --- a/src/luarocks/manif.lua +++ b/src/luarocks/manif.lua @@ -65,27 +65,6 @@ function manif.load_rock_manifest(name, version, root) return rock_manifest.rock_manifest end - -local function fetch_manifest_from(repo_url, filename) - local url = dir.path(repo_url, filename) - local name = repo_url:gsub("[/:]","_") - local cache_dir = dir.path(cfg.local_cache, name) - local ok = fs.make_dir(cache_dir) - if not ok then - cfg.local_cache = fs.make_temp_dir("local_cache") - cache_dir = dir.path(cfg.local_cache, name) - ok = fs.make_dir(cache_dir) - if not ok then - return nil, "Failed creating temporary cache directory "..cache_dir - end - end - local file, err, errcode, from_cache = fetch.fetch_url(url, dir.path(cache_dir, filename), true) - if not file then - return nil, "Failed fetching manifest for "..repo_url..(err and " - "..err or ""), errcode - end - return file, nil, nil, from_cache -end - --- Load a local or remote manifest describing a repository. -- All functions that use manifest tables assume they were obtained -- through this function. @@ -124,7 +103,7 @@ function manif.load_manifest(repo_url, lua_version, versioned_only) else local err, errcode for _, filename in ipairs(filenames) do - pathname, err, errcode, from_cache = fetch_manifest_from(repo_url, filename) + pathname, err, errcode, from_cache = fetch.fetch_caching(dir.path(repo_url, filename)) if pathname then break end -- cgit v1.2.3-55-g6feb