From ff58301bd9755f8e86e35dbc16d925b1fc65105a Mon Sep 17 00:00:00 2001 From: Hisham Muhammad Date: Tue, 13 Apr 2021 16:52:15 -0300 Subject: use mirrors when downloading rocks even if manifest succeeds LuaRocks generally only checked whether to use a mirror in the first operation, when it fetches the manifest. If the manifest fails to load, it switches to the mirror and everything works from there. But if the manifest fetches ok and the then actual rock download fails with a 504, it gives up, instead of trying that in a mirror as well. Changing that to make it retry every download on a mirror when the base URL matches one configured in cfg.rocks_servers should make it much more resilient. Fixes #1299. --- src/luarocks/fetch.lua | 76 +++++++++++++++++++++++++++++++++++++++++++++++--- src/luarocks/manif.lua | 2 +- 2 files changed, 73 insertions(+), 5 deletions(-) (limited to 'src') diff --git a/src/luarocks/fetch.lua b/src/luarocks/fetch.lua index 1882c348..c7f2d549 100644 --- a/src/luarocks/fetch.lua +++ b/src/luarocks/fetch.lua @@ -10,7 +10,25 @@ local persist = require("luarocks.persist") local util = require("luarocks.util") local cfg = require("luarocks.core.cfg") -function fetch.fetch_caching(url) + +--- Fetch a local or remote file, using a local cache directory. +-- Make a remote or local URL/pathname local, fetching the file if necessary. +-- Other "fetch" and "load" functions use this function to obtain files. +-- If a local pathname is given, it is returned as a result. +-- @param url string: a local pathname or a remote URL. +-- @param mirroring string: mirroring mode. +-- If set to "no_mirror", then rocks_servers mirror configuration is not used. +-- @return (string, nil, nil, boolean) or (nil, string, [string]): +-- in case of success: +-- * the absolute local pathname for the fetched file +-- * nil +-- * nil +-- * `true` if the file was fetched from cache +-- in case of failure: +-- * nil +-- * an error message +-- * an optional error code. +function fetch.fetch_caching(url, mirroring) local repo_url, filename = url:match("^(.*)/([^/]+)$") local name = repo_url:gsub("[/:]","_") local cache_dir = dir.path(cfg.local_cache, name) @@ -29,13 +47,56 @@ function fetch.fetch_caching(url) return cachefile, nil, nil, true end - local file, err, errcode, from_cache = fetch.fetch_url(url, cachefile, true) + local file, err, errcode, from_cache = fetch.fetch_url(url, cachefile, true, mirroring) if not file then return nil, err or "Failed downloading "..url, errcode end return file, nil, nil, from_cache end +local function ensure_trailing_slash(url) + return (url:gsub("/*$", "/")) +end + +local function is_url_relative_to_rocks_servers(url, servers) + for _, item in ipairs(servers) do + if type(item) == "table" then + for i, s in ipairs(item) do + local base = ensure_trailing_slash(s) + if string.find(url, base, 1, true) == 1 then + return i, url:sub(#base + 1), item + end + end + end + end +end + +local function download_with_mirrors(url, filename, cache, servers) + local idx, rest, mirrors = is_url_relative_to_rocks_servers(url, servers) + + if not idx then + -- URL is not from a rock server + return fs.download(url, filename, cache) + end + + -- URL is from a rock server: try to download it falling back to mirrors. + local err = "\n" + for i = idx, #mirrors do + local try_url = ensure_trailing_slash(mirrors[i]) .. rest + if i > idx then + util.warning("Failed downloading. Attempting mirror at " .. try_url) + end + local ok, name, from_cache = fs.download(try_url, filename, cache) + if ok then + return ok, name, from_cache + else + err = err .. name .. "\n" + end + end + + return nil, err +end + --- Fetch a local or remote file. -- Make a remote or local URL/pathname local, fetching the file if necessary. -- Other "fetch" and "load" functions use this function to obtain files. @@ -47,6 +108,8 @@ end -- filename can be given explicitly as this second argument. -- @param cache boolean: compare remote timestamps via HTTP HEAD prior to -- re-downloading the file. +-- @param mirroring string: mirroring mode. +-- If set to "no_mirror", then rocks_servers mirror configuration is not used. -- @return (string, nil, nil, boolean) or (nil, string, [string]): -- in case of success: -- * the absolute local pathname for the fetched file @@ -57,7 +120,7 @@ end -- * nil -- * an error message -- * an optional error code. -function fetch.fetch_url(url, filename, cache) +function fetch.fetch_url(url, filename, cache, mirroring) assert(type(url) == "string") assert(type(filename) == "string" or not filename) @@ -84,7 +147,12 @@ function fetch.fetch_url(url, filename, cache) return nil, "Failed copying local file " .. fullname .. " to " .. dstname .. ": " .. err end elseif dir.is_basic_protocol(protocol) then - local ok, name, from_cache = fs.download(url, filename, cache) + local ok, name, from_cache + if mirroring ~= "no_mirror" then + ok, name, from_cache = download_with_mirrors(url, filename, cache, cfg.rocks_servers) + else + ok, name, from_cache = fs.download(url, filename, cache) + end if not ok then return nil, "Failed downloading "..url..(name and " - "..name or ""), "network" end diff --git a/src/luarocks/manif.lua b/src/luarocks/manif.lua index 5790ef18..a4ddda11 100644 --- a/src/luarocks/manif.lua +++ b/src/luarocks/manif.lua @@ -105,7 +105,7 @@ function manif.load_manifest(repo_url, lua_version, versioned_only) else local err, errcode for _, filename in ipairs(filenames) do - pathname, err, errcode, from_cache = fetch.fetch_caching(dir.path(repo_url, filename)) + pathname, err, errcode, from_cache = fetch.fetch_caching(dir.path(repo_url, filename), "no_mirror") if pathname then break end -- cgit v1.2.3-55-g6feb