Cache remote manifest files locally and hit server with a HEAD request to check timestamp.

7 files changed, 132 insertions, 50 deletions

diff --git a/src/luarocks/cfg.lua b/src/luarocks/cfg.lua
index 94afe0a0..019d9293 100644
--- a/src/luarocks/cfg.lua
+++ b/src/luarocks/cfg.lua
@@ -315,6 +315,7 @@ if detected.windows then
    defaults.export_path_separator = ";"
    defaults.export_lua_path = "SET LUA_PATH=%s"
    defaults.export_lua_cpath = "SET LUA_CPATH=%s"
+   defaults.wrapper_suffix = ".bat"
    defaults.local_cache = home.."/cache/luarocks"
 end
 
@@ -372,6 +373,7 @@ if detected.unix then
    defaults.export_path_separator = ":"
    defaults.export_lua_path = "export LUA_PATH='%s'"
    defaults.export_lua_cpath = "export LUA_CPATH='%s'"
+   defaults.wrapper_suffix = ""
    defaults.local_cache = home.."/.cache/luarocks"
    if not defaults.variables.CFLAGS:match("-fPIC") then
       defaults.variables.CFLAGS = defaults.variables.CFLAGS.." -fPIC"
diff --git a/src/luarocks/fetch.lua b/src/luarocks/fetch.lua
index 3ba90647..e98d9cc2 100644
--- a/src/luarocks/fetch.lua
+++ b/src/luarocks/fetch.lua
@@ -27,7 +27,7 @@ end
 -- @return string or (nil, string, [string]): the absolute local pathname for the
 -- fetched file, or nil and a message in case of errors, followed by
 -- an optional error code.
-function fetch_url(url, filename)
+function fetch_url(url, filename, cache)
    assert(type(url) == "string")
    assert(type(filename) == "string" or not filename)
 
@@ -35,11 +35,11 @@ function fetch_url(url, filename)
    if protocol == "file" then
       return fs.absolute_name(pathname)
    elseif is_basic_protocol(protocol, true) then
-      local ok, err = fs.download(url, filename)
+      local ok, filename = fs.download(url, filename, cache)
       if not ok then
-         return nil, "Failed downloading "..url..(err and " - "..err or ""), "network"
+         return nil, "Failed downloading "..url..(filename and " - "..filename or ""), "network"
       end
-      return dir.path(fs.current_dir(), filename or dir.base_name(url))
+      return filename
    else
       return nil, "Unsupported protocol "..protocol
    end
diff --git a/src/luarocks/fs/lua.lua b/src/luarocks/fs/lua.lua
index 707b82bd..e637f6be 100644
--- a/src/luarocks/fs/lua.lua
+++ b/src/luarocks/fs/lua.lua
@@ -190,7 +190,7 @@ function pop_dir()
 end
 
 --- Create a directory if it does not already exist.
--- If any of the higher levels in the path name does not exist
+-- If any of the higher levels in the path name do not exist
 -- too, they are created as well.
 -- @param directory string: pathname of directory to create.
 -- @return boolean or (boolean, string): true on success or (false, error message) on failure.
@@ -506,7 +506,7 @@ local redirect_protocols = {
    https = luasec_ok and https,
 }
 
-local function http_request(url, http, loop_control)
+local function request(url, method, http, loop_control)
    local result = {}
    
    local proxy = cfg.proxy
@@ -516,14 +516,17 @@ local function http_request(url, http, loop_control)
       proxy = "http://" .. proxy
    end
    
-   io.write("Downloading "..url.." ...\n")
+   if cfg.show_downloads then
+      io.write(method.." "..url.." ...\n")
+   end
    local dots = 0
    local res, status, headers, err = http.request {
       url = url,
       proxy = proxy,
+      method = method,
       redirect = false,
       sink = ltn12.sink.table(result),
-      step = function(...)
+      step = cfg.show_downloads and function(...)
          io.write(".")
          io.flush()
          dots = dots + 1
@@ -537,7 +540,9 @@ local function http_request(url, http, loop_control)
          ["user-agent"] = cfg.user_agent.." via LuaSocket"
       },
    }
-   io.write("\n")
+   if cfg.show_downloads then
+      io.write("\n")
+   end
    if not res then
       return nil, status
    elseif status == 301 or status == 302 then
@@ -551,7 +556,7 @@ local function http_request(url, http, loop_control)
                return nil, "Redirection loop -- broken URL?"
             end
             loop_control[url] = true
-            return http_request(location, redirect_protocols[protocol], loop_control)
+            return request(location, method, redirect_protocols[protocol], loop_control)
          else
             return nil, "URL redirected to unsupported protocol - install luasec to get HTTPS support."
          end
@@ -560,7 +565,32 @@ local function http_request(url, http, loop_control)
    elseif status ~= 200 then
       return nil, err
    else
+      return result, status, headers, err
+   end
+end
+
+local function http_request(url, http, cached)
+   if cached then
+      local tsfd = io.open(cached..".timestamp", "r")
+      if tsfd then
+         local timestamp = tsfd:read("*a")
+         tsfd:close()
+         local result, status, headers, err = request(url, "HEAD", http)
+         if status == 200 and headers["last-modified"] == timestamp then
+            return true
+         end
+      end
+   end
+   local result, status, headers, err = request(url, "GET", http)
+   if result then
+      if cached and headers["last-modified"] then
+         local tsfd = io.open(cached..".timestamp", "w")
+         tsfd:write(headers["last-modified"])
+         tsfd:close()
+      end
       return table.concat(result)
+   else
+      return nil, status
    end
 end
 
@@ -570,27 +600,31 @@ end
 -- resulting local filename of the remote file as the basename of the URL;
 -- if that is not correct (due to a redirection, for example), the local
 -- filename can be given explicitly as this second argument.
--- @return boolean: true on success, false on failure.
-function download(url, filename)
+-- @return (boolean, string): true and the filename on success,
+-- false and the error message on failure.
+function download(url, filename, cache)
    assert(type(url) == "string")
    assert(type(filename) == "string" or not filename)
 
-   filename = dir.path(fs.current_dir(), filename or dir.base_name(url))
+   filename = fs.absolute_name(filename or dir.base_name(url))
    
    local content, err
    if util.starts_with(url, "http:") then
-      content, err = http_request(url, http)
+      content, err = http_request(url, http, cache and filename)
    elseif util.starts_with(url, "ftp:") then
       content, err = ftp.get(url)
    elseif util.starts_with(url, "https:") then
       if luasec_ok then
-         content, err = http_request(url, https)
+         content, err = http_request(url, https, cache and filename)
       else
          err = "Unsupported protocol - install luasec to get HTTPS support."
       end
    else
       err = "Unsupported protocol"
    end
+   if cache and content == true then
+      return true, filename
+   end
    if not content then
       return false, tostring(err)
    end
@@ -598,7 +632,7 @@ function download(url, filename)
    if not file then return false end
    file:write(content)
    file:close()
-   return true
+   return true, filename
 end
 
 end
diff --git a/src/luarocks/fs/unix/tools.lua b/src/luarocks/fs/unix/tools.lua
index 1b2931e9..2dfca715 100644
--- a/src/luarocks/fs/unix/tools.lua
+++ b/src/luarocks/fs/unix/tools.lua
@@ -239,21 +239,35 @@ end
 -- resulting local filename of the remote file as the basename of the URL;
 -- if that is not correct (due to a redirection, for example), the local
 -- filename can be given explicitly as this second argument.
--- @return boolean: true on success, false on failure.
-function download(url, filename)
+-- @return (boolean, string): true and the filename on success,
+-- false and the error message on failure.
+function download(url, filename, cache)
    assert(type(url) == "string")
    assert(type(filename) == "string" or not filename)
 
+   filename = fs.absolute_name(filename or dir.base_name(url))
+
+   local ok
    if cfg.downloader == "wget" then
-      local wget_cmd = vars.WGET.." --no-check-certificate --no-cache --user-agent='"..cfg.user_agent.." via wget' --quiet --continue "
-      if filename then
-         return fs.execute(wget_cmd.." --output-document ", filename, url)
+      local wget_cmd = vars.WGET.." --no-check-certificate --no-cache --user-agent='"..cfg.user_agent.." via wget' --quiet "
+      if cache then
+         -- --timestamping is incompatible with --output-document,
+         -- but that's not a problem for our use cases.
+         fs.change_dir(dir.dir_name(filename))
+         ok = fs.execute(wget_cmd.." --timestamping ", url)
+         fs.pop_dir()
+      elseif filename then
+         ok = fs.execute(wget_cmd.." --output-document "..fs.Q(filename), url)
       else
-         return fs.execute(wget_cmd, url)
+         ok = fs.execute(wget_cmd, url)
       end
    elseif cfg.downloader == "curl" then
-      filename = filename or dir.base_name(url)
-      return fs.execute_string(vars.CURL.." -L --user-agent '"..cfg.user_agent.." via curl' "..fs.Q(url).." 2> /dev/null 1> "..fs.Q(filename))
+      ok = fs.execute_string(vars.CURL.." -L --user-agent '"..cfg.user_agent.." via curl' "..fs.Q(url).." 2> /dev/null 1> "..fs.Q(filename))
+   end
+   if ok then
+      return true, filename
+   else
+      return false
    end
 end
 
diff --git a/src/luarocks/fs/win32/tools.lua b/src/luarocks/fs/win32/tools.lua
index 2e125923..a79421b6 100644
--- a/src/luarocks/fs/win32/tools.lua
+++ b/src/luarocks/fs/win32/tools.lua
@@ -262,22 +262,35 @@ end
 -- resulting local filename of the remote file as the basename of the URL;
 -- if that is not correct (due to a redirection, for example), the local
 -- filename can be given explicitly as this second argument.
--- @return boolean: true on success, false on failure.
-function download(url, filename)
+-- @return (boolean, string): true and the filename on success,
+-- false and the error message on failure.
+function download(url, filename, cache)
    assert(type(url) == "string")
    assert(type(filename) == "string" or not filename)
 
+   filename = fs.absolute_name(filename or dir.base_name(url))
+
+   local ok   
    if cfg.downloader == "wget" then
-      local wget_cmd = vars.WGET.." --no-check-certificate --no-cache --user-agent=\""..cfg.user_agent.." via wget\" --quiet --continue "
-      if filename then
-         wget_cmd = wget_cmd.." --output-document "..fs.Q(filename).." "..fs.Q(url)
+      local wget_cmd = vars.WGET.." --no-check-certificate --no-cache --user-agent=\""..cfg.user_agent.." via wget\" --quiet "
+      if cache then
+         -- --timestamping is incompatible with --output-document,
+         -- but that's not a problem for our use cases.
+         fs.change_dir(dir.dir_name(filename))
+         ok = fs.execute(wget_cmd.." --timestamping "..fs.Q(url).." 2> NUL 1> NUL")
+         fs.pop_dir()
+      elseif filename then
+         ok = fs.execute(wget_cmd.." --output-document "..fs.Q(filename)..fs.Q(url).." 2> NUL 1> NUL")
       else
-         wget_cmd = wget_cmd.." "..fs.Q(url)
+         ok = fs.execute(wget_cmd..fs.Q(url).." 2> NUL 1> NUL")
       end
-      return fs.execute(wget_cmd.." 2> NUL 1> NUL")
    elseif cfg.downloader == "curl" then
-      filename = filename or dir.base_name(url)
-      return fs.execute_string(vars.CURL.." -L --user-agent \""..cfg.user_agent.." via curl\" "..fs.Q(url).." 2> NUL 1> "..fs.Q(filename))
+      ok = fs.execute_string(vars.CURL.." -L --user-agent \""..cfg.user_agent.." via curl\" "..fs.Q(url).." 2> NUL 1> "..fs.Q(filename))
+   end
+   if ok then
+      return true, filename
+   else
+      return false
    end
 end
 
diff --git a/src/luarocks/manif.lua b/src/luarocks/manif.lua
index 03377d52..9feb35ce 100644
--- a/src/luarocks/manif.lua
+++ b/src/luarocks/manif.lua
@@ -84,7 +84,12 @@ end
 local function fetch_manifest_from(repo_url, filename)
    local url = dir.path(repo_url, filename)
    local name = repo_url:gsub("[/:]","_")
-   local file, err, errcode = fetch.fetch_url_at_temp_dir(url, "luarocks-manifest-"..name)
+   local cache_dir = dir.path(cfg.local_cache, name)
+   local ok = fs.make_dir(cache_dir)
+   if not ok then
+      return nil, "Failed creating temporary cache directory "..cache_dir
+   end
+   local file, err, errcode = fetch.fetch_url(url, dir.path(cache_dir, filename), true)
    if not file then
       return nil, "Failed fetching manifest for "..repo_url..(err and " - "..err or ""), errcode
    end
diff --git a/src/luarocks/repos.lua b/src/luarocks/repos.lua
index 08810015..86e82ec9 100644
--- a/src/luarocks/repos.lua
+++ b/src/luarocks/repos.lua
@@ -252,6 +252,20 @@ function deploy_files(name, version, wrap_bin_scripts)
    return ok, err
 end
 
+local function delete_suffixed(filename, suffix)
+   local filenames = { filename }
+   if suffix and suffix ~= "" then filenames = { filename..suffix, filename } end
+   for _, name in ipairs(filenames) do
+      local ok, err = fs.delete(name)
+      if ok then
+         return true, name
+      elseif fs.exists(name) then
+         return nil, "Failed deleting "..name, "fail"
+      end
+   end
+   return false, "File not found", "not found"
+end
+
 --- Delete a package from the local repository.
 -- Version numbers are compared as exact string comparison.
 -- @param name string: name of package
@@ -264,28 +278,28 @@ function delete_version(name, version, quick)
    assert(type(name) == "string")
    assert(type(version) == "string")
 
-   local function delete_deployed_file_tree(file_tree, deploy_dir)
+   local function delete_deployed_file_tree(file_tree, deploy_dir, suffix)
       return recurse_rock_manifest_tree(file_tree, 
          function(parent_path, parent_module, file)
             local target = dir.path(deploy_dir, parent_path, file)
             local versioned = path.versioned_name(target, deploy_dir, name, version)
-            if fs.exists(versioned) then
-               local ok = fs.delete(versioned)
+            local ok, name, err = delete_suffixed(versioned, suffix)
+            if ok then
                fs.remove_dir_tree_if_empty(dir.dir_name(versioned))
-               if not ok then return nil, "Failed deleting "..versioned end
-            else
-               local ok = fs.delete(target)
-               if not quick then
-                  local next_name, next_version = manif.find_next_provider(target)
-                  if next_name then
-                     local versioned = path.versioned_name(target, deploy_dir, next_name, next_version)
-                     fs.move(versioned, target)
-                     fs.remove_dir_tree_if_empty(dir.dir_name(versioned))
-                  end
+               return true
+            end
+            if err == "fail" then return nil, name end
+            ok, name, err = delete_suffixed(target, suffix)
+            if err == "fail" then return nil, name end
+            if not quick then
+               local next_name, next_version = manif.find_next_provider(target)
+               if next_name then
+                  local versioned = path.versioned_name(name, deploy_dir, next_name, next_version)
+                  fs.move(versioned, name)
+                  fs.remove_dir_tree_if_empty(dir.dir_name(versioned))
                end
-               fs.remove_dir_tree_if_empty(dir.dir_name(target))
-               if not ok then return nil, "Failed deleting "..target end
             end
+            fs.remove_dir_tree_if_empty(dir.dir_name(target))
             return true
          end
       )
@@ -298,7 +312,7 @@ function delete_version(name, version, quick)
    
    local ok, err = true
    if rock_manifest.bin then
-      ok, err = delete_deployed_file_tree(rock_manifest.bin, cfg.deploy_bin_dir)
+      ok, err = delete_deployed_file_tree(rock_manifest.bin, cfg.deploy_bin_dir, cfg.wrapper_suffix)
    end
    if ok and rock_manifest.lua then
       ok, err = delete_deployed_file_tree(rock_manifest.lua, cfg.deploy_lua_dir)