HTTPS support for downloading and uploading rocks.

HTTPS is set as default for uploading.
HTTP is still default for downloading to keep dependencies low and because the
HTTPS code is new; might be changed in the future.
See #273 and #240.

10 files changed, 180 insertions, 46 deletions

diff --git a/src/bin/luarocks b/src/bin/luarocks
index 3c9e1b74..9101e675 100755
--- a/src/bin/luarocks
+++ b/src/bin/luarocks
@@ -1,5 +1,9 @@
 #!/usr/bin/env lua
 
+-- this should be loaded first.
+local cfg = require("luarocks.cfg")
+cfg.init_package_paths()
+
 local loader = require("luarocks.loader")
 local command_line = require("luarocks.command_line")
 
diff --git a/src/bin/luarocks-admin b/src/bin/luarocks-admin
index e7f49c25..f49db920 100755
--- a/src/bin/luarocks-admin
+++ b/src/bin/luarocks-admin
@@ -1,5 +1,9 @@
 #!/usr/bin/env lua
 
+-- this should be loaded first.
+local cfg = require("luarocks.cfg")
+cfg.init_package_paths()
+
 local loader = require("luarocks.loader")
 local command_line = require("luarocks.command_line")
 
diff --git a/src/luarocks/cfg.lua b/src/luarocks/cfg.lua
index 2e86c3ca..8b5984ef 100644
--- a/src/luarocks/cfg.lua
+++ b/src/luarocks/cfg.lua
@@ -17,6 +17,8 @@ local rawset, next, table, pairs, require, io, os, setmetatable, pcall, ipairs,
 local cfg = {}
 package.loaded["luarocks.cfg"] = cfg
 
+local util = require("luarocks.util")
+
 cfg.lua_version = _VERSION:sub(5)
 local version_suffix = cfg.lua_version:gsub("%.", "_")
 
@@ -167,7 +169,6 @@ if not site_config.LUAROCKS_FORCE_CONFIG then
    end
    if home_overrides then
       home_config_ok = true
-      local util = require("luarocks.util")
       if home_overrides.rocks_trees then
          cfg.rocks_trees = nil
       end
@@ -217,12 +218,13 @@ local defaults = {
         "http://rocks.moonscript.org",
         "https://raw.githubusercontent.com/rocks-moonscript-org/moonrocks-mirror/master/",
         "http://luafr.org/moonrocks/",
+        "http://luarocks.logiceditor.com/rocks",
       }
    },
    disabled_servers = {},
    
    upload = {
-      server = "rocks.moonscript.org",
+      server = "https://rocks.moonscript.org",
       tool_version = "0.0.1",
       api_version = "1",
    },
@@ -527,20 +529,28 @@ local cfg_mt = {
 }
 setmetatable(cfg, cfg_mt)
 
+function cfg.make_paths_from_tree(tree)
+   local lua_path, lib_path, bin_path
+   if type(tree) == "string" then
+      lua_path = tree..cfg.lua_modules_path
+      lib_path = tree..cfg.lib_modules_path
+      bin_path = tree.."/bin"
+   else
+      lua_path = tree.lua_dir or tree.root..cfg.lua_modules_path
+      lib_path = tree.lib_dir or tree.root..cfg.lib_modules_path
+      bin_path = tree.bin_dir or tree.root.."/bin"
+   end
+   return lua_path, lib_path, bin_path
+end
+
 function cfg.package_paths()
    local new_path, new_cpath, new_bin = {}, {}, {}
    for _,tree in ipairs(cfg.rocks_trees) do
-     if type(tree) == "string" then
-        table.insert(new_path, tree..cfg.lua_modules_path.."/?.lua")
-        table.insert(new_path, tree..cfg.lua_modules_path.."/?/init.lua")
-        table.insert(new_cpath, tree..cfg.lib_modules_path.."/?."..cfg.lib_extension)
-        table.insert(new_bin, tree.."/bin")
-     else
-        table.insert(new_path, (tree.lua_dir or tree.root..cfg.lua_modules_path).."/?.lua")
-        table.insert(new_path, (tree.lua_dir or tree.root..cfg.lua_modules_path).."/?/init.lua")
-        table.insert(new_cpath, (tree.lib_dir or tree.root..cfg.lib_modules_path).."/?."..cfg.lib_extension)
-        table.insert(new_bin, (tree.bin_dir or tree.root.."/bin"))
-     end
+      local lua_path, lib_path, bin_path = cfg.make_paths_from_tree(tree)
+      table.insert(new_path, lua_path.."/?.lua")
+      table.insert(new_path, lua_path.."/?/init.lua")
+      table.insert(new_cpath, lib_path.."/?."..cfg.lib_extension)
+      table.insert(new_bin, bin_path)
    end
    if extra_luarocks_module_dir then 
      table.insert(new_path, extra_luarocks_module_dir)
@@ -548,6 +558,12 @@ function cfg.package_paths()
    return table.concat(new_path, ";"), table.concat(new_cpath, ";"), table.concat(new_bin, cfg.export_path_separator)
 end
 
+function cfg.init_package_paths()
+   local lr_path, lr_cpath, lr_bin = cfg.package_paths()
+   package.path = util.remove_path_dupes(package.path .. ";" .. lr_path, ";")
+   package.cpath = util.remove_path_dupes(package.cpath .. ";" .. lr_cpath, ";")
+end
+
 function cfg.which_config()
    return sys_config_file, sys_config_ok, home_config_file, home_config_ok
 end
diff --git a/src/luarocks/fs/lua.lua b/src/luarocks/fs/lua.lua
index f18deac3..b2619051 100644
--- a/src/luarocks/fs/lua.lua
+++ b/src/luarocks/fs/lua.lua
@@ -592,7 +592,7 @@ local function request(url, method, http, loop_control)
             loop_control[url] = true
             return request(location, method, redirect_protocols[protocol], loop_control)
          else
-            return nil, "URL redirected to unsupported protocol - install luasec to get HTTPS support."
+            return nil, "URL redirected to unsupported protocol - install luasec to get HTTPS support.", "https"
          end
       end
       return nil, err
@@ -614,7 +614,7 @@ local function http_request(url, http, cached)
             return true
          end
          if not result then
-            return nil, status
+            return nil, status, headers
          end
       end
    end
@@ -629,10 +629,12 @@ local function http_request(url, http, cached)
       end
       return table.concat(result)
    else
-      return nil, status
+      return nil, status, headers
    end
 end
 
+local downloader_warning = false
+
 --- Download a remote file.
 -- @param url string: URL to be fetched.
 -- @param filename string or nil: this function attempts to detect the
@@ -647,20 +649,27 @@ function fs_lua.download(url, filename, cache)
 
    filename = fs.absolute_name(filename or dir.base_name(url))
    
-   local content, err
+   local content, err, https_err
    if util.starts_with(url, "http:") then
-      content, err = http_request(url, http, cache and filename)
+      content, err, https_err = http_request(url, http, cache and filename)
    elseif util.starts_with(url, "ftp:") then
       content, err = ftp.get(url)
    elseif util.starts_with(url, "https:") then
       if luasec_ok then
          content, err = http_request(url, https, cache and filename)
       else
-         err = "Unsupported protocol - install luasec to get HTTPS support."
+         https_err = true
       end
    else
       err = "Unsupported protocol"
    end
+   if https_err then
+      if not downloader_warning then
+         util.printerr("Warning: falling back to "..cfg.downloader.." - install luasec to get native HTTPS support")
+         downloader_warning = true
+      end
+      return fs.use_downloader(url, filename, cache)
+   end
    if cache and content == true then
       return true, filename
    end
@@ -674,6 +683,12 @@ function fs_lua.download(url, filename, cache)
    return true, filename
 end
 
+else --...if socket_ok == false then
+
+function fs_lua.download(url, filename, cache)
+   return fs.use_downloader(url, filename, cache)
+end
+
 end
 ---------------------------------------------------------------------
 -- MD5 functions
diff --git a/src/luarocks/fs/unix/tools.lua b/src/luarocks/fs/unix/tools.lua
index 69466931..f36e815a 100644
--- a/src/luarocks/fs/unix/tools.lua
+++ b/src/luarocks/fs/unix/tools.lua
@@ -238,7 +238,7 @@ end
 -- filename can be given explicitly as this second argument.
 -- @return (boolean, string): true and the filename on success,
 -- false and the error message on failure.
-function tools.download(url, filename, cache)
+function tools.use_downloader(url, filename, cache)
    assert(type(url) == "string")
    assert(type(filename) == "string" or not filename)
 
diff --git a/src/luarocks/fs/win32/tools.lua b/src/luarocks/fs/win32/tools.lua
index a8b2a1db..f970f36a 100644
--- a/src/luarocks/fs/win32/tools.lua
+++ b/src/luarocks/fs/win32/tools.lua
@@ -248,7 +248,7 @@ end
 -- filename can be given explicitly as this second argument.
 -- @return (boolean, string): true and the filename on success,
 -- false and the error message on failure.
-function tools.download(url, filename, cache)
+function tools.use_downloader(url, filename, cache)
    assert(type(url) == "string")
    assert(type(filename) == "string" or not filename)
 
diff --git a/src/luarocks/loader.lua b/src/luarocks/loader.lua
index 3d36723f..c200d5ec 100644
--- a/src/luarocks/loader.lua
+++ b/src/luarocks/loader.lua
@@ -128,10 +128,11 @@ end
 -- (eg "luasocket"), the version (eg "2.0.2-1"), the path of the rocks tree
 -- (eg "/usr/local"), and the numeric index of the matching entry, so the
 -- filter function can know if the matching module was the first entry or not.
--- @return string, string, string: name of the rock containing the module
--- (eg. "luasocket"), version of the rock (eg. "2.0.2-1"),
--- name of the module (eg. "socket.core", or "socket.core_2_0_2" if file is
--- stored versioned).
+-- @return string, string, string, (string or table):
+-- * name of the rock containing the module (eg. "luasocket")
+-- * version of the rock (eg. "2.0.2-1")
+-- * name of the module (eg. "socket.core", or "socket.core_2_0_2" if file is stored versioned).
+-- * tree of the module (string or table in `rocks_trees` format)
 local function select_module(module, filter_module_name)
    --assert(type(module) == "string")
    --assert(type(filter_module_name) == "function")
@@ -155,7 +156,7 @@ local function select_module(module, filter_module_name)
                return name, version, module_name
             end
             version = deps.parse_version(version)
-            table.insert(providers, {name = name, version = version, module_name = module_name})
+            table.insert(providers, {name = name, version = version, module_name = module_name, tree = tree})
          end
       end
    end
@@ -163,16 +164,17 @@ local function select_module(module, filter_module_name)
    if next(providers) then
       table.sort(providers, sort_versions)
       local first = providers[1]
-      return first.name, first.version.string, first.module_name
+      return first.name, first.version.string, first.module_name, first.tree
    end
 end
 
 --- Search for a module
 -- @param module string: module name (eg. "socket.core")
--- @return string, string, string: name of the rock containing the module
--- (eg. "luasocket"), version of the rock (eg. "2.0.2-1"),
--- name of the module (eg. "socket.core", or "socket.core_2_0_2" if file is
--- stored versioned).
+-- @return string, string, string, (string or table):
+-- * name of the rock containing the module (eg. "luasocket")
+-- * version of the rock (eg. "2.0.2-1")
+-- * name of the module (eg. "socket.core", or "socket.core_2_0_2" if file is stored versioned).
+-- * tree of the module (string or table in `rocks_trees` format)
 local function pick_module(module)
    return
       select_module(module, function(module_name, name, version, tree, i)
diff --git a/src/luarocks/upload/api.lua b/src/luarocks/upload/api.lua
index 818d293e..4ab311f2 100644
--- a/src/luarocks/upload/api.lua
+++ b/src/luarocks/upload/api.lua
@@ -5,6 +5,7 @@ local cfg = require("luarocks.cfg")
 local fs = require("luarocks.fs")
 local util = require("luarocks.util")
 local persist = require("luarocks.persist")
+local multipart = require("luarocks.upload.multipart")
 
 local Api = {}
 
@@ -43,7 +44,7 @@ end
 function Api:check_version()
    if not self._server_tool_version then
       local tool_version = cfg.upload.tool_version
-      local res, err = self:request("http://" .. tostring(self.config.server) .. "/api/tool_version", {
+      local res, err = self:request(tostring(self.config.server) .. "/api/tool_version", {
         current = tool_version
       })
       if not res then
@@ -80,12 +81,11 @@ end
 
 function Api:raw_method(path, ...)
    self:check_version()
-   local url = "http://" .. tostring(self.config.server) .. "/api/" .. tostring(cfg.upload.api_version) .. "/" .. tostring(self.config.key) .. "/" .. tostring(path)
+   local url = tostring(self.config.server) .. "/api/" .. tostring(cfg.upload.api_version) .. "/" .. tostring(self.config.key) .. "/" .. tostring(path)
    return self:request(url, ...)
 end
 
 local function encode_query_string(t, sep)
-   local url = require("socket.url")
    if sep == nil then
       sep = "&"
    end
@@ -95,9 +95,9 @@ local function encode_query_string(t, sep)
       if type(k) == "number" and type(v) == "table" then
          k, v = v[1], v[2]
       end
-      buf[i + 1] = url.escape(k)
+      buf[i + 1] = multipart.url_escape(k)
       buf[i + 2] = "="
-      buf[i + 3] = url.escape(v)
+      buf[i + 3] = multipart.url_escape(v)
       buf[i + 4] = sep
       i = i + 4
    end
@@ -116,12 +116,99 @@ local function require_json()
    return nil
 end
 
+local ltn12_ok, ltn12 = pcall(require, "ltn12")
+if not ltn12_ok then -- If not using LuaSocket and/or LuaSec...
+
+function Api:request(url, params, post_params)
+   local vars = cfg.variables
+   local json_ok, json = require_json()
+   if not json_ok then return nil, "A JSON library is required for this command." end
+   
+   if cfg.downloader == "wget" then
+      local curl_ok = fs.execute_quiet(vars.CURL, "--version")
+      if not curl_ok then
+         return nil, "Missing network helper program 'curl'.\nMake sure 'curl' is installed and available from your path."
+      end
+   end
+
+   if not self.config.key then
+      return nil, "Must have API key before performing any actions."
+   end
+   local body
+   local headers = {}
+   if params and next(params) then
+      url = url .. ("?" .. encode_query_string(params))
+   end
+   local method = "GET"
+   local out 
+   local tmpfile = os.tmpname()
+   if post_params then
+      method = "POST"
+      local curl_cmd = fs.Q(vars.CURL).." -f -k -L --user-agent '"..cfg.user_agent.." via curl' "
+      for k,v in pairs(post_params) do
+         local var = v
+         if type(v) == "table" then
+            var = "@"..v.fname
+         end
+         curl_cmd = curl_cmd .. "--form '"..k.."="..var.."' "
+      end
+      if cfg.connection_timeout and cfg.connection_timeout > 0 then
+        curl_cmd = curl_cmd .. "--connect-timeout "..tonumber(cfg.connection_timeout).." " 
+      end
+      ok = fs.execute_string(curl_cmd..fs.Q(url).." 2> /dev/null 1> "..fs.Q(tmpfile))
+   else
+      local ok, err = fs.download(url, tmpfile)
+      if not ok then
+         return nil, "API failure: " .. tostring(err) .. " - " .. tostring(url)
+      end
+   end
+
+   local tmpfd = io.open(tmpfile)
+   if not tmpfd then
+      os.remove(tmpfile)
+      return nil, "API failure reading temporary file - " .. tostring(url)
+   end
+   out = tmpfd:read("*a")
+   tmpfd:close()
+   os.remove(tmpfile)
+
+   if self.debug then
+      util.printout("[" .. tostring(method) .. " via curl] " .. tostring(url) .. " ... ")
+   end
+
+   return json.decode(out)
+end
+
+else -- use LuaSocket and LuaSec
+
+local warned_luasec = false
+
 function Api:request(url, params, post_params)
-   local http_ok, http = pcall(require, "socket.http")
-   local ltn12_ok, ltn12 = pcall(require, "ltn12")
    local json_ok, json = require_json()
-   if not http_ok then return nil, "LuaSocket is required for this command." end
    if not json_ok then return nil, "A JSON library is required for this command." end
+   local server = tostring(self.config.server)
+   local http_ok, http
+   local via = "luasocket"
+   if server:match("^https://") then
+      http_ok, http = pcall(require, "ssl.https")
+      if http_ok then
+         via = "luasec"
+      else
+         if not warned_luasec then
+            util.printerr("LuaSec is not available; using plain HTTP. Install 'luasec' to enable HTTPS.")
+            warned_luasec = true
+         end
+         http_ok, http = pcall(require, "socket.http")
+         server = server:gsub("^https", "http")
+         url = url:gsub("^https", "http")
+         via = "luasocket"
+      end
+   else
+      http_ok, http = pcall(require, "socket.http")
+   end
+   if not http_ok then
+      return nil, "Failed loading socket library!"
+   end
    
    if not self.config.key then
       return nil, "Must have API key before performing any actions."
@@ -132,7 +219,6 @@ function Api:request(url, params, post_params)
       url = url .. ("?" .. encode_query_string(params))
    end
    if post_params then
-      local multipart = require("luarocks.upload.multipart")
       local boundary
       body, boundary = multipart.encode(post_params)
       headers["Content-length"] = #body
@@ -140,7 +226,7 @@ function Api:request(url, params, post_params)
    end
    local method = post_params and "POST" or "GET"
    if self.debug then
-      util.printout("[" .. tostring(method) .. "] " .. tostring(url) .. " ... ")
+      util.printout("[" .. tostring(method) .. " via "..via.."] " .. tostring(url) .. " ... ")
    end
    local out = {}
    local _, status = http.request({
@@ -159,6 +245,8 @@ function Api:request(url, params, post_params)
    return json.decode(table.concat(out))
 end
 
+end
+
 function api.new(flags, name)
    local self = {}
    setmetatable(self, { __index = Api })
@@ -169,7 +257,7 @@ function api.new(flags, name)
    self.debug = flags["debug"]
    if not self.config.key then
       return nil, "You need an API key to upload rocks.\n" ..
-                  "Navigate to http://"..self.config.server.."/settings to get a key\n" ..
+                  "Navigate to "..self.config.server.."/settings to get a key\n" ..
                   "and then pass it through the --api-key=<key> flag."
    end
    if flags["api-key"] then
diff --git a/src/luarocks/upload/multipart.lua b/src/luarocks/upload/multipart.lua
index 95afe1b3..56776570 100644
--- a/src/luarocks/upload/multipart.lua
+++ b/src/luarocks/upload/multipart.lua
@@ -1,14 +1,19 @@
 
 local multipart = {}
 
-local url = require("socket.url")
-
 local File = {}
 
 local unpack = unpack or table.unpack
 
 math.randomseed(os.time())
 
+-- socket.url.escape(s) from LuaSocket 3.0rc1
+function multipart.url_escape(s)
+   return (string.gsub(s, "([^A-Za-z0-9_])", function(c)
+      return string.format("%%%02x", string.byte(c))
+   end))
+end
+
 function File:mime()
    if not self.mimetype then
       local mimetypes_ok, mimetypes = pcall(require, "mimetypes")
@@ -64,7 +69,7 @@ function multipart.encode(params)
    local chunks = {}
    for _, tuple in ipairs(tuples) do
       local k,v = unpack(tuple)
-      k = url.escape(k)
+      k = multipart.url_escape(k)
       local buffer = { 'Content-Disposition: form-data; name="' .. k .. '"' }
       local content
       if type(v) == "table" and v.__class == File then
diff --git a/src/luarocks/util.lua b/src/luarocks/util.lua
index 1bf6533b..8772883b 100644
--- a/src/luarocks/util.lua
+++ b/src/luarocks/util.lua
@@ -392,7 +392,7 @@ function util.deps_mode_help(program)
 end
 
 function util.see_help(command, program)
-   return "See '"..util.this_program(program or "luarocks")..' help '..command.."'."
+   return "See '"..util.this_program(program or "luarocks")..' help'..(command and " "..command or "").."'."
 end
 
 -- from http://lua-users.org/wiki/SplitJoin