From 6ecf9ab8ee490cc562cdd199f1533a4cf3f1b918 Mon Sep 17 00:00:00 2001 From: Hisham Muhammad Date: Mon, 8 Nov 2010 14:58:18 -0200 Subject: Fixes for native zip packing module. --- src/luarocks/tools/zip.lua | 187 ++++++++++++++++++++++++--------------------- 1 file changed, 100 insertions(+), 87 deletions(-) diff --git a/src/luarocks/tools/zip.lua b/src/luarocks/tools/zip.lua index 18ad967e..caeb4ac9 100644 --- a/src/luarocks/tools/zip.lua +++ b/src/luarocks/tools/zip.lua @@ -1,7 +1,11 @@ module("luarocks.tools.zip", package.seeall) -local zlib = require "zlib" +local zlib = require("zlib") +local fs = require("luarocks.fs") +local dir = require("luarocks.dir") + +local size_buf = 65535 local function number_to_bytestring(number, nbytes) local out = {} @@ -13,34 +17,17 @@ local function number_to_bytestring(number, nbytes) return table.concat(out) end ---- Return a zip handle open for writing. --- @param name filename of the zipfile to be created. --- @return a zip handle, or nil in case of error. -function write_open(name) - - local zf = {} - - zf.ziphandle = io.open(name, "w") - if not zf.ziphandle then - return nil - end - zf.files = {} - zf.in_open_file = false - - return zf -end - --- Begin a new file to be stored inside the zipfile. --- @param zf handle of the zipfile being written. +-- @param self handle of the zipfile being written. -- @param filename filenome of the file to be added to the zipfile. -- @return true if succeeded, nil in case of failure. -function write_open_new_file_in_zip(zf, filename) - if zf.in_open_file then - close_file_in_zip(zf) +local function zipwriter_open_new_file_in_zip(self, filename) + if self.in_open_file then + self:close_file_in_zip() return nil end local lfh = {} - zf.local_file_header = lfh + self.local_file_header = lfh lfh.last_mod_file_time = 0 -- TODO lfh.last_mod_file_date = 0 -- TODO lfh.crc32 = 0 -- initial value @@ -49,40 +36,41 @@ function write_open_new_file_in_zip(zf, filename) lfh.file_name_length = #filename lfh.extra_field_length = 0 lfh.file_name = filename:gsub("\\", "/") - zf.in_open_file = true - zf.data = {} + lfh.external_attr = 0 -- TODO properly store permissions + self.in_open_file = true + self.data = {} return true end --- Write data to the file currently being stored in the zipfile. --- @param zf handle of the zipfile being written. +-- @param self handle of the zipfile being written. -- @param buf string containing data to be written. -- @return true if succeeded, nil in case of failure. -function write_in_file_in_zip(zf, buf) - if not zf.in_open_file then +local function zipwriter_write_file_in_zip(self, buf) + if not self.in_open_file then return nil end - local lfh = zf.local_file_header + local lfh = self.local_file_header local cbuf = zlib.compress(buf):sub(3, -5) lfh.crc32 = zlib.crc32(lfh.crc32, buf) lfh.compressed_size = lfh.compressed_size + #cbuf lfh.uncompressed_size = lfh.uncompressed_size + #buf - table.insert(zf.data, cbuf) + table.insert(self.data, cbuf) return true end --- Complete the writing of a file stored in the zipfile. --- @param zf handle of the zipfile being written. +-- @param self handle of the zipfile being written. -- @return true if succeeded, nil in case of failure. -function write_close_file_in_zip(zf) - local zh = zf.ziphandle +local function zipwriter_close_file_in_zip(self) + local zh = self.ziphandle - if not zf.in_open_file then + if not self.in_open_file then return nil end -- Local file header - local lfh = zf.local_file_header + local lfh = self.local_file_header lfh.offset = zh:seek() zh:write(number_to_bytestring(0x04034b50, 4)) -- signature zh:write(number_to_bytestring(20, 2)) -- version needed to extract: 2.0 @@ -98,7 +86,7 @@ function write_close_file_in_zip(zf) zh:write(lfh.file_name) -- File data - for _, cbuf in ipairs(zf.data) do + for _, cbuf in ipairs(self.data) do zh:write(cbuf) end @@ -107,23 +95,59 @@ function write_close_file_in_zip(zf) zh:write(number_to_bytestring(lfh.compressed_size, 4)) zh:write(number_to_bytestring(lfh.uncompressed_size, 4)) - table.insert(zf.files, lfh) - zf.in_open_file = false + table.insert(self.files, lfh) + self.in_open_file = false return true end +-- @return boolean or (boolean, string): true on success, +-- false and an error message on failure. +local function zipwriter_add(self, file) + local fin + local ok, err = self:open_new_file_in_zip(file) + if not ok then + err = "error in opening "..file.." in zipfile" + else + fin = io.open(file, "rb") + if not fin then + ok = false + err = "error opening "..file.." for reading" + end + end + while ok do + local buf = fin:read(size_buf) + if not buf then + break + end + ok = self:write_file_in_zip(buf) + if not ok then + err = "error in writing "..file.." in the zipfile" + end + end + if fin then + fin:close() + end + if ok then + ok = self:close_file_in_zip() + if not ok then + err = "error in writing "..file.." in the zipfile" + end + end + return ok == true, err +end + --- Complete the writing of the zipfile. --- @param zf handle of the zipfile being written. +-- @param self handle of the zipfile being written. -- @return true if succeeded, nil in case of failure. -function write_close(zf) - local zh = zf.ziphandle +local function zipwriter_close(self) + local zh = self.ziphandle local central_directory_offset = zh:seek() local size_of_central_directory = 0 -- Central directory structure - for _, lfh in ipairs(zf.files) do + for _, lfh in ipairs(self.files) do zh:write(number_to_bytestring(0x02014b50, 4)) -- signature zh:write(number_to_bytestring(3, 2)) -- version made by: UNIX zh:write(number_to_bytestring(20, 2)) -- version needed to extract: 2.0 @@ -139,7 +163,7 @@ function write_close(zf) zh:write(number_to_bytestring(0, 2)) -- file comment length zh:write(number_to_bytestring(0, 2)) -- disk number start zh:write(number_to_bytestring(0, 2)) -- internal file attributes - zh:write(number_to_bytestring(0, 4)) -- external file attributes + zh:write(number_to_bytestring(lfh.external_attr, 4)) -- external file attributes zh:write(number_to_bytestring(lfh.offset, 4)) -- relative offset of local header zh:write(lfh.file_name) size_of_central_directory = size_of_central_directory + 46 + lfh.file_name_length @@ -149,8 +173,8 @@ function write_close(zf) zh:write(number_to_bytestring(0x06054b50, 4)) -- signature zh:write(number_to_bytestring(0, 2)) -- number of this disk zh:write(number_to_bytestring(0, 2)) -- number of disk with start of central directory - zh:write(number_to_bytestring(#zf.files, 2)) -- total number of entries in the central dir on this disk - zh:write(number_to_bytestring(#zf.files, 2)) -- total number of entries in the central dir + zh:write(number_to_bytestring(#self.files, 2)) -- total number of entries in the central dir on this disk + zh:write(number_to_bytestring(#self.files, 2)) -- total number of entries in the central dir zh:write(number_to_bytestring(size_of_central_directory, 4)) zh:write(number_to_bytestring(central_directory_offset, 4)) zh:write(number_to_bytestring(0, 2)) -- zip file comment length @@ -159,40 +183,27 @@ function write_close(zf) return true end --- @return boolean or (boolean, string): true on success, --- false and an error message on failure. -local function add_to_zip(zf, file) - local fin - local ok, err = write_open_new_file_in_zip(zf, file) - if not ok then - err = "error in opening "..file.." in zipfile" - else - fin = io.open(file, "rb") - if not fin then - ok = false - err = "error opening "..file.." for reading" - end - end - while ok do - local buf = fin:read(size_buf) - if not buf then - break - end - ok = write_in_file_in_zip(zf, buf) - if not ok then - err = "error in writing "..file.." in the zipfile" - end - end - if fin then - fin:close() - end - if ok then - ok = write_close_file_in_zip(zf) - if not ok then - err = "error in writing "..file.." in the zipfile" - end +--- Return a zip handle open for writing. +-- @param name filename of the zipfile to be created. +-- @return a zip handle, or nil in case of error. +function new_zipwriter(name) + + local zw = {} + + zw.ziphandle = io.open(name, "w") + if not zw.ziphandle then + return nil end - return ok == true, err + zw.files = {} + zw.in_open_file = false + + zw.add = zipwriter_add + zw.close = zipwriter_close + zw.open_new_file_in_zip = zipwriter_open_new_file_in_zip + zw.write_file_in_zip = zipwriter_write_file_in_zip + zw.close_file_in_zip = zipwriter_close_file_in_zip + + return zw end --- Compress files in a .zip archive. @@ -202,29 +213,31 @@ end -- @return boolean or (boolean, string): true on success, -- false and an error message on failure. function zip(zipfile, ...) - local zf = write_open(filename) - if not zf then - return nil, "error opening "..filename + local zw = new_zipwriter(zipfile) + if not zw then + return nil, "error opening "..zipfile end local ok, err for _, file in pairs({...}) do if fs.is_dir(file) then - for _, file in pairs(fs.find(file)) do - if fs.is_file(file) then - ok, err = add_to_zip(file) + for _, entry in pairs(fs.find(file)) do + local fullname = dir.path(file, entry) + if fs.is_file(fullname) then + ok, err = zw:add(fullname) if not ok then break end end end else - ok, err = add_to_zip(file) + ok, err = zw:add(file) if not ok then break end end end - local ok = write_close(zf) + local ok = zw:close() if not ok then - return false, "error closing "..filename + return false, "error closing "..zipfile end return ok, err end + -- cgit v1.2.3-55-g6feb From 59423a667bdc69d97185886348188db6eb65ae68 Mon Sep 17 00:00:00 2001 From: Hisham Muhammad Date: Mon, 8 Nov 2010 14:59:10 -0200 Subject: Match "*" wildcard so that numbered libraries can be found, when using the cross-platform format in the libraries field from external_dependencies. --- src/luarocks/cfg.lua | 4 ++-- src/luarocks/deps.lua | 15 ++++++++++++--- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/src/luarocks/cfg.lua b/src/luarocks/cfg.lua index d141bdee..0d991849 100644 --- a/src/luarocks/cfg.lua +++ b/src/luarocks/cfg.lua @@ -239,12 +239,12 @@ if detected.unix then defaults.variables.LIBFLAG = "-shared" defaults.external_deps_patterns = { bin = { "?" }, - lib = { "lib?.a", "lib?.so" }, + lib = { "lib?.a", "lib?.so", "lib?.so.*" }, include = { "?.h" } } defaults.runtime_external_deps_patterns = { bin = { "?" }, - lib = { "lib?.so" }, + lib = { "lib?.so", "lib?.so.*" }, include = { "?.h" } } defaults.local_cache = home.."/.cache/luarocks" diff --git a/src/luarocks/deps.lua b/src/luarocks/deps.lua index b10594a3..4f16c2a6 100644 --- a/src/luarocks/deps.lua +++ b/src/luarocks/deps.lua @@ -543,9 +543,18 @@ function check_external_deps(rockspec, mode) if f:match("%.so$") or f:match("%.dylib$") or f:match("%.dll$") then f = f:gsub("%.[^.]+$", "."..cfg.external_lib_extension) end - local testfile = dir.path(dirdata.dir, f) - if fs.exists(testfile) then - found = true + if f:match("%*") then + local replaced = f:gsub("%.", "%%."):gsub("%*", ".*") + for _, entry in ipairs(fs.list_dir(dirdata.dir)) do + if entry:match(replaced) then + found = true + break + end + end + else + found = fs.exists(dir.path(dirdata.dir, f)) + end + if found then break else if failed_file then -- cgit v1.2.3-55-g6feb From b2f4e356b6ede8860b22501aa4ee2578ec9a1a03 Mon Sep 17 00:00:00 2001 From: Hisham Muhammad Date: Tue, 14 Dec 2010 12:12:26 -0200 Subject: Fix sorting and saving of persisted manifest table. --- src/luarocks/manif.lua | 4 ++-- src/luarocks/manif_core.lua | 4 ++-- src/luarocks/persist.lua | 10 +++------- src/luarocks/util.lua | 16 +++++++++++++++- 4 files changed, 22 insertions(+), 12 deletions(-) diff --git a/src/luarocks/manif.lua b/src/luarocks/manif.lua index c677a22c..a7211d70 100644 --- a/src/luarocks/manif.lua +++ b/src/luarocks/manif.lua @@ -304,9 +304,9 @@ local function find_providers(file, root) assert(type(file) == "string") root = root or cfg.root_dir - local manifest = manif_core.load_local_manifest(path.rocks_dir(root)) + local manifest, err = manif_core.load_local_manifest(path.rocks_dir(root)) if not manifest then - return nil, "manifest file is missing. Corrupted local rocks tree?" + return nil, err .. " -- corrupted local rocks tree?" end local deploy_bin = path.deploy_bin_dir(root) local deploy_lua = path.deploy_lua_dir(root) diff --git a/src/luarocks/manif_core.lua b/src/luarocks/manif_core.lua index d1eb4ebb..2e1a9518 100644 --- a/src/luarocks/manif_core.lua +++ b/src/luarocks/manif_core.lua @@ -16,9 +16,9 @@ manifest_cache = {} -- @param file string: The local filename of the manifest file. -- @param repo_url string: The repository identifier. function manifest_loader(file, repo_url, quick) - local manifest = persist.load_into_table(file) + local manifest, err = persist.load_into_table(file) if not manifest then - return nil, "Failed loading manifest for "..repo_url + return nil, "Failed loading manifest for "..repo_url..": "..err end if not quick then local ok, err = type_check.type_check_manifest(manifest) diff --git a/src/luarocks/persist.lua b/src/luarocks/persist.lua index d74a805e..c809b51c 100644 --- a/src/luarocks/persist.lua +++ b/src/luarocks/persist.lua @@ -28,10 +28,6 @@ function load_into_table(filename, tbl) return result end -local function string_sort(a,b) - return tostring(a) < tostring(b) -end - --- Write a table as Lua code representing a table to disk -- (that is, in curly brackets notation). -- This function handles only numbers, strings and tables @@ -43,7 +39,7 @@ local function write_table(out, tbl, level) local sep = "\n" local indent = true local i = 1 - for k, v in util.sortedpairs(tbl, string_sort) do + for k, v in util.sortedpairs(tbl) do out:write(sep) if indent then for n = 1,level do out:write(" ") end @@ -52,7 +48,7 @@ local function write_table(out, tbl, level) indent = true if type(k) == "number" then if k ~= i then - out:write(tostring(k).."=") + out:write('['..tostring(k).."]=") else i = i + 1 end @@ -95,7 +91,7 @@ function save_from_table(filename, tbl) if not out then return nil, "Cannot create file at "..filename end - for k, v in util.sortedpairs(tbl, string_sort) do + for k, v in util.sortedpairs(tbl) do out:write(k.." = ") write_table(out, v, 1) out:write("\n") diff --git a/src/luarocks/util.lua b/src/luarocks/util.lua index 10fc1e36..e0c01421 100644 --- a/src/luarocks/util.lua +++ b/src/luarocks/util.lua @@ -216,6 +216,20 @@ function keys(tbl) return ks end +local function default_sort(a, b) + local ta = type(a) + local tb = type(b) + if ta == "number" and tb == "number" then + return a < b + elseif ta == "number" then + return true + elseif tb == "number" then + return false + else + return tostring(a) < tostring(b) + end +end + -- The iterator function used internally by util.sortedpairs. -- @param tbl table: The table to be iterated. -- @param sort_function function or nil: An optional comparison function @@ -223,7 +237,7 @@ end -- @see sortedpairs local function sortedpairs_iterator(tbl, sort_function) local ks = keys(tbl) - table.sort(ks, sort_function) + table.sort(ks, sort_function or default_sort) for _, k in ipairs(ks) do coroutine.yield(k, tbl[k]) end -- cgit v1.2.3-55-g6feb From a0be4736ef870adc118a7d04931a9edaf7c2eda7 Mon Sep 17 00:00:00 2001 From: Hisham Muhammad Date: Tue, 21 Dec 2010 11:19:15 -0200 Subject: Fix error message on unsupported protocols --- src/luarocks/fs/lua.lua | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/src/luarocks/fs/lua.lua b/src/luarocks/fs/lua.lua index 5b3efd94..79baa07f 100644 --- a/src/luarocks/fs/lua.lua +++ b/src/luarocks/fs/lua.lua @@ -431,19 +431,19 @@ function unzip(zipfile) local files = zipfile:files() local file = files() repeat - if file.filename:sub(#file.filename) == "/" then - fs.make_dir(dir.path(fs.current_dir(), file.filename)) - else - local rf, err = zipfile:open(file.filename) - if not rf then zipfile:close(); return nil, err end - local contents = rf:read("*a") - rf:close() - local wf, err = io.open(dir.path(fs.current_dir(), file.filename), "wb") - if not wf then zipfile:close(); return nil, err end - wf:write(contents) - wf:close() - end - file = files() + if file.filename:sub(#file.filename) == "/" then + fs.make_dir(dir.path(fs.current_dir(), file.filename)) + else + local rf, err = zipfile:open(file.filename) + if not rf then zipfile:close(); return nil, err end + local contents = rf:read("*a") + rf:close() + local wf, err = io.open(dir.path(fs.current_dir(), file.filename), "wb") + if not wf then zipfile:close(); return nil, err end + wf:write(contents) + wf:close() + end + file = files() until not file zipfile:close() return true @@ -494,6 +494,8 @@ function download(url, filename) end elseif util.starts_with(url, "ftp:") then content, err = ftp.get(url) + else + err = "Unsupported protocol in URL: "..url end if not content then return false, "Failed downloading: " .. err -- cgit v1.2.3-55-g6feb From 24af3a8dd304b0e56c1bfe78ba51651dba50b765 Mon Sep 17 00:00:00 2001 From: Hisham Muhammad Date: Tue, 21 Dec 2010 11:34:37 -0200 Subject: Add HTTPS support to native fs.download function, using LuaSec. --- src/luarocks/fetch.lua | 4 +-- src/luarocks/fs/lua.lua | 84 +++++++++++++++++-------------------------------- 2 files changed, 31 insertions(+), 57 deletions(-) diff --git a/src/luarocks/fetch.lua b/src/luarocks/fetch.lua index 25add66a..ff77882e 100644 --- a/src/luarocks/fetch.lua +++ b/src/luarocks/fetch.lua @@ -30,9 +30,9 @@ function fetch_url(url, filename) if protocol == "file" then return fs.absolute_name(pathname) elseif protocol == "http" or protocol == "ftp" or protocol == "https" then - local ok = fs.download(url, filename) + local ok, err = fs.download(url, filename) if not ok then - return nil, "Failed downloading "..url, "network" + return nil, "Failed downloading "..url.." - "..err, "network" end return dir.path(fs.current_dir(), filename or dir.base_name(url)) else diff --git a/src/luarocks/fs/lua.lua b/src/luarocks/fs/lua.lua index 79baa07f..8d29f109 100644 --- a/src/luarocks/fs/lua.lua +++ b/src/luarocks/fs/lua.lua @@ -457,54 +457,28 @@ end if socket_ok then - local ltn12 = require("ltn12") - ---- Download a remote file. --- @param url string: URL to be fetched. --- @param filename string or nil: this function attempts to detect the --- resulting local filename of the remote file as the basename of the URL; --- if that is not correct (due to a redirection, for example), the local --- filename can be given explicitly as this second argument. --- @return boolean: true on success, false on failure. -function download(url, filename) - assert(type(url) == "string") - assert(type(filename) == "string" or not filename) - - filename = dir.path(fs.current_dir(), filename or dir.base_name(url)) - - local content, err - if util.starts_with(url, "http:") then - local proxy = cfg.proxy - local url_arg, proxy_result - if proxy then - proxy_result = {} - url_arg = { url = url, proxy = proxy, sink = ltn12.sink.table(proxy_result) } - else - url_arg = url - end - local res, status, headers, line = http.request(url_arg) - if not res then - err = status - elseif status ~= 200 then - err = line - else - if proxy_result then res = table.concat(proxy_result) end - content = res - end - elseif util.starts_with(url, "ftp:") then - content, err = ftp.get(url) +local luasec_ok, https = pcall(require, "ssl.https") + +local function http_request(url, http) + local proxy = cfg.proxy + local url_arg, proxy_result + if proxy then + proxy_result = {} + url_arg = { url = url, proxy = proxy, sink = ltn12.sink.table(proxy_result) } else - err = "Unsupported protocol in URL: "..url + url_arg = url end - if not content then - return false, "Failed downloading: " .. err + local res, status, headers, line = http.request(url_arg) + if not res then + err = status + elseif status ~= 200 then + err = line + else + if proxy_result then res = table.concat(proxy_result) end + content = res end - local file = io.open(filename, "wb") - if not file then return false end - file:write(content) - file:close() - return true + return content, err end --- Download a remote file. @@ -519,22 +493,23 @@ function download(url, filename) assert(type(filename) == "string" or not filename) filename = dir.path(fs.current_dir(), filename or dir.base_name(url)) - + local content, err if util.starts_with(url, "http:") then - local res, status, headers, line = http.request(url) - if not res then - err = status - elseif status ~= 200 then - err = line - else - content = res - end + content, err = http_request(url, http) elseif util.starts_with(url, "ftp:") then content, err = ftp.get(url) + elseif util.starts_with(url, "https:") then + if luasec_ok then + content, err = http_request(url, https) + else + err = "Unsupported protocol - install luasec to get HTTPS support." + end + else + err = "Unsupported protocol" end if not content then - return false, "Failed downloading: " .. err + return false, err end local file = io.open(filename, "wb") if not file then return false end @@ -543,7 +518,6 @@ function download(url, filename) return true end - end --------------------------------------------------------------------- -- MD5 functions -- cgit v1.2.3-55-g6feb