From a9c43e2efbfe05f5a71b68235d7115bbb06f407a Mon Sep 17 00:00:00 2001 From: V1K1NGbg Date: Thu, 22 Aug 2024 17:48:57 -0300 Subject: Teal: convert luarocks.fetch --- src/luarocks/fetch.lua | 610 ------------------------------------------------ src/luarocks/fetch.tl | 615 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 615 insertions(+), 610 deletions(-) delete mode 100644 src/luarocks/fetch.lua create mode 100644 src/luarocks/fetch.tl (limited to 'src') diff --git a/src/luarocks/fetch.lua b/src/luarocks/fetch.lua deleted file mode 100644 index 193e5e39..00000000 --- a/src/luarocks/fetch.lua +++ /dev/null @@ -1,610 +0,0 @@ - ---- Functions related to fetching and loading local and remote files. -local fetch = {} - -local fs = require("luarocks.fs") -local dir = require("luarocks.dir") -local rockspecs = require("luarocks.rockspecs") -local signing = require("luarocks.signing") -local persist = require("luarocks.persist") -local util = require("luarocks.util") -local cfg = require("luarocks.core.cfg") - - ---- Fetch a local or remote file, using a local cache directory. --- Make a remote or local URL/pathname local, fetching the file if necessary. --- Other "fetch" and "load" functions use this function to obtain files. --- If a local pathname is given, it is returned as a result. --- @param url string: a local pathname or a remote URL. --- @param mirroring string: mirroring mode. --- If set to "no_mirror", then rocks_servers mirror configuration is not used. --- @return (string, nil, nil, boolean) or (nil, string, [string]): --- in case of success: --- * the absolute local pathname for the fetched file --- * nil --- * nil --- * `true` if the file was fetched from cache --- in case of failure: --- * nil --- * an error message --- * an optional error code. -function fetch.fetch_caching(url, mirroring) - local repo_url, filename = url:match("^(.*)/([^/]+)$") - local name = repo_url:gsub("[/:]","_") - local cache_dir = dir.path(cfg.local_cache, name) - local ok = fs.make_dir(cache_dir) - - local cachefile = dir.path(cache_dir, filename) - local checkfile = cachefile .. ".check" - - if (fs.file_age(checkfile) < 10 or - cfg.aggressive_cache and (not name:match("^manifest"))) and fs.exists(cachefile) - then - return cachefile, nil, nil, true - end - - local lock, errlock - if ok then - lock, errlock = fs.lock_access(cache_dir) - end - - if not (ok and lock) then - cfg.local_cache = fs.make_temp_dir("local_cache") - if not cfg.local_cache then - return nil, "Failed creating temporary local_cache directory" - end - cache_dir = dir.path(cfg.local_cache, name) - ok = fs.make_dir(cache_dir) - if not ok then - return nil, "Failed creating temporary cache directory "..cache_dir - end - lock = fs.lock_access(cache_dir) - end - - local file, err, errcode, from_cache = fetch.fetch_url(url, cachefile, true, mirroring) - if not file then - fs.unlock_access(lock) - return nil, err or "Failed downloading "..url, errcode - end - - local fd, err = io.open(checkfile, "wb") - if err then - fs.unlock_access(lock) - return nil, err - end - fd:write("!") - fd:close() - - fs.unlock_access(lock) - return file, nil, nil, from_cache -end - -local function ensure_trailing_slash(url) - return (url:gsub("/*$", "/")) -end - -local function is_url_relative_to_rocks_servers(url, servers) - for _, item in ipairs(servers) do - if type(item) == "table" then - for i, s in ipairs(item) do - local base = ensure_trailing_slash(s) - if string.find(url, base, 1, true) == 1 then - return i, url:sub(#base + 1), item - end - end - end - end -end - -local function download_with_mirrors(url, filename, cache, servers) - local idx, rest, mirrors = is_url_relative_to_rocks_servers(url, servers) - - if not idx then - -- URL is not from a rock server - return fs.download(url, filename, cache) - end - - -- URL is from a rock server: try to download it falling back to mirrors. - local err = "\n" - for i = idx, #mirrors do - local try_url = ensure_trailing_slash(mirrors[i]) .. rest - if i > idx then - util.warning("Failed downloading. Attempting mirror at " .. try_url) - end - local ok, name, from_cache = fs.download(try_url, filename, cache) - if ok then - return ok, name, from_cache - else - err = err .. name .. "\n" - end - end - - return nil, err, "network" -end - ---- Fetch a local or remote file. --- Make a remote or local URL/pathname local, fetching the file if necessary. --- Other "fetch" and "load" functions use this function to obtain files. --- If a local pathname is given, it is returned as a result. --- @param url string: a local pathname or a remote URL. --- @param filename string or nil: this function attempts to detect the --- resulting local filename of the remote file as the basename of the URL; --- if that is not correct (due to a redirection, for example), the local --- filename can be given explicitly as this second argument. --- @param cache boolean: compare remote timestamps via HTTP HEAD prior to --- re-downloading the file. --- @param mirroring string: mirroring mode. --- If set to "no_mirror", then rocks_servers mirror configuration is not used. --- @return (string, nil, nil, boolean) or (nil, string, [string]): --- in case of success: --- * the absolute local pathname for the fetched file --- * nil --- * nil --- * `true` if the file was fetched from cache --- in case of failure: --- * nil --- * an error message --- * an optional error code. -function fetch.fetch_url(url, filename, cache, mirroring) - assert(type(url) == "string") - assert(type(filename) == "string" or not filename) - - local protocol, pathname = dir.split_url(url) - if protocol == "file" then - local fullname = fs.absolute_name(pathname) - if not fs.exists(fullname) then - local hint = (not pathname:match("^/")) - and (" - note that given path in rockspec is not absolute: " .. url) - or "" - return nil, "Local file not found: " .. fullname .. hint - end - filename = filename or dir.base_name(pathname) - local dstname = fs.absolute_name(dir.path(".", filename)) - local ok, err - if fullname == dstname then - ok = true - else - ok, err = fs.copy(fullname, dstname) - end - if ok then - return dstname - else - return nil, "Failed copying local file " .. fullname .. " to " .. dstname .. ": " .. err - end - elseif dir.is_basic_protocol(protocol) then - local ok, name, from_cache - if mirroring ~= "no_mirror" then - ok, name, from_cache = download_with_mirrors(url, filename, cache, cfg.rocks_servers) - else - ok, name, from_cache = fs.download(url, filename, cache) - end - if not ok then - return nil, "Failed downloading "..url..(name and " - "..name or ""), from_cache - end - return name, nil, nil, from_cache - else - return nil, "Unsupported protocol "..protocol - end -end - ---- For remote URLs, create a temporary directory and download URL inside it. --- This temporary directory will be deleted on program termination. --- For local URLs, just return the local pathname and its directory. --- @param url string: URL to be downloaded --- @param tmpname string: name pattern to use for avoiding conflicts --- when creating temporary directory. --- @param filename string or nil: local filename of URL to be downloaded, --- in case it can't be inferred from the URL. --- @return (string, string) or (nil, string, [string]): absolute local pathname of --- the fetched file and temporary directory name; or nil and an error message --- followed by an optional error code -function fetch.fetch_url_at_temp_dir(url, tmpname, filename, cache) - assert(type(url) == "string") - assert(type(tmpname) == "string") - assert(type(filename) == "string" or not filename) - filename = filename or dir.base_name(url) - - local protocol, pathname = dir.split_url(url) - if protocol == "file" then - if fs.exists(pathname) then - return pathname, dir.dir_name(fs.absolute_name(pathname)) - else - return nil, "File not found: "..pathname - end - else - local temp_dir, err = fs.make_temp_dir(tmpname) - if not temp_dir then - return nil, "Failed creating temporary directory "..tmpname..": "..err - end - util.schedule_function(fs.delete, temp_dir) - local ok, err = fs.change_dir(temp_dir) - if not ok then return nil, err end - - local file, err, errcode - - if cache then - local cachefile - cachefile, err, errcode = fetch.fetch_caching(url) - - if cachefile then - file = dir.path(temp_dir, filename) - fs.copy(cachefile, file) - end - end - - if not file then - file, err, errcode = fetch.fetch_url(url, filename, cache) - end - - fs.pop_dir() - if not file then - return nil, "Error fetching file: "..err, errcode - end - - return file, temp_dir - end -end - --- Determine base directory of a fetched URL by extracting its --- archive and looking for a directory in the root. --- @param file string: absolute local pathname of the fetched file --- @param temp_dir string: temporary directory in which URL was fetched. --- @param src_url string: URL to use when inferring base directory. --- @param src_dir string or nil: expected base directory (inferred --- from src_url if not given). --- @return (string, string) or (string, nil) or (nil, string): --- The inferred base directory and the one actually found (which may --- be nil if not found), or nil followed by an error message. --- The inferred dir is returned first to avoid confusion with errors, --- because it is never nil. -function fetch.find_base_dir(file, temp_dir, src_url, src_dir) - local ok, err = fs.change_dir(temp_dir) - if not ok then return nil, err end - fs.unpack_archive(file) - - if not src_dir then - local rockspec = { - source = { - file = file, - dir = src_dir, - url = src_url, - } - } - ok, err = fetch.find_rockspec_source_dir(rockspec, ".") - if ok then - src_dir = rockspec.source.dir - end - end - - local inferred_dir = src_dir or dir.deduce_base_dir(src_url) - local found_dir = nil - if fs.exists(inferred_dir) then - found_dir = inferred_dir - else - util.printerr("Directory "..inferred_dir.." not found") - local files = fs.list_dir() - if files then - table.sort(files) - for i,filename in ipairs(files) do - if fs.is_dir(filename) then - util.printerr("Found "..filename) - found_dir = filename - break - end - end - end - end - fs.pop_dir() - return inferred_dir, found_dir -end - -local function fetch_and_verify_signature_for(url, filename, tmpdir) - local sig_url = signing.signature_url(url) - local sig_file, err, errcode = fetch.fetch_url_at_temp_dir(sig_url, tmpdir) - if not sig_file then - return nil, "Could not fetch signature file for verification: " .. err, errcode - end - - local ok, err = signing.verify_signature(filename, sig_file) - if not ok then - return nil, "Failed signature verification: " .. err - end - - return fs.absolute_name(sig_file) -end - ---- Obtain a rock and unpack it. --- If a directory is not given, a temporary directory will be created, --- which will be deleted on program termination. --- @param rock_file string: URL or filename of the rock. --- @param dest string or nil: if given, directory will be used as --- a permanent destination. --- @param verify boolean: if true, download and verify signature for rockspec --- @return string or (nil, string, [string]): the directory containing the contents --- of the unpacked rock. -function fetch.fetch_and_unpack_rock(url, dest, verify) - assert(type(url) == "string") - assert(type(dest) == "string" or not dest) - - local name = dir.base_name(url):match("(.*)%.[^.]*%.rock") - local tmpname = "luarocks-rock-" .. name - - local rock_file, err, errcode = fetch.fetch_url_at_temp_dir(url, tmpname, nil, true) - if not rock_file then - return nil, "Could not fetch rock file: " .. err, errcode - end - - local sig_file - if verify then - sig_file, err = fetch_and_verify_signature_for(url, rock_file, tmpname) - if err then - return nil, err - end - end - - rock_file = fs.absolute_name(rock_file) - - local unpack_dir - if dest then - unpack_dir = dest - local ok, err = fs.make_dir(unpack_dir) - if not ok then - return nil, "Failed unpacking rock file: " .. err - end - else - unpack_dir, err = fs.make_temp_dir(name) - if not unpack_dir then - return nil, "Failed creating temporary dir: " .. err - end - end - if not dest then - util.schedule_function(fs.delete, unpack_dir) - end - local ok, err = fs.change_dir(unpack_dir) - if not ok then return nil, err end - ok, err = fs.unzip(rock_file) - if not ok then - return nil, "Failed unpacking rock file: " .. rock_file .. ": " .. err - end - if sig_file then - ok, err = fs.copy(sig_file, ".") - if not ok then - return nil, "Failed copying signature file" - end - end - fs.pop_dir() - return unpack_dir -end - ---- Back-end function that actually loads the local rockspec. --- Performs some validation and postprocessing of the rockspec contents. --- @param rel_filename string: The local filename of the rockspec file. --- @param quick boolean: if true, skips some steps when loading --- rockspec. --- @return table or (nil, string): A table representing the rockspec --- or nil followed by an error message. -function fetch.load_local_rockspec(rel_filename, quick) - assert(type(rel_filename) == "string") - local abs_filename = fs.absolute_name(rel_filename) - - local basename = dir.base_name(abs_filename) - if basename ~= "rockspec" then - if not basename:match("(.*)%-[^-]*%-[0-9]*") then - return nil, "Expected filename in format 'name-version-revision.rockspec'." - end - end - - local tbl, err = persist.load_into_table(abs_filename) - if not tbl then - return nil, "Could not load rockspec file "..abs_filename.." ("..err..")" - end - - local rockspec, err = rockspecs.from_persisted_table(abs_filename, tbl, err, quick) - if not rockspec then - return nil, abs_filename .. ": " .. err - end - - local name_version = rockspec.package:lower() .. "-" .. rockspec.version - if basename ~= "rockspec" and basename ~= name_version .. ".rockspec" then - return nil, "Inconsistency between rockspec filename ("..basename..") and its contents ("..name_version..".rockspec)." - end - - return rockspec -end - ---- Load a local or remote rockspec into a table. --- This is the entry point for the LuaRocks tools. --- Only the LuaRocks runtime loader should use --- load_local_rockspec directly. --- @param filename string: Local or remote filename of a rockspec. --- @param location string or nil: Where to download. If not given, --- a temporary dir is created. --- @param verify boolean: if true, download and verify signature for rockspec --- @return table or (nil, string, [string]): A table representing the rockspec --- or nil followed by an error message and optional error code. -function fetch.load_rockspec(url, location, verify) - assert(type(url) == "string") - - local name - local basename = dir.base_name(url) - if basename == "rockspec" then - name = "rockspec" - else - name = basename:match("(.*)%.rockspec") - if not name then - return nil, "Filename '"..url.."' does not look like a rockspec." - end - end - - local tmpname = "luarocks-rockspec-"..name - local filename, err, errcode - if location then - local ok, err = fs.change_dir(location) - if not ok then return nil, err end - filename, err = fetch.fetch_url(url) - fs.pop_dir() - else - filename, err, errcode = fetch.fetch_url_at_temp_dir(url, tmpname, nil, true) - end - if not filename then - return nil, err, errcode - end - - if verify then - local _, err = fetch_and_verify_signature_for(url, filename, tmpname) - if err then - return nil, err - end - end - - return fetch.load_local_rockspec(filename) -end - ---- Download sources for building a rock using the basic URL downloader. --- @param rockspec table: The rockspec table --- @param extract boolean: Whether to extract the sources from --- the fetched source tarball or not. --- @param dest_dir string or nil: If set, will extract to the given directory; --- if not given, will extract to a temporary directory. --- @return (string, string) or (nil, string, [string]): The absolute pathname of --- the fetched source tarball and the temporary directory created to --- store it; or nil and an error message and optional error code. -function fetch.get_sources(rockspec, extract, dest_dir) - assert(rockspec:type() == "rockspec") - assert(type(extract) == "boolean") - assert(type(dest_dir) == "string" or not dest_dir) - - local url = rockspec.source.url - local name = rockspec.name.."-"..rockspec.version - local filename = rockspec.source.file - local source_file, store_dir - local ok, err, errcode - if dest_dir then - ok, err = fs.change_dir(dest_dir) - if not ok then return nil, err, "dest_dir" end - source_file, err, errcode = fetch.fetch_url(url, filename) - fs.pop_dir() - store_dir = dest_dir - else - source_file, store_dir, errcode = fetch.fetch_url_at_temp_dir(url, "luarocks-source-"..name, filename) - end - if not source_file then - return nil, err or store_dir, errcode - end - if rockspec.source.md5 then - if not fs.check_md5(source_file, rockspec.source.md5) then - return nil, "MD5 check for "..filename.." has failed.", "md5" - end - end - if extract then - local ok, err = fs.change_dir(store_dir) - if not ok then return nil, err end - ok, err = fs.unpack_archive(rockspec.source.file) - if not ok then return nil, err end - ok, err = fetch.find_rockspec_source_dir(rockspec, ".") - if not ok then return nil, err end - fs.pop_dir() - end - return source_file, store_dir -end - -function fetch.find_rockspec_source_dir(rockspec, store_dir) - local ok, err = fs.change_dir(store_dir) - if not ok then return nil, err end - - local file_count, dir_count, found_dir = 0, 0, 0 - - if rockspec.source.dir and fs.exists(rockspec.source.dir) then - ok, err = true, nil - elseif rockspec.source.file and rockspec.source.dir then - ok, err = nil, "Directory "..rockspec.source.dir.." not found inside archive "..rockspec.source.file - elseif not rockspec.source.dir_set then -- and rockspec:format_is_at_least("3.0") then - - local name = dir.base_name(rockspec.source.file or rockspec.source.url or "") - - if name:match("%.lua$") or name:match("%.c$") then - if fs.is_file(name) then - rockspec.source.dir = "." - ok, err = true, nil - end - end - - if not rockspec.source.dir then - for file in fs.dir() do - file_count = file_count + 1 - if fs.is_dir(file) then - dir_count = dir_count + 1 - found_dir = file - end - end - - if dir_count == 1 then - rockspec.source.dir = found_dir - ok, err = true, nil - else - ok, err = nil, "Could not determine source directory from rock contents (" .. tostring(file_count).." file(s), "..tostring(dir_count).." dir(s))" - end - end - else - ok, err = nil, "Could not determine source directory, please set source.dir in rockspec." - end - - fs.pop_dir() - - assert(rockspec.source.dir or not ok) - return ok, err -end - ---- Download sources for building a rock, calling the appropriate protocol method. --- @param rockspec table: The rockspec table --- @param extract boolean: When downloading compressed formats, whether to extract --- the sources from the fetched archive or not. --- @param dest_dir string or nil: If set, will extract to the given directory. --- if not given, will extract to a temporary directory. --- @return (string, string) or (nil, string): The absolute pathname of --- the fetched source tarball and the temporary directory created to --- store it; or nil and an error message. -function fetch.fetch_sources(rockspec, extract, dest_dir) - assert(rockspec:type() == "rockspec") - assert(type(extract) == "boolean") - assert(type(dest_dir) == "string" or not dest_dir) - - -- auto-convert git://github.com URLs to use git+https - -- see https://github.blog/2021-09-01-improving-git-protocol-security-github/ - if rockspec.source.url:match("^git://github%.com/") - or rockspec.source.url:match("^git://www%.github%.com/") then - rockspec.source.url = rockspec.source.url:gsub("^git://", "git+https://") - rockspec.source.protocol = "git+https" - end - - local protocol = rockspec.source.protocol - local ok, err, proto - if dir.is_basic_protocol(protocol) then - proto = fetch - else - ok, proto = pcall(require, "luarocks.fetch."..protocol:gsub("[+-]", "_")) - if not ok then - return nil, "Unknown protocol "..protocol - end - end - - if cfg.only_sources_from - and rockspec.source.pathname - and #rockspec.source.pathname > 0 then - if #cfg.only_sources_from == 0 then - return nil, "Can't download "..rockspec.source.url.." -- download from remote servers disabled" - elseif rockspec.source.pathname:find(cfg.only_sources_from, 1, true) ~= 1 then - return nil, "Can't download "..rockspec.source.url.." -- only downloading from "..cfg.only_sources_from - end - end - - local source_file, store_dir = proto.get_sources(rockspec, extract, dest_dir) - if not source_file then return nil, store_dir end - - ok, err = fetch.find_rockspec_source_dir(rockspec, store_dir) - if not ok then return nil, err, "source.dir", source_file, store_dir end - - return source_file, store_dir -end - -return fetch diff --git a/src/luarocks/fetch.tl b/src/luarocks/fetch.tl new file mode 100644 index 00000000..222b2194 --- /dev/null +++ b/src/luarocks/fetch.tl @@ -0,0 +1,615 @@ + +--- Functions related to fetching and loading local and remote files. +local record fetch + fetch_caching: function(string, ?string): string, string, string, boolean + fetch_url: function(string, ?string, ?boolean, ?string): string, string, string, boolean + fetch_url_at_temp_dir: function(string, string, ?string, ?boolean): string, string, string + find_base_dir: function(string, string, string, ?string): string, string + fetch_and_unpack_rock: function(string, ?string, ?boolean): string, string, string + load_local_rockspec: function(string, ?boolean): Rockspec, string + load_rockspec: function(string, ?string, ?boolean): Rockspec, string, string + find_rockspec_source_dir: function(Rockspec, string): boolean, string + fetch_sources: function(Rockspec, boolean, ?string): string, string, string, string, string + record Fetch + get_sources: function(Rockspec, boolean, ?string): string, string + end +end + +local fs = require("luarocks.fs") +local dir = require("luarocks.dir") +local rockspecs = require("luarocks.rockspecs") +local signing = require("luarocks.signing") +local persist = require("luarocks.persist") +local util = require("luarocks.util") +local cfg = require("luarocks.core.cfg") + +local type Fetch = fetch.Fetch +local type Lock = fs.Lock +local type Rockspec = require("luarocks.core.types.rockspec").Rockspec + + +--- Fetch a local or remote file, using a local cache directory. +-- Make a remote or local URL/pathname local, fetching the file if necessary. +-- Other "fetch" and "load" functions use this function to obtain files. +-- If a local pathname is given, it is returned as a result. +-- @param url string: a local pathname or a remote URL. +-- @param mirroring string: mirroring mode. +-- If set to "no_mirror", then rocks_servers mirror configuration is not used. +-- @return (string, nil, nil, boolean) or (nil, string, [string]): +-- in case of success: +-- * the absolute local pathname for the fetched file +-- * nil +-- * nil +-- * `true` if the file was fetched from cache +-- in case of failure: +-- * nil +-- * an error message +-- * an optional error code. +function fetch.fetch_caching(url: string, mirroring?: string): string, string, string, boolean + local repo_url, filename = url:match("^(.*)/([^/]+)$") + local name = repo_url:gsub("[/:]","_") + local cache_dir = dir.path(cfg.local_cache, name) + local ok = fs.make_dir(cache_dir) + + local cachefile = dir.path(cache_dir, filename) + local checkfile = cachefile .. ".check" + + if (fs.file_age(checkfile) < 10 or + cfg.aggressive_cache and (not name:match("^manifest"))) and fs.exists(cachefile) + then + return cachefile, nil, nil, true + end + + local lock, errlock: Lock, string + if ok then + lock, errlock = fs.lock_access(cache_dir) + end + + if not (ok and lock) then + cfg.local_cache = fs.make_temp_dir("local_cache") + if not cfg.local_cache then + return nil, "Failed creating temporary local_cache directory" + end + cache_dir = dir.path(cfg.local_cache, name) + ok = fs.make_dir(cache_dir) + if not ok then + return nil, "Failed creating temporary cache directory "..cache_dir + end + lock = fs.lock_access(cache_dir) + end + + local file, err, errcode, from_cache = fetch.fetch_url(url, cachefile, true, mirroring) + if not file then + fs.unlock_access(lock) + return nil, err or "Failed downloading "..url, errcode + end + + local fd, erropen = io.open(checkfile, "wb") + if erropen then + fs.unlock_access(lock) + return nil, erropen + end + fd:write("!") + fd:close() + + fs.unlock_access(lock) + return file, nil, nil, from_cache +end + +local function ensure_trailing_slash(url: string): string + return (url:gsub("/*$", "/")) +end + +local function is_url_relative_to_rocks_servers(url: string, servers: {{string} | string}): integer, string, {string} + for _, item in ipairs(servers) do + if item is {string} then + for i, s in ipairs(item) do + local base = ensure_trailing_slash(s) + if string.find(url, base, 1, true) == 1 then + return i, url:sub(#base + 1), item + end + end + end + end +end + +local function download_with_mirrors(url: string, filename: string, cache: boolean, servers: {{string} | string}): string, string, string, boolean + local idx, rest, mirrors = is_url_relative_to_rocks_servers(url, servers) + + if not idx then + -- URL is not from a rock server + return fs.download(url, filename, cache) + end + + -- URL is from a rock server: try to download it falling back to mirrors. + local err = "\n" + for i = idx, #mirrors do + local try_url = ensure_trailing_slash(mirrors[i]) .. rest + if i > idx then + util.warning("Failed downloading. Attempting mirror at " .. try_url) + end + local name, _, _, from_cache = fs.download(try_url, filename, cache) + if name then + return name, nil, nil, from_cache + else + err = err .. name .. "\n" + end + end + + return nil, err, "network" +end + +--- Fetch a local or remote file. +-- Make a remote or local URL/pathname local, fetching the file if necessary. +-- Other "fetch" and "load" functions use this function to obtain files. +-- If a local pathname is given, it is returned as a result. +-- @param url string: a local pathname or a remote URL. +-- @param filename string or nil: this function attempts to detect the +-- resulting local filename of the remote file as the basename of the URL; +-- if that is not correct (due to a redirection, for example), the local +-- filename can be given explicitly as this second argument. +-- @param cache boolean: compare remote timestamps via HTTP HEAD prior to +-- re-downloading the file. +-- @param mirroring string: mirroring mode. +-- If set to "no_mirror", then rocks_servers mirror configuration is not used. +-- @return (string, nil, nil, boolean) or (nil, string, [string]): +-- in case of success: +-- * the absolute local pathname for the fetched file +-- * nil +-- * nil +-- * `true` if the file was fetched from cache +-- in case of failure: +-- * nil +-- * an error message +-- * an optional error code. +function fetch.fetch_url(url: string, filename?: string, cache?: boolean, mirroring?: string): string, string, string, boolean + + local protocol, pathname = dir.split_url(url) + if protocol == "file" then + local fullname = fs.absolute_name(pathname) + if not fs.exists(fullname) then + local hint = (not pathname:match("^/")) + and (" - note that given path in rockspec is not absolute: " .. url) + or "" + return nil, "Local file not found: " .. fullname .. hint + end + filename = filename or dir.base_name(pathname) + local dstname = fs.absolute_name(dir.path(".", filename)) + local ok, err: boolean, string + if fullname == dstname then + ok = true + else + ok, err = fs.copy(fullname, dstname) + end + if ok then + return dstname + else + return nil, "Failed copying local file " .. fullname .. " to " .. dstname .. ": " .. err + end + elseif dir.is_basic_protocol(protocol) then + local name, err, err_code, from_cache: string, string, string, boolean + if mirroring ~= "no_mirror" then + name, err, err_code, from_cache = download_with_mirrors(url, filename, cache, cfg.rocks_servers) + else + name, err, err_code, from_cache = fs.download(url, filename, cache) + end + if not name then + return nil, "Failed downloading "..url..(err and " - "..err or ""), err_code + end + return name, nil, nil, from_cache --! string or boolean + else + return nil, "Unsupported protocol "..protocol + end +end + +--- For remote URLs, create a temporary directory and download URL inside it. +-- This temporary directory will be deleted on program termination. +-- For local URLs, just return the local pathname and its directory. +-- @param url string: URL to be downloaded +-- @param tmpname string: name pattern to use for avoiding conflicts +-- when creating temporary directory. +-- @param filename string or nil: local filename of URL to be downloaded, +-- in case it can't be inferred from the URL. +-- @return (string, string) or (nil, string, [string]): absolute local pathname of +-- the fetched file and temporary directory name; or nil and an error message +-- followed by an optional error code +function fetch.fetch_url_at_temp_dir(url: string, tmpname: string, filename?: string, cache?: boolean): string, string, string + filename = filename or dir.base_name(url) + + local protocol, pathname = dir.split_url(url) + if protocol == "file" then + if fs.exists(pathname) then + return pathname, dir.dir_name(fs.absolute_name(pathname)) + else + return nil, "File not found: "..pathname + end + else + local temp_dir, errmake = fs.make_temp_dir(tmpname) + if not temp_dir then + return nil, "Failed creating temporary directory "..tmpname..": "..errmake + end + util.schedule_function(fs.delete, temp_dir) + local ok, errchange = fs.change_dir(temp_dir) + if not ok then return nil, errchange end + + local file, err, errcode: string, string, string + + if cache then + local cachefile: string + cachefile, err, errcode = fetch.fetch_caching(url) + + if cachefile then + file = dir.path(temp_dir, filename) + fs.copy(cachefile, file) + end + end + + if not file then + file, err, errcode = fetch.fetch_url(url, filename, cache) + end + + fs.pop_dir() + if not file then + return nil, "Error fetching file: "..err, errcode + end + + return file, temp_dir + end +end + +-- Determine base directory of a fetched URL by extracting its +-- archive and looking for a directory in the root. +-- @param file string: absolute local pathname of the fetched file +-- @param temp_dir string: temporary directory in which URL was fetched. +-- @param src_url string: URL to use when inferring base directory. +-- @param src_dir string or nil: expected base directory (inferred +-- from src_url if not given). +-- @return (string, string) or (string, nil) or (nil, string): +-- The inferred base directory and the one actually found (which may +-- be nil if not found), or nil followed by an error message. +-- The inferred dir is returned first to avoid confusion with errors, +-- because it is never nil. +function fetch.find_base_dir(file: string, temp_dir: string, src_url: string, src_dir?: string): string, string + local ok, err = fs.change_dir(temp_dir) + if not ok then return nil, err end + fs.unpack_archive(file) + + if not src_dir then + local rockspec = { + source = { + file = file, + dir = src_dir, + url = src_url, + } + } + ok, err = fetch.find_rockspec_source_dir(rockspec, ".") + if ok then + src_dir = rockspec.source.dir + end + end + + local inferred_dir = src_dir or dir.deduce_base_dir(src_url) + local found_dir: string = nil + if fs.exists(inferred_dir) then + found_dir = inferred_dir + else + util.printerr("Directory "..inferred_dir.." not found") + local files = fs.list_dir() + if files then + table.sort(files) + for _,filename in ipairs(files) do + if fs.is_dir(filename) then + util.printerr("Found "..filename) + found_dir = filename + break + end + end + end + end + fs.pop_dir() + return inferred_dir, found_dir +end + + +local function fetch_and_verify_signature_for(url: string, filename: string, tmpdir: string): string, string, string + local sig_url = signing.signature_url(url) + local sig_file, errfetch, errcode = fetch.fetch_url_at_temp_dir(sig_url, tmpdir) + if not sig_file then + return nil, "Could not fetch signature file for verification: " .. errfetch, errcode + end + + local ok, err = signing.verify_signature(filename, sig_file) + if not ok then + return nil, "Failed signature verification: " .. err + end + + return fs.absolute_name(sig_file) +end + +--- Obtain a rock and unpack it. +-- If a directory is not given, a temporary directory will be created, +-- which will be deleted on program termination. +-- @param rock_file string: URL or filename of the rock. +-- @param dest string or nil: if given, directory will be used as +-- a permanent destination. +-- @param verify boolean: if true, download and verify signature for rockspec +-- @return string or (nil, string, [string]): the directory containing the contents +-- of the unpacked rock. +function fetch.fetch_and_unpack_rock(url: string, dest?: string, verify?: boolean): string, string, string + + local name = dir.base_name(url):match("(.*)%.[^.]*%.rock") + local tmpname = "luarocks-rock-" .. name + + local rock_file, err, errcode = fetch.fetch_url_at_temp_dir(url, tmpname, nil, true) + if not rock_file then + return nil, "Could not fetch rock file: " .. err, errcode + end + + local sig_file: string + if verify then + sig_file, err = fetch_and_verify_signature_for(url, rock_file, tmpname) + if err then + return nil, err + end + end + + rock_file = fs.absolute_name(rock_file) + + local unpack_dir: string + if dest then + unpack_dir = dest + local ok, errmake = fs.make_dir(unpack_dir) + if not ok then + return nil, "Failed unpacking rock file: " .. errmake + end + else + unpack_dir, err = fs.make_temp_dir(name) + if not unpack_dir then + return nil, "Failed creating temporary dir: " .. err + end + end + if not dest then + util.schedule_function(fs.delete, unpack_dir) + end + local ok, errchange = fs.change_dir(unpack_dir) + if not ok then return nil, errchange end + ok, err = fs.unzip(rock_file) + if not ok then + return nil, "Failed unpacking rock file: " .. rock_file .. ": " .. err + end + if sig_file then + ok, err = fs.copy(sig_file, ".") + if not ok then + return nil, "Failed copying signature file" + end + end + fs.pop_dir() + return unpack_dir +end + +--- Back-end function that actually loads the local rockspec. +-- Performs some validation and postprocessing of the rockspec contents. +-- @param rel_filename string: The local filename of the rockspec file. +-- @param quick boolean: if true, skips some steps when loading +-- rockspec. +-- @return table or (nil, string): A table representing the rockspec +-- or nil followed by an error message. +function fetch.load_local_rockspec(rel_filename: string, quick?: boolean): Rockspec, string + local abs_filename = fs.absolute_name(rel_filename) + + local basename = dir.base_name(abs_filename) + if basename ~= "rockspec" then + if not basename:match("(.*)%-[^-]*%-[0-9]*") then + return nil, "Expected filename in format 'name-version-revision.rockspec'." + end + end + + local tbl, err = persist.load_into_table(abs_filename) + if not tbl and err is string then + return nil, "Could not load rockspec file "..abs_filename.." ("..err..")" + end + + local rockspec, errrock = rockspecs.from_persisted_table(abs_filename, tbl as Rockspec, err as {string: any}, quick) + if not rockspec then + return nil, abs_filename .. ": " .. errrock + end + + local name_version = rockspec.package:lower() .. "-" .. rockspec.version + if basename ~= "rockspec" and basename ~= name_version .. ".rockspec" then + return nil, "Inconsistency between rockspec filename ("..basename..") and its contents ("..name_version..".rockspec)." + end + + return rockspec +end + +--- Load a local or remote rockspec into a table. +-- This is the entry point for the LuaRocks tools. +-- Only the LuaRocks runtime loader should use +-- load_local_rockspec directly. +-- @param filename string: Local or remote filename of a rockspec. +-- @param location string or nil: Where to download. If not given, +-- a temporary dir is created. +-- @param verify boolean: if true, download and verify signature for rockspec +-- @return table or (nil, string, [string]): A table representing the rockspec +-- or nil followed by an error message and optional error code. +function fetch.load_rockspec(url: string, location?: string, verify?: boolean): Rockspec, string, string + + local name: string + local basename = dir.base_name(url) + if basename == "rockspec" then + name = "rockspec" + else + name = basename:match("(.*)%.rockspec") + if not name then + return nil, "Filename '"..url.."' does not look like a rockspec." + end + end + + local tmpname = "luarocks-rockspec-"..name + local filename, err, errcode, ok: string, string, string, boolean + if location then + ok, err = fs.change_dir(location) + if not ok then return nil, err end + filename, err = fetch.fetch_url(url) + fs.pop_dir() + else + filename, err, errcode = fetch.fetch_url_at_temp_dir(url, tmpname, nil, true) + end + if not filename then + return nil, err, errcode + end + + if verify then + local _, errfetch = fetch_and_verify_signature_for(url, filename, tmpname) + if err then + return nil, errfetch + end + end + + return fetch.load_local_rockspec(filename) +end + +--- Download sources for building a rock using the basic URL downloader. +-- @param rockspec table: The rockspec table +-- @param extract boolean: Whether to extract the sources from +-- the fetched source tarball or not. +-- @param dest_dir string or nil: If set, will extract to the given directory; +-- if not given, will extract to a temporary directory. +-- @return (string, string) or (nil, string, [string]): The absolute pathname of +-- the fetched source tarball and the temporary directory created to +-- store it; or nil and an error message and optional error code. +function fetch.get_sources(rockspec: Rockspec, extract: boolean, dest_dir?: string): string, string, string + + local url = rockspec.source.url + local name = rockspec.name.."-"..rockspec.version + local filename = rockspec.source.file + local source_file, store_dir: string, string + local ok, err, errcode: boolean, string, string + if dest_dir then + ok, err = fs.change_dir(dest_dir) + if not ok then return nil, err, "dest_dir" end + source_file, err, errcode = fetch.fetch_url(url, filename) + fs.pop_dir() + store_dir = dest_dir + else + source_file, store_dir, errcode = fetch.fetch_url_at_temp_dir(url, "luarocks-source-"..name, filename) + end + if not source_file then + return nil, err or store_dir, errcode + end + if rockspec.source.md5 then + if not fs.check_md5(source_file, rockspec.source.md5) then + return nil, "MD5 check for "..filename.." has failed.", "md5" + end + end + if extract then + ok, err = fs.change_dir(store_dir) + if not ok then return nil, err end + ok, err = fs.unpack_archive(rockspec.source.file) + if not ok then return nil, err end + ok, err = fetch.find_rockspec_source_dir(rockspec, ".") + if not ok then return nil, err end + fs.pop_dir() + end + return source_file, store_dir +end + +function fetch.find_rockspec_source_dir(rockspec: Rockspec, store_dir: string): boolean, string + local ok, err = fs.change_dir(store_dir) + if not ok then return nil, err end + + local file_count, dir_count = 0, 0 + local found_dir: string + + if rockspec.source.dir and fs.exists(rockspec.source.dir) then + ok, err = true, nil + elseif rockspec.source.file and rockspec.source.dir then + ok, err = nil, "Directory "..rockspec.source.dir.." not found inside archive "..rockspec.source.file + elseif not rockspec.source.dir_set then -- and rockspec:format_is_at_least("3.0") then + + local name = dir.base_name(rockspec.source.file or rockspec.source.url or "") + + if name:match("%.lua$") or name:match("%.c$") then + if fs.is_file(name) then + rockspec.source.dir = "." + ok, err = true, nil + end + end + + if not rockspec.source.dir then + for file in fs.dir() do + file_count = file_count + 1 + if fs.is_dir(file) then + dir_count = dir_count + 1 + found_dir = file + end + end + + if dir_count == 1 then + rockspec.source.dir = found_dir + ok, err = true, nil + else + ok, err = nil, "Could not determine source directory from rock contents (" .. tostring(file_count).." file(s), "..tostring(dir_count).." dir(s))" + end + end + else + ok, err = nil, "Could not determine source directory, please set source.dir in rockspec." + end + + fs.pop_dir() + + assert(rockspec.source.dir or not ok) + return ok, err +end + +--- Download sources for building a rock, calling the appropriate protocol method. +-- @param rockspec table: The rockspec table +-- @param extract boolean: When downloading compressed formats, whether to extract +-- the sources from the fetched archive or not. +-- @param dest_dir string or nil: If set, will extract to the given directory. +-- if not given, will extract to a temporary directory. +-- @return (string, string) or (nil, string): The absolute pathname of +-- the fetched source tarball and the temporary directory created to +-- store it; or nil and an error message. +function fetch.fetch_sources(rockspec: Rockspec, extract: boolean, dest_dir?: string): string, string, string, string, string + + -- auto-convert git://github.com URLs to use git+https + -- see https://github.blog/2021-09-01-improving-git-protocol-security-github/ + if rockspec.source.url:match("^git://github%.com/") + or rockspec.source.url:match("^git://www%.github%.com/") then + rockspec.source.url = rockspec.source.url:gsub("^git://", "git+https://") + rockspec.source.protocol = "git+https" + end + + local protocol = rockspec.source.protocol + local ok, err, proto: boolean, string, Fetch + + if dir.is_basic_protocol(protocol) then + proto = fetch as Fetch + else + ok, proto = pcall(require, "luarocks.fetch."..protocol:gsub("[+-]", "_")) as (boolean, Fetch) + if not ok then + return nil, "Unknown protocol "..protocol + end + end + + if cfg.only_sources_from + and rockspec.source.pathname + and #rockspec.source.pathname > 0 then + if #cfg.only_sources_from == 0 then + return nil, "Can't download "..rockspec.source.url.." -- download from remote servers disabled" + elseif rockspec.source.pathname:find(cfg.only_sources_from, 1, true) ~= 1 then + return nil, "Can't download "..rockspec.source.url.." -- only downloading from "..cfg.only_sources_from + end + end + + local source_file, store_dir: string, string = proto.get_sources(rockspec, extract, dest_dir) + if not source_file then return nil, store_dir end + + ok, err = fetch.find_rockspec_source_dir(rockspec, store_dir) + if not ok then return nil, err, "source.dir", source_file, store_dir end + + return source_file, store_dir +end + +return fetch -- cgit v1.2.3-55-g6feb