aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorV1K1NGbg <victor@ilchev.com>2024-08-07 13:48:53 +0300
committerV1K1NGbg <victor@ilchev.com>2024-08-07 13:48:53 +0300
commitac4953e52d2130838fc5de89b12b3664ddeaadc4 (patch)
tree8c4baf9268c7ff99dab367e0c097d8b8c970b5ce
parentf8b104cb552b823655fb2a35b0a9e66471de446c (diff)
downloadluarocks-ac4953e52d2130838fc5de89b12b3664ddeaadc4.tar.gz
luarocks-ac4953e52d2130838fc5de89b12b3664ddeaadc4.tar.bz2
luarocks-ac4953e52d2130838fc5de89b12b3664ddeaadc4.zip
fetch
-rw-r--r--src/luarocks/fetch-original.lua610
-rw-r--r--src/luarocks/fetch.lua575
-rw-r--r--src/luarocks/fetch.tl56
3 files changed, 981 insertions, 260 deletions
diff --git a/src/luarocks/fetch-original.lua b/src/luarocks/fetch-original.lua
new file mode 100644
index 00000000..4e0aa89a
--- /dev/null
+++ b/src/luarocks/fetch-original.lua
@@ -0,0 +1,610 @@
1
2--- Functions related to fetching and loading local and remote files.
3local fetch = {}
4
5local fs = require("luarocks.fs")
6local dir = require("luarocks.dir")
7local rockspecs = require("luarocks.rockspecs")
8local signing = require("luarocks.signing")
9local persist = require("luarocks.persist")
10local util = require("luarocks.util")
11local cfg = require("luarocks.core.cfg")
12
13
14--- Fetch a local or remote file, using a local cache directory.
15-- Make a remote or local URL/pathname local, fetching the file if necessary.
16-- Other "fetch" and "load" functions use this function to obtain files.
17-- If a local pathname is given, it is returned as a result.
18-- @param url string: a local pathname or a remote URL.
19-- @param mirroring string: mirroring mode.
20-- If set to "no_mirror", then rocks_servers mirror configuration is not used.
21-- @return (string, nil, nil, boolean) or (nil, string, [string]):
22-- in case of success:
23-- * the absolute local pathname for the fetched file
24-- * nil
25-- * nil
26-- * `true` if the file was fetched from cache
27-- in case of failure:
28-- * nil
29-- * an error message
30-- * an optional error code.
31function fetch.fetch_caching(url, mirroring)
32 local repo_url, filename = url:match("^(.*)/([^/]+)$")
33 local name = repo_url:gsub("[/:]","_")
34 local cache_dir = dir.path(cfg.local_cache, name)
35 local ok = fs.make_dir(cache_dir)
36
37 local cachefile = dir.path(cache_dir, filename)
38 local checkfile = cachefile .. ".check"
39
40 if (fs.file_age(checkfile) < 10 or
41 cfg.aggressive_cache and (not name:match("^manifest"))) and fs.exists(cachefile)
42 then
43 return cachefile, nil, nil, true
44 end
45
46 local lock, errlock
47 if ok then
48 lock, errlock = fs.lock_access(cache_dir)
49 end
50
51 if not (ok and lock) then
52 cfg.local_cache = fs.make_temp_dir("local_cache")
53 if not cfg.local_cache then
54 return nil, "Failed creating temporary local_cache directory"
55 end
56 cache_dir = dir.path(cfg.local_cache, name)
57 ok = fs.make_dir(cache_dir)
58 if not ok then
59 return nil, "Failed creating temporary cache directory "..cache_dir
60 end
61 lock = fs.lock_access(cache_dir)
62 end
63
64 local file, err, errcode, from_cache = fetch.fetch_url(url, cachefile, true, mirroring)
65 if not file then
66 fs.unlock_access(lock)
67 return nil, err or "Failed downloading "..url, errcode
68 end
69
70 local fd, err = io.open(checkfile, "wb")
71 if err then
72 fs.unlock_access(lock)
73 return nil, err
74 end
75 fd:write("!")
76 fd:close()
77
78 fs.unlock_access(lock)
79 return file, nil, nil, from_cache
80end
81
82local function ensure_trailing_slash(url)
83 return (url:gsub("/*$", "/"))
84end
85
86local function is_url_relative_to_rocks_servers(url, servers)
87 for _, item in ipairs(servers) do
88 if type(item) == "table" then
89 for i, s in ipairs(item) do
90 local base = ensure_trailing_slash(s)
91 if string.find(url, base, 1, true) == 1 then
92 return i, url:sub(#base + 1), item
93 end
94 end
95 end
96 end
97end
98
99local function download_with_mirrors(url, filename, cache, servers)
100 local idx, rest, mirrors = is_url_relative_to_rocks_servers(url, servers)
101
102 if not idx then
103 -- URL is not from a rock server
104 return fs.download(url, filename, cache)
105 end
106
107 -- URL is from a rock server: try to download it falling back to mirrors.
108 local err = "\n"
109 for i = idx, #mirrors do
110 local try_url = ensure_trailing_slash(mirrors[i]) .. rest
111 if i > idx then
112 util.warning("Failed downloading. Attempting mirror at " .. try_url)
113 end
114 local name, _, _, from_cache = fs.download(try_url, filename, cache)
115 if name then
116 return name, nil, nil, from_cache
117 else
118 err = err .. name .. "\n"
119 end
120 end
121
122 return nil, err, "network"
123end
124
125--- Fetch a local or remote file.
126-- Make a remote or local URL/pathname local, fetching the file if necessary.
127-- Other "fetch" and "load" functions use this function to obtain files.
128-- If a local pathname is given, it is returned as a result.
129-- @param url string: a local pathname or a remote URL.
130-- @param filename string or nil: this function attempts to detect the
131-- resulting local filename of the remote file as the basename of the URL;
132-- if that is not correct (due to a redirection, for example), the local
133-- filename can be given explicitly as this second argument.
134-- @param cache boolean: compare remote timestamps via HTTP HEAD prior to
135-- re-downloading the file.
136-- @param mirroring string: mirroring mode.
137-- If set to "no_mirror", then rocks_servers mirror configuration is not used.
138-- @return (string, nil, nil, boolean) or (nil, string, [string]):
139-- in case of success:
140-- * the absolute local pathname for the fetched file
141-- * nil
142-- * nil
143-- * `true` if the file was fetched from cache
144-- in case of failure:
145-- * nil
146-- * an error message
147-- * an optional error code.
148function fetch.fetch_url(url, filename, cache, mirroring)
149 assert(type(url) == "string")
150 assert(type(filename) == "string" or not filename)
151
152 local protocol, pathname = dir.split_url(url)
153 if protocol == "file" then
154 local fullname = fs.absolute_name(pathname)
155 if not fs.exists(fullname) then
156 local hint = (not pathname:match("^/"))
157 and (" - note that given path in rockspec is not absolute: " .. url)
158 or ""
159 return nil, "Local file not found: " .. fullname .. hint
160 end
161 filename = filename or dir.base_name(pathname)
162 local dstname = fs.absolute_name(dir.path(".", filename))
163 local ok, err
164 if fullname == dstname then
165 ok = true
166 else
167 ok, err = fs.copy(fullname, dstname)
168 end
169 if ok then
170 return dstname
171 else
172 return nil, "Failed copying local file " .. fullname .. " to " .. dstname .. ": " .. err
173 end
174 elseif dir.is_basic_protocol(protocol) then
175 local name, err, err_code, from_cache
176 if mirroring ~= "no_mirror" then
177 name, err, err_code, from_cache = download_with_mirrors(url, filename, cache, cfg.rocks_servers)
178 else
179 name, err, err_code, from_cache = fs.download(url, filename, cache)
180 end
181 if not name then
182 return nil, "Failed downloading "..url..(err and " - "..err or ""), err_code
183 end
184 return name, nil, nil, from_cache
185 else
186 return nil, "Unsupported protocol "..protocol
187 end
188end
189
190--- For remote URLs, create a temporary directory and download URL inside it.
191-- This temporary directory will be deleted on program termination.
192-- For local URLs, just return the local pathname and its directory.
193-- @param url string: URL to be downloaded
194-- @param tmpname string: name pattern to use for avoiding conflicts
195-- when creating temporary directory.
196-- @param filename string or nil: local filename of URL to be downloaded,
197-- in case it can't be inferred from the URL.
198-- @return (string, string) or (nil, string, [string]): absolute local pathname of
199-- the fetched file and temporary directory name; or nil and an error message
200-- followed by an optional error code
201function fetch.fetch_url_at_temp_dir(url, tmpname, filename, cache)
202 assert(type(url) == "string")
203 assert(type(tmpname) == "string")
204 assert(type(filename) == "string" or not filename)
205 filename = filename or dir.base_name(url)
206
207 local protocol, pathname = dir.split_url(url)
208 if protocol == "file" then
209 if fs.exists(pathname) then
210 return pathname, dir.dir_name(fs.absolute_name(pathname))
211 else
212 return nil, "File not found: "..pathname
213 end
214 else
215 local temp_dir, err = fs.make_temp_dir(tmpname)
216 if not temp_dir then
217 return nil, "Failed creating temporary directory "..tmpname..": "..err
218 end
219 util.schedule_function(fs.delete, temp_dir)
220 local ok, err = fs.change_dir(temp_dir)
221 if not ok then return nil, err end
222
223 local file, err, errcode
224
225 if cache then
226 local cachefile
227 cachefile, err, errcode = fetch.fetch_caching(url)
228
229 if cachefile then
230 file = dir.path(temp_dir, filename)
231 fs.copy(cachefile, file)
232 end
233 end
234
235 if not file then
236 file, err, errcode = fetch.fetch_url(url, filename, cache)
237 end
238
239 fs.pop_dir()
240 if not file then
241 return nil, "Error fetching file: "..err, errcode
242 end
243
244 return file, temp_dir
245 end
246end
247
248-- Determine base directory of a fetched URL by extracting its
249-- archive and looking for a directory in the root.
250-- @param file string: absolute local pathname of the fetched file
251-- @param temp_dir string: temporary directory in which URL was fetched.
252-- @param src_url string: URL to use when inferring base directory.
253-- @param src_dir string or nil: expected base directory (inferred
254-- from src_url if not given).
255-- @return (string, string) or (string, nil) or (nil, string):
256-- The inferred base directory and the one actually found (which may
257-- be nil if not found), or nil followed by an error message.
258-- The inferred dir is returned first to avoid confusion with errors,
259-- because it is never nil.
260function fetch.find_base_dir(file, temp_dir, src_url, src_dir)
261 local ok, err = fs.change_dir(temp_dir)
262 if not ok then return nil, err end
263 fs.unpack_archive(file)
264
265 if not src_dir then
266 local rockspec = {
267 source = {
268 file = file,
269 dir = src_dir,
270 url = src_url,
271 }
272 }
273 ok, err = fetch.find_rockspec_source_dir(rockspec, ".")
274 if ok then
275 src_dir = rockspec.source.dir
276 end
277 end
278
279 local inferred_dir = src_dir or dir.deduce_base_dir(src_url)
280 local found_dir = nil
281 if fs.exists(inferred_dir) then
282 found_dir = inferred_dir
283 else
284 util.printerr("Directory "..inferred_dir.." not found")
285 local files = fs.list_dir()
286 if files then
287 table.sort(files)
288 for i,filename in ipairs(files) do
289 if fs.is_dir(filename) then
290 util.printerr("Found "..filename)
291 found_dir = filename
292 break
293 end
294 end
295 end
296 end
297 fs.pop_dir()
298 return inferred_dir, found_dir
299end
300
301local function fetch_and_verify_signature_for(url, filename, tmpdir)
302 local sig_url = signing.signature_url(url)
303 local sig_file, err, errcode = fetch.fetch_url_at_temp_dir(sig_url, tmpdir)
304 if not sig_file then
305 return nil, "Could not fetch signature file for verification: " .. err, errcode
306 end
307
308 local ok, err = signing.verify_signature(filename, sig_file)
309 if not ok then
310 return nil, "Failed signature verification: " .. err
311 end
312
313 return fs.absolute_name(sig_file)
314end
315
316--- Obtain a rock and unpack it.
317-- If a directory is not given, a temporary directory will be created,
318-- which will be deleted on program termination.
319-- @param rock_file string: URL or filename of the rock.
320-- @param dest string or nil: if given, directory will be used as
321-- a permanent destination.
322-- @param verify boolean: if true, download and verify signature for rockspec
323-- @return string or (nil, string, [string]): the directory containing the contents
324-- of the unpacked rock.
325function fetch.fetch_and_unpack_rock(url, dest, verify)
326 assert(type(url) == "string")
327 assert(type(dest) == "string" or not dest)
328
329 local name = dir.base_name(url):match("(.*)%.[^.]*%.rock")
330 local tmpname = "luarocks-rock-" .. name
331
332 local rock_file, err, errcode = fetch.fetch_url_at_temp_dir(url, tmpname, nil, true)
333 if not rock_file then
334 return nil, "Could not fetch rock file: " .. err, errcode
335 end
336
337 local sig_file
338 if verify then
339 sig_file, err = fetch_and_verify_signature_for(url, rock_file, tmpname)
340 if err then
341 return nil, err
342 end
343 end
344
345 rock_file = fs.absolute_name(rock_file)
346
347 local unpack_dir
348 if dest then
349 unpack_dir = dest
350 local ok, err = fs.make_dir(unpack_dir)
351 if not ok then
352 return nil, "Failed unpacking rock file: " .. err
353 end
354 else
355 unpack_dir, err = fs.make_temp_dir(name)
356 if not unpack_dir then
357 return nil, "Failed creating temporary dir: " .. err
358 end
359 end
360 if not dest then
361 util.schedule_function(fs.delete, unpack_dir)
362 end
363 local ok, err = fs.change_dir(unpack_dir)
364 if not ok then return nil, err end
365 ok, err = fs.unzip(rock_file)
366 if not ok then
367 return nil, "Failed unpacking rock file: " .. rock_file .. ": " .. err
368 end
369 if sig_file then
370 ok, err = fs.copy(sig_file, ".")
371 if not ok then
372 return nil, "Failed copying signature file"
373 end
374 end
375 fs.pop_dir()
376 return unpack_dir
377end
378
379--- Back-end function that actually loads the local rockspec.
380-- Performs some validation and postprocessing of the rockspec contents.
381-- @param rel_filename string: The local filename of the rockspec file.
382-- @param quick boolean: if true, skips some steps when loading
383-- rockspec.
384-- @return table or (nil, string): A table representing the rockspec
385-- or nil followed by an error message.
386function fetch.load_local_rockspec(rel_filename, quick)
387 assert(type(rel_filename) == "string")
388 local abs_filename = fs.absolute_name(rel_filename)
389
390 local basename = dir.base_name(abs_filename)
391 if basename ~= "rockspec" then
392 if not basename:match("(.*)%-[^-]*%-[0-9]*") then
393 return nil, "Expected filename in format 'name-version-revision.rockspec'."
394 end
395 end
396
397 local tbl, err = persist.load_into_table(abs_filename)
398 if not tbl then
399 return nil, "Could not load rockspec file "..abs_filename.." ("..err..")"
400 end
401
402 local rockspec, err = rockspecs.from_persisted_table(abs_filename, tbl, err, quick)
403 if not rockspec then
404 return nil, abs_filename .. ": " .. err
405 end
406
407 local name_version = rockspec.package:lower() .. "-" .. rockspec.version
408 if basename ~= "rockspec" and basename ~= name_version .. ".rockspec" then
409 return nil, "Inconsistency between rockspec filename ("..basename..") and its contents ("..name_version..".rockspec)."
410 end
411
412 return rockspec
413end
414
415--- Load a local or remote rockspec into a table.
416-- This is the entry point for the LuaRocks tools.
417-- Only the LuaRocks runtime loader should use
418-- load_local_rockspec directly.
419-- @param filename string: Local or remote filename of a rockspec.
420-- @param location string or nil: Where to download. If not given,
421-- a temporary dir is created.
422-- @param verify boolean: if true, download and verify signature for rockspec
423-- @return table or (nil, string, [string]): A table representing the rockspec
424-- or nil followed by an error message and optional error code.
425function fetch.load_rockspec(url, location, verify)
426 assert(type(url) == "string")
427
428 local name
429 local basename = dir.base_name(url)
430 if basename == "rockspec" then
431 name = "rockspec"
432 else
433 name = basename:match("(.*)%.rockspec")
434 if not name then
435 return nil, "Filename '"..url.."' does not look like a rockspec."
436 end
437 end
438
439 local tmpname = "luarocks-rockspec-"..name
440 local filename, err, errcode
441 if location then
442 local ok, err = fs.change_dir(location)
443 if not ok then return nil, err end
444 filename, err = fetch.fetch_url(url)
445 fs.pop_dir()
446 else
447 filename, err, errcode = fetch.fetch_url_at_temp_dir(url, tmpname, nil, true)
448 end
449 if not filename then
450 return nil, err, errcode
451 end
452
453 if verify then
454 local _, err = fetch_and_verify_signature_for(url, filename, tmpname)
455 if err then
456 return nil, err
457 end
458 end
459
460 return fetch.load_local_rockspec(filename)
461end
462
463--- Download sources for building a rock using the basic URL downloader.
464-- @param rockspec table: The rockspec table
465-- @param extract boolean: Whether to extract the sources from
466-- the fetched source tarball or not.
467-- @param dest_dir string or nil: If set, will extract to the given directory;
468-- if not given, will extract to a temporary directory.
469-- @return (string, string) or (nil, string, [string]): The absolute pathname of
470-- the fetched source tarball and the temporary directory created to
471-- store it; or nil and an error message and optional error code.
472function fetch.get_sources(rockspec, extract, dest_dir)
473 assert(rockspec:type() == "rockspec")
474 assert(type(extract) == "boolean")
475 assert(type(dest_dir) == "string" or not dest_dir)
476
477 local url = rockspec.source.url
478 local name = rockspec.name.."-"..rockspec.version
479 local filename = rockspec.source.file
480 local source_file, store_dir
481 local ok, err, errcode
482 if dest_dir then
483 ok, err = fs.change_dir(dest_dir)
484 if not ok then return nil, err, "dest_dir" end
485 source_file, err, errcode = fetch.fetch_url(url, filename)
486 fs.pop_dir()
487 store_dir = dest_dir
488 else
489 source_file, store_dir, errcode = fetch.fetch_url_at_temp_dir(url, "luarocks-source-"..name, filename)
490 end
491 if not source_file then
492 return nil, err or store_dir, errcode
493 end
494 if rockspec.source.md5 then
495 if not fs.check_md5(source_file, rockspec.source.md5) then
496 return nil, "MD5 check for "..filename.." has failed.", "md5"
497 end
498 end
499 if extract then
500 local ok, err = fs.change_dir(store_dir)
501 if not ok then return nil, err end
502 ok, err = fs.unpack_archive(rockspec.source.file)
503 if not ok then return nil, err end
504 ok, err = fetch.find_rockspec_source_dir(rockspec, ".")
505 if not ok then return nil, err end
506 fs.pop_dir()
507 end
508 return source_file, store_dir
509end
510
511function fetch.find_rockspec_source_dir(rockspec, store_dir)
512 local ok, err = fs.change_dir(store_dir)
513 if not ok then return nil, err end
514
515 local file_count, dir_count, found_dir = 0, 0, 0
516
517 if rockspec.source.dir and fs.exists(rockspec.source.dir) then
518 ok, err = true, nil
519 elseif rockspec.source.file and rockspec.source.dir then
520 ok, err = nil, "Directory "..rockspec.source.dir.." not found inside archive "..rockspec.source.file
521 elseif not rockspec.source.dir_set then -- and rockspec:format_is_at_least("3.0") then
522
523 local name = dir.base_name(rockspec.source.file or rockspec.source.url or "")
524
525 if name:match("%.lua$") or name:match("%.c$") then
526 if fs.is_file(name) then
527 rockspec.source.dir = "."
528 ok, err = true, nil
529 end
530 end
531
532 if not rockspec.source.dir then
533 for file in fs.dir() do
534 file_count = file_count + 1
535 if fs.is_dir(file) then
536 dir_count = dir_count + 1
537 found_dir = file
538 end
539 end
540
541 if dir_count == 1 then
542 rockspec.source.dir = found_dir
543 ok, err = true, nil
544 else
545 ok, err = nil, "Could not determine source directory from rock contents (" .. tostring(file_count).." file(s), "..tostring(dir_count).." dir(s))"
546 end
547 end
548 else
549 ok, err = nil, "Could not determine source directory, please set source.dir in rockspec."
550 end
551
552 fs.pop_dir()
553
554 assert(rockspec.source.dir or not ok)
555 return ok, err
556end
557
558--- Download sources for building a rock, calling the appropriate protocol method.
559-- @param rockspec table: The rockspec table
560-- @param extract boolean: When downloading compressed formats, whether to extract
561-- the sources from the fetched archive or not.
562-- @param dest_dir string or nil: If set, will extract to the given directory.
563-- if not given, will extract to a temporary directory.
564-- @return (string, string) or (nil, string): The absolute pathname of
565-- the fetched source tarball and the temporary directory created to
566-- store it; or nil and an error message.
567function fetch.fetch_sources(rockspec, extract, dest_dir)
568 assert(rockspec:type() == "rockspec")
569 assert(type(extract) == "boolean")
570 assert(type(dest_dir) == "string" or not dest_dir)
571
572 -- auto-convert git://github.com URLs to use git+https
573 -- see https://github.blog/2021-09-01-improving-git-protocol-security-github/
574 if rockspec.source.url:match("^git://github%.com/")
575 or rockspec.source.url:match("^git://www%.github%.com/") then
576 rockspec.source.url = rockspec.source.url:gsub("^git://", "git+https://")
577 rockspec.source.protocol = "git+https"
578 end
579
580 local protocol = rockspec.source.protocol
581 local ok, err, proto
582 if dir.is_basic_protocol(protocol) then
583 proto = fetch
584 else
585 ok, proto = pcall(require, "luarocks.fetch."..protocol:gsub("[+-]", "_"))
586 if not ok then
587 return nil, "Unknown protocol "..protocol
588 end
589 end
590
591 if cfg.only_sources_from
592 and rockspec.source.pathname
593 and #rockspec.source.pathname > 0 then
594 if #cfg.only_sources_from == 0 then
595 return nil, "Can't download "..rockspec.source.url.." -- download from remote servers disabled"
596 elseif rockspec.source.pathname:find(cfg.only_sources_from, 1, true) ~= 1 then
597 return nil, "Can't download "..rockspec.source.url.." -- only downloading from "..cfg.only_sources_from
598 end
599 end
600
601 local source_file, store_dir = proto.get_sources(rockspec, extract, dest_dir)
602 if not source_file then return nil, store_dir end
603
604 ok, err = fetch.find_rockspec_source_dir(rockspec, store_dir)
605 if not ok then return nil, err, "source.dir", source_file, store_dir end
606
607 return source_file, store_dir
608end
609
610return fetch
diff --git a/src/luarocks/fetch.lua b/src/luarocks/fetch.lua
index 4e0aa89a..2c6f7e28 100644
--- a/src/luarocks/fetch.lua
+++ b/src/luarocks/fetch.lua
@@ -1,6 +1,10 @@
1local _tl_compat; if (tonumber((_VERSION or ''):match('[%d.]*$')) or 0) < 5.3 then local p, m = pcall(require, 'compat53.module'); if p then _tl_compat = m end end; local assert = _tl_compat and _tl_compat.assert or assert; local io = _tl_compat and _tl_compat.io or io; local ipairs = _tl_compat and _tl_compat.ipairs or ipairs; local pcall = _tl_compat and _tl_compat.pcall or pcall; local string = _tl_compat and _tl_compat.string or string; local table = _tl_compat and _tl_compat.table or table
2
3local fetch = {Fetch = {}, }
4
5
6
1 7
2--- Functions related to fetching and loading local and remote files.
3local fetch = {}
4 8
5local fs = require("luarocks.fs") 9local fs = require("luarocks.fs")
6local dir = require("luarocks.dir") 10local dir = require("luarocks.dir")
@@ -11,73 +15,10 @@ local util = require("luarocks.util")
11local cfg = require("luarocks.core.cfg") 15local cfg = require("luarocks.core.cfg")
12 16
13 17
14--- Fetch a local or remote file, using a local cache directory.
15-- Make a remote or local URL/pathname local, fetching the file if necessary.
16-- Other "fetch" and "load" functions use this function to obtain files.
17-- If a local pathname is given, it is returned as a result.
18-- @param url string: a local pathname or a remote URL.
19-- @param mirroring string: mirroring mode.
20-- If set to "no_mirror", then rocks_servers mirror configuration is not used.
21-- @return (string, nil, nil, boolean) or (nil, string, [string]):
22-- in case of success:
23-- * the absolute local pathname for the fetched file
24-- * nil
25-- * nil
26-- * `true` if the file was fetched from cache
27-- in case of failure:
28-- * nil
29-- * an error message
30-- * an optional error code.
31function fetch.fetch_caching(url, mirroring)
32 local repo_url, filename = url:match("^(.*)/([^/]+)$")
33 local name = repo_url:gsub("[/:]","_")
34 local cache_dir = dir.path(cfg.local_cache, name)
35 local ok = fs.make_dir(cache_dir)
36
37 local cachefile = dir.path(cache_dir, filename)
38 local checkfile = cachefile .. ".check"
39
40 if (fs.file_age(checkfile) < 10 or
41 cfg.aggressive_cache and (not name:match("^manifest"))) and fs.exists(cachefile)
42 then
43 return cachefile, nil, nil, true
44 end
45
46 local lock, errlock
47 if ok then
48 lock, errlock = fs.lock_access(cache_dir)
49 end
50 18
51 if not (ok and lock) then
52 cfg.local_cache = fs.make_temp_dir("local_cache")
53 if not cfg.local_cache then
54 return nil, "Failed creating temporary local_cache directory"
55 end
56 cache_dir = dir.path(cfg.local_cache, name)
57 ok = fs.make_dir(cache_dir)
58 if not ok then
59 return nil, "Failed creating temporary cache directory "..cache_dir
60 end
61 lock = fs.lock_access(cache_dir)
62 end
63 19
64 local file, err, errcode, from_cache = fetch.fetch_url(url, cachefile, true, mirroring)
65 if not file then
66 fs.unlock_access(lock)
67 return nil, err or "Failed downloading "..url, errcode
68 end
69 20
70 local fd, err = io.open(checkfile, "wb")
71 if err then
72 fs.unlock_access(lock)
73 return nil, err
74 end
75 fd:write("!")
76 fd:close()
77 21
78 fs.unlock_access(lock)
79 return file, nil, nil, from_cache
80end
81 22
82local function ensure_trailing_slash(url) 23local function ensure_trailing_slash(url)
83 return (url:gsub("/*$", "/")) 24 return (url:gsub("/*$", "/"))
@@ -100,11 +41,11 @@ local function download_with_mirrors(url, filename, cache, servers)
100 local idx, rest, mirrors = is_url_relative_to_rocks_servers(url, servers) 41 local idx, rest, mirrors = is_url_relative_to_rocks_servers(url, servers)
101 42
102 if not idx then 43 if not idx then
103 -- URL is not from a rock server 44
104 return fs.download(url, filename, cache) 45 return fs.download(url, filename, cache)
105 end 46 end
106 47
107 -- URL is from a rock server: try to download it falling back to mirrors. 48
108 local err = "\n" 49 local err = "\n"
109 for i = idx, #mirrors do 50 for i = idx, #mirrors do
110 local try_url = ensure_trailing_slash(mirrors[i]) .. rest 51 local try_url = ensure_trailing_slash(mirrors[i]) .. rest
@@ -122,40 +63,38 @@ local function download_with_mirrors(url, filename, cache, servers)
122 return nil, err, "network" 63 return nil, err, "network"
123end 64end
124 65
125--- Fetch a local or remote file. 66
126-- Make a remote or local URL/pathname local, fetching the file if necessary. 67
127-- Other "fetch" and "load" functions use this function to obtain files. 68
128-- If a local pathname is given, it is returned as a result. 69
129-- @param url string: a local pathname or a remote URL. 70
130-- @param filename string or nil: this function attempts to detect the 71
131-- resulting local filename of the remote file as the basename of the URL; 72
132-- if that is not correct (due to a redirection, for example), the local 73
133-- filename can be given explicitly as this second argument. 74
134-- @param cache boolean: compare remote timestamps via HTTP HEAD prior to 75
135-- re-downloading the file. 76
136-- @param mirroring string: mirroring mode. 77
137-- If set to "no_mirror", then rocks_servers mirror configuration is not used. 78
138-- @return (string, nil, nil, boolean) or (nil, string, [string]): 79
139-- in case of success: 80
140-- * the absolute local pathname for the fetched file 81
141-- * nil 82
142-- * nil 83
143-- * `true` if the file was fetched from cache 84
144-- in case of failure: 85
145-- * nil 86
146-- * an error message 87
147-- * an optional error code. 88
148function fetch.fetch_url(url, filename, cache, mirroring) 89function fetch.fetch_url(url, filename, cache, mirroring)
149 assert(type(url) == "string")
150 assert(type(filename) == "string" or not filename)
151 90
152 local protocol, pathname = dir.split_url(url) 91 local protocol, pathname = dir.split_url(url)
153 if protocol == "file" then 92 if protocol == "file" then
154 local fullname = fs.absolute_name(pathname) 93 local fullname = fs.absolute_name(pathname)
155 if not fs.exists(fullname) then 94 if not fs.exists(fullname) then
156 local hint = (not pathname:match("^/")) 95 local hint = (not pathname:match("^/")) and
157 and (" - note that given path in rockspec is not absolute: " .. url) 96 (" - note that given path in rockspec is not absolute: " .. url) or
158 or "" 97 ""
159 return nil, "Local file not found: " .. fullname .. hint 98 return nil, "Local file not found: " .. fullname .. hint
160 end 99 end
161 filename = filename or dir.base_name(pathname) 100 filename = filename or dir.base_name(pathname)
@@ -179,29 +118,96 @@ function fetch.fetch_url(url, filename, cache, mirroring)
179 name, err, err_code, from_cache = fs.download(url, filename, cache) 118 name, err, err_code, from_cache = fs.download(url, filename, cache)
180 end 119 end
181 if not name then 120 if not name then
182 return nil, "Failed downloading "..url..(err and " - "..err or ""), err_code 121 return nil, "Failed downloading " .. url .. (err and " - " .. err or ""), err_code
183 end 122 end
184 return name, nil, nil, from_cache 123 return name, nil, nil, from_cache
185 else 124 else
186 return nil, "Unsupported protocol "..protocol 125 return nil, "Unsupported protocol " .. protocol
187 end 126 end
188end 127end
189 128
190--- For remote URLs, create a temporary directory and download URL inside it. 129
191-- This temporary directory will be deleted on program termination. 130
192-- For local URLs, just return the local pathname and its directory. 131
193-- @param url string: URL to be downloaded 132
194-- @param tmpname string: name pattern to use for avoiding conflicts 133
195-- when creating temporary directory. 134
196-- @param filename string or nil: local filename of URL to be downloaded, 135
197-- in case it can't be inferred from the URL. 136
198-- @return (string, string) or (nil, string, [string]): absolute local pathname of 137
199-- the fetched file and temporary directory name; or nil and an error message 138
200-- followed by an optional error code 139
140
141
142
143
144
145
146
147function fetch.fetch_caching(url, mirroring)
148 local repo_url, filename = url:match("^(.*)/([^/]+)$")
149 local name = repo_url:gsub("[/:]", "_")
150 local cache_dir = dir.path(cfg.local_cache, name)
151 local ok = fs.make_dir(cache_dir)
152
153 local cachefile = dir.path(cache_dir, filename)
154 local checkfile = cachefile .. ".check"
155
156 if (fs.file_age(checkfile) < 10 or
157 cfg.aggressive_cache and (not name:match("^manifest"))) and fs.exists(cachefile) then
158
159 return cachefile, nil, nil, true
160 end
161
162 local lock, errlock
163 if ok then
164 lock, errlock = fs.lock_access(cache_dir)
165 end
166
167 if not (ok and lock) then
168 cfg.local_cache = fs.make_temp_dir("local_cache")
169 if not cfg.local_cache then
170 return nil, "Failed creating temporary local_cache directory"
171 end
172 cache_dir = dir.path(cfg.local_cache, name)
173 ok = fs.make_dir(cache_dir)
174 if not ok then
175 return nil, "Failed creating temporary cache directory " .. cache_dir
176 end
177 lock = fs.lock_access(cache_dir)
178 end
179
180 local file, err, errcode, from_cache = fetch.fetch_url(url, cachefile, true, mirroring)
181 if not file then
182 fs.unlock_access(lock)
183 return nil, err or "Failed downloading " .. url, errcode
184 end
185
186 local fd, erropen = io.open(checkfile, "wb")
187 if erropen then
188 fs.unlock_access(lock)
189 return nil, erropen
190 end
191 fd:write("!")
192 fd:close()
193
194 fs.unlock_access(lock)
195 return file, nil, nil, from_cache
196end
197
198
199
200
201
202
203
204
205
206
207
208
209
201function fetch.fetch_url_at_temp_dir(url, tmpname, filename, cache) 210function fetch.fetch_url_at_temp_dir(url, tmpname, filename, cache)
202 assert(type(url) == "string")
203 assert(type(tmpname) == "string")
204 assert(type(filename) == "string" or not filename)
205 filename = filename or dir.base_name(url) 211 filename = filename or dir.base_name(url)
206 212
207 local protocol, pathname = dir.split_url(url) 213 local protocol, pathname = dir.split_url(url)
@@ -209,16 +215,16 @@ function fetch.fetch_url_at_temp_dir(url, tmpname, filename, cache)
209 if fs.exists(pathname) then 215 if fs.exists(pathname) then
210 return pathname, dir.dir_name(fs.absolute_name(pathname)) 216 return pathname, dir.dir_name(fs.absolute_name(pathname))
211 else 217 else
212 return nil, "File not found: "..pathname 218 return nil, "File not found: " .. pathname
213 end 219 end
214 else 220 else
215 local temp_dir, err = fs.make_temp_dir(tmpname) 221 local temp_dir, errmake = fs.make_temp_dir(tmpname)
216 if not temp_dir then 222 if not temp_dir then
217 return nil, "Failed creating temporary directory "..tmpname..": "..err 223 return nil, "Failed creating temporary directory " .. tmpname .. ": " .. errmake
218 end 224 end
219 util.schedule_function(fs.delete, temp_dir) 225 util.schedule_function(fs.delete, temp_dir)
220 local ok, err = fs.change_dir(temp_dir) 226 local ok, errchange = fs.change_dir(temp_dir)
221 if not ok then return nil, err end 227 if not ok then return nil, errchange end
222 228
223 local file, err, errcode 229 local file, err, errcode
224 230
@@ -238,25 +244,73 @@ function fetch.fetch_url_at_temp_dir(url, tmpname, filename, cache)
238 244
239 fs.pop_dir() 245 fs.pop_dir()
240 if not file then 246 if not file then
241 return nil, "Error fetching file: "..err, errcode 247 return nil, "Error fetching file: " .. err, errcode
242 end 248 end
243 249
244 return file, temp_dir 250 return file, temp_dir
245 end 251 end
246end 252end
247 253
248-- Determine base directory of a fetched URL by extracting its 254function fetch.find_rockspec_source_dir(rockspec, store_dir)
249-- archive and looking for a directory in the root. 255 local ok, err = fs.change_dir(store_dir)
250-- @param file string: absolute local pathname of the fetched file 256 if not ok then return nil, err end
251-- @param temp_dir string: temporary directory in which URL was fetched. 257
252-- @param src_url string: URL to use when inferring base directory. 258 local file_count, dir_count = 0, 0
253-- @param src_dir string or nil: expected base directory (inferred 259 local found_dir
254-- from src_url if not given). 260
255-- @return (string, string) or (string, nil) or (nil, string): 261 if rockspec.source.dir and fs.exists(rockspec.source.dir) then
256-- The inferred base directory and the one actually found (which may 262 ok, err = true, nil
257-- be nil if not found), or nil followed by an error message. 263 elseif rockspec.source.file and rockspec.source.dir then
258-- The inferred dir is returned first to avoid confusion with errors, 264 ok, err = nil, "Directory " .. rockspec.source.dir .. " not found inside archive " .. rockspec.source.file
259-- because it is never nil. 265 elseif not rockspec.source.dir_set then
266
267 local name = dir.base_name(rockspec.source.file or rockspec.source.url or "")
268
269 if name:match("%.lua$") or name:match("%.c$") then
270 if fs.is_file(name) then
271 rockspec.source.dir = "."
272 ok, err = true, nil
273 end
274 end
275
276 if not rockspec.source.dir then
277 for file in fs.dir() do
278 file_count = file_count + 1
279 if fs.is_dir(file) then
280 dir_count = dir_count + 1
281 found_dir = file
282 end
283 end
284
285 if dir_count == 1 then
286 rockspec.source.dir = found_dir
287 ok, err = true, nil
288 else
289 ok, err = nil, "Could not determine source directory from rock contents (" .. tostring(file_count) .. " file(s), " .. tostring(dir_count) .. " dir(s))"
290 end
291 end
292 else
293 ok, err = nil, "Could not determine source directory, please set source.dir in rockspec."
294 end
295
296 fs.pop_dir()
297
298 assert(rockspec.source.dir or not ok)
299 return ok, err
300end
301
302
303
304
305
306
307
308
309
310
311
312
313
260function fetch.find_base_dir(file, temp_dir, src_url, src_dir) 314function fetch.find_base_dir(file, temp_dir, src_url, src_dir)
261 local ok, err = fs.change_dir(temp_dir) 315 local ok, err = fs.change_dir(temp_dir)
262 if not ok then return nil, err end 316 if not ok then return nil, err end
@@ -268,7 +322,7 @@ function fetch.find_base_dir(file, temp_dir, src_url, src_dir)
268 file = file, 322 file = file,
269 dir = src_dir, 323 dir = src_dir,
270 url = src_url, 324 url = src_url,
271 } 325 },
272 } 326 }
273 ok, err = fetch.find_rockspec_source_dir(rockspec, ".") 327 ok, err = fetch.find_rockspec_source_dir(rockspec, ".")
274 if ok then 328 if ok then
@@ -281,13 +335,13 @@ function fetch.find_base_dir(file, temp_dir, src_url, src_dir)
281 if fs.exists(inferred_dir) then 335 if fs.exists(inferred_dir) then
282 found_dir = inferred_dir 336 found_dir = inferred_dir
283 else 337 else
284 util.printerr("Directory "..inferred_dir.." not found") 338 util.printerr("Directory " .. inferred_dir .. " not found")
285 local files = fs.list_dir() 339 local files = fs.list_dir()
286 if files then 340 if files then
287 table.sort(files) 341 table.sort(files)
288 for i,filename in ipairs(files) do 342 for _, filename in ipairs(files) do
289 if fs.is_dir(filename) then 343 if fs.is_dir(filename) then
290 util.printerr("Found "..filename) 344 util.printerr("Found " .. filename)
291 found_dir = filename 345 found_dir = filename
292 break 346 break
293 end 347 end
@@ -300,9 +354,9 @@ end
300 354
301local function fetch_and_verify_signature_for(url, filename, tmpdir) 355local function fetch_and_verify_signature_for(url, filename, tmpdir)
302 local sig_url = signing.signature_url(url) 356 local sig_url = signing.signature_url(url)
303 local sig_file, err, errcode = fetch.fetch_url_at_temp_dir(sig_url, tmpdir) 357 local sig_file, errfetch, errcode = fetch.fetch_url_at_temp_dir(sig_url, tmpdir)
304 if not sig_file then 358 if not sig_file then
305 return nil, "Could not fetch signature file for verification: " .. err, errcode 359 return nil, "Could not fetch signature file for verification: " .. errfetch, errcode
306 end 360 end
307 361
308 local ok, err = signing.verify_signature(filename, sig_file) 362 local ok, err = signing.verify_signature(filename, sig_file)
@@ -313,18 +367,16 @@ local function fetch_and_verify_signature_for(url, filename, tmpdir)
313 return fs.absolute_name(sig_file) 367 return fs.absolute_name(sig_file)
314end 368end
315 369
316--- Obtain a rock and unpack it. 370
317-- If a directory is not given, a temporary directory will be created, 371
318-- which will be deleted on program termination. 372
319-- @param rock_file string: URL or filename of the rock. 373
320-- @param dest string or nil: if given, directory will be used as 374
321-- a permanent destination. 375
322-- @param verify boolean: if true, download and verify signature for rockspec 376
323-- @return string or (nil, string, [string]): the directory containing the contents 377
324-- of the unpacked rock. 378
325function fetch.fetch_and_unpack_rock(url, dest, verify) 379function fetch.fetch_and_unpack_rock(url, dest, verify)
326 assert(type(url) == "string")
327 assert(type(dest) == "string" or not dest)
328 380
329 local name = dir.base_name(url):match("(.*)%.[^.]*%.rock") 381 local name = dir.base_name(url):match("(.*)%.[^.]*%.rock")
330 local tmpname = "luarocks-rock-" .. name 382 local tmpname = "luarocks-rock-" .. name
@@ -347,9 +399,9 @@ function fetch.fetch_and_unpack_rock(url, dest, verify)
347 local unpack_dir 399 local unpack_dir
348 if dest then 400 if dest then
349 unpack_dir = dest 401 unpack_dir = dest
350 local ok, err = fs.make_dir(unpack_dir) 402 local ok, errmake = fs.make_dir(unpack_dir)
351 if not ok then 403 if not ok then
352 return nil, "Failed unpacking rock file: " .. err 404 return nil, "Failed unpacking rock file: " .. errmake
353 end 405 end
354 else 406 else
355 unpack_dir, err = fs.make_temp_dir(name) 407 unpack_dir, err = fs.make_temp_dir(name)
@@ -360,8 +412,8 @@ function fetch.fetch_and_unpack_rock(url, dest, verify)
360 if not dest then 412 if not dest then
361 util.schedule_function(fs.delete, unpack_dir) 413 util.schedule_function(fs.delete, unpack_dir)
362 end 414 end
363 local ok, err = fs.change_dir(unpack_dir) 415 local ok, errchange = fs.change_dir(unpack_dir)
364 if not ok then return nil, err end 416 if not ok then return nil, errchange end
365 ok, err = fs.unzip(rock_file) 417 ok, err = fs.unzip(rock_file)
366 if not ok then 418 if not ok then
367 return nil, "Failed unpacking rock file: " .. rock_file .. ": " .. err 419 return nil, "Failed unpacking rock file: " .. rock_file .. ": " .. err
@@ -376,13 +428,13 @@ function fetch.fetch_and_unpack_rock(url, dest, verify)
376 return unpack_dir 428 return unpack_dir
377end 429end
378 430
379--- Back-end function that actually loads the local rockspec. 431
380-- Performs some validation and postprocessing of the rockspec contents. 432
381-- @param rel_filename string: The local filename of the rockspec file. 433
382-- @param quick boolean: if true, skips some steps when loading 434
383-- rockspec. 435
384-- @return table or (nil, string): A table representing the rockspec 436
385-- or nil followed by an error message. 437
386function fetch.load_local_rockspec(rel_filename, quick) 438function fetch.load_local_rockspec(rel_filename, quick)
387 assert(type(rel_filename) == "string") 439 assert(type(rel_filename) == "string")
388 local abs_filename = fs.absolute_name(rel_filename) 440 local abs_filename = fs.absolute_name(rel_filename)
@@ -395,35 +447,34 @@ function fetch.load_local_rockspec(rel_filename, quick)
395 end 447 end
396 448
397 local tbl, err = persist.load_into_table(abs_filename) 449 local tbl, err = persist.load_into_table(abs_filename)
398 if not tbl then 450 if not tbl and type(err) == "string" then
399 return nil, "Could not load rockspec file "..abs_filename.." ("..err..")" 451 return nil, "Could not load rockspec file " .. abs_filename .. " (" .. err .. ")"
400 end 452 end
401 453
402 local rockspec, err = rockspecs.from_persisted_table(abs_filename, tbl, err, quick) 454 local rockspec, errrock = rockspecs.from_persisted_table(abs_filename, tbl, err, quick)
403 if not rockspec then 455 if not rockspec then
404 return nil, abs_filename .. ": " .. err 456 return nil, abs_filename .. ": " .. errrock
405 end 457 end
406 458
407 local name_version = rockspec.package:lower() .. "-" .. rockspec.version 459 local name_version = rockspec.package:lower() .. "-" .. rockspec.version
408 if basename ~= "rockspec" and basename ~= name_version .. ".rockspec" then 460 if basename ~= "rockspec" and basename ~= name_version .. ".rockspec" then
409 return nil, "Inconsistency between rockspec filename ("..basename..") and its contents ("..name_version..".rockspec)." 461 return nil, "Inconsistency between rockspec filename (" .. basename .. ") and its contents (" .. name_version .. ".rockspec)."
410 end 462 end
411 463
412 return rockspec 464 return rockspec
413end 465end
414 466
415--- Load a local or remote rockspec into a table. 467
416-- This is the entry point for the LuaRocks tools. 468
417-- Only the LuaRocks runtime loader should use 469
418-- load_local_rockspec directly. 470
419-- @param filename string: Local or remote filename of a rockspec. 471
420-- @param location string or nil: Where to download. If not given, 472
421-- a temporary dir is created. 473
422-- @param verify boolean: if true, download and verify signature for rockspec 474
423-- @return table or (nil, string, [string]): A table representing the rockspec 475
424-- or nil followed by an error message and optional error code. 476
425function fetch.load_rockspec(url, location, verify) 477function fetch.load_rockspec(url, location, verify)
426 assert(type(url) == "string")
427 478
428 local name 479 local name
429 local basename = dir.base_name(url) 480 local basename = dir.base_name(url)
@@ -432,14 +483,14 @@ function fetch.load_rockspec(url, location, verify)
432 else 483 else
433 name = basename:match("(.*)%.rockspec") 484 name = basename:match("(.*)%.rockspec")
434 if not name then 485 if not name then
435 return nil, "Filename '"..url.."' does not look like a rockspec." 486 return nil, "Filename '" .. url .. "' does not look like a rockspec."
436 end 487 end
437 end 488 end
438 489
439 local tmpname = "luarocks-rockspec-"..name 490 local tmpname = "luarocks-rockspec-" .. name
440 local filename, err, errcode 491 local filename, err, errcode, ok
441 if location then 492 if location then
442 local ok, err = fs.change_dir(location) 493 ok, err = fs.change_dir(location)
443 if not ok then return nil, err end 494 if not ok then return nil, err end
444 filename, err = fetch.fetch_url(url) 495 filename, err = fetch.fetch_url(url)
445 fs.pop_dir() 496 fs.pop_dir()
@@ -451,31 +502,28 @@ function fetch.load_rockspec(url, location, verify)
451 end 502 end
452 503
453 if verify then 504 if verify then
454 local _, err = fetch_and_verify_signature_for(url, filename, tmpname) 505 local _, errfetch = fetch_and_verify_signature_for(url, filename, tmpname)
455 if err then 506 if err then
456 return nil, err 507 return nil, errfetch
457 end 508 end
458 end 509 end
459 510
460 return fetch.load_local_rockspec(filename) 511 return fetch.load_local_rockspec(filename)
461end 512end
462 513
463--- Download sources for building a rock using the basic URL downloader. 514
464-- @param rockspec table: The rockspec table 515
465-- @param extract boolean: Whether to extract the sources from 516
466-- the fetched source tarball or not. 517
467-- @param dest_dir string or nil: If set, will extract to the given directory; 518
468-- if not given, will extract to a temporary directory. 519
469-- @return (string, string) or (nil, string, [string]): The absolute pathname of 520
470-- the fetched source tarball and the temporary directory created to 521
471-- store it; or nil and an error message and optional error code. 522
472function fetch.get_sources(rockspec, extract, dest_dir) 523function fetch.get_sources(rockspec, extract, dest_dir)
473 assert(rockspec:type() == "rockspec")
474 assert(type(extract) == "boolean")
475 assert(type(dest_dir) == "string" or not dest_dir)
476 524
477 local url = rockspec.source.url 525 local url = rockspec.source.url
478 local name = rockspec.name.."-"..rockspec.version 526 local name = rockspec.name .. "-" .. rockspec.version
479 local filename = rockspec.source.file 527 local filename = rockspec.source.file
480 local source_file, store_dir 528 local source_file, store_dir
481 local ok, err, errcode 529 local ok, err, errcode
@@ -486,18 +534,18 @@ function fetch.get_sources(rockspec, extract, dest_dir)
486 fs.pop_dir() 534 fs.pop_dir()
487 store_dir = dest_dir 535 store_dir = dest_dir
488 else 536 else
489 source_file, store_dir, errcode = fetch.fetch_url_at_temp_dir(url, "luarocks-source-"..name, filename) 537 source_file, store_dir, errcode = fetch.fetch_url_at_temp_dir(url, "luarocks-source-" .. name, filename)
490 end 538 end
491 if not source_file then 539 if not source_file then
492 return nil, err or store_dir, errcode 540 return nil, err or store_dir, errcode
493 end 541 end
494 if rockspec.source.md5 then 542 if rockspec.source.md5 then
495 if not fs.check_md5(source_file, rockspec.source.md5) then 543 if not fs.check_md5(source_file, rockspec.source.md5) then
496 return nil, "MD5 check for "..filename.." has failed.", "md5" 544 return nil, "MD5 check for " .. filename .. " has failed.", "md5"
497 end 545 end
498 end 546 end
499 if extract then 547 if extract then
500 local ok, err = fs.change_dir(store_dir) 548 ok, err = fs.change_dir(store_dir)
501 if not ok then return nil, err end 549 if not ok then return nil, err end
502 ok, err = fs.unpack_archive(rockspec.source.file) 550 ok, err = fs.unpack_archive(rockspec.source.file)
503 if not ok then return nil, err end 551 if not ok then return nil, err end
@@ -508,93 +556,104 @@ function fetch.get_sources(rockspec, extract, dest_dir)
508 return source_file, store_dir 556 return source_file, store_dir
509end 557end
510 558
511function fetch.find_rockspec_source_dir(rockspec, store_dir)
512 local ok, err = fs.change_dir(store_dir)
513 if not ok then return nil, err end
514 559
515 local file_count, dir_count, found_dir = 0, 0, 0
516 560
517 if rockspec.source.dir and fs.exists(rockspec.source.dir) then
518 ok, err = true, nil
519 elseif rockspec.source.file and rockspec.source.dir then
520 ok, err = nil, "Directory "..rockspec.source.dir.." not found inside archive "..rockspec.source.file
521 elseif not rockspec.source.dir_set then -- and rockspec:format_is_at_least("3.0") then
522 561
523 local name = dir.base_name(rockspec.source.file or rockspec.source.url or "")
524 562
525 if name:match("%.lua$") or name:match("%.c$") then
526 if fs.is_file(name) then
527 rockspec.source.dir = "."
528 ok, err = true, nil
529 end
530 end
531 563
532 if not rockspec.source.dir then
533 for file in fs.dir() do
534 file_count = file_count + 1
535 if fs.is_dir(file) then
536 dir_count = dir_count + 1
537 found_dir = file
538 end
539 end
540 564
541 if dir_count == 1 then
542 rockspec.source.dir = found_dir
543 ok, err = true, nil
544 else
545 ok, err = nil, "Could not determine source directory from rock contents (" .. tostring(file_count).." file(s), "..tostring(dir_count).." dir(s))"
546 end
547 end
548 else
549 ok, err = nil, "Could not determine source directory, please set source.dir in rockspec."
550 end
551 565
552 fs.pop_dir()
553 566
554 assert(rockspec.source.dir or not ok)
555 return ok, err
556end
557 567
558--- Download sources for building a rock, calling the appropriate protocol method.
559-- @param rockspec table: The rockspec table
560-- @param extract boolean: When downloading compressed formats, whether to extract
561-- the sources from the fetched archive or not.
562-- @param dest_dir string or nil: If set, will extract to the given directory.
563-- if not given, will extract to a temporary directory.
564-- @return (string, string) or (nil, string): The absolute pathname of
565-- the fetched source tarball and the temporary directory created to
566-- store it; or nil and an error message.
567function fetch.fetch_sources(rockspec, extract, dest_dir) 568function fetch.fetch_sources(rockspec, extract, dest_dir)
568 assert(rockspec:type() == "rockspec") 569
569 assert(type(extract) == "boolean") 570
570 assert(type(dest_dir) == "string" or not dest_dir) 571
571 572 if rockspec.source.url:match("^git://github%.com/") or
572 -- auto-convert git://github.com URLs to use git+https 573 rockspec.source.url:match("^git://www%.github%.com/") then
573 -- see https://github.blog/2021-09-01-improving-git-protocol-security-github/
574 if rockspec.source.url:match("^git://github%.com/")
575 or rockspec.source.url:match("^git://www%.github%.com/") then
576 rockspec.source.url = rockspec.source.url:gsub("^git://", "git+https://") 574 rockspec.source.url = rockspec.source.url:gsub("^git://", "git+https://")
577 rockspec.source.protocol = "git+https" 575 rockspec.source.protocol = "git+https"
578 end 576 end
579 577
580 local protocol = rockspec.source.protocol 578 local protocol = rockspec.source.protocol
581 local ok, err, proto 579 local ok, err, proto
580
582 if dir.is_basic_protocol(protocol) then 581 if dir.is_basic_protocol(protocol) then
583 proto = fetch 582 proto = fetch
584 else 583 else
585 ok, proto = pcall(require, "luarocks.fetch."..protocol:gsub("[+-]", "_")) 584 local protocolfilename = protocol:gsub("[+-]", "_")
586 if not ok then 585 if protocolfilename == "cvs" then
587 return nil, "Unknown protocol "..protocol 586 ok, proto = pcall(require, "luarocks.fetch.cvs")
587 if not ok then
588 return nil, "Unknown protocol " .. protocol
589 end
590 elseif protocolfilename == "git_file" then
591 ok, proto = pcall(require, "luarocks.fetch.git_file")
592 if not ok then
593 return nil, "Unknown protocol " .. protocol
594 end
595 elseif protocolfilename == "git_http" then
596 ok, proto = pcall(require, "luarocks.fetch.git_http")
597 if not ok then
598 return nil, "Unknown protocol " .. protocol
599 end
600 elseif protocolfilename == "git_https" then
601 ok, proto = pcall(require, "luarocks.fetch.git_https")
602 if not ok then
603 return nil, "Unknown protocol " .. protocol
604 end
605 elseif protocolfilename == "git_ssh" then
606 ok, proto = pcall(require, "luarocks.fetch.git_ssh")
607 if not ok then
608 return nil, "Unknown protocol " .. protocol
609 end
610 elseif protocolfilename == "git" then
611 ok, proto = pcall(require, "luarocks.fetch.git")
612 if not ok then
613 return nil, "Unknown protocol " .. protocol
614 end
615 elseif protocolfilename == "hg_http" then
616 ok, proto = pcall(require, "luarocks.fetch.hg_http")
617 if not ok then
618 return nil, "Unknown protocol " .. protocol
619 end
620 elseif protocolfilename == "hg_https" then
621 ok, proto = pcall(require, "luarocks.fetch.hg_https")
622 if not ok then
623 return nil, "Unknown protocol " .. protocol
624 end
625 elseif protocolfilename == "hg_ssh" then
626 ok, proto = pcall(require, "luarocks.fetch.hg_ssh")
627 if not ok then
628 return nil, "Unknown protocol " .. protocol
629 end
630 elseif protocolfilename == "hg" then
631 ok, proto = pcall(require, "luarocks.fetch.hg")
632 if not ok then
633 return nil, "Unknown protocol " .. protocol
634 end
635 elseif protocolfilename == "sscm" then
636 ok, proto = pcall(require, "luarocks.fetch.sscm")
637 if not ok then
638 return nil, "Unknown protocol " .. protocol
639 end
640 elseif protocolfilename == "svn" then
641 ok, proto = pcall(require, "luarocks.fetch.svn")
642 if not ok then
643 return nil, "Unknown protocol " .. protocol
644 end
645 else
646 return nil, "Unknown protocol " .. protocol
588 end 647 end
589 end 648 end
590 649
591 if cfg.only_sources_from 650 if cfg.only_sources_from and
592 and rockspec.source.pathname 651 rockspec.source.pathname and
593 and #rockspec.source.pathname > 0 then 652 #rockspec.source.pathname > 0 then
594 if #cfg.only_sources_from == 0 then 653 if #cfg.only_sources_from == 0 then
595 return nil, "Can't download "..rockspec.source.url.." -- download from remote servers disabled" 654 return nil, "Can't download " .. rockspec.source.url .. " -- download from remote servers disabled"
596 elseif rockspec.source.pathname:find(cfg.only_sources_from, 1, true) ~= 1 then 655 elseif rockspec.source.pathname:find(cfg.only_sources_from, 1, true) ~= 1 then
597 return nil, "Can't download "..rockspec.source.url.." -- only downloading from "..cfg.only_sources_from 656 return nil, "Can't download " .. rockspec.source.url .. " -- only downloading from " .. cfg.only_sources_from
598 end 657 end
599 end 658 end
600 659
diff --git a/src/luarocks/fetch.tl b/src/luarocks/fetch.tl
index 5e6acff1..5a42645b 100644
--- a/src/luarocks/fetch.tl
+++ b/src/luarocks/fetch.tl
@@ -587,11 +587,63 @@ function fetch.fetch_sources(rockspec: Rockspec, extract: boolean, dest_dir?: st
587 if not ok then 587 if not ok then
588 return nil, "Unknown protocol "..protocol 588 return nil, "Unknown protocol "..protocol
589 end 589 end
590 elseif protocolfilename == "cvs" then 590 elseif protocolfilename == "git_file" then
591 ok, proto = pcall(require, "luarocks.fetch.cvs") as (boolean, Fetch) 591 ok, proto = pcall(require, "luarocks.fetch.git_file") as (boolean, Fetch)
592 if not ok then
593 return nil, "Unknown protocol "..protocol
594 end
595 elseif protocolfilename == "git_http" then
596 ok, proto = pcall(require, "luarocks.fetch.git_http") as (boolean, Fetch)
597 if not ok then
598 return nil, "Unknown protocol "..protocol
599 end
600 elseif protocolfilename == "git_https" then
601 ok, proto = pcall(require, "luarocks.fetch.git_https") as (boolean, Fetch)
602 if not ok then
603 return nil, "Unknown protocol "..protocol
604 end
605 elseif protocolfilename == "git_ssh" then
606 ok, proto = pcall(require, "luarocks.fetch.git_ssh") as (boolean, Fetch)
607 if not ok then
608 return nil, "Unknown protocol "..protocol
609 end
610 elseif protocolfilename == "git" then
611 ok, proto = pcall(require, "luarocks.fetch.git") as (boolean, Fetch)
612 if not ok then
613 return nil, "Unknown protocol "..protocol
614 end
615 elseif protocolfilename == "hg_http" then
616 ok, proto = pcall(require, "luarocks.fetch.hg_http") as (boolean, Fetch)
592 if not ok then 617 if not ok then
593 return nil, "Unknown protocol "..protocol 618 return nil, "Unknown protocol "..protocol
594 end 619 end
620 elseif protocolfilename == "hg_https" then
621 ok, proto = pcall(require, "luarocks.fetch.hg_https") as (boolean, Fetch)
622 if not ok then
623 return nil, "Unknown protocol "..protocol
624 end
625 elseif protocolfilename == "hg_ssh" then
626 ok, proto = pcall(require, "luarocks.fetch.hg_ssh") as (boolean, Fetch)
627 if not ok then
628 return nil, "Unknown protocol "..protocol
629 end
630 elseif protocolfilename == "hg" then
631 ok, proto = pcall(require, "luarocks.fetch.hg") as (boolean, Fetch)
632 if not ok then
633 return nil, "Unknown protocol "..protocol
634 end
635 elseif protocolfilename == "sscm" then
636 ok, proto = pcall(require, "luarocks.fetch.sscm") as (boolean, Fetch)
637 if not ok then
638 return nil, "Unknown protocol "..protocol
639 end
640 elseif protocolfilename == "svn" then
641 ok, proto = pcall(require, "luarocks.fetch.svn") as (boolean, Fetch)
642 if not ok then
643 return nil, "Unknown protocol "..protocol
644 end
645 else
646 return nil, "Unknown protocol "..protocol
595 end 647 end
596 end 648 end
597 649