From 8dd1e1460282f5fc992d838ec2411f4d1bc7b2b7 Mon Sep 17 00:00:00 2001 From: Hisham Muhammad Date: Mon, 2 Jul 2018 19:27:48 -0300 Subject: Experimental Windows cross-compiled binary build This commit consolidates the work that has been ongoing over the last few weeks in producing the single-binary builds of LuaRocks based on the new distribution model described in https://github.com/luarocks/luarocks/wiki/Project:-LuaRocks-new-distribution-model The single-binary build is in a good shape for Linux, it's a work-in-progress for Windows (binaries do build, but some work on the dependencies is still necessary), and is untested in macOS. --- binary/Makefile.windows | 59 ++++++++++++++++ binary/all_in_one | 145 +++++++++++++++++++++++++------------- binary/lua-zlib-1.2-0.rockspec | 39 ++++++++++ binary/luasec-0.7alpha-2.rockspec | 9 ++- binary/static-gcc | 25 +++++-- 5 files changed, 221 insertions(+), 56 deletions(-) create mode 100644 binary/Makefile.windows create mode 100644 binary/lua-zlib-1.2-0.rockspec (limited to 'binary') diff --git a/binary/Makefile.windows b/binary/Makefile.windows new file mode 100644 index 00000000..ea8f0a60 --- /dev/null +++ b/binary/Makefile.windows @@ -0,0 +1,59 @@ + +# "i686-w64-mingw32" or "x86_64-w64-mingw32" +MINGW_PREFIX=i686-w64-mingw32 +# sysroot of your mingw-w64 installation +MINGW_SYSROOT=/usr/lib/mingw-w64-sysroot/$(MINGW_PREFIX) +# "mingw" or "mingw64" +OPENSSL_PLATFORM=mingw +# Versions of dependencies +LIBLUA_VERSION=5.3.4 +OPENSSL_VERSION=1.0.2o +ZLIB_VERSION=1.2.11 + +windows-binary: windows-deps/lib/liblua.a windows-deps/lib/libssl.a windows-deps/lib/libz.a + STATIC_GCC_AR=$(MINGW_PREFIX)-ar \ + STATIC_GCC_RANLIB=$(MINGW_PREFIX)-ranlib \ + STATIC_GCC_CC=$(MINGW_PREFIX)-gcc \ + STATIC_GCC_LIBDIR=$(MINGW_SYSROOT)/lib \ + LUAROCKS_CROSS_COMPILING=1 \ + make binary LUA_DIR=$(PWD)/windows-deps BINARY_CC=$(MINGW_PREFIX)-gcc BINARY_NM=$(MINGW_PREFIX)-nm BINARY_PLATFORM=windows BINARY_TARGET=build-windows-binary BINARY_SYSROOT=$(MINGW_SYSROOT) + +build-windows-deps/lua-$(LIBLUA_VERSION).tar.gz: + mkdir -p build-windows-deps + cd build-windows-deps && curl -OL https://www.lua.org/ftp/lua-$(LIBLUA_VERSION).tar.gz +build-windows-deps/lua-$(LIBLUA_VERSION): build-windows-deps/lua-$(LIBLUA_VERSION).tar.gz + cd build-windows-deps && tar zxvpf lua-$(LIBLUA_VERSION).tar.gz +windows-deps/lib/liblua.a: build-windows-deps/lua-$(LIBLUA_VERSION) + cd build-windows-deps/lua-$(LIBLUA_VERSION)/src && make LUA_A=liblua.a CC=$(MINGW_PREFIX)-gcc AR="$(MINGW_PREFIX)-ar rcu" RANLIB=$(MINGW_PREFIX)-ranlib SYSCFLAGS= SYSLIBS= SYSLDFLAGS= liblua.a + mkdir -p windows-deps/include + cd build-windows-deps/lua-$(LIBLUA_VERSION)/src && cp lauxlib.h lua.h lua.hpp luaconf.h lualib.h ../../../windows-deps/include + mkdir -p windows-deps/lib + cd build-windows-deps/lua-$(LIBLUA_VERSION)/src && cp liblua.a ../../../windows-deps/lib + +build-windows-deps/openssl-$(OPENSSL_VERSION).tar.gz: + mkdir -p build-windows-deps + cd build-windows-deps && curl -OL https://www.openssl.org/source/openssl-$(OPENSSL_VERSION).tar.gz +build-windows-deps/openssl-$(OPENSSL_VERSION): build-windows-deps/openssl-$(OPENSSL_VERSION).tar.gz + cd build-windows-deps && tar zxvpf openssl-$(OPENSSL_VERSION).tar.gz +windows-deps/lib/libssl.a: build-windows-deps/openssl-$(OPENSSL_VERSION) + cd build-windows-deps/openssl-$(OPENSSL_VERSION) && ./Configure --prefix=$(PWD)/windows-deps --cross-compile-prefix=$(MINGW_PREFIX)- $(OPENSSL_PLATFORM) + cd build-windows-deps/openssl-$(OPENSSL_VERSION) && make install_sw + +build-windows-deps/zlib-$(ZLIB_VERSION).tar.gz: + mkdir -p build-windows-deps + cd build-windows-deps && curl -OL https://www.zlib.net/zlib-$(ZLIB_VERSION).tar.gz +build-windows-deps/zlib-$(ZLIB_VERSION): build-windows-deps/zlib-$(ZLIB_VERSION).tar.gz + cd build-windows-deps && tar zxvpf zlib-$(ZLIB_VERSION).tar.gz +windows-deps/lib/libz.a: build-windows-deps/zlib-$(ZLIB_VERSION) + cd build-windows-deps/zlib-$(ZLIB_VERSION) && sed -ie "s,dllwrap,$(MINGW_PREFIX)-dllwrap," win32/Makefile.gcc + cd build-windows-deps/zlib-$(ZLIB_VERSION) && ./configure --prefix=$(PWD)/windows-deps --static + cd build-windows-deps/zlib-$(ZLIB_VERSION) && make -f win32/Makefile.gcc CC=$(MINGW_PREFIX)-gcc AR=$(MINGW_PREFIX)-ar RC=$(MINGW_PREFIX)-windres STRIP=$(MINGW_PREFIX)-strip IMPLIB=libz.dll.a + mkdir -p windows-deps/include + cd build-windows-deps/zlib-$(ZLIB_VERSION) && cp zlib.h zconf.h ../../windows-deps/include + cd build-windows-deps/zlib-$(ZLIB_VERSION) && $(MINGW_PREFIX)-strip -g libz.a + mkdir -p windows-deps/lib + cd build-windows-deps/zlib-$(ZLIB_VERSION) && cp libz.a ../../windows-deps/lib + +windows-clean: + rm -rf windows-deps + rm -rf build-windows-binary diff --git a/binary/all_in_one b/binary/all_in_one index de545d04..9c7a690c 100755 --- a/binary/all_in_one +++ b/binary/all_in_one @@ -30,6 +30,10 @@ local LUA_DIR = arg[2] or "/usr" local EXCLUDE = arg[3] or "^src/luarocks/admin/" local SYSCONFDIR = arg[4] or "/etc/luarocks" local TARGET = arg[5] or "binary-build" +local MY_PLATFORM = arg[6] or "unix" +local CC = arg[7] or "gcc" +local NM = arg[8] or "nm" +local CROSSCOMPILER_SYSROOT = arg[9] or "/usr/lib/mingw-w64-sysroot/i686-w64-mingw32" local LUA_MODULES = TARGET .. "/lua_modules" local CONFIG_DIR = TARGET .. "/.luarocks" @@ -48,6 +52,12 @@ local persist = require("luarocks.persist") -------------------------------------------------------------------------------- +local function if_platform(plat, val) + if MY_PLATFORM == plat then + return val + end +end + local function reindent_c(input) local out = {} local indent = 0 @@ -110,10 +120,18 @@ local c_preamble = [[ static int registry_key; +/* fatal error, from srlua */ +static void fatal(const char* message) { + alert(message); + exit(EXIT_FAILURE); +} + ]] local function bin2c_file(out, filename) - local content = string.dump(assert(loadfile(filename))) + local fd = io.open(filename, "rb") + local content = fd:read("*a"):gsub("^#![^\n]+\n", "") + fd:close() table.insert(out, ("static const unsigned char code[] = {")) table.insert(out, hexdump(content)) table.insert(out, ("};")) @@ -121,24 +139,32 @@ end local function write_hardcoded_module(dir) - local system = util.popen_read("uname -s") - local processor = util.popen_read("uname -m") + local system = if_platform("windows", "MINGW") + local processor = if_platform("windows", "x86") - if processor:match("i[%d]86") then - processor = "x86" - elseif processor:match("amd64") or processor:match("x86_64") then - processor = "x86_64" - elseif processor:match("Power Macintosh") then - processor = "powerpc" + if not system then + system = util.popen_read("uname -s") + processor = util.popen_read("uname -m") + + if processor:match("i[%d]86") then + processor = "x86" + elseif processor:match("amd64") or processor:match("x86_64") then + processor = "x86_64" + elseif processor:match("Power Macintosh") then + processor = "powerpc" + end end local hardcoded = { SYSTEM = system, PROCESSOR = processor, - SYSCONFDIR = SYSCONFDIR, - LUA_DIR = cfg.variables.LUA_DIR, - LUA_BINDIR = cfg.variables.LUA_BINDIR, - LUA_INTERPRETER = cfg.lua_interpreter, + + SYSCONFDIR = if_platform("unix", SYSCONFDIR), + + LUA_DIR = if_platform("unix", cfg.variables.LUA_DIR), + LUA_BINDIR = if_platform("unix", cfg.variables.LUA_BINDIR) + or if_platform("windows", "."), + LUA_INTERPRETER = if_platform("unix", cfg.lua_interpreter), } local name = dir .. "/luarocks/core/hardcoded.lua" @@ -187,7 +213,7 @@ local function declare_modules(out, dirs, skip) end local function nm(filename) - local pd = io.popen("nm " .. filename) + local pd = io.popen(NM .. " " .. filename) local out = pd:read("*a") pd:close() return out @@ -244,7 +270,9 @@ end local function load_main(out, main_program, program_name) table.insert(out, [[static void load_main(lua_State* L) {]]) bin2c_file(out, main_program) - table.insert(out, ("luaL_loadbuffer(L, code, sizeof(code), %q);"):format(program_name)) + table.insert(out, ("if(luaL_loadbuffer(L, code, sizeof(code), %q) != LUA_OK) {"):format(program_name)) + table.insert(out, (" fatal(lua_tostring(L, -1));")) + table.insert(out, ("}")) table.insert(out, [[}]]) table.insert(out, [[]]) end @@ -262,7 +290,7 @@ static int pkg_loader(lua_State* L) { lua_pop(L, 1); /* modname ? modules */ lua_pushvalue(L, 1); /* modname ? modules modname */ lua_pushliteral(L, ".init"); /* modname ? modules modname ".init" */ - lua_concat(L, 2); /* modname ? modules modname .. ".init" */ + lua_concat(L, 2); /* modname ? modules modname..".init" */ lua_gettable(L, -2); /* modname ? mod */ } return 1; @@ -306,12 +334,6 @@ static int pmain(lua_State *L) { return 0; } -/* fatal error, from srlua */ -static void fatal(const char* message) { - alert(message); - exit(EXIT_FAILURE); -} - /* error handler, from luac */ static int msghandler (lua_State *L) { /* is error object not a string? */ @@ -358,14 +380,25 @@ int main(int argc, char** argv) { ]] +local function filter_in(f, xs) + for i = #xs, 1, -1 do + if not f(xs[i]) then + table.remove(xs, i) + end + end + return xs +end + +local function nonnull(x) return x ~= nil end + local function generate(main_program, dir, skip) local program_name = main_program:gsub(".*/", "") local hardcoded = write_hardcoded_module(dir) local out = {} - table.insert(out, c_preamble) table.insert(out, ([[static const char* progname = %q;]]):format(program_name)) + table.insert(out, c_preamble) load_main(out, main_program, program_name) local lua_modules = LUA_MODULES .. "/share/lua/" .. cfg.lua_version declare_modules(out, { dir, lua_modules }, skip) @@ -378,17 +411,23 @@ local function generate(main_program, dir, skip) local fd = io.open(c_filename, "w") fd:write(reindent_c(table.concat(out, "\n"))) fd:close() + + deps.check_lua(cfg.variables) - cmd = table.concat({ - "gcc", "-o", TARGET .. "/" .. program_name .. ".exe", + cmd = table.concat(filter_in(nonnull, { + CC, "-o", TARGET .. "/" .. program_name .. ".exe", "-I", cfg.variables.LUA_INCDIR, - "-rdynamic", + if_platform("unix", "-rdynamic"), "-Os", c_filename, "-L", cfg.variables.LUA_LIBDIR, table.concat(a_files, " "), + --if_platform("unix", cfg.variables.LUA_LIBDIR .. "/" .. cfg.variables.LUALIB:gsub("%.so.*$", ".a")), + --if_platform("windows", "mingw/liblua.a"), -- FIXME cfg.variables.LUA_LIBDIR .. "/" .. cfg.variables.LUALIB:gsub("%.so.*$", ".a"), - "-ldl", "-lm"}, " ") + if_platform("unix", "-ldl"), + "-lm" + }), " ") print(cmd) os.execute(cmd) end @@ -397,46 +436,56 @@ end local function main() - cfg.init(cmd.find_lua(LUA_DIR)) + os.remove("src/luarocks/core/hardcoded.lua") + cfg.init() + cfg.variables.LUA_DIR = LUA_DIR + cfg.variables.LUA_INCDIR = nil -- let it autodetect later + cfg.variables.LUA_LIBDIR = nil -- let it autodetect later fs.init() - deps.check_lua(cfg.variables) path.use_tree("./" .. LUA_MODULES) - + local CONFIG_FILE = CONFIG_DIR .. "/config-" .. cfg.lua_version .. ".lua" fs.make_dir(CONFIG_DIR) - local fd = io.open(CONFIG_FILE, "w") - fd:write([[ - lib_extension = "a" - external_lib_extension = "a" - variables = { - CC = "]] .. fs.current_dir() .. [[/static-gcc", - LD = "]] .. fs.current_dir() .. [[/static-gcc", - LIB_EXTENSION = "a", - LIBFLAG = "-static", - } - ]]) - fd:close() + + persist.save_from_table(CONFIG_FILE, { + lib_extension = "a", + external_lib_extension = "a", + variables = { + CC = fs.current_dir() .. "/binary/static-gcc", + LD = fs.current_dir() .. "/binary/static-gcc", + LIB_EXTENSION = "a", + LIBFLAG = "-static", + }, + platforms = if_platform("windows", { "windows", "win32", "mingw32" }), + external_deps_dirs = if_platform("windows", { CROSSCOMPILER_SYSROOT, fs.current_dir().."/windows-deps" }), + }) local dependencies = { md5 = "md5", - luazip = "luazip", - luasec = "./luasec-0.7alpha-2.rockspec", - luaposix = "./luaposix-34.0.4-1.rockspec", + luazip = if_platform("unix", "luazip"), + luasec = "./binary/luasec-0.7alpha-2.rockspec", + luaposix = if_platform("unix", "./binary/luaposix-34.0.4-1.rockspec"), luasocket = "luasocket", - ["lua-zlib"] = "lua-zlib", + ["lua-zlib"] = "./binary/lua-zlib-1.2-0.rockspec", luafilesystem = "luafilesystem", } fs.make_dir(LUA_MODULES) for name, arg in pairs(dependencies) do + print("----------------------------------------------------------------") + print(name) + print("----------------------------------------------------------------") local vers = manif.get_versions(queries.from_dep_string(name), "one") if not next(vers) then - os.execute("LUAROCKS_CONFIG='" .. CONFIG_FILE .. "' ./luarocks install '--tree=" .. LUA_MODULES .. "' " .. arg) + local ok = os.execute("LUAROCKS_CONFIG='" .. CONFIG_FILE .. "' ./luarocks install '--tree=" .. LUA_MODULES .. "' " .. arg) + if ok ~= 0 and ok ~= true then + error("Failed building dependency: " .. name) + end end end - generate(MAIN_PROGRAM, "src", { EXCLUDE, "^bin/?" }) + generate(MAIN_PROGRAM, "src", { EXCLUDE, "core/site_config", "^bin/?" }) end main() diff --git a/binary/lua-zlib-1.2-0.rockspec b/binary/lua-zlib-1.2-0.rockspec new file mode 100644 index 00000000..dd0c4a15 --- /dev/null +++ b/binary/lua-zlib-1.2-0.rockspec @@ -0,0 +1,39 @@ +package = "lua-zlib" +version = "1.2-0" +source = { + url = "git://github.com/brimworks/lua-zlib.git", + tag = "v1.2", +} +description = { + summary = "Simple streaming interface to zlib for Lua.", + detailed = [[ + Simple streaming interface to zlib for Lua. + Consists of two functions: inflate and deflate. + Both functions return "stream functions" (takes a buffer of input and returns a buffer of output). + This project is hosted on github. + ]], + homepage = "https://github.com/brimworks/lua-zlib", + license = "MIT" +} +dependencies = { + "lua >= 5.1, <= 5.3" +} +external_dependencies = { + ZLIB = { + header = "zlib.h", + library = "z", + } +} + +build = { + type = "builtin", + modules = { + zlib = { + sources = { "lua_zlib.c" }, + libraries = { "z" }, + defines = { "LZLIB_COMPAT" }, + incdirs = { "$(ZLIB_INCDIR)" }, + libdirs = { "$(ZLIB_LIBDIR)" }, + } + }, +} diff --git a/binary/luasec-0.7alpha-2.rockspec b/binary/luasec-0.7alpha-2.rockspec index 2736ca15..89d516b5 100644 --- a/binary/luasec-0.7alpha-2.rockspec +++ b/binary/luasec-0.7alpha-2.rockspec @@ -26,6 +26,11 @@ external_dependencies = { header = "openssl/ssl.h", } }, + mingw32 = { + OPENSSL = { + library = "ssl", + } + }, } } build = { @@ -60,11 +65,11 @@ build = { } } }, - mingw = { + mingw32 = { modules = { ssl = { libraries = { - "ssl", "crypto", + "ssl", "crypto", "ws2_32", "gdi32" } } } diff --git a/binary/static-gcc b/binary/static-gcc index a4d865d5..c08f24b2 100755 --- a/binary/static-gcc +++ b/binary/static-gcc @@ -1,5 +1,10 @@ #!/usr/bin/env bash +STATIC_GCC_AR=${STATIC_GCC_AR:-ar} +STATIC_GCC_RANLIB=${STATIC_GCC_RANLIB:-ranlib} +STATIC_GCC_CC=${STATIC_GCC_CC:-gcc} +STATIC_GCC_LIBDIR=${STATIC_GCC_LIBDIR:-/usr/lib} + DIR="$( cd "$( dirname "$0" )" && pwd )" function log() { echo -- "$@" >> $DIR/log.txt; } @@ -13,7 +18,7 @@ allargs=() sources=() objects=() etc=() -libdirs=("/usr/lib") +libdirs=("$STATIC_GCC_LIBDIR") incdirs=() linking=0 @@ -94,15 +99,22 @@ done staticlibs=() for lib in "${libs[@]}" do + found=0 for libdir in "${libdirs[@]}" do staticlib="$libdir/lib$lib.a" if [ -e "$staticlib" ] then staticlibs+=("$staticlib") + found=1 break fi done + if [ "$found" = 0 ] + then + log "STATICLIB not found for $lib" + runlog exit 1 + fi done oflag=() @@ -119,7 +131,7 @@ then for source in "${sources[@]}" do object="${source%.c}.o" - runlog gcc "${incdirs[@]}" "${etc[@]}" -c -o "$object" "$source" + runlog $STATIC_GCC_CC "${incdirs[@]}" "${etc[@]}" -c -o "$object" "$source" [ "$?" = 0 ] || runlog exit $? objects+=("$object") done @@ -137,22 +149,23 @@ then done echo "SAVE" >> ar.script echo "END" >> ar.script - cat ar.script | ar -M + cat ar.script >> "$DIR/log.txt" + cat ar.script | $STATIC_GCC_AR -M [ "$?" = 0 ] || runlog exit $? [ -e "$output" ] || { exit 1 } - runlog ranlib "$output" + runlog $STATIC_GCC_RANLIB "$output" runlog exit $? elif [ "$object" = 1 ] then log OBJECT - runlog gcc "${oflag[@]}" "${incdirs[@]}" "${etc[@]}" "${sources[@]}" + runlog $STATIC_GCC_CC "${oflag[@]}" "${incdirs[@]}" "${etc[@]}" "${sources[@]}" runlog exit $? else log EXECUTABLE - runlog gcc "${allargs[@]}" + runlog $STATIC_GCC_CC "${allargs[@]}" runlog exit $? fi -- cgit v1.2.3-55-g6feb