Experimental Windows cross-compiled binary build

This commit consolidates the work that has been ongoing over the
last few weeks in producing the single-binary builds of LuaRocks
based on the new distribution model described in
https://github.com/luarocks/luarocks/wiki/Project:-LuaRocks-new-distribution-model

The single-binary build is in a good shape for Linux,
it's a work-in-progress for Windows (binaries do build,
but some work on the dependencies is still necessary),
and is untested in macOS.

5 files changed, 221 insertions, 56 deletions

diff --git a/binary/Makefile.windows b/binary/Makefile.windows
new file mode 100644
index 00000000..ea8f0a60
--- /dev/null
+++ b/binary/Makefile.windows
@@ -0,0 +1,59 @@
+
+# "i686-w64-mingw32" or "x86_64-w64-mingw32"
+MINGW_PREFIX=i686-w64-mingw32
+# sysroot of your mingw-w64 installation
+MINGW_SYSROOT=/usr/lib/mingw-w64-sysroot/$(MINGW_PREFIX)
+# "mingw" or "mingw64"
+OPENSSL_PLATFORM=mingw
+# Versions of dependencies
+LIBLUA_VERSION=5.3.4
+OPENSSL_VERSION=1.0.2o
+ZLIB_VERSION=1.2.11
+
+windows-binary: windows-deps/lib/liblua.a windows-deps/lib/libssl.a windows-deps/lib/libz.a
+        STATIC_GCC_AR=$(MINGW_PREFIX)-ar \
+        STATIC_GCC_RANLIB=$(MINGW_PREFIX)-ranlib \
+        STATIC_GCC_CC=$(MINGW_PREFIX)-gcc \
+        STATIC_GCC_LIBDIR=$(MINGW_SYSROOT)/lib \
+        LUAROCKS_CROSS_COMPILING=1 \
+        make binary LUA_DIR=$(PWD)/windows-deps BINARY_CC=$(MINGW_PREFIX)-gcc BINARY_NM=$(MINGW_PREFIX)-nm BINARY_PLATFORM=windows BINARY_TARGET=build-windows-binary BINARY_SYSROOT=$(MINGW_SYSROOT)
+
+build-windows-deps/lua-$(LIBLUA_VERSION).tar.gz:
+        mkdir -p build-windows-deps
+        cd build-windows-deps && curl -OL https://www.lua.org/ftp/lua-$(LIBLUA_VERSION).tar.gz
+build-windows-deps/lua-$(LIBLUA_VERSION): build-windows-deps/lua-$(LIBLUA_VERSION).tar.gz
+        cd build-windows-deps && tar zxvpf lua-$(LIBLUA_VERSION).tar.gz
+windows-deps/lib/liblua.a: build-windows-deps/lua-$(LIBLUA_VERSION)
+        cd build-windows-deps/lua-$(LIBLUA_VERSION)/src && make LUA_A=liblua.a CC=$(MINGW_PREFIX)-gcc AR="$(MINGW_PREFIX)-ar rcu" RANLIB=$(MINGW_PREFIX)-ranlib SYSCFLAGS= SYSLIBS= SYSLDFLAGS= liblua.a
+        mkdir -p windows-deps/include
+        cd build-windows-deps/lua-$(LIBLUA_VERSION)/src && cp lauxlib.h lua.h lua.hpp luaconf.h lualib.h ../../../windows-deps/include
+        mkdir -p windows-deps/lib
+        cd build-windows-deps/lua-$(LIBLUA_VERSION)/src && cp liblua.a ../../../windows-deps/lib
+
+build-windows-deps/openssl-$(OPENSSL_VERSION).tar.gz:
+        mkdir -p build-windows-deps
+        cd build-windows-deps && curl -OL https://www.openssl.org/source/openssl-$(OPENSSL_VERSION).tar.gz
+build-windows-deps/openssl-$(OPENSSL_VERSION): build-windows-deps/openssl-$(OPENSSL_VERSION).tar.gz
+        cd build-windows-deps && tar zxvpf openssl-$(OPENSSL_VERSION).tar.gz
+windows-deps/lib/libssl.a: build-windows-deps/openssl-$(OPENSSL_VERSION)
+        cd build-windows-deps/openssl-$(OPENSSL_VERSION) && ./Configure --prefix=$(PWD)/windows-deps --cross-compile-prefix=$(MINGW_PREFIX)- $(OPENSSL_PLATFORM)
+        cd build-windows-deps/openssl-$(OPENSSL_VERSION) && make install_sw
+
+build-windows-deps/zlib-$(ZLIB_VERSION).tar.gz:
+        mkdir -p build-windows-deps
+        cd build-windows-deps && curl -OL https://www.zlib.net/zlib-$(ZLIB_VERSION).tar.gz
+build-windows-deps/zlib-$(ZLIB_VERSION): build-windows-deps/zlib-$(ZLIB_VERSION).tar.gz
+        cd build-windows-deps && tar zxvpf zlib-$(ZLIB_VERSION).tar.gz
+windows-deps/lib/libz.a: build-windows-deps/zlib-$(ZLIB_VERSION)
+        cd build-windows-deps/zlib-$(ZLIB_VERSION) && sed -ie "s,dllwrap,$(MINGW_PREFIX)-dllwrap," win32/Makefile.gcc
+        cd build-windows-deps/zlib-$(ZLIB_VERSION) && ./configure --prefix=$(PWD)/windows-deps --static
+        cd build-windows-deps/zlib-$(ZLIB_VERSION) && make -f win32/Makefile.gcc CC=$(MINGW_PREFIX)-gcc AR=$(MINGW_PREFIX)-ar RC=$(MINGW_PREFIX)-windres STRIP=$(MINGW_PREFIX)-strip IMPLIB=libz.dll.a
+        mkdir -p windows-deps/include
+        cd build-windows-deps/zlib-$(ZLIB_VERSION) && cp zlib.h zconf.h ../../windows-deps/include
+        cd build-windows-deps/zlib-$(ZLIB_VERSION) && $(MINGW_PREFIX)-strip -g libz.a
+        mkdir -p windows-deps/lib
+        cd build-windows-deps/zlib-$(ZLIB_VERSION) && cp libz.a ../../windows-deps/lib
+
+windows-clean:
+        rm -rf windows-deps
+        rm -rf build-windows-binary
diff --git a/binary/all_in_one b/binary/all_in_one
index de545d04..9c7a690c 100755
--- a/binary/all_in_one
+++ b/binary/all_in_one
@@ -30,6 +30,10 @@ local LUA_DIR = arg[2] or "/usr"
 local EXCLUDE = arg[3] or "^src/luarocks/admin/"
 local SYSCONFDIR = arg[4] or "/etc/luarocks"
 local TARGET = arg[5] or "binary-build"
+local MY_PLATFORM = arg[6] or "unix"
+local CC = arg[7] or "gcc"
+local NM = arg[8] or "nm"
+local CROSSCOMPILER_SYSROOT = arg[9] or "/usr/lib/mingw-w64-sysroot/i686-w64-mingw32"
 
 local LUA_MODULES = TARGET .. "/lua_modules"
 local CONFIG_DIR = TARGET .. "/.luarocks"
@@ -48,6 +52,12 @@ local persist = require("luarocks.persist")
 
 --------------------------------------------------------------------------------
 
+local function if_platform(plat, val)
+   if MY_PLATFORM == plat then
+      return val
+   end
+end
+
 local function reindent_c(input)
    local out = {}
    local indent = 0
@@ -110,10 +120,18 @@ local c_preamble = [[
 
 static int registry_key;
 
+/* fatal error, from srlua */
+static void fatal(const char* message) {
+   alert(message);
+   exit(EXIT_FAILURE);
+}
+
 ]]
 
 local function bin2c_file(out, filename)
-   local content = string.dump(assert(loadfile(filename)))
+   local fd = io.open(filename, "rb")
+   local content = fd:read("*a"):gsub("^#![^\n]+\n", "")
+   fd:close()
    table.insert(out, ("static const unsigned char code[] = {"))
    table.insert(out, hexdump(content))
    table.insert(out, ("};"))
@@ -121,24 +139,32 @@ end
 
 local function write_hardcoded_module(dir)
 
-   local system = util.popen_read("uname -s")
-   local processor = util.popen_read("uname -m")
+   local system = if_platform("windows", "MINGW")
+   local processor = if_platform("windows", "x86")
 
-   if processor:match("i[%d]86") then
-      processor = "x86"
-   elseif processor:match("amd64") or processor:match("x86_64") then
-      processor = "x86_64"
-   elseif processor:match("Power Macintosh") then
-      processor = "powerpc"
+   if not system then
+      system = util.popen_read("uname -s")
+      processor = util.popen_read("uname -m")
+
+      if processor:match("i[%d]86") then
+         processor = "x86"
+      elseif processor:match("amd64") or processor:match("x86_64") then
+         processor = "x86_64"
+      elseif processor:match("Power Macintosh") then
+         processor = "powerpc"
+      end
    end
 
    local hardcoded = {
       SYSTEM = system,
       PROCESSOR = processor,
-      SYSCONFDIR = SYSCONFDIR,
-      LUA_DIR = cfg.variables.LUA_DIR,
-      LUA_BINDIR = cfg.variables.LUA_BINDIR,
-      LUA_INTERPRETER = cfg.lua_interpreter,
+
+      SYSCONFDIR = if_platform("unix", SYSCONFDIR),
+
+      LUA_DIR = if_platform("unix", cfg.variables.LUA_DIR),
+      LUA_BINDIR = if_platform("unix", cfg.variables.LUA_BINDIR)
+                or if_platform("windows", "."),
+      LUA_INTERPRETER = if_platform("unix", cfg.lua_interpreter),
    }
    
    local name = dir .. "/luarocks/core/hardcoded.lua"
@@ -187,7 +213,7 @@ local function declare_modules(out, dirs, skip)
 end
 
 local function nm(filename)
-   local pd = io.popen("nm " .. filename)
+   local pd = io.popen(NM .. " " .. filename)
    local out = pd:read("*a")
    pd:close()
    return out
@@ -244,7 +270,9 @@ end
 local function load_main(out, main_program, program_name)
    table.insert(out, [[static void load_main(lua_State* L) {]])
    bin2c_file(out, main_program)
-   table.insert(out, ("luaL_loadbuffer(L, code, sizeof(code), %q);"):format(program_name))
+   table.insert(out, ("if(luaL_loadbuffer(L, code, sizeof(code), %q) != LUA_OK) {"):format(program_name))
+   table.insert(out, ("   fatal(lua_tostring(L, -1));"))
+   table.insert(out, ("}"))
    table.insert(out, [[}]])
    table.insert(out, [[]])
 end
@@ -262,7 +290,7 @@ static int pkg_loader(lua_State* L) {
       lua_pop(L, 1);                                /* modname ? modules */
       lua_pushvalue(L, 1);                          /* modname ? modules modname */
       lua_pushliteral(L, ".init");                  /* modname ? modules modname ".init" */
-      lua_concat(L, 2);                             /* modname ? modules modname .. ".init" */
+      lua_concat(L, 2);                             /* modname ? modules modname..".init" */
       lua_gettable(L, -2);                          /* modname ? mod */
    }
    return 1;
@@ -306,12 +334,6 @@ static int pmain(lua_State *L) {
    return 0;
 }
 
-/* fatal error, from srlua */
-static void fatal(const char* message) {
-   alert(message);
-   exit(EXIT_FAILURE);
-}
-
 /* error handler, from luac */
 static int msghandler (lua_State *L) {
    /* is error object not a string? */
@@ -358,14 +380,25 @@ int main(int argc, char** argv) {
 
 ]]
 
+local function filter_in(f, xs)
+   for i = #xs, 1, -1 do
+      if not f(xs[i]) then
+         table.remove(xs, i)
+      end
+   end
+   return xs
+end
+
+local function nonnull(x) return x ~= nil end
+
 local function generate(main_program, dir, skip)
    local program_name = main_program:gsub(".*/", "")
 
    local hardcoded = write_hardcoded_module(dir)
    
    local out = {}
-   table.insert(out, c_preamble)
    table.insert(out, ([[static const char* progname = %q;]]):format(program_name))
+   table.insert(out, c_preamble)
    load_main(out, main_program, program_name)
    local lua_modules = LUA_MODULES .. "/share/lua/" .. cfg.lua_version
    declare_modules(out, { dir, lua_modules }, skip)
@@ -378,17 +411,23 @@ local function generate(main_program, dir, skip)
    local fd = io.open(c_filename, "w")
    fd:write(reindent_c(table.concat(out, "\n")))
    fd:close()
+
+   deps.check_lua(cfg.variables)
    
-   cmd = table.concat({
-      "gcc", "-o", TARGET .. "/" .. program_name .. ".exe",
+   cmd = table.concat(filter_in(nonnull, {
+      CC, "-o", TARGET .. "/" .. program_name .. ".exe",
       "-I", cfg.variables.LUA_INCDIR,
-      "-rdynamic",
+      if_platform("unix", "-rdynamic"),
       "-Os",
       c_filename,
       "-L", cfg.variables.LUA_LIBDIR,
       table.concat(a_files, " "),
+      --if_platform("unix", cfg.variables.LUA_LIBDIR .. "/" .. cfg.variables.LUALIB:gsub("%.so.*$", ".a")),
+      --if_platform("windows", "mingw/liblua.a"), -- FIXME
       cfg.variables.LUA_LIBDIR .. "/" .. cfg.variables.LUALIB:gsub("%.so.*$", ".a"),
-      "-ldl", "-lm"}, " ")
+      if_platform("unix", "-ldl"),
+      "-lm"
+   }), " ")
    print(cmd)
    os.execute(cmd)
 end
@@ -397,46 +436,56 @@ end
 
 local function main()
 
-   cfg.init(cmd.find_lua(LUA_DIR))
+   os.remove("src/luarocks/core/hardcoded.lua")
+   cfg.init()
+   cfg.variables.LUA_DIR = LUA_DIR
+   cfg.variables.LUA_INCDIR = nil -- let it autodetect later
+   cfg.variables.LUA_LIBDIR = nil -- let it autodetect later
    fs.init()
-   deps.check_lua(cfg.variables)
    path.use_tree("./" .. LUA_MODULES)
-   
+
    local CONFIG_FILE = CONFIG_DIR .. "/config-" .. cfg.lua_version .. ".lua"
    
    fs.make_dir(CONFIG_DIR)
-   local fd = io.open(CONFIG_FILE, "w")
-   fd:write([[
-   lib_extension = "a"
-   external_lib_extension = "a"
-   variables = {
-      CC = "]] .. fs.current_dir() .. [[/static-gcc",
-      LD = "]] .. fs.current_dir() .. [[/static-gcc",
-      LIB_EXTENSION = "a",
-      LIBFLAG = "-static",
-   }
-   ]])
-   fd:close()
+
+   persist.save_from_table(CONFIG_FILE, {
+      lib_extension = "a",
+      external_lib_extension = "a",
+      variables = {
+         CC = fs.current_dir() .. "/binary/static-gcc",
+         LD = fs.current_dir() .. "/binary/static-gcc",
+         LIB_EXTENSION = "a",
+         LIBFLAG = "-static",
+      },
+      platforms = if_platform("windows", { "windows", "win32", "mingw32" }),
+      external_deps_dirs = if_platform("windows", { CROSSCOMPILER_SYSROOT, fs.current_dir().."/windows-deps" }),
+   })
    
    local dependencies = {
       md5 = "md5",
-      luazip = "luazip",
-      luasec = "./luasec-0.7alpha-2.rockspec",
-      luaposix = "./luaposix-34.0.4-1.rockspec",
+      luazip = if_platform("unix", "luazip"),
+      luasec = "./binary/luasec-0.7alpha-2.rockspec",
+      luaposix = if_platform("unix", "./binary/luaposix-34.0.4-1.rockspec"),
       luasocket = "luasocket",
-      ["lua-zlib"] = "lua-zlib",
+      ["lua-zlib"] = "./binary/lua-zlib-1.2-0.rockspec",
       luafilesystem = "luafilesystem",
    }
    
    fs.make_dir(LUA_MODULES)
    for name, arg in pairs(dependencies) do
+      print("----------------------------------------------------------------")
+      print(name)
+      print("----------------------------------------------------------------")
       local vers = manif.get_versions(queries.from_dep_string(name), "one")
       if not next(vers) then
-         os.execute("LUAROCKS_CONFIG='" .. CONFIG_FILE .. "' ./luarocks install '--tree=" .. LUA_MODULES .. "' " .. arg)
+         local ok = os.execute("LUAROCKS_CONFIG='" .. CONFIG_FILE .. "' ./luarocks install '--tree=" .. LUA_MODULES .. "' " .. arg)
+         if ok ~= 0 and ok ~= true then
+            error("Failed building dependency: " .. name)
+         end
       end
    end
    
-   generate(MAIN_PROGRAM, "src", { EXCLUDE, "^bin/?" })
+   generate(MAIN_PROGRAM, "src", { EXCLUDE, "core/site_config", "^bin/?" })
 end
 
 main()
diff --git a/binary/lua-zlib-1.2-0.rockspec b/binary/lua-zlib-1.2-0.rockspec
new file mode 100644
index 00000000..dd0c4a15
--- /dev/null
+++ b/binary/lua-zlib-1.2-0.rockspec
@@ -0,0 +1,39 @@
+package = "lua-zlib"
+version = "1.2-0"
+source = {
+   url = "git://github.com/brimworks/lua-zlib.git",
+   tag = "v1.2",
+}
+description = {
+   summary = "Simple streaming interface to zlib for Lua.",
+   detailed = [[
+      Simple streaming interface to zlib for Lua.
+      Consists of two functions: inflate and deflate.
+      Both functions return "stream functions" (takes a buffer of input and returns a buffer of output).
+      This project is hosted on github.
+   ]],
+   homepage = "https://github.com/brimworks/lua-zlib",
+   license = "MIT"
+}
+dependencies = {
+   "lua >= 5.1, <= 5.3"
+}
+external_dependencies = {
+    ZLIB = {
+       header = "zlib.h",
+       library = "z",
+    }
+}
+
+build = {
+   type = "builtin",
+   modules = {
+      zlib = {
+         sources = { "lua_zlib.c" },
+         libraries = { "z" },
+         defines = { "LZLIB_COMPAT" },
+         incdirs = { "$(ZLIB_INCDIR)" },
+         libdirs = { "$(ZLIB_LIBDIR)" },
+      }
+   },
+}
diff --git a/binary/luasec-0.7alpha-2.rockspec b/binary/luasec-0.7alpha-2.rockspec
index 2736ca15..89d516b5 100644
--- a/binary/luasec-0.7alpha-2.rockspec
+++ b/binary/luasec-0.7alpha-2.rockspec
@@ -26,6 +26,11 @@ external_dependencies = {
             header = "openssl/ssl.h",
          }
       },
+      mingw32 = {
+         OPENSSL = {
+            library = "ssl",
+         }
+      },
    }
 }
 build = {
@@ -60,11 +65,11 @@ build = {
             }
          }
       },
-      mingw = {
+      mingw32 = {
          modules = {
             ssl = {
                libraries = {
-                  "ssl", "crypto",
+                  "ssl", "crypto", "ws2_32", "gdi32"
                }
             }
          }
diff --git a/binary/static-gcc b/binary/static-gcc
index a4d865d5..c08f24b2 100755
--- a/binary/static-gcc
+++ b/binary/static-gcc
@@ -1,5 +1,10 @@
 #!/usr/bin/env bash
 
+STATIC_GCC_AR=${STATIC_GCC_AR:-ar}
+STATIC_GCC_RANLIB=${STATIC_GCC_RANLIB:-ranlib}
+STATIC_GCC_CC=${STATIC_GCC_CC:-gcc}
+STATIC_GCC_LIBDIR=${STATIC_GCC_LIBDIR:-/usr/lib}
+
 DIR="$( cd "$( dirname "$0" )" && pwd )"
 
 function log() { echo -- "$@" >> $DIR/log.txt; }
@@ -13,7 +18,7 @@ allargs=()
 sources=()
 objects=()
 etc=()
-libdirs=("/usr/lib")
+libdirs=("$STATIC_GCC_LIBDIR")
 incdirs=()
 
 linking=0
@@ -94,15 +99,22 @@ done
 staticlibs=()
 for lib in "${libs[@]}"
 do
+   found=0
    for libdir in "${libdirs[@]}"
    do
       staticlib="$libdir/lib$lib.a"
       if [ -e "$staticlib" ]
       then
          staticlibs+=("$staticlib")
+         found=1
          break
       fi
    done
+   if [ "$found" = 0 ]
+   then
+      log "STATICLIB not found for $lib"
+      runlog exit 1
+   fi
 done
 
 oflag=()
@@ -119,7 +131,7 @@ then
       for source in "${sources[@]}"
       do
          object="${source%.c}.o"
-         runlog gcc "${incdirs[@]}" "${etc[@]}" -c -o "$object" "$source"
+         runlog $STATIC_GCC_CC "${incdirs[@]}" "${etc[@]}" -c -o "$object" "$source"
          [ "$?" = 0 ] || runlog exit $?
          objects+=("$object")
       done
@@ -137,22 +149,23 @@ then
    done
    echo "SAVE" >> ar.script
    echo "END" >> ar.script
-   cat ar.script | ar -M
+   cat ar.script >> "$DIR/log.txt"
+   cat ar.script | $STATIC_GCC_AR -M
    [ "$?" = 0 ] || runlog exit $?
 
    [ -e "$output" ] || {
       exit 1
    }
 
-   runlog ranlib "$output"
+   runlog $STATIC_GCC_RANLIB "$output"
    runlog exit $?
 elif [ "$object" = 1 ]
 then
    log OBJECT
-   runlog gcc "${oflag[@]}" "${incdirs[@]}" "${etc[@]}" "${sources[@]}"
+   runlog $STATIC_GCC_CC "${oflag[@]}" "${incdirs[@]}" "${etc[@]}" "${sources[@]}"
    runlog exit $?
 else
    log EXECUTABLE
-   runlog gcc "${allargs[@]}"
+   runlog $STATIC_GCC_CC "${allargs[@]}"
    runlog exit $?
 fi