WORKSPACE: use nix for python/go if available
This introduces Nix, the package manager, and nixpkgs, the package
collection, into hscloud's bazel build machinery.
There are two reasons behind this:
- on NixOS, it's painful or at least very difficult to run hscloud out
of the box. Especially with rules_go, that download a blob from the
Internet to get a Go toolchain, it just fails outright. This solves
this and allows hscloud to be used on NixOS.
- on non-NixOS platforms that still might have access to Nix this
allows to somewhat hermeticize the build. Notably, Python now comes
from nixpkgs, and is fabricobbled in a way that makes pip3_import
use Nix system dependencies for ncurses and libpq.
This has been tested to run ci_presubmit on NixOS 20.09pre and Gentoo
~amd64.
Change-Id: Ic16e4827cb52a05aea0df0eed84d80c5e9ae0e07
diff --git a/third_party/nix/BUILD b/third_party/nix/BUILD
new file mode 100644
index 0000000..c0821e6
--- /dev/null
+++ b/third_party/nix/BUILD
@@ -0,0 +1,21 @@
+load("@rules_python//python:defs.bzl", "py_runtime_pair")
+
+# Python3 toolchain definition that uses //third_party/nix:python.nix (via
+# external repository).
+
+py_runtime(
+ name = "py3_runtime",
+ interpreter = "@hscloud_nix_python3//:python3",
+ python_version = "PY3",
+)
+
+py_runtime_pair(
+ name = "py_runtime_pair",
+ py3_runtime = ":py3_runtime",
+)
+
+toolchain(
+ name = "py_toolchain",
+ toolchain = ":py_runtime_pair",
+ toolchain_type = "@rules_python//python:toolchain_type",
+)
diff --git a/third_party/nix/python.nix b/third_party/nix/python.nix
new file mode 100644
index 0000000..5571a60
--- /dev/null
+++ b/third_party/nix/python.nix
@@ -0,0 +1,46 @@
+# This is a Python interpreter wrapper that's passed to pip3_import under
+# NixOS.
+# It allows us to build some pip wheels under NixOS that require special
+# system libraries. This is quite hacky, it would be much better if we could
+# somehow tell pip3_import that a given package needs to be built within a
+# given environment.
+
+with import <nixpkgs> {};
+
+let
+ # Add cffi for import _cffi_backend in `cryptography` to work.
+ py = pkgs.python37.withPackages (ps: with ps; [ cffi ]);
+
+# We use mkDerivation instead of writeScript or writeScriptBin as we need a
+# derivation that both:
+# - has a directory structure (for rules_nixpkgs to be able to use it)
+# - has the Python interpreter directly in that structure and not in bin/, as
+# rules_python's pip3_import interpreter_path requires a file target, and
+# will not take an alias. Meanwhile, rules_nixpkgs only creates a BUILD file
+# in the root path of the external repository (which is populated with a
+# symlink tree from the nix derivation), so we can onlly directly reference
+# file in the root of a Nix derivation.
+in stdenv.mkDerivation {
+ name = "py-wrapper";
+ version = "1.0";
+ src = ./.;
+ unpackPhase = "";
+ buildPhase = ''
+ mkdir -p $out
+ cat > $out/python3 <<EOF
+#!/bin/bash
+
+# pyscopg wants libpq, and uses pg_config to find paths. Inject pg_config into
+# the Python interpreter's path.
+export PATH="${pkgs.postgresql}/bin:\$PATH"
+
+# uWSGI has a truly cheese-grade build system, and this is the only way to let
+# it know where to find ncurses.
+export LDFLAGS="-L${pkgs.ncurses}/lib"
+exec ${py}/bin/python3 "\$@"
+EOF
+ '';
+ installPhase = ''
+ chmod +x $out/python3
+ '';
+}
diff --git a/third_party/nix/repository_rules.bzl b/third_party/nix/repository_rules.bzl
new file mode 100644
index 0000000..1a02962
--- /dev/null
+++ b/third_party/nix/repository_rules.bzl
@@ -0,0 +1,97 @@
+load("@io_tweag_rules_nixpkgs//nixpkgs:repositories.bzl", "rules_nixpkgs_dependencies")
+load("@io_tweag_rules_nixpkgs//nixpkgs:nixpkgs.bzl", "nixpkgs_git_repository", "nixpkgs_package")
+
+def has_nix(ctx):
+ return ctx.which("nix-build") != None
+
+def _hscloud_gen_go_imports_impl(ctx):
+ ctx.file("BUILD", "")
+
+ imports_for_nix = """
+load("@io_tweag_rules_nixpkgs//nixpkgs:toolchains/go.bzl", "nixpkgs_go_configure")
+
+def hscloud_go_register_toolchains():
+ nixpkgs_go_configure(repository = "@nixpkgs")
+"""
+ imports_for_non_nix = """
+load("@io_bazel_rules_go//go:deps.bzl", "go_rules_dependencies", "go_register_toolchains")
+def hscloud_go_register_toolchains():
+ go_register_toolchains()
+"""
+
+ if has_nix(ctx):
+ ctx.file("imports.bzl", imports_for_nix)
+ else:
+ ctx.file("imports.bzl", imports_for_non_nix)
+
+# Generate repository containing either a call to go_register_toolchains() or
+# nixpkgs_go_configure(), depending on nix presence.
+hscloud_gen_go_imports = repository_rule(
+ implementation = _hscloud_gen_go_imports_impl,
+ attrs = dict(),
+)
+
+def _hscloud_gen_pip_imports_impl(ctx):
+ ctx.file("BUILD", "")
+
+ # For Nix, we have to both pass our interpreter to pip3_import, and also
+ # register it as a toolchain.
+ imports_for_nix = """
+load("@rules_python//python:pip.bzl", "pip3_import")
+def hscloud_pip3_import(name, requirements):
+ pip3_import(
+ name = name,
+ requirements = requirements,
+ python_interpreter_target = "@hscloud_nix_python3//:python3",
+ )
+ native.register_toolchains("//third_party/nix:py_toolchain")
+"""
+ imports_for_non_nix = """
+load("@rules_python//python:pip.bzl", "pip3_import")
+def hscloud_pip3_import(name, requirements):
+ pip3_import(
+ name = name,
+ requirements = requirements,
+ )
+"""
+ if has_nix(ctx):
+ ctx.file("imports.bzl", imports_for_nix)
+ else:
+ ctx.file("imports.bzl", imports_for_non_nix)
+
+# Generate repository containing a wrapped pip3_import that either uses the
+# host Python interpreter or one from nixpkgs, depending on nix presence.
+hscloud_gen_pip_imports = repository_rule(
+ implementation = _hscloud_gen_pip_imports_impl,
+ attrs = dict(),
+)
+
+def hscloud_setup_nix(revision, sha256):
+ rules_nixpkgs_dependencies()
+ nixpkgs_git_repository(
+ name = "nixpkgs",
+ revision = "1179840f9a88b8a548f4b11d1a03aa25a790c379",
+ sha256 = "8b64041bfb9760de9e797c0a985a4830880c21732489f397e217d877edd9a990",
+ )
+
+ # Load python3 from nixpkgs. Python is a large source of non-hermiticity,
+ # and loading it from nix vastly hermeticizes the build - well, at least to
+ # also be dependent on this Nix store state. That's still better than just
+ # grabbing whatever random system Python a user might have.
+ nixpkgs_package(
+ name = "hscloud_nix_python3",
+ repositories = { "nixpkgs": "@nixpkgs//:default.nix" },
+ nix_file = "//third_party/nix:python.nix",
+ build_file_content = """
+package(default_visibility = ["//visibility:public"])
+exports_files(["python3"])
+ """,
+ )
+
+ # Generate a Go toolchain setup workspace rule.
+ hscloud_gen_go_imports(
+ name = "hscloud_go_toolchain",
+ )
+ hscloud_gen_pip_imports(
+ name = "hscloud_pip_imports",
+ )