Merge "WORKSPACE: use nix for python/go if available"
diff --git a/WORKSPACE b/WORKSPACE
index 031d55a..6ecd658 100644
--- a/WORKSPACE
+++ b/WORKSPACE
@@ -7,7 +7,6 @@
# Protobuf deps (shared between many rules).
# Load this as early as possible, to avoid a different version being pulled in by deps of something else
-
http_archive(
name = "com_google_protobuf",
sha256 = "bb8ce9ba11eb7bccf080599fe7cad9cc461751c8dd1ba61701c0070d58cde973",
@@ -18,8 +17,27 @@
load("@com_google_protobuf//:protobuf_deps.bzl", "protobuf_deps")
protobuf_deps()
-# Go/Gazelle rules
+# Force rules_python at a bleeding edge version (for pip3_import).
+http_archive(
+ name = "rules_python",
+ url = "https://github.com/bazelbuild/rules_python/releases/download/0.0.3/rules_python-0.0.3.tar.gz",
+ sha256 = "e46612e9bb0dae8745de6a0643be69e8665a03f63163ac6610c210e80d14c3e4",
+)
+# Load and setup Nixpkgs, if Nix is present on the build system.
+http_archive(
+ name = "io_tweag_rules_nixpkgs",
+ strip_prefix = "rules_nixpkgs-dc24090573d74adcf38730422941fd69b87682c7",
+ urls = ["https://github.com/tweag/rules_nixpkgs/archive/dc24090573d74adcf38730422941fd69b87682c7.tar.gz"],
+ sha256 = "aca86baa64174478c57f74ed09d5c2313113abe94aa3af030486d1b14032d3ed",
+)
+load("//third_party/nix:repository_rules.bzl", "hscloud_setup_nix")
+hscloud_setup_nix(
+ revision = "1179841f9a88b8a548f4b11d1a03aa25a790c379",
+ sha256 = "8b64041bfb9760de9e797c0a985a4830880c21732489f397e217d877edd9a990",
+)
+
+# Download Go/Gazelle rules
http_archive(
name = "io_bazel_rules_go",
sha256 = "6a68e269802911fa419abb940c850734086869d7fe9bc8e12aaf60a09641c818",
@@ -28,7 +46,6 @@
"https://github.com/bazelbuild/rules_go/releases/download/v0.23.0/rules_go-v0.23.0.tar.gz",
],
)
-
http_archive(
name = "bazel_gazelle",
sha256 = "bfd86b3cbe855d6c16c6fce60d76bd51f5c8dbc9cfcaef7a2bb5c1aafd0710e8",
@@ -39,23 +56,15 @@
)
# Python rules
-
# Important: rules_python must be loaded before protobuf (and grpc) because they load an older version otherwise
-
-http_archive(
- name = "rules_python",
- url = "https://github.com/bazelbuild/rules_python/releases/download/0.0.2/rules_python-0.0.2.tar.gz",
- strip_prefix = "rules_python-0.0.2",
- sha256 = "b5668cde8bb6e3515057ef465a35ad712214962f0b3a314e551204266c7be90c",
-)
load("@rules_python//python:repositories.bzl", "py_repositories")
py_repositories()
load("@rules_python//python:pip.bzl", "pip_repositories")
pip_repositories()
-load("@rules_python//python:pip.bzl", "pip3_import")
-pip3_import(
+load("@hscloud_pip_imports//:imports.bzl", "hscloud_pip3_import")
+hscloud_pip3_import(
name = "pydeps",
requirements = "//third_party/py:requirements.txt",
)
@@ -63,6 +72,14 @@
load("@pydeps//:requirements.bzl", "pip_install")
pip_install()
+
+# Setup Go toolchain.
+# This workspace is generated by hscloud_setup_nixpkgs. It will either call
+# go_register_toolchains() to automagically get Go toolchains from the Internet
+# or, if nix is present, instead setup a toolchain from nixpkgs.
+load("@hscloud_go_toolchain//:imports.bzl", "hscloud_go_register_toolchains")
+hscloud_go_register_toolchains()
+
# IMPORTANT: match protobuf version above with the one loaded by grpc
http_archive(
name = "com_github_grpc_grpc",
@@ -71,21 +88,20 @@
urls = ["https://github.com/grpc/grpc/archive/v1.30.0.tar.gz"],
)
+# Load grpc deps after Go, to prevent overriding Go toolchains/SDK.
load("@com_github_grpc_grpc//bazel:grpc_deps.bzl", "grpc_deps")
grpc_deps()
load("@com_github_grpc_grpc//bazel:grpc_extra_deps.bzl", "grpc_extra_deps")
grpc_extra_deps()
-# Go rules dependencies and our own dependencies.
-
-load("@io_bazel_rules_go//go:deps.bzl", "go_rules_dependencies", "go_register_toolchains")
+load("@io_bazel_rules_go//go:deps.bzl", "go_rules_dependencies")
go_rules_dependencies()
-go_register_toolchains()
# gazelle:repository_macro third_party/go/repositories.bzl%go_repositories
load("@bazel_gazelle//:deps.bzl", "gazelle_dependencies")
gazelle_dependencies()
+# Load Go third-party packages.
load("//third_party/go:repositories.bzl", "go_repositories")
go_repositories()
diff --git a/third_party/nix/BUILD b/third_party/nix/BUILD
new file mode 100644
index 0000000..c0821e6
--- /dev/null
+++ b/third_party/nix/BUILD
@@ -0,0 +1,21 @@
+load("@rules_python//python:defs.bzl", "py_runtime_pair")
+
+# Python3 toolchain definition that uses //third_party/nix:python.nix (via
+# external repository).
+
+py_runtime(
+ name = "py3_runtime",
+ interpreter = "@hscloud_nix_python3//:python3",
+ python_version = "PY3",
+)
+
+py_runtime_pair(
+ name = "py_runtime_pair",
+ py3_runtime = ":py3_runtime",
+)
+
+toolchain(
+ name = "py_toolchain",
+ toolchain = ":py_runtime_pair",
+ toolchain_type = "@rules_python//python:toolchain_type",
+)
diff --git a/third_party/nix/python.nix b/third_party/nix/python.nix
new file mode 100644
index 0000000..5571a60
--- /dev/null
+++ b/third_party/nix/python.nix
@@ -0,0 +1,46 @@
+# This is a Python interpreter wrapper that's passed to pip3_import under
+# NixOS.
+# It allows us to build some pip wheels under NixOS that require special
+# system libraries. This is quite hacky, it would be much better if we could
+# somehow tell pip3_import that a given package needs to be built within a
+# given environment.
+
+with import <nixpkgs> {};
+
+let
+ # Add cffi for import _cffi_backend in `cryptography` to work.
+ py = pkgs.python37.withPackages (ps: with ps; [ cffi ]);
+
+# We use mkDerivation instead of writeScript or writeScriptBin as we need a
+# derivation that both:
+# - has a directory structure (for rules_nixpkgs to be able to use it)
+# - has the Python interpreter directly in that structure and not in bin/, as
+# rules_python's pip3_import interpreter_path requires a file target, and
+# will not take an alias. Meanwhile, rules_nixpkgs only creates a BUILD file
+# in the root path of the external repository (which is populated with a
+# symlink tree from the nix derivation), so we can onlly directly reference
+# file in the root of a Nix derivation.
+in stdenv.mkDerivation {
+ name = "py-wrapper";
+ version = "1.0";
+ src = ./.;
+ unpackPhase = "";
+ buildPhase = ''
+ mkdir -p $out
+ cat > $out/python3 <<EOF
+#!/bin/bash
+
+# pyscopg wants libpq, and uses pg_config to find paths. Inject pg_config into
+# the Python interpreter's path.
+export PATH="${pkgs.postgresql}/bin:\$PATH"
+
+# uWSGI has a truly cheese-grade build system, and this is the only way to let
+# it know where to find ncurses.
+export LDFLAGS="-L${pkgs.ncurses}/lib"
+exec ${py}/bin/python3 "\$@"
+EOF
+ '';
+ installPhase = ''
+ chmod +x $out/python3
+ '';
+}
diff --git a/third_party/nix/repository_rules.bzl b/third_party/nix/repository_rules.bzl
new file mode 100644
index 0000000..1a02962
--- /dev/null
+++ b/third_party/nix/repository_rules.bzl
@@ -0,0 +1,97 @@
+load("@io_tweag_rules_nixpkgs//nixpkgs:repositories.bzl", "rules_nixpkgs_dependencies")
+load("@io_tweag_rules_nixpkgs//nixpkgs:nixpkgs.bzl", "nixpkgs_git_repository", "nixpkgs_package")
+
+def has_nix(ctx):
+ return ctx.which("nix-build") != None
+
+def _hscloud_gen_go_imports_impl(ctx):
+ ctx.file("BUILD", "")
+
+ imports_for_nix = """
+load("@io_tweag_rules_nixpkgs//nixpkgs:toolchains/go.bzl", "nixpkgs_go_configure")
+
+def hscloud_go_register_toolchains():
+ nixpkgs_go_configure(repository = "@nixpkgs")
+"""
+ imports_for_non_nix = """
+load("@io_bazel_rules_go//go:deps.bzl", "go_rules_dependencies", "go_register_toolchains")
+def hscloud_go_register_toolchains():
+ go_register_toolchains()
+"""
+
+ if has_nix(ctx):
+ ctx.file("imports.bzl", imports_for_nix)
+ else:
+ ctx.file("imports.bzl", imports_for_non_nix)
+
+# Generate repository containing either a call to go_register_toolchains() or
+# nixpkgs_go_configure(), depending on nix presence.
+hscloud_gen_go_imports = repository_rule(
+ implementation = _hscloud_gen_go_imports_impl,
+ attrs = dict(),
+)
+
+def _hscloud_gen_pip_imports_impl(ctx):
+ ctx.file("BUILD", "")
+
+ # For Nix, we have to both pass our interpreter to pip3_import, and also
+ # register it as a toolchain.
+ imports_for_nix = """
+load("@rules_python//python:pip.bzl", "pip3_import")
+def hscloud_pip3_import(name, requirements):
+ pip3_import(
+ name = name,
+ requirements = requirements,
+ python_interpreter_target = "@hscloud_nix_python3//:python3",
+ )
+ native.register_toolchains("//third_party/nix:py_toolchain")
+"""
+ imports_for_non_nix = """
+load("@rules_python//python:pip.bzl", "pip3_import")
+def hscloud_pip3_import(name, requirements):
+ pip3_import(
+ name = name,
+ requirements = requirements,
+ )
+"""
+ if has_nix(ctx):
+ ctx.file("imports.bzl", imports_for_nix)
+ else:
+ ctx.file("imports.bzl", imports_for_non_nix)
+
+# Generate repository containing a wrapped pip3_import that either uses the
+# host Python interpreter or one from nixpkgs, depending on nix presence.
+hscloud_gen_pip_imports = repository_rule(
+ implementation = _hscloud_gen_pip_imports_impl,
+ attrs = dict(),
+)
+
+def hscloud_setup_nix(revision, sha256):
+ rules_nixpkgs_dependencies()
+ nixpkgs_git_repository(
+ name = "nixpkgs",
+ revision = "1179840f9a88b8a548f4b11d1a03aa25a790c379",
+ sha256 = "8b64041bfb9760de9e797c0a985a4830880c21732489f397e217d877edd9a990",
+ )
+
+ # Load python3 from nixpkgs. Python is a large source of non-hermiticity,
+ # and loading it from nix vastly hermeticizes the build - well, at least to
+ # also be dependent on this Nix store state. That's still better than just
+ # grabbing whatever random system Python a user might have.
+ nixpkgs_package(
+ name = "hscloud_nix_python3",
+ repositories = { "nixpkgs": "@nixpkgs//:default.nix" },
+ nix_file = "//third_party/nix:python.nix",
+ build_file_content = """
+package(default_visibility = ["//visibility:public"])
+exports_files(["python3"])
+ """,
+ )
+
+ # Generate a Go toolchain setup workspace rule.
+ hscloud_gen_go_imports(
+ name = "hscloud_go_toolchain",
+ )
+ hscloud_gen_pip_imports(
+ name = "hscloud_pip_imports",
+ )