Merge "ops, cluster: consolidate NixOS provisioning"
diff --git a/default.nix b/default.nix
index 6d9c0c4..f06a4d3 100644
--- a/default.nix
+++ b/default.nix
@@ -5,7 +5,7 @@
let
fix = f: let x = f x; in x;
- readTree = import ./nix/readtree.nix {};
+ readTree = import ./nix/readtree {};
# Tracking nixos-unstable as of 2021-08-11.
nixpkgsCommit = "e26c0ffdb013cd378fc2528a44689a8bf35d2a6c";
@@ -18,21 +18,12 @@
config.allowBroken = true;
};
-in fix (self: rec {
- config = {
- hscloud = self // {
- root = ./.;
- };
- pkgs = nixpkgs;
- pkgsSrc = nixpkgsSrc;
-
- inherit (nixpkgs) lib stdenv;
- };
-
- bgpwtf = readTree config ./bgpwtf;
- cluster = readTree config ./cluster;
- hswaw = readTree config ./hswaw;
- ops = readTree config ./ops;
-
+in fix (self: (readTree rec {
+ hscloud = self;
+ pkgs = nixpkgs;
+ pkgsSrc = nixpkgsSrc;
+ inherit (nixpkgs) lib stdenv;
+} ./.) // {
+ root = ./.;
pkgs = nixpkgs;
})
diff --git a/nix/readtree.nix b/nix/readtree.nix
deleted file mode 100644
index 066d326..0000000
--- a/nix/readtree.nix
+++ /dev/null
@@ -1,96 +0,0 @@
-# The MIT License (MIT)
-#
-# Copyright (c) 2019 Vincent Ambo
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in all
-# copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-# SOFTWARE.
-
-{ ... }:
-
-args: initPath:
-
-let
- inherit (builtins)
- attrNames
- baseNameOf
- filter
- hasAttr
- head
- isAttrs
- length
- listToAttrs
- map
- match
- readDir
- substring;
-
- argsWithPath = parts:
- let meta.locatedAt = parts;
- in meta // (if isAttrs args then args else args meta);
-
- readDirVisible = path:
- let
- children = readDir path;
- isVisible = f: f == ".skip-subtree" || (substring 0 1 f) != ".";
- names = filter isVisible (attrNames children);
- in listToAttrs (map (name: {
- inherit name;
- value = children.${name};
- }) names);
-
- # The marker is added to every set that was imported directly by
- # readTree.
- importWithMark = path: parts:
- let imported = import path (argsWithPath parts);
- in if (isAttrs imported)
- then imported // { __readTree = true; }
- else imported;
-
- nixFileName = file:
- let res = match "(.*)\.nix" file;
- in if res == null then null else head res;
-
- readTree = path: parts:
- let
- dir = readDirVisible path;
- self = importWithMark path parts;
- joinChild = c: path + ("/" + c);
-
- # Import subdirectories of the current one, unless the special
- # `.skip-subtree` file exists which makes readTree ignore the
- # children.
- #
- # This file can optionally contain information on why the tree
- # should be ignored, but its content is not inspected by
- # readTree
- filterDir = f: dir."${f}" == "directory";
- children = if hasAttr ".skip-subtree" dir then [] else map (c: {
- name = c;
- value = readTree (joinChild c) (parts ++ [ c ]);
- }) (filter filterDir (attrNames dir));
-
- # Import Nix files
- nixFiles = filter (f: f != null) (map nixFileName (attrNames dir));
- nixChildren = map (c: let p = joinChild (c + ".nix"); in {
- name = c;
- value = importWithMark p (parts ++ [ c ]);
- }) nixFiles;
- in if dir ? "default.nix"
- then (if isAttrs self then self // (listToAttrs children) else self)
- else listToAttrs (nixChildren ++ children);
-in readTree initPath [ (baseNameOf initPath) ]
diff --git a/nix/readtree/LICENSE b/nix/readtree/LICENSE
new file mode 100644
index 0000000..bdc72a2
--- /dev/null
+++ b/nix/readtree/LICENSE
@@ -0,0 +1,22 @@
+The MIT License (MIT)
+
+Copyright (c) 2019 Vincent Ambo
+Copyright (c) 2020-2021 The TVL Authors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/nix/readtree/README.md b/nix/readtree/README.md
new file mode 100644
index 0000000..138abbe
--- /dev/null
+++ b/nix/readtree/README.md
@@ -0,0 +1,84 @@
+readTree
+========
+
+This is a Nix program that builds up an attribute set tree for a large
+repository based on the filesystem layout.
+
+It is in fact the tool that lays out the attribute set of this repository.
+
+As an example, consider a root (`.`) of a repository and a layout such as:
+
+```
+.
+├── third_party
+│ ├── default.nix
+│ └── rustpkgs
+│ ├── aho-corasick.nix
+│ └── serde.nix
+└── tools
+ ├── cheddar
+ │ └── default.nix
+ └── roquefort.nix
+```
+
+When `readTree` is called on that tree, it will construct an attribute set with
+this shape:
+
+```nix
+{
+ tools = {
+ cheddar = ...;
+ roquefort = ...;
+ };
+
+ third_party = {
+ # the `default.nix` of this folder might have had arbitrary other
+ # attributes here, such as this:
+ favouriteColour = "orange";
+
+ rustpkgs = {
+ aho-corasick = ...;
+ serde = ...;
+ };
+ };
+}
+```
+
+Every imported Nix file that yields an attribute set will have a `__readTree =
+true;` attribute merged into it.
+
+## Traversal logic
+
+`readTree` will follow any subdirectories of a tree and import all Nix files,
+with some exceptions:
+
+* A folder can declare that its children are off-limit by containing a
+ `.skip-subtree` file. Since the content of the file is not checked, it can be
+ useful to leave a note for a human in the file.
+* If a folder contains a `default.nix` file, no *sibling* Nix files will be
+ imported - however children are traversed as normal.
+* If a folder contains a `default.nix` it is loaded and, if it evaluates to a
+ set, *merged* with the children. If it evaluates to anything else the children
+ are *not traversed*.
+* The `default.nix` of the top-level folder on which readTree is
+ called is **not** read to avoid infinite recursion (as, presumably,
+ this file is where readTree itself is called).
+
+Traversal is lazy, `readTree` will only build up the tree as requested. This
+currently has the downside that directories with no importable files end up in
+the tree as empty nodes (`{}`).
+
+## Import structure
+
+`readTree` is called with two parameters: The arguments to pass to all imports,
+and the initial path at which to start the traversal.
+
+The package headers in this repository follow the form `{ pkgs, ... }:` where
+`pkgs` is a fixed-point of the entire package tree (see the `default.nix` at the
+root of the depot).
+
+In theory `readTree` can pass arguments of different shapes, but I have found
+this to be a good solution for the most part.
+
+Note that `readTree` does not currently make functions overridable, though it is
+feasible that it could do that in the future.
diff --git a/nix/readtree/default.nix b/nix/readtree/default.nix
new file mode 100644
index 0000000..633915f
--- /dev/null
+++ b/nix/readtree/default.nix
@@ -0,0 +1,116 @@
+# Copyright (c) 2019 Vincent Ambo
+# Copyright (c) 2020-2021 The TVL Authors
+# SPDX-License-Identifier: MIT
+#
+# Provides a function to automatically read a a filesystem structure
+# into a Nix attribute set.
+#
+# Optionally accepts an argument `argsFilter` on import, which is a
+# function that receives the current tree location (as a list of
+# strings) and the argument set and can arbitrarily modify it.
+{ argsFilter ? (x: _parts: x)
+, ... }:
+
+let
+ inherit (builtins)
+ attrNames
+ baseNameOf
+ concatStringsSep
+ filter
+ hasAttr
+ head
+ isAttrs
+ length
+ listToAttrs
+ map
+ match
+ readDir
+ substring;
+
+ assertMsg = pred: msg:
+ if pred
+ then true
+ else builtins.trace msg false;
+
+ argsWithPath = args: parts:
+ let meta.locatedAt = parts;
+ in meta // (if isAttrs args then args else args meta);
+
+ readDirVisible = path:
+ let
+ children = readDir path;
+ isVisible = f: f == ".skip-subtree" || (substring 0 1 f) != ".";
+ names = filter isVisible (attrNames children);
+ in listToAttrs (map (name: {
+ inherit name;
+ value = children.${name};
+ }) names);
+
+ # Create a mark containing the location of this attribute.
+ marker = parts: {
+ __readTree = parts;
+ };
+
+ # The marker is added to every set that was imported directly by
+ # readTree.
+ importWithMark = args: path: parts:
+ let
+ importedFile = import path;
+ pathType = builtins.typeOf importedFile;
+ imported =
+ assert assertMsg
+ (pathType == "lambda")
+ "readTree: trying to import ${toString path}, but it’s a ${pathType}, you need to make it a function like { depot, pkgs, ... }";
+ importedFile (argsFilter (argsWithPath args parts) parts);
+ in if (isAttrs imported)
+ then imported // (marker parts)
+ else imported;
+
+ nixFileName = file:
+ let res = match "(.*)\\.nix" file;
+ in if res == null then null else head res;
+
+ readTree = { args, initPath, rootDir, parts }:
+ let
+ dir = readDirVisible initPath;
+ joinChild = c: initPath + ("/" + c);
+
+ self = if rootDir
+ then { __readTree = []; }
+ else importWithMark args initPath parts;
+
+ # Import subdirectories of the current one, unless the special
+ # `.skip-subtree` file exists which makes readTree ignore the
+ # children.
+ #
+ # This file can optionally contain information on why the tree
+ # should be ignored, but its content is not inspected by
+ # readTree
+ filterDir = f: dir."${f}" == "directory";
+ children = if hasAttr ".skip-subtree" dir then [] else map (c: {
+ name = c;
+ value = readTree {
+ args = args;
+ initPath = (joinChild c);
+ rootDir = false;
+ parts = (parts ++ [ c ]);
+ };
+ }) (filter filterDir (attrNames dir));
+
+ # Import Nix files
+ nixFiles = filter (f: f != null) (map nixFileName (attrNames dir));
+ nixChildren = map (c: let p = joinChild (c + ".nix"); in {
+ name = c;
+ value = importWithMark args p (parts ++ [ c ]);
+ }) nixFiles;
+ in if dir ? "default.nix"
+ then (if isAttrs self then self // (listToAttrs children) else self)
+ else (listToAttrs (nixChildren ++ children) // (marker parts));
+
+in {
+ __functor = _: args: initPath: readTree {
+ inherit args initPath;
+ rootDir = true;
+ parts = [];
+ };
+}
diff --git a/shell.nix b/shell.nix
index de3db3c..c664ca9 100644
--- a/shell.nix
+++ b/shell.nix
@@ -4,7 +4,7 @@
hscloud = import ./default.nix {};
-in with hscloud.config.pkgs; let
+in with hscloud.pkgs; let
wrapper = pkgs.writeScript "wrapper.sh"
''
@@ -12,6 +12,13 @@
source ${toString ./.}/env.sh
${toString ./.}/tools/install.sh
+ # Fancy colorful PS1 to make people notice easily they're in hscloud.
+ PS1='\[\033]0;\u/hscloud:\w\007\]'
+ if type -P dircolors >/dev/null ; then
+ PS1+='\[\033[01;35m\]\u/hscloud\[\033[01;34m\] \w \$\[\033[00m\] '
+ fi
+ export PS1
+
exec bash "$@"
'';