Merge "cluster/kube: remove ceph diff against k0 production"
diff --git a/cluster/clustercfg/clustercfg.py b/cluster/clustercfg/clustercfg.py
index 0adef40..d852d6a 100644
--- a/cluster/clustercfg/clustercfg.py
+++ b/cluster/clustercfg/clustercfg.py
@@ -206,10 +206,12 @@
         ca_admitomatic = ca.CA(ss, certs_root, 'admitomatic', 'admitomatic webhook CA')
         ca_admitomatic.make_cert('admitomatic-webhook', ou='Admitomatic Webhook', hosts=['admitomatic.admitomatic.svc'])
 
-    subprocess.check_call(["nix", "run",
-                           "-f", local_root,
-                           "cluster.nix.provision",
-                           "-c", "provision-{}".format(fqdn.split('.')[0])])
+    toplevel = subprocess.check_output([
+        "nix-build",
+        local_root,
+        "-A", "ops.machines.\"" + fqdn + "\".config.passthru.hscloud.provision",
+    ]).decode().strip()
+    subprocess.check_call([toplevel])
 
 
 def usage():
diff --git a/cluster/nix/provision.nix b/cluster/nix/provision.nix
deleted file mode 100644
index 7ab7e71..0000000
--- a/cluster/nix/provision.nix
+++ /dev/null
@@ -1,49 +0,0 @@
-{ hscloud, pkgs, ... }:
-
-with builtins;
-
-let 
-  machines = (import ./defs-machines.nix);
-  configurations = builtins.listToAttrs (map (machine: {
-    name = machine.fqdn;
-    value = pkgs.nixos ({ config, pkgs, ... }: {
-      networking.hostName = machine.name;
-      imports = [
-        ./modules/base.nix
-        ./modules/kubernetes.nix
-      ];
-    });
-  }) machines);
-
-  scriptForMachine = machine: let
-    configuration = configurations."${machine.fqdn}";
-  in ''
-   set -e
-   remote=root@${machine.fqdn}
-   echo "Configuration for ${machine.fqdn} is ${configuration.toplevel}"
-   nix copy --no-check-sigs -s --to ssh://$remote ${configuration.toplevel}
-   echo "/etc/systemd/system diff:"
-   ssh $remote diff -ur /var/run/current-system/etc/systemd/system ${configuration.toplevel}/etc/systemd/system || true
-   echo ""
-   echo ""
-   ssh $remote ${configuration.toplevel}/bin/switch-to-configuration dry-activate
-   read -p "Do you want to switch to this configuration? " -n 1 -r
-   echo
-   if [[ $REPLY =~ ^[Yy]$ ]]; then
-       ssh $remote ${configuration.toplevel}/bin/switch-to-configuration switch
-   fi
-  '';
-
-  provisioners = (map (machine:
-    pkgs.writeScriptBin "provision-${machine.name}" (scriptForMachine machine)
-  ) machines);
-
-  provision = pkgs.writeScriptBin "provision" (
-    ''
-      echo "Available provisioniers:"
-    '' + (concatStringsSep "\n" (map (machine: "echo '  provision-${machine.name}'") machines)));
-in
-pkgs.symlinkJoin {
-  name = "provision";
-  paths = [ provision ] ++ provisioners;
-}
diff --git a/default.nix b/default.nix
index 6d9c0c4..f06a4d3 100644
--- a/default.nix
+++ b/default.nix
@@ -5,7 +5,7 @@
 let
   fix = f: let x = f x; in x;
 
-  readTree = import ./nix/readtree.nix {};
+  readTree = import ./nix/readtree {};
 
   # Tracking nixos-unstable as of 2021-08-11.
   nixpkgsCommit = "e26c0ffdb013cd378fc2528a44689a8bf35d2a6c";
@@ -18,21 +18,12 @@
     config.allowBroken = true;
   };
 
-in fix (self: rec {
-  config = {
-    hscloud = self // {
-      root = ./.;
-    };
-    pkgs = nixpkgs;
-    pkgsSrc = nixpkgsSrc;
-
-    inherit (nixpkgs) lib stdenv;
-  };
-
-  bgpwtf = readTree config ./bgpwtf;
-  cluster = readTree config ./cluster;
-  hswaw = readTree config ./hswaw;
-  ops = readTree config ./ops;
-
+in fix (self: (readTree rec {
+  hscloud = self;
+  pkgs = nixpkgs;
+  pkgsSrc = nixpkgsSrc;
+  inherit (nixpkgs) lib stdenv;
+} ./.) // {
+  root = ./.;
   pkgs = nixpkgs;
 })
diff --git a/nix/readtree.nix b/nix/readtree.nix
deleted file mode 100644
index 066d326..0000000
--- a/nix/readtree.nix
+++ /dev/null
@@ -1,96 +0,0 @@
-# The MIT License (MIT)
-# 
-# Copyright (c) 2019 Vincent Ambo
-# 
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-# 
-# The above copyright notice and this permission notice shall be included in all
-# copies or substantial portions of the Software.
-# 
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-# SOFTWARE.
-
-{ ... }:
-
-args: initPath:
-
-let
-  inherit (builtins)
-    attrNames
-    baseNameOf
-    filter
-    hasAttr
-    head
-    isAttrs
-    length
-    listToAttrs
-    map
-    match
-    readDir
-    substring;
-
-  argsWithPath = parts:
-    let meta.locatedAt = parts;
-    in meta // (if isAttrs args then args else args meta);
-
-  readDirVisible = path:
-    let
-      children = readDir path;
-      isVisible = f: f == ".skip-subtree" || (substring 0 1 f) != ".";
-      names = filter isVisible (attrNames children);
-    in listToAttrs (map (name: {
-      inherit name;
-      value = children.${name};
-    }) names);
-
-  # The marker is added to every set that was imported directly by
-  # readTree.
-  importWithMark = path: parts:
-    let imported = import path (argsWithPath parts);
-    in if (isAttrs imported)
-      then imported // { __readTree = true; }
-      else imported;
-
-  nixFileName = file:
-    let res = match "(.*)\.nix" file;
-    in if res == null then null else head res;
-
-  readTree = path: parts:
-    let
-      dir = readDirVisible path;
-      self = importWithMark path parts;
-      joinChild = c: path + ("/" + c);
-
-      # Import subdirectories of the current one, unless the special
-      # `.skip-subtree` file exists which makes readTree ignore the
-      # children.
-      #
-      # This file can optionally contain information on why the tree
-      # should be ignored, but its content is not inspected by
-      # readTree
-      filterDir = f: dir."${f}" == "directory";
-      children = if hasAttr ".skip-subtree" dir then [] else map (c: {
-        name = c;
-        value = readTree (joinChild c) (parts ++ [ c ]);
-      }) (filter filterDir (attrNames dir));
-
-      # Import Nix files
-      nixFiles = filter (f: f != null) (map nixFileName (attrNames dir));
-      nixChildren = map (c: let p = joinChild (c + ".nix"); in {
-        name = c;
-        value = importWithMark p (parts ++ [ c ]);
-      }) nixFiles;
-    in if dir ? "default.nix"
-      then (if isAttrs self then self // (listToAttrs children) else self)
-      else listToAttrs (nixChildren ++ children);
-in readTree initPath [ (baseNameOf initPath) ]
diff --git a/nix/readtree/LICENSE b/nix/readtree/LICENSE
new file mode 100644
index 0000000..bdc72a2
--- /dev/null
+++ b/nix/readtree/LICENSE
@@ -0,0 +1,22 @@
+The MIT License (MIT)
+
+Copyright (c) 2019 Vincent Ambo
+Copyright (c) 2020-2021 The TVL Authors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/nix/readtree/README.md b/nix/readtree/README.md
new file mode 100644
index 0000000..138abbe
--- /dev/null
+++ b/nix/readtree/README.md
@@ -0,0 +1,84 @@
+readTree
+========
+
+This is a Nix program that builds up an attribute set tree for a large
+repository based on the filesystem layout.
+
+It is in fact the tool that lays out the attribute set of this repository.
+
+As an example, consider a root (`.`) of a repository and a layout such as:
+
+```
+.
+├── third_party
+│   ├── default.nix
+│   └── rustpkgs
+│       ├── aho-corasick.nix
+│       └── serde.nix
+└── tools
+    ├── cheddar
+    │   └── default.nix
+    └── roquefort.nix
+```
+
+When `readTree` is called on that tree, it will construct an attribute set with
+this shape:
+
+```nix
+{
+    tools = {
+        cheddar = ...;
+        roquefort = ...;
+    };
+
+    third_party = {
+        # the `default.nix` of this folder might have had arbitrary other
+        # attributes here, such as this:
+        favouriteColour = "orange";
+
+        rustpkgs = {
+            aho-corasick = ...;
+            serde = ...;
+        };
+    };
+}
+```
+
+Every imported Nix file that yields an attribute set will have a `__readTree =
+true;` attribute merged into it.
+
+## Traversal logic
+
+`readTree` will follow any subdirectories of a tree and import all Nix files,
+with some exceptions:
+
+* A folder can declare that its children are off-limit by containing a
+  `.skip-subtree` file. Since the content of the file is not checked, it can be
+  useful to leave a note for a human in the file.
+* If a folder contains a `default.nix` file, no *sibling* Nix files will be
+  imported - however children are traversed as normal.
+* If a folder contains a `default.nix` it is loaded and, if it evaluates to a
+  set, *merged* with the children. If it evaluates to anything else the children
+  are *not traversed*.
+* The `default.nix` of the top-level folder on which readTree is
+  called is **not** read to avoid infinite recursion (as, presumably,
+  this file is where readTree itself is called).
+
+Traversal is lazy, `readTree` will only build up the tree as requested. This
+currently has the downside that directories with no importable files end up in
+the tree as empty nodes (`{}`).
+
+## Import structure
+
+`readTree` is called with two parameters: The arguments to pass to all imports,
+and the initial path at which to start the traversal.
+
+The package headers in this repository follow the form `{ pkgs, ... }:` where
+`pkgs` is a fixed-point of the entire package tree (see the `default.nix` at the
+root of the depot).
+
+In theory `readTree` can pass arguments of different shapes, but I have found
+this to be a good solution for the most part.
+
+Note that `readTree` does not currently make functions overridable, though it is
+feasible that it could do that in the future.
diff --git a/nix/readtree/default.nix b/nix/readtree/default.nix
new file mode 100644
index 0000000..633915f
--- /dev/null
+++ b/nix/readtree/default.nix
@@ -0,0 +1,116 @@
+# Copyright (c) 2019 Vincent Ambo
+# Copyright (c) 2020-2021 The TVL Authors
+# SPDX-License-Identifier: MIT
+#
+# Provides a function to automatically read a a filesystem structure
+# into a Nix attribute set.
+#
+# Optionally accepts an argument `argsFilter` on import, which is a
+# function that receives the current tree location (as a list of
+# strings) and the argument set and can arbitrarily modify it.
+{ argsFilter ? (x: _parts: x)
+, ... }:
+
+let
+  inherit (builtins)
+    attrNames
+    baseNameOf
+    concatStringsSep
+    filter
+    hasAttr
+    head
+    isAttrs
+    length
+    listToAttrs
+    map
+    match
+    readDir
+    substring;
+
+  assertMsg = pred: msg:
+    if pred
+    then true
+    else builtins.trace msg false;
+
+  argsWithPath = args: parts:
+    let meta.locatedAt = parts;
+    in meta // (if isAttrs args then args else args meta);
+
+  readDirVisible = path:
+    let
+      children = readDir path;
+      isVisible = f: f == ".skip-subtree" || (substring 0 1 f) != ".";
+      names = filter isVisible (attrNames children);
+    in listToAttrs (map (name: {
+      inherit name;
+      value = children.${name};
+    }) names);
+
+  # Create a mark containing the location of this attribute.
+  marker = parts: {
+    __readTree = parts;
+  };
+
+  # The marker is added to every set that was imported directly by
+  # readTree.
+  importWithMark = args: path: parts:
+    let
+      importedFile = import path;
+      pathType = builtins.typeOf importedFile;
+      imported =
+        assert assertMsg
+          (pathType == "lambda")
+          "readTree: trying to import ${toString path}, but it’s a ${pathType}, you need to make it a function like { depot, pkgs, ... }";
+        importedFile (argsFilter (argsWithPath args parts) parts);
+    in if (isAttrs imported)
+      then imported // (marker parts)
+      else imported;
+
+  nixFileName = file:
+    let res = match "(.*)\\.nix" file;
+    in if res == null then null else head res;
+
+  readTree = { args, initPath, rootDir, parts }:
+    let
+      dir = readDirVisible initPath;
+      joinChild = c: initPath + ("/" + c);
+
+      self = if rootDir
+        then { __readTree = []; }
+        else importWithMark args initPath parts;
+
+      # Import subdirectories of the current one, unless the special
+      # `.skip-subtree` file exists which makes readTree ignore the
+      # children.
+      #
+      # This file can optionally contain information on why the tree
+      # should be ignored, but its content is not inspected by
+      # readTree
+      filterDir = f: dir."${f}" == "directory";
+      children = if hasAttr ".skip-subtree" dir then [] else map (c: {
+        name = c;
+        value = readTree {
+          args = args;
+          initPath = (joinChild c);
+          rootDir = false;
+          parts = (parts ++ [ c ]);
+        };
+      }) (filter filterDir (attrNames dir));
+
+      # Import Nix files
+      nixFiles = filter (f: f != null) (map nixFileName (attrNames dir));
+      nixChildren = map (c: let p = joinChild (c + ".nix"); in {
+        name = c;
+        value = importWithMark args p (parts ++ [ c ]);
+      }) nixFiles;
+    in if dir ? "default.nix"
+      then (if isAttrs self then self // (listToAttrs children) else self)
+      else (listToAttrs (nixChildren ++ children) // (marker parts));
+
+in {
+  __functor = _: args: initPath: readTree {
+    inherit args initPath;
+    rootDir = true;
+    parts = [];
+  };
+}
diff --git a/ops/README.md b/ops/README.md
new file mode 100644
index 0000000..d31f767
--- /dev/null
+++ b/ops/README.md
@@ -0,0 +1,23 @@
+Operations
+===
+
+Deploying NixOS machines
+---
+
+Machine configurations are in `ops/machines.nix`.
+
+Wrapper script to show all available machines and provision a single machine:
+
+     $ $(nix-build -A ops.provision)
+     Available machines:
+      - bc01n01.hswaw.net
+      - bc01n02.hswaw.net
+      - dcr01s22.hswaw.net
+      - dcr01s24.hswaw.net
+      - edge01.waw.bgp.wtf
+
+     $ $(nix-build -A ops.provision) edge01.waw.bgp.wtf
+
+This can be slow, as it evaluates/builds all machines' configs. If you just want to deploy one machine and possible iterate faster:
+
+    $ $(nix-build -A 'ops.machines."edge01.waw.bgp.wtf".config.passthru.hscloud.provision')
diff --git a/ops/machines.nix b/ops/machines.nix
index 0e63228..5401e30 100644
--- a/ops/machines.nix
+++ b/ops/machines.nix
@@ -3,30 +3,41 @@
 # This allows to have a common attrset of machines that can be deployed
 # in the same way.
 #
-# Currently building/deployment is still done in a half-assed way:
-#
-#    machine=edge01.waw.bgp.wtf
-#    d=$(nix-build -A 'ops.machines."'$machine'"'.toplevel)
-#
-# To then deploy derivation $d on $machine:
-#
-#    nix-copy-closure --to root@$machine $d
-#    ssh root@$machine $d/bin/switch-to-configuration dry-activate
-#    ssh root@$machine $d/bin/switch-to-configuration test
-#    ssh root@$machine nix-env -p /nix/var/nix/profiles/system --set $d
-#    ssh root@$machine $d/bin/switch-to-configuration boot
-#
-# TODO(q3k): merge this with //cluster/clustercfg - this should be unified!
+# For information about building/deploying machines see //ops/README.md.
 
 { hscloud, pkgs, ... }:
 
 let
+  # nixpkgs for cluster machines (.hswaw.net). Currently pinned to an old
+  # nixpkgs because NixOS modules for kubernetes changed enough that it's not
+  # super easy to use them as is.
+  #
+  # TODO(q3k): fix this: use an old nixpkgs for Kube modules while using
+  # hscloud nixpkgs for everything else.
+  nixpkgsCluster = import (pkgs.fetchFromGitHub {
+    owner = "nixos";
+    repo = "nixpkgs-channels";
+    rev = "44ad80ab1036c5cc83ada4bfa451dac9939f2a10";
+    sha256 = "1b61nzvy0d46cspy07szkc0rggacxiqg9v1py27pkqpj7rvawfsk";
+  }) {};
+
+  # edge01 still lives on an old nixpkgs checkout.
+  #
+  # TODO(b/3): unpin and deploy.
+  nixpkgsBgpwtf = import (pkgs.fetchFromGitHub {
+    owner = "nixos";
+    repo = "nixpkgs-channels";
+    rev = "c59ea8b8a0e7f927e7291c14ea6cd1bd3a16ff38";
+    sha256 = "1ak7jqx94fjhc68xh1lh35kh3w3ndbadprrb762qgvcfb8351x8v";
+  }) {};
+
   # Stopgap measure to import //cluster/nix machine definitions into new
-  # //ops/machines infrastructure.
+  # //ops/ infrastructure.
+  #
   # TODO(q3k): inject defs-cluster-k0.nix / defs-machines.nix content via
   # nixos options instead of having module definitions loading it themselves,
   # deduplicate list of machines below with defs-machines.nix somehow.
-  mkClusterMachine = name: pkgs.nixos ({ config, pkgs, ... }: {
+  clusterMachineConfig = name: [({ config, pkgs, ...}: {
     # The hostname is used by //cluster/nix machinery to load the appropriate
     # config from defs-machines into defs-cluster-k0.
     networking.hostName = name;
@@ -34,29 +45,71 @@
       ../cluster/nix/modules/base.nix
       ../cluster/nix/modules/kubernetes.nix
     ];
-  });
+  })];
 
+  # mkMachine builds NixOS modules into a NixOS derivation, and injects
+  # passthru.hscloud.provision which deploys that configuration over SSH to a
+  # production machine.
   mkMachine = pkgs: paths: pkgs.nixos ({ config, pkgs, ... }: {
     imports = paths;
+
+    config = let
+      name = config.networking.hostName;
+      domain = if (config.networking ? domain) && config.networking.domain != null then config.networking.domain else "hswaw.net";
+      fqdn = name + "." + domain;
+      toplevel = config.system.build.toplevel;
+
+      runProvision = ''
+        #!/bin/sh
+        set -eu
+        remote=root@${fqdn}
+        echo "Configuration for ${fqdn} is ${toplevel}"
+        nix copy -s --to ssh://$remote ${toplevel}
+
+        running="$(ssh $remote readlink -f /nix/var/nix/profiles/system)"
+        if [ "$running" == "${toplevel}" ]; then
+          echo "${fqdn} already running ${toplevel}."
+        else
+          echo "/etc/systemd/system diff:"
+          ssh $remote diff -ur /var/run/current-system/etc/systemd/system ${toplevel}/etc/systemd/system || true
+          echo ""
+          echo ""
+          echo "dry-activate diff:"
+          ssh $remote ${toplevel}/bin/switch-to-configuration dry-activate
+          read -p "Do you want to switch to this configuration? " -n 1 -r
+          echo
+          if ! [[ $REPLY =~ ^[Yy]$ ]]; then
+            exit 1
+          fi
+
+          echo -ne "\n\nswitch-to-configuration test...\n"
+          ssh $remote ${toplevel}/bin/switch-to-configuration test
+        fi
+
+        echo -ne "\n\n"
+        read -p "Do you want to set this configuration as boot? " -n 1 -r
+        echo
+        if ! [[ $REPLY =~ ^[Yy]$ ]]; then
+            exit 1
+        fi
+
+        echo -ne "\n\nsetting system profile...\n"
+        ssh $remote nix-env -p /nix/var/nix/profiles/system --set ${toplevel}
+
+        echo -ne "\n\nswitch-to-configuration boot...\n"
+        ssh $remote ${toplevel}/bin/switch-to-configuration boot
+      '';
+    in {
+      passthru.hscloud.provision = pkgs.writeScript "provision-${fqdn}" runProvision;
+    };
   });
-
 in {
-  "bc01n01.hswaw.net" = mkClusterMachine "bc01n01";
-  "bc01n02.hswaw.net" = mkClusterMachine "bc01n02";
-  "bc01n03.hswaw.net" = mkClusterMachine "bc01n03";
-  "dcr01s22.hswaw.net" = mkClusterMachine "dcr01s22";
-  "dcr01s24.hswaw.net" = mkClusterMachine "dcr01s24";
+  "bc01n01.hswaw.net"  = mkMachine nixpkgsCluster (clusterMachineConfig "bc01n01");
+  "bc01n02.hswaw.net"  = mkMachine nixpkgsCluster (clusterMachineConfig "bc01n02");
+  "dcr01s22.hswaw.net" = mkMachine nixpkgsCluster (clusterMachineConfig "dcr01s22");
+  "dcr01s24.hswaw.net" = mkMachine nixpkgsCluster (clusterMachineConfig "dcr01s24");
 
-  # edge01 still lives on an old nixpkgs checkout.
-  # TODO(b/3): unpin and deploy.
-  "edge01.waw.bgp.wtf" = mkMachine (
-    import (pkgs.fetchFromGitHub {
-      owner = "nixos";
-      repo = "nixpkgs-channels";
-      rev = "c59ea8b8a0e7f927e7291c14ea6cd1bd3a16ff38";
-      sha256 = "1ak7jqx94fjhc68xh1lh35kh3w3ndbadprrb762qgvcfb8351x8v";
-    }) {}
-  ) [
+  "edge01.waw.bgp.wtf" = mkMachine nixpkgsBgpwtf [
     ../bgpwtf/machines/edge01.waw.bgp.wtf.nix
     ../bgpwtf/machines/edge01.waw.bgp.wtf-hardware.nix
   ];
diff --git a/ops/provision.nix b/ops/provision.nix
new file mode 100644
index 0000000..76054c4
--- /dev/null
+++ b/ops/provision.nix
@@ -0,0 +1,74 @@
+# Top-level wrapper script for calling per-machine provisioners.
+#
+# Given ops.machines."edge01.waw.bgp.wtf".config.passthru.hscloud.provision,
+# this script allows to run it by doing:
+#   $ $(nix-build -A ops.provision) edge01.waw.bgp.wtf
+# Or, to first list all available machines by doing:
+#   $ $(nix-build -A ops.provision)
+#
+# The main logic of the provisioner script is in machines.nix.
+
+{ hscloud, pkgs, lib, ... }:
+
+with lib; with builtins;
+
+let
+
+  # All machines from ops.machines, keyed by FQDN.
+  machines = filterAttrs (n: _: n != "__readTree") hscloud.ops.machines;
+  # Machines' provisioner scripts, keyed by machine FQDN.
+  machineProvisioners = mapAttrs (_: v: v.config.passthru.hscloud.provision) machines;
+  # List of machine FQDNs.
+  machineNames = attrNames machines;
+
+  # User-friendly list of machines by FQDN.
+  machineList = concatStringsSep "\n"
+    (map
+      (name: "  - ${name}")
+      machineNames);
+
+  # Derivation containing bin/provision-FQDN symlinks to machines' provisioners.
+  forest = pkgs.linkFarm "provision-forest"
+    (mapAttrsToList
+      (fqdn: p: { name = "bin/provision-${fqdn}"; path = p; })
+      machineProvisioners);
+in
+
+pkgs.writeScript "provision" ''
+  #!/bin/sh
+  name="$1"
+
+  usage() {
+    echo >&2 "Usage: $0 machine|machine.hswaw.net"
+    echo >&2 "Available machines:"
+    echo >&2 "${machineList}"
+  }
+
+  if [ -z "$name" ]; then
+    usage
+    exit 1
+  fi
+
+  provisioner="${forest}/bin/provision-$name"
+  if [ ! -e "$provisioner" ]; then
+    name="$name.hswaw.net"
+    provisioner="${forest}/bin/provision-$name"
+  fi
+  if [ ! -e "$provisioner" ]; then
+    usage
+    exit 1
+  fi
+  # :^)
+  echo -ne "\e[34mh \e[31ms \e[33mc l \e[34mo \e[32mu \e[31md \e[0m"
+  echo ""
+  echo "Starting provisioner for $name..."
+  echo ""
+  echo "Too slow to evaluate? Equivalent faster command line that rebuilds just one node:"
+  echo "  \$(nix-build -A 'ops.machines.\"$name\".config.passthru.hscloud.provision')"
+  echo ""
+  echo "Or, if you want to deploy the same configuration on different machines, just run"
+  echo "this script again without re-evaluating nix:"
+  echo "  $0 $name"
+  echo ""
+  exec "$provisioner"
+''
diff --git a/shell.nix b/shell.nix
index de3db3c..c664ca9 100644
--- a/shell.nix
+++ b/shell.nix
@@ -4,7 +4,7 @@
 
   hscloud = import ./default.nix {};
 
-in with hscloud.config.pkgs; let
+in with hscloud.pkgs; let
 
   wrapper = pkgs.writeScript "wrapper.sh"
   ''
@@ -12,6 +12,13 @@
     source ${toString ./.}/env.sh
     ${toString ./.}/tools/install.sh
 
+    # Fancy colorful PS1 to make people notice easily they're in hscloud.
+    PS1='\[\033]0;\u/hscloud:\w\007\]'
+    if type -P dircolors >/dev/null ; then
+      PS1+='\[\033[01;35m\]\u/hscloud\[\033[01;34m\] \w \$\[\033[00m\] '
+    fi
+    export PS1
+
     exec bash "$@"
   '';