nix: upgrade readTree
Change-Id: I460800dc3d8095e2ae89b8bd6ed7c5f0c90b6ccf
diff --git a/nix/readtree/LICENSE b/nix/readtree/LICENSE
new file mode 100644
index 0000000..bdc72a2
--- /dev/null
+++ b/nix/readtree/LICENSE
@@ -0,0 +1,22 @@
+The MIT License (MIT)
+
+Copyright (c) 2019 Vincent Ambo
+Copyright (c) 2020-2021 The TVL Authors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/nix/readtree/README.md b/nix/readtree/README.md
new file mode 100644
index 0000000..138abbe
--- /dev/null
+++ b/nix/readtree/README.md
@@ -0,0 +1,84 @@
+readTree
+========
+
+This is a Nix program that builds up an attribute set tree for a large
+repository based on the filesystem layout.
+
+It is in fact the tool that lays out the attribute set of this repository.
+
+As an example, consider a root (`.`) of a repository and a layout such as:
+
+```
+.
+├── third_party
+│ ├── default.nix
+│ └── rustpkgs
+│ ├── aho-corasick.nix
+│ └── serde.nix
+└── tools
+ ├── cheddar
+ │ └── default.nix
+ └── roquefort.nix
+```
+
+When `readTree` is called on that tree, it will construct an attribute set with
+this shape:
+
+```nix
+{
+ tools = {
+ cheddar = ...;
+ roquefort = ...;
+ };
+
+ third_party = {
+ # the `default.nix` of this folder might have had arbitrary other
+ # attributes here, such as this:
+ favouriteColour = "orange";
+
+ rustpkgs = {
+ aho-corasick = ...;
+ serde = ...;
+ };
+ };
+}
+```
+
+Every imported Nix file that yields an attribute set will have a `__readTree =
+true;` attribute merged into it.
+
+## Traversal logic
+
+`readTree` will follow any subdirectories of a tree and import all Nix files,
+with some exceptions:
+
+* A folder can declare that its children are off-limit by containing a
+ `.skip-subtree` file. Since the content of the file is not checked, it can be
+ useful to leave a note for a human in the file.
+* If a folder contains a `default.nix` file, no *sibling* Nix files will be
+ imported - however children are traversed as normal.
+* If a folder contains a `default.nix` it is loaded and, if it evaluates to a
+ set, *merged* with the children. If it evaluates to anything else the children
+ are *not traversed*.
+* The `default.nix` of the top-level folder on which readTree is
+ called is **not** read to avoid infinite recursion (as, presumably,
+ this file is where readTree itself is called).
+
+Traversal is lazy, `readTree` will only build up the tree as requested. This
+currently has the downside that directories with no importable files end up in
+the tree as empty nodes (`{}`).
+
+## Import structure
+
+`readTree` is called with two parameters: The arguments to pass to all imports,
+and the initial path at which to start the traversal.
+
+The package headers in this repository follow the form `{ pkgs, ... }:` where
+`pkgs` is a fixed-point of the entire package tree (see the `default.nix` at the
+root of the depot).
+
+In theory `readTree` can pass arguments of different shapes, but I have found
+this to be a good solution for the most part.
+
+Note that `readTree` does not currently make functions overridable, though it is
+feasible that it could do that in the future.
diff --git a/nix/readtree/default.nix b/nix/readtree/default.nix
new file mode 100644
index 0000000..633915f
--- /dev/null
+++ b/nix/readtree/default.nix
@@ -0,0 +1,116 @@
+# Copyright (c) 2019 Vincent Ambo
+# Copyright (c) 2020-2021 The TVL Authors
+# SPDX-License-Identifier: MIT
+#
+# Provides a function to automatically read a a filesystem structure
+# into a Nix attribute set.
+#
+# Optionally accepts an argument `argsFilter` on import, which is a
+# function that receives the current tree location (as a list of
+# strings) and the argument set and can arbitrarily modify it.
+{ argsFilter ? (x: _parts: x)
+, ... }:
+
+let
+ inherit (builtins)
+ attrNames
+ baseNameOf
+ concatStringsSep
+ filter
+ hasAttr
+ head
+ isAttrs
+ length
+ listToAttrs
+ map
+ match
+ readDir
+ substring;
+
+ assertMsg = pred: msg:
+ if pred
+ then true
+ else builtins.trace msg false;
+
+ argsWithPath = args: parts:
+ let meta.locatedAt = parts;
+ in meta // (if isAttrs args then args else args meta);
+
+ readDirVisible = path:
+ let
+ children = readDir path;
+ isVisible = f: f == ".skip-subtree" || (substring 0 1 f) != ".";
+ names = filter isVisible (attrNames children);
+ in listToAttrs (map (name: {
+ inherit name;
+ value = children.${name};
+ }) names);
+
+ # Create a mark containing the location of this attribute.
+ marker = parts: {
+ __readTree = parts;
+ };
+
+ # The marker is added to every set that was imported directly by
+ # readTree.
+ importWithMark = args: path: parts:
+ let
+ importedFile = import path;
+ pathType = builtins.typeOf importedFile;
+ imported =
+ assert assertMsg
+ (pathType == "lambda")
+ "readTree: trying to import ${toString path}, but it’s a ${pathType}, you need to make it a function like { depot, pkgs, ... }";
+ importedFile (argsFilter (argsWithPath args parts) parts);
+ in if (isAttrs imported)
+ then imported // (marker parts)
+ else imported;
+
+ nixFileName = file:
+ let res = match "(.*)\\.nix" file;
+ in if res == null then null else head res;
+
+ readTree = { args, initPath, rootDir, parts }:
+ let
+ dir = readDirVisible initPath;
+ joinChild = c: initPath + ("/" + c);
+
+ self = if rootDir
+ then { __readTree = []; }
+ else importWithMark args initPath parts;
+
+ # Import subdirectories of the current one, unless the special
+ # `.skip-subtree` file exists which makes readTree ignore the
+ # children.
+ #
+ # This file can optionally contain information on why the tree
+ # should be ignored, but its content is not inspected by
+ # readTree
+ filterDir = f: dir."${f}" == "directory";
+ children = if hasAttr ".skip-subtree" dir then [] else map (c: {
+ name = c;
+ value = readTree {
+ args = args;
+ initPath = (joinChild c);
+ rootDir = false;
+ parts = (parts ++ [ c ]);
+ };
+ }) (filter filterDir (attrNames dir));
+
+ # Import Nix files
+ nixFiles = filter (f: f != null) (map nixFileName (attrNames dir));
+ nixChildren = map (c: let p = joinChild (c + ".nix"); in {
+ name = c;
+ value = importWithMark args p (parts ++ [ c ]);
+ }) nixFiles;
+ in if dir ? "default.nix"
+ then (if isAttrs self then self // (listToAttrs children) else self)
+ else (listToAttrs (nixChildren ++ children) // (marker parts));
+
+in {
+ __functor = _: args: initPath: readTree {
+ inherit args initPath;
+ rootDir = true;
+ parts = [];
+ };
+}