cluster: refactor nix machinery to fit //ops
This is a chonky refactor that get rids of the previous cluster-centric
defs-* plain nix file setup.
Now, nodes are configured individually in plain nixos modules, and are
provided a view of all other nodes in the 'machines' attribute. Cluster
logic is moved into modules which inspect this array to find other nodes
within the same cluster.
Kubernetes options are not fully clusterified yet (ie., they are still
hardcode to only provide the 'k0' cluster) but that can be fixed later.
The Ceph machinery is a good example of how that can be done.
The new NixOS configs are zero-diff against prod. While this is done
mostly by keeping the logic, we had to keep a few newly discovered
'bugs' around by adding some temporary options which keeps things as they
are. These will be removed in a future CL, then introducing a diff (but
no functional changes, hopefully).
We also remove the nix eval from clustercfg as it was not used anymore
(basically since we refactored certs at some point).
Change-Id: Id79772a96249b0e6344046f96f9c2cb481c4e1f4
Reviewed-on: https://gerrit.hackerspace.pl/c/hscloud/+/1322
Reviewed-by: informatic <informatic@hackerspace.pl>
diff --git a/cluster/machines/modules/base.nix b/cluster/machines/modules/base.nix
new file mode 100644
index 0000000..66335ef
--- /dev/null
+++ b/cluster/machines/modules/base.nix
@@ -0,0 +1,100 @@
+{ config, pkgs, lib, ... }:
+
+with lib;
+
+let
+ cfg = config.hscloud.base;
+
+in {
+ options.hscloud.base = {
+ fqdn = mkOption {
+ type = types.str;
+ description = "Node's FQDN.";
+ default = "${config.networking.hostName}.${config.networking.domain}";
+ };
+ mgmtIf = mkOption {
+ type = types.str;
+ description = "Main network interface. Called mgmtIf for legacy reasons.";
+ };
+ ipAddr = mkOption {
+ type = types.str;
+ description = "IPv4 address on main network interface.";
+ };
+ ipAddrBits = mkOption {
+ type = types.int;
+ description = "IPv4 CIDR mask bits.";
+ };
+ gw = mkOption {
+ type = types.str;
+ description = "IPv4 address of gateway.";
+ };
+ };
+ config = rec {
+ boot.loader.grub.enable = true;
+ boot.loader.grub.version = 2;
+
+ fileSystems."/" =
+ { # device = ""; needs to be defined
+ fsType = "ext4";
+ };
+ swapDevices = [ ];
+
+ boot.kernelPackages = pkgs.linuxPackages_latest;
+ boot.kernelParams = [ "boot.shell_on_fail" ];
+ boot.kernel.sysctl."net.ipv4.conf.all.rp_filter" = "0";
+ boot.kernel.sysctl."net.ipv4.conf.default.rp_filter" = "0";
+ boot.initrd.availableKernelModules = [ "uhci_hcd" "ehci_pci" "megaraid_sas" "usb_storage" "usbhid" "sd_mod" "sr_mod" ];
+ boot.kernelModules = [ "kvm-intel" ];
+ boot.extraModulePackages = [];
+ hardware.enableRedistributableFirmware = true;
+
+ time.timeZone = "Europe/Warsaw";
+
+ environment.systemPackages = with pkgs; [
+ wget vim htop tcpdump
+ rxvt_unicode.terminfo
+ ];
+ programs.mtr.enable = true;
+
+ networking.useDHCP = false;
+ networking.interfaces."${cfg.mgmtIf}" = {
+ ipv4.addresses = [
+ {
+ address = cfg.ipAddr;
+ prefixLength = cfg.ipAddrBits;
+ }
+ ];
+ };
+ networking.defaultGateway = cfg.gw;
+ networking.nameservers = ["185.236.240.1"];
+
+ # Instead of using nixpkgs from the root/nixos channel, use pkgs pin from this file.
+ nix.nixPath = [ "nixpkgs=${pkgs.path}" "nixos-config=/etc/nixos/configuration.nix" ];
+
+ # Otherwise fetchGit nixpkgs pin fails.
+ systemd.services.nixos-upgrade.path = [ pkgs.git ];
+
+ # Use Chrony instead of systemd-timesyncd
+ services.chrony.enable = true;
+
+ # Symlink lvm into /sbin/lvm on activation. This is needed by Rook OSD
+ # instances running on Kubernetes.
+ # See: https://github.com/rook/rook/commit/f3c4975e353e3ce3599c958ec6d2cae8ee8f6f61
+ system.activationScripts.sbinlvm =
+ ''
+ mkdir -m 0755 -p /sbin
+ ln -sfn ${pkgs.lvm2.bin}/bin/lvm /sbin/lvm
+ '';
+
+ # Enable the OpenSSH daemon.
+ services.openssh.enable = true;
+ users.users.root.openssh.authorizedKeys.keys = [
+ "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDD4VJXAXEHEXZk2dxNwehneuJcEGkfXG/U7z4fO79vDVIENdedtXQUyLyhZJc5RTEfHhQj66FwIqzl7mzBHd9x9PuDp6QAYXrkVNMj48s6JXqZqBvF6H/weRqFMf4a2TZv+hG8D0kpvmLheCwWAVRls7Jofnp/My+yDd57GMdsbG/yFEf6WPMiOnA7hxdSJSVihCsCSw2p8PD4GhBe8CVt7xIuinhutjm9zYBjV78NT8acjDUfJh0B1ODTjs7nuW1CC4jybSe2j/OU3Yczj4AxRxBNWuFxUq+jBo9BfpbKLh+Tt7re+zBkaicM77KM/oV6943JJxgHNBBOsv9scZE7 q3k@amnesia"
+ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIG599UildOrAq+LIOQjKqtGMwjgjIxozI1jtQQRKHtCP q3k@mimeomia"
+ "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDQb3YQoiYFZLKwvHYKbu1bMqzNeDCAszQhAe1+QI5SLDOotclyY/vFmOReZOsmyMFl71G2d7d+FbYNusUnNNjTxRYQ021tVc+RkMdLJaORRURmQfEFEKbai6QSFTwErXzuoIzyEPK0lbsQuGgqT9WaVnRzHJ2Q/4+qQbxAS34PuR5NqEkmn4G6LMo3OyJ5mwPkCj9lsqz4BcxRaMWFO3mNcwGDfSW+sqgc3E8N6LKrTpZq3ke7xacpQmcG5DU9VO+2QVPdltl9jWbs3gXjmF92YRNOuKPVfAOZBBsp8JOznfx8s9wDgs7RwPmDpjIAJEyoABqW5hlXfqRbTnfnMvuR informatic@InformaticPC"
+ "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDGkMgEVwQM8yeuFUYL2TwlJIq9yUNBmHnwce46zeL2PK2CkMz7sxT/om7sp/K5XDiqeD05Nioe+Dr3drP6B8uI33S5NgxPIfaqQsRS+CBEgk6cqFlcdlKETU/DT+/WsdoO173n7mgGeafPInEuQuGDUID0Fl099kIxtqfAhdeZFMM6/szAZEZsElLJ8K6dp1Ni/jmnXCZhjivZH3AZUlnqrmtDG7FY1bgcOfDXAal45LItughGPtrdiigXe9DK2fW3+9DBZZduh5DMJTNlphAZ+nfSrbyHVKUg6WsgMSprur4KdU47q1QwzqqvEj75JcdP1jOWoZi4F6VJDte9Wb9lhD1jGgjxY9O6Gs4CH35bx15W7CN9hgNa0C8NbPJe/fZYIeMZmJ1m7O2xmnYwP8j+t7RNJWu7Pa3Em4mOEXvhBF07Zfq+Ye/4SluoRgADy5eII2x5fFo5EBhInxK0/X8wF6XZvysalVifoCh7T4Edejoi91oAxFgYAxbboXGlod0eEHIi2hla8SM9+IBHOChmgawKBYp2kzAJyAmHNBF+Pah9G4arVCj/axp/SJZDZbJQoI7UT/fJzEtvlb5RWrHXRq+y6IvjpUq4pzpDWW04+9UMqEEXRmhWOakHfEVM9rN8h3aJBflLUBBnh0Z/hVsKNh8bCRHaKtah8TrD9i+wMw== patryk.jakuszew@gmail.com"
+ "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC33naG1ptCvUcRWX9cj9wXM1nW1lyQC4SvMJzWlr9aMD96O8hQ2JMkuIUgUJvorAY02QRplQ2BuoVoVkdkzwjMyi1bL3OdgcKo7Z1yByClGTTocqNJYY0lcUb6EJH8+6e6F9ydrQlSxNzL1uCaA7phZr+yPcmAmWbSfioXn98yXNkE0emHxzJv/nypJY56sDCMC2IXDRd8L2goDtPwgPEW7bWfAQdIFMJ75xOidZOTxJ8eqyXLw/kxY5UlyX66jdoYz1sE5XUHuoQl1AOG9UdlMo0aMhUvP4pX5l7r7EnA9OttKMFB3oWqkVK/R6ynZ52YNOU5BZ9V+Ppaj34W0xNu+p0mbHcCtXYCTrf/OU0hcZDbDaNTjs6Vtcm2wYw9iAKX7Tex+eOMwUwlrlcyPNRV5BTot7lGNYfauHCSIuWJKN4NhCLR/NtVNh4/94eKkPTwJsY6XqDcS7q49wPAs4DAH7BJgsbHPOqygVHrY0YYEfz3Pj0HTxJHQMCP/hQX4fXEGt0BjgoVJbXPAQtPyeg0JuxiUg+b4CgVVfQ6R060MlM1BZzhmh+FY5MJH6nJppS0aHYCvSg8Z68NUlCPKy0jpcyfuAIWQWwSGG1O010WShQG2ELsvNdg5/4HVdCGNl5mmoom6JOd72FOZyQlHDFfeQUQRn9HOeCq/c51rK99SQ== bartek@IHM"
+ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICTR292kx/2CNuWYIsZ6gykQ036aBGrmheIuZa6S1D2x implr@thonk"
+ ];
+ };
+}