Serge Bazanski | 6abe4fa | 2020-10-03 00:18:34 +0200 | [diff] [blame] | 1 | # Top-level file aggregating all machines managed from hscloud. |
| 2 | # |
| 3 | # This allows to have a common attrset of machines that can be deployed |
| 4 | # in the same way. |
| 5 | # |
Serge Bazanski | b3c6770 | 2021-09-10 22:27:24 +0000 | [diff] [blame] | 6 | # For information about building/deploying machines see //ops/README.md. |
Serge Bazanski | 6abe4fa | 2020-10-03 00:18:34 +0200 | [diff] [blame] | 7 | |
| 8 | { hscloud, pkgs, ... }: |
| 9 | |
| 10 | let |
Serge Bazanski | b3c6770 | 2021-09-10 22:27:24 +0000 | [diff] [blame] | 11 | # nixpkgs for cluster machines (.hswaw.net). Currently pinned to an old |
| 12 | # nixpkgs because NixOS modules for kubernetes changed enough that it's not |
| 13 | # super easy to use them as is. |
| 14 | # |
| 15 | # TODO(q3k): fix this: use an old nixpkgs for Kube modules while using |
| 16 | # hscloud nixpkgs for everything else. |
| 17 | nixpkgsCluster = import (pkgs.fetchFromGitHub { |
| 18 | owner = "nixos"; |
| 19 | repo = "nixpkgs-channels"; |
| 20 | rev = "44ad80ab1036c5cc83ada4bfa451dac9939f2a10"; |
| 21 | sha256 = "1b61nzvy0d46cspy07szkc0rggacxiqg9v1py27pkqpj7rvawfsk"; |
Serge Bazanski | 9848e7e | 2021-09-10 22:30:56 +0000 | [diff] [blame] | 22 | }) { |
| 23 | overlays = [ |
| 24 | (self: super: rec { |
| 25 | # Use a newer version of Ceph (16, Pacific, EOL 2023-06-01) than in |
| 26 | # this nixpkgs (15, Octopus, EOL 2022-06-01). |
| 27 | # |
| 28 | # This is to: |
| 29 | # 1. Fix a bug in which ceph-volume lvm create fails due to a rocksdb |
| 30 | # mismatch (https://tracker.ceph.com/issues/49815) |
| 31 | # 2. At the time of deployment not start out with an ancient version |
| 32 | # of Ceph. |
| 33 | # |
| 34 | # Once we unpin nixpkgsCluster past a version that contains this Ceph, |
| 35 | # this can be unoverlayed. |
| 36 | inherit (super.callPackages ./ceph { |
| 37 | boost = super.boost17x.override { enablePython = true; python = super.python3; }; |
| 38 | lua = super.lua5_4; |
| 39 | }) ceph ceph-client; |
| 40 | ceph-lib = ceph.lib; |
| 41 | }) |
| 42 | ]; |
| 43 | }; |
Serge Bazanski | b3c6770 | 2021-09-10 22:27:24 +0000 | [diff] [blame] | 44 | |
| 45 | # edge01 still lives on an old nixpkgs checkout. |
| 46 | # |
| 47 | # TODO(b/3): unpin and deploy. |
| 48 | nixpkgsBgpwtf = import (pkgs.fetchFromGitHub { |
| 49 | owner = "nixos"; |
| 50 | repo = "nixpkgs-channels"; |
| 51 | rev = "c59ea8b8a0e7f927e7291c14ea6cd1bd3a16ff38"; |
| 52 | sha256 = "1ak7jqx94fjhc68xh1lh35kh3w3ndbadprrb762qgvcfb8351x8v"; |
| 53 | }) {}; |
| 54 | |
Serge Bazanski | 6abe4fa | 2020-10-03 00:18:34 +0200 | [diff] [blame] | 55 | # Stopgap measure to import //cluster/nix machine definitions into new |
Serge Bazanski | b3c6770 | 2021-09-10 22:27:24 +0000 | [diff] [blame] | 56 | # //ops/ infrastructure. |
| 57 | # |
Serge Bazanski | 6abe4fa | 2020-10-03 00:18:34 +0200 | [diff] [blame] | 58 | # TODO(q3k): inject defs-cluster-k0.nix / defs-machines.nix content via |
| 59 | # nixos options instead of having module definitions loading it themselves, |
| 60 | # deduplicate list of machines below with defs-machines.nix somehow. |
Serge Bazanski | b3c6770 | 2021-09-10 22:27:24 +0000 | [diff] [blame] | 61 | clusterMachineConfig = name: [({ config, pkgs, ...}: { |
Serge Bazanski | 6abe4fa | 2020-10-03 00:18:34 +0200 | [diff] [blame] | 62 | # The hostname is used by //cluster/nix machinery to load the appropriate |
| 63 | # config from defs-machines into defs-cluster-k0. |
| 64 | networking.hostName = name; |
| 65 | imports = [ |
| 66 | ../cluster/nix/modules/base.nix |
| 67 | ../cluster/nix/modules/kubernetes.nix |
Serge Bazanski | 9848e7e | 2021-09-10 22:30:56 +0000 | [diff] [blame] | 68 | ../cluster/nix/modules/ceph.nix |
Serge Bazanski | 6abe4fa | 2020-10-03 00:18:34 +0200 | [diff] [blame] | 69 | ]; |
Serge Bazanski | b3c6770 | 2021-09-10 22:27:24 +0000 | [diff] [blame] | 70 | })]; |
Serge Bazanski | 6abe4fa | 2020-10-03 00:18:34 +0200 | [diff] [blame] | 71 | |
Serge Bazanski | b3c6770 | 2021-09-10 22:27:24 +0000 | [diff] [blame] | 72 | # mkMachine builds NixOS modules into a NixOS derivation, and injects |
| 73 | # passthru.hscloud.provision which deploys that configuration over SSH to a |
| 74 | # production machine. |
Serge Bazanski | a0332a7 | 2021-03-17 22:12:43 +0100 | [diff] [blame] | 75 | mkMachine = pkgs: paths: pkgs.nixos ({ config, pkgs, ... }: { |
Serge Bazanski | 6abe4fa | 2020-10-03 00:18:34 +0200 | [diff] [blame] | 76 | imports = paths; |
Serge Bazanski | b3c6770 | 2021-09-10 22:27:24 +0000 | [diff] [blame] | 77 | |
| 78 | config = let |
| 79 | name = config.networking.hostName; |
| 80 | domain = if (config.networking ? domain) && config.networking.domain != null then config.networking.domain else "hswaw.net"; |
| 81 | fqdn = name + "." + domain; |
| 82 | toplevel = config.system.build.toplevel; |
| 83 | |
| 84 | runProvision = '' |
| 85 | #!/bin/sh |
| 86 | set -eu |
| 87 | remote=root@${fqdn} |
| 88 | echo "Configuration for ${fqdn} is ${toplevel}" |
| 89 | nix copy -s --to ssh://$remote ${toplevel} |
| 90 | |
| 91 | running="$(ssh $remote readlink -f /nix/var/nix/profiles/system)" |
| 92 | if [ "$running" == "${toplevel}" ]; then |
| 93 | echo "${fqdn} already running ${toplevel}." |
| 94 | else |
| 95 | echo "/etc/systemd/system diff:" |
| 96 | ssh $remote diff -ur /var/run/current-system/etc/systemd/system ${toplevel}/etc/systemd/system || true |
| 97 | echo "" |
| 98 | echo "" |
| 99 | echo "dry-activate diff:" |
| 100 | ssh $remote ${toplevel}/bin/switch-to-configuration dry-activate |
| 101 | read -p "Do you want to switch to this configuration? " -n 1 -r |
| 102 | echo |
| 103 | if ! [[ $REPLY =~ ^[Yy]$ ]]; then |
| 104 | exit 1 |
| 105 | fi |
| 106 | |
| 107 | echo -ne "\n\nswitch-to-configuration test...\n" |
| 108 | ssh $remote ${toplevel}/bin/switch-to-configuration test |
| 109 | fi |
| 110 | |
| 111 | echo -ne "\n\n" |
| 112 | read -p "Do you want to set this configuration as boot? " -n 1 -r |
| 113 | echo |
| 114 | if ! [[ $REPLY =~ ^[Yy]$ ]]; then |
| 115 | exit 1 |
| 116 | fi |
| 117 | |
| 118 | echo -ne "\n\nsetting system profile...\n" |
| 119 | ssh $remote nix-env -p /nix/var/nix/profiles/system --set ${toplevel} |
| 120 | |
| 121 | echo -ne "\n\nswitch-to-configuration boot...\n" |
| 122 | ssh $remote ${toplevel}/bin/switch-to-configuration boot |
| 123 | ''; |
| 124 | in { |
| 125 | passthru.hscloud.provision = pkgs.writeScript "provision-${fqdn}" runProvision; |
| 126 | }; |
Serge Bazanski | 6abe4fa | 2020-10-03 00:18:34 +0200 | [diff] [blame] | 127 | }); |
Serge Bazanski | 6abe4fa | 2020-10-03 00:18:34 +0200 | [diff] [blame] | 128 | in { |
Serge Bazanski | b3c6770 | 2021-09-10 22:27:24 +0000 | [diff] [blame] | 129 | "bc01n01.hswaw.net" = mkMachine nixpkgsCluster (clusterMachineConfig "bc01n01"); |
| 130 | "bc01n02.hswaw.net" = mkMachine nixpkgsCluster (clusterMachineConfig "bc01n02"); |
| 131 | "dcr01s22.hswaw.net" = mkMachine nixpkgsCluster (clusterMachineConfig "dcr01s22"); |
| 132 | "dcr01s24.hswaw.net" = mkMachine nixpkgsCluster (clusterMachineConfig "dcr01s24"); |
Serge Bazanski | 6abe4fa | 2020-10-03 00:18:34 +0200 | [diff] [blame] | 133 | |
Serge Bazanski | b3c6770 | 2021-09-10 22:27:24 +0000 | [diff] [blame] | 134 | "edge01.waw.bgp.wtf" = mkMachine nixpkgsBgpwtf [ |
Serge Bazanski | 6abe4fa | 2020-10-03 00:18:34 +0200 | [diff] [blame] | 135 | ../bgpwtf/machines/edge01.waw.bgp.wtf.nix |
| 136 | ../bgpwtf/machines/edge01.waw.bgp.wtf-hardware.nix |
| 137 | ]; |
| 138 | } |