Sergiusz Bazanski | c78cc13 | 2020-02-02 22:31:53 +0100 | [diff] [blame] | 1 | machineName: |
| 2 | |
| 3 | let |
| 4 | machines = (import ./defs-machines.nix); |
| 5 | in rec { |
| 6 | domain = ".hswaw.net"; |
| 7 | k8sapi = "k0.hswaw.net"; |
| 8 | acmeEmail = "q3k@hackerspace.pl"; |
| 9 | |
| 10 | fqdn = machineName + domain; |
| 11 | machine = (builtins.head (builtins.filter (n: n.fqdn == fqdn) machines)); |
| 12 | otherMachines = (builtins.filter (n: n.fqdn != fqdn) machines); |
Serge Bazanski | 9848e7e | 2021-09-10 22:30:56 +0000 | [diff] [blame] | 13 | machinesByName = builtins.listToAttrs (map (m: { name = m.name; value = m; }) machines); |
Sergiusz Bazanski | c78cc13 | 2020-02-02 22:31:53 +0100 | [diff] [blame] | 14 | inherit machines; |
| 15 | |
Serge Bazanski | 9848e7e | 2021-09-10 22:30:56 +0000 | [diff] [blame] | 16 | # Ceph cluster to run systemd modules for. |
| 17 | cephCluster = { |
| 18 | fsid = "74592dc2-31b7-4dbe-88cf-40459dfeb354"; |
| 19 | name = "k0"; |
| 20 | |
| 21 | # Map from node name to mon configuration (currently always empty). |
| 22 | # |
| 23 | # Each mon also runs a mgr daemon (which is a leader-elected kitchen |
| 24 | # sink^W^Whousekeeping service hanging off of a mon cluster). |
| 25 | # |
| 26 | # Consult the Ceph documentation |
| 27 | # (https://docs.ceph.com/en/pacific/rados/operations/add-or-rm-mons/) on |
| 28 | # how to actually carry out mon-related maintenance operations. |
| 29 | mons = { |
| 30 | bc01n02 = {}; |
| 31 | }; |
| 32 | |
| 33 | # Map from node name to list of disks on node. |
| 34 | # Each disk is: |
| 35 | # id: OSD numerical ID, eg. 0 for osd.0. You get this after running |
| 36 | # ceph-lvm volume create. |
| 37 | # path: Filesystem path for disk backing drive. This should be something |
| 38 | # in /dev/disk/by-id for safety. This is only used to gate OSD |
| 39 | # daemon startup by disk presence. |
| 40 | # uuid: OSD uuid/fsid. You get this after running ceph-lvm volume create. |
| 41 | # |
| 42 | # Quick guide how to set up a new OSD (but please refer to the Ceph manual): |
| 43 | # 0. Copy /var/lib/ceph/bootstrap-osd/k0.keyring from another OSD node to |
| 44 | # the new OSD node, if this is a new node. Remember to chown ceph:ceph |
| 45 | # chmod 0600! |
| 46 | # 1. nix-shell -p ceph lvm2 cryptsetup (if on a node that's not yet an OSD) |
| 47 | # 2. ceph-volume --cluster k0 lvm create --bluestore --data /dev/sdX --no-systemd --dmcrypt |
| 48 | # 3. The above will mount a tmpfs on /var/lib/ceph/osd/k0-X. X is the new |
| 49 | # osd id. A file named fsid inside this directory is the new OSD fsid/uuid. |
| 50 | # 4. Configure osds below with the above information, redeploy node from nix. |
| 51 | osds = { |
| 52 | dcr01s22 = [ |
| 53 | { id = 0; path = "/dev/disk/by-id/scsi-35000c500850293e3"; uuid = "314034c5-474c-4d0d-ba41-36a881c52560";} |
| 54 | { id = 1; path = "/dev/disk/by-id/scsi-35000c500850312cb"; uuid = "a7f1baa0-0fc3-4ab1-9895-67abdc29de03";} |
| 55 | { id = 2; path = "/dev/disk/by-id/scsi-35000c5008508e3ef"; uuid = "11ac8316-6a87-48a7-a0c7-74c3cef6c2fa";} |
| 56 | { id = 3; path = "/dev/disk/by-id/scsi-35000c5008508e23f"; uuid = "c6b838d1-b08c-4788-936c-293041ed2d4d";} |
| 57 | ]; |
| 58 | dcr01s24 = [ |
| 59 | { id = 4; path = "/dev/disk/by-id/scsi-35000c5008509199b"; uuid = "a2b4663d-bd8f-49b3-b0b0-195c56ba252f";} |
| 60 | { id = 5; path = "/dev/disk/by-id/scsi-35000c50085046abf"; uuid = "a2242989-ccce-4367-8813-519b64b5afdb";} |
| 61 | { id = 6; path = "/dev/disk/by-id/scsi-35000c5008502929b"; uuid = "7deac89c-22dd-4c2b-b3cc-43ff7f990fd6";} |
| 62 | { id = 7; path = "/dev/disk/by-id/scsi-35000c5008502a323"; uuid = "e305ebb3-9cac-44d2-9f1d-bbb72c8ab51f";} |
| 63 | ]; |
| 64 | }; |
| 65 | }; |
| 66 | |
Sergiusz Bazanski | c78cc13 | 2020-02-02 22:31:53 +0100 | [diff] [blame] | 67 | pki = rec { |
| 68 | make = (radix: name: rec { |
| 69 | ca = ./../certs + "/ca-${radix}.crt"; |
| 70 | cert = ./../certs + "/${radix}-${name}.cert"; |
| 71 | key = ./../secrets/plain + "/${radix}-${name}.key"; |
| 72 | |
| 73 | json = (builtins.toJSON { |
| 74 | ca = (builtins.toString ca); |
| 75 | cert = (builtins.toString cert); |
| 76 | key = (builtins.toString key); |
| 77 | }); |
| 78 | }); |
| 79 | |
| 80 | etcdPeer = (make "etcdpeer" fqdn); |
| 81 | |
| 82 | etcd = { |
| 83 | server = (make "etcd" fqdn); |
| 84 | kube = (make "etcd" "kube"); |
| 85 | }; |
| 86 | |
| 87 | makeKube = (name: (make "kube" name) // { |
| 88 | config = { |
| 89 | server = "https://${k8sapi}:${toString ports.k8sAPIServerSecure}"; |
| 90 | certFile = (make "kube" name).cert; |
| 91 | keyFile = (make "kube" name).key; |
| 92 | }; |
| 93 | }); |
| 94 | |
| 95 | kube = rec { |
| 96 | ca = apiserver.ca; |
| 97 | |
| 98 | # Used to identify apiserver. |
| 99 | apiserver = (makeKube "apiserver"); |
| 100 | |
| 101 | # Used to identify controller-manager. |
| 102 | controllermanager = (makeKube "controllermanager"); |
| 103 | |
| 104 | # Used to identify scheduler. |
| 105 | scheduler = (makeKube "scheduler"); |
| 106 | |
| 107 | # Used to identify kube-proxy. |
| 108 | proxy = (makeKube "proxy"); |
| 109 | |
| 110 | # Used to identify kubelet. |
| 111 | kubelet = (makeKube "kubelet-${fqdn}"); |
| 112 | |
| 113 | # Used to encrypt service accounts. |
| 114 | serviceaccounts = (makeKube "serviceaccounts"); |
| 115 | }; |
| 116 | |
| 117 | kubeFront = { |
| 118 | apiserver = (make "kubefront" "apiserver"); |
| 119 | }; |
| 120 | }; |
| 121 | |
| 122 | ports = { |
| 123 | k8sAPIServerPlain = 4000; |
| 124 | k8sAPIServerSecure = 4001; |
Serge Bazanski | 1257389 | 2020-10-10 14:55:08 +0200 | [diff] [blame] | 125 | k8sControllerManagerPlain = 0; # would be 4002; do not serve plain http |
Sergiusz Bazanski | c78cc13 | 2020-02-02 22:31:53 +0100 | [diff] [blame] | 126 | k8sControllerManagerSecure = 4003; |
Serge Bazanski | 1257389 | 2020-10-10 14:55:08 +0200 | [diff] [blame] | 127 | k8sSchedulerPlain = 0; # would be 4004; do not serve plain http |
| 128 | k8sSchedulerSecure = 4005; |
Sergiusz Bazanski | c78cc13 | 2020-02-02 22:31:53 +0100 | [diff] [blame] | 129 | }; |
| 130 | } |