Serge Bazanski | 55a486a | 2022-06-11 18:27:01 +0000 | [diff] [blame] | 1 | { config, pkgs, lib, machines, ... }: |
| 2 | |
| 3 | with lib; |
| 4 | |
| 5 | let |
| 6 | cfg = config.hscloud.kube.control; |
| 7 | |
| 8 | # All control plane nodes. |
| 9 | allNodes = let |
| 10 | list = mapAttrsToList (_: v: v) machines; |
| 11 | filtered = filter (m: (m.config ? hscloud.kube.control) && (m.config.hscloud.kube.control.enable)) list; |
| 12 | sorted = sort (a: b: a.config.hscloud.base.fqdn < b.config.hscloud.base.fqdn) filtered; |
| 13 | in sorted; |
| 14 | |
| 15 | # All control plane nodes that aren't the node being evaluated. |
| 16 | otherNodes = (filter (m: m.config.networking.hostName != config.networking.hostName) allNodes); |
| 17 | |
| 18 | fqdn = config.hscloud.base.fqdn; |
| 19 | |
| 20 | pki = config.hscloud.kube.pki; |
| 21 | |
| 22 | in { |
| 23 | imports = [ |
| 24 | ./kube-common.nix |
| 25 | ]; |
| 26 | |
| 27 | options.hscloud.kube.control = { |
| 28 | enable = mkEnableOption "kubernetes control plane"; |
| 29 | portControllerManagerSecure = mkOption { |
| 30 | type = types.int; |
| 31 | description = "Port at which k8s controller-manager will listen."; |
| 32 | default = 4003; |
| 33 | }; |
| 34 | portSchedulerSecure = mkOption { |
| 35 | type = types.int; |
| 36 | description = "Port at which k8s scheduler will listen."; |
| 37 | default = 4005; |
| 38 | }; |
| 39 | }; |
| 40 | |
| 41 | config = mkIf cfg.enable { |
| 42 | networking.firewall.enable = false; |
| 43 | |
| 44 | # Point k8s apiserver address at ourselves, as we _are_ the apiserver. |
| 45 | networking.extraHosts = '' |
| 46 | 127.0.0.1 k0.hswaw.net |
| 47 | ''; |
| 48 | |
| 49 | services.etcd = rec { |
| 50 | enable = true; |
| 51 | name = fqdn; |
| 52 | listenClientUrls = ["https://0.0.0.0:2379"]; |
| 53 | advertiseClientUrls = ["https://${fqdn}:2379"]; |
| 54 | listenPeerUrls = ["https://0.0.0.0:2380"]; |
| 55 | initialAdvertisePeerUrls = ["https://${fqdn}:2380"]; |
| 56 | initialCluster = (map (n: "${n.config.hscloud.base.fqdn}=https://${n.config.hscloud.base.fqdn}:2380") allNodes); |
| 57 | initialClusterState = "existing"; |
| 58 | |
| 59 | clientCertAuth = true; |
| 60 | trustedCaFile = pki.etcd.server.ca; |
| 61 | certFile = pki.etcd.server.cert; |
| 62 | keyFile = pki.etcd.server.key; |
| 63 | |
| 64 | peerClientCertAuth = true; |
| 65 | peerTrustedCaFile = pki.etcd.peer.ca; |
| 66 | peerCertFile = pki.etcd.peer.cert; |
| 67 | peerKeyFile = pki.etcd.peer.key; |
| 68 | |
| 69 | extraConf = { |
| 70 | PEER_CLIENT_CERT_AUTH = "true"; |
| 71 | }; |
| 72 | }; |
| 73 | |
| 74 | # https://github.com/NixOS/nixpkgs/issues/60687 |
| 75 | systemd.services.kube-control-plane-online = { |
| 76 | preStart = pkgs.lib.mkForce ""; |
| 77 | }; |
| 78 | |
| 79 | services.kubernetes = { |
| 80 | package = config.hscloud.kube.package; |
| 81 | # We do not use any nixpkgs predefined roles for k8s. Instead, we enable |
| 82 | # k8s components manually. |
| 83 | roles = []; |
| 84 | addons.dns.enable = false; |
Serge Bazanski | 9251121 | 2023-04-01 13:50:02 +0000 | [diff] [blame] | 85 | caFile = pki.kube.apiserver.ca; |
Serge Bazanski | 55a486a | 2022-06-11 18:27:01 +0000 | [diff] [blame] | 86 | clusterCidr = "10.10.16.0/20"; |
| 87 | |
| 88 | apiserver = rec { |
| 89 | enable = true; |
| 90 | # BUG: should be 0. |
| 91 | insecurePort = 4000; |
| 92 | securePort = config.hscloud.kube.portAPIServerSecure; |
| 93 | advertiseAddress = config.hscloud.base.ipAddr; |
| 94 | |
| 95 | etcd = { |
| 96 | # Only point at our own etcd. |
| 97 | servers = [ "https://${fqdn}:2379" ]; |
| 98 | caFile = pki.etcd.kube.ca; |
| 99 | keyFile = pki.etcd.kube.key; |
| 100 | certFile = pki.etcd.kube.cert; |
| 101 | }; |
| 102 | |
| 103 | tlsCertFile = pki.kube.apiserver.cert; |
| 104 | tlsKeyFile = pki.kube.apiserver.key; |
Serge Bazanski | 9251121 | 2023-04-01 13:50:02 +0000 | [diff] [blame] | 105 | clientCaFile = pki.kube.apiserver.ca; |
Serge Bazanski | 55a486a | 2022-06-11 18:27:01 +0000 | [diff] [blame] | 106 | |
| 107 | kubeletHttps = true; |
| 108 | # Same CA as main APIServer CA. |
Serge Bazanski | 9251121 | 2023-04-01 13:50:02 +0000 | [diff] [blame] | 109 | kubeletClientCaFile = pki.kube.apiserver.ca; |
Serge Bazanski | 55a486a | 2022-06-11 18:27:01 +0000 | [diff] [blame] | 110 | kubeletClientCertFile = pki.kube.apiserver.cert; |
| 111 | kubeletClientKeyFile = pki.kube.apiserver.key; |
| 112 | |
| 113 | serviceAccountKeyFile = pki.kube.serviceaccounts.key; |
| 114 | |
| 115 | allowPrivileged = true; |
| 116 | serviceClusterIpRange = "10.10.12.0/24"; |
| 117 | runtimeConfig = "api/all,authentication.k8s.io/v1beta1"; |
| 118 | authorizationMode = [ |
| 119 | "Node" "RBAC" |
| 120 | ]; |
| 121 | enableAdmissionPlugins = [ |
| 122 | "NamespaceLifecycle" "NodeRestriction" "LimitRanger" "ServiceAccount" |
| 123 | "DefaultStorageClass" "ResourceQuota" "PodSecurityPolicy" |
| 124 | ]; |
| 125 | extraOpts = '' |
| 126 | --apiserver-count=5 \ |
| 127 | --proxy-client-cert-file=${pki.kubeFront.apiserver.cert} \ |
| 128 | --proxy-client-key-file=${pki.kubeFront.apiserver.key} \ |
| 129 | --requestheader-allowed-names= \ |
| 130 | --requestheader-client-ca-file=${pki.kubeFront.apiserver.ca} \ |
| 131 | --requestheader-extra-headers-prefix=X-Remote-Extra- \ |
| 132 | --requestheader-group-headers=X-Remote-Group \ |
| 133 | --requestheader-username-headers=X-Remote-User \ |
| 134 | -v=5 |
| 135 | ''; |
| 136 | }; |
| 137 | |
| 138 | controllerManager = let |
| 139 | top = config.services.kubernetes; |
| 140 | kubeconfig = top.lib.mkKubeConfig "controller-manager" pki.kube.controllermanager.config; |
| 141 | in { |
| 142 | enable = true; |
| 143 | bindAddress = "0.0.0.0"; |
| 144 | insecurePort = 0; |
| 145 | leaderElect = true; |
| 146 | serviceAccountKeyFile = pki.kube.serviceaccounts.key; |
| 147 | rootCaFile = pki.kube.ca; |
| 148 | extraOpts = '' |
| 149 | --service-cluster-ip-range=10.10.12.0/24 \ |
| 150 | --use-service-account-credentials=true \ |
| 151 | --secure-port=${toString cfg.portControllerManagerSecure}\ |
| 152 | --authentication-kubeconfig=${kubeconfig}\ |
| 153 | --authorization-kubeconfig=${kubeconfig}\ |
| 154 | ''; |
| 155 | kubeconfig = pki.kube.controllermanager.config; |
| 156 | }; |
| 157 | |
| 158 | scheduler = let |
| 159 | top = config.services.kubernetes; |
Serge Bazanski | 9251121 | 2023-04-01 13:50:02 +0000 | [diff] [blame] | 160 | kubeconfig = top.lib.mkKubeConfig "scheduler" pki.kube.scheduler.config; |
Serge Bazanski | 55a486a | 2022-06-11 18:27:01 +0000 | [diff] [blame] | 161 | in { |
| 162 | enable = true; |
| 163 | address = "0.0.0.0"; |
| 164 | port = 0; |
| 165 | leaderElect = true; |
| 166 | kubeconfig = pki.kube.scheduler.config; |
| 167 | extraOpts = '' |
| 168 | --secure-port=${toString cfg.portSchedulerSecure}\ |
| 169 | --authentication-kubeconfig=${kubeconfig}\ |
| 170 | --authorization-kubeconfig=${kubeconfig}\ |
| 171 | ''; |
| 172 | }; |
| 173 | }; |
| 174 | }; |
| 175 | } |
| 176 | |