Sergiusz Bazanski | c78cc13 | 2020-02-02 22:31:53 +0100 | [diff] [blame] | 1 | { config, pkgs, lib, ... }: |
| 2 | |
Serge Bazanski | fbe234b | 2020-10-03 00:13:28 +0200 | [diff] [blame] | 3 | with (( import ../defs-cluster-k0.nix ) config.networking.hostName); |
Sergiusz Bazanski | c78cc13 | 2020-02-02 22:31:53 +0100 | [diff] [blame] | 4 | let |
| 5 | # Pin for k8s packages. This is so that upagrading the system will not upgrade the k8s control or data planes. |
| 6 | k8spkgs = import (fetchGit { |
Serge Bazanski | e77f771 | 2020-10-10 22:39:50 +0200 | [diff] [blame] | 7 | # Now at 1.16.5 |
| 8 | name = "nixos-unstable-2020-01-22"; |
Sergiusz Bazanski | c78cc13 | 2020-02-02 22:31:53 +0100 | [diff] [blame] | 9 | url = https://github.com/nixos/nixpkgs-channels/; |
Serge Bazanski | e77f771 | 2020-10-10 22:39:50 +0200 | [diff] [blame] | 10 | rev = "a96ed5d70427bdc2fbb9e805784e1b9621157a98"; |
Sergiusz Bazanski | c78cc13 | 2020-02-02 22:31:53 +0100 | [diff] [blame] | 11 | }) {}; |
| 12 | # Pin for kubelet |
| 13 | k8spkgsKubelet = import (fetchGit { |
Serge Bazanski | e77f771 | 2020-10-10 22:39:50 +0200 | [diff] [blame] | 14 | # Now at 1.16.5 |
| 15 | name = "nixos-unstable-2020-01-22"; |
Sergiusz Bazanski | c78cc13 | 2020-02-02 22:31:53 +0100 | [diff] [blame] | 16 | url = https://github.com/nixos/nixpkgs-channels/; |
Serge Bazanski | e77f771 | 2020-10-10 22:39:50 +0200 | [diff] [blame] | 17 | rev = "a96ed5d70427bdc2fbb9e805784e1b9621157a98"; |
Sergiusz Bazanski | c78cc13 | 2020-02-02 22:31:53 +0100 | [diff] [blame] | 18 | }) {}; |
| 19 | |
| 20 | in rec { |
| 21 | # Disable kubelet service and bring in our own override. |
| 22 | # Also nuke flannel from the orbit. |
| 23 | disabledModules = [ |
| 24 | "services/cluster/kubernetes/kubelet.nix" |
| 25 | "services/cluster/kubernetes/flannel.nix" |
| 26 | ]; |
| 27 | |
| 28 | imports = |
| 29 | [ |
Serge Bazanski | fbe234b | 2020-10-03 00:13:28 +0200 | [diff] [blame] | 30 | ./kubelet.nix |
Sergiusz Bazanski | c78cc13 | 2020-02-02 22:31:53 +0100 | [diff] [blame] | 31 | ]; |
| 32 | |
Sergiusz Bazanski | c78cc13 | 2020-02-02 22:31:53 +0100 | [diff] [blame] | 33 | networking.firewall.enable = false; |
| 34 | |
| 35 | # Point k8s apiserver address at ourselves, as every machine runs an apiserver with this cert name. |
| 36 | networking.extraHosts = '' |
| 37 | 127.0.0.1 ${k8sapi} |
| 38 | ''; |
| 39 | |
Serge Bazanski | 3164117 | 2020-08-23 00:58:29 +0200 | [diff] [blame] | 40 | security.acme.acceptTerms = true; |
Sergiusz Bazanski | c78cc13 | 2020-02-02 22:31:53 +0100 | [diff] [blame] | 41 | security.acme.certs = { |
| 42 | host = { |
| 43 | email = acmeEmail; |
| 44 | domain = fqdn; |
| 45 | webroot = services.nginx.virtualHosts.host.root; |
| 46 | }; |
| 47 | }; |
| 48 | |
| 49 | services.nginx = { |
| 50 | enable = true; |
| 51 | virtualHosts.host = { |
| 52 | serverName = fqdn; |
| 53 | root = "/var/www/${fqdn}"; |
| 54 | }; |
| 55 | }; |
| 56 | |
| 57 | services.etcd = rec { |
| 58 | enable = true; |
| 59 | name = fqdn; |
| 60 | listenClientUrls = ["https://0.0.0.0:2379"]; |
| 61 | advertiseClientUrls = ["https://${fqdn}:2379"]; |
| 62 | listenPeerUrls = ["https://0.0.0.0:2380"]; |
| 63 | initialAdvertisePeerUrls = ["https://${fqdn}:2380"]; |
| 64 | initialCluster = (map (n: "${n.fqdn}=https://${n.fqdn}:2380") machines); |
| 65 | initialClusterState = "existing"; |
| 66 | |
| 67 | clientCertAuth = true; |
| 68 | trustedCaFile = pki.etcd.server.ca; |
| 69 | certFile = pki.etcd.server.cert; |
| 70 | keyFile = pki.etcd.server.key; |
| 71 | |
| 72 | peerClientCertAuth = true; |
| 73 | peerTrustedCaFile = pki.etcdPeer.ca; |
| 74 | peerCertFile = pki.etcdPeer.cert; |
| 75 | peerKeyFile = pki.etcdPeer.key; |
| 76 | |
| 77 | extraConf = { |
| 78 | PEER_CLIENT_CERT_AUTH = "true"; |
| 79 | }; |
| 80 | }; |
| 81 | |
| 82 | services.kubernetes = { |
| 83 | # Pin to specific k8s package. |
| 84 | package = k8spkgs.kubernetes; |
| 85 | roles = []; # We do not use any nixpkgs predefined roles for k8s. Instead, |
| 86 | # we enable k8s components manually. |
| 87 | |
| 88 | caFile = pki.kube.apiserver.ca; |
| 89 | clusterCidr = "10.10.16.0/20"; |
| 90 | |
| 91 | path = [ pkgs.e2fsprogs ]; # kubelet wants to mkfs.ext4 when mounting pvcs |
| 92 | |
| 93 | addons.dns.enable = false; |
| 94 | |
| 95 | apiserver = rec { |
| 96 | enable = true; |
| 97 | insecurePort = ports.k8sAPIServerPlain; |
| 98 | securePort = ports.k8sAPIServerSecure; |
| 99 | advertiseAddress = "${machine.ipAddr}"; |
| 100 | |
| 101 | etcd = { |
| 102 | # https://github.com/kubernetes/kubernetes/issues/72102 |
| 103 | servers = (map (n: "https://${n.fqdn}:2379") ( [ machine ] )); |
| 104 | caFile = pki.etcd.kube.ca; |
| 105 | keyFile = pki.etcd.kube.key; |
| 106 | certFile = pki.etcd.kube.cert; |
| 107 | }; |
| 108 | |
| 109 | tlsCertFile = pki.kube.apiserver.cert; |
| 110 | tlsKeyFile = pki.kube.apiserver.key; |
| 111 | |
| 112 | clientCaFile = pki.kube.apiserver.ca; |
| 113 | |
| 114 | kubeletHttps = true; |
| 115 | kubeletClientCaFile = pki.kube.apiserver.ca; |
| 116 | kubeletClientCertFile = pki.kube.apiserver.cert; |
| 117 | kubeletClientKeyFile = pki.kube.apiserver.key; |
| 118 | |
| 119 | serviceAccountKeyFile = pki.kube.serviceaccounts.key; |
| 120 | |
| 121 | allowPrivileged = true; |
| 122 | serviceClusterIpRange = "10.10.12.0/24"; |
| 123 | runtimeConfig = "api/all,authentication.k8s.io/v1beta1"; |
| 124 | authorizationMode = ["Node" "RBAC"]; |
| 125 | enableAdmissionPlugins = ["NamespaceLifecycle" "NodeRestriction" "LimitRanger" "ServiceAccount" "DefaultStorageClass" "ResourceQuota" "PodSecurityPolicy"]; |
| 126 | extraOpts = '' |
| 127 | --apiserver-count=5 \ |
| 128 | --proxy-client-cert-file=${pki.kubeFront.apiserver.cert} \ |
| 129 | --proxy-client-key-file=${pki.kubeFront.apiserver.key} \ |
| 130 | --requestheader-allowed-names= \ |
| 131 | --requestheader-client-ca-file=${pki.kubeFront.apiserver.ca} \ |
| 132 | --requestheader-extra-headers-prefix=X-Remote-Extra- \ |
| 133 | --requestheader-group-headers=X-Remote-Group \ |
| 134 | --requestheader-username-headers=X-Remote-User \ |
| 135 | -v=5 |
| 136 | ''; |
| 137 | }; |
| 138 | |
Serge Bazanski | 1257389 | 2020-10-10 14:55:08 +0200 | [diff] [blame] | 139 | controllerManager = let |
| 140 | top = config.services.kubernetes; |
| 141 | kubeconfig = top.lib.mkKubeConfig "controller-manager" pki.kube.controllermanager.config; |
| 142 | in { |
Sergiusz Bazanski | c78cc13 | 2020-02-02 22:31:53 +0100 | [diff] [blame] | 143 | enable = true; |
| 144 | bindAddress = "0.0.0.0"; |
| 145 | insecurePort = ports.k8sControllerManagerPlain; |
| 146 | leaderElect = true; |
| 147 | serviceAccountKeyFile = pki.kube.serviceaccounts.key; |
| 148 | rootCaFile = pki.kube.ca; |
| 149 | extraOpts = '' |
| 150 | --service-cluster-ip-range=10.10.12.0/24 \ |
| 151 | --use-service-account-credentials=true \ |
| 152 | --secure-port=${toString ports.k8sControllerManagerSecure}\ |
Serge Bazanski | 1257389 | 2020-10-10 14:55:08 +0200 | [diff] [blame] | 153 | --authentication-kubeconfig=${kubeconfig}\ |
| 154 | --authorization-kubeconfig=${kubeconfig}\ |
Sergiusz Bazanski | c78cc13 | 2020-02-02 22:31:53 +0100 | [diff] [blame] | 155 | ''; |
| 156 | kubeconfig = pki.kube.controllermanager.config; |
| 157 | }; |
| 158 | |
Serge Bazanski | 1257389 | 2020-10-10 14:55:08 +0200 | [diff] [blame] | 159 | scheduler = let |
| 160 | top = config.services.kubernetes; |
| 161 | kubeconfig = top.lib.mkKubeConfig "scheduler" pki.kube.controllermanager.config; |
| 162 | in { |
Sergiusz Bazanski | c78cc13 | 2020-02-02 22:31:53 +0100 | [diff] [blame] | 163 | enable = true; |
| 164 | address = "0.0.0.0"; |
Serge Bazanski | 1257389 | 2020-10-10 14:55:08 +0200 | [diff] [blame] | 165 | port = ports.k8sSchedulerPlain; |
Sergiusz Bazanski | c78cc13 | 2020-02-02 22:31:53 +0100 | [diff] [blame] | 166 | leaderElect = true; |
| 167 | kubeconfig = pki.kube.scheduler.config; |
Serge Bazanski | 1257389 | 2020-10-10 14:55:08 +0200 | [diff] [blame] | 168 | extraOpts = '' |
| 169 | --secure-port=${toString ports.k8sSchedulerSecure}\ |
| 170 | --authentication-kubeconfig=${kubeconfig}\ |
| 171 | --authorization-kubeconfig=${kubeconfig}\ |
| 172 | ''; |
Sergiusz Bazanski | c78cc13 | 2020-02-02 22:31:53 +0100 | [diff] [blame] | 173 | }; |
| 174 | |
| 175 | proxy = { |
| 176 | enable = true; |
| 177 | kubeconfig = pki.kube.proxy.config; |
| 178 | extraOpts = '' |
| 179 | --hostname-override=${fqdn}\ |
| 180 | --proxy-mode=iptables |
| 181 | ''; |
| 182 | }; |
| 183 | |
| 184 | kubelet = { |
| 185 | enable = true; |
| 186 | unschedulable = false; |
| 187 | hostname = fqdn; |
| 188 | tlsCertFile = pki.kube.kubelet.cert; |
| 189 | tlsKeyFile = pki.kube.kubelet.key; |
| 190 | clientCaFile = pki.kube.kubelet.ca; |
| 191 | nodeIp = machine.ipAddr; |
| 192 | networkPlugin = "cni"; |
| 193 | clusterDns = "10.10.12.254"; |
| 194 | kubeconfig = pki.kube.kubelet.config; |
| 195 | extraOpts = '' |
| 196 | --read-only-port=0 |
| 197 | ''; |
| 198 | package = k8spkgsKubelet.kubernetes; |
| 199 | }; |
| 200 | |
| 201 | }; |
| 202 | |
| 203 | # https://github.com/NixOS/nixpkgs/issues/60687 |
| 204 | systemd.services.kube-control-plane-online = { |
| 205 | preStart = pkgs.lib.mkForce ""; |
| 206 | }; |
| 207 | # this seems to depend on flannel |
| 208 | # TODO(q3k): file issue |
| 209 | systemd.services.kubelet-online = { |
| 210 | script = pkgs.lib.mkForce "sleep 1"; |
| 211 | }; |
| 212 | } |