Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 1 | { config, pkgs, lib, ... }: |
| 2 | |
| 3 | with ( import ./toplevel.nix ); |
| 4 | let |
| 5 | fqdn = config.networking.hostName + domain; |
| 6 | node = (builtins.head (builtins.filter (n: n.fqdn == fqdn) nodes)); |
| 7 | otherNodes = (builtins.filter (n: n.fqdn != fqdn) nodes); |
| 8 | |
Sergiusz Bazanski | b7e4bd4 | 2019-05-14 01:44:39 +0200 | [diff] [blame] | 9 | # Pin for k8s packages. This is so that upagrading the system will not upgrade the k8s control or data planes. |
| 10 | k8spkgs = import (fetchGit { |
| 11 | name = "nixos-unstable-2019-04-12"; |
| 12 | url = https://github.com/nixos/nixpkgs/; |
| 13 | rev = "1fc591f9a5bd1b016b5d66dfab29560073955a14"; |
| 14 | }) {}; |
| 15 | |
| 16 | |
Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 17 | in rec { |
| 18 | imports = |
| 19 | [ # Include the results of the hardware scan. |
| 20 | ./hardware-configuration.nix |
| 21 | ]; |
| 22 | |
| 23 | # Use the GRUB 2 boot loader. |
| 24 | boot.loader.grub.enable = true; |
| 25 | boot.loader.grub.version = 2; |
| 26 | boot.loader.grub.device = node.diskBoot; |
| 27 | |
Sergiusz Bazanski | b7e4bd4 | 2019-05-14 01:44:39 +0200 | [diff] [blame] | 28 | boot.kernelPackages = pkgs.linuxPackages_5_1; |
Sergiusz Bazanski | 4232c8b | 2019-04-28 17:12:54 +0200 | [diff] [blame] | 29 | boot.kernelParams = [ "boot.shell_on_fail" ]; |
| 30 | |
Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 31 | time.timeZone = "Europe/Warsaw"; |
| 32 | |
| 33 | # List packages installed in system profile. To search, run: |
| 34 | # $ nix search wget |
| 35 | environment.systemPackages = with pkgs; [ |
| 36 | wget vim htop tcpdump |
| 37 | rxvt_unicode.terminfo |
| 38 | ]; |
| 39 | |
| 40 | # Some programs need SUID wrappers, can be configured further or are |
| 41 | # started in user sessions. |
| 42 | programs.mtr.enable = true; |
| 43 | |
| 44 | # List services that you want to enable: |
| 45 | virtualisation.docker.enable = true; |
| 46 | virtualisation.docker.extraOptions = "--iptables=false --ip-masq=false --ip-forward=true"; |
| 47 | |
| 48 | # Docker 1.13 sets iptables FORWARD to DROP. Unfuck this. |
| 49 | systemd.services."docker-iptables-unfuck" = { |
| 50 | enable = true; |
| 51 | wantedBy = [ "kubernetes.target" ]; |
| 52 | description = "Docker iptable Unfuck"; |
| 53 | after = [ "docker.service" ]; |
| 54 | requires = [ "docker.service" ]; |
| 55 | path = [ pkgs.iptables ]; |
| 56 | script = '' |
| 57 | iptables -P FORWARD ACCEPT |
| 58 | ''; |
| 59 | serviceConfig.Type = "oneshot"; |
| 60 | }; |
Sergiusz Bazanski | b7e4bd4 | 2019-05-14 01:44:39 +0200 | [diff] [blame] | 61 | # Otherwise fetchGit nixpkgs pin fails. |
| 62 | systemd.services.nixos-upgrade.path = [ pkgs.git ]; |
Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 63 | |
| 64 | # Enable the OpenSSH daemon. |
| 65 | services.openssh.enable = true; |
| 66 | users.users.root.openssh.authorizedKeys.keys = [ |
| 67 | "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDD4VJXAXEHEXZk2dxNwehneuJcEGkfXG/U7z4fO79vDVIENdedtXQUyLyhZJc5RTEfHhQj66FwIqzl7mzBHd9x9PuDp6QAYXrkVNMj48s6JXqZqBvF6H/weRqFMf4a2TZv+hG8D0kpvmLheCwWAVRls7Jofnp/My+yDd57GMdsbG/yFEf6WPMiOnA7hxdSJSVihCsCSw2p8PD4GhBe8CVt7xIuinhutjm9zYBjV78NT8acjDUfJh0B1ODTjs7nuW1CC4jybSe2j/OU3Yczj4AxRxBNWuFxUq+jBo9BfpbKLh+Tt7re+zBkaicM77KM/oV6943JJxgHNBBOsv9scZE7 q3k@amnesia" |
| 68 | "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDQb3YQoiYFZLKwvHYKbu1bMqzNeDCAszQhAe1+QI5SLDOotclyY/vFmOReZOsmyMFl71G2d7d+FbYNusUnNNjTxRYQ021tVc+RkMdLJaORRURmQfEFEKbai6QSFTwErXzuoIzyEPK0lbsQuGgqT9WaVnRzHJ2Q/4+qQbxAS34PuR5NqEkmn4G6LMo3OyJ5mwPkCj9lsqz4BcxRaMWFO3mNcwGDfSW+sqgc3E8N6LKrTpZq3ke7xacpQmcG5DU9VO+2QVPdltl9jWbs3gXjmF92YRNOuKPVfAOZBBsp8JOznfx8s9wDgs7RwPmDpjIAJEyoABqW5hlXfqRbTnfnMvuR informatic@InformaticPC" |
| 69 | "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDGkMgEVwQM8yeuFUYL2TwlJIq9yUNBmHnwce46zeL2PK2CkMz7sxT/om7sp/K5XDiqeD05Nioe+Dr3drP6B8uI33S5NgxPIfaqQsRS+CBEgk6cqFlcdlKETU/DT+/WsdoO173n7mgGeafPInEuQuGDUID0Fl099kIxtqfAhdeZFMM6/szAZEZsElLJ8K6dp1Ni/jmnXCZhjivZH3AZUlnqrmtDG7FY1bgcOfDXAal45LItughGPtrdiigXe9DK2fW3+9DBZZduh5DMJTNlphAZ+nfSrbyHVKUg6WsgMSprur4KdU47q1QwzqqvEj75JcdP1jOWoZi4F6VJDte9Wb9lhD1jGgjxY9O6Gs4CH35bx15W7CN9hgNa0C8NbPJe/fZYIeMZmJ1m7O2xmnYwP8j+t7RNJWu7Pa3Em4mOEXvhBF07Zfq+Ye/4SluoRgADy5eII2x5fFo5EBhInxK0/X8wF6XZvysalVifoCh7T4Edejoi91oAxFgYAxbboXGlod0eEHIi2hla8SM9+IBHOChmgawKBYp2kzAJyAmHNBF+Pah9G4arVCj/axp/SJZDZbJQoI7UT/fJzEtvlb5RWrHXRq+y6IvjpUq4pzpDWW04+9UMqEEXRmhWOakHfEVM9rN8h3aJBflLUBBnh0Z/hVsKNh8bCRHaKtah8TrD9i+wMw== patryk.jakuszew@gmail.com" |
| 70 | "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC33naG1ptCvUcRWX9cj9wXM1nW1lyQC4SvMJzWlr9aMD96O8hQ2JMkuIUgUJvorAY02QRplQ2BuoVoVkdkzwjMyi1bL3OdgcKo7Z1yByClGTTocqNJYY0lcUb6EJH8+6e6F9ydrQlSxNzL1uCaA7phZr+yPcmAmWbSfioXn98yXNkE0emHxzJv/nypJY56sDCMC2IXDRd8L2goDtPwgPEW7bWfAQdIFMJ75xOidZOTxJ8eqyXLw/kxY5UlyX66jdoYz1sE5XUHuoQl1AOG9UdlMo0aMhUvP4pX5l7r7EnA9OttKMFB3oWqkVK/R6ynZ52YNOU5BZ9V+Ppaj34W0xNu+p0mbHcCtXYCTrf/OU0hcZDbDaNTjs6Vtcm2wYw9iAKX7Tex+eOMwUwlrlcyPNRV5BTot7lGNYfauHCSIuWJKN4NhCLR/NtVNh4/94eKkPTwJsY6XqDcS7q49wPAs4DAH7BJgsbHPOqygVHrY0YYEfz3Pj0HTxJHQMCP/hQX4fXEGt0BjgoVJbXPAQtPyeg0JuxiUg+b4CgVVfQ6R060MlM1BZzhmh+FY5MJH6nJppS0aHYCvSg8Z68NUlCPKy0jpcyfuAIWQWwSGG1O010WShQG2ELsvNdg5/4HVdCGNl5mmoom6JOd72FOZyQlHDFfeQUQRn9HOeCq/c51rK99SQ== bartek@IHM" |
Bartosz Stebel | caa770a | 2019-05-20 04:26:34 +0200 | [diff] [blame] | 71 | "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICTR292kx/2CNuWYIsZ6gykQ036aBGrmheIuZa6S1D2x implr@thonk" |
Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 72 | ]; |
| 73 | |
| 74 | networking.firewall.enable = false; |
| 75 | |
| 76 | # Point k8s apiserver address at ourselves, as every node runs an apiserver with this cert name. |
| 77 | networking.extraHosts = '' |
| 78 | 127.0.0.1 ${k8sapi} |
| 79 | ''; |
| 80 | |
| 81 | security.acme.certs = { |
| 82 | host = { |
| 83 | email = acmeEmail; |
| 84 | domain = fqdn; |
| 85 | webroot = services.nginx.virtualHosts.host.root; |
| 86 | }; |
| 87 | }; |
| 88 | |
| 89 | services.nginx = { |
| 90 | enable = true; |
| 91 | virtualHosts.host = { |
| 92 | serverName = fqdn; |
| 93 | root = "/var/www/${fqdn}"; |
| 94 | }; |
| 95 | }; |
| 96 | |
| 97 | services.etcd = { |
| 98 | enable = true; |
| 99 | name = fqdn; |
| 100 | listenClientUrls = ["https://0.0.0.0:2379"]; |
| 101 | advertiseClientUrls = ["https://${fqdn}:2379"]; |
| 102 | listenPeerUrls = ["https://0.0.0.0:2380"]; |
| 103 | initialAdvertisePeerUrls = ["https://${fqdn}:2380"]; |
| 104 | initialCluster = (map (n: "${n.fqdn}=https://${n.fqdn}:2380") nodes); |
| 105 | |
| 106 | clientCertAuth = true; |
| 107 | trustedCaFile = pki.etcd.server.ca; |
| 108 | certFile = pki.etcd.server.cert; |
| 109 | keyFile = pki.etcd.server.key; |
| 110 | |
| 111 | peerClientCertAuth = true; |
| 112 | peerTrustedCaFile = pki.etcdPeer.ca; |
| 113 | peerCertFile = pki.etcdPeer.cert; |
| 114 | peerKeyFile = pki.etcdPeer.key; |
| 115 | |
| 116 | extraConf = { |
| 117 | PEER_CLIENT_CERT_AUTH = "true"; |
| 118 | }; |
| 119 | }; |
| 120 | |
| 121 | services.kubernetes = { |
Sergiusz Bazanski | b7e4bd4 | 2019-05-14 01:44:39 +0200 | [diff] [blame] | 122 | # Pin to specific k8s package. |
| 123 | package = k8spkgs.kubernetes; |
Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 124 | roles = []; # We do not use any nixpkgs predefined roles for k8s. Instead, |
| 125 | # we enable k8s components manually. |
| 126 | |
| 127 | caFile = pki.kube.apiserver.ca; |
| 128 | clusterCidr = "10.10.16.0/20"; |
Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 129 | |
| 130 | path = [ pkgs.e2fsprogs ]; # kubelet wants to mkfs.ext4 when mounting pvcs |
| 131 | |
| 132 | addons.dns.enable = false; |
| 133 | |
Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 134 | apiserver = rec { |
| 135 | enable = true; |
Sergiusz Bazanski | 4232c8b | 2019-04-28 17:12:54 +0200 | [diff] [blame] | 136 | insecurePort = ports.k8sAPIServerPlain; |
Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 137 | securePort = ports.k8sAPIServerSecure; |
| 138 | advertiseAddress = "${node.ipAddr}"; |
| 139 | |
Sergiusz Bazanski | 4232c8b | 2019-04-28 17:12:54 +0200 | [diff] [blame] | 140 | etcd = { |
Sergiusz Bazanski | d07861b | 2019-08-08 17:48:25 +0200 | [diff] [blame] | 141 | # https://github.com/kubernetes/kubernetes/issues/72102 |
| 142 | servers = (map (n: "https://${n.fqdn}:2379") ( [ node ] )); |
Sergiusz Bazanski | 4232c8b | 2019-04-28 17:12:54 +0200 | [diff] [blame] | 143 | caFile = pki.etcd.kube.ca; |
| 144 | keyFile = pki.etcd.kube.key; |
| 145 | certFile = pki.etcd.kube.cert; |
| 146 | }; |
| 147 | |
Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 148 | tlsCertFile = pki.kube.apiserver.cert; |
| 149 | tlsKeyFile = pki.kube.apiserver.key; |
| 150 | |
| 151 | clientCaFile = pki.kube.apiserver.ca; |
| 152 | |
| 153 | kubeletHttps = true; |
| 154 | kubeletClientCaFile = pki.kube.apiserver.ca; |
| 155 | kubeletClientCertFile = pki.kube.apiserver.cert; |
| 156 | kubeletClientKeyFile = pki.kube.apiserver.key; |
| 157 | |
| 158 | serviceAccountKeyFile = pki.kube.serviceaccounts.key; |
| 159 | |
Sergiusz Bazanski | 4232c8b | 2019-04-28 17:12:54 +0200 | [diff] [blame] | 160 | allowPrivileged = true; |
Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 161 | serviceClusterIpRange = "10.10.12.0/24"; |
| 162 | runtimeConfig = "api/all,authentication.k8s.io/v1beta1"; |
| 163 | authorizationMode = ["Node" "RBAC"]; |
Sergiusz Bazanski | b13b7ff | 2019-08-29 20:12:24 +0200 | [diff] [blame] | 164 | enableAdmissionPlugins = ["Initializers" "NamespaceLifecycle" "NodeRestriction" "LimitRanger" "ServiceAccount" "DefaultStorageClass" "ResourceQuota" "PodSecurityPolicy"]; |
Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 165 | extraOpts = '' |
| 166 | --apiserver-count=3 \ |
| 167 | --proxy-client-cert-file=${pki.kubeFront.apiserver.cert} \ |
| 168 | --proxy-client-key-file=${pki.kubeFront.apiserver.key} \ |
| 169 | --requestheader-allowed-names= \ |
| 170 | --requestheader-client-ca-file=${pki.kubeFront.apiserver.ca} \ |
| 171 | --requestheader-extra-headers-prefix=X-Remote-Extra- \ |
| 172 | --requestheader-group-headers=X-Remote-Group \ |
| 173 | --requestheader-username-headers=X-Remote-User \ |
| 174 | -v=5 |
| 175 | ''; |
| 176 | }; |
| 177 | |
| 178 | controllerManager = { |
| 179 | enable = true; |
Sergiusz Bazanski | 4232c8b | 2019-04-28 17:12:54 +0200 | [diff] [blame] | 180 | bindAddress = "0.0.0.0"; |
| 181 | insecurePort = ports.k8sControllerManagerPlain; |
Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 182 | leaderElect = true; |
| 183 | serviceAccountKeyFile = pki.kube.serviceaccounts.key; |
| 184 | rootCaFile = pki.kube.ca; |
| 185 | extraOpts = '' |
| 186 | --service-cluster-ip-range=10.10.12.0/24 \ |
| 187 | --use-service-account-credentials=true \ |
| 188 | --secure-port=${toString ports.k8sControllerManagerSecure}\ |
| 189 | ''; |
| 190 | kubeconfig = pki.kube.controllermanager.config; |
| 191 | }; |
| 192 | |
| 193 | scheduler = { |
| 194 | enable = true; |
| 195 | address = "0.0.0.0"; |
| 196 | port = 0; |
| 197 | leaderElect = true; |
| 198 | kubeconfig = pki.kube.scheduler.config; |
| 199 | }; |
| 200 | |
| 201 | proxy = { |
| 202 | enable = true; |
| 203 | kubeconfig = pki.kube.proxy.config; |
| 204 | extraOpts = '' |
| 205 | --hostname-override=${fqdn}\ |
| 206 | --proxy-mode=iptables |
| 207 | ''; |
| 208 | }; |
| 209 | |
| 210 | kubelet = { |
| 211 | enable = true; |
| 212 | unschedulable = false; |
Sergiusz Bazanski | 4232c8b | 2019-04-28 17:12:54 +0200 | [diff] [blame] | 213 | allowPrivileged = true; |
Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 214 | hostname = fqdn; |
| 215 | tlsCertFile = pki.kube.kubelet.cert; |
| 216 | tlsKeyFile = pki.kube.kubelet.key; |
| 217 | clientCaFile = pki.kube.kubelet.ca; |
| 218 | nodeIp = node.ipAddr; |
| 219 | networkPlugin = "cni"; |
| 220 | clusterDns = "10.10.12.254"; |
| 221 | kubeconfig = pki.kube.kubelet.config; |
| 222 | extraOpts = '' |
| 223 | --cni-conf-dir=/opt/cni/conf \ |
| 224 | --cni-bin-dir=/opt/cni/bin |
| 225 | ''; |
| 226 | }; |
Sergiusz Bazanski | b7e4bd4 | 2019-05-14 01:44:39 +0200 | [diff] [blame] | 227 | |
| 228 | }; |
| 229 | |
| 230 | # https://github.com/NixOS/nixpkgs/issues/60687 |
| 231 | systemd.services.kube-control-plane-online = { |
| 232 | preStart = pkgs.lib.mkForce ""; |
| 233 | }; |
| 234 | # this seems to depend on flannel |
| 235 | # TODO(q3k): file issue |
| 236 | systemd.services.kubelet-online = { |
| 237 | script = pkgs.lib.mkForce "sleep 1"; |
Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 238 | }; |
Sergiusz Bazanski | f970a7e | 2019-06-20 12:51:39 +0200 | [diff] [blame] | 239 | # This by default removes all CNI plugins and replaces them with nix-defines ones |
| 240 | # Since we bring our own CNI plugins via containers with host mounts, this causes |
| 241 | # them to be removed on kubelet restart. |
Sergiusz Bazanski | 0de4368 | 2019-06-20 12:55:02 +0200 | [diff] [blame] | 242 | # TODO(https://github.com/NixOS/nixpkgs/issues/53601): fix when resolved |
Sergiusz Bazanski | f970a7e | 2019-06-20 12:51:39 +0200 | [diff] [blame] | 243 | systemd.services.kubelet = { |
| 244 | preStart = pkgs.lib.mkForce "sleep 1"; |
| 245 | }; |
Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 246 | } |