Serge Bazanski | ef3aab6 | 2022-11-18 14:39:45 +0000 | [diff] [blame] | 1 | # Vendored from nixpkgs git 44ad80ab1036c5cc83ada4bfa451dac9939f2a10 |
| 2 | # Copyright (c) 2003-2023 Eelco Dolstra and the Nixpkgs/NixOS contributors |
| 3 | # SPDX-License-Identifier: MIT |
| 4 | |
| 5 | { config, lib, pkgs, ... }: |
| 6 | |
| 7 | with lib; |
| 8 | |
| 9 | let |
| 10 | cfg = config.services.kubernetes; |
| 11 | |
| 12 | mkKubeConfig = name: conf: pkgs.writeText "${name}-kubeconfig" (builtins.toJSON { |
| 13 | apiVersion = "v1"; |
| 14 | kind = "Config"; |
| 15 | clusters = [{ |
| 16 | name = "local"; |
| 17 | cluster.certificate-authority = conf.caFile or cfg.caFile; |
| 18 | cluster.server = conf.server; |
| 19 | }]; |
| 20 | users = [{ |
| 21 | inherit name; |
| 22 | user = { |
| 23 | client-certificate = conf.certFile; |
| 24 | client-key = conf.keyFile; |
| 25 | }; |
| 26 | }]; |
| 27 | contexts = [{ |
| 28 | context = { |
| 29 | cluster = "local"; |
| 30 | user = name; |
| 31 | }; |
| 32 | current-context = "local"; |
| 33 | }]; |
| 34 | }); |
| 35 | |
| 36 | caCert = secret "ca"; |
| 37 | |
| 38 | etcdEndpoints = ["https://${cfg.masterAddress}:2379"]; |
| 39 | |
| 40 | mkCert = { name, CN, hosts ? [], fields ? {}, action ? "", |
| 41 | privateKeyOwner ? "kubernetes" }: rec { |
| 42 | inherit name caCert CN hosts fields action; |
| 43 | cert = secret name; |
| 44 | key = secret "${name}-key"; |
| 45 | privateKeyOptions = { |
| 46 | owner = privateKeyOwner; |
| 47 | group = "nogroup"; |
| 48 | mode = "0600"; |
| 49 | path = key; |
| 50 | }; |
| 51 | }; |
| 52 | |
| 53 | secret = name: "${cfg.secretsPath}/${name}.pem"; |
| 54 | |
| 55 | mkKubeConfigOptions = prefix: { |
| 56 | server = mkOption { |
| 57 | description = "${prefix} kube-apiserver server address."; |
| 58 | type = types.str; |
| 59 | }; |
| 60 | |
| 61 | caFile = mkOption { |
| 62 | description = "${prefix} certificate authority file used to connect to kube-apiserver."; |
| 63 | type = types.nullOr types.path; |
| 64 | default = cfg.caFile; |
| 65 | }; |
| 66 | |
| 67 | certFile = mkOption { |
| 68 | description = "${prefix} client certificate file used to connect to kube-apiserver."; |
| 69 | type = types.nullOr types.path; |
| 70 | default = null; |
| 71 | }; |
| 72 | |
| 73 | keyFile = mkOption { |
| 74 | description = "${prefix} client key file used to connect to kube-apiserver."; |
| 75 | type = types.nullOr types.path; |
| 76 | default = null; |
| 77 | }; |
| 78 | }; |
| 79 | in { |
| 80 | |
| 81 | imports = [ |
| 82 | (mkRemovedOptionModule [ "services" "kubernetes" "verbose" ] "") |
| 83 | ]; |
| 84 | |
| 85 | ###### interface |
| 86 | |
| 87 | options.services.kubernetes = { |
| 88 | roles = mkOption { |
| 89 | description = '' |
| 90 | Kubernetes role that this machine should take. |
| 91 | |
| 92 | Master role will enable etcd, apiserver, scheduler, controller manager |
| 93 | addon manager, flannel and proxy services. |
| 94 | Node role will enable flannel, docker, kubelet and proxy services. |
| 95 | ''; |
| 96 | default = []; |
| 97 | type = types.listOf (types.enum ["master" "node"]); |
| 98 | }; |
| 99 | |
| 100 | package = mkOption { |
| 101 | description = "Kubernetes package to use."; |
| 102 | type = types.package; |
| 103 | default = pkgs.kubernetes; |
| 104 | defaultText = "pkgs.kubernetes"; |
| 105 | }; |
| 106 | |
| 107 | kubeconfig = mkKubeConfigOptions "Default kubeconfig"; |
| 108 | |
| 109 | apiserverAddress = mkOption { |
| 110 | description = '' |
| 111 | Clusterwide accessible address for the kubernetes apiserver, |
| 112 | including protocol and optional port. |
| 113 | ''; |
| 114 | example = "https://kubernetes-apiserver.example.com:6443"; |
| 115 | type = types.str; |
| 116 | }; |
| 117 | |
| 118 | caFile = mkOption { |
| 119 | description = "Default kubernetes certificate authority"; |
| 120 | type = types.nullOr types.path; |
| 121 | default = null; |
| 122 | }; |
| 123 | |
| 124 | dataDir = mkOption { |
| 125 | description = "Kubernetes root directory for managing kubelet files."; |
| 126 | default = "/var/lib/kubernetes"; |
| 127 | type = types.path; |
| 128 | }; |
| 129 | |
| 130 | easyCerts = mkOption { |
| 131 | description = "Automatically setup x509 certificates and keys for the entire cluster."; |
| 132 | default = false; |
| 133 | type = types.bool; |
| 134 | }; |
| 135 | |
| 136 | featureGates = mkOption { |
| 137 | description = "List set of feature gates."; |
| 138 | default = []; |
| 139 | type = types.listOf types.str; |
| 140 | }; |
| 141 | |
| 142 | masterAddress = mkOption { |
| 143 | description = "Clusterwide available network address or hostname for the kubernetes master server."; |
| 144 | example = "master.example.com"; |
| 145 | type = types.str; |
| 146 | }; |
| 147 | |
| 148 | path = mkOption { |
| 149 | description = "Packages added to the services' PATH environment variable. Both the bin and sbin subdirectories of each package are added."; |
| 150 | type = types.listOf types.package; |
| 151 | default = []; |
| 152 | }; |
| 153 | |
| 154 | clusterCidr = mkOption { |
| 155 | description = "Kubernetes controller manager and proxy CIDR Range for Pods in cluster."; |
| 156 | default = "10.1.0.0/16"; |
| 157 | type = types.nullOr types.str; |
| 158 | }; |
| 159 | |
| 160 | lib = mkOption { |
| 161 | description = "Common functions for the kubernetes modules."; |
| 162 | default = { |
| 163 | inherit mkCert; |
| 164 | inherit mkKubeConfig; |
| 165 | inherit mkKubeConfigOptions; |
| 166 | }; |
| 167 | type = types.attrs; |
| 168 | }; |
| 169 | |
| 170 | secretsPath = mkOption { |
| 171 | description = "Default location for kubernetes secrets. Not a store location."; |
| 172 | type = types.path; |
| 173 | default = cfg.dataDir + "/secrets"; |
| 174 | }; |
| 175 | }; |
| 176 | |
| 177 | ###### implementation |
| 178 | |
| 179 | config = mkMerge [ |
| 180 | |
| 181 | (mkIf cfg.easyCerts { |
| 182 | services.kubernetes.pki.enable = mkDefault true; |
| 183 | services.kubernetes.caFile = caCert; |
| 184 | }) |
| 185 | |
| 186 | (mkIf (elem "master" cfg.roles) { |
| 187 | services.kubernetes.apiserver.enable = mkDefault true; |
| 188 | services.kubernetes.scheduler.enable = mkDefault true; |
| 189 | services.kubernetes.controllerManager.enable = mkDefault true; |
| 190 | services.kubernetes.addonManager.enable = mkDefault true; |
| 191 | services.kubernetes.proxy.enable = mkDefault true; |
| 192 | services.etcd.enable = true; # Cannot mkDefault because of flannel default options |
| 193 | services.kubernetes.kubelet = { |
| 194 | enable = mkDefault true; |
| 195 | taints = mkIf (!(elem "node" cfg.roles)) { |
| 196 | master = { |
| 197 | key = "node-role.kubernetes.io/master"; |
| 198 | value = "true"; |
| 199 | effect = "NoSchedule"; |
| 200 | }; |
| 201 | }; |
| 202 | }; |
| 203 | }) |
| 204 | |
| 205 | |
| 206 | (mkIf (all (el: el == "master") cfg.roles) { |
| 207 | # if this node is only a master make it unschedulable by default |
| 208 | services.kubernetes.kubelet.unschedulable = mkDefault true; |
| 209 | }) |
| 210 | |
| 211 | (mkIf (elem "node" cfg.roles) { |
| 212 | services.kubernetes.kubelet.enable = mkDefault true; |
| 213 | services.kubernetes.proxy.enable = mkDefault true; |
| 214 | }) |
| 215 | |
| 216 | # Using "services.kubernetes.roles" will automatically enable easyCerts and flannel |
| 217 | (mkIf (cfg.roles != []) { |
| 218 | services.kubernetes.flannel.enable = mkDefault true; |
| 219 | services.flannel.etcd.endpoints = mkDefault etcdEndpoints; |
| 220 | services.kubernetes.easyCerts = mkDefault true; |
| 221 | }) |
| 222 | |
| 223 | (mkIf cfg.apiserver.enable { |
| 224 | services.kubernetes.pki.etcClusterAdminKubeconfig = mkDefault "kubernetes/cluster-admin.kubeconfig"; |
| 225 | services.kubernetes.apiserver.etcd.servers = mkDefault etcdEndpoints; |
| 226 | }) |
| 227 | |
| 228 | (mkIf cfg.kubelet.enable { |
| 229 | virtualisation.docker = { |
| 230 | enable = mkDefault true; |
| 231 | |
| 232 | # kubernetes needs access to logs |
| 233 | logDriver = mkDefault "json-file"; |
| 234 | |
| 235 | # iptables must be disabled for kubernetes |
| 236 | extraOptions = "--iptables=false --ip-masq=false"; |
| 237 | }; |
| 238 | }) |
| 239 | |
| 240 | (mkIf (cfg.apiserver.enable || cfg.controllerManager.enable) { |
| 241 | services.kubernetes.pki.certs = { |
| 242 | serviceAccount = mkCert { |
| 243 | name = "service-account"; |
| 244 | CN = "system:service-account-signer"; |
| 245 | action = '' |
| 246 | systemctl reload \ |
| 247 | kube-apiserver.service \ |
| 248 | kube-controller-manager.service |
| 249 | ''; |
| 250 | }; |
| 251 | }; |
| 252 | }) |
| 253 | |
| 254 | (mkIf ( |
| 255 | cfg.apiserver.enable || |
| 256 | cfg.scheduler.enable || |
| 257 | cfg.controllerManager.enable || |
| 258 | cfg.kubelet.enable || |
| 259 | cfg.proxy.enable || |
| 260 | cfg.addonManager.enable |
| 261 | ) { |
| 262 | systemd.targets.kubernetes = { |
| 263 | description = "Kubernetes"; |
| 264 | wantedBy = [ "multi-user.target" ]; |
| 265 | }; |
| 266 | |
| 267 | systemd.tmpfiles.rules = [ |
| 268 | "d /opt/cni/bin 0755 root root -" |
| 269 | "d /run/kubernetes 0755 kubernetes kubernetes -" |
| 270 | "d /var/lib/kubernetes 0755 kubernetes kubernetes -" |
| 271 | ]; |
| 272 | |
| 273 | users.users.kubernetes = { |
| 274 | uid = config.ids.uids.kubernetes; |
| 275 | description = "Kubernetes user"; |
| 276 | extraGroups = [ "docker" ]; |
| 277 | group = "kubernetes"; |
| 278 | home = cfg.dataDir; |
| 279 | createHome = true; |
| 280 | }; |
| 281 | users.groups.kubernetes.gid = config.ids.gids.kubernetes; |
| 282 | |
| 283 | # dns addon is enabled by default |
| 284 | services.kubernetes.addons.dns.enable = mkDefault true; |
| 285 | |
| 286 | services.kubernetes.apiserverAddress = mkDefault ("https://${if cfg.apiserver.advertiseAddress != null |
| 287 | then cfg.apiserver.advertiseAddress |
| 288 | else "${cfg.masterAddress}:${toString cfg.apiserver.securePort}"}"); |
| 289 | }) |
| 290 | ]; |
| 291 | } |