Serge Bazanski | ef3aab6 | 2022-11-18 14:39:45 +0000 | [diff] [blame] | 1 | # Vendored from nixpkgs git 44ad80ab1036c5cc83ada4bfa451dac9939f2a10 |
| 2 | # Copyright (c) 2003-2023 Eelco Dolstra and the Nixpkgs/NixOS contributors |
| 3 | # SPDX-License-Identifier: MIT |
| 4 | |
| 5 | { config, lib, pkgs, ... }: |
| 6 | |
| 7 | with lib; |
| 8 | |
| 9 | let |
| 10 | top = config.services.kubernetes; |
| 11 | cfg = top.pki; |
| 12 | |
| 13 | csrCA = pkgs.writeText "kube-pki-cacert-csr.json" (builtins.toJSON { |
| 14 | key = { |
| 15 | algo = "rsa"; |
| 16 | size = 2048; |
| 17 | }; |
| 18 | names = singleton cfg.caSpec; |
| 19 | }); |
| 20 | |
| 21 | csrCfssl = pkgs.writeText "kube-pki-cfssl-csr.json" (builtins.toJSON { |
| 22 | key = { |
| 23 | algo = "rsa"; |
| 24 | size = 2048; |
| 25 | }; |
| 26 | CN = top.masterAddress; |
| 27 | hosts = [top.masterAddress] ++ cfg.cfsslAPIExtraSANs; |
| 28 | }); |
| 29 | |
| 30 | cfsslAPITokenBaseName = "apitoken.secret"; |
| 31 | cfsslAPITokenPath = "${config.services.cfssl.dataDir}/${cfsslAPITokenBaseName}"; |
| 32 | certmgrAPITokenPath = "${top.secretsPath}/${cfsslAPITokenBaseName}"; |
| 33 | cfsslAPITokenLength = 32; |
| 34 | |
| 35 | clusterAdminKubeconfig = with cfg.certs.clusterAdmin; |
| 36 | top.lib.mkKubeConfig "cluster-admin" { |
| 37 | server = top.apiserverAddress; |
| 38 | certFile = cert; |
| 39 | keyFile = key; |
| 40 | }; |
| 41 | |
| 42 | remote = with config.services; "https://${kubernetes.masterAddress}:${toString cfssl.port}"; |
| 43 | in |
| 44 | { |
| 45 | ###### interface |
| 46 | options.services.kubernetes.pki = with lib.types; { |
| 47 | |
| 48 | enable = mkEnableOption "easyCert issuer service"; |
| 49 | |
| 50 | certs = mkOption { |
| 51 | description = "List of certificate specs to feed to cert generator."; |
| 52 | default = {}; |
| 53 | type = attrs; |
| 54 | }; |
| 55 | |
| 56 | genCfsslCACert = mkOption { |
| 57 | description = '' |
| 58 | Whether to automatically generate cfssl CA certificate and key, |
| 59 | if they don't exist. |
| 60 | ''; |
| 61 | default = true; |
| 62 | type = bool; |
| 63 | }; |
| 64 | |
| 65 | genCfsslAPICerts = mkOption { |
| 66 | description = '' |
| 67 | Whether to automatically generate cfssl API webserver TLS cert and key, |
| 68 | if they don't exist. |
| 69 | ''; |
| 70 | default = true; |
| 71 | type = bool; |
| 72 | }; |
| 73 | |
| 74 | cfsslAPIExtraSANs = mkOption { |
| 75 | description = '' |
| 76 | Extra x509 Subject Alternative Names to be added to the cfssl API webserver TLS cert. |
| 77 | ''; |
| 78 | default = []; |
| 79 | example = [ "subdomain.example.com" ]; |
| 80 | type = listOf str; |
| 81 | }; |
| 82 | |
| 83 | genCfsslAPIToken = mkOption { |
| 84 | description = '' |
| 85 | Whether to automatically generate cfssl API-token secret, |
| 86 | if they doesn't exist. |
| 87 | ''; |
| 88 | default = true; |
| 89 | type = bool; |
| 90 | }; |
| 91 | |
| 92 | pkiTrustOnBootstrap = mkOption { |
| 93 | description = "Whether to always trust remote cfssl server upon initial PKI bootstrap."; |
| 94 | default = true; |
| 95 | type = bool; |
| 96 | }; |
| 97 | |
| 98 | caCertPathPrefix = mkOption { |
| 99 | description = '' |
| 100 | Path-prefrix for the CA-certificate to be used for cfssl signing. |
| 101 | Suffixes ".pem" and "-key.pem" will be automatically appended for |
| 102 | the public and private keys respectively. |
| 103 | ''; |
| 104 | default = "${config.services.cfssl.dataDir}/ca"; |
| 105 | type = str; |
| 106 | }; |
| 107 | |
| 108 | caSpec = mkOption { |
| 109 | description = "Certificate specification for the auto-generated CAcert."; |
| 110 | default = { |
| 111 | CN = "kubernetes-cluster-ca"; |
| 112 | O = "NixOS"; |
| 113 | OU = "services.kubernetes.pki.caSpec"; |
| 114 | L = "auto-generated"; |
| 115 | }; |
| 116 | type = attrs; |
| 117 | }; |
| 118 | |
| 119 | etcClusterAdminKubeconfig = mkOption { |
| 120 | description = '' |
| 121 | Symlink a kubeconfig with cluster-admin privileges to environment path |
| 122 | (/etc/<path>). |
| 123 | ''; |
| 124 | default = null; |
| 125 | type = nullOr str; |
| 126 | }; |
| 127 | |
| 128 | }; |
| 129 | |
| 130 | ###### implementation |
| 131 | config = mkIf cfg.enable |
| 132 | (let |
| 133 | cfsslCertPathPrefix = "${config.services.cfssl.dataDir}/cfssl"; |
| 134 | cfsslCert = "${cfsslCertPathPrefix}.pem"; |
| 135 | cfsslKey = "${cfsslCertPathPrefix}-key.pem"; |
| 136 | in |
| 137 | { |
| 138 | |
| 139 | services.cfssl = mkIf (top.apiserver.enable) { |
| 140 | enable = true; |
| 141 | address = "0.0.0.0"; |
| 142 | tlsCert = cfsslCert; |
| 143 | tlsKey = cfsslKey; |
| 144 | configFile = toString (pkgs.writeText "cfssl-config.json" (builtins.toJSON { |
| 145 | signing = { |
| 146 | profiles = { |
| 147 | default = { |
| 148 | usages = ["digital signature"]; |
| 149 | auth_key = "default"; |
| 150 | expiry = "720h"; |
| 151 | }; |
| 152 | }; |
| 153 | }; |
| 154 | auth_keys = { |
| 155 | default = { |
| 156 | type = "standard"; |
| 157 | key = "file:${cfsslAPITokenPath}"; |
| 158 | }; |
| 159 | }; |
| 160 | })); |
| 161 | }; |
| 162 | |
| 163 | systemd.services.cfssl.preStart = with pkgs; with config.services.cfssl; mkIf (top.apiserver.enable) |
| 164 | (concatStringsSep "\n" [ |
| 165 | "set -e" |
| 166 | (optionalString cfg.genCfsslCACert '' |
| 167 | if [ ! -f "${cfg.caCertPathPrefix}.pem" ]; then |
| 168 | ${cfssl}/bin/cfssl genkey -initca ${csrCA} | \ |
| 169 | ${cfssl}/bin/cfssljson -bare ${cfg.caCertPathPrefix} |
| 170 | fi |
| 171 | '') |
| 172 | (optionalString cfg.genCfsslAPICerts '' |
| 173 | if [ ! -f "${dataDir}/cfssl.pem" ]; then |
| 174 | ${cfssl}/bin/cfssl gencert -ca "${cfg.caCertPathPrefix}.pem" -ca-key "${cfg.caCertPathPrefix}-key.pem" ${csrCfssl} | \ |
| 175 | ${cfssl}/bin/cfssljson -bare ${cfsslCertPathPrefix} |
| 176 | fi |
| 177 | '') |
| 178 | (optionalString cfg.genCfsslAPIToken '' |
| 179 | if [ ! -f "${cfsslAPITokenPath}" ]; then |
| 180 | head -c ${toString (cfsslAPITokenLength / 2)} /dev/urandom | od -An -t x | tr -d ' ' >"${cfsslAPITokenPath}" |
| 181 | fi |
| 182 | chown cfssl "${cfsslAPITokenPath}" && chmod 400 "${cfsslAPITokenPath}" |
| 183 | '')]); |
| 184 | |
| 185 | systemd.services.kube-certmgr-bootstrap = { |
| 186 | description = "Kubernetes certmgr bootstrapper"; |
| 187 | wantedBy = [ "certmgr.service" ]; |
| 188 | after = [ "cfssl.target" ]; |
| 189 | script = concatStringsSep "\n" ['' |
| 190 | set -e |
| 191 | |
| 192 | # If there's a cfssl (cert issuer) running locally, then don't rely on user to |
| 193 | # manually paste it in place. Just symlink. |
| 194 | # otherwise, create the target file, ready for users to insert the token |
| 195 | |
| 196 | if [ -f "${cfsslAPITokenPath}" ]; then |
| 197 | ln -fs "${cfsslAPITokenPath}" "${certmgrAPITokenPath}" |
| 198 | else |
| 199 | touch "${certmgrAPITokenPath}" && chmod 600 "${certmgrAPITokenPath}" |
| 200 | fi |
| 201 | '' |
| 202 | (optionalString (cfg.pkiTrustOnBootstrap) '' |
| 203 | if [ ! -f "${top.caFile}" ] || [ $(cat "${top.caFile}" | wc -c) -lt 1 ]; then |
| 204 | ${pkgs.curl}/bin/curl --fail-early -f -kd '{}' ${remote}/api/v1/cfssl/info | \ |
| 205 | ${pkgs.cfssl}/bin/cfssljson -stdout >${top.caFile} |
| 206 | fi |
| 207 | '') |
| 208 | ]; |
| 209 | serviceConfig = { |
| 210 | RestartSec = "10s"; |
| 211 | Restart = "on-failure"; |
| 212 | }; |
| 213 | }; |
| 214 | |
| 215 | services.certmgr = { |
| 216 | enable = true; |
| 217 | package = pkgs.certmgr-selfsigned; |
| 218 | svcManager = "command"; |
| 219 | specs = |
| 220 | let |
| 221 | mkSpec = _: cert: { |
| 222 | inherit (cert) action; |
| 223 | authority = { |
| 224 | inherit remote; |
| 225 | file.path = cert.caCert; |
| 226 | root_ca = cert.caCert; |
| 227 | profile = "default"; |
| 228 | auth_key_file = certmgrAPITokenPath; |
| 229 | }; |
| 230 | certificate = { |
| 231 | path = cert.cert; |
| 232 | }; |
| 233 | private_key = cert.privateKeyOptions; |
| 234 | request = { |
| 235 | hosts = [cert.CN] ++ cert.hosts; |
| 236 | inherit (cert) CN; |
| 237 | key = { |
| 238 | algo = "rsa"; |
| 239 | size = 2048; |
| 240 | }; |
| 241 | names = [ cert.fields ]; |
| 242 | }; |
| 243 | }; |
| 244 | in |
| 245 | mapAttrs mkSpec cfg.certs; |
| 246 | }; |
| 247 | |
| 248 | #TODO: Get rid of kube-addon-manager in the future for the following reasons |
| 249 | # - it is basically just a shell script wrapped around kubectl |
| 250 | # - it assumes that it is clusterAdmin or can gain clusterAdmin rights through serviceAccount |
| 251 | # - it is designed to be used with k8s system components only |
| 252 | # - it would be better with a more Nix-oriented way of managing addons |
| 253 | systemd.services.kube-addon-manager = mkIf top.addonManager.enable (mkMerge [{ |
| 254 | environment.KUBECONFIG = with cfg.certs.addonManager; |
| 255 | top.lib.mkKubeConfig "addon-manager" { |
| 256 | server = top.apiserverAddress; |
| 257 | certFile = cert; |
| 258 | keyFile = key; |
| 259 | }; |
| 260 | } |
| 261 | |
| 262 | (optionalAttrs (top.addonManager.bootstrapAddons != {}) { |
| 263 | serviceConfig.PermissionsStartOnly = true; |
| 264 | preStart = with pkgs; |
| 265 | let |
| 266 | files = mapAttrsToList (n: v: writeText "${n}.json" (builtins.toJSON v)) |
| 267 | top.addonManager.bootstrapAddons; |
| 268 | in |
| 269 | '' |
| 270 | export KUBECONFIG=${clusterAdminKubeconfig} |
| 271 | ${kubectl}/bin/kubectl apply -f ${concatStringsSep " \\\n -f " files} |
| 272 | ''; |
| 273 | })]); |
| 274 | |
| 275 | environment.etc.${cfg.etcClusterAdminKubeconfig}.source = mkIf (!isNull cfg.etcClusterAdminKubeconfig) |
| 276 | clusterAdminKubeconfig; |
| 277 | |
| 278 | environment.systemPackages = mkIf (top.kubelet.enable || top.proxy.enable) [ |
| 279 | (pkgs.writeScriptBin "nixos-kubernetes-node-join" '' |
| 280 | set -e |
| 281 | exec 1>&2 |
| 282 | |
| 283 | if [ $# -gt 0 ]; then |
| 284 | echo "Usage: $(basename $0)" |
| 285 | echo "" |
| 286 | echo "No args. Apitoken must be provided on stdin." |
| 287 | echo "To get the apitoken, execute: 'sudo cat ${certmgrAPITokenPath}' on the master node." |
| 288 | exit 1 |
| 289 | fi |
| 290 | |
| 291 | if [ $(id -u) != 0 ]; then |
| 292 | echo "Run as root please." |
| 293 | exit 1 |
| 294 | fi |
| 295 | |
| 296 | read -r token |
| 297 | if [ ''${#token} != ${toString cfsslAPITokenLength} ]; then |
| 298 | echo "Token must be of length ${toString cfsslAPITokenLength}." |
| 299 | exit 1 |
| 300 | fi |
| 301 | |
| 302 | echo $token > ${certmgrAPITokenPath} |
| 303 | chmod 600 ${certmgrAPITokenPath} |
| 304 | |
| 305 | echo "Restarting certmgr..." >&1 |
| 306 | systemctl restart certmgr |
| 307 | |
| 308 | echo "Waiting for certs to appear..." >&1 |
| 309 | |
| 310 | ${optionalString top.kubelet.enable '' |
| 311 | while [ ! -f ${cfg.certs.kubelet.cert} ]; do sleep 1; done |
| 312 | echo "Restarting kubelet..." >&1 |
| 313 | systemctl restart kubelet |
| 314 | ''} |
| 315 | |
| 316 | ${optionalString top.proxy.enable '' |
| 317 | while [ ! -f ${cfg.certs.kubeProxyClient.cert} ]; do sleep 1; done |
| 318 | echo "Restarting kube-proxy..." >&1 |
| 319 | systemctl restart kube-proxy |
| 320 | ''} |
| 321 | |
| 322 | ${optionalString top.flannel.enable '' |
| 323 | while [ ! -f ${cfg.certs.flannelClient.cert} ]; do sleep 1; done |
| 324 | echo "Restarting flannel..." >&1 |
| 325 | systemctl restart flannel |
| 326 | ''} |
| 327 | |
| 328 | echo "Node joined succesfully" |
| 329 | '')]; |
| 330 | |
| 331 | # isolate etcd on loopback at the master node |
| 332 | # easyCerts doesn't support multimaster clusters anyway atm. |
| 333 | services.etcd = with cfg.certs.etcd; { |
| 334 | listenClientUrls = ["https://127.0.0.1:2379"]; |
| 335 | listenPeerUrls = ["https://127.0.0.1:2380"]; |
| 336 | advertiseClientUrls = ["https://etcd.local:2379"]; |
| 337 | initialCluster = ["${top.masterAddress}=https://etcd.local:2380"]; |
| 338 | initialAdvertisePeerUrls = ["https://etcd.local:2380"]; |
| 339 | certFile = mkDefault cert; |
| 340 | keyFile = mkDefault key; |
| 341 | trustedCaFile = mkDefault caCert; |
| 342 | }; |
| 343 | networking.extraHosts = mkIf (config.services.etcd.enable) '' |
| 344 | 127.0.0.1 etcd.${top.addons.dns.clusterDomain} etcd.local |
| 345 | ''; |
| 346 | |
| 347 | services.flannel = with cfg.certs.flannelClient; { |
| 348 | kubeconfig = top.lib.mkKubeConfig "flannel" { |
| 349 | server = top.apiserverAddress; |
| 350 | certFile = cert; |
| 351 | keyFile = key; |
| 352 | }; |
| 353 | }; |
| 354 | |
| 355 | services.kubernetes = { |
| 356 | |
| 357 | apiserver = mkIf top.apiserver.enable (with cfg.certs.apiServer; { |
| 358 | etcd = with cfg.certs.apiserverEtcdClient; { |
| 359 | servers = ["https://etcd.local:2379"]; |
| 360 | certFile = mkDefault cert; |
| 361 | keyFile = mkDefault key; |
| 362 | caFile = mkDefault caCert; |
| 363 | }; |
| 364 | clientCaFile = mkDefault caCert; |
| 365 | tlsCertFile = mkDefault cert; |
| 366 | tlsKeyFile = mkDefault key; |
| 367 | serviceAccountKeyFile = mkDefault cfg.certs.serviceAccount.cert; |
| 368 | kubeletClientCaFile = mkDefault caCert; |
| 369 | kubeletClientCertFile = mkDefault cfg.certs.apiserverKubeletClient.cert; |
| 370 | kubeletClientKeyFile = mkDefault cfg.certs.apiserverKubeletClient.key; |
| 371 | proxyClientCertFile = mkDefault cfg.certs.apiserverProxyClient.cert; |
| 372 | proxyClientKeyFile = mkDefault cfg.certs.apiserverProxyClient.key; |
| 373 | }); |
| 374 | controllerManager = mkIf top.controllerManager.enable { |
| 375 | serviceAccountKeyFile = mkDefault cfg.certs.serviceAccount.key; |
| 376 | rootCaFile = cfg.certs.controllerManagerClient.caCert; |
| 377 | kubeconfig = with cfg.certs.controllerManagerClient; { |
| 378 | certFile = mkDefault cert; |
| 379 | keyFile = mkDefault key; |
| 380 | }; |
| 381 | }; |
| 382 | scheduler = mkIf top.scheduler.enable { |
| 383 | kubeconfig = with cfg.certs.schedulerClient; { |
| 384 | certFile = mkDefault cert; |
| 385 | keyFile = mkDefault key; |
| 386 | }; |
| 387 | }; |
| 388 | kubelet = mkIf top.kubelet.enable { |
| 389 | clientCaFile = mkDefault cfg.certs.kubelet.caCert; |
| 390 | tlsCertFile = mkDefault cfg.certs.kubelet.cert; |
| 391 | tlsKeyFile = mkDefault cfg.certs.kubelet.key; |
| 392 | kubeconfig = with cfg.certs.kubeletClient; { |
| 393 | certFile = mkDefault cert; |
| 394 | keyFile = mkDefault key; |
| 395 | }; |
| 396 | }; |
| 397 | proxy = mkIf top.proxy.enable { |
| 398 | kubeconfig = with cfg.certs.kubeProxyClient; { |
| 399 | certFile = mkDefault cert; |
| 400 | keyFile = mkDefault key; |
| 401 | }; |
| 402 | }; |
| 403 | }; |
| 404 | }); |
| 405 | } |