| # Vendored from nixpkgs git 44ad80ab1036c5cc83ada4bfa451dac9939f2a10 |
| # Copyright (c) 2003-2023 Eelco Dolstra and the Nixpkgs/NixOS contributors |
| # SPDX-License-Identifier: MIT |
| |
| { config, lib, pkgs, ... }: |
| |
| with lib; |
| |
| let |
| top = config.services.kubernetes; |
| cfg = top.pki; |
| |
| csrCA = pkgs.writeText "kube-pki-cacert-csr.json" (builtins.toJSON { |
| key = { |
| algo = "rsa"; |
| size = 2048; |
| }; |
| names = singleton cfg.caSpec; |
| }); |
| |
| csrCfssl = pkgs.writeText "kube-pki-cfssl-csr.json" (builtins.toJSON { |
| key = { |
| algo = "rsa"; |
| size = 2048; |
| }; |
| CN = top.masterAddress; |
| hosts = [top.masterAddress] ++ cfg.cfsslAPIExtraSANs; |
| }); |
| |
| cfsslAPITokenBaseName = "apitoken.secret"; |
| cfsslAPITokenPath = "${config.services.cfssl.dataDir}/${cfsslAPITokenBaseName}"; |
| certmgrAPITokenPath = "${top.secretsPath}/${cfsslAPITokenBaseName}"; |
| cfsslAPITokenLength = 32; |
| |
| clusterAdminKubeconfig = with cfg.certs.clusterAdmin; |
| top.lib.mkKubeConfig "cluster-admin" { |
| server = top.apiserverAddress; |
| certFile = cert; |
| keyFile = key; |
| }; |
| |
| remote = with config.services; "https://${kubernetes.masterAddress}:${toString cfssl.port}"; |
| in |
| { |
| ###### interface |
| options.services.kubernetes.pki = with lib.types; { |
| |
| enable = mkEnableOption "easyCert issuer service"; |
| |
| certs = mkOption { |
| description = "List of certificate specs to feed to cert generator."; |
| default = {}; |
| type = attrs; |
| }; |
| |
| genCfsslCACert = mkOption { |
| description = '' |
| Whether to automatically generate cfssl CA certificate and key, |
| if they don't exist. |
| ''; |
| default = true; |
| type = bool; |
| }; |
| |
| genCfsslAPICerts = mkOption { |
| description = '' |
| Whether to automatically generate cfssl API webserver TLS cert and key, |
| if they don't exist. |
| ''; |
| default = true; |
| type = bool; |
| }; |
| |
| cfsslAPIExtraSANs = mkOption { |
| description = '' |
| Extra x509 Subject Alternative Names to be added to the cfssl API webserver TLS cert. |
| ''; |
| default = []; |
| example = [ "subdomain.example.com" ]; |
| type = listOf str; |
| }; |
| |
| genCfsslAPIToken = mkOption { |
| description = '' |
| Whether to automatically generate cfssl API-token secret, |
| if they doesn't exist. |
| ''; |
| default = true; |
| type = bool; |
| }; |
| |
| pkiTrustOnBootstrap = mkOption { |
| description = "Whether to always trust remote cfssl server upon initial PKI bootstrap."; |
| default = true; |
| type = bool; |
| }; |
| |
| caCertPathPrefix = mkOption { |
| description = '' |
| Path-prefrix for the CA-certificate to be used for cfssl signing. |
| Suffixes ".pem" and "-key.pem" will be automatically appended for |
| the public and private keys respectively. |
| ''; |
| default = "${config.services.cfssl.dataDir}/ca"; |
| type = str; |
| }; |
| |
| caSpec = mkOption { |
| description = "Certificate specification for the auto-generated CAcert."; |
| default = { |
| CN = "kubernetes-cluster-ca"; |
| O = "NixOS"; |
| OU = "services.kubernetes.pki.caSpec"; |
| L = "auto-generated"; |
| }; |
| type = attrs; |
| }; |
| |
| etcClusterAdminKubeconfig = mkOption { |
| description = '' |
| Symlink a kubeconfig with cluster-admin privileges to environment path |
| (/etc/<path>). |
| ''; |
| default = null; |
| type = nullOr str; |
| }; |
| |
| }; |
| |
| ###### implementation |
| config = mkIf cfg.enable |
| (let |
| cfsslCertPathPrefix = "${config.services.cfssl.dataDir}/cfssl"; |
| cfsslCert = "${cfsslCertPathPrefix}.pem"; |
| cfsslKey = "${cfsslCertPathPrefix}-key.pem"; |
| in |
| { |
| |
| services.cfssl = mkIf (top.apiserver.enable) { |
| enable = true; |
| address = "0.0.0.0"; |
| tlsCert = cfsslCert; |
| tlsKey = cfsslKey; |
| configFile = toString (pkgs.writeText "cfssl-config.json" (builtins.toJSON { |
| signing = { |
| profiles = { |
| default = { |
| usages = ["digital signature"]; |
| auth_key = "default"; |
| expiry = "720h"; |
| }; |
| }; |
| }; |
| auth_keys = { |
| default = { |
| type = "standard"; |
| key = "file:${cfsslAPITokenPath}"; |
| }; |
| }; |
| })); |
| }; |
| |
| systemd.services.cfssl.preStart = with pkgs; with config.services.cfssl; mkIf (top.apiserver.enable) |
| (concatStringsSep "\n" [ |
| "set -e" |
| (optionalString cfg.genCfsslCACert '' |
| if [ ! -f "${cfg.caCertPathPrefix}.pem" ]; then |
| ${cfssl}/bin/cfssl genkey -initca ${csrCA} | \ |
| ${cfssl}/bin/cfssljson -bare ${cfg.caCertPathPrefix} |
| fi |
| '') |
| (optionalString cfg.genCfsslAPICerts '' |
| if [ ! -f "${dataDir}/cfssl.pem" ]; then |
| ${cfssl}/bin/cfssl gencert -ca "${cfg.caCertPathPrefix}.pem" -ca-key "${cfg.caCertPathPrefix}-key.pem" ${csrCfssl} | \ |
| ${cfssl}/bin/cfssljson -bare ${cfsslCertPathPrefix} |
| fi |
| '') |
| (optionalString cfg.genCfsslAPIToken '' |
| if [ ! -f "${cfsslAPITokenPath}" ]; then |
| head -c ${toString (cfsslAPITokenLength / 2)} /dev/urandom | od -An -t x | tr -d ' ' >"${cfsslAPITokenPath}" |
| fi |
| chown cfssl "${cfsslAPITokenPath}" && chmod 400 "${cfsslAPITokenPath}" |
| '')]); |
| |
| systemd.services.kube-certmgr-bootstrap = { |
| description = "Kubernetes certmgr bootstrapper"; |
| wantedBy = [ "certmgr.service" ]; |
| after = [ "cfssl.target" ]; |
| script = concatStringsSep "\n" ['' |
| set -e |
| |
| # If there's a cfssl (cert issuer) running locally, then don't rely on user to |
| # manually paste it in place. Just symlink. |
| # otherwise, create the target file, ready for users to insert the token |
| |
| if [ -f "${cfsslAPITokenPath}" ]; then |
| ln -fs "${cfsslAPITokenPath}" "${certmgrAPITokenPath}" |
| else |
| touch "${certmgrAPITokenPath}" && chmod 600 "${certmgrAPITokenPath}" |
| fi |
| '' |
| (optionalString (cfg.pkiTrustOnBootstrap) '' |
| if [ ! -f "${top.caFile}" ] || [ $(cat "${top.caFile}" | wc -c) -lt 1 ]; then |
| ${pkgs.curl}/bin/curl --fail-early -f -kd '{}' ${remote}/api/v1/cfssl/info | \ |
| ${pkgs.cfssl}/bin/cfssljson -stdout >${top.caFile} |
| fi |
| '') |
| ]; |
| serviceConfig = { |
| RestartSec = "10s"; |
| Restart = "on-failure"; |
| }; |
| }; |
| |
| services.certmgr = { |
| enable = true; |
| package = pkgs.certmgr-selfsigned; |
| svcManager = "command"; |
| specs = |
| let |
| mkSpec = _: cert: { |
| inherit (cert) action; |
| authority = { |
| inherit remote; |
| file.path = cert.caCert; |
| root_ca = cert.caCert; |
| profile = "default"; |
| auth_key_file = certmgrAPITokenPath; |
| }; |
| certificate = { |
| path = cert.cert; |
| }; |
| private_key = cert.privateKeyOptions; |
| request = { |
| hosts = [cert.CN] ++ cert.hosts; |
| inherit (cert) CN; |
| key = { |
| algo = "rsa"; |
| size = 2048; |
| }; |
| names = [ cert.fields ]; |
| }; |
| }; |
| in |
| mapAttrs mkSpec cfg.certs; |
| }; |
| |
| #TODO: Get rid of kube-addon-manager in the future for the following reasons |
| # - it is basically just a shell script wrapped around kubectl |
| # - it assumes that it is clusterAdmin or can gain clusterAdmin rights through serviceAccount |
| # - it is designed to be used with k8s system components only |
| # - it would be better with a more Nix-oriented way of managing addons |
| systemd.services.kube-addon-manager = mkIf top.addonManager.enable (mkMerge [{ |
| environment.KUBECONFIG = with cfg.certs.addonManager; |
| top.lib.mkKubeConfig "addon-manager" { |
| server = top.apiserverAddress; |
| certFile = cert; |
| keyFile = key; |
| }; |
| } |
| |
| (optionalAttrs (top.addonManager.bootstrapAddons != {}) { |
| serviceConfig.PermissionsStartOnly = true; |
| preStart = with pkgs; |
| let |
| files = mapAttrsToList (n: v: writeText "${n}.json" (builtins.toJSON v)) |
| top.addonManager.bootstrapAddons; |
| in |
| '' |
| export KUBECONFIG=${clusterAdminKubeconfig} |
| ${kubectl}/bin/kubectl apply -f ${concatStringsSep " \\\n -f " files} |
| ''; |
| })]); |
| |
| environment.etc.${cfg.etcClusterAdminKubeconfig}.source = mkIf (!isNull cfg.etcClusterAdminKubeconfig) |
| clusterAdminKubeconfig; |
| |
| environment.systemPackages = mkIf (top.kubelet.enable || top.proxy.enable) [ |
| (pkgs.writeScriptBin "nixos-kubernetes-node-join" '' |
| set -e |
| exec 1>&2 |
| |
| if [ $# -gt 0 ]; then |
| echo "Usage: $(basename $0)" |
| echo "" |
| echo "No args. Apitoken must be provided on stdin." |
| echo "To get the apitoken, execute: 'sudo cat ${certmgrAPITokenPath}' on the master node." |
| exit 1 |
| fi |
| |
| if [ $(id -u) != 0 ]; then |
| echo "Run as root please." |
| exit 1 |
| fi |
| |
| read -r token |
| if [ ''${#token} != ${toString cfsslAPITokenLength} ]; then |
| echo "Token must be of length ${toString cfsslAPITokenLength}." |
| exit 1 |
| fi |
| |
| echo $token > ${certmgrAPITokenPath} |
| chmod 600 ${certmgrAPITokenPath} |
| |
| echo "Restarting certmgr..." >&1 |
| systemctl restart certmgr |
| |
| echo "Waiting for certs to appear..." >&1 |
| |
| ${optionalString top.kubelet.enable '' |
| while [ ! -f ${cfg.certs.kubelet.cert} ]; do sleep 1; done |
| echo "Restarting kubelet..." >&1 |
| systemctl restart kubelet |
| ''} |
| |
| ${optionalString top.proxy.enable '' |
| while [ ! -f ${cfg.certs.kubeProxyClient.cert} ]; do sleep 1; done |
| echo "Restarting kube-proxy..." >&1 |
| systemctl restart kube-proxy |
| ''} |
| |
| ${optionalString top.flannel.enable '' |
| while [ ! -f ${cfg.certs.flannelClient.cert} ]; do sleep 1; done |
| echo "Restarting flannel..." >&1 |
| systemctl restart flannel |
| ''} |
| |
| echo "Node joined succesfully" |
| '')]; |
| |
| # isolate etcd on loopback at the master node |
| # easyCerts doesn't support multimaster clusters anyway atm. |
| services.etcd = with cfg.certs.etcd; { |
| listenClientUrls = ["https://127.0.0.1:2379"]; |
| listenPeerUrls = ["https://127.0.0.1:2380"]; |
| advertiseClientUrls = ["https://etcd.local:2379"]; |
| initialCluster = ["${top.masterAddress}=https://etcd.local:2380"]; |
| initialAdvertisePeerUrls = ["https://etcd.local:2380"]; |
| certFile = mkDefault cert; |
| keyFile = mkDefault key; |
| trustedCaFile = mkDefault caCert; |
| }; |
| networking.extraHosts = mkIf (config.services.etcd.enable) '' |
| 127.0.0.1 etcd.${top.addons.dns.clusterDomain} etcd.local |
| ''; |
| |
| services.flannel = with cfg.certs.flannelClient; { |
| kubeconfig = top.lib.mkKubeConfig "flannel" { |
| server = top.apiserverAddress; |
| certFile = cert; |
| keyFile = key; |
| }; |
| }; |
| |
| services.kubernetes = { |
| |
| apiserver = mkIf top.apiserver.enable (with cfg.certs.apiServer; { |
| etcd = with cfg.certs.apiserverEtcdClient; { |
| servers = ["https://etcd.local:2379"]; |
| certFile = mkDefault cert; |
| keyFile = mkDefault key; |
| caFile = mkDefault caCert; |
| }; |
| clientCaFile = mkDefault caCert; |
| tlsCertFile = mkDefault cert; |
| tlsKeyFile = mkDefault key; |
| serviceAccountKeyFile = mkDefault cfg.certs.serviceAccount.cert; |
| kubeletClientCaFile = mkDefault caCert; |
| kubeletClientCertFile = mkDefault cfg.certs.apiserverKubeletClient.cert; |
| kubeletClientKeyFile = mkDefault cfg.certs.apiserverKubeletClient.key; |
| proxyClientCertFile = mkDefault cfg.certs.apiserverProxyClient.cert; |
| proxyClientKeyFile = mkDefault cfg.certs.apiserverProxyClient.key; |
| }); |
| controllerManager = mkIf top.controllerManager.enable { |
| serviceAccountKeyFile = mkDefault cfg.certs.serviceAccount.key; |
| rootCaFile = cfg.certs.controllerManagerClient.caCert; |
| kubeconfig = with cfg.certs.controllerManagerClient; { |
| certFile = mkDefault cert; |
| keyFile = mkDefault key; |
| }; |
| }; |
| scheduler = mkIf top.scheduler.enable { |
| kubeconfig = with cfg.certs.schedulerClient; { |
| certFile = mkDefault cert; |
| keyFile = mkDefault key; |
| }; |
| }; |
| kubelet = mkIf top.kubelet.enable { |
| clientCaFile = mkDefault cfg.certs.kubelet.caCert; |
| tlsCertFile = mkDefault cfg.certs.kubelet.cert; |
| tlsKeyFile = mkDefault cfg.certs.kubelet.key; |
| kubeconfig = with cfg.certs.kubeletClient; { |
| certFile = mkDefault cert; |
| keyFile = mkDefault key; |
| }; |
| }; |
| proxy = mkIf top.proxy.enable { |
| kubeconfig = with cfg.certs.kubeProxyClient; { |
| certFile = mkDefault cert; |
| keyFile = mkDefault key; |
| }; |
| }; |
| }; |
| }); |
| } |