k0: host os bump wip
This bumps it on bc01n01, but nowhere else yet.
We have to vendor some more kubelet bits unfortunately.
Change-Id: Ifb169dd9c2c19d60f88d946d065d4446141601b1
Reviewed-on: https://gerrit.hackerspace.pl/c/hscloud/+/1465
Reviewed-by: implr <implr@hackerspace.pl>
diff --git a/cluster/machines/modules/vendor/pki.nix b/cluster/machines/modules/vendor/pki.nix
new file mode 100644
index 0000000..fa40e88
--- /dev/null
+++ b/cluster/machines/modules/vendor/pki.nix
@@ -0,0 +1,405 @@
+# Vendored from nixpkgs git 44ad80ab1036c5cc83ada4bfa451dac9939f2a10
+# Copyright (c) 2003-2023 Eelco Dolstra and the Nixpkgs/NixOS contributors
+# SPDX-License-Identifier: MIT
+
+{ config, lib, pkgs, ... }:
+
+with lib;
+
+let
+ top = config.services.kubernetes;
+ cfg = top.pki;
+
+ csrCA = pkgs.writeText "kube-pki-cacert-csr.json" (builtins.toJSON {
+ key = {
+ algo = "rsa";
+ size = 2048;
+ };
+ names = singleton cfg.caSpec;
+ });
+
+ csrCfssl = pkgs.writeText "kube-pki-cfssl-csr.json" (builtins.toJSON {
+ key = {
+ algo = "rsa";
+ size = 2048;
+ };
+ CN = top.masterAddress;
+ hosts = [top.masterAddress] ++ cfg.cfsslAPIExtraSANs;
+ });
+
+ cfsslAPITokenBaseName = "apitoken.secret";
+ cfsslAPITokenPath = "${config.services.cfssl.dataDir}/${cfsslAPITokenBaseName}";
+ certmgrAPITokenPath = "${top.secretsPath}/${cfsslAPITokenBaseName}";
+ cfsslAPITokenLength = 32;
+
+ clusterAdminKubeconfig = with cfg.certs.clusterAdmin;
+ top.lib.mkKubeConfig "cluster-admin" {
+ server = top.apiserverAddress;
+ certFile = cert;
+ keyFile = key;
+ };
+
+ remote = with config.services; "https://${kubernetes.masterAddress}:${toString cfssl.port}";
+in
+{
+ ###### interface
+ options.services.kubernetes.pki = with lib.types; {
+
+ enable = mkEnableOption "easyCert issuer service";
+
+ certs = mkOption {
+ description = "List of certificate specs to feed to cert generator.";
+ default = {};
+ type = attrs;
+ };
+
+ genCfsslCACert = mkOption {
+ description = ''
+ Whether to automatically generate cfssl CA certificate and key,
+ if they don't exist.
+ '';
+ default = true;
+ type = bool;
+ };
+
+ genCfsslAPICerts = mkOption {
+ description = ''
+ Whether to automatically generate cfssl API webserver TLS cert and key,
+ if they don't exist.
+ '';
+ default = true;
+ type = bool;
+ };
+
+ cfsslAPIExtraSANs = mkOption {
+ description = ''
+ Extra x509 Subject Alternative Names to be added to the cfssl API webserver TLS cert.
+ '';
+ default = [];
+ example = [ "subdomain.example.com" ];
+ type = listOf str;
+ };
+
+ genCfsslAPIToken = mkOption {
+ description = ''
+ Whether to automatically generate cfssl API-token secret,
+ if they doesn't exist.
+ '';
+ default = true;
+ type = bool;
+ };
+
+ pkiTrustOnBootstrap = mkOption {
+ description = "Whether to always trust remote cfssl server upon initial PKI bootstrap.";
+ default = true;
+ type = bool;
+ };
+
+ caCertPathPrefix = mkOption {
+ description = ''
+ Path-prefrix for the CA-certificate to be used for cfssl signing.
+ Suffixes ".pem" and "-key.pem" will be automatically appended for
+ the public and private keys respectively.
+ '';
+ default = "${config.services.cfssl.dataDir}/ca";
+ type = str;
+ };
+
+ caSpec = mkOption {
+ description = "Certificate specification for the auto-generated CAcert.";
+ default = {
+ CN = "kubernetes-cluster-ca";
+ O = "NixOS";
+ OU = "services.kubernetes.pki.caSpec";
+ L = "auto-generated";
+ };
+ type = attrs;
+ };
+
+ etcClusterAdminKubeconfig = mkOption {
+ description = ''
+ Symlink a kubeconfig with cluster-admin privileges to environment path
+ (/etc/<path>).
+ '';
+ default = null;
+ type = nullOr str;
+ };
+
+ };
+
+ ###### implementation
+ config = mkIf cfg.enable
+ (let
+ cfsslCertPathPrefix = "${config.services.cfssl.dataDir}/cfssl";
+ cfsslCert = "${cfsslCertPathPrefix}.pem";
+ cfsslKey = "${cfsslCertPathPrefix}-key.pem";
+ in
+ {
+
+ services.cfssl = mkIf (top.apiserver.enable) {
+ enable = true;
+ address = "0.0.0.0";
+ tlsCert = cfsslCert;
+ tlsKey = cfsslKey;
+ configFile = toString (pkgs.writeText "cfssl-config.json" (builtins.toJSON {
+ signing = {
+ profiles = {
+ default = {
+ usages = ["digital signature"];
+ auth_key = "default";
+ expiry = "720h";
+ };
+ };
+ };
+ auth_keys = {
+ default = {
+ type = "standard";
+ key = "file:${cfsslAPITokenPath}";
+ };
+ };
+ }));
+ };
+
+ systemd.services.cfssl.preStart = with pkgs; with config.services.cfssl; mkIf (top.apiserver.enable)
+ (concatStringsSep "\n" [
+ "set -e"
+ (optionalString cfg.genCfsslCACert ''
+ if [ ! -f "${cfg.caCertPathPrefix}.pem" ]; then
+ ${cfssl}/bin/cfssl genkey -initca ${csrCA} | \
+ ${cfssl}/bin/cfssljson -bare ${cfg.caCertPathPrefix}
+ fi
+ '')
+ (optionalString cfg.genCfsslAPICerts ''
+ if [ ! -f "${dataDir}/cfssl.pem" ]; then
+ ${cfssl}/bin/cfssl gencert -ca "${cfg.caCertPathPrefix}.pem" -ca-key "${cfg.caCertPathPrefix}-key.pem" ${csrCfssl} | \
+ ${cfssl}/bin/cfssljson -bare ${cfsslCertPathPrefix}
+ fi
+ '')
+ (optionalString cfg.genCfsslAPIToken ''
+ if [ ! -f "${cfsslAPITokenPath}" ]; then
+ head -c ${toString (cfsslAPITokenLength / 2)} /dev/urandom | od -An -t x | tr -d ' ' >"${cfsslAPITokenPath}"
+ fi
+ chown cfssl "${cfsslAPITokenPath}" && chmod 400 "${cfsslAPITokenPath}"
+ '')]);
+
+ systemd.services.kube-certmgr-bootstrap = {
+ description = "Kubernetes certmgr bootstrapper";
+ wantedBy = [ "certmgr.service" ];
+ after = [ "cfssl.target" ];
+ script = concatStringsSep "\n" [''
+ set -e
+
+ # If there's a cfssl (cert issuer) running locally, then don't rely on user to
+ # manually paste it in place. Just symlink.
+ # otherwise, create the target file, ready for users to insert the token
+
+ if [ -f "${cfsslAPITokenPath}" ]; then
+ ln -fs "${cfsslAPITokenPath}" "${certmgrAPITokenPath}"
+ else
+ touch "${certmgrAPITokenPath}" && chmod 600 "${certmgrAPITokenPath}"
+ fi
+ ''
+ (optionalString (cfg.pkiTrustOnBootstrap) ''
+ if [ ! -f "${top.caFile}" ] || [ $(cat "${top.caFile}" | wc -c) -lt 1 ]; then
+ ${pkgs.curl}/bin/curl --fail-early -f -kd '{}' ${remote}/api/v1/cfssl/info | \
+ ${pkgs.cfssl}/bin/cfssljson -stdout >${top.caFile}
+ fi
+ '')
+ ];
+ serviceConfig = {
+ RestartSec = "10s";
+ Restart = "on-failure";
+ };
+ };
+
+ services.certmgr = {
+ enable = true;
+ package = pkgs.certmgr-selfsigned;
+ svcManager = "command";
+ specs =
+ let
+ mkSpec = _: cert: {
+ inherit (cert) action;
+ authority = {
+ inherit remote;
+ file.path = cert.caCert;
+ root_ca = cert.caCert;
+ profile = "default";
+ auth_key_file = certmgrAPITokenPath;
+ };
+ certificate = {
+ path = cert.cert;
+ };
+ private_key = cert.privateKeyOptions;
+ request = {
+ hosts = [cert.CN] ++ cert.hosts;
+ inherit (cert) CN;
+ key = {
+ algo = "rsa";
+ size = 2048;
+ };
+ names = [ cert.fields ];
+ };
+ };
+ in
+ mapAttrs mkSpec cfg.certs;
+ };
+
+ #TODO: Get rid of kube-addon-manager in the future for the following reasons
+ # - it is basically just a shell script wrapped around kubectl
+ # - it assumes that it is clusterAdmin or can gain clusterAdmin rights through serviceAccount
+ # - it is designed to be used with k8s system components only
+ # - it would be better with a more Nix-oriented way of managing addons
+ systemd.services.kube-addon-manager = mkIf top.addonManager.enable (mkMerge [{
+ environment.KUBECONFIG = with cfg.certs.addonManager;
+ top.lib.mkKubeConfig "addon-manager" {
+ server = top.apiserverAddress;
+ certFile = cert;
+ keyFile = key;
+ };
+ }
+
+ (optionalAttrs (top.addonManager.bootstrapAddons != {}) {
+ serviceConfig.PermissionsStartOnly = true;
+ preStart = with pkgs;
+ let
+ files = mapAttrsToList (n: v: writeText "${n}.json" (builtins.toJSON v))
+ top.addonManager.bootstrapAddons;
+ in
+ ''
+ export KUBECONFIG=${clusterAdminKubeconfig}
+ ${kubectl}/bin/kubectl apply -f ${concatStringsSep " \\\n -f " files}
+ '';
+ })]);
+
+ environment.etc.${cfg.etcClusterAdminKubeconfig}.source = mkIf (!isNull cfg.etcClusterAdminKubeconfig)
+ clusterAdminKubeconfig;
+
+ environment.systemPackages = mkIf (top.kubelet.enable || top.proxy.enable) [
+ (pkgs.writeScriptBin "nixos-kubernetes-node-join" ''
+ set -e
+ exec 1>&2
+
+ if [ $# -gt 0 ]; then
+ echo "Usage: $(basename $0)"
+ echo ""
+ echo "No args. Apitoken must be provided on stdin."
+ echo "To get the apitoken, execute: 'sudo cat ${certmgrAPITokenPath}' on the master node."
+ exit 1
+ fi
+
+ if [ $(id -u) != 0 ]; then
+ echo "Run as root please."
+ exit 1
+ fi
+
+ read -r token
+ if [ ''${#token} != ${toString cfsslAPITokenLength} ]; then
+ echo "Token must be of length ${toString cfsslAPITokenLength}."
+ exit 1
+ fi
+
+ echo $token > ${certmgrAPITokenPath}
+ chmod 600 ${certmgrAPITokenPath}
+
+ echo "Restarting certmgr..." >&1
+ systemctl restart certmgr
+
+ echo "Waiting for certs to appear..." >&1
+
+ ${optionalString top.kubelet.enable ''
+ while [ ! -f ${cfg.certs.kubelet.cert} ]; do sleep 1; done
+ echo "Restarting kubelet..." >&1
+ systemctl restart kubelet
+ ''}
+
+ ${optionalString top.proxy.enable ''
+ while [ ! -f ${cfg.certs.kubeProxyClient.cert} ]; do sleep 1; done
+ echo "Restarting kube-proxy..." >&1
+ systemctl restart kube-proxy
+ ''}
+
+ ${optionalString top.flannel.enable ''
+ while [ ! -f ${cfg.certs.flannelClient.cert} ]; do sleep 1; done
+ echo "Restarting flannel..." >&1
+ systemctl restart flannel
+ ''}
+
+ echo "Node joined succesfully"
+ '')];
+
+ # isolate etcd on loopback at the master node
+ # easyCerts doesn't support multimaster clusters anyway atm.
+ services.etcd = with cfg.certs.etcd; {
+ listenClientUrls = ["https://127.0.0.1:2379"];
+ listenPeerUrls = ["https://127.0.0.1:2380"];
+ advertiseClientUrls = ["https://etcd.local:2379"];
+ initialCluster = ["${top.masterAddress}=https://etcd.local:2380"];
+ initialAdvertisePeerUrls = ["https://etcd.local:2380"];
+ certFile = mkDefault cert;
+ keyFile = mkDefault key;
+ trustedCaFile = mkDefault caCert;
+ };
+ networking.extraHosts = mkIf (config.services.etcd.enable) ''
+ 127.0.0.1 etcd.${top.addons.dns.clusterDomain} etcd.local
+ '';
+
+ services.flannel = with cfg.certs.flannelClient; {
+ kubeconfig = top.lib.mkKubeConfig "flannel" {
+ server = top.apiserverAddress;
+ certFile = cert;
+ keyFile = key;
+ };
+ };
+
+ services.kubernetes = {
+
+ apiserver = mkIf top.apiserver.enable (with cfg.certs.apiServer; {
+ etcd = with cfg.certs.apiserverEtcdClient; {
+ servers = ["https://etcd.local:2379"];
+ certFile = mkDefault cert;
+ keyFile = mkDefault key;
+ caFile = mkDefault caCert;
+ };
+ clientCaFile = mkDefault caCert;
+ tlsCertFile = mkDefault cert;
+ tlsKeyFile = mkDefault key;
+ serviceAccountKeyFile = mkDefault cfg.certs.serviceAccount.cert;
+ kubeletClientCaFile = mkDefault caCert;
+ kubeletClientCertFile = mkDefault cfg.certs.apiserverKubeletClient.cert;
+ kubeletClientKeyFile = mkDefault cfg.certs.apiserverKubeletClient.key;
+ proxyClientCertFile = mkDefault cfg.certs.apiserverProxyClient.cert;
+ proxyClientKeyFile = mkDefault cfg.certs.apiserverProxyClient.key;
+ });
+ controllerManager = mkIf top.controllerManager.enable {
+ serviceAccountKeyFile = mkDefault cfg.certs.serviceAccount.key;
+ rootCaFile = cfg.certs.controllerManagerClient.caCert;
+ kubeconfig = with cfg.certs.controllerManagerClient; {
+ certFile = mkDefault cert;
+ keyFile = mkDefault key;
+ };
+ };
+ scheduler = mkIf top.scheduler.enable {
+ kubeconfig = with cfg.certs.schedulerClient; {
+ certFile = mkDefault cert;
+ keyFile = mkDefault key;
+ };
+ };
+ kubelet = mkIf top.kubelet.enable {
+ clientCaFile = mkDefault cfg.certs.kubelet.caCert;
+ tlsCertFile = mkDefault cfg.certs.kubelet.cert;
+ tlsKeyFile = mkDefault cfg.certs.kubelet.key;
+ kubeconfig = with cfg.certs.kubeletClient; {
+ certFile = mkDefault cert;
+ keyFile = mkDefault key;
+ };
+ };
+ proxy = mkIf top.proxy.enable {
+ kubeconfig = with cfg.certs.kubeProxyClient; {
+ certFile = mkDefault cert;
+ keyFile = mkDefault key;
+ };
+ };
+ };
+ });
+}