cluster/machines/modules/kubelet.nix - hscloud - Gitiles

 # Same as upstream kubelet.nix module from nixpkgs, but with the following
 # changes:
 #   - cni tunables nuked and replaced with static host dirs, so that calico
 #     running on k8s can drop CNI plugins there itself
 #   - package configurable separately from rest of kubernetes

 { config, lib, pkgs, ... }:

 with lib;

 let
   top = config.services.kubernetes;
   cfg = top.kubelet;

   infraContainer = pkgs.dockerTools.buildImage {
     name = "pause";
     tag = "latest";
     contents = top.package.pause;
     config.Cmd = ["/bin/pause"];
   };

   kubeconfig = top.lib.mkKubeConfig "kubelet" cfg.kubeconfig;

   manifestPath = "kubernetes/manifests";

   taintOptions = with lib.types; { name, ... }: {
     options = {
       key = mkOption {
         description = "Key of taint.";
         default = name;
         type = str;
       };
       value = mkOption {
         description = "Value of taint.";
         type = str;
       };
       effect = mkOption {
         description = "Effect of taint.";
         example = "NoSchedule";
         type = enum ["NoSchedule" "PreferNoSchedule" "NoExecute"];
       };
     };
   };

   taints = concatMapStringsSep "," (v: "${v.key}=${v.value}:${v.effect}") (mapAttrsToList (n: v: v) cfg.taints);
 in
 {
   # services/cluster/kubernetes/default.nix still wants to poke flannel,
   # but since we nuke that module we have to add a fake tunable for it.
   options.services.kubernetes.flannel = {
     enable = mkEnableOption "enable flannel networking";
   };

   ###### interface
   options.services.kubernetes.kubelet = with lib.types; {

     address = mkOption {
       description = "Kubernetes kubelet info server listening address.";
       default = "0.0.0.0";
       type = str;
     };

     clusterDns = mkOption {
       description = "Use alternative DNS.";
       default = "10.1.0.1";
       type = str;
     };

     clusterDomain = mkOption {
       description = "Use alternative domain.";
       default = config.services.kubernetes.addons.dns.clusterDomain;
       type = str;
     };

     clientCaFile = mkOption {
       description = "Kubernetes apiserver CA file for client authentication.";
       default = top.caFile;
       type = nullOr path;
     };

     enable = mkEnableOption "Kubernetes kubelet.";

     extraOpts = mkOption {
       description = "Kubernetes kubelet extra command line options.";
       default = "";
       type = str;
     };

     featureGates = mkOption {
       description = "List set of feature gates";
       default = top.featureGates;
       type = listOf str;
     };

     healthz = {
       bind = mkOption {
         description = "Kubernetes kubelet healthz listening address.";
         default = "127.0.0.1";
         type = str;
       };

       port = mkOption {
         description = "Kubernetes kubelet healthz port.";
         default = 10248;
         type = int;
       };
     };

     hostname = mkOption {
       description = "Kubernetes kubelet hostname override.";
       default = config.networking.hostName;
       type = str;
     };

     kubeconfig = top.lib.mkKubeConfigOptions "Kubelet";

     manifests = mkOption {
       description = "List of manifests to bootstrap with kubelet (only pods can be created as manifest entry)";
       type = attrsOf attrs;
       default = {};
     };

     networkPlugin = mkOption {
       description = "Network plugin to use by Kubernetes.";
       type = nullOr (enum ["cni" "kubenet"]);
       default = "kubenet";
     };

     nodeIp = mkOption {
       description = "IP address of the node. If set, kubelet will use this IP address for the node.";
       default = null;
       type = nullOr str;
     };

     registerNode = mkOption {
       description = "Whether to auto register kubelet with API server.";
       default = true;
       type = bool;
     };

     package = mkOption {
       description = "Kubernetes package to use.";
       type = types.package;
       default = pkgs.kubernetes;
       defaultText = "pkgs.kubernetes";
     };

     port = mkOption {
       description = "Kubernetes kubelet info server listening port.";
       default = 10250;
       type = int;
     };

     seedDockerImages = mkOption {
       description = "List of docker images to preload on system";
       default = [];
       type = listOf package;
     };

     taints = mkOption {
       description = "Node taints (https://kubernetes.io/docs/concepts/configuration/assign-pod-node/).";
       default = {};
       type = attrsOf (submodule [ taintOptions ]);
     };

     tlsCertFile = mkOption {
       description = "File containing x509 Certificate for HTTPS.";
       default = null;
       type = nullOr path;
     };

     tlsKeyFile = mkOption {
       description = "File containing x509 private key matching tlsCertFile.";
       default = null;
       type = nullOr path;
     };

     unschedulable = mkOption {
       description = "Whether to set node taint to unschedulable=true as it is the case of node that has only master role.";
       default = false;
       type = bool;
     };

     verbosity = mkOption {
       description = ''
         Optional glog verbosity level for logging statements. See
         <link xlink:href="https://github.com/kubernetes/community/blob/master/contributors/devel/logging.md"/>
       '';
       default = null;
       type = nullOr int;
     };

   };

   ###### implementation
   config = mkMerge [
     (mkIf cfg.enable {
       services.kubernetes.kubelet.seedDockerImages = [infraContainer];

       # Drop crictl into administrative command line.
       environment.systemPackages = with pkgs; [ cri-tools ];

       # Force disable Docker.
       virtualisation.docker.enable = false;

       # TODO(q3k): move to unified cgroups (cgroup v2) once we upgrade to
       # Kubelet 1.19.
       systemd.enableUnifiedCgroupHierarchy = false;

       # Run containerd service. This is exposes the CRI API that is consumed by
       # crictl and Kubelet.
       systemd.services.containerd = {
         description = "containerd container runtime";
         wantedBy = [ "kubernetes.target" ];
         after = [ "network.target" ];
         path = with pkgs; [ runc iptables ];
         serviceConfig = {
           Delegate = "yes";
           KillMode = "process";
           Restart = "always";
           RestartSec = "5";
           LimitNPROC = "infinity";
           LimitCORE = "infinity";
           # https://github.com/coreos/fedora-coreos-tracker/issues/329
           LimitNOFILE = "1048576";
           TasksMax = "infinity";
           OOMScoreAdjust = "-999";

           ExecStart = "${pkgs.containerd}/bin/containerd -c ${./containerd.toml}";
         };
       };

       systemd.services.kubelet = {
         description = "Kubernetes Kubelet Service";
         wantedBy = [ "kubernetes.target" ];
         after = [ "network.target" "containerd.service" "kube-apiserver.service" ];
         path = with pkgs; [ gitMinimal openssh utillinux iproute ethtool thin-provisioning-tools iptables socat cri-tools containerd gzip ] ++ top.path;

         # Mildly hacky - by moving over to OCI image build infrastructure in
         # NixOS we should be able to get rid of the gunzip.
         # TODO(q3k): figure this out, check if this is even being used by
         # kubelet.
         preStart = ''
           ${concatMapStrings (img: ''
             echo "Seeding OCI image: ${img}"
             cp ${img} /tmp/image.tar.gz
             rm -f /tmp/image.tar
             gunzip /tmp/image.tar.gz
             ctr -n=k8s.io images import /tmp/image.tar || true
             rm /tmp/image.tar
           '') cfg.seedDockerImages}
         '';
         serviceConfig = {
           Slice = "kubernetes.slice";
           CPUAccounting = true;
           MemoryAccounting = true;
           Restart = "on-failure";
           RestartSec = "1000ms";
           ExecStart = ''${cfg.package}/bin/kubelet \
             --cgroup-driver=systemd \
             --container-runtime=remote \
             --container-runtime-endpoint=unix:///var/run/containerd/containerd.sock \
             --address=${cfg.address} \
             --authentication-token-webhook \
             --authentication-token-webhook-cache-ttl="10s" \
             --authorization-mode=Webhook \
             ${optionalString (cfg.clientCaFile != null)
               "--client-ca-file=${cfg.clientCaFile}"} \
             ${optionalString (cfg.clusterDns != "")
               "--cluster-dns=${cfg.clusterDns}"} \
             ${optionalString (cfg.clusterDomain != "")
               "--cluster-domain=${cfg.clusterDomain}"} \
             --cni-conf-dir=/opt/cni/conf \
             --cni-bin-dir=/opt/cni/bin \
             ${optionalString (cfg.featureGates != [])
               "--feature-gates=${concatMapStringsSep "," (feature: "${feature}=true") cfg.featureGates}"} \
             --hairpin-mode=hairpin-veth \
             --healthz-bind-address=${cfg.healthz.bind} \
             --healthz-port=${toString cfg.healthz.port} \
             --hostname-override=${cfg.hostname} \
             --kubeconfig=${kubeconfig} \
             ${optionalString (cfg.networkPlugin != null)
               "--network-plugin=${cfg.networkPlugin}"} \
             ${optionalString (cfg.nodeIp != null)
               "--node-ip=${cfg.nodeIp}"} \
             --pod-infra-container-image=pause \
             ${optionalString (cfg.manifests != {})
               "--pod-manifest-path=/etc/${manifestPath}"} \
             --port=${toString cfg.port} \
             --register-node=${boolToString cfg.registerNode} \
             ${optionalString (taints != "")
               "--register-with-taints=${taints}"} \
             --root-dir=${top.dataDir} \
             ${optionalString (cfg.tlsCertFile != null)
               "--tls-cert-file=${cfg.tlsCertFile}"} \
             ${optionalString (cfg.tlsKeyFile != null)
               "--tls-private-key-file=${cfg.tlsKeyFile}"} \
             ${optionalString (cfg.verbosity != null) "--v=${toString cfg.verbosity}"} \
             ${cfg.extraOpts}
           '';
           WorkingDirectory = top.dataDir;
         };
       };

       boot.kernelModules = [ "br_netfilter" "overlay" ];
       boot.kernel.sysctl = {
         "net.ipv4.ip_forward" = "1";
         "vm.max_map_count" = "262144"; # Needed for running things such as ElasticSearch.
       };

       services.kubernetes.kubelet.hostname = with config.networking;
         mkDefault (hostName + optionalString (domain != null) ".${domain}");

       services.kubernetes.pki.certs = with top.lib; {
         kubelet = mkCert {
           name = "kubelet";
           CN = top.kubelet.hostname;
           action = "systemctl restart kubelet.service";

         };
         kubeletClient = mkCert {
           name = "kubelet-client";
           CN = "system:node:${top.kubelet.hostname}";
           fields = {
             O = "system:nodes";
           };
           action = "systemctl restart kubelet.service";
         };
       };

       services.kubernetes.kubelet.kubeconfig.server = mkDefault top.apiserverAddress;
     })

     (mkIf (cfg.enable && cfg.manifests != {}) {
       environment.etc = mapAttrs' (name: manifest:
         nameValuePair "${manifestPath}/${name}.json" {
           text = builtins.toJSON manifest;
           mode = "0755";
         }
       ) cfg.manifests;
     })

     (mkIf (cfg.unschedulable && cfg.enable) {
       services.kubernetes.kubelet.taints.unschedulable = {
         value = "true";
         effect = "NoSchedule";
       };
     })

   ];
 }
	# Same as upstream kubelet.nix module from nixpkgs, but with the following
	# changes:
	# - cni tunables nuked and replaced with static host dirs, so that calico
	# running on k8s can drop CNI plugins there itself
	# - package configurable separately from rest of kubernetes

	{ config, lib, pkgs, ... }:

	with lib;

	let
	top = config.services.kubernetes;
	cfg = top.kubelet;

	infraContainer = pkgs.dockerTools.buildImage {
	name = "pause";
	tag = "latest";
	contents = top.package.pause;
	config.Cmd = ["/bin/pause"];
	};

	kubeconfig = top.lib.mkKubeConfig "kubelet" cfg.kubeconfig;

	manifestPath = "kubernetes/manifests";

	taintOptions = with lib.types; { name, ... }: {
	options = {
	key = mkOption {
	description = "Key of taint.";
	default = name;
	type = str;
	};
	value = mkOption {
	description = "Value of taint.";
	type = str;
	};
	effect = mkOption {
	description = "Effect of taint.";
	example = "NoSchedule";
	type = enum ["NoSchedule" "PreferNoSchedule" "NoExecute"];
	};
	};
	};

	taints = concatMapStringsSep "," (v: "${v.key}=${v.value}:${v.effect}") (mapAttrsToList (n: v: v) cfg.taints);
	in
	{
	# services/cluster/kubernetes/default.nix still wants to poke flannel,
	# but since we nuke that module we have to add a fake tunable for it.
	options.services.kubernetes.flannel = {
	enable = mkEnableOption "enable flannel networking";
	};

	###### interface
	options.services.kubernetes.kubelet = with lib.types; {

	address = mkOption {
	description = "Kubernetes kubelet info server listening address.";
	default = "0.0.0.0";
	type = str;
	};

	clusterDns = mkOption {
	description = "Use alternative DNS.";
	default = "10.1.0.1";
	type = str;
	};

	clusterDomain = mkOption {
	description = "Use alternative domain.";
	default = config.services.kubernetes.addons.dns.clusterDomain;
	type = str;
	};

	clientCaFile = mkOption {
	description = "Kubernetes apiserver CA file for client authentication.";
	default = top.caFile;
	type = nullOr path;
	};

	enable = mkEnableOption "Kubernetes kubelet.";

	extraOpts = mkOption {
	description = "Kubernetes kubelet extra command line options.";
	default = "";
	type = str;
	};

	featureGates = mkOption {
	description = "List set of feature gates";
	default = top.featureGates;
	type = listOf str;
	};

	healthz = {
	bind = mkOption {
	description = "Kubernetes kubelet healthz listening address.";
	default = "127.0.0.1";
	type = str;
	};

	port = mkOption {
	description = "Kubernetes kubelet healthz port.";
	default = 10248;
	type = int;
	};
	};

	hostname = mkOption {
	description = "Kubernetes kubelet hostname override.";
	default = config.networking.hostName;
	type = str;
	};

	kubeconfig = top.lib.mkKubeConfigOptions "Kubelet";

	manifests = mkOption {
	description = "List of manifests to bootstrap with kubelet (only pods can be created as manifest entry)";
	type = attrsOf attrs;
	default = {};
	};

	networkPlugin = mkOption {
	description = "Network plugin to use by Kubernetes.";
	type = nullOr (enum ["cni" "kubenet"]);
	default = "kubenet";
	};

	nodeIp = mkOption {
	description = "IP address of the node. If set, kubelet will use this IP address for the node.";
	default = null;
	type = nullOr str;
	};

	registerNode = mkOption {
	description = "Whether to auto register kubelet with API server.";
	default = true;
	type = bool;
	};

	package = mkOption {
	description = "Kubernetes package to use.";
	type = types.package;
	default = pkgs.kubernetes;
	defaultText = "pkgs.kubernetes";
	};

	port = mkOption {
	description = "Kubernetes kubelet info server listening port.";
	default = 10250;
	type = int;
	};

	seedDockerImages = mkOption {
	description = "List of docker images to preload on system";
	default = [];
	type = listOf package;
	};

	taints = mkOption {
	description = "Node taints (https://kubernetes.io/docs/concepts/configuration/assign-pod-node/).";
	default = {};
	type = attrsOf (submodule [ taintOptions ]);
	};

	tlsCertFile = mkOption {
	description = "File containing x509 Certificate for HTTPS.";
	default = null;
	type = nullOr path;
	};

	tlsKeyFile = mkOption {
	description = "File containing x509 private key matching tlsCertFile.";
	default = null;
	type = nullOr path;
	};

	unschedulable = mkOption {
	description = "Whether to set node taint to unschedulable=true as it is the case of node that has only master role.";
	default = false;
	type = bool;
	};

	verbosity = mkOption {
	description = ''
	Optional glog verbosity level for logging statements. See
	<link xlink:href="https://github.com/kubernetes/community/blob/master/contributors/devel/logging.md"/>
	'';
	default = null;
	type = nullOr int;
	};

	};

	###### implementation
	config = mkMerge [
	(mkIf cfg.enable {
	services.kubernetes.kubelet.seedDockerImages = [infraContainer];

	# Drop crictl into administrative command line.
	environment.systemPackages = with pkgs; [ cri-tools ];

	# Force disable Docker.
	virtualisation.docker.enable = false;

	# TODO(q3k): move to unified cgroups (cgroup v2) once we upgrade to
	# Kubelet 1.19.
	systemd.enableUnifiedCgroupHierarchy = false;

	# Run containerd service. This is exposes the CRI API that is consumed by
	# crictl and Kubelet.
	systemd.services.containerd = {
	description = "containerd container runtime";
	wantedBy = [ "kubernetes.target" ];
	after = [ "network.target" ];
	path = with pkgs; [ runc iptables ];
	serviceConfig = {
	Delegate = "yes";
	KillMode = "process";
	Restart = "always";
	RestartSec = "5";
	LimitNPROC = "infinity";
	LimitCORE = "infinity";
	# https://github.com/coreos/fedora-coreos-tracker/issues/329
	LimitNOFILE = "1048576";
	TasksMax = "infinity";
	OOMScoreAdjust = "-999";

	ExecStart = "${pkgs.containerd}/bin/containerd -c ${./containerd.toml}";
	};
	};

	systemd.services.kubelet = {
	description = "Kubernetes Kubelet Service";
	wantedBy = [ "kubernetes.target" ];
	after = [ "network.target" "containerd.service" "kube-apiserver.service" ];
	path = with pkgs; [ gitMinimal openssh utillinux iproute ethtool thin-provisioning-tools iptables socat cri-tools containerd gzip ] ++ top.path;

	# Mildly hacky - by moving over to OCI image build infrastructure in
	# NixOS we should be able to get rid of the gunzip.
	# TODO(q3k): figure this out, check if this is even being used by
	# kubelet.
	preStart = ''
	${concatMapStrings (img: ''
	echo "Seeding OCI image: ${img}"
	cp ${img} /tmp/image.tar.gz
	rm -f /tmp/image.tar
	gunzip /tmp/image.tar.gz
	ctr -n=k8s.io images import /tmp/image.tar \|\| true
	rm /tmp/image.tar
	'') cfg.seedDockerImages}
	'';
	serviceConfig = {
	Slice = "kubernetes.slice";
	CPUAccounting = true;
	MemoryAccounting = true;
	Restart = "on-failure";
	RestartSec = "1000ms";
	ExecStart = ''${cfg.package}/bin/kubelet \
	--cgroup-driver=systemd \
	--container-runtime=remote \
	--container-runtime-endpoint=unix:///var/run/containerd/containerd.sock \
	--address=${cfg.address} \
	--authentication-token-webhook \
	--authentication-token-webhook-cache-ttl="10s" \
	--authorization-mode=Webhook \
	${optionalString (cfg.clientCaFile != null)
	"--client-ca-file=${cfg.clientCaFile}"} \
	${optionalString (cfg.clusterDns != "")
	"--cluster-dns=${cfg.clusterDns}"} \
	${optionalString (cfg.clusterDomain != "")
	"--cluster-domain=${cfg.clusterDomain}"} \
	--cni-conf-dir=/opt/cni/conf \
	--cni-bin-dir=/opt/cni/bin \
	${optionalString (cfg.featureGates != [])
	"--feature-gates=${concatMapStringsSep "," (feature: "${feature}=true") cfg.featureGates}"} \
	--hairpin-mode=hairpin-veth \
	--healthz-bind-address=${cfg.healthz.bind} \
	--healthz-port=${toString cfg.healthz.port} \
	--hostname-override=${cfg.hostname} \
	--kubeconfig=${kubeconfig} \
	${optionalString (cfg.networkPlugin != null)
	"--network-plugin=${cfg.networkPlugin}"} \
	${optionalString (cfg.nodeIp != null)
	"--node-ip=${cfg.nodeIp}"} \
	--pod-infra-container-image=pause \
	${optionalString (cfg.manifests != {})
	"--pod-manifest-path=/etc/${manifestPath}"} \
	--port=${toString cfg.port} \
	--register-node=${boolToString cfg.registerNode} \
	${optionalString (taints != "")
	"--register-with-taints=${taints}"} \
	--root-dir=${top.dataDir} \
	${optionalString (cfg.tlsCertFile != null)
	"--tls-cert-file=${cfg.tlsCertFile}"} \
	${optionalString (cfg.tlsKeyFile != null)
	"--tls-private-key-file=${cfg.tlsKeyFile}"} \
	${optionalString (cfg.verbosity != null) "--v=${toString cfg.verbosity}"} \
	${cfg.extraOpts}
	'';
	WorkingDirectory = top.dataDir;
	};
	};

	boot.kernelModules = [ "br_netfilter" "overlay" ];
	boot.kernel.sysctl = {
	"net.ipv4.ip_forward" = "1";
	"vm.max_map_count" = "262144"; # Needed for running things such as ElasticSearch.
	};

	services.kubernetes.kubelet.hostname = with config.networking;
	mkDefault (hostName + optionalString (domain != null) ".${domain}");

	services.kubernetes.pki.certs = with top.lib; {
	kubelet = mkCert {
	name = "kubelet";
	CN = top.kubelet.hostname;
	action = "systemctl restart kubelet.service";

	};
	kubeletClient = mkCert {
	name = "kubelet-client";
	CN = "system:node:${top.kubelet.hostname}";
	fields = {
	O = "system:nodes";
	};
	action = "systemctl restart kubelet.service";
	};
	};

	services.kubernetes.kubelet.kubeconfig.server = mkDefault top.apiserverAddress;
	})

	(mkIf (cfg.enable && cfg.manifests != {}) {
	environment.etc = mapAttrs' (name: manifest:
	nameValuePair "${manifestPath}/${name}.json" {
	text = builtins.toJSON manifest;
	mode = "0755";
	}
	) cfg.manifests;
	})

	(mkIf (cfg.unschedulable && cfg.enable) {
	services.kubernetes.kubelet.taints.unschedulable = {
	value = "true";
	effect = "NoSchedule";
	};
	})

	];
	}