| # Same as upstream kubelet.nix module from nixpkgs, but with the following |
| # changes: |
| # - cni tunables nuked and replaced with static host dirs, so that calico |
| # running on k8s can drop CNI plugins there itself |
| # - package configurable separately from rest of kubernetes |
| |
| { config, lib, pkgs, ... }: |
| |
| with lib; |
| |
| let |
| top = config.services.kubernetes; |
| cfg = top.kubelet; |
| |
| infraContainer = pkgs.dockerTools.buildImage { |
| name = "pause"; |
| tag = "latest"; |
| contents = top.package.pause; |
| config.Cmd = "/bin/pause"; |
| }; |
| |
| kubeconfig = top.lib.mkKubeConfig "kubelet" cfg.kubeconfig; |
| |
| manifestPath = "kubernetes/manifests"; |
| |
| taintOptions = with lib.types; { name, ... }: { |
| options = { |
| key = mkOption { |
| description = "Key of taint."; |
| default = name; |
| type = str; |
| }; |
| value = mkOption { |
| description = "Value of taint."; |
| type = str; |
| }; |
| effect = mkOption { |
| description = "Effect of taint."; |
| example = "NoSchedule"; |
| type = enum ["NoSchedule" "PreferNoSchedule" "NoExecute"]; |
| }; |
| }; |
| }; |
| |
| taints = concatMapStringsSep "," (v: "${v.key}=${v.value}:${v.effect}") (mapAttrsToList (n: v: v) cfg.taints); |
| in |
| { |
| imports = [ |
| #(mkRemovedOptionModule [ "services" "kubernetes" "kubelet" "applyManifests" ] "") |
| #(mkRemovedOptionModule [ "services" "kubernetes" "kubelet" "cadvisorPort" ] "") |
| #(mkRemovedOptionModule [ "services" "kubernetes" "kubelet" "allowPrivileged" ] "") |
| ]; |
| |
| # services/cluster/kubernetes/default.nix still wants to poke flannel, |
| # but since we nuke that module we have to add a fake tunable for it. |
| options.services.kubernetes.flannel = { |
| enable = mkEnableOption "enable flannel networking"; |
| }; |
| |
| ###### interface |
| options.services.kubernetes.kubelet = with lib.types; { |
| |
| address = mkOption { |
| description = "Kubernetes kubelet info server listening address."; |
| default = "0.0.0.0"; |
| type = str; |
| }; |
| |
| clusterDns = mkOption { |
| description = "Use alternative DNS."; |
| default = "10.1.0.1"; |
| type = str; |
| }; |
| |
| clusterDomain = mkOption { |
| description = "Use alternative domain."; |
| default = config.services.kubernetes.addons.dns.clusterDomain; |
| type = str; |
| }; |
| |
| clientCaFile = mkOption { |
| description = "Kubernetes apiserver CA file for client authentication."; |
| default = top.caFile; |
| type = nullOr path; |
| }; |
| |
| enable = mkEnableOption "Kubernetes kubelet."; |
| |
| extraOpts = mkOption { |
| description = "Kubernetes kubelet extra command line options."; |
| default = ""; |
| type = str; |
| }; |
| |
| featureGates = mkOption { |
| description = "List set of feature gates"; |
| default = top.featureGates; |
| type = listOf str; |
| }; |
| |
| healthz = { |
| bind = mkOption { |
| description = "Kubernetes kubelet healthz listening address."; |
| default = "127.0.0.1"; |
| type = str; |
| }; |
| |
| port = mkOption { |
| description = "Kubernetes kubelet healthz port."; |
| default = 10248; |
| type = int; |
| }; |
| }; |
| |
| hostname = mkOption { |
| description = "Kubernetes kubelet hostname override."; |
| default = config.networking.hostName; |
| type = str; |
| }; |
| |
| kubeconfig = top.lib.mkKubeConfigOptions "Kubelet"; |
| |
| manifests = mkOption { |
| description = "List of manifests to bootstrap with kubelet (only pods can be created as manifest entry)"; |
| type = attrsOf attrs; |
| default = {}; |
| }; |
| |
| networkPlugin = mkOption { |
| description = "Network plugin to use by Kubernetes."; |
| type = nullOr (enum ["cni" "kubenet"]); |
| default = "kubenet"; |
| }; |
| |
| nodeIp = mkOption { |
| description = "IP address of the node. If set, kubelet will use this IP address for the node."; |
| default = null; |
| type = nullOr str; |
| }; |
| |
| registerNode = mkOption { |
| description = "Whether to auto register kubelet with API server."; |
| default = true; |
| type = bool; |
| }; |
| |
| package = mkOption { |
| description = "Kubernetes package to use."; |
| type = types.package; |
| default = pkgs.kubernetes; |
| defaultText = "pkgs.kubernetes"; |
| }; |
| |
| port = mkOption { |
| description = "Kubernetes kubelet info server listening port."; |
| default = 10250; |
| type = int; |
| }; |
| |
| seedDockerImages = mkOption { |
| description = "List of docker images to preload on system"; |
| default = []; |
| type = listOf package; |
| }; |
| |
| taints = mkOption { |
| description = "Node taints (https://kubernetes.io/docs/concepts/configuration/assign-pod-node/)."; |
| default = {}; |
| type = attrsOf (submodule [ taintOptions ]); |
| }; |
| |
| tlsCertFile = mkOption { |
| description = "File containing x509 Certificate for HTTPS."; |
| default = null; |
| type = nullOr path; |
| }; |
| |
| tlsKeyFile = mkOption { |
| description = "File containing x509 private key matching tlsCertFile."; |
| default = null; |
| type = nullOr path; |
| }; |
| |
| unschedulable = mkOption { |
| description = "Whether to set node taint to unschedulable=true as it is the case of node that has only master role."; |
| default = false; |
| type = bool; |
| }; |
| |
| verbosity = mkOption { |
| description = '' |
| Optional glog verbosity level for logging statements. See |
| <link xlink:href="https://github.com/kubernetes/community/blob/master/contributors/devel/logging.md"/> |
| ''; |
| default = null; |
| type = nullOr int; |
| }; |
| |
| }; |
| |
| ###### implementation |
| config = mkMerge [ |
| (mkIf cfg.enable { |
| services.kubernetes.kubelet.seedDockerImages = [infraContainer]; |
| |
| systemd.services.kubelet = { |
| description = "Kubernetes Kubelet Service"; |
| wantedBy = [ "kubernetes.target" ]; |
| after = [ "network.target" "docker.service" "kube-apiserver.service" ]; |
| path = with pkgs; [ gitMinimal openssh docker utillinux iproute ethtool thin-provisioning-tools iptables socat ] ++ top.path; |
| preStart = '' |
| ${concatMapStrings (img: '' |
| echo "Seeding docker image: ${img}" |
| docker load <${img} |
| '') cfg.seedDockerImages} |
| ''; |
| serviceConfig = { |
| Slice = "kubernetes.slice"; |
| CPUAccounting = true; |
| MemoryAccounting = true; |
| Restart = "on-failure"; |
| RestartSec = "1000ms"; |
| ExecStart = ''${cfg.package}/bin/kubelet \ |
| --address=${cfg.address} \ |
| --authentication-token-webhook \ |
| --authentication-token-webhook-cache-ttl="10s" \ |
| --authorization-mode=Webhook \ |
| ${optionalString (cfg.clientCaFile != null) |
| "--client-ca-file=${cfg.clientCaFile}"} \ |
| ${optionalString (cfg.clusterDns != "") |
| "--cluster-dns=${cfg.clusterDns}"} \ |
| ${optionalString (cfg.clusterDomain != "") |
| "--cluster-domain=${cfg.clusterDomain}"} \ |
| --cni-conf-dir=/opt/cni/conf \ |
| --cni-bin-dir=/opt/cni/bin \ |
| ${optionalString (cfg.featureGates != []) |
| "--feature-gates=${concatMapStringsSep "," (feature: "${feature}=true") cfg.featureGates}"} \ |
| --hairpin-mode=hairpin-veth \ |
| --healthz-bind-address=${cfg.healthz.bind} \ |
| --healthz-port=${toString cfg.healthz.port} \ |
| --hostname-override=${cfg.hostname} \ |
| --kubeconfig=${kubeconfig} \ |
| ${optionalString (cfg.networkPlugin != null) |
| "--network-plugin=${cfg.networkPlugin}"} \ |
| ${optionalString (cfg.nodeIp != null) |
| "--node-ip=${cfg.nodeIp}"} \ |
| --pod-infra-container-image=pause \ |
| ${optionalString (cfg.manifests != {}) |
| "--pod-manifest-path=/etc/${manifestPath}"} \ |
| --port=${toString cfg.port} \ |
| --register-node=${boolToString cfg.registerNode} \ |
| ${optionalString (taints != "") |
| "--register-with-taints=${taints}"} \ |
| --root-dir=${top.dataDir} \ |
| ${optionalString (cfg.tlsCertFile != null) |
| "--tls-cert-file=${cfg.tlsCertFile}"} \ |
| ${optionalString (cfg.tlsKeyFile != null) |
| "--tls-private-key-file=${cfg.tlsKeyFile}"} \ |
| ${optionalString (cfg.verbosity != null) "--v=${toString cfg.verbosity}"} \ |
| ${cfg.extraOpts} |
| ''; |
| WorkingDirectory = top.dataDir; |
| }; |
| }; |
| |
| boot.kernelModules = ["br_netfilter"]; |
| |
| services.kubernetes.kubelet.hostname = with config.networking; |
| mkDefault (hostName + optionalString (domain != null) ".${domain}"); |
| |
| services.kubernetes.pki.certs = with top.lib; { |
| kubelet = mkCert { |
| name = "kubelet"; |
| CN = top.kubelet.hostname; |
| action = "systemctl restart kubelet.service"; |
| |
| }; |
| kubeletClient = mkCert { |
| name = "kubelet-client"; |
| CN = "system:node:${top.kubelet.hostname}"; |
| fields = { |
| O = "system:nodes"; |
| }; |
| action = "systemctl restart kubelet.service"; |
| }; |
| }; |
| |
| services.kubernetes.kubelet.kubeconfig.server = mkDefault top.apiserverAddress; |
| }) |
| |
| (mkIf (cfg.enable && cfg.manifests != {}) { |
| environment.etc = mapAttrs' (name: manifest: |
| nameValuePair "${manifestPath}/${name}.json" { |
| text = builtins.toJSON manifest; |
| mode = "0755"; |
| } |
| ) cfg.manifests; |
| }) |
| |
| (mkIf (cfg.unschedulable && cfg.enable) { |
| services.kubernetes.kubelet.taints.unschedulable = { |
| value = "true"; |
| effect = "NoSchedule"; |
| }; |
| }) |
| |
| ]; |
| } |