cluster/machines/modules/kubelet.nix

# Same as upstream kubelet.nix module from nixpkgs, but with the following
# changes:
#   - cni tunables nuked and replaced with static host dirs, so that calico
#     running on k8s can drop CNI plugins there itself
#   - package configurable separately from rest of kubernetes

{ config, lib, pkgs, ... }:

with lib;

let
  top = config.services.kubernetes;
  cfg = top.kubelet;

  infraContainer = pkgs.dockerTools.buildImage {
    name = "pause";
    tag = "latest";
    contents = top.package.pause;
    config.Cmd = ["/bin/pause"];
  };

  kubeconfig = top.lib.mkKubeConfig "kubelet" cfg.kubeconfig;

  manifestPath = "kubernetes/manifests";

  taintOptions = with lib.types; { name, ... }: {
    options = {
      key = mkOption {
        description = "Key of taint.";
        default = name;
        type = str;
      };
      value = mkOption {
        description = "Value of taint.";
        type = str;
      };
      effect = mkOption {
        description = "Effect of taint.";
        example = "NoSchedule";
        type = enum ["NoSchedule" "PreferNoSchedule" "NoExecute"];
      };
    };
  };

  taints = concatMapStringsSep "," (v: "${v.key}=${v.value}:${v.effect}") (mapAttrsToList (n: v: v) cfg.taints);
in
{
  # services/cluster/kubernetes/default.nix still wants to poke flannel,
  # but since we nuke that module we have to add a fake tunable for it.
  options.services.kubernetes.flannel = {
    enable = mkEnableOption "enable flannel networking";
  };

  ###### interface
  options.services.kubernetes.kubelet = with lib.types; {

    address = mkOption {
      description = "Kubernetes kubelet info server listening address.";
      default = "0.0.0.0";
      type = str;
    };

    clusterDns = mkOption {
      description = "Use alternative DNS.";
      default = "10.1.0.1";
      type = str;
    };

    clusterDomain = mkOption {
      description = "Use alternative domain.";
      default = config.services.kubernetes.addons.dns.clusterDomain;
      type = str;
    };

    clientCaFile = mkOption {
      description = "Kubernetes apiserver CA file for client authentication.";
      default = top.caFile;
      type = nullOr path;
    };

    enable = mkEnableOption "Kubernetes kubelet.";

    extraOpts = mkOption {
      description = "Kubernetes kubelet extra command line options.";
      default = "";
      type = str;
    };

    featureGates = mkOption {
      description = "List set of feature gates";
      default = top.featureGates;
      type = listOf str;
    };

    healthz = {
      bind = mkOption {
        description = "Kubernetes kubelet healthz listening address.";
        default = "127.0.0.1";
        type = str;
      };

      port = mkOption {
        description = "Kubernetes kubelet healthz port.";
        default = 10248;
        type = int;
      };
    };

    hostname = mkOption {
      description = "Kubernetes kubelet hostname override.";
      default = config.networking.hostName;
      type = str;
    };

    kubeconfig = top.lib.mkKubeConfigOptions "Kubelet";

    manifests = mkOption {
      description = "List of manifests to bootstrap with kubelet (only pods can be created as manifest entry)";
      type = attrsOf attrs;
      default = {};
    };

    networkPlugin = mkOption {
      description = "Network plugin to use by Kubernetes.";
      type = nullOr (enum ["cni" "kubenet"]);
      default = "kubenet";
    };

    nodeIp = mkOption {
      description = "IP address of the node. If set, kubelet will use this IP address for the node.";
      default = null;
      type = nullOr str;
    };

    registerNode = mkOption {
      description = "Whether to auto register kubelet with API server.";
      default = true;
      type = bool;
    };

    package = mkOption {
      description = "Kubernetes package to use.";
      type = types.package;
      default = pkgs.kubernetes;
      defaultText = "pkgs.kubernetes";
    };

    port = mkOption {
      description = "Kubernetes kubelet info server listening port.";
      default = 10250;
      type = int;
    };

    seedDockerImages = mkOption {
      description = "List of docker images to preload on system";
      default = [];
      type = listOf package;
    };

    taints = mkOption {
      description = "Node taints (https://kubernetes.io/docs/concepts/configuration/assign-pod-node/).";
      default = {};
      type = attrsOf (submodule [ taintOptions ]);
    };

    tlsCertFile = mkOption {
      description = "File containing x509 Certificate for HTTPS.";
      default = null;
      type = nullOr path;
    };

    tlsKeyFile = mkOption {
      description = "File containing x509 private key matching tlsCertFile.";
      default = null;
      type = nullOr path;
    };

    unschedulable = mkOption {
      description = "Whether to set node taint to unschedulable=true as it is the case of node that has only master role.";
      default = false;
      type = bool;
    };

    verbosity = mkOption {
      description = ''
        Optional glog verbosity level for logging statements. See
        <link xlink:href="https://github.com/kubernetes/community/blob/master/contributors/devel/logging.md"/>
      '';
      default = null;
      type = nullOr int;
    };

  };

  ###### implementation
  config = mkMerge [
    (mkIf cfg.enable {
      services.kubernetes.kubelet.seedDockerImages = [infraContainer];

      # Drop crictl into administrative command line.
      environment.systemPackages = with pkgs; [ cri-tools ];

      # Force disable Docker.
      virtualisation.docker.enable = false;

      # TODO(q3k): move to unified cgroups (cgroup v2) once we upgrade to
      # Kubelet 1.19.
      systemd.enableUnifiedCgroupHierarchy = false;

      # Run containerd service. This is exposes the CRI API that is consumed by
      # crictl and Kubelet.
      systemd.services.containerd = {
        description = "containerd container runtime";
        wantedBy = [ "kubernetes.target" ];
        after = [ "network.target" ];
        path = with pkgs; [ runc iptables ];
        serviceConfig = {
          Delegate = "yes";
          KillMode = "process";
          Restart = "always";
          RestartSec = "5";
          LimitNPROC = "infinity";
          LimitCORE = "infinity";
          # https://github.com/coreos/fedora-coreos-tracker/issues/329
          LimitNOFILE = "1048576";
          TasksMax = "infinity";
          OOMScoreAdjust = "-999";

          ExecStart = "${pkgs.containerd}/bin/containerd -c ${./containerd.toml}";
        };
      };

      systemd.services.kubelet = {
        description = "Kubernetes Kubelet Service";
        wantedBy = [ "kubernetes.target" ];
        after = [ "network.target" "containerd.service" "kube-apiserver.service" ];
        path = with pkgs; [ gitMinimal openssh utillinux iproute ethtool thin-provisioning-tools iptables socat cri-tools containerd gzip ] ++ top.path;

        # Mildly hacky - by moving over to OCI image build infrastructure in
        # NixOS we should be able to get rid of the gunzip.
        # TODO(q3k): figure this out, check if this is even being used by
        # kubelet.
        preStart = ''
          ${concatMapStrings (img: ''
            echo "Seeding OCI image: ${img}"
            cp ${img} /tmp/image.tar.gz
            rm -f /tmp/image.tar
            gunzip /tmp/image.tar.gz
            ctr -n=k8s.io images import /tmp/image.tar || true
            rm /tmp/image.tar
          '') cfg.seedDockerImages}
        '';
        serviceConfig = {
          Slice = "kubernetes.slice";
          CPUAccounting = true;
          MemoryAccounting = true;
          Restart = "on-failure";
          RestartSec = "1000ms";
          ExecStart = ''${cfg.package}/bin/kubelet \
            --cgroup-driver=systemd \
            --container-runtime=remote \
            --container-runtime-endpoint=unix:///var/run/containerd/containerd.sock \
            --address=${cfg.address} \
            --authentication-token-webhook \
            --authentication-token-webhook-cache-ttl="10s" \
            --authorization-mode=Webhook \
            ${optionalString (cfg.clientCaFile != null)
              "--client-ca-file=${cfg.clientCaFile}"} \
            ${optionalString (cfg.clusterDns != "")
              "--cluster-dns=${cfg.clusterDns}"} \
            ${optionalString (cfg.clusterDomain != "")
              "--cluster-domain=${cfg.clusterDomain}"} \
            --cni-conf-dir=/opt/cni/conf \
            --cni-bin-dir=/opt/cni/bin \
            ${optionalString (cfg.featureGates != [])
              "--feature-gates=${concatMapStringsSep "," (feature: "${feature}=true") cfg.featureGates}"} \
            --hairpin-mode=hairpin-veth \
            --healthz-bind-address=${cfg.healthz.bind} \
            --healthz-port=${toString cfg.healthz.port} \
            --hostname-override=${cfg.hostname} \
            --kubeconfig=${kubeconfig} \
            ${optionalString (cfg.networkPlugin != null)
              "--network-plugin=${cfg.networkPlugin}"} \
            ${optionalString (cfg.nodeIp != null)
              "--node-ip=${cfg.nodeIp}"} \
            --pod-infra-container-image=pause \
            ${optionalString (cfg.manifests != {})
              "--pod-manifest-path=/etc/${manifestPath}"} \
            --port=${toString cfg.port} \
            --register-node=${boolToString cfg.registerNode} \
            ${optionalString (taints != "")
              "--register-with-taints=${taints}"} \
            --root-dir=${top.dataDir} \
            ${optionalString (cfg.tlsCertFile != null)
              "--tls-cert-file=${cfg.tlsCertFile}"} \
            ${optionalString (cfg.tlsKeyFile != null)
              "--tls-private-key-file=${cfg.tlsKeyFile}"} \
            ${optionalString (cfg.verbosity != null) "--v=${toString cfg.verbosity}"} \
            ${cfg.extraOpts}
          '';
          WorkingDirectory = top.dataDir;
        };
      };

      boot.kernelModules = [ "br_netfilter" "overlay" ];
      boot.kernel.sysctl = {
        "net.ipv4.ip_forward" = "1";
        "vm.max_map_count" = "262144"; # Needed for running things such as ElasticSearch.
      };

      services.kubernetes.kubelet.hostname = with config.networking;
        mkDefault (hostName + optionalString (domain != null) ".${domain}");

      services.kubernetes.pki.certs = with top.lib; {
        kubelet = mkCert {
          name = "kubelet";
          CN = top.kubelet.hostname;
          action = "systemctl restart kubelet.service";

        };
        kubeletClient = mkCert {
          name = "kubelet-client";
          CN = "system:node:${top.kubelet.hostname}";
          fields = {
            O = "system:nodes";
          };
          action = "systemctl restart kubelet.service";
        };
      };

      services.kubernetes.kubelet.kubeconfig.server = mkDefault top.apiserverAddress;
    })

    (mkIf (cfg.enable && cfg.manifests != {}) {
      environment.etc = mapAttrs' (name: manifest:
        nameValuePair "${manifestPath}/${name}.json" {
          text = builtins.toJSON manifest;
          mode = "0755";
        }
      ) cfg.manifests;
    })

    (mkIf (cfg.unschedulable && cfg.enable) {
      services.kubernetes.kubelet.taints.unschedulable = {
        value = "true";
        effect = "NoSchedule";
      };
    })

  ];
}