cluster/machines/modules/kube-dataplane.nix

{ config, pkgs, lib, machines, ... }:

with lib;

let
  # Pin for kubelet and proxy.
  k8spkgs = import (fetchGit {
    # Now at 1.16.5
    name = "nixos-unstable-2020-01-22";
    url = https://github.com/nixos/nixpkgs-channels/;
    rev = "a96ed5d70427bdc2fbb9e805784e1b9621157a98";
  }) {};

  cfg = config.hscloud.kube.data;

  # All control plane nodes.
  controlNodes = let
    list = mapAttrsToList (_: v: v) machines;
    filtered = filter (m: (m.config ? hscloud.kube.control) && (m.config.hscloud.kube.control.enable)) list;
    sorted = sort (a: b: a.config.hscloud.base.fqdn < b.config.hscloud.base.fqdn) filtered;
  in sorted;

  fqdn = config.hscloud.base.fqdn;

  pki = config.hscloud.kube.pki;

in {
  options.hscloud.kube.data = {
    enable = mkEnableOption "kubernetes data plane";
    podNet = mkOption {
      type = types.str;
      description = "Subnet in which this node will run pods. Must be exclusive with podNets of other nodes.";
    };
  };

  # Disable kubelet service and bring in our own override.
  # Also nuke flannel from the orbit.
  disabledModules = [
    "services/cluster/kubernetes/kubelet.nix"
    "services/cluster/kubernetes/flannel.nix"
  ];

  imports = [
    ./kubelet.nix
    ./kube-common.nix
  ];


  config = mkIf cfg.enable {
    # If we're not running the control plane, render a hostsfile that points at
    # all other control plane nodes. Otherwise, the control plane module will
    # make this hostsfile contain the node itself.
    networking.extraHosts = mkIf (!config.hscloud.kube.control.enable) (concatStringsSep "\n" (map
      (n: ''
        ${n.config.hscloud.base.mgmtIf} ${n.config.hscloud.base.fqdn}
      '')
    controlNodes));

    # this seems to depend on flannel
    # TODO(q3k): file issue
    systemd.services.kubelet-online = {
      script = pkgs.lib.mkForce "sleep 1";
    };

    services.kubernetes = {
      # The kubelet wants to mkfs.ext4 when mounting pvcs.
      path = [ pkgs.e2fsprogs ];

      proxy = {
        enable = true;
        kubeconfig = pki.kube.proxy.config;
        extraOpts = ''
          --hostname-override=${fqdn}\
          --proxy-mode=iptables
        '';
      };

      kubelet = {
        enable = true;
        unschedulable = false;
        hostname = fqdn;
        tlsCertFile = pki.kube.kubelet.cert;
        tlsKeyFile = pki.kube.kubelet.key;
        clientCaFile = pki.kube.kubelet.ca;
        nodeIp = config.hscloud.base.ipAddr;
        networkPlugin = "cni";
        clusterDns = "10.10.12.254";
        kubeconfig = pki.kube.kubelet.config;
        extraOpts = ''
          --read-only-port=0
        '';
        package = config.hscloud.kube.packageKubelet;
      };
    };
  };
}