cluster/nix: locally build nixos derivations
We change the existing behaviour (copy files & run nixos-rebuild switch)
to something closer to nixops-style. This now means that provisioning
admin machines need Nix installed locally, but that's probably an okay
choice to make.
The upside of this approach is that it's easier to debug and test
derivations, as all data is local to the repo and the workstation, and
deploying just means copying a configuration closure and switching the
system to it. At some point we should even be able to run the entire
cluster within a set of test VMs.
We also bump the kubernetes control plane to 1.14. Kubelets are still at
1.13 and their upgrade is comint up today too.
Change-Id: Ia9832c47f258ee223d93893d27946d1161cc4bbd
diff --git a/cluster/nix/module-kubelet.nix b/cluster/nix/module-kubelet.nix
new file mode 100644
index 0000000..f475b5b
--- /dev/null
+++ b/cluster/nix/module-kubelet.nix
@@ -0,0 +1,308 @@
+# Same as upstream kubelet.nix module from nixpkgs, but with the following
+# changes:
+# - cni tunables nuked and replaced with static host dirs, so that calico
+# running on k8s can drop CNI plugins there itself
+# - package configurable separately from rest of kubernetes
+
+{ config, lib, pkgs, ... }:
+
+with lib;
+
+let
+ top = config.services.kubernetes;
+ cfg = top.kubelet;
+
+ infraContainer = pkgs.dockerTools.buildImage {
+ name = "pause";
+ tag = "latest";
+ contents = top.package.pause;
+ config.Cmd = "/bin/pause";
+ };
+
+ kubeconfig = top.lib.mkKubeConfig "kubelet" cfg.kubeconfig;
+
+ manifestPath = "kubernetes/manifests";
+
+ taintOptions = with lib.types; { name, ... }: {
+ options = {
+ key = mkOption {
+ description = "Key of taint.";
+ default = name;
+ type = str;
+ };
+ value = mkOption {
+ description = "Value of taint.";
+ type = str;
+ };
+ effect = mkOption {
+ description = "Effect of taint.";
+ example = "NoSchedule";
+ type = enum ["NoSchedule" "PreferNoSchedule" "NoExecute"];
+ };
+ };
+ };
+
+ taints = concatMapStringsSep "," (v: "${v.key}=${v.value}:${v.effect}") (mapAttrsToList (n: v: v) cfg.taints);
+in
+{
+ imports = [
+ #(mkRemovedOptionModule [ "services" "kubernetes" "kubelet" "applyManifests" ] "")
+ #(mkRemovedOptionModule [ "services" "kubernetes" "kubelet" "cadvisorPort" ] "")
+ #(mkRemovedOptionModule [ "services" "kubernetes" "kubelet" "allowPrivileged" ] "")
+ ];
+
+ # services/cluster/kubernetes/default.nix still wants to poke flannel,
+ # but since we nuke that module we have to add a fake tunable for it.
+ options.services.kubernetes.flannel = {
+ enable = mkEnableOption "enable flannel networking";
+ };
+
+ ###### interface
+ options.services.kubernetes.kubelet = with lib.types; {
+
+ address = mkOption {
+ description = "Kubernetes kubelet info server listening address.";
+ default = "0.0.0.0";
+ type = str;
+ };
+
+ clusterDns = mkOption {
+ description = "Use alternative DNS.";
+ default = "10.1.0.1";
+ type = str;
+ };
+
+ clusterDomain = mkOption {
+ description = "Use alternative domain.";
+ default = config.services.kubernetes.addons.dns.clusterDomain;
+ type = str;
+ };
+
+ clientCaFile = mkOption {
+ description = "Kubernetes apiserver CA file for client authentication.";
+ default = top.caFile;
+ type = nullOr path;
+ };
+
+ enable = mkEnableOption "Kubernetes kubelet.";
+
+ extraOpts = mkOption {
+ description = "Kubernetes kubelet extra command line options.";
+ default = "";
+ type = str;
+ };
+
+ featureGates = mkOption {
+ description = "List set of feature gates";
+ default = top.featureGates;
+ type = listOf str;
+ };
+
+ healthz = {
+ bind = mkOption {
+ description = "Kubernetes kubelet healthz listening address.";
+ default = "127.0.0.1";
+ type = str;
+ };
+
+ port = mkOption {
+ description = "Kubernetes kubelet healthz port.";
+ default = 10248;
+ type = int;
+ };
+ };
+
+ hostname = mkOption {
+ description = "Kubernetes kubelet hostname override.";
+ default = config.networking.hostName;
+ type = str;
+ };
+
+ kubeconfig = top.lib.mkKubeConfigOptions "Kubelet";
+
+ manifests = mkOption {
+ description = "List of manifests to bootstrap with kubelet (only pods can be created as manifest entry)";
+ type = attrsOf attrs;
+ default = {};
+ };
+
+ networkPlugin = mkOption {
+ description = "Network plugin to use by Kubernetes.";
+ type = nullOr (enum ["cni" "kubenet"]);
+ default = "kubenet";
+ };
+
+ nodeIp = mkOption {
+ description = "IP address of the node. If set, kubelet will use this IP address for the node.";
+ default = null;
+ type = nullOr str;
+ };
+
+ registerNode = mkOption {
+ description = "Whether to auto register kubelet with API server.";
+ default = true;
+ type = bool;
+ };
+
+ package = mkOption {
+ description = "Kubernetes package to use.";
+ type = types.package;
+ default = pkgs.kubernetes;
+ defaultText = "pkgs.kubernetes";
+ };
+
+ port = mkOption {
+ description = "Kubernetes kubelet info server listening port.";
+ default = 10250;
+ type = int;
+ };
+
+ seedDockerImages = mkOption {
+ description = "List of docker images to preload on system";
+ default = [];
+ type = listOf package;
+ };
+
+ taints = mkOption {
+ description = "Node taints (https://kubernetes.io/docs/concepts/configuration/assign-pod-node/).";
+ default = {};
+ type = attrsOf (submodule [ taintOptions ]);
+ };
+
+ tlsCertFile = mkOption {
+ description = "File containing x509 Certificate for HTTPS.";
+ default = null;
+ type = nullOr path;
+ };
+
+ tlsKeyFile = mkOption {
+ description = "File containing x509 private key matching tlsCertFile.";
+ default = null;
+ type = nullOr path;
+ };
+
+ unschedulable = mkOption {
+ description = "Whether to set node taint to unschedulable=true as it is the case of node that has only master role.";
+ default = false;
+ type = bool;
+ };
+
+ verbosity = mkOption {
+ description = ''
+ Optional glog verbosity level for logging statements. See
+ <link xlink:href="https://github.com/kubernetes/community/blob/master/contributors/devel/logging.md"/>
+ '';
+ default = null;
+ type = nullOr int;
+ };
+
+ };
+
+ ###### implementation
+ config = mkMerge [
+ (mkIf cfg.enable {
+ services.kubernetes.kubelet.seedDockerImages = [infraContainer];
+
+ systemd.services.kubelet = {
+ description = "Kubernetes Kubelet Service";
+ wantedBy = [ "kubernetes.target" ];
+ after = [ "network.target" "docker.service" "kube-apiserver.service" ];
+ path = with pkgs; [ gitMinimal openssh docker utillinux iproute ethtool thin-provisioning-tools iptables socat ] ++ top.path;
+ preStart = ''
+ ${concatMapStrings (img: ''
+ echo "Seeding docker image: ${img}"
+ docker load <${img}
+ '') cfg.seedDockerImages}
+ '';
+ serviceConfig = {
+ Slice = "kubernetes.slice";
+ CPUAccounting = true;
+ MemoryAccounting = true;
+ Restart = "on-failure";
+ RestartSec = "1000ms";
+ ExecStart = ''${cfg.package}/bin/kubelet \
+ --address=${cfg.address} \
+ --authentication-token-webhook \
+ --authentication-token-webhook-cache-ttl="10s" \
+ --authorization-mode=Webhook \
+ ${optionalString (cfg.clientCaFile != null)
+ "--client-ca-file=${cfg.clientCaFile}"} \
+ ${optionalString (cfg.clusterDns != "")
+ "--cluster-dns=${cfg.clusterDns}"} \
+ ${optionalString (cfg.clusterDomain != "")
+ "--cluster-domain=${cfg.clusterDomain}"} \
+ --cni-conf-dir=/opt/cni/conf \
+ --cni-bin-dir=/opt/cni/bin \
+ ${optionalString (cfg.featureGates != [])
+ "--feature-gates=${concatMapStringsSep "," (feature: "${feature}=true") cfg.featureGates}"} \
+ --hairpin-mode=hairpin-veth \
+ --healthz-bind-address=${cfg.healthz.bind} \
+ --healthz-port=${toString cfg.healthz.port} \
+ --hostname-override=${cfg.hostname} \
+ --kubeconfig=${kubeconfig} \
+ ${optionalString (cfg.networkPlugin != null)
+ "--network-plugin=${cfg.networkPlugin}"} \
+ ${optionalString (cfg.nodeIp != null)
+ "--node-ip=${cfg.nodeIp}"} \
+ --pod-infra-container-image=pause \
+ ${optionalString (cfg.manifests != {})
+ "--pod-manifest-path=/etc/${manifestPath}"} \
+ --port=${toString cfg.port} \
+ --register-node=${boolToString cfg.registerNode} \
+ ${optionalString (taints != "")
+ "--register-with-taints=${taints}"} \
+ --root-dir=${top.dataDir} \
+ ${optionalString (cfg.tlsCertFile != null)
+ "--tls-cert-file=${cfg.tlsCertFile}"} \
+ ${optionalString (cfg.tlsKeyFile != null)
+ "--tls-private-key-file=${cfg.tlsKeyFile}"} \
+ ${optionalString (cfg.verbosity != null) "--v=${toString cfg.verbosity}"} \
+ ${cfg.extraOpts}
+ '';
+ WorkingDirectory = top.dataDir;
+ };
+ };
+
+ boot.kernelModules = ["br_netfilter"];
+
+ services.kubernetes.kubelet.hostname = with config.networking;
+ mkDefault (hostName + optionalString (domain != null) ".${domain}");
+
+ services.kubernetes.pki.certs = with top.lib; {
+ kubelet = mkCert {
+ name = "kubelet";
+ CN = top.kubelet.hostname;
+ action = "systemctl restart kubelet.service";
+
+ };
+ kubeletClient = mkCert {
+ name = "kubelet-client";
+ CN = "system:node:${top.kubelet.hostname}";
+ fields = {
+ O = "system:nodes";
+ };
+ action = "systemctl restart kubelet.service";
+ };
+ };
+
+ services.kubernetes.kubelet.kubeconfig.server = mkDefault top.apiserverAddress;
+ })
+
+ (mkIf (cfg.enable && cfg.manifests != {}) {
+ environment.etc = mapAttrs' (name: manifest:
+ nameValuePair "${manifestPath}/${name}.json" {
+ text = builtins.toJSON manifest;
+ mode = "0755";
+ }
+ ) cfg.manifests;
+ })
+
+ (mkIf (cfg.unschedulable && cfg.enable) {
+ services.kubernetes.kubelet.taints.unschedulable = {
+ value = "true";
+ effect = "NoSchedule";
+ };
+ })
+
+ ];
+}