nix/ -> cluster/nix/
These are related to cluster bootstrapping, not generic language
libraries (like go/ and bzl/).
Change-Id: I03a83c64f3e0fa6cb615d36b4e618f5e92d886ec
diff --git a/cluster/clustercfg/clustercfg.py b/cluster/clustercfg/clustercfg.py
index 41614c7..24fa745 100644
--- a/cluster/clustercfg/clustercfg.py
+++ b/cluster/clustercfg/clustercfg.py
@@ -44,7 +44,7 @@
def pki_config(key):
raw = subprocess.check_output([
'nix', 'eval', '--raw',
- '( (import ' + local_root + '/nix/toplevel.nix ).pki.' + key + '.json )',
+ '( (import ' + local_root + '/cluster/nix/toplevel.nix ).pki.' + key + '.json )',
])
return json.loads(raw)
@@ -203,7 +203,7 @@
# Upload NixOS config
for f in ['toplevel', 'cluster-configuration']:
- r.put(local=os.path.join(local_root, 'nix/{}.nix'.format(f)),
+ r.put(local=os.path.join(local_root, 'cluster/nix/{}.nix'.format(f)),
remote='/etc/nixos/{}.nix'.format(f))
r.run('nixos-rebuild switch')
@@ -228,8 +228,10 @@
elif mode == "config":
print('etcd peer:')
print(json.dumps(pki_config('etcdPeer'), indent=2))
- print('etcd client:')
- print(json.dumps(pki_config('etcdClient'), indent=2))
+ print('etcd server:')
+ print(json.dumps(pki_config('etcd.server'), indent=2))
+ print('etcd client (kube):')
+ print(json.dumps(pki_config('etcd.kube'), indent=2))
else:
usage()
return 1
diff --git a/cluster/nix/cluster-configuration.nix b/cluster/nix/cluster-configuration.nix
new file mode 100644
index 0000000..a3601bd
--- /dev/null
+++ b/cluster/nix/cluster-configuration.nix
@@ -0,0 +1,245 @@
+{ config, pkgs, lib, ... }:
+
+with ( import ./toplevel.nix );
+let
+ fqdn = config.networking.hostName + domain;
+ node = (builtins.head (builtins.filter (n: n.fqdn == fqdn) nodes));
+ otherNodes = (builtins.filter (n: n.fqdn != fqdn) nodes);
+
+ # Pin for k8s packages. This is so that upagrading the system will not upgrade the k8s control or data planes.
+ k8spkgs = import (fetchGit {
+ name = "nixos-unstable-2019-04-12";
+ url = https://github.com/nixos/nixpkgs/;
+ rev = "1fc591f9a5bd1b016b5d66dfab29560073955a14";
+ }) {};
+
+
+in rec {
+ imports =
+ [ # Include the results of the hardware scan.
+ ./hardware-configuration.nix
+ ];
+
+ # Use the GRUB 2 boot loader.
+ boot.loader.grub.enable = true;
+ boot.loader.grub.version = 2;
+ boot.loader.grub.device = node.diskBoot;
+
+ boot.kernelPackages = pkgs.linuxPackages_5_1;
+ boot.kernelParams = [ "boot.shell_on_fail" ];
+
+ time.timeZone = "Europe/Warsaw";
+
+ # List packages installed in system profile. To search, run:
+ # $ nix search wget
+ environment.systemPackages = with pkgs; [
+ wget vim htop tcpdump
+ rxvt_unicode.terminfo
+ ];
+
+ # Some programs need SUID wrappers, can be configured further or are
+ # started in user sessions.
+ programs.mtr.enable = true;
+
+ # List services that you want to enable:
+ virtualisation.docker.enable = true;
+ virtualisation.docker.extraOptions = "--iptables=false --ip-masq=false --ip-forward=true";
+
+ # Docker 1.13 sets iptables FORWARD to DROP. Unfuck this.
+ systemd.services."docker-iptables-unfuck" = {
+ enable = true;
+ wantedBy = [ "kubernetes.target" ];
+ description = "Docker iptable Unfuck";
+ after = [ "docker.service" ];
+ requires = [ "docker.service" ];
+ path = [ pkgs.iptables ];
+ script = ''
+ iptables -P FORWARD ACCEPT
+ '';
+ serviceConfig.Type = "oneshot";
+ };
+ # Otherwise fetchGit nixpkgs pin fails.
+ systemd.services.nixos-upgrade.path = [ pkgs.git ];
+
+ # Enable the OpenSSH daemon.
+ services.openssh.enable = true;
+ users.users.root.openssh.authorizedKeys.keys = [
+ "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDD4VJXAXEHEXZk2dxNwehneuJcEGkfXG/U7z4fO79vDVIENdedtXQUyLyhZJc5RTEfHhQj66FwIqzl7mzBHd9x9PuDp6QAYXrkVNMj48s6JXqZqBvF6H/weRqFMf4a2TZv+hG8D0kpvmLheCwWAVRls7Jofnp/My+yDd57GMdsbG/yFEf6WPMiOnA7hxdSJSVihCsCSw2p8PD4GhBe8CVt7xIuinhutjm9zYBjV78NT8acjDUfJh0B1ODTjs7nuW1CC4jybSe2j/OU3Yczj4AxRxBNWuFxUq+jBo9BfpbKLh+Tt7re+zBkaicM77KM/oV6943JJxgHNBBOsv9scZE7 q3k@amnesia"
+ "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDQb3YQoiYFZLKwvHYKbu1bMqzNeDCAszQhAe1+QI5SLDOotclyY/vFmOReZOsmyMFl71G2d7d+FbYNusUnNNjTxRYQ021tVc+RkMdLJaORRURmQfEFEKbai6QSFTwErXzuoIzyEPK0lbsQuGgqT9WaVnRzHJ2Q/4+qQbxAS34PuR5NqEkmn4G6LMo3OyJ5mwPkCj9lsqz4BcxRaMWFO3mNcwGDfSW+sqgc3E8N6LKrTpZq3ke7xacpQmcG5DU9VO+2QVPdltl9jWbs3gXjmF92YRNOuKPVfAOZBBsp8JOznfx8s9wDgs7RwPmDpjIAJEyoABqW5hlXfqRbTnfnMvuR informatic@InformaticPC"
+ "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDGkMgEVwQM8yeuFUYL2TwlJIq9yUNBmHnwce46zeL2PK2CkMz7sxT/om7sp/K5XDiqeD05Nioe+Dr3drP6B8uI33S5NgxPIfaqQsRS+CBEgk6cqFlcdlKETU/DT+/WsdoO173n7mgGeafPInEuQuGDUID0Fl099kIxtqfAhdeZFMM6/szAZEZsElLJ8K6dp1Ni/jmnXCZhjivZH3AZUlnqrmtDG7FY1bgcOfDXAal45LItughGPtrdiigXe9DK2fW3+9DBZZduh5DMJTNlphAZ+nfSrbyHVKUg6WsgMSprur4KdU47q1QwzqqvEj75JcdP1jOWoZi4F6VJDte9Wb9lhD1jGgjxY9O6Gs4CH35bx15W7CN9hgNa0C8NbPJe/fZYIeMZmJ1m7O2xmnYwP8j+t7RNJWu7Pa3Em4mOEXvhBF07Zfq+Ye/4SluoRgADy5eII2x5fFo5EBhInxK0/X8wF6XZvysalVifoCh7T4Edejoi91oAxFgYAxbboXGlod0eEHIi2hla8SM9+IBHOChmgawKBYp2kzAJyAmHNBF+Pah9G4arVCj/axp/SJZDZbJQoI7UT/fJzEtvlb5RWrHXRq+y6IvjpUq4pzpDWW04+9UMqEEXRmhWOakHfEVM9rN8h3aJBflLUBBnh0Z/hVsKNh8bCRHaKtah8TrD9i+wMw== patryk.jakuszew@gmail.com"
+ "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC33naG1ptCvUcRWX9cj9wXM1nW1lyQC4SvMJzWlr9aMD96O8hQ2JMkuIUgUJvorAY02QRplQ2BuoVoVkdkzwjMyi1bL3OdgcKo7Z1yByClGTTocqNJYY0lcUb6EJH8+6e6F9ydrQlSxNzL1uCaA7phZr+yPcmAmWbSfioXn98yXNkE0emHxzJv/nypJY56sDCMC2IXDRd8L2goDtPwgPEW7bWfAQdIFMJ75xOidZOTxJ8eqyXLw/kxY5UlyX66jdoYz1sE5XUHuoQl1AOG9UdlMo0aMhUvP4pX5l7r7EnA9OttKMFB3oWqkVK/R6ynZ52YNOU5BZ9V+Ppaj34W0xNu+p0mbHcCtXYCTrf/OU0hcZDbDaNTjs6Vtcm2wYw9iAKX7Tex+eOMwUwlrlcyPNRV5BTot7lGNYfauHCSIuWJKN4NhCLR/NtVNh4/94eKkPTwJsY6XqDcS7q49wPAs4DAH7BJgsbHPOqygVHrY0YYEfz3Pj0HTxJHQMCP/hQX4fXEGt0BjgoVJbXPAQtPyeg0JuxiUg+b4CgVVfQ6R060MlM1BZzhmh+FY5MJH6nJppS0aHYCvSg8Z68NUlCPKy0jpcyfuAIWQWwSGG1O010WShQG2ELsvNdg5/4HVdCGNl5mmoom6JOd72FOZyQlHDFfeQUQRn9HOeCq/c51rK99SQ== bartek@IHM"
+ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICTR292kx/2CNuWYIsZ6gykQ036aBGrmheIuZa6S1D2x implr@thonk"
+ ];
+
+ networking.firewall.enable = false;
+
+ # Point k8s apiserver address at ourselves, as every node runs an apiserver with this cert name.
+ networking.extraHosts = ''
+ 127.0.0.1 ${k8sapi}
+ '';
+
+ security.acme.certs = {
+ host = {
+ email = acmeEmail;
+ domain = fqdn;
+ webroot = services.nginx.virtualHosts.host.root;
+ };
+ };
+
+ services.nginx = {
+ enable = true;
+ virtualHosts.host = {
+ serverName = fqdn;
+ root = "/var/www/${fqdn}";
+ };
+ };
+
+ services.etcd = {
+ enable = true;
+ name = fqdn;
+ listenClientUrls = ["https://0.0.0.0:2379"];
+ advertiseClientUrls = ["https://${fqdn}:2379"];
+ listenPeerUrls = ["https://0.0.0.0:2380"];
+ initialAdvertisePeerUrls = ["https://${fqdn}:2380"];
+ initialCluster = (map (n: "${n.fqdn}=https://${n.fqdn}:2380") nodes);
+
+ clientCertAuth = true;
+ trustedCaFile = pki.etcd.server.ca;
+ certFile = pki.etcd.server.cert;
+ keyFile = pki.etcd.server.key;
+
+ peerClientCertAuth = true;
+ peerTrustedCaFile = pki.etcdPeer.ca;
+ peerCertFile = pki.etcdPeer.cert;
+ peerKeyFile = pki.etcdPeer.key;
+
+ extraConf = {
+ PEER_CLIENT_CERT_AUTH = "true";
+ };
+ };
+
+ services.kubernetes = {
+ # Pin to specific k8s package.
+ package = k8spkgs.kubernetes;
+ roles = []; # We do not use any nixpkgs predefined roles for k8s. Instead,
+ # we enable k8s components manually.
+
+ caFile = pki.kube.apiserver.ca;
+ clusterCidr = "10.10.16.0/20";
+
+ path = [ pkgs.e2fsprogs ]; # kubelet wants to mkfs.ext4 when mounting pvcs
+
+ addons.dns.enable = false;
+
+ apiserver = rec {
+ enable = true;
+ insecurePort = ports.k8sAPIServerPlain;
+ securePort = ports.k8sAPIServerSecure;
+ advertiseAddress = "${node.ipAddr}";
+
+ etcd = {
+ servers = (map (n: "https://${n.fqdn}:2379") nodes);
+ caFile = pki.etcd.kube.ca;
+ keyFile = pki.etcd.kube.key;
+ certFile = pki.etcd.kube.cert;
+ };
+
+ tlsCertFile = pki.kube.apiserver.cert;
+ tlsKeyFile = pki.kube.apiserver.key;
+
+ clientCaFile = pki.kube.apiserver.ca;
+
+ kubeletHttps = true;
+ kubeletClientCaFile = pki.kube.apiserver.ca;
+ kubeletClientCertFile = pki.kube.apiserver.cert;
+ kubeletClientKeyFile = pki.kube.apiserver.key;
+
+ serviceAccountKeyFile = pki.kube.serviceaccounts.key;
+
+ allowPrivileged = true;
+ serviceClusterIpRange = "10.10.12.0/24";
+ runtimeConfig = "api/all,authentication.k8s.io/v1beta1";
+ authorizationMode = ["Node" "RBAC"];
+ enableAdmissionPlugins = ["Initializers" "NamespaceLifecycle" "NodeRestriction" "LimitRanger" "ServiceAccount" "DefaultStorageClass" "ResourceQuota"];
+ extraOpts = ''
+ --apiserver-count=3 \
+ --proxy-client-cert-file=${pki.kubeFront.apiserver.cert} \
+ --proxy-client-key-file=${pki.kubeFront.apiserver.key} \
+ --requestheader-allowed-names= \
+ --requestheader-client-ca-file=${pki.kubeFront.apiserver.ca} \
+ --requestheader-extra-headers-prefix=X-Remote-Extra- \
+ --requestheader-group-headers=X-Remote-Group \
+ --requestheader-username-headers=X-Remote-User \
+ -v=5
+ '';
+ };
+
+ controllerManager = {
+ enable = true;
+ bindAddress = "0.0.0.0";
+ insecurePort = ports.k8sControllerManagerPlain;
+ leaderElect = true;
+ serviceAccountKeyFile = pki.kube.serviceaccounts.key;
+ rootCaFile = pki.kube.ca;
+ extraOpts = ''
+ --service-cluster-ip-range=10.10.12.0/24 \
+ --use-service-account-credentials=true \
+ --secure-port=${toString ports.k8sControllerManagerSecure}\
+ '';
+ kubeconfig = pki.kube.controllermanager.config;
+ };
+
+ scheduler = {
+ enable = true;
+ address = "0.0.0.0";
+ port = 0;
+ leaderElect = true;
+ kubeconfig = pki.kube.scheduler.config;
+ };
+
+ proxy = {
+ enable = true;
+ kubeconfig = pki.kube.proxy.config;
+ extraOpts = ''
+ --hostname-override=${fqdn}\
+ --proxy-mode=iptables
+ '';
+ };
+
+ kubelet = {
+ enable = true;
+ unschedulable = false;
+ allowPrivileged = true;
+ hostname = fqdn;
+ tlsCertFile = pki.kube.kubelet.cert;
+ tlsKeyFile = pki.kube.kubelet.key;
+ clientCaFile = pki.kube.kubelet.ca;
+ nodeIp = node.ipAddr;
+ networkPlugin = "cni";
+ clusterDns = "10.10.12.254";
+ kubeconfig = pki.kube.kubelet.config;
+ extraOpts = ''
+ --cni-conf-dir=/opt/cni/conf \
+ --cni-bin-dir=/opt/cni/bin
+ '';
+ };
+
+ };
+
+ # https://github.com/NixOS/nixpkgs/issues/60687
+ systemd.services.kube-control-plane-online = {
+ preStart = pkgs.lib.mkForce "";
+ };
+ # this seems to depend on flannel
+ # TODO(q3k): file issue
+ systemd.services.kubelet-online = {
+ script = pkgs.lib.mkForce "sleep 1";
+ };
+ # This by default removes all CNI plugins and replaces them with nix-defines ones
+ # Since we bring our own CNI plugins via containers with host mounts, this causes
+ # them to be removed on kubelet restart.
+ # TODO(https://github.com/NixOS/nixpkgs/issues/53601): fix when resolved
+ systemd.services.kubelet = {
+ preStart = pkgs.lib.mkForce "sleep 1";
+ };
+}
diff --git a/cluster/nix/configuration.sample.nix b/cluster/nix/configuration.sample.nix
new file mode 100644
index 0000000..8681f76
--- /dev/null
+++ b/cluster/nix/configuration.sample.nix
@@ -0,0 +1,11 @@
+{ config, pkgs, ... }:
+
+{
+ imports = [
+ ./hardware-configuration.nix
+ ./hswaw-cluster.nix
+ ];
+
+ networking.hostName = "bc01n01";
+ system.stateVersion = "18.09";
+}
diff --git a/cluster/nix/toplevel.nix b/cluster/nix/toplevel.nix
new file mode 100644
index 0000000..15b552a
--- /dev/null
+++ b/cluster/nix/toplevel.nix
@@ -0,0 +1,90 @@
+rec {
+ domain = ".hswaw.net";
+ k8sapi = "k0.hswaw.net";
+ acmeEmail = "q3k@hackerspace.pl";
+
+ nodes = [
+ {
+ fqdn = "bc01n01.hswaw.net";
+ ipAddr = "185.236.240.35";
+ podNet = "10.10.16.0/24";
+ diskBoot = "/dev/sdb";
+ }
+ {
+ fqdn = "bc01n02.hswaw.net";
+ ipAddr = "185.236.240.36";
+ podNet = "10.10.17.0/24";
+ diskBoot = "/dev/sdb";
+ }
+ {
+ fqdn = "bc01n03.hswaw.net";
+ ipAddr = "185.236.240.37";
+ podNet = "10.10.18.0/24";
+ diskBoot = "/dev/sdb";
+ }
+ ];
+
+ pki = rec {
+ root = /opt/hscloud;
+
+ make = (radix: name: rec {
+ ca = root + "/${radix}-ca.crt";
+ cert = root + "/${radix}-${name}.crt";
+ key = root + "/${radix}-${name}.key";
+
+ json = (builtins.toJSON {
+ ca = (builtins.toString ca);
+ cert = (builtins.toString cert);
+ key = (builtins.toString key);
+ });
+ });
+
+ etcdPeer = (make "etcdpeer" "server");
+
+ etcd = {
+ server = (make "etcd" "server");
+ kube = (make "etcd" "kube");
+ };
+
+ makeKube = (name: (make "kube" name) // {
+ config = {
+ server = "https://${k8sapi}:${toString ports.k8sAPIServerSecure}";
+ certFile = (make "kube" name).cert;
+ keyFile = (make "kube" name).key;
+ };
+ });
+
+ kube = rec {
+ ca = apiserver.ca;
+
+ # Used to identify apiserver.
+ apiserver = (makeKube "apiserver");
+
+ # Used to identify controller-manager.
+ controllermanager = (makeKube "controller-manager");
+
+ # Used to identify scheduler.
+ scheduler = (makeKube "scheduler");
+
+ # Used to identify kube-proxy.
+ proxy = (makeKube "proxy");
+
+ # Used to identify kubelet.
+ kubelet = (makeKube "node");
+
+ # Used to encrypt service accounts.
+ serviceaccounts = (makeKube "serviceaccounts");
+ };
+
+ kubeFront = {
+ apiserver = (make "kubeFront" "apiserver");
+ };
+ };
+
+ ports = {
+ k8sAPIServerPlain = 4000;
+ k8sAPIServerSecure = 4001;
+ k8sControllerManagerPlain = 0; # 4002; do not serve plain http
+ k8sControllerManagerSecure = 4003;
+ };
+}