commit 1257389d3dba07fedfb43ce4b6a5862142d4b832
author Serge Bazanski <q3k@hackerspace.pl> Sat Oct 10 14:55:08 2020 +0200
committer q3k <q3k@hackerspace.pl> Sat Oct 10 16:00:15 2020 +0000
tree e11fdf90244b8fce0c17454fff29f30081c2513a
parent 36224c617a1e0924e95a64c047091d3678885d82

k0: expose controller-manager and scheduler metrics

We want to be able to scrape controller-manager and scheduler metrics
into Prometheus. For that, each of them needs to:

 1) listen on a secure port
 2) have authn enabled

With this, any k8s user with the right permissions (and a bearer token
or TLS certificate) can come in and access metrics over a node's public
IP address. Access without a certificate/token gets thrown into the
system:anonymous user, which as no access to any API.

Change-Id: I267680f92f748ba63b6762e6aaba3c417446e50b

cluster/nix/defs-cluster-k0.nix

diff --git a/cluster/nix/defs-cluster-k0.nix b/cluster/nix/defs-cluster-k0.nix
index 832c741..c3519cc 100644
--- a/cluster/nix/defs-cluster-k0.nix
+++ b/cluster/nix/defs-cluster-k0.nix
@@ -70,7 +70,9 @@
   ports = {
     k8sAPIServerPlain = 4000;
     k8sAPIServerSecure = 4001;
-    k8sControllerManagerPlain = 0; # 4002; do not serve plain http
+    k8sControllerManagerPlain = 0; # would be 4002; do not serve plain http
     k8sControllerManagerSecure = 4003;
+    k8sSchedulerPlain = 0; # would be 4004; do not serve plain http
+    k8sSchedulerSecure = 4005;
   };
 }

cluster/nix/modules/kubernetes.nix

diff --git a/cluster/nix/modules/kubernetes.nix b/cluster/nix/modules/kubernetes.nix
index 10560cd..b36e806 100644
--- a/cluster/nix/modules/kubernetes.nix
+++ b/cluster/nix/modules/kubernetes.nix
@@ -154,7 +154,10 @@
       '';
     };
 
-    controllerManager = {
+    controllerManager = let
+      top = config.services.kubernetes;
+      kubeconfig = top.lib.mkKubeConfig "controller-manager" pki.kube.controllermanager.config;
+    in {
       enable = true;
       bindAddress = "0.0.0.0";
       insecurePort = ports.k8sControllerManagerPlain;
@@ -165,16 +168,26 @@
         --service-cluster-ip-range=10.10.12.0/24 \
         --use-service-account-credentials=true \
         --secure-port=${toString ports.k8sControllerManagerSecure}\
+        --authentication-kubeconfig=${kubeconfig}\
+        --authorization-kubeconfig=${kubeconfig}\
       '';
       kubeconfig = pki.kube.controllermanager.config;
     };
 
-    scheduler = {
+    scheduler = let
+      top = config.services.kubernetes;
+      kubeconfig = top.lib.mkKubeConfig "scheduler" pki.kube.controllermanager.config;
+    in {
       enable = true;
       address = "0.0.0.0";
-      port = 0;
+      port = ports.k8sSchedulerPlain;
       leaderElect = true;
       kubeconfig = pki.kube.scheduler.config;
+      extraOpts = ''
+        --secure-port=${toString ports.k8sSchedulerSecure}\
+        --authentication-kubeconfig=${kubeconfig}\
+        --authorization-kubeconfig=${kubeconfig}\
+      '';
     };
 
     proxy = {