cluster: deploy coredns
diff --git a/cluster/certs/ca.srl b/cluster/certs/ca.srl
index 651f7dd..c15d2a2 100644
--- a/cluster/certs/ca.srl
+++ b/cluster/certs/ca.srl
@@ -1 +1 @@
-80F13FCE5DBBF730
+80F13FCE5DBBF736
diff --git a/cluster/certs/kube-apiserver.crt b/cluster/certs/kube-apiserver.crt
index 1777380..f7df90b 100644
--- a/cluster/certs/kube-apiserver.crt
+++ b/cluster/certs/kube-apiserver.crt
@@ -1,10 +1,10 @@
-----BEGIN CERTIFICATE-----
-MIIF0DCCA7igAwIBAgIJAIDxP85du/cnMA0GCSqGSIb3DQEBCwUAMIG3MQswCQYD
+MIIGCzCCA/OgAwIBAgIJAIDxP85du/c2MA0GCSqGSIb3DQEBCwUAMIG3MQswCQYD
VQQGEwJQTDEUMBIGA1UECAwLTWF6b3dpZWNraWUxETAPBgNVBAcMCFdhcnN6YXdh
MS4wLAYDVQQKDCVTdG93YXJ6eXN6ZW5pZSBXYXJzemF3c2tpIEhhY2tlcnNwYWNl
MRAwDgYDVQQLDAdoc2Nsb3VkMRowGAYDVQQDDBFCb290c3RyYXAgTm9kZSBDQTEh
-MB8GCSqGSIb3DQEJARYScTNrQGhhY2tlcnNwYWNlLnBsMB4XDTE5MDExMzE5NDU0
-NVoXDTIwMDExMzE5NDU0NVowgZYxCzAJBgNVBAYTAlBMMRQwEgYDVQQIDAtNYXpv
+MB8GCSqGSIb3DQEJARYScTNrQGhhY2tlcnNwYWNlLnBsMB4XDTE5MDExMzIyMzgy
+NVoXDTIwMDExMzIyMzgyNVowgZYxCzAJBgNVBAYTAlBMMRQwEgYDVQQIDAtNYXpv
d2llY2tpZTERMA8GA1UEBwwIV2Fyc3phd2ExLjAsBgNVBAoMJVN0b3dhcnp5c3pl
bmllIFdhcnN6YXdza2kgSGFja2Vyc3BhY2UxFzAVBgNVBAsMDkt1YmVybmV0ZXMg
QVBJMRUwEwYDVQQDDAxrMC5oc3dhdy5uZXQwggIiMA0GCSqGSIb3DQEBAQUAA4IC
@@ -19,16 +19,17 @@
wWs87E5VPZ7LuM5QJNg6ZBLJ7B81rvw3BYTar0H2YfLGeTjhktJ9fJVjx7gvAagB
RnipgSOLN4fiB68wTe8lyLLH+7+ZtfZl8myRzkoDvHc0iBeZa0Pr2iGCLfR5Fkqo
hU7nVRremTfIodygtTMdSozpOWRMaLJV1WJfMiB91rs+mwMBhncqa3Hp6QIDAQAB
-MA0GCSqGSIb3DQEBCwUAA4ICAQBMkv4dG3gybWdggc5aCZqyanp+CU506ejVpAd2
-oPgJnvcAR1DVnHer2hMFlRk4lt1rSPsRv1bqOQLgBkOEbUJhknaSD6CknmfriX1/
-ZdBwB9JHy7E/S4QDrm/8s6HiWcKYW6eK35aP4bF8ebDp+PBmYOrHRl85vNqtjeMJ
-iyXznQFL1kiuT2hBcMiQeVbEz4o0u/yAlNIxL3PXKXn0AVyW0LjLI+EAd8lCfGKy
-SkJf2gw/UWx3s+rEctA6qrB29PBR03PTHvXfb53ILh8KuIh3hU3+EED7puNhNvrS
-qWthIe5hAVOEaE9GfHCqdelQELrrYhAVMuO+PqtsGwZruEY6dpI493Aq+lfd+2TT
-pRG/isoGvGh+Lg+pwV3DLuGnnMH47iUHnPPXbYRBvSpnhC80vx8Bnbn4l7TpuZYo
-KLo5heP+Mb4sueG7KjuoOHRXcI3vHgKD2XjXFokdLBAy+75Ik0YNUWbvK76PiajE
-znic15ws1lTJiY16z+JPdjpLh+ddXf2DDnFhkWNy/Fxt+dIm0Bhdgin0rPKpE8Sv
-BIKiagL0VpDlVW5DUQe9ZVNW3zvyb3fvis+4+SmPcHDEq6ULgQQD6NjuPTKa7Suo
-pm7SxeMmP1c0B28S1wqZAh0mxrD/yUhKA/ZagRktAhCr+CXAgePEqeCZCx0XGqUw
-d/ZySw==
+ozkwNzAdBgNVHREEFjAUhwQKCgwBggxrMC5oc3dhdy5uZXQwCQYDVR0TBAIwADAL
+BgNVHQ8EBAMCBeAwDQYJKoZIhvcNAQELBQADggIBAAOx8hojlmZWH7cfLaqiSrHJ
+nzcUARJYeQzTqHYggOR0TBkI/kSOkgXeDky8mya/9zcn+/GdDvxfgivZ6J7ZLAtF
+JU6BXnBaVqG/N2oBM6qzXedU4SBDCtS+/3vhAdrk8e1gml5kAYEaw0IDeHqMtW63
+k8QtWeg3+I5/u28X46+xEhkWhk6ofXoDOs3ytSU0lUqi4FFI4JCeCM/fgHWDJUNh
+Z+5O3ovQmn0d2N/O7sK4toBPAPv7TcvaaBvM4OYnLPoZ4PSfbgK0ZFfhEaJ0jVNH
+5AApv/0eT1utOxuZX9xdCkrVrGHPmS+HJM/Q0n19Iqzxa6ZBCrqBlUjuo51DUisd
+knwTczSke7d2AEjgbKk00eGd10aSjXk/wFQwTxK6jbdmKztRvP88zq8FilRdYcqN
+4TX48bjFo6NjSmyo0kXgZJeIcHM4pKrNyOAz6WRt39da+KUZKXy1oy/3PXUhiOaX
+5rGt2H8FVxTg4XBsRBvwvKFxyse7mKFYyWuQa97ugfg66PxNLdmOn5WJgy/CcHYy
+89np4pgkNjQ6K/OE0OgMN5WbrVU231b+hzNkYl6XT3iB7vZOChLkl5afdiFXzFg1
+kbnQTWu3BwZhCg2mNGs9pGqJ0FvL1KHfgsLt3VBWjepFkOOWYvrmjqwxCxZaRIrn
+g0RSHtqS2xS9EAjqVG0/
-----END CERTIFICATE-----
diff --git a/cluster/kube/cluster.jsonnet b/cluster/kube/cluster.jsonnet
index 6b64240..9209cf1 100644
--- a/cluster/kube/cluster.jsonnet
+++ b/cluster/kube/cluster.jsonnet
@@ -1,6 +1,7 @@
# Top level cluster configuration.
local kube = import "../../kube/kube.libsonnet";
+local coredns = import "lib/coredns.libsonnet";
local Cluster(fqdn) = {
local cluster = self,
@@ -37,9 +38,14 @@
name: fqdn,
},
],
- }
+ },
+
+ // CoreDNS for this cluster.
+ dns: coredns.Environment {
+ },
};
+
{
k0: Cluster("k0.hswaw.net"),
}
diff --git a/cluster/kube/lib/coredns.libsonnet b/cluster/kube/lib/coredns.libsonnet
new file mode 100644
index 0000000..073c8ec
--- /dev/null
+++ b/cluster/kube/lib/coredns.libsonnet
@@ -0,0 +1,196 @@
+# Deploy a per-cluster CoreDNS
+
+local kube = import "../../../kube/kube.libsonnet";
+
+{
+ Environment: {
+ local env = self,
+ local cfg = env.cfg,
+ cfg:: {
+ image: "coredns/coredns:1.3.0",
+ namespace: "kube-system",
+ upstream_server: "185.236.240.1",
+ cluster_domain: "cluster.local",
+ reverse_cidrs: ["in-addr.arpa", "ip6.arpa"],
+ clusterIP: "10.10.12.254",
+ },
+
+ sa: kube.ServiceAccount("coredns") {
+ metadata+: {
+ namespace: cfg.namespace,
+ },
+ },
+
+ cr: kube.ClusterRole("system:coredns") {
+ metadata+: {
+ labels: {
+ "kubernetes.io/bootstrapping": "rbac-defaults",
+ },
+ },
+ rules: [
+ {
+ apiGroups: [""],
+ resources: ["endpoints", "services", "pods", "namespaces"],
+ verbs: ["list", "watch"],
+ },
+ {
+ apiGroups: [""],
+ resources: ["nodes"],
+ verbs: ["get"],
+ },
+ ],
+ },
+
+ crb: kube.ClusterRoleBinding("system:coredns") {
+ metadata+: {
+ labels: {
+ "kubernetes.io/bootstrapping": "rbac-defaults",
+ },
+ annotations+: {
+ "rbac.authorization.kubernetes.io/autoupdate": "true",
+ },
+ },
+ roleRef: {
+ apiGroup: "rbac.authorization.k8s.io",
+ kind: "ClusterRole",
+ name: env.cr.metadata.name,
+ },
+ subjects: [
+ {
+ kind: "ServiceAccount",
+ name: env.sa.metadata.name,
+ namespace: env.sa.metadata.namespace,
+ },
+ ],
+ },
+
+ cm: kube.ConfigMap("coredns") {
+ local map = self,
+
+ upstream_server:: cfg.upstream_server,
+ cluster_domain:: cfg.cluster_domain,
+ reverse_cidrs:: std.join(" ", cfg.reverse_cidrs),
+
+ metadata+: {
+ namespace: cfg.namespace,
+ },
+ data: {
+ Corefile: |||
+ .:53 {
+ log
+ errors
+ health
+ kubernetes %s %s {
+ pods insecure
+ upstream
+ fallthrough in-addr.arpa ip6.arpa
+ }
+ prometheus :9153
+ proxy . %s
+ cache 30
+ loop
+ reload
+ loadbalance
+ }
+ ||| % [map.cluster_domain, map.reverse_cidrs, map.upstream_server]
+ },
+ },
+
+ deployment: kube.Deployment("coredns") {
+ metadata+: {
+ namespace: cfg.namespace,
+ labels+: {
+ "k8s-app": "coredns",
+ },
+ },
+ spec+: {
+ replicas: 2,
+ strategy: {
+ type: "RollingUpdate",
+ rollingUpdate: { maxUnavailable: 1 },
+ },
+ template+: {
+ spec+: {
+ serviceAccountName: env.sa.metadata.name,
+ tolerations: [
+ { key: "CriticalAddonsOnly", operator: "Exists" },
+ ],
+ dnsPolicy: "Default",
+ volumes_: {
+ config: {
+ configMap: {
+ name: env.cm.metadata.name,
+ items: [ { key: "Corefile", path: "Corefile" } ],
+ },
+ },
+ },
+ containers_: {
+ coredns: kube.Container("coredns") {
+ local container = self,
+
+ image: cfg.image,
+ args: [
+ "-conf", "%s/Corefile" % container.volumeMounts[0].mountPath,
+ ],
+ imagePullPolicy: "IfNotPresent",
+ resources: {
+ limits: { memory: "170Mi" },
+ requests: { memory: "70Mi", cpu: "100m" },
+ },
+ volumeMounts_: {
+ config: {
+ mountPath: "/etc/coredns",
+ },
+ },
+ ports_: {
+ dns: {
+ containerPort: 53,
+ protocol: "UDP",
+ },
+ "dns-tcp": {
+ containerPort: 53,
+ protocol: "TCP",
+ },
+ metrics: {
+ containerPort: 9153,
+ protocol: "TCP",
+ },
+ },
+ securityContext: {
+ allowPrivilegeEscalation: false,
+ capabilities: {
+ add: ["NET_BIND_SERVICE"],
+ drop: ["all"],
+ },
+ readOnlyRootFilesystem: true,
+ },
+ livenessProbe: {
+ httpGet: {
+ path: "/health",
+ port: 8080,
+ scheme: "HTTP",
+ },
+ initialDelaySeconds: 60,
+ timeoutSeconds: 5,
+ successThreshold: 1,
+ failureThreshold: 5,
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ svc: kube.Service("coredns") {
+ local svc = self,
+ metadata+: {
+ namespace: cfg.namespace,
+ },
+ target_pod: env.deployment.spec.template,
+ spec+: {
+ ports: [ { name: p.name, port: p.containerPort, protocol: p.protocol } for p in svc.target_pod.spec.containers[0].ports ],
+ clusterIP: cfg.clusterIP,
+ },
+ },
+ },
+}
diff --git a/tools/clustercfg.py b/tools/clustercfg.py
index 29e42a4..a36664a 100644
--- a/tools/clustercfg.py
+++ b/tools/clustercfg.py
@@ -96,8 +96,20 @@
if san:
config.seek(0, 2)
config.write(b'\n[SAN]\n')
+ config.write(b'subjectAltName = @alt_names\n')
+ config.write(b'basicConstraints = CA:FALSE\nkeyUsage = nonRepudiation, digitalSignature, keyEncipherment\n')
+ config.write(b'[alt_names]\n')
+
+ ipcnt = 1
+ dnscnt = 1
for s in san:
- config.write('subjectAltName=DNS:{}\n'.format(s).encode())
+ parts = s.split(':')
+ if s.startswith('DNS'):
+ config.write('DNS.{} = {}\n'.format(dnscnt, parts[1]).encode())
+ dnscnt += 1
+ elif s.startswith('IP'):
+ config.write('IP.{} = {}\n'.format(ipcnt, parts[1]).encode())
+ ipcnt += 1
f = tempfile.NamedTemporaryFile(delete=False)
path = f.name
@@ -192,32 +204,30 @@
else:
generate_cert = True
- if not generate_cert:
- return False
+ if generate_cert:
+ local_csr_f = tempfile.NamedTemporaryFile(delete=False)
+ local_csr = local_csr_f.name
+ local_csr_f.close()
- local_csr_f = tempfile.NamedTemporaryFile(delete=False)
- local_csr = local_csr_f.name
- local_csr_f.close()
+ local_config = openssl_config(san)
- local_config = openssl_config(san)
+ subprocess.check_call([
+ 'openssl', 'req', '-new',
+ '-key', local_key,
+ '-out', local_csr,
+ '-subj', str(subj),
+ '-config', local_config,
+ ] + ([
+ '-reqexts', 'SAN',
+ ] if san else []))
- subprocess.check_call([
- 'openssl', 'req', '-new',
- '-key', local_key,
- '-out', local_csr,
- '-subj', str(subj),
- '-config', local_config,
- ] + ([
- '-reqexts', 'SAN',
- ] if san else []))
-
- pki.sign(local_csr, local_cert, local_config, days)
+ pki.sign(local_csr, local_cert, local_config, days)
+ os.remove(local_csr)
+ os.remove(local_config)
c.put(local=local_key, remote=remote_key)
c.put(local=local_cert, remote=remote_cert)
- os.remove(local_csr)
- os.remove(local_config)
return True
@@ -312,27 +322,15 @@
modified = False
modified |= remote_cert(p, c, fqdn, "node", Subject(Subject.hswaw, 'Node Certificate', fqdn))
- modified |= remote_cert(p, c, fqdn, "kube-node", Subject('system:nodes', 'Kubelet Certificate', 'system:node:' + fqdn), san=[fqdn,])
+ modified |= remote_cert(p, c, fqdn, "kube-node", Subject('system:nodes', 'Kubelet Certificate', 'system:node:' + fqdn), san=["DNS:"+fqdn,])
for component in ['controller-manager', 'proxy', 'scheduler']:
o = 'system:kube-{}'.format(component)
ou = 'Kuberneter Component {}'.format(component)
modified |= shared_cert(p, c, fqdn, 'kube-{}'.format(component), Subject(o, ou, o))
- modified |= shared_cert(p, c, fqdn, 'kube-apiserver', Subject(Subject.hswaw, 'Kubernetes API', cluster))
+ modified |= shared_cert(p, c, fqdn, 'kube-apiserver', Subject(Subject.hswaw, 'Kubernetes API', cluster), san=['IP:10.10.12.1', 'DNS:' + cluster])
modified |= shared_cert(p, c, fqdn, 'kube-serviceaccounts', Subject(Subject.hswaw, 'Kubernetes Service Account Signer', 'service-accounts'))
- if modified:
- logger.info('{}: cert(s) modified, restarting services...'.format(fqdn))
-
- services = [
- 'kubelet', 'kube-proxy',
- 'kube-apiserver', 'kube-controller-manager', 'kube-scheduler',
- 'etcd'
- ]
-
- for s in services:
- c.run('systemctl stop {}'.format(s))
- for s in services[::-1]:
- c.run('systemctl start {}'.format(s))
+ c.run('nixos-rebuild switch')
def usage():
sys.stderr.write("Usage: {} <nodestrap|admincreds>\n".format(sys.argv[0]))