Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 1 | #!/usr/bin/env python |
| 2 | |
| 3 | from builtins import object |
| 4 | |
| 5 | import datetime |
| 6 | from io import BytesIO |
| 7 | import json |
| 8 | import logging |
| 9 | import os |
| 10 | import tempfile |
| 11 | import subprocess |
| 12 | import sys |
| 13 | |
| 14 | from cryptography import x509 |
| 15 | from cryptography.hazmat.backends import default_backend |
| 16 | import fabric |
| 17 | |
| 18 | from tools import secretstore |
Serge Bazanski | 0f8e5a2 | 2021-10-16 20:53:51 +0000 | [diff] [blame] | 19 | from tools.hscloud import lib as hscloud |
Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 20 | |
| 21 | import ca |
| 22 | |
| 23 | |
Serge Bazanski | 0f8e5a2 | 2021-10-16 20:53:51 +0000 | [diff] [blame] | 24 | local_root = hscloud.workspace_location() |
Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 25 | |
| 26 | |
| 27 | cluster = 'k0.hswaw.net' |
Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 28 | ss = secretstore.SecretStore( |
| 29 | plain_root=os.path.join(local_root, 'cluster/secrets/plain'), |
| 30 | cipher_root=os.path.join(local_root, 'cluster/secrets/cipher')) |
| 31 | |
| 32 | |
| 33 | logger = logging.getLogger() |
| 34 | logger.setLevel(logging.INFO) |
| 35 | formatter = logging.Formatter('%(levelname)s - %(message)s') |
| 36 | sh = logging.StreamHandler() |
| 37 | sh.setFormatter(formatter) |
| 38 | logger.addHandler(sh) |
| 39 | |
| 40 | |
| 41 | |
Sergiusz Bazanski | c78cc13 | 2020-02-02 22:31:53 +0100 | [diff] [blame] | 42 | def pki_config(key, fqdn): |
| 43 | machine_name = fqdn.split('.')[0] |
Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 44 | raw = subprocess.check_output([ |
| 45 | 'nix', 'eval', '--raw', |
Sergiusz Bazanski | c78cc13 | 2020-02-02 22:31:53 +0100 | [diff] [blame] | 46 | '( ((import ' + local_root + '/cluster/nix/defs-cluster-k0.nix ) "' + machine_name + '").pki.' + key + '.json )', |
Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 47 | ]) |
| 48 | return json.loads(raw) |
| 49 | |
| 50 | |
| 51 | def _file_exists(c, filename): |
| 52 | res = c.run('stat "{}"'.format(filename), warn=True, hide=True) |
| 53 | return res.exited == 0 |
| 54 | |
| 55 | |
| 56 | def configure_k8s(username, ca, cert, key): |
| 57 | subprocess.check_call([ |
| 58 | 'kubectl', 'config', |
Sergiusz Bazanski | b13b7ff | 2019-08-29 20:12:24 +0200 | [diff] [blame] | 59 | 'set-cluster', 'admin.' + cluster, |
Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 60 | '--certificate-authority=' + ca, |
| 61 | '--embed-certs=true', |
| 62 | '--server=https://' + cluster + ':4001', |
| 63 | ]) |
| 64 | subprocess.check_call([ |
| 65 | 'kubectl', 'config', |
| 66 | 'set-credentials', username, |
| 67 | '--client-certificate=' + cert, |
| 68 | '--client-key=' + key, |
| 69 | '--embed-certs=true', |
| 70 | ]) |
| 71 | subprocess.check_call([ |
| 72 | 'kubectl', 'config', |
Sergiusz Bazanski | b13b7ff | 2019-08-29 20:12:24 +0200 | [diff] [blame] | 73 | 'set-context', 'admin.' + cluster, |
| 74 | '--cluster=' + 'admin.' + cluster, |
Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 75 | '--user=' + username, |
| 76 | ]) |
| 77 | subprocess.check_call([ |
| 78 | 'kubectl', 'config', |
Sergiusz Bazanski | b13b7ff | 2019-08-29 20:12:24 +0200 | [diff] [blame] | 79 | 'use-context', 'admin.' + cluster, |
Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 80 | ]) |
| 81 | |
| 82 | |
| 83 | def admincreds(args): |
| 84 | if len(args) != 1: |
| 85 | sys.stderr.write("Usage: admincreds q3k\n") |
| 86 | return 1 |
| 87 | username = args[0] |
Sergiusz Bazanski | b13b7ff | 2019-08-29 20:12:24 +0200 | [diff] [blame] | 88 | print("") |
| 89 | print("WARNING WARNING WARNING WARNING WARNING WARNING") |
| 90 | print("===============================================") |
| 91 | print("") |
| 92 | print("You are requesting ADMIN credentials.") |
| 93 | print("") |
| 94 | print("You likely shouldn't be doing this, and") |
| 95 | print("instead should be using `prodaccess`.") |
| 96 | print("") |
| 97 | print("===============================================") |
| 98 | print("WARNING WARNING WARNING WARNING WARNING WARNING") |
| 99 | print("") |
Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 100 | |
| 101 | ## Make kube certificates. |
| 102 | certs_root = os.path.join(local_root, 'cluster/certs') |
| 103 | ca_kube = ca.CA(ss, certs_root, 'kube', 'kubernetes main CA') |
| 104 | |
| 105 | local_key = os.path.join(local_root, '.kubectl/admin.key') |
| 106 | local_crt = os.path.join(local_root, '.kubectl/admin.crt') |
| 107 | |
| 108 | kubectl = os.path.join(local_root, '.kubectl') |
| 109 | if not os.path.exists(kubectl): |
| 110 | os.mkdir(kubectl) |
| 111 | |
| 112 | generate_cert = False |
| 113 | if not os.path.exists(local_key): |
| 114 | generate_cert = True |
| 115 | |
| 116 | if os.path.exists(local_crt): |
| 117 | with open(local_crt, 'rb') as f: |
| 118 | b = f.read() |
| 119 | cert = x509.load_pem_x509_certificate(b, default_backend()) |
| 120 | delta = cert.not_valid_after - datetime.datetime.now() |
| 121 | logger.info("admin: existing cert expiry: {}".format(delta)) |
| 122 | if delta.total_seconds() < 3600 * 24: |
| 123 | logger.info("admin: expires soon, regenerating") |
| 124 | generate_cert = True |
| 125 | else: |
| 126 | generate_cert = True |
| 127 | |
| 128 | if not generate_cert: |
| 129 | return configure_k8s(username, ca_kube._cert, local_crt, local_key) |
| 130 | |
| 131 | key, csr = ca_kube.gen_key(hosts=['admin', username], o='system:masters', ou='Kube Admin Account') |
| 132 | crt = ca_kube.sign(csr) |
| 133 | |
| 134 | with open(local_key, 'w') as f: |
| 135 | f.write(key) |
| 136 | |
| 137 | with open(local_crt, 'w') as f: |
| 138 | f.write(crt) |
| 139 | |
| 140 | configure_k8s(username, ca_kube._cert, local_crt, local_key) |
| 141 | |
| 142 | |
Sergiusz Bazanski | c0fc3ee | 2019-06-20 16:11:38 +0200 | [diff] [blame] | 143 | def nodestrap(args, nocerts=False): |
Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 144 | if len(args) != 1: |
| 145 | sys.stderr.write("Usage: nodestrap bc01n01.hswaw.net\n") |
| 146 | return 1 |
| 147 | fqdn = args[0] |
| 148 | |
| 149 | logger.info("Nodestrapping {}...".format(fqdn)) |
| 150 | r = fabric.Connection('root@{}'.format(fqdn)) |
| 151 | |
Sergiusz Bazanski | c0fc3ee | 2019-06-20 16:11:38 +0200 | [diff] [blame] | 152 | if not nocerts: |
Sergiusz Bazanski | c78cc13 | 2020-02-02 22:31:53 +0100 | [diff] [blame] | 153 | cfg = dict((k, pki_config(k, fqdn)) for k in [ |
Sergiusz Bazanski | c0fc3ee | 2019-06-20 16:11:38 +0200 | [diff] [blame] | 154 | 'etcdPeer', 'etcd.server', 'etcd.kube' |
| 155 | ]) |
| 156 | certs_root = os.path.join(local_root, 'cluster/certs') |
Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 157 | |
Sergiusz Bazanski | c0fc3ee | 2019-06-20 16:11:38 +0200 | [diff] [blame] | 158 | # Make etcd peer certificate for node. |
| 159 | ca_etcd_peer = ca.CA(ss, certs_root, 'etcdpeer', 'etcd peer ca') |
Sergiusz Bazanski | c78cc13 | 2020-02-02 22:31:53 +0100 | [diff] [blame] | 160 | ca_etcd_peer.make_cert('etcdpeer-{}'.format(fqdn), hosts=[fqdn], ou='node etcd peer certificate') |
Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 161 | |
Sergiusz Bazanski | c0fc3ee | 2019-06-20 16:11:38 +0200 | [diff] [blame] | 162 | # Make etcd server certificate for node and client certificate for kube. |
| 163 | ca_etcd = ca.CA(ss, certs_root, 'etcd', 'etcd ca') |
Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 164 | |
Sergiusz Bazanski | c78cc13 | 2020-02-02 22:31:53 +0100 | [diff] [blame] | 165 | ca_etcd.make_cert('etcd-{}'.format(fqdn), hosts=[fqdn], ou='node etcd server certificate') |
Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 166 | |
Sergiusz Bazanski | c78cc13 | 2020-02-02 22:31:53 +0100 | [diff] [blame] | 167 | ca_etcd.make_cert('etcd-kube', hosts=['kube'], ou='kube etcd client certificate') |
Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 168 | |
Sergiusz Bazanski | c0fc3ee | 2019-06-20 16:11:38 +0200 | [diff] [blame] | 169 | ca_etcd.make_cert('etcd-root', hosts=['root'], ou='root etcd client certificate') |
Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 170 | |
Sergiusz Bazanski | c0fc3ee | 2019-06-20 16:11:38 +0200 | [diff] [blame] | 171 | ca_etcd.make_cert('etcd-calico', hosts=['calico'], ou='root etcd client certificate') |
Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 172 | |
Sergiusz Bazanski | c0fc3ee | 2019-06-20 16:11:38 +0200 | [diff] [blame] | 173 | ## Make kube certificates. |
| 174 | ca_kube = ca.CA(ss, certs_root, 'kube', 'kubernetes main CA') |
Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 175 | |
Sergiusz Bazanski | b13b7ff | 2019-08-29 20:12:24 +0200 | [diff] [blame] | 176 | # Make prodvider intermediate CA. |
Sergiusz Bazanski | 0dcc702 | 2020-03-28 17:58:19 +0100 | [diff] [blame] | 177 | ca_kube.make_cert('ca-kube-prodvider', o='Warsaw Hackerspace', ou='kubernetes prodvider intermediate', hosts=['kubernetes prodvider intermediate CA'], profile='intermediate').ensure() |
Sergiusz Bazanski | b13b7ff | 2019-08-29 20:12:24 +0200 | [diff] [blame] | 178 | |
Sergiusz Bazanski | c0fc3ee | 2019-06-20 16:11:38 +0200 | [diff] [blame] | 179 | # Make kubelet certificate (per node). |
Sergiusz Bazanski | c78cc13 | 2020-02-02 22:31:53 +0100 | [diff] [blame] | 180 | ca_kube.make_cert('kube-kubelet-'+fqdn, o='system:nodes', ou='Kubelet', hosts=['system:node:'+fqdn, fqdn]) |
Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 181 | |
Sergiusz Bazanski | c0fc3ee | 2019-06-20 16:11:38 +0200 | [diff] [blame] | 182 | # Make apiserver certificate. |
Sergiusz Bazanski | c78cc13 | 2020-02-02 22:31:53 +0100 | [diff] [blame] | 183 | ca_kube.make_cert('kube-apiserver', ou='Kubernetes API', hosts=[cluster, 'kubernetes.default.svc.'+cluster, '10.10.12.1']) |
Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 184 | |
Sergiusz Bazanski | c0fc3ee | 2019-06-20 16:11:38 +0200 | [diff] [blame] | 185 | # Make service accounts decryption key (as cert for consistency). |
Sergiusz Bazanski | c78cc13 | 2020-02-02 22:31:53 +0100 | [diff] [blame] | 186 | ca_kube.make_cert('kube-serviceaccounts', ou='Kubernetes Service Accounts Signer', hosts=['serviceaccounts']) |
Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 187 | |
Sergiusz Bazanski | c0fc3ee | 2019-06-20 16:11:38 +0200 | [diff] [blame] | 188 | # Make kube component certificates. |
| 189 | kube_components = ['controllermanager', 'scheduler', 'proxy'] |
Sergiusz Bazanski | c78cc13 | 2020-02-02 22:31:53 +0100 | [diff] [blame] | 190 | cfg = dict((k, pki_config('kube.' + k, fqdn)) for k in kube_components) |
Sergiusz Bazanski | c0fc3ee | 2019-06-20 16:11:38 +0200 | [diff] [blame] | 191 | for k in kube_components: |
Sergiusz Bazanski | c0fc3ee | 2019-06-20 16:11:38 +0200 | [diff] [blame] | 192 | # meh |
| 193 | if k == 'controllermanager': |
| 194 | o = 'system:kube-controller-manager' |
| 195 | else: |
| 196 | o = 'system:kube-'+k |
| 197 | ou = 'Kubernetes Component '+k |
| 198 | c = ca_kube.make_cert('kube-'+k, ou=ou, o=o, hosts=[o,]) |
Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 199 | |
Sergiusz Bazanski | c0fc3ee | 2019-06-20 16:11:38 +0200 | [diff] [blame] | 200 | ## Make kubefront certificates. |
| 201 | ca_kubefront = ca.CA(ss, certs_root, 'kubefront', 'kubernetes frontend CA') |
Sergiusz Bazanski | c78cc13 | 2020-02-02 22:31:53 +0100 | [diff] [blame] | 202 | ca_kubefront.make_cert('kubefront-apiserver', ou='Kubernetes Frontend', hosts=['apiserver']) |
Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 203 | |
Serge Bazanski | 04604b2 | 2021-02-06 14:57:20 +0000 | [diff] [blame] | 204 | ## Make admitomatic (admission controller) certificates. |
| 205 | ca_admitomatic = ca.CA(ss, certs_root, 'admitomatic', 'admitomatic webhook CA') |
| 206 | ca_admitomatic.make_cert('admitomatic-webhook', ou='Admitomatic Webhook', hosts=['admitomatic.admitomatic.svc']) |
| 207 | |
Serge Bazanski | b3c6770 | 2021-09-10 22:27:24 +0000 | [diff] [blame] | 208 | toplevel = subprocess.check_output([ |
| 209 | "nix-build", |
| 210 | local_root, |
| 211 | "-A", "ops.machines.\"" + fqdn + "\".config.passthru.hscloud.provision", |
| 212 | ]).decode().strip() |
| 213 | subprocess.check_call([toplevel]) |
Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 214 | |
| 215 | |
| 216 | def usage(): |
Sergiusz Bazanski | c78cc13 | 2020-02-02 22:31:53 +0100 | [diff] [blame] | 217 | sys.stderr.write("Usage: clustercfg <nodestrap|admincreds>\n") |
Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 218 | |
| 219 | |
| 220 | def main(): |
| 221 | if len(sys.argv) < 2: |
| 222 | usage() |
| 223 | return 1 |
| 224 | |
| 225 | mode = sys.argv[1] |
| 226 | if mode == "nodestrap": |
| 227 | return nodestrap(sys.argv[2:]) |
Sergiusz Bazanski | c0fc3ee | 2019-06-20 16:11:38 +0200 | [diff] [blame] | 228 | elif mode == "nodestrap-nocerts": |
| 229 | return nodestrap(sys.argv[2:], nocerts=True) |
Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 230 | elif mode == "admincreds": |
| 231 | return admincreds(sys.argv[2:]) |
Serge Bazanski | e7fca3a | 2020-09-25 20:23:53 +0000 | [diff] [blame] | 232 | elif mode == "smoketest": |
| 233 | sys.stdout.write("Smoke test passed.") |
| 234 | return 0 |
Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 235 | else: |
| 236 | usage() |
| 237 | return 1 |
| 238 | |
| 239 | if __name__ == '__main__': |
| 240 | sys.exit(main() or 0) |