Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 1 | #!/usr/bin/env python |
| 2 | |
| 3 | from builtins import object |
| 4 | |
| 5 | import datetime |
| 6 | from io import BytesIO |
| 7 | import json |
| 8 | import logging |
| 9 | import os |
| 10 | import tempfile |
| 11 | import subprocess |
| 12 | import sys |
| 13 | |
| 14 | from cryptography import x509 |
| 15 | from cryptography.hazmat.backends import default_backend |
| 16 | import fabric |
| 17 | |
| 18 | from tools import secretstore |
| 19 | |
| 20 | import ca |
| 21 | |
| 22 | |
| 23 | local_root = os.getenv('hscloud_root') |
| 24 | if local_root is None: |
| 25 | raise Exception("Please source env.sh") |
| 26 | |
| 27 | |
| 28 | cluster = 'k0.hswaw.net' |
| 29 | remote_root = '/opt/hscloud' |
| 30 | ss = secretstore.SecretStore( |
| 31 | plain_root=os.path.join(local_root, 'cluster/secrets/plain'), |
| 32 | cipher_root=os.path.join(local_root, 'cluster/secrets/cipher')) |
| 33 | |
| 34 | |
| 35 | logger = logging.getLogger() |
| 36 | logger.setLevel(logging.INFO) |
| 37 | formatter = logging.Formatter('%(levelname)s - %(message)s') |
| 38 | sh = logging.StreamHandler() |
| 39 | sh.setFormatter(formatter) |
| 40 | logger.addHandler(sh) |
| 41 | |
| 42 | |
| 43 | |
| 44 | def pki_config(key): |
| 45 | raw = subprocess.check_output([ |
| 46 | 'nix', 'eval', '--raw', |
Sergiusz Bazanski | 116da98 | 2019-07-21 15:53:20 +0200 | [diff] [blame] | 47 | '( (import ' + local_root + '/cluster/nix/toplevel.nix ).pki.' + key + '.json )', |
Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 48 | ]) |
| 49 | return json.loads(raw) |
| 50 | |
| 51 | |
| 52 | def _file_exists(c, filename): |
| 53 | res = c.run('stat "{}"'.format(filename), warn=True, hide=True) |
| 54 | return res.exited == 0 |
| 55 | |
| 56 | |
| 57 | def configure_k8s(username, ca, cert, key): |
| 58 | subprocess.check_call([ |
| 59 | 'kubectl', 'config', |
Sergiusz Bazanski | b13b7ff | 2019-08-29 20:12:24 +0200 | [diff] [blame] | 60 | 'set-cluster', 'admin.' + cluster, |
Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 61 | '--certificate-authority=' + ca, |
| 62 | '--embed-certs=true', |
| 63 | '--server=https://' + cluster + ':4001', |
| 64 | ]) |
| 65 | subprocess.check_call([ |
| 66 | 'kubectl', 'config', |
| 67 | 'set-credentials', username, |
| 68 | '--client-certificate=' + cert, |
| 69 | '--client-key=' + key, |
| 70 | '--embed-certs=true', |
| 71 | ]) |
| 72 | subprocess.check_call([ |
| 73 | 'kubectl', 'config', |
Sergiusz Bazanski | b13b7ff | 2019-08-29 20:12:24 +0200 | [diff] [blame] | 74 | 'set-context', 'admin.' + cluster, |
| 75 | '--cluster=' + 'admin.' + cluster, |
Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 76 | '--user=' + username, |
| 77 | ]) |
| 78 | subprocess.check_call([ |
| 79 | 'kubectl', 'config', |
Sergiusz Bazanski | b13b7ff | 2019-08-29 20:12:24 +0200 | [diff] [blame] | 80 | 'use-context', 'admin.' + cluster, |
Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 81 | ]) |
| 82 | |
| 83 | |
| 84 | def admincreds(args): |
| 85 | if len(args) != 1: |
| 86 | sys.stderr.write("Usage: admincreds q3k\n") |
| 87 | return 1 |
| 88 | username = args[0] |
Sergiusz Bazanski | b13b7ff | 2019-08-29 20:12:24 +0200 | [diff] [blame] | 89 | print("") |
| 90 | print("WARNING WARNING WARNING WARNING WARNING WARNING") |
| 91 | print("===============================================") |
| 92 | print("") |
| 93 | print("You are requesting ADMIN credentials.") |
| 94 | print("") |
| 95 | print("You likely shouldn't be doing this, and") |
| 96 | print("instead should be using `prodaccess`.") |
| 97 | print("") |
| 98 | print("===============================================") |
| 99 | print("WARNING WARNING WARNING WARNING WARNING WARNING") |
| 100 | print("") |
Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 101 | |
| 102 | ## Make kube certificates. |
| 103 | certs_root = os.path.join(local_root, 'cluster/certs') |
| 104 | ca_kube = ca.CA(ss, certs_root, 'kube', 'kubernetes main CA') |
| 105 | |
| 106 | local_key = os.path.join(local_root, '.kubectl/admin.key') |
| 107 | local_crt = os.path.join(local_root, '.kubectl/admin.crt') |
| 108 | |
| 109 | kubectl = os.path.join(local_root, '.kubectl') |
| 110 | if not os.path.exists(kubectl): |
| 111 | os.mkdir(kubectl) |
| 112 | |
| 113 | generate_cert = False |
| 114 | if not os.path.exists(local_key): |
| 115 | generate_cert = True |
| 116 | |
| 117 | if os.path.exists(local_crt): |
| 118 | with open(local_crt, 'rb') as f: |
| 119 | b = f.read() |
| 120 | cert = x509.load_pem_x509_certificate(b, default_backend()) |
| 121 | delta = cert.not_valid_after - datetime.datetime.now() |
| 122 | logger.info("admin: existing cert expiry: {}".format(delta)) |
| 123 | if delta.total_seconds() < 3600 * 24: |
| 124 | logger.info("admin: expires soon, regenerating") |
| 125 | generate_cert = True |
| 126 | else: |
| 127 | generate_cert = True |
| 128 | |
| 129 | if not generate_cert: |
| 130 | return configure_k8s(username, ca_kube._cert, local_crt, local_key) |
| 131 | |
| 132 | key, csr = ca_kube.gen_key(hosts=['admin', username], o='system:masters', ou='Kube Admin Account') |
| 133 | crt = ca_kube.sign(csr) |
| 134 | |
| 135 | with open(local_key, 'w') as f: |
| 136 | f.write(key) |
| 137 | |
| 138 | with open(local_crt, 'w') as f: |
| 139 | f.write(crt) |
| 140 | |
| 141 | configure_k8s(username, ca_kube._cert, local_crt, local_key) |
| 142 | |
| 143 | |
Sergiusz Bazanski | c0fc3ee | 2019-06-20 16:11:38 +0200 | [diff] [blame] | 144 | def nodestrap(args, nocerts=False): |
Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 145 | if len(args) != 1: |
| 146 | sys.stderr.write("Usage: nodestrap bc01n01.hswaw.net\n") |
| 147 | return 1 |
| 148 | fqdn = args[0] |
| 149 | |
| 150 | logger.info("Nodestrapping {}...".format(fqdn)) |
| 151 | r = fabric.Connection('root@{}'.format(fqdn)) |
| 152 | |
Sergiusz Bazanski | c0fc3ee | 2019-06-20 16:11:38 +0200 | [diff] [blame] | 153 | if not nocerts: |
| 154 | cfg = dict((k, pki_config(k)) for k in [ |
| 155 | 'etcdPeer', 'etcd.server', 'etcd.kube' |
| 156 | ]) |
| 157 | certs_root = os.path.join(local_root, 'cluster/certs') |
Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 158 | |
Sergiusz Bazanski | c0fc3ee | 2019-06-20 16:11:38 +0200 | [diff] [blame] | 159 | # Make etcd peer certificate for node. |
| 160 | ca_etcd_peer = ca.CA(ss, certs_root, 'etcdpeer', 'etcd peer ca') |
| 161 | ca_etcd_peer.upload(r, cfg['etcdPeer']['ca']) |
| 162 | c = ca_etcd_peer.make_cert('etcdpeer-{}'.format(fqdn), hosts=[fqdn], ou='node etcd peer certificate') |
| 163 | c.upload_pki(r, cfg['etcdPeer']) |
Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 164 | |
Sergiusz Bazanski | c0fc3ee | 2019-06-20 16:11:38 +0200 | [diff] [blame] | 165 | # Make etcd server certificate for node and client certificate for kube. |
| 166 | ca_etcd = ca.CA(ss, certs_root, 'etcd', 'etcd ca') |
| 167 | ca_etcd.upload(r, cfg['etcd.server']['ca']) |
Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 168 | |
Sergiusz Bazanski | c0fc3ee | 2019-06-20 16:11:38 +0200 | [diff] [blame] | 169 | c = ca_etcd.make_cert('etcd-{}'.format(fqdn), hosts=[fqdn], ou='node etcd server certificate') |
| 170 | c.upload_pki(r, cfg['etcd.server']) |
Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 171 | |
Sergiusz Bazanski | c0fc3ee | 2019-06-20 16:11:38 +0200 | [diff] [blame] | 172 | c = ca_etcd.make_cert('etcd-kube', hosts=['kube'], ou='kube etcd client certificate') |
| 173 | c.upload_pki(r, cfg['etcd.kube']) |
Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 174 | |
Sergiusz Bazanski | c0fc3ee | 2019-06-20 16:11:38 +0200 | [diff] [blame] | 175 | # Make root etcd client (do not upload). |
| 176 | ca_etcd.make_cert('etcd-root', hosts=['root'], ou='root etcd client certificate') |
Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 177 | |
Sergiusz Bazanski | c0fc3ee | 2019-06-20 16:11:38 +0200 | [diff] [blame] | 178 | # Make calico etcd client (do not upload, used by jsonnet). |
| 179 | ca_etcd.make_cert('etcd-calico', hosts=['calico'], ou='root etcd client certificate') |
Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 180 | |
Sergiusz Bazanski | c0fc3ee | 2019-06-20 16:11:38 +0200 | [diff] [blame] | 181 | ## Make kube certificates. |
| 182 | ca_kube = ca.CA(ss, certs_root, 'kube', 'kubernetes main CA') |
Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 183 | |
Sergiusz Bazanski | b13b7ff | 2019-08-29 20:12:24 +0200 | [diff] [blame] | 184 | # Make prodvider intermediate CA. |
| 185 | c = ca_kube.make_cert('ca-kube-prodvider', o='Warsaw Hackerspace', ou='kubernetes prodvider intermediate', hosts=['kubernetes prodvider intermediate CA'], profile='intermediate') |
| 186 | c.ensure() |
| 187 | |
Sergiusz Bazanski | c0fc3ee | 2019-06-20 16:11:38 +0200 | [diff] [blame] | 188 | # Make kubelet certificate (per node). |
| 189 | c = ca_kube.make_cert('kube-kubelet-'+fqdn, o='system:nodes', ou='Kubelet', hosts=['system:node:'+fqdn, fqdn]) |
| 190 | c.upload_pki(r, pki_config('kube.kubelet')) |
Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 191 | |
Sergiusz Bazanski | c0fc3ee | 2019-06-20 16:11:38 +0200 | [diff] [blame] | 192 | # Make apiserver certificate. |
| 193 | c = ca_kube.make_cert('kube-apiserver', ou='Kubernetes API', hosts=[cluster, '10.10.12.1']) |
| 194 | c.upload_pki(r, pki_config('kube.apiserver'), concat_ca=True) |
Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 195 | |
Sergiusz Bazanski | c0fc3ee | 2019-06-20 16:11:38 +0200 | [diff] [blame] | 196 | # Make service accounts decryption key (as cert for consistency). |
| 197 | c = ca_kube.make_cert('kube-serviceaccounts', ou='Kubernetes Service Accounts Signer', hosts=['serviceaccounts']) |
| 198 | c.upload_pki(r, pki_config('kube.serviceaccounts')) |
Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 199 | |
Sergiusz Bazanski | c0fc3ee | 2019-06-20 16:11:38 +0200 | [diff] [blame] | 200 | # Make kube component certificates. |
| 201 | kube_components = ['controllermanager', 'scheduler', 'proxy'] |
| 202 | cfg = dict((k, pki_config('kube.' + k)) for k in kube_components) |
| 203 | for k in kube_components: |
| 204 | ca_kube.upload(r, cfg[k]['ca']) |
| 205 | # meh |
| 206 | if k == 'controllermanager': |
| 207 | o = 'system:kube-controller-manager' |
| 208 | else: |
| 209 | o = 'system:kube-'+k |
| 210 | ou = 'Kubernetes Component '+k |
| 211 | c = ca_kube.make_cert('kube-'+k, ou=ou, o=o, hosts=[o,]) |
| 212 | c.upload_pki(r, cfg[k]) |
Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 213 | |
Sergiusz Bazanski | c0fc3ee | 2019-06-20 16:11:38 +0200 | [diff] [blame] | 214 | ## Make kubefront certificates. |
| 215 | ca_kubefront = ca.CA(ss, certs_root, 'kubefront', 'kubernetes frontend CA') |
| 216 | ca_kubefront.upload(r, pki_config('kubeFront.apiserver')['ca']) |
| 217 | c = ca_kubefront.make_cert('kubefront-apiserver', ou='Kubernetes Frontend', hosts=['apiserver']) |
| 218 | c.upload_pki(r, pki_config('kubeFront.apiserver')) |
Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 219 | |
| 220 | # Upload NixOS config |
| 221 | for f in ['toplevel', 'cluster-configuration']: |
Sergiusz Bazanski | 116da98 | 2019-07-21 15:53:20 +0200 | [diff] [blame] | 222 | r.put(local=os.path.join(local_root, 'cluster/nix/{}.nix'.format(f)), |
Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 223 | remote='/etc/nixos/{}.nix'.format(f)) |
| 224 | |
| 225 | r.run('nixos-rebuild switch') |
| 226 | |
| 227 | |
| 228 | def usage(): |
Serge Bazanski | 2ce3676 | 2019-07-14 16:29:52 +0200 | [diff] [blame] | 229 | sys.stderr.write("Usage: clustercfg <nodestrap|admincreds|config>\n") |
Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 230 | |
| 231 | |
| 232 | def main(): |
| 233 | if len(sys.argv) < 2: |
| 234 | usage() |
| 235 | return 1 |
| 236 | |
| 237 | mode = sys.argv[1] |
| 238 | if mode == "nodestrap": |
| 239 | return nodestrap(sys.argv[2:]) |
Sergiusz Bazanski | c0fc3ee | 2019-06-20 16:11:38 +0200 | [diff] [blame] | 240 | elif mode == "nodestrap-nocerts": |
| 241 | return nodestrap(sys.argv[2:], nocerts=True) |
Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 242 | elif mode == "admincreds": |
| 243 | return admincreds(sys.argv[2:]) |
| 244 | elif mode == "config": |
| 245 | print('etcd peer:') |
| 246 | print(json.dumps(pki_config('etcdPeer'), indent=2)) |
Sergiusz Bazanski | 116da98 | 2019-07-21 15:53:20 +0200 | [diff] [blame] | 247 | print('etcd server:') |
| 248 | print(json.dumps(pki_config('etcd.server'), indent=2)) |
| 249 | print('etcd client (kube):') |
| 250 | print(json.dumps(pki_config('etcd.kube'), indent=2)) |
Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 251 | else: |
| 252 | usage() |
| 253 | return 1 |
| 254 | |
| 255 | if __name__ == '__main__': |
| 256 | sys.exit(main() or 0) |