Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 1 | #!/usr/bin/env python |
| 2 | |
| 3 | from builtins import object |
| 4 | |
| 5 | import datetime |
| 6 | from io import BytesIO |
| 7 | import json |
| 8 | import logging |
| 9 | import os |
| 10 | import tempfile |
| 11 | import subprocess |
| 12 | import sys |
| 13 | |
| 14 | from cryptography import x509 |
| 15 | from cryptography.hazmat.backends import default_backend |
| 16 | import fabric |
| 17 | |
| 18 | from tools import secretstore |
| 19 | |
| 20 | import ca |
| 21 | |
| 22 | |
| 23 | local_root = os.getenv('hscloud_root') |
| 24 | if local_root is None: |
| 25 | raise Exception("Please source env.sh") |
| 26 | |
| 27 | |
| 28 | cluster = 'k0.hswaw.net' |
| 29 | remote_root = '/opt/hscloud' |
| 30 | ss = secretstore.SecretStore( |
| 31 | plain_root=os.path.join(local_root, 'cluster/secrets/plain'), |
| 32 | cipher_root=os.path.join(local_root, 'cluster/secrets/cipher')) |
| 33 | |
| 34 | |
| 35 | logger = logging.getLogger() |
| 36 | logger.setLevel(logging.INFO) |
| 37 | formatter = logging.Formatter('%(levelname)s - %(message)s') |
| 38 | sh = logging.StreamHandler() |
| 39 | sh.setFormatter(formatter) |
| 40 | logger.addHandler(sh) |
| 41 | |
| 42 | |
| 43 | |
| 44 | def pki_config(key): |
| 45 | raw = subprocess.check_output([ |
| 46 | 'nix', 'eval', '--raw', |
| 47 | '( (import ' + local_root + '/nix/toplevel.nix ).pki.' + key + '.json )', |
| 48 | ]) |
| 49 | return json.loads(raw) |
| 50 | |
| 51 | |
| 52 | def _file_exists(c, filename): |
| 53 | res = c.run('stat "{}"'.format(filename), warn=True, hide=True) |
| 54 | return res.exited == 0 |
| 55 | |
| 56 | |
| 57 | def configure_k8s(username, ca, cert, key): |
| 58 | subprocess.check_call([ |
| 59 | 'kubectl', 'config', |
| 60 | 'set-cluster', cluster, |
| 61 | '--certificate-authority=' + ca, |
| 62 | '--embed-certs=true', |
| 63 | '--server=https://' + cluster + ':4001', |
| 64 | ]) |
| 65 | subprocess.check_call([ |
| 66 | 'kubectl', 'config', |
| 67 | 'set-credentials', username, |
| 68 | '--client-certificate=' + cert, |
| 69 | '--client-key=' + key, |
| 70 | '--embed-certs=true', |
| 71 | ]) |
| 72 | subprocess.check_call([ |
| 73 | 'kubectl', 'config', |
| 74 | 'set-context', cluster, |
| 75 | '--cluster=' + cluster, |
| 76 | '--user=' + username, |
| 77 | ]) |
| 78 | subprocess.check_call([ |
| 79 | 'kubectl', 'config', |
| 80 | 'use-context', cluster, |
| 81 | ]) |
| 82 | |
| 83 | |
| 84 | def admincreds(args): |
| 85 | if len(args) != 1: |
| 86 | sys.stderr.write("Usage: admincreds q3k\n") |
| 87 | return 1 |
| 88 | username = args[0] |
| 89 | |
| 90 | ## Make kube certificates. |
| 91 | certs_root = os.path.join(local_root, 'cluster/certs') |
| 92 | ca_kube = ca.CA(ss, certs_root, 'kube', 'kubernetes main CA') |
| 93 | |
| 94 | local_key = os.path.join(local_root, '.kubectl/admin.key') |
| 95 | local_crt = os.path.join(local_root, '.kubectl/admin.crt') |
| 96 | |
| 97 | kubectl = os.path.join(local_root, '.kubectl') |
| 98 | if not os.path.exists(kubectl): |
| 99 | os.mkdir(kubectl) |
| 100 | |
| 101 | generate_cert = False |
| 102 | if not os.path.exists(local_key): |
| 103 | generate_cert = True |
| 104 | |
| 105 | if os.path.exists(local_crt): |
| 106 | with open(local_crt, 'rb') as f: |
| 107 | b = f.read() |
| 108 | cert = x509.load_pem_x509_certificate(b, default_backend()) |
| 109 | delta = cert.not_valid_after - datetime.datetime.now() |
| 110 | logger.info("admin: existing cert expiry: {}".format(delta)) |
| 111 | if delta.total_seconds() < 3600 * 24: |
| 112 | logger.info("admin: expires soon, regenerating") |
| 113 | generate_cert = True |
| 114 | else: |
| 115 | generate_cert = True |
| 116 | |
| 117 | if not generate_cert: |
| 118 | return configure_k8s(username, ca_kube._cert, local_crt, local_key) |
| 119 | |
| 120 | key, csr = ca_kube.gen_key(hosts=['admin', username], o='system:masters', ou='Kube Admin Account') |
| 121 | crt = ca_kube.sign(csr) |
| 122 | |
| 123 | with open(local_key, 'w') as f: |
| 124 | f.write(key) |
| 125 | |
| 126 | with open(local_crt, 'w') as f: |
| 127 | f.write(crt) |
| 128 | |
| 129 | configure_k8s(username, ca_kube._cert, local_crt, local_key) |
| 130 | |
| 131 | |
Sergiusz Bazanski | c0fc3ee | 2019-06-20 16:11:38 +0200 | [diff] [blame] | 132 | def nodestrap(args, nocerts=False): |
Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 133 | if len(args) != 1: |
| 134 | sys.stderr.write("Usage: nodestrap bc01n01.hswaw.net\n") |
| 135 | return 1 |
| 136 | fqdn = args[0] |
| 137 | |
| 138 | logger.info("Nodestrapping {}...".format(fqdn)) |
| 139 | r = fabric.Connection('root@{}'.format(fqdn)) |
| 140 | |
Sergiusz Bazanski | c0fc3ee | 2019-06-20 16:11:38 +0200 | [diff] [blame] | 141 | if not nocerts: |
| 142 | cfg = dict((k, pki_config(k)) for k in [ |
| 143 | 'etcdPeer', 'etcd.server', 'etcd.kube' |
| 144 | ]) |
| 145 | certs_root = os.path.join(local_root, 'cluster/certs') |
Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 146 | |
Sergiusz Bazanski | c0fc3ee | 2019-06-20 16:11:38 +0200 | [diff] [blame] | 147 | # Make etcd peer certificate for node. |
| 148 | ca_etcd_peer = ca.CA(ss, certs_root, 'etcdpeer', 'etcd peer ca') |
| 149 | ca_etcd_peer.upload(r, cfg['etcdPeer']['ca']) |
| 150 | c = ca_etcd_peer.make_cert('etcdpeer-{}'.format(fqdn), hosts=[fqdn], ou='node etcd peer certificate') |
| 151 | c.upload_pki(r, cfg['etcdPeer']) |
Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 152 | |
Sergiusz Bazanski | c0fc3ee | 2019-06-20 16:11:38 +0200 | [diff] [blame] | 153 | # Make etcd server certificate for node and client certificate for kube. |
| 154 | ca_etcd = ca.CA(ss, certs_root, 'etcd', 'etcd ca') |
| 155 | ca_etcd.upload(r, cfg['etcd.server']['ca']) |
Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 156 | |
Sergiusz Bazanski | c0fc3ee | 2019-06-20 16:11:38 +0200 | [diff] [blame] | 157 | c = ca_etcd.make_cert('etcd-{}'.format(fqdn), hosts=[fqdn], ou='node etcd server certificate') |
| 158 | c.upload_pki(r, cfg['etcd.server']) |
Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 159 | |
Sergiusz Bazanski | c0fc3ee | 2019-06-20 16:11:38 +0200 | [diff] [blame] | 160 | c = ca_etcd.make_cert('etcd-kube', hosts=['kube'], ou='kube etcd client certificate') |
| 161 | c.upload_pki(r, cfg['etcd.kube']) |
Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 162 | |
Sergiusz Bazanski | c0fc3ee | 2019-06-20 16:11:38 +0200 | [diff] [blame] | 163 | # Make root etcd client (do not upload). |
| 164 | ca_etcd.make_cert('etcd-root', hosts=['root'], ou='root etcd client certificate') |
Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 165 | |
Sergiusz Bazanski | c0fc3ee | 2019-06-20 16:11:38 +0200 | [diff] [blame] | 166 | # Make calico etcd client (do not upload, used by jsonnet). |
| 167 | ca_etcd.make_cert('etcd-calico', hosts=['calico'], ou='root etcd client certificate') |
Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 168 | |
Sergiusz Bazanski | c0fc3ee | 2019-06-20 16:11:38 +0200 | [diff] [blame] | 169 | ## Make kube certificates. |
| 170 | ca_kube = ca.CA(ss, certs_root, 'kube', 'kubernetes main CA') |
Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 171 | |
Sergiusz Bazanski | c0fc3ee | 2019-06-20 16:11:38 +0200 | [diff] [blame] | 172 | # Make kubelet certificate (per node). |
| 173 | c = ca_kube.make_cert('kube-kubelet-'+fqdn, o='system:nodes', ou='Kubelet', hosts=['system:node:'+fqdn, fqdn]) |
| 174 | c.upload_pki(r, pki_config('kube.kubelet')) |
Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 175 | |
Sergiusz Bazanski | c0fc3ee | 2019-06-20 16:11:38 +0200 | [diff] [blame] | 176 | # Make apiserver certificate. |
| 177 | c = ca_kube.make_cert('kube-apiserver', ou='Kubernetes API', hosts=[cluster, '10.10.12.1']) |
| 178 | c.upload_pki(r, pki_config('kube.apiserver'), concat_ca=True) |
Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 179 | |
Sergiusz Bazanski | c0fc3ee | 2019-06-20 16:11:38 +0200 | [diff] [blame] | 180 | # Make service accounts decryption key (as cert for consistency). |
| 181 | c = ca_kube.make_cert('kube-serviceaccounts', ou='Kubernetes Service Accounts Signer', hosts=['serviceaccounts']) |
| 182 | c.upload_pki(r, pki_config('kube.serviceaccounts')) |
Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 183 | |
Sergiusz Bazanski | c0fc3ee | 2019-06-20 16:11:38 +0200 | [diff] [blame] | 184 | # Make kube component certificates. |
| 185 | kube_components = ['controllermanager', 'scheduler', 'proxy'] |
| 186 | cfg = dict((k, pki_config('kube.' + k)) for k in kube_components) |
| 187 | for k in kube_components: |
| 188 | ca_kube.upload(r, cfg[k]['ca']) |
| 189 | # meh |
| 190 | if k == 'controllermanager': |
| 191 | o = 'system:kube-controller-manager' |
| 192 | else: |
| 193 | o = 'system:kube-'+k |
| 194 | ou = 'Kubernetes Component '+k |
| 195 | c = ca_kube.make_cert('kube-'+k, ou=ou, o=o, hosts=[o,]) |
| 196 | c.upload_pki(r, cfg[k]) |
Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 197 | |
Sergiusz Bazanski | c0fc3ee | 2019-06-20 16:11:38 +0200 | [diff] [blame] | 198 | ## Make kubefront certificates. |
| 199 | ca_kubefront = ca.CA(ss, certs_root, 'kubefront', 'kubernetes frontend CA') |
| 200 | ca_kubefront.upload(r, pki_config('kubeFront.apiserver')['ca']) |
| 201 | c = ca_kubefront.make_cert('kubefront-apiserver', ou='Kubernetes Frontend', hosts=['apiserver']) |
| 202 | c.upload_pki(r, pki_config('kubeFront.apiserver')) |
Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 203 | |
| 204 | # Upload NixOS config |
| 205 | for f in ['toplevel', 'cluster-configuration']: |
| 206 | r.put(local=os.path.join(local_root, 'nix/{}.nix'.format(f)), |
| 207 | remote='/etc/nixos/{}.nix'.format(f)) |
| 208 | |
| 209 | r.run('nixos-rebuild switch') |
| 210 | |
| 211 | |
| 212 | def usage(): |
Serge Bazanski | 2ce3676 | 2019-07-14 16:29:52 +0200 | [diff] [blame] | 213 | sys.stderr.write("Usage: clustercfg <nodestrap|admincreds|config>\n") |
Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 214 | |
| 215 | |
| 216 | def main(): |
| 217 | if len(sys.argv) < 2: |
| 218 | usage() |
| 219 | return 1 |
| 220 | |
| 221 | mode = sys.argv[1] |
| 222 | if mode == "nodestrap": |
| 223 | return nodestrap(sys.argv[2:]) |
Sergiusz Bazanski | c0fc3ee | 2019-06-20 16:11:38 +0200 | [diff] [blame] | 224 | elif mode == "nodestrap-nocerts": |
| 225 | return nodestrap(sys.argv[2:], nocerts=True) |
Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 226 | elif mode == "admincreds": |
| 227 | return admincreds(sys.argv[2:]) |
| 228 | elif mode == "config": |
| 229 | print('etcd peer:') |
| 230 | print(json.dumps(pki_config('etcdPeer'), indent=2)) |
| 231 | print('etcd client:') |
| 232 | print(json.dumps(pki_config('etcdClient'), indent=2)) |
| 233 | else: |
| 234 | usage() |
| 235 | return 1 |
| 236 | |
| 237 | if __name__ == '__main__': |
| 238 | sys.exit(main() or 0) |