Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 1 | #!/usr/bin/env python |
| 2 | |
| 3 | from builtins import object |
| 4 | |
| 5 | import datetime |
| 6 | from io import BytesIO |
| 7 | import json |
| 8 | import logging |
| 9 | import os |
| 10 | import tempfile |
| 11 | import subprocess |
| 12 | import sys |
| 13 | |
| 14 | from cryptography import x509 |
| 15 | from cryptography.hazmat.backends import default_backend |
| 16 | import fabric |
| 17 | |
| 18 | from tools import secretstore |
| 19 | |
| 20 | import ca |
| 21 | |
| 22 | |
| 23 | local_root = os.getenv('hscloud_root') |
| 24 | if local_root is None: |
| 25 | raise Exception("Please source env.sh") |
| 26 | |
| 27 | |
| 28 | cluster = 'k0.hswaw.net' |
Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 29 | ss = secretstore.SecretStore( |
| 30 | plain_root=os.path.join(local_root, 'cluster/secrets/plain'), |
| 31 | cipher_root=os.path.join(local_root, 'cluster/secrets/cipher')) |
| 32 | |
| 33 | |
| 34 | logger = logging.getLogger() |
| 35 | logger.setLevel(logging.INFO) |
| 36 | formatter = logging.Formatter('%(levelname)s - %(message)s') |
| 37 | sh = logging.StreamHandler() |
| 38 | sh.setFormatter(formatter) |
| 39 | logger.addHandler(sh) |
| 40 | |
| 41 | |
| 42 | |
Sergiusz Bazanski | c78cc13 | 2020-02-02 22:31:53 +0100 | [diff] [blame] | 43 | def pki_config(key, fqdn): |
| 44 | machine_name = fqdn.split('.')[0] |
Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 45 | raw = subprocess.check_output([ |
| 46 | 'nix', 'eval', '--raw', |
Sergiusz Bazanski | c78cc13 | 2020-02-02 22:31:53 +0100 | [diff] [blame] | 47 | '( ((import ' + local_root + '/cluster/nix/defs-cluster-k0.nix ) "' + machine_name + '").pki.' + key + '.json )', |
Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 48 | ]) |
| 49 | return json.loads(raw) |
| 50 | |
| 51 | |
| 52 | def _file_exists(c, filename): |
| 53 | res = c.run('stat "{}"'.format(filename), warn=True, hide=True) |
| 54 | return res.exited == 0 |
| 55 | |
| 56 | |
| 57 | def configure_k8s(username, ca, cert, key): |
| 58 | subprocess.check_call([ |
| 59 | 'kubectl', 'config', |
Sergiusz Bazanski | b13b7ff | 2019-08-29 20:12:24 +0200 | [diff] [blame] | 60 | 'set-cluster', 'admin.' + cluster, |
Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 61 | '--certificate-authority=' + ca, |
| 62 | '--embed-certs=true', |
| 63 | '--server=https://' + cluster + ':4001', |
| 64 | ]) |
| 65 | subprocess.check_call([ |
| 66 | 'kubectl', 'config', |
| 67 | 'set-credentials', username, |
| 68 | '--client-certificate=' + cert, |
| 69 | '--client-key=' + key, |
| 70 | '--embed-certs=true', |
| 71 | ]) |
| 72 | subprocess.check_call([ |
| 73 | 'kubectl', 'config', |
Sergiusz Bazanski | b13b7ff | 2019-08-29 20:12:24 +0200 | [diff] [blame] | 74 | 'set-context', 'admin.' + cluster, |
| 75 | '--cluster=' + 'admin.' + cluster, |
Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 76 | '--user=' + username, |
| 77 | ]) |
| 78 | subprocess.check_call([ |
| 79 | 'kubectl', 'config', |
Sergiusz Bazanski | b13b7ff | 2019-08-29 20:12:24 +0200 | [diff] [blame] | 80 | 'use-context', 'admin.' + cluster, |
Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 81 | ]) |
| 82 | |
| 83 | |
| 84 | def admincreds(args): |
| 85 | if len(args) != 1: |
| 86 | sys.stderr.write("Usage: admincreds q3k\n") |
| 87 | return 1 |
| 88 | username = args[0] |
Sergiusz Bazanski | b13b7ff | 2019-08-29 20:12:24 +0200 | [diff] [blame] | 89 | print("") |
| 90 | print("WARNING WARNING WARNING WARNING WARNING WARNING") |
| 91 | print("===============================================") |
| 92 | print("") |
| 93 | print("You are requesting ADMIN credentials.") |
| 94 | print("") |
| 95 | print("You likely shouldn't be doing this, and") |
| 96 | print("instead should be using `prodaccess`.") |
| 97 | print("") |
| 98 | print("===============================================") |
| 99 | print("WARNING WARNING WARNING WARNING WARNING WARNING") |
| 100 | print("") |
Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 101 | |
| 102 | ## Make kube certificates. |
| 103 | certs_root = os.path.join(local_root, 'cluster/certs') |
| 104 | ca_kube = ca.CA(ss, certs_root, 'kube', 'kubernetes main CA') |
| 105 | |
| 106 | local_key = os.path.join(local_root, '.kubectl/admin.key') |
| 107 | local_crt = os.path.join(local_root, '.kubectl/admin.crt') |
| 108 | |
| 109 | kubectl = os.path.join(local_root, '.kubectl') |
| 110 | if not os.path.exists(kubectl): |
| 111 | os.mkdir(kubectl) |
| 112 | |
| 113 | generate_cert = False |
| 114 | if not os.path.exists(local_key): |
| 115 | generate_cert = True |
| 116 | |
| 117 | if os.path.exists(local_crt): |
| 118 | with open(local_crt, 'rb') as f: |
| 119 | b = f.read() |
| 120 | cert = x509.load_pem_x509_certificate(b, default_backend()) |
| 121 | delta = cert.not_valid_after - datetime.datetime.now() |
| 122 | logger.info("admin: existing cert expiry: {}".format(delta)) |
| 123 | if delta.total_seconds() < 3600 * 24: |
| 124 | logger.info("admin: expires soon, regenerating") |
| 125 | generate_cert = True |
| 126 | else: |
| 127 | generate_cert = True |
| 128 | |
| 129 | if not generate_cert: |
| 130 | return configure_k8s(username, ca_kube._cert, local_crt, local_key) |
| 131 | |
| 132 | key, csr = ca_kube.gen_key(hosts=['admin', username], o='system:masters', ou='Kube Admin Account') |
| 133 | crt = ca_kube.sign(csr) |
| 134 | |
| 135 | with open(local_key, 'w') as f: |
| 136 | f.write(key) |
| 137 | |
| 138 | with open(local_crt, 'w') as f: |
| 139 | f.write(crt) |
| 140 | |
| 141 | configure_k8s(username, ca_kube._cert, local_crt, local_key) |
| 142 | |
| 143 | |
Sergiusz Bazanski | c0fc3ee | 2019-06-20 16:11:38 +0200 | [diff] [blame] | 144 | def nodestrap(args, nocerts=False): |
Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 145 | if len(args) != 1: |
| 146 | sys.stderr.write("Usage: nodestrap bc01n01.hswaw.net\n") |
| 147 | return 1 |
| 148 | fqdn = args[0] |
| 149 | |
| 150 | logger.info("Nodestrapping {}...".format(fqdn)) |
| 151 | r = fabric.Connection('root@{}'.format(fqdn)) |
| 152 | |
Sergiusz Bazanski | c0fc3ee | 2019-06-20 16:11:38 +0200 | [diff] [blame] | 153 | if not nocerts: |
Sergiusz Bazanski | c78cc13 | 2020-02-02 22:31:53 +0100 | [diff] [blame] | 154 | cfg = dict((k, pki_config(k, fqdn)) for k in [ |
Sergiusz Bazanski | c0fc3ee | 2019-06-20 16:11:38 +0200 | [diff] [blame] | 155 | 'etcdPeer', 'etcd.server', 'etcd.kube' |
| 156 | ]) |
| 157 | certs_root = os.path.join(local_root, 'cluster/certs') |
Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 158 | |
Sergiusz Bazanski | c0fc3ee | 2019-06-20 16:11:38 +0200 | [diff] [blame] | 159 | # Make etcd peer certificate for node. |
| 160 | ca_etcd_peer = ca.CA(ss, certs_root, 'etcdpeer', 'etcd peer ca') |
Sergiusz Bazanski | c78cc13 | 2020-02-02 22:31:53 +0100 | [diff] [blame] | 161 | ca_etcd_peer.make_cert('etcdpeer-{}'.format(fqdn), hosts=[fqdn], ou='node etcd peer certificate') |
Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 162 | |
Sergiusz Bazanski | c0fc3ee | 2019-06-20 16:11:38 +0200 | [diff] [blame] | 163 | # Make etcd server certificate for node and client certificate for kube. |
| 164 | ca_etcd = ca.CA(ss, certs_root, 'etcd', 'etcd ca') |
Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 165 | |
Sergiusz Bazanski | c78cc13 | 2020-02-02 22:31:53 +0100 | [diff] [blame] | 166 | ca_etcd.make_cert('etcd-{}'.format(fqdn), hosts=[fqdn], ou='node etcd server certificate') |
Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 167 | |
Sergiusz Bazanski | c78cc13 | 2020-02-02 22:31:53 +0100 | [diff] [blame] | 168 | ca_etcd.make_cert('etcd-kube', hosts=['kube'], ou='kube etcd client certificate') |
Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 169 | |
Sergiusz Bazanski | c0fc3ee | 2019-06-20 16:11:38 +0200 | [diff] [blame] | 170 | ca_etcd.make_cert('etcd-root', hosts=['root'], ou='root etcd client certificate') |
Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 171 | |
Sergiusz Bazanski | c0fc3ee | 2019-06-20 16:11:38 +0200 | [diff] [blame] | 172 | ca_etcd.make_cert('etcd-calico', hosts=['calico'], ou='root etcd client certificate') |
Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 173 | |
Sergiusz Bazanski | c0fc3ee | 2019-06-20 16:11:38 +0200 | [diff] [blame] | 174 | ## Make kube certificates. |
| 175 | ca_kube = ca.CA(ss, certs_root, 'kube', 'kubernetes main CA') |
Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 176 | |
Sergiusz Bazanski | b13b7ff | 2019-08-29 20:12:24 +0200 | [diff] [blame] | 177 | # Make prodvider intermediate CA. |
Sergiusz Bazanski | 0dcc702 | 2020-03-28 17:58:19 +0100 | [diff] [blame] | 178 | ca_kube.make_cert('ca-kube-prodvider', o='Warsaw Hackerspace', ou='kubernetes prodvider intermediate', hosts=['kubernetes prodvider intermediate CA'], profile='intermediate').ensure() |
Sergiusz Bazanski | b13b7ff | 2019-08-29 20:12:24 +0200 | [diff] [blame] | 179 | |
Sergiusz Bazanski | c0fc3ee | 2019-06-20 16:11:38 +0200 | [diff] [blame] | 180 | # Make kubelet certificate (per node). |
Sergiusz Bazanski | c78cc13 | 2020-02-02 22:31:53 +0100 | [diff] [blame] | 181 | ca_kube.make_cert('kube-kubelet-'+fqdn, o='system:nodes', ou='Kubelet', hosts=['system:node:'+fqdn, fqdn]) |
Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 182 | |
Sergiusz Bazanski | c0fc3ee | 2019-06-20 16:11:38 +0200 | [diff] [blame] | 183 | # Make apiserver certificate. |
Sergiusz Bazanski | c78cc13 | 2020-02-02 22:31:53 +0100 | [diff] [blame] | 184 | ca_kube.make_cert('kube-apiserver', ou='Kubernetes API', hosts=[cluster, 'kubernetes.default.svc.'+cluster, '10.10.12.1']) |
Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 185 | |
Sergiusz Bazanski | c0fc3ee | 2019-06-20 16:11:38 +0200 | [diff] [blame] | 186 | # Make service accounts decryption key (as cert for consistency). |
Sergiusz Bazanski | c78cc13 | 2020-02-02 22:31:53 +0100 | [diff] [blame] | 187 | ca_kube.make_cert('kube-serviceaccounts', ou='Kubernetes Service Accounts Signer', hosts=['serviceaccounts']) |
Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 188 | |
Sergiusz Bazanski | c0fc3ee | 2019-06-20 16:11:38 +0200 | [diff] [blame] | 189 | # Make kube component certificates. |
| 190 | kube_components = ['controllermanager', 'scheduler', 'proxy'] |
Sergiusz Bazanski | c78cc13 | 2020-02-02 22:31:53 +0100 | [diff] [blame] | 191 | cfg = dict((k, pki_config('kube.' + k, fqdn)) for k in kube_components) |
Sergiusz Bazanski | c0fc3ee | 2019-06-20 16:11:38 +0200 | [diff] [blame] | 192 | for k in kube_components: |
Sergiusz Bazanski | c0fc3ee | 2019-06-20 16:11:38 +0200 | [diff] [blame] | 193 | # meh |
| 194 | if k == 'controllermanager': |
| 195 | o = 'system:kube-controller-manager' |
| 196 | else: |
| 197 | o = 'system:kube-'+k |
| 198 | ou = 'Kubernetes Component '+k |
| 199 | c = ca_kube.make_cert('kube-'+k, ou=ou, o=o, hosts=[o,]) |
Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 200 | |
Sergiusz Bazanski | c0fc3ee | 2019-06-20 16:11:38 +0200 | [diff] [blame] | 201 | ## Make kubefront certificates. |
| 202 | ca_kubefront = ca.CA(ss, certs_root, 'kubefront', 'kubernetes frontend CA') |
Sergiusz Bazanski | c78cc13 | 2020-02-02 22:31:53 +0100 | [diff] [blame] | 203 | ca_kubefront.make_cert('kubefront-apiserver', ou='Kubernetes Frontend', hosts=['apiserver']) |
Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 204 | |
Sergiusz Bazanski | c78cc13 | 2020-02-02 22:31:53 +0100 | [diff] [blame] | 205 | subprocess.check_call(["nix", "run", |
| 206 | "-f", os.path.join(local_root, "cluster/nix/default.nix"), |
| 207 | "provision", |
Serge Bazanski | 36224c6 | 2020-10-10 14:54:09 +0200 | [diff] [blame] | 208 | "-c", "provision-{}".format(fqdn.split('.')[0])]) |
Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 209 | |
| 210 | |
| 211 | def usage(): |
Sergiusz Bazanski | c78cc13 | 2020-02-02 22:31:53 +0100 | [diff] [blame] | 212 | sys.stderr.write("Usage: clustercfg <nodestrap|admincreds>\n") |
Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 213 | |
| 214 | |
| 215 | def main(): |
| 216 | if len(sys.argv) < 2: |
| 217 | usage() |
| 218 | return 1 |
| 219 | |
| 220 | mode = sys.argv[1] |
| 221 | if mode == "nodestrap": |
| 222 | return nodestrap(sys.argv[2:]) |
Sergiusz Bazanski | c0fc3ee | 2019-06-20 16:11:38 +0200 | [diff] [blame] | 223 | elif mode == "nodestrap-nocerts": |
| 224 | return nodestrap(sys.argv[2:], nocerts=True) |
Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 225 | elif mode == "admincreds": |
| 226 | return admincreds(sys.argv[2:]) |
Serge Bazanski | e7fca3a | 2020-09-25 20:23:53 +0000 | [diff] [blame] | 227 | elif mode == "smoketest": |
| 228 | sys.stdout.write("Smoke test passed.") |
| 229 | return 0 |
Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 230 | else: |
| 231 | usage() |
| 232 | return 1 |
| 233 | |
| 234 | if __name__ == '__main__': |
| 235 | sys.exit(main() or 0) |