blob: 8ebad52c330ff7ea7cc25054f804ebf077310230 [file] [log] [blame]
Sergiusz Bazanskie53e39a2019-06-20 23:36:35 +02001# Deploy a CockroachDB cluster in secure mode.
2# This creates an N-node cluster based on a given static topology.
Patryk Jakuszew5dfd4cc2019-05-22 23:54:02 +02003
Sergiusz Bazanski662a3cd2019-06-20 19:45:03 +02004# Can be used either in own namespace or in an existing one:
5# crdb: cockroachdb.Cluster("q3kdb") {
6# cfg+: {
7# namespace: "q3k", // if not given, will create 'q3kdb' namespace
Sergiusz Bazanskie53e39a2019-06-20 23:36:35 +02008# topology: [
Sergiusz Bazanski184678b2019-06-22 02:07:41 +02009# { name: "a", node: "bc01n01.hswaw.net" },
10# { name: "b", node: "bc01n02.hswaw.net" },
11# { name: "c", node: "bc01n03.hswaw.net" },
Sergiusz Bazanskie53e39a2019-06-20 23:36:35 +020012# ],
13# hostPath: "/var/db/cockroach-q3k",
Sergiusz Bazanski662a3cd2019-06-20 19:45:03 +020014# },
15#},
16#
17# After the cluster is up, you can get to an administrateive SQL shell:
Sergiusz Bazanskidec401c2019-06-21 22:31:13 +020018# $ NS=q3k kubectl -n $NS exec -it $(kubectl -n $NS get pods -o name | grep client- | cut -d/ -f 2) /cockroach/cockroach sql
Sergiusz Bazanski662a3cd2019-06-20 19:45:03 +020019# root@q3kdb-cockroachdb-0.q3kdb-internal.q3k.svc.cluster.local:26257/defaultdb>
20#
21# Then, you can create some users and databases for applications:
22# defaultdb> CREATE DATABASE wykop;
23# defaultdb> CREATE USER bialkov PASSWORD hackme;
Sergiusz Bazanskie53e39a2019-06-20 23:36:35 +020024# defaultdb> GRANT ALL ON DATABASE wykop TO bialkov;
Sergiusz Bazanski662a3cd2019-06-20 19:45:03 +020025#
26# You are then ready to access the database via the public service from your application.
27#
28# PGCLIENTENCODING=utf8 psql -h q3kdb-public -p 26257 -U bialkov wykop
29# Password for user bialkov:
30# psql (10.9 (Ubuntu 10.9-0ubuntu0.18.04.1), server 9.5.0)
31# SSL connection (protocol: TLSv1.2, cipher: ECDHE-RSA-AES128-GCM-SHA256, bits: 128, compression: off)
32# Type "help" for help.
33#
34# wykop=>
35
36
Sergiusz Bazanski224a50b2019-06-20 16:41:54 +020037local kube = import "../../../kube/kube.libsonnet";
Sergiusz Bazanskib13b7ff2019-08-29 20:12:24 +020038local policies = import "../../../kube/policies.libsonnet";
Patryk Jakuszew5dfd4cc2019-05-22 23:54:02 +020039
40{
Sergiusz Bazanski662a3cd2019-06-20 19:45:03 +020041 Cluster(name): {
42 local cluster = self,
Patryk Jakuszew5dfd4cc2019-05-22 23:54:02 +020043
Sergiusz Bazanski662a3cd2019-06-20 19:45:03 +020044 cfg:: {
45 image: "cockroachdb/cockroach:v19.1.0",
Sergiusz Bazanskie53e39a2019-06-20 23:36:35 +020046
47 # Must be unique per cluster.
48 portServe: 26257,
49 portHttp: 8080,
50 hostPath: error "hostPath must be defined",
51 topology: error "topology must be defined",
Sergiusz Bazanski1fad2e52019-08-01 20:16:27 +020052 clients: [],
Sergiusz Bazanskie53e39a2019-06-20 23:36:35 +020053
Sergiusz Bazanski662a3cd2019-06-20 19:45:03 +020054 namespace: null,
55 ownNamespace: cluster.cfg.namespace == null,
Serge Bazanski509ab6e2020-07-30 22:43:20 +020056 extraDNS: [],
Patryk Jakuszew5dfd4cc2019-05-22 23:54:02 +020057 },
Patryk Jakuszew5dfd4cc2019-05-22 23:54:02 +020058
Sergiusz Bazanski662a3cd2019-06-20 19:45:03 +020059 namespaceName:: if cluster.cfg.namespace != null then cluster.cfg.namespace else name,
60
61 metadata:: {
62 namespace: cluster.namespaceName,
63 labels: {
64 "app.kubernetes.io/name": "cockroachdb",
65 "app.kubernetes.io/managed-by": "kubecfg",
66 "app.kubernetes.io/component": "cockroachdb",
Patryk Jakuszew5dfd4cc2019-05-22 23:54:02 +020067 },
68 },
69
Sergiusz Bazanski662a3cd2019-06-20 19:45:03 +020070 namespace: {
71 [if cluster.cfg.ownNamespace then "ns"]: kube.Namespace(cluster.namespaceName),
72 },
73
Sergiusz Bazanskib13b7ff2019-08-29 20:12:24 +020074 insecurePolicy: policies.AllowNamespaceInsecure(cluster.namespaceName),
75
Sergiusz Bazanski662a3cd2019-06-20 19:45:03 +020076 name(suffix):: if cluster.cfg.ownNamespace then suffix else name + "-" + suffix,
77
Sergiusz Bazanski662a3cd2019-06-20 19:45:03 +020078 pki: {
Sergiusz Bazanskie31d64f2019-10-02 20:59:26 +020079 selfSignedIssuer: kube.Issuer(cluster.name("selfsigned")) {
Sergiusz Bazanski662a3cd2019-06-20 19:45:03 +020080 metadata+: cluster.metadata,
81 spec: {
82 selfSigned: {},
Patryk Jakuszew5dfd4cc2019-05-22 23:54:02 +020083 },
Patryk Jakuszew5dfd4cc2019-05-22 23:54:02 +020084 },
Patryk Jakuszew5dfd4cc2019-05-22 23:54:02 +020085
Sergiusz Bazanskie31d64f2019-10-02 20:59:26 +020086 selfSignedKeypair: kube.Certificate(cluster.name("cluster-ca")) {
Sergiusz Bazanski662a3cd2019-06-20 19:45:03 +020087 metadata+: cluster.metadata,
88 spec: {
89 secretName: cluster.name("cluster-ca"),
90 duration: "43800h0m0s", // 5 years
91 isCA: true,
92 issuerRef: {
93 name: cluster.pki.selfSignedIssuer.metadata.name,
94 },
95 commonName: "cockroachdb-cluster-ca",
96 },
97 },
98
Sergiusz Bazanskie31d64f2019-10-02 20:59:26 +020099 clusterIssuer: kube.Issuer(cluster.name("cluster-ca")) {
Sergiusz Bazanski662a3cd2019-06-20 19:45:03 +0200100 metadata+: cluster.metadata,
101 spec: {
102 ca: {
103 secretName: cluster.pki.selfSignedKeypair.metadata.name,
104 },
105 },
106 },
107
Sergiusz Bazanskie31d64f2019-10-02 20:59:26 +0200108 nodeCertificate: kube.Certificate(cluster.name("node")) {
Sergiusz Bazanski662a3cd2019-06-20 19:45:03 +0200109 metadata+: cluster.metadata,
110 spec: {
111 secretName: "cockroachdb-node-cert",
112 duration: "43800h0m0s", // 5 years
113 issuerRef: {
114 name: cluster.pki.clusterIssuer.metadata.name,
115 },
116 commonName: "node",
117 dnsNames: [
Sergiusz Bazanski662a3cd2019-06-20 19:45:03 +0200118 cluster.publicService.metadata.name,
119 std.join(".", [cluster.publicService.metadata.name, cluster.metadata.namespace ]),
Sergiusz Bazanskie53e39a2019-06-20 23:36:35 +0200120 cluster.publicService.host,
Sergiusz Bazanski662a3cd2019-06-20 19:45:03 +0200121 std.join(".", [cluster.publicService.host, "cluster.local" ]),
Sergiusz Bazanskie53e39a2019-06-20 23:36:35 +0200122 std.join(".", [cluster.publicService.metadata.name, cluster.metadata.namespace ]),
123 ] + [
124 "%s.cluster.local" % s.service.host
125 for s in cluster.servers
Serge Bazanski509ab6e2020-07-30 22:43:20 +0200126 ] + cluster.cfg.extraDNS,
Sergiusz Bazanski662a3cd2019-06-20 19:45:03 +0200127 },
128 },
129
Sergiusz Bazanskie31d64f2019-10-02 20:59:26 +0200130 clientCertificate: kube.Certificate(cluster.name("client")) {
Sergiusz Bazanski662a3cd2019-06-20 19:45:03 +0200131 metadata+: cluster.metadata,
132 spec: {
133 secretName: cluster.name("client-certificate"),
134 duration: "43800h0m0s", // 5 years
135 issuerRef: {
136 name: cluster.pki.clusterIssuer.metadata.name,
137 },
138 commonName: "root",
Patryk Jakuszew5dfd4cc2019-05-22 23:54:02 +0200139 },
140 },
141 },
142
Sergiusz Bazanski662a3cd2019-06-20 19:45:03 +0200143 serviceAccount: kube.ServiceAccount(cluster.name("cockroachdb")) {
144 metadata+: cluster.metadata,
145 },
146
147 role: kube.Role(cluster.name("cockroachdb")) {
148 metadata+: cluster.metadata,
149 rules: [
150 {
151 apiGroups: [ "" ],
152 resources: [ "secrets" ],
153 verbs: [ "get" ],
Patryk Jakuszew5dfd4cc2019-05-22 23:54:02 +0200154 },
Sergiusz Bazanski662a3cd2019-06-20 19:45:03 +0200155 ],
156 },
157
158 roleBinding: kube.RoleBinding(cluster.name("cockroachdb")) {
159 metadata+: cluster.metadata,
160 roleRef_: cluster.role,
161 subjects_: [cluster.serviceAccount],
162 },
163
164 publicService: kube.Service(cluster.name("public")) {
165 metadata+: cluster.metadata,
Sergiusz Bazanskie53e39a2019-06-20 23:36:35 +0200166 target_pod:: cluster.servers[0].deploy.spec.template,
Sergiusz Bazanski662a3cd2019-06-20 19:45:03 +0200167 spec+: {
168 ports: [
Sergiusz Bazanskie53e39a2019-06-20 23:36:35 +0200169 { name: "grpc", port: cluster.cfg.portServe, targetPort: cluster.cfg.portServe },
170 { name: "http", port: cluster.cfg.portHttp, targetPort: cluster.cfg.portHttp },
Patryk Jakuszew5dfd4cc2019-05-22 23:54:02 +0200171 ],
Sergiusz Bazanskid5338922019-08-09 14:13:50 +0200172 type: "LoadBalancer",
Patryk Jakuszew5dfd4cc2019-05-22 23:54:02 +0200173 },
174 },
175
Sergiusz Bazanski662a3cd2019-06-20 19:45:03 +0200176 podDisruptionBudget: kube.PodDisruptionBudget(cluster.name("pod")) {
177 metadata+: cluster.metadata,
Patryk Jakuszew5dfd4cc2019-05-22 23:54:02 +0200178 spec: {
Sergiusz Bazanski662a3cd2019-06-20 19:45:03 +0200179 selector: {
180 matchLabels: {
181 "app.kubernetes.io/component": "cockroachdb",
182 },
Patryk Jakuszew5dfd4cc2019-05-22 23:54:02 +0200183 },
Sergiusz Bazanski662a3cd2019-06-20 19:45:03 +0200184 maxUnavailable: 1,
Patryk Jakuszew5dfd4cc2019-05-22 23:54:02 +0200185 },
186 },
Patryk Jakuszew5dfd4cc2019-05-22 23:54:02 +0200187
Sergiusz Bazanskie53e39a2019-06-20 23:36:35 +0200188 servers: [
189 {
190 local server = self,
191 service: kube.Service(cluster.name("server-" + el.name)) {
192 metadata+: cluster.metadata + {
193 annotations+: {
194 "service.alpha.kubernetes.io/tolerate-unready-endpoints": "true",
195 "prometheus.io/scrape": "true",
196 "prometheus.io/path": "_status/vars",
197 "prometheus.io/port": std.toString(cluster.cfg.portHttp),
198 },
199 },
200 target_pod:: server.deploy.spec.template,
Sergiusz Bazanski662a3cd2019-06-20 19:45:03 +0200201 spec+: {
Sergiusz Bazanskie53e39a2019-06-20 23:36:35 +0200202 ports: [
203 { name: "grpc", port: cluster.cfg.portServe, targetPort: cluster.cfg.portServe },
204 { name: "http", port: cluster.cfg.portHttp, targetPort: cluster.cfg.portHttp },
205 ],
206 publishNotReadyAddresses: true,
207 clusterIP: "None",
208 },
209 },
210 deploy: kube.Deployment(cluster.name("server-" + el.name)) {
211 metadata+: cluster.metadata {
212 labels+: {
213 "app.kubernetes.io/component": "server",
214 "kubernetes.hackerspace.pl/cockroachdb-server": el.name,
215 },
216 },
217 spec+: {
218 template+: {
219 metadata: server.deploy.metadata,
220 spec+: {
221 dnsPolicy: "ClusterFirst",
222 serviceAccountName: cluster.serviceAccount.metadata.name,
223 nodeSelector: {
224 "kubernetes.io/hostname": el.node,
225 },
226 containers: [
227 kube.Container("cockroachdb") {
228 image: cluster.cfg.image,
229 imagePullPolicy: "IfNotPresent",
230 resources: {
231 requests: {
232 cpu: "2",
233 memory: "6Gi",
Sergiusz Bazanski662a3cd2019-06-20 19:45:03 +0200234 },
Sergiusz Bazanskie53e39a2019-06-20 23:36:35 +0200235 limits: {
236 memory: "6Gi",
237 },
238 },
239 ports_: {
240 "grpc": { containerPort: cluster.cfg.portServe },
241 "http": { containerPort: cluster.cfg.portHttp },
242 },
243 livenessProbe: {
244 httpGet: {
245 path: "/health",
246 port: "http",
247 },
248 initialDelaySeconds: 30,
249 periodSeconds: 5,
250 },
251 readinessProbe: {
252 httpGet: {
253 path: "/health?ready=1",
254 port: "http",
255 },
256 initialDelaySeconds: 10,
257 periodSeconds: 5,
258 failureThreshold: 2,
259 },
260 volumeMounts: [
261 {
262 name: "datadir",
263 mountPath: "/cockroach/cockroach-data",
264 },
265 {
266 name: "certs",
267 mountPath: "/cockroach/cockroach-certs/node.crt",
268 subPath: "tls.crt",
269 },
270 {
271 name: "certs",
272 mountPath: "/cockroach/cockroach-certs/node.key",
273 subPath: "tls.key",
274 },
275 {
276 name: "certs",
277 mountPath: "/cockroach/cockroach-certs/ca.crt",
278 subPath: "ca.crt",
279 },
280 ],
281 env_: {
282 "COCKROACH_CERTS_DIR": "/cockroach/cockroach-certs",
283 },
284 command: [
285 "/cockroach/cockroach", "start",
286 "--logtostderr",
287 "--certs-dir", "/cockroach/cockroach-certs",
288 "--advertise-host", "%s.cluster.local" % server.service.host,
289 "--cache", "25%", "--max-sql-memory", "25%",
290 "--join", std.join(",", ["%s.cluster.local:%d" % [s.service.host, cluster.cfg.portServe] for s in cluster.servers]),
291 "--listen-addr=0.0.0.0:%d" % cluster.cfg.portServe,
292 "--http-addr=0.0.0.0:%d" % cluster.cfg.portHttp,
293 ],
294 },
295 ],
296 terminationGracePeriodSeconds: 60,
297 volumes: [
298 {
299 name: "datadir",
300 hostPath: {
301 path: cluster.cfg.hostPath,
302 },
303 },
304 {
305 name: "certs",
306 secret: {
307 secretName: cluster.pki.nodeCertificate.spec.secretName,
308 defaultMode: kube.parseOctal("400"),
Patryk Jakuszew5dfd4cc2019-05-22 23:54:02 +0200309 },
Sergiusz Bazanski662a3cd2019-06-20 19:45:03 +0200310 },
311 ],
312 },
313 },
Patryk Jakuszew5dfd4cc2019-05-22 23:54:02 +0200314 },
Sergiusz Bazanskie53e39a2019-06-20 23:36:35 +0200315 }
316 }
317 for el in cluster.cfg.topology
318 ],
Patryk Jakuszew5dfd4cc2019-05-22 23:54:02 +0200319
Sergiusz Bazanski662a3cd2019-06-20 19:45:03 +0200320 initJob: kube.Job(cluster.name("init")) {
321 metadata+: cluster.metadata,
322 spec: {
323 template: {
324 metadata+: cluster.metadata,
325 spec+: {
326 serviceAccountName: cluster.serviceAccount.metadata.name,
327 containers: [
328 kube.Container("cluster-init") {
329 image: cluster.cfg.image,
330 imagePullPolicy: "IfNotPresent",
331 env_: {
332 "COCKROACH_CERTS_DIR": "/cockroach/cockroach-certs",
333 },
334 command: [
335 "/bin/bash",
336 "-ecx",
Sergiusz Bazanskie53e39a2019-06-20 23:36:35 +0200337 "/cockroach/cockroach init --host=%s.cluster.local:%d" % [cluster.servers[0].service.host, cluster.cfg.portServe],
Sergiusz Bazanski662a3cd2019-06-20 19:45:03 +0200338 ],
339 volumeMounts: [
340 {
341 name: "certs",
342 mountPath: "/cockroach/cockroach-certs/ca.crt",
343 subPath: "ca.crt",
344 },
345 {
346 name: "certs",
347 mountPath: "/cockroach/cockroach-certs/client.root.crt",
348 subPath: "tls.crt",
349 },
350 {
351 name: "certs",
352 mountPath: "/cockroach/cockroach-certs/client.root.key",
353 subPath: "tls.key",
354 },
355 ],
356 },
357 ],
358 restartPolicy: "OnFailure",
359 volumes: [
360 {
361 name: "certs",
362 secret: {
363 secretName: cluster.pki.clientCertificate.spec.secretName,
364 defaultMode: kube.parseOctal("400")
365 }
366 },
367 ],
Patryk Jakuszew5dfd4cc2019-05-22 23:54:02 +0200368 },
Patryk Jakuszew5dfd4cc2019-05-22 23:54:02 +0200369 },
Sergiusz Bazanski662a3cd2019-06-20 19:45:03 +0200370 },
371 },
372
Sergiusz Bazanski1fad2e52019-08-01 20:16:27 +0200373 Client(name):: {
Sergiusz Bazanskie31d64f2019-10-02 20:59:26 +0200374 certificate: kube.Certificate(cluster.name("client-%s" % name)) {
Sergiusz Bazanski1fad2e52019-08-01 20:16:27 +0200375 metadata+: cluster.metadata,
376 spec: {
377 secretName: cluster.name("client-%s-certificate" % name),
378 duration: "43800h0m0s", // 5 years
379 issuerRef: {
380 name: cluster.pki.clusterIssuer.metadata.name,
381 },
382 commonName: name,
383 },
384 },
385 },
386
Sergiusz Bazanskidec401c2019-06-21 22:31:13 +0200387 client: kube.Deployment(cluster.name("client")) {
Sergiusz Bazanski662a3cd2019-06-20 19:45:03 +0200388 metadata+: cluster.metadata {
389 labels+: {
390 "app.kubernetes.io/component": "client",
Patryk Jakuszew5dfd4cc2019-05-22 23:54:02 +0200391 },
Sergiusz Bazanski662a3cd2019-06-20 19:45:03 +0200392 },
Sergiusz Bazanskidec401c2019-06-21 22:31:13 +0200393 spec+: {
394 template: {
395 metadata: cluster.client.metadata,
396 spec+: {
397 terminationGracePeriodSeconds: 5,
398 containers: [
399 kube.Container("cockroachdb-client") {
400 image: cluster.cfg.image,
401 env_: {
402 "COCKROACH_CERTS_DIR": "/cockroach/cockroach-certs",
403 "COCKROACH_HOST": cluster.publicService.host,
404 "COCKROACH_PORT": std.toString(cluster.cfg.portServe),
405 },
406 command: ["sleep", "2147483648"], //(FIXME) keep the client pod running indefinitely
407 volumeMounts: [
408 {
409 name: "certs",
410 mountPath: "/cockroach/cockroach-certs/ca.crt",
411 subPath: "ca.crt",
412 },
413 {
414 name: "certs",
415 mountPath: "/cockroach/cockroach-certs/client.root.crt",
416 subPath: "tls.crt",
417 },
418 {
419 name: "certs",
420 mountPath: "/cockroach/cockroach-certs/client.root.key",
421 subPath: "tls.key",
422 },
423 ],
Sergiusz Bazanski662a3cd2019-06-20 19:45:03 +0200424 },
Sergiusz Bazanskidec401c2019-06-21 22:31:13 +0200425 ],
426 volumes: [
Sergiusz Bazanski662a3cd2019-06-20 19:45:03 +0200427 {
428 name: "certs",
Sergiusz Bazanskidec401c2019-06-21 22:31:13 +0200429 secret: {
430 secretName: cluster.pki.clientCertificate.spec.secretName,
431 defaultMode: kube.parseOctal("400")
432 }
Sergiusz Bazanski662a3cd2019-06-20 19:45:03 +0200433 },
434 ],
435 },
Sergiusz Bazanskidec401c2019-06-21 22:31:13 +0200436 },
Sergiusz Bazanski662a3cd2019-06-20 19:45:03 +0200437 },
Patryk Jakuszew5dfd4cc2019-05-22 23:54:02 +0200438 },
439 },
Sergiusz Bazanski224a50b2019-06-20 16:41:54 +0200440}