Gitiles
Code Review Sign In
gerrit.hackerspace.pl / hscloud / 13bb1bf4e3e557d43d8ce4347bf711b8439d6b5e / . / cluster / kube / lib / rook.libsonnet
blob: 8aa51a725218f8a27e1ac7f2c346a979f5246999 [file] [log] [blame]
Sergiusz Bazanskib7fcc672019-04-01 18:40:50 +0200[diff] [blame]1# Deploy Rook/Ceph Operator
2
3local kube = import "../../../kube/kube.libsonnet";
4
5{
Sergiusz Bazanskic6da1272019-04-02 00:06:13 +0200[diff] [blame]6 Operator: {
Sergiusz Bazanskib7fcc672019-04-01 18:40:50 +0200[diff] [blame]7 local env = self,
8 local cfg = env.cfg,
9 cfg:: {
Sergiusz Bazanski71a21c72019-08-30 21:30:42 +0200[diff] [blame]10 image: "rook/ceph:v1.0.5",
Sergiusz Bazanskib7fcc672019-04-01 18:40:50 +0200[diff] [blame]11 namespace: "rook-ceph-system",
12 },
13
14 metadata:: {
15 namespace: cfg.namespace,
16 labels: {
17 "operator": "rook",
18 "storage-backend": "ceph",
19 },
20 },
21
22 namespace: kube.Namespace(cfg.namespace),
23
24 crds: {
25 cephclusters: kube.CustomResourceDefinition("ceph.rook.io", "v1", "CephCluster") {
26 spec+: {
27 additionalPrinterColumns: [
28 { name: "DataDirHostPath", type: "string", description: "Directory used on the K8s nodes", JSONPath: ".spec.dataDirHostPath" },
29 { name: "MonCount", type: "string", description: "Number of MONs", JSONPath: ".spec.mon.count" },
30 { name: "Age", type: "date", JSONPath: ".metadata.creationTimestamp" },
31 { name: "State", type: "string", description: "Current State", JSONPath: ".status.state" },
32 ],
33 validation: {
34 # Converted from official operator YAML
35 "openAPIV3Schema": {
36 "properties": {
37 "spec": {
38 "properties": {
39 "cephVersion": {
40 "properties": {
41 "allowUnsupported": {
42 "type": "boolean"
43 },
44 "image": {
45 "type": "string"
46 },
47 "name": {
48 "pattern": "^(luminous|mimic|nautilus)$",
49 "type": "string"
50 }
51 }
52 },
53 "dashboard": {
54 "properties": {
55 "enabled": {
56 "type": "boolean"
57 },
58 "urlPrefix": {
59 "type": "string"
60 },
61 "port": {
62 "type": "integer"
63 }
64 }
65 },
66 "dataDirHostPath": {
67 "pattern": "^/(\\S+)",
68 "type": "string"
69 },
70 "mon": {
71 "properties": {
72 "allowMultiplePerNode": {
73 "type": "boolean"
74 },
75 "count": {
76 "maximum": 9,
77 "minimum": 1,
78 "type": "integer"
79 },
80 "preferredCount": {
81 "maximum": 9,
82 "minimum": 0,
83 "type": "integer"
84 }
85 },
86 "required": [
87 "count"
88 ]
89 },
90 "network": {
91 "properties": {
92 "hostNetwork": {
93 "type": "boolean"
94 }
95 }
96 },
97 "storage": {
98 "properties": {
99 "nodes": {
100 "items": {},
101 "type": "array"
102 },
103 "useAllDevices": {},
104 "useAllNodes": {
105 "type": "boolean"
106 }
107 }
108 }
109 },
110 "required": [
111 "mon"
112 ]
113 }
114 }
115 }
116 }
117 },
118 },
119 cephfilesystems: kube.CustomResourceDefinition("ceph.rook.io", "v1", "CephFilesystem") {
120 spec+: {
121 additionalPrinterColumns: [
122 { name: "MdsCount", type: "string", description: "Number of MDs", JSONPath: ".spec.metadataServer.activeCount" },
123 { name: "Age", type: "date", JSONPath: ".metadata.creationTimestamp" },
124 ],
125 },
126 },
127 cephnfses: kube.CustomResourceDefinition("ceph.rook.io", "v1", "CephNFS") {
128 spec+: {
129 names+: {
Sergiusz Bazanskic6da1272019-04-02 00:06:13 +0200[diff] [blame]130 plural: "cephnfses",
Sergiusz Bazanskib7fcc672019-04-01 18:40:50 +0200[diff] [blame]131 shortNames: ["nfs"],
132 },
133 },
134 },
135 cephobjectstores: kube.CustomResourceDefinition("ceph.rook.io", "v1", "CephObjectStore"),
136 cephobjectstoreusers: kube.CustomResourceDefinition("ceph.rook.io", "v1", "CephObjectStoreUser"),
137 cephblockpools: kube.CustomResourceDefinition("ceph.rook.io", "v1", "CephBlockPool"),
138 volumes: kube.CustomResourceDefinition("rook.io", "v1alpha2", "Volume") {
139 spec+: {
140 names+: {
141 shortNames: ["rv"],
142 },
143 },
144 },
145 },
146
Sergiusz Bazanskicdfafaf2019-04-01 19:16:18 +0200[diff] [blame]147 sa: kube.ServiceAccount("rook-ceph-system") {
148 metadata+: env.metadata,
149 },
150
151 crs: {
152 clusterMgmt: kube.ClusterRole("rook-ceph-cluster-mgmt") {
Sergiusz Bazanskic6da1272019-04-02 00:06:13 +0200[diff] [blame]153 metadata+: env.metadata { namespace:: null },
Sergiusz Bazanskicdfafaf2019-04-01 19:16:18 +0200[diff] [blame]154 rules: [
155 {
156 apiGroups: [""],
157 resources: ["secrets", "pods", "pods/log", "services", "configmaps"],
158 verbs: ["get", "list", "watch", "patch", "create", "update", "delete"],
159 },
160 {
161 apiGroups: ["apps"],
162 resources: ["deployments", "daemonsets", "replicasets"],
163 verbs: ["get", "list", "watch", "create", "update", "delete"],
164 },
165 ],
166 },
167 global: kube.ClusterRole("rook-ceph-global") {
Sergiusz Bazanskic6da1272019-04-02 00:06:13 +0200[diff] [blame]168 metadata+: env.metadata { namespace:: null },
Sergiusz Bazanskicdfafaf2019-04-01 19:16:18 +0200[diff] [blame]169 rules: [
170 {
171 apiGroups: [""],
172 resources: ["pods", "nodes", "nodes/proxy"],
173 verbs: ["get", "list", "watch"],
174 },
175 {
176 apiGroups: [""],
177 resources: ["events", "persistentvolumes", "persistentvolumeclaims", "endpoints"],
178 verbs: ["get", "list", "watch", "patch", "create", "update", "delete"],
179 },
180 {
181 apiGroups: ["storage.k8s.io"],
182 resources: ["storageclasses"],
183 verbs: ["get", "list", "watch", "create", "update", "delete"],
184 },
185 {
Sergiusz Bazanskic6da1272019-04-02 00:06:13 +0200[diff] [blame]186 apiGroups: ["batch"],
187 resources: ["jobs"],
188 verbs: ["get", "list", "watch", "create", "update", "delete"],
189 },
190 {
Sergiusz Bazanskicdfafaf2019-04-01 19:16:18 +0200[diff] [blame]191 apiGroups: ["ceph.rook.io"],
192 resources: ["*"],
193 verbs: ["*"],
194 },
195 {
196 apiGroups: ["rook.io"],
197 resources: ["*"],
198 verbs: ["*"],
199 },
200 ],
201 },
202 mgrCluster: kube.ClusterRole("rook-ceph-mgr-cluster") {
Sergiusz Bazanskic6da1272019-04-02 00:06:13 +0200[diff] [blame]203 metadata+: env.metadata { namespace:: null },
Sergiusz Bazanskicdfafaf2019-04-01 19:16:18 +0200[diff] [blame]204 rules: [
205 {
206 apiGroups: [""],
207 resources: ["configmaps", "nodes", "nodes/proxy"],
208 verbs: ["get", "list", "watch"],
209 },
210 ]
211 },
212 },
213
214 crb: kube.ClusterRoleBinding("ceph-rook-global") {
Sergiusz Bazanskic6da1272019-04-02 00:06:13 +0200[diff] [blame]215 metadata+: env.metadata { namespace:: null },
Sergiusz Bazanski13bb1bf2019-08-31 16:33:29 +0200[diff] [blame^]216 roleRef_: env.crs.global,
217 subjects_: [env.sa],
Sergiusz Bazanskicdfafaf2019-04-01 19:16:18 +0200[diff] [blame]218 },
219
220 role: kube.Role("ceph-rook-system") {
221 metadata+: env.metadata,
222 rules: [
223 {
224 apiGroups: [""],
225 resources: ["pods", "configmaps"],
226 verbs: ["get", "list", "watch", "patch", "create", "update", "delete"],
227 },
228 {
229 apiGroups: ["apps"],
230 resources: ["daemonsets"],
231 verbs: ["get", "list", "watch", "create", "update", "delete"],
232 },
233 ],
234 },
235
236 rb: kube.RoleBinding("ceph-rook-system") {
237 metadata+: env.metadata,
Sergiusz Bazanski13bb1bf2019-08-31 16:33:29 +0200[diff] [blame^]238 roleRef_: env.role,
239 subjects_: [env.sa],
Sergiusz Bazanskicdfafaf2019-04-01 19:16:18 +0200[diff] [blame]240 },
241
242 operator: kube.Deployment("rook-ceph-operator") {
243 metadata+: env.metadata,
244 spec+: {
245 template+: {
246 spec+: {
247 serviceAccountName: env.sa.metadata.name,
248 containers_: {
249 operator: kube.Container("rook-ceph-operator") {
250 image: cfg.image,
251 args: ["ceph", "operator"],
252 volumeMounts_: {
253 "rook-config": { mountPath: "/var/lib/rook" },
254 "default-config-dir": { mountPath: "/etc/ceph" },
255 },
256 env_: {
257 LIB_MODULES_DIR_PATH: "/run/current-system/kernel-modules/lib/modules/",
258 ROOK_ALLOW_MULTIPLE_FILESYSTEMS: "false",
Sergiusz Bazanski321fad92019-04-19 14:14:36 +0200[diff] [blame]259 ROOK_LOG_LEVEL: "INFO",
Sergiusz Bazanskicdfafaf2019-04-01 19:16:18 +0200[diff] [blame]260 ROOK_MON_HEALTHCHECK_INTERVAL: "45s",
261 ROOK_MON_OUT_TIMEOUT: "600s",
262 ROOK_DISCOVER_DEVICES_INTERVAL: "60m",
263 ROOK_HOSTPATH_REQUIRES_PRIVILEGED: "false",
264 ROOK_ENABLE_SELINUX_RELABELING: "true",
265 ROOK_ENABLE_FSGROUP: "true",
266 NODE_NAME: kube.FieldRef("spec.nodeName"),
267 POD_NAME: kube.FieldRef("metadata.name"),
268 POD_NAMESPACE: kube.FieldRef("metadata.namespace"),
269 },
270 },
271 },
272 volumes_: {
273 "rook-config": { emptyDir: {} },
274 "default-config-dir": { emptyDir: {} },
275 },
276 },
277 },
278 },
279 },
Sergiusz Bazanskib7fcc672019-04-01 18:40:50 +0200[diff] [blame]280 },
Sergiusz Bazanskic6da1272019-04-02 00:06:13 +0200[diff] [blame]281
282 // Create a new Ceph cluster in a new namespace.
283 Cluster(operator, name):: {
284 local cluster = self,
285 spec:: error "please define cluster spec",
286
287
288 metadata:: {
289 namespace: name,
290 },
291
292 name(suffix):: cluster.metadata.namespace + "-" + suffix,
293
294 namespace: kube.Namespace(cluster.metadata.namespace),
295
296 sa: {
297 // service accounts need to be hardcoded, see operator source.
298 osd: kube.ServiceAccount("rook-ceph-osd") {
299 metadata+: cluster.metadata,
300 },
301 mgr: kube.ServiceAccount("rook-ceph-mgr") {
302 metadata+: cluster.metadata,
303 },
304 },
305
306 roles: {
307 osd: kube.Role(cluster.name("osd")) {
308 metadata+: cluster.metadata,
309 rules: [
310 {
311 apiGroups: [""],
312 resources: ["configmaps"],
313 verbs: ["get", "list", "watch", "create", "update", "delete"],
314 }
315 ],
316 },
317 mgr: kube.Role(cluster.name("mgr")) {
318 metadata+: cluster.metadata,
319 rules: [
320 {
321 apiGroups: [""],
322 resources: ["pods", "services"],
323 verbs: ["get", "list", "watch"],
324 },
325 {
326 apiGroups: ["batch"],
327 resources: ["jobs"],
328 verbs: ["get", "list", "watch", "create", "update", "delete"],
329 },
330 {
331 apiGroups: ["ceph.rook.io"],
332 resources: ["*"],
333 verbs: ["*"],
334 },
335 ],
336 },
337 mgrSystem: kube.ClusterRole(cluster.name("mgr-system")) {
338 metadata+: cluster.metadata { namespace:: null },
339 rules: [
340 {
341 apiGroups: [""],
342 resources: ["configmaps"],
343 verbs: ["get", "list", "watch"],
344 }
345 ],
346 },
347 },
348
349 rbs: [
350 kube.RoleBinding(cluster.name(el.name)) {
351 metadata+: cluster.metadata,
Sergiusz Bazanski13bb1bf2019-08-31 16:33:29 +0200[diff] [blame^]352 roleRef_: el.role,
353 subjects_: [el.sa],
Sergiusz Bazanskic6da1272019-04-02 00:06:13 +0200[diff] [blame]354 },
355 for el in [
356 // Allow Operator SA to perform Cluster Mgmt in this namespace.
357 { name: "cluster-mgmt", role: operator.crs.clusterMgmt, sa: operator.sa },
Sergiusz Bazanski13bb1bf2019-08-31 16:33:29 +0200[diff] [blame^]358 { name: "osd", role: cluster.roles.osd, sa: cluster.sa.osd },
Sergiusz Bazanskic6da1272019-04-02 00:06:13 +0200[diff] [blame]359 { name: "mgr", role: cluster.roles.mgr, sa: cluster.sa.mgr },
360 { name: "mgr-cluster", role: operator.crs.mgrCluster, sa: cluster.sa.mgr },
361 ]
362 ],
363
364 mgrSystemRB: kube.RoleBinding(cluster.name("mgr-system")) {
365 metadata+: {
366 namespace: operator.cfg.namespace,
367 },
Sergiusz Bazanski13bb1bf2019-08-31 16:33:29 +0200[diff] [blame^]368 roleRef_: cluster.roles.mgrSystem,
369 subjects_: [cluster.sa.mgr],
Sergiusz Bazanskic6da1272019-04-02 00:06:13 +0200[diff] [blame]370 },
371
372 cluster: kube._Object("ceph.rook.io/v1", "CephCluster", name) {
373 metadata+: cluster.metadata,
374 spec: {
375 cephVersion: {
Sergiusz Bazanskied2e6702019-04-19 13:27:20 +0200[diff] [blame]376 # https://github.com/rook/rook/issues/2945#issuecomment-483964014
377 #image: "ceph/ceph:v13.2.5-20190319",
Sergiusz Bazanski71a21c72019-08-30 21:30:42 +0200[diff] [blame]378 image: "ceph/ceph:v14.2.2-20190830",
Sergiusz Bazanskied2e6702019-04-19 13:27:20 +0200[diff] [blame]379 allowUnsupported: true,
Sergiusz Bazanskic6da1272019-04-02 00:06:13 +0200[diff] [blame]380 },
381 dataDirHostPath: "/var/lib/rook",
382 dashboard: {
383 ssl: false,
384 enabled: true,
385 port: 8080,
386 },
387 } + cluster.spec,
388 },
389
390 dashboardService: kube.Service(cluster.name("dashboard")) {
391 metadata+: cluster.metadata,
392 spec: {
393 ports: [
Sergiusz Bazanski13bb1bf2019-08-31 16:33:29 +0200[diff] [blame^]394 { name: "dashboard", port: 80, targetPort: 8080, protocol: "TCP" },
Sergiusz Bazanskic6da1272019-04-02 00:06:13 +0200[diff] [blame]395 ],
396 selector: {
397 app: "rook-ceph-mgr",
398 rook_cluster: name,
399 },
400 type: "ClusterIP",
401 },
402 },
403
404 dashboardIngress: kube.Ingress(cluster.name("dashboard")) {
Piotr Dobrowolski3187c592019-04-02 14:44:04 +0200[diff] [blame]405 metadata+: cluster.metadata {
406 annotations+: {
407 "kubernetes.io/tls-acme": "true",
408 "certmanager.k8s.io/cluster-issuer": "letsencrypt-prod",
409 },
410 },
Sergiusz Bazanskic6da1272019-04-02 00:06:13 +0200[diff] [blame]411 spec+: {
Piotr Dobrowolski3187c592019-04-02 14:44:04 +0200[diff] [blame]412 tls: [
413 {
414 hosts: ["%s.hswaw.net" % name],
415 secretName: cluster.name("dashboard"),
416 },
417 ],
Sergiusz Bazanskic6da1272019-04-02 00:06:13 +0200[diff] [blame]418 rules: [
419 {
420 host: "%s.hswaw.net" % name,
421 http: {
422 paths: [
423 { path: "/", backend: cluster.dashboardService.name_port },
424 ]
425 },
426 }
427 ],
428 },
Sergiusz Bazanski13bb1bf2019-08-31 16:33:29 +0200[diff] [blame^]429 },
430
431 # Benji is a backup tool, external to rook, that we use for backing up
432 # RBDs.
433 benji: {
434 sa: kube.ServiceAccount(cluster.name("benji")) {
435 metadata+: cluster.metadata,
436 },
437
438 cr: kube.ClusterRole(cluster.name("benji")) {
439 rules: [
440 {
441 apiGroups: [""],
442 resources: [
443 "persistentvolumes",
444 "persistentvolumeclaims"
445 ],
446 verbs: ["list", "get"],
447 },
448 {
449 apiGroups: [""],
450 resources: [
451 "events",
452 ],
453 verbs: ["create", "update"],
454 },
455 ],
456 },
457
458 crb: kube.ClusterRoleBinding(cluster.name("benji")) {
459 roleRef_: cluster.benji.cr,
460 subjects_: [cluster.benji.sa],
461 },
462
463 config: kube.Secret(cluster.name("benji-config")) {
464 metadata+: cluster.metadata,
465 data_: {
466 "benji.yaml": std.manifestJson({
467 configurationVersion: '1',
468 databaseEngine: 'sqlite:////data/benji.sqlite',
469 defaultStorage: 'wasabi',
470 storages: [
471 {
472 name: "wasabi",
473 storageId: 1,
474 module: "s3",
475 configuration: cluster.spec.benji.s3Configuration {
476 activeTransforms: ["encrypt"],
477 },
478 },
479 ],
480 transforms: [
481 {
482 name: "encrypt",
483 module: "aes_256_gcm",
484 configuration: {
485 # not secret.
486 kdfSalt: "T2huZzZpcGhhaWM3QWVwaDhybzRhaDNhbzFpc2VpOWFobDNSZWVQaGVvTWV1bmVaYWVsNHRoYWg5QWVENHNoYWg0ZGFoN3Rlb3NvcHVuZzNpZXZpMm9vTG9vbmc1YWlmb0RlZXAwYmFobDlab294b2hjaG9odjRhbzFsYWkwYWk=",
487 kdfIterations: 2137,
488 password: cluster.spec.benji.encryptionPassword,
489 },
490 },
491 ],
492 ios: [
493 { name: pool, module: "rbd" }
494 for pool in cluster.spec.benji.pools
495 ],
496 }),
497 },
498 },
499
500 # Yes, Benji keeps data (backup metadata) on the ceph cluster that
501 # it backs up. However:
502 # - we add a command to benji-k8s to also copy over the sqlite
503 # database over to s3
504 # - benji can, in a pinch, restore without a database if a version
505 # is known: https://benji-backup.me/restore.html#restoring-without-a-database
506 data: kube.PersistentVolumeClaim(cluster.name("benji-data")) {
507 metadata+: cluster.metadata,
508 spec+: {
509 storageClassName: cluster.spec.benji.metadataStorageClass,
510 accessModes: [ "ReadWriteOnce" ],
511 resources: {
512 requests: {
513 storage: "1Gi",
514 },
515 },
516 },
517 },
518
519 # Extra scripts.
520 extrabins: kube.ConfigMap(cluster.name("benji-extrabins")) {
521 metadata+: cluster.metadata,
522 data: {
523 "metabackup.sh" : |||
524 # Make backups of sqlite3 metadata used by Benji.
525 # The backups live in the same bucket as backups, and the metabackups
526 # are named `metabackup-0..10`, where 0 is the newest backup. Any time
527 # this script is called, backups get shifted one way to the left (9 to 10,
528 # 8 to 9, etc). This ensures we have at least 10 backup replicas.
529
530 set -e
531
532 which s3cmd || pip install --upgrade s3cmd
533
534 AWS_ACCESS_KEY_ID=$(jq -r .storages[0].configuration.awsAccessKeyId < /etc/benji/benji.yaml)
535 AWS_SECRET_ACCESS_KEY=$(jq -r .storages[0].configuration.awsSecretAccessKey < /etc/benji/benji.yaml)
536 BUCKET=$(jq -r .storages[0].configuration.bucketName < /etc/benji/benji.yaml)
537
538 s3() {
539 s3cmd --host=s3.wasabisys.com \
540 "--host-bucket=%(bucket)s.s3.wasabisys.com" \
541 --region=eu-central-1 \
542 --access_key=$AWS_ACCESS_KEY_ID \
543 --secret_key=$AWS_SECRET_ACCESS_KEY \
544 "$@"
545 }
546
547 # Copy over old backups, if they exist.
548 for i in `seq 9 -1 0`; do
549 from="s3://$BUCKET/metabackup-$i.sqlite3"
550 to="s3://$BUCKET/metabackup-$((i+1)).sqlite3"
551
552 if [[ $(s3 ls $from | wc -l) -eq 0 ]]; then
553 echo "$from does not exist, skipping shift."
554 continue
555 fi
556 echo "Moving $from to $to..."
557 s3 mv $from $to
558 done
559
560 # Make new metabackup.
561 s3 put /data/benji.sqlite s3://$BUCKET/metabackup-0.sqlite3
562
563 |||,
564 "get-rook-creds.sh": |||
565 # Based on the Rook Toolbox /usr/local/bin/toolbox.sh script.
566 # Copyright 2016 The Rook Authors. All rights reserved.
567
568 CEPH_CONFIG="/etc/ceph/ceph.conf"
569 MON_CONFIG="/etc/rook/mon-endpoints"
570 KEYRING_FILE="/etc/ceph/keyring"
571
572 # create a ceph config file in its default location so ceph/rados tools can be used
573 # without specifying any arguments
574 write_endpoints() {
575 endpoints=$(cat ${MON_CONFIG})
576
577 # filter out the mon names
578 mon_endpoints=$(echo ${endpoints} | sed 's/[a-z]\+=//g')
579
580 # filter out the legacy mon names
581 mon_endpoints=$(echo ${mon_endpoints} | sed 's/rook-ceph-mon[0-9]\+=//g')
582
583 DATE=$(date)
584 echo "$DATE writing mon endpoints to ${CEPH_CONFIG}: ${endpoints}"
585 cat <<EOF > ${CEPH_CONFIG}
586 [global]
587 mon_host = ${mon_endpoints}
588
589 [client.admin]
590 keyring = ${KEYRING_FILE}
591 EOF
592 }
593
594 # watch the endpoints config file and update if the mon endpoints ever change
595 watch_endpoints() {
596 # get the timestamp for the target of the soft link
597 real_path=$(realpath ${MON_CONFIG})
598 initial_time=$(stat -c %Z ${real_path})
599 while true; do
600 real_path=$(realpath ${MON_CONFIG})
601 latest_time=$(stat -c %Z ${real_path})
602
603 if [[ "${latest_time}" != "${initial_time}" ]]; then
604 write_endpoints
605 initial_time=${latest_time}
606 fi
607 sleep 10
608 done
609 }
610
611 # create the keyring file
612 cat <<EOF > ${KEYRING_FILE}
613 [client.admin]
614 key = ${ROOK_ADMIN_SECRET}
615 EOF
616
617 # write the initial config file
618 write_endpoints
619
620 # continuously update the mon endpoints if they fail over
621 watch_endpoints &
622 |||
623 },
624 },
625
626 cronjob: kube.CronJob(cluster.name("benji")) {
627 metadata+: cluster.metadata,
628 spec+: { # CronJob Spec
629 schedule: "42 * * * *", # Hourly at 42 minute past.
630 jobTemplate+: {
631 spec+: { # Job Spec
632 selector:: null,
633 template+: {
634 spec+: { # PodSpec
635 serviceAccountName: cluster.benji.sa.metadata.name,
636 containers_: {
637 benji: kube.Container(cluster.name("benji")) {
638 # TODO(q3k): switch back to upstream after pull/52 goes in.
639 # Currently this is being built from github.com/q3k/benji.
640 # https://github.com/elemental-lf/benji/pull/52
641 image: "registry.k0.hswaw.net/q3k/benji-k8s:20190831-1351",
642 volumeMounts_: {
643 extrabins: { mountPath: "/usr/local/extrabins" },
644 monendpoints: { mountPath: "/etc/rook" },
645 benjiconfig: { mountPath: "/etc/benji" },
646 data: { mountPath: "/data" },
647 },
648 env_: {
649 ROOK_ADMIN_SECRET: { secretKeyRef: { name: "rook-ceph-mon", key: "admin-secret" }},
650 },
651 command: [
652 "bash", "-c", |||
653 bash /usr/local/extrabins/get-rook-creds.sh
654 benji-backup-pvc
655 benji-command enforce latest3,hours48,days7,months12
656 benji-command cleanup
657 bash /usr/local/extrabins/metabackup.sh
658 |||,
659 ],
660 },
661 },
662 volumes_: {
663 data: kube.PersistentVolumeClaimVolume(cluster.benji.data),
664 benjiconfig: kube.SecretVolume(cluster.benji.config),
665 extrabins: kube.ConfigMapVolume(cluster.benji.extrabins),
666 monendpoints: {
667 configMap: {
668 name: "rook-ceph-mon-endpoints",
669 items: [
670 { key: "data", path: "mon-endpoints" },
671 ],
672 },
673 },
674 },
675 },
676 },
677 },
678 },
679 },
680 },
681 },
Sergiusz Bazanski65f3b1d2019-04-02 01:05:38 +0200[diff] [blame]682 },
683
Sergiusz Bazanski36cc4fb2019-05-17 18:08:48 +0200[diff] [blame]684 ReplicatedBlockPool(cluster, name):: {
685 local pool = self,
Sergiusz Bazanskid07861b2019-08-08 17:48:25 +0200[diff] [blame]686 name:: name,
687
Sergiusz Bazanski36cc4fb2019-05-17 18:08:48 +0200[diff] [blame]688 spec:: error "spec must be specified",
689
690 pool: kube._Object("ceph.rook.io/v1", "CephBlockPool", name) {
691 metadata+: cluster.metadata,
692 spec: pool.spec,
693 },
694
695 storageClass: kube.StorageClass(name) {
696 provisioner: "ceph.rook.io/block",
697 parameters: {
698 blockPool: pool.pool.metadata.name,
699 clusterNamespace: pool.pool.metadata.namespace,
700 fstype: "ext4",
701 },
702 reclaimPolicy: "Retain",
703 },
704 },
705
Sergiusz Bazanski65f3b1d2019-04-02 01:05:38 +0200[diff] [blame]706 ECBlockPool(cluster, name):: {
707 local pool = self,
Sergiusz Bazanski4d61d202019-07-21 16:56:41 +0200[diff] [blame]708 name:: name,
709
Sergiusz Bazanski65f3b1d2019-04-02 01:05:38 +0200[diff] [blame]710 spec:: error "spec must be specified",
711
712 pool: kube._Object("ceph.rook.io/v1", "CephBlockPool", name) {
713 metadata+: cluster.metadata,
714 spec: pool.spec,
715 },
716 metapool: kube._Object("ceph.rook.io/v1", "CephBlockPool", name + "-metadata") {
717 metadata+: cluster.metadata,
718 spec: {
719 failureDomain: "host",
720 replicated: {
721 size: 3,
722 },
723 },
724 },
725
726 storageClass: kube.StorageClass(name) {
727 provisioner: "ceph.rook.io/block",
728 parameters: {
729 blockPool: pool.metapool.metadata.name,
730 dataBlockPool: pool.pool.metadata.name,
731 clusterNamespace: pool.pool.metadata.namespace,
732 fstype: "ext4",
733 },
734 reclaimPolicy: "Retain",
Sergiusz Bazanskic6da1272019-04-02 00:06:13 +0200[diff] [blame]735 },
736 },
Piotr Dobrowolski5ac85c62019-04-09 21:45:32 +0200[diff] [blame]737
738 S3ObjectStore(cluster, name):: {
739 local store = self,
740 spec:: error "spec must be specified",
741 objectStore: kube._Object("ceph.rook.io/v1", "CephObjectStore", name) {
742 metadata+: cluster.metadata,
743 spec: store.spec {
744 gateway: {
745 type: "s3",
746 port: 80,
747 instances: 1,
748 allNodes: false,
749 },
750 },
751 },
752
753 objectIngress: kube.Ingress(name) {
754 metadata+: cluster.metadata {
755 annotations+: {
756 "kubernetes.io/tls-acme": "true",
757 "certmanager.k8s.io/cluster-issuer": "letsencrypt-prod",
758 "nginx.ingress.kubernetes.io/proxy-body-size": "0",
759 },
760 },
761 spec+: {
762 tls: [
763 {
764 hosts: ["object.%s.hswaw.net" % [cluster.metadata.namespace]],
765 secretName: "%s-tls" % [name],
766 },
767 ],
768 rules: [
769 {
770 host: "object.%s.hswaw.net" % [cluster.metadata.namespace],
771 http: {
772 paths: [
773 {
774 path: "/",
775 backend: {
776 serviceName: "rook-ceph-rgw-%s" % [name],
777 servicePort: 80,
778 },
779 },
780 ]
781 },
782 }
783 ],
784 },
785 },
786 },
Sergiusz Bazanskib7fcc672019-04-01 18:40:50 +0200[diff] [blame]787}