Gitiles
Code Review Sign In
gerrit.hackerspace.pl / hscloud / a01c487a6e16bdc7c2bef25e3414c3720368b7ca / . / cluster / kube / lib / rook.libsonnet
blob: 92daf1a7d8550889aeeecb749b137fec4079a534 [file] [log] [blame]
Sergiusz Bazanskib7fcc672019-04-01 18:40:50 +0200[diff] [blame]1# Deploy Rook/Ceph Operator
2
3local kube = import "../../../kube/kube.libsonnet";
Sergiusz Bazanskia01c4872019-09-05 18:00:02 +0200[diff] [blame^]4local policies = import "../../../kube/policies.libsonnet";
Sergiusz Bazanskib7fcc672019-04-01 18:40:50 +0200[diff] [blame]5
6{
Sergiusz Bazanskic6da1272019-04-02 00:06:13 +0200[diff] [blame]7 Operator: {
Sergiusz Bazanskib7fcc672019-04-01 18:40:50 +0200[diff] [blame]8 local env = self,
9 local cfg = env.cfg,
10 cfg:: {
Sergiusz Bazanski71a21c72019-08-30 21:30:42 +0200[diff] [blame]11 image: "rook/ceph:v1.0.5",
Sergiusz Bazanskib7fcc672019-04-01 18:40:50 +0200[diff] [blame]12 namespace: "rook-ceph-system",
13 },
14
15 metadata:: {
16 namespace: cfg.namespace,
17 labels: {
18 "operator": "rook",
19 "storage-backend": "ceph",
20 },
21 },
22
23 namespace: kube.Namespace(cfg.namespace),
24
Sergiusz Bazanskia01c4872019-09-05 18:00:02 +0200[diff] [blame^]25 policyInsecure: policies.AllowNamespaceInsecure(cfg.namespace),
26
Sergiusz Bazanskib7fcc672019-04-01 18:40:50 +0200[diff] [blame]27 crds: {
28 cephclusters: kube.CustomResourceDefinition("ceph.rook.io", "v1", "CephCluster") {
29 spec+: {
30 additionalPrinterColumns: [
31 { name: "DataDirHostPath", type: "string", description: "Directory used on the K8s nodes", JSONPath: ".spec.dataDirHostPath" },
32 { name: "MonCount", type: "string", description: "Number of MONs", JSONPath: ".spec.mon.count" },
33 { name: "Age", type: "date", JSONPath: ".metadata.creationTimestamp" },
34 { name: "State", type: "string", description: "Current State", JSONPath: ".status.state" },
35 ],
36 validation: {
37 # Converted from official operator YAML
38 "openAPIV3Schema": {
39 "properties": {
40 "spec": {
41 "properties": {
42 "cephVersion": {
43 "properties": {
44 "allowUnsupported": {
45 "type": "boolean"
46 },
47 "image": {
48 "type": "string"
49 },
50 "name": {
51 "pattern": "^(luminous|mimic|nautilus)$",
52 "type": "string"
53 }
54 }
55 },
56 "dashboard": {
57 "properties": {
58 "enabled": {
59 "type": "boolean"
60 },
61 "urlPrefix": {
62 "type": "string"
63 },
64 "port": {
65 "type": "integer"
66 }
67 }
68 },
69 "dataDirHostPath": {
70 "pattern": "^/(\\S+)",
71 "type": "string"
72 },
73 "mon": {
74 "properties": {
75 "allowMultiplePerNode": {
76 "type": "boolean"
77 },
78 "count": {
79 "maximum": 9,
80 "minimum": 1,
81 "type": "integer"
82 },
83 "preferredCount": {
84 "maximum": 9,
85 "minimum": 0,
86 "type": "integer"
87 }
88 },
89 "required": [
90 "count"
91 ]
92 },
93 "network": {
94 "properties": {
95 "hostNetwork": {
96 "type": "boolean"
97 }
98 }
99 },
100 "storage": {
101 "properties": {
102 "nodes": {
103 "items": {},
104 "type": "array"
105 },
106 "useAllDevices": {},
107 "useAllNodes": {
108 "type": "boolean"
109 }
110 }
111 }
112 },
113 "required": [
114 "mon"
115 ]
116 }
117 }
118 }
119 }
120 },
121 },
122 cephfilesystems: kube.CustomResourceDefinition("ceph.rook.io", "v1", "CephFilesystem") {
123 spec+: {
124 additionalPrinterColumns: [
125 { name: "MdsCount", type: "string", description: "Number of MDs", JSONPath: ".spec.metadataServer.activeCount" },
126 { name: "Age", type: "date", JSONPath: ".metadata.creationTimestamp" },
127 ],
128 },
129 },
130 cephnfses: kube.CustomResourceDefinition("ceph.rook.io", "v1", "CephNFS") {
131 spec+: {
132 names+: {
Sergiusz Bazanskic6da1272019-04-02 00:06:13 +0200[diff] [blame]133 plural: "cephnfses",
Sergiusz Bazanskib7fcc672019-04-01 18:40:50 +0200[diff] [blame]134 shortNames: ["nfs"],
135 },
136 },
137 },
138 cephobjectstores: kube.CustomResourceDefinition("ceph.rook.io", "v1", "CephObjectStore"),
139 cephobjectstoreusers: kube.CustomResourceDefinition("ceph.rook.io", "v1", "CephObjectStoreUser"),
140 cephblockpools: kube.CustomResourceDefinition("ceph.rook.io", "v1", "CephBlockPool"),
141 volumes: kube.CustomResourceDefinition("rook.io", "v1alpha2", "Volume") {
142 spec+: {
143 names+: {
144 shortNames: ["rv"],
145 },
146 },
147 },
148 },
149
Sergiusz Bazanskicdfafaf2019-04-01 19:16:18 +0200[diff] [blame]150 sa: kube.ServiceAccount("rook-ceph-system") {
151 metadata+: env.metadata,
152 },
153
154 crs: {
155 clusterMgmt: kube.ClusterRole("rook-ceph-cluster-mgmt") {
Sergiusz Bazanskic6da1272019-04-02 00:06:13 +0200[diff] [blame]156 metadata+: env.metadata { namespace:: null },
Sergiusz Bazanskicdfafaf2019-04-01 19:16:18 +0200[diff] [blame]157 rules: [
158 {
159 apiGroups: [""],
160 resources: ["secrets", "pods", "pods/log", "services", "configmaps"],
161 verbs: ["get", "list", "watch", "patch", "create", "update", "delete"],
162 },
163 {
164 apiGroups: ["apps"],
165 resources: ["deployments", "daemonsets", "replicasets"],
166 verbs: ["get", "list", "watch", "create", "update", "delete"],
167 },
168 ],
169 },
170 global: kube.ClusterRole("rook-ceph-global") {
Sergiusz Bazanskic6da1272019-04-02 00:06:13 +0200[diff] [blame]171 metadata+: env.metadata { namespace:: null },
Sergiusz Bazanskicdfafaf2019-04-01 19:16:18 +0200[diff] [blame]172 rules: [
173 {
174 apiGroups: [""],
175 resources: ["pods", "nodes", "nodes/proxy"],
176 verbs: ["get", "list", "watch"],
177 },
178 {
179 apiGroups: [""],
180 resources: ["events", "persistentvolumes", "persistentvolumeclaims", "endpoints"],
181 verbs: ["get", "list", "watch", "patch", "create", "update", "delete"],
182 },
183 {
184 apiGroups: ["storage.k8s.io"],
185 resources: ["storageclasses"],
186 verbs: ["get", "list", "watch", "create", "update", "delete"],
187 },
188 {
Sergiusz Bazanskic6da1272019-04-02 00:06:13 +0200[diff] [blame]189 apiGroups: ["batch"],
190 resources: ["jobs"],
191 verbs: ["get", "list", "watch", "create", "update", "delete"],
192 },
193 {
Sergiusz Bazanskicdfafaf2019-04-01 19:16:18 +0200[diff] [blame]194 apiGroups: ["ceph.rook.io"],
195 resources: ["*"],
196 verbs: ["*"],
197 },
198 {
199 apiGroups: ["rook.io"],
200 resources: ["*"],
201 verbs: ["*"],
202 },
203 ],
204 },
205 mgrCluster: kube.ClusterRole("rook-ceph-mgr-cluster") {
Sergiusz Bazanskic6da1272019-04-02 00:06:13 +0200[diff] [blame]206 metadata+: env.metadata { namespace:: null },
Sergiusz Bazanskicdfafaf2019-04-01 19:16:18 +0200[diff] [blame]207 rules: [
208 {
209 apiGroups: [""],
210 resources: ["configmaps", "nodes", "nodes/proxy"],
211 verbs: ["get", "list", "watch"],
212 },
213 ]
214 },
215 },
216
217 crb: kube.ClusterRoleBinding("ceph-rook-global") {
Sergiusz Bazanskic6da1272019-04-02 00:06:13 +0200[diff] [blame]218 metadata+: env.metadata { namespace:: null },
Sergiusz Bazanskicdfafaf2019-04-01 19:16:18 +0200[diff] [blame]219 roleRef: {
220 apiGroup: "rbac.authorization.k8s.io",
221 kind: "ClusterRole",
222 name: env.crs.global.metadata.name,
223 },
224 subjects: [
225 {
226 kind: "ServiceAccount",
227 name: env.sa.metadata.name,
228 namespace: env.sa.metadata.namespace,
229 },
230 ],
231 },
232
233 role: kube.Role("ceph-rook-system") {
234 metadata+: env.metadata,
235 rules: [
236 {
237 apiGroups: [""],
238 resources: ["pods", "configmaps"],
239 verbs: ["get", "list", "watch", "patch", "create", "update", "delete"],
240 },
241 {
242 apiGroups: ["apps"],
243 resources: ["daemonsets"],
244 verbs: ["get", "list", "watch", "create", "update", "delete"],
245 },
246 ],
247 },
248
249 rb: kube.RoleBinding("ceph-rook-system") {
250 metadata+: env.metadata,
251 roleRef: {
252 apiGroup: "rbac.authorization.k8s.io",
253 kind: "Role",
254 name: env.role.metadata.name,
255 },
256 subjects: [
257 {
258 kind: "ServiceAccount",
259 name: env.sa.metadata.name,
260 namespace: env.sa.metadata.namespace,
261 },
262 ],
263 },
264
265 operator: kube.Deployment("rook-ceph-operator") {
266 metadata+: env.metadata,
267 spec+: {
268 template+: {
269 spec+: {
270 serviceAccountName: env.sa.metadata.name,
271 containers_: {
272 operator: kube.Container("rook-ceph-operator") {
273 image: cfg.image,
274 args: ["ceph", "operator"],
275 volumeMounts_: {
276 "rook-config": { mountPath: "/var/lib/rook" },
277 "default-config-dir": { mountPath: "/etc/ceph" },
278 },
279 env_: {
280 LIB_MODULES_DIR_PATH: "/run/current-system/kernel-modules/lib/modules/",
281 ROOK_ALLOW_MULTIPLE_FILESYSTEMS: "false",
Sergiusz Bazanski321fad92019-04-19 14:14:36 +0200[diff] [blame]282 ROOK_LOG_LEVEL: "INFO",
Sergiusz Bazanskicdfafaf2019-04-01 19:16:18 +0200[diff] [blame]283 ROOK_MON_HEALTHCHECK_INTERVAL: "45s",
284 ROOK_MON_OUT_TIMEOUT: "600s",
285 ROOK_DISCOVER_DEVICES_INTERVAL: "60m",
286 ROOK_HOSTPATH_REQUIRES_PRIVILEGED: "false",
287 ROOK_ENABLE_SELINUX_RELABELING: "true",
288 ROOK_ENABLE_FSGROUP: "true",
289 NODE_NAME: kube.FieldRef("spec.nodeName"),
290 POD_NAME: kube.FieldRef("metadata.name"),
291 POD_NAMESPACE: kube.FieldRef("metadata.namespace"),
292 },
293 },
294 },
295 volumes_: {
296 "rook-config": { emptyDir: {} },
297 "default-config-dir": { emptyDir: {} },
298 },
299 },
300 },
301 },
302 },
Sergiusz Bazanskib7fcc672019-04-01 18:40:50 +0200[diff] [blame]303 },
Sergiusz Bazanskic6da1272019-04-02 00:06:13 +0200[diff] [blame]304
305 // Create a new Ceph cluster in a new namespace.
306 Cluster(operator, name):: {
307 local cluster = self,
308 spec:: error "please define cluster spec",
309
310
311 metadata:: {
312 namespace: name,
313 },
314
315 name(suffix):: cluster.metadata.namespace + "-" + suffix,
316
317 namespace: kube.Namespace(cluster.metadata.namespace),
318
319 sa: {
320 // service accounts need to be hardcoded, see operator source.
321 osd: kube.ServiceAccount("rook-ceph-osd") {
322 metadata+: cluster.metadata,
323 },
324 mgr: kube.ServiceAccount("rook-ceph-mgr") {
325 metadata+: cluster.metadata,
326 },
327 },
328
329 roles: {
330 osd: kube.Role(cluster.name("osd")) {
331 metadata+: cluster.metadata,
332 rules: [
333 {
334 apiGroups: [""],
335 resources: ["configmaps"],
336 verbs: ["get", "list", "watch", "create", "update", "delete"],
337 }
338 ],
339 },
340 mgr: kube.Role(cluster.name("mgr")) {
341 metadata+: cluster.metadata,
342 rules: [
343 {
344 apiGroups: [""],
345 resources: ["pods", "services"],
346 verbs: ["get", "list", "watch"],
347 },
348 {
349 apiGroups: ["batch"],
350 resources: ["jobs"],
351 verbs: ["get", "list", "watch", "create", "update", "delete"],
352 },
353 {
354 apiGroups: ["ceph.rook.io"],
355 resources: ["*"],
356 verbs: ["*"],
357 },
358 ],
359 },
360 mgrSystem: kube.ClusterRole(cluster.name("mgr-system")) {
361 metadata+: cluster.metadata { namespace:: null },
362 rules: [
363 {
364 apiGroups: [""],
365 resources: ["configmaps"],
366 verbs: ["get", "list", "watch"],
367 }
368 ],
369 },
370 },
371
372 rbs: [
373 kube.RoleBinding(cluster.name(el.name)) {
374 metadata+: cluster.metadata,
375 roleRef: {
376 apiGroup: "rbac.authorization.k8s.io",
377 kind: el.role.kind,
378 name: el.role.metadata.name,
379 },
380 subjects: [
381 {
382 kind: el.sa.kind,
383 name: el.sa.metadata.name,
384 namespace: el.sa.metadata.namespace,
385 },
386 ],
387 },
388 for el in [
389 // Allow Operator SA to perform Cluster Mgmt in this namespace.
390 { name: "cluster-mgmt", role: operator.crs.clusterMgmt, sa: operator.sa },
391 { name: "osd", role: cluster.roles.osd, sa: cluster.sa.osd },
392 { name: "mgr", role: cluster.roles.mgr, sa: cluster.sa.mgr },
393 { name: "mgr-cluster", role: operator.crs.mgrCluster, sa: cluster.sa.mgr },
394 ]
395 ],
396
397 mgrSystemRB: kube.RoleBinding(cluster.name("mgr-system")) {
398 metadata+: {
399 namespace: operator.cfg.namespace,
400 },
401 roleRef: {
402 apiGroup: "rbac.authorization.k8s.io",
403 kind: cluster.roles.mgrSystem.kind,
404 name: cluster.roles.mgrSystem.metadata.name,
405 },
406 subjects: [
407 {
408 kind: cluster.sa.mgr.kind,
409 name: cluster.sa.mgr.metadata.name,
410 namespace: cluster.sa.mgr.metadata.namespace,
411 },
412 ],
413 },
414
415 cluster: kube._Object("ceph.rook.io/v1", "CephCluster", name) {
416 metadata+: cluster.metadata,
417 spec: {
418 cephVersion: {
Sergiusz Bazanskied2e6702019-04-19 13:27:20 +0200[diff] [blame]419 # https://github.com/rook/rook/issues/2945#issuecomment-483964014
420 #image: "ceph/ceph:v13.2.5-20190319",
Sergiusz Bazanski71a21c72019-08-30 21:30:42 +0200[diff] [blame]421 image: "ceph/ceph:v14.2.2-20190830",
Sergiusz Bazanskied2e6702019-04-19 13:27:20 +0200[diff] [blame]422 allowUnsupported: true,
Sergiusz Bazanskic6da1272019-04-02 00:06:13 +0200[diff] [blame]423 },
424 dataDirHostPath: "/var/lib/rook",
425 dashboard: {
426 ssl: false,
427 enabled: true,
428 port: 8080,
429 },
430 } + cluster.spec,
431 },
432
433 dashboardService: kube.Service(cluster.name("dashboard")) {
434 metadata+: cluster.metadata,
435 spec: {
436 ports: [
437 { name: "dashboard", port: 80, targetPort: 8080, protocol: "TCP" },
438 ],
439 selector: {
440 app: "rook-ceph-mgr",
441 rook_cluster: name,
442 },
443 type: "ClusterIP",
444 },
445 },
446
447 dashboardIngress: kube.Ingress(cluster.name("dashboard")) {
Piotr Dobrowolski3187c592019-04-02 14:44:04 +0200[diff] [blame]448 metadata+: cluster.metadata {
449 annotations+: {
450 "kubernetes.io/tls-acme": "true",
451 "certmanager.k8s.io/cluster-issuer": "letsencrypt-prod",
452 },
453 },
Sergiusz Bazanskic6da1272019-04-02 00:06:13 +0200[diff] [blame]454 spec+: {
Piotr Dobrowolski3187c592019-04-02 14:44:04 +0200[diff] [blame]455 tls: [
456 {
457 hosts: ["%s.hswaw.net" % name],
458 secretName: cluster.name("dashboard"),
459 },
460 ],
Sergiusz Bazanskic6da1272019-04-02 00:06:13 +0200[diff] [blame]461 rules: [
462 {
463 host: "%s.hswaw.net" % name,
464 http: {
465 paths: [
466 { path: "/", backend: cluster.dashboardService.name_port },
467 ]
468 },
469 }
470 ],
471 },
Sergiusz Bazanski65f3b1d2019-04-02 01:05:38 +0200[diff] [blame]472 }
473 },
474
Sergiusz Bazanski36cc4fb2019-05-17 18:08:48 +0200[diff] [blame]475 ReplicatedBlockPool(cluster, name):: {
476 local pool = self,
Sergiusz Bazanskid07861b2019-08-08 17:48:25 +0200[diff] [blame]477 name:: name,
478
Sergiusz Bazanski36cc4fb2019-05-17 18:08:48 +0200[diff] [blame]479 spec:: error "spec must be specified",
480
481 pool: kube._Object("ceph.rook.io/v1", "CephBlockPool", name) {
482 metadata+: cluster.metadata,
483 spec: pool.spec,
484 },
485
486 storageClass: kube.StorageClass(name) {
487 provisioner: "ceph.rook.io/block",
488 parameters: {
489 blockPool: pool.pool.metadata.name,
490 clusterNamespace: pool.pool.metadata.namespace,
491 fstype: "ext4",
492 },
493 reclaimPolicy: "Retain",
494 },
495 },
496
Sergiusz Bazanski65f3b1d2019-04-02 01:05:38 +0200[diff] [blame]497 ECBlockPool(cluster, name):: {
498 local pool = self,
Sergiusz Bazanski4d61d202019-07-21 16:56:41 +0200[diff] [blame]499 name:: name,
500
Sergiusz Bazanski65f3b1d2019-04-02 01:05:38 +0200[diff] [blame]501 spec:: error "spec must be specified",
502
503 pool: kube._Object("ceph.rook.io/v1", "CephBlockPool", name) {
504 metadata+: cluster.metadata,
505 spec: pool.spec,
506 },
507 metapool: kube._Object("ceph.rook.io/v1", "CephBlockPool", name + "-metadata") {
508 metadata+: cluster.metadata,
509 spec: {
510 failureDomain: "host",
511 replicated: {
512 size: 3,
513 },
514 },
515 },
516
517 storageClass: kube.StorageClass(name) {
518 provisioner: "ceph.rook.io/block",
519 parameters: {
520 blockPool: pool.metapool.metadata.name,
521 dataBlockPool: pool.pool.metadata.name,
522 clusterNamespace: pool.pool.metadata.namespace,
523 fstype: "ext4",
524 },
525 reclaimPolicy: "Retain",
Sergiusz Bazanskic6da1272019-04-02 00:06:13 +0200[diff] [blame]526 },
527 },
Piotr Dobrowolski5ac85c62019-04-09 21:45:32 +0200[diff] [blame]528
529 S3ObjectStore(cluster, name):: {
530 local store = self,
531 spec:: error "spec must be specified",
532 objectStore: kube._Object("ceph.rook.io/v1", "CephObjectStore", name) {
533 metadata+: cluster.metadata,
534 spec: store.spec {
535 gateway: {
536 type: "s3",
537 port: 80,
538 instances: 1,
539 allNodes: false,
540 },
541 },
542 },
543
544 objectIngress: kube.Ingress(name) {
545 metadata+: cluster.metadata {
546 annotations+: {
547 "kubernetes.io/tls-acme": "true",
548 "certmanager.k8s.io/cluster-issuer": "letsencrypt-prod",
549 "nginx.ingress.kubernetes.io/proxy-body-size": "0",
550 },
551 },
552 spec+: {
553 tls: [
554 {
555 hosts: ["object.%s.hswaw.net" % [cluster.metadata.namespace]],
556 secretName: "%s-tls" % [name],
557 },
558 ],
559 rules: [
560 {
561 host: "object.%s.hswaw.net" % [cluster.metadata.namespace],
562 http: {
563 paths: [
564 {
565 path: "/",
566 backend: {
567 serviceName: "rook-ceph-rgw-%s" % [name],
568 servicePort: 80,
569 },
570 },
571 ]
572 },
573 }
574 ],
575 },
576 },
577 },
Sergiusz Bazanskib7fcc672019-04-01 18:40:50 +0200[diff] [blame]578}