Gitiles
Code Review Sign In
gerrit.hackerspace.pl / hscloud / 321fad98652bd6f7a4eca16902386140cc6d04b5 / . / cluster / kube / lib / rook.libsonnet
blob: f75e9f0c0c68179a3d01eba8962fc351599a44e8 [file] [log] [blame]
Sergiusz Bazanskib7fcc672019-04-01 18:40:50 +0200[diff] [blame]1# Deploy Rook/Ceph Operator
2
3local kube = import "../../../kube/kube.libsonnet";
4
5{
Sergiusz Bazanskic6da1272019-04-02 00:06:13 +0200[diff] [blame]6 Operator: {
Sergiusz Bazanskib7fcc672019-04-01 18:40:50 +0200[diff] [blame]7 local env = self,
8 local cfg = env.cfg,
9 cfg:: {
Sergiusz Bazanskied2e6702019-04-19 13:27:20 +0200[diff] [blame]10 image: "rook/ceph:v0.9.0-465.g5f6de03",
Sergiusz Bazanskib7fcc672019-04-01 18:40:50 +0200[diff] [blame]11 namespace: "rook-ceph-system",
12 },
13
14 metadata:: {
15 namespace: cfg.namespace,
16 labels: {
17 "operator": "rook",
18 "storage-backend": "ceph",
19 },
20 },
21
22 namespace: kube.Namespace(cfg.namespace),
23
24 crds: {
25 cephclusters: kube.CustomResourceDefinition("ceph.rook.io", "v1", "CephCluster") {
26 spec+: {
27 additionalPrinterColumns: [
28 { name: "DataDirHostPath", type: "string", description: "Directory used on the K8s nodes", JSONPath: ".spec.dataDirHostPath" },
29 { name: "MonCount", type: "string", description: "Number of MONs", JSONPath: ".spec.mon.count" },
30 { name: "Age", type: "date", JSONPath: ".metadata.creationTimestamp" },
31 { name: "State", type: "string", description: "Current State", JSONPath: ".status.state" },
32 ],
33 validation: {
34 # Converted from official operator YAML
35 "openAPIV3Schema": {
36 "properties": {
37 "spec": {
38 "properties": {
39 "cephVersion": {
40 "properties": {
41 "allowUnsupported": {
42 "type": "boolean"
43 },
44 "image": {
45 "type": "string"
46 },
47 "name": {
48 "pattern": "^(luminous|mimic|nautilus)$",
49 "type": "string"
50 }
51 }
52 },
53 "dashboard": {
54 "properties": {
55 "enabled": {
56 "type": "boolean"
57 },
58 "urlPrefix": {
59 "type": "string"
60 },
61 "port": {
62 "type": "integer"
63 }
64 }
65 },
66 "dataDirHostPath": {
67 "pattern": "^/(\\S+)",
68 "type": "string"
69 },
70 "mon": {
71 "properties": {
72 "allowMultiplePerNode": {
73 "type": "boolean"
74 },
75 "count": {
76 "maximum": 9,
77 "minimum": 1,
78 "type": "integer"
79 },
80 "preferredCount": {
81 "maximum": 9,
82 "minimum": 0,
83 "type": "integer"
84 }
85 },
86 "required": [
87 "count"
88 ]
89 },
90 "network": {
91 "properties": {
92 "hostNetwork": {
93 "type": "boolean"
94 }
95 }
96 },
97 "storage": {
98 "properties": {
99 "nodes": {
100 "items": {},
101 "type": "array"
102 },
103 "useAllDevices": {},
104 "useAllNodes": {
105 "type": "boolean"
106 }
107 }
108 }
109 },
110 "required": [
111 "mon"
112 ]
113 }
114 }
115 }
116 }
117 },
118 },
119 cephfilesystems: kube.CustomResourceDefinition("ceph.rook.io", "v1", "CephFilesystem") {
120 spec+: {
121 additionalPrinterColumns: [
122 { name: "MdsCount", type: "string", description: "Number of MDs", JSONPath: ".spec.metadataServer.activeCount" },
123 { name: "Age", type: "date", JSONPath: ".metadata.creationTimestamp" },
124 ],
125 },
126 },
127 cephnfses: kube.CustomResourceDefinition("ceph.rook.io", "v1", "CephNFS") {
128 spec+: {
129 names+: {
Sergiusz Bazanskic6da1272019-04-02 00:06:13 +0200[diff] [blame]130 plural: "cephnfses",
Sergiusz Bazanskib7fcc672019-04-01 18:40:50 +0200[diff] [blame]131 shortNames: ["nfs"],
132 },
133 },
134 },
135 cephobjectstores: kube.CustomResourceDefinition("ceph.rook.io", "v1", "CephObjectStore"),
136 cephobjectstoreusers: kube.CustomResourceDefinition("ceph.rook.io", "v1", "CephObjectStoreUser"),
137 cephblockpools: kube.CustomResourceDefinition("ceph.rook.io", "v1", "CephBlockPool"),
138 volumes: kube.CustomResourceDefinition("rook.io", "v1alpha2", "Volume") {
139 spec+: {
140 names+: {
141 shortNames: ["rv"],
142 },
143 },
144 },
145 },
146
Sergiusz Bazanskicdfafaf2019-04-01 19:16:18 +0200[diff] [blame]147 sa: kube.ServiceAccount("rook-ceph-system") {
148 metadata+: env.metadata,
149 },
150
151 crs: {
152 clusterMgmt: kube.ClusterRole("rook-ceph-cluster-mgmt") {
Sergiusz Bazanskic6da1272019-04-02 00:06:13 +0200[diff] [blame]153 metadata+: env.metadata { namespace:: null },
Sergiusz Bazanskicdfafaf2019-04-01 19:16:18 +0200[diff] [blame]154 rules: [
155 {
156 apiGroups: [""],
157 resources: ["secrets", "pods", "pods/log", "services", "configmaps"],
158 verbs: ["get", "list", "watch", "patch", "create", "update", "delete"],
159 },
160 {
161 apiGroups: ["apps"],
162 resources: ["deployments", "daemonsets", "replicasets"],
163 verbs: ["get", "list", "watch", "create", "update", "delete"],
164 },
165 ],
166 },
167 global: kube.ClusterRole("rook-ceph-global") {
Sergiusz Bazanskic6da1272019-04-02 00:06:13 +0200[diff] [blame]168 metadata+: env.metadata { namespace:: null },
Sergiusz Bazanskicdfafaf2019-04-01 19:16:18 +0200[diff] [blame]169 rules: [
170 {
171 apiGroups: [""],
172 resources: ["pods", "nodes", "nodes/proxy"],
173 verbs: ["get", "list", "watch"],
174 },
175 {
176 apiGroups: [""],
177 resources: ["events", "persistentvolumes", "persistentvolumeclaims", "endpoints"],
178 verbs: ["get", "list", "watch", "patch", "create", "update", "delete"],
179 },
180 {
181 apiGroups: ["storage.k8s.io"],
182 resources: ["storageclasses"],
183 verbs: ["get", "list", "watch", "create", "update", "delete"],
184 },
185 {
Sergiusz Bazanskic6da1272019-04-02 00:06:13 +0200[diff] [blame]186 apiGroups: ["batch"],
187 resources: ["jobs"],
188 verbs: ["get", "list", "watch", "create", "update", "delete"],
189 },
190 {
Sergiusz Bazanskicdfafaf2019-04-01 19:16:18 +0200[diff] [blame]191 apiGroups: ["ceph.rook.io"],
192 resources: ["*"],
193 verbs: ["*"],
194 },
195 {
196 apiGroups: ["rook.io"],
197 resources: ["*"],
198 verbs: ["*"],
199 },
200 ],
201 },
202 mgrCluster: kube.ClusterRole("rook-ceph-mgr-cluster") {
Sergiusz Bazanskic6da1272019-04-02 00:06:13 +0200[diff] [blame]203 metadata+: env.metadata { namespace:: null },
Sergiusz Bazanskicdfafaf2019-04-01 19:16:18 +0200[diff] [blame]204 rules: [
205 {
206 apiGroups: [""],
207 resources: ["configmaps", "nodes", "nodes/proxy"],
208 verbs: ["get", "list", "watch"],
209 },
210 ]
211 },
212 },
213
214 crb: kube.ClusterRoleBinding("ceph-rook-global") {
Sergiusz Bazanskic6da1272019-04-02 00:06:13 +0200[diff] [blame]215 metadata+: env.metadata { namespace:: null },
Sergiusz Bazanskicdfafaf2019-04-01 19:16:18 +0200[diff] [blame]216 roleRef: {
217 apiGroup: "rbac.authorization.k8s.io",
218 kind: "ClusterRole",
219 name: env.crs.global.metadata.name,
220 },
221 subjects: [
222 {
223 kind: "ServiceAccount",
224 name: env.sa.metadata.name,
225 namespace: env.sa.metadata.namespace,
226 },
227 ],
228 },
229
230 role: kube.Role("ceph-rook-system") {
231 metadata+: env.metadata,
232 rules: [
233 {
234 apiGroups: [""],
235 resources: ["pods", "configmaps"],
236 verbs: ["get", "list", "watch", "patch", "create", "update", "delete"],
237 },
238 {
239 apiGroups: ["apps"],
240 resources: ["daemonsets"],
241 verbs: ["get", "list", "watch", "create", "update", "delete"],
242 },
243 ],
244 },
245
246 rb: kube.RoleBinding("ceph-rook-system") {
247 metadata+: env.metadata,
248 roleRef: {
249 apiGroup: "rbac.authorization.k8s.io",
250 kind: "Role",
251 name: env.role.metadata.name,
252 },
253 subjects: [
254 {
255 kind: "ServiceAccount",
256 name: env.sa.metadata.name,
257 namespace: env.sa.metadata.namespace,
258 },
259 ],
260 },
261
262 operator: kube.Deployment("rook-ceph-operator") {
263 metadata+: env.metadata,
264 spec+: {
265 template+: {
266 spec+: {
267 serviceAccountName: env.sa.metadata.name,
268 containers_: {
269 operator: kube.Container("rook-ceph-operator") {
270 image: cfg.image,
271 args: ["ceph", "operator"],
272 volumeMounts_: {
273 "rook-config": { mountPath: "/var/lib/rook" },
274 "default-config-dir": { mountPath: "/etc/ceph" },
275 },
276 env_: {
277 LIB_MODULES_DIR_PATH: "/run/current-system/kernel-modules/lib/modules/",
278 ROOK_ALLOW_MULTIPLE_FILESYSTEMS: "false",
Sergiusz Bazanski321fad92019-04-19 14:14:36 +0200[diff] [blame^]279 ROOK_LOG_LEVEL: "INFO",
Sergiusz Bazanskicdfafaf2019-04-01 19:16:18 +0200[diff] [blame]280 ROOK_MON_HEALTHCHECK_INTERVAL: "45s",
281 ROOK_MON_OUT_TIMEOUT: "600s",
282 ROOK_DISCOVER_DEVICES_INTERVAL: "60m",
283 ROOK_HOSTPATH_REQUIRES_PRIVILEGED: "false",
284 ROOK_ENABLE_SELINUX_RELABELING: "true",
285 ROOK_ENABLE_FSGROUP: "true",
286 NODE_NAME: kube.FieldRef("spec.nodeName"),
287 POD_NAME: kube.FieldRef("metadata.name"),
288 POD_NAMESPACE: kube.FieldRef("metadata.namespace"),
289 },
290 },
291 },
292 volumes_: {
293 "rook-config": { emptyDir: {} },
294 "default-config-dir": { emptyDir: {} },
295 },
296 },
297 },
298 },
299 },
Sergiusz Bazanskib7fcc672019-04-01 18:40:50 +0200[diff] [blame]300 },
Sergiusz Bazanskic6da1272019-04-02 00:06:13 +0200[diff] [blame]301
302 // Create a new Ceph cluster in a new namespace.
303 Cluster(operator, name):: {
304 local cluster = self,
305 spec:: error "please define cluster spec",
306
307
308 metadata:: {
309 namespace: name,
310 },
311
312 name(suffix):: cluster.metadata.namespace + "-" + suffix,
313
314 namespace: kube.Namespace(cluster.metadata.namespace),
315
316 sa: {
317 // service accounts need to be hardcoded, see operator source.
318 osd: kube.ServiceAccount("rook-ceph-osd") {
319 metadata+: cluster.metadata,
320 },
321 mgr: kube.ServiceAccount("rook-ceph-mgr") {
322 metadata+: cluster.metadata,
323 },
324 },
325
326 roles: {
327 osd: kube.Role(cluster.name("osd")) {
328 metadata+: cluster.metadata,
329 rules: [
330 {
331 apiGroups: [""],
332 resources: ["configmaps"],
333 verbs: ["get", "list", "watch", "create", "update", "delete"],
334 }
335 ],
336 },
337 mgr: kube.Role(cluster.name("mgr")) {
338 metadata+: cluster.metadata,
339 rules: [
340 {
341 apiGroups: [""],
342 resources: ["pods", "services"],
343 verbs: ["get", "list", "watch"],
344 },
345 {
346 apiGroups: ["batch"],
347 resources: ["jobs"],
348 verbs: ["get", "list", "watch", "create", "update", "delete"],
349 },
350 {
351 apiGroups: ["ceph.rook.io"],
352 resources: ["*"],
353 verbs: ["*"],
354 },
355 ],
356 },
357 mgrSystem: kube.ClusterRole(cluster.name("mgr-system")) {
358 metadata+: cluster.metadata { namespace:: null },
359 rules: [
360 {
361 apiGroups: [""],
362 resources: ["configmaps"],
363 verbs: ["get", "list", "watch"],
364 }
365 ],
366 },
367 },
368
369 rbs: [
370 kube.RoleBinding(cluster.name(el.name)) {
371 metadata+: cluster.metadata,
372 roleRef: {
373 apiGroup: "rbac.authorization.k8s.io",
374 kind: el.role.kind,
375 name: el.role.metadata.name,
376 },
377 subjects: [
378 {
379 kind: el.sa.kind,
380 name: el.sa.metadata.name,
381 namespace: el.sa.metadata.namespace,
382 },
383 ],
384 },
385 for el in [
386 // Allow Operator SA to perform Cluster Mgmt in this namespace.
387 { name: "cluster-mgmt", role: operator.crs.clusterMgmt, sa: operator.sa },
388 { name: "osd", role: cluster.roles.osd, sa: cluster.sa.osd },
389 { name: "mgr", role: cluster.roles.mgr, sa: cluster.sa.mgr },
390 { name: "mgr-cluster", role: operator.crs.mgrCluster, sa: cluster.sa.mgr },
391 ]
392 ],
393
394 mgrSystemRB: kube.RoleBinding(cluster.name("mgr-system")) {
395 metadata+: {
396 namespace: operator.cfg.namespace,
397 },
398 roleRef: {
399 apiGroup: "rbac.authorization.k8s.io",
400 kind: cluster.roles.mgrSystem.kind,
401 name: cluster.roles.mgrSystem.metadata.name,
402 },
403 subjects: [
404 {
405 kind: cluster.sa.mgr.kind,
406 name: cluster.sa.mgr.metadata.name,
407 namespace: cluster.sa.mgr.metadata.namespace,
408 },
409 ],
410 },
411
412 cluster: kube._Object("ceph.rook.io/v1", "CephCluster", name) {
413 metadata+: cluster.metadata,
414 spec: {
415 cephVersion: {
Sergiusz Bazanskied2e6702019-04-19 13:27:20 +0200[diff] [blame]416 # https://github.com/rook/rook/issues/2945#issuecomment-483964014
417 #image: "ceph/ceph:v13.2.5-20190319",
418 image: "ceph/ceph:v14",
419 allowUnsupported: true,
Sergiusz Bazanskic6da1272019-04-02 00:06:13 +0200[diff] [blame]420 },
421 dataDirHostPath: "/var/lib/rook",
422 dashboard: {
423 ssl: false,
424 enabled: true,
425 port: 8080,
426 },
427 } + cluster.spec,
428 },
429
430 dashboardService: kube.Service(cluster.name("dashboard")) {
431 metadata+: cluster.metadata,
432 spec: {
433 ports: [
434 { name: "dashboard", port: 80, targetPort: 8080, protocol: "TCP" },
435 ],
436 selector: {
437 app: "rook-ceph-mgr",
438 rook_cluster: name,
439 },
440 type: "ClusterIP",
441 },
442 },
443
444 dashboardIngress: kube.Ingress(cluster.name("dashboard")) {
Piotr Dobrowolski3187c592019-04-02 14:44:04 +0200[diff] [blame]445 metadata+: cluster.metadata {
446 annotations+: {
447 "kubernetes.io/tls-acme": "true",
448 "certmanager.k8s.io/cluster-issuer": "letsencrypt-prod",
449 },
450 },
Sergiusz Bazanskic6da1272019-04-02 00:06:13 +0200[diff] [blame]451 spec+: {
Piotr Dobrowolski3187c592019-04-02 14:44:04 +0200[diff] [blame]452 tls: [
453 {
454 hosts: ["%s.hswaw.net" % name],
455 secretName: cluster.name("dashboard"),
456 },
457 ],
Sergiusz Bazanskic6da1272019-04-02 00:06:13 +0200[diff] [blame]458 rules: [
459 {
460 host: "%s.hswaw.net" % name,
461 http: {
462 paths: [
463 { path: "/", backend: cluster.dashboardService.name_port },
464 ]
465 },
466 }
467 ],
468 },
Sergiusz Bazanski65f3b1d2019-04-02 01:05:38 +0200[diff] [blame]469 }
470 },
471
472 ECBlockPool(cluster, name):: {
473 local pool = self,
474 spec:: error "spec must be specified",
475
476 pool: kube._Object("ceph.rook.io/v1", "CephBlockPool", name) {
477 metadata+: cluster.metadata,
478 spec: pool.spec,
479 },
480 metapool: kube._Object("ceph.rook.io/v1", "CephBlockPool", name + "-metadata") {
481 metadata+: cluster.metadata,
482 spec: {
483 failureDomain: "host",
484 replicated: {
485 size: 3,
486 },
487 },
488 },
489
490 storageClass: kube.StorageClass(name) {
491 provisioner: "ceph.rook.io/block",
492 parameters: {
493 blockPool: pool.metapool.metadata.name,
494 dataBlockPool: pool.pool.metadata.name,
495 clusterNamespace: pool.pool.metadata.namespace,
496 fstype: "ext4",
497 },
498 reclaimPolicy: "Retain",
Sergiusz Bazanskic6da1272019-04-02 00:06:13 +0200[diff] [blame]499 },
500 },
Piotr Dobrowolski5ac85c62019-04-09 21:45:32 +0200[diff] [blame]501
502 S3ObjectStore(cluster, name):: {
503 local store = self,
504 spec:: error "spec must be specified",
505 objectStore: kube._Object("ceph.rook.io/v1", "CephObjectStore", name) {
506 metadata+: cluster.metadata,
507 spec: store.spec {
508 gateway: {
509 type: "s3",
510 port: 80,
511 instances: 1,
512 allNodes: false,
513 },
514 },
515 },
516
517 objectIngress: kube.Ingress(name) {
518 metadata+: cluster.metadata {
519 annotations+: {
520 "kubernetes.io/tls-acme": "true",
521 "certmanager.k8s.io/cluster-issuer": "letsencrypt-prod",
522 "nginx.ingress.kubernetes.io/proxy-body-size": "0",
523 },
524 },
525 spec+: {
526 tls: [
527 {
528 hosts: ["object.%s.hswaw.net" % [cluster.metadata.namespace]],
529 secretName: "%s-tls" % [name],
530 },
531 ],
532 rules: [
533 {
534 host: "object.%s.hswaw.net" % [cluster.metadata.namespace],
535 http: {
536 paths: [
537 {
538 path: "/",
539 backend: {
540 serviceName: "rook-ceph-rgw-%s" % [name],
541 servicePort: 80,
542 },
543 },
544 ]
545 },
546 }
547 ],
548 },
549 },
550 },
Sergiusz Bazanskib7fcc672019-04-01 18:40:50 +0200[diff] [blame]551}