Gitiles
Code Review Sign In
gerrit.hackerspace.pl / hscloud / 65f3b1d8abe634bc7041c895a8ac1b6628234510 / . / cluster / kube / lib / rook.libsonnet
blob: 9c7b1e99f6acfb06e7d3e3539aba8d1fd51733c0 [file] [log] [blame]
Sergiusz Bazanskib7fcc672019-04-01 18:40:50 +0200[diff] [blame]1# Deploy Rook/Ceph Operator
2
3local kube = import "../../../kube/kube.libsonnet";
4
5{
Sergiusz Bazanskic6da1272019-04-02 00:06:13 +0200[diff] [blame]6 Operator: {
Sergiusz Bazanskib7fcc672019-04-01 18:40:50 +0200[diff] [blame]7 local env = self,
8 local cfg = env.cfg,
9 cfg:: {
10 image: "rook/ceph:master",
11 namespace: "rook-ceph-system",
12 },
13
14 metadata:: {
15 namespace: cfg.namespace,
16 labels: {
17 "operator": "rook",
18 "storage-backend": "ceph",
19 },
20 },
21
22 namespace: kube.Namespace(cfg.namespace),
23
24 crds: {
25 cephclusters: kube.CustomResourceDefinition("ceph.rook.io", "v1", "CephCluster") {
26 spec+: {
27 additionalPrinterColumns: [
28 { name: "DataDirHostPath", type: "string", description: "Directory used on the K8s nodes", JSONPath: ".spec.dataDirHostPath" },
29 { name: "MonCount", type: "string", description: "Number of MONs", JSONPath: ".spec.mon.count" },
30 { name: "Age", type: "date", JSONPath: ".metadata.creationTimestamp" },
31 { name: "State", type: "string", description: "Current State", JSONPath: ".status.state" },
32 ],
33 validation: {
34 # Converted from official operator YAML
35 "openAPIV3Schema": {
36 "properties": {
37 "spec": {
38 "properties": {
39 "cephVersion": {
40 "properties": {
41 "allowUnsupported": {
42 "type": "boolean"
43 },
44 "image": {
45 "type": "string"
46 },
47 "name": {
48 "pattern": "^(luminous|mimic|nautilus)$",
49 "type": "string"
50 }
51 }
52 },
53 "dashboard": {
54 "properties": {
55 "enabled": {
56 "type": "boolean"
57 },
58 "urlPrefix": {
59 "type": "string"
60 },
61 "port": {
62 "type": "integer"
63 }
64 }
65 },
66 "dataDirHostPath": {
67 "pattern": "^/(\\S+)",
68 "type": "string"
69 },
70 "mon": {
71 "properties": {
72 "allowMultiplePerNode": {
73 "type": "boolean"
74 },
75 "count": {
76 "maximum": 9,
77 "minimum": 1,
78 "type": "integer"
79 },
80 "preferredCount": {
81 "maximum": 9,
82 "minimum": 0,
83 "type": "integer"
84 }
85 },
86 "required": [
87 "count"
88 ]
89 },
90 "network": {
91 "properties": {
92 "hostNetwork": {
93 "type": "boolean"
94 }
95 }
96 },
97 "storage": {
98 "properties": {
99 "nodes": {
100 "items": {},
101 "type": "array"
102 },
103 "useAllDevices": {},
104 "useAllNodes": {
105 "type": "boolean"
106 }
107 }
108 }
109 },
110 "required": [
111 "mon"
112 ]
113 }
114 }
115 }
116 }
117 },
118 },
119 cephfilesystems: kube.CustomResourceDefinition("ceph.rook.io", "v1", "CephFilesystem") {
120 spec+: {
121 additionalPrinterColumns: [
122 { name: "MdsCount", type: "string", description: "Number of MDs", JSONPath: ".spec.metadataServer.activeCount" },
123 { name: "Age", type: "date", JSONPath: ".metadata.creationTimestamp" },
124 ],
125 },
126 },
127 cephnfses: kube.CustomResourceDefinition("ceph.rook.io", "v1", "CephNFS") {
128 spec+: {
129 names+: {
Sergiusz Bazanskic6da1272019-04-02 00:06:13 +0200[diff] [blame]130 plural: "cephnfses",
Sergiusz Bazanskib7fcc672019-04-01 18:40:50 +0200[diff] [blame]131 shortNames: ["nfs"],
132 },
133 },
134 },
135 cephobjectstores: kube.CustomResourceDefinition("ceph.rook.io", "v1", "CephObjectStore"),
136 cephobjectstoreusers: kube.CustomResourceDefinition("ceph.rook.io", "v1", "CephObjectStoreUser"),
137 cephblockpools: kube.CustomResourceDefinition("ceph.rook.io", "v1", "CephBlockPool"),
138 volumes: kube.CustomResourceDefinition("rook.io", "v1alpha2", "Volume") {
139 spec+: {
140 names+: {
141 shortNames: ["rv"],
142 },
143 },
144 },
145 },
146
Sergiusz Bazanskicdfafaf2019-04-01 19:16:18 +0200[diff] [blame]147 sa: kube.ServiceAccount("rook-ceph-system") {
148 metadata+: env.metadata,
149 },
150
151 crs: {
152 clusterMgmt: kube.ClusterRole("rook-ceph-cluster-mgmt") {
Sergiusz Bazanskic6da1272019-04-02 00:06:13 +0200[diff] [blame]153 metadata+: env.metadata { namespace:: null },
Sergiusz Bazanskicdfafaf2019-04-01 19:16:18 +0200[diff] [blame]154 rules: [
155 {
156 apiGroups: [""],
157 resources: ["secrets", "pods", "pods/log", "services", "configmaps"],
158 verbs: ["get", "list", "watch", "patch", "create", "update", "delete"],
159 },
160 {
161 apiGroups: ["apps"],
162 resources: ["deployments", "daemonsets", "replicasets"],
163 verbs: ["get", "list", "watch", "create", "update", "delete"],
164 },
165 ],
166 },
167 global: kube.ClusterRole("rook-ceph-global") {
Sergiusz Bazanskic6da1272019-04-02 00:06:13 +0200[diff] [blame]168 metadata+: env.metadata { namespace:: null },
Sergiusz Bazanskicdfafaf2019-04-01 19:16:18 +0200[diff] [blame]169 rules: [
170 {
171 apiGroups: [""],
172 resources: ["pods", "nodes", "nodes/proxy"],
173 verbs: ["get", "list", "watch"],
174 },
175 {
176 apiGroups: [""],
177 resources: ["events", "persistentvolumes", "persistentvolumeclaims", "endpoints"],
178 verbs: ["get", "list", "watch", "patch", "create", "update", "delete"],
179 },
180 {
181 apiGroups: ["storage.k8s.io"],
182 resources: ["storageclasses"],
183 verbs: ["get", "list", "watch", "create", "update", "delete"],
184 },
185 {
Sergiusz Bazanskic6da1272019-04-02 00:06:13 +0200[diff] [blame]186 apiGroups: ["batch"],
187 resources: ["jobs"],
188 verbs: ["get", "list", "watch", "create", "update", "delete"],
189 },
190 {
Sergiusz Bazanskicdfafaf2019-04-01 19:16:18 +0200[diff] [blame]191 apiGroups: ["ceph.rook.io"],
192 resources: ["*"],
193 verbs: ["*"],
194 },
195 {
196 apiGroups: ["rook.io"],
197 resources: ["*"],
198 verbs: ["*"],
199 },
200 ],
201 },
202 mgrCluster: kube.ClusterRole("rook-ceph-mgr-cluster") {
Sergiusz Bazanskic6da1272019-04-02 00:06:13 +0200[diff] [blame]203 metadata+: env.metadata { namespace:: null },
Sergiusz Bazanskicdfafaf2019-04-01 19:16:18 +0200[diff] [blame]204 rules: [
205 {
206 apiGroups: [""],
207 resources: ["configmaps", "nodes", "nodes/proxy"],
208 verbs: ["get", "list", "watch"],
209 },
210 ]
211 },
212 },
213
214 crb: kube.ClusterRoleBinding("ceph-rook-global") {
Sergiusz Bazanskic6da1272019-04-02 00:06:13 +0200[diff] [blame]215 metadata+: env.metadata { namespace:: null },
Sergiusz Bazanskicdfafaf2019-04-01 19:16:18 +0200[diff] [blame]216 roleRef: {
217 apiGroup: "rbac.authorization.k8s.io",
218 kind: "ClusterRole",
219 name: env.crs.global.metadata.name,
220 },
221 subjects: [
222 {
223 kind: "ServiceAccount",
224 name: env.sa.metadata.name,
225 namespace: env.sa.metadata.namespace,
226 },
227 ],
228 },
229
230 role: kube.Role("ceph-rook-system") {
231 metadata+: env.metadata,
232 rules: [
233 {
234 apiGroups: [""],
235 resources: ["pods", "configmaps"],
236 verbs: ["get", "list", "watch", "patch", "create", "update", "delete"],
237 },
238 {
239 apiGroups: ["apps"],
240 resources: ["daemonsets"],
241 verbs: ["get", "list", "watch", "create", "update", "delete"],
242 },
243 ],
244 },
245
246 rb: kube.RoleBinding("ceph-rook-system") {
247 metadata+: env.metadata,
248 roleRef: {
249 apiGroup: "rbac.authorization.k8s.io",
250 kind: "Role",
251 name: env.role.metadata.name,
252 },
253 subjects: [
254 {
255 kind: "ServiceAccount",
256 name: env.sa.metadata.name,
257 namespace: env.sa.metadata.namespace,
258 },
259 ],
260 },
261
262 operator: kube.Deployment("rook-ceph-operator") {
263 metadata+: env.metadata,
264 spec+: {
265 template+: {
266 spec+: {
267 serviceAccountName: env.sa.metadata.name,
268 containers_: {
269 operator: kube.Container("rook-ceph-operator") {
270 image: cfg.image,
271 args: ["ceph", "operator"],
272 volumeMounts_: {
273 "rook-config": { mountPath: "/var/lib/rook" },
274 "default-config-dir": { mountPath: "/etc/ceph" },
275 },
276 env_: {
277 LIB_MODULES_DIR_PATH: "/run/current-system/kernel-modules/lib/modules/",
278 ROOK_ALLOW_MULTIPLE_FILESYSTEMS: "false",
Sergiusz Bazanskic6da1272019-04-02 00:06:13 +0200[diff] [blame]279 ROOK_LOG_LEVEL: "DEBUG",
Sergiusz Bazanskicdfafaf2019-04-01 19:16:18 +0200[diff] [blame]280 ROOK_MON_HEALTHCHECK_INTERVAL: "45s",
281 ROOK_MON_OUT_TIMEOUT: "600s",
282 ROOK_DISCOVER_DEVICES_INTERVAL: "60m",
283 ROOK_HOSTPATH_REQUIRES_PRIVILEGED: "false",
284 ROOK_ENABLE_SELINUX_RELABELING: "true",
285 ROOK_ENABLE_FSGROUP: "true",
286 NODE_NAME: kube.FieldRef("spec.nodeName"),
287 POD_NAME: kube.FieldRef("metadata.name"),
288 POD_NAMESPACE: kube.FieldRef("metadata.namespace"),
289 },
290 },
291 },
292 volumes_: {
293 "rook-config": { emptyDir: {} },
294 "default-config-dir": { emptyDir: {} },
295 },
296 },
297 },
298 },
299 },
Sergiusz Bazanskib7fcc672019-04-01 18:40:50 +0200[diff] [blame]300 },
Sergiusz Bazanskic6da1272019-04-02 00:06:13 +0200[diff] [blame]301
302 // Create a new Ceph cluster in a new namespace.
303 Cluster(operator, name):: {
304 local cluster = self,
305 spec:: error "please define cluster spec",
306
307
308 metadata:: {
309 namespace: name,
310 },
311
312 name(suffix):: cluster.metadata.namespace + "-" + suffix,
313
314 namespace: kube.Namespace(cluster.metadata.namespace),
315
316 sa: {
317 // service accounts need to be hardcoded, see operator source.
318 osd: kube.ServiceAccount("rook-ceph-osd") {
319 metadata+: cluster.metadata,
320 },
321 mgr: kube.ServiceAccount("rook-ceph-mgr") {
322 metadata+: cluster.metadata,
323 },
324 },
325
326 roles: {
327 osd: kube.Role(cluster.name("osd")) {
328 metadata+: cluster.metadata,
329 rules: [
330 {
331 apiGroups: [""],
332 resources: ["configmaps"],
333 verbs: ["get", "list", "watch", "create", "update", "delete"],
334 }
335 ],
336 },
337 mgr: kube.Role(cluster.name("mgr")) {
338 metadata+: cluster.metadata,
339 rules: [
340 {
341 apiGroups: [""],
342 resources: ["pods", "services"],
343 verbs: ["get", "list", "watch"],
344 },
345 {
346 apiGroups: ["batch"],
347 resources: ["jobs"],
348 verbs: ["get", "list", "watch", "create", "update", "delete"],
349 },
350 {
351 apiGroups: ["ceph.rook.io"],
352 resources: ["*"],
353 verbs: ["*"],
354 },
355 ],
356 },
357 mgrSystem: kube.ClusterRole(cluster.name("mgr-system")) {
358 metadata+: cluster.metadata { namespace:: null },
359 rules: [
360 {
361 apiGroups: [""],
362 resources: ["configmaps"],
363 verbs: ["get", "list", "watch"],
364 }
365 ],
366 },
367 },
368
369 rbs: [
370 kube.RoleBinding(cluster.name(el.name)) {
371 metadata+: cluster.metadata,
372 roleRef: {
373 apiGroup: "rbac.authorization.k8s.io",
374 kind: el.role.kind,
375 name: el.role.metadata.name,
376 },
377 subjects: [
378 {
379 kind: el.sa.kind,
380 name: el.sa.metadata.name,
381 namespace: el.sa.metadata.namespace,
382 },
383 ],
384 },
385 for el in [
386 // Allow Operator SA to perform Cluster Mgmt in this namespace.
387 { name: "cluster-mgmt", role: operator.crs.clusterMgmt, sa: operator.sa },
388 { name: "osd", role: cluster.roles.osd, sa: cluster.sa.osd },
389 { name: "mgr", role: cluster.roles.mgr, sa: cluster.sa.mgr },
390 { name: "mgr-cluster", role: operator.crs.mgrCluster, sa: cluster.sa.mgr },
391 ]
392 ],
393
394 mgrSystemRB: kube.RoleBinding(cluster.name("mgr-system")) {
395 metadata+: {
396 namespace: operator.cfg.namespace,
397 },
398 roleRef: {
399 apiGroup: "rbac.authorization.k8s.io",
400 kind: cluster.roles.mgrSystem.kind,
401 name: cluster.roles.mgrSystem.metadata.name,
402 },
403 subjects: [
404 {
405 kind: cluster.sa.mgr.kind,
406 name: cluster.sa.mgr.metadata.name,
407 namespace: cluster.sa.mgr.metadata.namespace,
408 },
409 ],
410 },
411
412 cluster: kube._Object("ceph.rook.io/v1", "CephCluster", name) {
413 metadata+: cluster.metadata,
414 spec: {
415 cephVersion: {
416 image: "ceph/ceph:v13.2.5-20190319",
417 },
418 dataDirHostPath: "/var/lib/rook",
419 dashboard: {
420 ssl: false,
421 enabled: true,
422 port: 8080,
423 },
424 } + cluster.spec,
425 },
426
427 dashboardService: kube.Service(cluster.name("dashboard")) {
428 metadata+: cluster.metadata,
429 spec: {
430 ports: [
431 { name: "dashboard", port: 80, targetPort: 8080, protocol: "TCP" },
432 ],
433 selector: {
434 app: "rook-ceph-mgr",
435 rook_cluster: name,
436 },
437 type: "ClusterIP",
438 },
439 },
440
441 dashboardIngress: kube.Ingress(cluster.name("dashboard")) {
442 metadata+: cluster.metadata,
443 spec+: {
444 rules: [
445 {
446 host: "%s.hswaw.net" % name,
447 http: {
448 paths: [
449 { path: "/", backend: cluster.dashboardService.name_port },
450 ]
451 },
452 }
453 ],
454 },
Sergiusz Bazanski65f3b1d2019-04-02 01:05:38 +0200[diff] [blame^]455 }
456 },
457
458 ECBlockPool(cluster, name):: {
459 local pool = self,
460 spec:: error "spec must be specified",
461
462 pool: kube._Object("ceph.rook.io/v1", "CephBlockPool", name) {
463 metadata+: cluster.metadata,
464 spec: pool.spec,
465 },
466 metapool: kube._Object("ceph.rook.io/v1", "CephBlockPool", name + "-metadata") {
467 metadata+: cluster.metadata,
468 spec: {
469 failureDomain: "host",
470 replicated: {
471 size: 3,
472 },
473 },
474 },
475
476 storageClass: kube.StorageClass(name) {
477 provisioner: "ceph.rook.io/block",
478 parameters: {
479 blockPool: pool.metapool.metadata.name,
480 dataBlockPool: pool.pool.metadata.name,
481 clusterNamespace: pool.pool.metadata.namespace,
482 fstype: "ext4",
483 },
484 reclaimPolicy: "Retain",
Sergiusz Bazanskic6da1272019-04-02 00:06:13 +0200[diff] [blame]485 },
486 },
Sergiusz Bazanskib7fcc672019-04-01 18:40:50 +0200[diff] [blame]487}