| # Deploy Rook/Ceph Operator |
| |
| local kube = import "../../../kube/kube.libsonnet"; |
| |
| { |
| Operator: { |
| local env = self, |
| local cfg = env.cfg, |
| cfg:: { |
| image: "rook/ceph:master", |
| namespace: "rook-ceph-system", |
| }, |
| |
| metadata:: { |
| namespace: cfg.namespace, |
| labels: { |
| "operator": "rook", |
| "storage-backend": "ceph", |
| }, |
| }, |
| |
| namespace: kube.Namespace(cfg.namespace), |
| |
| crds: { |
| cephclusters: kube.CustomResourceDefinition("ceph.rook.io", "v1", "CephCluster") { |
| spec+: { |
| additionalPrinterColumns: [ |
| { name: "DataDirHostPath", type: "string", description: "Directory used on the K8s nodes", JSONPath: ".spec.dataDirHostPath" }, |
| { name: "MonCount", type: "string", description: "Number of MONs", JSONPath: ".spec.mon.count" }, |
| { name: "Age", type: "date", JSONPath: ".metadata.creationTimestamp" }, |
| { name: "State", type: "string", description: "Current State", JSONPath: ".status.state" }, |
| ], |
| validation: { |
| # Converted from official operator YAML |
| "openAPIV3Schema": { |
| "properties": { |
| "spec": { |
| "properties": { |
| "cephVersion": { |
| "properties": { |
| "allowUnsupported": { |
| "type": "boolean" |
| }, |
| "image": { |
| "type": "string" |
| }, |
| "name": { |
| "pattern": "^(luminous|mimic|nautilus)$", |
| "type": "string" |
| } |
| } |
| }, |
| "dashboard": { |
| "properties": { |
| "enabled": { |
| "type": "boolean" |
| }, |
| "urlPrefix": { |
| "type": "string" |
| }, |
| "port": { |
| "type": "integer" |
| } |
| } |
| }, |
| "dataDirHostPath": { |
| "pattern": "^/(\\S+)", |
| "type": "string" |
| }, |
| "mon": { |
| "properties": { |
| "allowMultiplePerNode": { |
| "type": "boolean" |
| }, |
| "count": { |
| "maximum": 9, |
| "minimum": 1, |
| "type": "integer" |
| }, |
| "preferredCount": { |
| "maximum": 9, |
| "minimum": 0, |
| "type": "integer" |
| } |
| }, |
| "required": [ |
| "count" |
| ] |
| }, |
| "network": { |
| "properties": { |
| "hostNetwork": { |
| "type": "boolean" |
| } |
| } |
| }, |
| "storage": { |
| "properties": { |
| "nodes": { |
| "items": {}, |
| "type": "array" |
| }, |
| "useAllDevices": {}, |
| "useAllNodes": { |
| "type": "boolean" |
| } |
| } |
| } |
| }, |
| "required": [ |
| "mon" |
| ] |
| } |
| } |
| } |
| } |
| }, |
| }, |
| cephfilesystems: kube.CustomResourceDefinition("ceph.rook.io", "v1", "CephFilesystem") { |
| spec+: { |
| additionalPrinterColumns: [ |
| { name: "MdsCount", type: "string", description: "Number of MDs", JSONPath: ".spec.metadataServer.activeCount" }, |
| { name: "Age", type: "date", JSONPath: ".metadata.creationTimestamp" }, |
| ], |
| }, |
| }, |
| cephnfses: kube.CustomResourceDefinition("ceph.rook.io", "v1", "CephNFS") { |
| spec+: { |
| names+: { |
| plural: "cephnfses", |
| shortNames: ["nfs"], |
| }, |
| }, |
| }, |
| cephobjectstores: kube.CustomResourceDefinition("ceph.rook.io", "v1", "CephObjectStore"), |
| cephobjectstoreusers: kube.CustomResourceDefinition("ceph.rook.io", "v1", "CephObjectStoreUser"), |
| cephblockpools: kube.CustomResourceDefinition("ceph.rook.io", "v1", "CephBlockPool"), |
| volumes: kube.CustomResourceDefinition("rook.io", "v1alpha2", "Volume") { |
| spec+: { |
| names+: { |
| shortNames: ["rv"], |
| }, |
| }, |
| }, |
| }, |
| |
| sa: kube.ServiceAccount("rook-ceph-system") { |
| metadata+: env.metadata, |
| }, |
| |
| crs: { |
| clusterMgmt: kube.ClusterRole("rook-ceph-cluster-mgmt") { |
| metadata+: env.metadata { namespace:: null }, |
| rules: [ |
| { |
| apiGroups: [""], |
| resources: ["secrets", "pods", "pods/log", "services", "configmaps"], |
| verbs: ["get", "list", "watch", "patch", "create", "update", "delete"], |
| }, |
| { |
| apiGroups: ["apps"], |
| resources: ["deployments", "daemonsets", "replicasets"], |
| verbs: ["get", "list", "watch", "create", "update", "delete"], |
| }, |
| ], |
| }, |
| global: kube.ClusterRole("rook-ceph-global") { |
| metadata+: env.metadata { namespace:: null }, |
| rules: [ |
| { |
| apiGroups: [""], |
| resources: ["pods", "nodes", "nodes/proxy"], |
| verbs: ["get", "list", "watch"], |
| }, |
| { |
| apiGroups: [""], |
| resources: ["events", "persistentvolumes", "persistentvolumeclaims", "endpoints"], |
| verbs: ["get", "list", "watch", "patch", "create", "update", "delete"], |
| }, |
| { |
| apiGroups: ["storage.k8s.io"], |
| resources: ["storageclasses"], |
| verbs: ["get", "list", "watch", "create", "update", "delete"], |
| }, |
| { |
| apiGroups: ["batch"], |
| resources: ["jobs"], |
| verbs: ["get", "list", "watch", "create", "update", "delete"], |
| }, |
| { |
| apiGroups: ["ceph.rook.io"], |
| resources: ["*"], |
| verbs: ["*"], |
| }, |
| { |
| apiGroups: ["rook.io"], |
| resources: ["*"], |
| verbs: ["*"], |
| }, |
| ], |
| }, |
| mgrCluster: kube.ClusterRole("rook-ceph-mgr-cluster") { |
| metadata+: env.metadata { namespace:: null }, |
| rules: [ |
| { |
| apiGroups: [""], |
| resources: ["configmaps", "nodes", "nodes/proxy"], |
| verbs: ["get", "list", "watch"], |
| }, |
| ] |
| }, |
| }, |
| |
| crb: kube.ClusterRoleBinding("ceph-rook-global") { |
| metadata+: env.metadata { namespace:: null }, |
| roleRef: { |
| apiGroup: "rbac.authorization.k8s.io", |
| kind: "ClusterRole", |
| name: env.crs.global.metadata.name, |
| }, |
| subjects: [ |
| { |
| kind: "ServiceAccount", |
| name: env.sa.metadata.name, |
| namespace: env.sa.metadata.namespace, |
| }, |
| ], |
| }, |
| |
| role: kube.Role("ceph-rook-system") { |
| metadata+: env.metadata, |
| rules: [ |
| { |
| apiGroups: [""], |
| resources: ["pods", "configmaps"], |
| verbs: ["get", "list", "watch", "patch", "create", "update", "delete"], |
| }, |
| { |
| apiGroups: ["apps"], |
| resources: ["daemonsets"], |
| verbs: ["get", "list", "watch", "create", "update", "delete"], |
| }, |
| ], |
| }, |
| |
| rb: kube.RoleBinding("ceph-rook-system") { |
| metadata+: env.metadata, |
| roleRef: { |
| apiGroup: "rbac.authorization.k8s.io", |
| kind: "Role", |
| name: env.role.metadata.name, |
| }, |
| subjects: [ |
| { |
| kind: "ServiceAccount", |
| name: env.sa.metadata.name, |
| namespace: env.sa.metadata.namespace, |
| }, |
| ], |
| }, |
| |
| operator: kube.Deployment("rook-ceph-operator") { |
| metadata+: env.metadata, |
| spec+: { |
| template+: { |
| spec+: { |
| serviceAccountName: env.sa.metadata.name, |
| containers_: { |
| operator: kube.Container("rook-ceph-operator") { |
| image: cfg.image, |
| args: ["ceph", "operator"], |
| volumeMounts_: { |
| "rook-config": { mountPath: "/var/lib/rook" }, |
| "default-config-dir": { mountPath: "/etc/ceph" }, |
| }, |
| env_: { |
| LIB_MODULES_DIR_PATH: "/run/current-system/kernel-modules/lib/modules/", |
| ROOK_ALLOW_MULTIPLE_FILESYSTEMS: "false", |
| ROOK_LOG_LEVEL: "DEBUG", |
| ROOK_MON_HEALTHCHECK_INTERVAL: "45s", |
| ROOK_MON_OUT_TIMEOUT: "600s", |
| ROOK_DISCOVER_DEVICES_INTERVAL: "60m", |
| ROOK_HOSTPATH_REQUIRES_PRIVILEGED: "false", |
| ROOK_ENABLE_SELINUX_RELABELING: "true", |
| ROOK_ENABLE_FSGROUP: "true", |
| NODE_NAME: kube.FieldRef("spec.nodeName"), |
| POD_NAME: kube.FieldRef("metadata.name"), |
| POD_NAMESPACE: kube.FieldRef("metadata.namespace"), |
| }, |
| }, |
| }, |
| volumes_: { |
| "rook-config": { emptyDir: {} }, |
| "default-config-dir": { emptyDir: {} }, |
| }, |
| }, |
| }, |
| }, |
| }, |
| }, |
| |
| // Create a new Ceph cluster in a new namespace. |
| Cluster(operator, name):: { |
| local cluster = self, |
| spec:: error "please define cluster spec", |
| |
| |
| metadata:: { |
| namespace: name, |
| }, |
| |
| name(suffix):: cluster.metadata.namespace + "-" + suffix, |
| |
| namespace: kube.Namespace(cluster.metadata.namespace), |
| |
| sa: { |
| // service accounts need to be hardcoded, see operator source. |
| osd: kube.ServiceAccount("rook-ceph-osd") { |
| metadata+: cluster.metadata, |
| }, |
| mgr: kube.ServiceAccount("rook-ceph-mgr") { |
| metadata+: cluster.metadata, |
| }, |
| }, |
| |
| roles: { |
| osd: kube.Role(cluster.name("osd")) { |
| metadata+: cluster.metadata, |
| rules: [ |
| { |
| apiGroups: [""], |
| resources: ["configmaps"], |
| verbs: ["get", "list", "watch", "create", "update", "delete"], |
| } |
| ], |
| }, |
| mgr: kube.Role(cluster.name("mgr")) { |
| metadata+: cluster.metadata, |
| rules: [ |
| { |
| apiGroups: [""], |
| resources: ["pods", "services"], |
| verbs: ["get", "list", "watch"], |
| }, |
| { |
| apiGroups: ["batch"], |
| resources: ["jobs"], |
| verbs: ["get", "list", "watch", "create", "update", "delete"], |
| }, |
| { |
| apiGroups: ["ceph.rook.io"], |
| resources: ["*"], |
| verbs: ["*"], |
| }, |
| ], |
| }, |
| mgrSystem: kube.ClusterRole(cluster.name("mgr-system")) { |
| metadata+: cluster.metadata { namespace:: null }, |
| rules: [ |
| { |
| apiGroups: [""], |
| resources: ["configmaps"], |
| verbs: ["get", "list", "watch"], |
| } |
| ], |
| }, |
| }, |
| |
| rbs: [ |
| kube.RoleBinding(cluster.name(el.name)) { |
| metadata+: cluster.metadata, |
| roleRef: { |
| apiGroup: "rbac.authorization.k8s.io", |
| kind: el.role.kind, |
| name: el.role.metadata.name, |
| }, |
| subjects: [ |
| { |
| kind: el.sa.kind, |
| name: el.sa.metadata.name, |
| namespace: el.sa.metadata.namespace, |
| }, |
| ], |
| }, |
| for el in [ |
| // Allow Operator SA to perform Cluster Mgmt in this namespace. |
| { name: "cluster-mgmt", role: operator.crs.clusterMgmt, sa: operator.sa }, |
| { name: "osd", role: cluster.roles.osd, sa: cluster.sa.osd }, |
| { name: "mgr", role: cluster.roles.mgr, sa: cluster.sa.mgr }, |
| { name: "mgr-cluster", role: operator.crs.mgrCluster, sa: cluster.sa.mgr }, |
| ] |
| ], |
| |
| mgrSystemRB: kube.RoleBinding(cluster.name("mgr-system")) { |
| metadata+: { |
| namespace: operator.cfg.namespace, |
| }, |
| roleRef: { |
| apiGroup: "rbac.authorization.k8s.io", |
| kind: cluster.roles.mgrSystem.kind, |
| name: cluster.roles.mgrSystem.metadata.name, |
| }, |
| subjects: [ |
| { |
| kind: cluster.sa.mgr.kind, |
| name: cluster.sa.mgr.metadata.name, |
| namespace: cluster.sa.mgr.metadata.namespace, |
| }, |
| ], |
| }, |
| |
| cluster: kube._Object("ceph.rook.io/v1", "CephCluster", name) { |
| metadata+: cluster.metadata, |
| spec: { |
| cephVersion: { |
| image: "ceph/ceph:v13.2.5-20190319", |
| }, |
| dataDirHostPath: "/var/lib/rook", |
| dashboard: { |
| ssl: false, |
| enabled: true, |
| port: 8080, |
| }, |
| } + cluster.spec, |
| }, |
| |
| dashboardService: kube.Service(cluster.name("dashboard")) { |
| metadata+: cluster.metadata, |
| spec: { |
| ports: [ |
| { name: "dashboard", port: 80, targetPort: 8080, protocol: "TCP" }, |
| ], |
| selector: { |
| app: "rook-ceph-mgr", |
| rook_cluster: name, |
| }, |
| type: "ClusterIP", |
| }, |
| }, |
| |
| dashboardIngress: kube.Ingress(cluster.name("dashboard")) { |
| metadata+: cluster.metadata, |
| spec+: { |
| rules: [ |
| { |
| host: "%s.hswaw.net" % name, |
| http: { |
| paths: [ |
| { path: "/", backend: cluster.dashboardService.name_port }, |
| ] |
| }, |
| } |
| ], |
| }, |
| } |
| }, |
| |
| ECBlockPool(cluster, name):: { |
| local pool = self, |
| spec:: error "spec must be specified", |
| |
| pool: kube._Object("ceph.rook.io/v1", "CephBlockPool", name) { |
| metadata+: cluster.metadata, |
| spec: pool.spec, |
| }, |
| metapool: kube._Object("ceph.rook.io/v1", "CephBlockPool", name + "-metadata") { |
| metadata+: cluster.metadata, |
| spec: { |
| failureDomain: "host", |
| replicated: { |
| size: 3, |
| }, |
| }, |
| }, |
| |
| storageClass: kube.StorageClass(name) { |
| provisioner: "ceph.rook.io/block", |
| parameters: { |
| blockPool: pool.metapool.metadata.name, |
| dataBlockPool: pool.pool.metadata.name, |
| clusterNamespace: pool.pool.metadata.namespace, |
| fstype: "ext4", |
| }, |
| reclaimPolicy: "Retain", |
| }, |
| }, |
| } |