| # Deploy Rook/Ceph Operator |
| |
| local kube = import "../../../kube/hscloud.libsonnet"; |
| local policies = import "../../../kube/policies.libsonnet"; |
| |
| local oa = kube.OpenAPI; |
| |
| { |
| Operator: { |
| local env = self, |
| local cfg = env.cfg, |
| cfg:: { |
| image: "rook/ceph:v1.6.9", |
| namespace: "rook-ceph-system", |
| }, |
| |
| metadata:: { |
| namespace: cfg.namespace, |
| labels: { |
| "operator": "rook", |
| "storage-backend": "ceph", |
| }, |
| }, |
| |
| namespace: kube.Namespace(cfg.namespace), |
| |
| policyInsecure: policies.AllowNamespaceInsecure(cfg.namespace), |
| |
| // Grab CRDs from upstream YAML. |
| // |
| // We use a bit of jsonnet to remove some fields that kubebuilder (used |
| // upstream) added and to override preserveUnknownFIelds (as some older |
| // deployment apparently set it to true, which doesn't work for new |
| // CRDs that have default values). |
| crds: [ |
| (el { |
| metadata+: { |
| annotations:: null, |
| creationTimestamp:: null, |
| }, |
| status:: null, |
| spec+: { |
| preserveUnknownFields: false, |
| }, |
| }) |
| for el in (std.native("parseYaml")(importstr "rook-crds.yaml")) if el != null |
| ], |
| |
| sa: { |
| system: kube.ServiceAccount("rook-ceph-system") { |
| metadata+: env.metadata, |
| }, |
| csiCephfsPlugin: kube.ServiceAccount("rook-csi-cephfs-plugin-sa") { |
| metadata+: env.metadata, |
| }, |
| csiCephfsProvisioner: kube.ServiceAccount("rook-csi-cephfs-provisioner-sa") { |
| metadata+: env.metadata, |
| }, |
| csiRbdPlugin: kube.ServiceAccount("rook-csi-rbd-plugin-sa") { |
| metadata+: env.metadata, |
| }, |
| csiRbdProvisioner: kube.ServiceAccount("rook-csi-rbd-provisioner-sa") { |
| metadata+: env.metadata, |
| }, |
| }, |
| |
| crs: { |
| clusterMgmt: kube.ClusterRole("rook-ceph-cluster-mgmt") { |
| metadata+: env.metadata { namespace:: null }, |
| rules: [ |
| { |
| apiGroups: ["", "apps", "extensions"], |
| resources: ["secrets", "pods", "pods/log", "services", "configmaps", "deployments", "daemonsets"], |
| verbs: ["get", "list", "watch", "patch", "create", "update", "delete"], |
| }, |
| ], |
| }, |
| global: kube.ClusterRole("rook-ceph-global") { |
| metadata+: env.metadata { |
| namespace:: null, |
| labels+: { |
| "rbac.ceph.rook.io/aggregate-to-rook-ceph-global": "true", |
| }, |
| }, |
| rules: [ |
| { |
| apiGroups: [""], |
| resources: ["pods", "nodes", "nodes/proxy", "services"], |
| verbs: ["get", "list", "watch"], |
| }, |
| { |
| apiGroups: [""], |
| resources: ["events", "persistentvolumes", "persistentvolumeclaims", "endpoints"], |
| verbs: ["get", "list", "watch", "patch", "create", "update", "delete"], |
| }, |
| { |
| apiGroups: ["storage.k8s.io"], |
| resources: ["storageclasses"], |
| verbs: ["get", "list", "watch"], |
| }, |
| { |
| apiGroups: ["batch"], |
| resources: ["jobs", "cronjobs"], |
| verbs: ["get", "list", "watch", "create", "update", "delete"], |
| }, |
| { |
| apiGroups: ["ceph.rook.io"], |
| resources: ["*"], |
| verbs: ["*"], |
| }, |
| { |
| apiGroups: ["rook.io"], |
| resources: ["*"], |
| verbs: ["*"], |
| }, |
| { |
| apiGroups: ["policy", "apps", "extensions"], |
| resources: ["poddisruptionbudgets", "deployments", "replicasets"], |
| verbs: ["*"], |
| }, |
| { |
| apiGroups: ["healthchecking.openshift.io"], |
| resources: ["machinedisruptionbudgets"], |
| verbs: ["get", "list", "watch", "create", "update", "delete"], |
| }, |
| { |
| apiGroups: ["machine.openshift.io"], |
| resources: ["machines"], |
| verbs: ["get", "list", "watch", "create", "update", "delete"], |
| }, |
| { |
| apiGroups: ["storage.k8s.io"], |
| resources: ["csidrivers"], |
| verbs: ["create", "delete", "get", "update"], |
| }, |
| { |
| apiGroups: ["k8s.cni.cncf.io"], |
| resources: ["network-attachment-definitions"], |
| verbs: ["get"], |
| }, |
| ], |
| }, |
| |
| mgrCluster: kube.ClusterRole("rook-ceph-mgr-cluster") { |
| metadata+: env.metadata { namespace:: null }, |
| rules: [ |
| { |
| apiGroups: [""], |
| resources: ["configmaps", "nodes", "nodes/proxy"], |
| verbs: ["get", "list", "watch"], |
| }, |
| { |
| apiGroups: [""], |
| resources: ["events"], |
| verbs: ["create", "patch", "list", "get", "watch"], |
| }, |
| ] |
| }, |
| objectBucket: kube.ClusterRole("rook-ceph-object-bucket") { |
| metadata+: env.metadata { |
| namespace:: null, |
| }, |
| rules: [ |
| { |
| apiGroups: [""], |
| resources: ["secrets", "configmaps"], |
| verbs: ["*"], |
| }, |
| { |
| apiGroups: ["storage.k8s.io"], |
| resources: ["storageclasses"], |
| verbs: ["get", "list", "watch"], |
| }, |
| { |
| apiGroups: ["objectbucket.io"], |
| resources: ["*"], |
| verbs: ["*"], |
| }, |
| ], |
| }, |
| |
| cephfsCSINodeplugin: kube.ClusterRole("cephfs-csi-nodeplugin") { |
| metadata+: env.metadata { namespace:: null }, |
| rules: [ |
| { |
| apiGroups: [""], |
| resources: ["nodes"], |
| verbs: ["get", "list", "update"], |
| }, |
| { |
| apiGroups: [""], |
| resources: ["namespaces"], |
| verbs: ["get", "list"], |
| }, |
| { |
| apiGroups: [""], |
| resources: ["persistentvolumes"], |
| verbs: ["get", "list", "watch", "update"], |
| }, |
| { |
| apiGroups: ["storage.k8s.io"], |
| resources: ["volumeattachments"], |
| verbs: ["get", "list", "watch", "update"], |
| }, |
| { |
| apiGroups: [""], |
| resources: ["configmaps"], |
| verbs: ["get", "list"], |
| }, |
| ], |
| }, |
| |
| cephfsExternalProvisionerRunner: kube.ClusterRole("cephfs-external-provisioner-runner") { |
| metadata+: env.metadata { |
| namespace:: null, |
| labels+: { |
| "rbac.ceph.rook.io/aggregate-to-cephfs-external-provisioner-runner": "true", |
| }, |
| }, |
| rules: [ |
| { |
| apiGroups: [""], |
| resources: ["secrets"], |
| verbs: ["get", "list"], |
| }, |
| { |
| apiGroups: [""], |
| resources: ["persistentvolumes"], |
| verbs: ["get", "list", "watch", "create", "delete", "update", "patch"], |
| }, |
| { |
| apiGroups: [""], |
| resources: ["persistentvolumeclaims"], |
| verbs: ["get", "list", "watch", "update"], |
| }, |
| { |
| apiGroups: ["storage.k8s.io"], |
| resources: ["storageclasses"], |
| verbs: ["get", "list", "watch"], |
| }, |
| { |
| apiGroups: [""], |
| resources: ["events"], |
| verbs: ["list", "watch", "create", "update", "patch"], |
| }, |
| { |
| apiGroups: ["snapshot.storage.k8s.io"], |
| resources: ["volumesnapshots"], |
| verbs: ["get", "list", "watch", "update"], |
| }, |
| { |
| apiGroups: ["snapshot.storage.k8s.io"], |
| resources: ["volumesnapshotcontents"], |
| verbs: ["create", "get", "list", "watch", "update", "delete"], |
| }, |
| { |
| apiGroups: ["snapshot.storage.k8s.io"], |
| resources: ["volumesnapshotclasses"], |
| verbs: ["get", "list", "watch"], |
| }, |
| { |
| apiGroups: ["snapshot.storage.k8s.io"], |
| resources: ["volumesnapshotcontents/status"], |
| verbs: ["update"], |
| }, |
| { |
| apiGroups: ["apiextensions.k8s.io"], |
| resources: ["customresourcedefinitions"], |
| verbs: ["create", "list", "watch", "delete", "get", "update"], |
| }, |
| { |
| apiGroups: ["snapshot.storage.k8s.io"], |
| resources: ["volumesnapshots/status"], |
| verbs: ["update"], |
| }, |
| { |
| apiGroups: ["storage.k8s.io"], |
| resources: ["volumeattachments"], |
| verbs: ["get", "list", "watch", "update", "patch"], |
| }, |
| { |
| apiGroups: ["storage.k8s.io"], |
| resources: ["volumeattachments/status"], |
| verbs: ["patch"], |
| }, |
| { |
| apiGroups: [""], |
| resources: ["nodes"], |
| verbs: ["get", "list", "watch"], |
| }, |
| { |
| apiGroups: [""], |
| resources: ["persistentvolumeclaims/status"], |
| verbs: ["update", "patch"], |
| }, |
| ], |
| }, |
| |
| rbdCSINodeplugin: kube.ClusterRole("rbd-csi-nodeplugin") { |
| metadata+: env.metadata { namespace:: null }, |
| rules: [ |
| { |
| apiGroups: [""], |
| resources: ["secrets"], |
| verbs: ["get", "list"], |
| }, |
| { |
| apiGroups: [""], |
| resources: ["nodes"], |
| verbs: ["get", "list", "update"], |
| }, |
| { |
| apiGroups: [""], |
| resources: ["namespaces"], |
| verbs: ["get", "list"], |
| }, |
| { |
| apiGroups: [""], |
| resources: ["persistentvolumes"], |
| verbs: ["get", "list", "watch", "update"], |
| }, |
| { |
| apiGroups: ["storage.k8s.io"], |
| resources: ["volumeattachments"], |
| verbs: ["get", "list", "watch", "update"], |
| }, |
| { |
| apiGroups: [""], |
| resources: ["configmaps"], |
| verbs: ["get", "list"], |
| }, |
| ], |
| }, |
| |
| rbdExternalProvisionerRunner: kube.ClusterRole("rbd-external-provisioner-runner") { |
| metadata+: env.metadata { |
| namespace:: null, |
| labels+: { |
| "rbac.ceph.rook.io/aggregate-to-rbd-external-provisioner-runner": "true", |
| }, |
| }, |
| rules: [ |
| { |
| apiGroups: [""], |
| resources: ["secrets"], |
| verbs: ["get", "list", "watch"], |
| }, |
| { |
| apiGroups: [""], |
| resources: ["persistentvolumes"], |
| verbs: ["get", "list", "watch", "create", "delete", "update", "patch"], |
| }, |
| { |
| apiGroups: [""], |
| resources: ["persistentvolumeclaims"], |
| verbs: ["get", "list", "watch", "update"], |
| }, |
| { |
| apiGroups: ["storage.k8s.io"], |
| resources: ["volumeattachments"], |
| verbs: ["get", "list", "watch", "update", "patch"], |
| }, |
| { |
| apiGroups: ["storage.k8s.io"], |
| resources: ["volumeattachments/status"], |
| verbs: ["patch"], |
| }, |
| { |
| apiGroups: [""], |
| resources: ["nodes"], |
| verbs: ["get", "list", "watch"], |
| }, |
| { |
| apiGroups: ["storage.k8s.io"], |
| resources: ["storageclasses"], |
| verbs: ["get", "list", "watch"] |
| }, |
| { |
| apiGroups: [""], |
| resources: ["events"], |
| verbs: ["list", "watch", "create", "update", "patch"], |
| }, |
| { |
| apiGroups: ["snapshot.storage.k8s.io"], |
| resources: ["volumesnapshots"], |
| verbs: ["get", "list", "watch", "update"], |
| }, |
| { |
| apiGroups: ["snapshot.storage.k8s.io"], |
| resources: ["volumesnapshotcontents"], |
| verbs: ["create", "get", "list", "watch", "update", "delete"], |
| }, |
| { |
| apiGroups: ["snapshot.storage.k8s.io"], |
| resources: ["volumesnapshotclasses"], |
| verbs: ["get", "list", "watch"], |
| }, |
| { |
| apiGroups: ["snapshot.storage.k8s.io"], |
| resources: ["volumesnapshotcontents/status"], |
| verbs: ["update"], |
| }, |
| { |
| apiGroups: ["apiextensions.k8s.io"], |
| resources: ["customresourcedefinitions"], |
| verbs: ["create", "list", "watch", "delete", "get", "update"], |
| }, |
| { |
| apiGroups: ["snapshot.storage.k8s.io"], |
| resources: ["volumesnapshots/status"], |
| verbs: ["update"], |
| }, |
| { |
| apiGroups: [""], |
| resources: ["persistentvolumeclaims/status"], |
| verbs: ["update", "patch"], |
| }, |
| { |
| apiGroups: [""], |
| resources: ["configmaps"], |
| verbs: ["get"], |
| }, |
| { |
| apiGroups: ["replication.storage.openshift.io"], |
| resources: ["volumereplications", "volumereplicationclasses"], |
| verbs: ["create", "delete", "get", "list", "patch", "update", "watch"], |
| }, |
| { |
| apiGroups: ["replication.storage.openshift.io"], |
| resources: ["volumereplications/finalizers"], |
| verbs: ["update"], |
| }, |
| { |
| apiGroups: ["replication.storage.openshift.io"], |
| resources: ["volumereplications/status"], |
| verbs: ["get", "patch", "update"], |
| }, |
| { |
| apiGroups: ["replication.storage.openshift.io"], |
| resources: ["volumereplicationclasses/status"], |
| verbs: ["get"], |
| }, |
| ], |
| }, |
| }, |
| |
| crbs: { |
| global: kube.ClusterRoleBinding("ceph-rook-global") { |
| metadata+: env.metadata { namespace:: null }, |
| roleRef_: env.crs.global, |
| subjects_: [env.sa.system], |
| }, |
| objectBucket: kube.ClusterRoleBinding("rook-ceph-object-bucket") { |
| metadata+: env.metadata { namespace:: null }, |
| roleRef_: env.crs.objectBucket, |
| subjects_: [env.sa.system], |
| }, |
| cephfsCSINodeplugin: kube.ClusterRoleBinding("cephfs-csi-nodeplugin") { |
| metadata+: env.metadata { namespace:: null }, |
| roleRef_: env.crs.cephfsCSINodeplugin, |
| subjects_: [env.sa.csiCephfsPlugin], |
| }, |
| cephfsCSIProvisioner: kube.ClusterRoleBinding("cephfs-csi-provisioner") { |
| metadata+: env.metadata { namespace:: null }, |
| roleRef_: env.crs.cephfsExternalProvisionerRunner, |
| subjects_: [env.sa.csiCephfsProvisioner], |
| }, |
| rbdCSINodeplugin: kube.ClusterRoleBinding("rbd-csi-nodeplugin") { |
| metadata+: env.metadata { namespace:: null }, |
| roleRef_: env.crs.rbdCSINodeplugin, |
| subjects_: [env.sa.csiRbdPlugin], |
| }, |
| rbdCSIProvisioner: kube.ClusterRoleBinding("rbd-csi-provisioner") { |
| metadata+: env.metadata { namespace:: null }, |
| roleRef_: env.crs.rbdExternalProvisionerRunner, |
| subjects_: [env.sa.csiRbdProvisioner], |
| }, |
| }, |
| |
| roles: { |
| system: kube.Role("ceph-rook-system") { |
| metadata+: env.metadata, |
| rules: [ |
| { |
| apiGroups: [""], |
| resources: ["pods", "configmaps", "services"], |
| verbs: ["get", "list", "watch", "patch", "create", "update", "delete"], |
| }, |
| { |
| apiGroups: ["apps"], |
| resources: ["daemonsets", "statefulsets", "deployments"], |
| verbs: ["get", "list", "watch", "create", "update", "delete"], |
| }, |
| { |
| apiGroups: ["k8s.cni.cncf.io"], |
| resources: ["network-attachment-definitions"], |
| verbs: ["get"], |
| }, |
| ], |
| }, |
| cephfsExternalProvisioner: kube.Role("cephfs-external-provisioner-cfg") { |
| metadata+: env.metadata, |
| rules: [ |
| { |
| apiGroups: [""], |
| resources: ["endpoints"], |
| verbs: ["get", "watch", "list", "delete", "update", "create"], |
| }, |
| { |
| apiGroups: [""], |
| resources: ["configmaps"], |
| verbs: ["get", "list", "create", "delete"], |
| }, |
| { |
| apiGroups: ["coordination.k8s.io"], |
| resources: ["leases"], |
| verbs: ["get" ,"watch", "list", "delete", "update", "create"], |
| }, |
| ], |
| }, |
| rbdExternalProvisioner: kube.Role("rbd-external-provisioner-cfg") { |
| metadata+: env.metadata, |
| rules: [ |
| { |
| apiGroups: [""], |
| resources: ["endpoints"], |
| verbs: ["get", "watch", "list", "delete", "update", "create"], |
| }, |
| { |
| apiGroups: [""], |
| resources: ["configmaps"], |
| verbs: ["get", "list", "watch", "create", "delete", "update"], |
| }, |
| { |
| apiGroups: ["coordination.k8s.io"], |
| resources: ["leases"], |
| verbs: ["get" ,"watch", "list", "delete", "update", "create"], |
| }, |
| ], |
| }, |
| }, |
| |
| rbs: { |
| system: kube.RoleBinding("ceph-rook-system") { |
| metadata+: env.metadata, |
| roleRef_: env.roles.system, |
| subjects_: [env.sa.system], |
| }, |
| cephfsCSIProvisioner: kube.RoleBinding("cephfs-csi-provisioner-role-cfg") { |
| metadata+: env.metadata, |
| roleRef_: env.roles.cephfsExternalProvisioner, |
| subjects_: [env.sa.csiCephfsProvisioner], |
| }, |
| rbdCSIProvisioner: kube.RoleBinding("rbd-csi-provisioner-role-cfg") { |
| metadata+: env.metadata, |
| roleRef_: env.roles.rbdExternalProvisioner, |
| subjects_: [env.sa.csiRbdProvisioner], |
| }, |
| }, |
| |
| operator: kube.Deployment("rook-ceph-operator") { |
| metadata+: env.metadata, |
| spec+: { |
| template+: { |
| spec+: { |
| serviceAccountName: env.sa.system.metadata.name, |
| containers_: { |
| operator: kube.Container("rook-ceph-operator") { |
| image: cfg.image, |
| args: ["ceph", "operator"], |
| volumeMounts_: { |
| "rook-config": { mountPath: "/var/lib/rook" }, |
| "default-config-dir": { mountPath: "/etc/ceph" }, |
| }, |
| env_: { |
| LIB_MODULES_DIR_PATH: "/run/current-system/kernel-modules/lib/modules/", |
| ROOK_ALLOW_MULTIPLE_FILESYSTEMS: "false", |
| ROOK_LOG_LEVEL: "INFO", |
| ROOK_MON_HEALTHCHECK_INTERVAL: "45s", |
| ROOK_MON_OUT_TIMEOUT: "600s", |
| ROOK_DISCOVER_DEVICES_INTERVAL: "60m", |
| ROOK_HOSTPATH_REQUIRES_PRIVILEGED: "false", |
| ROOK_ENABLE_SELINUX_RELABELING: "true", |
| ROOK_ENABLE_FSGROUP: "true", |
| NODE_NAME: kube.FieldRef("spec.nodeName"), |
| POD_NAME: kube.FieldRef("metadata.name"), |
| POD_NAMESPACE: kube.FieldRef("metadata.namespace"), |
| ROOK_CSI_KUBELET_DIR_PATH: "/var/lib/kubernetes", |
| ROOK_ENABLE_FLEX_DRIVER: "true", |
| }, |
| }, |
| }, |
| volumes_: { |
| "rook-config": { emptyDir: {} }, |
| "default-config-dir": { emptyDir: {} }, |
| }, |
| }, |
| }, |
| }, |
| }, |
| }, |
| |
| // Create a new Ceph cluster in a new namespace. |
| Cluster(operator, name):: { |
| local cluster = self, |
| spec:: error "please define cluster spec", |
| |
| |
| metadata:: { |
| namespace: name, |
| }, |
| |
| name(suffix):: cluster.metadata.namespace + "-" + suffix, |
| |
| namespace: kube.Namespace(cluster.metadata.namespace), |
| |
| sa: { |
| // service accounts need to be hardcoded, see operator source. |
| osd: kube.ServiceAccount("rook-ceph-osd") { |
| metadata+: cluster.metadata, |
| }, |
| mgr: kube.ServiceAccount("rook-ceph-mgr") { |
| metadata+: cluster.metadata, |
| }, |
| cmdReporter: kube.ServiceAccount("rook-ceph-cmd-reporter") { |
| metadata+: cluster.metadata, |
| }, |
| }, |
| |
| roles: { |
| osd: kube.Role(cluster.name("osd")) { |
| metadata+: cluster.metadata, |
| rules: [ |
| { |
| apiGroups: [""], |
| resources: ["configmaps"], |
| verbs: ["get", "list", "watch", "create", "update", "delete"], |
| }, |
| ], |
| }, |
| osdCluster: kube.ClusterRole(cluster.name("osd-cluster")) { |
| metadata+: cluster.metadata { namespace:: null }, |
| rules: [ |
| { |
| apiGroups: [""], |
| resources: ["nodes"], |
| verbs: ["get", "list"], |
| }, |
| ], |
| }, |
| mgr: kube.Role(cluster.name("mgr")) { |
| metadata+: cluster.metadata, |
| rules: [ |
| { |
| apiGroups: [""], |
| resources: ["pods", "services"], |
| verbs: ["get", "list", "watch"], |
| }, |
| { |
| apiGroups: ["batch"], |
| resources: ["jobs"], |
| verbs: ["get", "list", "watch", "create", "update", "delete"], |
| }, |
| { |
| apiGroups: ["ceph.rook.io"], |
| resources: ["*"], |
| verbs: ["*"], |
| }, |
| ], |
| }, |
| cmdReporter: kube.Role(cluster.name("cmd-reporter")) { |
| metadata+: cluster.metadata, |
| rules: [ |
| { |
| apiGroups: [""], |
| resources: ["pods", "configmaps"], |
| verbs: ["get", "list", "watch", "create", "update", "delete"], |
| }, |
| ], |
| }, |
| mgrSystem: kube.ClusterRole(cluster.name("mgr-system")) { |
| metadata+: cluster.metadata { namespace:: null }, |
| rules: [ |
| { |
| apiGroups: [""], |
| resources: ["configmaps"], |
| verbs: ["get", "list", "watch"], |
| } |
| ], |
| }, |
| }, |
| |
| rbs: [ |
| kube.RoleBinding(cluster.name(el.name)) { |
| metadata+: cluster.metadata, |
| roleRef_: el.role, |
| subjects_: [el.sa], |
| }, |
| for el in [ |
| // Allow Operator SA to perform Cluster Mgmt in this namespace. |
| { name: "cluster-mgmt", role: operator.crs.clusterMgmt, sa: operator.sa.system }, |
| { name: "osd", role: cluster.roles.osd, sa: cluster.sa.osd }, |
| { name: "mgr", role: cluster.roles.mgr, sa: cluster.sa.mgr }, |
| { name: "cmd-reporter", role: cluster.roles.cmdReporter, sa: cluster.sa.cmdReporter }, |
| { name: "mgr-cluster", role: operator.crs.mgrCluster, sa: cluster.sa.mgr }, |
| ] |
| ], |
| |
| mgrSystemRB: kube.RoleBinding(cluster.name("mgr-system")) { |
| metadata+: { |
| namespace: operator.cfg.namespace, |
| }, |
| roleRef_: cluster.roles.mgrSystem, |
| subjects_: [cluster.sa.mgr], |
| }, |
| |
| osdClusterRB: kube.ClusterRoleBinding(cluster.name("osd-cluster")) { |
| metadata+: { |
| namespace:: null, |
| }, |
| roleRef_: cluster.roles.osdCluster, |
| subjects_: [cluster.sa.osd], |
| }, |
| |
| |
| cluster: kube._Object("ceph.rook.io/v1", "CephCluster", name) { |
| metadata+: cluster.metadata, |
| spec: { |
| cephVersion: { |
| image: "quay.io/ceph/ceph:v16.2.5", |
| allowUnsupported: true, |
| }, |
| dataDirHostPath: if name == "ceph-waw2" then "/var/lib/rook" else "/var/lib/rook-%s" % [name], |
| dashboard: { |
| ssl: false, |
| enabled: true, |
| port: 8080, |
| }, |
| } + cluster.spec, |
| }, |
| |
| dashboardService: kube.Service(cluster.name("dashboard")) { |
| metadata+: cluster.metadata, |
| spec: { |
| ports: [ |
| { name: "dashboard", port: 80, targetPort: 8080, protocol: "TCP" }, |
| ], |
| selector: { |
| app: "rook-ceph-mgr", |
| rook_cluster: name, |
| }, |
| type: "ClusterIP", |
| }, |
| }, |
| |
| dashboardIngress: kube.SimpleIngress(cluster.name("dashboard")) { |
| hosts:: ["%s.hswaw.net" % name], |
| target_service:: cluster.dashboardService, |
| }, |
| |
| # Benji is a backup tool, external to rook, that we use for backing up |
| # RBDs. |
| benji: { |
| sa: kube.ServiceAccount(cluster.name("benji")) { |
| metadata+: cluster.metadata, |
| }, |
| |
| cr: kube.ClusterRole(cluster.name("benji")) { |
| rules: [ |
| { |
| apiGroups: [""], |
| resources: [ |
| "persistentvolumes", |
| "persistentvolumeclaims" |
| ], |
| verbs: ["list", "get"], |
| }, |
| { |
| apiGroups: [""], |
| resources: [ |
| "events", |
| ], |
| verbs: ["create", "update"], |
| }, |
| ], |
| }, |
| |
| crb: kube.ClusterRoleBinding(cluster.name("benji")) { |
| roleRef_: cluster.benji.cr, |
| subjects_: [cluster.benji.sa], |
| }, |
| |
| config: kube.Secret(cluster.name("benji-config")) { |
| metadata+: cluster.metadata, |
| data_: { |
| "benji.yaml": std.manifestJson({ |
| configurationVersion: '1', |
| databaseEngine: 'sqlite:////data/benji.sqlite', |
| defaultStorage: 'wasabi', |
| storages: [ |
| { |
| name: "wasabi", |
| storageId: 1, |
| module: "s3", |
| configuration: cluster.spec.benji.s3Configuration { |
| activeTransforms: ["encrypt"], |
| }, |
| }, |
| ], |
| transforms: [ |
| { |
| name: "encrypt", |
| module: "aes_256_gcm", |
| configuration: { |
| # not secret. |
| kdfSalt: "T2huZzZpcGhhaWM3QWVwaDhybzRhaDNhbzFpc2VpOWFobDNSZWVQaGVvTWV1bmVaYWVsNHRoYWg5QWVENHNoYWg0ZGFoN3Rlb3NvcHVuZzNpZXZpMm9vTG9vbmc1YWlmb0RlZXAwYmFobDlab294b2hjaG9odjRhbzFsYWkwYWk=", |
| kdfIterations: 2137, |
| password: cluster.spec.benji.encryptionPassword, |
| }, |
| }, |
| ], |
| ios: [ |
| { name: pool, module: "rbd" } |
| for pool in cluster.spec.benji.pools |
| ], |
| }), |
| }, |
| }, |
| |
| # Yes, Benji keeps data (backup metadata) on the ceph cluster that |
| # it backs up. However: |
| # - we add a command to benji-k8s to also copy over the sqlite |
| # database over to s3 |
| # - benji can, in a pinch, restore without a database if a version |
| # is known: https://benji-backup.me/restore.html#restoring-without-a-database |
| data: kube.PersistentVolumeClaim(cluster.name("benji-data")) { |
| metadata+: cluster.metadata, |
| spec+: { |
| storageClassName: cluster.spec.benji.metadataStorageClass, |
| accessModes: [ "ReadWriteOnce" ], |
| resources: { |
| requests: { |
| storage: "1Gi", |
| }, |
| }, |
| }, |
| }, |
| |
| # Extra scripts. |
| extrabins: kube.ConfigMap(cluster.name("benji-extrabins")) { |
| metadata+: cluster.metadata, |
| data: { |
| "metabackup.sh" : ||| |
| # Make backups of sqlite3 metadata used by Benji. |
| # The backups live in the same bucket as backups, and the metabackups |
| # are named `metabackup-0..10`, where 0 is the newest backup. Any time |
| # this script is called, backups get shifted one way to the left (9 to 10, |
| # 8 to 9, etc). This ensures we have at least 10 backup replicas. |
| |
| set -e |
| |
| which s3cmd || pip install --upgrade s3cmd |
| |
| AWS_ACCESS_KEY_ID=$(jq -r .storages[0].configuration.awsAccessKeyId < /etc/benji/benji.yaml) |
| AWS_SECRET_ACCESS_KEY=$(jq -r .storages[0].configuration.awsSecretAccessKey < /etc/benji/benji.yaml) |
| BUCKET=$(jq -r .storages[0].configuration.bucketName < /etc/benji/benji.yaml) |
| |
| s3() { |
| s3cmd --host=s3.wasabisys.com \ |
| "--host-bucket=%(bucket)s.s3.wasabisys.com" \ |
| --region=eu-central-1 \ |
| --access_key=$AWS_ACCESS_KEY_ID \ |
| --secret_key=$AWS_SECRET_ACCESS_KEY \ |
| "$@" |
| } |
| |
| # Copy over old backups, if they exist. |
| for i in `seq 9 -1 0`; do |
| from="s3://$BUCKET/metabackup-$i.sqlite3" |
| to="s3://$BUCKET/metabackup-$((i+1)).sqlite3" |
| |
| if [[ $(s3 ls $from | wc -l) -eq 0 ]]; then |
| echo "$from does not exist, skipping shift." |
| continue |
| fi |
| echo "Moving $from to $to..." |
| s3 mv $from $to |
| done |
| |
| # Make new metabackup. |
| s3 put /data/benji.sqlite s3://$BUCKET/metabackup-0.sqlite3 |
| |
| |||, |
| "get-rook-creds.sh": ||| |
| # Based on the Rook Toolbox /usr/local/bin/toolbox.sh script. |
| # Copyright 2016 The Rook Authors. All rights reserved. |
| |
| CEPH_CONFIG="/etc/ceph/ceph.conf" |
| MON_CONFIG="/etc/rook/mon-endpoints" |
| KEYRING_FILE="/etc/ceph/keyring" |
| |
| # create a ceph config file in its default location so ceph/rados tools can be used |
| # without specifying any arguments |
| write_endpoints() { |
| endpoints=$(cat ${MON_CONFIG}) |
| |
| # filter out the mon names |
| mon_endpoints=$(echo ${endpoints} | sed 's/[a-z]\+=//g') |
| |
| # filter out the legacy mon names |
| mon_endpoints=$(echo ${mon_endpoints} | sed 's/rook-ceph-mon[0-9]\+=//g') |
| |
| DATE=$(date) |
| echo "$DATE writing mon endpoints to ${CEPH_CONFIG}: ${endpoints}" |
| cat <<EOF > ${CEPH_CONFIG} |
| [global] |
| mon_host = ${mon_endpoints} |
| |
| [client.admin] |
| keyring = ${KEYRING_FILE} |
| EOF |
| } |
| |
| # watch the endpoints config file and update if the mon endpoints ever change |
| watch_endpoints() { |
| # get the timestamp for the target of the soft link |
| real_path=$(realpath ${MON_CONFIG}) |
| initial_time=$(stat -c %Z ${real_path}) |
| while true; do |
| real_path=$(realpath ${MON_CONFIG}) |
| latest_time=$(stat -c %Z ${real_path}) |
| |
| if [[ "${latest_time}" != "${initial_time}" ]]; then |
| write_endpoints |
| initial_time=${latest_time} |
| fi |
| sleep 10 |
| done |
| } |
| |
| # create the keyring file |
| cat <<EOF > ${KEYRING_FILE} |
| [client.admin] |
| key = ${ROOK_ADMIN_SECRET} |
| EOF |
| |
| # write the initial config file |
| write_endpoints |
| |
| # continuously update the mon endpoints if they fail over |
| watch_endpoints & |
| ||| |
| }, |
| }, |
| |
| cronjob: kube.CronJob(cluster.name("benji")) { |
| metadata+: cluster.metadata, |
| spec+: { # CronJob Spec |
| schedule: "42 0 * * *", # Daily at 42 minute past midnight. |
| jobTemplate+: { |
| spec+: { # Job Spec |
| selector:: null, |
| template+: { |
| spec+: { # PodSpec |
| serviceAccountName: cluster.benji.sa.metadata.name, |
| containers_: { |
| benji: kube.Container(cluster.name("benji")) { |
| image: "ghcr.io/elemental-lf/benji-k8s@sha256:8ab74b210bfdf6ebbf0017144b34fb840ad9ad225edf27dd3394fbd8332e649d", |
| volumeMounts_: { |
| extrabins: { mountPath: "/usr/local/extrabins" }, |
| monendpoints: { mountPath: "/etc/rook" }, |
| benjiconfig: { mountPath: "/etc/benji" }, |
| data: { mountPath: "/data" }, |
| }, |
| env_: { |
| ROOK_ADMIN_SECRET: { secretKeyRef: { name: "rook-ceph-mon", key: "admin-secret" }}, |
| }, |
| command: [ |
| "bash", "-c", ||| |
| bash /usr/local/extrabins/get-rook-creds.sh |
| benji-backup-pvc |
| benji-command enforce latest3,hours48,days7,months12 |
| benji-command cleanup |
| bash /usr/local/extrabins/metabackup.sh |
| ||| |
| ], |
| }, |
| }, |
| volumes_: { |
| data: kube.PersistentVolumeClaimVolume(cluster.benji.data), |
| benjiconfig: kube.SecretVolume(cluster.benji.config), |
| extrabins: kube.ConfigMapVolume(cluster.benji.extrabins), |
| monendpoints: { |
| configMap: { |
| name: "rook-ceph-mon-endpoints", |
| items: [ |
| { key: "data", path: "mon-endpoints" }, |
| ], |
| }, |
| }, |
| }, |
| }, |
| }, |
| }, |
| }, |
| }, |
| }, |
| }, |
| }, |
| |
| ReplicatedBlockPool(cluster, name):: { |
| local pool = self, |
| name:: name, |
| |
| spec:: error "spec must be specified", |
| |
| pool: kube._Object("ceph.rook.io/v1", "CephBlockPool", name) { |
| metadata+: cluster.metadata, |
| spec: pool.spec, |
| }, |
| |
| storageClass: kube.StorageClass(name) { |
| provisioner: "ceph.rook.io/block", |
| parameters: { |
| blockPool: pool.pool.metadata.name, |
| clusterNamespace: pool.pool.metadata.namespace, |
| fstype: "ext4", |
| }, |
| reclaimPolicy: "Retain", |
| }, |
| }, |
| |
| ECBlockPool(cluster, name):: { |
| local pool = self, |
| name:: name, |
| metadataReplicas:: 3, |
| |
| spec:: error "spec must be specified", |
| |
| pool: kube._Object("ceph.rook.io/v1", "CephBlockPool", name) { |
| metadata+: cluster.metadata, |
| spec: pool.spec, |
| }, |
| metapool: kube._Object("ceph.rook.io/v1", "CephBlockPool", name + "-metadata") { |
| metadata+: cluster.metadata, |
| spec: { |
| failureDomain: "host", |
| replicated: { |
| size: pool.metadataReplicas, |
| }, |
| }, |
| }, |
| |
| storageClass: kube.StorageClass(name) { |
| provisioner: "ceph.rook.io/block", |
| parameters: { |
| blockPool: pool.metapool.metadata.name, |
| dataBlockPool: pool.pool.metadata.name, |
| clusterNamespace: pool.pool.metadata.namespace, |
| fstype: "ext4", |
| }, |
| reclaimPolicy: "Retain", |
| }, |
| }, |
| |
| // This is a rook CephObjectRealm which corresponds to a radosgw realm. |
| // |
| // A realm is a 'world' of radosgw user-facing metadata, like credentials, |
| // buckets, and underlying structures like zones and zonegroups. A realm |
| // contains zonegroups and zones, but a single Ceph cluster can actually |
| // serve multiple realms, by running multiple radosgw instances. |
| S3ObjectRealm(cluster, name):: { |
| cluster:: cluster, |
| realm: kube._Object("ceph.rook.io/v1", "CephObjectRealm", name) { |
| metadata+: cluster.metadata, |
| }, |
| }, |
| |
| // This is a rook CephObjectZoneGroup which corresponds to a radosgw |
| // zonegroup. |
| // |
| // A zonegroup contains zones, and zones within a zonegroup will serve a |
| // concise view of objects in buckets, and will sync between eachother to |
| // eventually contain the same data. |
| // |
| // A single zonegroup within a realm must be a 'master' zonegroup, and will |
| // then hold and replicate the metadata of this realm. All realm operations |
| // via radosgw-admin must be performed within the master zonegroup. |
| S3ObjectZoneGroup(realm, name):: { |
| realm:: realm, |
| zonegroup: kube._Object("ceph.rook.io/v1", "CephObjectZoneGroup", name) { |
| metadata+: realm.cluster.metadata, |
| spec+: { |
| realm: realm.realm.metadata.name, |
| }, |
| }, |
| }, |
| |
| // This is a CephObjectZone but also a CephObjectStore. |
| // |
| // Rook attempts to hide away Ceph's radosgw multisite structures |
| // (realm/zonegroup/zone) by presenting a single CRD named |
| // 'CephObjectStore'. When such a resource is created, Rook will create a |
| // realm, zonegroup and zone under the hood, as a radosgw zone is required |
| // to serve data, and a radosgw zone cannot exist without a zonegroup, and |
| // a radosgw zonegroup cannot exist without a realm. |
| // |
| // However, rook also exposes the lower-level API by letting the user |
| // specify 'zone' in the ObjectStore's spec, which should point to a |
| // CephObjectZone. Then, an entirely different reconciliation codepath is |
| // taken and instead users are expected to manage |
| // CephObject{Realm,ZoneGroup,Zone} manually at Ceph's native abstraction |
| // level. |
| // |
| // CephObjectStore not only represents a Ceph zone (and possibly |
| // zonegroup/realm), but also pods and services that are required to servev |
| // radosgw data publicly. That's why S3ObjectStore takes parameters like |
| // 'public port' and 'instance number'. |
| // |
| // To add to the confusion, our S3ObjectStore wrapper also sprinkles in an |
| // Ingress with TLS to terminate the above service, and automatically |
| // creates a CephObjectZone. |
| // |
| // This whole jsonent abstraction basically forces users to manually create |
| // realms and zonegroups, but makes it very easy to do so. By forcing these |
| // to be explicitly created by rook objects, only the 'multi-site' |
| // reconciliation codepath is taken in rook, making things a bit more |
| // predictable. |
| S3ObjectStore(zonegroup, name):: { |
| local store = self, |
| spec:: { |
| dataPool: error "spec.dataPool must be specified", |
| metadataPool: error "spec.metadataPool must be specified", |
| }, |
| |
| cfg:: { |
| // We want to have each rgw run under a domain corresponding to the |
| // zone it's running in, but also to the zonegroup it's running in. |
| // This will allow us to DNS loadbalance a zonegroup to be backed |
| // by multiple zone ingresses. |
| domainParts: [ |
| zonegroup.zone.metadata.name, |
| zonegroup.zonegroup.metadata.name, |
| ], |
| domains: [ |
| "object.ceph-%s.hswaw.net" % [part] |
| for part in cfg.domainParts |
| ], |
| }, |
| local cfg = self.cfg, |
| |
| zone: kube._Object("ceph.rook.io/v1", "CephObjectZone", name) { |
| metadata+: zonegroup.realm.cluster.metadata, |
| spec: store.spec { |
| zoneGroup: zonegroup.zonegroup.metadata.name, |
| }, |
| }, |
| |
| objectStore: kube._Object("ceph.rook.io/v1", "CephObjectStore", name) { |
| metadata+: zonegroup.realm.cluster.metadata, |
| spec: { |
| gateway: { |
| port: 80, |
| instances: 1, |
| allNodes: false, |
| }, |
| zone: { |
| name: name, |
| }, |
| preservePoolsOnDelete: true, |
| }, |
| }, |
| |
| objectIngress: kube.Ingress(name) { |
| metadata+: zonegroup.realm.cluster.metadata { |
| annotations+: { |
| "kubernetes.io/tls-acme": "true", |
| "cert-manager.io/cluster-issuer": "letsencrypt-prod", |
| "nginx.ingress.kubernetes.io/proxy-body-size": "0", |
| }, |
| }, |
| |
| spec+: { |
| tls: [ |
| { |
| hosts: cfg.domains, |
| secretName: "%s-tls" % [name], |
| }, |
| ], |
| rules: [ |
| { |
| host: domain, |
| http: { |
| paths: [ |
| { |
| path: "/", |
| backend: { |
| serviceName: "rook-ceph-rgw-%s" % [name], |
| servicePort: 80, |
| }, |
| }, |
| ] |
| }, |
| } |
| for domain in cfg.domains |
| ], |
| }, |
| }, |
| }, |
| } |