cluster/kube/lib/rook.libsonnet

# Deploy Rook/Ceph Operator

local kube = import "../../../kube/kube.libsonnet";
local policies = import "../../../kube/policies.libsonnet";

local oa = kube.OpenAPI;

{
    Operator: {
        local env = self,
        local cfg = env.cfg,
        cfg:: {
            image: "rook/ceph:v1.1.9",
            namespace: "rook-ceph-system",
        },

        metadata:: {
            namespace: cfg.namespace,
            labels: {
                "operator": "rook",
                "storage-backend": "ceph",
            },
        },

        namespace: kube.Namespace(cfg.namespace),

        policyInsecure: policies.AllowNamespaceInsecure(cfg.namespace),

        crds: {
            # BUG: cannot control this because of:
            # ERROR Error updating customresourcedefinitions cephclusters.ceph.rook.io: expected kind, but got map
            # TODO(q3k): debug and fix kubecfg (it's _not_ just https://github.com/bitnami/kubecfg/issues/259 )
            cephclusters:: kube.CustomResourceDefinition("ceph.rook.io", "v1", "CephCluster") {
                spec+: {
                    additionalPrinterColumns: [
                        { name: "DataDirHostPath", type: "string", description: "Directory used on the K8s nodes", JSONPath: ".spec.dataDirHostPath" },
                        { name: "MonCount", type: "string", description: "Number of MONs", JSONPath: ".spec.mon.count" },
                        { name: "Age", type: "date", JSONPath: ".metadata.creationTimestamp" },
                        { name: "State", type: "string", description: "Current State", JSONPath: ".status.state" },
                        { name: "Health", type: "string", description: "Ceaph Health", JSONPath: ".status.ceph.health" },
                    ],
                    validation: oa.Validation(oa.Dict {
                        spec: oa.Dict {
                            annotations: oa.Any,
                            cephVersion: oa.Dict {
                                allowUnsupported: oa.Boolean,
                                image: oa.String,
                            },
                            dashboard: oa.Dict {
                                enabled: oa.Boolean,
                                urlPrefix: oa.String,
                                port: oa.Integer { minimum: 0, maximum: 65535 },
                                ssl: oa.Boolean,
                            },
                            dataDirHostPath: oa.String { pattern: "^/(\\S+)" },
                            skipUpgradeChecks: oa.Boolean,
                            mon: oa.Dict {
                                allowMultiplePerNode: oa.Boolean,
                                count: oa.Integer { minimum: 0, maximum: 9 },
                                preferredCount: oa.Integer { minimum: 0, maximum: 9 },
                            },
                            mgr: oa.Dict {
                                modules: oa.Array(oa.Dict {
                                    name: oa.String,
                                    enabled: oa.Boolean,
                                }),
                            },
                            network: oa.Dict {
                                hostNetwork: oa.Boolean,
                            },
                            storage: oa.Dict {
                                disruptionManagement: oa.Dict {
                                    managePodBudgets: oa.Boolean,
                                    osdMaintenanceTimeout: oa.Integer,
                                    manageMachineDisruptionBudgets: oa.Boolean,
                                },
                                useAllNodes: oa.Boolean,
                                nodes: oa.Array(oa.Dict {
                                    name: oa.String,
                                    config: oa.Dict {
                                        metadataDevice: oa.String,
                                        storeType: oa.String { pattern: "^(filestore|bluestore)$" },
                                        databaseSizeMB: oa.String,
                                        walSizeMB: oa.String,
                                        journalSizeMB: oa.String,
                                        osdsPerDevice: oa.String,
                                        encryptedDevice: oa.String { pattern: "^(true|false)$" },
                                    },
                                    useAllDevices: oa.Boolean,
                                    deviceFilter: oa.Any,
                                    directories: oa.Array(oa.Dict {
                                        path: oa.String,
                                    }),
                                    devices: oa.Array(oa.Dict {
                                        name: oa.String,
                                    }),
                                    location: oa.Any,
                                    resources: oa.Any,
                                }),
                                useAllDevices: oa.Boolean,
                                deviceFilter: oa.Any,
                                location: oa.Any,
                                directories: oa.Array(oa.Dict {
                                    path: oa.String,
                                }),
                                config: oa.Any,
                                topologyAware: oa.Boolean,
                            },
                            monitoring: oa.Dict {
                                enabled: oa.Boolean,
                                rulesNamespace: oa.String,
                            },
                            rbdMirroring: oa.Dict {
                                workers: oa.Integer,
                            },
                            placement: oa.Any,
                            resources: oa.Any,
                        },
                    }),
                },
            },
            cephfilesystems: kube.CustomResourceDefinition("ceph.rook.io", "v1", "CephFilesystem") {
                spec+: {
                    additionalPrinterColumns: [
                        { name: "ActiveMDS", type: "string", description: "Number of desired active MDS daemons", JSONPath: ".spec.metadataServer.activeCount" },
                        { name: "Age", type: "date", JSONPath: ".metadata.creationTimestamp" },
                    ],
                    validation: oa.Validation(oa.Dict {
                        spec: oa.Dict {
                            metadataServer: oa.Dict {
                                activeCount: oa.Integer,
                                activeStandby: oa.Boolean,
                                annotations: oa.Any,
                                placement: oa.Any,
                                resources: oa.Any,
                            },
                            metadataPool: oa.Dict {
                                failureDomain: oa.String,
                                replicated: oa.Dict {
                                    size: oa.Integer,
                                },
                                erasureCoded: oa.Dict {
                                    dataChunks: oa.Integer,
                                    codingChunks: oa.Integer,
                                },
                            },
                            dataPools: oa.Array(oa.Dict {
                                failureDomain: oa.String,
                                replicated: oa.Dict {
                                    site: oa.Integer,
                                    erasureCoded: oa.Dict {
                                        dataChunks: oa.Integer,
                                        codingChunks: oa.Integer,
                                    },
                                },
                            })
                        },
                    }),
                },
            },
            cephnfses: kube.CustomResourceDefinition("ceph.rook.io", "v1", "CephNFS") {
                spec+: {
                    names+: {
                        plural: "cephnfses",
                        shortNames: ["nfs"],
                    },
                    validation: oa.Validation(oa.Dict {
                        spec: oa.Dict {
                            rados: oa.Dict {
                                pool: oa.String,
                                namespace: oa.String,
                            },
                            server: oa.Dict {
                                active: oa.Integer,
                                annotations: oa.Any,
                                placement: oa.Any,
                                resources: oa.Any,
                            },
                        },
                    }),
                },
            },
            cephobjectstores: kube.CustomResourceDefinition("ceph.rook.io", "v1", "CephObjectStore") {
                spec+: {
                    validation: oa.Validation(oa.Dict {
                        spec: oa.Dict {
                            gateway: oa.Dict {
                                type: oa.String,
                                sslCertificateRef: oa.Any,
                                port: oa.Integer,
                                securePort: oa.Any,
                                instances: oa.Integer,
                                annotations: oa.Any,
                                placement: oa.Any,
                                resources: oa.Any,
                            },
                            local poolDef = oa.Dict {
                                failureDomain: oa.String,
                                replicated: oa.Dict {
                                    size: oa.Integer,
                                },
                                erasureCoded: oa.Dict {
                                    dataChunks: oa.Integer,
                                    codingChunks: oa.Integer,
                                },
                            },
                            metadataPool: poolDef,
                            dataPool: poolDef,
                        },
                    }),
                },
            },
            cephobjectstoreusers: kube.CustomResourceDefinition("ceph.rook.io", "v1", "CephObjectStoreUser"),
            cephblockpools: kube.CustomResourceDefinition("ceph.rook.io", "v1", "CephBlockPool"),
            volumes: kube.CustomResourceDefinition("rook.io", "v1alpha2", "Volume") {
                spec+: {
                    names+: {
                        shortNames: ["rv"],
                    },
                },
            },
            objectbuckets: kube.CustomResourceDefinition("objectbucket.io", "v1alpha1", "ObjectBucket") {
                spec+: {
                    names+: {
                        shortNames: ["ob", "obs"],
                    },
                    scope: "Cluster",
                    subresources: { status: {} },
                },
            },
            objectbucketclaims: kube.CustomResourceDefinition("objectbucket.io", "v1alpha1", "ObjectBucketClaim") {
                spec+: {
                    names+: {
                        shortNames: ["obc", "obcs"],
                    },
                    subresources: { status: {} },
                },
            },
        },

        sa: {
            system: kube.ServiceAccount("rook-ceph-system") {
                metadata+: env.metadata,
            },
            csiCephfsPlugin: kube.ServiceAccount("rook-csi-cephfs-plugin-sa") {
                metadata+: env.metadata,
            },
            csiCephfsProvisioner: kube.ServiceAccount("rook-csi-cephfs-provisioner-sa") {
                metadata+: env.metadata,
            },
            csiRbdPlugin: kube.ServiceAccount("rook-csi-rbd-plugin-sa") {
                metadata+: env.metadata,
            },
            csiRbdProvisioner: kube.ServiceAccount("rook-csi-rbd-provisioner-sa") {
                metadata+: env.metadata,
            },
        },

        crs: {
            clusterMgmt: kube.ClusterRole("rook-ceph-cluster-mgmt") {
                metadata+: env.metadata { namespace:: null },
                rules: [
                    {
                        apiGroups: [""],
                        resources: ["secrets", "pods", "pods/log", "services", "configmaps"],
                        verbs: ["get", "list", "watch", "patch", "create", "update", "delete"],
                    },
                    {
                        apiGroups: ["apps"],
                        resources: ["deployments", "daemonsets", "replicasets"],
                        verbs: ["get", "list", "watch", "create", "update", "delete"],
                    },
                ],
            },
            global: kube.ClusterRole("rook-ceph-global") {
                metadata+: env.metadata { namespace:: null },
                rules: [
                    {
                        apiGroups: [""],
                        resources: ["pods", "nodes", "nodes/proxy"],
                        verbs: ["get", "list", "watch"],
                    },
                    {
                        apiGroups: [""],
                        resources: ["events", "persistentvolumes", "persistentvolumeclaims", "endpoints"],
                        verbs: ["get", "list", "watch", "patch", "create", "update", "delete"],
                    },
                    {
                        apiGroups: ["storage.k8s.io"],
                        resources: ["storageclasses"],
                        verbs: ["get", "list", "watch"],
                    },
                    {
                        apiGroups: ["batch"],
                        resources: ["jobs"],
                        verbs: ["get", "list", "watch", "create", "update", "delete"],
                    },
                    {
                        apiGroups: ["ceph.rook.io"],
                        resources: ["*"],
                        verbs: ["*"],
                    },
                    {
                        apiGroups: ["rook.io"],
                        resources: ["*"],
                        verbs: ["*"],
                    },
                    {
                        apiGroups: ["policy", "apps"],
                        resources: ["poddisruptionbudgets", "deployments"],
                        verbs: ["*"],
                    },
                ],
            },

            // Upstream rook uses split ClusterRoles, with the 'main' role (eg rook-ceph-cluster-mgmt)
            // using aggregationRules to point to a '-rules' role (eg rook-ceph-cluster-mgmt-rules) which
            // contains the actual role rules. This was done to permit for a bettr upgrade experience on
            // systems that only allow for a recreation of a clusterroles (see https://github.com/rook/rook/issues/2634
            // for more background information).
            // We do not use this split because our update mechanism is not broken. However, it seems
            // that Rook started to use these split rules for other reasons, too. For instance, the
            // mgr-cluster role in upstream not only aggregates its equivalent -rules role, but also
            // the rook-ceph-object-bucket role. As such, we split mgr-cluster as they do in upstream.
            // In the future, we may split the rest of the roles in order to stay consisdent with upsteam.

            mgrCluster: kube.ClusterRole("rook-ceph-mgr-cluster") {
                metadata+: env.metadata { namespace:: null },
                aggregationRule: {
                    clusterRoleSelectors: [
                        { matchLabels: { "rbac.ceph.rook.io/aggregate-to-rook-ceph-mgr-cluster": "true" }},
                    ],
                },
            },
            mgrClusterRules: kube.ClusterRole("rook-ceph-mgr-cluster-rules") {
                metadata+: env.metadata {
                    namespace:: null,
                    labels+: {
                        "rbac.ceph.rook.io/aggregate-to-rook-ceph-mgr-cluster": "true",
                    },
                },
                rules: [
                    {
                        apiGroups: [""],
                        resources: ["configmaps", "nodes", "nodes/proxy"],
                        verbs: ["get", "list", "watch"],
                    },
                    {
                        apiGroups: [""],
                        resources: ["events"],
                        verbs: ["create", "patch", "list", "get", "watch"],
                    },
                ]
            },
            objectBucket: kube.ClusterRole("rook-ceph-object-bucket") {
                metadata+: env.metadata {
                    namespace:: null,
                    labels+: {
                        "rbac.ceph.rook.io/aggregate-to-rook-ceph-mgr-cluster": "true",
                    },
                },
                rules: [
                    {
                        apiGroups: [""],
                        resources: ["secrets", "configmaps"],
                        verbs: ["*"],
                    },
                    {
                        apiGroups: ["storage.k8s.io"],
                        resources: ["storageclasses"],
                        verbs: ["get", "list", "watch"],
                    },
                    {
                        apiGroups: ["objectbucket.io"],
                        resources: ["*"],
                        verbs: ["*"],
                    },
                ],
            },

            cephfsCSINodeplugin: kube.ClusterRole("cephfs-csi-nodeplugin") {
                metadata+: env.metadata { namespace:: null },
                rules: [
                    {
                        apiGroups: [""],
                        resources: ["nodes"],
                        verbs: ["get", "list", "update"],
                    },
                    {
                        apiGroups: [""],
                        resources: ["namespaces"],
                        verbs: ["get", "list"],
                    },
                    {
                        apiGroups: [""],
                        resources: ["persistentvolumes"],
                        verbs: ["get", "list", "watch", "update"],
                    },
                    {
                        apiGroups: ["storage.k8s.io"],
                        resources: ["volumeattachments"],
                        verbs: ["get", "list", "watch", "update"],
                    },
                    {
                        apiGroups: [""],
                        resources: ["configmaps"],
                        verbs: ["get", "list"],
                    },
                ],
            },

            cephfsExternalProvisionerRunner: kube.ClusterRole("cephfs-external-provisioner-runner") {
                metadata+: env.metadata { namespace:: null },
                rules: [
                    {
                        apiGroups: [""],
                        resources: ["secrets"],
                        verbs: ["get", "list"],
                    },
                    {
                        apiGroups: [""],
                        resources: ["persistentvolumes"],
                        verbs: ["get", "list", "watch", "create", "update", "delete"],
                    },
                    {
                        apiGroups: [""],
                        resources: ["persistentvolumeclaims"],
                        verbs: ["get", "list", "watch", "update"],
                    },
                    {
                        apiGroups: ["storage.k8s.io"],
                        resources: ["storageclasses"],
                        verbs: ["get", "list", "watch"],
                    },
                    {
                        apiGroups: [""],
                        resources: ["events"],
                        verbs: ["list", "watch", "create", "update", "patch"],
                    },
                    {
                        apiGroups: ["storage.k8s.io"],
                        resources: ["volumeattachments"],
                        verbs: ["get", "list", "watch", "update"],
                    },
                    {
                        apiGroups: [""],
                        resources: ["nodes"],
                        verbs: ["get", "list", "watch"],
                    },
                ],
            },

            rbdCSINodeplugin: kube.ClusterRole("rbd-csi-nodeplugin") {
                metadata+: env.metadata { namespace:: null },
                rules: [
                    {
                        apiGroups: [""],
                        resources: ["secrets"],
                        verbs: ["get", "list"],
                    },
                    {
                        apiGroups: [""],
                        resources: ["nodes"],
                        verbs: ["get", "list", "update"],
                    },
                    {
                        apiGroups: [""],
                        resources: ["namespaces"],
                        verbs: ["get", "list"],
                    },
                    {
                        apiGroups: [""],
                        resources: ["persistentvolumes"],
                        verbs: ["get", "list", "watch", "update"],
                    },
                    {
                        apiGroups: ["storage.k8s.io"],
                        resources: ["volumeattachments"],
                        verbs: ["get", "list", "watch", "update"],
                    },
                    {
                        apiGroups: [""],
                        resources: ["configmaps"],
                        verbs: ["get", "list"],
                    },
                ],
            },

            rbdExternalProvisionerRunner: kube.ClusterRole("rbd-external-provisioner-runner") {
                metadata+: env.metadata { namespace:: null },
                rules: [
                    {
                        apiGroups: [""],
                        resources: ["secrets"],
                        verbs: ["get", "list"],
                    },
                    {
                        apiGroups: [""],
                        resources: ["persistentvolumes"],
                        verbs: ["get", "list", "watch", "create", "update", "delete"],
                    },
                    {
                        apiGroups: [""],
                        resources: ["persistentvolumeclaims"],
                        verbs: ["get", "list", "watch", "update"],
                    },
                    {
                        apiGroups: ["storage.k8s.io"],
                        resources: ["volumeattachments"],
                        verbs: ["get", "list", "watch", "update"],
                    },
                    {
                        apiGroups: [""],
                        resources: ["nodes"],
                        verbs: ["get", "list", "watch"],
                    },
                    {
                        apiGroups: ["storage.k8s.io"],
                        resources: ["storageclasses"],
                        verbs: ["get", "list", "watch"]
                    },
                    {
                        apiGroups: [""],
                        resources: ["events"],
                        verbs: ["list", "watch", "create", "update", "patch"],
                    },
                    {
                        apiGroups: ["snapshot.storage.k8s.io"],
                        resources: ["volumesnapshotcontents"],
                        verbs: ["create", "get", "list", "watch", "update", "delete"],
                    },
                    {
                        apiGroups: ["snapshot.storage.k8s.io"],
                        resources: ["volumesnapshotclasses"],
                        verbs: ["get", "list", "watch"],
                    },
                    {
                        apiGroups: ["apiextensions.k8s.io"],
                        resources: ["customresourcedefinitions"],
                        verbs: ["create", "list", "watch", "delete", "get", "update"],
                    },
                    {
                        apiGroups: ["snapshot.storage.k8s.io"],
                        resources: ["volumesnapshots/status"],
                        verbs: ["update"],
                    },
                ],
            },
        },

        crbs: {
            global: kube.ClusterRoleBinding("ceph-rook-global") {
                metadata+: env.metadata { namespace:: null },
                roleRef_: env.crs.global,
                subjects_: [env.sa.system],
            },
            objectBucket: kube.ClusterRoleBinding("rook-ceph-object-bucket") {
                metadata+: env.metadata { namespace:: null },
                roleRef_: env.crs.objectBucket,
                subjects_: [env.sa.system],
            },
            cephfsCSINodeplugin: kube.ClusterRoleBinding("cepfs-csi-nodeplugin") {
                metadata+: env.metadata { namespace:: null },
                roleRef_: env.crs.cephfsCSINodeplugin,
                subjects_: [env.sa.csiCephfsPlugin],
            },
            cephfsCSIProvisioner: kube.ClusterRoleBinding("cephfs-csi-provisioner") {
                metadata+: env.metadata { namespace:: null },
                roleRef_: env.crs.cephfsExternalProvisionerRunner,
                subjects_: [env.sa.csiCephfsProvisioner],
            },
            rbdCSINodeplugin: kube.ClusterRoleBinding("rbd-csi-nodeplugin") {
                metadata+: env.metadata { namespace:: null },
                roleRef_: env.crs.rbdCSINodeplugin,
                subjects_: [env.sa.csiRbdPlugin],
            },
            rbdCSIProvisioner: kube.ClusterRoleBinding("rbd-csi-provisioner") {
                metadata+: env.metadata { namespace:: null },
                roleRef_: env.crs.rbdExternalProvisionerRunner,
                subjects_: [env.sa.csiRbdProvisioner],
            },
        },

        roles: {
            system: kube.Role("ceph-rook-system") {
                metadata+: env.metadata,
                rules: [
                    {
                        apiGroups: [""],
                        resources: ["pods", "configmaps", "services"],
                        verbs: ["get", "list", "watch", "patch", "create", "update", "delete"],
                    },
                    {
                        apiGroups: ["apps"],
                        resources: ["deployments", "statefulsets", "daemonsets"],
                        verbs: ["get", "list", "watch", "create", "update", "delete"],
                    },
                ],
            },
            cephfsExternalProvisioner: kube.Role("cephfs-external-provisioner-cfg") {
                metadata+: env.metadata,
                rules: [
                    {
                        apiGroups: [""],
                        resources: ["endpoints"],
                        verbs: ["get", "watch", "list", "delete", "update", "create"],
                    },
                    {
                        apiGroups: [""],
                        resources: ["configmaps"],
                        verbs: ["get", "list", "create", "delete"],
                    },
                    {
                        apiGroups: ["coordination.k8s.io"],
                        resources: ["leases"],
                        verbs: ["get" ,"watch", "list", "delete", "update", "create"],
                    },
                ],
            },
            rbdExternalProvisioner: kube.Role("rbd-external-provisioner-cfg") {
                metadata+: env.metadata,
                rules: [
                    {
                        apiGroups: [""],
                        resources: ["endpoints"],
                        verbs: ["get", "watch", "list", "delete", "update", "create"],
                    },
                    {
                        apiGroups: [""],
                        resources: ["configmaps"],
                        verbs: ["get", "list", "watch", "create", "delete"],
                    },
                    {
                        apiGroups: ["coordination.k8s.io"],
                        resources: ["leases"],
                        verbs: ["get" ,"watch", "list", "delete", "update", "create"],
                    },
                ],
            },
        },

        rbs: {
            system: kube.RoleBinding("ceph-rook-system") {
                metadata+: env.metadata,
                roleRef_: env.roles.system,
                subjects_: [env.sa.system],
            },
            cephfsCSIProvisioner: kube.RoleBinding("cephfs-csi-provisioner-role-cfg") {
                metadata+: env.metadata,
                roleRef_: env.roles.cephfsExternalProvisioner,
                subjects_: [env.sa.csiCephfsProvisioner],
            },
            rbdCSIProvisioner: kube.RoleBinding("rbd-csi-provisioner-role-cfg") {
                metadata+: env.metadata,
                roleRef_: env.roles.rbdExternalProvisioner,
                subjects_: [env.sa.csiRbdProvisioner],
            },
        },

        operator: kube.Deployment("rook-ceph-operator") {
            metadata+: env.metadata,
            spec+: {
                template+: {
                    spec+: {
                        serviceAccountName: env.sa.system.metadata.name,
                        containers_: {
                            operator: kube.Container("rook-ceph-operator") {
                                image: cfg.image,
                                args: ["ceph", "operator"],
                                volumeMounts_: {
                                    "rook-config": { mountPath: "/var/lib/rook" },
                                    "default-config-dir": { mountPath: "/etc/ceph" },
                                },
                                env_: {
                                    LIB_MODULES_DIR_PATH: "/run/current-system/kernel-modules/lib/modules/",
                                    ROOK_ALLOW_MULTIPLE_FILESYSTEMS: "false",
                                    ROOK_LOG_LEVEL: "INFO",
                                    ROOK_MON_HEALTHCHECK_INTERVAL: "45s",
                                    ROOK_MON_OUT_TIMEOUT: "600s",
                                    ROOK_DISCOVER_DEVICES_INTERVAL: "60m",
                                    ROOK_HOSTPATH_REQUIRES_PRIVILEGED: "false",
                                    ROOK_ENABLE_SELINUX_RELABELING: "true",
                                    ROOK_ENABLE_FSGROUP: "true",
                                    NODE_NAME: kube.FieldRef("spec.nodeName"),
                                    POD_NAME: kube.FieldRef("metadata.name"),
                                    POD_NAMESPACE: kube.FieldRef("metadata.namespace"),
                                    ROOK_CSI_KUBELET_DIR_PATH: "/var/lib/kubernetes"
                                },
                            },
                        },
                        volumes_: {
                            "rook-config": { emptyDir: {} },
                            "default-config-dir": { emptyDir: {} },
                        },
                    },
                },
            },
        },
    },

    // Create a new Ceph cluster in a new namespace.
    Cluster(operator, name):: {
        local cluster = self,
        spec:: error "please define cluster spec",


        metadata:: {
            namespace: name,
        },

        name(suffix):: cluster.metadata.namespace + "-" + suffix,

        namespace: kube.Namespace(cluster.metadata.namespace),

        sa: {
            // service accounts need to be hardcoded, see operator source.
            osd: kube.ServiceAccount("rook-ceph-osd") {
                metadata+: cluster.metadata,
            },
            mgr: kube.ServiceAccount("rook-ceph-mgr") {
                metadata+: cluster.metadata,
            },
            cmdReporter: kube.ServiceAccount("rook-ceph-cmd-reporter") {
                metadata+: cluster.metadata,
            },
        },

        roles: {
            osd: kube.Role(cluster.name("osd")) {
                metadata+: cluster.metadata,
                rules: [
                    {
                        apiGroups: [""],
                        resources: ["configmaps"],
                        verbs: ["get", "list", "watch", "create", "update", "delete"],
                    }
                ],
            },
            mgr: kube.Role(cluster.name("mgr")) {
                metadata+: cluster.metadata,
                rules: [
                    {
                        apiGroups: [""],
                        resources: ["pods", "services"],
                        verbs: ["get", "list", "watch"],
                    },
                    {
                        apiGroups: ["batch"],
                        resources: ["jobs"],
                        verbs: ["get", "list", "watch", "create", "update", "delete"],
                    },
                    {
                        apiGroups: ["ceph.rook.io"],
                        resources: ["*"],
                        verbs: ["*"],
                    },
                ],
            },
            cmdReporter: kube.Role(cluster.name("cmd-reporter")) {
                metadata+: cluster.metadata,
                rules: [
                    {
                        apiGroups: [""],
                        resources: ["pods", "configmaps"],
                        verbs: ["get", "list", "watch", "create", "update", "delete"],
                    },
                ],
            },
            mgrSystem: kube.ClusterRole(cluster.name("mgr-system")) {
                metadata+: cluster.metadata { namespace:: null },
                rules: [
                    {
                        apiGroups: [""],
                        resources: ["configmaps"],
                        verbs: ["get", "list", "watch"],
                    }
                ],
            },
        },

        rbs: [
            kube.RoleBinding(cluster.name(el.name)) {
                metadata+: cluster.metadata,
                roleRef_: el.role,
                subjects_: [el.sa],
            },
            for el in [
                // Allow Operator SA to perform Cluster Mgmt in this namespace.
                { name: "cluster-mgmt", role: operator.crs.clusterMgmt, sa: operator.sa.system },
                { name: "osd", role: cluster.roles.osd, sa: cluster.sa.osd },
                { name: "mgr", role: cluster.roles.mgr, sa: cluster.sa.mgr },
                { name: "cmd-reporter", role: cluster.roles.cmdReporter, sa: cluster.sa.cmdReporter },
                { name: "mgr-cluster", role: operator.crs.mgrCluster, sa: cluster.sa.mgr },
            ]
        ],

        mgrSystemRB: kube.RoleBinding(cluster.name("mgr-system")) {
            metadata+: {
                namespace: operator.cfg.namespace,
            },
            roleRef_: cluster.roles.mgrSystem,
            subjects_: [cluster.sa.mgr],
        },

        cluster: kube._Object("ceph.rook.io/v1", "CephCluster", name) {
            metadata+: cluster.metadata,
            spec: {
                cephVersion: {
                    # https://github.com/rook/rook/issues/2945#issuecomment-483964014
                    #image: "ceph/ceph:v13.2.5-20190319",
                    image: "ceph/ceph:v14.2.2-20190830",
                    allowUnsupported: true,
                },
                dataDirHostPath: if name == "ceph-waw2" then "/var/lib/rook" else "/var/lib/rook-%s" % [name],
                dashboard: {
                    ssl: false,
                    enabled: true,
                    port: 8080,
                },
            } + cluster.spec,
        },

        dashboardService: kube.Service(cluster.name("dashboard")) {
            metadata+: cluster.metadata,
            spec: {
                ports: [
                    { name: "dashboard", port: 80, targetPort: 8080, protocol: "TCP" },
                ],
                selector: {
                    app: "rook-ceph-mgr",
                    rook_cluster: name,
                },
                type: "ClusterIP",
            },
        },

        dashboardIngress: kube.Ingress(cluster.name("dashboard")) {
            metadata+: cluster.metadata {
                annotations+: {
                    "kubernetes.io/tls-acme": "true",
                    "certmanager.k8s.io/cluster-issuer": "letsencrypt-prod",
                },
            },
            spec+: {
                tls: [
                    {
                        hosts: ["%s.hswaw.net" % name],
                        secretName: cluster.name("dashboard"),
                    },
                ],
                rules: [
                    {
                        host: "%s.hswaw.net" % name,
                        http: {
                            paths: [
                                { path: "/", backend: cluster.dashboardService.name_port },
                            ]
                        },
                    }
                ],
            },
        },

        # Benji is a backup tool, external to rook, that we use for backing up
        # RBDs.
        benji: {
            sa: kube.ServiceAccount(cluster.name("benji")) {
                metadata+: cluster.metadata,
            },

            cr: kube.ClusterRole(cluster.name("benji")) {
                rules: [
                    {
                        apiGroups: [""],
                        resources: [
                            "persistentvolumes",
                            "persistentvolumeclaims"
                        ],
                        verbs: ["list", "get"],
                    },
                    {
                        apiGroups: [""],
                        resources: [
                            "events",
                        ],
                        verbs: ["create", "update"],
                    },
                ],
            },

            crb: kube.ClusterRoleBinding(cluster.name("benji")) {
                roleRef_: cluster.benji.cr,
                subjects_: [cluster.benji.sa],
            },

            config: kube.Secret(cluster.name("benji-config")) {
                metadata+: cluster.metadata,
                data_: {
                    "benji.yaml": std.manifestJson({
                        configurationVersion: '1',
                        databaseEngine: 'sqlite:////data/benji.sqlite',
                        defaultStorage: 'wasabi',
                        storages: [
                            {
                                name: "wasabi",
                                storageId: 1,
                                module: "s3",
                                configuration: cluster.spec.benji.s3Configuration {
                                    activeTransforms: ["encrypt"],
                                },
                            },
                        ],
                        transforms: [
                            {
                                name: "encrypt",
                                module: "aes_256_gcm",
                                configuration: {
                                    # not secret.
                                    kdfSalt: "T2huZzZpcGhhaWM3QWVwaDhybzRhaDNhbzFpc2VpOWFobDNSZWVQaGVvTWV1bmVaYWVsNHRoYWg5QWVENHNoYWg0ZGFoN3Rlb3NvcHVuZzNpZXZpMm9vTG9vbmc1YWlmb0RlZXAwYmFobDlab294b2hjaG9odjRhbzFsYWkwYWk=",
                                    kdfIterations: 2137,
                                    password: cluster.spec.benji.encryptionPassword,
                                },
                            },
                        ],
                        ios: [
                            { name: pool, module: "rbd" }
                            for pool in cluster.spec.benji.pools
                        ],
                    }),
                },
            },

            # Yes, Benji keeps data (backup metadata) on the ceph cluster that
            # it backs up. However:
            #  - we add a command to benji-k8s to also copy over the sqlite
            #    database over to s3
            #  - benji can, in a pinch, restore without a database if a version
            #    is known: https://benji-backup.me/restore.html#restoring-without-a-database
            data: kube.PersistentVolumeClaim(cluster.name("benji-data")) {
                metadata+: cluster.metadata,
                spec+: {
                    storageClassName: cluster.spec.benji.metadataStorageClass,
                    accessModes: [ "ReadWriteOnce" ],
                    resources: {
                        requests: {
                            storage: "1Gi",
                        },
                    },
                },
            },

            # Extra scripts.
            extrabins: kube.ConfigMap(cluster.name("benji-extrabins")) {
                metadata+: cluster.metadata,
                data: {
                    "metabackup.sh" : |||
                        # Make backups of sqlite3 metadata used by Benji.
                        # The backups live in the same bucket as backups, and the metabackups
                        # are named `metabackup-0..10`, where 0 is the newest backup. Any time
                        # this script is called, backups get shifted one way to the left (9 to 10,
                        # 8 to 9, etc). This ensures we have at least 10 backup replicas.

                        set -e

                        which s3cmd || pip install --upgrade s3cmd

                        AWS_ACCESS_KEY_ID=$(jq -r .storages[0].configuration.awsAccessKeyId < /etc/benji/benji.yaml)
                        AWS_SECRET_ACCESS_KEY=$(jq -r .storages[0].configuration.awsSecretAccessKey < /etc/benji/benji.yaml)
                        BUCKET=$(jq -r .storages[0].configuration.bucketName < /etc/benji/benji.yaml)

                        s3() {
                            s3cmd --host=s3.wasabisys.com \
                                "--host-bucket=%(bucket)s.s3.wasabisys.com" \
                                --region=eu-central-1 \
                                --access_key=$AWS_ACCESS_KEY_ID \
                                --secret_key=$AWS_SECRET_ACCESS_KEY \
                                "$@"
                        }

                        # Copy over old backups, if they exist.
                        for i in `seq 9 -1 0`; do
                            from="s3://$BUCKET/metabackup-$i.sqlite3"
                            to="s3://$BUCKET/metabackup-$((i+1)).sqlite3"

                            if [[ $(s3 ls $from | wc -l) -eq 0 ]]; then
                                echo "$from does not exist, skipping shift."
                                continue
                            fi
                            echo "Moving $from to $to..."
                            s3 mv $from $to
                        done

                        # Make new metabackup.
                        s3 put /data/benji.sqlite s3://$BUCKET/metabackup-0.sqlite3

                    |||,
                    "get-rook-creds.sh": |||
                        # Based on the Rook Toolbox /usr/local/bin/toolbox.sh script.
                        # Copyright 2016 The Rook Authors. All rights reserved.

                        CEPH_CONFIG="/etc/ceph/ceph.conf"
                        MON_CONFIG="/etc/rook/mon-endpoints"
                        KEYRING_FILE="/etc/ceph/keyring"

                        # create a ceph config file in its default location so ceph/rados tools can be used
                        # without specifying any arguments
                        write_endpoints() {
                            endpoints=$(cat ${MON_CONFIG})

                            # filter out the mon names
                            mon_endpoints=$(echo ${endpoints} | sed 's/[a-z]\+=//g')

                            # filter out the legacy mon names
                            mon_endpoints=$(echo ${mon_endpoints} | sed 's/rook-ceph-mon[0-9]\+=//g')

                            DATE=$(date)
                            echo "$DATE writing mon endpoints to ${CEPH_CONFIG}: ${endpoints}"
                            cat <<EOF > ${CEPH_CONFIG}
                        [global]
                        mon_host = ${mon_endpoints}

                        [client.admin]
                        keyring = ${KEYRING_FILE}
                        EOF
                        }

                        # watch the endpoints config file and update if the mon endpoints ever change
                        watch_endpoints() {
                            # get the timestamp for the target of the soft link
                            real_path=$(realpath ${MON_CONFIG})
                            initial_time=$(stat -c %Z ${real_path})
                            while true; do
                               real_path=$(realpath ${MON_CONFIG})
                               latest_time=$(stat -c %Z ${real_path})

                               if [[ "${latest_time}" != "${initial_time}" ]]; then
                                 write_endpoints
                                 initial_time=${latest_time}
                               fi
                               sleep 10
                            done
                        }

                        # create the keyring file
                        cat <<EOF > ${KEYRING_FILE}
                        [client.admin]
                        key = ${ROOK_ADMIN_SECRET}
                        EOF

                        # write the initial config file
                        write_endpoints

                        # continuously update the mon endpoints if they fail over
                        watch_endpoints &
                    |||
                },
            },

            cronjob: kube.CronJob(cluster.name("benji")) {
                metadata+: cluster.metadata,
                spec+: { # CronJob Spec
                    schedule: "42 0 * * *", # Daily at 42 minute past midnight.
                    jobTemplate+: {
                        spec+: { # Job Spec
                            selector:: null,
                            template+: {
                                spec+: { # PodSpec
                                    serviceAccountName: cluster.benji.sa.metadata.name,
                                    containers_: {
                                        benji: kube.Container(cluster.name("benji")) {
                                            # TODO(q3k): switch back to upstream after pull/65 goes in.
                                            # Currently this is being built from github.com/q3k/benji.
                                            # https://github.com/elemental-lf/benji/pull/65
                                            image: "registry.k0.hswaw.net/q3k/benji-k8s:20191221-2336",
                                            volumeMounts_: {
                                                extrabins: { mountPath: "/usr/local/extrabins" },
                                                monendpoints: { mountPath: "/etc/rook" },
                                                benjiconfig: { mountPath: "/etc/benji" },
                                                data: { mountPath: "/data" },
                                            },
                                            env_: {
                                                ROOK_ADMIN_SECRET: { secretKeyRef: { name: "rook-ceph-mon", key: "admin-secret" }},
                                            },
                                            command: [
                                                "bash", "-c", |||
                                                    bash /usr/local/extrabins/get-rook-creds.sh
                                                    benji-backup-pvc %s
                                                    benji-command enforce latest3,hours48,days7,months12
                                                    benji-command cleanup
                                                    bash /usr/local/extrabins/metabackup.sh
                                                ||| % [std.join(" ", ["--pool-filter=%s" % [p] for p in cluster.spec.benji.pools])],
                                            ],
                                        },
                                    },
                                    volumes_: {
                                        data: kube.PersistentVolumeClaimVolume(cluster.benji.data),
                                        benjiconfig: kube.SecretVolume(cluster.benji.config),
                                        extrabins: kube.ConfigMapVolume(cluster.benji.extrabins),
                                        monendpoints: {
                                            configMap: {
                                                name: "rook-ceph-mon-endpoints",
                                                items: [
                                                    { key: "data", path: "mon-endpoints" },
                                                ],
                                            },
                                        },
                                    },
                                },
                            },
                        },
                    },
                },
            },
        },
    },

    ReplicatedBlockPool(cluster, name):: {
        local pool = self,
        name:: name,

        spec:: error "spec must be specified",

        pool: kube._Object("ceph.rook.io/v1", "CephBlockPool", name) {
            metadata+: cluster.metadata,
            spec: pool.spec,
        },

        storageClass: kube.StorageClass(name) {
            provisioner: "ceph.rook.io/block",
            parameters: {
                blockPool: pool.pool.metadata.name,
                clusterNamespace: pool.pool.metadata.namespace,
                fstype: "ext4",
            },
            reclaimPolicy: "Retain",
        },
    },

    ECBlockPool(cluster, name):: {
        local pool = self,
        name:: name,
        metadataReplicas:: 3,

        spec:: error "spec must be specified",

        pool: kube._Object("ceph.rook.io/v1", "CephBlockPool", name) {
            metadata+: cluster.metadata,
            spec: pool.spec,
        },
        metapool: kube._Object("ceph.rook.io/v1", "CephBlockPool", name + "-metadata") {
            metadata+: cluster.metadata,
            spec: {
                failureDomain: "host",
                replicated: {
                    size: pool.metadataReplicas,
                },
            },
        },

        storageClass: kube.StorageClass(name) {
            provisioner: "ceph.rook.io/block",
            parameters: {
                blockPool: pool.metapool.metadata.name,
                dataBlockPool: pool.pool.metadata.name,
                clusterNamespace: pool.pool.metadata.namespace,
                fstype: "ext4",
            },
            reclaimPolicy: "Retain",
        },
    },

    S3ObjectStore(cluster, name):: {
        local store = self,
        spec:: error "spec must be specified",
        objectStore: kube._Object("ceph.rook.io/v1", "CephObjectStore", name) {
            metadata+: cluster.metadata,
            spec: store.spec {
                gateway: {
                    type: "s3",
                    port: 80,
                    instances: 1,
                    allNodes: false,
                },
            },
        },

        objectIngress: kube.Ingress(name) {
            metadata+: cluster.metadata {
                annotations+: {
                    "kubernetes.io/tls-acme": "true",
                    "certmanager.k8s.io/cluster-issuer": "letsencrypt-prod",
                    "nginx.ingress.kubernetes.io/proxy-body-size": "0",
                },
            },
            spec+: {
                tls: [
                    {
                        hosts: ["object.%s.hswaw.net" % [cluster.metadata.namespace]],
                        secretName: "%s-tls" % [name],
                    },
                ],
                rules: [
                    {
                        host: "object.%s.hswaw.net" % [cluster.metadata.namespace],
                        http: {
                            paths: [
                                {
                                    path: "/",
                                    backend: {
                                        serviceName: "rook-ceph-rgw-%s" % [name],
                                        servicePort: 80,
                                    },
                                },
                            ]
                        },
                    }
                ],
            },
        },
    },
}