cluster/kube/lib/rook.libsonnet

# Deploy Rook/Ceph Operator

local kube = import "../../../kube/kube.libsonnet";

{
    Environment: {
        local env = self,
        local cfg = env.cfg,
        cfg:: {
            image: "rook/ceph:master",
            namespace: "rook-ceph-system",
        },

        metadata:: {
            namespace: cfg.namespace,
            labels: {
                "operator": "rook",
                "storage-backend": "ceph",
            },
        },

        namespace: kube.Namespace(cfg.namespace),

        crds: {
            cephclusters: kube.CustomResourceDefinition("ceph.rook.io", "v1", "CephCluster") {
                spec+: {
                    additionalPrinterColumns: [
                        { name: "DataDirHostPath", type: "string", description: "Directory used on the K8s nodes", JSONPath: ".spec.dataDirHostPath" },
                        { name: "MonCount", type: "string", description: "Number of MONs", JSONPath: ".spec.mon.count" },
                        { name: "Age", type: "date", JSONPath: ".metadata.creationTimestamp" },
                        { name: "State", type: "string", description: "Current State", JSONPath: ".status.state" },
                    ],
                    validation: {
                        # Converted from official operator YAML
                        "openAPIV3Schema": {
                            "properties": {
                                "spec": {
                                    "properties": {
                                        "cephVersion": {
                                            "properties": {
                                                "allowUnsupported": {
                                                    "type": "boolean"
                                                },
                                                "image": {
                                                    "type": "string"
                                                },
                                                "name": {
                                                    "pattern": "^(luminous|mimic|nautilus)$",
                                                    "type": "string"
                                                }
                                            }
                                        },
                                        "dashboard": {
                                            "properties": {
                                                "enabled": {
                                                    "type": "boolean"
                                                },
                                                "urlPrefix": {
                                                    "type": "string"
                                                },
                                                "port": {
                                                    "type": "integer"
                                                }
                                            }
                                        },
                                        "dataDirHostPath": {
                                            "pattern": "^/(\\S+)",
                                            "type": "string"
                                        },
                                        "mon": {
                                            "properties": {
                                                "allowMultiplePerNode": {
                                                    "type": "boolean"
                                                },
                                                "count": {
                                                    "maximum": 9,
                                                    "minimum": 1,
                                                    "type": "integer"
                                                },
                                                "preferredCount": {
                                                    "maximum": 9,
                                                    "minimum": 0,
                                                    "type": "integer"
                                                }
                                            },
                                            "required": [
                                                "count"
                                            ]
                                        },
                                        "network": {
                                            "properties": {
                                                "hostNetwork": {
                                                    "type": "boolean"
                                                }
                                            }
                                        },
                                        "storage": {
                                            "properties": {
                                                "nodes": {
                                                    "items": {},
                                                    "type": "array"
                                                },
                                                "useAllDevices": {},
                                                "useAllNodes": {
                                                    "type": "boolean"
                                                }
                                            }
                                        }
                                    },
                                    "required": [
                                        "mon"
                                    ]
                                }
                            }
                        }
                    }
                },
            },
            cephfilesystems: kube.CustomResourceDefinition("ceph.rook.io", "v1", "CephFilesystem") {
                spec+: {
                    additionalPrinterColumns: [
                        { name: "MdsCount", type: "string", description: "Number of MDs", JSONPath: ".spec.metadataServer.activeCount" },
                        { name: "Age", type: "date", JSONPath: ".metadata.creationTimestamp" },
                    ],
                },
            },
            cephnfses: kube.CustomResourceDefinition("ceph.rook.io", "v1", "CephNFS") {
                spec+: {
                    names+: {
                        shortNames: ["nfs"],
                    },
                },
            },
            cephobjectstores: kube.CustomResourceDefinition("ceph.rook.io", "v1", "CephObjectStore"),
            cephobjectstoreusers: kube.CustomResourceDefinition("ceph.rook.io", "v1", "CephObjectStoreUser"),
            cephblockpools: kube.CustomResourceDefinition("ceph.rook.io", "v1", "CephBlockPool"),
            volumes: kube.CustomResourceDefinition("rook.io", "v1alpha2", "Volume") {
                spec+: {
                    names+: {
                        shortNames: ["rv"],
                    },
                },
            },
        },

        sa: kube.ServiceAccount("rook-ceph-system") {
            metadata+: env.metadata,
        },

        crs: {
            clusterMgmt: kube.ClusterRole("rook-ceph-cluster-mgmt") {
                metadata+: env.metadata {
                    namespace:: null,
                },
                rules: [
                    {
                        apiGroups: [""],
                        resources: ["secrets", "pods", "pods/log", "services", "configmaps"],
                        verbs: ["get", "list", "watch", "patch", "create", "update", "delete"],
                    },
                    {
                        apiGroups: ["apps"],
                        resources: ["deployments", "daemonsets", "replicasets"],
                        verbs: ["get", "list", "watch", "create", "update", "delete"],
                    },
                ],
            },
            global: kube.ClusterRole("rook-ceph-global") {
                metadata+: env.metadata {
                    namespace:: null,
                },
                rules: [
                    {
                        apiGroups: [""],
                        resources: ["pods", "nodes", "nodes/proxy"],
                        verbs: ["get", "list", "watch"],
                    },
                    {
                        apiGroups: [""],
                        resources: ["events", "persistentvolumes", "persistentvolumeclaims", "endpoints"],
                        verbs: ["get", "list", "watch", "patch", "create", "update", "delete"],
                    },
                    {
                        apiGroups: ["storage.k8s.io"],
                        resources: ["storageclasses"],
                        verbs: ["get", "list", "watch", "create", "update", "delete"],
                    },
                    {
                        apiGroups: ["ceph.rook.io"],
                        resources: ["*"],
                        verbs: ["*"],
                    },
                    {
                        apiGroups: ["rook.io"],
                        resources: ["*"],
                        verbs: ["*"],
                    },
                ],
            },
            mgrCluster: kube.ClusterRole("rook-ceph-mgr-cluster") {
                metadata+: env.metadata {
                    namespace:: null,
                },
                rules: [
                    {
                        apiGroups: [""],
                        resources: ["configmaps", "nodes", "nodes/proxy"],
                        verbs: ["get", "list", "watch"],
                    },
                ]
            },
        },

        crb: kube.ClusterRoleBinding("ceph-rook-global") {
            metadata+: env.metadata,
            roleRef: {
                apiGroup: "rbac.authorization.k8s.io",
                kind: "ClusterRole",
                name: env.crs.global.metadata.name,
            },
            subjects: [
                {
                    kind: "ServiceAccount",
                    name: env.sa.metadata.name,
                    namespace: env.sa.metadata.namespace,
                },
            ],
        },

        role: kube.Role("ceph-rook-system") {
            metadata+: env.metadata,
            rules: [
                {
                    apiGroups: [""],
                    resources: ["pods", "configmaps"],
                    verbs: ["get", "list", "watch", "patch", "create", "update", "delete"],
                },
                {
                    apiGroups: ["apps"],
                    resources: ["daemonsets"],
                    verbs: ["get", "list", "watch", "create", "update", "delete"],
                },
            ],
        },

        rb: kube.RoleBinding("ceph-rook-system") {
            metadata+: env.metadata,
            roleRef: {
                apiGroup: "rbac.authorization.k8s.io",
                kind: "Role",
                name: env.role.metadata.name,
            },
            subjects: [
                {
                    kind: "ServiceAccount",
                    name: env.sa.metadata.name,
                    namespace: env.sa.metadata.namespace,
                },
            ],
        },

        operator: kube.Deployment("rook-ceph-operator") {
            metadata+: env.metadata,
            spec+: {
                template+: {
                    spec+: {
                        serviceAccountName: env.sa.metadata.name,
                        containers_: {
                            operator: kube.Container("rook-ceph-operator") {
                                image: cfg.image,
                                args: ["ceph", "operator"],
                                volumeMounts_: {
                                    "rook-config": { mountPath: "/var/lib/rook" },
                                    "default-config-dir": { mountPath: "/etc/ceph" },
                                },
                                env_: {
                                    LIB_MODULES_DIR_PATH: "/run/current-system/kernel-modules/lib/modules/",
                                    ROOK_ALLOW_MULTIPLE_FILESYSTEMS: "false",
                                    ROOK_LOG_LEVEL: "info",
                                    ROOK_MON_HEALTHCHECK_INTERVAL: "45s",
                                    ROOK_MON_OUT_TIMEOUT: "600s",
                                    ROOK_DISCOVER_DEVICES_INTERVAL: "60m",
                                    ROOK_HOSTPATH_REQUIRES_PRIVILEGED: "false",
                                    ROOK_ENABLE_SELINUX_RELABELING: "true",
                                    ROOK_ENABLE_FSGROUP: "true",
                                    NODE_NAME: kube.FieldRef("spec.nodeName"),
                                    POD_NAME: kube.FieldRef("metadata.name"),
                                    POD_NAMESPACE: kube.FieldRef("metadata.namespace"),
                                },
                            },
                        },
                        volumes_: {
                            "rook-config": { emptyDir: {} },
                            "default-config-dir": { emptyDir: {} },
                        },
                    },
                },
            },
        },
    },
}