# Deploy Rook/Ceph Operator

local kube = import "../../../kube/kube.libsonnet";

{
    Operator: {
        local env = self,
        local cfg = env.cfg,
        cfg:: {
            image: "rook/ceph:v1.0.5",
            namespace: "rook-ceph-system",
        },

        metadata:: {
            namespace: cfg.namespace,
            labels: {
                "operator": "rook",
                "storage-backend": "ceph",
            },
        },

        namespace: kube.Namespace(cfg.namespace),

        crds: {
            cephclusters: kube.CustomResourceDefinition("ceph.rook.io", "v1", "CephCluster") {
                spec+: {
                    additionalPrinterColumns: [
                        { name: "DataDirHostPath", type: "string", description: "Directory used on the K8s nodes", JSONPath: ".spec.dataDirHostPath" },
                        { name: "MonCount", type: "string", description: "Number of MONs", JSONPath: ".spec.mon.count" },
                        { name: "Age", type: "date", JSONPath: ".metadata.creationTimestamp" },
                        { name: "State", type: "string", description: "Current State", JSONPath: ".status.state" },
                    ],
                    validation: {
                        # Converted from official operator YAML
                        "openAPIV3Schema": {
                            "properties": {
                                "spec": {
                                    "properties": {
                                        "cephVersion": {
                                            "properties": {
                                                "allowUnsupported": {
                                                    "type": "boolean"
                                                },
                                                "image": {
                                                    "type": "string"
                                                },
                                                "name": {
                                                    "pattern": "^(luminous|mimic|nautilus)$",
                                                    "type": "string"
                                                }
                                            }
                                        },
                                        "dashboard": {
                                            "properties": {
                                                "enabled": {
                                                    "type": "boolean"
                                                },
                                                "urlPrefix": {
                                                    "type": "string"
                                                },
                                                "port": {
                                                    "type": "integer"
                                                }
                                            }
                                        },
                                        "dataDirHostPath": {
                                            "pattern": "^/(\\S+)",
                                            "type": "string"
                                        },
                                        "mon": {
                                            "properties": {
                                                "allowMultiplePerNode": {
                                                    "type": "boolean"
                                                },
                                                "count": {
                                                    "maximum": 9,
                                                    "minimum": 1,
                                                    "type": "integer"
                                                },
                                                "preferredCount": {
                                                    "maximum": 9,
                                                    "minimum": 0,
                                                    "type": "integer"
                                                }
                                            },
                                            "required": [
                                                "count"
                                            ]
                                        },
                                        "network": {
                                            "properties": {
                                                "hostNetwork": {
                                                    "type": "boolean"
                                                }
                                            }
                                        },
                                        "storage": {
                                            "properties": {
                                                "nodes": {
                                                    "items": {},
                                                    "type": "array"
                                                },
                                                "useAllDevices": {},
                                                "useAllNodes": {
                                                    "type": "boolean"
                                                }
                                            }
                                        }
                                    },
                                    "required": [
                                        "mon"
                                    ]
                                }
                            }
                        }
                    }
                },
            },
            cephfilesystems: kube.CustomResourceDefinition("ceph.rook.io", "v1", "CephFilesystem") {
                spec+: {
                    additionalPrinterColumns: [
                        { name: "MdsCount", type: "string", description: "Number of MDs", JSONPath: ".spec.metadataServer.activeCount" },
                        { name: "Age", type: "date", JSONPath: ".metadata.creationTimestamp" },
                    ],
                },
            },
            cephnfses: kube.CustomResourceDefinition("ceph.rook.io", "v1", "CephNFS") {
                spec+: {
                    names+: {
                        plural: "cephnfses",
                        shortNames: ["nfs"],
                    },
                },
            },
            cephobjectstores: kube.CustomResourceDefinition("ceph.rook.io", "v1", "CephObjectStore"),
            cephobjectstoreusers: kube.CustomResourceDefinition("ceph.rook.io", "v1", "CephObjectStoreUser"),
            cephblockpools: kube.CustomResourceDefinition("ceph.rook.io", "v1", "CephBlockPool"),
            volumes: kube.CustomResourceDefinition("rook.io", "v1alpha2", "Volume") {
                spec+: {
                    names+: {
                        shortNames: ["rv"],
                    },
                },
            },
        },

        sa: kube.ServiceAccount("rook-ceph-system") {
            metadata+: env.metadata,
        },

        crs: {
            clusterMgmt: kube.ClusterRole("rook-ceph-cluster-mgmt") {
                metadata+: env.metadata { namespace:: null },
                rules: [
                    {
                        apiGroups: [""],
                        resources: ["secrets", "pods", "pods/log", "services", "configmaps"],
                        verbs: ["get", "list", "watch", "patch", "create", "update", "delete"],
                    },
                    {
                        apiGroups: ["apps"],
                        resources: ["deployments", "daemonsets", "replicasets"],
                        verbs: ["get", "list", "watch", "create", "update", "delete"],
                    },
                ],
            },
            global: kube.ClusterRole("rook-ceph-global") {
                metadata+: env.metadata { namespace:: null },
                rules: [
                    {
                        apiGroups: [""],
                        resources: ["pods", "nodes", "nodes/proxy"],
                        verbs: ["get", "list", "watch"],
                    },
                    {
                        apiGroups: [""],
                        resources: ["events", "persistentvolumes", "persistentvolumeclaims", "endpoints"],
                        verbs: ["get", "list", "watch", "patch", "create", "update", "delete"],
                    },
                    {
                        apiGroups: ["storage.k8s.io"],
                        resources: ["storageclasses"],
                        verbs: ["get", "list", "watch", "create", "update", "delete"],
                    },
                    {
                        apiGroups: ["batch"],
                        resources: ["jobs"],
                        verbs: ["get", "list", "watch", "create", "update", "delete"],
                    },
                    {
                        apiGroups: ["ceph.rook.io"],
                        resources: ["*"],
                        verbs: ["*"],
                    },
                    {
                        apiGroups: ["rook.io"],
                        resources: ["*"],
                        verbs: ["*"],
                    },
                ],
            },
            mgrCluster: kube.ClusterRole("rook-ceph-mgr-cluster") {
                metadata+: env.metadata { namespace:: null },
                rules: [
                    {
                        apiGroups: [""],
                        resources: ["configmaps", "nodes", "nodes/proxy"],
                        verbs: ["get", "list", "watch"],
                    },
                ]
            },
        },

        crb: kube.ClusterRoleBinding("ceph-rook-global") {
            metadata+: env.metadata { namespace:: null },
            roleRef: {
                apiGroup: "rbac.authorization.k8s.io",
                kind: "ClusterRole",
                name: env.crs.global.metadata.name,
            },
            subjects: [
                {
                    kind: "ServiceAccount",
                    name: env.sa.metadata.name,
                    namespace: env.sa.metadata.namespace,
                },
            ],
        },

        role: kube.Role("ceph-rook-system") {
            metadata+: env.metadata,
            rules: [
                {
                    apiGroups: [""],
                    resources: ["pods", "configmaps"],
                    verbs: ["get", "list", "watch", "patch", "create", "update", "delete"],
                },
                {
                    apiGroups: ["apps"],
                    resources: ["daemonsets"],
                    verbs: ["get", "list", "watch", "create", "update", "delete"],
                },
            ],
        },

        rb: kube.RoleBinding("ceph-rook-system") {
            metadata+: env.metadata,
            roleRef: {
                apiGroup: "rbac.authorization.k8s.io",
                kind: "Role",
                name: env.role.metadata.name,
            },
            subjects: [
                {
                    kind: "ServiceAccount",
                    name: env.sa.metadata.name,
                    namespace: env.sa.metadata.namespace,
                },
            ],
        },

        operator: kube.Deployment("rook-ceph-operator") {
            metadata+: env.metadata,
            spec+: {
                template+: {
                    spec+: {
                        serviceAccountName: env.sa.metadata.name,
                        containers_: {
                            operator: kube.Container("rook-ceph-operator") {
                                image: cfg.image,
                                args: ["ceph", "operator"],
                                volumeMounts_: {
                                    "rook-config": { mountPath: "/var/lib/rook" },
                                    "default-config-dir": { mountPath: "/etc/ceph" },
                                },
                                env_: {
                                    LIB_MODULES_DIR_PATH: "/run/current-system/kernel-modules/lib/modules/",
                                    ROOK_ALLOW_MULTIPLE_FILESYSTEMS: "false",
                                    ROOK_LOG_LEVEL: "INFO",
                                    ROOK_MON_HEALTHCHECK_INTERVAL: "45s",
                                    ROOK_MON_OUT_TIMEOUT: "600s",
                                    ROOK_DISCOVER_DEVICES_INTERVAL: "60m",
                                    ROOK_HOSTPATH_REQUIRES_PRIVILEGED: "false",
                                    ROOK_ENABLE_SELINUX_RELABELING: "true",
                                    ROOK_ENABLE_FSGROUP: "true",
                                    NODE_NAME: kube.FieldRef("spec.nodeName"),
                                    POD_NAME: kube.FieldRef("metadata.name"),
                                    POD_NAMESPACE: kube.FieldRef("metadata.namespace"),
                                },
                            },
                        },
                        volumes_: {
                            "rook-config": { emptyDir: {} },
                            "default-config-dir": { emptyDir: {} },
                        },
                    },
                },
            },
        },
    },

    // Create a new Ceph cluster in a new namespace.
    Cluster(operator, name):: {
        local cluster = self,
        spec:: error "please define cluster spec",


        metadata:: {
            namespace: name,
        },

        name(suffix):: cluster.metadata.namespace + "-" + suffix,

        namespace: kube.Namespace(cluster.metadata.namespace),

        sa: {
            // service accounts need to be hardcoded, see operator source.
            osd: kube.ServiceAccount("rook-ceph-osd") {
                metadata+: cluster.metadata,
            },
            mgr: kube.ServiceAccount("rook-ceph-mgr") {
                metadata+: cluster.metadata,
            },
        },

        roles: {
            osd: kube.Role(cluster.name("osd")) {
                metadata+: cluster.metadata,
                rules: [
                    {
                        apiGroups: [""],
                        resources: ["configmaps"],
                        verbs: ["get", "list", "watch", "create", "update", "delete"],
                    }
                ],
            },
            mgr: kube.Role(cluster.name("mgr")) {
                metadata+: cluster.metadata,
                rules: [
                    {
                        apiGroups: [""],
                        resources: ["pods", "services"],
                        verbs: ["get", "list", "watch"],
                    },
                    {
                        apiGroups: ["batch"],
                        resources: ["jobs"],
                        verbs: ["get", "list", "watch", "create", "update", "delete"],
                    },
                    {
                        apiGroups: ["ceph.rook.io"],
                        resources: ["*"],
                        verbs: ["*"],
                    },
                ],
            },
            mgrSystem: kube.ClusterRole(cluster.name("mgr-system")) {
                metadata+: cluster.metadata { namespace:: null },
                rules: [
                    {
                        apiGroups: [""],
                        resources: ["configmaps"],
                        verbs: ["get", "list", "watch"],
                    }
                ],
            },
        },

        rbs: [
            kube.RoleBinding(cluster.name(el.name)) {
                metadata+: cluster.metadata,
                roleRef: {
                    apiGroup: "rbac.authorization.k8s.io",
                    kind: el.role.kind,
                    name: el.role.metadata.name,
                },
                subjects: [
                    {
                        kind: el.sa.kind,
                        name: el.sa.metadata.name,
                        namespace: el.sa.metadata.namespace,
                    },
                ],
            },
            for el in [
                // Allow Operator SA to perform Cluster Mgmt in this namespace.
                { name: "cluster-mgmt", role: operator.crs.clusterMgmt, sa: operator.sa },
                { name: "osd", role: cluster.roles.osd, sa: cluster.sa.osd }, 
                { name: "mgr", role: cluster.roles.mgr, sa: cluster.sa.mgr },
                { name: "mgr-cluster", role: operator.crs.mgrCluster, sa: cluster.sa.mgr },
            ]
        ],

        mgrSystemRB: kube.RoleBinding(cluster.name("mgr-system")) {
            metadata+: {
                namespace: operator.cfg.namespace,
            },
            roleRef: {
                apiGroup: "rbac.authorization.k8s.io",
                kind: cluster.roles.mgrSystem.kind,
                name: cluster.roles.mgrSystem.metadata.name,
            },
            subjects: [
                {
                    kind: cluster.sa.mgr.kind,
                    name: cluster.sa.mgr.metadata.name,
                    namespace: cluster.sa.mgr.metadata.namespace,
                },
            ],
        },

        cluster: kube._Object("ceph.rook.io/v1", "CephCluster", name) {
            metadata+: cluster.metadata,
            spec: {
                cephVersion: {
                    # https://github.com/rook/rook/issues/2945#issuecomment-483964014
                    #image: "ceph/ceph:v13.2.5-20190319",
                    image: "ceph/ceph:v14.2.2-20190830",
                    allowUnsupported: true,
                },
                dataDirHostPath: "/var/lib/rook",
                dashboard: {
                    ssl: false,
                    enabled: true,
                    port: 8080,
                },
            } + cluster.spec,
        },

        dashboardService: kube.Service(cluster.name("dashboard")) {
            metadata+: cluster.metadata,
            spec: {
                ports: [
                    { name: "dashboard", port: 80, targetPort: 8080, protocol: "TCP" }, 
                ],
                selector: {
                    app: "rook-ceph-mgr",
                    rook_cluster: name,
                },
                type: "ClusterIP",
            },
        },

        dashboardIngress: kube.Ingress(cluster.name("dashboard")) {
            metadata+: cluster.metadata {
                annotations+: {
                    "kubernetes.io/tls-acme": "true",
                    "certmanager.k8s.io/cluster-issuer": "letsencrypt-prod",
                },
            },
            spec+: {
                tls: [
                    {
                        hosts: ["%s.hswaw.net" % name],
                        secretName: cluster.name("dashboard"),
                    },
                ],
                rules: [
                    {
                        host: "%s.hswaw.net" % name,
                        http: {
                            paths: [
                                { path: "/", backend: cluster.dashboardService.name_port },
                            ]
                        },
                    }
                ],
            },
        }
    },

    ReplicatedBlockPool(cluster, name):: {
        local pool = self,
        name:: name,

        spec:: error "spec must be specified",

        pool: kube._Object("ceph.rook.io/v1", "CephBlockPool", name) {
            metadata+: cluster.metadata,
            spec: pool.spec,
        },

        storageClass: kube.StorageClass(name) {
            provisioner: "ceph.rook.io/block",
            parameters: {
                blockPool: pool.pool.metadata.name,
                clusterNamespace: pool.pool.metadata.namespace,
                fstype: "ext4",
            },
            reclaimPolicy: "Retain",
        },
    },

    ECBlockPool(cluster, name):: {
        local pool = self,
        name:: name,

        spec:: error "spec must be specified",

        pool: kube._Object("ceph.rook.io/v1", "CephBlockPool", name) {
            metadata+: cluster.metadata,
            spec: pool.spec,
        },
        metapool: kube._Object("ceph.rook.io/v1", "CephBlockPool", name + "-metadata") {
            metadata+: cluster.metadata,
            spec: {
                failureDomain: "host",
                replicated: {
                    size: 3,
                },
            },
        },

        storageClass: kube.StorageClass(name) {
            provisioner: "ceph.rook.io/block",
            parameters: {
                blockPool: pool.metapool.metadata.name,
                dataBlockPool: pool.pool.metadata.name,
                clusterNamespace: pool.pool.metadata.namespace,
                fstype: "ext4",
            },
            reclaimPolicy: "Retain",
        },
    },

    S3ObjectStore(cluster, name):: {
        local store = self,
        spec:: error "spec must be specified",
        objectStore: kube._Object("ceph.rook.io/v1", "CephObjectStore", name) {
            metadata+: cluster.metadata,
            spec: store.spec {
                gateway: {
                    type: "s3",
                    port: 80,
                    instances: 1,
                    allNodes: false,
                },
            },
        },

        objectIngress: kube.Ingress(name) {
            metadata+: cluster.metadata {
                annotations+: {
                    "kubernetes.io/tls-acme": "true",
                    "certmanager.k8s.io/cluster-issuer": "letsencrypt-prod",
                    "nginx.ingress.kubernetes.io/proxy-body-size": "0",
                },
            },
            spec+: {
                tls: [
                    {
                        hosts: ["object.%s.hswaw.net" % [cluster.metadata.namespace]],
                        secretName: "%s-tls" % [name],
                    },
                ],
                rules: [
                    {
                        host: "object.%s.hswaw.net" % [cluster.metadata.namespace],
                        http: {
                            paths: [
                                {
                                    path: "/",
                                    backend: {
                                        serviceName: "rook-ceph-rgw-%s" % [name],
                                        servicePort: 80,
                                    },
                                },
                            ]
                        },
                    }
                ],
            },
        },
    },
}