| # Deploy Rook/Ceph Operator |
| |
| local kube = import "../../../kube/kube.libsonnet"; |
| local policies = import "../../../kube/policies.libsonnet"; |
| |
| { |
| Operator: { |
| local env = self, |
| local cfg = env.cfg, |
| cfg:: { |
| image: "rook/ceph:v1.0.6", |
| namespace: "rook-ceph-system", |
| }, |
| |
| metadata:: { |
| namespace: cfg.namespace, |
| labels: { |
| "operator": "rook", |
| "storage-backend": "ceph", |
| }, |
| }, |
| |
| namespace: kube.Namespace(cfg.namespace), |
| |
| policyInsecure: policies.AllowNamespaceInsecure(cfg.namespace), |
| |
| crds: { |
| cephclusters: kube.CustomResourceDefinition("ceph.rook.io", "v1", "CephCluster") { |
| spec+: { |
| additionalPrinterColumns: [ |
| { name: "DataDirHostPath", type: "string", description: "Directory used on the K8s nodes", JSONPath: ".spec.dataDirHostPath" }, |
| { name: "MonCount", type: "string", description: "Number of MONs", JSONPath: ".spec.mon.count" }, |
| { name: "Age", type: "date", JSONPath: ".metadata.creationTimestamp" }, |
| { name: "State", type: "string", description: "Current State", JSONPath: ".status.state" }, |
| ], |
| validation: { |
| # Converted from official operator YAML |
| "openAPIV3Schema": { |
| "properties": { |
| "spec": { |
| "properties": { |
| "cephVersion": { |
| "properties": { |
| "allowUnsupported": { |
| "type": "boolean" |
| }, |
| "image": { |
| "type": "string" |
| }, |
| "name": { |
| "pattern": "^(luminous|mimic|nautilus)$", |
| "type": "string" |
| } |
| } |
| }, |
| "dashboard": { |
| "properties": { |
| "enabled": { |
| "type": "boolean" |
| }, |
| "urlPrefix": { |
| "type": "string" |
| }, |
| "port": { |
| "type": "integer" |
| } |
| } |
| }, |
| "dataDirHostPath": { |
| "pattern": "^/(\\S+)", |
| "type": "string" |
| }, |
| "mon": { |
| "properties": { |
| "allowMultiplePerNode": { |
| "type": "boolean" |
| }, |
| "count": { |
| "maximum": 9, |
| "minimum": 1, |
| "type": "integer" |
| }, |
| "preferredCount": { |
| "maximum": 9, |
| "minimum": 0, |
| "type": "integer" |
| } |
| }, |
| "required": [ |
| "count" |
| ] |
| }, |
| "network": { |
| "properties": { |
| "hostNetwork": { |
| "type": "boolean" |
| } |
| } |
| }, |
| "storage": { |
| "properties": { |
| "nodes": { |
| "items": {}, |
| "type": "array" |
| }, |
| "useAllDevices": {}, |
| "useAllNodes": { |
| "type": "boolean" |
| } |
| } |
| } |
| }, |
| "required": [ |
| "mon" |
| ] |
| } |
| } |
| } |
| } |
| }, |
| }, |
| cephfilesystems: kube.CustomResourceDefinition("ceph.rook.io", "v1", "CephFilesystem") { |
| spec+: { |
| additionalPrinterColumns: [ |
| { name: "MdsCount", type: "string", description: "Number of MDs", JSONPath: ".spec.metadataServer.activeCount" }, |
| { name: "Age", type: "date", JSONPath: ".metadata.creationTimestamp" }, |
| ], |
| }, |
| }, |
| cephnfses: kube.CustomResourceDefinition("ceph.rook.io", "v1", "CephNFS") { |
| spec+: { |
| names+: { |
| plural: "cephnfses", |
| shortNames: ["nfs"], |
| }, |
| }, |
| }, |
| cephobjectstores: kube.CustomResourceDefinition("ceph.rook.io", "v1", "CephObjectStore"), |
| cephobjectstoreusers: kube.CustomResourceDefinition("ceph.rook.io", "v1", "CephObjectStoreUser"), |
| cephblockpools: kube.CustomResourceDefinition("ceph.rook.io", "v1", "CephBlockPool"), |
| volumes: kube.CustomResourceDefinition("rook.io", "v1alpha2", "Volume") { |
| spec+: { |
| names+: { |
| shortNames: ["rv"], |
| }, |
| }, |
| }, |
| }, |
| |
| sa: kube.ServiceAccount("rook-ceph-system") { |
| metadata+: env.metadata, |
| }, |
| |
| crs: { |
| clusterMgmt: kube.ClusterRole("rook-ceph-cluster-mgmt") { |
| metadata+: env.metadata { namespace:: null }, |
| rules: [ |
| { |
| apiGroups: [""], |
| resources: ["secrets", "pods", "pods/log", "services", "configmaps"], |
| verbs: ["get", "list", "watch", "patch", "create", "update", "delete"], |
| }, |
| { |
| apiGroups: ["apps"], |
| resources: ["deployments", "daemonsets", "replicasets"], |
| verbs: ["get", "list", "watch", "create", "update", "delete"], |
| }, |
| ], |
| }, |
| global: kube.ClusterRole("rook-ceph-global") { |
| metadata+: env.metadata { namespace:: null }, |
| rules: [ |
| { |
| apiGroups: [""], |
| resources: ["pods", "nodes", "nodes/proxy"], |
| verbs: ["get", "list", "watch"], |
| }, |
| { |
| apiGroups: [""], |
| resources: ["events", "persistentvolumes", "persistentvolumeclaims", "endpoints"], |
| verbs: ["get", "list", "watch", "patch", "create", "update", "delete"], |
| }, |
| { |
| apiGroups: ["storage.k8s.io"], |
| resources: ["storageclasses"], |
| verbs: ["get", "list", "watch", "create", "update", "delete"], |
| }, |
| { |
| apiGroups: ["batch"], |
| resources: ["jobs"], |
| verbs: ["get", "list", "watch", "create", "update", "delete"], |
| }, |
| { |
| apiGroups: ["ceph.rook.io"], |
| resources: ["*"], |
| verbs: ["*"], |
| }, |
| { |
| apiGroups: ["rook.io"], |
| resources: ["*"], |
| verbs: ["*"], |
| }, |
| ], |
| }, |
| mgrCluster: kube.ClusterRole("rook-ceph-mgr-cluster") { |
| metadata+: env.metadata { namespace:: null }, |
| rules: [ |
| { |
| apiGroups: [""], |
| resources: ["configmaps", "nodes", "nodes/proxy"], |
| verbs: ["get", "list", "watch"], |
| }, |
| ] |
| }, |
| }, |
| |
| crb: kube.ClusterRoleBinding("ceph-rook-global") { |
| metadata+: env.metadata { namespace:: null }, |
| roleRef_: env.crs.global, |
| subjects_: [env.sa], |
| }, |
| |
| role: kube.Role("ceph-rook-system") { |
| metadata+: env.metadata, |
| rules: [ |
| { |
| apiGroups: [""], |
| resources: ["pods", "configmaps"], |
| verbs: ["get", "list", "watch", "patch", "create", "update", "delete"], |
| }, |
| { |
| apiGroups: ["apps"], |
| resources: ["daemonsets"], |
| verbs: ["get", "list", "watch", "create", "update", "delete"], |
| }, |
| ], |
| }, |
| |
| rb: kube.RoleBinding("ceph-rook-system") { |
| metadata+: env.metadata, |
| roleRef_: env.role, |
| subjects_: [env.sa], |
| }, |
| |
| operator: kube.Deployment("rook-ceph-operator") { |
| metadata+: env.metadata, |
| spec+: { |
| template+: { |
| spec+: { |
| serviceAccountName: env.sa.metadata.name, |
| containers_: { |
| operator: kube.Container("rook-ceph-operator") { |
| image: cfg.image, |
| args: ["ceph", "operator"], |
| volumeMounts_: { |
| "rook-config": { mountPath: "/var/lib/rook" }, |
| "default-config-dir": { mountPath: "/etc/ceph" }, |
| }, |
| env_: { |
| LIB_MODULES_DIR_PATH: "/run/current-system/kernel-modules/lib/modules/", |
| ROOK_ALLOW_MULTIPLE_FILESYSTEMS: "false", |
| ROOK_LOG_LEVEL: "INFO", |
| ROOK_MON_HEALTHCHECK_INTERVAL: "45s", |
| ROOK_MON_OUT_TIMEOUT: "600s", |
| ROOK_DISCOVER_DEVICES_INTERVAL: "60m", |
| ROOK_HOSTPATH_REQUIRES_PRIVILEGED: "false", |
| ROOK_ENABLE_SELINUX_RELABELING: "true", |
| ROOK_ENABLE_FSGROUP: "true", |
| NODE_NAME: kube.FieldRef("spec.nodeName"), |
| POD_NAME: kube.FieldRef("metadata.name"), |
| POD_NAMESPACE: kube.FieldRef("metadata.namespace"), |
| }, |
| }, |
| }, |
| volumes_: { |
| "rook-config": { emptyDir: {} }, |
| "default-config-dir": { emptyDir: {} }, |
| }, |
| }, |
| }, |
| }, |
| }, |
| }, |
| |
| // Create a new Ceph cluster in a new namespace. |
| Cluster(operator, name):: { |
| local cluster = self, |
| spec:: error "please define cluster spec", |
| |
| |
| metadata:: { |
| namespace: name, |
| }, |
| |
| name(suffix):: cluster.metadata.namespace + "-" + suffix, |
| |
| namespace: kube.Namespace(cluster.metadata.namespace), |
| |
| sa: { |
| // service accounts need to be hardcoded, see operator source. |
| osd: kube.ServiceAccount("rook-ceph-osd") { |
| metadata+: cluster.metadata, |
| }, |
| mgr: kube.ServiceAccount("rook-ceph-mgr") { |
| metadata+: cluster.metadata, |
| }, |
| }, |
| |
| roles: { |
| osd: kube.Role(cluster.name("osd")) { |
| metadata+: cluster.metadata, |
| rules: [ |
| { |
| apiGroups: [""], |
| resources: ["configmaps"], |
| verbs: ["get", "list", "watch", "create", "update", "delete"], |
| } |
| ], |
| }, |
| mgr: kube.Role(cluster.name("mgr")) { |
| metadata+: cluster.metadata, |
| rules: [ |
| { |
| apiGroups: [""], |
| resources: ["pods", "services"], |
| verbs: ["get", "list", "watch"], |
| }, |
| { |
| apiGroups: ["batch"], |
| resources: ["jobs"], |
| verbs: ["get", "list", "watch", "create", "update", "delete"], |
| }, |
| { |
| apiGroups: ["ceph.rook.io"], |
| resources: ["*"], |
| verbs: ["*"], |
| }, |
| ], |
| }, |
| mgrSystem: kube.ClusterRole(cluster.name("mgr-system")) { |
| metadata+: cluster.metadata { namespace:: null }, |
| rules: [ |
| { |
| apiGroups: [""], |
| resources: ["configmaps"], |
| verbs: ["get", "list", "watch"], |
| } |
| ], |
| }, |
| }, |
| |
| rbs: [ |
| kube.RoleBinding(cluster.name(el.name)) { |
| metadata+: cluster.metadata, |
| roleRef_: el.role, |
| subjects_: [el.sa], |
| }, |
| for el in [ |
| // Allow Operator SA to perform Cluster Mgmt in this namespace. |
| { name: "cluster-mgmt", role: operator.crs.clusterMgmt, sa: operator.sa }, |
| { name: "osd", role: cluster.roles.osd, sa: cluster.sa.osd }, |
| { name: "mgr", role: cluster.roles.mgr, sa: cluster.sa.mgr }, |
| { name: "mgr-cluster", role: operator.crs.mgrCluster, sa: cluster.sa.mgr }, |
| ] |
| ], |
| |
| mgrSystemRB: kube.RoleBinding(cluster.name("mgr-system")) { |
| metadata+: { |
| namespace: operator.cfg.namespace, |
| }, |
| roleRef_: cluster.roles.mgrSystem, |
| subjects_: [cluster.sa.mgr], |
| }, |
| |
| cluster: kube._Object("ceph.rook.io/v1", "CephCluster", name) { |
| metadata+: cluster.metadata, |
| spec: { |
| cephVersion: { |
| # https://github.com/rook/rook/issues/2945#issuecomment-483964014 |
| #image: "ceph/ceph:v13.2.5-20190319", |
| image: "ceph/ceph:v14.2.2-20190830", |
| allowUnsupported: true, |
| }, |
| dataDirHostPath: if name == "ceph-waw2" then "/var/lib/rook" else "/var/lib/rook-%s" % [name], |
| dashboard: { |
| ssl: false, |
| enabled: true, |
| port: 8080, |
| }, |
| } + cluster.spec, |
| }, |
| |
| dashboardService: kube.Service(cluster.name("dashboard")) { |
| metadata+: cluster.metadata, |
| spec: { |
| ports: [ |
| { name: "dashboard", port: 80, targetPort: 8080, protocol: "TCP" }, |
| ], |
| selector: { |
| app: "rook-ceph-mgr", |
| rook_cluster: name, |
| }, |
| type: "ClusterIP", |
| }, |
| }, |
| |
| dashboardIngress: kube.Ingress(cluster.name("dashboard")) { |
| metadata+: cluster.metadata { |
| annotations+: { |
| "kubernetes.io/tls-acme": "true", |
| "certmanager.k8s.io/cluster-issuer": "letsencrypt-prod", |
| }, |
| }, |
| spec+: { |
| tls: [ |
| { |
| hosts: ["%s.hswaw.net" % name], |
| secretName: cluster.name("dashboard"), |
| }, |
| ], |
| rules: [ |
| { |
| host: "%s.hswaw.net" % name, |
| http: { |
| paths: [ |
| { path: "/", backend: cluster.dashboardService.name_port }, |
| ] |
| }, |
| } |
| ], |
| }, |
| }, |
| |
| # Benji is a backup tool, external to rook, that we use for backing up |
| # RBDs. |
| benji: { |
| sa: kube.ServiceAccount(cluster.name("benji")) { |
| metadata+: cluster.metadata, |
| }, |
| |
| cr: kube.ClusterRole(cluster.name("benji")) { |
| rules: [ |
| { |
| apiGroups: [""], |
| resources: [ |
| "persistentvolumes", |
| "persistentvolumeclaims" |
| ], |
| verbs: ["list", "get"], |
| }, |
| { |
| apiGroups: [""], |
| resources: [ |
| "events", |
| ], |
| verbs: ["create", "update"], |
| }, |
| ], |
| }, |
| |
| crb: kube.ClusterRoleBinding(cluster.name("benji")) { |
| roleRef_: cluster.benji.cr, |
| subjects_: [cluster.benji.sa], |
| }, |
| |
| config: kube.Secret(cluster.name("benji-config")) { |
| metadata+: cluster.metadata, |
| data_: { |
| "benji.yaml": std.manifestJson({ |
| configurationVersion: '1', |
| databaseEngine: 'sqlite:////data/benji.sqlite', |
| defaultStorage: 'wasabi', |
| storages: [ |
| { |
| name: "wasabi", |
| storageId: 1, |
| module: "s3", |
| configuration: cluster.spec.benji.s3Configuration { |
| activeTransforms: ["encrypt"], |
| }, |
| }, |
| ], |
| transforms: [ |
| { |
| name: "encrypt", |
| module: "aes_256_gcm", |
| configuration: { |
| # not secret. |
| kdfSalt: "T2huZzZpcGhhaWM3QWVwaDhybzRhaDNhbzFpc2VpOWFobDNSZWVQaGVvTWV1bmVaYWVsNHRoYWg5QWVENHNoYWg0ZGFoN3Rlb3NvcHVuZzNpZXZpMm9vTG9vbmc1YWlmb0RlZXAwYmFobDlab294b2hjaG9odjRhbzFsYWkwYWk=", |
| kdfIterations: 2137, |
| password: cluster.spec.benji.encryptionPassword, |
| }, |
| }, |
| ], |
| ios: [ |
| { name: pool, module: "rbd" } |
| for pool in cluster.spec.benji.pools |
| ], |
| }), |
| }, |
| }, |
| |
| # Yes, Benji keeps data (backup metadata) on the ceph cluster that |
| # it backs up. However: |
| # - we add a command to benji-k8s to also copy over the sqlite |
| # database over to s3 |
| # - benji can, in a pinch, restore without a database if a version |
| # is known: https://benji-backup.me/restore.html#restoring-without-a-database |
| data: kube.PersistentVolumeClaim(cluster.name("benji-data")) { |
| metadata+: cluster.metadata, |
| spec+: { |
| storageClassName: cluster.spec.benji.metadataStorageClass, |
| accessModes: [ "ReadWriteOnce" ], |
| resources: { |
| requests: { |
| storage: "1Gi", |
| }, |
| }, |
| }, |
| }, |
| |
| # Extra scripts. |
| extrabins: kube.ConfigMap(cluster.name("benji-extrabins")) { |
| metadata+: cluster.metadata, |
| data: { |
| "metabackup.sh" : ||| |
| # Make backups of sqlite3 metadata used by Benji. |
| # The backups live in the same bucket as backups, and the metabackups |
| # are named `metabackup-0..10`, where 0 is the newest backup. Any time |
| # this script is called, backups get shifted one way to the left (9 to 10, |
| # 8 to 9, etc). This ensures we have at least 10 backup replicas. |
| |
| set -e |
| |
| which s3cmd || pip install --upgrade s3cmd |
| |
| AWS_ACCESS_KEY_ID=$(jq -r .storages[0].configuration.awsAccessKeyId < /etc/benji/benji.yaml) |
| AWS_SECRET_ACCESS_KEY=$(jq -r .storages[0].configuration.awsSecretAccessKey < /etc/benji/benji.yaml) |
| BUCKET=$(jq -r .storages[0].configuration.bucketName < /etc/benji/benji.yaml) |
| |
| s3() { |
| s3cmd --host=s3.wasabisys.com \ |
| "--host-bucket=%(bucket)s.s3.wasabisys.com" \ |
| --region=eu-central-1 \ |
| --access_key=$AWS_ACCESS_KEY_ID \ |
| --secret_key=$AWS_SECRET_ACCESS_KEY \ |
| "$@" |
| } |
| |
| # Copy over old backups, if they exist. |
| for i in `seq 9 -1 0`; do |
| from="s3://$BUCKET/metabackup-$i.sqlite3" |
| to="s3://$BUCKET/metabackup-$((i+1)).sqlite3" |
| |
| if [[ $(s3 ls $from | wc -l) -eq 0 ]]; then |
| echo "$from does not exist, skipping shift." |
| continue |
| fi |
| echo "Moving $from to $to..." |
| s3 mv $from $to |
| done |
| |
| # Make new metabackup. |
| s3 put /data/benji.sqlite s3://$BUCKET/metabackup-0.sqlite3 |
| |
| |||, |
| "get-rook-creds.sh": ||| |
| # Based on the Rook Toolbox /usr/local/bin/toolbox.sh script. |
| # Copyright 2016 The Rook Authors. All rights reserved. |
| |
| CEPH_CONFIG="/etc/ceph/ceph.conf" |
| MON_CONFIG="/etc/rook/mon-endpoints" |
| KEYRING_FILE="/etc/ceph/keyring" |
| |
| # create a ceph config file in its default location so ceph/rados tools can be used |
| # without specifying any arguments |
| write_endpoints() { |
| endpoints=$(cat ${MON_CONFIG}) |
| |
| # filter out the mon names |
| mon_endpoints=$(echo ${endpoints} | sed 's/[a-z]\+=//g') |
| |
| # filter out the legacy mon names |
| mon_endpoints=$(echo ${mon_endpoints} | sed 's/rook-ceph-mon[0-9]\+=//g') |
| |
| DATE=$(date) |
| echo "$DATE writing mon endpoints to ${CEPH_CONFIG}: ${endpoints}" |
| cat <<EOF > ${CEPH_CONFIG} |
| [global] |
| mon_host = ${mon_endpoints} |
| |
| [client.admin] |
| keyring = ${KEYRING_FILE} |
| EOF |
| } |
| |
| # watch the endpoints config file and update if the mon endpoints ever change |
| watch_endpoints() { |
| # get the timestamp for the target of the soft link |
| real_path=$(realpath ${MON_CONFIG}) |
| initial_time=$(stat -c %Z ${real_path}) |
| while true; do |
| real_path=$(realpath ${MON_CONFIG}) |
| latest_time=$(stat -c %Z ${real_path}) |
| |
| if [[ "${latest_time}" != "${initial_time}" ]]; then |
| write_endpoints |
| initial_time=${latest_time} |
| fi |
| sleep 10 |
| done |
| } |
| |
| # create the keyring file |
| cat <<EOF > ${KEYRING_FILE} |
| [client.admin] |
| key = ${ROOK_ADMIN_SECRET} |
| EOF |
| |
| # write the initial config file |
| write_endpoints |
| |
| # continuously update the mon endpoints if they fail over |
| watch_endpoints & |
| ||| |
| }, |
| }, |
| |
| cronjob: kube.CronJob(cluster.name("benji")) { |
| metadata+: cluster.metadata, |
| spec+: { # CronJob Spec |
| schedule: "42 0 * * *", # Daily at 42 minute past midnight. |
| jobTemplate+: { |
| spec+: { # Job Spec |
| selector:: null, |
| template+: { |
| spec+: { # PodSpec |
| serviceAccountName: cluster.benji.sa.metadata.name, |
| containers_: { |
| benji: kube.Container(cluster.name("benji")) { |
| # TODO(q3k): switch back to upstream after pull/65 goes in. |
| # Currently this is being built from github.com/q3k/benji. |
| # https://github.com/elemental-lf/benji/pull/65 |
| image: "registry.k0.hswaw.net/q3k/benji-k8s:20191221-2336", |
| volumeMounts_: { |
| extrabins: { mountPath: "/usr/local/extrabins" }, |
| monendpoints: { mountPath: "/etc/rook" }, |
| benjiconfig: { mountPath: "/etc/benji" }, |
| data: { mountPath: "/data" }, |
| }, |
| env_: { |
| ROOK_ADMIN_SECRET: { secretKeyRef: { name: "rook-ceph-mon", key: "admin-secret" }}, |
| }, |
| command: [ |
| "bash", "-c", ||| |
| bash /usr/local/extrabins/get-rook-creds.sh |
| benji-backup-pvc %s |
| benji-command enforce latest3,hours48,days7,months12 |
| benji-command cleanup |
| bash /usr/local/extrabins/metabackup.sh |
| ||| % [std.join(" ", ["--pool-filter=%s" % [p] for p in cluster.spec.benji.pools])], |
| ], |
| }, |
| }, |
| volumes_: { |
| data: kube.PersistentVolumeClaimVolume(cluster.benji.data), |
| benjiconfig: kube.SecretVolume(cluster.benji.config), |
| extrabins: kube.ConfigMapVolume(cluster.benji.extrabins), |
| monendpoints: { |
| configMap: { |
| name: "rook-ceph-mon-endpoints", |
| items: [ |
| { key: "data", path: "mon-endpoints" }, |
| ], |
| }, |
| }, |
| }, |
| }, |
| }, |
| }, |
| }, |
| }, |
| }, |
| }, |
| }, |
| |
| ReplicatedBlockPool(cluster, name):: { |
| local pool = self, |
| name:: name, |
| |
| spec:: error "spec must be specified", |
| |
| pool: kube._Object("ceph.rook.io/v1", "CephBlockPool", name) { |
| metadata+: cluster.metadata, |
| spec: pool.spec, |
| }, |
| |
| storageClass: kube.StorageClass(name) { |
| provisioner: "ceph.rook.io/block", |
| parameters: { |
| blockPool: pool.pool.metadata.name, |
| clusterNamespace: pool.pool.metadata.namespace, |
| fstype: "ext4", |
| }, |
| reclaimPolicy: "Retain", |
| }, |
| }, |
| |
| ECBlockPool(cluster, name):: { |
| local pool = self, |
| name:: name, |
| metadataReplicas:: 3, |
| |
| spec:: error "spec must be specified", |
| |
| pool: kube._Object("ceph.rook.io/v1", "CephBlockPool", name) { |
| metadata+: cluster.metadata, |
| spec: pool.spec, |
| }, |
| metapool: kube._Object("ceph.rook.io/v1", "CephBlockPool", name + "-metadata") { |
| metadata+: cluster.metadata, |
| spec: { |
| failureDomain: "host", |
| replicated: { |
| size: pool.metadataReplicas, |
| }, |
| }, |
| }, |
| |
| storageClass: kube.StorageClass(name) { |
| provisioner: "ceph.rook.io/block", |
| parameters: { |
| blockPool: pool.metapool.metadata.name, |
| dataBlockPool: pool.pool.metadata.name, |
| clusterNamespace: pool.pool.metadata.namespace, |
| fstype: "ext4", |
| }, |
| reclaimPolicy: "Retain", |
| }, |
| }, |
| |
| S3ObjectStore(cluster, name):: { |
| local store = self, |
| spec:: error "spec must be specified", |
| objectStore: kube._Object("ceph.rook.io/v1", "CephObjectStore", name) { |
| metadata+: cluster.metadata, |
| spec: store.spec { |
| gateway: { |
| type: "s3", |
| port: 80, |
| instances: 1, |
| allNodes: false, |
| }, |
| }, |
| }, |
| |
| objectIngress: kube.Ingress(name) { |
| metadata+: cluster.metadata { |
| annotations+: { |
| "kubernetes.io/tls-acme": "true", |
| "certmanager.k8s.io/cluster-issuer": "letsencrypt-prod", |
| "nginx.ingress.kubernetes.io/proxy-body-size": "0", |
| }, |
| }, |
| spec+: { |
| tls: [ |
| { |
| hosts: ["object.%s.hswaw.net" % [cluster.metadata.namespace]], |
| secretName: "%s-tls" % [name], |
| }, |
| ], |
| rules: [ |
| { |
| host: "object.%s.hswaw.net" % [cluster.metadata.namespace], |
| http: { |
| paths: [ |
| { |
| path: "/", |
| backend: { |
| serviceName: "rook-ceph-rgw-%s" % [name], |
| servicePort: 80, |
| }, |
| }, |
| ] |
| }, |
| } |
| ], |
| }, |
| }, |
| }, |
| } |