blob: 0392edad94465ca84522f14692bac583fd91a229 [file] [log] [blame]
# Deploy Rook/Ceph Operator
local kube = import "../../../kube/hscloud.libsonnet";
local policies = import "../../../kube/policies.libsonnet";
local oa = kube.OpenAPI;
{
Operator: {
local env = self,
local cfg = env.cfg,
cfg:: {
image: "rook/ceph:v1.6.9",
namespace: "rook-ceph-system",
},
metadata:: {
namespace: cfg.namespace,
labels: {
"operator": "rook",
"storage-backend": "ceph",
},
},
namespace: kube.Namespace(cfg.namespace),
policyInsecure: policies.AllowNamespaceInsecure(cfg.namespace),
// Grab CRDs from upstream YAML.
//
// We use a bit of jsonnet to remove some fields that kubebuilder (used
// upstream) added and to override preserveUnknownFIelds (as some older
// deployment apparently set it to true, which doesn't work for new
// CRDs that have default values).
crds: [
(el {
metadata+: {
annotations:: null,
creationTimestamp:: null,
},
status:: null,
spec+: {
preserveUnknownFields: false,
},
})
for el in (std.native("parseYaml")(importstr "rook-crds.yaml")) if el != null
],
sa: {
system: kube.ServiceAccount("rook-ceph-system") {
metadata+: env.metadata,
},
csiCephfsPlugin: kube.ServiceAccount("rook-csi-cephfs-plugin-sa") {
metadata+: env.metadata,
},
csiCephfsProvisioner: kube.ServiceAccount("rook-csi-cephfs-provisioner-sa") {
metadata+: env.metadata,
},
csiRbdPlugin: kube.ServiceAccount("rook-csi-rbd-plugin-sa") {
metadata+: env.metadata,
},
csiRbdProvisioner: kube.ServiceAccount("rook-csi-rbd-provisioner-sa") {
metadata+: env.metadata,
},
},
crs: {
clusterMgmt: kube.ClusterRole("rook-ceph-cluster-mgmt") {
metadata+: env.metadata { namespace:: null },
rules: [
{
apiGroups: ["", "apps", "extensions"],
resources: ["secrets", "pods", "pods/log", "services", "configmaps", "deployments", "daemonsets"],
verbs: ["get", "list", "watch", "patch", "create", "update", "delete"],
},
],
},
global: kube.ClusterRole("rook-ceph-global") {
metadata+: env.metadata {
namespace:: null,
labels+: {
"rbac.ceph.rook.io/aggregate-to-rook-ceph-global": "true",
},
},
rules: [
{
apiGroups: [""],
resources: ["pods", "nodes", "nodes/proxy", "services"],
verbs: ["get", "list", "watch"],
},
{
apiGroups: [""],
resources: ["events", "persistentvolumes", "persistentvolumeclaims", "endpoints"],
verbs: ["get", "list", "watch", "patch", "create", "update", "delete"],
},
{
apiGroups: ["storage.k8s.io"],
resources: ["storageclasses"],
verbs: ["get", "list", "watch"],
},
{
apiGroups: ["batch"],
resources: ["jobs", "cronjobs"],
verbs: ["get", "list", "watch", "create", "update", "delete"],
},
{
apiGroups: ["ceph.rook.io"],
resources: ["*"],
verbs: ["*"],
},
{
apiGroups: ["rook.io"],
resources: ["*"],
verbs: ["*"],
},
{
apiGroups: ["policy", "apps", "extensions"],
resources: ["poddisruptionbudgets", "deployments", "replicasets"],
verbs: ["*"],
},
{
apiGroups: ["healthchecking.openshift.io"],
resources: ["machinedisruptionbudgets"],
verbs: ["get", "list", "watch", "create", "update", "delete"],
},
{
apiGroups: ["machine.openshift.io"],
resources: ["machines"],
verbs: ["get", "list", "watch", "create", "update", "delete"],
},
{
apiGroups: ["storage.k8s.io"],
resources: ["csidrivers"],
verbs: ["create", "delete", "get", "update"],
},
{
apiGroups: ["k8s.cni.cncf.io"],
resources: ["network-attachment-definitions"],
verbs: ["get"],
},
],
},
mgrCluster: kube.ClusterRole("rook-ceph-mgr-cluster") {
metadata+: env.metadata { namespace:: null },
rules: [
{
apiGroups: [""],
resources: ["configmaps", "nodes", "nodes/proxy"],
verbs: ["get", "list", "watch"],
},
{
apiGroups: [""],
resources: ["events"],
verbs: ["create", "patch", "list", "get", "watch"],
},
]
},
objectBucket: kube.ClusterRole("rook-ceph-object-bucket") {
metadata+: env.metadata {
namespace:: null,
},
rules: [
{
apiGroups: [""],
resources: ["secrets", "configmaps"],
verbs: ["*"],
},
{
apiGroups: ["storage.k8s.io"],
resources: ["storageclasses"],
verbs: ["get", "list", "watch"],
},
{
apiGroups: ["objectbucket.io"],
resources: ["*"],
verbs: ["*"],
},
],
},
cephfsCSINodeplugin: kube.ClusterRole("cephfs-csi-nodeplugin") {
metadata+: env.metadata { namespace:: null },
rules: [
{
apiGroups: [""],
resources: ["nodes"],
verbs: ["get", "list", "update"],
},
{
apiGroups: [""],
resources: ["namespaces"],
verbs: ["get", "list"],
},
{
apiGroups: [""],
resources: ["persistentvolumes"],
verbs: ["get", "list", "watch", "update"],
},
{
apiGroups: ["storage.k8s.io"],
resources: ["volumeattachments"],
verbs: ["get", "list", "watch", "update"],
},
{
apiGroups: [""],
resources: ["configmaps"],
verbs: ["get", "list"],
},
],
},
cephfsExternalProvisionerRunner: kube.ClusterRole("cephfs-external-provisioner-runner") {
metadata+: env.metadata {
namespace:: null,
labels+: {
"rbac.ceph.rook.io/aggregate-to-cephfs-external-provisioner-runner": "true",
},
},
rules: [
{
apiGroups: [""],
resources: ["secrets"],
verbs: ["get", "list"],
},
{
apiGroups: [""],
resources: ["persistentvolumes"],
verbs: ["get", "list", "watch", "create", "delete", "update", "patch"],
},
{
apiGroups: [""],
resources: ["persistentvolumeclaims"],
verbs: ["get", "list", "watch", "update"],
},
{
apiGroups: ["storage.k8s.io"],
resources: ["storageclasses"],
verbs: ["get", "list", "watch"],
},
{
apiGroups: [""],
resources: ["events"],
verbs: ["list", "watch", "create", "update", "patch"],
},
{
apiGroups: ["snapshot.storage.k8s.io"],
resources: ["volumesnapshots"],
verbs: ["get", "list", "watch", "update"],
},
{
apiGroups: ["snapshot.storage.k8s.io"],
resources: ["volumesnapshotcontents"],
verbs: ["create", "get", "list", "watch", "update", "delete"],
},
{
apiGroups: ["snapshot.storage.k8s.io"],
resources: ["volumesnapshotclasses"],
verbs: ["get", "list", "watch"],
},
{
apiGroups: ["snapshot.storage.k8s.io"],
resources: ["volumesnapshotcontents/status"],
verbs: ["update"],
},
{
apiGroups: ["apiextensions.k8s.io"],
resources: ["customresourcedefinitions"],
verbs: ["create", "list", "watch", "delete", "get", "update"],
},
{
apiGroups: ["snapshot.storage.k8s.io"],
resources: ["volumesnapshots/status"],
verbs: ["update"],
},
{
apiGroups: ["storage.k8s.io"],
resources: ["volumeattachments"],
verbs: ["get", "list", "watch", "update", "patch"],
},
{
apiGroups: ["storage.k8s.io"],
resources: ["volumeattachments/status"],
verbs: ["patch"],
},
{
apiGroups: [""],
resources: ["nodes"],
verbs: ["get", "list", "watch"],
},
{
apiGroups: [""],
resources: ["persistentvolumeclaims/status"],
verbs: ["update", "patch"],
},
],
},
rbdCSINodeplugin: kube.ClusterRole("rbd-csi-nodeplugin") {
metadata+: env.metadata { namespace:: null },
rules: [
{
apiGroups: [""],
resources: ["secrets"],
verbs: ["get", "list"],
},
{
apiGroups: [""],
resources: ["nodes"],
verbs: ["get", "list", "update"],
},
{
apiGroups: [""],
resources: ["namespaces"],
verbs: ["get", "list"],
},
{
apiGroups: [""],
resources: ["persistentvolumes"],
verbs: ["get", "list", "watch", "update"],
},
{
apiGroups: ["storage.k8s.io"],
resources: ["volumeattachments"],
verbs: ["get", "list", "watch", "update"],
},
{
apiGroups: [""],
resources: ["configmaps"],
verbs: ["get", "list"],
},
],
},
rbdExternalProvisionerRunner: kube.ClusterRole("rbd-external-provisioner-runner") {
metadata+: env.metadata {
namespace:: null,
labels+: {
"rbac.ceph.rook.io/aggregate-to-rbd-external-provisioner-runner": "true",
},
},
rules: [
{
apiGroups: [""],
resources: ["secrets"],
verbs: ["get", "list", "watch"],
},
{
apiGroups: [""],
resources: ["persistentvolumes"],
verbs: ["get", "list", "watch", "create", "delete", "update", "patch"],
},
{
apiGroups: [""],
resources: ["persistentvolumeclaims"],
verbs: ["get", "list", "watch", "update"],
},
{
apiGroups: ["storage.k8s.io"],
resources: ["volumeattachments"],
verbs: ["get", "list", "watch", "update", "patch"],
},
{
apiGroups: ["storage.k8s.io"],
resources: ["volumeattachments/status"],
verbs: ["patch"],
},
{
apiGroups: [""],
resources: ["nodes"],
verbs: ["get", "list", "watch"],
},
{
apiGroups: ["storage.k8s.io"],
resources: ["storageclasses"],
verbs: ["get", "list", "watch"]
},
{
apiGroups: [""],
resources: ["events"],
verbs: ["list", "watch", "create", "update", "patch"],
},
{
apiGroups: ["snapshot.storage.k8s.io"],
resources: ["volumesnapshots"],
verbs: ["get", "list", "watch", "update"],
},
{
apiGroups: ["snapshot.storage.k8s.io"],
resources: ["volumesnapshotcontents"],
verbs: ["create", "get", "list", "watch", "update", "delete"],
},
{
apiGroups: ["snapshot.storage.k8s.io"],
resources: ["volumesnapshotclasses"],
verbs: ["get", "list", "watch"],
},
{
apiGroups: ["snapshot.storage.k8s.io"],
resources: ["volumesnapshotcontents/status"],
verbs: ["update"],
},
{
apiGroups: ["apiextensions.k8s.io"],
resources: ["customresourcedefinitions"],
verbs: ["create", "list", "watch", "delete", "get", "update"],
},
{
apiGroups: ["snapshot.storage.k8s.io"],
resources: ["volumesnapshots/status"],
verbs: ["update"],
},
{
apiGroups: [""],
resources: ["persistentvolumeclaims/status"],
verbs: ["update", "patch"],
},
{
apiGroups: [""],
resources: ["configmaps"],
verbs: ["get"],
},
{
apiGroups: ["replication.storage.openshift.io"],
resources: ["volumereplications", "volumereplicationclasses"],
verbs: ["create", "delete", "get", "list", "patch", "update", "watch"],
},
{
apiGroups: ["replication.storage.openshift.io"],
resources: ["volumereplications/finalizers"],
verbs: ["update"],
},
{
apiGroups: ["replication.storage.openshift.io"],
resources: ["volumereplications/status"],
verbs: ["get", "patch", "update"],
},
{
apiGroups: ["replication.storage.openshift.io"],
resources: ["volumereplicationclasses/status"],
verbs: ["get"],
},
],
},
},
crbs: {
global: kube.ClusterRoleBinding("ceph-rook-global") {
metadata+: env.metadata { namespace:: null },
roleRef_: env.crs.global,
subjects_: [env.sa.system],
},
objectBucket: kube.ClusterRoleBinding("rook-ceph-object-bucket") {
metadata+: env.metadata { namespace:: null },
roleRef_: env.crs.objectBucket,
subjects_: [env.sa.system],
},
cephfsCSINodeplugin: kube.ClusterRoleBinding("cephfs-csi-nodeplugin") {
metadata+: env.metadata { namespace:: null },
roleRef_: env.crs.cephfsCSINodeplugin,
subjects_: [env.sa.csiCephfsPlugin],
},
cephfsCSIProvisioner: kube.ClusterRoleBinding("cephfs-csi-provisioner") {
metadata+: env.metadata { namespace:: null },
roleRef_: env.crs.cephfsExternalProvisionerRunner,
subjects_: [env.sa.csiCephfsProvisioner],
},
rbdCSINodeplugin: kube.ClusterRoleBinding("rbd-csi-nodeplugin") {
metadata+: env.metadata { namespace:: null },
roleRef_: env.crs.rbdCSINodeplugin,
subjects_: [env.sa.csiRbdPlugin],
},
rbdCSIProvisioner: kube.ClusterRoleBinding("rbd-csi-provisioner") {
metadata+: env.metadata { namespace:: null },
roleRef_: env.crs.rbdExternalProvisionerRunner,
subjects_: [env.sa.csiRbdProvisioner],
},
},
roles: {
system: kube.Role("ceph-rook-system") {
metadata+: env.metadata,
rules: [
{
apiGroups: [""],
resources: ["pods", "configmaps", "services"],
verbs: ["get", "list", "watch", "patch", "create", "update", "delete"],
},
{
apiGroups: ["apps"],
resources: ["daemonsets", "statefulsets", "deployments"],
verbs: ["get", "list", "watch", "create", "update", "delete"],
},
{
apiGroups: ["k8s.cni.cncf.io"],
resources: ["network-attachment-definitions"],
verbs: ["get"],
},
],
},
cephfsExternalProvisioner: kube.Role("cephfs-external-provisioner-cfg") {
metadata+: env.metadata,
rules: [
{
apiGroups: [""],
resources: ["endpoints"],
verbs: ["get", "watch", "list", "delete", "update", "create"],
},
{
apiGroups: [""],
resources: ["configmaps"],
verbs: ["get", "list", "create", "delete"],
},
{
apiGroups: ["coordination.k8s.io"],
resources: ["leases"],
verbs: ["get" ,"watch", "list", "delete", "update", "create"],
},
],
},
rbdExternalProvisioner: kube.Role("rbd-external-provisioner-cfg") {
metadata+: env.metadata,
rules: [
{
apiGroups: [""],
resources: ["endpoints"],
verbs: ["get", "watch", "list", "delete", "update", "create"],
},
{
apiGroups: [""],
resources: ["configmaps"],
verbs: ["get", "list", "watch", "create", "delete", "update"],
},
{
apiGroups: ["coordination.k8s.io"],
resources: ["leases"],
verbs: ["get" ,"watch", "list", "delete", "update", "create"],
},
],
},
},
rbs: {
system: kube.RoleBinding("ceph-rook-system") {
metadata+: env.metadata,
roleRef_: env.roles.system,
subjects_: [env.sa.system],
},
cephfsCSIProvisioner: kube.RoleBinding("cephfs-csi-provisioner-role-cfg") {
metadata+: env.metadata,
roleRef_: env.roles.cephfsExternalProvisioner,
subjects_: [env.sa.csiCephfsProvisioner],
},
rbdCSIProvisioner: kube.RoleBinding("rbd-csi-provisioner-role-cfg") {
metadata+: env.metadata,
roleRef_: env.roles.rbdExternalProvisioner,
subjects_: [env.sa.csiRbdProvisioner],
},
},
operator: kube.Deployment("rook-ceph-operator") {
metadata+: env.metadata,
spec+: {
template+: {
spec+: {
serviceAccountName: env.sa.system.metadata.name,
containers_: {
operator: kube.Container("rook-ceph-operator") {
image: cfg.image,
args: ["ceph", "operator"],
volumeMounts_: {
"rook-config": { mountPath: "/var/lib/rook" },
"default-config-dir": { mountPath: "/etc/ceph" },
},
env_: {
LIB_MODULES_DIR_PATH: "/run/current-system/kernel-modules/lib/modules/",
ROOK_ALLOW_MULTIPLE_FILESYSTEMS: "false",
ROOK_LOG_LEVEL: "INFO",
ROOK_MON_HEALTHCHECK_INTERVAL: "45s",
ROOK_MON_OUT_TIMEOUT: "600s",
ROOK_DISCOVER_DEVICES_INTERVAL: "60m",
ROOK_HOSTPATH_REQUIRES_PRIVILEGED: "false",
ROOK_ENABLE_SELINUX_RELABELING: "true",
ROOK_ENABLE_FSGROUP: "true",
NODE_NAME: kube.FieldRef("spec.nodeName"),
POD_NAME: kube.FieldRef("metadata.name"),
POD_NAMESPACE: kube.FieldRef("metadata.namespace"),
ROOK_CSI_KUBELET_DIR_PATH: "/var/lib/kubernetes",
ROOK_ENABLE_FLEX_DRIVER: "true",
},
},
},
volumes_: {
"rook-config": { emptyDir: {} },
"default-config-dir": { emptyDir: {} },
},
},
},
},
},
},
// Create a new Ceph cluster in a new namespace.
Cluster(operator, name):: {
local cluster = self,
spec:: error "please define cluster spec",
metadata:: {
namespace: name,
},
name(suffix):: cluster.metadata.namespace + "-" + suffix,
namespace: kube.Namespace(cluster.metadata.namespace),
sa: {
// service accounts need to be hardcoded, see operator source.
osd: kube.ServiceAccount("rook-ceph-osd") {
metadata+: cluster.metadata,
},
mgr: kube.ServiceAccount("rook-ceph-mgr") {
metadata+: cluster.metadata,
},
cmdReporter: kube.ServiceAccount("rook-ceph-cmd-reporter") {
metadata+: cluster.metadata,
},
},
roles: {
osd: kube.Role(cluster.name("osd")) {
metadata+: cluster.metadata,
rules: [
{
apiGroups: [""],
resources: ["configmaps"],
verbs: ["get", "list", "watch", "create", "update", "delete"],
},
],
},
osdCluster: kube.ClusterRole(cluster.name("osd-cluster")) {
metadata+: cluster.metadata { namespace:: null },
rules: [
{
apiGroups: [""],
resources: ["nodes"],
verbs: ["get", "list"],
},
],
},
mgr: kube.Role(cluster.name("mgr")) {
metadata+: cluster.metadata,
rules: [
{
apiGroups: [""],
resources: ["pods", "services"],
verbs: ["get", "list", "watch"],
},
{
apiGroups: ["batch"],
resources: ["jobs"],
verbs: ["get", "list", "watch", "create", "update", "delete"],
},
{
apiGroups: ["ceph.rook.io"],
resources: ["*"],
verbs: ["*"],
},
],
},
cmdReporter: kube.Role(cluster.name("cmd-reporter")) {
metadata+: cluster.metadata,
rules: [
{
apiGroups: [""],
resources: ["pods", "configmaps"],
verbs: ["get", "list", "watch", "create", "update", "delete"],
},
],
},
mgrSystem: kube.ClusterRole(cluster.name("mgr-system")) {
metadata+: cluster.metadata { namespace:: null },
rules: [
{
apiGroups: [""],
resources: ["configmaps"],
verbs: ["get", "list", "watch"],
}
],
},
},
rbs: [
kube.RoleBinding(cluster.name(el.name)) {
metadata+: cluster.metadata,
roleRef_: el.role,
subjects_: [el.sa],
},
for el in [
// Allow Operator SA to perform Cluster Mgmt in this namespace.
{ name: "cluster-mgmt", role: operator.crs.clusterMgmt, sa: operator.sa.system },
{ name: "osd", role: cluster.roles.osd, sa: cluster.sa.osd },
{ name: "mgr", role: cluster.roles.mgr, sa: cluster.sa.mgr },
{ name: "cmd-reporter", role: cluster.roles.cmdReporter, sa: cluster.sa.cmdReporter },
{ name: "mgr-cluster", role: operator.crs.mgrCluster, sa: cluster.sa.mgr },
]
],
mgrSystemRB: kube.RoleBinding(cluster.name("mgr-system")) {
metadata+: {
namespace: operator.cfg.namespace,
},
roleRef_: cluster.roles.mgrSystem,
subjects_: [cluster.sa.mgr],
},
osdClusterRB: kube.ClusterRoleBinding(cluster.name("osd-cluster")) {
metadata+: {
namespace:: null,
},
roleRef_: cluster.roles.osdCluster,
subjects_: [cluster.sa.osd],
},
cluster: kube._Object("ceph.rook.io/v1", "CephCluster", name) {
metadata+: cluster.metadata,
spec: {
cephVersion: {
image: "quay.io/ceph/ceph:v16.2.5",
allowUnsupported: true,
},
dataDirHostPath: if name == "ceph-waw2" then "/var/lib/rook" else "/var/lib/rook-%s" % [name],
dashboard: {
ssl: false,
enabled: true,
port: 8080,
},
} + cluster.spec,
},
dashboardService: kube.Service(cluster.name("dashboard")) {
metadata+: cluster.metadata,
spec: {
ports: [
{ name: "dashboard", port: 80, targetPort: 8080, protocol: "TCP" },
],
selector: {
app: "rook-ceph-mgr",
rook_cluster: name,
},
type: "ClusterIP",
},
},
dashboardIngress: kube.SimpleIngress(cluster.name("dashboard")) {
hosts:: ["%s.hswaw.net" % name],
target_service:: cluster.dashboardService,
},
# Benji is a backup tool, external to rook, that we use for backing up
# RBDs.
benji: {
sa: kube.ServiceAccount(cluster.name("benji")) {
metadata+: cluster.metadata,
},
cr: kube.ClusterRole(cluster.name("benji")) {
rules: [
{
apiGroups: [""],
resources: [
"persistentvolumes",
"persistentvolumeclaims"
],
verbs: ["list", "get"],
},
{
apiGroups: [""],
resources: [
"events",
],
verbs: ["create", "update"],
},
],
},
crb: kube.ClusterRoleBinding(cluster.name("benji")) {
roleRef_: cluster.benji.cr,
subjects_: [cluster.benji.sa],
},
config: kube.Secret(cluster.name("benji-config")) {
metadata+: cluster.metadata,
data_: {
"benji.yaml": std.manifestJson({
configurationVersion: '1',
databaseEngine: 'sqlite:////data/benji.sqlite',
defaultStorage: 'wasabi',
storages: [
{
name: "wasabi",
storageId: 1,
module: "s3",
configuration: cluster.spec.benji.s3Configuration {
activeTransforms: ["encrypt"],
},
},
],
transforms: [
{
name: "encrypt",
module: "aes_256_gcm",
configuration: {
# not secret.
kdfSalt: "T2huZzZpcGhhaWM3QWVwaDhybzRhaDNhbzFpc2VpOWFobDNSZWVQaGVvTWV1bmVaYWVsNHRoYWg5QWVENHNoYWg0ZGFoN3Rlb3NvcHVuZzNpZXZpMm9vTG9vbmc1YWlmb0RlZXAwYmFobDlab294b2hjaG9odjRhbzFsYWkwYWk=",
kdfIterations: 2137,
password: cluster.spec.benji.encryptionPassword,
},
},
],
ios: [
{ name: pool, module: "rbd" }
for pool in cluster.spec.benji.pools
],
}),
},
},
# Yes, Benji keeps data (backup metadata) on the ceph cluster that
# it backs up. However:
# - we add a command to benji-k8s to also copy over the sqlite
# database over to s3
# - benji can, in a pinch, restore without a database if a version
# is known: https://benji-backup.me/restore.html#restoring-without-a-database
data: kube.PersistentVolumeClaim(cluster.name("benji-data")) {
metadata+: cluster.metadata,
spec+: {
storageClassName: cluster.spec.benji.metadataStorageClass,
accessModes: [ "ReadWriteOnce" ],
resources: {
requests: {
storage: "1Gi",
},
},
},
},
# Extra scripts.
extrabins: kube.ConfigMap(cluster.name("benji-extrabins")) {
metadata+: cluster.metadata,
data: {
"metabackup.sh" : |||
# Make backups of sqlite3 metadata used by Benji.
# The backups live in the same bucket as backups, and the metabackups
# are named `metabackup-0..10`, where 0 is the newest backup. Any time
# this script is called, backups get shifted one way to the left (9 to 10,
# 8 to 9, etc). This ensures we have at least 10 backup replicas.
set -e
which s3cmd || pip install --upgrade s3cmd
AWS_ACCESS_KEY_ID=$(jq -r .storages[0].configuration.awsAccessKeyId < /etc/benji/benji.yaml)
AWS_SECRET_ACCESS_KEY=$(jq -r .storages[0].configuration.awsSecretAccessKey < /etc/benji/benji.yaml)
BUCKET=$(jq -r .storages[0].configuration.bucketName < /etc/benji/benji.yaml)
s3() {
s3cmd --host=s3.wasabisys.com \
"--host-bucket=%(bucket)s.s3.wasabisys.com" \
--region=eu-central-1 \
--access_key=$AWS_ACCESS_KEY_ID \
--secret_key=$AWS_SECRET_ACCESS_KEY \
"$@"
}
# Copy over old backups, if they exist.
for i in `seq 9 -1 0`; do
from="s3://$BUCKET/metabackup-$i.sqlite3"
to="s3://$BUCKET/metabackup-$((i+1)).sqlite3"
if [[ $(s3 ls $from | wc -l) -eq 0 ]]; then
echo "$from does not exist, skipping shift."
continue
fi
echo "Moving $from to $to..."
s3 mv $from $to
done
# Make new metabackup.
s3 put /data/benji.sqlite s3://$BUCKET/metabackup-0.sqlite3
|||,
"get-rook-creds.sh": |||
# Based on the Rook Toolbox /usr/local/bin/toolbox.sh script.
# Copyright 2016 The Rook Authors. All rights reserved.
CEPH_CONFIG="/etc/ceph/ceph.conf"
MON_CONFIG="/etc/rook/mon-endpoints"
KEYRING_FILE="/etc/ceph/keyring"
# create a ceph config file in its default location so ceph/rados tools can be used
# without specifying any arguments
write_endpoints() {
endpoints=$(cat ${MON_CONFIG})
# filter out the mon names
mon_endpoints=$(echo ${endpoints} | sed 's/[a-z]\+=//g')
# filter out the legacy mon names
mon_endpoints=$(echo ${mon_endpoints} | sed 's/rook-ceph-mon[0-9]\+=//g')
DATE=$(date)
echo "$DATE writing mon endpoints to ${CEPH_CONFIG}: ${endpoints}"
cat <<EOF > ${CEPH_CONFIG}
[global]
mon_host = ${mon_endpoints}
[client.admin]
keyring = ${KEYRING_FILE}
EOF
}
# watch the endpoints config file and update if the mon endpoints ever change
watch_endpoints() {
# get the timestamp for the target of the soft link
real_path=$(realpath ${MON_CONFIG})
initial_time=$(stat -c %Z ${real_path})
while true; do
real_path=$(realpath ${MON_CONFIG})
latest_time=$(stat -c %Z ${real_path})
if [[ "${latest_time}" != "${initial_time}" ]]; then
write_endpoints
initial_time=${latest_time}
fi
sleep 10
done
}
# create the keyring file
cat <<EOF > ${KEYRING_FILE}
[client.admin]
key = ${ROOK_ADMIN_SECRET}
EOF
# write the initial config file
write_endpoints
# continuously update the mon endpoints if they fail over
watch_endpoints &
|||
},
},
cronjob: kube.CronJob(cluster.name("benji")) {
metadata+: cluster.metadata,
spec+: { # CronJob Spec
schedule: "42 0 * * *", # Daily at 42 minute past midnight.
jobTemplate+: {
spec+: { # Job Spec
selector:: null,
template+: {
spec+: { # PodSpec
serviceAccountName: cluster.benji.sa.metadata.name,
containers_: {
benji: kube.Container(cluster.name("benji")) {
image: "ghcr.io/elemental-lf/benji-k8s@sha256:8ab74b210bfdf6ebbf0017144b34fb840ad9ad225edf27dd3394fbd8332e649d",
volumeMounts_: {
extrabins: { mountPath: "/usr/local/extrabins" },
monendpoints: { mountPath: "/etc/rook" },
benjiconfig: { mountPath: "/etc/benji" },
data: { mountPath: "/data" },
},
env_: {
ROOK_ADMIN_SECRET: { secretKeyRef: { name: "rook-ceph-mon", key: "admin-secret" }},
},
command: [
"bash", "-c", |||
bash /usr/local/extrabins/get-rook-creds.sh
benji-backup-pvc
benji-command enforce latest3,hours48,days7,months12
benji-command cleanup
bash /usr/local/extrabins/metabackup.sh
|||
],
},
},
volumes_: {
data: kube.PersistentVolumeClaimVolume(cluster.benji.data),
benjiconfig: kube.SecretVolume(cluster.benji.config),
extrabins: kube.ConfigMapVolume(cluster.benji.extrabins),
monendpoints: {
configMap: {
name: "rook-ceph-mon-endpoints",
items: [
{ key: "data", path: "mon-endpoints" },
],
},
},
},
},
},
},
},
},
},
},
},
ReplicatedBlockPool(cluster, name):: {
local pool = self,
name:: name,
spec:: error "spec must be specified",
pool: kube._Object("ceph.rook.io/v1", "CephBlockPool", name) {
metadata+: cluster.metadata,
spec: pool.spec,
},
storageClass: kube.StorageClass(name) {
provisioner: "ceph.rook.io/block",
parameters: {
blockPool: pool.pool.metadata.name,
clusterNamespace: pool.pool.metadata.namespace,
fstype: "ext4",
},
reclaimPolicy: "Retain",
},
},
ECBlockPool(cluster, name):: {
local pool = self,
name:: name,
metadataReplicas:: 3,
spec:: error "spec must be specified",
pool: kube._Object("ceph.rook.io/v1", "CephBlockPool", name) {
metadata+: cluster.metadata,
spec: pool.spec,
},
metapool: kube._Object("ceph.rook.io/v1", "CephBlockPool", name + "-metadata") {
metadata+: cluster.metadata,
spec: {
failureDomain: "host",
replicated: {
size: pool.metadataReplicas,
},
},
},
storageClass: kube.StorageClass(name) {
provisioner: "ceph.rook.io/block",
parameters: {
blockPool: pool.metapool.metadata.name,
dataBlockPool: pool.pool.metadata.name,
clusterNamespace: pool.pool.metadata.namespace,
fstype: "ext4",
},
reclaimPolicy: "Retain",
},
},
// This is a rook CephObjectRealm which corresponds to a radosgw realm.
//
// A realm is a 'world' of radosgw user-facing metadata, like credentials,
// buckets, and underlying structures like zones and zonegroups. A realm
// contains zonegroups and zones, but a single Ceph cluster can actually
// serve multiple realms, by running multiple radosgw instances.
S3ObjectRealm(cluster, name):: {
cluster:: cluster,
realm: kube._Object("ceph.rook.io/v1", "CephObjectRealm", name) {
metadata+: cluster.metadata,
},
},
// This is a rook CephObjectZoneGroup which corresponds to a radosgw
// zonegroup.
//
// A zonegroup contains zones, and zones within a zonegroup will serve a
// concise view of objects in buckets, and will sync between eachother to
// eventually contain the same data.
//
// A single zonegroup within a realm must be a 'master' zonegroup, and will
// then hold and replicate the metadata of this realm. All realm operations
// via radosgw-admin must be performed within the master zonegroup.
S3ObjectZoneGroup(realm, name):: {
realm:: realm,
zonegroup: kube._Object("ceph.rook.io/v1", "CephObjectZoneGroup", name) {
metadata+: realm.cluster.metadata,
spec+: {
realm: realm.realm.metadata.name,
},
},
},
// This is a CephObjectZone but also a CephObjectStore.
//
// Rook attempts to hide away Ceph's radosgw multisite structures
// (realm/zonegroup/zone) by presenting a single CRD named
// 'CephObjectStore'. When such a resource is created, Rook will create a
// realm, zonegroup and zone under the hood, as a radosgw zone is required
// to serve data, and a radosgw zone cannot exist without a zonegroup, and
// a radosgw zonegroup cannot exist without a realm.
//
// However, rook also exposes the lower-level API by letting the user
// specify 'zone' in the ObjectStore's spec, which should point to a
// CephObjectZone. Then, an entirely different reconciliation codepath is
// taken and instead users are expected to manage
// CephObject{Realm,ZoneGroup,Zone} manually at Ceph's native abstraction
// level.
//
// CephObjectStore not only represents a Ceph zone (and possibly
// zonegroup/realm), but also pods and services that are required to servev
// radosgw data publicly. That's why S3ObjectStore takes parameters like
// 'public port' and 'instance number'.
//
// To add to the confusion, our S3ObjectStore wrapper also sprinkles in an
// Ingress with TLS to terminate the above service, and automatically
// creates a CephObjectZone.
//
// This whole jsonent abstraction basically forces users to manually create
// realms and zonegroups, but makes it very easy to do so. By forcing these
// to be explicitly created by rook objects, only the 'multi-site'
// reconciliation codepath is taken in rook, making things a bit more
// predictable.
S3ObjectStore(zonegroup, name):: {
local store = self,
spec:: {
dataPool: error "spec.dataPool must be specified",
metadataPool: error "spec.metadataPool must be specified",
},
cfg:: {
// We want to have each rgw run under a domain corresponding to the
// zone it's running in, but also to the zonegroup it's running in.
// This will allow us to DNS loadbalance a zonegroup to be backed
// by multiple zone ingresses.
domainParts: [
zonegroup.zone.metadata.name,
zonegroup.zonegroup.metadata.name,
],
domains: [
"object.ceph-%s.hswaw.net" % [part]
for part in cfg.domainParts
],
},
local cfg = self.cfg,
zone: kube._Object("ceph.rook.io/v1", "CephObjectZone", name) {
metadata+: zonegroup.realm.cluster.metadata,
spec: store.spec {
zoneGroup: zonegroup.zonegroup.metadata.name,
},
},
objectStore: kube._Object("ceph.rook.io/v1", "CephObjectStore", name) {
metadata+: zonegroup.realm.cluster.metadata,
spec: {
gateway: {
port: 80,
instances: 1,
allNodes: false,
},
zone: {
name: name,
},
preservePoolsOnDelete: true,
},
},
objectIngress: kube.Ingress(name) {
metadata+: zonegroup.realm.cluster.metadata {
annotations+: {
"kubernetes.io/tls-acme": "true",
"cert-manager.io/cluster-issuer": "letsencrypt-prod",
"nginx.ingress.kubernetes.io/proxy-body-size": "0",
},
},
spec+: {
tls: [
{
hosts: cfg.domains,
secretName: "%s-tls" % [name],
},
],
rules: [
{
host: domain,
http: {
paths: [
{
path: "/",
backend: {
serviceName: "rook-ceph-rgw-%s" % [name],
servicePort: 80,
},
},
]
},
}
for domain in cfg.domains
],
},
},
},
}