cluster/kube/lib/cockroachdb.libsonnet

# Deploy a 3-node CockroachDB cluster in secure mode.

local kube = import "../../../kube/kube.libsonnet";
local cm = import "cert-manager.libsonnet";

{
    local cockroachdb = self,
    local crdb = cockroachdb,
    local cfg = crdb.cfg,

    cfg:: {
        namespace: error "namespace must be set", 
        appName: error "app name must be set",
        prefix: "", # if set, should be 'foo-',

        image: "cockroachdb/cockroach:v19.1.0",
        database: error "database name must be set",
        username: error "username must be set",
        password: error "password must be set",
    },

    makeName(suffix):: cfg.prefix + suffix,

    metadata:: {
        namespace: cfg.namespace,
        labels: {
            "app.kubernetes.io/name": cfg.appName,
            "app.kubernetes.io/managed-by": "kubecfg",
            "app.kubernetes.io/component": "cockroachdb",
        },
    },

    pki: {
        selfSignedIssuer: cm.Issuer("cockroachdb-selfsigned-issuer") {
            metadata+: crdb.metadata,
            spec: {
                selfSigned: {},
            },
        },

        selfSignedKeypair: cm.Certificate("cockroachdb-cluster-ca-keypair") {
            metadata+: crdb.metadata,
            spec: {
                secretName: "cockroachdb-cluster-ca-keypair",
                duration: "43800h0m0s", // 5 years
                isCA: true,
                issuerRef: {
                    name: crdb.pki.selfSignedIssuer.metadata.name,
                },
                commonName: "cockroachdb-cluster-ca",
            },
        },

        clusterIssuer: cm.Issuer("cockroachdb-cluster-ca") {
            metadata+: crdb.metadata,
            spec: {
                ca: {
                    secretName: crdb.pki.selfSignedKeypair.metadata.name,
                },
            },
        },

        nodeCertificate: cm.Certificate("cockroachdb-node-cert") {
            metadata+: crdb.metadata,
            spec: {
                secretName: "cockroachdb-node-cert",
                duration: "43800h0m0s", // 5 years
                issuerRef: {
                    name: crdb.pki.clusterIssuer.metadata.name,
                },
                commonName: "node",
                dnsNames: [
                    "localhost",
                    "127.0.0.1",
                    crdb.publicService.metadata.name,
                    std.join(".", [crdb.publicService.metadata.name, cfg.namespace ]),
                    std.join(".", [crdb.publicService.host, "cluster.local" ]),
                    std.join(".", [ "*", crdb.internalService.metadata.name ]),
                    std.join(".", [ "*", crdb.internalService.metadata.name, cfg.namespace ]),
                    std.join(".", [ "*", crdb.internalService.host, "cluster.local" ]),
                ],
            },
        },

        clientCertificate: cm.Certificate("cockroachdb-client-cert") {
            metadata+: crdb.metadata,
            spec: {
                secretName: "cockroachdb-client-cert",
                duration: "43800h0m0s", // 5 years
                issuerRef: {
                    name: crdb.pki.clusterIssuer.metadata.name,
                },
                commonName: "root",
            },
        },
    },

    serviceAccount: kube.ServiceAccount("cockroachdb") {
        metadata+: crdb.metadata,
    },

    role: kube.Role("cockroachdb") {
        metadata+: crdb.metadata,
        rules: [
            {
                apiGroups: [ "" ],
                resources: [ "secrets" ],
                verbs: [ "get" ],
            },
        ],
    },

    roleBinding: kube.RoleBinding("cockroachdb") {
        metadata+: crdb.metadata,
        roleRef: {
            apiGroup: "rbac.authorization.k8s.io",
            kind: "Role",
            name: "cockroachdb",
        },
        subjects: [
            {
                kind: "ServiceAccount",
                name: crdb.serviceAccount.metadata.name,
                namespace: cfg.namespace,
            },
        ],
    },

    publicService: kube.Service(crdb.makeName("cockroachdb-public")) {
        metadata+: crdb.metadata,
        target_pod:: crdb.statefulSet.spec.template,
        spec+: {
            ports: [
                { name: "grpc", port: 26257, targetPort: 26257 },
                { name: "http", port: 8080, targetPort: 8080 },
            ],
        },
    },

    internalService: kube.Service(crdb.makeName("cockroachdb")) {
        metadata+: crdb.metadata + {
            annotations+: {
                "service.alpha.kubernetes.io/tolerate-unready-endpoints": "true",
                "prometheus.io/scrape": "true",
                "prometheus.io/path": "_status/vars",
                "prometheus.io/port": "8080",
            },
        },
        target_pod:: crdb.statefulSet.spec.template,
        spec+: {
            ports: [
                { name: "grpc", port: 26257, targetPort: 26257 },
                { name: "http", port: 8080, targetPort: 8080 },
            ],
            publishNotReadyAddresses: true,
            clusterIP: "None",
        },
    },

    podDisruptionBudget: kube.PodDisruptionBudget(crdb.makeName("cockroachdb-budget")) {
        metadata+: crdb.metadata,
        spec: {
            selector: {
                matchLabels: {
                    "app.kubernetes.io/component": "cockroachdb",
                },
            },
            maxUnavailable: 1,
        },
    },

    statefulSet: kube.StatefulSet(crdb.makeName("cockroachdb")) {
        metadata+: crdb.metadata,
        spec+: {
            serviceName: crdb.internalService.metadata.name,
            replicas: 3,
            template: {
                metadata+: crdb.metadata,
                spec+: {
                    dnsPolicy: "ClusterFirst",
                    serviceAccountName: crdb.serviceAccount.metadata.name,
                    affinity: {
                        podAntiAffinity: {
                            preferredDuringSchedulingIgnoredDuringExecution: [
                                {
                                    weight: 100,
                                    podAffinityTerm: {
                                        labelSelector: {
                                            matchExpressions: [
                                                {
                                                    key: "app.kubernetes.io/component",
                                                    operator: "In",
                                                    values: [ "cockroachdb" ],
                                                },
                                            ],
                                        },
                                        topologyKey: "kubernetes.io/hostname",
                                    },
                                },
                            ],
                        },
                    },
                    containers: [
                        kube.Container("cockroachdb") {
                            image: cfg.image,
                            imagePullPolicy: "IfNotPresent",
                            resources: {
                                requests: {
                                    cpu: "2",
                                    memory: "6Gi",
                                },
                                limits: {
                                    memory: "6Gi",
                                },
                            },
                            ports_: {
                                "grpc": { containerPort: 26257 },
                                "http": { containerPort: 8080 },
                            },
                            livenessProbe: {
                                httpGet: {
                                    path: "/health",
                                    port: "http",
                                },
                                initialDelaySeconds: 30,
                                periodSeconds: 5,
                            },
                            readinessProbe: {
                                httpGet: {
                                    path: "/health?ready=1",
                                    port: "http",
                                },
                                initialDelaySeconds: 10,
                                periodSeconds: 5,
                                failureThreshold: 2,
                            },
                            volumeMounts: [
                                {
                                    name: "datadir",
                                    mountPath: "/cockroach/cockroach-data",
                                },
                                {
                                    name: "certs",
                                    mountPath: "/cockroach/cockroach-certs/node.crt",
                                    subPath: "tls.crt",
                                },
                                {
                                    name: "certs",
                                    mountPath: "/cockroach/cockroach-certs/node.key",
                                    subPath: "tls.key",
                                },
                                {
                                    name: "certs",
                                    mountPath: "/cockroach/cockroach-certs/ca.crt",
                                    subPath: "ca.crt",
                                },
                            ],
                            env_: {
                                "COCKROACH_CERTS_DIR": "/cockroach/cockroach-certs",
                            },
                            command: [
                                "/bin/bash",
                                "-ecx",
                                "exec /cockroach/cockroach start --logtostderr --certs-dir /cockroach/cockroach-certs --advertise-host $(hostname -f) --http-addr 0.0.0.0 --join cockroachdb-0.cockroachdb,cockroachdb-1.cockroachdb,cockroachdb-2.cockroachdb --cache 25% --max-sql-memory 25%",
                            ],
                        },
                    ],
                    terminationGracePeriodSeconds: 60,
                    volumes: [
                        {
                            name: "datadir",
                            emptyDir: {},
                        },
                        {
                            name: "certs",
                            secret: {
                                secretName: crdb.pki.nodeCertificate.spec.secretName,
                                defaultMode: kube.parseOctal("400"),
                            },
                        },
                    ],
                },
            },
            podManagementPolicy: "Parallel",
            updateStrategy: {
                type: "RollingUpdate",
            },
        },
    },

    initJob: kube.Job(crdb.makeName("cockroachdb-init")) {
        metadata+: crdb.metadata,
        spec: {
            template: {
                metadata+: crdb.metadata,
                spec+: {
                    serviceAccountName: crdb.serviceAccount.metadata.name,
                    initContainers: [
                        kube.Container("cluster-init") {
                            image: cfg.image,
                            imagePullPolicy: "IfNotPresent",
                            env_: {
                                "COCKROACH_CERTS_DIR": "/cockroach/cockroach-certs",
                            },
                            command: [
                                "/bin/bash",
                                "-ecx",
                                "/cockroach/cockroach init --host=cockroachdb-0.cockroachdb",
                            ],
                            volumeMounts: [
                                {
                                    name: "certs",
                                    mountPath: "/cockroach/cockroach-certs/ca.crt",
                                    subPath: "ca.crt",
                                },
                                {
                                    name: "certs",
                                    mountPath: "/cockroach/cockroach-certs/client.root.crt",
                                    subPath: "tls.crt",
                                },
                                {
                                    name: "certs",
                                    mountPath: "/cockroach/cockroach-certs/client.root.key",
                                    subPath: "tls.key",
                                },
                            ],
                        },
                    ],
                    containers: [
                        kube.Container("db-init") {
                            image: cfg.image,
                            imagePullPolicy: "IfNotPresent",
                            env_: {
                                "COCKROACH_CERTS_DIR": "/cockroach/cockroach-certs",
                                "DB_NAME": cfg.database,
                                "DB_USERNAME": cfg.username,
                                "DB_PASSWORD": cfg.password,
                            },
                            command: [
                                "/bin/bash",
                                "-ec",
                                "/cockroach/cockroach sql -e \"CREATE DATABASE ${DB_NAME}; CREATE USER ${DB_USERNAME} PASSWORD '${DB_PASSWORD}'; GRANT ALL ON DATABASE ${DB_NAME} TO ${DB_USERNAME};\" --host=cockroachdb-0.cockroachdb",
                            ],
                            volumeMounts: [
                                {
                                    name: "certs",
                                    mountPath: "/cockroach/cockroach-certs/ca.crt",
                                    subPath: "ca.crt",
                                },
                                {
                                    name: "certs",
                                    mountPath: "/cockroach/cockroach-certs/client.root.crt",
                                    subPath: "tls.crt",
                                },
                                {
                                    name: "certs",
                                    mountPath: "/cockroach/cockroach-certs/client.root.key",
                                    subPath: "tls.key",
                                },
                            ],
                        },
                    ],
                    restartPolicy: "OnFailure",
                    volumes: [
                        {
                            name: "certs",
                            secret: {
                                secretName: crdb.pki.clientCertificate.spec.secretName,
                                defaultMode: kube.parseOctal("400")
                            }
                        },
                    ],
                },
            },
        },
    },

    clientPod: kube.Pod(crdb.makeName("cockroachdb-client")) {
        metadata+: crdb.metadata,
        spec: {
            terminationGracePeriodSeconds: 5,
            containers: [
                kube.Container("cockroachdb-client") {
                    image: cfg.image,
                    env_: {
                        "COCKROACH_CERTS_DIR": "/cockroach/cockroach-certs",
                    },
                    command: ["sleep", "2147483648"], //(FIXME) keep the client pod running indefinitely
                    volumeMounts: [
                        {
                            name: "certs",
                            mountPath: "/cockroach/cockroach-certs/ca.crt",
                            subPath: "ca.crt",
                        },
                        {
                            name: "certs",
                            mountPath: "/cockroach/cockroach-certs/client.root.crt",
                            subPath: "tls.crt",
                        },
                        {
                            name: "certs",
                            mountPath: "/cockroach/cockroach-certs/client.root.key",
                            subPath: "tls.key",
                        },
                    ],
                },
            ],
            volumes: [
                {
                    name: "certs",
                    secret: {
                        secretName: crdb.pki.clientCertificate.spec.secretName,
                        defaultMode: kube.parseOctal("400")
                    }
                },
            ],
        },
    },
}