cluster/kube/lib/cockroachdb.libsonnet

# Deploy a CockroachDB cluster in secure mode.
# This creates an N-node cluster based on a given static topology.

# Can be used either in own namespace or in an existing one:
# crdb: cockroachdb.Cluster("q3kdb") {
#    cfg+: {
#        namespace: "q3k", // if not given, will create 'q3kdb' namespace
#        topology: [
#            { name: "a", node: "bc01n01.hswaw.net" },
#            { name: "b", node: "bc01n02.hswaw.net" },
#            { name: "c", node: "bc01n03.hswaw.net" },
#        ],
#        hostPath: "/var/db/cockroach-q3k",
#    },
#},
#
# After the cluster is up, you can get to an administrateive SQL shell:
#   $ NS=q3k kubectl -n $NS exec -it $(kubectl -n $NS get pods -o name | grep client- | cut -d/ -f 2) /cockroach/cockroach sql
#   root@q3kdb-cockroachdb-0.q3kdb-internal.q3k.svc.cluster.local:26257/defaultdb>
#
# Then, you can create some users and databases for applications:
#   defaultdb> CREATE DATABASE wykop;
#   defaultdb> CREATE USER bialkov PASSWORD hackme;
#   defaultdb> GRANT ALL ON DATABASE wykop TO bialkov;
#
# You are then ready to access the database via the public service from your application.
#
#   PGCLIENTENCODING=utf8 psql -h q3kdb-public -p 26257 -U bialkov wykop
#   Password for user bialkov:
#   psql (10.9 (Ubuntu 10.9-0ubuntu0.18.04.1), server 9.5.0)
#   SSL connection (protocol: TLSv1.2, cipher: ECDHE-RSA-AES128-GCM-SHA256, bits: 128, compression: off)
#   Type "help" for help.
#
#   wykop=>


local kube = import "../../../kube/kube.libsonnet";
local policies = import "../../../kube/policies.libsonnet";

{
    Cluster(name): {
        local cluster = self,

        cfg:: {
            image: "cockroachdb/cockroach:v21.1.21",

            # Must be unique per cluster.
            portServe: 26257,
            portHttp: 8080,
            hostPath: error "hostPath must be defined",
            topology: error "topology must be defined",
            clients: [],

            namespace: null,
            ownNamespace: cluster.cfg.namespace == null,
            extraDNS: [],
        },

        namespaceName:: if cluster.cfg.namespace != null then cluster.cfg.namespace else name,

        metadata:: {
            namespace: cluster.namespaceName,
            labels: {
                "app.kubernetes.io/name": "cockroachdb",
                "app.kubernetes.io/managed-by": "kubecfg",
                "app.kubernetes.io/component": "cockroachdb",
            },
        },

        namespace: {
            [if cluster.cfg.ownNamespace then "ns"]: kube.Namespace(cluster.namespaceName),
        },

        insecurePolicy: policies.AllowNamespaceInsecure(cluster.namespaceName),

        name(suffix):: if cluster.cfg.ownNamespace then suffix else name + "-" + suffix,

        pki: {
            selfSignedIssuer: kube.Issuer(cluster.name("selfsigned")) {
                metadata+: cluster.metadata,
                spec: {
                    selfSigned: {},
                },
            },

            selfSignedKeypair: kube.Certificate(cluster.name("cluster-ca")) {
                metadata+: cluster.metadata,
                spec: {
                    secretName: cluster.name("cluster-ca"),
                    duration: "43800h0m0s", // 5 years
                    isCA: true,
                    issuerRef: {
                        name: cluster.pki.selfSignedIssuer.metadata.name,
                    },
                    commonName: "cockroachdb-cluster-ca",
                },
            },

            clusterIssuer: kube.Issuer(cluster.name("cluster-ca")) {
                metadata+: cluster.metadata,
                spec: {
                    ca: {
                        secretName: cluster.pki.selfSignedKeypair.metadata.name,
                    },
                },
            },

            nodeCertificate: kube.Certificate(cluster.name("node")) {
                metadata+: cluster.metadata,
                spec: {
                    secretName: "cockroachdb-node-cert",
                    duration: "43800h0m0s", // 5 years
                    issuerRef: {
                        name: cluster.pki.clusterIssuer.metadata.name,
                    },
                    commonName: "node",
                    dnsNames: [
                        cluster.publicService.metadata.name,
                        std.join(".", [cluster.publicService.metadata.name, cluster.metadata.namespace ]),
                        cluster.publicService.host,
                        std.join(".", [cluster.publicService.host, "cluster.local" ]),
                        std.join(".", [cluster.publicService.metadata.name, cluster.metadata.namespace ]),
                    ] + [
                        "%s.cluster.local" % s.service.host
                        for s in cluster.servers
                    ] + cluster.cfg.extraDNS,
                },
            },

            clientCertificate: kube.Certificate(cluster.name("client")) {
                metadata+: cluster.metadata,
                spec: {
                    secretName: cluster.name("client-certificate"),
                    duration: "43800h0m0s", // 5 years
                    issuerRef: {
                        name: cluster.pki.clusterIssuer.metadata.name,
                    },
                    commonName: "root",
                },
            },
        },

        serviceAccount: kube.ServiceAccount(cluster.name("cockroachdb")) {
            metadata+: cluster.metadata,
        },

        role: kube.Role(cluster.name("cockroachdb")) {
            metadata+: cluster.metadata,
            rules: [
                {
                    apiGroups: [ "" ],
                    resources: [ "secrets" ],
                    verbs: [ "get" ],
                },
            ],
        },

        roleBinding: kube.RoleBinding(cluster.name("cockroachdb")) {
            metadata+: cluster.metadata,
            roleRef_: cluster.role,
            subjects_: [cluster.serviceAccount],
        },

        publicService: kube.Service(cluster.name("public")) {
            metadata+: cluster.metadata,
            target_pod:: cluster.servers[0].deploy.spec.template + {
                // no easy way to *drop* a field in jsonnet: https://github.com/google/jsonnet/issues/312
                // so hide it (the :: is critical, the null doesn't matter much)
                metadata+: {
                    labels+: {
                        // this is different for each node, so we want to *not* select on it
                        "kubernetes.hackerspace.pl/cockroachdb-server":: null,
                    }
                }
            },
            spec+: {
                ports: [
                    { name: "grpc", port: cluster.cfg.portServe, targetPort: cluster.cfg.portServe },
                    { name: "http", port: cluster.cfg.portHttp, targetPort: cluster.cfg.portHttp },
                ],
                type: "LoadBalancer",
            },
        },

        podDisruptionBudget: kube.PodDisruptionBudget(cluster.name("pod")) {
            metadata+: cluster.metadata,
            spec: {
                selector: {
                    matchLabels: {
                        "app.kubernetes.io/component": "cockroachdb",
                    },
                },
                maxUnavailable: 1,
            },
        },

        servers: [
            {
                local server = self,
                service: kube.Service(cluster.name("server-" + el.name)) {
                    metadata+: cluster.metadata + {
                        annotations+: {
                            "service.alpha.kubernetes.io/tolerate-unready-endpoints": "true",
                            "prometheus.io/scrape": "true",
                            "prometheus.io/path": "_status/vars",
                            "prometheus.io/port": std.toString(cluster.cfg.portHttp),
                        },
                    },
                    target:: server.deploy,
                    spec+: {
                        ports: [
                            { name: "grpc", port: cluster.cfg.portServe, targetPort: cluster.cfg.portServe },
                            { name: "http", port: cluster.cfg.portHttp, targetPort: cluster.cfg.portHttp },
                        ],
                        publishNotReadyAddresses: true,
                        clusterIP: "None",
                    },
                },
                deploy: kube.Deployment(cluster.name("server-" + el.name)) {
                    metadata+: cluster.metadata {
                        labels+: {
                            "app.kubernetes.io/component": "server",
                            "kubernetes.hackerspace.pl/cockroachdb-server": el.name,
                        },
                    },
                    spec+: {
                        strategy+: {
                          type: "RollingUpdate",
                          rollingUpdate: {
                            maxSurge: 0,
                            maxUnavailable: 1,
                          },
                        },
                        template+: {
                            metadata: server.deploy.metadata,
                            spec+: {
                                dnsPolicy: "ClusterFirst",
                                serviceAccountName: cluster.serviceAccount.metadata.name,
                                nodeSelector: {
                                    "kubernetes.io/hostname": el.node,
                                },
                                containers: [
                                    kube.Container("cockroachdb") {
                                        image: cluster.cfg.image,
                                        imagePullPolicy: "IfNotPresent",
                                        resources: {
                                            requests: {
                                                cpu: "2",
                                                memory: "6Gi",
                                            },
                                            limits: {
                                                memory: "6Gi",
                                            },
                                        },
                                        ports_: {
                                            "grpc": { containerPort: cluster.cfg.portServe },
                                            "http": { containerPort: cluster.cfg.portHttp },
                                        },
                                        livenessProbe: {
                                            httpGet: {
                                                path: "/health",
                                                port: "http",
                                            },
                                            initialDelaySeconds: 30,
                                            periodSeconds: 5,
                                        },
                                        readinessProbe: {
                                            httpGet: {
                                                path: "/health?ready=1",
                                                port: "http",
                                            },
                                            initialDelaySeconds: 10,
                                            periodSeconds: 5,
                                            failureThreshold: 2,
                                        },
                                        volumeMounts: [
                                            {
                                                name: "datadir",
                                                mountPath: "/cockroach/cockroach-data",
                                            },
                                            {
                                                name: "certs",
                                                mountPath: "/cockroach/cockroach-certs/node.crt",
                                                subPath: "tls.crt",
                                            },
                                            {
                                                name: "certs",
                                                mountPath: "/cockroach/cockroach-certs/node.key",
                                                subPath: "tls.key",
                                            },
                                            {
                                                name: "certs",
                                                mountPath: "/cockroach/cockroach-certs/ca.crt",
                                                subPath: "ca.crt",
                                            },
                                        ],
                                        env_: {
                                            "COCKROACH_CERTS_DIR": "/cockroach/cockroach-certs",
                                        },
                                        command: [
                                            "/cockroach/cockroach", "start",
                                            "--certs-dir", "/cockroach/cockroach-certs",
                                            "--advertise-host", "%s.cluster.local" % server.service.host,
                                            "--cache", "25%", "--max-sql-memory", "25%",
                                            "--join", std.join(",", ["%s.cluster.local:%d" % [s.service.host, cluster.cfg.portServe] for s in cluster.servers if s.service.host != server.service.host]),
                                            "--listen-addr=0.0.0.0:%d" % cluster.cfg.portServe,
                                            "--http-addr=0.0.0.0:%d" % cluster.cfg.portHttp,
                                        ],
                                    },
                                ],
                                terminationGracePeriodSeconds: 60,
                                volumes: [
                                    {
                                        name: "datadir",
                                        hostPath: {
                                            path: cluster.cfg.hostPath,
                                        },
                                    },
                                    {
                                        name: "certs",
                                        secret: {
                                            secretName: cluster.pki.nodeCertificate.spec.secretName,
                                            defaultMode: kube.parseOctal("400"),
                                        },
                                    },
                                ],
                            },
                        },
                    },
                }
            }
            for el in cluster.cfg.topology
        ],

        initJob: kube.Job(cluster.name("init")) {
            metadata+: cluster.metadata,
            spec: {
                template: {
                    metadata+: cluster.metadata,
                    spec+: {
                        serviceAccountName: cluster.serviceAccount.metadata.name,
                        containers: [
                            kube.Container("cluster-init") {
                                image: cluster.cfg.image,
                                imagePullPolicy: "IfNotPresent",
                                env_: {
                                    "COCKROACH_CERTS_DIR": "/cockroach/cockroach-certs",
                                },
                                command: [
                                    "/bin/bash",
                                    "-ecx",
                                    "/cockroach/cockroach init --host=%s.cluster.local:%d || true" % [cluster.servers[0].service.host, cluster.cfg.portServe],
                                ],
                                volumeMounts: [
                                    {
                                        name: "certs",
                                        mountPath: "/cockroach/cockroach-certs/ca.crt",
                                        subPath: "ca.crt",
                                    },
                                    {
                                        name: "certs",
                                        mountPath: "/cockroach/cockroach-certs/client.root.crt",
                                        subPath: "tls.crt",
                                    },
                                    {
                                        name: "certs",
                                        mountPath: "/cockroach/cockroach-certs/client.root.key",
                                        subPath: "tls.key",
                                    },
                                ],
                            },
                        ],
                        restartPolicy: "OnFailure",
                        volumes: [
                            {
                                name: "certs",
                                secret: {
                                    secretName: cluster.pki.clientCertificate.spec.secretName,
                                    defaultMode: kube.parseOctal("400")
                                }
                            },
                        ],
                    },
                },
            },
        },

        Client(name):: {
            certificate: kube.Certificate(cluster.name("client-%s" % name)) {
                metadata+: cluster.metadata,
                spec: {
                    secretName: cluster.name("client-%s-certificate" % name),
                    duration: "43800h0m0s", // 5 years
                    issuerRef: {
                        name: cluster.pki.clusterIssuer.metadata.name,
                    },
                    commonName: name,
                },
            },
        },

        client: kube.Deployment(cluster.name("client")) {
            metadata+: cluster.metadata {
                labels+: {
                    "app.kubernetes.io/component": "client",
                },
            },
            spec+: {
                template: {
                    metadata: cluster.client.metadata,
                    spec+: {
                        terminationGracePeriodSeconds: 5,
                        containers: [
                            kube.Container("cockroachdb-client") {
                                image: cluster.cfg.image,
                                env_: {
                                    "COCKROACH_CERTS_DIR": "/cockroach/cockroach-certs",
                                    "COCKROACH_HOST": cluster.publicService.host,
                                    "COCKROACH_PORT": std.toString(cluster.cfg.portServe),
                                },
                                command: ["sleep", "2147483648"], //(FIXME) keep the client pod running indefinitely
                                volumeMounts: [
                                    {
                                        name: "certs",
                                        mountPath: "/cockroach/cockroach-certs/ca.crt",
                                        subPath: "ca.crt",
                                    },
                                    {
                                        name: "certs",
                                        mountPath: "/cockroach/cockroach-certs/client.root.crt",
                                        subPath: "tls.crt",
                                    },
                                    {
                                        name: "certs",
                                        mountPath: "/cockroach/cockroach-certs/client.root.key",
                                        subPath: "tls.key",
                                    },
                                ],
                            },
                        ],
                        volumes: [
                            {
                                name: "certs",
                                secret: {
                                    secretName: cluster.pki.clientCertificate.spec.secretName,
                                    defaultMode: kube.parseOctal("400")
                                }
                            },
                        ],
                    },
                },
            },
        },
    },
}