cluster/kube/lib/cockroachdb.libsonnet

# Deploy a 3-node CockroachDB cluster in secure mode.

# Can be used either in own namespace or in an existing one:
# crdb: cockroachdb.Cluster("q3kdb") {
#    cfg+: {
#        namespace: "q3k", // if not given, will create 'q3kdb' namespace
#    },
#},
#
# After the cluster is up, you can get to an administrateive SQL shell:
#   $ kubectl -n q3k exec -it q3kdb-client /cockroach/cockroach sql
#   root@q3kdb-cockroachdb-0.q3kdb-internal.q3k.svc.cluster.local:26257/defaultdb>
#
# Then, you can create some users and databases for applications:
#   defaultdb> CREATE DATABASE wykop;
#   defaultdb> CREATE USER bialkov PASSWORD hackme;
#   defaultdb> GRANT ALL ON DATABASE wykop to bialkov;
#
# You are then ready to access the database via the public service from your application.
#
#   PGCLIENTENCODING=utf8 psql -h q3kdb-public -p 26257 -U bialkov wykop
#   Password for user bialkov: 
#   psql (10.9 (Ubuntu 10.9-0ubuntu0.18.04.1), server 9.5.0)
#   SSL connection (protocol: TLSv1.2, cipher: ECDHE-RSA-AES128-GCM-SHA256, bits: 128, compression: off)
#   Type "help" for help.
#   
#   wykop=>


local kube = import "../../../kube/kube.libsonnet";
local cm = import "cert-manager.libsonnet";

{
    Cluster(name): {
        local cluster = self,

        cfg:: {
            image: "cockroachdb/cockroach:v19.1.0",
            namespace: null,
            ownNamespace: cluster.cfg.namespace == null,
        },

        namespaceName:: if cluster.cfg.namespace != null then cluster.cfg.namespace else name,

        metadata:: {
            namespace: cluster.namespaceName,
            labels: {
                "app.kubernetes.io/name": "cockroachdb",
                "app.kubernetes.io/managed-by": "kubecfg",
                "app.kubernetes.io/component": "cockroachdb",
            },
        },

        namespace: {
            [if cluster.cfg.ownNamespace then "ns"]: kube.Namespace(cluster.namespaceName),
        },

        name(suffix):: if cluster.cfg.ownNamespace then suffix else name + "-" + suffix,

        hosts:: ["%s-%d.%s.cluster.local" % [cluster.statefulSet.metadata.name, n, cluster.internalService.host] for n in std.range(0, cluster.statefulSet.spec.replicas)],

        pki: {
            selfSignedIssuer: cm.Issuer(cluster.name("selfsigned")) {
                metadata+: cluster.metadata,
                spec: {
                    selfSigned: {},
                },
            },

            selfSignedKeypair: cm.Certificate(cluster.name("cluster-ca")) {
                metadata+: cluster.metadata,
                spec: {
                    secretName: cluster.name("cluster-ca"),
                    duration: "43800h0m0s", // 5 years
                    isCA: true,
                    issuerRef: {
                        name: cluster.pki.selfSignedIssuer.metadata.name,
                    },
                    commonName: "cockroachdb-cluster-ca",
                },
            },

            clusterIssuer: cm.Issuer(cluster.name("cluster-ca")) {
                metadata+: cluster.metadata,
                spec: {
                    ca: {
                        secretName: cluster.pki.selfSignedKeypair.metadata.name,
                    },
                },
            },

            nodeCertificate: cm.Certificate(cluster.name("node")) {
                metadata+: cluster.metadata,
                spec: {
                    secretName: "cockroachdb-node-cert",
                    duration: "43800h0m0s", // 5 years
                    issuerRef: {
                        name: cluster.pki.clusterIssuer.metadata.name,
                    },
                    commonName: "node",
                    dnsNames: [
                        "localhost",
                        "127.0.0.1",
                        cluster.publicService.metadata.name,
                        std.join(".", [cluster.publicService.metadata.name, cluster.metadata.namespace ]),
                        std.join(".", [cluster.publicService.host, "cluster.local" ]),
                        std.join(".", [ "*", cluster.internalService.metadata.name ]),
                        std.join(".", [ "*", cluster.internalService.metadata.name, cluster.metadata.namespace ]),
                        std.join(".", [ "*", cluster.internalService.host, "cluster.local" ]),
                    ],
                },
            },

            clientCertificate: cm.Certificate(cluster.name("client")) {
                metadata+: cluster.metadata,
                spec: {
                    secretName: cluster.name("client-certificate"),
                    duration: "43800h0m0s", // 5 years
                    issuerRef: {
                        name: cluster.pki.clusterIssuer.metadata.name,
                    },
                    commonName: "root",
                },
            },
        },

        serviceAccount: kube.ServiceAccount(cluster.name("cockroachdb")) {
            metadata+: cluster.metadata,
        },

        role: kube.Role(cluster.name("cockroachdb")) {
            metadata+: cluster.metadata,
            rules: [
                {
                    apiGroups: [ "" ],
                    resources: [ "secrets" ],
                    verbs: [ "get" ],
                },
            ],
        },

        roleBinding: kube.RoleBinding(cluster.name("cockroachdb")) {
            metadata+: cluster.metadata,
            roleRef_: cluster.role,
            subjects_: [cluster.serviceAccount],
        },

        publicService: kube.Service(cluster.name("public")) {
            metadata+: cluster.metadata,
            target_pod:: cluster.statefulSet.spec.template,
            spec+: {
                ports: [
                    { name: "grpc", port: 26257, targetPort: 26257 },
                    { name: "http", port: 8080, targetPort: 8080 },
                ],
            },
        },

        internalService: kube.Service(cluster.name("internal")) {
            metadata+: cluster.metadata + {
                annotations+: {
                    "service.alpha.kubernetes.io/tolerate-unready-endpoints": "true",
                    "prometheus.io/scrape": "true",
                    "prometheus.io/path": "_status/vars",
                    "prometheus.io/port": "8080",
                },
            },
            target_pod:: cluster.statefulSet.spec.template,
            spec+: {
                ports: [
                    { name: "grpc", port: 26257, targetPort: 26257 },
                    { name: "http", port: 8080, targetPort: 8080 },
                ],
                publishNotReadyAddresses: true,
                clusterIP: "None",
            },
        },

        podDisruptionBudget: kube.PodDisruptionBudget(cluster.name("pod")) {
            metadata+: cluster.metadata,
            spec: {
                selector: {
                    matchLabels: {
                        "app.kubernetes.io/component": "cockroachdb",
                    },
                },
                maxUnavailable: 1,
            },
        },

        statefulSet: kube.StatefulSet(cluster.name("cockroachdb")) {
            metadata+: cluster.metadata {
                labels+: {
                    "app.kubernetes.io/component": "server",
                },
            },
            spec+: {
                serviceName: cluster.internalService.metadata.name,
                replicas: 3,
                template: {
                    metadata: cluster.statefulSet.metadata,
                    spec+: {
                        dnsPolicy: "ClusterFirst",
                        serviceAccountName: cluster.serviceAccount.metadata.name,
                        affinity: {
                            podAntiAffinity: {
                                preferredDuringSchedulingIgnoredDuringExecution: [
                                    {
                                        weight: 100,
                                        podAffinityTerm: {
                                            labelSelector: {
                                                matchExpressions: [
                                                    {
                                                        key: "app.kubernetes.io/component",
                                                        operator: "In",
                                                        values: [ "cockroachdb" ],
                                                    },
                                                ],
                                            },
                                            topologyKey: "kubernetes.io/hostname",
                                        },
                                    },
                                ],
                            },
                        },
                        containers: [
                            kube.Container("cockroachdb") {
                                image: cluster.cfg.image,
                                imagePullPolicy: "IfNotPresent",
                                resources: {
                                    requests: {
                                        cpu: "2",
                                        memory: "6Gi",
                                    },
                                    limits: {
                                        memory: "6Gi",
                                    },
                                },
                                ports_: {
                                    "grpc": { containerPort: 26257 },
                                    "http": { containerPort: 8080 },
                                },
                                livenessProbe: {
                                    httpGet: {
                                        path: "/health",
                                        port: "http",
                                    },
                                    initialDelaySeconds: 30,
                                    periodSeconds: 5,
                                },
                                readinessProbe: {
                                    httpGet: {
                                        path: "/health?ready=1",
                                        port: "http",
                                    },
                                    initialDelaySeconds: 10,
                                    periodSeconds: 5,
                                    failureThreshold: 2,
                                },
                                volumeMounts: [
                                    {
                                        name: "datadir",
                                        mountPath: "/cockroach/cockroach-data",
                                    },
                                    {
                                        name: "certs",
                                        mountPath: "/cockroach/cockroach-certs/node.crt",
                                        subPath: "tls.crt",
                                    },
                                    {
                                        name: "certs",
                                        mountPath: "/cockroach/cockroach-certs/node.key",
                                        subPath: "tls.key",
                                    },
                                    {
                                        name: "certs",
                                        mountPath: "/cockroach/cockroach-certs/ca.crt",
                                        subPath: "ca.crt",
                                    },
                                ],
                                env_: {
                                    "COCKROACH_CERTS_DIR": "/cockroach/cockroach-certs",
                                },
                                command: [
                                    "/bin/bash",
                                    "-ecx",
                                    "exec /cockroach/cockroach start --logtostderr --certs-dir /cockroach/cockroach-certs --advertise-host $(hostname -f) --http-addr 0.0.0.0 --cache 25% --max-sql-memory 25% --join " + std.join(",", cluster.hosts),
                                ],
                            },
                        ],
                        terminationGracePeriodSeconds: 60,
                        volumes: [
                            {
                                name: "datadir",
                                emptyDir: {},
                            },
                            {
                                name: "certs",
                                secret: {
                                    secretName: cluster.pki.nodeCertificate.spec.secretName,
                                    defaultMode: kube.parseOctal("400"),
                                },
                            },
                        ],
                    },
                },
                podManagementPolicy: "Parallel",
                updateStrategy: {
                    type: "RollingUpdate",
                },
            },
        },

        initJob: kube.Job(cluster.name("init")) {
            metadata+: cluster.metadata,
            spec: {
                template: {
                    metadata+: cluster.metadata,
                    spec+: {
                        serviceAccountName: cluster.serviceAccount.metadata.name,
                        containers: [
                            kube.Container("cluster-init") {
                                image: cluster.cfg.image,
                                imagePullPolicy: "IfNotPresent",
                                env_: {
                                    "COCKROACH_CERTS_DIR": "/cockroach/cockroach-certs",
                                },
                                command: [
                                    "/bin/bash",
                                    "-ecx",
                                    "/cockroach/cockroach init --host=" + cluster.hosts[0],
                                ],
                                volumeMounts: [
                                    {
                                        name: "certs",
                                        mountPath: "/cockroach/cockroach-certs/ca.crt",
                                        subPath: "ca.crt",
                                    },
                                    {
                                        name: "certs",
                                        mountPath: "/cockroach/cockroach-certs/client.root.crt",
                                        subPath: "tls.crt",
                                    },
                                    {
                                        name: "certs",
                                        mountPath: "/cockroach/cockroach-certs/client.root.key",
                                        subPath: "tls.key",
                                    },
                                ],
                            },
                        ],
                        restartPolicy: "OnFailure",
                        volumes: [
                            {
                                name: "certs",
                                secret: {
                                    secretName: cluster.pki.clientCertificate.spec.secretName,
                                    defaultMode: kube.parseOctal("400")
                                }
                            },
                        ],
                    },
                },
            },
        },

        clientPod: kube.Pod(cluster.name("client")) {
            metadata+: cluster.metadata {
                labels+: {
                    "app.kubernetes.io/component": "client",
                },
            },
            spec: {
                terminationGracePeriodSeconds: 5,
                containers: [
                    kube.Container("cockroachdb-client") {
                        image: cluster.cfg.image,
                        env_: {
                            "COCKROACH_CERTS_DIR": "/cockroach/cockroach-certs",
                            "COCKROACH_HOST": cluster.hosts[0],
                        },
                        command: ["sleep", "2147483648"], //(FIXME) keep the client pod running indefinitely
                        volumeMounts: [
                            {
                                name: "certs",
                                mountPath: "/cockroach/cockroach-certs/ca.crt",
                                subPath: "ca.crt",
                            },
                            {
                                name: "certs",
                                mountPath: "/cockroach/cockroach-certs/client.root.crt",
                                subPath: "tls.crt",
                            },
                            {
                                name: "certs",
                                mountPath: "/cockroach/cockroach-certs/client.root.key",
                                subPath: "tls.key",
                            },
                        ],
                    },
                ],
                volumes: [
                    {
                        name: "certs",
                        secret: {
                            secretName: cluster.pki.clientCertificate.spec.secretName,
                            defaultMode: kube.parseOctal("400")
                        }
                    },
                ],
            },
        },
    },
}