| # Deploy a 3-node CockroachDB cluster in secure mode. |
| |
| local kube = import "../../../kube/kube.libsonnet"; |
| local cm = import "cert-manager.libsonnet"; |
| |
| { |
| local cockroachdb = self, |
| local crdb = cockroachdb, |
| local cfg = crdb.cfg, |
| |
| cfg:: { |
| namespace: error "namespace must be set", |
| appName: error "app name must be set", |
| prefix: "", # if set, should be 'foo-', |
| |
| image: "cockroachdb/cockroach:v19.1.0", |
| database: error "database name must be set", |
| username: error "username must be set", |
| password: error "password must be set", |
| }, |
| |
| makeName(suffix):: cfg.prefix + suffix, |
| |
| metadata:: { |
| namespace: cfg.namespace, |
| labels: { |
| "app.kubernetes.io/name": cfg.appName, |
| "app.kubernetes.io/managed-by": "kubecfg", |
| "app.kubernetes.io/component": "cockroachdb", |
| }, |
| }, |
| |
| pki: { |
| selfSignedIssuer: cm.Issuer("cockroachdb-selfsigned-issuer") { |
| metadata+: crdb.metadata, |
| spec: { |
| selfSigned: {}, |
| }, |
| }, |
| |
| selfSignedKeypair: cm.Certificate("cockroachdb-cluster-ca-keypair") { |
| metadata+: crdb.metadata, |
| spec: { |
| secretName: "cockroachdb-cluster-ca-keypair", |
| duration: "43800h0m0s", // 5 years |
| isCA: true, |
| issuerRef: { |
| name: crdb.pki.selfSignedIssuer.metadata.name, |
| }, |
| commonName: "cockroachdb-cluster-ca", |
| }, |
| }, |
| |
| clusterIssuer: cm.Issuer("cockroachdb-cluster-ca") { |
| metadata+: crdb.metadata, |
| spec: { |
| ca: { |
| secretName: crdb.pki.selfSignedKeypair.metadata.name, |
| }, |
| }, |
| }, |
| |
| nodeCertificate: cm.Certificate("cockroachdb-node-cert") { |
| metadata+: crdb.metadata, |
| spec: { |
| secretName: "cockroachdb-node-cert", |
| duration: "43800h0m0s", // 5 years |
| issuerRef: { |
| name: crdb.pki.clusterIssuer.metadata.name, |
| }, |
| commonName: "node", |
| dnsNames: [ |
| "localhost", |
| "127.0.0.1", |
| crdb.publicService.metadata.name, |
| std.join(".", [crdb.publicService.metadata.name, cfg.namespace ]), |
| std.join(".", [crdb.publicService.host, "cluster.local" ]), |
| std.join(".", [ "*", crdb.internalService.metadata.name ]), |
| std.join(".", [ "*", crdb.internalService.metadata.name, cfg.namespace ]), |
| std.join(".", [ "*", crdb.internalService.host, "cluster.local" ]), |
| ], |
| }, |
| }, |
| |
| clientCertificate: cm.Certificate("cockroachdb-client-cert") { |
| metadata+: crdb.metadata, |
| spec: { |
| secretName: "cockroachdb-client-cert", |
| duration: "43800h0m0s", // 5 years |
| issuerRef: { |
| name: crdb.pki.clusterIssuer.metadata.name, |
| }, |
| commonName: "root", |
| }, |
| }, |
| }, |
| |
| serviceAccount: kube.ServiceAccount("cockroachdb") { |
| metadata+: crdb.metadata, |
| }, |
| |
| role: kube.Role("cockroachdb") { |
| metadata+: crdb.metadata, |
| rules: [ |
| { |
| apiGroups: [ "" ], |
| resources: [ "secrets" ], |
| verbs: [ "get" ], |
| }, |
| ], |
| }, |
| |
| roleBinding: kube.RoleBinding("cockroachdb") { |
| metadata+: crdb.metadata, |
| roleRef: { |
| apiGroup: "rbac.authorization.k8s.io", |
| kind: "Role", |
| name: "cockroachdb", |
| }, |
| subjects: [ |
| { |
| kind: "ServiceAccount", |
| name: crdb.serviceAccount.metadata.name, |
| namespace: cfg.namespace, |
| }, |
| ], |
| }, |
| |
| publicService: kube.Service(crdb.makeName("cockroachdb-public")) { |
| metadata+: crdb.metadata, |
| target_pod:: crdb.statefulSet.spec.template, |
| spec+: { |
| ports: [ |
| { name: "grpc", port: 26257, targetPort: 26257 }, |
| { name: "http", port: 8080, targetPort: 8080 }, |
| ], |
| }, |
| }, |
| |
| internalService: kube.Service(crdb.makeName("cockroachdb")) { |
| metadata+: crdb.metadata + { |
| annotations+: { |
| "service.alpha.kubernetes.io/tolerate-unready-endpoints": "true", |
| "prometheus.io/scrape": "true", |
| "prometheus.io/path": "_status/vars", |
| "prometheus.io/port": "8080", |
| }, |
| }, |
| target_pod:: crdb.statefulSet.spec.template, |
| spec+: { |
| ports: [ |
| { name: "grpc", port: 26257, targetPort: 26257 }, |
| { name: "http", port: 8080, targetPort: 8080 }, |
| ], |
| publishNotReadyAddresses: true, |
| clusterIP: "None", |
| }, |
| }, |
| |
| podDisruptionBudget: kube.PodDisruptionBudget(crdb.makeName("cockroachdb-budget")) { |
| metadata+: crdb.metadata, |
| spec: { |
| selector: { |
| matchLabels: { |
| "app.kubernetes.io/component": "cockroachdb", |
| }, |
| }, |
| maxUnavailable: 1, |
| }, |
| }, |
| |
| statefulSet: kube.StatefulSet(crdb.makeName("cockroachdb")) { |
| metadata+: crdb.metadata, |
| spec+: { |
| serviceName: crdb.internalService.metadata.name, |
| replicas: 3, |
| template: { |
| metadata+: crdb.metadata, |
| spec+: { |
| dnsPolicy: "ClusterFirst", |
| serviceAccountName: crdb.serviceAccount.metadata.name, |
| affinity: { |
| podAntiAffinity: { |
| preferredDuringSchedulingIgnoredDuringExecution: [ |
| { |
| weight: 100, |
| podAffinityTerm: { |
| labelSelector: { |
| matchExpressions: [ |
| { |
| key: "app.kubernetes.io/component", |
| operator: "In", |
| values: [ "cockroachdb" ], |
| }, |
| ], |
| }, |
| topologyKey: "kubernetes.io/hostname", |
| }, |
| }, |
| ], |
| }, |
| }, |
| containers: [ |
| kube.Container("cockroachdb") { |
| image: cfg.image, |
| imagePullPolicy: "IfNotPresent", |
| resources: { |
| requests: { |
| cpu: "2", |
| memory: "6Gi", |
| }, |
| limits: { |
| memory: "6Gi", |
| }, |
| }, |
| ports_: { |
| "grpc": { containerPort: 26257 }, |
| "http": { containerPort: 8080 }, |
| }, |
| livenessProbe: { |
| httpGet: { |
| path: "/health", |
| port: "http", |
| }, |
| initialDelaySeconds: 30, |
| periodSeconds: 5, |
| }, |
| readinessProbe: { |
| httpGet: { |
| path: "/health?ready=1", |
| port: "http", |
| }, |
| initialDelaySeconds: 10, |
| periodSeconds: 5, |
| failureThreshold: 2, |
| }, |
| volumeMounts: [ |
| { |
| name: "datadir", |
| mountPath: "/cockroach/cockroach-data", |
| }, |
| { |
| name: "certs", |
| mountPath: "/cockroach/cockroach-certs/node.crt", |
| subPath: "tls.crt", |
| }, |
| { |
| name: "certs", |
| mountPath: "/cockroach/cockroach-certs/node.key", |
| subPath: "tls.key", |
| }, |
| { |
| name: "certs", |
| mountPath: "/cockroach/cockroach-certs/ca.crt", |
| subPath: "ca.crt", |
| }, |
| ], |
| env_: { |
| "COCKROACH_CERTS_DIR": "/cockroach/cockroach-certs", |
| }, |
| command: [ |
| "/bin/bash", |
| "-ecx", |
| "exec /cockroach/cockroach start --logtostderr --certs-dir /cockroach/cockroach-certs --advertise-host $(hostname -f) --http-addr 0.0.0.0 --join cockroachdb-0.cockroachdb,cockroachdb-1.cockroachdb,cockroachdb-2.cockroachdb --cache 25% --max-sql-memory 25%", |
| ], |
| }, |
| ], |
| terminationGracePeriodSeconds: 60, |
| volumes: [ |
| { |
| name: "datadir", |
| emptyDir: {}, |
| }, |
| { |
| name: "certs", |
| secret: { |
| secretName: crdb.pki.nodeCertificate.spec.secretName, |
| defaultMode: kube.parseOctal("400"), |
| }, |
| }, |
| ], |
| }, |
| }, |
| podManagementPolicy: "Parallel", |
| updateStrategy: { |
| type: "RollingUpdate", |
| }, |
| }, |
| }, |
| |
| initJob: kube.Job(crdb.makeName("cockroachdb-init")) { |
| metadata+: crdb.metadata, |
| spec: { |
| template: { |
| metadata+: crdb.metadata, |
| spec+: { |
| serviceAccountName: crdb.serviceAccount.metadata.name, |
| initContainers: [ |
| kube.Container("cluster-init") { |
| image: cfg.image, |
| imagePullPolicy: "IfNotPresent", |
| env_: { |
| "COCKROACH_CERTS_DIR": "/cockroach/cockroach-certs", |
| }, |
| command: [ |
| "/bin/bash", |
| "-ecx", |
| "/cockroach/cockroach init --host=cockroachdb-0.cockroachdb", |
| ], |
| volumeMounts: [ |
| { |
| name: "certs", |
| mountPath: "/cockroach/cockroach-certs/ca.crt", |
| subPath: "ca.crt", |
| }, |
| { |
| name: "certs", |
| mountPath: "/cockroach/cockroach-certs/client.root.crt", |
| subPath: "tls.crt", |
| }, |
| { |
| name: "certs", |
| mountPath: "/cockroach/cockroach-certs/client.root.key", |
| subPath: "tls.key", |
| }, |
| ], |
| }, |
| ], |
| containers: [ |
| kube.Container("db-init") { |
| image: cfg.image, |
| imagePullPolicy: "IfNotPresent", |
| env_: { |
| "COCKROACH_CERTS_DIR": "/cockroach/cockroach-certs", |
| "DB_NAME": cfg.database, |
| "DB_USERNAME": cfg.username, |
| "DB_PASSWORD": cfg.password, |
| }, |
| command: [ |
| "/bin/bash", |
| "-ec", |
| "/cockroach/cockroach sql -e \"CREATE DATABASE ${DB_NAME}; CREATE USER ${DB_USERNAME} PASSWORD '${DB_PASSWORD}'; GRANT ALL ON DATABASE ${DB_NAME} TO ${DB_USERNAME};\" --host=cockroachdb-0.cockroachdb", |
| ], |
| volumeMounts: [ |
| { |
| name: "certs", |
| mountPath: "/cockroach/cockroach-certs/ca.crt", |
| subPath: "ca.crt", |
| }, |
| { |
| name: "certs", |
| mountPath: "/cockroach/cockroach-certs/client.root.crt", |
| subPath: "tls.crt", |
| }, |
| { |
| name: "certs", |
| mountPath: "/cockroach/cockroach-certs/client.root.key", |
| subPath: "tls.key", |
| }, |
| ], |
| }, |
| ], |
| restartPolicy: "OnFailure", |
| volumes: [ |
| { |
| name: "certs", |
| secret: { |
| secretName: crdb.pki.clientCertificate.spec.secretName, |
| defaultMode: kube.parseOctal("400") |
| } |
| }, |
| ], |
| }, |
| }, |
| }, |
| }, |
| |
| clientPod: kube.Pod(crdb.makeName("cockroachdb-client")) { |
| metadata+: crdb.metadata, |
| spec: { |
| terminationGracePeriodSeconds: 5, |
| containers: [ |
| kube.Container("cockroachdb-client") { |
| image: cfg.image, |
| env_: { |
| "COCKROACH_CERTS_DIR": "/cockroach/cockroach-certs", |
| }, |
| command: ["sleep", "2147483648"], //(FIXME) keep the client pod running indefinitely |
| volumeMounts: [ |
| { |
| name: "certs", |
| mountPath: "/cockroach/cockroach-certs/ca.crt", |
| subPath: "ca.crt", |
| }, |
| { |
| name: "certs", |
| mountPath: "/cockroach/cockroach-certs/client.root.crt", |
| subPath: "tls.crt", |
| }, |
| { |
| name: "certs", |
| mountPath: "/cockroach/cockroach-certs/client.root.key", |
| subPath: "tls.key", |
| }, |
| ], |
| }, |
| ], |
| volumes: [ |
| { |
| name: "certs", |
| secret: { |
| secretName: crdb.pki.clientCertificate.spec.secretName, |
| defaultMode: kube.parseOctal("400") |
| } |
| }, |
| ], |
| }, |
| }, |
| } |