| # Deploy a 3-node CockroachDB cluster in secure mode. |
| |
| # Can be used either in own namespace or in an existing one: |
| # crdb: cockroachdb.Cluster("q3kdb") { |
| # cfg+: { |
| # namespace: "q3k", // if not given, will create 'q3kdb' namespace |
| # }, |
| #}, |
| # |
| # After the cluster is up, you can get to an administrateive SQL shell: |
| # $ kubectl -n q3k exec -it q3kdb-client /cockroach/cockroach sql |
| # root@q3kdb-cockroachdb-0.q3kdb-internal.q3k.svc.cluster.local:26257/defaultdb> |
| # |
| # Then, you can create some users and databases for applications: |
| # defaultdb> CREATE DATABASE wykop; |
| # defaultdb> CREATE USER bialkov PASSWORD hackme; |
| # defaultdb> GRANT ALL ON DATABASE wykop to bialkov; |
| # |
| # You are then ready to access the database via the public service from your application. |
| # |
| # PGCLIENTENCODING=utf8 psql -h q3kdb-public -p 26257 -U bialkov wykop |
| # Password for user bialkov: |
| # psql (10.9 (Ubuntu 10.9-0ubuntu0.18.04.1), server 9.5.0) |
| # SSL connection (protocol: TLSv1.2, cipher: ECDHE-RSA-AES128-GCM-SHA256, bits: 128, compression: off) |
| # Type "help" for help. |
| # |
| # wykop=> |
| |
| |
| local kube = import "../../../kube/kube.libsonnet"; |
| local cm = import "cert-manager.libsonnet"; |
| |
| { |
| Cluster(name): { |
| local cluster = self, |
| |
| cfg:: { |
| image: "cockroachdb/cockroach:v19.1.0", |
| namespace: null, |
| ownNamespace: cluster.cfg.namespace == null, |
| }, |
| |
| namespaceName:: if cluster.cfg.namespace != null then cluster.cfg.namespace else name, |
| |
| metadata:: { |
| namespace: cluster.namespaceName, |
| labels: { |
| "app.kubernetes.io/name": "cockroachdb", |
| "app.kubernetes.io/managed-by": "kubecfg", |
| "app.kubernetes.io/component": "cockroachdb", |
| }, |
| }, |
| |
| namespace: { |
| [if cluster.cfg.ownNamespace then "ns"]: kube.Namespace(cluster.namespaceName), |
| }, |
| |
| name(suffix):: if cluster.cfg.ownNamespace then suffix else name + "-" + suffix, |
| |
| hosts:: ["%s-%d.%s.cluster.local" % [cluster.statefulSet.metadata.name, n, cluster.internalService.host] for n in std.range(0, cluster.statefulSet.spec.replicas)], |
| |
| pki: { |
| selfSignedIssuer: cm.Issuer(cluster.name("selfsigned")) { |
| metadata+: cluster.metadata, |
| spec: { |
| selfSigned: {}, |
| }, |
| }, |
| |
| selfSignedKeypair: cm.Certificate(cluster.name("cluster-ca")) { |
| metadata+: cluster.metadata, |
| spec: { |
| secretName: cluster.name("cluster-ca"), |
| duration: "43800h0m0s", // 5 years |
| isCA: true, |
| issuerRef: { |
| name: cluster.pki.selfSignedIssuer.metadata.name, |
| }, |
| commonName: "cockroachdb-cluster-ca", |
| }, |
| }, |
| |
| clusterIssuer: cm.Issuer(cluster.name("cluster-ca")) { |
| metadata+: cluster.metadata, |
| spec: { |
| ca: { |
| secretName: cluster.pki.selfSignedKeypair.metadata.name, |
| }, |
| }, |
| }, |
| |
| nodeCertificate: cm.Certificate(cluster.name("node")) { |
| metadata+: cluster.metadata, |
| spec: { |
| secretName: "cockroachdb-node-cert", |
| duration: "43800h0m0s", // 5 years |
| issuerRef: { |
| name: cluster.pki.clusterIssuer.metadata.name, |
| }, |
| commonName: "node", |
| dnsNames: [ |
| "localhost", |
| "127.0.0.1", |
| cluster.publicService.metadata.name, |
| std.join(".", [cluster.publicService.metadata.name, cluster.metadata.namespace ]), |
| std.join(".", [cluster.publicService.host, "cluster.local" ]), |
| std.join(".", [ "*", cluster.internalService.metadata.name ]), |
| std.join(".", [ "*", cluster.internalService.metadata.name, cluster.metadata.namespace ]), |
| std.join(".", [ "*", cluster.internalService.host, "cluster.local" ]), |
| ], |
| }, |
| }, |
| |
| clientCertificate: cm.Certificate(cluster.name("client")) { |
| metadata+: cluster.metadata, |
| spec: { |
| secretName: cluster.name("client-certificate"), |
| duration: "43800h0m0s", // 5 years |
| issuerRef: { |
| name: cluster.pki.clusterIssuer.metadata.name, |
| }, |
| commonName: "root", |
| }, |
| }, |
| }, |
| |
| serviceAccount: kube.ServiceAccount(cluster.name("cockroachdb")) { |
| metadata+: cluster.metadata, |
| }, |
| |
| role: kube.Role(cluster.name("cockroachdb")) { |
| metadata+: cluster.metadata, |
| rules: [ |
| { |
| apiGroups: [ "" ], |
| resources: [ "secrets" ], |
| verbs: [ "get" ], |
| }, |
| ], |
| }, |
| |
| roleBinding: kube.RoleBinding(cluster.name("cockroachdb")) { |
| metadata+: cluster.metadata, |
| roleRef_: cluster.role, |
| subjects_: [cluster.serviceAccount], |
| }, |
| |
| publicService: kube.Service(cluster.name("public")) { |
| metadata+: cluster.metadata, |
| target_pod:: cluster.statefulSet.spec.template, |
| spec+: { |
| ports: [ |
| { name: "grpc", port: 26257, targetPort: 26257 }, |
| { name: "http", port: 8080, targetPort: 8080 }, |
| ], |
| }, |
| }, |
| |
| internalService: kube.Service(cluster.name("internal")) { |
| metadata+: cluster.metadata + { |
| annotations+: { |
| "service.alpha.kubernetes.io/tolerate-unready-endpoints": "true", |
| "prometheus.io/scrape": "true", |
| "prometheus.io/path": "_status/vars", |
| "prometheus.io/port": "8080", |
| }, |
| }, |
| target_pod:: cluster.statefulSet.spec.template, |
| spec+: { |
| ports: [ |
| { name: "grpc", port: 26257, targetPort: 26257 }, |
| { name: "http", port: 8080, targetPort: 8080 }, |
| ], |
| publishNotReadyAddresses: true, |
| clusterIP: "None", |
| }, |
| }, |
| |
| podDisruptionBudget: kube.PodDisruptionBudget(cluster.name("pod")) { |
| metadata+: cluster.metadata, |
| spec: { |
| selector: { |
| matchLabels: { |
| "app.kubernetes.io/component": "cockroachdb", |
| }, |
| }, |
| maxUnavailable: 1, |
| }, |
| }, |
| |
| statefulSet: kube.StatefulSet(cluster.name("cockroachdb")) { |
| metadata+: cluster.metadata { |
| labels+: { |
| "app.kubernetes.io/component": "server", |
| }, |
| }, |
| spec+: { |
| serviceName: cluster.internalService.metadata.name, |
| replicas: 3, |
| template: { |
| metadata: cluster.statefulSet.metadata, |
| spec+: { |
| dnsPolicy: "ClusterFirst", |
| serviceAccountName: cluster.serviceAccount.metadata.name, |
| affinity: { |
| podAntiAffinity: { |
| preferredDuringSchedulingIgnoredDuringExecution: [ |
| { |
| weight: 100, |
| podAffinityTerm: { |
| labelSelector: { |
| matchExpressions: [ |
| { |
| key: "app.kubernetes.io/component", |
| operator: "In", |
| values: [ "cockroachdb" ], |
| }, |
| ], |
| }, |
| topologyKey: "kubernetes.io/hostname", |
| }, |
| }, |
| ], |
| }, |
| }, |
| containers: [ |
| kube.Container("cockroachdb") { |
| image: cluster.cfg.image, |
| imagePullPolicy: "IfNotPresent", |
| resources: { |
| requests: { |
| cpu: "2", |
| memory: "6Gi", |
| }, |
| limits: { |
| memory: "6Gi", |
| }, |
| }, |
| ports_: { |
| "grpc": { containerPort: 26257 }, |
| "http": { containerPort: 8080 }, |
| }, |
| livenessProbe: { |
| httpGet: { |
| path: "/health", |
| port: "http", |
| }, |
| initialDelaySeconds: 30, |
| periodSeconds: 5, |
| }, |
| readinessProbe: { |
| httpGet: { |
| path: "/health?ready=1", |
| port: "http", |
| }, |
| initialDelaySeconds: 10, |
| periodSeconds: 5, |
| failureThreshold: 2, |
| }, |
| volumeMounts: [ |
| { |
| name: "datadir", |
| mountPath: "/cockroach/cockroach-data", |
| }, |
| { |
| name: "certs", |
| mountPath: "/cockroach/cockroach-certs/node.crt", |
| subPath: "tls.crt", |
| }, |
| { |
| name: "certs", |
| mountPath: "/cockroach/cockroach-certs/node.key", |
| subPath: "tls.key", |
| }, |
| { |
| name: "certs", |
| mountPath: "/cockroach/cockroach-certs/ca.crt", |
| subPath: "ca.crt", |
| }, |
| ], |
| env_: { |
| "COCKROACH_CERTS_DIR": "/cockroach/cockroach-certs", |
| }, |
| command: [ |
| "/bin/bash", |
| "-ecx", |
| "exec /cockroach/cockroach start --logtostderr --certs-dir /cockroach/cockroach-certs --advertise-host $(hostname -f) --http-addr 0.0.0.0 --cache 25% --max-sql-memory 25% --join " + std.join(",", cluster.hosts), |
| ], |
| }, |
| ], |
| terminationGracePeriodSeconds: 60, |
| volumes: [ |
| { |
| name: "datadir", |
| emptyDir: {}, |
| }, |
| { |
| name: "certs", |
| secret: { |
| secretName: cluster.pki.nodeCertificate.spec.secretName, |
| defaultMode: kube.parseOctal("400"), |
| }, |
| }, |
| ], |
| }, |
| }, |
| podManagementPolicy: "Parallel", |
| updateStrategy: { |
| type: "RollingUpdate", |
| }, |
| }, |
| }, |
| |
| initJob: kube.Job(cluster.name("init")) { |
| metadata+: cluster.metadata, |
| spec: { |
| template: { |
| metadata+: cluster.metadata, |
| spec+: { |
| serviceAccountName: cluster.serviceAccount.metadata.name, |
| containers: [ |
| kube.Container("cluster-init") { |
| image: cluster.cfg.image, |
| imagePullPolicy: "IfNotPresent", |
| env_: { |
| "COCKROACH_CERTS_DIR": "/cockroach/cockroach-certs", |
| }, |
| command: [ |
| "/bin/bash", |
| "-ecx", |
| "/cockroach/cockroach init --host=" + cluster.hosts[0], |
| ], |
| volumeMounts: [ |
| { |
| name: "certs", |
| mountPath: "/cockroach/cockroach-certs/ca.crt", |
| subPath: "ca.crt", |
| }, |
| { |
| name: "certs", |
| mountPath: "/cockroach/cockroach-certs/client.root.crt", |
| subPath: "tls.crt", |
| }, |
| { |
| name: "certs", |
| mountPath: "/cockroach/cockroach-certs/client.root.key", |
| subPath: "tls.key", |
| }, |
| ], |
| }, |
| ], |
| restartPolicy: "OnFailure", |
| volumes: [ |
| { |
| name: "certs", |
| secret: { |
| secretName: cluster.pki.clientCertificate.spec.secretName, |
| defaultMode: kube.parseOctal("400") |
| } |
| }, |
| ], |
| }, |
| }, |
| }, |
| }, |
| |
| clientPod: kube.Pod(cluster.name("client")) { |
| metadata+: cluster.metadata { |
| labels+: { |
| "app.kubernetes.io/component": "client", |
| }, |
| }, |
| spec: { |
| terminationGracePeriodSeconds: 5, |
| containers: [ |
| kube.Container("cockroachdb-client") { |
| image: cluster.cfg.image, |
| env_: { |
| "COCKROACH_CERTS_DIR": "/cockroach/cockroach-certs", |
| "COCKROACH_HOST": cluster.hosts[0], |
| }, |
| command: ["sleep", "2147483648"], //(FIXME) keep the client pod running indefinitely |
| volumeMounts: [ |
| { |
| name: "certs", |
| mountPath: "/cockroach/cockroach-certs/ca.crt", |
| subPath: "ca.crt", |
| }, |
| { |
| name: "certs", |
| mountPath: "/cockroach/cockroach-certs/client.root.crt", |
| subPath: "tls.crt", |
| }, |
| { |
| name: "certs", |
| mountPath: "/cockroach/cockroach-certs/client.root.key", |
| subPath: "tls.key", |
| }, |
| ], |
| }, |
| ], |
| volumes: [ |
| { |
| name: "certs", |
| secret: { |
| secretName: cluster.pki.clientCertificate.spec.secretName, |
| defaultMode: kube.parseOctal("400") |
| } |
| }, |
| ], |
| }, |
| }, |
| }, |
| } |