cluster/kube/lib/cockroachdb: refactor
We refactor this library to:
- support multiple databases, but with a strong suggestion of having
one per k8s cluster
- drop the database creation logic
- redo naming (allowing for two options: multiple clusters per
namespace or an exclusive namespace for the cluster)
- unhardcode dns names
diff --git a/cluster/kube/lib/cockroachdb.libsonnet b/cluster/kube/lib/cockroachdb.libsonnet
index b3aed8a..def9bcb 100644
--- a/cluster/kube/lib/cockroachdb.libsonnet
+++ b/cluster/kube/lib/cockroachdb.libsonnet
@@ -1,419 +1,414 @@
# Deploy a 3-node CockroachDB cluster in secure mode.
+# Can be used either in own namespace or in an existing one:
+# crdb: cockroachdb.Cluster("q3kdb") {
+# cfg+: {
+# namespace: "q3k", // if not given, will create 'q3kdb' namespace
+# },
+#},
+#
+# After the cluster is up, you can get to an administrateive SQL shell:
+# $ kubectl -n q3k exec -it q3kdb-client /cockroach/cockroach sql
+# root@q3kdb-cockroachdb-0.q3kdb-internal.q3k.svc.cluster.local:26257/defaultdb>
+#
+# Then, you can create some users and databases for applications:
+# defaultdb> CREATE DATABASE wykop;
+# defaultdb> CREATE USER bialkov PASSWORD hackme;
+# defaultdb> GRANT ALL ON DATABASE wykop to bialkov;
+#
+# You are then ready to access the database via the public service from your application.
+#
+# PGCLIENTENCODING=utf8 psql -h q3kdb-public -p 26257 -U bialkov wykop
+# Password for user bialkov:
+# psql (10.9 (Ubuntu 10.9-0ubuntu0.18.04.1), server 9.5.0)
+# SSL connection (protocol: TLSv1.2, cipher: ECDHE-RSA-AES128-GCM-SHA256, bits: 128, compression: off)
+# Type "help" for help.
+#
+# wykop=>
+
+
local kube = import "../../../kube/kube.libsonnet";
local cm = import "cert-manager.libsonnet";
{
- local cockroachdb = self,
- local crdb = cockroachdb,
- local cfg = crdb.cfg,
+ Cluster(name): {
+ local cluster = self,
- cfg:: {
- namespace: error "namespace must be set",
- appName: error "app name must be set",
- prefix: "", # if set, should be 'foo-',
-
- image: "cockroachdb/cockroach:v19.1.0",
- database: error "database name must be set",
- username: error "username must be set",
- password: error "password must be set",
- },
-
- makeName(suffix):: cfg.prefix + suffix,
-
- metadata:: {
- namespace: cfg.namespace,
- labels: {
- "app.kubernetes.io/name": cfg.appName,
- "app.kubernetes.io/managed-by": "kubecfg",
- "app.kubernetes.io/component": "cockroachdb",
+ cfg:: {
+ image: "cockroachdb/cockroach:v19.1.0",
+ namespace: null,
+ ownNamespace: cluster.cfg.namespace == null,
},
- },
- pki: {
- selfSignedIssuer: cm.Issuer("cockroachdb-selfsigned-issuer") {
- metadata+: crdb.metadata,
- spec: {
- selfSigned: {},
+ namespaceName:: if cluster.cfg.namespace != null then cluster.cfg.namespace else name,
+
+ metadata:: {
+ namespace: cluster.namespaceName,
+ labels: {
+ "app.kubernetes.io/name": "cockroachdb",
+ "app.kubernetes.io/managed-by": "kubecfg",
+ "app.kubernetes.io/component": "cockroachdb",
},
},
- selfSignedKeypair: cm.Certificate("cockroachdb-cluster-ca-keypair") {
- metadata+: crdb.metadata,
- spec: {
- secretName: "cockroachdb-cluster-ca-keypair",
- duration: "43800h0m0s", // 5 years
- isCA: true,
- issuerRef: {
- name: crdb.pki.selfSignedIssuer.metadata.name,
+ namespace: {
+ [if cluster.cfg.ownNamespace then "ns"]: kube.Namespace(cluster.namespaceName),
+ },
+
+ name(suffix):: if cluster.cfg.ownNamespace then suffix else name + "-" + suffix,
+
+ hosts:: ["%s-%d.%s.cluster.local" % [cluster.statefulSet.metadata.name, n, cluster.internalService.host] for n in std.range(0, cluster.statefulSet.spec.replicas)],
+
+ pki: {
+ selfSignedIssuer: cm.Issuer(cluster.name("selfsigned")) {
+ metadata+: cluster.metadata,
+ spec: {
+ selfSigned: {},
},
- commonName: "cockroachdb-cluster-ca",
},
- },
- clusterIssuer: cm.Issuer("cockroachdb-cluster-ca") {
- metadata+: crdb.metadata,
- spec: {
- ca: {
- secretName: crdb.pki.selfSignedKeypair.metadata.name,
+ selfSignedKeypair: cm.Certificate(cluster.name("cluster-ca")) {
+ metadata+: cluster.metadata,
+ spec: {
+ secretName: cluster.name("cluster-ca"),
+ duration: "43800h0m0s", // 5 years
+ isCA: true,
+ issuerRef: {
+ name: cluster.pki.selfSignedIssuer.metadata.name,
+ },
+ commonName: "cockroachdb-cluster-ca",
+ },
+ },
+
+ clusterIssuer: cm.Issuer(cluster.name("cluster-ca")) {
+ metadata+: cluster.metadata,
+ spec: {
+ ca: {
+ secretName: cluster.pki.selfSignedKeypair.metadata.name,
+ },
+ },
+ },
+
+ nodeCertificate: cm.Certificate(cluster.name("node")) {
+ metadata+: cluster.metadata,
+ spec: {
+ secretName: "cockroachdb-node-cert",
+ duration: "43800h0m0s", // 5 years
+ issuerRef: {
+ name: cluster.pki.clusterIssuer.metadata.name,
+ },
+ commonName: "node",
+ dnsNames: [
+ "localhost",
+ "127.0.0.1",
+ cluster.publicService.metadata.name,
+ std.join(".", [cluster.publicService.metadata.name, cluster.metadata.namespace ]),
+ std.join(".", [cluster.publicService.host, "cluster.local" ]),
+ std.join(".", [ "*", cluster.internalService.metadata.name ]),
+ std.join(".", [ "*", cluster.internalService.metadata.name, cluster.metadata.namespace ]),
+ std.join(".", [ "*", cluster.internalService.host, "cluster.local" ]),
+ ],
+ },
+ },
+
+ clientCertificate: cm.Certificate(cluster.name("client")) {
+ metadata+: cluster.metadata,
+ spec: {
+ secretName: cluster.name("client-certificate"),
+ duration: "43800h0m0s", // 5 years
+ issuerRef: {
+ name: cluster.pki.clusterIssuer.metadata.name,
+ },
+ commonName: "root",
},
},
},
- nodeCertificate: cm.Certificate("cockroachdb-node-cert") {
- metadata+: crdb.metadata,
- spec: {
- secretName: "cockroachdb-node-cert",
- duration: "43800h0m0s", // 5 years
- issuerRef: {
- name: crdb.pki.clusterIssuer.metadata.name,
+ serviceAccount: kube.ServiceAccount(cluster.name("cockroachdb")) {
+ metadata+: cluster.metadata,
+ },
+
+ role: kube.Role(cluster.name("cockroachdb")) {
+ metadata+: cluster.metadata,
+ rules: [
+ {
+ apiGroups: [ "" ],
+ resources: [ "secrets" ],
+ verbs: [ "get" ],
},
- commonName: "node",
- dnsNames: [
- "localhost",
- "127.0.0.1",
- crdb.publicService.metadata.name,
- std.join(".", [crdb.publicService.metadata.name, cfg.namespace ]),
- std.join(".", [crdb.publicService.host, "cluster.local" ]),
- std.join(".", [ "*", crdb.internalService.metadata.name ]),
- std.join(".", [ "*", crdb.internalService.metadata.name, cfg.namespace ]),
- std.join(".", [ "*", crdb.internalService.host, "cluster.local" ]),
+ ],
+ },
+
+ roleBinding: kube.RoleBinding(cluster.name("cockroachdb")) {
+ metadata+: cluster.metadata,
+ roleRef_: cluster.role,
+ subjects_: [cluster.serviceAccount],
+ },
+
+ publicService: kube.Service(cluster.name("public")) {
+ metadata+: cluster.metadata,
+ target_pod:: cluster.statefulSet.spec.template,
+ spec+: {
+ ports: [
+ { name: "grpc", port: 26257, targetPort: 26257 },
+ { name: "http", port: 8080, targetPort: 8080 },
],
},
},
- clientCertificate: cm.Certificate("cockroachdb-client-cert") {
- metadata+: crdb.metadata,
+ internalService: kube.Service(cluster.name("internal")) {
+ metadata+: cluster.metadata + {
+ annotations+: {
+ "service.alpha.kubernetes.io/tolerate-unready-endpoints": "true",
+ "prometheus.io/scrape": "true",
+ "prometheus.io/path": "_status/vars",
+ "prometheus.io/port": "8080",
+ },
+ },
+ target_pod:: cluster.statefulSet.spec.template,
+ spec+: {
+ ports: [
+ { name: "grpc", port: 26257, targetPort: 26257 },
+ { name: "http", port: 8080, targetPort: 8080 },
+ ],
+ publishNotReadyAddresses: true,
+ clusterIP: "None",
+ },
+ },
+
+ podDisruptionBudget: kube.PodDisruptionBudget(cluster.name("pod")) {
+ metadata+: cluster.metadata,
spec: {
- secretName: "cockroachdb-client-cert",
- duration: "43800h0m0s", // 5 years
- issuerRef: {
- name: crdb.pki.clusterIssuer.metadata.name,
+ selector: {
+ matchLabels: {
+ "app.kubernetes.io/component": "cockroachdb",
+ },
},
- commonName: "root",
+ maxUnavailable: 1,
},
},
- },
- serviceAccount: kube.ServiceAccount("cockroachdb") {
- metadata+: crdb.metadata,
- },
-
- role: kube.Role("cockroachdb") {
- metadata+: crdb.metadata,
- rules: [
- {
- apiGroups: [ "" ],
- resources: [ "secrets" ],
- verbs: [ "get" ],
- },
- ],
- },
-
- roleBinding: kube.RoleBinding("cockroachdb") {
- metadata+: crdb.metadata,
- roleRef: {
- apiGroup: "rbac.authorization.k8s.io",
- kind: "Role",
- name: "cockroachdb",
- },
- subjects: [
- {
- kind: "ServiceAccount",
- name: crdb.serviceAccount.metadata.name,
- namespace: cfg.namespace,
- },
- ],
- },
-
- publicService: kube.Service(crdb.makeName("cockroachdb-public")) {
- metadata+: crdb.metadata,
- target_pod:: crdb.statefulSet.spec.template,
- spec+: {
- ports: [
- { name: "grpc", port: 26257, targetPort: 26257 },
- { name: "http", port: 8080, targetPort: 8080 },
- ],
- },
- },
-
- internalService: kube.Service(crdb.makeName("cockroachdb")) {
- metadata+: crdb.metadata + {
- annotations+: {
- "service.alpha.kubernetes.io/tolerate-unready-endpoints": "true",
- "prometheus.io/scrape": "true",
- "prometheus.io/path": "_status/vars",
- "prometheus.io/port": "8080",
- },
- },
- target_pod:: crdb.statefulSet.spec.template,
- spec+: {
- ports: [
- { name: "grpc", port: 26257, targetPort: 26257 },
- { name: "http", port: 8080, targetPort: 8080 },
- ],
- publishNotReadyAddresses: true,
- clusterIP: "None",
- },
- },
-
- podDisruptionBudget: kube.PodDisruptionBudget(crdb.makeName("cockroachdb-budget")) {
- metadata+: crdb.metadata,
- spec: {
- selector: {
- matchLabels: {
- "app.kubernetes.io/component": "cockroachdb",
+ statefulSet: kube.StatefulSet(cluster.name("cockroachdb")) {
+ metadata+: cluster.metadata {
+ labels+: {
+ "app.kubernetes.io/component": "server",
},
},
- maxUnavailable: 1,
- },
- },
-
- statefulSet: kube.StatefulSet(crdb.makeName("cockroachdb")) {
- metadata+: crdb.metadata,
- spec+: {
- serviceName: crdb.internalService.metadata.name,
- replicas: 3,
- template: {
- metadata+: crdb.metadata,
- spec+: {
- dnsPolicy: "ClusterFirst",
- serviceAccountName: crdb.serviceAccount.metadata.name,
- affinity: {
- podAntiAffinity: {
- preferredDuringSchedulingIgnoredDuringExecution: [
- {
- weight: 100,
- podAffinityTerm: {
- labelSelector: {
- matchExpressions: [
- {
- key: "app.kubernetes.io/component",
- operator: "In",
- values: [ "cockroachdb" ],
- },
- ],
+ spec+: {
+ serviceName: cluster.internalService.metadata.name,
+ replicas: 3,
+ template: {
+ metadata: cluster.statefulSet.metadata,
+ spec+: {
+ dnsPolicy: "ClusterFirst",
+ serviceAccountName: cluster.serviceAccount.metadata.name,
+ affinity: {
+ podAntiAffinity: {
+ preferredDuringSchedulingIgnoredDuringExecution: [
+ {
+ weight: 100,
+ podAffinityTerm: {
+ labelSelector: {
+ matchExpressions: [
+ {
+ key: "app.kubernetes.io/component",
+ operator: "In",
+ values: [ "cockroachdb" ],
+ },
+ ],
+ },
+ topologyKey: "kubernetes.io/hostname",
},
- topologyKey: "kubernetes.io/hostname",
+ },
+ ],
+ },
+ },
+ containers: [
+ kube.Container("cockroachdb") {
+ image: cluster.cfg.image,
+ imagePullPolicy: "IfNotPresent",
+ resources: {
+ requests: {
+ cpu: "2",
+ memory: "6Gi",
+ },
+ limits: {
+ memory: "6Gi",
},
},
- ],
- },
+ ports_: {
+ "grpc": { containerPort: 26257 },
+ "http": { containerPort: 8080 },
+ },
+ livenessProbe: {
+ httpGet: {
+ path: "/health",
+ port: "http",
+ },
+ initialDelaySeconds: 30,
+ periodSeconds: 5,
+ },
+ readinessProbe: {
+ httpGet: {
+ path: "/health?ready=1",
+ port: "http",
+ },
+ initialDelaySeconds: 10,
+ periodSeconds: 5,
+ failureThreshold: 2,
+ },
+ volumeMounts: [
+ {
+ name: "datadir",
+ mountPath: "/cockroach/cockroach-data",
+ },
+ {
+ name: "certs",
+ mountPath: "/cockroach/cockroach-certs/node.crt",
+ subPath: "tls.crt",
+ },
+ {
+ name: "certs",
+ mountPath: "/cockroach/cockroach-certs/node.key",
+ subPath: "tls.key",
+ },
+ {
+ name: "certs",
+ mountPath: "/cockroach/cockroach-certs/ca.crt",
+ subPath: "ca.crt",
+ },
+ ],
+ env_: {
+ "COCKROACH_CERTS_DIR": "/cockroach/cockroach-certs",
+ },
+ command: [
+ "/bin/bash",
+ "-ecx",
+ "exec /cockroach/cockroach start --logtostderr --certs-dir /cockroach/cockroach-certs --advertise-host $(hostname -f) --http-addr 0.0.0.0 --cache 25% --max-sql-memory 25% --join " + std.join(",", cluster.hosts),
+ ],
+ },
+ ],
+ terminationGracePeriodSeconds: 60,
+ volumes: [
+ {
+ name: "datadir",
+ emptyDir: {},
+ },
+ {
+ name: "certs",
+ secret: {
+ secretName: cluster.pki.nodeCertificate.spec.secretName,
+ defaultMode: kube.parseOctal("400"),
+ },
+ },
+ ],
},
- containers: [
- kube.Container("cockroachdb") {
- image: cfg.image,
- imagePullPolicy: "IfNotPresent",
- resources: {
- requests: {
- cpu: "2",
- memory: "6Gi",
- },
- limits: {
- memory: "6Gi",
- },
- },
- ports_: {
- "grpc": { containerPort: 26257 },
- "http": { containerPort: 8080 },
- },
- livenessProbe: {
- httpGet: {
- path: "/health",
- port: "http",
- },
- initialDelaySeconds: 30,
- periodSeconds: 5,
- },
- readinessProbe: {
- httpGet: {
- path: "/health?ready=1",
- port: "http",
- },
- initialDelaySeconds: 10,
- periodSeconds: 5,
- failureThreshold: 2,
- },
- volumeMounts: [
- {
- name: "datadir",
- mountPath: "/cockroach/cockroach-data",
- },
- {
- name: "certs",
- mountPath: "/cockroach/cockroach-certs/node.crt",
- subPath: "tls.crt",
- },
- {
- name: "certs",
- mountPath: "/cockroach/cockroach-certs/node.key",
- subPath: "tls.key",
- },
- {
- name: "certs",
- mountPath: "/cockroach/cockroach-certs/ca.crt",
- subPath: "ca.crt",
- },
- ],
- env_: {
- "COCKROACH_CERTS_DIR": "/cockroach/cockroach-certs",
- },
- command: [
- "/bin/bash",
- "-ecx",
- "exec /cockroach/cockroach start --logtostderr --certs-dir /cockroach/cockroach-certs --advertise-host $(hostname -f) --http-addr 0.0.0.0 --join cockroachdb-0.cockroachdb,cockroachdb-1.cockroachdb,cockroachdb-2.cockroachdb --cache 25% --max-sql-memory 25%",
- ],
- },
- ],
- terminationGracePeriodSeconds: 60,
- volumes: [
- {
- name: "datadir",
- emptyDir: {},
- },
- {
- name: "certs",
- secret: {
- secretName: crdb.pki.nodeCertificate.spec.secretName,
- defaultMode: kube.parseOctal("400"),
- },
- },
- ],
},
- },
- podManagementPolicy: "Parallel",
- updateStrategy: {
- type: "RollingUpdate",
- },
- },
- },
-
- initJob: kube.Job(crdb.makeName("cockroachdb-init")) {
- metadata+: crdb.metadata,
- spec: {
- template: {
- metadata+: crdb.metadata,
- spec+: {
- serviceAccountName: crdb.serviceAccount.metadata.name,
- initContainers: [
- kube.Container("cluster-init") {
- image: cfg.image,
- imagePullPolicy: "IfNotPresent",
- env_: {
- "COCKROACH_CERTS_DIR": "/cockroach/cockroach-certs",
- },
- command: [
- "/bin/bash",
- "-ecx",
- "/cockroach/cockroach init --host=cockroachdb-0.cockroachdb",
- ],
- volumeMounts: [
- {
- name: "certs",
- mountPath: "/cockroach/cockroach-certs/ca.crt",
- subPath: "ca.crt",
- },
- {
- name: "certs",
- mountPath: "/cockroach/cockroach-certs/client.root.crt",
- subPath: "tls.crt",
- },
- {
- name: "certs",
- mountPath: "/cockroach/cockroach-certs/client.root.key",
- subPath: "tls.key",
- },
- ],
- },
- ],
- containers: [
- kube.Container("db-init") {
- image: cfg.image,
- imagePullPolicy: "IfNotPresent",
- env_: {
- "COCKROACH_CERTS_DIR": "/cockroach/cockroach-certs",
- "DB_NAME": cfg.database,
- "DB_USERNAME": cfg.username,
- "DB_PASSWORD": cfg.password,
- },
- command: [
- "/bin/bash",
- "-ec",
- "/cockroach/cockroach sql -e \"CREATE DATABASE ${DB_NAME}; CREATE USER ${DB_USERNAME} PASSWORD '${DB_PASSWORD}'; GRANT ALL ON DATABASE ${DB_NAME} TO ${DB_USERNAME};\" --host=cockroachdb-0.cockroachdb",
- ],
- volumeMounts: [
- {
- name: "certs",
- mountPath: "/cockroach/cockroach-certs/ca.crt",
- subPath: "ca.crt",
- },
- {
- name: "certs",
- mountPath: "/cockroach/cockroach-certs/client.root.crt",
- subPath: "tls.crt",
- },
- {
- name: "certs",
- mountPath: "/cockroach/cockroach-certs/client.root.key",
- subPath: "tls.key",
- },
- ],
- },
- ],
- restartPolicy: "OnFailure",
- volumes: [
- {
- name: "certs",
- secret: {
- secretName: crdb.pki.clientCertificate.spec.secretName,
- defaultMode: kube.parseOctal("400")
- }
- },
- ],
+ podManagementPolicy: "Parallel",
+ updateStrategy: {
+ type: "RollingUpdate",
},
},
},
- },
- clientPod: kube.Pod(crdb.makeName("cockroachdb-client")) {
- metadata+: crdb.metadata,
- spec: {
- terminationGracePeriodSeconds: 5,
- containers: [
- kube.Container("cockroachdb-client") {
- image: cfg.image,
- env_: {
- "COCKROACH_CERTS_DIR": "/cockroach/cockroach-certs",
+ initJob: kube.Job(cluster.name("init")) {
+ metadata+: cluster.metadata,
+ spec: {
+ template: {
+ metadata+: cluster.metadata,
+ spec+: {
+ serviceAccountName: cluster.serviceAccount.metadata.name,
+ containers: [
+ kube.Container("cluster-init") {
+ image: cluster.cfg.image,
+ imagePullPolicy: "IfNotPresent",
+ env_: {
+ "COCKROACH_CERTS_DIR": "/cockroach/cockroach-certs",
+ },
+ command: [
+ "/bin/bash",
+ "-ecx",
+ "/cockroach/cockroach init --host=" + cluster.hosts[0],
+ ],
+ volumeMounts: [
+ {
+ name: "certs",
+ mountPath: "/cockroach/cockroach-certs/ca.crt",
+ subPath: "ca.crt",
+ },
+ {
+ name: "certs",
+ mountPath: "/cockroach/cockroach-certs/client.root.crt",
+ subPath: "tls.crt",
+ },
+ {
+ name: "certs",
+ mountPath: "/cockroach/cockroach-certs/client.root.key",
+ subPath: "tls.key",
+ },
+ ],
+ },
+ ],
+ restartPolicy: "OnFailure",
+ volumes: [
+ {
+ name: "certs",
+ secret: {
+ secretName: cluster.pki.clientCertificate.spec.secretName,
+ defaultMode: kube.parseOctal("400")
+ }
+ },
+ ],
},
- command: ["sleep", "2147483648"], //(FIXME) keep the client pod running indefinitely
- volumeMounts: [
- {
- name: "certs",
- mountPath: "/cockroach/cockroach-certs/ca.crt",
- subPath: "ca.crt",
- },
- {
- name: "certs",
- mountPath: "/cockroach/cockroach-certs/client.root.crt",
- subPath: "tls.crt",
- },
- {
- name: "certs",
- mountPath: "/cockroach/cockroach-certs/client.root.key",
- subPath: "tls.key",
- },
- ],
},
- ],
- volumes: [
- {
- name: "certs",
- secret: {
- secretName: crdb.pki.clientCertificate.spec.secretName,
- defaultMode: kube.parseOctal("400")
- }
+ },
+ },
+
+ clientPod: kube.Pod(cluster.name("client")) {
+ metadata+: cluster.metadata {
+ labels+: {
+ "app.kubernetes.io/component": "client",
},
- ],
+ },
+ spec: {
+ terminationGracePeriodSeconds: 5,
+ containers: [
+ kube.Container("cockroachdb-client") {
+ image: cluster.cfg.image,
+ env_: {
+ "COCKROACH_CERTS_DIR": "/cockroach/cockroach-certs",
+ "COCKROACH_HOST": cluster.hosts[0],
+ },
+ command: ["sleep", "2147483648"], //(FIXME) keep the client pod running indefinitely
+ volumeMounts: [
+ {
+ name: "certs",
+ mountPath: "/cockroach/cockroach-certs/ca.crt",
+ subPath: "ca.crt",
+ },
+ {
+ name: "certs",
+ mountPath: "/cockroach/cockroach-certs/client.root.crt",
+ subPath: "tls.crt",
+ },
+ {
+ name: "certs",
+ mountPath: "/cockroach/cockroach-certs/client.root.key",
+ subPath: "tls.key",
+ },
+ ],
+ },
+ ],
+ volumes: [
+ {
+ name: "certs",
+ secret: {
+ secretName: cluster.pki.clientCertificate.spec.secretName,
+ defaultMode: kube.parseOctal("400")
+ }
+ },
+ ],
+ },
},
},
}