| # Deploy a CockroachDB cluster in secure mode. |
| # This creates an N-node cluster based on a given static topology. |
| |
| # Can be used either in own namespace or in an existing one: |
| # crdb: cockroachdb.Cluster("q3kdb") { |
| # cfg+: { |
| # namespace: "q3k", // if not given, will create 'q3kdb' namespace |
| # topology: [ |
| # { name: "a", node: "bc01n01.hswaw.net" }, |
| # { name: "b", node: "bc01n02.hswaw.net" }, |
| # { name: "c", node: "bc01n03.hswaw.net" }, |
| # ], |
| # hostPath: "/var/db/cockroach-q3k", |
| # }, |
| #}, |
| # |
| # After the cluster is up, you can get to an administrateive SQL shell: |
| # $ NS=q3k kubectl -n $NS exec -it $(kubectl -n $NS get pods -o name | grep client- | cut -d/ -f 2) /cockroach/cockroach sql |
| # root@q3kdb-cockroachdb-0.q3kdb-internal.q3k.svc.cluster.local:26257/defaultdb> |
| # |
| # Then, you can create some users and databases for applications: |
| # defaultdb> CREATE DATABASE wykop; |
| # defaultdb> CREATE USER bialkov PASSWORD hackme; |
| # defaultdb> GRANT ALL ON DATABASE wykop TO bialkov; |
| # |
| # You are then ready to access the database via the public service from your application. |
| # |
| # PGCLIENTENCODING=utf8 psql -h q3kdb-public -p 26257 -U bialkov wykop |
| # Password for user bialkov: |
| # psql (10.9 (Ubuntu 10.9-0ubuntu0.18.04.1), server 9.5.0) |
| # SSL connection (protocol: TLSv1.2, cipher: ECDHE-RSA-AES128-GCM-SHA256, bits: 128, compression: off) |
| # Type "help" for help. |
| # |
| # wykop=> |
| |
| |
| local kube = import "../../../kube/kube.libsonnet"; |
| local policies = import "../../../kube/policies.libsonnet"; |
| |
| { |
| Cluster(name): { |
| local cluster = self, |
| |
| cfg:: { |
| image: "cockroachdb/cockroach:v19.1.0", |
| |
| # Must be unique per cluster. |
| portServe: 26257, |
| portHttp: 8080, |
| hostPath: error "hostPath must be defined", |
| topology: error "topology must be defined", |
| clients: [], |
| |
| namespace: null, |
| ownNamespace: cluster.cfg.namespace == null, |
| }, |
| |
| namespaceName:: if cluster.cfg.namespace != null then cluster.cfg.namespace else name, |
| |
| metadata:: { |
| namespace: cluster.namespaceName, |
| labels: { |
| "app.kubernetes.io/name": "cockroachdb", |
| "app.kubernetes.io/managed-by": "kubecfg", |
| "app.kubernetes.io/component": "cockroachdb", |
| }, |
| }, |
| |
| namespace: { |
| [if cluster.cfg.ownNamespace then "ns"]: kube.Namespace(cluster.namespaceName), |
| }, |
| |
| insecurePolicy: policies.AllowNamespaceInsecure(cluster.namespaceName), |
| |
| name(suffix):: if cluster.cfg.ownNamespace then suffix else name + "-" + suffix, |
| |
| pki: { |
| selfSignedIssuer: kube.Issuer(cluster.name("selfsigned")) { |
| metadata+: cluster.metadata, |
| spec: { |
| selfSigned: {}, |
| }, |
| }, |
| |
| selfSignedKeypair: kube.Certificate(cluster.name("cluster-ca")) { |
| metadata+: cluster.metadata, |
| spec: { |
| secretName: cluster.name("cluster-ca"), |
| duration: "43800h0m0s", // 5 years |
| isCA: true, |
| issuerRef: { |
| name: cluster.pki.selfSignedIssuer.metadata.name, |
| }, |
| commonName: "cockroachdb-cluster-ca", |
| }, |
| }, |
| |
| clusterIssuer: kube.Issuer(cluster.name("cluster-ca")) { |
| metadata+: cluster.metadata, |
| spec: { |
| ca: { |
| secretName: cluster.pki.selfSignedKeypair.metadata.name, |
| }, |
| }, |
| }, |
| |
| nodeCertificate: kube.Certificate(cluster.name("node")) { |
| metadata+: cluster.metadata, |
| spec: { |
| secretName: "cockroachdb-node-cert", |
| duration: "43800h0m0s", // 5 years |
| issuerRef: { |
| name: cluster.pki.clusterIssuer.metadata.name, |
| }, |
| commonName: "node", |
| dnsNames: [ |
| cluster.publicService.metadata.name, |
| std.join(".", [cluster.publicService.metadata.name, cluster.metadata.namespace ]), |
| cluster.publicService.host, |
| std.join(".", [cluster.publicService.host, "cluster.local" ]), |
| std.join(".", [cluster.publicService.metadata.name, cluster.metadata.namespace ]), |
| ] + [ |
| "%s.cluster.local" % s.service.host |
| for s in cluster.servers |
| ], |
| }, |
| }, |
| |
| clientCertificate: kube.Certificate(cluster.name("client")) { |
| metadata+: cluster.metadata, |
| spec: { |
| secretName: cluster.name("client-certificate"), |
| duration: "43800h0m0s", // 5 years |
| issuerRef: { |
| name: cluster.pki.clusterIssuer.metadata.name, |
| }, |
| commonName: "root", |
| }, |
| }, |
| }, |
| |
| serviceAccount: kube.ServiceAccount(cluster.name("cockroachdb")) { |
| metadata+: cluster.metadata, |
| }, |
| |
| role: kube.Role(cluster.name("cockroachdb")) { |
| metadata+: cluster.metadata, |
| rules: [ |
| { |
| apiGroups: [ "" ], |
| resources: [ "secrets" ], |
| verbs: [ "get" ], |
| }, |
| ], |
| }, |
| |
| roleBinding: kube.RoleBinding(cluster.name("cockroachdb")) { |
| metadata+: cluster.metadata, |
| roleRef_: cluster.role, |
| subjects_: [cluster.serviceAccount], |
| }, |
| |
| publicService: kube.Service(cluster.name("public")) { |
| metadata+: cluster.metadata, |
| target_pod:: cluster.servers[0].deploy.spec.template, |
| spec+: { |
| ports: [ |
| { name: "grpc", port: cluster.cfg.portServe, targetPort: cluster.cfg.portServe }, |
| { name: "http", port: cluster.cfg.portHttp, targetPort: cluster.cfg.portHttp }, |
| ], |
| type: "LoadBalancer", |
| }, |
| }, |
| |
| podDisruptionBudget: kube.PodDisruptionBudget(cluster.name("pod")) { |
| metadata+: cluster.metadata, |
| spec: { |
| selector: { |
| matchLabels: { |
| "app.kubernetes.io/component": "cockroachdb", |
| }, |
| }, |
| maxUnavailable: 1, |
| }, |
| }, |
| |
| servers: [ |
| { |
| local server = self, |
| service: kube.Service(cluster.name("server-" + el.name)) { |
| metadata+: cluster.metadata + { |
| annotations+: { |
| "service.alpha.kubernetes.io/tolerate-unready-endpoints": "true", |
| "prometheus.io/scrape": "true", |
| "prometheus.io/path": "_status/vars", |
| "prometheus.io/port": std.toString(cluster.cfg.portHttp), |
| }, |
| }, |
| target_pod:: server.deploy.spec.template, |
| spec+: { |
| ports: [ |
| { name: "grpc", port: cluster.cfg.portServe, targetPort: cluster.cfg.portServe }, |
| { name: "http", port: cluster.cfg.portHttp, targetPort: cluster.cfg.portHttp }, |
| ], |
| publishNotReadyAddresses: true, |
| clusterIP: "None", |
| }, |
| }, |
| deploy: kube.Deployment(cluster.name("server-" + el.name)) { |
| metadata+: cluster.metadata { |
| labels+: { |
| "app.kubernetes.io/component": "server", |
| "kubernetes.hackerspace.pl/cockroachdb-server": el.name, |
| }, |
| }, |
| spec+: { |
| template+: { |
| metadata: server.deploy.metadata, |
| spec+: { |
| dnsPolicy: "ClusterFirst", |
| serviceAccountName: cluster.serviceAccount.metadata.name, |
| nodeSelector: { |
| "kubernetes.io/hostname": el.node, |
| }, |
| containers: [ |
| kube.Container("cockroachdb") { |
| image: cluster.cfg.image, |
| imagePullPolicy: "IfNotPresent", |
| resources: { |
| requests: { |
| cpu: "2", |
| memory: "6Gi", |
| }, |
| limits: { |
| memory: "6Gi", |
| }, |
| }, |
| ports_: { |
| "grpc": { containerPort: cluster.cfg.portServe }, |
| "http": { containerPort: cluster.cfg.portHttp }, |
| }, |
| livenessProbe: { |
| httpGet: { |
| path: "/health", |
| port: "http", |
| }, |
| initialDelaySeconds: 30, |
| periodSeconds: 5, |
| }, |
| readinessProbe: { |
| httpGet: { |
| path: "/health?ready=1", |
| port: "http", |
| }, |
| initialDelaySeconds: 10, |
| periodSeconds: 5, |
| failureThreshold: 2, |
| }, |
| volumeMounts: [ |
| { |
| name: "datadir", |
| mountPath: "/cockroach/cockroach-data", |
| }, |
| { |
| name: "certs", |
| mountPath: "/cockroach/cockroach-certs/node.crt", |
| subPath: "tls.crt", |
| }, |
| { |
| name: "certs", |
| mountPath: "/cockroach/cockroach-certs/node.key", |
| subPath: "tls.key", |
| }, |
| { |
| name: "certs", |
| mountPath: "/cockroach/cockroach-certs/ca.crt", |
| subPath: "ca.crt", |
| }, |
| ], |
| env_: { |
| "COCKROACH_CERTS_DIR": "/cockroach/cockroach-certs", |
| }, |
| command: [ |
| "/cockroach/cockroach", "start", |
| "--logtostderr", |
| "--certs-dir", "/cockroach/cockroach-certs", |
| "--advertise-host", "%s.cluster.local" % server.service.host, |
| "--cache", "25%", "--max-sql-memory", "25%", |
| "--join", std.join(",", ["%s.cluster.local:%d" % [s.service.host, cluster.cfg.portServe] for s in cluster.servers]), |
| "--listen-addr=0.0.0.0:%d" % cluster.cfg.portServe, |
| "--http-addr=0.0.0.0:%d" % cluster.cfg.portHttp, |
| ], |
| }, |
| ], |
| terminationGracePeriodSeconds: 60, |
| volumes: [ |
| { |
| name: "datadir", |
| hostPath: { |
| path: cluster.cfg.hostPath, |
| }, |
| }, |
| { |
| name: "certs", |
| secret: { |
| secretName: cluster.pki.nodeCertificate.spec.secretName, |
| defaultMode: kube.parseOctal("400"), |
| }, |
| }, |
| ], |
| }, |
| }, |
| }, |
| } |
| } |
| for el in cluster.cfg.topology |
| ], |
| |
| initJob: kube.Job(cluster.name("init")) { |
| metadata+: cluster.metadata, |
| spec: { |
| template: { |
| metadata+: cluster.metadata, |
| spec+: { |
| serviceAccountName: cluster.serviceAccount.metadata.name, |
| containers: [ |
| kube.Container("cluster-init") { |
| image: cluster.cfg.image, |
| imagePullPolicy: "IfNotPresent", |
| env_: { |
| "COCKROACH_CERTS_DIR": "/cockroach/cockroach-certs", |
| }, |
| command: [ |
| "/bin/bash", |
| "-ecx", |
| "/cockroach/cockroach init --host=%s.cluster.local:%d" % [cluster.servers[0].service.host, cluster.cfg.portServe], |
| ], |
| volumeMounts: [ |
| { |
| name: "certs", |
| mountPath: "/cockroach/cockroach-certs/ca.crt", |
| subPath: "ca.crt", |
| }, |
| { |
| name: "certs", |
| mountPath: "/cockroach/cockroach-certs/client.root.crt", |
| subPath: "tls.crt", |
| }, |
| { |
| name: "certs", |
| mountPath: "/cockroach/cockroach-certs/client.root.key", |
| subPath: "tls.key", |
| }, |
| ], |
| }, |
| ], |
| restartPolicy: "OnFailure", |
| volumes: [ |
| { |
| name: "certs", |
| secret: { |
| secretName: cluster.pki.clientCertificate.spec.secretName, |
| defaultMode: kube.parseOctal("400") |
| } |
| }, |
| ], |
| }, |
| }, |
| }, |
| }, |
| |
| Client(name):: { |
| certificate: kube.Certificate(cluster.name("client-%s" % name)) { |
| metadata+: cluster.metadata, |
| spec: { |
| secretName: cluster.name("client-%s-certificate" % name), |
| duration: "43800h0m0s", // 5 years |
| issuerRef: { |
| name: cluster.pki.clusterIssuer.metadata.name, |
| }, |
| commonName: name, |
| }, |
| }, |
| }, |
| |
| client: kube.Deployment(cluster.name("client")) { |
| metadata+: cluster.metadata { |
| labels+: { |
| "app.kubernetes.io/component": "client", |
| }, |
| }, |
| spec+: { |
| template: { |
| metadata: cluster.client.metadata, |
| spec+: { |
| terminationGracePeriodSeconds: 5, |
| containers: [ |
| kube.Container("cockroachdb-client") { |
| image: cluster.cfg.image, |
| env_: { |
| "COCKROACH_CERTS_DIR": "/cockroach/cockroach-certs", |
| "COCKROACH_HOST": cluster.publicService.host, |
| "COCKROACH_PORT": std.toString(cluster.cfg.portServe), |
| }, |
| command: ["sleep", "2147483648"], //(FIXME) keep the client pod running indefinitely |
| volumeMounts: [ |
| { |
| name: "certs", |
| mountPath: "/cockroach/cockroach-certs/ca.crt", |
| subPath: "ca.crt", |
| }, |
| { |
| name: "certs", |
| mountPath: "/cockroach/cockroach-certs/client.root.crt", |
| subPath: "tls.crt", |
| }, |
| { |
| name: "certs", |
| mountPath: "/cockroach/cockroach-certs/client.root.key", |
| subPath: "tls.key", |
| }, |
| ], |
| }, |
| ], |
| volumes: [ |
| { |
| name: "certs", |
| secret: { |
| secretName: cluster.pki.clientCertificate.spec.secretName, |
| defaultMode: kube.parseOctal("400") |
| } |
| }, |
| ], |
| }, |
| }, |
| }, |
| }, |
| }, |
| } |