app/registry: integrate into cluster/kube
This makes a registry be automatically part of the cluster
infrastructure.
Tested by running kubecfg diff, no diffs (apart from out-of-date ACLs)
found.
Change-Id: Ic0635e789cf3fb851f410bcf2865326f1fa87545
diff --git a/cluster/kube/lib/registry.libsonnet b/cluster/kube/lib/registry.libsonnet
new file mode 100644
index 0000000..8b57dd7
--- /dev/null
+++ b/cluster/kube/lib/registry.libsonnet
@@ -0,0 +1,321 @@
+# Deploy a Docker Registry in a cluster.
+
+# This needs an oauth2 secret provisioned, create with:
+# kubectl -n registry create secret generic auth --from-literal=oauth2_secret=...
+# kubectl get secrets rook-ceph-object-user-<ceph-pool>-object-registry -n <ceph-namespace> -o yaml --export | kubectl replace -f - -n registry
+
+local kube = import "../../../kube/kube.libsonnet";
+local cm = import "cert-manager.libsonnet";
+
+{
+ Environment: {
+ local env = self,
+ local cfg = env.cfg,
+ cfg:: {
+ namespace: "registry",
+ domain: error "domain must be set",
+ storageClassName: error "storageClassName must be set",
+ },
+
+ metadata(component):: {
+ namespace: cfg.namespace,
+ labels: {
+ "app.kubernetes.io/name": "registry",
+ "app.kubernetes.io/managed-by": "kubecfg",
+ "app.kubernetes.io/component": component,
+ },
+ },
+
+ namespace: kube.Namespace(cfg.namespace),
+
+ registryIssuer: cm.Issuer("registry-issuer") {
+ metadata+: env.metadata("registry-issuer"),
+ spec: {
+ selfSigned: {},
+ },
+ },
+ authCertificate: cm.Certificate("auth") {
+ metadata+: env.metadata("auth"),
+ spec: {
+ secretName: "auth-internal",
+ duration: "43800h0m0s", // 5 years
+ issuerRef: {
+ name: env.registryIssuer.metadata.name,
+ },
+ commonName: "auth.registry",
+ },
+ },
+ registryCertificate: cm.Certificate("registry") {
+ metadata+: env.metadata("registry"),
+ spec: {
+ secretName: "registry-internal",
+ duration: "43800h0m0s", // 5 years
+ issuerRef: {
+ name: env.registryIssuer.metadata.name,
+ },
+ commonName: "registry.registry",
+ },
+ },
+
+ registryConfig: kube.ConfigMap("registry-config") {
+ metadata+: env.metadata("registry-config"),
+ data: {
+ "config.yml": std.manifestYamlDoc({
+ version: "0.1",
+ log: {
+ fields: {
+ service: "registry",
+ },
+ },
+ storage: {
+ cache: {
+ blobdescriptor: "inmemory",
+ },
+ s3: {
+ regionendpoint: "https://object.ceph-waw1.hswaw.net",
+ bucket: "registry",
+ region: "waw-hdd-redunant-1-object:default-placement",
+ },
+ },
+ http: {
+ addr: ":5000",
+ headers: {
+ "X-Content-Type-Options": ["nosniff"],
+ },
+ tls: {
+ certificate: "/certs/tls.crt",
+ key: "/certs/tls.key",
+ },
+ debug: {
+ addr: "localhost:5001",
+ },
+ },
+ health: {
+ storagedriver: {
+ enabled: true,
+ interval: "10s",
+ threshold: 3,
+ },
+ },
+ auth: {
+ token: {
+ realm: "https://%s/auth" % [cfg.domain],
+ service: "my.docker.registry",
+ issuer: "%s auth server" % [cfg.domain],
+ rootcertbundle: "/authcerts/tls.crt",
+ },
+ },
+ }),
+ },
+ },
+
+ authVolumeClaim: kube.PersistentVolumeClaim("auth-token-storage") {
+ metadata+: env.metadata("auth-token-storage"),
+ spec+: {
+ storageClassName: cfg.storageClassName,
+ accessModes: [ "ReadWriteOnce" ],
+ resources: {
+ requests: {
+ storage: "1Gi",
+ },
+ },
+ },
+ },
+
+ authConfig: kube.ConfigMap("auth-config") {
+ metadata+: env.metadata("auth-config"),
+ data: {
+ "auth_config.yml": std.manifestYamlDoc({
+ server: {
+ addr: ":5001",
+ certificate: "/certs/tls.crt",
+ key: "/certs/tls.key",
+ },
+ token: {
+ issuer: "%s auth server" % [cfg.domain],
+ expiration: 900,
+ },
+ oauth2: {
+ client_id: "registry",
+ client_secret_file: "/secrets/oauth2_secret",
+ authorize_url: "https://sso.hackerspace.pl/oauth/authorize",
+ access_token_url: "https://sso.hackerspace.pl/oauth/token",
+ profile_url: "https://sso.hackerspace.pl/api/1/profile",
+ redirect_url: "https://registry.k0.hswaw.net/oauth2",
+ username_key: "username",
+ token_db: "/data/oauth2_tokens.ldb",
+ registry_url: "https://registry.k0.hswaw.net",
+ },
+ users: {
+ [""]: {}, // '' user are anonymous users.
+ },
+ local data = self,
+ pushers:: [
+ { who: ["q3k", "inf"], what: "vms/*" },
+ { who: ["q3k", "inf"], what: "app/*" },
+ { who: ["q3k", "inf"], what: "go/svc/*" },
+ ],
+ acl: [
+ {
+ match: {account: "/.+/", name: "${account}/*"},
+ actions: ["*"],
+ comment: "Logged in users have full access to images that are in their 'namespace'",
+ },
+ {
+ match: {account: "/.+/", type: "registry", name: "catalog"},
+ actions: ["*"],
+ comment: "Logged in users can query the catalog.",
+ },
+ {
+ match: {account: ""},
+ actions: ["pull"],
+ comment: "Anyone can pull all images.",
+ },
+ ] + [
+ {
+ match: {
+ account: "/(%s)/" % std.join("|", p.who),
+ name: p.what,
+ },
+ actions: ["*"],
+ comment: "%s can push to %s" % [std.join(", ", p.who), p.what],
+ }
+ for p in data.pushers
+ ],
+ }),
+ }
+ },
+
+ authDeployment: kube.Deployment("auth") {
+ metadata+: env.metadata("auth"),
+ spec+: {
+ replicas: 1,
+ template+: {
+ spec+: {
+ volumes_: {
+ data: kube.PersistentVolumeClaimVolume(env.authVolumeClaim),
+ config: kube.ConfigMapVolume(env.authConfig),
+ certs: {
+ secret: { secretName: env.authCertificate.spec.secretName },
+ },
+ secrets: {
+ secret: { secretName: "auth" },
+ },
+ },
+ containers_: {
+ auth: kube.Container("auth") {
+ image: "informatic/docker_auth:2019040307",
+ volumeMounts_: {
+ config: { mountPath: "/config" },
+ certs: { mountPath: "/certs" },
+ secrets: { mountPath: "/secrets" },
+ data: { mountPath: "/data" },
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ authService: kube.Service("auth") {
+ metadata+: env.metadata("auth"),
+ target_pod:: env.authDeployment.spec.template,
+ spec+: {
+ type: "ClusterIP",
+ ports: [
+ { name: "auth", port: 5001, targetPort: 5001, protocol: "TCP" },
+ ],
+ }
+ },
+ registryDeployment: kube.Deployment("docker-registry") {
+ metadata+: env.metadata("docker-registry"),
+ spec+: {
+ replicas: 1,
+ template+: {
+ spec+: {
+ volumes_: {
+ config: kube.ConfigMapVolume(env.registryConfig),
+ certs: {
+ secret: { secretName: env.registryCertificate.spec.secretName },
+ },
+ authcerts: {
+ secret: { secretName: env.authCertificate.spec.secretName },
+ },
+ },
+ containers_: {
+ registry: kube.Container("docker-registry") {
+ image: "registry:2",
+ args: ["/config/config.yml"],
+ volumeMounts_: {
+ config: { mountPath: "/config" },
+ certs: { mountPath: "/certs" },
+ authcerts: { mountPath: "/authcerts" },
+ },
+ env_: {
+ REGISTRY_STORAGE_S3_ACCESSKEY: { secretKeyRef: {
+ name: "rook-ceph-object-user-waw-hdd-redundant-1-object-registry",
+ key: "AccessKey"
+ }},
+ REGISTRY_STORAGE_S3_SECRETKEY: { secretKeyRef: {
+ name: "rook-ceph-object-user-waw-hdd-redundant-1-object-registry",
+ key: "SecretKey",
+ }},
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ registryService: kube.Service("docker-registry") {
+ metadata+: env.metadata("docker-registry"),
+ target_pod:: env.registryDeployment.spec.template,
+ spec+: {
+ type: "ClusterIP",
+ ports: [
+ { name: "registry", port: 5000, targetPort: 5000, protocol: "TCP" },
+ ],
+ }
+ },
+ registryIngress: kube.Ingress("registry") {
+ metadata+: env.metadata("registry") {
+ annotations+: {
+ "kubernetes.io/tls-acme": "true",
+ "certmanager.k8s.io/cluster-issuer": "letsencrypt-prod",
+ "nginx.ingress.kubernetes.io/backend-protocol": "HTTPS",
+ "nginx.ingress.kubernetes.io/proxy-body-size": "0",
+ },
+ },
+ spec+: {
+ tls: [
+ {
+ hosts: [cfg.domain],
+ secretName: "registry-tls",
+ },
+ ],
+ rules: [
+ {
+ host: cfg.domain,
+ http: {
+ paths: [
+ { path: "/auth", backend: env.authService.name_port },
+ { path: "/", backend: env.authService.name_port },
+ { path: "/v2/", backend: env.registryService.name_port },
+ ]
+ },
+ }
+ ],
+ },
+ },
+
+ registryStorageUser: kube._Object("ceph.rook.io/v1", "CephObjectStoreUser", "registry") {
+ metadata+: {
+ namespace: "ceph-waw1",
+ },
+ spec: {
+ store: "waw-hdd-redundant-1-object",
+ displayName: "docker-registry user",
+ },
+ },
+ }
+}