| # Deploy a Docker Registry in a cluster. |
| |
| # This needs an oauth2 secret provisioned, create with: |
| # kubectl -n registry create secret generic auth --from-literal=oauth2_secret=... |
| # kubectl get secrets rook-ceph-object-user-<ceph-pool>-object-registry -n <ceph-namespace> -o yaml --export | kubectl replace -f - -n registry |
| |
| local kube = import "../../../kube/kube.libsonnet"; |
| |
| { |
| Environment: { |
| local env = self, |
| local cfg = env.cfg, |
| cfg:: { |
| namespace: "registry", |
| domain: error "domain must be set", |
| storageClassName: error "storageClassName must be set", |
| objectStoreName: error "objectStoreName must be set", |
| }, |
| |
| metadata(component):: { |
| namespace: cfg.namespace, |
| labels: { |
| "app.kubernetes.io/name": "registry", |
| "app.kubernetes.io/managed-by": "kubecfg", |
| "app.kubernetes.io/component": component, |
| }, |
| }, |
| |
| namespace: kube.Namespace(cfg.namespace), |
| |
| registryIssuer: kube.Issuer("registry-issuer") { |
| metadata+: env.metadata("registry-issuer"), |
| spec: { |
| selfSigned: {}, |
| }, |
| }, |
| authCertificate: kube.Certificate("auth") { |
| metadata+: env.metadata("auth"), |
| spec: { |
| secretName: "auth-internal", |
| duration: "43800h0m0s", // 5 years |
| issuerRef: { |
| name: env.registryIssuer.metadata.name, |
| }, |
| commonName: "auth.registry", |
| }, |
| }, |
| registryCertificate: kube.Certificate("registry") { |
| metadata+: env.metadata("registry"), |
| spec: { |
| secretName: "registry-internal", |
| duration: "43800h0m0s", // 5 years |
| issuerRef: { |
| name: env.registryIssuer.metadata.name, |
| }, |
| commonName: "registry.registry", |
| }, |
| }, |
| |
| registryConfig: kube.ConfigMap("registry-config") { |
| metadata+: env.metadata("registry-config"), |
| data: { |
| "config.yml": std.manifestYamlDoc({ |
| version: "0.1", |
| log: { |
| level: "debug", |
| fields: { |
| service: "registry", |
| }, |
| }, |
| storage: { |
| cache: { |
| blobdescriptor: "inmemory", |
| }, |
| s3: { |
| regionendpoint: "https://object.ceph-waw2.hswaw.net", |
| bucket: "registry", |
| region: "waw-hdd-redunant-2-object:default-placement", |
| }, |
| }, |
| http: { |
| addr: ":5000", |
| headers: { |
| "X-Content-Type-Options": ["nosniff"], |
| }, |
| tls: { |
| certificate: "/certs/tls.crt", |
| key: "/certs/tls.key", |
| }, |
| debug: { |
| addr: "localhost:5001", |
| }, |
| }, |
| health: { |
| storagedriver: { |
| enabled: true, |
| interval: "10s", |
| threshold: 3, |
| }, |
| }, |
| auth: { |
| token: { |
| realm: "https://%s/auth" % [cfg.domain], |
| service: "my.docker.registry", |
| issuer: "%s auth server" % [cfg.domain], |
| rootcertbundle: "/authcerts/tls.crt", |
| }, |
| }, |
| }), |
| }, |
| }, |
| |
| authVolumeClaim: kube.PersistentVolumeClaim("auth-token-storage") { |
| metadata+: env.metadata("auth-token-storage"), |
| spec+: { |
| storageClassName: cfg.storageClassName, |
| accessModes: [ "ReadWriteOnce" ], |
| resources: { |
| requests: { |
| storage: "1Gi", |
| }, |
| }, |
| }, |
| }, |
| |
| authConfig: kube.ConfigMap("auth-config") { |
| metadata+: env.metadata("auth-config"), |
| data: { |
| "auth_config.yml": std.manifestYamlDoc({ |
| server: { |
| addr: ":5001", |
| certificate: "/certs/tls.crt", |
| key: "/certs/tls.key", |
| }, |
| token: { |
| issuer: "%s auth server" % [cfg.domain], |
| expiration: 900, |
| }, |
| oauth2: { |
| client_id: "registry", |
| client_secret_file: "/secrets/oauth2_secret", |
| authorize_url: "https://sso.hackerspace.pl/oauth/authorize", |
| access_token_url: "https://sso.hackerspace.pl/oauth/token", |
| profile_url: "https://sso.hackerspace.pl/api/1/profile", |
| redirect_url: "https://registry.k0.hswaw.net/oauth2", |
| username_key: "username", |
| token_db: "/data/oauth2_tokens.ldb", |
| registry_url: "https://registry.k0.hswaw.net", |
| }, |
| users: { |
| [""]: {}, // '' user are anonymous users. |
| }, |
| local data = self, |
| pushers:: [ |
| { who: ["q3k", "informatic"], what: "vms/*" }, |
| { who: ["q3k", "informatic"], what: "app/*" }, |
| { who: ["q3k", "informatic"], what: "go/svc/*" }, |
| { who: ["q3k"], what: "bgpwtf/*" }, |
| { who: ["q3k"], what: "devtools/*" }, |
| { who: ["q3k", "informatic"], what: "cluster/*" }, |
| ], |
| acl: [ |
| { |
| match: {account: "/.+/", name: "${account}/*"}, |
| actions: ["*"], |
| comment: "Logged in users have full access to images that are in their 'namespace'", |
| }, |
| { |
| match: {account: "/.+/", type: "registry", name: "catalog"}, |
| actions: ["*"], |
| comment: "Logged in users can query the catalog.", |
| }, |
| { |
| match: {account: "/.*/"}, |
| actions: ["pull"], |
| comment: "Anyone can pull all images.", |
| }, |
| ] + [ |
| { |
| match: { |
| account: "/(%s)/" % std.join("|", p.who), |
| name: p.what, |
| }, |
| actions: ["*"], |
| comment: "%s can push to %s" % [std.join(", ", p.who), p.what], |
| } |
| for p in data.pushers |
| ], |
| }), |
| } |
| }, |
| |
| authDeployment: kube.Deployment("auth") { |
| metadata+: env.metadata("auth"), |
| spec+: { |
| replicas: 1, |
| template+: { |
| spec+: { |
| volumes_: { |
| data: kube.PersistentVolumeClaimVolume(env.authVolumeClaim), |
| config: kube.ConfigMapVolume(env.authConfig), |
| certs: { |
| secret: { secretName: env.authCertificate.spec.secretName }, |
| }, |
| secrets: { |
| secret: { secretName: "auth" }, |
| }, |
| }, |
| containers_: { |
| auth: kube.Container("auth") { |
| image: "informatic/docker_auth:2019040307", |
| volumeMounts_: { |
| config: { mountPath: "/config" }, |
| certs: { mountPath: "/certs" }, |
| secrets: { mountPath: "/secrets" }, |
| data: { mountPath: "/data" }, |
| }, |
| }, |
| }, |
| }, |
| }, |
| }, |
| }, |
| authService: kube.Service("auth") { |
| metadata+: env.metadata("auth"), |
| target_pod:: env.authDeployment.spec.template, |
| spec+: { |
| type: "ClusterIP", |
| ports: [ |
| { name: "auth", port: 5001, targetPort: 5001, protocol: "TCP" }, |
| ], |
| } |
| }, |
| registryDeployment: kube.Deployment("docker-registry") { |
| metadata+: env.metadata("docker-registry"), |
| spec+: { |
| replicas: 1, |
| template+: { |
| spec+: { |
| volumes_: { |
| config: kube.ConfigMapVolume(env.registryConfig), |
| certs: { |
| secret: { secretName: env.registryCertificate.spec.secretName }, |
| }, |
| authcerts: { |
| secret: { secretName: env.authCertificate.spec.secretName }, |
| }, |
| }, |
| containers_: { |
| registry: kube.Container("docker-registry") { |
| image: "registry:2.7.1", |
| args: ["/config/config.yml"], |
| volumeMounts_: { |
| config: { mountPath: "/config" }, |
| certs: { mountPath: "/certs" }, |
| authcerts: { mountPath: "/authcerts" }, |
| }, |
| env_: { |
| REGISTRY_STORAGE_S3_ACCESSKEY: { secretKeyRef: { |
| name: "rook-ceph-object-user-%(objectStorageName)s-registry" % {objectStorageName: cfg.objectStorageName}, |
| key: "AccessKey" |
| }}, |
| REGISTRY_STORAGE_S3_SECRETKEY: { secretKeyRef: { |
| name: "rook-ceph-object-user-%(objectStorageName)s-registry" % {objectStorageName: cfg.objectStorageName}, |
| key: "SecretKey", |
| }}, |
| }, |
| }, |
| }, |
| }, |
| }, |
| }, |
| }, |
| registryService: kube.Service("docker-registry") { |
| metadata+: env.metadata("docker-registry"), |
| target_pod:: env.registryDeployment.spec.template, |
| spec+: { |
| type: "ClusterIP", |
| ports: [ |
| { name: "registry", port: 5000, targetPort: 5000, protocol: "TCP" }, |
| ], |
| } |
| }, |
| registryIngress: kube.Ingress("registry") { |
| metadata+: env.metadata("registry") { |
| annotations+: { |
| "kubernetes.io/tls-acme": "true", |
| "certmanager.k8s.io/cluster-issuer": "letsencrypt-prod", |
| "nginx.ingress.kubernetes.io/backend-protocol": "HTTPS", |
| "nginx.ingress.kubernetes.io/proxy-body-size": "0", |
| }, |
| }, |
| spec+: { |
| tls: [ |
| { |
| hosts: [cfg.domain], |
| secretName: "registry-tls", |
| }, |
| ], |
| rules: [ |
| { |
| host: cfg.domain, |
| http: { |
| paths: [ |
| { path: "/auth", backend: env.authService.name_port }, |
| { path: "/", backend: env.authService.name_port }, |
| { path: "/v2/", backend: env.registryService.name_port }, |
| ] |
| }, |
| } |
| ], |
| }, |
| }, |
| |
| registryStorageUser: kube.CephObjectStoreUser("registry") { |
| metadata+: { |
| namespace: "ceph-waw2", |
| }, |
| spec: { |
| store: cfg.objectStorageName, |
| displayName: "docker-registry user", |
| }, |
| }, |
| } |
| } |