cluster/kube/lib/registry.libsonnet

# Deploy a Docker Registry in a cluster.

# This needs an oauth2 secret provisioned, create with:
#    kubectl -n registry create secret generic auth --from-literal=oauth2_secret=...
#    kubectl get secrets rook-ceph-object-user-<ceph-pool>-object-registry -n <ceph-namespace> -o yaml --export | kubectl replace -f - -n registry

local kube = import "../../../kube/kube.libsonnet";
local cm = import "cert-manager.libsonnet";

{
    Environment: {
        local env = self,
        local cfg = env.cfg,
        cfg:: {
            namespace: "registry",
            domain: error "domain must be set",
            storageClassName: error "storageClassName must be set",
            objectStoreName: error "objectStoreName must be set",
        },

        metadata(component):: {
            namespace: cfg.namespace,
            labels: {
                "app.kubernetes.io/name": "registry",
                "app.kubernetes.io/managed-by": "kubecfg",
                "app.kubernetes.io/component": component,
            },
        },

        namespace: kube.Namespace(cfg.namespace),

        registryIssuer: cm.Issuer("registry-issuer") {
            metadata+: env.metadata("registry-issuer"),
            spec: {
                selfSigned: {},
            },
        },
        authCertificate: cm.Certificate("auth") {
            metadata+: env.metadata("auth"),
            spec: {
                secretName: "auth-internal",
                duration: "43800h0m0s", // 5 years
                issuerRef: {
                    name: env.registryIssuer.metadata.name,
                },
                commonName: "auth.registry",
            },
        },
        registryCertificate: cm.Certificate("registry") {
            metadata+: env.metadata("registry"),
            spec: {
                secretName: "registry-internal",
                duration: "43800h0m0s", // 5 years
                issuerRef: {
                    name: env.registryIssuer.metadata.name,
                },
                commonName: "registry.registry",
            },
        },

        registryConfig: kube.ConfigMap("registry-config") {
            metadata+: env.metadata("registry-config"),
            data: {
                "config.yml": std.manifestYamlDoc({
                    version: "0.1",
                    log: {
                        level: "debug",
                        fields: {
                            service: "registry",
                        },
                    },
                    storage: {
                        cache: {
                            blobdescriptor: "inmemory",
                        },
                        s3: {
                            regionendpoint: "https://object.ceph-waw2.hswaw.net",
                            bucket: "registry",
                            region: "waw-hdd-redunant-2-object:default-placement",
                        },
                    },
                    http: {
                        addr: ":5000",
                        headers: {
                            "X-Content-Type-Options": ["nosniff"],
                        },
                        tls: {
                            certificate: "/certs/tls.crt",
                            key: "/certs/tls.key",
                        },
                        debug: {
                            addr: "localhost:5001",
                        },
                    },
                    health: {
                        storagedriver: {
                            enabled: true,
                            interval: "10s",
                            threshold: 3,
                        },
                    },
                    auth: {
                        token: {
                            realm: "https://%s/auth" % [cfg.domain],
                            service: "my.docker.registry",
                            issuer: "%s auth server" % [cfg.domain],
                            rootcertbundle: "/authcerts/tls.crt",
                        },
                    },
                }),
            },
        },

        authVolumeClaim: kube.PersistentVolumeClaim("auth-token-storage") {
            metadata+: env.metadata("auth-token-storage"),
            spec+: {
                storageClassName: cfg.storageClassName,
                accessModes: [ "ReadWriteOnce" ],
                resources: {
                    requests: {
                        storage: "1Gi",
                    },
                },
            },
        },

        authConfig: kube.ConfigMap("auth-config") {
            metadata+: env.metadata("auth-config"),
            data: {
                "auth_config.yml": std.manifestYamlDoc({
                    server: {
                        addr: ":5001",
                        certificate: "/certs/tls.crt",
                        key: "/certs/tls.key",
                    },
                    token: {
                        issuer: "%s auth server" % [cfg.domain],
                        expiration: 900,
                    },
                    oauth2: {
                        client_id: "registry",
                        client_secret_file: "/secrets/oauth2_secret",
                        authorize_url: "https://sso.hackerspace.pl/oauth/authorize",
                        access_token_url: "https://sso.hackerspace.pl/oauth/token",
                        profile_url: "https://sso.hackerspace.pl/api/1/profile",
                        redirect_url: "https://registry.k0.hswaw.net/oauth2",
                        username_key: "username",
                        token_db: "/data/oauth2_tokens.ldb",
                        registry_url: "https://registry.k0.hswaw.net",
                    },
                    users: {
                        [""]: {}, // '' user are anonymous users.
                    },
                    local data = self,
                    pushers:: [
                            { who: ["q3k", "informatic"], what: "vms/*" },
                            { who: ["q3k", "informatic"], what: "app/*" },
                            { who: ["q3k", "informatic"], what: "go/svc/*" },
                            { who: ["q3k"], what: "bgpwtf/*" },
                            { who: ["q3k"], what: "devtools/*" },
                            { who: ["q3k", "informatic"], what: "cluster/*" },
                    ],
                    acl: [
                        {
                            match: {account: "/.+/", name: "${account}/*"},
                            actions: ["*"],
                            comment: "Logged in users have full access to images that are in their 'namespace'",
                        },
                        {
                            match: {account: "/.+/", type: "registry", name: "catalog"},
                            actions: ["*"],
                            comment: "Logged in users can query the catalog.",
                        },
                        {
                            match: {account: ""},
                            actions: ["pull"],
                            comment: "Anyone can pull all images.",
                        },
                    ] + [
                        {
                            match: {
                                account: "/(%s)/" % std.join("|", p.who),
                                name: p.what,
                            },
                            actions: ["*"],
                            comment: "%s can push to %s" % [std.join(", ", p.who), p.what],
                        }
                        for p in data.pushers
                    ],
                }),
            }
        },

        authDeployment: kube.Deployment("auth") {
            metadata+: env.metadata("auth"),
            spec+: {
                replicas: 1,
                template+: {
                    spec+: {
                        volumes_: {
                            data: kube.PersistentVolumeClaimVolume(env.authVolumeClaim),
                            config: kube.ConfigMapVolume(env.authConfig),
                            certs: {
                                secret: { secretName: env.authCertificate.spec.secretName },
                            },
                            secrets: {
                                secret: { secretName: "auth" },
                            },
                        },
                        containers_: {
                            auth: kube.Container("auth") {
                                image: "informatic/docker_auth:2019040307",
                                volumeMounts_: {
                                    config: { mountPath: "/config" },
                                    certs: { mountPath: "/certs" },
                                    secrets: { mountPath: "/secrets" },
                                    data: { mountPath: "/data" },
                                },
                            },
                        },
                    },
                },
            },
        },
        authService: kube.Service("auth") {
            metadata+: env.metadata("auth"),
            target_pod:: env.authDeployment.spec.template,
            spec+: {
                type: "ClusterIP",
                ports: [
                    { name: "auth", port: 5001, targetPort: 5001, protocol: "TCP" },
                ],
            }
        },
        registryDeployment: kube.Deployment("docker-registry") {
            metadata+: env.metadata("docker-registry"),
            spec+: {
                replicas: 1,
                template+: {
                    spec+: {
                        volumes_: {
                            config: kube.ConfigMapVolume(env.registryConfig),
                            certs: {
                                secret: { secretName: env.registryCertificate.spec.secretName },
                            },
                            authcerts: {
                                secret: { secretName: env.authCertificate.spec.secretName },
                            },
                        },
                        containers_: {
                            registry: kube.Container("docker-registry") {
                                image: "registry:2.7.1",
                                args: ["/config/config.yml"],
                                volumeMounts_: {
                                    config: { mountPath: "/config" },
                                    certs: { mountPath: "/certs" },
                                    authcerts: { mountPath: "/authcerts" },
                                },
                                env_: {
                                    REGISTRY_STORAGE_S3_ACCESSKEY: { secretKeyRef: {
                                        name: "rook-ceph-object-user-%(objectStorageName)s-registry" % {objectStorageName: cfg.objectStorageName},
                                        key: "AccessKey"
                                    }},
                                    REGISTRY_STORAGE_S3_SECRETKEY: { secretKeyRef: {
                                        name: "rook-ceph-object-user-%(objectStorageName)s-registry" % {objectStorageName: cfg.objectStorageName},
                                        key: "SecretKey",
                                    }},
                                },
                            },
                        },
                    },
                },
            },
        },
        registryService: kube.Service("docker-registry") {
            metadata+: env.metadata("docker-registry"),
            target_pod:: env.registryDeployment.spec.template,
            spec+: {
                type: "ClusterIP",
                ports: [
                    { name: "registry", port: 5000, targetPort: 5000, protocol: "TCP" },
                ],
            }
        },
        registryIngress: kube.Ingress("registry") {
            metadata+: env.metadata("registry") {
                annotations+: {
                    "kubernetes.io/tls-acme": "true",
                    "certmanager.k8s.io/cluster-issuer": "letsencrypt-prod",
                    "nginx.ingress.kubernetes.io/backend-protocol": "HTTPS",
                    "nginx.ingress.kubernetes.io/proxy-body-size": "0",
                },
            },
            spec+: {
                tls: [
                    {
                        hosts: [cfg.domain],
                        secretName: "registry-tls",
                    },
                ],
                rules: [
                    {
                        host: cfg.domain,
                        http: {
                            paths: [
                                { path: "/auth", backend: env.authService.name_port },
                                { path: "/", backend: env.authService.name_port },
                                { path: "/v2/", backend: env.registryService.name_port },
                            ]
                        },
                    }
                ],
            },
        },

        registryStorageUser: kube._Object("ceph.rook.io/v1", "CephObjectStoreUser", "registry") {
            metadata+: {
                namespace: "ceph-waw2",
            },
            spec: {
                store: cfg.objectStorageName,
                displayName: "docker-registry user",
            },
        },
    }
}