app/registry/prod.jsonnet

# registry.k0.hswaw.net, a private docker registry
# This needs an oauth2 secret provisioned, create with:
#    kubectl -n registry create secret generic auth --from-literal=oauth2_secret=...
#    kubectl get secrets rook-ceph-object-user-waw-hdd-redundant-1-object-registry -n ceph-waw1 -o yaml --export | kubectl replace -f - -n registry

local kube = import "../../kube/kube.libsonnet";
local cm = import "../../cluster/kube/lib/cert-manager.libsonnet";

{
    local app = self,
    local cfg = app.cfg,
    cfg:: {
        namespace: "registry",
        domain: "k0.hswaw.net",
        storageClassName: "waw-hdd-redundant-1",
    },

    metadata(component):: {
        namespace: cfg.namespace,
        labels: {
            "app.kubernetes.io/name": "registry",
            "app.kubernetes.io/managed-by": "kubecfg",
            "app.kubernetes.io/component": component,
        },
    },

    namespace: kube.Namespace(cfg.namespace),

    registryIssuer: cm.Issuer("registry-issuer") {
        metadata+: app.metadata("registry-issuer"),
        spec: {
            selfSigned: {},
        },
    },
    authCertificate: cm.Certificate("auth") {
        metadata+: app.metadata("auth"),
        spec: {
            secretName: "auth-internal",
            duration: "43800h0m0s", // 5 years
            issuerRef: {
                name: app.registryIssuer.metadata.name,
            },
            commonName: "auth.registry",
        },
    },
    registryCertificate: cm.Certificate("registry") {
        metadata+: app.metadata("registry"),
        spec: {
            secretName: "registry-internal",
            duration: "43800h0m0s", // 5 years
            issuerRef: {
                name: app.registryIssuer.metadata.name,
            },
            commonName: "registry.registry",
        },
    },

    registryConfig: kube.ConfigMap("registry-config") {
        metadata+: app.metadata("registry-config"),
        data: {
            "config.yml": std.manifestYamlDoc({
                version: "0.1",
                log: {
                    fields: {
                        service: "registry",
                    },
                },
                storage: {
                    cache: {
                        blobdescriptor: "inmemory",
                    },
                    s3: {
                        regionendpoint: "https://object.ceph-waw1.hswaw.net",
                        bucket: "registry",
                        region: "waw-hdd-redunant-1-object:default-placement",
                    },
                },
                http: {
                    addr: ":5000",
                    headers: {
                        "X-Content-Type-Options": ["nosniff"],
                    },
                    tls: {
                        certificate: "/certs/tls.crt",
                        key: "/certs/tls.key",
                    },
                    debug: {
                        addr: "localhost:5001",
                    },
                },
                health: {
                    storagedriver: {
                        enabled: true,
                        interval: "10s",
                        threshold: 3,
                    },
                },
                auth: {
                    token: {
                        realm: "https://registry.%s/auth" % [cfg.domain],
                        service: "my.docker.registry",
                        issuer: "registry.%s auth server" % [cfg.domain],
                        rootcertbundle: "/authcerts/tls.crt",
                    },
                },
            }),
        },
    },

    authVolumeClaim: kube.PersistentVolumeClaim("auth-token-storage") {
        metadata+: app.metadata("auth-token-storage"),
        spec+: {
            storageClassName: cfg.storageClassName,
            accessModes: [ "ReadWriteOnce" ],
            resources: {
                requests: {
                    storage: "1Gi",
                },
            },
        },
    },

    authConfig: kube.ConfigMap("auth-config") {
        metadata+: app.metadata("auth-config"),
        data: {
            "auth_config.yml": std.manifestYamlDoc({
                server: {
                    addr: ":5001",
                    certificate: "/certs/tls.crt",
                    key: "/certs/tls.key",
                },
                token: {
                    issuer: "registry.%s auth server" % [cfg.domain],
                    expiration: 900,
                },
                oauth2: {
                    client_id: "registry",
                    client_secret_file: "/secrets/oauth2_secret",
                    authorize_url: "https://sso.hackerspace.pl/oauth/authorize",
                    access_token_url: "https://sso.hackerspace.pl/oauth/token",
                    profile_url: "https://sso.hackerspace.pl/api/1/profile",
                    redirect_url: "https://registry.k0.hswaw.net/oauth2",
                    username_key: "username",
                    token_db: "/data/oauth2_tokens.ldb",
                    registry_url: "https://registry.k0.hswaw.net",
                },
                users: {
                    [""]: {}, // '' user are anonymous users.
                },
                local data = self,
                pushers:: [
                        { who: ["q3k", "inf"], what: "vms/*" },
                        { who: ["q3k", "inf"], what: "app/*" },
                        { who: ["q3k", "inf"], what: "go/svc/*" },
                ],
                acl: [
                    {
                        match: {account: "/.+/", name: "${account}/*"},
                        actions: ["*"],
                        comment: "Logged in users have full access to images that are in their 'namespace'",
                    },
                    {
                        match: {account: "/.+/", type: "registry", name: "catalog"},
                        actions: ["*"],
                        comment: "Logged in users can query the catalog.",
                    },
                    {
                        match: {account: ""},
                        actions: ["pull"],
                        comment: "Anyone can pull all images.",
                    },
                ] + [
                    {
                        match: {
                            account: "/(%s)/" % std.join("|", p.who),
                            name: p.what,
                        },
                        actions: ["*"],
                        comment: "%s can push to %s" % [std.join(", ", p.who), p.what],
                    }
                    for p in data.pushers
                ],
            }),
        }
    },

    authDeployment: kube.Deployment("auth") {
        metadata+: app.metadata("auth"),
        spec+: {
            replicas: 1,
            template+: {
                spec+: {
                    volumes_: {
                        data: kube.PersistentVolumeClaimVolume(app.authVolumeClaim),
                        config: kube.ConfigMapVolume(app.authConfig),
                        certs: {
                            secret: { secretName: app.authCertificate.spec.secretName },
                        },
                        secrets: {
                            secret: { secretName: "auth" },
                        },
                    },
                    containers_: {
                        auth: kube.Container("auth") {
                            image: "informatic/docker_auth:2019040307",
                            volumeMounts_: {
                                config: { mountPath: "/config" },
                                certs: { mountPath: "/certs" },
                                secrets: { mountPath: "/secrets" },
                                data: { mountPath: "/data" },
                            },
                        },
                    },
                },
            },
        },
    },
    authService: kube.Service("auth") {
        metadata+: app.metadata("auth"),
        target_pod:: app.authDeployment.spec.template,
        spec+: {
            type: "ClusterIP",
            ports: [
                { name: "auth", port: 5001, targetPort: 5001, protocol: "TCP" },
            ],
        }
    },
    registryDeployment: kube.Deployment("docker-registry") {
        metadata+: app.metadata("docker-registry"),
        spec+: {
            replicas: 1,
            template+: {
                spec+: {
                    volumes_: {
                        config: kube.ConfigMapVolume(app.registryConfig),
                        certs: {
                            secret: { secretName: app.registryCertificate.spec.secretName },
                        },
                        authcerts: {
                            secret: { secretName: app.authCertificate.spec.secretName },
                        },
                    },
                    containers_: {
                        registry: kube.Container("docker-registry") {
                            image: "registry:2",
                            args: ["/config/config.yml"],
                            volumeMounts_: {
                                config: { mountPath: "/config" },
                                certs: { mountPath: "/certs" },
                                authcerts: { mountPath: "/authcerts" },
                            },
                            env_: {
                                REGISTRY_STORAGE_S3_ACCESSKEY: { secretKeyRef: {
                                    name: "rook-ceph-object-user-waw-hdd-redundant-1-object-registry",
                                    key: "AccessKey"
                                }},
                                REGISTRY_STORAGE_S3_SECRETKEY: { secretKeyRef: {
                                    name: "rook-ceph-object-user-waw-hdd-redundant-1-object-registry",
                                    key: "SecretKey",
                                }},
                            },
                        },
                    },
                },
            },
        },
    },
    registryService: kube.Service("docker-registry") {
        metadata+: app.metadata("docker-registry"),
        target_pod:: app.registryDeployment.spec.template,
        spec+: {
            type: "ClusterIP",
            ports: [
                { name: "registry", port: 5000, targetPort: 5000, protocol: "TCP" },
            ],
        }
    },
    registryIngress: kube.Ingress("registry") {
        metadata+: app.metadata("registry") {
            annotations+: {
                "kubernetes.io/tls-acme": "true",
                "certmanager.k8s.io/cluster-issuer": "letsencrypt-prod",
                "nginx.ingress.kubernetes.io/backend-protocol": "HTTPS",
                "nginx.ingress.kubernetes.io/proxy-body-size": "0",
            },
        },
        spec+: {
            tls: [
                {
                    hosts: ["registry.%s" % [cfg.domain]],
                    secretName: "registry-tls",
                },
            ],
            rules: [
                {
                    host: "registry.%s" % [cfg.domain],
                    http: {
                        paths: [
                            { path: "/auth", backend: app.authService.name_port },
                            { path: "/", backend: app.authService.name_port },
                            { path: "/v2/", backend: app.registryService.name_port },
                        ]
                    },
                }
            ],
        },
    },

    registryStorageUser: kube._Object("ceph.rook.io/v1", "CephObjectStoreUser", "registry") {
        metadata+: {
            namespace: "ceph-waw1",
        },
        spec: {
            store: "waw-hdd-redundant-1-object",
            displayName: "docker-registry user",
        },
    },
}