blob: f2916073cdbc8820d2475f46828e527cf8569980 [file] [log] [blame]
# Deploy a Docker Registry in a cluster.
# This needs an oauth2 secret provisioned, create with:
# kubectl -n registry create secret generic auth --from-literal=oauth2_secret=...
# kubectl get secrets rook-ceph-object-user-<ceph-pool>-object-registry -n <ceph-namespace> -o yaml --export | kubectl replace -f - -n registry
local kube = import "../../../kube/kube.libsonnet";
{
Environment: {
local env = self,
local cfg = env.cfg,
cfg:: {
namespace: "registry",
domain: error "domain must be set",
storageClassName: error "storageClassName must be set",
objectStoreName: error "objectStoreName must be set",
},
metadata(component):: {
namespace: cfg.namespace,
labels: {
"app.kubernetes.io/name": "registry",
"app.kubernetes.io/managed-by": "kubecfg",
"app.kubernetes.io/component": component,
},
},
namespace: kube.Namespace(cfg.namespace),
registryIssuer: kube.Issuer("registry-issuer") {
metadata+: env.metadata("registry-issuer"),
spec: {
selfSigned: {},
},
},
authCertificate: kube.Certificate("auth") {
metadata+: env.metadata("auth"),
spec: {
secretName: "auth-internal",
duration: "43800h0m0s", // 5 years
issuerRef: {
name: env.registryIssuer.metadata.name,
},
commonName: "auth.registry",
},
},
registryCertificate: kube.Certificate("registry") {
metadata+: env.metadata("registry"),
spec: {
secretName: "registry-internal",
duration: "43800h0m0s", // 5 years
issuerRef: {
name: env.registryIssuer.metadata.name,
},
commonName: "registry.registry",
},
},
registryConfig: kube.ConfigMap("registry-config") {
metadata+: env.metadata("registry-config"),
data: {
"config.yml": std.manifestYamlDoc({
version: "0.1",
log: {
level: "debug",
fields: {
service: "registry",
},
},
storage: {
cache: {
blobdescriptor: "inmemory",
},
s3: {
regionendpoint: "https://object.ceph-waw3.hswaw.net",
bucket: "registry",
region: "waw-hdd-redunant-3-object:default-placement",
},
},
http: {
addr: ":5000",
headers: {
"X-Content-Type-Options": ["nosniff"],
},
tls: {
certificate: "/certs/tls.crt",
key: "/certs/tls.key",
},
debug: {
addr: "localhost:5001",
},
},
health: {
storagedriver: {
enabled: true,
interval: "10s",
threshold: 3,
},
},
auth: {
token: {
realm: "https://%s/auth" % [cfg.domain],
service: "my.docker.registry",
issuer: "%s auth server" % [cfg.domain],
rootcertbundle: "/authcerts/tls.crt",
},
},
}),
},
},
authVolumeClaim: kube.PersistentVolumeClaim("auth-token-storage-3") {
metadata+: env.metadata("auth-token-storage-3"),
storage: "1Gi",
storageClass: cfg.storageClassName,
},
authConfig: kube.ConfigMap("auth-config") {
metadata+: env.metadata("auth-config"),
data: {
"auth_config.yml": std.manifestYamlDoc({
server: {
addr: ":5001",
certificate: "/certs/tls.crt",
key: "/certs/tls.key",
},
token: {
issuer: "%s auth server" % [cfg.domain],
expiration: 900,
},
oauth2: {
client_id: "registry",
client_secret_file: "/secrets/oauth2_secret",
authorize_url: "https://sso.hackerspace.pl/oauth/authorize",
access_token_url: "https://sso.hackerspace.pl/oauth/token",
profile_url: "https://sso.hackerspace.pl/api/1/profile",
redirect_url: "https://registry.k0.hswaw.net/oauth2",
username_key: "username",
token_db: "/data/oauth2_tokens.ldb",
registry_url: "https://registry.k0.hswaw.net",
},
users: {
[""]: {}, // '' user are anonymous users.
},
local data = self,
pushers:: [
{ who: ["q3k", "informatic"], what: "vms/*" },
{ who: ["q3k", "informatic"], what: "app/*" },
{ who: ["q3k", "informatic"], what: "go/svc/*" },
{ who: ["q3k"], what: "bgpwtf/*" },
{ who: ["q3k"], what: "devtools/*" },
{ who: ["q3k"], what: "games/factorio/*" },
{ who: ["q3k", "informatic"], what: "cluster/*" },
],
acl: [
{
match: {
account: "/(%s)/" % std.join("|", p.who),
name: p.what,
},
actions: ["*"],
comment: "%s can push to %s" % [std.join(", ", p.who), p.what],
}
for p in data.pushers
] + [
{
match: {account: "/.+/", name: "${account}/*"},
actions: ["*"],
comment: "Logged in users have full access to images that are in their 'namespace'",
},
{
match: {account: "/.+/", type: "registry", name: "catalog"},
actions: ["*"],
comment: "Logged in users can query the catalog.",
},
{
match: {account: "/.*/"},
actions: ["pull"],
comment: "Anyone can pull all images.",
},
],
}),
}
},
authDeployment: kube.Deployment("auth") {
metadata+: env.metadata("auth"),
spec+: {
replicas: 1,
template+: {
spec+: {
volumes_: {
data: kube.PersistentVolumeClaimVolume(env.authVolumeClaim),
config: kube.ConfigMapVolume(env.authConfig),
certs: {
secret: { secretName: env.authCertificate.spec.secretName },
},
secrets: {
secret: { secretName: "auth" },
},
},
containers_: {
auth: kube.Container("auth") {
image: "informatic/docker_auth:2019040307",
volumeMounts_: {
config: { mountPath: "/config" },
certs: { mountPath: "/certs" },
secrets: { mountPath: "/secrets" },
data: { mountPath: "/data" },
},
},
},
},
},
},
},
authService: kube.Service("auth") {
metadata+: env.metadata("auth"),
target:: env.authDeployment,
spec+: {
type: "ClusterIP",
ports: [
{ name: "auth", port: 5001, targetPort: 5001, protocol: "TCP" },
],
}
},
registryDeployment: kube.Deployment("docker-registry") {
metadata+: env.metadata("docker-registry"),
spec+: {
replicas: 1,
template+: {
spec+: {
volumes_: {
config: kube.ConfigMapVolume(env.registryConfig),
certs: {
secret: { secretName: env.registryCertificate.spec.secretName },
},
authcerts: {
secret: { secretName: env.authCertificate.spec.secretName },
},
},
containers_: {
registry: kube.Container("docker-registry") {
image: "registry:2.7.1",
args: ["/config/config.yml"],
volumeMounts_: {
config: { mountPath: "/config" },
certs: { mountPath: "/certs" },
authcerts: { mountPath: "/authcerts" },
},
env_: {
REGISTRY_STORAGE_S3_ACCESSKEY: { secretKeyRef: {
name: "rook-ceph-object-user-%(objectStorageName)s-registry" % {objectStorageName: cfg.objectStorageName},
key: "AccessKey"
}},
REGISTRY_STORAGE_S3_SECRETKEY: { secretKeyRef: {
name: "rook-ceph-object-user-%(objectStorageName)s-registry" % {objectStorageName: cfg.objectStorageName},
key: "SecretKey",
}},
},
},
},
},
},
},
},
registryService: kube.Service("docker-registry") {
metadata+: env.metadata("docker-registry"),
target:: env.registryDeployment,
spec+: {
type: "ClusterIP",
ports: [
{ name: "registry", port: 5000, targetPort: 5000, protocol: "TCP" },
],
}
},
registryIngress: kube.Ingress("registry") {
metadata+: env.metadata("registry") {
annotations+: {
"kubernetes.io/tls-acme": "true",
"cert-manager.io/cluster-issuer": "letsencrypt-prod",
"nginx.ingress.kubernetes.io/backend-protocol": "HTTPS",
"nginx.ingress.kubernetes.io/proxy-body-size": "0",
},
},
spec+: {
tls: [
{
hosts: [cfg.domain],
secretName: "registry-tls",
},
],
rules: [
{
host: cfg.domain,
http: {
paths: [
{ path: "/auth", backend: env.authService.name_port },
{ path: "/", backend: env.authService.name_port },
{ path: "/v2/", backend: env.registryService.name_port },
]
},
}
],
},
},
registryStorageUser: kube.CephObjectStoreUser("registry") {
metadata+: {
namespace: "ceph-waw3",
},
spec: {
store: cfg.objectStorageName,
displayName: "docker-registry user",
},
},
}
}