Sergiusz Bazanski | 4d61d20 | 2019-07-21 16:56:41 +0200 | [diff] [blame] | 1 | # Deploy a Docker Registry in a cluster. |
| 2 | |
| 3 | # This needs an oauth2 secret provisioned, create with: |
| 4 | # kubectl -n registry create secret generic auth --from-literal=oauth2_secret=... |
| 5 | # kubectl get secrets rook-ceph-object-user-<ceph-pool>-object-registry -n <ceph-namespace> -o yaml --export | kubectl replace -f - -n registry |
| 6 | |
| 7 | local kube = import "../../../kube/kube.libsonnet"; |
Sergiusz Bazanski | 4d61d20 | 2019-07-21 16:56:41 +0200 | [diff] [blame] | 8 | |
| 9 | { |
| 10 | Environment: { |
| 11 | local env = self, |
| 12 | local cfg = env.cfg, |
| 13 | cfg:: { |
| 14 | namespace: "registry", |
| 15 | domain: error "domain must be set", |
| 16 | storageClassName: error "storageClassName must be set", |
Sergiusz Bazanski | d07861b | 2019-08-08 17:48:25 +0200 | [diff] [blame] | 17 | objectStoreName: error "objectStoreName must be set", |
Sergiusz Bazanski | 4d61d20 | 2019-07-21 16:56:41 +0200 | [diff] [blame] | 18 | }, |
| 19 | |
| 20 | metadata(component):: { |
| 21 | namespace: cfg.namespace, |
| 22 | labels: { |
| 23 | "app.kubernetes.io/name": "registry", |
| 24 | "app.kubernetes.io/managed-by": "kubecfg", |
| 25 | "app.kubernetes.io/component": component, |
| 26 | }, |
| 27 | }, |
| 28 | |
| 29 | namespace: kube.Namespace(cfg.namespace), |
| 30 | |
Sergiusz Bazanski | e31d64f | 2019-10-02 20:59:26 +0200 | [diff] [blame] | 31 | registryIssuer: kube.Issuer("registry-issuer") { |
Sergiusz Bazanski | 4d61d20 | 2019-07-21 16:56:41 +0200 | [diff] [blame] | 32 | metadata+: env.metadata("registry-issuer"), |
| 33 | spec: { |
| 34 | selfSigned: {}, |
| 35 | }, |
| 36 | }, |
Sergiusz Bazanski | e31d64f | 2019-10-02 20:59:26 +0200 | [diff] [blame] | 37 | authCertificate: kube.Certificate("auth") { |
Sergiusz Bazanski | 4d61d20 | 2019-07-21 16:56:41 +0200 | [diff] [blame] | 38 | metadata+: env.metadata("auth"), |
| 39 | spec: { |
| 40 | secretName: "auth-internal", |
| 41 | duration: "43800h0m0s", // 5 years |
| 42 | issuerRef: { |
| 43 | name: env.registryIssuer.metadata.name, |
| 44 | }, |
| 45 | commonName: "auth.registry", |
| 46 | }, |
| 47 | }, |
Sergiusz Bazanski | e31d64f | 2019-10-02 20:59:26 +0200 | [diff] [blame] | 48 | registryCertificate: kube.Certificate("registry") { |
Sergiusz Bazanski | 4d61d20 | 2019-07-21 16:56:41 +0200 | [diff] [blame] | 49 | metadata+: env.metadata("registry"), |
| 50 | spec: { |
| 51 | secretName: "registry-internal", |
| 52 | duration: "43800h0m0s", // 5 years |
| 53 | issuerRef: { |
| 54 | name: env.registryIssuer.metadata.name, |
| 55 | }, |
| 56 | commonName: "registry.registry", |
| 57 | }, |
| 58 | }, |
| 59 | |
| 60 | registryConfig: kube.ConfigMap("registry-config") { |
| 61 | metadata+: env.metadata("registry-config"), |
| 62 | data: { |
| 63 | "config.yml": std.manifestYamlDoc({ |
| 64 | version: "0.1", |
| 65 | log: { |
Sergiusz Bazanski | 5f3a5e0 | 2019-09-25 02:51:51 +0200 | [diff] [blame] | 66 | level: "debug", |
Sergiusz Bazanski | 4d61d20 | 2019-07-21 16:56:41 +0200 | [diff] [blame] | 67 | fields: { |
| 68 | service: "registry", |
| 69 | }, |
| 70 | }, |
| 71 | storage: { |
| 72 | cache: { |
| 73 | blobdescriptor: "inmemory", |
| 74 | }, |
| 75 | s3: { |
Serge Bazanski | 3d29484 | 2020-08-04 01:34:07 +0200 | [diff] [blame] | 76 | regionendpoint: "https://object.ceph-waw3.hswaw.net", |
Sergiusz Bazanski | 4d61d20 | 2019-07-21 16:56:41 +0200 | [diff] [blame] | 77 | bucket: "registry", |
Serge Bazanski | 3d29484 | 2020-08-04 01:34:07 +0200 | [diff] [blame] | 78 | region: "waw-hdd-redunant-3-object:default-placement", |
Sergiusz Bazanski | 4d61d20 | 2019-07-21 16:56:41 +0200 | [diff] [blame] | 79 | }, |
| 80 | }, |
| 81 | http: { |
| 82 | addr: ":5000", |
| 83 | headers: { |
| 84 | "X-Content-Type-Options": ["nosniff"], |
| 85 | }, |
| 86 | tls: { |
| 87 | certificate: "/certs/tls.crt", |
| 88 | key: "/certs/tls.key", |
| 89 | }, |
| 90 | debug: { |
| 91 | addr: "localhost:5001", |
| 92 | }, |
| 93 | }, |
| 94 | health: { |
| 95 | storagedriver: { |
| 96 | enabled: true, |
| 97 | interval: "10s", |
| 98 | threshold: 3, |
| 99 | }, |
| 100 | }, |
| 101 | auth: { |
| 102 | token: { |
| 103 | realm: "https://%s/auth" % [cfg.domain], |
| 104 | service: "my.docker.registry", |
| 105 | issuer: "%s auth server" % [cfg.domain], |
| 106 | rootcertbundle: "/authcerts/tls.crt", |
| 107 | }, |
| 108 | }, |
| 109 | }), |
| 110 | }, |
| 111 | }, |
| 112 | |
Serge Bazanski | 3d29484 | 2020-08-04 01:34:07 +0200 | [diff] [blame] | 113 | authVolumeClaim: kube.PersistentVolumeClaim("auth-token-storage-3") { |
| 114 | metadata+: env.metadata("auth-token-storage-3"), |
radex | 36964dc | 2023-11-24 11:19:46 +0100 | [diff] [blame] | 115 | storage:: "1Gi", |
| 116 | storageClass:: cfg.storageClassName, |
Sergiusz Bazanski | 4d61d20 | 2019-07-21 16:56:41 +0200 | [diff] [blame] | 117 | }, |
| 118 | |
| 119 | authConfig: kube.ConfigMap("auth-config") { |
| 120 | metadata+: env.metadata("auth-config"), |
| 121 | data: { |
| 122 | "auth_config.yml": std.manifestYamlDoc({ |
| 123 | server: { |
| 124 | addr: ":5001", |
| 125 | certificate: "/certs/tls.crt", |
| 126 | key: "/certs/tls.key", |
| 127 | }, |
| 128 | token: { |
| 129 | issuer: "%s auth server" % [cfg.domain], |
| 130 | expiration: 900, |
| 131 | }, |
| 132 | oauth2: { |
| 133 | client_id: "registry", |
| 134 | client_secret_file: "/secrets/oauth2_secret", |
| 135 | authorize_url: "https://sso.hackerspace.pl/oauth/authorize", |
| 136 | access_token_url: "https://sso.hackerspace.pl/oauth/token", |
| 137 | profile_url: "https://sso.hackerspace.pl/api/1/profile", |
| 138 | redirect_url: "https://registry.k0.hswaw.net/oauth2", |
| 139 | username_key: "username", |
| 140 | token_db: "/data/oauth2_tokens.ldb", |
| 141 | registry_url: "https://registry.k0.hswaw.net", |
| 142 | }, |
| 143 | users: { |
| 144 | [""]: {}, // '' user are anonymous users. |
| 145 | }, |
| 146 | local data = self, |
| 147 | pushers:: [ |
Sergiusz Bazanski | b13b7ff | 2019-08-29 20:12:24 +0200 | [diff] [blame] | 148 | { who: ["q3k", "informatic"], what: "vms/*" }, |
| 149 | { who: ["q3k", "informatic"], what: "app/*" }, |
| 150 | { who: ["q3k", "informatic"], what: "go/svc/*" }, |
Sergiusz Bazanski | d07861b | 2019-08-08 17:48:25 +0200 | [diff] [blame] | 151 | { who: ["q3k"], what: "bgpwtf/*" }, |
| 152 | { who: ["q3k"], what: "devtools/*" }, |
Sergiusz Bazanski | 0581bbf | 2020-05-11 03:21:32 +0200 | [diff] [blame] | 153 | { who: ["q3k"], what: "games/factorio/*" }, |
Sergiusz Bazanski | b13b7ff | 2019-08-29 20:12:24 +0200 | [diff] [blame] | 154 | { who: ["q3k", "informatic"], what: "cluster/*" }, |
Sergiusz Bazanski | 4d61d20 | 2019-07-21 16:56:41 +0200 | [diff] [blame] | 155 | ], |
| 156 | acl: [ |
| 157 | { |
Piotr Dobrowolski | 10384cd | 2023-06-20 00:42:15 +0200 | [diff] [blame] | 158 | match: { |
| 159 | account: "/(%s)/" % std.join("|", p.who), |
| 160 | name: p.what, |
| 161 | }, |
| 162 | actions: ["*"], |
| 163 | comment: "%s can push to %s" % [std.join(", ", p.who), p.what], |
| 164 | } |
| 165 | for p in data.pushers |
| 166 | ] + [ |
| 167 | { |
Sergiusz Bazanski | 4d61d20 | 2019-07-21 16:56:41 +0200 | [diff] [blame] | 168 | match: {account: "/.+/", name: "${account}/*"}, |
| 169 | actions: ["*"], |
| 170 | comment: "Logged in users have full access to images that are in their 'namespace'", |
| 171 | }, |
| 172 | { |
| 173 | match: {account: "/.+/", type: "registry", name: "catalog"}, |
| 174 | actions: ["*"], |
| 175 | comment: "Logged in users can query the catalog.", |
| 176 | }, |
| 177 | { |
Piotr Dobrowolski | 0697e01 | 2020-07-02 18:32:24 +0200 | [diff] [blame] | 178 | match: {account: "/.*/"}, |
Sergiusz Bazanski | 4d61d20 | 2019-07-21 16:56:41 +0200 | [diff] [blame] | 179 | actions: ["pull"], |
| 180 | comment: "Anyone can pull all images.", |
| 181 | }, |
Sergiusz Bazanski | 4d61d20 | 2019-07-21 16:56:41 +0200 | [diff] [blame] | 182 | ], |
| 183 | }), |
| 184 | } |
| 185 | }, |
| 186 | |
| 187 | authDeployment: kube.Deployment("auth") { |
| 188 | metadata+: env.metadata("auth"), |
| 189 | spec+: { |
| 190 | replicas: 1, |
| 191 | template+: { |
| 192 | spec+: { |
| 193 | volumes_: { |
| 194 | data: kube.PersistentVolumeClaimVolume(env.authVolumeClaim), |
| 195 | config: kube.ConfigMapVolume(env.authConfig), |
| 196 | certs: { |
| 197 | secret: { secretName: env.authCertificate.spec.secretName }, |
| 198 | }, |
| 199 | secrets: { |
| 200 | secret: { secretName: "auth" }, |
| 201 | }, |
| 202 | }, |
| 203 | containers_: { |
| 204 | auth: kube.Container("auth") { |
| 205 | image: "informatic/docker_auth:2019040307", |
| 206 | volumeMounts_: { |
| 207 | config: { mountPath: "/config" }, |
| 208 | certs: { mountPath: "/certs" }, |
| 209 | secrets: { mountPath: "/secrets" }, |
| 210 | data: { mountPath: "/data" }, |
| 211 | }, |
| 212 | }, |
| 213 | }, |
| 214 | }, |
| 215 | }, |
| 216 | }, |
| 217 | }, |
| 218 | authService: kube.Service("auth") { |
| 219 | metadata+: env.metadata("auth"), |
radex | 8b8f387 | 2023-11-24 11:09:46 +0100 | [diff] [blame] | 220 | target:: env.authDeployment, |
Sergiusz Bazanski | 4d61d20 | 2019-07-21 16:56:41 +0200 | [diff] [blame] | 221 | spec+: { |
| 222 | type: "ClusterIP", |
| 223 | ports: [ |
| 224 | { name: "auth", port: 5001, targetPort: 5001, protocol: "TCP" }, |
| 225 | ], |
| 226 | } |
| 227 | }, |
| 228 | registryDeployment: kube.Deployment("docker-registry") { |
| 229 | metadata+: env.metadata("docker-registry"), |
| 230 | spec+: { |
| 231 | replicas: 1, |
| 232 | template+: { |
| 233 | spec+: { |
| 234 | volumes_: { |
| 235 | config: kube.ConfigMapVolume(env.registryConfig), |
| 236 | certs: { |
| 237 | secret: { secretName: env.registryCertificate.spec.secretName }, |
| 238 | }, |
| 239 | authcerts: { |
| 240 | secret: { secretName: env.authCertificate.spec.secretName }, |
| 241 | }, |
| 242 | }, |
| 243 | containers_: { |
| 244 | registry: kube.Container("docker-registry") { |
Sergiusz Bazanski | 5f3a5e0 | 2019-09-25 02:51:51 +0200 | [diff] [blame] | 245 | image: "registry:2.7.1", |
Sergiusz Bazanski | 4d61d20 | 2019-07-21 16:56:41 +0200 | [diff] [blame] | 246 | args: ["/config/config.yml"], |
| 247 | volumeMounts_: { |
| 248 | config: { mountPath: "/config" }, |
| 249 | certs: { mountPath: "/certs" }, |
| 250 | authcerts: { mountPath: "/authcerts" }, |
| 251 | }, |
| 252 | env_: { |
| 253 | REGISTRY_STORAGE_S3_ACCESSKEY: { secretKeyRef: { |
Sergiusz Bazanski | d07861b | 2019-08-08 17:48:25 +0200 | [diff] [blame] | 254 | name: "rook-ceph-object-user-%(objectStorageName)s-registry" % {objectStorageName: cfg.objectStorageName}, |
Sergiusz Bazanski | 4d61d20 | 2019-07-21 16:56:41 +0200 | [diff] [blame] | 255 | key: "AccessKey" |
| 256 | }}, |
| 257 | REGISTRY_STORAGE_S3_SECRETKEY: { secretKeyRef: { |
Sergiusz Bazanski | d07861b | 2019-08-08 17:48:25 +0200 | [diff] [blame] | 258 | name: "rook-ceph-object-user-%(objectStorageName)s-registry" % {objectStorageName: cfg.objectStorageName}, |
Sergiusz Bazanski | 4d61d20 | 2019-07-21 16:56:41 +0200 | [diff] [blame] | 259 | key: "SecretKey", |
| 260 | }}, |
| 261 | }, |
| 262 | }, |
| 263 | }, |
| 264 | }, |
| 265 | }, |
| 266 | }, |
| 267 | }, |
| 268 | registryService: kube.Service("docker-registry") { |
| 269 | metadata+: env.metadata("docker-registry"), |
radex | 8b8f387 | 2023-11-24 11:09:46 +0100 | [diff] [blame] | 270 | target:: env.registryDeployment, |
Sergiusz Bazanski | 4d61d20 | 2019-07-21 16:56:41 +0200 | [diff] [blame] | 271 | spec+: { |
| 272 | type: "ClusterIP", |
| 273 | ports: [ |
| 274 | { name: "registry", port: 5000, targetPort: 5000, protocol: "TCP" }, |
| 275 | ], |
| 276 | } |
| 277 | }, |
| 278 | registryIngress: kube.Ingress("registry") { |
| 279 | metadata+: env.metadata("registry") { |
| 280 | annotations+: { |
| 281 | "kubernetes.io/tls-acme": "true", |
Piotr Dobrowolski | 7e84106 | 2023-04-23 11:36:15 +0200 | [diff] [blame] | 282 | "cert-manager.io/cluster-issuer": "letsencrypt-prod", |
Sergiusz Bazanski | 4d61d20 | 2019-07-21 16:56:41 +0200 | [diff] [blame] | 283 | "nginx.ingress.kubernetes.io/backend-protocol": "HTTPS", |
| 284 | "nginx.ingress.kubernetes.io/proxy-body-size": "0", |
| 285 | }, |
| 286 | }, |
| 287 | spec+: { |
| 288 | tls: [ |
| 289 | { |
| 290 | hosts: [cfg.domain], |
| 291 | secretName: "registry-tls", |
| 292 | }, |
| 293 | ], |
| 294 | rules: [ |
| 295 | { |
| 296 | host: cfg.domain, |
| 297 | http: { |
| 298 | paths: [ |
| 299 | { path: "/auth", backend: env.authService.name_port }, |
| 300 | { path: "/", backend: env.authService.name_port }, |
| 301 | { path: "/v2/", backend: env.registryService.name_port }, |
| 302 | ] |
| 303 | }, |
| 304 | } |
| 305 | ], |
| 306 | }, |
| 307 | }, |
| 308 | |
Sergiusz Bazanski | e186c87 | 2020-02-21 12:57:02 +0100 | [diff] [blame] | 309 | registryStorageUser: kube.CephObjectStoreUser("registry") { |
Sergiusz Bazanski | 4d61d20 | 2019-07-21 16:56:41 +0200 | [diff] [blame] | 310 | metadata+: { |
Serge Bazanski | 3d29484 | 2020-08-04 01:34:07 +0200 | [diff] [blame] | 311 | namespace: "ceph-waw3", |
Sergiusz Bazanski | 4d61d20 | 2019-07-21 16:56:41 +0200 | [diff] [blame] | 312 | }, |
| 313 | spec: { |
Sergiusz Bazanski | d07861b | 2019-08-08 17:48:25 +0200 | [diff] [blame] | 314 | store: cfg.objectStorageName, |
Sergiusz Bazanski | 4d61d20 | 2019-07-21 16:56:41 +0200 | [diff] [blame] | 315 | displayName: "docker-registry user", |
| 316 | }, |
| 317 | }, |
| 318 | } |
| 319 | } |