Gitiles
Code Review Sign In
gerrit.hackerspace.pl / hscloud / 4d61d20aec3564207ebc23ef601e9de3187d2353 / . / cluster / kube / lib / registry.libsonnet
blob: 8b57dd7e733984131d1380edf61304f1a4921ff1 [file] [log] [blame]
Sergiusz Bazanski4d61d202019-07-21 16:56:41 +0200[diff] [blame^]1# Deploy a Docker Registry in a cluster.
2
3# This needs an oauth2 secret provisioned, create with:
4# kubectl -n registry create secret generic auth --from-literal=oauth2_secret=...
5# kubectl get secrets rook-ceph-object-user-<ceph-pool>-object-registry -n <ceph-namespace> -o yaml --export | kubectl replace -f - -n registry
6
7local kube = import "../../../kube/kube.libsonnet";
8local cm = import "cert-manager.libsonnet";
9
10{
11 Environment: {
12 local env = self,
13 local cfg = env.cfg,
14 cfg:: {
15 namespace: "registry",
16 domain: error "domain must be set",
17 storageClassName: error "storageClassName must be set",
18 },
19
20 metadata(component):: {
21 namespace: cfg.namespace,
22 labels: {
23 "app.kubernetes.io/name": "registry",
24 "app.kubernetes.io/managed-by": "kubecfg",
25 "app.kubernetes.io/component": component,
26 },
27 },
28
29 namespace: kube.Namespace(cfg.namespace),
30
31 registryIssuer: cm.Issuer("registry-issuer") {
32 metadata+: env.metadata("registry-issuer"),
33 spec: {
34 selfSigned: {},
35 },
36 },
37 authCertificate: cm.Certificate("auth") {
38 metadata+: env.metadata("auth"),
39 spec: {
40 secretName: "auth-internal",
41 duration: "43800h0m0s", // 5 years
42 issuerRef: {
43 name: env.registryIssuer.metadata.name,
44 },
45 commonName: "auth.registry",
46 },
47 },
48 registryCertificate: cm.Certificate("registry") {
49 metadata+: env.metadata("registry"),
50 spec: {
51 secretName: "registry-internal",
52 duration: "43800h0m0s", // 5 years
53 issuerRef: {
54 name: env.registryIssuer.metadata.name,
55 },
56 commonName: "registry.registry",
57 },
58 },
59
60 registryConfig: kube.ConfigMap("registry-config") {
61 metadata+: env.metadata("registry-config"),
62 data: {
63 "config.yml": std.manifestYamlDoc({
64 version: "0.1",
65 log: {
66 fields: {
67 service: "registry",
68 },
69 },
70 storage: {
71 cache: {
72 blobdescriptor: "inmemory",
73 },
74 s3: {
75 regionendpoint: "https://object.ceph-waw1.hswaw.net",
76 bucket: "registry",
77 region: "waw-hdd-redunant-1-object:default-placement",
78 },
79 },
80 http: {
81 addr: ":5000",
82 headers: {
83 "X-Content-Type-Options": ["nosniff"],
84 },
85 tls: {
86 certificate: "/certs/tls.crt",
87 key: "/certs/tls.key",
88 },
89 debug: {
90 addr: "localhost:5001",
91 },
92 },
93 health: {
94 storagedriver: {
95 enabled: true,
96 interval: "10s",
97 threshold: 3,
98 },
99 },
100 auth: {
101 token: {
102 realm: "https://%s/auth" % [cfg.domain],
103 service: "my.docker.registry",
104 issuer: "%s auth server" % [cfg.domain],
105 rootcertbundle: "/authcerts/tls.crt",
106 },
107 },
108 }),
109 },
110 },
111
112 authVolumeClaim: kube.PersistentVolumeClaim("auth-token-storage") {
113 metadata+: env.metadata("auth-token-storage"),
114 spec+: {
115 storageClassName: cfg.storageClassName,
116 accessModes: [ "ReadWriteOnce" ],
117 resources: {
118 requests: {
119 storage: "1Gi",
120 },
121 },
122 },
123 },
124
125 authConfig: kube.ConfigMap("auth-config") {
126 metadata+: env.metadata("auth-config"),
127 data: {
128 "auth_config.yml": std.manifestYamlDoc({
129 server: {
130 addr: ":5001",
131 certificate: "/certs/tls.crt",
132 key: "/certs/tls.key",
133 },
134 token: {
135 issuer: "%s auth server" % [cfg.domain],
136 expiration: 900,
137 },
138 oauth2: {
139 client_id: "registry",
140 client_secret_file: "/secrets/oauth2_secret",
141 authorize_url: "https://sso.hackerspace.pl/oauth/authorize",
142 access_token_url: "https://sso.hackerspace.pl/oauth/token",
143 profile_url: "https://sso.hackerspace.pl/api/1/profile",
144 redirect_url: "https://registry.k0.hswaw.net/oauth2",
145 username_key: "username",
146 token_db: "/data/oauth2_tokens.ldb",
147 registry_url: "https://registry.k0.hswaw.net",
148 },
149 users: {
150 [""]: {}, // '' user are anonymous users.
151 },
152 local data = self,
153 pushers:: [
154 { who: ["q3k", "inf"], what: "vms/*" },
155 { who: ["q3k", "inf"], what: "app/*" },
156 { who: ["q3k", "inf"], what: "go/svc/*" },
157 ],
158 acl: [
159 {
160 match: {account: "/.+/", name: "${account}/*"},
161 actions: ["*"],
162 comment: "Logged in users have full access to images that are in their 'namespace'",
163 },
164 {
165 match: {account: "/.+/", type: "registry", name: "catalog"},
166 actions: ["*"],
167 comment: "Logged in users can query the catalog.",
168 },
169 {
170 match: {account: ""},
171 actions: ["pull"],
172 comment: "Anyone can pull all images.",
173 },
174 ] + [
175 {
176 match: {
177 account: "/(%s)/" % std.join("|", p.who),
178 name: p.what,
179 },
180 actions: ["*"],
181 comment: "%s can push to %s" % [std.join(", ", p.who), p.what],
182 }
183 for p in data.pushers
184 ],
185 }),
186 }
187 },
188
189 authDeployment: kube.Deployment("auth") {
190 metadata+: env.metadata("auth"),
191 spec+: {
192 replicas: 1,
193 template+: {
194 spec+: {
195 volumes_: {
196 data: kube.PersistentVolumeClaimVolume(env.authVolumeClaim),
197 config: kube.ConfigMapVolume(env.authConfig),
198 certs: {
199 secret: { secretName: env.authCertificate.spec.secretName },
200 },
201 secrets: {
202 secret: { secretName: "auth" },
203 },
204 },
205 containers_: {
206 auth: kube.Container("auth") {
207 image: "informatic/docker_auth:2019040307",
208 volumeMounts_: {
209 config: { mountPath: "/config" },
210 certs: { mountPath: "/certs" },
211 secrets: { mountPath: "/secrets" },
212 data: { mountPath: "/data" },
213 },
214 },
215 },
216 },
217 },
218 },
219 },
220 authService: kube.Service("auth") {
221 metadata+: env.metadata("auth"),
222 target_pod:: env.authDeployment.spec.template,
223 spec+: {
224 type: "ClusterIP",
225 ports: [
226 { name: "auth", port: 5001, targetPort: 5001, protocol: "TCP" },
227 ],
228 }
229 },
230 registryDeployment: kube.Deployment("docker-registry") {
231 metadata+: env.metadata("docker-registry"),
232 spec+: {
233 replicas: 1,
234 template+: {
235 spec+: {
236 volumes_: {
237 config: kube.ConfigMapVolume(env.registryConfig),
238 certs: {
239 secret: { secretName: env.registryCertificate.spec.secretName },
240 },
241 authcerts: {
242 secret: { secretName: env.authCertificate.spec.secretName },
243 },
244 },
245 containers_: {
246 registry: kube.Container("docker-registry") {
247 image: "registry:2",
248 args: ["/config/config.yml"],
249 volumeMounts_: {
250 config: { mountPath: "/config" },
251 certs: { mountPath: "/certs" },
252 authcerts: { mountPath: "/authcerts" },
253 },
254 env_: {
255 REGISTRY_STORAGE_S3_ACCESSKEY: { secretKeyRef: {
256 name: "rook-ceph-object-user-waw-hdd-redundant-1-object-registry",
257 key: "AccessKey"
258 }},
259 REGISTRY_STORAGE_S3_SECRETKEY: { secretKeyRef: {
260 name: "rook-ceph-object-user-waw-hdd-redundant-1-object-registry",
261 key: "SecretKey",
262 }},
263 },
264 },
265 },
266 },
267 },
268 },
269 },
270 registryService: kube.Service("docker-registry") {
271 metadata+: env.metadata("docker-registry"),
272 target_pod:: env.registryDeployment.spec.template,
273 spec+: {
274 type: "ClusterIP",
275 ports: [
276 { name: "registry", port: 5000, targetPort: 5000, protocol: "TCP" },
277 ],
278 }
279 },
280 registryIngress: kube.Ingress("registry") {
281 metadata+: env.metadata("registry") {
282 annotations+: {
283 "kubernetes.io/tls-acme": "true",
284 "certmanager.k8s.io/cluster-issuer": "letsencrypt-prod",
285 "nginx.ingress.kubernetes.io/backend-protocol": "HTTPS",
286 "nginx.ingress.kubernetes.io/proxy-body-size": "0",
287 },
288 },
289 spec+: {
290 tls: [
291 {
292 hosts: [cfg.domain],
293 secretName: "registry-tls",
294 },
295 ],
296 rules: [
297 {
298 host: cfg.domain,
299 http: {
300 paths: [
301 { path: "/auth", backend: env.authService.name_port },
302 { path: "/", backend: env.authService.name_port },
303 { path: "/v2/", backend: env.registryService.name_port },
304 ]
305 },
306 }
307 ],
308 },
309 },
310
311 registryStorageUser: kube._Object("ceph.rook.io/v1", "CephObjectStoreUser", "registry") {
312 metadata+: {
313 namespace: "ceph-waw1",
314 },
315 spec: {
316 store: "waw-hdd-redundant-1-object",
317 displayName: "docker-registry user",
318 },
319 },
320 }
321}