blob: 9887f58fce59d041b9c719425cb5d064ac58b1b7 [file] [log] [blame]
Serge Bazanski60076c72020-11-03 19:17:25 +01001# Matrix server (synapse).
Piotr Dobrowolskia2226912019-05-14 18:49:29 +02002# This needs a secret provisioned, create with:
Serge Bazanski60076c72020-11-03 19:17:25 +01003# ns=matrix
4# kubectl -n $ns create secret generic synapse --from-literal=postgres_password=$(pwgen 24 1) --from-literal=macaroon_secret_key=$(pwgen 32 1) --from-literal=registration_shared_secret=$(pwgen 32 1)
5# kubectl -n $ns create secret generic oauth2-cas-proxy --from-literal=oauth2_secret=...
Serge Bazanskide627512020-08-24 21:17:55 +00006#
Serge Bazanski21a96162020-11-03 23:34:36 +01007# After starting, re-create the postgres database (because docker/postgres won't let you set ENCODING):
8# postgres=# drop database synapse;
9# DROP DATABASE
10# postgres=# CREATE DATABASE synapse
11# postgres-# ENCODING 'UTF8'
12# postgres-# LC_COLLATE='C'
13# postgres-# LC_CTYPE='C'
14# postgres-# template=template0
15# postgres-# OWNER synapse;
16# CREATE DATABASE
17#
Serge Bazanskide627512020-08-24 21:17:55 +000018# Sequencing appservices is fun. The appservice needs to run first (for
19# instance, via a bootstrap job), and on startup it will spit out a
20# registration file. This registration file then needs to be fed to synapse -
21# this is done via specialy named secrets (appservice-X-registration, for X key
22# in the appservices object).
23#
24# For appservice-irc instances, you can use this oneliner magic to get the
25# registration YAML from logs.
Piotr Dobrowolski3ea979d2019-05-23 16:11:52 +020026# kubectl -n matrix create secret generic appservice-irc-freenode-registration --from-file=registration.yaml=<(kubectl logs -n matrix $(kubectl get pods -n matrix --selector=job-name=appservice-irc-freenode-bootstrap --output=jsonpath='{.items[*].metadata.name}') | tail -n +4 | sed -r 's/(.*aliases:.*)/ group_id: "+freenode:hackerspace.pl"\n\1/')
Serge Bazanskide627512020-08-24 21:17:55 +000027#
28# For appservice-telegram instances, you can use this oneliner magic:
29# kubectl -n matrix create secret generic appservice-telegram-prod-registration --from-file=registration.yaml=<(kubectl -n matrix logs job/appservice-telegram-prod-bootstrap | grep -A 100 SNIPSNIP | grep -v SNIPSNIP)
Piotr Dobrowolskia2226912019-05-14 18:49:29 +020030
Serge Bazanski60076c72020-11-03 19:17:25 +010031local kube = import "../../../kube/kube.libsonnet";
32local postgres = import "../../../kube/postgres.libsonnet";
Serge Bazanskicdba2912020-08-24 19:11:10 +000033
Piotr Dobrowolskia2226912019-05-14 18:49:29 +020034{
35 local app = self,
36 local cfg = app.cfg,
37 cfg:: {
Serge Bazanski60076c72020-11-03 19:17:25 +010038 namespace: error "cfg.namespace must be set",
39 # webDomain is the domain name at which element will run
40 webDomain: error "cfg.webDomain must be set",
41 # serverName is the server part of the MXID this homeserver will cover
42 serverName: error "cfg.serverName must be set",
Serge Bazanskide627512020-08-24 21:17:55 +000043 storageClassName: "waw-hdd-redundant-3",
Piotr Dobrowolskifef4c122019-05-16 21:05:02 +020044
Serge Bazanskiace32c02020-11-03 22:04:06 +010045 images: {
46 synapse: "matrixdotorg/synapse:v1.19.2",
47 riot: "vectorim/riot-web:v1.7.7",
48 casProxy: "registry.k0.hswaw.net/q3k/oauth2-cas-proxy:0.1.4",
49 appserviceIRC: "matrixdotorg/matrix-appservice-irc:release-0.17.1",
50 # That's v0.8.2 - we just don't trust that host to not re-tag images.
51 appserviceTelegram: "dock.mau.dev/tulir/mautrix-telegram@sha256:9e68eaa80c9e4a75d9a09ec92dc4898b12d48390e01efa4de40ce882a6f7e330",
52 },
53
54 cas: {
55 # whether to enable the CAS proxy (ie. connect to hswaw sso via OAuth)
56 enable: false,
57 },
Piotr Dobrowolskia2226912019-05-14 18:49:29 +020058 },
59
60 metadata(component):: {
Piotr Dobrowolski4b4231d2019-05-15 11:41:21 +020061 namespace: cfg.namespace,
Piotr Dobrowolskia2226912019-05-14 18:49:29 +020062 labels: {
63 "app.kubernetes.io/name": "matrix",
64 "app.kubernetes.io/managed-by": "kubecfg",
65 "app.kubernetes.io/component": component,
66 },
67 },
68
Piotr Dobrowolski4b4231d2019-05-15 11:41:21 +020069 namespace: kube.Namespace(cfg.namespace),
Piotr Dobrowolskia2226912019-05-14 18:49:29 +020070
Serge Bazanskic0c037a2020-08-23 01:24:03 +000071 postgres3: postgres {
Piotr Dobrowolskia2226912019-05-14 18:49:29 +020072 cfg+: {
73 namespace: cfg.namespace,
74 appName: "synapse",
75 database: "synapse",
76 username: "synapse",
Serge Bazanskic0c037a2020-08-23 01:24:03 +000077 prefix: "waw3-",
Piotr Dobrowolskia2226912019-05-14 18:49:29 +020078 password: { secretKeyRef: { name: "synapse", key: "postgres_password" } },
Serge Bazanskide627512020-08-24 21:17:55 +000079 storageClassName: cfg.storageClassName,
Serge Bazanskic0c037a2020-08-23 01:24:03 +000080 storageSize: "100Gi",
Piotr Dobrowolskia2226912019-05-14 18:49:29 +020081 },
82 },
83
Serge Bazanskic0c037a2020-08-23 01:24:03 +000084 dataVolume: kube.PersistentVolumeClaim("synapse-data-waw3") {
Piotr Dobrowolskia2226912019-05-14 18:49:29 +020085 metadata+: app.metadata("synapse-data"),
86 spec+: {
Serge Bazanskide627512020-08-24 21:17:55 +000087 storageClassName: cfg.storageClassName,
Piotr Dobrowolskia2226912019-05-14 18:49:29 +020088 accessModes: [ "ReadWriteOnce" ],
89 resources: {
90 requests: {
91 storage: "50Gi",
92 },
93 },
94 },
95 },
Piotr Dobrowolskiffbb47c2019-05-16 12:18:39 +020096
Serge Bazanski21a96162020-11-03 23:34:36 +010097 // homeserver.yaml that will be used to run synapse (in synapseConfigMap).
Serge Bazanskiace32c02020-11-03 22:04:06 +010098 // This is based off of //app/matrix/lib/synapse/homeserver.yaml with some fields overriden per
99 // deployment.
100 // Note this is a templated yaml - {{}}/{%%} style. This templatization is consumed by the Docker
101 // container startup magic.
Serge Bazanski21a96162020-11-03 23:34:36 +0100102 synapseConfig:: (std.native("parseYaml"))(importstr "synapse/homeserver.yaml")[0] {
Serge Bazanskiace32c02020-11-03 22:04:06 +0100103 server_name: cfg.serverName,
104 public_baseurl: "https://%s" % [cfg.webDomain],
105 signing_key_path: "/data/%s.signing.key" % [cfg.serverName],
Serge Bazanski21a96162020-11-03 23:34:36 +0100106 app_service_config_files: [
107 "/appservices/%s/registration.yaml" % [k]
108 for k in std.objectFields(app.appservices)
109 ],
110 } + (if cfg.cas.enable then {
111 cas_config: {
Serge Bazanskiace32c02020-11-03 22:04:06 +0100112 enabled: true,
113 server_url: "https://%s/_cas" % [cfg.webDomain],
114 service_url: "https://%s" % [cfg.webDomain],
Serge Bazanski21a96162020-11-03 23:34:36 +0100115 },
116 } else {}),
Serge Bazanskiace32c02020-11-03 22:04:06 +0100117
Serge Bazanski21a96162020-11-03 23:34:36 +0100118 synapseConfigMap: kube.ConfigMap("synapse") {
Piotr Dobrowolskic39fb042019-05-17 09:13:56 +0200119 metadata+: app.metadata("synapse"),
120 data: {
Serge Bazanski21a96162020-11-03 23:34:36 +0100121 "homeserver.yaml": std.manifestYamlDoc(app.synapseConfig),
Serge Bazanski60076c72020-11-03 19:17:25 +0100122 "log.config": importstr "synapse/log.config",
Piotr Dobrowolskic39fb042019-05-17 09:13:56 +0200123 },
124 },
125
Serge Bazanskiace32c02020-11-03 22:04:06 +0100126 casDeployment: if cfg.cas.enable then kube.Deployment("oauth2-cas-proxy") {
Piotr Dobrowolskic39fb042019-05-17 09:13:56 +0200127 metadata+: app.metadata("oauth2-cas-proxy"),
128 spec+: {
129 replicas: 1,
130 template+: {
131 spec+: {
132 containers_: {
133 proxy: kube.Container("oauth2-cas-proxy") {
Serge Bazanskiace32c02020-11-03 22:04:06 +0100134 image: cfg.images.casProxy,
Piotr Dobrowolskic39fb042019-05-17 09:13:56 +0200135 ports_: {
136 http: { containerPort: 5000 },
137 },
138 env_: {
Serge Bazanski60076c72020-11-03 19:17:25 +0100139 BASE_URL: "https://%s" % [cfg.webDomain],
140 SERVICE_URL: "https://%s" % [cfg.webDomain],
Piotr Dobrowolskic39fb042019-05-17 09:13:56 +0200141 OAUTH2_CLIENT: "matrix",
142 OAUTH2_SECRET: { secretKeyRef: { name: "oauth2-cas-proxy", key: "oauth2_secret" } },
143 },
144 },
145 },
146 },
147 },
148 },
149 },
150
Serge Bazanskiace32c02020-11-03 22:04:06 +0100151 casSvc: if cfg.cas.enable then kube.Service("oauth2-cas-proxy") {
Piotr Dobrowolskic39fb042019-05-17 09:13:56 +0200152 metadata+: app.metadata("oauth2-cas-proxy"),
153 target_pod:: app.casDeployment.spec.template,
154 },
155
Piotr Dobrowolskifef4c122019-05-16 21:05:02 +0200156 synapseDeployment: kube.Deployment("synapse") {
Piotr Dobrowolskia2226912019-05-14 18:49:29 +0200157 metadata+: app.metadata("synapse"),
158 spec+: {
159 replicas: 1,
160 template+: {
161 spec+: {
162 volumes_: {
163 data: kube.PersistentVolumeClaimVolume(app.dataVolume),
Serge Bazanski21a96162020-11-03 23:34:36 +0100164 config: kube.ConfigMapVolume(app.synapseConfigMap),
Piotr Dobrowolskifef4c122019-05-16 21:05:02 +0200165 } + {
166 [k]: { secret: { secretName: "appservice-%s-registration" % [k] } }
167 for k in std.objectFields(app.appservices)
Piotr Dobrowolskia2226912019-05-14 18:49:29 +0200168 },
169 containers_: {
170 web: kube.Container("synapse") {
Serge Bazanskiace32c02020-11-03 22:04:06 +0100171 image: cfg.images.synapse,
Piotr Dobrowolski8ebfc1d2020-03-03 21:01:18 +0100172 command: ["/bin/sh", "-c", "/start.py migrate_config && exec /start.py"],
Piotr Dobrowolskia2226912019-05-14 18:49:29 +0200173 ports_: {
174 http: { containerPort: 8008 },
Serge Bazanski1230ac32020-09-12 22:09:46 +0000175 metrics: { containerPort: 9092 },
Piotr Dobrowolskia2226912019-05-14 18:49:29 +0200176 },
177 env_: {
Serge Bazanski21a96162020-11-03 23:34:36 +0100178 SYNAPSE_CONFIG_DIR: "/tmp/config",
179 SYNAPSE_CONFIG_PATH: "/tmp/config/homeserver.yaml",
Piotr Dobrowolskia2226912019-05-14 18:49:29 +0200180
Piotr Dobrowolski8ebfc1d2020-03-03 21:01:18 +0100181 # These values are not used in a template, but
182 # are required by /start.py migrate_config
Serge Bazanski60076c72020-11-03 19:17:25 +0100183 SYNAPSE_SERVER_NAME: cfg.serverName,
Piotr Dobrowolski8ebfc1d2020-03-03 21:01:18 +0100184 SYNAPSE_REPORT_STATS: "no",
185
186 SYNAPSE_MACAROON_SECRET_KEY: { secretKeyRef: { name: "synapse", key: "macaroon_secret_key" } },
187 SYNAPSE_REGISTRATION_SHARED_SECRET: { secretKeyRef: { name: "synapse", key: "registration_shared_secret" } },
188 POSTGRES_PASSWORD: { secretKeyRef: { name: "synapse", key: "postgres_password" } },
Piotr Dobrowolskia2226912019-05-14 18:49:29 +0200189 },
190 volumeMounts_: {
191 data: { mountPath: "/data" },
Serge Bazanski21a96162020-11-03 23:34:36 +0100192 config: { mountPath: "/conf", },
Piotr Dobrowolskifef4c122019-05-16 21:05:02 +0200193 } + {
194 [k]: { mountPath: "/appservices/%s" % [k] }
195 for k in std.objectFields(app.appservices)
Piotr Dobrowolskia2226912019-05-14 18:49:29 +0200196 },
197 },
198 },
Serge Bazanski21a96162020-11-03 23:34:36 +0100199 securityContext: {
200 runAsUser: 991,
201 runAsGroup: 991,
202 fsGroup: 991,
203 },
Piotr Dobrowolskia2226912019-05-14 18:49:29 +0200204 },
205 },
206 },
207 },
208
Piotr Dobrowolskifef4c122019-05-16 21:05:02 +0200209 synapseSvc: kube.Service("synapse") {
Piotr Dobrowolskiffbb47c2019-05-16 12:18:39 +0200210 metadata+: app.metadata("synapse"),
Piotr Dobrowolskifef4c122019-05-16 21:05:02 +0200211 target_pod:: app.synapseDeployment.spec.template,
Piotr Dobrowolskiffbb47c2019-05-16 12:18:39 +0200212 },
213
Norbert Szulc1ef56002020-11-08 16:13:58 +0100214 riotConfig:: {
215 "default_hs_url": "https://%s" % [cfg.webDomain],
216 "disable_custom_urls": false,
217 "disable_guests": false,
218 "disable_login_language_selector": false,
219 "disable_3pid_login": true,
220 "brand": "Riot",
221 "integrations_ui_url": "https://scalar.vector.im/",
222 "integrations_rest_url": "https://scalar.vector.im/api",
223 "integrations_jitsi_widget_url": "https://scalar.vector.im/api/widgets/jitsi.html",
224
225 "bug_report_endpoint_url": "https://riot.im/bugreports/submit",
226 "features": {
227 "feature_groups": "labs",
228 "feature_pinning": "labs",
229 "feature_reactions": "labs"
230 },
231 "default_federate": true,
232 "default_theme": "light",
233 "roomDirectory": {
234 "servers": [
235 cfg.serverName,
236 ]
237 },
238 "welcomeUserId": "@riot-bot:matrix.org",
239 "enable_presence_by_hs_url": {
240 "https://matrix.org": false
241 }
242 },
243
244 riotConfigMap: kube.ConfigMap("riot-web-config") {
Piotr Dobrowolskia2226912019-05-14 18:49:29 +0200245 metadata+: app.metadata("riot-web-config"),
246 data: {
Norbert Szulc1ef56002020-11-08 16:13:58 +0100247 "config.json": std.manifestJsonEx(app.riotConfig, ""),
Serge Bazanski21a96162020-11-03 23:34:36 +0100248 // Standard nginx.conf, made to work when running as unprivileged user.
249 "nginx.conf": |||
250 worker_processes auto;
251
252 error_log /tmp/nginx_error.log warn;
253 pid /tmp/nginx.pid;
254
255 events {
256 worker_connections 1024;
257 }
258
259
260 http {
261 client_body_temp_path /tmp/nginx_client_temp;
262 proxy_temp_path /tmp/nginx_proxy_temp;
263 fastcgi_temp_path /tmp/nginx_fastcgi_temp;
264 uwsgi_temp_path /tmp/nginx_uwsgi_temp;
265 scgi_temp_path /tmp/nginx_scgi_temp;
266
267 include /etc/nginx/mime.types;
268 default_type application/octet-stream;
269 log_format main '$remote_addr - $remote_user [$time_local] "$request" '
270 '$status $body_bytes_sent "$http_referer" '
271 '"$http_user_agent" "$http_x_forwarded_for"';
272 access_log /tmp/nginx_access.log main;
273 sendfile on;
274 keepalive_timeout 65;
275
276 server {
277 listen 8080;
278 server_name localhost;
279
280 location / {
281 root /usr/share/nginx/html;
282 index index.html index.htm;
283 }
284
285 error_page 500 502 503 504 /50x.html;
286 location = /50x.html {
287 root /usr/share/nginx/html;
288 }
289 }
290 }
291 |||,
Piotr Dobrowolskia2226912019-05-14 18:49:29 +0200292 },
293 },
294
295 riotDeployment: kube.Deployment("riot-web") {
296 metadata+: app.metadata("riot-web"),
297 spec+: {
298 replicas: 1,
299 template+: {
300 spec+: {
301 volumes_: {
Norbert Szulc1ef56002020-11-08 16:13:58 +0100302 config: kube.ConfigMapVolume(app.riotConfigMap),
Piotr Dobrowolskia2226912019-05-14 18:49:29 +0200303 },
304 containers_: {
Piotr Dobrowolskifef4c122019-05-16 21:05:02 +0200305 web: kube.Container("riot-web") {
Serge Bazanskiace32c02020-11-03 22:04:06 +0100306 image: cfg.images.riot,
Piotr Dobrowolskia2226912019-05-14 18:49:29 +0200307 ports_: {
Serge Bazanski21a96162020-11-03 23:34:36 +0100308 http: { containerPort: 8080 },
Piotr Dobrowolskia2226912019-05-14 18:49:29 +0200309 },
Serge Bazanski21a96162020-11-03 23:34:36 +0100310 volumeMounts: [
311 {
312 name: "config",
Piotr Dobrowolskiaca7e282020-03-21 22:14:38 +0100313 mountPath: "/app/config.json",
Piotr Dobrowolskia2226912019-05-14 18:49:29 +0200314 subPath: "config.json",
315 },
Serge Bazanski21a96162020-11-03 23:34:36 +0100316 {
317 name: "config",
318 mountPath: "/etc/nginx/nginx.conf",
319 subPath: "nginx.conf",
320 },
321 ],
Piotr Dobrowolskia2226912019-05-14 18:49:29 +0200322 },
323 },
Serge Bazanski21a96162020-11-03 23:34:36 +0100324 securityContext: {
325 // nginx:nginx
326 runAsUser: 101,
327 runAsGroup: 101,
328 },
Piotr Dobrowolskia2226912019-05-14 18:49:29 +0200329 },
330 },
331 },
332 },
333
Piotr Dobrowolskia2226912019-05-14 18:49:29 +0200334 riotSvc: kube.Service("riot-web") {
335 metadata+: app.metadata("riot-web"),
336 target_pod:: app.riotDeployment.spec.template,
Piotr Dobrowolskia2226912019-05-14 18:49:29 +0200337 },
338
Serge Bazanskide627512020-08-24 21:17:55 +0000339 // Any appservice you add here will require an appservice-X-registration
340 // secret containing a registration.yaml file. Adding something to this
341 // dictionary will cause Synapse to not start until that secret is
342 // available - so change things carefully!
343 // If bootstrapping a new appservice, just keep it out of this dictionary
344 // until it spits you a registration YAML and you feed that to a secret.
Serge Bazanski60076c72020-11-03 19:17:25 +0100345 appservices: {},
Piotr Dobrowolskifef4c122019-05-16 21:05:02 +0200346
347 ingress: kube.Ingress("matrix") {
348 metadata+: app.metadata("matrix") {
Piotr Dobrowolskia2226912019-05-14 18:49:29 +0200349 annotations+: {
350 "kubernetes.io/tls-acme": "true",
351 "certmanager.k8s.io/cluster-issuer": "letsencrypt-prod",
352 "nginx.ingress.kubernetes.io/proxy-body-size": "0",
353 },
354 },
355 spec+: {
356 tls: [
357 {
Serge Bazanski60076c72020-11-03 19:17:25 +0100358 hosts: [cfg.webDomain],
Piotr Dobrowolskia2226912019-05-14 18:49:29 +0200359 secretName: "synapse-tls",
360 },
361 ],
362 rules: [
363 {
Serge Bazanski60076c72020-11-03 19:17:25 +0100364 host: cfg.webDomain,
Piotr Dobrowolskia2226912019-05-14 18:49:29 +0200365 http: {
366 paths: [
367 { path: "/", backend: app.riotSvc.name_port },
Piotr Dobrowolskifef4c122019-05-16 21:05:02 +0200368 { path: "/_matrix", backend: app.synapseSvc.name_port },
Serge Bazanskiace32c02020-11-03 22:04:06 +0100369 ] + (if cfg.cas.enable then [
Piotr Dobrowolskic39fb042019-05-17 09:13:56 +0200370 { path: "/_cas", backend: app.casSvc.name_port },
Serge Bazanskiace32c02020-11-03 22:04:06 +0100371 ] else [])
Piotr Dobrowolskia2226912019-05-14 18:49:29 +0200372 },
373 }
374 ],
375 },
376 },
Piotr Dobrowolskifef4c122019-05-16 21:05:02 +0200377
Piotr Dobrowolskia2226912019-05-14 18:49:29 +0200378}