blob: a881aafb83bb894e5102db7eac8d5dcb7ae90801 [file] [log] [blame]
Serge Bazanski60076c72020-11-03 19:17:25 +01001# Matrix server (synapse).
Piotr Dobrowolskia2226912019-05-14 18:49:29 +02002# This needs a secret provisioned, create with:
Serge Bazanski60076c72020-11-03 19:17:25 +01003# ns=matrix
4# kubectl -n $ns create secret generic synapse --from-literal=postgres_password=$(pwgen 24 1) --from-literal=macaroon_secret_key=$(pwgen 32 1) --from-literal=registration_shared_secret=$(pwgen 32 1)
5# kubectl -n $ns create secret generic oauth2-cas-proxy --from-literal=oauth2_secret=...
Serge Bazanskide627512020-08-24 21:17:55 +00006#
Serge Bazanski21a96162020-11-03 23:34:36 +01007# After starting, re-create the postgres database (because docker/postgres won't let you set ENCODING):
8# postgres=# drop database synapse;
9# DROP DATABASE
10# postgres=# CREATE DATABASE synapse
11# postgres-# ENCODING 'UTF8'
12# postgres-# LC_COLLATE='C'
13# postgres-# LC_CTYPE='C'
14# postgres-# template=template0
15# postgres-# OWNER synapse;
16# CREATE DATABASE
17#
Serge Bazanskide627512020-08-24 21:17:55 +000018# Sequencing appservices is fun. The appservice needs to run first (for
19# instance, via a bootstrap job), and on startup it will spit out a
20# registration file. This registration file then needs to be fed to synapse -
21# this is done via specialy named secrets (appservice-X-registration, for X key
22# in the appservices object).
23#
24# For appservice-irc instances, you can use this oneliner magic to get the
25# registration YAML from logs.
Piotr Dobrowolski3ea979d2019-05-23 16:11:52 +020026# kubectl -n matrix create secret generic appservice-irc-freenode-registration --from-file=registration.yaml=<(kubectl logs -n matrix $(kubectl get pods -n matrix --selector=job-name=appservice-irc-freenode-bootstrap --output=jsonpath='{.items[*].metadata.name}') | tail -n +4 | sed -r 's/(.*aliases:.*)/ group_id: "+freenode:hackerspace.pl"\n\1/')
Serge Bazanskide627512020-08-24 21:17:55 +000027#
28# For appservice-telegram instances, you can use this oneliner magic:
29# kubectl -n matrix create secret generic appservice-telegram-prod-registration --from-file=registration.yaml=<(kubectl -n matrix logs job/appservice-telegram-prod-bootstrap | grep -A 100 SNIPSNIP | grep -v SNIPSNIP)
Piotr Dobrowolskia2226912019-05-14 18:49:29 +020030
Serge Bazanski60076c72020-11-03 19:17:25 +010031local kube = import "../../../kube/kube.libsonnet";
32local postgres = import "../../../kube/postgres.libsonnet";
Serge Bazanskicdba2912020-08-24 19:11:10 +000033
Piotr Dobrowolskia2226912019-05-14 18:49:29 +020034{
35 local app = self,
36 local cfg = app.cfg,
37 cfg:: {
Serge Bazanski60076c72020-11-03 19:17:25 +010038 namespace: error "cfg.namespace must be set",
39 # webDomain is the domain name at which element will run
40 webDomain: error "cfg.webDomain must be set",
41 # serverName is the server part of the MXID this homeserver will cover
42 serverName: error "cfg.serverName must be set",
Serge Bazanskide627512020-08-24 21:17:55 +000043 storageClassName: "waw-hdd-redundant-3",
Piotr Dobrowolskifef4c122019-05-16 21:05:02 +020044
Serge Bazanskiace32c02020-11-03 22:04:06 +010045 images: {
46 synapse: "matrixdotorg/synapse:v1.19.2",
47 riot: "vectorim/riot-web:v1.7.7",
48 casProxy: "registry.k0.hswaw.net/q3k/oauth2-cas-proxy:0.1.4",
49 appserviceIRC: "matrixdotorg/matrix-appservice-irc:release-0.17.1",
50 # That's v0.8.2 - we just don't trust that host to not re-tag images.
51 appserviceTelegram: "dock.mau.dev/tulir/mautrix-telegram@sha256:9e68eaa80c9e4a75d9a09ec92dc4898b12d48390e01efa4de40ce882a6f7e330",
52 },
53
54 cas: {
55 # whether to enable the CAS proxy (ie. connect to hswaw sso via OAuth)
56 enable: false,
57 },
Piotr Dobrowolskia2226912019-05-14 18:49:29 +020058 },
59
60 metadata(component):: {
Piotr Dobrowolski4b4231d2019-05-15 11:41:21 +020061 namespace: cfg.namespace,
Piotr Dobrowolskia2226912019-05-14 18:49:29 +020062 labels: {
63 "app.kubernetes.io/name": "matrix",
64 "app.kubernetes.io/managed-by": "kubecfg",
65 "app.kubernetes.io/component": component,
66 },
67 },
68
Piotr Dobrowolski4b4231d2019-05-15 11:41:21 +020069 namespace: kube.Namespace(cfg.namespace),
Piotr Dobrowolskia2226912019-05-14 18:49:29 +020070
Serge Bazanskic0c037a2020-08-23 01:24:03 +000071 postgres3: postgres {
Piotr Dobrowolskia2226912019-05-14 18:49:29 +020072 cfg+: {
73 namespace: cfg.namespace,
74 appName: "synapse",
75 database: "synapse",
76 username: "synapse",
Serge Bazanskic0c037a2020-08-23 01:24:03 +000077 prefix: "waw3-",
Piotr Dobrowolskia2226912019-05-14 18:49:29 +020078 password: { secretKeyRef: { name: "synapse", key: "postgres_password" } },
Serge Bazanskide627512020-08-24 21:17:55 +000079 storageClassName: cfg.storageClassName,
Serge Bazanskic0c037a2020-08-23 01:24:03 +000080 storageSize: "100Gi",
Piotr Dobrowolskia2226912019-05-14 18:49:29 +020081 },
82 },
83
Serge Bazanskic0c037a2020-08-23 01:24:03 +000084 dataVolume: kube.PersistentVolumeClaim("synapse-data-waw3") {
Piotr Dobrowolskia2226912019-05-14 18:49:29 +020085 metadata+: app.metadata("synapse-data"),
86 spec+: {
Serge Bazanskide627512020-08-24 21:17:55 +000087 storageClassName: cfg.storageClassName,
Piotr Dobrowolskia2226912019-05-14 18:49:29 +020088 accessModes: [ "ReadWriteOnce" ],
89 resources: {
90 requests: {
91 storage: "50Gi",
92 },
93 },
94 },
95 },
Piotr Dobrowolskiffbb47c2019-05-16 12:18:39 +020096
Serge Bazanski21a96162020-11-03 23:34:36 +010097 // homeserver.yaml that will be used to run synapse (in synapseConfigMap).
Serge Bazanskiace32c02020-11-03 22:04:06 +010098 // This is based off of //app/matrix/lib/synapse/homeserver.yaml with some fields overriden per
99 // deployment.
100 // Note this is a templated yaml - {{}}/{%%} style. This templatization is consumed by the Docker
101 // container startup magic.
Serge Bazanski21a96162020-11-03 23:34:36 +0100102 synapseConfig:: (std.native("parseYaml"))(importstr "synapse/homeserver.yaml")[0] {
Serge Bazanskiace32c02020-11-03 22:04:06 +0100103 server_name: cfg.serverName,
104 public_baseurl: "https://%s" % [cfg.webDomain],
105 signing_key_path: "/data/%s.signing.key" % [cfg.serverName],
Serge Bazanski21a96162020-11-03 23:34:36 +0100106 app_service_config_files: [
107 "/appservices/%s/registration.yaml" % [k]
108 for k in std.objectFields(app.appservices)
109 ],
110 } + (if cfg.cas.enable then {
111 cas_config: {
Serge Bazanskiace32c02020-11-03 22:04:06 +0100112 enabled: true,
113 server_url: "https://%s/_cas" % [cfg.webDomain],
114 service_url: "https://%s" % [cfg.webDomain],
Serge Bazanski21a96162020-11-03 23:34:36 +0100115 },
116 } else {}),
Serge Bazanskiace32c02020-11-03 22:04:06 +0100117
Serge Bazanski21a96162020-11-03 23:34:36 +0100118 synapseConfigMap: kube.ConfigMap("synapse") {
Piotr Dobrowolskic39fb042019-05-17 09:13:56 +0200119 metadata+: app.metadata("synapse"),
120 data: {
Serge Bazanski21a96162020-11-03 23:34:36 +0100121 "homeserver.yaml": std.manifestYamlDoc(app.synapseConfig),
Serge Bazanski60076c72020-11-03 19:17:25 +0100122 "log.config": importstr "synapse/log.config",
Piotr Dobrowolskic39fb042019-05-17 09:13:56 +0200123 },
124 },
125
Serge Bazanskiace32c02020-11-03 22:04:06 +0100126 casDeployment: if cfg.cas.enable then kube.Deployment("oauth2-cas-proxy") {
Piotr Dobrowolskic39fb042019-05-17 09:13:56 +0200127 metadata+: app.metadata("oauth2-cas-proxy"),
128 spec+: {
129 replicas: 1,
130 template+: {
131 spec+: {
132 containers_: {
133 proxy: kube.Container("oauth2-cas-proxy") {
Serge Bazanskiace32c02020-11-03 22:04:06 +0100134 image: cfg.images.casProxy,
Piotr Dobrowolskic39fb042019-05-17 09:13:56 +0200135 ports_: {
136 http: { containerPort: 5000 },
137 },
138 env_: {
Serge Bazanski60076c72020-11-03 19:17:25 +0100139 BASE_URL: "https://%s" % [cfg.webDomain],
140 SERVICE_URL: "https://%s" % [cfg.webDomain],
Piotr Dobrowolskic39fb042019-05-17 09:13:56 +0200141 OAUTH2_CLIENT: "matrix",
142 OAUTH2_SECRET: { secretKeyRef: { name: "oauth2-cas-proxy", key: "oauth2_secret" } },
143 },
144 },
145 },
146 },
147 },
148 },
149 },
150
Serge Bazanskiace32c02020-11-03 22:04:06 +0100151 casSvc: if cfg.cas.enable then kube.Service("oauth2-cas-proxy") {
Piotr Dobrowolskic39fb042019-05-17 09:13:56 +0200152 metadata+: app.metadata("oauth2-cas-proxy"),
153 target_pod:: app.casDeployment.spec.template,
154 },
155
Piotr Dobrowolskifef4c122019-05-16 21:05:02 +0200156 synapseDeployment: kube.Deployment("synapse") {
Piotr Dobrowolskia2226912019-05-14 18:49:29 +0200157 metadata+: app.metadata("synapse"),
158 spec+: {
159 replicas: 1,
160 template+: {
161 spec+: {
162 volumes_: {
163 data: kube.PersistentVolumeClaimVolume(app.dataVolume),
Serge Bazanski21a96162020-11-03 23:34:36 +0100164 config: kube.ConfigMapVolume(app.synapseConfigMap),
Piotr Dobrowolskifef4c122019-05-16 21:05:02 +0200165 } + {
166 [k]: { secret: { secretName: "appservice-%s-registration" % [k] } }
167 for k in std.objectFields(app.appservices)
Piotr Dobrowolskia2226912019-05-14 18:49:29 +0200168 },
169 containers_: {
170 web: kube.Container("synapse") {
Serge Bazanskiace32c02020-11-03 22:04:06 +0100171 image: cfg.images.synapse,
Piotr Dobrowolski8ebfc1d2020-03-03 21:01:18 +0100172 command: ["/bin/sh", "-c", "/start.py migrate_config && exec /start.py"],
Piotr Dobrowolskia2226912019-05-14 18:49:29 +0200173 ports_: {
174 http: { containerPort: 8008 },
Serge Bazanski1230ac32020-09-12 22:09:46 +0000175 metrics: { containerPort: 9092 },
Piotr Dobrowolskia2226912019-05-14 18:49:29 +0200176 },
177 env_: {
Serge Bazanski21a96162020-11-03 23:34:36 +0100178 SYNAPSE_CONFIG_DIR: "/tmp/config",
179 SYNAPSE_CONFIG_PATH: "/tmp/config/homeserver.yaml",
Piotr Dobrowolskia2226912019-05-14 18:49:29 +0200180
Piotr Dobrowolski8ebfc1d2020-03-03 21:01:18 +0100181 # These values are not used in a template, but
182 # are required by /start.py migrate_config
Serge Bazanski60076c72020-11-03 19:17:25 +0100183 SYNAPSE_SERVER_NAME: cfg.serverName,
Piotr Dobrowolski8ebfc1d2020-03-03 21:01:18 +0100184 SYNAPSE_REPORT_STATS: "no",
185
186 SYNAPSE_MACAROON_SECRET_KEY: { secretKeyRef: { name: "synapse", key: "macaroon_secret_key" } },
187 SYNAPSE_REGISTRATION_SHARED_SECRET: { secretKeyRef: { name: "synapse", key: "registration_shared_secret" } },
188 POSTGRES_PASSWORD: { secretKeyRef: { name: "synapse", key: "postgres_password" } },
Piotr Dobrowolskia2226912019-05-14 18:49:29 +0200189 },
190 volumeMounts_: {
191 data: { mountPath: "/data" },
Serge Bazanski21a96162020-11-03 23:34:36 +0100192 config: { mountPath: "/conf", },
Piotr Dobrowolskifef4c122019-05-16 21:05:02 +0200193 } + {
194 [k]: { mountPath: "/appservices/%s" % [k] }
195 for k in std.objectFields(app.appservices)
Piotr Dobrowolskia2226912019-05-14 18:49:29 +0200196 },
197 },
198 },
Serge Bazanski21a96162020-11-03 23:34:36 +0100199 securityContext: {
200 runAsUser: 991,
201 runAsGroup: 991,
202 fsGroup: 991,
203 },
Piotr Dobrowolskia2226912019-05-14 18:49:29 +0200204 },
205 },
206 },
207 },
208
Piotr Dobrowolskifef4c122019-05-16 21:05:02 +0200209 synapseSvc: kube.Service("synapse") {
Piotr Dobrowolskiffbb47c2019-05-16 12:18:39 +0200210 metadata+: app.metadata("synapse"),
Piotr Dobrowolskifef4c122019-05-16 21:05:02 +0200211 target_pod:: app.synapseDeployment.spec.template,
Piotr Dobrowolskiffbb47c2019-05-16 12:18:39 +0200212 },
213
Piotr Dobrowolskia2226912019-05-14 18:49:29 +0200214 riotConfig: kube.ConfigMap("riot-web-config") {
215 metadata+: app.metadata("riot-web-config"),
216 data: {
217 "config.json": std.manifestJsonEx({
Serge Bazanski60076c72020-11-03 19:17:25 +0100218 "default_hs_url": "https://%s" % [cfg.webDomain],
Piotr Dobrowolskia2226912019-05-14 18:49:29 +0200219 "disable_custom_urls": false,
220 "disable_guests": false,
221 "disable_login_language_selector": false,
Piotr Dobrowolski4b4231d2019-05-15 11:41:21 +0200222 "disable_3pid_login": true,
Piotr Dobrowolskia2226912019-05-14 18:49:29 +0200223 "brand": "Riot",
224 "integrations_ui_url": "https://scalar.vector.im/",
225 "integrations_rest_url": "https://scalar.vector.im/api",
226 "integrations_jitsi_widget_url": "https://scalar.vector.im/api/widgets/jitsi.html",
Piotr Dobrowolski4b4231d2019-05-15 11:41:21 +0200227
Piotr Dobrowolskia2226912019-05-14 18:49:29 +0200228 "bug_report_endpoint_url": "https://riot.im/bugreports/submit",
229 "features": {
230 "feature_groups": "labs",
231 "feature_pinning": "labs",
232 "feature_reactions": "labs"
233 },
234 "default_federate": true,
235 "default_theme": "light",
236 "roomDirectory": {
237 "servers": [
Serge Bazanski60076c72020-11-03 19:17:25 +0100238 cfg.serverName,
Piotr Dobrowolskia2226912019-05-14 18:49:29 +0200239 ]
240 },
241 "welcomeUserId": "@riot-bot:matrix.org",
Piotr Dobrowolskia2226912019-05-14 18:49:29 +0200242 "enable_presence_by_hs_url": {
243 "https://matrix.org": false
244 }
245 }, ""),
Serge Bazanski21a96162020-11-03 23:34:36 +0100246 // Standard nginx.conf, made to work when running as unprivileged user.
247 "nginx.conf": |||
248 worker_processes auto;
249
250 error_log /tmp/nginx_error.log warn;
251 pid /tmp/nginx.pid;
252
253 events {
254 worker_connections 1024;
255 }
256
257
258 http {
259 client_body_temp_path /tmp/nginx_client_temp;
260 proxy_temp_path /tmp/nginx_proxy_temp;
261 fastcgi_temp_path /tmp/nginx_fastcgi_temp;
262 uwsgi_temp_path /tmp/nginx_uwsgi_temp;
263 scgi_temp_path /tmp/nginx_scgi_temp;
264
265 include /etc/nginx/mime.types;
266 default_type application/octet-stream;
267 log_format main '$remote_addr - $remote_user [$time_local] "$request" '
268 '$status $body_bytes_sent "$http_referer" '
269 '"$http_user_agent" "$http_x_forwarded_for"';
270 access_log /tmp/nginx_access.log main;
271 sendfile on;
272 keepalive_timeout 65;
273
274 server {
275 listen 8080;
276 server_name localhost;
277
278 location / {
279 root /usr/share/nginx/html;
280 index index.html index.htm;
281 }
282
283 error_page 500 502 503 504 /50x.html;
284 location = /50x.html {
285 root /usr/share/nginx/html;
286 }
287 }
288 }
289 |||,
Piotr Dobrowolskia2226912019-05-14 18:49:29 +0200290 },
291 },
292
293 riotDeployment: kube.Deployment("riot-web") {
294 metadata+: app.metadata("riot-web"),
295 spec+: {
296 replicas: 1,
297 template+: {
298 spec+: {
299 volumes_: {
300 config: kube.ConfigMapVolume(app.riotConfig),
301 },
302 containers_: {
Piotr Dobrowolskifef4c122019-05-16 21:05:02 +0200303 web: kube.Container("riot-web") {
Serge Bazanskiace32c02020-11-03 22:04:06 +0100304 image: cfg.images.riot,
Piotr Dobrowolskia2226912019-05-14 18:49:29 +0200305 ports_: {
Serge Bazanski21a96162020-11-03 23:34:36 +0100306 http: { containerPort: 8080 },
Piotr Dobrowolskia2226912019-05-14 18:49:29 +0200307 },
Serge Bazanski21a96162020-11-03 23:34:36 +0100308 volumeMounts: [
309 {
310 name: "config",
Piotr Dobrowolskiaca7e282020-03-21 22:14:38 +0100311 mountPath: "/app/config.json",
Piotr Dobrowolskia2226912019-05-14 18:49:29 +0200312 subPath: "config.json",
313 },
Serge Bazanski21a96162020-11-03 23:34:36 +0100314 {
315 name: "config",
316 mountPath: "/etc/nginx/nginx.conf",
317 subPath: "nginx.conf",
318 },
319 ],
Piotr Dobrowolskia2226912019-05-14 18:49:29 +0200320 },
321 },
Serge Bazanski21a96162020-11-03 23:34:36 +0100322 securityContext: {
323 // nginx:nginx
324 runAsUser: 101,
325 runAsGroup: 101,
326 },
Piotr Dobrowolskia2226912019-05-14 18:49:29 +0200327 },
328 },
329 },
330 },
331
Piotr Dobrowolskia2226912019-05-14 18:49:29 +0200332 riotSvc: kube.Service("riot-web") {
333 metadata+: app.metadata("riot-web"),
334 target_pod:: app.riotDeployment.spec.template,
Piotr Dobrowolskia2226912019-05-14 18:49:29 +0200335 },
336
Serge Bazanskide627512020-08-24 21:17:55 +0000337 // Any appservice you add here will require an appservice-X-registration
338 // secret containing a registration.yaml file. Adding something to this
339 // dictionary will cause Synapse to not start until that secret is
340 // available - so change things carefully!
341 // If bootstrapping a new appservice, just keep it out of this dictionary
342 // until it spits you a registration YAML and you feed that to a secret.
Serge Bazanski60076c72020-11-03 19:17:25 +0100343 appservices: {},
Piotr Dobrowolskifef4c122019-05-16 21:05:02 +0200344
345 ingress: kube.Ingress("matrix") {
346 metadata+: app.metadata("matrix") {
Piotr Dobrowolskia2226912019-05-14 18:49:29 +0200347 annotations+: {
348 "kubernetes.io/tls-acme": "true",
349 "certmanager.k8s.io/cluster-issuer": "letsencrypt-prod",
350 "nginx.ingress.kubernetes.io/proxy-body-size": "0",
351 },
352 },
353 spec+: {
354 tls: [
355 {
Serge Bazanski60076c72020-11-03 19:17:25 +0100356 hosts: [cfg.webDomain],
Piotr Dobrowolskia2226912019-05-14 18:49:29 +0200357 secretName: "synapse-tls",
358 },
359 ],
360 rules: [
361 {
Serge Bazanski60076c72020-11-03 19:17:25 +0100362 host: cfg.webDomain,
Piotr Dobrowolskia2226912019-05-14 18:49:29 +0200363 http: {
364 paths: [
365 { path: "/", backend: app.riotSvc.name_port },
Piotr Dobrowolskifef4c122019-05-16 21:05:02 +0200366 { path: "/_matrix", backend: app.synapseSvc.name_port },
Serge Bazanskiace32c02020-11-03 22:04:06 +0100367 ] + (if cfg.cas.enable then [
Piotr Dobrowolskic39fb042019-05-17 09:13:56 +0200368 { path: "/_cas", backend: app.casSvc.name_port },
Serge Bazanskiace32c02020-11-03 22:04:06 +0100369 ] else [])
Piotr Dobrowolskia2226912019-05-14 18:49:29 +0200370 },
371 }
372 ],
373 },
374 },
Piotr Dobrowolskifef4c122019-05-16 21:05:02 +0200375
Piotr Dobrowolskia2226912019-05-14 18:49:29 +0200376}