blob: 4190941be940960657ee2a61fdb308e1537ba2b8 [file] [log] [blame]
Serge Bazanski60076c72020-11-03 19:17:25 +01001# Matrix server (synapse).
Piotr Dobrowolskia2226912019-05-14 18:49:29 +02002# This needs a secret provisioned, create with:
Serge Bazanski60076c72020-11-03 19:17:25 +01003# ns=matrix
4# kubectl -n $ns create secret generic synapse --from-literal=postgres_password=$(pwgen 24 1) --from-literal=macaroon_secret_key=$(pwgen 32 1) --from-literal=registration_shared_secret=$(pwgen 32 1)
5# kubectl -n $ns create secret generic oauth2-cas-proxy --from-literal=oauth2_secret=...
Serge Bazanskide627512020-08-24 21:17:55 +00006#
Serge Bazanski21a96162020-11-03 23:34:36 +01007# After starting, re-create the postgres database (because docker/postgres won't let you set ENCODING):
8# postgres=# drop database synapse;
9# DROP DATABASE
10# postgres=# CREATE DATABASE synapse
11# postgres-# ENCODING 'UTF8'
12# postgres-# LC_COLLATE='C'
13# postgres-# LC_CTYPE='C'
14# postgres-# template=template0
15# postgres-# OWNER synapse;
16# CREATE DATABASE
17#
Serge Bazanskide627512020-08-24 21:17:55 +000018# Sequencing appservices is fun. The appservice needs to run first (for
19# instance, via a bootstrap job), and on startup it will spit out a
20# registration file. This registration file then needs to be fed to synapse -
21# this is done via specialy named secrets (appservice-X-registration, for X key
22# in the appservices object).
23#
24# For appservice-irc instances, you can use this oneliner magic to get the
25# registration YAML from logs.
Piotr Dobrowolski3ea979d2019-05-23 16:11:52 +020026# kubectl -n matrix create secret generic appservice-irc-freenode-registration --from-file=registration.yaml=<(kubectl logs -n matrix $(kubectl get pods -n matrix --selector=job-name=appservice-irc-freenode-bootstrap --output=jsonpath='{.items[*].metadata.name}') | tail -n +4 | sed -r 's/(.*aliases:.*)/ group_id: "+freenode:hackerspace.pl"\n\1/')
Serge Bazanskide627512020-08-24 21:17:55 +000027#
28# For appservice-telegram instances, you can use this oneliner magic:
29# kubectl -n matrix create secret generic appservice-telegram-prod-registration --from-file=registration.yaml=<(kubectl -n matrix logs job/appservice-telegram-prod-bootstrap | grep -A 100 SNIPSNIP | grep -v SNIPSNIP)
Piotr Dobrowolskia2226912019-05-14 18:49:29 +020030
Serge Bazanski60076c72020-11-03 19:17:25 +010031local kube = import "../../../kube/kube.libsonnet";
32local postgres = import "../../../kube/postgres.libsonnet";
Serge Bazanskicdba2912020-08-24 19:11:10 +000033
Piotr Dobrowolskia2226912019-05-14 18:49:29 +020034{
35 local app = self,
36 local cfg = app.cfg,
37 cfg:: {
Serge Bazanski60076c72020-11-03 19:17:25 +010038 namespace: error "cfg.namespace must be set",
39 # webDomain is the domain name at which element will run
40 webDomain: error "cfg.webDomain must be set",
41 # serverName is the server part of the MXID this homeserver will cover
42 serverName: error "cfg.serverName must be set",
Serge Bazanskide627512020-08-24 21:17:55 +000043 storageClassName: "waw-hdd-redundant-3",
Piotr Dobrowolskifef4c122019-05-16 21:05:02 +020044
Serge Bazanskiace32c02020-11-03 22:04:06 +010045 images: {
46 synapse: "matrixdotorg/synapse:v1.19.2",
47 riot: "vectorim/riot-web:v1.7.7",
48 casProxy: "registry.k0.hswaw.net/q3k/oauth2-cas-proxy:0.1.4",
49 appserviceIRC: "matrixdotorg/matrix-appservice-irc:release-0.17.1",
50 # That's v0.8.2 - we just don't trust that host to not re-tag images.
51 appserviceTelegram: "dock.mau.dev/tulir/mautrix-telegram@sha256:9e68eaa80c9e4a75d9a09ec92dc4898b12d48390e01efa4de40ce882a6f7e330",
52 },
53
54 cas: {
55 # whether to enable the CAS proxy (ie. connect to hswaw sso via OAuth)
56 enable: false,
Serge Bazanski8483d372020-11-10 22:07:30 +010057 # generate client ID and secret in with your OAuth2 provider, refer to https://www.oauth.com/oauth2-servers/client-registration/client-id-secret/
58 oauth2: {
59 clientID: error "cas.oauth2.clientID must be set",
60 clientSecret: error "cas.oauth2.clientSecret must be set",
61 scope: error "cas.oauth2.scope must be set",
62 authorizeURL: error "cas.oauth2.authorizeURL must be set",
63 tokenURL: error "cas.oauth2.tokenURL must be set",
64 userinfoURL: error "cas.oauth2.userinfoURL must be set",
65 },
Serge Bazanskiace32c02020-11-03 22:04:06 +010066 },
Piotr Dobrowolskia2226912019-05-14 18:49:29 +020067 },
68
69 metadata(component):: {
Piotr Dobrowolski4b4231d2019-05-15 11:41:21 +020070 namespace: cfg.namespace,
Piotr Dobrowolskia2226912019-05-14 18:49:29 +020071 labels: {
72 "app.kubernetes.io/name": "matrix",
73 "app.kubernetes.io/managed-by": "kubecfg",
74 "app.kubernetes.io/component": component,
75 },
76 },
77
Piotr Dobrowolski4b4231d2019-05-15 11:41:21 +020078 namespace: kube.Namespace(cfg.namespace),
Piotr Dobrowolskia2226912019-05-14 18:49:29 +020079
Serge Bazanskic0c037a2020-08-23 01:24:03 +000080 postgres3: postgres {
Piotr Dobrowolskia2226912019-05-14 18:49:29 +020081 cfg+: {
82 namespace: cfg.namespace,
83 appName: "synapse",
84 database: "synapse",
85 username: "synapse",
Serge Bazanskic0c037a2020-08-23 01:24:03 +000086 prefix: "waw3-",
Piotr Dobrowolskia2226912019-05-14 18:49:29 +020087 password: { secretKeyRef: { name: "synapse", key: "postgres_password" } },
Serge Bazanskide627512020-08-24 21:17:55 +000088 storageClassName: cfg.storageClassName,
Serge Bazanskic0c037a2020-08-23 01:24:03 +000089 storageSize: "100Gi",
Piotr Dobrowolskia2226912019-05-14 18:49:29 +020090 },
91 },
92
Serge Bazanskic0c037a2020-08-23 01:24:03 +000093 dataVolume: kube.PersistentVolumeClaim("synapse-data-waw3") {
Piotr Dobrowolskia2226912019-05-14 18:49:29 +020094 metadata+: app.metadata("synapse-data"),
95 spec+: {
Serge Bazanskide627512020-08-24 21:17:55 +000096 storageClassName: cfg.storageClassName,
Piotr Dobrowolskia2226912019-05-14 18:49:29 +020097 accessModes: [ "ReadWriteOnce" ],
98 resources: {
99 requests: {
100 storage: "50Gi",
101 },
102 },
103 },
104 },
Piotr Dobrowolskiffbb47c2019-05-16 12:18:39 +0200105
Serge Bazanski21a96162020-11-03 23:34:36 +0100106 // homeserver.yaml that will be used to run synapse (in synapseConfigMap).
Serge Bazanskiace32c02020-11-03 22:04:06 +0100107 // This is based off of //app/matrix/lib/synapse/homeserver.yaml with some fields overriden per
108 // deployment.
109 // Note this is a templated yaml - {{}}/{%%} style. This templatization is consumed by the Docker
110 // container startup magic.
Serge Bazanski21a96162020-11-03 23:34:36 +0100111 synapseConfig:: (std.native("parseYaml"))(importstr "synapse/homeserver.yaml")[0] {
Serge Bazanskiace32c02020-11-03 22:04:06 +0100112 server_name: cfg.serverName,
113 public_baseurl: "https://%s" % [cfg.webDomain],
114 signing_key_path: "/data/%s.signing.key" % [cfg.serverName],
Serge Bazanski21a96162020-11-03 23:34:36 +0100115 app_service_config_files: [
116 "/appservices/%s/registration.yaml" % [k]
117 for k in std.objectFields(app.appservices)
118 ],
119 } + (if cfg.cas.enable then {
120 cas_config: {
Serge Bazanskiace32c02020-11-03 22:04:06 +0100121 enabled: true,
122 server_url: "https://%s/_cas" % [cfg.webDomain],
123 service_url: "https://%s" % [cfg.webDomain],
Serge Bazanski21a96162020-11-03 23:34:36 +0100124 },
125 } else {}),
Serge Bazanskiace32c02020-11-03 22:04:06 +0100126
Serge Bazanski21a96162020-11-03 23:34:36 +0100127 synapseConfigMap: kube.ConfigMap("synapse") {
Piotr Dobrowolskic39fb042019-05-17 09:13:56 +0200128 metadata+: app.metadata("synapse"),
129 data: {
Serge Bazanski21a96162020-11-03 23:34:36 +0100130 "homeserver.yaml": std.manifestYamlDoc(app.synapseConfig),
Serge Bazanski60076c72020-11-03 19:17:25 +0100131 "log.config": importstr "synapse/log.config",
Piotr Dobrowolskic39fb042019-05-17 09:13:56 +0200132 },
133 },
134
Serge Bazanskiace32c02020-11-03 22:04:06 +0100135 casDeployment: if cfg.cas.enable then kube.Deployment("oauth2-cas-proxy") {
Piotr Dobrowolskic39fb042019-05-17 09:13:56 +0200136 metadata+: app.metadata("oauth2-cas-proxy"),
137 spec+: {
138 replicas: 1,
139 template+: {
140 spec+: {
141 containers_: {
142 proxy: kube.Container("oauth2-cas-proxy") {
Serge Bazanskiace32c02020-11-03 22:04:06 +0100143 image: cfg.images.casProxy,
Piotr Dobrowolskic39fb042019-05-17 09:13:56 +0200144 ports_: {
145 http: { containerPort: 5000 },
146 },
147 env_: {
Serge Bazanski60076c72020-11-03 19:17:25 +0100148 BASE_URL: "https://%s" % [cfg.webDomain],
149 SERVICE_URL: "https://%s" % [cfg.webDomain],
Serge Bazanski8483d372020-11-10 22:07:30 +0100150 OAUTH2_CLIENT: cfg.cas.oauth2.clientID,
151 OAUTH2_SECRET: cfg.cas.oauth2.clientSecret,
152 OAUTH2_SCOPE: cfg.cas.oauth2.scope,
153 OAUTH2_AUTHORIZE: cfg.cas.oauth2.authorizeURL,
154 OAUTH2_TOKEN: cfg.cas.oauth2.tokenURL,
155 OAUTH2_USERINFO: cfg.cas.oauth2.userinfoURL,
Piotr Dobrowolskic39fb042019-05-17 09:13:56 +0200156 },
157 },
158 },
159 },
160 },
161 },
162 },
163
Serge Bazanskiace32c02020-11-03 22:04:06 +0100164 casSvc: if cfg.cas.enable then kube.Service("oauth2-cas-proxy") {
Piotr Dobrowolskic39fb042019-05-17 09:13:56 +0200165 metadata+: app.metadata("oauth2-cas-proxy"),
166 target_pod:: app.casDeployment.spec.template,
167 },
168
Piotr Dobrowolskifef4c122019-05-16 21:05:02 +0200169 synapseDeployment: kube.Deployment("synapse") {
Piotr Dobrowolskia2226912019-05-14 18:49:29 +0200170 metadata+: app.metadata("synapse"),
171 spec+: {
172 replicas: 1,
173 template+: {
174 spec+: {
175 volumes_: {
176 data: kube.PersistentVolumeClaimVolume(app.dataVolume),
Serge Bazanski21a96162020-11-03 23:34:36 +0100177 config: kube.ConfigMapVolume(app.synapseConfigMap),
Piotr Dobrowolskifef4c122019-05-16 21:05:02 +0200178 } + {
179 [k]: { secret: { secretName: "appservice-%s-registration" % [k] } }
180 for k in std.objectFields(app.appservices)
Piotr Dobrowolskia2226912019-05-14 18:49:29 +0200181 },
182 containers_: {
183 web: kube.Container("synapse") {
Serge Bazanskiace32c02020-11-03 22:04:06 +0100184 image: cfg.images.synapse,
Piotr Dobrowolski8ebfc1d2020-03-03 21:01:18 +0100185 command: ["/bin/sh", "-c", "/start.py migrate_config && exec /start.py"],
Piotr Dobrowolskia2226912019-05-14 18:49:29 +0200186 ports_: {
187 http: { containerPort: 8008 },
Serge Bazanski1230ac32020-09-12 22:09:46 +0000188 metrics: { containerPort: 9092 },
Piotr Dobrowolskia2226912019-05-14 18:49:29 +0200189 },
190 env_: {
Serge Bazanski21a96162020-11-03 23:34:36 +0100191 SYNAPSE_CONFIG_DIR: "/tmp/config",
192 SYNAPSE_CONFIG_PATH: "/tmp/config/homeserver.yaml",
Piotr Dobrowolskia2226912019-05-14 18:49:29 +0200193
Piotr Dobrowolski8ebfc1d2020-03-03 21:01:18 +0100194 # These values are not used in a template, but
195 # are required by /start.py migrate_config
Serge Bazanski60076c72020-11-03 19:17:25 +0100196 SYNAPSE_SERVER_NAME: cfg.serverName,
Piotr Dobrowolski8ebfc1d2020-03-03 21:01:18 +0100197 SYNAPSE_REPORT_STATS: "no",
198
199 SYNAPSE_MACAROON_SECRET_KEY: { secretKeyRef: { name: "synapse", key: "macaroon_secret_key" } },
200 SYNAPSE_REGISTRATION_SHARED_SECRET: { secretKeyRef: { name: "synapse", key: "registration_shared_secret" } },
201 POSTGRES_PASSWORD: { secretKeyRef: { name: "synapse", key: "postgres_password" } },
Piotr Dobrowolskia2226912019-05-14 18:49:29 +0200202 },
203 volumeMounts_: {
204 data: { mountPath: "/data" },
Serge Bazanski21a96162020-11-03 23:34:36 +0100205 config: { mountPath: "/conf", },
Piotr Dobrowolskifef4c122019-05-16 21:05:02 +0200206 } + {
207 [k]: { mountPath: "/appservices/%s" % [k] }
208 for k in std.objectFields(app.appservices)
Piotr Dobrowolskia2226912019-05-14 18:49:29 +0200209 },
210 },
211 },
Serge Bazanski21a96162020-11-03 23:34:36 +0100212 securityContext: {
213 runAsUser: 991,
214 runAsGroup: 991,
215 fsGroup: 991,
216 },
Piotr Dobrowolskia2226912019-05-14 18:49:29 +0200217 },
218 },
219 },
220 },
221
Piotr Dobrowolskifef4c122019-05-16 21:05:02 +0200222 synapseSvc: kube.Service("synapse") {
Piotr Dobrowolskiffbb47c2019-05-16 12:18:39 +0200223 metadata+: app.metadata("synapse"),
Piotr Dobrowolskifef4c122019-05-16 21:05:02 +0200224 target_pod:: app.synapseDeployment.spec.template,
Piotr Dobrowolskiffbb47c2019-05-16 12:18:39 +0200225 },
226
Norbert Szulc1ef56002020-11-08 16:13:58 +0100227 riotConfig:: {
228 "default_hs_url": "https://%s" % [cfg.webDomain],
229 "disable_custom_urls": false,
230 "disable_guests": false,
231 "disable_login_language_selector": false,
232 "disable_3pid_login": true,
233 "brand": "Riot",
234 "integrations_ui_url": "https://scalar.vector.im/",
235 "integrations_rest_url": "https://scalar.vector.im/api",
236 "integrations_jitsi_widget_url": "https://scalar.vector.im/api/widgets/jitsi.html",
237
238 "bug_report_endpoint_url": "https://riot.im/bugreports/submit",
239 "features": {
240 "feature_groups": "labs",
241 "feature_pinning": "labs",
242 "feature_reactions": "labs"
243 },
244 "default_federate": true,
245 "default_theme": "light",
246 "roomDirectory": {
247 "servers": [
248 cfg.serverName,
249 ]
250 },
251 "welcomeUserId": "@riot-bot:matrix.org",
252 "enable_presence_by_hs_url": {
253 "https://matrix.org": false
254 }
255 },
256
257 riotConfigMap: kube.ConfigMap("riot-web-config") {
Piotr Dobrowolskia2226912019-05-14 18:49:29 +0200258 metadata+: app.metadata("riot-web-config"),
259 data: {
Norbert Szulc1ef56002020-11-08 16:13:58 +0100260 "config.json": std.manifestJsonEx(app.riotConfig, ""),
Serge Bazanski21a96162020-11-03 23:34:36 +0100261 // Standard nginx.conf, made to work when running as unprivileged user.
262 "nginx.conf": |||
263 worker_processes auto;
264
265 error_log /tmp/nginx_error.log warn;
266 pid /tmp/nginx.pid;
267
268 events {
269 worker_connections 1024;
270 }
271
272
273 http {
274 client_body_temp_path /tmp/nginx_client_temp;
275 proxy_temp_path /tmp/nginx_proxy_temp;
276 fastcgi_temp_path /tmp/nginx_fastcgi_temp;
277 uwsgi_temp_path /tmp/nginx_uwsgi_temp;
278 scgi_temp_path /tmp/nginx_scgi_temp;
279
280 include /etc/nginx/mime.types;
281 default_type application/octet-stream;
282 log_format main '$remote_addr - $remote_user [$time_local] "$request" '
283 '$status $body_bytes_sent "$http_referer" '
284 '"$http_user_agent" "$http_x_forwarded_for"';
285 access_log /tmp/nginx_access.log main;
286 sendfile on;
287 keepalive_timeout 65;
288
289 server {
290 listen 8080;
291 server_name localhost;
292
293 location / {
294 root /usr/share/nginx/html;
295 index index.html index.htm;
296 }
297
298 error_page 500 502 503 504 /50x.html;
299 location = /50x.html {
300 root /usr/share/nginx/html;
301 }
302 }
303 }
304 |||,
Piotr Dobrowolskia2226912019-05-14 18:49:29 +0200305 },
306 },
307
308 riotDeployment: kube.Deployment("riot-web") {
309 metadata+: app.metadata("riot-web"),
310 spec+: {
311 replicas: 1,
312 template+: {
313 spec+: {
314 volumes_: {
Norbert Szulc1ef56002020-11-08 16:13:58 +0100315 config: kube.ConfigMapVolume(app.riotConfigMap),
Piotr Dobrowolskia2226912019-05-14 18:49:29 +0200316 },
317 containers_: {
Piotr Dobrowolskifef4c122019-05-16 21:05:02 +0200318 web: kube.Container("riot-web") {
Serge Bazanskiace32c02020-11-03 22:04:06 +0100319 image: cfg.images.riot,
Piotr Dobrowolskia2226912019-05-14 18:49:29 +0200320 ports_: {
Serge Bazanski21a96162020-11-03 23:34:36 +0100321 http: { containerPort: 8080 },
Piotr Dobrowolskia2226912019-05-14 18:49:29 +0200322 },
Serge Bazanski21a96162020-11-03 23:34:36 +0100323 volumeMounts: [
324 {
325 name: "config",
Piotr Dobrowolskiaca7e282020-03-21 22:14:38 +0100326 mountPath: "/app/config.json",
Piotr Dobrowolskia2226912019-05-14 18:49:29 +0200327 subPath: "config.json",
328 },
Serge Bazanski21a96162020-11-03 23:34:36 +0100329 {
330 name: "config",
331 mountPath: "/etc/nginx/nginx.conf",
332 subPath: "nginx.conf",
333 },
334 ],
Piotr Dobrowolskia2226912019-05-14 18:49:29 +0200335 },
336 },
Serge Bazanski21a96162020-11-03 23:34:36 +0100337 securityContext: {
338 // nginx:nginx
339 runAsUser: 101,
340 runAsGroup: 101,
341 },
Piotr Dobrowolskia2226912019-05-14 18:49:29 +0200342 },
343 },
344 },
345 },
346
Piotr Dobrowolskia2226912019-05-14 18:49:29 +0200347 riotSvc: kube.Service("riot-web") {
348 metadata+: app.metadata("riot-web"),
349 target_pod:: app.riotDeployment.spec.template,
Piotr Dobrowolskia2226912019-05-14 18:49:29 +0200350 },
351
Serge Bazanskide627512020-08-24 21:17:55 +0000352 // Any appservice you add here will require an appservice-X-registration
353 // secret containing a registration.yaml file. Adding something to this
354 // dictionary will cause Synapse to not start until that secret is
355 // available - so change things carefully!
356 // If bootstrapping a new appservice, just keep it out of this dictionary
357 // until it spits you a registration YAML and you feed that to a secret.
Serge Bazanski60076c72020-11-03 19:17:25 +0100358 appservices: {},
Piotr Dobrowolskifef4c122019-05-16 21:05:02 +0200359
360 ingress: kube.Ingress("matrix") {
361 metadata+: app.metadata("matrix") {
Piotr Dobrowolskia2226912019-05-14 18:49:29 +0200362 annotations+: {
363 "kubernetes.io/tls-acme": "true",
364 "certmanager.k8s.io/cluster-issuer": "letsencrypt-prod",
365 "nginx.ingress.kubernetes.io/proxy-body-size": "0",
366 },
367 },
368 spec+: {
369 tls: [
370 {
Serge Bazanski60076c72020-11-03 19:17:25 +0100371 hosts: [cfg.webDomain],
Piotr Dobrowolskia2226912019-05-14 18:49:29 +0200372 secretName: "synapse-tls",
373 },
374 ],
375 rules: [
376 {
Serge Bazanski60076c72020-11-03 19:17:25 +0100377 host: cfg.webDomain,
Piotr Dobrowolskia2226912019-05-14 18:49:29 +0200378 http: {
379 paths: [
380 { path: "/", backend: app.riotSvc.name_port },
Piotr Dobrowolskifef4c122019-05-16 21:05:02 +0200381 { path: "/_matrix", backend: app.synapseSvc.name_port },
Serge Bazanskiace32c02020-11-03 22:04:06 +0100382 ] + (if cfg.cas.enable then [
Piotr Dobrowolskic39fb042019-05-17 09:13:56 +0200383 { path: "/_cas", backend: app.casSvc.name_port },
Serge Bazanskiace32c02020-11-03 22:04:06 +0100384 ] else [])
Piotr Dobrowolskia2226912019-05-14 18:49:29 +0200385 },
386 }
387 ],
388 },
389 },
Piotr Dobrowolskifef4c122019-05-16 21:05:02 +0200390
Piotr Dobrowolskia2226912019-05-14 18:49:29 +0200391}