Piotr Dobrowolski | b67ae48 | 2021-01-31 10:35:38 +0100 | [diff] [blame] | 1 | local kube = import "../../../kube/kube.libsonnet"; |
| 2 | |
| 3 | { |
| 4 | local app = self, |
| 5 | local cfg = app.cfg, |
| 6 | cfg:: { |
| 7 | image: error "cfg.image needs to be set", |
| 8 | storageClassName: error "cfg.storrageClassName needs to be set", |
| 9 | |
| 10 | # webDomain is the domain name at which synapse instance will run |
| 11 | webDomain: error "cfg.webDomain must be set", |
| 12 | # serverName is the server part of the MXID this homeserver will cover |
| 13 | serverName: error "cfg.serverName must be set", |
| 14 | |
| 15 | cas: { enable: false }, |
| 16 | oidc: { enable: false }, |
| 17 | |
| 18 | macaroonSecretKey: error "cfg.macaroonSecretKey needs to be set", |
| 19 | registrationSharedSecret: error "cfg.registationSharedSecret needs to be set", |
| 20 | workerReplicationSecret: error "cfg.workerReplicationSecret needs to be set", |
| 21 | }, |
| 22 | |
| 23 | ns:: error "ns needs to be provided", |
| 24 | postgres:: error "postgres needs to be provided", |
| 25 | redis:: error "redis needs to be provided", |
| 26 | |
| 27 | // See matrix-ng.libsonnet for description |
| 28 | appservices:: error "appservices need to be provided", |
| 29 | |
| 30 | dataVolume: app.ns.Contain(kube.PersistentVolumeClaim("synapse-data-waw3")) { |
| 31 | spec+: { |
| 32 | storageClassName: cfg.storageClassName, |
| 33 | accessModes: [ "ReadWriteOnce" ], |
| 34 | resources: { |
| 35 | requests: { |
| 36 | storage: "50Gi", |
| 37 | }, |
| 38 | }, |
| 39 | }, |
| 40 | }, |
| 41 | |
| 42 | // homeserver.yaml that will be used to run synapse (in configMap). |
| 43 | // This is based off of //app/matrix/lib/synapse/homeserver.yaml with some fields overriden per |
| 44 | // deployment. |
| 45 | config:: (std.native("parseYaml"))(importstr "synapse/homeserver-ng.yaml")[0] { |
| 46 | server_name: cfg.serverName, |
| 47 | public_baseurl: "https://%s" % [cfg.webDomain], |
| 48 | signing_key_path: "/secrets/homeserver_signing_key", |
| 49 | app_service_config_files: [ |
| 50 | "/appservices/%s/registration.yaml" % [k] |
| 51 | for k in std.objectFields(app.appservices) |
| 52 | ], |
| 53 | } + (if cfg.cas.enable then { |
| 54 | cas_config: { |
| 55 | enabled: true, |
| 56 | server_url: "https://%s/_cas" % [cfg.webDomain], |
| 57 | service_url: "https://%s" % [cfg.webDomain], |
| 58 | }, |
| 59 | } else {}), |
| 60 | |
| 61 | configMap: app.ns.Contain(kube.ConfigMap("synapse")) { |
| 62 | data: { |
| 63 | "homeserver.yaml": std.manifestYamlDoc(app.config), |
| 64 | "log.config": importstr "synapse/log.config", |
| 65 | }, |
| 66 | }, |
| 67 | |
| 68 | // homeserver-secrets.yaml contains all the templated secret variables from |
| 69 | // base homeserver.yaml passed as yaml-encoded environment variable. |
| 70 | // $(ENVVAR)-encoded variables are resolved by Kubernetes on pod startup |
| 71 | secretsConfig:: (std.native("parseYaml"))(importstr "synapse/homeserver-secrets.yaml")[0] { |
| 72 | } + (if cfg.oidc.enable then { |
| 73 | oidc_config: cfg.oidc.config { |
| 74 | enabled: true, |
| 75 | client_secret: "$(OIDC_CLIENT_SECRET)", |
| 76 | }, |
| 77 | } else {}), |
| 78 | |
| 79 | # Synapse process Deployment/StatefulSet base resource. |
| 80 | SynapseWorker(name, workerType, builder):: app.ns.Contain(builder(name)) { |
| 81 | local worker = self, |
| 82 | cfg:: { |
| 83 | # Configuration customization. Can contain environment substitution |
| 84 | # syntax, as used in worker_name value. |
| 85 | localConfig: { |
| 86 | worker_app: workerType, |
| 87 | worker_name: "$(POD_NAME)", |
| 88 | |
| 89 | # The replication listener on the main synapse process. |
| 90 | worker_replication_host: "synapse-replication-master", |
| 91 | worker_replication_http_port: 9093, |
| 92 | }, |
| 93 | |
| 94 | # Mount app.dataVolume in /data |
| 95 | mountData: false, |
| 96 | }, |
| 97 | |
| 98 | spec+: { |
| 99 | replicas: 1, |
| 100 | template+: { |
| 101 | spec+: { |
| 102 | volumes_: { |
| 103 | config: kube.ConfigMapVolume(app.configMap), |
| 104 | secrets: { secret: { secretName: "synapse" } }, |
| 105 | } + { |
| 106 | [k]: { secret: { secretName: "appservice-%s-registration" % [k] } } |
| 107 | for k in std.objectFields(app.appservices) |
| 108 | } + if worker.cfg.mountData then { |
| 109 | data: kube.PersistentVolumeClaimVolume(app.dataVolume), |
| 110 | } else {}, |
| 111 | containers_: { |
| 112 | web: kube.Container("synapse") { |
| 113 | image: cfg.image, |
| 114 | command: [ |
| 115 | "/bin/sh", "-c", ||| |
| 116 | set -e |
| 117 | echo "${X_SECRETS_CONFIG}" > /tmp/secrets.yaml |
| 118 | echo "${X_LOCAL_CONFIG}" > /tmp/local.yaml |
| 119 | exec python -m ${SYNAPSE_WORKER} --config-path /conf/homeserver.yaml --config-path /tmp/secrets.yaml --config-path /tmp/local.yaml |
| 120 | ||| |
| 121 | ], |
| 122 | ports_: { |
| 123 | http: { containerPort: 8008 }, |
| 124 | metrics: { containerPort: 9092 }, |
| 125 | replication: { containerPort: 9093 }, |
| 126 | }, |
| 127 | env_: { |
| 128 | SYNAPSE_WORKER: workerType, |
| 129 | |
| 130 | SYNAPSE_MACAROON_SECRET_KEY: cfg.macaroonSecretKey, |
| 131 | SYNAPSE_REGISTRATION_SHARED_SECRET: cfg.registrationSharedSecret, |
| 132 | WORKER_REPLICATION_SECRET: cfg.workerReplicationSecret, |
| 133 | POSTGRES_PASSWORD: app.postgres.cfg.password, |
| 134 | REDIS_PASSWORD: app.redis.cfg.password, |
| 135 | POD_NAME: { fieldRef: { fieldPath: "metadata.name" } }, |
| 136 | OIDC_CLIENT_SECRET: if cfg.oidc.enable then cfg.oidc.config.client_secret else "", |
| 137 | |
| 138 | X_SECRETS_CONFIG: std.manifestYamlDoc(app.secretsConfig), |
| 139 | X_LOCAL_CONFIG: std.manifestYamlDoc(worker.cfg.localConfig), |
| 140 | }, |
| 141 | volumeMounts_: { |
| 142 | config: { mountPath: "/conf", }, |
| 143 | secrets: { mountPath: "/secrets" }, |
| 144 | } + { |
| 145 | [k]: { mountPath: "/appservices/%s" % [k] } |
| 146 | for k in std.objectFields(app.appservices) |
| 147 | } + if worker.cfg.mountData then { |
| 148 | data: { mountPath: "/data" }, |
| 149 | } else {}, |
Piotr Dobrowolski | 77af94d | 2021-09-16 22:17:58 +0200 | [diff] [blame] | 150 | readinessProbe: { |
| 151 | httpGet: { |
| 152 | path: "/health", |
| 153 | port: "http", |
| 154 | }, |
| 155 | initialDelaySeconds: 5, |
| 156 | periodSeconds: 10, |
| 157 | }, |
| 158 | livenessProbe: { |
| 159 | httpGet: { |
| 160 | path: "/health", |
| 161 | port: "http", |
| 162 | }, |
| 163 | initialDelaySeconds: 60, |
| 164 | periodSeconds: 30, |
| 165 | }, |
Piotr Dobrowolski | b67ae48 | 2021-01-31 10:35:38 +0100 | [diff] [blame] | 166 | }, |
| 167 | }, |
| 168 | securityContext: { |
| 169 | runAsUser: 991, |
| 170 | runAsGroup: 991, |
| 171 | fsGroup: 991, |
| 172 | }, |
| 173 | }, |
| 174 | }, |
| 175 | }, |
| 176 | }, |
| 177 | |
| 178 | # Synapse main process |
| 179 | main: { |
| 180 | deployment: app.SynapseWorker("synapse", "synapse.app.homeserver", kube.Deployment) { |
| 181 | cfg+: { |
| 182 | # Main process doesn't need any configuration customization |
| 183 | localConfig: {} |
| 184 | }, |
| 185 | }, |
| 186 | svc: app.ns.Contain(kube.Service("synapse")) { |
| 187 | target_pod:: app.main.deployment.spec.template, |
| 188 | }, |
| 189 | replicationSvc: app.ns.Contain(kube.Service("synapse-replication-master")) { |
| 190 | target_pod:: app.main.deployment.spec.template, |
| 191 | spec+: { |
| 192 | ports: [ |
| 193 | { port: 9093, name: 'replication', targetPort: 9093 }, |
| 194 | ], |
| 195 | }, |
| 196 | }, |
| 197 | }, |
| 198 | |
| 199 | genericWorker: { |
| 200 | # Synapse generic worker deployment |
| 201 | deployment: app.SynapseWorker("synapse-generic", "synapse.app.generic_worker", kube.StatefulSet) { |
| 202 | cfg+: { |
| 203 | localConfig+: { |
| 204 | worker_listeners: [{ |
| 205 | type: "http", |
| 206 | port: 8008, |
| 207 | x_forwarded: true, |
| 208 | bind_addresses: ["::"], |
| 209 | resources: [{ names: ["client", "federation"]}], |
| 210 | }], |
| 211 | }, |
| 212 | }, |
| 213 | }, |
| 214 | svc: app.ns.Contain(kube.Service("synapse-generic")) { |
| 215 | target_pod:: app.genericWorker.deployment.spec.template, |
| 216 | }, |
| 217 | |
| 218 | # Following paths can be handled by generic workers. |
| 219 | # See: https://github.com/matrix-org/synapse/blob/master/docs/workers.md |
| 220 | paths:: [ |
| 221 | "/_matrix/client/(v2_alpha|r0)/sync", |
| 222 | "/_matrix/client/(api/v1|v2_alpha|r0)/events", |
| 223 | "/_matrix/client/(api/v1|r0)/initialSync", |
| 224 | "/_matrix/client/(api/v1|r0)/rooms/[^/]+/initialSync", |
| 225 | "/_matrix/client/(api/v1|r0|unstable)/publicRooms", |
| 226 | "/_matrix/client/(api/v1|r0|unstable)/rooms/.*/joined_members", |
| 227 | "/_matrix/client/(api/v1|r0|unstable)/rooms/.*/context/.*", |
| 228 | "/_matrix/client/(api/v1|r0|unstable)/rooms/.*/members", |
| 229 | "/_matrix/client/(api/v1|r0|unstable)/rooms/.*/state", |
| 230 | "/_matrix/client/(api/v1|r0|unstable)/account/3pid", |
| 231 | "/_matrix/client/(api/v1|r0|unstable)/keys/query", |
| 232 | "/_matrix/client/(api/v1|r0|unstable)/keys/changes", |
| 233 | "/_matrix/client/versions", |
| 234 | "/_matrix/client/(api/v1|r0|unstable)/voip/turnServer", |
| 235 | "/_matrix/client/(api/v1|r0|unstable)/joined_groups", |
| 236 | "/_matrix/client/(api/v1|r0|unstable)/publicised_groups", |
| 237 | "/_matrix/client/(api/v1|r0|unstable)/publicised_groups/", |
| 238 | # Blocked by https://github.com/matrix-org/synapse/issues/8966 |
| 239 | # "/_matrix/client/(api/v1|r0|unstable)/login", |
| 240 | # "/_matrix/client/(r0|unstable)/register", |
| 241 | # "/_matrix/client/(r0|unstable)/auth/.*/fallback/web", |
| 242 | "/_matrix/client/(api/v1|r0|unstable)/rooms/.*/send", |
| 243 | "/_matrix/client/(api/v1|r0|unstable)/rooms/.*/state/", |
| 244 | "/_matrix/client/(api/v1|r0|unstable)/rooms/.*/(join|invite|leave|ban|unban|kick)", |
| 245 | "/_matrix/client/(api/v1|r0|unstable)/join/", |
| 246 | "/_matrix/client/(api/v1|r0|unstable)/profile/", |
| 247 | "/_matrix/federation/v1/event/", |
| 248 | "/_matrix/federation/v1/state/", |
| 249 | "/_matrix/federation/v1/state_ids/", |
| 250 | "/_matrix/federation/v1/backfill/", |
| 251 | "/_matrix/federation/v1/get_missing_events/", |
| 252 | "/_matrix/federation/v1/publicRooms", |
| 253 | "/_matrix/federation/v1/query/", |
| 254 | "/_matrix/federation/v1/make_join/", |
| 255 | "/_matrix/federation/v1/make_leave/", |
| 256 | "/_matrix/federation/v1/send_join/", |
| 257 | "/_matrix/federation/v2/send_join/", |
| 258 | "/_matrix/federation/v1/send_leave/", |
| 259 | "/_matrix/federation/v2/send_leave/", |
| 260 | "/_matrix/federation/v1/invite/", |
| 261 | "/_matrix/federation/v2/invite/", |
| 262 | "/_matrix/federation/v1/query_auth/", |
| 263 | "/_matrix/federation/v1/event_auth/", |
| 264 | "/_matrix/federation/v1/exchange_third_party_invite/", |
| 265 | "/_matrix/federation/v1/user/devices/", |
| 266 | "/_matrix/federation/v1/get_groups_publicised", |
| 267 | "/_matrix/key/v2/query", |
| 268 | "/_matrix/federation/v1/send/", |
| 269 | ], |
| 270 | }, |
| 271 | |
| 272 | # Synapse media worker. This handles access to uploads and media stored in app.dataVolume |
| 273 | mediaWorker: { |
| 274 | deployment: app.SynapseWorker("synapse-media", "synapse.app.media_repository", kube.StatefulSet) { |
| 275 | cfg+: { |
| 276 | mountData: true, |
| 277 | localConfig+: { |
| 278 | worker_listeners: [{ |
| 279 | type: "http", |
| 280 | port: 8008, |
| 281 | x_forwarded: true, |
| 282 | bind_addresses: ["::"], |
| 283 | resources: [{ names: ["media"]}], |
| 284 | }], |
| 285 | }, |
| 286 | }, |
| 287 | }, |
| 288 | svc: app.ns.Contain(kube.Service("synapse-media")) { |
| 289 | target_pod:: app.mediaWorker.deployment.spec.template, |
| 290 | }, |
| 291 | }, |
| 292 | } |