app/matrix: matrix-media-repo RGW-based media storage
Change-Id: I459bd78eee52fd349a16f31a48346d3258ef50a4
Reviewed-on: https://gerrit.hackerspace.pl/c/hscloud/+/1081
Reviewed-by: q3k <q3k@hackerspace.pl>
diff --git a/app/matrix/lib/matrix-ng.libsonnet b/app/matrix/lib/matrix-ng.libsonnet
index 941c46a..3455b11 100644
--- a/app/matrix/lib/matrix-ng.libsonnet
+++ b/app/matrix/lib/matrix-ng.libsonnet
@@ -28,6 +28,51 @@
# * https://{homeserver}/_synapse/oidc/callback is added to allowed callback URLs list
# * openid scope is enabled for configured client
#
+# In order to deploy matrix-media-repo as a replacement for synapse built-in
+# media workers the following steps need to be carried out:
+#
+# 1. Generate password and bootstrap extra postgres user
+# pwgen 32 1 > secrets/plain/media-repo-$ns-postgres
+# echo "create database mediarepo; create user mediarepo with password '$(cat secrets/plain/media-repo-$ns-postgres)'; grant all privileges on database mediarepo to mediarepo;" | kubectl -n $ns exec -it deploy/waw3-postgres psql
+# secretstore sync secrets
+#
+# 2. Fetch Ceph RGW credentials
+# kubectl get secrets -n ceph-waw3 rook-ceph-object-user-waw-hdd-redundant-3-object-$ns -o json | jq '.data|map_values(@base64d)' > secrets/plain/media-repo-$ns-ceph.json
+# secretstore sync secrets
+#
+# 3. Create an apropriate bucket using s3cmd
+# s3cmd --access_key="$(jq -r '.AccessKey' secrets/plain/media-repo-$ns-ceph.json)" --secret_key="$(jq -r '.SecretKey' secrets/plain/media-repo-$ns-ceph.json)" --host=object.ceph-waw3.hswaw.net --host-bucket=object.ceph-waw3.hswaw.net mb s3://media-repo-$ns
+#
+# 4. Add relevant configuration overrides in cfg.mediaRepo key for your
+# deployment configuration file:
+#
+# mediaRepo+: {
+# enable: true,
+# route: false,
+# s3+: {
+# endpoint: std.strReplace((import "secrets/plain/media-repo-$ns-ceph.json").Endpoint, "http://", ""),
+# accessKey: (import "secrets/plain/media-repo-$ns-ceph.json").AccessKey,
+# secretKey: (import "secrets/plain/media-repo-$ns-ceph.json").SecretKey,
+# bucketName: "media-repo-$ns",
+# region: "eu",
+# },
+# db+: {
+# password: std.strReplace(importstr "secrets/plain/media-repo-$ns-postgres", "\n", ""),
+# },
+# },
+#
+# 5. Additionally, when migrating from already deployed synapse media worker the
+# following command needs to be run in order to import existing media files:
+# kubectl -n $ns exec deploy/media-repo -- import_synapse -baseUrl http://synapse-media:8008 -dbHost waw3-postgres -dbPassword "$(kubectl -n $ns get secret synapse -o json | jq -r '.data.postgres_password | @base64d')" -config /config/config.yaml -serverName 'SERVER_NAME'
+#
+# 6. After migrating data over from native synapse media worker storage traffic
+# can be rerouted to matrix-media-repo by switching cfg.mediaRepo.route flag
+# to true
+#
+# 7. Run import step #5 again to make sure no media were left missing in old
+# media worker deployment - import operation is indempotent and can be ran
+# against a synapse media worker that's not handling user traffic anymore.
+#
# Sequencing appservices is fun. The appservice needs to run first (for
# instance, via a bootstrap job), and on startup it will spit out a
# registration file. This registration file then needs to be fed to synapse -
@@ -49,6 +94,7 @@
local cas = import "./cas.libsonnet";
local wellKnown = import "./wellknown.libsonnet";
local synapse = import "./synapse.libsonnet";
+local mediaRepo = import "./media-repo.libsonnet";
{
local app = self,
@@ -68,6 +114,7 @@
appserviceIRC: "matrixdotorg/matrix-appservice-irc:release-v0.29.0",
appserviceTelegram: "dock.mau.dev/tulir/mautrix-telegram@sha256:c6e25cb57e1b67027069e8dc2627338df35d156315c004a6f2b34b6aeaa79f77",
wellKnown: "registry.k0.hswaw.net/q3k/wellknown:1611960794-adbf560851a46ad0e58b42f0daad7ef19535687c",
+ mediaRepo: "turt2live/matrix-media-repo:v1.2.8",
},
# OpenID Connect provider configuration.
@@ -119,6 +166,33 @@
# Serve /.well-known/matrix configuration endpoints required when using
# cfg.webDomain directly as mxid.
wellKnown: false,
+
+ # matrix-media-repo S3-based media storage container
+ mediaRepo: {
+ enable: false,
+
+ # Route /_matrix/media/ endpoints to matrix-media-repo. Set this
+ # to true after migrating media files to matrix-media-repo.
+ route: false,
+
+ s3: {
+ endpoint: error "mediaRepo.s3.endpoint needs to be set",
+ accessKey: error "mediaRepo.s3.accessKey needs to be set",
+ secretKey: error "mediaRepo.s3.secretKey needs to be set",
+ bucketName: error "mediaRepo.s3.bucketName needs to be set",
+ region: error "mediaRepo.s3.region needs to be set",
+ },
+
+ db: {
+ username: "mediarepo",
+ password: error "mediaRepo.db.password needs to be set",
+ database: "mediarepo",
+ host: "waw3-postgres",
+ },
+ },
+
+ # List of administrative users MXIDs (used in matrix-media-repo only)
+ admins: [],
},
# DEPRECATED: this needs to be removed in favor of namespace.Contain() in
@@ -184,6 +258,21 @@
},
} else {},
+ mediaRepo: if cfg.mediaRepo.enable then mediaRepo {
+ ns: app.namespace,
+ cfg+: {
+ image: cfg.images.mediaRepo,
+
+ homeservers: [
+ {name: cfg.serverName, csApi: "https://" + cfg.webDomain}
+ ],
+ admins: cfg.admins,
+
+ s3: cfg.mediaRepo.s3,
+ db: cfg.mediaRepo.db,
+ },
+ } else {},
+
synapse: synapse {
ns: app.namespace,
postgres: app.postgres3,
@@ -231,7 +320,7 @@
for path in app.synapse.genericWorker.paths
] + [
{ path: "/", backend: app.riot.svc.name_port },
- { path: "/_matrix/media/", backend: app.synapse.mediaWorker.svc.name_port },
+ { path: "/_matrix/media/", backend: if cfg.mediaRepo.route then app.mediaRepo.svc.name_port else app.synapse.mediaWorker.svc.name_port },
{ path: "/_matrix/", backend: app.synapse.main.svc.name_port },
# Used by OpenID Connect login flow