kube/postgres: pgupgrade automation
Change-Id: Ibcbddf57b8cdcac75ce366a95db63817bec42a22
Reviewed-on: https://gerrit.hackerspace.pl/c/hscloud/+/1698
Reviewed-by: q3k <q3k@hackerspace.pl>
diff --git a/kube/postgres.libsonnet b/kube/postgres.libsonnet
index edb1cae..2935213 100644
--- a/kube/postgres.libsonnet
+++ b/kube/postgres.libsonnet
@@ -29,6 +29,30 @@
# string values.
# Example: { max_connections: "300" }
opts: {},
+
+ # Postgres cluster upgrade automation. In order to update running
+ # postgres version:
+ # * set image to target postgres version
+ # * pgupgrade.from to previous/current major version
+ # * pgupgrade.to to target major version number (optional, this should
+ # be figured out from image version)
+ # * switch pgupgrade.enable to true.
+ #
+ # While we do have some countermeasures to prevent stupid typos, you
+ # should still probably make a database backup, eg. using:
+ # kubectl exec deploy/postgres -- pg_dumpall > dump.sql
+ #
+ # After succesful upgrade /var/lib/postgresql/data/pgdata-old directory
+ # needs to be removed by hand. In case a rollback is needed pgdata needs
+ # to be swapped with the pgdata-old directory and the postgres image
+ # needs to be adjusted accordingly.
+ pgupgrade: {
+ enable: false,
+ from: "10",
+ # Extract target version from image name, supported:
+ # postgres:1.2-suffix, postgres:1-suffix, postgres:1.2, postgres:1
+ to: std.native('regexSubst')("^[^:]+:([^.]+).*$", cfg.image, "${1}"),
+ },
},
makeName(suffix):: cfg.prefix + suffix,
@@ -88,6 +112,62 @@
},
},
},
+
+ initContainers_: if cfg.pgupgrade.enable then {
+ pgupgrade: kube.Container(postgres.makeName("pgupgrade")) {
+ image: "tianon/postgres-upgrade:%s-to-%s" % [cfg.pgupgrade.from, cfg.pgupgrade.to],
+ command: [
+ "bash", "-c", |||
+ set -e -x -o pipefail
+
+ CURRENT_VERSION="$(cat "$PGDATA/PG_VERSION")"
+ if [[ "$CURRENT_VERSION" == "$VERSION_TO" ]]; then
+ echo "Already running target version ($VERSION_TO)"
+ exit 0
+ fi
+
+ if [[ "$CURRENT_VERSION" != "$VERSION_FROM" ]]; then
+ echo "Running unexpected source version, wanted $VERSION_FROM, got $CURRENT_VERSION"
+ exit 1
+ fi
+
+ rm -rf $PGDATANEXT || true
+
+ if [ ! -s "$PGDATANEXT/PG_VERSION" ]; then
+ echo "Initializing new database..."
+ PGDATA="$PGDATANEXT" eval "initdb $POSTGRES_INITDB_ARGS"
+ fi
+
+ chmod 700 $PGDATA $PGDATANEXT
+
+ echo "Running upgrade..."
+ pg_upgrade --link --old-datadir $PGDATA --new-datadir $PGDATANEXT || (sleep 3600 ; exit 1)
+
+ echo "Copying pg_hba.conf"
+ cp $PGDATA/pg_hba.conf $PGDATANEXT/pg_hba.conf
+
+ echo "Done, swapping..."
+ mv $PGDATA $PGDATAOLD
+ mv $PGDATANEXT $PGDATA
+ |||
+ ],
+ env_: postgres.deployment.spec.template.spec.containers_.postgres.env_ + {
+ VERSION_TO: cfg.pgupgrade.to,
+ VERSION_FROM: cfg.pgupgrade.from,
+
+ # pg_upgrade target directory, swapped with
+ # PGDATA after cluster data upgrade is finished
+ PGDATANEXT: "/var/lib/postgresql/data/pgdata-next",
+
+ # Directory used to stash previous pgdata
+ # version
+ PGDATAOLD: "/var/lib/postgresql/data/pgdata-old",
+ },
+ volumeMounts_: {
+ data: { mountPath: "/var/lib/postgresql/data" },
+ },
+ },
+ } else {},
securityContext: {
runAsUser: 999,
},