ceph-waw1 -> ceph-waw2
Change-Id: I03d6244b9697a9efc06492114ef90cdb01e17601
diff --git a/app/matrix/prod.jsonnet b/app/matrix/prod.jsonnet
index 62ed291..7185c59 100644
--- a/app/matrix/prod.jsonnet
+++ b/app/matrix/prod.jsonnet
@@ -17,7 +17,7 @@
namespace: "matrix",
domain: "matrix.hackerspace.pl",
serverName: "hackerspace.pl",
- storageClassName: "waw-hdd-redundant-1",
+ storageClassName: "waw-hdd-paranoid-2",
synapseImage: "matrixdotorg/synapse:v0.99.4",
riotImage: "bubuntux/riot-web:v1.1.2",
@@ -267,7 +267,7 @@
image: "registry.k0.hswaw.net/informatic/matrix-appservice-irc:0.11.2",
metadata: {},
config: std.native("parseYaml")(importstr "appservice-irc.yaml")[0],
- storageClassName: "waw-hdd-redundant-1",
+ storageClassName: "waw-hdd-paranoid-2",
},
config: kube.ConfigMap("appservice-irc-%s" % [name]) {
diff --git a/cluster/kube/cluster.jsonnet b/cluster/kube/cluster.jsonnet
index d0b77bd..0156672 100644
--- a/cluster/kube/cluster.jsonnet
+++ b/cluster/kube/cluster.jsonnet
@@ -93,7 +93,7 @@
spec+: {
// TODO(q3k): Bring up the operator again when stability gets fixed
// See: https://github.com/rook/rook/issues/3059#issuecomment-492378873
- replicas: 0,
+ replicas: 1,
},
},
},
@@ -102,7 +102,8 @@
registry: registry.Environment {
cfg+: {
domain: "registry.%s" % [fqdn],
- storageClassName: cfg.storageClassNameRedundant,
+ storageClassName: cfg.storageClassNameParanoid,
+ objectStorageName: "waw-hdd-redundant-2-object",
},
},
};
@@ -113,24 +114,25 @@
local k0 = self,
cluster: Cluster("k0.hswaw.net") {
cfg+: {
- storageClassNameRedundant: k0.ceph.blockRedundant.name,
+ storageClassNameParanoid: k0.ceph.blockParanoid.name,
},
},
cockroach: {
- waw1: cockroachdb.Cluster("crdb-waw1") {
+ waw2: cockroachdb.Cluster("crdb-waw2") {
cfg+: {
topology: [
{ name: "bc01n01", node: "bc01n01.hswaw.net" },
{ name: "bc01n02", node: "bc01n02.hswaw.net" },
{ name: "bc01n03", node: "bc01n03.hswaw.net" },
],
- hostPath: "/var/db/crdb-waw1",
+ hostPath: "/var/db/crdb-waw2",
},
},
},
ceph: {
- // waw1 cluster
- waw1: rook.Cluster(k0.cluster.rook, "ceph-waw1") {
+ // waw1 cluster - dead as of 2019/08/06, data corruption
+ // waw2 cluster
+ waw2: rook.Cluster(k0.cluster.rook, "ceph-waw2") {
spec: {
mon: {
count: 3,
@@ -164,7 +166,7 @@
},
},
// redundant block storage
- blockRedundant: rook.ECBlockPool(k0.ceph.waw1, "waw-hdd-redundant-1") {
+ blockRedundant: rook.ECBlockPool(k0.ceph.waw2, "waw-hdd-redundant-2") {
spec: {
failureDomain: "host",
erasureCoded: {
@@ -173,8 +175,17 @@
},
},
},
+ // paranoid block storage (3 replicas)
+ blockParanoid: rook.ReplicatedBlockPool(k0.ceph.waw2, "waw-hdd-paranoid-2") {
+ spec: {
+ failureDomain: "host",
+ replicated: {
+ size: 3,
+ },
+ },
+ },
// yolo block storage (no replicas!)
- blockYolo: rook.ReplicatedBlockPool(k0.ceph.waw1, "waw-hdd-yolo-1") {
+ blockYolo: rook.ReplicatedBlockPool(k0.ceph.waw2, "waw-hdd-yolo-2") {
spec: {
failureDomain: "host",
replicated: {
@@ -182,7 +193,7 @@
},
},
},
- objectRedundant: rook.S3ObjectStore(k0.ceph.waw1, "waw-hdd-redundant-1-object") {
+ objectRedundant: rook.S3ObjectStore(k0.ceph.waw2, "waw-hdd-redundant-2-object") {
spec: {
metadataPool: {
failureDomain: "host",
diff --git a/cluster/kube/lib/registry.libsonnet b/cluster/kube/lib/registry.libsonnet
index 8b57dd7..1ce022d 100644
--- a/cluster/kube/lib/registry.libsonnet
+++ b/cluster/kube/lib/registry.libsonnet
@@ -15,6 +15,7 @@
namespace: "registry",
domain: error "domain must be set",
storageClassName: error "storageClassName must be set",
+ objectStoreName: error "objectStoreName must be set",
},
metadata(component):: {
@@ -72,9 +73,9 @@
blobdescriptor: "inmemory",
},
s3: {
- regionendpoint: "https://object.ceph-waw1.hswaw.net",
+ regionendpoint: "https://object.ceph-waw2.hswaw.net",
bucket: "registry",
- region: "waw-hdd-redunant-1-object:default-placement",
+ region: "waw-hdd-redunant-2-object:default-placement",
},
},
http: {
@@ -154,6 +155,8 @@
{ who: ["q3k", "inf"], what: "vms/*" },
{ who: ["q3k", "inf"], what: "app/*" },
{ who: ["q3k", "inf"], what: "go/svc/*" },
+ { who: ["q3k"], what: "bgpwtf/*" },
+ { who: ["q3k"], what: "devtools/*" },
],
acl: [
{
@@ -253,11 +256,11 @@
},
env_: {
REGISTRY_STORAGE_S3_ACCESSKEY: { secretKeyRef: {
- name: "rook-ceph-object-user-waw-hdd-redundant-1-object-registry",
+ name: "rook-ceph-object-user-%(objectStorageName)s-registry" % {objectStorageName: cfg.objectStorageName},
key: "AccessKey"
}},
REGISTRY_STORAGE_S3_SECRETKEY: { secretKeyRef: {
- name: "rook-ceph-object-user-waw-hdd-redundant-1-object-registry",
+ name: "rook-ceph-object-user-%(objectStorageName)s-registry" % {objectStorageName: cfg.objectStorageName},
key: "SecretKey",
}},
},
@@ -310,10 +313,10 @@
registryStorageUser: kube._Object("ceph.rook.io/v1", "CephObjectStoreUser", "registry") {
metadata+: {
- namespace: "ceph-waw1",
+ namespace: "ceph-waw2",
},
spec: {
- store: "waw-hdd-redundant-1-object",
+ store: cfg.objectStorageName,
displayName: "docker-registry user",
},
},
diff --git a/cluster/kube/lib/rook.libsonnet b/cluster/kube/lib/rook.libsonnet
index 512c6a0..063bd17 100644
--- a/cluster/kube/lib/rook.libsonnet
+++ b/cluster/kube/lib/rook.libsonnet
@@ -471,6 +471,8 @@
ReplicatedBlockPool(cluster, name):: {
local pool = self,
+ name:: name,
+
spec:: error "spec must be specified",
pool: kube._Object("ceph.rook.io/v1", "CephBlockPool", name) {
diff --git a/cluster/nix/cluster-configuration.nix b/cluster/nix/cluster-configuration.nix
index a3601bd..7357f14 100644
--- a/cluster/nix/cluster-configuration.nix
+++ b/cluster/nix/cluster-configuration.nix
@@ -138,7 +138,8 @@
advertiseAddress = "${node.ipAddr}";
etcd = {
- servers = (map (n: "https://${n.fqdn}:2379") nodes);
+ # https://github.com/kubernetes/kubernetes/issues/72102
+ servers = (map (n: "https://${n.fqdn}:2379") ( [ node ] ));
caFile = pki.etcd.kube.ca;
keyFile = pki.etcd.kube.key;
certFile = pki.etcd.kube.cert;
diff --git a/devtools/gerrit/kube/prod.jsonnet b/devtools/gerrit/kube/prod.jsonnet
index 565772f..68f68cc 100644
--- a/devtools/gerrit/kube/prod.jsonnet
+++ b/devtools/gerrit/kube/prod.jsonnet
@@ -11,7 +11,7 @@
domain: "gerrit.hackerspace.pl",
identity: "7b6244cf-e30b-42c5-ba91-c329ef4e6cf1",
- storageClassName: "waw-hdd-redundant-1",
+ storageClassName: "waw-hdd-paranoid-2",
secureSecret: "gerrit",
},
diff --git a/kube/postgres.libsonnet b/kube/postgres.libsonnet
index bc087cf..d82a89a 100644
--- a/kube/postgres.libsonnet
+++ b/kube/postgres.libsonnet
@@ -8,7 +8,7 @@
cfg:: {
namespace: error "namespace must be set",
appName: error "app name must be set",
- storageClassName: "waw-hdd-redundant-1",
+ storageClassName: "waw-hdd-paranoid-2",
prefix: "", # if set, should be 'foo-'
image: "postgres:10.4",