cluster: add ceph-waw3, move metallb to bgp
Change-Id: Iebf369f9a02e44be163ef4afc2e0f23c4b009898
diff --git a/cluster/kube/cluster.jsonnet b/cluster/kube/cluster.jsonnet
index e89a801..0706d31 100644
--- a/cluster/kube/cluster.jsonnet
+++ b/cluster/kube/cluster.jsonnet
@@ -138,8 +138,10 @@
allowInsecureNamespaces: [
policies.AllowNamespaceInsecure("kube-system"),
+ policies.AllowNamespaceInsecure("metallb-system"),
# TODO(q3k): fix this?
policies.AllowNamespaceInsecure("ceph-waw2"),
+ policies.AllowNamespaceInsecure("ceph-waw3"),
policies.AllowNamespaceInsecure("matrix"),
policies.AllowNamespaceInsecure("registry"),
policies.AllowNamespaceInsecure("internet"),
@@ -173,8 +175,28 @@
// Metal Load Balancer
metallb: metallb.Environment {
cfg+: {
+ peers: [
+ {
+ "peer-address": "185.236.240.33",
+ "peer-asn": 65001,
+ "my-asn": 65002,
+ },
+ ],
addressPools: [
- { name: "public-v4-1", protocol: "layer2", addresses: ["185.236.240.50-185.236.240.63"] },
+ {
+ name: "public-v4-1",
+ protocol: "bgp",
+ addresses: [
+ "185.236.240.48/28",
+ ],
+ },
+ {
+ name: "public-v4-2",
+ protocol: "bgp",
+ addresses: [
+ "185.236.240.112/28"
+ ],
+ },
],
},
},
@@ -231,7 +253,7 @@
local k0 = self,
cluster: Cluster("k0", "hswaw.net") {
cfg+: {
- storageClassNameParanoid: k0.ceph.blockParanoid.name,
+ storageClassNameParanoid: k0.ceph.waw2Pools.blockParanoid.name,
},
},
cockroach: {
@@ -302,41 +324,10 @@
}
},
},
- // redundant block storage
- blockRedundant: rook.ECBlockPool(k0.ceph.waw2, "waw-hdd-redundant-2") {
- spec: {
- failureDomain: "host",
- erasureCoded: {
- dataChunks: 2,
- codingChunks: 1,
- },
- },
- },
- // paranoid block storage (3 replicas)
- blockParanoid: rook.ReplicatedBlockPool(k0.ceph.waw2, "waw-hdd-paranoid-2") {
- spec: {
- failureDomain: "host",
- replicated: {
- size: 3,
- },
- },
- },
- // yolo block storage (no replicas!)
- blockYolo: rook.ReplicatedBlockPool(k0.ceph.waw2, "waw-hdd-yolo-2") {
- spec: {
- failureDomain: "host",
- replicated: {
- size: 1,
- },
- },
- },
- objectRedundant: rook.S3ObjectStore(k0.ceph.waw2, "waw-hdd-redundant-2-object") {
- spec: {
- metadataPool: {
- failureDomain: "host",
- replicated: { size: 3 },
- },
- dataPool: {
+ waw2Pools: {
+ // redundant block storage
+ blockRedundant: rook.ECBlockPool(k0.ceph.waw2, "waw-hdd-redundant-2") {
+ spec: {
failureDomain: "host",
erasureCoded: {
dataChunks: 2,
@@ -344,6 +335,139 @@
},
},
},
+ // paranoid block storage (3 replicas)
+ blockParanoid: rook.ReplicatedBlockPool(k0.ceph.waw2, "waw-hdd-paranoid-2") {
+ spec: {
+ failureDomain: "host",
+ replicated: {
+ size: 3,
+ },
+ },
+ },
+ // yolo block storage (no replicas!)
+ blockYolo: rook.ReplicatedBlockPool(k0.ceph.waw2, "waw-hdd-yolo-2") {
+ spec: {
+ failureDomain: "host",
+ replicated: {
+ size: 1,
+ },
+ },
+ },
+ objectRedundant: rook.S3ObjectStore(k0.ceph.waw2, "waw-hdd-redundant-2-object") {
+ spec: {
+ metadataPool: {
+ failureDomain: "host",
+ replicated: { size: 3 },
+ },
+ dataPool: {
+ failureDomain: "host",
+ erasureCoded: {
+ dataChunks: 2,
+ codingChunks: 1,
+ },
+ },
+ },
+ },
+ },
+ waw3: rook.Cluster(k0.cluster.rook, "ceph-waw3") {
+ spec: {
+ mon: {
+ count: 3,
+ allowMultiplePerNode: false,
+ },
+ storage: {
+ useAllNodes: false,
+ useAllDevices: false,
+ config: {
+ databaseSizeMB: "1024",
+ journalSizeMB: "1024",
+ },
+ nodes: [
+ {
+ name: "dcr01s22.hswaw.net",
+ location: "rack=dcr01 host=dcr01s22",
+ devices: [
+ // https://github.com/rook/rook/issues/1228
+ //{ name: "disk/by-id/wwan-0x" + wwan }
+ //for wwan in [
+ // "5000c5008508c433",
+ // "5000c500850989cf",
+ // "5000c5008508f843",
+ // "5000c5008508baf7",
+ //]
+ { name: "sdn" },
+ { name: "sda" },
+ { name: "sdb" },
+ { name: "sdc" },
+ ],
+ },
+ {
+ name: "dcr01s24.hswaw.net",
+ location: "rack=dcr01 host=dcr01s22",
+ devices: [
+ // https://github.com/rook/rook/issues/1228
+ //{ name: "disk/by-id/wwan-0x" + wwan }
+ //for wwan in [
+ // "5000c5008508ee03",
+ // "5000c5008508c9ef",
+ // "5000c5008508df33",
+ // "5000c5008508dd3b",
+ //]
+ { name: "sdm" },
+ { name: "sda" },
+ { name: "sdb" },
+ { name: "sdc" },
+ ],
+ },
+ ],
+ },
+ benji:: {
+ metadataStorageClass: "waw-hdd-paranoid-3",
+ encryptionPassword: std.split((importstr "../secrets/plain/k0-benji-encryption-password"), '\n')[0],
+ pools: [
+ ],
+ s3Configuration: {
+ awsAccessKeyId: "RPYZIROFXNLQVU2WJ4R3",
+ awsSecretAccessKey: std.split((importstr "../secrets/plain/k0-benji-secret-access-key"), '\n')[0],
+ bucketName: "benji-k0-backups-waw3",
+ endpointUrl: "https://s3.eu-central-1.wasabisys.com/",
+ },
+ }
+ },
+ },
+ waw3Pools: {
+ // redundant block storage
+ blockRedundant: rook.ECBlockPool(k0.ceph.waw3, "waw-hdd-redundant-3") {
+ metadataReplicas: 2,
+ spec: {
+ failureDomain: "host",
+ replicated: {
+ size: 2,
+ },
+ },
+ },
+ // yolo block storage (low usage, no host redundancy)
+ blockYolo: rook.ReplicatedBlockPool(k0.ceph.waw3, "waw-hdd-yolo-3") {
+ spec: {
+ failureDomain: "osd",
+ erasureCoded: {
+ dataChunks: 12,
+ codingChunks: 4,
+ },
+ },
+ },
+ objectRedundant: rook.S3ObjectStore(k0.ceph.waw3, "waw-hdd-redundant-3-object") {
+ spec: {
+ metadataPool: {
+ failureDomain: "host",
+ replicated: { size: 2 },
+ },
+ dataPool: {
+ failureDomain: "host",
+ replicated: { size: 2 },
+ },
+ },
+ },
},
},
diff --git a/cluster/kube/lib/metallb.libsonnet b/cluster/kube/lib/metallb.libsonnet
index 7f3d746..ad58964 100644
--- a/cluster/kube/lib/metallb.libsonnet
+++ b/cluster/kube/lib/metallb.libsonnet
@@ -25,9 +25,10 @@
cfg:: {
namespace: "metallb-system",
namespaceCreate: true,
- version:: "v0.7.3",
+ version:: "v0.8.3",
imageController: "metallb/controller:" + cfg.version,
- imageSpeaker: "metallb/speaker:" + cfg.version,
+ //imageSpeaker: "metallb/speaker:" + cfg.version,
+ imageSpeaker: "derq3k/metallb-speaker:20191101-180123",
addressPools: error "addressPools must be set in config",
},
@@ -76,6 +77,11 @@
resources: ["services", "endpoints", "nodes"],
verbs: ["get", "list", "watch"],
},
+ {
+ apiGroups: [""],
+ resources: ["events"],
+ verbs: ["create", "patch"],
+ },
],
},
@@ -165,6 +171,7 @@
args: [ "--port=7472", "--config=config" ],
env_: {
METALLB_NODE_NAME: kube.FieldRef("spec.nodeName"),
+ METALLB_HOST: kube.FieldRef("status.hostIP"),
},
ports: [
{ name: "monitoring", containerPort: 7472 },
@@ -174,7 +181,7 @@
},
securityContext: {
allowPrivilegeEscalation: false,
- capabilities: { drop: [ "all" ], add: [ "net_raw" ] },
+ capabilities: { drop: [ "all" ], add: [ "NET_ADMIN", "NET_RAW", "SYS_ADMIN" ] },
readOnlyRootFilesystem: true,
},
},
@@ -192,6 +199,7 @@
data: {
config: std.manifestYamlDoc({
"address-pools": cfg.addressPools,
+ "peers": cfg.peers,
}),
},
},
diff --git a/cluster/kube/lib/rook.libsonnet b/cluster/kube/lib/rook.libsonnet
index 21a0f56..e594e3d 100644
--- a/cluster/kube/lib/rook.libsonnet
+++ b/cluster/kube/lib/rook.libsonnet
@@ -381,7 +381,7 @@
image: "ceph/ceph:v14.2.2-20190830",
allowUnsupported: true,
},
- dataDirHostPath: "/var/lib/rook",
+ dataDirHostPath: if name == "ceph-waw2" then "/var/lib/rook" else "/var/lib/rook-%s" % [name],
dashboard: {
ssl: false,
enabled: true,
@@ -709,6 +709,7 @@
ECBlockPool(cluster, name):: {
local pool = self,
name:: name,
+ metadataReplicas:: 3,
spec:: error "spec must be specified",
@@ -721,7 +722,7 @@
spec: {
failureDomain: "host",
replicated: {
- size: 3,
+ size: pool.metadataReplicas,
},
},
},