Gitiles
Code Review Sign In
gerrit.hackerspace.pl / hscloud / 96074a78599be980f967f3b07e16dc5b9b56723d / . / cluster / kube / k0.libsonnet
blob: 06671b4734da6861484c911af227e2d4a59ed286 [file] [log] [blame]
Sergiusz Bazanskidbfa9882020-06-06 01:21:45 +0200[diff] [blame]1// k0.hswaw.net kubernetes cluster
2// This defines the cluster as a single object.
3// Use the sibling k0*.jsonnet 'view' files to actually apply the configuration.
4
5local kube = import "../../kube/kube.libsonnet";
6local policies = import "../../kube/policies.libsonnet";
7
8local cluster = import "cluster.libsonnet";
9
10local cockroachdb = import "lib/cockroachdb.libsonnet";
11local registry = import "lib/registry.libsonnet";
12local rook = import "lib/rook.libsonnet";
13
14{
15 k0: {
16 local k0 = self,
17 cluster: cluster.Cluster("k0", "hswaw.net") {
18 cfg+: {
19 storageClassNameParanoid: k0.ceph.waw2Pools.blockParanoid.name,
20 },
21 metallb+: {
22 cfg+: {
23 peers: [
24 {
25 "peer-address": "185.236.240.33",
26 "peer-asn": 65001,
27 "my-asn": 65002,
28 },
29 ],
30 addressPools: [
31 {
32 name: "public-v4-1",
33 protocol: "bgp",
34 addresses: [
35 "185.236.240.48/28",
36 ],
37 },
38 {
39 name: "public-v4-2",
40 protocol: "bgp",
41 addresses: [
42 "185.236.240.112/28"
43 ],
44 },
45 ],
46 },
47 },
48 },
49
50 // Docker registry
51 registry: registry.Environment {
52 cfg+: {
53 domain: "registry.%s" % [k0.cluster.fqdn],
54 storageClassName: k0.cluster.cfg.storageClassNameParanoid,
55 objectStorageName: "waw-hdd-redundant-2-object",
56 },
57 },
58
59 // CockroachDB, running on bc01n{01,02,03}.
60 cockroach: {
61 waw2: cockroachdb.Cluster("crdb-waw1") {
62 cfg+: {
63 topology: [
64 { name: "bc01n01", node: "bc01n01.hswaw.net" },
65 { name: "bc01n02", node: "bc01n02.hswaw.net" },
66 { name: "bc01n03", node: "bc01n03.hswaw.net" },
67 ],
68 // Host path on SSD.
69 hostPath: "/var/db/crdb-waw1",
70 },
71 },
72 clients: {
73 cccampix: k0.cockroach.waw2.Client("cccampix"),
74 cccampixDev: k0.cockroach.waw2.Client("cccampix-dev"),
75 buglessDev: k0.cockroach.waw2.Client("bugless-dev"),
76 sso: k0.cockroach.waw2.Client("sso"),
77 },
78 },
79
80 ceph: {
81 // waw1 cluster - dead as of 2019/08/06, data corruption
82 // waw2 cluster: shitty 7200RPM 2.5" HDDs
83 waw2: rook.Cluster(k0.cluster.rook, "ceph-waw2") {
84 spec: {
85 mon: {
86 count: 3,
87 allowMultiplePerNode: false,
88 },
89 storage: {
90 useAllNodes: false,
91 useAllDevices: false,
92 config: {
93 databaseSizeMB: "1024",
94 journalSizeMB: "1024",
95 },
96 nodes: [
97 {
98 name: "bc01n01.hswaw.net",
99 location: "rack=dcr01 chassis=bc01 host=bc01n01",
100 devices: [ { name: "sda" } ],
101 },
102 {
103 name: "bc01n02.hswaw.net",
104 location: "rack=dcr01 chassis=bc01 host=bc01n02",
105 devices: [ { name: "sda" } ],
106 },
107 {
108 name: "bc01n03.hswaw.net",
109 location: "rack=dcr01 chassis=bc01 host=bc01n03",
110 devices: [ { name: "sda" } ],
111 },
112 ],
113 },
114 benji:: {
115 metadataStorageClass: "waw-hdd-paranoid-2",
116 encryptionPassword: std.split((importstr "../secrets/plain/k0-benji-encryption-password"), '\n')[0],
117 pools: [
118 "waw-hdd-redundant-2",
119 "waw-hdd-redundant-2-metadata",
120 "waw-hdd-paranoid-2",
121 "waw-hdd-yolo-2",
122 ],
123 s3Configuration: {
124 awsAccessKeyId: "RPYZIROFXNLQVU2WJ4R3",
125 awsSecretAccessKey: std.split((importstr "../secrets/plain/k0-benji-secret-access-key"), '\n')[0],
126 bucketName: "benji-k0-backups",
127 endpointUrl: "https://s3.eu-central-1.wasabisys.com/",
128 },
129 }
130 },
131 },
132 waw2Pools: {
133 // redundant block storage
134 blockRedundant: rook.ECBlockPool(k0.ceph.waw2, "waw-hdd-redundant-2") {
135 spec: {
136 failureDomain: "host",
137 erasureCoded: {
138 dataChunks: 2,
139 codingChunks: 1,
140 },
141 },
142 },
143 // paranoid block storage (3 replicas)
144 blockParanoid: rook.ReplicatedBlockPool(k0.ceph.waw2, "waw-hdd-paranoid-2") {
145 spec: {
146 failureDomain: "host",
147 replicated: {
148 size: 3,
149 },
150 },
151 },
152 // yolo block storage (no replicas!)
153 blockYolo: rook.ReplicatedBlockPool(k0.ceph.waw2, "waw-hdd-yolo-2") {
154 spec: {
155 failureDomain: "host",
156 replicated: {
157 size: 1,
158 },
159 },
160 },
161 objectRedundant: rook.S3ObjectStore(k0.ceph.waw2, "waw-hdd-redundant-2-object") {
162 spec: {
163 metadataPool: {
164 failureDomain: "host",
165 replicated: { size: 3 },
166 },
167 dataPool: {
168 failureDomain: "host",
169 erasureCoded: {
170 dataChunks: 2,
171 codingChunks: 1,
172 },
173 },
174 },
175 },
176 },
177
178 // waw3: 6TB SAS 3.5" HDDs
179 waw3: rook.Cluster(k0.cluster.rook, "ceph-waw3") {
180 spec: {
181 mon: {
182 count: 3,
183 allowMultiplePerNode: false,
184 },
185 storage: {
186 useAllNodes: false,
187 useAllDevices: false,
188 config: {
189 databaseSizeMB: "1024",
190 journalSizeMB: "1024",
191 },
192 nodes: [
193 {
194 name: "dcr01s22.hswaw.net",
195 location: "rack=dcr01 host=dcr01s22",
196 devices: [
197 // https://github.com/rook/rook/issues/1228
198 //{ name: "disk/by-id/wwan-0x" + wwan }
199 //for wwan in [
200 // "5000c5008508c433",
201 // "5000c500850989cf",
202 // "5000c5008508f843",
203 // "5000c5008508baf7",
204 //]
205 { name: "sdn" },
206 { name: "sda" },
207 { name: "sdb" },
208 { name: "sdc" },
209 ],
210 },
211 {
212 name: "dcr01s24.hswaw.net",
213 location: "rack=dcr01 host=dcr01s22",
214 devices: [
215 // https://github.com/rook/rook/issues/1228
216 //{ name: "disk/by-id/wwan-0x" + wwan }
217 //for wwan in [
218 // "5000c5008508ee03",
219 // "5000c5008508c9ef",
220 // "5000c5008508df33",
221 // "5000c5008508dd3b",
222 //]
223 { name: "sdm" },
224 { name: "sda" },
225 { name: "sdb" },
226 { name: "sdc" },
227 ],
228 },
229 ],
230 },
231 benji:: {
232 metadataStorageClass: "waw-hdd-redundant-3",
233 encryptionPassword: std.split((importstr "../secrets/plain/k0-benji-encryption-password"), '\n')[0],
234 pools: [
235 "waw-hdd-redundant-3",
236 "waw-hdd-redundant-3-metadata",
237 "waw-hdd-yolo-3",
238 ],
239 s3Configuration: {
240 awsAccessKeyId: "RPYZIROFXNLQVU2WJ4R3",
241 awsSecretAccessKey: std.split((importstr "../secrets/plain/k0-benji-secret-access-key"), '\n')[0],
242 bucketName: "benji-k0-backups-waw3",
243 endpointUrl: "https://s3.eu-central-1.wasabisys.com/",
244 },
245 }
246 },
247 },
248 waw3Pools: {
249 // redundant block storage
250 blockRedundant: rook.ECBlockPool(k0.ceph.waw3, "waw-hdd-redundant-3") {
251 metadataReplicas: 2,
252 spec: {
253 failureDomain: "host",
254 replicated: {
255 size: 2,
256 },
257 },
258 },
259 // yolo block storage (low usage, no host redundancy)
260 blockYolo: rook.ReplicatedBlockPool(k0.ceph.waw3, "waw-hdd-yolo-3") {
261 spec: {
262 failureDomain: "osd",
263 erasureCoded: {
264 dataChunks: 12,
265 codingChunks: 4,
266 },
267 },
268 },
269 objectRedundant: rook.S3ObjectStore(k0.ceph.waw3, "waw-hdd-redundant-3-object") {
270 spec: {
271 metadataPool: {
272 failureDomain: "host",
273 replicated: { size: 2 },
274 },
275 dataPool: {
276 failureDomain: "host",
277 replicated: { size: 2 },
278 },
279 },
280 },
281 },
282
283 // Clients for S3/radosgw storage.
284 clients: {
285 # Used for owncloud.hackerspace.pl, which for now lives on boston-packets.hackerspace.pl.
286 nextcloudWaw3: kube.CephObjectStoreUser("nextcloud") {
287 metadata+: {
288 namespace: "ceph-waw3",
289 },
290 spec: {
291 store: "waw-hdd-redundant-3-object",
292 displayName: "nextcloud",
293 },
294 },
295
296 # nuke@hackerspace.pl's personal storage.
297 nukePersonalWaw3: kube.CephObjectStoreUser("nuke-personal") {
298 metadata+: {
299 namespace: "ceph-waw3",
300 },
301 spec: {
302 store: "waw-hdd-redundant-3-object",
303 displayName: "nuke-personal",
304 },
305 },
306
307 # patryk@hackerspace.pl's ArmA3 mod bucket.
308 cz2ArmaModsWaw3: kube.CephObjectStoreUser("cz2-arma3mods") {
309 metadata+: {
310 namespace: "ceph-waw3",
311 },
312 spec: {
313 store: "waw-hdd-redundant-3-object",
314 displayName: "cz2-arma3mods",
315 },
316 },
Bartosz Stebeld9df5872020-06-13 21:19:40 +0200[diff] [blame]317 # Buckets for spark pipelines
318 # TODO(implr): consider a second yolo-backed one for temp data
319 implrSparkWaw3: kube.CephObjectStoreUser("implr-spark") {
320 metadata+: {
321 namespace: "ceph-waw3",
322 },
323 spec: {
324 store: "waw-hdd-redundant-3-object",
325 displayName: "implr-spark",
326 },
327 },
Sergiusz Bazanskib1aadd82020-06-24 19:06:17 +0200[diff] [blame]328 # q3k's personal user
329 q3kWaw3: kube.CephObjectStoreUser("q3k") {
330 metadata+: {
331 namespace: "ceph-waw3",
332 },
333 spec: {
334 store: "waw-hdd-redundant-3-object",
335 displayName: "q3k",
336 },
337 },
Sergiusz Bazanskidbfa9882020-06-06 01:21:45 +0200[diff] [blame]338 },
339 },
340
341
342 # These are policies allowing for Insecure pods in some namespaces.
343 # A lot of them are spurious and come from the fact that we deployed
344 # these namespaces before we deployed the draconian PodSecurityPolicy
345 # we have now. This should be fixed by setting up some more granular
346 # policies, or fixing the workloads to not need some of the permission
347 # bits they use, whatever those might be.
348 # TODO(q3k): fix this?
349 unnecessarilyInsecureNamespaces: [
350 policies.AllowNamespaceInsecure("ceph-waw2"),
351 policies.AllowNamespaceInsecure("ceph-waw3"),
352 policies.AllowNamespaceInsecure("matrix"),
353 policies.AllowNamespaceInsecure("registry"),
354 policies.AllowNamespaceInsecure("internet"),
355 # TODO(implr): restricted policy with CAP_NET_ADMIN and tuntap, but no full root
356 policies.AllowNamespaceInsecure("implr-vpn"),
357 ],
358 },
359}