blob: f1d49707ca1f13c51d0ae86c503df45ae7b9131a [file] [log] [blame]
// k0.hswaw.net kubernetes cluster
// This defines the cluster as a single object.
// Use the sibling k0*.jsonnet 'view' files to actually apply the configuration.
local kube = import "../../kube/kube.libsonnet";
local policies = import "../../kube/policies.libsonnet";
local cluster = import "cluster.libsonnet";
local admitomatic = import "lib/admitomatic.libsonnet";
local cockroachdb = import "lib/cockroachdb.libsonnet";
local registry = import "lib/registry.libsonnet";
local rook = import "lib/rook.libsonnet";
{
k0: {
local k0 = self,
cluster: cluster.Cluster("k0", "hswaw.net") {
cfg+: {
storageClassNameParanoid: k0.ceph.waw3Pools.blockRedundant.name,
},
metallb+: {
cfg+: {
// Peer with calico running on same node.
peers: [
{
"peer-address": "127.0.0.1",
"peer-asn": 65003,
"my-asn": 65002,
},
],
// Public IP address pools. Keep in sync with k0.calico.yaml.
addressPools: [
{
name: "public-v4-1",
protocol: "bgp",
addresses: [
"185.236.240.48/28",
],
},
{
name: "public-v4-2",
protocol: "bgp",
addresses: [
"185.236.240.112/28"
],
},
],
},
},
},
// Docker registry
registry: registry.Environment {
cfg+: {
domain: "registry.%s" % [k0.cluster.fqdn],
storageClassName: k0.cluster.cfg.storageClassNameParanoid,
objectStorageName: "waw-hdd-redundant-3-object",
},
},
// CockroachDB, running on bc01n{01,02,03}.
cockroach: {
waw2: cockroachdb.Cluster("crdb-waw1") {
cfg+: {
topology: [
{ name: "bc01n01", node: "bc01n01.hswaw.net" },
{ name: "bc01n02", node: "bc01n02.hswaw.net" },
{ name: "dcr01s22", node: "dcr01s22.hswaw.net" },
],
// Host path on SSD.
hostPath: "/var/db/crdb-waw1",
extraDNS: [
"crdb-waw1.hswaw.net",
],
},
},
clients: {
cccampix: k0.cockroach.waw2.Client("cccampix"),
cccampixDev: k0.cockroach.waw2.Client("cccampix-dev"),
buglessDev: k0.cockroach.waw2.Client("bugless-dev"),
sso: k0.cockroach.waw2.Client("sso"),
herpDev: k0.cockroach.waw2.Client("herp-dev"),
gitea: k0.cockroach.waw2.Client("gitea"),
issues: k0.cockroach.waw2.Client("issues"),
dns: k0.cockroach.waw2.Client("dns"),
},
},
ceph: {
// waw1 cluster - dead as of 2019/08/06, data corruption
// waw2 cluster - dead as of 2021/01/22, torn down (horrible M610 RAID controllers are horrible)
// waw3: 6TB SAS 3.5" HDDs
waw3: rook.Cluster(k0.cluster.rook, "ceph-waw3") {
spec: {
mon: {
count: 1,
allowMultiplePerNode: false,
},
resources: {
osd: {
requests: {
cpu: "2",
memory: "6G",
},
limits: {
cpu: "2",
memory: "8G",
},
},
},
storage: {
useAllNodes: false,
useAllDevices: false,
config: {
databaseSizeMB: "1024",
journalSizeMB: "1024",
},
nodes: [
{
name: "dcr01s22.hswaw.net",
location: "rack=dcr01 host=dcr01s22",
devices: [
// https://github.com/rook/rook/issues/1228
//{ name: "disk/by-id/wwan-0x" + wwan }
//for wwan in [
// "5000c5008508c433",
// "5000c500850989cf",
// "5000c5008508f843",
// "5000c5008508baf7",
//]
{ name: "sdn" },
{ name: "sda" },
{ name: "sdb" },
{ name: "sdc" },
],
},
{
name: "dcr01s24.hswaw.net",
location: "rack=dcr01 host=dcr01s22",
devices: [
// https://github.com/rook/rook/issues/1228
//{ name: "disk/by-id/wwan-0x" + wwan }
//for wwan in [
// "5000c5008508ee03",
// "5000c5008508c9ef",
// "5000c5008508df33",
// "5000c5008508dd3b",
//]
{ name: "sdm" },
{ name: "sda" },
{ name: "sdb" },
{ name: "sdc" },
],
},
],
},
benji:: {
metadataStorageClass: "waw-hdd-redundant-3",
encryptionPassword: std.split((importstr "../secrets/plain/k0-benji-encryption-password"), '\n')[0],
pools: [
"waw-hdd-redundant-3",
"waw-hdd-redundant-3-metadata",
"waw-hdd-yolo-3",
],
s3Configuration: {
awsAccessKeyId: "RPYZIROFXNLQVU2WJ4R3",
awsSecretAccessKey: std.split((importstr "../secrets/plain/k0-benji-secret-access-key"), '\n')[0],
bucketName: "benji-k0-backups-waw3",
endpointUrl: "https://s3.eu-central-1.wasabisys.com/",
},
}
},
},
waw3Pools: {
// redundant block storage
blockRedundant: rook.ECBlockPool(k0.ceph.waw3, "waw-hdd-redundant-3") {
metadataReplicas: 2,
spec: {
failureDomain: "host",
replicated: {
size: 2,
},
},
},
// yolo block storage (low usage, no host redundancy)
blockYolo: rook.ReplicatedBlockPool(k0.ceph.waw3, "waw-hdd-yolo-3") {
spec: {
failureDomain: "osd",
erasureCoded: {
dataChunks: 2,
codingChunks: 1,
},
},
},
// q3k's personal pool, used externally from k8s.
q3kRedundant: rook.ECBlockPool(k0.ceph.waw3, "waw-hdd-redundant-q3k-3") {
metadataReplicas: 2,
spec: {
failureDomain: "host",
replicated: {
size: 2,
},
},
},
objectRedundant: rook.S3ObjectStore(k0.ceph.waw3, "waw-hdd-redundant-3-object") {
spec: {
metadataPool: {
failureDomain: "host",
replicated: { size: 2 },
},
dataPool: {
failureDomain: "host",
replicated: { size: 2 },
},
},
},
},
// Clients for S3/radosgw storage.
clients: {
# Used for owncloud.hackerspace.pl, which for now lives on boston-packets.hackerspace.pl.
nextcloudWaw3: kube.CephObjectStoreUser("nextcloud") {
metadata+: {
namespace: "ceph-waw3",
},
spec: {
store: "waw-hdd-redundant-3-object",
displayName: "nextcloud",
},
},
# issues.hackerspace.pl (redmine) attachments bucket
issuesWaw3: kube.CephObjectStoreUser("issues") {
metadata+: {
namespace: "ceph-waw3",
},
spec: {
store: "waw-hdd-redundant-3-object",
displayName: "issues",
},
},
# nuke@hackerspace.pl's personal storage.
nukePersonalWaw3: kube.CephObjectStoreUser("nuke-personal") {
metadata+: {
namespace: "ceph-waw3",
},
spec: {
store: "waw-hdd-redundant-3-object",
displayName: "nuke-personal",
},
},
# patryk@hackerspace.pl's ArmA3 mod bucket.
cz2ArmaModsWaw3: kube.CephObjectStoreUser("cz2-arma3mods") {
metadata+: {
namespace: "ceph-waw3",
},
spec: {
store: "waw-hdd-redundant-3-object",
displayName: "cz2-arma3mods",
},
},
# Buckets for spark pipelines
# TODO(implr): consider a second yolo-backed one for temp data
implrSparkWaw3: kube.CephObjectStoreUser("implr-spark") {
metadata+: {
namespace: "ceph-waw3",
},
spec: {
store: "waw-hdd-redundant-3-object",
displayName: "implr-spark",
},
},
# q3k's personal user
q3kWaw3: kube.CephObjectStoreUser("q3k") {
metadata+: {
namespace: "ceph-waw3",
},
spec: {
store: "waw-hdd-redundant-3-object",
displayName: "q3k",
},
},
# woju's personal user
wojuWaw3: kube.CephObjectStoreUser("woju") {
metadata+: {
namespace: "ceph-waw3",
},
spec: {
store: "waw-hdd-redundant-3-object",
displayName: "woju",
},
},
# cz3's (patryk@hackerspace.pl) personal user
cz3Waw3: kube.CephObjectStoreUser("cz3") {
metadata+: {
namespace: "ceph-waw3",
},
spec: {
store: "waw-hdd-redundant-3-object",
displayName: "cz3",
},
},
},
},
# These are policies allowing for Insecure pods in some namespaces.
# A lot of them are spurious and come from the fact that we deployed
# these namespaces before we deployed the draconian PodSecurityPolicy
# we have now. This should be fixed by setting up some more granular
# policies, or fixing the workloads to not need some of the permission
# bits they use, whatever those might be.
# TODO(q3k): fix this?
unnecessarilyInsecureNamespaces: [
policies.AllowNamespaceInsecure("ceph-waw3"),
policies.AllowNamespaceInsecure("matrix"),
policies.AllowNamespaceInsecure("registry"),
policies.AllowNamespaceInsecure("internet"),
# TODO(implr): restricted policy with CAP_NET_ADMIN and tuntap, but no full root
policies.AllowNamespaceInsecure("implr-vpn"),
],
# Admission controller that permits non-privileged users to manage
# their namespaces without danger of hijacking important URLs.
admitomatic: admitomatic.Environment {
cfg+: {
proto: {
// Domains allowed in given namespaces. If a domain exists
// anywhere, ingresses will only be permitted to be created
// within namespaces in which it appears here. This works
// the same way for wildcards, if a wildcard exists in this
// list it blocks all unauthorized uses of that domain
// elsewhere.
//
// See //cluster/admitomatic for more information.
//
// Or, tl;dr:
//
// If you do a wildcard CNAME onto the k0 ingress, you
// should explicitly state *.your.name.com here.
//
// If you just want to protect your host from being
// hijacked by other cluster users, you should also state
// it here (either as a wildcard, or unary domains).
allow_domain: [
{ namespace: "covid-formity", dns: "covid19.hackerspace.pl" },
{ namespace: "covid-formity", dns: "covid.hackerspace.pl" },
{ namespace: "covid-formity", dns: "www.covid.hackerspace.pl" },
{ namespace: "devtools-prod", dns: "hackdoc.hackerspace.pl" },
{ namespace: "devtools-prod", dns: "cs.hackerspace.pl" },
{ namespace: "engelsystem-prod", dns: "engelsystem.hackerspace.pl" },
{ namespace: "gerrit", dns: "gerrit.hackerspace.pl" },
{ namespace: "gitea-prod", dns: "gitea.hackerspace.pl" },
{ namespace: "hswaw-prod", dns: "*.hackerspace.pl" },
{ namespace: "internet", dns: "internet.hackerspace.pl" },
{ namespace: "matrix", dns: "matrix.hackerspace.pl" },
{ namespace: "onlyoffice-prod", dns: "office.hackerspace.pl" },
{ namespace: "redmine", dns: "issues.hackerspace.pl" },
{ namespace: "redmine", dns: "b.hackerspace.pl" },
{ namespace: "redmine", dns: "b.hswaw.net" },
{ namespace: "redmine", dns: "xn--137h.hackerspace.pl" },
{ namespace: "redmine", dns: "xn--137h.hswaw.net" },
{ namespace: "speedtest", dns: "speedtest.hackerspace.pl" },
{ namespace: "sso", dns: "sso.hackerspace.pl" },
{ namespace: "ceph-waw3", dns: "ceph-waw3.hswaw.net" },
{ namespace: "ceph-waw3", dns: "object.ceph-waw3.hswaw.net" },
{ namespace: "monitoring-global-k0", dns: "*.hswaw.net" },
{ namespace: "registry", dns: "*.hswaw.net" },
// q3k's legacy namespace (pre-prodvider)
{ namespace: "q3k", dns: "*.q3k.org" },
{ namespace: "personal-q3k", dns: "*.q3k.org" },
],
},
},
},
},
}