cluster/kube/cluster.jsonnet

# Top level cluster configuration.

local kube = import "../../kube/kube.libsonnet";
local policies = import "../../kube/policies.libsonnet";

local calico = import "lib/calico.libsonnet";
local certmanager = import "lib/cert-manager.libsonnet";
local cockroachdb = import "lib/cockroachdb.libsonnet";
local coredns = import "lib/coredns.libsonnet";
local metallb = import "lib/metallb.libsonnet";
local metrics = import "lib/metrics.libsonnet";
local nginx = import "lib/nginx.libsonnet";
local prodvider = import "lib/prodvider.libsonnet";
local registry = import "lib/registry.libsonnet";
local rook = import "lib/rook.libsonnet";
local pki = import "lib/pki.libsonnet";

local Cluster(short, realm) = {
    local cluster = self,
    local cfg = cluster.cfg,

    short:: short,
    realm:: realm,
    fqdn:: "%s.%s" % [cluster.short, cluster.realm],

    cfg:: {
        // Storage class used for internal services (like registry). This must
        // be set to a valid storage class. This can either be a cloud provider class
        // (when running on GKE &co) or a storage class created using rook.
        storageClassNameRedundant: error "storageClassNameRedundant must be set",
    },

    // These are required to let the API Server contact kubelets.
    crAPIServerToKubelet: kube.ClusterRole("system:kube-apiserver-to-kubelet") {
        metadata+: {
            annotations+: {
                "rbac.authorization.kubernetes.io/autoupdate": "true",
            },
            labels+: {
                "kubernetes.io/bootstrapping": "rbac-defaults",
            },
        },
        rules: [
            {
                apiGroups: [""],
                resources: ["nodes/%s" % r for r in [ "proxy", "stats", "log", "spec", "metrics" ]],
                verbs: ["*"],
            },
        ],
    },
    crbAPIServer: kube.ClusterRoleBinding("system:kube-apiserver") {
        roleRef: {
            apiGroup: "rbac.authorization.k8s.io",
            kind: "ClusterRole",
            name: cluster.crAPIServerToKubelet.metadata.name,
        },
        subjects: [
            {
                apiGroup: "rbac.authorization.k8s.io",
                kind: "User",
                # A cluster API Server authenticates with a certificate whose CN is == to the FQDN of the cluster.
                name: cluster.fqdn,
            },
        ],
    },

    // This ClusteRole is bound to all humans that log in via prodaccess/prodvider/SSO.
    // It should allow viewing of non-sensitive data for debugability and openness.
    crViewer: kube.ClusterRole("system:viewer") {
        rules: [
            {
                apiGroups: [""],
                resources: [
                    "nodes",
                    "namespaces",
                    "pods",
                    "configmaps",
                    "services",
                ],
                verbs: ["list"],
            },
            {
                apiGroups: ["metrics.k8s.io"],
                resources: [
                    "nodes",
                    "pods",
                ],
                verbs: ["list"],
            },
            {
                apiGroups: ["apps"],
                resources: [
                    "statefulsets",
                ],
                verbs: ["list"],
            },
            {
                apiGroups: ["extensions"],
                resources: [
                    "deployments",
                    "ingresses",
                ],
                verbs: ["list"],
            }
        ],
    },
    // This ClusterRole is applied (scoped to personal namespace) to all humans.
    crFullInNamespace: kube.ClusterRole("system:admin-namespace") {
        rules: [
            {
                apiGroups: ["", "extensions", "apps"],
                resources: ["*"],
                verbs: ["*"],
            },
            {
                apiGroups: ["batch"],
                resources: ["jobs", "cronjobs"],
                verbs: ["*"],
            },
        ],
    },
    // This ClusterRoleBindings allows root access to cluster admins.
    crbAdmins: kube.ClusterRoleBinding("system:admins") {
        roleRef: {
            apiGroup: "rbac.authorization.k8s.io",
            kind: "ClusterRole",
            name: "cluster-admin",
        },
        subjects: [
            {
                apiGroup: "rbac.authorization.k8s.io",
                kind: "User",
                name: user + "@hackerspace.pl",
            } for user in [
                "q3k",
                "implr",
                "informatic",
            ]
        ],
    },

    podSecurityPolicies: policies.Cluster {},

    allowInsecureNamespaces: [
        policies.AllowNamespaceInsecure("kube-system"),
        policies.AllowNamespaceInsecure("metallb-system"),
        # TODO(q3k): fix this?
        policies.AllowNamespaceInsecure("ceph-waw2"),
        policies.AllowNamespaceInsecure("ceph-waw3"),
        policies.AllowNamespaceInsecure("matrix"),
        policies.AllowNamespaceInsecure("registry"),
        policies.AllowNamespaceInsecure("internet"),
        # TODO(implr): restricted policy with CAP_NET_ADMIN and tuntap, but no full root
        policies.AllowNamespaceInsecure("implr-vpn"),
    ],

    // Allow all service accounts (thus all controllers) to create secure pods.
    crbAllowServiceAccountsSecure: kube.ClusterRoleBinding("policy:allow-all-secure") {
        roleRef_: cluster.podSecurityPolicies.secureRole,
        subjects: [
            {
                kind: "Group",
                apiGroup: "rbac.authorization.k8s.io",
                name: "system:serviceaccounts",
            }
        ],
    },

    // Calico network fabric
    calico: calico.Environment {},
    // CoreDNS for this cluster.
    dns: coredns.Environment {
        cfg+: {
            cluster_domains: [
                "cluster.local",
                cluster.fqdn,
            ],
        },
    },
    // Metrics Server
    metrics: metrics.Environment {},
    // Metal Load Balancer
    metallb: metallb.Environment {
        cfg+: {
            peers: [
                {
                    "peer-address": "185.236.240.33",
                    "peer-asn": 65001,
                    "my-asn": 65002,
                },
            ],
            addressPools: [
                {
                    name: "public-v4-1",
                    protocol: "bgp",
                    addresses: [
                        "185.236.240.48/28",
                    ],
                },
                {
                    name: "public-v4-2",
                    protocol: "bgp",
                    addresses: [
                        "185.236.240.112/28"
                    ],
                },
            ],
        },
    },
    // Main nginx Ingress Controller
    nginx: nginx.Environment {},
    certmanager: certmanager.Environment {},
    issuer: kube.ClusterIssuer("letsencrypt-prod") {
        spec: {
            acme: {
                server: "https://acme-v02.api.letsencrypt.org/directory",
                email: "bofh@hackerspace.pl",
                privateKeySecretRef: {
                    name: "letsencrypt-prod"
                },
                http01: {},
            },
        },
    },

    // Rook Ceph storage
    rook: rook.Operator {
        operator+: {
            spec+: {
                // TODO(q3k): Bring up the operator again when stability gets fixed
                // See: https://github.com/rook/rook/issues/3059#issuecomment-492378873
                replicas: 1,
            },
        },
    },

    // Docker registry
    registry: registry.Environment {
        cfg+: {
            domain: "registry.%s" % [cluster.fqdn],
            storageClassName: cfg.storageClassNameParanoid,
            objectStorageName: "waw-hdd-redundant-2-object",
        },
    },

    // TLS PKI machinery
    pki: pki.Environment(cluster.short, cluster.realm),

    // Prodvider
    prodvider: prodvider.Environment {
        cfg+: {
            apiEndpoint: "kubernetes.default.svc.%s" % [cluster.fqdn],
        },
    },
};


{
    k0: {
        local k0 = self,
        cluster: Cluster("k0", "hswaw.net") {
            cfg+: {
                storageClassNameParanoid: k0.ceph.waw2Pools.blockParanoid.name,
            },
        },
        cockroach: {
            waw2: cockroachdb.Cluster("crdb-waw1") {
                cfg+: {
                    topology: [
                        { name: "bc01n01", node: "bc01n01.hswaw.net" },
                        { name: "bc01n02", node: "bc01n02.hswaw.net" },
                        { name: "bc01n03", node: "bc01n03.hswaw.net" },
                    ],
                    hostPath: "/var/db/crdb-waw1",
                },
            },
            clients: {
                cccampix: k0.cockroach.waw2.Client("cccampix"),
                cccampixDev: k0.cockroach.waw2.Client("cccampix-dev"),
                buglessDev: k0.cockroach.waw2.Client("bugless-dev"),
            },
        },
        ceph: {
            // waw1 cluster - dead as of 2019/08/06, data corruption
            // waw2 cluster
            waw2: rook.Cluster(k0.cluster.rook, "ceph-waw2") {
                spec: {
                    mon: {
                        count: 3,
                        allowMultiplePerNode: false,
                    },
                    storage: {
                        useAllNodes: false,
                        useAllDevices: false,
                        config: {
                            databaseSizeMB: "1024",
                            journalSizeMB: "1024",
                        },
                        nodes: [
                            {
                                name: "bc01n01.hswaw.net",
                                location: "rack=dcr01 chassis=bc01 host=bc01n01",
                                devices: [ { name: "sda" } ],
                            },
                            {
                                name: "bc01n02.hswaw.net",
                                location: "rack=dcr01 chassis=bc01 host=bc01n02",
                                devices: [ { name: "sda" } ],
                            },
                            {
                                name: "bc01n03.hswaw.net",
                                location: "rack=dcr01 chassis=bc01 host=bc01n03",
                                devices: [ { name: "sda" } ],
                            },
                        ],
                    },
                    benji:: {
                        metadataStorageClass: "waw-hdd-paranoid-2",
                        encryptionPassword: std.split((importstr "../secrets/plain/k0-benji-encryption-password"), '\n')[0],
                        pools: [
                            "waw-hdd-redundant-2",
                            "waw-hdd-redundant-2-metadata",
                            "waw-hdd-paranoid-2",
                            "waw-hdd-yolo-2",
                        ],
                        s3Configuration: {
                            awsAccessKeyId: "RPYZIROFXNLQVU2WJ4R3",
                            awsSecretAccessKey: std.split((importstr "../secrets/plain/k0-benji-secret-access-key"), '\n')[0],
                            bucketName: "benji-k0-backups",
                            endpointUrl: "https://s3.eu-central-1.wasabisys.com/",
                        },
                    }
                },
            },
            waw2Pools: {
                // redundant block storage
                blockRedundant: rook.ECBlockPool(k0.ceph.waw2, "waw-hdd-redundant-2") {
                    spec: {
                        failureDomain: "host",
                        erasureCoded: {
                            dataChunks: 2,
                            codingChunks: 1,
                        },
                    },
                },
                // paranoid block storage (3 replicas)
                blockParanoid: rook.ReplicatedBlockPool(k0.ceph.waw2, "waw-hdd-paranoid-2") {
                    spec: {
                        failureDomain: "host",
                        replicated: {
                            size: 3,
                        },
                    },
                },
                // yolo block storage (no replicas!)
                blockYolo: rook.ReplicatedBlockPool(k0.ceph.waw2, "waw-hdd-yolo-2") {
                    spec: {
                        failureDomain: "host",
                        replicated: {
                            size: 1,
                        },
                    },
                },
                objectRedundant: rook.S3ObjectStore(k0.ceph.waw2, "waw-hdd-redundant-2-object") {
                    spec: {
                        metadataPool: {
                            failureDomain: "host",
                            replicated: { size: 3 },
                        },
                        dataPool: {
                            failureDomain: "host",
                            erasureCoded: {
                                dataChunks: 2,
                                codingChunks: 1,
                            },
                        },
                    },
                },
            },
            waw3: rook.Cluster(k0.cluster.rook, "ceph-waw3") {
                spec: {
                    mon: {
                        count: 3,
                        allowMultiplePerNode: false,
                    },
                    storage: {
                        useAllNodes: false,
                        useAllDevices: false,
                        config: {
                            databaseSizeMB: "1024",
                            journalSizeMB: "1024",
                        },
                        nodes: [
                            {
                                name: "dcr01s22.hswaw.net",
                                location: "rack=dcr01 host=dcr01s22",
                                devices: [
                                    // https://github.com/rook/rook/issues/1228
                                    //{ name: "disk/by-id/wwan-0x" + wwan }
                                    //for wwan in [
                                    //    "5000c5008508c433",
                                    //    "5000c500850989cf",
                                    //    "5000c5008508f843",
                                    //    "5000c5008508baf7",
                                    //]
                                    { name: "sdn" },
                                    { name: "sda" },
                                    { name: "sdb" },
                                    { name: "sdc" },
                                ],
                            },
                            {
                                name: "dcr01s24.hswaw.net",
                                location: "rack=dcr01 host=dcr01s22",
                                devices: [
                                    // https://github.com/rook/rook/issues/1228
                                    //{ name: "disk/by-id/wwan-0x" + wwan }
                                    //for wwan in [
                                    //    "5000c5008508ee03",
                                    //    "5000c5008508c9ef",
                                    //    "5000c5008508df33",
                                    //    "5000c5008508dd3b",
                                    //]
                                    { name: "sdm" },
                                    { name: "sda" },
                                    { name: "sdb" },
                                    { name: "sdc" },
                                ],
                            },
                        ],
                    },
                    benji:: {
                        metadataStorageClass: "waw-hdd-redundant-3",
                        encryptionPassword: std.split((importstr "../secrets/plain/k0-benji-encryption-password"), '\n')[0],
                        pools: [
                            "waw-hdd-redundant-3",
                            "waw-hdd-redundant-3-metadata",
                            "waw-hdd-yolo-3",
                        ],
                        s3Configuration: {
                            awsAccessKeyId: "RPYZIROFXNLQVU2WJ4R3",
                            awsSecretAccessKey: std.split((importstr "../secrets/plain/k0-benji-secret-access-key"), '\n')[0],
                            bucketName: "benji-k0-backups-waw3",
                            endpointUrl: "https://s3.eu-central-1.wasabisys.com/",
                        },
                    }
                },
            },
            waw3Pools: {
                // redundant block storage
                blockRedundant: rook.ECBlockPool(k0.ceph.waw3, "waw-hdd-redundant-3") {
                    metadataReplicas: 2,
                    spec: {
                        failureDomain: "host",
                        replicated: {
                          size: 2,
                        },
                    },
                },
                // yolo block storage (low usage, no host redundancy)
                blockYolo: rook.ReplicatedBlockPool(k0.ceph.waw3, "waw-hdd-yolo-3") {
                    spec: {
                        failureDomain: "osd",
                        erasureCoded: {
                            dataChunks: 12,
                            codingChunks: 4,
                        },
                    },
                },
                objectRedundant: rook.S3ObjectStore(k0.ceph.waw3, "waw-hdd-redundant-3-object") {
                    spec: {
                        metadataPool: {
                            failureDomain: "host",
                            replicated: { size: 2 },
                        },
                        dataPool: {
                            failureDomain: "host",
                            replicated: { size: 2 },
                        },
                    },
                },
            },
        },

        # Used for owncloud.hackerspace.pl, which for now lices on boston-packets.hackerspace.pl.
        nextcloudWaw3: kube.CephObjectStoreUser("nextcloud") {
            metadata+: {
                namespace: "ceph-waw3",
            },
            spec: {
                store: "waw-hdd-redundant-3-object",
                displayName: "nextcloud",
            },
        },

        # nuke@hackerspace.pl's personal storage.
        nukePersonalWaw3: kube.CephObjectStoreuser("nuke-personal") {
            metadata+: {
                namespace: "ceph-waw3",
            },
            spec: {
                store: "waw-hdd-redundant-3-object",
                displayName: "nuke-personal",
            },
        },
    },
}