cluster/kube/cluster.jsonnet

# Top level cluster configuration.

local kube = import "../../kube/kube.libsonnet";
local policies = import "../../kube/policies.libsonnet";

local calico = import "lib/calico.libsonnet";
local certmanager = import "lib/cert-manager.libsonnet";
local cockroachdb = import "lib/cockroachdb.libsonnet";
local coredns = import "lib/coredns.libsonnet";
local metallb = import "lib/metallb.libsonnet";
local metrics = import "lib/metrics.libsonnet";
local nginx = import "lib/nginx.libsonnet";
local prodvider = import "lib/prodvider.libsonnet";
local registry = import "lib/registry.libsonnet";
local rook = import "lib/rook.libsonnet";

local Cluster(fqdn) = {
    local cluster = self,
    local cfg = cluster.cfg,

    cfg:: {
        // Storage class used for internal services (like registry). This must
        // be set to a valid storage class. This can either be a cloud provider class
        // (when running on GKE &co) or a storage class created using rook.
        storageClassNameRedundant: error "storageClassNameRedundant must be set",
    },

    // These are required to let the API Server contact kubelets.
    crAPIServerToKubelet: kube.ClusterRole("system:kube-apiserver-to-kubelet") {
        metadata+: {
            annotations+: {
                "rbac.authorization.kubernetes.io/autoupdate": "true",
            },
            labels+: {
                "kubernetes.io/bootstrapping": "rbac-defaults",
            },
        },
        rules: [
            {
                apiGroups: [""],
                resources: ["nodes/%s" % r for r in [ "proxy", "stats", "log", "spec", "metrics" ]],
                verbs: ["*"],
            },
        ],
    },
    crbAPIServer: kube.ClusterRoleBinding("system:kube-apiserver") {
        roleRef: {
            apiGroup: "rbac.authorization.k8s.io",
            kind: "ClusterRole",
            name: cluster.crAPIServerToKubelet.metadata.name,
        },
        subjects: [
            {
                apiGroup: "rbac.authorization.k8s.io",
                kind: "User",
                # A cluster API Server authenticates with a certificate whose CN is == to the FQDN of the cluster.
                name: fqdn,
            },
        ],
    },

    // This ClusteRole is bound to all humans that log in via prodaccess/prodvider/SSO.
    // It should allow viewing of non-sensitive data for debugability and openness.
    crViewer: kube.ClusterRole("system:viewer") {
        rules: [
            {
                apiGroups: [""],
                resources: [
                    "nodes",
                    "namespaces",
                    "pods",
                    "configmaps",
                    "services",
                ],
                verbs: ["list"],
            },
            {
                apiGroups: ["metrics.k8s.io"],
                resources: [
                    "nodes",
                    "pods",
                ],
                verbs: ["list"],
            },
            {
                apiGroups: ["apps"],
                resources: [
                    "statefulsets",
                ],
                verbs: ["list"],
            },
            {
                apiGroups: ["extensions"],
                resources: [
                    "deployments",
                    "ingresses",
                ],
                verbs: ["list"],
            }
        ],
    },
    // This ClusterRole is applied (scoped to personal namespace) to all humans.
    crFullInNamespace: kube.ClusterRole("system:admin-namespace") {
        rules: [
            {
                apiGroups: ["*"],
                resources: ["*"],
                verbs: ["*"],
            },
        ],
    },
    // This ClusterRoleBindings allows root access to cluster admins.
    crbAdmins: kube.ClusterRoleBinding("system:admins") {
        roleRef: {
            apiGroup: "rbac.authorization.k8s.io",
            kind: "ClusterRole",
            name: "cluster-admin",
        },
        subjects: [
            {
                apiGroup: "rbac.authorization.k8s.io",
                kind: "User",
                name: user + "@hackerspace.pl",
            } for user in [
                "q3k",
                "implr",
                "informatic",
            ]
        ],
    },

    podSecurityPolicies: policies.Cluster {},

    allowInsecureNamespaces: [
        policies.AllowNamespaceInsecure("kube-system"),
        # TODO(q3k): fix this?
        policies.AllowNamespaceInsecure("ceph-waw2"),
    ],

    // Allow all service accounts (thus all controllers) to create secure pods.
    crbAllowServiceAccountsSecure: kube.ClusterRoleBinding("policy:allow-all-secure") {
        roleRef_: cluster.podSecurityPolicies.secureRole,
        subjects: [
            {
                kind: "Group",
                apiGroup: "rbac.authorization.k8s.io",
                name: "system:serviceaccounts",
            }
        ],
    },

    // Calico network fabric
    calico: calico.Environment {},
    // CoreDNS for this cluster.
    dns: coredns.Environment {},
    // Metrics Server
    metrics: metrics.Environment {},
    // Metal Load Balancer
    metallb: metallb.Environment {
        cfg+: {
            addressPools: [
                { name: "public-v4-1", protocol: "layer2", addresses: ["185.236.240.50-185.236.240.63"] },
            ],
        },
    },
    // Main nginx Ingress Controller
    nginx: nginx.Environment {},
    certmanager: certmanager.Environment {},
    issuer: certmanager.ClusterIssuer("letsencrypt-prod") {
        spec: {
            acme: {
                server: "https://acme-v02.api.letsencrypt.org/directory",
                email: "bofh@hackerspace.pl",
                privateKeySecretRef: {
                    name: "letsencrypt-prod"
                },
                http01: {},
            },
        },
    },

    // Rook Ceph storage
    rook: rook.Operator {
        operator+: {
            spec+: {
                // TODO(q3k): Bring up the operator again when stability gets fixed
                // See: https://github.com/rook/rook/issues/3059#issuecomment-492378873
                replicas: 1,
            },
        },
    },

    // Docker registry
    registry: registry.Environment {
        cfg+: {
            domain: "registry.%s" % [fqdn],
            storageClassName: cfg.storageClassNameParanoid,
            objectStorageName: "waw-hdd-redundant-2-object",
        },
    },

    // Prodvider
    prodvider: prodvider.Environment {},
};


{
    k0: {
        local k0 = self,
        cluster: Cluster("k0.hswaw.net") {
            cfg+: {
                storageClassNameParanoid: k0.ceph.blockParanoid.name,
            },
        },
        cockroach: {
            waw2: cockroachdb.Cluster("crdb-waw1") {
                cfg+: {
                    topology: [
                        { name: "bc01n01", node: "bc01n01.hswaw.net" },
                        { name: "bc01n02", node: "bc01n02.hswaw.net" },
                        { name: "bc01n03", node: "bc01n03.hswaw.net" },
                    ],
                    hostPath: "/var/db/crdb-waw1",
                },
            },
            clients: {
                cccampix: k0.cockroach.waw2.Client("cccampix"),
                cccampixDev: k0.cockroach.waw2.Client("cccampix-dev"),
            },
        },
        ceph: {
            // waw1 cluster - dead as of 2019/08/06, data corruption
            // waw2 cluster
            waw2: rook.Cluster(k0.cluster.rook, "ceph-waw2") {
                spec: {
                    mon: {
                        count: 3,
                        allowMultiplePerNode: false,
                    },
                    storage: {
                        useAllNodes: false,
                        useAllDevices: false,
                        config: {
                            databaseSizeMB: "1024",
                            journalSizeMB: "1024",
                        },
                        nodes: [
                            {
                                name: "bc01n01.hswaw.net",
                                location: "rack=dcr01 chassis=bc01 host=bc01n01",
                                devices: [ { name: "sda" } ],
                            },
                            {
                                name: "bc01n02.hswaw.net",
                                location: "rack=dcr01 chassis=bc01 host=bc01n02",
                                devices: [ { name: "sda" } ],
                            },
                            {
                                name: "bc01n03.hswaw.net",
                                location: "rack=dcr01 chassis=bc01 host=bc01n03",
                                devices: [ { name: "sda" } ],
                            },
                        ],
                    },
                },
            },
            // redundant block storage
            blockRedundant: rook.ECBlockPool(k0.ceph.waw2, "waw-hdd-redundant-2") {
                spec: {
                    failureDomain: "host",
                    erasureCoded: {
                        dataChunks: 2,
                        codingChunks: 1,
                    },
                },
            },
            // paranoid block storage (3 replicas)
            blockParanoid: rook.ReplicatedBlockPool(k0.ceph.waw2, "waw-hdd-paranoid-2") {
                spec: {
                    failureDomain: "host",
                    replicated: {
                        size: 3,
                    },
                },
            },
            // yolo block storage (no replicas!)
            blockYolo: rook.ReplicatedBlockPool(k0.ceph.waw2, "waw-hdd-yolo-2") {
                spec: {
                    failureDomain: "host",
                    replicated: {
                        size: 1,
                    },
                },
            },
            objectRedundant: rook.S3ObjectStore(k0.ceph.waw2, "waw-hdd-redundant-2-object") {
                spec: {
                    metadataPool: {
                        failureDomain: "host",
                        replicated: { size: 3 },
                    },
                    dataPool: {
                        failureDomain: "host",
                        erasureCoded: {
                            dataChunks: 2,
                            codingChunks: 1,
                        },
                    },
                },
            },
        },
    },
}