cluster/kube/cluster.jsonnet

# Top level cluster configuration.

local kube = import "../../kube/kube.libsonnet";
local coredns = import "lib/coredns.libsonnet";
local metrics = import "lib/metrics.libsonnet";
local calico = import "lib/calico.libsonnet";
local metallb = import "lib/metallb.libsonnet";
local nginx = import "lib/nginx.libsonnet";
local rook = import "lib/rook.libsonnet";
local certmanager = import "lib/cert-manager.libsonnet";

local Cluster(fqdn) = {
    local cluster = self,

    // These are required to let the API Server contact kubelets.
    crAPIServerToKubelet: kube.ClusterRole("system:kube-apiserver-to-kubelet") {
        metadata+: {
            annotations+: {
                "rbac.authorization.kubernetes.io/autoupdate": "true",
            },
            labels+: {
                "kubernets.io/bootstrapping": "rbac-defaults",
            },
        },
        rules: [
            {
                apiGroups: [""],
                resources: ["nodes/%s" % r for r in [ "proxy", "stats", "log", "spec", "metrics" ]],
                verbs: ["*"],
            },
        ],
    },
    crbAPIServer: kube.ClusterRoleBinding("system:kube-apiserver") {
        roleRef: {
            apiGroup: "rbac.authorization.k8s.io",
            kind: "ClusterRole",
            name: cluster.crAPIServerToKubelet.metadata.name,
        },
        subjects: [
            {
                apiGroup: "rbac.authorization.k8s.io",
                kind: "User",
                # A cluster API Server authenticates with a certificate whose CN is == to the FQDN of the cluster.
                name: fqdn,
            },
        ],
    },

    // Calico network fabric
    calico: calico.Environment {},
    // CoreDNS for this cluster.
    dns: coredns.Environment {},
    // Metrics Server
    metrics: metrics.Environment {},
    // Metal Load Balancer
    metallb: metallb.Environment {
        cfg+: {
            addressPools: [
                { name: "public-v4-1", protocol: "layer2", addresses: ["185.236.240.50-185.236.240.63"] },
            ],
        },
    },
    // Main nginx Ingress Controller
    nginx: nginx.Environment {},
    certmanager: certmanager.Environment {},

    // Rook Ceph storage
    rook: rook.Operator {},
    // waw1 ceph cluster
    cephWaw1: rook.Cluster(cluster.rook, "ceph-waw1") {
        spec: {
            mon: {
                count: 3,
                allowMultiplePerNode: false,
            },
            storage: {
                useAllNodes: false,
                useAllDevices: false,
                config: {
                    databaseSizeMB: "1024",
                    journalSizeMB: "1024",
                },
                nodes: [
                    {
                        name: "bc01n01.hswaw.net",
                        location: "rack=dcr01 chassis=bc01 host=bc01n01",
                        devices: [ { name: "sda" } ],
                    },
                    {
                        name: "bc01n02.hswaw.net",
                        location: "rack=dcr01 chassis=bc01 host=bc01n02",
                        devices: [ { name: "sda" } ],
                    },
                    {
                        name: "bc01n03.hswaw.net",
                        location: "rack=dcr01 chassis=bc01 host=bc01n03",
                        devices: [ { name: "sda" } ],
                    },
                ],
            },
        },
    },
    // redundant block storage
    cephWaw1Redundant: rook.ECBlockPool(cluster.cephWaw1, "waw-hdd-redundant-1") {
        spec: {
            failureDomain: "host",
            erasureCoded: {
                dataChunks: 2,
                codingChunks: 1,
            },
        },
    },
};


{
    k0: Cluster("k0.hswaw.net"),
}