cluster/kube/lib/calico.libsonnet

# Deploy hosted calico with its own etcd.

local kube = import "../../../kube/kube.libsonnet";

local bindServiceAccountClusterRole(sa, cr) = kube.ClusterRoleBinding(cr.metadata.name) {
    roleRef: {
        apiGroup: "rbac.authorization.k8s.io",
        kind: "ClusterRole",
        name: cr.metadata.name,
    },
    subjects: [
        {
            kind: "ServiceAccount",
            name: sa.metadata.name,
            namespace: sa.metadata.namespace,
        },
    ],
};

{
    Environment: {
        local env = self,
        local cfg = env.cfg,
        cfg:: {
            namespace: "kube-system",
            version: "v3.15.5",
            imageController: "calico/kube-controllers:" + cfg.version,
            imageCNI: "calico/cni:" + cfg.version,
            imageNode: "calico/node:" + cfg.version,
            // TODO(implr): migrate calico from etcd to apiserver
            etcd: {
                endpoints: [
                    "https://bc01n01.hswaw.net:2379",
                    "https://bc01n02.hswaw.net:2379",
                    "https://dcr01s22.hswaw.net:2379",
                    "https://dcr01s24.hswaw.net:2379",
                ],
                ca: importstr "../../certs/ca-etcd.crt",
                cert: importstr "../../certs/etcd-calico.cert",
                key: importstr "../../secrets/plain/etcd-calico.key",
            },
        },

        cm: kube.ConfigMap("calico-config") {
            local cm = self,
            secretPrefix:: "/calico-secrets/",

            metadata+: {
                namespace: cfg.namespace,
            },

            data: {
                etcd_endpoints: std.join(",", cfg.etcd.endpoints),

                etcd_ca: cm.secretPrefix + "etcd-ca",
                etcd_cert: cm.secretPrefix + "etcd-cert",
                etcd_key: cm.secretPrefix + "etcd-key",

                calico_backend: "bird",
                veth_mtu: "1440",

                typha_service_name: "none",

                cni_network_config: |||
                   {
                     "name": "k8s-pod-network",
                     "cniVersion": "0.3.1",
                     "plugins": [
                       {
                         "type": "calico",
                         "log_level": "info",
                         "etcd_endpoints": "__ETCD_ENDPOINTS__",
                         "etcd_key_file": "__ETCD_KEY_FILE__",
                         "etcd_cert_file": "__ETCD_CERT_FILE__",
                         "etcd_ca_cert_file": "__ETCD_CA_CERT_FILE__",
                         "datastore_type": "etcdv3",
                         "mtu": __CNI_MTU__,
                         "ipam": {
                             "type": "calico-ipam"
                         },
                         "policy": {
                             "type": "k8s"
                         },
                         "kubernetes": {
                             "kubeconfig": "__KUBECONFIG_FILEPATH__"
                         }
                       },
                       {
                         "type": "portmap",
                         "snat": true,
                         "capabilities": {"portMappings": true}
                       },
                       {
                         "type": "bandwidth",
                         "capabilities": {"bandwidth": true}
                       }
                     ]
                   }
                |||
            },
        },

        secrets: kube.Secret("calico-secrets") {
            metadata+: {
                namespace: cfg.namespace,
            },

            data_: {
                "etcd-ca": cfg.etcd.ca,
                "etcd-cert": cfg.etcd.cert,
                "etcd-key": cfg.etcd.key,
            },
        },

        saNode: kube.ServiceAccount("calico-node") {
            metadata+: {
                namespace: cfg.namespace,
            },
        },

        crNode: kube.ClusterRole("calico-node") {
            rules: [
                {
                    apiGroups: [""],
                    resources: ["pods", "nodes", "namespaces"],
                    verbs: ["get"],
                },
                {
                    apiGroups: [""],
                    resources: ["endpoints", "services"],
                    verbs: ["watch", "list", "get"],
                },
                {
                    apiGroups: [""],
                    resources: ["configmaps"],
                    verbs: ["get"],
                },
                {
                    apiGroups: [""],
                    resources: ["nodes/status"],
                    verbs: ["patch", "update"],
                },
                {
                    apiGroups: [""],
                    resources: ["pods/status"],
                    verbs: ["patch"],
                },
                {
                    apiGroups: [""],
                    resources: ["nodes"],
                    verbs: ["get", "list", "watch"],
                },
            ],
        },

        crbNode: bindServiceAccountClusterRole(env.saNode, env.crNode),

        saController: kube.ServiceAccount("calico-kube-controllers") {
            metadata+: {
                namespace: cfg.namespace,
            },
        },

        crController: kube.ClusterRole("calico-kube-controllers") {
            rules: [
                {
                    apiGroups: [""],
                    resources: ["nodes", "pods", "namespaces", "serviceaccounts"],
                    verbs: ["watch", "list", "get"],
                },
                {
                    apiGroups: ["networking.k8s.io"],
                    resources: ["networkpolicies"],
                    verbs: ["watch", "list"],
                },
            ],
        },

        crbController: bindServiceAccountClusterRole(env.saController, env.crController),

        controller: kube.Deployment("calico-kube-controllers") {
            metadata+: {
                namespace: cfg.namespace,
                labels+: {
                    "k8s-app": "calico-kube-controllers",
                },
            },
            spec+: {
                replicas: 1,
                strategy: { type: "Recreate" },
                template+: {
                    spec+: {
                        nodeSelector: {
                            "kubernetes.io/os": "linux"
                        },
                        tolerations: [
                            { key: "CriticalAddonsOnly", operator: "Exists" },
                            { key: "node-role.kubernetes.io/master", effect: "NoSchedule" },
                        ],
                        serviceAccountName: env.saController.metadata.name,
                        priorityClassName: "system-cluster-critical",
                        hostNetwork: true,
                        containers_: {
                            "calico-kube-controllers": kube.Container("calico-kube-controllers") {
                                image: cfg.imageController,
                                env_: {
                                    ETCD_ENDPOINTS: kube.ConfigMapRef(env.cm, "etcd_endpoints"),
                                    ETCD_CA_CERT_FILE: kube.ConfigMapRef(env.cm, "etcd_ca"),
                                    ETCD_KEY_FILE: kube.ConfigMapRef(env.cm, "etcd_key"),
                                    ETCD_CERT_FILE: kube.ConfigMapRef(env.cm, "etcd_cert"),
                                    ENABLED_CONTROLLERS: "policy,namespace,serviceaccount,workloadendpoint,node",
                                },
                                volumeMounts_: {
                                    secrets: {
                                        mountPath: env.cm.secretPrefix,
                                    },
                                },
                                readinessProbe: {
                                    exec: {
                                        command: [ "/usr/bin/check-status", "-r" ],
                                    },
                                },
                            },
                        },
                        volumes_: {
                            secrets: kube.SecretVolume(env.secrets),
                        },
                    },
                },
            },
        },

        # ConfigMap that holds overriden bird.cfg.template and bird_ipam.cfg.template.
        calicoMetallbBird: kube.ConfigMap("calico-metallb-bird") {
            metadata+: {
                namespace: cfg.namespace,
            },
            data: {
                "bird.cfg.template": (importstr "calico-bird.cfg.template"),
                "bird_ipam.cfg.template": (importstr "calico-bird-ipam.cfg.template"),
            },
        },

        nodeDaemon: kube.DaemonSet("calico-node") {
            metadata+: {
                namespace: cfg.namespace,
                labels+: {
                    "k8s-app": "calico-node",
                },
            },
            spec+: {
                template+: {
                    spec+: {
                        nodeSelector: {
                            "kubernetes.io/os": "linux"
                        },
                        hostNetwork: true,
                        tolerations: [
                            { effect: "NoSchedule", operator: "Exists" },
                            { key: "CriticalAddonsOnly", operator: "Exists" },
                            { effect: "NoExecute", operator: "Exists" },
                        ],
                        serviceAccountName: env.saNode.metadata.name,
                        terminationGracePeriodSeconds: 0,
                        priorityClassName: "system-cluster-critical",
                        volumes_: {
                            lib_modules: kube.HostPathVolume("/run/current-system/kernel-modules/lib/modules"),
                            var_run_calico: kube.HostPathVolume("/var/run/calico"),
                            var_lib_calico: kube.HostPathVolume("/var/lib/calico"),
                            xtables_lock: kube.HostPathVolume("/run/xtables.lock"),
                            cni_bin: kube.HostPathVolume("/opt/cni/bin"),
                            cni_config: kube.HostPathVolume("/opt/cni/conf"),
                            secrets: kube.SecretVolume(env.secrets),
                            bird_cfg_template: kube.ConfigMapVolume(env.calicoMetallbBird),
                            # TODO flexvol-driver-host, policysync
                        },
                        initContainers_: {
                            installCNI: kube.Container("install-cni") {
                                image: cfg.imageCNI,
                                command: ["/install-cni.sh"],
                                env_: {
                                    CNI_CONF_NAME: "10-calico.conflist",
                                    CNI_NETWORK_CONFIG: kube.ConfigMapRef(env.cm, "cni_network_config"),
                                    ETCD_ENDPOINTS: kube.ConfigMapRef(env.cm, "etcd_endpoints"),
                                    CNI_MTU: kube.ConfigMapRef(env.cm, "veth_mtu"),
                                    # Important: our directory is changed from the default (/etc/cni/net.d)
                                    # to inside /opt/ above in the cni_config HostPathVolume.
                                    # See projectcalico/cni-plugin//k8s-install/scripts/install-cni.sh:24 for reference.
                                    CNI_NET_DIR: "/opt/cni/conf",
                                    # TODO(implr) needed?
                                    CNI_CONF_ETCD_CA_CERT_FILE: kube.ConfigMapRef(env.cm, "etcd_ca"),
                                    CNI_CONF_ETCD_KEY_FILE: kube.ConfigMapRef(env.cm, "etcd_key"),
                                    CNI_CONF_ETCD_CERT_FILE: kube.ConfigMapRef(env.cm, "etcd_cert"),
                                    SLEEP: "false",
                                    KUBERNETES_NODE_NAME: { fieldRef: { fieldPath: "spec.nodeName" } },
                                },
                                volumeMounts_: {
                                    cni_bin: { mountPath: "/host/opt/cni/bin" },
                                    cni_config: { mountPath: "/host/etc/cni/net.d" },
                                    secrets: { mountPath: env.cm.secretPrefix },
                                },
                                securityContext: {
                                    privileged: true,
                                },
                            },
                        },
                        containers_: {
                            calicoNode: kube.Container("calico-node") {
                                image: cfg.imageNode,
                                env_: {
                                    DATASTORE_TYPE: "etcdv3",
                                    ETCD_ENDPOINTS: kube.ConfigMapRef(env.cm, "etcd_endpoints"),
                                    ETCD_CA_CERT_FILE: kube.ConfigMapRef(env.cm, "etcd_ca"),
                                    ETCD_KEY_FILE: kube.ConfigMapRef(env.cm, "etcd_key"),
                                    ETCD_CERT_FILE: kube.ConfigMapRef(env.cm, "etcd_cert"),
                                    CALICO_K8S_NODE_REF: kube.FieldRef("spec.nodeName"),
                                    CALICO_NETWORKING_BACKEND: kube.ConfigMapRef(env.cm, "calico_backend"),
                                    CLUSTER_TYPE: "k8s,bgp",
                                    IP: "autodetect",
                                    IP_AUTODETECTION_METHOD: "can-reach=185.236.240.1",
                                    CALICO_IPV4POOL_IPIP: "Always",
                                    FELIX_IPINIPMTU: kube.ConfigMapRef(env.cm, "veth_mtu"),
                                    FELIX_WIREGUARDMTU: kube.ConfigMapRef(env.cm, "veth_mtu"),
                                    CALICO_IPV4POOL_CIDR: "10.10.24.0/21",
                                    CALICO_DISABLE_FILE_LOGGING: "true",
                                    FELIX_DEFAULTENDPOINTTOHOSTACTION: "ACCEPT",
                                    FELIX_LOGSEVERITYSCREEN: "info",
                                    FELIX_IPV6SUPPORT: "false",
                                    FELIX_HEALTHENABLED: "true",
                                    FELIX_HEALTHHOST: "127.0.0.1",
                                    CALICO_ADVERTISE_CLUSTER_IPS: "10.10.12.0/24",
                                    KUBERNETES_NODE_NAME: { fieldRef: { fieldPath: "spec.nodeName" } },
                                },
                                securityContext: {
                                    privileged: true,
                                },
                                resources: {
                                    requests: { cpu: "250m" },
                                },
                                livenessProbe: {
                                    exec: {
                                        command: ["/bin/calico-node", "-bird-live", "-felix-live"],
                                    },
                                    periodSeconds: 10,
                                    initialDelaySeconds: 10,
                                    failureThreshold: 6,
                                },
                                readinessProbe: {
                                    exec: {
                                        command: ["/bin/calico-node", "-bird-ready", "-felix-ready"],
                                    },
                                    periodSeconds: 10,
                                },
                                volumeMounts_: {
                                    lib_modules: { mountPath: "/lib/modules" },
                                    xtables_lock: { mountPath: "/run/xtables.lock" },
                                    var_run_calico: { mountPath: "/var/run/calico" },
                                    var_lib_calico: { mountPath: "/var/lib/calico" },
                                    secrets: { mountPath: env.cm.secretPrefix },
                                },
                                volumeMounts+: [
                                    { name: "bird-cfg-template",
                                      mountPath: "/etc/calico/confd/templates/bird.cfg.template",
                                      subPath: "bird.cfg.template"
                                    },
                                    { name: "bird-cfg-template",
                                      mountPath: "/etc/calico/confd/templates/bird_ipam.cfg.template",
                                      subPath: "bird_ipam.cfg.template"
                                    },
                                ],
                            },
                        },
                    },
                },
            },
        },
    },
}