| # Deploy hosted calico with its own etcd. |
| |
| local kube = import "../../../kube/kube.libsonnet"; |
| local crdYaml = (std.native("parseYaml"))(importstr "./calico_crd.yml"); |
| local crdMap = { |
| [x.metadata.name]: x |
| for x in crdYaml if x != null |
| }; |
| |
| local bindServiceAccountClusterRole(sa, cr) = kube.ClusterRoleBinding(cr.metadata.name) { |
| roleRef: { |
| apiGroup: "rbac.authorization.k8s.io", |
| kind: "ClusterRole", |
| name: cr.metadata.name, |
| }, |
| subjects: [ |
| { |
| kind: "ServiceAccount", |
| name: sa.metadata.name, |
| namespace: sa.metadata.namespace, |
| }, |
| ], |
| }; |
| |
| { |
| Environment: { |
| local env = self, |
| local cfg = env.cfg, |
| cfg:: { |
| namespace: "kube-system", |
| version: "v3.15.5", |
| imageController: "calico/kube-controllers:" + cfg.version, |
| imageCNI: "calico/cni:" + cfg.version, |
| imageNode: "calico/node:" + cfg.version, |
| }, |
| |
| crds: crdMap, |
| |
| cm: kube.ConfigMap("calico-config") { |
| local cm = self, |
| secretPrefix:: "/calico-secrets/", |
| |
| metadata+: { |
| namespace: cfg.namespace, |
| }, |
| |
| data: { |
| calico_backend: "bird", |
| veth_mtu: "1440", |
| |
| typha_service_name: "none", |
| |
| cni_network_config: ||| |
| { |
| "name": "k8s-pod-network", |
| "cniVersion": "0.3.1", |
| "plugins": [ |
| { |
| "type": "calico", |
| "log_level": "info", |
| "datastore_type": "kubernetes", |
| "nodename": "__KUBERNETES_NODE_NAME__", |
| "mtu": __CNI_MTU__, |
| "ipam": { |
| "type": "calico-ipam" |
| }, |
| "policy": { |
| "type": "k8s" |
| }, |
| "kubernetes": { |
| "kubeconfig": "__KUBECONFIG_FILEPATH__" |
| } |
| }, |
| { |
| "type": "portmap", |
| "snat": true, |
| "capabilities": {"portMappings": true} |
| }, |
| { |
| "type": "bandwidth", |
| "capabilities": {"bandwidth": true} |
| } |
| ] |
| } |
| ||| |
| }, |
| }, |
| |
| secrets: kube.Secret("calico-secrets") { |
| metadata+: { |
| namespace: cfg.namespace, |
| }, |
| |
| data_: { |
| }, |
| }, |
| |
| saNode: kube.ServiceAccount("calico-node") { |
| metadata+: { |
| namespace: cfg.namespace, |
| }, |
| }, |
| |
| crNode: kube.ClusterRole("calico-node") { |
| rules: [ |
| { |
| apiGroups: [""], |
| resources: ["pods", "nodes", "namespaces"], |
| verbs: ["get"], |
| }, |
| { |
| apiGroups: [""], |
| resources: ["endpoints", "services"], |
| verbs: ["watch", "list", "get"], |
| }, |
| { |
| apiGroups: [""], |
| resources: ["configmaps"], |
| verbs: ["get"], |
| }, |
| { |
| apiGroups: [""], |
| resources: ["nodes/status"], |
| verbs: ["patch", "update"], |
| }, |
| { |
| apiGroups: ["networking.k8s.io"], |
| resources: ["networkpolicies"], |
| verbs: ["watch", "list"], |
| }, |
| { |
| apiGroups: [""], |
| resources: ["pods", "namespaces", "serviceaccounts"], |
| verbs: ["list", "watch"], |
| }, |
| { |
| apiGroups: [""], |
| resources: ["pods/status"], |
| verbs: ["patch"], |
| }, |
| { |
| apiGroups: ["crd.projectcalico.org"], |
| resources: ["globalfelixconfigs", "felixconfigurations", "bgppeers", "globalbgpconfigs", "bgpconfigurations", "ippools", "ipamblocks", "globalnetworkpolicies", "globalnetworksets", "networkpolicies", "networksets", "clusterinformations", "hostendpoints", "blockaffinities"], |
| verbs: ["get", "list", "watch"], |
| }, |
| { |
| apiGroups: ["crd.projectcalico.org"], |
| resources: ["ippools", "felixconfigurations", "clusterinformations"], |
| verbs: ["create", "update"], |
| }, |
| { |
| apiGroups: [""], |
| resources: ["nodes"], |
| verbs: ["get", "list", "watch"], |
| }, |
| { |
| apiGroups: ["crd.projectcalico.org"], |
| resources: ["blockaffinities", "ipamblocks", "ipamhandles"], |
| verbs: ["get", "list", "create", "update", "delete"], |
| }, |
| { |
| apiGroups: ["crd.projectcalico.org"], |
| resources: ["ipamconfigs"], |
| verbs: ["get"], |
| }, |
| { |
| apiGroups: ["crd.projectcalico.org"], |
| resources: ["blockaffinities"], |
| verbs: ["watch"], |
| }, |
| { |
| apiGroups: ["apps"], |
| resources: ["daemonsets"], |
| verbs: ["get"], |
| }, |
| ], |
| }, |
| |
| crbNode: bindServiceAccountClusterRole(env.saNode, env.crNode), |
| |
| saController: kube.ServiceAccount("calico-kube-controllers") { |
| metadata+: { |
| namespace: cfg.namespace, |
| }, |
| }, |
| |
| crController: kube.ClusterRole("calico-kube-controllers") { |
| rules: [ |
| { |
| apiGroups: [""], |
| resources: ["nodes"], |
| verbs: ["watch", "list", "get"], |
| }, |
| { |
| apiGroups: [""], |
| resources: ["pods"], |
| verbs: ["get"], |
| }, |
| { |
| apiGroups: ["crd.projectcalico.org"], |
| resources: ["ippools"], |
| verbs: ["list"], |
| }, |
| { |
| apiGroups: ["crd.projectcalico.org"], |
| resources: ["blockaffinities", "ipamblocks", "ipamhandles"], |
| verbs: ["get", "list", "create", "update", "delete"], |
| }, |
| { |
| apiGroups: ["crd.projectcalico.org"], |
| resources: ["hostendpoints"], |
| verbs: ["get", "list", "create", "update", "delete"], |
| }, |
| { |
| apiGroups: ["crd.projectcalico.org"], |
| resources: ["clusterinformations"], |
| verbs: ["get", "create", "update"], |
| }, |
| { |
| apiGroups: ["crd.projectcalico.org"], |
| resources: ["kubecontrollersconfigurations"], |
| verbs: ["get", "create", "update", "watch"], |
| }, |
| ], |
| }, |
| |
| crbController: bindServiceAccountClusterRole(env.saController, env.crController), |
| |
| controller: kube.Deployment("calico-kube-controllers") { |
| metadata+: { |
| namespace: cfg.namespace, |
| labels+: { |
| "k8s-app": "calico-kube-controllers", |
| }, |
| }, |
| spec+: { |
| replicas: 1, |
| strategy: { type: "Recreate" }, |
| template+: { |
| spec+: { |
| nodeSelector: { |
| "kubernetes.io/os": "linux" |
| }, |
| tolerations: [ |
| { key: "CriticalAddonsOnly", operator: "Exists" }, |
| { key: "node-role.kubernetes.io/master", effect: "NoSchedule" }, |
| ], |
| serviceAccountName: env.saController.metadata.name, |
| priorityClassName: "system-cluster-critical", |
| containers_: { |
| "calico-kube-controllers": kube.Container("calico-kube-controllers") { |
| image: cfg.imageController, |
| env_: { |
| DATASTORE_TYPE: "kubernetes", |
| ENABLED_CONTROLLERS: "node", |
| }, |
| volumeMounts_: { |
| secrets: { |
| mountPath: env.cm.secretPrefix, |
| }, |
| }, |
| readinessProbe: { |
| exec: { |
| command: [ "/usr/bin/check-status", "-r" ], |
| }, |
| }, |
| }, |
| }, |
| volumes_: { |
| secrets: kube.SecretVolume(env.secrets), |
| }, |
| }, |
| }, |
| }, |
| }, |
| |
| # ConfigMap that holds overriden bird.cfg.template and bird_ipam.cfg.template. |
| calicoMetallbBird: kube.ConfigMap("calico-metallb-bird") { |
| metadata+: { |
| namespace: cfg.namespace, |
| }, |
| data: { |
| "bird.cfg.template": (importstr "calico-bird.cfg.template"), |
| "bird_ipam.cfg.template": (importstr "calico-bird-ipam.cfg.template"), |
| }, |
| }, |
| |
| nodeDaemon: kube.DaemonSet("calico-node") { |
| metadata+: { |
| namespace: cfg.namespace, |
| labels+: { |
| "k8s-app": "calico-node", |
| }, |
| }, |
| spec+: { |
| template+: { |
| spec+: { |
| nodeSelector: { |
| "kubernetes.io/os": "linux" |
| }, |
| hostNetwork: true, |
| tolerations: [ |
| { effect: "NoSchedule", operator: "Exists" }, |
| { key: "CriticalAddonsOnly", operator: "Exists" }, |
| { effect: "NoExecute", operator: "Exists" }, |
| ], |
| serviceAccountName: env.saNode.metadata.name, |
| terminationGracePeriodSeconds: 0, |
| priorityClassName: "system-cluster-critical", |
| volumes_: { |
| lib_modules: kube.HostPathVolume("/run/current-system/kernel-modules/lib/modules"), |
| var_run_calico: kube.HostPathVolume("/var/run/calico"), |
| var_lib_calico: kube.HostPathVolume("/var/lib/calico"), |
| xtables_lock: kube.HostPathVolume("/run/xtables.lock"), |
| cni_bin: kube.HostPathVolume("/opt/cni/bin"), |
| cni_config: kube.HostPathVolume("/opt/cni/conf"), |
| secrets: kube.SecretVolume(env.secrets), |
| bird_cfg_template: kube.ConfigMapVolume(env.calicoMetallbBird), |
| # TODO flexvol-driver-host, policysync |
| }, |
| initContainers_: { |
| installCNI: kube.Container("install-cni") { |
| image: cfg.imageCNI, |
| command: ["/install-cni.sh"], |
| env_: { |
| CNI_CONF_NAME: "10-calico.conflist", |
| CNI_NETWORK_CONFIG: kube.ConfigMapRef(env.cm, "cni_network_config"), |
| CNI_MTU: kube.ConfigMapRef(env.cm, "veth_mtu"), |
| # Important: our directory is changed from the default (/etc/cni/net.d) |
| # to inside /opt/ above in the cni_config HostPathVolume. |
| # See projectcalico/cni-plugin//k8s-install/scripts/install-cni.sh:24 for reference. |
| CNI_NET_DIR: "/opt/cni/conf", |
| SLEEP: "false", |
| KUBERNETES_NODE_NAME: { fieldRef: { fieldPath: "spec.nodeName" } }, |
| }, |
| volumeMounts_: { |
| cni_bin: { mountPath: "/host/opt/cni/bin" }, |
| cni_config: { mountPath: "/host/etc/cni/net.d" }, |
| secrets: { mountPath: env.cm.secretPrefix }, |
| }, |
| securityContext: { |
| privileged: true, |
| }, |
| }, |
| }, |
| containers_: { |
| calicoNode: kube.Container("calico-node") { |
| image: cfg.imageNode, |
| env_: { |
| WAIT_FOR_DATASTORE: "true", |
| NODENAME: kube.FieldRef("spec.nodeName"), |
| DATASTORE_TYPE: "kubernetes", |
| CALICO_NETWORKING_BACKEND: kube.ConfigMapRef(env.cm, "calico_backend"), |
| CLUSTER_TYPE: "k8s,bgp", |
| IP: "autodetect", |
| IP_AUTODETECTION_METHOD: "can-reach=185.236.240.1", |
| CALICO_IPV4POOL_IPIP: "Always", |
| FELIX_IPINIPMTU: kube.ConfigMapRef(env.cm, "veth_mtu"), |
| FELIX_WIREGUARDMTU: kube.ConfigMapRef(env.cm, "veth_mtu"), |
| CALICO_IPV4POOL_CIDR: "10.10.24.0/21", |
| CALICO_DISABLE_FILE_LOGGING: "true", |
| FELIX_DEFAULTENDPOINTTOHOSTACTION: "ACCEPT", |
| FELIX_LOGSEVERITYSCREEN: "info", |
| FELIX_IPV6SUPPORT: "false", |
| FELIX_HEALTHENABLED: "true", |
| FELIX_HEALTHHOST: "127.0.0.1", |
| CALICO_ADVERTISE_CLUSTER_IPS: "10.10.12.0/24", |
| KUBERNETES_NODE_NAME: { fieldRef: { fieldPath: "spec.nodeName" } }, |
| }, |
| securityContext: { |
| privileged: true, |
| }, |
| resources: { |
| requests: { cpu: "250m" }, |
| }, |
| livenessProbe: { |
| exec: { |
| command: ["/bin/calico-node", "-bird-live", "-felix-live"], |
| }, |
| periodSeconds: 10, |
| initialDelaySeconds: 10, |
| failureThreshold: 6, |
| }, |
| readinessProbe: { |
| exec: { |
| command: ["/bin/calico-node", "-bird-ready", "-felix-ready"], |
| }, |
| periodSeconds: 10, |
| }, |
| volumeMounts_: { |
| lib_modules: { mountPath: "/lib/modules" }, |
| xtables_lock: { mountPath: "/run/xtables.lock" }, |
| var_run_calico: { mountPath: "/var/run/calico" }, |
| var_lib_calico: { mountPath: "/var/lib/calico" }, |
| secrets: { mountPath: env.cm.secretPrefix }, |
| }, |
| volumeMounts+: [ |
| { name: "bird-cfg-template", |
| mountPath: "/etc/calico/confd/templates/bird.cfg.template", |
| subPath: "bird.cfg.template" |
| }, |
| { name: "bird-cfg-template", |
| mountPath: "/etc/calico/confd/templates/bird_ipam.cfg.template", |
| subPath: "bird_ipam.cfg.template" |
| }, |
| ], |
| }, |
| }, |
| }, |
| }, |
| }, |
| }, |
| }, |
| } |