Sergiusz Bazanski | af3be42 | 2019-01-17 18:57:19 +0100 | [diff] [blame] | 1 | # Deploy hosted calico with its own etcd. |
| 2 | |
| 3 | local kube = import "../../../kube/kube.libsonnet"; |
| 4 | |
| 5 | local bindServiceAccountClusterRole(sa, cr) = kube.ClusterRoleBinding(cr.metadata.name) { |
| 6 | roleRef: { |
| 7 | apiGroup: "rbac.authorization.k8s.io", |
| 8 | kind: "ClusterRole", |
| 9 | name: cr.metadata.name, |
| 10 | }, |
| 11 | subjects: [ |
| 12 | { |
| 13 | kind: "ServiceAccount", |
| 14 | name: sa.metadata.name, |
| 15 | namespace: sa.metadata.namespace, |
| 16 | }, |
| 17 | ], |
| 18 | }; |
| 19 | |
| 20 | { |
| 21 | Environment: { |
| 22 | local env = self, |
| 23 | local cfg = env.cfg, |
| 24 | cfg:: { |
| 25 | namespace: "kube-system", |
Sergiusz Bazanski | d81bf72 | 2020-05-28 16:38:52 +0200 | [diff] [blame] | 26 | version: "v3.14.0", |
| 27 | imageController: "calico/kube-controllers:" + cfg.version, |
| 28 | imageCNI: "calico/cni:" + cfg.version, |
| 29 | imageNode: "calico/node:" + cfg.version, |
Sergiusz Bazanski | af3be42 | 2019-01-17 18:57:19 +0100 | [diff] [blame] | 30 | // TODO(q3k): Separate etcd for calico |
| 31 | etcd: { |
| 32 | endpoints: ["https://bc01n%02d.hswaw.net:2379" % n for n in std.range(1, 3)], |
Sergiusz Bazanski | 73cef11 | 2019-04-07 00:06:23 +0200 | [diff] [blame] | 33 | ca: importstr "../../certs/ca-etcd.crt", |
| 34 | cert: importstr "../../certs/etcd-calico.cert", |
| 35 | key: importstr "../../secrets/plain/etcd-calico.key", |
Sergiusz Bazanski | af3be42 | 2019-01-17 18:57:19 +0100 | [diff] [blame] | 36 | }, |
| 37 | }, |
| 38 | |
| 39 | cm: kube.ConfigMap("calico-config") { |
| 40 | local cm = self, |
| 41 | secretPrefix:: "/calico-secrets/", |
| 42 | |
| 43 | metadata+: { |
| 44 | namespace: cfg.namespace, |
| 45 | }, |
| 46 | |
| 47 | data: { |
| 48 | etcd_endpoints: std.join(",", cfg.etcd.endpoints), |
| 49 | |
| 50 | etcd_ca: cm.secretPrefix + "etcd-ca", |
| 51 | etcd_cert: cm.secretPrefix + "etcd-cert", |
| 52 | etcd_key: cm.secretPrefix + "etcd-key", |
| 53 | |
| 54 | calico_backend: "bird", |
| 55 | veth_mtu: "1440", |
| 56 | |
Sergiusz Bazanski | d81bf72 | 2020-05-28 16:38:52 +0200 | [diff] [blame] | 57 | typha_service_name: "none", |
| 58 | |
Sergiusz Bazanski | af3be42 | 2019-01-17 18:57:19 +0100 | [diff] [blame] | 59 | cni_network_config: ||| |
| 60 | { |
| 61 | "name": "k8s-pod-network", |
Sergiusz Bazanski | d81bf72 | 2020-05-28 16:38:52 +0200 | [diff] [blame] | 62 | "cniVersion": "0.3.1", |
Sergiusz Bazanski | af3be42 | 2019-01-17 18:57:19 +0100 | [diff] [blame] | 63 | "plugins": [ |
| 64 | { |
| 65 | "type": "calico", |
| 66 | "log_level": "info", |
| 67 | "etcd_endpoints": "__ETCD_ENDPOINTS__", |
| 68 | "etcd_key_file": "__ETCD_KEY_FILE__", |
| 69 | "etcd_cert_file": "__ETCD_CERT_FILE__", |
| 70 | "etcd_ca_cert_file": "__ETCD_CA_CERT_FILE__", |
Sergiusz Bazanski | d81bf72 | 2020-05-28 16:38:52 +0200 | [diff] [blame] | 71 | "datastore_type": "etcdv3", |
Sergiusz Bazanski | af3be42 | 2019-01-17 18:57:19 +0100 | [diff] [blame] | 72 | "mtu": __CNI_MTU__, |
| 73 | "ipam": { |
| 74 | "type": "calico-ipam" |
| 75 | }, |
| 76 | "policy": { |
| 77 | "type": "k8s" |
| 78 | }, |
| 79 | "kubernetes": { |
| 80 | "kubeconfig": "__KUBECONFIG_FILEPATH__" |
| 81 | } |
| 82 | }, |
| 83 | { |
| 84 | "type": "portmap", |
| 85 | "snat": true, |
| 86 | "capabilities": {"portMappings": true} |
Sergiusz Bazanski | d81bf72 | 2020-05-28 16:38:52 +0200 | [diff] [blame] | 87 | }, |
| 88 | { |
| 89 | "type": "bandwidth", |
| 90 | "capabilities": {"bandwidth": true} |
Sergiusz Bazanski | af3be42 | 2019-01-17 18:57:19 +0100 | [diff] [blame] | 91 | } |
| 92 | ] |
| 93 | } |
| 94 | ||| |
| 95 | }, |
| 96 | }, |
| 97 | |
| 98 | secrets: kube.Secret("calico-secrets") { |
| 99 | metadata+: { |
| 100 | namespace: cfg.namespace, |
| 101 | }, |
| 102 | |
| 103 | data_: { |
| 104 | "etcd-ca": cfg.etcd.ca, |
| 105 | "etcd-cert": cfg.etcd.cert, |
| 106 | "etcd-key": cfg.etcd.key, |
| 107 | }, |
| 108 | }, |
| 109 | |
| 110 | saNode: kube.ServiceAccount("calico-node") { |
| 111 | metadata+: { |
| 112 | namespace: cfg.namespace, |
| 113 | }, |
| 114 | }, |
| 115 | |
| 116 | crNode: kube.ClusterRole("calico-node") { |
| 117 | rules: [ |
| 118 | { |
| 119 | apiGroups: [""], |
| 120 | resources: ["pods", "nodes", "namespaces"], |
| 121 | verbs: ["get"], |
| 122 | }, |
| 123 | { |
| 124 | apiGroups: [""], |
| 125 | resources: ["endpoints", "services"], |
Sergiusz Bazanski | d81bf72 | 2020-05-28 16:38:52 +0200 | [diff] [blame] | 126 | verbs: ["watch", "list", "get"], |
| 127 | }, |
| 128 | { |
| 129 | apiGroups: [""], |
| 130 | resources: ["configmaps"], |
| 131 | verbs: ["get"], |
Sergiusz Bazanski | af3be42 | 2019-01-17 18:57:19 +0100 | [diff] [blame] | 132 | }, |
| 133 | { |
| 134 | apiGroups: [""], |
| 135 | resources: ["nodes/status"], |
Sergiusz Bazanski | d81bf72 | 2020-05-28 16:38:52 +0200 | [diff] [blame] | 136 | verbs: ["patch", "update"], |
| 137 | }, |
| 138 | { |
| 139 | apiGroups: ["networking.k8s.io"], |
| 140 | resources: ["networkpolicies"], |
| 141 | verbs: ["watch", "list"], |
| 142 | }, |
| 143 | { |
| 144 | apiGroups: [""], |
| 145 | resources: ["pods", "namespaces", "serviceaccounts"], |
| 146 | verbs: ["watch", "list"], |
| 147 | }, |
| 148 | { |
| 149 | apiGroups: [""], |
| 150 | resources: ["pods/status"], |
Sergiusz Bazanski | af3be42 | 2019-01-17 18:57:19 +0100 | [diff] [blame] | 151 | verbs: ["patch"], |
| 152 | }, |
Sergiusz Bazanski | d81bf72 | 2020-05-28 16:38:52 +0200 | [diff] [blame] | 153 | { |
| 154 | apiGroups: [""], |
| 155 | resources: ["nodes"], |
| 156 | verbs: ["get", "list", "watch"], |
| 157 | }, |
Sergiusz Bazanski | af3be42 | 2019-01-17 18:57:19 +0100 | [diff] [blame] | 158 | ], |
| 159 | }, |
| 160 | |
| 161 | crbNode: bindServiceAccountClusterRole(env.saNode, env.crNode), |
| 162 | |
| 163 | saController: kube.ServiceAccount("calico-kube-controllers") { |
| 164 | metadata+: { |
| 165 | namespace: cfg.namespace, |
| 166 | }, |
| 167 | }, |
| 168 | |
| 169 | crController: kube.ClusterRole("calico-kube-controllers") { |
| 170 | rules: [ |
| 171 | { |
| 172 | apiGroups: [""], |
Sergiusz Bazanski | e55493f | 2020-05-30 17:57:05 +0200 | [diff] [blame] | 173 | resources: ["nodes", "pods", "namespaces", "serviceaccounts"], |
Sergiusz Bazanski | d81bf72 | 2020-05-28 16:38:52 +0200 | [diff] [blame] | 174 | verbs: ["watch", "list", "get"], |
| 175 | }, |
| 176 | { |
Sergiusz Bazanski | af3be42 | 2019-01-17 18:57:19 +0100 | [diff] [blame] | 177 | apiGroups: ["networking.k8s.io"], |
| 178 | resources: ["networkpolicies"], |
| 179 | verbs: ["watch", "list"], |
| 180 | }, |
| 181 | ], |
| 182 | }, |
| 183 | |
| 184 | crbController: bindServiceAccountClusterRole(env.saController, env.crController), |
| 185 | |
| 186 | controller: kube.Deployment("calico-kube-controllers") { |
| 187 | metadata+: { |
| 188 | namespace: cfg.namespace, |
| 189 | annotations+: { |
| 190 | "scheduler.alpha.kubernetes.io/critical-pod": "", |
| 191 | }, |
| 192 | }, |
| 193 | spec+: { |
| 194 | replicas: 1, |
| 195 | strategy: { type: "Recreate" }, |
| 196 | template+: { |
| 197 | spec+: { |
| 198 | hostNetwork: true, |
| 199 | tolerations: [ |
| 200 | { key: "CriticalAddonsOnly", operator: "Exists" }, |
| 201 | ], |
| 202 | serviceAccountName: env.saController.metadata.name, |
| 203 | volumes_: { |
| 204 | secrets: kube.SecretVolume(env.secrets), |
| 205 | }, |
| 206 | containers_: { |
| 207 | "calico-kube-controllers": kube.Container("calico-kube-controllers") { |
| 208 | image: cfg.imageController, |
| 209 | env_: { |
| 210 | ETCD_ENDPOINTS: kube.ConfigMapRef(env.cm, "etcd_endpoints"), |
| 211 | ETCD_CA_CERT_FILE: kube.ConfigMapRef(env.cm, "etcd_ca"), |
| 212 | ETCD_KEY_FILE: kube.ConfigMapRef(env.cm, "etcd_key"), |
| 213 | ETCD_CERT_FILE: kube.ConfigMapRef(env.cm, "etcd_cert"), |
| 214 | ENABLED_CONTROLLERS: "policy,namespace,serviceaccount,workloadendpoint,node", |
| 215 | }, |
| 216 | volumeMounts_: { |
| 217 | secrets: { |
| 218 | mountPath: env.cm.secretPrefix, |
| 219 | }, |
| 220 | }, |
| 221 | readinessProbe: { |
| 222 | exec: { |
| 223 | command: [ "/usr/bin/check-status", "-r" ], |
| 224 | }, |
| 225 | }, |
| 226 | }, |
| 227 | }, |
| 228 | }, |
| 229 | }, |
| 230 | }, |
| 231 | }, |
| 232 | |
Serge Bazanski | a5ed644 | 2020-09-20 22:52:57 +0000 | [diff] [blame] | 233 | # ConfigMap that holds overriden bird.cfg.template and bird_ipam.cfg.template. |
| 234 | calicoMetallbBird: kube.ConfigMap("calico-metallb-bird") { |
| 235 | metadata+: { |
| 236 | namespace: cfg.namespace, |
| 237 | }, |
| 238 | data: { |
| 239 | "bird.cfg.template": (importstr "calico-bird.cfg.template"), |
| 240 | "bird_ipam.cfg.template": (importstr "calico-bird-ipam.cfg.template"), |
| 241 | }, |
| 242 | }, |
| 243 | |
Sergiusz Bazanski | af3be42 | 2019-01-17 18:57:19 +0100 | [diff] [blame] | 244 | nodeDaemon: kube.DaemonSet("calico-node") { |
| 245 | metadata+: { |
| 246 | namespace: cfg.namespace, |
| 247 | }, |
| 248 | spec+: { |
| 249 | template+: { |
| 250 | metadata+: { |
| 251 | annotations+: { |
| 252 | "scheduler.alpha.kubernetes.io/critical-pod": "", |
| 253 | }, |
| 254 | }, |
| 255 | spec+: { |
| 256 | hostNetwork: true, |
| 257 | tolerations: [ |
| 258 | { key: "CriticalAddonsOnly", operator: "Exists" }, |
Serge Bazanski | d493ab6 | 2019-10-31 17:07:19 +0100 | [diff] [blame] | 259 | { effect: "NoExecute", operator: "Exists" }, |
| 260 | { effect: "NoSchedule", operator: "Exists" }, |
Sergiusz Bazanski | af3be42 | 2019-01-17 18:57:19 +0100 | [diff] [blame] | 261 | ], |
| 262 | serviceAccountName: env.saNode.metadata.name, |
| 263 | terminationGracePeriodSeconds: 0, |
| 264 | volumes_: { |
| 265 | cni_bin: kube.HostPathVolume("/opt/cni/bin"), |
| 266 | cni_config: kube.HostPathVolume("/opt/cni/conf"), |
| 267 | secrets: kube.SecretVolume(env.secrets), |
| 268 | lib_modules: kube.HostPathVolume("/run/current-system/kernel-modules/lib/modules"), |
| 269 | xtables_lock: kube.HostPathVolume("/run/xtables.lock"), |
| 270 | var_run_calico: kube.HostPathVolume("/var/run/calico"), |
| 271 | var_lib_calico: kube.HostPathVolume("/var/lib/calico"), |
Serge Bazanski | a5ed644 | 2020-09-20 22:52:57 +0000 | [diff] [blame] | 272 | bird_cfg_template: kube.ConfigMapVolume(env.calicoMetallbBird), |
Sergiusz Bazanski | af3be42 | 2019-01-17 18:57:19 +0100 | [diff] [blame] | 273 | }, |
| 274 | initContainers_: { |
| 275 | installCNI: kube.Container("install-cni") { |
| 276 | image: cfg.imageCNI, |
| 277 | command: ["/install-cni.sh"], |
| 278 | env_: { |
| 279 | ETCD_ENDPOINTS: kube.ConfigMapRef(env.cm, "etcd_endpoints"), |
| 280 | CNI_CONF_NAME: "10-calico.conflist", |
| 281 | CNI_NETWORK_CONFIG: kube.ConfigMapRef(env.cm, "cni_network_config"), |
| 282 | CNI_CONF_ETCD_CA_CERT_FILE: kube.ConfigMapRef(env.cm, "etcd_ca"), |
| 283 | CNI_CONF_ETCD_KEY_FILE: kube.ConfigMapRef(env.cm, "etcd_key"), |
| 284 | CNI_CONF_ETCD_CERT_FILE: kube.ConfigMapRef(env.cm, "etcd_cert"), |
| 285 | CNI_MTU: kube.ConfigMapRef(env.cm, "veth_mtu"), |
| 286 | CNI_NET_DIR: "/opt/cni/conf", |
| 287 | SLEEP: "false", |
Sergiusz Bazanski | d81bf72 | 2020-05-28 16:38:52 +0200 | [diff] [blame] | 288 | KUBERNETES_NODE_NAME: { fieldRef: { fieldPath: "spec.nodeName" } }, |
Sergiusz Bazanski | af3be42 | 2019-01-17 18:57:19 +0100 | [diff] [blame] | 289 | }, |
| 290 | volumeMounts_: { |
| 291 | cni_bin: { mountPath: "/host/opt/cni/bin" }, |
| 292 | cni_config: { mountPath: "/host/etc/cni/net.d" }, |
| 293 | secrets: { mountPath: env.cm.secretPrefix }, |
| 294 | }, |
| 295 | }, |
| 296 | }, |
| 297 | containers_: { |
| 298 | calicoNode: kube.Container("calico-node") { |
| 299 | image: cfg.imageNode, |
| 300 | env_: { |
Sergiusz Bazanski | d81bf72 | 2020-05-28 16:38:52 +0200 | [diff] [blame] | 301 | DATASTORE_TYPE: "etcdv3", |
Sergiusz Bazanski | af3be42 | 2019-01-17 18:57:19 +0100 | [diff] [blame] | 302 | ETCD_ENDPOINTS: kube.ConfigMapRef(env.cm, "etcd_endpoints"), |
| 303 | ETCD_CA_CERT_FILE: kube.ConfigMapRef(env.cm, "etcd_ca"), |
| 304 | ETCD_KEY_FILE: kube.ConfigMapRef(env.cm, "etcd_key"), |
| 305 | ETCD_CERT_FILE: kube.ConfigMapRef(env.cm, "etcd_cert"), |
| 306 | CALICO_K8S_NODE_REF: kube.FieldRef("spec.nodeName"), |
Sergiusz Bazanski | d81bf72 | 2020-05-28 16:38:52 +0200 | [diff] [blame] | 307 | CALICO_NETWORKING_BACKEND: kube.ConfigMapRef(env.cm, "calico_backend"), |
Sergiusz Bazanski | af3be42 | 2019-01-17 18:57:19 +0100 | [diff] [blame] | 308 | CLUSTER_TYPE: "k8s,bgp", |
Sergiusz Bazanski | e3af1eb | 2019-01-18 09:39:57 +0100 | [diff] [blame] | 309 | IP: "autodetect", |
Serge Bazanski | d493ab6 | 2019-10-31 17:07:19 +0100 | [diff] [blame] | 310 | IP_AUTODETECTION_METHOD: "can-reach=185.236.240.1", |
Sergiusz Bazanski | af3be42 | 2019-01-17 18:57:19 +0100 | [diff] [blame] | 311 | CALICO_IPV4POOL_IPIP: "Always", |
| 312 | FELIX_IPINIPMTU: kube.ConfigMapRef(env.cm, "veth_mtu"), |
| 313 | CALICO_IPV4POOL_CIDR: "10.10.24.0/21", |
| 314 | CALICO_DISABLE_FILE_LOGGING: "true", |
| 315 | FELIX_DEFAULTENDPOINTTOHOSTACTION: "ACCEPT", |
| 316 | FELIX_IPV6SUPPORT: "false", |
| 317 | FELIX_LOGSEVERITYSCREEN: "info", |
| 318 | FELIX_HEALTHENABLED: "true", |
Serge Bazanski | d493ab6 | 2019-10-31 17:07:19 +0100 | [diff] [blame] | 319 | FELIX_HEALTHHOST: "127.0.0.1", |
Sergiusz Bazanski | af3be42 | 2019-01-17 18:57:19 +0100 | [diff] [blame] | 320 | CALICO_ADVERTISE_CLUSTER_IPS: "10.10.12.0/24", |
Sergiusz Bazanski | d81bf72 | 2020-05-28 16:38:52 +0200 | [diff] [blame] | 321 | KUBERNETES_NODE_NAME: { fieldRef: { fieldPath: "spec.nodeName" } }, |
Sergiusz Bazanski | af3be42 | 2019-01-17 18:57:19 +0100 | [diff] [blame] | 322 | }, |
| 323 | securityContext: { |
| 324 | privileged: true, |
| 325 | }, |
| 326 | resources: { |
| 327 | requests: { cpu: "250m" }, |
| 328 | }, |
| 329 | livenessProbe: { |
Sergiusz Bazanski | d81bf72 | 2020-05-28 16:38:52 +0200 | [diff] [blame] | 330 | exec: { |
| 331 | command: ["/bin/calico-node", "-bird-live", "-felix-live"], |
Sergiusz Bazanski | af3be42 | 2019-01-17 18:57:19 +0100 | [diff] [blame] | 332 | }, |
| 333 | periodSeconds: 10, |
| 334 | initialDelaySeconds: 10, |
| 335 | failureThreshold: 6, |
| 336 | }, |
| 337 | readinessProbe: { |
| 338 | exec: { |
| 339 | command: ["/bin/calico-node", "-bird-ready", "-felix-ready"], |
| 340 | }, |
| 341 | periodSeconds: 10, |
| 342 | }, |
| 343 | volumeMounts_: { |
| 344 | lib_modules: { mountPath: "/lib/modules" }, |
| 345 | xtables_lock: { mountPath: "/run/xtables.lock" }, |
| 346 | var_run_calico: { mountPath: "/var/run/calico" }, |
| 347 | var_lib_calico: { mountPath: "/var/lib/calico" }, |
| 348 | secrets: { mountPath: env.cm.secretPrefix }, |
| 349 | }, |
Serge Bazanski | a5ed644 | 2020-09-20 22:52:57 +0000 | [diff] [blame] | 350 | volumeMounts+: [ |
| 351 | { name: "bird-cfg-template", |
| 352 | mountPath: "/etc/calico/confd/templates/bird.cfg.template", |
| 353 | subPath: "bird.cfg.template" |
| 354 | }, |
| 355 | { name: "bird-cfg-template", |
| 356 | mountPath: "/etc/calico/confd/templates/bird_ipam.cfg.template", |
| 357 | subPath: "bird_ipam.cfg.template" |
| 358 | }, |
| 359 | ], |
Sergiusz Bazanski | af3be42 | 2019-01-17 18:57:19 +0100 | [diff] [blame] | 360 | }, |
| 361 | }, |
| 362 | }, |
| 363 | }, |
| 364 | }, |
| 365 | }, |
| 366 | }, |
| 367 | } |