blob: 43b56161f14267ac6b183f5c8c3f913c81624fb3 [file] [log] [blame]
# Deploy hosted calico with its own etcd.
local kube = import "../../../kube/kube.libsonnet";
local crdYaml = (std.native("parseYaml"))(importstr "./calico_crd.yml");
local crdMap = {
[x.metadata.name]: x
for x in crdYaml if x != null
};
local bindServiceAccountClusterRole(sa, cr) = kube.ClusterRoleBinding(cr.metadata.name) {
roleRef: {
apiGroup: "rbac.authorization.k8s.io",
kind: "ClusterRole",
name: cr.metadata.name,
},
subjects: [
{
kind: "ServiceAccount",
name: sa.metadata.name,
namespace: sa.metadata.namespace,
},
],
};
{
Environment: {
local env = self,
local cfg = env.cfg,
cfg:: {
namespace: "kube-system",
version: "v3.15.5",
imageController: "calico/kube-controllers:" + cfg.version,
imageCNI: "calico/cni:" + cfg.version,
imageNode: "calico/node:" + cfg.version,
},
crds: crdMap,
cm: kube.ConfigMap("calico-config") {
local cm = self,
secretPrefix:: "/calico-secrets/",
metadata+: {
namespace: cfg.namespace,
},
data: {
calico_backend: "bird",
veth_mtu: "1440",
typha_service_name: "none",
cni_network_config: |||
{
"name": "k8s-pod-network",
"cniVersion": "0.3.1",
"plugins": [
{
"type": "calico",
"log_level": "info",
"datastore_type": "kubernetes",
"nodename": "__KUBERNETES_NODE_NAME__",
"mtu": __CNI_MTU__,
"ipam": {
"type": "calico-ipam"
},
"policy": {
"type": "k8s"
},
"kubernetes": {
"kubeconfig": "__KUBECONFIG_FILEPATH__"
}
},
{
"type": "portmap",
"snat": true,
"capabilities": {"portMappings": true}
},
{
"type": "bandwidth",
"capabilities": {"bandwidth": true}
}
]
}
|||
},
},
secrets: kube.Secret("calico-secrets") {
metadata+: {
namespace: cfg.namespace,
},
data_: {
},
},
saNode: kube.ServiceAccount("calico-node") {
metadata+: {
namespace: cfg.namespace,
},
},
crNode: kube.ClusterRole("calico-node") {
rules: [
{
apiGroups: [""],
resources: ["pods", "nodes", "namespaces"],
verbs: ["get"],
},
{
apiGroups: [""],
resources: ["endpoints", "services"],
verbs: ["watch", "list", "get"],
},
{
apiGroups: [""],
resources: ["configmaps"],
verbs: ["get"],
},
{
apiGroups: [""],
resources: ["nodes/status"],
verbs: ["patch", "update"],
},
{
apiGroups: ["networking.k8s.io"],
resources: ["networkpolicies"],
verbs: ["watch", "list"],
},
{
apiGroups: [""],
resources: ["pods", "namespaces", "serviceaccounts"],
verbs: ["list", "watch"],
},
{
apiGroups: [""],
resources: ["pods/status"],
verbs: ["patch"],
},
{
apiGroups: ["crd.projectcalico.org"],
resources: ["globalfelixconfigs", "felixconfigurations", "bgppeers", "globalbgpconfigs", "bgpconfigurations", "ippools", "ipamblocks", "globalnetworkpolicies", "globalnetworksets", "networkpolicies", "networksets", "clusterinformations", "hostendpoints", "blockaffinities"],
verbs: ["get", "list", "watch"],
},
{
apiGroups: ["crd.projectcalico.org"],
resources: ["ippools", "felixconfigurations", "clusterinformations"],
verbs: ["create", "update"],
},
{
apiGroups: [""],
resources: ["nodes"],
verbs: ["get", "list", "watch"],
},
{
apiGroups: ["crd.projectcalico.org"],
resources: ["blockaffinities", "ipamblocks", "ipamhandles"],
verbs: ["get", "list", "create", "update", "delete"],
},
{
apiGroups: ["crd.projectcalico.org"],
resources: ["ipamconfigs"],
verbs: ["get"],
},
{
apiGroups: ["crd.projectcalico.org"],
resources: ["blockaffinities"],
verbs: ["watch"],
},
{
apiGroups: ["apps"],
resources: ["daemonsets"],
verbs: ["get"],
},
],
},
crbNode: bindServiceAccountClusterRole(env.saNode, env.crNode),
saController: kube.ServiceAccount("calico-kube-controllers") {
metadata+: {
namespace: cfg.namespace,
},
},
crController: kube.ClusterRole("calico-kube-controllers") {
rules: [
{
apiGroups: [""],
resources: ["nodes"],
verbs: ["watch", "list", "get"],
},
{
apiGroups: [""],
resources: ["pods"],
verbs: ["get"],
},
{
apiGroups: ["crd.projectcalico.org"],
resources: ["ippools"],
verbs: ["list"],
},
{
apiGroups: ["crd.projectcalico.org"],
resources: ["blockaffinities", "ipamblocks", "ipamhandles"],
verbs: ["get", "list", "create", "update", "delete"],
},
{
apiGroups: ["crd.projectcalico.org"],
resources: ["hostendpoints"],
verbs: ["get", "list", "create", "update", "delete"],
},
{
apiGroups: ["crd.projectcalico.org"],
resources: ["clusterinformations"],
verbs: ["get", "create", "update"],
},
{
apiGroups: ["crd.projectcalico.org"],
resources: ["kubecontrollersconfigurations"],
verbs: ["get", "create", "update", "watch"],
},
],
},
crbController: bindServiceAccountClusterRole(env.saController, env.crController),
controller: kube.Deployment("calico-kube-controllers") {
metadata+: {
namespace: cfg.namespace,
labels+: {
"k8s-app": "calico-kube-controllers",
},
},
spec+: {
replicas: 1,
strategy: { type: "Recreate" },
template+: {
spec+: {
nodeSelector: {
"kubernetes.io/os": "linux"
},
tolerations: [
{ key: "CriticalAddonsOnly", operator: "Exists" },
{ key: "node-role.kubernetes.io/master", effect: "NoSchedule" },
],
serviceAccountName: env.saController.metadata.name,
priorityClassName: "system-cluster-critical",
containers_: {
"calico-kube-controllers": kube.Container("calico-kube-controllers") {
image: cfg.imageController,
env_: {
DATASTORE_TYPE: "kubernetes",
ENABLED_CONTROLLERS: "node",
},
volumeMounts_: {
secrets: {
mountPath: env.cm.secretPrefix,
},
},
readinessProbe: {
exec: {
command: [ "/usr/bin/check-status", "-r" ],
},
},
},
},
volumes_: {
secrets: kube.SecretVolume(env.secrets),
},
},
},
},
},
# ConfigMap that holds overriden bird.cfg.template and bird_ipam.cfg.template.
calicoMetallbBird: kube.ConfigMap("calico-metallb-bird") {
metadata+: {
namespace: cfg.namespace,
},
data: {
"bird.cfg.template": (importstr "calico-bird.cfg.template"),
"bird_ipam.cfg.template": (importstr "calico-bird-ipam.cfg.template"),
},
},
nodeDaemon: kube.DaemonSet("calico-node") {
metadata+: {
namespace: cfg.namespace,
labels+: {
"k8s-app": "calico-node",
},
},
spec+: {
template+: {
spec+: {
nodeSelector: {
"kubernetes.io/os": "linux"
},
hostNetwork: true,
tolerations: [
{ effect: "NoSchedule", operator: "Exists" },
{ key: "CriticalAddonsOnly", operator: "Exists" },
{ effect: "NoExecute", operator: "Exists" },
],
serviceAccountName: env.saNode.metadata.name,
terminationGracePeriodSeconds: 0,
priorityClassName: "system-cluster-critical",
volumes_: {
lib_modules: kube.HostPathVolume("/run/current-system/kernel-modules/lib/modules"),
var_run_calico: kube.HostPathVolume("/var/run/calico"),
var_lib_calico: kube.HostPathVolume("/var/lib/calico"),
xtables_lock: kube.HostPathVolume("/run/xtables.lock"),
cni_bin: kube.HostPathVolume("/opt/cni/bin"),
cni_config: kube.HostPathVolume("/opt/cni/conf"),
secrets: kube.SecretVolume(env.secrets),
bird_cfg_template: env.calicoMetallbBird.volume,
# TODO flexvol-driver-host, policysync
},
initContainers_: {
installCNI: kube.Container("install-cni") {
image: cfg.imageCNI,
command: ["/install-cni.sh"],
env_: {
CNI_CONF_NAME: "10-calico.conflist",
CNI_NETWORK_CONFIG: kube.ConfigMapRef(env.cm, "cni_network_config"),
CNI_MTU: kube.ConfigMapRef(env.cm, "veth_mtu"),
# Important: our directory is changed from the default (/etc/cni/net.d)
# to inside /opt/ above in the cni_config HostPathVolume.
# See projectcalico/cni-plugin//k8s-install/scripts/install-cni.sh:24 for reference.
CNI_NET_DIR: "/opt/cni/conf",
SLEEP: "false",
KUBERNETES_NODE_NAME: { fieldRef: { fieldPath: "spec.nodeName" } },
},
volumeMounts_: {
cni_bin: { mountPath: "/host/opt/cni/bin" },
cni_config: { mountPath: "/host/etc/cni/net.d" },
secrets: { mountPath: env.cm.secretPrefix },
},
securityContext: {
privileged: true,
},
},
},
containers_: {
calicoNode: kube.Container("calico-node") {
image: cfg.imageNode,
env_: {
WAIT_FOR_DATASTORE: "true",
NODENAME: kube.FieldRef("spec.nodeName"),
DATASTORE_TYPE: "kubernetes",
CALICO_NETWORKING_BACKEND: kube.ConfigMapRef(env.cm, "calico_backend"),
CLUSTER_TYPE: "k8s,bgp",
IP: "autodetect",
IP_AUTODETECTION_METHOD: "can-reach=185.236.240.1",
CALICO_IPV4POOL_IPIP: "Always",
FELIX_IPINIPMTU: kube.ConfigMapRef(env.cm, "veth_mtu"),
FELIX_WIREGUARDMTU: kube.ConfigMapRef(env.cm, "veth_mtu"),
CALICO_IPV4POOL_CIDR: "10.10.24.0/21",
CALICO_DISABLE_FILE_LOGGING: "true",
FELIX_DEFAULTENDPOINTTOHOSTACTION: "ACCEPT",
FELIX_LOGSEVERITYSCREEN: "info",
FELIX_IPV6SUPPORT: "false",
FELIX_HEALTHENABLED: "true",
FELIX_HEALTHHOST: "127.0.0.1",
CALICO_ADVERTISE_CLUSTER_IPS: "10.10.12.0/24",
KUBERNETES_NODE_NAME: { fieldRef: { fieldPath: "spec.nodeName" } },
},
securityContext: {
privileged: true,
},
resources: {
requests: { cpu: "250m" },
},
livenessProbe: {
exec: {
command: ["/bin/calico-node", "-bird-live", "-felix-live"],
},
periodSeconds: 10,
initialDelaySeconds: 10,
failureThreshold: 6,
},
readinessProbe: {
exec: {
command: ["/bin/calico-node", "-bird-ready", "-felix-ready"],
},
periodSeconds: 10,
},
volumeMounts_: {
lib_modules: { mountPath: "/lib/modules" },
xtables_lock: { mountPath: "/run/xtables.lock" },
var_run_calico: { mountPath: "/var/run/calico" },
var_lib_calico: { mountPath: "/var/lib/calico" },
secrets: { mountPath: env.cm.secretPrefix },
},
volumeMounts+: [
{ name: "bird-cfg-template",
mountPath: "/etc/calico/confd/templates/bird.cfg.template",
subPath: "bird.cfg.template"
},
{ name: "bird-cfg-template",
mountPath: "/etc/calico/confd/templates/bird_ipam.cfg.template",
subPath: "bird_ipam.cfg.template"
},
],
},
},
},
},
},
},
},
}