calico: move from etcd to crd
Leaving the CRD definitions as YAML, extracted without modifications
from the original install file - this should make upgrades simpler.
Change-Id: I7211d2711e2af014b36dd887a951abb9e1032eb9
Reviewed-on: https://gerrit.hackerspace.pl/c/hscloud/+/1179
Reviewed-by: q3k <q3k@hackerspace.pl>
diff --git a/cluster/kube/lib/calico.libsonnet b/cluster/kube/lib/calico.libsonnet
index eb813a8..0c3673f 100644
--- a/cluster/kube/lib/calico.libsonnet
+++ b/cluster/kube/lib/calico.libsonnet
@@ -1,6 +1,11 @@
# Deploy hosted calico with its own etcd.
local kube = import "../../../kube/kube.libsonnet";
+local crdYaml = (std.native("parseYaml"))(importstr "./calico_crd.yml");
+local crdMap = {
+ [x.metadata.name]: x
+ for x in crdYaml if x != null
+};
local bindServiceAccountClusterRole(sa, cr) = kube.ClusterRoleBinding(cr.metadata.name) {
roleRef: {
@@ -27,20 +32,10 @@
imageController: "calico/kube-controllers:" + cfg.version,
imageCNI: "calico/cni:" + cfg.version,
imageNode: "calico/node:" + cfg.version,
- // TODO(implr): migrate calico from etcd to apiserver
- etcd: {
- endpoints: [
- "https://bc01n01.hswaw.net:2379",
- "https://bc01n02.hswaw.net:2379",
- "https://dcr01s22.hswaw.net:2379",
- "https://dcr01s24.hswaw.net:2379",
- ],
- ca: importstr "../../certs/ca-etcd.crt",
- cert: importstr "../../certs/etcd-calico.cert",
- key: importstr "../../secrets/plain/etcd-calico.key",
- },
},
+ crds: crdMap,
+
cm: kube.ConfigMap("calico-config") {
local cm = self,
secretPrefix:: "/calico-secrets/",
@@ -50,12 +45,6 @@
},
data: {
- etcd_endpoints: std.join(",", cfg.etcd.endpoints),
-
- etcd_ca: cm.secretPrefix + "etcd-ca",
- etcd_cert: cm.secretPrefix + "etcd-cert",
- etcd_key: cm.secretPrefix + "etcd-key",
-
calico_backend: "bird",
veth_mtu: "1440",
@@ -69,11 +58,8 @@
{
"type": "calico",
"log_level": "info",
- "etcd_endpoints": "__ETCD_ENDPOINTS__",
- "etcd_key_file": "__ETCD_KEY_FILE__",
- "etcd_cert_file": "__ETCD_CERT_FILE__",
- "etcd_ca_cert_file": "__ETCD_CA_CERT_FILE__",
- "datastore_type": "etcdv3",
+ "datastore_type": "kubernetes",
+ "nodename": "__KUBERNETES_NODE_NAME__",
"mtu": __CNI_MTU__,
"ipam": {
"type": "calico-ipam"
@@ -106,9 +92,6 @@
},
data_: {
- "etcd-ca": cfg.etcd.ca,
- "etcd-cert": cfg.etcd.cert,
- "etcd-key": cfg.etcd.key,
},
},
@@ -141,15 +124,55 @@
verbs: ["patch", "update"],
},
{
+ apiGroups: ["networking.k8s.io"],
+ resources: ["networkpolicies"],
+ verbs: ["watch", "list"],
+ },
+ {
+ apiGroups: [""],
+ resources: ["pods", "namespaces", "serviceaccounts"],
+ verbs: ["list", "watch"],
+ },
+ {
apiGroups: [""],
resources: ["pods/status"],
verbs: ["patch"],
},
{
+ apiGroups: ["crd.projectcalico.org"],
+ resources: ["globalfelixconfigs", "felixconfigurations", "bgppeers", "globalbgpconfigs", "bgpconfigurations", "ippools", "ipamblocks", "globalnetworkpolicies", "globalnetworksets", "networkpolicies", "networksets", "clusterinformations", "hostendpoints", "blockaffinities"],
+ verbs: ["get", "list", "watch"],
+ },
+ {
+ apiGroups: ["crd.projectcalico.org"],
+ resources: ["ippools", "felixconfigurations", "clusterinformations"],
+ verbs: ["create", "update"],
+ },
+ {
apiGroups: [""],
resources: ["nodes"],
verbs: ["get", "list", "watch"],
},
+ {
+ apiGroups: ["crd.projectcalico.org"],
+ resources: ["blockaffinities", "ipamblocks", "ipamhandles"],
+ verbs: ["get", "list", "create", "update", "delete"],
+ },
+ {
+ apiGroups: ["crd.projectcalico.org"],
+ resources: ["ipamconfigs"],
+ verbs: ["get"],
+ },
+ {
+ apiGroups: ["crd.projectcalico.org"],
+ resources: ["blockaffinities"],
+ verbs: ["watch"],
+ },
+ {
+ apiGroups: ["apps"],
+ resources: ["daemonsets"],
+ verbs: ["get"],
+ },
],
},
@@ -165,13 +188,38 @@
rules: [
{
apiGroups: [""],
- resources: ["nodes", "pods", "namespaces", "serviceaccounts"],
+ resources: ["nodes"],
verbs: ["watch", "list", "get"],
},
{
- apiGroups: ["networking.k8s.io"],
- resources: ["networkpolicies"],
- verbs: ["watch", "list"],
+ apiGroups: [""],
+ resources: ["pods"],
+ verbs: ["get"],
+ },
+ {
+ apiGroups: ["crd.projectcalico.org"],
+ resources: ["ippools"],
+ verbs: ["list"],
+ },
+ {
+ apiGroups: ["crd.projectcalico.org"],
+ resources: ["blockaffinities", "ipamblocks", "ipamhandles"],
+ verbs: ["get", "list", "create", "update", "delete"],
+ },
+ {
+ apiGroups: ["crd.projectcalico.org"],
+ resources: ["hostendpoints"],
+ verbs: ["get", "list", "create", "update", "delete"],
+ },
+ {
+ apiGroups: ["crd.projectcalico.org"],
+ resources: ["clusterinformations"],
+ verbs: ["get", "create", "update"],
+ },
+ {
+ apiGroups: ["crd.projectcalico.org"],
+ resources: ["kubecontrollersconfigurations"],
+ verbs: ["get", "create", "update", "watch"],
},
],
},
@@ -199,16 +247,12 @@
],
serviceAccountName: env.saController.metadata.name,
priorityClassName: "system-cluster-critical",
- hostNetwork: true,
containers_: {
"calico-kube-controllers": kube.Container("calico-kube-controllers") {
image: cfg.imageController,
env_: {
- ETCD_ENDPOINTS: kube.ConfigMapRef(env.cm, "etcd_endpoints"),
- ETCD_CA_CERT_FILE: kube.ConfigMapRef(env.cm, "etcd_ca"),
- ETCD_KEY_FILE: kube.ConfigMapRef(env.cm, "etcd_key"),
- ETCD_CERT_FILE: kube.ConfigMapRef(env.cm, "etcd_cert"),
- ENABLED_CONTROLLERS: "policy,namespace,serviceaccount,workloadendpoint,node",
+ DATASTORE_TYPE: "kubernetes",
+ ENABLED_CONTROLLERS: "node",
},
volumeMounts_: {
secrets: {
@@ -281,16 +325,11 @@
env_: {
CNI_CONF_NAME: "10-calico.conflist",
CNI_NETWORK_CONFIG: kube.ConfigMapRef(env.cm, "cni_network_config"),
- ETCD_ENDPOINTS: kube.ConfigMapRef(env.cm, "etcd_endpoints"),
CNI_MTU: kube.ConfigMapRef(env.cm, "veth_mtu"),
# Important: our directory is changed from the default (/etc/cni/net.d)
# to inside /opt/ above in the cni_config HostPathVolume.
# See projectcalico/cni-plugin//k8s-install/scripts/install-cni.sh:24 for reference.
CNI_NET_DIR: "/opt/cni/conf",
- # TODO(implr) needed?
- CNI_CONF_ETCD_CA_CERT_FILE: kube.ConfigMapRef(env.cm, "etcd_ca"),
- CNI_CONF_ETCD_KEY_FILE: kube.ConfigMapRef(env.cm, "etcd_key"),
- CNI_CONF_ETCD_CERT_FILE: kube.ConfigMapRef(env.cm, "etcd_cert"),
SLEEP: "false",
KUBERNETES_NODE_NAME: { fieldRef: { fieldPath: "spec.nodeName" } },
},
@@ -308,12 +347,9 @@
calicoNode: kube.Container("calico-node") {
image: cfg.imageNode,
env_: {
- DATASTORE_TYPE: "etcdv3",
- ETCD_ENDPOINTS: kube.ConfigMapRef(env.cm, "etcd_endpoints"),
- ETCD_CA_CERT_FILE: kube.ConfigMapRef(env.cm, "etcd_ca"),
- ETCD_KEY_FILE: kube.ConfigMapRef(env.cm, "etcd_key"),
- ETCD_CERT_FILE: kube.ConfigMapRef(env.cm, "etcd_cert"),
- CALICO_K8S_NODE_REF: kube.FieldRef("spec.nodeName"),
+ WAIT_FOR_DATASTORE: "true",
+ NODENAME: kube.FieldRef("spec.nodeName"),
+ DATASTORE_TYPE: "kubernetes",
CALICO_NETWORKING_BACKEND: kube.ConfigMapRef(env.cm, "calico_backend"),
CLUSTER_TYPE: "k8s,bgp",
IP: "autodetect",