Sergiusz Bazanski | 2022ac2 | 2020-06-06 17:04:07 +0200 | [diff] [blame] | 1 | local kube = import "../../../kube/kube.libsonnet"; |
Sergiusz Bazanski | ce81c39 | 2020-06-06 12:35:06 +0200 | [diff] [blame] | 2 | |
| 3 | { |
Sergiusz Bazanski | 2022ac2 | 2020-06-06 17:04:07 +0200 | [diff] [blame] | 4 | // Cluster sets up all cluster-specific monitoring resources in their own namespace. |
Serge Bazanski | 363bf4f | 2020-08-24 21:00:56 +0200 | [diff] [blame] | 5 | // |
Sergiusz Bazanski | 2022ac2 | 2020-06-06 17:04:07 +0200 | [diff] [blame] | 6 | // Currently this consists of a prometheus server that scrapes k8s nodes for kubelet |
Serge Bazanski | 363bf4f | 2020-08-24 21:00:56 +0200 | [diff] [blame] | 7 | // and cAdvisor metrics, and possibly ships over metrics to the global tier via set |
| 8 | // upstreams. |
Sergiusz Bazanski | ce81c39 | 2020-06-06 12:35:06 +0200 | [diff] [blame] | 9 | Cluster(name):: { |
| 10 | local cluster = self, |
| 11 | local cfg = cluster.cfg, |
| 12 | cfg:: { |
| 13 | name: name, |
| 14 | namespace: "monitoring-cluster", |
| 15 | |
| 16 | images: { |
| 17 | prometheus: "prom/prometheus:v2.18.1", |
| 18 | }, |
| 19 | |
| 20 | storageClasses: { |
Sergiusz Bazanski | 2022ac2 | 2020-06-06 17:04:07 +0200 | [diff] [blame] | 21 | prometheus: error "storageClasses.prometheus must be set", |
Sergiusz Bazanski | ce81c39 | 2020-06-06 12:35:06 +0200 | [diff] [blame] | 22 | }, |
Serge Bazanski | 363bf4f | 2020-08-24 21:00:56 +0200 | [diff] [blame] | 23 | |
| 24 | // Username used to authenticate to upstreams. |
| 25 | username: error "username must be set", |
| 26 | |
| 27 | // Global tier upstreams that this cluster should ship metrics off to. |
| 28 | // List of |
| 29 | // { |
| 30 | // remote: URL of upstream |
| 31 | // password: password used to authenticate, in conjunction with cfg.username. |
| 32 | // |
| 33 | upstreams: [], |
Sergiusz Bazanski | ce81c39 | 2020-06-06 12:35:06 +0200 | [diff] [blame] | 34 | }, |
| 35 | |
| 36 | namespace: kube.Namespace(cfg.namespace), |
| 37 | |
| 38 | prometheus: { |
| 39 | local prometheus = self, |
| 40 | |
| 41 | // Configuration that's going to be emitted as prometheus.yml and passed to the |
| 42 | // prometheus server for this cluster. |
| 43 | configuration:: { |
| 44 | global: { |
| 45 | external_labels: { |
| 46 | cluster: cluster.cfg.name, |
| 47 | }, |
| 48 | }, |
| 49 | |
| 50 | // Constructor for a Kubernetes scrape job that uses the pod's service account and |
| 51 | // TLS configuration, selecting the given k8s scrape 'role'. |
| 52 | local kubeScrapeConfig = function(name, role) { |
| 53 | job_name: name, |
| 54 | scheme: "https", |
| 55 | scrape_interval: "30s", |
| 56 | kubernetes_sd_configs: [ { role: role }, ], |
| 57 | tls_config: { |
| 58 | ca_file: "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt", |
| 59 | }, |
| 60 | bearer_token_file: "/var/run/secrets/kubernetes.io/serviceaccount/token", |
| 61 | }, |
| 62 | |
| 63 | scrape_configs: [ |
| 64 | // When scraping node-based metrics (ie. node and cadvisor metrics) we contact |
| 65 | // the metrics endpoints on the kubelet via the API server. This is done by |
| 66 | // relabeling _address__ and __metrics_path__ to point at the k8s API server, |
| 67 | // and at the API server proxy path to reach a node's metrics endpoint. |
| 68 | // |
| 69 | // This approach was lifted from the prometheus examples for Kubernetes, and |
| 70 | // while the benefits outlined there do not matter that much to us (our |
| 71 | // kubelets listen on public addresses, anyway), we still enjoy this approach |
| 72 | // for the fact that we don't have to hardcode the kubelet TLS port. |
| 73 | // |
| 74 | // https://github.com/prometheus/prometheus/blob/master/documentation/examples/prometheus-kubernetes.yml |
| 75 | // |
| 76 | // When contacting the API server, we hardcode the 'hswaw.net' DNS suffix as |
| 77 | // our API server's TLS certificate only has a CN/SAN for its full FQDN, not |
| 78 | // the .svc.cluster.local shorthand (see //cluster/clustercfg:clustercfg.py). |
| 79 | |
| 80 | // Scrape Kubernetes node metrics via apiserver. This emites kube_node_* metrics. |
| 81 | kubeScrapeConfig("cluster_node_metrics", "node") { |
| 82 | relabel_configs: [ |
| 83 | { |
| 84 | action: "labelmap", |
| 85 | regex: "__meta_kubernetes_node_label_(.+)", |
| 86 | }, |
| 87 | { |
| 88 | action: "replace", |
| 89 | target_label: "__address__", |
| 90 | replacement: "kubernetes.default.svc.%s.hswaw.net:443" % [cluster.cfg.name], |
| 91 | }, |
| 92 | { |
| 93 | target_label: "__metrics_path__", |
| 94 | source_labels: ["__meta_kubernetes_node_name"], |
| 95 | regex: "(.+)", |
| 96 | replacement: "/api/v1/nodes/${1}/proxy/metrics", |
| 97 | }, |
| 98 | ], |
| 99 | }, |
| 100 | // Scrape Kubernetes node cadvisor metrics via apiserver. This emits container_* metrics. |
| 101 | kubeScrapeConfig("cluster_cadvisor_metrics", "node") { |
| 102 | relabel_configs: [ |
| 103 | { |
| 104 | action: "labelmap", |
| 105 | regex: "__meta_kubernetes_node_label_(.+)", |
| 106 | }, |
| 107 | { |
| 108 | action: "replace", |
| 109 | target_label: "__address__", |
| 110 | replacement: "kubernetes.default.svc.%s.hswaw.net:443" % [cluster.cfg.name], |
| 111 | }, |
| 112 | { |
| 113 | target_label: "__metrics_path__", |
| 114 | source_labels: ["__meta_kubernetes_node_name"], |
| 115 | regex: "(.+)", |
| 116 | replacement: "/api/v1/nodes/${1}/proxy/metrics/cadvisor", |
| 117 | }, |
| 118 | ], |
| 119 | }, |
| 120 | ], |
Serge Bazanski | 363bf4f | 2020-08-24 21:00:56 +0200 | [diff] [blame] | 121 | |
| 122 | remote_write: [ |
| 123 | { |
| 124 | url: u.remote, |
| 125 | basic_auth: { |
| 126 | username: cluster.cfg.username, |
| 127 | password: u.password, |
| 128 | }, |
| 129 | } |
| 130 | for u in cluster.cfg.upstreams |
| 131 | ], |
Sergiusz Bazanski | ce81c39 | 2020-06-06 12:35:06 +0200 | [diff] [blame] | 132 | }, |
| 133 | |
| 134 | configmap: kube.ConfigMap("prometheus-cluster") { |
| 135 | metadata+: { |
| 136 | namespace: cfg.namespace, |
| 137 | }, |
| 138 | data: { |
| 139 | "prometheus.yml": std.manifestYamlDoc(prometheus.configuration), |
| 140 | }, |
| 141 | }, |
| 142 | |
| 143 | sa: kube.ServiceAccount("prometheus-cluster") { |
| 144 | metadata+: { |
| 145 | namespace: cfg.namespace, |
| 146 | }, |
| 147 | }, |
| 148 | |
| 149 | cr: kube.ClusterRole("monitoring-cluster-prometheus-server-%s" % [cfg.name]) { |
| 150 | rules: [ |
| 151 | // Allow access to all metrics. |
| 152 | { nonResourceURLs: ["/metrics"], verbs: ["get"], }, |
| 153 | // Allow to access node details for discovery. |
| 154 | { apiGroups: [""], resources: ["nodes"], verbs: ["list", "watch", "get"], }, |
| 155 | // Allow to proxy to bare node HTTP to access per-node metrics endpoints. |
| 156 | { apiGroups: [""], resources: ["nodes/proxy"], verbs: ["get"], }, |
| 157 | ], |
| 158 | }, |
| 159 | |
| 160 | crb: kube.ClusterRoleBinding("monitoring-cluster-prometheus-server-%s" % [cfg.name]) { |
| 161 | subjects_: [prometheus.sa], |
| 162 | roleRef_: prometheus.cr, |
| 163 | }, |
| 164 | |
| 165 | deploy: kube.Deployment("prometheus-cluster") { |
| 166 | metadata+: { |
| 167 | namespace: cfg.namespace, |
| 168 | }, |
| 169 | spec+: { |
| 170 | template+: { |
| 171 | spec+: { |
| 172 | containers_: { |
| 173 | default: kube.Container("default") { |
| 174 | image: cfg.images.prometheus, |
| 175 | command: [ |
| 176 | "/bin/prometheus", |
| 177 | "--config.file=/etc/prometheus/prometheus.yml", |
| 178 | "--storage.tsdb.path=/prometheus", |
Serge Bazanski | 363bf4f | 2020-08-24 21:00:56 +0200 | [diff] [blame] | 179 | "--storage.tsdb.retention.size=10GB", |
Sergiusz Bazanski | ce81c39 | 2020-06-06 12:35:06 +0200 | [diff] [blame] | 180 | "--web.console.libraries=/usr/share/prometheus/console_libraries", |
| 181 | "--web.console.templates=/usr/share/prometheus/consoles", |
| 182 | "--web.enable-lifecycle", |
| 183 | ], |
| 184 | resources: { |
| 185 | requests: { |
| 186 | memory: "256Mi", |
| 187 | cpu: "100m", |
| 188 | }, |
| 189 | limits: { |
| 190 | memory: "1Gi", |
| 191 | cpu: "1", |
| 192 | }, |
| 193 | }, |
| 194 | volumeMounts_: { |
| 195 | data: { mountPath: "/prometheus", }, |
| 196 | configmap: { mountPath: "/etc/prometheus", }, |
| 197 | }, |
| 198 | }, |
| 199 | }, |
| 200 | serviceAccountName: prometheus.sa.metadata.name, |
| 201 | tolerations: [ |
| 202 | { key: "CriticalAddonsOnly", operator: "Exists" }, |
| 203 | ], |
| 204 | volumes_: { |
| 205 | data: kube.PersistentVolumeClaimVolume(prometheus.pvc), |
| 206 | configmap: kube.ConfigMapVolume(prometheus.configmap), |
| 207 | }, |
| 208 | }, |
| 209 | }, |
| 210 | }, |
| 211 | }, |
| 212 | |
| 213 | // Kubernetes metric storage volume. |
| 214 | pvc: kube.PersistentVolumeClaim("prometheus-cluster") { |
| 215 | metadata+: { |
| 216 | namespace: cfg.namespace, |
| 217 | }, |
| 218 | spec+: { |
| 219 | storageClassName: cfg.storageClasses.prometheus, |
| 220 | accessModes: ["ReadWriteOnce"], |
| 221 | resources: { |
| 222 | requests: { |
Serge Bazanski | 363bf4f | 2020-08-24 21:00:56 +0200 | [diff] [blame] | 223 | storage: "16Gi", |
Sergiusz Bazanski | ce81c39 | 2020-06-06 12:35:06 +0200 | [diff] [blame] | 224 | }, |
| 225 | }, |
| 226 | }, |
| 227 | }, |
| 228 | |
| 229 | // Network Policy governing access to the prometheus server. |
| 230 | np: kube.NetworkPolicy("prometheus-cluster") { |
| 231 | metadata+: { |
| 232 | namespace: cfg.namespace, |
| 233 | }, |
| 234 | spec+: kube.podLabelsSelector(prometheus.deploy) { |
| 235 | ingress_: { |
| 236 | // Deny all inbound traffic to pod. |
| 237 | // This will be augmented to allow access from some other pod/namespace |
| 238 | // in the future. |
| 239 | }, |
| 240 | egress_: { |
| 241 | // Allow all outbound traffic from pod. |
| 242 | outboundAll: {}, |
| 243 | }, |
| 244 | policyTypes: ["Ingress", "Egress"], |
| 245 | }, |
| 246 | }, |
| 247 | }, |
| 248 | }, |
Sergiusz Bazanski | ce81c39 | 2020-06-06 12:35:06 +0200 | [diff] [blame] | 249 | } |