| local kube = import "../../../kube/kube.libsonnet"; |
| |
| { |
| // Cluster sets up all cluster-specific monitoring resources in their own namespace. |
| // |
| // Currently this consists of a prometheus server that scrapes k8s nodes for kubelet |
| // and cAdvisor metrics, and possibly ships over metrics to the global tier via set |
| // upstreams. |
| Cluster(name):: { |
| local cluster = self, |
| local cfg = cluster.cfg, |
| cfg:: { |
| name: name, |
| namespace: "monitoring-cluster", |
| |
| images: { |
| prometheus: "prom/prometheus:v2.18.1", |
| }, |
| |
| storageClasses: { |
| prometheus: error "storageClasses.prometheus must be set", |
| }, |
| |
| // Username used to authenticate to upstreams. |
| username: error "username must be set", |
| |
| // Global tier upstreams that this cluster should ship metrics off to. |
| // List of |
| // { |
| // remote: URL of upstream |
| // password: password used to authenticate, in conjunction with cfg.username. |
| // |
| upstreams: [], |
| }, |
| |
| namespace: kube.Namespace(cfg.namespace), |
| |
| prometheus: { |
| local prometheus = self, |
| |
| // Configuration that's going to be emitted as prometheus.yml and passed to the |
| // prometheus server for this cluster. |
| configuration:: { |
| global: { |
| external_labels: { |
| cluster: cluster.cfg.name, |
| }, |
| }, |
| |
| // Constructor for a Kubernetes scrape job that uses the pod's service account and |
| // TLS configuration, selecting the given k8s scrape 'role'. |
| local kubeScrapeConfig = function(name, role) { |
| job_name: name, |
| scheme: "https", |
| scrape_interval: "30s", |
| kubernetes_sd_configs: [ { role: role }, ], |
| tls_config: { |
| ca_file: "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt", |
| }, |
| bearer_token_file: "/var/run/secrets/kubernetes.io/serviceaccount/token", |
| }, |
| |
| |
| // When scraping node-based metrics (ie. node and cadvisor metrics) we contact |
| // the metrics endpoints on the kubelet via the API server. This is done by |
| // relabeling _address__ and __metrics_path__ to point at the k8s API server, |
| // and at the API server proxy path to reach a node's metrics endpoint. |
| // |
| // This approach was lifted from the prometheus examples for Kubernetes, and |
| // while the benefits outlined there do not matter that much to us (our |
| // kubelets listen on public addresses, anyway), we still enjoy this approach |
| // for the fact that we don't have to hardcode the kubelet TLS port. |
| // |
| // https://github.com/prometheus/prometheus/blob/master/documentation/examples/prometheus-kubernetes.yml |
| // |
| // When contacting the API server, we hardcode the 'hswaw.net' DNS suffix as |
| // our API server's TLS certificate only has a CN/SAN for its full FQDN, not |
| // the .svc.cluster.local shorthand (see //cluster/clustercfg). |
| local kubeScrapeNodeMetrics = function(name, path) kubeScrapeConfig(name, "node") { |
| relabel_configs: [ |
| { |
| action: "labelmap", |
| regex: "__meta_kubernetes_node_label_(.+)", |
| }, |
| { |
| action: "replace", |
| target_label: "__address__", |
| replacement: "kubernetes.default.svc.%s.hswaw.net:443" % [cluster.cfg.name], |
| }, |
| { |
| target_label: "__metrics_path__", |
| source_labels: ["__meta_kubernetes_node_name"], |
| regex: "(.+)", |
| replacement: "/api/v1/nodes/${1}/proxy" + path, |
| }, |
| ], |
| }, |
| |
| // When scraping API server-colocated metrics (ie. metrics from nixos services running alongside |
| // APIserver instances), we contact the metrics endpoints directly over the node's IP addresses |
| // and an external port. The node IP addresses are discovered via Prometheus kubernetes endpoint |
| // discovery which selects all endpoints for the default/kubernetes service. This service is |
| // backed by apiserver instances on public IP addresses. We can then rewrite the received port |
| // by the port of the service we're interested in to get to that service. |
| local kubeScrapeAPIServerColocated = function(name, port) kubeScrapeConfig(name, "endpoints") { |
| relabel_configs: [ |
| // Select only endpoints that back the default/kubernetes service. These are all |
| // public IP addresses of nodes that run the API server. |
| { |
| action: "keep", |
| regex: "default;kubernetes;https", |
| source_labels: [ |
| "__meta_kubernetes_namespace", |
| "__meta_kubernetes_service_name", |
| "__meta_kubernetes_endpoint_port_name", |
| ], |
| }, |
| ] + (if port == 4001 then [] else [ |
| // Replace endpoint port with requested port, if the requested port is not the apiserver's |
| // port 4001, which is the one returned by default for the these endpoints. |
| { |
| action: "replace", |
| regex: "([^:]+):.+", |
| replacement: "$1:%d" % [port], |
| source_labels: [ |
| "__address__", |
| ], |
| target_label: "__address__", |
| }, |
| ]), |
| // We disable server-side TLS certificate verification. |
| // Unfortunately, all apiserver-colocated services run with TLS certificates that do not have |
| // the right IP address SAN. Unfortunately, we can't override the TLS ServerName for a scrape |
| // target [1], so the only two choiced we are left with are: |
| // 1) re-emit relevant certificates with IP address SANs that allow for access by IP. |
| // 2) disable TLS verification. |
| // We choose 2), knowing that if someone manages to hijack a target IP address they can end up |
| // stealing our bearer token and impersonating the service account with which Prometheus is |
| // running. In the long term, we hope for [1] to be resolved. |
| // |
| // TODO(q3k): revisit this once [1] gets fixed. |
| // [1] - https://github.com/prometheus/prometheus/issues/4827 |
| tls_config: { |
| insecure_skip_verify: true, |
| }, |
| }, |
| |
| scrape_configs: [ |
| /// Scrape per-node metrics, proxied via the APIServer.. |
| // Scrape Kubernetes node metrics via apiserver. This emits kube_node_* metrics. |
| kubeScrapeNodeMetrics("cluster_node_metrics", "/metrics"), |
| // Scrape Kubernetes node cadvisor metrics via apiserver. This emits container_* metrics. |
| kubeScrapeNodeMetrics("cluster_cadvisor_metrics", "/metrics/cadvisor"), |
| |
| /// Scape apiserver-colocated ('master node') metrics, over nodes' public IP addresses. |
| /// (currently all nodes are 'master' nodes) |
| // Scrape Kubernetes apiserver metrics. |
| kubeScrapeAPIServerColocated("cluster_apiserver_metrics", 4001), |
| // Scrape Kubernetes controller-manager metrics. |
| kubeScrapeAPIServerColocated("cluster_controllermanager_metrics", 4003), |
| // Scrape Kubernetes scheduler metrics. |
| kubeScrapeAPIServerColocated("cluster_scheduler_metrics", 4005), |
| |
| // Scrape core Kubernetes service (without service discovery, relying on DNS instead) |
| { |
| job_name: "kubernetes-core-services", |
| scheme: "http", |
| scrape_interval: "30s", |
| static_configs: [ |
| { targets: ["rook-ceph-mgr.ceph-waw3.svc.cluster.local:9283"], labels: { core_service: "ceph-mgr", ceph_cluster: "waw3" } } |
| ], |
| } |
| ], |
| |
| remote_write: [ |
| { |
| url: u.remote, |
| basic_auth: { |
| username: cluster.cfg.username, |
| password: u.password, |
| }, |
| } |
| for u in cluster.cfg.upstreams |
| ], |
| }, |
| |
| configmap: kube.ConfigMap("prometheus-cluster") { |
| metadata+: { |
| namespace: cfg.namespace, |
| }, |
| data: { |
| "prometheus.yml": std.manifestYamlDoc(prometheus.configuration), |
| }, |
| }, |
| |
| sa: kube.ServiceAccount("prometheus-cluster") { |
| metadata+: { |
| namespace: cfg.namespace, |
| }, |
| }, |
| |
| cr: kube.ClusterRole("monitoring-cluster-prometheus-server-%s" % [cfg.name]) { |
| rules: [ |
| // Allow access to all metrics. |
| { nonResourceURLs: ["/metrics"], verbs: ["get"], }, |
| // Allow to access node details for discovery. |
| { apiGroups: [""], resources: ["nodes"], verbs: ["list", "watch", "get"], }, |
| { apiGroups: [""], resources: ["endpoints", "services", "pods"], verbs: ["list", "watch", "get"], }, |
| // Allow to proxy to bare node HTTP to access per-node metrics endpoints. |
| { apiGroups: [""], resources: ["nodes/proxy"], verbs: ["get"], }, |
| ], |
| }, |
| |
| crb: kube.ClusterRoleBinding("monitoring-cluster-prometheus-server-%s" % [cfg.name]) { |
| subjects_: [prometheus.sa], |
| roleRef_: prometheus.cr, |
| }, |
| |
| deploy: kube.Deployment("prometheus-cluster") { |
| metadata+: { |
| namespace: cfg.namespace, |
| }, |
| spec+: { |
| template+: { |
| spec+: { |
| containers_: { |
| default: kube.Container("default") { |
| image: cfg.images.prometheus, |
| command: [ |
| "/bin/prometheus", |
| "--config.file=/etc/prometheus/prometheus.yml", |
| "--storage.tsdb.path=/prometheus", |
| "--storage.tsdb.retention.size=10GB", |
| "--web.console.libraries=/usr/share/prometheus/console_libraries", |
| "--web.console.templates=/usr/share/prometheus/consoles", |
| "--web.enable-lifecycle", |
| ], |
| resources: { |
| requests: { |
| memory: "3Gi", |
| cpu: "100m", |
| }, |
| limits: { |
| memory: "3Gi", |
| cpu: "1", |
| }, |
| }, |
| volumeMounts_: { |
| data: { mountPath: "/prometheus", }, |
| configmap: { mountPath: "/etc/prometheus", }, |
| }, |
| }, |
| }, |
| serviceAccountName: prometheus.sa.metadata.name, |
| tolerations: [ |
| { key: "CriticalAddonsOnly", operator: "Exists" }, |
| ], |
| volumes_: { |
| data: prometheus.pvc.volume, |
| configmap: prometheus.configmap.volume, |
| }, |
| }, |
| }, |
| }, |
| }, |
| |
| // Kubernetes metric storage volume. |
| pvc: kube.PersistentVolumeClaim("prometheus-cluster") { |
| metadata+: { |
| namespace: cfg.namespace, |
| }, |
| storage:: "16Gi", |
| storageClass:: cfg.storageClasses.prometheus, |
| }, |
| |
| // Network Policy governing access to the prometheus server. |
| np: kube.NetworkPolicy("prometheus-cluster") { |
| metadata+: { |
| namespace: cfg.namespace, |
| }, |
| spec+: kube.podLabelsSelector(prometheus.deploy) { |
| ingress_: { |
| // Deny all inbound traffic to pod. |
| // This will be augmented to allow access from some other pod/namespace |
| // in the future. |
| }, |
| egress_: { |
| // Allow all outbound traffic from pod. |
| outboundAll: {}, |
| }, |
| policyTypes: ["Ingress", "Egress"], |
| }, |
| }, |
| }, |
| }, |
| } |