Sergiusz Bazanski | a9c7e86 | 2019-04-01 17:56:28 +0200 | [diff] [blame] | 1 | # Deploy a per-cluster Nginx Ingress Controller |
| 2 | |
| 3 | local kube = import "../../../kube/kube.libsonnet"; |
Sergiusz Bazanski | b13b7ff | 2019-08-29 20:12:24 +0200 | [diff] [blame] | 4 | local policies = import "../../../kube/policies.libsonnet"; |
Sergiusz Bazanski | a9c7e86 | 2019-04-01 17:56:28 +0200 | [diff] [blame] | 5 | |
| 6 | { |
| 7 | Environment: { |
| 8 | local env = self, |
| 9 | local cfg = env.cfg, |
| 10 | cfg:: { |
Serge Bazanski | 2e8d24b | 2021-03-25 18:39:52 +0100 | [diff] [blame] | 11 | # Built from nginx-ingress-controller/Dockerfile: |
| 12 | # |
| 13 | # $ cd cluster/kube/lib/nginx-ingress-controller |
| 14 | # $ docker build -t eu.gcr.io/bgpwtf/nginx-ingress-controller:v0.44.0-r1 . |
| 15 | # [..] |
| 16 | # (2/8) Upgrading libcrypto1.1 (1.1.1i-r0 -> 1.1.1k-r0) |
| 17 | # (3/8) Upgrading libssl1.1 (1.1.1i-r0 -> 1.1.1k-r0 |
| 18 | # [...] |
| 19 | # (8/8) Upgrading openssl (1.1.1i-r0 -> 1.1.1k-r0) |
| 20 | # $ docker push eu.gcr.io/bgpwtf/nginx-ingress-controller:v0.44.0-r1 |
| 21 | # |
| 22 | # TODO(q3k): unfork this once openssl 1.1.1k lands in upstream |
| 23 | # nginx-ingress-controller. |
| 24 | image: "eu.gcr.io/bgpwtf/nginx-ingress-controller:v0.44.0-r1", |
Sergiusz Bazanski | a9c7e86 | 2019-04-01 17:56:28 +0200 | [diff] [blame] | 25 | namespace: "nginx-system", |
| 26 | }, |
| 27 | |
| 28 | metadata:: { |
| 29 | namespace: cfg.namespace, |
| 30 | labels: { |
| 31 | "app.kubernetes.io/name": "ingress-nginx", |
| 32 | "app.kubernetes.io/part-of": "ingress-nginx", |
| 33 | }, |
| 34 | }, |
| 35 | |
| 36 | namespace: kube.Namespace(cfg.namespace), |
| 37 | |
Sergiusz Bazanski | b13b7ff | 2019-08-29 20:12:24 +0200 | [diff] [blame] | 38 | allowInsecure: policies.AllowNamespaceInsecure(cfg.namespace), |
| 39 | |
Sergiusz Bazanski | a9c7e86 | 2019-04-01 17:56:28 +0200 | [diff] [blame] | 40 | maps: { |
| 41 | make(name):: kube.ConfigMap(name) { |
| 42 | metadata+: env.metadata, |
| 43 | }, |
| 44 | configuration: env.maps.make("nginx-configuration"), |
Sergiusz Bazanski | 543b412 | 2019-06-29 22:42:39 +0200 | [diff] [blame] | 45 | tcp: env.maps.make("tcp-services") { |
| 46 | data: { |
Piotr Dobrowolski | f00edf6 | 2020-07-02 18:30:38 +0200 | [diff] [blame] | 47 | "22": "gerrit/gerrit:22", |
Sergiusz Bazanski | 543b412 | 2019-06-29 22:42:39 +0200 | [diff] [blame] | 48 | } |
| 49 | }, |
Sergiusz Bazanski | a9c7e86 | 2019-04-01 17:56:28 +0200 | [diff] [blame] | 50 | udp: env.maps.make("udp-services"), |
| 51 | }, |
| 52 | |
| 53 | sa: kube.ServiceAccount("nginx-ingress-serviceaccount") { |
| 54 | metadata+: env.metadata, |
| 55 | }, |
| 56 | |
| 57 | cr: kube.ClusterRole("nginx-ingress-clusterrole") { |
| 58 | metadata+: env.metadata { |
| 59 | namespace:: null, |
| 60 | }, |
| 61 | rules: [ |
| 62 | { |
| 63 | apiGroups: [""], |
| 64 | resources: ["configmaps", "endpoints", "nodes", "pods", "secrets"], |
| 65 | verbs: ["list", "watch"], |
| 66 | }, |
| 67 | { |
| 68 | apiGroups: [""], |
| 69 | resources: ["nodes"], |
| 70 | verbs: ["get"], |
| 71 | }, |
| 72 | { |
| 73 | apiGroups: [""], |
| 74 | resources: ["services"], |
| 75 | verbs: ["get", "list", "watch"], |
| 76 | }, |
| 77 | { |
Serge Bazanski | 2e8d24b | 2021-03-25 18:39:52 +0100 | [diff] [blame] | 78 | apiGroups: ["extensions", "networking.k8s.io"], |
Sergiusz Bazanski | a9c7e86 | 2019-04-01 17:56:28 +0200 | [diff] [blame] | 79 | resources: ["ingresses"], |
| 80 | verbs: ["get", "list", "watch"], |
| 81 | }, |
| 82 | { |
| 83 | apiGroups: [""], |
| 84 | resources: ["events"], |
| 85 | verbs: ["create", "patch"], |
| 86 | }, |
| 87 | { |
Serge Bazanski | 2e8d24b | 2021-03-25 18:39:52 +0100 | [diff] [blame] | 88 | apiGroups: ["extensions", "networking.k8s.io"], |
Sergiusz Bazanski | a9c7e86 | 2019-04-01 17:56:28 +0200 | [diff] [blame] | 89 | resources: ["ingresses/status"], |
| 90 | verbs: ["update"], |
| 91 | }, |
Serge Bazanski | 2e8d24b | 2021-03-25 18:39:52 +0100 | [diff] [blame] | 92 | { |
| 93 | apiGroups: ["extensions", "networking.k8s.io"], |
| 94 | resources: ["ingressclasses"], |
| 95 | verbs: ["get", "list", "watch"], |
| 96 | }, |
Sergiusz Bazanski | a9c7e86 | 2019-04-01 17:56:28 +0200 | [diff] [blame] | 97 | ], |
| 98 | }, |
| 99 | |
| 100 | crb: kube.ClusterRoleBinding("nginx-ingress-clusterrole-nisa-binding") { |
| 101 | metadata+: env.metadata { |
| 102 | namespace:: null, |
| 103 | }, |
| 104 | roleRef: { |
| 105 | apiGroup: "rbac.authorization.k8s.io", |
| 106 | kind: "ClusterRole", |
| 107 | name: env.cr.metadata.name, |
| 108 | }, |
| 109 | subjects: [ |
| 110 | { |
| 111 | kind: "ServiceAccount", |
| 112 | name: env.sa.metadata.name, |
| 113 | namespace: env.sa.metadata.namespace, |
| 114 | }, |
| 115 | ], |
| 116 | }, |
| 117 | |
| 118 | role: kube.Role("nginx-ingress-role") { |
| 119 | metadata+: env.metadata, |
| 120 | rules : [ |
| 121 | { |
| 122 | apiGroups: [""], |
Serge Bazanski | 2e8d24b | 2021-03-25 18:39:52 +0100 | [diff] [blame] | 123 | resources: ["namespaces"], |
Sergiusz Bazanski | a9c7e86 | 2019-04-01 17:56:28 +0200 | [diff] [blame] | 124 | verbs: ["get"], |
| 125 | }, |
| 126 | { |
| 127 | apiGroups: [""], |
Serge Bazanski | 2e8d24b | 2021-03-25 18:39:52 +0100 | [diff] [blame] | 128 | resources: ["configmaps", "pods", "secrets", "endpoints"], |
| 129 | verbs: ["get", "list", "watch"], |
| 130 | }, |
| 131 | { |
| 132 | apiGroups: [""], |
| 133 | resources: ["services"], |
| 134 | verbs: ["get", "list", "watch"], |
| 135 | }, |
| 136 | { |
| 137 | apiGroups: ["extensions", "networking.k8s.io"], |
| 138 | resources: ["ingresses"], |
| 139 | verbs: ["get", "list", "watch"], |
| 140 | }, |
| 141 | { |
| 142 | apiGroups: ["extensions", "networking.k8s.io"], |
| 143 | resources: ["ingresses/status"], |
| 144 | verbs: ["update"], |
| 145 | }, |
| 146 | { |
| 147 | apiGroups: ["extensions", "networking.k8s.io"], |
| 148 | resources: ["ingressclasses"], |
| 149 | verbs: ["get", "list", "watch"], |
| 150 | }, |
| 151 | { |
| 152 | apiGroups: [""], |
Sergiusz Bazanski | a9c7e86 | 2019-04-01 17:56:28 +0200 | [diff] [blame] | 153 | resources: ["configmaps"], |
| 154 | resourceNames: ["ingress-controller-leader-nginx"], |
| 155 | verbs: ["get", "update"], |
| 156 | }, |
| 157 | { |
| 158 | apiGroups: [""], |
| 159 | resources: ["configmaps"], |
| 160 | verbs: ["create"], |
| 161 | }, |
| 162 | { |
| 163 | apiGroups: [""], |
Serge Bazanski | 2e8d24b | 2021-03-25 18:39:52 +0100 | [diff] [blame] | 164 | resources: ["events"], |
| 165 | verbs: ["create", "patch"], |
Sergiusz Bazanski | a9c7e86 | 2019-04-01 17:56:28 +0200 | [diff] [blame] | 166 | }, |
| 167 | ], |
| 168 | }, |
| 169 | |
| 170 | roleb: kube.RoleBinding("nginx-ingress-role-nisa-binding") { |
| 171 | metadata+: env.metadata, |
| 172 | roleRef: { |
| 173 | apiGroup: "rbac.authorization.k8s.io", |
| 174 | kind: "Role", |
| 175 | name: env.role.metadata.name, |
| 176 | }, |
| 177 | subjects: [ |
| 178 | { |
| 179 | kind: "ServiceAccount", |
| 180 | name: env.sa.metadata.name, |
| 181 | namespace: env.sa.metadata.namespace, |
| 182 | }, |
| 183 | ], |
| 184 | }, |
| 185 | |
| 186 | service: kube.Service("ingress-nginx") { |
| 187 | metadata+: env.metadata, |
| 188 | target_pod:: env.deployment.spec.template, |
| 189 | spec+: { |
| 190 | type: "LoadBalancer", |
| 191 | ports: [ |
Sergiusz Bazanski | 543b412 | 2019-06-29 22:42:39 +0200 | [diff] [blame] | 192 | { name: "ssh", port: 22, targetPort: 22, protocol: "TCP" }, |
Sergiusz Bazanski | a9c7e86 | 2019-04-01 17:56:28 +0200 | [diff] [blame] | 193 | { name: "http", port: 80, targetPort: 80, protocol: "TCP" }, |
| 194 | { name: "https", port: 443, targetPort: 443, protocol: "TCP" }, |
| 195 | ], |
| 196 | }, |
| 197 | }, |
| 198 | |
Piotr Dobrowolski | f00edf6 | 2020-07-02 18:30:38 +0200 | [diff] [blame] | 199 | serviceGitea: kube.Service("ingress-nginx-gitea") { |
| 200 | metadata+: env.metadata, |
| 201 | target_pod:: env.deployment.spec.template, |
| 202 | spec+: { |
| 203 | type: "LoadBalancer", |
| 204 | loadBalancerIP: "185.236.240.60", |
| 205 | ports: [ |
| 206 | { name: "ssh", port: 22, targetPort: 222, protocol: "TCP" }, |
| 207 | { name: "http", port: 80, targetPort: 80, protocol: "TCP" }, |
| 208 | { name: "https", port: 443, targetPort: 443, protocol: "TCP" }, |
| 209 | ], |
| 210 | }, |
| 211 | }, |
| 212 | |
Sergiusz Bazanski | a9c7e86 | 2019-04-01 17:56:28 +0200 | [diff] [blame] | 213 | deployment: kube.Deployment("nginx-ingress-controller") { |
| 214 | metadata+: env.metadata, |
| 215 | spec+: { |
Sergiusz Bazanski | fd323a0 | 2019-11-17 19:49:04 +0100 | [diff] [blame] | 216 | replicas: 5, |
Sergiusz Bazanski | a9c7e86 | 2019-04-01 17:56:28 +0200 | [diff] [blame] | 217 | template+: { |
| 218 | spec+: { |
| 219 | serviceAccountName: env.sa.metadata.name, |
| 220 | containers_: { |
| 221 | controller: kube.Container("nginx-ingress-controller") { |
| 222 | image: cfg.image, |
Serge Bazanski | 2e8d24b | 2021-03-25 18:39:52 +0100 | [diff] [blame] | 223 | imagePullPolicy: "IfNotPresent", |
| 224 | lifecycle: { |
| 225 | preStop: { |
| 226 | exec: { |
| 227 | command: [ "/wait-shutdown" ], |
| 228 | }, |
| 229 | }, |
| 230 | }, |
Sergiusz Bazanski | a9c7e86 | 2019-04-01 17:56:28 +0200 | [diff] [blame] | 231 | args: [ |
| 232 | "/nginx-ingress-controller", |
Serge Bazanski | 2e8d24b | 2021-03-25 18:39:52 +0100 | [diff] [blame] | 233 | "--election-id=ingress-controller-leader", |
| 234 | "--ingress-class=nginx", |
Sergiusz Bazanski | a9c7e86 | 2019-04-01 17:56:28 +0200 | [diff] [blame] | 235 | "--configmap=%s/%s" % [cfg.namespace, env.maps.configuration.metadata.name], |
| 236 | "--tcp-services-configmap=%s/%s" % [cfg.namespace, env.maps.tcp.metadata.name], |
| 237 | "--udp-services-configmap=%s/%s" % [cfg.namespace, env.maps.udp.metadata.name], |
| 238 | "--publish-service=%s/%s" % [cfg.namespace, env.service.metadata.name], |
| 239 | "--annotations-prefix=nginx.ingress.kubernetes.io", |
| 240 | ], |
| 241 | env_: { |
| 242 | POD_NAME: kube.FieldRef("metadata.name"), |
| 243 | POD_NAMESPACE: kube.FieldRef("metadata.namespace"), |
| 244 | }, |
| 245 | ports_: { |
| 246 | http: { containerPort: 80 }, |
| 247 | https: { containerPort: 443 }, |
| 248 | }, |
| 249 | livenessProbe: { |
| 250 | failureThreshold: 3, |
| 251 | httpGet: { |
| 252 | path: "/healthz", |
| 253 | port: 10254, |
| 254 | scheme: "HTTP", |
| 255 | }, |
| 256 | initialDelaySeconds: 10, |
| 257 | periodSeconds: 10, |
| 258 | successThreshold: 1, |
| 259 | timeoutSeconds: 10, |
| 260 | }, |
| 261 | readinessProbe: { |
| 262 | failureThreshold: 3, |
| 263 | httpGet: { |
| 264 | path: "/healthz", |
| 265 | port: 10254, |
| 266 | scheme: "HTTP", |
| 267 | }, |
| 268 | periodSeconds: 10, |
| 269 | successThreshold: 1, |
| 270 | timeoutSeconds: 10, |
| 271 | }, |
| 272 | securityContext: { |
| 273 | allowPrivilegeEscalation: true, |
| 274 | capabilities: { |
| 275 | drop: ["ALL"], |
| 276 | add: ["NET_BIND_SERVICE"], |
| 277 | }, |
Serge Bazanski | 2e8d24b | 2021-03-25 18:39:52 +0100 | [diff] [blame] | 278 | runAsUser: 101, |
Sergiusz Bazanski | a9c7e86 | 2019-04-01 17:56:28 +0200 | [diff] [blame] | 279 | }, |
Serge Bazanski | 059fdfe | 2020-09-12 21:44:53 +0000 | [diff] [blame] | 280 | resources: { |
| 281 | limits: { cpu: "2", memory: "4G" }, |
| 282 | requests: { cpu: "1", memory: "1G" }, |
| 283 | }, |
Sergiusz Bazanski | a9c7e86 | 2019-04-01 17:56:28 +0200 | [diff] [blame] | 284 | }, |
| 285 | }, |
| 286 | }, |
| 287 | }, |
| 288 | }, |
| 289 | }, |
| 290 | }, |
| 291 | } |