blob: acb7b44b8794338217b64e52de9e7a5d32e28440 [file] [log] [blame]
# Deploy a per-cluster Nginx Ingress Controller
local kube = import "../../../kube/kube.libsonnet";
local policies = import "../../../kube/policies.libsonnet";
{
Environment: {
local env = self,
local cfg = env.cfg,
cfg:: {
# Built from nginx-ingress-controller/Dockerfile:
#
# $ cd cluster/kube/lib/nginx-ingress-controller
# $ docker build -t eu.gcr.io/bgpwtf/nginx-ingress-controller:v0.44.0-r1 .
# [..]
# (2/8) Upgrading libcrypto1.1 (1.1.1i-r0 -> 1.1.1k-r0)
# (3/8) Upgrading libssl1.1 (1.1.1i-r0 -> 1.1.1k-r0
# [...]
# (8/8) Upgrading openssl (1.1.1i-r0 -> 1.1.1k-r0)
# $ docker push eu.gcr.io/bgpwtf/nginx-ingress-controller:v0.44.0-r1
#
# TODO(q3k): unfork this once openssl 1.1.1k lands in upstream
# nginx-ingress-controller.
image: "eu.gcr.io/bgpwtf/nginx-ingress-controller:v0.44.0-r1",
namespace: "nginx-system",
},
metadata:: {
namespace: cfg.namespace,
labels: {
"app.kubernetes.io/name": "ingress-nginx",
"app.kubernetes.io/part-of": "ingress-nginx",
},
},
namespace: kube.Namespace(cfg.namespace),
allowInsecure: policies.AllowNamespaceInsecure(cfg.namespace),
maps: {
make(name):: kube.ConfigMap(name) {
metadata+: env.metadata,
},
configuration: env.maps.make("nginx-configuration") {
data: {
"proxy-set-headers": "%s/nginx-custom-headers" % [cfg.namespace],
},
},
customHeaders: env.maps.make("nginx-custom-headers") {
data: {
# RFC6648 deprecates X-prefixed headers as a convention in
# multiple application protocols, including HTTP. It
# recommends that any new headers should just start off
# with a final standardized name, ie. suggests to use
# Toaster-ID instead of X-Toaster-ID.
#
# However, it also acknowledges that headers likely to
# never be standardized can still be prefixed with OrgName-
# or other constructs. And since we're not even attempting
# to standardize anything here, this is what we use to
# prefix hscloud-specific headers.
#
# Hscloud == hscloud, this repository.
# Nic == nginx-ingress-controller, this ingress controller.
# Set source port/addr. Source-IP duplicates
# X-Forwarded-For, but is added for consistency with
# Source-Port.
#
# Source-IP is an IP address in two possible formats:
# IPv4: "1.2.3.4"
# IPv6: "2a0d:1234::42"
# Any other format received by services should be
# considered invalid, and the service should assume a
# misconfiguration of the N-I-C.
"Hscloud-Nic-Source-IP": "${remote_addr}",
# Source-Port is a stringified TCP port, encoding a port
# number from 1 to 65535. Any other value received by
# services should be considered invalid, and the service
# should assume a misconfiguration of the N-I-C.
"Hscloud-Nic-Source-Port": "${remote_port}",
},
},
tcp: env.maps.make("tcp-services") {
data: {
"22": "gerrit/gerrit:22",
}
},
udp: env.maps.make("udp-services"),
},
sa: kube.ServiceAccount("nginx-ingress-serviceaccount") {
metadata+: env.metadata,
},
cr: kube.ClusterRole("nginx-ingress-clusterrole") {
metadata+: env.metadata {
namespace:: null,
},
rules: [
{
apiGroups: [""],
resources: ["configmaps", "endpoints", "nodes", "pods", "secrets"],
verbs: ["list", "watch"],
},
{
apiGroups: [""],
resources: ["nodes"],
verbs: ["get"],
},
{
apiGroups: [""],
resources: ["services"],
verbs: ["get", "list", "watch"],
},
{
apiGroups: ["extensions", "networking.k8s.io"],
resources: ["ingresses"],
verbs: ["get", "list", "watch"],
},
{
apiGroups: [""],
resources: ["events"],
verbs: ["create", "patch"],
},
{
apiGroups: ["extensions", "networking.k8s.io"],
resources: ["ingresses/status"],
verbs: ["update"],
},
{
apiGroups: ["extensions", "networking.k8s.io"],
resources: ["ingressclasses"],
verbs: ["get", "list", "watch"],
},
],
},
crb: kube.ClusterRoleBinding("nginx-ingress-clusterrole-nisa-binding") {
metadata+: env.metadata {
namespace:: null,
},
roleRef: {
apiGroup: "rbac.authorization.k8s.io",
kind: "ClusterRole",
name: env.cr.metadata.name,
},
subjects: [
{
kind: "ServiceAccount",
name: env.sa.metadata.name,
namespace: env.sa.metadata.namespace,
},
],
},
role: kube.Role("nginx-ingress-role") {
metadata+: env.metadata,
rules : [
{
apiGroups: [""],
resources: ["namespaces"],
verbs: ["get"],
},
{
apiGroups: [""],
resources: ["configmaps", "pods", "secrets", "endpoints"],
verbs: ["get", "list", "watch"],
},
{
apiGroups: [""],
resources: ["services"],
verbs: ["get", "list", "watch"],
},
{
apiGroups: ["extensions", "networking.k8s.io"],
resources: ["ingresses"],
verbs: ["get", "list", "watch"],
},
{
apiGroups: ["extensions", "networking.k8s.io"],
resources: ["ingresses/status"],
verbs: ["update"],
},
{
apiGroups: ["extensions", "networking.k8s.io"],
resources: ["ingressclasses"],
verbs: ["get", "list", "watch"],
},
{
apiGroups: [""],
resources: ["configmaps"],
resourceNames: ["ingress-controller-leader-nginx"],
verbs: ["get", "update"],
},
{
apiGroups: [""],
resources: ["configmaps"],
verbs: ["create"],
},
{
apiGroups: [""],
resources: ["events"],
verbs: ["create", "patch"],
},
],
},
roleb: kube.RoleBinding("nginx-ingress-role-nisa-binding") {
metadata+: env.metadata,
roleRef: {
apiGroup: "rbac.authorization.k8s.io",
kind: "Role",
name: env.role.metadata.name,
},
subjects: [
{
kind: "ServiceAccount",
name: env.sa.metadata.name,
namespace: env.sa.metadata.namespace,
},
],
},
service: kube.Service("ingress-nginx") {
metadata+: env.metadata,
target:: env.deployment,
spec+: {
type: "LoadBalancer",
ports: [
{ name: "ssh", port: 22, targetPort: 22, protocol: "TCP" },
{ name: "http", port: 80, targetPort: 80, protocol: "TCP" },
{ name: "https", port: 443, targetPort: 443, protocol: "TCP" },
],
},
},
serviceGitea: kube.Service("ingress-nginx-gitea") {
metadata+: env.metadata,
target:: env.deployment,
spec+: {
type: "LoadBalancer",
loadBalancerIP: "185.236.240.60",
ports: [
{ name: "ssh", port: 22, targetPort: 222, protocol: "TCP" },
{ name: "http", port: 80, targetPort: 80, protocol: "TCP" },
{ name: "https", port: 443, targetPort: 443, protocol: "TCP" },
],
},
},
deployment: kube.Deployment("nginx-ingress-controller") {
metadata+: env.metadata,
spec+: {
replicas: 5,
template+: {
spec+: {
serviceAccountName: env.sa.metadata.name,
containers_: {
controller: kube.Container("nginx-ingress-controller") {
image: cfg.image,
imagePullPolicy: "IfNotPresent",
lifecycle: {
preStop: {
exec: {
command: [ "/wait-shutdown" ],
},
},
},
args: [
"/nginx-ingress-controller",
"--election-id=ingress-controller-leader",
"--ingress-class=nginx",
"--configmap=%s/%s" % [cfg.namespace, env.maps.configuration.metadata.name],
"--tcp-services-configmap=%s/%s" % [cfg.namespace, env.maps.tcp.metadata.name],
"--udp-services-configmap=%s/%s" % [cfg.namespace, env.maps.udp.metadata.name],
"--publish-service=%s/%s" % [cfg.namespace, env.service.metadata.name],
"--annotations-prefix=nginx.ingress.kubernetes.io",
],
env_: {
POD_NAME: kube.FieldRef("metadata.name"),
POD_NAMESPACE: kube.FieldRef("metadata.namespace"),
},
ports_: {
http: { containerPort: 80 },
https: { containerPort: 443 },
},
livenessProbe: {
failureThreshold: 3,
httpGet: {
path: "/healthz",
port: 10254,
scheme: "HTTP",
},
initialDelaySeconds: 10,
periodSeconds: 10,
successThreshold: 1,
timeoutSeconds: 10,
},
readinessProbe: {
failureThreshold: 3,
httpGet: {
path: "/healthz",
port: 10254,
scheme: "HTTP",
},
periodSeconds: 10,
successThreshold: 1,
timeoutSeconds: 10,
},
securityContext: {
allowPrivilegeEscalation: true,
capabilities: {
drop: ["ALL"],
add: ["NET_BIND_SERVICE"],
},
runAsUser: 101,
},
resources: {
limits: { cpu: "2", memory: "4G" },
requests: { cpu: "1", memory: "1G" },
},
},
},
},
},
},
},
},
}