cluster/kube/lib/nginx.libsonnet

# Deploy a per-cluster Nginx Ingress Controller

local kube = import "../../../kube/kube.libsonnet";
local policies = import "../../../kube/policies.libsonnet";

{
    Environment: {
        local env = self,
        local cfg = env.cfg,
        cfg:: {
            # Built from nginx-ingress-controller/Dockerfile:
            #
            #   $ cd cluster/kube/lib/nginx-ingress-controller
            #   $ docker build -t eu.gcr.io/bgpwtf/nginx-ingress-controller:v0.44.0-r1 .
            #   [..]
            #   (2/8) Upgrading libcrypto1.1 (1.1.1i-r0 -> 1.1.1k-r0)
            #   (3/8) Upgrading libssl1.1 (1.1.1i-r0 -> 1.1.1k-r0
            #   [...]
            #   (8/8) Upgrading openssl (1.1.1i-r0 -> 1.1.1k-r0)
            #   $ docker push eu.gcr.io/bgpwtf/nginx-ingress-controller:v0.44.0-r1
            #
            # TODO(q3k): unfork this once openssl 1.1.1k lands in upstream
            # nginx-ingress-controller.
            image: "eu.gcr.io/bgpwtf/nginx-ingress-controller:v0.44.0-r1",
            namespace: "nginx-system",
        },

        metadata:: {
            namespace: cfg.namespace,
            labels: {
                "app.kubernetes.io/name": "ingress-nginx",
                "app.kubernetes.io/part-of": "ingress-nginx",
            },
        },

        namespace: kube.Namespace(cfg.namespace),

        allowInsecure: policies.AllowNamespaceInsecure(cfg.namespace),

        maps: {
            make(name):: kube.ConfigMap(name) {
                metadata+: env.metadata,
            },
            configuration: env.maps.make("nginx-configuration") {
                data: {
                    "proxy-set-headers": "%s/nginx-custom-headers" % [cfg.namespace],
                },
            },
            customHeaders: env.maps.make("nginx-custom-headers") {
                data: {
                    # RFC6648 deprecates X-prefixed headers as a convention in
                    # multiple application protocols, including HTTP. It
                    # recommends that any new headers should just start off
                    # with a final standardized name, ie. suggests to use
                    # Toaster-ID instead of X-Toaster-ID.
                    #
                    # However, it also acknowledges that headers likely to
                    # never be standardized can still be prefixed with OrgName-
                    # or other constructs. And since we're not even attempting
                    # to standardize anything here, this is what we use to
                    # prefix hscloud-specific headers.
                    #
                    # Hscloud == hscloud, this repository.
                    # Nic == nginx-ingress-controller, this ingress controller.

                    # Set source port/addr. Source-IP duplicates
                    # X-Forwarded-For, but is added for consistency with
                    # Source-Port.
                    #
                    # Source-IP is an IP address in two possible formats:
                    #   IPv4: "1.2.3.4"
                    #   IPv6: "2a0d:1234::42"
                    # Any other format received by services should be
                    # considered invalid, and the service should assume a
                    # misconfiguration of the N-I-C.
                    "Hscloud-Nic-Source-IP": "${remote_addr}",
                    # Source-Port is a stringified TCP port, encoding a port
                    # number from 1 to 65535. Any other value received by
                    # services should be considered invalid, and the service
                    # should assume a misconfiguration of the N-I-C.
                    "Hscloud-Nic-Source-Port": "${remote_port}",
                },
            },
            tcp: env.maps.make("tcp-services") {
                data: {
                    "22": "gerrit/gerrit:22",
                }
            },
            udp: env.maps.make("udp-services"),
        },

        sa: kube.ServiceAccount("nginx-ingress-serviceaccount") {
            metadata+: env.metadata,
        },

        cr: kube.ClusterRole("nginx-ingress-clusterrole") {
            metadata+: env.metadata {
                namespace:: null,
            },
            rules: [
                {
                    apiGroups: [""],
                    resources: ["configmaps", "endpoints", "nodes", "pods", "secrets"],
                    verbs: ["list", "watch"],
                },
                {
                    apiGroups: [""],
                    resources: ["nodes"],
                    verbs: ["get"],
                },
                {
                    apiGroups: [""],
                    resources: ["services"],
                    verbs: ["get", "list", "watch"],
                },
                {
                    apiGroups: ["extensions", "networking.k8s.io"],
                    resources: ["ingresses"],
                    verbs: ["get", "list", "watch"],
                },
                {
                    apiGroups: [""],
                    resources: ["events"],
                    verbs: ["create", "patch"],
                },
                {
                    apiGroups: ["extensions", "networking.k8s.io"],
                    resources: ["ingresses/status"],
                    verbs: ["update"],
                },
                {
                    apiGroups: ["extensions", "networking.k8s.io"],
                    resources: ["ingressclasses"],
                    verbs: ["get", "list", "watch"],
                },
            ],
        },

        crb: kube.ClusterRoleBinding("nginx-ingress-clusterrole-nisa-binding") {
            metadata+: env.metadata {
                namespace:: null,
            },
            roleRef: {
                apiGroup: "rbac.authorization.k8s.io",
                kind: "ClusterRole",
                name: env.cr.metadata.name,
            },
            subjects: [
                {
                    kind: "ServiceAccount",
                    name: env.sa.metadata.name,
                    namespace: env.sa.metadata.namespace,
                },
            ],
        },

        role: kube.Role("nginx-ingress-role") {
            metadata+: env.metadata,
            rules : [
                {
                    apiGroups: [""],
                    resources: ["namespaces"],
                    verbs: ["get"],
                },
                {
                    apiGroups: [""],
                    resources: ["configmaps", "pods", "secrets", "endpoints"],
                    verbs: ["get", "list", "watch"],
                },
                {
                    apiGroups: [""],
                    resources: ["services"],
                    verbs: ["get", "list", "watch"],
                },
                {
                    apiGroups: ["extensions", "networking.k8s.io"],
                    resources: ["ingresses"],
                    verbs: ["get", "list", "watch"],
                },
                {
                    apiGroups: ["extensions", "networking.k8s.io"],
                    resources: ["ingresses/status"],
                    verbs: ["update"],
                },
                {
                    apiGroups: ["extensions", "networking.k8s.io"],
                    resources: ["ingressclasses"],
                    verbs: ["get", "list", "watch"],
                },
                {
                    apiGroups: [""],
                    resources: ["configmaps"],
                    resourceNames: ["ingress-controller-leader-nginx"],
                    verbs: ["get", "update"],
                },
                {
                    apiGroups: [""],
                    resources: ["configmaps"],
                    verbs: ["create"],
                },
                {
                    apiGroups: [""],
                    resources: ["events"],
                    verbs: ["create", "patch"],
                },
            ],
        },

        roleb: kube.RoleBinding("nginx-ingress-role-nisa-binding") {
            metadata+: env.metadata,
            roleRef: {
                apiGroup: "rbac.authorization.k8s.io",
                kind: "Role",
                name: env.role.metadata.name,
            },
            subjects: [
                {
                    kind: "ServiceAccount",
                    name: env.sa.metadata.name,
                    namespace: env.sa.metadata.namespace,
                },
            ],
        },

        service: kube.Service("ingress-nginx") {
            metadata+: env.metadata,
            target_pod:: env.deployment.spec.template,
            spec+: {
                type: "LoadBalancer",
                ports: [
                    { name: "ssh", port: 22, targetPort: 22, protocol: "TCP" },
                    { name: "http", port: 80, targetPort: 80, protocol: "TCP" },
                    { name: "https", port: 443, targetPort: 443, protocol: "TCP" },
                ],
            },
        },

        serviceGitea: kube.Service("ingress-nginx-gitea") {
            metadata+: env.metadata,
            target_pod:: env.deployment.spec.template,
            spec+: {
                type: "LoadBalancer",
                loadBalancerIP: "185.236.240.60",
                ports: [
                    { name: "ssh", port: 22, targetPort: 222, protocol: "TCP" },
                    { name: "http", port: 80, targetPort: 80, protocol: "TCP" },
                    { name: "https", port: 443, targetPort: 443, protocol: "TCP" },
                ],
            },
        },

        deployment: kube.Deployment("nginx-ingress-controller") {
            metadata+: env.metadata,
            spec+: {
                replicas: 5,
                template+: {
                    spec+: {
                        serviceAccountName: env.sa.metadata.name,
                        containers_: {
                            controller: kube.Container("nginx-ingress-controller") {
                                image: cfg.image,
                                imagePullPolicy: "IfNotPresent",
                                lifecycle: {
                                    preStop: {
                                        exec: {
                                            command: [ "/wait-shutdown" ],
                                        },
                                    },
                                },
                                args: [
                                    "/nginx-ingress-controller",
                                    "--election-id=ingress-controller-leader",
                                    "--ingress-class=nginx",
                                    "--configmap=%s/%s" % [cfg.namespace, env.maps.configuration.metadata.name],
                                    "--tcp-services-configmap=%s/%s" % [cfg.namespace, env.maps.tcp.metadata.name],
                                    "--udp-services-configmap=%s/%s" % [cfg.namespace, env.maps.udp.metadata.name],
                                    "--publish-service=%s/%s" % [cfg.namespace, env.service.metadata.name],
                                    "--annotations-prefix=nginx.ingress.kubernetes.io",
                                ],
                                env_: {
                                    POD_NAME: kube.FieldRef("metadata.name"),
                                    POD_NAMESPACE: kube.FieldRef("metadata.namespace"),
                                },
                                ports_: {
                                    http: { containerPort: 80 },
                                    https: { containerPort: 443 },
                                },
                                livenessProbe: {
                                    failureThreshold: 3,
                                    httpGet: {
                                        path: "/healthz",
                                        port: 10254,
                                        scheme: "HTTP",
                                    },
                                    initialDelaySeconds: 10,
                                    periodSeconds: 10,
                                    successThreshold: 1,
                                    timeoutSeconds: 10,
                                },
                                readinessProbe: {
                                    failureThreshold: 3,
                                    httpGet: {
                                        path: "/healthz",
                                        port: 10254,
                                        scheme: "HTTP",
                                    },
                                    periodSeconds: 10,
                                    successThreshold: 1,
                                    timeoutSeconds: 10,
                                },
                                securityContext: {
                                    allowPrivilegeEscalation: true,
                                    capabilities: {
                                        drop: ["ALL"],
                                        add: ["NET_BIND_SERVICE"],
                                    },
                                    runAsUser: 101,
                                },
                                resources: {
                                    limits: { cpu: "2", memory: "4G" },
                                    requests: { cpu: "1", memory: "1G" },
                                },
                            },
                        },
                    },
                },
            },
        },
    },
}