cluster/kube/lib/nginx: use Local traffic policy

Diff against prod:

  - live services nginx-system.ingress-nginx
  + config services nginx-system.ingress-nginx
    {
      "apiVersion": "v1",
      "kind": "Service",
      "metadata": {
        "annotations": {},
        "labels": {
          "app.kubernetes.io/name": "ingress-nginx",
          "app.kubernetes.io/part-of": "ingress-nginx"
        },
        "name": "ingress-nginx",
        "namespace": "nginx-system"
      },
      "spec": {
  -     "externalTrafficPolicy": "Cluster",
  +     "externalTrafficPolicy": "Local",
        "ports": [
          {
            "name": "ssh",
            "port": 22,
            "protocol": "TCP",
            "targetPort": 22
          },
          {
            "name": "http",
            "port": 80,
            "protocol": "TCP",
            "targetPort": 80
          },
          {
            "name": "https",
            "port": 443,
            "protocol": "TCP",
            "targetPort": 443
          }
        ],
        "selector": {
          "app.kubernetes.io/name": "ingress-nginx",
          "app.kubernetes.io/part-of": "ingress-nginx"
        },
        "type": "LoadBalancer"
      }
    }

Change-Id: I0dd66e3f1643efa975d6180cc163a265d4b484ef
diff --git a/cluster/kube/lib/nginx.libsonnet b/cluster/kube/lib/nginx.libsonnet
index a871b96..52ff93c 100644
--- a/cluster/kube/lib/nginx.libsonnet
+++ b/cluster/kube/lib/nginx.libsonnet
@@ -142,6 +142,10 @@
             target_pod:: env.deployment.spec.template,
             spec+: {
                 type: "LoadBalancer",
+                // The nginx ingress instrance is single-instance and metallb l2 (which we run)
+                // does SNAT on 'Cluster', thereby losing the source IP address.
+                // See: https://metallb.universe.tf/usage/#cluster-traffic-policy
+                externalTrafficPolicy: "Local",
                 ports: [
                     { name: "ssh", port: 22, targetPort: 22, protocol: "TCP" },
                     { name: "http", port: 80, targetPort: 80, protocol: "TCP" },