bgpwtf/machines/edge01.waw.bgp.wtf.nix

# Main configuration file for edge01.waw.bgp.wtf.
# This includes everything needed to run the machine, except for hardware
# configuration, which is defined in //bgpwtf/machines/
# edge01.waw.bgp.wtf-hardware.nix.
#
# Any changes here can be tested in a local NixOS test by running the following:
#
#  nix-build -A bgpwtf.machines.tests.edge01-waw
#
# To deploy changes, see //ops:machines.nix.

{ config, pkgs, ... }:

with builtins;

let
  passwords = import ./secrets/plain/passwords.nix;

in rec {
  networking.hostName = "edge01";
  networking.domain = "waw.bgp.wtf";

  imports = [
    ./modules/router.nix
    ./modules/anchorvm.nix
    # Private configuration data - notably, customer data.
    ./secrets/plain/edge01.waw.bgp.wtf-private.nix
  ];

  # TODO(q3k): make this generic, move to modules/router.nix.
  services.unbound = {
    enable = true;
    interfaces = [
      "185.236.240.1"
      "2a0d:eb00:2137::1"
      "127.0.0.1"
    ];
    allowedAccess = [
      "185.236.240.0/22"
      "2a0d:eb00::0/29"
      "127.0.0.0/8"
    ];
    extraConfig = ''
      outgoing-interface: 185.236.240.1
      outgoing-interface: 2a0d:eb00:2137::1
      cache-max-negative-ttl: 30

      # Disable DoH in Firefox
      local-zone: "use-application-dns.net" static

      # Rejestr Stron Hazardowych.
      # Populated by the rsh-unbound daemon.
      include: "/var/lib/unbound/rsh.conf"

      remote-control:
         control-enable: yes
         control-interface: /var/run/unbound.ctl
    '';
  };
  # Fix `systemctl reload unbound`.
  systemd.services.unbound.reload = "${pkgs.unbound}/bin/unbound-control -c /var/lib/unbound/unbound.conf reload";
  hscloud.rsh = {
    enable = true;
    out = "/var/lib/unbound/rsh.conf";
  };

  networking.wireguard.interfaces = {
    wg-fmt = {
      ips = [
        "185.236.240.68/31"
        "2a0d:eb00:2137:1::e/127"
      ];
      allowedIPsAsRoutes = false;
      listenPort = 51820;
      generatePrivateKeyFile = true;
      privateKeyFile = "/root/fmt-wg";
      peers = [
        {
          publicKey = "zxL/1Jr0LLwJwXDm8ZOWkuY3ZkHO3sC7TdSBh89CsWc=";
          allowedIPs = [
            "0.0.0.0/0"
            "::/0"
          ];
          endpoint = "[2a00:6340:4000:10::10]:41521";
        }
      ];
    };
  };

  hscloud.renameInterfaces = {
    # Link to Nitronet CPE.
    e1-nnet.mac = "ac:1f:6b:1c:d7:ae";
    # Link to HSWAW Customs.
    e2-customs.mac = "ac:1f:6b:1c:d7:af";
    # Link to management switch.
    e3-mgmt.mac = "ac:1f:6b:1c:d7:b0";
    # Link to oob1.
    e4-oob.mac = "ac:1f:6b:1c:d7:b1";
    e5.mac = "ac:1f:6b:1c:d7:b2";
    e6.mac = "ac:1f:6b:1c:d7:b3";
    # Link to dcsw01.hswaw.net
    e7-dcsw.mac = "ac:1f:6b:1c:db:06";
    e8.mac = "ac:1f:6b:1c:db:07";
  };
  networking.interfaces.e7-dcsw.mtu = 9000;

  networking.vlans = {
    "vl-globalmix" = { interface = "e1-nnet"; id = 466; };
    "vl-polmix" = { interface = "e1-nnet"; id = 2486; };
    "vl-openpeering" = { interface = "e1-nnet"; id = 992; };

    "vl-dcsw-l3" = { interface = "e7-dcsw"; id = 4001; };
    "vl-dist-l3" = { interface = "e7-dcsw"; id = 3006; };

    # Extra vlans contained in //bgpwtf/machines/secrets/plain/edge01.waw.bgp.wtf-private.nix
  };
  networking.interfaces = {
    lo = {
      ipv4.addresses = [ { address = "185.236.240.1"; prefixLength = 32; } ];
      ipv6.addresses = [ { address = "2a0d:eb00:2137::1"; prefixLength = 64; } ];
    };
    ## EPIX links via Nitronet.
    "vl-globalmix" = {
      ipv4.addresses = [ { address = "185.235.70.45"; prefixLength = 31; } ];
      ipv6.addresses = [ { address = "2001:67c:778:fd40::b9eb:462d"; prefixLength = 127; } ];
    };
    "vl-polmix" = {
      ipv4.addresses = [ { address = "94.246.185.175"; prefixLength = 31; } ];
      ipv6.addresses = [ { address = "2001:67c:778:fa40::5ef6:b9af"; prefixLength = 127; } ];
    };
    "vl-openpeering" = {
      ipv4.addresses = [ { address = "89.46.145.61"; prefixLength = 21; } ];
      ipv6.addresses = [ { address = "2001:678:3ac::313"; prefixLength = 48; } ];
    };

    ## L3/mgmt links..
    # To customs.hackerspace.pl.
    "e2-customs" = {
      ipv4.addresses = [ { address = "185.236.240.4"; prefixLength = 31; } ];
      ipv6.addresses = [ { address = "2a0d:eb00:2137:1::2"; prefixLength = 127; } ];
    };
    # To mgmt.
    "e3-mgmt" = {
      ipv4.addresses = [ { address = "10.10.10.1"; prefixLength = 24; } ];
    };
    # To obb1.
    "e4-oob" = {
      ipv4.addresses = [ { address = "185.236.240.74"; prefixLength = 29; } ];
    };
    # To dcsw01, L3 (BGP).
    "vl-dcsw-l3" = {
      mtu = 9000;
      ipv4.addresses = [ { address = "185.236.240.6"; prefixLength = 31; } ];
      ipv6.addresses = [ { address = "2a0d:eb00:2137:1::6"; prefixLength = 127; } ];
    };
    # To dist02, L3 (BGP).
    "vl-dist-l3" = {
      ipv4.addresses = [ { address = "185.236.240.14"; prefixLength = 31; } ];
      ipv6.addresses = [ { address = "2a0d:eb00:2137:1::a"; prefixLength = 127; } ];
    };
    # VM bridge
    "br0" = {
      ipv4.addresses = [ { address = "185.236.240.17"; prefixLength = 29; } ];
      ipv6.addresses = [ { address = "2a0d:eb00:2137:3::1"; prefixLength = 64; } ];
    };

    # Extra interface configs contained in //bgpwtf/machines/secrets/plain/edge01.waw.bgp.wtf-private.nix
  };
  networking.bridges = {
    "br0" = {
      interfaces = [];
    };
  };
  hscloud.anchorvm = {
    bridge = "br0";
  };

  hscloud.routing.enable = true;
  hscloud.routing.routerID = "185.236.240.1";
  hscloud.routing.asn = 204880;
  # Use default master4/master6 tables so that `birdc show route` works.
  hscloud.routing.tables.master.program = true;
  hscloud.routing.tables.master.programSourceV4 = "185.236.240.1";
  hscloud.routing.tables.master.programSourceV6 = "2a0d:eb00:2137::1";

  hscloud.routing.extra = ''
    function net_martian_v4() {
      return net ~ [ 169.254.0.0/16+, 172.16.0.0/12+, 192.168.0.0/16+, 10.0.0.0/8+,
        127.0.0.0/8+, 224.0.0.0/4+, 240.0.0.0/4+, 0.0.0.0/32-, 0.0.0.0/0{25,32}, 0.0.0.0/0{0,7} ];
    }
    function net_as204480_waw_v4() {
      return net ~ [ 185.236.240.0/23+ ];
    }
    function net_martian_v6() {
      return net ~ [ fc00::/7+, fec0::/10+, ::/128-, ::/0{0,15}, ::/0{49,128} ];
    }
    function net_as204480_waw_v6() {
      return net ~ [ 2a0d:eb00::/32 ];
    }

  '';
  hscloud.routing.originate = {
    # WAW prefixes, exposed into internet BGP table.
    v4.waw = { table = "internet"; address = "185.236.240.0"; prefixLength = 23; };
    v6.waw = { table = "internet"; address = "2a0d:eb00::"; prefixLength = 32; };

    # Default gateway via us, exposed into aggregated table.
    v4.default = { table = "aggregate"; address = "0.0.0.0"; prefixLength = 0; };
    v6.default = { table = "aggregate"; address = "::"; prefixLength = 0; };

    # Loopbacks for IGP table.
    # Alternatively we could add 'lo' as a stub interface into IGP OSPF, but
    # that would also add 127.0.0.1...
    v4.loopbacks = { table = "igp"; address = "185.236.240.1"; prefixLength = 32; };
    v6.loopbacks = { table = "igp"; address = "2a0d:eb00:2137::1"; prefixLength = 128; };
  };
  hscloud.routing.pipe = let
    copySourcesToKernel = sources: table: extra: {
      table = "master";
      peerTable = table;
      filterIn = ''
        ${extra}
        ${concatStringsSep "\n" (map (v: "if source = RTS_${v} then accept;") sources)}
        reject;
      '';
    };
  in {
    v4."internet_to_kernel" = copySourcesToKernel ["BGP" "OSPF"] "internet" "";
    v4."aggregate_to_kernel" = copySourcesToKernel ["BGP" "OSPF"] "aggregate" ''
      # Static v4 routes for customers.
      if proto ~ "static_static_ipv4_customer_*" then accept;
    '';
    v6."internet_to_kernel" = copySourcesToKernel ["BGP" "OSPF"] "internet" "";
    v6."aggregate_to_kernel" = copySourcesToKernel ["BGP" "OSPF"] "aggregate" ''
      # Static v6 routes for customers.
      if proto ~ "static_static_ipv6_customer_*" then accept;
    '';
  };

  hscloud.routing.ospf.v6.main = {
    area."0.0.0.0".interfaces = {
      "e2-customs" = {
        type = "bcast";
      };
      "e4-oob" = {
        type = "bcast";
        stub = true;
      };
    };
    table = "aggregate";
    filterIn = ''
      # hswaw prefix from e2-customs
      if net ~ [ 2a0d:eb00:4242::/48+ ] then accept;
      # e2-customs link
      if net ~ [ 2a0d:eb00:2137:1::2/127+ ] then accept;
    '';
  };
  hscloud.routing.ospf.v4.main = {
    area."0.0.0.0".interfaces = {
      "e4-oob" = {
        type = "bcast";
        stub = true;
      };
    };
    table = "aggregate";
    filterIn = ''
      # e4-oob link
      if net ~ [ 185.236.240.72/29+ ] then accept;
    '';
  };
  hscloud.routing.ospf.v6.igp = {
    area."0.0.0.0".interfaces = {
      "wg-fmt" = {
        type = "ptmp";
        neighbors = [
          "2a0d:eb00:2137:1::f"
        ];
      };
    };
    table = "igp";
    filterIn = ''
      # fmt networks
      if net ~ [ 2a0d:eb01::/48+ ] then accept;
    '';
  };

  hscloud.routing.bgpSessions.v4 = let
    filterInUpstream = ''
      if net_martian_v4() then reject;
      if net_as204480_waw_v4() then reject;
      accept;
    '';
    filterOutUpstream = ''
      # Accept AS204880-announced prefixes.
      if (net ~ [ 185.236.240.0/22+ ]) then accept;
      reject;
    '';
  in {
    "waw_globalmix" = {
      description = "UPSTREAM EPIX.WAR GlobalMix";
      table = "internet";
      local = "185.235.70.45";
      neighbors = [
        { address = "185.235.70.44"; asn = 62081; }
      ];
      prepend = 2; pref = 100;
      filterIn = filterInUpstream;
      filterOut = filterOutUpstream;
    };
    "waw_polmix" = {
      description = "UPSTREAM EPIX.WAR PolMix";
      table = "internet";
      local = "94.246.185.175";
      neighbors = [
        { address = "94.246.185.174"; asn = 201054; }
      ];
      prepend = 1; pref = 200;
      filterIn = filterInUpstream;
      filterOut = filterOutUpstream;
    };
    "waw_openpeering" = {
      description = "IXP EPIX.WAR OpenPeering";
      table = "internet";
      local = "89.46.145.61";
      neighbors = [
        { address = "89.46.144.11"; asn = 48850; }
        { address = "89.46.144.12"; asn = 48850; }
      ];
      prepend = 0; pref = 300;
      filterIn = filterInUpstream;
      filterOut = filterOutUpstream;
    };
    "waw_google" = {
      description = "PEER Google AS15169 (EPIX)";
      table = "internet";
      local = "89.46.145.61";
      neighbors = [
        # TODO(q3k): secretify the password.
        { address = "89.46.144.185"; asn = 15169; password = passwords."edge01.waw-bgp-google"; }
      ];
      prepend = 0; pref = 300;
      filterIn = filterInUpstream;
      filterOut = filterOutUpstream;
    };
    # hscloud spine switch (dcsw01.hswaw.net).
    "waw_hscloud" = {
      description = "AGGREGATE CUSTOMER hscloud/dcsw01";
      table = "aggregate";
      local = "185.236.240.6";
      asn = 65000;
      neighbors = [
        { address = "185.236.240.7"; asn = 65001; }
      ];
      filterIn = ''
        # wieloryb prefix
        if net ~ [ 185.236.240.8/31+ ] then accept;
        # dcsw01 l2 general purpose
        if net ~ [ 185.236.240.24/29+ ] then accept;
        # hscloud l2 general purpose
        if net ~ [ 185.236.240.32/28+ ] then accept;
        # k0 metallb pools
        if net ~ [ 185.236.240.48/28+, 185.236.240.112/28+ ] then accept;
        #  dcsw01.hswaw.net / dcr03sw48.hswaw.net 
        if net ~ [ 185.236.240.66/31 ] then accept;
        reject;
      '';
    };
    # bgp.wtf internet customer router on W2A, floor 3 (dist02.bgp.wtf).
    "waw_dist02" = {
      description = "AGGREGATE CUSTOMER bgpwtf/dist02";
      table = "aggregate";
      local = "185.236.240.14";
      asn = 65000;
      neighbors = [
        { address = "185.236.240.15"; asn = 65002; }
      ];
      filterIn = ''
        # dist02 customer routed
        if net ~ [  185.236.240.80/28+ ] then accept;
        reject;
      '';
    };
    # backup LTE link to edge01.fra
    "fra_edge01" = {
      description = "IBGP edge01.fra";
      table = "internet";
      local = "185.236.240.74";
      direct = true;
      neighbors = [
        { address = "185.236.240.75"; asn = 204880; }
      ];
      pref = 50;
      filterIn = filterInUpstream;
      filterOut = filterOutUpstream;
    };
  };
  hscloud.routing.bgpSessions.v6 = let
    filterInUpstream = ''
      if net_martian_v6() then reject;
      if net_as204480_waw_v6() then reject;
      accept;
    '';
    filterOutUpstream = ''
      # Accept AS204880-announced prefixes.
      if (net ~ [ 2a0d:eb00::/29+ ]) then accept;
      reject;
    '';
  in {
    "waw_globalmix" = {
      description = "UPSTREAM EPIX.WAR GlobalMix";
      table = "internet";
      local = "2001:67c:778:fd40::b9eb:462d";
      neighbors = [
        { address = "2001:67c:778:fd40::b9eb:462c"; asn = 62081; }
      ];
      prepend = 2; pref = 100;
      filterIn = filterInUpstream;
      filterOut = filterOutUpstream;
    };
    "waw_polmix" = {
      description = "UPSTREAM EPIX.WAR PolMix";
      table = "internet";
      local = "2001:67c:778:fa40::5ef6:b9af";
      neighbors = [
        { address = "2001:67c:778:fa40::5ef6:b9ae"; asn = 201054; }
      ];
      prepend = 1; pref = 200;
      filterIn = filterInUpstream;
      filterOut = filterOutUpstream;
    };
    "waw_openpeering" = {
      description = "IXP EPIX.WAR OpenPeering";
      table = "internet";
      local = "2001:678:3ac::313";
      neighbors = [
        { address = "2001:678:3ac::11"; asn = 48850; }
        { address = "2001:678:3ac::12"; asn = 48850; }
      ];
      prepend = 0; pref = 300;
      filterIn = filterInUpstream;
      filterOut = filterOutUpstream;
    };
    "waw_google" = {
      description = "PEER Google AS15169 (EPIX)";
      table = "internet";
      local = "2001:678:3ac::313";
      neighbors = [
        { address = "2001:678:3ac::185"; asn = 15169; password = passwords."edge01.waw-bgp-google"; }
      ];
      prepend = 0; pref = 300;
      filterIn = filterInUpstream;
      filterOut = filterOutUpstream;
    };
    # hscloud spine switch (dcsw01.hswaw.net).
    "waw_hscloud" = {
      description = "AGGREGATE CUSTOMER dcsw01.hswaw.net";
      table = "aggregate";
      local = "2a0d:eb00:2137:1::6";
      asn = 65000;
      neighbors = [
        { address = "2a0d:eb00:2137:1::7"; asn = 65001; }
      ];
      filterIn = ''
        # dcsw01 l2 general purpose
        if net ~ [ 2a0d:eb00:2137::/48+ ] then accept;
        # customer
        if net ~ [ 2a0d:eb00:8004::/48+ ] then accept;
        reject;
      '';
    };
    # bgp.wtf internet customer router on W2A, floor 3 (dist02.bgp.wtf).
    "waw_dist02" = {
      description = "AGGREGATE CUSTOMER dist02.bgp.wtf";
      table = "aggregate";
      local = "2a0d:eb00:2137:1::a";
      asn = 65000;
      neighbors = [
        { address = "2a0d:eb00:2137:1::b"; asn = 65002; }
      ];
      filterIn = ''
        # dist02 customers.
        if net ~ [ 2a0d:eb00:8002::/48 ] then accept;
        reject;
      '';
    };
  };
}