Serge Bazanski | 6abe4fa | 2020-10-03 00:18:34 +0200 | [diff] [blame] | 1 | # Smoke test edge01.waw in a multi-VM NixOS test. |
| 2 | # |
| 3 | # This brings up three VMs: |
| 4 | # - dut/edge01 |
| 5 | # - bgpspeaker, which simulates bgp upstreams |
| 6 | # - customs, which simulates customs.hackerspace.pl. |
| 7 | # |
Serge Bazanski | cc769a5 | 2021-02-13 13:13:41 +0100 | [diff] [blame] | 8 | # We use GRETap to build up virtual ethernet links between the machines, and to |
| 9 | # run VLANs on that. We don't just use plain 'vlans' from NixOS tests as we |
| 10 | # actually want to run 802.1q ourselves from the edge01 config. |
Serge Bazanski | 6abe4fa | 2020-10-03 00:18:34 +0200 | [diff] [blame] | 11 | # |
| 12 | # Everything else is pretty much straightforward. Bring up everything, ping |
| 13 | # stuff. We don't really test much else than internet routing. |
| 14 | # |
| 15 | # To run this: |
Serge Bazanski | e1aa63c | 2022-06-12 12:27:56 +0200 | [diff] [blame] | 16 | # nix-build -A bgpwtf.machines.tests.edge01-waw-bgp |
Serge Bazanski | 6abe4fa | 2020-10-03 00:18:34 +0200 | [diff] [blame] | 17 | # |
| 18 | # To debug this: |
Serge Bazanski | e1aa63c | 2022-06-12 12:27:56 +0200 | [diff] [blame] | 19 | # nix-build -A bgpwtf.machines.tests.edge01-waw-bgp.driver && result/bin/nixos-test-driver |
Serge Bazanski | 6abe4fa | 2020-10-03 00:18:34 +0200 | [diff] [blame] | 20 | # >>> start_all() |
| 21 | |
Serge Bazanski | d635dc8 | 2022-06-11 21:54:04 +0200 | [diff] [blame] | 22 | { hscloud, ... }: |
Serge Bazanski | 6abe4fa | 2020-10-03 00:18:34 +0200 | [diff] [blame] | 23 | |
Serge Bazanski | d635dc8 | 2022-06-11 21:54:04 +0200 | [diff] [blame] | 24 | # Use pkgs that edge01 is using. Perhaps we shouldn't use them for |
| 25 | # _everything_, but this will have to do. |
| 26 | let |
| 27 | pkgs = hscloud.ops.machines."edge01.waw.bgp.wtf".pkgs; |
| 28 | pkgsSrc = pkgs.path; |
| 29 | lib = pkgs.lib; |
| 30 | |
| 31 | in with lib; |
Serge Bazanski | 6abe4fa | 2020-10-03 00:18:34 +0200 | [diff] [blame] | 32 | |
| 33 | let |
| 34 | |
| 35 | mkBGPSpeaker = let |
| 36 | in { config, pkgs, ... }: { |
| 37 | networking.hostName = "bgpspeaker"; |
| 38 | virtualisation.memorySize = 1024; |
| 39 | virtualisation.vlans = [ 1 ]; |
| 40 | imports = [ |
Serge Bazanski | cc769a5 | 2021-02-13 13:13:41 +0100 | [diff] [blame] | 41 | ../modules/gretap.nix |
Serge Bazanski | 6abe4fa | 2020-10-03 00:18:34 +0200 | [diff] [blame] | 42 | ]; |
Serge Bazanski | cc769a5 | 2021-02-13 13:13:41 +0100 | [diff] [blame] | 43 | hscloud.gretap.interfaces."nnet" = { |
Serge Bazanski | 6abe4fa | 2020-10-03 00:18:34 +0200 | [diff] [blame] | 44 | parent = "eth1"; |
| 45 | localV4 = "192.168.1.3"; |
| 46 | remoteV4 = "192.168.1.2"; |
| 47 | id = 100; |
| 48 | }; |
Serge Bazanski | 6abe4fa | 2020-10-03 00:18:34 +0200 | [diff] [blame] | 49 | networking.vlans = { |
| 50 | "vl-globalmix" = { interface = "nnet"; id = 466; }; |
| 51 | }; |
| 52 | networking.interfaces."vl-globalmix" = { |
| 53 | ipv4.addresses = [{ address = "185.235.70.44"; prefixLength = 31; }]; |
| 54 | ipv6.addresses = [{ address = "2001:67c:778:fd40::b9eb:462c"; prefixLength = 127; }]; |
| 55 | }; |
| 56 | |
| 57 | services.bird2 = { |
| 58 | enable = true; |
| 59 | config = '' |
| 60 | log syslog all; |
| 61 | debug protocols { states, interfaces, events } |
| 62 | router id 185.235.70.44; |
| 63 | |
| 64 | protocol device { |
| 65 | scan time 10; |
| 66 | }; |
| 67 | protocol kernel kernel_v4 { |
| 68 | ipv4 { |
| 69 | import none; |
| 70 | export all; |
| 71 | }; |
| 72 | } |
| 73 | protocol kernel kernel_v6 { |
| 74 | ipv6 { |
| 75 | import none; |
| 76 | export all; |
| 77 | }; |
| 78 | } |
| 79 | ipv4 table globalmix4; |
| 80 | ipv6 table globalmix6; |
| 81 | |
| 82 | protocol pipe pipe_globalmix4 { |
| 83 | table master4; |
| 84 | peer table globalmix4; |
| 85 | import all; |
| 86 | export none; |
| 87 | }; |
| 88 | protocol pipe pipe_globalmix6 { |
| 89 | table master6; |
| 90 | peer table globalmix6; |
| 91 | import all; |
| 92 | export none; |
| 93 | }; |
| 94 | |
| 95 | protocol static static_globalmix_originate_v4 { |
| 96 | ipv4 { |
| 97 | table globalmix4; |
| 98 | import all; |
| 99 | }; |
| 100 | route 8.8.8.0/24 blackhole; |
| 101 | } |
| 102 | protocol static static_globalmix_originate_v6 { |
| 103 | ipv6 { |
| 104 | table globalmix6; |
| 105 | import all; |
| 106 | }; |
| 107 | route 2a00:1450:4016::/48 blackhole; |
| 108 | } |
| 109 | protocol bgp bgp_globalmix_v4 { |
| 110 | ipv4 { |
| 111 | table globalmix4; |
| 112 | export all; |
| 113 | import all; |
| 114 | }; |
| 115 | local 185.235.70.44 as 62081; |
| 116 | neighbor 185.235.70.45 as 204880; |
| 117 | }; |
| 118 | protocol bgp bgp_globalmix_v6 { |
| 119 | ipv6 { |
| 120 | table globalmix6; |
| 121 | export all; |
| 122 | import all; |
| 123 | }; |
| 124 | local 2001:67c:778:fd40::b9eb:462c as 62081; |
| 125 | neighbor 2001:67c:778:fd40::b9eb:462d as 204880; |
| 126 | }; |
| 127 | ''; |
| 128 | }; |
| 129 | networking.firewall.enable = false; |
| 130 | networking.useDHCP = false; |
| 131 | networking.interfaces.lo.ipv4.addresses = [ { address = "8.8.8.1"; prefixLength = 32; } ]; |
| 132 | networking.interfaces.lo.ipv6.addresses = [ { address = "2a00:1450:4016:801::200e"; prefixLength = 128; } ]; |
| 133 | environment.systemPackages = with pkgs; [ |
| 134 | tcpdump htop dstat file |
| 135 | ]; |
| 136 | |
| 137 | }; |
| 138 | |
| 139 | |
| 140 | test = import "${pkgsSrc}/nixos/tests/make-test-python.nix" ({ pkgs, libs, ... }: { |
Serge Bazanski | e1aa63c | 2022-06-12 12:27:56 +0200 | [diff] [blame] | 141 | name = "test-edge01-waw-bgp"; |
Serge Bazanski | 6abe4fa | 2020-10-03 00:18:34 +0200 | [diff] [blame] | 142 | |
| 143 | nodes = { |
| 144 | dut = { config, pkgs, ... }: { |
| 145 | imports = [ |
| 146 | ../edge01.waw.bgp.wtf.nix |
Serge Bazanski | cc769a5 | 2021-02-13 13:13:41 +0100 | [diff] [blame] | 147 | ../modules/gretap.nix |
Serge Bazanski | 6abe4fa | 2020-10-03 00:18:34 +0200 | [diff] [blame] | 148 | ]; |
| 149 | virtualisation.memorySize = 1024; |
| 150 | virtualisation.vlans = [ |
| 151 | 1 2 |
| 152 | ]; |
| 153 | |
Serge Bazanski | cc769a5 | 2021-02-13 13:13:41 +0100 | [diff] [blame] | 154 | hscloud.gretap.interfaces = { |
Serge Bazanski | 6abe4fa | 2020-10-03 00:18:34 +0200 | [diff] [blame] | 155 | "e1-nnet" = { parent = "eth1"; localV4 = "192.168.1.2"; remoteV4 = "192.168.1.3"; id = 100; }; |
| 156 | "e2-customs" = { parent = "eth2"; localV4 = "192.168.2.2"; remoteV4 = "192.168.2.1"; id = 200; }; |
| 157 | "e3-mgmt" = { parent = "eth2"; localV4 = "192.168.2.2"; remoteV4 = "192.168.2.111"; id = 300; }; # not connected |
| 158 | "e4-oob" = { parent = "eth2"; localV4 = "192.168.2.2"; remoteV4 = "192.168.2.112"; id = 400; }; # not connected |
| 159 | "e7-dcsw" = { parent = "eth2"; localV4 = "192.168.2.2"; remoteV4 = "192.168.2.113"; id = 500; }; # not connected |
| 160 | }; |
Bartosz Stebel | c726798 | 2020-12-10 15:38:29 +0100 | [diff] [blame] | 161 | hscloud.anchorvm = { |
| 162 | blkdev = "/anchor.img"; |
| 163 | ram = 32; |
| 164 | }; |
| 165 | systemd.services.anchorTestImg = { |
| 166 | requiredBy = [ "anchorvm.service" ]; |
| 167 | serviceConfig = { |
| 168 | Type = "oneshot"; |
| 169 | ExecStart = "${pkgs.coreutils}/bin/truncate -s 128m /anchor.img"; |
| 170 | }; |
| 171 | }; |
Serge Bazanski | 6abe4fa | 2020-10-03 00:18:34 +0200 | [diff] [blame] | 172 | }; |
| 173 | |
| 174 | speaker = mkBGPSpeaker; |
| 175 | |
| 176 | customs = { config, pkgs, ... }: { |
| 177 | imports = [ |
Serge Bazanski | cc769a5 | 2021-02-13 13:13:41 +0100 | [diff] [blame] | 178 | ../modules/gretap.nix |
Serge Bazanski | 6abe4fa | 2020-10-03 00:18:34 +0200 | [diff] [blame] | 179 | ]; |
| 180 | environment.systemPackages = with pkgs; [ |
| 181 | tcpdump htop dstat file dhcpcd |
| 182 | ]; |
| 183 | virtualisation.memorySize = 1024; |
| 184 | virtualisation.vlans = [ |
| 185 | 2 |
| 186 | ]; |
| 187 | networking.firewall.enable = false; |
| 188 | networking.useDHCP = false; |
| 189 | networking.defaultGateway = "185.236.240.4"; |
| 190 | networking.defaultGateway6 = "2a0d:eb00:2137:1::2"; |
| 191 | networking.interfaces."edge" = { |
Serge Bazanski | 6abe4fa | 2020-10-03 00:18:34 +0200 | [diff] [blame] | 192 | ipv4.addresses = [{ address = "185.236.240.5"; prefixLength = 31; }]; |
| 193 | ipv6.addresses = [{ address = "2a0d:eb00:2137:1::3"; prefixLength = 127; }]; |
| 194 | }; |
Serge Bazanski | cc769a5 | 2021-02-13 13:13:41 +0100 | [diff] [blame] | 195 | hscloud.gretap.interfaces."edge" = { |
Serge Bazanski | 6abe4fa | 2020-10-03 00:18:34 +0200 | [diff] [blame] | 196 | parent = "eth2"; |
| 197 | localV4 = "192.168.2.1"; |
| 198 | remoteV4 = "192.168.2.2"; |
| 199 | id = 200; |
| 200 | }; |
| 201 | networking.bridges."lan".interfaces = []; |
| 202 | networking.interfaces."lan" = { |
| 203 | ipv4.addresses = [{ address = "10.8.1.2"; prefixLength = 23; }]; |
| 204 | ipv6.addresses = [{ address = "2a0d:eb00:4242::1"; prefixLength = 64; }]; |
| 205 | }; |
| 206 | services.bird2 = { |
| 207 | enable = true; |
| 208 | config = '' |
| 209 | log syslog all; |
| 210 | debug protocols { states, interfaces, events } |
| 211 | router id 185.236.240.5; |
| 212 | |
| 213 | protocol device { |
| 214 | scan time 10; |
| 215 | }; |
| 216 | protocol kernel kernel_v4 { |
| 217 | ipv4 { |
| 218 | import none; |
| 219 | export all; |
| 220 | }; |
| 221 | } |
| 222 | protocol kernel kernel_v6 { |
| 223 | ipv6 { |
| 224 | import none; |
| 225 | export all; |
| 226 | }; |
| 227 | } |
| 228 | |
| 229 | protocol ospf v3 ospf_hswaw { |
| 230 | ipv6 { |
| 231 | import all; |
| 232 | export all; |
| 233 | }; |
| 234 | area 0.0.0.0 { |
| 235 | interface "edge" { |
| 236 | cost 10; |
| 237 | type bcast; |
| 238 | }; |
| 239 | interface "lan" { |
| 240 | cost 10; |
| 241 | stub yes; |
| 242 | type bcast; |
| 243 | check link no; |
| 244 | }; |
| 245 | }; |
| 246 | } |
| 247 | ''; |
| 248 | }; |
| 249 | }; |
| 250 | }; |
| 251 | |
| 252 | testScript = '' |
| 253 | start_all() |
| 254 | |
| 255 | edge01.wait_for_unit("bird2.service") |
| 256 | # Wait for BGP to settle. |
| 257 | edge01.wait_until_succeeds("ping 185.235.70.44 -c 1 -w 2") |
| 258 | edge01.wait_until_succeeds("birdc show route for 8.8.8.1 table all | grep via") |
| 259 | edge01.wait_until_succeeds( |
| 260 | "birdc show route for 2a00:1450:4016:801::200e table all | grep via" |
| 261 | ) |
| 262 | edge01.succeed("ping 8.8.8.1 -c 1 -w 2") |
| 263 | |
| 264 | # ping from customs to globalmix must succeed. |
| 265 | customs.succeed("ping 8.8.8.1 -c 1 -w 2") |
| 266 | customs.succeed("ping 2a00:1450:4016:801::200e -c 1 -w 2") |
| 267 | |
| 268 | # edge01 must announce exactly one v4 prefix. |
| 269 | bgpspeaker.succeed("birdc show route protocol bgp_globalmix_v4 | grep unicast") |
| 270 | bgpspeaker.fail( |
Serge Bazanski | d635dc8 | 2022-06-11 21:54:04 +0200 | [diff] [blame] | 271 | "birdc show route protocol bgp_globalmix_v4 | grep unicast | grep -v 185.236.240.0/23" |
Serge Bazanski | 6abe4fa | 2020-10-03 00:18:34 +0200 | [diff] [blame] | 272 | ) |
| 273 | |
| 274 | # edge01 must announce exactly one v6 prefix. |
| 275 | bgpspeaker.succeed("birdc show route protocol bgp_globalmix_v6 | grep unicast") |
| 276 | bgpspeaker.fail( |
| 277 | "birdc show route protocol bgp_globalmix_v6 | grep unicast | grep -v 2a0d:eb00::/32" |
| 278 | ) |
| 279 | |
| 280 | # customer networks must be reachable from globalmix |
| 281 | bgpspeaker.succeed("ping 185.236.240.10 -c 1 -w 2") |
| 282 | bgpspeaker.succeed("ping 2a0d:eb00:8000::1 -c 1 -w 2") |
| 283 | bgpspeaker.succeed("ping 185.236.240.12 -c 1 -w 2") |
| 284 | bgpspeaker.succeed("ping 185.236.240.105 -c 1 -w 2") |
| 285 | bgpspeaker.succeed("ping 2a0d:eb00:8003::1 -c 1 -w 2") |
| 286 | |
| 287 | # dhcp agent must be reachable |
| 288 | customs.succeed("ping 185.236.240.18 -c 1 -w 2") |
| 289 | ''; |
| 290 | }); |
| 291 | |
| 292 | in test { inherit pkgs; inherit (pkgs) libs; } |