# xlab-gateway networking # Bond (balance-xor) → VLANs (lan254, wan99, mgmt) → MACVLAN (wan99.0) # WireGuard tunnels with policy routing ("freedom" tables) # NAT/masquerade on wan99.0 { config, pkgs, ... }: let xlabWg = pkgs.writeShellApplication { name = "xlab-wg"; runtimeInputs = with pkgs; [ coreutils gnugrep gawk iproute2 systemd wireguard-tools ]; text = '' set -euo pipefail state=/var/lib/xlab-wireguard command=status target=all if (( $# >= 1 )); then command=$1; shift; fi if (( $# >= 1 )); then target=$1; shift; fi if (( $# != 0 )); then echo "usage: sudo xlab-wg {status|enable|disable|restart|health} [wgnet|skyworks|all]" >&2 exit 2 fi if (( EUID != 0 )); then echo "xlab-wg must run as root" >&2 exit 1 fi install -d -m 0750 "$state" configured() { local iface=$1 port peer case "$iface" in wg-to-wgnet) port=51998 peer='H+PAPw+1MsE50Of4VMMPzMbzGG731CkNrIgaXbxcFwk=' ;; wg-to-skyworks) port=46961 peer='yyHQ8fg1riI9BJPUjydh/C2MTiA/p0Pb1f9Hc88BuCk=' ;; *) return 2 ;; esac ip link show dev "$iface" >/dev/null 2>&1 \ && [[ "$(wg show "$iface" listen-port 2>/dev/null)" == "$port" ]] \ && [[ "$(wg show "$iface" peers 2>/dev/null)" == "$peer" ]] } ensure_configured() { local iface=$1 configured "$iface" && return 0 echo "$iface is missing or incomplete; reloading networkd configuration" >&2 networkctl reload sleep 2 networkctl reconfigure "$iface" 2>/dev/null || true sleep 2 configured "$iface" } trigger_handshake() { case "$1" in wg-to-wgnet) ping -q -c 1 -W 1 -I wg-to-wgnet 10.0.0.1 >/dev/null 2>&1 || true ;; wg-to-skyworks) ping -q -c 1 -W 1 -I wg-to-skyworks 10.239.0.1 >/dev/null 2>&1 || true ;; esac } handshake_age() { local iface=$1 now latest now=$(date +%s) latest=$(wg show "$iface" latest-handshakes 2>/dev/null | gawk 'NR == 1 { print $2 }') [[ "$latest" =~ ^[0-9]+$ ]] || latest=0 if (( latest == 0 )); then echo 2147483647 else echo $((now - latest)) fi } wait_for_handshake() { local iface=$1 age for _ in $(seq 1 10); do trigger_handshake "$iface" sleep 2 age=$(handshake_age "$iface") if (( age <= 180 )); then return 0 fi done return 1 } disabled() { [[ -e "$state/disabled-$1" ]] } health_one() { local iface=$1 now last=0 age if disabled "$iface"; then ip link set dev "$iface" down 2>/dev/null || true echo "$iface is administratively disabled" return 0 fi ensure_configured "$iface" || { echo "$iface configuration is incomplete after networkd reload" >&2 return 1 } ip link set dev "$iface" up age=$(handshake_age "$iface") if (( age <= 180 )); then echo "$iface is healthy; latest handshake is ''${age}s old" return 0 fi now=$(date +%s) if [[ -s "$state/last-recovery-$iface" ]]; then read -r last < "$state/last-recovery-$iface" || last=0 fi if (( now - last < 300 )); then echo "$iface handshake is stale and recovery is cooling down" >&2 return 1 fi echo "$now" > "$state/last-recovery-$iface" echo "$iface handshake is stale; cycling the link after WAN readiness" >&2 ip link set dev "$iface" down sleep 1 ip link set dev "$iface" up if wait_for_handshake "$iface"; then age=$(handshake_age "$iface") echo "$iface recovered; latest handshake is ''${age}s old" return 0 fi echo "$iface did not handshake after recovery" >&2 return 1 } status_one() { local iface=$1 desired=enabled age=unavailable disabled "$iface" && desired=disabled if ip link show dev "$iface" >/dev/null 2>&1; then age=$(handshake_age "$iface") fi printf '%s: desired=%s handshake_age=%ss\n' "$iface" "$desired" "$age" ip -brief link show dev "$iface" 2>/dev/null || true wg show "$iface" 2>/dev/null || true } disable_one() { local iface=$1 touch "$state/disabled-$iface" ip link set dev "$iface" down 2>/dev/null || true echo "$iface disabled; automatic recovery will leave it down" } enable_one() { local iface=$1 rm -f "$state/disabled-$iface" "$state/last-recovery-$iface" ensure_configured "$iface" ip link set dev "$iface" up trigger_handshake "$iface" echo "$iface enabled" } restart_one() { local iface=$1 if disabled "$iface"; then echo "$iface is disabled; use 'xlab-wg enable' instead" >&2 return 1 fi ensure_configured "$iface" ip link set dev "$iface" down sleep 1 ip link set dev "$iface" up trigger_handshake "$iface" rm -f "$state/last-recovery-$iface" echo "$iface restarted" } run_for_target() { local action=$1 rc=0 case "$target" in wgnet|wg-to-wgnet) "$action" wg-to-wgnet || rc=1 ;; skyworks|wg-to-skyworks) "$action" wg-to-skyworks || rc=1 ;; all) "$action" wg-to-wgnet || rc=1 "$action" wg-to-skyworks || rc=1 ;; *) echo "unknown WireGuard target: $target" >&2 return 2 ;; esac return "$rc" } wait_for_wan() { for _ in $(seq 1 60); do if ip -4 route get 166.111.17.108 2>/dev/null | grep -q 'dev wan99.0'; then return 0 fi sleep 2 done echo "campus WAN route did not become ready" >&2 return 1 } case "$command" in status) run_for_target status_one ;; disable|down) systemctl stop xlab-wireguard-readiness.service 2>/dev/null || true run_for_target disable_one ;; enable|up) wait_for_wan run_for_target enable_one systemctl restart xlab-wireguard-readiness.service ;; restart) wait_for_wan systemctl stop xlab-wireguard-readiness.service 2>/dev/null || true run_for_target restart_one systemctl restart xlab-wireguard-readiness.service ;; health) wait_for_wan run_for_target health_one ;; *) echo "usage: sudo xlab-wg {status|enable|disable|restart|health} [wgnet|skyworks|all]" >&2 exit 2 ;; esac ''; }; cnDirectSetScript = '' set -euo pipefail umask 027 state=/var/lib/xlab-cn-direct delegated="$state/delegated-apnic-extended-latest" url=https://ftp.apnic.net/stats/apnic/delegated-apnic-extended-latest v4=$(mktemp --tmpdir="$state" cn-v4.XXXXXX) v6=$(mktemp --tmpdir="$state" cn-v6.XXXXXX) batch=$(mktemp --tmpdir="$state" cn-sets.XXXXXX) candidate= trap 'rm -f "$v4" "$v6" "$batch" "$candidate"' EXIT load_data() { local input=$1 label=$2 bytes v4_count v6_count bytes=$(wc -c < "$input") if [ "$bytes" -lt 1000000 ] || [ "$bytes" -gt 67108864 ]; then echo "Refusing $label APNIC data with unexpected size: $bytes bytes" >&2 return 1 fi # Treat registry data as untrusted input. Only canonical IPv4 CIDRs are # emitted, and every count must be a power of two aligned to its start. if ! gawk -F '|' ' function reject(reason) { printf "Invalid APNIC IPv4 record at line %d: %s\n", NR, reason > "/dev/stderr" bad = 1 valid = 0 } !/^#/ && $2 == "CN" && $3 == "ipv4" && ($7 == "assigned" || $7 == "allocated") { valid = 1 parts = split($4, octet, ".") if (parts != 4) reject("address field") address = 0 for (i = 1; valid && i <= 4; i++) { if (octet[i] !~ /^[0-9]+$/ || length(octet[i]) > 3 || octet[i] + 0 > 255) reject("address octet") else address = address * 256 + octet[i] } if ($5 !~ /^[0-9]+$/ || length($5) > 10) reject("count field") if (!valid) next count = $5 + 0 if (count < 1 || count > 4294967296) reject("count range") units = count prefix = 32 while (valid && units > 1 && int(units / 2) * 2 == units) { units = int(units / 2) prefix-- } if (valid && units != 1) reject("count is not a power of two") if (valid && address - int(address / count) * count != 0) reject("network is not count-aligned") if (valid) printf "%d.%d.%d.%d/%d\n", octet[1], octet[2], octet[3], octet[4], prefix } END { if (bad) exit 1 } ' "$input" > "$v4"; then return 1 fi # The character allow-list prevents command injection; nft --check # performs the complete IPv6 syntax and network-prefix validation. if ! gawk -F '|' ' !/^#/ && $2 == "CN" && $3 == "ipv6" && ($7 == "assigned" || $7 == "allocated") { if (length($4) > 39 || $4 !~ /^[0-9A-Fa-f:]+$/ || index($4, ":") == 0 || $5 !~ /^[0-9]+$/ || length($5) > 3 || $5 + 0 > 128) { printf "Invalid APNIC IPv6 record at line %d\n", NR > "/dev/stderr" bad = 1 next } printf "%s/%d\n", tolower($4), $5 + 0 } END { if (bad) exit 1 } ' "$input" > "$v6"; then return 1 fi v4_count=$(wc -l < "$v4") v6_count=$(wc -l < "$v6") if [ "$v4_count" -lt 8000 ] || [ "$v6_count" -lt 1500 ]; then echo "Refusing undersized $label APNIC sets: v4=$v4_count v6=$v6_count" >&2 return 1 fi { echo 'flush set inet xlab_pbr cn_direct_v4' echo 'flush set inet xlab_pbr cn_direct_v6' gawk 'BEGIN { printf "add element inet xlab_pbr cn_direct_v4 { " } { printf "%s%s", sep, $0; sep = ", " } END { print " }" }' "$v4" gawk 'BEGIN { printf "add element inet xlab_pbr cn_direct_v6 { " } { printf "%s%s", sep, $0; sep = ", " } END { print " }" }' "$v6" } > "$batch" # Both commands are transactions. A parse/check/load failure therefore # retains the currently loaded sets and the last-known-good cache. nft -c -f "$batch" nft -f "$batch" echo "Loaded $label APNIC-CN direct sets: v4=$v4_count v6=$v6_count" } cache_valid=false if [ -s "$delegated" ]; then if load_data "$delegated" cached; then cache_valid=true else echo "Cached APNIC data is invalid; attempting a fresh download" >&2 fi fi refresh=true if $cache_valid && [ -n "$(find "$delegated" -mmin -1440 -print -quit)" ]; then refresh=false fi if $refresh; then candidate=$(mktemp --tmpdir="$state" delegated.XXXXXX) if curl --fail --silent --show-error --location \ --proto '=https' --proto-redir '=https' --max-filesize 67108864 \ --retry 3 --retry-delay 2 --connect-timeout 10 --max-time 180 \ "$url" --output "$candidate" && load_data "$candidate" downloaded; then # Promotion happens only after complete schema, size, count, nft # syntax, and live transaction validation. mv -f "$candidate" "$delegated" candidate= cache_valid=true else echo "APNIC refresh rejected; retaining the last-known-good cache" >&2 fi fi if ! $cache_valid; then echo "No validated APNIC data; leaving CN sets empty" >&2 exit 1 fi ''; in { networking = { useNetworkd = true; useDHCP = false; nftables = { enable = true; ruleset = '' table ip filter { set bogons_v4 { type ipv4_addr flags interval elements = { 0.0.0.0/8, 10.0.0.0/8, 100.64.0.0/10, 127.0.0.0/8, 169.254.0.0/16, 172.16.0.0/12, 192.0.0.0/24, 192.0.2.0/24, 192.168.0.0/16, 198.18.0.0/15, 198.51.100.0/24, 203.0.113.0/24, 224.0.0.0/4, 240.0.0.0/4 } } chain prerouting { type filter hook prerouting priority raw; policy accept; iifname "wan99.0" ip saddr @bogons_v4 drop } chain forward { type filter hook forward priority filter; policy drop; iifname "bond.lan254" ip saddr != 10.253.254.0/24 counter drop comment "Drop spoofed LAN source" ct state established,related accept iifname "bond.lan254" accept iifname "wg-to-wgnet" ip saddr 10.0.0.0/8 ip daddr 10.253.254.0/24 accept comment "Internal WG clients to xlab LAN" } } table inet input_filter { chain input { type filter hook input priority filter; policy drop; ct state invalid drop # Reject forged LAN/management sources before service ACLs. DHCPv4 # legitimately starts at 0.0.0.0. IPv6 DAD probes use the # unspecified source, while normal control traffic is link-local. iifname "bond.lan254" ip saddr != 10.253.254.0/24 ip saddr != 0.0.0.0 drop iifname "bond.lan254" ip6 saddr :: ip6 nexthdr ipv6-icmp accept comment "Allow IPv6 duplicate-address detection" iifname "bond.lan254" ip6 saddr != fd99:23eb:1682:1::/64 ip6 saddr != fe80::/10 drop iifname "bond.mgmt" ip saddr != 192.168.1.0/24 drop ct state established,related accept iif lo accept # The router only exposes DHCP and SSH on trusted access networks. iifname "bond.lan254" udp dport 67 accept iifname "bond.lan254" meta l4proto { tcp, udp } th dport 53 accept comment "LAN DNS resolver" iifname { "bond.lan254", "bond.mgmt" } tcp dport 22 accept iifname "wg-to-wgnet" ip saddr 10.0.0.0/8 tcp dport 22 accept iifname "wg-to-wgnet" ip6 saddr fd99:23eb:1682::/48 tcp dport 22 accept iifname "wg-to-skyworks" tcp dport 22 accept # WAN — DHCP client replies (v4 + v6) iifname "wan99.0" udp sport 67 udp dport 68 accept iifname "wan99.0" udp sport 547 udp dport 546 accept iifname "wan99.0" udp dport { 46961, 51998 } accept comment "WireGuard handshakes" iifname "wan99.0" ip saddr { 166.111.17.81, 166.111.17.108 } tcp dport 22 accept comment "Break-glass SSH from server3 or main" # ICMP/ICMPv6 for path MTU discovery and diagnostics ip protocol icmp accept ip6 nexthdr icmpv6 accept } } table ip6 filter { chain forward { type filter hook forward priority filter; policy drop; iifname "bond.lan254" ip6 saddr != fd99:23eb:1682:1::/64 counter drop comment "Drop spoofed LAN v6 source" ct state established,related accept iifname "bond.lan254" accept iifname "wg-to-wgnet" ip6 saddr fd99:23eb:1682::/48 ip6 daddr fd99:23eb:1682:1::/64 accept comment "Internal WG clients to xlab LAN v6" } } # Destination policy for xlab clients. APNIC-CN sets are populated by # xlab-cn-direct-sets.service; an empty set safely preserves the old # behavior (traffic continues through wg-to-wgnet and is split at .1). table inet xlab_pbr { set cn_direct_v4 { type ipv4_addr; flags interval; } set cn_direct_v6 { type ipv6_addr; flags interval; } # These campus and Alibaba AS45102 ranges need the same failover # semantics as the APNIC-CN data. Marking them selects main while a # WAN default exists, then falls through to table 1002 when it does # not. A throw route would skip table 1002 and blackhole an outage. set regional_direct_v4 { type ipv4_addr flags interval elements = { 166.111.0.0/16, 101.5.0.0/16, 101.6.0.0/16, 59.66.0.0/16, 183.172.0.0/16, 183.173.0.0/16, 47.246.0.0/16, 43.109.0.0/16, 163.181.0.0/16, 139.95.0.0/16 } } set regional_direct_v6 { type ipv6_addr flags interval elements = { 2402:f000::/32, 2001:da8::/32, 2001:250::/35 } } chain prerouting { type filter hook prerouting priority mangle; policy accept; iifname "bond.lan254" ip daddr @cn_direct_v4 counter meta mark set 0x10 comment "APNIC-CN direct via xlab WAN" iifname "bond.lan254" ip6 daddr @cn_direct_v6 counter meta mark set 0x10 comment "APNIC-CN v6 direct via xlab WAN" iifname "bond.lan254" ip daddr @regional_direct_v4 counter meta mark set 0x10 comment "Campus and Alibaba direct via xlab WAN" iifname "bond.lan254" ip6 daddr @regional_direct_v6 counter meta mark set 0x10 comment "CERNET v6 direct via xlab WAN" } } table inet nat { chain postrouting { type nat hook postrouting priority srcnat; policy accept; oifname "wan99.0" masquerade oifname "bond.mgmt" masquerade } } table inet mangle { chain forward_mss { type filter hook forward priority mangle; policy accept; # MSS clamping for forwarded TCP, both directions. Old rule # `& (syn|ack) == syn` ONLY matched plain SYN — SYN-ACK from # the server came back unclamped, full-MTU segments overflowed # the WG path, and large pages (YouTube, Microsoft) silently # truncated. Use `& (syn|rst) == syn` so SYN-ACK still matches # (it has SYN+ACK), only RST packets are excluded. # # Outbound SYN/SYN-ACK uses the tunnel route MTU. For packets # arriving from WireGuard the egress LAN route is 1500, so clamp # those explicitly to the 1420-byte tunnel ceilings as well. oifname { "wg-to-wgnet", "wg-to-skyworks" } tcp flags & (syn|rst) == syn tcp option maxseg size set rt mtu iifname { "wg-to-wgnet", "wg-to-skyworks" } meta nfproto ipv4 tcp flags & (syn|rst) == syn tcp option maxseg size set 1380 iifname { "wg-to-wgnet", "wg-to-skyworks" } meta nfproto ipv6 tcp flags & (syn|rst) == syn tcp option maxseg size set 1360 } chain postrouting { type filter hook postrouting priority filter; policy accept; } } ''; }; firewall.enable = false; # Using nftables directly }; # Custom routing table names environment.etc."iproute2/rt_tables.d/skyworks.conf".text = '' 1000 freedom 1001 freedom-extra 1002 freedom-wgnet 1003 wg-to-skyworks ''; # =========================================================================== # NETDEVS # =========================================================================== systemd.network.netdevs = { "10-bond" = { netdevConfig = { Name = "bond"; Kind = "bond"; Description = "Main aggregated interface"; }; bondConfig = { Mode = "balance-xor"; MIIMonitorSec = "100ms"; TransmitHashPolicy = "layer3+4"; }; }; "20-bond-lan254" = { netdevConfig = { Name = "bond.lan254"; Kind = "vlan"; }; vlanConfig.Id = 10; }; "20-bond-wan99" = { netdevConfig = { Name = "bond.wan99"; Kind = "vlan"; }; vlanConfig.Id = 99; }; "20-bond-mgmt" = { netdevConfig = { Name = "bond.mgmt"; Kind = "vlan"; }; vlanConfig.Id = 1; }; "30-wan99-0" = { netdevConfig = { Name = "wan99.0"; Kind = "macvlan"; MACAddress = "90:e2:ba:54:26:90"; }; macvlanConfig.Mode = "bridge"; }; "40-wg-to-wgnet" = { netdevConfig = { Name = "wg-to-wgnet"; Kind = "wireguard"; }; wireguardConfig = { PrivateKeyFile = config.age.secrets.xlab-wg-wgnet.path; ListenPort = 51998; }; wireguardPeers = [ { PublicKey = "H+PAPw+1MsE50Of4VMMPzMbzGG731CkNrIgaXbxcFwk="; PresharedKeyFile = config.age.secrets.xlab-wg-wgnet-psk.path; AllowedIPs = [ "0.0.0.0/0" "::/0" ]; Endpoint = "166.111.17.108:51820"; PersistentKeepalive = 16; } ]; }; "40-wg-to-skyworks" = { netdevConfig = { Name = "wg-to-skyworks"; Kind = "wireguard"; }; wireguardConfig = { PrivateKeyFile = config.age.secrets.xlab-wg-skyworks.path; ListenPort = 46961; }; wireguardPeers = [ { PublicKey = "yyHQ8fg1riI9BJPUjydh/C2MTiA/p0Pb1f9Hc88BuCk="; AllowedIPs = [ "10.0.2.144/32" "10.74.0.0/16" "10.239.0.0/16" "fd5b:8249:afe:cb9a::/64" ]; # Avoid a boot-time dependency cycle: this tunnel previously waited # indefinitely for DNS that was itself routed through wg-to-wgnet. Endpoint = "166.111.17.81:16777"; PersistentKeepalive = 20; } ]; }; }; # =========================================================================== # NETWORKS # =========================================================================== systemd.network.networks = { # Bond slaves "10-bond-slaves" = { matchConfig.Name = "enp3s0f0 enp3s0f1"; networkConfig = { Bond = "bond"; IPv6AcceptRA = false; LLDP = false; }; }; # Bond master "20-bond" = { matchConfig.Name = "bond"; linkConfig.RequiredForOnline = "no"; networkConfig = { VLAN = [ "bond.mgmt" "bond.wan99" "bond.lan254" ]; DHCP = "no"; IPv6AcceptRA = false; }; }; # LAN - 10.253.254.0/24 "30-bond-lan254" = { matchConfig.Name = "bond.lan254"; networkConfig = { DHCP = "no"; Address = [ "10.253.254.1/24" "fd99:23eb:1682:1::1/64" ]; IPv6AcceptRA = false; }; routes = [ # Keep the local management VLAN outside the foreign tunnel. { Destination = "192.168.1.0/24"; Type = "throw"; Table = 1002; } ]; routingPolicyRules = [ # nft table xlab_pbr marks APNIC-CN destinations for the local WAN. # If the WAN default disappears, lookup falls through to the existing # source-based WG rule below, preserving connectivity through .1. { FirewallMark = "0x10"; Table = 254; Priority = 90; Family = "ipv4"; } { FirewallMark = "0x10"; Table = 254; Priority = 90; Family = "ipv6"; } # LAN IPv4 → suppress default route in main table, fall through to freedom-wgnet { From = "10.253.254.0/24"; SuppressPrefixLength = 0; Family = "ipv4"; Priority = 100; } # LAN IPv6 → same { From = "fd99:23eb:1682:1::/64"; SuppressPrefixLength = 0; Family = "ipv6"; Priority = 100; } # LAN IPv4 → WireGuard routing table { From = "10.253.254.0/24"; Table = 1002; Priority = 20000; Family = "ipv4"; } # LAN IPv6 → WireGuard routing table { From = "fd99:23eb:1682:1::/64"; Table = 1002; Priority = 20000; Family = "ipv6"; } ]; }; # Management - 192.168.1.0/24 "30-bond-mgmt" = { matchConfig.Name = "bond.mgmt"; networkConfig.DHCP = "no"; addresses = [{ Address = "192.168.1.13/24"; }]; }; # WAN VLAN trunk "30-bond-wan99" = { matchConfig.Name = "bond.wan99"; linkConfig.RequiredForOnline = "no"; networkConfig = { MACVLAN = [ "wan99.0" ]; DHCP = "no"; IPv6AcceptRA = false; }; }; # WAN uplink (DHCP) "40-wan99-0" = { matchConfig.Name = "wan99.0"; networkConfig = { NTP = [ "ntp.tuna.tsinghua.edu.cn" "ntp1.aliyun.com" "ntp.ntsc.ac.cn" ]; # Don't set link-specific DNS on the WAN. Tsinghua's resolvers # (166.111.8.28/29) are subject to GFW DNS poisoning, and link-DNS # would override the global services.resolved policy. Global DNS # uses main's IPv4 and IPv6 MosDNS listeners. Numeric WireGuard # endpoints keep tunnel bootstrap independent of DNS. DHCP = "yes"; }; dhcpV4Config = { RouteMTUBytes = "1500"; DUIDType = "link-layer-time"; UseDNS = false; }; dhcpV6Config = { DUIDType = "link-layer-time"; UseDNS = false; }; ipv6AcceptRAConfig = { UseAutonomousPrefix = false; UseDNS = false; }; }; # WireGuard: wg-to-wgnet (main tunnel for "freedom" routing) "50-wg-to-wgnet" = { matchConfig.Name = "wg-to-wgnet"; linkConfig = { ActivationPolicy = "manual"; RequiredForOnline = "no"; }; addresses = [ { Address = "10.253.1.37/32"; } { Address = "fd99:23eb:1682:fd::128/128"; } ]; routes = [ { Destination = "0.0.0.0/0"; Scope = "link"; Table = 1002; } { Destination = "::/0"; Table = 1002; } # Internal ULA via tunnel (main table) so gateway + LAN can reach 10.0.0.1's IPv6 { Destination = "fd99:23eb:1682::/48"; } # Internal networks are reached through the main gateway tunnel. { Destination = "10.0.0.0/16"; Scope = "link"; } { Destination = "10.5.0.0/24"; Scope = "link"; } { Destination = "10.253.0.0/24"; Scope = "link"; } { Destination = "10.254.0.0/24"; Scope = "link"; } ]; }; # WireGuard: wg-to-skyworks "50-wg-to-skyworks" = { matchConfig.Name = "wg-to-skyworks"; linkConfig = { ActivationPolicy = "manual"; RequiredForOnline = "no"; }; addresses = [ { Address = "10.239.0.9/16"; } { Address = "fd5b:8249:afe:cb9a::9/64"; } ]; routes = [ { Destination = "10.0.2.144/32"; } { Destination = "10.74.0.0/16"; } ]; }; }; environment.systemPackages = [ xlabWg ]; # Networkd creates and configures the interfaces but leaves their # administrative state alone. This unit waits for the campus route, raises # enabled tunnels, validates real handshakes, and cycles only stale peers. # Persistent disable markers let operators intentionally keep either path # down without racing an automatic recovery timer. systemd.services.xlab-wireguard-readiness = { description = "Recover and verify xlab WireGuard handshakes"; wants = [ "systemd-networkd.service" "nftables.service" ]; after = [ "systemd-networkd.service" "nftables.service" ]; wantedBy = [ "multi-user.target" ]; serviceConfig = { Type = "oneshot"; ExecStart = "${xlabWg}/bin/xlab-wg health all"; StateDirectory = "xlab-wireguard"; StateDirectoryMode = "0750"; CapabilityBoundingSet = [ "CAP_NET_ADMIN" "CAP_NET_RAW" ]; NoNewPrivileges = true; PrivateTmp = true; ProtectHome = true; ProtectKernelModules = true; ProtectSystem = "strict"; RestrictAddressFamilies = [ "AF_INET" "AF_INET6" "AF_NETLINK" "AF_UNIX" ]; }; }; systemd.timers.xlab-wireguard-readiness = { description = "Periodically verify xlab WireGuard netdevs"; wantedBy = [ "timers.target" ]; timerConfig = { OnBootSec = "30s"; OnUnitInactiveSec = "2min"; AccuracySec = "15s"; RandomizedDelaySec = "10s"; Persistent = true; }; }; # Keep a local APNIC-CN destination set so xlab clients use xlab's own # campus uplink for domestic traffic instead of hairpinning through .1. # The cache is loaded before any network refresh, then refreshed daily. # Propagated reloads repopulate the dynamic sets after nftables flushes them. systemd.services.nftables.unitConfig.PropagatesReloadTo = [ "xlab-cn-direct-sets.service" ]; systemd.services.xlab-cn-direct-sets = { description = "Populate nftables APNIC-CN direct-routing sets"; requires = [ "nftables.service" ]; after = [ "nftables.service" ]; partOf = [ "nftables.service" ]; wantedBy = [ "multi-user.target" "nftables.service" ]; path = with pkgs; [ coreutils curl findutils gawk nftables ]; serviceConfig = { Type = "oneshot"; RemainAfterExit = true; StateDirectory = "xlab-cn-direct"; StateDirectoryMode = "0750"; CapabilityBoundingSet = [ "CAP_NET_ADMIN" ]; NoNewPrivileges = true; PrivateTmp = true; ProtectControlGroups = true; ProtectHome = true; ProtectKernelModules = true; ProtectKernelTunables = true; ProtectSystem = "strict"; RestrictAddressFamilies = [ "AF_INET" "AF_INET6" "AF_NETLINK" "AF_UNIX" ]; RestrictSUIDSGID = true; }; script = cnDirectSetScript; reload = cnDirectSetScript; }; # Timers start services rather than reload active oneshot services, so use a # small trigger unit to run the loader's ExecReload (or restart after error). systemd.services.xlab-cn-direct-sets-refresh = { description = "Trigger APNIC-CN direct-routing set refresh"; after = [ "nftables.service" ]; serviceConfig = { Type = "oneshot"; ExecStart = "${pkgs.systemd}/bin/systemctl reload-or-restart xlab-cn-direct-sets.service"; }; }; systemd.timers.xlab-cn-direct-sets-refresh = { description = "Refresh nftables APNIC-CN direct-routing sets"; wantedBy = [ "timers.target" ]; timerConfig = { OnBootSec = "5min"; OnUnitInactiveSec = "1h"; RandomizedDelaySec = "5min"; Persistent = true; }; }; # Verify the complete client path independently of the loader. This catches # stale registry data and policy loss after either nftables or networkd is # reloaded, even when the population unit itself still reports success. systemd.services.xlab-cn-direct-health = { description = "Verify xlab CN-direct policy and foreign-tunnel fallback"; wants = [ "xlab-cn-direct-sets.service" ]; after = [ "nftables.service" "systemd-networkd.service" "xlab-cn-direct-sets.service" ]; path = with pkgs; [ coreutils findutils gnugrep iproute2 jq nftables ]; serviceConfig = { Type = "oneshot"; CapabilityBoundingSet = [ "CAP_NET_ADMIN" ]; NoNewPrivileges = true; PrivateTmp = true; ProtectHome = true; ProtectSystem = "strict"; RestrictAddressFamilies = [ "AF_INET" "AF_INET6" "AF_NETLINK" "AF_UNIX" ]; }; script = '' set -euo pipefail cache=/var/lib/xlab-cn-direct/delegated-apnic-extended-latest fail() { echo "CN-direct health check: $*" >&2 exit 1 } [[ -s "$cache" ]] || fail "validated APNIC cache is missing" [[ -n "$(find "$cache" -mmin -2880 -print -quit)" ]] || fail "APNIC cache is older than 48 hours" set_count() { nft -j list set inet xlab_pbr "$1" \ | jq '[.nftables[] | .set?.elem? // empty | .[]] | length' } v4_count=$(set_count cn_direct_v4) v6_count=$(set_count cn_direct_v6) regional_v4_count=$(set_count regional_direct_v4) regional_v6_count=$(set_count regional_direct_v6) ((v4_count >= 8000)) || fail "IPv4 set is undersized: $v4_count" ((v6_count >= 1500)) || fail "IPv6 set is undersized: $v6_count" ((regional_v4_count >= 10)) || fail "regional IPv4 set is undersized: $regional_v4_count" ((regional_v6_count >= 3)) || fail "regional IPv6 set is undersized: $regional_v6_count" ip -4 rule show | grep -Eq '^90:.*fwmark 0x10.*lookup main' \ || fail "IPv4 mark rule is missing" ip -6 rule show | grep -Eq '^90:.*fwmark 0x10.*lookup main' \ || fail "IPv6 mark rule is missing" ip -4 route get 223.5.5.5 from 10.253.254.2 iif bond.lan254 mark 0x10 \ | grep -q 'dev wan99.0' || fail "marked CN IPv4 does not use the campus WAN" ip -6 route get 2400:3200::1 from fd99:23eb:1682:1::2 iif bond.lan254 mark 0x10 \ | grep -q 'dev wan99.0' || fail "marked CN IPv6 does not use the campus WAN" ip -4 route get 47.246.1.1 from 10.253.254.2 iif bond.lan254 mark 0x10 \ | grep -q 'dev wan99.0' || fail "Alibaba exception does not use the campus WAN" ip -4 route get 1.1.1.1 from 10.253.254.2 iif bond.lan254 \ | grep -q 'dev wg-to-wgnet' || fail "foreign IPv4 does not use WireGuard" ip -6 route get 2606:4700:4700::1111 from fd99:23eb:1682:1::2 iif bond.lan254 \ | grep -q 'dev wg-to-wgnet' || fail "foreign IPv6 does not use WireGuard" echo "CN-direct healthy: APNIC v4=$v4_count v6=$v6_count; regional v4=$regional_v4_count v6=$regional_v6_count" ''; }; systemd.timers.xlab-cn-direct-health = { description = "Periodically verify xlab client routing policy"; wantedBy = [ "timers.target" ]; timerConfig = { OnBootSec = "10min"; OnUnitInactiveSec = "15min"; RandomizedDelaySec = "2min"; Persistent = true; }; }; # =========================================================================== # DNS RESOLUTION # =========================================================================== # Route DNS through the network's local mosdns at 10.0.0.1 so this host # inherits CN-aware split routing and the analytics blocking policy. # Do not add a mainland resolver as a same-priority "secondary": resolved # load-balances global servers rather than treating their order as failover, # so polluted foreign answers intermittently reach clients. Both addresses # below terminate on main's filtered MosDNS over the primary WireGuard path. # WireGuard endpoints are numeric, so tunnel recovery has no DNS dependency. services.resolved = { enable = true; settings.Resolve = { DNS = [ "10.0.0.1" "fd99:23eb:1682::1" ]; FallbackDNS = ""; DNSStubListenerExtra = [ "10.253.254.1" "fd99:23eb:1682:1::1" ]; LLMNR = false; MulticastDNS = false; }; }; # =========================================================================== # AGENIX SECRETS # =========================================================================== age.secrets = { xlab-wg-wgnet = { file = ../../secrets/xlab-wg-wgnet.age; owner = "systemd-network"; mode = "0400"; }; xlab-wg-wgnet-psk = { file = ../../secrets/xlab-wg-wgnet-psk.age; owner = "systemd-network"; mode = "0400"; }; xlab-wg-skyworks = { file = ../../secrets/xlab-wg-skyworks.age; owner = "systemd-network"; mode = "0400"; }; }; }