diff --git a/hosts/xlab-gateway/networking.nix b/hosts/xlab-gateway/networking.nix index a82c4bf..b4b3a8e 100644 --- a/hosts/xlab-gateway/networking.nix +++ b/hosts/xlab-gateway/networking.nix @@ -2,8 +2,146 @@ # Bond (balance-xor) → VLANs (lan254, wan99, mgmt) → MACVLAN (wan99.0) # WireGuard tunnels with policy routing ("freedom" tables) # NAT/masquerade on wan99.0 -{ config, ... }: +{ config, pkgs, ... }: +let + cnDirectSetScript = '' + set -euo pipefail + umask 027 + + state=/var/lib/xlab-cn-direct + delegated="$state/delegated-apnic-extended-latest" + url=https://ftp.apnic.net/stats/apnic/delegated-apnic-extended-latest + v4=$(mktemp --tmpdir="$state" cn-v4.XXXXXX) + v6=$(mktemp --tmpdir="$state" cn-v6.XXXXXX) + batch=$(mktemp --tmpdir="$state" cn-sets.XXXXXX) + candidate= + trap 'rm -f "$v4" "$v6" "$batch" "$candidate"' EXIT + + load_data() { + local input=$1 label=$2 bytes v4_count v6_count + + bytes=$(wc -c < "$input") + if [ "$bytes" -lt 1000000 ] || [ "$bytes" -gt 67108864 ]; then + echo "Refusing $label APNIC data with unexpected size: $bytes bytes" >&2 + return 1 + fi + + # Treat registry data as untrusted input. Only canonical IPv4 CIDRs are + # emitted, and every count must be a power of two aligned to its start. + if ! gawk -F '|' ' + function reject(reason) { + printf "Invalid APNIC IPv4 record at line %d: %s\n", NR, reason > "/dev/stderr" + bad = 1 + valid = 0 + } + !/^#/ && $2 == "CN" && $3 == "ipv4" && ($7 == "assigned" || $7 == "allocated") { + valid = 1 + parts = split($4, octet, ".") + if (parts != 4) reject("address field") + address = 0 + for (i = 1; valid && i <= 4; i++) { + if (octet[i] !~ /^[0-9]+$/ || length(octet[i]) > 3 || octet[i] + 0 > 255) + reject("address octet") + else + address = address * 256 + octet[i] + } + if ($5 !~ /^[0-9]+$/ || length($5) > 10) reject("count field") + if (!valid) next + + count = $5 + 0 + if (count < 1 || count > 4294967296) reject("count range") + units = count + prefix = 32 + while (valid && units > 1 && int(units / 2) * 2 == units) { + units = int(units / 2) + prefix-- + } + if (valid && units != 1) reject("count is not a power of two") + if (valid && address - int(address / count) * count != 0) + reject("network is not count-aligned") + if (valid) + printf "%d.%d.%d.%d/%d\n", octet[1], octet[2], octet[3], octet[4], prefix + } + END { if (bad) exit 1 } + ' "$input" > "$v4"; then + return 1 + fi + + # The character allow-list prevents command injection; nft --check + # performs the complete IPv6 syntax and network-prefix validation. + if ! gawk -F '|' ' + !/^#/ && $2 == "CN" && $3 == "ipv6" && ($7 == "assigned" || $7 == "allocated") { + if (length($4) > 39 || $4 !~ /^[0-9A-Fa-f:]+$/ || index($4, ":") == 0 || + $5 !~ /^[0-9]+$/ || length($5) > 3 || $5 + 0 > 128) { + printf "Invalid APNIC IPv6 record at line %d\n", NR > "/dev/stderr" + bad = 1 + next + } + printf "%s/%d\n", tolower($4), $5 + 0 + } + END { if (bad) exit 1 } + ' "$input" > "$v6"; then + return 1 + fi + + v4_count=$(wc -l < "$v4") + v6_count=$(wc -l < "$v6") + if [ "$v4_count" -lt 8000 ] || [ "$v6_count" -lt 1500 ]; then + echo "Refusing undersized $label APNIC sets: v4=$v4_count v6=$v6_count" >&2 + return 1 + fi + + { + echo 'flush set inet xlab_pbr cn_direct_v4' + echo 'flush set inet xlab_pbr cn_direct_v6' + gawk 'BEGIN { printf "add element inet xlab_pbr cn_direct_v4 { " } { printf "%s%s", sep, $0; sep = ", " } END { print " }" }' "$v4" + gawk 'BEGIN { printf "add element inet xlab_pbr cn_direct_v6 { " } { printf "%s%s", sep, $0; sep = ", " } END { print " }" }' "$v6" + } > "$batch" + + # Both commands are transactions. A parse/check/load failure therefore + # retains the currently loaded sets and the last-known-good cache. + nft -c -f "$batch" + nft -f "$batch" + echo "Loaded $label APNIC-CN direct sets: v4=$v4_count v6=$v6_count" + } + + cache_valid=false + if [ -s "$delegated" ]; then + if load_data "$delegated" cached; then + cache_valid=true + else + echo "Cached APNIC data is invalid; attempting a fresh download" >&2 + fi + fi + + refresh=true + if $cache_valid && [ -n "$(find "$delegated" -mmin -1440 -print -quit)" ]; then + refresh=false + fi + + if $refresh; then + candidate=$(mktemp --tmpdir="$state" delegated.XXXXXX) + if curl --fail --silent --show-error --location \ + --proto '=https' --proto-redir '=https' --max-filesize 67108864 \ + --retry 3 --retry-delay 2 --connect-timeout 10 --max-time 180 \ + "$url" --output "$candidate" && load_data "$candidate" downloaded; then + # Promotion happens only after complete schema, size, count, nft + # syntax, and live transaction validation. + mv -f "$candidate" "$delegated" + candidate= + cache_valid=true + else + echo "APNIC refresh rejected; retaining the last-known-good cache" >&2 + fi + fi + + if ! $cache_valid; then + echo "No validated APNIC data; leaving CN sets empty" >&2 + exit 1 + fi + ''; +in { networking = { useNetworkd = true; @@ -31,6 +169,7 @@ chain forward { type filter hook forward priority filter; policy drop; + iifname "bond.lan254" ip saddr != 10.253.254.0/24 counter drop comment "Drop spoofed LAN source" iifname { "bond.lan254", "wg-to-wgnet" } accept ct state established,related accept } @@ -58,11 +197,26 @@ table ip6 filter { chain forward { type filter hook forward priority filter; policy drop; + iifname "bond.lan254" ip6 saddr != fd99:23eb:1682:1::/64 counter drop comment "Drop spoofed LAN v6 source" iifname { "bond.lan254", "wg-to-wgnet" } accept ct state established,related accept } } + # Destination policy for xlab clients. APNIC-CN sets are populated by + # xlab-cn-direct-sets.service; an empty set safely preserves the old + # behavior (traffic continues through wg-to-wgnet and is split at .1). + table inet xlab_pbr { + set cn_direct_v4 { type ipv4_addr; flags interval; } + set cn_direct_v6 { type ipv6_addr; flags interval; } + + chain prerouting { + type filter hook prerouting priority mangle; policy accept; + iifname "bond.lan254" ip daddr @cn_direct_v4 counter meta mark set 0x10 comment "APNIC-CN direct via xlab WAN" + iifname "bond.lan254" ip6 daddr @cn_direct_v6 counter meta mark set 0x10 comment "APNIC-CN v6 direct via xlab WAN" + } + } + table inet nat { chain postrouting { type nat hook postrouting priority srcnat; policy accept; @@ -236,6 +390,16 @@ { Destination = "59.66.0.0/16"; Type = "throw"; Table = 1002; } { Destination = "183.172.0.0/16"; Type = "throw"; Table = 1002; } { Destination = "183.173.0.0/16"; Type = "throw"; Table = 1002; } + # Alibaba overseas CDN ranges used by Taobao/Tmall HTTPDNS. These + # addresses are outside chnroute, but sending them through the abroad + # tunnel makes Alibaba's geo/risk controls see a foreign source. A + # throw in the WG table falls through to main, so they leave directly + # through the campus WAN. Ported from nix-infra commit + # 8692bebcfaf59be065d84d064c367cb01dddc691. + { Destination = "47.246.0.0/16"; Type = "throw"; Table = 1002; } + { Destination = "43.109.0.0/16"; Type = "throw"; Table = 1002; } + { Destination = "163.181.0.0/16"; Type = "throw"; Table = 1002; } + { Destination = "139.95.0.0/16"; Type = "throw"; Table = 1002; } # ========= 新增下面这一行 ========= # 告诉策略路由表:如果是去往管理网段,不要走隧道,跳回 main 表处理 { Destination = "192.168.1.0/24"; Type = "throw"; Table = 1002; } @@ -246,6 +410,21 @@ { Destination = "2001:250::/35"; Type = "throw"; Table = 1002; } ]; routingPolicyRules = [ + # nft table xlab_pbr marks APNIC-CN destinations for the local WAN. + # If the WAN default disappears, lookup falls through to the existing + # source-based WG rule below, preserving connectivity through .1. + { + FirewallMark = "0x10"; + Table = 254; + Priority = 90; + Family = "ipv4"; + } + { + FirewallMark = "0x10"; + Table = 254; + Priority = 90; + Family = "ipv6"; + } # LAN IPv4 → suppress default route in main table, fall through to freedom-wgnet { From = "10.253.254.0/24"; @@ -363,6 +542,63 @@ }; }; + # Keep a local APNIC-CN destination set so xlab clients use xlab's own + # campus uplink for domestic traffic instead of hairpinning through .1. + # The cache is loaded before any network refresh, then refreshed daily. + # Propagated reloads repopulate the dynamic sets after nftables flushes them. + systemd.services.nftables.unitConfig.PropagatesReloadTo = [ + "xlab-cn-direct-sets.service" + ]; + + systemd.services.xlab-cn-direct-sets = { + description = "Populate nftables APNIC-CN direct-routing sets"; + requires = [ "nftables.service" ]; + after = [ "nftables.service" ]; + partOf = [ "nftables.service" ]; + wantedBy = [ "multi-user.target" "nftables.service" ]; + path = with pkgs; [ coreutils curl findutils gawk nftables ]; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + StateDirectory = "xlab-cn-direct"; + StateDirectoryMode = "0750"; + CapabilityBoundingSet = [ "CAP_NET_ADMIN" ]; + NoNewPrivileges = true; + PrivateTmp = true; + ProtectControlGroups = true; + ProtectHome = true; + ProtectKernelModules = true; + ProtectKernelTunables = true; + ProtectSystem = "strict"; + RestrictAddressFamilies = [ "AF_INET" "AF_INET6" "AF_NETLINK" "AF_UNIX" ]; + RestrictSUIDSGID = true; + }; + script = cnDirectSetScript; + reload = cnDirectSetScript; + }; + + # Timers start services rather than reload active oneshot services, so use a + # small trigger unit to run the loader's ExecReload (or restart after error). + systemd.services.xlab-cn-direct-sets-refresh = { + description = "Trigger APNIC-CN direct-routing set refresh"; + after = [ "nftables.service" ]; + serviceConfig = { + Type = "oneshot"; + ExecStart = "${pkgs.systemd}/bin/systemctl reload-or-restart xlab-cn-direct-sets.service"; + }; + }; + + systemd.timers.xlab-cn-direct-sets-refresh = { + description = "Refresh nftables APNIC-CN direct-routing sets"; + wantedBy = [ "timers.target" ]; + timerConfig = { + OnBootSec = "5min"; + OnUnitInactiveSec = "1h"; + RandomizedDelaySec = "5min"; + Persistent = true; + }; + }; + # =========================================================================== # DNS RESOLUTION # ===========================================================================