# xlab-gateway networking
# Bond (balance-xor) → VLANs (lan254, wan99, mgmt) → MACVLAN (wan99.0)
# WireGuard tunnels with policy routing ("freedom" tables)
# NAT/masquerade on wan99.0
{ config, pkgs, ... }:
let
cnDirectSetScript = ''
set -euo pipefail
umask 027
state=/var/lib/xlab-cn-direct
delegated="$state/delegated-apnic-extended-latest"
url=https://ftp.apnic.net/stats/apnic/delegated-apnic-extended-latest
v4=$(mktemp --tmpdir="$state" cn-v4.XXXXXX)
v6=$(mktemp --tmpdir="$state" cn-v6.XXXXXX)
batch=$(mktemp --tmpdir="$state" cn-sets.XXXXXX)
candidate=
trap 'rm -f "$v4" "$v6" "$batch" "$candidate"' EXIT
load_data() {
local input=$1 label=$2 bytes v4_count v6_count
bytes=$(wc -c < "$input")
if [ "$bytes" -lt 1000000 ] || [ "$bytes" -gt 67108864 ]; then
echo "Refusing $label APNIC data with unexpected size: $bytes bytes" >&2
return 1
fi
# Treat registry data as untrusted input. Only canonical IPv4 CIDRs are
# emitted, and every count must be a power of two aligned to its start.
if ! gawk -F '|' '
function reject(reason) {
printf "Invalid APNIC IPv4 record at line %d: %s\n", NR, reason > "/dev/stderr"
bad = 1
valid = 0
}
!/^#/ && $2 == "CN" && $3 == "ipv4" && ($7 == "assigned" || $7 == "allocated") {
valid = 1
parts = split($4, octet, ".")
if (parts != 4) reject("address field")
address = 0
for (i = 1; valid && i <= 4; i++) {
if (octet[i] !~ /^[0-9]+$/ || length(octet[i]) > 3 || octet[i] + 0 > 255)
reject("address octet")
else
address = address * 256 + octet[i]
}
if ($5 !~ /^[0-9]+$/ || length($5) > 10) reject("count field")
if (!valid) next
count = $5 + 0
if (count < 1 || count > 4294967296) reject("count range")
units = count
prefix = 32
while (valid && units > 1 && int(units / 2) * 2 == units) {
units = int(units / 2)
prefix--
}
if (valid && units != 1) reject("count is not a power of two")
if (valid && address - int(address / count) * count != 0)
reject("network is not count-aligned")
if (valid)
printf "%d.%d.%d.%d/%d\n", octet[1], octet[2], octet[3], octet[4], prefix
}
END { if (bad) exit 1 }
' "$input" > "$v4"; then
return 1
fi
# The character allow-list prevents command injection; nft --check
# performs the complete IPv6 syntax and network-prefix validation.
if ! gawk -F '|' '
!/^#/ && $2 == "CN" && $3 == "ipv6" && ($7 == "assigned" || $7 == "allocated") {
if (length($4) > 39 || $4 !~ /^[0-9A-Fa-f:]+$/ || index($4, ":") == 0 ||
$5 !~ /^[0-9]+$/ || length($5) > 3 || $5 + 0 > 128) {
printf "Invalid APNIC IPv6 record at line %d\n", NR > "/dev/stderr"
bad = 1
next
}
printf "%s/%d\n", tolower($4), $5 + 0
}
END { if (bad) exit 1 }
' "$input" > "$v6"; then
return 1
fi
v4_count=$(wc -l < "$v4")
v6_count=$(wc -l < "$v6")
if [ "$v4_count" -lt 8000 ] || [ "$v6_count" -lt 1500 ]; then
echo "Refusing undersized $label APNIC sets: v4=$v4_count v6=$v6_count" >&2
return 1
fi
{
echo 'flush set inet xlab_pbr cn_direct_v4'
echo 'flush set inet xlab_pbr cn_direct_v6'
gawk 'BEGIN { printf "add element inet xlab_pbr cn_direct_v4 { " } { printf "%s%s", sep, $0; sep = ", " } END { print " }" }' "$v4"
gawk 'BEGIN { printf "add element inet xlab_pbr cn_direct_v6 { " } { printf "%s%s", sep, $0; sep = ", " } END { print " }" }' "$v6"
} > "$batch"
# Both commands are transactions. A parse/check/load failure therefore
# retains the currently loaded sets and the last-known-good cache.
nft -c -f "$batch"
nft -f "$batch"
echo "Loaded $label APNIC-CN direct sets: v4=$v4_count v6=$v6_count"
}
cache_valid=false
if [ -s "$delegated" ]; then
if load_data "$delegated" cached; then
cache_valid=true
else
echo "Cached APNIC data is invalid; attempting a fresh download" >&2
fi
fi
refresh=true
if $cache_valid && [ -n "$(find "$delegated" -mmin -1440 -print -quit)" ]; then
refresh=false
fi
if $refresh; then
candidate=$(mktemp --tmpdir="$state" delegated.XXXXXX)
if curl --fail --silent --show-error --location \
--proto '=https' --proto-redir '=https' --max-filesize 67108864 \
--retry 3 --retry-delay 2 --connect-timeout 10 --max-time 180 \
"$url" --output "$candidate" && load_data "$candidate" downloaded; then
# Promotion happens only after complete schema, size, count, nft
# syntax, and live transaction validation.
mv -f "$candidate" "$delegated"
candidate=
cache_valid=true
else
echo "APNIC refresh rejected; retaining the last-known-good cache" >&2
fi
fi
if ! $cache_valid; then
echo "No validated APNIC data; leaving CN sets empty" >&2
exit 1
fi
'';
in
{
networking = {
useNetworkd = true;
useDHCP = false;
nftables = {
enable = true;
ruleset = ''
table ip filter {
set bogons_v4 {
type ipv4_addr
flags interval
elements = {
0.0.0.0/8, 10.0.0.0/8, 100.64.0.0/10, 127.0.0.0/8,
169.254.0.0/16, 172.16.0.0/12, 192.0.0.0/24, 192.0.2.0/24,
192.168.0.0/16, 198.18.0.0/15, 198.51.100.0/24, 203.0.113.0/24,
224.0.0.0/4, 240.0.0.0/4
}
}
chain prerouting {
type filter hook prerouting priority raw; policy accept;
iifname "wan99.0" ip saddr @bogons_v4 drop
}
chain forward {
type filter hook forward priority filter; policy drop;
iifname "bond.lan254" ip saddr != 10.253.254.0/24 counter drop comment "Drop spoofed LAN source"
iifname { "bond.lan254", "wg-to-wgnet" } accept
ct state established,related accept
}
}
table inet input_filter {
chain input {
type filter hook input priority filter; policy drop;
ct state established,related accept
iif lo accept
# LAN, management, and WireGuard — trust fully
iifname { "bond.lan254", "bond.mgmt", "wg-to-wgnet", "wg-to-skyworks" } accept
# WAN — DHCP client replies (v4 + v6)
iifname "wan99.0" udp sport 67 udp dport 68 accept
iifname "wan99.0" udp sport 547 udp dport 546 accept
# ICMP/ICMPv6 for path MTU discovery and diagnostics
ip protocol icmp accept
ip6 nexthdr icmpv6 accept
}
}
table ip6 filter {
chain forward {
type filter hook forward priority filter; policy drop;
iifname "bond.lan254" ip6 saddr != fd99:23eb:1682:1::/64 counter drop comment "Drop spoofed LAN v6 source"
iifname { "bond.lan254", "wg-to-wgnet" } accept
ct state established,related accept
}
}
# Destination policy for xlab clients. APNIC-CN sets are populated by
# xlab-cn-direct-sets.service; an empty set safely preserves the old
# behavior (traffic continues through wg-to-wgnet and is split at .1).
table inet xlab_pbr {
set cn_direct_v4 { type ipv4_addr; flags interval; }
set cn_direct_v6 { type ipv6_addr; flags interval; }
chain prerouting {
type filter hook prerouting priority mangle; policy accept;
iifname "bond.lan254" ip daddr @cn_direct_v4 counter meta mark set 0x10 comment "APNIC-CN direct via xlab WAN"
iifname "bond.lan254" ip6 daddr @cn_direct_v6 counter meta mark set 0x10 comment "APNIC-CN v6 direct via xlab WAN"
}
}
table inet nat {
chain postrouting {
type nat hook postrouting priority srcnat; policy accept;
oifname "wan99.0" masquerade
oifname "bond.mgmt" masquerade
}
}
table inet mangle {
chain forward_mss {
type filter hook forward priority mangle; policy accept;
# MSS clamping for forwarded TCP, both directions. Old rule
# `& (syn|ack) == syn` ONLY matched plain SYN — SYN-ACK from
# the server came back unclamped, full-MTU segments overflowed
# the WG path, and large pages (YouTube, Microsoft) silently
# truncated. Use `& (syn|rst) == syn` so SYN-ACK still matches
# (it has SYN+ACK), only RST packets are excluded.
#
# `set rt mtu` picks MSS from the egress route's MTU
# (wg-to-wgnet=1420 → 1380 v4 / 1360 v6), instead of the
# one-size-fits-all 1280 that was too low AND one-directional.
tcp flags & (syn|rst) == syn tcp option maxseg size set rt mtu
}
chain postrouting {
type filter hook postrouting priority filter; policy accept;
}
}
'';
};
firewall.enable = false; # Using nftables directly
};
# Custom routing table names
environment.etc."iproute2/rt_tables.d/skyworks.conf".text = ''
1000 freedom
1001 freedom-extra
1002 freedom-wgnet
1003 wg-to-skyworks
'';
# ===========================================================================
# NETDEVS
# ===========================================================================
systemd.network.netdevs = {
"10-bond" = {
netdevConfig = {
Name = "bond";
Kind = "bond";
Description = "Main aggregated interface";
};
bondConfig = {
Mode = "balance-xor";
};
};
"20-bond-lan254" = {
netdevConfig = {
Name = "bond.lan254";
Kind = "vlan";
};
vlanConfig.Id = 10;
};
"20-bond-wan99" = {
netdevConfig = {
Name = "bond.wan99";
Kind = "vlan";
};
vlanConfig.Id = 99;
};
"20-bond-mgmt" = {
netdevConfig = {
Name = "bond.mgmt";
Kind = "vlan";
};
vlanConfig.Id = 1;
};
"30-wan99-0" = {
netdevConfig = {
Name = "wan99.0";
Kind = "macvlan";
MACAddress = "90:e2:ba:54:26:90";
};
macvlanConfig.Mode = "bridge";
};
"40-wg-to-wgnet" = {
netdevConfig = {
Name = "wg-to-wgnet";
Kind = "wireguard";
};
wireguardConfig = {
PrivateKeyFile = config.age.secrets.xlab-wg-wgnet.path;
};
wireguardPeers = [
{
PublicKey = "H+PAPw+1MsE50Of4VMMPzMbzGG731CkNrIgaXbxcFwk=";
PresharedKeyFile = config.age.secrets.xlab-wg-wgnet-psk.path;
AllowedIPs = [ "0.0.0.0/0" "::/0" ];
Endpoint = "166.111.17.108:51820";
PersistentKeepalive = 16;
}
];
};
"40-wg-to-skyworks" = {
netdevConfig = {
Name = "wg-to-skyworks";
Kind = "wireguard";
};
wireguardConfig = {
PrivateKeyFile = config.age.secrets.xlab-wg-skyworks.path;
};
wireguardPeers = [
{
PublicKey = "yyHQ8fg1riI9BJPUjydh/C2MTiA/p0Pb1f9Hc88BuCk=";
AllowedIPs = [ "0.0.0.0/0" "::/0" ];
Endpoint = "wgep.thu-skyworks.org:16777";
PersistentKeepalive = 20;
}
];
};
};
# ===========================================================================
# NETWORKS
# ===========================================================================
systemd.network.networks = {
# Bond slaves
"10-bond-slaves" = {
matchConfig.Name = "enp3s0f0 enp3s0f1";
networkConfig = {
Bond = "bond";
IPv6AcceptRA = false;
LLDP = false;
};
};
# Bond master
"20-bond" = {
matchConfig.Name = "bond";
linkConfig.RequiredForOnline = "no";
networkConfig = {
VLAN = [ "bond.mgmt" "bond.wan99" "bond.lan254" ];
DHCP = "no";
IPv6AcceptRA = false;
};
};
# LAN - 10.253.254.0/24
"30-bond-lan254" = {
matchConfig.Name = "bond.lan254";
networkConfig = {
DHCP = "no";
Address = [
"10.253.254.1/24"
"fd99:23eb:1682:1::1/64"
];
IPv6AcceptRA = false;
};
routes = [
#{ Destination = "10.0.0.0/16"; Gateway = "10.253.0.1"; }
# Throw routes for Tsinghua ranges → punches holes in WireGuard table
{ Destination = "166.111.0.0/16"; Type = "throw"; Table = 1002; }
{ Destination = "101.5.0.0/16"; Type = "throw"; Table = 1002; }
{ Destination = "101.6.0.0/16"; Type = "throw"; Table = 1002; }
{ Destination = "59.66.0.0/16"; Type = "throw"; Table = 1002; }
{ Destination = "183.172.0.0/16"; Type = "throw"; Table = 1002; }
{ Destination = "183.173.0.0/16"; Type = "throw"; Table = 1002; }
# Alibaba overseas CDN ranges used by Taobao/Tmall HTTPDNS. These
# addresses are outside chnroute, but sending them through the abroad
# tunnel makes Alibaba's geo/risk controls see a foreign source. A
# throw in the WG table falls through to main, so they leave directly
# through the campus WAN. Ported from nix-infra commit
# 8692bebcfaf59be065d84d064c367cb01dddc691.
{ Destination = "47.246.0.0/16"; Type = "throw"; Table = 1002; }
{ Destination = "43.109.0.0/16"; Type = "throw"; Table = 1002; }
{ Destination = "163.181.0.0/16"; Type = "throw"; Table = 1002; }
{ Destination = "139.95.0.0/16"; Type = "throw"; Table = 1002; }
# ========= 新增下面这一行 =========
# 告诉策略路由表：如果是去往管理网段，不要走隧道，跳回 main 表处理
{ Destination = "192.168.1.0/24"; Type = "throw"; Table = 1002; }
# ==================================
# v6 CERNET throw routes → CERNET/Tsinghua v6 (incl. tuna 2402:f000::/32) direct via wan99.0, not the wg tunnel
{ Destination = "2402:f000::/32"; Type = "throw"; Table = 1002; }
{ Destination = "2001:da8::/32"; Type = "throw"; Table = 1002; }
{ Destination = "2001:250::/35"; Type = "throw"; Table = 1002; }
];
routingPolicyRules = [
# nft table xlab_pbr marks APNIC-CN destinations for the local WAN.
# If the WAN default disappears, lookup falls through to the existing
# source-based WG rule below, preserving connectivity through .1.
{
FirewallMark = "0x10";
Table = 254;
Priority = 90;
Family = "ipv4";
}
{
FirewallMark = "0x10";
Table = 254;
Priority = 90;
Family = "ipv6";
}
# LAN IPv4 → suppress default route in main table, fall through to freedom-wgnet
{
From = "10.253.254.0/24";
SuppressPrefixLength = 0;
Family = "ipv4";
Priority = 100;
}
# LAN IPv6 → same
{
From = "fd99:23eb:1682:1::/64";
SuppressPrefixLength = 0;
Family = "ipv6";
Priority = 100;
}
# LAN IPv4 → WireGuard routing table
{
From = "10.253.254.0/24";
Table = 1002;
Priority = 20000;
Family = "ipv4";
}
# LAN IPv6 → WireGuard routing table
{
From = "fd99:23eb:1682:1::/64";
Table = 1002;
Priority = 20000;
Family = "ipv6";
}
];
};
# Management - 192.168.1.0/24
"30-bond-mgmt" = {
matchConfig.Name = "bond.mgmt";
networkConfig.DHCP = "no";
addresses = [{ Address = "192.168.1.13/24"; }];
};
# WAN VLAN trunk
"30-bond-wan99" = {
matchConfig.Name = "bond.wan99";
linkConfig.RequiredForOnline = "no";
networkConfig = {
MACVLAN = [ "wan99.0" ];
DHCP = "no";
IPv6AcceptRA = false;
};
};
# WAN uplink (DHCP)
"40-wan99-0" = {
matchConfig.Name = "wan99.0";
networkConfig = {
NTP = [
"ntp.tuna.tsinghua.edu.cn"
"ntp1.aliyun.com"
"ntp.ntsc.ac.cn"
];
# Don't set link-specific DNS on the WAN. Tsinghua's resolvers
# (166.111.8.28/29) are subject to GFW DNS poisoning, and link-DNS
# would override the global services.resolved policy. Global DNS
# (10.0.0.1 → mosdns) handles CN routing internally; resolved's
# fallbackDns covers the case when 10.0.0.1 is unreachable.
DHCP = "yes";
};
dhcpV4Config = {
RouteMTUBytes = "1500";
DUIDType = "link-layer-time";
UseDNS = false;
};
dhcpV6Config = {
DUIDType = "link-layer-time";
UseDNS = false;
};
ipv6AcceptRAConfig = {
UseAutonomousPrefix = false;
UseDNS = false;
};
};
# WireGuard: wg-to-wgnet (main tunnel for "freedom" routing)
"50-wg-to-wgnet" = {
matchConfig.Name = "wg-to-wgnet";
addresses = [
{ Address = "10.253.1.37/32"; }
{ Address = "fd99:23eb:1682:fd::128/128"; }
];
routes = [
{ Destination = "0.0.0.0/0"; Scope = "link"; Table = 1002; }
{ Destination = "::/0"; Scope = "link"; Table = 1002; }
# Internal ULA via tunnel (main table) so gateway + LAN can reach 10.0.0.1's IPv6
{ Destination = "fd99:23eb:1682::/48"; Scope = "link"; }
# ========= 新增下面这两行 =========
{ Destination = "10.0.0.0/16"; Scope = "link"; }
{ Destination = "10.5.0.0/24"; Scope = "link"; }
{ Destination = "10.253.0.0/24"; Scope = "link"; }
{ Destination = "10.254.0.0/24"; Scope = "link"; }
# ==================================
];
};
# WireGuard: wg-to-skyworks
"50-wg-to-skyworks" = {
matchConfig.Name = "wg-to-skyworks";
addresses = [
{ Address = "10.239.0.9/16"; }
{ Address = "fd5b:8249:afe:cb9a::9/64"; }
];
routes = [
{ Destination = "10.0.2.144/32"; }
{ Destination = "10.74.0.0/16"; }
];
};
};
# Keep a local APNIC-CN destination set so xlab clients use xlab's own
# campus uplink for domestic traffic instead of hairpinning through .1.
# The cache is loaded before any network refresh, then refreshed daily.
# Propagated reloads repopulate the dynamic sets after nftables flushes them.
systemd.services.nftables.unitConfig.PropagatesReloadTo = [
"xlab-cn-direct-sets.service"
];
systemd.services.xlab-cn-direct-sets = {
description = "Populate nftables APNIC-CN direct-routing sets";
requires = [ "nftables.service" ];
after = [ "nftables.service" ];
partOf = [ "nftables.service" ];
wantedBy = [ "multi-user.target" "nftables.service" ];
path = with pkgs; [ coreutils curl findutils gawk nftables ];
serviceConfig = {
Type = "oneshot";
RemainAfterExit = true;
StateDirectory = "xlab-cn-direct";
StateDirectoryMode = "0750";
CapabilityBoundingSet = [ "CAP_NET_ADMIN" ];
NoNewPrivileges = true;
PrivateTmp = true;
ProtectControlGroups = true;
ProtectHome = true;
ProtectKernelModules = true;
ProtectKernelTunables = true;
ProtectSystem = "strict";
RestrictAddressFamilies = [ "AF_INET" "AF_INET6" "AF_NETLINK" "AF_UNIX" ];
RestrictSUIDSGID = true;
};
script = cnDirectSetScript;
reload = cnDirectSetScript;
};
# Timers start services rather than reload active oneshot services, so use a
# small trigger unit to run the loader's ExecReload (or restart after error).
systemd.services.xlab-cn-direct-sets-refresh = {
description = "Trigger APNIC-CN direct-routing set refresh";
after = [ "nftables.service" ];
serviceConfig = {
Type = "oneshot";
ExecStart = "${pkgs.systemd}/bin/systemctl reload-or-restart xlab-cn-direct-sets.service";
};
};
systemd.timers.xlab-cn-direct-sets-refresh = {
description = "Refresh nftables APNIC-CN direct-routing sets";
wantedBy = [ "timers.target" ];
timerConfig = {
OnBootSec = "5min";
OnUnitInactiveSec = "1h";
RandomizedDelaySec = "5min";
Persistent = true;
};
};
# ===========================================================================
# DNS RESOLUTION
# ===========================================================================
# Route DNS through the network's local mosdns at 10.0.0.1 so this host
# inherits CN-aware split routing and the analytics blocking policy.
# Cloudflare is retained as fallback in case 10.0.0.1 is unreachable
# (bootstrap, maintenance, or partial outage).
services.resolved = {
enable = true;
fallbackDns = [ "1.1.1.1" "2606:4700:4700::1111" ];
extraConfig = ''
DNS=10.0.0.1
'';
};
# ===========================================================================
# AGENIX SECRETS
# ===========================================================================
age.secrets = {
xlab-wg-wgnet = {
file = ../../secrets/xlab-wg-wgnet.age;
owner = "systemd-network";
mode = "0400";
};
xlab-wg-wgnet-psk = {
file = ../../secrets/xlab-wg-wgnet-psk.age;
owner = "systemd-network";
mode = "0400";
};
xlab-wg-skyworks = {
file = ../../secrets/xlab-wg-skyworks.age;
owner = "systemd-network";
mode = "0400";
};
};
}
