# xlab-gateway networking
# Bond (balance-xor) → VLANs (lan254, wan99, mgmt) → MACVLAN (wan99.0)
# WireGuard tunnels with policy routing ("freedom" tables)
# NAT/masquerade on wan99.0
{ config, pkgs, ... }:
let
xlabWg = pkgs.writeShellApplication {
name = "xlab-wg";
runtimeInputs = with pkgs; [ coreutils gnugrep gawk iproute2 systemd wireguard-tools ];
text = ''
set -euo pipefail
state=/var/lib/xlab-wireguard
command=status
target=all
if (( $# >= 1 )); then command=$1; shift; fi
if (( $# >= 1 )); then target=$1; shift; fi
if (( $# != 0 )); then
echo "usage: sudo xlab-wg {status|enable|disable|restart|health} [wgnet|skyworks|all]" >&2
exit 2
fi
if (( EUID != 0 )); then
echo "xlab-wg must run as root" >&2
exit 1
fi
install -d -m 0750 "$state"
configured() {
local iface=$1 port peer
case "$iface" in
wg-to-wgnet)
port=51998
peer='H+PAPw+1MsE50Of4VMMPzMbzGG731CkNrIgaXbxcFwk='
;;
wg-to-skyworks)
port=46961
peer='yyHQ8fg1riI9BJPUjydh/C2MTiA/p0Pb1f9Hc88BuCk='
;;
*) return 2 ;;
esac
ip link show dev "$iface" >/dev/null 2>&1 \
&& [[ "$(wg show "$iface" listen-port 2>/dev/null)" == "$port" ]] \
&& [[ "$(wg show "$iface" peers 2>/dev/null)" == "$peer" ]]
}
ensure_configured() {
local iface=$1
configured "$iface" && return 0
echo "$iface is missing or incomplete; reloading networkd configuration" >&2
networkctl reload
sleep 2
networkctl reconfigure "$iface" 2>/dev/null || true
sleep 2
configured "$iface"
}
trigger_handshake() {
case "$1" in
wg-to-wgnet)
ping -q -c 1 -W 1 -I wg-to-wgnet 10.0.0.1 >/dev/null 2>&1 || true
;;
wg-to-skyworks)
ping -q -c 1 -W 1 -I wg-to-skyworks 10.239.0.1 >/dev/null 2>&1 || true
;;
esac
}
handshake_age() {
local iface=$1 now latest
now=$(date +%s)
latest=$(wg show "$iface" latest-handshakes 2>/dev/null | gawk 'NR == 1 { print $2 }')
[[ "$latest" =~ ^[0-9]+$ ]] || latest=0
if (( latest == 0 )); then
echo 2147483647
else
echo $((now - latest))
fi
}
wait_for_handshake() {
local iface=$1 age
for _ in $(seq 1 10); do
trigger_handshake "$iface"
sleep 2
age=$(handshake_age "$iface")
if (( age <= 180 )); then
return 0
fi
done
return 1
}
disabled() {
[[ -e "$state/disabled-$1" ]]
}
health_one() {
local iface=$1 now last=0 age
if disabled "$iface"; then
ip link set dev "$iface" down 2>/dev/null || true
echo "$iface is administratively disabled"
return 0
fi
ensure_configured "$iface" || {
echo "$iface configuration is incomplete after networkd reload" >&2
return 1
}
ip link set dev "$iface" up
age=$(handshake_age "$iface")
if (( age <= 180 )); then
echo "$iface is healthy; latest handshake is ''${age}s old"
return 0
fi
now=$(date +%s)
if [[ -s "$state/last-recovery-$iface" ]]; then
read -r last < "$state/last-recovery-$iface" || last=0
fi
if (( now - last < 300 )); then
echo "$iface handshake is stale and recovery is cooling down" >&2
return 1
fi
echo "$now" > "$state/last-recovery-$iface"
echo "$iface handshake is stale; cycling the link after WAN readiness" >&2
ip link set dev "$iface" down
sleep 1
ip link set dev "$iface" up
if wait_for_handshake "$iface"; then
age=$(handshake_age "$iface")
echo "$iface recovered; latest handshake is ''${age}s old"
return 0
fi
echo "$iface did not handshake after recovery" >&2
return 1
}
status_one() {
local iface=$1 desired=enabled age=unavailable
disabled "$iface" && desired=disabled
if ip link show dev "$iface" >/dev/null 2>&1; then
age=$(handshake_age "$iface")
fi
printf '%s: desired=%s handshake_age=%ss\n' "$iface" "$desired" "$age"
ip -brief link show dev "$iface" 2>/dev/null || true
wg show "$iface" 2>/dev/null || true
}
disable_one() {
local iface=$1
touch "$state/disabled-$iface"
ip link set dev "$iface" down 2>/dev/null || true
echo "$iface disabled; automatic recovery will leave it down"
}
enable_one() {
local iface=$1
rm -f "$state/disabled-$iface" "$state/last-recovery-$iface"
ensure_configured "$iface"
ip link set dev "$iface" up
trigger_handshake "$iface"
echo "$iface enabled"
}
restart_one() {
local iface=$1
if disabled "$iface"; then
echo "$iface is disabled; use 'xlab-wg enable' instead" >&2
return 1
fi
ensure_configured "$iface"
ip link set dev "$iface" down
sleep 1
ip link set dev "$iface" up
trigger_handshake "$iface"
rm -f "$state/last-recovery-$iface"
echo "$iface restarted"
}
run_for_target() {
local action=$1 rc=0
case "$target" in
wgnet|wg-to-wgnet) "$action" wg-to-wgnet || rc=1 ;;
skyworks|wg-to-skyworks) "$action" wg-to-skyworks || rc=1 ;;
all)
"$action" wg-to-wgnet || rc=1
"$action" wg-to-skyworks || rc=1
;;
*)
echo "unknown WireGuard target: $target" >&2
return 2
;;
esac
return "$rc"
}
wait_for_wan() {
for _ in $(seq 1 60); do
if ip -4 route get 166.111.17.108 2>/dev/null | grep -q 'dev wan99.0'; then
return 0
fi
sleep 2
done
echo "campus WAN route did not become ready" >&2
return 1
}
case "$command" in
status)
run_for_target status_one
;;
disable|down)
systemctl stop xlab-wireguard-readiness.service 2>/dev/null || true
run_for_target disable_one
;;
enable|up)
wait_for_wan
run_for_target enable_one
systemctl restart xlab-wireguard-readiness.service
;;
restart)
wait_for_wan
systemctl stop xlab-wireguard-readiness.service 2>/dev/null || true
run_for_target restart_one
systemctl restart xlab-wireguard-readiness.service
;;
health)
wait_for_wan
run_for_target health_one
;;
*)
echo "usage: sudo xlab-wg {status|enable|disable|restart|health} [wgnet|skyworks|all]" >&2
exit 2
;;
esac
'';
};
cnDirectSetScript = ''
set -euo pipefail
umask 027
state=/var/lib/xlab-cn-direct
delegated="$state/delegated-apnic-extended-latest"
url=https://ftp.apnic.net/stats/apnic/delegated-apnic-extended-latest
v4=$(mktemp --tmpdir="$state" cn-v4.XXXXXX)
v6=$(mktemp --tmpdir="$state" cn-v6.XXXXXX)
batch=$(mktemp --tmpdir="$state" cn-sets.XXXXXX)
candidate=
trap 'rm -f "$v4" "$v6" "$batch" "$candidate"' EXIT
load_data() {
local input=$1 label=$2 bytes v4_count v6_count
bytes=$(wc -c < "$input")
if [ "$bytes" -lt 1000000 ] || [ "$bytes" -gt 67108864 ]; then
echo "Refusing $label APNIC data with unexpected size: $bytes bytes" >&2
return 1
fi
# Treat registry data as untrusted input. Only canonical IPv4 CIDRs are
# emitted, and every count must be a power of two aligned to its start.
if ! gawk -F '|' '
function reject(reason) {
printf "Invalid APNIC IPv4 record at line %d: %s\n", NR, reason > "/dev/stderr"
bad = 1
valid = 0
}
!/^#/ && $2 == "CN" && $3 == "ipv4" && ($7 == "assigned" || $7 == "allocated") {
valid = 1
parts = split($4, octet, ".")
if (parts != 4) reject("address field")
address = 0
for (i = 1; valid && i <= 4; i++) {
if (octet[i] !~ /^[0-9]+$/ || length(octet[i]) > 3 || octet[i] + 0 > 255)
reject("address octet")
else
address = address * 256 + octet[i]
}
if ($5 !~ /^[0-9]+$/ || length($5) > 10) reject("count field")
if (!valid) next
count = $5 + 0
if (count < 1 || count > 4294967296) reject("count range")
units = count
prefix = 32
while (valid && units > 1 && int(units / 2) * 2 == units) {
units = int(units / 2)
prefix--
}
if (valid && units != 1) reject("count is not a power of two")
if (valid && address - int(address / count) * count != 0)
reject("network is not count-aligned")
if (valid)
printf "%d.%d.%d.%d/%d\n", octet[1], octet[2], octet[3], octet[4], prefix
}
END { if (bad) exit 1 }
' "$input" > "$v4"; then
return 1
fi
# The character allow-list prevents command injection; nft --check
# performs the complete IPv6 syntax and network-prefix validation.
if ! gawk -F '|' '
!/^#/ && $2 == "CN" && $3 == "ipv6" && ($7 == "assigned" || $7 == "allocated") {
if (length($4) > 39 || $4 !~ /^[0-9A-Fa-f:]+$/ || index($4, ":") == 0 ||
$5 !~ /^[0-9]+$/ || length($5) > 3 || $5 + 0 > 128) {
printf "Invalid APNIC IPv6 record at line %d\n", NR > "/dev/stderr"
bad = 1
next
}
printf "%s/%d\n", tolower($4), $5 + 0
}
END { if (bad) exit 1 }
' "$input" > "$v6"; then
return 1
fi
v4_count=$(wc -l < "$v4")
v6_count=$(wc -l < "$v6")
if [ "$v4_count" -lt 8000 ] || [ "$v6_count" -lt 1500 ]; then
echo "Refusing undersized $label APNIC sets: v4=$v4_count v6=$v6_count" >&2
return 1
fi
{
echo 'flush set inet xlab_pbr cn_direct_v4'
echo 'flush set inet xlab_pbr cn_direct_v6'
gawk 'BEGIN { printf "add element inet xlab_pbr cn_direct_v4 { " } { printf "%s%s", sep, $0; sep = ", " } END { print " }" }' "$v4"
gawk 'BEGIN { printf "add element inet xlab_pbr cn_direct_v6 { " } { printf "%s%s", sep, $0; sep = ", " } END { print " }" }' "$v6"
} > "$batch"
# Both commands are transactions. A parse/check/load failure therefore
# retains the currently loaded sets and the last-known-good cache.
nft -c -f "$batch"
nft -f "$batch"
echo "Loaded $label APNIC-CN direct sets: v4=$v4_count v6=$v6_count"
}
cache_valid=false
if [ -s "$delegated" ]; then
if load_data "$delegated" cached; then
cache_valid=true
else
echo "Cached APNIC data is invalid; attempting a fresh download" >&2
fi
fi
refresh=true
if $cache_valid && [ -n "$(find "$delegated" -mmin -1440 -print -quit)" ]; then
refresh=false
fi
if $refresh; then
candidate=$(mktemp --tmpdir="$state" delegated.XXXXXX)
if curl --fail --silent --show-error --location \
--proto '=https' --proto-redir '=https' --max-filesize 67108864 \
--retry 3 --retry-delay 2 --connect-timeout 10 --max-time 180 \
"$url" --output "$candidate" && load_data "$candidate" downloaded; then
# Promotion happens only after complete schema, size, count, nft
# syntax, and live transaction validation.
mv -f "$candidate" "$delegated"
candidate=
cache_valid=true
else
echo "APNIC refresh rejected; retaining the last-known-good cache" >&2
fi
fi
if ! $cache_valid; then
echo "No validated APNIC data; leaving CN sets empty" >&2
exit 1
fi
'';
in
{
networking = {
useNetworkd = true;
useDHCP = false;
nftables = {
enable = true;
ruleset = ''
table ip filter {
set bogons_v4 {
type ipv4_addr
flags interval
elements = {
0.0.0.0/8, 10.0.0.0/8, 100.64.0.0/10, 127.0.0.0/8,
169.254.0.0/16, 172.16.0.0/12, 192.0.0.0/24, 192.0.2.0/24,
192.168.0.0/16, 198.18.0.0/15, 198.51.100.0/24, 203.0.113.0/24,
224.0.0.0/4, 240.0.0.0/4
}
}
chain prerouting {
type filter hook prerouting priority raw; policy accept;
iifname "wan99.0" ip saddr @bogons_v4 drop
}
chain forward {
type filter hook forward priority filter; policy drop;
iifname "bond.lan254" ip saddr != 10.253.254.0/24 counter drop comment "Drop spoofed LAN source"
ct state established,related accept
iifname "bond.lan254" accept
iifname "wg-to-wgnet" ip saddr 10.0.0.0/8 ip daddr 10.253.254.0/24 accept comment "Internal WG clients to xlab LAN"
}
}
table inet input_filter {
chain input {
type filter hook input priority filter; policy drop;
ct state invalid drop
# Reject forged LAN/management sources before service ACLs. DHCPv4
# legitimately starts at 0.0.0.0. IPv6 DAD probes use the
# unspecified source, while normal control traffic is link-local.
iifname "bond.lan254" ip saddr != 10.253.254.0/24 ip saddr != 0.0.0.0 drop
iifname "bond.lan254" ip6 saddr :: ip6 nexthdr ipv6-icmp accept comment "Allow IPv6 duplicate-address detection"
iifname "bond.lan254" ip6 saddr != fd99:23eb:1682:1::/64 ip6 saddr != fe80::/10 drop
iifname "bond.mgmt" ip saddr != 192.168.1.0/24 drop
ct state established,related accept
iif lo accept
# The router only exposes DHCP and SSH on trusted access networks.
iifname "bond.lan254" udp dport 67 accept
iifname "bond.lan254" meta l4proto { tcp, udp } th dport 53 accept comment "LAN DNS resolver"
iifname { "bond.lan254", "bond.mgmt" } tcp dport 22 accept
iifname "wg-to-wgnet" ip saddr 10.0.0.0/8 tcp dport 22 accept
iifname "wg-to-wgnet" ip6 saddr fd99:23eb:1682::/48 tcp dport 22 accept
iifname "wg-to-skyworks" tcp dport 22 accept
# WAN — DHCP client replies (v4 + v6)
iifname "wan99.0" udp sport 67 udp dport 68 accept
iifname "wan99.0" udp sport 547 udp dport 546 accept
iifname "wan99.0" udp dport { 46961, 51998 } accept comment "WireGuard handshakes"
iifname "wan99.0" ip saddr { 166.111.17.81, 166.111.17.108 } tcp dport 22 accept comment "Break-glass SSH from server3 or main"
# ICMP/ICMPv6 for path MTU discovery and diagnostics
ip protocol icmp accept
ip6 nexthdr icmpv6 accept
}
}
table ip6 filter {
chain forward {
type filter hook forward priority filter; policy drop;
iifname "bond.lan254" ip6 saddr != fd99:23eb:1682:1::/64 counter drop comment "Drop spoofed LAN v6 source"
ct state established,related accept
iifname "bond.lan254" accept
iifname "wg-to-wgnet" ip6 saddr fd99:23eb:1682::/48 ip6 daddr fd99:23eb:1682:1::/64 accept comment "Internal WG clients to xlab LAN v6"
}
}
# Destination policy for xlab clients. APNIC-CN sets are populated by
# xlab-cn-direct-sets.service; an empty set safely preserves the old
# behavior (traffic continues through wg-to-wgnet and is split at .1).
table inet xlab_pbr {
set cn_direct_v4 { type ipv4_addr; flags interval; }
set cn_direct_v6 { type ipv6_addr; flags interval; }
# These campus and Alibaba AS45102 ranges need the same failover
# semantics as the APNIC-CN data. Marking them selects main while a
# WAN default exists, then falls through to table 1002 when it does
# not. A throw route would skip table 1002 and blackhole an outage.
set regional_direct_v4 {
type ipv4_addr
flags interval
elements = {
166.111.0.0/16, 101.5.0.0/16, 101.6.0.0/16,
59.66.0.0/16, 183.172.0.0/16, 183.173.0.0/16,
47.246.0.0/16, 43.109.0.0/16, 163.181.0.0/16,
139.95.0.0/16
}
}
set regional_direct_v6 {
type ipv6_addr
flags interval
elements = {
2402:f000::/32, 2001:da8::/32, 2001:250::/35
}
}
chain prerouting {
type filter hook prerouting priority mangle; policy accept;
iifname "bond.lan254" ip daddr @cn_direct_v4 counter meta mark set 0x10 comment "APNIC-CN direct via xlab WAN"
iifname "bond.lan254" ip6 daddr @cn_direct_v6 counter meta mark set 0x10 comment "APNIC-CN v6 direct via xlab WAN"
iifname "bond.lan254" ip daddr @regional_direct_v4 counter meta mark set 0x10 comment "Campus and Alibaba direct via xlab WAN"
iifname "bond.lan254" ip6 daddr @regional_direct_v6 counter meta mark set 0x10 comment "CERNET v6 direct via xlab WAN"
}
}
table inet nat {
chain postrouting {
type nat hook postrouting priority srcnat; policy accept;
oifname "wan99.0" masquerade
oifname "bond.mgmt" masquerade
}
}
table inet mangle {
chain forward_mss {
type filter hook forward priority mangle; policy accept;
# MSS clamping for forwarded TCP, both directions. Old rule
# `& (syn|ack) == syn` ONLY matched plain SYN — SYN-ACK from
# the server came back unclamped, full-MTU segments overflowed
# the WG path, and large pages (YouTube, Microsoft) silently
# truncated. Use `& (syn|rst) == syn` so SYN-ACK still matches
# (it has SYN+ACK), only RST packets are excluded.
#
# Outbound SYN/SYN-ACK uses the tunnel route MTU. For packets
# arriving from WireGuard the egress LAN route is 1500, so clamp
# those explicitly to the 1420-byte tunnel ceilings as well.
oifname { "wg-to-wgnet", "wg-to-skyworks" } tcp flags & (syn|rst) == syn tcp option maxseg size set rt mtu
iifname { "wg-to-wgnet", "wg-to-skyworks" } meta nfproto ipv4 tcp flags & (syn|rst) == syn tcp option maxseg size set 1380
iifname { "wg-to-wgnet", "wg-to-skyworks" } meta nfproto ipv6 tcp flags & (syn|rst) == syn tcp option maxseg size set 1360
}
chain postrouting {
type filter hook postrouting priority filter; policy accept;
}
}
'';
};
firewall.enable = false; # Using nftables directly
};
# Custom routing table names
environment.etc."iproute2/rt_tables.d/skyworks.conf".text = ''
1000 freedom
1001 freedom-extra
1002 freedom-wgnet
1003 wg-to-skyworks
'';
# ===========================================================================
# NETDEVS
# ===========================================================================
systemd.network.netdevs = {
"10-bond" = {
netdevConfig = {
Name = "bond";
Kind = "bond";
Description = "Main aggregated interface";
};
bondConfig = {
Mode = "balance-xor";
MIIMonitorSec = "100ms";
TransmitHashPolicy = "layer3+4";
};
};
"20-bond-lan254" = {
netdevConfig = {
Name = "bond.lan254";
Kind = "vlan";
};
vlanConfig.Id = 10;
};
"20-bond-wan99" = {
netdevConfig = {
Name = "bond.wan99";
Kind = "vlan";
};
vlanConfig.Id = 99;
};
"20-bond-mgmt" = {
netdevConfig = {
Name = "bond.mgmt";
Kind = "vlan";
};
vlanConfig.Id = 1;
};
"30-wan99-0" = {
netdevConfig = {
Name = "wan99.0";
Kind = "macvlan";
MACAddress = "90:e2:ba:54:26:90";
};
macvlanConfig.Mode = "bridge";
};
"40-wg-to-wgnet" = {
netdevConfig = {
Name = "wg-to-wgnet";
Kind = "wireguard";
};
wireguardConfig = {
PrivateKeyFile = config.age.secrets.xlab-wg-wgnet.path;
ListenPort = 51998;
};
wireguardPeers = [
{
PublicKey = "H+PAPw+1MsE50Of4VMMPzMbzGG731CkNrIgaXbxcFwk=";
PresharedKeyFile = config.age.secrets.xlab-wg-wgnet-psk.path;
AllowedIPs = [ "0.0.0.0/0" "::/0" ];
Endpoint = "166.111.17.108:51820";
PersistentKeepalive = 16;
}
];
};
"40-wg-to-skyworks" = {
netdevConfig = {
Name = "wg-to-skyworks";
Kind = "wireguard";
};
wireguardConfig = {
PrivateKeyFile = config.age.secrets.xlab-wg-skyworks.path;
ListenPort = 46961;
};
wireguardPeers = [
{
PublicKey = "yyHQ8fg1riI9BJPUjydh/C2MTiA/p0Pb1f9Hc88BuCk=";
AllowedIPs = [
"10.0.2.144/32"
"10.74.0.0/16"
"10.239.0.0/16"
"fd5b:8249:afe:cb9a::/64"
];
# Avoid a boot-time dependency cycle: this tunnel previously waited
# indefinitely for DNS that was itself routed through wg-to-wgnet.
Endpoint = "166.111.17.81:16777";
PersistentKeepalive = 20;
}
];
};
};
# ===========================================================================
# NETWORKS
# ===========================================================================
systemd.network.networks = {
# Bond slaves
"10-bond-slaves" = {
matchConfig.Name = "enp3s0f0 enp3s0f1";
networkConfig = {
Bond = "bond";
IPv6AcceptRA = false;
LLDP = false;
};
};
# Bond master
"20-bond" = {
matchConfig.Name = "bond";
linkConfig.RequiredForOnline = "no";
networkConfig = {
VLAN = [ "bond.mgmt" "bond.wan99" "bond.lan254" ];
DHCP = "no";
IPv6AcceptRA = false;
};
};
# LAN - 10.253.254.0/24
"30-bond-lan254" = {
matchConfig.Name = "bond.lan254";
networkConfig = {
DHCP = "no";
Address = [
"10.253.254.1/24"
"fd99:23eb:1682:1::1/64"
];
IPv6AcceptRA = false;
};
routes = [
# Keep the local management VLAN outside the foreign tunnel.
{ Destination = "192.168.1.0/24"; Type = "throw"; Table = 1002; }
];
routingPolicyRules = [
# nft table xlab_pbr marks APNIC-CN destinations for the local WAN.
# If the WAN default disappears, lookup falls through to the existing
# source-based WG rule below, preserving connectivity through .1.
{
FirewallMark = "0x10";
Table = 254;
Priority = 90;
Family = "ipv4";
}
{
FirewallMark = "0x10";
Table = 254;
Priority = 90;
Family = "ipv6";
}
# LAN IPv4 → suppress default route in main table, fall through to freedom-wgnet
{
From = "10.253.254.0/24";
SuppressPrefixLength = 0;
Family = "ipv4";
Priority = 100;
}
# LAN IPv6 → same
{
From = "fd99:23eb:1682:1::/64";
SuppressPrefixLength = 0;
Family = "ipv6";
Priority = 100;
}
# LAN IPv4 → WireGuard routing table
{
From = "10.253.254.0/24";
Table = 1002;
Priority = 20000;
Family = "ipv4";
}
# LAN IPv6 → WireGuard routing table
{
From = "fd99:23eb:1682:1::/64";
Table = 1002;
Priority = 20000;
Family = "ipv6";
}
];
};
# Management - 192.168.1.0/24
"30-bond-mgmt" = {
matchConfig.Name = "bond.mgmt";
networkConfig.DHCP = "no";
addresses = [{ Address = "192.168.1.13/24"; }];
};
# WAN VLAN trunk
"30-bond-wan99" = {
matchConfig.Name = "bond.wan99";
linkConfig.RequiredForOnline = "no";
networkConfig = {
MACVLAN = [ "wan99.0" ];
DHCP = "no";
IPv6AcceptRA = false;
};
};
# WAN uplink (DHCP)
"40-wan99-0" = {
matchConfig.Name = "wan99.0";
networkConfig = {
NTP = [
"ntp.tuna.tsinghua.edu.cn"
"ntp1.aliyun.com"
"ntp.ntsc.ac.cn"
];
# Don't set link-specific DNS on the WAN. Tsinghua's resolvers
# (166.111.8.28/29) are subject to GFW DNS poisoning, and link-DNS
# would override the global services.resolved policy. Global DNS
# uses main's IPv4 and IPv6 MosDNS listeners. Numeric WireGuard
# endpoints keep tunnel bootstrap independent of DNS.
DHCP = "yes";
};
dhcpV4Config = {
RouteMTUBytes = "1500";
DUIDType = "link-layer-time";
UseDNS = false;
};
dhcpV6Config = {
DUIDType = "link-layer-time";
UseDNS = false;
};
ipv6AcceptRAConfig = {
UseAutonomousPrefix = false;
UseDNS = false;
};
};
# WireGuard: wg-to-wgnet (main tunnel for "freedom" routing)
"50-wg-to-wgnet" = {
matchConfig.Name = "wg-to-wgnet";
linkConfig = {
ActivationPolicy = "manual";
RequiredForOnline = "no";
};
addresses = [
{ Address = "10.253.1.37/32"; }
{ Address = "fd99:23eb:1682:fd::128/128"; }
];
routes = [
{ Destination = "0.0.0.0/0"; Scope = "link"; Table = 1002; }
{ Destination = "::/0"; Table = 1002; }
# Internal ULA via tunnel (main table) so gateway + LAN can reach 10.0.0.1's IPv6
{ Destination = "fd99:23eb:1682::/48"; }
# Internal networks are reached through the main gateway tunnel.
{ Destination = "10.0.0.0/16"; Scope = "link"; }
{ Destination = "10.5.0.0/24"; Scope = "link"; }
{ Destination = "10.253.0.0/24"; Scope = "link"; }
{ Destination = "10.254.0.0/24"; Scope = "link"; }
];
};
# WireGuard: wg-to-skyworks
"50-wg-to-skyworks" = {
matchConfig.Name = "wg-to-skyworks";
linkConfig = {
ActivationPolicy = "manual";
RequiredForOnline = "no";
};
addresses = [
{ Address = "10.239.0.9/16"; }
{ Address = "fd5b:8249:afe:cb9a::9/64"; }
];
routes = [
{ Destination = "10.0.2.144/32"; }
{ Destination = "10.74.0.0/16"; }
];
};
};
environment.systemPackages = [ xlabWg ];
# Networkd creates and configures the interfaces but leaves their
# administrative state alone. This unit waits for the campus route, raises
# enabled tunnels, validates real handshakes, and cycles only stale peers.
# Persistent disable markers let operators intentionally keep either path
# down without racing an automatic recovery timer.
systemd.services.xlab-wireguard-readiness = {
description = "Recover and verify xlab WireGuard handshakes";
wants = [ "systemd-networkd.service" "nftables.service" ];
after = [ "systemd-networkd.service" "nftables.service" ];
wantedBy = [ "multi-user.target" ];
serviceConfig = {
Type = "oneshot";
ExecStart = "${xlabWg}/bin/xlab-wg health all";
StateDirectory = "xlab-wireguard";
StateDirectoryMode = "0750";
CapabilityBoundingSet = [ "CAP_NET_ADMIN" "CAP_NET_RAW" ];
NoNewPrivileges = true;
PrivateTmp = true;
ProtectHome = true;
ProtectKernelModules = true;
ProtectSystem = "strict";
RestrictAddressFamilies = [ "AF_INET" "AF_INET6" "AF_NETLINK" "AF_UNIX" ];
};
};
systemd.timers.xlab-wireguard-readiness = {
description = "Periodically verify xlab WireGuard netdevs";
wantedBy = [ "timers.target" ];
timerConfig = {
OnBootSec = "30s";
OnUnitInactiveSec = "2min";
AccuracySec = "15s";
RandomizedDelaySec = "10s";
Persistent = true;
};
};
# Keep a local APNIC-CN destination set so xlab clients use xlab's own
# campus uplink for domestic traffic instead of hairpinning through .1.
# The cache is loaded before any network refresh, then refreshed daily.
# Propagated reloads repopulate the dynamic sets after nftables flushes them.
systemd.services.nftables.unitConfig.PropagatesReloadTo = [
"xlab-cn-direct-sets.service"
];
systemd.services.xlab-cn-direct-sets = {
description = "Populate nftables APNIC-CN direct-routing sets";
requires = [ "nftables.service" ];
after = [ "nftables.service" ];
partOf = [ "nftables.service" ];
wantedBy = [ "multi-user.target" "nftables.service" ];
path = with pkgs; [ coreutils curl findutils gawk nftables ];
serviceConfig = {
Type = "oneshot";
RemainAfterExit = true;
StateDirectory = "xlab-cn-direct";
StateDirectoryMode = "0750";
CapabilityBoundingSet = [ "CAP_NET_ADMIN" ];
NoNewPrivileges = true;
PrivateTmp = true;
ProtectControlGroups = true;
ProtectHome = true;
ProtectKernelModules = true;
ProtectKernelTunables = true;
ProtectSystem = "strict";
RestrictAddressFamilies = [ "AF_INET" "AF_INET6" "AF_NETLINK" "AF_UNIX" ];
RestrictSUIDSGID = true;
};
script = cnDirectSetScript;
reload = cnDirectSetScript;
};
# Timers start services rather than reload active oneshot services, so use a
# small trigger unit to run the loader's ExecReload (or restart after error).
systemd.services.xlab-cn-direct-sets-refresh = {
description = "Trigger APNIC-CN direct-routing set refresh";
after = [ "nftables.service" ];
serviceConfig = {
Type = "oneshot";
ExecStart = "${pkgs.systemd}/bin/systemctl reload-or-restart xlab-cn-direct-sets.service";
};
};
systemd.timers.xlab-cn-direct-sets-refresh = {
description = "Refresh nftables APNIC-CN direct-routing sets";
wantedBy = [ "timers.target" ];
timerConfig = {
OnBootSec = "5min";
OnUnitInactiveSec = "1h";
RandomizedDelaySec = "5min";
Persistent = true;
};
};
# Verify the complete client path independently of the loader. This catches
# stale registry data and policy loss after either nftables or networkd is
# reloaded, even when the population unit itself still reports success.
systemd.services.xlab-cn-direct-health = {
description = "Verify xlab CN-direct policy and foreign-tunnel fallback";
wants = [ "xlab-cn-direct-sets.service" ];
after = [
"nftables.service"
"systemd-networkd.service"
"xlab-cn-direct-sets.service"
];
path = with pkgs; [ coreutils findutils gnugrep iproute2 jq nftables ];
serviceConfig = {
Type = "oneshot";
CapabilityBoundingSet = [ "CAP_NET_ADMIN" ];
NoNewPrivileges = true;
PrivateTmp = true;
ProtectHome = true;
ProtectSystem = "strict";
RestrictAddressFamilies = [ "AF_INET" "AF_INET6" "AF_NETLINK" "AF_UNIX" ];
};
script = ''
set -euo pipefail
cache=/var/lib/xlab-cn-direct/delegated-apnic-extended-latest
fail() {
echo "CN-direct health check: $*" >&2
exit 1
}
[[ -s "$cache" ]] || fail "validated APNIC cache is missing"
[[ -n "$(find "$cache" -mmin -2880 -print -quit)" ]] || fail "APNIC cache is older than 48 hours"
set_count() {
nft -j list set inet xlab_pbr "$1" \
| jq '[.nftables[] | .set?.elem? // empty | .[]] | length'
}
v4_count=$(set_count cn_direct_v4)
v6_count=$(set_count cn_direct_v6)
regional_v4_count=$(set_count regional_direct_v4)
regional_v6_count=$(set_count regional_direct_v6)
((v4_count >= 8000)) || fail "IPv4 set is undersized: $v4_count"
((v6_count >= 1500)) || fail "IPv6 set is undersized: $v6_count"
((regional_v4_count >= 10)) || fail "regional IPv4 set is undersized: $regional_v4_count"
((regional_v6_count >= 3)) || fail "regional IPv6 set is undersized: $regional_v6_count"
ip -4 rule show | grep -Eq '^90:.*fwmark 0x10.*lookup main' \
|| fail "IPv4 mark rule is missing"
ip -6 rule show | grep -Eq '^90:.*fwmark 0x10.*lookup main' \
|| fail "IPv6 mark rule is missing"
ip -4 route get 223.5.5.5 from 10.253.254.2 iif bond.lan254 mark 0x10 \
| grep -q 'dev wan99.0' || fail "marked CN IPv4 does not use the campus WAN"
ip -6 route get 2400:3200::1 from fd99:23eb:1682:1::2 iif bond.lan254 mark 0x10 \
| grep -q 'dev wan99.0' || fail "marked CN IPv6 does not use the campus WAN"
ip -4 route get 47.246.1.1 from 10.253.254.2 iif bond.lan254 mark 0x10 \
| grep -q 'dev wan99.0' || fail "Alibaba exception does not use the campus WAN"
ip -4 route get 1.1.1.1 from 10.253.254.2 iif bond.lan254 \
| grep -q 'dev wg-to-wgnet' || fail "foreign IPv4 does not use WireGuard"
ip -6 route get 2606:4700:4700::1111 from fd99:23eb:1682:1::2 iif bond.lan254 \
| grep -q 'dev wg-to-wgnet' || fail "foreign IPv6 does not use WireGuard"
echo "CN-direct healthy: APNIC v4=$v4_count v6=$v6_count; regional v4=$regional_v4_count v6=$regional_v6_count"
'';
};
systemd.timers.xlab-cn-direct-health = {
description = "Periodically verify xlab client routing policy";
wantedBy = [ "timers.target" ];
timerConfig = {
OnBootSec = "10min";
OnUnitInactiveSec = "15min";
RandomizedDelaySec = "2min";
Persistent = true;
};
};
# ===========================================================================
# DNS RESOLUTION
# ===========================================================================
# Route DNS through the network's local mosdns at 10.0.0.1 so this host
# inherits CN-aware split routing and the analytics blocking policy.
# Do not add a mainland resolver as a same-priority "secondary": resolved
# load-balances global servers rather than treating their order as failover,
# so polluted foreign answers intermittently reach clients. Both addresses
# below terminate on main's filtered MosDNS over the primary WireGuard path.
# WireGuard endpoints are numeric, so tunnel recovery has no DNS dependency.
services.resolved = {
enable = true;
settings.Resolve = {
DNS = [ "10.0.0.1" "fd99:23eb:1682::1" ];
FallbackDNS = "";
DNSStubListenerExtra = [ "10.253.254.1" "fd99:23eb:1682:1::1" ];
LLMNR = false;
MulticastDNS = false;
};
};
# ===========================================================================
# AGENIX SECRETS
# ===========================================================================
age.secrets = {
xlab-wg-wgnet = {
file = ../../secrets/xlab-wg-wgnet.age;
owner = "systemd-network";
mode = "0400";
};
xlab-wg-wgnet-psk = {
file = ../../secrets/xlab-wg-wgnet-psk.age;
owner = "systemd-network";
mode = "0400";
};
xlab-wg-skyworks = {
file = ../../secrets/xlab-wg-skyworks.age;
owner = "systemd-network";
mode = "0400";
};
};
}
