diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..a0aabf8 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +result +result-* +.direnv diff --git a/flake.nix b/flake.nix new file mode 100644 index 0000000..4cf79ac --- /dev/null +++ b/flake.nix @@ -0,0 +1,78 @@ +{ + description = "Skyworks infrastructure"; + + inputs = { + nixpkgs.url = "github:NixOS/nixpkgs/nixos-24.11"; + + disko = { + url = "github:nix-community/disko"; + inputs.nixpkgs.follows = "nixpkgs"; + }; + + agenix = { + url = "github:ryantm/agenix"; + inputs.nixpkgs.follows = "nixpkgs"; + }; + + deploy-rs = { + url = "github:serokell/deploy-rs"; + inputs.nixpkgs.follows = "nixpkgs"; + }; + }; + + outputs = { self, nixpkgs, disko, agenix, deploy-rs, ... }: + let + commonModules = [ + agenix.nixosModules.default + ./modules/common.nix + ./modules/users.nix + ./modules/ssh.nix + ]; + in { + nixosConfigurations = { + skydick = nixpkgs.lib.nixosSystem { + system = "x86_64-linux"; + modules = commonModules ++ [ + disko.nixosModules.disko + ./hosts/skydick/disko.nix + ./hosts/skydick/default.nix + ]; + }; + + xlab-gateway = nixpkgs.lib.nixosSystem { + system = "x86_64-linux"; + modules = commonModules ++ [ + disko.nixosModules.disko + ./hosts/xlab-gateway/disko.nix + ./hosts/xlab-gateway/default.nix + ]; + }; + }; + + deploy.nodes = { + skydick = { + hostname = "10.0.1.1"; + sshUser = "ldx"; + profiles.system = { + user = "root"; + path = deploy-rs.lib.x86_64-linux.activate.nixos + self.nixosConfigurations.skydick; + }; + }; + + xlab-gateway = { + hostname = "10.253.254.1"; + sshUser = "ldx"; + profiles.system = { + user = "root"; + path = deploy-rs.lib.x86_64-linux.activate.nixos + self.nixosConfigurations.xlab-gateway; + }; + }; + }; + + checks = builtins.mapAttrs + (system: deployLib: deployLib.deployChecks self.deploy) + deploy-rs.lib; + }; +} diff --git a/hosts/skydick/default.nix b/hosts/skydick/default.nix new file mode 100644 index 0000000..785ff48 --- /dev/null +++ b/hosts/skydick/default.nix @@ -0,0 +1,350 @@ +# Skydick Storage Server - NixOS Configuration +# Hardware: Dual E5-2699 v3, 256GB RAM, 36-bay SAS chassis (Inventec K800G3-10G) +{ config, pkgs, lib, ... }: + +{ + # ========================================================================== + # SYSTEM IDENTITY + # ========================================================================== + networking.hostName = "skydick"; + networking.hostId = "8425e349"; # Required for ZFS + + # ========================================================================== + # HARDWARE CONFIGURATION + # ========================================================================== + nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux"; + hardware.cpu.intel.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware; + hardware.enableRedistributableFirmware = true; + + # ========================================================================== + # BOOT CONFIGURATION + # ========================================================================== + boot = { + loader = { + systemd-boot.enable = true; + efi.canTouchEfiVariables = true; + timeout = 3; + }; + + supportedFilesystems = [ "zfs" ]; + kernelPackages = pkgs.linuxPackages_6_6; # LTS kernel (Dec 2026), best ZFS stability + kernelModules = [ "kvm-intel" ]; + + # ZFS tunables for 256GB RAM storage server + kernelParams = [ + "zfs.zfs_arc_max=137438953472" # 128GB ARC max + "zfs.zfs_arc_min=34359738368" # 32GB ARC min + "zfs.zfs_txg_timeout=5" + "zfs.zfs_vdev_scrub_min_active=1" + "zfs.zfs_vdev_scrub_max_active=2" + "zfs.zfs_dirty_data_max_percent=25" + "zfs.zfs_vdev_async_read_max_active=8" + "zfs.zfs_vdev_async_write_max_active=8" + "zfs.l2arc_write_max=536870912" # 512MB/s L2ARC write + "zfs.l2arc_write_boost=1073741824" # 1GB/s L2ARC warmup + ]; + + initrd = { + supportedFilesystems = [ "zfs" ]; + availableKernelModules = [ + "xhci_pci" "ehci_pci" "ahci" "mpt3sas" # SAS HBA + "sd_mod" "sr_mod" + "usb_storage" "usbhid" + "mlx5_core" # Mellanox ConnectX-4/5 + ]; + }; + + zfs = { + devNodes = "/dev/disk/by-id"; + forceImportRoot = false; + extraPools = [ "dick" ]; + }; + }; + + # ========================================================================== + # NETWORK CONFIGURATION + # ========================================================================== + networking = { + useDHCP = false; + useNetworkd = true; + + bonds.bond0 = { + interfaces = [ "enp4s0f0np0" "enp4s0f1np1" ]; + driverOptions = { + mode = "active-backup"; + primary = "enp4s0f0np0"; + miimon = "100"; + fail_over_mac = "active"; + }; + }; + + interfaces.bond0 = { + ipv4.addresses = [{ + address = "10.0.1.1"; + prefixLength = 16; + }]; + mtu = 9000; # Jumbo frames for storage traffic + }; + + defaultGateway = { + address = "10.0.0.1"; + interface = "bond0"; + }; + nameservers = [ "10.0.0.1" "223.5.5.5" ]; + + firewall = { + enable = true; + allowedTCPPorts = [ + 22 # SSH + 111 # RPC (NFS) + 2049 # NFS + 445 # SMB + 139 # NetBIOS (SMB) + 3260 # iSCSI + ]; + allowedUDPPorts = [ + 111 # RPC (NFS) + 2049 # NFS (NFSv4.1+) + 137 # NetBIOS Name Service + 138 # NetBIOS Datagram + ]; + allowedTCPPortRanges = [{ from = 20000; to = 20005; }]; + allowedUDPPortRanges = [{ from = 20000; to = 20005; }]; + }; + }; + + services.rpcbind.enable = true; + + # ========================================================================== + # KERNEL PERFORMANCE TUNING + # ========================================================================== + powerManagement.cpuFreqGovernor = "performance"; + + services.udev.extraRules = '' + # SAS/SATA HDDs - use mq-deadline + ACTION=="add|change", KERNEL=="sd[c-z]", ATTR{queue/rotational}=="1", ATTR{queue/scheduler}="mq-deadline" + # SSDs/NVMe - use none + ACTION=="add|change", KERNEL=="sd[a-b]", ATTR{queue/rotational}=="0", ATTR{queue/scheduler}="none" + ACTION=="add|change", KERNEL=="nvme[0-9]*", ATTR{queue/scheduler}="none" + ''; + + boot.kernel.sysctl = { + # Network buffers for high-throughput storage + "net.core.rmem_max" = 134217728; + "net.core.wmem_max" = 134217728; + "net.core.rmem_default" = 16777216; + "net.core.wmem_default" = 16777216; + "net.core.netdev_max_backlog" = 30000; + "net.core.optmem_max" = 67108864; + + # TCP tuning + "net.ipv4.tcp_rmem" = "4096 1048576 134217728"; + "net.ipv4.tcp_wmem" = "4096 1048576 134217728"; + "net.ipv4.tcp_congestion_control" = "bbr"; + "net.ipv4.tcp_mtu_probing" = 1; + "net.ipv4.tcp_window_scaling" = 1; + "net.ipv4.tcp_timestamps" = 1; + "net.ipv4.tcp_sack" = 1; + "net.ipv4.tcp_slow_start_after_idle" = 0; + + # Low-latency network polling + "net.core.busy_read" = 50; + "net.core.busy_poll" = 50; + + # Memory management for large RAM + "vm.swappiness" = 5; + "vm.dirty_ratio" = 40; + "vm.dirty_background_ratio" = 10; + "vm.vfs_cache_pressure" = 50; + "vm.min_free_kbytes" = 1048576; + "vm.zone_reclaim_mode" = 0; + + # NFS server tuning + "sunrpc.tcp_slot_table_entries" = 128; + "sunrpc.udp_slot_table_entries" = 128; + + # File descriptor limits + "fs.file-max" = 2097152; + "fs.nr_open" = 2097152; + }; + + security.pam.loginLimits = [ + { domain = "*"; type = "soft"; item = "nofile"; value = "1048576"; } + { domain = "*"; type = "hard"; item = "nofile"; value = "1048576"; } + ]; + + # ========================================================================== + # ZFS SERVICES + # ========================================================================== + services.zfs = { + autoScrub = { + enable = true; + interval = "Sun *-*-01..07 02:00:00"; + pools = [ "rpool" "dick" ]; + }; + + autoSnapshot = { + enable = true; + flags = "-k -p --utc"; + frequent = 4; + hourly = 24; + daily = 7; + weekly = 4; + monthly = 12; + }; + + trim = { + enable = true; + interval = "weekly"; + }; + }; + + # ========================================================================== + # NFS SERVER + # ========================================================================== + services.nfs.server = { + enable = true; + statdPort = 20001; + lockdPort = 20002; + mountdPort = 20003; + + exports = '' + /srv 10.0.0.0/16(rw,sync,fsid=0,crossmnt,no_subtree_check,root_squash) + /srv/share 10.0.0.0/16(rw,sync,no_subtree_check,root_squash) + /srv/media 10.0.0.0/16(ro,async,no_subtree_check,root_squash) + /srv/backup 10.0.0.0/16(rw,sync,no_subtree_check,no_root_squash) + ''; + }; + + services.nfs.idmapd.settings = { + General = { + Domain = "skydick.local"; + }; + Mapping = { + Nobody-User = "nobody"; + Nobody-Group = "nogroup"; + }; + }; + + systemd.tmpfiles.rules = [ + "d /srv 0755 root root -" + "d /srv/share 0755 nobody nogroup -" + "d /srv/media 0755 nobody nogroup -" + "d /srv/backup 0700 root root -" + ]; + + # ========================================================================== + # SAMBA SERVER + # ========================================================================== + services.samba = { + enable = true; + openFirewall = false; + + settings = { + global = { + workgroup = "WORKGROUP"; + "server string" = "Skydick Storage"; + "netbios name" = "SKYDICK"; + security = "user"; + "hosts allow" = "10.0. 127."; + "hosts deny" = "ALL"; + + "socket options" = "TCP_NODELAY IPTOS_LOWDELAY SO_RCVBUF=131072 SO_SNDBUF=131072"; + "use sendfile" = "yes"; + "aio read size" = "16384"; + "aio write size" = "16384"; + + "map to guest" = "never"; + "server min protocol" = "SMB2_10"; + + "load printers" = "no"; + }; + + share = { + path = "/srv/share"; + browseable = "yes"; + "read only" = "no"; + "guest ok" = "no"; + "valid users" = "@storage"; + "create mask" = "0664"; + "directory mask" = "0775"; + }; + + media = { + path = "/srv/media"; + browseable = "yes"; + "read only" = "yes"; + "guest users" = "@storage"; + }; + }; + }; + + users.groups.storage = {}; + + services.samba-wsdd = { + enable = true; + openFirewall = false; + }; + + # ========================================================================== + # iSCSI TARGET (LIO) + # ========================================================================== + services.target.enable = true; + + # ========================================================================== + # HOST-SPECIFIC USERS + # ========================================================================== + users.users.ldx = { + extraGroups = [ "storage" ]; + hashedPassword = "$y$j9T$hHcj2QYj1.AXbLEALbvr/.$WuDsa.hRDcBWN5s.dJX.KHm9rgkgP/NpNlp3bs2vvs3"; + }; + + users.users.ye-lw21 = { + isNormalUser = true; + extraGroups = [ "wheel" "storage" ]; + hashedPassword = "$y$j9T$hia.9h7L/5q7G4QdKFHOA1$fAFFSpJRf57ZEvCVjDjwM1WH8UPR5E1Xy28KeJQ.gfD"; + openssh.authorizedKeys.keys = [ + "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDGMOAuXpGQcwZDOiAbqnVOkBB4jjSs+awGa0WHpDhuZFo61mQNIoE3lozAO58HikY49+JwJccL+oJgA7VNzpy1hl711wbqj8CV3wjJflpnTla86XnXI30w6dlx7qd07JDoXYr6JvCOCk0nbwomXeekxm8Yw1hwjb476ryvZJDvFhIpSx+7j8oP+qlsUDKOhaMdpiDN71b8YihS7jTZvyw9958XB2AH1WjjV1foWk1ux0KIfkqTTZTPkvbkKBsuHFhHzp08OVXTNaROBz+iDcOm8JUodAO/Qbjcaw2pwENjThxwtuXDCYwYv9CN1rQ3lTXo5zC+8MqvGCZBbnhgIvTV8C5W1nRTpOC1d07r0CrNqq4PVPPW1FKBkhUhbuhJCawNjxgQvK2fwP9hZYstrYiXHVgGNmnv9Utg8ENB4fXYKHgFktGhWXMb8oYJkV9wSQuh86hnr3WAjmQu7dIRT02Sp+KPYskFIGQkDoC79PjUVGJC0HZAFLPIBozliF0u/58= ylw-laptop@YLW-LAPTOP" + ]; + }; + + # ========================================================================== + # MONITORING + # ========================================================================== + services.smartd = { + enable = true; + autodetect = true; + }; + + # ========================================================================== + # PACKAGES + # ========================================================================== + environment.systemPackages = with pkgs; [ + # ZFS & storage + zfs + targetcli + sg3_utils + sdparm + nvme-cli + + # Monitoring + iotop + iftop + smartmontools + lm_sensors + sysstat + dstat + + # Network + iperf3 + ethtool + tcpdump + + # Performance & NUMA + numactl + perf-tools + linuxPackages_6_6.perf + ]; + + system.stateVersion = "25.11"; +} diff --git a/hosts/skydick/disko.nix b/hosts/skydick/disko.nix new file mode 100644 index 0000000..aeaf530 --- /dev/null +++ b/hosts/skydick/disko.nix @@ -0,0 +1,113 @@ +# Disko: System disk partitioning for skydick +# Primary: Samsung SSD 860 EVO 250GB (wwn-0x5002538e4099ad35) +# Future: Add second disk to mirror rpool +{ ... }: + +{ + disko.devices = { + disk.system = { + device = "/dev/disk/by-id/wwn-0x5002538e4099ad35"; + type = "disk"; + content = { + type = "gpt"; + partitions = { + esp = { + size = "2G"; + type = "EF00"; + content = { + type = "filesystem"; + format = "vfat"; + mountpoint = "/boot"; + mountOptions = [ "defaults" "umask=0077" ]; + }; + }; + swap = { + size = "32G"; + content = { + type = "swap"; + randomEncryption = true; + }; + }; + root = { + size = "100%"; + content = { + type = "zfs"; + pool = "rpool"; + }; + }; + }; + }; + }; + + zpool.rpool = { + type = "zpool"; + options = { + ashift = "12"; + autotrim = "on"; + failmode = "continue"; + }; + rootFsOptions = { + compression = "lz4"; + atime = "off"; + relatime = "on"; + xattr = "sa"; + acltype = "posixacl"; + dnodesize = "auto"; + normalization = "formD"; + mountpoint = "none"; + canmount = "off"; + }; + datasets = { + "nixos" = { + type = "zfs_fs"; + options = { + mountpoint = "none"; + canmount = "off"; + }; + }; + "nixos/root" = { + type = "zfs_fs"; + mountpoint = "/"; + options = { + canmount = "on"; + "com.sun:auto-snapshot" = "true"; + }; + }; + "nixos/nix" = { + type = "zfs_fs"; + mountpoint = "/nix"; + options = { + canmount = "on"; + atime = "off"; + "com.sun:auto-snapshot" = "false"; + }; + }; + "nixos/var" = { + type = "zfs_fs"; + mountpoint = "/var"; + options = { + canmount = "on"; + "com.sun:auto-snapshot" = "true"; + }; + }; + "nixos/var/log" = { + type = "zfs_fs"; + mountpoint = "/var/log"; + options = { + canmount = "on"; + compression = "zstd"; + "com.sun:auto-snapshot" = "false"; + }; + }; + "nixos/home" = { + type = "zfs_fs"; + mountpoint = "/home"; + options = { + canmount = "on"; + "com.sun:auto-snapshot" = "true"; + }; + }; + }; + }; + }; +} diff --git a/hosts/xlab-gateway/default.nix b/hosts/xlab-gateway/default.nix new file mode 100644 index 0000000..ea1451d --- /dev/null +++ b/hosts/xlab-gateway/default.nix @@ -0,0 +1,48 @@ +# xlab-gateway - Lab Gateway / Router +# TODO: Migrate from Debian 12 to NixOS +# Current services: Kea DHCP4/6, DDNS, radvd, WireGuard, NAT, policy routing +{ config, pkgs, lib, ... }: + +{ + imports = [ + ./networking.nix + ./dhcp.nix + ]; + + networking.hostName = "xlab-gateway"; + + nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux"; + + boot = { + loader = { + systemd-boot.enable = true; + efi.canTouchEfiVariables = true; + }; + kernel.sysctl = { + "net.ipv4.ip_forward" = 1; + "net.ipv6.conf.all.forwarding" = 1; + }; + }; + + users.users.ldx = { + extraGroups = [ "networkmanager" ]; + }; + + environment.systemPackages = with pkgs; [ + wireguard-tools + iperf3 + ethtool + tcpdump + nftables + iproute2 + glances + smartmontools + ]; + + services.smartd = { + enable = true; + autodetect = true; + }; + + system.stateVersion = "25.11"; +} diff --git a/hosts/xlab-gateway/dhcp.nix b/hosts/xlab-gateway/dhcp.nix new file mode 100644 index 0000000..9061974 --- /dev/null +++ b/hosts/xlab-gateway/dhcp.nix @@ -0,0 +1,195 @@ +# xlab-gateway DHCP + DDNS + radvd +# Kea DHCPv4/v6 on bond.lan254, DDNS forwarding to BIND9 at 10.0.0.1:5353 +{ config, pkgs, ... }: + +{ + # =========================================================================== + # Kea DHCPv4 + # =========================================================================== + services.kea.dhcp4 = { + enable = true; + settings = { + interfaces-config.interfaces = [ "bond.lan254" ]; + + lease-database = { + type = "memfile"; + name = "/var/lib/kea/kea-leases4.csv"; + persist = true; + lfc-interval = 3600; + }; + + expired-leases-processing = { + reclaim-timer-wait-time = 10; + flush-reclaimed-timer-wait-time = 25; + hold-reclaimed-time = 3600; + max-reclaim-leases = 100; + max-reclaim-time = 250; + }; + + valid-lifetime = 86400; + renew-timer = 21600; + rebind-timer = 43200; + + option-data = [ + { name = "routers"; data = "10.253.254.1"; } + { name = "domain-name-servers"; data = "10.0.0.1"; } + { name = "domain-name"; data = "dev.skyw.top"; } + { name = "domain-search"; data = "dev.skyw.top"; } + ]; + + subnet4 = [ + { + id = 1; + subnet = "10.253.254.0/24"; + pools = [ + { pool = "10.253.254.100 - 10.253.254.240"; } + ]; + option-data = [ + { name = "subnet-mask"; data = "255.255.255.0"; } + { name = "routers"; data = "10.253.254.1"; } + { name = "domain-name-servers"; data = "10.0.0.1"; } + # Classless static routes: 10.0.0.0/16 via 10.253.254.1, default via 10.253.254.1 + { code = 121; csv-format = false; data = "100A000AFDFE01000AFDFE01"; } + # MS classless static routes (same) + { code = 249; csv-format = false; data = "100A000AFDFE01000AFDFE01"; } + ]; + reservations = []; + } + ]; + + ddns-send-updates = true; + ddns-qualifying-suffix = "dev.skyw.top."; + + dhcp-ddns = { + enable-updates = true; + max-queue-size = 1024; + ncr-protocol = "UDP"; + ncr-format = "JSON"; + sender-ip = "127.0.0.1"; + sender-port = 0; + server-ip = "127.0.0.1"; + server-port = 53001; + }; + }; + }; + + # =========================================================================== + # Kea DHCPv6 + # =========================================================================== + services.kea.dhcp6 = { + enable = true; + settings = { + interfaces-config.interfaces = [ "bond.lan254" ]; + + lease-database = { + type = "memfile"; + name = "/var/lib/kea/kea-leases6.csv"; + persist = true; + lfc-interval = 3600; + }; + + expired-leases-processing = { + reclaim-timer-wait-time = 10; + flush-reclaimed-timer-wait-time = 25; + hold-reclaimed-time = 3600; + max-reclaim-leases = 100; + max-reclaim-time = 250; + }; + + valid-lifetime = 86400; + preferred-lifetime = 72000; + renew-timer = 21600; + rebind-timer = 43200; + + subnet6 = [ + { + id = 1; + subnet = "fd99:23eb:1682:fd:df00::/72"; + pools = [ + { pool = "fd99:23eb:1682:fd:df00::100 - fd99:23eb:1682:fd:df00::ffff"; } + ]; + option-data = [ + { name = "dns-servers"; data = "fd99:23eb:1682::1"; } + { name = "domain-search"; data = "dev.skyw.top"; } + ]; + } + ]; + + ddns-send-updates = true; + ddns-qualifying-suffix = "dev.skyw.top."; + + dhcp-ddns = { + enable-updates = true; + max-queue-size = 1024; + ncr-protocol = "UDP"; + ncr-format = "JSON"; + sender-ip = "::1"; + sender-port = 0; + server-ip = "::1"; + server-port = 53001; + }; + }; + }; + + # =========================================================================== + # Kea DHCP-DDNS (D2) - Forwards DNS updates to BIND9 + # =========================================================================== + services.kea.dhcp-ddns = { + enable = true; + settings = { + ip-address = "127.0.0.1"; + port = 53001; + + tsig-keys = [ + { + name = "edge-ddns-key"; + algorithm = "HMAC-SHA256"; + # TODO: Move TSIG secret to agenix + secret = "qq+zsTGsWG4ENW9mazyE3/JFKhsUiUR1ex4geYv8OIo="; + } + ]; + + forward-ddns.ddns-domains = [ + { + name = "dev.skyw.top."; + key-name = "edge-ddns-key"; + dns-servers = [{ ip-address = "10.0.0.1"; port = 5353; }]; + } + ]; + + reverse-ddns.ddns-domains = [ + { + name = "10.in-addr.arpa."; + key-name = "edge-ddns-key"; + dns-servers = [{ ip-address = "10.0.0.1"; port = 5353; }]; + } + { + name = "2.8.6.1.b.e.3.2.9.9.d.f.ip6.arpa."; + key-name = "edge-ddns-key"; + dns-servers = [{ ip-address = "10.0.0.1"; port = 5353; }]; + } + ]; + }; + }; + + # =========================================================================== + # radvd - IPv6 Router Advertisements + # =========================================================================== + services.radvd = { + enable = true; + config = '' + interface bond.lan254 { + AdvSendAdvert on; + AdvManagedFlag on; + AdvOtherConfigFlag on; + MinRtrAdvInterval 3; + MaxRtrAdvInterval 10; + prefix fd99:23eb:1682:fd:df::/80 { + AdvOnLink on; + AdvAutonomous off; + AdvRouterAddr on; + }; + }; + ''; + }; +} diff --git a/hosts/xlab-gateway/disko.nix b/hosts/xlab-gateway/disko.nix new file mode 100644 index 0000000..590d987 --- /dev/null +++ b/hosts/xlab-gateway/disko.nix @@ -0,0 +1,43 @@ +# Disko: System disk partitioning for xlab-gateway +# Intel Optane MEMPEK1W016GA 13.4GB NVMe +{ ... }: + +{ + disko.devices = { + disk.system = { + device = "/dev/disk/by-id/nvme-INTEL_MEMPEK1W016GA_PHBT713102AA016D"; + type = "disk"; + content = { + type = "gpt"; + partitions = { + esp = { + size = "512M"; + type = "EF00"; + content = { + type = "filesystem"; + format = "vfat"; + mountpoint = "/boot"; + mountOptions = [ "defaults" "umask=0077" ]; + }; + }; + swap = { + size = "1G"; + content = { + type = "swap"; + randomEncryption = true; + }; + }; + root = { + size = "100%"; + content = { + type = "filesystem"; + format = "ext4"; + mountpoint = "/"; + mountOptions = [ "defaults" "noatime" ]; + }; + }; + }; + }; + }; + }; +} diff --git a/hosts/xlab-gateway/networking.nix b/hosts/xlab-gateway/networking.nix new file mode 100644 index 0000000..a3e3a0b --- /dev/null +++ b/hosts/xlab-gateway/networking.nix @@ -0,0 +1,315 @@ +# xlab-gateway networking +# Bond (balance-xor) → VLANs (lan254, wan99, mgmt) → MACVLAN (wan99.0) +# WireGuard tunnels with policy routing ("freedom" tables) +# NAT/masquerade on wan99.0 +{ config, ... }: + +{ + networking = { + useNetworkd = true; + useDHCP = false; + + nftables = { + enable = true; + ruleset = '' + table ip filter { + set bogons_v4 { + type ipv4_addr + flags interval + elements = { + 0.0.0.0/8, 10.0.0.0/8, 100.64.0.0/10, 127.0.0.0/8, + 169.254.0.0/16, 172.16.0.0/12, 192.0.0.0/24, 192.0.2.0/24, + 192.168.0.0/16, 198.18.0.0/15, 198.51.100.0/24, 203.0.113.0/24, + 224.0.0.0/4, 240.0.0.0/4 + } + } + + chain prerouting { + type filter hook prerouting priority raw; policy accept; + iifname "wan99.0" ip saddr @bogons_v4 drop + } + + chain forward { + type filter hook forward priority filter; policy drop; + iifname { "bond.lan254", "wg-to-wgnet" } accept + iifname "wan99.0" ct state established,related accept + } + } + + table ip6 filter { + chain forward { + type filter hook forward priority filter; policy drop; + iifname { "bond.lan254", "wg-to-wgnet" } accept + iifname "wan99.0" ct state established,related accept + } + } + + table inet nat { + chain postrouting { + type nat hook postrouting priority filter; policy accept; + oifname "wan99.0" masquerade + } + } + + table inet mangle { + chain postrouting { + type filter hook postrouting priority filter; policy accept; + oifname "wg-*" tcp flags syn tcp option maxseg size set 1380 + } + } + ''; + }; + + firewall.enable = false; # Using nftables directly + }; + + # Custom routing table names + environment.etc."iproute2/rt_tables.d/skyworks.conf".text = '' + 1000 freedom + 1001 freedom-extra + 1002 wg-to-skywme + 1003 wg-to-skyworks + ''; + + # =========================================================================== + # NETDEVS + # =========================================================================== + systemd.network.netdevs = { + "10-bond" = { + netdevConfig = { + Name = "bond"; + Kind = "bond"; + Description = "Main aggregated interface"; + }; + bondConfig = { + Mode = "balance-xor"; + }; + }; + + "20-bond-lan254" = { + netdevConfig = { + Name = "bond.lan254"; + Kind = "vlan"; + }; + vlanConfig.Id = 10; + }; + + "20-bond-wan99" = { + netdevConfig = { + Name = "bond.wan99"; + Kind = "vlan"; + }; + vlanConfig.Id = 99; + }; + + "20-bond-mgmt" = { + netdevConfig = { + Name = "bond.mgmt"; + Kind = "vlan"; + }; + vlanConfig.Id = 1; + }; + + "30-wan99-0" = { + netdevConfig = { + Name = "wan99.0"; + Kind = "macvlan"; + MACAddress = "90:e2:ba:54:26:90"; + }; + macvlanConfig.Mode = "bridge"; + }; + + "40-wg-to-wgnet" = { + netdevConfig = { + Name = "wg-to-wgnet"; + Kind = "wireguard"; + }; + wireguardConfig = { + PrivateKeyFile = config.age.secrets.xlab-wg-wgnet.path; + }; + wireguardPeers = [ + { + PublicKey = "H+PAPw+1MsE50Of4VMMPzMbzGG731CkNrIgaXbxcFwk="; + PresharedKeyFile = config.age.secrets.xlab-wg-wgnet-psk.path; + AllowedIPs = [ "0.0.0.0/0" "::/0" ]; + Endpoint = "166.111.17.108:51820"; + PersistentKeepalive = 16; + } + ]; + }; + + "40-wg-to-skyworks" = { + netdevConfig = { + Name = "wg-to-skyworks"; + Kind = "wireguard"; + }; + wireguardConfig = { + PrivateKeyFile = config.age.secrets.xlab-wg-skyworks.path; + }; + wireguardPeers = [ + { + PublicKey = "yyHQ8fg1riI9BJPUjydh/C2MTiA/p0Pb1f9Hc88BuCk="; + AllowedIPs = [ "0.0.0.0/0" "::/0" ]; + Endpoint = "wgep.thu-skyworks.org:16777"; + PersistentKeepalive = 20; + } + ]; + }; + }; + + # =========================================================================== + # NETWORKS + # =========================================================================== + systemd.network.networks = { + # Bond slaves + "10-bond-slaves" = { + matchConfig.Name = "enp3s0f0 enp3s0f1"; + networkConfig = { + Bond = "bond"; + IPv6AcceptRA = false; + LLDP = false; + }; + }; + + # Bond master + "20-bond" = { + matchConfig.Name = "bond"; + linkConfig.RequiredForOnline = "no"; + networkConfig = { + VLAN = [ "bond.mgmt" "bond.wan99" "bond.lan254" ]; + DHCP = "no"; + IPv6AcceptRA = false; + }; + }; + + # LAN - 10.253.254.0/24 + "30-bond-lan254" = { + matchConfig.Name = "bond.lan254"; + networkConfig = { + DHCP = "no"; + Address = [ + "10.253.254.1/24" + "fd99:23eb:1682:fd:df::1/80" + ]; + IPv6AcceptRA = false; + }; + routes = [ + { Destination = "10.0.0.0/16"; Gateway = "10.253.0.1"; } + # Throw routes for Tsinghua ranges → punches holes in WireGuard table + { Destination = "166.111.0.0/16"; Type = "throw"; Table = 1002; } + { Destination = "101.5.0.0/16"; Type = "throw"; Table = 1002; } + { Destination = "101.6.0.0/16"; Type = "throw"; Table = 1002; } + { Destination = "59.66.0.0/16"; Type = "throw"; Table = 1002; } + { Destination = "183.172.0.0/16"; Type = "throw"; Table = 1002; } + { Destination = "183.173.0.0/16"; Type = "throw"; Table = 1002; } + { Destination = "2402:f000::/32"; Type = "throw"; Table = 1002; } + ]; + routingPolicyRules = [ + # Use main table only for specific routes (not default) + { + SuppressPrefixLength = 0; + Family = "both"; + Priority = 100; + } + # All traffic → WireGuard routing table + { + Table = 1002; + Priority = 20000; + Family = "both"; + } + ]; + }; + + # Management - 192.168.1.0/24 + "30-bond-mgmt" = { + matchConfig.Name = "bond.mgmt"; + networkConfig.DHCP = "no"; + addresses = [{ Address = "192.168.1.13/24"; }]; + }; + + # WAN VLAN trunk + "30-bond-wan99" = { + matchConfig.Name = "bond.wan99"; + linkConfig.RequiredForOnline = "no"; + networkConfig = { + MACVLAN = [ "wan99.0" ]; + DHCP = "no"; + IPv6AcceptRA = false; + }; + }; + + # WAN uplink (DHCP) + "40-wan99-0" = { + matchConfig.Name = "wan99.0"; + networkConfig = { + NTP = [ + "ntp.tuna.tsinghua.edu.cn" + "ntp1.aliyun.com" + "ntp.ntsc.ac.cn" + ]; + DNS = [ "166.111.8.28" "166.111.8.29" ]; + DHCP = "yes"; + }; + dhcpV4Config = { + RouteMTUBytes = "1500"; + DUIDType = "link-layer-time"; + UseDNS = false; + }; + dhcpV6Config = { + DUIDType = "link-layer-time"; + UseDNS = false; + }; + ipv6AcceptRAConfig = { + UseAutonomousPrefix = false; + UseDNS = false; + }; + }; + + # WireGuard: wg-to-wgnet (main tunnel for "freedom" routing) + "50-wg-to-wgnet" = { + matchConfig.Name = "wg-to-wgnet"; + addresses = [ + { Address = "10.253.1.37/32"; } + { Address = "fd99:23eb:1682:fd::128/128"; } + ]; + routes = [ + { Destination = "0.0.0.0/0"; Scope = "link"; Table = 1002; } + { Destination = "::/0"; Scope = "link"; Table = 1002; } + ]; + }; + + # WireGuard: wg-to-skyworks + "50-wg-to-skyworks" = { + matchConfig.Name = "wg-to-skyworks"; + addresses = [ + { Address = "10.239.0.9/16"; } + { Address = "fd5b:8249:afe:cb9a::9/64"; } + ]; + routes = [ + { Destination = "10.0.2.144/32"; } + { Destination = "10.74.0.0/16"; } + ]; + }; + }; + + # =========================================================================== + # AGENIX SECRETS + # =========================================================================== + age.secrets = { + xlab-wg-wgnet = { + file = ../../secrets/xlab-wg-wgnet.age; + owner = "systemd-network"; + mode = "0400"; + }; + xlab-wg-wgnet-psk = { + file = ../../secrets/xlab-wg-wgnet-psk.age; + owner = "systemd-network"; + mode = "0400"; + }; + xlab-wg-skyworks = { + file = ../../secrets/xlab-wg-skyworks.age; + owner = "systemd-network"; + mode = "0400"; + }; + }; +} diff --git a/modules/common.nix b/modules/common.nix new file mode 100644 index 0000000..6aefe9f --- /dev/null +++ b/modules/common.nix @@ -0,0 +1,22 @@ +{ pkgs, ... }: + +{ + time.timeZone = "Asia/Shanghai"; + i18n.defaultLocale = "en_US.UTF-8"; + + nix.settings = { + experimental-features = [ "nix-command" "flakes" ]; + auto-optimise-store = true; + }; + + environment.systemPackages = with pkgs; [ + curl + wget + git + vim + tmux + htop + pciutils + usbutils + ]; +} diff --git a/modules/ssh.nix b/modules/ssh.nix new file mode 100644 index 0000000..aa5daa9 --- /dev/null +++ b/modules/ssh.nix @@ -0,0 +1,23 @@ +{ ... }: + +{ + services.openssh = { + enable = true; + settings = { + PermitRootLogin = "no"; + PasswordAuthentication = false; + KbdInteractiveAuthentication = false; + X11Forwarding = false; + MaxAuthTries = 3; + ClientAliveInterval = 300; + ClientAliveCountMax = 2; + }; + }; + + services.fail2ban = { + enable = true; + maxretry = 5; + bantime = "1h"; + bantime-increment.enable = true; + }; +} diff --git a/modules/users.nix b/modules/users.nix new file mode 100644 index 0000000..8113d40 --- /dev/null +++ b/modules/users.nix @@ -0,0 +1,13 @@ +{ ... }: + +{ + users.users.ldx = { + isNormalUser = true; + extraGroups = [ "wheel" ]; + openssh.authorizedKeys.keys = [ + "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKMNHFTC5HMO3IsggHpA+eVSCyhZSmDZz7aV62IFt7sj" + ]; + }; + + security.sudo.wheelNeedsPassword = true; +} diff --git a/secrets/secrets.nix b/secrets/secrets.nix new file mode 100644 index 0000000..88f2618 --- /dev/null +++ b/secrets/secrets.nix @@ -0,0 +1,14 @@ +let + # Admin key (ldx's SSH key) + admin = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKMNHFTC5HMO3IsggHpA+eVSCyhZSmDZz7aV62IFt7sj"; + + # Host keys + skydick = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDnuvNClEXwMEP0IVNZ8GM1V93eU+QMmBqM5R8TM1Sx2"; + xlab-gateway = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAII+EKDpUPWahclzYF6o26AWfrRyZ3bW7D3l9oMo2J6Eg"; +in { + "skydick-wg.age".publicKeys = [ admin skydick ]; + "xlab-wg-skyworks.age".publicKeys = [ admin xlab-gateway ]; + "xlab-wg-wgnet.age".publicKeys = [ admin xlab-gateway ]; + "xlab-wg-wgnet-psk.age".publicKeys = [ admin xlab-gateway ]; + "xlab-wg-warp.age".publicKeys = [ admin xlab-gateway ]; +}