# Skydick Storage Server - NixOS Configuration
# Hardware: Dual E5-2699 v3, 256GB RAM, 36-bay SAS chassis (Inventec K800G3-10G)
{ config, pkgs, lib, ... }:
{
# ==========================================================================
# SYSTEM IDENTITY
# ==========================================================================
networking.hostName = "skydick";
networking.hostId = "8425e349"; # Required for ZFS
# Time sync: pin local CERNET NTP. The NixOS default pool resolved to an
# abroad Vultr server (0.nixos.pool.ntp.org, 137ms delay / 42ms jitter ->
# ~60ms offset that flapped the clock-drift alert). ntp.tuna.tsinghua.edu.cn
# is on-campus (sub-ms); ntp.ntsc.ac.cn (NTSC stratum-1, ~19ms) is the backup.
networking.timeServers = [ "ntp.tuna.tsinghua.edu.cn" "ntp.ntsc.ac.cn" ];
# ==========================================================================
# HARDWARE CONFIGURATION
# ==========================================================================
nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
hardware.cpu.intel.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
hardware.enableRedistributableFirmware = true;
# ==========================================================================
# BOOT CONFIGURATION
# ==========================================================================
boot = {
loader = {
systemd-boot.enable = true;
efi.canTouchEfiVariables = true;
timeout = 3;
};
supportedFilesystems = [ "zfs" ];
# 6.18 contains MM/LRU fixes missing from 6.6 and is supported by the
# pinned OpenZFS 2.4.3 package.
kernelPackages = pkgs.linuxPackages_6_18;
kernelModules = [ "kvm-intel" "ipmi_si" "ipmi_watchdog" ];
extraModprobeConfig = ''
options ipmi_watchdog action=reset timeout=120 panic_wdt_timeout=60
'';
# ZFS tunables for 256GB RAM storage server
kernelParams = [
"zfs.zfs_arc_max=137438953472" # 128GB ARC max
"zfs.zfs_arc_min=34359738368" # 32GB ARC min
"zfs.zfs_txg_timeout=5"
"zfs.zfs_vdev_scrub_min_active=1"
"zfs.zfs_vdev_scrub_max_active=2"
"zfs.zfs_dirty_data_max_percent=25"
"zfs.zfs_vdev_async_read_max_active=32"
"zfs.zfs_vdev_async_read_min_active=4"
"zfs.zfs_vdev_async_write_max_active=8"
"zfs.zfetch_max_streams=16"
"zfs.l2arc_write_max=536870912" # 512MB/s L2ARC write
"zfs.l2arc_write_boost=1073741824" # 1GB/s L2ARC warmup
"panic=60"
];
initrd = {
supportedFilesystems = [ "zfs" ];
availableKernelModules = [
"xhci_pci" "ehci_pci" "ahci" "mpt3sas" # SAS HBA
"sd_mod" "sr_mod"
"usb_storage" "usbhid"
"mlx5_core" # Mellanox ConnectX-4/5
];
};
zfs = {
devNodes = "/dev/disk/by-id";
forceImportRoot = false;
extraPools = [ "dick" ];
};
};
# ==========================================================================
# NETWORK CONFIGURATION
# ==========================================================================
networking = {
useDHCP = false;
useNetworkd = true;
# bond40g: 2× 40G ConnectX-3 (enp130s0 + enp130s0d1) in LACP 802.3ad
# layer3+4 to cisco Po5 (Eth1/51 + Eth1/52). Carries 10.0.1.1/16 and the
# default route since the 2026-05-15 cutover from the original 25G
# ConnectX-4 LX bond0 (active-backup), which was torn down — its
# ports enp4s0f0np0 / enp4s0f1np1 are now standalone DOWN.
bonds.bond40g = {
interfaces = [ "enp130s0" "enp130s0d1" ];
driverOptions = {
mode = "802.3ad";
miimon = "100";
xmit_hash_policy = "layer3+4";
lacp_rate = "fast";
};
};
# bond40g carries the host IP (moved off bond0 at the 2026-05-15 cutover).
interfaces.bond40g = {
ipv4.addresses = [{
address = "10.0.1.1";
prefixLength = 16;
}];
mtu = 9200;
# Disable IPv6 privacy/temporary addresses: NixOS otherwise emits
# net.ipv6.conf.bond40g.use_tempaddr=2, which makes the kernel generate
# and prefer a rotating temporary address over the stable ::d1c0 token.
# A storage server wants a fixed source address (ACLs, logging, rDNS).
tempAddress = "disabled";
};
defaultGateway = {
address = "10.0.0.1";
interface = "bond40g";
};
# IPv6 is enabled for SLAAC on bond40g. Keep DNS pinned to main MosDNS:
# RA/DHCPv6-provided DNS is disabled in the networkd link config below.
enableIPv6 = true;
firewall = {
enable = true;
backend = "nftables";
# This host has a globally routed SLAAC address. Keep storage and
# monitoring services scoped to their actual clients instead of opening
# them on every IPv4 and IPv6 address.
extraInputRules = ''
iifname "bond40g" ip saddr 10.0.0.0/8 tcp dport 22 accept comment "SSH from routed private networks"
iifname "bond40g" ip6 saddr { fd99:23eb:1682::/48, 2a0c:b641:69c:ada0::/64 } tcp dport 22 accept comment "SSH from Skyworks IPv6 networks"
iifname "bond40g" ip saddr 10.0.0.0/16 meta l4proto { tcp, udp } th dport { 111, 2049, 20001-20003 } accept comment "NFS IPv4 clients"
iifname "bond40g" ip6 saddr fd99:23eb:1682::75:15 meta l4proto { tcp, udp } th dport { 111, 2049, 20001-20003 } accept comment "door-pek NFS over ULA"
iifname "bond40g" ip6 saddr 2a0c:b641:69c:ada0::/64 meta l4proto { tcp, udp } th dport { 111, 2049, 20001-20003 } accept comment "declared NFS global IPv6 clients"
iifname "bond40g" ip saddr { 10.0.0.0/16, 10.253.0.0/16 } tcp dport 445 accept comment "SMB client networks"
iifname "bond40g" ip saddr 10.0.75.15 tcp dport 8086 accept comment "door1 InfluxDB and Grafana"
iifname "bond40g" ip saddr { 10.0.0.1, 10.0.75.15 } tcp dport 9100 accept comment "node exporter scrapers"
'';
};
};
networking.nftables.enable = true;
services.openssh.openFirewall = false;
# systemd-networkd: accept RA on bond40g for SLAAC, but keep DNS/search
# domains and the default route under the explicit IPv4 config above.
systemd.network.networks."40-bond40g" = {
networkConfig = {
IPv6AcceptRA = true;
# NixOS unconditionally sets IPv6PrivacyExtensions = "kernel" (normal
# priority) for every interface in networking.interfaces, so mkForce is
# required to disable privacy extensions and pin the stable ::d1c0 token
# address as the source — a plain `false` collides with that default.
IPv6PrivacyExtensions = lib.mkForce false;
};
ipv6AcceptRAConfig = {
Token = "::d1c0";
UseDNS = false;
UseGateway = false;
UseDomains = false;
DHCPv6Client = false;
};
};
# Use only main's filtered MosDNS endpoints. Global DNS servers are peers in
# resolved, not ordered failovers; adding AliDNS here can intermittently
# return polluted foreign-domain answers, while adding a public resolver
# bypasses the network's split-DNS and blocking policy.
services.resolved = {
enable = true;
settings.Resolve = {
DNS = [ "10.0.0.1" "fd99:23eb:1682::1" ];
FallbackDNS = "";
LLMNR = false;
MulticastDNS = false;
};
};
# Trust the private CA used by the LDAP service on the main gateway.
security.pki.certificateFiles = [ ../../certs/skyw-ldap-ca.crt ];
# Wait only for bond40g, not individual member ports — a disconnected port
# (cable maintenance) should not stall boot by 2 minutes.
systemd.network.wait-online.anyInterface = true;
# ==========================================================================
# KERNEL PERFORMANCE TUNING
# ==========================================================================
powerManagement.cpuFreqGovernor = "performance";
# The BMC exposes a hardware watchdog through the in-band IPMI interface.
# Reboot a wedged host even when the scheduler can no longer run userspace.
systemd.settings.Manager = {
WatchdogDevice = "/dev/watchdog";
RuntimeWatchdogSec = "2min";
};
services.udev.extraRules = ''
# I/O schedulers by media type (no kernel-name assumptions)
ACTION=="add|change", KERNEL=="sd*", ATTR{queue/rotational}=="1", ATTR{queue/scheduler}="mq-deadline"
ACTION=="add|change", KERNEL=="sd*", ATTR{queue/rotational}=="0", ATTR{queue/scheduler}="none"
ACTION=="add|change", KERNEL=="nvme[0-9]*", ATTR{queue/scheduler}="none"
# The BMC virtual-media xHCI root hub loops in usb_runtime_idle while
# suspending, consuming roughly one third of a CPU. Keep only this
# controller's root hubs awake; other USB devices retain runtime PM.
ACTION=="add|bind", SUBSYSTEM=="usb", KERNEL=="usb[0-9]*", KERNELS=="0000:00:14.0", ATTR{idVendor}=="1d6b", TEST=="power/control", ATTR{power/control}="on"
'';
# Belt-and-suspenders: enforce I/O schedulers and NIC ring buffers at boot.
# The udev rules above handle hotplug; this service catches anything the
# rules miss due to driver-level scheduler resets on SAS devices.
systemd.services.storage-tuning = {
description = "Storage I/O scheduler and NIC ring buffer tuning";
after = [ "systemd-udevd.service" "network.target" ];
wantedBy = [ "multi-user.target" ];
serviceConfig = {
Type = "oneshot";
RemainAfterExit = true;
};
path = [ pkgs.ethtool ];
script = ''
# mq-deadline for rotational drives (SAS Mach2 HDDs)
for dev in /sys/block/sd*; do
[ -d "$dev/queue" ] || continue
if [ "$(cat "$dev/queue/rotational" 2>/dev/null)" = "1" ]; then
echo mq-deadline > "$dev/queue/scheduler" 2>/dev/null || true
fi
done
# Keep both active 40GbE bond members at their verified hardware maximum.
for nic in enp130s0 enp130s0d1; do
ethtool -G "$nic" rx 8192 tx 8192 2>/dev/null || true
done
'';
};
boot.kernel.sysctl = {
# Network buffers for high-throughput storage
"net.core.rmem_max" = 134217728;
"net.core.wmem_max" = 134217728;
"net.core.rmem_default" = 16777216;
"net.core.wmem_default" = 16777216;
"net.core.netdev_max_backlog" = 30000;
"net.core.optmem_max" = 67108864;
# TCP tuning
"net.ipv4.tcp_rmem" = "4096 1048576 134217728";
"net.ipv4.tcp_wmem" = "4096 1048576 134217728";
"net.ipv4.tcp_congestion_control" = "bbr";
"net.ipv4.tcp_mtu_probing" = 1;
"net.ipv4.tcp_window_scaling" = 1;
"net.ipv4.tcp_timestamps" = 1;
"net.ipv4.tcp_sack" = 1;
"net.ipv4.tcp_slow_start_after_idle" = 0;
# Low-latency network polling
"net.core.busy_read" = 50;
"net.core.busy_poll" = 50;
# Memory management for large RAM
"vm.swappiness" = 5;
"vm.dirty_ratio" = 40;
"vm.dirty_background_ratio" = 10;
"vm.vfs_cache_pressure" = 50;
"vm.min_free_kbytes" = 1048576;
"vm.zone_reclaim_mode" = 0;
# No huge pages are configured on this storage server. Proactive
# compaction supplied little value and was the trigger that walked a
# previously corrupted ZFS-backed folio during the 2026-07-22 lockup.
"vm.compaction_proactiveness" = 0;
# Continuing after an Oops risks further corruption. Panic, persist the
# trace through pstore/ERST, then let panic=60 or the IPMI watchdog reboot.
"kernel.panic_on_oops" = 1;
"kernel.softlockup_panic" = 1;
"kernel.hardlockup_panic" = 1;
# NFS server tuning
"sunrpc.tcp_slot_table_entries" = 128;
"sunrpc.udp_slot_table_entries" = 128;
# File descriptor limits
"fs.file-max" = 2097152;
"fs.nr_open" = 2097152;
};
security.pam.loginLimits = [
{ domain = "*"; type = "soft"; item = "nofile"; value = "1048576"; }
{ domain = "*"; type = "hard"; item = "nofile"; value = "1048576"; }
];
# ==========================================================================
# ZFS SERVICES
# ==========================================================================
services.zfs = {
autoScrub = {
enable = true;
interval = "Sun *-*-01..07 02:00:00";
pools = [ "rpool" "dick" ];
};
autoSnapshot = {
enable = true;
flags = "-k -p --utc";
frequent = 4;
hourly = 24;
daily = 7;
weekly = 4;
monthly = 12;
};
trim = {
enable = true;
interval = "weekly";
};
};
# ==========================================================================
# HOST-SPECIFIC USERS
# ==========================================================================
users.users.ldx = {
extraGroups = [ "storage" ];
hashedPassword = "$y$j9T$hHcj2QYj1.AXbLEALbvr/.$WuDsa.hRDcBWN5s.dJX.KHm9rgkgP/NpNlp3bs2vvs3";
};
users.users."ye-lw21" = {
uid = 1002;
extraGroups = [ "storage" ];
hashedPassword = "$y$j9T$hia.9h7L/5q7G4QdKFHOA1$fAFFSpJRf57ZEvCVjDjwM1WH8UPR5E1Xy28KeJQ.gfD";
};
users.users.ajax = {
isNormalUser = true;
extraGroups = [ "wheel" "storage" ];
hashedPassword = "$y$j9T$Y2Noz4MN.XlE5lu1E0axQ/$MJzDruFklLRjK.BJpr4ws.x/AomIGOnD8NavLy.jgH1";
openssh.authorizedKeys.keys = [
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGTFzIwnvYEXjBdr4K2Wg/ujrel7moGpto4mOOI984EU Main SSH Key"
];
};
# zhuyz24 (朱驭之) — LDAP-backed data-pool user. Local override pins the Unix
# UID to the LDAP uidNumber (2200000020) so on-disk ownership stays consistent
# whether resolved via local NSS or pure LDAP. SSH-key only, no wheel, no
# password. SMB password lives in LDAP after one-time `smbpasswd -a zhuyz24`.
users.users.zhuyz24 = {
isNormalUser = true;
uid = 2200000020;
extraGroups = [ "storage" ];
openssh.authorizedKeys.keys = [
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHaHj2lCOktkU0MnUxQo9ElhBw7/iZ5C2e+GkHdwP38t [email protected]"
];
};
# ==========================================================================
# LDAP IDENTITY
# ==========================================================================
age.secrets.skydick-ldap-bind = {
file = ../../secrets/skydick-ldap-bind.age;
owner = "nslcd";
group = "nslcd";
mode = "0400";
};
age.secrets.skydick-samba-ldap-admin = {
file = ../../secrets/skydick-samba-ldap-admin.age;
owner = "root";
group = "root";
mode = "0400";
};
# LDAP is used for POSIX identity lookups. SMB account data is also stored in
# LDAP via Samba's ldapsam backend. Same-name local admin overlays still win
# for the final Unix UID/GID on skydick, while SMB password data remains in
# LDAP.
users.ldap = {
enable = true;
loginPam = false;
nsswitch = true;
daemon.enable = true;
server = "ldap://ldap.skyw.top/";
base = "dc=skyw,dc=top";
useTLS = true;
timeLimit = 5;
bind = {
distinguishedName = "cn=query_user,dc=skyw,dc=top";
passwordFile = config.age.secrets.skydick-ldap-bind.path;
timeLimit = 5;
policy = "hard_open";
};
daemon.extraConfig = ''
ssl start_tls
tls_reqcert demand
tls_cacertfile /etc/ssl/certs/ca-certificates.crt
nss_initgroups_ignoreusers ALLLOCAL
'';
};
# The LDAP module only sees the stable /run/agenix path. Track the encrypted
# source so rotating the bind secret restarts nslcd and rewrites its config.
systemd.services.nslcd.restartTriggers = [
config.age.secrets.skydick-ldap-bind.file
];
# ==========================================================================
# MONITORING
# ==========================================================================
services.smartd = {
enable = true;
autodetect = true;
};
# ==========================================================================
# PACKAGES
# ==========================================================================
environment.systemPackages = with pkgs; [
# ZFS & storage
zfs
targetcli-fb
sg3_utils
sdparm
nvme-cli
# Monitoring
btop
iotop
iftop
smartmontools
lm_sensors
sysstat
dool
ipmitool
# Network
iperf3
ethtool
tcpdump
openldap
# Performance & NUMA
numactl
perf-tools
perf
];
# ==========================================================================
# INFLUXDB + TELEGRAF MONITORING
# ==========================================================================
skyworks.influxdb = {
enable = true;
openFirewall = false;
};
skyworks.monitoring = {
enable = true;
influxUrl = "http://127.0.0.1:8086";
bucket = "skydick";
netInterfaces = [ "bond40g" ];
nodeExporter = {
enable = true;
openFirewall = false;
};
};
system.stateVersion = "25.11";
}
