diff --git a/hosts/skydick/default.nix b/hosts/skydick/default.nix index 9f01c82..c599d05 100644 --- a/hosts/skydick/default.nix +++ b/hosts/skydick/default.nix @@ -33,8 +33,13 @@ }; supportedFilesystems = [ "zfs" ]; - kernelPackages = pkgs.linuxPackages_6_6; # LTS kernel (Dec 2026), best ZFS stability - kernelModules = [ "kvm-intel" ]; + # 6.18 contains MM/LRU fixes missing from 6.6 and is supported by the + # pinned OpenZFS 2.4.3 package. + kernelPackages = pkgs.linuxPackages_6_18; + kernelModules = [ "kvm-intel" "ipmi_si" "ipmi_watchdog" ]; + extraModprobeConfig = '' + options ipmi_watchdog action=reset timeout=120 panic_wdt_timeout=60 + ''; # ZFS tunables for 256GB RAM storage server kernelParams = [ @@ -50,6 +55,7 @@ "zfs.zfetch_max_streams=16" "zfs.l2arc_write_max=536870912" # 512MB/s L2ARC write "zfs.l2arc_write_boost=1073741824" # 1GB/s L2ARC warmup + "panic=60" ]; initrd = { @@ -184,11 +190,23 @@ # ========================================================================== powerManagement.cpuFreqGovernor = "performance"; + # The BMC exposes a hardware watchdog through the in-band IPMI interface. + # Reboot a wedged host even when the scheduler can no longer run userspace. + systemd.settings.Manager = { + WatchdogDevice = "/dev/watchdog"; + RuntimeWatchdogSec = "2min"; + }; + services.udev.extraRules = '' # I/O schedulers by media type (no kernel-name assumptions) ACTION=="add|change", KERNEL=="sd*", ATTR{queue/rotational}=="1", ATTR{queue/scheduler}="mq-deadline" ACTION=="add|change", KERNEL=="sd*", ATTR{queue/rotational}=="0", ATTR{queue/scheduler}="none" ACTION=="add|change", KERNEL=="nvme[0-9]*", ATTR{queue/scheduler}="none" + + # The BMC virtual-media xHCI root hub loops in usb_runtime_idle while + # suspending, consuming roughly one third of a CPU. Keep only this + # controller's root hubs awake; other USB devices retain runtime PM. + ACTION=="add|bind", SUBSYSTEM=="usb", KERNEL=="usb[0-9]*", KERNELS=="0000:00:14.0", ATTR{idVendor}=="1d6b", TEST=="power/control", ATTR{power/control}="on" ''; # Belt-and-suspenders: enforce I/O schedulers and NIC ring buffers at boot. @@ -249,6 +267,16 @@ "vm.vfs_cache_pressure" = 50; "vm.min_free_kbytes" = 1048576; "vm.zone_reclaim_mode" = 0; + # No huge pages are configured on this storage server. Proactive + # compaction supplied little value and was the trigger that walked a + # previously corrupted ZFS-backed folio during the 2026-07-22 lockup. + "vm.compaction_proactiveness" = 0; + + # Continuing after an Oops risks further corruption. Panic, persist the + # trace through pstore/ERST, then let panic=60 or the IPMI watchdog reboot. + "kernel.panic_on_oops" = 1; + "kernel.softlockup_panic" = 1; + "kernel.hardlockup_panic" = 1; # NFS server tuning "sunrpc.tcp_slot_table_entries" = 128; @@ -405,6 +433,7 @@ lm_sensors sysstat dool + ipmitool # Network iperf3