Newer
Older
skyworks-Nix-infra / hosts / skydick / default.nix
# Skydick Storage Server - NixOS Configuration
# Hardware: Dual E5-2699 v3, 256GB RAM, 36-bay SAS chassis (Inventec K800G3-10G)
{ config, pkgs, lib, ... }:

{
  # ==========================================================================
  # SYSTEM IDENTITY
  # ==========================================================================
  networking.hostName = "skydick";
  networking.hostId = "8425e349";  # Required for ZFS

  # ==========================================================================
  # HARDWARE CONFIGURATION
  # ==========================================================================
  nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
  hardware.cpu.intel.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
  hardware.enableRedistributableFirmware = true;

  # ==========================================================================
  # BOOT CONFIGURATION
  # ==========================================================================
  boot = {
    loader = {
      systemd-boot.enable = true;
      efi.canTouchEfiVariables = true;
      timeout = 3;
    };

    supportedFilesystems = [ "zfs" ];
    kernelPackages = pkgs.linuxPackages_6_6;  # LTS kernel (Dec 2026), best ZFS stability
    kernelModules = [ "kvm-intel" ];

    # ZFS tunables for 256GB RAM storage server
    kernelParams = [
      "zfs.zfs_arc_max=137438953472"      # 128GB ARC max
      "zfs.zfs_arc_min=34359738368"       # 32GB ARC min
      "zfs.zfs_txg_timeout=5"
      "zfs.zfs_vdev_scrub_min_active=1"
      "zfs.zfs_vdev_scrub_max_active=2"
      "zfs.zfs_dirty_data_max_percent=25"
      "zfs.zfs_vdev_async_read_max_active=8"
      "zfs.zfs_vdev_async_write_max_active=8"
      "zfs.l2arc_write_max=536870912"     # 512MB/s L2ARC write
      "zfs.l2arc_write_boost=1073741824"  # 1GB/s L2ARC warmup
    ];

    initrd = {
      supportedFilesystems = [ "zfs" ];
      availableKernelModules = [
        "xhci_pci" "ehci_pci" "ahci" "mpt3sas"  # SAS HBA
        "sd_mod" "sr_mod"
        "usb_storage" "usbhid"
        "mlx5_core"  # Mellanox ConnectX-4/5
      ];
    };

    zfs = {
      devNodes = "/dev/disk/by-id";
      forceImportRoot = false;
      extraPools = [ "dick" ];
    };
  };

  # ==========================================================================
  # NETWORK CONFIGURATION
  # ==========================================================================
  networking = {
    useDHCP = false;
    useNetworkd = true;

    bonds.bond0 = {
      interfaces = [ "enp4s0f0np0" "enp4s0f1np1" ];
      driverOptions = {
        mode = "active-backup";
        primary = "enp4s0f0np0";
        miimon = "100";
        fail_over_mac = "active";
      };
    };

    interfaces.bond0 = {
      ipv4.addresses = [{
        address = "10.0.1.1";
        prefixLength = 16;
      }];
      mtu = 9000;  # Jumbo frames for storage traffic
    };

    defaultGateway = {
      address = "10.0.0.1";
      interface = "bond0";
    };
    nameservers = [ "10.0.0.1" "223.5.5.5" ];

    # Prefer IPv4 for outbound connections — IPv6 goes through wg-outbound
    # on 10.0.0.1 which masquerades with ULA (not globally routable)
    enableIPv6 = false;

    firewall = {
      enable = true;
      allowedTCPPorts = [ 22 ];
    };
  };

  # ==========================================================================
  # KERNEL PERFORMANCE TUNING
  # ==========================================================================
  powerManagement.cpuFreqGovernor = "performance";

  services.udev.extraRules = ''
    # SAS/SATA HDDs - use mq-deadline
    ACTION=="add|change", KERNEL=="sd[c-z]", ATTR{queue/rotational}=="1", ATTR{queue/scheduler}="mq-deadline"
    # SSDs/NVMe - use none
    ACTION=="add|change", KERNEL=="sd[a-b]", ATTR{queue/rotational}=="0", ATTR{queue/scheduler}="none"
    ACTION=="add|change", KERNEL=="nvme[0-9]*", ATTR{queue/scheduler}="none"
  '';

  boot.kernel.sysctl = {
    # Network buffers for high-throughput storage
    "net.core.rmem_max" = 134217728;
    "net.core.wmem_max" = 134217728;
    "net.core.rmem_default" = 16777216;
    "net.core.wmem_default" = 16777216;
    "net.core.netdev_max_backlog" = 30000;
    "net.core.optmem_max" = 67108864;

    # TCP tuning
    "net.ipv4.tcp_rmem" = "4096 1048576 134217728";
    "net.ipv4.tcp_wmem" = "4096 1048576 134217728";
    "net.ipv4.tcp_congestion_control" = "bbr";
    "net.ipv4.tcp_mtu_probing" = 1;
    "net.ipv4.tcp_window_scaling" = 1;
    "net.ipv4.tcp_timestamps" = 1;
    "net.ipv4.tcp_sack" = 1;
    "net.ipv4.tcp_slow_start_after_idle" = 0;

    # Low-latency network polling
    "net.core.busy_read" = 50;
    "net.core.busy_poll" = 50;

    # Memory management for large RAM
    "vm.swappiness" = 5;
    "vm.dirty_ratio" = 40;
    "vm.dirty_background_ratio" = 10;
    "vm.vfs_cache_pressure" = 50;
    "vm.min_free_kbytes" = 1048576;
    "vm.zone_reclaim_mode" = 0;

    # NFS server tuning
    "sunrpc.tcp_slot_table_entries" = 128;
    "sunrpc.udp_slot_table_entries" = 128;

    # File descriptor limits
    "fs.file-max" = 2097152;
    "fs.nr_open" = 2097152;
  };

  security.pam.loginLimits = [
    { domain = "*"; type = "soft"; item = "nofile"; value = "1048576"; }
    { domain = "*"; type = "hard"; item = "nofile"; value = "1048576"; }
  ];

  # ==========================================================================
  # ZFS SERVICES
  # ==========================================================================
  services.zfs = {
    autoScrub = {
      enable = true;
      interval = "Sun *-*-01..07 02:00:00";
      pools = [ "rpool" "dick" ];
    };

    autoSnapshot = {
      enable = true;
      flags = "-k -p --utc";
      frequent = 4;
      hourly = 24;
      daily = 7;
      weekly = 4;
      monthly = 12;
    };

    trim = {
      enable = true;
      interval = "weekly";
    };
  };

  # ==========================================================================
  # HOST-SPECIFIC USERS
  # ==========================================================================
  users.users.ldx = {
    extraGroups = [ "storage" ];
    hashedPassword = "$y$j9T$hHcj2QYj1.AXbLEALbvr/.$WuDsa.hRDcBWN5s.dJX.KHm9rgkgP/NpNlp3bs2vvs3";
  };

  users.users.ylw = {
    extraGroups = [ "storage" ];
    hashedPassword = "$y$j9T$hia.9h7L/5q7G4QdKFHOA1$fAFFSpJRf57ZEvCVjDjwM1WH8UPR5E1Xy28KeJQ.gfD";
  };

  # ==========================================================================
  # LDAP IDENTITY
  # ==========================================================================
  age.secrets.skydick-ldap-bind = {
    file = ../../secrets/skydick-ldap-bind.age;
    owner = "nslcd";
    group = "nslcd";
    mode = "0400";
  };

  # LDAP is used here for POSIX identity lookups only. SMB still uses Samba's
  # local passdb until sambaSamAccount objects are provisioned in LDAP.
  users.ldap = {
    enable = true;
    loginPam = false;
    nsswitch = true;
    daemon.enable = true;
    server = "ldap://10.0.0.1/";
    base = "dc=skyw,dc=top";
    useTLS = false;
    timeLimit = 5;

    bind = {
      distinguishedName = "cn=query_user,dc=skyw,dc=top";
      passwordFile = config.age.secrets.skydick-ldap-bind.path;
      timeLimit = 5;
      policy = "hard_open";
    };

    daemon.extraConfig = ''
      nss_initgroups_ignoreusers ALLLOCAL
    '';
  };

  # ==========================================================================
  # MONITORING
  # ==========================================================================
  services.smartd = {
    enable = true;
    autodetect = true;
  };

  # ==========================================================================
  # PACKAGES
  # ==========================================================================
  environment.systemPackages = with pkgs; [
    # ZFS & storage
    zfs
    targetcli
    sg3_utils
    sdparm
    nvme-cli

    # Monitoring
    btop
    iotop
    iftop
    smartmontools
    lm_sensors
    sysstat
    dstat

    # Network
    iperf3
    ethtool
    tcpdump
    openldap

    # Performance & NUMA
    numactl
    perf-tools
    linuxPackages_6_6.perf
  ];

  # ==========================================================================
  # TELEGRAF MONITORING
  # ==========================================================================
  skyworks.monitoring = {
    enable = true;
    bucket = "skydick";
    netInterfaces = [ "bond0" ];
  };

  system.stateVersion = "25.11";
}