Newer
Older
skyworks-Nix-infra / hosts / skydick / default.nix
# Skydick Storage Server - NixOS Configuration
# Hardware: Dual E5-2699 v3, 256GB RAM, 36-bay SAS chassis (Inventec K800G3-10G)
{ config, pkgs, lib, ... }:

{
  # ==========================================================================
  # SYSTEM IDENTITY
  # ==========================================================================
  networking.hostName = "skydick";
  networking.hostId = "8425e349";  # Required for ZFS

  # ==========================================================================
  # HARDWARE CONFIGURATION
  # ==========================================================================
  nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
  hardware.cpu.intel.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
  hardware.enableRedistributableFirmware = true;

  # ==========================================================================
  # BOOT CONFIGURATION
  # ==========================================================================
  boot = {
    loader = {
      systemd-boot.enable = true;
      efi.canTouchEfiVariables = true;
      timeout = 3;
    };

    supportedFilesystems = [ "zfs" ];
    kernelPackages = pkgs.linuxPackages_6_6;  # LTS kernel (Dec 2026), best ZFS stability
    kernelModules = [ "kvm-intel" ];

    # ZFS tunables for 256GB RAM storage server
    kernelParams = [
      "zfs.zfs_arc_max=137438953472"      # 128GB ARC max
      "zfs.zfs_arc_min=34359738368"       # 32GB ARC min
      "zfs.zfs_txg_timeout=5"
      "zfs.zfs_vdev_scrub_min_active=1"
      "zfs.zfs_vdev_scrub_max_active=2"
      "zfs.zfs_dirty_data_max_percent=25"
      "zfs.zfs_vdev_async_read_max_active=8"
      "zfs.zfs_vdev_async_write_max_active=8"
      "zfs.l2arc_write_max=536870912"     # 512MB/s L2ARC write
      "zfs.l2arc_write_boost=1073741824"  # 1GB/s L2ARC warmup
    ];

    initrd = {
      supportedFilesystems = [ "zfs" ];
      availableKernelModules = [
        "xhci_pci" "ehci_pci" "ahci" "mpt3sas"  # SAS HBA
        "sd_mod" "sr_mod"
        "usb_storage" "usbhid"
        "mlx5_core"  # Mellanox ConnectX-4/5
      ];
    };

    zfs = {
      devNodes = "/dev/disk/by-id";
      forceImportRoot = false;
      extraPools = [ "dick" ];
    };
  };

  # ==========================================================================
  # NETWORK CONFIGURATION
  # ==========================================================================
  networking = {
    useDHCP = false;
    useNetworkd = true;

    bonds.bond0 = {
      interfaces = [ "enp4s0f0np0" "enp4s0f1np1" ];
      driverOptions = {
        mode = "active-backup";
        primary = "enp4s0f0np0";
        miimon = "100";
        fail_over_mac = "active";
      };
    };

    interfaces.bond0 = {
      ipv4.addresses = [{
        address = "10.0.1.1";
        prefixLength = 16;
      }];
      mtu = 9000;  # Jumbo frames for storage traffic
    };

    defaultGateway = {
      address = "10.0.0.1";
      interface = "bond0";
    };
    nameservers = [ "10.0.0.1" "223.5.5.5" ];

    firewall = {
      enable = true;
      allowedTCPPorts = [
        22    # SSH
        111   # RPC (NFS)
        2049  # NFS
        445   # SMB
        139   # NetBIOS (SMB)
        3260  # iSCSI
      ];
      allowedUDPPorts = [
        111   # RPC (NFS)
        2049  # NFS (NFSv4.1+)
        137   # NetBIOS Name Service
        138   # NetBIOS Datagram
      ];
      allowedTCPPortRanges = [{ from = 20000; to = 20005; }];
      allowedUDPPortRanges = [{ from = 20000; to = 20005; }];
    };
  };

  services.rpcbind.enable = true;

  # ==========================================================================
  # KERNEL PERFORMANCE TUNING
  # ==========================================================================
  powerManagement.cpuFreqGovernor = "performance";

  services.udev.extraRules = ''
    # SAS/SATA HDDs - use mq-deadline
    ACTION=="add|change", KERNEL=="sd[c-z]", ATTR{queue/rotational}=="1", ATTR{queue/scheduler}="mq-deadline"
    # SSDs/NVMe - use none
    ACTION=="add|change", KERNEL=="sd[a-b]", ATTR{queue/rotational}=="0", ATTR{queue/scheduler}="none"
    ACTION=="add|change", KERNEL=="nvme[0-9]*", ATTR{queue/scheduler}="none"
  '';

  boot.kernel.sysctl = {
    # Network buffers for high-throughput storage
    "net.core.rmem_max" = 134217728;
    "net.core.wmem_max" = 134217728;
    "net.core.rmem_default" = 16777216;
    "net.core.wmem_default" = 16777216;
    "net.core.netdev_max_backlog" = 30000;
    "net.core.optmem_max" = 67108864;

    # TCP tuning
    "net.ipv4.tcp_rmem" = "4096 1048576 134217728";
    "net.ipv4.tcp_wmem" = "4096 1048576 134217728";
    "net.ipv4.tcp_congestion_control" = "bbr";
    "net.ipv4.tcp_mtu_probing" = 1;
    "net.ipv4.tcp_window_scaling" = 1;
    "net.ipv4.tcp_timestamps" = 1;
    "net.ipv4.tcp_sack" = 1;
    "net.ipv4.tcp_slow_start_after_idle" = 0;

    # Low-latency network polling
    "net.core.busy_read" = 50;
    "net.core.busy_poll" = 50;

    # Memory management for large RAM
    "vm.swappiness" = 5;
    "vm.dirty_ratio" = 40;
    "vm.dirty_background_ratio" = 10;
    "vm.vfs_cache_pressure" = 50;
    "vm.min_free_kbytes" = 1048576;
    "vm.zone_reclaim_mode" = 0;

    # NFS server tuning
    "sunrpc.tcp_slot_table_entries" = 128;
    "sunrpc.udp_slot_table_entries" = 128;

    # File descriptor limits
    "fs.file-max" = 2097152;
    "fs.nr_open" = 2097152;
  };

  security.pam.loginLimits = [
    { domain = "*"; type = "soft"; item = "nofile"; value = "1048576"; }
    { domain = "*"; type = "hard"; item = "nofile"; value = "1048576"; }
  ];

  # ==========================================================================
  # ZFS SERVICES
  # ==========================================================================
  services.zfs = {
    autoScrub = {
      enable = true;
      interval = "Sun *-*-01..07 02:00:00";
      pools = [ "rpool" "dick" ];
    };

    autoSnapshot = {
      enable = true;
      flags = "-k -p --utc";
      frequent = 4;
      hourly = 24;
      daily = 7;
      weekly = 4;
      monthly = 12;
    };

    trim = {
      enable = true;
      interval = "weekly";
    };
  };

  # ==========================================================================
  # NFS SERVER
  # ==========================================================================
  services.nfs.server = {
    enable = true;
    statdPort = 20001;
    lockdPort = 20002;
    mountdPort = 20003;

    exports = ''
      /srv          10.0.0.0/16(rw,sync,fsid=0,crossmnt,no_subtree_check,root_squash)
      /srv/share    10.0.0.0/16(rw,sync,no_subtree_check,root_squash)
      /srv/media    10.0.0.0/16(ro,async,no_subtree_check,root_squash)
      /srv/backup   10.0.0.0/16(rw,sync,no_subtree_check,no_root_squash)
    '';
  };

  services.nfs.idmapd.settings = {
    General = {
      Domain = "skydick.local";
    };
    Mapping = {
      Nobody-User = "nobody";
      Nobody-Group = "nogroup";
    };
  };

  systemd.tmpfiles.rules = [
    "d /srv 0755 root root -"
    "d /srv/share 0755 nobody nogroup -"
    "d /srv/media 0755 nobody nogroup -"
    "d /srv/backup 0700 root root -"
  ];

  # ==========================================================================
  # SAMBA SERVER
  # ==========================================================================
  services.samba = {
    enable = true;
    openFirewall = false;

    settings = {
      global = {
        workgroup = "WORKGROUP";
        "server string" = "Skydick Storage";
        "netbios name" = "SKYDICK";
        security = "user";
        "hosts allow" = "10.0. 127.";
        "hosts deny" = "ALL";

        "socket options" = "TCP_NODELAY IPTOS_LOWDELAY SO_RCVBUF=131072 SO_SNDBUF=131072";
        "use sendfile" = "yes";
        "aio read size" = "16384";
        "aio write size" = "16384";

        "map to guest" = "never";
        "server min protocol" = "SMB2_10";

        "load printers" = "no";
      };

      share = {
        path = "/srv/share";
        browseable = "yes";
        "read only" = "no";
        "guest ok" = "no";
        "valid users" = "@storage";
        "create mask" = "0664";
        "directory mask" = "0775";
      };

      media = {
        path = "/srv/media";
        browseable = "yes";
        "read only" = "yes";
        "guest users" = "@storage";
      };
    };
  };

  users.groups.storage = {};

  services.samba-wsdd = {
    enable = true;
    openFirewall = false;
  };

  # ==========================================================================
  # iSCSI TARGET (LIO)
  # ==========================================================================
  services.target.enable = true;

  # ==========================================================================
  # HOST-SPECIFIC USERS
  # ==========================================================================
  users.users.ldx = {
    extraGroups = [ "storage" ];
    hashedPassword = "$y$j9T$hHcj2QYj1.AXbLEALbvr/.$WuDsa.hRDcBWN5s.dJX.KHm9rgkgP/NpNlp3bs2vvs3";
  };

  users.users.ye-lw21 = {
    isNormalUser = true;
    extraGroups = [ "wheel" "storage" ];
    hashedPassword = "$y$j9T$hia.9h7L/5q7G4QdKFHOA1$fAFFSpJRf57ZEvCVjDjwM1WH8UPR5E1Xy28KeJQ.gfD";
    openssh.authorizedKeys.keys = [
      "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDGMOAuXpGQcwZDOiAbqnVOkBB4jjSs+awGa0WHpDhuZFo61mQNIoE3lozAO58HikY49+JwJccL+oJgA7VNzpy1hl711wbqj8CV3wjJflpnTla86XnXI30w6dlx7qd07JDoXYr6JvCOCk0nbwomXeekxm8Yw1hwjb476ryvZJDvFhIpSx+7j8oP+qlsUDKOhaMdpiDN71b8YihS7jTZvyw9958XB2AH1WjjV1foWk1ux0KIfkqTTZTPkvbkKBsuHFhHzp08OVXTNaROBz+iDcOm8JUodAO/Qbjcaw2pwENjThxwtuXDCYwYv9CN1rQ3lTXo5zC+8MqvGCZBbnhgIvTV8C5W1nRTpOC1d07r0CrNqq4PVPPW1FKBkhUhbuhJCawNjxgQvK2fwP9hZYstrYiXHVgGNmnv9Utg8ENB4fXYKHgFktGhWXMb8oYJkV9wSQuh86hnr3WAjmQu7dIRT02Sp+KPYskFIGQkDoC79PjUVGJC0HZAFLPIBozliF0u/58= ylw-laptop@YLW-LAPTOP"
    ];
  };

  # ==========================================================================
  # MONITORING
  # ==========================================================================
  services.smartd = {
    enable = true;
    autodetect = true;
  };

  # ==========================================================================
  # PACKAGES
  # ==========================================================================
  environment.systemPackages = with pkgs; [
    # ZFS & storage
    zfs
    targetcli
    sg3_utils
    sdparm
    nvme-cli

    # Monitoring
    iotop
    iftop
    smartmontools
    lm_sensors
    sysstat
    dstat

    # Network
    iperf3
    ethtool
    tcpdump

    # Performance & NUMA
    numactl
    perf-tools
    linuxPackages_6_6.perf
  ];

  system.stateVersion = "25.11";
}