# Skydick Storage Server - NixOS Configuration
# Hardware: Dual E5-2699 v3, 256GB RAM, 36-bay SAS chassis (Inventec K800G3-10G)
{ config, pkgs, lib, ... }:

{
  # ==========================================================================
  # SYSTEM IDENTITY
  # ==========================================================================
  networking.hostName = "skydick";
  networking.hostId = "8425e349";  # Required for ZFS

  # Time sync: pin local CERNET NTP. The NixOS default pool resolved to an
  # abroad Vultr server (0.nixos.pool.ntp.org, 137ms delay / 42ms jitter ->
  # ~60ms offset that flapped the clock-drift alert). ntp.tuna.tsinghua.edu.cn
  # is on-campus (sub-ms); ntp.ntsc.ac.cn (NTSC stratum-1, ~19ms) is the backup.
  networking.timeServers = [ "ntp.tuna.tsinghua.edu.cn" "ntp.ntsc.ac.cn" ];

  # ==========================================================================
  # HARDWARE CONFIGURATION
  # ==========================================================================
  nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
  hardware.cpu.intel.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
  hardware.enableRedistributableFirmware = true;

  # ==========================================================================
  # BOOT CONFIGURATION
  # ==========================================================================
  boot = {
    loader = {
      systemd-boot.enable = true;
      efi.canTouchEfiVariables = true;
      timeout = 3;
    };

    supportedFilesystems = [ "zfs" ];
    kernelPackages = pkgs.linuxPackages_6_6;  # LTS kernel (Dec 2026), best ZFS stability
    kernelModules = [ "kvm-intel" ];

    # ZFS tunables for 256GB RAM storage server
    kernelParams = [
      "zfs.zfs_arc_max=137438953472"      # 128GB ARC max
      "zfs.zfs_arc_min=34359738368"       # 32GB ARC min
      "zfs.zfs_txg_timeout=5"
      "zfs.zfs_vdev_scrub_min_active=1"
      "zfs.zfs_vdev_scrub_max_active=2"
      "zfs.zfs_dirty_data_max_percent=25"
      "zfs.zfs_vdev_async_read_max_active=32"
      "zfs.zfs_vdev_async_read_min_active=4"
      "zfs.zfs_vdev_async_write_max_active=8"
      "zfs.zfetch_max_streams=16"
      "zfs.l2arc_write_max=536870912"     # 512MB/s L2ARC write
      "zfs.l2arc_write_boost=1073741824"  # 1GB/s L2ARC warmup
    ];

    initrd = {
      supportedFilesystems = [ "zfs" ];
      availableKernelModules = [
        "xhci_pci" "ehci_pci" "ahci" "mpt3sas"  # SAS HBA
        "sd_mod" "sr_mod"
        "usb_storage" "usbhid"
        "mlx5_core"  # Mellanox ConnectX-4/5
      ];
    };

    zfs = {
      devNodes = "/dev/disk/by-id";
      forceImportRoot = false;
      extraPools = [ "dick" ];
    };
  };

  # ==========================================================================
  # NETWORK CONFIGURATION
  # ==========================================================================
  networking = {
    useDHCP = false;
    useNetworkd = true;

    # bond40g: 2× 40G ConnectX-3 (enp130s0 + enp130s0d1) in LACP 802.3ad
    # layer3+4 to cisco Po5 (Eth1/51 + Eth1/52). Carries 10.0.1.1/16 and the
    # default route since the 2026-05-15 cutover from the original 25G
    # ConnectX-4 LX bond0 (active-backup), which was torn down — its
    # ports enp4s0f0np0 / enp4s0f1np1 are now standalone DOWN.
    bonds.bond40g = {
      interfaces = [ "enp130s0" "enp130s0d1" ];
      driverOptions = {
        mode = "802.3ad";
        miimon = "100";
        xmit_hash_policy = "layer3+4";
        lacp_rate = "fast";
      };
    };

    # bond40g carries the host IP (moved off bond0 at the 2026-05-15 cutover).
    interfaces.bond40g = {
      ipv4.addresses = [{
        address = "10.0.1.1";
        prefixLength = 16;
      }];
      mtu = 9200;
      # Disable IPv6 privacy/temporary addresses: NixOS otherwise emits
      # net.ipv6.conf.bond40g.use_tempaddr=2, which makes the kernel generate
      # and prefer a rotating temporary address over the stable ::d1c0 token.
      # A storage server wants a fixed source address (ACLs, logging, rDNS).
      tempAddress = "disabled";
    };

    defaultGateway = {
      address = "10.0.0.1";
      interface = "bond40g";
    };
    # IPv6 is enabled for SLAAC on bond40g. Keep DNS pinned to main MosDNS:
    # RA/DHCPv6-provided DNS is disabled in the networkd link config below.
    enableIPv6 = true;

    firewall = {
      enable = true;
      backend = "nftables";

      # This host has a globally routed SLAAC address. Keep storage and
      # monitoring services scoped to their actual clients instead of opening
      # them on every IPv4 and IPv6 address.
      extraInputRules = ''
        iifname "bond40g" ip saddr 10.0.0.0/8 tcp dport 22 accept comment "SSH from routed private networks"
        iifname "bond40g" ip6 saddr { fd99:23eb:1682::/48, 2a0c:b641:69c:ada0::/64 } tcp dport 22 accept comment "SSH from Skyworks IPv6 networks"

        iifname "bond40g" ip saddr 10.0.0.0/16 meta l4proto { tcp, udp } th dport { 111, 2049, 20001-20003 } accept comment "NFS IPv4 clients"
        iifname "bond40g" ip6 saddr fd99:23eb:1682::75:15 meta l4proto { tcp, udp } th dport { 111, 2049, 20001-20003 } accept comment "door-pek NFS over ULA"
        iifname "bond40g" ip6 saddr 2a0c:b641:69c:ada0::/64 meta l4proto { tcp, udp } th dport { 111, 2049, 20001-20003 } accept comment "declared NFS global IPv6 clients"

        iifname "bond40g" ip saddr { 10.0.0.0/16, 10.253.0.0/16 } tcp dport 445 accept comment "SMB client networks"
        iifname "bond40g" ip saddr 10.0.200.11 tcp dport 3260 accept comment "rh1288v3 iSCSI initiator"
        iifname "bond40g" ip saddr 10.0.75.15 tcp dport 8086 accept comment "door1 InfluxDB and Grafana"
        iifname "bond40g" ip saddr { 10.0.0.1, 10.0.75.15 } tcp dport 9100 accept comment "node exporter scrapers"
      '';
    };
  };

  networking.nftables.enable = true;
  services.openssh.openFirewall = false;

  # systemd-networkd: accept RA on bond40g for SLAAC, but keep DNS/search
  # domains and the default route under the explicit IPv4 config above.
  systemd.network.networks."40-bond40g" = {
    networkConfig = {
      IPv6AcceptRA = true;
      # NixOS unconditionally sets IPv6PrivacyExtensions = "kernel" (normal
      # priority) for every interface in networking.interfaces, so mkForce is
      # required to disable privacy extensions and pin the stable ::d1c0 token
      # address as the source — a plain `false` collides with that default.
      IPv6PrivacyExtensions = lib.mkForce false;
    };
    ipv6AcceptRAConfig = {
      Token = "::d1c0";
      UseDNS = false;
      UseGateway = false;
      UseDomains = false;
      DHCPv6Client = false;
    };
  };

  # Use only main's filtered MosDNS endpoints. Global DNS servers are peers in
  # resolved, not ordered failovers; adding AliDNS here can intermittently
  # return polluted foreign-domain answers, while adding a public resolver
  # bypasses the network's split-DNS and blocking policy.
  services.resolved = {
    enable = true;
    settings.Resolve = {
      DNS = [ "10.0.0.1" "fd99:23eb:1682::1" ];
      FallbackDNS = "";
      LLMNR = false;
      MulticastDNS = false;
    };
  };

  # Trust the private CA used by the LDAP service on the main gateway.
  security.pki.certificateFiles = [ ../../certs/skyw-ldap-ca.crt ];

  # Wait only for bond40g, not individual member ports — a disconnected port
  # (cable maintenance) should not stall boot by 2 minutes.
  systemd.network.wait-online.anyInterface = true;

  # ==========================================================================
  # KERNEL PERFORMANCE TUNING
  # ==========================================================================
  powerManagement.cpuFreqGovernor = "performance";

  services.udev.extraRules = ''
    # I/O schedulers by media type (no kernel-name assumptions)
    ACTION=="add|change", KERNEL=="sd*", ATTR{queue/rotational}=="1", ATTR{queue/scheduler}="mq-deadline"
    ACTION=="add|change", KERNEL=="sd*", ATTR{queue/rotational}=="0", ATTR{queue/scheduler}="none"
    ACTION=="add|change", KERNEL=="nvme[0-9]*", ATTR{queue/scheduler}="none"
  '';

  # Belt-and-suspenders: enforce I/O schedulers and NIC ring buffers at boot.
  # The udev rules above handle hotplug; this service catches anything the
  # rules miss due to driver-level scheduler resets on SAS devices.
  systemd.services.storage-tuning = {
    description = "Storage I/O scheduler and NIC ring buffer tuning";
    after = [ "systemd-udevd.service" "network.target" ];
    wantedBy = [ "multi-user.target" ];
    serviceConfig = {
      Type = "oneshot";
      RemainAfterExit = true;
    };
    path = [ pkgs.ethtool ];
    script = ''
      # mq-deadline for rotational drives (SAS Mach2 HDDs)
      for dev in /sys/block/sd*; do
        [ -d "$dev/queue" ] || continue
        if [ "$(cat "$dev/queue/rotational" 2>/dev/null)" = "1" ]; then
          echo mq-deadline > "$dev/queue/scheduler" 2>/dev/null || true
        fi
      done

      # Keep both active 40GbE bond members at their verified hardware maximum.
      for nic in enp130s0 enp130s0d1; do
        ethtool -G "$nic" rx 8192 tx 8192 2>/dev/null || true
      done
    '';
  };

  boot.kernel.sysctl = {
    # Network buffers for high-throughput storage
    "net.core.rmem_max" = 134217728;
    "net.core.wmem_max" = 134217728;
    "net.core.rmem_default" = 16777216;
    "net.core.wmem_default" = 16777216;
    "net.core.netdev_max_backlog" = 30000;
    "net.core.optmem_max" = 67108864;

    # TCP tuning
    "net.ipv4.tcp_rmem" = "4096 1048576 134217728";
    "net.ipv4.tcp_wmem" = "4096 1048576 134217728";
    "net.ipv4.tcp_congestion_control" = "bbr";
    "net.ipv4.tcp_mtu_probing" = 1;
    "net.ipv4.tcp_window_scaling" = 1;
    "net.ipv4.tcp_timestamps" = 1;
    "net.ipv4.tcp_sack" = 1;
    "net.ipv4.tcp_slow_start_after_idle" = 0;

    # Low-latency network polling
    "net.core.busy_read" = 50;
    "net.core.busy_poll" = 50;

    # Memory management for large RAM
    "vm.swappiness" = 5;
    "vm.dirty_ratio" = 40;
    "vm.dirty_background_ratio" = 10;
    "vm.vfs_cache_pressure" = 50;
    "vm.min_free_kbytes" = 1048576;
    "vm.zone_reclaim_mode" = 0;

    # NFS server tuning
    "sunrpc.tcp_slot_table_entries" = 128;
    "sunrpc.udp_slot_table_entries" = 128;

    # File descriptor limits
    "fs.file-max" = 2097152;
    "fs.nr_open" = 2097152;
  };

  security.pam.loginLimits = [
    { domain = "*"; type = "soft"; item = "nofile"; value = "1048576"; }
    { domain = "*"; type = "hard"; item = "nofile"; value = "1048576"; }
  ];

  # ==========================================================================
  # ZFS SERVICES
  # ==========================================================================
  services.zfs = {
    autoScrub = {
      enable = true;
      interval = "Sun *-*-01..07 02:00:00";
      pools = [ "rpool" "dick" ];
    };

    autoSnapshot = {
      enable = true;
      flags = "-k -p --utc";
      frequent = 4;
      hourly = 24;
      daily = 7;
      weekly = 4;
      monthly = 12;
    };

    trim = {
      enable = true;
      interval = "weekly";
    };
  };

  # ==========================================================================
  # HOST-SPECIFIC USERS
  # ==========================================================================
  users.users.ldx = {
    extraGroups = [ "storage" ];
    hashedPassword = "$y$j9T$hHcj2QYj1.AXbLEALbvr/.$WuDsa.hRDcBWN5s.dJX.KHm9rgkgP/NpNlp3bs2vvs3";
  };

  users.users."ye-lw21" = {
    uid = 1002;
    extraGroups = [ "storage" ];
    hashedPassword = "$y$j9T$hia.9h7L/5q7G4QdKFHOA1$fAFFSpJRf57ZEvCVjDjwM1WH8UPR5E1Xy28KeJQ.gfD";
  };

  users.users.ajax = {
    isNormalUser = true;
    extraGroups = [ "wheel" "storage" ];
    hashedPassword = "$y$j9T$Y2Noz4MN.XlE5lu1E0axQ/$MJzDruFklLRjK.BJpr4ws.x/AomIGOnD8NavLy.jgH1";
    openssh.authorizedKeys.keys = [
      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGTFzIwnvYEXjBdr4K2Wg/ujrel7moGpto4mOOI984EU Main SSH Key"
    ];
  };

  # zhuyz24 (朱驭之) — LDAP-backed data-pool user. Local override pins the Unix
  # UID to the LDAP uidNumber (2200000020) so on-disk ownership stays consistent
  # whether resolved via local NSS or pure LDAP. SSH-key only, no wheel, no
  # password. SMB password lives in LDAP after one-time `smbpasswd -a zhuyz24`.
  users.users.zhuyz24 = {
    isNormalUser = true;
    uid = 2200000020;
    extraGroups = [ "storage" ];
    openssh.authorizedKeys.keys = [
      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHaHj2lCOktkU0MnUxQo9ElhBw7/iZ5C2e+GkHdwP38t [email protected]"
    ];
  };

  # ==========================================================================
  # LDAP IDENTITY
  # ==========================================================================
  age.secrets.skydick-ldap-bind = {
    file = ../../secrets/skydick-ldap-bind.age;
    owner = "nslcd";
    group = "nslcd";
    mode = "0400";
  };

  age.secrets.skydick-samba-ldap-admin = {
    file = ../../secrets/skydick-samba-ldap-admin.age;
    owner = "root";
    group = "root";
    mode = "0400";
  };

  # LDAP is used for POSIX identity lookups. SMB account data is also stored in
  # LDAP via Samba's ldapsam backend. Same-name local admin overlays still win
  # for the final Unix UID/GID on skydick, while SMB password data remains in
  # LDAP.
  users.ldap = {
    enable = true;
    loginPam = false;
    nsswitch = true;
    daemon.enable = true;
    server = "ldap://ldap.skyw.top/";
    base = "dc=skyw,dc=top";
    useTLS = true;
    timeLimit = 5;

    bind = {
      distinguishedName = "cn=query_user,dc=skyw,dc=top";
      passwordFile = config.age.secrets.skydick-ldap-bind.path;
      timeLimit = 5;
      policy = "hard_open";
    };

    daemon.extraConfig = ''
      ssl start_tls
      tls_reqcert demand
      tls_cacertfile /etc/ssl/certs/ca-certificates.crt
      nss_initgroups_ignoreusers ALLLOCAL
    '';
  };

  # The LDAP module only sees the stable /run/agenix path. Track the encrypted
  # source so rotating the bind secret restarts nslcd and rewrites its config.
  systemd.services.nslcd.restartTriggers = [
    config.age.secrets.skydick-ldap-bind.file
  ];

  # ==========================================================================
  # MONITORING
  # ==========================================================================
  services.smartd = {
    enable = true;
    autodetect = true;
  };

  # ==========================================================================
  # PACKAGES
  # ==========================================================================
  environment.systemPackages = with pkgs; [
    # ZFS & storage
    zfs
    targetcli-fb
    sg3_utils
    sdparm
    nvme-cli

    # Monitoring
    btop
    iotop
    iftop
    smartmontools
    lm_sensors
    sysstat
    dool

    # Network
    iperf3
    ethtool
    tcpdump
    openldap

    # Performance & NUMA
    numactl
    perf-tools
    perf
  ];

  # ==========================================================================
  # INFLUXDB + TELEGRAF MONITORING
  # ==========================================================================
  skyworks.influxdb = {
    enable = true;
    openFirewall = false;
  };

  skyworks.monitoring = {
    enable = true;
    influxUrl = "http://127.0.0.1:8086";
    bucket = "skydick";
    netInterfaces = [ "bond40g" ];
    nodeExporter = {
      enable = true;
      openFirewall = false;
    };
  };

  system.stateVersion = "25.11";
}