# Seagate Mach2 SAS dual-actuator data pool ("dick") # # Each Mach2 drive presents TWO LUNs over SAS. CRITICAL: never place both # LUNs of the same physical drive in the same mirror vdev — a single drive # failure would kill the entire vdev. # # SAS WWN identification: # LUN0: wwn-0x6000c500XXXXXXXX0000000000000000 # LUN1: wwn-0x6000c500XXXXXXXX0001000000000000 # Same prefix = same physical drive. Pair DIFFERENT drives in each mirror. # # Drive inventory (ST14000NM0001, 14TB SAS Mach2, all sg_format'd to 4Kn): # # drive1: cab9587b (serial ZKL05VPS...FMAC) # drive2: caf74697 (serial ZKL05VM0...GSP3) # drive3: cb3613eb (serial ZKL09S4X...FA4P) # drive4: cb3957ab (serial ZKL09QXG...50KE) # drive5: cb9dd2eb (serial ZKL0EP06...6LLW) [spare] # drive6: d76a9e4f (serial ZKL0YA0X...4QB) # drive7: d76aa413 (serial ZKL0Y9WR...31Y) # drive8: d82293b3 (serial ZKL1FGTX...D63) # drive9: d822944f (serial ZKL1FGTM...63J) # drive10: c9bcfa0f (serial ZKL01BH9...9J1) [spare] # # Layout (8 active mirrors + 2 hot spares, ~50.9T usable): # # mirror-0: drive1-LUN0 drive2-LUN0 # mirror-1: drive1-LUN1 drive2-LUN1 # mirror-2: drive3-LUN0 drive4-LUN0 # mirror-3: drive3-LUN1 drive4-LUN1 # mirror-5: drive6-LUN0 drive7-LUN0 # mirror-6: drive6-LUN1 drive7-LUN1 # mirror-7: drive8-LUN0 drive9-LUN0 # mirror-8: drive8-LUN1 drive9-LUN1 # spare: drive5-LUN0 drive5-LUN1 drive10-LUN0 drive10-LUN1 # # If driveN fails: the two mirrors carrying its LUNs each lose one member # but stay online via the paired drive. A spare auto-replaces one # degraded member. # # === Pool creation === # # zpool create -o ashift=12 -o autotrim=on -o failmode=continue \ # -O compression=zstd -O atime=off \ # -O xattr=sa -O acltype=posixacl -O dnodesize=auto \ # -O normalization=formD -O redundant_metadata=most \ # -O mountpoint=none -O canmount=off \ # dick \ # mirror wwn-0x6000c500cab9587b0000000000000000 wwn-0x6000c500caf746970000000000000000 \ # mirror wwn-0x6000c500cab9587b0001000000000000 wwn-0x6000c500caf746970001000000000000 \ # mirror wwn-0x6000c500cb3613eb0000000000000000 wwn-0x6000c500cb3957ab0000000000000000 \ # mirror wwn-0x6000c500cb3613eb0001000000000000 wwn-0x6000c500cb3957ab0001000000000000 \ # spare wwn-0x6000c500cb9dd2eb0000000000000000 wwn-0x6000c500cb9dd2eb0001000000000000 # # Expansion (2026-03-23, added 4 drives = 4 mirror vdevs): # zpool add dick \ # mirror wwn-0x6000c500d76a9e4f0000000000000000 wwn-0x6000c500d76aa4130000000000000000 \ # mirror wwn-0x6000c500d76a9e4f0001000000000000 wwn-0x6000c500d76aa4130001000000000000 \ # mirror wwn-0x6000c500d82293b30000000000000000 wwn-0x6000c500d822944f0000000000000000 \ # mirror wwn-0x6000c500d82293b30001000000000000 wwn-0x6000c500d822944f0001000000000000 # # Spare addition (2026-03-24, drive10 sg_format completed): # zpool add dick \ # spare wwn-0x6000c500c9bcfa0f0000000000000000 wwn-0x6000c500c9bcfa0f0001000000000000 # # === NVMe acceleration (2 × Intel DC P4600 750GB, Optane) === # # nvme0: PHKE0163008K750BGN (SSDPE21K750GAC) — 4Kn native # nvme1: PHKE8136008M750BGN (SSDPE21K750GA) — 4Kn (nvme format --lbaf=3) # # Partition layout (both drives identical, 4096-byte sectors): # sfdisk /dev/nvmeXn1 <. # # # ye-lw21 (local+LDAP account, local UID 1002/GID 100 on skydick) # zfs create -o mountpoint=/srv/users/ye-lw21 -o quota=10T dick/users/ye-lw21 # zfs create -o recordsize=128K -o mountpoint=/srv/users/ye-lw21/files dick/users/ye-lw21/files # zfs create -o recordsize=16K -o mountpoint=/srv/users/ye-lw21/bt-state dick/users/ye-lw21/bt-state # zfs create -o recordsize=64K -o mountpoint=/srv/users/ye-lw21/vm dick/users/ye-lw21/vm # mkdir -p /srv/users/ye-lw21/vm/files # chown ye-lw21:users /srv/users/ye-lw21 && chmod 0700 /srv/users/ye-lw21 # for d in files bt-state vm vm/files; do chown ye-lw21:users /srv/users/ye-lw21/$d && chmod 0750 /srv/users/ye-lw21/$d; done # # File-backed VM images live under /srv/users/ye-lw21/vm/files. # # Block LUNs are zvol children of dick/users/ye-lw21/vm/. # # # zhuyz24 (local+LDAP account, local UID 2200000020/GID 100 on skydick; # # LDAP gidNumber is 10000 but local NSS wins and pins GID to 100) # zfs create -o mountpoint=/srv/users/zhuyz24 -o quota=10T dick/users/zhuyz24 # zfs create -o recordsize=128K -o mountpoint=/srv/users/zhuyz24/files dick/users/zhuyz24/files # zfs create -o recordsize=16K -o mountpoint=/srv/users/zhuyz24/bt-state dick/users/zhuyz24/bt-state # zfs create -o recordsize=64K -o mountpoint=/srv/users/zhuyz24/vm dick/users/zhuyz24/vm # mkdir -p /srv/users/zhuyz24/vm/files # chown zhuyz24:users /srv/users/zhuyz24 && chmod 0700 /srv/users/zhuyz24 # for d in files bt-state vm vm/files; do chown zhuyz24:users /srv/users/zhuyz24/$d && chmod 0750 /srv/users/zhuyz24/$d; done # # File-backed VM images live under /srv/users/zhuyz24/vm/files. # # Block LUNs are zvol children of dick/users/zhuyz24/vm/. # # System: # zfs create -o mountpoint=none -o canmount=off dick/system # zfs create -o recordsize=1M -o compression=zstd-3 -o mountpoint=/srv/system/backup dick/system/backup # zfs create -o recordsize=128K -o mountpoint=/srv/system/influxdb -o quota=500G dick/system/influxdb # chown influxdb:influxdb /srv/system/influxdb # zfs create -o recordsize=64K -o mountpoint=/srv/system/vm dick/system/vm # mkdir -p /srv/system/vm/files # zfs create -o recordsize=64K -o readonly=on -o mountpoint=/srv/templates/vm dick/templates/vm # chown root:root /srv/system/{backup,vm} /srv/templates/vm && chmod 0700 /srv/system/{backup,vm} # chown root:root /srv/system/vm/files && chmod 0700 /srv/system/vm/files # # File-backed VM images live under /srv/system/vm/files. # # Block LUNs are zvol children of dick/system/vm/. # # iSCSI zvols (block service — never the same bytes as SMB/NFS): # zfs create -V -o volblocksize=16K dick/users//vm/ # zfs create -V -o volblocksize=16K dick/system/vm/ # # === Expanding the pool === # # Add another pair of Mach2 drives (driveN + driveM): # 1. sg_format both drives to 4Kn if not already done: # sg_format --format --size=4096 /dev/sdX # Wait for completion (monitor with: sg_requests -p /dev/sdX) # 2. Wipe any leftover partition tables: # wipefs -a /dev/sdX # 3. Add as mirror pair using stable WWN paths: # zpool add dick \ # mirror \ # mirror # # === Service model === # # File services (SMB + NFS share the same filesystem datasets): # Public: root:storage 2775, NFS root_squash, Samba [public] @storage # Media: qbittorrent:storage, NFS rw /srv/media all_squash(900), # NFS reader /srv/media/library ro, Samba [media] ro @storage # Home: : 0700, explicit per-user NFS exports, Samba [homes] # BT-state: : 0750, NFS all_squash(uid), no Samba # VM files: : 0750, NFS all_squash(uid), no Samba # TM: : 0750, Samba [timemachine] fruit:time machine, no NFS # # Block services (iSCSI — separate zvols, never shared with SMB/NFS): # dick/users//vm/ — user-owned zvols # dick/system/vm/ — centrally managed zvols # # Quotas: # ZFS quota on dick/users/ caps total across all child datasets. # dick/media is shared — no per-user quota; manage via service-level controls. # # Access time: # dick should carry atime=off so child datasets inherit it by default. This # avoids metadata writes on reads across SMB/NFS-serving datasets. # # Auth: # NFS all_squash provides UID mapping, not authentication. Per-user NFS here # uses one explicit export per user, which is acceptable for a small fixed set # but scales linearly with user count and client ACL maintenance. # Samba [homes] valid users = %S gives real per-user auth via LDAP-backed # Samba accounts (ldapsam). The intended model is LDAP-only SMB users; if a # same-name local account exists on skydick, local NSS still wins for the # final Unix authorization step. For stronger NFS isolation: use sec=krb5 or # tighter per-client IP restrictions. { config, lib, pkgs, ... }: let storageMounts = [ { dataset = "dick/public"; mountPoint = "/srv/public"; } { dataset = "dick/public/datasets"; mountPoint = "/srv/public/datasets"; } { dataset = "dick/public/models"; mountPoint = "/srv/public/models"; } { dataset = "dick/public/sdk"; mountPoint = "/srv/public/sdk"; } { dataset = "dick/media"; mountPoint = "/srv/media"; } { dataset = "dick/system/backup"; mountPoint = "/srv/system/backup"; } { dataset = "dick/system/influxdb"; mountPoint = "/srv/system/influxdb"; } { dataset = "dick/system/vm"; mountPoint = "/srv/system/vm"; } { dataset = "dick/templates/vm"; mountPoint = "/srv/templates/vm"; } { dataset = "dick/users/ldx"; mountPoint = "/srv/users/ldx"; } { dataset = "dick/users/ldx/files"; mountPoint = "/srv/users/ldx/files"; } { dataset = "dick/users/ldx/bt-state"; mountPoint = "/srv/users/ldx/bt-state"; } { dataset = "dick/users/ldx/vm"; mountPoint = "/srv/users/ldx/vm"; } { dataset = "dick/users/ldx/timemachine"; mountPoint = "/srv/users/ldx/timemachine"; } { dataset = "dick/users/ye-lw21"; mountPoint = "/srv/users/ye-lw21"; } { dataset = "dick/users/ye-lw21/files"; mountPoint = "/srv/users/ye-lw21/files"; } { dataset = "dick/users/ye-lw21/bt-state"; mountPoint = "/srv/users/ye-lw21/bt-state"; } { dataset = "dick/users/ye-lw21/vm"; mountPoint = "/srv/users/ye-lw21/vm"; } { dataset = "dick/users/zhuyz24"; mountPoint = "/srv/users/zhuyz24"; } { dataset = "dick/users/zhuyz24/files"; mountPoint = "/srv/users/zhuyz24/files"; } { dataset = "dick/users/zhuyz24/bt-state"; mountPoint = "/srv/users/zhuyz24/bt-state"; } { dataset = "dick/users/zhuyz24/vm"; mountPoint = "/srv/users/zhuyz24/vm"; } ]; storageMountPoints = map (mount: mount.mountPoint) storageMounts; storageServices = [ "nfs-server.service" "samba-smbd.service" "influxdb2.service" "iscsi-target.service" ]; iscsiConfigGuard = pkgs.writeShellApplication { name = "skydick-iscsi-config-guard"; runtimeInputs = [ pkgs.coreutils pkgs.jq ]; text = '' set -euo pipefail check_devices=true if [[ "''${1:-}" == "--no-devices" ]]; then check_devices=false shift fi config="''${1:-/etc/target/saveconfig.json}" fail() { echo "iSCSI configuration guard: $*" >&2 exit 1 } [[ -f "$config" ]] || fail "$config is missing or is not a regular file" [[ -s "$config" ]] || fail "$config is empty" [[ "$(stat -c '%u:%g' "$config")" == "0:0" ]] || fail "$config must be owned by root:root" [[ "$(stat -c '%a' "$config")" == "600" ]] || fail "$config must have mode 0600" jq -e ' type == "object" and (.storage_objects | type == "array" and length > 0) and all(.storage_objects[]; .plugin == "block" and (.dev | type == "string" and length > 0) ) and (.targets | type == "array" and length > 0) and all(.targets[]; (.tpgs | type == "array" and length > 0) and all(.tpgs[]; .attributes.authentication == 1 and .attributes.generate_node_acls == 0 and .attributes.cache_dynamic_acls == 0 and (.portals | type == "array" and length > 0) and all(.portals[]; .port == 3260 and .ip_address != "0.0.0.0" and .ip_address != "::0" ) and (.luns | type == "array" and length > 0) and (.node_acls | type == "array" and length > 0) and all(.node_acls[]; (.node_wwn | type == "string" and length > 0) and (.chap_userid | type == "string" and length > 0) and (.chap_password | type == "string" and length >= 12) and (.mapped_luns | type == "array" and length > 0) ) ) ) ' "$config" >/dev/null || fail "$config lacks a secured block target, portal, ACL, CHAP credential, or LUN mapping" if $check_devices; then device_count=0 while IFS= read -r device; do [[ -b "$device" ]] || fail "configured backstore $device is not a block device" ((device_count += 1)) done < <(jq -r '.. | objects | select(.plugin? == "block") | .dev? // empty' "$config") ((device_count > 0)) || fail "$config has no usable block backstore" fi ''; }; in { # Build sambaFull with Spotlight/tracker support. # Fixes for tinysparql 3.x (tracker-sparql-3.0) compatibility: # (1) waf only checks tracker-sparql-{2.0..0.14} — add 3.0 # (2) tracker 2.x API (get_async/get_finish) removed in 3.x — replace # with bus_new_async/bus_new_finish connecting to localsearch miner # (3) disable tevent_glib_tracker test (uses same removed API, test-only) # (4) add icu for Unicode normalisation required by Spotlight # # SMB Direct (SMB3-over-RDMA) is NOT enabled: upstream Samba's userspace # smbd does not implement an RDMA transport, even with --with-smb-direct # passed (waf silently accepts the flag but the produced smbd contains # no ibverbs code). The kernel server `ksmbd` does support SMB Direct, # but switching would lose LDAP-backed posix users, Spotlight, fruit # metadata and Time Machine — features that all live in userspace Samba. # See: NFS-over-RDMA is on the same fabric and covers the bulk-throughput # use case; SMB stays on TCP for now. nixpkgs.overlays = [ (final: prev: { sambaFull = prev.sambaFull.overrideAttrs (old: { buildInputs = old.buildInputs ++ [ final.tinysparql.dev final.glib.dev final.icu.dev ]; postPatch = (old.postPatch or "") + '' substituteInPlace source3/wscript \ --replace-fail "tracker_versions = ['2.0', '1.0', '0.16', '0.14']" \ "tracker_versions = ['3.0', '2.0', '1.0', '0.16', '0.14']" # Patch rpcd_mdssvc to use tinysparql 3.x bus API substituteInPlace source3/rpc_server/mdssvc/mdssvc_tracker.c \ --replace-fail \ 'tracker_sparql_connection_get_finish(res, &error)' \ 'tracker_sparql_connection_bus_new_finish(res, &error)' \ --replace-fail \ 'tracker_sparql_connection_get_async(ctx->gcancellable,' \ 'tracker_sparql_connection_bus_new_async("org.freedesktop.Tracker3.Miner.Files", NULL, NULL, ctx->gcancellable,' # Disable test binary that also uses removed API (not installed) substituteInPlace source3/wscript_build \ --replace-fail "enabled=bld.CONFIG_SET('HAVE_TRACKER') and bld.CONFIG_SET('WITH_TEVENT_GLIB_GLUE')" \ "enabled=False" ''; }); }) ]; # Keep a fixed local GID for on-disk ownership and the qbittorrent service # account. Shared-user membership policy is carried by the matching LDAP # posixGroup cn=storage,ou=posix_groups,dc=skyw,dc=top. users.groups.storage = { gid = 997; }; # Service account for the shared media writer (qbittorrent + *arr stack) users.users.qbittorrent = { uid = 900; group = "storage"; isSystemUser = true; shell = "/run/current-system/sw/bin/nologin"; }; systemd.tmpfiles.rules = [ "d /srv 0755 root root -" # Shared "d /srv/public 2775 root storage -" "d /srv/media 2775 root storage -" "d /srv/media/data 2775 qbittorrent storage -" "d /srv/media/library 2775 qbittorrent storage -" # Per-user trees "d /srv/users 0755 root root -" "d /srv/users/ldx 0700 ldx users -" "d /srv/users/ldx/files 0750 ldx users -" "d /srv/users/ldx/bt-state 0750 ldx users -" "d /srv/users/ldx/vm 0750 ldx users -" "d /srv/users/ldx/vm/files 0750 ldx users -" "d /srv/users/ldx/timemachine 0750 ldx users -" "d /srv/users/ldx/.cache 0750 ldx users -" "d /srv/users/ldx/.local 0750 ldx users -" "d /srv/users/ldx/.local/share 0750 ldx users -" "f /var/lib/systemd/linger/ldx" "d /srv/users/ye-lw21 0700 ye-lw21 users -" "d /srv/users/ye-lw21/files 0750 ye-lw21 users -" "d /srv/users/ye-lw21/bt-state 0750 ye-lw21 users -" "d /srv/users/ye-lw21/vm 0750 ye-lw21 users -" "d /srv/users/ye-lw21/vm/files 0750 ye-lw21 users -" "d /srv/users/zhuyz24 0700 zhuyz24 users -" "d /srv/users/zhuyz24/files 0750 zhuyz24 users -" "d /srv/users/zhuyz24/bt-state 0750 zhuyz24 users -" "d /srv/users/zhuyz24/vm 0750 zhuyz24 users -" "d /srv/users/zhuyz24/vm/files 0750 zhuyz24 users -" # System "d /srv/system 0700 root root -" "d /srv/system/backup 0700 root root -" "d /srv/system/vm 0700 root root -" "d /srv/system/vm/files 0700 root root -" "d /srv/templates 0755 root root -" "d /srv/templates/vm 0755 root root -" ]; # Fail closed if ZFS did not mount the real datasets. tmpfiles deliberately # creates these paths, so path existence alone cannot distinguish a dataset # from an empty placeholder on the root filesystem. systemd.services.dick-storage-ready = { description = "Verify skydick datapool datasets are mounted"; wants = [ "zfs-mount.service" ]; after = [ "zfs-mount.service" ]; before = storageServices; requiredBy = storageServices; unitConfig.RequiresMountsFor = storageMountPoints; serviceConfig = { Type = "oneshot"; RemainAfterExit = true; }; script = '' set -euo pipefail fail() { echo "Datapool readiness check: $*" >&2 exit 1 } ${pkgs.zfs}/bin/zpool list -H -o name dick >/dev/null 2>&1 || fail "pool dick is not imported" check_mount() { local dataset="$1" local mountpoint="$2" local actual actual="$(${pkgs.util-linux}/bin/findmnt --noheadings --raw --mountpoint "$mountpoint" --output SOURCE,FSTYPE 2>/dev/null || true)" [[ "$actual" == "$dataset zfs" ]] || fail "$mountpoint is '$actual', expected '$dataset zfs'" [[ "$(${pkgs.zfs}/bin/zfs get -H -o value mounted "$dataset")" == "yes" ]] || fail "$dataset is not mounted" } ${lib.concatMapStringsSep "\n" (mount: '' check_mount ${lib.escapeShellArg mount.dataset} ${lib.escapeShellArg mount.mountPoint} '') storageMounts} ''; }; # Keep the manually created dick pool aligned with the documented policy. systemd.services.dick-zfs-properties = { description = "Apply ZFS properties for the dick datapool"; requires = [ "dick-storage-ready.service" ]; after = [ "dick-storage-ready.service" ]; before = storageServices; requiredBy = storageServices; wantedBy = [ "multi-user.target" ]; serviceConfig = { Type = "oneshot"; RemainAfterExit = true; }; script = '' set -euo pipefail # Child datasets on the datapool currently inherit the pool default. # Setting the root keeps read-heavy SMB/NFS trees from generating atime # metadata writes without having to stamp every descendant explicitly. ${pkgs.zfs}/bin/zfs set atime=off dick # Route metadata + small blocks (≤128K) to the mirrored NVMe special # vdev. Subtitles, NFO, thumbnails, directory entries, and small config # files all land on Optane instead of spinning rust. ${pkgs.zfs}/bin/zfs set special_small_blocks=128K dick ''; }; systemd.services.nfs-server = { requires = [ "dick-storage-ready.service" "dick-zfs-properties.service" ]; after = [ "dick-storage-ready.service" "dick-zfs-properties.service" ]; unitConfig.RequiresMountsFor = storageMountPoints; }; systemd.services.samba-smbd = { requires = [ "dick-storage-ready.service" "dick-zfs-properties.service" ]; after = [ "dick-storage-ready.service" "dick-zfs-properties.service" ]; unitConfig.RequiresMountsFor = storageMountPoints; restartTriggers = [ config.age.secrets.skydick-samba-ldap-admin.file ]; }; systemd.services.samba-winbindd.restartTriggers = [ config.age.secrets.skydick-samba-ldap-admin.file ]; systemd.services.influxdb2 = { requires = [ "dick-storage-ready.service" "dick-zfs-properties.service" ]; after = [ "dick-storage-ready.service" "dick-zfs-properties.service" ]; unitConfig.RequiresMountsFor = [ "/srv/system/influxdb" ]; }; systemd.services.samba-ldap-admin-password = { description = "Seed Samba LDAP admin password into secrets.tdb"; wants = [ "network-online.target" ]; after = [ "network-online.target" ]; before = [ "samba-winbindd.service" "samba-smbd.service" ]; requiredBy = [ "samba-winbindd.service" "samba-smbd.service" ]; restartTriggers = [ config.environment.etc."samba/smb.conf".source config.age.secrets.skydick-samba-ldap-admin.file ]; serviceConfig = { Type = "oneshot"; RemainAfterExit = true; ConditionPathExists = config.age.secrets.skydick-samba-ldap-admin.path; }; script = '' set -euo pipefail password="$(${pkgs.coreutils}/bin/cat ${config.age.secrets.skydick-samba-ldap-admin.path})" ${pkgs.coreutils}/bin/printf '%s\n%s\n' "$password" "$password" | ${config.services.samba.package}/bin/smbpasswd -s -W ''; }; # NFS — primary protocol for all datasets services.rpcbind.enable = true; # NFS-over-RDMA listener (additive to TCP — clients pick one transport # per mount). skydick's ConnectX-3 Pro / mlx4_0 (bond40g, enp130s0) RoCEv2 fabric is the # transport; nfsd binds to UDP/TCP 2049 by default plus RDMA 20049 # when this oneshot writes to /proc/fs/nfsd/portlist. Clients use # the RDMA path with `proto=rdma,port=20049` mount options. Saves a # TCP/IP stack traversal + uses native NIC parallelism (no nconnect # equivalent needed — RDMA QPs are intrinsically parallel). boot.kernelModules = [ "rpcrdma" ]; # RoCEv2 QoS — DSCP-marking half (decision "C", 2026-06-25). Stamp ToS 96 = # DSCP 24 on RDMA-CM connections so the switch + the mlx5 RDMA clients classify # skydick's RoCE egress into priority 3 (the PFC/lossless queue). Only affects # NEW rdma_cm connections, and must be set before nfsd accepts client QPs. # MARKING ONLY: end-to-end lossless also needs PFC on prio 3 at every hop; # skydick's ConnectX-3 (mlx4) does PCP-based PFC with no upstream trust-dscp, so # the PFC half is deferred (move RoCE to the idle ConnectX-4-LX/mlx5 for a clean # DSCP/PFC fabric, or mlx4 pfctx/pfcrx — open decision A/B). systemd.services.roce-rdma-cm-tos = { description = "RoCEv2 RDMA-CM default ToS = 96 (DSCP 24) for prio-3 QoS"; requires = [ "sys-kernel-config.mount" ]; after = [ "sys-kernel-config.mount" "systemd-modules-load.service" ]; before = [ "nfsd-rdma-listener.service" "nfs-server.service" ]; wantedBy = [ "multi-user.target" ]; path = [ pkgs.coreutils ]; serviceConfig = { Type = "oneshot"; RemainAfterExit = true; ExecStartPre = "${pkgs.kmod}/bin/modprobe rdma_cm"; }; # Best-effort over all present RDMA devices (mlx4_0 now; covers mlx5 if option # B ever brings it up). Always exits 0 so a transient device state can't trip # deploy-rs magic-rollback. script = '' for d in /sys/class/infiniband/*; do [ -e "$d" ] || continue dev=$(basename "$d") mkdir -p /sys/kernel/config/rdma_cm/"$dev" 2>/dev/null || continue for tos in /sys/kernel/config/rdma_cm/"$dev"/ports/*/default_roce_tos; do [ -e "$tos" ] || continue echo 96 > "$tos" 2>/dev/null && echo "set $tos = 96" done done exit 0 ''; }; systemd.services.nfsd-rdma-listener = { description = "Enable NFS-over-RDMA listener on port 20049"; # `after` orders the unit at boot. # `wants` brings it up when the target activates. # `partOf` (NEW) propagates restart: whenever nfs-server.service # cycles, systemd re-runs THIS unit's ExecStart. Without partOf # the listener stays in "exited (success)" state, but nfsd's # kernel-side portlist gets wiped on every nfs-server stop, so # the RDMA port goes dead silently. Caught 2026-05-26 after a # `nixos-rebuild switch` left port 20049 down for ~14h — every # NFS-RDMA client (door-pek) hung on every mount-touching read. after = [ "nfs-server.service" ]; wants = [ "nfs-server.service" ]; partOf = [ "nfs-server.service" ]; wantedBy = [ "multi-user.target" "nfs-server.service" ]; serviceConfig = { Type = "oneshot"; RemainAfterExit = true; }; # If the listener is already added (re-run after rebuild), nfsd # returns EINVAL — that's fine, the listener stays up. script = '' ${pkgs.gnugrep}/bin/grep -q "^rdma 20049$" /proc/fs/nfsd/portlist && { echo "rdma 20049 already listening" exit 0 } echo "rdma 20049" > /proc/fs/nfsd/portlist echo "added RDMA listener on 20049" ''; }; services.nfs.server = { enable = true; nproc = 64; statdPort = 20001; lockdPort = 20002; mountdPort = 20003; exports = '' # Traversal-only NFSv4 pseudo-root. Do not use crossmnt here: it would # implicitly publish every present and future ZFS child below /srv. /srv 10.0.0.0/16(ro,sync,fsid=0,no_subtree_check,root_squash) fd99:23eb:1682::75:15(ro,sync,fsid=0,no_subtree_check,root_squash) 2a0c:b641:69c:ada0::/64(ro,sync,fsid=0,no_subtree_check,root_squash) # Shared /srv/public 10.0.0.0/16(rw,sync,no_subtree_check,root_squash) /srv/public/datasets 10.0.0.0/16(rw,sync,nohide,no_subtree_check,root_squash) /srv/public/models 10.0.0.0/16(rw,sync,nohide,no_subtree_check,root_squash) /srv/public/sdk 10.0.0.0/16(rw,sync,nohide,no_subtree_check,root_squash) # door-pek is the sole qBittorrent/*arr writer observed on this export. /srv/media 10.0.75.15(rw,async,no_subtree_check,all_squash,anonuid=900,anongid=997) fd99:23eb:1682::75:15(rw,async,no_subtree_check,all_squash,anonuid=900,anongid=997) /srv/media/library 10.0.0.0/16(ro,sync,no_subtree_check,root_squash) # Per-user exports. Each mounted child is explicit, so an unrelated new # dataset cannot inherit owner-mapped access through crossmnt. /srv/users/ldx 10.0.75.15(rw,sync,no_subtree_check,all_squash,anonuid=1000,anongid=100) fd99:23eb:1682::75:15(rw,sync,no_subtree_check,all_squash,anonuid=1000,anongid=100) /srv/users/ldx/files 10.0.75.15(rw,sync,nohide,no_subtree_check,all_squash,anonuid=1000,anongid=100) fd99:23eb:1682::75:15(rw,sync,nohide,no_subtree_check,all_squash,anonuid=1000,anongid=100) /srv/users/ldx/bt-state 10.0.75.15(rw,sync,nohide,no_subtree_check,all_squash,anonuid=1000,anongid=100) fd99:23eb:1682::75:15(rw,sync,nohide,no_subtree_check,all_squash,anonuid=1000,anongid=100) /srv/users/ldx/vm 10.0.75.15(rw,sync,nohide,no_subtree_check,all_squash,anonuid=1000,anongid=100) fd99:23eb:1682::75:15(rw,sync,nohide,no_subtree_check,all_squash,anonuid=1000,anongid=100) /srv/users/ye-lw21 10.0.0.0/16(rw,sync,no_subtree_check,all_squash,anonuid=1002,anongid=100) /srv/users/ye-lw21/files 10.0.0.0/16(rw,sync,nohide,no_subtree_check,all_squash,anonuid=1002,anongid=100) /srv/users/ye-lw21/bt-state 10.0.0.0/16(rw,sync,nohide,no_subtree_check,all_squash,anonuid=1002,anongid=100) /srv/users/ye-lw21/vm 10.0.0.0/16(rw,sync,nohide,no_subtree_check,all_squash,anonuid=1002,anongid=100) # node1 (.10) and node2 (.20) currently hold open files in this tree, but # that snapshot does not prove they are the user's only valid clients. /srv/users/zhuyz24 10.0.0.0/16(rw,sync,no_subtree_check,all_squash,anonuid=2200000020,anongid=100) /srv/users/zhuyz24/files 10.0.0.0/16(rw,sync,nohide,no_subtree_check,all_squash,anonuid=2200000020,anongid=100) /srv/users/zhuyz24/bt-state 10.0.0.0/16(rw,sync,nohide,no_subtree_check,all_squash,anonuid=2200000020,anongid=100) /srv/users/zhuyz24/vm 10.0.0.0/16(rw,sync,nohide,no_subtree_check,all_squash,anonuid=2200000020,anongid=100) # Exact global-v6 client addresses are not yet inventoried; retain this # existing ACL until they can replace the /64 without breaking clients. /srv/users/zhuyz24 2a0c:b641:69c:ada0::/64(rw,sync,no_subtree_check,all_squash,anonuid=2200000020,anongid=100) /srv/users/zhuyz24/files 2a0c:b641:69c:ada0::/64(rw,sync,nohide,no_subtree_check,all_squash,anonuid=2200000020,anongid=100) /srv/users/zhuyz24/bt-state 2a0c:b641:69c:ada0::/64(rw,sync,nohide,no_subtree_check,all_squash,anonuid=2200000020,anongid=100) /srv/users/zhuyz24/vm 2a0c:b641:69c:ada0::/64(rw,sync,nohide,no_subtree_check,all_squash,anonuid=2200000020,anongid=100) # System /srv/system/backup 10.0.0.1(rw,sync,no_subtree_check,no_root_squash) /srv/system/vm 10.0.0.0/16(rw,sync,no_subtree_check,no_root_squash) /srv/templates/vm 10.0.0.0/16(ro,sync,no_subtree_check,root_squash) ''; }; services.nfs.idmapd.settings = { General.Domain = "skydick.local"; Mapping = { Nobody-User = "nobody"; Nobody-Group = "nogroup"; }; }; # Samba — file-level access (SMB + NFS share the same datasets) services.samba = { enable = true; package = pkgs.sambaFull; openFirewall = false; nmbd.enable = false; settings = { global = { workgroup = "WORKGROUP"; "server string" = "Skydick Storage"; "netbios name" = "SKYDICK"; security = "user"; "passdb backend" = "ldapsam:ldap://ldap.skyw.top"; "ldap admin dn" = "cn=admin,dc=skyw,dc=top"; "ldap suffix" = "dc=skyw,dc=top"; "ldap user suffix" = "ou=people"; "ldap group suffix" = "ou=posix_groups"; "ldap machine suffix" = "ou=machines"; "ldap delete dn" = "no"; "ldap passwd sync" = "only"; "ldap ssl" = "start tls"; "ldap server require strong auth" = "yes"; "ldap connection timeout" = "5"; "ldap timeout" = "5"; "hosts allow" = "10.0. 127. 10.253."; "hosts deny" = "ALL"; # SMB Multichannel: just enable the feature. Earlier attempt to # advertise RSS+speed via the `interfaces` directive (commit # 4f21721) used the format `lo;capability=DYNAMIC,speed=1 ...` # but Samba's parser split on semicolons too aggressively and # tried to resolve `capability=DYNAMIC` as a hostname (logged # "interpret_string_addr_internal: getaddrinfo failed for name # capability=DYNAMIC"), which corrupted the interfaces table # and broke auth (rpcd_classic crash-looped on disconnected # endpoints, smbd rejected all client auth as `Authentication # error`). Reverted 2026-05-16. Clients still get multichannel # support via `server multi channel support`; they just open # one channel by default without the RSS hint. Re-enable # capability advertising only with a verified working syntax. "server multi channel support" = "yes"; "map to guest" = "never"; "server min protocol" = "SMB2_10"; "load printers" = "no"; "vfs objects" = "catia fruit streams_xattr recycle"; "fruit:metadata" = "stream"; "fruit:nfs_aces" = "no"; # Stop macOS Finder turds accumulating on the rw shares (public, homes): # veto .DS_Store + the AppleDouble/dot-junk family at the SMB layer so # they are never written or shown. fruit:metadata=stream keeps real Apple # metadata in xattr streams (not ._ files), so vetoing ._* is safe. # `delete veto files` lets a dir still be removed if it only holds these. "veto files" = "/._*/.DS_Store/.AppleDouble/.Trashes/.TemporaryItems/.fseventsd/.apdisk/"; "delete veto files" = "yes"; "recycle:repository" = ".recycle/%U"; "recycle:keeptree" = "yes"; "recycle:versions" = "yes"; "recycle:touch" = "yes"; "recycle:maxsize" = "0"; "spotlight" = "yes"; "spotlight backend" = "tracker"; }; # Shared datasets public = { path = "/srv/public"; browseable = "yes"; "read only" = "no"; "guest ok" = "no"; "valid users" = "@storage"; "create mask" = "0664"; "directory mask" = "2775"; }; media = { path = "/srv/media/library"; browseable = "yes"; "read only" = "yes"; "valid users" = "@storage"; "vfs objects" = "catia fruit streams_xattr"; }; # Raw torrent payload — read-only, browseable so every @storage user can # SEE the per-tracker download tree (BYRBT, M-Team, TJUPT, IPTorrents, # qbit, tv, movies, music, …) and re-seed the same bytes without # re-downloading. Seeding needs only READ access, so this share never # grants write: door-pek's qBittorrent (uid 900), writing /srv/media over # the rw,all_squash NFS export, stays the SOLE writer. Pointed at # /srv/media/data directly (not /srv/media whole) to exclude the backups/, # baidu/, immich/ and *.zfs.zst clutter in the /srv/media root. On-disk # perms already permit @storage reads (payload 0640, dirs 0750/2775, all # group-readable/traversable by storage gid 997). `recycle` is omitted — # meaningless on a read-only share (mirrors the [media] block). torrents = { path = "/srv/media/data"; browseable = "yes"; "read only" = "yes"; "valid users" = "@storage"; "vfs objects" = "catia fruit streams_xattr"; }; # Per-user homes — Samba auto-creates \\SKYDICK\ from this template homes = { path = "/srv/users/%S/files"; browseable = "no"; "read only" = "no"; "valid users" = "%S"; "create mask" = "0640"; "directory mask" = "0750"; }; # Per-user Time Machine — ldx's Macs back up here. # Separate share so fruit:time machine writes sparsebundles into the # dedicated ZFS dataset (dick/users/ldx/timemachine, recordsize=64K, # compression=zstd) instead of polluting the files dataset. "ldx-timemachine" = { path = "/srv/users/ldx/timemachine"; browseable = "yes"; "read only" = "no"; "valid users" = "ldx"; "vfs objects" = "catia fruit streams_xattr"; "fruit:time machine" = "yes"; "fruit:metadata" = "stream"; "fruit:model" = "TimeCapsule"; "spotlight" = "no"; }; }; }; # Localsearch filesystem miner for Samba Spotlight search. # Runs as ldx on ldx's session bus so smbd workers (setuid to the # connecting user) can reach the miner via /run/user/1000/bus. # Lingering for ldx is enabled in tmpfiles above. environment.systemPackages = with pkgs; [ tinysparql localsearch ]; systemd.services.localsearch-miner = { description = "Localsearch filesystem miner for Samba Spotlight"; requires = [ "user@1000.service" ]; after = [ "user@1000.service" "samba-smbd.service" "local-fs.target" ]; wants = [ "samba-smbd.service" ]; wantedBy = [ "multi-user.target" ]; environment = { DBUS_SESSION_BUS_ADDRESS = "unix:path=/run/user/1000/bus"; XDG_DATA_DIRS = builtins.concatStringsSep ":" [ "${pkgs.localsearch}/share/gsettings-schemas/${pkgs.localsearch.name}" "${pkgs.tinysparql}/share/gsettings-schemas/${pkgs.tinysparql.name}" "/run/current-system/sw/share" ]; XDG_CACHE_HOME = "/srv/users/ldx/.cache"; XDG_DATA_HOME = "/srv/users/ldx/.local/share"; }; serviceConfig = { User = "ldx"; Group = "users"; ExecStart = "${pkgs.localsearch}/libexec/localsearch-3"; Restart = "on-failure"; RestartSec = 10; }; }; systemd.services.localsearch-paths = { description = "Register Spotlight index paths with localsearch"; requires = [ "localsearch-miner.service" ]; after = [ "localsearch-miner.service" ]; wantedBy = [ "multi-user.target" ]; environment = { DBUS_SESSION_BUS_ADDRESS = "unix:path=/run/user/1000/bus"; XDG_DATA_DIRS = builtins.concatStringsSep ":" [ "${pkgs.localsearch}/share/gsettings-schemas/${pkgs.localsearch.name}" "${pkgs.tinysparql}/share/gsettings-schemas/${pkgs.tinysparql.name}" "/run/current-system/sw/share" ]; }; serviceConfig = { Type = "oneshot"; RemainAfterExit = true; User = "ldx"; Group = "users"; }; script = '' ${pkgs.localsearch}/bin/localsearch index --add /srv/public ${pkgs.localsearch}/bin/localsearch index --add /srv/media/library ${pkgs.localsearch}/bin/localsearch index --add /srv/users/ldx/files ''; }; services.samba-wsdd = { # Discovery was firewall-blocked and had no usable network surface. enable = false; openFirewall = false; }; # iSCSI state is intentionally imperative because targetcli can contain CHAP # material. Keep the entire directory outside the Nix store: targetcli uses # an atomic rename, so a directory symlink is required instead of a file # symlink. The activation dependency migrates the current /etc/target before # NixOS replaces the formerly generated saveconfig.json. environment.etc."target/saveconfig.json".enable = lib.mkForce false; environment.etc.target = { source = "/var/lib/target"; mode = "direct-symlink"; }; system.activationScripts.iscsiTargetState = { deps = [ "users" "groups" "specialfs" ]; text = '' state_dir=/var/lib/target legacy_dir=/etc/target if [[ -L "$legacy_dir" ]]; then if [[ "$(${pkgs.coreutils}/bin/readlink -f "$legacy_dir")" != "$state_dir" ]]; then echo "Refusing to replace unexpected $legacy_dir symlink" >&2 exit 1 fi elif [[ -d "$legacy_dir" ]]; then if [[ -e "$state_dir" ]]; then echo "Both $legacy_dir and $state_dir exist; refusing an ambiguous iSCSI state merge" >&2 exit 1 fi ${iscsiConfigGuard}/bin/skydick-iscsi-config-guard --no-devices "$legacy_dir/saveconfig.json" ${pkgs.coreutils}/bin/install -d -m 0755 /var/lib ${pkgs.coreutils}/bin/mv "$legacy_dir" "$state_dir" ${pkgs.coreutils}/bin/chmod 0700 "$state_dir" elif [[ ! -d "$state_dir" ]]; then echo "No persistent iSCSI configuration exists at $legacy_dir or $state_dir" >&2 exit 1 fi ${iscsiConfigGuard}/bin/skydick-iscsi-config-guard --no-devices "$state_dir/saveconfig.json" # The previous NixOS generation tracked this copied file in /etc/.clean. # Remove only its exact bookkeeping record before setup-etc follows the # new directory symlink and mistakes the persistent state for an obsolete # generated file. if [[ -f /etc/.clean ]]; then ${pkgs.gnused}/bin/sed -i '\|^target/saveconfig.json$|d' /etc/.clean fi ''; }; system.activationScripts.etc.deps = [ "iscsiTargetState" ]; systemd.services.iscsi-target-config-ready = { description = "Verify persistent LIO target configuration"; requires = [ "dick-storage-ready.service" ]; after = [ "dick-storage-ready.service" ]; before = [ "iscsi-target.service" ]; requiredBy = [ "iscsi-target.service" ]; unitConfig.RequiresMountsFor = [ "/srv/system/vm" ]; serviceConfig = { Type = "oneshot"; RemainAfterExit = true; }; script = '' ${iscsiConfigGuard}/bin/skydick-iscsi-config-guard /etc/target/saveconfig.json ''; }; # iSCSI — vm zvols only. A switch must never clear an active target/session; # changed units take effect at the next controlled restart or reboot. services.target.enable = true; systemd.services.iscsi-target = { requires = [ "dick-storage-ready.service" "dick-zfs-properties.service" "iscsi-target-config-ready.service" ]; after = [ "dick-storage-ready.service" "dick-zfs-properties.service" "iscsi-target-config-ready.service" ]; unitConfig.RequiresMountsFor = [ "/srv/system/vm" ]; restartIfChanged = false; stopIfChanged = false; }; }