diff --git a/hosts/skydick/datapool.nix b/hosts/skydick/datapool.nix index d0cd745..cdc06c9 100644 --- a/hosts/skydick/datapool.nix +++ b/hosts/skydick/datapool.nix @@ -393,7 +393,7 @@ services.rpcbind.enable = true; # NFS-over-RDMA listener (additive to TCP — clients pick one transport - # per mount). skydick's ConnectX-5 / mlx5_bond_0 RoCEv2 fabric is the + # per mount). skydick's ConnectX-3 Pro / mlx4_0 (bond40g, enp130s0) RoCEv2 fabric is the # transport; nfsd binds to UDP/TCP 2049 by default plus RDMA 20049 # when this oneshot writes to /proc/fs/nfsd/portlist. Clients use # the RDMA path with `proto=rdma,port=20049` mount options. Saves a @@ -401,6 +401,43 @@ # equivalent needed — RDMA QPs are intrinsically parallel). boot.kernelModules = [ "rpcrdma" ]; + # RoCEv2 QoS — DSCP-marking half (decision "C", 2026-06-25). Stamp ToS 96 = + # DSCP 24 on RDMA-CM connections so the switch + the mlx5 RDMA clients classify + # skydick's RoCE egress into priority 3 (the PFC/lossless queue). Only affects + # NEW rdma_cm connections, and must be set before nfsd accepts client QPs. + # MARKING ONLY: end-to-end lossless also needs PFC on prio 3 at every hop; + # skydick's ConnectX-3 (mlx4) does PCP-based PFC with no upstream trust-dscp, so + # the PFC half is deferred (move RoCE to the idle ConnectX-4-LX/mlx5 for a clean + # DSCP/PFC fabric, or mlx4 pfctx/pfcrx — open decision A/B). + systemd.services.roce-rdma-cm-tos = { + description = "RoCEv2 RDMA-CM default ToS = 96 (DSCP 24) for prio-3 QoS"; + requires = [ "sys-kernel-config.mount" ]; + after = [ "sys-kernel-config.mount" "systemd-modules-load.service" ]; + before = [ "nfsd-rdma-listener.service" "nfs-server.service" ]; + wantedBy = [ "multi-user.target" ]; + path = [ pkgs.coreutils ]; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + ExecStartPre = "${pkgs.kmod}/bin/modprobe rdma_cm"; + }; + # Best-effort over all present RDMA devices (mlx4_0 now; covers mlx5 if option + # B ever brings it up). Always exits 0 so a transient device state can't trip + # deploy-rs magic-rollback. + script = '' + for d in /sys/class/infiniband/*; do + [ -e "$d" ] || continue + dev=$(basename "$d") + mkdir -p /sys/kernel/config/rdma_cm/"$dev" 2>/dev/null || continue + for tos in /sys/kernel/config/rdma_cm/"$dev"/ports/*/default_roce_tos; do + [ -e "$tos" ] || continue + echo 96 > "$tos" 2>/dev/null && echo "set $tos = 96" + done + done + exit 0 + ''; + }; + systemd.services.nfsd-rdma-listener = { description = "Enable NFS-over-RDMA listener on port 20049"; # `after` orders the unit at boot.