diff --git a/modules/monitoring.nix b/modules/monitoring.nix index 7865202..2a7f7b5 100644 --- a/modules/monitoring.nix +++ b/modules/monitoring.nix @@ -60,21 +60,6 @@ fi done ''; - - # Physical pool IOPS/bandwidth straight from the vdev layer. The telegraf - # `inputs.zfs` poolMetrics emits per-objset *logical* counters, which include - # ARC cache hits — so RAM-served reads show up as huge "pool IOPS" with zero - # actual disk activity (this misled a 2026-05-30 mountd investigation). This - # collector reports what `zpool iostat` reports: real disk ops. The second - # block of `iostat 1 2` is the live 1s sample; the operations columns are - # already per-second rates, so these are gauges (no derivative in Grafana). - zpoolIostatScript = pkgs.writeShellScript "zpool-iostat" '' - npools=$(${pkgs.zfs}/bin/zpool list -H -o name | ${pkgs.coreutils}/bin/wc -l) - ${pkgs.zfs}/bin/zpool iostat -Hp 1 2 | ${pkgs.coreutils}/bin/tail -n "$npools" \ - | while read -r name alloc free rops wops rbytes wbytes; do - echo "zpool_io,pool=$name read_ops=''${rops}i,write_ops=''${wops}i,read_bytes=''${rbytes}i,write_bytes=''${wbytes}i" - done - ''; in { options.skyworks.monitoring = { enable = lib.mkEnableOption "Telegraf monitoring to InfluxDB"; @@ -166,14 +151,6 @@ timeout = "60s"; data_format = "influx"; } - { - # Physical pool IOPS (measurement `zpool_io`) — the honest version - # of the ARC-inclusive `zfs` poolMetrics counters. - commands = [ "${zpoolIostatScript}" ]; - interval = "10s"; - timeout = "10s"; - data_format = "influx"; - } ]; }; }; @@ -279,6 +256,31 @@ } > "$TMP_ZFS" chmod 0644 "$TMP_ZFS" mv -f "$TMP_ZFS" "$OUT_DIR/zpool_health.prom" + + # ZFS pool -> block-device mapping. Lets Grafana compute *physical* + # per-pool IOPS by joining node_disk_*_completed_total on `device` + # (node_zfs_zpool_dataset_* is LOGICAL — it counts ARC cache hits, so + # RAM-served reads look like huge phantom pool IOPS). Re-read from live + # `zpool status` each run, so kernel device renames across reboots + # can't break the dashboard. Partition suffix is stripped to match + # node-exporter's whole-disk `device` label (sdb1->sdb, nvme0n1p1->nvme0n1). + TMP_MEM=$(mktemp "$OUT_DIR/zpool_members.prom.XXXXXX") + { + echo "# HELP skyw_zpool_member 1 if the block device is a member of the pool" + echo "# TYPE skyw_zpool_member gauge" + ${pkgs.zfs}/bin/zpool status -LP 2>/dev/null | ${pkgs.gawk}/bin/awk ' + /^[[:space:]]*pool:/ { pool = $2; next } + /\/dev\// { + for (i = 1; i <= NF; i++) if ($i ~ /^\/dev\//) { + dev = $i; sub(/^\/dev\//, "", dev) + if (dev ~ /^nvme/) sub(/p[0-9]+$/, "", dev); else sub(/[0-9]+$/, "", dev) + line = "skyw_zpool_member{pool=\"" pool "\",device=\"" dev "\"} 1" + if (!(line in seen)) { seen[line] = 1; print line } + } + }' + } > "$TMP_MEM" + chmod 0644 "$TMP_MEM" + mv -f "$TMP_MEM" "$OUT_DIR/zpool_members.prom" ''; }; };