{ lib, pkgs, config, ... }: { services.grafana.provision.alerting.rules.settings.groups = [ { name = "InodeUsageAlerts"; folder = "System Alerts"; interval = "1m"; rules = [ { uid = "high-inode-usage-alert-uid"; title = "HighInodeUsage"; condition = "D"; # Condition is now D data = [ # Query A: Calculate inode usage percentage { refId = "A"; datasourceUid = "vm-datasource-uid"; queryType = "prometheus"; relativeTimeRange = { from = 60; to = 0; }; model = { expr = '' ( node_filesystem_files{fstype!~"tmpfs|rootfs",mountpoint!=""} - node_filesystem_files_free{fstype!~"tmpfs|rootfs",mountpoint!=""} ) / (node_filesystem_files{fstype!~"tmpfs|rootfs",mountpoint!=""} > 0) * 100 and node_filesystem_files{fstype!~"tmpfs|rootfs",mountpoint!=""} and node_filesystem_files_free{fstype!~"tmpfs|rootfs",mountpoint!=""} ''; legendFormat = "{{mountpoint}} on {{instance}}"; instant = false; }; } # Expression C: Reduce Query A to its last value, preserving labels { refId = "C"; datasourceUid = "__expr__"; model = { type = "reduce"; expression = "A"; # Input is Query A reducer = "last"; # Get the last value of each series in A }; } # Expression D: Apply math condition to the reduced values from C { refId = "D"; datasourceUid = "__expr__"; model = { type = "math"; expression = "$C > 80"; # Alert if inode usage from C is > 80% }; } ]; for = "30m"; # Duration the condition must be met noDataState = "NoData"; execErrState = "Error"; annotations = { summary = "High inode usage on {{ $labels.instance }} at {{ $labels.mountpoint }}"; description = ''Inode usage on {{ $labels.instance }} for mount point {{ $labels.mountpoint }} (fstype: {{ $labels.fstype }}) has been above 80% for more than 30 minutes. Current value: {{ if $values.C }}{{ $values.C | humanizePercentage }}{{ else }}N/A{{ end }}%.''; }; labels = { severity = "warning"; category = "capacity"; }; } ]; } ]; }