{ lib, pkgs, config, ... }: { services.grafana.provision.alerting.rules.settings.groups = [ { # orgId = 1; # Defaults to 1 for provisioned rules name = "DiskUsageAlerts"; # Name of the rule group folder = "System Alerts"; # The folder these rules belong to in Grafana UI interval = "1m"; # How often to evaluate rules in this group rules = [ { uid = "high-disk-usage-alert-uid"; # Optional: provide a stable UID for the rule itself title = "HighDiskUsage"; # Name of the alert rule (was 'alert' in vmalert) # Condition for the alert to fire. 'C' refers to the refId of the threshold expression. condition = "D"; # Condition is now D # Removed rule-level relativeTimeRange # Data queries and expressions data = [ # Query A: Calculate disk usage percentage { refId = "A"; datasourceUid = "vm-datasource-uid"; # UID of the VictoriaMetrics datasource queryType = "prometheus"; # Explicitly set, though often inferred relativeTimeRange = { from = 60; to = 0; }; # Query-level, integer seconds model = { expr = '' ( node_filesystem_size_bytes{fstype!~"tmpfs|rootfs",mountpoint!=""} - node_filesystem_avail_bytes{fstype!~"tmpfs|rootfs",mountpoint!=""} ) / (node_filesystem_size_bytes{fstype!~"tmpfs|rootfs",mountpoint!=""} > 0) * 100 and node_filesystem_size_bytes{fstype!~"tmpfs|rootfs",mountpoint!=""} and node_filesystem_avail_bytes{fstype!~"tmpfs|rootfs",mountpoint!=""} ''; legendFormat = "{{mountpoint}} on {{instance}}"; # Example legend instant = false; # For range queries, default is false }; } # Expression C: Reduce Query A to its last value, preserving labels { refId = "C"; datasourceUid = "__expr__"; model = { type = "reduce"; expression = "A"; # Input is Query A reducer = "last"; # Get the last value of each series in A }; } # Expression D: Apply math condition to the reduced values from C { refId = "D"; datasourceUid = "__expr__"; model = { type = "math"; expression = "$C > 85"; # Check if the last value from each series in C is > 85 }; } ]; for = "15m"; # Duration the condition must be met (same as vmalert) # How to handle states where data is missing or query errors noDataState = "NoData"; # Options: NoData, Alerting, OK execErrState = "Error"; # Options: Error, Alerting, OK annotations = { summary = "High disk usage on {{ $labels.instance }} at {{ $labels.mountpoint }}"; description = '' Disk usage on {{ $labels.instance }} for mount point {{ $labels.mountpoint }} (fstype: {{ $labels.fstype }}) has been above 85% for more than 15 minutes. Current value: {{ if $values.C }}{{ $values.C | humanizePercentage }}{{ else }}N/A{{ end }}%. ''; # Using $values.C as it's the input to the math condition D }; labels = { severity = "warning"; category = "capacity"; # Grafana automatically adds labels from the query result (instance, mountpoint, etc.) # and labels from the rule group/folder. }; # isPaused = false; # Default is not paused } ]; } ]; }