Cloonar/nixos
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: refactor Grafana alerting rules into a consolidated system module and update individual alert files

Browse Source
This commit is contained in:
Dominik Polakovics Polakovics
2025-05-31 09:57:03 +02:00
parent 8b5fb0861d
commit 35fa61ef34
7 changed files with 281 additions and 299 deletions
Download Patch File Download Diff File Expand all files Collapse all files

137
hosts/web-arm/modules/grafana/alerting/disk_usage.nix
View File

@@ -1,85 +1,76 @@
{ lib, pkgs, config, ... }:
{
services.grafana.provision.alerting.rules.settings.groups = [
grafanaAlertRuleDefinitions = [
{
# orgId = 1; # Defaults to 1 for provisioned rules
name = "DiskUsageAlerts"; # Name of the rule group
folder = "System Alerts"; # The folder these rules belong to in Grafana UI
interval = "1m"; # How often to evaluate rules in this group
uid = "high-disk-usage-alert-uid"; # Optional: provide a stable UID for the rule itself
title = "HighDiskUsage"; # Name of the alert rule (was 'alert' in vmalert)
rules = [
# Condition for the alert to fire. 'D' refers to the refId of the threshold expression.
condition = "D"; # Condition is now D
# Removed rule-level relativeTimeRange
# Data queries and expressions
data = [
# Query A: Calculate disk usage percentage
{
uid = "high-disk-usage-alert-uid"; # Optional: provide a stable UID for the rule itself
title = "HighDiskUsage"; # Name of the alert rule (was 'alert' in vmalert)
# Condition for the alert to fire. 'C' refers to the refId of the threshold expression.
condition = "D"; # Condition is now D
# Removed rule-level relativeTimeRange
# Data queries and expressions
data = [
# Query A: Calculate disk usage percentage
{
refId = "A";
datasourceUid = "vm-datasource-uid"; # UID of the VictoriaMetrics datasource
queryType = "prometheus"; # Explicitly set, though often inferred
relativeTimeRange = { from = 60; to = 0; }; # Query-level, integer seconds
model = {
expr = ''
(
node_filesystem_size_bytes{fstype!~"tmpfs|rootfs",mountpoint!=""} - node_filesystem_avail_bytes{fstype!~"tmpfs|rootfs",mountpoint!=""}
) / (node_filesystem_size_bytes{fstype!~"tmpfs|rootfs",mountpoint!=""} > 0) * 100
and node_filesystem_size_bytes{fstype!~"tmpfs|rootfs",mountpoint!=""}
and node_filesystem_avail_bytes{fstype!~"tmpfs|rootfs",mountpoint!=""}
'';
legendFormat = "{{mountpoint}} on {{instance}}"; # Example legend
instant = false; # For range queries, default is false
};
}
# Expression C: Reduce Query A to its last value, preserving labels
{
refId = "C";
datasourceUid = "__expr__";
model = {
type = "reduce";
expression = "A"; # Input is Query A
reducer = "last"; # Get the last value of each series in A
};
}
# Expression D: Apply math condition to the reduced values from C
{
refId = "D";
datasourceUid = "__expr__";
model = {
type = "math";
expression = "$C > 85"; # Check if the last value from each series in C is > 85
};
}
];
for = "15m"; # Duration the condition must be met (same as vmalert)
# How to handle states where data is missing or query errors
noDataState = "NoData"; # Options: NoData, Alerting, OK
execErrState = "Error"; # Options: Error, Alerting, OK
annotations = {
summary = "High disk usage on {{ $labels.instance }} at {{ $labels.mountpoint }}";
description = ''
Disk usage on {{ $labels.instance }} for mount point {{ $labels.mountpoint }}
(fstype: {{ $labels.fstype }}) has been above 85% for more than 15 minutes.
Current value: {{ if $values.C }}{{ $values.C | humanizePercentage }}{{ else }}N/A{{ end }}%.
''; # Using $values.C as it's the input to the math condition D
refId = "A";
datasourceUid = "vm-datasource-uid"; # UID of the VictoriaMetrics datasource
queryType = "prometheus"; # Explicitly set, though often inferred
relativeTimeRange = { from = 60; to = 0; }; # Query-level, integer seconds
model = {
expr = ''
(
node_filesystem_size_bytes{fstype!~"tmpfs|rootfs",mountpoint!=""} - node_filesystem_avail_bytes{fstype!~"tmpfs|rootfs",mountpoint!=""}
) / (node_filesystem_size_bytes{fstype!~"tmpfs|rootfs",mountpoint!=""} > 0) * 100
and node_filesystem_size_bytes{fstype!~"tmpfs|rootfs",mountpoint!=""}
and node_filesystem_avail_bytes{fstype!~"tmpfs|rootfs",mountpoint!=""}
'';
legendFormat = "{{mountpoint}} on {{instance}}"; # Example legend
instant = false; # For range queries, default is false
};
labels = {
severity = "warning";
category = "capacity";
# Grafana automatically adds labels from the query result (instance, mountpoint, etc.)
# and labels from the rule group/folder.
}
# Expression C: Reduce Query A to its last value, preserving labels
{
refId = "C";
datasourceUid = "__expr__";
model = {
type = "reduce";
expression = "A"; # Input is Query A
reducer = "last"; # Get the last value of each series in A
};
}
# Expression D: Apply math condition to the reduced values from C
{
refId = "D";
datasourceUid = "__expr__";
model = {
type = "math";
expression = "$C > 85"; # Check if the last value from each series in C is > 85
};
# isPaused = false; # Default is not paused
}
];
for = "15m"; # Duration the condition must be met (same as vmalert)
# How to handle states where data is missing or query errors
noDataState = "NoData"; # Options: NoData, Alerting, OK
execErrState = "Error"; # Options: Error, Alerting, OK
annotations = {
summary = "High disk usage on {{ $labels.instance }} at {{ $labels.mountpoint }}";
description = ''
Disk usage on {{ $labels.instance }} for mount point {{ $labels.mountpoint }}
(fstype: {{ $labels.fstype }}) has been above 85% for more than 15 minutes.
Current value: {{ if $values.C }}{{ $values.C | humanizePercentage }}{{ else }}N/A{{ end }}%.
''; # Using $values.C as it's the input to the math condition D
};
labels = {
severity = "warning";
category = "capacity";
# Grafana automatically adds labels from the query result (instance, mountpoint, etc.)
# and labels from the rule group/folder.
};
# isPaused = false; # Default is not paused
}
];
}