refactor: Grafana alerting rules: consolidate and reorganize alert definitions
- Deleted individual alert files for host down, inode usage, and RAM usage. - Merged service down alerts into a new structure with separate files for each service (Gitea, Postfix, Dovecot, OpenLDAP, WireGuard). - Introduced a new system alert structure consolidating CPU, disk, host down, inode, and RAM usage alerts. - Updated alert conditions to use 'D' for thresholds and adjusted expressions accordingly. - Improved annotations and labels for clarity and consistency across alerts.
This commit is contained in:
54
hosts/web-arm/modules/grafana/alerting/system/host_down.nix
Normal file
54
hosts/web-arm/modules/grafana/alerting/system/host_down.nix
Normal file
@@ -0,0 +1,54 @@
|
||||
{ lib, pkgs, config, ... }:
|
||||
{
|
||||
grafanaAlertRuleDefinitions = [
|
||||
{
|
||||
uid = "host-down-alert-uid";
|
||||
title = "HostDown";
|
||||
condition = "C";
|
||||
|
||||
data = [
|
||||
{
|
||||
refId = "A";
|
||||
datasourceUid = "vm-datasource-uid";
|
||||
queryType = "prometheus";
|
||||
relativeTimeRange = { from = 60; to = 0; }; # Query over the last minute
|
||||
model = {
|
||||
expr = ''up'';
|
||||
legendFormat = "{{instance}} ({{job}})";
|
||||
instant = false; # Changed from true, as relativeTimeRange is used
|
||||
};
|
||||
}
|
||||
{ # New Expression B: Reduce Query A
|
||||
refId = "B";
|
||||
datasourceUid = "__expr__";
|
||||
model = {
|
||||
type = "reduce";
|
||||
expression = "A"; # Input is Query A
|
||||
reducer = "last"; # Get the last value of each series in A
|
||||
};
|
||||
}
|
||||
{ # Modified Expression C: Math condition based on B
|
||||
refId = "C";
|
||||
datasourceUid = "__expr__";
|
||||
model = {
|
||||
type = "math";
|
||||
expression = "$B == 0"; # Check if the last value from B is 0
|
||||
};
|
||||
}
|
||||
];
|
||||
|
||||
for = "2m";
|
||||
noDataState = "Alerting";
|
||||
execErrState = "Error";
|
||||
|
||||
annotations = {
|
||||
summary = "Host {{ $labels.instance }} is down";
|
||||
description = ''Host {{ $labels.instance }} (job: {{ $labels.job }}) has been down for more than 2 minutes.'';
|
||||
};
|
||||
labels = {
|
||||
severity = "critical";
|
||||
category = "availability";
|
||||
};
|
||||
}
|
||||
];
|
||||
}
|
||||
Reference in New Issue
Block a user