Dominik Polakovics
b6b90bca7d
- Deleted individual alert files for host down, inode usage, and RAM usage. - Merged service down alerts into a new structure with separate files for each service (Gitea, Postfix, Dovecot, OpenLDAP, WireGuard). - Introduced a new system alert structure consolidating CPU, disk, host down, inode, and RAM usage alerts. - Updated alert conditions to use 'D' for thresholds and adjusted expressions accordingly. - Improved annotations and labels for clarity and consistency across alerts.
62 lines
1.9 KiB
Nix
62 lines
1.9 KiB
Nix
{ lib, pkgs, config, ... }:
|
|
{
|
|
grafanaAlertRuleDefinitions = [
|
|
{
|
|
uid = "high-ram-usage-alert-uid";
|
|
title = "HighRAMUsage";
|
|
condition = "D"; # Condition is now D
|
|
|
|
data = [
|
|
# Query A: Calculate RAM usage percentage
|
|
{
|
|
refId = "A";
|
|
datasourceUid = "vm-datasource-uid";
|
|
queryType = "prometheus";
|
|
relativeTimeRange = { from = 60; to = 0; };
|
|
model = {
|
|
expr = ''
|
|
(1 - node_memory_MemAvailable_bytes / (node_memory_MemTotal_bytes > 0)) * 100
|
|
and node_memory_MemAvailable_bytes
|
|
and node_memory_MemTotal_bytes
|
|
'';
|
|
legendFormat = "RAM usage on {{instance}} ({{job}})";
|
|
instant = false;
|
|
};
|
|
}
|
|
# Expression C: Reduce Query A to its last value, preserving labels
|
|
{
|
|
refId = "C";
|
|
datasourceUid = "__expr__";
|
|
model = {
|
|
type = "reduce";
|
|
expression = "A"; # Input is Query A
|
|
reducer = "last"; # Get the last value of each series in A
|
|
};
|
|
}
|
|
# Expression D: Apply math condition to the reduced values from C
|
|
{
|
|
refId = "D";
|
|
datasourceUid = "__expr__";
|
|
model = {
|
|
type = "math";
|
|
expression = "$C > 90"; # Alert if RAM usage from C is > 90%
|
|
};
|
|
}
|
|
];
|
|
|
|
for = "10m"; # Duration the condition must be met
|
|
noDataState = "NoData";
|
|
execErrState = "Error";
|
|
|
|
annotations = {
|
|
summary = "High RAM usage on {{ $labels.instance }}";
|
|
description = ''RAM usage on {{ $labels.instance }} (job: {{ $labels.job }}) has been above 90% for more than 10 minutes. Current value: {{ if $values.C }}{{ $values.C | humanizePercentage }}{{ else }}N/A{{ end }}%.'';
|
|
};
|
|
labels = {
|
|
severity = "warning";
|
|
category = "performance";
|
|
};
|
|
}
|
|
];
|
|
}
|