Cloonar/nixos
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

feat: implement centralized alerting with vmalert and Grafana, add alert rules for CPU, disk, inode, RAM usage, and host status

Browse source
This commit is contained in:
Dominik Polakovics Polakovics 2025-05-30 21:39:58 +02:00
parent fa42667c2a
commit 17a3602d3c
8 changed files with 231 additions and 25 deletions
Download patch file Download diff file Expand all files Collapse all files

26
hosts/web-arm/modules/vmalert/rules/cpu_usage.nix Normal file
View file

@ -0,0 +1,26 @@
{ lib, pkgs, config, ... }: # Standard module arguments
{
# This module contributes its rule group to a list that will be
# collected and processed by the main vmalert module.
services.vmalert.rules.groups = [
{
name = "CPUUsageAlerts";
# interval = "60s"; # Optional: group-level interval
rules = [ # This MUST be a list of rule attribute sets
{
alert = "HighCPUUsage";
expr = "(1 - avg by (instance, job) (rate(node_cpu_seconds_total{mode=\"idle\"}[1m]))) * 100 > 90";
for = "5m";
labels = {
severity = "warning";
category = "performance";
};
annotations = {
summary = "High CPU usage on {{ $labels.instance }}";
description = "CPU usage on {{ $labels.instance }} (job: {{ $labels.job }}) has been above 90% for more than 5 minutes. Current value: {{ $value | printf \"%.2f\" }}%.";
};
}
];
}
];
}

27
hosts/web-arm/modules/vmalert/rules/disk_usage.nix Normal file
View file

@ -0,0 +1,27 @@
{ lib, pkgs, config, ... }: # Standard module arguments
{
services.vmalert.rules.groups = [
{
name = "DiskUsageAlerts";
rules = [
{
alert = "HighDiskUsage";
expr = ''
(
node_filesystem_size_bytes{fstype!~"tmpfs|rootfs",mountpoint!=""} - node_filesystem_avail_bytes{fstype!~"tmpfs|rootfs",mountpoint!=""}
) / node_filesystem_size_bytes{fstype!~"tmpfs|rootfs",mountpoint!=""} * 100 > 85
'';
for = "15m";
labels = {
severity = "warning";
category = "capacity";
};
annotations = {
summary = "High disk usage on {{ $labels.instance }} at {{ $labels.mountpoint }}";
description = "Disk usage on {{ $labels.instance }} for mount point {{ $labels.mountpoint }} (fstype: {{ $labels.fstype }}) has been above 85% for more than 15 minutes. Current value: {{ $value | printf \"%.2f\" }}%.";
};
}
];
}
];
}

23
hosts/web-arm/modules/vmalert/rules/host_down.nix Normal file
View file

@ -0,0 +1,23 @@
{ lib, pkgs, config, ... }: # Standard module arguments
{
services.vmalert.rules.groups = [
{
name = "HostStatusAlerts";
rules = [
{
alert = "HostDown";
expr = "up == 0";
for = "2m";
labels = {
severity = "critical";
category = "availability";
};
annotations = {
summary = "Host {{ $labels.instance }} is down";
description = "Host {{ $labels.instance }} (job: {{ $labels.job }}) has been down for more than 2 minutes.";
};
}
];
}
];
}

27
hosts/web-arm/modules/vmalert/rules/inode_usage.nix Normal file
View file

@ -0,0 +1,27 @@
{ lib, pkgs, config, ... }: # Standard module arguments
{
services.vmalert.rules.groups = [
{
name = "InodeUsageAlerts";
rules = [
{
alert = "HighInodeUsage";
expr = ''
(
node_filesystem_files{fstype!~"tmpfs|rootfs",mountpoint!=""} - node_filesystem_files_free{fstype!~"tmpfs|rootfs",mountpoint!=""}
) / node_filesystem_files{fstype!~"tmpfs|rootfs",mountpoint!=""} * 100 > 80
'';
for = "30m";
labels = {
severity = "warning";
category = "capacity";
};
annotations = {
summary = "High inode usage on {{ $labels.instance }} at {{ $labels.mountpoint }}";
description = "Inode usage on {{ $labels.instance }} for mount point {{ $labels.mountpoint }} (fstype: {{ $labels.fstype }}) has been above 80% for more than 30 minutes. Current value: {{ $value | printf \"%.2f\" }}%.";
};
}
];
}
];
}

23
hosts/web-arm/modules/vmalert/rules/ram_usage.nix Normal file
View file

@ -0,0 +1,23 @@
{ lib, pkgs, config, ... }: # Standard module arguments
{
services.vmalert.rules.groups = [
{
name = "RAMUsageAlerts";
rules = [
{
alert = "HighRAMUsage";
expr = "(1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 > 90";
for = "10m";
labels = {
severity = "warning";
category = "performance";
};
annotations = {
summary = "High RAM usage on {{ $labels.instance }}";
description = "RAM usage on {{ $labels.instance }} (job: {{ $labels.job }}) has been above 90% for more than 10 minutes. Current value: {{ $value | printf \"%.2f\" }}%.";
};
}
];
}
];
}