Cloonar/nixos
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: implement centralized alerting with vmalert and Grafana, add alert rules for CPU, disk, inode, RAM usage, and host status

Browse Source
This commit is contained in:
Dominik Polakovics Polakovics
2025-05-30 21:39:58 +02:00
parent fa42667c2a
commit 17a3602d3c
8 changed files with 231 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

38
hosts/web-arm/modules/vmalert/default.nix Normal file
View File

@@ -0,0 +1,38 @@
{ config, pkgs, lib, ... }:
{
imports = [
./rules/cpu_usage.nix
./rules/disk_usage.nix
./rules/host_down.nix
./rules/inode_usage.nix
./rules/ram_usage.nix
];
# Standard vmalert service configuration
services.vmalert = {
enable = true;
settings = {
"datasource.url" = "http://localhost:8428"; # VictoriaMetrics address
"notifier.url" = [ "http://localhost:3001/api/alertmanager/grafana/api/v2/alerts" ]; # Must be a list of strings
};
# 'rules' is now set by the mkMerge block above.
};
# Override the User and Group for the systemd service managed by the official vmalert module.
systemd.services.vmalert = {
serviceConfig = {
User = "victoriametrics";
Group = "victoriametrics";
};
};
# Ensure the user/group itself exists on the system.
users.users.victoriametrics = lib.mkIf (config.services.victoriametrics.enable || config.services.vmalert.enable) {
isSystemUser = true;
group = "victoriametrics"; # Primary group for the user
home = "/var/lib/victoriametrics"; # Standard home for VictoriaMetrics components
};
users.groups.victoriametrics = lib.mkIf (config.services.victoriametrics.enable || config.services.vmalert.enable) {
# Ensures the group exists.
};
}

26
hosts/web-arm/modules/vmalert/rules/cpu_usage.nix Normal file
View File

@@ -0,0 +1,26 @@
{ lib, pkgs, config, ... }: # Standard module arguments
{
# This module contributes its rule group to a list that will be
# collected and processed by the main vmalert module.
services.vmalert.rules.groups = [
{
name = "CPUUsageAlerts";
# interval = "60s"; # Optional: group-level interval
rules = [ # This MUST be a list of rule attribute sets
{
alert = "HighCPUUsage";
expr = "(1 - avg by (instance, job) (rate(node_cpu_seconds_total{mode=\"idle\"}[1m]))) * 100 > 90";
for = "5m";
labels = {
severity = "warning";
category = "performance";
};
annotations = {
summary = "High CPU usage on {{ $labels.instance }}";
description = "CPU usage on {{ $labels.instance }} (job: {{ $labels.job }}) has been above 90% for more than 5 minutes. Current value: {{ $value | printf \"%.2f\" }}%.";
};
}
];
}
];
}

27
hosts/web-arm/modules/vmalert/rules/disk_usage.nix Normal file
View File

@@ -0,0 +1,27 @@
{ lib, pkgs, config, ... }: # Standard module arguments
{
services.vmalert.rules.groups = [
{
name = "DiskUsageAlerts";
rules = [
{
alert = "HighDiskUsage";
expr = ''
(
node_filesystem_size_bytes{fstype!~"tmpfs|rootfs",mountpoint!=""} - node_filesystem_avail_bytes{fstype!~"tmpfs|rootfs",mountpoint!=""}
) / node_filesystem_size_bytes{fstype!~"tmpfs|rootfs",mountpoint!=""} * 100 > 85
'';
for = "15m";
labels = {
severity = "warning";
category = "capacity";
};
annotations = {
summary = "High disk usage on {{ $labels.instance }} at {{ $labels.mountpoint }}";
description = "Disk usage on {{ $labels.instance }} for mount point {{ $labels.mountpoint }} (fstype: {{ $labels.fstype }}) has been above 85% for more than 15 minutes. Current value: {{ $value | printf \"%.2f\" }}%.";
};
}
];
}
];
}

23
hosts/web-arm/modules/vmalert/rules/host_down.nix Normal file
View File

@@ -0,0 +1,23 @@
{ lib, pkgs, config, ... }: # Standard module arguments
{
services.vmalert.rules.groups = [
{
name = "HostStatusAlerts";
rules = [
{
alert = "HostDown";
expr = "up == 0";
for = "2m";
labels = {
severity = "critical";
category = "availability";
};
annotations = {
summary = "Host {{ $labels.instance }} is down";
description = "Host {{ $labels.instance }} (job: {{ $labels.job }}) has been down for more than 2 minutes.";
};
}
];
}
];
}

27
hosts/web-arm/modules/vmalert/rules/inode_usage.nix Normal file
View File

@@ -0,0 +1,27 @@
{ lib, pkgs, config, ... }: # Standard module arguments
{
services.vmalert.rules.groups = [
{
name = "InodeUsageAlerts";
rules = [
{
alert = "HighInodeUsage";
expr = ''
(
node_filesystem_files{fstype!~"tmpfs|rootfs",mountpoint!=""} - node_filesystem_files_free{fstype!~"tmpfs|rootfs",mountpoint!=""}
) / node_filesystem_files{fstype!~"tmpfs|rootfs",mountpoint!=""} * 100 > 80
'';
for = "30m";
labels = {
severity = "warning";
category = "capacity";
};
annotations = {
summary = "High inode usage on {{ $labels.instance }} at {{ $labels.mountpoint }}";
description = "Inode usage on {{ $labels.instance }} for mount point {{ $labels.mountpoint }} (fstype: {{ $labels.fstype }}) has been above 80% for more than 30 minutes. Current value: {{ $value | printf \"%.2f\" }}%.";
};
}
];
}
];
}

23
hosts/web-arm/modules/vmalert/rules/ram_usage.nix Normal file
View File

@@ -0,0 +1,23 @@
{ lib, pkgs, config, ... }: # Standard module arguments
{
services.vmalert.rules.groups = [
{
name = "RAMUsageAlerts";
rules = [
{
alert = "HighRAMUsage";
expr = "(1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 > 90";
for = "10m";
labels = {
severity = "warning";
category = "performance";
};
annotations = {
summary = "High RAM usage on {{ $labels.instance }}";
description = "RAM usage on {{ $labels.instance }} (job: {{ $labels.job }}) has been above 90% for more than 10 minutes. Current value: {{ $value | printf \"%.2f\" }}%.";
};
}
];
}
];
}