Cloonar/nixos
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: add smart alerting and noatime to disks

Browse Source
This commit is contained in:
Dominik Polakovics Polakovics
2025-11-28 23:50:24 +01:00
parent dbada3c509
commit 537f144885
8 changed files with 642 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
hosts/web-arm/modules/grafana/alerting/storage/default.nix Normal file
View File

@@ -0,0 +1,17 @@
{ lib, pkgs, config, ... }:
let
smartAlertRules = (import ./smart_alerts.nix { inherit lib pkgs config; }).grafanaAlertRuleDefinitions;
raidAlertRules = (import ./raid_alerts.nix { inherit lib pkgs config; }).grafanaAlertRuleDefinitions;
allStorageRules = smartAlertRules ++ raidAlertRules;
in
{
services.grafana.provision.alerting.rules.settings.groups = [
{
name = "Storage Alerts";
folder = "Storage Alerts";
interval = "5m"; # Check every 5 minutes (metrics collected every 20 min)
rules = allStorageRules;
}
];
}

102
hosts/web-arm/modules/grafana/alerting/storage/raid_alerts.nix Normal file
View File

@@ -0,0 +1,102 @@
{ lib, pkgs, config, ... }:
{
grafanaAlertRuleDefinitions = [
# RAID array degraded - critical
{
uid = "raid-array-degraded-uid";
title = "RaidArrayDegraded";
condition = "D";
data = [
{
refId = "A";
datasourceUid = "vm-datasource-uid";
relativeTimeRange = { from = 300; to = 0; };
model = {
expr = ''mdadm_array_state == 0'';
instant = false;
};
}
{
refId = "C";
datasourceUid = "__expr__";
model = {
type = "reduce";
expression = "A";
reducer = "last";
};
}
{
refId = "D";
datasourceUid = "__expr__";
model = {
type = "math";
expression = "$C == 0";
};
}
];
for = "0s";
noDataState = "NoData";
execErrState = "Error";
annotations = {
summary = "RAID array {{ $labels.array }} is degraded";
description = ''
RAID array {{ $labels.array }} on {{ $labels.instance }} is in state "{{ $labels.state }}".
The array is not in a healthy state. Check for failed disks immediately!
'';
};
labels = {
severity = "critical";
category = "storage";
};
}
# RAID missing devices - critical
{
uid = "raid-missing-devices-uid";
title = "RaidMissingDevices";
condition = "D";
data = [
{
refId = "A";
datasourceUid = "vm-datasource-uid";
relativeTimeRange = { from = 300; to = 0; };
model = {
expr = ''mdadm_array_devices_active < mdadm_array_devices_total'';
instant = false;
};
}
{
refId = "C";
datasourceUid = "__expr__";
model = {
type = "reduce";
expression = "A";
reducer = "last";
};
}
{
refId = "D";
datasourceUid = "__expr__";
model = {
type = "math";
expression = "$C > 0";
};
}
];
for = "0s";
noDataState = "NoData";
execErrState = "Error";
annotations = {
summary = "RAID array {{ $labels.array }} has missing devices";
description = ''
RAID array {{ $labels.array }} on {{ $labels.instance }} has fewer active devices than expected.
A disk may have failed or been removed. Check array status immediately!
'';
};
labels = {
severity = "critical";
category = "storage";
};
}
];
}

298
hosts/web-arm/modules/grafana/alerting/storage/smart_alerts.nix Normal file
View File

@@ -0,0 +1,298 @@
{ lib, pkgs, config, ... }:
{
grafanaAlertRuleDefinitions = [
# S.M.A.R.T. overall health failed - critical
{
uid = "smart-health-failed-uid";
title = "DiskSmartHealthFailed";
condition = "D";
data = [
{
refId = "A";
datasourceUid = "vm-datasource-uid";
relativeTimeRange = { from = 300; to = 0; };
model = {
expr = ''smart_health_passed == 0'';
instant = false;
};
}
{
refId = "C";
datasourceUid = "__expr__";
model = {
type = "reduce";
expression = "A";
reducer = "last";
};
}
{
refId = "D";
datasourceUid = "__expr__";
model = {
type = "math";
expression = "$C == 0";
};
}
];
for = "0s";
noDataState = "NoData";
execErrState = "Error";
annotations = {
summary = "S.M.A.R.T. health check FAILED on {{ $labels.device }}";
description = ''
Disk {{ $labels.device }} ({{ $labels.serial }}) on {{ $labels.instance }} has failed its S.M.A.R.T. health check.
This indicates imminent disk failure. Replace the disk immediately!
'';
};
labels = {
severity = "critical";
category = "storage";
};
}
# Reallocated sectors - warning (any count > 0 is concerning)
{
uid = "smart-reallocated-sectors-uid";
title = "DiskReallocatedSectors";
condition = "D";
data = [
{
refId = "A";
datasourceUid = "vm-datasource-uid";
relativeTimeRange = { from = 300; to = 0; };
model = {
expr = ''smart_reallocated_sector_ct > 0'';
instant = false;
};
}
{
refId = "C";
datasourceUid = "__expr__";
model = {
type = "reduce";
expression = "A";
reducer = "last";
};
}
{
refId = "D";
datasourceUid = "__expr__";
model = {
type = "math";
expression = "$C > 0";
};
}
];
for = "0s";
noDataState = "NoData";
execErrState = "Error";
annotations = {
summary = "Reallocated sectors detected on {{ $labels.device }}";
description = ''
Disk {{ $labels.device }} ({{ $labels.serial }}) on {{ $labels.instance }} has reallocated sectors.
This indicates disk surface damage. Monitor closely and plan replacement.
'';
};
labels = {
severity = "warning";
category = "storage";
};
}
# Current pending sectors
{
uid = "smart-pending-sectors-uid";
title = "DiskPendingSectors";
condition = "D";
data = [
{
refId = "A";
datasourceUid = "vm-datasource-uid";
relativeTimeRange = { from = 300; to = 0; };
model = {
expr = ''smart_current_pending_sector > 0'';
instant = false;
};
}
{
refId = "C";
datasourceUid = "__expr__";
model = {
type = "reduce";
expression = "A";
reducer = "last";
};
}
{
refId = "D";
datasourceUid = "__expr__";
model = {
type = "math";
expression = "$C > 0";
};
}
];
for = "0s";
noDataState = "NoData";
execErrState = "Error";
annotations = {
summary = "Pending sectors detected on {{ $labels.device }}";
description = ''
Disk {{ $labels.device }} ({{ $labels.serial }}) on {{ $labels.instance }} has pending sectors.
These sectors could not be read and may be reallocated. Monitor for increase.
'';
};
labels = {
severity = "warning";
category = "storage";
};
}
# Offline uncorrectable errors
{
uid = "smart-offline-uncorrectable-uid";
title = "DiskOfflineUncorrectable";
condition = "D";
data = [
{
refId = "A";
datasourceUid = "vm-datasource-uid";
relativeTimeRange = { from = 300; to = 0; };
model = {
expr = ''smart_offline_uncorrectable > 0'';
instant = false;
};
}
{
refId = "C";
datasourceUid = "__expr__";
model = {
type = "reduce";
expression = "A";
reducer = "last";
};
}
{
refId = "D";
datasourceUid = "__expr__";
model = {
type = "math";
expression = "$C > 0";
};
}
];
for = "0s";
noDataState = "NoData";
execErrState = "Error";
annotations = {
summary = "Offline uncorrectable errors on {{ $labels.device }}";
description = ''
Disk {{ $labels.device }} ({{ $labels.serial }}) on {{ $labels.instance }} has offline uncorrectable errors.
This indicates data integrity issues. Consider replacement.
'';
};
labels = {
severity = "warning";
category = "storage";
};
}
# High temperature (Seagate enterprise: warning at 50C)
{
uid = "smart-high-temperature-uid";
title = "DiskHighTemperature";
condition = "D";
data = [
{
refId = "A";
datasourceUid = "vm-datasource-uid";
relativeTimeRange = { from = 600; to = 0; };
model = {
expr = ''smart_temperature_celsius > 50'';
instant = false;
};
}
{
refId = "C";
datasourceUid = "__expr__";
model = {
type = "reduce";
expression = "A";
reducer = "last";
};
}
{
refId = "D";
datasourceUid = "__expr__";
model = {
type = "math";
expression = "$C > 0";
};
}
];
for = "10m";
noDataState = "NoData";
execErrState = "Error";
annotations = {
summary = "High temperature on {{ $labels.device }}";
description = ''
Disk {{ $labels.device }} ({{ $labels.serial }}) on {{ $labels.instance }} temperature exceeds 50°C.
Check cooling and ventilation.
'';
};
labels = {
severity = "warning";
category = "storage";
};
}
# UDMA CRC errors (cable/connection issues)
{
uid = "smart-udma-crc-errors-uid";
title = "DiskUDMACRCErrors";
condition = "D";
data = [
{
refId = "A";
datasourceUid = "vm-datasource-uid";
relativeTimeRange = { from = 86400; to = 0; };
model = {
expr = ''increase(smart_udma_crc_error_count[24h]) > 0'';
instant = false;
};
}
{
refId = "C";
datasourceUid = "__expr__";
model = {
type = "reduce";
expression = "A";
reducer = "last";
};
}
{
refId = "D";
datasourceUid = "__expr__";
model = {
type = "math";
expression = "$C > 0";
};
}
];
for = "0s";
noDataState = "NoData";
execErrState = "Error";
annotations = {
summary = "UDMA CRC errors on {{ $labels.device }}";
description = ''
Disk {{ $labels.device }} ({{ $labels.serial }}) on {{ $labels.instance }} has new CRC errors.
This typically indicates SATA cable or connection issues. Check cables.
'';
};
labels = {
severity = "warning";
category = "storage";
};
}
];
}