Cloonar/nixos
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Compare commits

base: Cloonar:b6b90bca7d602851770126ff4839a9e88aa544e5
Branches Tags
Cloonar:master
..
compare: Cloonar:7fc3c3db631ad0dc5d35a92dc1665fc6701cbd46
Branches Tags
Cloonar:master

No commits in common. "b6b90bca7d602851770126ff4839a9e88aa544e5" and "7fc3c3db631ad0dc5d35a92dc1665fc6701cbd46" have entirely different histories.
b6b90bca7d ... 7fc3c3db63

16 changed files with 330 additions and 377 deletions
Download patch file Download diff file Expand all files Collapse all files

0
hosts/web-arm/modules/grafana/alerting/system/cpu_usage.nix → hosts/web-arm/modules/grafana/alerting/cpu_usage.nix
View file

0
hosts/web-arm/modules/grafana/alerting/system/disk_usage.nix → hosts/web-arm/modules/grafana/alerting/disk_usage.nix
View file

0
hosts/web-arm/modules/grafana/alerting/system/host_down.nix → hosts/web-arm/modules/grafana/alerting/host_down.nix
View file

0
hosts/web-arm/modules/grafana/alerting/system/inode_usage.nix → hosts/web-arm/modules/grafana/alerting/inode_usage.nix
View file

0
hosts/web-arm/modules/grafana/alerting/system/ram_usage.nix → hosts/web-arm/modules/grafana/alerting/ram_usage.nix
View file

14
hosts/web-arm/modules/grafana/alerting/service/default.nix
View file

@ -1,18 +1,8 @@
{ lib, pkgs, config, ... }: { lib, pkgs, config, ... }:
let let
giteaDownAlertRules = (import ./gitea_down.nix { inherit lib pkgs config; }).grafanaAlertRuleDefinitions; serviceDownAlertRules = (import ./service_down.nix { inherit lib pkgs config; }).grafanaAlertRuleDefinitions;
giteaRunnerDownAlertRules = (import ./gitea_runner_down.nix { inherit lib pkgs config; }).grafanaAlertRuleDefinitions;
postfixDownAlertRules = (import ./postfix_down.nix { inherit lib pkgs config; }).grafanaAlertRuleDefinitions;
dovecotDownAlertRules = (import ./dovecot_down.nix { inherit lib pkgs config; }).grafanaAlertRuleDefinitions;
openldapDownAlertRules = (import ./openldap_down.nix { inherit lib pkgs config; }).grafanaAlertRuleDefinitions;
wireguardDownAlertRules = (import ./wireguard_down.nix { inherit lib pkgs config; }).grafanaAlertRuleDefinitions;
allServiceRules = giteaDownAlertRules allServiceRules = serviceDownAlertRules;
++ giteaRunnerDownAlertRules
++ postfixDownAlertRules
++ dovecotDownAlertRules
++ openldapDownAlertRules
++ wireguardDownAlertRules;
in in
{ {
services.grafana.provision.alerting.rules.settings.groups = [ services.grafana.provision.alerting.rules.settings.groups = [

57
hosts/web-arm/modules/grafana/alerting/service/dovecot_down.nix
View file

@ -1,57 +0,0 @@
{ lib, pkgs, config, ... }:
{
grafanaAlertRuleDefinitions = [
{
uid = "dovecot-service-down-alert-uid";
title = "Dovecot Service Down";
condition = "C";
data = [
{
refId = "A";
relativeTimeRange = {
from = 300;
to = 0;
};
datasourceUid = "vm-datasource-uid";
model = {
editorMode = "code";
expr = "node_systemd_unit_state{state=\"active\", name=\"dovecot2.service\"} OR on() vector(0)";
hide = false;
intervalMs = 1000;
legendFormat = "__auto";
maxDataPoints = 43200;
range = true;
refId = "A";
};
}
{
refId = "B";
datasourceUid = "__expr__";
model = {
type = "reduce";
expression = "A";
reducer = "last";
};
}
{
refId = "C";
datasourceUid = "__expr__";
model = {
type = "math";
expression = "$B < 1";
};
}
];
noDataState = "Alerting";
execErrState = "Alerting";
for = "5m";
annotations = {
description = "Dovecot service is down on {{ $labels.instance }}";
summary = "Dovecot Service Down";
};
labels = {
severity = "critical";
};
}
];
}

57
hosts/web-arm/modules/grafana/alerting/service/gitea_down.nix
View file

@ -1,57 +0,0 @@
{ lib, pkgs, config, ... }:
{
grafanaAlertRuleDefinitions = [
{
uid = "gitea-service-down-alert-uid";
title = "Gitea Service Down";
condition = "C";
data = [
{
refId = "A";
relativeTimeRange = {
from = 300;
to = 0;
};
datasourceUid = "vm-datasource-uid";
model = {
editorMode = "code";
expr = "node_systemd_unit_state{state=\"active\", name=\"container@git.service\"} OR on() vector(0)";
hide = false;
intervalMs = 1000;
legendFormat = "__auto";
maxDataPoints = 43200;
range = true;
refId = "A";
};
}
{
refId = "B";
datasourceUid = "__expr__";
model = {
type = "reduce";
expression = "A";
reducer = "last";
};
}
{
refId = "C";
datasourceUid = "__expr__";
model = {
type = "math";
expression = "$B < 1";
};
}
];
noDataState = "Alerting";
execErrState = "Alerting";
for = "5m";
annotations = {
description = "Gitea service is down on {{ $labels.instance }}";
summary = "Gitea Service Down";
};
labels = {
severity = "critical";
};
}
];
}

57
hosts/web-arm/modules/grafana/alerting/service/gitea_runner_down.nix
View file

@ -1,57 +0,0 @@
{ lib, pkgs, config, ... }:
{
grafanaAlertRuleDefinitions = [
{
uid = "gitea-runner-service-down-alert-uid";
title = "Gitea Runner Service Down";
condition = "C";
data = [
{
refId = "A";
relativeTimeRange = {
from = 300;
to = 0;
};
datasourceUid = "vm-datasource-uid";
model = {
editorMode = "code";
expr = "node_systemd_unit_state{state=\"active\", name=\"microvm@git-runner-1.service\"} OR on() vector(0)";
hide = false;
intervalMs = 1000;
legendFormat = "__auto";
maxDataPoints = 43200;
range = true;
refId = "A";
};
}
{
refId = "B";
datasourceUid = "__expr__";
model = {
type = "reduce";
expression = "A";
reducer = "last";
};
}
{
refId = "C";
datasourceUid = "__expr__";
model = {
type = "math";
expression = "$B < 1";
};
}
];
noDataState = "Alerting";
execErrState = "Alerting";
for = "5m";
annotations = {
description = "Gitea Runner service is down on {{ $labels.instance }}";
summary = "Gitea Runner Service Down";
};
labels = {
severity = "critical";
};
}
];
}

57
hosts/web-arm/modules/grafana/alerting/service/openldap_down.nix
View file

@ -1,57 +0,0 @@
{ lib, pkgs, config, ... }:
{
grafanaAlertRuleDefinitions = [
{
uid = "openldap-service-down-alert-uid";
title = "OpenLDAP Service Down";
condition = "C";
data = [
{
refId = "A";
relativeTimeRange = {
from = 300;
to = 0;
};
datasourceUid = "vm-datasource-uid";
model = {
editorMode = "code";
expr = "node_systemd_unit_state{state=\"active\", name=\"openldap.service\"} OR on() vector(0)";
hide = false;
intervalMs = 1000;
legendFormat = "__auto";
maxDataPoints = 43200;
range = true;
refId = "A";
};
}
{
refId = "B";
datasourceUid = "__expr__";
model = {
type = "reduce";
expression = "A";
reducer = "last";
};
}
{
refId = "C";
datasourceUid = "__expr__";
model = {
type = "math";
expression = "$B < 1";
};
}
];
noDataState = "Alerting";
execErrState = "Alerting";
for = "5m";
annotations = {
description = "OpenLDAP service is down on {{ $labels.instance }}";
summary = "OpenLDAP Service Down";
};
labels = {
severity = "critical";
};
}
];
}

57
hosts/web-arm/modules/grafana/alerting/service/postfix_down.nix
View file

@ -1,57 +0,0 @@
{ lib, pkgs, config, ... }:
{
grafanaAlertRuleDefinitions = [
{
uid = "postfix-service-down-alert-uid";
title = "Postfix Service Down";
condition = "C";
data = [
{
refId = "A";
relativeTimeRange = {
from = 300;
to = 0;
};
datasourceUid = "vm-datasource-uid";
model = {
editorMode = "code";
expr = "node_systemd_unit_state{state=\"active\", name=\"postfix.service\"} OR on() vector(0)";
hide = false;
intervalMs = 1000;
legendFormat = "__auto";
maxDataPoints = 43200;
range = true;
refId = "A";
};
}
{
refId = "B";
datasourceUid = "__expr__";
model = {
type = "reduce";
expression = "A";
reducer = "last";
};
}
{
refId = "C";
datasourceUid = "__expr__";
model = {
type = "math";
expression = "$B < 1";
};
}
];
noDataState = "Alerting";
execErrState = "Alerting";
for = "5m";
annotations = {
description = "Postfix service is down on {{ $labels.instance }}";
summary = "Postfix Service Down";
};
labels = {
severity = "critical";
};
}
];
}

318
hosts/web-arm/modules/grafana/alerting/service/service_down.nix Normal file
View file

@ -0,0 +1,318 @@
{ lib, pkgs, config, ... }:
{
grafanaAlertRuleDefinitions = [
# Systemd service monitoring alerts
{
uid = "gitea-service-down-alert-uid";
title = "Gitea Service Down";
condition = "C";
data = [
{
refId = "A";
relativeTimeRange = {
from = 300;
to = 0;
};
datasourceUid = "vm-datasource-uid";
model = {
editorMode = "code";
expr = "node_systemd_unit_state{state=\"active\", name=\"container@git.service\"} OR on() vector(0)";
hide = false;
intervalMs = 1000;
legendFormat = "__auto";
maxDataPoints = 43200;
range = true;
refId = "A";
};
}
{
refId = "B";
datasourceUid = "__expr__";
model = {
type = "reduce";
expression = "A";
reducer = "last";
};
}
{
refId = "C";
datasourceUid = "__expr__";
model = {
type = "math";
expression = "$B < 1";
};
}
];
noDataState = "Alerting";
execErrState = "Alerting";
for = "5m";
annotations = {
description = "Gitea service is down on {{ $labels.instance }}";
summary = "Gitea Service Down";
};
labels = {
severity = "critical";
};
}
{
uid = "gitea-runner-service-down-alert-uid";
title = "Gitea Runner Service Down";
condition = "C";
data = [
{
refId = "A";
relativeTimeRange = {
from = 300;
to = 0;
};
datasourceUid = "vm-datasource-uid";
model = {
editorMode = "code";
expr = "node_systemd_unit_state{state=\"active\", name=\"microvm@git-runner-1.service\"} OR on() vector(0)";
hide = false;
intervalMs = 1000;
legendFormat = "__auto";
maxDataPoints = 43200;
range = true;
refId = "A";
};
}
{
refId = "B";
datasourceUid = "__expr__";
model = {
type = "reduce";
expression = "A";
reducer = "last";
};
}
{
refId = "C";
datasourceUid = "__expr__";
model = {
type = "math";
expression = "$B < 1";
};
}
];
noDataState = "Alerting";
execErrState = "Alerting";
for = "5m";
annotations = {
description = "Gitea Runner service is down on {{ $labels.instance }}";
summary = "Gitea Runner Service Down";
};
labels = {
severity = "critical";
};
}
{
uid = "postfix-service-down-alert-uid";
title = "Postfix Service Down";
condition = "C";
data = [
{
refId = "A";
relativeTimeRange = {
from = 300;
to = 0;
};
datasourceUid = "vm-datasource-uid";
model = {
editorMode = "code";
expr = "node_systemd_unit_state{state=\"active\", name=\"postfix.service\"} OR on() vector(0)";
hide = false;
intervalMs = 1000;
legendFormat = "__auto";
maxDataPoints = 43200;
range = true;
refId = "A";
};
}
{
refId = "B";
datasourceUid = "__expr__";
model = {
type = "reduce";
expression = "A";
reducer = "last";
};
}
{
refId = "C";
datasourceUid = "__expr__";
model = {
type = "math";
expression = "$B < 1";
};
}
];
noDataState = "Alerting";
execErrState = "Alerting";
for = "5m";
annotations = {
description = "Postfix service is down on {{ $labels.instance }}";
summary = "Postfix Service Down";
};
labels = {
severity = "critical";
};
}
{
uid = "dovecot-service-down-alert-uid";
title = "Dovecot Service Down";
condition = "C";
data = [
{
refId = "A";
relativeTimeRange = {
from = 300;
to = 0;
};
datasourceUid = "vm-datasource-uid";
model = {
editorMode = "code";
expr = "node_systemd_unit_state{state=\"active\", name=\"dovecot2.service\"} OR on() vector(0)";
hide = false;
intervalMs = 1000;
legendFormat = "__auto";
maxDataPoints = 43200;
range = true;
refId = "A";
};
}
{
refId = "B";
datasourceUid = "__expr__";
model = {
type = "reduce";
expression = "A";
reducer = "last";
};
}
{
refId = "C";
datasourceUid = "__expr__";
model = {
type = "math";
expression = "$B < 1";
};
}
];
noDataState = "Alerting";
execErrState = "Alerting";
for = "5m";
annotations = {
description = "Dovecot service is down on {{ $labels.instance }}";
summary = "Dovecot Service Down";
};
labels = {
severity = "critical";
};
}
{
uid = "openldap-service-down-alert-uid";
title = "OpenLDAP Service Down";
condition = "C";
data = [
{
refId = "A";
relativeTimeRange = {
from = 300;
to = 0;
};
datasourceUid = "vm-datasource-uid";
model = {
editorMode = "code";
expr = "node_systemd_unit_state{state=\"active\", name=\"openldap.service\"} OR on() vector(0)";
hide = false;
intervalMs = 1000;
legendFormat = "__auto";
maxDataPoints = 43200;
range = true;
refId = "A";
};
}
{
refId = "B";
datasourceUid = "__expr__";
model = {
type = "reduce";
expression = "A";
reducer = "last";
};
}
{
refId = "C";
datasourceUid = "__expr__";
model = {
type = "math";
expression = "$B < 1";
};
}
];
noDataState = "Alerting";
execErrState = "Alerting";
for = "5m";
annotations = {
description = "OpenLDAP service is down on {{ $labels.instance }}";
summary = "OpenLDAP Service Down";
};
labels = {
severity = "critical";
};
}
{
uid = "wireguard-service-down-alert-uid";
title = "WireGuard Service Down";
condition = "C";
data = [
{
refId = "A";
relativeTimeRange = {
from = 300;
to = 0;
};
datasourceUid = "vm-datasource-uid";
model = {
editorMode = "code";
expr = "node_systemd_unit_state{state=\"active\", name=\"wireguard-wg_cloonar.service\"} OR on() vector(0)";
hide = false;
intervalMs = 1000;
legendFormat = "__auto";
maxDataPoints = 43200;
range = true;
refId = "A";
};
}
{
refId = "B";
datasourceUid = "__expr__";
model = {
type = "reduce";
expression = "A";
reducer = "last";
};
}
{
refId = "C";
datasourceUid = "__expr__";
model = {
type = "math";
expression = "$B < 1";
};
}
];
noDataState = "Alerting";
execErrState = "Alerting";
for = "5m";
annotations = {
description = "WireGuard service is down on {{ $labels.instance }}";
summary = "WireGuard Service Down";
};
labels = {
severity = "critical";
};
}
];
}

57
hosts/web-arm/modules/grafana/alerting/service/wireguard_down.nix
View file

@ -1,57 +0,0 @@
{ lib, pkgs, config, ... }:
{
grafanaAlertRuleDefinitions = [
{
uid = "wireguard-service-down-alert-uid";
title = "WireGuard Service Down";
condition = "C";
data = [
{
refId = "A";
relativeTimeRange = {
from = 300;
to = 0;
};
datasourceUid = "vm-datasource-uid";
model = {
editorMode = "code";
expr = "node_systemd_unit_state{state=\"active\", name=\"wireguard-wg_cloonar.service\"} OR on() vector(0)";
hide = false;
intervalMs = 1000;
legendFormat = "__auto";
maxDataPoints = 43200;
range = true;
refId = "A";
};
}
{
refId = "B";
datasourceUid = "__expr__";
model = {
type = "reduce";
expression = "A";
reducer = "last";
};
}
{
refId = "C";
datasourceUid = "__expr__";
model = {
type = "math";
expression = "$B < 1";
};
}
];
noDataState = "Alerting";
execErrState = "Alerting";
for = "5m";
annotations = {
description = "WireGuard service is down on {{ $labels.instance }}";
summary = "WireGuard Service Down";
};
labels = {
severity = "critical";
};
}
];
}

10
hosts/web-arm/modules/grafana/alerting/system/default.nix
View file

@ -1,11 +1,11 @@
{ lib, pkgs, config, ... }: { lib, pkgs, config, ... }:
let let
# Import rule definitions from refactored alert files in the parent 'alerting' directory # Import rule definitions from refactored alert files in the parent 'alerting' directory
cpuAlertRules = (import ./cpu_usage.nix { inherit lib pkgs config; }).grafanaAlertRuleDefinitions; cpuAlertRules = (import ../cpu_usage.nix { inherit lib pkgs config; }).grafanaAlertRuleDefinitions;
diskAlertRules = (import ./disk_usage.nix { inherit lib pkgs config; }).grafanaAlertRuleDefinitions; diskAlertRules = (import ../disk_usage.nix { inherit lib pkgs config; }).grafanaAlertRuleDefinitions;
hostDownAlertRules = (import ./host_down.nix { inherit lib pkgs config; }).grafanaAlertRuleDefinitions; hostDownAlertRules = (import ../host_down.nix { inherit lib pkgs config; }).grafanaAlertRuleDefinitions;
inodeAlertRules = (import ./inode_usage.nix { inherit lib pkgs config; }).grafanaAlertRuleDefinitions; inodeAlertRules = (import ../inode_usage.nix { inherit lib pkgs config; }).grafanaAlertRuleDefinitions;
ramAlertRules = (import ./ram_usage.nix { inherit lib pkgs config; }).grafanaAlertRuleDefinitions; ramAlertRules = (import ../ram_usage.nix { inherit lib pkgs config; }).grafanaAlertRuleDefinitions;
allSystemRules = cpuAlertRules ++ diskAlertRules ++ hostDownAlertRules ++ inodeAlertRules ++ ramAlertRules; allSystemRules = cpuAlertRules ++ diskAlertRules ++ hostDownAlertRules ++ inodeAlertRules ++ ramAlertRules;
in in

17
hosts/web-arm/modules/grafana/datasources/loki.nix
View file

@ -1,17 +0,0 @@
{ lib, pkgs, config, ... }:
{
services.grafana.provision.datasources.settings.datasources = [
{
name = "Loki";
uid = "loki-datasource-uid"; # Stable UID
type = "loki";
url = "http://localhost:3100"; # Loki's local address
access = "proxy"; # Grafana proxies requests
isDefault = false; # Can be set to true if desired
jsonData = {
# Loki-specific options (e.g., derivedFields) can go here
};
editable = false; # Recommended for provisioned datasources
}
];
}

6
hosts/web-arm/modules/grafana/default.nix
View file

@ -29,11 +29,15 @@ in
{ {
imports = [ imports = [
# Individual alert files removed, now handled by alerting/system/default.nix # Individual alert files removed, now handled by alerting/system/default.nix
# ./alerting/disk_usage.nix
# ./alerting/cpu_usage.nix
# ./alerting/host_down.nix
# ./alerting/inode_usage.nix
# ./alerting/ram_usage.nix
./alerting/system/default.nix # Added: Imports the consolidated system alerts module ./alerting/system/default.nix # Added: Imports the consolidated system alerts module
./alerting/service/default.nix # Added: Imports the new service alerts module ./alerting/service/default.nix # Added: Imports the new service alerts module
# ... other rule files can be added here ... # ... other rule files can be added here ...
./datasources/victoriametrics.nix ./datasources/victoriametrics.nix
./datasources/loki.nix # Add Loki datasource
]; ];
systemd.services.grafana.script = lib.mkBefore '' systemd.services.grafana.script = lib.mkBefore ''