refactor: Grafana alerting rules: consolidate and reorganize alert definitions

- Deleted individual alert files for host down, inode usage, and RAM usage.
- Merged service down alerts into a new structure with separate files for each service (Gitea, Postfix, Dovecot, OpenLDAP, WireGuard).
- Introduced a new system alert structure consolidating CPU, disk, host down, inode, and RAM usage alerts.
- Updated alert conditions to use 'D' for thresholds and adjusted expressions accordingly.
- Improved annotations and labels for clarity and consistency across alerts.

hosts/web-arm/modules/grafana/alerting/service/default.nix

@@ -1,8 +1,18 @@
 { lib, pkgs, config, ... }:
 let
-  serviceDownAlertRules = (import ./service_down.nix { inherit lib pkgs config; }).grafanaAlertRuleDefinitions;
+  giteaDownAlertRules = (import ./gitea_down.nix { inherit lib pkgs config; }).grafanaAlertRuleDefinitions;
+  giteaRunnerDownAlertRules = (import ./gitea_runner_down.nix { inherit lib pkgs config; }).grafanaAlertRuleDefinitions;
+  postfixDownAlertRules = (import ./postfix_down.nix { inherit lib pkgs config; }).grafanaAlertRuleDefinitions;
+  dovecotDownAlertRules = (import ./dovecot_down.nix { inherit lib pkgs config; }).grafanaAlertRuleDefinitions;
+  openldapDownAlertRules = (import ./openldap_down.nix { inherit lib pkgs config; }).grafanaAlertRuleDefinitions;
+  wireguardDownAlertRules = (import ./wireguard_down.nix { inherit lib pkgs config; }).grafanaAlertRuleDefinitions;
 
-  allServiceRules = serviceDownAlertRules;
+  allServiceRules = giteaDownAlertRules
+                   ++ giteaRunnerDownAlertRules
+                   ++ postfixDownAlertRules
+                   ++ dovecotDownAlertRules
+                   ++ openldapDownAlertRules
+                   ++ wireguardDownAlertRules;
 in
 {
   services.grafana.provision.alerting.rules.settings.groups = [

hosts/web-arm/modules/grafana/alerting/service/dovecot_down.nix

@@ -0,0 +1,57 @@
+{ lib, pkgs, config, ... }:
+{
+  grafanaAlertRuleDefinitions = [
+    {
+      uid = "dovecot-service-down-alert-uid";
+      title = "Dovecot Service Down";
+      condition = "C";
+      data = [
+        {
+          refId = "A";
+          relativeTimeRange = {
+            from = 300;
+            to = 0;
+          };
+          datasourceUid = "vm-datasource-uid";
+          model = {
+            editorMode = "code";
+            expr = "node_systemd_unit_state{state=\"active\", name=\"dovecot2.service\"} OR on() vector(0)";
+            hide = false;
+            intervalMs = 1000;
+            legendFormat = "__auto";
+            maxDataPoints = 43200;
+            range = true;
+            refId = "A";
+          };
+        }
+        {
+          refId = "B";
+          datasourceUid = "__expr__";
+          model = {
+            type = "reduce";
+            expression = "A";
+            reducer = "last";
+          };
+        }
+        {
+          refId = "C";
+          datasourceUid = "__expr__";
+          model = {
+            type = "math";
+            expression = "$B < 1";
+          };
+        }
+      ];
+      noDataState = "Alerting";
+      execErrState = "Alerting";
+      for = "5m";
+      annotations = {
+        description = "Dovecot service is down on {{ $labels.instance }}";
+        summary = "Dovecot Service Down";
+      };
+      labels = {
+        severity = "critical";
+      };
+    }
+  ];
+}

hosts/web-arm/modules/grafana/alerting/service/gitea_down.nix

@@ -0,0 +1,57 @@
+{ lib, pkgs, config, ... }:
+{
+  grafanaAlertRuleDefinitions = [
+    {
+      uid = "gitea-service-down-alert-uid";
+      title = "Gitea Service Down";
+      condition = "C";
+      data = [
+        {
+          refId = "A";
+          relativeTimeRange = {
+            from = 300;
+            to = 0;
+          };
+          datasourceUid = "vm-datasource-uid";
+          model = {
+            editorMode = "code";
+            expr = "node_systemd_unit_state{state=\"active\", name=\"container@git.service\"} OR on() vector(0)";
+            hide = false;
+            intervalMs = 1000;
+            legendFormat = "__auto";
+            maxDataPoints = 43200;
+            range = true;
+            refId = "A";
+          };
+        }
+        {
+          refId = "B";
+          datasourceUid = "__expr__";
+          model = {
+            type = "reduce";
+            expression = "A";
+            reducer = "last";
+          };
+        }
+        {
+          refId = "C";
+          datasourceUid = "__expr__";
+          model = {
+            type = "math";
+            expression = "$B < 1";
+          };
+        }
+      ];
+      noDataState = "Alerting";
+      execErrState = "Alerting";
+      for = "5m";
+      annotations = {
+        description = "Gitea service is down on {{ $labels.instance }}";
+        summary = "Gitea Service Down";
+      };
+      labels = {
+        severity = "critical";
+      };
+    }
+  ];
+}

hosts/web-arm/modules/grafana/alerting/service/gitea_runner_down.nix

@@ -0,0 +1,57 @@
+{ lib, pkgs, config, ... }:
+{
+  grafanaAlertRuleDefinitions = [
+    {
+      uid = "gitea-runner-service-down-alert-uid";
+      title = "Gitea Runner Service Down";
+      condition = "C";
+      data = [
+        {
+          refId = "A";
+          relativeTimeRange = {
+            from = 300;
+            to = 0;
+          };
+          datasourceUid = "vm-datasource-uid";
+          model = {
+            editorMode = "code";
+            expr = "node_systemd_unit_state{state=\"active\", name=\"microvm@git-runner-1.service\"} OR on() vector(0)";
+            hide = false;
+            intervalMs = 1000;
+            legendFormat = "__auto";
+            maxDataPoints = 43200;
+            range = true;
+            refId = "A";
+          };
+        }
+        {
+          refId = "B";
+          datasourceUid = "__expr__";
+          model = {
+            type = "reduce";
+            expression = "A";
+            reducer = "last";
+          };
+        }
+        {
+          refId = "C";
+          datasourceUid = "__expr__";
+          model = {
+            type = "math";
+            expression = "$B < 1";
+          };
+        }
+      ];
+      noDataState = "Alerting";
+      execErrState = "Alerting";
+      for = "5m";
+      annotations = {
+        description = "Gitea Runner service is down on {{ $labels.instance }}";
+        summary = "Gitea Runner Service Down";
+      };
+      labels = {
+        severity = "critical";
+      };
+    }
+  ];
+}

hosts/web-arm/modules/grafana/alerting/service/openldap_down.nix

@@ -0,0 +1,57 @@
+{ lib, pkgs, config, ... }:
+{
+  grafanaAlertRuleDefinitions = [
+    {
+      uid = "openldap-service-down-alert-uid";
+      title = "OpenLDAP Service Down";
+      condition = "C";
+      data = [
+        {
+          refId = "A";
+          relativeTimeRange = {
+            from = 300;
+            to = 0;
+          };
+          datasourceUid = "vm-datasource-uid";
+          model = {
+            editorMode = "code";
+            expr = "node_systemd_unit_state{state=\"active\", name=\"openldap.service\"} OR on() vector(0)";
+            hide = false;
+            intervalMs = 1000;
+            legendFormat = "__auto";
+            maxDataPoints = 43200;
+            range = true;
+            refId = "A";
+          };
+        }
+        {
+          refId = "B";
+          datasourceUid = "__expr__";
+          model = {
+            type = "reduce";
+            expression = "A";
+            reducer = "last";
+          };
+        }
+        {
+          refId = "C";
+          datasourceUid = "__expr__";
+          model = {
+            type = "math";
+            expression = "$B < 1";
+          };
+        }
+      ];
+      noDataState = "Alerting";
+      execErrState = "Alerting";
+      for = "5m";
+      annotations = {
+        description = "OpenLDAP service is down on {{ $labels.instance }}";
+        summary = "OpenLDAP Service Down";
+      };
+      labels = {
+        severity = "critical";
+      };
+    }
+  ];
+}

hosts/web-arm/modules/grafana/alerting/service/postfix_down.nix

@@ -0,0 +1,57 @@
+{ lib, pkgs, config, ... }:
+{
+  grafanaAlertRuleDefinitions = [
+    {
+      uid = "postfix-service-down-alert-uid";
+      title = "Postfix Service Down";
+      condition = "C";
+      data = [
+        {
+          refId = "A";
+          relativeTimeRange = {
+            from = 300;
+            to = 0;
+          };
+          datasourceUid = "vm-datasource-uid";
+          model = {
+            editorMode = "code";
+            expr = "node_systemd_unit_state{state=\"active\", name=\"postfix.service\"} OR on() vector(0)";
+            hide = false;
+            intervalMs = 1000;
+            legendFormat = "__auto";
+            maxDataPoints = 43200;
+            range = true;
+            refId = "A";
+          };
+        }
+        {
+          refId = "B";
+          datasourceUid = "__expr__";
+          model = {
+            type = "reduce";
+            expression = "A";
+            reducer = "last";
+          };
+        }
+        {
+          refId = "C";
+          datasourceUid = "__expr__";
+          model = {
+            type = "math";
+            expression = "$B < 1";
+          };
+        }
+      ];
+      noDataState = "Alerting";
+      execErrState = "Alerting";
+      for = "5m";
+      annotations = {
+        description = "Postfix service is down on {{ $labels.instance }}";
+        summary = "Postfix Service Down";
+      };
+      labels = {
+        severity = "critical";
+      };
+    }
+  ];
+}

hosts/web-arm/modules/grafana/alerting/service/service_down.nix

@@ -1,318 +0,0 @@
-{ lib, pkgs, config, ... }:
-{
-  grafanaAlertRuleDefinitions = [
-    # Systemd service monitoring alerts
-    {
-      uid = "gitea-service-down-alert-uid";
-      title = "Gitea Service Down";
-      condition = "C";
-      data = [
-        {
-          refId = "A";
-          relativeTimeRange = {
-            from = 300;
-            to = 0;
-          };
-          datasourceUid = "vm-datasource-uid";
-          model = {
-            editorMode = "code";
-            expr = "node_systemd_unit_state{state=\"active\", name=\"container@git.service\"} OR on() vector(0)";
-            hide = false;
-            intervalMs = 1000;
-            legendFormat = "__auto";
-            maxDataPoints = 43200;
-            range = true;
-            refId = "A";
-          };
-        }
-        {
-          refId = "B";
-          datasourceUid = "__expr__";
-          model = {
-            type = "reduce";
-            expression = "A";
-            reducer = "last";
-          };
-        }
-        {
-          refId = "C";
-          datasourceUid = "__expr__";
-          model = {
-            type = "math";
-            expression = "$B < 1";
-          };
-        }
-      ];
-      noDataState = "Alerting";
-      execErrState = "Alerting";
-      for = "5m";
-      annotations = {
-        description = "Gitea service is down on {{ $labels.instance }}";
-        summary = "Gitea Service Down";
-      };
-      labels = {
-        severity = "critical";
-      };
-    }
-    {
-      uid = "gitea-runner-service-down-alert-uid";
-      title = "Gitea Runner Service Down";
-      condition = "C";
-      data = [
-        {
-          refId = "A";
-          relativeTimeRange = {
-            from = 300;
-            to = 0;
-          };
-          datasourceUid = "vm-datasource-uid";
-          model = {
-            editorMode = "code";
-            expr = "node_systemd_unit_state{state=\"active\", name=\"microvm@git-runner-1.service\"} OR on() vector(0)";
-            hide = false;
-            intervalMs = 1000;
-            legendFormat = "__auto";
-            maxDataPoints = 43200;
-            range = true;
-            refId = "A";
-          };
-        }
-        {
-          refId = "B";
-          datasourceUid = "__expr__";
-          model = {
-            type = "reduce";
-            expression = "A";
-            reducer = "last";
-          };
-        }
-        {
-          refId = "C";
-          datasourceUid = "__expr__";
-          model = {
-            type = "math";
-            expression = "$B < 1";
-          };
-        }
-      ];
-      noDataState = "Alerting";
-      execErrState = "Alerting";
-      for = "5m";
-      annotations = {
-        description = "Gitea Runner service is down on {{ $labels.instance }}";
-        summary = "Gitea Runner Service Down";
-      };
-      labels = {
-        severity = "critical";
-      };
-    }
-    {
-      uid = "postfix-service-down-alert-uid";
-      title = "Postfix Service Down";
-      condition = "C";
-      data = [
-        {
-          refId = "A";
-          relativeTimeRange = {
-            from = 300;
-            to = 0;
-          };
-          datasourceUid = "vm-datasource-uid";
-          model = {
-            editorMode = "code";
-            expr = "node_systemd_unit_state{state=\"active\", name=\"postfix.service\"} OR on() vector(0)";
-            hide = false;
-            intervalMs = 1000;
-            legendFormat = "__auto";
-            maxDataPoints = 43200;
-            range = true;
-            refId = "A";
-          };
-        }
-        {
-          refId = "B";
-          datasourceUid = "__expr__";
-          model = {
-            type = "reduce";
-            expression = "A";
-            reducer = "last";
-          };
-        }
-        {
-          refId = "C";
-          datasourceUid = "__expr__";
-          model = {
-            type = "math";
-            expression = "$B < 1";
-          };
-        }
-      ];
-      noDataState = "Alerting";
-      execErrState = "Alerting";
-      for = "5m";
-      annotations = {
-        description = "Postfix service is down on {{ $labels.instance }}";
-        summary = "Postfix Service Down";
-      };
-      labels = {
-        severity = "critical";
-      };
-    }
-    {
-      uid = "dovecot-service-down-alert-uid";
-      title = "Dovecot Service Down";
-      condition = "C";
-      data = [
-        {
-          refId = "A";
-          relativeTimeRange = {
-            from = 300;
-            to = 0;
-          };
-          datasourceUid = "vm-datasource-uid";
-          model = {
-            editorMode = "code";
-            expr = "node_systemd_unit_state{state=\"active\", name=\"dovecot2.service\"} OR on() vector(0)";
-            hide = false;
-            intervalMs = 1000;
-            legendFormat = "__auto";
-            maxDataPoints = 43200;
-            range = true;
-            refId = "A";
-          };
-        }
-        {
-          refId = "B";
-          datasourceUid = "__expr__";
-          model = {
-            type = "reduce";
-            expression = "A";
-            reducer = "last";
-          };
-        }
-        {
-          refId = "C";
-          datasourceUid = "__expr__";
-          model = {
-            type = "math";
-            expression = "$B < 1";
-          };
-        }
-      ];
-      noDataState = "Alerting";
-      execErrState = "Alerting";
-      for = "5m";
-      annotations = {
-        description = "Dovecot service is down on {{ $labels.instance }}";
-        summary = "Dovecot Service Down";
-      };
-      labels = {
-        severity = "critical";
-      };
-    }
-    {
-      uid = "openldap-service-down-alert-uid";
-      title = "OpenLDAP Service Down";
-      condition = "C";
-      data = [
-        {
-          refId = "A";
-          relativeTimeRange = {
-            from = 300;
-            to = 0;
-          };
-          datasourceUid = "vm-datasource-uid";
-          model = {
-            editorMode = "code";
-            expr = "node_systemd_unit_state{state=\"active\", name=\"openldap.service\"} OR on() vector(0)";
-            hide = false;
-            intervalMs = 1000;
-            legendFormat = "__auto";
-            maxDataPoints = 43200;
-            range = true;
-            refId = "A";
-          };
-        }
-        {
-          refId = "B";
-          datasourceUid = "__expr__";
-          model = {
-            type = "reduce";
-            expression = "A";
-            reducer = "last";
-          };
-        }
-        {
-          refId = "C";
-          datasourceUid = "__expr__";
-          model = {
-            type = "math";
-            expression = "$B < 1";
-          };
-        }
-      ];
-      noDataState = "Alerting";
-      execErrState = "Alerting";
-      for = "5m";
-      annotations = {
-        description = "OpenLDAP service is down on {{ $labels.instance }}";
-        summary = "OpenLDAP Service Down";
-      };
-      labels = {
-        severity = "critical";
-      };
-    }
-    {
-      uid = "wireguard-service-down-alert-uid";
-      title = "WireGuard Service Down";
-      condition = "C";
-      data = [
-        {
-          refId = "A";
-          relativeTimeRange = {
-            from = 300;
-            to = 0;
-          };
-          datasourceUid = "vm-datasource-uid";
-          model = {
-            editorMode = "code";
-            expr = "node_systemd_unit_state{state=\"active\", name=\"wireguard-wg_cloonar.service\"} OR on() vector(0)";
-            hide = false;
-            intervalMs = 1000;
-            legendFormat = "__auto";
-            maxDataPoints = 43200;
-            range = true;
-            refId = "A";
-          };
-        }
-        {
-          refId = "B";
-          datasourceUid = "__expr__";
-          model = {
-            type = "reduce";
-            expression = "A";
-            reducer = "last";
-          };
-        }
-        {
-          refId = "C";
-          datasourceUid = "__expr__";
-          model = {
-            type = "math";
-            expression = "$B < 1";
-          };
-        }
-      ];
-      noDataState = "Alerting";
-      execErrState = "Alerting";
-      for = "5m";
-      annotations = {
-        description = "WireGuard service is down on {{ $labels.instance }}";
-        summary = "WireGuard Service Down";
-      };
-      labels = {
-        severity = "critical";
-      };
-    }
-  ];
-}

hosts/web-arm/modules/grafana/alerting/service/wireguard_down.nix

@@ -0,0 +1,57 @@
+{ lib, pkgs, config, ... }:
+{
+  grafanaAlertRuleDefinitions = [
+    {
+      uid = "wireguard-service-down-alert-uid";
+      title = "WireGuard Service Down";
+      condition = "C";
+      data = [
+        {
+          refId = "A";
+          relativeTimeRange = {
+            from = 300;
+            to = 0;
+          };
+          datasourceUid = "vm-datasource-uid";
+          model = {
+            editorMode = "code";
+            expr = "node_systemd_unit_state{state=\"active\", name=\"wireguard-wg_cloonar.service\"} OR on() vector(0)";
+            hide = false;
+            intervalMs = 1000;
+            legendFormat = "__auto";
+            maxDataPoints = 43200;
+            range = true;
+            refId = "A";
+          };
+        }
+        {
+          refId = "B";
+          datasourceUid = "__expr__";
+          model = {
+            type = "reduce";
+            expression = "A";
+            reducer = "last";
+          };
+        }
+        {
+          refId = "C";
+          datasourceUid = "__expr__";
+          model = {
+            type = "math";
+            expression = "$B < 1";
+          };
+        }
+      ];
+      noDataState = "Alerting";
+      execErrState = "Alerting";
+      for = "5m";
+      annotations = {
+        description = "WireGuard service is down on {{ $labels.instance }}";
+        summary = "WireGuard Service Down";
+      };
+      labels = {
+        severity = "critical";
+      };
+    }
+  ];
+}

hosts/web-arm/modules/grafana/alerting/system/default.nix

@@ -1,11 +1,11 @@
 { lib, pkgs, config, ... }:
 let
   # Import rule definitions from refactored alert files in the parent 'alerting' directory
-  cpuAlertRules = (import ../cpu_usage.nix { inherit lib pkgs config; }).grafanaAlertRuleDefinitions;
-  diskAlertRules = (import ../disk_usage.nix { inherit lib pkgs config; }).grafanaAlertRuleDefinitions;
-  hostDownAlertRules = (import ../host_down.nix { inherit lib pkgs config; }).grafanaAlertRuleDefinitions;
-  inodeAlertRules = (import ../inode_usage.nix { inherit lib pkgs config; }).grafanaAlertRuleDefinitions;
-  ramAlertRules = (import ../ram_usage.nix { inherit lib pkgs config; }).grafanaAlertRuleDefinitions;
+  cpuAlertRules = (import ./cpu_usage.nix { inherit lib pkgs config; }).grafanaAlertRuleDefinitions;
+  diskAlertRules = (import ./disk_usage.nix { inherit lib pkgs config; }).grafanaAlertRuleDefinitions;
+  hostDownAlertRules = (import ./host_down.nix { inherit lib pkgs config; }).grafanaAlertRuleDefinitions;
+  inodeAlertRules = (import ./inode_usage.nix { inherit lib pkgs config; }).grafanaAlertRuleDefinitions;
+  ramAlertRules = (import ./ram_usage.nix { inherit lib pkgs config; }).grafanaAlertRuleDefinitions;
 
   allSystemRules = cpuAlertRules ++ diskAlertRules ++ hostDownAlertRules ++ inodeAlertRules ++ ramAlertRules;
 in

hosts/web-arm/modules/grafana/datasources/loki.nix

@@ -0,0 +1,17 @@
+{ lib, pkgs, config, ... }:
+{
+  services.grafana.provision.datasources.settings.datasources = [
+    {
+      name = "Loki";
+      uid = "loki-datasource-uid"; # Stable UID
+      type = "loki";
+      url = "http://localhost:3100"; # Loki's local address
+      access = "proxy"; # Grafana proxies requests
+      isDefault = false; # Can be set to true if desired
+      jsonData = {
+        # Loki-specific options (e.g., derivedFields) can go here
+      };
+      editable = false; # Recommended for provisioned datasources
+    }
+  ];
+}

hosts/web-arm/modules/grafana/default.nix

@@ -29,15 +29,11 @@ in
 {
   imports = [
     # Individual alert files removed, now handled by alerting/system/default.nix
-    # ./alerting/disk_usage.nix
-    # ./alerting/cpu_usage.nix
-    # ./alerting/host_down.nix
-    # ./alerting/inode_usage.nix
-    # ./alerting/ram_usage.nix
     ./alerting/system/default.nix # Added: Imports the consolidated system alerts module
     ./alerting/service/default.nix # Added: Imports the new service alerts module
     # ... other rule files can be added here ...
     ./datasources/victoriametrics.nix
+    ./datasources/loki.nix # Add Loki datasource
   ];
 
   systemd.services.grafana.script = lib.mkBefore ''