feat: implement centralized alerting with vmalert and Grafana, add alert rules for CPU, disk, inode, RAM usage, and host status

hosts/web-arm/modules/grafana.nix

@@ -89,32 +89,73 @@ in
     };
     provision = {
       alerting = {
-        contactPoints.settings = {
-          apiVersion = 1;
-
-          contactPoints = [{
-            orgId = 1;
-            name = "cp_dominik";
-            receivers = [{
-              uid = "dominik";
-              type = "pushover";
-              settings = {
-                security.apiToken = "$__file{${config.sops.secrets.pushover-api-token.path}}";
-                security.userKey = "$__file{${config.sops.secrets.pushover-user-key.path}}";
-                apiToken = "\${PUSHOVER_API_TOKEN}";
-                userKey = "\${PUSHOVER_USER_KEY}";
-                device = "iphone";
-                priority = "2";
-                retry = "30";
-                expire = "120";
-                sound = "siren";
-                okSound = "magic";
-                message = ''
-                  {{ template "default.message" . }}
-                '';
-              };
+        contactPoints = {
+          settings = {
+            apiVersion = 1; # As per Grafana provisioning API
+            contactPoints = [{
+              orgId = 1;
+              name = "cp_dominik";
+              receivers = [{
+                uid = "dominik_pushover_cp_receiver"; # Made UID even more specific
+                type = "pushover";
+                settings = {
+                  apiToken = "\${PUSHOVER_API_TOKEN}";
+                  userKey = "\${PUSHOVER_USER_KEY}";
+                  device = "iphone";
+                  priority = 2;
+                  retry = "30s";
+                  expire = "2m";
+                  sound = "siren";
+                  okSound = "magic";
+                  message = ''
+                    {{ template "default.message" . }}
+                  '';
+                };
+              }];
             }];
-          }];
+          };
+        };
+
+        policies = { # Corrected from notificationPolicies to policies
+          settings = {
+            apiVersion = 1; # As per Grafana provisioning API
+
+            # Grafana's new unified alerting expects a single policy tree per org.
+            # For OrgID 1 (default), this defines the root of that tree.
+            # The NixOS module should translate this into the correct YAML structure.
+            # The `policies` attribute within `settings` usually takes a list of policy trees.
+            # For a single default organization, we define one policy tree.
+            # Grafana's own YAML examples show a top-level 'route' for the default policy,
+            # or a list under 'policies' if you're managing multiple policy sets (less common for basic setup).
+            # Given the NixOS option `services.grafana.provision.alerting.policies.settings.policies`,
+            # it's likely expecting a list here.
+            policies = [{ # This outer list corresponds to the `policies` option
+              # orgId = 1; # Usually implicit for the default policy file, but can be specified
+              receiver = "cp_dominik"; # This sets the default receiver for the root route
+
+              # The actual routing tree starts here.
+              # For a simple setup where all alerts go to one receiver,
+              # just setting the top-level 'receiver' is often enough.
+              # If more complex routing is needed, 'routes' would be defined here.
+              # Example:
+              # route = {
+              #   receiver = "cp_dominik";
+              #   group_by = [ "alertname", "job" ];
+              #   # ... other root route settings
+              #   routes = [
+              #     {
+              #       matcher_re = { severity = "critical" };
+              #       receiver = "critical_alerts_receiver"; # Another contact point
+              #       continue = false;
+              #     },
+              #     # ... other specific routes
+              #   ];
+              # };
+              # For the simplest case, just defining the receiver at this level should work
+              # as the root policy for the default organization.
+            }];
+            # resetPolicies = false; # Default, set to true to remove existing policies not in this config.
+          };
         };
       };
     };