feat: add service monitoring alerts for Gitea, Postfix, Dovecot, OpenLDAP, and WireGuard, and consolidate alerting rules in Grafana

utils/modules/victoriametrics/default.nix

@@ -2,18 +2,45 @@
 let
   configure_prom = builtins.toFile "prometheus.yml" ''
     scrape_configs:
-    - job_name: 'server'
+    # System metrics
+    - job_name: 'node'
       stream_parse: true
       static_configs:
       - targets:
         - ${config.networking.hostName}:9100
+
+    # Systemd service monitoring
+    - job_name: 'systemd'
+      metrics_path: /metrics
+      params:
+        collect[]:
+          - 'systemd.service.state'
+          - 'systemd.service.start_time_seconds'
+          - 'systemd.unit_file.state'
+      static_configs:
+      - targets:
+        - ${config.networking.hostName}:9100
+      relabel_configs:
+        # Filter for specific services we want to monitor
+        - source_labels: [__name__]
+          regex: 'node_systemd_unit_state'
+          action: keep
+        - source_labels: [name]
+          regex: '(container@git|microvm@git-runner-1|postfix|dovecot|openldap|wireguard-wg_cloonar).*\.service'
+          action: keep
   '';
 in {
   sops.secrets.victoria-agent-env = {
     sopsFile = ./secrets.yaml;
   };
 
-  services.prometheus.exporters.node.enable = true;
+  # Node exporter for system metrics
+  services.prometheus.exporters.node = {
+    enable = true;
+    enabledCollectors = [
+      "systemd"  # Enable systemd collector for service monitoring
+    ];
+  };
   
   systemd.services.export-to-prometheus = {
     path = with pkgs; [victoriametrics];