feat: add VictoriaMetrics module, update Dovecot Sieve extensions, and fix Grafana service expression

hosts/fw/configuration.nix

@@ -26,6 +26,7 @@
     ./modules/podman.nix
     ./modules/omada.nix
     ./modules/ddclient.nix
+    ./utils/modules/victoriametrics
     # ./modules/wol.nix
 
 
@@ -42,7 +43,7 @@
 
     # git
     ./modules/gitea.nix
-    ./modules/fwmetrics.nix
+    # ./modules/fwmetrics.nix
 
     # ha customers
     ./modules/ha-customers

hosts/mail/modules/dovecot.nix

@@ -99,6 +99,9 @@ in
     mailLocation = "maildir:/var/vmail/%d/%n/Maildir";
     mailUser = "vmail";
     mailGroup = "vmail";
+
+    sieve.extensions = [ "copy" "editheader" "relational" "reject" "fileinto" "imap4flags" "vacation-seconds" ];
+
     extraConfig = ''
       ssl = yes
       ssl_cert = </var/lib/acme/imap.${domain}/fullchain.pem

hosts/web-arm/modules/grafana/alerting/service/default.nix

@@ -0,0 +1,16 @@
+{ lib, pkgs, config, ... }:
+let
+  serviceDownAlertRules = (import ./service_down.nix { inherit lib pkgs config; }).grafanaAlertRuleDefinitions;
+
+  allServiceRules = serviceDownAlertRules;
+in
+{
+  services.grafana.provision.alerting.rules.settings.groups = [
+    {
+      name = "Service Alerts";
+      folder = "Service Monitoring";
+      interval = "1m";
+      rules = allServiceRules;
+    }
+  ];
+}

hosts/web-arm/modules/grafana/alerting/service/service_down.nix

@@ -0,0 +1,318 @@
+{ lib, pkgs, config, ... }:
+{
+  grafanaAlertRuleDefinitions = [
+    # Systemd service monitoring alerts
+    {
+      uid = "gitea-service-down-alert-uid";
+      title = "Gitea Service Down";
+      condition = "C";
+      data = [
+        {
+          refId = "A";
+          relativeTimeRange = {
+            from = 300;
+            to = 0;
+          };
+          datasourceUid = "vm-datasource-uid";
+          model = {
+            editorMode = "code";
+            expr = "node_systemd_unit_state{state=\"active\", name=\"container@git.service\"} OR on() vector(0)";
+            hide = false;
+            intervalMs = 1000;
+            legendFormat = "__auto";
+            maxDataPoints = 43200;
+            range = true;
+            refId = "A";
+          };
+        }
+        {
+          refId = "B";
+          datasourceUid = "__expr__";
+          model = {
+            type = "reduce";
+            expression = "A";
+            reducer = "last";
+          };
+        }
+        {
+          refId = "C";
+          datasourceUid = "__expr__";
+          model = {
+            type = "math";
+            expression = "$B < 1";
+          };
+        }
+      ];
+      noDataState = "Alerting";
+      execErrState = "Alerting";
+      for = "5m";
+      annotations = {
+        description = "Gitea service is down on {{ $labels.instance }}";
+        summary = "Gitea Service Down";
+      };
+      labels = {
+        severity = "critical";
+      };
+    }
+    {
+      uid = "gitea-runner-service-down-alert-uid";
+      title = "Gitea Runner Service Down";
+      condition = "C";
+      data = [
+        {
+          refId = "A";
+          relativeTimeRange = {
+            from = 300;
+            to = 0;
+          };
+          datasourceUid = "vm-datasource-uid";
+          model = {
+            editorMode = "code";
+            expr = "node_systemd_unit_state{state=\"active\", name=\"microvm@git-runner-1.service\"} OR on() vector(0)";
+            hide = false;
+            intervalMs = 1000;
+            legendFormat = "__auto";
+            maxDataPoints = 43200;
+            range = true;
+            refId = "A";
+          };
+        }
+        {
+          refId = "B";
+          datasourceUid = "__expr__";
+          model = {
+            type = "reduce";
+            expression = "A";
+            reducer = "last";
+          };
+        }
+        {
+          refId = "C";
+          datasourceUid = "__expr__";
+          model = {
+            type = "math";
+            expression = "$B < 1";
+          };
+        }
+      ];
+      noDataState = "Alerting";
+      execErrState = "Alerting";
+      for = "5m";
+      annotations = {
+        description = "Gitea Runner service is down on {{ $labels.instance }}";
+        summary = "Gitea Runner Service Down";
+      };
+      labels = {
+        severity = "critical";
+      };
+    }
+    {
+      uid = "postfix-service-down-alert-uid";
+      title = "Postfix Service Down";
+      condition = "C";
+      data = [
+        {
+          refId = "A";
+          relativeTimeRange = {
+            from = 300;
+            to = 0;
+          };
+          datasourceUid = "vm-datasource-uid";
+          model = {
+            editorMode = "code";
+            expr = "node_systemd_unit_state{state=\"active\", name=\"postfix.service\"} OR on() vector(0)";
+            hide = false;
+            intervalMs = 1000;
+            legendFormat = "__auto";
+            maxDataPoints = 43200;
+            range = true;
+            refId = "A";
+          };
+        }
+        {
+          refId = "B";
+          datasourceUid = "__expr__";
+          model = {
+            type = "reduce";
+            expression = "A";
+            reducer = "last";
+          };
+        }
+        {
+          refId = "C";
+          datasourceUid = "__expr__";
+          model = {
+            type = "math";
+            expression = "$B < 1";
+          };
+        }
+      ];
+      noDataState = "Alerting";
+      execErrState = "Alerting";
+      for = "5m";
+      annotations = {
+        description = "Postfix service is down on {{ $labels.instance }}";
+        summary = "Postfix Service Down";
+      };
+      labels = {
+        severity = "critical";
+      };
+    }
+    {
+      uid = "dovecot-service-down-alert-uid";
+      title = "Dovecot Service Down";
+      condition = "C";
+      data = [
+        {
+          refId = "A";
+          relativeTimeRange = {
+            from = 300;
+            to = 0;
+          };
+          datasourceUid = "vm-datasource-uid";
+          model = {
+            editorMode = "code";
+            expr = "node_systemd_unit_state{state=\"active\", name=\"dovecot2.service\"} OR on() vector(0)";
+            hide = false;
+            intervalMs = 1000;
+            legendFormat = "__auto";
+            maxDataPoints = 43200;
+            range = true;
+            refId = "A";
+          };
+        }
+        {
+          refId = "B";
+          datasourceUid = "__expr__";
+          model = {
+            type = "reduce";
+            expression = "A";
+            reducer = "last";
+          };
+        }
+        {
+          refId = "C";
+          datasourceUid = "__expr__";
+          model = {
+            type = "math";
+            expression = "$B < 1";
+          };
+        }
+      ];
+      noDataState = "Alerting";
+      execErrState = "Alerting";
+      for = "5m";
+      annotations = {
+        description = "Dovecot service is down on {{ $labels.instance }}";
+        summary = "Dovecot Service Down";
+      };
+      labels = {
+        severity = "critical";
+      };
+    }
+    {
+      uid = "openldap-service-down-alert-uid";
+      title = "OpenLDAP Service Down";
+      condition = "C";
+      data = [
+        {
+          refId = "A";
+          relativeTimeRange = {
+            from = 300;
+            to = 0;
+          };
+          datasourceUid = "vm-datasource-uid";
+          model = {
+            editorMode = "code";
+            expr = "node_systemd_unit_state{state=\"active\", name=\"openldap.service\"} OR on() vector(0)";
+            hide = false;
+            intervalMs = 1000;
+            legendFormat = "__auto";
+            maxDataPoints = 43200;
+            range = true;
+            refId = "A";
+          };
+        }
+        {
+          refId = "B";
+          datasourceUid = "__expr__";
+          model = {
+            type = "reduce";
+            expression = "A";
+            reducer = "last";
+          };
+        }
+        {
+          refId = "C";
+          datasourceUid = "__expr__";
+          model = {
+            type = "math";
+            expression = "$B < 1";
+          };
+        }
+      ];
+      noDataState = "Alerting";
+      execErrState = "Alerting";
+      for = "5m";
+      annotations = {
+        description = "OpenLDAP service is down on {{ $labels.instance }}";
+        summary = "OpenLDAP Service Down";
+      };
+      labels = {
+        severity = "critical";
+      };
+    }
+    {
+      uid = "wireguard-service-down-alert-uid";
+      title = "WireGuard Service Down";
+      condition = "C";
+      data = [
+        {
+          refId = "A";
+          relativeTimeRange = {
+            from = 300;
+            to = 0;
+          };
+          datasourceUid = "vm-datasource-uid";
+          model = {
+            editorMode = "code";
+            expr = "node_systemd_unit_state{state=\"active\", name=\"wireguard-wg_cloonar.service\"} OR on() vector(0)";
+            hide = false;
+            intervalMs = 1000;
+            legendFormat = "__auto";
+            maxDataPoints = 43200;
+            range = true;
+            refId = "A";
+          };
+        }
+        {
+          refId = "B";
+          datasourceUid = "__expr__";
+          model = {
+            type = "reduce";
+            expression = "A";
+            reducer = "last";
+          };
+        }
+        {
+          refId = "C";
+          datasourceUid = "__expr__";
+          model = {
+            type = "math";
+            expression = "$B < 1";
+          };
+        }
+      ];
+      noDataState = "Alerting";
+      execErrState = "Alerting";
+      for = "5m";
+      annotations = {
+        description = "WireGuard service is down on {{ $labels.instance }}";
+        summary = "WireGuard Service Down";
+      };
+      labels = {
+        severity = "critical";
+      };
+    }
+  ];
+}

hosts/web-arm/modules/grafana/default.nix

@@ -35,6 +35,7 @@ in
     # ./alerting/inode_usage.nix
     # ./alerting/ram_usage.nix
     ./alerting/system/default.nix # Added: Imports the consolidated system alerts module
+    ./alerting/service/default.nix # Added: Imports the new service alerts module
     # ... other rule files can be added here ...
     ./datasources/victoriametrics.nix
   ];

utils/modules/victoriametrics/default.nix

@@ -2,18 +2,45 @@
 let
   configure_prom = builtins.toFile "prometheus.yml" ''
     scrape_configs:
-    - job_name: 'server'
+    # System metrics
+    - job_name: 'node'
       stream_parse: true
       static_configs:
       - targets:
         - ${config.networking.hostName}:9100
+
+    # Systemd service monitoring
+    - job_name: 'systemd'
+      metrics_path: /metrics
+      params:
+        collect[]:
+          - 'systemd.service.state'
+          - 'systemd.service.start_time_seconds'
+          - 'systemd.unit_file.state'
+      static_configs:
+      - targets:
+        - ${config.networking.hostName}:9100
+      relabel_configs:
+        # Filter for specific services we want to monitor
+        - source_labels: [__name__]
+          regex: 'node_systemd_unit_state'
+          action: keep
+        - source_labels: [name]
+          regex: '(container@git|microvm@git-runner-|postfix|dovecot|openldap|wireguard-wg_cloonar).*\.service'
+          action: keep
   '';
 in {
   sops.secrets.victoria-agent-env = {
     sopsFile = ./secrets.yaml;
   };
 
-  services.prometheus.exporters.node.enable = true;
+  # Node exporter for system metrics
+  services.prometheus.exporters.node = {
+    enable = true;
+    enabledCollectors = [
+      "systemd"  # Enable systemd collector for service monitoring
+    ];
+  };
   
   systemd.services.export-to-prometheus = {
     path = with pkgs; [victoriametrics];