feat: implement website alerting plan with Blackbox Exporter and VictoriaMetrics integration

hosts/web-arm/modules/victoriametrics.nix

@@ -1,43 +1,84 @@
-{ config, ... }:
+{ config, lib, ... }:
+with lib;
 let
+  # configure_prom = builtins.toFile "prometheus.yml" ''
+  #   scrape_configs:
+  #   - job_name: 'server'
+  #     stream_parse: true
+  #     static_configs:
+  #     - targets:
+  #       - ${config.networking.hostName}:9100
+  # '';
   configure_prom = builtins.toFile "prometheus.yml" ''
     scrape_configs:
-    - job_name: 'server'
+    # System metrics
+    - job_name: 'node'
       stream_parse: true
       static_configs:
       - targets:
         - ${config.networking.hostName}:9100
+
+    # Systemd service monitoring
+    - job_name: 'systemd'
+      metrics_path: /metrics
+      params:
+        collect[]:
+          - 'systemd.service.state'
+          - 'systemd.service.start_time_seconds'
+          - 'systemd.unit_file.state'
+      static_configs:
+      - targets:
+        - ${config.networking.hostName}:9100
+      relabel_configs:
+        # Filter for specific services we want to monitor
+        - source_labels: [__name__]
+          regex: 'node_systemd_unit_state'
+          action: keep
+        - source_labels: [name]
+          regex: '(container@git|microvm@git-runner-|postfix|dovecot|openldap|wireguard-wg_cloonar).*\.service'
+          action: keep
+
+    ${concatStringsSep "\n" config.services.victoriametrics.extraScrapeConfigs}
   '';
 in {
-  services.prometheus.exporters.node.enable = true;
-
-  sops.secrets.victoria-nginx-password.owner = "nginx";
-
-  services.victoriametrics = {
-    enable = true;
-    extraOptions = [
-      "-promscrape.config=${configure_prom}"
-    ];
-  };
-
-  services.nginx.virtualHosts."victoria-server.cloonar.com" = {
-    forceSSL = true;
-    enableACME = true;
-    acmeRoot = null;
-    locations."/" = {
-      proxyWebsockets = true;
-      extraConfig = ''
-        auth_basic "Victoria password";
-        auth_basic_user_file ${config.sops.secrets.victoria-nginx-password.path};
-
-        proxy_read_timeout 1800s;
-        proxy_redirect off;
-        proxy_connect_timeout 1600s;
-
-        access_log off;
-        proxy_pass http://127.0.0.1:8428;
-      '';
+  options.services.victoriametrics = {
+    extraScrapeConfigs = mkOption {
+      type = types.listOf types.str;
+      default = [];
+      description = "Additional Prometheus scrape job YAML snippets for Blackbox Exporter probes";
     };
   };
 
+  config = {
+    services.prometheus.exporters.node.enable = true;
+
+    sops.secrets.victoria-nginx-password.owner = "nginx";
+
+    services.victoriametrics = {
+      enable = true;
+      extraOptions = [
+        "-promscrape.config=${configure_prom}"
+      ];
+    };
+
+    services.nginx.virtualHosts."victoria-server.cloonar.com" = {
+      forceSSL = true;
+      enableACME = true;
+      acmeRoot = null;
+      locations."/" = {
+        proxyWebsockets = true;
+        extraConfig = ''
+          auth_basic "Victoria password";
+          auth_basic_user_file ${config.sops.secrets.victoria-nginx-password.path};
+
+          proxy_read_timeout 1800s;
+          proxy_redirect off;
+          proxy_connect_timeout 1600s;
+
+          access_log off;
+          proxy_pass http://127.0.0.1:8428;
+        '';
+      };
+    };
+  };
 }