feat: implement website alerting plan with Blackbox Exporter and VictoriaMetrics integration

hosts/web-arm/modules/blackbox-exporter.nix

@@ -0,0 +1,56 @@
+{ config, pkgs, lib, ... }:
+
+with lib;
+
+let
+  hostname = config.networking.hostName;
+
+  nginxVHosts = config.services.nginx.virtualHosts or {};
+  allDomains = lib.attrNames nginxVHosts;
+  httpsDomains = lib.map (d: "https://${d}") allDomains;
+  domainsString = builtins.concatStringsSep "\n          "
+    (map (d: "\"${d}\",") httpsDomains);
+in {
+  config = {
+    # Systemd service for Blackbox Exporter
+    systemd.services.blackbox-exporter = {
+      description = "Blackbox Exporter";
+      after = [ "network-online.target" ];
+      wantedBy = [ "multi-user.target" ];
+      serviceConfig.ExecStart = ''
+        ${pkgs.prometheus-blackbox-exporter}/bin/blackbox_exporter \
+          --config.file=/etc/blackbox_exporter/blackbox.yml
+      '';
+    };
+
+    # Configuration file for Blackbox Exporter
+    environment.etc."blackbox_exporter/blackbox.yml".text = ''
+      modules:
+        http_2xx:
+          prober: http
+    '';
+
+    # Add scrape config for VictoriaMetrics agent
+    services.victoriametrics.extraScrapeConfigs = [
+      ''
+      - job_name: "blackbox_http_all_domains"
+        metrics_path: "/probe"
+        params:
+          module: ["http_2xx"]
+
+        static_configs:
+          - targets:
+              [
+                ${domainsString}
+              ]
+
+        relabel_configs:
+          - source_labels: ["__address__"]
+            target_label: "__param_target"
+            replacement: "$$1"
+          - source_labels: ["__param_target"]
+            target_label: "instance"
+      ''
+    ];
+  };
+}