feat: add alerting for amz ebs server and websites blackbox

hosts/amzebs-01/configuration.nix

@@ -7,6 +7,7 @@
     ./modules/mysql.nix
     ./modules/web/stack.nix
     ./modules/laravel-storage.nix
+    ./modules/blackbox-exporter.nix
 
     ./utils/modules/autoupgrade.nix
     ./utils/modules/promtail

hosts/amzebs-01/modules/blackbox-exporter.nix

@@ -0,0 +1,83 @@
+{ config, pkgs, lib, ... }:
+
+with lib;
+
+let
+  hostname = config.networking.hostName;
+
+  cfg = config.services.blackbox-exporter;
+  nginxVHosts = config.services.nginx.virtualHosts or {};
+  allDomains = lib.attrNames nginxVHosts;
+  filteredDomains = builtins.filter (d: !builtins.elem d cfg.blacklistDomains) allDomains;
+  httpsDomains = lib.map (d: "https://${d}") filteredDomains;
+  domainsString = builtins.concatStringsSep "\n          "
+    (map (d: "\"${d}\",") httpsDomains);
+in {
+  options.services.blackbox-exporter.blacklistDomains = mkOption {
+    type = types.listOf types.str;
+    default = [];
+    description = "List of domains to exclude from Blackbox Exporter monitoring";
+  };
+
+  config = {
+    services.blackbox-exporter = {
+      blacklistDomains = [
+        # Currently no domains blacklisted - monitoring all nginx virtualHosts
+      ];
+    };
+
+    # Systemd service for Blackbox Exporter
+    systemd.services.blackbox-exporter = {
+      description = "Blackbox Exporter";
+      after = [ "network-online.target" ];
+      wantedBy = [ "multi-user.target" ];
+      serviceConfig.ExecStart = ''
+        ${pkgs.prometheus-blackbox-exporter}/bin/blackbox_exporter \
+          --config.file=/etc/blackbox_exporter/blackbox.yml
+      '';
+    };
+
+    # Configuration file for Blackbox Exporter
+    environment.etc."blackbox_exporter/blackbox.yml".text = ''
+      modules:
+        http_200_final:
+          prober: http
+          http:
+            method: GET
+            follow_redirects: true
+            preferred_ip_protocol: "ip4"   # avoid blanket IPv6 failures
+            valid_http_versions: ["HTTP/1.1", "HTTP/2.0"]
+            valid_status_codes: [200]
+    '';
+
+    # Add scrape config for VictoriaMetrics agent
+    services.victoriametrics.extraScrapeConfigs = [
+      ''
+        - job_name: "blackbox_http_all_domains"
+          metrics_path: "/probe"
+          params:
+            module: ["http_200_final"]
+
+          static_configs:
+            - targets:
+                [
+                  ${domainsString}
+                ]
+
+          relabel_configs:
+            - source_labels: ["__address__"]
+              target_label: "__param_target"
+              regex: '(.*)'
+              replacement: "$1"
+            - source_labels: ["__param_target"]
+              target_label: "instance"
+            - target_label: "__address__"
+              replacement: "127.0.0.1:9115"
+            - source_labels: ["__address__"]
+              regex: "127\\.0\\.0\\.1:9115"
+              target_label: "__scheme__"
+              replacement: "http"
+      ''
+    ];
+  };
+}