fix: update Blackbox Exporter scrape config and Grafana alert expressions for improved monitoring accuracy

hosts/web-arm/modules/blackbox-exporter.nix

@@ -33,23 +33,30 @@ in {
     # Add scrape config for VictoriaMetrics agent
     services.victoriametrics.extraScrapeConfigs = [
       ''
-      - job_name: "blackbox_http_all_domains"
+        - job_name: "blackbox_http_all_domains"
-        metrics_path: "/probe"
+          metrics_path: "/probe"
-        params:
+          params:
-          module: ["http_2xx"]
+            module: ["http_2xx"]
 
-        static_configs:
+          static_configs:
-          - targets:
+            - targets:
-              [
+                [
-                ${domainsString}
+                  ${domainsString}
-              ]
+                ]
 
-        relabel_configs:
+          relabel_configs:
-          - source_labels: ["__address__"]
+            - source_labels: ["__address__"]
-            target_label: "__param_target"
+              target_label: "__param_target"
-            replacement: "$$1"
+              regex: '(.*)'
-          - source_labels: ["__param_target"]
+              replacement: "$1"
-            target_label: "instance"
+            - source_labels: ["__param_target"]
+              target_label: "instance"
+            - target_label: "__address__"
+              replacement: "127.0.0.1:9115"
+            - source_labels: ["__address__"]
+              regex: "127\\.0\\.0\\.1:9115"
+              target_label: "__scheme__"
+              replacement: "http"
       ''
     ];
   };

hosts/web-arm/modules/grafana/alerting/websites/default.nix

@@ -20,7 +20,7 @@ let
           datasourceUid     = "vm-datasource-uid";
           model = {
             editorMode     = "code";
-            expr           = "probe_success{target=\"" + target + "\"} OR on() vector(0)";
+            expr           = "probe_success{instance=\"" + target + "\"} OR on() vector(0)";
             hide           = false;
             intervalMs     = 1000;
             legendFormat   = target;
@@ -43,7 +43,7 @@ let
           datasourceUid = "__expr__";
           model = {
             type       = "math";
-            expression = "$B < 1";
+            expression = "$B == 0";
           };
         }
       ];

hosts/web-arm/modules/victoriametrics.nix

@@ -37,8 +37,7 @@ let
         - source_labels: [name]
           regex: '(container@git|microvm@git-runner-|postfix|dovecot|openldap|wireguard-wg_cloonar).*\.service'
           action: keep
-
+${concatStringsSep "\n" config.services.victoriametrics.extraScrapeConfigs}
-    ${concatStringsSep "\n" config.services.victoriametrics.extraScrapeConfigs}
   '';
 in {
   options.services.victoriametrics = {