From faad280aa0fe4a4b53d099f269d3d12895d01584 Mon Sep 17 00:00:00 2001
From: Dominik Polakovics <dominik.polakovics@cloonar.com>
Date: Sun, 1 Jun 2025 09:08:36 +0200
Subject: [PATCH] fix: update Blackbox Exporter scrape config and Grafana alert
 expressions for improved monitoring accuracy

---
 hosts/web-arm/modules/blackbox-exporter.nix   | 37 +++++++++++--------
 .../grafana/alerting/websites/default.nix     |  4 +-
 hosts/web-arm/modules/victoriametrics.nix     |  3 +-
 3 files changed, 25 insertions(+), 19 deletions(-)

diff --git a/hosts/web-arm/modules/blackbox-exporter.nix b/hosts/web-arm/modules/blackbox-exporter.nix
index 9b35d67..ddf9557 100644
--- a/hosts/web-arm/modules/blackbox-exporter.nix
+++ b/hosts/web-arm/modules/blackbox-exporter.nix
@@ -33,23 +33,30 @@ in {
     # Add scrape config for VictoriaMetrics agent
     services.victoriametrics.extraScrapeConfigs = [
       ''
-      - job_name: "blackbox_http_all_domains"
-        metrics_path: "/probe"
-        params:
-          module: ["http_2xx"]
+        - job_name: "blackbox_http_all_domains"
+          metrics_path: "/probe"
+          params:
+            module: ["http_2xx"]
 
-        static_configs:
-          - targets:
-              [
-                ${domainsString}
-              ]
+          static_configs:
+            - targets:
+                [
+                  ${domainsString}
+                ]
 
-        relabel_configs:
-          - source_labels: ["__address__"]
-            target_label: "__param_target"
-            replacement: "$$1"
-          - source_labels: ["__param_target"]
-            target_label: "instance"
+          relabel_configs:
+            - source_labels: ["__address__"]
+              target_label: "__param_target"
+              regex: '(.*)'
+              replacement: "$1"
+            - source_labels: ["__param_target"]
+              target_label: "instance"
+            - target_label: "__address__"
+              replacement: "127.0.0.1:9115"
+            - source_labels: ["__address__"]
+              regex: "127\\.0\\.0\\.1:9115"
+              target_label: "__scheme__"
+              replacement: "http"
       ''
     ];
   };
diff --git a/hosts/web-arm/modules/grafana/alerting/websites/default.nix b/hosts/web-arm/modules/grafana/alerting/websites/default.nix
index c495194..6681423 100644
--- a/hosts/web-arm/modules/grafana/alerting/websites/default.nix
+++ b/hosts/web-arm/modules/grafana/alerting/websites/default.nix
@@ -20,7 +20,7 @@ let
           datasourceUid     = "vm-datasource-uid";
           model = {
             editorMode     = "code";
-            expr           = "probe_success{target=\"" + target + "\"} OR on() vector(0)";
+            expr           = "probe_success{instance=\"" + target + "\"} OR on() vector(0)";
             hide           = false;
             intervalMs     = 1000;
             legendFormat   = target;
@@ -43,7 +43,7 @@ let
           datasourceUid = "__expr__";
           model = {
             type       = "math";
-            expression = "$B < 1";
+            expression = "$B == 0";
           };
         }
       ];
diff --git a/hosts/web-arm/modules/victoriametrics.nix b/hosts/web-arm/modules/victoriametrics.nix
index 0dbd915..e37567f 100644
--- a/hosts/web-arm/modules/victoriametrics.nix
+++ b/hosts/web-arm/modules/victoriametrics.nix
@@ -37,8 +37,7 @@ let
         - source_labels: [name]
           regex: '(container@git|microvm@git-runner-|postfix|dovecot|openldap|wireguard-wg_cloonar).*\.service'
           action: keep
-
-    ${concatStringsSep "\n" config.services.victoriametrics.extraScrapeConfigs}
+${concatStringsSep "\n" config.services.victoriametrics.extraScrapeConfigs}
   '';
 in {
   options.services.victoriametrics = {