{ lib, pkgs, config, ... }: let cfg = config.services.blackbox-exporter; nginxVHosts = config.services.nginx.virtualHosts or {}; allDomains = (lib.attrNames nginxVHosts) ++ [ "foundry-vtt.cloonar.com" ]; filteredDomains = builtins.filter (d: !builtins.elem d cfg.blacklistDomains) allDomains; httpsDomains = lib.map (d: "https://${d}") filteredDomains; websiteAlertRules = lib.map (target: let domain = lib.replaceStrings ["://" "." "-" "/" ] ["-" "-" "_" "_"] target + "-down-alert"; uid = builtins.hashString "sha1" domain; in { uid = uid; title = "Website " + target + " Down"; condition = "C"; data = [ { refId = "A"; relativeTimeRange = { from = 300; to = 0; }; datasourceUid = "vm-datasource-uid"; model = { editorMode = "code"; expr = "probe_success{instance=\"" + target + "\"} OR on() vector(0)"; hide = false; intervalMs = 1000; legendFormat = target; maxDataPoints = 43200; range = true; refId = "A"; }; } { refId = "B"; datasourceUid = "__expr__"; model = { type = "reduce"; expression = "A"; reducer = "last"; }; } { refId = "C"; datasourceUid = "__expr__"; model = { type = "math"; expression = "$B == 0"; }; } ]; noDataState = "Alerting"; execErrState = "Alerting"; for = "5m"; annotations = { description = "Website " + target + " is unreachable."; summary = "Website Down"; }; labels = { severity = "critical"; website_url = target; }; } ) httpsDomains; in { services.grafana.provision.alerting.rules.settings.groups = [ { name = "Website Alerts"; folder = "Websites"; interval = "1m"; rules = websiteAlertRules; } ]; }