nixos/hosts/web-arm/modules/grafana/alerting/service/service_down.nix

{ lib, pkgs, config, ... }:
{
  grafanaAlertRuleDefinitions = [
    # Systemd service monitoring alerts
    {
      uid = "gitea-service-down-alert-uid";
      title = "Gitea Service Down";
      condition = "C";
      data = [
        {
          refId = "A";
          relativeTimeRange = {
            from = 300;
            to = 0;
          };
          datasourceUid = "vm-datasource-uid";
          model = {
            editorMode = "code";
            expr = "node_systemd_unit_state{state=\"active\", name=\"container@git.service\"} OR on() vector(0)";
            hide = false;
            intervalMs = 1000;
            legendFormat = "__auto";
            maxDataPoints = 43200;
            range = true;
            refId = "A";
          };
        }
        {
          refId = "B";
          datasourceUid = "__expr__";
          model = {
            type = "reduce";
            expression = "A";
            reducer = "last";
          };
        }
        {
          refId = "C";
          datasourceUid = "__expr__";
          model = {
            type = "math";
            expression = "$B < 1";
          };
        }
      ];
      noDataState = "Alerting";
      execErrState = "Alerting";
      for = "5m";
      annotations = {
        description = "Gitea service is down on {{ $labels.instance }}";
        summary = "Gitea Service Down";
      };
      labels = {
        severity = "critical";
      };
    }
    {
      uid = "gitea-runner-service-down-alert-uid";
      title = "Gitea Runner Service Down";
      condition = "C";
      data = [
        {
          refId = "A";
          relativeTimeRange = {
            from = 300;
            to = 0;
          };
          datasourceUid = "vm-datasource-uid";
          model = {
            editorMode = "code";
            expr = "node_systemd_unit_state{state=\"active\", name=\"microvm@git-runner-1.service\"} OR on() vector(0)";
            hide = false;
            intervalMs = 1000;
            legendFormat = "__auto";
            maxDataPoints = 43200;
            range = true;
            refId = "A";
          };
        }
        {
          refId = "B";
          datasourceUid = "__expr__";
          model = {
            type = "reduce";
            expression = "A";
            reducer = "last";
          };
        }
        {
          refId = "C";
          datasourceUid = "__expr__";
          model = {
            type = "math";
            expression = "$B < 1";
          };
        }
      ];
      noDataState = "Alerting";
      execErrState = "Alerting";
      for = "5m";
      annotations = {
        description = "Gitea Runner service is down on {{ $labels.instance }}";
        summary = "Gitea Runner Service Down";
      };
      labels = {
        severity = "critical";
      };
    }
    {
      uid = "postfix-service-down-alert-uid";
      title = "Postfix Service Down";
      condition = "C";
      data = [
        {
          refId = "A";
          relativeTimeRange = {
            from = 300;
            to = 0;
          };
          datasourceUid = "vm-datasource-uid";
          model = {
            editorMode = "code";
            expr = "node_systemd_unit_state{state=\"active\", name=\"postfix.service\"} OR on() vector(0)";
            hide = false;
            intervalMs = 1000;
            legendFormat = "__auto";
            maxDataPoints = 43200;
            range = true;
            refId = "A";
          };
        }
        {
          refId = "B";
          datasourceUid = "__expr__";
          model = {
            type = "reduce";
            expression = "A";
            reducer = "last";
          };
        }
        {
          refId = "C";
          datasourceUid = "__expr__";
          model = {
            type = "math";
            expression = "$B < 1";
          };
        }
      ];
      noDataState = "Alerting";
      execErrState = "Alerting";
      for = "5m";
      annotations = {
        description = "Postfix service is down on {{ $labels.instance }}";
        summary = "Postfix Service Down";
      };
      labels = {
        severity = "critical";
      };
    }
    {
      uid = "dovecot-service-down-alert-uid";
      title = "Dovecot Service Down";
      condition = "C";
      data = [
        {
          refId = "A";
          relativeTimeRange = {
            from = 300;
            to = 0;
          };
          datasourceUid = "vm-datasource-uid";
          model = {
            editorMode = "code";
            expr = "node_systemd_unit_state{state=\"active\", name=\"dovecot2.service\"} OR on() vector(0)";
            hide = false;
            intervalMs = 1000;
            legendFormat = "__auto";
            maxDataPoints = 43200;
            range = true;
            refId = "A";
          };
        }
        {
          refId = "B";
          datasourceUid = "__expr__";
          model = {
            type = "reduce";
            expression = "A";
            reducer = "last";
          };
        }
        {
          refId = "C";
          datasourceUid = "__expr__";
          model = {
            type = "math";
            expression = "$B < 1";
          };
        }
      ];
      noDataState = "Alerting";
      execErrState = "Alerting";
      for = "5m";
      annotations = {
        description = "Dovecot service is down on {{ $labels.instance }}";
        summary = "Dovecot Service Down";
      };
      labels = {
        severity = "critical";
      };
    }
    {
      uid = "openldap-service-down-alert-uid";
      title = "OpenLDAP Service Down";
      condition = "C";
      data = [
        {
          refId = "A";
          relativeTimeRange = {
            from = 300;
            to = 0;
          };
          datasourceUid = "vm-datasource-uid";
          model = {
            editorMode = "code";
            expr = "node_systemd_unit_state{state=\"active\", name=\"openldap.service\"} OR on() vector(0)";
            hide = false;
            intervalMs = 1000;
            legendFormat = "__auto";
            maxDataPoints = 43200;
            range = true;
            refId = "A";
          };
        }
        {
          refId = "B";
          datasourceUid = "__expr__";
          model = {
            type = "reduce";
            expression = "A";
            reducer = "last";
          };
        }
        {
          refId = "C";
          datasourceUid = "__expr__";
          model = {
            type = "math";
            expression = "$B < 1";
          };
        }
      ];
      noDataState = "Alerting";
      execErrState = "Alerting";
      for = "5m";
      annotations = {
        description = "OpenLDAP service is down on {{ $labels.instance }}";
        summary = "OpenLDAP Service Down";
      };
      labels = {
        severity = "critical";
      };
    }
    {
      uid = "wireguard-service-down-alert-uid";
      title = "WireGuard Service Down";
      condition = "C";
      data = [
        {
          refId = "A";
          relativeTimeRange = {
            from = 300;
            to = 0;
          };
          datasourceUid = "vm-datasource-uid";
          model = {
            editorMode = "code";
            expr = "node_systemd_unit_state{state=\"active\", name=\"wireguard-wg_cloonar.service\"} OR on() vector(0)";
            hide = false;
            intervalMs = 1000;
            legendFormat = "__auto";
            maxDataPoints = 43200;
            range = true;
            refId = "A";
          };
        }
        {
          refId = "B";
          datasourceUid = "__expr__";
          model = {
            type = "reduce";
            expression = "A";
            reducer = "last";
          };
        }
        {
          refId = "C";
          datasourceUid = "__expr__";
          model = {
            type = "math";
            expression = "$B < 1";
          };
        }
      ];
      noDataState = "Alerting";
      execErrState = "Alerting";
      for = "5m";
      annotations = {
        description = "WireGuard service is down on {{ $labels.instance }}";
        summary = "WireGuard Service Down";
      };
      labels = {
        severity = "critical";
      };
    }
  ];
}