feat: restructure Grafana configuration, migrate alert rules to new format and add VictoriaMetrics datasource

hosts/web-arm/modules/grafana/alerting/disk_usage.nix

@@ -0,0 +1,85 @@
+{ lib, pkgs, config, ... }:
+{
+  services.grafana.provision.alerting.rules.settings.groups = [
+    {
+      # orgId = 1; # Defaults to 1 for provisioned rules
+      name = "DiskUsageAlerts";      # Name of the rule group
+      folder = "System Alerts";       # The folder these rules belong to in Grafana UI
+      interval = "1m";              # How often to evaluate rules in this group
+
+      rules = [
+        {
+          uid = "high-disk-usage-alert-uid"; # Optional: provide a stable UID for the rule itself
+          title = "HighDiskUsage";          # Name of the alert rule (was 'alert' in vmalert)
+
+          # Condition for the alert to fire. 'C' refers to the refId of the threshold expression.
+          condition = "D"; # Condition is now D
+          # Removed rule-level relativeTimeRange
+
+          # Data queries and expressions
+          data = [
+            # Query A: Calculate disk usage percentage
+            {
+              refId = "A";
+              datasourceUid = "vm-datasource-uid"; # UID of the VictoriaMetrics datasource
+              queryType = "prometheus"; # Explicitly set, though often inferred
+              relativeTimeRange = { from = 60; to = 0; }; # Query-level, integer seconds
+              model = {
+                expr = ''
+                  (
+                    node_filesystem_size_bytes{fstype!~"tmpfs|rootfs",mountpoint!=""} - node_filesystem_avail_bytes{fstype!~"tmpfs|rootfs",mountpoint!=""}
+                  ) / (node_filesystem_size_bytes{fstype!~"tmpfs|rootfs",mountpoint!=""} > 0) * 100
+                  and node_filesystem_size_bytes{fstype!~"tmpfs|rootfs",mountpoint!=""}
+                  and node_filesystem_avail_bytes{fstype!~"tmpfs|rootfs",mountpoint!=""}
+                '';
+                legendFormat = "{{mountpoint}} on {{instance}}"; # Example legend
+                instant = false; # For range queries, default is false
+              };
+            }
+            # Expression C: Reduce Query A to its last value, preserving labels
+            {
+              refId = "C";
+              datasourceUid = "__expr__";
+              model = {
+                type = "reduce";
+                expression = "A"; # Input is Query A
+                reducer = "last"; # Get the last value of each series in A
+              };
+            }
+            # Expression D: Apply math condition to the reduced values from C
+            {
+              refId = "D";
+              datasourceUid = "__expr__";
+              model = {
+                type = "math";
+                expression = "$C > 85"; # Check if the last value from each series in C is > 85
+              };
+            }
+          ];
+          
+          for = "15m"; # Duration the condition must be met (same as vmalert)
+          
+          # How to handle states where data is missing or query errors
+          noDataState = "NoData";   # Options: NoData, Alerting, OK
+          execErrState = "Error"; # Options: Error, Alerting, OK
+
+          annotations = {
+            summary = "High disk usage on {{ $labels.instance }} at {{ $labels.mountpoint }}";
+            description = ''
+              Disk usage on {{ $labels.instance }} for mount point {{ $labels.mountpoint }}
+              (fstype: {{ $labels.fstype }}) has been above 85% for more than 15 minutes.
+              Current value: {{ if $values.C }}{{ $values.C | humanizePercentage }}{{ else }}N/A{{ end }}%.
+            ''; # Using $values.C as it's the input to the math condition D
+          };
+          labels = {
+            severity = "warning";
+            category = "capacity";
+            # Grafana automatically adds labels from the query result (instance, mountpoint, etc.)
+            # and labels from the rule group/folder.
+          };
+          # isPaused = false; # Default is not paused
+        }
+      ];
+    }
+  ];
+}