diff --git a/.sops.yaml b/.sops.yaml index d93b121..6538e51 100644 --- a/.sops.yaml +++ b/.sops.yaml @@ -151,4 +151,5 @@ creation_rules: - *netboot - *fw - *fw-new + - *nas - *amzebs-01 diff --git a/hosts/nas/configuration.nix b/hosts/nas/configuration.nix index d07c2e5..7cddbcb 100644 --- a/hosts/nas/configuration.nix +++ b/hosts/nas/configuration.nix @@ -9,9 +9,12 @@ in { "${impermanence}/nixos.nix" ./utils/bento.nix ./utils/modules/sops.nix + ./utils/modules/victoriametrics/default.nix ./modules/pyload.nix ./modules/jellyfin.nix + ./modules/power-management.nix + ./modules/disk-monitoring.nix ./hardware-configuration.nix ]; diff --git a/hosts/nas/hardware-configuration.nix b/hosts/nas/hardware-configuration.nix index a3c3edd..077f5e9 100644 --- a/hosts/nas/hardware-configuration.nix +++ b/hosts/nas/hardware-configuration.nix @@ -16,6 +16,14 @@ boot.kernelModules = [ "kvm-intel" ]; boot.extraModulePackages = [ ]; + # Power management kernel parameters + boot.kernelParams = [ + "intel_pstate=passive" # Better with powersave governor + "i915.enable_rc6=1" # GPU deep sleep states + "i915.enable_dc=2" # Display C-states (deepest) + "i915.enable_fbc=1" # Frame buffer compression + ]; + # RAID 1 array for data storage boot.swraid = { enable = true; @@ -78,11 +86,13 @@ fileSystems."/var/lib/downloads" = { device = "/dev/vg-data/lv-downloads"; fsType = "xfs"; + options = [ "noatime" ]; }; fileSystems."/var/lib/multimedia" = { device = "/dev/vg-data/lv-multimedia"; fsType = "xfs"; + options = [ "noatime" ]; }; # DHCP networking diff --git a/hosts/nas/modules/disk-monitoring.nix b/hosts/nas/modules/disk-monitoring.nix new file mode 100644 index 0000000..da42bcb --- /dev/null +++ b/hosts/nas/modules/disk-monitoring.nix @@ -0,0 +1,192 @@ +# Disk monitoring for NAS +# - S.M.A.R.T. metrics collection (respects disk spindown) +# - mdadm RAID array status +# - Exports metrics via node_exporter textfile collector +{ config, lib, pkgs, ... }: + +let + # Disk identifiers from hardware-configuration.nix + disks = [ + "/dev/disk/by-id/ata-ST18000NM000J-2TV103_ZR52TBSB" + "/dev/disk/by-id/ata-ST18000NM000J-2TV103_ZR52V9QX" + ]; + + textfileDir = "/var/lib/prometheus-node-exporter"; + + # Script to collect S.M.A.R.T. and mdadm metrics + collectMetricsScript = pkgs.writeShellScript "collect-disk-metrics" '' + set -euo pipefail + + TEXTFILE_DIR="${textfileDir}" + METRICS_FILE="$TEXTFILE_DIR/disk_health.prom" + TEMP_FILE="$TEXTFILE_DIR/disk_health.prom.tmp" + + mkdir -p "$TEXTFILE_DIR" + : > "$TEMP_FILE" + + # Timestamp of collection + echo "# HELP disk_metrics_last_update Unix timestamp of last metrics collection" >> "$TEMP_FILE" + echo "# TYPE disk_metrics_last_update gauge" >> "$TEMP_FILE" + echo "disk_metrics_last_update $(date +%s)" >> "$TEMP_FILE" + + echo "" >> "$TEMP_FILE" + echo "# HELP smart_device_active Whether the disk was active (1) or sleeping (0) when checked" >> "$TEMP_FILE" + echo "# TYPE smart_device_active gauge" >> "$TEMP_FILE" + + # S.M.A.R.T. metrics for each disk + for disk in ${lib.concatStringsSep " " disks}; do + if [[ ! -e "$disk" ]]; then + echo "Warning: Disk $disk not found, skipping" >&2 + continue + fi + + # Resolve symlink to get actual device + device=$(readlink -f "$disk") + short_name=$(basename "$device") + + # Extract serial from disk ID for labels + serial=$(basename "$disk" | sed 's/ata-ST18000NM000J-2TV103_//') + + # Check power state without waking disk + power_state=$(${pkgs.hdparm}/bin/hdparm -C "$device" 2>/dev/null | grep -oP '(standby|active/idle|active|idle)' | head -1 || echo "unknown") + + if [[ "$power_state" == "standby" ]]; then + # Disk is sleeping - don't wake it, report inactive + echo "smart_device_active{device=\"$short_name\",serial=\"$serial\"} 0" >> "$TEMP_FILE" + echo "Disk $short_name is in standby, skipping S.M.A.R.T. collection" >&2 + continue + fi + + # Disk is active - collect S.M.A.R.T. data + echo "smart_device_active{device=\"$short_name\",serial=\"$serial\"} 1" >> "$TEMP_FILE" + + # Get S.M.A.R.T. health status + if ${pkgs.smartmontools}/bin/smartctl -H "$device" 2>/dev/null | grep -q "PASSED"; then + health=1 + else + health=0 + fi + + # Get S.M.A.R.T. attributes + smartctl_output=$(${pkgs.smartmontools}/bin/smartctl -A "$device" 2>/dev/null || true) + + # Parse key attributes + # Format: ID# ATTRIBUTE_NAME FLAG VALUE WORST THRESH TYPE UPDATED WHEN_FAILED RAW_VALUE + + get_raw_value() { + local attr_id="$1" + echo "$smartctl_output" | awk -v id="$attr_id" '$1 == id { print $10 }' | head -1 + } + + reallocated=$(get_raw_value "5") + power_on_hours=$(get_raw_value "9") + temperature=$(get_raw_value "194") + reallocated_event=$(get_raw_value "196") + pending_sector=$(get_raw_value "197") + offline_uncorrectable=$(get_raw_value "198") + udma_crc_error=$(get_raw_value "199") + + # Output metrics + cat >> "$TEMP_FILE" << EOF + +# S.M.A.R.T. metrics for $short_name +smart_health_passed{device="$short_name",serial="$serial"} $health +EOF + + [[ -n "$reallocated" ]] && echo "smart_reallocated_sector_ct{device=\"$short_name\",serial=\"$serial\"} $reallocated" >> "$TEMP_FILE" + [[ -n "$power_on_hours" ]] && echo "smart_power_on_hours{device=\"$short_name\",serial=\"$serial\"} $power_on_hours" >> "$TEMP_FILE" + [[ -n "$temperature" ]] && echo "smart_temperature_celsius{device=\"$short_name\",serial=\"$serial\"} $temperature" >> "$TEMP_FILE" + [[ -n "$reallocated_event" ]] && echo "smart_reallocated_event_count{device=\"$short_name\",serial=\"$serial\"} $reallocated_event" >> "$TEMP_FILE" + [[ -n "$pending_sector" ]] && echo "smart_current_pending_sector{device=\"$short_name\",serial=\"$serial\"} $pending_sector" >> "$TEMP_FILE" + [[ -n "$offline_uncorrectable" ]] && echo "smart_offline_uncorrectable{device=\"$short_name\",serial=\"$serial\"} $offline_uncorrectable" >> "$TEMP_FILE" + [[ -n "$udma_crc_error" ]] && echo "smart_udma_crc_error_count{device=\"$short_name\",serial=\"$serial\"} $udma_crc_error" >> "$TEMP_FILE" + done + + # mdadm RAID array status (doesn't access disks) + echo "" >> "$TEMP_FILE" + echo "# HELP mdadm_array_state RAID array state (1=clean, 0=degraded/other)" >> "$TEMP_FILE" + echo "# TYPE mdadm_array_state gauge" >> "$TEMP_FILE" + echo "# HELP mdadm_array_devices_total Total devices in RAID array" >> "$TEMP_FILE" + echo "# TYPE mdadm_array_devices_total gauge" >> "$TEMP_FILE" + echo "# HELP mdadm_array_devices_active Active devices in RAID array" >> "$TEMP_FILE" + echo "# TYPE mdadm_array_devices_active gauge" >> "$TEMP_FILE" + + # Find RAID arrays + for md_device in /dev/md/*; do + [[ -e "$md_device" ]] || continue + + array_name=$(basename "$md_device") + + # Get array details + mdadm_output=$(${pkgs.mdadm}/bin/mdadm --detail "$md_device" 2>/dev/null || continue) + + # Parse state + state=$(echo "$mdadm_output" | grep "State :" | sed 's/.*State : //' | tr -d ' ') + if [[ "$state" == "clean" ]] || [[ "$state" == "active" ]]; then + state_value=1 + else + state_value=0 + fi + + # Parse device counts + total_devices=$(echo "$mdadm_output" | grep "Raid Devices" | awk '{print $4}') + active_devices=$(echo "$mdadm_output" | grep "Active Devices" | awk '{print $4}') + + echo "mdadm_array_state{array=\"$array_name\",state=\"$state\"} $state_value" >> "$TEMP_FILE" + [[ -n "$total_devices" ]] && echo "mdadm_array_devices_total{array=\"$array_name\"} $total_devices" >> "$TEMP_FILE" + [[ -n "$active_devices" ]] && echo "mdadm_array_devices_active{array=\"$array_name\"} $active_devices" >> "$TEMP_FILE" + done + + # Atomically replace the metrics file + mv "$TEMP_FILE" "$METRICS_FILE" + + echo "Disk metrics collection complete" + ''; +in +{ + # Required packages + environment.systemPackages = with pkgs; [ + smartmontools + hdparm + mdadm + ]; + + # Node exporter with textfile collector + services.prometheus.exporters.node = { + enable = true; + enabledCollectors = [ + "textfile" + "systemd" + ]; + extraFlags = [ + "--collector.textfile.directory=${textfileDir}" + ]; + }; + + # Systemd service to collect metrics + systemd.services.disk-metrics = { + description = "Collect S.M.A.R.T. and RAID metrics"; + path = with pkgs; [ coreutils gawk gnugrep gnused ]; + serviceConfig = { + Type = "oneshot"; + ExecStart = "${collectMetricsScript}"; + # Run as root to access disk devices + User = "root"; + }; + }; + + # Timer to run every 20 minutes (5min buffer for 15min spindown) + systemd.timers.disk-metrics = { + wantedBy = [ "timers.target" ]; + timerConfig = { + OnCalendar = "*:0/20"; # Every 20 minutes + RandomizedDelaySec = "1min"; + Persistent = true; + }; + }; + + # Ensure textfile directory exists and is persisted + systemd.tmpfiles.rules = [ + "d ${textfileDir} 0755 root root -" + ]; +} diff --git a/hosts/nas/modules/power-management.nix b/hosts/nas/modules/power-management.nix new file mode 100644 index 0000000..1d0b338 --- /dev/null +++ b/hosts/nas/modules/power-management.nix @@ -0,0 +1,19 @@ +# Power management for NAS +# - CPU powersave governor (scales up on demand for transcoding) +# - Disk spindown after 15 minutes idle +{ config, lib, pkgs, ... }: + +{ + # CPU Power Management - powersave scales up on demand for transcoding + powerManagement.cpuFreqGovernor = "powersave"; + + # Disk spindown - hdparm for Seagate 18TB drives + environment.systemPackages = [ pkgs.hdparm ]; + + services.udev.extraRules = '' + # Seagate 18TB NAS drives - APM 127 allows spindown, -S 180 = 15 min + ACTION=="add", KERNEL=="sd[a-z]", SUBSYSTEM=="block", \ + ATTRS{model}=="ST18000NM000J*", \ + RUN+="${pkgs.hdparm}/bin/hdparm -B 127 -S 180 /dev/%k" + ''; +} diff --git a/hosts/web-arm/modules/grafana/alerting/storage/default.nix b/hosts/web-arm/modules/grafana/alerting/storage/default.nix new file mode 100644 index 0000000..8b63271 --- /dev/null +++ b/hosts/web-arm/modules/grafana/alerting/storage/default.nix @@ -0,0 +1,17 @@ +{ lib, pkgs, config, ... }: +let + smartAlertRules = (import ./smart_alerts.nix { inherit lib pkgs config; }).grafanaAlertRuleDefinitions; + raidAlertRules = (import ./raid_alerts.nix { inherit lib pkgs config; }).grafanaAlertRuleDefinitions; + + allStorageRules = smartAlertRules ++ raidAlertRules; +in +{ + services.grafana.provision.alerting.rules.settings.groups = [ + { + name = "Storage Alerts"; + folder = "Storage Alerts"; + interval = "5m"; # Check every 5 minutes (metrics collected every 20 min) + rules = allStorageRules; + } + ]; +} diff --git a/hosts/web-arm/modules/grafana/alerting/storage/raid_alerts.nix b/hosts/web-arm/modules/grafana/alerting/storage/raid_alerts.nix new file mode 100644 index 0000000..82ad73e --- /dev/null +++ b/hosts/web-arm/modules/grafana/alerting/storage/raid_alerts.nix @@ -0,0 +1,102 @@ +{ lib, pkgs, config, ... }: +{ + grafanaAlertRuleDefinitions = [ + # RAID array degraded - critical + { + uid = "raid-array-degraded-uid"; + title = "RaidArrayDegraded"; + condition = "D"; + data = [ + { + refId = "A"; + datasourceUid = "vm-datasource-uid"; + relativeTimeRange = { from = 300; to = 0; }; + model = { + expr = ''mdadm_array_state == 0''; + instant = false; + }; + } + { + refId = "C"; + datasourceUid = "__expr__"; + model = { + type = "reduce"; + expression = "A"; + reducer = "last"; + }; + } + { + refId = "D"; + datasourceUid = "__expr__"; + model = { + type = "math"; + expression = "$C == 0"; + }; + } + ]; + for = "0s"; + noDataState = "NoData"; + execErrState = "Error"; + annotations = { + summary = "RAID array {{ $labels.array }} is degraded"; + description = '' + RAID array {{ $labels.array }} on {{ $labels.instance }} is in state "{{ $labels.state }}". + The array is not in a healthy state. Check for failed disks immediately! + ''; + }; + labels = { + severity = "critical"; + category = "storage"; + }; + } + + # RAID missing devices - critical + { + uid = "raid-missing-devices-uid"; + title = "RaidMissingDevices"; + condition = "D"; + data = [ + { + refId = "A"; + datasourceUid = "vm-datasource-uid"; + relativeTimeRange = { from = 300; to = 0; }; + model = { + expr = ''mdadm_array_devices_active < mdadm_array_devices_total''; + instant = false; + }; + } + { + refId = "C"; + datasourceUid = "__expr__"; + model = { + type = "reduce"; + expression = "A"; + reducer = "last"; + }; + } + { + refId = "D"; + datasourceUid = "__expr__"; + model = { + type = "math"; + expression = "$C > 0"; + }; + } + ]; + for = "0s"; + noDataState = "NoData"; + execErrState = "Error"; + annotations = { + summary = "RAID array {{ $labels.array }} has missing devices"; + description = '' + RAID array {{ $labels.array }} on {{ $labels.instance }} has fewer active devices than expected. + A disk may have failed or been removed. Check array status immediately! + ''; + }; + labels = { + severity = "critical"; + category = "storage"; + }; + } + ]; +} diff --git a/hosts/web-arm/modules/grafana/alerting/storage/smart_alerts.nix b/hosts/web-arm/modules/grafana/alerting/storage/smart_alerts.nix new file mode 100644 index 0000000..dd36462 --- /dev/null +++ b/hosts/web-arm/modules/grafana/alerting/storage/smart_alerts.nix @@ -0,0 +1,298 @@ +{ lib, pkgs, config, ... }: +{ + grafanaAlertRuleDefinitions = [ + # S.M.A.R.T. overall health failed - critical + { + uid = "smart-health-failed-uid"; + title = "DiskSmartHealthFailed"; + condition = "D"; + data = [ + { + refId = "A"; + datasourceUid = "vm-datasource-uid"; + relativeTimeRange = { from = 300; to = 0; }; + model = { + expr = ''smart_health_passed == 0''; + instant = false; + }; + } + { + refId = "C"; + datasourceUid = "__expr__"; + model = { + type = "reduce"; + expression = "A"; + reducer = "last"; + }; + } + { + refId = "D"; + datasourceUid = "__expr__"; + model = { + type = "math"; + expression = "$C == 0"; + }; + } + ]; + for = "0s"; + noDataState = "NoData"; + execErrState = "Error"; + annotations = { + summary = "S.M.A.R.T. health check FAILED on {{ $labels.device }}"; + description = '' + Disk {{ $labels.device }} ({{ $labels.serial }}) on {{ $labels.instance }} has failed its S.M.A.R.T. health check. + This indicates imminent disk failure. Replace the disk immediately! + ''; + }; + labels = { + severity = "critical"; + category = "storage"; + }; + } + + # Reallocated sectors - warning (any count > 0 is concerning) + { + uid = "smart-reallocated-sectors-uid"; + title = "DiskReallocatedSectors"; + condition = "D"; + data = [ + { + refId = "A"; + datasourceUid = "vm-datasource-uid"; + relativeTimeRange = { from = 300; to = 0; }; + model = { + expr = ''smart_reallocated_sector_ct > 0''; + instant = false; + }; + } + { + refId = "C"; + datasourceUid = "__expr__"; + model = { + type = "reduce"; + expression = "A"; + reducer = "last"; + }; + } + { + refId = "D"; + datasourceUid = "__expr__"; + model = { + type = "math"; + expression = "$C > 0"; + }; + } + ]; + for = "0s"; + noDataState = "NoData"; + execErrState = "Error"; + annotations = { + summary = "Reallocated sectors detected on {{ $labels.device }}"; + description = '' + Disk {{ $labels.device }} ({{ $labels.serial }}) on {{ $labels.instance }} has reallocated sectors. + This indicates disk surface damage. Monitor closely and plan replacement. + ''; + }; + labels = { + severity = "warning"; + category = "storage"; + }; + } + + # Current pending sectors + { + uid = "smart-pending-sectors-uid"; + title = "DiskPendingSectors"; + condition = "D"; + data = [ + { + refId = "A"; + datasourceUid = "vm-datasource-uid"; + relativeTimeRange = { from = 300; to = 0; }; + model = { + expr = ''smart_current_pending_sector > 0''; + instant = false; + }; + } + { + refId = "C"; + datasourceUid = "__expr__"; + model = { + type = "reduce"; + expression = "A"; + reducer = "last"; + }; + } + { + refId = "D"; + datasourceUid = "__expr__"; + model = { + type = "math"; + expression = "$C > 0"; + }; + } + ]; + for = "0s"; + noDataState = "NoData"; + execErrState = "Error"; + annotations = { + summary = "Pending sectors detected on {{ $labels.device }}"; + description = '' + Disk {{ $labels.device }} ({{ $labels.serial }}) on {{ $labels.instance }} has pending sectors. + These sectors could not be read and may be reallocated. Monitor for increase. + ''; + }; + labels = { + severity = "warning"; + category = "storage"; + }; + } + + # Offline uncorrectable errors + { + uid = "smart-offline-uncorrectable-uid"; + title = "DiskOfflineUncorrectable"; + condition = "D"; + data = [ + { + refId = "A"; + datasourceUid = "vm-datasource-uid"; + relativeTimeRange = { from = 300; to = 0; }; + model = { + expr = ''smart_offline_uncorrectable > 0''; + instant = false; + }; + } + { + refId = "C"; + datasourceUid = "__expr__"; + model = { + type = "reduce"; + expression = "A"; + reducer = "last"; + }; + } + { + refId = "D"; + datasourceUid = "__expr__"; + model = { + type = "math"; + expression = "$C > 0"; + }; + } + ]; + for = "0s"; + noDataState = "NoData"; + execErrState = "Error"; + annotations = { + summary = "Offline uncorrectable errors on {{ $labels.device }}"; + description = '' + Disk {{ $labels.device }} ({{ $labels.serial }}) on {{ $labels.instance }} has offline uncorrectable errors. + This indicates data integrity issues. Consider replacement. + ''; + }; + labels = { + severity = "warning"; + category = "storage"; + }; + } + + # High temperature (Seagate enterprise: warning at 50C) + { + uid = "smart-high-temperature-uid"; + title = "DiskHighTemperature"; + condition = "D"; + data = [ + { + refId = "A"; + datasourceUid = "vm-datasource-uid"; + relativeTimeRange = { from = 600; to = 0; }; + model = { + expr = ''smart_temperature_celsius > 50''; + instant = false; + }; + } + { + refId = "C"; + datasourceUid = "__expr__"; + model = { + type = "reduce"; + expression = "A"; + reducer = "last"; + }; + } + { + refId = "D"; + datasourceUid = "__expr__"; + model = { + type = "math"; + expression = "$C > 0"; + }; + } + ]; + for = "10m"; + noDataState = "NoData"; + execErrState = "Error"; + annotations = { + summary = "High temperature on {{ $labels.device }}"; + description = '' + Disk {{ $labels.device }} ({{ $labels.serial }}) on {{ $labels.instance }} temperature exceeds 50°C. + Check cooling and ventilation. + ''; + }; + labels = { + severity = "warning"; + category = "storage"; + }; + } + + # UDMA CRC errors (cable/connection issues) + { + uid = "smart-udma-crc-errors-uid"; + title = "DiskUDMACRCErrors"; + condition = "D"; + data = [ + { + refId = "A"; + datasourceUid = "vm-datasource-uid"; + relativeTimeRange = { from = 86400; to = 0; }; + model = { + expr = ''increase(smart_udma_crc_error_count[24h]) > 0''; + instant = false; + }; + } + { + refId = "C"; + datasourceUid = "__expr__"; + model = { + type = "reduce"; + expression = "A"; + reducer = "last"; + }; + } + { + refId = "D"; + datasourceUid = "__expr__"; + model = { + type = "math"; + expression = "$C > 0"; + }; + } + ]; + for = "0s"; + noDataState = "NoData"; + execErrState = "Error"; + annotations = { + summary = "UDMA CRC errors on {{ $labels.device }}"; + description = '' + Disk {{ $labels.device }} ({{ $labels.serial }}) on {{ $labels.instance }} has new CRC errors. + This typically indicates SATA cable or connection issues. Check cables. + ''; + }; + labels = { + severity = "warning"; + category = "storage"; + }; + } + ]; +} diff --git a/hosts/web-arm/modules/grafana/default.nix b/hosts/web-arm/modules/grafana/default.nix index 3ac5e10..881f802 100644 --- a/hosts/web-arm/modules/grafana/default.nix +++ b/hosts/web-arm/modules/grafana/default.nix @@ -31,6 +31,7 @@ in ./alerting/system/default.nix ./alerting/service/default.nix ./alerting/websites/default.nix + # ./alerting/storage/default.nix ./datasources/victoriametrics.nix ./datasources/loki.nix diff --git a/utils/modules/victoriametrics/secrets.yaml b/utils/modules/victoriametrics/secrets.yaml index b673f65..5f542d9 100644 --- a/utils/modules/victoriametrics/secrets.yaml +++ b/utils/modules/victoriametrics/secrets.yaml @@ -1,88 +1,97 @@ -victoria-agent-env: ENC[AES256_GCM,data:m+o9GgDSm2qYVk90199H6J+RqE0fZH92G7uFjP0Al1JlvclOFjHnNlFJ7y8YfgBcPUFutIU45HN4+I2X2k0+GFyKlrKxevH3wUKNIWG/l/6VlOmZr7SMRQAlVv88aT0EeBIEh+AOilCcWsD4egQPS/faP5yolsqrm/sXltsdbdI6i7EVXBKUUryBjogE6tv5nroN0ter0hXtyspkV4oBxcZIqTK4/t6staEq1OcY/augdpqz6z1aBFVun3c0q70vS5VBP1KIKd+nKVABWWwt,iv:7mnSPuP3jh06uIFQbWUI2VRFF5wbGFLoTS4rf7PUF7M=,tag:BtVruJm2+9pGNYOzQOV32A==,type:str] +victoria-agent-env: ENC[AES256_GCM,data:kkbtEi4nVqv7jvL0Y3XCsil0InrVsL1zasBcOSKpA32ix73q0bRIwT8JVXX5hI6G8REYvp1oGNRlMBmSPy7H5YxxUd3RsK/5SbLkPfifTAxl8qz1STKPOmq6dZn016809AIkBiDWuu4FOHvzoSVkaO5EXluzb9wVwGH1HM/5pYBDFzqtkgWbCGbeGRXComgD4bqlFjDg/HRX2CaZf0sBByFQxH+NFvXCwdBeEZ82x5gBmXsaPnsAmh8fJotK7Eyu4+Urnahcp8V/4cTadhvg,iv:eizMQCL7vuTn3F0lY23fQfsvxiE8P3CdHqImth9X4JA=,tag:BMltfLO93YMEKv3sycDF2A==,type:str] sops: age: - recipient: age14grjcxaq4h55yfnjxvnqhtswxhj9sfdcvyas4lwvpa8py27pjy2sv3g6v7 enc: | -----BEGIN AGE ENCRYPTED FILE----- - YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBhaFZ6dXVqbHppZ21NeWFi - NU1kV2FOUkVDWlc4RVdUdGNZU21jODdUUXpVCnU4aDdTbXlWNlZ2OUVaOGEwcDkr - aGNrSzVuZTVyU1BMNXArZTBSWnBacG8KLS0tIFVCMUNDeWdTL0VxLzdYNjgyMHY3 - QmNCNGdaeFRHOTY4S2Z4RE9LZVB4TG8KQAe6ensRM4QVJKnDgbnFk9ZYoLk4L7iQ - 4V8jODObt9m2WDCqASJdt/8m/l84E7FTw4g9aimr4fBOU4zOqpGe2w== + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBhSjNMUlh2TFB4L2JBSWU3 + U2xyQTVoUmlQYUV5R1Y5MWtOa3VSVkkyTURzClN6QVFpZjZDdXdrU2ZUWDUyZDN0 + S2RIY1pjdk9vSjg2cUZyTUpQbE51a28KLS0tIDFxSUNiZTQ3UWMySnNCWVZ0MzNZ + NnI1RFhXN3pqdTYyQ0NFZG95UHdvWkUKD23hIRYhZ0BRamLcjfKOpcHIzeatwEFb + 0dvIm8RWcZEYK4w96GY5EQPkJsvcoBNWIyMFrJkgtcRfie3+W6kdag== -----END AGE ENCRYPTED FILE----- - recipient: age1exny8unxynaw03yu8ppahu5z28uermghr8ag34e7kdqnaduq9stsyettzz enc: | -----BEGIN AGE ENCRYPTED FILE----- - YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBCYnhvamdLYjN2SG9DRVl1 - aEdGSWxQTUwrSTBtQURLS3RiQjl3by8xN1hzCjVLc2hQV1dCbTd1UkRaUC9ZN25V - aDJMczFSNGhCZTZFWnpLSW9sSENoWUEKLS0tIG5wRnBvRytKTXo5TXZ3bEdlVkVX - bDF4UUR5bE1JK0tGejJWanFkVDJCMUUKuIbnEMXPsDkJ2eDriJ6gKmwC4gj4ijfU - vmuAEsdpjo2UzP2Sjvm6tFSK85LvZi9lDu5NXdVTdwcq13+OYX1TLw== + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA5dlY4c1FPU21VbkI2bzMx + WS9BNDJSU0JyTEVvY1VFQkU4L3J4U0hrd0NZCmlkQWw1N3RxU1MyZVVuNVZpVStk + UXlyLzlUaFlWTEdDdUZBV2Q5RGpvV28KLS0tIFRHTkwxWTRuMlFJWTV1SVQwU0Ji + c3JnQmlPd1JMQU9pcnVoaG9teVlBWWcKMOIDN36cLqOemzP62uRgKe/myhRs4F4g + MiNVYUL+d3WR3HQEvhovUU5RowlCD84U6Za8z+ss2sApJ6jdUcSP5A== -----END AGE ENCRYPTED FILE----- - recipient: age1v6p8dan2t3w9h94fz4flldl32082j3s9x6zqq7u5j66keth9aphsd6pvch enc: | -----BEGIN AGE ENCRYPTED FILE----- - YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA1aHJlbDFBZlAvbjhRNHkr - angxd01kWkpVckZHcXA3VzJCa0xRSEpuZWlRCjNrUVV5NnM5VWV0YkRmQnE4aU1y - SDc1eXBDM1hsZHh4UkFYREZmTTFLMW8KLS0tIEJZSmhWa1l2SFpqdXVQWHZuWkc2 - ZFNPcDhyc3dIaWx6TlhTMlIrK1NLWmsKhMpNkZEvlaIKWauZsVoLzwYWx0k1sbmk - KO+pNAUz2RMX+N4ykCRgFfeV6SDMxbaOACFm1/6yyDXHgvaI7zrQTQ== + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBUMS9CbzFrZENWam5BOW5t + K3ExNGplNWZLNTVqMndnQ2ZSOFVIL2YvdHlvClkyS1RvZ3FaQWVkNjlWWExkbDZQ + NFVPWmkyNkdoTXdjY0xvS3QrMmFLeXMKLS0tIDhHV2hmMmIzY3dKeTVsUnB5ZGhR + cXpPcklDUDRkcytIcytDY1NYYXNQcTAKvjGjCCrj//KRijPCLOEDAxiexY0CnnqY + 80l7bSnQPOOs27HO/9YlJerQlhPBRZo/KJ+cY3T2fAIfad/f7XN04g== -----END AGE ENCRYPTED FILE----- - recipient: age1ylrpaytkm0k5kcecsxvyv5xd9ts4md0uap48g6wsmj9pwm4lf5esffu0gw enc: | -----BEGIN AGE ENCRYPTED FILE----- - YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA1WkQxb3h1aUVDVGJTMUd1 - RWdXMUQvWUk2TDYwbzd3ZW5ZSEVGZFhUOVg0CmxEd0lOV3EvN014NUFQUEdaWjNq - ZVE5MGZ4SVpXNTRwRnplMHhrQlBHMm8KLS0tIFpUeWZQZmQvWlBrcG9oNERIVEZt - UlBxcFgxVlc2WFRxVkhjWmZCdUdjSEUKPbadYyvWy0Kfbs5EovcpL3Aukj7wiZPJ - VlDkELrTtbvFYp1aAASFZOQb+0NYUyOCtM/OI5qNYigR7TgJBAf/Ig== + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBaMHUrQWZMNmxtSWtxVllM + U1VsMGxVOGt4UmtJSTE5TUdQaEdXUVRIc1FBCk1vTlU2c0lHUDh4VmYwMlFwZU44 + Q1p6N1l5ZWlveGY3VFZJNDNpTFpqNEEKLS0tIHJxdlJtTDkrWHFTcmZXbnVCQ0ti + K2ovOUo4UTN0Y0dubllQM013SW05QmMKQUN03J+ju/JVs6geEMWLnnnlzjPfZOqS + rn3ldZv1aC7ckAp2JiVoznkpsBHFOV8T0pxE9vnmDipAU263k/4ktg== -----END AGE ENCRYPTED FILE----- - recipient: age1jyeppc8yl2twnv8fwcewutd5gjewnxl59lmhev6ygds9qel8zf8syt7zz4 enc: | -----BEGIN AGE ENCRYPTED FILE----- - YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAwcnZrUklWYUZBbm11S2RX - b0F6UjNGQVlXQm83NDBCTFoybHhhTWd6NWs4ClM2dXNXT3pPZkpOYnpRR1BwV1k1 - NWlXNXpBeFJHalhxaExybTJ5NG5nMlkKLS0tIDdXa09CKzdVeVp6TFdpei9PempS - dVNPL051aXpYN09EVVErZkV6czI2dkEK2UZPimVhLwjgjXjj1m7Qc/w36xYDe7sQ - D/5gub0EFuycIQj3lw0y59Jds4GoxBImExC0AKIaX9XBGW1BfBFtPQ== + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBwa1hoalhrQVdsc0NrNzN6 + ZU5JVTdDRjhBWFUzMXhJM2FiaU5HcmlvZzE4CnJ5NDlUYmZmbzZsdC9jYXJBc3Rx + c1RDZDFOdnVHYjZSc2VNc2tOTGF4aTgKLS0tIFhEK1JaRWk3VzhtUE1GYUxoOVJD + ODRHRUxheVRuaEVoZ01sQkNOZGkzMUEKUBpUd30EjSkRFK2cbCARdycf9hHamoVG + XjCfIf1BLGe76+c88zDKaPvp/iAWfGypQkA71tRUoe6pEtrAT8sRgQ== -----END AGE ENCRYPTED FILE----- - recipient: age14uarclad0ty5supc8ep09793xrnwkv8a4h9j0fq8d8lc92n2dadqkf64vw enc: | -----BEGIN AGE ENCRYPTED FILE----- - YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBKM29sYklYMDUxTGJRZGlw - V2tVQTFaZXhoa0xSb2VNZldpUER5a0hWQVVVCktiVDRORDBPM25xSjVucTFsOUha - dHdVMjMxWDcvbVVoTDR5czFrMnVXd00KLS0tIEc2V3RtUVJBbHpKWkhneFRzcCti - NkN5SkZNWlp2dFBmclNjcW1iWXZZY2cK6fqN6xbVFLSTRPfDhvRALVt1yxizvyzI - C32AxiKQo9XrXGBmD5Zi6dtTy+Kdm4PqZjk7M1vW0LOJCErlVI0AjQ== + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA4a1d1UVVmNDVkaWtiOWlN + enA5RS9WSGgyT0ZqNWNvZ0lVbGpkcWxESzNJClFMMlJJRDJDSnRyVVZmRUZIU0JU + RFlTUm1nSzBFWVFEOGtROFVWMit1R1UKLS0tIEtRRXZvZFBKa0kxaDRXazJEcW1P + ajdFVHVpSWtjVU1ZSmlmUmxIY2NxMmcKVMc+JpeCxXs9RGjg7RN6c1TC9ndIOvWw + uFCTBRbTZJHyDC5fYrQQdhMQprm8UT3Fr51i1RWVWjsHz8GZEBwQVg== -----END AGE ENCRYPTED FILE----- - recipient: age1wq82xjyj80htz33x7agxddjfumr3wkwh3r24tasagepxw7ka893sau68df enc: | -----BEGIN AGE ENCRYPTED FILE----- - YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBNT1dBY3kycWVINUV1Wm93 - VEMvNTRSQzV5U2N4ODhiSERHcnVVK3c3Q0hFCmxwWXE3Q1BCRFpIMUlFemRQOTRB - TERiVzdOZGZDbmxXT2ZYZHdhY0FpOEEKLS0tIHRVVkYvVW9nYnJ3WkFhNEQ4UkVY - US96T0JUUlJjVFcxc1ZSUjY1aVg4NmMKc4jD2CtrKc43wArBm566h117ko1vrsHG - ONq9zOp4z90WPvY9octFOEn9cZ2tJvcGYDhWyBGH3rVRYN5hzTRLdg== + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBtTm8xeUlpbC8xRndpSGQw + S2RDSE93ZG8wcnlHZFBXbjNzMXlpeWRIVjBBCnlVRDMzL2paRmFWL2toZVFBOFRk + bi9LRjhWM2ZBcTBFMk9ydTFhaTdrblEKLS0tIFQzUjRpSGR5NzNYcFNVNUNVUks0 + R3AreG1Cd3d1NDNIWnlkcm94L3h6NGcKurqFEsNJklMgu3G7mDNq449o2vMX/2t1 + aUOLRLzi5GTJoQzmuJbveQA+HIgv/B9aNC762YH6df4N0Yk1GRt7pw== -----END AGE ENCRYPTED FILE----- - recipient: age12msc2c6drsaw0yk2hjlaw0q0lyq0emjx5e8rq7qc7ql689k593kqfmhss2 enc: | -----BEGIN AGE ENCRYPTED FILE----- - YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBUd241b3c1MFUvUTRRc2R3 - dFNLREZ0Qi9kcWlVait0bVhJZ05PZ21ScEFjCitab1NaWDlEWkEwZlpzTmFFQ3RV - YTVWUmx4VEJ2Q1J1TllZZ2VTaVF0OFUKLS0tIGZxdGNaM1Q0Z1JEbkFpQm0xaW5T - bHkyeHVtbCtzb2pKOTUvcm91MnBockkKCrwgT0auKuuRYNwajiqBVzpgG13li8KO - W52Fy9lIc9poyxEnvyIqpPz29gQdL4aqGHX735f2+n2fq/SIgiKHfw== + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAwUHZ0Yk14c2NBQm9SelNR + cjY0K0xYUWREUkhWQVNRUVFUczlvZjZ4V1VnClV2b3hhMnlhTUt2U1FLTERlVThv + K2Nqd0dPVWZLTXI2Y2VhaG9Yano1d1kKLS0tIExRY3NlZ0x2VWtDZWhJc0dyM0ow + dlp5SzFqZmNncGQzYzR0THkyeVR6WXcKUb9DJ1u3/VnYKXIgzpTdImSgE2lWdCOD + yVOJXwY5IrU6NJgBYbZlbIysIS+ihO9pLodjaYhbZduNCasdrhwu7A== + -----END AGE ENCRYPTED FILE----- + - recipient: age1x3elhtccp4u8ha5ry32juj9fkpg0qg7qqx4gduuehgwwnnhcxp8s892hek + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA5cFJka2xuSDhyaXowSzVw + cXJSR1ZUWnhDdWRwbFpmenJDZURmbXhDNVhRCm1GMTZ3VHA1S0o3ekVsR2loS3Bi + SmJrOEFBUzJZVUlxazh1TVhkZ0kzVm8KLS0tIDZwajVOUWlIS1FHWHNpZy9jSXhK + RHBjNjRBRzVaUlMrL0d5ZGZTdUcvYkUK0hf09zou0OqZSV81G8HMeNRx+uz9Nade + dhyQ1CRVWT5Q5HW/4t/abB0hSxFId8+X7ufJIjjVSbywLZ7WHLKCSw== -----END AGE ENCRYPTED FILE----- - recipient: age1xcgc6u7fmc2trgxtdtf5nhrd7axzweuxlg0ya9jre3sdrg6c6easecue9w enc: | -----BEGIN AGE ENCRYPTED FILE----- - YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBrNzB5bHVMU2MySmtaOHA4 - K0N3RnZINWpTbkxiTWJnVTZ4Z3d1L1N3T0hjCmxieHdhc1VFWXM1WHExdWNPdW16 - ZVQwTzdGcFVWamMva1RpZGV6U2hpSUEKLS0tIER1T3ZJYUQ0RE5TVTk2dVdmYXhU - eUNKWkczb3dwMnB3SS8rVjRHU1VBQkkKOdh1+qrx9WX0NGSVrGdptFNU8C1ZZFNi - 3lJmzNvJRyVMxXNjWqpXUfCDWQONvCKEGuAvt7Zra+z7/2GtcFTNZw== + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBndjYrRnFRd2RDeTRDQ2Uz + UXk2WE1ORERTOWtOcXI3eXVHZzJJQ2RMRTMwClBPR2J6cmMzSzR1cXJWaHJJUlJa + ZVd1S3hpWEdFYXpvUXd3ZUs3SVJrZHcKLS0tIGN5MUFET0M3WWZkcHRlV1QyTEQx + eTFJUnZpM2haMUZBekh3SjBQRGFsQUUKUI4laym7zGzu1iIlMYnnKjyDHUqblcn6 + K49AdBT+U8WUnVNwAOc5Irf86GfNvJ0S0qk6+v4CpPDgKFksc3flFg== -----END AGE ENCRYPTED FILE----- - lastmodified: "2025-11-14T12:05:25Z" - mac: ENC[AES256_GCM,data:DuD/sqYIwfW2XW+QojfuwLOKI+Y3lXCYOmI5ayCqQlWU9F0W3H1WSnVyfp4YanwY9zjNJukZy9vQNB7wk7DoPGrYTgPihEFmt1GWV10kCFhOxQCMaB9Pt8a7JucF1k8+4jbT4SW282KLAoITDqIAe4qXBPEKKx1+SH34VhKZh3Y=,iv:j5U41i0LHVntwyhTjHExkS65K/gWpBdNy68Cb+bIXmg=,tag:Wu46HI+T+ZzeTb4Rf35vdg==,type:str] + lastmodified: "2025-11-28T22:51:04Z" + mac: ENC[AES256_GCM,data:pXXNkrHSC4mp+K2NfiJ7RucJ299BkVv7wQiWlV2SboLwS7le753CKf2yBGsOYEgxelhLAnIFW3biDR/v+P+7kvUinRUPIYyxeYzMT864Rz8Fb+FBjf4Bd/j/oBdSWk1QM6aOPQRcAiNAol+AcdajhSmy68prUp/lR0gvDPGuczA=,iv:t18AtziHtA58uMoVSsDXsn4PPQUSJCNCG6jg9VBvM0w=,tag:RKQRWjI3NE3Rzb+JdMrOng==,type:str] unencrypted_suffix: _unencrypted version: 3.11.0