add loki and promtail to web host

.sops.yaml

@@ -90,3 +90,14 @@ creation_rules:
     - age:
       - *dominik
       - *web-01-server
+  - path_regex: utils/modules/promtail/[^/]+\.yaml$
+    key_groups:
+    - age:
+      - *dominik
+      - *git-server
+      - *web-01-server
+      - *home-assistant-server
+      - *ldap-server-arm
+      - *ldap-server-test
+      - *testmodules
+      - *netboot

hosts/web-01.cloonar.com/configuration.nix

@@ -13,6 +13,8 @@
     ./utils/modules/nextcloud
     ./modules/postgresql.nix
     ./modules/grafana.nix
+    ./modules/loki.nix
+    ./utils/modules/promtail
 
     ./utils/modules/borgbackup.nix
     ./utils/modules/netdata.nix

hosts/web-01.cloonar.com/modules/prometheus.nix

@@ -0,0 +1,248 @@
+{ config, ... }:
+{
+  sops.secrets.alertmanager = { };
+  sops.secrets.hass-token.owner = "prometheus";
+
+  # imports = [
+  #   ./matrix-alertmanager.nix
+  #   ./irc-alertmanager.nix
+  #   ./rules.nix
+  # ];
+
+  services.prometheus = {
+    webExternalUrl = "https://prometheus.cloonar.com";
+    scrapeConfigs = [
+      {
+        job_name = "telegraf";
+        scrape_interval = "60s";
+        metrics_path = "/metrics";
+        static_configs = [
+          {
+            targets = [
+              "turingmachine.r:9273"
+              "bernie.r:9273"
+              #"rock.r:9273"
+            ];
+            labels.type = "mobile";
+          }
+          {
+            targets = [
+              "eva.r:9273"
+              "eve.r:9273"
+              "blob64.r:9273"
+              "matchbox.r:9273"
+              "alertmanager.r:80"
+              "prometheus.r:80"
+              #"rock.r:9273"
+            ];
+          }
+          {
+            targets = [
+              "rauter.r:9273"
+            ];
+            # to make it compatible with the node-exporter dashboard
+            labels.host = "rauter.r:9273";
+          }
+          {
+            targets = [
+              "prism.r:9273"
+              "gum.r:9273"
+              "kelle.r:9273"
+            ];
+
+            labels.org = "krebs";
+          }
+          {
+            targets = [
+              "clan.lol:9273"
+            ];
+
+            labels.org = "clan-lol";
+          }
+          #{
+          #  targets = [
+          #    "dev1.numtide.com.r:9273"
+          #  ];
+
+          #  labels.org = "numtide";
+          #}
+          {
+            targets = map (host: "${host}.r:9273") [
+              # university
+              "amy"
+              "clara"
+              "rose"
+
+              "astrid"
+              "dan"
+              "mickey"
+              "bill"
+              "nardole"
+              "yasmin"
+              "ryan"
+              "graham"
+
+              "astrid"
+              "dan"
+              "mickey"
+
+              "jackson"
+              "christina"
+              "adelaide"
+              "wilfred"
+              "river"
+              "jack"
+
+              "ruby"
+            ];
+
+            labels.org = "uni";
+          }
+        ];
+      }
+      {
+        job_name = "homeassistant";
+        scrape_interval = "60s";
+        metrics_path = "/api/prometheus";
+
+        authorization.credentials_file = config.sops.secrets.hass-token.path;
+
+        scheme = "https";
+        static_configs = [
+          {
+            targets = [
+              "home-assistant.cloonar.com:443"
+            ];
+          }
+        ];
+      }
+      {
+        job_name = "gitea";
+        scrape_interval = "60s";
+        metrics_path = "/metrics";
+
+        scheme = "https";
+        static_configs = [
+          {
+            targets = [
+              "git.cloonar.com:443"
+            ];
+          }
+        ];
+      }
+    ];
+    alertmanagers = [
+      {
+        static_configs = [
+          {
+            targets = [ "localhost:9093" ];
+          }
+        ];
+      }
+    ];
+  };
+  services.prometheus.alertmanager = {
+    enable = true;
+    environmentFile = config.sops.secrets.alertmanager.path;
+    webExternalUrl = "https://alertmanager.cloonar.com";
+    listenAddress = "[::1]";
+    configuration = {
+      global = {
+        # The smarthost and SMTP sender used for mail notifications.
+        smtp_smarthost = "mail.cloonar.com:587";
+        smtp_from = "alertmanager@cloonar.com";
+        smtp_auth_username = "alertmanager@cloonar.com";
+        smtp_auth_password = "$SMTP_PASSWORD";
+      };
+      route = {
+        receiver = "default";
+        routes = [
+          {
+            group_by = [ "host" ];
+            match_re.org = "krebs";
+            group_wait = "5m";
+            group_interval = "5m";
+            repeat_interval = "4h";
+            receiver = "krebs";
+          }
+          {
+            group_by = [ "host" ];
+            match_re.org = "nix-community";
+            group_wait = "5m";
+            group_interval = "5m";
+            repeat_interval = "4h";
+            receiver = "nix-community";
+          }
+          {
+            group_by = [ "host" ];
+            match_re.org = "clan-lol";
+            group_wait = "5m";
+            group_interval = "5m";
+            repeat_interval = "4h";
+            receiver = "clan-lol";
+          }
+          {
+            group_by = [ "host" ];
+            group_wait = "30s";
+            group_interval = "2m";
+            repeat_interval = "2h";
+            receiver = "all";
+          }
+        ];
+      };
+      receivers = [
+        {
+          name = "krebs";
+          webhook_configs = [
+            {
+              url = "http://127.0.0.1:9223/";
+              max_alerts = 5;
+            }
+          ];
+        }
+        #{
+        #  name = "numtide";
+        #  slack_configs = [
+        #    {
+        #      token = "$SLACK_TOKEN";
+        #      api_url = "https://";
+        #    }
+        #  ];
+        #}
+        {
+          name = "nix-community";
+          webhook_configs = [
+            {
+              url = "http://localhost:9088/alert";
+              max_alerts = 5;
+            }
+          ];
+        }
+        {
+          name = "clan-lol";
+          webhook_configs = [
+            # TODO
+            #{
+            #  url = "http://localhost:4050/services/hooks/YWxlcnRtYW5hZ2VyX3NlcnZpY2U";
+            #  max_alerts = 5;
+            #}
+          ];
+        }
+        {
+          name = "all";
+          pushover_configs = [
+            {
+              user_key = "$PUSHOVER_USER_KEY";
+              token = "$PUSHOVER_TOKEN";
+              priority = "0";
+            }
+          ];
+        }
+        {
+          name = "default";
+        }
+      ];
+    };
+  };
+
+}

hosts/web-01.cloonar.com/secrets.yaml

@@ -2,6 +2,7 @@ borg-passphrase: ENC[AES256_GCM,data:V77hfP5jk/DXcvRiZKu6RLAqsJhlIelkQwA6ClYJKNm
 borg-ssh-key: ENC[AES256_GCM,data:7F7uUlTP3ZKkpySj6/AGfH3K1/8/GzIdfp+ch1hU55zX51KgRs/KuGmj+yKyH9ua41oR4FR94MoiTb3u3MRpUj6dqO4VjVm6fRgJFNXOBhTUcelRR98Nq2QClkRqcNmPiHQC4bxjyW9C1tZfCA52AIILGh9O1Q9XnYAz2q8JwbrY+eTXS/U/1Fh1D+0Y9q1n+oingehal2huHzeVsLxFlip3TmGJx8QBlnAbANABKrMFxkHAlkvAVCrF1MULzeDeeWscHi5T80OvJfOZBSonNEZosw6QJVscMYxh047Yyl6YJ/sxIJjZjC7GiLuG/FC0FCLKRhD8PkZFjrDt/6xwrqePxIb7yIKZRpsvNVCplDDdVOB/V0AqRWAeZh3z813Zi+tnjynHPYaphso/qY2r/HZKlGInzW+QCUBdPVifhVL+y2TKytCWcp4A+c/3Dc2Ut59sfO+hqE946nYGl2S+kpNASHhBa7o6cnaMtfz8NT6rVtYN/3l1snlxeOUZo85XAuYCIerGsMkdVENg5RIJwIzwM0oaGwNzruq8cul5MfAf8hFMXpoLggYECynHk9TNdhsNtUzmsS9cXAyPnnzZT6HGI2/5cgU1weCHbbXAq+ZU9WZwOT41Fpwda/WrNJuMJYFCOFer2hcSLEyxsCfqmPSdwpYRRSYHiaCuVghV6lWyi1IVV5EsX9H6hD+Uetux1SEN95ga8edME16W2dubA0yukcOq+XHI7PMEHAs20H+dybsmVx1XjIsiV/XBWkrooFBXQp950p93XOK49vwNmHNohhCvmERkH0dczGQ5vTAMXwIYqydTqXWERDRzJVvK+pbzKiecEzhFHFge345poUKQS7BWNsv+eehgBAH4HEddtzzzjcPGYMHAhafAm1VqO2BvKg2r3XnzElUCjxWlGfrMkU+LhIgAjom8Mk9+Kha+OD84+iZnuoq73TjpqaUgC0TbkxLAm0DPgk8LsLE9XfJaroftKnrE7P7kyNGSQZfGYcl7ssLck5LJA07dTvDtytlL1Kk8RJe4FwFNPLtdiAnCVvKBe0kcClvCWrrxRt1QT85b+9IwG1yhRxlxheleePX1Fwlg/d2xj6AB3Cz7kAVO2phGPAF16Ke/UQeMxhbOIcy20g7RoiCpfx9jvdwExhDmbmUR7cleL1seFH+wgAM6uQwh80+q8IDJWPNyeXOwiA2siPwexb20xN+n1kIMa03U5Bn+lBqYoh0xXhlDHP3FiKew6xjSducNUtzyKm24kQaWSOfYIqTxYFRTI0bDgRu6xWGmZL37VSMWqDj3TIWOMBoLy0JBsRl6Jj07keJ0CwaNEa4cCWZ0s6nu49mp04JfJWF2r9ksX+2OVeDjSgX5JBB/V79j3I8iu4Nlp0pjyBKPlST/FUDAl1jT12X1grkYjXO8UyJ9AQqqnqK5lM+KYVR15Ui1lBb/vISPqiiw45bcUyHfVAAt7hcFpYLaYB0om3mernccN2fi26lawhhambRd7FGkILKA3byE9ytqGvDQ2CxtjA30kbGxeqXVFsyzROahR0c3KdKlnuCO9Uar4J5VEXgv2obNlNUfSMa9uleWDuhveBaE+2jtKUJd2P4wIIzF/VJGxgWGSge/ji0EV36EFfMg/Tyizdw5wtv4rQF0M+Uu6j/n/l62SmHnT/30H8IFCFuXWmtEo1xcssMymY4ricU3kJIgjGO9h+DrBP1GBczj7yVjLHTpEhRF0yP790xgsJrP4IQB2lOtGf5MCXLrBDYkOZ5xM3+Rq5ZNH0SAHRU/qtFUmLtfkcidjkPiwdqJ0e1LUtLwmSlsot1CkPs7hKORassyUrug7dCtv9QjRMDbtW61PlIbXqa59Aimql4IUcWyUybx12E8wRqmgcX5kG2wGfd9ZFUj6mXhQINFqChsTjXSavQw3u3m8kvd1mJXfjKS12ajr2X1e5wPDUEpLzc3wTOvVgZizGeykKTcKG+GXSjPXLR61ueAO26rboYiDAeSd73shFX/vvut/aB47kZobMOooljGVtnfZjVGY8dWzdNeGvBeLws7vnFrH1u/WXPxpktGz2/eJ0L8ANJd+BZ2+wC+R3OnHeDiXHfofm3dZJTncgxYPqboKKCXRzMviSCWB3poQTm5vEltsQOR6Lj17UmHu5MX7os7t7TrfW/op4Qko9ViWT5vtrUrCZzVqJS+d8hq4lm6ANMhi3Ql+nIMIxK+hjGZNBuKZEyKY+r+Lhz9E/xdOBGVB8QH4g32DWsYvaHfpcvJC8CkGO1nRIGFyrMc10lrv++XQYtiZ1Z4a0oOtQGAXaPGQTJB4KzwlGWc0+kguV+lK4h0QIvHvuorghYC4EJerNdRUviRxZB5mTDyJc1gGq6YgMr4d/a4tyXwoEzPbPGc/3YJATZPYLXOMaGDo0rd2961CFfrsneElNIZ71DWoy41P1fJG7qrfN0gLjU0aSC1vVnzDM8GQvoJGV35cONT7N7u+hjjFBFciLBNEE1DKge156EnP6VbR2LSptWrHeq3zOe9fN3FFR1WWor+lOl+SFztt7uVHCLuWxYPV+csOeBLQi6OhFnh9r4mLTc43wjPkbuci2jqEVM6Bf5gXQoEGzybhDb/3HfQK00NXovru5uEdREYDaj5NVyzyBhOT29JuOvqX7wEMNDqE//Me6Tx8bU12cuUM7Lne7grgYQMbYfLs3gKzRbeuYCqGCIz6KQMDOZvTR38EXqJxysrPb/HuiPfoSGHoAIHviJeIsy6bF7hKA8BikrMmoNc4762pKNKNzBGR+/HeQ1trDBCbYrTmlqoXPJaeANUZlD9NdwiopTJCBzjP/F61s5bpurAiJF6Ymx5yUHljZGVPFOH/ZawfhEcfYGvMUnAOup+EJCih5WQsrn2vodbVYFJ5GNMrDH9//uMi/cwqFWlBqnKGgnGdyqzXlLYTTMlv7fuh3XxkjRRTh6ZVuLyBqEBnXzSWwlcLOUdw3r/48JJ0JC2/rTLVnb+R7T2/+7uXVQISIrBx1ijtRCzFwaFWAPqsL+nMevAHnjpWl9NvanNWKIPIQ6cHGywxdi/7ynEnCYlY13fyIpagFKrgywsHcHuez8cyNtGLpg6hwbSEeN2wZYld+DQc5UH9pqPvvTuNHu3J62WJHZlkc6yBs/hiZBT/sEZUrL2Bf3SNFjzt/IcNSjlVbpP6dd6HpIH6tuN+lQJNypRe//TgKviHBkN0mzmw7IuRIDz1tcVA28COCo8NkzyD194zxDJ8yYKfKH/YwN5q8/R/r/EuUcY4qbeAokaS7XcIkOp/QFKHVM6AYKZ3SPyB5ZjkHFoHBG7YNDrx8AHDNemG+WUev/flmsEv3ykgTztgOoCmTqH/rduS+BGcKwsNW0iMwni9mUiJfuNdYLqhWohGLWb/mkUVoTgycXtsJ8x5BGkk4wSuGSwGCur8yVel7+Fr6CzsVGiPJOa0RHO6sN+Y9jnSMPIVm+mx16j3Q==,iv:ZGV3C0nvqdEnukiPkeMxDD66OjeXQF4anQLkALmBno8=,tag:ELar6NeP5bjL5L/Z5m7Piw==,type:str]
 grafana-ldap-password: ENC[AES256_GCM,data:hNB6CRtXW98yqUqInD3LsZ75sA+lVfmbooehni0UKL60qE/XCZm5B9JVO9pjxbIYZN6Eu/RFX+9L9cJVa5jnEo2MVeLS4CSjqC8BHLArlOuEdA5v8vqqJofBpBfXXN5Ca5xeUDJKz2HgtoTg7G5nTkegGZPGrmj5QQiL1xzco38=,iv:ViQAPTGxEWnjLkJlGCdCq5wW+fbr/O9er8/71VjL/GE=,tag:+Mow4cw7tvtkXvV2iSHeQw==,type:str]
 grafana-admin-password: ENC[AES256_GCM,data:365efRy8xD7SHBnVz6ZJO3l8/lfiZ5vZPZZbxnUmjKKJTMeebLY+P54moStY0wsbU9vk7sCKATCxrS5xy+FQJSgKLoajfz50OMA4+1k3Shl+skbeIikHKwFxqrljFa6HRQ2HTW6KLDPu6Z5Agkima5xdfrtc5R1SnOFg5b6D5NU=,iv:0yZGZVQd35Itj66Ff5hDfDYYx5xsNs/wc887bgMV1MY=,tag:9t8Iffg7kxSjE5eo7iv/RQ==,type:str]
+promtail-nginx-password: ENC[AES256_GCM,data:zk/Wq+Nss6Md0GdhoOcysPrDBqfoAobmqb4LMDkJBjpCn/mdP3/HPiIYdZnZ0vV0JmYpQVqgVFPMlA==,iv:TA19kKllw0Vco6RRlbW4eUqeGQ0SQJRr/TATmyZBMrs=,tag:10/87/svXdL1hpUcTOtY0w==,type:str]
 sops:
     kms: []
     gcp_kms: []
@@ -26,8 +27,8 @@ sops:
             elpwY3Q3dnRzR0loN1BiVk44TTF2VDQKs8Si2LHZ4L4oQqkYUhCI6affE0aTrWmE
             L+am++gYdygVURIh0Z6ftUuhYHPwhlCgmKxx51mKRV2ydraOdUUw0g==
             -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2023-08-18T23:19:22Z"
-    mac: ENC[AES256_GCM,data:sWtJUW19HleKalg/Mfysk/b0N6YxdFcC/66BLmbcchI6s5MeGMLdYIJkNm7RKRQM5PY25d3saOqvsm5qK+keOBa0H9v0DwmFuS9cBJGa5KV6/IDoMvO8VtgDzCZ9HLtrSVTuh84bv7XL3cRd99BfSlSyHBJRpV7kJTudid2O9vo=,iv:8sOMUnsm8hyJlLvc5zG72wjKXtcbK7qnEd7Og0+yJt4=,tag:4XirU7fx0UmJSNkKgmJp8g==,type:str]
+    lastmodified: "2023-08-19T00:09:27Z"
+    mac: ENC[AES256_GCM,data:4FjX8XngdwYBbifM4xmdW/7a3tf43/AdD6ujpYa9M7c7EJ+4ipf6S/eu1CuVk4XAr84rkCAfF+PpGXWeZCJ47YhbXI3yg6HRjGt//5X4Jn6tUYre8vk5Fy7C3dwDKgqHLqOm0hFE89m82xfkfe6VuDeCSbLFUucEtQ3d+rKcGvY=,iv:ufx9eQNNOXcRQISLvdfLK2RUinQPTgjiYpGUWYiqDZc=,tag:A2MoB+/NUFiEee4nTNpAXg==,type:str]
     pgp: []
     unencrypted_suffix: _unencrypted
     version: 3.7.3

utils/modules/promtail/default.nix

@@ -0,0 +1,98 @@
+{ config, ... }: {
+  sops.secrets.promtail-password = {
+    owner = "promtail";
+    sopsFile = ./secrets.yaml;
+  };
+  services.promtail = {
+    enable = true;
+    configuration = {
+      server.http_listen_port = 9080;
+      server.grpc_listen_port = 0;
+
+      clients = [
+        {
+          basic_auth.username = "promtail@cloonar.com";
+          basic_auth.password_file = config.sops.secrets.promtail-password.path;
+          url = "https://loki.cloonar.com/loki/api/v1/push";
+        }
+      ];
+
+      scrape_configs = [
+        {
+          job_name = "journal";
+          journal = {
+            json = true;
+            max_age = "12h";
+            labels.job = "systemd-journal";
+          };
+          pipeline_stages = [
+            {
+              json.expressions = {
+                transport = "_TRANSPORT";
+                unit = "_SYSTEMD_UNIT";
+                msg = "MESSAGE";
+                coredump_cgroup = "COREDUMP_CGROUP";
+                coredump_exe = "COREDUMP_EXE";
+                coredump_cmdline = "COREDUMP_CMDLINE";
+                coredump_uid = "COREDUMP_UID";
+                coredump_gid = "COREDUMP_GID";
+              };
+            }
+            {
+              # Set the unit (defaulting to the transport like audit and kernel)
+              template = {
+                source = "unit";
+                template = "{{if .unit}}{{.unit}}{{else}}{{.transport}}{{end}}";
+              };
+            }
+            {
+              regex = {
+                expression = "(?P<coredump_unit>[^/]+)$";
+                source = "coredump_cgroup";
+              };
+            }
+            {
+              template = {
+                source = "msg";
+                # FIXME would be cleaner to have this in a match block, but could not get it to work
+                template = "{{if .coredump_exe}}{{.coredump_exe}} core dumped (user: {{.coredump_uid}}/{{.coredump_gid}}, command: {{.coredump_cmdline}}){{else}}{{.msg}}{{end}}";
+              };
+            }
+            {
+              labels.coredump_unit = "coredump_unit";
+            }
+            {
+              # Normalize session IDs (session-1234.scope -> session.scope) to limit number of label values
+              replace = {
+                source = "unit";
+                expression = "^(session-\\d+.scope)$";
+                replace = "session.scope";
+              };
+            }
+            {
+              labels.unit = "unit";
+            }
+            {
+              # Write the proper message instead of JSON
+              output.source = "msg";
+            }
+            # silence nscd:
+            # Oct 24 18:20:19 nardole nscd[1812]: 1812 ignored inotify event for `/etc/netgroup` (file exists)
+            { drop.expression = "ignored inotify event for"; }
+            # messages from rpi3
+            { drop.expression = "hwmon hwmon1: Undervoltage detected!"; }
+            { drop.expression = "hwmon hwmon1: Voltage normalised"; }
+            # ignore random portscans on the internet
+            { drop.expression = "refused connection: IN="; }
+          ];
+          relabel_configs = [
+            {
+              source_labels = [ "__journal__hostname" ];
+              target_label = "host";
+            }
+          ];
+        }
+      ];
+    };
+  };
+}

utils/modules/promtail/secrets.yaml

@@ -0,0 +1,84 @@
+promtail-password: ENC[AES256_GCM,data:+KjooDZPlJ7UiSPusdzeTP+6DMfmdSM+T5ID8FOWA3u1/PKqPpgrpnKUzdwKJT0KcmaAVQI723Qjsk0q6UTYYfhkLosyAXnAz3/31EylCaJy4M0CG222xnJDjQ4VA4D/Fp/t+zWWShnf1vUL5wXnWw3rfZ5PEZ34U1M9DtP340Y=,iv:OawqwactX5qncggFvZCWna17SEN+pbF2HRSxnXqa9Xo=,tag:wT2BR5zruIOMKgtmWfRBFg==,type:str]
+sops:
+    kms: []
+    gcp_kms: []
+    azure_kv: []
+    hc_vault: []
+    age:
+        - recipient: age16veg3fmvpfm7a89a9fc8dvvsxmsthlm70nfxqspr6t8vnf9wkcwsvdq38d
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAzdXQ0cmJTUUplaFBYMENJ
+            eWM3RklTKzNiVVJiU2c0VHNPdWpNZ1VNV2pVClZxTGN1RG5hT1JqZ2ovajBiVFlB
+            c28xWnNjRG56MUJnaVJxNndheUppTFEKLS0tIHhVQUVacnl1bHdhTGZkQlVJQVlo
+            TEV0MEdnTG9Tek5rOXdEODUwWmdoMW8K2VRvIBDY3o2SD/ToKk0Zvo3uLVF41Vs0
+            AqJND7wbavF9ZFu6XrVfcrxucrvqLsLCmcFzO2fCauSWgd6lcjlivg==
+            -----END AGE ENCRYPTED FILE-----
+        - recipient: age106n5n3rrrss45eqqzz8pq90la3kqdtnw63uw0sfa2mahk5xpe30sxs5x58
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB6d0pDN1J4VzN4WXhMbzRF
+            ODdDWDlkYWdZRHJkR0JXazlKTG11KzNrTW5JClJ5ZmUyNEFkRFNVaTBEdmNzWlha
+            SXpvdWVmcDB5RWlDWWtWejJ4M0pGYVEKLS0tIEdrNVlZZmxyd3Y2aHVLV2lMeFp3
+            NmRjaWRZQTF3MUpHRWJTZ2J6ajhSYWMKlAnM5DgaFC75JmLa18WXO7DJsfLzXbAq
+            jr+FCIFTEUbCAfNeEcqz9hI5MOD7kVGuCJ5ZpgVjwaTRhfgloJI+IA==
+            -----END AGE ENCRYPTED FILE-----
+        - recipient: age1y6lvl5jkwc47p5ae9yz9j9kuwhy7rtttua5xhygrgmr7ehd49svsszyt42
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB2b1NSNzVOdkt6ZXgzb084
+            Z2tXaEpIcXp0dDhjcURQcVQ3dnZwQkpwd1ZrCjJZTkVkOEdwaFpheG9OV2t2SWwz
+            ZFhVN042RDVuN29kL1ZFTGtzL2dPeUUKLS0tIDVwUFEyZ0RHUlM4UW9WVkdzbVBz
+            TnB4YXV2anlpaXpjblQzUHlINHljRlkKhAUhR2YSDd28L3MjtuUTSrHIRcHIzgHx
+            jhb+Ynji43X5CUNeQQJNGPuDRa3sRC+4SWHfVS16GbJTcxmFsSmS4g==
+            -----END AGE ENCRYPTED FILE-----
+        - recipient: age1ezq2j34qngky22enhnslx6hzh4ekwk8dtmn6c9us0uqxqpn7hgpsspjz58
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBHV1hlbTUyYW9vSnBqVVE1
+            VlVmM1JiRTl3NmFBTFJIU2tEUUhmQnkrbm5rCm85R1M3ODhObmtlVThkTDh0ZHdp
+            cjBwek85MnIxOGdqTTJRUGRKRm1TUncKLS0tIFl6bm1CRTg5ak8xc3NOd0xkdnNE
+            dElTc1IzbjVtNmFOU3hRR1hyTkk1V2cKYQvD9SzJhZEc51YiOqc5cSHa8XREVU2y
+            KbItb9QohlTEohmOoGZoP3LQNm4ZbvQ6jyBHLccHpckV84jxv0ljJg==
+            -----END AGE ENCRYPTED FILE-----
+        - recipient: age1jyeppc8yl2twnv8fwcewutd5gjewnxl59lmhev6ygds9qel8zf8syt7zz4
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBoaDV4WUsvdjZPQi9tL3FR
+            N24zSXUxMVJpWkpGRi80VGNSNkR2U1Myc2tZCjNIOVdoOFFjNEFnMUFYZEp4enJI
+            R1o1K2F2L1I0TlJ2amkrbG14RjRRQ2cKLS0tIDZkZ2lhVlZ0L3NkMXdNZmJDdUR6
+            TkZNNzZMMm9OendQY2txbGczam94T2sKBpHDDHP/NRT4yOAD0JDA3TIFUXQWvdXt
+            kYIpOGfq3hwxcGUGMPTz6K/MlzpZGwiXvmVqhTLOj9X2jt+DcWfphw==
+            -----END AGE ENCRYPTED FILE-----
+        - recipient: age1azmxsw5llmp2nnsv3yc2l8paelmq9rfepxd8jvmswgsmax0qyyxqdnsc7t
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBMdEYzTnR1SWpPZjVYZXR2
+            amgzMDZBeXRMdnBwS3A2VitvYlN5cUIyaEdFCk9yVStMVklXVGVPdWs3b3Vjb1Vz
+            bnhmYW0wZU1vOEVtU3NBdFVrSmNjeTgKLS0tIFpIZjJBbHEwaVN6Y1A4NEJPSHNw
+            Z1NxSU9OYzVFbGtwTVJEaGpQOC80TVEKPwNbehHyR9kua8a3xj2bo8CysrPPvj+b
+            0G5HEw5Ol3BKIqg+eNbuNQY4/mC3VZMDbFFQ/GHbWYFXeAwJre5/Fw==
+            -----END AGE ENCRYPTED FILE-----
+        - recipient: age1zkzpnfeakyvg3fqtyay32sushjx2hqe28y6hs6ss7plemzqjqa5s6s5yu3
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAxRk50cnFXVUpRY3JxTjQ5
+            WjFBZlQ0M01RN2I4TUtkeFJVYVlLVFhIYURFCjdWdlkzNXBXUkRPbDY4alh5TC9p
+            VnFKNWZydjlOQWhjTjlYeW9PeWtwY1UKLS0tIEk3eVZzQ2tYTE4rOGhxcWYxQVZy
+            UFJoYklHQ3E3cXJHQU1MN2ZwM2R5RTAK3+F3OcR7beKeF91YCQLwLz5QLZ78xVQd
+            sEKPbGyRiMXrT9wMZz8WjE253P/iaCmaf6wABK2PW64wI36/kR+62Q==
+            -----END AGE ENCRYPTED FILE-----
+        - recipient: age14uarclad0ty5supc8ep09793xrnwkv8a4h9j0fq8d8lc92n2dadqkf64vw
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB0U3hzNDNtZ2ZDMnptM2pY
+            NVhmVjVqTzN1aGxaSUdHQWliMUxsYnNKamtjCkg4a01pd0RDeEhQZytLNzdKb1dE
+            ck4wV2p2VFc3YjJHRFIwbnF0Wms0R0UKLS0tIGZOY3BMekZSVWlxUmIzbENNejc1
+            WDc1QkkwRnJhMTNkVjFpaGJNaHJHVzQKtlXo8afV++E5uwGSPY7RCj8TY0XbQqRv
+            +DygK0wq4hj0IZsR49W42ORSWxAxkFiLMDAr8o0X0MZx226y90NdVw==
+            -----END AGE ENCRYPTED FILE-----
+    lastmodified: "2023-08-19T00:05:00Z"
+    mac: ENC[AES256_GCM,data:BO3WZzW4MzXpOLKTi6vzVq5lFMATANvIH8Kl9HJPH4bRTRf+z/IX3GmmowjOQis8aGnbfbMja6K0hBWMSY0mY7WzMN9W2ARHTgbXre9/5l6PfFrW4q36sLwXOJU/mzLVz4errHSt6A3Te5AOqThlULuJO/F4pPX2i0Sgs2F1tVA=,iv:zOTWgbuUzuIhYbJFKocwEdR9DxZ3enjc2aIchkovfuA=,tag:x+2jEytk5XrSAGWvbB6bKw==,type:str]
+    pgp: []
+    unencrypted_suffix: _unencrypted
+    version: 3.7.3