many changes

2024-09-03 14:47:06 +02:00
parent fb32b88798
commit 92099bd1e9
44 changed files with 900 additions and 658 deletions
--- a/hosts/web-arm/configuration.nix
+++ b/hosts/web-arm/configuration.nix
@@ -1,4 +1,4 @@
-{ ... }: {
+{ lib, pkgs, ... }: {
  imports = [
    ./utils/bento.nix
    ./utils/modules/sops.nix
@@ -10,7 +10,7 @@
    ./modules/bitwarden
    ./modules/authelia
    ./modules/collabora.nix
-    # ./modules/nextcloud
+    ./modules/nextcloud
    ./modules/rustdesk.nix
    ./modules/postgresql.nix
    ./modules/grafana.nix
@@ -28,10 +28,12 @@
    ./modules/web/stack.nix

    ./sites/autoconfig.cloonar.com.nix
+    ./sites/feeds.cloonar.com.nix

    ./sites/cloonar.com.nix
    ./sites/gbv-aktuell.at.nix
    ./sites/matomo.cloonar.com.nix
+    ./sites/support.cloonar.dev.nix

    ./sites/cloonar.dev.nix
    ./sites/paraclub.cloonar.dev.nix
@@ -48,6 +50,14 @@
    "openssl-1.1.1w"
  ];

+  environment.systemPackages = with pkgs; [
+    vim
+    davfs2
+    screen
+    ucommon
+  ];
+  services.davfs2.enable = true;
+
  time.timeZone = "Europe/Vienna";

  services.logind.extraConfig = "RuntimeDirectorySize=2G";
@@ -70,9 +80,7 @@
  ];

  # backups
-  borgbackup.repo = "u149513-sub5@u149513-sub5.your-backup.de:borg";
-  services.borgbackup.jobs.default.startAt = "Fri 2012-11-23 11:12:13"
-
+  borgbackup.repo = "u149513-sub8@u149513-sub8.your-backup.de:borg";

  networking.firewall = {
    enable = true;
--- a/hosts/web-arm/hardware-configuration.nix
+++ b/hosts/web-arm/hardware-configuration.nix
@@ -1,8 +1,14 @@
 { modulesPath, ... }:
 {
  imports = [ (modulesPath + "/profiles/qemu-guest.nix") ];
-  boot.loader.grub.device = "/dev/sda";
-  boot.initrd.availableKernelModules = [ "ata_piix" "uhci_hcd" "vmw_pvscsi" "xen_blkfront" ];
+  boot.loader.grub = {
+    efiSupport = true;
+    efiInstallAsRemovable = true;
+    device = "nodev";
+    configurationLimit = 2;
+  };
+  fileSystems."/boot" = { device = "/dev/disk/by-uuid/82F0-EC7D"; fsType = "vfat"; };
+  boot.initrd.availableKernelModules = [ "ata_piix" "uhci_hcd" "xen_blkfront" ];
  boot.initrd.kernelModules = [ "nvme" ];
  fileSystems."/" = { device = "/dev/sda1"; fsType = "ext4"; };
  
--- a/hosts/web-arm/modules/authelia/default.nix
+++ b/hosts/web-arm/modules/authelia/default.nix
@@ -3,31 +3,24 @@
 {
  sops.secrets.authelia-jwt-secret = {
    owner = "authelia-main";
-    sopsFile = ./secrets.yaml;
  };
  sops.secrets.authelia-backend-ldap-password = {
    owner = "authelia-main";
-    sopsFile = ./secrets.yaml;
  };
  sops.secrets.authelia-storage-encryption-key = {
    owner = "authelia-main";
-    sopsFile = ./secrets.yaml;
  };
  sops.secrets.authelia-session-secret = {
    owner = "authelia-main";
-    sopsFile = ./secrets.yaml;
  };
  sops.secrets.authelia-identity-providers-oidc-hmac-secret = {
    owner = "authelia-main";
-    sopsFile = ./secrets.yaml;
  };
  sops.secrets.authelia-identity-providers-oidc-issuer-certificate-chain = {
    owner = "authelia-main";
-    sopsFile = ./secrets.yaml;
  };
  sops.secrets.authelia-identity-providers-oidc-issuer-private-key = {
    owner = "authelia-main";
-    sopsFile = ./secrets.yaml;
  };

  services.authelia.instances.main = {
@@ -150,9 +143,6 @@
  
      notifier = {
        disable_startup_check = false;
-        # filesystem = {
-        #   filename = "/var/lib/authelia-main/notification.txt";
-        # };
        smtp = {
          host = "mail.cloonar.com";
          port = 25;
@@ -164,6 +154,21 @@
        oidc = {
          ## The other portions of the mandatory OpenID Connect 1.0 configuration go here.
          ## See: https://www.authelia.com/c/oidc
+          # authorization_policies = {
+          #   support = {
+          #     default_policy = "deny";
+          #     rules = [
+          #       {
+          #         policy = "two_factor";
+          #         subject = "group:support"; # Deny access to users of services group
+          #       }
+          #       {
+          #         policy = "two_factor";
+          #         subject = "group:admin"; # Deny access to users of services group
+          #       }
+          #     ];
+          #   };
+          # };
          clients = [
            {
              id = "gitea";
@@ -183,12 +188,11 @@
            {
              id = "nextcloud";
              description = "Nextcloud";
-              secret = "$pbkdf2-sha512$310000$UqX35Fh.7uTZLQqD.mk5wg$e139D4g9SGUFc.ZdKt3RAZljC8A7C9nixUQd7rQoHFMKop643SuwfazjNn0ehdyAjydM2zV.KzKnMLgSajo.xw";
+              secret = "$pbkdf2-sha512$310000$jPzRYxmYCCDC/Go0Xti9rg$5K70qyNktBEs6PVnJYMrve4insptBzRD1eTi76zFVnJ2aFEc1.7f3yzRTpQ9HVWfEfxRoowXNMNdLxHeDcbLDw";
              public = false;
              authorization_policy = "one_factor";
              redirect_uris = [ 
                "https://nextcloud.cloonar.com/apps/oidc_login/oidc"
-                "https://cloud.cloonar.com/apps/user_oidc/code"
              ];
              pre_configured_consent_duration = "1y";
              scopes = [
@@ -215,6 +219,22 @@
              ];
              userinfo_signing_algorithm = "none";
            }
+            {
+              id = "freescout";
+              description = "FreeScout Support platform";
+              secret = "$pbkdf2-sha512$310000$5D3wUR7CnuoeHu3eNWfETw$SY0GTnZor3BlZKPyU3evH9QTlQG6Bm32RoPAlUgdIRJ8HmL3jRLVtmPLxOcJj06ZS/dDTRfkYej2RmD5cA3T4A";
+              public = false;
+              authorization_policy = "one_factor";
+              redirect_uris = [ "https://support.cloonar.dev/oauth-login/callback/fryg87l64" ];
+              pre_configured_consent_duration = "1y";
+              scopes = [
+                "openid"
+                "profile"
+                "email"
+                "groups"
+              ];
+              userinfo_signing_algorithm = "none";
+            }
          ];
        };
      };
--- a/hosts/web-arm/modules/bitwarden/default.nix
+++ b/hosts/web-arm/modules/bitwarden/default.nix
@@ -50,7 +50,7 @@ in {
  };

  systemd.services.vaultwarden.serviceConfig = {
-    EnvironmentFile = [config.sops.secrets.bitwarden-smtp-password.path];
+    EnvironmentFile = [config.sops.secrets.vaultwarden-env.path];
  };

  systemd.services.vaultwarden_ldap = {
@@ -58,8 +58,8 @@ in {

    preStart = ''
      sed \
-        -e "s=@LDAP_PASSWORD@=$(<${config.sops.secrets.bitwarden-ldap-password.path})=" \
-        -e "s=@ADMIN_TOKEN@=$(<${config.sops.secrets.bitwarden-admin-token.path})=" \
+        -e "s=@LDAP_PASSWORD@=$(<${config.sops.secrets.vaultwarden-ldap-password.path})=" \
+        -e "s=@ADMIN_TOKEN@=$(<${config.sops.secrets.vaultwarden-admin-token.path})=" \
        ${ldapConfigFile} \
        > /run/vaultwarden_ldap/config.toml
    '';
@@ -97,10 +97,9 @@ in {
  };

  sops.secrets = {
-    bitwarden-admin-token.owner = "vaultwarden_ldap";
-    bitwarden-ldap-password.owner = "vaultwarden_ldap";
-    bitwarden-db-password.owner = "vaultwarden";
-    bitwarden-smtp-password.owner = "vaultwarden";
+    vaultwarden-admin-token.owner = "vaultwarden_ldap";
+    vaultwarden-ldap-password.owner = "vaultwarden_ldap";
+    vaultwarden-env.owner = "vaultwarden";
  };

  users.users.vaultwarden_ldap = {
@@ -110,5 +109,16 @@ in {

  users.groups.vaultwarden_ldap = {};

-  services.mysqlBackup.databases = [ "bitwarden" ];
+  services.mysql = {
+    ensureUsers = [
+      {
+        name = "vaultwarden";
+        ensurePermissions = {
+          "vaultwarden.*" = "ALL PRIVILEGES";
+        };
+      }
+    ];
+    ensureDatabases = [ "vaultwarden" ];
+  };
+  services.mysqlBackup.databases = [ "vaultwarden" ];
 }
--- a/hosts/web-arm/modules/collabora.nix
+++ b/hosts/web-arm/modules/collabora.nix
@@ -6,7 +6,8 @@
    ports = [ "9980:9980/tcp" ];
    environment = {
      server_name = "code.cloonar.com";
-      aliasgroup1 = "https://cloud.cloonar.com:443";
+      aliasgroup1 = "https://nextcloud.cloonar.com:443";
+      aliasgroup2 = "https://cloud.cloonar.com:443";
      dictionaries = "en_US";
      extra_params = "--o:ssl.enable=false --o:ssl.termination=true";
    };
--- a/hosts/web-arm/modules/grafana.nix
+++ b/hosts/web-arm/modules/grafana.nix
@@ -15,7 +15,7 @@ let
      [servers.attributes]
      name = "givenName"
      surname = "sn"
-      username = "uid"
+      username = "mail"
      email =  "mail"
      member_of = "memberOf"

@@ -27,13 +27,17 @@ let
  };
 in
 {
-  systemd.services.grafana.script = lib.mkBefore "export GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET=$(cat /run/secrets/grafana-oauth-secret)";
+  systemd.services.grafana.script = lib.mkBefore ''
+    export GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET=$(cat /run/secrets/grafana-oauth-secret)
+    export PUSHOVER_API_TOKEN=$(cat /run/secrets/pushover-api-token)
+    export PUSHOVER_USER_KEY=$(cat /run/secrets/pushover-user-key)
+  '';
  services.grafana = {
    enable = true;
    settings = {
      analytics.reporting_enabled = false;
-      # "auth.ldap".enabled = true;
-      # "auth.ldap".config_file = toString ldap;
+      "auth.ldap".enabled = true;
+      "auth.ldap".config_file = toString ldap;

      "auth.generic_oauth" = {
        enabled = true;
@@ -47,6 +51,8 @@ in
        api_url = "https://auth.cloonar.com/api/oidc/userinfo";
        login_attribute_path = "preferred_username";
        groups_attribute_path = "groups";
+        role_attribute_path = "contains(groups, 'Administrators') && 'Admin' || contains(groups, 'editor') && 'Editor' || 'Viewer'";
+        allow_assign_grafana_admin = true;
        name_attribute_path = "name";
        use_pkce = true;
      };
@@ -81,6 +87,37 @@ in

      security.admin_password = "$__file{${config.sops.secrets.grafana-admin-password.path}}";
    };
+    provision = {
+      alerting = {
+        contactPoints.settings = {
+          apiVersion = 1;
+
+          contactPoints = [{
+            orgId = 1;
+            name = "cp_dominik";
+            receivers = [{
+              uid = "dominik";
+              type = "pushover";
+              settings = {
+                security.apiToken = "$__file{${config.sops.secrets.pushover-api-token.path}}";
+                security.userKey = "$__file{${config.sops.secrets.pushover-user-key.path}}";
+                apiToken = "\${PUSHOVER_API_TOKEN}";
+                userKey = "\${PUSHOVER_USER_KEY}";
+                device = "iphone";
+                priority = "2";
+                retry = "30";
+                expire = "120";
+                sound = "siren";
+                okSound = "magic";
+                message = ''
+                  {{ template "default.message" . }}
+                '';
+              };
+            }];
+          }];
+        };
+      };
+    };
  };

  services.nginx.virtualHosts."grafana.cloonar.com" = {
@@ -103,5 +140,7 @@ in
    grafana-admin-password.owner = "grafana";
    grafana-ldap-password.owner = "grafana";
    grafana-oauth-secret.owner = "grafana";
+    pushover-api-token.owner = "grafana";
+    pushover-user-key.owner = "grafana";
  };
 }
--- a/hosts/web-arm/modules/loki.nix
+++ b/hosts/web-arm/modules/loki.nix
@@ -103,15 +103,15 @@ in

      limits_config.ingestion_burst_size_mb = 16;

-      # ruler = {
-      #   storage = {
-      #     type = "local";
-      #     local.directory = rulerDir;
-      #   };
-      #   rule_path = "/var/lib/loki/ruler";
-      #   alertmanager_url = "http://alertmanager.cloonar.com";
-      #   ring.kvstore.store = "inmemory";
-      # };
+      ruler = {
+        storage = {
+          type = "local";
+          local.directory = rulerDir;
+        };
+        rule_path = "/var/lib/loki/ruler";
+        alertmanager_url = "http://alertmanager.cloonar.com";
+        ring.kvstore.store = "inmemory";
+      };

      query_range.cache_results = true;
      query_range.parallelise_shardable_queries = false;
--- a/hosts/web-arm/modules/mysql.nix
+++ b/hosts/web-arm/modules/mysql.nix
@@ -69,6 +69,8 @@ in {
    settings = {
      mysqld = {
        max_allowed_packet = "64M";
+        transaction_isolation = "READ-COMMITTED";
+        binlog_format = "ROW";
      };
    };
  };
--- a/hosts/web-arm/modules/nextcloud/default.nix
+++ b/hosts/web-arm/modules/nextcloud/default.nix
@@ -1,30 +1,67 @@
-{ pkgs, config, ... }:
+{ pkgs, config, ... }: 
 {
  sops.secrets.nextcloud-adminpass.owner = "nextcloud";
+  sops.secrets.nextcloud-secrets.owner = "nextcloud";
+  sops.secrets.nextcloud-smb-credentials.owner = "nextcloud";

  services.nextcloud = {                
    enable = true;                   
    hostName = "nextcloud.cloonar.com";
    https = true;
-    package = pkgs.nextcloud27;
+    package = pkgs.nextcloud29;
    # Instead of using pkgs.nextcloud27Packages.apps,
    # we'll reference the package version specified above
-    extraApps = with config.services.nextcloud.package.packages.apps; {
-      inherit contacts calendar tasks deck;
+    extraApps = {
+      inherit (config.services.nextcloud.package.packages.apps) calendar contacts deck groupfolders mail richdocuments tasks;
      oidc_login = pkgs.fetchNextcloudApp rec {
-        url = "https://github.com/pulsejet/nextcloud-oidc-login/releases/download/v2.6.0/oidc_login.tar.gz";
-        sha256 = "sha256-MZ/Pgqrb8Y9aH1vd3BfuPhfLOmYyZQO2xVasdj+rCo4=";
+        url = "https://github.com/pulsejet/nextcloud-oidc-login/releases/download/v3.1.1/oidc_login.tar.gz";
+        sha256 = "sha256-EVHDDFtz92lZviuTqr+St7agfBWok83HpfuL6DFCoTE=";
+        license = "gpl3";
      };
    };
+    autoUpdateApps.enable = true;
    extraAppsEnable = true;
-
    database.createLocally = true;
-    enableBrokenCiphersForSSE = false;

+    caching.apcu = true;
+    configureRedis = true;
+    phpOptions."opcache.interned_strings_buffer" = "23";
    config = {
      adminpassFile = config.sops.secrets.nextcloud-adminpass.path;
      dbtype = "mysql";
    };
+
+    secretFile = config.sops.secrets.nextcloud-secrets.path;
+
+    settings = {
+      log_type = "errorlog";
+      allow_user_to_change_display_name = false;
+      maintenance_window_start = 1;
+      lost_password_link = "disabled";
+      oidc_login_provider_url = "https://auth.cloonar.com";
+      oidc_login_client_id = "nextcloud";
+      oidc_login_button_text = "Log in with Authelia";
+      oidc_login_auto_redirect = false;
+      oidc_login_proxy_ldap = true;
+      oidc_login_attributes = {
+        id = "preferred_username";
+        name = "name";
+        mail = "email";
+        groups = "groups";
+        ldap_uid = "email";
+      };
+      oidc_login_scope = "openid profile email groups";
+      default_phone_region = "AT";
+    };
+  };
+
+  environment.systemPackages = [ pkgs.cifs-utils ];
+  fileSystems."/var/lib/nextcloud/data" = {
+    device = "//u149513.your-backup.de/u149513-sub4/";
+    fsType = "cifs";
+    options = let
+      automount_opts = "x-systemd.automount,noauto,x-systemd.idle-timeout=60,x-systemd.device-timeout=5s,x-systemd.mount-timeout=5s,user,users,file_mode=0770,dir_mode=0770";
+      in ["${automount_opts},credentials=${config.sops.secrets.nextcloud-smb-credentials.path},uid=983,gid=964"];
  };

  services.nginx.virtualHosts.${config.services.nextcloud.hostName} = {
@@ -33,5 +70,17 @@
    acmeRoot = null;
  };

+  services.mysql = {
+    ensureUsers = [
+      {
+        name = "nextcloud";
+        ensurePermissions = {
+          "nextcloud.*" = "ALL PRIVILEGES";
+        };
+      }
+    ];
+    ensureDatabases = [ "nextcloud" ];
+  };
+
  services.mysqlBackup.databases = [ "nextcloud" ];
 }
--- a/hosts/web-arm/modules/zammad/default.nix
+++ b/hosts/web-arm/modules/zammad/default.nix
@@ -1,117 +0,0 @@
-{ config, pkgs, ... }:
-
-{
-  services.zammad = {
-    enable = true;
-    port = 3010;
-    secretKeyBaseFile = config.sops.secrets.zammad-key-base.path;
-    database = {
-      createLocally = true;
-    };
-  };
-  
-  services.nginx.enable = true;
-  services.nginx.virtualHosts."support.cloonar.com" = {
-    forceSSL = true;
-    enableACME = true;
-    acmeRoot = null;
-
-    extraConfig = ''
-      # Virtual endpoint created by nginx to forward auth requests.
-      location /authelia {
-        internal;
-        set $upstream_authelia http://127.0.0.1:9091/api/verify;
-        proxy_pass_request_body off;
-        proxy_pass $upstream_authelia;    
-        proxy_set_header Content-Length "";
-
-        # Timeout if the real server is dead
-        proxy_next_upstream error timeout invalid_header http_500 http_502 http_503;
-
-        # [REQUIRED] Needed by Authelia to check authorizations of the resource.
-        # Provide either X-Original-URL and X-Forwarded-Proto or
-        # X-Forwarded-Proto, X-Forwarded-Host and X-Forwarded-Uri or both.
-        # Those headers will be used by Authelia to deduce the target url of the     user.
-        # Basic Proxy Config
-        client_body_buffer_size 128k;
-        proxy_set_header Host $host;
-        proxy_set_header X-Original-URL $scheme://$http_host$request_uri;
-        proxy_set_header X-Real-IP $remote_addr;
-        proxy_set_header X-Forwarded-For $remote_addr; 
-        proxy_set_header X-Forwarded-Proto $scheme;
-        proxy_set_header X-Forwarded-Host $http_host;
-        proxy_set_header X-Forwarded-Uri $request_uri;
-        proxy_set_header X-Forwarded-Ssl on;
-        proxy_redirect  http://  $scheme://;
-        proxy_http_version 1.1;
-        proxy_set_header Connection "";
-        proxy_cache_bypass $cookie_session;
-        proxy_no_cache $cookie_session;
-        proxy_buffers 4 32k;
-
-        # Advanced Proxy Config
-        send_timeout 5m;
-        proxy_read_timeout 240;
-        proxy_send_timeout 240;
-        proxy_connect_timeout 240;
-      }
-    '';
-
-    locations."/" = {
-      proxyPass = "http://127.0.0.1:3010";
-      proxyWebsockets = true;
-      extraConfig =
-        "proxy_connect_timeout  300;" +
-        "proxy_send_timeout     300;" +
-        "proxy_read_timeout     300;" +
-        "send_timeout           300;"
-      ;
-    };
-    locations."/auth/sso" = {
-      proxyPass = "http://127.0.0.1:3010";
-      proxyWebsockets = true;
-
-      extraConfig = ''
-        # Basic Authelia Config
-        # Send a subsequent request to Authelia to verify if the user is authenticated
-        # and has the right permissions to access the resource.
-        auth_request /authelia;
-        # Set the `target_url` variable based on the request. It will be used to build the portal
-        # URL with the correct redirection parameter.
-        auth_request_set $target_url $scheme://$http_host$request_uri;
-        # Set the X-Forwarded-User and X-Forwarded-Groups with the headers
-        # returned by Authelia for the backends which can consume them.
-        # This is not safe, as the backend must make sure that they come from the
-        # proxy. In the future, it's gonna be safe to just use OAuth.
-        auth_request_set $user $upstream_http_remote_user;
-        auth_request_set $groups $upstream_http_remote_groups;
-        auth_request_set $name $upstream_http_remote_name;
-        auth_request_set $email $upstream_http_remote_email;
-        proxy_set_header Remote-User $user;
-        proxy_set_header Remote-Groups $groups;
-        proxy_set_header Remote-Name $name;
-        proxy_set_header Remote-Email $email;
-        # If Authelia returns 401, then nginx redirects the user to the login portal.
-        # If it returns 200, then the request pass through to the backend.
-        # For other type of errors, nginx will handle them as usual.
-        error_page 401 =302 https://auth.cloonar.com/?rd=$target_url;
-      '';
-    };
-    locations."/ws" = {
-      proxyPass = "http://127.0.0.1:6042";
-      proxyWebsockets = true;
-      extraConfig =
-        "proxy_read_timeout     86400;" +
-        "send_timeout           300;"
-      ;
-    };
-  };
-
-  sops.secrets = {
-    zammad-db-password.sopsFile = ./secrets.yaml;
-    zammad-key-base.owner = "zammad";
-  };
-
-  services.postgresqlBackup.enable = true;
-  services.postgresqlBackup.databases = [ "zammad" ];
-}
--- a/hosts/web-arm/modules/zammad/secrets.yaml
+++ b/hosts/web-arm/modules/zammad/secrets.yaml
@@ -1,40 +0,0 @@
-zammad-db-password: ENC[AES256_GCM,data:FFsTnwQcL8V1ZWvZ9a15FWcHnsrC7nuDW155reSmfg/IRhRfrtnvbCDQ0N3AMh7TBiyG3x5za/6orV04CplUgQ==,iv:inQXkwlTbGaKgU3nfOtIYMcheBdGv8xa7dCad8WrGEc=,tag:fxjNRCUpS6RMipk4D08new==,type:str]
-zammad-key-base: ENC[AES256_GCM,data:z2v1GrjRFoaDY9tPaAsUJPVRHZhSOrXWCZhhm5E6rmH4s6QWU1EW7aY4PPgditdcathLRWkDlBT5c3SQ8Cd2DPLp/SVn9Xd8w8g/lrplhNC2sJXUyB+CUgdEnBBN0XPMsFWNx9EIrqGrF/A8js5eKtQON9fCNytaHMOsCCc0rNE=,iv:oHKiXE9U0h846XVpCrcD/dFJ1MAXCYrnM80CwaWgALc=,tag:W88DsRWvdudMscH+UBPy/Q==,type:str]
-sops:
-    kms: []
-    gcp_kms: []
-    azure_kv: []
-    hc_vault: []
-    age:
-        - recipient: age16veg3fmvpfm7a89a9fc8dvvsxmsthlm70nfxqspr6t8vnf9wkcwsvdq38d
-          enc: |
-            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBUc0RlQUt4VHU1eWZrdlF5
-            UFhjSU5TWFlGbTIwbzVlaStHaWRTdS92d0YwCkJQRlh0eWVNRW9SdUFXQUZzNFYw
-            dktoSmFqbWxDbXR0dDNTNy8zTHYwQUEKLS0tIFFwQkdvK2QvSmFGaVRBaVFMeEFi
-            YUZ6b1dzUGZkL2t4aU5tTjA4UC9KU3cKmhugvvIexQqpVtGp7aLKU7WSQNxk0cTO
-            +8MWF1v0mztJlGbiWk5OOzT9L8TO7GDGXfi8GyMVgVBvaA7tFF709w==
-            -----END AGE ENCRYPTED FILE-----
-        - recipient: age1v6p8dan2t3w9h94fz4flldl32082j3s9x6zqq7u5j66keth9aphsd6pvch
-          enc: |
-            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBFZGRWbnVxVUdHWndEanlk
-            Wmp4WS8yUjdrVSsxTHFNcjFUWm5IZytaZVRzCmorZTJRSnBRTE5qK2xiZGtYNXZH
-            RjBDdWE5NjE3ZWtXRU5Fc2FaVFkzNUEKLS0tIGwvUjVBL2NpdTFsY04zbktJRGxF
-            QWo1Vm56dnZWQ2l1K3hzVlZDL3BaTHMKw9CjtbS9hyW42prUhlTIcmcb4Z6OaxRr
-            T7RJZxXefEr4myJYI5B3pqbXlBpSLLwS4lgtoqHmmYuSNjL8/xoksw==
-            -----END AGE ENCRYPTED FILE-----
-        - recipient: age1y6lvl5jkwc47p5ae9yz9j9kuwhy7rtttua5xhygrgmr7ehd49svsszyt42
-          enc: |
-            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBicStLZGZvdGJyMyszMkFo
-            S2xTeUM5ZEIrbUxqbXBxQTUyeHhJVTAzUm40Ck5KbngvdWYvVk5VYTRCUWhZeFkw
-            eFJKVEZ3VnpuL3BmOFVQdCs2K3hoTUUKLS0tIEhFRXZyRlpPZUpEanFMU1oweVJ2
-            RVJjc0FUb0NFMHk2M3gxTmhMYjlrTDgKR0tfq1CWU8OdeeigOsKqNx2sszVtPWjH
-            yXcqe/jLAnvS/Ut/afEyfGYEiyyzJXLp9TGjV1fAp9y2K2noD8/TwQ==
-            -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2022-11-29T10:54:56Z"
-    mac: ENC[AES256_GCM,data:OX49RTucGWdH1RkbXfkiMLH2Lj65v554WSfJxkCkIu/dFagCH90QSRiX/15HTsI//ffwKVurDivC6H6OByK2eWdaeCYTEn2029GjdL4RhJhXy0RLXEq5D/KVRu73O9Xe6M36asc/OenzPcmbHAvddD14y9vaOsVTL0H15ydVrwg=,iv:+uBt1Mvj+WMM4CvAOwmOXhZJVZBXVDCXA8iSXpdjktU=,tag:AeipsBJ8PA22OfUxXA8bIA==,type:str]
-    pgp: []
-    unencrypted_suffix: _unencrypted
-    version: 3.7.3
--- a/hosts/web-arm/secrets.yaml
+++ b/hosts/web-arm/secrets.yaml
@@ -1,9 +1,8 @@
-borg-passphrase: ENC[AES256_GCM,data:V77hfP5jk/DXcvRiZKu6RLAqsJhlIelkQwA6ClYJKNmMtvAXG+g6794YJ+ooof1h8qcnMoctEWMUcsBetjaguA==,iv:OyJF/dftfEaGUnmbzrcn0P0tvnUZX4l6Vk0Qf0NwwfE=,tag:AAkRMD+jq01BPq2LSYPQGA==,type:str]
-borg-ssh-key: ENC[AES256_GCM,data:7F7uUlTP3ZKkpySj6/AGfH3K1/8/GzIdfp+ch1hU55zX51KgRs/KuGmj+yKyH9ua41oR4FR94MoiTb3u3MRpUj6dqO4VjVm6fRgJFNXOBhTUcelRR98Nq2QClkRqcNmPiHQC4bxjyW9C1tZfCA52AIILGh9O1Q9XnYAz2q8JwbrY+eTXS/U/1Fh1D+0Y9q1n+oingehal2huHzeVsLxFlip3TmGJx8QBlnAbANABKrMFxkHAlkvAVCrF1MULzeDeeWscHi5T80OvJfOZBSonNEZosw6QJVscMYxh047Yyl6YJ/sxIJjZjC7GiLuG/FC0FCLKRhD8PkZFjrDt/6xwrqePxIb7yIKZRpsvNVCplDDdVOB/V0AqRWAeZh3z813Zi+tnjynHPYaphso/qY2r/HZKlGInzW+QCUBdPVifhVL+y2TKytCWcp4A+c/3Dc2Ut59sfO+hqE946nYGl2S+kpNASHhBa7o6cnaMtfz8NT6rVtYN/3l1snlxeOUZo85XAuYCIerGsMkdVENg5RIJwIzwM0oaGwNzruq8cul5MfAf8hFMXpoLggYECynHk9TNdhsNtUzmsS9cXAyPnnzZT6HGI2/5cgU1weCHbbXAq+ZU9WZwOT41Fpwda/WrNJuMJYFCOFer2hcSLEyxsCfqmPSdwpYRRSYHiaCuVghV6lWyi1IVV5EsX9H6hD+Uetux1SEN95ga8edME16W2dubA0yukcOq+XHI7PMEHAs20H+dybsmVx1XjIsiV/XBWkrooFBXQp950p93XOK49vwNmHNohhCvmERkH0dczGQ5vTAMXwIYqydTqXWERDRzJVvK+pbzKiecEzhFHFge345poUKQS7BWNsv+eehgBAH4HEddtzzzjcPGYMHAhafAm1VqO2BvKg2r3XnzElUCjxWlGfrMkU+LhIgAjom8Mk9+Kha+OD84+iZnuoq73TjpqaUgC0TbkxLAm0DPgk8LsLE9XfJaroftKnrE7P7kyNGSQZfGYcl7ssLck5LJA07dTvDtytlL1Kk8RJe4FwFNPLtdiAnCVvKBe0kcClvCWrrxRt1QT85b+9IwG1yhRxlxheleePX1Fwlg/d2xj6AB3Cz7kAVO2phGPAF16Ke/UQeMxhbOIcy20g7RoiCpfx9jvdwExhDmbmUR7cleL1seFH+wgAM6uQwh80+q8IDJWPNyeXOwiA2siPwexb20xN+n1kIMa03U5Bn+lBqYoh0xXhlDHP3FiKew6xjSducNUtzyKm24kQaWSOfYIqTxYFRTI0bDgRu6xWGmZL37VSMWqDj3TIWOMBoLy0JBsRl6Jj07keJ0CwaNEa4cCWZ0s6nu49mp04JfJWF2r9ksX+2OVeDjSgX5JBB/V79j3I8iu4Nlp0pjyBKPlST/FUDAl1jT12X1grkYjXO8UyJ9AQqqnqK5lM+KYVR15Ui1lBb/vISPqiiw45bcUyHfVAAt7hcFpYLaYB0om3mernccN2fi26lawhhambRd7FGkILKA3byE9ytqGvDQ2CxtjA30kbGxeqXVFsyzROahR0c3KdKlnuCO9Uar4J5VEXgv2obNlNUfSMa9uleWDuhveBaE+2jtKUJd2P4wIIzF/VJGxgWGSge/ji0EV36EFfMg/Tyizdw5wtv4rQF0M+Uu6j/n/l62SmHnT/30H8IFCFuXWmtEo1xcssMymY4ricU3kJIgjGO9h+DrBP1GBczj7yVjLHTpEhRF0yP790xgsJrP4IQB2lOtGf5MCXLrBDYkOZ5xM3+Rq5ZNH0SAHRU/qtFUmLtfkcidjkPiwdqJ0e1LUtLwmSlsot1CkPs7hKORassyUrug7dCtv9QjRMDbtW61PlIbXqa59Aimql4IUcWyUybx12E8wRqmgcX5kG2wGfd9ZFUj6mXhQINFqChsTjXSavQw3u3m8kvd1mJXfjKS12ajr2X1e5wPDUEpLzc3wTOvVgZizGeykKTcKG+GXSjPXLR61ueAO26rboYiDAeSd73shFX/vvut/aB47kZobMOooljGVtnfZjVGY8dWzdNeGvBeLws7vnFrH1u/WXPxpktGz2/eJ0L8ANJd+BZ2+wC+R3OnHeDiXHfofm3dZJTncgxYPqboKKCXRzMviSCWB3poQTm5vEltsQOR6Lj17UmHu5MX7os7t7TrfW/op4Qko9ViWT5vtrUrCZzVqJS+d8hq4lm6ANMhi3Ql+nIMIxK+hjGZNBuKZEyKY+r+Lhz9E/xdOBGVB8QH4g32DWsYvaHfpcvJC8CkGO1nRIGFyrMc10lrv++XQYtiZ1Z4a0oOtQGAXaPGQTJB4KzwlGWc0+kguV+lK4h0QIvHvuorghYC4EJerNdRUviRxZB5mTDyJc1gGq6YgMr4d/a4tyXwoEzPbPGc/3YJATZPYLXOMaGDo0rd2961CFfrsneElNIZ71DWoy41P1fJG7qrfN0gLjU0aSC1vVnzDM8GQvoJGV35cONT7N7u+hjjFBFciLBNEE1DKge156EnP6VbR2LSptWrHeq3zOe9fN3FFR1WWor+lOl+SFztt7uVHCLuWxYPV+csOeBLQi6OhFnh9r4mLTc43wjPkbuci2jqEVM6Bf5gXQoEGzybhDb/3HfQK00NXovru5uEdREYDaj5NVyzyBhOT29JuOvqX7wEMNDqE//Me6Tx8bU12cuUM7Lne7grgYQMbYfLs3gKzRbeuYCqGCIz6KQMDOZvTR38EXqJxysrPb/HuiPfoSGHoAIHviJeIsy6bF7hKA8BikrMmoNc4762pKNKNzBGR+/HeQ1trDBCbYrTmlqoXPJaeANUZlD9NdwiopTJCBzjP/F61s5bpurAiJF6Ymx5yUHljZGVPFOH/ZawfhEcfYGvMUnAOup+EJCih5WQsrn2vodbVYFJ5GNMrDH9//uMi/cwqFWlBqnKGgnGdyqzXlLYTTMlv7fuh3XxkjRRTh6ZVuLyBqEBnXzSWwlcLOUdw3r/48JJ0JC2/rTLVnb+R7T2/+7uXVQISIrBx1ijtRCzFwaFWAPqsL+nMevAHnjpWl9NvanNWKIPIQ6cHGywxdi/7ynEnCYlY13fyIpagFKrgywsHcHuez8cyNtGLpg6hwbSEeN2wZYld+DQc5UH9pqPvvTuNHu3J62WJHZlkc6yBs/hiZBT/sEZUrL2Bf3SNFjzt/IcNSjlVbpP6dd6HpIH6tuN+lQJNypRe//TgKviHBkN0mzmw7IuRIDz1tcVA28COCo8NkzyD194zxDJ8yYKfKH/YwN5q8/R/r/EuUcY4qbeAokaS7XcIkOp/QFKHVM6AYKZ3SPyB5ZjkHFoHBG7YNDrx8AHDNemG+WUev/flmsEv3ykgTztgOoCmTqH/rduS+BGcKwsNW0iMwni9mUiJfuNdYLqhWohGLWb/mkUVoTgycXtsJ8x5BGkk4wSuGSwGCur8yVel7+Fr6CzsVGiPJOa0RHO6sN+Y9jnSMPIVm+mx16j3Q==,iv:ZGV3C0nvqdEnukiPkeMxDD66OjeXQF4anQLkALmBno8=,tag:ELar6NeP5bjL5L/Z5m7Piw==,type:str]
-bitwarden-admin-token: ENC[AES256_GCM,data:WWkkhaSwJA423FSeSoEmssACB6qjyM2usKFQhGqzP+es5bIbr4SxpC1vhWHoS3om+OndVsWzQe4NZ9bNvWAefw==,iv:S/JBDXLZDaCG6EvFigIdSv6GvmFAL8w0BJZFYoGgkl8=,tag:bc7bjJUlcyHEsO3AEd4sxQ==,type:str]
-bitwarden-db-password: ENC[AES256_GCM,data:ues1754DstLekOtmjbi1LgpA4nV+4i9xUcUH05xPQSa1osvig1prh3JVnyYxJpy2zOqeRF0adZuRyb7/P/SLpA==,iv:AZG8FGPrcgfgNCtYjCVvIEHI3bkIjWVf82QRJ+qQdRA=,tag:IHnlKpWdyAjrgrzYaJtYiA==,type:str]
-bitwarden-ldap-password: ENC[AES256_GCM,data:gz8ntl7mwA9f2I8LjTR2lBky7J3xYYTyQwXBrunF8/6eEgAme0zxeA5u3DTUrQ4BNfUqPfxHOIX38IxiLKRyzg==,iv:5J+KIER7R+93wdaiK7FAfS5+m8qFDruyTYh2a3n6PIg=,tag:dsT7s2TKWKcwgl3yOE3I5g==,type:str]
-bitwarden-smtp-password: ENC[AES256_GCM,data:og0n7HJhplyAUDY45iuKtjnOOwmW9wD2UUwrt7/Mf/DgWbhLiYJH/NVPiUhSERMimZjTkjuHHp3bNGiIPRojX0ukJTbfiX01/BipDon1TVleLNq/tYB+VjL9KDoYi5Og5gg2ZG0DfXu8IKYshF0UD9gpYHmmxDWlZ+ZTi19cDKkiVErj44ov3Bia7hs22FHqg2J946PmWJbWDTuYKRqyynAoOtfwmrSXVW+Q+xmHNYIfOiNHo/33V1xj0Ldl49g3ry3nFBP9OGnPKOOYmekv14ehJ4eixDuZQT9gpU5m2zdHRAcapW3T8TGZIibOGlMeYRbPzBoISOr+q419bsAuB90lzpGLZfkvriHxuxtpGSg=,iv:WTvc7i4hrDi5aSc+PCL+gTuf4KKZehwk6WfgXumnRPE=,tag:TOHJsAJi2t6L9ahrikS67Q==,type:str]
+borg-passphrase: ENC[AES256_GCM,data:l4yO6pdxeaiyyYhTVpr00HPwCgBsrvJzN+YjQb+sf0HdmSiXpiwVvD7MuVcN1OgdupJ3uVR4sfDa0fM0wXJZjw==,iv:xp8rcuBicg/JhOvBe9dv7vUzqjlp6qAR3/gDt0dX8ZY=,tag:yYXPN8GVbYbi1DrLbDjgNw==,type:str]
+borg-ssh-key: ENC[AES256_GCM,data:bS84F9jvxuUR3190aBOhTuKbC+ieKmTby64PUTKQiWuhI9aChRSrgmPRPEwGW1aV7lWzC3rBmWwC9BRMuL7J0ZqYlijssO0h8cMZ5RR3REhmZf9ypfS015EBmyB3cUk0sYuQSSzNV6yYNvaparPlK+LOqCguWNtpHnstfh1doHbvaFXcm4dULU+n4b5yKu2vOG7c4HOJIefFeN06ZH1ikOtW5u9sVxPVB5vCvyM1GE6A/pBVZN6fNisnPNTUZLEwDnM01FS/jdkBIHJtV44tspWxwUcCySCL04VTa991TGqVFz0LzSByNmdFcpvdzTBYdUu5e6mRXwRZ6NUytRDcAL/iA6IWK9MOceY+dF0ykaR4y7XAs9TnfUk69Lveb9xFF1Tv43EsLW2yLwkm28p2FLgdhv1jck97mQa7wJKU73mV0YLYPkourEeiMMdyYLlK9wfGluAekRwMAxjfmAZzu2+s8JpvNHmggYXV4McvuWkPiF7yOGFeijl6BOaOxJKiNSpdmszpxxmI52X9nO8L,iv:Evd5X9tDKytlfn5WZt8+IOAm0vA2k6gAUYXcZoevGoQ=,tag:SFjOnpeNDufZBvgQjzLJKQ==,type:str]
+vaultwarden-admin-token: ENC[AES256_GCM,data:61qZTUh60L09dAn5Cbj1VVPIrs0xiw5oWDxvJ7CvPHi3urtjnU0QarclgMtYDo7ES0yBV8omFLb27hQZ4OY7ww==,iv:QHtGNhzJiefIWXdaq50dnLKl6qsD3k3zGI7BZ91N0ic=,tag:R3vsVE6vXrB45rb2RaepBQ==,type:str]
+vaultwarden-ldap-password: ENC[AES256_GCM,data:ZHSGJslZLG4Zan+LZo4FxHjVjOv8u/s3Mu2572VR4HyxtsFzzsIU/1G4As0Dj9HnWx9Jad695udtzSsREf2TGw==,iv:ruyPj++6tLxbzRoOTl1bT+4M8ZpV5gu1GLj4EavMuVU=,tag:y3kaiXrGjuhwx2SOJ9ZCMA==,type:str]
+vaultwarden-env: ENC[AES256_GCM,data:wDhfHW7BcO0eGq30jJNT6OLv5+LLOQskQrJjtKzUXQg6ptMGQuPmDlgetUq602HplP5mDWULoleyUBYCQB44FrU2nq+FXCCeUk/5N9cyfCsnXcpF4eZhLLT6jVFfSOZMs6vPYVl/Er+wklwbztguyLMtBHqc4p8XqVSigs8Bod6iLBO7Pbr1qm8gWUNcy++cLl9ZHncpLbcIfIqo/rQXMujXu/+wTSQPwR6x/cRV9zqUdPB2EEOCoGBSbYfHIxLBpOK5nbFoi4m03XHF8e68z+TE2e8KmhqoLH0J8MYGtH50WCc172pDTWzQ17BKr87KrL2YgSqe2g==,iv:XZKq5AC5ziaxtdd5/KVteEY0O7CcF8KKX3V7lyxVR5w=,tag:3r26CTdVvYsuYOam4wwQsg==,type:str]
 authelia-jwt-secret: ENC[AES256_GCM,data:sr3+B5UPtPsAYq8Dwqrbb/hXKuY49nWKhkQ11DGfSSgdIEOnDHP7jnyDCB1Mt536djovmrl1AlOG6/JKyxvakQ==,iv:r/LtU4sef4bwSY+T9TFccZq+bKrcdZ/lPsY9QInQ3xk=,tag:GNC4kVLRuxxShLwIPGKZmg==,type:str]
 authelia-backend-ldap-password: ENC[AES256_GCM,data:36qJ5r/ddjgxzq82/EkvYVM8VAKoHpNUbIKlimm7eABk2FkEw+U/7h5ZLjFPmKtKkbOUSI7R48xY0cKkodKwuA==,iv:jG0rXAX8Yi2okp1Y6ZSiGgSSAVFJakKEI781EpVgOLc=,tag:cPd4wmAaF81KbVsnmIy+NQ==,type:str]
 authelia-storage-encryption-key: ENC[AES256_GCM,data:A0w+CuVEUZZruXYbPiM3Mv7DcsXlu0+PvzLUS0oX71YAX7jnYBrJBFQ+sg7Y19JhQOvugCn2VJoSkcXErPq7Fg==,iv:p90bnFfoXOVEZ+BalN+Qs6PMWG8cIAqHE8jGQAaJAJU=,tag:1yp9z6UyrasKPYHHTRyHlA==,type:str]
@@ -17,6 +16,10 @@ grafana-oauth-secret: ENC[AES256_GCM,data:OXsKChjgnDEKG58LarUpdJlDy4FJTrs1lrHH9I
 promtail-nginx-password: ENC[AES256_GCM,data:zk/Wq+Nss6Md0GdhoOcysPrDBqfoAobmqb4LMDkJBjpCn/mdP3/HPiIYdZnZ0vV0JmYpQVqgVFPMlA==,iv:TA19kKllw0Vco6RRlbW4eUqeGQ0SQJRr/TATmyZBMrs=,tag:10/87/svXdL1hpUcTOtY0w==,type:str]
 victoria-nginx-password: ENC[AES256_GCM,data:+rKDzML5eQX47JF1i/ZU9jwdeLgRXPyzwSCt+iDzsCx8RKSn+omTESs/P4lj9dBPO0zjo6w=,iv:o4JW6EIwTMt3SAqhGscnc9iQBwWr6VYFSIA5sc86+pc=,tag:OvupW1Py8pCu5IAemdc81w==,type:str]
 nextcloud-adminpass: ENC[AES256_GCM,data:/vt17v+aaucz8sq/uYUA0hlj1urKNYcmCN0LbgGAMhWoTiTwzYr5FzrygOuZWZBeaAFH1pWItTZRXj74OX8XqutLPlYDg/jZqLszU0/9HgSBoHb5ZnPUpzIjNI9dpMttPphpo5TVrYKoh/vR3OWjJa3ObcpGLdvMQc1r8ABEvvg=,iv:0xW7++80CwZy0O4J3bFElqp0ZMC+RpO5kcczshM1pzg=,tag:PJj5PHfkoHE8jRbS4mpq6Q==,type:str]
+nextcloud-secrets: ENC[AES256_GCM,data:FwP+z4B03m0VEFEb8c/UwBKMcWXo+2dnlBAuO4SCVXNBLdq3IK+e8gGzKima+sac+WZ3k3ncPAqyIomBLwEmIUB/24xYx4SL6AddwDoyytZbVDv5Zt7Vpvy6aheOvARoqez3pWMaC+rW11JFVw==,iv:BT9eGRUhHMbwkhuQ+cC32zHICRbm2hQQeVfIHrCB+JM=,tag:GNpdz1QYEcfVvmkjFJY1vg==,type:str]
+nextcloud-smb-credentials: ENC[AES256_GCM,data:Ra1iVCP/Y1G87oDrn01JxorTQy6d80POKIVEbHPttrd6x5QgEvvyWIz6rCiK4mEH,iv:6wXHBSwq9P+tHrkB82ZReFXsUOF0rDi2hpZ8jXLU7OE=,tag:Fu4RB0hPyHFpN6YLTtfGDQ==,type:str]
+pushover-api-token: ENC[AES256_GCM,data:itcWlyaJi+saBmhLabOOgbOej9yxQgCIiwU9uuOg,iv:dnD12MPZsENogsnCMGpZe1F0cC4eFfefSx7sP9Fl9Mw=,tag:lk1+pkvNab6yG0Sv/+TVIQ==,type:str]
+pushover-user-key: ENC[AES256_GCM,data:swXKXMAeCyYbBQNAEEpDTJXjdNmFFVWnhExAqfnn,iv:AZd6phibpwEX97U/SzeiRoFFL3TviSONwOWkPsXdcKc=,tag:+mzfrxHpTWOzb3bEzN3D5Q==,type:str]
 zammad-db-password: ENC[AES256_GCM,data:4LkMM06cs9H/ricsE+2LNin8PIn4MLbi+TaYpESeAhUz7M6JFcoLGdn2Rws3crGuCWVLColh1bv0hALLSYQs2Q==,iv:MIufiAixz6wLp1byQ2tiAx27jJGUAnVGs8KLWLaqk+4=,tag:Wbq6V3661r3Ue942q1jBRg==,type:str]
 zammad-key-base: ENC[AES256_GCM,data:IERHJKzK/kRa4P6EfpSzt/9Xj1I0/YGl/Fj8ISA/WQFn4+hu9VqdJzMoVgZexbjhpB+fPWmxwyGBhrsJRf77zJGosRzG+4MPWPw6Yggai6TGbZkxj5St+I7nm9KZbtkCbo3pH3YLXhKCFVZJuSNtBb9Y3sqd0h8XcygMQbaf2Js=,iv:FEZUOBulpPDGUuJztod+r/17MEmojKrOe+HptecMdTo=,tag:ZsFKuUKaCgc01/iDJgbkNQ==,type:str]
 sops:
@@ -28,23 +31,32 @@ sops:
        - recipient: age16veg3fmvpfm7a89a9fc8dvvsxmsthlm70nfxqspr6t8vnf9wkcwsvdq38d
          enc: |
            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBTRTZCcnlXaHhiVlRhTlo2
-            RWlWQkY0bkJseHdVU1BvVHRwcHhNNU1Yb2xvCm1ZeU1KY3Y4WkZPRmlvQ01HdTVP
-            b3lDTjZLVTRnV0NxQkU4ZVg2ck9FYTAKLS0tIFhnaTRSVVlpM28zaGI1OFJ6VkpW
-            QkNTd1hzSm9zNnlmTzlpQ1hsa2loeXMKfWYt6gtlXRv97kmSeT31fSA+JfQFAeH/
-            e+Z8maFTUte0NF/toqsxDJPyLG8TPaWMiS+75RCRPXyvxtt58H5iOQ==
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA4ZjFGT0N2SGZkQWx6NS9x
+            OWpsU2RsOG51akRUYlRoL1Rwd0NyZkl4ZTNFCmowdXl0QlplMlFhYlg4bFpwYVZm
+            OTRXWE5wY0JJQzlaQVRuVE40Qi9XcE0KLS0tIGJ1MmhIc3RuR0U0WWhDTjl1aFJG
+            aGNabFJKR216dlFETmRPMkdnT0J2bEkKDz9UCFSUgFxPHJLvs8Olm/UYowbuCEl8
+            wDCJFobtV7AYYB3gJmXA46DHefsC+7rbUJ2E5y50SFIeofcEK/oorQ==
            -----END AGE ENCRYPTED FILE-----
-        - recipient: age136s4znrmkheztq6mps46dj5z4avy2umzz3the58fqtlsksvx5skq9ljqgk
+        - recipient: age1v6p8dan2t3w9h94fz4flldl32082j3s9x6zqq7u5j66keth9aphsd6pvch
          enc: |
            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBSM2NzUVVWZGZ2alJ4L0lC
-            TE9UWHEvNmtaT0pnVS9mUUh5VHdkQ1lIaDBNClVmendCYW5PZUNqUTFEYUhldnRZ
-            UWJqVTU1ajJNa0FtcnBDdThFYnBETUUKLS0tIFFROVRoTFNUOHVLSjN6elZzb1RQ
-            MXlOWjQ0cU1mUEhhTGlLWVNyS2V5c28KDNN6eK17Z+RZtb1/pH/tr8y9qk34cHPg
-            UGKimFTU2o0CvZY7ZnA24XV2RgfKs2J7COUc8I34b1kWPge57yQbJw==
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA3MWk3dEpkYkZxdjJpVDdI
+            RTdvSElPcG9JdjdzVGI5RTJWNUtBTEt6cFRjCnJpQjBtdUdpTHlHUXZMaVIraDdE
+            YlBKYWdBNzA3L09oVzRPdDEzOGJiOVUKLS0tIEpzTWhuUnR6VHpMMVhVdEpWKzBy
+            WGo2cHEyZFg5UnZEcUlmeHoycERCUDQKEPymfQ8YOkDtamYtyXkws5H5yuylOjtD
+            7C6nmKruZzFNIUs8Wf6u2TLtEPEsR2AX9k70ZjtOoygIHhvIpqY1uA==
            -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2023-08-19T12:12:46Z"
-    mac: ENC[AES256_GCM,data:W7MGnXfVxBgS/AQ5Xl6PcK3P4rH+1OjbWGBJBlz7KaG3uZXf8rnZGb7OUgYadu1KjhWZIJf8i3iyOBSqPTnBbd2xYKRMmxJj1qMlGY6dx8eGv4Zlvahs4pzT0iGqhC9Ce0+mc1QQwiD7paq0PSgNAy8q2XudITCS6iIL9woc+CM=,iv:SyTmDoG49wp1WPYUsnjw6u28Ch4N8a3T6EFncCgel5I=,tag:xJk//KA/Zhq3bjy1GG1L3g==,type:str]
+        - recipient: age1ylrpaytkm0k5kcecsxvyv5xd9ts4md0uap48g6wsmj9pwm4lf5esffu0gw
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBJc0FIMmwyTFpHNlNOSEhv
+            TmM2eW5IZlZpekNwTERuLzRLbW5ZYk8vaFVnCnlTQlNiaVFwdU9BWHFrc1VnSWc2
+            d0R1eHhWbTJPQjBsV0J0WjFyckZXL00KLS0tIEdwY1oyeGsxZzBLeStJSmZKVW1V
+            ZFBWTUhrbVJHTERMMkdDaDBkMU16VEUKi214s0sjzOR8wTK55lZelBKO+ar03lG2
+            Ue4Rx1utf3DDskRY6ELqSroIYEMIWDk9rxQTovIQD978mP9vpXgfPA==
+            -----END AGE ENCRYPTED FILE-----
+    lastmodified: "2024-08-31T15:45:49Z"
+    mac: ENC[AES256_GCM,data:BjoytvHEO/mvFUdAN/jf3EnwIjmWzSbY3TgCOjIp4zlVi+QrtwLD1G6fTN6q8tWOrYUBETS93q5FsCHKqCh58TEp/JZSnw2OhODBAn8LmdNvbvXX3dNFkVvjLsLH9rl4knMD5gr2fSc+YqHsTcb1sKBom7pFEsRppnnbU6h+FZo=,iv:ipoiDA2Er9gaqKg5bbjvVSC3RTiUV+t7J72ns5IEdac=,tag:1/4KRpnUnm14jGAdS6EoxA==,type:str]
    pgp: []
    unencrypted_suffix: _unencrypted
-    version: 3.7.3
+    version: 3.8.1
--- a/hosts/web-arm/sites/feeds.cloonar.com.nix
+++ b/hosts/web-arm/sites/feeds.cloonar.com.nix
@@ -0,0 +1,54 @@
+{ pkgs, lib, config, ... }:
+let
+  domain = "feeds.cloonar.com";
+  dataDir = "/var/www/${domain}";
+in {
+
+  sops.secrets.nextcloud-smb-credentials-feeds = {
+    owner = "feeds_cloonar_com";
+    key = "nextcloud-smb-credentials";
+  };
+  fileSystems."${dataDir}/public" = {
+    device = "//u149513.your-backup.de/u149513-sub4/dominik.polakovics@cloonar.com/files/Audio\ Books/";
+    fsType = "cifs";
+    options = let
+      automount_opts = "x-systemd.automount,noauto,x-systemd.idle-timeout=60,x-systemd.device-timeout=5s,x-systemd.mount-timeout=5s,user,users,file_mode=0770,dir_mode=0770";
+      in ["${automount_opts},credentials=${config.sops.secrets.nextcloud-smb-credentials-feeds.path},uid=1011,gid=60"];
+  };
+
+  services.borgbackup.jobs.default.exclude = [
+    "${dataDir}/public"
+  ];
+
+
+  services.webstack.instances."${domain}" = {
+    enableDefaultLocations = false;
+    enableMysql = false;
+    authorizedKeys = [
+      "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDDBGlzrg6NP5ezRFVu1CV8r0uCcS2oIgYG2/u6Cit++ARWQRO5Y0+9qC1Y2RNUaLPbvTmXg7ShskolUeuLryqvp10K2kXQ4E9NlmJ3BNLiAfWCzfe6gAgr6u5unVlXHttnP0leYpGGMUCKuiJpzy/bR6rMIUrCQC6W/MeXkwysNWKvL+ZD0IeQbogtfMFZmag9PO04RKZZvuUn9YvlgkTEK97g5dtyP1NxdtE9dDYf0G+0HcHITcw+lVmGNNwi43nAoUHieQd1kWc8YmxFB+y5O+vRH2O6pZBSdr0tdK6bPcezxd3Gk6i3a54yZfbvSislWA+o7s6uw/qExocpZb7xWa5ymPrGlEPbpYdT1y3hFO25+L1lR4QdG9oUNtJ974bL+EmYmHU+j32K3f8fxDg6BRo8FuriLtAzP7/2/7W8K4nIdMoosS+Ond2JE6XFkg1kSrXCivDBQoetZLO2y+ZPYcsQwIZsdjOnZqVr76nTepqCGIKYCuNM/9sl4AWCsyU="
+    ];
+    extraConfig = ''
+      add_header X-Frame-Options "SAMEORIGIN";
+      add_header X-Content-Type-Options "nosniff";
+
+      index index.php
+
+      charset utf-8;
+
+      error_page 404 /index.php;
+    '';
+    locations."/favicon.ico".extraConfig = ''
+      log_not_found off;
+      access_log off;
+    '';
+    locations."/robots.txt".extraConfig = ''
+      access_log off;
+      log_not_found off;
+    '';
+
+    locations."/".extraConfig = ''
+      try_files $uri $uri/ /index.php$is_args$args;
+    '';
+    phpPackage = pkgs.php82;
+  };
+}
--- a/hosts/web-arm/sites/matomo.cloonar.com.nix
+++ b/hosts/web-arm/sites/matomo.cloonar.com.nix
@@ -113,5 +113,16 @@ in {
    };
  };

+  services.mysql = {
+    ensureUsers = [
+      {
+        name = "matomo";
+        ensurePermissions = {
+          "matomo.*" = "ALL PRIVILEGES";
+        };
+      }
+    ];
+    ensureDatabases = [ "matomo" ];
+  };
  services.mysqlBackup.databases = [ "matomo" ];
 }
--- a/hosts/web-arm/sites/stage.myhidden.life.nix
+++ b/hosts/web-arm/sites/stage.myhidden.life.nix
@@ -38,12 +38,12 @@
    '';
  };

-  systemd.services."stage-myhidden-life-schedule" = {
-    startAt = "*:0/1:0";
-    serviceConfig = {
-      Type = "oneshot";
-      User = "stage_myhidden_life";
-      ExecStart = "${pkgs.php83}/bin/php /var/www/stage.myhidden.life/artisan schedule:run";
-    };
-  };
+  # systemd.services."stage-myhidden-life-schedule" = {
+  #   startAt = "*:0/1:0";
+  #   serviceConfig = {
+  #     Type = "oneshot";
+  #     User = "stage_myhidden_life";
+  #     ExecStart = "${pkgs.php83}/bin/php /var/www/stage.myhidden.life/artisan schedule:run";
+  #   };
+  # };
 }
--- a/hosts/web-arm/sites/support.cloonar.dev.nix
+++ b/hosts/web-arm/sites/support.cloonar.dev.nix
@@ -0,0 +1,72 @@
+{ pkgs, lib, config, ... }:
+let
+  domain = "support.cloonar.dev";
+  user = "support_cloonar_dev";
+
+  # phpPackage = pkgs.php82.withExtensions ({ enabled, all }:
+  #     enabled ++ [ all.imagick all.pcntl all.mbstring ]);
+
+  phpPackage = pkgs.php83.buildEnv {
+    extensions = ({ enabled, all }: enabled ++ (with all; [
+      imagick
+      mbstring
+      pcntl
+      imap
+      gd
+      curl
+      intl
+      zip
+    ]));
+  };
+
+in {
+  services.webstack.instances."${domain}" = {
+    enableDefaultLocations = false;
+    enableMysql = true;
+    authorizedKeys = [
+      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIChPB1wdZUO/VTt2J9e0+mLYhXcsWSL487HNQfmt23vB"
+    ];
+    extraConfig = ''
+      add_header X-Frame-Options "SAMEORIGIN";
+      add_header X-Content-Type-Options "nosniff";
+
+      index index.php
+
+      charset utf-8;
+
+      error_page 404 /index.php;
+    '';
+    locations."/favicon.ico".extraConfig = ''
+      log_not_found off;
+      access_log off;
+    '';
+    locations."/robots.txt".extraConfig = ''
+      access_log off;
+      log_not_found off;
+    '';
+
+    locations."/".extraConfig = ''
+      try_files $uri $uri/ /index.php$is_args$args;
+    '';
+    phpPackage = phpPackage;
+  };
+
+  systemd.services."freescout-worker" = {
+    enable = true;
+    serviceConfig = {
+      User = "${user}";
+      ExecStart = "${phpPackage}/bin/php /var/www/${domain}/artisan queue:work --queue=emails,default,a5b8cea21bd06d071d2d8da9307d9e04 --sleep=5 --tries=1 --timeout=1800";
+    };
+  };
+
+  systemd.services."freescout-cron" = {
+    startAt = "*:*";
+    wants = [ "freescout-worker.service" ];
+    serviceConfig = {
+      Type = "oneshot";
+      User = "${user}";
+      ExecStart = "${phpPackage}/bin/php /var/www/${domain}/artisan schedule:run --no-interaction";
+    };
+  };
+
+}