add victoriametrics

.sops.yaml

@@ -101,3 +101,14 @@ creation_rules:
       - *ldap-server-test
       - *testmodules
       - *netboot
+  - path_regex: utils/modules/victoriametrics/[^/]+\.yaml$
+    key_groups:
+    - age:
+      - *dominik
+      - *git-server
+      - *web-01-server
+      - *home-assistant-server
+      - *ldap-server-arm
+      - *ldap-server-test
+      - *testmodules
+      - *netboot

hosts/web-01.cloonar.com/configuration.nix

@@ -14,7 +14,9 @@
     ./modules/postgresql.nix
     ./modules/grafana.nix
     ./modules/loki.nix
+    ./modules/victoriametrics.nix
     ./utils/modules/promtail
+    ./utils/modules/victoriametrics
 
     ./utils/modules/borgbackup.nix
     ./utils/modules/netdata.nix

hosts/web-01.cloonar.com/modules/victoriametrics.nix

@@ -0,0 +1,26 @@
+{ config, ... }:
+{
+  services.victoriametrics.enable = true;
+  services.prometheus.exporters.node.enable = true;
+
+  services.nginx.virtualHosts."victoria-server.cloonar.com" = {
+    forceSSL = true;
+    enableACME = true;
+    acmeRoot = null;
+    locations."/" = {
+      proxyWebsockets = true;
+      extraConfig = ''
+        auth_basic "Victoria password";
+        auth_basic_user_file ${config.sops.secrets.victoria-nginx-password.path};
+
+        proxy_read_timeout 1800s;
+        proxy_redirect off;
+        proxy_connect_timeout 1600s;
+
+        access_log off;
+        proxy_pass http://127.0.0.1:8428;
+      '';
+    };
+  };
+
+}

hosts/web-01.cloonar.com/secrets.yaml

@@ -3,6 +3,7 @@ borg-ssh-key: ENC[AES256_GCM,data:7F7uUlTP3ZKkpySj6/AGfH3K1/8/GzIdfp+ch1hU55zX51
 grafana-ldap-password: ENC[AES256_GCM,data:hNB6CRtXW98yqUqInD3LsZ75sA+lVfmbooehni0UKL60qE/XCZm5B9JVO9pjxbIYZN6Eu/RFX+9L9cJVa5jnEo2MVeLS4CSjqC8BHLArlOuEdA5v8vqqJofBpBfXXN5Ca5xeUDJKz2HgtoTg7G5nTkegGZPGrmj5QQiL1xzco38=,iv:ViQAPTGxEWnjLkJlGCdCq5wW+fbr/O9er8/71VjL/GE=,tag:+Mow4cw7tvtkXvV2iSHeQw==,type:str]
 grafana-admin-password: ENC[AES256_GCM,data:365efRy8xD7SHBnVz6ZJO3l8/lfiZ5vZPZZbxnUmjKKJTMeebLY+P54moStY0wsbU9vk7sCKATCxrS5xy+FQJSgKLoajfz50OMA4+1k3Shl+skbeIikHKwFxqrljFa6HRQ2HTW6KLDPu6Z5Agkima5xdfrtc5R1SnOFg5b6D5NU=,iv:0yZGZVQd35Itj66Ff5hDfDYYx5xsNs/wc887bgMV1MY=,tag:9t8Iffg7kxSjE5eo7iv/RQ==,type:str]
 promtail-nginx-password: ENC[AES256_GCM,data:zk/Wq+Nss6Md0GdhoOcysPrDBqfoAobmqb4LMDkJBjpCn/mdP3/HPiIYdZnZ0vV0JmYpQVqgVFPMlA==,iv:TA19kKllw0Vco6RRlbW4eUqeGQ0SQJRr/TATmyZBMrs=,tag:10/87/svXdL1hpUcTOtY0w==,type:str]
+victoria-nginx-password: ENC[AES256_GCM,data:5J7bqqtqd/KEqnDLJUpZ4uF8OZ22JiSUhhCEm5T9fsNs1EHGsbdamiXB8HHFNNQQMSMADdnFgfK3w2NLk2qWOiQ=,iv:3nTXWHKQgfz8hGPdgBM+w8fNFZutybPibdyu/slp2WQ=,tag:0+uvo3altTFMu52FpR+pIQ==,type:str]
 sops:
     kms: []
     gcp_kms: []
@@ -27,8 +28,8 @@ sops:
             elpwY3Q3dnRzR0loN1BiVk44TTF2VDQKs8Si2LHZ4L4oQqkYUhCI6affE0aTrWmE
             L+am++gYdygVURIh0Z6ftUuhYHPwhlCgmKxx51mKRV2ydraOdUUw0g==
             -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2023-08-19T00:09:27Z"
-    mac: ENC[AES256_GCM,data:4FjX8XngdwYBbifM4xmdW/7a3tf43/AdD6ujpYa9M7c7EJ+4ipf6S/eu1CuVk4XAr84rkCAfF+PpGXWeZCJ47YhbXI3yg6HRjGt//5X4Jn6tUYre8vk5Fy7C3dwDKgqHLqOm0hFE89m82xfkfe6VuDeCSbLFUucEtQ3d+rKcGvY=,iv:ufx9eQNNOXcRQISLvdfLK2RUinQPTgjiYpGUWYiqDZc=,tag:A2MoB+/NUFiEee4nTNpAXg==,type:str]
+    lastmodified: "2023-08-19T04:18:59Z"
+    mac: ENC[AES256_GCM,data:PENs4IS9EjGlC7ib4l+raa9Q6y1rFBIT90Pf2yMqTTsBhup48vG2NpLzOm64SBIBJGusU2naPBh2lAB0yv4yTqouz2HxTHh0MrXg+se7Zgg9HVhtC4Ct08MD+kD1N/V5S03d4gIA+dwEiirAokyJo9FiWE64ksWFrtgs7tQnypw=,iv:BMzOpFMScWlSOgSark1MN+NNTHU4nRQ1bMrYSfvh9BI=,tag:EPvQr5dg6N2/yDbM8ZqmWw==,type:str]
     pgp: []
     unencrypted_suffix: _unencrypted
     version: 3.7.3

utils/modules/victoriametrics/default.nix

@@ -0,0 +1,26 @@
+{ config, pkgs, ... }:
+let
+  configure_prom = builtins.toFile "prometheus.yml" ''
+    scrape_configs:
+    - job_name: '${config.networking.hostName}'
+      stream_parse: true
+      static_configs:
+      - targets:
+        - 127.0.0.1:9100
+  '';
+in {
+  sops.secrets.victoria-agent-env = {
+    sopsFile = ./secrets.yaml;
+  };
+
+  services.prometheus.exporters.node.enable = true;
+  
+  systemd.services.export-to-prometheus = {
+    path = with pkgs; [victoriametrics];
+    enable = true;
+    after = ["network-online.target"];
+    wantedBy = ["multi-user.target"];
+    EnvironmentFile=config.sops.secrets.victoria-agent-env.path;
+    script = "vmagent -promscrape.config=${configure_prom} -envflag.enable -remoteWrite.url=https://victoria-server.cloonar.com/api/v1/write";
+  };
+}

utils/modules/victoriametrics/secrets.yaml

@@ -0,0 +1,84 @@
+victoria-agent-env: ENC[AES256_GCM,data:15GlckIcI29qf5iCPZwwtxXJX+vODvZs6b8DwThYe7x9IDEcs2Y5JU1u6u85VoDksWWs8WHD1w6KKHhRcd5L4zBTS7IY2lkfbANEXWWWW8ZsO21LXTtfvekux5FcXS1XIw4eDs4527yVRyehhMyWDIZoXVz4y8jJhq17DXbOZUlQnLZ72pS21ciAEkH2vkHKTIWJxOkfSZzKA5pdGJFgjxf/VjQKAcf/sSLYQi8xb/N4H2SFHyAsPgexNm+AOqvsKQ==,iv:Aekqh6505/+1Edpxfo9YF9f2vBrn9aVK5cilf3JC98g=,tag:HMG3NahsXItqhLqPCaF+kw==,type:str]
+sops:
+    kms: []
+    gcp_kms: []
+    azure_kv: []
+    hc_vault: []
+    age:
+        - recipient: age16veg3fmvpfm7a89a9fc8dvvsxmsthlm70nfxqspr6t8vnf9wkcwsvdq38d
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBmLy84dnJEM1daSWdlNGtC
+            TUhkamk2NlBEblVxejhuU0I2aWw1Ym9JSWxFCm00bVJ1UUdxSUhENU15L000bkJK
+            ZXhwOVF0UzRZWlg5QzQ5S3VFaUdFTUEKLS0tIDFPMGtaZ09oWVQxTmd5STdMeU8y
+            aDYwZUxzUC8rc3Ezclhwbkl3MTJtVGMK3VrOYdJmvbuCAHeb3EpczuxpZokHutgT
+            Cz1vQzQZnmhEWMDpK+HgOfOW6drrAWEfP33p9oRpUsQBeuo6Z/mShw==
+            -----END AGE ENCRYPTED FILE-----
+        - recipient: age106n5n3rrrss45eqqzz8pq90la3kqdtnw63uw0sfa2mahk5xpe30sxs5x58
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBKbDg0Q2RkQWdkSzRYckFK
+            ZlNZSDQzL1pQTVEwaXJHeTZBUUFYall6cG1RCnpSbFlQYXA4RjNyUGZvbW1SeUZs
+            cUxuSWFrVE1GYkNiOXp3TERyaWg2R3cKLS0tIEZ0bm9WaFVITmV5V2JXczFsNEo2
+            SXVOM2pWLzdIdnRLZ0p3R0pCMjQ5Nk0KPosyPjEgU9zrgxWzj3RCzRCip631FYnc
+            CjtRbFq4i/BC1guqmiwrvI0+B4abmgfEnAQp1jHnR6qLxMv35pwlZQ==
+            -----END AGE ENCRYPTED FILE-----
+        - recipient: age1y6lvl5jkwc47p5ae9yz9j9kuwhy7rtttua5xhygrgmr7ehd49svsszyt42
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBNb1JqdzVkSWU2c2NuNjJ2
+            MUpWVFNaSThFaHNHOU9qU3V0eG5KRk9BaTM4CndTdStVbHZ4M2ZQaDVwMnVsYnIy
+            R2EvUE14OEJ2SllTY1FCQzY1QkRNeUUKLS0tIGM0OU4vQkRXaG1vc3hkY2xPemFT
+            N0xYY05GUXVQY0hRL3ZwMlZzc3dIRFEK+Z2k+IC4QFhRLuWnAtnFVcAwWhGwnB76
+            Z8v3Vs4R83KzWEd+Lp0vzFmnxlBv37Vdr+wF7Y5vYPgejbRx7I9T5w==
+            -----END AGE ENCRYPTED FILE-----
+        - recipient: age1ezq2j34qngky22enhnslx6hzh4ekwk8dtmn6c9us0uqxqpn7hgpsspjz58
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBVNytzZGpLRkh4c0VudWJC
+            S2praTRnMmxvZDZkNzI1SDFNbDJHVnM4VTEwCi94RGxhOFRGZHhBU243dHZ5VkJi
+            a1hHTjRmYjRBYis1MGpkMS9IaHNyR2MKLS0tIFRVNzlPUENtN2RZTkJXclVUY210
+            c2d3NVlJZ0hNczZmaDdwTzE2aThlUXcKfZ3l2jBtOdNvlabLijjaUxtJZq9DAXvR
+            FVr9yw4HSUMQB2dBBQY/aReLHtxSwI9/ap9uo7YmgdtoiImcBa+xQg==
+            -----END AGE ENCRYPTED FILE-----
+        - recipient: age1jyeppc8yl2twnv8fwcewutd5gjewnxl59lmhev6ygds9qel8zf8syt7zz4
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA2SUpYSjluRlhmMUVYNk53
+            Q21MVlpIcWRsYWovTzd0MXVGbEVDeHdIZldBCnM0U3EwV1poQmgvdWZBNU5IdSt1
+            MFRYUVVwNFJhbUQxd2phWlNNWXd2NkEKLS0tIHp0alBvMms0TWduYTRtZ1I5cEsr
+            aUNJeFZkY1BvWEVLNnVlSFlwaEZtLzgK16wv8+6ue4xfkUSA+yw7MLn4zIDpdbRw
+            +ZvlL+s341cwuOxMuXmW+E0dHUrOM7GfS6jG/mNYPAhzQPYBI7RVIg==
+            -----END AGE ENCRYPTED FILE-----
+        - recipient: age1azmxsw5llmp2nnsv3yc2l8paelmq9rfepxd8jvmswgsmax0qyyxqdnsc7t
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBhU1FyYUplNlVrSjQyTXgz
+            eVVhOHVJOXU2elZ1by9ZOUxEYzJtdTdKdVdzCmtTY0d6S25GMndNZkFtVXY3Lytq
+            L0VyUy83ZGoxek1BNlo5b0ZyMnp6Mk0KLS0tIHpNQWRJSHloSEpremVTTGJycXp5
+            TUhkbUpiSzllUlpsYXNmb3FITFE2aU0K6muCw6pQ9s+lCswbOME+8k0Z2d9wJ/1G
+            xu5t0S0S49u7RMTMmQ22zOEHUtW2scqEa6xNMdMWt+WrqejyI+xRWQ==
+            -----END AGE ENCRYPTED FILE-----
+        - recipient: age1zkzpnfeakyvg3fqtyay32sushjx2hqe28y6hs6ss7plemzqjqa5s6s5yu3
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBDL3NOOFRFekphUTBBYjFH
+            d2pUN2hQelBYN1JITTVwMFJROFJ3VStTUGs4Cm41eUN6eFVWMkM5WVR3OFFOdmdS
+            am5zajEzclZwRDNZUkZqaE03R3l0ajQKLS0tIGRjZ3FvVmowelMyMkVEczVBQXNR
+            TVlTYmR2UFpINHdMa2RVclI1MjBXSVEK5gb/FVUnSy9pdnrc7RlzwI64+ObYbV2B
+            q0DsX2v2EtmxCGBgk+T330t6WbM6fS6T4xWZ8+hhwouTwhIaXyOuVw==
+            -----END AGE ENCRYPTED FILE-----
+        - recipient: age14uarclad0ty5supc8ep09793xrnwkv8a4h9j0fq8d8lc92n2dadqkf64vw
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA4WEpSZFM2Nm82M2FZbkNw
+            eVVnVDdzeXZtamhnY3ZlVEJoTFA5RkdPSFMwCjQvVVoyM2xjWkhyT2F5WVZHenR2
+            SnZsRldVQlhCT3ZRWGR6VGt0SytWRFEKLS0tIFJUbklYb2FiYWUxaHBNZW1QOG16
+            NkMzbGlkQTJGd3NIUVFuRm1YMEczK0EKSpNf9PeifY82y4HKm8s6AgYUVNXHtEUT
+            T+NijcHxoaCb8i+Z91ob5KZOLP8Zv2HfsAxPkWNB6OQSdn0yuCrNog==
+            -----END AGE ENCRYPTED FILE-----
+    lastmodified: "2023-08-19T04:17:42Z"
+    mac: ENC[AES256_GCM,data:3gxdFmVPyPvVr/bRNieUefDPz6olWnhDVEY4Hzps3N80NbJw/ZdX+OpxuBVjGYv13MtotNexe0DpNHpzyfSrH5BKu+WB601nbh9y45Y8dIIzy7EgnGYreRydAn+uTM95j5Zr6R1zF0su0G8a5M1jWbB+eCQ9uTd6nbKaupWESN0=,iv:ta51MnEETIEQcZI47goi4rYT7VVzdTRmDiTLY44aD0w=,tag:tcpGYOdlKddeVmQ8Bvf7gg==,type:str]
+    pgp: []
+    unencrypted_suffix: _unencrypted
+    version: 3.7.3