add loki and promtail to web host

2023-08-19 02:10:37 +02:00
parent 9f48ac2f3c
commit 6d42a0ecac
6 changed files with 446 additions and 2 deletions
--- a/hosts/web-01.cloonar.com/configuration.nix
+++ b/hosts/web-01.cloonar.com/configuration.nix
@@ -13,6 +13,8 @@
    ./utils/modules/nextcloud
    ./modules/postgresql.nix
    ./modules/grafana.nix
+    ./modules/loki.nix
+    ./utils/modules/promtail

    ./utils/modules/borgbackup.nix
    ./utils/modules/netdata.nix
--- a/hosts/web-01.cloonar.com/modules/prometheus.nix
+++ b/hosts/web-01.cloonar.com/modules/prometheus.nix
@@ -0,0 +1,248 @@
+{ config, ... }:
+{
+  sops.secrets.alertmanager = { };
+  sops.secrets.hass-token.owner = "prometheus";
+
+  # imports = [
+  #   ./matrix-alertmanager.nix
+  #   ./irc-alertmanager.nix
+  #   ./rules.nix
+  # ];
+
+  services.prometheus = {
+    webExternalUrl = "https://prometheus.cloonar.com";
+    scrapeConfigs = [
+      {
+        job_name = "telegraf";
+        scrape_interval = "60s";
+        metrics_path = "/metrics";
+        static_configs = [
+          {
+            targets = [
+              "turingmachine.r:9273"
+              "bernie.r:9273"
+              #"rock.r:9273"
+            ];
+            labels.type = "mobile";
+          }
+          {
+            targets = [
+              "eva.r:9273"
+              "eve.r:9273"
+              "blob64.r:9273"
+              "matchbox.r:9273"
+              "alertmanager.r:80"
+              "prometheus.r:80"
+              #"rock.r:9273"
+            ];
+          }
+          {
+            targets = [
+              "rauter.r:9273"
+            ];
+            # to make it compatible with the node-exporter dashboard
+            labels.host = "rauter.r:9273";
+          }
+          {
+            targets = [
+              "prism.r:9273"
+              "gum.r:9273"
+              "kelle.r:9273"
+            ];
+
+            labels.org = "krebs";
+          }
+          {
+            targets = [
+              "clan.lol:9273"
+            ];
+
+            labels.org = "clan-lol";
+          }
+          #{
+          #  targets = [
+          #    "dev1.numtide.com.r:9273"
+          #  ];
+
+          #  labels.org = "numtide";
+          #}
+          {
+            targets = map (host: "${host}.r:9273") [
+              # university
+              "amy"
+              "clara"
+              "rose"
+
+              "astrid"
+              "dan"
+              "mickey"
+              "bill"
+              "nardole"
+              "yasmin"
+              "ryan"
+              "graham"
+
+              "astrid"
+              "dan"
+              "mickey"
+
+              "jackson"
+              "christina"
+              "adelaide"
+              "wilfred"
+              "river"
+              "jack"
+
+              "ruby"
+            ];
+
+            labels.org = "uni";
+          }
+        ];
+      }
+      {
+        job_name = "homeassistant";
+        scrape_interval = "60s";
+        metrics_path = "/api/prometheus";
+
+        authorization.credentials_file = config.sops.secrets.hass-token.path;
+
+        scheme = "https";
+        static_configs = [
+          {
+            targets = [
+              "home-assistant.cloonar.com:443"
+            ];
+          }
+        ];
+      }
+      {
+        job_name = "gitea";
+        scrape_interval = "60s";
+        metrics_path = "/metrics";
+
+        scheme = "https";
+        static_configs = [
+          {
+            targets = [
+              "git.cloonar.com:443"
+            ];
+          }
+        ];
+      }
+    ];
+    alertmanagers = [
+      {
+        static_configs = [
+          {
+            targets = [ "localhost:9093" ];
+          }
+        ];
+      }
+    ];
+  };
+  services.prometheus.alertmanager = {
+    enable = true;
+    environmentFile = config.sops.secrets.alertmanager.path;
+    webExternalUrl = "https://alertmanager.cloonar.com";
+    listenAddress = "[::1]";
+    configuration = {
+      global = {
+        # The smarthost and SMTP sender used for mail notifications.
+        smtp_smarthost = "mail.cloonar.com:587";
+        smtp_from = "alertmanager@cloonar.com";
+        smtp_auth_username = "alertmanager@cloonar.com";
+        smtp_auth_password = "$SMTP_PASSWORD";
+      };
+      route = {
+        receiver = "default";
+        routes = [
+          {
+            group_by = [ "host" ];
+            match_re.org = "krebs";
+            group_wait = "5m";
+            group_interval = "5m";
+            repeat_interval = "4h";
+            receiver = "krebs";
+          }
+          {
+            group_by = [ "host" ];
+            match_re.org = "nix-community";
+            group_wait = "5m";
+            group_interval = "5m";
+            repeat_interval = "4h";
+            receiver = "nix-community";
+          }
+          {
+            group_by = [ "host" ];
+            match_re.org = "clan-lol";
+            group_wait = "5m";
+            group_interval = "5m";
+            repeat_interval = "4h";
+            receiver = "clan-lol";
+          }
+          {
+            group_by = [ "host" ];
+            group_wait = "30s";
+            group_interval = "2m";
+            repeat_interval = "2h";
+            receiver = "all";
+          }
+        ];
+      };
+      receivers = [
+        {
+          name = "krebs";
+          webhook_configs = [
+            {
+              url = "http://127.0.0.1:9223/";
+              max_alerts = 5;
+            }
+          ];
+        }
+        #{
+        #  name = "numtide";
+        #  slack_configs = [
+        #    {
+        #      token = "$SLACK_TOKEN";
+        #      api_url = "https://";
+        #    }
+        #  ];
+        #}
+        {
+          name = "nix-community";
+          webhook_configs = [
+            {
+              url = "http://localhost:9088/alert";
+              max_alerts = 5;
+            }
+          ];
+        }
+        {
+          name = "clan-lol";
+          webhook_configs = [
+            # TODO
+            #{
+            #  url = "http://localhost:4050/services/hooks/YWxlcnRtYW5hZ2VyX3NlcnZpY2U";
+            #  max_alerts = 5;
+            #}
+          ];
+        }
+        {
+          name = "all";
+          pushover_configs = [
+            {
+              user_key = "$PUSHOVER_USER_KEY";
+              token = "$PUSHOVER_TOKEN";
+              priority = "0";
+            }
+          ];
+        }
+        {
+          name = "default";
+        }
+      ];
+    };
+  };
+
+}
--- a/hosts/web-01.cloonar.com/secrets.yaml
+++ b/hosts/web-01.cloonar.com/secrets.yaml
@@ -2,6 +2,7 @@ borg-passphrase: ENC[AES256_GCM,data:V77hfP5jk/DXcvRiZKu6RLAqsJhlIelkQwA6ClYJKNm
 borg-ssh-key: ENC[AES256_GCM,data:7F7uUlTP3ZKkpySj6/AGfH3K1/8/GzIdfp+ch1hU55zX51KgRs/KuGmj+yKyH9ua41oR4FR94MoiTb3u3MRpUj6dqO4VjVm6fRgJFNXOBhTUcelRR98Nq2QClkRqcNmPiHQC4bxjyW9C1tZfCA52AIILGh9O1Q9XnYAz2q8JwbrY+eTXS/U/1Fh1D+0Y9q1n+oingehal2huHzeVsLxFlip3TmGJx8QBlnAbANABKrMFxkHAlkvAVCrF1MULzeDeeWscHi5T80OvJfOZBSonNEZosw6QJVscMYxh047Yyl6YJ/sxIJjZjC7GiLuG/FC0FCLKRhD8PkZFjrDt/6xwrqePxIb7yIKZRpsvNVCplDDdVOB/V0AqRWAeZh3z813Zi+tnjynHPYaphso/qY2r/HZKlGInzW+QCUBdPVifhVL+y2TKytCWcp4A+c/3Dc2Ut59sfO+hqE946nYGl2S+kpNASHhBa7o6cnaMtfz8NT6rVtYN/3l1snlxeOUZo85XAuYCIerGsMkdVENg5RIJwIzwM0oaGwNzruq8cul5MfAf8hFMXpoLggYECynHk9TNdhsNtUzmsS9cXAyPnnzZT6HGI2/5cgU1weCHbbXAq+ZU9WZwOT41Fpwda/WrNJuMJYFCOFer2hcSLEyxsCfqmPSdwpYRRSYHiaCuVghV6lWyi1IVV5EsX9H6hD+Uetux1SEN95ga8edME16W2dubA0yukcOq+XHI7PMEHAs20H+dybsmVx1XjIsiV/XBWkrooFBXQp950p93XOK49vwNmHNohhCvmERkH0dczGQ5vTAMXwIYqydTqXWERDRzJVvK+pbzKiecEzhFHFge345poUKQS7BWNsv+eehgBAH4HEddtzzzjcPGYMHAhafAm1VqO2BvKg2r3XnzElUCjxWlGfrMkU+LhIgAjom8Mk9+Kha+OD84+iZnuoq73TjpqaUgC0TbkxLAm0DPgk8LsLE9XfJaroftKnrE7P7kyNGSQZfGYcl7ssLck5LJA07dTvDtytlL1Kk8RJe4FwFNPLtdiAnCVvKBe0kcClvCWrrxRt1QT85b+9IwG1yhRxlxheleePX1Fwlg/d2xj6AB3Cz7kAVO2phGPAF16Ke/UQeMxhbOIcy20g7RoiCpfx9jvdwExhDmbmUR7cleL1seFH+wgAM6uQwh80+q8IDJWPNyeXOwiA2siPwexb20xN+n1kIMa03U5Bn+lBqYoh0xXhlDHP3FiKew6xjSducNUtzyKm24kQaWSOfYIqTxYFRTI0bDgRu6xWGmZL37VSMWqDj3TIWOMBoLy0JBsRl6Jj07keJ0CwaNEa4cCWZ0s6nu49mp04JfJWF2r9ksX+2OVeDjSgX5JBB/V79j3I8iu4Nlp0pjyBKPlST/FUDAl1jT12X1grkYjXO8UyJ9AQqqnqK5lM+KYVR15Ui1lBb/vISPqiiw45bcUyHfVAAt7hcFpYLaYB0om3mernccN2fi26lawhhambRd7FGkILKA3byE9ytqGvDQ2CxtjA30kbGxeqXVFsyzROahR0c3KdKlnuCO9Uar4J5VEXgv2obNlNUfSMa9uleWDuhveBaE+2jtKUJd2P4wIIzF/VJGxgWGSge/ji0EV36EFfMg/Tyizdw5wtv4rQF0M+Uu6j/n/l62SmHnT/30H8IFCFuXWmtEo1xcssMymY4ricU3kJIgjGO9h+DrBP1GBczj7yVjLHTpEhRF0yP790xgsJrP4IQB2lOtGf5MCXLrBDYkOZ5xM3+Rq5ZNH0SAHRU/qtFUmLtfkcidjkPiwdqJ0e1LUtLwmSlsot1CkPs7hKORassyUrug7dCtv9QjRMDbtW61PlIbXqa59Aimql4IUcWyUybx12E8wRqmgcX5kG2wGfd9ZFUj6mXhQINFqChsTjXSavQw3u3m8kvd1mJXfjKS12ajr2X1e5wPDUEpLzc3wTOvVgZizGeykKTcKG+GXSjPXLR61ueAO26rboYiDAeSd73shFX/vvut/aB47kZobMOooljGVtnfZjVGY8dWzdNeGvBeLws7vnFrH1u/WXPxpktGz2/eJ0L8ANJd+BZ2+wC+R3OnHeDiXHfofm3dZJTncgxYPqboKKCXRzMviSCWB3poQTm5vEltsQOR6Lj17UmHu5MX7os7t7TrfW/op4Qko9ViWT5vtrUrCZzVqJS+d8hq4lm6ANMhi3Ql+nIMIxK+hjGZNBuKZEyKY+r+Lhz9E/xdOBGVB8QH4g32DWsYvaHfpcvJC8CkGO1nRIGFyrMc10lrv++XQYtiZ1Z4a0oOtQGAXaPGQTJB4KzwlGWc0+kguV+lK4h0QIvHvuorghYC4EJerNdRUviRxZB5mTDyJc1gGq6YgMr4d/a4tyXwoEzPbPGc/3YJATZPYLXOMaGDo0rd2961CFfrsneElNIZ71DWoy41P1fJG7qrfN0gLjU0aSC1vVnzDM8GQvoJGV35cONT7N7u+hjjFBFciLBNEE1DKge156EnP6VbR2LSptWrHeq3zOe9fN3FFR1WWor+lOl+SFztt7uVHCLuWxYPV+csOeBLQi6OhFnh9r4mLTc43wjPkbuci2jqEVM6Bf5gXQoEGzybhDb/3HfQK00NXovru5uEdREYDaj5NVyzyBhOT29JuOvqX7wEMNDqE//Me6Tx8bU12cuUM7Lne7grgYQMbYfLs3gKzRbeuYCqGCIz6KQMDOZvTR38EXqJxysrPb/HuiPfoSGHoAIHviJeIsy6bF7hKA8BikrMmoNc4762pKNKNzBGR+/HeQ1trDBCbYrTmlqoXPJaeANUZlD9NdwiopTJCBzjP/F61s5bpurAiJF6Ymx5yUHljZGVPFOH/ZawfhEcfYGvMUnAOup+EJCih5WQsrn2vodbVYFJ5GNMrDH9//uMi/cwqFWlBqnKGgnGdyqzXlLYTTMlv7fuh3XxkjRRTh6ZVuLyBqEBnXzSWwlcLOUdw3r/48JJ0JC2/rTLVnb+R7T2/+7uXVQISIrBx1ijtRCzFwaFWAPqsL+nMevAHnjpWl9NvanNWKIPIQ6cHGywxdi/7ynEnCYlY13fyIpagFKrgywsHcHuez8cyNtGLpg6hwbSEeN2wZYld+DQc5UH9pqPvvTuNHu3J62WJHZlkc6yBs/hiZBT/sEZUrL2Bf3SNFjzt/IcNSjlVbpP6dd6HpIH6tuN+lQJNypRe//TgKviHBkN0mzmw7IuRIDz1tcVA28COCo8NkzyD194zxDJ8yYKfKH/YwN5q8/R/r/EuUcY4qbeAokaS7XcIkOp/QFKHVM6AYKZ3SPyB5ZjkHFoHBG7YNDrx8AHDNemG+WUev/flmsEv3ykgTztgOoCmTqH/rduS+BGcKwsNW0iMwni9mUiJfuNdYLqhWohGLWb/mkUVoTgycXtsJ8x5BGkk4wSuGSwGCur8yVel7+Fr6CzsVGiPJOa0RHO6sN+Y9jnSMPIVm+mx16j3Q==,iv:ZGV3C0nvqdEnukiPkeMxDD66OjeXQF4anQLkALmBno8=,tag:ELar6NeP5bjL5L/Z5m7Piw==,type:str]
 grafana-ldap-password: ENC[AES256_GCM,data:hNB6CRtXW98yqUqInD3LsZ75sA+lVfmbooehni0UKL60qE/XCZm5B9JVO9pjxbIYZN6Eu/RFX+9L9cJVa5jnEo2MVeLS4CSjqC8BHLArlOuEdA5v8vqqJofBpBfXXN5Ca5xeUDJKz2HgtoTg7G5nTkegGZPGrmj5QQiL1xzco38=,iv:ViQAPTGxEWnjLkJlGCdCq5wW+fbr/O9er8/71VjL/GE=,tag:+Mow4cw7tvtkXvV2iSHeQw==,type:str]
 grafana-admin-password: ENC[AES256_GCM,data:365efRy8xD7SHBnVz6ZJO3l8/lfiZ5vZPZZbxnUmjKKJTMeebLY+P54moStY0wsbU9vk7sCKATCxrS5xy+FQJSgKLoajfz50OMA4+1k3Shl+skbeIikHKwFxqrljFa6HRQ2HTW6KLDPu6Z5Agkima5xdfrtc5R1SnOFg5b6D5NU=,iv:0yZGZVQd35Itj66Ff5hDfDYYx5xsNs/wc887bgMV1MY=,tag:9t8Iffg7kxSjE5eo7iv/RQ==,type:str]
+promtail-nginx-password: ENC[AES256_GCM,data:zk/Wq+Nss6Md0GdhoOcysPrDBqfoAobmqb4LMDkJBjpCn/mdP3/HPiIYdZnZ0vV0JmYpQVqgVFPMlA==,iv:TA19kKllw0Vco6RRlbW4eUqeGQ0SQJRr/TATmyZBMrs=,tag:10/87/svXdL1hpUcTOtY0w==,type:str]
 sops:
    kms: []
    gcp_kms: []
@@ -26,8 +27,8 @@ sops:
            elpwY3Q3dnRzR0loN1BiVk44TTF2VDQKs8Si2LHZ4L4oQqkYUhCI6affE0aTrWmE
            L+am++gYdygVURIh0Z6ftUuhYHPwhlCgmKxx51mKRV2ydraOdUUw0g==
            -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2023-08-18T23:19:22Z"
-    mac: ENC[AES256_GCM,data:sWtJUW19HleKalg/Mfysk/b0N6YxdFcC/66BLmbcchI6s5MeGMLdYIJkNm7RKRQM5PY25d3saOqvsm5qK+keOBa0H9v0DwmFuS9cBJGa5KV6/IDoMvO8VtgDzCZ9HLtrSVTuh84bv7XL3cRd99BfSlSyHBJRpV7kJTudid2O9vo=,iv:8sOMUnsm8hyJlLvc5zG72wjKXtcbK7qnEd7Og0+yJt4=,tag:4XirU7fx0UmJSNkKgmJp8g==,type:str]
+    lastmodified: "2023-08-19T00:09:27Z"
+    mac: ENC[AES256_GCM,data:4FjX8XngdwYBbifM4xmdW/7a3tf43/AdD6ujpYa9M7c7EJ+4ipf6S/eu1CuVk4XAr84rkCAfF+PpGXWeZCJ47YhbXI3yg6HRjGt//5X4Jn6tUYre8vk5Fy7C3dwDKgqHLqOm0hFE89m82xfkfe6VuDeCSbLFUucEtQ3d+rKcGvY=,iv:ufx9eQNNOXcRQISLvdfLK2RUinQPTgjiYpGUWYiqDZc=,tag:A2MoB+/NUFiEee4nTNpAXg==,type:str]
    pgp: []
    unencrypted_suffix: _unencrypted
    version: 3.7.3