From 6dc3798dc7cb4e18ccd2243579f27fadccf1727e Mon Sep 17 00:00:00 2001
From: Dominik Polakovics <dominik.polakovics@cloonar.com>
Date: Sat, 19 Aug 2023 01:19:55 +0200
Subject: [PATCH] add grafana admin password

---
 hosts/web-01.cloonar.com/modules/loki.nix | 133 ++++++++++++++++++++++
 hosts/web-01.cloonar.com/secrets.yaml     |   5 +-
 2 files changed, 136 insertions(+), 2 deletions(-)
 create mode 100644 hosts/web-01.cloonar.com/modules/loki.nix

diff --git a/hosts/web-01.cloonar.com/modules/loki.nix b/hosts/web-01.cloonar.com/modules/loki.nix
new file mode 100644
index 0000000..4d7fa08
--- /dev/null
+++ b/hosts/web-01.cloonar.com/modules/loki.nix
@@ -0,0 +1,133 @@
+{ config, pkgs, ... }:
+let
+  rulerConfig = {
+    groups = [
+      {
+        name = "general";
+        rules = [
+          {
+            alert = "Coredumps";
+            # filter out failed build gitlab CI runner, users or nix build sandboxes
+            expr = ''sum by (host) (count_over_time({unit=~"systemd-coredump.*"} !~ "(/runner/_work|/home|/build|/scratch)" |~ "core dumped"[10m])) > 0'';
+            for = "10s";
+            annotations.description = ''{{ $labels.instance }} {{ $labels.coredump_unit }} core dumped in last 10min.'';
+          }
+        ];
+      }
+    ];
+  };
+
+  rulerDir = pkgs.writeTextDir "ruler/ruler.yml" (builtins.toJSON rulerConfig);
+in
+{
+  systemd.tmpfiles.rules = [
+    "d /var/lib/loki 0700 loki loki - -"
+    "d /var/lib/loki/ruler 0700 loki loki - -"
+  ];
+  services.loki = {
+    enable = true;
+    configuration = {
+      # Basic stuff
+      auth_enabled = false;
+      server = {
+        http_listen_port = 3100;
+        log_level = "warn";
+      };
+
+      # Distributor
+      distributor.ring.kvstore.store = "inmemory";
+
+      # Ingester
+      ingester = {
+        lifecycler.ring = {
+          kvstore.store = "inmemory";
+          replication_factor = 1;
+        };
+        lifecycler.interface_names = [ "eth0" "en0" "ens192" ];
+        chunk_encoding = "snappy";
+        # Disable block transfers on shutdown
+        max_transfer_retries = 0;
+      };
+
+      # Storage
+      storage_config = {
+        boltdb.directory = "/var/lib/loki/boltdb";
+        filesystem.directory = "/var/lib/loki/storage";
+      };
+
+      limits_config.retention_period = "120h";
+
+      # Table manager
+      table_manager = {
+        retention_deletes_enabled = true;
+        retention_period = "120h";
+      };
+
+      compactor = {
+        retention_enabled = true;
+        compaction_interval = "10m";
+        working_directory = "/var/lib/loki/compactor";
+      };
+
+      # Schema
+      schema_config.configs = [
+        {
+          from = "2020-11-08";
+          store = "boltdb";
+          object_store = "filesystem";
+          schema = "v11";
+          index.prefix = "index_";
+          index.period = "120h";
+        }
+      ];
+
+      limits_config.ingestion_burst_size_mb = 16;
+
+      ruler = {
+        storage = {
+          type = "local";
+          local.directory = rulerDir;
+        };
+        rule_path = "/var/lib/loki/ruler";
+        alertmanager_url = "http://alertmanager.r";
+        ring.kvstore.store = "inmemory";
+      };
+
+      query_range.cache_results = true;
+      limits_config.split_queries_by_interval = "24h";
+    };
+  };
+
+  sops.secrets.promtail-nginx-password.owner = "nginx";
+
+  security.acme.certs."loki.r".server = config.retiolum.ca.acmeURL;
+  services.nginx.virtualHosts."loki.cloonar.com" = {
+    forceSSL = true;
+    enableACME = true;
+    acmeRoot = null;
+    locations."/" = {
+      proxyWebsockets = true;
+      extraConfig = ''
+        auth_basic "Loki password";
+        auth_basic_user_file ${config.sops.secrets.promtail-nginx-password.path};
+
+        proxy_read_timeout 1800s;
+        proxy_redirect off;
+        proxy_connect_timeout 1600s;
+
+        access_log off;
+        proxy_pass http://127.0.0.1:3100;
+      '';
+    };
+    locations."/ready" = {
+      proxyWebsockets = true;
+      extraConfig = ''
+        auth_basic off;
+        access_log off;
+        proxy_pass http://127.0.0.1:3100;
+      '';
+    };
+  };
+
+  networking.firewall.interfaces."tinc.retiolum".allowedTCPPorts = [ 80 ];
+}
diff --git a/hosts/web-01.cloonar.com/secrets.yaml b/hosts/web-01.cloonar.com/secrets.yaml
index aeaacef..17005c6 100644
--- a/hosts/web-01.cloonar.com/secrets.yaml
+++ b/hosts/web-01.cloonar.com/secrets.yaml
@@ -1,6 +1,7 @@
 borg-passphrase: ENC[AES256_GCM,data:V77hfP5jk/DXcvRiZKu6RLAqsJhlIelkQwA6ClYJKNmMtvAXG+g6794YJ+ooof1h8qcnMoctEWMUcsBetjaguA==,iv:OyJF/dftfEaGUnmbzrcn0P0tvnUZX4l6Vk0Qf0NwwfE=,tag:AAkRMD+jq01BPq2LSYPQGA==,type:str]
 borg-ssh-key: ENC[AES256_GCM,data:7F7uUlTP3ZKkpySj6/AGfH3K1/8/GzIdfp+ch1hU55zX51KgRs/KuGmj+yKyH9ua41oR4FR94MoiTb3u3MRpUj6dqO4VjVm6fRgJFNXOBhTUcelRR98Nq2QClkRqcNmPiHQC4bxjyW9C1tZfCA52AIILGh9O1Q9XnYAz2q8JwbrY+eTXS/U/1Fh1D+0Y9q1n+oingehal2huHzeVsLxFlip3TmGJx8QBlnAbANABKrMFxkHAlkvAVCrF1MULzeDeeWscHi5T80OvJfOZBSonNEZosw6QJVscMYxh047Yyl6YJ/sxIJjZjC7GiLuG/FC0FCLKRhD8PkZFjrDt/6xwrqePxIb7yIKZRpsvNVCplDDdVOB/V0AqRWAeZh3z813Zi+tnjynHPYaphso/qY2r/HZKlGInzW+QCUBdPVifhVL+y2TKytCWcp4A+c/3Dc2Ut59sfO+hqE946nYGl2S+kpNASHhBa7o6cnaMtfz8NT6rVtYN/3l1snlxeOUZo85XAuYCIerGsMkdVENg5RIJwIzwM0oaGwNzruq8cul5MfAf8hFMXpoLggYECynHk9TNdhsNtUzmsS9cXAyPnnzZT6HGI2/5cgU1weCHbbXAq+ZU9WZwOT41Fpwda/WrNJuMJYFCOFer2hcSLEyxsCfqmPSdwpYRRSYHiaCuVghV6lWyi1IVV5EsX9H6hD+Uetux1SEN95ga8edME16W2dubA0yukcOq+XHI7PMEHAs20H+dybsmVx1XjIsiV/XBWkrooFBXQp950p93XOK49vwNmHNohhCvmERkH0dczGQ5vTAMXwIYqydTqXWERDRzJVvK+pbzKiecEzhFHFge345poUKQS7BWNsv+eehgBAH4HEddtzzzjcPGYMHAhafAm1VqO2BvKg2r3XnzElUCjxWlGfrMkU+LhIgAjom8Mk9+Kha+OD84+iZnuoq73TjpqaUgC0TbkxLAm0DPgk8LsLE9XfJaroftKnrE7P7kyNGSQZfGYcl7ssLck5LJA07dTvDtytlL1Kk8RJe4FwFNPLtdiAnCVvKBe0kcClvCWrrxRt1QT85b+9IwG1yhRxlxheleePX1Fwlg/d2xj6AB3Cz7kAVO2phGPAF16Ke/UQeMxhbOIcy20g7RoiCpfx9jvdwExhDmbmUR7cleL1seFH+wgAM6uQwh80+q8IDJWPNyeXOwiA2siPwexb20xN+n1kIMa03U5Bn+lBqYoh0xXhlDHP3FiKew6xjSducNUtzyKm24kQaWSOfYIqTxYFRTI0bDgRu6xWGmZL37VSMWqDj3TIWOMBoLy0JBsRl6Jj07keJ0CwaNEa4cCWZ0s6nu49mp04JfJWF2r9ksX+2OVeDjSgX5JBB/V79j3I8iu4Nlp0pjyBKPlST/FUDAl1jT12X1grkYjXO8UyJ9AQqqnqK5lM+KYVR15Ui1lBb/vISPqiiw45bcUyHfVAAt7hcFpYLaYB0om3mernccN2fi26lawhhambRd7FGkILKA3byE9ytqGvDQ2CxtjA30kbGxeqXVFsyzROahR0c3KdKlnuCO9Uar4J5VEXgv2obNlNUfSMa9uleWDuhveBaE+2jtKUJd2P4wIIzF/VJGxgWGSge/ji0EV36EFfMg/Tyizdw5wtv4rQF0M+Uu6j/n/l62SmHnT/30H8IFCFuXWmtEo1xcssMymY4ricU3kJIgjGO9h+DrBP1GBczj7yVjLHTpEhRF0yP790xgsJrP4IQB2lOtGf5MCXLrBDYkOZ5xM3+Rq5ZNH0SAHRU/qtFUmLtfkcidjkPiwdqJ0e1LUtLwmSlsot1CkPs7hKORassyUrug7dCtv9QjRMDbtW61PlIbXqa59Aimql4IUcWyUybx12E8wRqmgcX5kG2wGfd9ZFUj6mXhQINFqChsTjXSavQw3u3m8kvd1mJXfjKS12ajr2X1e5wPDUEpLzc3wTOvVgZizGeykKTcKG+GXSjPXLR61ueAO26rboYiDAeSd73shFX/vvut/aB47kZobMOooljGVtnfZjVGY8dWzdNeGvBeLws7vnFrH1u/WXPxpktGz2/eJ0L8ANJd+BZ2+wC+R3OnHeDiXHfofm3dZJTncgxYPqboKKCXRzMviSCWB3poQTm5vEltsQOR6Lj17UmHu5MX7os7t7TrfW/op4Qko9ViWT5vtrUrCZzVqJS+d8hq4lm6ANMhi3Ql+nIMIxK+hjGZNBuKZEyKY+r+Lhz9E/xdOBGVB8QH4g32DWsYvaHfpcvJC8CkGO1nRIGFyrMc10lrv++XQYtiZ1Z4a0oOtQGAXaPGQTJB4KzwlGWc0+kguV+lK4h0QIvHvuorghYC4EJerNdRUviRxZB5mTDyJc1gGq6YgMr4d/a4tyXwoEzPbPGc/3YJATZPYLXOMaGDo0rd2961CFfrsneElNIZ71DWoy41P1fJG7qrfN0gLjU0aSC1vVnzDM8GQvoJGV35cONT7N7u+hjjFBFciLBNEE1DKge156EnP6VbR2LSptWrHeq3zOe9fN3FFR1WWor+lOl+SFztt7uVHCLuWxYPV+csOeBLQi6OhFnh9r4mLTc43wjPkbuci2jqEVM6Bf5gXQoEGzybhDb/3HfQK00NXovru5uEdREYDaj5NVyzyBhOT29JuOvqX7wEMNDqE//Me6Tx8bU12cuUM7Lne7grgYQMbYfLs3gKzRbeuYCqGCIz6KQMDOZvTR38EXqJxysrPb/HuiPfoSGHoAIHviJeIsy6bF7hKA8BikrMmoNc4762pKNKNzBGR+/HeQ1trDBCbYrTmlqoXPJaeANUZlD9NdwiopTJCBzjP/F61s5bpurAiJF6Ymx5yUHljZGVPFOH/ZawfhEcfYGvMUnAOup+EJCih5WQsrn2vodbVYFJ5GNMrDH9//uMi/cwqFWlBqnKGgnGdyqzXlLYTTMlv7fuh3XxkjRRTh6ZVuLyBqEBnXzSWwlcLOUdw3r/48JJ0JC2/rTLVnb+R7T2/+7uXVQISIrBx1ijtRCzFwaFWAPqsL+nMevAHnjpWl9NvanNWKIPIQ6cHGywxdi/7ynEnCYlY13fyIpagFKrgywsHcHuez8cyNtGLpg6hwbSEeN2wZYld+DQc5UH9pqPvvTuNHu3J62WJHZlkc6yBs/hiZBT/sEZUrL2Bf3SNFjzt/IcNSjlVbpP6dd6HpIH6tuN+lQJNypRe//TgKviHBkN0mzmw7IuRIDz1tcVA28COCo8NkzyD194zxDJ8yYKfKH/YwN5q8/R/r/EuUcY4qbeAokaS7XcIkOp/QFKHVM6AYKZ3SPyB5ZjkHFoHBG7YNDrx8AHDNemG+WUev/flmsEv3ykgTztgOoCmTqH/rduS+BGcKwsNW0iMwni9mUiJfuNdYLqhWohGLWb/mkUVoTgycXtsJ8x5BGkk4wSuGSwGCur8yVel7+Fr6CzsVGiPJOa0RHO6sN+Y9jnSMPIVm+mx16j3Q==,iv:ZGV3C0nvqdEnukiPkeMxDD66OjeXQF4anQLkALmBno8=,tag:ELar6NeP5bjL5L/Z5m7Piw==,type:str]
 grafana-ldap-password: ENC[AES256_GCM,data:hNB6CRtXW98yqUqInD3LsZ75sA+lVfmbooehni0UKL60qE/XCZm5B9JVO9pjxbIYZN6Eu/RFX+9L9cJVa5jnEo2MVeLS4CSjqC8BHLArlOuEdA5v8vqqJofBpBfXXN5Ca5xeUDJKz2HgtoTg7G5nTkegGZPGrmj5QQiL1xzco38=,iv:ViQAPTGxEWnjLkJlGCdCq5wW+fbr/O9er8/71VjL/GE=,tag:+Mow4cw7tvtkXvV2iSHeQw==,type:str]
+grafana-admin-password: ENC[AES256_GCM,data:365efRy8xD7SHBnVz6ZJO3l8/lfiZ5vZPZZbxnUmjKKJTMeebLY+P54moStY0wsbU9vk7sCKATCxrS5xy+FQJSgKLoajfz50OMA4+1k3Shl+skbeIikHKwFxqrljFa6HRQ2HTW6KLDPu6Z5Agkima5xdfrtc5R1SnOFg5b6D5NU=,iv:0yZGZVQd35Itj66Ff5hDfDYYx5xsNs/wc887bgMV1MY=,tag:9t8Iffg7kxSjE5eo7iv/RQ==,type:str]
 sops:
     kms: []
     gcp_kms: []
@@ -25,8 +26,8 @@ sops:
             elpwY3Q3dnRzR0loN1BiVk44TTF2VDQKs8Si2LHZ4L4oQqkYUhCI6affE0aTrWmE
             L+am++gYdygVURIh0Z6ftUuhYHPwhlCgmKxx51mKRV2ydraOdUUw0g==
             -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2023-08-18T23:07:18Z"
-    mac: ENC[AES256_GCM,data:nBSL5yMMkdotUYxjQyKw25PHRW31nrpV7XerzNcXj7+tosgYGd8yGKLLKufBG3B3w7wCmDEBD25vK95vW8mlZhCFiVitVg1sI4ZPI9gl0xQFeVNLeeKlQa0Ywnpye+4BktYcEvcZeQSMWEzvh8IjfZWssL43Q35ZROUnsWUjMiE=,iv:ixvpw/oG7lSzZO64uMWyXdtmAIzo8CKEA1h30GbaShg=,tag:Rdb/Z6VW9u6fTzZ3vC+Ljw==,type:str]
+    lastmodified: "2023-08-18T23:19:22Z"
+    mac: ENC[AES256_GCM,data:sWtJUW19HleKalg/Mfysk/b0N6YxdFcC/66BLmbcchI6s5MeGMLdYIJkNm7RKRQM5PY25d3saOqvsm5qK+keOBa0H9v0DwmFuS9cBJGa5KV6/IDoMvO8VtgDzCZ9HLtrSVTuh84bv7XL3cRd99BfSlSyHBJRpV7kJTudid2O9vo=,iv:8sOMUnsm8hyJlLvc5zG72wjKXtcbK7qnEd7Og0+yJt4=,tag:4XirU7fx0UmJSNkKgmJp8g==,type:str]
     pgp: []
     unencrypted_suffix: _unencrypted
     version: 3.7.3