{ config, pkgs, ... }:
let
  rulerConfig = {
    groups = [
      {
        name = "general";
        rules = [
          {
            alert = "Coredumps";
            # filter out failed build gitlab CI runner, users or nix build sandboxes
            expr = ''sum by (host) (count_over_time({unit=~"systemd-coredump.*"} !~ "(/runner/_work|/home|/build|/scratch)" |~ "core dumped"[10m])) > 0'';
            for = "10s";
            annotations.description = ''{{ $labels.instance }} {{ $labels.coredump_unit }} core dumped in last 10min.'';
          }
        ];
      }
    ];
  };

  rulerDir = pkgs.writeTextDir "ruler/ruler.yml" (builtins.toJSON rulerConfig);
in
{
  systemd.tmpfiles.rules = [
    "d /var/lib/loki 0700 loki loki - -"
    "d /var/lib/loki/ruler 0700 loki loki - -"
  ];
  services.loki = {
    enable = true;
    configuration = {
      # Basic stuff
      auth_enabled = false;
      server = {
        http_listen_port = 3100;
        log_level = "warn";
      };

      # Distributor
      distributor.ring.kvstore.store = "inmemory";

      # Ingester
      ingester = {
        lifecycler.address = "0.0.0.0";
        lifecycler.ring = {
          kvstore.store = "inmemory";
          replication_factor = 1;
        };
        chunk_encoding = "snappy";
        # Disable block transfers on shutdown
      };

      # Storage
      storage_config = {
        boltdb.directory = "/var/lib/loki/boltdb";
        boltdb_shipper = {
          active_index_directory = "/var/lib/loki/index";
          cache_location = "/var/lib/loki/boltdb-cache";
        };
        tsdb_shipper = {
          active_index_directory = "/var/lib/loki/tsdb-index";
          cache_location = "/var/lib/loki/tsdb-cache";

        };
        filesystem.directory = "/var/lib/loki/storage";

      };

      limits_config.retention_period = "48h";

      # Table manager
      table_manager = {
        retention_deletes_enabled = true;
        retention_period = "48h";
      };

      compactor = {
        retention_enabled = true;
        compaction_interval = "10m";
        working_directory = "/var/lib/loki/compactor";
        retention_delete_delay = "2h";
        retention_delete_worker_count = 150;
        delete_request_store = "filesystem";
      };

      # Schema
      schema_config.configs = [
        {
          from = "2020-11-08";
          store = "boltdb-shipper";
          object_store = "filesystem";
          schema = "v13";
          index.prefix = "index_";
          index.period = "24h";
        }
        {
          from = "2024-04-01";
          store = "tsdb";
          object_store = "filesystem";
          schema = "v13";
          index.prefix = "index_";
          index.period = "24h";
        }
      ];

      limits_config.ingestion_burst_size_mb = 16;

      ruler = {
        storage = {
          type = "local";
          local.directory = rulerDir;
        };
        rule_path = "/var/lib/loki/ruler";
        alertmanager_url = "http://alertmanager.cloonar.com";
        ring.kvstore.store = "inmemory";
      };

      query_range.cache_results = true;
      query_range.parallelise_shardable_queries = false;
      limits_config.split_queries_by_interval = "24h";
    };
  };

  sops.secrets.promtail-nginx-password.owner = "nginx";

  services.nginx.virtualHosts."loki.cloonar.com" = {
    forceSSL = true;
    enableACME = true;
    acmeRoot = null;
    locations."/" = {
      proxyWebsockets = true;
      extraConfig = ''
        auth_basic "Loki password";
        auth_basic_user_file ${config.sops.secrets.promtail-nginx-password.path};

        proxy_read_timeout 1800s;
        proxy_redirect off;
        proxy_connect_timeout 1600s;

        access_log off;
        proxy_pass http://127.0.0.1:3100;
      '';
    };
    locations."/ready" = {
      proxyWebsockets = true;
      extraConfig = ''
        auth_basic off;
        access_log off;
        proxy_pass http://127.0.0.1:3100;
      '';
    };
  };
}