diff --git a/hosts/fw/configuration.nix b/hosts/fw/configuration.nix index 8a00734..930d428 100644 --- a/hosts/fw/configuration.nix +++ b/hosts/fw/configuration.nix @@ -9,6 +9,7 @@ ./utils/modules/autoupgrade.nix ./utils/modules/promtail ./utils/modules/borgbackup.nix + # ./utils/modules/netdata.nix # fw ./modules/network-prefix.nix diff --git a/hosts/mail/configuration.nix b/hosts/mail/configuration.nix index e4fd7e2..0bc9d7b 100644 --- a/hosts/mail/configuration.nix +++ b/hosts/mail/configuration.nix @@ -14,6 +14,7 @@ ./utils/modules/borgbackup.nix ./utils/modules/promtail ./utils/modules/victoriametrics + ./utils/modules/netdata.nix ./modules/set-nix-channel.nix # Automatically manage nix-channel from /var/bento/channel ./hardware-configuration.nix diff --git a/hosts/web-arm/configuration.nix b/hosts/web-arm/configuration.nix index be836b4..54c74d9 100644 --- a/hosts/web-arm/configuration.nix +++ b/hosts/web-arm/configuration.nix @@ -1,4 +1,4 @@ -{ config, lib, pkgs, ... }: { +{ lib, pkgs, ... }: { imports = [ ./utils/bento.nix ./utils/modules/sops.nix @@ -17,12 +17,12 @@ ./modules/grafana/default.nix ./modules/loki.nix ./modules/victoriametrics.nix - ./modules/blackbox-exporter.nix ./modules/updns.nix ./utils/modules/autoupgrade.nix ./utils/modules/promtail ./utils/modules/borgbackup.nix + ./utils/modules/netdata.nix ./modules/set-nix-channel.nix # Automatically manage nix-channel from /var/bento/channel ./hardware-configuration.nix diff --git a/hosts/web-arm/modules/blackbox-exporter.nix b/hosts/web-arm/modules/blackbox-exporter.nix deleted file mode 100644 index 27f265a..0000000 --- a/hosts/web-arm/modules/blackbox-exporter.nix +++ /dev/null @@ -1,82 +0,0 @@ -{ config, pkgs, lib, ... }: - -with lib; - -let - hostname = config.networking.hostName; - - cfg = config.services.blackbox-exporter; - nginxVHosts = config.services.nginx.virtualHosts or {}; - allDomains = lib.attrNames nginxVHosts; - filteredDomains = builtins.filter (d: !builtins.elem d cfg.blacklistDomains) allDomains; - httpsDomains = lib.map (d: "https://${d}") filteredDomains; - domainsString = builtins.concatStringsSep "\n " - (map (d: "\"${d}\",") httpsDomains); -in { - options.services.blackbox-exporter.blacklistDomains = mkOption { - type = types.listOf types.str; - default = []; - description = "List of domains to monitor with Blackbox Exporter"; - }; - - config = { - services.blackbox-exporter = { - blacklistDomains = [ - "autoconfig.cloonar.com" - "cloonar.dev" - "loki.cloonar.com" - "stage.korean-skin.care" - "victoria-server.cloonar.com" - "updns.cloonar.com" - "feeds.jordanrannells.com" - ]; - }; - # Systemd service for Blackbox Exporter - systemd.services.blackbox-exporter = { - description = "Blackbox Exporter"; - after = [ "network-online.target" ]; - wantedBy = [ "multi-user.target" ]; - serviceConfig.ExecStart = '' - ${pkgs.prometheus-blackbox-exporter}/bin/blackbox_exporter \ - --config.file=/etc/blackbox_exporter/blackbox.yml - ''; - }; - - # Configuration file for Blackbox Exporter - environment.etc."blackbox_exporter/blackbox.yml".text = '' - modules: - http_2xx: - prober: http - ''; - - # Add scrape config for VictoriaMetrics agent - services.victoriametrics.extraScrapeConfigs = [ - '' - - job_name: "blackbox_http_all_domains" - metrics_path: "/probe" - params: - module: ["http_2xx"] - - static_configs: - - targets: - [ - ${domainsString} - ] - - relabel_configs: - - source_labels: ["__address__"] - target_label: "__param_target" - regex: '(.*)' - replacement: "$1" - - source_labels: ["__param_target"] - target_label: "instance" - - target_label: "__address__" - replacement: "127.0.0.1:9115" - - source_labels: ["__address__"] - regex: "127\\.0\\.0\\.1:9115" - target_label: "__scheme__" - replacement: "http" - '' - ]; - }; -} diff --git a/hosts/web-arm/modules/grafana/alerting/websites/default.nix b/hosts/web-arm/modules/grafana/alerting/websites/default.nix deleted file mode 100644 index b6139f6..0000000 --- a/hosts/web-arm/modules/grafana/alerting/websites/default.nix +++ /dev/null @@ -1,74 +0,0 @@ -{ lib, pkgs, config, ... }: - -let - cfg = config.services.blackbox-exporter; - nginxVHosts = config.services.nginx.virtualHosts or {}; - allDomains = lib.attrNames nginxVHosts; - filteredDomains = builtins.filter (d: !builtins.elem d cfg.blacklistDomains) allDomains; - httpsDomains = lib.map (d: "https://${d}") filteredDomains; - websiteAlertRules = lib.map (target: - let - domain = lib.replaceStrings ["://" "." "-" "/" ] ["-" "-" "_" "_"] target + "-down-alert"; - uid = builtins.hashString "sha1" domain; - in { - uid = uid; - title = "Website " + target + " Down"; - condition = "C"; - - data = [ - { - refId = "A"; - relativeTimeRange = { from = 300; to = 0; }; - datasourceUid = "vm-datasource-uid"; - model = { - editorMode = "code"; - expr = "probe_success{instance=\"" + target + "\"} OR on() vector(0)"; - hide = false; - intervalMs = 1000; - legendFormat = target; - maxDataPoints = 43200; - range = true; - refId = "A"; - }; - } - { - refId = "B"; - datasourceUid = "__expr__"; - model = { - type = "reduce"; - expression = "A"; - reducer = "last"; - }; - } - { - refId = "C"; - datasourceUid = "__expr__"; - model = { - type = "math"; - expression = "$B == 0"; - }; - } - ]; - noDataState = "Alerting"; - execErrState = "Alerting"; - for = "5m"; - annotations = { - description = "Website " + target + " is unreachable."; - summary = "Website Down"; - }; - labels = { - severity = "critical"; - website_url = target; - }; - } - ) httpsDomains; -in { - services.grafana.provision.alerting.rules.settings.groups = [ - { - name = "Website Alerts"; - folder = "Websites"; - interval = "1m"; - rules = websiteAlertRules; - } - ]; -} diff --git a/hosts/web-arm/modules/grafana/default.nix b/hosts/web-arm/modules/grafana/default.nix index db3286b..8bf0771 100644 --- a/hosts/web-arm/modules/grafana/default.nix +++ b/hosts/web-arm/modules/grafana/default.nix @@ -31,7 +31,6 @@ in # Individual alert files removed, now handled by alerting/system/default.nix ./alerting/system/default.nix # Added: Imports the consolidated system alerts module ./alerting/service/default.nix # Added: Imports the new service alerts module - ./alerting/websites/default.nix # Added: Imports the new websites alerts module # ... other rule files can be added here ... ./datasources/victoriametrics.nix ./datasources/loki.nix # Add Loki datasource @@ -67,7 +66,9 @@ in use_pkce = true; }; - "auth.anonymous".enabled = false; + "auth.anonymous".enabled = true; + "auth.anonymous".org_name = "Cloonar e.U."; + "auth.anonymous".org_role = "Viewer"; server = { root_url = "https://grafana.cloonar.com"; diff --git a/hosts/web-arm/modules/victoriametrics.nix b/hosts/web-arm/modules/victoriametrics.nix index 257e02f..a2788e7 100644 --- a/hosts/web-arm/modules/victoriametrics.nix +++ b/hosts/web-arm/modules/victoriametrics.nix @@ -1,83 +1,43 @@ -{ config, lib, ... }: -with lib; +{ config, ... }: let - # configure_prom = builtins.toFile "prometheus.yml" '' - # scrape_configs: - # - job_name: 'server' - # stream_parse: true - # static_configs: - # - targets: - # - ${config.networking.hostName}:9100 - # ''; configure_prom = builtins.toFile "prometheus.yml" '' scrape_configs: - # System metrics - - job_name: 'node' + - job_name: 'server' stream_parse: true static_configs: - targets: - ${config.networking.hostName}:9100 - - # Systemd service monitoring - - job_name: 'systemd' - metrics_path: /metrics - params: - collect[]: - - 'systemd.service.state' - - 'systemd.service.start_time_seconds' - - 'systemd.unit_file.state' - static_configs: - - targets: - - ${config.networking.hostName}:9100 - relabel_configs: - # Filter for specific services we want to monitor - - source_labels: [__name__] - regex: 'node_systemd_unit_state' - action: keep - - source_labels: [name] - regex: '(container@git|microvm@git-runner-|postfix|dovecot|openldap|wireguard-wg_cloonar).*\.service' - action: keep - ${concatStringsSep "\n" config.services.victoriametrics.extraScrapeConfigs} ''; in { - options.services.victoriametrics = { - extraScrapeConfigs = mkOption { - type = types.listOf types.str; - default = []; - description = "Additional Prometheus scrape job YAML snippets for Blackbox Exporter probes"; + services.prometheus.exporters.node.enable = true; + + sops.secrets.victoria-nginx-password.owner = "nginx"; + + services.victoriametrics = { + enable = true; + extraOptions = [ + "-promscrape.config=${configure_prom}" + ]; + }; + + services.nginx.virtualHosts."victoria-server.cloonar.com" = { + forceSSL = true; + enableACME = true; + acmeRoot = null; + locations."/" = { + proxyWebsockets = true; + extraConfig = '' + auth_basic "Victoria password"; + auth_basic_user_file ${config.sops.secrets.victoria-nginx-password.path}; + + proxy_read_timeout 1800s; + proxy_redirect off; + proxy_connect_timeout 1600s; + + access_log off; + proxy_pass http://127.0.0.1:8428; + ''; }; }; - config = { - services.prometheus.exporters.node.enable = true; - - sops.secrets.victoria-nginx-password.owner = "nginx"; - - services.victoriametrics = { - enable = true; - extraOptions = [ - "-promscrape.config=${configure_prom}" - ]; - }; - - services.nginx.virtualHosts."victoria-server.cloonar.com" = { - forceSSL = true; - enableACME = true; - acmeRoot = null; - locations."/" = { - proxyWebsockets = true; - extraConfig = '' - auth_basic "Victoria password"; - auth_basic_user_file ${config.sops.secrets.victoria-nginx-password.path}; - - proxy_read_timeout 1800s; - proxy_redirect off; - proxy_connect_timeout 1600s; - - access_log off; - proxy_pass http://127.0.0.1:8428; - ''; - }; - }; - }; } diff --git a/hosts/web-arm/modules/web/typo3.nix b/hosts/web-arm/modules/web/typo3.nix index 409228f..1e7db9c 100644 --- a/hosts/web-arm/modules/web/typo3.nix +++ b/hosts/web-arm/modules/web/typo3.nix @@ -97,6 +97,18 @@ in }; config = { + # systemd.services = mapAttrs' (instance: instanceOpts: + # let + # domain = if instanceOpts.domain != null then instanceOpts.domain else instance; + # in + # nameValuePair "phpfpm-${domain}" { + # serviceConfig = { + # ProtectHome = lib.mkForce "tmpfs"; + # BindPaths = "BindPaths=/var/www/${domain}:/var/www/${domain}"; + # }; + # } + # ) cfg.instances; + systemd.timers = mapAttrs' (instance: instanceOpts: let domain = if instanceOpts.domain != null then instanceOpts.domain else instance; @@ -232,6 +244,45 @@ in } ''; + # locations."/typo3/login" = { + # extraConfig = '' + # # Basic Authelia Config + # # Send a subsequent request to Authelia to verify if the user is authenticated + # # and has the right permissions to access the resource. + # auth_request /authelia; + # # Set the `target_url` variable based on the request. It will be used to build the portal + # # URL with the correct redirection parameter. + # auth_request_set $target_url $scheme://$http_host$request_uri; + # # Set the X-Forwarded-User and X-Forwarded-Groups with the headers + # # returned by Authelia for the backends which can consume them. + # # This is not safe, as the backend must make sure that they come from the + # # proxy. In the future, it's gonna be safe to just use OAuth. + # auth_request_set $user $upstream_http_remote_user; + # auth_request_set $groups $upstream_http_remote_groups; + # auth_request_set $name $upstream_http_remote_name; + # auth_request_set $email $upstream_http_remote_email; + # proxy_set_header Remote-User $user; + # proxy_set_header Remote-Groups $groups; + # proxy_set_header Remote-Name $name; + # proxy_set_header Remote-Email $email; + # # If Authelia returns 401, then nginx redirects the user to the login portal. + # # If it returns 200, then the request pass through to the backend. + # # For other type of errors, nginx will handle them as usual. + # error_page 401 =302 https://auth.cloonar.com/?rd=$target_url; + # + # fastcgi_param REMOTE_USER $user; + # + # include ${pkgs.nginx}/conf/fastcgi.conf; + # fastcgi_buffer_size 32k; + # fastcgi_buffers 8 16k; + # fastcgi_connect_timeout 240s; + # fastcgi_read_timeout 240s; + # fastcgi_send_timeout 240s; + # fastcgi_pass unix:${config.services.phpfpm.pools."${domain}".socket}; + # fastcgi_param SCRIPT_FILENAME ${cfg.dataDir}/${domain}/public/typo3/index.php; + # ''; + # }; + locations."/favicon.ico".extraConfig = '' log_not_found off; access_log off; diff --git a/hosts/web-arm/sites/autoconfig.cloonar.com.nix b/hosts/web-arm/sites/autoconfig.cloonar.com.nix index d45fb8f..8b40ad0 100644 --- a/hosts/web-arm/sites/autoconfig.cloonar.com.nix +++ b/hosts/web-arm/sites/autoconfig.cloonar.com.nix @@ -27,4 +27,10 @@ in proxyPass = "http://localhost:1323/"; }; }; + services.nginx.virtualHosts."autoconfig.superbros.tv".extraConfig = '' + return 301 https://autoconfig.cloonar.com$request_uri; + ''; + services.nginx.virtualHosts."autoconfig.korean-skin.care".extraConfig = '' + return 301 https://autoconfig.cloonar.com$request_uri; + ''; } diff --git a/hosts/web-arm/sites/autoconfig.nix b/hosts/web-arm/sites/autoconfig.nix index 465c69e..963d2b8 100644 --- a/hosts/web-arm/sites/autoconfig.nix +++ b/hosts/web-arm/sites/autoconfig.nix @@ -1,5 +1,11 @@ { pkgs, lib, config, ... }: let + domains = [ + "cloonar.com" + "ghetto.at" + "optiprot.eu" + ]; + vhostConfig = { forceSSL = true; enableACME = true; @@ -42,6 +48,10 @@ let in { services.nginx.virtualHosts."autoconfig.cloonar.com" = vhostConfig; + services.nginx.virtualHosts."autoconfig.ghetto.at" = vhostConfig; + services.nginx.virtualHosts."autoconfig.optiprot.eu" = vhostConfig; + services.nginx.virtualHosts."autoconfig.superbros.tv" = vhostConfig; + services.nginx.virtualHosts."autoconfig.korean-skin.care" = vhostConfig; systemd.services."phpfpm-autoconfig".serviceConfig.ProtectHome = lib.mkForce false; diff --git a/scripts/test-configuration b/scripts/test-configuration index 6c6dcd1..f01e24f 100755 --- a/scripts/test-configuration +++ b/scripts/test-configuration @@ -45,7 +45,7 @@ fi # Execute nixos-rebuild dry-build # Store the output and error streams, and the exit code -NIX_OUTPUT_ERR=$(nixos-rebuild dry-build $SHOW_TRACE_OPT -I nixos-config="$CONFIG_PATH" --show-trace 2>&1) +NIX_OUTPUT_ERR=$(nixos-rebuild dry-build $SHOW_TRACE_OPT -I nixos-config="$CONFIG_PATH" 2>&1) NIX_EXIT_STATUS=$? # Check the exit status @@ -61,4 +61,4 @@ else echo "Output from nixos-rebuild:" >&2 echo "$NIX_OUTPUT_ERR" >&2 exit "$NIX_EXIT_STATUS" -fi +fi \ No newline at end of file diff --git a/utils/modules/victoriametrics/default.nix b/utils/modules/victoriametrics/default.nix index b39821c..fd14825 100644 --- a/utils/modules/victoriametrics/default.nix +++ b/utils/modules/victoriametrics/default.nix @@ -1,5 +1,4 @@ -{ config, lib, pkgs, ... }: -with lib; +{ config, pkgs, ... }: let configure_prom = builtins.toFile "prometheus.yml" '' scrape_configs: @@ -29,41 +28,29 @@ let - source_labels: [name] regex: '(container@git|microvm@git-runner-|postfix|dovecot|openldap|wireguard-wg_cloonar).*\.service' action: keep - - ${concatStringsSep "\n " config.services.victoriametrics.extraScrapeConfigs} ''; in { - options.services.victoriametrics = { - extraScrapeConfigs = mkOption { - type = types.listOf types.str; - default = []; - description = "Additional Prometheus scrape job YAML snippets for Blackbox Exporter probes"; - }; + sops.secrets.victoria-agent-env = { + sopsFile = ./secrets.yaml; }; - config = { - sops.secrets.victoria-agent-env = { - sopsFile = ./secrets.yaml; - }; + # Node exporter for system metrics + services.prometheus.exporters.node = { + enable = true; + enabledCollectors = [ + "systemd" # Enable systemd collector for service monitoring + ]; + }; + + systemd.services.export-to-prometheus = { + path = with pkgs; [victoriametrics]; + enable = true; + after = ["network-online.target"]; + wantedBy = ["multi-user.target"]; + script = "vmagent -promscrape.config=${configure_prom} -envflag.enable -remoteWrite.url=https://victoria-server.cloonar.com/api/v1/write"; - # Node exporter for system metrics - services.prometheus.exporters.node = { - enable = true; - enabledCollectors = [ - "systemd" # Enable systemd collector for service monitoring - ]; - }; - - systemd.services.export-to-prometheus = { - path = with pkgs; [victoriametrics]; - enable = true; - after = ["network-online.target"]; - wantedBy = ["multi-user.target"]; - script = "vmagent -promscrape.config=${configure_prom} -envflag.enable -remoteWrite.url=https://victoria-server.cloonar.com/api/v1/write"; - - serviceConfig = { - EnvironmentFile=config.sops.secrets.victoria-agent-env.path; - }; + serviceConfig = { + EnvironmentFile=config.sops.secrets.victoria-agent-env.path; }; }; }