add loki and promtail to web host
This commit is contained in:
11
.sops.yaml
11
.sops.yaml
@@ -90,3 +90,14 @@ creation_rules:
|
|||||||
- age:
|
- age:
|
||||||
- *dominik
|
- *dominik
|
||||||
- *web-01-server
|
- *web-01-server
|
||||||
|
- path_regex: utils/modules/promtail/[^/]+\.yaml$
|
||||||
|
key_groups:
|
||||||
|
- age:
|
||||||
|
- *dominik
|
||||||
|
- *git-server
|
||||||
|
- *web-01-server
|
||||||
|
- *home-assistant-server
|
||||||
|
- *ldap-server-arm
|
||||||
|
- *ldap-server-test
|
||||||
|
- *testmodules
|
||||||
|
- *netboot
|
||||||
|
|||||||
@@ -13,6 +13,8 @@
|
|||||||
./utils/modules/nextcloud
|
./utils/modules/nextcloud
|
||||||
./modules/postgresql.nix
|
./modules/postgresql.nix
|
||||||
./modules/grafana.nix
|
./modules/grafana.nix
|
||||||
|
./modules/loki.nix
|
||||||
|
./utils/modules/promtail
|
||||||
|
|
||||||
./utils/modules/borgbackup.nix
|
./utils/modules/borgbackup.nix
|
||||||
./utils/modules/netdata.nix
|
./utils/modules/netdata.nix
|
||||||
|
|||||||
248
hosts/web-01.cloonar.com/modules/prometheus.nix
Normal file
248
hosts/web-01.cloonar.com/modules/prometheus.nix
Normal file
@@ -0,0 +1,248 @@
|
|||||||
|
{ config, ... }:
|
||||||
|
{
|
||||||
|
sops.secrets.alertmanager = { };
|
||||||
|
sops.secrets.hass-token.owner = "prometheus";
|
||||||
|
|
||||||
|
# imports = [
|
||||||
|
# ./matrix-alertmanager.nix
|
||||||
|
# ./irc-alertmanager.nix
|
||||||
|
# ./rules.nix
|
||||||
|
# ];
|
||||||
|
|
||||||
|
services.prometheus = {
|
||||||
|
webExternalUrl = "https://prometheus.cloonar.com";
|
||||||
|
scrapeConfigs = [
|
||||||
|
{
|
||||||
|
job_name = "telegraf";
|
||||||
|
scrape_interval = "60s";
|
||||||
|
metrics_path = "/metrics";
|
||||||
|
static_configs = [
|
||||||
|
{
|
||||||
|
targets = [
|
||||||
|
"turingmachine.r:9273"
|
||||||
|
"bernie.r:9273"
|
||||||
|
#"rock.r:9273"
|
||||||
|
];
|
||||||
|
labels.type = "mobile";
|
||||||
|
}
|
||||||
|
{
|
||||||
|
targets = [
|
||||||
|
"eva.r:9273"
|
||||||
|
"eve.r:9273"
|
||||||
|
"blob64.r:9273"
|
||||||
|
"matchbox.r:9273"
|
||||||
|
"alertmanager.r:80"
|
||||||
|
"prometheus.r:80"
|
||||||
|
#"rock.r:9273"
|
||||||
|
];
|
||||||
|
}
|
||||||
|
{
|
||||||
|
targets = [
|
||||||
|
"rauter.r:9273"
|
||||||
|
];
|
||||||
|
# to make it compatible with the node-exporter dashboard
|
||||||
|
labels.host = "rauter.r:9273";
|
||||||
|
}
|
||||||
|
{
|
||||||
|
targets = [
|
||||||
|
"prism.r:9273"
|
||||||
|
"gum.r:9273"
|
||||||
|
"kelle.r:9273"
|
||||||
|
];
|
||||||
|
|
||||||
|
labels.org = "krebs";
|
||||||
|
}
|
||||||
|
{
|
||||||
|
targets = [
|
||||||
|
"clan.lol:9273"
|
||||||
|
];
|
||||||
|
|
||||||
|
labels.org = "clan-lol";
|
||||||
|
}
|
||||||
|
#{
|
||||||
|
# targets = [
|
||||||
|
# "dev1.numtide.com.r:9273"
|
||||||
|
# ];
|
||||||
|
|
||||||
|
# labels.org = "numtide";
|
||||||
|
#}
|
||||||
|
{
|
||||||
|
targets = map (host: "${host}.r:9273") [
|
||||||
|
# university
|
||||||
|
"amy"
|
||||||
|
"clara"
|
||||||
|
"rose"
|
||||||
|
|
||||||
|
"astrid"
|
||||||
|
"dan"
|
||||||
|
"mickey"
|
||||||
|
"bill"
|
||||||
|
"nardole"
|
||||||
|
"yasmin"
|
||||||
|
"ryan"
|
||||||
|
"graham"
|
||||||
|
|
||||||
|
"astrid"
|
||||||
|
"dan"
|
||||||
|
"mickey"
|
||||||
|
|
||||||
|
"jackson"
|
||||||
|
"christina"
|
||||||
|
"adelaide"
|
||||||
|
"wilfred"
|
||||||
|
"river"
|
||||||
|
"jack"
|
||||||
|
|
||||||
|
"ruby"
|
||||||
|
];
|
||||||
|
|
||||||
|
labels.org = "uni";
|
||||||
|
}
|
||||||
|
];
|
||||||
|
}
|
||||||
|
{
|
||||||
|
job_name = "homeassistant";
|
||||||
|
scrape_interval = "60s";
|
||||||
|
metrics_path = "/api/prometheus";
|
||||||
|
|
||||||
|
authorization.credentials_file = config.sops.secrets.hass-token.path;
|
||||||
|
|
||||||
|
scheme = "https";
|
||||||
|
static_configs = [
|
||||||
|
{
|
||||||
|
targets = [
|
||||||
|
"home-assistant.cloonar.com:443"
|
||||||
|
];
|
||||||
|
}
|
||||||
|
];
|
||||||
|
}
|
||||||
|
{
|
||||||
|
job_name = "gitea";
|
||||||
|
scrape_interval = "60s";
|
||||||
|
metrics_path = "/metrics";
|
||||||
|
|
||||||
|
scheme = "https";
|
||||||
|
static_configs = [
|
||||||
|
{
|
||||||
|
targets = [
|
||||||
|
"git.cloonar.com:443"
|
||||||
|
];
|
||||||
|
}
|
||||||
|
];
|
||||||
|
}
|
||||||
|
];
|
||||||
|
alertmanagers = [
|
||||||
|
{
|
||||||
|
static_configs = [
|
||||||
|
{
|
||||||
|
targets = [ "localhost:9093" ];
|
||||||
|
}
|
||||||
|
];
|
||||||
|
}
|
||||||
|
];
|
||||||
|
};
|
||||||
|
services.prometheus.alertmanager = {
|
||||||
|
enable = true;
|
||||||
|
environmentFile = config.sops.secrets.alertmanager.path;
|
||||||
|
webExternalUrl = "https://alertmanager.cloonar.com";
|
||||||
|
listenAddress = "[::1]";
|
||||||
|
configuration = {
|
||||||
|
global = {
|
||||||
|
# The smarthost and SMTP sender used for mail notifications.
|
||||||
|
smtp_smarthost = "mail.cloonar.com:587";
|
||||||
|
smtp_from = "alertmanager@cloonar.com";
|
||||||
|
smtp_auth_username = "alertmanager@cloonar.com";
|
||||||
|
smtp_auth_password = "$SMTP_PASSWORD";
|
||||||
|
};
|
||||||
|
route = {
|
||||||
|
receiver = "default";
|
||||||
|
routes = [
|
||||||
|
{
|
||||||
|
group_by = [ "host" ];
|
||||||
|
match_re.org = "krebs";
|
||||||
|
group_wait = "5m";
|
||||||
|
group_interval = "5m";
|
||||||
|
repeat_interval = "4h";
|
||||||
|
receiver = "krebs";
|
||||||
|
}
|
||||||
|
{
|
||||||
|
group_by = [ "host" ];
|
||||||
|
match_re.org = "nix-community";
|
||||||
|
group_wait = "5m";
|
||||||
|
group_interval = "5m";
|
||||||
|
repeat_interval = "4h";
|
||||||
|
receiver = "nix-community";
|
||||||
|
}
|
||||||
|
{
|
||||||
|
group_by = [ "host" ];
|
||||||
|
match_re.org = "clan-lol";
|
||||||
|
group_wait = "5m";
|
||||||
|
group_interval = "5m";
|
||||||
|
repeat_interval = "4h";
|
||||||
|
receiver = "clan-lol";
|
||||||
|
}
|
||||||
|
{
|
||||||
|
group_by = [ "host" ];
|
||||||
|
group_wait = "30s";
|
||||||
|
group_interval = "2m";
|
||||||
|
repeat_interval = "2h";
|
||||||
|
receiver = "all";
|
||||||
|
}
|
||||||
|
];
|
||||||
|
};
|
||||||
|
receivers = [
|
||||||
|
{
|
||||||
|
name = "krebs";
|
||||||
|
webhook_configs = [
|
||||||
|
{
|
||||||
|
url = "http://127.0.0.1:9223/";
|
||||||
|
max_alerts = 5;
|
||||||
|
}
|
||||||
|
];
|
||||||
|
}
|
||||||
|
#{
|
||||||
|
# name = "numtide";
|
||||||
|
# slack_configs = [
|
||||||
|
# {
|
||||||
|
# token = "$SLACK_TOKEN";
|
||||||
|
# api_url = "https://";
|
||||||
|
# }
|
||||||
|
# ];
|
||||||
|
#}
|
||||||
|
{
|
||||||
|
name = "nix-community";
|
||||||
|
webhook_configs = [
|
||||||
|
{
|
||||||
|
url = "http://localhost:9088/alert";
|
||||||
|
max_alerts = 5;
|
||||||
|
}
|
||||||
|
];
|
||||||
|
}
|
||||||
|
{
|
||||||
|
name = "clan-lol";
|
||||||
|
webhook_configs = [
|
||||||
|
# TODO
|
||||||
|
#{
|
||||||
|
# url = "http://localhost:4050/services/hooks/YWxlcnRtYW5hZ2VyX3NlcnZpY2U";
|
||||||
|
# max_alerts = 5;
|
||||||
|
#}
|
||||||
|
];
|
||||||
|
}
|
||||||
|
{
|
||||||
|
name = "all";
|
||||||
|
pushover_configs = [
|
||||||
|
{
|
||||||
|
user_key = "$PUSHOVER_USER_KEY";
|
||||||
|
token = "$PUSHOVER_TOKEN";
|
||||||
|
priority = "0";
|
||||||
|
}
|
||||||
|
];
|
||||||
|
}
|
||||||
|
{
|
||||||
|
name = "default";
|
||||||
|
}
|
||||||
|
];
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
}
|
||||||
@@ -2,6 +2,7 @@ borg-passphrase: ENC[AES256_GCM,data:V77hfP5jk/DXcvRiZKu6RLAqsJhlIelkQwA6ClYJKNm
|
|||||||
borg-ssh-key: ENC[AES256_GCM,data:7F7uUlTP3ZKkpySj6/AGfH3K1/8/GzIdfp+ch1hU55zX51KgRs/KuGmj+yKyH9ua41oR4FR94MoiTb3u3MRpUj6dqO4VjVm6fRgJFNXOBhTUcelRR98Nq2QClkRqcNmPiHQC4bxjyW9C1tZfCA52AIILGh9O1Q9XnYAz2q8JwbrY+eTXS/U/1Fh1D+0Y9q1n+oingehal2huHzeVsLxFlip3TmGJx8QBlnAbANABKrMFxkHAlkvAVCrF1MULzeDeeWscHi5T80OvJfOZBSonNEZosw6QJVscMYxh047Yyl6YJ/sxIJjZjC7GiLuG/FC0FCLKRhD8PkZFjrDt/6xwrqePxIb7yIKZRpsvNVCplDDdVOB/V0AqRWAeZh3z813Zi+tnjynHPYaphso/qY2r/HZKlGInzW+QCUBdPVifhVL+y2TKytCWcp4A+c/3Dc2Ut59sfO+hqE946nYGl2S+kpNASHhBa7o6cnaMtfz8NT6rVtYN/3l1snlxeOUZo85XAuYCIerGsMkdVENg5RIJwIzwM0oaGwNzruq8cul5MfAf8hFMXpoLggYECynHk9TNdhsNtUzmsS9cXAyPnnzZT6HGI2/5cgU1weCHbbXAq+ZU9WZwOT41Fpwda/WrNJuMJYFCOFer2hcSLEyxsCfqmPSdwpYRRSYHiaCuVghV6lWyi1IVV5EsX9H6hD+Uetux1SEN95ga8edME16W2dubA0yukcOq+XHI7PMEHAs20H+dybsmVx1XjIsiV/XBWkrooFBXQp950p93XOK49vwNmHNohhCvmERkH0dczGQ5vTAMXwIYqydTqXWERDRzJVvK+pbzKiecEzhFHFge345poUKQS7BWNsv+eehgBAH4HEddtzzzjcPGYMHAhafAm1VqO2BvKg2r3XnzElUCjxWlGfrMkU+LhIgAjom8Mk9+Kha+OD84+iZnuoq73TjpqaUgC0TbkxLAm0DPgk8LsLE9XfJaroftKnrE7P7kyNGSQZfGYcl7ssLck5LJA07dTvDtytlL1Kk8RJe4FwFNPLtdiAnCVvKBe0kcClvCWrrxRt1QT85b+9IwG1yhRxlxheleePX1Fwlg/d2xj6AB3Cz7kAVO2phGPAF16Ke/UQeMxhbOIcy20g7RoiCpfx9jvdwExhDmbmUR7cleL1seFH+wgAM6uQwh80+q8IDJWPNyeXOwiA2siPwexb20xN+n1kIMa03U5Bn+lBqYoh0xXhlDHP3FiKew6xjSducNUtzyKm24kQaWSOfYIqTxYFRTI0bDgRu6xWGmZL37VSMWqDj3TIWOMBoLy0JBsRl6Jj07keJ0CwaNEa4cCWZ0s6nu49mp04JfJWF2r9ksX+2OVeDjSgX5JBB/V79j3I8iu4Nlp0pjyBKPlST/FUDAl1jT12X1grkYjXO8UyJ9AQqqnqK5lM+KYVR15Ui1lBb/vISPqiiw45bcUyHfVAAt7hcFpYLaYB0om3mernccN2fi26lawhhambRd7FGkILKA3byE9ytqGvDQ2CxtjA30kbGxeqXVFsyzROahR0c3KdKlnuCO9Uar4J5VEXgv2obNlNUfSMa9uleWDuhveBaE+2jtKUJd2P4wIIzF/VJGxgWGSge/ji0EV36EFfMg/Tyizdw5wtv4rQF0M+Uu6j/n/l62SmHnT/30H8IFCFuXWmtEo1xcssMymY4ricU3kJIgjGO9h+DrBP1GBczj7yVjLHTpEhRF0yP790xgsJrP4IQB2lOtGf5MCXLrBDYkOZ5xM3+Rq5ZNH0SAHRU/qtFUmLtfkcidjkPiwdqJ0e1LUtLwmSlsot1CkPs7hKORassyUrug7dCtv9QjRMDbtW61PlIbXqa59Aimql4IUcWyUybx12E8wRqmgcX5kG2wGfd9ZFUj6mXhQINFqChsTjXSavQw3u3m8kvd1mJXfjKS12ajr2X1e5wPDUEpLzc3wTOvVgZizGeykKTcKG+GXSjPXLR61ueAO26rboYiDAeSd73shFX/vvut/aB47kZobMOooljGVtnfZjVGY8dWzdNeGvBeLws7vnFrH1u/WXPxpktGz2/eJ0L8ANJd+BZ2+wC+R3OnHeDiXHfofm3dZJTncgxYPqboKKCXRzMviSCWB3poQTm5vEltsQOR6Lj17UmHu5MX7os7t7TrfW/op4Qko9ViWT5vtrUrCZzVqJS+d8hq4lm6ANMhi3Ql+nIMIxK+hjGZNBuKZEyKY+r+Lhz9E/xdOBGVB8QH4g32DWsYvaHfpcvJC8CkGO1nRIGFyrMc10lrv++XQYtiZ1Z4a0oOtQGAXaPGQTJB4KzwlGWc0+kguV+lK4h0QIvHvuorghYC4EJerNdRUviRxZB5mTDyJc1gGq6YgMr4d/a4tyXwoEzPbPGc/3YJATZPYLXOMaGDo0rd2961CFfrsneElNIZ71DWoy41P1fJG7qrfN0gLjU0aSC1vVnzDM8GQvoJGV35cONT7N7u+hjjFBFciLBNEE1DKge156EnP6VbR2LSptWrHeq3zOe9fN3FFR1WWor+lOl+SFztt7uVHCLuWxYPV+csOeBLQi6OhFnh9r4mLTc43wjPkbuci2jqEVM6Bf5gXQoEGzybhDb/3HfQK00NXovru5uEdREYDaj5NVyzyBhOT29JuOvqX7wEMNDqE//Me6Tx8bU12cuUM7Lne7grgYQMbYfLs3gKzRbeuYCqGCIz6KQMDOZvTR38EXqJxysrPb/HuiPfoSGHoAIHviJeIsy6bF7hKA8BikrMmoNc4762pKNKNzBGR+/HeQ1trDBCbYrTmlqoXPJaeANUZlD9NdwiopTJCBzjP/F61s5bpurAiJF6Ymx5yUHljZGVPFOH/ZawfhEcfYGvMUnAOup+EJCih5WQsrn2vodbVYFJ5GNMrDH9//uMi/cwqFWlBqnKGgnGdyqzXlLYTTMlv7fuh3XxkjRRTh6ZVuLyBqEBnXzSWwlcLOUdw3r/48JJ0JC2/rTLVnb+R7T2/+7uXVQISIrBx1ijtRCzFwaFWAPqsL+nMevAHnjpWl9NvanNWKIPIQ6cHGywxdi/7ynEnCYlY13fyIpagFKrgywsHcHuez8cyNtGLpg6hwbSEeN2wZYld+DQc5UH9pqPvvTuNHu3J62WJHZlkc6yBs/hiZBT/sEZUrL2Bf3SNFjzt/IcNSjlVbpP6dd6HpIH6tuN+lQJNypRe//TgKviHBkN0mzmw7IuRIDz1tcVA28COCo8NkzyD194zxDJ8yYKfKH/YwN5q8/R/r/EuUcY4qbeAokaS7XcIkOp/QFKHVM6AYKZ3SPyB5ZjkHFoHBG7YNDrx8AHDNemG+WUev/flmsEv3ykgTztgOoCmTqH/rduS+BGcKwsNW0iMwni9mUiJfuNdYLqhWohGLWb/mkUVoTgycXtsJ8x5BGkk4wSuGSwGCur8yVel7+Fr6CzsVGiPJOa0RHO6sN+Y9jnSMPIVm+mx16j3Q==,iv:ZGV3C0nvqdEnukiPkeMxDD66OjeXQF4anQLkALmBno8=,tag:ELar6NeP5bjL5L/Z5m7Piw==,type:str]
|
borg-ssh-key: ENC[AES256_GCM,data:7F7uUlTP3ZKkpySj6/AGfH3K1/8/GzIdfp+ch1hU55zX51KgRs/KuGmj+yKyH9ua41oR4FR94MoiTb3u3MRpUj6dqO4VjVm6fRgJFNXOBhTUcelRR98Nq2QClkRqcNmPiHQC4bxjyW9C1tZfCA52AIILGh9O1Q9XnYAz2q8JwbrY+eTXS/U/1Fh1D+0Y9q1n+oingehal2huHzeVsLxFlip3TmGJx8QBlnAbANABKrMFxkHAlkvAVCrF1MULzeDeeWscHi5T80OvJfOZBSonNEZosw6QJVscMYxh047Yyl6YJ/sxIJjZjC7GiLuG/FC0FCLKRhD8PkZFjrDt/6xwrqePxIb7yIKZRpsvNVCplDDdVOB/V0AqRWAeZh3z813Zi+tnjynHPYaphso/qY2r/HZKlGInzW+QCUBdPVifhVL+y2TKytCWcp4A+c/3Dc2Ut59sfO+hqE946nYGl2S+kpNASHhBa7o6cnaMtfz8NT6rVtYN/3l1snlxeOUZo85XAuYCIerGsMkdVENg5RIJwIzwM0oaGwNzruq8cul5MfAf8hFMXpoLggYECynHk9TNdhsNtUzmsS9cXAyPnnzZT6HGI2/5cgU1weCHbbXAq+ZU9WZwOT41Fpwda/WrNJuMJYFCOFer2hcSLEyxsCfqmPSdwpYRRSYHiaCuVghV6lWyi1IVV5EsX9H6hD+Uetux1SEN95ga8edME16W2dubA0yukcOq+XHI7PMEHAs20H+dybsmVx1XjIsiV/XBWkrooFBXQp950p93XOK49vwNmHNohhCvmERkH0dczGQ5vTAMXwIYqydTqXWERDRzJVvK+pbzKiecEzhFHFge345poUKQS7BWNsv+eehgBAH4HEddtzzzjcPGYMHAhafAm1VqO2BvKg2r3XnzElUCjxWlGfrMkU+LhIgAjom8Mk9+Kha+OD84+iZnuoq73TjpqaUgC0TbkxLAm0DPgk8LsLE9XfJaroftKnrE7P7kyNGSQZfGYcl7ssLck5LJA07dTvDtytlL1Kk8RJe4FwFNPLtdiAnCVvKBe0kcClvCWrrxRt1QT85b+9IwG1yhRxlxheleePX1Fwlg/d2xj6AB3Cz7kAVO2phGPAF16Ke/UQeMxhbOIcy20g7RoiCpfx9jvdwExhDmbmUR7cleL1seFH+wgAM6uQwh80+q8IDJWPNyeXOwiA2siPwexb20xN+n1kIMa03U5Bn+lBqYoh0xXhlDHP3FiKew6xjSducNUtzyKm24kQaWSOfYIqTxYFRTI0bDgRu6xWGmZL37VSMWqDj3TIWOMBoLy0JBsRl6Jj07keJ0CwaNEa4cCWZ0s6nu49mp04JfJWF2r9ksX+2OVeDjSgX5JBB/V79j3I8iu4Nlp0pjyBKPlST/FUDAl1jT12X1grkYjXO8UyJ9AQqqnqK5lM+KYVR15Ui1lBb/vISPqiiw45bcUyHfVAAt7hcFpYLaYB0om3mernccN2fi26lawhhambRd7FGkILKA3byE9ytqGvDQ2CxtjA30kbGxeqXVFsyzROahR0c3KdKlnuCO9Uar4J5VEXgv2obNlNUfSMa9uleWDuhveBaE+2jtKUJd2P4wIIzF/VJGxgWGSge/ji0EV36EFfMg/Tyizdw5wtv4rQF0M+Uu6j/n/l62SmHnT/30H8IFCFuXWmtEo1xcssMymY4ricU3kJIgjGO9h+DrBP1GBczj7yVjLHTpEhRF0yP790xgsJrP4IQB2lOtGf5MCXLrBDYkOZ5xM3+Rq5ZNH0SAHRU/qtFUmLtfkcidjkPiwdqJ0e1LUtLwmSlsot1CkPs7hKORassyUrug7dCtv9QjRMDbtW61PlIbXqa59Aimql4IUcWyUybx12E8wRqmgcX5kG2wGfd9ZFUj6mXhQINFqChsTjXSavQw3u3m8kvd1mJXfjKS12ajr2X1e5wPDUEpLzc3wTOvVgZizGeykKTcKG+GXSjPXLR61ueAO26rboYiDAeSd73shFX/vvut/aB47kZobMOooljGVtnfZjVGY8dWzdNeGvBeLws7vnFrH1u/WXPxpktGz2/eJ0L8ANJd+BZ2+wC+R3OnHeDiXHfofm3dZJTncgxYPqboKKCXRzMviSCWB3poQTm5vEltsQOR6Lj17UmHu5MX7os7t7TrfW/op4Qko9ViWT5vtrUrCZzVqJS+d8hq4lm6ANMhi3Ql+nIMIxK+hjGZNBuKZEyKY+r+Lhz9E/xdOBGVB8QH4g32DWsYvaHfpcvJC8CkGO1nRIGFyrMc10lrv++XQYtiZ1Z4a0oOtQGAXaPGQTJB4KzwlGWc0+kguV+lK4h0QIvHvuorghYC4EJerNdRUviRxZB5mTDyJc1gGq6YgMr4d/a4tyXwoEzPbPGc/3YJATZPYLXOMaGDo0rd2961CFfrsneElNIZ71DWoy41P1fJG7qrfN0gLjU0aSC1vVnzDM8GQvoJGV35cONT7N7u+hjjFBFciLBNEE1DKge156EnP6VbR2LSptWrHeq3zOe9fN3FFR1WWor+lOl+SFztt7uVHCLuWxYPV+csOeBLQi6OhFnh9r4mLTc43wjPkbuci2jqEVM6Bf5gXQoEGzybhDb/3HfQK00NXovru5uEdREYDaj5NVyzyBhOT29JuOvqX7wEMNDqE//Me6Tx8bU12cuUM7Lne7grgYQMbYfLs3gKzRbeuYCqGCIz6KQMDOZvTR38EXqJxysrPb/HuiPfoSGHoAIHviJeIsy6bF7hKA8BikrMmoNc4762pKNKNzBGR+/HeQ1trDBCbYrTmlqoXPJaeANUZlD9NdwiopTJCBzjP/F61s5bpurAiJF6Ymx5yUHljZGVPFOH/ZawfhEcfYGvMUnAOup+EJCih5WQsrn2vodbVYFJ5GNMrDH9//uMi/cwqFWlBqnKGgnGdyqzXlLYTTMlv7fuh3XxkjRRTh6ZVuLyBqEBnXzSWwlcLOUdw3r/48JJ0JC2/rTLVnb+R7T2/+7uXVQISIrBx1ijtRCzFwaFWAPqsL+nMevAHnjpWl9NvanNWKIPIQ6cHGywxdi/7ynEnCYlY13fyIpagFKrgywsHcHuez8cyNtGLpg6hwbSEeN2wZYld+DQc5UH9pqPvvTuNHu3J62WJHZlkc6yBs/hiZBT/sEZUrL2Bf3SNFjzt/IcNSjlVbpP6dd6HpIH6tuN+lQJNypRe//TgKviHBkN0mzmw7IuRIDz1tcVA28COCo8NkzyD194zxDJ8yYKfKH/YwN5q8/R/r/EuUcY4qbeAokaS7XcIkOp/QFKHVM6AYKZ3SPyB5ZjkHFoHBG7YNDrx8AHDNemG+WUev/flmsEv3ykgTztgOoCmTqH/rduS+BGcKwsNW0iMwni9mUiJfuNdYLqhWohGLWb/mkUVoTgycXtsJ8x5BGkk4wSuGSwGCur8yVel7+Fr6CzsVGiPJOa0RHO6sN+Y9jnSMPIVm+mx16j3Q==,iv:ZGV3C0nvqdEnukiPkeMxDD66OjeXQF4anQLkALmBno8=,tag:ELar6NeP5bjL5L/Z5m7Piw==,type:str]
|
||||||
grafana-ldap-password: ENC[AES256_GCM,data:hNB6CRtXW98yqUqInD3LsZ75sA+lVfmbooehni0UKL60qE/XCZm5B9JVO9pjxbIYZN6Eu/RFX+9L9cJVa5jnEo2MVeLS4CSjqC8BHLArlOuEdA5v8vqqJofBpBfXXN5Ca5xeUDJKz2HgtoTg7G5nTkegGZPGrmj5QQiL1xzco38=,iv:ViQAPTGxEWnjLkJlGCdCq5wW+fbr/O9er8/71VjL/GE=,tag:+Mow4cw7tvtkXvV2iSHeQw==,type:str]
|
grafana-ldap-password: ENC[AES256_GCM,data:hNB6CRtXW98yqUqInD3LsZ75sA+lVfmbooehni0UKL60qE/XCZm5B9JVO9pjxbIYZN6Eu/RFX+9L9cJVa5jnEo2MVeLS4CSjqC8BHLArlOuEdA5v8vqqJofBpBfXXN5Ca5xeUDJKz2HgtoTg7G5nTkegGZPGrmj5QQiL1xzco38=,iv:ViQAPTGxEWnjLkJlGCdCq5wW+fbr/O9er8/71VjL/GE=,tag:+Mow4cw7tvtkXvV2iSHeQw==,type:str]
|
||||||
grafana-admin-password: ENC[AES256_GCM,data:365efRy8xD7SHBnVz6ZJO3l8/lfiZ5vZPZZbxnUmjKKJTMeebLY+P54moStY0wsbU9vk7sCKATCxrS5xy+FQJSgKLoajfz50OMA4+1k3Shl+skbeIikHKwFxqrljFa6HRQ2HTW6KLDPu6Z5Agkima5xdfrtc5R1SnOFg5b6D5NU=,iv:0yZGZVQd35Itj66Ff5hDfDYYx5xsNs/wc887bgMV1MY=,tag:9t8Iffg7kxSjE5eo7iv/RQ==,type:str]
|
grafana-admin-password: ENC[AES256_GCM,data:365efRy8xD7SHBnVz6ZJO3l8/lfiZ5vZPZZbxnUmjKKJTMeebLY+P54moStY0wsbU9vk7sCKATCxrS5xy+FQJSgKLoajfz50OMA4+1k3Shl+skbeIikHKwFxqrljFa6HRQ2HTW6KLDPu6Z5Agkima5xdfrtc5R1SnOFg5b6D5NU=,iv:0yZGZVQd35Itj66Ff5hDfDYYx5xsNs/wc887bgMV1MY=,tag:9t8Iffg7kxSjE5eo7iv/RQ==,type:str]
|
||||||
|
promtail-nginx-password: ENC[AES256_GCM,data:zk/Wq+Nss6Md0GdhoOcysPrDBqfoAobmqb4LMDkJBjpCn/mdP3/HPiIYdZnZ0vV0JmYpQVqgVFPMlA==,iv:TA19kKllw0Vco6RRlbW4eUqeGQ0SQJRr/TATmyZBMrs=,tag:10/87/svXdL1hpUcTOtY0w==,type:str]
|
||||||
sops:
|
sops:
|
||||||
kms: []
|
kms: []
|
||||||
gcp_kms: []
|
gcp_kms: []
|
||||||
@@ -26,8 +27,8 @@ sops:
|
|||||||
elpwY3Q3dnRzR0loN1BiVk44TTF2VDQKs8Si2LHZ4L4oQqkYUhCI6affE0aTrWmE
|
elpwY3Q3dnRzR0loN1BiVk44TTF2VDQKs8Si2LHZ4L4oQqkYUhCI6affE0aTrWmE
|
||||||
L+am++gYdygVURIh0Z6ftUuhYHPwhlCgmKxx51mKRV2ydraOdUUw0g==
|
L+am++gYdygVURIh0Z6ftUuhYHPwhlCgmKxx51mKRV2ydraOdUUw0g==
|
||||||
-----END AGE ENCRYPTED FILE-----
|
-----END AGE ENCRYPTED FILE-----
|
||||||
lastmodified: "2023-08-18T23:19:22Z"
|
lastmodified: "2023-08-19T00:09:27Z"
|
||||||
mac: ENC[AES256_GCM,data:sWtJUW19HleKalg/Mfysk/b0N6YxdFcC/66BLmbcchI6s5MeGMLdYIJkNm7RKRQM5PY25d3saOqvsm5qK+keOBa0H9v0DwmFuS9cBJGa5KV6/IDoMvO8VtgDzCZ9HLtrSVTuh84bv7XL3cRd99BfSlSyHBJRpV7kJTudid2O9vo=,iv:8sOMUnsm8hyJlLvc5zG72wjKXtcbK7qnEd7Og0+yJt4=,tag:4XirU7fx0UmJSNkKgmJp8g==,type:str]
|
mac: ENC[AES256_GCM,data:4FjX8XngdwYBbifM4xmdW/7a3tf43/AdD6ujpYa9M7c7EJ+4ipf6S/eu1CuVk4XAr84rkCAfF+PpGXWeZCJ47YhbXI3yg6HRjGt//5X4Jn6tUYre8vk5Fy7C3dwDKgqHLqOm0hFE89m82xfkfe6VuDeCSbLFUucEtQ3d+rKcGvY=,iv:ufx9eQNNOXcRQISLvdfLK2RUinQPTgjiYpGUWYiqDZc=,tag:A2MoB+/NUFiEee4nTNpAXg==,type:str]
|
||||||
pgp: []
|
pgp: []
|
||||||
unencrypted_suffix: _unencrypted
|
unencrypted_suffix: _unencrypted
|
||||||
version: 3.7.3
|
version: 3.7.3
|
||||||
|
|||||||
98
utils/modules/promtail/default.nix
Normal file
98
utils/modules/promtail/default.nix
Normal file
@@ -0,0 +1,98 @@
|
|||||||
|
{ config, ... }: {
|
||||||
|
sops.secrets.promtail-password = {
|
||||||
|
owner = "promtail";
|
||||||
|
sopsFile = ./secrets.yaml;
|
||||||
|
};
|
||||||
|
services.promtail = {
|
||||||
|
enable = true;
|
||||||
|
configuration = {
|
||||||
|
server.http_listen_port = 9080;
|
||||||
|
server.grpc_listen_port = 0;
|
||||||
|
|
||||||
|
clients = [
|
||||||
|
{
|
||||||
|
basic_auth.username = "promtail@cloonar.com";
|
||||||
|
basic_auth.password_file = config.sops.secrets.promtail-password.path;
|
||||||
|
url = "https://loki.cloonar.com/loki/api/v1/push";
|
||||||
|
}
|
||||||
|
];
|
||||||
|
|
||||||
|
scrape_configs = [
|
||||||
|
{
|
||||||
|
job_name = "journal";
|
||||||
|
journal = {
|
||||||
|
json = true;
|
||||||
|
max_age = "12h";
|
||||||
|
labels.job = "systemd-journal";
|
||||||
|
};
|
||||||
|
pipeline_stages = [
|
||||||
|
{
|
||||||
|
json.expressions = {
|
||||||
|
transport = "_TRANSPORT";
|
||||||
|
unit = "_SYSTEMD_UNIT";
|
||||||
|
msg = "MESSAGE";
|
||||||
|
coredump_cgroup = "COREDUMP_CGROUP";
|
||||||
|
coredump_exe = "COREDUMP_EXE";
|
||||||
|
coredump_cmdline = "COREDUMP_CMDLINE";
|
||||||
|
coredump_uid = "COREDUMP_UID";
|
||||||
|
coredump_gid = "COREDUMP_GID";
|
||||||
|
};
|
||||||
|
}
|
||||||
|
{
|
||||||
|
# Set the unit (defaulting to the transport like audit and kernel)
|
||||||
|
template = {
|
||||||
|
source = "unit";
|
||||||
|
template = "{{if .unit}}{{.unit}}{{else}}{{.transport}}{{end}}";
|
||||||
|
};
|
||||||
|
}
|
||||||
|
{
|
||||||
|
regex = {
|
||||||
|
expression = "(?P<coredump_unit>[^/]+)$";
|
||||||
|
source = "coredump_cgroup";
|
||||||
|
};
|
||||||
|
}
|
||||||
|
{
|
||||||
|
template = {
|
||||||
|
source = "msg";
|
||||||
|
# FIXME would be cleaner to have this in a match block, but could not get it to work
|
||||||
|
template = "{{if .coredump_exe}}{{.coredump_exe}} core dumped (user: {{.coredump_uid}}/{{.coredump_gid}}, command: {{.coredump_cmdline}}){{else}}{{.msg}}{{end}}";
|
||||||
|
};
|
||||||
|
}
|
||||||
|
{
|
||||||
|
labels.coredump_unit = "coredump_unit";
|
||||||
|
}
|
||||||
|
{
|
||||||
|
# Normalize session IDs (session-1234.scope -> session.scope) to limit number of label values
|
||||||
|
replace = {
|
||||||
|
source = "unit";
|
||||||
|
expression = "^(session-\\d+.scope)$";
|
||||||
|
replace = "session.scope";
|
||||||
|
};
|
||||||
|
}
|
||||||
|
{
|
||||||
|
labels.unit = "unit";
|
||||||
|
}
|
||||||
|
{
|
||||||
|
# Write the proper message instead of JSON
|
||||||
|
output.source = "msg";
|
||||||
|
}
|
||||||
|
# silence nscd:
|
||||||
|
# Oct 24 18:20:19 nardole nscd[1812]: 1812 ignored inotify event for `/etc/netgroup` (file exists)
|
||||||
|
{ drop.expression = "ignored inotify event for"; }
|
||||||
|
# messages from rpi3
|
||||||
|
{ drop.expression = "hwmon hwmon1: Undervoltage detected!"; }
|
||||||
|
{ drop.expression = "hwmon hwmon1: Voltage normalised"; }
|
||||||
|
# ignore random portscans on the internet
|
||||||
|
{ drop.expression = "refused connection: IN="; }
|
||||||
|
];
|
||||||
|
relabel_configs = [
|
||||||
|
{
|
||||||
|
source_labels = [ "__journal__hostname" ];
|
||||||
|
target_label = "host";
|
||||||
|
}
|
||||||
|
];
|
||||||
|
}
|
||||||
|
];
|
||||||
|
};
|
||||||
|
};
|
||||||
|
}
|
||||||
84
utils/modules/promtail/secrets.yaml
Normal file
84
utils/modules/promtail/secrets.yaml
Normal file
@@ -0,0 +1,84 @@
|
|||||||
|
promtail-password: ENC[AES256_GCM,data:+KjooDZPlJ7UiSPusdzeTP+6DMfmdSM+T5ID8FOWA3u1/PKqPpgrpnKUzdwKJT0KcmaAVQI723Qjsk0q6UTYYfhkLosyAXnAz3/31EylCaJy4M0CG222xnJDjQ4VA4D/Fp/t+zWWShnf1vUL5wXnWw3rfZ5PEZ34U1M9DtP340Y=,iv:OawqwactX5qncggFvZCWna17SEN+pbF2HRSxnXqa9Xo=,tag:wT2BR5zruIOMKgtmWfRBFg==,type:str]
|
||||||
|
sops:
|
||||||
|
kms: []
|
||||||
|
gcp_kms: []
|
||||||
|
azure_kv: []
|
||||||
|
hc_vault: []
|
||||||
|
age:
|
||||||
|
- recipient: age16veg3fmvpfm7a89a9fc8dvvsxmsthlm70nfxqspr6t8vnf9wkcwsvdq38d
|
||||||
|
enc: |
|
||||||
|
-----BEGIN AGE ENCRYPTED FILE-----
|
||||||
|
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAzdXQ0cmJTUUplaFBYMENJ
|
||||||
|
eWM3RklTKzNiVVJiU2c0VHNPdWpNZ1VNV2pVClZxTGN1RG5hT1JqZ2ovajBiVFlB
|
||||||
|
c28xWnNjRG56MUJnaVJxNndheUppTFEKLS0tIHhVQUVacnl1bHdhTGZkQlVJQVlo
|
||||||
|
TEV0MEdnTG9Tek5rOXdEODUwWmdoMW8K2VRvIBDY3o2SD/ToKk0Zvo3uLVF41Vs0
|
||||||
|
AqJND7wbavF9ZFu6XrVfcrxucrvqLsLCmcFzO2fCauSWgd6lcjlivg==
|
||||||
|
-----END AGE ENCRYPTED FILE-----
|
||||||
|
- recipient: age106n5n3rrrss45eqqzz8pq90la3kqdtnw63uw0sfa2mahk5xpe30sxs5x58
|
||||||
|
enc: |
|
||||||
|
-----BEGIN AGE ENCRYPTED FILE-----
|
||||||
|
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB6d0pDN1J4VzN4WXhMbzRF
|
||||||
|
ODdDWDlkYWdZRHJkR0JXazlKTG11KzNrTW5JClJ5ZmUyNEFkRFNVaTBEdmNzWlha
|
||||||
|
SXpvdWVmcDB5RWlDWWtWejJ4M0pGYVEKLS0tIEdrNVlZZmxyd3Y2aHVLV2lMeFp3
|
||||||
|
NmRjaWRZQTF3MUpHRWJTZ2J6ajhSYWMKlAnM5DgaFC75JmLa18WXO7DJsfLzXbAq
|
||||||
|
jr+FCIFTEUbCAfNeEcqz9hI5MOD7kVGuCJ5ZpgVjwaTRhfgloJI+IA==
|
||||||
|
-----END AGE ENCRYPTED FILE-----
|
||||||
|
- recipient: age1y6lvl5jkwc47p5ae9yz9j9kuwhy7rtttua5xhygrgmr7ehd49svsszyt42
|
||||||
|
enc: |
|
||||||
|
-----BEGIN AGE ENCRYPTED FILE-----
|
||||||
|
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB2b1NSNzVOdkt6ZXgzb084
|
||||||
|
Z2tXaEpIcXp0dDhjcURQcVQ3dnZwQkpwd1ZrCjJZTkVkOEdwaFpheG9OV2t2SWwz
|
||||||
|
ZFhVN042RDVuN29kL1ZFTGtzL2dPeUUKLS0tIDVwUFEyZ0RHUlM4UW9WVkdzbVBz
|
||||||
|
TnB4YXV2anlpaXpjblQzUHlINHljRlkKhAUhR2YSDd28L3MjtuUTSrHIRcHIzgHx
|
||||||
|
jhb+Ynji43X5CUNeQQJNGPuDRa3sRC+4SWHfVS16GbJTcxmFsSmS4g==
|
||||||
|
-----END AGE ENCRYPTED FILE-----
|
||||||
|
- recipient: age1ezq2j34qngky22enhnslx6hzh4ekwk8dtmn6c9us0uqxqpn7hgpsspjz58
|
||||||
|
enc: |
|
||||||
|
-----BEGIN AGE ENCRYPTED FILE-----
|
||||||
|
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBHV1hlbTUyYW9vSnBqVVE1
|
||||||
|
VlVmM1JiRTl3NmFBTFJIU2tEUUhmQnkrbm5rCm85R1M3ODhObmtlVThkTDh0ZHdp
|
||||||
|
cjBwek85MnIxOGdqTTJRUGRKRm1TUncKLS0tIFl6bm1CRTg5ak8xc3NOd0xkdnNE
|
||||||
|
dElTc1IzbjVtNmFOU3hRR1hyTkk1V2cKYQvD9SzJhZEc51YiOqc5cSHa8XREVU2y
|
||||||
|
KbItb9QohlTEohmOoGZoP3LQNm4ZbvQ6jyBHLccHpckV84jxv0ljJg==
|
||||||
|
-----END AGE ENCRYPTED FILE-----
|
||||||
|
- recipient: age1jyeppc8yl2twnv8fwcewutd5gjewnxl59lmhev6ygds9qel8zf8syt7zz4
|
||||||
|
enc: |
|
||||||
|
-----BEGIN AGE ENCRYPTED FILE-----
|
||||||
|
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBoaDV4WUsvdjZPQi9tL3FR
|
||||||
|
N24zSXUxMVJpWkpGRi80VGNSNkR2U1Myc2tZCjNIOVdoOFFjNEFnMUFYZEp4enJI
|
||||||
|
R1o1K2F2L1I0TlJ2amkrbG14RjRRQ2cKLS0tIDZkZ2lhVlZ0L3NkMXdNZmJDdUR6
|
||||||
|
TkZNNzZMMm9OendQY2txbGczam94T2sKBpHDDHP/NRT4yOAD0JDA3TIFUXQWvdXt
|
||||||
|
kYIpOGfq3hwxcGUGMPTz6K/MlzpZGwiXvmVqhTLOj9X2jt+DcWfphw==
|
||||||
|
-----END AGE ENCRYPTED FILE-----
|
||||||
|
- recipient: age1azmxsw5llmp2nnsv3yc2l8paelmq9rfepxd8jvmswgsmax0qyyxqdnsc7t
|
||||||
|
enc: |
|
||||||
|
-----BEGIN AGE ENCRYPTED FILE-----
|
||||||
|
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBMdEYzTnR1SWpPZjVYZXR2
|
||||||
|
amgzMDZBeXRMdnBwS3A2VitvYlN5cUIyaEdFCk9yVStMVklXVGVPdWs3b3Vjb1Vz
|
||||||
|
bnhmYW0wZU1vOEVtU3NBdFVrSmNjeTgKLS0tIFpIZjJBbHEwaVN6Y1A4NEJPSHNw
|
||||||
|
Z1NxSU9OYzVFbGtwTVJEaGpQOC80TVEKPwNbehHyR9kua8a3xj2bo8CysrPPvj+b
|
||||||
|
0G5HEw5Ol3BKIqg+eNbuNQY4/mC3VZMDbFFQ/GHbWYFXeAwJre5/Fw==
|
||||||
|
-----END AGE ENCRYPTED FILE-----
|
||||||
|
- recipient: age1zkzpnfeakyvg3fqtyay32sushjx2hqe28y6hs6ss7plemzqjqa5s6s5yu3
|
||||||
|
enc: |
|
||||||
|
-----BEGIN AGE ENCRYPTED FILE-----
|
||||||
|
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAxRk50cnFXVUpRY3JxTjQ5
|
||||||
|
WjFBZlQ0M01RN2I4TUtkeFJVYVlLVFhIYURFCjdWdlkzNXBXUkRPbDY4alh5TC9p
|
||||||
|
VnFKNWZydjlOQWhjTjlYeW9PeWtwY1UKLS0tIEk3eVZzQ2tYTE4rOGhxcWYxQVZy
|
||||||
|
UFJoYklHQ3E3cXJHQU1MN2ZwM2R5RTAK3+F3OcR7beKeF91YCQLwLz5QLZ78xVQd
|
||||||
|
sEKPbGyRiMXrT9wMZz8WjE253P/iaCmaf6wABK2PW64wI36/kR+62Q==
|
||||||
|
-----END AGE ENCRYPTED FILE-----
|
||||||
|
- recipient: age14uarclad0ty5supc8ep09793xrnwkv8a4h9j0fq8d8lc92n2dadqkf64vw
|
||||||
|
enc: |
|
||||||
|
-----BEGIN AGE ENCRYPTED FILE-----
|
||||||
|
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB0U3hzNDNtZ2ZDMnptM2pY
|
||||||
|
NVhmVjVqTzN1aGxaSUdHQWliMUxsYnNKamtjCkg4a01pd0RDeEhQZytLNzdKb1dE
|
||||||
|
ck4wV2p2VFc3YjJHRFIwbnF0Wms0R0UKLS0tIGZOY3BMekZSVWlxUmIzbENNejc1
|
||||||
|
WDc1QkkwRnJhMTNkVjFpaGJNaHJHVzQKtlXo8afV++E5uwGSPY7RCj8TY0XbQqRv
|
||||||
|
+DygK0wq4hj0IZsR49W42ORSWxAxkFiLMDAr8o0X0MZx226y90NdVw==
|
||||||
|
-----END AGE ENCRYPTED FILE-----
|
||||||
|
lastmodified: "2023-08-19T00:05:00Z"
|
||||||
|
mac: ENC[AES256_GCM,data:BO3WZzW4MzXpOLKTi6vzVq5lFMATANvIH8Kl9HJPH4bRTRf+z/IX3GmmowjOQis8aGnbfbMja6K0hBWMSY0mY7WzMN9W2ARHTgbXre9/5l6PfFrW4q36sLwXOJU/mzLVz4errHSt6A3Te5AOqThlULuJO/F4pPX2i0Sgs2F1tVA=,iv:zOTWgbuUzuIhYbJFKocwEdR9DxZ3enjc2aIchkovfuA=,tag:x+2jEytk5XrSAGWvbB6bKw==,type:str]
|
||||||
|
pgp: []
|
||||||
|
unencrypted_suffix: _unencrypted
|
||||||
|
version: 3.7.3
|
||||||
Reference in New Issue
Block a user