diff --git a/machines/overwatch/configuration.nix b/machines/overwatch/configuration.nix index 280133b..dfe722a 100644 --- a/machines/overwatch/configuration.nix +++ b/machines/overwatch/configuration.nix @@ -7,11 +7,21 @@ let in { sops.defaultSopsFile = ./secrets.yaml; + sops.secrets = { grafana_smtp = { owner = "grafana"; group = "grafana"; }; + + # Contains private targets for blackbox monitoring + # It is used below in the blackbox-http-private0 job + # More of these can be created such as private1, private2, ... + private0_blackbox_config = { + owner = "prometheus"; + group = "prometheus"; + path = "/var/lib/prometheus2/private0.json"; + }; }; networking = { @@ -108,6 +118,10 @@ in retentionTime = "1y"; port = 9001; + # We have to disable the checks since we use file_sd_configs provided through + # sops secrets. These are not available during buildtime which leads to errors during the check + checkConfig = false; + exporters.blackbox = { enable = true; configFile = ./blackbox.yaml; @@ -133,6 +147,35 @@ in "https://zines.malobeo.org" ]; }]; + + relabel_configs = [ + { + source_labels = ["__address__"]; + target_label = "__param_target"; + } + { + source_labels = ["__param_target"]; + target_label = "instance"; + } + { + target_label = "__address__"; + replacement = "127.0.0.1:9115"; + } + ]; + } + { + job_name = "blackbox-http-private0"; + metrics_path = "/probe"; + params = { + module = ["http_2xx"]; + }; + file_sd_configs = [ + { + files = [ + config.sops.secrets.private0_blackbox_config.path + ]; + } + ]; relabel_configs = [ { source_labels = ["__address__"]; diff --git a/machines/overwatch/dummy.yaml b/machines/overwatch/dummy.yaml index 8538eaf..994f0bb 100644 --- a/machines/overwatch/dummy.yaml +++ b/machines/overwatch/dummy.yaml @@ -1,9 +1,9 @@ grafana_admin: ENC[AES256_GCM,data:zO7Tfmo=,iv:xa456cUb14VjJPEKClqGJiYR7cSsjQPIld+O2E0YNM8=,tag:dFgGtqMYhg4uweoLZPu48Q==,type:str] grafana_smtp: ENC[AES256_GCM,data:xndLMw==,iv:0P4INbDD2/Teo8Dv6lIfLhXvNAkcneEDCeB5OAY25wI=,tag:ilE0Noleb+uL5gqF4bU70w==,type:str] +private0_blackbox_config: ENC[AES256_GCM,data:0QdJTdKxMdYPoGQxyCYaXeAIwqEP4HG9gVbEiITnXqXotsdj4p7nBnCAuFasHoBOa9svR+sqibHQaP0iFUDUR7O/tjB9/7Bz4U3b7fpNA9EIN3hpDfXH/B48fI5A7FDRRnWLQN+Hcbty6pWce/l66tpthw==,iv:HNXIIu22M4NRqshbfrhQYF+lNM9hstqx4qaPQy1Nl9o=,tag:oRhTVTtvTI/9qcTQbUc2OA==,type:str] sops: age: - - recipient: age18jn5mrfs4gqrnv0e2sxsgh3kq4sgxx39hwr8z7mz9kt7wlgaasjqlr88ng - enc: | + - enc: | -----BEGIN AGE ENCRYPTED FILE----- YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBPMXlEbWlabzVUNlRkVFFC ZktpQUx2alRNN01qYmhFYnhMRDBkYmFRdzNnCi9YbjBDUlRhc1kwVjZWY0NGZERQ @@ -11,8 +11,8 @@ sops: eDFDVStZS0o0WVh4cmJPa2o3eWNseGMKc0ydK8/2Gzxe2IYG96saVosJTrRcWizR lcMbsh8mKlVCt5Fx9EQOtj50ylOGk2h2Itk4uLYG4UAySDU/Aw3iQw== -----END AGE ENCRYPTED FILE----- - - recipient: age1ljpdczmg5ctqyeezn739hv589fwhssjjnuqf7276fqun6kc62v3qmhkd0c - enc: | + recipient: age18jn5mrfs4gqrnv0e2sxsgh3kq4sgxx39hwr8z7mz9kt7wlgaasjqlr88ng + - enc: | -----BEGIN AGE ENCRYPTED FILE----- YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB1QmsxNERCclRBeFo0WGZ1 OGlHTGxDMURsWXdDRXpoUk10S0JpeGk2QURvCjBkaVFHTHo1eGtKbERPWmhNZEpL @@ -20,8 +20,9 @@ sops: QXZwTVd5NnNSdUxRSzNLMHk2SWYxVnMKZyZvZV5qhJj4KATFV2eh9aN9XMbSQL7i kRGQI9y5ADv6UsLh/y+rWe0xF4BPOGF0xL8JUnHfTy/RsKfhB3aTYQ== -----END AGE ENCRYPTED FILE----- - lastmodified: "2026-01-24T17:01:57Z" - mac: ENC[AES256_GCM,data:KySo9sJgOLBPtxQZ1gXY8aCmCHbn44aiLKiK1mGziKsBoPkBdEivCT7dDQ1tXQj9wz3gly6raS4FEmR6ikcbtAE2EN86cDLNuu9sSiW2OYiJRe5iZvet2veqtA00K7TtPZgAX+ccsUE5iy0W55bu+kpK3J8MPNy3LUK0EbSKW8I=,iv:7BQKCNyWe4azTelyKlV6XvvgPIrspVbwITyycsW/c44=,tag:1XbSxju44GCKcGw8Ln9iuQ==,type:str] + recipient: age1ljpdczmg5ctqyeezn739hv589fwhssjjnuqf7276fqun6kc62v3qmhkd0c + lastmodified: "2026-09-21T11:06:39Z" + mac: ENC[AES256_GCM,data:pnoz3zIl8cOhq4jgCrWU+j9KkhPao8F4XGCjAblMkayPkF1x9tCC7lf+fS0BeEIADhl7ttUQimjTN62dk77iu0nJzMFfckPt4RRLrYI07yc61cBUPGz0NEDFUE5DxnoxUrpzBGtq4Y55wuD88hBjLZowmk7Eun9v7VAU9tyXrOM=,iv:Iz/fz0k0mhmuFGxsRChRPOi46ywIGTqKJ0AuKcdkCc8=,tag:NBjCzszMoSBrn+o5BHK1tg==,type:str] pgp: - created_at: "2026-01-24T17:01:26Z" enc: |- @@ -62,4 +63,4 @@ sops: -----END PGP MESSAGE----- fp: aef8d6c7e4761fc297cda833df13aebb1011b5d4 unencrypted_suffix: _unencrypted - version: 3.11.0 + version: 3.13.3 diff --git a/machines/overwatch/secrets.yaml b/machines/overwatch/secrets.yaml index 86af99f..3dc8f2e 100644 --- a/machines/overwatch/secrets.yaml +++ b/machines/overwatch/secrets.yaml @@ -1,9 +1,9 @@ grafana_admin: ENC[AES256_GCM,data:c+ZnOyxSXrG4eiK8ETKHheadiSz98LLHYwxb,iv:Ut2qFD2p6OmKDWjLMjFxyISxzTdJpZpgIB7obW5bgkY=,tag:HdayzjXQ1Zc7w9ITLzKLxA==,type:str] grafana_smtp: ENC[AES256_GCM,data:gOER9SzqRACIWe3PchyKguX3RdW6xhSZDzLd727CK1w=,iv:KDOyXGYGXnUu92hvt6eBqI8zeKP+JRDsF8Ir5X/9TDk=,tag:wRexcIJp8XEURKcZnHCRtQ==,type:str] +private0_blackbox_config: ENC[AES256_GCM,data:KzRe0DxBGK0khU9wJ6znLELS6BsoNrdUDu8/n6RnGgAm/r0CQT66AxZZJiImlDtMgHLqB+x/w1hTET0ThzDURr5lL1ckSP7AD2w3vbn/4TmV+/4IeLHeOtiDHmbwoPcOWwSpIelfq1MbhnDjgRITix2IWgOncoenVgo6txIpePxsMhJCM0SZb4Tujy7P79vm,iv:QKK5Q89Glp5nbNOmCmQ2xYVs82X3ErTPjdFCFmEoYXE=,tag:UZW0jMOuEZwS3VXFlJhhoQ==,type:str] sops: age: - - recipient: age1ljpdczmg5ctqyeezn739hv589fwhssjjnuqf7276fqun6kc62v3qmhkd0c - enc: | + - enc: | -----BEGIN AGE ENCRYPTED FILE----- YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBWdVhpdTdJNnh5VCtmNHh5 M1ZTKzk1WlVYYmRWKzJxd1ZUTi9IMW1ta1VvCjVZQVQ1K1lFeC9QZkI2MGZXNk81 @@ -11,8 +11,8 @@ sops: NmRNSit1TkRRbUFQNzUvYThUczAxRzAKvpDK4R7m/GpfOzM4nU5vSYDXZGUy4D+c xouTxOguMdId8GiKb1AJnVv2q/YYdr2M9yA/FpKn7kBfuQw/Hrj1rg== -----END AGE ENCRYPTED FILE----- - - recipient: age1hq75x3dpnfqat9sgtfjf8lep49qvkdgza3xwp7ugft3kd74pdfnqfsmmdn - enc: | + recipient: age1ljpdczmg5ctqyeezn739hv589fwhssjjnuqf7276fqun6kc62v3qmhkd0c + - enc: | -----BEGIN AGE ENCRYPTED FILE----- YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA0bEhCQitwNVpYSHRJOEMr ZnZLQ3lELzdtREVpSHVGd3BHZ3p6WU9BK3lFCkFlYXNsa2JWZmZFVjRmK3BTejc1 @@ -20,8 +20,9 @@ sops: VnRIYVVIZktISENseEVLVWlsenFhcVkKvj8i349iNX2YNL5G0w7F9IuW/5ZRP6Z7 XC8TrYzCSOEOXt5nMnZg2yQUJHi6Qq8Wu2e2KV7Cd9J8Q/39dRv2cg== -----END AGE ENCRYPTED FILE----- - lastmodified: "2026-01-24T16:52:54Z" - mac: ENC[AES256_GCM,data:BoJwPldIVZf2NZALbeMTq6DvuBMwOwyvmr5UjsPvQrdbbzYjLTl5dplkhT6QRHPDCFDT4Ibd1UfJ8pQYRkGdO2EjlUyuk3hvAYdR+PUmVz8afFE07sNsGw2EDGVHmsFjUVHLT69YqXtGpVNTW6KjqYuRhz/Ik0wEZLxzDSiy86c=,iv:TQU0si/AYaMpShF4l3CzUN3qajQpE0ZWa7mLOqhRP00=,tag:Oyapkoo4+Q7x08m6RTXfcQ==,type:str] + recipient: age1hq75x3dpnfqat9sgtfjf8lep49qvkdgza3xwp7ugft3kd74pdfnqfsmmdn + lastmodified: "2026-09-21T11:30:50Z" + mac: ENC[AES256_GCM,data:/LmnSheGjhxznLQa1eL67qrlOdujY9fluugjsnCOZov0vJLDRgh5mItxkyH4h7TmMmisU7Pty9s/kRzjLoGmLH0WDmFgg5SnB9Bw6Bt5mn7N4xTiGlfy6RsvJhIZefExeg5kTBo3zQ84pQellGZy/1a0/Vhc6j8hLLPPoO/eL7g=,iv:d/MR5Z7ngB75gMzQiCMAEnxbee8Wcwemd9uKESKvt6E=,tag:QAGB3kiKlxV395o1ZA6ePQ==,type:str] pgp: - created_at: "2026-01-24T18:05:47Z" enc: |- @@ -62,4 +63,4 @@ sops: -----END PGP MESSAGE----- fp: aef8d6c7e4761fc297cda833df13aebb1011b5d4 unencrypted_suffix: _unencrypted - version: 3.11.0 + version: 3.13.3