[overwatch] add new blackbox target private0

This allows configuring blackbox monitoring of domains provided through
sops secrets. If new targets should be added we can add private1,
private2, ...

machines/overwatch/configuration.nix

@@ -7,11 +7,21 @@ let
 in
 {
   sops.defaultSopsFile = ./secrets.yaml;
+
   sops.secrets = {
     grafana_smtp = {
       owner = "grafana";
       group = "grafana";
     };
+
+    # Contains private targets for blackbox monitoring
+    # It is used below in the blackbox-http-private0 job
+    # More of these can be created such as private1, private2, ...
+    private0_blackbox_config = {
+      owner = "prometheus";
+      group = "prometheus";
+      path = "/var/lib/prometheus2/private0.json";
+    };
   };
 
   networking = {
@@ -108,6 +118,10 @@ in
     retentionTime = "1y";
     port = 9001;
 
+    # We have to disable the checks since we use file_sd_configs provided through
+    # sops secrets. These are not available during buildtime which leads to errors during the check
+    checkConfig = false;
+
     exporters.blackbox = {
       enable = true;
       configFile = ./blackbox.yaml;
@@ -133,6 +147,35 @@ in
             "https://zines.malobeo.org"
            ];
         }];
+
+        relabel_configs = [
+          {
+            source_labels = ["__address__"];
+            target_label = "__param_target";
+          }
+          {
+            source_labels = ["__param_target"];
+            target_label = "instance";
+          }
+          {
+            target_label = "__address__";
+            replacement = "127.0.0.1:9115";
+          }
+        ];
+      }
+      {
+        job_name = "blackbox-http-private0";
+        metrics_path = "/probe";
+        params = {
+          module = ["http_2xx"];
+        };
+        file_sd_configs = [
+          {
+            files = [ 
+              config.sops.secrets.private0_blackbox_config.path
+            ];
+          }
+        ];
         relabel_configs = [
           {
             source_labels = ["__address__"];

machines/overwatch/dummy.yaml

@@ -1,9 +1,9 @@
 grafana_admin: ENC[AES256_GCM,data:zO7Tfmo=,iv:xa456cUb14VjJPEKClqGJiYR7cSsjQPIld+O2E0YNM8=,tag:dFgGtqMYhg4uweoLZPu48Q==,type:str]
 grafana_smtp: ENC[AES256_GCM,data:xndLMw==,iv:0P4INbDD2/Teo8Dv6lIfLhXvNAkcneEDCeB5OAY25wI=,tag:ilE0Noleb+uL5gqF4bU70w==,type:str]
+private0_blackbox_config: ENC[AES256_GCM,data:0QdJTdKxMdYPoGQxyCYaXeAIwqEP4HG9gVbEiITnXqXotsdj4p7nBnCAuFasHoBOa9svR+sqibHQaP0iFUDUR7O/tjB9/7Bz4U3b7fpNA9EIN3hpDfXH/B48fI5A7FDRRnWLQN+Hcbty6pWce/l66tpthw==,iv:HNXIIu22M4NRqshbfrhQYF+lNM9hstqx4qaPQy1Nl9o=,tag:oRhTVTtvTI/9qcTQbUc2OA==,type:str]
 sops:
     age:
-        - recipient: age18jn5mrfs4gqrnv0e2sxsgh3kq4sgxx39hwr8z7mz9kt7wlgaasjqlr88ng
-          enc: |
+        - enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
             YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBPMXlEbWlabzVUNlRkVFFC
             ZktpQUx2alRNN01qYmhFYnhMRDBkYmFRdzNnCi9YbjBDUlRhc1kwVjZWY0NGZERQ
@@ -11,8 +11,8 @@ sops:
             eDFDVStZS0o0WVh4cmJPa2o3eWNseGMKc0ydK8/2Gzxe2IYG96saVosJTrRcWizR
             lcMbsh8mKlVCt5Fx9EQOtj50ylOGk2h2Itk4uLYG4UAySDU/Aw3iQw==
             -----END AGE ENCRYPTED FILE-----
-        - recipient: age1ljpdczmg5ctqyeezn739hv589fwhssjjnuqf7276fqun6kc62v3qmhkd0c
-          enc: |
+          recipient: age18jn5mrfs4gqrnv0e2sxsgh3kq4sgxx39hwr8z7mz9kt7wlgaasjqlr88ng
+        - enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
             YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB1QmsxNERCclRBeFo0WGZ1
             OGlHTGxDMURsWXdDRXpoUk10S0JpeGk2QURvCjBkaVFHTHo1eGtKbERPWmhNZEpL
@@ -20,8 +20,9 @@ sops:
             QXZwTVd5NnNSdUxRSzNLMHk2SWYxVnMKZyZvZV5qhJj4KATFV2eh9aN9XMbSQL7i
             kRGQI9y5ADv6UsLh/y+rWe0xF4BPOGF0xL8JUnHfTy/RsKfhB3aTYQ==
             -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2026-01-24T17:01:57Z"
-    mac: ENC[AES256_GCM,data:KySo9sJgOLBPtxQZ1gXY8aCmCHbn44aiLKiK1mGziKsBoPkBdEivCT7dDQ1tXQj9wz3gly6raS4FEmR6ikcbtAE2EN86cDLNuu9sSiW2OYiJRe5iZvet2veqtA00K7TtPZgAX+ccsUE5iy0W55bu+kpK3J8MPNy3LUK0EbSKW8I=,iv:7BQKCNyWe4azTelyKlV6XvvgPIrspVbwITyycsW/c44=,tag:1XbSxju44GCKcGw8Ln9iuQ==,type:str]
+          recipient: age1ljpdczmg5ctqyeezn739hv589fwhssjjnuqf7276fqun6kc62v3qmhkd0c
+    lastmodified: "2026-09-21T11:06:39Z"
+    mac: ENC[AES256_GCM,data:pnoz3zIl8cOhq4jgCrWU+j9KkhPao8F4XGCjAblMkayPkF1x9tCC7lf+fS0BeEIADhl7ttUQimjTN62dk77iu0nJzMFfckPt4RRLrYI07yc61cBUPGz0NEDFUE5DxnoxUrpzBGtq4Y55wuD88hBjLZowmk7Eun9v7VAU9tyXrOM=,iv:Iz/fz0k0mhmuFGxsRChRPOi46ywIGTqKJ0AuKcdkCc8=,tag:NBjCzszMoSBrn+o5BHK1tg==,type:str]
     pgp:
         - created_at: "2026-01-24T17:01:26Z"
           enc: |-
@@ -62,4 +63,4 @@ sops:
             -----END PGP MESSAGE-----
           fp: aef8d6c7e4761fc297cda833df13aebb1011b5d4
     unencrypted_suffix: _unencrypted
-    version: 3.11.0
+    version: 3.13.3

machines/overwatch/secrets.yaml

@@ -1,9 +1,9 @@
 grafana_admin: ENC[AES256_GCM,data:c+ZnOyxSXrG4eiK8ETKHheadiSz98LLHYwxb,iv:Ut2qFD2p6OmKDWjLMjFxyISxzTdJpZpgIB7obW5bgkY=,tag:HdayzjXQ1Zc7w9ITLzKLxA==,type:str]
 grafana_smtp: ENC[AES256_GCM,data:gOER9SzqRACIWe3PchyKguX3RdW6xhSZDzLd727CK1w=,iv:KDOyXGYGXnUu92hvt6eBqI8zeKP+JRDsF8Ir5X/9TDk=,tag:wRexcIJp8XEURKcZnHCRtQ==,type:str]
+private0_blackbox_config: ENC[AES256_GCM,data:KzRe0DxBGK0khU9wJ6znLELS6BsoNrdUDu8/n6RnGgAm/r0CQT66AxZZJiImlDtMgHLqB+x/w1hTET0ThzDURr5lL1ckSP7AD2w3vbn/4TmV+/4IeLHeOtiDHmbwoPcOWwSpIelfq1MbhnDjgRITix2IWgOncoenVgo6txIpePxsMhJCM0SZb4Tujy7P79vm,iv:QKK5Q89Glp5nbNOmCmQ2xYVs82X3ErTPjdFCFmEoYXE=,tag:UZW0jMOuEZwS3VXFlJhhoQ==,type:str]
 sops:
     age:
-        - recipient: age1ljpdczmg5ctqyeezn739hv589fwhssjjnuqf7276fqun6kc62v3qmhkd0c
-          enc: |
+        - enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
             YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBWdVhpdTdJNnh5VCtmNHh5
             M1ZTKzk1WlVYYmRWKzJxd1ZUTi9IMW1ta1VvCjVZQVQ1K1lFeC9QZkI2MGZXNk81
@@ -11,8 +11,8 @@ sops:
             NmRNSit1TkRRbUFQNzUvYThUczAxRzAKvpDK4R7m/GpfOzM4nU5vSYDXZGUy4D+c
             xouTxOguMdId8GiKb1AJnVv2q/YYdr2M9yA/FpKn7kBfuQw/Hrj1rg==
             -----END AGE ENCRYPTED FILE-----
-        - recipient: age1hq75x3dpnfqat9sgtfjf8lep49qvkdgza3xwp7ugft3kd74pdfnqfsmmdn
-          enc: |
+          recipient: age1ljpdczmg5ctqyeezn739hv589fwhssjjnuqf7276fqun6kc62v3qmhkd0c
+        - enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
             YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA0bEhCQitwNVpYSHRJOEMr
             ZnZLQ3lELzdtREVpSHVGd3BHZ3p6WU9BK3lFCkFlYXNsa2JWZmZFVjRmK3BTejc1
@@ -20,8 +20,9 @@ sops:
             VnRIYVVIZktISENseEVLVWlsenFhcVkKvj8i349iNX2YNL5G0w7F9IuW/5ZRP6Z7
             XC8TrYzCSOEOXt5nMnZg2yQUJHi6Qq8Wu2e2KV7Cd9J8Q/39dRv2cg==
             -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2026-01-24T16:52:54Z"
-    mac: ENC[AES256_GCM,data:BoJwPldIVZf2NZALbeMTq6DvuBMwOwyvmr5UjsPvQrdbbzYjLTl5dplkhT6QRHPDCFDT4Ibd1UfJ8pQYRkGdO2EjlUyuk3hvAYdR+PUmVz8afFE07sNsGw2EDGVHmsFjUVHLT69YqXtGpVNTW6KjqYuRhz/Ik0wEZLxzDSiy86c=,iv:TQU0si/AYaMpShF4l3CzUN3qajQpE0ZWa7mLOqhRP00=,tag:Oyapkoo4+Q7x08m6RTXfcQ==,type:str]
+          recipient: age1hq75x3dpnfqat9sgtfjf8lep49qvkdgza3xwp7ugft3kd74pdfnqfsmmdn
+    lastmodified: "2026-09-21T11:30:50Z"
+    mac: ENC[AES256_GCM,data:/LmnSheGjhxznLQa1eL67qrlOdujY9fluugjsnCOZov0vJLDRgh5mItxkyH4h7TmMmisU7Pty9s/kRzjLoGmLH0WDmFgg5SnB9Bw6Bt5mn7N4xTiGlfy6RsvJhIZefExeg5kTBo3zQ84pQellGZy/1a0/Vhc6j8hLLPPoO/eL7g=,iv:d/MR5Z7ngB75gMzQiCMAEnxbee8Wcwemd9uKESKvt6E=,tag:QAGB3kiKlxV395o1ZA6ePQ==,type:str]
     pgp:
         - created_at: "2026-01-24T18:05:47Z"
           enc: |-
@@ -62,4 +63,4 @@ sops:
             -----END PGP MESSAGE-----
           fp: aef8d6c7e4761fc297cda833df13aebb1011b5d4
     unencrypted_suffix: _unencrypted
-    version: 3.11.0
+    version: 3.13.3