Merge branch 'add_blackbox_metrics'

machines/albert/hydra.nix

@@ -35,9 +35,17 @@
       Include ${config.sops.secrets.gitea_token.path}
       using_frontend_proxy 1
       base_uri hydra.malobeo.org
+      <hydra_notify>
+        <prometheus>
+          listen_address = 0.0.0.0
+          port = 9199
+        </prometheus>
+      </hydra_notify>
+      queue_runner_metrics_address = 0.0.0.0:9198
     '';
   };
-
+  networking.firewall.allowedTCPPorts = [ 9199 9198 ];
+  
   systemd.services.hydra-manual-setup = {
     description = "Create Admin User for Hydra";
     serviceConfig.Type = "oneshot";

machines/overwatch/blackbox.yaml

@@ -0,0 +1,78 @@
+modules:
+  http_2xx:
+    prober: http
+    http:
+      preferred_ip_protocol: "ip4"
+  http_post_2xx:
+    prober: http
+    http:
+      method: POST
+  tcp_connect:
+    prober: tcp
+  pop3s_banner:
+    prober: tcp
+    tcp:
+      query_response:
+      - expect: "^+OK"
+      tls: true
+      tls_config:
+        insecure_skip_verify: false
+  grpc:
+    prober: grpc
+    grpc:
+      tls: true
+      preferred_ip_protocol: "ip4"
+  grpc_plain:
+    prober: grpc
+    grpc:
+      tls: false
+      service: "service1"
+  ssh_banner:
+    prober: tcp
+    tcp:
+      query_response:
+      - expect: "^SSH-2.0-"
+      - send: "SSH-2.0-blackbox-ssh-check"
+  ssh_banner_extract:
+    prober: tcp
+    timeout: 5s
+    tcp:
+      query_response:
+      - expect: "^SSH-2.0-([^ -]+)(?: (.*))?$"
+        labels:
+        - name: ssh_version
+          value: "${1}"
+        - name: ssh_comments
+          value: "${2}"
+  irc_banner:
+    prober: tcp
+    tcp:
+      query_response:
+      - send: "NICK prober"
+      - send: "USER prober prober prober :prober"
+      - expect: "PING :([^ ]+)"
+        send: "PONG ${1}"
+      - expect: "^:[^ ]+ 001"
+  icmp:
+    prober: icmp
+  icmp_ttl5:
+    prober: icmp
+    timeout: 5s
+    icmp:
+      ttl: 5
+  websocket:
+    prober: websocket
+  http_3xx:
+    prober: http
+    http:
+      preferred_ip_protocol: "ip4"
+      enable_http3: true
+      enable_http2: false
+      valid_http_versions: ["HTTP/3.0"]
+  postgresql:
+    prober: tcp
+    tcp:
+      query_response:
+      - send: !!binary AAAACATSFi8= # 0x00, 0x00, 0x00, 0x08, 0x04, 0xD2, 0x16, 0x2F - PostgreSQL SSLRequest
+      - expect_bytes: S # 0x53 - Reply will be 'S' if SSL is enabled, and 'N' if it is not.
+      - starttls: true

machines/overwatch/configuration.nix

@@ -108,7 +108,58 @@ in
     retentionTime = "1y";
     port = 9001;
 
+    exporters.blackbox = {
+      enable = true;
+      configFile = ./blackbox.yaml;
+    };
+
     scrapeConfigs = [
+      {
+        job_name = "blackbox-http";
+        metrics_path = "/probe";
+        params = {
+          module = ["http_2xx"];
+        };
+        static_configs = [{
+          targets = [ 
+            "https://malobeo.org"
+            "https://cloud.malobeo.org"
+            "https://antamap.malobeo.org"
+            "https://docs.malobeo.org"
+            "https://events.malobeo.org"
+            "https://hydra.malobeo.org"
+            "https://keys.malobeo.org"
+            "https://tasklist.malobeo.org"
+            "https://zines.malobeo.org"
+           ];
+        }];
+        relabel_configs = [
+          {
+            source_labels = ["__address__"];
+            target_label = "__param_target";
+          }
+          {
+            source_labels = ["__param_target"];
+            target_label = "instance";
+          }
+          {
+            target_label = "__address__";
+            replacement = "127.0.0.1:9115";
+          }
+        ];
+      }
+      {
+        job_name = "hydra";
+        static_configs = [{
+          targets = [ "${hosts.malobeo.hosts.albert.network.address}:9199" ];
+        }];
+      }
+      {
+        job_name = "hydra-queue-runner";
+        static_configs = [{
+          targets = [ "${hosts.malobeo.hosts.albert.network.address}:9198" ];
+        }];
+      }
       {
         job_name = "overwatch";
         static_configs = [{