From d50a9ddbbfc4257198e01789f57098e1737fca4b Mon Sep 17 00:00:00 2001 From: ahtlon Date: Tue, 1 Sep 2026 19:31:49 +0200 Subject: [PATCH 1/3] [overwatch] add the blackbox exporter to monitor websites --- machines/overwatch/blackbox.yaml | 78 ++++++++++++++++++++++++++++ machines/overwatch/configuration.nix | 32 ++++++++++++ 2 files changed, 110 insertions(+) create mode 100644 machines/overwatch/blackbox.yaml diff --git a/machines/overwatch/blackbox.yaml b/machines/overwatch/blackbox.yaml new file mode 100644 index 0000000..7606508 --- /dev/null +++ b/machines/overwatch/blackbox.yaml @@ -0,0 +1,78 @@ +modules: + http_2xx: + prober: http + http: + preferred_ip_protocol: "ip4" + http_post_2xx: + prober: http + http: + method: POST + tcp_connect: + prober: tcp + pop3s_banner: + prober: tcp + tcp: + query_response: + - expect: "^+OK" + tls: true + tls_config: + insecure_skip_verify: false + grpc: + prober: grpc + grpc: + tls: true + preferred_ip_protocol: "ip4" + grpc_plain: + prober: grpc + grpc: + tls: false + service: "service1" + ssh_banner: + prober: tcp + tcp: + query_response: + - expect: "^SSH-2.0-" + - send: "SSH-2.0-blackbox-ssh-check" + ssh_banner_extract: + prober: tcp + timeout: 5s + tcp: + query_response: + - expect: "^SSH-2.0-([^ -]+)(?: (.*))?$" + labels: + - name: ssh_version + value: "${1}" + - name: ssh_comments + value: "${2}" + irc_banner: + prober: tcp + tcp: + query_response: + - send: "NICK prober" + - send: "USER prober prober prober :prober" + - expect: "PING :([^ ]+)" + send: "PONG ${1}" + - expect: "^:[^ ]+ 001" + icmp: + prober: icmp + icmp_ttl5: + prober: icmp + timeout: 5s + icmp: + ttl: 5 + websocket: + prober: websocket + http_3xx: + prober: http + http: + preferred_ip_protocol: "ip4" + enable_http3: true + enable_http2: false + valid_http_versions: ["HTTP/3.0"] + postgresql: + prober: tcp + tcp: + query_response: + - send: !!binary AAAACATSFi8= # 0x00, 0x00, 0x00, 0x08, 0x04, 0xD2, 0x16, 0x2F - PostgreSQL SSLRequest + - expect_bytes: S # 0x53 - Reply will be 'S' if SSL is enabled, and 'N' if it is not. + - starttls: true \ No newline at end of file diff --git a/machines/overwatch/configuration.nix b/machines/overwatch/configuration.nix index 5a7b44f..148ae43 100644 --- a/machines/overwatch/configuration.nix +++ b/machines/overwatch/configuration.nix @@ -108,7 +108,39 @@ in retentionTime = "1y"; port = 9001; + exporters.blackbox = { + enable = true; + configFile = ./blackbox.yaml; + }; + scrapeConfigs = [ + { + job_name = "blackbox-http"; + metrics_path = "/probe"; + params = { + module = ["http_2xx"]; + }; + static_configs = [{ + targets = [ + "https://malobeo.org" + "https://cloud.malobeo.org" + ]; + }]; + relabel_configs = [ + { + source_labels = ["__address__"]; + target_label = "__param_target"; + } + { + source_labels = ["__param_target"]; + target_label = "instance"; + } + { + target_label = "__address__"; + replacement = "127.0.0.1:9115"; + } + ]; + } { job_name = "overwatch"; static_configs = [{ -- 2.54.0 From 4c8253e8e3966f4bf58225fa73c4cd22ec9da5f2 Mon Sep 17 00:00:00 2001 From: ahtlon Date: Fri, 4 Sep 2026 18:21:39 +0200 Subject: [PATCH 2/3] [overwatch] enable and scrape metrics from hydra --- machines/albert/hydra.nix | 10 +++++++++- machines/overwatch/configuration.nix | 12 ++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/machines/albert/hydra.nix b/machines/albert/hydra.nix index e8c9472..b796071 100644 --- a/machines/albert/hydra.nix +++ b/machines/albert/hydra.nix @@ -33,9 +33,17 @@ binary_cache_secret_key_file = /etc/nix/albert/secret binary_cache_dir = /var/lib/hydra/cache Include ${config.sops.secrets.gitea_token.path} + + + listen_address = 0.0.0.0 + port = 9199 + + + queue_runner_metrics_address = 0.0.0.0:9198 ''; }; - + networking.firewall.allowedTCPPorts = [ 9199 9198 ]; + systemd.services.hydra-manual-setup = { description = "Create Admin User for Hydra"; serviceConfig.Type = "oneshot"; diff --git a/machines/overwatch/configuration.nix b/machines/overwatch/configuration.nix index 148ae43..d2d897a 100644 --- a/machines/overwatch/configuration.nix +++ b/machines/overwatch/configuration.nix @@ -141,6 +141,18 @@ in } ]; } + { + job_name = "hydra"; + static_configs = [{ + targets = [ "${hosts.malobeo.hosts.albert.network.address}:9199" ]; + }]; + } + { + job_name = "hydra-queue-runner"; + static_configs = [{ + targets = [ "${hosts.malobeo.hosts.albert.network.address}:9198" ]; + }]; + } { job_name = "overwatch"; static_configs = [{ -- 2.54.0 From 8ee0a99ce717be1d76ba20e5c76d9461f7be18d5 Mon Sep 17 00:00:00 2001 From: ahtlon Date: Mon, 7 Sep 2026 16:59:01 +0200 Subject: [PATCH 3/3] [overwatch] blackbox: add more domains --- machines/overwatch/configuration.nix | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/machines/overwatch/configuration.nix b/machines/overwatch/configuration.nix index d2d897a..280133b 100644 --- a/machines/overwatch/configuration.nix +++ b/machines/overwatch/configuration.nix @@ -124,6 +124,13 @@ in targets = [ "https://malobeo.org" "https://cloud.malobeo.org" + "https://antamap.malobeo.org" + "https://docs.malobeo.org" + "https://events.malobeo.org" + "https://hydra.malobeo.org" + "https://keys.malobeo.org" + "https://tasklist.malobeo.org" + "https://zines.malobeo.org" ]; }]; relabel_configs = [ -- 2.54.0