[overwatch] blackbox: add more domains

Reviewed-on: #174

.gitea/workflows/hydra-callback.yml

@@ -40,7 +40,7 @@ jobs:
               timeout-minutes: 200
               run: |
                 echo "Running now @ pr no $PR"
-                nix run nixpkgs#hydra-cli -- -H https://hydra.dynamicdiscord.de jobset-wait malobeo2 "$PR"
+                nix run nixpkgs#hydra-cli -- -H https://hydra.malobeo.org jobset-wait malobeo "$PR"
 
     on_push:
         if: github.event.push
@@ -60,4 +60,4 @@ jobs:
               timeout-minutes: 200
               run: |
                 echo "Running now @ master"
-                nix run nixpkgs#hydra-cli -- -H https://hydra.dynamicdiscord.de jobset-wait malobeo2 master
+                nix run nixpkgs#hydra-cli -- -H https://hydra.malobeo.org jobset-wait malobeo master

machines/albert/configuration.nix

@@ -6,6 +6,10 @@ let
 in
 {
   networking.hostName = "albert";
+  networking.nameservers = [
+    "192.168.1.1"
+    "1.1.1.1"
+  ];
 
   sops.defaultSopsFile = ./secrets.yaml;
   sops.secrets.wg_private = {};
@@ -45,7 +49,7 @@ in
     enable = true;
     url = "https://hydra.malobeo.org";
     project = "malobeo";
-    jobset = "infrastructure";
+    jobset = "master";
     cacheurl = "https://cache.malobeo.org";
   };
 
@@ -77,14 +81,13 @@ in
 
   networking = {
     firewall = {
-      allowedTCPPorts = [ 80 443 3000 ];
+      allowedTCPPorts = [ 80 443 3000 5000];
     };
   };
 
   nixpkgs.config.allowUnfree = true; 
 
   services.acpid.enable = true;
-  networking.networkmanager.enable = true;
   time.timeZone = "Europe/Berlin";
   system.stateVersion = "26.05"; # Do.. Not.. Change..
 }

machines/albert/hydra.nix

@@ -33,9 +33,17 @@
       binary_cache_secret_key_file = /etc/nix/albert/secret
       binary_cache_dir = /var/lib/hydra/cache
       Include ${config.sops.secrets.gitea_token.path}
+      <hydra_notify>
+        <prometheus>
+          listen_address = 0.0.0.0
+          port = 9199
+        </prometheus>
+      </hydra_notify>
+      queue_runner_metrics_address = 0.0.0.0:9198
     '';
   };
-
+  networking.firewall.allowedTCPPorts = [ 9199 9198 ];
+  
   systemd.services.hydra-manual-setup = {
     description = "Create Admin User for Hydra";
     serviceConfig.Type = "oneshot";
@@ -95,11 +103,5 @@
     "git+ssh://git.dynamicdiscord.de/"
     "https://git.dynamicdiscord.de"
   ];
-
-  services.nginx.virtualHosts."hydra.malobeo.org" = {
-    forceSSL = true;
-    enableACME = true;
-    locations."/".proxyPass = "http://localhost:3000";
-  };
 }
 

machines/antamap/configuration.nix

@@ -0,0 +1,59 @@
+{ self, config, lib, pkgs, inputs, ... }:
+
+with lib;
+
+let
+  antamap-pkg = pkgs.stdenv.mkDerivation {
+    name = "antamap";
+    src = pkgs.fetchgit {
+      url = "https://0xacab.org/AND/antamap.git";
+      branchName = "master";
+      rev = "f07f4522e8658ee5dc113ea3feffb67418e5f403";
+      sha256 = "sha256-ODEey7PwoBAwVxcQm02Q8/a42UxzOcQB2Dqr3FYxqyo=";
+    };
+
+    installPhase = "mkdir -p $out; cp -R * $out/";
+  };
+in
+{
+  networking = {
+    hostName = mkDefault "antamap";
+    useDHCP = false;
+  };
+
+  imports = [
+    self.nixosModules.malobeo.users
+    ../modules/sshd.nix
+  ];
+
+  malobeo.users = {
+    malobeo = false;
+    admin = true;
+  };
+
+  services.nginx = {
+    enable = true;
+    virtualHosts."_" = {
+      listen = [ 
+        { addr = "0.0.0.0"; port = 80; } 
+      ];
+      root = "${antamap-pkg}";
+      extraConfig = ''
+        proxy_buffering off;
+        proxy_cache off;
+        proxy_http_version 1.1; 
+        proxy_set_header Host $host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Proto $scheme;
+      '';
+    };
+  };
+
+  networking.firewall.allowedTCPPorts = [ 80 ];
+
+  time.timeZone = "Europe/Berlin";
+
+  system.stateVersion = "22.11"; # Did you read the comment?
+}
+

machines/bakunin/configuration.nix

@@ -19,7 +19,7 @@ in
     enable = true;
     url = "https://hydra.malobeo.org";
     project = "malobeo";
-    jobset = "infrastructure";
+    jobset = "master";
     cacheurl = "https://cache.malobeo.org";
   };
 
@@ -83,7 +83,6 @@ in
   services.acpid.enable = true;
 
   networking.hostName = "bakunin";
-  networking.networkmanager.enable = true;
 
   security.rtkit.enable = true;
   services.pipewire = {

machines/fanny/configuration.nix

@@ -44,7 +44,7 @@ in
     enable = true;
     url = "https://hydra.malobeo.org";
     project = "malobeo";
-    jobset = "infrastructure";
+    jobset = "master";
     cacheurl = "https://cache.malobeo.org";
   };
 
@@ -146,6 +146,7 @@ in
     "zineshop"
     "vaultwarden"
     "pretalx"
+    "antamap" 
   ];
 
   networking = {
@@ -286,6 +287,38 @@ in
         '';
       };
     };
+
+    virtualHosts."antamap.malobeo.org" = {
+      forceSSL = true;
+      enableACME = true;
+      locations."/" = {
+        proxyPass = "http://${hosts.malobeo.hosts.antamap.network.address}:80";
+        extraConfig = ''
+          proxy_set_header Host $host;
+        '';
+      };
+    };
+    
+    virtualHosts."hydra.malobeo.org" = {
+      forceSSL = true;
+      enableACME = true;
+      locations."/" = {
+        proxyPass = "http://${hosts.malobeo.hosts.albert.network.address}:3000";
+        extraConfig = ''
+          proxy_set_header Host $host;
+        '';
+      };
+    };
+    virtualHosts."cache.malobeo.org" = {
+      forceSSL = true;
+      enableACME = true;
+      locations."/" = {
+        proxyPass = "http://${hosts.malobeo.hosts.albert.network.address}:5000";
+        extraConfig = ''
+          proxy_set_header Host $host;
+        '';
+      };
+    };
   };
 
   services.tor = {
@@ -299,7 +332,6 @@ in
   services.acpid.enable = true;
 
   networking.hostName = "fanny";
-  networking.networkmanager.enable = true;
 
   virtualisation.vmVariant.virtualisation.graphics = false;
 

machines/fanny/dyndns.nix

@@ -2,12 +2,15 @@
 {
   sops.secrets.njalacloud = {};
   sops.secrets.njalazines = {};
+  sops.secrets.njalaantamap = {};
   systemd.services."dyndns" = {
     script = ''
       KEYCLOUD=$(cat /run/secrets/njalacloud)
       KEYZINES=$(cat /run/secrets/njalazines)
+      KEYANATAMAP=$(cat /run/secrets/njalaantamap)
       ${pkgs.curl}/bin/curl --fail --silent --show-error "https://njal.la/update/?h=cloud.malobeo.org&k="$KEYCLOUD"&auto"
       ${pkgs.curl}/bin/curl --fail --silent --show-error "https://njal.la/update/?h=zines.malobeo.org&k="$KEYZINES"&auto"
+      ${pkgs.curl}/bin/curl --fail --silent --show-error "https://njal.la/update/?h=antamap.malobeo.org&k="$KEYANATAMAP"&auto"
     '';
     serviceConfig = {
       Type = "oneshot";

machines/fanny/secrets.yaml

@@ -3,11 +3,11 @@ shop_cleartext: ENC[AES256_GCM,data:sifpX/R6JCcNKgwN2M4Dbflgnfs5CqB8ez5fULPohuFS
 shop_auth: ENC[AES256_GCM,data:0NDIRjmGwlSFls12sCb5OlgyGTCHpPQIjycEJGhYlZsWKhEYXV2u3g1RHMkF8Ny913jarjf0BgwSq5pBD9rgPL9t8X8=,iv:3jgCv/Gg93Mhdm4eYzwF9QrK14QL2bcC4wwSajCA88o=,tag:h8dhMK46hABv9gYW4johkA==,type:str]
 njalacloud: ENC[AES256_GCM,data:sp79Ij1vd9pQZuPUR1phmw==,iv:AWKZoOfBA/n16pWQCfA0dZmH1KajCztnLvYItoZZbgA=,tag:BIUrobBoO96pxUz1sjIYIw==,type:str]
 njalazines: ENC[AES256_GCM,data:fnObUEnXYvdj9HtkZNzXVA==,iv:0Zj2n2we9w4fj/n7e1ayd9XgFEMAGCHk4QLTu1IlRnQ=,tag:zeOLAB0oE6XbxqdqhdRNxw==,type:str]
+njalaantamap: ENC[AES256_GCM,data:nUc+E7HNbW0EDclzNDV6+w==,iv:H8nqeuOrvvu8O8QhGWkBkYWAQXwkBehf/jU6u/grs1A=,tag:mqX6jn47DyeSuk8xzG0e/Q==,type:str]
 njala_api_key: ENC[AES256_GCM,data:ohSVzQUvFjia/s9WceqnZCdLyk3N1Lm2BCBmXeBlkWD2dyrohKCnd9GiJ499IORpuYcOXyM=,iv:Uczk8op5mgqe8gefxgU9YuTqOsYvjzHCKvzA7GDsgio=,tag:XA7JRq/LsGkpHcQSO36Whg==,type:str]
 sops:
     age:
-        - recipient: age136sz3lzhxf74ryruvq34d4tmmxnezkqkgu6zqa3dm582c22fgejqagrqxk
-          enc: |
+        - enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
             YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB2ZFBYMHMzTFRMLzhCbnBE
             MXkreklWSUVOckl5OTJ0VzlWS2tIOFBRRVVJCk90OXJoMHQza0hTSGt5VUphNjY1
@@ -15,8 +15,8 @@ sops:
             TStjbTBkMTNOcHBja0JRYUdvSWJUN00KtOPBH8xZy/GD9Ua3H6jisoluCR+UzaeE
             pAWM9Y6Gn6f7jv2BPKVTaWsyrafsYP7cDabQe2ancAuuKvkng/jrEw==
             -----END AGE ENCRYPTED FILE-----
-        - recipient: age1ljpdczmg5ctqyeezn739hv589fwhssjjnuqf7276fqun6kc62v3qmhkd0c
-          enc: |
+          recipient: age136sz3lzhxf74ryruvq34d4tmmxnezkqkgu6zqa3dm582c22fgejqagrqxk
+        - enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
             YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBhc282T2VVamFGcG1Ub3hp
             S1VwKzVsWW1sRXczZnRNdkxDWE5Sd0hhVUJRCkovNGZ1ZlN0c1VyMXV0WThJMGFi
@@ -24,8 +24,9 @@ sops:
             QVZyNWVOMTh3ejBha21Qb2xCRkFERGMKH9nMQUoS5bGcLUx2T1dOmKd9jshttTrP
             SKFx7MXcjFRLKS2Ij12V8ftjL3Uod6be5zoMibkxK19KmXY/514Jww==
             -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2026-02-20T19:25:05Z"
-    mac: ENC[AES256_GCM,data:g+bFYqJN1X8F52tpIO60S2WKxLG27ZrP399fsfE6o7rPtIMimZou/4oUo7i+kpNtygEuCr3+suP8TPas4x5zMXhRjnjJuwJwL/NwdciHZU0O3rPJgucCEWqr9OdAtxezDM9c2vv+jzqZxWT9t0fIpB9RxO5oy1pHZs0RCgjAJR4=,iv:v6RdTMeQUxSdjIVNFbx2HtxCsdVgFTQTzMXS5Fj62is=,tag:pLoZMBRIXYElO5rY+xX9zg==,type:str]
+          recipient: age1ljpdczmg5ctqyeezn739hv589fwhssjjnuqf7276fqun6kc62v3qmhkd0c
+    lastmodified: "2026-08-28T12:19:14Z"
+    mac: ENC[AES256_GCM,data:TnElCRaL/b238clfFVEq+Ot1lwg5MLMchCkmhi5PmADRn2exA42Zeq5v1M/IM5sHt4xEyFLn5R/88Y49DAfehRFwu6rddV+3lnFV40k2rWRtMfQOUXSrCd7lGxZdFbMU0eIXF5jaATFbcP0wO03FohNA69/u6kNsS0y0jK3vuLQ=,iv:xPeNXUIavVYFfAv1fs5TXCHXmvnXQDs4mJZMHOX7qG8=,tag:85hrBTKMh1ckTtXpOxj5lg==,type:str]
     pgp:
         - created_at: "2025-02-11T18:32:49Z"
           enc: |-
@@ -66,4 +67,4 @@ sops:
             -----END PGP MESSAGE-----
           fp: aef8d6c7e4761fc297cda833df13aebb1011b5d4
     unencrypted_suffix: _unencrypted
-    version: 3.11.0
+    version: 3.13.1

machines/hosts.nix

@@ -29,6 +29,8 @@ in
           local = true;
           hostId = "11";
           address = "192.168.1.101";
+          gateway = "192.168.1.1";
+          nameservers = [ "192.168.1.1" "1.1.1.1" ];
         };
       };
 
@@ -38,6 +40,8 @@ in
           local = true;
           hostId = "12";
           address = "192.168.1.102";
+          gateway = "192.168.1.1";
+          nameservers = [ "192.168.1.1" "1.1.1.1" ];
         };
       };
   
@@ -47,6 +51,8 @@ in
           local = true;
           hostId = "13";
           address = "192.168.1.103";
+          gateway = "192.168.1.1";
+          nameservers = [ "192.168.1.1" "1.1.1.1" ];
         };
       };
   
@@ -56,6 +62,8 @@ in
           local = true;
           hostId = "15";
           address = "192.168.1.105";
+          gateway = "192.168.1.1";
+          nameservers = [ "192.168.1.1" "1.1.1.1" ];
         };
       };
 
@@ -65,6 +73,8 @@ in
           local = true;
           hostId = "19";
           address = "192.168.1.111";
+          gateway = "192.168.1.1";
+          nameservers = [ "192.168.1.1" "1.1.1.1" ];
         };
       };
   
@@ -118,7 +128,12 @@ in
         network = createMaloNet "18" "52:DA:0D:F9:EF:F4";
       };
 
-      # last host id: 19
+      antamap = {
+        type = "microvm";
+        network = createMaloNet "20" "52:DA:0D:F9:EF:F6";
+      };
+
+      # last host id: 20
     };
   };
 }

machines/louise/configuration.nix

@@ -16,7 +16,7 @@
     enable = true;
     url = "https://hydra.malobeo.org";
     project = "malobeo";
-    jobset = "infrastructure";
+    jobset = "master";
     cacheurl = "https://cache.malobeo.org";
   };
 
@@ -71,7 +71,6 @@
   services.acpid.enable = true;
 
   networking.hostName = "louise";
-  networking.networkmanager.enable = true;
 
   security.rtkit.enable = true;
   services.pipewire = {

machines/modules/autoupdate.nix

@@ -92,6 +92,7 @@ in
       '';
       # don't let the switch kill this service, aborting the switch
       restartIfChanged = false;
+      serviceConfig.RemainAfterExit = true;
       unitConfig.X-StopOnRemoval = false;
       # create timer
       startAt = "hourly";

machines/modules/host_builder.nix

@@ -31,8 +31,8 @@ rec {
                 "https://cache.nixos.org/"
               ];
               trusted-public-keys = [
-                # TODO: add cache.malobeo.org key after creation
                 "cache.dynamicdiscord.de:DKueZicqi2NhJJXz9MYgUbiyobMs10fTyHCgAUibRP4="
+                "cache.malobeo.org:+eSv8F63uj94A0fvmdyPcDDH0qI3CYR91fsbA/eLFGw="
               ];
               trusted-users = [ "root" "@wheel" ];
             };
@@ -45,6 +45,20 @@ rec {
   ];
   defaultModules = baseModules;
 
+  makeNetwork = network: [{
+    systemd.network.enable = true;
+    
+    systemd.network.networks."20-lan" = {
+      matchConfig.Type = "ether";
+      networkConfig = {
+        Address = [ "${network.address}/24" ];
+        Gateway = "${network.gateway}";
+        DNS = network.nameservers;
+        DHCP = "no";
+      };
+    };
+  }];
+
   makeMicroVM = hostName: network: modules: [
     {
       microvm = {
@@ -89,18 +103,6 @@ rec {
           }
         ];
       };
-
-      systemd.network.enable = true;
-      
-      systemd.network.networks."20-lan" = {
-        matchConfig.Type = "ether";
-        networkConfig = {
-          Address = [ "${network.address}/24" ];
-          Gateway = "${network.gateway}";
-          DNS = network.nameservers;
-          DHCP = "no";
-        };
-      };
     }
   ] ++ defaultModules ++ modules;
 
@@ -259,9 +261,9 @@ rec {
     specialArgs.inputs = inputsMod;
     specialArgs.self = self;
     modules = (if (settings.type != "microvm") then
-      defaultModules ++ [ ../${host}/configuration.nix ]
+      (makeNetwork settings.network) ++ defaultModules ++ [ ../${host}/configuration.nix ]
       else
-      makeMicroVM "${host}" settings.network [
+      (makeNetwork settings.network) ++ makeMicroVM "${host}" settings.network [
         inputs.microvm.nixosModules.microvm
         ../${host}/configuration.nix
       ]);

machines/overwatch/blackbox.yaml

@@ -0,0 +1,78 @@
+modules:
+  http_2xx:
+    prober: http
+    http:
+      preferred_ip_protocol: "ip4"
+  http_post_2xx:
+    prober: http
+    http:
+      method: POST
+  tcp_connect:
+    prober: tcp
+  pop3s_banner:
+    prober: tcp
+    tcp:
+      query_response:
+      - expect: "^+OK"
+      tls: true
+      tls_config:
+        insecure_skip_verify: false
+  grpc:
+    prober: grpc
+    grpc:
+      tls: true
+      preferred_ip_protocol: "ip4"
+  grpc_plain:
+    prober: grpc
+    grpc:
+      tls: false
+      service: "service1"
+  ssh_banner:
+    prober: tcp
+    tcp:
+      query_response:
+      - expect: "^SSH-2.0-"
+      - send: "SSH-2.0-blackbox-ssh-check"
+  ssh_banner_extract:
+    prober: tcp
+    timeout: 5s
+    tcp:
+      query_response:
+      - expect: "^SSH-2.0-([^ -]+)(?: (.*))?$"
+        labels:
+        - name: ssh_version
+          value: "${1}"
+        - name: ssh_comments
+          value: "${2}"
+  irc_banner:
+    prober: tcp
+    tcp:
+      query_response:
+      - send: "NICK prober"
+      - send: "USER prober prober prober :prober"
+      - expect: "PING :([^ ]+)"
+        send: "PONG ${1}"
+      - expect: "^:[^ ]+ 001"
+  icmp:
+    prober: icmp
+  icmp_ttl5:
+    prober: icmp
+    timeout: 5s
+    icmp:
+      ttl: 5
+  websocket:
+    prober: websocket
+  http_3xx:
+    prober: http
+    http:
+      preferred_ip_protocol: "ip4"
+      enable_http3: true
+      enable_http2: false
+      valid_http_versions: ["HTTP/3.0"]
+  postgresql:
+    prober: tcp
+    tcp:
+      query_response:
+      - send: !!binary AAAACATSFi8= # 0x00, 0x00, 0x00, 0x08, 0x04, 0xD2, 0x16, 0x2F - PostgreSQL SSLRequest
+      - expect_bytes: S # 0x53 - Reply will be 'S' if SSL is enabled, and 'N' if it is not.
+      - starttls: true

machines/overwatch/configuration.nix

@@ -108,7 +108,58 @@ in
     retentionTime = "1y";
     port = 9001;
 
+    exporters.blackbox = {
+      enable = true;
+      configFile = ./blackbox.yaml;
+    };
+
     scrapeConfigs = [
+      {
+        job_name = "blackbox-http";
+        metrics_path = "/probe";
+        params = {
+          module = ["http_2xx"];
+        };
+        static_configs = [{
+          targets = [ 
+            "https://malobeo.org"
+            "https://cloud.malobeo.org"
+            "https://antamap.malobeo.org"
+            "https://docs.malobeo.org"
+            "https://events.malobeo.org"
+            "https://hydra.malobeo.org"
+            "https://keys.malobeo.org"
+            "https://tasklist.malobeo.org"
+            "https://zines.malobeo.org"
+           ];
+        }];
+        relabel_configs = [
+          {
+            source_labels = ["__address__"];
+            target_label = "__param_target";
+          }
+          {
+            source_labels = ["__param_target"];
+            target_label = "instance";
+          }
+          {
+            target_label = "__address__";
+            replacement = "127.0.0.1:9115";
+          }
+        ];
+      }
+      {
+        job_name = "hydra";
+        static_configs = [{
+          targets = [ "${hosts.malobeo.hosts.albert.network.address}:9199" ];
+        }];
+      }
+      {
+        job_name = "hydra-queue-runner";
+        static_configs = [{
+          targets = [ "${hosts.malobeo.hosts.albert.network.address}:9198" ];
+        }];
+      }
       {
         job_name = "overwatch";
         static_configs = [{
@@ -163,6 +214,12 @@ in
           targets = [ "${hosts.malobeo.hosts.fanny.network.address}:9002" ];
         }];
       }
+      {
+        job_name = "albert";
+        static_configs = [{
+          targets = [ "${hosts.malobeo.hosts.albert.network.address}:9002" ];
+        }];
+      }
       # add vpn - check how to reach it first. most probably 10.100.0.1
     ];
   };

machines/testvm/configuration.nix

@@ -51,7 +51,6 @@ in
   services.acpid.enable = true;
 
   networking.hostName = "testvm";
-  networking.networkmanager.enable = true;
 
   time.timeZone = "Europe/Berlin";
   system.stateVersion = "23.05"; # Do.. Not.. Change..