infrastructure/machines/overwatch/configuration.nix

{ config, self, lib, pkgs, inputs, ... }:

with lib;

let
  hosts = import ../hosts.nix {};
in
{
  sops.defaultSopsFile = ./secrets.yaml;
  sops.secrets = {
    grafana_smtp = {
      owner = "grafana";
      group = "grafana";
    };
  };

  networking = {
    hostName = mkDefault "overwatch";
    useDHCP = false;
  };

  imports = [
    self.nixosModules.malobeo.metrics
    ../modules/malobeo_user.nix
    ../modules/sshd.nix
    ./printer_module.nix
  ];

  networking.firewall.allowedTCPPorts = [ 80 3100 ];

  malobeo.metrics = {
    enable = true;
    enablePromtail = true;
    logNginx = false;
  };

  services.grafana = {
    enable = true;
    settings = {
      security.secret_key = "SW2YcwTIb9zpOOhoPsMm";
      server = {
        domain = "grafana.malobeo.org";
        http_port = 2342;
        http_addr = "127.0.0.1";
      };

      smtp = {
        enabled = true;
        host = "mail.systemli.org:465";
        user = "malobot@systemli.org";
        from_address = "malobot@systemli.org";
        from_name = "malobot";
        password = "$__file{${config.sops.secrets.grafana_smtp.path}}";
      };
    };

    provision.datasources.settings = {
      apiVersion = 1;

      datasources = [
        {
          name = "loki";
          type = "loki";
          access = "proxy";
          uid = "eeakiack8nqwwc";
          url = "http://localhost:3100";
          editable = false;
        }
        {
          name = "prometheus";
          type = "prometheus";
          access = "proxy";
          uid = "feakib1gq7ugwc";
          url = "http://localhost:9001";
          editable = false;

        }
      ];
    };

    provision.dashboards.settings = {
      apiVersion = 1;
      providers = [{
        name = "default";
        options.path = ./dashboards;
      }];
    };
  };

  services.nginx = {
    enable = true;
    virtualHosts.${config.services.grafana.settings.server.domain} = {
      locations."/" = {
          proxyPass = "http://127.0.0.1:${toString config.services.grafana.settings.server.http_port}";
          proxyWebsockets = true;

      extraConfig = ''
        proxy_set_header Host $host;
      '';
    };
      };
  };
  
  printer_scraping.enable = true;

  services.prometheus = {
    enable = true;
    retentionTime = "1y";
    port = 9001;

    exporters.blackbox = {
      enable = true;
      configFile = ./blackbox.yaml;
    };

    scrapeConfigs = [
      {
        job_name = "blackbox-http";
        metrics_path = "/probe";
        params = {
          module = ["http_2xx"];
        };
        static_configs = [{
          targets = [ 
            "https://malobeo.org"
            "https://cloud.malobeo.org"
            "https://antamap.malobeo.org"
            "https://docs.malobeo.org"
            "https://events.malobeo.org"
            "https://hydra.malobeo.org"
            "https://keys.malobeo.org"
            "https://tasklist.malobeo.org"
            "https://zines.malobeo.org"
           ];
        }];
        relabel_configs = [
          {
            source_labels = ["__address__"];
            target_label = "__param_target";
          }
          {
            source_labels = ["__param_target"];
            target_label = "instance";
          }
          {
            target_label = "__address__";
            replacement = "127.0.0.1:9115";
          }
        ];
      }
      {
        job_name = "hydra";
        static_configs = [{
          targets = [ "${hosts.malobeo.hosts.albert.network.address}:9199" ];
        }];
      }
      {
        job_name = "hydra-queue-runner";
        static_configs = [{
          targets = [ "${hosts.malobeo.hosts.albert.network.address}:9198" ];
        }];
      }
      {
        job_name = "overwatch";
        static_configs = [{
          targets = [ "127.0.0.1:9002" ];
        }];
      }
      {
        job_name = "printer";
        static_configs = [{
          targets = [ "127.0.0.1:9091" ];
        }];
      }
      {
        job_name = "durruti";
        static_configs = [{
          targets = [ "${hosts.malobeo.hosts.durruti.network.address}:9002" ];
        }];
      }
      {
        job_name = "infradocs";
        static_configs = [{
          targets = [ "${hosts.malobeo.hosts.infradocs.network.address}:9002" ];
        }];
      }
      {
        job_name = "nextcloud";
        static_configs = [{
          targets = [ "${hosts.malobeo.hosts.nextcloud.network.address}:9002" ];
        }];
      }
      {
        job_name = "zineshop";
        static_configs = [{
          targets = [ "${hosts.malobeo.hosts.zineshop.network.address}:9002" ];
        }];
      }
      {
        job_name = "vaultwarden";
        static_configs = [{
          targets = [ "${hosts.malobeo.hosts.vaultwarden.network.address}:9002" ];
        }];
      }
      {
        job_name = "pretalx";
        static_configs = [{
          targets = [ "${hosts.malobeo.hosts.pretalx.network.address}:9002" ];
        }];
      }
      {
        job_name = "fanny";
        static_configs = [{
          targets = [ "${hosts.malobeo.hosts.fanny.network.address}:9002" ];
        }];
      }
      {
        job_name = "albert";
        static_configs = [{
          targets = [ "${hosts.malobeo.hosts.albert.network.address}:9002" ];
        }];
      }
      # add vpn - check how to reach it first. most probably 10.100.0.1
    ];
  };

  services.loki = {
    enable = true;
    configFile = ./loki.yaml;
  };

  users.users.promtail.extraGroups = [ "nginx" "systemd-journal" ];



  system.stateVersion = "22.11"; # Did you read the comment?
}