[overwatch] init

still most of it is quite out of date...

doc/src/Index.md

@@ -1 +1,82 @@
-# Index
+# malobeo infrastructure
+
+this repository nxios configurations of the digital malobeo infrastructure. it should be used to setup, test, build and deploy different hosts in a reproducible manner.
+
+the file structure is based on this [blog post](https://samleathers.com/posts/2022-02-03-my-new-network-and-deploy-rs.html)
+
+### deploying configuration
+#### local deployment
+``` shell
+nixos-rebuild switch --use-remote-sudo
+```
+
+#### remote deployment
+you need the hostname and ip address of the host:
+``` shell
+ nixos-rebuild switch --flake .#<hostname> --target-host root@<ip_address> --build-host localhost
+```
+
+in this case 'localhost' is used as buildhost which can be usefull if the target host is low systemresources
+
+
+## development
+
+### requirements
+we use flake based configurations for our hosts. if you want to build configurations on you own machine you have to enable flakes first by adding the following to your *configuration.nix* or *nix.conf*
+``` nix
+nix.extraOptions = ''
+  experimental-features = nix-command flakes
+'';
+```
+
+More information about flakes can be found [here](https://nixos.wiki/wiki/Flakes)
+
+### dev shell
+a development shell with the correct environment can be created by running ```nix develop ```
+
+If you're using direnv you can add flake support by following those steps: [link](https://nixos.wiki/wiki/Flakes#Direnv_integration)
+
+### build a configuration
+
+to build a configuration run the following command (replace ```<hostname>``` with the actual hostname):
+
+``` shell
+nix build .#nixosConfigurations.<hostname>.config.system.build.toplevel
+```
+
+### building raspberry image
+
+for the raspberry it is possible to build the whole configuration as an sd-card image which then can be flashed directly. more information about building arm on nixos can be found [here](https://nixos.wiki/wiki/NixOS_on_ARM).
+
+to be able to build the image you need to enable qemu emulation on the machine you are building with. therefore it is necessary to add the following to your configuration.nix:
+
+``` nix
+boot.binfmt.emulatedSystems = [ "aarch64-linux" ];
+```
+
+then you can build the image with:
+
+``` shell
+nix build .#nixosConfigurations.rpi1_base_image.config.system.build.sdImage
+```
+
+### run a configuration as vm
+
+to run a vm we have to build it first using the following command (replace ```<hostname>``` with the actual hostname):
+
+``` shell
+nix build .#nixosConfigurations.<hostname>.config.system.build.vm
+```
+
+afterwards run the following command to start the vm:
+
+``` shell
+./result/bin/run-<hostname>-vm
+```
+
+### documentation
+
+for documentation we currently just use README.md files.
+
+the devshell provides the python package ['grip'](https://github.com/joeyespo/grip) which can be used to preview different README.md files in the browser.
+the usage is simple, just run ```grip``` in the same folder as the README.md you wanna preview. then open your browser at ```http://localhost:6419 ```.

machines/configuration.nix

@@ -170,6 +170,16 @@ in
     ];
   };
 
+  overwatch = nixosSystem {
+    system = "x86_64-linux";
+    specialArgs.inputs = inputs;
+    specialArgs.self = self;
+    modules = makeMicroVM "overwatch" "10.0.0.13" "D0:E5:CA:F0:D7:E9" [
+      ./overwatch/configuration.nix
+    ];
+  };
+
+
   testvm = nixosSystem {
     system = "x86_64-linux";
     specialArgs.inputs = inputs;

machines/durruti/configuration.nix

@@ -8,7 +8,6 @@ with lib;
   networking = {
     hostName = mkDefault "durruti";
     useDHCP = false;
-    nameservers = [ "1.1.1.1" ];
   };
 
   networking.firewall.allowedTCPPorts = [ 8080 ];

machines/infradocs/configuration.nix

@@ -6,7 +6,6 @@ with lib;
   networking = {
     hostName = mkDefault "infradocs";
     useDHCP = false;
-    nameservers = [ "1.1.1.1" ];
   };
 
   imports = [

machines/modules/malobeo/promtail_config.nix

@@ -0,0 +1,49 @@
+{ logNginx, lokiAddress, config, pkgs, ... }:
+
+let
+  basecfg = ''
+    server:
+      http_listen_port: 9080
+      grpc_listen_port: 0
+    
+    positions:
+      filename: /tmp/positions.yaml
+    
+    clients:
+      - url: http://${lokiAddress}:3100/loki/api/v1/push
+  '';
+
+  withNginx = ''
+    scrape_configs:
+      - job_name: journal
+        journal:
+          max_age: 12h
+          labels:
+            job: systemd-journal
+            host: ${config.networking.hostName}
+        relabel_configs:
+          - source_labels: ["__journal__systemd_unit"]
+            target_label: "unit"
+      - job_name: nginx
+        static_configs:
+        - targets:
+            - localhost
+          labels:
+            job: nginx
+            __path__: /var/log/nginx/*log
+  '';
+
+  withoutNginx = ''
+    scrape_configs:
+      - job_name: journal
+        journal:
+          max_age: 12h
+          labels:
+            job: systemd-journal
+            host: ${config.networking.hostName}
+        relabel_configs:
+          - source_labels: ["__journal__systemd_unit"]
+            target_label: "unit"
+  '';
+in
+pkgs.writeText "promtailcfg.yaml" (if logNginx then ''${basecfg}${withNginx}'' else ''${basecfg}${withoutNginx}'')

machines/overwatch/configuration.nix

@@ -0,0 +1,87 @@
+{ config, lib, pkgs, inputs, ... }:
+
+with lib;
+
+{
+  networking = {
+    hostName = mkDefault "overwatch";
+    useDHCP = false;
+  };
+
+  imports = [
+    ../modules/malobeo_user.nix
+    ../modules/sshd.nix
+  ];
+
+  networking.firewall.allowedTCPPorts = [ 80 9080 9001 3100 ];
+
+  services.grafana = {
+    enable = true;
+    domain = "grafana.malobeo.org";
+    port = 2342;
+    addr = "127.0.0.1";
+  };
+
+  services.nginx = {
+    enable = true;
+    virtualHosts.${config.services.grafana.domain} = {
+      locations."/" = {
+          proxyPass = "http://127.0.0.1:${toString config.services.grafana.port}";
+          proxyWebsockets = true;
+
+      extraConfig = ''
+        proxy_set_header Host $host;
+      '';
+    };
+      };
+  };
+
+  services.prometheus = {
+    enable = true;
+    port = 9001;
+    exporters = {
+      node = {
+        enable = true;
+        enabledCollectors = [ "systemd" "processes" ];
+        port = 9002;
+      };
+    };
+
+    scrapeConfigs = [
+      {
+        job_name = "overwatch";
+        static_configs = [{
+          targets = [ "127.0.0.1:9002" ];
+        }];
+      }
+      {
+        job_name = "infradocs";
+        static_configs = [{
+          targets = [ "10.0.0.11:9002" ];
+        }];
+      }
+    ];
+  };
+
+  services.loki = {
+    enable = true;
+    configFile = ./loki.yaml;
+  };
+
+  services.promtail = {
+    enable = true;
+    configFile = import ../modules/malobeo/promtail_config.nix { 
+      lokiAddress = "10.0.0.13";
+      logNginx = false;
+      config = config;
+      pkgs = pkgs;
+    };
+  };
+
+  users.users.promtail.extraGroups = [ "nginx" "systemd-journal" ];
+
+
+
+  system.stateVersion = "22.11"; # Did you read the comment?
+}
+

machines/overwatch/loki.yaml

@@ -0,0 +1,60 @@
+auth_enabled: false
+
+server:
+  http_listen_port: 3100
+  grpc_listen_port: 9096
+  log_level: debug
+  grpc_server_max_concurrent_streams: 1000
+
+common:
+  instance_addr: 127.0.0.1
+  path_prefix: /tmp/loki
+  storage:
+    filesystem:
+      chunks_directory: /tmp/loki/chunks
+      rules_directory: /tmp/loki/rules
+  replication_factor: 1
+  ring:
+    kvstore:
+      store: inmemory
+
+query_range:
+  results_cache:
+    cache:
+      embedded_cache:
+        enabled: true
+        max_size_mb: 100
+
+schema_config:
+  configs:
+    - from: 2020-10-24
+      store: tsdb
+      object_store: filesystem
+      schema: v13
+      index:
+        prefix: index_
+        period: 24h
+
+pattern_ingester:
+  enabled: true
+  metric_aggregation:
+    loki_address: localhost:3100
+
+ruler:
+  alertmanager_url: http://localhost:9093
+
+frontend:
+  encoding: protobuf
+
+# By default, Loki will send anonymous, but uniquely-identifiable usage and configuration
+# analytics to Grafana Labs. These statistics are sent to https://stats.grafana.org/
+#
+# Statistics help us better understand how Loki is used, and they show us performance
+# levels for most users. This helps us prioritize features and documentation.
+# For more information on what's sent, look at
+# https://github.com/grafana/loki/blob/main/pkg/analytics/stats.go
+# Refer to the buildReport method to see what goes into a report.
+#
+# If you would like to disable reporting, uncomment the following lines:
+analytics:
+  reporting_enabled: false

machines/overwatch/promtail.yaml

@@ -0,0 +1,29 @@
+server:
+  http_listen_port: 9080
+  grpc_listen_port: 0
+
+positions:
+  filename: /tmp/positions.yaml
+
+clients:
+  - url: http://10.0.0.13:3100/loki/api/v1/push
+
+  
+scrape_configs:
+  - job_name: journal
+    journal:
+      max_age: 12h
+      labels:
+        job: systemd-journal
+        host: overwatch
+    relabel_configs:
+      - source_labels: ["__journal__systemd_unit"]
+        target_label: "unit"
+  - job_name: nginx
+    static_configs:
+    - targets:
+        - localhost
+      labels:
+        job: nginx
+        __path__: /var/log/nginx/*log
+

machines/uptimekuma/configuration.nix

@@ -6,7 +6,6 @@ with lib;
   networking = {
     hostName = mkDefault "uptimekuma";
     useDHCP = false;
-    nameservers = [ "1.1.1.1" ];
   };
 
   imports = [

outputs.nix

@@ -20,7 +20,6 @@ in (utils.lib.eachSystem (builtins.filter filter_system utils.lib.defaultSystems
     let
       sops = sops-nix.packages."${pkgs.system}";
       microvmpkg = microvm.packages."${pkgs.system}";
-      installed = builtins.attrNames self.packages."${pkgs.system}".scripts;
     in
     pkgs.mkShell {
       sopsPGPKeyDirs = [
@@ -38,13 +37,11 @@ in (utils.lib.eachSystem (builtins.filter filter_system utils.lib.defaultSystems
         pkgs.mdbook
         microvmpkg.microvm
       ];
-      packages = builtins.map (pkgName: self.packages."${pkgs.system}".scripts.${pkgName}) installed;
-      shellHook = ''echo "Available scripts: ${builtins.concatStringsSep " " installed}"'';
     };
 
     packages = {
-      scripts.remote-install = pkgs.writeShellScriptBin "remote-install" (builtins.readFile ./scripts/remote-install-encrypt.sh);
+      remote-install = pkgs.writeShellScriptBin "remote-install" (builtins.readFile ./scripts/remote-install-encrypt.sh);
-      scripts.boot-unlock = pkgs.writeShellScriptBin "boot-unlock" (builtins.readFile ./scripts/unlock-boot.sh);
+      boot-unlock = pkgs.writeShellScriptBin "boot-unlock" (builtins.readFile ./scripts/unlock-boot.sh);
       docs = pkgs.stdenv.mkDerivation {
         name = "malobeo-docs";
         phases = [ "buildPhase" ];
@@ -81,11 +78,6 @@ in (utils.lib.eachSystem (builtins.filter filter_system utils.lib.defaultSystems
                 source = "/nix/store";
                 mountPoint = "/nix/.ro-store";
               }];
-              interfaces = pkgs.lib.mkForce [{
-                type = "user";
-                id = "eth0";
-                mac = "02:23:de:ad:be:ef";
-              }];
             };
             boot.isContainer = pkgs.lib.mkForce false;
             users.users.root.password = "";

scripts/remote-install-encrypt.sh

@@ -1,4 +1,5 @@
 set -o errexit
+set -o nounset
 set -o pipefail
 
 if [ $# -lt 2 ]; then
@@ -8,21 +9,6 @@ if [ $# -lt 2 ]; then
   exit 1
 fi
 
-if [ ! -e flake.nix ]
-    then
-    echo "flake.nix not found. Searching down."
-    while [ ! -e flake.nix ]
-        do
-        if [ $PWD = "/" ]
-            then
-            echo "Found root. Aborting."
-            exit 1
-            else
-            cd ..
-        fi
-    done
-fi
-
 hostname=$1
 ipaddress=$2
 

scripts/unlock-boot.sh

@@ -4,33 +4,19 @@ set -o pipefail
 sshoptions="-o StrictHostKeyChecking=no -o ServerAliveInterval=1 -o ServerAliveCountMax=1 -p 222 -T"
 HOSTNAME=$1
 
-if [ ! -e flake.nix ]
-    then
-    echo "flake.nix not found. Searching down."
-    while [ ! -e flake.nix ]
-        do
-        if [ $PWD = "/" ]
-            then
-            echo "Found root. Aborting."
-            exit 1
-            else
-            cd ..
-        fi
-    done
-fi
-
 echo
+diskkey=$(sops -d machines/$HOSTNAME/disk.key)
+
 if [ $# = 1 ]
     then
-    diskkey=$(sops -d machines/$HOSTNAME/disk.key)
     echo "$diskkey" | ssh $sshoptions root@$HOSTNAME-initrd "systemd-tty-ask-password-agent" #storage
 
     echo "$diskkey" | ssh $sshoptions root@$HOSTNAME-initrd "systemd-tty-ask-password-agent" #root
 
 elif [ $# = 2 ]
     then
-    diskkey=$(sops -d machines/$HOSTNAME/disk.key)
     IP=$2
+    
     echo "$diskkey" | ssh $sshoptions root@$IP "systemd-tty-ask-password-agent" #storage
 
     echo "$diskkey" | ssh $sshoptions root@$IP "systemd-tty-ask-password-agent" #root