[scripts] only need to unlock once

currently only allows forwarding to port 80, i was to lazy to handle two
arguments in bash

doc/src/anleitung/microvm.md

@@ -12,13 +12,31 @@ Use durruti as orientation:
 "10.0.0.5" is the IP assigned to its tap interface.  
 
 ### Testing MicroVMs locally
-MicroVMs can be built and run easily on your local host, but they are not persistent!
+MicroVMs can be built and run easily on your localhost for development.  
-For durruti for example this is done by:
+We provide the script ```run-vm``` to handle stuff like development (dummy) secrets, sharing directories, ect. easily.
+Usage examples:
 ``` bash
-nix run .\#durruti-vm
+# run without args to get available options and usage info
+run-vm
+
+# run nextcloud locally with dummy secrets
+run-vm nextcloud --dummy-secrets
+
+# share a local folder as /var/lib dir so that nextcloud application data stays persistent between boots
+mkdir /tmp/nextcloud
+run-vm nextcloud --dummy-secrets --varlib /tmp/nextcloud
+
+# enable networking to provide connectivity between multiple vms
+# for that the malobeo hostBridge must be enabled on your host
+# this example deploys persistent grafana on overwatch and fetches metrics from infradocs
+mkdir overwatch
+run-vm overwatch --networking --varlib /tmp/overwatch
+run-vm infradocs --networking
 ```
 
-### Testing persistent microvms
+
+
+### Fully deploy microvms on local host
 In order to test persistent microvms locally we need to create them using the ```microvm``` command.  
 This is necessary to be able to mount persistent /etc and /var volumes on those hosts.  
 Do the following:

flake.lock

@@ -341,11 +341,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1736184101,
+        "lastModified": 1737548421,
-        "narHash": "sha256-HAX+TkDXzyNp6SAsKwjNFql7KzAtxximpQSv+GmP8KQ=",
+        "narHash": "sha256-gmlqJdC+v86vXc2yMhiza1mvsqh3vMfrEsiw+tV5MXg=",
         "ref": "refs/heads/master",
-        "rev": "9cdab949f44301553e3817cf1f38287ad947e00c",
+        "rev": "c5fff78c83959841ac724980a13597dcfa6dc26d",
-        "revCount": 28,
+        "revCount": 29,
         "type": "git",
         "url": "https://git.dynamicdiscord.de/kalipso/tasklist"
       },

machines/.sops.yaml

@@ -11,6 +11,7 @@ keys:
   - &machine_durruti age1xu6kxpf8p0r8d6sgyl0m20p5hmw35nserl7rejuzm66eql0ur4mq03u0vp
   - &machine_vpn age1v6uxwej4nlrpfanr9js7x6059mtvyg4fw50pzt0a2kt3ahk7edlslafeuh
   - &machine_fanny age14dpm6vaycd6u34dkndcktpamqgdyj4aqccjnl5533dsza05hxuds0tjfnf
+  - &machine_nextcloud age1w07s4y2uh0xd322ralyyh79545lvxzqncd0s65q9cx4ttlqv5u9s7y78gr
   #this dummy key is used for testing.
   - &machine_dummy age18jn5mrfs4gqrnv0e2sxsgh3kq4sgxx39hwr8z7mz9kt7wlgaasjqlr88ng
 creation_rules:
@@ -84,12 +85,18 @@ creation_rules:
       - *admin_kalipso_dsktp
       age:
       - *admin_atlan
-
   - path_regex: nextcloud/secrets.yaml$
     key_groups:
     - pgp:
       - *admin_kalipso
       - *admin_kalipso_dsktp
-      - *machine_durruti
       age:
-      - *admin_atlan
+      - *admin_atlan
+      - *machine_nextcloud
+  - path_regex: overwatch/secrets.yaml$
+    key_groups:
+    - pgp:
+      - *admin_kalipso
+      - *admin_kalipso_dsktp
+      age:
+      - *admin_atlan

machines/configuration.nix

@@ -1,189 +0,0 @@
-{ self
-, nixpkgs-unstable
-, nixpkgs
-, sops-nix
-, inputs
-, microvm
-, nixos-hardware
-, home-manager
-, ...
-}:
-let
-  nixosSystem = nixpkgs.lib.makeOverridable nixpkgs.lib.nixosSystem;
-  nixosSystemUnstable = nixpkgs-unstable.lib.makeOverridable nixpkgs-unstable.lib.nixosSystem;
-
-  baseModules = [
-    # make flake inputs accessiable in NixOS
-    { _module.args.inputs = inputs; }
-    {
-      imports = [
-        ({ pkgs, ... }: {
-          nix = {
-            extraOptions = ''
-              experimental-features = nix-command flakes
-            '';
-
-            settings = {
-              substituters = [
-                "https://cache.dynamicdiscord.de"
-                "https://cache.nixos.org/"
-              ];
-              trusted-public-keys = [
-                "cache.dynamicdiscord.de:DKueZicqi2NhJJXz9MYgUbiyobMs10fTyHCgAUibRP4="
-              ];
-              trusted-users = [ "root" "@wheel" ];
-            };
-          };
-        })
-        sops-nix.nixosModules.sops
-        microvm.nixosModules.microvm
-      ];
-    }
-  ];
-  defaultModules = baseModules;
-
-  makeMicroVM = hostName: ipv4Addr: macAddr: modules: [
-    {
-      microvm = {
-        hypervisor = "cloud-hypervisor";
-        mem = 2560;
-        shares = [
-          {
-            source = "/nix/store";
-            mountPoint = "/nix/.ro-store";
-            tag = "store";
-            proto = "virtiofs";
-            socket = "store.socket";
-          }
-          {
-            source = "/var/lib/microvms/${hostName}/etc";
-            mountPoint = "/etc";
-            tag = "etc";
-            proto = "virtiofs";
-            socket = "etc.socket";
-          }
-          {
-            source = "/var/lib/microvms/${hostName}/var";
-            mountPoint = "/var";
-            tag = "var";
-            proto = "virtiofs";
-            socket = "var.socket";
-          }
-        ];
-
-        interfaces = [
-          {
-            type = "tap";
-            id = "vm-${hostName}";
-            mac = "${macAddr}";
-          }
-        ];
-      };
-      
-      systemd.network.enable = true;
-      
-      systemd.network.networks."20-lan" = {
-        matchConfig.Type = "ether";
-        networkConfig = {
-          Address = [ "${ipv4Addr}/24" ];
-          Gateway = "10.0.0.1";
-          DNS = ["1.1.1.1"];
-          DHCP = "no";
-        };
-      };
-    }
-  ] ++ defaultModules ++ modules;
-
-  inputsMod = inputs // { malobeo = self; };
-in
-{
-  louise = nixosSystem {
-    system = "x86_64-linux";
-    specialArgs.inputs = inputs;
-    modules = defaultModules ++ [
-      ./louise/configuration.nix
-    ];
-  };
-
-  bakunin = nixosSystem {
-    system = "x86_64-linux";
-    specialArgs.inputs = inputs;
-    modules = defaultModules ++ [
-      ./bakunin/configuration.nix
-      inputs.disko.nixosModules.disko
-    ];
-  };
-
-  lucia = nixosSystem {
-    system = "aarch64-linux";
-    specialArgs.inputs = inputs;
-    modules = defaultModules ++ [
-      ./lucia/configuration.nix
-      ./lucia/hardware_configuration.nix
-    ];
-  };
-
-  fanny = nixosSystem {
-    system = "x86_64-linux";
-    specialArgs.inputs = inputsMod;
-    modules = defaultModules ++ [
-      self.nixosModules.malobeo.vpn
-      ./fanny/configuration.nix
-    ];
-  };
-
-  durruti = nixosSystem {
-    system = "x86_64-linux";
-    specialArgs.inputs = inputs;
-    specialArgs.self = self;
-    modules = makeMicroVM "durruti" "10.0.0.5" "52:DA:0D:F9:EF:F9" [
-      ./durruti/configuration.nix
-    ];
-  };
-
-  vpn = nixosSystem {
-    system = "x86_64-linux";
-    specialArgs.inputs = inputs;
-    specialArgs.self = self;
-    modules = makeMicroVM "vpn" "10.0.0.10" "D0:E5:CA:F0:D7:E6" [
-      self.nixosModules.malobeo.vpn
-      ./vpn/configuration.nix
-    ];
-  };
-
-  infradocs = nixosSystem {
-    system = "x86_64-linux";
-    specialArgs.inputs = inputs;
-    specialArgs.self = self;
-    modules = makeMicroVM "infradocs" "10.0.0.11" "D0:E5:CA:F0:D7:E7" [
-      self.nixosModules.malobeo.vpn
-      ./infradocs/configuration.nix
-    ];
-  };
-
-  uptimekuma = nixosSystem {
-    system = "x86_64-linux";
-    specialArgs.inputs = inputs;
-    specialArgs.self = self;
-    modules = makeMicroVM "uptimekuma" "10.0.0.12" "D0:E5:CA:F0:D7:E8" [
-      ./uptimekuma/configuration.nix
-    ];
-  };
-
-  nextcloud = nixosSystem {
-    system = "x86_64-linux";
-    specialArgs.inputs = inputs;
-    specialArgs.self = self;
-    modules = makeMicroVM "nextcloud" "10.0.0.13" "D0:E5:CA:F0:D7:E9" [
-      ./nextcloud/configuration.nix
-    ];
-  };
-
-  testvm = nixosSystem {
-    system = "x86_64-linux";
-    specialArgs.inputs = inputs;
-    specialArgs.self = self;
-    modules = defaultModules ++ [ ./testvm ];
-  };
-
-}

machines/durruti/configuration.nix

@@ -1,4 +1,4 @@
-{ config, lib, pkgs, inputs, ... }:
+{ config, self, lib, pkgs, inputs, ... }:
 
 with lib;
 
@@ -6,7 +6,6 @@ with lib;
   networking = {
     hostName = mkDefault "durruti";
     useDHCP = false;
-    nameservers = [ "1.1.1.1" ];
   };
 
   networking.firewall.allowedTCPPorts = [ 8080 ];
@@ -18,15 +17,21 @@ with lib;
   ];
 
   imports = [
+    self.nixosModules.malobeo.metrics
     inputs.tasklist.nixosModules.malobeo-tasklist
 
-    ./documentation.nix
-
     ../modules/malobeo_user.nix
     ../modules/sshd.nix
     ../modules/minimal_tools.nix
   ];
 
+  malobeo.metrics = {
+    enable = true;
+    enablePromtail = true;
+    logNginx = true;
+    lokiHost = "10.0.0.14";
+  };
+
   services.malobeo-tasklist.enable = true;
 
   system.stateVersion = "22.11"; # Did you read the comment?

machines/durruti/host_config.nix

@@ -43,6 +43,36 @@ in
       };
     };
 
+    services.nginx.virtualHosts."cloud.malobeo.org" = {
+      forceSSL = true;
+      enableACME= true;
+      locations."/" = {
+        proxyPass = "http://10.0.0.10";
+        extraConfig = ''
+        '';
+      };
+    };
+
+    services.nginx.virtualHosts."grafana.malobeo.org" = {
+      forceSSL = true;
+      enableACME= true;
+      locations."/" = {
+        proxyPass = "http://10.0.0.10";
+        extraConfig = ''
+        '';
+      };
+    };
+
+    services.nginx.virtualHosts."tasklist.malobeo.org" = {
+      forceSSL = true;
+      enableACME= true;
+      locations."/" = {
+        proxyPass = "http://10.0.0.10";
+        extraConfig = ''
+        '';
+      };
+    };
+
     services.nginx.virtualHosts."status.malobeo.org" = {
       forceSSL = true;
       enableACME= true;
@@ -52,17 +82,5 @@ in
         '';
       };
     };
-
-    services.nginx.virtualHosts."tasklist.malobeo.org" = {
-      forceSSL = true;
-      enableACME= true;
-      locations."/".proxyPass = "http://${cfg.host_ip}:8080";
-    };
-
-    services.nginx.virtualHosts."booking.dynamicdiscord.de" = {
-      forceSSL = true;
-      enableACME= true;
-      locations."/".proxyPass = "http://${cfg.host_ip}:80";
-    };
   };
 }

machines/fanny/configuration.nix

@@ -13,11 +13,20 @@ in
       ../modules/sshd.nix
       ../modules/minimal_tools.nix
       ../modules/autoupdate.nix
+      inputs.self.nixosModules.malobeo.vpn
       inputs.self.nixosModules.malobeo.initssh
       inputs.self.nixosModules.malobeo.disko
       inputs.self.nixosModules.malobeo.microvm
+      inputs.self.nixosModules.malobeo.metrics
     ];
 
+  malobeo.metrics = {
+    enable = true;
+    enablePromtail = true;
+    logNginx = true;
+    lokiHost = "10.0.0.14";
+  };
+
   malobeo.autoUpdate = {
     enable = true;
     url = "https://hydra.dynamicdiscord.de";
@@ -26,7 +35,14 @@ in
     cacheurl = "https://cache.dynamicdiscord.de";
   };
 
-  nix.settings.experimental-features = [ "nix-command" "flakes" ];
+  nix = {
+    settings.experimental-features = [ "nix-command" "flakes" ];
+    #always update microvms
+    extraOptions = ''
+      tarball-ttl = 0
+    '';
+  };
+
 
   malobeo.disks = {
     enable = true;
@@ -53,9 +69,15 @@ in
   };
 
   services.malobeo.microvm.enableHostBridge = true;
-  services.malobeo.microvm.deployHosts = [ "infradocs" "nextcloud" ];
+  services.malobeo.microvm.deployHosts = [ "overwatch" "infradocs" "nextcloud" "durruti" ];
 
   networking = {
+    nat = {
+      enable = true;
+      externalInterface = "enp1s0";
+      internalInterfaces = [ "microvm" ];
+    };
+
     firewall = {
       allowedTCPPorts = [ 80 ];
     };
@@ -67,6 +89,7 @@ in
       locations."/" = {
         proxyPass = "http://10.0.0.11:9000";
         extraConfig = ''
+          proxy_set_header Host $host;
         '';
       };
     };
@@ -75,6 +98,25 @@ in
       locations."/" = {
         proxyPass = "http://10.0.0.13";
         extraConfig = ''
+          proxy_set_header Host $host;
+        '';
+      };
+    };
+
+    virtualHosts."grafana.malobeo.org" = {
+      locations."/" = {
+        proxyPass = "http://10.0.0.14";
+        extraConfig = ''
+          proxy_set_header Host $host;
+        '';
+      };
+    };
+
+    virtualHosts."tasklist.malobeo.org" = {
+      locations."/" = {
+        proxyPass = "http://10.0.0.5:8080";
+        extraConfig = ''
+          proxy_set_header Host $host;
         '';
       };
     };

machines/hosts.nix

@@ -0,0 +1,75 @@
+{ ... }:
+
+{
+  malobeo = {
+    hosts = {
+      louise = {
+        type = "host";
+      };
+
+      bakunin = {
+        type = "host";
+      };
+
+      fanny = {
+        type = "host";
+      };
+
+      lucia = {
+        type = "rpi";
+      };
+
+      durruti = {
+        type = "microvm";
+        network = {
+          address = "10.0.0.5";
+          mac = "52:DA:0D:F9:EF:F9";
+        };
+      };
+
+      vpn = {
+        type = "microvm";
+        network = {
+          address = "10.0.0.10";
+          mac = "D0:E5:CA:F0:D7:E6";
+        };
+      };
+
+      infradocs = {
+        type = "microvm";
+        network = {
+          address = "10.0.0.11";
+          mac = "D0:E5:CA:F0:D7:E7";
+        };
+      };
+
+      uptimekuma = {
+        type = "microvm";
+        network = {
+          address = "10.0.0.12";
+          mac = "D0:E5:CA:F0:D7:E8";
+        };
+      };
+
+      nextcloud = {
+        type = "microvm";
+        network = {
+          address = "10.0.0.13";
+          mac = "D0:E5:CA:F0:D7:E9";
+        };
+      };
+
+      overwatch = {
+        type = "microvm";
+        network = {
+          address = "10.0.0.14";
+          mac = "D0:E5:CA:F0:D7:E0";
+        };
+      };
+
+      testvm = {
+        type = "host";
+      };
+    };
+  };
+}

machines/infradocs/configuration.nix

@@ -1,4 +1,4 @@
-{ config, lib, pkgs, inputs, ... }:
+{ self, config, lib, pkgs, inputs, ... }:
 
 with lib;
 
@@ -6,15 +6,22 @@ with lib;
   networking = {
     hostName = mkDefault "infradocs";
     useDHCP = false;
-    nameservers = [ "1.1.1.1" ];
   };
 
   imports = [
+    inputs.malobeo.nixosModules.malobeo.metrics
     ../durruti/documentation.nix
     ../modules/malobeo_user.nix
     ../modules/sshd.nix
   ];
 
+  malobeo.metrics = {
+    enable = true;
+    enablePromtail = true;
+    logNginx = true;
+    lokiHost = "10.0.0.14";
+  };
+
   system.stateVersion = "22.11"; # Did you read the comment?
 }
 

machines/lucia/configuration.nix

@@ -6,6 +6,7 @@ in
 {
   imports =
     [ # Include the results of the hardware scan.
+      ./hardware_configuration.nix
       ../modules/malobeo_user.nix
     ];
 

machines/modules/disko/btrfs-laptop.nix

@@ -1,63 +0,0 @@
-{ config, self, inputs, ... }:
-
-{
-  imports = [
-    inputs.disko.nixosModules.disko
-  ];
-
-  # https://github.com/nix-community/disko/blob/master/example/luks-btrfs-subvolumes.nix
-  disko.devices = {
-    disk = {
-      main = {
-        type = "disk";
-        # When using disko-install, we will overwrite this value from the commandline
-        device = "/dev/disk/by-id/some-disk-id";
-        content = {
-          type = "gpt";
-          partitions = {
-            ESP = {
-              size = "512M";
-              type = "EF00";
-              content = {
-                type = "filesystem";
-                format = "vfat";
-                mountpoint = "/boot";
-                mountOptions = [ "umask=0077" ];
-              };
-            };
-            luks = {
-              size = "100%";
-              content = {
-                type = "luks";
-                name = "crypted";
-                passwordFile = /tmp/secret.key; # Interactive
-                content = {
-                  type = "btrfs";
-                  extraArgs = [ "-f" ];
-                  subvolumes = {
-                    "/root" = {
-                      mountpoint = "/";
-                      mountOptions = [ "compress=zstd" "noatime" ];
-                    };
-                    "/home" = {
-                      mountpoint = "/home";
-                      mountOptions = [ "compress=zstd" "noatime" ];
-                    };
-                    "/nix" = {
-                      mountpoint = "/nix";
-                      mountOptions = [ "compress=zstd" "noatime" ];
-                    };
-                    "/swap" = {
-                      mountpoint = "/.swapvol";
-                      swap.swapfile.size = "20M";
-                    };
-                  };
-                };
-              };
-            };
-          };
-        };
-      };
-    };
-  };
-}

machines/modules/disko/default.nix

@@ -187,6 +187,7 @@ in
               postCreateHook = lib.mkIf cfg.encryption ''
                 zfs set keylocation="prompt" zroot/encrypted;
               '';
+
             };
             "encrypted/root" = {
               type = "zfs_fs";
@@ -244,13 +245,12 @@ in
               };
               # use this to read the key during boot
               postCreateHook = lib.mkIf cfg.encryption ''
-                zfs set keylocation="prompt" storage/encrypted;
+                zfs set keylocation="file:///root/secret.key" storage/encrypted;
               '';
             };
             "encrypted/data" = {
               type = "zfs_fs";
               mountpoint = "/data";
-              options.mountpoint = "legacy";
             };
             reserved = {
               # for cow delete if pool is full
@@ -267,7 +267,7 @@ in
     };
 
     boot.zfs.devNodes = lib.mkDefault cfg.devNodes;
-
+    boot.zfs.extraPools = lib.mkIf cfg.storage.enable [ "storage" ];
     fileSystems."/".neededForBoot = true;
     fileSystems."/etc".neededForBoot = true;
     fileSystems."/boot".neededForBoot = true;

machines/modules/host_builder.nix

@@ -0,0 +1,243 @@
+{ self
+, nixpkgs-unstable
+, nixpkgs
+, sops-nix
+, inputs
+, hosts
+, ...
+}:
+let
+  pkgs = nixpkgs.legacyPackages."x86_64-linux";
+in
+rec {
+  nixosSystem = nixpkgs.lib.makeOverridable nixpkgs.lib.nixosSystem;
+  nixosSystemUnstable = nixpkgs-unstable.lib.makeOverridable nixpkgs-unstable.lib.nixosSystem;
+
+  baseModules = [
+    # make flake inputs accessiable in NixOS
+    { _module.args.inputs = inputs; }
+    {
+      imports = [
+        ({ pkgs, ... }: {
+          nix = {
+            extraOptions = ''
+              experimental-features = nix-command flakes
+            '';
+
+            settings = {
+              substituters = [
+                "https://cache.dynamicdiscord.de"
+                "https://cache.nixos.org/"
+              ];
+              trusted-public-keys = [
+                "cache.dynamicdiscord.de:DKueZicqi2NhJJXz9MYgUbiyobMs10fTyHCgAUibRP4="
+              ];
+              trusted-users = [ "root" "@wheel" ];
+            };
+          };
+        })
+
+        sops-nix.nixosModules.sops
+      ];
+    }
+  ];
+  defaultModules = baseModules;
+
+  makeMicroVM = hostName: ipv4Addr: macAddr: modules: [
+    {
+      microvm = {
+        hypervisor = "cloud-hypervisor";
+        mem = 2560;
+        shares = [
+          {
+            source = "/nix/store";
+            mountPoint = "/nix/.ro-store";
+            tag = "store";
+            proto = "virtiofs";
+            socket = "store.socket";
+          }
+          {
+            source = "/var/lib/microvms/${hostName}/etc";
+            mountPoint = "/etc";
+            tag = "etc";
+            proto = "virtiofs";
+            socket = "etc.socket";
+          }
+          {
+            source = "/var/lib/microvms/${hostName}/var";
+            mountPoint = "/var";
+            tag = "var";
+            proto = "virtiofs";
+            socket = "var.socket";
+          }
+        ];
+
+        interfaces = [
+          {
+            type = "tap";
+            id = "vm-${hostName}";
+            mac = "${macAddr}";
+          }
+        ];
+      };
+
+      systemd.network.enable = true;
+      
+      systemd.network.networks."20-lan" = {
+        matchConfig.Type = "ether";
+        networkConfig = {
+          Address = [ "${ipv4Addr}/24" ];
+          Gateway = "10.0.0.1";
+          DNS = ["1.1.1.1"];
+          DHCP = "no";
+        };
+      };
+    }
+  ] ++ defaultModules ++ modules;
+
+  inputsMod = inputs // { malobeo = self; };
+
+
+    vmMicroVMOverwrites = hostname: options: {
+      microvm = rec {
+        mem = pkgs.lib.mkForce 4096;
+        hypervisor = pkgs.lib.mkForce "qemu";
+        socket = pkgs.lib.mkForce null;
+
+
+        #needed for hosts that deploy imperative microvms (for example fanny)
+        writableStoreOverlay = pkgs.lib.mkIf options.writableStore "/nix/.rw-store";
+        volumes = pkgs.lib.mkIf options.writableStore [ {
+          image = "nix-store-overlay.img";
+          mountPoint = writableStoreOverlay;
+          size = 2048;
+        } ];
+
+        shares = pkgs.lib.mkForce (pkgs.lib.optionals (!options.writableStore) [
+          {
+            tag = "ro-store";
+            source = "/nix/store";
+            mountPoint = "/nix/.ro-store";
+          }
+        ] ++ pkgs.lib.optionals (options.varPath != "") [
+          {
+            source = "${options.varPath}";
+            securityModel = "mapped";
+            mountPoint = "/var";
+            tag = "var";
+          }
+        ]);
+
+        interfaces = pkgs.lib.mkIf (!options.withNetworking) (pkgs.lib.mkForce [{
+          type = "user";
+          id = "eth0";
+          mac = "02:23:de:ad:be:ef";
+        }]);
+
+        #if networking is disabled forward port 80 to still have access to webservices
+        forwardPorts = pkgs.lib.mkIf (!options.withNetworking && options.fwdPort != 0) (pkgs.lib.mkForce [
+          { from = "host"; host.port = options.fwdPort; guest.port = 80; }
+        ]);
+
+      };
+
+      fileSystems = {
+        "/".fsType = pkgs.lib.mkForce "tmpfs";
+
+        # prometheus uses a memory mapped file which doesnt seem supported by 9p shares
+        # therefore we mount a tmpfs inside the datadir
+        "/var/lib/prometheus2/data" = pkgs.lib.mkIf (hostname == "overwatch" && options.varPath != "") (pkgs.lib.mkForce {
+          fsType = pkgs.lib.mkForce "tmpfs";
+        });
+      };
+
+      boot.isContainer = pkgs.lib.mkForce false;
+      services.timesyncd.enable = false;
+      users.users.root.password = "";
+      services.getty.helpLine = ''
+        Log in as "root" with an empty password.
+        Use "reboot" to shut qemu down.
+      '';
+    };
+
+    vmDiskoOverwrites = {
+      boot.initrd = {
+        secrets = pkgs.lib.mkForce {};
+        network.ssh.enable = pkgs.lib.mkForce false;
+      };
+
+      malobeo.disks.enable = pkgs.lib.mkForce false;
+      networking.hostId = "a3c3101f";
+    };
+
+    vmSopsOverwrites = host: {
+      sops.defaultSopsFile = pkgs.lib.mkForce ../${host}/dummy.yaml;
+
+      environment.etc = {
+        devHostKey = {
+          source = ../secrets/devkey_ed25519;
+          mode = "0600";
+        };
+      };
+
+      services.openssh.hostKeys = [{
+        path = "/etc/devHostKey";
+        type = "ed25519";
+      }];
+    };
+
+    vmNestedMicroVMOverwrites = host: sopsDummy: {
+
+      services.malobeo.microvm.deployHosts = pkgs.lib.mkForce [];
+      microvm.vms = 
+      let
+        # Map the values to each hostname to then generate an Attrset using listToAttrs
+        mapperFunc = name: { inherit name; value = {
+          specialArgs.inputs = inputsMod;
+          specialArgs.self = self;
+          config = {
+            imports = (makeMicroVM "${name}" 
+                                   "${hosts.malobeo.hosts.${name}.network.address}"
+                                   "${hosts.malobeo.hosts.${name}.network.mac}" [
+              ../${name}/configuration.nix     
+              (vmMicroVMOverwrites name { 
+                withNetworking = true; 
+                varPath = ""; 
+                writableStore = false; })
+              (if sopsDummy then (vmSopsOverwrites name) else {})
+            ]);
+          };
+        }; };
+      in
+      builtins.listToAttrs (map mapperFunc self.nixosConfigurations.${host}.config.services.malobeo.microvm.deployHosts);
+    };
+
+    buildVM = host: networking: sopsDummy: disableDisko: varPath: writableStore: fwdPort: (self.nixosConfigurations.${host}.extendModules {
+        modules = [
+            (vmMicroVMOverwrites host { 
+              withNetworking = networking; 
+              varPath = "${varPath}"; 
+              writableStore = writableStore;
+              fwdPort = fwdPort; })
+            (if sopsDummy then (vmSopsOverwrites host) else {})
+            (if disableDisko then vmDiskoOverwrites else {})
+          ] ++ pkgs.lib.optionals (hosts.malobeo.hosts.${host}.type != "microvm") [
+            inputs.microvm.nixosModules.microvm
+          ] ++ pkgs.lib.optionals (self.nixosConfigurations.${host}.config ? services.malobeo.microvm.deployHosts) [
+            (vmNestedMicroVMOverwrites host sopsDummy)
+        ];
+      });
+
+  buildHost = hosts: (builtins.mapAttrs (host: settings: nixosSystem {
+    system = if (settings.type == "rpi") then "aarch64-linux" else "x86_64-linux";
+    specialArgs.inputs = inputsMod;
+    specialArgs.self = self;
+    modules = (if (settings.type != "microvm") then
+      defaultModules ++ [ ../${host}/configuration.nix ]
+      else
+      makeMicroVM "${host}" "${settings.network.address}" "${settings.network.mac}" [
+        inputs.microvm.nixosModules.microvm
+        ../${host}/configuration.nix
+      ]);
+  }) hosts);
+}

machines/modules/malobeo/initssh.nix

@@ -30,9 +30,7 @@ in
       loader.efi.canTouchEfiVariables = true;
       supportedFilesystems = [ "vfat" "zfs" ];
       zfs = {
-        forceImportAll = true;
         requestEncryptionCredentials = true;
-        
       };
       initrd = {
         availableKernelModules = cfg.ethernetDrivers;

machines/modules/malobeo/metrics.nix

@@ -0,0 +1,56 @@
+{ config, lib, pkgs, ... }:
+let 
+  cfg = config.malobeo.metrics;
+in
+{
+  options.malobeo.metrics = {
+    enable = lib.mkOption {
+      type = lib.types.bool;
+      default = false;
+      description = "Enable sharing metrics";
+    };
+    enablePromtail = lib.mkOption {
+      type = lib.types.bool;
+      default = true;
+      description = "Enable sharing logs";
+    };
+    logNginx = lib.mkOption {
+      type = lib.types.bool;
+      default = false;
+      description = "Share nginx logs";
+    };
+    lokiHost = lib.mkOption {
+      type = lib.types.str;
+      default = "10.0.0.14";
+      description = "Address of loki host";
+    };
+  };
+  
+  config = lib.mkIf (cfg.enable) {
+
+    networking.firewall.allowedTCPPorts = [ 9002 ];
+
+    services.prometheus = {
+      exporters = {
+        node = {
+          enable = true;
+          enabledCollectors = [ "systemd" "processes" ];
+          port = 9002;
+        };
+      };
+    };
+
+    services.promtail = {
+      enable = cfg.enablePromtail;
+      configFile = import ./promtail_config.nix { 
+        lokiAddress = cfg.lokiHost;
+        logNginx = cfg.logNginx;
+        config = config;
+        pkgs = pkgs;
+      };
+    };
+
+    users.users.promtail.extraGroups = [ "systemd-journal" ] ++ (lib.optionals cfg.logNginx [ "nginx" ]) ;
+
+  };
+}

machines/modules/malobeo/promtail_config.nix

@@ -0,0 +1,49 @@
+{ logNginx, lokiAddress, config, pkgs, ... }:
+
+let
+  basecfg = ''
+    server:
+      http_listen_port: 9080
+      grpc_listen_port: 0
+    
+    positions:
+      filename: /tmp/positions.yaml
+    
+    clients:
+      - url: http://${lokiAddress}:3100/loki/api/v1/push
+  '';
+
+  withNginx = ''
+    scrape_configs:
+      - job_name: journal
+        journal:
+          max_age: 12h
+          labels:
+            job: systemd-journal
+            host: ${config.networking.hostName}
+        relabel_configs:
+          - source_labels: ["__journal__systemd_unit"]
+            target_label: "unit"
+      - job_name: nginx
+        static_configs:
+        - targets:
+            - localhost
+          labels:
+            job: nginx
+            __path__: /var/log/nginx/*log
+  '';
+
+  withoutNginx = ''
+    scrape_configs:
+      - job_name: journal
+        journal:
+          max_age: 12h
+          labels:
+            job: systemd-journal
+            host: ${config.networking.hostName}
+        relabel_configs:
+          - source_labels: ["__journal__systemd_unit"]
+            target_label: "unit"
+  '';
+in
+pkgs.writeText "promtailcfg.yaml" (if logNginx then ''${basecfg}${withNginx}'' else ''${basecfg}${withoutNginx}'')

machines/nextcloud/configuration.nix

@@ -1,4 +1,4 @@
-{ config, lib, pkgs,  ... }:
+{ config, self, lib, pkgs,  ... }:
 
 with lib;
 
@@ -17,12 +17,20 @@ with lib;
   };
 
   imports = [
+    self.nixosModules.malobeo.metrics
     ../modules/malobeo_user.nix
     ../modules/sshd.nix
     ../modules/minimal_tools.nix
     ../modules/autoupdate.nix
   ];
 
+  malobeo.metrics = {
+    enable = true;
+    enablePromtail = true;
+    logNginx = true;
+    lokiHost = "10.0.0.14";
+  };
+
   services.nextcloud = {
     enable = true;
     package = pkgs.nextcloud30;
@@ -38,7 +46,7 @@ with lib;
     };
     extraAppsEnable = true;
     extraApps = {
-      inherit (config.services.nextcloud.package.packages.apps) contacts calendar;
+      inherit (config.services.nextcloud.package.packages.apps) contacts calendar deck polls;
       collectives = pkgs.fetchNextcloudApp {
         sha256 = "sha256-cj/8FhzxOACJaUEu0eG9r7iAQmnOG62yFHeyUICalFY=";
         url = "https://github.com/nextcloud/collectives/releases/download/v2.15.2/collectives-2.15.2.tar.gz";

machines/nextcloud/secrets.yaml

@@ -8,72 +8,61 @@ sops:
         - recipient: age1ljpdczmg5ctqyeezn739hv589fwhssjjnuqf7276fqun6kc62v3qmhkd0c
           enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBmVGxsNmZ3Z0RIYmMyL0Mr
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBqSk9GWktrZ3FsRHpOcTJp
-            UUpaMEZLTCtQaGFrL1YwOVBicEtNRTVaVGhRCmhDSUgxYXpRcldaMngvOWJDdnNo
+            Y3VWMytTRlhxVXJma1puT1lMRTN2NHBNV2xrCi8xYTFWeVN6RWl0Um9mZXpoKzFh
-            b2ZFbUdmcE9EV2E3SkMvZ1RpKzZmeU0KLS0tIE5hNmVFTXpBZFZ3bHYwQlJQaUtw
+            SjVFcGJRNlhkVUZQYXpEb0EwYzUvUjQKLS0tIGEvdGdMRGxvcndxMllZTWZqKzg1
-            UFJmTVFaOTJXN09QLzY4emh5Z3hqRjAKXk1PSwR2x0H2cMN06fyigiusz8v2IRIg
+            aWlJOTdYV1JMM0dIWEFDSHRuQWdlcVUKsdwGZ3SkJEf4ALDhHUlSQJNKrFyWd7fW
-            S4ZTq/JX39U4QQHgWA1dFPfC636LNBo+QKdl/2mjwnXW7duqDJ+5kA==
+            WTGk66NJ2yD8ko/6OyB9J9U0WPbFLgr972H+klBq/IDmOx0hClbYNA==
+            -----END AGE ENCRYPTED FILE-----
+        - recipient: age1w07s4y2uh0xd322ralyyh79545lvxzqncd0s65q9cx4ttlqv5u9s7y78gr
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBoNzdib3Ztd0g0MlVqYVF6
+            cUtjZzEyY2FJYVRoT1p5RlJwYVQwUXVOUkNVCkp4V3hMYlJsaVN4RjlwQXNWS1Jt
+            aitzWVdOcUdrNHorenZGZU1iWFZzVjgKLS0tIGNGcTU5OUJLM3VzQk1uODFwS1hO
+            WG16Y25tMDkreGFnSFRKN1AybyttYWcKcLHJScp2Ozh0jIdi7Hb/tSjaCGorqXaC
+            9DIrQPHbPP1RIc6Ak8Kn30/BHEWV3VaiBCT3vfS9pNJQNjB4T+901g==
             -----END AGE ENCRYPTED FILE-----
     lastmodified: "2024-11-26T20:00:50Z"
     mac: ENC[AES256_GCM,data:qoY9SfpoU+8HfvD5v/1S6BOkbnZUmHIbtwr0tTSuPETjnFNgr1VVw9mnRatJKPYYFb9/rMZQWIqTY+iUIEkcTVyVXhd6ki5CHW+uxCeBIyMzq33rtEa/btkEUoii4iPieamBCIY21W0znE+edxfR04yRJtLxMICEbuW4Hjf6bwk=,iv:nG42fRgjpuIjPMYnn/6egEdzYolcUBsspaZ8zMv4888=,tag:C6apGoAvVLsWdLWSCwrx6w==,type:str]
     pgp:
-        - created_at: "2024-11-26T19:59:36Z"
+        - created_at: "2025-01-21T21:04:08Z"
           enc: |-
             -----BEGIN PGP MESSAGE-----
 
-            hQGMA5HdvEwzh/H7AQv+KkX46UGQzLvhrk/VUCnnMdLEcNbYfk4h+sZJzs1riOGA
+            hQGMA5HdvEwzh/H7AQv/ejIylIgs3yeVcZriQTA8d/xyXTdFw6On422lTCDk3d0W
-            LAKYNeaeN6iLLeZX+T2/s5OT4WkIEKGg8/gziurdx01BR70M96Faubp6EVtdK44+
+            GOdV44vAzUzNX5tziQtLjectLUrKh9Qb9WaP4VnTCGI0XJ/dEtYRCkYMx8MjjbLl
-            6F5BLLrDhlEKDNOx48qPwJdFjbYW4wZLWmv5nzwPmmRCKO7MoI9UHKq69msCor1i
+            8GqFi3Hw958Uykp9wt0iiP6BQ42Fo77EPxVcn21eHKZY0zg/vaeRXXeXSzkjzANs
-            ralbjlVHyKSuRfvflKAlxFoEqeB6H+ryc54g3stk1j2eFiMNuF/oKDJZT+XI5LHZ
+            NN/KFS06uFRJhmp+0z6hDRrHnpb0wd5JGjHOp96jK9LmpwfZZZlVpAHp04hOhlPV
-            Ai80DAWoUBYgpP4aWiNC075GPutdPlZ3mrGf5+7QnNm7GmNUdJN5VAWmI2NUGr1J
+            cMmdjg9IRSubvbraTbDrgwB0h3JKdqovFDnAP/KvT+rw5xnVUVMq/3tUNq4MbfZb
-            BLopnPFo4juWNsZkLMj2aAuKvGTkhz2PuFKfLj6Erpu82RAjadpFWx239n+i4Ryq
+            CvQrXsjQJQbEhY+eAJZVRO07kX0+zMvIin4ss7Xt++qlo4/OvFvuGbnUhJE+hrBb
-            wSquYshpuiecLEejntTBKLEacwp+aPx8IHKnOOKBTdJj+YYaISiznQAlkF7WS+lg
+            nkyGhbDrjpsfa3djCEZ0UxMAWtPeIQ7T8QMkGY+UKeJKxfOGSchARnfCtGD/rtsj
-            MTZR85BvCxiPogujL7uhYSx1wM5FVkuAIPf1JOJCRvQt30eRRrR0VMrmqQ1Kl5OT
+            wuhqGya7g7WP78WzwASzlPwB5jpdQ29/zLWXR60lNCYu0UYSVYmlspZnKEB0FkLO
-            VMzZRIGIoC5vrKGeIIjJ0lgBWQ3bYFh/LGrwKetku6TRAH29mp/XwQqBC97RsUYb
+            TNUrwXXMrM0XwMVaG/sF0lgBEPE6CTuE85evCHFyu6zhEAa7YimKAPIowcwYLSJ2
-            EOxft5sUWaYrXK+z2yzCxOQBWKJISPgcyhdoKfYGnRkHXHi2Uay84oQP4co72eVF
+            46KfttJAYnRnb68Kk9N5xcFyvhKyTx/6eMdxkgr2LMoSTBDUgZfG3rDQC+ZbFE3m
-            cAhEJOxMw36e
+            bUOvx3Ho80EC
-            =bSaN
+            =oQd6
             -----END PGP MESSAGE-----
           fp: c4639370c41133a738f643a591ddbc4c3387f1fb
-        - created_at: "2024-11-26T19:59:36Z"
+        - created_at: "2025-01-21T21:04:08Z"
           enc: |-
             -----BEGIN PGP MESSAGE-----
 
-            hQIMA98TrrsQEbXUARAAmG88ZDt43zj6dCJkYYVj7MGIhIviJzilTvX4+EfNobtA
+            hQIMA98TrrsQEbXUAQ//eu7YkPL7dU4AYWCZI7THsiJ51SOMahOXp/qC5yL18aZY
-            tll60GYfRotKnwbuqzSVaaIcV+6cDQ5I1hG5WNFJSXm7DpJ0W1Ir1x2hpxektXFa
+            r4SpyNhFezGIJfMuhwBSZZBI/MNW6M+zMwIJ2wkioxUDnDvfVi10/cV6p85U75Jn
-            fQ+9HiCOfEqUu5PEynCAD1jN6CQLdl87hLQx9TqbZnHuUYPSH1o9Y6kbA/Vp3bpy
+            59e1afN+eekG2DCI6sWPmLy8jmYh4CQRdEurtfzquDOARZ4IHZjotP5AWI8OPHlM
-            evJc8qa66WHYH1kjdEw+qneD5HzZQOLOtXZ7xkxjGbyMcYex9JfyGHohO5dpLg6B
+            FdK2jGXFVevQY0m619CNm78D2NEdlGe1QtLVSazWQ8MsDLfMnHTYFUy3EoSihzat
-            3XrLlIWWERVz04MlnzlaMKfzhoMCU9ByqJSQ3VBm9kblQqu54fOZD2sN8j9ACEfL
+            QkcR//8whzlLT/NcqKlnBDNBU7FvPov+ZdUmIw1mx2wp5f2sGp4m737Yhoey2aFL
-            YNC7Jm2rasVSqv09G1kso9/VNDw3kNCLvjnpE5rJRP7Ckfj+4FxQN/zPVUwQ1e1k
+            qLXHDc91nVRcw95FBDNYlSH8a2AzT4sm4vFR5EkC6vrfz+v1pdg1Fc3dc++hPgE0
-            upoQ8MHyf1bJr8vspm/prm9zp+PRRTUwY1Yyts/ffj+CF5ec9M3jr/RSeEAdswsL
+            MYWn6f4v8lDhPhw2kpmAP4Oz4uPdmPgdfXKiIzr7qf3O5lIC6ZIIwoqhj2f0odj6
-            6dLKBL1LuLAjKXOuVnQ7E6gN940Y994sDFkbqEmzzCUHGcfxSF3IDn/qpkQlqerU
+            7anDUN5C3B5ruFU3UNJEBLrZelbmg4zf2hAtzfoi0L9paIZX5SCLP3PDbvdRbADc
-            B/D43Yef+rtsUDyTA5RUpxKleGORcS4sV0BhQrNXeFclaMTyMr+AbOei4Y77qlD1
+            oyC3Gw/DeddQ9ZeP+wYiwJ/614zRBmZRzQr9RFowf0gJBSS7TaWPCONfUJ/3eekX
-            x/fHB3IT4Intvp9k4m6jJ86RtLpVhEoA4cHEdCCiXHzUpA6aVtNHVAOqT/aBykrf
+            or8JpLTD5PMQNoS0L4S41Cj+yOg/AlmHF/9yvj1GVTKT9rBj3Snki9NOmY2ZUQo3
-            uSm1wu/nl6yKbIwTJueli1OfQYKEYcUdjOrEOwXb+UDQKSohWZrMg0sj7/S6Pl/S
+            BDdnsftA3w4q4iu06ojQkrjn/FJjmNzb83XR2WxrHFUAaY//nISyY/9uTsEhwFbS
-            WAG1BZ20HXD2ZrVqESV87Pl04nKMqswrio+BINfAT9X3ya7L3DF69MR18bDt+ZIB
+            WAFlKfmyVc7nLBI12i0yWLLy/tcVF3c8gtGfNmyoe/RIr+6EQmzUi0v+X49Tnzpj
-            0F3+9WUREGI5in4S3hXNxrgfLNFl1YLklfWLYcx0HXJN3z6F2eJOUvM=
+            8JAnE+4Jzm2ijqF4Ats5KoXqFiLUenJZQHJ3IFoI36n+hM4P/ICeZ4k=
-            =aT3U
+            =s9pl
             -----END PGP MESSAGE-----
           fp: aef8d6c7e4761fc297cda833df13aebb1011b5d4
-        - created_at: "2024-11-26T19:59:36Z"
-          enc: |-
-            -----BEGIN PGP MESSAGE-----
-
-            hQIMA1kR3vWkIYVnARAAqFyuCtvu6AidWg/9+btEcjWv0sBZaIRpYfX3p2QECwCu
-            UYAtssvSHgHdBEQzU27MA/5CGmEreB3NhWrjGquv88RojLO1JuhNHGYPZKeIcBKr
-            I2oS79RKuYs+d2Qu0KUYDaVoY9M5YJsfkju2FOXqMNYlbqX+lDuWnisigj3n2N4e
-            OEBnVIpfPBQE6c1Z9DaQJE7MyBbKfg5YeWjlwwh+fCf1dV/nGp+QdD88F3dzWMoK
-            xNGt69TwZ8JUVmElAIJqLJTpyDI5xHQUw2A6ddPSTk/u363eHhOnZZUNAAm3FdO5
-            0x+4QhcBaH59S8WDZhw4MVmZN7v4+3l3mf7Rx/TXSz4oJg+U7RMgvc291/gowNVm
-            /cVhBlMYz4Ogx/OYR/t+nzq35r+eBungTB+dRXw7qTTfkCtNgp34JMCkGAq5WWnY
-            57H2HtssGiMF0qN4SfWxw7317oUmqHI2XvG0yWt42G++jNgIGbDOtuc/7wATEbhK
-            SBX2aLqDIB1OUwLHQeawyKkB0qGmRSVPkPg8JLwRp43ICETH1WPkY5m/a2slVlDj
-            qgdw00clTI5Fgu/5G5QBD4Ds9f9ZwjrMD4v+NYfGxa0ajisXl1X6CL1+YvQ6Uicf
-            QmIRJYxyVd0VoXScZnsk0T/XTKjJB/fRLRalA2PmlZ1v+gisCUz2dhM+OHtSjGTS
-            WAG5znRbP8UMVt02O0PgbzHYtIUAtQLCuBnzfEKJn721rqCXf7DXU3jrR73Ys6ce
-            VJzkVBMnBszF71GN56t0PaUYIDOnaGvgjMtHHtOCLQHSK7asnm/Bc+E=
-            =Znii
-            -----END PGP MESSAGE-----
-          fp: 4095412245b6efc14cf92ca25911def5a4218567
     unencrypted_suffix: _unencrypted
     version: 3.8.1

machines/overwatch/configuration.nix

@@ -0,0 +1,131 @@
+{ config, self, lib, pkgs, inputs, ... }:
+
+with lib;
+
+{
+  networking = {
+    hostName = mkDefault "overwatch";
+    useDHCP = false;
+  };
+
+  imports = [
+    self.nixosModules.malobeo.metrics
+    ../modules/malobeo_user.nix
+    ../modules/sshd.nix
+  ];
+
+  networking.firewall.allowedTCPPorts = [ 80 3100 ];
+
+  malobeo.metrics = {
+    enable = true;
+    enablePromtail = true;
+    logNginx = false;
+    lokiHost = "10.0.0.14";
+  };
+
+  services.grafana = {
+    enable = true;
+    settings.server = {
+      domain = "grafana.malobeo.org";
+      http_port = 2342;
+      http_addr = "127.0.0.1";
+    };
+
+    provision.datasources.settings = {
+      apiVersion = 1;
+
+      datasources = [
+        {
+          name = "loki";
+          type = "loki";
+          access = "proxy";
+          uid = "eeakiack8nqwwc";
+          url = "http://localhost:3100";
+          editable = false;
+        }
+        {
+          name = "prometheus";
+          type = "prometheus";
+          access = "proxy";
+          uid = "feakib1gq7ugwc";
+          url = "http://localhost:9001";
+          editable = false;
+
+        }
+      ];
+    };
+
+    provision.dashboards.settings = {
+      apiVersion = 1;
+      providers = [{
+        name = "default";
+        options.path = ./dashboards;
+      }];
+    };
+  };
+
+  services.nginx = {
+    enable = true;
+    virtualHosts.${config.services.grafana.domain} = {
+      locations."/" = {
+          proxyPass = "http://127.0.0.1:${toString config.services.grafana.port}";
+          proxyWebsockets = true;
+
+      extraConfig = ''
+        proxy_set_header Host $host;
+      '';
+    };
+      };
+  };
+
+  services.prometheus = {
+    enable = true;
+    port = 9001;
+
+    scrapeConfigs = [
+      {
+        job_name = "overwatch";
+        static_configs = [{
+          targets = [ "127.0.0.1:9002" ];
+        }];
+      }
+      {
+        job_name = "durruti";
+        static_configs = [{
+          targets = [ "10.0.0.5:9002" ];
+        }];
+      }
+      {
+        job_name = "infradocs";
+        static_configs = [{
+          targets = [ "10.0.0.11:9002" ];
+        }];
+      }
+      {
+        job_name = "nextcloud";
+        static_configs = [{
+          targets = [ "10.0.0.13:9002" ];
+        }];
+      }
+      {
+        job_name = "fanny";
+        static_configs = [{
+          targets = [ "10.0.0.1:9002" ];
+        }];
+      }
+      # add vpn - check how to reach it first. most probably 10.100.0.1
+    ];
+  };
+
+  services.loki = {
+    enable = true;
+    configFile = ./loki.yaml;
+  };
+
+  users.users.promtail.extraGroups = [ "nginx" "systemd-journal" ];
+
+
+
+  system.stateVersion = "22.11"; # Did you read the comment?
+}
+

machines/overwatch/loki.yaml

@@ -0,0 +1,60 @@
+auth_enabled: false
+
+server:
+  http_listen_port: 3100
+  grpc_listen_port: 9096
+  log_level: info
+  grpc_server_max_concurrent_streams: 1000
+
+common:
+  instance_addr: 127.0.0.1
+  path_prefix: /tmp/loki
+  storage:
+    filesystem:
+      chunks_directory: /tmp/loki/chunks
+      rules_directory: /tmp/loki/rules
+  replication_factor: 1
+  ring:
+    kvstore:
+      store: inmemory
+
+query_range:
+  results_cache:
+    cache:
+      embedded_cache:
+        enabled: true
+        max_size_mb: 100
+
+schema_config:
+  configs:
+    - from: 2020-10-24
+      store: tsdb
+      object_store: filesystem
+      schema: v13
+      index:
+        prefix: index_
+        period: 24h
+
+pattern_ingester:
+  enabled: true
+  metric_aggregation:
+    loki_address: localhost:3100
+
+ruler:
+  alertmanager_url: http://localhost:9093
+
+frontend:
+  encoding: protobuf
+
+# By default, Loki will send anonymous, but uniquely-identifiable usage and configuration
+# analytics to Grafana Labs. These statistics are sent to https://stats.grafana.org/
+#
+# Statistics help us better understand how Loki is used, and they show us performance
+# levels for most users. This helps us prioritize features and documentation.
+# For more information on what's sent, look at
+# https://github.com/grafana/loki/blob/main/pkg/analytics/stats.go
+# Refer to the buildReport method to see what goes into a report.
+#
+# If you would like to disable reporting, uncomment the following lines:
+analytics:
+  reporting_enabled: false

machines/overwatch/promtail.yaml

@@ -0,0 +1,29 @@
+server:
+  http_listen_port: 9080
+  grpc_listen_port: 0
+
+positions:
+  filename: /tmp/positions.yaml
+
+clients:
+  - url: http://10.0.0.13:3100/loki/api/v1/push
+
+  
+scrape_configs:
+  - job_name: journal
+    journal:
+      max_age: 12h
+      labels:
+        job: systemd-journal
+        host: overwatch
+    relabel_configs:
+      - source_labels: ["__journal__systemd_unit"]
+        target_label: "unit"
+  - job_name: nginx
+    static_configs:
+    - targets:
+        - localhost
+      labels:
+        job: nginx
+        __path__: /var/log/nginx/*log
+

machines/overwatch/secrets.yaml

@@ -0,0 +1,59 @@
+grafana_admin: ENC[AES256_GCM,data:c+ZnOyxSXrG4eiK8ETKHheadiSz98LLHYwxb,iv:Ut2qFD2p6OmKDWjLMjFxyISxzTdJpZpgIB7obW5bgkY=,tag:HdayzjXQ1Zc7w9ITLzKLxA==,type:str]
+sops:
+    kms: []
+    gcp_kms: []
+    azure_kv: []
+    hc_vault: []
+    age:
+        - recipient: age1ljpdczmg5ctqyeezn739hv589fwhssjjnuqf7276fqun6kc62v3qmhkd0c
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAxN1VURXJuMENJV1Z2eEtS
+            bUR2cTNmNUdhU1B4SHZNMW9KRDV2dW5VNmlNCjdmYXpZb05mMEdPdjN4c0VWOUhV
+            RW4vV05CMno1MmJmYzdESjN5MFVFcjQKLS0tIDNxTE1KaW1EVGhtOEQwWXZndk53
+            bFBCMExGdEdMb2Z0TzF0Yk02MUpkN0kKIUm9iUvU/xu1Xl6yoYSVGcIXKnGsp/D/
+            RjVQ7tgJIbrupubny/fg4v2sz5HOs5uzmEq4ZKgBWrBeMPss4gYstA==
+            -----END AGE ENCRYPTED FILE-----
+    lastmodified: "2025-01-22T00:51:32Z"
+    mac: ENC[AES256_GCM,data:TEEyPmVxIJxC49hDqDbwzTZZ/tNymFr0dMvWn6DRli70Kp5XXNCLTpicAbiFh3WoyzbDpN/5c2yxVNGjhB8nXgKpCZdffdONMY6eSCpPbblYwJS7hNsjW+u2wysSFPDAk5apwbNXJcKnlI1tBcGQRHlym9ShSw6fT7K7afWYWqo=,iv:583DWNug8yNF/vZZN4btT6P1yUa0b1UN4frvAX4UKv0=,tag:YI5KIAe5P5Bx0TZU4wG8ag==,type:str]
+    pgp:
+        - created_at: "2025-01-22T00:51:09Z"
+          enc: |-
+            -----BEGIN PGP MESSAGE-----
+
+            hQGMA5HdvEwzh/H7AQv+PxajhJgXHcxwJ7Mk0gjqFV0dGmNJ1m0eY3gPyIS38GSB
+            Rjto1zUd6EARu1GnxSrVrSZYlQaL6x3l2DuFIP7mtymvlFrmhiAoDz/si0zlzsJp
+            WZyQZdepnt9FyYJAwTzbmfVdpZDYajuMI38byMJqzUhS7SEOsPwiU1KRoTHcf4se
+            2E+9v8OwTVT2UoDxyiVJuDAA+K+Jh2RjHk3p/uVnZDqqQpI9UAI8LrCpun9uALpH
+            +29wyhkCZ9RIHU7nDQNVvwHkbYCyRUwR44bciSwITpjp7GuZCcZvzSSimPktkC9q
+            VZkHA6rHgHgcu6mnMfpP0+j4gB0dU0t4hGF41klV3YpEGfYcFIsKV10lfa6aMNmW
+            08RuLdCtnQyplYhgBm1zQvYHJsIuwK9s1B2dz3Z8l3o2eg8AqFuIL+MlZOvf5A/2
+            MOXffyXbOM5Dhy7DdUckTOYYfwWe1mStw3vx3I3mAFzuOOR7HQuzlc9Bf1oxh36T
+            6e/qOijjPPqkLeR2mufo0lgBBFTQFt2jVvMo1lrCB8Amj4yj/4noXTzglkYTYBKs
+            S513kUdhAGtWoNrqcItOYAn/gl+CPGY2Op3tJBVCWM6aT/KO6M/LPJ7wiQk2zlOL
+            pp7SnCKvv9eQ
+            =N+uj
+            -----END PGP MESSAGE-----
+          fp: c4639370c41133a738f643a591ddbc4c3387f1fb
+        - created_at: "2025-01-22T00:51:09Z"
+          enc: |-
+            -----BEGIN PGP MESSAGE-----
+
+            hQIMA98TrrsQEbXUAQ//fm/TcxgsAMY6P8jvOQMcRBmjjVwfLP1UTTnkRUa36Mmn
+            2V9619zsH1dzTp0gUVV8mvIKUtpz2nCBTZBJw803wW9KorxW6f4e3+mIWelZT5WM
+            sLcQPFWAYKDVnSfk5j8kRSmpM6k2xFRB9DTEMIyFH9PZL99Fztp2hjTn553YTQOo
+            pdw+AMzgptYQghW/Pl/32wXHwCO+bL8tyk62VIQO13l3tX83oSslXNkFzNQYt3jv
+            xXFUaQEGD/1lFLeFnIuJZzfjWt6n0fShJboUcuk/ZIcYdwrbG0pyLLoBoObSRQtG
+            t+7VpWpfl3rnk822SU9z9YcaMNy9HD3Kz9Qh0BRQiN3scCMzm1LyzlXLqlc/gPiq
+            JyCxy98vJXIxmlLQZpDFfTMe3xsc8jSsHI6av+wEFKGEJAOUkDtRYvLfZdgDTfiB
+            XTAhQ3ixnlBxdZZ9DjBXyVfM8q9iB8bggFi7g2SjO32LKhXRFUqZR+avddCyKR/V
+            hRpWjDRn+eX2tl7LPawvX6tIow3aZiezkMVyeRXfcZCvpicq64b80LrYR+JJUfJE
+            vxHaekKImrdJ9ocii0wW91ZmESJwL6m5lt3ZsCY5GTlEDt4wBse5uhj48mtuK8sh
+            g8uQMKp7SiiCtV5a6O1SQLDJeAt6VCcRyudLToO9S4gwrGXNPcGxsHj07XAN2PHS
+            WAE6wUhufXfpa8UgSWy7fmEZt4L03XlRfC7bm/ycwaFww3A7w4+B1gkW6gOon6sy
+            nOyIxUZfU6abZWKzH+OIuViKH7xPiULDy75gEmkRHjKu5BiC3Tx0eO4=
+            =+PPr
+            -----END PGP MESSAGE-----
+          fp: aef8d6c7e4761fc297cda833df13aebb1011b5d4
+    unencrypted_suffix: _unencrypted
+    version: 3.9.2

machines/testvm/configuration.nix

@@ -24,7 +24,7 @@ in
 
   malobeo.disks = {
     enable = true;
-    encryption = false;
+    encryption = true;
     hostId = "83abc8cb";
     devNodes = "/dev/disk/by-path/";
     root = {

machines/uptimekuma/configuration.nix

@@ -6,7 +6,6 @@ with lib;
   networking = {
     hostName = mkDefault "uptimekuma";
     useDHCP = false;
-    nameservers = [ "1.1.1.1" ];
   };
 
   imports = [

machines/vpn/configuration.nix

@@ -17,6 +17,7 @@ with lib;
   };
 
   imports = [
+    inputs.self.nixosModules.malobeo.vpn
     ../modules/malobeo_user.nix
     ../modules/sshd.nix
     ../modules/minimal_tools.nix
@@ -34,9 +35,36 @@ with lib;
       locations."/" = {
         proxyPass = "http://10.100.0.101";
         extraConfig = ''
+          proxy_set_header Host $host;
         '';
       };
+    };
 
+    virtualHosts."cloud.malobeo.org" = {
+      locations."/" = {
+        proxyPass = "http://10.100.0.101";
+        extraConfig = ''
+          proxy_set_header Host $host;
+        '';
+      };
+    };
+
+    virtualHosts."grafana.malobeo.org" = {
+      locations."/" = {
+        proxyPass = "http://10.100.0.101";
+        extraConfig = ''
+          proxy_set_header Host $host;
+        '';
+      };
+    };
+
+    virtualHosts."tasklist.malobeo.org" = {
+      locations."/" = {
+        proxyPass = "http://10.100.0.101";
+        extraConfig = ''
+          proxy_set_header Host $host;
+        '';
+      };
     };
   };
 

outputs.nix

@@ -15,69 +15,8 @@ in (utils.lib.eachSystem (builtins.filter filter_system utils.lib.defaultSystems
     pkgs-unstable = nixpkgs-unstable.legacyPackages."${system}";
     pkgs = nixpkgs.legacyPackages."${system}";
 
-    vmMicroVMOverwrites = options: {
+    hosts = import ./machines/hosts.nix ( inputs // { inherit inputs; self = self; });
-      microvm = {
+    utils = import ./machines/modules/host_builder.nix ( inputs // { inherit inputs; self = self; hosts = hosts; });
-        mem = pkgs.lib.mkForce 4096;
-        hypervisor = pkgs.lib.mkForce "qemu";
-        socket = pkgs.lib.mkForce null;
-        shares = pkgs.lib.mkForce [
-          {
-            tag = "ro-store";
-            source = "/nix/store";
-            mountPoint = "/nix/.ro-store";
-          }
-        ];
-        interfaces = pkgs.lib.mkIf (!options.withNetworking) (pkgs.lib.mkForce [{
-          type = "user";
-          id = "eth0";
-          mac = "02:23:de:ad:be:ef";
-        }]);
-      };
-
-      boot.isContainer = pkgs.lib.mkForce false;
-      users.users.root.password = "";
-      fileSystems."/".fsType = pkgs.lib.mkForce "tmpfs";
-      services.getty.helpLine = ''
-        Log in as "root" with an empty password.
-        Use "reboot" to shut qemu down.
-      '';
-    };
-
-    vmDiskoOverwrites = {
-      boot.initrd = {
-        secrets = pkgs.lib.mkForce {};
-        network.ssh.enable = pkgs.lib.mkForce false;
-      };
-
-      malobeo.disks.enable = pkgs.lib.mkForce false;
-      networking.hostId = "a3c3101f";
-    };
-
-    vmSopsOverwrites = host: {
-      sops.defaultSopsFile = pkgs.lib.mkForce ./machines/${host}/dummy.yaml;
-
-      environment.etc = {
-        devHostKey = {
-          source = ./machines/secrets/devkey_ed25519;
-          mode = "0600";
-        };
-      };
-
-      services.openssh.hostKeys = [{
-        path = "/etc/devHostKey";
-        type = "ed25519";
-      }];
-    };
-
-    buildVM = host: networking: sopsDummy: disableDisko: (self.nixosConfigurations.${host}.extendModules {
-        modules = [
-          (vmMicroVMOverwrites { withNetworking = networking; })
-          (if sopsDummy then (vmSopsOverwrites host) else {})
-          (if disableDisko then vmDiskoOverwrites else {})
-        ] ++ pkgs.lib.optionals (! self.nixosConfigurations.${host}.config ? microvm) [
-          microvm.nixosModules.microvm
-        ];
-      }).config.microvm.declaredRunner;
   in
   {
     devShells.default =
@@ -113,7 +52,7 @@ in (utils.lib.eachSystem (builtins.filter filter_system utils.lib.defaultSystems
       scripts.run-vm = self.packages.${system}.run-vm;
     };
 
-    vmBuilder = buildVM;
+    vmBuilder = utils.buildVM;
 
     packages = {
       docs = pkgs.stdenv.mkDerivation {
@@ -133,73 +72,46 @@ in (utils.lib.eachSystem (builtins.filter filter_system utils.lib.defaultSystems
         '';
       };
 
-      run-vm = pkgs.writeShellScriptBin "run-vm" ''
+      run-vm = pkgs.writeShellScriptBin "run-vm"  (builtins.readFile ./scripts/run-vm.sh);
-        usage() {
-            echo "Usage: run-vm <hostname> [--networking] [--dummy-secrets] [--no-disko]"
-            echo "ATTENTION: This script must be run from the flakes root directory"
-            echo "--networking setup interfaces. requires root and hostbridge enabled on the host"
-            echo "--dummy-secrets deploy dummy sops secrets"
-            echo "--no-disko disable disko and initrd secrets. needed for actual hosts like fanny"
-            exit 1
-        }
-        
-        # check at least one arg was given
-        if [ "$#" -lt 1 ]; then
-            usage
-        fi
-        
-        HOSTNAME=$1
-        
-        # Optionale Argumente
-        NETWORK=false
-        DUMMY_SECRETS=false
-        NO_DISKO=false
-        
-        # check argws
-        shift
-        while [[ "$#" -gt 0 ]]; do
-            case $1 in
-                --networking) NETWORK=true ;;
-                --dummy-secrets) DUMMY_SECRETS=true ;;
-                --no-disko) NO_DISKO=true ;;
-                *) echo "Unknown argument: $1"; usage ;;
-            esac
-            shift
-        done
-        echo "starting host $HOSTNAME"
-        echo "enable networking: $NETWORK"
-        echo "deploy dummy secrets: $DUMMY_SECRETS"
-        echo "disable disko and initrd secrets: $NO_DISKO"
-        
-        ${pkgs.nix}/bin/nix run --impure --expr "((builtins.getFlake \"$(pwd)\").vmBuilder.x86_64-linux \"$HOSTNAME\" $NETWORK $DUMMY_SECRETS $NO_DISKO)"
-      '';
     };
 
     apps = {
       docs = {
         type = "app";
         program = builtins.toString (pkgs.writeShellScript "docs" ''
+          ${pkgs.xdg-utils}/bin/xdg-open "${self.packages.${system}.docs}/share/doc/index.html"
+        '');
+      };
+
+
+      docsDev = {
+        type = "app";
+        program = builtins.toString (pkgs.writeShellScript "docs" ''
+          echo "needs to run from infrastuctre root folder"
           ${pkgs.mdbook}/bin/mdbook serve --open ./doc
         '');
       };
 
       run-vm = {
         type = "app";
-        program = self.packages.${system}.run-vm;
+        program = "${self.packages.${system}.run-vm}/bin/run-vm";
       };
     };
 
-  })) // {
+  })) // (
-    nixosConfigurations = import ./machines/configuration.nix (inputs // {
+  let
-      inherit inputs;
+    hosts = import ./machines/hosts.nix ( inputs // { inherit inputs; self = self; });
-      self = self;
+    utils = import ./machines/modules/host_builder.nix ( inputs // { inherit inputs; self = self; hosts = hosts; });
-    });
+  in
+  {
+    nixosConfigurations = utils.buildHost hosts.malobeo.hosts;
 
     nixosModules.malobeo = {
       host.imports = [ ./machines/durruti/host_config.nix ];
       microvm.imports = [ ./machines/modules/malobeo/microvm_host.nix ];
       vpn.imports = [ ./machines/modules/malobeo/wireguard.nix ];
       initssh.imports = [ ./machines/modules/malobeo/initssh.nix ];
+      metrics.imports = [ ./machines/modules/malobeo/metrics.nix ];
       disko.imports = [ ./machines/modules/disko ];
     };
 
@@ -214,4 +126,4 @@ in (utils.lib.eachSystem (builtins.filter filter_system utils.lib.defaultSystems
       nixpkgs.lib.mapAttrs getBuildEntry self.nixosConfigurations
 
       );
-}
+})

scripts/remote-install-encrypt.sh

@@ -37,9 +37,11 @@ trap cleanup EXIT
 
 # Create the directory where sshd expects to find the host keys
 install -d -m755 "$temp/etc/ssh/"
+install -d -m755 "$temp/root/"
 
 diskKey=$(sops -d machines/$hostname/disk.key)
 echo "$diskKey" > /tmp/secret.key
+echo "$diskKey" > $temp/root/secret.key
 
 ssh-keygen -f $temp/etc/ssh/"$hostname" -t ed25519 -N ""
 ssh-keygen -f $temp/etc/ssh/initrd -t ed25519 -N ""

scripts/run-vm.sh

@@ -0,0 +1,67 @@
+usage() {
+    echo "Usage: run-vm <hostname> [--networking] [--dummy-secrets] [--no-disko]"
+    echo "ATTENTION: This script must be run from the flakes root directory"
+    echo "--networking setup interfaces. requires root and hostbridge enabled on the host"
+    echo "--dummy-secrets run vm with dummy sops secrets"
+    echo "--no-disko disable disko and initrd secrets. needed for real hosts like fanny"
+    echo "--writable-store enables writable store. necessary for host with nested imperative microvms like fanny"
+    echo "--var path to directory that should be shared as /var. may require root otherwise some systemd units fail within vm. if dir is empty vm will populate"
+    echo "--fwd-port forwards the given port to port 80 on vm"
+    exit 1
+}
+
+# check at least one arg was given
+if [ "$#" -lt 1 ]; then
+    usage
+fi
+
+HOSTNAME=$1
+
+# Optionale Argumente
+NETWORK=false
+DUMMY_SECRETS=false
+NO_DISKO=false
+RW_STORE=false
+VAR_PATH=""
+FWD_PORT=0
+
+# check argws
+shift
+while [[ "$#" -gt 0 ]]; do
+    case $1 in
+        --networking) NETWORK=true ;;
+        --dummy-secrets) DUMMY_SECRETS=true ;;
+        --no-disko) NO_DISKO=true ;;
+        --writable-store) RW_STORE=true ;;
+        --var) 
+          if [[ -n "$2" && ! "$2" =~ ^- ]]; then
+              VAR_PATH="$2"
+              shift
+          else
+              echo "Error: --var requires a non-empty string argument."
+              usage
+          fi
+          ;;
+        --fwd-port) 
+          if [[ -n "$2" && ! "$2" =~ ^- ]]; then
+              FWD_PORT="$2"
+              shift
+          else
+              echo "Error: --var requires a non-empty string argument."
+              usage
+          fi
+          ;;
+        *) echo "Unknown argument: $1"; usage ;;
+    esac
+    shift
+done
+echo "starting host $HOSTNAME"
+echo "enable networking: $NETWORK"
+echo "deploy dummy secrets: $DUMMY_SECRETS"
+echo "disable disko and initrd secrets: $NO_DISKO"
+echo "use writable store: $RW_STORE"
+if [ -n "$VAR_PATH" ]; then
+  echo "sharing var directory: $VAR_PATH"
+fi
+
+nix run --show-trace --impure --expr "((builtins.getFlake \"$(pwd)\").vmBuilder.x86_64-linux \"$HOSTNAME\" $NETWORK $DUMMY_SECRETS $NO_DISKO \"$VAR_PATH\" $RW_STORE $FWD_PORT).config.microvm.declaredRunner"

scripts/unlock-boot.sh

@@ -23,18 +23,14 @@ echo
 if [ $# = 1 ]
     then
     diskkey=$(sops -d machines/$HOSTNAME/disk.key)
-    echo "$diskkey" | ssh $sshoptions root@$HOSTNAME-initrd "systemd-tty-ask-password-agent" #storage
-
     echo "$diskkey" | ssh $sshoptions root@$HOSTNAME-initrd "systemd-tty-ask-password-agent" #root
 
 elif [ $# = 2 ]
     then
     diskkey=$(sops -d machines/$HOSTNAME/disk.key)
     IP=$2
-    echo "$diskkey" | ssh $sshoptions root@$IP "systemd-tty-ask-password-agent" #storage
-
     echo "$diskkey" | ssh $sshoptions root@$IP "systemd-tty-ask-password-agent" #root
-
+    
 else
     echo
     echo "Unlock the root disk on a remote host."