wip

sharing /var somehow doesnt work. for example nginx fails because of
lacking permissions to access /var/log/nginx. this also happens when
run-vm is started as root. thats why only /var/lib is shared which still
allows application persistency between tests

machines/configuration.nix

@@ -43,6 +43,7 @@ let
   defaultModules = baseModules;
 
   makeMicroVM = hostName: ipv4Addr: macAddr: modules: [
+    self.nixosModules.malobeo.metrics
     {
       microvm = {
         hypervisor = "cloud-hypervisor";
@@ -79,6 +80,13 @@ let
           }
         ];
       };
+
+      malobeo.metrics = {
+        enable = true;
+        enablePromtail = true;
+        logNginx = false;
+        lokiHost = "10.0.0.14";
+      };
       
       systemd.network.enable = true;
       
@@ -179,11 +187,19 @@ in
     ];
   };
 
+  overwatch = nixosSystem {
+    system = "x86_64-linux";
+    specialArgs.inputs = inputs;
+    specialArgs.self = self;
+    modules = makeMicroVM "overwatch" "10.0.0.14" "D0:E5:CA:F0:D7:E0" [
+      ./overwatch/configuration.nix
+    ];
+  };
+
   testvm = nixosSystem {
     system = "x86_64-linux";
     specialArgs.inputs = inputs;
     specialArgs.self = self;
     modules = defaultModules ++ [ ./testvm ];
   };
-
 }

machines/durruti/configuration.nix

@@ -6,7 +6,6 @@ with lib;
   networking = {
     hostName = mkDefault "durruti";
     useDHCP = false;
-    nameservers = [ "1.1.1.1" ];
   };
 
   networking.firewall.allowedTCPPorts = [ 8080 ];

machines/fanny/configuration.nix

@@ -53,7 +53,7 @@ in
   };
 
   services.malobeo.microvm.enableHostBridge = true;
-  services.malobeo.microvm.deployHosts = [ "infradocs" "nextcloud" ];
+  services.malobeo.microvm.deployHosts = [ "infradocs" "nextcloud" "overwatch" ];
 
   networking = {
     firewall = {

machines/infradocs/configuration.nix

@@ -6,7 +6,6 @@ with lib;
   networking = {
     hostName = mkDefault "infradocs";
     useDHCP = false;
-    nameservers = [ "1.1.1.1" ];
   };
 
   imports = [
@@ -15,6 +14,12 @@ with lib;
     ../modules/sshd.nix
   ];
 
+  networking.firewall.allowedTCPPorts = [ 9002 ];
+
+  malobeo.metrics.logNginx = lib.mkForce true;
+
+  users.users.promtail.extraGroups = [ "nginx" "systemd-journal" ];
+
   system.stateVersion = "22.11"; # Did you read the comment?
 }
 

machines/modules/malobeo/metrics.nix

@@ -0,0 +1,56 @@
+{ config, lib, pkgs, ... }:
+let 
+  cfg = config.malobeo.metrics;
+in
+{
+  options.malobeo.metrics = {
+    enable = lib.mkOption {
+      type = lib.types.bool;
+      default = false;
+      description = "Enable sharing metrics";
+    };
+    enablePromtail = lib.mkOption {
+      type = lib.types.bool;
+      default = true;
+      description = "Enable sharing logs";
+    };
+    logNginx = lib.mkOption {
+      type = lib.types.bool;
+      default = false;
+      description = "Share nginx logs";
+    };
+    lokiHost = lib.mkOption {
+      type = lib.types.str;
+      default = "10.0.0.14";
+      description = "Address of loki host";
+    };
+  };
+  
+  config = lib.mkIf (cfg.enable) {
+
+    networking.firewall.allowedTCPPorts = [ 9002 ];
+
+    services.prometheus = {
+      exporters = {
+        node = {
+          enable = true;
+          enabledCollectors = [ "systemd" "processes" ];
+          port = 9002;
+        };
+      };
+    };
+
+    services.promtail = {
+      enable = cfg.enablePromtail;
+      configFile = import ./promtail_config.nix { 
+        lokiAddress = cfg.lokiHost;
+        logNginx = cfg.logNginx;
+        config = config;
+        pkgs = pkgs;
+      };
+    };
+
+    users.users.promtail.extraGroups = [ "systemd-journal" ] ++ (lib.optionals cfg.logNginx [ "nginx" ]) ;
+
+  };
+}

machines/modules/malobeo/microvm_host.nix

@@ -86,6 +86,12 @@ in
     in
     builtins.listToAttrs (map mapperFunc cfg.deployHosts);
 
+    systemd.tmpfiles.rules = builtins.concatLists (map (name: [
+      "d /var/lib/microvms/${name}/var 0755 root root - -"
+      "d /var/lib/microvms/${name}/etc 0755 root root - -"
+      "d /${name} 0755 root root - -"
+    ]) cfg.deployHosts);
+
     systemd.services = builtins.foldl' (services: name: services // {
       "microvm-update@${name}" = {
         description = "Update MicroVMs automatically";

machines/modules/malobeo/promtail_config.nix

@@ -0,0 +1,49 @@
+{ logNginx, lokiAddress, config, pkgs, ... }:
+
+let
+  basecfg = ''
+    server:
+      http_listen_port: 9080
+      grpc_listen_port: 0
+    
+    positions:
+      filename: /tmp/positions.yaml
+    
+    clients:
+      - url: http://${lokiAddress}:3100/loki/api/v1/push
+  '';
+
+  withNginx = ''
+    scrape_configs:
+      - job_name: journal
+        journal:
+          max_age: 12h
+          labels:
+            job: systemd-journal
+            host: ${config.networking.hostName}
+        relabel_configs:
+          - source_labels: ["__journal__systemd_unit"]
+            target_label: "unit"
+      - job_name: nginx
+        static_configs:
+        - targets:
+            - localhost
+          labels:
+            job: nginx
+            __path__: /var/log/nginx/*log
+  '';
+
+  withoutNginx = ''
+    scrape_configs:
+      - job_name: journal
+        journal:
+          max_age: 12h
+          labels:
+            job: systemd-journal
+            host: ${config.networking.hostName}
+        relabel_configs:
+          - source_labels: ["__journal__systemd_unit"]
+            target_label: "unit"
+  '';
+in
+pkgs.writeText "promtailcfg.yaml" (if logNginx then ''${basecfg}${withNginx}'' else ''${basecfg}${withoutNginx}'')

machines/overwatch/configuration.nix

@@ -0,0 +1,115 @@
+{ config, lib, pkgs, inputs, ... }:
+
+with lib;
+
+{
+  networking = {
+    hostName = mkDefault "overwatch";
+    useDHCP = false;
+  };
+
+  imports = [
+    ../modules/malobeo_user.nix
+    ../modules/sshd.nix
+  ];
+
+  networking.firewall.allowedTCPPorts = [ 80 9080 9001 3100 ];
+
+  services.grafana = {
+    enable = true;
+    domain = "grafana.malobeo.org";
+    port = 2342;
+    addr = "127.0.0.1";
+
+    provision.datasources.settings = {
+      apiVersion = 1;
+
+      datasources = [
+        {
+          name = "loki";
+          type = "loki";
+          access = "proxy";
+          uid = "eeakiack8nqwwc";
+          url = "http://localhost:3100";
+          editable = false;
+        }
+        {
+          name = "prometheus";
+          type = "prometheus";
+          access = "proxy";
+          uid = "feakib1gq7ugwc";
+          url = "http://localhost:9001";
+          editable = false;
+
+        }
+      ];
+    };
+
+    provision.dashboards.settings = {
+      apiVersion = 1;
+      providers = [{
+        name = "default";
+        options.path = ./dashboards;
+      }];
+    };
+  };
+
+  services.nginx = {
+    enable = true;
+    virtualHosts.${config.services.grafana.domain} = {
+      locations."/" = {
+          proxyPass = "http://127.0.0.1:${toString config.services.grafana.port}";
+          proxyWebsockets = true;
+
+      extraConfig = ''
+        proxy_set_header Host $host;
+      '';
+    };
+      };
+  };
+
+  services.prometheus = {
+    enable = true;
+    port = 9001;
+
+    scrapeConfigs = [
+      {
+        job_name = "overwatch";
+        static_configs = [{
+          targets = [ "127.0.0.1:9002" ];
+        }];
+      }
+      {
+        job_name = "durruti";
+        static_configs = [{
+          targets = [ "10.0.0.5:9002" ];
+        }];
+      }
+      {
+        job_name = "infradocs";
+        static_configs = [{
+          targets = [ "10.0.0.11:9002" ];
+        }];
+      }
+      {
+        job_name = "nextcloud";
+        static_configs = [{
+          targets = [ "10.0.0.13:9002" ];
+        }];
+      }
+      # add vpn - check how to reach it first. most probably 10.100.0.1
+    ];
+  };
+
+  services.loki = {
+    enable = true;
+    configFile = ./loki.yaml;
+  };
+
+  users.users.promtail.extraGroups = [ "nginx" "systemd-journal" ];
+
+
+
+  system.stateVersion = "22.11"; # Did you read the comment?
+}
+

machines/overwatch/loki.yaml

@@ -0,0 +1,60 @@
+auth_enabled: false
+
+server:
+  http_listen_port: 3100
+  grpc_listen_port: 9096
+  log_level: debug
+  grpc_server_max_concurrent_streams: 1000
+
+common:
+  instance_addr: 127.0.0.1
+  path_prefix: /tmp/loki
+  storage:
+    filesystem:
+      chunks_directory: /tmp/loki/chunks
+      rules_directory: /tmp/loki/rules
+  replication_factor: 1
+  ring:
+    kvstore:
+      store: inmemory
+
+query_range:
+  results_cache:
+    cache:
+      embedded_cache:
+        enabled: true
+        max_size_mb: 100
+
+schema_config:
+  configs:
+    - from: 2020-10-24
+      store: tsdb
+      object_store: filesystem
+      schema: v13
+      index:
+        prefix: index_
+        period: 24h
+
+pattern_ingester:
+  enabled: true
+  metric_aggregation:
+    loki_address: localhost:3100
+
+ruler:
+  alertmanager_url: http://localhost:9093
+
+frontend:
+  encoding: protobuf
+
+# By default, Loki will send anonymous, but uniquely-identifiable usage and configuration
+# analytics to Grafana Labs. These statistics are sent to https://stats.grafana.org/
+#
+# Statistics help us better understand how Loki is used, and they show us performance
+# levels for most users. This helps us prioritize features and documentation.
+# For more information on what's sent, look at
+# https://github.com/grafana/loki/blob/main/pkg/analytics/stats.go
+# Refer to the buildReport method to see what goes into a report.
+#
+# If you would like to disable reporting, uncomment the following lines:
+analytics:
+  reporting_enabled: false

machines/overwatch/promtail.yaml

@@ -0,0 +1,29 @@
+server:
+  http_listen_port: 9080
+  grpc_listen_port: 0
+
+positions:
+  filename: /tmp/positions.yaml
+
+clients:
+  - url: http://10.0.0.13:3100/loki/api/v1/push
+
+  
+scrape_configs:
+  - job_name: journal
+    journal:
+      max_age: 12h
+      labels:
+        job: systemd-journal
+        host: overwatch
+    relabel_configs:
+      - source_labels: ["__journal__systemd_unit"]
+        target_label: "unit"
+  - job_name: nginx
+    static_configs:
+    - targets:
+        - localhost
+      labels:
+        job: nginx
+        __path__: /var/log/nginx/*log
+

machines/uptimekuma/configuration.nix

@@ -6,7 +6,6 @@ with lib;
   networking = {
     hostName = mkDefault "uptimekuma";
     useDHCP = false;
-    nameservers = [ "1.1.1.1" ];
   };
 
   imports = [

machines/vpn/configuration.nix

@@ -17,6 +17,7 @@ with lib;
   };
 
   imports = [
+    inputs.self.nixosModules.malobeo.vpn
     ../modules/malobeo_user.nix
     ../modules/sshd.nix
     ../modules/minimal_tools.nix

outputs.nix

@@ -12,15 +12,116 @@
 let filter_system = name: if name == utils.lib.system.i686-linux then false else true;
 in (utils.lib.eachSystem (builtins.filter filter_system utils.lib.defaultSystems) ( system:
   let
+    baseModules = [
+      # make flake inputs accessiable in NixOS
+      { _module.args.inputs = inputs; }
+      {
+        imports = [
+          ({ pkgs, ... }: {
+            nix = {
+              extraOptions = ''
+                experimental-features = nix-command flakes
+              '';
+
+              settings = {
+                substituters = [
+                  "https://cache.dynamicdiscord.de"
+                  "https://cache.nixos.org/"
+                ];
+                trusted-public-keys = [
+                  "cache.dynamicdiscord.de:DKueZicqi2NhJJXz9MYgUbiyobMs10fTyHCgAUibRP4="
+                ];
+                trusted-users = [ "root" "@wheel" ];
+              };
+            };
+          })
+          sops-nix.nixosModules.sops
+          #microvm.nixosModules.microvm
+        ];
+      }
+    ];
+    defaultModules = baseModules;
+
+  makeMicroVM = hostName: ipv4Addr: macAddr: modules: [
+    self.nixosModules.malobeo.metrics
+    {
+      microvm = {
+        hypervisor = "cloud-hypervisor";
+        mem = 2560;
+        shares = [
+          {
+            source = "/nix/store";
+            mountPoint = "/nix/.ro-store";
+            tag = "store";
+            proto = "virtiofs";
+            socket = "store.socket";
+          }
+          {
+            source = "/var/lib/microvms/${hostName}/etc";
+            mountPoint = "/etc";
+            tag = "etc";
+            proto = "virtiofs";
+            socket = "etc.socket";
+          }
+          {
+            source = "/var/lib/microvms/${hostName}/var";
+            mountPoint = "/var";
+            tag = "var";
+            proto = "virtiofs";
+            socket = "var.socket";
+          }
+        ];
+
+        interfaces = [
+          {
+            type = "tap";
+            id = "vm-${hostName}";
+            mac = "${macAddr}";
+          }
+        ];
+      };
+
+      malobeo.metrics = {
+        enable = true;
+        enablePromtail = true;
+        logNginx = false;
+        lokiHost = "10.0.0.14";
+      };
+      
+      systemd.network.enable = true;
+      
+      systemd.network.networks."20-lan" = {
+        matchConfig.Type = "ether";
+        networkConfig = {
+          Address = [ "${ipv4Addr}/24" ];
+          Gateway = "10.0.0.1";
+          DNS = ["1.1.1.1"];
+          DHCP = "no";
+        };
+      };
+    }
+  ] ++ defaultModules ++ modules;
+
+
     pkgs-unstable = nixpkgs-unstable.legacyPackages."${system}";
     pkgs = nixpkgs.legacyPackages."${system}";
 
-    vmMicroVMOverwrites = options: {
+    vmMicroVMOverwrites = hostname: options: {
       microvm = {
         mem = pkgs.lib.mkForce 4096;
         hypervisor = pkgs.lib.mkForce "qemu";
         socket = pkgs.lib.mkForce null;
-        shares = pkgs.lib.mkForce ([
+
+
+        #needed for hosts that deploy imperative microvms (for example fanny)
+        writableStoreOverlay = pkgs.lib.mkIf options.writableStore "/nix/.rw-store";
+        volumes = pkgs.lib.mkIf options.writableStore [ {
+          image = "nix-store-overlay.img";
+          mountPoint = self.nixosConfigurations.${hostname}.config.microvm.writableStoreOverlay;
+          size = 2048;
+        } ];
+
+        shares = pkgs.lib.mkForce (pkgs.lib.optionals (!options.writableStore) [
           {
             tag = "ro-store";
             source = "/nix/store";
@@ -29,22 +130,33 @@ in (utils.lib.eachSystem (builtins.filter filter_system utils.lib.defaultSystems
         ] ++ pkgs.lib.optionals (options.varPath != "") [
           {
             source = "${options.varPath}";
-            mountPoint = "/var/lib";
-            tag = "varlib";
+            securityModel = "mapped";
+            mountPoint = "/var";
+            tag = "var";
           }
         ]);
+
         interfaces = pkgs.lib.mkIf (!options.withNetworking) (pkgs.lib.mkForce [{
           type = "user";
           id = "eth0";
           mac = "02:23:de:ad:be:ef";
         }]);
+
+        #if networking is disabled forward port 80 to still have access to webservices
+        forwardPorts = pkgs.lib.mkIf (!options.withNetworking) (pkgs.lib.mkForce [
+          { from = "host"; host.port = 8080; guest.port = 80; }
+        ]);
+
       };
 
       fileSystems = {
         "/".fsType = pkgs.lib.mkForce "tmpfs";
-        "/var/lib" = pkgs.lib.mkIf (options.varPath != "") {
-          depends = [ "/var" ];
-        };
+
+        # prometheus uses a memory mapped file which doesnt seem supported by 9p shares
+        # therefore we mount a tmpfs inside the datadir
+        "/var/lib/prometheus2/data" = pkgs.lib.mkIf (hostname == "overwatch" && options.varPath != "") (pkgs.lib.mkForce {
+          fsType = pkgs.lib.mkForce "tmpfs";
+        });
       };
 
       boot.isContainer = pkgs.lib.mkForce false;
@@ -82,15 +194,59 @@ in (utils.lib.eachSystem (builtins.filter filter_system utils.lib.defaultSystems
       }];
     };
 
-    buildVM = host: networking: sopsDummy: disableDisko: varPath: (self.nixosConfigurations.${host}.extendModules {
+    buildVM = host: networking: sopsDummy: disableDisko: varPath: writableStore: (self.nixosConfigurations.${host}.extendModules {
         modules = [
-          (vmMicroVMOverwrites { withNetworking = networking; varPath = "${varPath}"; })
+          (vmMicroVMOverwrites host { 
+            withNetworking = networking; 
+            varPath = "${varPath}"; 
+            writableStore = writableStore; })
           (if sopsDummy then (vmSopsOverwrites host) else {})
           (if disableDisko then vmDiskoOverwrites else {})
         ] ++ pkgs.lib.optionals (! self.nixosConfigurations.${host}.config ? microvm) [
-          microvm.nixosModules.microvm
-        ];
-      }).config.microvm.declaredRunner;
+          #microvm.nixosModules.microvm
+        ] ++ pkgs.lib.optionals (self.nixosConfigurations.${host}.config ? services.malobeo.microvm.deployHosts) [
+          #microvm.nixosModules.host
+          {
+          services.malobeo.microvm.deployHosts = pkgs.lib.mkForce [];
+          systemd.tmpfiles.rules = builtins.concatLists (map (name: [
+            "q /var/lib/microvms/${name}/var 0755 root root - -"
+            "q /var/lib/microvms/${name}/etc 0755 root root - -"
+            "q /var/${name}/wow/it/works 0755 root root - -"
+            "q /var/lib/${name} 0755 root root - -"
+            "d /${name} 0755 root root - -"
+          ]) self.nixosConfigurations.${host}.config.services.malobeo.microvm.deployHosts);
+
+
+          microvm.vms = 
+          let
+            # Map the values to each hostname to then generate an Attrset using listToAttrs
+            mapperFunc = name: { inherit name; value = {
+              #pkgs = import self.nixosConfigurations.${name}.config.nixpkgs;
+
+              #pkgs = (buildVM name networking sopsDummy false "" false).config.nixpkgs;
+              #config = (buildVM name networking sopsDummy false "" false);
+              #pkgs = pkgs;
+              #config = self.nixosConfigurations.${name};
+              specialArgs.inputs = inputs;
+              specialArgs.self = self;
+              config = {
+                imports = (makeMicroVM "${name}" "10.0.0.11" "D0:E5:CA:F0:D7:E7" [
+              
+                  #(vmMicroVMOverwrites name { 
+                  #  withNetworking = true; 
+                  #  varPath = ""; 
+                  #  writableStore = false; })
+                  (if sopsDummy then (vmSopsOverwrites name) else {})
+
+                  
+                ]);
+
+              };
+            }; };
+          in
+          builtins.listToAttrs (map mapperFunc self.nixosConfigurations.${host}.config.services.malobeo.microvm.deployHosts);
+        }];
+      });
   in
   {
     devShells.default =
@@ -153,7 +309,8 @@ in (utils.lib.eachSystem (builtins.filter filter_system utils.lib.defaultSystems
             echo "--networking setup interfaces. requires root and hostbridge enabled on the host"
             echo "--dummy-secrets run vm with dummy sops secrets"
             echo "--no-disko disable disko and initrd secrets. needed for real hosts like fanny"
-            echo "--varlib path to directory that should be shared as /var/lib. may require root otherwise some systemd units fail within vm. if dir is empty vm will populate"
+            echo "--writable-store enables writable store. necessary for host with nested imperative microvms like fanny"
+            echo "--var path to directory that should be shared as /var. may require root otherwise some systemd units fail within vm. if dir is empty vm will populate"
             exit 1
         }
         
@@ -168,6 +325,7 @@ in (utils.lib.eachSystem (builtins.filter filter_system utils.lib.defaultSystems
         NETWORK=false
         DUMMY_SECRETS=false
         NO_DISKO=false
+        RW_STORE=false
         VAR_PATH=""
         
         # check argws
@@ -177,7 +335,8 @@ in (utils.lib.eachSystem (builtins.filter filter_system utils.lib.defaultSystems
                 --networking) NETWORK=true ;;
                 --dummy-secrets) DUMMY_SECRETS=true ;;
                 --no-disko) NO_DISKO=true ;;
-                --varlib) 
+                --writable-store) RW_STORE=true ;;
+                --var) 
                   if [[ -n "$2" && ! "$2" =~ ^- ]]; then
                       VAR_PATH="$2"
                       shift
@@ -194,11 +353,12 @@ in (utils.lib.eachSystem (builtins.filter filter_system utils.lib.defaultSystems
         echo "enable networking: $NETWORK"
         echo "deploy dummy secrets: $DUMMY_SECRETS"
         echo "disable disko and initrd secrets: $NO_DISKO"
+        echo "use writable store: $RW_STORE"
         if [ -n "$VAR_PATH" ]; then
           echo "sharing var directory: $VAR_PATH"
         fi
         
-        ${pkgs.nix}/bin/nix run --show-trace --impure --expr "((builtins.getFlake \"$(pwd)\").vmBuilder.x86_64-linux \"$HOSTNAME\" $NETWORK $DUMMY_SECRETS $NO_DISKO \"$VAR_PATH\")"
+        ${pkgs.nix}/bin/nix run --show-trace --impure --expr "((builtins.getFlake \"$(pwd)\").vmBuilder.x86_64-linux \"$HOSTNAME\" $NETWORK $DUMMY_SECRETS $NO_DISKO \"$VAR_PATH\" $RW_STORE).config.microvm.declaredRunner"
       '';
     };
 
@@ -206,6 +366,15 @@ in (utils.lib.eachSystem (builtins.filter filter_system utils.lib.defaultSystems
       docs = {
         type = "app";
         program = builtins.toString (pkgs.writeShellScript "docs" ''
+          ${pkgs.xdg-utils}/bin/xdg-open "${self.packages.${system}.docs}/share/doc/index.html"
+        '');
+      };
+
+
+      docsDev = {
+        type = "app";
+        program = builtins.toString (pkgs.writeShellScript "docs" ''
+          echo "needs to run from infrastuctre root folder"
           ${pkgs.mdbook}/bin/mdbook serve --open ./doc
         '');
       };
@@ -227,6 +396,7 @@ in (utils.lib.eachSystem (builtins.filter filter_system utils.lib.defaultSystems
       microvm.imports = [ ./machines/modules/malobeo/microvm_host.nix ];
       vpn.imports = [ ./machines/modules/malobeo/wireguard.nix ];
       initssh.imports = [ ./machines/modules/malobeo/initssh.nix ];
+      metrics.imports = [ ./machines/modules/malobeo/metrics.nix ];
       disko.imports = [ ./machines/modules/disko ];
     };