[run-vm] handle edgecase for prometheus mmaped file on 9p share

sharing /var somehow doesnt work. for example nginx fails because of
lacking permissions to access /var/log/nginx. this also happens when
run-vm is started as root. thats why only /var/lib is shared which still
allows application persistency between tests

machines/configuration.nix

@@ -43,7 +43,6 @@ let
   defaultModules = baseModules;
 
   makeMicroVM = hostName: ipv4Addr: macAddr: modules: [
-    self.nixosModules.malobeo.metrics
     {
       microvm = {
         hypervisor = "cloud-hypervisor";
@@ -81,13 +80,6 @@ let
         ];
       };
       
-      malobeo.metrics = {
-        enable = true;
-        enablePromtail = true;
-        logNginx = false;
-        lokiHost = "10.0.0.14";
-      };
-      
       systemd.network.enable = true;
       
       systemd.network.networks."20-lan" = {
@@ -187,19 +179,11 @@ in
     ];
   };
 
-  overwatch = nixosSystem {
-    system = "x86_64-linux";
-    specialArgs.inputs = inputs;
-    specialArgs.self = self;
-    modules = makeMicroVM "overwatch" "10.0.0.14" "D0:E5:CA:F0:D7:E0" [
-      ./overwatch/configuration.nix
-    ];
-  };
-
   testvm = nixosSystem {
     system = "x86_64-linux";
     specialArgs.inputs = inputs;
     specialArgs.self = self;
     modules = defaultModules ++ [ ./testvm ];
   };
+
 }

machines/durruti/configuration.nix

@@ -6,6 +6,7 @@ with lib;
   networking = {
     hostName = mkDefault "durruti";
     useDHCP = false;
+    nameservers = [ "1.1.1.1" ];
   };
 
   networking.firewall.allowedTCPPorts = [ 8080 ];

machines/fanny/configuration.nix

@@ -53,7 +53,7 @@ in
   };
 
   services.malobeo.microvm.enableHostBridge = true;
-  services.malobeo.microvm.deployHosts = [ "infradocs" "nextcloud" "overwatch" ];
+  services.malobeo.microvm.deployHosts = [ "infradocs" "nextcloud" ];
 
   networking = {
     firewall = {

machines/infradocs/configuration.nix

@@ -6,6 +6,7 @@ with lib;
   networking = {
     hostName = mkDefault "infradocs";
     useDHCP = false;
+    nameservers = [ "1.1.1.1" ];
   };
 
   imports = [
@@ -14,12 +15,6 @@ with lib;
     ../modules/sshd.nix
   ];
 
-  networking.firewall.allowedTCPPorts = [ 9002 ];
-
-  malobeo.metrics.logNginx = lib.mkForce true;
-
-  users.users.promtail.extraGroups = [ "nginx" "systemd-journal" ];
-
   system.stateVersion = "22.11"; # Did you read the comment?
 }
 

machines/modules/malobeo/metrics.nix

@@ -1,56 +0,0 @@
-{ config, lib, pkgs, ... }:
-let 
-  cfg = config.malobeo.metrics;
-in
-{
-  options.malobeo.metrics = {
-    enable = lib.mkOption {
-      type = lib.types.bool;
-      default = false;
-      description = "Enable sharing metrics";
-    };
-    enablePromtail = lib.mkOption {
-      type = lib.types.bool;
-      default = true;
-      description = "Enable sharing logs";
-    };
-    logNginx = lib.mkOption {
-      type = lib.types.bool;
-      default = false;
-      description = "Share nginx logs";
-    };
-    lokiHost = lib.mkOption {
-      type = lib.types.str;
-      default = "10.0.0.14";
-      description = "Address of loki host";
-    };
-  };
-  
-  config = lib.mkIf (cfg.enable) {
-
-    networking.firewall.allowedTCPPorts = [ 9002 ];
-
-    services.prometheus = {
-      exporters = {
-        node = {
-          enable = true;
-          enabledCollectors = [ "systemd" "processes" ];
-          port = 9002;
-        };
-      };
-    };
-
-    services.promtail = {
-      enable = cfg.enablePromtail;
-      configFile = import ./promtail_config.nix { 
-        lokiAddress = cfg.lokiHost;
-        logNginx = cfg.logNginx;
-        config = config;
-        pkgs = pkgs;
-      };
-    };
-
-    users.users.promtail.extraGroups = [ "systemd-journal" ] ++ (lib.optionals cfg.logNginx [ "nginx" ]) ;
-
-  };
-}

machines/modules/malobeo/microvm_host.nix

@@ -86,12 +86,6 @@ in
     in
     builtins.listToAttrs (map mapperFunc cfg.deployHosts);
 
-    systemd.tmpfiles.rules = builtins.concatLists (map (name: [
-      "d /var/lib/microvms/${name}/var 0755 root root - -"
-      "d /var/lib/microvms/${name}/etc 0755 root root - -"
-      "d /${name} 0755 root root - -"
-    ]) cfg.deployHosts);
-
     systemd.services = builtins.foldl' (services: name: services // {
       "microvm-update@${name}" = {
         description = "Update MicroVMs automatically";

machines/modules/malobeo/promtail_config.nix

@@ -1,49 +0,0 @@
-{ logNginx, lokiAddress, config, pkgs, ... }:
-
-let
-  basecfg = ''
-    server:
-      http_listen_port: 9080
-      grpc_listen_port: 0
-    
-    positions:
-      filename: /tmp/positions.yaml
-    
-    clients:
-      - url: http://${lokiAddress}:3100/loki/api/v1/push
-  '';
-
-  withNginx = ''
-    scrape_configs:
-      - job_name: journal
-        journal:
-          max_age: 12h
-          labels:
-            job: systemd-journal
-            host: ${config.networking.hostName}
-        relabel_configs:
-          - source_labels: ["__journal__systemd_unit"]
-            target_label: "unit"
-      - job_name: nginx
-        static_configs:
-        - targets:
-            - localhost
-          labels:
-            job: nginx
-            __path__: /var/log/nginx/*log
-  '';
-
-  withoutNginx = ''
-    scrape_configs:
-      - job_name: journal
-        journal:
-          max_age: 12h
-          labels:
-            job: systemd-journal
-            host: ${config.networking.hostName}
-        relabel_configs:
-          - source_labels: ["__journal__systemd_unit"]
-            target_label: "unit"
-  '';
-in
-pkgs.writeText "promtailcfg.yaml" (if logNginx then ''${basecfg}${withNginx}'' else ''${basecfg}${withoutNginx}'')

machines/overwatch/configuration.nix

@@ -1,115 +0,0 @@
-{ config, lib, pkgs, inputs, ... }:
-
-with lib;
-
-{
-  networking = {
-    hostName = mkDefault "overwatch";
-    useDHCP = false;
-  };
-
-  imports = [
-    ../modules/malobeo_user.nix
-    ../modules/sshd.nix
-  ];
-
-  networking.firewall.allowedTCPPorts = [ 80 9080 9001 3100 ];
-
-  services.grafana = {
-    enable = true;
-    domain = "grafana.malobeo.org";
-    port = 2342;
-    addr = "127.0.0.1";
-
-    provision.datasources.settings = {
-      apiVersion = 1;
-
-      datasources = [
-        {
-          name = "loki";
-          type = "loki";
-          access = "proxy";
-          uid = "eeakiack8nqwwc";
-          url = "http://localhost:3100";
-          editable = false;
-        }
-        {
-          name = "prometheus";
-          type = "prometheus";
-          access = "proxy";
-          uid = "feakib1gq7ugwc";
-          url = "http://localhost:9001";
-          editable = false;
-
-        }
-      ];
-    };
-
-    provision.dashboards.settings = {
-      apiVersion = 1;
-      providers = [{
-        name = "default";
-        options.path = ./dashboards;
-      }];
-    };
-  };
-
-  services.nginx = {
-    enable = true;
-    virtualHosts.${config.services.grafana.domain} = {
-      locations."/" = {
-          proxyPass = "http://127.0.0.1:${toString config.services.grafana.port}";
-          proxyWebsockets = true;
-
-      extraConfig = ''
-        proxy_set_header Host $host;
-      '';
-    };
-      };
-  };
-
-  services.prometheus = {
-    enable = true;
-    port = 9001;
-
-    scrapeConfigs = [
-      {
-        job_name = "overwatch";
-        static_configs = [{
-          targets = [ "127.0.0.1:9002" ];
-        }];
-      }
-      {
-        job_name = "durruti";
-        static_configs = [{
-          targets = [ "10.0.0.5:9002" ];
-        }];
-      }
-      {
-        job_name = "infradocs";
-        static_configs = [{
-          targets = [ "10.0.0.11:9002" ];
-        }];
-      }
-      {
-        job_name = "nextcloud";
-        static_configs = [{
-          targets = [ "10.0.0.13:9002" ];
-        }];
-      }
-      # add vpn - check how to reach it first. most probably 10.100.0.1
-    ];
-  };
-
-  services.loki = {
-    enable = true;
-    configFile = ./loki.yaml;
-  };
-
-  users.users.promtail.extraGroups = [ "nginx" "systemd-journal" ];
-
-
-
-  system.stateVersion = "22.11"; # Did you read the comment?
-}
-

machines/overwatch/loki.yaml

@@ -1,60 +0,0 @@
-auth_enabled: false
-
-server:
-  http_listen_port: 3100
-  grpc_listen_port: 9096
-  log_level: debug
-  grpc_server_max_concurrent_streams: 1000
-
-common:
-  instance_addr: 127.0.0.1
-  path_prefix: /tmp/loki
-  storage:
-    filesystem:
-      chunks_directory: /tmp/loki/chunks
-      rules_directory: /tmp/loki/rules
-  replication_factor: 1
-  ring:
-    kvstore:
-      store: inmemory
-
-query_range:
-  results_cache:
-    cache:
-      embedded_cache:
-        enabled: true
-        max_size_mb: 100
-
-schema_config:
-  configs:
-    - from: 2020-10-24
-      store: tsdb
-      object_store: filesystem
-      schema: v13
-      index:
-        prefix: index_
-        period: 24h
-
-pattern_ingester:
-  enabled: true
-  metric_aggregation:
-    loki_address: localhost:3100
-
-ruler:
-  alertmanager_url: http://localhost:9093
-
-frontend:
-  encoding: protobuf
-
-# By default, Loki will send anonymous, but uniquely-identifiable usage and configuration
-# analytics to Grafana Labs. These statistics are sent to https://stats.grafana.org/
-#
-# Statistics help us better understand how Loki is used, and they show us performance
-# levels for most users. This helps us prioritize features and documentation.
-# For more information on what's sent, look at
-# https://github.com/grafana/loki/blob/main/pkg/analytics/stats.go
-# Refer to the buildReport method to see what goes into a report.
-#
-# If you would like to disable reporting, uncomment the following lines:
-analytics:
-  reporting_enabled: false

machines/overwatch/promtail.yaml

@@ -1,29 +0,0 @@
-server:
-  http_listen_port: 9080
-  grpc_listen_port: 0
-
-positions:
-  filename: /tmp/positions.yaml
-
-clients:
-  - url: http://10.0.0.13:3100/loki/api/v1/push
-
-  
-scrape_configs:
-  - job_name: journal
-    journal:
-      max_age: 12h
-      labels:
-        job: systemd-journal
-        host: overwatch
-    relabel_configs:
-      - source_labels: ["__journal__systemd_unit"]
-        target_label: "unit"
-  - job_name: nginx
-    static_configs:
-    - targets:
-        - localhost
-      labels:
-        job: nginx
-        __path__: /var/log/nginx/*log
-

machines/uptimekuma/configuration.nix

@@ -6,6 +6,7 @@ with lib;
   networking = {
     hostName = mkDefault "uptimekuma";
     useDHCP = false;
+    nameservers = [ "1.1.1.1" ];
   };
 
   imports = [

machines/vpn/configuration.nix

@@ -17,7 +17,6 @@ with lib;
   };
 
   imports = [
-    inputs.self.nixosModules.malobeo.vpn
     ../modules/malobeo_user.nix
     ../modules/sshd.nix
     ../modules/minimal_tools.nix

outputs.nix

@@ -12,97 +12,6 @@
 let filter_system = name: if name == utils.lib.system.i686-linux then false else true;
 in (utils.lib.eachSystem (builtins.filter filter_system utils.lib.defaultSystems) ( system:
   let
-    baseModules = [
-      # make flake inputs accessiable in NixOS
-      { _module.args.inputs = inputs; }
-      {
-        imports = [
-          ({ pkgs, ... }: {
-            nix = {
-              extraOptions = ''
-                experimental-features = nix-command flakes
-              '';
-
-              settings = {
-                substituters = [
-                  "https://cache.dynamicdiscord.de"
-                  "https://cache.nixos.org/"
-                ];
-                trusted-public-keys = [
-                  "cache.dynamicdiscord.de:DKueZicqi2NhJJXz9MYgUbiyobMs10fTyHCgAUibRP4="
-                ];
-                trusted-users = [ "root" "@wheel" ];
-              };
-            };
-          })
-          sops-nix.nixosModules.sops
-          #microvm.nixosModules.microvm
-        ];
-      }
-    ];
-    defaultModules = baseModules;
-
-  makeMicroVM = hostName: ipv4Addr: macAddr: modules: [
-    self.nixosModules.malobeo.metrics
-    {
-      microvm = {
-        hypervisor = "cloud-hypervisor";
-        mem = 2560;
-        shares = [
-          {
-            source = "/nix/store";
-            mountPoint = "/nix/.ro-store";
-            tag = "store";
-            proto = "virtiofs";
-            socket = "store.socket";
-          }
-          {
-            source = "/var/lib/microvms/${hostName}/etc";
-            mountPoint = "/etc";
-            tag = "etc";
-            proto = "virtiofs";
-            socket = "etc.socket";
-          }
-          {
-            source = "/var/lib/microvms/${hostName}/var";
-            mountPoint = "/var";
-            tag = "var";
-            proto = "virtiofs";
-            socket = "var.socket";
-          }
-        ];
-
-        interfaces = [
-          {
-            type = "tap";
-            id = "vm-${hostName}";
-            mac = "${macAddr}";
-          }
-        ];
-      };
-
-      malobeo.metrics = {
-        enable = true;
-        enablePromtail = true;
-        logNginx = false;
-        lokiHost = "10.0.0.14";
-      };
-      
-      systemd.network.enable = true;
-      
-      systemd.network.networks."20-lan" = {
-        matchConfig.Type = "ether";
-        networkConfig = {
-          Address = [ "${ipv4Addr}/24" ];
-          Gateway = "10.0.0.1";
-          DNS = ["1.1.1.1"];
-          DHCP = "no";
-        };
-      };
-    }
-  ] ++ defaultModules ++ modules;
-
-
     pkgs-unstable = nixpkgs-unstable.legacyPackages."${system}";
     pkgs = nixpkgs.legacyPackages."${system}";
 
@@ -111,17 +20,7 @@ in (utils.lib.eachSystem (builtins.filter filter_system utils.lib.defaultSystems
         mem = pkgs.lib.mkForce 4096;
         hypervisor = pkgs.lib.mkForce "qemu";
         socket = pkgs.lib.mkForce null;
-
-
-        #needed for hosts that deploy imperative microvms (for example fanny)
-        writableStoreOverlay = pkgs.lib.mkIf options.writableStore "/nix/.rw-store";
-        volumes = pkgs.lib.mkIf options.writableStore [ {
-          image = "nix-store-overlay.img";
-          mountPoint = self.nixosConfigurations.${hostname}.config.microvm.writableStoreOverlay;
-          size = 2048;
-        } ];
-
-        shares = pkgs.lib.mkForce (pkgs.lib.optionals (!options.writableStore) [
+        shares = pkgs.lib.mkForce ([
           {
             tag = "ro-store";
             source = "/nix/store";
@@ -135,18 +34,11 @@ in (utils.lib.eachSystem (builtins.filter filter_system utils.lib.defaultSystems
             tag = "var";
           }
         ]);
-
         interfaces = pkgs.lib.mkIf (!options.withNetworking) (pkgs.lib.mkForce [{
           type = "user";
           id = "eth0";
           mac = "02:23:de:ad:be:ef";
         }]);
-
-        #if networking is disabled forward port 80 to still have access to webservices
-        forwardPorts = pkgs.lib.mkIf (!options.withNetworking) (pkgs.lib.mkForce [
-          { from = "host"; host.port = 8080; guest.port = 80; }
-        ]);
-
       };
 
       fileSystems = {
@@ -194,59 +86,15 @@ in (utils.lib.eachSystem (builtins.filter filter_system utils.lib.defaultSystems
       }];
     };
 
-    buildVM = host: networking: sopsDummy: disableDisko: varPath: writableStore: (self.nixosConfigurations.${host}.extendModules {
+    buildVM = host: networking: sopsDummy: disableDisko: varPath: (self.nixosConfigurations.${host}.extendModules {
         modules = [
-          (vmMicroVMOverwrites host { 
-            withNetworking = networking; 
-            varPath = "${varPath}"; 
-            writableStore = writableStore; })
+          (vmMicroVMOverwrites host { withNetworking = networking; varPath = "${varPath}"; })
           (if sopsDummy then (vmSopsOverwrites host) else {})
           (if disableDisko then vmDiskoOverwrites else {})
         ] ++ pkgs.lib.optionals (! self.nixosConfigurations.${host}.config ? microvm) [
-          #microvm.nixosModules.microvm
-        ] ++ pkgs.lib.optionals (self.nixosConfigurations.${host}.config ? services.malobeo.microvm.deployHosts) [
-          #microvm.nixosModules.host
-          {
-          services.malobeo.microvm.deployHosts = pkgs.lib.mkForce [];
-          systemd.tmpfiles.rules = builtins.concatLists (map (name: [
-            "q /var/lib/microvms/${name}/var 0755 root root - -"
-            "q /var/lib/microvms/${name}/etc 0755 root root - -"
-            "q /var/${name}/wow/it/works 0755 root root - -"
-            "q /var/lib/${name} 0755 root root - -"
-            "d /${name} 0755 root root - -"
-          ]) self.nixosConfigurations.${host}.config.services.malobeo.microvm.deployHosts);
-
-
-          microvm.vms = 
-          let
-            # Map the values to each hostname to then generate an Attrset using listToAttrs
-            mapperFunc = name: { inherit name; value = {
-              #pkgs = import self.nixosConfigurations.${name}.config.nixpkgs;
-
-              #pkgs = (buildVM name networking sopsDummy false "" false).config.nixpkgs;
-              #config = (buildVM name networking sopsDummy false "" false);
-              #pkgs = pkgs;
-              #config = self.nixosConfigurations.${name};
-              specialArgs.inputs = inputs;
-              specialArgs.self = self;
-              config = {
-                imports = (makeMicroVM "${name}" "10.0.0.11" "D0:E5:CA:F0:D7:E7" [
-              
-                  #(vmMicroVMOverwrites name { 
-                  #  withNetworking = true; 
-                  #  varPath = ""; 
-                  #  writableStore = false; })
-                  (if sopsDummy then (vmSopsOverwrites name) else {})
-
-                  
-                ]);
-
-              };
-            }; };
-          in
-          builtins.listToAttrs (map mapperFunc self.nixosConfigurations.${host}.config.services.malobeo.microvm.deployHosts);
-        }];
-      });
+          microvm.nixosModules.microvm
+        ];
+      }).config.microvm.declaredRunner;
   in
   {
     devShells.default =
@@ -309,7 +157,6 @@ in (utils.lib.eachSystem (builtins.filter filter_system utils.lib.defaultSystems
             echo "--networking setup interfaces. requires root and hostbridge enabled on the host"
             echo "--dummy-secrets run vm with dummy sops secrets"
             echo "--no-disko disable disko and initrd secrets. needed for real hosts like fanny"
-            echo "--writable-store enables writable store. necessary for host with nested imperative microvms like fanny"
             echo "--var path to directory that should be shared as /var. may require root otherwise some systemd units fail within vm. if dir is empty vm will populate"
             exit 1
         }
@@ -325,7 +172,6 @@ in (utils.lib.eachSystem (builtins.filter filter_system utils.lib.defaultSystems
         NETWORK=false
         DUMMY_SECRETS=false
         NO_DISKO=false
-        RW_STORE=false
         VAR_PATH=""
         
         # check argws
@@ -335,7 +181,6 @@ in (utils.lib.eachSystem (builtins.filter filter_system utils.lib.defaultSystems
                 --networking) NETWORK=true ;;
                 --dummy-secrets) DUMMY_SECRETS=true ;;
                 --no-disko) NO_DISKO=true ;;
-                --writable-store) RW_STORE=true ;;
                 --var) 
                   if [[ -n "$2" && ! "$2" =~ ^- ]]; then
                       VAR_PATH="$2"
@@ -353,12 +198,11 @@ in (utils.lib.eachSystem (builtins.filter filter_system utils.lib.defaultSystems
         echo "enable networking: $NETWORK"
         echo "deploy dummy secrets: $DUMMY_SECRETS"
         echo "disable disko and initrd secrets: $NO_DISKO"
-        echo "use writable store: $RW_STORE"
         if [ -n "$VAR_PATH" ]; then
           echo "sharing var directory: $VAR_PATH"
         fi
         
-        ${pkgs.nix}/bin/nix run --show-trace --impure --expr "((builtins.getFlake \"$(pwd)\").vmBuilder.x86_64-linux \"$HOSTNAME\" $NETWORK $DUMMY_SECRETS $NO_DISKO \"$VAR_PATH\" $RW_STORE).config.microvm.declaredRunner"
+        ${pkgs.nix}/bin/nix run --show-trace --impure --expr "((builtins.getFlake \"$(pwd)\").vmBuilder.x86_64-linux \"$HOSTNAME\" $NETWORK $DUMMY_SECRETS $NO_DISKO \"$VAR_PATH\")"
       '';
     };
 
@@ -396,7 +240,6 @@ in (utils.lib.eachSystem (builtins.filter filter_system utils.lib.defaultSystems
       microvm.imports = [ ./machines/modules/malobeo/microvm_host.nix ];
       vpn.imports = [ ./machines/modules/malobeo/wireguard.nix ];
       initssh.imports = [ ./machines/modules/malobeo/initssh.nix ];
-      metrics.imports = [ ./machines/modules/malobeo/metrics.nix ];
       disko.imports = [ ./machines/modules/disko ];
     };