[nix] mv buildVM to host_builder

flake.lock

@@ -341,11 +341,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1737548421,
+        "lastModified": 1736184101,
-        "narHash": "sha256-gmlqJdC+v86vXc2yMhiza1mvsqh3vMfrEsiw+tV5MXg=",
+        "narHash": "sha256-HAX+TkDXzyNp6SAsKwjNFql7KzAtxximpQSv+GmP8KQ=",
         "ref": "refs/heads/master",
-        "rev": "c5fff78c83959841ac724980a13597dcfa6dc26d",
+        "rev": "9cdab949f44301553e3817cf1f38287ad947e00c",
-        "revCount": 29,
+        "revCount": 28,
         "type": "git",
         "url": "https://git.dynamicdiscord.de/kalipso/tasklist"
       },

machines/configuration.nix

@@ -0,0 +1,211 @@
+{ self
+, nixpkgs-unstable
+, nixpkgs
+, sops-nix
+, inputs
+, nixos-hardware
+, home-manager
+, ...
+}:
+let
+  nixosSystem = nixpkgs.lib.makeOverridable nixpkgs.lib.nixosSystem;
+  nixosSystemUnstable = nixpkgs-unstable.lib.makeOverridable nixpkgs-unstable.lib.nixosSystem;
+
+  baseModules = [
+    # make flake inputs accessiable in NixOS
+    { _module.args.inputs = inputs; }
+    {
+      imports = [
+        ({ pkgs, ... }: {
+          nix = {
+            extraOptions = ''
+              experimental-features = nix-command flakes
+            '';
+
+            settings = {
+              substituters = [
+                "https://cache.dynamicdiscord.de"
+                "https://cache.nixos.org/"
+              ];
+              trusted-public-keys = [
+                "cache.dynamicdiscord.de:DKueZicqi2NhJJXz9MYgUbiyobMs10fTyHCgAUibRP4="
+              ];
+              trusted-users = [ "root" "@wheel" ];
+            };
+          };
+        })
+
+        sops-nix.nixosModules.sops
+      ];
+    }
+  ];
+  defaultModules = baseModules;
+
+  makeMicroVM = hostName: ipv4Addr: macAddr: modules: [
+    inputs.microvm.nixosModules.microvm
+    {
+      microvm = {
+        hypervisor = "cloud-hypervisor";
+        mem = 2560;
+        shares = [
+          {
+            source = "/nix/store";
+            mountPoint = "/nix/.ro-store";
+            tag = "store";
+            proto = "virtiofs";
+            socket = "store.socket";
+          }
+          {
+            source = "/var/lib/microvms/${hostName}/etc";
+            mountPoint = "/etc";
+            tag = "etc";
+            proto = "virtiofs";
+            socket = "etc.socket";
+          }
+          {
+            source = "/var/lib/microvms/${hostName}/var";
+            mountPoint = "/var";
+            tag = "var";
+            proto = "virtiofs";
+            socket = "var.socket";
+          }
+        ];
+
+        interfaces = [
+          {
+            type = "tap";
+            id = "vm-${hostName}";
+            mac = "${macAddr}";
+          }
+        ];
+      };
+
+      systemd.network.enable = true;
+      
+      systemd.network.networks."20-lan" = {
+        matchConfig.Type = "ether";
+        networkConfig = {
+          Address = [ "${ipv4Addr}/24" ];
+          Gateway = "10.0.0.1";
+          DNS = ["1.1.1.1"];
+          DHCP = "no";
+        };
+      };
+    }
+  ] ++ defaultModules ++ modules;
+
+  inputsMod = inputs // { malobeo = self; };
+
+  hosts = import ./modules/host_builder.nix {};
+in
+  builtins.mapAttrs (host: settings: nixosSystem {
+    system = if (settings.type == "rpi") then "aarch64-linux" else "x86_64-linux";
+    specialArgs.inputs = inputs;
+    modules = (if (settings.type != "microvm") then
+      defaultModules ++ [ ./${host}/configuration.nix ]
+      else
+      makeMicroVM "${host}" "${settings.network.address}" "${settings.network.mac}" [
+        ./${host}/configuration.nix
+      ]);
+  }) hosts.malobeo.hosts // 
+  {
+    testvm = nixosSystem {
+      system = "x86_64-linux";
+      specialArgs.inputs = inputs;
+      specialArgs.self = self;
+      modules = defaultModules ++ [ ./testvm ];
+    };
+  }
+#{
+#  louise = nixosSystem {
+#    system = "x86_64-linux";
+#    specialArgs.inputs = inputs;
+#    modules = defaultModules ++ [
+#      ./louise/configuration.nix
+#    ];
+#  };
+#
+#  bakunin = nixosSystem {
+#    system = "x86_64-linux";
+#    specialArgs.inputs = inputs;
+#    modules = defaultModules ++ [
+#      ./bakunin/configuration.nix
+#      inputs.disko.nixosModules.disko
+#    ];
+#  };
+#
+#  lucia = nixosSystem {
+#    system = "aarch64-linux";
+#    specialArgs.inputs = inputs;
+#    modules = defaultModules ++ [
+#      ./lucia/configuration.nix
+#      ./lucia/hardware_configuration.nix
+#    ];
+#  };
+#
+#  fanny = nixosSystem {
+#    system = "x86_64-linux";
+#    specialArgs.inputs = inputsMod;
+#    modules = defaultModules ++ [
+#      self.nixosModules.malobeo.vpn
+#      ./fanny/configuration.nix
+#    ];
+#  };
+#
+#  durruti = nixosSystem {
+#    system = "x86_64-linux";
+#    specialArgs.inputs = inputs;
+#    specialArgs.self = self;
+#    modules = makeMicroVM "durruti" "10.0.0.5" "52:DA:0D:F9:EF:F9" [
+#      ./durruti/configuration.nix
+#    ];
+#  };
+#
+#  vpn = nixosSystem {
+#    system = "x86_64-linux";
+#    specialArgs.inputs = inputs;
+#    specialArgs.self = self;
+#    modules = makeMicroVM "vpn" "10.0.0.10" "D0:E5:CA:F0:D7:E6" [
+#      self.nixosModules.malobeo.vpn
+#      ./vpn/configuration.nix
+#    ];
+#  };
+#
+#  infradocs = nixosSystem {
+#    system = "x86_64-linux";
+#    specialArgs.inputs = inputs;
+#    specialArgs.self = self;
+#    modules = makeMicroVM "infradocs" "10.0.0.11" "D0:E5:CA:F0:D7:E7" [
+#      self.nixosModules.malobeo.vpn
+#      ./infradocs/configuration.nix
+#    ];
+#  };
+#
+#  uptimekuma = nixosSystem {
+#    system = "x86_64-linux";
+#    specialArgs.inputs = inputs;
+#    specialArgs.self = self;
+#    modules = makeMicroVM "uptimekuma" "10.0.0.12" "D0:E5:CA:F0:D7:E8" [
+#      ./uptimekuma/configuration.nix
+#    ];
+#  };
+#
+#  nextcloud = nixosSystem {
+#    system = "x86_64-linux";
+#    specialArgs.inputs = inputs;
+#    specialArgs.self = self;
+#    modules = makeMicroVM "nextcloud" "10.0.0.13" "D0:E5:CA:F0:D7:E9" [
+#      ./nextcloud/configuration.nix
+#    ];
+#  };
+#
+#  overwatch = nixosSystem {
+#    system = "x86_64-linux";
+#    specialArgs.inputs = inputs;
+#    specialArgs.self = self;
+#    modules = makeMicroVM "overwatch" "10.0.0.14" "D0:E5:CA:F0:D7:E0" [
+#      ./overwatch/configuration.nix
+#    ];
+#  };
+#
+#}

machines/fanny/configuration.nix

@@ -69,16 +69,9 @@ in
   };
 
   services.malobeo.microvm.enableHostBridge = true;
-  services.malobeo.microvm.deployHosts = [ "overwatch" "infradocs" "durruti" ];
+  services.malobeo.microvm.deployHosts = [ "overwatch" "infradocs" "nextcloud" "durruti" ];
-  services.malobeo.microvm.client.nextcloud.enable = true;
 
   networking = {
-    nat = {
-      enable = true;
-      externalInterface = "enp1s0";
-      internalInterfaces = [ "microvm" ];
-    };
-
     firewall = {
       allowedTCPPorts = [ 80 ];
     };

machines/hosts.nix

@@ -66,10 +66,6 @@
           mac = "D0:E5:CA:F0:D7:E0";
         };
       };
-
-      testvm = {
-        type = "host";
-      };
     };
   };
 }

machines/modules/disko/btrfs-laptop.nix

@@ -0,0 +1,63 @@
+{ config, self, inputs, ... }:
+
+{
+  imports = [
+    inputs.disko.nixosModules.disko
+  ];
+
+  # https://github.com/nix-community/disko/blob/master/example/luks-btrfs-subvolumes.nix
+  disko.devices = {
+    disk = {
+      main = {
+        type = "disk";
+        # When using disko-install, we will overwrite this value from the commandline
+        device = "/dev/disk/by-id/some-disk-id";
+        content = {
+          type = "gpt";
+          partitions = {
+            ESP = {
+              size = "512M";
+              type = "EF00";
+              content = {
+                type = "filesystem";
+                format = "vfat";
+                mountpoint = "/boot";
+                mountOptions = [ "umask=0077" ];
+              };
+            };
+            luks = {
+              size = "100%";
+              content = {
+                type = "luks";
+                name = "crypted";
+                passwordFile = /tmp/secret.key; # Interactive
+                content = {
+                  type = "btrfs";
+                  extraArgs = [ "-f" ];
+                  subvolumes = {
+                    "/root" = {
+                      mountpoint = "/";
+                      mountOptions = [ "compress=zstd" "noatime" ];
+                    };
+                    "/home" = {
+                      mountpoint = "/home";
+                      mountOptions = [ "compress=zstd" "noatime" ];
+                    };
+                    "/nix" = {
+                      mountpoint = "/nix";
+                      mountOptions = [ "compress=zstd" "noatime" ];
+                    };
+                    "/swap" = {
+                      mountpoint = "/.swapvol";
+                      swap.swapfile.size = "20M";
+                    };
+                  };
+                };
+              };
+            };
+          };
+        };
+      };
+    };
+  };
+}

machines/modules/disko/default.nix

@@ -187,7 +187,6 @@ in
               postCreateHook = lib.mkIf cfg.encryption ''
                 zfs set keylocation="prompt" zroot/encrypted;
               '';
-
             };
             "encrypted/root" = {
               type = "zfs_fs";
@@ -245,12 +244,13 @@ in
               };
               # use this to read the key during boot
               postCreateHook = lib.mkIf cfg.encryption ''
-                zfs set keylocation="file:///root/secret.key" storage/encrypted;
+                zfs set keylocation="prompt" storage/encrypted;
               '';
             };
             "encrypted/data" = {
               type = "zfs_fs";
               mountpoint = "/data";
+              options.mountpoint = "legacy";
             };
             reserved = {
               # for cow delete if pool is full
@@ -267,7 +267,7 @@ in
     };
 
     boot.zfs.devNodes = lib.mkDefault cfg.devNodes;
-    boot.zfs.extraPools = lib.mkIf cfg.storage.enable [ "storage" ];
+
     fileSystems."/".neededForBoot = true;
     fileSystems."/etc".neededForBoot = true;
     fileSystems."/boot".neededForBoot = true;

machines/modules/host_builder.nix

@@ -135,8 +135,8 @@ rec {
         }]);
 
         #if networking is disabled forward port 80 to still have access to webservices
-        forwardPorts = pkgs.lib.mkIf (!options.withNetworking && options.fwdPort != 0) (pkgs.lib.mkForce [
+        forwardPorts = pkgs.lib.mkIf (!options.withNetworking) (pkgs.lib.mkForce [
-          { from = "host"; host.port = options.fwdPort; guest.port = 80; }
+          { from = "host"; host.port = 8080; guest.port = 80; }
         ]);
 
       };
@@ -212,13 +212,12 @@ rec {
       builtins.listToAttrs (map mapperFunc self.nixosConfigurations.${host}.config.services.malobeo.microvm.deployHosts);
     };
 
-    buildVM = host: networking: sopsDummy: disableDisko: varPath: writableStore: fwdPort: (self.nixosConfigurations.${host}.extendModules {
+    buildVM = host: networking: sopsDummy: disableDisko: varPath: writableStore: (self.nixosConfigurations.${host}.extendModules {
         modules = [
             (vmMicroVMOverwrites host { 
               withNetworking = networking; 
               varPath = "${varPath}"; 
-              writableStore = writableStore;
+              writableStore = writableStore; })
-              fwdPort = fwdPort; })
             (if sopsDummy then (vmSopsOverwrites host) else {})
             (if disableDisko then vmDiskoOverwrites else {})
           ] ++ pkgs.lib.optionals (hosts.malobeo.hosts.${host}.type != "microvm") [

machines/modules/malobeo/initssh.nix

@@ -30,7 +30,9 @@ in
       loader.efi.canTouchEfiVariables = true;
       supportedFilesystems = [ "vfat" "zfs" ];
       zfs = {
+        forceImportAll = true;
         requestEncryptionCredentials = true;
+        
       };
       initrd = {
         availableKernelModules = cfg.ethernetDrivers;

machines/modules/malobeo/microvm_client.nix

@@ -1,28 +0,0 @@
-{config, lib, pkgs, ...}:
-let
-  cfg = config.services.malobeo.microvm.client;
-in
-{
-  options.services.malobeo.microvm.client = {
-    nextcloud = {
-      enable = lib.mkEnableOption "enable the nextcloud microvm wrapper";
-      datadir = lib.mkOption {
-        type = lib.types.string;
-        default = "/data/services/nextcloud/";
-        description = "set a custom datadir";
-      };
-    };
-  };
-
-  config = lib.mkMerge [
-    (lib.mkIf cfg.nextcloud.enable { #add check for run-vm?
-      services.malobeo.microvm.deployHosts = ["nextcloud"];
-      microvm.vms.nextcloud.config.microvm.shares = lib.mkAfter [{
-        source = cfg.datadir;
-        mountPoint = "/datadir";
-        tag = "nc-datadir";
-        proto = "virtiofs";
-      }];
-    })
-  ];
-}

machines/nextcloud/configuration.nix

@@ -37,7 +37,6 @@ with lib;
     hostName = "cloud.malobeo.org";
     config.adminpassFile = config.sops.secrets.nextcloudAdminPass.path;
     #https = true; #disable for testing
-    datadir = "/datadir";
     database.createLocally = true;
     config.dbtype = "pgsql";
     configureRedis = true;
@@ -47,7 +46,7 @@ with lib;
     };
     extraAppsEnable = true;
     extraApps = {
-      inherit (config.services.nextcloud.package.packages.apps) contacts calendar deck polls;
+      inherit (config.services.nextcloud.package.packages.apps) contacts calendar;
       collectives = pkgs.fetchNextcloudApp {
         sha256 = "sha256-cj/8FhzxOACJaUEu0eG9r7iAQmnOG62yFHeyUICalFY=";
         url = "https://github.com/nextcloud/collectives/releases/download/v2.15.2/collectives-2.15.2.tar.gz";
@@ -56,12 +55,6 @@ with lib;
     };
     settings = {
       trusted_domains = ["10.0.0.13"];
-      "maintenance_window_start" = "1";
-      "default_phone_region" = "DE";
-    };
-    phpOptions = {
-      "realpath_cache_size" = "0";
-      "opcache.interned_strings_buffer" = "23";
     };
   };
 

machines/overwatch/loki.yaml

@@ -3,7 +3,7 @@ auth_enabled: false
 server:
   http_listen_port: 3100
   grpc_listen_port: 9096
-  log_level: info
+  log_level: debug
   grpc_server_max_concurrent_streams: 1000
 
 common:

machines/testvm/default.nix

@@ -24,7 +24,7 @@ in
 
   malobeo.disks = {
     enable = true;
-    encryption = true;
+    encryption = false;
     hostId = "83abc8cb";
     devNodes = "/dev/disk/by-path/";
     root = {

outputs.nix

@@ -72,7 +72,64 @@ in (utils.lib.eachSystem (builtins.filter filter_system utils.lib.defaultSystems
         '';
       };
 
-      run-vm = pkgs.writeShellScriptBin "run-vm"  (builtins.readFile ./scripts/run-vm.sh);
+      run-vm = pkgs.writeShellScriptBin "run-vm" ''
+        usage() {
+            echo "Usage: run-vm <hostname> [--networking] [--dummy-secrets] [--no-disko]"
+            echo "ATTENTION: This script must be run from the flakes root directory"
+            echo "--networking setup interfaces. requires root and hostbridge enabled on the host"
+            echo "--dummy-secrets run vm with dummy sops secrets"
+            echo "--no-disko disable disko and initrd secrets. needed for real hosts like fanny"
+            echo "--writable-store enables writable store. necessary for host with nested imperative microvms like fanny"
+            echo "--var path to directory that should be shared as /var. may require root otherwise some systemd units fail within vm. if dir is empty vm will populate"
+            exit 1
+        }
+        
+        # check at least one arg was given
+        if [ "$#" -lt 1 ]; then
+            usage
+        fi
+        
+        HOSTNAME=$1
+        
+        # Optionale Argumente
+        NETWORK=false
+        DUMMY_SECRETS=false
+        NO_DISKO=false
+        RW_STORE=false
+        VAR_PATH=""
+        
+        # check argws
+        shift
+        while [[ "$#" -gt 0 ]]; do
+            case $1 in
+                --networking) NETWORK=true ;;
+                --dummy-secrets) DUMMY_SECRETS=true ;;
+                --no-disko) NO_DISKO=true ;;
+                --writable-store) RW_STORE=true ;;
+                --var) 
+                  if [[ -n "$2" && ! "$2" =~ ^- ]]; then
+                      VAR_PATH="$2"
+                      shift
+                  else
+                      echo "Error: --var requires a non-empty string argument."
+                      usage
+                  fi
+                  ;;
+                *) echo "Unknown argument: $1"; usage ;;
+            esac
+            shift
+        done
+        echo "starting host $HOSTNAME"
+        echo "enable networking: $NETWORK"
+        echo "deploy dummy secrets: $DUMMY_SECRETS"
+        echo "disable disko and initrd secrets: $NO_DISKO"
+        echo "use writable store: $RW_STORE"
+        if [ -n "$VAR_PATH" ]; then
+          echo "sharing var directory: $VAR_PATH"
+        fi
+        
+        ${pkgs.nix}/bin/nix run --show-trace --impure --expr "((builtins.getFlake \"$(pwd)\").vmBuilder.x86_64-linux \"$HOSTNAME\" $NETWORK $DUMMY_SECRETS $NO_DISKO \"$VAR_PATH\" $RW_STORE).config.microvm.declaredRunner"
+      '';
     };
 
     apps = {
@@ -105,10 +162,14 @@ in (utils.lib.eachSystem (builtins.filter filter_system utils.lib.defaultSystems
   in
   {
     nixosConfigurations = utils.buildHost hosts.malobeo.hosts;
+    #nixosConfigurations = import ./machines/configuration.nix (inputs // {
+    #  inherit inputs;
+    #  self = self;
+    #});
 
     nixosModules.malobeo = {
       host.imports = [ ./machines/durruti/host_config.nix ];
-      microvm.imports = [ ./machines/modules/malobeo/microvm_host.nix ./machines/modules/malobeo/microvm_client.nix];
+      microvm.imports = [ ./machines/modules/malobeo/microvm_host.nix ];
       vpn.imports = [ ./machines/modules/malobeo/wireguard.nix ];
       initssh.imports = [ ./machines/modules/malobeo/initssh.nix ];
       metrics.imports = [ ./machines/modules/malobeo/metrics.nix ];

scripts/remote-install-encrypt.sh

@@ -37,11 +37,9 @@ trap cleanup EXIT
 
 # Create the directory where sshd expects to find the host keys
 install -d -m755 "$temp/etc/ssh/"
-install -d -m755 "$temp/root/"
 
 diskKey=$(sops -d machines/$hostname/disk.key)
 echo "$diskKey" > /tmp/secret.key
-echo "$diskKey" > $temp/root/secret.key
 
 ssh-keygen -f $temp/etc/ssh/"$hostname" -t ed25519 -N ""
 ssh-keygen -f $temp/etc/ssh/initrd -t ed25519 -N ""

scripts/run-vm.sh

@@ -1,67 +0,0 @@
-usage() {
-    echo "Usage: run-vm <hostname> [--networking] [--dummy-secrets] [--no-disko]"
-    echo "ATTENTION: This script must be run from the flakes root directory"
-    echo "--networking setup interfaces. requires root and hostbridge enabled on the host"
-    echo "--dummy-secrets run vm with dummy sops secrets"
-    echo "--no-disko disable disko and initrd secrets. needed for real hosts like fanny"
-    echo "--writable-store enables writable store. necessary for host with nested imperative microvms like fanny"
-    echo "--var path to directory that should be shared as /var. may require root otherwise some systemd units fail within vm. if dir is empty vm will populate"
-    echo "--fwd-port forwards the given port to port 80 on vm"
-    exit 1
-}
-
-# check at least one arg was given
-if [ "$#" -lt 1 ]; then
-    usage
-fi
-
-HOSTNAME=$1
-
-# Optionale Argumente
-NETWORK=false
-DUMMY_SECRETS=false
-NO_DISKO=false
-RW_STORE=false
-VAR_PATH=""
-FWD_PORT=0
-
-# check argws
-shift
-while [[ "$#" -gt 0 ]]; do
-    case $1 in
-        --networking) NETWORK=true ;;
-        --dummy-secrets) DUMMY_SECRETS=true ;;
-        --no-disko) NO_DISKO=true ;;
-        --writable-store) RW_STORE=true ;;
-        --var) 
-          if [[ -n "$2" && ! "$2" =~ ^- ]]; then
-              VAR_PATH="$2"
-              shift
-          else
-              echo "Error: --var requires a non-empty string argument."
-              usage
-          fi
-          ;;
-        --fwd-port) 
-          if [[ -n "$2" && ! "$2" =~ ^- ]]; then
-              FWD_PORT="$2"
-              shift
-          else
-              echo "Error: --var requires a non-empty string argument."
-              usage
-          fi
-          ;;
-        *) echo "Unknown argument: $1"; usage ;;
-    esac
-    shift
-done
-echo "starting host $HOSTNAME"
-echo "enable networking: $NETWORK"
-echo "deploy dummy secrets: $DUMMY_SECRETS"
-echo "disable disko and initrd secrets: $NO_DISKO"
-echo "use writable store: $RW_STORE"
-if [ -n "$VAR_PATH" ]; then
-  echo "sharing var directory: $VAR_PATH"
-fi
-
-nix run --show-trace --impure --expr "((builtins.getFlake \"$(pwd)\").vmBuilder.x86_64-linux \"$HOSTNAME\" $NETWORK $DUMMY_SECRETS $NO_DISKO \"$VAR_PATH\" $RW_STORE $FWD_PORT).config.microvm.declaredRunner"

scripts/unlock-boot.sh

@@ -23,14 +23,18 @@ echo
 if [ $# = 1 ]
     then
     diskkey=$(sops -d machines/$HOSTNAME/disk.key)
+    echo "$diskkey" | ssh $sshoptions root@$HOSTNAME-initrd "systemd-tty-ask-password-agent" #storage
+
     echo "$diskkey" | ssh $sshoptions root@$HOSTNAME-initrd "systemd-tty-ask-password-agent" #root
 
 elif [ $# = 2 ]
     then
     diskkey=$(sops -d machines/$HOSTNAME/disk.key)
     IP=$2
+    echo "$diskkey" | ssh $sshoptions root@$IP "systemd-tty-ask-password-agent" #storage
+
     echo "$diskkey" | ssh $sshoptions root@$IP "systemd-tty-ask-password-agent" #root
-    
+
 else
     echo
     echo "Unlock the root disk on a remote host."