refactor host creation #62

Merged
kalipso merged 11 commits from host-builder into staging 2025-01-23 16:48:03 +01:00
9 changed files with 385 additions and 340 deletions

View File

@@ -1,198 +0,0 @@
{ self
, nixpkgs-unstable
, nixpkgs
, sops-nix
, inputs
, nixos-hardware
, home-manager
, ...
}:
let
nixosSystem = nixpkgs.lib.makeOverridable nixpkgs.lib.nixosSystem;
nixosSystemUnstable = nixpkgs-unstable.lib.makeOverridable nixpkgs-unstable.lib.nixosSystem;
baseModules = [
# make flake inputs accessiable in NixOS
{ _module.args.inputs = inputs; }
{
imports = [
({ pkgs, ... }: {
nix = {
extraOptions = ''
experimental-features = nix-command flakes
'';
settings = {
substituters = [
"https://cache.dynamicdiscord.de"
"https://cache.nixos.org/"
];
trusted-public-keys = [
"cache.dynamicdiscord.de:DKueZicqi2NhJJXz9MYgUbiyobMs10fTyHCgAUibRP4="
];
trusted-users = [ "root" "@wheel" ];
};
};
})
sops-nix.nixosModules.sops
];
}
];
defaultModules = baseModules;
makeMicroVM = hostName: ipv4Addr: macAddr: modules: [
inputs.microvm.nixosModules.microvm
{
microvm = {
hypervisor = "cloud-hypervisor";
mem = 2560;
shares = [
{
source = "/nix/store";
mountPoint = "/nix/.ro-store";
tag = "store";
proto = "virtiofs";
socket = "store.socket";
}
{
source = "/var/lib/microvms/${hostName}/etc";
mountPoint = "/etc";
tag = "etc";
proto = "virtiofs";
socket = "etc.socket";
}
{
source = "/var/lib/microvms/${hostName}/var";
mountPoint = "/var";
tag = "var";
proto = "virtiofs";
socket = "var.socket";
}
];
interfaces = [
{
type = "tap";
id = "vm-${hostName}";
mac = "${macAddr}";
}
];
};
systemd.network.enable = true;
systemd.network.networks."20-lan" = {
matchConfig.Type = "ether";
networkConfig = {
Address = [ "${ipv4Addr}/24" ];
Gateway = "10.0.0.1";
DNS = ["1.1.1.1"];
DHCP = "no";
};
};
}
] ++ defaultModules ++ modules;
inputsMod = inputs // { malobeo = self; };
in
{
louise = nixosSystem {
system = "x86_64-linux";
specialArgs.inputs = inputs;
modules = defaultModules ++ [
./louise/configuration.nix
];
};
bakunin = nixosSystem {
system = "x86_64-linux";
specialArgs.inputs = inputs;
modules = defaultModules ++ [
./bakunin/configuration.nix
inputs.disko.nixosModules.disko
];
};
lucia = nixosSystem {
system = "aarch64-linux";
specialArgs.inputs = inputs;
modules = defaultModules ++ [
./lucia/configuration.nix
./lucia/hardware_configuration.nix
];
};
fanny = nixosSystem {
system = "x86_64-linux";
specialArgs.inputs = inputsMod;
modules = defaultModules ++ [
self.nixosModules.malobeo.vpn
./fanny/configuration.nix
];
};
durruti = nixosSystem {
system = "x86_64-linux";
specialArgs.inputs = inputs;
specialArgs.self = self;
modules = makeMicroVM "durruti" "10.0.0.5" "52:DA:0D:F9:EF:F9" [
./durruti/configuration.nix
];
};
vpn = nixosSystem {
system = "x86_64-linux";
specialArgs.inputs = inputs;
specialArgs.self = self;
modules = makeMicroVM "vpn" "10.0.0.10" "D0:E5:CA:F0:D7:E6" [
self.nixosModules.malobeo.vpn
./vpn/configuration.nix
];
};
infradocs = nixosSystem {
system = "x86_64-linux";
specialArgs.inputs = inputs;
specialArgs.self = self;
modules = makeMicroVM "infradocs" "10.0.0.11" "D0:E5:CA:F0:D7:E7" [
self.nixosModules.malobeo.vpn
./infradocs/configuration.nix
];
};
uptimekuma = nixosSystem {
system = "x86_64-linux";
specialArgs.inputs = inputs;
specialArgs.self = self;
modules = makeMicroVM "uptimekuma" "10.0.0.12" "D0:E5:CA:F0:D7:E8" [
./uptimekuma/configuration.nix
];
};
nextcloud = nixosSystem {
system = "x86_64-linux";
specialArgs.inputs = inputs;
specialArgs.self = self;
modules = makeMicroVM "nextcloud" "10.0.0.13" "D0:E5:CA:F0:D7:E9" [
./nextcloud/configuration.nix
];
};
overwatch = nixosSystem {
system = "x86_64-linux";
specialArgs.inputs = inputs;
specialArgs.self = self;
modules = makeMicroVM "overwatch" "10.0.0.14" "D0:E5:CA:F0:D7:E0" [
./overwatch/configuration.nix
];
};
testvm = nixosSystem {
system = "x86_64-linux";
specialArgs.inputs = inputs;
specialArgs.self = self;
modules = defaultModules ++ [ ./testvm ];
};
}

View File

@@ -13,6 +13,7 @@ in
../modules/sshd.nix
../modules/minimal_tools.nix
../modules/autoupdate.nix
inputs.self.nixosModules.malobeo.vpn
inputs.self.nixosModules.malobeo.initssh
inputs.self.nixosModules.malobeo.disko
inputs.self.nixosModules.malobeo.microvm

71
machines/hosts.nix Normal file
View File

@@ -0,0 +1,71 @@
{ ... }:
{
malobeo = {
hosts = {
louise = {
type = "host";
};
bakunin = {
type = "host";
};
fanny = {
type = "host";
};
lucia = {
type = "rpi";
};
durruti = {
type = "microvm";
network = {
address = "10.0.0.5";
mac = "52:DA:0D:F9:EF:F9";
};
};
vpn = {
type = "microvm";
network = {
address = "10.0.0.10";
mac = "D0:E5:CA:F0:D7:E6";
};
};
infradocs = {
type = "microvm";
network = {
address = "10.0.0.11";
mac = "D0:E5:CA:F0:D7:E7";
};
};
uptimekuma = {
type = "microvm";
network = {
address = "10.0.0.12";
mac = "D0:E5:CA:F0:D7:E8";
};
};
nextcloud = {
type = "microvm";
network = {
address = "10.0.0.13";
mac = "D0:E5:CA:F0:D7:E9";
};
};
overwatch = {
type = "microvm";
network = {
address = "10.0.0.14";
mac = "D0:E5:CA:F0:D7:E0";
};
};
};
};
}

View File

@@ -9,7 +9,7 @@ with lib;
};
imports = [
self.nixosModules.malobeo.metrics
inputs.malobeo.nixosModules.malobeo.metrics
../durruti/documentation.nix
../modules/malobeo_user.nix
../modules/sshd.nix

View File

@@ -6,6 +6,7 @@ in
{
imports =
[ # Include the results of the hardware scan.
./hardware_configuration.nix
../modules/malobeo_user.nix
];

View File

@@ -0,0 +1,242 @@
{ self
, nixpkgs-unstable
, nixpkgs
, sops-nix
, inputs
, hosts
, ...
}:
let
pkgs = nixpkgs.legacyPackages."x86_64-linux";
in
rec {
nixosSystem = nixpkgs.lib.makeOverridable nixpkgs.lib.nixosSystem;
nixosSystemUnstable = nixpkgs-unstable.lib.makeOverridable nixpkgs-unstable.lib.nixosSystem;
baseModules = [
# make flake inputs accessiable in NixOS
{ _module.args.inputs = inputs; }
{
imports = [
({ pkgs, ... }: {
nix = {
extraOptions = ''
experimental-features = nix-command flakes
'';
settings = {
substituters = [
"https://cache.dynamicdiscord.de"
"https://cache.nixos.org/"
];
trusted-public-keys = [
"cache.dynamicdiscord.de:DKueZicqi2NhJJXz9MYgUbiyobMs10fTyHCgAUibRP4="
];
trusted-users = [ "root" "@wheel" ];
};
};
})
sops-nix.nixosModules.sops
];
}
];
defaultModules = baseModules;
makeMicroVM = hostName: ipv4Addr: macAddr: modules: [
{
microvm = {
hypervisor = "cloud-hypervisor";
mem = 2560;
shares = [
{
source = "/nix/store";
mountPoint = "/nix/.ro-store";
tag = "store";
proto = "virtiofs";
socket = "store.socket";
}
{
source = "/var/lib/microvms/${hostName}/etc";
mountPoint = "/etc";
tag = "etc";
proto = "virtiofs";
socket = "etc.socket";
}
{
source = "/var/lib/microvms/${hostName}/var";
mountPoint = "/var";
tag = "var";
proto = "virtiofs";
socket = "var.socket";
}
];
interfaces = [
{
type = "tap";
id = "vm-${hostName}";
mac = "${macAddr}";
}
];
};
systemd.network.enable = true;
systemd.network.networks."20-lan" = {
matchConfig.Type = "ether";
networkConfig = {
Address = [ "${ipv4Addr}/24" ];
Gateway = "10.0.0.1";
DNS = ["1.1.1.1"];
DHCP = "no";
};
};
}
] ++ defaultModules ++ modules;
inputsMod = inputs // { malobeo = self; };
vmMicroVMOverwrites = hostname: options: {
microvm = rec {
mem = pkgs.lib.mkForce 4096;
hypervisor = pkgs.lib.mkForce "qemu";
socket = pkgs.lib.mkForce null;
#needed for hosts that deploy imperative microvms (for example fanny)
writableStoreOverlay = pkgs.lib.mkIf options.writableStore "/nix/.rw-store";
volumes = pkgs.lib.mkIf options.writableStore [ {
image = "nix-store-overlay.img";
mountPoint = writableStoreOverlay;
size = 2048;
} ];
shares = pkgs.lib.mkForce (pkgs.lib.optionals (!options.writableStore) [
{
tag = "ro-store";
source = "/nix/store";
mountPoint = "/nix/.ro-store";
}
] ++ pkgs.lib.optionals (options.varPath != "") [
{
source = "${options.varPath}";
securityModel = "mapped";
mountPoint = "/var";
tag = "var";
}
]);
interfaces = pkgs.lib.mkIf (!options.withNetworking) (pkgs.lib.mkForce [{
type = "user";
id = "eth0";
mac = "02:23:de:ad:be:ef";
}]);
#if networking is disabled forward port 80 to still have access to webservices
forwardPorts = pkgs.lib.mkIf (!options.withNetworking) (pkgs.lib.mkForce [
{ from = "host"; host.port = 8080; guest.port = 80; }
]);
};
fileSystems = {
"/".fsType = pkgs.lib.mkForce "tmpfs";
# prometheus uses a memory mapped file which doesnt seem supported by 9p shares
# therefore we mount a tmpfs inside the datadir
"/var/lib/prometheus2/data" = pkgs.lib.mkIf (hostname == "overwatch" && options.varPath != "") (pkgs.lib.mkForce {
fsType = pkgs.lib.mkForce "tmpfs";
});
};
boot.isContainer = pkgs.lib.mkForce false;
services.timesyncd.enable = false;
users.users.root.password = "";
services.getty.helpLine = ''
Log in as "root" with an empty password.
Use "reboot" to shut qemu down.
'';
};
vmDiskoOverwrites = {
boot.initrd = {
secrets = pkgs.lib.mkForce {};
network.ssh.enable = pkgs.lib.mkForce false;
};
malobeo.disks.enable = pkgs.lib.mkForce false;
networking.hostId = "a3c3101f";
};
vmSopsOverwrites = host: {
sops.defaultSopsFile = pkgs.lib.mkForce ../${host}/dummy.yaml;
environment.etc = {
devHostKey = {
source = ../secrets/devkey_ed25519;
mode = "0600";
};
};
services.openssh.hostKeys = [{
path = "/etc/devHostKey";
type = "ed25519";
}];
};
vmNestedMicroVMOverwrites = host: sopsDummy: {
services.malobeo.microvm.deployHosts = pkgs.lib.mkForce [];
microvm.vms =
let
# Map the values to each hostname to then generate an Attrset using listToAttrs
mapperFunc = name: { inherit name; value = {
specialArgs.inputs = inputsMod;
specialArgs.self = self;
config = {
imports = (makeMicroVM "${name}"
"${hosts.malobeo.hosts.${name}.network.address}"
"${hosts.malobeo.hosts.${name}.network.mac}" [
../${name}/configuration.nix
(vmMicroVMOverwrites name {
withNetworking = true;
varPath = "";
writableStore = false; })
(if sopsDummy then (vmSopsOverwrites name) else {})
]);
};
}; };
in
builtins.listToAttrs (map mapperFunc self.nixosConfigurations.${host}.config.services.malobeo.microvm.deployHosts);
};
buildVM = host: networking: sopsDummy: disableDisko: varPath: writableStore: (self.nixosConfigurations.${host}.extendModules {
modules = [
(vmMicroVMOverwrites host {
withNetworking = networking;
varPath = "${varPath}";
writableStore = writableStore; })
(if sopsDummy then (vmSopsOverwrites host) else {})
(if disableDisko then vmDiskoOverwrites else {})
] ++ pkgs.lib.optionals (hosts.malobeo.hosts.${host}.type != "microvm") [
inputs.microvm.nixosModules.microvm
] ++ pkgs.lib.optionals (self.nixosConfigurations.${host}.config ? services.malobeo.microvm.deployHosts) [
(vmNestedMicroVMOverwrites host sopsDummy)
];
});
buildHost = hosts: (builtins.mapAttrs (host: settings: nixosSystem {
system = if (settings.type == "rpi") then "aarch64-linux" else "x86_64-linux";
specialArgs.inputs = inputsMod;
specialArgs.self = self;
modules = (if (settings.type != "microvm") then
defaultModules ++ [ ../${host}/configuration.nix ]
else
makeMicroVM "${host}" "${settings.network.address}" "${settings.network.mac}" [
inputs.microvm.nixosModules.microvm
../${host}/configuration.nix
]);
}) hosts);
}

View File

@@ -17,6 +17,7 @@ with lib;
};
imports = [
inputs.self.nixosModules.malobeo.vpn
../modules/malobeo_user.nix
../modules/sshd.nix
../modules/minimal_tools.nix

View File

@@ -15,86 +15,8 @@ in (utils.lib.eachSystem (builtins.filter filter_system utils.lib.defaultSystems
pkgs-unstable = nixpkgs-unstable.legacyPackages."${system}";
pkgs = nixpkgs.legacyPackages."${system}";
vmMicroVMOverwrites = hostname: options: {
microvm = {
mem = pkgs.lib.mkForce 4096;
hypervisor = pkgs.lib.mkForce "qemu";
socket = pkgs.lib.mkForce null;
shares = pkgs.lib.mkForce ([
{
tag = "ro-store";
source = "/nix/store";
mountPoint = "/nix/.ro-store";
}
] ++ pkgs.lib.optionals (options.varPath != "") [
{
source = "${options.varPath}";
securityModel = "mapped";
mountPoint = "/var";
tag = "var";
}
]);
interfaces = pkgs.lib.mkIf (!options.withNetworking) (pkgs.lib.mkForce [{
type = "user";
id = "eth0";
mac = "02:23:de:ad:be:ef";
}]);
};
fileSystems = {
"/".fsType = pkgs.lib.mkForce "tmpfs";
# prometheus uses a memory mapped file which doesnt seem supported by 9p shares
# therefore we mount a tmpfs inside the datadir
"/var/lib/prometheus2/data" = pkgs.lib.mkIf (hostname == "overwatch" && options.varPath != "") (pkgs.lib.mkForce {
fsType = pkgs.lib.mkForce "tmpfs";
});
};
boot.isContainer = pkgs.lib.mkForce false;
services.timesyncd.enable = false;
users.users.root.password = "";
services.getty.helpLine = ''
Log in as "root" with an empty password.
Use "reboot" to shut qemu down.
'';
};
vmDiskoOverwrites = {
boot.initrd = {
secrets = pkgs.lib.mkForce {};
network.ssh.enable = pkgs.lib.mkForce false;
};
malobeo.disks.enable = pkgs.lib.mkForce false;
networking.hostId = "a3c3101f";
};
vmSopsOverwrites = host: {
sops.defaultSopsFile = pkgs.lib.mkForce ./machines/${host}/dummy.yaml;
environment.etc = {
devHostKey = {
source = ./machines/secrets/devkey_ed25519;
mode = "0600";
};
};
services.openssh.hostKeys = [{
path = "/etc/devHostKey";
type = "ed25519";
}];
};
buildVM = host: networking: sopsDummy: disableDisko: varPath: (self.nixosConfigurations.${host}.extendModules {
modules = [
(vmMicroVMOverwrites host { withNetworking = networking; varPath = "${varPath}"; })
(if sopsDummy then (vmSopsOverwrites host) else {})
(if disableDisko then vmDiskoOverwrites else {})
] ++ pkgs.lib.optionals (! self.nixosConfigurations.${host}.config ? microvm) [
microvm.nixosModules.microvm
];
}).config.microvm.declaredRunner;
hosts = import ./machines/hosts.nix ( inputs // { inherit inputs; self = self; });
utils = import ./machines/modules/host_builder.nix ( inputs // { inherit inputs; self = self; hosts = hosts; });
in
{
devShells.default =
@@ -130,7 +52,7 @@ in (utils.lib.eachSystem (builtins.filter filter_system utils.lib.defaultSystems
scripts.run-vm = self.packages.${system}.run-vm;
};
vmBuilder = buildVM;
vmBuilder = utils.buildVM;
packages = {
docs = pkgs.stdenv.mkDerivation {
@@ -150,60 +72,7 @@ in (utils.lib.eachSystem (builtins.filter filter_system utils.lib.defaultSystems
'';
};
run-vm = pkgs.writeShellScriptBin "run-vm" ''
usage() {
echo "Usage: run-vm <hostname> [--networking] [--dummy-secrets] [--no-disko]"
echo "ATTENTION: This script must be run from the flakes root directory"
echo "--networking setup interfaces. requires root and hostbridge enabled on the host"
echo "--dummy-secrets run vm with dummy sops secrets"
echo "--no-disko disable disko and initrd secrets. needed for real hosts like fanny"
echo "--var path to directory that should be shared as /var. may require root otherwise some systemd units fail within vm. if dir is empty vm will populate"
exit 1
}
# check at least one arg was given
if [ "$#" -lt 1 ]; then
usage
fi
HOSTNAME=$1
# Optionale Argumente
NETWORK=false
DUMMY_SECRETS=false
NO_DISKO=false
VAR_PATH=""
# check argws
shift
while [[ "$#" -gt 0 ]]; do
case $1 in
--networking) NETWORK=true ;;
--dummy-secrets) DUMMY_SECRETS=true ;;
--no-disko) NO_DISKO=true ;;
--var)
if [[ -n "$2" && ! "$2" =~ ^- ]]; then
VAR_PATH="$2"
shift
else
echo "Error: --var requires a non-empty string argument."
usage
fi
;;
*) echo "Unknown argument: $1"; usage ;;
esac
shift
done
echo "starting host $HOSTNAME"
echo "enable networking: $NETWORK"
echo "deploy dummy secrets: $DUMMY_SECRETS"
echo "disable disko and initrd secrets: $NO_DISKO"
if [ -n "$VAR_PATH" ]; then
echo "sharing var directory: $VAR_PATH"
fi
${pkgs.nix}/bin/nix run --show-trace --impure --expr "((builtins.getFlake \"$(pwd)\").vmBuilder.x86_64-linux \"$HOSTNAME\" $NETWORK $DUMMY_SECRETS $NO_DISKO \"$VAR_PATH\")"
'';
run-vm = pkgs.writeShellScriptBin "run-vm" (builtins.readFile ./scripts/run-vm.sh);
};
apps = {
@@ -229,11 +98,13 @@ in (utils.lib.eachSystem (builtins.filter filter_system utils.lib.defaultSystems
};
};
})) // {
nixosConfigurations = import ./machines/configuration.nix (inputs // {
inherit inputs;
self = self;
});
})) // (
let
hosts = import ./machines/hosts.nix ( inputs // { inherit inputs; self = self; });
utils = import ./machines/modules/host_builder.nix ( inputs // { inherit inputs; self = self; hosts = hosts; });
in
{
nixosConfigurations = utils.buildHost hosts.malobeo.hosts;
nixosModules.malobeo = {
host.imports = [ ./machines/durruti/host_config.nix ];
@@ -255,4 +126,4 @@ in (utils.lib.eachSystem (builtins.filter filter_system utils.lib.defaultSystems
nixpkgs.lib.mapAttrs getBuildEntry self.nixosConfigurations
);
}
})

56
scripts/run-vm.sh Normal file
View File

@@ -0,0 +1,56 @@
usage() {
echo "Usage: run-vm <hostname> [--networking] [--dummy-secrets] [--no-disko]"
echo "ATTENTION: This script must be run from the flakes root directory"
echo "--networking setup interfaces. requires root and hostbridge enabled on the host"
echo "--dummy-secrets run vm with dummy sops secrets"
echo "--no-disko disable disko and initrd secrets. needed for real hosts like fanny"
echo "--writable-store enables writable store. necessary for host with nested imperative microvms like fanny"
echo "--var path to directory that should be shared as /var. may require root otherwise some systemd units fail within vm. if dir is empty vm will populate"
exit 1
}
# check at least one arg was given
if [ "$#" -lt 1 ]; then
usage
fi
HOSTNAME=$1
# Optionale Argumente
NETWORK=false
DUMMY_SECRETS=false
NO_DISKO=false
RW_STORE=false
VAR_PATH=""
# check argws
shift
while [[ "$#" -gt 0 ]]; do
case $1 in
--networking) NETWORK=true ;;
--dummy-secrets) DUMMY_SECRETS=true ;;
--no-disko) NO_DISKO=true ;;
--writable-store) RW_STORE=true ;;
--var)
if [[ -n "$2" && ! "$2" =~ ^- ]]; then
VAR_PATH="$2"
shift
else
echo "Error: --var requires a non-empty string argument."
usage
fi
;;
*) echo "Unknown argument: $1"; usage ;;
esac
shift
done
echo "starting host $HOSTNAME"
echo "enable networking: $NETWORK"
echo "deploy dummy secrets: $DUMMY_SECRETS"
echo "disable disko and initrd secrets: $NO_DISKO"
echo "use writable store: $RW_STORE"
if [ -n "$VAR_PATH" ]; then
echo "sharing var directory: $VAR_PATH"
fi
nix run --show-trace --impure --expr "((builtins.getFlake \"$(pwd)\").vmBuilder.x86_64-linux \"$HOSTNAME\" $NETWORK $DUMMY_SECRETS $NO_DISKO \"$VAR_PATH\" $RW_STORE).config.microvm.declaredRunner"