From 683622c11fb5eaf3c308eae69be9b9015dcc0871 Mon Sep 17 00:00:00 2001 From: kalipso Date: Wed, 22 Jan 2025 03:28:04 +0100 Subject: [PATCH 01/11] [nix] init host_builder.nix --- machines/modules/host_builer.nix | 71 ++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 machines/modules/host_builer.nix diff --git a/machines/modules/host_builer.nix b/machines/modules/host_builer.nix new file mode 100644 index 0000000..46f961a --- /dev/null +++ b/machines/modules/host_builer.nix @@ -0,0 +1,71 @@ +{ ... }: + +{ + malobeo = { + hosts = { + louise = { + type = "host"; + }; + + bakunin = { + type = "host"; + }; + + fanny = { + type = "host"; + }; + + lucia = { + type = "rpi"; + }; + + durruti = { + type = "microvm"; + network = { + address = "10.0.0.5"; + mac = "52:DA:0D:F9:EF:F9"; + }; + }; + + vpn = { + type = "microvm"; + network = { + address = "10.0.0.10"; + mac = "D0:E5:CA:F0:D7:E6"; + }; + }; + + infradocs = { + type = "microvm"; + network = { + address = "10.0.0.11"; + mac = "D0:E5:CA:F0:D7:E7"; + }; + }; + + uptimekuma = { + type = "microvm"; + network = { + address = "10.0.0.12"; + mac = "D0:E5:CA:F0:D7:E8"; + }; + }; + + nextcloud = { + type = "microvm"; + network = { + address = "10.0.0.13"; + mac = "D0:E5:CA:F0:D7:E9"; + }; + }; + + overwatch = { + type = "microvm"; + network = { + address = "10.0.0.14"; + mac = "D0:E5:CA:F0:D7:E0"; + }; + }; + }; + }; +} -- 2.51.2 From 4da6b467bd2613484e316f9110405d7ec4beb94d Mon Sep 17 00:00:00 2001 From: kalipso Date: Wed, 22 Jan 2025 04:22:43 +0100 Subject: [PATCH 02/11] [nix] generate hosts --- machines/configuration.nix | 211 ++++++++++++++++++++----------------- 1 file changed, 112 insertions(+), 99 deletions(-) diff --git a/machines/configuration.nix b/machines/configuration.nix index 0e7fd4b..887023e 100644 --- a/machines/configuration.nix +++ b/machines/configuration.nix @@ -95,104 +95,117 @@ let ] ++ defaultModules ++ modules; inputsMod = inputs // { malobeo = self; }; + + hosts = import ./modules/host_builer.nix {}; in -{ - louise = nixosSystem { - system = "x86_64-linux"; + builtins.mapAttrs (host: settings: nixosSystem { + system = if (settings.type == "rpi") then "aarch64-linux" else "x86_64-linux"; specialArgs.inputs = inputs; - modules = defaultModules ++ [ - ./louise/configuration.nix - ]; - }; - - bakunin = nixosSystem { - system = "x86_64-linux"; - specialArgs.inputs = inputs; - modules = defaultModules ++ [ - ./bakunin/configuration.nix - inputs.disko.nixosModules.disko - ]; - }; - - lucia = nixosSystem { - system = "aarch64-linux"; - specialArgs.inputs = inputs; - modules = defaultModules ++ [ - ./lucia/configuration.nix - ./lucia/hardware_configuration.nix - ]; - }; - - fanny = nixosSystem { - system = "x86_64-linux"; - specialArgs.inputs = inputsMod; - modules = defaultModules ++ [ - self.nixosModules.malobeo.vpn - ./fanny/configuration.nix - ]; - }; - - durruti = nixosSystem { - system = "x86_64-linux"; - specialArgs.inputs = inputs; - specialArgs.self = self; - modules = makeMicroVM "durruti" "10.0.0.5" "52:DA:0D:F9:EF:F9" [ - ./durruti/configuration.nix - ]; - }; - - vpn = nixosSystem { - system = "x86_64-linux"; - specialArgs.inputs = inputs; - specialArgs.self = self; - modules = makeMicroVM "vpn" "10.0.0.10" "D0:E5:CA:F0:D7:E6" [ - self.nixosModules.malobeo.vpn - ./vpn/configuration.nix - ]; - }; - - infradocs = nixosSystem { - system = "x86_64-linux"; - specialArgs.inputs = inputs; - specialArgs.self = self; - modules = makeMicroVM "infradocs" "10.0.0.11" "D0:E5:CA:F0:D7:E7" [ - self.nixosModules.malobeo.vpn - ./infradocs/configuration.nix - ]; - }; - - uptimekuma = nixosSystem { - system = "x86_64-linux"; - specialArgs.inputs = inputs; - specialArgs.self = self; - modules = makeMicroVM "uptimekuma" "10.0.0.12" "D0:E5:CA:F0:D7:E8" [ - ./uptimekuma/configuration.nix - ]; - }; - - nextcloud = nixosSystem { - system = "x86_64-linux"; - specialArgs.inputs = inputs; - specialArgs.self = self; - modules = makeMicroVM "nextcloud" "10.0.0.13" "D0:E5:CA:F0:D7:E9" [ - ./nextcloud/configuration.nix - ]; - }; - - overwatch = nixosSystem { - system = "x86_64-linux"; - specialArgs.inputs = inputs; - specialArgs.self = self; - modules = makeMicroVM "overwatch" "10.0.0.14" "D0:E5:CA:F0:D7:E0" [ - ./overwatch/configuration.nix - ]; - }; - - testvm = nixosSystem { - system = "x86_64-linux"; - specialArgs.inputs = inputs; - specialArgs.self = self; - modules = defaultModules ++ [ ./testvm ]; - }; - -} + modules = (if (settings.type != "microvm") then + defaultModules ++ [ ./${host}/configuration.nix ] + else + makeMicroVM "${host}" "${settings.network.address}" "${settings.network.mac}" [ + ./${host}/configuration.nix + ]); + }) hosts.malobeo.hosts // + { + testvm = nixosSystem { + system = "x86_64-linux"; + specialArgs.inputs = inputs; + specialArgs.self = self; + modules = defaultModules ++ [ ./testvm ]; + }; + } +#{ +# louise = nixosSystem { +# system = "x86_64-linux"; +# specialArgs.inputs = inputs; +# modules = defaultModules ++ [ +# ./louise/configuration.nix +# ]; +# }; +# +# bakunin = nixosSystem { +# system = "x86_64-linux"; +# specialArgs.inputs = inputs; +# modules = defaultModules ++ [ +# ./bakunin/configuration.nix +# inputs.disko.nixosModules.disko +# ]; +# }; +# +# lucia = nixosSystem { +# system = "aarch64-linux"; +# specialArgs.inputs = inputs; +# modules = defaultModules ++ [ +# ./lucia/configuration.nix +# ./lucia/hardware_configuration.nix +# ]; +# }; +# +# fanny = nixosSystem { +# system = "x86_64-linux"; +# specialArgs.inputs = inputsMod; +# modules = defaultModules ++ [ +# self.nixosModules.malobeo.vpn +# ./fanny/configuration.nix +# ]; +# }; +# +# durruti = nixosSystem { +# system = "x86_64-linux"; +# specialArgs.inputs = inputs; +# specialArgs.self = self; +# modules = makeMicroVM "durruti" "10.0.0.5" "52:DA:0D:F9:EF:F9" [ +# ./durruti/configuration.nix +# ]; +# }; +# +# vpn = nixosSystem { +# system = "x86_64-linux"; +# specialArgs.inputs = inputs; +# specialArgs.self = self; +# modules = makeMicroVM "vpn" "10.0.0.10" "D0:E5:CA:F0:D7:E6" [ +# self.nixosModules.malobeo.vpn +# ./vpn/configuration.nix +# ]; +# }; +# +# infradocs = nixosSystem { +# system = "x86_64-linux"; +# specialArgs.inputs = inputs; +# specialArgs.self = self; +# modules = makeMicroVM "infradocs" "10.0.0.11" "D0:E5:CA:F0:D7:E7" [ +# self.nixosModules.malobeo.vpn +# ./infradocs/configuration.nix +# ]; +# }; +# +# uptimekuma = nixosSystem { +# system = "x86_64-linux"; +# specialArgs.inputs = inputs; +# specialArgs.self = self; +# modules = makeMicroVM "uptimekuma" "10.0.0.12" "D0:E5:CA:F0:D7:E8" [ +# ./uptimekuma/configuration.nix +# ]; +# }; +# +# nextcloud = nixosSystem { +# system = "x86_64-linux"; +# specialArgs.inputs = inputs; +# specialArgs.self = self; +# modules = makeMicroVM "nextcloud" "10.0.0.13" "D0:E5:CA:F0:D7:E9" [ +# ./nextcloud/configuration.nix +# ]; +# }; +# +# overwatch = nixosSystem { +# system = "x86_64-linux"; +# specialArgs.inputs = inputs; +# specialArgs.self = self; +# modules = makeMicroVM "overwatch" "10.0.0.14" "D0:E5:CA:F0:D7:E0" [ +# ./overwatch/configuration.nix +# ]; +# }; +# +#} -- 2.51.2 From 96ab36977b9499808e126a4fd16c36b0b96fc410 Mon Sep 17 00:00:00 2001 From: kalipso Date: Wed, 22 Jan 2025 11:52:47 +0100 Subject: [PATCH 03/11] [nix] mv host declarations to hosts.nix, add util to host_builer.nix --- machines/hosts.nix | 71 +++++++++++++ machines/modules/host_builer.nix | 165 +++++++++++++++++++------------ 2 files changed, 173 insertions(+), 63 deletions(-) create mode 100644 machines/hosts.nix diff --git a/machines/hosts.nix b/machines/hosts.nix new file mode 100644 index 0000000..46f961a --- /dev/null +++ b/machines/hosts.nix @@ -0,0 +1,71 @@ +{ ... }: + +{ + malobeo = { + hosts = { + louise = { + type = "host"; + }; + + bakunin = { + type = "host"; + }; + + fanny = { + type = "host"; + }; + + lucia = { + type = "rpi"; + }; + + durruti = { + type = "microvm"; + network = { + address = "10.0.0.5"; + mac = "52:DA:0D:F9:EF:F9"; + }; + }; + + vpn = { + type = "microvm"; + network = { + address = "10.0.0.10"; + mac = "D0:E5:CA:F0:D7:E6"; + }; + }; + + infradocs = { + type = "microvm"; + network = { + address = "10.0.0.11"; + mac = "D0:E5:CA:F0:D7:E7"; + }; + }; + + uptimekuma = { + type = "microvm"; + network = { + address = "10.0.0.12"; + mac = "D0:E5:CA:F0:D7:E8"; + }; + }; + + nextcloud = { + type = "microvm"; + network = { + address = "10.0.0.13"; + mac = "D0:E5:CA:F0:D7:E9"; + }; + }; + + overwatch = { + type = "microvm"; + network = { + address = "10.0.0.14"; + mac = "D0:E5:CA:F0:D7:E0"; + }; + }; + }; + }; +} diff --git a/machines/modules/host_builer.nix b/machines/modules/host_builer.nix index 46f961a..e65e198 100644 --- a/machines/modules/host_builer.nix +++ b/machines/modules/host_builer.nix @@ -1,71 +1,110 @@ -{ ... }: +{ self +, nixpkgs-unstable +, nixpkgs +, sops-nix +, inputs +, nixos-hardware +, home-manager +, ... +}: -{ - malobeo = { - hosts = { - louise = { - type = "host"; +rec { + nixosSystem = nixpkgs.lib.makeOverridable nixpkgs.lib.nixosSystem; + nixosSystemUnstable = nixpkgs-unstable.lib.makeOverridable nixpkgs-unstable.lib.nixosSystem; + + baseModules = [ + # make flake inputs accessiable in NixOS + { _module.args.inputs = inputs; } + { + imports = [ + ({ pkgs, ... }: { + nix = { + extraOptions = '' + experimental-features = nix-command flakes + ''; + + settings = { + substituters = [ + "https://cache.dynamicdiscord.de" + "https://cache.nixos.org/" + ]; + trusted-public-keys = [ + "cache.dynamicdiscord.de:DKueZicqi2NhJJXz9MYgUbiyobMs10fTyHCgAUibRP4=" + ]; + trusted-users = [ "root" "@wheel" ]; + }; + }; + }) + + sops-nix.nixosModules.sops + ]; + } + ]; + defaultModules = baseModules; + + makeMicroVM = hostName: ipv4Addr: macAddr: modules: [ + inputs.microvm.nixosModules.microvm + { + microvm = { + hypervisor = "cloud-hypervisor"; + mem = 2560; + shares = [ + { + source = "/nix/store"; + mountPoint = "/nix/.ro-store"; + tag = "store"; + proto = "virtiofs"; + socket = "store.socket"; + } + { + source = "/var/lib/microvms/${hostName}/etc"; + mountPoint = "/etc"; + tag = "etc"; + proto = "virtiofs"; + socket = "etc.socket"; + } + { + source = "/var/lib/microvms/${hostName}/var"; + mountPoint = "/var"; + tag = "var"; + proto = "virtiofs"; + socket = "var.socket"; + } + ]; + + interfaces = [ + { + type = "tap"; + id = "vm-${hostName}"; + mac = "${macAddr}"; + } + ]; }; - bakunin = { - type = "host"; - }; - - fanny = { - type = "host"; - }; - - lucia = { - type = "rpi"; - }; - - durruti = { - type = "microvm"; - network = { - address = "10.0.0.5"; - mac = "52:DA:0D:F9:EF:F9"; + systemd.network.enable = true; + + systemd.network.networks."20-lan" = { + matchConfig.Type = "ether"; + networkConfig = { + Address = [ "${ipv4Addr}/24" ]; + Gateway = "10.0.0.1"; + DNS = ["1.1.1.1"]; + DHCP = "no"; }; }; + } + ] ++ defaultModules ++ modules; - vpn = { - type = "microvm"; - network = { - address = "10.0.0.10"; - mac = "D0:E5:CA:F0:D7:E6"; - }; - }; + inputsMod = inputs // { malobeo = self; }; - infradocs = { - type = "microvm"; - network = { - address = "10.0.0.11"; - mac = "D0:E5:CA:F0:D7:E7"; - }; - }; - - uptimekuma = { - type = "microvm"; - network = { - address = "10.0.0.12"; - mac = "D0:E5:CA:F0:D7:E8"; - }; - }; - - nextcloud = { - type = "microvm"; - network = { - address = "10.0.0.13"; - mac = "D0:E5:CA:F0:D7:E9"; - }; - }; - - overwatch = { - type = "microvm"; - network = { - address = "10.0.0.14"; - mac = "D0:E5:CA:F0:D7:E0"; - }; - }; - }; - }; + buildHost = hosts: (builtins.mapAttrs (host: settings: nixosSystem { + system = if (settings.type == "rpi") then "aarch64-linux" else "x86_64-linux"; + specialArgs.inputs = inputsMod; + modules = (if (settings.type != "microvm") then + defaultModules ++ [ ../${host}/configuration.nix ] + else + makeMicroVM "${host}" "${settings.network.address}" "${settings.network.mac}" [ + ./${host}/configuration.nix + ]); + }) hosts); } -- 2.51.2 From 607ad465015de609e7055eee13f1c6c47341b8a1 Mon Sep 17 00:00:00 2001 From: kalipso Date: Wed, 22 Jan 2025 11:53:41 +0100 Subject: [PATCH 04/11] [nix] create nixosConfigurations using malobeo.hosts --- outputs.nix | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/outputs.nix b/outputs.nix index 9cfbf03..8a9a301 100644 --- a/outputs.nix +++ b/outputs.nix @@ -229,11 +229,17 @@ in (utils.lib.eachSystem (builtins.filter filter_system utils.lib.defaultSystems }; }; - })) // { - nixosConfigurations = import ./machines/configuration.nix (inputs // { - inherit inputs; - self = self; - }); + })) // ( + let + utils = import ./machines/modules/host_builer.nix ( inputs // { inherit inputs; self = self; }); + hosts = import ./machines/hosts.nix ( inputs // { inherit inputs; self = self; }); + in + { + nixosConfigurations = utils.buildHost hosts.malobeo.hosts; + #nixosConfigurations = import ./machines/configuration.nix (inputs // { + # inherit inputs; + # self = self; + #}); nixosModules.malobeo = { host.imports = [ ./machines/durruti/host_config.nix ]; @@ -255,4 +261,4 @@ in (utils.lib.eachSystem (builtins.filter filter_system utils.lib.defaultSystems nixpkgs.lib.mapAttrs getBuildEntry self.nixosConfigurations ); -} +}) -- 2.51.2 From c42ce41c71c081f3ca2df390663bd2969faa4ac3 Mon Sep 17 00:00:00 2001 From: kalipso Date: Wed, 22 Jan 2025 11:55:24 +0100 Subject: [PATCH 05/11] [nix] mv host_builer.nix host_builder.nix --- machines/configuration.nix | 2 +- machines/modules/{host_builer.nix => host_builder.nix} | 0 outputs.nix | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) rename machines/modules/{host_builer.nix => host_builder.nix} (100%) diff --git a/machines/configuration.nix b/machines/configuration.nix index 887023e..e046d59 100644 --- a/machines/configuration.nix +++ b/machines/configuration.nix @@ -96,7 +96,7 @@ let inputsMod = inputs // { malobeo = self; }; - hosts = import ./modules/host_builer.nix {}; + hosts = import ./modules/host_builder.nix {}; in builtins.mapAttrs (host: settings: nixosSystem { system = if (settings.type == "rpi") then "aarch64-linux" else "x86_64-linux"; diff --git a/machines/modules/host_builer.nix b/machines/modules/host_builder.nix similarity index 100% rename from machines/modules/host_builer.nix rename to machines/modules/host_builder.nix diff --git a/outputs.nix b/outputs.nix index 8a9a301..d75afbd 100644 --- a/outputs.nix +++ b/outputs.nix @@ -231,7 +231,7 @@ in (utils.lib.eachSystem (builtins.filter filter_system utils.lib.defaultSystems })) // ( let - utils = import ./machines/modules/host_builer.nix ( inputs // { inherit inputs; self = self; }); + utils = import ./machines/modules/host_builder.nix ( inputs // { inherit inputs; self = self; }); hosts = import ./machines/hosts.nix ( inputs // { inherit inputs; self = self; }); in { -- 2.51.2 From 0db92ce92a1163f69edad40b037b2c60c5fc5eb3 Mon Sep 17 00:00:00 2001 From: kalipso Date: Wed, 22 Jan 2025 12:21:33 +0100 Subject: [PATCH 06/11] [nix] mv vm overwrites to host_builder --- outputs.nix | 83 ++--------------------------------------------------- 1 file changed, 2 insertions(+), 81 deletions(-) diff --git a/outputs.nix b/outputs.nix index d75afbd..3c2433f 100644 --- a/outputs.nix +++ b/outputs.nix @@ -15,86 +15,7 @@ in (utils.lib.eachSystem (builtins.filter filter_system utils.lib.defaultSystems pkgs-unstable = nixpkgs-unstable.legacyPackages."${system}"; pkgs = nixpkgs.legacyPackages."${system}"; - vmMicroVMOverwrites = hostname: options: { - microvm = { - mem = pkgs.lib.mkForce 4096; - hypervisor = pkgs.lib.mkForce "qemu"; - socket = pkgs.lib.mkForce null; - shares = pkgs.lib.mkForce ([ - { - tag = "ro-store"; - source = "/nix/store"; - mountPoint = "/nix/.ro-store"; - } - ] ++ pkgs.lib.optionals (options.varPath != "") [ - { - source = "${options.varPath}"; - securityModel = "mapped"; - mountPoint = "/var"; - tag = "var"; - } - ]); - interfaces = pkgs.lib.mkIf (!options.withNetworking) (pkgs.lib.mkForce [{ - type = "user"; - id = "eth0"; - mac = "02:23:de:ad:be:ef"; - }]); - }; - - fileSystems = { - "/".fsType = pkgs.lib.mkForce "tmpfs"; - - # prometheus uses a memory mapped file which doesnt seem supported by 9p shares - # therefore we mount a tmpfs inside the datadir - "/var/lib/prometheus2/data" = pkgs.lib.mkIf (hostname == "overwatch" && options.varPath != "") (pkgs.lib.mkForce { - fsType = pkgs.lib.mkForce "tmpfs"; - }); - }; - - boot.isContainer = pkgs.lib.mkForce false; - services.timesyncd.enable = false; - users.users.root.password = ""; - services.getty.helpLine = '' - Log in as "root" with an empty password. - Use "reboot" to shut qemu down. - ''; - }; - - vmDiskoOverwrites = { - boot.initrd = { - secrets = pkgs.lib.mkForce {}; - network.ssh.enable = pkgs.lib.mkForce false; - }; - - malobeo.disks.enable = pkgs.lib.mkForce false; - networking.hostId = "a3c3101f"; - }; - - vmSopsOverwrites = host: { - sops.defaultSopsFile = pkgs.lib.mkForce ./machines/${host}/dummy.yaml; - - environment.etc = { - devHostKey = { - source = ./machines/secrets/devkey_ed25519; - mode = "0600"; - }; - }; - - services.openssh.hostKeys = [{ - path = "/etc/devHostKey"; - type = "ed25519"; - }]; - }; - - buildVM = host: networking: sopsDummy: disableDisko: varPath: (self.nixosConfigurations.${host}.extendModules { - modules = [ - (vmMicroVMOverwrites host { withNetworking = networking; varPath = "${varPath}"; }) - (if sopsDummy then (vmSopsOverwrites host) else {}) - (if disableDisko then vmDiskoOverwrites else {}) - ] ++ pkgs.lib.optionals (! self.nixosConfigurations.${host}.config ? microvm) [ - microvm.nixosModules.microvm - ]; - }).config.microvm.declaredRunner; + utils = import ./machines/modules/host_builder.nix ( inputs // { inherit inputs; self = self; }); in { devShells.default = @@ -130,7 +51,7 @@ in (utils.lib.eachSystem (builtins.filter filter_system utils.lib.defaultSystems scripts.run-vm = self.packages.${system}.run-vm; }; - vmBuilder = buildVM; + vmBuilder = utils.buildVM; packages = { docs = pkgs.stdenv.mkDerivation { -- 2.51.2 From a7446c35f6551eadf1d5e165baccf6cd3b68a921 Mon Sep 17 00:00:00 2001 From: kalipso Date: Wed, 22 Jan 2025 12:29:11 +0100 Subject: [PATCH 07/11] [nix] fix imports --- machines/fanny/configuration.nix | 1 + machines/infradocs/configuration.nix | 2 +- machines/lucia/configuration.nix | 1 + machines/vpn/configuration.nix | 1 + 4 files changed, 4 insertions(+), 1 deletion(-) diff --git a/machines/fanny/configuration.nix b/machines/fanny/configuration.nix index 6c19159..6d215a3 100644 --- a/machines/fanny/configuration.nix +++ b/machines/fanny/configuration.nix @@ -13,6 +13,7 @@ in ../modules/sshd.nix ../modules/minimal_tools.nix ../modules/autoupdate.nix + inputs.self.nixosModules.malobeo.vpn inputs.self.nixosModules.malobeo.initssh inputs.self.nixosModules.malobeo.disko inputs.self.nixosModules.malobeo.microvm diff --git a/machines/infradocs/configuration.nix b/machines/infradocs/configuration.nix index fece076..d1cc2fa 100644 --- a/machines/infradocs/configuration.nix +++ b/machines/infradocs/configuration.nix @@ -9,7 +9,7 @@ with lib; }; imports = [ - self.nixosModules.malobeo.metrics + inputs.malobeo.nixosModules.malobeo.metrics ../durruti/documentation.nix ../modules/malobeo_user.nix ../modules/sshd.nix diff --git a/machines/lucia/configuration.nix b/machines/lucia/configuration.nix index 7184763..3688f84 100644 --- a/machines/lucia/configuration.nix +++ b/machines/lucia/configuration.nix @@ -6,6 +6,7 @@ in { imports = [ # Include the results of the hardware scan. + ./hardware_configuration.nix ../modules/malobeo_user.nix ]; diff --git a/machines/vpn/configuration.nix b/machines/vpn/configuration.nix index 82914b4..6caeed1 100644 --- a/machines/vpn/configuration.nix +++ b/machines/vpn/configuration.nix @@ -17,6 +17,7 @@ with lib; }; imports = [ + inputs.self.nixosModules.malobeo.vpn ../modules/malobeo_user.nix ../modules/sshd.nix ../modules/minimal_tools.nix -- 2.51.2 From 28b0a84b1012737249e2213de053b0ec16b4fbba Mon Sep 17 00:00:00 2001 From: kalipso Date: Wed, 22 Jan 2025 12:30:17 +0100 Subject: [PATCH 08/11] [vmBuilder] add writable store flag --- outputs.nix | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/outputs.nix b/outputs.nix index 3c2433f..86be7f6 100644 --- a/outputs.nix +++ b/outputs.nix @@ -78,6 +78,7 @@ in (utils.lib.eachSystem (builtins.filter filter_system utils.lib.defaultSystems echo "--networking setup interfaces. requires root and hostbridge enabled on the host" echo "--dummy-secrets run vm with dummy sops secrets" echo "--no-disko disable disko and initrd secrets. needed for real hosts like fanny" + echo "--writable-store enables writable store. necessary for host with nested imperative microvms like fanny" echo "--var path to directory that should be shared as /var. may require root otherwise some systemd units fail within vm. if dir is empty vm will populate" exit 1 } @@ -93,6 +94,7 @@ in (utils.lib.eachSystem (builtins.filter filter_system utils.lib.defaultSystems NETWORK=false DUMMY_SECRETS=false NO_DISKO=false + RW_STORE=false VAR_PATH="" # check argws @@ -102,6 +104,7 @@ in (utils.lib.eachSystem (builtins.filter filter_system utils.lib.defaultSystems --networking) NETWORK=true ;; --dummy-secrets) DUMMY_SECRETS=true ;; --no-disko) NO_DISKO=true ;; + --writable-store) RW_STORE=true ;; --var) if [[ -n "$2" && ! "$2" =~ ^- ]]; then VAR_PATH="$2" @@ -119,11 +122,12 @@ in (utils.lib.eachSystem (builtins.filter filter_system utils.lib.defaultSystems echo "enable networking: $NETWORK" echo "deploy dummy secrets: $DUMMY_SECRETS" echo "disable disko and initrd secrets: $NO_DISKO" + echo "use writable store: $RW_STORE" if [ -n "$VAR_PATH" ]; then echo "sharing var directory: $VAR_PATH" fi - ${pkgs.nix}/bin/nix run --show-trace --impure --expr "((builtins.getFlake \"$(pwd)\").vmBuilder.x86_64-linux \"$HOSTNAME\" $NETWORK $DUMMY_SECRETS $NO_DISKO \"$VAR_PATH\")" + ${pkgs.nix}/bin/nix run --show-trace --impure --expr "((builtins.getFlake \"$(pwd)\").vmBuilder.x86_64-linux \"$HOSTNAME\" $NETWORK $DUMMY_SECRETS $NO_DISKO \"$VAR_PATH\" $RW_STORE).config.microvm.declaredRunner" ''; }; -- 2.51.2 From 489aee9fb3323fdef1c70645da28277798ca81fa Mon Sep 17 00:00:00 2001 From: kalipso Date: Wed, 22 Jan 2025 17:17:16 +0100 Subject: [PATCH 09/11] [nix] mv buildVM to host_builder --- machines/modules/host_builder.nix | 142 ++++++++++++++++++++++++++++-- outputs.nix | 5 +- 2 files changed, 140 insertions(+), 7 deletions(-) diff --git a/machines/modules/host_builder.nix b/machines/modules/host_builder.nix index e65e198..98e7bb4 100644 --- a/machines/modules/host_builder.nix +++ b/machines/modules/host_builder.nix @@ -3,11 +3,12 @@ , nixpkgs , sops-nix , inputs -, nixos-hardware -, home-manager +, hosts , ... }: - +let + pkgs = nixpkgs.legacyPackages."x86_64-linux"; +in rec { nixosSystem = nixpkgs.lib.makeOverridable nixpkgs.lib.nixosSystem; nixosSystemUnstable = nixpkgs-unstable.lib.makeOverridable nixpkgs-unstable.lib.nixosSystem; @@ -43,7 +44,6 @@ rec { defaultModules = baseModules; makeMicroVM = hostName: ipv4Addr: macAddr: modules: [ - inputs.microvm.nixosModules.microvm { microvm = { hypervisor = "cloud-hypervisor"; @@ -97,14 +97,146 @@ rec { inputsMod = inputs // { malobeo = self; }; + + vmMicroVMOverwrites = hostname: options: { + microvm = rec { + mem = pkgs.lib.mkForce 4096; + hypervisor = pkgs.lib.mkForce "qemu"; + socket = pkgs.lib.mkForce null; + + + #needed for hosts that deploy imperative microvms (for example fanny) + writableStoreOverlay = pkgs.lib.mkIf options.writableStore "/nix/.rw-store"; + volumes = pkgs.lib.mkIf options.writableStore [ { + image = "nix-store-overlay.img"; + mountPoint = writableStoreOverlay; + size = 2048; + } ]; + + shares = pkgs.lib.mkForce (pkgs.lib.optionals (!options.writableStore) [ + { + tag = "ro-store"; + source = "/nix/store"; + mountPoint = "/nix/.ro-store"; + } + ] ++ pkgs.lib.optionals (options.varPath != "") [ + { + source = "${options.varPath}"; + securityModel = "mapped"; + mountPoint = "/var"; + tag = "var"; + } + ]); + + interfaces = pkgs.lib.mkIf (!options.withNetworking) (pkgs.lib.mkForce [{ + type = "user"; + id = "eth0"; + mac = "02:23:de:ad:be:ef"; + }]); + + #if networking is disabled forward port 80 to still have access to webservices + forwardPorts = pkgs.lib.mkIf (!options.withNetworking) (pkgs.lib.mkForce [ + { from = "host"; host.port = 8080; guest.port = 80; } + ]); + + }; + + fileSystems = { + "/".fsType = pkgs.lib.mkForce "tmpfs"; + + # prometheus uses a memory mapped file which doesnt seem supported by 9p shares + # therefore we mount a tmpfs inside the datadir + "/var/lib/prometheus2/data" = pkgs.lib.mkIf (hostname == "overwatch" && options.varPath != "") (pkgs.lib.mkForce { + fsType = pkgs.lib.mkForce "tmpfs"; + }); + }; + + boot.isContainer = pkgs.lib.mkForce false; + services.timesyncd.enable = false; + users.users.root.password = ""; + services.getty.helpLine = '' + Log in as "root" with an empty password. + Use "reboot" to shut qemu down. + ''; + }; + + vmDiskoOverwrites = { + boot.initrd = { + secrets = pkgs.lib.mkForce {}; + network.ssh.enable = pkgs.lib.mkForce false; + }; + + malobeo.disks.enable = pkgs.lib.mkForce false; + networking.hostId = "a3c3101f"; + }; + + vmSopsOverwrites = host: { + sops.defaultSopsFile = pkgs.lib.mkForce ../${host}/dummy.yaml; + + environment.etc = { + devHostKey = { + source = ../secrets/devkey_ed25519; + mode = "0600"; + }; + }; + + services.openssh.hostKeys = [{ + path = "/etc/devHostKey"; + type = "ed25519"; + }]; + }; + + vmNestedMicroVMOverwrites = host: sopsDummy: { + + services.malobeo.microvm.deployHosts = pkgs.lib.mkForce []; + microvm.vms = + let + # Map the values to each hostname to then generate an Attrset using listToAttrs + mapperFunc = name: { inherit name; value = { + specialArgs.inputs = inputsMod; + specialArgs.self = self; + config = { + imports = (makeMicroVM "${name}" + "${hosts.malobeo.hosts.${name}.network.address}" + "${hosts.malobeo.hosts.${name}.network.mac}" [ + ../${name}/configuration.nix + (vmMicroVMOverwrites name { + withNetworking = true; + varPath = ""; + writableStore = false; }) + (if sopsDummy then (vmSopsOverwrites name) else {}) + ]); + }; + }; }; + in + builtins.listToAttrs (map mapperFunc self.nixosConfigurations.${host}.config.services.malobeo.microvm.deployHosts); + }; + + buildVM = host: networking: sopsDummy: disableDisko: varPath: writableStore: (self.nixosConfigurations.${host}.extendModules { + modules = [ + (vmMicroVMOverwrites host { + withNetworking = networking; + varPath = "${varPath}"; + writableStore = writableStore; }) + (if sopsDummy then (vmSopsOverwrites host) else {}) + (if disableDisko then vmDiskoOverwrites else {}) + ] ++ pkgs.lib.optionals (hosts.malobeo.hosts.${host}.type != "microvm") [ + inputs.microvm.nixosModules.microvm + ] ++ pkgs.lib.optionals (self.nixosConfigurations.${host}.config ? services.malobeo.microvm.deployHosts) [ + (vmNestedMicroVMOverwrites host sopsDummy) + ]; + }); + buildHost = hosts: (builtins.mapAttrs (host: settings: nixosSystem { system = if (settings.type == "rpi") then "aarch64-linux" else "x86_64-linux"; specialArgs.inputs = inputsMod; + specialArgs.self = self; modules = (if (settings.type != "microvm") then defaultModules ++ [ ../${host}/configuration.nix ] else makeMicroVM "${host}" "${settings.network.address}" "${settings.network.mac}" [ - ./${host}/configuration.nix + inputs.microvm.nixosModules.microvm + ../${host}/configuration.nix ]); }) hosts); } diff --git a/outputs.nix b/outputs.nix index 86be7f6..248d332 100644 --- a/outputs.nix +++ b/outputs.nix @@ -15,7 +15,8 @@ in (utils.lib.eachSystem (builtins.filter filter_system utils.lib.defaultSystems pkgs-unstable = nixpkgs-unstable.legacyPackages."${system}"; pkgs = nixpkgs.legacyPackages."${system}"; - utils = import ./machines/modules/host_builder.nix ( inputs // { inherit inputs; self = self; }); + hosts = import ./machines/hosts.nix ( inputs // { inherit inputs; self = self; }); + utils = import ./machines/modules/host_builder.nix ( inputs // { inherit inputs; self = self; hosts = hosts; }); in { devShells.default = @@ -156,8 +157,8 @@ in (utils.lib.eachSystem (builtins.filter filter_system utils.lib.defaultSystems })) // ( let - utils = import ./machines/modules/host_builder.nix ( inputs // { inherit inputs; self = self; }); hosts = import ./machines/hosts.nix ( inputs // { inherit inputs; self = self; }); + utils = import ./machines/modules/host_builder.nix ( inputs // { inherit inputs; self = self; hosts = hosts; }); in { nixosConfigurations = utils.buildHost hosts.malobeo.hosts; -- 2.51.2 From b136a0bbdb16c0497a3c9177b21aca244c755347 Mon Sep 17 00:00:00 2001 From: kalipso Date: Wed, 22 Jan 2025 17:18:46 +0100 Subject: [PATCH 10/11] [nix] rm machines/configuration.nix --- machines/configuration.nix | 211 ------------------------------------- outputs.nix | 4 - 2 files changed, 215 deletions(-) delete mode 100644 machines/configuration.nix diff --git a/machines/configuration.nix b/machines/configuration.nix deleted file mode 100644 index e046d59..0000000 --- a/machines/configuration.nix +++ /dev/null @@ -1,211 +0,0 @@ -{ self -, nixpkgs-unstable -, nixpkgs -, sops-nix -, inputs -, nixos-hardware -, home-manager -, ... -}: -let - nixosSystem = nixpkgs.lib.makeOverridable nixpkgs.lib.nixosSystem; - nixosSystemUnstable = nixpkgs-unstable.lib.makeOverridable nixpkgs-unstable.lib.nixosSystem; - - baseModules = [ - # make flake inputs accessiable in NixOS - { _module.args.inputs = inputs; } - { - imports = [ - ({ pkgs, ... }: { - nix = { - extraOptions = '' - experimental-features = nix-command flakes - ''; - - settings = { - substituters = [ - "https://cache.dynamicdiscord.de" - "https://cache.nixos.org/" - ]; - trusted-public-keys = [ - "cache.dynamicdiscord.de:DKueZicqi2NhJJXz9MYgUbiyobMs10fTyHCgAUibRP4=" - ]; - trusted-users = [ "root" "@wheel" ]; - }; - }; - }) - - sops-nix.nixosModules.sops - ]; - } - ]; - defaultModules = baseModules; - - makeMicroVM = hostName: ipv4Addr: macAddr: modules: [ - inputs.microvm.nixosModules.microvm - { - microvm = { - hypervisor = "cloud-hypervisor"; - mem = 2560; - shares = [ - { - source = "/nix/store"; - mountPoint = "/nix/.ro-store"; - tag = "store"; - proto = "virtiofs"; - socket = "store.socket"; - } - { - source = "/var/lib/microvms/${hostName}/etc"; - mountPoint = "/etc"; - tag = "etc"; - proto = "virtiofs"; - socket = "etc.socket"; - } - { - source = "/var/lib/microvms/${hostName}/var"; - mountPoint = "/var"; - tag = "var"; - proto = "virtiofs"; - socket = "var.socket"; - } - ]; - - interfaces = [ - { - type = "tap"; - id = "vm-${hostName}"; - mac = "${macAddr}"; - } - ]; - }; - - systemd.network.enable = true; - - systemd.network.networks."20-lan" = { - matchConfig.Type = "ether"; - networkConfig = { - Address = [ "${ipv4Addr}/24" ]; - Gateway = "10.0.0.1"; - DNS = ["1.1.1.1"]; - DHCP = "no"; - }; - }; - } - ] ++ defaultModules ++ modules; - - inputsMod = inputs // { malobeo = self; }; - - hosts = import ./modules/host_builder.nix {}; -in - builtins.mapAttrs (host: settings: nixosSystem { - system = if (settings.type == "rpi") then "aarch64-linux" else "x86_64-linux"; - specialArgs.inputs = inputs; - modules = (if (settings.type != "microvm") then - defaultModules ++ [ ./${host}/configuration.nix ] - else - makeMicroVM "${host}" "${settings.network.address}" "${settings.network.mac}" [ - ./${host}/configuration.nix - ]); - }) hosts.malobeo.hosts // - { - testvm = nixosSystem { - system = "x86_64-linux"; - specialArgs.inputs = inputs; - specialArgs.self = self; - modules = defaultModules ++ [ ./testvm ]; - }; - } -#{ -# louise = nixosSystem { -# system = "x86_64-linux"; -# specialArgs.inputs = inputs; -# modules = defaultModules ++ [ -# ./louise/configuration.nix -# ]; -# }; -# -# bakunin = nixosSystem { -# system = "x86_64-linux"; -# specialArgs.inputs = inputs; -# modules = defaultModules ++ [ -# ./bakunin/configuration.nix -# inputs.disko.nixosModules.disko -# ]; -# }; -# -# lucia = nixosSystem { -# system = "aarch64-linux"; -# specialArgs.inputs = inputs; -# modules = defaultModules ++ [ -# ./lucia/configuration.nix -# ./lucia/hardware_configuration.nix -# ]; -# }; -# -# fanny = nixosSystem { -# system = "x86_64-linux"; -# specialArgs.inputs = inputsMod; -# modules = defaultModules ++ [ -# self.nixosModules.malobeo.vpn -# ./fanny/configuration.nix -# ]; -# }; -# -# durruti = nixosSystem { -# system = "x86_64-linux"; -# specialArgs.inputs = inputs; -# specialArgs.self = self; -# modules = makeMicroVM "durruti" "10.0.0.5" "52:DA:0D:F9:EF:F9" [ -# ./durruti/configuration.nix -# ]; -# }; -# -# vpn = nixosSystem { -# system = "x86_64-linux"; -# specialArgs.inputs = inputs; -# specialArgs.self = self; -# modules = makeMicroVM "vpn" "10.0.0.10" "D0:E5:CA:F0:D7:E6" [ -# self.nixosModules.malobeo.vpn -# ./vpn/configuration.nix -# ]; -# }; -# -# infradocs = nixosSystem { -# system = "x86_64-linux"; -# specialArgs.inputs = inputs; -# specialArgs.self = self; -# modules = makeMicroVM "infradocs" "10.0.0.11" "D0:E5:CA:F0:D7:E7" [ -# self.nixosModules.malobeo.vpn -# ./infradocs/configuration.nix -# ]; -# }; -# -# uptimekuma = nixosSystem { -# system = "x86_64-linux"; -# specialArgs.inputs = inputs; -# specialArgs.self = self; -# modules = makeMicroVM "uptimekuma" "10.0.0.12" "D0:E5:CA:F0:D7:E8" [ -# ./uptimekuma/configuration.nix -# ]; -# }; -# -# nextcloud = nixosSystem { -# system = "x86_64-linux"; -# specialArgs.inputs = inputs; -# specialArgs.self = self; -# modules = makeMicroVM "nextcloud" "10.0.0.13" "D0:E5:CA:F0:D7:E9" [ -# ./nextcloud/configuration.nix -# ]; -# }; -# -# overwatch = nixosSystem { -# system = "x86_64-linux"; -# specialArgs.inputs = inputs; -# specialArgs.self = self; -# modules = makeMicroVM "overwatch" "10.0.0.14" "D0:E5:CA:F0:D7:E0" [ -# ./overwatch/configuration.nix -# ]; -# }; -# -#} diff --git a/outputs.nix b/outputs.nix index 248d332..a846a48 100644 --- a/outputs.nix +++ b/outputs.nix @@ -162,10 +162,6 @@ in (utils.lib.eachSystem (builtins.filter filter_system utils.lib.defaultSystems in { nixosConfigurations = utils.buildHost hosts.malobeo.hosts; - #nixosConfigurations = import ./machines/configuration.nix (inputs // { - # inherit inputs; - # self = self; - #}); nixosModules.malobeo = { host.imports = [ ./machines/durruti/host_config.nix ]; -- 2.51.2 From ecefcc609c478bc4e66ffd7f91d54e8fa2527bc1 Mon Sep 17 00:00:00 2001 From: kalipso Date: Wed, 22 Jan 2025 17:33:27 +0100 Subject: [PATCH 11/11] [run-vm] mv to /scripts --- outputs.nix | 59 +---------------------------------------------- scripts/run-vm.sh | 56 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 58 deletions(-) create mode 100644 scripts/run-vm.sh diff --git a/outputs.nix b/outputs.nix index a846a48..c8dac17 100644 --- a/outputs.nix +++ b/outputs.nix @@ -72,64 +72,7 @@ in (utils.lib.eachSystem (builtins.filter filter_system utils.lib.defaultSystems ''; }; - run-vm = pkgs.writeShellScriptBin "run-vm" '' - usage() { - echo "Usage: run-vm [--networking] [--dummy-secrets] [--no-disko]" - echo "ATTENTION: This script must be run from the flakes root directory" - echo "--networking setup interfaces. requires root and hostbridge enabled on the host" - echo "--dummy-secrets run vm with dummy sops secrets" - echo "--no-disko disable disko and initrd secrets. needed for real hosts like fanny" - echo "--writable-store enables writable store. necessary for host with nested imperative microvms like fanny" - echo "--var path to directory that should be shared as /var. may require root otherwise some systemd units fail within vm. if dir is empty vm will populate" - exit 1 - } - - # check at least one arg was given - if [ "$#" -lt 1 ]; then - usage - fi - - HOSTNAME=$1 - - # Optionale Argumente - NETWORK=false - DUMMY_SECRETS=false - NO_DISKO=false - RW_STORE=false - VAR_PATH="" - - # check argws - shift - while [[ "$#" -gt 0 ]]; do - case $1 in - --networking) NETWORK=true ;; - --dummy-secrets) DUMMY_SECRETS=true ;; - --no-disko) NO_DISKO=true ;; - --writable-store) RW_STORE=true ;; - --var) - if [[ -n "$2" && ! "$2" =~ ^- ]]; then - VAR_PATH="$2" - shift - else - echo "Error: --var requires a non-empty string argument." - usage - fi - ;; - *) echo "Unknown argument: $1"; usage ;; - esac - shift - done - echo "starting host $HOSTNAME" - echo "enable networking: $NETWORK" - echo "deploy dummy secrets: $DUMMY_SECRETS" - echo "disable disko and initrd secrets: $NO_DISKO" - echo "use writable store: $RW_STORE" - if [ -n "$VAR_PATH" ]; then - echo "sharing var directory: $VAR_PATH" - fi - - ${pkgs.nix}/bin/nix run --show-trace --impure --expr "((builtins.getFlake \"$(pwd)\").vmBuilder.x86_64-linux \"$HOSTNAME\" $NETWORK $DUMMY_SECRETS $NO_DISKO \"$VAR_PATH\" $RW_STORE).config.microvm.declaredRunner" - ''; + run-vm = pkgs.writeShellScriptBin "run-vm" (builtins.readFile ./scripts/run-vm.sh); }; apps = { diff --git a/scripts/run-vm.sh b/scripts/run-vm.sh new file mode 100644 index 0000000..c9eee2a --- /dev/null +++ b/scripts/run-vm.sh @@ -0,0 +1,56 @@ +usage() { + echo "Usage: run-vm [--networking] [--dummy-secrets] [--no-disko]" + echo "ATTENTION: This script must be run from the flakes root directory" + echo "--networking setup interfaces. requires root and hostbridge enabled on the host" + echo "--dummy-secrets run vm with dummy sops secrets" + echo "--no-disko disable disko and initrd secrets. needed for real hosts like fanny" + echo "--writable-store enables writable store. necessary for host with nested imperative microvms like fanny" + echo "--var path to directory that should be shared as /var. may require root otherwise some systemd units fail within vm. if dir is empty vm will populate" + exit 1 +} + +# check at least one arg was given +if [ "$#" -lt 1 ]; then + usage +fi + +HOSTNAME=$1 + +# Optionale Argumente +NETWORK=false +DUMMY_SECRETS=false +NO_DISKO=false +RW_STORE=false +VAR_PATH="" + +# check argws +shift +while [[ "$#" -gt 0 ]]; do + case $1 in + --networking) NETWORK=true ;; + --dummy-secrets) DUMMY_SECRETS=true ;; + --no-disko) NO_DISKO=true ;; + --writable-store) RW_STORE=true ;; + --var) + if [[ -n "$2" && ! "$2" =~ ^- ]]; then + VAR_PATH="$2" + shift + else + echo "Error: --var requires a non-empty string argument." + usage + fi + ;; + *) echo "Unknown argument: $1"; usage ;; + esac + shift +done +echo "starting host $HOSTNAME" +echo "enable networking: $NETWORK" +echo "deploy dummy secrets: $DUMMY_SECRETS" +echo "disable disko and initrd secrets: $NO_DISKO" +echo "use writable store: $RW_STORE" +if [ -n "$VAR_PATH" ]; then + echo "sharing var directory: $VAR_PATH" +fi + +nix run --show-trace --impure --expr "((builtins.getFlake \"$(pwd)\").vmBuilder.x86_64-linux \"$HOSTNAME\" $NETWORK $DUMMY_SECRETS $NO_DISKO \"$VAR_PATH\" $RW_STORE).config.microvm.declaredRunner" -- 2.51.2