WIP ssh-ng://

WIP: Avoid custom logic copying outputs from the remote builder
We need a replacement for the nar member logic, however. And maybe also a test that fails until this is fixed (this one should not be passing).
2025-02-14 18:51:49 -05:00 · 2025-02-14 18:50:38 -05:00 · 2025-02-14 18:50:38 -05:00 · 2025-02-14 18:48:46 -05:00 · 2025-02-14 18:15:22 -05:00 · 2025-02-13 22:13:32 -05:00
35 changed files with 283 additions and 858 deletions
--- a/flake.lock
+++ b/flake.lock
@@ -1,51 +1,11 @@
 {
  "nodes": {
-    "flake-parts": {
-      "inputs": {
-        "nixpkgs-lib": [
-          "nix-eval-jobs",
-          "nixpkgs"
-        ]
-      },
-      "locked": {
-        "lastModified": 1722555600,
-        "narHash": "sha256-XOQkdLafnb/p9ij77byFQjDf5m5QYl9b2REiVClC+x4=",
-        "owner": "hercules-ci",
-        "repo": "flake-parts",
-        "rev": "8471fe90ad337a8074e957b69ca4d0089218391d",
-        "type": "github"
-      },
-      "original": {
-        "owner": "hercules-ci",
-        "repo": "flake-parts",
-        "type": "github"
-      }
-    },
-    "libgit2": {
-      "flake": false,
-      "locked": {
-        "lastModified": 1715853528,
-        "narHash": "sha256-J2rCxTecyLbbDdsyBWn9w7r3pbKRMkI9E7RvRgAqBdY=",
-        "owner": "libgit2",
-        "repo": "libgit2",
-        "rev": "36f7e21ad757a3dacc58cf7944329da6bc1d6e96",
-        "type": "github"
-      },
-      "original": {
-        "owner": "libgit2",
-        "ref": "v1.8.1",
-        "repo": "libgit2",
-        "type": "github"
-      }
-    },
    "nix": {
      "inputs": {
        "flake-compat": [],
        "flake-parts": [],
        "git-hooks-nix": [],
-        "libgit2": [
-          "libgit2"
-        ],
+        "nixfmt": [],
        "nixpkgs": [
          "nixpkgs"
        ],
@@ -53,88 +13,59 @@
        "nixpkgs-regression": []
      },
      "locked": {
-        "lastModified": 1726787955,
-        "narHash": "sha256-XFznzb8L4SdUm9u+w3DPpMWJhffuv+/6+aiVl00slns=",
+        "lastModified": 1739571938,
+        "narHash": "sha256-NlaLAed/xei6RWpU2HIIbDjILRC4l1NIfGeyrn7ALQs=",
        "owner": "NixOS",
        "repo": "nix",
-        "rev": "a7fdef6858dd45b9d7bda7c92324c63faee7f509",
+        "rev": "ffc649d2eabdd3e678b5bcc211dd59fd06debf3e",
        "type": "github"
      },
      "original": {
        "owner": "NixOS",
-        "ref": "2.24-maintenance",
+        "ref": "ssh-ng-extensions-for-hydra",
        "repo": "nix",
        "type": "github"
      }
    },
    "nix-eval-jobs": {
-      "inputs": {
-        "flake-parts": "flake-parts",
-        "nix-github-actions": [],
-        "nixpkgs": [
-          "nixpkgs"
-        ],
-        "treefmt-nix": "treefmt-nix"
-      },
+      "flake": false,
      "locked": {
-        "lastModified": 1733814344,
-        "narHash": "sha256-3wwtKpS5tUBdjaGeSia7CotonbiRB6K5Kp0dsUt3nzU=",
-        "owner": "nix-community",
+        "lastModified": 1739499741,
+        "narHash": "sha256-dNMJY6+G3PwE8lIAhwetPJdA2DxCEKRXPY/EtHmdDh4=",
+        "owner": "Ericson2314",
        "repo": "nix-eval-jobs",
-        "rev": "889ea1406736b53cf165b6c28398aae3969418d1",
+        "rev": "de345eb4518d952c2d86261b270f2c31edecd3de",
        "type": "github"
      },
      "original": {
-        "owner": "nix-community",
-        "ref": "release-2.24",
+        "owner": "Ericson2314",
+        "ref": "nix-2.27",
        "repo": "nix-eval-jobs",
        "type": "github"
      }
    },
    "nixpkgs": {
      "locked": {
-        "lastModified": 1726688310,
-        "narHash": "sha256-Xc9lEtentPCEtxc/F1e6jIZsd4MPDYv4Kugl9WtXlz0=",
+        "lastModified": 1739461644,
+        "narHash": "sha256-1o1qR0KYozYGRrnqytSpAhVBYLNBHX+Lv6I39zGRzKM=",
        "owner": "NixOS",
        "repo": "nixpkgs",
-        "rev": "dbebdd67a6006bb145d98c8debf9140ac7e651d0",
+        "rev": "97a719c9f0a07923c957cf51b20b329f9fb9d43f",
        "type": "github"
      },
      "original": {
        "owner": "NixOS",
-        "ref": "nixos-24.05-small",
+        "ref": "nixos-24.11-small",
        "repo": "nixpkgs",
        "type": "github"
      }
    },
    "root": {
      "inputs": {
-        "libgit2": "libgit2",
        "nix": "nix",
        "nix-eval-jobs": "nix-eval-jobs",
        "nixpkgs": "nixpkgs"
      }
-    },
-    "treefmt-nix": {
-      "inputs": {
-        "nixpkgs": [
-          "nix-eval-jobs",
-          "nixpkgs"
-        ]
-      },
-      "locked": {
-        "lastModified": 1723303070,
-        "narHash": "sha256-krGNVA30yptyRonohQ+i9cnK+CfCpedg6z3qzqVJcTs=",
-        "owner": "numtide",
-        "repo": "treefmt-nix",
-        "rev": "14c092e0326de759e16b37535161b3cb9770cea3",
-        "type": "github"
-      },
-      "original": {
-        "owner": "numtide",
-        "repo": "treefmt-nix",
-        "type": "github"
-      }
    }
  },
  "root": "root",
--- a/flake.nix
+++ b/flake.nix
@@ -1,25 +1,26 @@
 {
  description = "A Nix-based continuous build system";

-  inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-24.05-small";
+  inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-24.11-small";

-  inputs.libgit2 = { url = "github:libgit2/libgit2/v1.8.1"; flake = false; };
-  inputs.nix.url = "github:NixOS/nix/2.24-maintenance";
-  inputs.nix.inputs.nixpkgs.follows = "nixpkgs";
-  inputs.nix.inputs.libgit2.follows = "libgit2";
+  inputs.nix = {
+    url = "github:NixOS/nix/ssh-ng-extensions-for-hydra";
+    inputs.nixpkgs.follows = "nixpkgs";

-  inputs.nix-eval-jobs.url = "github:nix-community/nix-eval-jobs/release-2.24";
-  inputs.nix-eval-jobs.inputs.nixpkgs.follows = "nixpkgs";
+    # hide nix dev tooling from our lock file
+    inputs.flake-parts.follows = "";
+    inputs.git-hooks-nix.follows = "";
+    inputs.nixpkgs-regression.follows = "";
+    inputs.nixpkgs-23-11.follows = "";
+    inputs.flake-compat.follows = "";
+    inputs.nixfmt.follows = "";
+  };

-  # hide nix dev tooling from our lock file
-  inputs.nix.inputs.flake-parts.follows = "";
-  inputs.nix.inputs.git-hooks-nix.follows = "";
-  inputs.nix.inputs.nixpkgs-regression.follows = "";
-  inputs.nix.inputs.nixpkgs-23-11.follows = "";
-  inputs.nix.inputs.flake-compat.follows = "";
-
-  # hide nix-eval-jobs dev tooling from our lock file
-  inputs.nix-eval-jobs.inputs.nix-github-actions.follows = "";
+  inputs.nix-eval-jobs = {
+    url = "github:Ericson2314/nix-eval-jobs/nix-2.27";
+    # We want to control the deps precisely
+    flake = false;
+  };

  outputs = { self, nixpkgs, nix, nix-eval-jobs, ... }:
    let
@@ -30,9 +31,9 @@

      # A Nixpkgs overlay that provides a 'hydra' package.
      overlays.default = final: prev: {
+        nix-eval-jobs = final.callPackage nix-eval-jobs {};
        hydra = final.callPackage ./package.nix {
          inherit (nixpkgs.lib) fileset;
-          nix-eval-jobs = nix-eval-jobs.packages.${final.system}.default;
          rawSrc = self;
          nix-perl-bindings = final.nixComponents.nix-perl-bindings;
        };
@@ -74,11 +75,19 @@
      });

      packages = forEachSystem (system: {
+        nix-eval-jobs = nixpkgs.legacyPackages.${system}.callPackage nix-eval-jobs {
+          nix = nix.packages.${system}.nix;
+        };
        hydra = nixpkgs.legacyPackages.${system}.callPackage ./package.nix {
          inherit (nixpkgs.lib) fileset;
-          nix-eval-jobs = nix-eval-jobs.packages.${system}.default;
+          inherit (self.packages.${system}) nix-eval-jobs;
          rawSrc = self;
-          nix = nix.packages.${system}.nix;
+          inherit (nix.packages.${system})
+            nix-util
+            nix-store
+            nix-main
+            nix-cli
+            ;
          nix-perl-bindings = nix.hydraJobs.perlBindings.${system};
        };
        default = self.packages.${system}.hydra;
--- a/meson.build
+++ b/meson.build
@@ -8,22 +8,22 @@ project('hydra', 'cpp',
  ],
 )

+nix_util_dep = dependency('nix-util', required: true)
 nix_store_dep = dependency('nix-store', required: true)
 nix_main_dep = dependency('nix-main', required: true)
-nix_expr_dep = dependency('nix-expr', required: true)
-nix_flake_dep = dependency('nix-flake', required: true)
-nix_cmd_dep = dependency('nix-cmd', required: true)

 # Nix need extra flags not provided in its pkg-config files.
 nix_dep = declare_dependency(
  dependencies: [
+    nix_util_dep,
    nix_store_dep,
    nix_main_dep,
-    nix_expr_dep,
-    nix_flake_dep,
-    nix_cmd_dep,
  ],
-  compile_args: ['-include', 'nix/config.h'],
+  compile_args: [
+    '-include', 'nix/config-util.hh',
+    '-include', 'nix/config-store.hh',
+    '-include', 'nix/config-main.hh',
+  ],
 )

 pqxx_dep = dependency('libpqxx', required: true)
--- a/package.nix
+++ b/package.nix
@@ -8,7 +8,10 @@

 , perlPackages

-, nix
+, nix-util
+, nix-store
+, nix-main
+, nix-cli
 , nix-perl-bindings
 , git

@@ -90,7 +93,6 @@ let
        DateTime
        DBDPg
        DBDSQLite
-        DBIxClassHelpers
        DigestSHA1
        EmailMIME
        EmailSender
@@ -163,7 +165,7 @@ stdenv.mkDerivation (finalAttrs: {
    nukeReferences
    pkg-config
    mdbook
-    nix
+    nix-cli
    perlDeps
    perl
    unzip
@@ -173,7 +175,9 @@ stdenv.mkDerivation (finalAttrs: {
    libpqxx
    openssl
    libxslt
-    nix
+    nix-util
+    nix-store
+    nix-main
    perlDeps
    perl
    boost
@@ -200,13 +204,14 @@ stdenv.mkDerivation (finalAttrs: {
    glibcLocales
    libressl.nc
    python3
+    nix-cli
  ];

  hydraPath = lib.makeBinPath (
    [
      subversion
      openssh
-      nix
+      nix-cli
      coreutils
      findutils
      pixz
@@ -267,7 +272,7 @@ stdenv.mkDerivation (finalAttrs: {
            --prefix PATH ':' $out/bin:$hydraPath \
            --set HYDRA_RELEASE ${version} \
            --set HYDRA_HOME $out/libexec/hydra \
-            --set NIX_RELEASE ${nix.name or "unknown"} \
+            --set NIX_RELEASE ${nix-cli.name or "unknown"} \
            --set NIX_EVAL_JOBS_RELEASE ${nix-eval-jobs.name or "unknown"}
    done
  '';
@@ -275,5 +280,5 @@ stdenv.mkDerivation (finalAttrs: {
  dontStrip = true;

  meta.description = "Build of Hydra on ${stdenv.system}";
-  passthru = { inherit perlDeps nix; };
+  passthru = { inherit perlDeps; };
 })
--- a/src/hydra-queue-runner/build-remote.cc
+++ b/src/hydra-queue-runner/build-remote.cc
@@ -7,140 +7,35 @@

 #include "build-result.hh"
 #include "path.hh"
+#include "ssh-store.hh"
 #include "serve-protocol.hh"
-#include "serve-protocol-impl.hh"
 #include "state.hh"
 #include "current-process.hh"
 #include "processes.hh"
 #include "util.hh"
-#include "serve-protocol.hh"
-#include "serve-protocol-impl.hh"
 #include "ssh.hh"
 #include "finally.hh"
 #include "url.hh"

 using namespace nix;

+bool ::Machine::isLocalhost() const
+{
+    return storeUri.params.empty() && std::visit(overloaded {
+        [](const StoreReference::Auto &) {
+            return true;
+        },
+        [](const StoreReference::Specified & s) {
+            return
+               (s.scheme == "local" || s.scheme == "unix") ||
+               ((s.scheme == "ssh" || s.scheme == "ssh-ng") &&
+                s.authority == "localhost");
+        },
+    }, storeUri.variant);
+}
+
 namespace nix::build_remote {

-static Strings extraStoreArgs(std::string & machine)
-{
-    Strings result;
-    try {
-        auto parsed = parseURL(machine);
-        if (parsed.scheme != "ssh") {
-            throw SysError("Currently, only (legacy-)ssh stores are supported!");
-        }
-        machine = parsed.authority.value_or("");
-        auto remoteStore = parsed.query.find("remote-store");
-        if (remoteStore != parsed.query.end()) {
-            result = {"--store", shellEscape(remoteStore->second)};
-        }
-    } catch (BadURL &) {
-        // We just try to continue with `machine->sshName` here for backwards compat.
-    }
-
-    return result;
-}
-
-static std::unique_ptr<SSHMaster::Connection> openConnection(
-    ::Machine::ptr machine, SSHMaster & master)
-{
-    Strings command = {"nix-store", "--serve", "--write"};
-    if (machine->isLocalhost()) {
-        command.push_back("--builders");
-        command.push_back("");
-    } else {
-        command.splice(command.end(), extraStoreArgs(machine->sshName));
-    }
-
-    auto ret = master.startCommand(std::move(command), {
-        "-a", "-oBatchMode=yes", "-oConnectTimeout=60", "-oTCPKeepAlive=yes"
-    });
-
-    // XXX: determine the actual max value we can use from /proc.
-
-    // FIXME: Should this be upstreamed into `startCommand` in Nix?
-
-    int pipesize = 1024 * 1024;
-
-    fcntl(ret->in.get(), F_SETPIPE_SZ, &pipesize);
-    fcntl(ret->out.get(), F_SETPIPE_SZ, &pipesize);
-
-    return ret;
-}
-
-
-static void copyClosureTo(
-    ::Machine::Connection & conn,
-    Store & destStore,
-    const StorePathSet & paths,
-    SubstituteFlag useSubstitutes = NoSubstitute)
-{
-    StorePathSet closure;
-    destStore.computeFSClosure(paths, closure);
-
-    /* Send the "query valid paths" command with the "lock" option
-       enabled. This prevents a race where the remote host
-       garbage-collect paths that are already there. Optionally, ask
-       the remote host to substitute missing paths. */
-    // FIXME: substitute output pollutes our build log
-    /* Get back the set of paths that are already valid on the remote
-       host. */
-    auto present = conn.queryValidPaths(
-        destStore, true, closure, useSubstitutes);
-
-    if (present.size() == closure.size()) return;
-
-    auto sorted = destStore.topoSortPaths(closure);
-
-    StorePathSet missing;
-    for (auto i = sorted.rbegin(); i != sorted.rend(); ++i)
-        if (!present.count(*i)) missing.insert(*i);
-
-    printMsg(lvlDebug, "sending %d missing paths", missing.size());
-
-    std::unique_lock<std::timed_mutex> sendLock(conn.machine->state->sendLock,
-        std::chrono::seconds(600));
-
-    conn.to << ServeProto::Command::ImportPaths;
-    destStore.exportPaths(missing, conn.to);
-    conn.to.flush();
-
-    if (readInt(conn.from) != 1)
-        throw Error("remote machine failed to import closure");
-}
-
-
-// FIXME: use Store::topoSortPaths().
-static StorePaths reverseTopoSortPaths(const std::map<StorePath, UnkeyedValidPathInfo> & paths)
-{
-    StorePaths sorted;
-    StorePathSet visited;
-
-    std::function<void(const StorePath & path)> dfsVisit;
-
-    dfsVisit = [&](const StorePath & path) {
-        if (!visited.insert(path).second) return;
-
-        auto info = paths.find(path);
-        auto references = info == paths.end() ? StorePathSet() : info->second.references;
-
-        for (auto & i : references)
-            /* Don't traverse into paths that don't exist.  That can
-               happen due to substitutes for non-existent paths. */
-            if (i != path && paths.count(i))
-                dfsVisit(i);
-
-        sorted.push_back(path);
-    };
-
-    for (auto & i : paths)
-        dfsVisit(i.first);
-
-    return sorted;
-}
-
 static std::pair<Path, AutoCloseFD> openLogFile(const std::string & logDir, const StorePath & drvPath)
 {
    std::string base(drvPath.to_string());
@@ -198,18 +93,18 @@ static BasicDerivation sendInputs(
        MaintainCount<counter> mc2(nrStepsCopyingTo);

        printMsg(lvlDebug, "sending closure of ‘%s’ to ‘%s’",
-            localStore.printStorePath(step.drvPath), conn.machine->sshName);
+            localStore.printStorePath(step.drvPath), conn.machine->storeUri.render());

        auto now1 = std::chrono::steady_clock::now();

        /* Copy the input closure. */
-        if (conn.machine->isLocalhost()) {
-            StorePathSet closure;
-            destStore.computeFSClosure(basicDrv.inputSrcs, closure);
-            copyPaths(destStore, localStore, closure, NoRepair, NoCheckSigs, NoSubstitute);
-        } else {
-            copyClosureTo(conn, destStore, basicDrv.inputSrcs, Substitute);
-        }
+        copyClosure(
+            destStore,
+            conn.machine->isLocalhost() ? localStore : *conn.store,
+            basicDrv.inputSrcs,
+            NoRepair,
+            NoCheckSigs,
+            Substitute);

        auto now2 = std::chrono::steady_clock::now();

@@ -224,11 +119,10 @@ static BuildResult performBuild(
    Store & localStore,
    StorePath drvPath,
    const BasicDerivation & drv,
-    const ServeProto::BuildOptions & options,
    counter & nrStepsBuilding
 )
 {
-    conn.putBuildDerivationRequest(localStore, drvPath, drv, options);
+    auto kont = conn.store->buildDerivationAsync(drvPath, drv, bmNormal);

    BuildResult result;

@@ -237,7 +131,10 @@ static BuildResult performBuild(
    startTime = time(0);
    {
        MaintainCount<counter> mc(nrStepsBuilding);
-        result = ServeProto::Serialise<BuildResult>::read(localStore, conn);
+        result = kont();
+        // Without proper call-once functions, we need to manually
+        // delete after calling.
+        kont = {};
    }
    stopTime = time(0);

@@ -253,7 +150,7 @@ static BuildResult performBuild(

    // If the protocol was too old to give us `builtOutputs`, initialize
    // it manually by introspecting the derivation.
-    if (GET_PROTOCOL_MINOR(conn.remoteVersion) < 6)
+    if (GET_PROTOCOL_MINOR(conn.store->getProtocol()) < 6)
    {
        // If the remote is too old to handle CA derivations, we can’t get this
        // far anyways
@@ -278,81 +175,6 @@ static BuildResult performBuild(
    return result;
 }

-static std::map<StorePath, UnkeyedValidPathInfo> queryPathInfos(
-    ::Machine::Connection & conn,
-    Store & localStore,
-    StorePathSet & outputs,
-    size_t & totalNarSize
-)
-{
-
-    /* Get info about each output path. */
-    std::map<StorePath, UnkeyedValidPathInfo> infos;
-    conn.to << ServeProto::Command::QueryPathInfos;
-    ServeProto::write(localStore, conn, outputs);
-    conn.to.flush();
-    while (true) {
-        auto storePathS = readString(conn.from);
-        if (storePathS == "") break;
-
-        auto storePath = localStore.parseStorePath(storePathS);
-        auto info = ServeProto::Serialise<UnkeyedValidPathInfo>::read(localStore, conn);
-        totalNarSize += info.narSize;
-        infos.insert_or_assign(std::move(storePath), std::move(info));
-    }
-
-    return infos;
-}
-
-static void copyPathFromRemote(
-    ::Machine::Connection & conn,
-    NarMemberDatas & narMembers,
-    Store & localStore,
-    Store & destStore,
-    const ValidPathInfo & info
-)
-{
-      /* Receive the NAR from the remote and add it to the
-          destination store. Meanwhile, extract all the info from the
-          NAR that getBuildOutput() needs. */
-      auto source2 = sinkToSource([&](Sink & sink)
-      {
-          /* Note: we should only send the command to dump the store
-              path to the remote if the NAR is actually going to get read
-              by the destination store, which won't happen if this path
-              is already valid on the destination store. Since this
-              lambda function only gets executed if someone tries to read
-              from source2, we will send the command from here rather
-              than outside the lambda. */
-          conn.to << ServeProto::Command::DumpStorePath << localStore.printStorePath(info.path);
-          conn.to.flush();
-
-          TeeSource tee(conn.from, sink);
-          extractNarData(tee, localStore.printStorePath(info.path), narMembers);
-      });
-
-      destStore.addToStore(info, *source2, NoRepair, NoCheckSigs);
-}
-
-static void copyPathsFromRemote(
-    ::Machine::Connection & conn,
-    NarMemberDatas & narMembers,
-    Store & localStore,
-    Store & destStore,
-    const std::map<StorePath, UnkeyedValidPathInfo> & infos
-)
-{
-      auto pathsSorted = reverseTopoSortPaths(infos);
-
-      for (auto & path : pathsSorted) {
-          auto & info = infos.find(path)->second;
-          copyPathFromRemote(
-              conn, narMembers, localStore, destStore,
-              ValidPathInfo { path, info });
-      }
-
-}
-
 }

 /* using namespace nix::build_remote; */
@@ -412,21 +234,9 @@ void RemoteResult::updateWithBuildResult(const nix::BuildResult & buildResult)

 }

-/* Utility guard object to auto-release a semaphore on destruction. */
-template <typename T>
-class SemaphoreReleaser {
-public:
-    SemaphoreReleaser(T* s) : sem(s) {}
-    ~SemaphoreReleaser() { sem->release(); }
-
-private:
-    T* sem;
-};

 void State::buildRemote(ref<Store> destStore,
-    MachineReservation::ptr & reservation,
    ::Machine::ptr machine, Step::ptr step,
-    const ServeProto::BuildOptions & buildOptions,
    RemoteResult & result, std::shared_ptr<ActiveStep> activeStep,
    std::function<void(StepState)> updateStep,
    NarMemberDatas & narMembers)
@@ -441,27 +251,43 @@ void State::buildRemote(ref<Store> destStore,

        updateStep(ssConnecting);

-        SSHMaster master {
-            machine->sshName,
-            machine->sshKey,
-            machine->sshPublicHostKey,
-            false, // no SSH master yet
-            false, // no compression yet
-            logFD.get(),
-        };
-
        // FIXME: rewrite to use Store.
-        auto child = build_remote::openConnection(machine, master);
+        ::Machine::Connection conn {
+            .machine = machine,
+            .store = [&]{
+                auto * pSpecified = std::get_if<StoreReference::Specified>(&machine->storeUri.variant);
+                if (!pSpecified || pSpecified->scheme != "ssh-ng") {
+                    throw Error("Currently, only ssh-ng:// stores are supported!");
+                }
+
+                auto remoteStore = machine->openStore().dynamic_pointer_cast<RemoteStore>();
+                auto remoteStoreConfig = std::dynamic_pointer_cast<SSHStoreConfig>(remoteStore);
+                assert(remoteStore);
+
+                if (machine->isLocalhost()) {
+                    auto rp_new = remoteStoreConfig->remoteProgram.get();
+                    rp_new.push_back("--builders");
+                    rp_new.push_back("");
+                    const_cast<nix::Setting<Strings> &>(remoteStoreConfig->remoteProgram).assign(rp_new);
+                }
+                remoteStoreConfig->extraSshArgs = {
+                    "-a", "-oBatchMode=yes", "-oConnectTimeout=60", "-oTCPKeepAlive=yes"
+                };
+
+                // TODO logging
+                //const_cast<nix::Setting<int> &>(remoteStore->logFD).assign(logFD.get());
+
+                return nix::ref{remoteStore};
+            }(),
+        };

        {
            auto activeStepState(activeStep->state_.lock());
            if (activeStepState->cancelled) throw Error("step cancelled");
-            activeStepState->pid = child->sshPid;
        }

        Finally clearPid([&]() {
            auto activeStepState(activeStep->state_.lock());
-            activeStepState->pid = -1;

            /* FIXME: there is a slight race here with step
               cancellation in State::processQueueChange(), which
@@ -471,41 +297,13 @@ void State::buildRemote(ref<Store> destStore,
               process. Meh. */
        });

-        ::Machine::Connection conn {
-            {
-                .to = child->in.get(),
-                .from = child->out.get(),
-                /* Handshake. */
-                .remoteVersion = 0xdadbeef, // FIXME avoid dummy initialize
-            },
-            /*.machine =*/ machine,
-        };
-
        Finally updateStats([&]() {
-            bytesReceived += conn.from.read;
-            bytesSent += conn.to.written;
+            // TODO
+            //auto stats = conn.store->getConnectionStats();
+            //bytesReceived += stats.bytesReceived;
+            //bytesSent += stats.bytesSent;
        });

-        constexpr ServeProto::Version our_version = 0x206;
-
-        try {
-            conn.remoteVersion = decltype(conn)::handshake(
-                conn.to,
-                conn.from,
-                our_version,
-                machine->sshName);
-        } catch (EndOfFile & e) {
-            child->sshPid.wait();
-            std::string s = chomp(readFile(result.logFile));
-            throw Error("cannot connect to ‘%1%’: %2%", machine->sshName, s);
-        }
-
-        // Do not attempt to speak a newer version of the protocol.
-        //
-        // Per https://github.com/NixOS/nix/issues/9584 should be handled as
-        // part of `handshake` in upstream nix.
-        conn.remoteVersion = std::min(conn.remoteVersion, our_version);
-
        {
            auto info(machine->state->connectInfo.lock());
            info->consecutiveFailures = 0;
@@ -534,7 +332,7 @@ void State::buildRemote(ref<Store> destStore,
        /* Do the build. */
        printMsg(lvlDebug, "building ‘%s’ on ‘%s’",
            localStore->printStorePath(step->drvPath),
-            machine->sshName);
+            machine->storeUri.render());

        updateStep(ssBuilding);

@@ -543,7 +341,6 @@ void State::buildRemote(ref<Store> destStore,
            *localStore,
            step->drvPath,
            resolvedDrv,
-            buildOptions,
            nrStepsBuilding
        );

@@ -557,29 +354,11 @@ void State::buildRemote(ref<Store> destStore,
           get a build log. */
        if (result.isCached) {
            printMsg(lvlInfo, "outputs of ‘%s’ substituted or already valid on ‘%s’",
-                localStore->printStorePath(step->drvPath), machine->sshName);
+                localStore->printStorePath(step->drvPath), machine->storeUri.render());
            unlink(result.logFile.c_str());
            result.logFile = "";
        }

-        /* Throttle CPU-bound work. Opportunistically skip updating the current
-         * step, since this requires a DB roundtrip. */
-        if (!localWorkThrottler.try_acquire()) {
-            MaintainCount<counter> mc(nrStepsWaitingForDownloadSlot);
-            updateStep(ssWaitingForLocalSlot);
-            localWorkThrottler.acquire();
-        }
-        SemaphoreReleaser releaser(&localWorkThrottler);
-
-        /* Once we've started copying outputs, release the machine reservation
-         * so further builds can happen. We do not release the machine earlier
-         * to avoid situations where the queue runner is bottlenecked on
-         * copying outputs and we end up building too many things that we
-         * haven't been able to allow copy slots for. */
-        assert(reservation.unique());
-        reservation = 0;
-        wakeDispatcher();
-
        StorePathSet outputs;
        for (auto & [_, realisation] : buildResult.builtOutputs)
            outputs.insert(realisation.outPath);
@@ -592,19 +371,12 @@ void State::buildRemote(ref<Store> destStore,

            auto now1 = std::chrono::steady_clock::now();

-            size_t totalNarSize = 0;
-            auto infos = build_remote::queryPathInfos(conn, *localStore, outputs, totalNarSize);
-
-            if (totalNarSize > maxOutputSize) {
-                result.stepStatus = bsNarSizeLimitExceeded;
-                return;
-            }
-
            /* Copy each path. */
-            printMsg(lvlDebug, "copying outputs of ‘%s’ from ‘%s’ (%d bytes)",
-                localStore->printStorePath(step->drvPath), machine->sshName, totalNarSize);
+            printMsg(lvlDebug, "copying outputs of ‘%s’ from ‘%s’",
+                localStore->printStorePath(step->drvPath), machine->storeUri.render());
+
+            copyClosure(*conn.store, *destStore, outputs);

-            build_remote::copyPathsFromRemote(conn, narMembers, *localStore, *destStore, infos);
            auto now2 = std::chrono::steady_clock::now();

            result.overhead += std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count();
@@ -625,9 +397,11 @@ void State::buildRemote(ref<Store> destStore,
            }
        }

-        /* Shut down the connection. */
-        child->in = -1;
-        child->sshPid.wait();
+        /* Shut down the connection done by RAII.
+
+           Only difference is kill() instead of wait() (i.e. send signal
+           then wait())
+         */

    } catch (Error & e) {
        /* Disable this machine until a certain period of time has
@@ -641,7 +415,7 @@ void State::buildRemote(ref<Store> destStore,
            info->consecutiveFailures = std::min(info->consecutiveFailures + 1, (unsigned int) 4);
            info->lastFailure = now;
            int delta = retryInterval * std::pow(retryBackoff, info->consecutiveFailures - 1) + (rand() % 30);
-            printMsg(lvlInfo, "will disable machine ‘%1%’ for %2%s", machine->sshName, delta);
+            printMsg(lvlInfo, "will disable machine ‘%1%’ for %2%s", machine->storeUri.render(), delta);
            info->disabledUntil = now + std::chrono::seconds(delta);
        }
        throw;
--- a/src/hydra-queue-runner/builder.cc
+++ b/src/hydra-queue-runner/builder.cc
@@ -37,22 +37,19 @@ void State::builder(MachineReservation::ptr reservation)

        try {
            auto destStore = getDestStore();
-            // Might release the reservation.
            res = doBuildStep(destStore, reservation, activeStep);
        } catch (std::exception & e) {
            printMsg(lvlError, "uncaught exception building ‘%s’ on ‘%s’: %s",
-                localStore->printStorePath(activeStep->step->drvPath),
-                reservation ? reservation->machine->sshName : std::string("(no machine)"),
+                localStore->printStorePath(reservation->step->drvPath),
+                reservation->machine->storeUri.render(),
                e.what());
        }
    }

-    /* If the machine hasn't been released yet, release and wake up the dispatcher. */
-    if (reservation) {
-        assert(reservation.unique());
-        reservation = 0;
-        wakeDispatcher();
-    }
+    /* Release the machine and wake up the dispatcher. */
+    assert(reservation.unique());
+    reservation = 0;
+    wakeDispatcher();

    /* If there was a temporary failure, retry the step after an
       exponentially increasing interval. */
@@ -75,11 +72,11 @@ void State::builder(MachineReservation::ptr reservation)


 State::StepResult State::doBuildStep(nix::ref<Store> destStore,
-    MachineReservation::ptr & reservation,
+    MachineReservation::ptr reservation,
    std::shared_ptr<ActiveStep> activeStep)
 {
-    auto step(reservation->step);
-    auto machine(reservation->machine);
+    auto & step(reservation->step);
+    auto & machine(reservation->machine);

    {
        auto step_(step->state.lock());
@@ -101,13 +98,6 @@ State::StepResult State::doBuildStep(nix::ref<Store> destStore,
       it). */
    BuildID buildId;
    std::optional<StorePath> buildDrvPath;
-    // Other fields set below
-    nix::ServeProto::BuildOptions buildOptions {
-        .maxLogSize = maxLogSize,
-        .nrRepeats = step->isDeterministic ? 1u : 0u,
-        .enforceDeterminism = step->isDeterministic,
-        .keepFailed = false,
-    };

    auto conn(dbPool.get());

@@ -142,18 +132,19 @@ State::StepResult State::doBuildStep(nix::ref<Store> destStore,
            {
                auto i = jobsetRepeats.find(std::make_pair(build2->projectName, build2->jobsetName));
                if (i != jobsetRepeats.end())
-                    buildOptions.nrRepeats = std::max(buildOptions.nrRepeats, i->second);
+                    warn("jobset repeats is deprecated; nix stopped supporting this correctly a long time ago.");
            }
        }
        if (!build) build = *dependents.begin();

        buildId = build->id;
        buildDrvPath = build->drvPath;
-        buildOptions.maxSilentTime = build->maxSilentTime;
-        buildOptions.buildTimeout = build->buildTimeout;
+        settings.maxLogSize = maxLogSize;
+        settings.maxSilentTime = build->maxSilentTime;
+        settings.buildTimeout = build->buildTimeout;

        printInfo("performing step ‘%s’ %d times on ‘%s’ (needed by build %d and %d others)",
-            localStore->printStorePath(step->drvPath), buildOptions.nrRepeats + 1, machine->sshName, buildId, (dependents.size() - 1));
+            localStore->printStorePath(step->drvPath), 1, machine->storeUri.render(), buildId, (dependents.size() - 1));
    }

    if (!buildOneDone)
@@ -181,7 +172,7 @@ State::StepResult State::doBuildStep(nix::ref<Store> destStore,
                    unlink(result.logFile.c_str());
                }
            } catch (...) {
-                ignoreException();
+                ignoreExceptionInDestructor();
            }
        }
    });
@@ -199,7 +190,7 @@ State::StepResult State::doBuildStep(nix::ref<Store> destStore,
        {
            auto mc = startDbUpdate();
            pqxx::work txn(*conn);
-            stepNr = createBuildStep(txn, result.startTime, buildId, step, machine->sshName, bsBusy);
+            stepNr = createBuildStep(txn, result.startTime, buildId, step, machine->storeUri.render(), bsBusy);
            txn.commit();
        }

@@ -214,7 +205,7 @@ State::StepResult State::doBuildStep(nix::ref<Store> destStore,

        try {
            /* FIXME: referring builds may have conflicting timeouts. */
-            buildRemote(destStore, reservation, machine, step, buildOptions, result, activeStep, updateStep, narMembers);
+            buildRemote(destStore, machine, step, result, activeStep, updateStep, narMembers);
        } catch (Error & e) {
            if (activeStep->state_.lock()->cancelled) {
                printInfo("marking step %d of build %d as cancelled", stepNr, buildId);
@@ -256,7 +247,7 @@ State::StepResult State::doBuildStep(nix::ref<Store> destStore,
    /* Finish the step in the database. */
    if (stepNr) {
        pqxx::work txn(*conn);
-        finishBuildStep(txn, result, buildId, stepNr, machine->sshName);
+        finishBuildStep(txn, result, buildId, stepNr, machine->storeUri.render());
        txn.commit();
    }

@@ -264,7 +255,7 @@ State::StepResult State::doBuildStep(nix::ref<Store> destStore,
       issue). Retry a number of times. */
    if (result.canRetry) {
        printMsg(lvlError, "possibly transient failure building ‘%s’ on ‘%s’: %s",
-            localStore->printStorePath(step->drvPath), machine->sshName, result.errorMsg);
+            localStore->printStorePath(step->drvPath), machine->storeUri.render(), result.errorMsg);
        assert(stepNr);
        bool retry;
        {
@@ -455,7 +446,7 @@ void State::failStep(
                    build->finishedInDB)
                    continue;
                createBuildStep(txn,
-                    0, build->id, step, machine ? machine->sshName : "",
+                    0, build->id, step, machine ? machine->storeUri.render() : "",
                    result.stepStatus, result.errorMsg, buildId == build->id ? 0 : buildId);
            }

--- a/src/hydra-queue-runner/dispatcher.cc
+++ b/src/hydra-queue-runner/dispatcher.cc
@@ -40,15 +40,13 @@ void State::dispatcher()
            printMsg(lvlDebug, "dispatcher woken up");
            nrDispatcherWakeups++;

-            auto t_before_work = std::chrono::steady_clock::now();
+            auto now1 = std::chrono::steady_clock::now();

            auto sleepUntil = doDispatch();

-            auto t_after_work = std::chrono::steady_clock::now();
+            auto now2 = std::chrono::steady_clock::now();

-            prom.dispatcher_time_spent_running.Increment(
-                std::chrono::duration_cast<std::chrono::microseconds>(t_after_work - t_before_work).count());
-            dispatchTimeMs += std::chrono::duration_cast<std::chrono::milliseconds>(t_after_work - t_before_work).count();
+            dispatchTimeMs += std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count();

            /* Sleep until we're woken up (either because a runnable build
               is added, or because a build finishes). */
@@ -62,10 +60,6 @@ void State::dispatcher()
                *dispatcherWakeup_ = false;
            }

-            auto t_after_sleep = std::chrono::steady_clock::now();
-            prom.dispatcher_time_spent_waiting.Increment(
-                std::chrono::duration_cast<std::chrono::microseconds>(t_after_sleep - t_after_work).count());
-
        } catch (std::exception & e) {
            printError("dispatcher: %s", e.what());
            sleep(1);
@@ -262,7 +256,7 @@ system_time State::doDispatch()
                /* Can this machine do this step? */
                if (!mi.machine->supportsStep(step)) {
                    debug("machine '%s' does not support step '%s' (system type '%s')",
-                        mi.machine->sshName, localStore->printStorePath(step->drvPath), step->drv->platform);
+                        mi.machine->storeUri.render(), localStore->printStorePath(step->drvPath), step->drv->platform);
                    continue;
                }

--- a/src/hydra-queue-runner/hydra-queue-runner.cc
+++ b/src/hydra-queue-runner/hydra-queue-runner.cc
@@ -70,31 +70,10 @@ State::PromMetrics::PromMetrics()
            .Register(*registry)
            .Add({})
    )
-    , dispatcher_time_spent_running(
-        prometheus::BuildCounter()
-            .Name("hydraqueuerunner_dispatcher_time_spent_running")
-            .Help("Time (in micros) spent running the dispatcher")
-            .Register(*registry)
-            .Add({})
-    )
-    , dispatcher_time_spent_waiting(
-        prometheus::BuildCounter()
-            .Name("hydraqueuerunner_dispatcher_time_spent_waiting")
-            .Help("Time (in micros) spent waiting for the dispatcher to obtain work")
-            .Register(*registry)
-            .Add({})
-    )
-    , queue_monitor_time_spent_running(
-        prometheus::BuildCounter()
-            .Name("hydraqueuerunner_queue_monitor_time_spent_running")
-            .Help("Time (in micros) spent running the queue monitor")
-            .Register(*registry)
-            .Add({})
-    )
-    , queue_monitor_time_spent_waiting(
-        prometheus::BuildCounter()
-            .Name("hydraqueuerunner_queue_monitor_time_spent_waiting")
-            .Help("Time (in micros) spent waiting for the queue monitor to obtain work")
+    , queue_max_id(
+        prometheus::BuildGauge()
+            .Name("hydraqueuerunner_queue_max_build_id_info")
+            .Help("Maximum build record ID in the queue")
            .Register(*registry)
            .Add({})
    )
@@ -106,7 +85,6 @@ State::State(std::optional<std::string> metricsAddrOpt)
    : config(std::make_unique<HydraConfig>())
    , maxUnsupportedTime(config->getIntOption("max_unsupported_time", 0))
    , dbPool(config->getIntOption("max_db_connections", 128))
-    , localWorkThrottler(config->getIntOption("max_local_worker_threads", std::min(maxSupportedLocalWorkers, std::max(4u, std::thread::hardware_concurrency()) - 2)))
    , maxOutputSize(config->getIntOption("max_output_size", 2ULL << 30))
    , maxLogSize(config->getIntOption("max_log_size", 64ULL << 20))
    , uploadLogsToBinaryCache(config->getBoolOption("upload_logs_to_binary_cache", false))
@@ -157,65 +135,26 @@ void State::parseMachines(const std::string & contents)
        oldMachines = *machines_;
    }

-    for (auto line : tokenizeString<Strings>(contents, "\n")) {
-        line = trim(std::string(line, 0, line.find('#')));
-        auto tokens = tokenizeString<std::vector<std::string>>(line);
-        if (tokens.size() < 3) continue;
-        tokens.resize(8);
-
-        if (tokens[5] == "-") tokens[5] = "";
-        auto supportedFeatures = tokenizeString<StringSet>(tokens[5], ",");
-
-        if (tokens[6] == "-") tokens[6] = "";
-        auto mandatoryFeatures = tokenizeString<StringSet>(tokens[6], ",");
-
-        for (auto & f : mandatoryFeatures)
-            supportedFeatures.insert(f);
-
-        using MaxJobs = std::remove_const<decltype(nix::Machine::maxJobs)>::type;
-
-        auto machine = std::make_shared<::Machine>(nix::Machine {
-            // `storeUri`, not yet used
-            "",
-            // `systemTypes`
-            tokenizeString<StringSet>(tokens[1], ","),
-            // `sshKey`
-            tokens[2] == "-" ? "" : tokens[2],
-            // `maxJobs`
-            tokens[3] != ""
-                ? string2Int<MaxJobs>(tokens[3]).value()
-                : 1,
-            // `speedFactor`
-            std::stof(tokens[4].c_str()),
-            // `supportedFeatures`
-            std::move(supportedFeatures),
-            // `mandatoryFeatures`
-            std::move(mandatoryFeatures),
-            // `sshPublicHostKey`
-            tokens[7] != "" && tokens[7] != "-"
-                ? tokens[7]
-                : "",
-        });
-
-        machine->sshName = tokens[0];
+    for (auto && machine_ : nix::Machine::parseConfig({}, contents)) {
+        auto machine = std::make_shared<::Machine>(std::move(machine_));

        /* Re-use the State object of the previous machine with the
           same name. */
-        auto i = oldMachines.find(machine->sshName);
+        auto i = oldMachines.find(machine->storeUri.variant);
        if (i == oldMachines.end())
-            printMsg(lvlChatty, "adding new machine ‘%1%’", machine->sshName);
+            printMsg(lvlChatty, "adding new machine ‘%1%’", machine->storeUri.render());
        else
-            printMsg(lvlChatty, "updating machine ‘%1%’", machine->sshName);
+            printMsg(lvlChatty, "updating machine ‘%1%’", machine->storeUri.render());
        machine->state = i == oldMachines.end()
            ? std::make_shared<::Machine::State>()
            : i->second->state;
-        newMachines[machine->sshName] = machine;
+        newMachines[machine->storeUri.variant] = machine;
    }

    for (auto & m : oldMachines)
        if (newMachines.find(m.first) == newMachines.end()) {
            if (m.second->enabled)
-                printInfo("removing machine ‘%1%’", m.first);
+                printInfo("removing machine ‘%1%’", m.second->storeUri.render());
            /* Add a disabled ::Machine object to make sure stats are
               maintained. */
            auto machine = std::make_shared<::Machine>(*(m.second));
@@ -243,7 +182,7 @@ void State::monitorMachinesFile()
        getEnv("NIX_REMOTE_SYSTEMS").value_or(pathExists(defaultMachinesFile) ? defaultMachinesFile : ""), ":");

    if (machinesFiles.empty()) {
-        parseMachines("localhost " +
+        parseMachines("ssh-ng://localhost " +
            (settings.thisSystem == "x86_64-linux" ? "x86_64-linux,i686-linux" : settings.thisSystem.get())
            + " - " + std::to_string(settings.maxBuildJobs) + " 1 "
            + concatStringsSep(",", StoreConfig::getDefaultSystemFeatures()));
@@ -612,7 +551,6 @@ void State::dumpStatus(Connection & conn)
        {"nrActiveSteps", activeSteps_.lock()->size()},
        {"nrStepsBuilding", nrStepsBuilding.load()},
        {"nrStepsCopyingTo", nrStepsCopyingTo.load()},
-        {"nrStepsWaitingForDownloadSlot", nrStepsWaitingForDownloadSlot.load()},
        {"nrStepsCopyingFrom", nrStepsCopyingFrom.load()},
        {"nrStepsWaiting", nrStepsWaiting.load()},
        {"nrUnsupportedSteps", nrUnsupportedSteps.load()},
@@ -680,7 +618,7 @@ void State::dumpStatus(Connection & conn)
                    machine["avgStepTime"] = (float) s->totalStepTime / s->nrStepsDone;
                    machine["avgStepBuildTime"] = (float) s->totalStepBuildTime / s->nrStepsDone;
                }
-                statusJson["machines"][m->sshName] = machine;
+                statusJson["machines"][m->storeUri.render()] = machine;
            }
        }

--- a/src/hydra-queue-runner/queue-monitor.cc
+++ b/src/hydra-queue-runner/queue-monitor.cc
@@ -1,7 +1,6 @@
 #include "state.hh"
 #include "hydra-build-result.hh"
 #include "globals.hh"
-#include "thread-pool.hh"

 #include <cstring>

@@ -38,21 +37,16 @@ void State::queueMonitorLoop(Connection & conn)

    auto destStore = getDestStore();

+    unsigned int lastBuildId = 0;
+
    bool quit = false;
    while (!quit) {
-        auto t_before_work = std::chrono::steady_clock::now();
-
        localStore->clearPathInfoCache();

-        bool done = getQueuedBuilds(conn, destStore);
+        bool done = getQueuedBuilds(conn, destStore, lastBuildId);

        if (buildOne && buildOneDone) quit = true;

-        auto t_after_work = std::chrono::steady_clock::now();
-
-        prom.queue_monitor_time_spent_running.Increment(
-            std::chrono::duration_cast<std::chrono::microseconds>(t_after_work - t_before_work).count());
-
        /* Sleep until we get notification from the database about an
           event. */
        if (done && !quit) {
@@ -62,10 +56,12 @@ void State::queueMonitorLoop(Connection & conn)
            conn.get_notifs();

        if (auto lowestId = buildsAdded.get()) {
+            lastBuildId = std::min(lastBuildId, static_cast<unsigned>(std::stoul(*lowestId) - 1));
            printMsg(lvlTalkative, "got notification: new builds added to the queue");
        }
        if (buildsRestarted.get()) {
            printMsg(lvlTalkative, "got notification: builds restarted");
+            lastBuildId = 0; // check all builds
        }
        if (buildsCancelled.get() || buildsDeleted.get() || buildsBumped.get()) {
            printMsg(lvlTalkative, "got notification: builds cancelled or bumped");
@@ -75,10 +71,6 @@ void State::queueMonitorLoop(Connection & conn)
            printMsg(lvlTalkative, "got notification: jobset shares changed");
            processJobsetSharesChange(conn);
        }
-
-        auto t_after_sleep = std::chrono::steady_clock::now();
-        prom.queue_monitor_time_spent_waiting.Increment(
-            std::chrono::duration_cast<std::chrono::microseconds>(t_after_sleep - t_after_work).count());
    }

    exit(0);
@@ -92,18 +84,20 @@ struct PreviousFailure : public std::exception {


 bool State::getQueuedBuilds(Connection & conn,
-    ref<Store> destStore)
+    ref<Store> destStore, unsigned int & lastBuildId)
 {
    prom.queue_checks_started.Increment();

-    printInfo("checking the queue for builds...");
+    printInfo("checking the queue for builds > %d...", lastBuildId);

    /* Grab the queued builds from the database, but don't process
       them yet (since we don't want a long-running transaction). */
    std::vector<BuildID> newIDs;
-    std::unordered_map<BuildID, Build::ptr> newBuildsByID;
+    std::map<BuildID, Build::ptr> newBuildsByID;
    std::multimap<StorePath, BuildID> newBuildsByPath;

+    unsigned int newLastBuildId = lastBuildId;
+
    {
        pqxx::work txn(conn);

@@ -112,12 +106,17 @@ bool State::getQueuedBuilds(Connection & conn,
             "jobsets.name as jobset, job, drvPath, maxsilent, timeout, timestamp, "
             "globalPriority, priority from Builds "
             "inner join jobsets on builds.jobset_id = jobsets.id "
-             "where finished = 0 order by globalPriority desc, random()");
+             "where builds.id > $1 and finished = 0 order by globalPriority desc, builds.id",
+            lastBuildId);

        for (auto const & row : res) {
            auto builds_(builds.lock());
            BuildID id = row["id"].as<BuildID>();
            if (buildOne && id != buildOne) continue;
+            if (id > newLastBuildId) {
+                newLastBuildId = id;
+                prom.queue_max_id.Set(id);
+            }
            if (builds_->count(id)) continue;

            auto build = std::make_shared<Build>(
@@ -319,13 +318,15 @@ bool State::getQueuedBuilds(Connection & conn,

        /* Stop after a certain time to allow priority bumps to be
           processed. */
-        if (std::chrono::system_clock::now() > start + std::chrono::seconds(60)) {
+        if (std::chrono::system_clock::now() > start + std::chrono::seconds(600)) {
            prom.queue_checks_early_exits.Increment();
            break;
        }
    }

    prom.queue_checks_finished.Increment();
+
+    lastBuildId = newBuildsByID.empty() ? newLastBuildId : newBuildsByID.begin()->first - 1;
    return newBuildsByID.empty();
 }

@@ -404,34 +405,6 @@ void State::processQueueChange(Connection & conn)
 }


-std::map<DrvOutput, std::optional<StorePath>> State::getMissingRemotePaths(
-    ref<Store> destStore,
-    const std::map<DrvOutput, std::optional<StorePath>> & paths)
-{
-    Sync<std::map<DrvOutput, std::optional<StorePath>>> missing_;
-    ThreadPool tp;
-
-    for (auto & [output, maybeOutputPath] : paths) {
-        if (!maybeOutputPath) {
-            auto missing(missing_.lock());
-            missing->insert({output, maybeOutputPath});
-        } else {
-            tp.enqueue([&] {
-                if (!destStore->isValidPath(*maybeOutputPath)) {
-                    auto missing(missing_.lock());
-                    missing->insert({output, maybeOutputPath});
-                }
-            });
-        }
-    }
-
-    tp.process();
-
-    auto missing(missing_.lock());
-    return *missing;
-}
-
-
 Step::ptr State::createStep(ref<Store> destStore,
    Connection & conn, Build::ptr build, const StorePath & drvPath,
    Build::ptr referringBuild, Step::ptr referringStep, std::set<StorePath> & finishedDrvs,
@@ -512,15 +485,16 @@ Step::ptr State::createStep(ref<Store> destStore,

    /* Are all outputs valid? */
    auto outputHashes = staticOutputHashes(*localStore, *(step->drv));
-    std::map<DrvOutput, std::optional<StorePath>> paths;
+    bool valid = true;
+    std::map<DrvOutput, std::optional<StorePath>> missing;
    for (auto & [outputName, maybeOutputPath] : destStore->queryPartialDerivationOutputMap(drvPath, &*localStore)) {
        auto outputHash = outputHashes.at(outputName);
-        paths.insert({{outputHash, outputName}, maybeOutputPath});
+        if (maybeOutputPath && destStore->isValidPath(*maybeOutputPath))
+            continue;
+        valid = false;
+        missing.insert({{outputHash, outputName}, maybeOutputPath});
    }

-    auto missing = getMissingRemotePaths(destStore, paths);
-    bool valid = missing.empty();
-
    /* Try to copy the missing paths from the local store or from
       substitutes. */
    if (!missing.empty()) {
--- a/src/hydra-queue-runner/state.hh
+++ b/src/hydra-queue-runner/state.hh
@@ -6,8 +6,6 @@
 #include <map>
 #include <memory>
 #include <queue>
-#include <regex>
-#include <semaphore>

 #include <prometheus/counter.h>
 #include <prometheus/gauge.h>
@@ -22,9 +20,7 @@
 #include "store-api.hh"
 #include "sync.hh"
 #include "nar-extractor.hh"
-#include "serve-protocol.hh"
-#include "serve-protocol-impl.hh"
-#include "serve-protocol-connection.hh"
+#include "ssh-store.hh"
 #include "machines.hh"


@@ -59,7 +55,6 @@ typedef enum {
    ssConnecting = 10,
    ssSendingInputs = 20,
    ssBuilding = 30,
-    ssWaitingForLocalSlot = 35,
    ssReceivingOutputs = 40,
    ssPostProcessing = 50,
 } StepState;
@@ -243,10 +238,6 @@ struct Machine : nix::Machine
 {
    typedef std::shared_ptr<Machine> ptr;

-    /* TODO Get rid of: `nix::Machine::storeUri` is normalized in a way
-       we are not yet used to, but once we are, we don't need this. */
-    std::string sshName;
-
    struct State {
        typedef std::shared_ptr<State> ptr;
        counter currentJobs{0};
@@ -296,21 +287,19 @@ struct Machine : nix::Machine
        return true;
    }

-    bool isLocalhost()
-    {
-        std::regex r("^(ssh://|ssh-ng://)?localhost$");
-        return std::regex_search(sshName, r);
-    }
+    bool isLocalhost() const;

    // A connection to a machine
-    struct Connection : nix::ServeProto::BasicClientConnection {
+    struct Connection {
        // Backpointer to the machine
        ptr machine;
+        // Opened store
+        nix::ref<nix::RemoteStore> store;
    };
 };


-class HydraConfig;
+struct HydraConfig;


 class State
@@ -360,13 +349,9 @@ private:

    /* The build machines. */
    std::mutex machinesReadyLock;
-    typedef std::map<std::string, Machine::ptr> Machines;
+    typedef std::map<nix::StoreReference::Variant, Machine::ptr> Machines;
    nix::Sync<Machines> machines; // FIXME: use atomic_shared_ptr

-    /* Throttler for CPU-bound local work. */
-    static constexpr unsigned int maxSupportedLocalWorkers = 1024;
-    std::counting_semaphore<maxSupportedLocalWorkers> localWorkThrottler;
-
    /* Various stats. */
    time_t startedAt;
    counter nrBuildsRead{0};
@@ -376,7 +361,6 @@ private:
    counter nrStepsDone{0};
    counter nrStepsBuilding{0};
    counter nrStepsCopyingTo{0};
-    counter nrStepsWaitingForDownloadSlot{0};
    counter nrStepsCopyingFrom{0};
    counter nrStepsWaiting{0};
    counter nrUnsupportedSteps{0};
@@ -465,12 +449,7 @@ private:
        prometheus::Counter& queue_steps_created;
        prometheus::Counter& queue_checks_early_exits;
        prometheus::Counter& queue_checks_finished;
-
-        prometheus::Counter& dispatcher_time_spent_running;
-        prometheus::Counter& dispatcher_time_spent_waiting;
-
-        prometheus::Counter& queue_monitor_time_spent_running;
-        prometheus::Counter& queue_monitor_time_spent_waiting;
+        prometheus::Gauge& queue_max_id;

        PromMetrics();
    };
@@ -514,7 +493,8 @@ private:
    void queueMonitorLoop(Connection & conn);

    /* Check the queue for new builds. */
-    bool getQueuedBuilds(Connection & conn, nix::ref<nix::Store> destStore);
+    bool getQueuedBuilds(Connection & conn,
+        nix::ref<nix::Store> destStore, unsigned int & lastBuildId);

    /* Handle cancellation, deletion and priority bumps. */
    void processQueueChange(Connection & conn);
@@ -522,12 +502,6 @@ private:
    BuildOutput getBuildOutputCached(Connection & conn, nix::ref<nix::Store> destStore,
        const nix::StorePath & drvPath);

-    /* Returns paths missing from the remote store. Paths are processed in
-     * parallel to work around the possible latency of remote stores. */
-    std::map<nix::DrvOutput, std::optional<nix::StorePath>> getMissingRemotePaths(
-        nix::ref<nix::Store> destStore,
-        const std::map<nix::DrvOutput, std::optional<nix::StorePath>> & paths);
-
    Step::ptr createStep(nix::ref<nix::Store> store,
        Connection & conn, Build::ptr build, const nix::StorePath & drvPath,
        Build::ptr referringBuild, Step::ptr referringStep, std::set<nix::StorePath> & finishedDrvs,
@@ -563,13 +537,11 @@ private:
       retried. */
    enum StepResult { sDone, sRetry, sMaybeCancelled };
    StepResult doBuildStep(nix::ref<nix::Store> destStore,
-        MachineReservation::ptr & reservation,
+        MachineReservation::ptr reservation,
        std::shared_ptr<ActiveStep> activeStep);

    void buildRemote(nix::ref<nix::Store> destStore,
-        MachineReservation::ptr & reservation,
        Machine::ptr machine, Step::ptr step,
-        const nix::ServeProto::BuildOptions & buildOptions,
        RemoteResult & result, std::shared_ptr<ActiveStep> activeStep,
        std::function<void(StepState)> updateStep,
        NarMemberDatas & narMembers);
--- a/src/lib/Hydra/Controller/Build.pm
+++ b/src/lib/Hydra/Controller/Build.pm
@@ -238,7 +238,7 @@ sub serveFile {
        # XSS hole.
        $c->response->header('Content-Security-Policy' => 'sandbox allow-scripts');

-        $c->stash->{'plain'} = { data => readIntoSocket(cmd => ["nix", "--experimental-features", "nix-command",
+        $c->stash->{'plain'} = { data => grab(cmd => ["nix", "--experimental-features", "nix-command",
                                                      "store", "cat", "--store", getStoreUri(), "$path"]) };

        # Detect MIME type.
--- a/src/lib/Hydra/Controller/Jobset.pm
+++ b/src/lib/Hydra/Controller/Jobset.pm
@@ -364,21 +364,6 @@ sub evals_GET {
    );
 }

-sub errors :Chained('jobsetChain') :PathPart('errors') :Args(0) :ActionClass('REST') { }
-
-sub errors_GET {
-    my ($self, $c) = @_;
-
-    $c->stash->{template} = 'eval-error.tt';
-
-    my $jobsetName = $c->stash->{params}->{name};
-    $c->stash->{jobset} = $c->stash->{project}->jobsets->find(
-        { name => $jobsetName },
-        { '+columns' => { 'errormsg' => 'errormsg' } }
-    );
-
-    $self->status_ok($c, entity => $c->stash->{jobset});
-}

 # Redirect to the latest finished evaluation of this jobset.
 sub latest_eval : Chained('jobsetChain') PathPart('latest-eval') {
--- a/src/lib/Hydra/Controller/JobsetEval.pm
+++ b/src/lib/Hydra/Controller/JobsetEval.pm
@@ -86,17 +86,6 @@ sub view_GET {
    );
 }

-sub errors :Chained('evalChain') :PathPart('errors') :Args(0) :ActionClass('REST') { }
-
-sub errors_GET {
-    my ($self, $c) = @_;
-
-    $c->stash->{template} = 'eval-error.tt';
-
-    $c->stash->{eval} = $c->model('DB::JobsetEvals')->find($c->stash->{eval}->id, { prefetch => 'evaluationerror' });
-
-    $self->status_ok($c, entity => $c->stash->{eval});
-}

 sub create_jobset : Chained('evalChain') PathPart('create-jobset') Args(0) {
    my ($self, $c) = @_;
--- a/src/lib/Hydra/Controller/Root.pm
+++ b/src/lib/Hydra/Controller/Root.pm
@@ -162,7 +162,7 @@ sub status_GET {
            { "buildsteps.busy" => { '!=', 0 } },
            { order_by => ["globalpriority DESC", "id"],
              join => "buildsteps",
-              columns => [@buildListColumns, 'buildsteps.drvpath', 'buildsteps.type']
+              columns => [@buildListColumns]
            })]
    );
 }
--- a/src/lib/Hydra/Helper/BuildDiff.pm
+++ b/src/lib/Hydra/Helper/BuildDiff.pm
@@ -37,16 +37,7 @@ sub buildDiff {

    my $n = 0;
    foreach my $build (@{$builds}) {
-        my $aborted = $build->finished != 0 && (
-            # aborted
-            $build->buildstatus == 3
-            # cancelled
-            || $build->buildstatus == 4
-            # timeout
-            || $build->buildstatus == 7
-            # log limit exceeded
-            || $build->buildstatus == 10
-        );
+        my $aborted = $build->finished != 0 && ($build->buildstatus == 3 || $build->buildstatus == 4);
        my $d;
        my $found = 0;
        while ($n < scalar(@{$builds2})) {
@@ -88,4 +79,4 @@ sub buildDiff {
    return $ret;
 }

-1;
+1;
--- a/src/lib/Hydra/Helper/Nix.pm
+++ b/src/lib/Hydra/Helper/Nix.pm
@@ -36,7 +36,6 @@ our @EXPORT = qw(
    jobsetOverview
    jobsetOverview_
    pathIsInsidePrefix
-    readIntoSocket
    readNixFile
    registerRoot
    restartBuilds
@@ -297,7 +296,8 @@ sub getEvals {

    my @evals = $evals_result_set->search(
        { hasnewbuilds => 1 },
-        { order_by => "$me.id DESC", rows => $rows, offset => $offset });
+        { order_by => "$me.id DESC", rows => $rows, offset => $offset
+        , prefetch => { evaluationerror => [ ] } });
    my @res = ();
    my $cache = {};

@@ -417,16 +417,6 @@ sub pathIsInsidePrefix {
    return $cur;
 }

-sub readIntoSocket{
-    my (%args) = @_;
-    my $sock;
-
-    eval {
-        open($sock, "-|", @{$args{cmd}}) or die q(failed to open socket from command:\n $x);
-    };
-
-    return $sock;
-}



--- a/src/lib/Hydra/Schema/Result/EvaluationErrors.pm
+++ b/src/lib/Hydra/Schema/Result/EvaluationErrors.pm
@@ -105,6 +105,4 @@ __PACKAGE__->add_column(
    "+id" => { retrieve_on_insert => 1 }
 );

-__PACKAGE__->mk_group_accessors('column' => 'has_error');
-
 1;
--- a/src/lib/Hydra/Schema/Result/Jobsets.pm
+++ b/src/lib/Hydra/Schema/Result/Jobsets.pm
@@ -386,8 +386,6 @@ __PACKAGE__->add_column(
    "+id" => { retrieve_on_insert => 1 }
 );

-__PACKAGE__->mk_group_accessors('column' => 'has_error');
-
 sub supportsDynamicRunCommand {
  my ($self) = @_;

--- a/src/lib/Hydra/Schema/ResultSet/EvaluationErrors.pm
+++ b/src/lib/Hydra/Schema/ResultSet/EvaluationErrors.pm
@@ -1,30 +0,0 @@
-package Hydra::Schema::ResultSet::EvaluationErrors;
-
-use strict;
-use utf8;
-use warnings;
-
-use parent 'DBIx::Class::ResultSet';
-
-use Storable qw(dclone);
-
-__PACKAGE__->load_components('Helper::ResultSet::RemoveColumns');
-
-# Exclude expensive error message values unless explicitly requested, and
-# replace them with a summary field describing their presence/absence.
-sub search_rs {
-  my ( $class, $query, $attrs ) = @_;
-
-  if ($attrs) {
-    $attrs = dclone($attrs);
-  }
-
-  unless (exists $attrs->{'select'} || exists $attrs->{'columns'}) {
-    $attrs->{'+columns'}->{'has_error'} = "errormsg != ''";
-  }
-  unless (exists $attrs->{'+columns'}->{'errormsg'}) {
-    push @{ $attrs->{'remove_columns'} }, 'errormsg';
-  }
-
-  return $class->next::method($query, $attrs);
-}
--- a/src/lib/Hydra/Schema/ResultSet/Jobsets.pm
+++ b/src/lib/Hydra/Schema/ResultSet/Jobsets.pm
@@ -1,30 +0,0 @@
-package Hydra::Schema::ResultSet::Jobsets;
-
-use strict;
-use utf8;
-use warnings;
-
-use parent 'DBIx::Class::ResultSet';
-
-use Storable qw(dclone);
-
-__PACKAGE__->load_components('Helper::ResultSet::RemoveColumns');
-
-# Exclude expensive error message values unless explicitly requested, and
-# replace them with a summary field describing their presence/absence.
-sub search_rs {
-  my ( $class, $query, $attrs ) = @_;
-
-  if ($attrs) {
-    $attrs = dclone($attrs);
-  }
-
-  unless (exists $attrs->{'select'} || exists $attrs->{'columns'}) {
-    $attrs->{'+columns'}->{'has_error'} = "errormsg != ''";
-  }
-  unless (exists $attrs->{'+columns'}->{'errormsg'}) {
-    push @{ $attrs->{'remove_columns'} }, 'errormsg';
-  }
-
-  return $class->next::method($query, $attrs);
-}
--- a/src/root/build.tt
+++ b/src/root/build.tt
@@ -61,7 +61,21 @@ END;
            <td>[% IF step.busy != 0 || ((step.machine || step.starttime) && (step.status == 0 || step.status == 1 || step.status == 3 || step.status == 4 || step.status == 7)); INCLUDE renderMachineName machine=step.machine; ELSE; "<em>n/a</em>"; END %]</td>
            <td class="step-status">
              [% IF step.busy != 0 %]
-                [% INCLUDE renderBusyStatus %]
+                [% IF step.busy == 1 %]
+                  <strong>Preparing</strong>
+                [% ELSIF step.busy == 10 %]
+                  <strong>Connecting</strong>
+                [% ELSIF step.busy == 20 %]
+                  <strong>Sending inputs</strong>
+                [% ELSIF step.busy == 30 %]
+                  <strong>Building</strong>
+                [% ELSIF step.busy == 40 %]
+                  <strong>Receiving outputs</strong>
+                [% ELSIF step.busy == 50 %]
+                  <strong>Post-processing</strong>
+                [% ELSE %]
+                  <strong>Unknown state</strong>
+                [% END %]
              [% ELSIF step.status == 0 %]
                [% IF step.isnondeterministic %]
                  <span class="warn">Succeeded with non-determistic result</span>
--- a/src/root/common.tt
+++ b/src/root/common.tt
@@ -91,17 +91,6 @@ BLOCK renderDuration;
  duration % 60 %]s[%
 END;

-BLOCK renderDrvInfo;
-  drvname = step.drvpath
-    .substr(11) # strip `/nix/store/`
-    .split('-').slice(1).join("-") # strip hash part
-    .substr(0, -4); # strip `.drv`
-  IF drvname != releasename;
-    IF step.type == 0; action = "Build"; ELSE; action = "Substitution"; END;
-    IF drvname; %]<em> ([% action %] of [% drvname %])</em>[% END;
-  END;
-END;
-

 BLOCK renderBuildListHeader %]
  <table class="table table-striped table-condensed clickable-rows">
@@ -142,12 +131,7 @@ BLOCK renderBuildListBody;
      [% END %]
      <td><a class="row-link" href="[% link %]">[% build.id %]</a></td>
      [% IF !hideJobName %]
-        <td>
-          <a href="[%link%]">[% IF !hideJobsetName %][%build.jobset.get_column("project")%]:[%build.jobset.get_column("name")%]:[% END %][%build.get_column("job")%]</a>
-          [% IF showStepName %]
-            [% INCLUDE renderDrvInfo step=build.buildsteps releasename=build.nixname %]
-          [% END %]
-        </td>
+        <td><a href="[%link%]">[% IF !hideJobsetName %][%build.jobset.get_column("project")%]:[%build.jobset.get_column("name")%]:[% END %][%build.get_column("job")%]</td>
      [% END %]
      <td class="nowrap">[% t = showSchedulingInfo ? build.timestamp : build.stoptime; IF t; INCLUDE renderRelativeDate timestamp=(showSchedulingInfo ? build.timestamp : build.stoptime); ELSE; "-"; END %]</td>
      <td>[% !showSchedulingInfo and build.get_column('releasename') ? build.get_column('releasename') : build.nixname %]</td>
@@ -261,27 +245,6 @@ BLOCK renderBuildStatusIcon;
 END;


-BLOCK renderBusyStatus;
-  IF step.busy == 1 %]
-    <strong>Preparing</strong>
-  [% ELSIF step.busy == 10 %]
-    <strong>Connecting</strong>
-  [% ELSIF step.busy == 20 %]
-    <strong>Sending inputs</strong>
-  [% ELSIF step.busy == 30 %]
-    <strong>Building</strong>
-  [% ELSIF step.busy == 35 %]
-    <strong>Waiting to receive outputs</strong>
-  [% ELSIF step.busy == 40 %]
-    <strong>Receiving outputs</strong>
-  [% ELSIF step.busy == 50 %]
-    <strong>Post-processing</strong>
-  [% ELSE %]
-    <strong>Unknown state</strong>
-  [% END;
-END;
-
-
 BLOCK renderStatus;
  IF build.finished;
    buildstatus = build.buildstatus;
@@ -513,7 +476,7 @@ BLOCK renderEvals %]
            ELSE %]
             -
            [% END %]
-            [% IF eval.evaluationerror.has_error %]
+            [% IF eval.evaluationerror.errormsg %]
              <span class="badge badge-warning">Eval Errors</span>
            [% END %]
          </td>
@@ -639,7 +602,7 @@ BLOCK renderJobsetOverview %]
        <td>[% HTML.escape(j.description) %]</td>
        <td>[% IF j.lastcheckedtime;
                 INCLUDE renderDateTime timestamp = j.lastcheckedtime;
-                 IF j.has_error || j.fetcherrormsg; %]&nbsp;<span class = 'badge badge-warning'>Error</span>[% END;
+                 IF j.errormsg || j.fetcherrormsg; %]&nbsp;<span class = 'badge badge-warning'>Error</span>[% END;
                 ELSE; "-";
               END %]</td>
        [% IF j.get_column('nrtotal') > 0 %]
--- a/src/root/eval-error.tt
+++ b/src/root/eval-error.tt
@@ -1,26 +0,0 @@
-[% PROCESS common.tt %]
-<!DOCTYPE html>
-
-<html lang="en">
-
-  <head>
-    <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
-    <meta http-equiv="X-UA-Compatible" content="IE=Edge" />
-    [% INCLUDE style.tt %]
-  </head>
-
-  <body>
-
-    <div class="tab-content tab-pane">
-        <div id="tabs-errors" class="">
-          [% IF jobset %]
-              <p>Errors occurred at [% INCLUDE renderDateTime timestamp=(jobset.errortime || jobset.lastcheckedtime) %].</p>
-              <div class="card bg-light"><div class="card-body"><pre>[% HTML.escape(jobset.fetcherrormsg || jobset.errormsg) %]</pre></div></div>
-          [% ELSIF eval %]
-              <p>Errors occurred at [% INCLUDE renderDateTime timestamp=(eval.evaluationerror.errortime || eval.timestamp) %].</p>
-              <div class="card bg-light"><div class="card-body"><pre>[% HTML.escape(eval.evaluationerror.errormsg) %]</pre></div></div>
-          [% END %]
-        </div>
-    </div>
-  </body>
-</html>
--- a/src/root/jobset-eval.tt
+++ b/src/root/jobset-eval.tt
@@ -65,7 +65,7 @@ c.uri_for(c.controller('JobsetEval').action_for('view'),
  [% END %]

  [% IF aborted.size > 0 %]
-    <li class="nav-item"><a class="nav-link" href="#tabs-aborted" data-toggle="tab"><span class="text-warning">Aborted / Timed out Jobs ([% aborted.size %])</span></a></li>
+    <li class="nav-item"><a class="nav-link" href="#tabs-aborted" data-toggle="tab"><span class="text-warning">Aborted Jobs ([% aborted.size %])</span></a></li>
  [% END %]
  [% IF nowFail.size > 0 %]
    <li class="nav-item"><a class="nav-link" href="#tabs-now-fail" data-toggle="tab"><span class="text-warning">Newly Failing Jobs ([% nowFail.size %])</span></a></li>
@@ -90,7 +90,7 @@ c.uri_for(c.controller('JobsetEval').action_for('view'),
  [% END %]
  <li class="nav-item"><a class="nav-link" href="#tabs-inputs" data-toggle="tab">Inputs</a></li>

-  [% IF eval.evaluationerror.has_error %]
+  [% IF eval.evaluationerror.errormsg %]
    <li class="nav-item"><a class="nav-link" href="#tabs-errors" data-toggle="tab"><span class="text-warning">Evaluation Errors</span></a></li>
  [% END %]
 </ul>
@@ -108,6 +108,13 @@ c.uri_for(c.controller('JobsetEval').action_for('view'),

 <div class="tab-content">

+  [% IF eval.evaluationerror.errormsg %]
+    <div id="tabs-errors" class="tab-pane">
+      <p>Errors occurred at [% INCLUDE renderDateTime timestamp=(eval.evaluationerror.errortime || eval.timestamp) %].</p>
+      <div class="card bg-light"><div class="card-body"><pre>[% HTML.escape(eval.evaluationerror.errormsg) %]</pre></div></div>
+    </div>
+  [% END %]
+
  <div id="tabs-aborted" class="tab-pane">
    [% INCLUDE renderSome builds=aborted tabname="#tabs-aborted" %]
  </div>
@@ -165,9 +172,10 @@ c.uri_for(c.controller('JobsetEval').action_for('view'),
    [% END %]
  </div>

-  [% IF eval.evaluationerror.has_error %]
+  [% IF eval.evaluationerror.errormsg %]
    <div id="tabs-errors" class="tab-pane">
-      <iframe src="[% c.uri_for(c.controller('JobsetEval').action_for('errors'), [eval.id], params) %]" loading="lazy" frameBorder="0" width="100%"></iframe>
+      <p>Errors occurred at [% INCLUDE renderDateTime timestamp=(eval.evaluationerror.errortime || eval.timestamp) %].</p>
+      <div class="card bg-light"><div class="card-body"><pre>[% HTML.escape(eval.evaluationerror.errormsg) %]</pre></div></div>
    </div>
  [% END %]
 </div>
--- a/src/root/jobset.tt
+++ b/src/root/jobset.tt
@@ -61,7 +61,7 @@
  [% END %]

  <li class="nav-item"><a class="nav-link active" href="#tabs-evaluations" data-toggle="tab">Evaluations</a></li>
-  [% IF jobset.has_error || jobset.fetcherrormsg %]
+  [% IF jobset.errormsg || jobset.fetcherrormsg %]
    <li class="nav-item"><a class="nav-link" href="#tabs-errors" data-toggle="tab"><span class="text-warning">Evaluation Errors</span></a></li>
  [% END %]
  <li class="nav-item"><a class="nav-link" href="#tabs-jobs" data-toggle="tab">Jobs</a></li>
@@ -79,7 +79,7 @@
        <th>Last checked:</th>
        <td>
          [% IF jobset.lastcheckedtime %]
-            [% INCLUDE renderDateTime timestamp = jobset.lastcheckedtime %], [% IF jobset.has_error || jobset.fetcherrormsg %]<em class="text-warning">with errors!</em>[% ELSE %]<em>no errors</em>[% END %]
+            [% INCLUDE renderDateTime timestamp = jobset.lastcheckedtime %], [% IF jobset.errormsg || jobset.fetcherrormsg %]<em class="text-warning">with errors!</em>[% ELSE %]<em>no errors</em>[% END %]
          [% ELSE %]
            <em>never</em>
          [% END %]
@@ -117,9 +117,10 @@

  </div>

-  [% IF jobset.has_error || jobset.fetcherrormsg %]
+  [% IF jobset.errormsg || jobset.fetcherrormsg %]
    <div id="tabs-errors" class="tab-pane">
-      <iframe src="[% c.uri_for('/jobset' project.name jobset.name "errors") %]" loading="lazy" frameBorder="0" width="100%"></iframe>
+      <p>Errors occurred at [% INCLUDE renderDateTime timestamp=(jobset.errortime || jobset.lastcheckedtime) %].</p>
+      <div class="card bg-light"><div class="card-body"><pre>[% HTML.escape(jobset.fetcherrormsg || jobset.errormsg) %]</pre></div></div>
    </div>
  [% END %]

--- a/src/root/layout.tt
+++ b/src/root/layout.tt
@@ -10,7 +10,31 @@

    <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
    <meta http-equiv="X-UA-Compatible" content="IE=Edge" />
-    [% INCLUDE style.tt %]
+
+    <script type="text/javascript" src="[% c.uri_for("/static/js/jquery/jquery-3.4.1.min.js") %]"></script>
+    <script type="text/javascript" src="[% c.uri_for("/static/js/jquery/jquery-ui-1.10.4.min.js") %]"></script>
+    <script type="text/javascript" src="[% c.uri_for("/static/js/moment/moment-2.24.0.min.js") %]"></script>
+
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+
+    <link href="[% c.uri_for("/static/fontawesome/css/all.css") %]" rel="stylesheet" />
+    <script type="text/javascript" src="[% c.uri_for("/static/js/popper.min.js") %]"></script>
+    <script type="text/javascript" src="[% c.uri_for("/static/bootstrap/js/bootstrap.min.js") %]"></script>
+    <link href="[% c.uri_for("/static/bootstrap/css/bootstrap.min.css") %]" rel="stylesheet" />
+
+    <!-- hydra.css may need to be moved to before boostrap to make the @media rule work. -->
+    <link rel="stylesheet" href="[% c.uri_for("/static/css/hydra.css") %]" type="text/css" />
+    <link rel="stylesheet" href="[% c.uri_for("/static/css/rotated-th.css") %]" type="text/css" />
+
+    <style>
+      .popover { max-width: 40%; }
+    </style>
+
+    <script type="text/javascript" src="[% c.uri_for("/static/js/bootbox.min.js") %]"></script>
+
+    <link rel="stylesheet" href="[% c.uri_for("/static/css/tree.css") %]" type="text/css" />
+
+    <script type="text/javascript" src="[% c.uri_for("/static/js/common.js") %]"></script>

    [% IF c.config.enable_google_login %]
      <meta name="google-signin-client_id" content="[% c.config.google_client_id %]">
--- a/src/root/machine-status.tt
+++ b/src/root/machine-status.tt
@@ -6,10 +6,10 @@
  <thead>
    <tr>
      <th>Job</th>
+      <th>System</th>
      <th>Build</th>
      <th>Step</th>
      <th>What</th>
-      <th>Status</th>
      <th>Since</th>
    </tr>
  </thead>
@@ -40,10 +40,10 @@
          [% idle = 0 %]
          <tr>
            <td><tt>[% INCLUDE renderFullJobName project=step.project jobset=step.jobset job=step.job %]</tt></td>
+            <td><tt>[% step.system %]</tt></td>
            <td><a href="[% c.uri_for('/build' step.build) %]">[% step.build %]</a></td>
            <td>[% IF step.busy >= 30 %]<a class="row-link" href="[% c.uri_for('/build' step.build 'nixlog' step.stepnr 'tail') %]">[% step.stepnr %]</a>[% ELSE; step.stepnr; END %]</td>
            <td><tt>[% step.drvpath.match('-(.*)').0 %]</tt></td>
-            <td>[% INCLUDE renderBusyStatus %]</td>
            <td style="width: 10em">[% INCLUDE renderDuration duration = curTime - step.starttime %] </td>
          </tr>
        [% END %]
--- a/src/root/static/js/common.js
+++ b/src/root/static/js/common.js
@@ -129,12 +129,6 @@ $(document).ready(function() {
            el.addClass("is-local");
        }
    });
-
-    [...document.getElementsByTagName("iframe")].forEach((element) => {
-        element.contentWindow.addEventListener("DOMContentLoaded", (_) => {
-            element.style.height = element.contentWindow.document.body.scrollHeight + 'px';
-        })
-    })
 });

 var tabsLoaded = {};
--- a/src/root/status.tt
+++ b/src/root/status.tt
@@ -7,7 +7,7 @@

 [% ELSE %]

-  [% INCLUDE renderBuildList builds=resource showSchedulingInfo=1 hideResultInfo=1 busy=1 showStepName=1 %]
+  [% INCLUDE renderBuildList builds=resource showSchedulingInfo=1 hideResultInfo=1 busy=1 %]

 [% END %]

--- a/src/root/style.tt
+++ b/src/root/style.tt
@@ -1,24 +0,0 @@
-<script type="text/javascript" src="[% c.uri_for("/static/js/jquery/jquery-3.4.1.min.js") %]"></script>
-<script type="text/javascript" src="[% c.uri_for("/static/js/jquery/jquery-ui-1.10.4.min.js") %]"></script>
-<script type="text/javascript" src="[% c.uri_for("/static/js/moment/moment-2.24.0.min.js") %]"></script>
-
-<meta name="viewport" content="width=device-width, initial-scale=1.0" />
-
-<link href="[% c.uri_for("/static/fontawesome/css/all.css") %]" rel="stylesheet" />
-<script type="text/javascript" src="[% c.uri_for("/static/js/popper.min.js") %]"></script>
-<script type="text/javascript" src="[% c.uri_for("/static/bootstrap/js/bootstrap.min.js") %]"></script>
-<link href="[% c.uri_for("/static/bootstrap/css/bootstrap.min.css") %]" rel="stylesheet" />
-
-<!-- hydra.css may need to be moved to before boostrap to make the @media rule work. -->
-<link rel="stylesheet" href="[% c.uri_for("/static/css/hydra.css") %]" type="text/css" />
-<link rel="stylesheet" href="[% c.uri_for("/static/css/rotated-th.css") %]" type="text/css" />
-
-<style>
-  .popover { max-width: 40%; }
-</style>
-
-<script type="text/javascript" src="[% c.uri_for("/static/js/bootbox.min.js") %]"></script>
-
-<link rel="stylesheet" href="[% c.uri_for("/static/css/tree.css") %]" type="text/css" />
-
-<script type="text/javascript" src="[% c.uri_for("/static/js/common.js") %]"></script>
--- a/t/Hydra/Controller/Jobset/evals.t
+++ b/t/Hydra/Controller/Jobset/evals.t
@@ -32,9 +32,4 @@ subtest "/jobset/PROJECT/JOBSET/evals" => sub {
    ok($jobsetevals->is_success, "The page showing the jobset evals returns 200.");
 };

-subtest "/jobset/PROJECT/JOBSET/errors" => sub {
-    my $jobsetevals = request(GET '/jobset/' . $project->name . '/' . $jobset->name . '/errors');
-    ok($jobsetevals->is_success, "The page showing the jobset eval errors returns 200.");
-};
-
 done_testing;
--- a/t/Hydra/Controller/JobsetEval/fetch.t
+++ b/t/Hydra/Controller/JobsetEval/fetch.t
@@ -35,10 +35,6 @@ subtest "Fetching the eval's overview" => sub {
    is($fetch->code, 200, "channel page is 200");
 };

-subtest "Fetching the eval's overview" => sub {
-    my $fetch = request(GET '/eval/' . $eval->id, '/errors');
-    is($fetch->code, 200, "errors page is 200");
-};


 done_testing;
--- a/t/lib/CliRunners.pm
+++ b/t/lib/CliRunners.pm
@@ -14,7 +14,7 @@ our @EXPORT = qw(
 sub evalSucceeds {
    my ($jobset) = @_;
    my ($res, $stdout, $stderr) = captureStdoutStderr(60, ("hydra-eval-jobset", $jobset->project->name, $jobset->name));
-    $jobset->discard_changes({ '+columns' => {'errormsg' => 'errormsg'} });  # refresh from DB
+    $jobset->discard_changes;  # refresh from DB
    if ($res) {
        chomp $stdout; chomp $stderr;
        utf8::decode($stdout) or die "Invalid unicode in stdout.";
@@ -29,7 +29,7 @@ sub evalSucceeds {
 sub evalFails {
    my ($jobset) = @_;
    my ($res, $stdout, $stderr) = captureStdoutStderr(60, ("hydra-eval-jobset", $jobset->project->name, $jobset->name));
-    $jobset->discard_changes({ '+columns' => {'errormsg' => 'errormsg'} });  # refresh from DB
+    $jobset->discard_changes;  # refresh from DB
    if (!$res) {
        chomp $stdout; chomp $stderr;
        utf8::decode($stdout) or die "Invalid unicode in stdout.";
--- a/t/meson.build
+++ b/t/meson.build
@@ -27,6 +27,7 @@ testenv.prepend('PERL5LIB',
  separator: ':'
 )
 testenv.prepend('PATH',
+  fs.parent(find_program('nix').full_path()),
  fs.parent(hydra_evaluator.full_path()),
  fs.parent(hydra_queue_runner.full_path()),
  meson.project_source_root() / 'src/script',
--- a/t/queue-runner/direct-indirect-constituents.t
+++ b/t/queue-runner/direct-indirect-constituents.t
@@ -13,7 +13,7 @@ my $constituentBuildA = $builds->{"constituentA"};
 my $constituentBuildB = $builds->{"constituentB"};

 my $eval = $constituentBuildA->jobsetevals->first();
-is($eval->evaluationerror->has_error, 0);
+is($eval->evaluationerror->errormsg, "");

 subtest "Verifying the direct aggregate" => sub {
    my $aggBuild = $builds->{"direct_aggregate"};
Author	SHA1	Message	Date
John Ericson	f3dc4e9228	WIP ssh-ng://	2025-02-14 18:51:49 -05:00
John Ericson	45d075e5db	WIP: Avoid custom logic copying outputs from the remote builder We need a replacement for the nar member logic, however. And maybe also a test that fails until this is fixed (this one should not be passing).	2025-02-14 18:50:38 -05:00
John Ericson	9e162dcf52	Avoid custom logic to copy inputs to the remote builder	2025-02-14 18:50:38 -05:00
John Ericson	4c173daec7	Use `LegacySSHStore` In https://github.com/NixOS/nix/pull/10748 it is extended with everything we need.	2025-02-14 18:48:46 -05:00
John Ericson	8675aee25b	WIP TEMP nix update, don't mere this! Not until https://github.com/NixOS/nix/pull/10748 lands Flake lock file updates: • Updated input 'nix': 'github:NixOS/nix/970942f45836172fda410a638853382952189eb9?narHash=sha256-jGFuyYKJjJZsBRoi7ZcaVKt1OYxusz/ld1HA7VD2w/0%3D' (2025-02-12) → 'github:NixOS/nix/5eade4825221d3284fc6555cb20de2c7aa171d72?narHash=sha256-n5kdS1C24tlJxDV6Wm1iBlyvGk%2Bp0gMXRcWVCAipYLs%3D' (2025-02-14) • Updated input 'nix-eval-jobs': 'github:Ericson2314/nix-eval-jobs/5e27c2724a4b07862e7ff1a198aa2ed68dea3e2c?narHash=sha256-7xgSdKnQW11eWd59MnpUNS%2BgwgtOJH2ShzLwByev3rg%3D' (2025-02-14) → 'github:Ericson2314/nix-eval-jobs/de345eb4518d952c2d86261b270f2c31edecd3de?narHash=sha256-dNMJY6%2BG3PwE8lIAhwetPJdA2DxCEKRXPY/EtHmdDh4%3D' (2025-02-14)	2025-02-14 18:15:22 -05:00
John Ericson	51944a5fa5	Merge pull request #1443 from NixOS/nix-2.26 Nix 2.26	2025-02-13 22:13:32 -05:00
John Ericson	341b2f1309	Update build system to depend on Nix 2.26	2025-02-13 21:54:35 -05:00
John Ericson	4dc0f11379	Update flake.nix for Nix 2.26 Flake lock file updates: • Removed input 'libgit2' • Updated input 'nix': 'github:NixOS/nix/d652513e4519ed4eb48c92f8670e5a71c7793fc3?narHash=sha256-mIpJgIwPS4o4xYhN1B%2B/fHESEXoxpu6nVoZTzZ0MfTg%3D' (2025-02-12) → 'github:NixOS/nix/970942f45836172fda410a638853382952189eb9?narHash=sha256-jGFuyYKJjJZsBRoi7ZcaVKt1OYxusz/ld1HA7VD2w/0%3D' (2025-02-12) • Removed input 'nix/libgit2' • Updated input 'nix-eval-jobs': 'github:nix-community/nix-eval-jobs/6d4fd5a93d7bc953ffa4dcd6d53ad7056a71eff7?narHash=sha256-1dZLPw%2BnlFQzzswfyTxW%2B8VF1AJ4ZvoYvLTjlHiz1SA%3D' (2025-02-13) → 'github:nix-community/nix-eval-jobs/4b392b284877d203ae262e16af269f702df036bc?narHash=sha256-3wIReAqdTALv39gkWXLMZQvHyBOc3yPkWT2ZsItxedY%3D' (2025-02-14) • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/dbebdd67a6006bb145d98c8debf9140ac7e651d0?narHash=sha256-Xc9lEtentPCEtxc/F1e6jIZsd4MPDYv4Kugl9WtXlz0%3D' (2024-09-18) → 'github:NixOS/nixpkgs/97a719c9f0a07923c957cf51b20b329f9fb9d43f?narHash=sha256-1o1qR0KYozYGRrnqytSpAhVBYLNBHX%2BLv6I39zGRzKM%3D' (2025-02-13)	2025-02-13 21:54:31 -05:00
John Ericson	ea09952b7e	Merge pull request #1442 from NixOS/clean-up-flake-lockfile Clean up flake lockfile stuff	2025-02-13 20:52:40 -05:00
John Ericson	81d21979ef	Clean up flake lockfile stuff The `flake = false;` for `nix-eval-jobs` didn't fully take before. Flake lock file updates: • Removed input 'nix-eval-jobs/flake-parts' • Removed input 'nix-eval-jobs/flake-parts/nixpkgs-lib' • Removed input 'nix-eval-jobs/nix-github-actions' • Removed input 'nix-eval-jobs/nixpkgs' • Removed input 'nix-eval-jobs/treefmt-nix' • Removed input 'nix-eval-jobs/treefmt-nix/nixpkgs'	2025-02-13 20:23:08 -05:00
John Ericson	0ed9a82912	Merge pull request #1441 from NixOS/nix-2.25 Nix 2.25	2025-02-13 19:53:07 -05:00
John Ericson	80241fc8be	Make code change necessary for building with Nix 2.25	2025-02-13 19:10:09 -05:00
John Ericson	4347833f45	Rework to synchronize deps Bypass `nix-eval-job`'s flake, and just call-package it, for fine-grained control.	2025-02-13 19:03:37 -05:00
John Ericson	8835cbd10f	flake.lock: Update Flake lock file updates: • Updated input 'nix': 'github:NixOS/nix/a7fdef6858dd45b9d7bda7c92324c63faee7f509?narHash=sha256-XFznzb8L4SdUm9u%2Bw3DPpMWJhffuv%2B/6%2BaiVl00slns%3D' (2024-09-19) → 'github:NixOS/nix/d652513e4519ed4eb48c92f8670e5a71c7793fc3?narHash=sha256-mIpJgIwPS4o4xYhN1B%2B/fHESEXoxpu6nVoZTzZ0MfTg%3D' (2025-02-12) • Updated input 'nix-eval-jobs': 'github:nix-community/nix-eval-jobs/889ea1406736b53cf165b6c28398aae3969418d1?narHash=sha256-3wwtKpS5tUBdjaGeSia7CotonbiRB6K5Kp0dsUt3nzU%3D' (2024-12-10) → 'github:nix-community/nix-eval-jobs/6d4fd5a93d7bc953ffa4dcd6d53ad7056a71eff7?narHash=sha256-1dZLPw%2BnlFQzzswfyTxW%2B8VF1AJ4ZvoYvLTjlHiz1SA%3D' (2025-02-13)	2025-02-13 18:46:36 -05:00
John Ericson	9ad8ac586c	Merge pull request #1440 from NixOS/legacy-ssh-expose-ssh-master Use new `CommonSSHStoreConfig::createSSHMaster`	2025-02-13 18:30:41 -05:00
John Ericson	9a6928d93b	Use new `CommonSSHStoreConfig::createSSHMaster` This avoids some duplicated code, leveraging the same `StoreReference` type that also undergirds the machine file dedup we just did prior. By using `LegacySSHStoreConfig`, we're also taking a baby step towards using the store interface rather than messing around with the protocol internals.	2025-02-13 18:13:38 -05:00
John Ericson	810781a802	Merge pull request #1439 from NixOS/nix-next Dedup machine file parsing, and other improvements	2025-02-13 18:10:08 -05:00
John Ericson	af9b0663f2	Merge branch 'master' into nix-next	2025-02-13 17:54:15 -05:00
John Ericson	2feddd8511	flake.lock: Update Flake lock file updates: • Updated input 'nix': 'github:NixOS/nix/2c42e7b8d9ea32e59c01334852599b548b214d31' (2024-05-23) → 'github:NixOS/nix/ef5c846e257e1e284ad47ed6be4308d190fe6531' (2024-05-29)	2024-05-29 17:05:41 -04:00
John Ericson	cd925e876f	Merge branch 'master' into nix-next	2024-05-29 17:05:04 -04:00
John Ericson	91bb72e323	Merge pull request #1386 from NixOS/machine-dedup Dedup with nix: use `nix::Machine::parseConfig`	2024-05-23 11:21:41 -04:00
John Ericson	09a1e64ed2	Dedup with nix: use `nix::Machine::parseConfig` Companion to https://github.com/NixOS/nix/pull/10763	2024-05-23 09:59:46 -04:00
John Ericson	bede2a141a	flake.lock: Update Flake lock file updates: • Updated input 'nix': 'github:NixOS/nix/5845fd59c34198ad52a7f7bcb6d3ea7176ca437b' (2024-05-22) → 'github:NixOS/nix/2c42e7b8d9ea32e59c01334852599b548b214d31' (2024-05-23)	2024-05-23 09:59:32 -04:00
John Ericson	b75bf5c882	Merge pull request #1385 from NixOS/machine-dedup Utilize `nix::Machine` more fully	2024-05-23 00:00:58 -04:00
John Ericson	d55bea2a1e	Utilize `nix::Machine` more fully With https://github.com/NixOS/nix/pull/9839, the `storeUri` field is much better structured, so we can use it while still opening the SSH connection ourselves.	2024-05-22 22:02:46 -04:00
John Ericson	346badc66f	flake.lock: Update Flake lock file updates: • Updated input 'nix': 'github:NixOS/nix/a57abbd143f8ed44e823c3244e93507f64020878' (2024-05-20) → 'github:NixOS/nix/5845fd59c34198ad52a7f7bcb6d3ea7176ca437b' (2024-05-22)	2024-05-22 22:00:38 -04:00
John Ericson	a940450875	Merge branch 'master' into nix-next	2024-05-22 22:00:25 -04:00
John Ericson	af120e7195	Merge pull request #1384 from NixOS/more-serve-proto-factor-out Dedup more protocol code	2024-05-20 21:49:06 -04:00
John Ericson	71c4e2dc5b	Dedup more protocol code Use https://github.com/NixOS/nix/pull/10749	2024-05-20 18:19:59 -04:00
John Ericson	e4552ddf91	flake.lock: Update Flake lock file updates: • Updated input 'nix': 'github:NixOS/nix/beb3c2bc7ab781c1b8907b647c6e72b72fa9f56b' (2024-05-17) → 'github:NixOS/nix/a57abbd143f8ed44e823c3244e93507f64020878' (2024-05-20)	2024-05-20 18:11:37 -04:00
John Ericson	e4f2c84f8d	flake.lock: Update Flake lock file updates: • Updated input 'nix': 'github:NixOS/nix/0930058189f350a3729cd5aef2ffc8dae2ad436e' (2024-05-08) → 'github:NixOS/nix/beb3c2bc7ab781c1b8907b647c6e72b72fa9f56b' (2024-05-17)	2024-05-17 20:02:54 -04:00
John Ericson	e10fc2bd13	Merge branch 'master' into nix-next	2024-05-17 19:59:41 -04:00
John Ericson	5e910fa2ce	flake.lock: Update Flake lock file updates: • Updated input 'nix': 'github:NixOS/nix/00ca2b05b8fbbef09be5d1e4820857605d4c31b6' (2024-05-03) → 'github:NixOS/nix/0930058189f350a3729cd5aef2ffc8dae2ad436e' (2024-05-08)	2024-05-08 11:25:14 -04:00
John Ericson	4b767aa9a2	Merge branch 'master' into nix-next	2024-05-08 11:25:04 -04:00
John Ericson	2926aa1d64	Merge branch 'factor-out-tests' into nix-next	2024-05-03 12:44:51 -04:00
John Ericson	555ea44a7a	Merge branch 'master' into nix-next	2024-05-03 12:35:06 -04:00
John Ericson	410077a26e	Merge branch 'nix-2.22' into nix-next	2024-05-03 10:49:28 -04:00
John Ericson	39a4e4791e	Switch (back) to Nix master Re-creating `nix-next` after using it in #1375. Flake lock file updates: • Updated input 'nix': 'github:NixOS/nix/60824fa97c588a0faf68ea61260a47e388b0a4e5' (2024-04-11) → 'github:NixOS/nix/aa438b8fbaebbbdb922655127053c4e8ea3e55bb' (2024-04-12)	2024-04-12 17:30:57 -04:00