Create eval-jobset role and guard /api/push route

(cherry picked from commit f730433789)
api: Require POST for /api/push
2024-08-27 21:53:17 +02:00 · 2024-08-27 21:53:13 +02:00 · 2024-04-21 17:36:16 +02:00 · 2024-04-21 16:14:24 +02:00 · 2024-04-21 01:55:19 +02:00 · 2024-04-20 23:05:26 +02:00
47 changed files with 714 additions and 284 deletions
--- a/doc/manual/src/configuration.md
+++ b/doc/manual/src/configuration.md
@@ -208,7 +208,8 @@ Example configuration:
  <role_mapping>
    # Make all users in the hydra_admin group Hydra admins
    hydra_admin = admin
-    # Allow all users in the dev group to restart jobs and cancel builds
+    # Allow all users in the dev group to eval jobsets, restart jobs and cancel builds
+    dev = eval-jobset
    dev = restart-jobs
    dev = cancel-build
  </role_mapping>
--- a/flake.lock
+++ b/flake.lock
@@ -16,58 +16,58 @@
        "type": "github"
      }
    },
-    "libgit2": {
+    "lowdown-src": {
      "flake": false,
      "locked": {
-        "lastModified": 1697646580,
-        "narHash": "sha256-oX4Z3S9WtJlwvj0uH9HlYcWv+x1hqp8mhXl7HsLu2f0=",
-        "owner": "libgit2",
-        "repo": "libgit2",
-        "rev": "45fd9ed7ae1a9b74b957ef4f337bc3c8b3df01b5",
+        "lastModified": 1633514407,
+        "narHash": "sha256-Dw32tiMjdK9t3ETl5fzGrutQTzh2rufgZV4A/BbxuD4=",
+        "owner": "kristapsdz",
+        "repo": "lowdown",
+        "rev": "d2c2b44ff6c27b936ec27358a2653caaef8f73b8",
        "type": "github"
      },
      "original": {
-        "owner": "libgit2",
-        "repo": "libgit2",
+        "owner": "kristapsdz",
+        "repo": "lowdown",
        "type": "github"
      }
    },
    "nix": {
      "inputs": {
        "flake-compat": "flake-compat",
-        "libgit2": "libgit2",
+        "lowdown-src": "lowdown-src",
        "nixpkgs": [
          "nixpkgs"
        ],
        "nixpkgs-regression": "nixpkgs-regression"
      },
      "locked": {
-        "lastModified": 1715805674,
-        "narHash": "sha256-0CIBMECsA3ISJZrJcOTzi6wa3QENTKGLtOpYIOoxwxo=",
+        "lastModified": 1706208340,
+        "narHash": "sha256-wNyHUEIiKKVs6UXrUzhP7RSJQv0A8jckgcuylzftl8k=",
        "owner": "NixOS",
        "repo": "nix",
-        "rev": "ab48ea416a203e9ccefb70aa634e27477e4c1ac4",
+        "rev": "2c4bb93ba5a97e7078896ebc36385ce172960e4e",
        "type": "github"
      },
      "original": {
        "owner": "NixOS",
-        "ref": "2.20-maintenance",
+        "ref": "2.19-maintenance",
        "repo": "nix",
        "type": "github"
      }
    },
    "nixpkgs": {
      "locked": {
-        "lastModified": 1705033721,
-        "narHash": "sha256-K5eJHmL1/kev6WuqyqqbS1cdNnSidIZ3jeqJ7GbrYnQ=",
+        "lastModified": 1701615100,
+        "narHash": "sha256-7VI84NGBvlCTduw2aHLVB62NvCiZUlALLqBe5v684Aw=",
        "owner": "NixOS",
        "repo": "nixpkgs",
-        "rev": "a1982c92d8980a0114372973cbdfe0a307f1bdea",
+        "rev": "e9f06adb793d1cca5384907b3b8a4071d5d7cb19",
        "type": "github"
      },
      "original": {
        "owner": "NixOS",
-        "ref": "nixos-23.05-small",
+        "ref": "nixos-23.05",
        "repo": "nixpkgs",
        "type": "github"
      }
--- a/flake.nix
+++ b/flake.nix
@@ -1,8 +1,8 @@
 {
  description = "A Nix-based continuous build system";

-  inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.05-small";
-  inputs.nix.url = "github:NixOS/nix/2.20-maintenance";
+  inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.05";
+  inputs.nix.url = "github:NixOS/nix/2.19-maintenance";
  inputs.nix.inputs.nixpkgs.follows = "nixpkgs";

  # TODO get rid of this once https://github.com/NixOS/nix/pull/9546 is
@@ -180,9 +180,13 @@
                root=d7f16a3412e01a43a414535b16007c6931d3a9c7
                </gitea_authorization>
              '';
-              nixpkgs.config.permittedInsecurePackages = [ "gitea-1.19.4" ];
              nix = {
-                settings.substituters = [ ];
+                distributedBuilds = true;
+                buildMachines = [{
+                  hostName = "localhost";
+                  systems = [ system ];
+                }];
+                binaryCaches = [ ];
              };
              services.gitea = {
                enable = true;
@@ -198,7 +202,7 @@
            testScript =
              let
                scripts.mktoken = pkgs.writeText "token.sql" ''
-                  INSERT INTO access_token (id, uid, name, created_unix, updated_unix, token_hash, token_salt, token_last_eight, scope) VALUES (1, 1, 'hydra', 1617107360, 1617107360, 'a930f319ca362d7b49a4040ac0af74521c3a3c3303a86f327b01994430672d33b6ec53e4ea774253208686c712495e12a486', 'XRjWE9YW0g', '31d3a9c7', 'all');
+                  INSERT INTO access_token (id, uid, name, created_unix, updated_unix, token_hash, token_salt, token_last_eight) VALUES (1, 1, 'hydra', 1617107360, 1617107360, 'a930f319ca362d7b49a4040ac0af74521c3a3c3303a86f327b01994430672d33b6ec53e4ea774253208686c712495e12a486', 'XRjWE9YW0g', '31d3a9c7');
                '';

                scripts.git-setup = pkgs.writeShellScript "setup.sh" ''
@@ -353,9 +357,9 @@

                response = json.loads(data)

-                assert len(response) == 2, "Expected exactly three status updates for latest commit (queued, finished)!"
-                assert response[0]['status'] == "success", "Expected finished status to be success!"
-                assert response[1]['status'] == "pending", "Expected queued status to be pending!"
+                assert len(response) == 2, "Expected exactly two status updates for latest commit!"
+                assert response[0]['status'] == "success", "Expected latest status to be success!"
+                assert response[1]['status'] == "pending", "Expected first status to be pending!"

                machine.shutdown()
              '';
--- a/hydra-api.yaml
+++ b/hydra-api.yaml
@@ -70,7 +70,7 @@ paths:
                  $ref: '#/components/examples/projects-success'

  /api/push:
-    put:
+    post:
      summary: trigger jobsets
      parameters:
        - in: query
--- a/package.nix
+++ b/package.nix
@@ -87,6 +87,7 @@ let
        DateTime
        DBDPg
        DBDSQLite
+        DBIxClassHelpers
        DigestSHA1
        EmailMIME
        EmailSender
--- a/src/hydra-eval-jobs/hydra-eval-jobs.cc
+++ b/src/hydra-eval-jobs/hydra-eval-jobs.cc
@@ -89,7 +89,7 @@ struct MyArgs : MixEvalArgs, MixCommonArgs, RootArgs

 static MyArgs myArgs;

-static std::string queryMetaStrings(EvalState & state, PackageInfo & drv, const std::string & name, const std::string & subAttribute)
+static std::string queryMetaStrings(EvalState & state, DrvInfo & drv, const std::string & name, const std::string & subAttribute)
 {
    Strings res;
    std::function<void(Value & v)> rec;
@@ -181,7 +181,7 @@ static void worker(
                // CA derivations do not have static output paths, so we
                // have to defensively not query output paths in case we
                // encounter one.
-                PackageInfo::Outputs outputs = drv->queryOutputs(
+                DrvInfo::Outputs outputs = drv->queryOutputs(
                    !experimentalFeatureSettings.isEnabled(Xp::CaDerivations));

                if (drv->querySystem() == "unknown")
--- a/src/hydra-queue-runner/build-remote.cc
+++ b/src/hydra-queue-runner/build-remote.cc
@@ -8,7 +8,6 @@
 #include "build-result.hh"
 #include "path.hh"
 #include "serve-protocol.hh"
-#include "serve-protocol-impl.hh"
 #include "state.hh"
 #include "current-process.hh"
 #include "processes.hh"
@@ -21,6 +20,12 @@

 using namespace nix;

+
+static void append(Strings & dst, const Strings & src)
+{
+    dst.insert(dst.end(), src.begin(), src.end());
+}
+
 namespace nix::build_remote {

 static Strings extraStoreArgs(std::string & machine)
@@ -43,20 +48,63 @@ static Strings extraStoreArgs(std::string & machine)
    return result;
 }

-static std::unique_ptr<SSHMaster::Connection> openConnection(
-    ::Machine::ptr machine, SSHMaster & master)
+static void openConnection(::Machine::ptr machine, Path tmpDir, int stderrFD, SSHMaster::Connection & child)
 {
-    Strings command = {"nix-store", "--serve", "--write"};
+    std::string pgmName;
+    Pipe to, from;
+    to.create();
+    from.create();
+
+    Strings argv;
    if (machine->isLocalhost()) {
-        command.push_back("--builders");
-        command.push_back("");
+        pgmName = "nix-store";
+        argv = {"nix-store", "--builders", "", "--serve", "--write"};
    } else {
-        command.splice(command.end(), extraStoreArgs(machine->sshName));
+        pgmName = "ssh";
+        auto sshName = machine->sshName;
+        Strings extraArgs = extraStoreArgs(sshName);
+        argv = {"ssh", sshName};
+        if (machine->sshKey != "") append(argv, {"-i", machine->sshKey});
+        if (machine->sshPublicHostKey != "") {
+            Path fileName = tmpDir + "/host-key";
+            auto p = machine->sshName.find("@");
+            std::string host = p != std::string::npos ? std::string(machine->sshName, p + 1) : machine->sshName;
+            writeFile(fileName, host + " " + machine->sshPublicHostKey + "\n");
+            append(argv, {"-oUserKnownHostsFile=" + fileName});
+        }
+        append(argv,
+            { "-x", "-a", "-oBatchMode=yes", "-oConnectTimeout=60", "-oTCPKeepAlive=yes"
+            , "--", "nix-store", "--serve", "--write" });
+        append(argv, extraArgs);
    }

-    return master.startCommand(std::move(command), {
-        "-a", "-oBatchMode=yes", "-oConnectTimeout=60", "-oTCPKeepAlive=yes"
+    child.sshPid = startProcess([&]() {
+        restoreProcessContext();
+
+        if (dup2(to.readSide.get(), STDIN_FILENO) == -1)
+            throw SysError("cannot dup input pipe to stdin");
+
+        if (dup2(from.writeSide.get(), STDOUT_FILENO) == -1)
+            throw SysError("cannot dup output pipe to stdout");
+
+        if (dup2(stderrFD, STDERR_FILENO) == -1)
+            throw SysError("cannot dup stderr");
+
+        execvp(argv.front().c_str(), (char * *) stringsToCharPtrs(argv).data()); // FIXME: remove cast
+
+        throw SysError("cannot start %s", pgmName);
    });
+
+    to.readSide = -1;
+    from.writeSide = -1;
+
+    child.in = to.writeSide.release();
+    child.out = from.readSide.release();
+
+    // XXX: determine the actual max value we can use from /proc.
+    int pipesize = 1024 * 1024;
+    fcntl(child.in.get(), F_SETPIPE_SZ, &pipesize);
+    fcntl(child.out.get(), F_SETPIPE_SZ, &pipesize);
 }


@@ -74,10 +122,13 @@ static void copyClosureTo(
       garbage-collect paths that are already there. Optionally, ask
       the remote host to substitute missing paths. */
    // FIXME: substitute output pollutes our build log
+    conn.to << ServeProto::Command::QueryValidPaths << 1 << useSubstitutes;
+    ServeProto::write(destStore, conn, closure);
+    conn.to.flush();
+
    /* Get back the set of paths that are already valid on the remote
       host. */
-    auto present = conn.queryValidPaths(
-        destStore, true, closure, useSubstitutes);
+    auto present = ServeProto::Serialise<StorePathSet>::read(destStore, conn);

    if (present.size() == closure.size()) return;

@@ -102,7 +153,7 @@ static void copyClosureTo(


 // FIXME: use Store::topoSortPaths().
-static StorePaths reverseTopoSortPaths(const std::map<StorePath, UnkeyedValidPathInfo> & paths)
+static StorePaths reverseTopoSortPaths(const std::map<StorePath, ValidPathInfo> & paths)
 {
    StorePaths sorted;
    StorePathSet visited;
@@ -143,6 +194,28 @@ static std::pair<Path, AutoCloseFD> openLogFile(const std::string & logDir, cons
    return {std::move(logFile), std::move(logFD)};
 }

+/**
+ * @param conn is not fully initialized; it is this functions job to set
+ * the `remoteVersion` field after the handshake is completed.
+ * Therefore, no `ServeProto::Serialize` functions can be used until
+ * that field is set.
+ */
+static void handshake(::Machine::Connection & conn, unsigned int repeats)
+{
+    conn.to << SERVE_MAGIC_1 << 0x206;
+    conn.to.flush();
+
+    unsigned int magic = readInt(conn.from);
+    if (magic != SERVE_MAGIC_2)
+        throw Error("protocol mismatch with ‘nix-store --serve’ on ‘%1%’", conn.machine->sshName);
+    conn.remoteVersion = readInt(conn.from);
+    // Now `conn` is initialized.
+    if (GET_PROTOCOL_MAJOR(conn.remoteVersion) != 0x200)
+        throw Error("unsupported ‘nix-store --serve’ protocol version on ‘%1%’", conn.machine->sshName);
+    if (GET_PROTOCOL_MINOR(conn.remoteVersion) < 3 && repeats > 0)
+        throw Error("machine ‘%1%’ does not support repeating a build; please upgrade it to Nix 1.12", conn.machine->sshName);
+}
+
 static BasicDerivation sendInputs(
    State & state,
    Step & step,
@@ -213,11 +286,21 @@ static BuildResult performBuild(
    Store & localStore,
    StorePath drvPath,
    const BasicDerivation & drv,
-    const ServeProto::BuildOptions & options,
+    const State::BuildOptions & options,
    counter & nrStepsBuilding
 )
 {
-    conn.putBuildDerivationRequest(localStore, drvPath, drv, options);
+    conn.to << ServeProto::Command::BuildDerivation << localStore.printStorePath(drvPath);
+    writeDerivation(conn.to, localStore, drv);
+    conn.to << options.maxSilentTime << options.buildTimeout;
+    if (GET_PROTOCOL_MINOR(conn.remoteVersion) >= 2)
+        conn.to << options.maxLogSize;
+    if (GET_PROTOCOL_MINOR(conn.remoteVersion) >= 3) {
+        conn.to
+            << options.repeats // == build-repeat
+            << options.enforceDeterminism;
+    }
+    conn.to.flush();

    BuildResult result;

@@ -267,7 +350,7 @@ static BuildResult performBuild(
    return result;
 }

-static std::map<StorePath, UnkeyedValidPathInfo> queryPathInfos(
+static std::map<StorePath, ValidPathInfo> queryPathInfos(
    ::Machine::Connection & conn,
    Store & localStore,
    StorePathSet & outputs,
@@ -276,18 +359,30 @@ static std::map<StorePath, UnkeyedValidPathInfo> queryPathInfos(
 {

    /* Get info about each output path. */
-    std::map<StorePath, UnkeyedValidPathInfo> infos;
+    std::map<StorePath, ValidPathInfo> infos;
    conn.to << ServeProto::Command::QueryPathInfos;
    ServeProto::write(localStore, conn, outputs);
    conn.to.flush();
    while (true) {
        auto storePathS = readString(conn.from);
        if (storePathS == "") break;
-
-        auto storePath = localStore.parseStorePath(storePathS);
-        auto info = ServeProto::Serialise<UnkeyedValidPathInfo>::read(localStore, conn);
+        auto deriver = readString(conn.from); // deriver
+        auto references = ServeProto::Serialise<StorePathSet>::read(localStore, conn);
+        readLongLong(conn.from); // download size
+        auto narSize = readLongLong(conn.from);
+        auto narHash = Hash::parseAny(readString(conn.from), htSHA256);
+        auto ca = ContentAddress::parseOpt(readString(conn.from));
+        readStrings<StringSet>(conn.from); // sigs
+        ValidPathInfo info(localStore.parseStorePath(storePathS), narHash);
+        assert(outputs.count(info.path));
+        info.references = references;
+        info.narSize = narSize;
        totalNarSize += info.narSize;
-        infos.insert_or_assign(std::move(storePath), std::move(info));
+        info.narHash = narHash;
+        info.ca = ca;
+        if (deriver != "")
+            info.deriver = localStore.parseStorePath(deriver);
+        infos.insert_or_assign(info.path, info);
    }

    return infos;
@@ -328,16 +423,14 @@ static void copyPathsFromRemote(
    NarMemberDatas & narMembers,
    Store & localStore,
    Store & destStore,
-    const std::map<StorePath, UnkeyedValidPathInfo> & infos
+    const std::map<StorePath, ValidPathInfo> & infos
 )
 {
      auto pathsSorted = reverseTopoSortPaths(infos);

      for (auto & path : pathsSorted) {
          auto & info = infos.find(path)->second;
-          copyPathFromRemote(
-              conn, narMembers, localStore, destStore,
-              ValidPathInfo { path, info });
+          copyPathFromRemote(conn, narMembers, localStore, destStore, info);
      }

 }
@@ -401,10 +494,21 @@ void RemoteResult::updateWithBuildResult(const nix::BuildResult & buildResult)

 }

+/* Utility guard object to auto-release a semaphore on destruction. */
+template <typename T>
+class SemaphoreReleaser {
+public:
+    SemaphoreReleaser(T* s) : sem(s) {}
+    ~SemaphoreReleaser() { sem->release(); }
+
+private:
+    T* sem;
+};

 void State::buildRemote(ref<Store> destStore,
+    MachineReservation::ptr & reservation,
    ::Machine::ptr machine, Step::ptr step,
-    const ServeProto::BuildOptions & buildOptions,
+    const BuildOptions & buildOptions,
    RemoteResult & result, std::shared_ptr<ActiveStep> activeStep,
    std::function<void(StepState)> updateStep,
    NarMemberDatas & narMembers)
@@ -415,26 +519,21 @@ void State::buildRemote(ref<Store> destStore,
    AutoDelete logFileDel(logFile, false);
    result.logFile = logFile;

+    nix::Path tmpDir = createTempDir();
+    AutoDelete tmpDirDel(tmpDir, true);
+
    try {

        updateStep(ssConnecting);

-        SSHMaster master {
-            machine->sshName,
-            machine->sshKey,
-            machine->sshPublicHostKey,
-            false, // no SSH master yet
-            false, // no compression yet
-            logFD.get(),
-        };
-
        // FIXME: rewrite to use Store.
-        auto child = build_remote::openConnection(machine, master);
+        SSHMaster::Connection child;
+        build_remote::openConnection(machine, tmpDir, logFD.get(), child);

        {
            auto activeStepState(activeStep->state_.lock());
            if (activeStepState->cancelled) throw Error("step cancelled");
-            activeStepState->pid = child->sshPid;
+            activeStepState->pid = child.sshPid;
        }

        Finally clearPid([&]() {
@@ -450,13 +549,9 @@ void State::buildRemote(ref<Store> destStore,
        });

        ::Machine::Connection conn {
-            {
-                .to = child->in.get(),
-                .from = child->out.get(),
-                /* Handshake. */
-                .remoteVersion = 0xdadbeef, // FIXME avoid dummy initialize
-            },
-            /*.machine =*/ machine,
+            .from = child.out.get(),
+            .to = child.in.get(),
+            .machine = machine,
        };

        Finally updateStats([&]() {
@@ -464,26 +559,14 @@ void State::buildRemote(ref<Store> destStore,
            bytesSent += conn.to.written;
        });

-        constexpr ServeProto::Version our_version = 0x206;
-
        try {
-            conn.remoteVersion = decltype(conn)::handshake(
-                conn.to,
-                conn.from,
-                our_version,
-                machine->sshName);
+          build_remote::handshake(conn, buildOptions.repeats);
        } catch (EndOfFile & e) {
-            child->sshPid.wait();
+            child.sshPid.wait();
            std::string s = chomp(readFile(result.logFile));
            throw Error("cannot connect to ‘%1%’: %2%", machine->sshName, s);
        }

-        // Do not attempt to speak a newer version of the protocol.
-        //
-        // Per https://github.com/NixOS/nix/issues/9584 should be handled as
-        // part of `handshake` in upstream nix.
-        conn.remoteVersion = std::min(conn.remoteVersion, our_version);
-
        {
            auto info(machine->state->connectInfo.lock());
            info->consecutiveFailures = 0;
@@ -540,6 +623,23 @@ void State::buildRemote(ref<Store> destStore,
            result.logFile = "";
        }

+        /* Throttle CPU-bound work. Opportunistically skip updating the current
+         * step, since this requires a DB roundtrip. */
+        if (!localWorkThrottler.try_acquire()) {
+            updateStep(ssWaitingForLocalSlot);
+            localWorkThrottler.acquire();
+        }
+        SemaphoreReleaser releaser(&localWorkThrottler);
+
+        /* Once we've started copying outputs, release the machine reservation
+         * so further builds can happen. We do not release the machine earlier
+         * to avoid situations where the queue runner is bottlenecked on
+         * copying outputs and we end up building too many things that we
+         * haven't been able to allow copy slots for. */
+        assert(reservation.unique());
+        reservation = 0;
+        wakeDispatcher();
+
        StorePathSet outputs;
        for (auto & [_, realisation] : buildResult.builtOutputs)
            outputs.insert(realisation.outPath);
@@ -586,8 +686,8 @@ void State::buildRemote(ref<Store> destStore,
        }

        /* Shut down the connection. */
-        child->in = -1;
-        child->sshPid.wait();
+        child.in = -1;
+        child.sshPid.wait();

    } catch (Error & e) {
        /* Disable this machine until a certain period of time has
--- a/src/hydra-queue-runner/builder.cc
+++ b/src/hydra-queue-runner/builder.cc
@@ -46,10 +46,12 @@ void State::builder(MachineReservation::ptr reservation)
        }
    }

-    /* Release the machine and wake up the dispatcher. */
-    assert(reservation.unique());
-    reservation = 0;
-    wakeDispatcher();
+    /* If the machine hasn't been released yet, release and wake up the dispatcher. */
+    if (reservation) {
+        assert(reservation.unique());
+        reservation = 0;
+        wakeDispatcher();
+    }

    /* If there was a temporary failure, retry the step after an
       exponentially increasing interval. */
@@ -72,11 +74,11 @@ void State::builder(MachineReservation::ptr reservation)


 State::StepResult State::doBuildStep(nix::ref<Store> destStore,
-    MachineReservation::ptr reservation,
+    MachineReservation::ptr & reservation,
    std::shared_ptr<ActiveStep> activeStep)
 {
-    auto & step(reservation->step);
-    auto & machine(reservation->machine);
+    auto step(reservation->step);
+    auto machine(reservation->machine);

    {
        auto step_(step->state.lock());
@@ -98,13 +100,10 @@ State::StepResult State::doBuildStep(nix::ref<Store> destStore,
       it). */
    BuildID buildId;
    std::optional<StorePath> buildDrvPath;
-    // Other fields set below
-    nix::ServeProto::BuildOptions buildOptions {
-        .maxLogSize = maxLogSize,
-        .nrRepeats = step->isDeterministic ? 1u : 0u,
-        .enforceDeterminism = step->isDeterministic,
-        .keepFailed = false,
-    };
+    BuildOptions buildOptions;
+    buildOptions.repeats = step->isDeterministic ? 1 : 0;
+    buildOptions.maxLogSize = maxLogSize;
+    buildOptions.enforceDeterminism = step->isDeterministic;

    auto conn(dbPool.get());

@@ -139,7 +138,7 @@ State::StepResult State::doBuildStep(nix::ref<Store> destStore,
            {
                auto i = jobsetRepeats.find(std::make_pair(build2->projectName, build2->jobsetName));
                if (i != jobsetRepeats.end())
-                    buildOptions.nrRepeats = std::max(buildOptions.nrRepeats, i->second);
+                    buildOptions.repeats = std::max(buildOptions.repeats, i->second);
            }
        }
        if (!build) build = *dependents.begin();
@@ -150,7 +149,7 @@ State::StepResult State::doBuildStep(nix::ref<Store> destStore,
        buildOptions.buildTimeout = build->buildTimeout;

        printInfo("performing step ‘%s’ %d times on ‘%s’ (needed by build %d and %d others)",
-            localStore->printStorePath(step->drvPath), buildOptions.nrRepeats + 1, machine->sshName, buildId, (dependents.size() - 1));
+            localStore->printStorePath(step->drvPath), buildOptions.repeats + 1, machine->sshName, buildId, (dependents.size() - 1));
    }

    if (!buildOneDone)
@@ -211,7 +210,7 @@ State::StepResult State::doBuildStep(nix::ref<Store> destStore,

        try {
            /* FIXME: referring builds may have conflicting timeouts. */
-            buildRemote(destStore, machine, step, buildOptions, result, activeStep, updateStep, narMembers);
+            buildRemote(destStore, reservation, machine, step, buildOptions, result, activeStep, updateStep, narMembers);
        } catch (Error & e) {
            if (activeStep->state_.lock()->cancelled) {
                printInfo("marking step %d of build %d as cancelled", stepNr, buildId);
--- a/src/hydra-queue-runner/dispatcher.cc
+++ b/src/hydra-queue-runner/dispatcher.cc
@@ -39,13 +39,15 @@ void State::dispatcher()
            printMsg(lvlDebug, "dispatcher woken up");
            nrDispatcherWakeups++;

-            auto now1 = std::chrono::steady_clock::now();
+            auto t_before_work = std::chrono::steady_clock::now();

            auto sleepUntil = doDispatch();

-            auto now2 = std::chrono::steady_clock::now();
+            auto t_after_work = std::chrono::steady_clock::now();

-            dispatchTimeMs += std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count();
+            prom.dispatcher_time_spent_running.Increment(
+                std::chrono::duration_cast<std::chrono::microseconds>(t_after_work - t_before_work).count());
+            dispatchTimeMs += std::chrono::duration_cast<std::chrono::milliseconds>(t_after_work - t_before_work).count();

            /* Sleep until we're woken up (either because a runnable build
               is added, or because a build finishes). */
@@ -59,6 +61,10 @@ void State::dispatcher()
                *dispatcherWakeup_ = false;
            }

+            auto t_after_sleep = std::chrono::steady_clock::now();
+            prom.dispatcher_time_spent_waiting.Increment(
+                std::chrono::duration_cast<std::chrono::microseconds>(t_after_sleep - t_after_work).count());
+
        } catch (std::exception & e) {
            printError("dispatcher: %s", e.what());
            sleep(1);
@@ -127,6 +133,8 @@ system_time State::doDispatch()
           comparator is a partial ordering (see MachineInfo). */
        int highestGlobalPriority;
        int highestLocalPriority;
+        size_t numRequiredSystemFeatures;
+        size_t numRevDeps;
        BuildID lowestBuildID;

        StepInfo(Step::ptr step, Step::State & step_) : step(step)
@@ -135,6 +143,8 @@ system_time State::doDispatch()
                lowestShareUsed = std::min(lowestShareUsed, jobset->shareUsed());
            highestGlobalPriority = step_.highestGlobalPriority;
            highestLocalPriority = step_.highestLocalPriority;
+            numRequiredSystemFeatures = step->requiredSystemFeatures.size();
+            numRevDeps = step_.rdeps.size();
            lowestBuildID = step_.lowestBuildID;
        }
    };
@@ -187,6 +197,8 @@ system_time State::doDispatch()
                a.highestGlobalPriority != b.highestGlobalPriority ? a.highestGlobalPriority > b.highestGlobalPriority :
                a.lowestShareUsed != b.lowestShareUsed ? a.lowestShareUsed < b.lowestShareUsed :
                a.highestLocalPriority != b.highestLocalPriority ? a.highestLocalPriority > b.highestLocalPriority :
+                a.numRequiredSystemFeatures != b.numRequiredSystemFeatures ? a.numRequiredSystemFeatures > b.numRequiredSystemFeatures :
+                a.numRevDeps != b.numRevDeps ? a.numRevDeps > b.numRevDeps :
                a.lowestBuildID < b.lowestBuildID;
        });

@@ -231,11 +243,11 @@ system_time State::doDispatch()
        sort(machinesSorted.begin(), machinesSorted.end(),
            [](const MachineInfo & a, const MachineInfo & b) -> bool
            {
-                float ta = std::round(a.currentJobs / a.machine->speedFactor);
-                float tb = std::round(b.currentJobs / b.machine->speedFactor);
+                float ta = std::round(a.currentJobs / a.machine->speedFactorFloat);
+                float tb = std::round(b.currentJobs / b.machine->speedFactorFloat);
                return
                    ta != tb ? ta < tb :
-                    a.machine->speedFactor != b.machine->speedFactor ? a.machine->speedFactor > b.machine->speedFactor :
+                    a.machine->speedFactorFloat != b.machine->speedFactorFloat ? a.machine->speedFactorFloat > b.machine->speedFactorFloat :
                    a.currentJobs > b.currentJobs;
            });

--- a/src/hydra-queue-runner/hydra-queue-runner.cc
+++ b/src/hydra-queue-runner/hydra-queue-runner.cc
@@ -70,10 +70,31 @@ State::PromMetrics::PromMetrics()
            .Register(*registry)
            .Add({})
    )
-    , queue_max_id(
-        prometheus::BuildGauge()
-            .Name("hydraqueuerunner_queue_max_build_id_info")
-            .Help("Maximum build record ID in the queue")
+    , dispatcher_time_spent_running(
+        prometheus::BuildCounter()
+            .Name("hydraqueuerunner_dispatcher_time_spent_running")
+            .Help("Time (in micros) spent running the dispatcher")
+            .Register(*registry)
+            .Add({})
+    )
+    , dispatcher_time_spent_waiting(
+        prometheus::BuildCounter()
+            .Name("hydraqueuerunner_dispatcher_time_spent_waiting")
+            .Help("Time (in micros) spent waiting for the dispatcher to obtain work")
+            .Register(*registry)
+            .Add({})
+    )
+    , queue_monitor_time_spent_running(
+        prometheus::BuildCounter()
+            .Name("hydraqueuerunner_queue_monitor_time_spent_running")
+            .Help("Time (in micros) spent running the queue monitor")
+            .Register(*registry)
+            .Add({})
+    )
+    , queue_monitor_time_spent_waiting(
+        prometheus::BuildCounter()
+            .Name("hydraqueuerunner_queue_monitor_time_spent_waiting")
+            .Help("Time (in micros) spent waiting for the queue monitor to obtain work")
            .Register(*registry)
            .Add({})
    )
@@ -85,6 +106,7 @@ State::State(std::optional<std::string> metricsAddrOpt)
    : config(std::make_unique<HydraConfig>())
    , maxUnsupportedTime(config->getIntOption("max_unsupported_time", 0))
    , dbPool(config->getIntOption("max_db_connections", 128))
+    , localWorkThrottler(config->getIntOption("max_local_worker_threads", std::min(maxSupportedLocalWorkers, std::max(4u, std::thread::hardware_concurrency()) - 2)))
    , maxOutputSize(config->getIntOption("max_output_size", 2ULL << 30))
    , maxLogSize(config->getIntOption("max_log_size", 64ULL << 20))
    , uploadLogsToBinaryCache(config->getBoolOption("upload_logs_to_binary_cache", false))
@@ -155,27 +177,29 @@ void State::parseMachines(const std::string & contents)
        auto machine = std::make_shared<::Machine>(nix::Machine {
            // `storeUri`, not yet used
            "",
-            // `systemTypes`
-            tokenizeString<StringSet>(tokens[1], ","),
+            // `systemTypes`, not yet used
+            {},
            // `sshKey`
            tokens[2] == "-" ? "" : tokens[2],
            // `maxJobs`
            tokens[3] != ""
                ? string2Int<MaxJobs>(tokens[3]).value()
                : 1,
-            // `speedFactor`
-            atof(tokens[4].c_str()),
+            // `speedFactor`, not yet used
+            1,
            // `supportedFeatures`
            std::move(supportedFeatures),
            // `mandatoryFeatures`
            std::move(mandatoryFeatures),
            // `sshPublicHostKey`
            tokens[7] != "" && tokens[7] != "-"
-                ? tokens[7]
+                ? base64Decode(tokens[7])
                : "",
        });

        machine->sshName = tokens[0];
+        machine->systemTypesSet = tokenizeString<StringSet>(tokens[1], ",");
+        machine->speedFactorFloat = atof(tokens[4].c_str());

        /* Re-use the State object of the previous machine with the
           same name. */
@@ -639,7 +663,7 @@ void State::dumpStatus(Connection & conn)

                json machine = {
                    {"enabled",  m->enabled},
-                    {"systemTypes", m->systemTypes},
+                    {"systemTypes", m->systemTypesSet},
                    {"supportedFeatures", m->supportedFeatures},
                    {"mandatoryFeatures", m->mandatoryFeatures},
                    {"nrStepsDone", s->nrStepsDone.load()},
--- a/src/hydra-queue-runner/nar-extractor.cc
+++ b/src/hydra-queue-runner/nar-extractor.cc
@@ -6,46 +6,7 @@

 using namespace nix;

-
-struct NarMemberConstructor : CreateRegularFileSink
-{
-    NarMemberData & curMember;
-
-    HashSink hashSink = HashSink { HashAlgorithm::SHA256 };
-
-    std::optional<uint64_t> expectedSize;
-
-    NarMemberConstructor(NarMemberData & curMember)
-        : curMember(curMember)
-    { }
-
-    void isExecutable() override
-    {
-    }
-
-    void preallocateContents(uint64_t size) override
-    {
-        expectedSize = size;
-    }
-
-    void operator () (std::string_view data) override
-    {
-        assert(expectedSize);
-        *curMember.fileSize += data.size();
-        hashSink(data);
-        if (curMember.contents) {
-            curMember.contents->append(data);
-        }
-        assert(curMember.fileSize <= expectedSize);
-        if (curMember.fileSize == expectedSize) {
-            auto [hash, len] = hashSink.finish();
-            assert(curMember.fileSize == len);
-            curMember.sha256 = hash;
-        }
-    }
-};
-
-struct Extractor : FileSystemObjectSink
+struct Extractor : ParseSink
 {
    std::unordered_set<Path> filesToKeep {
        "/nix-support/hydra-build-products",
@@ -54,6 +15,7 @@ struct Extractor : FileSystemObjectSink
    };

    NarMemberDatas & members;
+    NarMemberData * curMember = nullptr;
    Path prefix;

    Extractor(NarMemberDatas & members, const Path & prefix)
@@ -65,22 +27,53 @@ struct Extractor : FileSystemObjectSink
        members.insert_or_assign(prefix + path, NarMemberData { .type = SourceAccessor::Type::tDirectory });
    }

-    void createRegularFile(const Path & path, std::function<void(CreateRegularFileSink &)> func) override
+    void createRegularFile(const Path & path) override
    {
-        NarMemberConstructor nmc {
-            members.insert_or_assign(prefix + path, NarMemberData {
-                .type = SourceAccessor::Type::tRegular,
-                .fileSize = 0,
-                .contents = filesToKeep.count(path) ? std::optional("") : std::nullopt,
-            }).first->second,
-        };
-        func(nmc);
+        curMember = &members.insert_or_assign(prefix + path, NarMemberData {
+            .type = SourceAccessor::Type::tRegular,
+            .fileSize = 0,
+            .contents = filesToKeep.count(path) ? std::optional("") : std::nullopt,
+        }).first->second;
+    }
+
+    std::optional<uint64_t> expectedSize;
+    std::unique_ptr<HashSink> hashSink;
+
+    void preallocateContents(uint64_t size) override
+    {
+        expectedSize = size;
+        hashSink = std::make_unique<HashSink>(htSHA256);
+    }
+
+    void receiveContents(std::string_view data) override
+    {
+        assert(expectedSize);
+        assert(curMember);
+        assert(hashSink);
+        *curMember->fileSize += data.size();
+        (*hashSink)(data);
+        if (curMember->contents) {
+            curMember->contents->append(data);
+        }
+        assert(curMember->fileSize <= expectedSize);
+        if (curMember->fileSize == expectedSize) {
+            auto [hash, len] = hashSink->finish();
+            assert(curMember->fileSize == len);
+            curMember->sha256 = hash;
+            hashSink.reset();
+        }
    }

    void createSymlink(const Path & path, const std::string & target) override
    {
        members.insert_or_assign(prefix + path, NarMemberData { .type = SourceAccessor::Type::tSymlink });
    }
+
+    void isExecutable() override
+    { }
+
+    void closeRegularFile() override
+    { }
 };


--- a/src/hydra-queue-runner/queue-monitor.cc
+++ b/src/hydra-queue-runner/queue-monitor.cc
@@ -1,6 +1,7 @@
 #include "state.hh"
 #include "hydra-build-result.hh"
 #include "globals.hh"
+#include "thread-pool.hh"

 #include <cstring>

@@ -37,16 +38,21 @@ void State::queueMonitorLoop(Connection & conn)

    auto destStore = getDestStore();

-    unsigned int lastBuildId = 0;
-
    bool quit = false;
    while (!quit) {
+        auto t_before_work = std::chrono::steady_clock::now();
+
        localStore->clearPathInfoCache();

-        bool done = getQueuedBuilds(conn, destStore, lastBuildId);
+        bool done = getQueuedBuilds(conn, destStore);

        if (buildOne && buildOneDone) quit = true;

+        auto t_after_work = std::chrono::steady_clock::now();
+
+        prom.queue_monitor_time_spent_running.Increment(
+            std::chrono::duration_cast<std::chrono::microseconds>(t_after_work - t_before_work).count());
+
        /* Sleep until we get notification from the database about an
           event. */
        if (done && !quit) {
@@ -56,12 +62,10 @@ void State::queueMonitorLoop(Connection & conn)
            conn.get_notifs();

        if (auto lowestId = buildsAdded.get()) {
-            lastBuildId = std::min(lastBuildId, static_cast<unsigned>(std::stoul(*lowestId) - 1));
            printMsg(lvlTalkative, "got notification: new builds added to the queue");
        }
        if (buildsRestarted.get()) {
            printMsg(lvlTalkative, "got notification: builds restarted");
-            lastBuildId = 0; // check all builds
        }
        if (buildsCancelled.get() || buildsDeleted.get() || buildsBumped.get()) {
            printMsg(lvlTalkative, "got notification: builds cancelled or bumped");
@@ -71,6 +75,10 @@ void State::queueMonitorLoop(Connection & conn)
            printMsg(lvlTalkative, "got notification: jobset shares changed");
            processJobsetSharesChange(conn);
        }
+
+        auto t_after_sleep = std::chrono::steady_clock::now();
+        prom.queue_monitor_time_spent_waiting.Increment(
+            std::chrono::duration_cast<std::chrono::microseconds>(t_after_sleep - t_after_work).count());
    }

    exit(0);
@@ -84,20 +92,18 @@ struct PreviousFailure : public std::exception {


 bool State::getQueuedBuilds(Connection & conn,
-    ref<Store> destStore, unsigned int & lastBuildId)
+    ref<Store> destStore)
 {
    prom.queue_checks_started.Increment();

-    printInfo("checking the queue for builds > %d...", lastBuildId);
+    printInfo("checking the queue for builds...");

    /* Grab the queued builds from the database, but don't process
       them yet (since we don't want a long-running transaction). */
    std::vector<BuildID> newIDs;
-    std::map<BuildID, Build::ptr> newBuildsByID;
+    std::unordered_map<BuildID, Build::ptr> newBuildsByID;
    std::multimap<StorePath, BuildID> newBuildsByPath;

-    unsigned int newLastBuildId = lastBuildId;
-
    {
        pqxx::work txn(conn);

@@ -106,17 +112,12 @@ bool State::getQueuedBuilds(Connection & conn,
             "jobsets.name as jobset, job, drvPath, maxsilent, timeout, timestamp, "
             "globalPriority, priority from Builds "
             "inner join jobsets on builds.jobset_id = jobsets.id "
-             "where builds.id > $1 and finished = 0 order by globalPriority desc, builds.id",
-            lastBuildId);
+             "where finished = 0 order by globalPriority desc, random()");

        for (auto const & row : res) {
            auto builds_(builds.lock());
            BuildID id = row["id"].as<BuildID>();
            if (buildOne && id != buildOne) continue;
-            if (id > newLastBuildId) {
-                newLastBuildId = id;
-                prom.queue_max_id.Set(id);
-            }
            if (builds_->count(id)) continue;

            auto build = std::make_shared<Build>(
@@ -318,15 +319,13 @@ bool State::getQueuedBuilds(Connection & conn,

        /* Stop after a certain time to allow priority bumps to be
           processed. */
-        if (std::chrono::system_clock::now() > start + std::chrono::seconds(600)) {
+        if (std::chrono::system_clock::now() > start + std::chrono::seconds(60)) {
            prom.queue_checks_early_exits.Increment();
            break;
        }
    }

    prom.queue_checks_finished.Increment();
-
-    lastBuildId = newBuildsByID.empty() ? newLastBuildId : newBuildsByID.begin()->first - 1;
    return newBuildsByID.empty();
 }

@@ -405,6 +404,34 @@ void State::processQueueChange(Connection & conn)
 }


+std::map<DrvOutput, std::optional<StorePath>> State::getMissingRemotePaths(
+    ref<Store> destStore,
+    const std::map<DrvOutput, std::optional<StorePath>> & paths)
+{
+    Sync<std::map<DrvOutput, std::optional<StorePath>>> missing_;
+    ThreadPool tp;
+
+    for (auto & [output, maybeOutputPath] : paths) {
+        if (!maybeOutputPath) {
+            auto missing(missing_.lock());
+            missing->insert({output, maybeOutputPath});
+        } else {
+            tp.enqueue([&] {
+                if (!destStore->isValidPath(*maybeOutputPath)) {
+                    auto missing(missing_.lock());
+                    missing->insert({output, maybeOutputPath});
+                }
+            });
+        }
+    }
+
+    tp.process();
+
+    auto missing(missing_.lock());
+    return *missing;
+}
+
+
 Step::ptr State::createStep(ref<Store> destStore,
    Connection & conn, Build::ptr build, const StorePath & drvPath,
    Build::ptr referringBuild, Step::ptr referringStep, std::set<StorePath> & finishedDrvs,
@@ -485,16 +512,15 @@ Step::ptr State::createStep(ref<Store> destStore,

    /* Are all outputs valid? */
    auto outputHashes = staticOutputHashes(*localStore, *(step->drv));
-    bool valid = true;
-    std::map<DrvOutput, std::optional<StorePath>> missing;
+    std::map<DrvOutput, std::optional<StorePath>> paths;
    for (auto & [outputName, maybeOutputPath] : destStore->queryPartialDerivationOutputMap(drvPath, &*localStore)) {
        auto outputHash = outputHashes.at(outputName);
-        if (maybeOutputPath && destStore->isValidPath(*maybeOutputPath))
-            continue;
-        valid = false;
-        missing.insert({{outputHash, outputName}, maybeOutputPath});
+        paths.insert({{outputHash, outputName}, maybeOutputPath});
    }

+    auto missing = getMissingRemotePaths(destStore, paths);
+    bool valid = missing.empty();
+
    /* Try to copy the missing paths from the local store or from
       substitutes. */
    if (!missing.empty()) {
@@ -700,7 +726,7 @@ BuildOutput State::getBuildOutputCached(Connection & conn, nix::ref<nix::Store>
                product.fileSize = row[2].as<off_t>();
            }
            if (!row[3].is_null())
-                product.sha256hash = Hash::parseAny(row[3].as<std::string>(), HashAlgorithm::SHA256);
+                product.sha256hash = Hash::parseAny(row[3].as<std::string>(), htSHA256);
            if (!row[4].is_null())
                product.path = row[4].as<std::string>();
            product.name = row[5].as<std::string>();
--- a/src/hydra-queue-runner/state.hh
+++ b/src/hydra-queue-runner/state.hh
@@ -7,6 +7,7 @@
 #include <memory>
 #include <queue>
 #include <regex>
+#include <semaphore>

 #include <prometheus/counter.h>
 #include <prometheus/gauge.h>
@@ -22,7 +23,6 @@
 #include "sync.hh"
 #include "nar-extractor.hh"
 #include "serve-protocol.hh"
-#include "serve-protocol-impl.hh"
 #include "machines.hh"


@@ -57,6 +57,7 @@ typedef enum {
    ssConnecting = 10,
    ssSendingInputs = 20,
    ssBuilding = 30,
+    ssWaitingForLocalSlot = 35,
    ssReceivingOutputs = 40,
    ssPostProcessing = 50,
 } StepState;
@@ -244,6 +245,14 @@ struct Machine : nix::Machine
       we are not yet used to, but once we are, we don't need this. */
    std::string sshName;

+    /* TODO Get rid once `nix::Machine::systemTypes` is a set not
+       vector. */
+    std::set<std::string> systemTypesSet;
+
+    /* TODO Get rid once `nix::Machine::systemTypes` is a `float` not
+       an `int`. */
+    float speedFactorFloat = 1.0;
+
    struct State {
        typedef std::shared_ptr<State> ptr;
        counter currentJobs{0};
@@ -270,7 +279,7 @@ struct Machine : nix::Machine
    {
        /* Check that this machine is of the type required by the
           step. */
-        if (!systemTypes.count(step->drv->platform == "builtin" ? nix::settings.thisSystem : step->drv->platform))
+        if (!systemTypesSet.count(step->drv->platform == "builtin" ? nix::settings.thisSystem : step->drv->platform))
            return false;

        /* Check that the step requires all mandatory features of this
@@ -300,9 +309,29 @@ struct Machine : nix::Machine
    }

    // A connection to a machine
-    struct Connection : nix::ServeProto::BasicClientConnection {
+    struct Connection {
+        nix::FdSource from;
+        nix::FdSink to;
+        nix::ServeProto::Version remoteVersion;
+
        // Backpointer to the machine
        ptr machine;
+
+        operator nix::ServeProto::ReadConn ()
+        {
+            return {
+                .from = from,
+                .version = remoteVersion,
+            };
+        }
+
+        operator nix::ServeProto::WriteConn ()
+        {
+            return {
+                .to = to,
+                .version = remoteVersion,
+            };
+        }
    };
 };

@@ -360,6 +389,10 @@ private:
    typedef std::map<std::string, Machine::ptr> Machines;
    nix::Sync<Machines> machines; // FIXME: use atomic_shared_ptr

+    /* Throttler for CPU-bound local work. */
+    static constexpr unsigned int maxSupportedLocalWorkers = 1024;
+    std::counting_semaphore<maxSupportedLocalWorkers> localWorkThrottler;
+
    /* Various stats. */
    time_t startedAt;
    counter nrBuildsRead{0};
@@ -437,7 +470,7 @@ private:

    /* How often the build steps of a jobset should be repeated in
       order to detect non-determinism. */
-    std::map<std::pair<std::string, std::string>, size_t> jobsetRepeats;
+    std::map<std::pair<std::string, std::string>, unsigned int> jobsetRepeats;

    bool uploadLogsToBinaryCache;

@@ -457,7 +490,12 @@ private:
        prometheus::Counter& queue_steps_created;
        prometheus::Counter& queue_checks_early_exits;
        prometheus::Counter& queue_checks_finished;
-        prometheus::Gauge& queue_max_id;
+
+        prometheus::Counter& dispatcher_time_spent_running;
+        prometheus::Counter& dispatcher_time_spent_waiting;
+
+        prometheus::Counter& queue_monitor_time_spent_running;
+        prometheus::Counter& queue_monitor_time_spent_waiting;

        PromMetrics();
    };
@@ -466,6 +504,12 @@ private:
 public:
    State(std::optional<std::string> metricsAddrOpt);

+    struct BuildOptions {
+        unsigned int maxSilentTime, buildTimeout, repeats;
+        size_t maxLogSize;
+        bool enforceDeterminism;
+    };
+
 private:

    nix::MaintainCount<counter> startDbUpdate();
@@ -501,8 +545,7 @@ private:
    void queueMonitorLoop(Connection & conn);

    /* Check the queue for new builds. */
-    bool getQueuedBuilds(Connection & conn,
-        nix::ref<nix::Store> destStore, unsigned int & lastBuildId);
+    bool getQueuedBuilds(Connection & conn, nix::ref<nix::Store> destStore);

    /* Handle cancellation, deletion and priority bumps. */
    void processQueueChange(Connection & conn);
@@ -510,6 +553,12 @@ private:
    BuildOutput getBuildOutputCached(Connection & conn, nix::ref<nix::Store> destStore,
        const nix::StorePath & drvPath);

+    /* Returns paths missing from the remote store. Paths are processed in
+     * parallel to work around the possible latency of remote stores. */
+    std::map<nix::DrvOutput, std::optional<nix::StorePath>> getMissingRemotePaths(
+        nix::ref<nix::Store> destStore,
+        const std::map<nix::DrvOutput, std::optional<nix::StorePath>> & paths);
+
    Step::ptr createStep(nix::ref<nix::Store> store,
        Connection & conn, Build::ptr build, const nix::StorePath & drvPath,
        Build::ptr referringBuild, Step::ptr referringStep, std::set<nix::StorePath> & finishedDrvs,
@@ -545,12 +594,13 @@ private:
       retried. */
    enum StepResult { sDone, sRetry, sMaybeCancelled };
    StepResult doBuildStep(nix::ref<nix::Store> destStore,
-        MachineReservation::ptr reservation,
+        MachineReservation::ptr & reservation,
        std::shared_ptr<ActiveStep> activeStep);

    void buildRemote(nix::ref<nix::Store> destStore,
+        MachineReservation::ptr & reservation,
        Machine::ptr machine, Step::ptr step,
-        const nix::ServeProto::BuildOptions & buildOptions,
+        const BuildOptions & buildOptions,
        RemoteResult & result, std::shared_ptr<ActiveStep> activeStep,
        std::function<void(StepState)> updateStep,
        NarMemberDatas & narMembers);
--- a/src/lib/Hydra/Config.pm
+++ b/src/lib/Hydra/Config.pm
@@ -95,6 +95,7 @@ sub get_legacy_ldap_config {
            "hydra_bump-to-front" => [ "bump-to-front" ],
            "hydra_cancel-build" => [ "cancel-build" ],
            "hydra_create-projects" => [ "create-projects" ],
+            "hydra_eval-jobset" => [ "eval-jobset" ],
            "hydra_restart-jobs" => [ "restart-jobs" ],
        },
    };
@@ -159,6 +160,7 @@ sub valid_roles {
        "bump-to-front",
        "cancel-build",
        "create-projects",
+        "eval-jobset",
        "restart-jobs",
    ];
 }
--- a/src/lib/Hydra/Controller/API.pm
+++ b/src/lib/Hydra/Controller/API.pm
@@ -239,6 +239,8 @@ sub triggerJobset {
 sub push : Chained('api') PathPart('push') Args(0) {
    my ($self, $c) = @_;

+    requirePost($c);
+
    $c->{stash}->{json}->{jobsetsTriggered} = [];

    my $force = exists $c->request->query_params->{force};
@@ -246,19 +248,24 @@ sub push : Chained('api') PathPart('push') Args(0) {
    foreach my $s (@jobsets) {
        my ($p, $j) = parseJobsetName($s);
        my $jobset = $c->model('DB::Jobsets')->find($p, $j);
+        requireEvalJobsetPrivileges($c, $jobset->project);
        next unless defined $jobset && ($force || ($jobset->project->enabled && $jobset->enabled));
        triggerJobset($self, $c, $jobset, $force);
    }

    my @repos = split /,/, ($c->request->query_params->{repos} // "");
    foreach my $r (@repos) {
-        triggerJobset($self, $c, $_, $force) foreach $c->model('DB::Jobsets')->search(
+        my @jobsets = $c->model('DB::Jobsets')->search(
            { 'project.enabled' => 1, 'me.enabled' => 1 },
            {
                join => 'project',
                where => \ [ 'exists (select 1 from JobsetInputAlts where project = me.project and jobset = me.name and value = ?)', [ 'value', $r ] ],
                order_by => 'me.id DESC'
            });
+        foreach my $jobset (@jobsets) {
+            requireEvalJobsetPrivileges($c, $jobset->project);
+            triggerJobset($self, $c, $jobset, $force)
+        }
    }

    $self->status_ok(
--- a/src/lib/Hydra/Controller/Build.pm
+++ b/src/lib/Hydra/Controller/Build.pm
@@ -236,6 +236,9 @@ sub serveFile {
    }

    elsif ($ls->{type} eq "regular") {
+        # Have the hosted data considered its own origin to avoid being a giant
+        # XSS hole.
+        $c->response->header('Content-Security-Policy' => 'sandbox allow-scripts');

        $c->stash->{'plain'} = { data => grab(cmd => ["nix", "--experimental-features", "nix-command",
                                                      "store", "cat", "--store", getStoreUri(), "$path"]) };
--- a/src/lib/Hydra/Controller/Jobset.pm
+++ b/src/lib/Hydra/Controller/Jobset.pm
@@ -364,6 +364,21 @@ sub evals_GET {
    );
 }

+sub errors :Chained('jobsetChain') :PathPart('errors') :Args(0) :ActionClass('REST') { }
+
+sub errors_GET {
+    my ($self, $c) = @_;
+
+    $c->stash->{template} = 'eval-error.tt';
+
+    my $jobsetName = $c->stash->{params}->{name};
+    $c->stash->{jobset} = $c->stash->{project}->jobsets->find(
+        { name => $jobsetName },
+        { '+columns' => { 'errormsg' => 'errormsg' } }
+    );
+
+    $self->status_ok($c, entity => $c->stash->{jobset});
+}

 # Redirect to the latest finished evaluation of this jobset.
 sub latest_eval : Chained('jobsetChain') PathPart('latest-eval') {
--- a/src/lib/Hydra/Controller/JobsetEval.pm
+++ b/src/lib/Hydra/Controller/JobsetEval.pm
@@ -86,6 +86,17 @@ sub view_GET {
    );
 }

+sub errors :Chained('evalChain') :PathPart('errors') :Args(0) :ActionClass('REST') { }
+
+sub errors_GET {
+    my ($self, $c) = @_;
+
+    $c->stash->{template} = 'eval-error.tt';
+
+    $c->stash->{eval} = $c->model('DB::JobsetEvals')->find($c->stash->{eval}->id, { prefetch => 'evaluationerror' });
+
+    $self->status_ok($c, entity => $c->stash->{eval});
+}

 sub create_jobset : Chained('evalChain') PathPart('create-jobset') Args(0) {
    my ($self, $c) = @_;
--- a/src/lib/Hydra/Controller/Root.pm
+++ b/src/lib/Hydra/Controller/Root.pm
@@ -160,7 +160,7 @@ sub status_GET {
            { "buildsteps.busy" => { '!=', 0 } },
            { order_by => ["globalpriority DESC", "id"],
              join => "buildsteps",
-              columns => [@buildListColumns]
+              columns => [@buildListColumns, 'buildsteps.drvpath', 'buildsteps.type']
            })]
    );
 }
--- a/src/lib/Hydra/Helper/BuildDiff.pm
+++ b/src/lib/Hydra/Helper/BuildDiff.pm
@@ -37,7 +37,16 @@ sub buildDiff {

    my $n = 0;
    foreach my $build (@{$builds}) {
-        my $aborted = $build->finished != 0 && ($build->buildstatus == 3 || $build->buildstatus == 4);
+        my $aborted = $build->finished != 0 && (
+            # aborted
+            $build->buildstatus == 3
+            # cancelled
+            || $build->buildstatus == 4
+            # timeout
+            || $build->buildstatus == 7
+            # log limit exceeded
+            || $build->buildstatus == 10
+        );
        my $d;
        my $found = 0;
        while ($n < scalar(@{$builds2})) {
@@ -79,4 +88,4 @@ sub buildDiff {
    return $ret;
 }

-1;
+1;
--- a/src/lib/Hydra/Helper/CatalystUtils.pm
+++ b/src/lib/Hydra/Helper/CatalystUtils.pm
@@ -15,6 +15,7 @@ our @EXPORT = qw(
    forceLogin requireUser requireProjectOwner requireRestartPrivileges requireAdmin requirePost isAdmin isProjectOwner
    requireBumpPrivileges
    requireCancelBuildPrivileges
+    requireEvalJobsetPrivileges
    trim
    getLatestFinishedEval getFirstEval
    paramToList
@@ -186,6 +187,27 @@ sub isProjectOwner {
         defined $c->model('DB::ProjectMembers')->find({ project => $project, userName => $c->user->username }));
 }

+sub hasEvalJobsetRole {
+    my ($c) = @_;
+    return $c->user_exists && $c->check_user_roles("eval-jobset");
+}
+
+sub mayEvalJobset {
+    my ($c, $project) = @_;
+    return
+        $c->user_exists &&
+        (isAdmin($c) ||
+         hasEvalJobsetRole($c) ||
+         isProjectOwner($c, $project));
+}
+
+sub requireEvalJobsetPrivileges {
+    my ($c, $project) = @_;
+    requireUser($c);
+    accessDenied($c, "Only the project members, administrators, and accounts with eval-jobset privileges can perform this operation.")
+        unless mayEvalJobset($c, $project);
+}
+
 sub hasCancelBuildRole {
    my ($c) = @_;
    return $c->user_exists && $c->check_user_roles('cancel-build');
@@ -272,7 +294,7 @@ sub requireAdmin {

 sub requirePost {
    my ($c) = @_;
-    error($c, "Request must be POSTed.") if $c->request->method ne "POST";
+    error($c, "Request must be POSTed.", 405) if $c->request->method ne "POST";
 }


--- a/src/lib/Hydra/Helper/Nix.pm
+++ b/src/lib/Hydra/Helper/Nix.pm
@@ -286,8 +286,7 @@ sub getEvals {

    my @evals = $evals_result_set->search(
        { hasnewbuilds => 1 },
-        { order_by => "$me.id DESC", rows => $rows, offset => $offset
-        , prefetch => { evaluationerror => [ ] } });
+        { order_by => "$me.id DESC", rows => $rows, offset => $offset });
    my @res = ();
    my $cache = {};

--- a/src/lib/Hydra/Plugin/GiteaStatus.pm
+++ b/src/lib/Hydra/Plugin/GiteaStatus.pm
@@ -88,6 +88,10 @@ sub buildQueued {
    common(@_, [], 0);
 }

+sub buildStarted {
+    common(@_, [], 1);
+}
+
 sub buildFinished {
    common(@_, 2);
 }
--- a/src/lib/Hydra/Schema/Result/EvaluationErrors.pm
+++ b/src/lib/Hydra/Schema/Result/EvaluationErrors.pm
@@ -105,4 +105,6 @@ __PACKAGE__->add_column(
    "+id" => { retrieve_on_insert => 1 }
 );

+__PACKAGE__->mk_group_accessors('column' => 'has_error');
+
 1;
--- a/src/lib/Hydra/Schema/Result/Jobsets.pm
+++ b/src/lib/Hydra/Schema/Result/Jobsets.pm
@@ -386,6 +386,8 @@ __PACKAGE__->add_column(
    "+id" => { retrieve_on_insert => 1 }
 );

+__PACKAGE__->mk_group_accessors('column' => 'has_error');
+
 sub supportsDynamicRunCommand {
  my ($self) = @_;

--- a/src/lib/Hydra/Schema/ResultSet/EvaluationErrors.pm
+++ b/src/lib/Hydra/Schema/ResultSet/EvaluationErrors.pm
@@ -0,0 +1,30 @@
+package Hydra::Schema::ResultSet::EvaluationErrors;
+
+use strict;
+use utf8;
+use warnings;
+
+use parent 'DBIx::Class::ResultSet';
+
+use Storable qw(dclone);
+
+__PACKAGE__->load_components('Helper::ResultSet::RemoveColumns');
+
+# Exclude expensive error message values unless explicitly requested, and
+# replace them with a summary field describing their presence/absence.
+sub search_rs {
+  my ( $class, $query, $attrs ) = @_;
+
+  if ($attrs) {
+    $attrs = dclone($attrs);
+  }
+
+  unless (exists $attrs->{'select'} || exists $attrs->{'columns'}) {
+    $attrs->{'+columns'}->{'has_error'} = "errormsg != ''";
+  }
+  unless (exists $attrs->{'+columns'}->{'errormsg'}) {
+    push @{ $attrs->{'remove_columns'} }, 'errormsg';
+  }
+
+  return $class->next::method($query, $attrs);
+}
--- a/src/lib/Hydra/Schema/ResultSet/Jobsets.pm
+++ b/src/lib/Hydra/Schema/ResultSet/Jobsets.pm
@@ -0,0 +1,30 @@
+package Hydra::Schema::ResultSet::Jobsets;
+
+use strict;
+use utf8;
+use warnings;
+
+use parent 'DBIx::Class::ResultSet';
+
+use Storable qw(dclone);
+
+__PACKAGE__->load_components('Helper::ResultSet::RemoveColumns');
+
+# Exclude expensive error message values unless explicitly requested, and
+# replace them with a summary field describing their presence/absence.
+sub search_rs {
+  my ( $class, $query, $attrs ) = @_;
+
+  if ($attrs) {
+    $attrs = dclone($attrs);
+  }
+
+  unless (exists $attrs->{'select'} || exists $attrs->{'columns'}) {
+    $attrs->{'+columns'}->{'has_error'} = "errormsg != ''";
+  }
+  unless (exists $attrs->{'+columns'}->{'errormsg'}) {
+    push @{ $attrs->{'remove_columns'} }, 'errormsg';
+  }
+
+  return $class->next::method($query, $attrs);
+}
--- a/src/root/build.tt
+++ b/src/root/build.tt
@@ -61,21 +61,7 @@ END;
            <td>[% IF step.busy != 0 || ((step.machine || step.starttime) && (step.status == 0 || step.status == 1 || step.status == 3 || step.status == 4 || step.status == 7)); INCLUDE renderMachineName machine=step.machine; ELSE; "<em>n/a</em>"; END %]</td>
            <td class="step-status">
              [% IF step.busy != 0 %]
-                [% IF step.busy == 1 %]
-                  <strong>Preparing</strong>
-                [% ELSIF step.busy == 10 %]
-                  <strong>Connecting</strong>
-                [% ELSIF step.busy == 20 %]
-                  <strong>Sending inputs</strong>
-                [% ELSIF step.busy == 30 %]
-                  <strong>Building</strong>
-                [% ELSIF step.busy == 40 %]
-                  <strong>Receiving outputs</strong>
-                [% ELSIF step.busy == 50 %]
-                  <strong>Post-processing</strong>
-                [% ELSE %]
-                  <strong>Unknown state</strong>
-                [% END %]
+                [% INCLUDE renderBusyStatus %]
              [% ELSIF step.status == 0 %]
                [% IF step.isnondeterministic %]
                  <span class="warn">Succeeded with non-determistic result</span>
--- a/src/root/common.tt
+++ b/src/root/common.tt
@@ -91,6 +91,17 @@ BLOCK renderDuration;
  duration % 60 %]s[%
 END;

+BLOCK renderDrvInfo;
+  drvname = step.drvpath
+    .substr(11) # strip `/nix/store/`
+    .split('-').slice(1).join("-") # strip hash part
+    .substr(0, -4); # strip `.drv`
+  IF drvname != releasename;
+    IF step.type == 0; action = "Build"; ELSE; action = "Substitution"; END;
+    IF drvname; %]<em> ([% action %] of [% drvname %])</em>[% END;
+  END;
+END;
+

 BLOCK renderBuildListHeader %]
  <table class="table table-striped table-condensed clickable-rows">
@@ -131,7 +142,12 @@ BLOCK renderBuildListBody;
      [% END %]
      <td><a class="row-link" href="[% link %]">[% build.id %]</a></td>
      [% IF !hideJobName %]
-        <td><a href="[%link%]">[% IF !hideJobsetName %][%build.jobset.get_column("project")%]:[%build.jobset.get_column("name")%]:[% END %][%build.get_column("job")%]</td>
+        <td>
+          <a href="[%link%]">[% IF !hideJobsetName %][%build.jobset.get_column("project")%]:[%build.jobset.get_column("name")%]:[% END %][%build.get_column("job")%]</a>
+          [% IF showStepName %]
+            [% INCLUDE renderDrvInfo step=build.buildsteps releasename=build.nixname %]
+          [% END %]
+        </td>
      [% END %]
      <td class="nowrap">[% t = showSchedulingInfo ? build.timestamp : build.stoptime; IF t; INCLUDE renderRelativeDate timestamp=(showSchedulingInfo ? build.timestamp : build.stoptime); ELSE; "-"; END %]</td>
      <td>[% !showSchedulingInfo and build.get_column('releasename') ? build.get_column('releasename') : build.nixname %]</td>
@@ -245,6 +261,27 @@ BLOCK renderBuildStatusIcon;
 END;


+BLOCK renderBusyStatus;
+  IF step.busy == 1 %]
+    <strong>Preparing</strong>
+  [% ELSIF step.busy == 10 %]
+    <strong>Connecting</strong>
+  [% ELSIF step.busy == 20 %]
+    <strong>Sending inputs</strong>
+  [% ELSIF step.busy == 30 %]
+    <strong>Building</strong>
+  [% ELSIF step.busy == 35 %]
+    <strong>Waiting to receive outputs</strong>
+  [% ELSIF step.busy == 40 %]
+    <strong>Receiving outputs</strong>
+  [% ELSIF step.busy == 50 %]
+    <strong>Post-processing</strong>
+  [% ELSE %]
+    <strong>Unknown state</strong>
+  [% END;
+END;
+
+
 BLOCK renderStatus;
  IF build.finished;
    buildstatus = build.buildstatus;
@@ -476,7 +513,7 @@ BLOCK renderEvals %]
            ELSE %]
             -
            [% END %]
-            [% IF eval.evaluationerror.errormsg %]
+            [% IF eval.evaluationerror.has_error %]
              <span class="badge badge-warning">Eval Errors</span>
            [% END %]
          </td>
@@ -602,7 +639,7 @@ BLOCK renderJobsetOverview %]
        <td>[% HTML.escape(j.description) %]</td>
        <td>[% IF j.lastcheckedtime;
                 INCLUDE renderDateTime timestamp = j.lastcheckedtime;
-                 IF j.errormsg || j.fetcherrormsg; %]&nbsp;<span class = 'badge badge-warning'>Error</span>[% END;
+                 IF j.has_error || j.fetcherrormsg; %]&nbsp;<span class = 'badge badge-warning'>Error</span>[% END;
                 ELSE; "-";
               END %]</td>
        [% IF j.get_column('nrtotal') > 0 %]
--- a/src/root/eval-error.tt
+++ b/src/root/eval-error.tt
@@ -0,0 +1,26 @@
+[% PROCESS common.tt %]
+<!DOCTYPE html>
+
+<html lang="en">
+
+  <head>
+    <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
+    <meta http-equiv="X-UA-Compatible" content="IE=Edge" />
+    [% INCLUDE style.tt %]
+  </head>
+
+  <body>
+
+    <div class="tab-content tab-pane">
+        <div id="tabs-errors" class="">
+          [% IF jobset %]
+              <p>Errors occurred at [% INCLUDE renderDateTime timestamp=(jobset.errortime || jobset.lastcheckedtime) %].</p>
+              <div class="card bg-light"><div class="card-body"><pre>[% HTML.escape(jobset.fetcherrormsg || jobset.errormsg) %]</pre></div></div>
+          [% ELSIF eval %]
+              <p>Errors occurred at [% INCLUDE renderDateTime timestamp=(eval.evaluationerror.errortime || eval.timestamp) %].</p>
+              <div class="card bg-light"><div class="card-body"><pre>[% HTML.escape(eval.evaluationerror.errormsg) %]</pre></div></div>
+          [% END %]
+        </div>
+    </div>
+  </body>
+</html>
--- a/src/root/jobset-eval.tt
+++ b/src/root/jobset-eval.tt
@@ -65,7 +65,7 @@ c.uri_for(c.controller('JobsetEval').action_for('view'),
  [% END %]

  [% IF aborted.size > 0 %]
-    <li class="nav-item"><a class="nav-link" href="#tabs-aborted" data-toggle="tab"><span class="text-warning">Aborted Jobs ([% aborted.size %])</span></a></li>
+    <li class="nav-item"><a class="nav-link" href="#tabs-aborted" data-toggle="tab"><span class="text-warning">Aborted / Timed out Jobs ([% aborted.size %])</span></a></li>
  [% END %]
  [% IF nowFail.size > 0 %]
    <li class="nav-item"><a class="nav-link" href="#tabs-now-fail" data-toggle="tab"><span class="text-warning">Newly Failing Jobs ([% nowFail.size %])</span></a></li>
@@ -90,7 +90,7 @@ c.uri_for(c.controller('JobsetEval').action_for('view'),
  [% END %]
  <li class="nav-item"><a class="nav-link" href="#tabs-inputs" data-toggle="tab">Inputs</a></li>

-  [% IF eval.evaluationerror.errormsg %]
+  [% IF eval.evaluationerror.has_error %]
    <li class="nav-item"><a class="nav-link" href="#tabs-errors" data-toggle="tab"><span class="text-warning">Evaluation Errors</span></a></li>
  [% END %]
 </ul>
@@ -108,13 +108,6 @@ c.uri_for(c.controller('JobsetEval').action_for('view'),

 <div class="tab-content">

-  [% IF eval.evaluationerror.errormsg %]
-    <div id="tabs-errors" class="tab-pane">
-      <p>Errors occurred at [% INCLUDE renderDateTime timestamp=(eval.evaluationerror.errortime || eval.timestamp) %].</p>
-      <div class="card bg-light"><div class="card-body"><pre>[% HTML.escape(eval.evaluationerror.errormsg) %]</pre></div></div>
-    </div>
-  [% END %]
-
  <div id="tabs-aborted" class="tab-pane">
    [% INCLUDE renderSome builds=aborted tabname="#tabs-aborted" %]
  </div>
@@ -172,10 +165,9 @@ c.uri_for(c.controller('JobsetEval').action_for('view'),
    [% END %]
  </div>

-  [% IF eval.evaluationerror.errormsg %]
+  [% IF eval.evaluationerror.has_error %]
    <div id="tabs-errors" class="tab-pane">
-      <p>Errors occurred at [% INCLUDE renderDateTime timestamp=(eval.evaluationerror.errortime || eval.timestamp) %].</p>
-      <div class="card bg-light"><div class="card-body"><pre>[% HTML.escape(eval.evaluationerror.errormsg) %]</pre></div></div>
+      <iframe src="[% c.uri_for(c.controller('JobsetEval').action_for('errors'), [eval.id], params) %]" loading="lazy" frameBorder="0" width="100%"></iframe>
    </div>
  [% END %]
 </div>
--- a/src/root/jobset.tt
+++ b/src/root/jobset.tt
@@ -61,7 +61,7 @@
  [% END %]

  <li class="nav-item"><a class="nav-link active" href="#tabs-evaluations" data-toggle="tab">Evaluations</a></li>
-  [% IF jobset.errormsg || jobset.fetcherrormsg %]
+  [% IF jobset.has_error || jobset.fetcherrormsg %]
    <li class="nav-item"><a class="nav-link" href="#tabs-errors" data-toggle="tab"><span class="text-warning">Evaluation Errors</span></a></li>
  [% END %]
  <li class="nav-item"><a class="nav-link" href="#tabs-jobs" data-toggle="tab">Jobs</a></li>
@@ -79,7 +79,7 @@
        <th>Last checked:</th>
        <td>
          [% IF jobset.lastcheckedtime %]
-            [% INCLUDE renderDateTime timestamp = jobset.lastcheckedtime %], [% IF jobset.errormsg || jobset.fetcherrormsg %]<em class="text-warning">with errors!</em>[% ELSE %]<em>no errors</em>[% END %]
+            [% INCLUDE renderDateTime timestamp = jobset.lastcheckedtime %], [% IF jobset.has_error || jobset.fetcherrormsg %]<em class="text-warning">with errors!</em>[% ELSE %]<em>no errors</em>[% END %]
          [% ELSE %]
            <em>never</em>
          [% END %]
@@ -117,10 +117,9 @@

  </div>

-  [% IF jobset.errormsg || jobset.fetcherrormsg %]
+  [% IF jobset.has_error || jobset.fetcherrormsg %]
    <div id="tabs-errors" class="tab-pane">
-      <p>Errors occurred at [% INCLUDE renderDateTime timestamp=(jobset.errortime || jobset.lastcheckedtime) %].</p>
-      <div class="card bg-light"><div class="card-body"><pre>[% HTML.escape(jobset.fetcherrormsg || jobset.errormsg) %]</pre></div></div>
+      <iframe src="[% c.uri_for('/jobset' project.name jobset.name "errors") %]" loading="lazy" frameBorder="0" width="100%"></iframe>
    </div>
  [% END %]

@@ -205,6 +204,7 @@
        if (!c) return;
        requestJSON({
          url: "[% HTML.escape(c.uri_for('/api/push', { jobsets = project.name _ ':' _ jobset.name, force = "1" })) %]",
+          type: 'POST',
          success: function(data) {
            bootbox.alert("The jobset has been scheduled for evaluation.");
          }
--- a/src/root/layout.tt
+++ b/src/root/layout.tt
@@ -10,31 +10,7 @@

    <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
    <meta http-equiv="X-UA-Compatible" content="IE=Edge" />
-
-    <script type="text/javascript" src="[% c.uri_for("/static/js/jquery/jquery-3.4.1.min.js") %]"></script>
-    <script type="text/javascript" src="[% c.uri_for("/static/js/jquery/jquery-ui-1.10.4.min.js") %]"></script>
-    <script type="text/javascript" src="[% c.uri_for("/static/js/moment/moment-2.24.0.min.js") %]"></script>
-
-    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-
-    <link href="[% c.uri_for("/static/fontawesome/css/all.css") %]" rel="stylesheet" />
-    <script type="text/javascript" src="[% c.uri_for("/static/js/popper.min.js") %]"></script>
-    <script type="text/javascript" src="[% c.uri_for("/static/bootstrap/js/bootstrap.min.js") %]"></script>
-    <link href="[% c.uri_for("/static/bootstrap/css/bootstrap.min.css") %]" rel="stylesheet" />
-
-    <!-- hydra.css may need to be moved to before boostrap to make the @media rule work. -->
-    <link rel="stylesheet" href="[% c.uri_for("/static/css/hydra.css") %]" type="text/css" />
-    <link rel="stylesheet" href="[% c.uri_for("/static/css/rotated-th.css") %]" type="text/css" />
-
-    <style>
-      .popover { max-width: 40%; }
-    </style>
-
-    <script type="text/javascript" src="[% c.uri_for("/static/js/bootbox.min.js") %]"></script>
-
-    <link rel="stylesheet" href="[% c.uri_for("/static/css/tree.css") %]" type="text/css" />
-
-    <script type="text/javascript" src="[% c.uri_for("/static/js/common.js") %]"></script>
+    [% INCLUDE style.tt %]

    [% IF c.config.enable_google_login %]
      <meta name="google-signin-client_id" content="[% c.config.google_client_id %]">
--- a/src/root/machine-status.tt
+++ b/src/root/machine-status.tt
@@ -6,10 +6,10 @@
  <thead>
    <tr>
      <th>Job</th>
-      <th>System</th>
      <th>Build</th>
      <th>Step</th>
      <th>What</th>
+      <th>Status</th>
      <th>Since</th>
    </tr>
  </thead>
@@ -40,10 +40,10 @@
          [% idle = 0 %]
          <tr>
            <td><tt>[% INCLUDE renderFullJobName project=step.project jobset=step.jobset job=step.job %]</tt></td>
-            <td><tt>[% step.system %]</tt></td>
            <td><a href="[% c.uri_for('/build' step.build) %]">[% step.build %]</a></td>
            <td>[% IF step.busy >= 30 %]<a class="row-link" href="[% c.uri_for('/build' step.build 'nixlog' step.stepnr 'tail') %]">[% step.stepnr %]</a>[% ELSE; step.stepnr; END %]</td>
            <td><tt>[% step.drvpath.match('-(.*)').0 %]</tt></td>
+            <td>[% INCLUDE renderBusyStatus %]</td>
            <td style="width: 10em">[% INCLUDE renderDuration duration = curTime - step.starttime %] </td>
          </tr>
        [% END %]
--- a/src/root/static/js/common.js
+++ b/src/root/static/js/common.js
@@ -129,6 +129,12 @@ $(document).ready(function() {
            el.addClass("is-local");
        }
    });
+
+    [...document.getElementsByTagName("iframe")].forEach((element) => {
+        element.contentWindow.addEventListener("DOMContentLoaded", (_) => {
+            element.style.height = element.contentWindow.document.body.scrollHeight + 'px';
+        })
+    })
 });

 var tabsLoaded = {};
--- a/src/root/status.tt
+++ b/src/root/status.tt
@@ -7,7 +7,7 @@

 [% ELSE %]

-  [% INCLUDE renderBuildList builds=resource showSchedulingInfo=1 hideResultInfo=1 busy=1 %]
+  [% INCLUDE renderBuildList builds=resource showSchedulingInfo=1 hideResultInfo=1 busy=1 showStepName=1 %]

 [% END %]

--- a/src/root/style.tt
+++ b/src/root/style.tt
@@ -0,0 +1,24 @@
+<script type="text/javascript" src="[% c.uri_for("/static/js/jquery/jquery-3.4.1.min.js") %]"></script>
+<script type="text/javascript" src="[% c.uri_for("/static/js/jquery/jquery-ui-1.10.4.min.js") %]"></script>
+<script type="text/javascript" src="[% c.uri_for("/static/js/moment/moment-2.24.0.min.js") %]"></script>
+
+<meta name="viewport" content="width=device-width, initial-scale=1.0" />
+
+<link href="[% c.uri_for("/static/fontawesome/css/all.css") %]" rel="stylesheet" />
+<script type="text/javascript" src="[% c.uri_for("/static/js/popper.min.js") %]"></script>
+<script type="text/javascript" src="[% c.uri_for("/static/bootstrap/js/bootstrap.min.js") %]"></script>
+<link href="[% c.uri_for("/static/bootstrap/css/bootstrap.min.css") %]" rel="stylesheet" />
+
+<!-- hydra.css may need to be moved to before boostrap to make the @media rule work. -->
+<link rel="stylesheet" href="[% c.uri_for("/static/css/hydra.css") %]" type="text/css" />
+<link rel="stylesheet" href="[% c.uri_for("/static/css/rotated-th.css") %]" type="text/css" />
+
+<style>
+  .popover { max-width: 40%; }
+</style>
+
+<script type="text/javascript" src="[% c.uri_for("/static/js/bootbox.min.js") %]"></script>
+
+<link rel="stylesheet" href="[% c.uri_for("/static/css/tree.css") %]" type="text/css" />
+
+<script type="text/javascript" src="[% c.uri_for("/static/js/common.js") %]"></script>
--- a/src/root/user.tt
+++ b/src/root/user.tt
@@ -91,6 +91,7 @@
          [% INCLUDE roleoption mutable=mutable role="restart-jobs" %]
          [% INCLUDE roleoption mutable=mutable role="bump-to-front" %]
          [% INCLUDE roleoption mutable=mutable role="cancel-build" %]
+          [% INCLUDE roleoption mutable=mutable role="eval-jobset" %]
        </p>
      </div>
    </div>
--- a/t/Hydra/Config/ldap_role_map.t
+++ b/t/Hydra/Config/ldap_role_map.t
@@ -57,6 +57,7 @@ subtest "getLDAPConfig" => sub {
                            "hydra_cancel-build" => [ "cancel-build" ],
                            "hydra_create-projects" => [ "create-projects" ],
                            "hydra_restart-jobs" => [ "restart-jobs" ],
+                            "hydra_eval-jobset" => [ "eval-jobset" ],
                        }
                    },
                    "The empty file and set env var make legacy mode active."
@@ -177,6 +178,7 @@ subtest "get_legacy_ldap_config" => sub {
                "hydra_cancel-build" => [ "cancel-build" ],
                "hydra_create-projects" => [ "create-projects" ],
                "hydra_restart-jobs" => [ "restart-jobs" ],
+                "hydra_eval-jobset" => [ "eval-jobset" ],
            }
        },
        "Legacy, default role maps are applied."
--- a/t/Hydra/Controller/API/checks.t
+++ b/t/Hydra/Controller/API/checks.t
@@ -22,9 +22,24 @@ sub is_json {
 }

 my $ctx = test_context();
-
 Catalyst::Test->import('Hydra');

+# Create a user to log in to
+my $user = $ctx->db->resultset('Users')->create({ username => 'alice', emailaddress => 'alice@example.com', password => '!' });
+$user->setPassword('foobar');
+$user->userroles->update_or_create({ role => 'admin' });
+
+# Login and save cookie for future requests
+my $req = request(POST '/login',
+    Referer => 'http://localhost/',
+    Content => {
+        username => 'alice',
+        password => 'foobar'
+    }
+);
+is($req->code, 302, "The login redirects");
+my $cookie = $req->header("set-cookie");
+
 my $finishedBuilds = $ctx->makeAndEvaluateJobset(
    expression => "one-job.nix",
    build => 1
@@ -109,7 +124,10 @@ subtest "/api/push" => sub {
        my $jobsetName = $jobset->name;
        is($jobset->forceeval, undef, "The existing jobset is not set to be forced to eval");

-        my $response = request(GET "/api/push?jobsets=$projectName:$jobsetName&force=1");
+        my $response = request(POST "/api/push?jobsets=$projectName:$jobsetName&force=1",
+            Cookie => $cookie,
+            Referer => 'http://localhost/',
+        );
        ok($response->is_success, "The API enpdoint for triggering jobsets returns 200.");

        my $data = is_json($response);
@@ -128,7 +146,10 @@ subtest "/api/push" => sub {

        print STDERR $repo;

-        my $response = request(GET "/api/push?repos=$repo&force=1");
+        my $response = request(POST "/api/push?repos=$repo&force=1",
+            Cookie => $cookie,
+            Referer => 'http://localhost/',
+        );
        ok($response->is_success, "The API enpdoint for triggering jobsets returns 200.");

        my $data = is_json($response);
--- a/t/Hydra/Controller/Jobset/evals.t
+++ b/t/Hydra/Controller/Jobset/evals.t
@@ -32,4 +32,9 @@ subtest "/jobset/PROJECT/JOBSET/evals" => sub {
    ok($jobsetevals->is_success, "The page showing the jobset evals returns 200.");
 };

+subtest "/jobset/PROJECT/JOBSET/errors" => sub {
+    my $jobsetevals = request(GET '/jobset/' . $project->name . '/' . $jobset->name . '/errors');
+    ok($jobsetevals->is_success, "The page showing the jobset eval errors returns 200.");
+};
+
 done_testing;
--- a/t/Hydra/Controller/JobsetEval/fetch.t
+++ b/t/Hydra/Controller/JobsetEval/fetch.t
@@ -35,6 +35,10 @@ subtest "Fetching the eval's overview" => sub {
    is($fetch->code, 200, "channel page is 200");
 };

+subtest "Fetching the eval's overview" => sub {
+    my $fetch = request(GET '/eval/' . $eval->id, '/errors');
+    is($fetch->code, 200, "errors page is 200");
+};


 done_testing;
--- a/t/Hydra/Controller/User/ldap-legacy.t
+++ b/t/Hydra/Controller/User/ldap-legacy.t
@@ -24,6 +24,7 @@ $ldap->add_group("hydra_create-projects", $users->{"many_roles"}->{"username"});
 $ldap->add_group("hydra_restart-jobs", $users->{"many_roles"}->{"username"});
 $ldap->add_group("hydra_bump-to-front", $users->{"many_roles"}->{"username"});
 $ldap->add_group("hydra_cancel-build", $users->{"many_roles"}->{"username"});
+$ldap->add_group("hydra_eval-jobset", $users->{"many_roles"}->{"username"});

 my $hydra_ldap_config = "${\$ldap->tmpdir()}/hydra_ldap_config.yaml";
 LDAPContext::write_file($hydra_ldap_config, <<YAML);
@@ -68,7 +69,7 @@ subtest "Valid login attempts" => sub {
        unrelated => [],
         admin => ["admin"],
         not_admin => [],
-         many_roles => [ "create-projects", "restart-jobs", "bump-to-front", "cancel-build" ],
+         many_roles => [ "create-projects", "restart-jobs", "bump-to-front", "cancel-build", "eval-jobset" ],
    );
    for my $username (keys %users_to_roles) {
        my $user = $users->{$username};
--- a/t/Hydra/Controller/User/ldap.t
+++ b/t/Hydra/Controller/User/ldap.t
@@ -24,6 +24,7 @@ $ldap->add_group("hydra_create-projects", $users->{"many_roles"}->{"username"});
 $ldap->add_group("hydra_restart-jobs", $users->{"many_roles"}->{"username"});
 $ldap->add_group("hydra_bump-to-front", $users->{"many_roles"}->{"username"});
 $ldap->add_group("hydra_cancel-build", $users->{"many_roles"}->{"username"});
+$ldap->add_group("hydra_eval-jobset", $users->{"many_roles"}->{"username"});


 my $ctx = test_context(
@@ -76,10 +77,12 @@ my $ctx = test_context(
                hydra_cancel-build = cancel-build
                hydra_bump-to-front = bump-to-front
                hydra_restart-jobs = restart-jobs
+                hydra_eval-jobset = eval-jobset

                hydra_one_group_many_roles = create-projects
                hydra_one_group_many_roles = cancel-build
                hydra_one_group_many_roles = bump-to-front
+                hydra_one_group_many-roles = eval-jobset
            </role_mapping>
        </ldap>
 CFG
@@ -92,7 +95,7 @@ subtest "Valid login attempts" => sub {
        unrelated => [],
        admin => ["admin"],
        not_admin => [],
-        many_roles => [ "create-projects", "restart-jobs", "bump-to-front", "cancel-build" ],
+        many_roles => [ "create-projects", "restart-jobs", "bump-to-front", "cancel-build", "eval-jobset" ],
        many_roles_one_group => [ "create-projects", "bump-to-front", "cancel-build" ],
    );
    for my $username (keys %users_to_roles) {
--- a/t/evaluator/evaluate-constituents-broken.t
+++ b/t/evaluator/evaluate-constituents-broken.t
@@ -22,7 +22,7 @@ like(
    "The stderr record includes a relevant error message"
 );

-$jobset->discard_changes;  # refresh from DB
+$jobset->discard_changes({ '+columns' => {'errormsg' => 'errormsg'} });  # refresh from DB
 like(
    $jobset->errormsg,
    qr/aggregate job ‘mixed_aggregate’ failed with the error: constituentA: does not exist/,
--- a/t/lib/CliRunners.pm
+++ b/t/lib/CliRunners.pm
@@ -14,7 +14,7 @@ our @EXPORT = qw(
 sub evalSucceeds {
    my ($jobset) = @_;
    my ($res, $stdout, $stderr) = captureStdoutStderr(60, ("hydra-eval-jobset", $jobset->project->name, $jobset->name));
-    $jobset->discard_changes;  # refresh from DB
+    $jobset->discard_changes({ '+columns' => {'errormsg' => 'errormsg'} });  # refresh from DB
    if ($res) {
        chomp $stdout; chomp $stderr;
        utf8::decode($stdout) or die "Invalid unicode in stdout.";
@@ -29,7 +29,7 @@ sub evalSucceeds {
 sub evalFails {
    my ($jobset) = @_;
    my ($res, $stdout, $stderr) = captureStdoutStderr(60, ("hydra-eval-jobset", $jobset->project->name, $jobset->name));
-    $jobset->discard_changes;  # refresh from DB
+    $jobset->discard_changes({ '+columns' => {'errormsg' => 'errormsg'} });  # refresh from DB
    if (!$res) {
        chomp $stdout; chomp $stderr;
        utf8::decode($stdout) or die "Invalid unicode in stdout.";
--- a/t/queue-runner/direct-indirect-constituents.t
+++ b/t/queue-runner/direct-indirect-constituents.t
@@ -13,7 +13,7 @@ my $constituentBuildA = $builds->{"constituentA"};
 my $constituentBuildB = $builds->{"constituentB"};

 my $eval = $constituentBuildA->jobsetevals->first();
-is($eval->evaluationerror->errormsg, "");
+is($eval->evaluationerror->has_error, 0);

 subtest "Verifying the direct aggregate" => sub {
    my $aggBuild = $builds->{"direct_aggregate"};
Author	SHA1	Message	Date
Martin Weinelt	bc6f08d233	Create eval-jobset role and guard /api/push route (cherry picked from commit `f730433789`)	2024-08-27 21:53:17 +02:00
Janne Heß	88d9898588	api: Require POST for /api/push (cherry picked from commit `916531dc9c`)	2024-08-27 21:53:13 +02:00
Pierre Bourdon	b8d03adaf4	queue runner: attempt at slightly smarter scheduling criteria Instead of just going for "whatever is the oldest build we know of", use the following first: - Is the step more constrained? If so, schedule it first to avoid filling up "more desirable" build slots with less constrained builds. - Does the step have more dependents? If so, schedule it first to try and maximize open parallelism and breadth of scheduling options.	2024-04-21 17:36:16 +02:00
Pierre Bourdon	ee1a7a7813	web: serveFile: also serve a CSP putting served HTML in its own origin	2024-04-21 16:14:24 +02:00
Pierre Bourdon	5c3e508e55	queue-runner: release machine reservation while copying outputs This allows for better builder usage when the queue runner is busy. To avoid running into uncontrollable imbalances between builder/queue runner, we only release the machine reservation after the local throttler has found a slot to start copying the outputs for that build.	2024-04-21 01:55:19 +02:00
Pierre Bourdon	026e3a3103	queue-runner: switch to pseudorandom ordering of builds processing We don't rely on sequential / monotonic build IDs processing anymore, so randomizing actually has the advantage of mixing builds for different systems together, to avoid only one chunk of builds for a single system getting processed while builders for other systems are starved.	2024-04-20 23:05:26 +02:00
Pierre Bourdon	6606a7f86e	queue runner: introduce some parallelism for remote paths lookup Each output for a given step being ingested is looked up in parallel, which should basically multiply the speed of builds ingestion by the average number of outputs per derivation.	2024-04-20 22:28:18 +02:00
Pierre Bourdon	f31b95d371	queue-runner: reduce the time between queue monitor restarts This will induce more DB queries (though these are fairly cheap), but at the benefit of processing bumps within 1m instead of within 10m.	2024-04-20 16:58:10 +02:00
Pierre Bourdon	54f8daf6b1	queue-runner: remove id > X from new builds query Running the query with/without it shows that it makes no difference to postgres, since there's an index on finished=0 already. This allows a few simplifications, but also paves the way towards running multiple parallel monitor threads in the future.	2024-04-20 16:53:52 +02:00
Pierre Bourdon	cc6bafe538	queue-runner: add prom metrics to allow detecting internal bottlenecks By looking at the ratio of running vs. waiting for the dispatcher and the queue monitor, we should get better visibility into what hydra is currently bottlenecked on. There are other side effects we can try to measure to get to the same result, but having a simple way doesn't cost us much.	2024-04-20 16:48:03 +02:00
Pierre Bourdon	6189ba9c5e	web: replace 'errormsg' with 'errormsg IS NULL' in most cases This is implement in an extremely hacky way due to poor DBIx feature support. Ideally, what we'd need is a way to tell DBIx to ignore the errormsg column unless explicitly requested, and to automatically add a computed 'errormsg IS NULL' column in others. Since it does not support that, this commit instead hacks some support via method overrides while taking care to not break anything obvious.	2024-04-12 20:14:09 +02:00
Pierre Bourdon	258e9314a9	web: include current step status on /machines	2024-04-11 17:15:58 +02:00
Pierre Bourdon	a51bd392a2	queue-runner: limit parallelism of CPU intensive operations My current theory is that running more parallel xz than available CPU cores is reducing our overall throughput by requiring more scheduling overhead and more cache thrashing.	2024-04-11 16:43:01 +02:00
Maximilian Bosch	a596d6c3c1	Only show stepname if it doesn't equal the name of the drv When building e.g. nixpkgs, the "Running builds" view will mostly look like this hello.x86_64-linux (Build of hello-X.Y) exa.x86_64-linux (Build of exa-X.Y) ... This doesn't provide any useful information. Showing the step name only makes sense if it's not a child of the job's derivation. With this patch, that information will only be shown if the drv name (i.e. w/o `/nix/store/` prefix, .drv ext & hash) is not equal to the drv name of the job itself (build.nixname).	2024-03-18 18:46:01 +01:00
Maximilian Bosch	415f9f2daa	Running builds view: show build step names When using Hydra to build machine configurations, you'll often see "nixosConfigurations.foo" five times, i.e. for each build step being run. This isn't very helpful I think because in such a case, a single build step can also be compiling the Linux kernel. This change also fetches the `drvpath` and `type` from the `buildsteps` relation. We're already joining it, so this doesn't make much difference (confirmed via query logging that this doesn't cause extra SQL queries). Unfortunately build steps don't have a human readable name, so I'm deriving it from the drvpath by stripping away the hash (assuming that it'll never contain a `-` and that `/nix/store/` is used as prefix). I decided against using the Nix bindings for that to avoid too much overhead due to store operations for each build step.	2024-03-18 18:46:01 +01:00
Maximilian Bosch	9b465e7a67	Make "timed out" and "log limit exceeded" builds aborted In `73694087a0` I gave builds that failed because of a timeout or exceeded log limit a stop sign and I stand by that reasoning: with that it's possible to distinguish between actual build failures and rather transient things such as timeouts. Back then I considered it a feature that these are shown in a different tab, but I don't think that's a good idea anymore. When using a jobset to e.g. track the regressions from a mass rebuild (like a compiler or gcc update), "Newly failed builds" should exclusively display regressions (and flaky builds of course, not much I can do about that). Also, when a bunch of builds fail in such a jobset because of e.g. a broken connection to a builder that results in a timeout, I want to be able to restart them all w/o rebuilding actual regressions. To make it clear that we not only have "Aborted" builds in the tab, I renamed the label to "Aborted / Timed out".	2024-03-16 22:10:40 +01:00
Maximilian Bosch	9b62c52e5c	hydra-queue-runner: drop broken connections from pool Closes #1336 When restarting postgresql, the connections are still reused in `hydra-queue-runner` causing errors like this main thread: Lost connection to the database server. queue monitor: Lost connection to the database server. and no more builds being processed. `hydra-evaluator` doesn't have that issue since it crashes right away. We could let it retry indefinitely as well (see below), but I don't want to change too much. If the DB is still unreachable 10s later, the process will stop with a non-zero exit code because of a missing DB connection. This however isn't such a big deal because it will be immediately restarted afterwards. With the current configuration, Hydra will never give up, but restart (and retry) infinitely. To me that seems reasonable, i.e. to retry DB connections on a long-running process. If this doesn't work out, the monitoring should fire anyways because the queue fills up, but I'm open to discuss that. Please note that this isn't reproducible with the DB and the queue runner on the same machine when using `services.hydra-dev`, because of the `Requires=` dependency `hydra-queue-runner.service` -> `hydra-init.service` -> `postgresql.service` that causes the queue runner to be restarted on `systemctl restart postgresql`. Internally, Hydra uses Nix's pool data structure: it basically has N slots (here DB connections) and whenever a new one is requested, an idle slot is provided or a new one is created (when N slots are active, it'll be waited until one slot is free). The issue in the code here is however that whenever an error is encountered, the slot is released, however the same broken connection will be reused the next time. By using `Pool::Handle::markBad`, Nix will drop a broken slot. This is now being done when `pqxx::broken_connection` was caught.	2024-03-16 22:10:40 +01:00
Maximilian Bosch	ef6be80f54	Use `submit` event in login form It's a pet peeve from me when logging into my personal Hydra that I always have to press the button rather than hitting Return after entering my password. Reason for that is that the form doesn't have a "submit" button, so far it was always listened to the "click" event. Submit does that and you can hit Return alternatively.	2024-03-16 22:10:40 +01:00
K900	969eb3eeac	urlencode drv names when fetching logs Otherwise names with special characters like + break things.	2024-03-16 22:10:40 +01:00
Pierre Bourdon	18466e8326	queue-runner: try larger pipe buffer sizes	2024-03-16 22:10:40 +01:00
ajs124	6ed21490ee	lazy-load evaluation errors Closes #1362	2024-03-16 22:10:40 +01:00