Merge branch 'nix-2.20' into nix-2.21

Nix expects a base64 encoded hostkey in SSHMaster, so make sure we don't
decode this prematurely in hydra.

Reported-By: Puck Meerburg <puck@puck.moe>

flake.lock

@@ -42,48 +42,32 @@
         "nixpkgs-regression": "nixpkgs-regression"
       },
       "locked": {
-        "lastModified": 1715805674,
-        "narHash": "sha256-0CIBMECsA3ISJZrJcOTzi6wa3QENTKGLtOpYIOoxwxo=",
+        "lastModified": 1715845907,
+        "narHash": "sha256-1OigUcZGDInTVZJBTioo9vwRt70yvcfAkSRUeAD/mfg=",
         "owner": "NixOS",
         "repo": "nix",
-        "rev": "ab48ea416a203e9ccefb70aa634e27477e4c1ac4",
+        "rev": "1ebc34e9c54b740ea4f4466443047d709dccf5b2",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
-        "ref": "2.20-maintenance",
+        "ref": "2.21-maintenance",
         "repo": "nix",
         "type": "github"
       }
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1705033721,
-        "narHash": "sha256-K5eJHmL1/kev6WuqyqqbS1cdNnSidIZ3jeqJ7GbrYnQ=",
+        "lastModified": 1712848736,
+        "narHash": "sha256-CzZwhqyLlebljv1zFS2KWVH/3byHND0LfaO1jKsGuVo=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "a1982c92d8980a0114372973cbdfe0a307f1bdea",
+        "rev": "1d6a23f11e44d0fb64b3237569b87658a9eb5643",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
-        "ref": "nixos-23.05-small",
-        "repo": "nixpkgs",
-        "type": "github"
-      }
-    },
-    "nixpkgs-for-fileset": {
-      "locked": {
-        "lastModified": 1706098335,
-        "narHash": "sha256-r3dWjT8P9/Ah5m5ul4WqIWD8muj5F+/gbCdjiNVBKmU=",
-        "owner": "NixOS",
-        "repo": "nixpkgs",
-        "rev": "a77ab169a83a4175169d78684ddd2e54486ac651",
-        "type": "github"
-      },
-      "original": {
-        "owner": "NixOS",
-        "ref": "nixos-23.11",
+        "ref": "nixos-23.11-small",
         "repo": "nixpkgs",
         "type": "github"
       }
@@ -107,8 +91,7 @@
     "root": {
       "inputs": {
         "nix": "nix",
-        "nixpkgs": "nixpkgs",
-        "nixpkgs-for-fileset": "nixpkgs-for-fileset"
+        "nixpkgs": "nixpkgs"
       }
     }
   },

flake.nix

@@ -1,16 +1,11 @@
 {
   description = "A Nix-based continuous build system";
 
-  inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.05-small";
-  inputs.nix.url = "github:NixOS/nix/2.20-maintenance";
+  inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.11-small";
+  inputs.nix.url = "github:NixOS/nix/2.21-maintenance";
   inputs.nix.inputs.nixpkgs.follows = "nixpkgs";
 
-  # TODO get rid of this once https://github.com/NixOS/nix/pull/9546 is
-  # mered and we upgrade or Nix, so the main `nixpkgs` input is at least
-  # 23.11 and has `lib.fileset`.
-  inputs.nixpkgs-for-fileset.url = "github:NixOS/nixpkgs/nixos-23.11";
-
-  outputs = { self, nixpkgs, nix, nixpkgs-for-fileset }:
+  outputs = { self, nixpkgs, nix }:
     let
       systems = [ "x86_64-linux" "aarch64-linux" ];
       forEachSystem = nixpkgs.lib.genAttrs systems;
@@ -67,7 +62,7 @@
         };
 
         hydra = final.callPackage ./package.nix {
-          inherit (nixpkgs-for-fileset.lib) fileset;
+          inherit (nixpkgs.lib) fileset;
           rawSrc = self;
         };
       };

package.nix

@@ -87,7 +87,6 @@ let
         DateTime
         DBDPg
         DBDSQLite
-        DBIxClassHelpers
         DigestSHA1
         EmailMIME
         EmailSender

src/hydra-eval-jobs/hydra-eval-jobs.cc

@@ -185,7 +185,7 @@ static void worker(
                     !experimentalFeatureSettings.isEnabled(Xp::CaDerivations));
 
                 if (drv->querySystem() == "unknown")
-                    throw EvalError("derivation must have a 'system' attribute");
+                    state.error<EvalError>("derivation must have a 'system' attribute").debugThrow();
 
                 auto drvPath = state.store->printStorePath(drv->requireDrvPath());
 
@@ -208,7 +208,7 @@ static void worker(
                 if (a && state.forceBool(*a->value, a->pos, "while evaluating the `_hydraAggregate` attribute")) {
                     auto a = v->attrs->get(state.symbols.create("constituents"));
                     if (!a)
-                        throw EvalError("derivation must have a ‘constituents’ attribute");
+                        state.error<EvalError>("derivation must have a ‘constituents’ attribute").debugThrow();
 
                     NixStringContext context;
                     state.coerceToString(a->pos, *a->value, context, "while evaluating the `constituents` attribute", true, false);
@@ -274,7 +274,7 @@ static void worker(
             else if (v->type() == nNull)
                 ;
 
-            else throw TypeError("attribute '%s' is %s, which is not supported", attrPath, showType(*v));
+            else state.error<TypeError>("attribute '%s' is %s, which is not supported", attrPath, showType(*v)).debugThrow();
 
         } catch (EvalError & e) {
             auto msg = e.msg();

src/hydra-evaluator/hydra-evaluator.cc

@@ -38,7 +38,7 @@ class JobsetId {
     friend bool operator!= (const JobsetId & lhs, const JobsetName & rhs);
 
     std::string display() const {
-        return str(format("%1%:%2% (jobset#%3%)") % project % jobset % id);
+        return boost::str(boost::format("%1%:%2% (jobset#%3%)") % project % jobset % id);
     }
 };
 bool operator==(const JobsetId & lhs, const JobsetId & rhs)

src/hydra-queue-runner/build-remote.cc

@@ -54,20 +54,9 @@ static std::unique_ptr<SSHMaster::Connection> openConnection(
         command.splice(command.end(), extraStoreArgs(machine->sshName));
     }
 
-    auto ret = master.startCommand(std::move(command), {
+    return master.startCommand(std::move(command), {
         "-a", "-oBatchMode=yes", "-oConnectTimeout=60", "-oTCPKeepAlive=yes"
     });
-
-    // XXX: determine the actual max value we can use from /proc.
-
-    // FIXME: Should this be upstreamed into `startCommand` in Nix?
-
-    int pipesize = 1024 * 1024;
-
-    fcntl(ret->in.get(), F_SETPIPE_SZ, &pipesize);
-    fcntl(ret->out.get(), F_SETPIPE_SZ, &pipesize);
-
-    return ret;
 }
 
 
@@ -412,19 +401,8 @@ void RemoteResult::updateWithBuildResult(const nix::BuildResult & buildResult)
 
 }
 
-/* Utility guard object to auto-release a semaphore on destruction. */
-template <typename T>
-class SemaphoreReleaser {
-public:
-    SemaphoreReleaser(T* s) : sem(s) {}
-    ~SemaphoreReleaser() { sem->release(); }
-
-private:
-    T* sem;
-};
 
 void State::buildRemote(ref<Store> destStore,
-    MachineReservation::ptr & reservation,
     ::Machine::ptr machine, Step::ptr step,
     const ServeProto::BuildOptions & buildOptions,
     RemoteResult & result, std::shared_ptr<ActiveStep> activeStep,
@@ -562,23 +540,6 @@ void State::buildRemote(ref<Store> destStore,
             result.logFile = "";
         }
 
-        /* Throttle CPU-bound work. Opportunistically skip updating the current
-         * step, since this requires a DB roundtrip. */
-        if (!localWorkThrottler.try_acquire()) {
-            updateStep(ssWaitingForLocalSlot);
-            localWorkThrottler.acquire();
-        }
-        SemaphoreReleaser releaser(&localWorkThrottler);
-
-        /* Once we've started copying outputs, release the machine reservation
-         * so further builds can happen. We do not release the machine earlier
-         * to avoid situations where the queue runner is bottlenecked on
-         * copying outputs and we end up building too many things that we
-         * haven't been able to allow copy slots for. */
-        assert(reservation.unique());
-        reservation = 0;
-        wakeDispatcher();
-
         StorePathSet outputs;
         for (auto & [_, realisation] : buildResult.builtOutputs)
             outputs.insert(realisation.outPath);

src/hydra-queue-runner/builder.cc

@@ -46,12 +46,10 @@ void State::builder(MachineReservation::ptr reservation)
         }
     }
 
-    /* If the machine hasn't been released yet, release and wake up the dispatcher. */
-    if (reservation) {
-        assert(reservation.unique());
-        reservation = 0;
-        wakeDispatcher();
-    }
+    /* Release the machine and wake up the dispatcher. */
+    assert(reservation.unique());
+    reservation = 0;
+    wakeDispatcher();
 
     /* If there was a temporary failure, retry the step after an
        exponentially increasing interval. */
@@ -74,11 +72,11 @@ void State::builder(MachineReservation::ptr reservation)
 
 
 State::StepResult State::doBuildStep(nix::ref<Store> destStore,
-    MachineReservation::ptr & reservation,
+    MachineReservation::ptr reservation,
     std::shared_ptr<ActiveStep> activeStep)
 {
-    auto step(reservation->step);
-    auto machine(reservation->machine);
+    auto & step(reservation->step);
+    auto & machine(reservation->machine);
 
     {
         auto step_(step->state.lock());
@@ -213,7 +211,7 @@ State::StepResult State::doBuildStep(nix::ref<Store> destStore,
 
         try {
             /* FIXME: referring builds may have conflicting timeouts. */
-            buildRemote(destStore, reservation, machine, step, buildOptions, result, activeStep, updateStep, narMembers);
+            buildRemote(destStore, machine, step, buildOptions, result, activeStep, updateStep, narMembers);
         } catch (Error & e) {
             if (activeStep->state_.lock()->cancelled) {
                 printInfo("marking step %d of build %d as cancelled", stepNr, buildId);

src/hydra-queue-runner/dispatcher.cc

@@ -39,15 +39,13 @@ void State::dispatcher()
             printMsg(lvlDebug, "dispatcher woken up");
             nrDispatcherWakeups++;
 
-            auto t_before_work = std::chrono::steady_clock::now();
+            auto now1 = std::chrono::steady_clock::now();
 
             auto sleepUntil = doDispatch();
 
-            auto t_after_work = std::chrono::steady_clock::now();
+            auto now2 = std::chrono::steady_clock::now();
 
-            prom.dispatcher_time_spent_running.Increment(
-                std::chrono::duration_cast<std::chrono::microseconds>(t_after_work - t_before_work).count());
-            dispatchTimeMs += std::chrono::duration_cast<std::chrono::milliseconds>(t_after_work - t_before_work).count();
+            dispatchTimeMs += std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count();
 
             /* Sleep until we're woken up (either because a runnable build
                is added, or because a build finishes). */
@@ -61,10 +59,6 @@ void State::dispatcher()
                 *dispatcherWakeup_ = false;
             }
 
-            auto t_after_sleep = std::chrono::steady_clock::now();
-            prom.dispatcher_time_spent_waiting.Increment(
-                std::chrono::duration_cast<std::chrono::microseconds>(t_after_sleep - t_after_work).count());
-
         } catch (std::exception & e) {
             printError("dispatcher: %s", e.what());
             sleep(1);
@@ -133,8 +127,6 @@ system_time State::doDispatch()
            comparator is a partial ordering (see MachineInfo). */
         int highestGlobalPriority;
         int highestLocalPriority;
-        size_t numRequiredSystemFeatures;
-        size_t numRevDeps;
         BuildID lowestBuildID;
 
         StepInfo(Step::ptr step, Step::State & step_) : step(step)
@@ -143,8 +135,6 @@ system_time State::doDispatch()
                 lowestShareUsed = std::min(lowestShareUsed, jobset->shareUsed());
             highestGlobalPriority = step_.highestGlobalPriority;
             highestLocalPriority = step_.highestLocalPriority;
-            numRequiredSystemFeatures = step->requiredSystemFeatures.size();
-            numRevDeps = step_.rdeps.size();
             lowestBuildID = step_.lowestBuildID;
         }
     };
@@ -197,8 +187,6 @@ system_time State::doDispatch()
                 a.highestGlobalPriority != b.highestGlobalPriority ? a.highestGlobalPriority > b.highestGlobalPriority :
                 a.lowestShareUsed != b.lowestShareUsed ? a.lowestShareUsed < b.lowestShareUsed :
                 a.highestLocalPriority != b.highestLocalPriority ? a.highestLocalPriority > b.highestLocalPriority :
-                a.numRequiredSystemFeatures != b.numRequiredSystemFeatures ? a.numRequiredSystemFeatures > b.numRequiredSystemFeatures :
-                a.numRevDeps != b.numRevDeps ? a.numRevDeps > b.numRevDeps :
                 a.lowestBuildID < b.lowestBuildID;
         });
 

src/hydra-queue-runner/hydra-queue-runner.cc

@@ -70,31 +70,10 @@ State::PromMetrics::PromMetrics()
             .Register(*registry)
             .Add({})
     )
-    , dispatcher_time_spent_running(
-        prometheus::BuildCounter()
-            .Name("hydraqueuerunner_dispatcher_time_spent_running")
-            .Help("Time (in micros) spent running the dispatcher")
-            .Register(*registry)
-            .Add({})
-    )
-    , dispatcher_time_spent_waiting(
-        prometheus::BuildCounter()
-            .Name("hydraqueuerunner_dispatcher_time_spent_waiting")
-            .Help("Time (in micros) spent waiting for the dispatcher to obtain work")
-            .Register(*registry)
-            .Add({})
-    )
-    , queue_monitor_time_spent_running(
-        prometheus::BuildCounter()
-            .Name("hydraqueuerunner_queue_monitor_time_spent_running")
-            .Help("Time (in micros) spent running the queue monitor")
-            .Register(*registry)
-            .Add({})
-    )
-    , queue_monitor_time_spent_waiting(
-        prometheus::BuildCounter()
-            .Name("hydraqueuerunner_queue_monitor_time_spent_waiting")
-            .Help("Time (in micros) spent waiting for the queue monitor to obtain work")
+    , queue_max_id(
+        prometheus::BuildGauge()
+            .Name("hydraqueuerunner_queue_max_build_id_info")
+            .Help("Maximum build record ID in the queue")
             .Register(*registry)
             .Add({})
     )
@@ -106,7 +85,6 @@ State::State(std::optional<std::string> metricsAddrOpt)
     : config(std::make_unique<HydraConfig>())
     , maxUnsupportedTime(config->getIntOption("max_unsupported_time", 0))
     , dbPool(config->getIntOption("max_db_connections", 128))
-    , localWorkThrottler(config->getIntOption("max_local_worker_threads", std::min(maxSupportedLocalWorkers, std::max(4u, std::thread::hardware_concurrency()) - 2)))
     , maxOutputSize(config->getIntOption("max_output_size", 2ULL << 30))
     , maxLogSize(config->getIntOption("max_log_size", 64ULL << 20))
     , uploadLogsToBinaryCache(config->getBoolOption("upload_logs_to_binary_cache", false))
@@ -193,7 +171,7 @@ void State::parseMachines(const std::string & contents)
             std::move(mandatoryFeatures),
             // `sshPublicHostKey`
             tokens[7] != "" && tokens[7] != "-"
-                ? base64Decode(tokens[7])
+                ? tokens[7]
                 : "",
         });
 

src/hydra-queue-runner/queue-monitor.cc

@@ -1,7 +1,6 @@
 #include "state.hh"
 #include "hydra-build-result.hh"
 #include "globals.hh"
-#include "thread-pool.hh"
 
 #include <cstring>
 
@@ -38,21 +37,16 @@ void State::queueMonitorLoop(Connection & conn)
 
     auto destStore = getDestStore();
 
+    unsigned int lastBuildId = 0;
+
     bool quit = false;
     while (!quit) {
-        auto t_before_work = std::chrono::steady_clock::now();
-
         localStore->clearPathInfoCache();
 
-        bool done = getQueuedBuilds(conn, destStore);
+        bool done = getQueuedBuilds(conn, destStore, lastBuildId);
 
         if (buildOne && buildOneDone) quit = true;
 
-        auto t_after_work = std::chrono::steady_clock::now();
-
-        prom.queue_monitor_time_spent_running.Increment(
-            std::chrono::duration_cast<std::chrono::microseconds>(t_after_work - t_before_work).count());
-
         /* Sleep until we get notification from the database about an
            event. */
         if (done && !quit) {
@@ -62,10 +56,12 @@ void State::queueMonitorLoop(Connection & conn)
             conn.get_notifs();
 
         if (auto lowestId = buildsAdded.get()) {
+            lastBuildId = std::min(lastBuildId, static_cast<unsigned>(std::stoul(*lowestId) - 1));
             printMsg(lvlTalkative, "got notification: new builds added to the queue");
         }
         if (buildsRestarted.get()) {
             printMsg(lvlTalkative, "got notification: builds restarted");
+            lastBuildId = 0; // check all builds
         }
         if (buildsCancelled.get() || buildsDeleted.get() || buildsBumped.get()) {
             printMsg(lvlTalkative, "got notification: builds cancelled or bumped");
@@ -75,10 +71,6 @@ void State::queueMonitorLoop(Connection & conn)
             printMsg(lvlTalkative, "got notification: jobset shares changed");
             processJobsetSharesChange(conn);
         }
-
-        auto t_after_sleep = std::chrono::steady_clock::now();
-        prom.queue_monitor_time_spent_waiting.Increment(
-            std::chrono::duration_cast<std::chrono::microseconds>(t_after_sleep - t_after_work).count());
     }
 
     exit(0);
@@ -92,18 +84,20 @@ struct PreviousFailure : public std::exception {
 
 
 bool State::getQueuedBuilds(Connection & conn,
-    ref<Store> destStore)
+    ref<Store> destStore, unsigned int & lastBuildId)
 {
     prom.queue_checks_started.Increment();
 
-    printInfo("checking the queue for builds...");
+    printInfo("checking the queue for builds > %d...", lastBuildId);
 
     /* Grab the queued builds from the database, but don't process
        them yet (since we don't want a long-running transaction). */
     std::vector<BuildID> newIDs;
-    std::unordered_map<BuildID, Build::ptr> newBuildsByID;
+    std::map<BuildID, Build::ptr> newBuildsByID;
     std::multimap<StorePath, BuildID> newBuildsByPath;
 
+    unsigned int newLastBuildId = lastBuildId;
+
     {
         pqxx::work txn(conn);
 
@@ -112,12 +106,17 @@ bool State::getQueuedBuilds(Connection & conn,
              "jobsets.name as jobset, job, drvPath, maxsilent, timeout, timestamp, "
              "globalPriority, priority from Builds "
              "inner join jobsets on builds.jobset_id = jobsets.id "
-             "where finished = 0 order by globalPriority desc, random()");
+             "where builds.id > $1 and finished = 0 order by globalPriority desc, builds.id",
+            lastBuildId);
 
         for (auto const & row : res) {
             auto builds_(builds.lock());
             BuildID id = row["id"].as<BuildID>();
             if (buildOne && id != buildOne) continue;
+            if (id > newLastBuildId) {
+                newLastBuildId = id;
+                prom.queue_max_id.Set(id);
+            }
             if (builds_->count(id)) continue;
 
             auto build = std::make_shared<Build>(
@@ -299,7 +298,7 @@ bool State::getQueuedBuilds(Connection & conn,
         try {
             createBuild(build);
         } catch (Error & e) {
-            e.addTrace({}, hintfmt("while loading build %d: ", build->id));
+            e.addTrace({}, HintFmt("while loading build %d: ", build->id));
             throw;
         }
 
@@ -319,13 +318,15 @@ bool State::getQueuedBuilds(Connection & conn,
 
         /* Stop after a certain time to allow priority bumps to be
            processed. */
-        if (std::chrono::system_clock::now() > start + std::chrono::seconds(60)) {
+        if (std::chrono::system_clock::now() > start + std::chrono::seconds(600)) {
             prom.queue_checks_early_exits.Increment();
             break;
         }
     }
 
     prom.queue_checks_finished.Increment();
+
+    lastBuildId = newBuildsByID.empty() ? newLastBuildId : newBuildsByID.begin()->first - 1;
     return newBuildsByID.empty();
 }
 
@@ -404,34 +405,6 @@ void State::processQueueChange(Connection & conn)
 }
 
 
-std::map<DrvOutput, std::optional<StorePath>> State::getMissingRemotePaths(
-    ref<Store> destStore,
-    const std::map<DrvOutput, std::optional<StorePath>> & paths)
-{
-    Sync<std::map<DrvOutput, std::optional<StorePath>>> missing_;
-    ThreadPool tp;
-
-    for (auto & [output, maybeOutputPath] : paths) {
-        if (!maybeOutputPath) {
-            auto missing(missing_.lock());
-            missing->insert({output, maybeOutputPath});
-        } else {
-            tp.enqueue([&] {
-                if (!destStore->isValidPath(*maybeOutputPath)) {
-                    auto missing(missing_.lock());
-                    missing->insert({output, maybeOutputPath});
-                }
-            });
-        }
-    }
-
-    tp.process();
-
-    auto missing(missing_.lock());
-    return *missing;
-}
-
-
 Step::ptr State::createStep(ref<Store> destStore,
     Connection & conn, Build::ptr build, const StorePath & drvPath,
     Build::ptr referringBuild, Step::ptr referringStep, std::set<StorePath> & finishedDrvs,
@@ -512,15 +485,16 @@ Step::ptr State::createStep(ref<Store> destStore,
 
     /* Are all outputs valid? */
     auto outputHashes = staticOutputHashes(*localStore, *(step->drv));
-    std::map<DrvOutput, std::optional<StorePath>> paths;
+    bool valid = true;
+    std::map<DrvOutput, std::optional<StorePath>> missing;
     for (auto & [outputName, maybeOutputPath] : destStore->queryPartialDerivationOutputMap(drvPath, &*localStore)) {
         auto outputHash = outputHashes.at(outputName);
-        paths.insert({{outputHash, outputName}, maybeOutputPath});
+        if (maybeOutputPath && destStore->isValidPath(*maybeOutputPath))
+            continue;
+        valid = false;
+        missing.insert({{outputHash, outputName}, maybeOutputPath});
     }
 
-    auto missing = getMissingRemotePaths(destStore, paths);
-    bool valid = missing.empty();
-
     /* Try to copy the missing paths from the local store or from
        substitutes. */
     if (!missing.empty()) {

src/hydra-queue-runner/state.hh

@@ -7,7 +7,6 @@
 #include <memory>
 #include <queue>
 #include <regex>
-#include <semaphore>
 
 #include <prometheus/counter.h>
 #include <prometheus/gauge.h>
@@ -58,7 +57,6 @@ typedef enum {
     ssConnecting = 10,
     ssSendingInputs = 20,
     ssBuilding = 30,
-    ssWaitingForLocalSlot = 35,
     ssReceivingOutputs = 40,
     ssPostProcessing = 50,
 } StepState;
@@ -362,10 +360,6 @@ private:
     typedef std::map<std::string, Machine::ptr> Machines;
     nix::Sync<Machines> machines; // FIXME: use atomic_shared_ptr
 
-    /* Throttler for CPU-bound local work. */
-    static constexpr unsigned int maxSupportedLocalWorkers = 1024;
-    std::counting_semaphore<maxSupportedLocalWorkers> localWorkThrottler;
-
     /* Various stats. */
     time_t startedAt;
     counter nrBuildsRead{0};
@@ -463,12 +457,7 @@ private:
         prometheus::Counter& queue_steps_created;
         prometheus::Counter& queue_checks_early_exits;
         prometheus::Counter& queue_checks_finished;
-
-        prometheus::Counter& dispatcher_time_spent_running;
-        prometheus::Counter& dispatcher_time_spent_waiting;
-
-        prometheus::Counter& queue_monitor_time_spent_running;
-        prometheus::Counter& queue_monitor_time_spent_waiting;
+        prometheus::Gauge& queue_max_id;
 
         PromMetrics();
     };
@@ -512,7 +501,8 @@ private:
     void queueMonitorLoop(Connection & conn);
 
     /* Check the queue for new builds. */
-    bool getQueuedBuilds(Connection & conn, nix::ref<nix::Store> destStore);
+    bool getQueuedBuilds(Connection & conn,
+        nix::ref<nix::Store> destStore, unsigned int & lastBuildId);
 
     /* Handle cancellation, deletion and priority bumps. */
     void processQueueChange(Connection & conn);
@@ -520,12 +510,6 @@ private:
     BuildOutput getBuildOutputCached(Connection & conn, nix::ref<nix::Store> destStore,
         const nix::StorePath & drvPath);
 
-    /* Returns paths missing from the remote store. Paths are processed in
-     * parallel to work around the possible latency of remote stores. */
-    std::map<nix::DrvOutput, std::optional<nix::StorePath>> getMissingRemotePaths(
-        nix::ref<nix::Store> destStore,
-        const std::map<nix::DrvOutput, std::optional<nix::StorePath>> & paths);
-
     Step::ptr createStep(nix::ref<nix::Store> store,
         Connection & conn, Build::ptr build, const nix::StorePath & drvPath,
         Build::ptr referringBuild, Step::ptr referringStep, std::set<nix::StorePath> & finishedDrvs,
@@ -561,11 +545,10 @@ private:
        retried. */
     enum StepResult { sDone, sRetry, sMaybeCancelled };
     StepResult doBuildStep(nix::ref<nix::Store> destStore,
-        MachineReservation::ptr & reservation,
+        MachineReservation::ptr reservation,
         std::shared_ptr<ActiveStep> activeStep);
 
     void buildRemote(nix::ref<nix::Store> destStore,
-        MachineReservation::ptr & reservation,
         Machine::ptr machine, Step::ptr step,
         const nix::ServeProto::BuildOptions & buildOptions,
         RemoteResult & result, std::shared_ptr<ActiveStep> activeStep,

src/lib/Hydra/Base/Controller/NixChannel.pm

@@ -4,7 +4,6 @@ use strict;
 use warnings;
 use base 'Hydra::Base::Controller::REST';
 use List::SomeUtils qw(any);
-use Nix::Store;
 use Hydra::Helper::Nix;
 use Hydra::Helper::CatalystUtils;
 
@@ -30,7 +29,7 @@ sub getChannelData {
         my $outputs = {};
         foreach my $output (@outputs) {
             my $outPath = $output->get_column("outpath");
-            next if $checkValidity && !isValidPath($outPath);
+            next if $checkValidity && !$MACHINE_LOCAL_STORE->isValidPath($outPath);
             $outputs->{$output->get_column("outname")} = $outPath;
             push @storePaths, $outPath;
             # Put the system type in the manifest (for top-level

src/lib/Hydra/Controller/Build.pm

@@ -10,8 +10,6 @@ use File::Basename;
 use File::LibMagic;
 use File::stat;
 use Data::Dump qw(dump);
-use Nix::Store;
-use Nix::Config;
 use List::SomeUtils qw(all);
 use Encode;
 use JSON::PP;
@@ -83,9 +81,9 @@ sub build_GET {
     # false because `$_->path` will be empty
     $c->stash->{available} =
         $c->stash->{isLocalStore}
-        ? all { $_->path && isValidPath($_->path) } $build->buildoutputs->all
+        ? all { $_->path && $MACHINE_LOCAL_STORE->isValidPath($_->path) } $build->buildoutputs->all
         : 1;
-    $c->stash->{drvAvailable} = isValidPath $build->drvpath;
+    $c->stash->{drvAvailable} = $MACHINE_LOCAL_STORE->isValidPath($build->drvpath);
 
     if ($build->finished && $build->iscachedbuild) {
         my $path = ($build->buildoutputs)[0]->path or undef;
@@ -312,7 +310,7 @@ sub output : Chained('buildChain') PathPart Args(1) {
     error($c, "This build is not finished yet.") unless $build->finished;
     my $output = $build->buildoutputs->find({name => $outputName});
     notFound($c, "This build has no output named ‘$outputName’") unless defined $output;
-    gone($c, "Output is no longer available.") unless isValidPath $output->path;
+    gone($c, "Output is no longer available.") unless $MACHINE_LOCAL_STORE->isValidPath($output->path);
 
     $c->response->header('Content-Disposition', "attachment; filename=\"build-${\$build->id}-${\$outputName}.nar.bz2\"");
     $c->stash->{current_view} = 'NixNAR';
@@ -429,7 +427,7 @@ sub getDependencyGraph {
             };
         $$done{$path} = $node;
         my @refs;
-        foreach my $ref (queryReferences($path)) {
+        foreach my $ref ($MACHINE_LOCAL_STORE->queryReferences($path)) {
             next if $ref eq $path;
             next unless $runtime || $ref =~ /\.drv$/;
             getDependencyGraph($self, $c, $runtime, $done, $ref);
@@ -437,7 +435,7 @@ sub getDependencyGraph {
         }
         # Show in reverse topological order to flatten the graph.
         # Should probably do a proper BFS.
-        my @sorted = reverse topoSortPaths(@refs);
+        my @sorted = reverse $MACHINE_LOCAL_STORE->topoSortPaths(@refs);
         $node->{refs} = [map { $$done{$_} } @sorted];
     }
 
@@ -450,7 +448,7 @@ sub build_deps : Chained('buildChain') PathPart('build-deps') {
     my $build = $c->stash->{build};
     my $drvPath = $build->drvpath;
 
-    error($c, "Derivation no longer available.") unless isValidPath $drvPath;
+    error($c, "Derivation no longer available.") unless $MACHINE_LOCAL_STORE->isValidPath($drvPath);
 
     $c->stash->{buildTimeGraph} = getDependencyGraph($self, $c, 0, {}, $drvPath);
 
@@ -465,7 +463,7 @@ sub runtime_deps : Chained('buildChain') PathPart('runtime-deps') {
 
     requireLocalStore($c);
 
-    error($c, "Build outputs no longer available.") unless all { isValidPath($_) } @outPaths;
+    error($c, "Build outputs no longer available.") unless all { $MACHINE_LOCAL_STORE->isValidPath($_) } @outPaths;
 
     my $done = {};
     $c->stash->{runtimeGraph} = [ map { getDependencyGraph($self, $c, 1, $done, $_) } @outPaths ];
@@ -485,7 +483,7 @@ sub nix : Chained('buildChain') PathPart('nix') CaptureArgs(0) {
     if (isLocalStore) {
         foreach my $out ($build->buildoutputs) {
             notFound($c, "Path " . $out->path . " is no longer available.")
-                unless isValidPath($out->path);
+                unless $MACHINE_LOCAL_STORE->isValidPath($out->path);
         }
     }
 

src/lib/Hydra/Controller/Jobset.pm

@@ -364,21 +364,6 @@ sub evals_GET {
     );
 }
 
-sub errors :Chained('jobsetChain') :PathPart('errors') :Args(0) :ActionClass('REST') { }
-
-sub errors_GET {
-    my ($self, $c) = @_;
-
-    $c->stash->{template} = 'eval-error.tt';
-
-    my $jobsetName = $c->stash->{params}->{name};
-    $c->stash->{jobset} = $c->stash->{project}->jobsets->find(
-        { name => $jobsetName },
-        { '+columns' => { 'errormsg' => 'errormsg' } }
-    );
-
-    $self->status_ok($c, entity => $c->stash->{jobset});
-}
 
 # Redirect to the latest finished evaluation of this jobset.
 sub latest_eval : Chained('jobsetChain') PathPart('latest-eval') {

src/lib/Hydra/Controller/JobsetEval.pm

@@ -86,17 +86,6 @@ sub view_GET {
     );
 }
 
-sub errors :Chained('evalChain') :PathPart('errors') :Args(0) :ActionClass('REST') { }
-
-sub errors_GET {
-    my ($self, $c) = @_;
-
-    $c->stash->{template} = 'eval-error.tt';
-
-    $c->stash->{eval} = $c->model('DB::JobsetEvals')->find($c->stash->{eval}->id, { prefetch => 'evaluationerror' });
-
-    $self->status_ok($c, entity => $c->stash->{eval});
-}
 
 sub create_jobset : Chained('evalChain') PathPart('create-jobset') Args(0) {
     my ($self, $c) = @_;

src/lib/Hydra/Controller/Root.pm

@@ -160,7 +160,7 @@ sub status_GET {
             { "buildsteps.busy" => { '!=', 0 } },
             { order_by => ["globalpriority DESC", "id"],
               join => "buildsteps",
-              columns => [@buildListColumns, 'buildsteps.drvpath', 'buildsteps.type']
+              columns => [@buildListColumns]
             })]
     );
 }
@@ -329,7 +329,7 @@ sub nar :Local :Args(1) {
     else {
         $path = $Nix::Config::storeDir . "/$path";
 
-        gone($c, "Path " . $path . " is no longer available.") unless isValidPath($path);
+        gone($c, "Path " . $path . " is no longer available.") unless $MACHINE_LOCAL_STORE->isValidPath($path);
 
         $c->stash->{current_view} = 'NixNAR';
         $c->stash->{storePath} = $path;
@@ -396,7 +396,7 @@ sub narinfo :Path :Args(StrMatch[NARINFO_REGEX]) {
         my ($hash) = $narinfo =~ NARINFO_REGEX;
 
         die("Hash length was not 32") if length($hash) != 32;
-        my $path = queryPathFromHashPart($hash);
+        my $path = $MACHINE_LOCAL_STORE->queryPathFromHashPart($hash);
 
         if (!$path) {
             $c->response->status(404);

src/lib/Hydra/Helper/BuildDiff.pm

@@ -37,16 +37,7 @@ sub buildDiff {
 
     my $n = 0;
     foreach my $build (@{$builds}) {
-        my $aborted = $build->finished != 0 && (
-            # aborted
-            $build->buildstatus == 3
-            # cancelled
-            || $build->buildstatus == 4
-            # timeout
-            || $build->buildstatus == 7
-            # log limit exceeded
-            || $build->buildstatus == 10
-        );
+        my $aborted = $build->finished != 0 && ($build->buildstatus == 3 || $build->buildstatus == 4);
         my $d;
         my $found = 0;
         while ($n < scalar(@{$builds2})) {
@@ -88,4 +79,4 @@ sub buildDiff {
     return $ret;
 }
 
-1;
+1;

src/lib/Hydra/Helper/Nix.pm

@@ -40,8 +40,11 @@ our @EXPORT = qw(
     registerRoot
     restartBuilds
     run
+    $MACHINE_LOCAL_STORE
     );
 
+our $MACHINE_LOCAL_STORE = Nix::Store->new();
+
 
 sub getHydraHome {
     my $dir = $ENV{"HYDRA_HOME"} or die "The HYDRA_HOME directory does not exist!\n";
@@ -187,6 +190,10 @@ sub findLog {
 
     return undef if scalar @outPaths == 0;
 
+    # Filter out any NULLs. Content-addressed derivations
+    # that haven't built yet or failed to build may have a NULL outPath.
+    @outPaths = grep {defined} @outPaths;
+
     my @steps = $c->model('DB::BuildSteps')->search(
         { path => { -in => [@outPaths] } },
         { select => ["drvpath"]
@@ -286,7 +293,8 @@ sub getEvals {
 
     my @evals = $evals_result_set->search(
         { hasnewbuilds => 1 },
-        { order_by => "$me.id DESC", rows => $rows, offset => $offset });
+        { order_by => "$me.id DESC", rows => $rows, offset => $offset
+        , prefetch => { evaluationerror => [ ] } });
     my @res = ();
     my $cache = {};
 
@@ -493,7 +501,7 @@ sub restartBuilds {
     $builds = $builds->search({ finished => 1 });
 
     foreach my $build ($builds->search({}, { columns => ["drvpath"] })) {
-        next if !isValidPath($build->drvpath);
+        next if !$MACHINE_LOCAL_STORE->isValidPath($build->drvpath);
         registerRoot $build->drvpath;
     }
 

src/lib/Hydra/Plugin/BazaarInput.pm

@@ -7,7 +7,6 @@ use Digest::SHA qw(sha256_hex);
 use File::Path;
 use Hydra::Helper::Exec;
 use Hydra::Helper::Nix;
-use Nix::Store;
 
 sub supportedInputTypes {
     my ($self, $inputTypes) = @_;
@@ -38,9 +37,9 @@ sub fetchInput {
     (my $cachedInput) = $self->{db}->resultset('CachedBazaarInputs')->search(
         {uri => $uri, revision => $revision});
 
-    addTempRoot($cachedInput->storepath) if defined $cachedInput;
+    $MACHINE_LOCAL_STORE->addTempRoot($cachedInput->storepath) if defined $cachedInput;
 
-    if (defined $cachedInput && isValidPath($cachedInput->storepath)) {
+    if (defined $cachedInput && $MACHINE_LOCAL_STORE->isValidPath($cachedInput->storepath)) {
         $storePath = $cachedInput->storepath;
         $sha256 = $cachedInput->sha256hash;
     } else {
@@ -58,7 +57,7 @@ sub fetchInput {
         ($sha256, $storePath) = split ' ', $stdout;
 
         # FIXME: time window between nix-prefetch-bzr and addTempRoot.
-        addTempRoot($storePath);
+        $MACHINE_LOCAL_STORE->addTempRoot($storePath);
 
         $self->{db}->txn_do(sub {
             $self->{db}->resultset('CachedBazaarInputs')->create(

src/lib/Hydra/Plugin/DarcsInput.pm

@@ -7,7 +7,6 @@ use Digest::SHA qw(sha256_hex);
 use File::Path;
 use Hydra::Helper::Exec;
 use Hydra::Helper::Nix;
-use Nix::Store;
 
 sub supportedInputTypes {
     my ($self, $inputTypes) = @_;
@@ -58,7 +57,7 @@ sub fetchInput {
         {uri => $uri, revision => $revision},
         {rows => 1});
 
-    if (defined $cachedInput && isValidPath($cachedInput->storepath)) {
+    if (defined $cachedInput && $MACHINE_LOCAL_STORE->isValidPath($cachedInput->storepath)) {
         $storePath = $cachedInput->storepath;
         $sha256 = $cachedInput->sha256hash;
         $revision = $cachedInput->revision;
@@ -75,8 +74,8 @@ sub fetchInput {
         die "darcs changes --count failed" if $? != 0;
 
         system "rm", "-rf", "$tmpDir/export/_darcs";
-        $storePath = addToStore("$tmpDir/export", 1, "sha256");
-        $sha256 = queryPathHash($storePath);
+        $storePath = $MACHINE_LOCAL_STORE->addToStore("$tmpDir/export", 1, "sha256");
+        $sha256 = $MACHINE_LOCAL_STORE->queryPathHash($storePath);
         $sha256 =~ s/sha256://;
 
         $self->{db}->txn_do(sub {

src/lib/Hydra/Plugin/GitInput.pm

@@ -186,9 +186,9 @@ sub fetchInput {
         {uri => $uri, branch => $branch, revision => $revision, isdeepclone => defined($deepClone) ? 1 : 0},
         {rows => 1});
 
-    addTempRoot($cachedInput->storepath) if defined $cachedInput;
+    $MACHINE_LOCAL_STORE->addTempRoot($cachedInput->storepath) if defined $cachedInput;
 
-    if (defined $cachedInput && isValidPath($cachedInput->storepath)) {
+    if (defined $cachedInput && $MACHINE_LOCAL_STORE->isValidPath($cachedInput->storepath)) {
         $storePath = $cachedInput->storepath;
         $sha256 = $cachedInput->sha256hash;
         $revision = $cachedInput->revision;
@@ -217,7 +217,7 @@ sub fetchInput {
         ($sha256, $storePath) = split ' ', grab(cmd => ["nix-prefetch-git", $clonePath, $revision], chomp => 1);
 
         # FIXME: time window between nix-prefetch-git and addTempRoot.
-        addTempRoot($storePath);
+        $MACHINE_LOCAL_STORE->addTempRoot($storePath);
 
         $self->{db}->txn_do(sub {
             $self->{db}->resultset('CachedGitInputs')->update_or_create(

src/lib/Hydra/Plugin/MercurialInput.pm

@@ -7,7 +7,6 @@ use Digest::SHA qw(sha256_hex);
 use File::Path;
 use Hydra::Helper::Nix;
 use Hydra::Helper::Exec;
-use Nix::Store;
 use Fcntl qw(:flock);
 
 sub supportedInputTypes {
@@ -68,9 +67,9 @@ sub fetchInput {
     (my $cachedInput) = $self->{db}->resultset('CachedHgInputs')->search(
         {uri => $uri, branch => $branch, revision => $revision});
 
-    addTempRoot($cachedInput->storepath) if defined $cachedInput;
+    $MACHINE_LOCAL_STORE->addTempRoot($cachedInput->storepath) if defined $cachedInput;
 
-    if (defined $cachedInput && isValidPath($cachedInput->storepath)) {
+    if (defined $cachedInput && $MACHINE_LOCAL_STORE->isValidPath($cachedInput->storepath)) {
         $storePath = $cachedInput->storepath;
         $sha256 = $cachedInput->sha256hash;
     } else {
@@ -85,7 +84,7 @@ sub fetchInput {
         ($sha256, $storePath) = split ' ', $stdout;
 
         # FIXME: time window between nix-prefetch-hg and addTempRoot.
-        addTempRoot($storePath);
+        $MACHINE_LOCAL_STORE->addTempRoot($storePath);
 
         $self->{db}->txn_do(sub {
             $self->{db}->resultset('CachedHgInputs')->update_or_create(

src/lib/Hydra/Plugin/PathInput.pm

@@ -5,7 +5,6 @@ use warnings;
 use parent 'Hydra::Plugin';
 use POSIX qw(strftime);
 use Hydra::Helper::Nix;
-use Nix::Store;
 
 sub supportedInputTypes {
     my ($self, $inputTypes) = @_;
@@ -30,7 +29,7 @@ sub fetchInput {
         {srcpath => $uri, lastseen => {">", $timestamp - $timeout}},
         {rows => 1, order_by => "lastseen DESC"});
 
-    if (defined $cachedInput && isValidPath($cachedInput->storepath)) {
+    if (defined $cachedInput && $MACHINE_LOCAL_STORE->isValidPath($cachedInput->storepath)) {
         $storePath = $cachedInput->storepath;
         $sha256 = $cachedInput->sha256hash;
         $timestamp = $cachedInput->timestamp;
@@ -46,7 +45,7 @@ sub fetchInput {
         }
         chomp $storePath;
 
-        $sha256 = (queryPathInfo($storePath, 0))[1] or die;
+        $sha256 = ($MACHINE_LOCAL_STORE->queryPathInfo($storePath, 0))[1] or die;
 
         ($cachedInput) = $self->{db}->resultset('CachedPathInputs')->search(
             {srcpath => $uri, sha256hash => $sha256});

src/lib/Hydra/Plugin/S3Backup.pm

@@ -92,7 +92,7 @@ sub buildFinished {
         my $hash = substr basename($path), 0, 32;
         my ($deriver, $narHash, $time, $narSize, $refs) = queryPathInfo($path, 0);
         my $system;
-        if (defined $deriver and isValidPath($deriver)) {
+        if (defined $deriver and $MACHINE_LOCAL_STORE->isValidPath($deriver)) {
             $system = derivationFromPath($deriver)->{platform};
         }
         foreach my $reference (@{$refs}) {

src/lib/Hydra/Plugin/SubversionInput.pm

@@ -7,7 +7,6 @@ use Digest::SHA qw(sha256_hex);
 use Hydra::Helper::Exec;
 use Hydra::Helper::Nix;
 use IPC::Run;
-use Nix::Store;
 
 sub supportedInputTypes {
     my ($self, $inputTypes) = @_;
@@ -45,9 +44,9 @@ sub fetchInput {
     (my $cachedInput) = $self->{db}->resultset('CachedSubversionInputs')->search(
         {uri => $uri, revision => $revision});
 
-    addTempRoot($cachedInput->storepath) if defined $cachedInput;
+    $MACHINE_LOCAL_STORE->addTempRoot($cachedInput->storepath) if defined $cachedInput;
 
-    if (defined $cachedInput && isValidPath($cachedInput->storepath)) {
+    if (defined $cachedInput && $MACHINE_LOCAL_STORE->isValidPath($cachedInput->storepath)) {
         $storePath = $cachedInput->storepath;
         $sha256 = $cachedInput->sha256hash;
     } else {
@@ -62,16 +61,16 @@ sub fetchInput {
         die "error checking out Subversion repo at `$uri':\n$stderr" if $res;
 
         if ($type eq "svn-checkout") {
-            $storePath = addToStore($wcPath, 1, "sha256");
+            $storePath = $MACHINE_LOCAL_STORE->addToStore($wcPath, 1, "sha256");
         } else {
             # Hm, if the Nix Perl bindings supported filters in
             # addToStore(), then we wouldn't need to make a copy here.
             my $tmpDir = File::Temp->newdir("hydra-svn-export.XXXXXX", CLEANUP => 1, TMPDIR => 1) or die;
             (system "svn", "export", $wcPath, "$tmpDir/source", "--quiet") == 0 or die "svn export failed";
-            $storePath = addToStore("$tmpDir/source", 1, "sha256");
+            $storePath = $MACHINE_LOCAL_STORE->addToStore("$tmpDir/source", 1, "sha256");
         }
 
-        $sha256 = queryPathHash($storePath); $sha256 =~ s/sha256://;
+        $sha256 = $MACHINE_LOCAL_STORE->queryPathHash($storePath); $sha256 =~ s/sha256://;
 
         $self->{db}->txn_do(sub {
             $self->{db}->resultset('CachedSubversionInputs')->update_or_create(

src/lib/Hydra/Schema/Result/EvaluationErrors.pm

@@ -105,6 +105,4 @@ __PACKAGE__->add_column(
     "+id" => { retrieve_on_insert => 1 }
 );
 
-__PACKAGE__->mk_group_accessors('column' => 'has_error');
-
 1;

src/lib/Hydra/Schema/Result/Jobsets.pm

@@ -386,8 +386,6 @@ __PACKAGE__->add_column(
     "+id" => { retrieve_on_insert => 1 }
 );
 
-__PACKAGE__->mk_group_accessors('column' => 'has_error');
-
 sub supportsDynamicRunCommand {
   my ($self) = @_;
 

src/lib/Hydra/Schema/ResultSet/EvaluationErrors.pm

@@ -1,30 +0,0 @@
-package Hydra::Schema::ResultSet::EvaluationErrors;
-
-use strict;
-use utf8;
-use warnings;
-
-use parent 'DBIx::Class::ResultSet';
-
-use Storable qw(dclone);
-
-__PACKAGE__->load_components('Helper::ResultSet::RemoveColumns');
-
-# Exclude expensive error message values unless explicitly requested, and
-# replace them with a summary field describing their presence/absence.
-sub search_rs {
-  my ( $class, $query, $attrs ) = @_;
-
-  if ($attrs) {
-    $attrs = dclone($attrs);
-  }
-
-  unless (exists $attrs->{'select'} || exists $attrs->{'columns'}) {
-    $attrs->{'+columns'}->{'has_error'} = "errormsg != ''";
-  }
-  unless (exists $attrs->{'+columns'}->{'errormsg'}) {
-    push @{ $attrs->{'remove_columns'} }, 'errormsg';
-  }
-
-  return $class->next::method($query, $attrs);
-}

src/lib/Hydra/Schema/ResultSet/Jobsets.pm

@@ -1,30 +0,0 @@
-package Hydra::Schema::ResultSet::Jobsets;
-
-use strict;
-use utf8;
-use warnings;
-
-use parent 'DBIx::Class::ResultSet';
-
-use Storable qw(dclone);
-
-__PACKAGE__->load_components('Helper::ResultSet::RemoveColumns');
-
-# Exclude expensive error message values unless explicitly requested, and
-# replace them with a summary field describing their presence/absence.
-sub search_rs {
-  my ( $class, $query, $attrs ) = @_;
-
-  if ($attrs) {
-    $attrs = dclone($attrs);
-  }
-
-  unless (exists $attrs->{'select'} || exists $attrs->{'columns'}) {
-    $attrs->{'+columns'}->{'has_error'} = "errormsg != ''";
-  }
-  unless (exists $attrs->{'+columns'}->{'errormsg'}) {
-    push @{ $attrs->{'remove_columns'} }, 'errormsg';
-  }
-
-  return $class->next::method($query, $attrs);
-}

src/lib/Hydra/View/NARInfo.pm

@@ -8,6 +8,7 @@ use MIME::Base64;
 use Nix::Manifest;
 use Nix::Store;
 use Nix::Utils;
+use Hydra::Helper::Nix;
 use base qw/Catalyst::View/;
 
 sub process {
@@ -17,7 +18,7 @@ sub process {
 
     $c->response->content_type('text/x-nix-narinfo'); # !!! check MIME type
 
-    my ($deriver, $narHash, $time, $narSize, $refs) = queryPathInfo($storePath, 1);
+    my ($deriver, $narHash, $time, $narSize, $refs) = $MACHINE_LOCAL_STORE->queryPathInfo($storePath, 1);
 
     my $info;
     $info .= "StorePath: $storePath\n";
@@ -28,8 +29,8 @@ sub process {
     $info .= "References: " . join(" ", map { basename $_ } @{$refs}) . "\n";
     if (defined $deriver) {
         $info .= "Deriver: " . basename $deriver . "\n";
-        if (isValidPath($deriver)) {
-            my $drv = derivationFromPath($deriver);
+        if ($MACHINE_LOCAL_STORE->isValidPath($deriver)) {
+            my $drv = $MACHINE_LOCAL_STORE->derivationFromPath($deriver);
             $info .= "System: $drv->{platform}\n";
         }
     }

src/root/build.tt

@@ -61,7 +61,21 @@ END;
             <td>[% IF step.busy != 0 || ((step.machine || step.starttime) && (step.status == 0 || step.status == 1 || step.status == 3 || step.status == 4 || step.status == 7)); INCLUDE renderMachineName machine=step.machine; ELSE; "<em>n/a</em>"; END %]</td>
             <td class="step-status">
               [% IF step.busy != 0 %]
-                [% INCLUDE renderBusyStatus %]
+                [% IF step.busy == 1 %]
+                  <strong>Preparing</strong>
+                [% ELSIF step.busy == 10 %]
+                  <strong>Connecting</strong>
+                [% ELSIF step.busy == 20 %]
+                  <strong>Sending inputs</strong>
+                [% ELSIF step.busy == 30 %]
+                  <strong>Building</strong>
+                [% ELSIF step.busy == 40 %]
+                  <strong>Receiving outputs</strong>
+                [% ELSIF step.busy == 50 %]
+                  <strong>Post-processing</strong>
+                [% ELSE %]
+                  <strong>Unknown state</strong>
+                [% END %]
               [% ELSIF step.status == 0 %]
                 [% IF step.isnondeterministic %]
                   <span class="warn">Succeeded with non-determistic result</span>

src/root/common.tt

@@ -91,17 +91,6 @@ BLOCK renderDuration;
   duration % 60 %]s[%
 END;
 
-BLOCK renderDrvInfo;
-  drvname = step.drvpath
-    .substr(11) # strip `/nix/store/`
-    .split('-').slice(1).join("-") # strip hash part
-    .substr(0, -4); # strip `.drv`
-  IF drvname != releasename;
-    IF step.type == 0; action = "Build"; ELSE; action = "Substitution"; END;
-    IF drvname; %]<em> ([% action %] of [% drvname %])</em>[% END;
-  END;
-END;
-
 
 BLOCK renderBuildListHeader %]
   <table class="table table-striped table-condensed clickable-rows">
@@ -142,12 +131,7 @@ BLOCK renderBuildListBody;
       [% END %]
       <td><a class="row-link" href="[% link %]">[% build.id %]</a></td>
       [% IF !hideJobName %]
-        <td>
-          <a href="[%link%]">[% IF !hideJobsetName %][%build.jobset.get_column("project")%]:[%build.jobset.get_column("name")%]:[% END %][%build.get_column("job")%]</a>
-          [% IF showStepName %]
-            [% INCLUDE renderDrvInfo step=build.buildsteps releasename=build.nixname %]
-          [% END %]
-        </td>
+        <td><a href="[%link%]">[% IF !hideJobsetName %][%build.jobset.get_column("project")%]:[%build.jobset.get_column("name")%]:[% END %][%build.get_column("job")%]</td>
       [% END %]
       <td class="nowrap">[% t = showSchedulingInfo ? build.timestamp : build.stoptime; IF t; INCLUDE renderRelativeDate timestamp=(showSchedulingInfo ? build.timestamp : build.stoptime); ELSE; "-"; END %]</td>
       <td>[% !showSchedulingInfo and build.get_column('releasename') ? build.get_column('releasename') : build.nixname %]</td>
@@ -261,27 +245,6 @@ BLOCK renderBuildStatusIcon;
 END;
 
 
-BLOCK renderBusyStatus;
-  IF step.busy == 1 %]
-    <strong>Preparing</strong>
-  [% ELSIF step.busy == 10 %]
-    <strong>Connecting</strong>
-  [% ELSIF step.busy == 20 %]
-    <strong>Sending inputs</strong>
-  [% ELSIF step.busy == 30 %]
-    <strong>Building</strong>
-  [% ELSIF step.busy == 35 %]
-    <strong>Waiting to receive outputs</strong>
-  [% ELSIF step.busy == 40 %]
-    <strong>Receiving outputs</strong>
-  [% ELSIF step.busy == 50 %]
-    <strong>Post-processing</strong>
-  [% ELSE %]
-    <strong>Unknown state</strong>
-  [% END;
-END;
-
-
 BLOCK renderStatus;
   IF build.finished;
     buildstatus = build.buildstatus;
@@ -513,7 +476,7 @@ BLOCK renderEvals %]
             ELSE %]
              -
             [% END %]
-            [% IF eval.evaluationerror.has_error %]
+            [% IF eval.evaluationerror.errormsg %]
               <span class="badge badge-warning">Eval Errors</span>
             [% END %]
           </td>
@@ -639,7 +602,7 @@ BLOCK renderJobsetOverview %]
         <td>[% HTML.escape(j.description) %]</td>
         <td>[% IF j.lastcheckedtime;
                  INCLUDE renderDateTime timestamp = j.lastcheckedtime;
-                 IF j.has_error || j.fetcherrormsg; %]&nbsp;<span class = 'badge badge-warning'>Error</span>[% END;
+                 IF j.errormsg || j.fetcherrormsg; %]&nbsp;<span class = 'badge badge-warning'>Error</span>[% END;
                  ELSE; "-";
                END %]</td>
         [% IF j.get_column('nrtotal') > 0 %]

src/root/eval-error.tt

@@ -1,26 +0,0 @@
-[% PROCESS common.tt %]
-<!DOCTYPE html>
-
-<html lang="en">
-
-  <head>
-    <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
-    <meta http-equiv="X-UA-Compatible" content="IE=Edge" />
-    [% INCLUDE style.tt %]
-  </head>
-
-  <body>
-
-    <div class="tab-content tab-pane">
-        <div id="tabs-errors" class="">
-          [% IF jobset %]
-              <p>Errors occurred at [% INCLUDE renderDateTime timestamp=(jobset.errortime || jobset.lastcheckedtime) %].</p>
-              <div class="card bg-light"><div class="card-body"><pre>[% HTML.escape(jobset.fetcherrormsg || jobset.errormsg) %]</pre></div></div>
-          [% ELSIF eval %]
-              <p>Errors occurred at [% INCLUDE renderDateTime timestamp=(eval.evaluationerror.errortime || eval.timestamp) %].</p>
-              <div class="card bg-light"><div class="card-body"><pre>[% HTML.escape(eval.evaluationerror.errormsg) %]</pre></div></div>
-          [% END %]
-        </div>
-    </div>
-  </body>
-</html>

src/root/jobset-eval.tt

@@ -65,7 +65,7 @@ c.uri_for(c.controller('JobsetEval').action_for('view'),
   [% END %]
 
   [% IF aborted.size > 0 %]
-    <li class="nav-item"><a class="nav-link" href="#tabs-aborted" data-toggle="tab"><span class="text-warning">Aborted / Timed out Jobs ([% aborted.size %])</span></a></li>
+    <li class="nav-item"><a class="nav-link" href="#tabs-aborted" data-toggle="tab"><span class="text-warning">Aborted Jobs ([% aborted.size %])</span></a></li>
   [% END %]
   [% IF nowFail.size > 0 %]
     <li class="nav-item"><a class="nav-link" href="#tabs-now-fail" data-toggle="tab"><span class="text-warning">Newly Failing Jobs ([% nowFail.size %])</span></a></li>
@@ -90,7 +90,7 @@ c.uri_for(c.controller('JobsetEval').action_for('view'),
   [% END %]
   <li class="nav-item"><a class="nav-link" href="#tabs-inputs" data-toggle="tab">Inputs</a></li>
 
-  [% IF eval.evaluationerror.has_error %]
+  [% IF eval.evaluationerror.errormsg %]
     <li class="nav-item"><a class="nav-link" href="#tabs-errors" data-toggle="tab"><span class="text-warning">Evaluation Errors</span></a></li>
   [% END %]
 </ul>
@@ -108,6 +108,13 @@ c.uri_for(c.controller('JobsetEval').action_for('view'),
 
 <div class="tab-content">
 
+  [% IF eval.evaluationerror.errormsg %]
+    <div id="tabs-errors" class="tab-pane">
+      <p>Errors occurred at [% INCLUDE renderDateTime timestamp=(eval.evaluationerror.errortime || eval.timestamp) %].</p>
+      <div class="card bg-light"><div class="card-body"><pre>[% HTML.escape(eval.evaluationerror.errormsg) %]</pre></div></div>
+    </div>
+  [% END %]
+
   <div id="tabs-aborted" class="tab-pane">
     [% INCLUDE renderSome builds=aborted tabname="#tabs-aborted" %]
   </div>
@@ -165,9 +172,10 @@ c.uri_for(c.controller('JobsetEval').action_for('view'),
     [% END %]
   </div>
 
-  [% IF eval.evaluationerror.has_error %]
+  [% IF eval.evaluationerror.errormsg %]
     <div id="tabs-errors" class="tab-pane">
-      <iframe src="[% c.uri_for(c.controller('JobsetEval').action_for('errors'), [eval.id], params) %]" loading="lazy" frameBorder="0" width="100%"></iframe>
+      <p>Errors occurred at [% INCLUDE renderDateTime timestamp=(eval.evaluationerror.errortime || eval.timestamp) %].</p>
+      <div class="card bg-light"><div class="card-body"><pre>[% HTML.escape(eval.evaluationerror.errormsg) %]</pre></div></div>
     </div>
   [% END %]
 </div>

src/root/jobset.tt

@@ -61,7 +61,7 @@
   [% END %]
 
   <li class="nav-item"><a class="nav-link active" href="#tabs-evaluations" data-toggle="tab">Evaluations</a></li>
-  [% IF jobset.has_error || jobset.fetcherrormsg %]
+  [% IF jobset.errormsg || jobset.fetcherrormsg %]
     <li class="nav-item"><a class="nav-link" href="#tabs-errors" data-toggle="tab"><span class="text-warning">Evaluation Errors</span></a></li>
   [% END %]
   <li class="nav-item"><a class="nav-link" href="#tabs-jobs" data-toggle="tab">Jobs</a></li>
@@ -79,7 +79,7 @@
         <th>Last checked:</th>
         <td>
           [% IF jobset.lastcheckedtime %]
-            [% INCLUDE renderDateTime timestamp = jobset.lastcheckedtime %], [% IF jobset.has_error || jobset.fetcherrormsg %]<em class="text-warning">with errors!</em>[% ELSE %]<em>no errors</em>[% END %]
+            [% INCLUDE renderDateTime timestamp = jobset.lastcheckedtime %], [% IF jobset.errormsg || jobset.fetcherrormsg %]<em class="text-warning">with errors!</em>[% ELSE %]<em>no errors</em>[% END %]
           [% ELSE %]
             <em>never</em>
           [% END %]
@@ -117,9 +117,10 @@
 
   </div>
 
-  [% IF jobset.has_error || jobset.fetcherrormsg %]
+  [% IF jobset.errormsg || jobset.fetcherrormsg %]
     <div id="tabs-errors" class="tab-pane">
-      <iframe src="[% c.uri_for('/jobset' project.name jobset.name "errors") %]" loading="lazy" frameBorder="0" width="100%"></iframe>
+      <p>Errors occurred at [% INCLUDE renderDateTime timestamp=(jobset.errortime || jobset.lastcheckedtime) %].</p>
+      <div class="card bg-light"><div class="card-body"><pre>[% HTML.escape(jobset.fetcherrormsg || jobset.errormsg) %]</pre></div></div>
     </div>
   [% END %]
 

src/root/layout.tt

@@ -10,7 +10,31 @@
 
     <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
     <meta http-equiv="X-UA-Compatible" content="IE=Edge" />
-    [% INCLUDE style.tt %]
+
+    <script type="text/javascript" src="[% c.uri_for("/static/js/jquery/jquery-3.4.1.min.js") %]"></script>
+    <script type="text/javascript" src="[% c.uri_for("/static/js/jquery/jquery-ui-1.10.4.min.js") %]"></script>
+    <script type="text/javascript" src="[% c.uri_for("/static/js/moment/moment-2.24.0.min.js") %]"></script>
+
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+
+    <link href="[% c.uri_for("/static/fontawesome/css/all.css") %]" rel="stylesheet" />
+    <script type="text/javascript" src="[% c.uri_for("/static/js/popper.min.js") %]"></script>
+    <script type="text/javascript" src="[% c.uri_for("/static/bootstrap/js/bootstrap.min.js") %]"></script>
+    <link href="[% c.uri_for("/static/bootstrap/css/bootstrap.min.css") %]" rel="stylesheet" />
+
+    <!-- hydra.css may need to be moved to before boostrap to make the @media rule work. -->
+    <link rel="stylesheet" href="[% c.uri_for("/static/css/hydra.css") %]" type="text/css" />
+    <link rel="stylesheet" href="[% c.uri_for("/static/css/rotated-th.css") %]" type="text/css" />
+
+    <style>
+      .popover { max-width: 40%; }
+    </style>
+
+    <script type="text/javascript" src="[% c.uri_for("/static/js/bootbox.min.js") %]"></script>
+
+    <link rel="stylesheet" href="[% c.uri_for("/static/css/tree.css") %]" type="text/css" />
+
+    <script type="text/javascript" src="[% c.uri_for("/static/js/common.js") %]"></script>
 
     [% IF c.config.enable_google_login %]
       <meta name="google-signin-client_id" content="[% c.config.google_client_id %]">

src/root/machine-status.tt

@@ -6,10 +6,10 @@
   <thead>
     <tr>
       <th>Job</th>
+      <th>System</th>
       <th>Build</th>
       <th>Step</th>
       <th>What</th>
-      <th>Status</th>
       <th>Since</th>
     </tr>
   </thead>
@@ -40,10 +40,10 @@
           [% idle = 0 %]
           <tr>
             <td><tt>[% INCLUDE renderFullJobName project=step.project jobset=step.jobset job=step.job %]</tt></td>
+            <td><tt>[% step.system %]</tt></td>
             <td><a href="[% c.uri_for('/build' step.build) %]">[% step.build %]</a></td>
             <td>[% IF step.busy >= 30 %]<a class="row-link" href="[% c.uri_for('/build' step.build 'nixlog' step.stepnr 'tail') %]">[% step.stepnr %]</a>[% ELSE; step.stepnr; END %]</td>
             <td><tt>[% step.drvpath.match('-(.*)').0 %]</tt></td>
-            <td>[% INCLUDE renderBusyStatus %]</td>
             <td style="width: 10em">[% INCLUDE renderDuration duration = curTime - step.starttime %] </td>
           </tr>
         [% END %]

src/root/static/js/common.js

@@ -129,12 +129,6 @@ $(document).ready(function() {
             el.addClass("is-local");
         }
     });
-
-    [...document.getElementsByTagName("iframe")].forEach((element) => {
-        element.contentWindow.addEventListener("DOMContentLoaded", (_) => {
-            element.style.height = element.contentWindow.document.body.scrollHeight + 'px';
-        })
-    })
 });
 
 var tabsLoaded = {};

src/root/status.tt

@@ -7,7 +7,7 @@
 
 [% ELSE %]
 
-  [% INCLUDE renderBuildList builds=resource showSchedulingInfo=1 hideResultInfo=1 busy=1 showStepName=1 %]
+  [% INCLUDE renderBuildList builds=resource showSchedulingInfo=1 hideResultInfo=1 busy=1 %]
 
 [% END %]
 

src/root/style.tt

@@ -1,24 +0,0 @@
-<script type="text/javascript" src="[% c.uri_for("/static/js/jquery/jquery-3.4.1.min.js") %]"></script>
-<script type="text/javascript" src="[% c.uri_for("/static/js/jquery/jquery-ui-1.10.4.min.js") %]"></script>
-<script type="text/javascript" src="[% c.uri_for("/static/js/moment/moment-2.24.0.min.js") %]"></script>
-
-<meta name="viewport" content="width=device-width, initial-scale=1.0" />
-
-<link href="[% c.uri_for("/static/fontawesome/css/all.css") %]" rel="stylesheet" />
-<script type="text/javascript" src="[% c.uri_for("/static/js/popper.min.js") %]"></script>
-<script type="text/javascript" src="[% c.uri_for("/static/bootstrap/js/bootstrap.min.js") %]"></script>
-<link href="[% c.uri_for("/static/bootstrap/css/bootstrap.min.css") %]" rel="stylesheet" />
-
-<!-- hydra.css may need to be moved to before boostrap to make the @media rule work. -->
-<link rel="stylesheet" href="[% c.uri_for("/static/css/hydra.css") %]" type="text/css" />
-<link rel="stylesheet" href="[% c.uri_for("/static/css/rotated-th.css") %]" type="text/css" />
-
-<style>
-  .popover { max-width: 40%; }
-</style>
-
-<script type="text/javascript" src="[% c.uri_for("/static/js/bootbox.min.js") %]"></script>
-
-<link rel="stylesheet" href="[% c.uri_for("/static/css/tree.css") %]" type="text/css" />
-
-<script type="text/javascript" src="[% c.uri_for("/static/js/common.js") %]"></script>

src/script/hydra-eval-jobset

@@ -85,14 +85,14 @@ sub attrsToSQL {
 # Fetch a store path from 'eval_substituter' if not already present.
 sub getPath {
     my ($path) = @_;
-    return 1 if isValidPath($path);
+    return 1 if $MACHINE_LOCAL_STORE->isValidPath($path);
 
     my $substituter = $config->{eval_substituter};
 
     system("nix", "--experimental-features", "nix-command", "copy", "--from", $substituter, "--", $path)
         if defined $substituter;
 
-    return isValidPath($path);
+    return $MACHINE_LOCAL_STORE->isValidPath($path);
 }
 
 
@@ -143,7 +143,7 @@ sub fetchInputBuild {
         , version => $version
         , outputName => $mainOutput->name
         };
-    if (isValidPath($prevBuild->drvpath)) {
+    if ($MACHINE_LOCAL_STORE->isValidPath($prevBuild->drvpath)) {
         $result->{drvPath} = $prevBuild->drvpath;
     }
 
@@ -233,7 +233,7 @@ sub fetchInputEval {
         my $out = $build->buildoutputs->find({ name => "out" });
         next unless defined $out;
         # FIXME: Should we fail if the path is not valid?
-        next unless isValidPath($out->path);
+        next unless $MACHINE_LOCAL_STORE->isValidPath($out->path);
         $jobs->{$build->get_column('job')} = $out->path;
     }
 

src/script/hydra-update-gc-roots

@@ -5,7 +5,6 @@ use warnings;
 use File::Path;
 use File::stat;
 use File::Basename;
-use Nix::Store;
 use Hydra::Config;
 use Hydra::Schema;
 use Hydra::Helper::Nix;
@@ -47,7 +46,7 @@ sub keepBuild {
         $build->finished && ($build->buildstatus == 0 || $build->buildstatus == 6))
     {
         foreach my $path (split / /, $build->get_column('outpaths')) {
-            if (isValidPath($path)) {
+            if ($MACHINE_LOCAL_STORE->isValidPath($path)) {
                 addRoot $path;
             } else {
                 print STDERR "    warning: output ", $path, " has disappeared\n" if $build->finished;
@@ -55,7 +54,7 @@ sub keepBuild {
         }
     }
     if (!$build->finished || ($keepFailedDrvs && $build->buildstatus != 0)) {
-        if (isValidPath($build->drvpath)) {
+        if ($MACHINE_LOCAL_STORE->isValidPath($build->drvpath)) {
             addRoot $build->drvpath;
         } else {
             print STDERR "    warning: derivation ", $build->drvpath, " has disappeared\n";

t/Hydra/Controller/Jobset/evals.t

@@ -32,9 +32,4 @@ subtest "/jobset/PROJECT/JOBSET/evals" => sub {
     ok($jobsetevals->is_success, "The page showing the jobset evals returns 200.");
 };
 
-subtest "/jobset/PROJECT/JOBSET/errors" => sub {
-    my $jobsetevals = request(GET '/jobset/' . $project->name . '/' . $jobset->name . '/errors');
-    ok($jobsetevals->is_success, "The page showing the jobset eval errors returns 200.");
-};
-
 done_testing;

t/Hydra/Controller/JobsetEval/fetch.t

@@ -35,10 +35,6 @@ subtest "Fetching the eval's overview" => sub {
     is($fetch->code, 200, "channel page is 200");
 };
 
-subtest "Fetching the eval's overview" => sub {
-    my $fetch = request(GET '/eval/' . $eval->id, '/errors');
-    is($fetch->code, 200, "errors page is 200");
-};
 
 
 done_testing;

t/content-addressed/basic.t

@@ -27,13 +27,13 @@ my $project = $db->resultset('Projects')->create({name => "tests", displayname =
 my $jobset = createBaseJobset("content-addressed", "content-addressed.nix", $ctx{jobsdir});
 
 ok(evalSucceeds($jobset), "Evaluating jobs/content-addressed.nix should exit with return code 0");
-is(nrQueuedBuildsForJobset($jobset), 5, "Evaluating jobs/content-addressed.nix should result in 4 builds");
+is(nrQueuedBuildsForJobset($jobset), 6, "Evaluating jobs/content-addressed.nix should result in 6 builds");
 
 for my $build (queuedBuildsForJobset($jobset)) {
     ok(runBuild($build), "Build '".$build->job."' from jobs/content-addressed.nix should exit with code 0");
     my $newbuild = $db->resultset('Builds')->find($build->id);
     is($newbuild->finished, 1, "Build '".$build->job."' from jobs/content-addressed.nix should be finished.");
-    my $expected = $build->job eq "fails" ? 1 : $build->job =~ /with_failed/ ? 6 : 0;
+    my $expected = $build->job eq "fails" ? 1 : $build->job =~ /with_failed/ ? 6 : $build->job =~ /FailingCA/ ? 2 : 0;
     is($newbuild->buildstatus, $expected, "Build '".$build->job."' from jobs/content-addressed.nix should have buildstatus $expected.");
 
     my $response = request("/build/".$build->id);
@@ -55,6 +55,8 @@ for my $build (queuedBuildsForJobset($jobset)) {
 
 }
 
+# XXX: deststoredir is undefined: Use of uninitialized value $ctx{"deststoredir"} in concatenation (.) or string at t/content-addressed/basic.t line 58.
+# XXX: This test seems to not do what it seems to be doing. See documentation: https://metacpan.org/pod/Test2::V0#isnt($got,-$do_not_want,-$name)
 isnt(<$ctx{deststoredir}/realisations/*>, "", "The destination store should have the realisations of the built derivations registered");
 
 done_testing;

t/evaluator/evaluate-constituents-broken.t

@@ -22,7 +22,7 @@ like(
     "The stderr record includes a relevant error message"
 );
 
-$jobset->discard_changes({ '+columns' => {'errormsg' => 'errormsg'} });  # refresh from DB
+$jobset->discard_changes;  # refresh from DB
 like(
     $jobset->errormsg,
     qr/aggregate job ‘mixed_aggregate’ failed with the error: constituentA: does not exist/,

t/jobs/content-addressed.nix

@@ -25,6 +25,13 @@ rec {
       FOO = empty_dir;
     };
 
+  caDependingOnFailingCA =
+    cfg.mkContentAddressedDerivation {
+      name = "ca-depending-on-failing-ca";
+      builder = ./dir-with-file-builder.sh;
+      FOO = fails;
+    };
+
   nonCaDependingOnCA =
     cfg.mkDerivation {
       name = "non-ca-depending-on-ca";

t/lib/CliRunners.pm

@@ -14,7 +14,7 @@ our @EXPORT = qw(
 sub evalSucceeds {
     my ($jobset) = @_;
     my ($res, $stdout, $stderr) = captureStdoutStderr(60, ("hydra-eval-jobset", $jobset->project->name, $jobset->name));
-    $jobset->discard_changes({ '+columns' => {'errormsg' => 'errormsg'} });  # refresh from DB
+    $jobset->discard_changes;  # refresh from DB
     if ($res) {
         chomp $stdout; chomp $stderr;
         utf8::decode($stdout) or die "Invalid unicode in stdout.";
@@ -29,7 +29,7 @@ sub evalSucceeds {
 sub evalFails {
     my ($jobset) = @_;
     my ($res, $stdout, $stderr) = captureStdoutStderr(60, ("hydra-eval-jobset", $jobset->project->name, $jobset->name));
-    $jobset->discard_changes({ '+columns' => {'errormsg' => 'errormsg'} });  # refresh from DB
+    $jobset->discard_changes;  # refresh from DB
     if (!$res) {
         chomp $stdout; chomp $stderr;
         utf8::decode($stdout) or die "Invalid unicode in stdout.";

t/queue-runner/direct-indirect-constituents.t

@@ -13,7 +13,7 @@ my $constituentBuildA = $builds->{"constituentA"};
 my $constituentBuildB = $builds->{"constituentB"};
 
 my $eval = $constituentBuildA->jobsetevals->first();
-is($eval->evaluationerror->has_error, 0);
+is($eval->evaluationerror->errormsg, "");
 
 subtest "Verifying the direct aggregate" => sub {
     my $aggBuild = $builds->{"direct_aggregate"};

t/s3-backup-test.pl

@@ -3,7 +3,6 @@ use warnings;
 use File::Basename;
 use Hydra::Model::DB;
 use Hydra::Helper::Nix;
-use Nix::Store;
 use Cwd;
 
 my $db = Hydra::Model::DB->new;