From c35791fcc276459e74510513fa030eeac2b62013 Mon Sep 17 00:00:00 2001 From: Andreas Rammhold Date: Tue, 9 Feb 2021 14:10:08 +0100 Subject: [PATCH 001/313] Add Queue Runner Status to the topbar I've been searching for this waaay too often in the past and I simply do not see a reason not to include it in the topbar by default. --- src/root/topbar.tt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/root/topbar.tt b/src/root/topbar.tt index f8366917..ec2ee08c 100644 --- a/src/root/topbar.tt +++ b/src/root/topbar.tt @@ -34,6 +34,9 @@ [% INCLUDE menuItem uri = c.uri_for(c.controller('Root').action_for('steps')) title = "Latest steps" %] + [% INCLUDE menuItem + uri = c.uri_for(c.controller('Root').action_for('queue_runner_status')) + title = "Queue Runner Status" %] [% END %] [% IF project %] From 54675a0d94adae6c6164e93ba088edcc1f527647 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Janne=20He=C3=9F?= Date: Sun, 13 Feb 2022 14:24:36 +0100 Subject: [PATCH 002/313] Fix local store detection and related issues - Add localStore into the stash because it's used in templates - Hide the Channels button for non-local stores because the link 404s anyway - Fix a style issue when having popovers in dark mode --- src/lib/Hydra/Controller/Root.pm | 1 + src/root/static/css/hydra.css | 2 +- src/root/topbar.tt | 6 +++--- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/lib/Hydra/Controller/Root.pm b/src/lib/Hydra/Controller/Root.pm index c6843d29..641002e7 100644 --- a/src/lib/Hydra/Controller/Root.pm +++ b/src/lib/Hydra/Controller/Root.pm @@ -52,6 +52,7 @@ sub begin :Private { $c->stash->{tracker} = defined $c->config->{tracker} ? $c->config->{tracker} : ""; $c->stash->{flashMsg} = $c->flash->{flashMsg}; $c->stash->{successMsg} = $c->flash->{successMsg}; + $c->stash->{localStore} = isLocalStore; $c->stash->{isPrivateHydra} = $c->config->{private} // "0" ne "0"; diff --git a/src/root/static/css/hydra.css b/src/root/static/css/hydra.css index 475d61c2..252ba815 100644 --- a/src/root/static/css/hydra.css +++ b/src/root/static/css/hydra.css @@ -151,7 +151,7 @@ td.step-status span.warn { html { background-color: #1f1f1f; } - body, div.popover { + body, div.popover, div.popover-body { background-color: #1f1f1f; color: #fafafa !important; } diff --git a/src/root/topbar.tt b/src/root/topbar.tt index fdfbf431..7165aa15 100644 --- a/src/root/topbar.tt +++ b/src/root/topbar.tt @@ -42,7 +42,7 @@
[% INCLUDE menuItem uri = c.uri_for(c.controller('Project').action_for('project'), [project.name]) title = "Overview" %] [% INCLUDE menuItem uri = c.uri_for(c.controller('Project').action_for('all'), [project.name]) title = "Latest builds" %] - [% INCLUDE menuItem uri = c.uri_for('/project' project.name 'channel' 'latest') title = "Channel" %] + [% IF localStore %][% INCLUDE menuItem uri = c.uri_for('/project' project.name 'channel' 'latest') title = "Channel" %][% END %] [% END %] [% END %] @@ -59,7 +59,7 @@ [% INCLUDE menuItem uri = c.uri_for(c.controller('Jobset').action_for('all'), [project.name, jobset.name]) title = "Latest builds" %] - [% INCLUDE menuItem uri = c.uri_for('/jobset' project.name jobset.name 'channel' 'latest') title = "Channel" %] + [% IF localStore %][% INCLUDE menuItem uri = c.uri_for('/jobset' project.name jobset.name 'channel' 'latest') title = "Channel" %][% END %] [% END %] [% END %] @@ -73,7 +73,7 @@ [% INCLUDE menuItem uri = c.uri_for(c.controller('Job').action_for('all'), [project.name, jobset.name, job]) title = "Latest builds" %] - [% INCLUDE menuItem uri = c.uri_for('/job' project.name jobset.name job 'channel' 'latest') title = "Channel" %] + [% IF localStore %][% INCLUDE menuItem uri = c.uri_for('/job' project.name jobset.name job 'channel' 'latest') title = "Channel" %][% END %] [% END %] [% END %] From 5db8642224e996d0ebe3492cd55dfef6276d5a15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Mon, 21 Mar 2022 16:02:51 +0100 Subject: [PATCH 003/313] Factor out a struct representing a connection to a machine --- src/hydra-queue-runner/state.hh | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/hydra-queue-runner/state.hh b/src/hydra-queue-runner/state.hh index 8f303d28..059b03a1 100644 --- a/src/hydra-queue-runner/state.hh +++ b/src/hydra-queue-runner/state.hh @@ -290,6 +290,16 @@ struct Machine { return sshName == "localhost"; } + + // A connection to a machine + struct Connection { + nix::FdSink to; + nix::FdSource from; + unsigned int remoteVersion; + + // Backpointer to the machine + ptr machine; + }; }; From 2f494b783425d6703f23bc4f2cdbc70592a31990 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Mon, 21 Mar 2022 10:42:44 +0100 Subject: [PATCH 004/313] Factor out the creation of the log file --- src/hydra-queue-runner/build-remote.cc | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc index 464a35c8..2e258484 100644 --- a/src/hydra-queue-runner/build-remote.cc +++ b/src/hydra-queue-runner/build-remote.cc @@ -175,6 +175,18 @@ StorePaths reverseTopoSortPaths(const std::map & paths return sorted; } +std::pair openLogFile(const std::string & logDir, const StorePath & drvPath) +{ + string base(drvPath.to_string()); + auto logFile = logDir + "/" + string(base, 0, 2) + "/" + string(base, 2); + + createDirs(dirOf(logFile)); + + AutoCloseFD logFD = open(logFile.c_str(), O_CREAT | O_TRUNC | O_WRONLY, 0666); + if (!logFD) throw SysError("creating log file ‘%s’", logFile); + + return {std::move(logFile), std::move(logFD)}; +} void State::buildRemote(ref destStore, Machine::ptr machine, Step::ptr step, @@ -185,14 +197,9 @@ void State::buildRemote(ref destStore, { assert(BuildResult::TimedOut == 8); - string base(step->drvPath.to_string()); - result.logFile = logDir + "/" + string(base, 0, 2) + "/" + string(base, 2); - AutoDelete autoDelete(result.logFile, false); - - createDirs(dirOf(result.logFile)); - - AutoCloseFD logFD = open(result.logFile.c_str(), O_CREAT | O_TRUNC | O_WRONLY, 0666); - if (!logFD) throw SysError("creating log file ‘%s’", result.logFile); + auto [logFile, logFD] = openLogFile(logDir, step->drvPath); + AutoDelete logFileDel(logFile, false); + result.logFile = logFile; nix::Path tmpDir = createTempDir(); AutoDelete tmpDirDel(tmpDir, true); @@ -316,7 +323,7 @@ void State::buildRemote(ref destStore, result.overhead += std::chrono::duration_cast(now2 - now1).count(); } - autoDelete.cancel(); + logFileDel.cancel(); /* Truncate the log to get rid of messages about substitutions etc. on the remote system. */ From 9f1b911625cbc92023b6aec936a505a46af3172f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Mon, 21 Mar 2022 11:35:38 +0100 Subject: [PATCH 005/313] Factor more stuff out --- src/hydra-queue-runner/build-remote.cc | 224 ++++++++++++++----------- 1 file changed, 122 insertions(+), 102 deletions(-) diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc index 2e258484..360a8ef7 100644 --- a/src/hydra-queue-runner/build-remote.cc +++ b/src/hydra-queue-runner/build-remote.cc @@ -188,6 +188,87 @@ std::pair openLogFile(const std::string & logDir, const Store return {std::move(logFile), std::move(logFD)}; } +void handshake(Machine::Connection & conn, unsigned int repeats) +{ + conn.to << SERVE_MAGIC_1 << 0x204; + conn.to.flush(); + + unsigned int magic = readInt(conn.from); + if (magic != SERVE_MAGIC_2) + throw Error("protocol mismatch with ‘nix-store --serve’ on ‘%1%’", conn.machine->sshName); + conn.remoteVersion = readInt(conn.from); + if (GET_PROTOCOL_MAJOR(conn.remoteVersion) != 0x200) + throw Error("unsupported ‘nix-store --serve’ protocol version on ‘%1%’", conn.machine->sshName); + if (GET_PROTOCOL_MINOR(conn.remoteVersion) < 3 && repeats > 0) + throw Error("machine ‘%1%’ does not support repeating a build; please upgrade it to Nix 1.12", conn.machine->sshName); +} + +StorePathSet sendInputs( + State & state, + Step & step, + Store & localStore, + Store & destStore, + Machine::Connection & conn, + unsigned int & overhead, + counter & nrStepsWaiting, + counter & nrStepsCopyingTo +) +{ + + StorePathSet inputs; + BasicDerivation basicDrv(*step.drv); + + for (auto & p : step.drv->inputSrcs) + inputs.insert(p); + + for (auto & input : step.drv->inputDrvs) { + auto drv2 = localStore.readDerivation(input.first); + for (auto & name : input.second) { + if (auto i = get(drv2.outputs, name)) { + auto outPath = i->path(localStore, drv2.name, name); + inputs.insert(*outPath); + basicDrv.inputSrcs.insert(*outPath); + } + } + } + + /* Ensure that the inputs exist in the destination store. This is + a no-op for regular stores, but for the binary cache store, + this will copy the inputs to the binary cache from the local + store. */ + if (localStore.getUri() != destStore.getUri()) { + StorePathSet closure; + localStore.computeFSClosure(step.drv->inputSrcs, closure); + copyPaths(localStore, destStore, closure, NoRepair, NoCheckSigs, NoSubstitute); + } + + { + auto mc1 = std::make_shared>(nrStepsWaiting); + mc1.reset(); + MaintainCount mc2(nrStepsCopyingTo); + + printMsg(lvlDebug, "sending closure of ‘%s’ to ‘%s’", + localStore.printStorePath(step.drvPath), conn.machine->sshName); + + auto now1 = std::chrono::steady_clock::now(); + + /* Copy the input closure. */ + if (conn.machine->isLocalhost()) { + StorePathSet closure; + destStore.computeFSClosure(inputs, closure); + copyPaths(destStore, localStore, closure, NoRepair, NoCheckSigs, NoSubstitute); + } else { + copyClosureTo(conn.machine->state->sendLock, destStore, conn.from, conn.to, inputs, true); + } + + auto now2 = std::chrono::steady_clock::now(); + + overhead += std::chrono::duration_cast(now2 - now1).count(); + } + + return inputs; +} + void State::buildRemote(ref destStore, Machine::ptr machine, Step::ptr step, unsigned int maxSilentTime, unsigned int buildTimeout, unsigned int repeats, @@ -230,33 +311,21 @@ void State::buildRemote(ref destStore, process. Meh. */ }); - FdSource from(child.from.get()); - FdSink to(child.to.get()); + Machine::Connection conn; + conn.from = child.from.get(); + conn.to = child.to.get(); + conn.machine = machine; Finally updateStats([&]() { - bytesReceived += from.read; - bytesSent += to.written; + bytesReceived += conn.from.read; + bytesSent += conn.to.written; }); - /* Handshake. */ - unsigned int remoteVersion; - try { - to << SERVE_MAGIC_1 << 0x204; - to.flush(); - - unsigned int magic = readInt(from); - if (magic != SERVE_MAGIC_2) - throw Error("protocol mismatch with ‘nix-store --serve’ on ‘%1%’", machine->sshName); - remoteVersion = readInt(from); - if (GET_PROTOCOL_MAJOR(remoteVersion) != 0x200) - throw Error("unsupported ‘nix-store --serve’ protocol version on ‘%1%’", machine->sshName); - if (GET_PROTOCOL_MINOR(remoteVersion) < 3 && repeats > 0) - throw Error("machine ‘%1%’ does not support repeating a build; please upgrade it to Nix 1.12", machine->sshName); - + handshake(conn, repeats); } catch (EndOfFile & e) { child.pid.wait(); - string s = chomp(readFile(result.logFile)); + std::string s = chomp(readFile(result.logFile)); throw Error("cannot connect to ‘%1%’: %2%", machine->sshName, s); } @@ -272,61 +341,12 @@ void State::buildRemote(ref destStore, outputs of the input derivations. */ updateStep(ssSendingInputs); - StorePathSet inputs; - BasicDerivation basicDrv(*step->drv); - - for (auto & p : step->drv->inputSrcs) - inputs.insert(p); - - for (auto & input : step->drv->inputDrvs) { - auto drv2 = localStore->readDerivation(input.first); - for (auto & name : input.second) { - if (auto i = get(drv2.outputs, name)) { - auto outPath = i->path(*localStore, drv2.name, name); - inputs.insert(*outPath); - basicDrv.inputSrcs.insert(*outPath); - } - } - } - - /* Ensure that the inputs exist in the destination store. This is - a no-op for regular stores, but for the binary cache store, - this will copy the inputs to the binary cache from the local - store. */ - if (localStore != std::shared_ptr(destStore)) { - StorePathSet closure; - localStore->computeFSClosure(step->drv->inputSrcs, closure); - copyPaths(*localStore, *destStore, closure, NoRepair, NoCheckSigs, NoSubstitute); - } - - { - auto mc1 = std::make_shared>(nrStepsWaiting); - mc1.reset(); - MaintainCount mc2(nrStepsCopyingTo); - - printMsg(lvlDebug, "sending closure of ‘%s’ to ‘%s’", - localStore->printStorePath(step->drvPath), machine->sshName); - - auto now1 = std::chrono::steady_clock::now(); - - /* Copy the input closure. */ - if (machine->isLocalhost()) { - StorePathSet closure; - destStore->computeFSClosure(inputs, closure); - copyPaths(*destStore, *localStore, closure, NoRepair, NoCheckSigs, NoSubstitute); - } else { - copyClosureTo(machine->state->sendLock, *destStore, from, to, inputs, true); - } - - auto now2 = std::chrono::steady_clock::now(); - - result.overhead += std::chrono::duration_cast(now2 - now1).count(); - } + StorePathSet inputs = sendInputs(*this, *step, *localStore, *destStore, conn, result.overhead, nrStepsWaiting, nrStepsCopyingTo); logFileDel.cancel(); /* Truncate the log to get rid of messages about substitutions - etc. on the remote system. */ + etc. on the remote system. */ if (lseek(logFD.get(), SEEK_SET, 0) != 0) throw SysError("seeking to the start of log file ‘%s’", result.logFile); @@ -342,31 +362,31 @@ void State::buildRemote(ref destStore, updateStep(ssBuilding); - to << cmdBuildDerivation << localStore->printStorePath(step->drvPath); - writeDerivation(to, *localStore, basicDrv); - to << maxSilentTime << buildTimeout; - if (GET_PROTOCOL_MINOR(remoteVersion) >= 2) - to << maxLogSize; - if (GET_PROTOCOL_MINOR(remoteVersion) >= 3) { - to << repeats // == build-repeat + conn.to << cmdBuildDerivation << localStore->printStorePath(step->drvPath); + writeDerivation(conn.to, *localStore, BasicDerivation(*step->drv)); + conn.to << maxSilentTime << buildTimeout; + if (GET_PROTOCOL_MINOR(conn.remoteVersion) >= 2) + conn.to << maxLogSize; + if (GET_PROTOCOL_MINOR(conn.remoteVersion) >= 3) { + conn.to << repeats // == build-repeat << step->isDeterministic; // == enforce-determinism } - to.flush(); + conn.to.flush(); result.startTime = time(0); int res; { MaintainCount mc(nrStepsBuilding); - res = readInt(from); + res = readInt(conn.from); } result.stopTime = time(0); - result.errorMsg = readString(from); - if (GET_PROTOCOL_MINOR(remoteVersion) >= 3) { - result.timesBuilt = readInt(from); - result.isNonDeterministic = readInt(from); - auto start = readInt(from); - auto stop = readInt(from); + result.errorMsg = readString(conn.from); + if (GET_PROTOCOL_MINOR(conn.remoteVersion) >= 3) { + result.timesBuilt = readInt(conn.from); + result.isNonDeterministic = readInt(conn.from); + auto start = readInt(conn.from); + auto stop = readInt(conn.from); if (start && start) { /* Note: this represents the duration of a single round, rather than all rounds. */ @@ -374,8 +394,8 @@ void State::buildRemote(ref destStore, result.stopTime = stop; } } - if (GET_PROTOCOL_MINOR(remoteVersion) >= 6) { - worker_proto::read(*localStore, from, Phantom {}); + if (GET_PROTOCOL_MINOR(conn.remoteVersion) >= 6) { + worker_proto::read(*localStore, conn.from, Phantom {}); } switch ((BuildResult::Status) res) { case BuildResult::Built: @@ -451,19 +471,19 @@ void State::buildRemote(ref destStore, /* Get info about each output path. */ std::map infos; size_t totalNarSize = 0; - to << cmdQueryPathInfos; - worker_proto::write(*localStore, to, outputs); - to.flush(); + conn.to << cmdQueryPathInfos; + worker_proto::write(*localStore, conn.to, outputs); + conn.to.flush(); while (true) { - auto storePathS = readString(from); + auto storePathS = readString(conn.from); if (storePathS == "") break; - auto deriver = readString(from); // deriver - auto references = worker_proto::read(*localStore, from, Phantom {}); - readLongLong(from); // download size - auto narSize = readLongLong(from); - auto narHash = Hash::parseAny(readString(from), htSHA256); - auto ca = parseContentAddressOpt(readString(from)); - readStrings(from); // sigs + auto deriver = readString(conn.from); // deriver + auto references = worker_proto::read(*localStore, conn.from, Phantom {}); + readLongLong(conn.from); // download size + auto narSize = readLongLong(conn.from); + auto narHash = Hash::parseAny(readString(conn.from), htSHA256); + auto ca = parseContentAddressOpt(readString(conn.from)); + readStrings(conn.from); // sigs ValidPathInfo info(localStore->parseStorePath(storePathS), narHash); assert(outputs.count(info.path)); info.references = references; @@ -502,10 +522,10 @@ void State::buildRemote(ref destStore, lambda function only gets executed if someone tries to read from source2, we will send the command from here rather than outside the lambda. */ - to << cmdDumpStorePath << localStore->printStorePath(path); - to.flush(); + conn.to << cmdDumpStorePath << localStore->printStorePath(path); + conn.to.flush(); - TeeSource tee(from, sink); + TeeSource tee(conn.from, sink); extractNarData(tee, localStore->printStorePath(path), narMembers); }); From 365776f5d77112842d4a094a596cb725123b535d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Mon, 21 Mar 2022 12:14:37 +0100 Subject: [PATCH 006/313] Factor out the building part --- src/hydra-queue-runner/build-remote.cc | 207 ++++++++++++++++--------- src/hydra-queue-runner/state.hh | 2 + 2 files changed, 132 insertions(+), 77 deletions(-) diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc index 360a8ef7..7934d401 100644 --- a/src/hydra-queue-runner/build-remote.cc +++ b/src/hydra-queue-runner/build-remote.cc @@ -269,6 +269,121 @@ StorePathSet sendInputs( return inputs; } +struct BuildOptions { + unsigned int maxSilentTime, buildTimeout, repeats; + size_t maxLogSize; + bool enforceDeterminism; +}; + +void RemoteResult::updateWithBuildResult(const nix::BuildResult & buildResult) +{ + RemoteResult thisArrow; + + startTime = buildResult.startTime; + stopTime = buildResult.stopTime; + timesBuilt = buildResult.timesBuilt; + errorMsg = buildResult.errorMsg; + isNonDeterministic = buildResult.isNonDeterministic; + + switch ((BuildResult::Status) buildResult.status) { + case BuildResult::Built: + stepStatus = bsSuccess; + break; + case BuildResult::Substituted: + case BuildResult::AlreadyValid: + stepStatus = bsSuccess; + isCached = true; + break; + case BuildResult::PermanentFailure: + stepStatus = bsFailed; + canCache = true; + errorMsg = ""; + break; + case BuildResult::InputRejected: + case BuildResult::OutputRejected: + stepStatus = bsFailed; + canCache = true; + break; + case BuildResult::TransientFailure: + stepStatus = bsFailed; + canRetry = true; + errorMsg = ""; + break; + case BuildResult::TimedOut: + stepStatus = bsTimedOut; + errorMsg = ""; + break; + case BuildResult::MiscFailure: + stepStatus = bsAborted; + canRetry = true; + break; + case BuildResult::LogLimitExceeded: + stepStatus = bsLogLimitExceeded; + break; + case BuildResult::NotDeterministic: + stepStatus = bsNotDeterministic; + canRetry = false; + canCache = true; + break; + default: + stepStatus = bsAborted; + break; + } + +} + +BuildResult performBuild( + Machine::Connection & conn, + Store & localStore, + StorePath drvPath, + const BasicDerivation & drv, + const BuildOptions & options, + counter & nrStepsBuilding +) +{ + + BuildResult result; + + conn.to << cmdBuildDerivation << localStore.printStorePath(drvPath); + writeDerivation(conn.to, localStore, drv); + conn.to << options.maxSilentTime << options.buildTimeout; + if (GET_PROTOCOL_MINOR(conn.remoteVersion) >= 2) + conn.to << options.maxLogSize; + if (GET_PROTOCOL_MINOR(conn.remoteVersion) >= 3) { + conn.to << options.repeats // == build-repeat + << options.enforceDeterminism; + } + conn.to.flush(); + + result.startTime = time(0); + + { + MaintainCount mc(nrStepsBuilding); + result.status = (BuildResult::Status)readInt(conn.from); + } + result.stopTime = time(0); + + + result.errorMsg = readString(conn.from); + if (GET_PROTOCOL_MINOR(conn.remoteVersion) >= 3) { + result.timesBuilt = readInt(conn.from); + result.isNonDeterministic = readInt(conn.from); + auto start = readInt(conn.from); + auto stop = readInt(conn.from); + if (start && start) { + /* Note: this represents the duration of a single + round, rather than all rounds. */ + result.startTime = start; + result.stopTime = stop; + } + } + if (GET_PROTOCOL_MINOR(conn.remoteVersion) >= 6) { + result.builtOutputs = worker_proto::read(localStore, conn.from, Phantom {}); + } + + return result; +} + void State::buildRemote(ref destStore, Machine::ptr machine, Step::ptr step, unsigned int maxSilentTime, unsigned int buildTimeout, unsigned int repeats, @@ -362,85 +477,23 @@ void State::buildRemote(ref destStore, updateStep(ssBuilding); - conn.to << cmdBuildDerivation << localStore->printStorePath(step->drvPath); - writeDerivation(conn.to, *localStore, BasicDerivation(*step->drv)); - conn.to << maxSilentTime << buildTimeout; - if (GET_PROTOCOL_MINOR(conn.remoteVersion) >= 2) - conn.to << maxLogSize; - if (GET_PROTOCOL_MINOR(conn.remoteVersion) >= 3) { - conn.to << repeats // == build-repeat - << step->isDeterministic; // == enforce-determinism - } - conn.to.flush(); + BuildResult buildResult = performBuild( + conn, + *localStore, + step->drvPath, + BasicDerivation(*step->drv), + { + .maxSilentTime = maxSilentTime, + .buildTimeout = buildTimeout, + .repeats = repeats, + .maxLogSize = maxLogSize, + .enforceDeterminism = step->isDeterministic, + }, + nrStepsBuilding + ); - result.startTime = time(0); - int res; - { - MaintainCount mc(nrStepsBuilding); - res = readInt(conn.from); - } - result.stopTime = time(0); + result.updateWithBuildResult(buildResult); - result.errorMsg = readString(conn.from); - if (GET_PROTOCOL_MINOR(conn.remoteVersion) >= 3) { - result.timesBuilt = readInt(conn.from); - result.isNonDeterministic = readInt(conn.from); - auto start = readInt(conn.from); - auto stop = readInt(conn.from); - if (start && start) { - /* Note: this represents the duration of a single - round, rather than all rounds. */ - result.startTime = start; - result.stopTime = stop; - } - } - if (GET_PROTOCOL_MINOR(conn.remoteVersion) >= 6) { - worker_proto::read(*localStore, conn.from, Phantom {}); - } - switch ((BuildResult::Status) res) { - case BuildResult::Built: - result.stepStatus = bsSuccess; - break; - case BuildResult::Substituted: - case BuildResult::AlreadyValid: - result.stepStatus = bsSuccess; - result.isCached = true; - break; - case BuildResult::PermanentFailure: - result.stepStatus = bsFailed; - result.canCache = true; - result.errorMsg = ""; - break; - case BuildResult::InputRejected: - case BuildResult::OutputRejected: - result.stepStatus = bsFailed; - result.canCache = true; - break; - case BuildResult::TransientFailure: - result.stepStatus = bsFailed; - result.canRetry = true; - result.errorMsg = ""; - break; - case BuildResult::TimedOut: - result.stepStatus = bsTimedOut; - result.errorMsg = ""; - break; - case BuildResult::MiscFailure: - result.stepStatus = bsAborted; - result.canRetry = true; - break; - case BuildResult::LogLimitExceeded: - result.stepStatus = bsLogLimitExceeded; - break; - case BuildResult::NotDeterministic: - result.stepStatus = bsNotDeterministic; - result.canRetry = false; - result.canCache = true; - break; - default: - result.stepStatus = bsAborted; - break; - } if (result.stepStatus != bsSuccess) return; result.errorMsg = ""; diff --git a/src/hydra-queue-runner/state.hh b/src/hydra-queue-runner/state.hh index 059b03a1..6292a2db 100644 --- a/src/hydra-queue-runner/state.hh +++ b/src/hydra-queue-runner/state.hh @@ -72,6 +72,8 @@ struct RemoteResult { return stepStatus == bsCachedFailure ? bsFailed : stepStatus; } + + void updateWithBuildResult(const nix::BuildResult &); }; From a778a89f0424b5bffa66818cf0b46ffa945403c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Mon, 21 Mar 2022 15:16:32 +0100 Subject: [PATCH 007/313] Factor out the `queryPathInfos` part of the build --- src/hydra-queue-runner/build-remote.cc | 65 +++++++++++++++----------- 1 file changed, 39 insertions(+), 26 deletions(-) diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc index 7934d401..0451ccb7 100644 --- a/src/hydra-queue-runner/build-remote.cc +++ b/src/hydra-queue-runner/build-remote.cc @@ -384,6 +384,44 @@ BuildResult performBuild( return result; } +std::map queryPathInfos( + Machine::Connection & conn, + Store & localStore, + StorePathSet & outputs, + size_t & totalNarSize +) +{ + + /* Get info about each output path. */ + std::map infos; + conn.to << cmdQueryPathInfos; + worker_proto::write(localStore, conn.to, outputs); + conn.to.flush(); + while (true) { + auto storePathS = readString(conn.from); + if (storePathS == "") break; + auto deriver = readString(conn.from); // deriver + auto references = worker_proto::read(localStore, conn.from, Phantom {}); + readLongLong(conn.from); // download size + auto narSize = readLongLong(conn.from); + auto narHash = Hash::parseAny(readString(conn.from), htSHA256); + auto ca = parseContentAddressOpt(readString(conn.from)); + readStrings(conn.from); // sigs + ValidPathInfo info(localStore.parseStorePath(storePathS), narHash); + assert(outputs.count(info.path)); + info.references = references; + info.narSize = narSize; + totalNarSize += info.narSize; + info.narHash = narHash; + info.ca = ca; + if (deriver != "") + info.deriver = localStore.parseStorePath(deriver); + infos.insert_or_assign(info.path, info); + } + + return infos; +} + void State::buildRemote(ref destStore, Machine::ptr machine, Step::ptr step, unsigned int maxSilentTime, unsigned int buildTimeout, unsigned int repeats, @@ -521,33 +559,8 @@ void State::buildRemote(ref destStore, outputs.insert(*i.second.second); } - /* Get info about each output path. */ - std::map infos; size_t totalNarSize = 0; - conn.to << cmdQueryPathInfos; - worker_proto::write(*localStore, conn.to, outputs); - conn.to.flush(); - while (true) { - auto storePathS = readString(conn.from); - if (storePathS == "") break; - auto deriver = readString(conn.from); // deriver - auto references = worker_proto::read(*localStore, conn.from, Phantom {}); - readLongLong(conn.from); // download size - auto narSize = readLongLong(conn.from); - auto narHash = Hash::parseAny(readString(conn.from), htSHA256); - auto ca = parseContentAddressOpt(readString(conn.from)); - readStrings(conn.from); // sigs - ValidPathInfo info(localStore->parseStorePath(storePathS), narHash); - assert(outputs.count(info.path)); - info.references = references; - info.narSize = narSize; - totalNarSize += info.narSize; - info.narHash = narHash; - info.ca = ca; - if (deriver != "") - info.deriver = localStore->parseStorePath(deriver); - infos.insert_or_assign(info.path, info); - } + auto infos = queryPathInfos(conn, *localStore, outputs, totalNarSize); if (totalNarSize > maxOutputSize) { result.stepStatus = bsNarSizeLimitExceeded; From fd0ae78eba058ab456590da698b45082fcafb519 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Mon, 21 Mar 2022 15:26:31 +0100 Subject: [PATCH 008/313] Factor out the copying from the build store --- src/hydra-queue-runner/build-remote.cc | 76 +++++++++++++++++--------- 1 file changed, 49 insertions(+), 27 deletions(-) diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc index 0451ccb7..79e5a231 100644 --- a/src/hydra-queue-runner/build-remote.cc +++ b/src/hydra-queue-runner/build-remote.cc @@ -422,6 +422,54 @@ std::map queryPathInfos( return infos; } +void copyPathFromRemote( + Machine::Connection & conn, + NarMemberDatas & narMembers, + Store & localStore, + Store & destStore, + const ValidPathInfo & info +) +{ + /* Receive the NAR from the remote and add it to the + destination store. Meanwhile, extract all the info from the + NAR that getBuildOutput() needs. */ + auto source2 = sinkToSource([&](Sink & sink) + { + /* Note: we should only send the command to dump the store + path to the remote if the NAR is actually going to get read + by the destination store, which won't happen if this path + is already valid on the destination store. Since this + lambda function only gets executed if someone tries to read + from source2, we will send the command from here rather + than outside the lambda. */ + conn.to << cmdDumpStorePath << localStore.printStorePath(info.path); + conn.to.flush(); + + TeeSource tee(conn.from, sink); + extractNarData(tee, localStore.printStorePath(info.path), narMembers); + }); + + destStore.addToStore(info, *source2, NoRepair, NoCheckSigs); +} + +void copyPathsFromRemote( + Machine::Connection & conn, + NarMemberDatas & narMembers, + Store & localStore, + Store & destStore, + const std::map & infos +) +{ + auto pathsSorted = reverseTopoSortPaths(infos); + + for (auto & path : pathsSorted) { + auto & info = infos.find(path)->second; + copyPathFromRemote(conn, narMembers, localStore, destStore, info); + } + +} + + void State::buildRemote(ref destStore, Machine::ptr machine, Step::ptr step, unsigned int maxSilentTime, unsigned int buildTimeout, unsigned int repeats, @@ -571,33 +619,7 @@ void State::buildRemote(ref destStore, printMsg(lvlDebug, "copying outputs of ‘%s’ from ‘%s’ (%d bytes)", localStore->printStorePath(step->drvPath), machine->sshName, totalNarSize); - auto pathsSorted = reverseTopoSortPaths(infos); - - for (auto & path : pathsSorted) { - auto & info = infos.find(path)->second; - - /* Receive the NAR from the remote and add it to the - destination store. Meanwhile, extract all the info from the - NAR that getBuildOutput() needs. */ - auto source2 = sinkToSource([&](Sink & sink) - { - /* Note: we should only send the command to dump the store - path to the remote if the NAR is actually going to get read - by the destination store, which won't happen if this path - is already valid on the destination store. Since this - lambda function only gets executed if someone tries to read - from source2, we will send the command from here rather - than outside the lambda. */ - conn.to << cmdDumpStorePath << localStore->printStorePath(path); - conn.to.flush(); - - TeeSource tee(conn.from, sink); - extractNarData(tee, localStore->printStorePath(path), narMembers); - }); - - destStore->addToStore(info, *source2, NoRepair, NoCheckSigs); - } - + copyPathsFromRemote(conn, narMembers, *localStore, *destStore, infos); auto now2 = std::chrono::steady_clock::now(); result.overhead += std::chrono::duration_cast(now2 - now1).count(); From b430d41afd6a4ca0e38343ae2eee4aa6307cf980 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Mon, 21 Mar 2022 16:33:25 +0100 Subject: [PATCH 009/313] Use the `BuildOptions` more eagerly --- src/hydra-queue-runner/build-remote.cc | 20 ++++---------------- src/hydra-queue-runner/builder.cc | 16 +++++++++------- src/hydra-queue-runner/state.hh | 9 +++++++-- 3 files changed, 20 insertions(+), 25 deletions(-) diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc index 79e5a231..1e06e501 100644 --- a/src/hydra-queue-runner/build-remote.cc +++ b/src/hydra-queue-runner/build-remote.cc @@ -269,12 +269,6 @@ StorePathSet sendInputs( return inputs; } -struct BuildOptions { - unsigned int maxSilentTime, buildTimeout, repeats; - size_t maxLogSize; - bool enforceDeterminism; -}; - void RemoteResult::updateWithBuildResult(const nix::BuildResult & buildResult) { RemoteResult thisArrow; @@ -337,7 +331,7 @@ BuildResult performBuild( Store & localStore, StorePath drvPath, const BasicDerivation & drv, - const BuildOptions & options, + const State::BuildOptions & options, counter & nrStepsBuilding ) { @@ -472,7 +466,7 @@ void copyPathsFromRemote( void State::buildRemote(ref destStore, Machine::ptr machine, Step::ptr step, - unsigned int maxSilentTime, unsigned int buildTimeout, unsigned int repeats, + const BuildOptions & buildOptions, RemoteResult & result, std::shared_ptr activeStep, std::function updateStep, NarMemberDatas & narMembers) @@ -523,7 +517,7 @@ void State::buildRemote(ref destStore, }); try { - handshake(conn, repeats); + handshake(conn, buildOptions.repeats); } catch (EndOfFile & e) { child.pid.wait(); std::string s = chomp(readFile(result.logFile)); @@ -568,13 +562,7 @@ void State::buildRemote(ref destStore, *localStore, step->drvPath, BasicDerivation(*step->drv), - { - .maxSilentTime = maxSilentTime, - .buildTimeout = buildTimeout, - .repeats = repeats, - .maxLogSize = maxLogSize, - .enforceDeterminism = step->isDeterministic, - }, + buildOptions, nrStepsBuilding ); diff --git a/src/hydra-queue-runner/builder.cc b/src/hydra-queue-runner/builder.cc index 89aa7d15..b25b4e63 100644 --- a/src/hydra-queue-runner/builder.cc +++ b/src/hydra-queue-runner/builder.cc @@ -98,8 +98,10 @@ State::StepResult State::doBuildStep(nix::ref destStore, it). */ BuildID buildId; std::optional buildDrvPath; - unsigned int maxSilentTime, buildTimeout; - unsigned int repeats = step->isDeterministic ? 1 : 0; + BuildOptions buildOptions; + buildOptions.repeats = step->isDeterministic ? 1 : 0; + buildOptions.maxLogSize = maxLogSize; + buildOptions.enforceDeterminism = step->isDeterministic; auto conn(dbPool.get()); @@ -134,18 +136,18 @@ State::StepResult State::doBuildStep(nix::ref destStore, { auto i = jobsetRepeats.find(std::make_pair(build2->projectName, build2->jobsetName)); if (i != jobsetRepeats.end()) - repeats = std::max(repeats, i->second); + buildOptions.repeats = std::max(buildOptions.repeats, i->second); } } if (!build) build = *dependents.begin(); buildId = build->id; buildDrvPath = build->drvPath; - maxSilentTime = build->maxSilentTime; - buildTimeout = build->buildTimeout; + buildOptions.maxSilentTime = build->maxSilentTime; + buildOptions.buildTimeout = build->buildTimeout; printInfo("performing step ‘%s’ %d times on ‘%s’ (needed by build %d and %d others)", - localStore->printStorePath(step->drvPath), repeats + 1, machine->sshName, buildId, (dependents.size() - 1)); + localStore->printStorePath(step->drvPath), buildOptions.repeats + 1, machine->sshName, buildId, (dependents.size() - 1)); } if (!buildOneDone) @@ -206,7 +208,7 @@ State::StepResult State::doBuildStep(nix::ref destStore, try { /* FIXME: referring builds may have conflicting timeouts. */ - buildRemote(destStore, machine, step, maxSilentTime, buildTimeout, repeats, result, activeStep, updateStep, narMembers); + buildRemote(destStore, machine, step, buildOptions, result, activeStep, updateStep, narMembers); } catch (Error & e) { if (activeStep->state_.lock()->cancelled) { printInfo("marking step %d of build %d as cancelled", stepNr, buildId); diff --git a/src/hydra-queue-runner/state.hh b/src/hydra-queue-runner/state.hh index 6292a2db..f4d8ccce 100644 --- a/src/hydra-queue-runner/state.hh +++ b/src/hydra-queue-runner/state.hh @@ -447,6 +447,12 @@ private: public: State(); + struct BuildOptions { + unsigned int maxSilentTime, buildTimeout, repeats; + size_t maxLogSize; + bool enforceDeterminism; + }; + private: nix::MaintainCount startDbUpdate(); @@ -531,8 +537,7 @@ private: void buildRemote(nix::ref destStore, Machine::ptr machine, Step::ptr step, - unsigned int maxSilentTime, unsigned int buildTimeout, - unsigned int repeats, + const BuildOptions & buildOptions, RemoteResult & result, std::shared_ptr activeStep, std::function updateStep, NarMemberDatas & narMembers); From 92b627ac1b3e0f53f6b35fc5940406c12fa977da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= <7226587+thufschmitt@users.noreply.github.com> Date: Thu, 24 Mar 2022 09:39:24 +0100 Subject: [PATCH 010/313] Remove an accidental re-indenting of a comment Co-authored-by: Eelco Dolstra --- src/hydra-queue-runner/build-remote.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc index 1e06e501..901bbc89 100644 --- a/src/hydra-queue-runner/build-remote.cc +++ b/src/hydra-queue-runner/build-remote.cc @@ -233,9 +233,9 @@ StorePathSet sendInputs( } /* Ensure that the inputs exist in the destination store. This is - a no-op for regular stores, but for the binary cache store, - this will copy the inputs to the binary cache from the local - store. */ + a no-op for regular stores, but for the binary cache store, + this will copy the inputs to the binary cache from the local + store. */ if (localStore.getUri() != destStore.getUri()) { StorePathSet closure; localStore.computeFSClosure(step.drv->inputSrcs, closure); From 6e571e26ff068386bc949eaab6646e90613b23c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Thu, 24 Mar 2022 14:27:45 +0100 Subject: [PATCH 011/313] Build the resolved derivation and not the original one --- src/hydra-queue-runner/build-remote.cc | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc index 901bbc89..62461a65 100644 --- a/src/hydra-queue-runner/build-remote.cc +++ b/src/hydra-queue-runner/build-remote.cc @@ -203,7 +203,7 @@ void handshake(Machine::Connection & conn, unsigned int repeats) throw Error("machine ‘%1%’ does not support repeating a build; please upgrade it to Nix 1.12", conn.machine->sshName); } -StorePathSet sendInputs( +BasicDerivation sendInputs( State & state, Step & step, Store & localStore, @@ -214,19 +214,13 @@ StorePathSet sendInputs( counter & nrStepsCopyingTo ) { - - StorePathSet inputs; BasicDerivation basicDrv(*step.drv); - for (auto & p : step.drv->inputSrcs) - inputs.insert(p); - for (auto & input : step.drv->inputDrvs) { auto drv2 = localStore.readDerivation(input.first); for (auto & name : input.second) { if (auto i = get(drv2.outputs, name)) { auto outPath = i->path(localStore, drv2.name, name); - inputs.insert(*outPath); basicDrv.inputSrcs.insert(*outPath); } } @@ -255,10 +249,10 @@ StorePathSet sendInputs( /* Copy the input closure. */ if (conn.machine->isLocalhost()) { StorePathSet closure; - destStore.computeFSClosure(inputs, closure); + destStore.computeFSClosure(basicDrv.inputSrcs, closure); copyPaths(destStore, localStore, closure, NoRepair, NoCheckSigs, NoSubstitute); } else { - copyClosureTo(conn.machine->state->sendLock, destStore, conn.from, conn.to, inputs, true); + copyClosureTo(conn.machine->state->sendLock, destStore, conn.from, conn.to, basicDrv.inputSrcs, true); } auto now2 = std::chrono::steady_clock::now(); @@ -266,7 +260,7 @@ StorePathSet sendInputs( overhead += std::chrono::duration_cast(now2 - now1).count(); } - return inputs; + return basicDrv; } void RemoteResult::updateWithBuildResult(const nix::BuildResult & buildResult) @@ -535,8 +529,7 @@ void State::buildRemote(ref destStore, copy the immediate sources of the derivation and the required outputs of the input derivations. */ updateStep(ssSendingInputs); - - StorePathSet inputs = sendInputs(*this, *step, *localStore, *destStore, conn, result.overhead, nrStepsWaiting, nrStepsCopyingTo); + BasicDerivation resolvedDrv = sendInputs(*this, *step, *localStore, *destStore, conn, result.overhead, nrStepsWaiting, nrStepsCopyingTo); logFileDel.cancel(); @@ -561,7 +554,7 @@ void State::buildRemote(ref destStore, conn, *localStore, step->drvPath, - BasicDerivation(*step->drv), + resolvedDrv, buildOptions, nrStepsBuilding ); From cba85a6a196d26eb1090af95e12a81434947cef8 Mon Sep 17 00:00:00 2001 From: Ivor Wanders Date: Wed, 4 May 2022 13:31:31 -0400 Subject: [PATCH 012/313] Add a link to the raw log. --- src/root/log.tt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/root/log.tt b/src/root/log.tt index 8bb4954d..1c5fd3b5 100644 --- a/src/root/log.tt +++ b/src/root/log.tt @@ -11,7 +11,8 @@ [% ELSE %] is [% END %] - the build log of derivation [% IF step; step.drvpath; ELSE; build.drvpath; END %]. + the build log (raw) of derivation [% IF step; step.drvpath; ELSE; build.drvpath; END %]. [% IF step && step.machine %] It was built on [% step.machine %]. [% END %] From 750978a19232583e17620a1bd80435e957e7213a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sandro=20J=C3=A4ckel?= Date: Sat, 18 Jun 2022 13:22:42 +0200 Subject: [PATCH 013/313] Add gitea push hook --- doc/manual/src/webhooks.md | 20 +++++++++++++++++--- src/lib/Hydra/Controller/API.pm | 16 ++++++++++++++++ src/lib/Hydra/Controller/Root.pm | 3 ++- 3 files changed, 35 insertions(+), 4 deletions(-) diff --git a/doc/manual/src/webhooks.md b/doc/manual/src/webhooks.md index 2b26cd61..674e1064 100644 --- a/doc/manual/src/webhooks.md +++ b/doc/manual/src/webhooks.md @@ -1,9 +1,12 @@ # Webhooks -Hydra can be notified by github's webhook to trigger a new evaluation when a +Hydra can be notified by github or gitea with webhooks to trigger a new evaluation when a jobset has a github repo in its input. -To set up a github webhook go to `https://github.com///settings` and in the `Webhooks` tab -click on `Add webhook`. + +## GitHub + +To set up a webhook for a GitHub repository go to `https://github.com///settings` +and in the `Webhooks` tab click on `Add webhook`. - In `Payload URL` fill in `https:///api/push-github`. - In `Content type` switch to `application/json`. @@ -11,3 +14,14 @@ click on `Add webhook`. - For `Which events would you like to trigger this webhook?` keep the default option for events on `Just the push event.`. Then add the hook with `Add webhook`. + +## Gitea + +To set up a webhook for a Gitea repository go to the settings of the repository in your Gitea instance +and in the `Webhooks` tab click on `Add Webhook` and choose `Gitea` in the drop down. + +- In `Target URL` fill in `https:///api/push-gitea`. +- Keep HTTP method `POST`, POST Content Type `application/json` and Trigger On `Push Events`. +- Change the branch filter to match the git branch hydra builds. + +Then add the hook with `Add webhook`. diff --git a/src/lib/Hydra/Controller/API.pm b/src/lib/Hydra/Controller/API.pm index 6f10ef57..12073595 100644 --- a/src/lib/Hydra/Controller/API.pm +++ b/src/lib/Hydra/Controller/API.pm @@ -285,6 +285,22 @@ sub push_github : Chained('api') PathPart('push-github') Args(0) { $c->response->body(""); } +sub push_gitea : Chained('api') PathPart('push-gitea') Args(0) { + my ($self, $c) = @_; + + $c->{stash}->{json}->{jobsetsTriggered} = []; + + my $in = $c->request->{data}; + my $url = $in->{repository}->{clone_url} or die; + print STDERR "got push from Gitea repository $url\n"; + + triggerJobset($self, $c, $_, 0) foreach $c->model('DB::Jobsets')->search( + { 'project.enabled' => 1, 'me.enabled' => 1 }, + { join => 'project' + , where => \ [ 'me.flake like ? or exists (select 1 from JobsetInputAlts where project = me.project and jobset = me.name and value like ?)', [ 'flake', "%$url%"], [ 'value', "%$url%" ] ] + }); + $c->response->body(""); +} 1; diff --git a/src/lib/Hydra/Controller/Root.pm b/src/lib/Hydra/Controller/Root.pm index c6843d29..1b33db2a 100644 --- a/src/lib/Hydra/Controller/Root.pm +++ b/src/lib/Hydra/Controller/Root.pm @@ -32,6 +32,7 @@ sub noLoginNeeded { return $whitelisted || $c->request->path eq "api/push-github" || + $c->request->path eq "api/push-gitea" || $c->request->path eq "google-login" || $c->request->path eq "github-redirect" || $c->request->path eq "github-login" || @@ -77,7 +78,7 @@ sub begin :Private { $_->supportedInputTypes($c->stash->{inputTypes}) foreach @{$c->hydra_plugins}; # XSRF protection: require POST requests to have the same origin. - if ($c->req->method eq "POST" && $c->req->path ne "api/push-github") { + if ($c->req->method eq "POST" && $c->req->path ne "api/push-github" && $c->req->path ne "api/push-gitea") { my $referer = $c->req->header('Referer'); $referer //= $c->req->header('Origin'); my $base = $c->req->base; From a81c6a3a80d1055aa80934ab229e2dc49594edd2 Mon Sep 17 00:00:00 2001 From: Sandro Date: Fri, 1 Jul 2022 22:21:32 +0200 Subject: [PATCH 014/313] Match URIs that don't end in .git Co-authored-by: Charlotte --- src/lib/Hydra/Controller/API.pm | 1 + 1 file changed, 1 insertion(+) diff --git a/src/lib/Hydra/Controller/API.pm b/src/lib/Hydra/Controller/API.pm index 12073595..5eeb0c04 100644 --- a/src/lib/Hydra/Controller/API.pm +++ b/src/lib/Hydra/Controller/API.pm @@ -292,6 +292,7 @@ sub push_gitea : Chained('api') PathPart('push-gitea') Args(0) { my $in = $c->request->{data}; my $url = $in->{repository}->{clone_url} or die; + $url =~ s/.git$//; print STDERR "got push from Gitea repository $url\n"; triggerJobset($self, $c, $_, 0) foreach $c->model('DB::Jobsets')->search( From 143c31734fc19d72a3ccfb7c064d829af9bfca72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Tue, 25 Oct 2022 10:04:29 +0200 Subject: [PATCH 015/313] Move all the build remote utils to their namespace Just don't pollute the global one --- src/hydra-queue-runner/build-remote.cc | 134 +++++++++++++------------ 1 file changed, 70 insertions(+), 64 deletions(-) diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc index 62461a65..a429467b 100644 --- a/src/hydra-queue-runner/build-remote.cc +++ b/src/hydra-queue-runner/build-remote.cc @@ -27,6 +27,8 @@ static void append(Strings & dst, const Strings & src) dst.insert(dst.end(), src.begin(), src.end()); } +namespace nix::build_remote { + static Strings extraStoreArgs(std::string & machine) { Strings result; @@ -263,63 +265,6 @@ BasicDerivation sendInputs( return basicDrv; } -void RemoteResult::updateWithBuildResult(const nix::BuildResult & buildResult) -{ - RemoteResult thisArrow; - - startTime = buildResult.startTime; - stopTime = buildResult.stopTime; - timesBuilt = buildResult.timesBuilt; - errorMsg = buildResult.errorMsg; - isNonDeterministic = buildResult.isNonDeterministic; - - switch ((BuildResult::Status) buildResult.status) { - case BuildResult::Built: - stepStatus = bsSuccess; - break; - case BuildResult::Substituted: - case BuildResult::AlreadyValid: - stepStatus = bsSuccess; - isCached = true; - break; - case BuildResult::PermanentFailure: - stepStatus = bsFailed; - canCache = true; - errorMsg = ""; - break; - case BuildResult::InputRejected: - case BuildResult::OutputRejected: - stepStatus = bsFailed; - canCache = true; - break; - case BuildResult::TransientFailure: - stepStatus = bsFailed; - canRetry = true; - errorMsg = ""; - break; - case BuildResult::TimedOut: - stepStatus = bsTimedOut; - errorMsg = ""; - break; - case BuildResult::MiscFailure: - stepStatus = bsAborted; - canRetry = true; - break; - case BuildResult::LogLimitExceeded: - stepStatus = bsLogLimitExceeded; - break; - case BuildResult::NotDeterministic: - stepStatus = bsNotDeterministic; - canRetry = false; - canCache = true; - break; - default: - stepStatus = bsAborted; - break; - } - -} - BuildResult performBuild( Machine::Connection & conn, Store & localStore, @@ -457,6 +402,67 @@ void copyPathsFromRemote( } +} + +/* using namespace nix::build_remote; */ + +void RemoteResult::updateWithBuildResult(const nix::BuildResult & buildResult) +{ + RemoteResult thisArrow; + + startTime = buildResult.startTime; + stopTime = buildResult.stopTime; + timesBuilt = buildResult.timesBuilt; + errorMsg = buildResult.errorMsg; + isNonDeterministic = buildResult.isNonDeterministic; + + switch ((BuildResult::Status) buildResult.status) { + case BuildResult::Built: + stepStatus = bsSuccess; + break; + case BuildResult::Substituted: + case BuildResult::AlreadyValid: + stepStatus = bsSuccess; + isCached = true; + break; + case BuildResult::PermanentFailure: + stepStatus = bsFailed; + canCache = true; + errorMsg = ""; + break; + case BuildResult::InputRejected: + case BuildResult::OutputRejected: + stepStatus = bsFailed; + canCache = true; + break; + case BuildResult::TransientFailure: + stepStatus = bsFailed; + canRetry = true; + errorMsg = ""; + break; + case BuildResult::TimedOut: + stepStatus = bsTimedOut; + errorMsg = ""; + break; + case BuildResult::MiscFailure: + stepStatus = bsAborted; + canRetry = true; + break; + case BuildResult::LogLimitExceeded: + stepStatus = bsLogLimitExceeded; + break; + case BuildResult::NotDeterministic: + stepStatus = bsNotDeterministic; + canRetry = false; + canCache = true; + break; + default: + stepStatus = bsAborted; + break; + } + +} + void State::buildRemote(ref destStore, Machine::ptr machine, Step::ptr step, @@ -467,7 +473,7 @@ void State::buildRemote(ref destStore, { assert(BuildResult::TimedOut == 8); - auto [logFile, logFD] = openLogFile(logDir, step->drvPath); + auto [logFile, logFD] = build_remote::openLogFile(logDir, step->drvPath); AutoDelete logFileDel(logFile, false); result.logFile = logFile; @@ -480,7 +486,7 @@ void State::buildRemote(ref destStore, // FIXME: rewrite to use Store. Child child; - openConnection(machine, tmpDir, logFD.get(), child); + build_remote::openConnection(machine, tmpDir, logFD.get(), child); { auto activeStepState(activeStep->state_.lock()); @@ -511,7 +517,7 @@ void State::buildRemote(ref destStore, }); try { - handshake(conn, buildOptions.repeats); + build_remote::handshake(conn, buildOptions.repeats); } catch (EndOfFile & e) { child.pid.wait(); std::string s = chomp(readFile(result.logFile)); @@ -529,7 +535,7 @@ void State::buildRemote(ref destStore, copy the immediate sources of the derivation and the required outputs of the input derivations. */ updateStep(ssSendingInputs); - BasicDerivation resolvedDrv = sendInputs(*this, *step, *localStore, *destStore, conn, result.overhead, nrStepsWaiting, nrStepsCopyingTo); + BasicDerivation resolvedDrv = build_remote::sendInputs(*this, *step, *localStore, *destStore, conn, result.overhead, nrStepsWaiting, nrStepsCopyingTo); logFileDel.cancel(); @@ -550,7 +556,7 @@ void State::buildRemote(ref destStore, updateStep(ssBuilding); - BuildResult buildResult = performBuild( + BuildResult buildResult = build_remote::performBuild( conn, *localStore, step->drvPath, @@ -589,7 +595,7 @@ void State::buildRemote(ref destStore, } size_t totalNarSize = 0; - auto infos = queryPathInfos(conn, *localStore, outputs, totalNarSize); + auto infos = build_remote::queryPathInfos(conn, *localStore, outputs, totalNarSize); if (totalNarSize > maxOutputSize) { result.stepStatus = bsNarSizeLimitExceeded; @@ -600,7 +606,7 @@ void State::buildRemote(ref destStore, printMsg(lvlDebug, "copying outputs of ‘%s’ from ‘%s’ (%d bytes)", localStore->printStorePath(step->drvPath), machine->sshName, totalNarSize); - copyPathsFromRemote(conn, narMembers, *localStore, *destStore, infos); + build_remote::copyPathsFromRemote(conn, narMembers, *localStore, *destStore, infos); auto now2 = std::chrono::steady_clock::now(); result.overhead += std::chrono::duration_cast(now2 - now1).count(); From d1d171ee9010be514bb7b95866ac275213ed4286 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niklas=20Hamb=C3=BCchen?= Date: Wed, 2 Nov 2022 17:30:32 +0100 Subject: [PATCH 016/313] renderInputDiff: Increase git hash length 6 -> 8 Nixpkgs has so many commits that length 6 is often ambiguous, making the use of the shown values with git difficult. --- src/root/common.tt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/root/common.tt b/src/root/common.tt index 32d6c8cc..4487cbe3 100644 --- a/src/root/common.tt +++ b/src/root/common.tt @@ -374,7 +374,7 @@ BLOCK renderInputDiff; %] [% ELSIF bi1.uri == bi2.uri && bi1.revision != bi2.revision %] [% IF bi1.type == "git" %] - [% bi1.name %][% INCLUDE renderDiffUri contents=(bi1.revision.substr(0, 6) _ ' to ' _ bi2.revision.substr(0, 6)) %] + [% bi1.name %][% INCLUDE renderDiffUri contents=(bi1.revision.substr(0, 8) _ ' to ' _ bi2.revision.substr(0, 8)) %] [% ELSE %] From ad99d3366f8a0088782904fc66958fdf614c12db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Josef=20Kemetm=C3=BCller?= Date: Thu, 29 Dec 2022 22:26:59 +0100 Subject: [PATCH 017/313] Fix MIME types when serving .js and .css To correctly render HTML reports we make sure to return the following MIME types instead of "text/plain" - *.css: "text/css" - *.js: "application/javascript" Fixes: #1267 --- src/lib/Hydra/Controller/Build.pm | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/lib/Hydra/Controller/Build.pm b/src/lib/Hydra/Controller/Build.pm index 18a0eba3..2d74f86a 100644 --- a/src/lib/Hydra/Controller/Build.pm +++ b/src/lib/Hydra/Controller/Build.pm @@ -238,9 +238,17 @@ sub serveFile { "store", "cat", "--store", getStoreUri(), "$path"]) }; # Detect MIME type. - state $magic = File::LibMagic->new(follow_symlinks => 1); - my $info = $magic->info_from_filename($path); - my $type = $info->{mime_with_encoding}; + my $type = "text/plain"; + if ($path =~ /.*\.(\S{1,})$/xms) { + my $ext = $1; + my $mimeTypes = MIME::Types->new(only_complete => 1); + my $t = $mimeTypes->mimeTypeOf($ext); + $type = ref $t ? $t->type : $t if $t; + } else { + state $magic = File::LibMagic->new(follow_symlinks => 1); + my $info = $magic->info_from_filename($path); + $type = $info->{mime_with_encoding}; + } $c->response->content_type($type); $c->forward('Hydra::View::Plain'); } From 96e36201ebb7748d64f895947d198b370968edd0 Mon Sep 17 00:00:00 2001 From: Linus Heckemann Date: Tue, 29 Nov 2022 18:13:15 +0100 Subject: [PATCH 018/313] hydra-queue-runner: adapt to nlohmann::json --- src/hydra-queue-runner/hydra-queue-runner.cc | 249 +++++++++---------- 1 file changed, 114 insertions(+), 135 deletions(-) diff --git a/src/hydra-queue-runner/hydra-queue-runner.cc b/src/hydra-queue-runner/hydra-queue-runner.cc index 723bf223..b16fd770 100644 --- a/src/hydra-queue-runner/hydra-queue-runner.cc +++ b/src/hydra-queue-runner/hydra-queue-runner.cc @@ -8,6 +8,8 @@ #include +#include + #include "state.hh" #include "hydra-build-result.hh" #include "store-api.hh" @@ -15,20 +17,11 @@ #include "globals.hh" #include "hydra-config.hh" -#include "json.hh" #include "s3-binary-cache-store.hh" #include "shared.hh" using namespace nix; - - -namespace nix { - -template<> void toJSON>(std::ostream & str, const std::atomic & n) { str << n; } -template<> void toJSON>(std::ostream & str, const std::atomic & n) { str << n; } -template<> void toJSON(std::ostream & str, const double & n) { str << n; } - -} +using nlohmann::json; std::string getEnvOrDie(const std::string & key) @@ -542,181 +535,167 @@ std::shared_ptr State::acquireGlobalLock() void State::dumpStatus(Connection & conn) { - std::ostringstream out; - + auto root = json::object(); { - JSONObject root(out); time_t now = time(0); - root.attr("status", "up"); - root.attr("time", time(0)); - root.attr("uptime", now - startedAt); - root.attr("pid", getpid()); + root["status"] = "up"; + root["time"] = time(0); + root["uptime"] = now - startedAt; + root["pid"] = getpid(); { auto builds_(builds.lock()); - root.attr("nrQueuedBuilds", builds_->size()); + root["nrQueuedBuilds"] = builds_->size(); } { auto steps_(steps.lock()); for (auto i = steps_->begin(); i != steps_->end(); ) if (i->second.lock()) ++i; else i = steps_->erase(i); - root.attr("nrUnfinishedSteps", steps_->size()); + root["nrUnfinishedSteps"] = steps_->size(); } { auto runnable_(runnable.lock()); for (auto i = runnable_->begin(); i != runnable_->end(); ) if (i->lock()) ++i; else i = runnable_->erase(i); - root.attr("nrRunnableSteps", runnable_->size()); + root["nrRunnableSteps"] = runnable_->size(); } - root.attr("nrActiveSteps", activeSteps_.lock()->size()); - root.attr("nrStepsBuilding", nrStepsBuilding); - root.attr("nrStepsCopyingTo", nrStepsCopyingTo); - root.attr("nrStepsCopyingFrom", nrStepsCopyingFrom); - root.attr("nrStepsWaiting", nrStepsWaiting); - root.attr("nrUnsupportedSteps", nrUnsupportedSteps); - root.attr("bytesSent", bytesSent); - root.attr("bytesReceived", bytesReceived); - root.attr("nrBuildsRead", nrBuildsRead); - root.attr("buildReadTimeMs", buildReadTimeMs); - root.attr("buildReadTimeAvgMs", nrBuildsRead == 0 ? 0.0 : (float) buildReadTimeMs / nrBuildsRead); - root.attr("nrBuildsDone", nrBuildsDone); - root.attr("nrStepsStarted", nrStepsStarted); - root.attr("nrStepsDone", nrStepsDone); - root.attr("nrRetries", nrRetries); - root.attr("maxNrRetries", maxNrRetries); + root["nrActiveSteps"] = activeSteps_.lock()->size(); + root["nrStepsBuilding"] = nrStepsBuilding.load(); + root["nrStepsCopyingTo"] = nrStepsCopyingTo.load(); + root["nrStepsCopyingFrom"] = nrStepsCopyingFrom.load(); + root["nrStepsWaiting"] = nrStepsWaiting.load(); + root["nrUnsupportedSteps"] = nrUnsupportedSteps.load(); + root["bytesSent"] = bytesSent.load(); + root["bytesReceived"] = bytesReceived.load(); + root["nrBuildsRead"] = nrBuildsRead.load(); + root["buildReadTimeMs"] = buildReadTimeMs.load(); + root["buildReadTimeAvgMs"] = nrBuildsRead == 0 ? 0.0 : (float) buildReadTimeMs / nrBuildsRead; + root["nrBuildsDone"] = nrBuildsDone.load(); + root["nrStepsStarted"] = nrStepsStarted.load(); + root["nrStepsDone"] = nrStepsDone.load(); + root["nrRetries"] = nrRetries.load(); + root["maxNrRetries"] = maxNrRetries.load(); if (nrStepsDone) { - root.attr("totalStepTime", totalStepTime); - root.attr("totalStepBuildTime", totalStepBuildTime); - root.attr("avgStepTime", (float) totalStepTime / nrStepsDone); - root.attr("avgStepBuildTime", (float) totalStepBuildTime / nrStepsDone); + root["totalStepTime"] = totalStepTime.load(); + root["totalStepBuildTime"] = totalStepBuildTime.load(); + root["avgStepTime"] = (float) totalStepTime / nrStepsDone; + root["avgStepBuildTime"] = (float) totalStepBuildTime / nrStepsDone; } - root.attr("nrQueueWakeups", nrQueueWakeups); - root.attr("nrDispatcherWakeups", nrDispatcherWakeups); - root.attr("dispatchTimeMs", dispatchTimeMs); - root.attr("dispatchTimeAvgMs", nrDispatcherWakeups == 0 ? 0.0 : (float) dispatchTimeMs / nrDispatcherWakeups); - root.attr("nrDbConnections", dbPool.count()); - root.attr("nrActiveDbUpdates", nrActiveDbUpdates); + root["nrQueueWakeups"] = nrQueueWakeups.load(); + root["nrDispatcherWakeups"] = nrDispatcherWakeups.load(); + root["dispatchTimeMs"] = dispatchTimeMs.load(); + root["dispatchTimeAvgMs"] = nrDispatcherWakeups == 0 ? 0.0 : (float) dispatchTimeMs / nrDispatcherWakeups; + root["nrDbConnections"] = dbPool.count(); + root["nrActiveDbUpdates"] = nrActiveDbUpdates.load(); { - auto nested = root.object("machines"); + auto nested = root["machines"]; auto machines_(machines.lock()); for (auto & i : *machines_) { auto & m(i.second); auto & s(m->state); - auto nested2 = nested.object(m->sshName); - nested2.attr("enabled", m->enabled); - - { - auto list = nested2.list("systemTypes"); - for (auto & s : m->systemTypes) - list.elem(s); - } - - { - auto list = nested2.list("supportedFeatures"); - for (auto & s : m->supportedFeatures) - list.elem(s); - } - - { - auto list = nested2.list("mandatoryFeatures"); - for (auto & s : m->mandatoryFeatures) - list.elem(s); - } - - nested2.attr("currentJobs", s->currentJobs); - if (s->currentJobs == 0) - nested2.attr("idleSince", s->idleSince); - nested2.attr("nrStepsDone", s->nrStepsDone); - if (m->state->nrStepsDone) { - nested2.attr("totalStepTime", s->totalStepTime); - nested2.attr("totalStepBuildTime", s->totalStepBuildTime); - nested2.attr("avgStepTime", (float) s->totalStepTime / s->nrStepsDone); - nested2.attr("avgStepBuildTime", (float) s->totalStepBuildTime / s->nrStepsDone); - } - auto info(m->state->connectInfo.lock()); - nested2.attr("disabledUntil", std::chrono::system_clock::to_time_t(info->disabledUntil)); - nested2.attr("lastFailure", std::chrono::system_clock::to_time_t(info->lastFailure)); - nested2.attr("consecutiveFailures", info->consecutiveFailures); + auto machine = nested[m->sshName] = { + {"enabled", m->enabled}, + {"systemTypes", m->systemTypes}, + {"supportedFeatures", m->supportedFeatures}, + {"mandatoryFeatures", m->mandatoryFeatures}, + {"nrStepsDone", s->nrStepsDone.load()}, + {"currentJobs", s->currentJobs.load()}, + {"disabledUntil", std::chrono::system_clock::to_time_t(info->disabledUntil)}, + {"lastFailure", std::chrono::system_clock::to_time_t(info->lastFailure)}, + {"consecutiveFailures", info->consecutiveFailures}, + }; + + if (s->currentJobs == 0) + machine["idleSince"] = s->idleSince.load(); + if (m->state->nrStepsDone) { + machine["totalStepTime"] = s->totalStepTime.load(); + machine["totalStepBuildTime"] = s->totalStepBuildTime.load(); + machine["avgStepTime"] = (float) s->totalStepTime / s->nrStepsDone; + machine["avgStepBuildTime"] = (float) s->totalStepBuildTime / s->nrStepsDone; + } } } { - auto nested = root.object("jobsets"); + auto jobsets_json = root["jobsets"]; auto jobsets_(jobsets.lock()); for (auto & jobset : *jobsets_) { - auto nested2 = nested.object(jobset.first.first + ":" + jobset.first.second); - nested2.attr("shareUsed", jobset.second->shareUsed()); - nested2.attr("seconds", jobset.second->getSeconds()); + jobsets_json[jobset.first.first + ":" + jobset.first.second] = { + {"shareUsed", jobset.second->shareUsed()}, + {"seconds", jobset.second->getSeconds()}, + }; } } { - auto nested = root.object("machineTypes"); + auto machineTypesJson = root["machineTypes"]; auto machineTypes_(machineTypes.lock()); for (auto & i : *machineTypes_) { - auto nested2 = nested.object(i.first); - nested2.attr("runnable", i.second.runnable); - nested2.attr("running", i.second.running); + auto machineTypeJson = machineTypesJson[i.first] = { + {"runnable", i.second.runnable}, + {"running", i.second.running}, + }; if (i.second.runnable > 0) - nested2.attr("waitTime", i.second.waitTime.count() + - i.second.runnable * (time(0) - lastDispatcherCheck)); + machineTypeJson["waitTime"] = i.second.waitTime.count() + + i.second.runnable * (time(0) - lastDispatcherCheck); if (i.second.running == 0) - nested2.attr("lastActive", std::chrono::system_clock::to_time_t(i.second.lastActive)); + machineTypeJson["lastActive"] = std::chrono::system_clock::to_time_t(i.second.lastActive); } } auto store = getDestStore(); - auto nested = root.object("store"); - auto & stats = store->getStats(); - nested.attr("narInfoRead", stats.narInfoRead); - nested.attr("narInfoReadAverted", stats.narInfoReadAverted); - nested.attr("narInfoMissing", stats.narInfoMissing); - nested.attr("narInfoWrite", stats.narInfoWrite); - nested.attr("narInfoCacheSize", stats.pathInfoCacheSize); - nested.attr("narRead", stats.narRead); - nested.attr("narReadBytes", stats.narReadBytes); - nested.attr("narReadCompressedBytes", stats.narReadCompressedBytes); - nested.attr("narWrite", stats.narWrite); - nested.attr("narWriteAverted", stats.narWriteAverted); - nested.attr("narWriteBytes", stats.narWriteBytes); - nested.attr("narWriteCompressedBytes", stats.narWriteCompressedBytes); - nested.attr("narWriteCompressionTimeMs", stats.narWriteCompressionTimeMs); - nested.attr("narCompressionSavings", - stats.narWriteBytes - ? 1.0 - (double) stats.narWriteCompressedBytes / stats.narWriteBytes - : 0.0); - nested.attr("narCompressionSpeed", // MiB/s + root["store"] = { + {"narInfoRead", stats.narInfoRead.load()}, + {"narInfoReadAverted", stats.narInfoReadAverted.load()}, + {"narInfoMissing", stats.narInfoMissing.load()}, + {"narInfoWrite", stats.narInfoWrite.load()}, + {"narInfoCacheSize", stats.pathInfoCacheSize.load()}, + {"narRead", stats.narRead.load()}, + {"narReadBytes", stats.narReadBytes.load()}, + {"narReadCompressedBytes", stats.narReadCompressedBytes.load()}, + {"narWrite", stats.narWrite.load()}, + {"narWriteAverted", stats.narWriteAverted.load()}, + {"narWriteBytes", stats.narWriteBytes.load()}, + {"narWriteCompressedBytes", stats.narWriteCompressedBytes.load()}, + {"narWriteCompressionTimeMs", stats.narWriteCompressionTimeMs.load()}, + {"narCompressionSavings", + stats.narWriteBytes + ? 1.0 - (double) stats.narWriteCompressedBytes / stats.narWriteBytes + : 0.0}, + {"narCompressionSpeed", // MiB/s stats.narWriteCompressionTimeMs ? (double) stats.narWriteBytes / stats.narWriteCompressionTimeMs * 1000.0 / (1024.0 * 1024.0) - : 0.0); + : 0.0}, + }; auto s3Store = dynamic_cast(&*store); if (s3Store) { - auto nested2 = nested.object("s3"); auto & s3Stats = s3Store->getS3Stats(); - nested2.attr("put", s3Stats.put); - nested2.attr("putBytes", s3Stats.putBytes); - nested2.attr("putTimeMs", s3Stats.putTimeMs); - nested2.attr("putSpeed", - s3Stats.putTimeMs - ? (double) s3Stats.putBytes / s3Stats.putTimeMs * 1000.0 / (1024.0 * 1024.0) - : 0.0); - nested2.attr("get", s3Stats.get); - nested2.attr("getBytes", s3Stats.getBytes); - nested2.attr("getTimeMs", s3Stats.getTimeMs); - nested2.attr("getSpeed", - s3Stats.getTimeMs - ? (double) s3Stats.getBytes / s3Stats.getTimeMs * 1000.0 / (1024.0 * 1024.0) - : 0.0); - nested2.attr("head", s3Stats.head); - nested2.attr("costDollarApprox", - (s3Stats.get + s3Stats.head) / 10000.0 * 0.004 - + s3Stats.put / 1000.0 * 0.005 + - + s3Stats.getBytes / (1024.0 * 1024.0 * 1024.0) * 0.09); + auto jsonS3 = root["s3"] = { + {"put", s3Stats.put.load()}, + {"putBytes", s3Stats.putBytes.load()}, + {"putTimeMs", s3Stats.putTimeMs.load()}, + {"putSpeed", + s3Stats.putTimeMs + ? (double) s3Stats.putBytes / s3Stats.putTimeMs * 1000.0 / (1024.0 * 1024.0) + : 0.0}, + {"get", s3Stats.get.load()}, + {"getBytes", s3Stats.getBytes.load()}, + {"getTimeMs", s3Stats.getTimeMs.load()}, + {"getSpeed", + s3Stats.getTimeMs + ? (double) s3Stats.getBytes / s3Stats.getTimeMs * 1000.0 / (1024.0 * 1024.0) + : 0.0}, + {"head", s3Stats.head.load()}, + {"costDollarApprox", + (s3Stats.get + s3Stats.head) / 10000.0 * 0.004 + + s3Stats.put / 1000.0 * 0.005 + + + s3Stats.getBytes / (1024.0 * 1024.0 * 1024.0) * 0.09}, + }; } } @@ -725,7 +704,7 @@ void State::dumpStatus(Connection & conn) pqxx::work txn(conn); // FIXME: use PostgreSQL 9.5 upsert. txn.exec("delete from SystemStatus where what = 'queue-runner'"); - txn.exec_params0("insert into SystemStatus values ('queue-runner', $1)", out.str()); + txn.exec_params0("insert into SystemStatus values ('queue-runner', $1)", root.dump()); txn.exec("notify status_dumped"); txn.commit(); } From 5b35e1389885cf4c5d1058dc60017564b58e7e4c Mon Sep 17 00:00:00 2001 From: Linus Heckemann Date: Sat, 28 Jan 2023 12:58:41 +0100 Subject: [PATCH 019/313] hydra-queue-runner: use initializer lists for constructing JSON And also fix the parts that were broken --- src/hydra-queue-runner/hydra-queue-runner.cc | 89 ++++++++++---------- 1 file changed, 44 insertions(+), 45 deletions(-) diff --git a/src/hydra-queue-runner/hydra-queue-runner.cc b/src/hydra-queue-runner/hydra-queue-runner.cc index b16fd770..b84681d5 100644 --- a/src/hydra-queue-runner/hydra-queue-runner.cc +++ b/src/hydra-queue-runner/hydra-queue-runner.cc @@ -535,67 +535,65 @@ std::shared_ptr State::acquireGlobalLock() void State::dumpStatus(Connection & conn) { - auto root = json::object(); + time_t now = time(0); + json statusJson = { + {"status", "up"}, + {"time", time(0)}, + {"uptime", now - startedAt}, + {"pid", getpid()}, + + {"nrQueuedBuilds", builds.lock()->size()}, + {"nrActiveSteps", activeSteps_.lock()->size()}, + {"nrStepsBuilding", nrStepsBuilding.load()}, + {"nrStepsCopyingTo", nrStepsCopyingTo.load()}, + {"nrStepsCopyingFrom", nrStepsCopyingFrom.load()}, + {"nrStepsWaiting", nrStepsWaiting.load()}, + {"nrUnsupportedSteps", nrUnsupportedSteps.load()}, + {"bytesSent", bytesSent.load()}, + {"bytesReceived", bytesReceived.load()}, + {"nrBuildsRead", nrBuildsRead.load()}, + {"buildReadTimeMs", buildReadTimeMs.load()}, + {"buildReadTimeAvgMs", nrBuildsRead == 0 ? 0.0 : (float) buildReadTimeMs / nrBuildsRead}, + {"nrBuildsDone", nrBuildsDone.load()}, + {"nrStepsStarted", nrStepsStarted.load()}, + {"nrStepsDone", nrStepsDone.load()}, + {"nrRetries", nrRetries.load()}, + {"maxNrRetries", maxNrRetries.load()}, + {"nrQueueWakeups", nrQueueWakeups.load()}, + {"nrDispatcherWakeups", nrDispatcherWakeups.load()}, + {"dispatchTimeMs", dispatchTimeMs.load()}, + {"dispatchTimeAvgMs", nrDispatcherWakeups == 0 ? 0.0 : (float) dispatchTimeMs / nrDispatcherWakeups}, + {"nrDbConnections", dbPool.count()}, + {"nrActiveDbUpdates", nrActiveDbUpdates.load()}, + }; { - time_t now = time(0); - root["status"] = "up"; - root["time"] = time(0); - root["uptime"] = now - startedAt; - root["pid"] = getpid(); - { - auto builds_(builds.lock()); - root["nrQueuedBuilds"] = builds_->size(); - } { auto steps_(steps.lock()); for (auto i = steps_->begin(); i != steps_->end(); ) if (i->second.lock()) ++i; else i = steps_->erase(i); - root["nrUnfinishedSteps"] = steps_->size(); + statusJson["nrUnfinishedSteps"] = steps_->size(); } { auto runnable_(runnable.lock()); for (auto i = runnable_->begin(); i != runnable_->end(); ) if (i->lock()) ++i; else i = runnable_->erase(i); - root["nrRunnableSteps"] = runnable_->size(); + statusJson["nrRunnableSteps"] = runnable_->size(); } - root["nrActiveSteps"] = activeSteps_.lock()->size(); - root["nrStepsBuilding"] = nrStepsBuilding.load(); - root["nrStepsCopyingTo"] = nrStepsCopyingTo.load(); - root["nrStepsCopyingFrom"] = nrStepsCopyingFrom.load(); - root["nrStepsWaiting"] = nrStepsWaiting.load(); - root["nrUnsupportedSteps"] = nrUnsupportedSteps.load(); - root["bytesSent"] = bytesSent.load(); - root["bytesReceived"] = bytesReceived.load(); - root["nrBuildsRead"] = nrBuildsRead.load(); - root["buildReadTimeMs"] = buildReadTimeMs.load(); - root["buildReadTimeAvgMs"] = nrBuildsRead == 0 ? 0.0 : (float) buildReadTimeMs / nrBuildsRead; - root["nrBuildsDone"] = nrBuildsDone.load(); - root["nrStepsStarted"] = nrStepsStarted.load(); - root["nrStepsDone"] = nrStepsDone.load(); - root["nrRetries"] = nrRetries.load(); - root["maxNrRetries"] = maxNrRetries.load(); if (nrStepsDone) { - root["totalStepTime"] = totalStepTime.load(); - root["totalStepBuildTime"] = totalStepBuildTime.load(); - root["avgStepTime"] = (float) totalStepTime / nrStepsDone; - root["avgStepBuildTime"] = (float) totalStepBuildTime / nrStepsDone; + statusJson["totalStepTime"] = totalStepTime.load(); + statusJson["totalStepBuildTime"] = totalStepBuildTime.load(); + statusJson["avgStepTime"] = (float) totalStepTime / nrStepsDone; + statusJson["avgStepBuildTime"] = (float) totalStepBuildTime / nrStepsDone; } - root["nrQueueWakeups"] = nrQueueWakeups.load(); - root["nrDispatcherWakeups"] = nrDispatcherWakeups.load(); - root["dispatchTimeMs"] = dispatchTimeMs.load(); - root["dispatchTimeAvgMs"] = nrDispatcherWakeups == 0 ? 0.0 : (float) dispatchTimeMs / nrDispatcherWakeups; - root["nrDbConnections"] = dbPool.count(); - root["nrActiveDbUpdates"] = nrActiveDbUpdates.load(); { - auto nested = root["machines"]; auto machines_(machines.lock()); for (auto & i : *machines_) { auto & m(i.second); auto & s(m->state); auto info(m->state->connectInfo.lock()); - auto machine = nested[m->sshName] = { + json machine = { {"enabled", m->enabled}, {"systemTypes", m->systemTypes}, {"supportedFeatures", m->supportedFeatures}, @@ -615,11 +613,12 @@ void State::dumpStatus(Connection & conn) machine["avgStepTime"] = (float) s->totalStepTime / s->nrStepsDone; machine["avgStepBuildTime"] = (float) s->totalStepBuildTime / s->nrStepsDone; } + statusJson["machines"][m->sshName] = machine; } } { - auto jobsets_json = root["jobsets"]; + auto jobsets_json = statusJson["jobsets"] = json::object(); auto jobsets_(jobsets.lock()); for (auto & jobset : *jobsets_) { jobsets_json[jobset.first.first + ":" + jobset.first.second] = { @@ -630,7 +629,7 @@ void State::dumpStatus(Connection & conn) } { - auto machineTypesJson = root["machineTypes"]; + auto machineTypesJson = statusJson["machineTypes"] = json::object(); auto machineTypes_(machineTypes.lock()); for (auto & i : *machineTypes_) { auto machineTypeJson = machineTypesJson[i.first] = { @@ -648,7 +647,7 @@ void State::dumpStatus(Connection & conn) auto store = getDestStore(); auto & stats = store->getStats(); - root["store"] = { + statusJson["store"] = { {"narInfoRead", stats.narInfoRead.load()}, {"narInfoReadAverted", stats.narInfoReadAverted.load()}, {"narInfoMissing", stats.narInfoMissing.load()}, @@ -675,7 +674,7 @@ void State::dumpStatus(Connection & conn) auto s3Store = dynamic_cast(&*store); if (s3Store) { auto & s3Stats = s3Store->getS3Stats(); - auto jsonS3 = root["s3"] = { + auto jsonS3 = statusJson["s3"] = { {"put", s3Stats.put.load()}, {"putBytes", s3Stats.putBytes.load()}, {"putTimeMs", s3Stats.putTimeMs.load()}, @@ -704,7 +703,7 @@ void State::dumpStatus(Connection & conn) pqxx::work txn(conn); // FIXME: use PostgreSQL 9.5 upsert. txn.exec("delete from SystemStatus where what = 'queue-runner'"); - txn.exec_params0("insert into SystemStatus values ('queue-runner', $1)", root.dump()); + txn.exec_params0("insert into SystemStatus values ('queue-runner', $1)", statusJson.dump()); txn.exec("notify status_dumped"); txn.commit(); } From c7716817a92031f6d94259a3f9d411dd1f062b1e Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Sat, 28 Jan 2023 09:27:48 +0100 Subject: [PATCH 020/313] Update Nix to 2.13 --- flake.lock | 20 ++++++++++---------- flake.nix | 2 +- src/hydra-eval-jobs/hydra-eval-jobs.cc | 2 +- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/flake.lock b/flake.lock index b41b843a..75023b95 100644 --- a/flake.lock +++ b/flake.lock @@ -23,32 +23,32 @@ "nixpkgs-regression": "nixpkgs-regression" }, "locked": { - "lastModified": 1661606874, - "narHash": "sha256-9+rpYzI+SmxJn+EbYxjGv68Ucp22bdFUSy/4LkHkkDQ=", - "owner": "NixOS", + "lastModified": 1675514340, + "narHash": "sha256-JjnneK+TkhkxFoh6EEVKAzEBdxz0iucZsJ6+PWTTReQ=", + "owner": "nixos", "repo": "nix", - "rev": "11e45768b34fdafdcf019ddbd337afa16127ff0f", + "rev": "9157f94e775936798c1f8783eab929e77904e5ed", "type": "github" }, "original": { - "owner": "NixOS", - "ref": "2.11.0", + "owner": "nixos", + "ref": "2.13-maintenance", "repo": "nix", "type": "github" } }, "nixpkgs": { "locked": { - "lastModified": 1657693803, - "narHash": "sha256-G++2CJ9u0E7NNTAi9n5G8TdDmGJXcIjkJ3NF8cetQB8=", + "lastModified": 1670461440, + "narHash": "sha256-jy1LB8HOMKGJEGXgzFRLDU1CBGL0/LlkolgnqIsF0D8=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "365e1b3a859281cf11b94f87231adeabbdd878a2", + "rev": "04a75b2eecc0acf6239acf9dd04485ff8d14f425", "type": "github" }, "original": { "owner": "NixOS", - "ref": "nixos-22.05-small", + "ref": "nixos-22.11-small", "repo": "nixpkgs", "type": "github" } diff --git a/flake.nix b/flake.nix index cd9f094d..208d9017 100644 --- a/flake.nix +++ b/flake.nix @@ -2,7 +2,7 @@ description = "A Nix-based continuous build system"; inputs.nixpkgs.follows = "nix/nixpkgs"; - inputs.nix.url = "github:NixOS/nix/2.11.0"; + inputs.nix.url = "github:nixos/nix/2.13-maintenance"; outputs = { self, nixpkgs, nix }: let diff --git a/src/hydra-eval-jobs/hydra-eval-jobs.cc b/src/hydra-eval-jobs/hydra-eval-jobs.cc index 18d39620..de7ae7ba 100644 --- a/src/hydra-eval-jobs/hydra-eval-jobs.cc +++ b/src/hydra-eval-jobs/hydra-eval-jobs.cc @@ -129,7 +129,7 @@ static void worker( LockFlags { .updateLockFile = false, .useRegistries = false, - .allowMutable = false, + .allowUnlocked = false, }); callFlake(state, lockedFlake, *vFlake); From ddd3ac3a4d57549c88812a0d1bad8a196899309d Mon Sep 17 00:00:00 2001 From: Linus Heckemann Date: Tue, 29 Nov 2022 15:48:42 +0100 Subject: [PATCH 021/313] name tests --- flake.nix | 3 +++ 1 file changed, 3 insertions(+) diff --git a/flake.nix b/flake.nix index 208d9017..ec9a708e 100644 --- a/flake.nix +++ b/flake.nix @@ -272,6 +272,7 @@ tests.install = forEachSystem (system: with import (nixpkgs + "/nixos/lib/testing-python.nix") { inherit system; }; simpleTest { + name = "hydra-install"; nodes.machine = hydraServer; testScript = '' @@ -288,6 +289,7 @@ let pkgs = pkgsBySystem.${system}; in with import (nixpkgs + "/nixos/lib/testing-python.nix") { inherit system; }; simpleTest { + name = "hydra-notifications"; nodes.machine = { pkgs, ... }: { imports = [ hydraServer ]; services.hydra-dev.extraConfig = '' @@ -346,6 +348,7 @@ let pkgs = pkgsBySystem.${system}; in with import (nixpkgs + "/nixos/lib/testing-python.nix") { inherit system; }; makeTest { + name = "hydra-gitea"; nodes.machine = { pkgs, ... }: { imports = [ hydraServer ]; services.hydra-dev.extraConfig = '' From 73dff150397bbc3de94e6c4bf4d7ac59b3a681ab Mon Sep 17 00:00:00 2001 From: Linus Heckemann Date: Thu, 1 Dec 2022 23:32:47 +0100 Subject: [PATCH 022/313] tests: ports are numbers --- flake.nix | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.nix b/flake.nix index ec9a708e..5f0a7d24 100644 --- a/flake.nix +++ b/flake.nix @@ -280,7 +280,7 @@ machine.wait_for_job("hydra-server") machine.wait_for_job("hydra-evaluator") machine.wait_for_job("hydra-queue-runner") - machine.wait_for_open_port("3000") + machine.wait_for_open_port(3000) machine.succeed("curl --fail http://localhost:3000/") ''; }); @@ -317,7 +317,7 @@ # Wait until InfluxDB can receive web requests machine.wait_for_job("influxdb") - machine.wait_for_open_port("8086") + machine.wait_for_open_port(8086) # Create an InfluxDB database where hydra will write to machine.succeed( @@ -327,7 +327,7 @@ # Wait until hydra-server can receive HTTP requests machine.wait_for_job("hydra-server") - machine.wait_for_open_port("3000") + machine.wait_for_open_port(3000) # Setup the project and jobset machine.succeed( From 65c1249227dd33fec8856dc1680dfdd5b3598d0a Mon Sep 17 00:00:00 2001 From: Rick van Schijndel Date: Thu, 16 Feb 2023 19:24:53 +0100 Subject: [PATCH 023/313] systemd: hydra-queue-runner: wait for network-online This prevents eval errors when a machine is just started and the network isn't yet online. I'm running hydra on a laptop and the network takes a bit of time to come online (WLAN), so it's nice if the evaluator starts only when the network actually goes online. Otherwise an error like this can happen on the first eval(s): ``` error fetching latest change from git repo at `https://github.com/nixos/nixpkgs.git': fatal: unable to access 'https://github.com/nixos/nixpkgs.git/': Could not resolve host: github.com ``` --- hydra-module.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hydra-module.nix b/hydra-module.nix index 8e02dcbb..70e17284 100644 --- a/hydra-module.nix +++ b/hydra-module.nix @@ -340,7 +340,7 @@ in systemd.services.hydra-queue-runner = { wantedBy = [ "multi-user.target" ]; requires = [ "hydra-init.service" ]; - after = [ "hydra-init.service" "network.target" ]; + after = [ "hydra-init.service" "network-online.target" ]; path = [ cfg.package pkgs.nettools pkgs.openssh pkgs.bzip2 config.nix.package ]; restartTriggers = [ hydraConf ]; environment = env // { From f44d3d6ec9b3ca0546a82d0230a9afee4ac179c8 Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Sat, 4 Mar 2023 12:07:34 +0100 Subject: [PATCH 024/313] Update Nix to 2.13.3 Includes the following required fixes: * perl-bindings are correctly initialized: https://github.com/NixOS/nix/commit/77d8066e83ec6120c954ce34290ee1ffe00da133 * /etc/ must be unwritable in build sandbox: https://github.com/NixOS/nix/commit/4acc684ef7b3117c6d6ac12837398a0008a53d85 --- flake.lock | 8 ++++---- flake.nix | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/flake.lock b/flake.lock index 75023b95..08fd86ad 100644 --- a/flake.lock +++ b/flake.lock @@ -23,16 +23,16 @@ "nixpkgs-regression": "nixpkgs-regression" }, "locked": { - "lastModified": 1675514340, - "narHash": "sha256-JjnneK+TkhkxFoh6EEVKAzEBdxz0iucZsJ6+PWTTReQ=", + "lastModified": 1677045134, + "narHash": "sha256-jUc2ccTR8f6MGY2pUKgujm+lxSPNGm/ZAP+toX+nMNc=", "owner": "nixos", "repo": "nix", - "rev": "9157f94e775936798c1f8783eab929e77904e5ed", + "rev": "4acc684ef7b3117c6d6ac12837398a0008a53d85", "type": "github" }, "original": { "owner": "nixos", - "ref": "2.13-maintenance", + "ref": "2.13.3", "repo": "nix", "type": "github" } diff --git a/flake.nix b/flake.nix index 5f0a7d24..beeb90c1 100644 --- a/flake.nix +++ b/flake.nix @@ -2,7 +2,7 @@ description = "A Nix-based continuous build system"; inputs.nixpkgs.follows = "nix/nixpkgs"; - inputs.nix.url = "github:nixos/nix/2.13-maintenance"; + inputs.nix.url = "github:nixos/nix/2.13.3"; outputs = { self, nixpkgs, nix }: let From 810d2e6b51af82a948a12eae505a66c2e0f6f09f Mon Sep 17 00:00:00 2001 From: Cole Helbling Date: Mon, 6 Mar 2023 07:45:03 -0800 Subject: [PATCH 025/313] Drop unused IndexBuildOutputsOnPath index Also it's larger than the actual table it's indexing lol. -[ RECORD 30 ]----------+----------------------------------------- table_name | buildoutputs index_name | indexbuildoutputsonpath index_scans_count | 0 index_size | 31 GB table_reads_index_count | 2128699937 table_reads_seq_count | 0 table_reads_count | 2128699937 table_writes_count | 22442976 table_size | 28 GB --- src/sql/upgrade-83.sql | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 src/sql/upgrade-83.sql diff --git a/src/sql/upgrade-83.sql b/src/sql/upgrade-83.sql new file mode 100644 index 00000000..01603e78 --- /dev/null +++ b/src/sql/upgrade-83.sql @@ -0,0 +1,3 @@ +-- This index was introduced in a migration but was never recorded in +-- hydra.sql (the source of truth), which is why `if exists` is required. +drop index if exists IndexBuildOutputsOnPath; From 8d53c3ca11855234e32ca9f1da0f544491e6cf09 Mon Sep 17 00:00:00 2001 From: Cole Helbling Date: Mon, 6 Mar 2023 07:47:35 -0800 Subject: [PATCH 026/313] test: use ubuntu-latest --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0f5f43da..42cb6843 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -4,7 +4,7 @@ on: push: jobs: tests: - runs-on: ubuntu-18.04 + runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 with: From a084e204ae8b8ffcf3a4a1912582a00c15914136 Mon Sep 17 00:00:00 2001 From: Rick van Schijndel Date: Tue, 7 Mar 2023 21:56:20 +0100 Subject: [PATCH 027/313] systemd: hydra-queue-runner: wait for network.target too Co-authored-by: Sandro --- hydra-module.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hydra-module.nix b/hydra-module.nix index 70e17284..1f0792d7 100644 --- a/hydra-module.nix +++ b/hydra-module.nix @@ -340,7 +340,7 @@ in systemd.services.hydra-queue-runner = { wantedBy = [ "multi-user.target" ]; requires = [ "hydra-init.service" ]; - after = [ "hydra-init.service" "network-online.target" ]; + after = [ "hydra-init.service" "network.target" "network-online.target" ]; path = [ cfg.package pkgs.nettools pkgs.openssh pkgs.bzip2 config.nix.package ]; restartTriggers = [ hydraConf ]; environment = env // { From f88bef15ed57c36dc33d220c8cdf1d5021b8fdbb Mon Sep 17 00:00:00 2001 From: Rob Vermaas Date: Mon, 13 Mar 2023 16:44:09 +0100 Subject: [PATCH 028/313] Use new Google for Web signin, the old way will be deprecated Mar 31st 2023 --- src/root/auth.tt | 5 +---- src/root/topbar.tt | 6 ++++-- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/src/root/auth.tt b/src/root/auth.tt index 360904d9..d1539765 100644 --- a/src/root/auth.tt +++ b/src/root/auth.tt @@ -82,7 +82,7 @@ function onGoogleSignIn(googleUser) { requestJSON({ url: "[% c.uri_for('/google-login') %]", - data: "id_token=" + googleUser.getAuthResponse().id_token, + data: "id_token=" + googleUser.credential, type: 'POST', success: function(data) { window.location.reload(); @@ -91,9 +91,6 @@ return false; }; - $("#google-signin").click(function() { - $(".g-signin2:first-child > div").click(); - }); [% END %] diff --git a/src/root/topbar.tt b/src/root/topbar.tt index fdfbf431..1771222d 100644 --- a/src/root/topbar.tt +++ b/src/root/topbar.tt @@ -133,8 +133,10 @@ [% ELSE %] [% WRAPPER makeSubMenu title="Sign in" id="sign-in-menu" align="right" %] [% IF c.config.enable_google_login %] - - Sign in with Google + +
+
+
[% END %] [% IF c.config.github_client_id %] From b4099df91ec542a2b2eed38d07ff1a64c265711f Mon Sep 17 00:00:00 2001 From: Julien Malka Date: Mon, 24 Apr 2023 16:30:03 +0200 Subject: [PATCH 029/313] hydra-eval-jobs: fix jobs containing a dot being dropped --- src/hydra-eval-jobs/hydra-eval-jobs.cc | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/hydra-eval-jobs/hydra-eval-jobs.cc b/src/hydra-eval-jobs/hydra-eval-jobs.cc index de7ae7ba..af839bba 100644 --- a/src/hydra-eval-jobs/hydra-eval-jobs.cc +++ b/src/hydra-eval-jobs/hydra-eval-jobs.cc @@ -245,7 +245,7 @@ static void worker( StringSet ss; for (auto & i : v->attrs->lexicographicOrder(state.symbols)) { std::string name(state.symbols[i->name]); - if (name.find('.') != std::string::npos || name.find(' ') != std::string::npos) { + if (name.find(' ') != std::string::npos) { printError("skipping job with illegal name '%s'", name); continue; } @@ -416,7 +416,11 @@ int main(int argc, char * * argv) if (response.find("attrs") != response.end()) { for (auto & i : response["attrs"]) { - auto s = (attrPath.empty() ? "" : attrPath + ".") + (std::string) i; + std::string path = i; + if (path.find(".") != std::string::npos){ + path = "\"" + path + "\""; + } + auto s = (attrPath.empty() ? "" : attrPath + ".") + (std::string) path; newAttrs.insert(s); } } From a0c8440a5c6eee911479c220706f74fd17e0c55f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 23 Jun 2023 13:14:49 +0200 Subject: [PATCH 030/313] Update to Nix 2.16 and NixOS 23.05 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nix': 'github:nixos/nix/4acc684ef7b3117c6d6ac12837398a0008a53d85' (2023-02-22) → 'github:NixOS/nix/84050709ea18f3285a85d729f40c8f8eddf5008e' (2023-06-06) • Added input 'nix/flake-compat': 'github:edolstra/flake-compat/35bb57c0c8d8b62bbfd284272c928ceb64ddbde9' (2023-01-17) • Updated input 'nixpkgs': follows 'nix/nixpkgs' → 'github:NixOS/nixpkgs/ef0bc3976340dab9a4e087a0bcff661a8b2e87f3' (2023-06-21) --- flake.lock | 46 +++++++++++++++++++++++++++++++--------------- flake.nix | 5 +++-- 2 files changed, 34 insertions(+), 17 deletions(-) diff --git a/flake.lock b/flake.lock index 08fd86ad..ee85f6fa 100644 --- a/flake.lock +++ b/flake.lock @@ -1,5 +1,21 @@ { "nodes": { + "flake-compat": { + "flake": false, + "locked": { + "lastModified": 1673956053, + "narHash": "sha256-4gtG9iQuiKITOjNQQeQIpoIB6b16fm+504Ch3sNKLd8=", + "owner": "edolstra", + "repo": "flake-compat", + "rev": "35bb57c0c8d8b62bbfd284272c928ceb64ddbde9", + "type": "github" + }, + "original": { + "owner": "edolstra", + "repo": "flake-compat", + "type": "github" + } + }, "lowdown-src": { "flake": false, "locked": { @@ -18,37 +34,40 @@ }, "nix": { "inputs": { + "flake-compat": "flake-compat", "lowdown-src": "lowdown-src", - "nixpkgs": "nixpkgs", + "nixpkgs": [ + "nixpkgs" + ], "nixpkgs-regression": "nixpkgs-regression" }, "locked": { - "lastModified": 1677045134, - "narHash": "sha256-jUc2ccTR8f6MGY2pUKgujm+lxSPNGm/ZAP+toX+nMNc=", - "owner": "nixos", + "lastModified": 1686048923, + "narHash": "sha256-/XCWa2osNFIpPC5MkxlX6qTZf/DaTLwS3LWN0SRFiuU=", + "owner": "NixOS", "repo": "nix", - "rev": "4acc684ef7b3117c6d6ac12837398a0008a53d85", + "rev": "84050709ea18f3285a85d729f40c8f8eddf5008e", "type": "github" }, "original": { - "owner": "nixos", - "ref": "2.13.3", + "owner": "NixOS", + "ref": "2.16.1", "repo": "nix", "type": "github" } }, "nixpkgs": { "locked": { - "lastModified": 1670461440, - "narHash": "sha256-jy1LB8HOMKGJEGXgzFRLDU1CBGL0/LlkolgnqIsF0D8=", + "lastModified": 1687379288, + "narHash": "sha256-cSuwfiqYfeVyqzCRkU9AvLTysmEuSal8nh6CYr+xWog=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "04a75b2eecc0acf6239acf9dd04485ff8d14f425", + "rev": "ef0bc3976340dab9a4e087a0bcff661a8b2e87f3", "type": "github" }, "original": { "owner": "NixOS", - "ref": "nixos-22.11-small", + "ref": "nixos-23.05", "repo": "nixpkgs", "type": "github" } @@ -72,10 +91,7 @@ "root": { "inputs": { "nix": "nix", - "nixpkgs": [ - "nix", - "nixpkgs" - ] + "nixpkgs": "nixpkgs" } } }, diff --git a/flake.nix b/flake.nix index beeb90c1..6bbec9b0 100644 --- a/flake.nix +++ b/flake.nix @@ -1,8 +1,9 @@ { description = "A Nix-based continuous build system"; - inputs.nixpkgs.follows = "nix/nixpkgs"; - inputs.nix.url = "github:nixos/nix/2.13.3"; + inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.05"; + inputs.nix.url = "github:NixOS/nix/2.16.1"; + inputs.nix.inputs.nixpkgs.follows = "nixpkgs"; outputs = { self, nixpkgs, nix }: let From 9f69bb5c2c132e9ac7b8155972096b425155c6e1 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 23 Jun 2023 15:06:34 +0200 Subject: [PATCH 031/313] Fix compilation against Nix 2.16 --- configure.ac | 2 -- src/hydra-eval-jobs/hydra-eval-jobs.cc | 31 ++++++++++++-------- src/hydra-queue-runner/build-remote.cc | 12 ++++---- src/hydra-queue-runner/builder.cc | 4 +-- src/hydra-queue-runner/dispatcher.cc | 8 ++--- src/hydra-queue-runner/hydra-queue-runner.cc | 7 ++--- src/hydra-queue-runner/queue-monitor.cc | 14 ++++----- 7 files changed, 40 insertions(+), 38 deletions(-) diff --git a/configure.ac b/configure.ac index 0c823696..eec647c3 100644 --- a/configure.ac +++ b/configure.ac @@ -10,8 +10,6 @@ AC_PROG_LN_S AC_PROG_LIBTOOL AC_PROG_CXX -CXXFLAGS+=" -std=c++17" - AC_PATH_PROG([XSLTPROC], [xsltproc]) AC_ARG_WITH([docbook-xsl], diff --git a/src/hydra-eval-jobs/hydra-eval-jobs.cc b/src/hydra-eval-jobs/hydra-eval-jobs.cc index af839bba..79523944 100644 --- a/src/hydra-eval-jobs/hydra-eval-jobs.cc +++ b/src/hydra-eval-jobs/hydra-eval-jobs.cc @@ -25,7 +25,8 @@ #include -void check_pid_status_nonblocking(pid_t check_pid) { +void check_pid_status_nonblocking(pid_t check_pid) +{ // Only check 'initialized' and known PID's if (check_pid <= 0) { return; } @@ -100,7 +101,7 @@ static std::string queryMetaStrings(EvalState & state, DrvInfo & drv, const std: else if (v.type() == nAttrs) { auto a = v.attrs->find(state.symbols.create(subAttribute)); if (a != v.attrs->end()) - res.push_back(std::string(state.forceString(*a->value))); + res.push_back(std::string(state.forceString(*a->value, a->pos, "while evaluating meta attributes"))); } }; @@ -197,26 +198,30 @@ static void worker( /* If this is an aggregate, then get its constituents. */ auto a = v->attrs->get(state.symbols.create("_hydraAggregate")); - if (a && state.forceBool(*a->value, a->pos)) { + if (a && state.forceBool(*a->value, a->pos, "while evaluating the `_hydraAggregate` attribute")) { auto a = v->attrs->get(state.symbols.create("constituents")); if (!a) throw EvalError("derivation must have a ‘constituents’ attribute"); + NixStringContext context; + state.coerceToString(a->pos, *a->value, context, "while evaluating the `constituents` attribute", true, false); + for (auto & c : context) + std::visit(overloaded { + [&](const NixStringContextElem::Built & b) { + job["constituents"].push_back(state.store->printStorePath(b.drvPath)); + }, + [&](const NixStringContextElem::Opaque & o) { + }, + [&](const NixStringContextElem::DrvDeep & d) { + }, + }, c.raw()); - PathSet context; - state.coerceToString(a->pos, *a->value, context, true, false); - for (auto & i : context) - if (i.at(0) == '!') { - size_t index = i.find("!", 1); - job["constituents"].push_back(std::string(i, index + 1)); - } - - state.forceList(*a->value, a->pos); + state.forceList(*a->value, a->pos, "while evaluating the `constituents` attribute"); for (unsigned int n = 0; n < a->value->listSize(); ++n) { auto v = a->value->listElems()[n]; state.forceValue(*v, noPos); if (v->type() == nString) - job["namedConstituents"].push_back(state.forceStringNoCtx(*v)); + job["namedConstituents"].push_back(v->str()); } } diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc index 21a6c331..6baff7df 100644 --- a/src/hydra-queue-runner/build-remote.cc +++ b/src/hydra-queue-runner/build-remote.cc @@ -116,12 +116,12 @@ static void copyClosureTo(std::timed_mutex & sendMutex, Store & destStore, the remote host to substitute missing paths. */ // FIXME: substitute output pollutes our build log to << cmdQueryValidPaths << 1 << useSubstitutes; - worker_proto::write(destStore, to, closure); + workerProtoWrite(destStore, to, closure); to.flush(); /* Get back the set of paths that are already valid on the remote host. */ - auto present = worker_proto::read(destStore, from, Phantom {}); + auto present = WorkerProto::read(destStore, from); if (present.size() == closure.size()) return; @@ -367,7 +367,7 @@ void State::buildRemote(ref destStore, } } if (GET_PROTOCOL_MINOR(remoteVersion) >= 6) { - worker_proto::read(*localStore, from, Phantom {}); + WorkerProto::read(*localStore, from); } switch ((BuildResult::Status) res) { case BuildResult::Built: @@ -444,17 +444,17 @@ void State::buildRemote(ref destStore, std::map infos; size_t totalNarSize = 0; to << cmdQueryPathInfos; - worker_proto::write(*localStore, to, outputs); + workerProtoWrite(*localStore, to, outputs); to.flush(); while (true) { auto storePathS = readString(from); if (storePathS == "") break; auto deriver = readString(from); // deriver - auto references = worker_proto::read(*localStore, from, Phantom {}); + auto references = WorkerProto::read(*localStore, from); readLongLong(from); // download size auto narSize = readLongLong(from); auto narHash = Hash::parseAny(readString(from), htSHA256); - auto ca = parseContentAddressOpt(readString(from)); + auto ca = ContentAddress::parseOpt(readString(from)); readStrings(from); // sigs ValidPathInfo info(localStore->parseStorePath(storePathS), narHash); assert(outputs.count(info.path)); diff --git a/src/hydra-queue-runner/builder.cc b/src/hydra-queue-runner/builder.cc index 37022522..89aec323 100644 --- a/src/hydra-queue-runner/builder.cc +++ b/src/hydra-queue-runner/builder.cc @@ -323,7 +323,7 @@ State::StepResult State::doBuildStep(nix::ref destStore, pqxx::work txn(*conn); for (auto & b : direct) { - printMsg(lvlInfo, format("marking build %1% as succeeded") % b->id); + printInfo("marking build %1% as succeeded", b->id); markSucceededBuild(txn, b, res, buildId != b->id || result.isCached, result.startTime, result.stopTime); } @@ -451,7 +451,7 @@ void State::failStep( /* Mark all builds that depend on this derivation as failed. */ for (auto & build : indirect) { if (build->finishedInDB) continue; - printMsg(lvlError, format("marking build %1% as failed") % build->id); + printError("marking build %1% as failed", build->id); txn.exec_params0 ("update Builds set finished = 1, buildStatus = $2, startTime = $3, stopTime = $4, isCachedBuild = $5, notificationPendingSince = $4 where id = $1 and finished = 0", build->id, diff --git a/src/hydra-queue-runner/dispatcher.cc b/src/hydra-queue-runner/dispatcher.cc index d2bb3c90..1e40fa69 100644 --- a/src/hydra-queue-runner/dispatcher.cc +++ b/src/hydra-queue-runner/dispatcher.cc @@ -52,7 +52,7 @@ void State::dispatcher() { auto dispatcherWakeup_(dispatcherWakeup.lock()); if (!*dispatcherWakeup_) { - printMsg(lvlDebug, format("dispatcher sleeping for %1%s") % + debug("dispatcher sleeping for %1%s", std::chrono::duration_cast(sleepUntil - std::chrono::system_clock::now()).count()); dispatcherWakeup_.wait_until(dispatcherWakeupCV, sleepUntil); } @@ -60,7 +60,7 @@ void State::dispatcher() } } catch (std::exception & e) { - printMsg(lvlError, format("dispatcher: %1%") % e.what()); + printError("dispatcher: %s", e.what()); sleep(1); } @@ -80,8 +80,8 @@ system_time State::doDispatch() jobset.second->pruneSteps(); auto s2 = jobset.second->shareUsed(); if (s1 != s2) - printMsg(lvlDebug, format("pruned scheduling window of ‘%1%:%2%’ from %3% to %4%") - % jobset.first.first % jobset.first.second % s1 % s2); + debug("pruned scheduling window of ‘%1%:%2%’ from %3% to %4%", + jobset.first.first, jobset.first.second, s1, s2); } } diff --git a/src/hydra-queue-runner/hydra-queue-runner.cc b/src/hydra-queue-runner/hydra-queue-runner.cc index b84681d5..acf1282e 100644 --- a/src/hydra-queue-runner/hydra-queue-runner.cc +++ b/src/hydra-queue-runner/hydra-queue-runner.cc @@ -161,9 +161,9 @@ void State::parseMachines(const std::string & contents) same name. */ auto i = oldMachines.find(machine->sshName); if (i == oldMachines.end()) - printMsg(lvlChatty, format("adding new machine ‘%1%’") % machine->sshName); + printMsg(lvlChatty, "adding new machine ‘%1%’", machine->sshName); else - printMsg(lvlChatty, format("updating machine ‘%1%’") % machine->sshName); + printMsg(lvlChatty, "updating machine ‘%1%’", machine->sshName); machine->state = i == oldMachines.end() ? std::make_shared() : i->second->state; @@ -173,7 +173,7 @@ void State::parseMachines(const std::string & contents) for (auto & m : oldMachines) if (newMachines.find(m.first) == newMachines.end()) { if (m.second->enabled) - printMsg(lvlInfo, format("removing machine ‘%1%’") % m.first); + printInfo("removing machine ‘%1%’", m.first); /* Add a disabled Machine object to make sure stats are maintained. */ auto machine = std::make_shared(*(m.second)); @@ -928,7 +928,6 @@ int main(int argc, char * * argv) }); settings.verboseBuild = true; - settings.lockCPU = false; State state{metricsAddrOpt}; if (status) diff --git a/src/hydra-queue-runner/queue-monitor.cc b/src/hydra-queue-runner/queue-monitor.cc index 12d55b79..0bb167a2 100644 --- a/src/hydra-queue-runner/queue-monitor.cc +++ b/src/hydra-queue-runner/queue-monitor.cc @@ -13,7 +13,7 @@ void State::queueMonitor() try { queueMonitorLoop(); } catch (std::exception & e) { - printMsg(lvlError, format("queue monitor: %1%") % e.what()); + printError("queue monitor: %s", e.what()); sleep(10); // probably a DB problem, so don't retry right away } } @@ -142,13 +142,13 @@ bool State::getQueuedBuilds(Connection & conn, createBuild = [&](Build::ptr build) { prom.queue_build_loads.Increment(); - printMsg(lvlTalkative, format("loading build %1% (%2%)") % build->id % build->fullJobName()); + printMsg(lvlTalkative, "loading build %1% (%2%)", build->id, build->fullJobName()); nrAdded++; newBuildsByID.erase(build->id); if (!localStore->isValidPath(build->drvPath)) { /* Derivation has been GC'ed prematurely. */ - printMsg(lvlError, format("aborting GC'ed build %1%") % build->id); + printError("aborting GC'ed build %1%", build->id); if (!build->finishedInDB) { auto mc = startDbUpdate(); pqxx::work txn(conn); @@ -302,7 +302,7 @@ bool State::getQueuedBuilds(Connection & conn, /* Add the new runnable build steps to ‘runnable’ and wake up the builder threads. */ - printMsg(lvlChatty, format("got %1% new runnable steps from %2% new builds") % newRunnable.size() % nrAdded); + printMsg(lvlChatty, "got %1% new runnable steps from %2% new builds", newRunnable.size(), nrAdded); for (auto & r : newRunnable) makeRunnable(r); @@ -358,13 +358,13 @@ void State::processQueueChange(Connection & conn) for (auto i = builds_->begin(); i != builds_->end(); ) { auto b = currentIds.find(i->first); if (b == currentIds.end()) { - printMsg(lvlInfo, format("discarding cancelled build %1%") % i->first); + printInfo("discarding cancelled build %1%", i->first); i = builds_->erase(i); // FIXME: ideally we would interrupt active build steps here. continue; } if (i->second->globalPriority < b->second) { - printMsg(lvlInfo, format("priority of build %1% increased") % i->first); + printInfo("priority of build %1% increased", i->first); i->second->globalPriority = b->second; i->second->propagatePriorities(); } @@ -654,7 +654,7 @@ BuildOutput State::getBuildOutputCached(Connection & conn, nix::ref if (r.empty()) continue; BuildID id = r[0][0].as(); - printMsg(lvlInfo, format("reusing build %d") % id); + printInfo("reusing build %d", id); BuildOutput res; res.failed = r[0][1].as() == bsFailedWithOutput; From ce001bb1420bb0c774ea08cd21fd624ccea04788 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 23 Jun 2023 15:09:09 +0200 Subject: [PATCH 032/313] Relax time interval checks I saw one of these failing randomly. --- t/Hydra/Plugin/RunCommand/basic.t | 4 ++-- t/Hydra/Plugin/RunCommand/errno.t | 4 ++-- t/Hydra/Schema/Result/RunCommandLogs.t | 22 +++++++++++----------- t/Hydra/Schema/Result/TaskRetries.t | 4 ++-- t/Hydra/Schema/ResultSet/TaskRetries.t | 2 +- 5 files changed, 18 insertions(+), 18 deletions(-) diff --git a/t/Hydra/Plugin/RunCommand/basic.t b/t/Hydra/Plugin/RunCommand/basic.t index e9fc730b..2c0eec68 100644 --- a/t/Hydra/Plugin/RunCommand/basic.t +++ b/t/Hydra/Plugin/RunCommand/basic.t @@ -57,8 +57,8 @@ subtest "Validate a run log was created" => sub { ok($runlog->did_succeed(), "The process did succeed."); is($runlog->job_matcher, "*:*:*", "An unspecified job matcher is defaulted to *:*:*"); is($runlog->command, 'cp "$HYDRA_JSON" "$HYDRA_DATA/joboutput.json"', "The executed command is saved."); - is($runlog->start_time, within(time() - 1, 2), "The start time is recent."); - is($runlog->end_time, within(time() - 1, 2), "The end time is also recent."); + is($runlog->start_time, within(time() - 1, 5), "The start time is recent."); + is($runlog->end_time, within(time() - 1, 5), "The end time is also recent."); is($runlog->exit_code, 0, "This command should have succeeded."); subtest "Validate the run log file exists" => sub { diff --git a/t/Hydra/Plugin/RunCommand/errno.t b/t/Hydra/Plugin/RunCommand/errno.t index 9e06f9bb..6b05d457 100644 --- a/t/Hydra/Plugin/RunCommand/errno.t +++ b/t/Hydra/Plugin/RunCommand/errno.t @@ -43,8 +43,8 @@ subtest "Validate a run log was created" => sub { ok($runlog->did_fail_with_exec_error(), "The process failed to start due to an exec error."); is($runlog->job_matcher, "*:*:*", "An unspecified job matcher is defaulted to *:*:*"); is($runlog->command, 'invalid-command-this-does-not-exist', "The executed command is saved."); - is($runlog->start_time, within(time() - 1, 2), "The start time is recent."); - is($runlog->end_time, within(time() - 1, 2), "The end time is also recent."); + is($runlog->start_time, within(time() - 1, 5), "The start time is recent."); + is($runlog->end_time, within(time() - 1, 5), "The end time is also recent."); is($runlog->exit_code, undef, "This command should not have executed."); is($runlog->error_number, 2, "This command failed to exec."); }; diff --git a/t/Hydra/Schema/Result/RunCommandLogs.t b/t/Hydra/Schema/Result/RunCommandLogs.t index 80589549..f702fcf9 100644 --- a/t/Hydra/Schema/Result/RunCommandLogs.t +++ b/t/Hydra/Schema/Result/RunCommandLogs.t @@ -55,7 +55,7 @@ subtest "Starting a process" => sub { ok($runlog->is_running(), "The process is running."); ok(!$runlog->did_fail_with_signal(), "The process was not killed by a signal."); ok(!$runlog->did_fail_with_exec_error(), "The process did not fail to start due to an exec error."); - is($runlog->start_time, within(time() - 1, 2), "The start time is recent."); + is($runlog->start_time, within(time() - 1, 5), "The start time is recent."); is($runlog->end_time, undef, "The end time is undefined."); is($runlog->exit_code, undef, "The exit code is undefined."); is($runlog->signal, undef, "The signal is undefined."); @@ -70,8 +70,8 @@ subtest "The process completed (success)" => sub { ok(!$runlog->is_running(), "The process is not running."); ok(!$runlog->did_fail_with_signal(), "The process was not killed by a signal."); ok(!$runlog->did_fail_with_exec_error(), "The process did not fail to start due to an exec error."); - is($runlog->start_time, within(time() - 1, 2), "The start time is recent."); - is($runlog->end_time, within(time() - 1, 2), "The end time is recent."); + is($runlog->start_time, within(time() - 1, 5), "The start time is recent."); + is($runlog->end_time, within(time() - 1, 5), "The end time is recent."); is($runlog->error_number, undef, "The error number is undefined"); is($runlog->exit_code, 0, "The exit code is 0."); is($runlog->signal, undef, "The signal is undefined."); @@ -86,8 +86,8 @@ subtest "The process completed (errored)" => sub { ok(!$runlog->is_running(), "The process is not running."); ok(!$runlog->did_fail_with_signal(), "The process was not killed by a signal."); ok(!$runlog->did_fail_with_exec_error(), "The process did not fail to start due to an exec error."); - is($runlog->start_time, within(time() - 1, 2), "The start time is recent."); - is($runlog->end_time, within(time() - 1, 2), "The end time is recent."); + is($runlog->start_time, within(time() - 1, 5), "The start time is recent."); + is($runlog->end_time, within(time() - 1, 5), "The end time is recent."); is($runlog->error_number, undef, "The error number is undefined"); is($runlog->exit_code, 85, "The exit code is 85."); is($runlog->signal, undef, "The signal is undefined."); @@ -102,8 +102,8 @@ subtest "The process completed (status 15, child error 0)" => sub { ok(!$runlog->is_running(), "The process is not running."); ok($runlog->did_fail_with_signal(), "The process was killed by a signal."); ok(!$runlog->did_fail_with_exec_error(), "The process did not fail to start due to an exec error."); - is($runlog->start_time, within(time() - 1, 2), "The start time is recent."); - is($runlog->end_time, within(time() - 1, 2), "The end time is recent."); + is($runlog->start_time, within(time() - 1, 5), "The start time is recent."); + is($runlog->end_time, within(time() - 1, 5), "The end time is recent."); is($runlog->error_number, undef, "The error number is undefined"); is($runlog->exit_code, undef, "The exit code is undefined."); is($runlog->signal, 15, "Signal 15 was sent."); @@ -118,8 +118,8 @@ subtest "The process completed (signaled)" => sub { ok(!$runlog->is_running(), "The process is not running."); ok($runlog->did_fail_with_signal(), "The process was killed by a signal."); ok(!$runlog->did_fail_with_exec_error(), "The process did not fail to start due to an exec error."); - is($runlog->start_time, within(time() - 1, 2), "The start time is recent."); - is($runlog->end_time, within(time() - 1, 2), "The end time is recent."); + is($runlog->start_time, within(time() - 1, 5), "The start time is recent."); + is($runlog->end_time, within(time() - 1, 5), "The end time is recent."); is($runlog->error_number, undef, "The error number is undefined"); is($runlog->exit_code, undef, "The exit code is undefined."); is($runlog->signal, 9, "The signal is 9."); @@ -134,8 +134,8 @@ subtest "The process failed to start" => sub { ok(!$runlog->is_running(), "The process is running."); ok(!$runlog->did_fail_with_signal(), "The process was not killed by a signal."); ok($runlog->did_fail_with_exec_error(), "The process failed to start due to an exec error."); - is($runlog->start_time, within(time() - 1, 2), "The start time is recent."); - is($runlog->end_time, within(time() - 1, 2), "The end time is recent."); + is($runlog->start_time, within(time() - 1, 5), "The start time is recent."); + is($runlog->end_time, within(time() - 1, 5), "The end time is recent."); is($runlog->error_number, 2, "The error number is saved"); is($runlog->exit_code, undef, "The exit code is undefined."); is($runlog->signal, undef, "The signal is undefined."); diff --git a/t/Hydra/Schema/Result/TaskRetries.t b/t/Hydra/Schema/Result/TaskRetries.t index 0425f11c..a9c9f132 100644 --- a/t/Hydra/Schema/Result/TaskRetries.t +++ b/t/Hydra/Schema/Result/TaskRetries.t @@ -25,11 +25,11 @@ subtest "requeue" => sub { $task->requeue(); is($task->attempts, 2, "We should have stored a second retry"); - is($task->retry_at, within(time() + 4, 2), "Delayed two exponential backoff step"); + is($task->retry_at, within(time() + 4, 5), "Delayed two exponential backoff step"); $task->requeue(); is($task->attempts, 3, "We should have stored a third retry"); - is($task->retry_at, within(time() + 8, 2), "Delayed a third exponential backoff step"); + is($task->retry_at, within(time() + 8, 5), "Delayed a third exponential backoff step"); }; done_testing; diff --git a/t/Hydra/Schema/ResultSet/TaskRetries.t b/t/Hydra/Schema/ResultSet/TaskRetries.t index 4555832c..a9354896 100644 --- a/t/Hydra/Schema/ResultSet/TaskRetries.t +++ b/t/Hydra/Schema/ResultSet/TaskRetries.t @@ -101,7 +101,7 @@ subtest "save_task" => sub { is($retry->pluginname, "FooPluginName", "Plugin name should match"); is($retry->payload, "1", "Payload should match"); is($retry->attempts, 1, "We've had one attempt"); - is($retry->retry_at, within(time() + 1, 2), "The retry at should be approximately one second away"); + is($retry->retry_at, within(time() + 1, 5), "The retry at should be approximately one second away"); }; done_testing; From 5c35d1be2005ee0471076df5c904c72f4c748d66 Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Sun, 25 Jun 2023 17:25:43 +0200 Subject: [PATCH 033/313] hydra-queue-runner: fix stats --- src/hydra-queue-runner/hydra-queue-runner.cc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/hydra-queue-runner/hydra-queue-runner.cc b/src/hydra-queue-runner/hydra-queue-runner.cc index b84681d5..30fad267 100644 --- a/src/hydra-queue-runner/hydra-queue-runner.cc +++ b/src/hydra-queue-runner/hydra-queue-runner.cc @@ -618,7 +618,7 @@ void State::dumpStatus(Connection & conn) } { - auto jobsets_json = statusJson["jobsets"] = json::object(); + auto jobsets_json = json::object(); auto jobsets_(jobsets.lock()); for (auto & jobset : *jobsets_) { jobsets_json[jobset.first.first + ":" + jobset.first.second] = { @@ -626,10 +626,11 @@ void State::dumpStatus(Connection & conn) {"seconds", jobset.second->getSeconds()}, }; } + statusJson["jobsets"] = jobsets_json; } { - auto machineTypesJson = statusJson["machineTypes"] = json::object(); + auto machineTypesJson = json::object(); auto machineTypes_(machineTypes.lock()); for (auto & i : *machineTypes_) { auto machineTypeJson = machineTypesJson[i.first] = { @@ -642,6 +643,7 @@ void State::dumpStatus(Connection & conn) if (i.second.running == 0) machineTypeJson["lastActive"] = std::chrono::system_clock::to_time_t(i.second.lastActive); } + statusJson["machineTypes"] = machineTypesJson; } auto store = getDestStore(); From 46246dcae3ad05dfd85fcf4f61b44f5c2114634a Mon Sep 17 00:00:00 2001 From: Arian van Putten Date: Wed, 19 Jul 2023 15:13:25 +0200 Subject: [PATCH 034/313] Fix docs for /eval/{id} endpoint You need to pass it an eval-id, not a build-id pretty sure --- hydra-api.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/hydra-api.yaml b/hydra-api.yaml index ce7e0f9a..c3068d66 100644 --- a/hydra-api.yaml +++ b/hydra-api.yaml @@ -533,13 +533,13 @@ paths: schema: $ref: '#/components/schemas/Error' - /eval/{build-id}: + /eval/{eval-id}: get: - summary: Retrieves evaluations identified by build id + summary: Retrieves evaluations identified by eval id parameters: - - name: build-id + - name: eval-id in: path - description: build identifier + description: eval identifier required: true schema: type: integer From a78664f1b5faac4ade5b3a2f3c0e6c6e59a94c51 Mon Sep 17 00:00:00 2001 From: Arian van Putten Date: Thu, 20 Jul 2023 14:43:03 +0200 Subject: [PATCH 035/313] Fix documentation of defaultpath in api docs --- hydra-api.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hydra-api.yaml b/hydra-api.yaml index ce7e0f9a..2646a275 100644 --- a/hydra-api.yaml +++ b/hydra-api.yaml @@ -870,7 +870,7 @@ components: description: Size of the produced file type: integer defaultpath: - description: This is a Git/Mercurial commit hash or a Subversion revision number + description: if path is a directory, the default file relative to path to be served type: string 'type': description: Types of build product (user defined) From b23431a657d8a9b2f478c95dd81034780751a262 Mon Sep 17 00:00:00 2001 From: Linus Heckemann Date: Fri, 4 Aug 2023 15:53:06 +0200 Subject: [PATCH 036/313] Support Nix 2.17 --- flake.lock | 8 ++++---- flake.nix | 2 +- src/hydra-queue-runner/build-remote.cc | 25 +++++++++++++++---------- 3 files changed, 20 insertions(+), 15 deletions(-) diff --git a/flake.lock b/flake.lock index ee85f6fa..4b18fbb4 100644 --- a/flake.lock +++ b/flake.lock @@ -42,16 +42,16 @@ "nixpkgs-regression": "nixpkgs-regression" }, "locked": { - "lastModified": 1686048923, - "narHash": "sha256-/XCWa2osNFIpPC5MkxlX6qTZf/DaTLwS3LWN0SRFiuU=", + "lastModified": 1690219894, + "narHash": "sha256-QMYAkdtU+g9HlZKtoJ+AI6TbWzzovKGnPZJHfZdclc8=", "owner": "NixOS", "repo": "nix", - "rev": "84050709ea18f3285a85d729f40c8f8eddf5008e", + "rev": "a212300a1d9f9c7b0daf19c00c87fc50480f54f4", "type": "github" }, "original": { "owner": "NixOS", - "ref": "2.16.1", + "ref": "2.17.0", "repo": "nix", "type": "github" } diff --git a/flake.nix b/flake.nix index 6bbec9b0..7e7d50e2 100644 --- a/flake.nix +++ b/flake.nix @@ -2,7 +2,7 @@ description = "A Nix-based continuous build system"; inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.05"; - inputs.nix.url = "github:NixOS/nix/2.16.1"; + inputs.nix.url = "github:NixOS/nix/2.17.0"; inputs.nix.inputs.nixpkgs.follows = "nixpkgs"; outputs = { self, nixpkgs, nix }: diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc index 6baff7df..6bbd22e2 100644 --- a/src/hydra-queue-runner/build-remote.cc +++ b/src/hydra-queue-runner/build-remote.cc @@ -6,6 +6,7 @@ #include #include "build-result.hh" +#include "path.hh" #include "serve-protocol.hh" #include "state.hh" #include "util.hh" @@ -110,18 +111,20 @@ static void copyClosureTo(std::timed_mutex & sendMutex, Store & destStore, StorePathSet closure; destStore.computeFSClosure(paths, closure); + WorkerProto::WriteConn wconn { .to = to }; + WorkerProto::ReadConn rconn { .from = from }; /* Send the "query valid paths" command with the "lock" option enabled. This prevents a race where the remote host garbage-collect paths that are already there. Optionally, ask the remote host to substitute missing paths. */ // FIXME: substitute output pollutes our build log - to << cmdQueryValidPaths << 1 << useSubstitutes; - workerProtoWrite(destStore, to, closure); + to << ServeProto::Command::QueryValidPaths << 1 << useSubstitutes; + WorkerProto::write(destStore, wconn, closure); to.flush(); /* Get back the set of paths that are already valid on the remote host. */ - auto present = WorkerProto::read(destStore, from); + auto present = WorkerProto::Serialise::read(destStore, rconn); if (present.size() == closure.size()) return; @@ -136,7 +139,7 @@ static void copyClosureTo(std::timed_mutex & sendMutex, Store & destStore, std::unique_lock sendLock(sendMutex, std::chrono::seconds(600)); - to << cmdImportPaths; + to << ServeProto::Command::ImportPaths; destStore.exportPaths(missing, to); to.flush(); @@ -223,7 +226,9 @@ void State::buildRemote(ref destStore, }); FdSource from(child.from.get()); + WorkerProto::ReadConn rconn { .from = from }; FdSink to(child.to.get()); + WorkerProto::WriteConn wconn { .to = to }; Finally updateStats([&]() { bytesReceived += from.read; @@ -334,7 +339,7 @@ void State::buildRemote(ref destStore, updateStep(ssBuilding); - to << cmdBuildDerivation << localStore->printStorePath(step->drvPath); + to << ServeProto::Command::BuildDerivation << localStore->printStorePath(step->drvPath); writeDerivation(to, *localStore, basicDrv); to << maxSilentTime << buildTimeout; if (GET_PROTOCOL_MINOR(remoteVersion) >= 2) @@ -367,7 +372,7 @@ void State::buildRemote(ref destStore, } } if (GET_PROTOCOL_MINOR(remoteVersion) >= 6) { - WorkerProto::read(*localStore, from); + WorkerProto::Serialise::read(*localStore, rconn); } switch ((BuildResult::Status) res) { case BuildResult::Built: @@ -443,14 +448,14 @@ void State::buildRemote(ref destStore, /* Get info about each output path. */ std::map infos; size_t totalNarSize = 0; - to << cmdQueryPathInfos; - workerProtoWrite(*localStore, to, outputs); + to << ServeProto::Command::QueryPathInfos; + WorkerProto::write(*localStore, wconn, outputs); to.flush(); while (true) { auto storePathS = readString(from); if (storePathS == "") break; auto deriver = readString(from); // deriver - auto references = WorkerProto::read(*localStore, from); + auto references = WorkerProto::Serialise::read(*localStore, rconn); readLongLong(from); // download size auto narSize = readLongLong(from); auto narHash = Hash::parseAny(readString(from), htSHA256); @@ -494,7 +499,7 @@ void State::buildRemote(ref destStore, lambda function only gets executed if someone tries to read from source2, we will send the command from here rather than outside the lambda. */ - to << cmdDumpStorePath << localStore->printStorePath(path); + to << ServeProto::Command::DumpStorePath << localStore->printStorePath(path); to.flush(); TeeSource tee(from, sink); From 9f0427385fa9f306c4cd651dc97377624b90997c Mon Sep 17 00:00:00 2001 From: Linus Heckemann Date: Sun, 20 Aug 2023 14:55:56 +0200 Subject: [PATCH 037/313] Apply LTO fix suggested by Ericson2314 --- src/hydra-queue-runner/build-remote.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc index 6bbd22e2..46c94e60 100644 --- a/src/hydra-queue-runner/build-remote.cc +++ b/src/hydra-queue-runner/build-remote.cc @@ -11,6 +11,7 @@ #include "state.hh" #include "util.hh" #include "worker-protocol.hh" +#include "worker-protocol-impl.hh" #include "finally.hh" #include "url.hh" From 35ccc9ebb22796270aafe1b567420e13529e3be5 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 23 Aug 2023 17:04:45 +0200 Subject: [PATCH 038/313] Fix indentation Co-authored-by: John Ericson --- src/hydra-queue-runner/build-remote.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc index 46c94e60..56ce1ccf 100644 --- a/src/hydra-queue-runner/build-remote.cc +++ b/src/hydra-queue-runner/build-remote.cc @@ -373,7 +373,7 @@ void State::buildRemote(ref destStore, } } if (GET_PROTOCOL_MINOR(remoteVersion) >= 6) { - WorkerProto::Serialise::read(*localStore, rconn); + WorkerProto::Serialise::read(*localStore, rconn); } switch ((BuildResult::Status) res) { case BuildResult::Built: From 113836ebae04c518b9c787983da9d43af4e27f73 Mon Sep 17 00:00:00 2001 From: sternenseemann Date: Wed, 30 Aug 2023 15:06:48 +0200 Subject: [PATCH 039/313] hydra-api.yaml: name JobsetEval parameter eval-id This is more accurate since the id space is not shared between build and eval ids. --- hydra-api.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/hydra-api.yaml b/hydra-api.yaml index 2646a275..f61a27aa 100644 --- a/hydra-api.yaml +++ b/hydra-api.yaml @@ -533,13 +533,13 @@ paths: schema: $ref: '#/components/schemas/Error' - /eval/{build-id}: + /eval/{eval-id}: get: - summary: Retrieves evaluations identified by build id + summary: Retrieves evaluations identified by eval id parameters: - - name: build-id + - name: eval-id in: path - description: build identifier + description: eval identifier required: true schema: type: integer From e2195c46d12e1d5ff14dd3d9319e405e2b2b49ab Mon Sep 17 00:00:00 2001 From: sternenseemann Date: Wed, 30 Aug 2023 15:08:11 +0200 Subject: [PATCH 040/313] hydra-api.yaml: document all_builds (/eval/{eval-id}/builds) --- hydra-api.yaml | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/hydra-api.yaml b/hydra-api.yaml index f61a27aa..623c9082 100644 --- a/hydra-api.yaml +++ b/hydra-api.yaml @@ -551,6 +551,24 @@ paths: schema: $ref: '#/components/schemas/JobsetEval' + /eval/{eval-id}/builds: + get: + summary: Retrieves all builds belonging to an evaluation identified by eval id + parameters: + - name: eval-id + in: path + description: eval identifier + required: true + schema: + type: integer + responses: + '200': + description: builds + content: + application/json: + schema: + $ref: '#/components/schemas/JobsetEvalBuilds' + components: schemas: @@ -796,6 +814,13 @@ components: additionalProperties: $ref: '#/components/schemas/JobsetEvalInput' + JobsetEvalBuilds: + type: array + items: + type: object + additionalProperties: + $ref: '#/components/schemas/Build' + JobsetOverview: type: array items: From b7c864c515faee58a11a67ea8619e464e883ad34 Mon Sep 17 00:00:00 2001 From: Pierre Bourdon Date: Fri, 8 Sep 2023 23:38:30 +0200 Subject: [PATCH 041/313] queue-runner: only re-sort runnables by prio once per dispatch cycle MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous implementation was O(N²lg(N)) due to sorting the full runnables priority list once per runnable being scheduled. While not confirmed, this is suspected to cause performance issues and bottlenecking with the queue runner when the runnable list gets large enough. This commit changes the dispatcher to instead only sort runnables per priority once per dispatch cycle. This has the drawback of being less reactive to runnable priority changes: the previous code would react immediately, while this might end up using "old" priorities until the next dispatch cycle. However, dispatch cycles are not supposed to take very long (seconds, not minutes/hours), so this is not expected to have much or any practical impact. Ideally runnables would be maintained in a sorted data structure instead of the current approach of copying + sorting in the scheduler. This would however be a much more invasive change to implement, and might have to wait until we can confirm where the queue runner bottlenecks actually lie. --- src/hydra-queue-runner/dispatcher.cc | 205 ++++++++++++++------------- 1 file changed, 106 insertions(+), 99 deletions(-) diff --git a/src/hydra-queue-runner/dispatcher.cc b/src/hydra-queue-runner/dispatcher.cc index 1e40fa69..af21dcbd 100644 --- a/src/hydra-queue-runner/dispatcher.cc +++ b/src/hydra-queue-runner/dispatcher.cc @@ -85,12 +85,113 @@ system_time State::doDispatch() } } + system_time now = std::chrono::system_clock::now(); + /* Start steps until we're out of steps or slots. */ auto sleepUntil = system_time::max(); bool keepGoing; + /* Sort the runnable steps by priority. Priority is establised + as follows (in order of precedence): + + - The global priority of the builds that depend on the + step. This allows admins to bump a build to the front of + the queue. + + - The lowest used scheduling share of the jobsets depending + on the step. + + - The local priority of the build, as set via the build's + meta.schedulingPriority field. Note that this is not + quite correct: the local priority should only be used to + establish priority between builds in the same jobset, but + here it's used between steps in different jobsets if they + happen to have the same lowest used scheduling share. But + that's not very likely. + + - The lowest ID of the builds depending on the step; + i.e. older builds take priority over new ones. + + FIXME: O(n lg n); obviously, it would be better to keep a + runnable queue sorted by priority. */ + struct StepInfo + { + Step::ptr step; + bool alreadyScheduled = false; + + /* The lowest share used of any jobset depending on this + step. */ + double lowestShareUsed = 1e9; + + /* Info copied from step->state to ensure that the + comparator is a partial ordering (see MachineInfo). */ + int highestGlobalPriority; + int highestLocalPriority; + BuildID lowestBuildID; + + StepInfo(Step::ptr step, Step::State & step_) : step(step) + { + for (auto & jobset : step_.jobsets) + lowestShareUsed = std::min(lowestShareUsed, jobset->shareUsed()); + highestGlobalPriority = step_.highestGlobalPriority; + highestLocalPriority = step_.highestLocalPriority; + lowestBuildID = step_.lowestBuildID; + } + }; + + std::vector runnableSorted; + + struct RunnablePerType + { + unsigned int count{0}; + std::chrono::seconds waitTime{0}; + }; + + std::unordered_map runnablePerType; + + { + auto runnable_(runnable.lock()); + runnableSorted.reserve(runnable_->size()); + for (auto i = runnable_->begin(); i != runnable_->end(); ) { + auto step = i->lock(); + + /* Remove dead steps. */ + if (!step) { + i = runnable_->erase(i); + continue; + } + + ++i; + + auto & r = runnablePerType[step->systemType]; + r.count++; + + /* Skip previously failed steps that aren't ready + to be retried. */ + auto step_(step->state.lock()); + r.waitTime += std::chrono::duration_cast(now - step_->runnableSince); + if (step_->tries > 0 && step_->after > now) { + if (step_->after < sleepUntil) + sleepUntil = step_->after; + continue; + } + + runnableSorted.emplace_back(step, *step_); + } + } + + sort(runnableSorted.begin(), runnableSorted.end(), + [](const StepInfo & a, const StepInfo & b) + { + return + a.highestGlobalPriority != b.highestGlobalPriority ? a.highestGlobalPriority > b.highestGlobalPriority : + a.lowestShareUsed != b.lowestShareUsed ? a.lowestShareUsed < b.lowestShareUsed : + a.highestLocalPriority != b.highestLocalPriority ? a.highestLocalPriority > b.highestLocalPriority : + a.lowestBuildID < b.lowestBuildID; + }); + do { - system_time now = std::chrono::system_clock::now(); + now = std::chrono::system_clock::now(); /* Copy the currentJobs field of each machine. This is necessary to ensure that the sort comparator below is @@ -138,104 +239,6 @@ system_time State::doDispatch() a.currentJobs > b.currentJobs; }); - /* Sort the runnable steps by priority. Priority is establised - as follows (in order of precedence): - - - The global priority of the builds that depend on the - step. This allows admins to bump a build to the front of - the queue. - - - The lowest used scheduling share of the jobsets depending - on the step. - - - The local priority of the build, as set via the build's - meta.schedulingPriority field. Note that this is not - quite correct: the local priority should only be used to - establish priority between builds in the same jobset, but - here it's used between steps in different jobsets if they - happen to have the same lowest used scheduling share. But - that's not very likely. - - - The lowest ID of the builds depending on the step; - i.e. older builds take priority over new ones. - - FIXME: O(n lg n); obviously, it would be better to keep a - runnable queue sorted by priority. */ - struct StepInfo - { - Step::ptr step; - - /* The lowest share used of any jobset depending on this - step. */ - double lowestShareUsed = 1e9; - - /* Info copied from step->state to ensure that the - comparator is a partial ordering (see MachineInfo). */ - int highestGlobalPriority; - int highestLocalPriority; - BuildID lowestBuildID; - - StepInfo(Step::ptr step, Step::State & step_) : step(step) - { - for (auto & jobset : step_.jobsets) - lowestShareUsed = std::min(lowestShareUsed, jobset->shareUsed()); - highestGlobalPriority = step_.highestGlobalPriority; - highestLocalPriority = step_.highestLocalPriority; - lowestBuildID = step_.lowestBuildID; - } - }; - - std::vector runnableSorted; - - struct RunnablePerType - { - unsigned int count{0}; - std::chrono::seconds waitTime{0}; - }; - - std::unordered_map runnablePerType; - - { - auto runnable_(runnable.lock()); - runnableSorted.reserve(runnable_->size()); - for (auto i = runnable_->begin(); i != runnable_->end(); ) { - auto step = i->lock(); - - /* Remove dead steps. */ - if (!step) { - i = runnable_->erase(i); - continue; - } - - ++i; - - auto & r = runnablePerType[step->systemType]; - r.count++; - - /* Skip previously failed steps that aren't ready - to be retried. */ - auto step_(step->state.lock()); - r.waitTime += std::chrono::duration_cast(now - step_->runnableSince); - if (step_->tries > 0 && step_->after > now) { - if (step_->after < sleepUntil) - sleepUntil = step_->after; - continue; - } - - runnableSorted.emplace_back(step, *step_); - } - } - - sort(runnableSorted.begin(), runnableSorted.end(), - [](const StepInfo & a, const StepInfo & b) - { - return - a.highestGlobalPriority != b.highestGlobalPriority ? a.highestGlobalPriority > b.highestGlobalPriority : - a.lowestShareUsed != b.lowestShareUsed ? a.lowestShareUsed < b.lowestShareUsed : - a.highestLocalPriority != b.highestLocalPriority ? a.highestLocalPriority > b.highestLocalPriority : - a.lowestBuildID < b.lowestBuildID; - }); - /* Find a machine with a free slot and find a step to run on it. Once we find such a pair, we restart the outer loop because the machine sorting will have changed. */ @@ -245,6 +248,8 @@ system_time State::doDispatch() if (mi.machine->state->currentJobs >= mi.machine->maxJobs) continue; for (auto & stepInfo : runnableSorted) { + if (stepInfo.alreadyScheduled) continue; + auto & step(stepInfo.step); /* Can this machine do this step? */ @@ -271,6 +276,8 @@ system_time State::doDispatch() r.count--; } + stepInfo.alreadyScheduled = true; + /* Make a slot reservation and start a thread to do the build. */ auto builderThread = std::thread(&State::builder, this, From 6a5fb9efaea35ca29836371307f5083576f421ab Mon Sep 17 00:00:00 2001 From: Stig Palmquist Date: Fri, 20 Oct 2023 00:09:28 +0200 Subject: [PATCH 042/313] Set output length of C::P::Argon2 hashes to 16 Since the default lengths in Crypt::Passphrase::Argon2 changed from 16 to 32 in in 0.009, some tests that expected the passphrase to be unchanged started failing. --- src/lib/Hydra/Schema/Result/Users.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/Hydra/Schema/Result/Users.pm b/src/lib/Hydra/Schema/Result/Users.pm index b3de6543..c28ae931 100644 --- a/src/lib/Hydra/Schema/Result/Users.pm +++ b/src/lib/Hydra/Schema/Result/Users.pm @@ -216,7 +216,7 @@ sub json_hint { sub _authenticator() { my $authenticator = Crypt::Passphrase->new( - encoder => 'Argon2', + encoder => { module => 'Argon2', output_size => 16 }, validators => [ (sub { my ($password, $hash) = @_; From e9da80fff6234fab2458173272ee0bedbe8935c3 Mon Sep 17 00:00:00 2001 From: chayleaf Date: Tue, 21 Nov 2023 18:41:52 +0700 Subject: [PATCH 043/313] support nix 2.18 --- flake.lock | 8 ++++---- flake.nix | 2 +- src/hydra-eval-jobs/hydra-eval-jobs.cc | 7 ++++--- src/hydra-queue-runner/build-remote.cc | 6 +++--- src/hydra-queue-runner/queue-monitor.cc | 4 ++-- 5 files changed, 14 insertions(+), 13 deletions(-) diff --git a/flake.lock b/flake.lock index 4b18fbb4..af913ea3 100644 --- a/flake.lock +++ b/flake.lock @@ -42,16 +42,16 @@ "nixpkgs-regression": "nixpkgs-regression" }, "locked": { - "lastModified": 1690219894, - "narHash": "sha256-QMYAkdtU+g9HlZKtoJ+AI6TbWzzovKGnPZJHfZdclc8=", + "lastModified": 1696259154, + "narHash": "sha256-WNmifcTsN9aG1ONkv+l2BC4sHZZxtNKy0keqBHXXQ7w=", "owner": "NixOS", "repo": "nix", - "rev": "a212300a1d9f9c7b0daf19c00c87fc50480f54f4", + "rev": "f5f4de6a550327b4b1a06123c2e450f1b92c73b6", "type": "github" }, "original": { "owner": "NixOS", - "ref": "2.17.0", + "ref": "2.18.1", "repo": "nix", "type": "github" } diff --git a/flake.nix b/flake.nix index 7e7d50e2..2f2abe62 100644 --- a/flake.nix +++ b/flake.nix @@ -2,7 +2,7 @@ description = "A Nix-based continuous build system"; inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.05"; - inputs.nix.url = "github:NixOS/nix/2.17.0"; + inputs.nix.url = "github:NixOS/nix/2.18.1"; inputs.nix.inputs.nixpkgs.follows = "nixpkgs"; outputs = { self, nixpkgs, nix }: diff --git a/src/hydra-eval-jobs/hydra-eval-jobs.cc b/src/hydra-eval-jobs/hydra-eval-jobs.cc index 79523944..30ab9740 100644 --- a/src/hydra-eval-jobs/hydra-eval-jobs.cc +++ b/src/hydra-eval-jobs/hydra-eval-jobs.cc @@ -7,6 +7,7 @@ #include "store-api.hh" #include "eval.hh" #include "eval-inline.hh" +#include "eval-settings.hh" #include "util.hh" #include "get-drvs.hh" #include "globals.hh" @@ -208,13 +209,13 @@ static void worker( for (auto & c : context) std::visit(overloaded { [&](const NixStringContextElem::Built & b) { - job["constituents"].push_back(state.store->printStorePath(b.drvPath)); + job["constituents"].push_back(b.drvPath->to_string(*state.store)); }, [&](const NixStringContextElem::Opaque & o) { }, [&](const NixStringContextElem::DrvDeep & d) { }, - }, c.raw()); + }, c.raw); state.forceList(*a->value, a->pos, "while evaluating the `constituents` attribute"); for (unsigned int n = 0; n < a->value->listSize(); ++n) { @@ -516,7 +517,7 @@ int main(int argc, char * * argv) auto drvPath2 = store->parseStorePath((std::string) (*job2)["drvPath"]); auto drv2 = store->readDerivation(drvPath2); job["constituents"].push_back(store->printStorePath(drvPath2)); - drv.inputDrvs[drvPath2] = {drv2.outputs.begin()->first}; + drv.inputDrvs.map[drvPath2].value = {drv2.outputs.begin()->first}; } if (brokenJobs.empty()) { diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc index 56ce1ccf..92438349 100644 --- a/src/hydra-queue-runner/build-remote.cc +++ b/src/hydra-queue-runner/build-remote.cc @@ -276,9 +276,9 @@ void State::buildRemote(ref destStore, for (auto & p : step->drv->inputSrcs) inputs.insert(p); - for (auto & input : step->drv->inputDrvs) { - auto drv2 = localStore->readDerivation(input.first); - for (auto & name : input.second) { + for (auto & [drvPath, node] : step->drv->inputDrvs.map) { + auto drv2 = localStore->readDerivation(drvPath); + for (auto & name : node.value) { if (auto i = get(drv2.outputs, name)) { auto outPath = i->path(*localStore, drv2.name, name); inputs.insert(*outPath); diff --git a/src/hydra-queue-runner/queue-monitor.cc b/src/hydra-queue-runner/queue-monitor.cc index 0bb167a2..6c339af6 100644 --- a/src/hydra-queue-runner/queue-monitor.cc +++ b/src/hydra-queue-runner/queue-monitor.cc @@ -315,7 +315,7 @@ bool State::getQueuedBuilds(Connection & conn, if (std::chrono::system_clock::now() > start + std::chrono::seconds(600)) { prom.queue_checks_early_exits.Increment(); break; - } + } } prom.queue_checks_finished.Increment(); @@ -561,7 +561,7 @@ Step::ptr State::createStep(ref destStore, printMsg(lvlDebug, "creating build step ‘%1%’", localStore->printStorePath(drvPath)); /* Create steps for the dependencies. */ - for (auto & i : step->drv->inputDrvs) { + for (auto & i : step->drv->inputDrvs.map) { auto dep = createStep(destStore, conn, build, i.first, 0, step, finishedDrvs, newSteps, newRunnable); if (dep) { auto step_(step->state.lock()); From 5db374cb500b687039ba4701b205ca7dfa67caba Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 30 Nov 2023 10:40:57 -0500 Subject: [PATCH 044/313] Cleanup deps - `nativeBuildInputs` vs `buildInputs` - narrow down `with`s for clarity - use `autoreconfHook` not `bootstrap` script These sorts of changes have also been done in the Nix repo. --- README.md | 2 +- bootstrap | 2 -- flake.nix | 31 +++++++++++++++++-------------- 3 files changed, 18 insertions(+), 17 deletions(-) delete mode 100755 bootstrap diff --git a/README.md b/README.md index 54cb9a93..2a085325 100644 --- a/README.md +++ b/README.md @@ -80,7 +80,7 @@ $ nix-build You can use the provided shell.nix to get a working development environment: ``` $ nix-shell -$ ./bootstrap +$ autoreconfPhase $ configurePhase # NOTE: not ./configure $ make ``` diff --git a/bootstrap b/bootstrap deleted file mode 100755 index 091b0ee4..00000000 --- a/bootstrap +++ /dev/null @@ -1,2 +0,0 @@ -#! /bin/sh -e -exec autoreconf -vfi diff --git a/flake.nix b/flake.nix index 7e7d50e2..ea96ac2e 100644 --- a/flake.nix +++ b/flake.nix @@ -61,10 +61,11 @@ }; - hydra = with final; let - perlDeps = buildEnv { + hydra = let + inherit (final) lib stdenv; + perlDeps = final.buildEnv { name = "hydra-perl-deps"; - paths = with perlPackages; lib.closePropagation + paths = with final.perlPackages; lib.closePropagation [ AuthenSASL CatalystActionREST @@ -98,7 +99,7 @@ FileSlurper FileWhich final.nix.perl-bindings - git + final.git IOCompress IPCRun IPCRun3 @@ -141,15 +142,20 @@ src = self; - buildInputs = - [ + nativeBuildInputs = + with final.buildPackages; [ makeWrapper - autoconf + autoreconfHook automake libtool - unzip nukeReferences pkg-config + mdbook + ]; + + buildInputs = + with final; [ + unzip libpqxx top-git mercurial @@ -162,7 +168,6 @@ final.nix perlDeps perl - mdbook pixz boost postgresql_13 @@ -172,7 +177,7 @@ prometheus-cpp ]; - checkInputs = [ + checkInputs = with final; [ cacert foreman glibcLocales @@ -181,7 +186,7 @@ python3 ]; - hydraPath = lib.makeBinPath ( + hydraPath = with final; lib.makeBinPath ( [ subversion openssh @@ -203,7 +208,7 @@ ] ++ lib.optionals stdenv.isLinux [ rpm dpkg cdrkit ] ); - OPENLDAP_ROOT = openldap; + OPENLDAP_ROOT = final.openldap; shellHook = '' pushd $(git rev-parse --show-toplevel) >/dev/null @@ -218,8 +223,6 @@ popd >/dev/null ''; - preConfigure = "autoreconf -vfi"; - NIX_LDFLAGS = [ "-lpthread" ]; enableParallelBuilding = true; From 2bda7ca642528e1a7c7188d72460fe90c89c4ebf Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 30 Nov 2023 11:31:58 -0500 Subject: [PATCH 045/313] Further use `Machine::Connection` to deduplicate --- src/hydra-queue-runner/build-remote.cc | 28 +++++++++++++------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc index a82a95e6..8f8d6532 100644 --- a/src/hydra-queue-runner/build-remote.cc +++ b/src/hydra-queue-runner/build-remote.cc @@ -107,27 +107,27 @@ static void openConnection(Machine::ptr machine, Path tmpDir, int stderrFD, Chil } -static void copyClosureTo(std::timed_mutex & sendMutex, Store & destStore, - FdSource & from, FdSink & to, const StorePathSet & paths, +static void copyClosureTo( + Machine::Connection & conn, + Store & destStore, + const StorePathSet & paths, bool useSubstitutes = false) { StorePathSet closure; destStore.computeFSClosure(paths, closure); - WorkerProto::WriteConn wconn { .to = to }; - WorkerProto::ReadConn rconn { .from = from }; /* Send the "query valid paths" command with the "lock" option enabled. This prevents a race where the remote host garbage-collect paths that are already there. Optionally, ask the remote host to substitute missing paths. */ // FIXME: substitute output pollutes our build log - to << ServeProto::Command::QueryValidPaths << 1 << useSubstitutes; - WorkerProto::write(destStore, wconn, closure); - to.flush(); + conn.to << ServeProto::Command::QueryValidPaths << 1 << useSubstitutes; + WorkerProto::write(destStore, conn, closure); + conn.to.flush(); /* Get back the set of paths that are already valid on the remote host. */ - auto present = WorkerProto::Serialise::read(destStore, rconn); + auto present = WorkerProto::Serialise::read(destStore, conn); if (present.size() == closure.size()) return; @@ -139,14 +139,14 @@ static void copyClosureTo(std::timed_mutex & sendMutex, Store & destStore, printMsg(lvlDebug, "sending %d missing paths", missing.size()); - std::unique_lock sendLock(sendMutex, + std::unique_lock sendLock(conn.machine->state->sendLock, std::chrono::seconds(600)); - to << ServeProto::Command::ImportPaths; - destStore.exportPaths(missing, to); - to.flush(); + conn.to << ServeProto::Command::ImportPaths; + destStore.exportPaths(missing, conn.to); + conn.to.flush(); - if (readInt(from) != 1) + if (readInt(conn.from) != 1) throw Error("remote machine failed to import closure"); } @@ -257,7 +257,7 @@ BasicDerivation sendInputs( destStore.computeFSClosure(basicDrv.inputSrcs, closure); copyPaths(destStore, localStore, closure, NoRepair, NoCheckSigs, NoSubstitute); } else { - copyClosureTo(conn.machine->state->sendLock, destStore, conn.from, conn.to, basicDrv.inputSrcs, true); + copyClosureTo(conn, destStore, basicDrv.inputSrcs, true); } auto now2 = std::chrono::steady_clock::now(); From 0917145622c521fff5c9af1b26053c2217bbd02b Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 30 Nov 2023 12:19:05 -0500 Subject: [PATCH 046/313] Make new functions not in header `static` --- src/hydra-queue-runner/build-remote.cc | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc index 8f8d6532..a5892016 100644 --- a/src/hydra-queue-runner/build-remote.cc +++ b/src/hydra-queue-runner/build-remote.cc @@ -152,7 +152,7 @@ static void copyClosureTo( // FIXME: use Store::topoSortPaths(). -StorePaths reverseTopoSortPaths(const std::map & paths) +static StorePaths reverseTopoSortPaths(const std::map & paths) { StorePaths sorted; StorePathSet visited; @@ -180,7 +180,7 @@ StorePaths reverseTopoSortPaths(const std::map & paths return sorted; } -std::pair openLogFile(const std::string & logDir, const StorePath & drvPath) +static std::pair openLogFile(const std::string & logDir, const StorePath & drvPath) { std::string base(drvPath.to_string()); auto logFile = logDir + "/" + std::string(base, 0, 2) + "/" + std::string(base, 2); @@ -193,7 +193,7 @@ std::pair openLogFile(const std::string & logDir, const Store return {std::move(logFile), std::move(logFD)}; } -void handshake(Machine::Connection & conn, unsigned int repeats) +static void handshake(Machine::Connection & conn, unsigned int repeats) { conn.to << SERVE_MAGIC_1 << 0x206; conn.to.flush(); @@ -208,7 +208,7 @@ void handshake(Machine::Connection & conn, unsigned int repeats) throw Error("machine ‘%1%’ does not support repeating a build; please upgrade it to Nix 1.12", conn.machine->sshName); } -BasicDerivation sendInputs( +static BasicDerivation sendInputs( State & state, Step & step, Store & localStore, @@ -268,7 +268,7 @@ BasicDerivation sendInputs( return basicDrv; } -BuildResult performBuild( +static BuildResult performBuild( Machine::Connection & conn, Store & localStore, StorePath drvPath, @@ -321,7 +321,7 @@ BuildResult performBuild( return result; } -std::map queryPathInfos( +static std::map queryPathInfos( Machine::Connection & conn, Store & localStore, StorePathSet & outputs, @@ -359,7 +359,7 @@ std::map queryPathInfos( return infos; } -void copyPathFromRemote( +static void copyPathFromRemote( Machine::Connection & conn, NarMemberDatas & narMembers, Store & localStore, @@ -389,7 +389,7 @@ void copyPathFromRemote( destStore.addToStore(info, *source2, NoRepair, NoCheckSigs); } -void copyPathsFromRemote( +static void copyPathsFromRemote( Machine::Connection & conn, NarMemberDatas & narMembers, Store & localStore, From e172461e550676a9a295cf917f9b6fab623af3e6 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 30 Nov 2023 12:19:20 -0500 Subject: [PATCH 047/313] Use `const` in for loop As requested by @teh --- src/hydra-queue-runner/build-remote.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc index a5892016..ca508b8b 100644 --- a/src/hydra-queue-runner/build-remote.cc +++ b/src/hydra-queue-runner/build-remote.cc @@ -221,7 +221,7 @@ static BasicDerivation sendInputs( { BasicDerivation basicDrv(*step.drv); - for (auto & input : step.drv->inputDrvs) { + for (const auto & input : step.drv->inputDrvs) { auto drv2 = localStore.readDerivation(input.first); for (auto & name : input.second) { if (auto i = get(drv2.outputs, name)) { From c922e73c11449759c8e1945ed01fb4cd120ef57d Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 30 Nov 2023 14:38:26 -0500 Subject: [PATCH 048/313] Update to Nix 2.19 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nix': 'github:NixOS/nix/f5f4de6a550327b4b1a06123c2e450f1b92c73b6' (2023-10-02) → 'github:NixOS/nix/50f8f1c8bc019a4c0fd098b9ac674b94cfc6af0d' (2023-11-27) --- flake.lock | 8 ++-- flake.nix | 2 +- src/hydra-eval-jobs/hydra-eval-jobs.cc | 8 ++-- src/hydra-evaluator/hydra-evaluator.cc | 1 + src/hydra-queue-runner/build-remote.cc | 43 +++++++++++++------- src/hydra-queue-runner/build-result.cc | 12 +++--- src/hydra-queue-runner/hydra-queue-runner.cc | 3 +- src/hydra-queue-runner/nar-extractor.cc | 12 ++++-- src/hydra-queue-runner/nar-extractor.hh | 4 +- src/libhydra/db.hh | 1 + src/libhydra/hydra-config.hh | 1 + 11 files changed, 61 insertions(+), 34 deletions(-) diff --git a/flake.lock b/flake.lock index af913ea3..9a9046c8 100644 --- a/flake.lock +++ b/flake.lock @@ -42,16 +42,16 @@ "nixpkgs-regression": "nixpkgs-regression" }, "locked": { - "lastModified": 1696259154, - "narHash": "sha256-WNmifcTsN9aG1ONkv+l2BC4sHZZxtNKy0keqBHXXQ7w=", + "lastModified": 1701122567, + "narHash": "sha256-iA8DqS+W2fWTfR+nNJSvMHqQ+4NpYMRT3b+2zS6JTvE=", "owner": "NixOS", "repo": "nix", - "rev": "f5f4de6a550327b4b1a06123c2e450f1b92c73b6", + "rev": "50f8f1c8bc019a4c0fd098b9ac674b94cfc6af0d", "type": "github" }, "original": { "owner": "NixOS", - "ref": "2.18.1", + "ref": "2.19.2", "repo": "nix", "type": "github" } diff --git a/flake.nix b/flake.nix index 2f2abe62..4e8f6dd0 100644 --- a/flake.nix +++ b/flake.nix @@ -2,7 +2,7 @@ description = "A Nix-based continuous build system"; inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.05"; - inputs.nix.url = "github:NixOS/nix/2.18.1"; + inputs.nix.url = "github:NixOS/nix/2.19.2"; inputs.nix.inputs.nixpkgs.follows = "nixpkgs"; outputs = { self, nixpkgs, nix }: diff --git a/src/hydra-eval-jobs/hydra-eval-jobs.cc b/src/hydra-eval-jobs/hydra-eval-jobs.cc index 30ab9740..2fe2c80f 100644 --- a/src/hydra-eval-jobs/hydra-eval-jobs.cc +++ b/src/hydra-eval-jobs/hydra-eval-jobs.cc @@ -8,6 +8,8 @@ #include "eval.hh" #include "eval-inline.hh" #include "eval-settings.hh" +#include "signals.hh" +#include "terminal.hh" #include "util.hh" #include "get-drvs.hh" #include "globals.hh" @@ -54,7 +56,7 @@ using namespace nix; static Path gcRootsDir; static size_t maxMemorySize; -struct MyArgs : MixEvalArgs, MixCommonArgs +struct MyArgs : MixEvalArgs, MixCommonArgs, RootArgs { Path releaseExpr; bool flake = false; @@ -95,7 +97,7 @@ static std::string queryMetaStrings(EvalState & state, DrvInfo & drv, const std: rec = [&](Value & v) { state.forceValue(v, noPos); if (v.type() == nString) - res.push_back(v.string.s); + res.emplace_back(v.string_view()); else if (v.isList()) for (unsigned int n = 0; n < v.listSize(); ++n) rec(*v.listElems()[n]); @@ -222,7 +224,7 @@ static void worker( auto v = a->value->listElems()[n]; state.forceValue(*v, noPos); if (v->type() == nString) - job["namedConstituents"].push_back(v->str()); + job["namedConstituents"].push_back(v->string_view()); } } diff --git a/src/hydra-evaluator/hydra-evaluator.cc b/src/hydra-evaluator/hydra-evaluator.cc index a1ccf047..75506ff8 100644 --- a/src/hydra-evaluator/hydra-evaluator.cc +++ b/src/hydra-evaluator/hydra-evaluator.cc @@ -2,6 +2,7 @@ #include "hydra-config.hh" #include "pool.hh" #include "shared.hh" +#include "signals.hh" #include #include diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc index 92438349..5552e91a 100644 --- a/src/hydra-queue-runner/build-remote.cc +++ b/src/hydra-queue-runner/build-remote.cc @@ -9,9 +9,11 @@ #include "path.hh" #include "serve-protocol.hh" #include "state.hh" +#include "current-process.hh" +#include "processes.hh" #include "util.hh" -#include "worker-protocol.hh" -#include "worker-protocol-impl.hh" +#include "serve-protocol.hh" +#include "serve-protocol-impl.hh" #include "finally.hh" #include "url.hh" @@ -106,26 +108,32 @@ static void openConnection(Machine::ptr machine, Path tmpDir, int stderrFD, Chil static void copyClosureTo(std::timed_mutex & sendMutex, Store & destStore, - FdSource & from, FdSink & to, const StorePathSet & paths, + FdSource & from, FdSink & to, ServeProto::Version remoteVersion, const StorePathSet & paths, bool useSubstitutes = false) { StorePathSet closure; destStore.computeFSClosure(paths, closure); - WorkerProto::WriteConn wconn { .to = to }; - WorkerProto::ReadConn rconn { .from = from }; + ServeProto::WriteConn wconn { + .to = to, + .version = remoteVersion, + }; + ServeProto::ReadConn rconn { + .from = from, + .version = remoteVersion, + }; /* Send the "query valid paths" command with the "lock" option enabled. This prevents a race where the remote host garbage-collect paths that are already there. Optionally, ask the remote host to substitute missing paths. */ // FIXME: substitute output pollutes our build log to << ServeProto::Command::QueryValidPaths << 1 << useSubstitutes; - WorkerProto::write(destStore, wconn, closure); + ServeProto::write(destStore, wconn, closure); to.flush(); /* Get back the set of paths that are already valid on the remote host. */ - auto present = WorkerProto::Serialise::read(destStore, rconn); + auto present = ServeProto::Serialise::read(destStore, rconn); if (present.size() == closure.size()) return; @@ -227,9 +235,7 @@ void State::buildRemote(ref destStore, }); FdSource from(child.from.get()); - WorkerProto::ReadConn rconn { .from = from }; FdSink to(child.to.get()); - WorkerProto::WriteConn wconn { .to = to }; Finally updateStats([&]() { bytesReceived += from.read; @@ -237,7 +243,7 @@ void State::buildRemote(ref destStore, }); /* Handshake. */ - unsigned int remoteVersion; + ServeProto::Version remoteVersion; try { to << SERVE_MAGIC_1 << 0x206; @@ -258,6 +264,15 @@ void State::buildRemote(ref destStore, throw Error("cannot connect to ‘%1%’: %2%", machine->sshName, s); } + ServeProto::ReadConn rconn { + .from = from, + .version = remoteVersion, + }; + ServeProto::WriteConn wconn { + .to = to, + .version = remoteVersion, + }; + { auto info(machine->state->connectInfo.lock()); info->consecutiveFailures = 0; @@ -313,7 +328,7 @@ void State::buildRemote(ref destStore, destStore->computeFSClosure(inputs, closure); copyPaths(*destStore, *localStore, closure, NoRepair, NoCheckSigs, NoSubstitute); } else { - copyClosureTo(machine->state->sendLock, *destStore, from, to, inputs, true); + copyClosureTo(machine->state->sendLock, *destStore, from, to, remoteVersion, inputs, true); } auto now2 = std::chrono::steady_clock::now(); @@ -373,7 +388,7 @@ void State::buildRemote(ref destStore, } } if (GET_PROTOCOL_MINOR(remoteVersion) >= 6) { - WorkerProto::Serialise::read(*localStore, rconn); + ServeProto::Serialise::read(*localStore, rconn); } switch ((BuildResult::Status) res) { case BuildResult::Built: @@ -450,13 +465,13 @@ void State::buildRemote(ref destStore, std::map infos; size_t totalNarSize = 0; to << ServeProto::Command::QueryPathInfos; - WorkerProto::write(*localStore, wconn, outputs); + ServeProto::write(*localStore, wconn, outputs); to.flush(); while (true) { auto storePathS = readString(from); if (storePathS == "") break; auto deriver = readString(from); // deriver - auto references = WorkerProto::Serialise::read(*localStore, rconn); + auto references = ServeProto::Serialise::read(*localStore, rconn); readLongLong(from); // download size auto narSize = readLongLong(from); auto narHash = Hash::parseAny(readString(from), htSHA256); diff --git a/src/hydra-queue-runner/build-result.cc b/src/hydra-queue-runner/build-result.cc index ea8b4a6a..691c1f19 100644 --- a/src/hydra-queue-runner/build-result.cc +++ b/src/hydra-queue-runner/build-result.cc @@ -1,7 +1,7 @@ #include "hydra-build-result.hh" #include "store-api.hh" #include "util.hh" -#include "fs-accessor.hh" +#include "source-accessor.hh" #include @@ -63,7 +63,7 @@ BuildOutput getBuildOutput( auto productsFile = narMembers.find(outputS + "/nix-support/hydra-build-products"); if (productsFile == narMembers.end() || - productsFile->second.type != FSAccessor::Type::tRegular) + productsFile->second.type != SourceAccessor::Type::tRegular) continue; assert(productsFile->second.contents); @@ -94,7 +94,7 @@ BuildOutput getBuildOutput( product.name = product.path == store->printStorePath(output) ? "" : baseNameOf(product.path); - if (file->second.type == FSAccessor::Type::tRegular) { + if (file->second.type == SourceAccessor::Type::tRegular) { product.isRegular = true; product.fileSize = file->second.fileSize.value(); product.sha256hash = file->second.sha256.value(); @@ -117,7 +117,7 @@ BuildOutput getBuildOutput( auto file = narMembers.find(product.path); assert(file != narMembers.end()); - if (file->second.type == FSAccessor::Type::tDirectory) + if (file->second.type == SourceAccessor::Type::tDirectory) res.products.push_back(product); } } @@ -126,7 +126,7 @@ BuildOutput getBuildOutput( for (auto & output : outputs) { auto file = narMembers.find(store->printStorePath(output) + "/nix-support/hydra-release-name"); if (file == narMembers.end() || - file->second.type != FSAccessor::Type::tRegular) + file->second.type != SourceAccessor::Type::tRegular) continue; res.releaseName = trim(file->second.contents.value()); // FIXME: validate release name @@ -136,7 +136,7 @@ BuildOutput getBuildOutput( for (auto & output : outputs) { auto file = narMembers.find(store->printStorePath(output) + "/nix-support/hydra-metrics"); if (file == narMembers.end() || - file->second.type != FSAccessor::Type::tRegular) + file->second.type != SourceAccessor::Type::tRegular) continue; for (auto & line : tokenizeString(file->second.contents.value(), "\n")) { auto fields = tokenizeString>(line); diff --git a/src/hydra-queue-runner/hydra-queue-runner.cc b/src/hydra-queue-runner/hydra-queue-runner.cc index 91779288..1d54bb93 100644 --- a/src/hydra-queue-runner/hydra-queue-runner.cc +++ b/src/hydra-queue-runner/hydra-queue-runner.cc @@ -10,6 +10,7 @@ #include +#include "signals.hh" #include "state.hh" #include "hydra-build-result.hh" #include "store-api.hh" @@ -467,7 +468,7 @@ void State::markSucceededBuild(pqxx::work & txn, Build::ptr build, product.type, product.subtype, product.fileSize ? std::make_optional(*product.fileSize) : std::nullopt, - product.sha256hash ? std::make_optional(product.sha256hash->to_string(Base16, false)) : std::nullopt, + product.sha256hash ? std::make_optional(product.sha256hash->to_string(HashFormat::Base16, false)) : std::nullopt, product.path, product.name, product.defaultPath); diff --git a/src/hydra-queue-runner/nar-extractor.cc b/src/hydra-queue-runner/nar-extractor.cc index 9f0eb431..3c6857bf 100644 --- a/src/hydra-queue-runner/nar-extractor.cc +++ b/src/hydra-queue-runner/nar-extractor.cc @@ -24,13 +24,13 @@ struct Extractor : ParseSink void createDirectory(const Path & path) override { - members.insert_or_assign(prefix + path, NarMemberData { .type = FSAccessor::Type::tDirectory }); + members.insert_or_assign(prefix + path, NarMemberData { .type = SourceAccessor::Type::tDirectory }); } void createRegularFile(const Path & path) override { curMember = &members.insert_or_assign(prefix + path, NarMemberData { - .type = FSAccessor::Type::tRegular, + .type = SourceAccessor::Type::tRegular, .fileSize = 0, .contents = filesToKeep.count(path) ? std::optional("") : std::nullopt, }).first->second; @@ -66,8 +66,14 @@ struct Extractor : ParseSink void createSymlink(const Path & path, const std::string & target) override { - members.insert_or_assign(prefix + path, NarMemberData { .type = FSAccessor::Type::tSymlink }); + members.insert_or_assign(prefix + path, NarMemberData { .type = SourceAccessor::Type::tSymlink }); } + + void isExecutable() override + { } + + void closeRegularFile() override + { } }; diff --git a/src/hydra-queue-runner/nar-extractor.hh b/src/hydra-queue-runner/nar-extractor.hh index 45b2706c..2634135b 100644 --- a/src/hydra-queue-runner/nar-extractor.hh +++ b/src/hydra-queue-runner/nar-extractor.hh @@ -1,13 +1,13 @@ #pragma once -#include "fs-accessor.hh" +#include "source-accessor.hh" #include "types.hh" #include "serialise.hh" #include "hash.hh" struct NarMemberData { - nix::FSAccessor::Type type; + nix::SourceAccessor::Type type; std::optional fileSize; std::optional contents; std::optional sha256; diff --git a/src/libhydra/db.hh b/src/libhydra/db.hh index 00e8f406..1e927573 100644 --- a/src/libhydra/db.hh +++ b/src/libhydra/db.hh @@ -2,6 +2,7 @@ #include +#include "environment-variables.hh" #include "util.hh" diff --git a/src/libhydra/hydra-config.hh b/src/libhydra/hydra-config.hh index 1688c278..b1275896 100644 --- a/src/libhydra/hydra-config.hh +++ b/src/libhydra/hydra-config.hh @@ -2,6 +2,7 @@ #include +#include "file-system.hh" #include "util.hh" struct HydraConfig From 622c25e3c4da9a6f4fd44bfe425137597b81705a Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 4 Dec 2023 08:56:06 -0500 Subject: [PATCH 049/313] Sedding prior to merge --- src/hydra-queue-runner/build-remote.cc | 14 +++++++------- src/hydra-queue-runner/state.hh | 6 +++--- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc index ca508b8b..f679db0c 100644 --- a/src/hydra-queue-runner/build-remote.cc +++ b/src/hydra-queue-runner/build-remote.cc @@ -10,8 +10,8 @@ #include "serve-protocol.hh" #include "state.hh" #include "util.hh" -#include "worker-protocol.hh" -#include "worker-protocol-impl.hh" +#include "serve-protocol.hh" +#include "serve-protocol-impl.hh" #include "finally.hh" #include "url.hh" @@ -122,12 +122,12 @@ static void copyClosureTo( the remote host to substitute missing paths. */ // FIXME: substitute output pollutes our build log conn.to << ServeProto::Command::QueryValidPaths << 1 << useSubstitutes; - WorkerProto::write(destStore, conn, closure); + ServeProto::write(destStore, conn, closure); conn.to.flush(); /* Get back the set of paths that are already valid on the remote host. */ - auto present = WorkerProto::Serialise::read(destStore, conn); + auto present = ServeProto::Serialise::read(destStore, conn); if (present.size() == closure.size()) return; @@ -315,7 +315,7 @@ static BuildResult performBuild( } } if (GET_PROTOCOL_MINOR(conn.remoteVersion) >= 6) { - WorkerProto::Serialise::read(localStore, conn); + ServeProto::Serialise::read(localStore, conn); } return result; @@ -332,13 +332,13 @@ static std::map queryPathInfos( /* Get info about each output path. */ std::map infos; conn.to << ServeProto::Command::QueryPathInfos; - WorkerProto::write(localStore, conn, outputs); + ServeProto::write(localStore, conn, outputs); conn.to.flush(); while (true) { auto storePathS = readString(conn.from); if (storePathS == "") break; auto deriver = readString(conn.from); // deriver - auto references = WorkerProto::Serialise::read(localStore, conn); + auto references = ServeProto::Serialise::read(localStore, conn); readLongLong(conn.from); // download size auto narSize = readLongLong(conn.from); auto narHash = Hash::parseAny(readString(conn.from), htSHA256); diff --git a/src/hydra-queue-runner/state.hh b/src/hydra-queue-runner/state.hh index ce795700..dfeaefe7 100644 --- a/src/hydra-queue-runner/state.hh +++ b/src/hydra-queue-runner/state.hh @@ -21,7 +21,7 @@ #include "store-api.hh" #include "sync.hh" #include "nar-extractor.hh" -#include "worker-protocol.hh" +#include "serve-protocol.hh" typedef unsigned int BuildID; @@ -310,12 +310,12 @@ struct Machine // Backpointer to the machine ptr machine; - operator nix::WorkerProto::ReadConn () + operator nix::ServeProto::ReadConn () { return { .from = from }; } - operator nix::WorkerProto::WriteConn () + operator nix::ServeProto::WriteConn () { return { .to = to }; } From 104baef503df3afcd09615d9b014923696e9ecf9 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 4 Dec 2023 09:42:04 -0500 Subject: [PATCH 050/313] Document the connection initialization process --- src/hydra-queue-runner/build-remote.cc | 16 ++++++++++++---- src/hydra-queue-runner/state.hh | 2 +- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc index 032c2733..baf35d86 100644 --- a/src/hydra-queue-runner/build-remote.cc +++ b/src/hydra-queue-runner/build-remote.cc @@ -195,6 +195,12 @@ static std::pair openLogFile(const std::string & logDir, cons return {std::move(logFile), std::move(logFD)}; } +/** + * @param conn is not fully initialized; it is this functions job to set + * the `remoteVersion` field after the handshake is completed. + * Therefore, no `ServeProto::Serialize` functions can be used until + * that field is set. + */ static void handshake(Machine::Connection & conn, unsigned int repeats) { conn.to << SERVE_MAGIC_1 << 0x206; @@ -204,6 +210,7 @@ static void handshake(Machine::Connection & conn, unsigned int repeats) if (magic != SERVE_MAGIC_2) throw Error("protocol mismatch with ‘nix-store --serve’ on ‘%1%’", conn.machine->sshName); conn.remoteVersion = readInt(conn.from); + // Now `conn` is initialized. if (GET_PROTOCOL_MAJOR(conn.remoteVersion) != 0x200) throw Error("unsupported ‘nix-store --serve’ protocol version on ‘%1%’", conn.machine->sshName); if (GET_PROTOCOL_MINOR(conn.remoteVersion) < 3 && repeats > 0) @@ -512,10 +519,11 @@ void State::buildRemote(ref destStore, process. Meh. */ }); - Machine::Connection conn; - conn.from = child.from.get(); - conn.to = child.to.get(); - conn.machine = machine; + Machine::Connection conn { + .from = child.from.get(), + .to = child.to.get(), + .machine = machine, + }; Finally updateStats([&]() { bytesReceived += conn.from.read; diff --git a/src/hydra-queue-runner/state.hh b/src/hydra-queue-runner/state.hh index 03923917..6359063a 100644 --- a/src/hydra-queue-runner/state.hh +++ b/src/hydra-queue-runner/state.hh @@ -303,8 +303,8 @@ struct Machine // A connection to a machine struct Connection { - nix::FdSink to; nix::FdSource from; + nix::FdSink to; nix::ServeProto::Version remoteVersion; // Backpointer to the machine From 162b538912fc940b3c737634c23c99c574adf4ad Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 4 Dec 2023 11:27:39 -0500 Subject: [PATCH 051/313] Remove unused `thisArrow` variable --- src/hydra-queue-runner/build-remote.cc | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc index baf35d86..f1575170 100644 --- a/src/hydra-queue-runner/build-remote.cc +++ b/src/hydra-queue-runner/build-remote.cc @@ -421,8 +421,6 @@ static void copyPathsFromRemote( void RemoteResult::updateWithBuildResult(const nix::BuildResult & buildResult) { - RemoteResult thisArrow; - startTime = buildResult.startTime; stopTime = buildResult.stopTime; timesBuilt = buildResult.timesBuilt; From 363604846adf27db017d2d599b373f1e0839dfd6 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 4 Dec 2023 11:31:05 -0500 Subject: [PATCH 052/313] Again, use `const` in for loop As requested by @teh. Was lost in merge with master, now added back. --- src/hydra-queue-runner/build-remote.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc index f1575170..73f46a53 100644 --- a/src/hydra-queue-runner/build-remote.cc +++ b/src/hydra-queue-runner/build-remote.cc @@ -230,7 +230,7 @@ static BasicDerivation sendInputs( { BasicDerivation basicDrv(*step.drv); - for (auto & [drvPath, node] : step.drv->inputDrvs.map) { + for (const auto & [drvPath, node] : step.drv->inputDrvs.map) { auto drv2 = localStore.readDerivation(drvPath); for (auto & name : node.value) { if (auto i = get(drv2.outputs, name)) { From 9ba4417940ffdd0fadea43f68c61ef948a4b8d39 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 4 Dec 2023 16:05:50 -0500 Subject: [PATCH 053/313] Prepare for CA derivation support with lower impact changes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is just C++ changes without any Perl / Frontend / SQL Schema changes. The idea is that it should be possible to redeploy Hydra with these chnages with (a) no schema migration and also (b) no regressions. We should be able to much more safely deploy these to a staging server and then production `hydra.nixos.org`. Extracted from #875 Co-Authored-By: Théophane Hufschmitt Co-Authored-By: Alexander Sosedkin Co-Authored-By: Andrea Ciceri Co-Authored-By: Charlotte 🦝 Delenk Mlotte@chir.rs> Co-Authored-By: Sandro Jäckel --- src/hydra-eval-jobs/hydra-eval-jobs.cc | 16 ++- src/hydra-queue-runner/build-remote.cc | 105 +++++++++++--- src/hydra-queue-runner/build-result.cc | 19 ++- src/hydra-queue-runner/builder.cc | 8 +- src/hydra-queue-runner/hydra-build-result.hh | 4 +- src/hydra-queue-runner/hydra-queue-runner.cc | 27 +++- src/hydra-queue-runner/queue-monitor.cc | 137 ++++++++++--------- src/hydra-queue-runner/state.hh | 4 +- 8 files changed, 213 insertions(+), 107 deletions(-) diff --git a/src/hydra-eval-jobs/hydra-eval-jobs.cc b/src/hydra-eval-jobs/hydra-eval-jobs.cc index 2fe2c80f..d007189d 100644 --- a/src/hydra-eval-jobs/hydra-eval-jobs.cc +++ b/src/hydra-eval-jobs/hydra-eval-jobs.cc @@ -178,7 +178,11 @@ static void worker( if (auto drv = getDerivation(state, *v, false)) { - DrvInfo::Outputs outputs = drv->queryOutputs(); + // CA derivations do not have static output paths, so we + // have to defensively not query output paths in case we + // encounter one. + DrvInfo::Outputs outputs = drv->queryOutputs( + !experimentalFeatureSettings.isEnabled(Xp::CaDerivations)); if (drv->querySystem() == "unknown") throw EvalError("derivation must have a 'system' attribute"); @@ -239,12 +243,12 @@ static void worker( } nlohmann::json out; - for (auto & j : outputs) - // FIXME: handle CA/impure builds. - if (j.second) - out[j.first] = state.store->printStorePath(*j.second); + for (auto & [outputName, optOutputPath] : outputs) + if (optOutputPath) + out[outputName] = state.store->printStorePath(*optOutputPath); + else + out[outputName] = ""; job["outputs"] = std::move(out); - reply["job"] = std::move(job); } diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc index 73f46a53..26f6d63f 100644 --- a/src/hydra-queue-runner/build-remote.cc +++ b/src/hydra-queue-runner/build-remote.cc @@ -182,6 +182,41 @@ static StorePaths reverseTopoSortPaths(const std::map return sorted; } +/** + * Replace the input derivations by their output paths to send a minimal closure + * to the builder. + * + * If we can afford it, resolve it, so that the newly generated derivation still + * has some sensible output paths. + */ +BasicDerivation inlineInputDerivations(Store & store, Derivation & drv, const StorePath & drvPath) +{ + BasicDerivation ret; + auto outputHashes = staticOutputHashes(store, drv); + if (!drv.type().hasKnownOutputPaths()) { + auto maybeBasicDrv = drv.tryResolve(store); + if (!maybeBasicDrv) + throw Error( + "the derivation '%s' can’t be resolved. It’s probably " + "missing some outputs", + store.printStorePath(drvPath)); + ret = *maybeBasicDrv; + } else { + // If the derivation is a real `InputAddressed` derivation, we must + // resolve it manually to keep the original output paths + ret = BasicDerivation(drv); + for (auto & [drvPath, node] : drv.inputDrvs.map) { + auto drv2 = store.readDerivation(drvPath); + auto drv2Outputs = drv2.outputsAndOptPaths(store); + for (auto & name : node.value) { + auto inputPath = drv2Outputs.at(name); + ret.inputSrcs.insert(*inputPath.second); + } + } + } + return ret; +} + static std::pair openLogFile(const std::string & logDir, const StorePath & drvPath) { std::string base(drvPath.to_string()); @@ -228,17 +263,7 @@ static BasicDerivation sendInputs( counter & nrStepsCopyingTo ) { - BasicDerivation basicDrv(*step.drv); - - for (const auto & [drvPath, node] : step.drv->inputDrvs.map) { - auto drv2 = localStore.readDerivation(drvPath); - for (auto & name : node.value) { - if (auto i = get(drv2.outputs, name)) { - auto outPath = i->path(localStore, drv2.name, name); - basicDrv.inputSrcs.insert(*outPath); - } - } - } + BasicDerivation basicDrv = inlineInputDerivations(localStore, *step.drv, step.drvPath); /* Ensure that the inputs exist in the destination store. This is a no-op for regular stores, but for the binary cache store, @@ -323,8 +348,33 @@ static BuildResult performBuild( result.stopTime = stop; } } + + // Get the newly built outputs, either from the remote if it + // supports it, or by introspecting the derivation if the remote is + // too old. if (GET_PROTOCOL_MINOR(conn.remoteVersion) >= 6) { - ServeProto::Serialise::read(localStore, conn); + auto builtOutputs = ServeProto::Serialise::read(localStore, conn); + for (auto && [output, realisation] : builtOutputs) + result.builtOutputs.insert_or_assign( + std::move(output.outputName), + std::move(realisation)); + } else { + // If the remote is too old to handle CA derivations, we can’t get this + // far anyways + assert(drv.type().hasKnownOutputPaths()); + DerivationOutputsAndOptPaths drvOutputs = drv.outputsAndOptPaths(localStore); + auto outputHashes = staticOutputHashes(localStore, drv); + for (auto & [outputName, output] : drvOutputs) { + auto outputPath = output.second; + // We’ve just asserted that the output paths of the derivation + // were known + assert(outputPath); + auto outputHash = outputHashes.at(outputName); + auto drvOutput = DrvOutput { outputHash, outputName }; + result.builtOutputs.insert_or_assign( + std::move(outputName), + Realisation { drvOutput, *outputPath }); + } } return result; @@ -376,6 +426,7 @@ static void copyPathFromRemote( const ValidPathInfo & info ) { + for (auto * store : {&destStore, &localStore}) { /* Receive the NAR from the remote and add it to the destination store. Meanwhile, extract all the info from the NAR that getBuildOutput() needs. */ @@ -395,7 +446,8 @@ static void copyPathFromRemote( extractNarData(tee, localStore.printStorePath(info.path), narMembers); }); - destStore.addToStore(info, *source2, NoRepair, NoCheckSigs); + store->addToStore(info, *source2, NoRepair, NoCheckSigs); + } } static void copyPathsFromRemote( @@ -592,6 +644,10 @@ void State::buildRemote(ref destStore, result.logFile = ""; } + StorePathSet outputs; + for (auto & [_, realisation] : buildResult.builtOutputs) + outputs.insert(realisation.outPath); + /* Copy the output paths. */ if (!machine->isLocalhost() || localStore != std::shared_ptr(destStore)) { updateStep(ssReceivingOutputs); @@ -600,12 +656,6 @@ void State::buildRemote(ref destStore, auto now1 = std::chrono::steady_clock::now(); - StorePathSet outputs; - for (auto & i : step->drv->outputsAndOptPaths(*localStore)) { - if (i.second.second) - outputs.insert(*i.second.second); - } - size_t totalNarSize = 0; auto infos = build_remote::queryPathInfos(conn, *localStore, outputs, totalNarSize); @@ -624,6 +674,23 @@ void State::buildRemote(ref destStore, result.overhead += std::chrono::duration_cast(now2 - now1).count(); } + /* Register the outputs of the newly built drv */ + if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { + auto outputHashes = staticOutputHashes(*localStore, *step->drv); + for (auto & [outputName, realisation] : buildResult.builtOutputs) { + // Register the resolved drv output + localStore->registerDrvOutput(realisation); + destStore->registerDrvOutput(realisation); + + // Also register the unresolved one + auto unresolvedRealisation = realisation; + unresolvedRealisation.signatures.clear(); + unresolvedRealisation.id.drvHash = outputHashes.at(outputName); + localStore->registerDrvOutput(unresolvedRealisation); + destStore->registerDrvOutput(unresolvedRealisation); + } + } + /* Shut down the connection. */ child.to = -1; child.pid.wait(); diff --git a/src/hydra-queue-runner/build-result.cc b/src/hydra-queue-runner/build-result.cc index 691c1f19..ffdc37b7 100644 --- a/src/hydra-queue-runner/build-result.cc +++ b/src/hydra-queue-runner/build-result.cc @@ -11,18 +11,18 @@ using namespace nix; BuildOutput getBuildOutput( nix::ref store, NarMemberDatas & narMembers, - const Derivation & drv) + const OutputPathMap derivationOutputs) { BuildOutput res; /* Compute the closure size. */ StorePathSet outputs; StorePathSet closure; - for (auto & i : drv.outputsAndOptPaths(*store)) - if (i.second.second) { - store->computeFSClosure(*i.second.second, closure); - outputs.insert(*i.second.second); - } + for (auto& [outputName, outputPath] : derivationOutputs) { + store->computeFSClosure(outputPath, closure); + outputs.insert(outputPath); + res.outputs.insert({outputName, outputPath}); + } for (auto & path : closure) { auto info = store->queryPathInfo(path); res.closureSize += info->narSize; @@ -107,13 +107,12 @@ BuildOutput getBuildOutput( /* If no build products were explicitly declared, then add all outputs as a product of type "nix-build". */ if (!explicitProducts) { - for (auto & [name, output] : drv.outputs) { + for (auto & [name, output] : derivationOutputs) { BuildProduct product; - auto outPath = output.path(*store, drv.name, name); - product.path = store->printStorePath(*outPath); + product.path = store->printStorePath(output); product.type = "nix-build"; product.subtype = name == "out" ? "" : name; - product.name = outPath->name(); + product.name = output.name(); auto file = narMembers.find(product.path); assert(file != narMembers.end()); diff --git a/src/hydra-queue-runner/builder.cc b/src/hydra-queue-runner/builder.cc index 307eee8e..42983941 100644 --- a/src/hydra-queue-runner/builder.cc +++ b/src/hydra-queue-runner/builder.cc @@ -223,7 +223,7 @@ State::StepResult State::doBuildStep(nix::ref destStore, if (result.stepStatus == bsSuccess) { updateStep(ssPostProcessing); - res = getBuildOutput(destStore, narMembers, *step->drv); + res = getBuildOutput(destStore, narMembers, localStore->queryDerivationOutputMap(step->drvPath)); } } @@ -277,9 +277,9 @@ State::StepResult State::doBuildStep(nix::ref destStore, assert(stepNr); - for (auto & i : step->drv->outputsAndOptPaths(*localStore)) { - if (i.second.second) - addRoot(*i.second.second); + for (auto & i : localStore->queryPartialDerivationOutputMap(step->drvPath)) { + if (i.second) + addRoot(*i.second); } /* Register success in the database for all Build objects that diff --git a/src/hydra-queue-runner/hydra-build-result.hh b/src/hydra-queue-runner/hydra-build-result.hh index a3f71ae9..7d47f67c 100644 --- a/src/hydra-queue-runner/hydra-build-result.hh +++ b/src/hydra-queue-runner/hydra-build-result.hh @@ -36,10 +36,12 @@ struct BuildOutput std::list products; + std::map outputs; + std::map metrics; }; BuildOutput getBuildOutput( nix::ref store, NarMemberDatas & narMembers, - const nix::Derivation & drv); + const nix::OutputPathMap derivationOutputs); diff --git a/src/hydra-queue-runner/hydra-queue-runner.cc b/src/hydra-queue-runner/hydra-queue-runner.cc index 1d54bb93..2f2c6091 100644 --- a/src/hydra-queue-runner/hydra-queue-runner.cc +++ b/src/hydra-queue-runner/hydra-queue-runner.cc @@ -312,10 +312,10 @@ unsigned int State::createBuildStep(pqxx::work & txn, time_t startTime, BuildID if (r.affected_rows() == 0) goto restart; - for (auto & [name, output] : step->drv->outputs) + for (auto & [name, output] : localStore->queryPartialDerivationOutputMap(step->drvPath)) txn.exec_params0 ("insert into BuildStepOutputs (build, stepnr, name, path) values ($1, $2, $3, $4)", - buildId, stepNr, name, localStore->printStorePath(*output.path(*localStore, step->drv->name, name))); + buildId, stepNr, name, output ? localStore->printStorePath(*output) : ""); if (status == bsBusy) txn.exec(fmt("notify step_started, '%d\t%d'", buildId, stepNr)); @@ -352,11 +352,23 @@ void State::finishBuildStep(pqxx::work & txn, const RemoteResult & result, assert(result.logFile.find('\t') == std::string::npos); txn.exec(fmt("notify step_finished, '%d\t%d\t%s'", buildId, stepNr, result.logFile)); + + if (result.stepStatus == bsSuccess) { + // Update the corresponding `BuildStepOutputs` row to add the output path + auto res = txn.exec_params1("select drvPath from BuildSteps where build = $1 and stepnr = $2", buildId, stepNr); + assert(res.size()); + StorePath drvPath = localStore->parseStorePath(res[0].as()); + // If we've finished building, all the paths should be known + for (auto & [name, output] : localStore->queryDerivationOutputMap(drvPath)) + txn.exec_params0 + ("update BuildStepOutputs set path = $4 where build = $1 and stepnr = $2 and name = $3", + buildId, stepNr, name, localStore->printStorePath(output)); + } } int State::createSubstitutionStep(pqxx::work & txn, time_t startTime, time_t stopTime, - Build::ptr build, const StorePath & drvPath, const std::string & outputName, const StorePath & storePath) + Build::ptr build, const StorePath & drvPath, const nix::Derivation drv, const std::string & outputName, const StorePath & storePath) { restart: auto stepNr = allocBuildStep(txn, build->id); @@ -457,6 +469,15 @@ void State::markSucceededBuild(pqxx::work & txn, Build::ptr build, res.releaseName != "" ? std::make_optional(res.releaseName) : std::nullopt, isCachedBuild ? 1 : 0); + for (auto & [outputName, outputPath] : res.outputs) { + txn.exec_params0 + ("update BuildOutputs set path = $3 where build = $1 and name = $2", + build->id, + outputName, + localStore->printStorePath(outputPath) + ); + } + txn.exec_params0("delete from BuildProducts where build = $1", build->id); unsigned int productNr = 1; diff --git a/src/hydra-queue-runner/queue-monitor.cc b/src/hydra-queue-runner/queue-monitor.cc index 6c339af6..cee5b6cc 100644 --- a/src/hydra-queue-runner/queue-monitor.cc +++ b/src/hydra-queue-runner/queue-monitor.cc @@ -192,15 +192,14 @@ bool State::getQueuedBuilds(Connection & conn, if (!res[0].is_null()) propagatedFrom = res[0].as(); if (!propagatedFrom) { - for (auto & i : ex.step->drv->outputsAndOptPaths(*localStore)) { - if (i.second.second) { - auto res = txn.exec_params - ("select max(s.build) from BuildSteps s join BuildStepOutputs o on s.build = o.build where path = $1 and startTime != 0 and stopTime != 0 and status = 1", - localStore->printStorePath(*i.second.second)); - if (!res[0][0].is_null()) { - propagatedFrom = res[0][0].as(); - break; - } + for (auto & i : localStore->queryPartialDerivationOutputMap(ex.step->drvPath)) { + auto res = txn.exec_params + ("select max(s.build) from BuildSteps s join BuildStepOutputs o on s.build = o.build where drvPath = $1 and name = $2 and startTime != 0 and stopTime != 0 and status = 1", + localStore->printStorePath(ex.step->drvPath), + i.first); + if (!res[0][0].is_null()) { + propagatedFrom = res[0][0].as(); + break; } } } @@ -236,12 +235,10 @@ bool State::getQueuedBuilds(Connection & conn, /* If we didn't get a step, it means the step's outputs are all valid. So we mark this as a finished, cached build. */ if (!step) { - auto drv = localStore->readDerivation(build->drvPath); - BuildOutput res = getBuildOutputCached(conn, destStore, drv); + BuildOutput res = getBuildOutputCached(conn, destStore, build->drvPath); - for (auto & i : drv.outputsAndOptPaths(*localStore)) - if (i.second.second) - addRoot(*i.second.second); + for (auto & i : localStore->queryDerivationOutputMap(build->drvPath)) + addRoot(i.second); { auto mc = startDbUpdate(); @@ -481,26 +478,39 @@ Step::ptr State::createStep(ref destStore, throw PreviousFailure{step}; /* Are all outputs valid? */ + auto outputHashes = staticOutputHashes(*localStore, *(step->drv)); bool valid = true; - DerivationOutputs missing; - for (auto & i : step->drv->outputs) - if (!destStore->isValidPath(*i.second.path(*localStore, step->drv->name, i.first))) { - valid = false; - missing.insert_or_assign(i.first, i.second); + std::map> missing; + for (auto &[outputName, maybeOutputPath] : step->drv->outputsAndOptPaths(*destStore)) { + auto outputHash = outputHashes.at(outputName); + if (maybeOutputPath.second) { + if (!destStore->isValidPath(*maybeOutputPath.second)) { + valid = false; + missing.insert({{outputHash, outputName}, maybeOutputPath.second}); + } + } else { + experimentalFeatureSettings.require(Xp::CaDerivations); + if (!destStore->queryRealisation(DrvOutput{outputHash, outputName})) { + valid = false; + missing.insert({{outputHash, outputName}, std::nullopt}); + } } + } /* Try to copy the missing paths from the local store or from substitutes. */ if (!missing.empty()) { size_t avail = 0; - for (auto & i : missing) { - auto path = i.second.path(*localStore, step->drv->name, i.first); - if (/* localStore != destStore && */ localStore->isValidPath(*path)) + for (auto & [i, maybePath] : missing) { + if ((maybePath && localStore->isValidPath(*maybePath))) avail++; - else if (useSubstitutes) { + else if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations) && localStore->queryRealisation(i)) { + maybePath = localStore->queryRealisation(i)->outPath; + avail++; + } else if (useSubstitutes && maybePath) { SubstitutablePathInfos infos; - localStore->querySubstitutablePathInfos({{*path, {}}}, infos); + localStore->querySubstitutablePathInfos({{*maybePath, {}}}, infos); if (infos.size() == 1) avail++; } @@ -508,44 +518,44 @@ Step::ptr State::createStep(ref destStore, if (missing.size() == avail) { valid = true; - for (auto & i : missing) { - auto path = i.second.path(*localStore, step->drv->name, i.first); + for (auto & [i, path] : missing) { + if (path) { + try { + time_t startTime = time(0); - try { - time_t startTime = time(0); + if (localStore->isValidPath(*path)) + printInfo("copying output ‘%1%’ of ‘%2%’ from local store", + localStore->printStorePath(*path), + localStore->printStorePath(drvPath)); + else { + printInfo("substituting output ‘%1%’ of ‘%2%’", + localStore->printStorePath(*path), + localStore->printStorePath(drvPath)); + localStore->ensurePath(*path); + // FIXME: should copy directly from substituter to destStore. + } - if (localStore->isValidPath(*path)) - printInfo("copying output ‘%1%’ of ‘%2%’ from local store", + StorePathSet closure; + localStore->computeFSClosure({*path}, closure); + copyPaths(*localStore, *destStore, closure, NoRepair, CheckSigs, NoSubstitute); + + time_t stopTime = time(0); + + { + auto mc = startDbUpdate(); + pqxx::work txn(conn); + createSubstitutionStep(txn, startTime, stopTime, build, drvPath, *(step->drv), "out", *path); + txn.commit(); + } + + } catch (Error & e) { + printError("while copying/substituting output ‘%s’ of ‘%s’: %s", localStore->printStorePath(*path), - localStore->printStorePath(drvPath)); - else { - printInfo("substituting output ‘%1%’ of ‘%2%’", - localStore->printStorePath(*path), - localStore->printStorePath(drvPath)); - localStore->ensurePath(*path); - // FIXME: should copy directly from substituter to destStore. + localStore->printStorePath(drvPath), + e.what()); + valid = false; + break; } - - copyClosure(*localStore, *destStore, - StorePathSet { *path }, - NoRepair, CheckSigs, NoSubstitute); - - time_t stopTime = time(0); - - { - auto mc = startDbUpdate(); - pqxx::work txn(conn); - createSubstitutionStep(txn, startTime, stopTime, build, drvPath, "out", *path); - txn.commit(); - } - - } catch (Error & e) { - printError("while copying/substituting output ‘%s’ of ‘%s’: %s", - localStore->printStorePath(*path), - localStore->printStorePath(drvPath), - e.what()); - valid = false; - break; } } } @@ -640,17 +650,20 @@ void State::processJobsetSharesChange(Connection & conn) } -BuildOutput State::getBuildOutputCached(Connection & conn, nix::ref destStore, const nix::Derivation & drv) +BuildOutput State::getBuildOutputCached(Connection & conn, nix::ref destStore, const nix::StorePath & drvPath) { + + auto derivationOutputs = localStore->queryDerivationOutputMap(drvPath); + { pqxx::work txn(conn); - for (auto & [name, output] : drv.outputsAndOptPaths(*localStore)) { + for (auto & [name, output] : derivationOutputs) { auto r = txn.exec_params ("select id, buildStatus, releaseName, closureSize, size from Builds b " "join BuildOutputs o on b.id = o.build " "where finished = 1 and (buildStatus = 0 or buildStatus = 6) and path = $1", - localStore->printStorePath(*output.second)); + localStore->printStorePath(output)); if (r.empty()) continue; BuildID id = r[0][0].as(); @@ -704,5 +717,5 @@ BuildOutput State::getBuildOutputCached(Connection & conn, nix::ref } NarMemberDatas narMembers; - return getBuildOutput(destStore, narMembers, drv); + return getBuildOutput(destStore, narMembers, derivationOutputs); } diff --git a/src/hydra-queue-runner/state.hh b/src/hydra-queue-runner/state.hh index 6359063a..5f01de1e 100644 --- a/src/hydra-queue-runner/state.hh +++ b/src/hydra-queue-runner/state.hh @@ -520,7 +520,7 @@ private: const std::string & machine); int createSubstitutionStep(pqxx::work & txn, time_t startTime, time_t stopTime, - Build::ptr build, const nix::StorePath & drvPath, const std::string & outputName, const nix::StorePath & storePath); + Build::ptr build, const nix::StorePath & drvPath, const nix::Derivation drv, const std::string & outputName, const nix::StorePath & storePath); void updateBuild(pqxx::work & txn, Build::ptr build, BuildStatus status); @@ -536,7 +536,7 @@ private: void processQueueChange(Connection & conn); BuildOutput getBuildOutputCached(Connection & conn, nix::ref destStore, - const nix::Derivation & drv); + const nix::StorePath & drvPath); Step::ptr createStep(nix::ref store, Connection & conn, Build::ptr build, const nix::StorePath & drvPath, From 8046ec266851d8b8d3f08900cf9ba905ed344ed8 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 4 Dec 2023 16:21:56 -0500 Subject: [PATCH 054/313] Remove unused `outputHashes` variable This looks like a stray copy paste. --- src/hydra-queue-runner/build-remote.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc index 26f6d63f..e26a8697 100644 --- a/src/hydra-queue-runner/build-remote.cc +++ b/src/hydra-queue-runner/build-remote.cc @@ -192,7 +192,6 @@ static StorePaths reverseTopoSortPaths(const std::map BasicDerivation inlineInputDerivations(Store & store, Derivation & drv, const StorePath & drvPath) { BasicDerivation ret; - auto outputHashes = staticOutputHashes(store, drv); if (!drv.type().hasKnownOutputPaths()) { auto maybeBasicDrv = drv.tryResolve(store); if (!maybeBasicDrv) From e3443cd22a3bc20c7d17f4e76d3868ea755b4b37 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 4 Dec 2023 17:41:11 -0500 Subject: [PATCH 055/313] Put back nicer `copyClosure` instead of manual closure + copy It looks like we accidentally got the old code back, probably after a merge conflict resolution. --- src/hydra-queue-runner/queue-monitor.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/hydra-queue-runner/queue-monitor.cc b/src/hydra-queue-runner/queue-monitor.cc index cee5b6cc..04917932 100644 --- a/src/hydra-queue-runner/queue-monitor.cc +++ b/src/hydra-queue-runner/queue-monitor.cc @@ -535,9 +535,9 @@ Step::ptr State::createStep(ref destStore, // FIXME: should copy directly from substituter to destStore. } - StorePathSet closure; - localStore->computeFSClosure({*path}, closure); - copyPaths(*localStore, *destStore, closure, NoRepair, CheckSigs, NoSubstitute); + copyClosure(*localStore, *destStore, + StorePathSet { *path }, + NoRepair, CheckSigs, NoSubstitute); time_t stopTime = time(0); From 069b7775c565f5999fe33e8c3f28c7b9306039ca Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 5 Dec 2023 11:26:26 -0500 Subject: [PATCH 056/313] hydra-eval-jobs: Ensure we have output path if ca-derivations is disabled Brought up by @thufschmitt in https://github.com/NixOS/hydra/pull/1316#discussion_r1415111329 . This makes this closer to what was originally there --- which just dispatched off the experimental feature rather than the presence/absense of the output, too. --- src/hydra-eval-jobs/hydra-eval-jobs.cc | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/hydra-eval-jobs/hydra-eval-jobs.cc b/src/hydra-eval-jobs/hydra-eval-jobs.cc index d007189d..b44ccbb3 100644 --- a/src/hydra-eval-jobs/hydra-eval-jobs.cc +++ b/src/hydra-eval-jobs/hydra-eval-jobs.cc @@ -243,11 +243,16 @@ static void worker( } nlohmann::json out; - for (auto & [outputName, optOutputPath] : outputs) - if (optOutputPath) + for (auto & [outputName, optOutputPath] : outputs) { + if (optOutputPath) { out[outputName] = state.store->printStorePath(*optOutputPath); - else + } else { + // See the `queryOutputs` call above; we should + // not encounter missing output paths otherwise. + assert(experimentalFeatureSettings.isEnabled(Xp::CaDerivations)); out[outputName] = ""; + } + } job["outputs"] = std::move(out); reply["job"] = std::move(job); } From 3df8feb3a2b30b733b141f6e6e43572998311524 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 5 Dec 2023 11:27:52 -0500 Subject: [PATCH 057/313] Add TODO about setting `null` instead of empty string in JSON An empty string is a sneaky way to avoid hard failures --- things that expect strings still get strings, but it does conversely open the door up to soft failures (spooky-action-at-a-distance ones because the string did not have the expected invariants). "Fail fast" with null will ultimately make the system more robust, but force us to fix more things up front, and I don't want to change this without also fixing those things up front, especially as this commit is for now just part of the the preparatory PR for which this is dead code. --- src/hydra-eval-jobs/hydra-eval-jobs.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/hydra-eval-jobs/hydra-eval-jobs.cc b/src/hydra-eval-jobs/hydra-eval-jobs.cc index b44ccbb3..2794cc62 100644 --- a/src/hydra-eval-jobs/hydra-eval-jobs.cc +++ b/src/hydra-eval-jobs/hydra-eval-jobs.cc @@ -250,6 +250,10 @@ static void worker( // See the `queryOutputs` call above; we should // not encounter missing output paths otherwise. assert(experimentalFeatureSettings.isEnabled(Xp::CaDerivations)); + // TODO it would be better to set `null` than an + // empty string here, to force the consumer of + // this JSON to more explicitly handle this + // case. out[outputName] = ""; } } From 11f8030b0f4c75ed7640563d68200cf3ec59edec Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 6 Dec 2023 17:59:25 -0500 Subject: [PATCH 058/313] Add comment from GitHub about adding to store as code comment --- src/hydra-queue-runner/build-remote.cc | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc index e26a8697..94a01676 100644 --- a/src/hydra-queue-runner/build-remote.cc +++ b/src/hydra-queue-runner/build-remote.cc @@ -425,6 +425,19 @@ static void copyPathFromRemote( const ValidPathInfo & info ) { + // Why both stores? @thufschmitt says: + // + // > I think it's an easy (and terribly inefficient 😬) way of + // making sure that `localStore.queryRealisations` will succeed + // (which we IIRC we need later to get back some metadata about the + // path to put it in the db). + // > + // > To be honest, we shouldn't do that but instead carry the needed + // metadata in memory until the point where we need it (but that can + // come later once we're confident that this is at least correct) + // + // TODO make the above change to avoid copying excess data back and + // forth. for (auto * store : {&destStore, &localStore}) { /* Receive the NAR from the remote and add it to the destination store. Meanwhile, extract all the info from the From 86cd5e907683a8c4b0f6cd75d4879648d8313b98 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 20 Feb 2022 20:06:22 +0000 Subject: [PATCH 059/313] `copyClosureTo`: Use `SubstituteFlag` instead of `bool` This matches Nix (in the same serialization logic in `src/libstore/legacy-ssh-store.cc`) and adds clarity. --- src/hydra-queue-runner/build-remote.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc index 73f46a53..f41585c8 100644 --- a/src/hydra-queue-runner/build-remote.cc +++ b/src/hydra-queue-runner/build-remote.cc @@ -113,7 +113,7 @@ static void copyClosureTo( Machine::Connection & conn, Store & destStore, const StorePathSet & paths, - bool useSubstitutes = false) + SubstituteFlag useSubstitutes = NoSubstitute) { StorePathSet closure; destStore.computeFSClosure(paths, closure); @@ -266,7 +266,7 @@ static BasicDerivation sendInputs( destStore.computeFSClosure(basicDrv.inputSrcs, closure); copyPaths(destStore, localStore, closure, NoRepair, NoCheckSigs, NoSubstitute); } else { - copyClosureTo(conn, destStore, basicDrv.inputSrcs, true); + copyClosureTo(conn, destStore, basicDrv.inputSrcs, Substitute); } auto now2 = std::chrono::steady_clock::now(); From 6a54ab24e26b4e387c41e6d8958e7a9ec36c8398 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 7 Dec 2023 02:00:22 -0500 Subject: [PATCH 060/313] Use factored-out `BuildResult` serializer For the record, here is the Nix 2.19 version: https://github.com/NixOS/nix/blob/2.19-maintenance/src/libstore/serve-protocol.cc, which is what we would initially use. It is a more complete version of what Hydra has today except for one thing: it always unconditionally sets the start/stop times. I think that is correct at the other end seems to unconditionally measure them, but just to be extra careful, I reproduced the old behavior of falling back on Hydra's own measurements if `startTime` is 0. The only difference is that the fallback `stopTime` is now measured from after the entire `BuildResult` is transferred over the wire, but I think that should be negligible if it is measurable at all. (And remember, this is fallback case I already suspect is dead code.) --- src/hydra-queue-runner/build-remote.cc | 36 ++++++++++---------------- 1 file changed, 14 insertions(+), 22 deletions(-) diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc index 73f46a53..5206ae5a 100644 --- a/src/hydra-queue-runner/build-remote.cc +++ b/src/hydra-queue-runner/build-remote.cc @@ -286,9 +286,6 @@ static BuildResult performBuild( counter & nrStepsBuilding ) { - - BuildResult result; - conn.to << ServeProto::Command::BuildDerivation << localStore.printStorePath(drvPath); writeDerivation(conn.to, localStore, drv); conn.to << options.maxSilentTime << options.buildTimeout; @@ -301,30 +298,25 @@ static BuildResult performBuild( } conn.to.flush(); - result.startTime = time(0); + BuildResult result; + time_t startTime, stopTime; + + startTime = time(0); { MaintainCount mc(nrStepsBuilding); - result.status = (BuildResult::Status)readInt(conn.from); + result = ServeProto::Serialise::read(localStore, conn); } - result.stopTime = time(0); + stopTime = time(0); - - result.errorMsg = readString(conn.from); - if (GET_PROTOCOL_MINOR(conn.remoteVersion) >= 3) { - result.timesBuilt = readInt(conn.from); - result.isNonDeterministic = readInt(conn.from); - auto start = readInt(conn.from); - auto stop = readInt(conn.from); - if (start && start) { - /* Note: this represents the duration of a single - round, rather than all rounds. */ - result.startTime = start; - result.stopTime = stop; - } - } - if (GET_PROTOCOL_MINOR(conn.remoteVersion) >= 6) { - ServeProto::Serialise::read(localStore, conn); + if (!result.startTime) { + // If the builder gave `startTime = 0`, use our measurements + // instead of the builder's. + // + // Note: this represents the duration of a single round, rather + // than all rounds. + result.startTime = startTime; + result.stopTime = stopTime; } return result; From a45a27851b6cfc22113a660df75e556199c20c84 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 7 Dec 2023 12:20:55 -0500 Subject: [PATCH 061/313] flake.lock: Update Nixpkgs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This will be required for upgrading Nix beyond 2.19. Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/ef0bc3976340dab9a4e087a0bcff661a8b2e87f3' (2023-06-21) → 'github:NixOS/nixpkgs/e9f06adb793d1cca5384907b3b8a4071d5d7cb19' (2023-12-03) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 9a9046c8..2871e70a 100644 --- a/flake.lock +++ b/flake.lock @@ -58,11 +58,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1687379288, - "narHash": "sha256-cSuwfiqYfeVyqzCRkU9AvLTysmEuSal8nh6CYr+xWog=", + "lastModified": 1701615100, + "narHash": "sha256-7VI84NGBvlCTduw2aHLVB62NvCiZUlALLqBe5v684Aw=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "ef0bc3976340dab9a4e087a0bcff661a8b2e87f3", + "rev": "e9f06adb793d1cca5384907b3b8a4071d5d7cb19", "type": "github" }, "original": { From 20c8263e3c993d4ba0a2a0bedd531a7f32861441 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 7 Dec 2023 11:40:30 -0500 Subject: [PATCH 062/313] Update to Nix master MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The point of this branch is to always track Nix master, so we are proactively ready to upgrade to the next Nix release when it is ready. Flake lock file updates: • Updated input 'nix': 'github:NixOS/nix/50f8f1c8bc019a4c0fd098b9ac674b94cfc6af0d' (2023-11-27) → 'github:NixOS/nix/c3827ff6348a4d5199eaddf8dbc2ca2e2ef46ec5' (2023-12-07) • Added input 'nix/libgit2': 'github:libgit2/libgit2/45fd9ed7ae1a9b74b957ef4f337bc3c8b3df01b5' (2023-10-18) --- flake.lock | 24 ++++++++++++++++++++---- flake.nix | 2 +- src/hydra-queue-runner/build-remote.cc | 2 +- src/hydra-queue-runner/nar-extractor.cc | 2 +- src/hydra-queue-runner/queue-monitor.cc | 2 +- 5 files changed, 24 insertions(+), 8 deletions(-) diff --git a/flake.lock b/flake.lock index 2871e70a..ae4f60eb 100644 --- a/flake.lock +++ b/flake.lock @@ -16,6 +16,22 @@ "type": "github" } }, + "libgit2": { + "flake": false, + "locked": { + "lastModified": 1697646580, + "narHash": "sha256-oX4Z3S9WtJlwvj0uH9HlYcWv+x1hqp8mhXl7HsLu2f0=", + "owner": "libgit2", + "repo": "libgit2", + "rev": "45fd9ed7ae1a9b74b957ef4f337bc3c8b3df01b5", + "type": "github" + }, + "original": { + "owner": "libgit2", + "repo": "libgit2", + "type": "github" + } + }, "lowdown-src": { "flake": false, "locked": { @@ -35,6 +51,7 @@ "nix": { "inputs": { "flake-compat": "flake-compat", + "libgit2": "libgit2", "lowdown-src": "lowdown-src", "nixpkgs": [ "nixpkgs" @@ -42,16 +59,15 @@ "nixpkgs-regression": "nixpkgs-regression" }, "locked": { - "lastModified": 1701122567, - "narHash": "sha256-iA8DqS+W2fWTfR+nNJSvMHqQ+4NpYMRT3b+2zS6JTvE=", + "lastModified": 1701948782, + "narHash": "sha256-rEu4hLHZIy3fgf88BpiaVfl79hNSESRfQNbmxAO5uzg=", "owner": "NixOS", "repo": "nix", - "rev": "50f8f1c8bc019a4c0fd098b9ac674b94cfc6af0d", + "rev": "c3827ff6348a4d5199eaddf8dbc2ca2e2ef46ec5", "type": "github" }, "original": { "owner": "NixOS", - "ref": "2.19.2", "repo": "nix", "type": "github" } diff --git a/flake.nix b/flake.nix index 5a5aef55..fdeaa6ca 100644 --- a/flake.nix +++ b/flake.nix @@ -2,7 +2,7 @@ description = "A Nix-based continuous build system"; inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.05"; - inputs.nix.url = "github:NixOS/nix/2.19.2"; + inputs.nix.url = "github:NixOS/nix"; inputs.nix.inputs.nixpkgs.follows = "nixpkgs"; outputs = { self, nixpkgs, nix }: diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc index f41585c8..dad414bf 100644 --- a/src/hydra-queue-runner/build-remote.cc +++ b/src/hydra-queue-runner/build-remote.cc @@ -350,7 +350,7 @@ static std::map queryPathInfos( auto references = ServeProto::Serialise::read(localStore, conn); readLongLong(conn.from); // download size auto narSize = readLongLong(conn.from); - auto narHash = Hash::parseAny(readString(conn.from), htSHA256); + auto narHash = Hash::parseAny(readString(conn.from), HashAlgorithm::SHA256); auto ca = ContentAddress::parseOpt(readString(conn.from)); readStrings(conn.from); // sigs ValidPathInfo info(localStore.parseStorePath(storePathS), narHash); diff --git a/src/hydra-queue-runner/nar-extractor.cc b/src/hydra-queue-runner/nar-extractor.cc index 3c6857bf..ff4aa268 100644 --- a/src/hydra-queue-runner/nar-extractor.cc +++ b/src/hydra-queue-runner/nar-extractor.cc @@ -42,7 +42,7 @@ struct Extractor : ParseSink void preallocateContents(uint64_t size) override { expectedSize = size; - hashSink = std::make_unique(htSHA256); + hashSink = std::make_unique(HashAlgorithm::SHA256); } void receiveContents(std::string_view data) override diff --git a/src/hydra-queue-runner/queue-monitor.cc b/src/hydra-queue-runner/queue-monitor.cc index 6c339af6..3ce9bba2 100644 --- a/src/hydra-queue-runner/queue-monitor.cc +++ b/src/hydra-queue-runner/queue-monitor.cc @@ -677,7 +677,7 @@ BuildOutput State::getBuildOutputCached(Connection & conn, nix::ref product.fileSize = row[2].as(); } if (!row[3].is_null()) - product.sha256hash = Hash::parseAny(row[3].as(), htSHA256); + product.sha256hash = Hash::parseAny(row[3].as(), HashAlgorithm::SHA256); if (!row[4].is_null()) product.path = row[4].as(); product.name = row[5].as(); From 2ee0068fdca7d9fb8bef23b3671a86b8f0fcff07 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 7 Dec 2023 15:05:03 -0500 Subject: [PATCH 063/313] Do not copy for both stores for now It has a performance cost, and as the comment says we should be doing the better solution. We want to land this preparatory change on prod while the rest is still on staging, so we should just skip it for now. Skipping it will not affect regular fixed-output and input-addressed derivations, which are the only ones prod would deal with upon getting this code. The main CA derivations support branch will revert this commit so it still works. --- src/hydra-queue-runner/build-remote.cc | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc index 1d0bcc35..77484244 100644 --- a/src/hydra-queue-runner/build-remote.cc +++ b/src/hydra-queue-runner/build-remote.cc @@ -425,20 +425,6 @@ static void copyPathFromRemote( const ValidPathInfo & info ) { - // Why both stores? @thufschmitt says: - // - // > I think it's an easy (and terribly inefficient 😬) way of - // making sure that `localStore.queryRealisations` will succeed - // (which we IIRC we need later to get back some metadata about the - // path to put it in the db). - // > - // > To be honest, we shouldn't do that but instead carry the needed - // metadata in memory until the point where we need it (but that can - // come later once we're confident that this is at least correct) - // - // TODO make the above change to avoid copying excess data back and - // forth. - for (auto * store : {&destStore, &localStore}) { /* Receive the NAR from the remote and add it to the destination store. Meanwhile, extract all the info from the NAR that getBuildOutput() needs. */ @@ -458,8 +444,7 @@ static void copyPathFromRemote( extractNarData(tee, localStore.printStorePath(info.path), narMembers); }); - store->addToStore(info, *source2, NoRepair, NoCheckSigs); - } + destStore.addToStore(info, *source2, NoRepair, NoCheckSigs); } static void copyPathsFromRemote( From 411e4d0c2458ec5319b8ea88129dff807793f7d7 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 8 Dec 2023 11:30:31 -0500 Subject: [PATCH 064/313] Let tests themselves intentionally leak temp dir (#1320) * Let tests themselves intentionally leak temp dir By default Yath will clean up temporary files, so the result is the same. But `--keep-dirs` can be passed to `yath test` telling Yath to *not* clean them up instead. This is very useful for debugging. * Update t/lib/HydraTestContext.pm Co-authored-by: Cole Helbling --- t/lib/HydraTestContext.pm | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/t/lib/HydraTestContext.pm b/t/lib/HydraTestContext.pm index 53eaa0f7..a22c3df1 100644 --- a/t/lib/HydraTestContext.pm +++ b/t/lib/HydraTestContext.pm @@ -39,7 +39,9 @@ use Hydra::Helper::Exec; sub new { my ($class, %opts) = @_; - my $dir = File::Temp->newdir(); + # Cleanup will be managed by yath. By the default it will be cleaned + # up, but can be kept to aid in debugging test failures. + my $dir = File::Temp->newdir(CLEANUP => 0); $ENV{'HYDRA_DATA'} = "$dir/hydra-data"; mkdir $ENV{'HYDRA_DATA'}; From aaa0e128c1e85885ff55f6abd3a3115ab4b194be Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 9 Dec 2023 11:58:09 -0500 Subject: [PATCH 065/313] flake.lock: Update Nix master MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nix': 'github:NixOS/nix/c3827ff6348a4d5199eaddf8dbc2ca2e2ef46ec5' (2023-12-07) → 'github:NixOS/nix/c8458bd731eb1c74159bebe459ea00165e056b65' (2023-12-09) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index ae4f60eb..424ed3b0 100644 --- a/flake.lock +++ b/flake.lock @@ -59,11 +59,11 @@ "nixpkgs-regression": "nixpkgs-regression" }, "locked": { - "lastModified": 1701948782, - "narHash": "sha256-rEu4hLHZIy3fgf88BpiaVfl79hNSESRfQNbmxAO5uzg=", + "lastModified": 1702090558, + "narHash": "sha256-JzuGOltp5Ht9flroZ7g0erD5ud2okNQChw9YlxWDX3c=", "owner": "NixOS", "repo": "nix", - "rev": "c3827ff6348a4d5199eaddf8dbc2ca2e2ef46ec5", + "rev": "c8458bd731eb1c74159bebe459ea00165e056b65", "type": "github" }, "original": { From 3f932a6731cfd40919258d0731aad235b77e470c Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 7 Dec 2023 14:10:28 -0500 Subject: [PATCH 066/313] build-remote: Use `std::map` It is less denormalized --- src/hydra-queue-runner/build-remote.cc | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc index 05bfdb66..62d321bb 100644 --- a/src/hydra-queue-runner/build-remote.cc +++ b/src/hydra-queue-runner/build-remote.cc @@ -154,7 +154,7 @@ static void copyClosureTo( // FIXME: use Store::topoSortPaths(). -static StorePaths reverseTopoSortPaths(const std::map & paths) +static StorePaths reverseTopoSortPaths(const std::map & paths) { StorePaths sorted; StorePathSet visited; @@ -322,7 +322,7 @@ static BuildResult performBuild( return result; } -static std::map queryPathInfos( +static std::map queryPathInfos( Machine::Connection & conn, Store & localStore, StorePathSet & outputs, @@ -331,7 +331,7 @@ static std::map queryPathInfos( { /* Get info about each output path. */ - std::map infos; + std::map infos; conn.to << ServeProto::Command::QueryPathInfos; ServeProto::write(localStore, conn, outputs); conn.to.flush(); @@ -395,14 +395,16 @@ static void copyPathsFromRemote( NarMemberDatas & narMembers, Store & localStore, Store & destStore, - const std::map & infos + const std::map & infos ) { auto pathsSorted = reverseTopoSortPaths(infos); for (auto & path : pathsSorted) { auto & info = infos.find(path)->second; - copyPathFromRemote(conn, narMembers, localStore, destStore, info); + copyPathFromRemote( + conn, narMembers, localStore, destStore, + ValidPathInfo { path, info }); } } From d0d3b0a2986915ab7aa96d3fce8371a5012c9021 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 7 Dec 2023 14:13:06 -0500 Subject: [PATCH 067/313] Use `ServeProto::Serialise` for `QueryValidPaths` Companion to already-merged https://github.com/NixOS/nix/pull/9560 --- src/hydra-queue-runner/build-remote.cc | 21 ++++----------------- 1 file changed, 4 insertions(+), 17 deletions(-) diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc index 62d321bb..5818727b 100644 --- a/src/hydra-queue-runner/build-remote.cc +++ b/src/hydra-queue-runner/build-remote.cc @@ -338,23 +338,10 @@ static std::map queryPathInfos( while (true) { auto storePathS = readString(conn.from); if (storePathS == "") break; - auto deriver = readString(conn.from); // deriver - auto references = ServeProto::Serialise::read(localStore, conn); - readLongLong(conn.from); // download size - auto narSize = readLongLong(conn.from); - auto narHash = Hash::parseAny(readString(conn.from), HashAlgorithm::SHA256); - auto ca = ContentAddress::parseOpt(readString(conn.from)); - readStrings(conn.from); // sigs - ValidPathInfo info(localStore.parseStorePath(storePathS), narHash); - assert(outputs.count(info.path)); - info.references = references; - info.narSize = narSize; - totalNarSize += info.narSize; - info.narHash = narHash; - info.ca = ca; - if (deriver != "") - info.deriver = localStore.parseStorePath(deriver); - infos.insert_or_assign(info.path, info); + + auto storePath = localStore.parseStorePath(storePathS); + auto info = ServeProto::Serialise::read(localStore, conn); + infos.insert_or_assign(storePath, info); } return infos; From f6f817926a5ee98af4251a4244d01f5ba1f31695 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 9 Dec 2023 12:11:04 -0500 Subject: [PATCH 068/313] `std::move` the into the path info map --- src/hydra-queue-runner/build-remote.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc index 5818727b..a8047182 100644 --- a/src/hydra-queue-runner/build-remote.cc +++ b/src/hydra-queue-runner/build-remote.cc @@ -341,7 +341,7 @@ static std::map queryPathInfos( auto storePath = localStore.parseStorePath(storePathS); auto info = ServeProto::Serialise::read(localStore, conn); - infos.insert_or_assign(storePath, info); + infos.insert_or_assign(std::move(storePath), std::move(info)); } return infos; From 1d80b72ffb9d9619a0c146d4662d27e3bd45850b Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 10 Dec 2023 13:00:43 -0500 Subject: [PATCH 069/313] flake.lock: Update Nix master MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nix': 'github:NixOS/nix/c8458bd731eb1c74159bebe459ea00165e056b65' (2023-12-09) → 'github:NixOS/nix/b7e016ab2464ad2e7e2e856ad0f173157135aae0' (2023-12-10) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 424ed3b0..d99954a9 100644 --- a/flake.lock +++ b/flake.lock @@ -59,11 +59,11 @@ "nixpkgs-regression": "nixpkgs-regression" }, "locked": { - "lastModified": 1702090558, - "narHash": "sha256-JzuGOltp5Ht9flroZ7g0erD5ud2okNQChw9YlxWDX3c=", + "lastModified": 1702228562, + "narHash": "sha256-VwvfxT8LZcEm3H+VCMCrKzl7Ip+dHumxPfugAx3tjk8=", "owner": "NixOS", "repo": "nix", - "rev": "c8458bd731eb1c74159bebe459ea00165e056b65", + "rev": "b7e016ab2464ad2e7e2e856ad0f173157135aae0", "type": "github" }, "original": { From 69a5b00e60df0a40ac8f193f33937b7214872327 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 7 Dec 2023 14:18:00 -0500 Subject: [PATCH 070/313] Use `ServeProto::BuildOption` More deduplication with Nix. --- src/hydra-queue-runner/build-remote.cc | 8 ++++---- src/hydra-queue-runner/builder.cc | 15 +++++++++------ src/hydra-queue-runner/state.hh | 10 ++-------- 3 files changed, 15 insertions(+), 18 deletions(-) diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc index a8047182..1fdd5a23 100644 --- a/src/hydra-queue-runner/build-remote.cc +++ b/src/hydra-queue-runner/build-remote.cc @@ -282,7 +282,7 @@ static BuildResult performBuild( Store & localStore, StorePath drvPath, const BasicDerivation & drv, - const State::BuildOptions & options, + const ServeProto::BuildOptions & options, counter & nrStepsBuilding ) { @@ -293,7 +293,7 @@ static BuildResult performBuild( conn.to << options.maxLogSize; if (GET_PROTOCOL_MINOR(conn.remoteVersion) >= 3) { conn.to - << options.repeats // == build-repeat + << options.nrRepeats << options.enforceDeterminism; } conn.to.flush(); @@ -458,7 +458,7 @@ void RemoteResult::updateWithBuildResult(const nix::BuildResult & buildResult) void State::buildRemote(ref destStore, Machine::ptr machine, Step::ptr step, - const BuildOptions & buildOptions, + const ServeProto::BuildOptions & buildOptions, RemoteResult & result, std::shared_ptr activeStep, std::function updateStep, NarMemberDatas & narMembers) @@ -510,7 +510,7 @@ void State::buildRemote(ref destStore, }); try { - build_remote::handshake(conn, buildOptions.repeats); + build_remote::handshake(conn, buildOptions.nrRepeats); } catch (EndOfFile & e) { child.pid.wait(); std::string s = chomp(readFile(result.logFile)); diff --git a/src/hydra-queue-runner/builder.cc b/src/hydra-queue-runner/builder.cc index 307eee8e..cfa95cef 100644 --- a/src/hydra-queue-runner/builder.cc +++ b/src/hydra-queue-runner/builder.cc @@ -98,10 +98,13 @@ State::StepResult State::doBuildStep(nix::ref destStore, it). */ BuildID buildId; std::optional buildDrvPath; - BuildOptions buildOptions; - buildOptions.repeats = step->isDeterministic ? 1 : 0; - buildOptions.maxLogSize = maxLogSize; - buildOptions.enforceDeterminism = step->isDeterministic; + // Other fields set below + nix::ServeProto::BuildOptions buildOptions { + .maxLogSize = maxLogSize, + .nrRepeats = step->isDeterministic ? 1u : 0u, + .enforceDeterminism = step->isDeterministic, + .keepFailed = false, + }; auto conn(dbPool.get()); @@ -136,7 +139,7 @@ State::StepResult State::doBuildStep(nix::ref destStore, { auto i = jobsetRepeats.find(std::make_pair(build2->projectName, build2->jobsetName)); if (i != jobsetRepeats.end()) - buildOptions.repeats = std::max(buildOptions.repeats, i->second); + buildOptions.nrRepeats = std::max(buildOptions.nrRepeats, i->second); } } if (!build) build = *dependents.begin(); @@ -147,7 +150,7 @@ State::StepResult State::doBuildStep(nix::ref destStore, buildOptions.buildTimeout = build->buildTimeout; printInfo("performing step ‘%s’ %d times on ‘%s’ (needed by build %d and %d others)", - localStore->printStorePath(step->drvPath), buildOptions.repeats + 1, machine->sshName, buildId, (dependents.size() - 1)); + localStore->printStorePath(step->drvPath), buildOptions.nrRepeats + 1, machine->sshName, buildId, (dependents.size() - 1)); } if (!buildOneDone) diff --git a/src/hydra-queue-runner/state.hh b/src/hydra-queue-runner/state.hh index 6359063a..830f0598 100644 --- a/src/hydra-queue-runner/state.hh +++ b/src/hydra-queue-runner/state.hh @@ -459,7 +459,7 @@ private: /* How often the build steps of a jobset should be repeated in order to detect non-determinism. */ - std::map, unsigned int> jobsetRepeats; + std::map, size_t> jobsetRepeats; bool uploadLogsToBinaryCache; @@ -488,12 +488,6 @@ private: public: State(std::optional metricsAddrOpt); - struct BuildOptions { - unsigned int maxSilentTime, buildTimeout, repeats; - size_t maxLogSize; - bool enforceDeterminism; - }; - private: nix::MaintainCount startDbUpdate(); @@ -578,7 +572,7 @@ private: void buildRemote(nix::ref destStore, Machine::ptr machine, Step::ptr step, - const BuildOptions & buildOptions, + const nix::ServeProto::BuildOptions & buildOptions, RemoteResult & result, std::shared_ptr activeStep, std::function updateStep, NarMemberDatas & narMembers); From b56d2383c1e0c39418af35665f7d5b21cf49d381 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 7 Dec 2023 14:30:44 -0500 Subject: [PATCH 071/313] Do not attempt to speak a newer version of the protocol Both sides need to agree on a version (with `std::min`) for anything to work. Somehow... we've never done this. With this comment, the next commit succeeds. Without this commit, the next commit fails. This is because the next commit exposes serializers which do different things for proto version 2.7, and we're currently requesting 2.6. Opened https://github.com/NixOS/nix/issues/9584 to track this issue --- src/hydra-queue-runner/build-remote.cc | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc index 1fdd5a23..659618b7 100644 --- a/src/hydra-queue-runner/build-remote.cc +++ b/src/hydra-queue-runner/build-remote.cc @@ -203,7 +203,9 @@ static std::pair openLogFile(const std::string & logDir, cons */ static void handshake(Machine::Connection & conn, unsigned int repeats) { - conn.to << SERVE_MAGIC_1 << 0x206; + constexpr ServeProto::Version our_version = 0x206; + + conn.to << SERVE_MAGIC_1 << our_version; conn.to.flush(); unsigned int magic = readInt(conn.from); @@ -215,6 +217,9 @@ static void handshake(Machine::Connection & conn, unsigned int repeats) throw Error("unsupported ‘nix-store --serve’ protocol version on ‘%1%’", conn.machine->sshName); if (GET_PROTOCOL_MINOR(conn.remoteVersion) < 3 && repeats > 0) throw Error("machine ‘%1%’ does not support repeating a build; please upgrade it to Nix 1.12", conn.machine->sshName); + + // Do not attempt to speak a newer version of the protocol + conn.remoteVersion = std::min(conn.remoteVersion, our_version); } static BasicDerivation sendInputs( From 20f5a2120cf0663bd52df9e82c42d26942017c1a Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 9 Dec 2023 12:26:11 -0500 Subject: [PATCH 072/313] Use `ServeProto::Serialise` --- src/hydra-queue-runner/build-remote.cc | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc index 659618b7..a4882f5c 100644 --- a/src/hydra-queue-runner/build-remote.cc +++ b/src/hydra-queue-runner/build-remote.cc @@ -293,14 +293,7 @@ static BuildResult performBuild( { conn.to << ServeProto::Command::BuildDerivation << localStore.printStorePath(drvPath); writeDerivation(conn.to, localStore, drv); - conn.to << options.maxSilentTime << options.buildTimeout; - if (GET_PROTOCOL_MINOR(conn.remoteVersion) >= 2) - conn.to << options.maxLogSize; - if (GET_PROTOCOL_MINOR(conn.remoteVersion) >= 3) { - conn.to - << options.nrRepeats - << options.enforceDeterminism; - } + ServeProto::write(localStore, conn, options); conn.to.flush(); BuildResult result; From 8c10331ee818a21699a8b774cccdd32f476f8ba6 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 10 Dec 2023 14:05:26 -0500 Subject: [PATCH 073/313] Fix `totalNarSize` summation I accidentally removed it in d0d3b0a2986915ab7aa96d3fce8371a5012c9021. --- src/hydra-queue-runner/build-remote.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc index 1fdd5a23..41c7c453 100644 --- a/src/hydra-queue-runner/build-remote.cc +++ b/src/hydra-queue-runner/build-remote.cc @@ -341,6 +341,7 @@ static std::map queryPathInfos( auto storePath = localStore.parseStorePath(storePathS); auto info = ServeProto::Serialise::read(localStore, conn); + totalNarSize += info.narSize; infos.insert_or_assign(std::move(storePath), std::move(info)); } From ebfefb9161046c83bbbebd26f4cb37ac69628b40 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 11 Dec 2023 12:46:36 -0500 Subject: [PATCH 074/313] Sync up with some changes done to the main CA branch --- flake.lock | 8 +- flake.nix | 2 +- src/hydra-queue-runner/build-remote.cc | 55 +++------ src/hydra-queue-runner/builder.cc | 11 +- src/hydra-queue-runner/hydra-queue-runner.cc | 4 +- src/hydra-queue-runner/queue-monitor.cc | 119 ++++++++++--------- 6 files changed, 94 insertions(+), 105 deletions(-) diff --git a/flake.lock b/flake.lock index 2871e70a..e7f9224c 100644 --- a/flake.lock +++ b/flake.lock @@ -42,16 +42,16 @@ "nixpkgs-regression": "nixpkgs-regression" }, "locked": { - "lastModified": 1701122567, - "narHash": "sha256-iA8DqS+W2fWTfR+nNJSvMHqQ+4NpYMRT3b+2zS6JTvE=", + "lastModified": 1702314838, + "narHash": "sha256-calxK+fZ4/tZy1fbph8qyx4ePUAf4ZdvIugpzWeFIGE=", "owner": "NixOS", "repo": "nix", - "rev": "50f8f1c8bc019a4c0fd098b9ac674b94cfc6af0d", + "rev": "ae451e2247b18be6bd36b9d85e41b632e774f40b", "type": "github" }, "original": { "owner": "NixOS", - "ref": "2.19.2", + "ref": "2.19-maintenance", "repo": "nix", "type": "github" } diff --git a/flake.nix b/flake.nix index 5a5aef55..8280e076 100644 --- a/flake.nix +++ b/flake.nix @@ -2,7 +2,7 @@ description = "A Nix-based continuous build system"; inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.05"; - inputs.nix.url = "github:NixOS/nix/2.19.2"; + inputs.nix.url = "github:NixOS/nix/2.19-maintenance"; inputs.nix.inputs.nixpkgs.follows = "nixpkgs"; outputs = { self, nixpkgs, nix }: diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc index 377bdb56..65f9f3fb 100644 --- a/src/hydra-queue-runner/build-remote.cc +++ b/src/hydra-queue-runner/build-remote.cc @@ -182,40 +182,6 @@ static StorePaths reverseTopoSortPaths(const std::map return sorted; } -/** - * Replace the input derivations by their output paths to send a minimal closure - * to the builder. - * - * If we can afford it, resolve it, so that the newly generated derivation still - * has some sensible output paths. - */ -BasicDerivation inlineInputDerivations(Store & store, Derivation & drv, const StorePath & drvPath) -{ - BasicDerivation ret; - if (!drv.type().hasKnownOutputPaths()) { - auto maybeBasicDrv = drv.tryResolve(store); - if (!maybeBasicDrv) - throw Error( - "the derivation '%s' can’t be resolved. It’s probably " - "missing some outputs", - store.printStorePath(drvPath)); - ret = *maybeBasicDrv; - } else { - // If the derivation is a real `InputAddressed` derivation, we must - // resolve it manually to keep the original output paths - ret = BasicDerivation(drv); - for (auto & [drvPath, node] : drv.inputDrvs.map) { - auto drv2 = store.readDerivation(drvPath); - auto drv2Outputs = drv2.outputsAndOptPaths(store); - for (auto & name : node.value) { - auto inputPath = drv2Outputs.at(name); - ret.inputSrcs.insert(*inputPath.second); - } - } - } - return ret; -} - static std::pair openLogFile(const std::string & logDir, const StorePath & drvPath) { std::string base(drvPath.to_string()); @@ -262,7 +228,22 @@ static BasicDerivation sendInputs( counter & nrStepsCopyingTo ) { - BasicDerivation basicDrv = inlineInputDerivations(localStore, *step.drv, step.drvPath); + /* Replace the input derivations by their output paths to send a + minimal closure to the builder. + + `tryResolve` currently does *not* rewrite input addresses, so it + is safe to do this in all cases. (It should probably have a mode + to do that, however, but we would not use it here.) + */ + BasicDerivation basicDrv = ({ + auto maybeBasicDrv = step.drv->tryResolve(destStore, &localStore); + if (!maybeBasicDrv) + throw Error( + "the derivation '%s' can’t be resolved. It’s probably " + "missing some outputs", + localStore.printStorePath(step.drvPath)); + *maybeBasicDrv; + }); /* Ensure that the inputs exist in the destination store. This is a no-op for regular stores, but for the binary cache store, @@ -351,6 +332,8 @@ static BuildResult performBuild( // far anyways assert(drv.type().hasKnownOutputPaths()); DerivationOutputsAndOptPaths drvOutputs = drv.outputsAndOptPaths(localStore); + // Since this a `BasicDerivation`, `staticOutputHashes` will not + // do any real work. auto outputHashes = staticOutputHashes(localStore, drv); for (auto & [outputName, output] : drvOutputs) { auto outputPath = output.second; @@ -665,14 +648,12 @@ void State::buildRemote(ref destStore, auto outputHashes = staticOutputHashes(*localStore, *step->drv); for (auto & [outputName, realisation] : buildResult.builtOutputs) { // Register the resolved drv output - localStore->registerDrvOutput(realisation); destStore->registerDrvOutput(realisation); // Also register the unresolved one auto unresolvedRealisation = realisation; unresolvedRealisation.signatures.clear(); unresolvedRealisation.id.drvHash = outputHashes.at(outputName); - localStore->registerDrvOutput(unresolvedRealisation); destStore->registerDrvOutput(unresolvedRealisation); } } diff --git a/src/hydra-queue-runner/builder.cc b/src/hydra-queue-runner/builder.cc index 42983941..b1b58c05 100644 --- a/src/hydra-queue-runner/builder.cc +++ b/src/hydra-queue-runner/builder.cc @@ -223,7 +223,7 @@ State::StepResult State::doBuildStep(nix::ref destStore, if (result.stepStatus == bsSuccess) { updateStep(ssPostProcessing); - res = getBuildOutput(destStore, narMembers, localStore->queryDerivationOutputMap(step->drvPath)); + res = getBuildOutput(destStore, narMembers, destStore->queryDerivationOutputMap(step->drvPath, &*localStore)); } } @@ -277,9 +277,12 @@ State::StepResult State::doBuildStep(nix::ref destStore, assert(stepNr); - for (auto & i : localStore->queryPartialDerivationOutputMap(step->drvPath)) { - if (i.second) - addRoot(*i.second); + for (auto & [outputName, optOutputPath] : destStore->queryPartialDerivationOutputMap(step->drvPath, &*localStore)) { + if (!optOutputPath) + throw Error( + "Missing output %s for derivation %d which was supposed to have succeeded", + outputName, localStore->printStorePath(step->drvPath)); + addRoot(*optOutputPath); } /* Register success in the database for all Build objects that diff --git a/src/hydra-queue-runner/hydra-queue-runner.cc b/src/hydra-queue-runner/hydra-queue-runner.cc index 2f2c6091..d514ecfa 100644 --- a/src/hydra-queue-runner/hydra-queue-runner.cc +++ b/src/hydra-queue-runner/hydra-queue-runner.cc @@ -312,7 +312,7 @@ unsigned int State::createBuildStep(pqxx::work & txn, time_t startTime, BuildID if (r.affected_rows() == 0) goto restart; - for (auto & [name, output] : localStore->queryPartialDerivationOutputMap(step->drvPath)) + for (auto & [name, output] : getDestStore()->queryPartialDerivationOutputMap(step->drvPath, &*localStore)) txn.exec_params0 ("insert into BuildStepOutputs (build, stepnr, name, path) values ($1, $2, $3, $4)", buildId, stepNr, name, output ? localStore->printStorePath(*output) : ""); @@ -359,7 +359,7 @@ void State::finishBuildStep(pqxx::work & txn, const RemoteResult & result, assert(res.size()); StorePath drvPath = localStore->parseStorePath(res[0].as()); // If we've finished building, all the paths should be known - for (auto & [name, output] : localStore->queryDerivationOutputMap(drvPath)) + for (auto & [name, output] : getDestStore()->queryDerivationOutputMap(drvPath, &*localStore)) txn.exec_params0 ("update BuildStepOutputs set path = $4 where build = $1 and stepnr = $2 and name = $3", buildId, stepNr, name, localStore->printStorePath(output)); diff --git a/src/hydra-queue-runner/queue-monitor.cc b/src/hydra-queue-runner/queue-monitor.cc index 04917932..a4d9b4dc 100644 --- a/src/hydra-queue-runner/queue-monitor.cc +++ b/src/hydra-queue-runner/queue-monitor.cc @@ -192,11 +192,11 @@ bool State::getQueuedBuilds(Connection & conn, if (!res[0].is_null()) propagatedFrom = res[0].as(); if (!propagatedFrom) { - for (auto & i : localStore->queryPartialDerivationOutputMap(ex.step->drvPath)) { + for (auto & [outputName, _] : destStore->queryPartialDerivationOutputMap(ex.step->drvPath, &*localStore)) { auto res = txn.exec_params ("select max(s.build) from BuildSteps s join BuildStepOutputs o on s.build = o.build where drvPath = $1 and name = $2 and startTime != 0 and stopTime != 0 and status = 1", localStore->printStorePath(ex.step->drvPath), - i.first); + outputName); if (!res[0][0].is_null()) { propagatedFrom = res[0][0].as(); break; @@ -237,7 +237,7 @@ bool State::getQueuedBuilds(Connection & conn, if (!step) { BuildOutput res = getBuildOutputCached(conn, destStore, build->drvPath); - for (auto & i : localStore->queryDerivationOutputMap(build->drvPath)) + for (auto & i : destStore->queryDerivationOutputMap(build->drvPath, &*localStore)) addRoot(i.second); { @@ -481,20 +481,12 @@ Step::ptr State::createStep(ref destStore, auto outputHashes = staticOutputHashes(*localStore, *(step->drv)); bool valid = true; std::map> missing; - for (auto &[outputName, maybeOutputPath] : step->drv->outputsAndOptPaths(*destStore)) { + for (auto & [outputName, maybeOutputPath] : destStore->queryPartialDerivationOutputMap(drvPath, &*localStore)) { auto outputHash = outputHashes.at(outputName); - if (maybeOutputPath.second) { - if (!destStore->isValidPath(*maybeOutputPath.second)) { - valid = false; - missing.insert({{outputHash, outputName}, maybeOutputPath.second}); - } - } else { - experimentalFeatureSettings.require(Xp::CaDerivations); - if (!destStore->queryRealisation(DrvOutput{outputHash, outputName})) { - valid = false; - missing.insert({{outputHash, outputName}, std::nullopt}); - } - } + if (maybeOutputPath && destStore->isValidPath(*maybeOutputPath)) + continue; + valid = false; + missing.insert({{outputHash, outputName}, maybeOutputPath}); } /* Try to copy the missing paths from the local store or from @@ -503,14 +495,24 @@ Step::ptr State::createStep(ref destStore, size_t avail = 0; for (auto & [i, maybePath] : missing) { - if ((maybePath && localStore->isValidPath(*maybePath))) + // If we don't know the output path from the destination + // store, see if the local store can tell us. + if (/* localStore != destStore && */ !maybePath && experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) + if (auto maybeRealisation = localStore->queryRealisation(i)) + maybePath = maybeRealisation->outPath; + + if (!maybePath) { + // No hope of getting the store object if we don't know + // the path. + continue; + } + auto & path = *maybePath; + + if (/* localStore != destStore && */ localStore->isValidPath(path)) avail++; - else if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations) && localStore->queryRealisation(i)) { - maybePath = localStore->queryRealisation(i)->outPath; - avail++; - } else if (useSubstitutes && maybePath) { + else if (useSubstitutes) { SubstitutablePathInfos infos; - localStore->querySubstitutablePathInfos({{*maybePath, {}}}, infos); + localStore->querySubstitutablePathInfos({{path, {}}}, infos); if (infos.size() == 1) avail++; } @@ -518,44 +520,47 @@ Step::ptr State::createStep(ref destStore, if (missing.size() == avail) { valid = true; - for (auto & [i, path] : missing) { - if (path) { - try { - time_t startTime = time(0); + for (auto & [i, maybePath] : missing) { + // If we found everything, then we should know the path + // to every missing store object now. + assert(maybePath); + auto & path = *maybePath; - if (localStore->isValidPath(*path)) - printInfo("copying output ‘%1%’ of ‘%2%’ from local store", - localStore->printStorePath(*path), - localStore->printStorePath(drvPath)); - else { - printInfo("substituting output ‘%1%’ of ‘%2%’", - localStore->printStorePath(*path), - localStore->printStorePath(drvPath)); - localStore->ensurePath(*path); - // FIXME: should copy directly from substituter to destStore. - } + try { + time_t startTime = time(0); - copyClosure(*localStore, *destStore, - StorePathSet { *path }, - NoRepair, CheckSigs, NoSubstitute); - - time_t stopTime = time(0); - - { - auto mc = startDbUpdate(); - pqxx::work txn(conn); - createSubstitutionStep(txn, startTime, stopTime, build, drvPath, *(step->drv), "out", *path); - txn.commit(); - } - - } catch (Error & e) { - printError("while copying/substituting output ‘%s’ of ‘%s’: %s", - localStore->printStorePath(*path), - localStore->printStorePath(drvPath), - e.what()); - valid = false; - break; + if (localStore->isValidPath(path)) + printInfo("copying output ‘%1%’ of ‘%2%’ from local store", + localStore->printStorePath(path), + localStore->printStorePath(drvPath)); + else { + printInfo("substituting output ‘%1%’ of ‘%2%’", + localStore->printStorePath(path), + localStore->printStorePath(drvPath)); + localStore->ensurePath(path); + // FIXME: should copy directly from substituter to destStore. } + + copyClosure(*localStore, *destStore, + StorePathSet { path }, + NoRepair, CheckSigs, NoSubstitute); + + time_t stopTime = time(0); + + { + auto mc = startDbUpdate(); + pqxx::work txn(conn); + createSubstitutionStep(txn, startTime, stopTime, build, drvPath, *(step->drv), "out", path); + txn.commit(); + } + + } catch (Error & e) { + printError("while copying/substituting output ‘%s’ of ‘%s’: %s", + localStore->printStorePath(path), + localStore->printStorePath(drvPath), + e.what()); + valid = false; + break; } } } From a6b6c5a5392a7e705825e8dac7cb818a1f9944fe Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 11 Dec 2023 12:55:57 -0500 Subject: [PATCH 075/313] Revert query -- those columns don't exist yet! --- src/hydra-queue-runner/queue-monitor.cc | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/hydra-queue-runner/queue-monitor.cc b/src/hydra-queue-runner/queue-monitor.cc index a4d9b4dc..fc344244 100644 --- a/src/hydra-queue-runner/queue-monitor.cc +++ b/src/hydra-queue-runner/queue-monitor.cc @@ -192,11 +192,12 @@ bool State::getQueuedBuilds(Connection & conn, if (!res[0].is_null()) propagatedFrom = res[0].as(); if (!propagatedFrom) { - for (auto & [outputName, _] : destStore->queryPartialDerivationOutputMap(ex.step->drvPath, &*localStore)) { + for (auto & [outputName, optOutputPath] : destStore->queryPartialDerivationOutputMap(ex.step->drvPath, &*localStore)) { + // ca-derivations not actually supported yet + assert(optOutputPath); auto res = txn.exec_params - ("select max(s.build) from BuildSteps s join BuildStepOutputs o on s.build = o.build where drvPath = $1 and name = $2 and startTime != 0 and stopTime != 0 and status = 1", - localStore->printStorePath(ex.step->drvPath), - outputName); + ("select max(s.build) from BuildSteps s join BuildStepOutputs o on s.build = o.build where path = $1 and startTime != 0 and stopTime != 0 and status = 1", + localStore->printStorePath(*optOutputPath)); if (!res[0][0].is_null()) { propagatedFrom = res[0][0].as(); break; From 6e67884ff164d9d1c115aef066e18f8de0c5fc44 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 11 Dec 2023 14:05:18 -0500 Subject: [PATCH 076/313] One more `queryDerivationOutputMap` should use the eval store param --- src/hydra-queue-runner/queue-monitor.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/hydra-queue-runner/queue-monitor.cc b/src/hydra-queue-runner/queue-monitor.cc index fc344244..81678fe1 100644 --- a/src/hydra-queue-runner/queue-monitor.cc +++ b/src/hydra-queue-runner/queue-monitor.cc @@ -658,8 +658,7 @@ void State::processJobsetSharesChange(Connection & conn) BuildOutput State::getBuildOutputCached(Connection & conn, nix::ref destStore, const nix::StorePath & drvPath) { - - auto derivationOutputs = localStore->queryDerivationOutputMap(drvPath); + auto derivationOutputs = destStore->queryDerivationOutputMap(drvPath, &*localStore); { pqxx::work txn(conn); From 7517c134c57c7ce967a7b603482a23f4952331a0 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 14 Dec 2023 00:29:23 -0500 Subject: [PATCH 077/313] flake.lock: Update Nix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Newer 2.19-maintenance has some `restricted-eval` fixes that benefit Hydra. Flake lock file updates: • Updated input 'nix': 'github:NixOS/nix/50f8f1c8bc019a4c0fd098b9ac674b94cfc6af0d' (2023-12-11) → 'github:NixOS/nix/b38e5a665e9d0aa7975beb0ed12e42d13a392e74' (2023-12-13) --- flake.lock | 8 ++++---- flake.nix | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/flake.lock b/flake.lock index 2871e70a..66e30323 100644 --- a/flake.lock +++ b/flake.lock @@ -42,16 +42,16 @@ "nixpkgs-regression": "nixpkgs-regression" }, "locked": { - "lastModified": 1701122567, - "narHash": "sha256-iA8DqS+W2fWTfR+nNJSvMHqQ+4NpYMRT3b+2zS6JTvE=", + "lastModified": 1702510710, + "narHash": "sha256-9K+w1mQgmUxCmEsPaSFkpYsj/cxjO2PSwTCPkNZ/NiU=", "owner": "NixOS", "repo": "nix", - "rev": "50f8f1c8bc019a4c0fd098b9ac674b94cfc6af0d", + "rev": "b38e5a665e9d0aa7975beb0ed12e42d13a392e74", "type": "github" }, "original": { "owner": "NixOS", - "ref": "2.19.2", + "ref": "2.19-maintenance", "repo": "nix", "type": "github" } diff --git a/flake.nix b/flake.nix index 5a5aef55..8280e076 100644 --- a/flake.nix +++ b/flake.nix @@ -2,7 +2,7 @@ description = "A Nix-based continuous build system"; inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.05"; - inputs.nix.url = "github:NixOS/nix/2.19.2"; + inputs.nix.url = "github:NixOS/nix/2.19-maintenance"; inputs.nix.inputs.nixpkgs.follows = "nixpkgs"; outputs = { self, nixpkgs, nix }: From abd858d3dcd8dbcb2ee2d17ff5b03dbce46821e7 Mon Sep 17 00:00:00 2001 From: Jack Kelly Date: Tue, 19 Dec 2023 07:54:40 +1000 Subject: [PATCH 078/313] Document the `store_uri` parameter by way of example --- doc/manual/src/configuration.md | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/doc/manual/src/configuration.md b/doc/manual/src/configuration.md index 02210449..4954040c 100644 --- a/doc/manual/src/configuration.md +++ b/doc/manual/src/configuration.md @@ -74,6 +74,30 @@ following: } } +Populating a Cache +------------------ + +A common use for Hydra is to pre-build and cache derivations which +take a long time to build. While it is possible to direcly access the +Hydra server's store over SSH, a more scalable option is to upload +built derivations to a remote store like an [S3-compatible object +store](https://nixos.org/manual/nix/stable/command-ref/new-cli/nix3-help-stores.html#s3-binary-cache-store). Setting +the `store_uri` parameter will cause Hydra to sign and upload +derivations as they are built: + +``` +store_uri = s3://cache-bucket-name?compression=zstd¶llel-compression=true&write-nar-listing=1&ls-compression=br&log-compression=br&secret-key=/path/to/cache/private/key +``` + +This example uses [Zstandard](https://github.com/facebook/zstd) +compression on derivations to reduce CPU usage on the server, but +[Brotli](https://brotli.org/) compression for derivation listings and +build logs because it has better browser support. + +See [`nix help +stores`](https://nixos.org/manual/nix/stable/command-ref/new-cli/nix3-help-stores.html) +for a description of the store URI format. + Statsd Configuration -------------------- From 75f26f1fc4f114b7f930c57ba37212ea67990de1 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 23 Dec 2023 19:10:58 -0500 Subject: [PATCH 079/313] Clean up `std::optional` dereferencing in the queue runner Instead of doing this partial operation a number of times, assert (with a comment, get a reference to the thing inside, and use that just once. (This refactor was done twice, "just once" for each time.) --- src/hydra-queue-runner/queue-monitor.cc | 26 ++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/src/hydra-queue-runner/queue-monitor.cc b/src/hydra-queue-runner/queue-monitor.cc index 6c339af6..09a2871e 100644 --- a/src/hydra-queue-runner/queue-monitor.cc +++ b/src/hydra-queue-runner/queue-monitor.cc @@ -495,12 +495,14 @@ Step::ptr State::createStep(ref destStore, size_t avail = 0; for (auto & i : missing) { - auto path = i.second.path(*localStore, step->drv->name, i.first); - if (/* localStore != destStore && */ localStore->isValidPath(*path)) + auto pathOpt = i.second.path(*localStore, step->drv->name, i.first); + assert(pathOpt); // CA derivations not yet supported + auto & path = *pathOpt; + if (/* localStore != destStore && */ localStore->isValidPath(path)) avail++; else if (useSubstitutes) { SubstitutablePathInfos infos; - localStore->querySubstitutablePathInfos({{*path, {}}}, infos); + localStore->querySubstitutablePathInfos({{path, {}}}, infos); if (infos.size() == 1) avail++; } @@ -509,25 +511,27 @@ Step::ptr State::createStep(ref destStore, if (missing.size() == avail) { valid = true; for (auto & i : missing) { - auto path = i.second.path(*localStore, step->drv->name, i.first); + auto pathOpt = i.second.path(*localStore, step->drv->name, i.first); + assert(pathOpt); // CA derivations not yet supported + auto & path = *pathOpt; try { time_t startTime = time(0); - if (localStore->isValidPath(*path)) + if (localStore->isValidPath(path)) printInfo("copying output ‘%1%’ of ‘%2%’ from local store", - localStore->printStorePath(*path), + localStore->printStorePath(path), localStore->printStorePath(drvPath)); else { printInfo("substituting output ‘%1%’ of ‘%2%’", - localStore->printStorePath(*path), + localStore->printStorePath(path), localStore->printStorePath(drvPath)); - localStore->ensurePath(*path); + localStore->ensurePath(path); // FIXME: should copy directly from substituter to destStore. } copyClosure(*localStore, *destStore, - StorePathSet { *path }, + StorePathSet { path }, NoRepair, CheckSigs, NoSubstitute); time_t stopTime = time(0); @@ -535,13 +539,13 @@ Step::ptr State::createStep(ref destStore, { auto mc = startDbUpdate(); pqxx::work txn(conn); - createSubstitutionStep(txn, startTime, stopTime, build, drvPath, "out", *path); + createSubstitutionStep(txn, startTime, stopTime, build, drvPath, "out", path); txn.commit(); } } catch (Error & e) { printError("while copying/substituting output ‘%s’ of ‘%s’: %s", - localStore->printStorePath(*path), + localStore->printStorePath(path), localStore->printStorePath(drvPath), e.what()); valid = false; From db7aa01b8d90439081ee60dce998a8b30ad5f5b2 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 22 Jan 2024 13:49:05 -0500 Subject: [PATCH 080/313] Update to newer Nix master MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nix': 'github:NixOS/nix/b7e016ab2464ad2e7e2e856ad0f173157135aae0' (2023-12-10) → 'github:NixOS/nix/74534829f23b668fb9b2f2a14ff6afa4d5e71d4a' (2024-01-22) • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/e9f06adb793d1cca5384907b3b8a4071d5d7cb19' (2023-12-03) → 'github:NixOS/nixpkgs/a1982c92d8980a0114372973cbdfe0a307f1bdea' (2024-01-12) • Removed input 'nix/lowdown-src' --- flake.lock | 31 ++++++-------------------- flake.nix | 2 +- src/hydra-eval-jobs/hydra-eval-jobs.cc | 4 ++-- 3 files changed, 10 insertions(+), 27 deletions(-) diff --git a/flake.lock b/flake.lock index d99954a9..8b15a882 100644 --- a/flake.lock +++ b/flake.lock @@ -32,38 +32,21 @@ "type": "github" } }, - "lowdown-src": { - "flake": false, - "locked": { - "lastModified": 1633514407, - "narHash": "sha256-Dw32tiMjdK9t3ETl5fzGrutQTzh2rufgZV4A/BbxuD4=", - "owner": "kristapsdz", - "repo": "lowdown", - "rev": "d2c2b44ff6c27b936ec27358a2653caaef8f73b8", - "type": "github" - }, - "original": { - "owner": "kristapsdz", - "repo": "lowdown", - "type": "github" - } - }, "nix": { "inputs": { "flake-compat": "flake-compat", "libgit2": "libgit2", - "lowdown-src": "lowdown-src", "nixpkgs": [ "nixpkgs" ], "nixpkgs-regression": "nixpkgs-regression" }, "locked": { - "lastModified": 1702228562, - "narHash": "sha256-VwvfxT8LZcEm3H+VCMCrKzl7Ip+dHumxPfugAx3tjk8=", + "lastModified": 1705948977, + "narHash": "sha256-8ZUbb/vf2R0kKKHJyLTIh0HYKzgkV2pq/unDj0kzKG8=", "owner": "NixOS", "repo": "nix", - "rev": "b7e016ab2464ad2e7e2e856ad0f173157135aae0", + "rev": "74534829f23b668fb9b2f2a14ff6afa4d5e71d4a", "type": "github" }, "original": { @@ -74,16 +57,16 @@ }, "nixpkgs": { "locked": { - "lastModified": 1701615100, - "narHash": "sha256-7VI84NGBvlCTduw2aHLVB62NvCiZUlALLqBe5v684Aw=", + "lastModified": 1705033721, + "narHash": "sha256-K5eJHmL1/kev6WuqyqqbS1cdNnSidIZ3jeqJ7GbrYnQ=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "e9f06adb793d1cca5384907b3b8a4071d5d7cb19", + "rev": "a1982c92d8980a0114372973cbdfe0a307f1bdea", "type": "github" }, "original": { "owner": "NixOS", - "ref": "nixos-23.05", + "ref": "nixos-23.05-small", "repo": "nixpkgs", "type": "github" } diff --git a/flake.nix b/flake.nix index fdeaa6ca..c9af0174 100644 --- a/flake.nix +++ b/flake.nix @@ -1,7 +1,7 @@ { description = "A Nix-based continuous build system"; - inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.05"; + inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.05-small"; inputs.nix.url = "github:NixOS/nix"; inputs.nix.inputs.nixpkgs.follows = "nixpkgs"; diff --git a/src/hydra-eval-jobs/hydra-eval-jobs.cc b/src/hydra-eval-jobs/hydra-eval-jobs.cc index 2fe2c80f..1c4b5d93 100644 --- a/src/hydra-eval-jobs/hydra-eval-jobs.cc +++ b/src/hydra-eval-jobs/hydra-eval-jobs.cc @@ -89,7 +89,7 @@ struct MyArgs : MixEvalArgs, MixCommonArgs, RootArgs static MyArgs myArgs; -static std::string queryMetaStrings(EvalState & state, DrvInfo & drv, const std::string & name, const std::string & subAttribute) +static std::string queryMetaStrings(EvalState & state, PackageInfo & drv, const std::string & name, const std::string & subAttribute) { Strings res; std::function rec; @@ -178,7 +178,7 @@ static void worker( if (auto drv = getDerivation(state, *v, false)) { - DrvInfo::Outputs outputs = drv->queryOutputs(); + PackageInfo::Outputs outputs = drv->queryOutputs(); if (drv->querySystem() == "unknown") throw EvalError("derivation must have a 'system' attribute"); From 4ac31c89df85a6a72d82a392b729b0c90bcd5d33 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 20 Feb 2022 19:51:28 +0000 Subject: [PATCH 081/313] Use `nix::serv_proto::BasicConnection` in build_remote.cc - Use the type itself This lays the foundation for being able to dedup the protocol code. - Use `BasicConnection::handshake`, replacing ours. - Use `BasicConnection::queryValidPaths` - Use `BasicConnection::putBuildDerivationRequest` --- src/hydra-queue-runner/build-remote.cc | 64 ++++++++++---------------- src/hydra-queue-runner/state.hh | 23 +-------- 2 files changed, 26 insertions(+), 61 deletions(-) diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc index f19008dc..058af02c 100644 --- a/src/hydra-queue-runner/build-remote.cc +++ b/src/hydra-queue-runner/build-remote.cc @@ -8,6 +8,7 @@ #include "build-result.hh" #include "path.hh" #include "serve-protocol.hh" +#include "serve-protocol-impl.hh" #include "state.hh" #include "current-process.hh" #include "processes.hh" @@ -123,13 +124,10 @@ static void copyClosureTo( garbage-collect paths that are already there. Optionally, ask the remote host to substitute missing paths. */ // FIXME: substitute output pollutes our build log - conn.to << ServeProto::Command::QueryValidPaths << 1 << useSubstitutes; - ServeProto::write(destStore, conn, closure); - conn.to.flush(); - /* Get back the set of paths that are already valid on the remote host. */ - auto present = ServeProto::Serialise::read(destStore, conn); + auto present = conn.queryValidPaths( + destStore, true, closure, useSubstitutes); if (present.size() == closure.size()) return; @@ -195,33 +193,6 @@ static std::pair openLogFile(const std::string & logDir, cons return {std::move(logFile), std::move(logFD)}; } -/** - * @param conn is not fully initialized; it is this functions job to set - * the `remoteVersion` field after the handshake is completed. - * Therefore, no `ServeProto::Serialize` functions can be used until - * that field is set. - */ -static void handshake(Machine::Connection & conn, unsigned int repeats) -{ - constexpr ServeProto::Version our_version = 0x206; - - conn.to << SERVE_MAGIC_1 << our_version; - conn.to.flush(); - - unsigned int magic = readInt(conn.from); - if (magic != SERVE_MAGIC_2) - throw Error("protocol mismatch with ‘nix-store --serve’ on ‘%1%’", conn.machine->sshName); - conn.remoteVersion = readInt(conn.from); - // Now `conn` is initialized. - if (GET_PROTOCOL_MAJOR(conn.remoteVersion) != 0x200) - throw Error("unsupported ‘nix-store --serve’ protocol version on ‘%1%’", conn.machine->sshName); - if (GET_PROTOCOL_MINOR(conn.remoteVersion) < 3 && repeats > 0) - throw Error("machine ‘%1%’ does not support repeating a build; please upgrade it to Nix 1.12", conn.machine->sshName); - - // Do not attempt to speak a newer version of the protocol - conn.remoteVersion = std::min(conn.remoteVersion, our_version); -} - static BasicDerivation sendInputs( State & state, Step & step, @@ -291,10 +262,7 @@ static BuildResult performBuild( counter & nrStepsBuilding ) { - conn.to << ServeProto::Command::BuildDerivation << localStore.printStorePath(drvPath); - writeDerivation(conn.to, localStore, drv); - ServeProto::write(localStore, conn, options); - conn.to.flush(); + conn.putBuildDerivationRequest(localStore, drvPath, drv, options); BuildResult result; @@ -498,9 +466,13 @@ void State::buildRemote(ref destStore, }); Machine::Connection conn { - .from = child.from.get(), - .to = child.to.get(), - .machine = machine, + { + .to = child.to.get(), + .from = child.from.get(), + /* Handshake. */ + .remoteVersion = 0xdadbeef, // FIXME avoid dummy initialize + }, + /*.machine =*/ machine, }; Finally updateStats([&]() { @@ -508,14 +480,26 @@ void State::buildRemote(ref destStore, bytesSent += conn.to.written; }); + constexpr ServeProto::Version our_version = 0x206; + try { - build_remote::handshake(conn, buildOptions.nrRepeats); + conn.remoteVersion = decltype(conn)::handshake( + conn.to, + conn.from, + our_version, + machine->sshName); } catch (EndOfFile & e) { child.pid.wait(); std::string s = chomp(readFile(result.logFile)); throw Error("cannot connect to ‘%1%’: %2%", machine->sshName, s); } + // Do not attempt to speak a newer version of the protocol. + // + // Per https://github.com/NixOS/nix/issues/9584 should be handled as + // part of `handshake` in upstream nix. + conn.remoteVersion = std::min(conn.remoteVersion, our_version); + { auto info(machine->state->connectInfo.lock()); info->consecutiveFailures = 0; diff --git a/src/hydra-queue-runner/state.hh b/src/hydra-queue-runner/state.hh index 830f0598..82481ab5 100644 --- a/src/hydra-queue-runner/state.hh +++ b/src/hydra-queue-runner/state.hh @@ -22,6 +22,7 @@ #include "sync.hh" #include "nar-extractor.hh" #include "serve-protocol.hh" +#include "serve-protocol-impl.hh" typedef unsigned int BuildID; @@ -302,29 +303,9 @@ struct Machine } // A connection to a machine - struct Connection { - nix::FdSource from; - nix::FdSink to; - nix::ServeProto::Version remoteVersion; - + struct Connection : nix::ServeProto::BasicClientConnection { // Backpointer to the machine ptr machine; - - operator nix::ServeProto::ReadConn () - { - return { - .from = from, - .version = remoteVersion, - }; - } - - operator nix::ServeProto::WriteConn () - { - return { - .to = to, - .version = remoteVersion, - }; - } }; }; From 4e8fbaa3d688a4ffa95ccbaa0af889569c918633 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 23 Jan 2024 00:00:23 -0500 Subject: [PATCH 082/313] Replace `Child` with `SSHMaster::Connection` Nix defines basically an identical struct for the same purpose, so let's just use that. --- src/hydra-queue-runner/build-remote.cc | 30 +++++++++++--------------- 1 file changed, 12 insertions(+), 18 deletions(-) diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc index b117fdad..a4f3a35c 100644 --- a/src/hydra-queue-runner/build-remote.cc +++ b/src/hydra-queue-runner/build-remote.cc @@ -14,19 +14,13 @@ #include "util.hh" #include "serve-protocol.hh" #include "serve-protocol-impl.hh" +#include "ssh.hh" #include "finally.hh" #include "url.hh" using namespace nix; -struct Child -{ - Pid pid; - AutoCloseFD to, from; -}; - - static void append(Strings & dst, const Strings & src) { dst.insert(dst.end(), src.begin(), src.end()); @@ -54,7 +48,7 @@ static Strings extraStoreArgs(std::string & machine) return result; } -static void openConnection(Machine::ptr machine, Path tmpDir, int stderrFD, Child & child) +static void openConnection(Machine::ptr machine, Path tmpDir, int stderrFD, SSHMaster::Connection & child) { std::string pgmName; Pipe to, from; @@ -84,7 +78,7 @@ static void openConnection(Machine::ptr machine, Path tmpDir, int stderrFD, Chil append(argv, extraArgs); } - child.pid = startProcess([&]() { + child.sshPid = startProcess([&]() { restoreProcessContext(); if (dup2(to.readSide.get(), STDIN_FILENO) == -1) @@ -104,8 +98,8 @@ static void openConnection(Machine::ptr machine, Path tmpDir, int stderrFD, Chil to.readSide = -1; from.writeSide = -1; - child.to = to.writeSide.release(); - child.from = from.readSide.release(); + child.in = to.writeSide.release(); + child.out = from.readSide.release(); } @@ -488,13 +482,13 @@ void State::buildRemote(ref destStore, updateStep(ssConnecting); // FIXME: rewrite to use Store. - Child child; + SSHMaster::Connection child; build_remote::openConnection(machine, tmpDir, logFD.get(), child); { auto activeStepState(activeStep->state_.lock()); if (activeStepState->cancelled) throw Error("step cancelled"); - activeStepState->pid = child.pid; + activeStepState->pid = child.sshPid; } Finally clearPid([&]() { @@ -510,8 +504,8 @@ void State::buildRemote(ref destStore, }); Machine::Connection conn { - .from = child.from.get(), - .to = child.to.get(), + .from = child.out.get(), + .to = child.in.get(), .machine = machine, }; @@ -523,7 +517,7 @@ void State::buildRemote(ref destStore, try { build_remote::handshake(conn, buildOptions.repeats); } catch (EndOfFile & e) { - child.pid.wait(); + child.sshPid.wait(); std::string s = chomp(readFile(result.logFile)); throw Error("cannot connect to ‘%1%’: %2%", machine->sshName, s); } @@ -617,8 +611,8 @@ void State::buildRemote(ref destStore, } /* Shut down the connection. */ - child.to = -1; - child.pid.wait(); + child.in = -1; + child.sshPid.wait(); } catch (Error & e) { /* Disable this machine until a certain period of time has From 84c46b6b6865721bb23a38580f4e190548a5a380 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 23 Jan 2024 09:43:43 -0500 Subject: [PATCH 083/313] Update to newer Nix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nix': 'github:NixOS/nix/74534829f23b668fb9b2f2a14ff6afa4d5e71d4a' (2024-01-22) → 'github:NixOS/nix/b6aee9a93f6646bbffd919d362a5c75c37bb9caa' (2024-01-23) --- flake.lock | 6 +- src/hydra-queue-runner/nar-extractor.cc | 91 +++++++++++++------------ 2 files changed, 52 insertions(+), 45 deletions(-) diff --git a/flake.lock b/flake.lock index 8b15a882..c78d5203 100644 --- a/flake.lock +++ b/flake.lock @@ -42,11 +42,11 @@ "nixpkgs-regression": "nixpkgs-regression" }, "locked": { - "lastModified": 1705948977, - "narHash": "sha256-8ZUbb/vf2R0kKKHJyLTIh0HYKzgkV2pq/unDj0kzKG8=", + "lastModified": 1706016881, + "narHash": "sha256-XZz6tfTuZcrrX27vI+jAereNVWmnCrzFhBFhaKwEdoY=", "owner": "NixOS", "repo": "nix", - "rev": "74534829f23b668fb9b2f2a14ff6afa4d5e71d4a", + "rev": "b6aee9a93f6646bbffd919d362a5c75c37bb9caa", "type": "github" }, "original": { diff --git a/src/hydra-queue-runner/nar-extractor.cc b/src/hydra-queue-runner/nar-extractor.cc index ff4aa268..61299ecd 100644 --- a/src/hydra-queue-runner/nar-extractor.cc +++ b/src/hydra-queue-runner/nar-extractor.cc @@ -6,7 +6,46 @@ using namespace nix; -struct Extractor : ParseSink + +struct NarMemberConstructor : CreateRegularFileSink +{ + NarMemberData & curMember; + + HashSink hashSink = HashSink { HashAlgorithm::SHA256 }; + + std::optional expectedSize; + + NarMemberConstructor(NarMemberData & curMember) + : curMember(curMember) + { } + + void isExecutable() override + { + } + + void preallocateContents(uint64_t size) override + { + expectedSize = size; + } + + void operator () (std::string_view data) override + { + assert(expectedSize); + *curMember.fileSize += data.size(); + hashSink(data); + if (curMember.contents) { + curMember.contents->append(data); + } + assert(curMember.fileSize <= expectedSize); + if (curMember.fileSize == expectedSize) { + auto [hash, len] = hashSink.finish(); + assert(curMember.fileSize == len); + curMember.sha256 = hash; + } + } +}; + +struct Extractor : FileSystemObjectSink { std::unordered_set filesToKeep { "/nix-support/hydra-build-products", @@ -15,7 +54,6 @@ struct Extractor : ParseSink }; NarMemberDatas & members; - NarMemberData * curMember = nullptr; Path prefix; Extractor(NarMemberDatas & members, const Path & prefix) @@ -27,53 +65,22 @@ struct Extractor : ParseSink members.insert_or_assign(prefix + path, NarMemberData { .type = SourceAccessor::Type::tDirectory }); } - void createRegularFile(const Path & path) override + void createRegularFile(const Path & path, std::function func) override { - curMember = &members.insert_or_assign(prefix + path, NarMemberData { - .type = SourceAccessor::Type::tRegular, - .fileSize = 0, - .contents = filesToKeep.count(path) ? std::optional("") : std::nullopt, - }).first->second; - } - - std::optional expectedSize; - std::unique_ptr hashSink; - - void preallocateContents(uint64_t size) override - { - expectedSize = size; - hashSink = std::make_unique(HashAlgorithm::SHA256); - } - - void receiveContents(std::string_view data) override - { - assert(expectedSize); - assert(curMember); - assert(hashSink); - *curMember->fileSize += data.size(); - (*hashSink)(data); - if (curMember->contents) { - curMember->contents->append(data); - } - assert(curMember->fileSize <= expectedSize); - if (curMember->fileSize == expectedSize) { - auto [hash, len] = hashSink->finish(); - assert(curMember->fileSize == len); - curMember->sha256 = hash; - hashSink.reset(); - } + NarMemberConstructor nmc { + members.insert_or_assign(prefix + path, NarMemberData { + .type = SourceAccessor::Type::tRegular, + .fileSize = 0, + .contents = filesToKeep.count(path) ? std::optional("") : std::nullopt, + }).first->second, + }; + func(nmc); } void createSymlink(const Path & path, const std::string & target) override { members.insert_or_assign(prefix + path, NarMemberData { .type = SourceAccessor::Type::tSymlink }); } - - void isExecutable() override - { } - - void closeRegularFile() override - { } }; From 7386caaecfd68fbf70099eb8f3a6f8e296d1fb5b Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 22 Jan 2024 18:38:39 -0500 Subject: [PATCH 084/313] Use Nix's `SSHMaster` --- src/hydra-queue-runner/build-remote.cc | 87 +++++++------------------- 1 file changed, 24 insertions(+), 63 deletions(-) diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc index 68a339f3..7f547a0b 100644 --- a/src/hydra-queue-runner/build-remote.cc +++ b/src/hydra-queue-runner/build-remote.cc @@ -21,12 +21,6 @@ using namespace nix; - -static void append(Strings & dst, const Strings & src) -{ - dst.insert(dst.end(), src.begin(), src.end()); -} - namespace nix::build_remote { static Strings extraStoreArgs(std::string & machine) @@ -49,58 +43,20 @@ static Strings extraStoreArgs(std::string & machine) return result; } -static void openConnection(Machine::ptr machine, Path tmpDir, int stderrFD, SSHMaster::Connection & child) +static std::unique_ptr openConnection( + Machine::ptr machine, SSHMaster & master) { - std::string pgmName; - Pipe to, from; - to.create(); - from.create(); - - Strings argv; + Strings command = {"nix-store", "--serve", "--write"}; if (machine->isLocalhost()) { - pgmName = "nix-store"; - argv = {"nix-store", "--builders", "", "--serve", "--write"}; + command.push_back("--builders"); + command.push_back(""); } else { - pgmName = "ssh"; - auto sshName = machine->sshName; - Strings extraArgs = extraStoreArgs(sshName); - argv = {"ssh", sshName}; - if (machine->sshKey != "") append(argv, {"-i", machine->sshKey}); - if (machine->sshPublicHostKey != "") { - Path fileName = tmpDir + "/host-key"; - auto p = machine->sshName.find("@"); - std::string host = p != std::string::npos ? std::string(machine->sshName, p + 1) : machine->sshName; - writeFile(fileName, host + " " + machine->sshPublicHostKey + "\n"); - append(argv, {"-oUserKnownHostsFile=" + fileName}); - } - append(argv, - { "-x", "-a", "-oBatchMode=yes", "-oConnectTimeout=60", "-oTCPKeepAlive=yes" - , "--", "nix-store", "--serve", "--write" }); - append(argv, extraArgs); + command.splice(command.end(), extraStoreArgs(machine->sshName)); } - child.sshPid = startProcess([&]() { - restoreProcessContext(); - - if (dup2(to.readSide.get(), STDIN_FILENO) == -1) - throw SysError("cannot dup input pipe to stdin"); - - if (dup2(from.writeSide.get(), STDOUT_FILENO) == -1) - throw SysError("cannot dup output pipe to stdout"); - - if (dup2(stderrFD, STDERR_FILENO) == -1) - throw SysError("cannot dup stderr"); - - execvp(argv.front().c_str(), (char * *) stringsToCharPtrs(argv).data()); // FIXME: remove cast - - throw SysError("cannot start %s", pgmName); + return master.startCommand(std::move(command), { + "-a", "-oBatchMode=yes", "-oConnectTimeout=60", "-oTCPKeepAlive=yes" }); - - to.readSide = -1; - from.writeSide = -1; - - child.in = to.writeSide.release(); - child.out = from.readSide.release(); } @@ -430,21 +386,26 @@ void State::buildRemote(ref destStore, AutoDelete logFileDel(logFile, false); result.logFile = logFile; - nix::Path tmpDir = createTempDir(); - AutoDelete tmpDirDel(tmpDir, true); - try { updateStep(ssConnecting); + SSHMaster master { + machine->sshName, + machine->sshKey, + machine->sshPublicHostKey, + false, // no SSH master yet + false, // no compression yet + logFD.get(), + }; + // FIXME: rewrite to use Store. - SSHMaster::Connection child; - build_remote::openConnection(machine, tmpDir, logFD.get(), child); + auto child = build_remote::openConnection(machine, master); { auto activeStepState(activeStep->state_.lock()); if (activeStepState->cancelled) throw Error("step cancelled"); - activeStepState->pid = child.sshPid; + activeStepState->pid = child->sshPid; } Finally clearPid([&]() { @@ -461,8 +422,8 @@ void State::buildRemote(ref destStore, Machine::Connection conn { { - .to = child.in.get(), - .from = child.out.get(), + .to = child->in.get(), + .from = child->out.get(), /* Handshake. */ .remoteVersion = 0xdadbeef, // FIXME avoid dummy initialize }, @@ -483,7 +444,7 @@ void State::buildRemote(ref destStore, our_version, machine->sshName); } catch (EndOfFile & e) { - child.sshPid.wait(); + child->sshPid.wait(); std::string s = chomp(readFile(result.logFile)); throw Error("cannot connect to ‘%1%’: %2%", machine->sshName, s); } @@ -583,8 +544,8 @@ void State::buildRemote(ref destStore, } /* Shut down the connection. */ - child.in = -1; - child.sshPid.wait(); + child->in = -1; + child->sshPid.wait(); } catch (Error & e) { /* Disable this machine until a certain period of time has From 2e6ee28f9bef00ef08e6b6125f04af82cfca9fd3 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 23 Jan 2024 11:03:19 -0500 Subject: [PATCH 085/313] `Machine` -> `::Machine` so we don't conflict with Nix's --- src/hydra-queue-runner/build-remote.cc | 20 ++++++++++---------- src/hydra-queue-runner/builder.cc | 2 +- src/hydra-queue-runner/dispatcher.cc | 4 ++-- src/hydra-queue-runner/hydra-queue-runner.cc | 8 ++++---- 4 files changed, 17 insertions(+), 17 deletions(-) diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc index a4f3a35c..5cc994fc 100644 --- a/src/hydra-queue-runner/build-remote.cc +++ b/src/hydra-queue-runner/build-remote.cc @@ -48,7 +48,7 @@ static Strings extraStoreArgs(std::string & machine) return result; } -static void openConnection(Machine::ptr machine, Path tmpDir, int stderrFD, SSHMaster::Connection & child) +static void openConnection(::Machine::ptr machine, Path tmpDir, int stderrFD, SSHMaster::Connection & child) { std::string pgmName; Pipe to, from; @@ -104,7 +104,7 @@ static void openConnection(Machine::ptr machine, Path tmpDir, int stderrFD, SSHM static void copyClosureTo( - Machine::Connection & conn, + ::Machine::Connection & conn, Store & destStore, const StorePathSet & paths, SubstituteFlag useSubstitutes = NoSubstitute) @@ -195,7 +195,7 @@ static std::pair openLogFile(const std::string & logDir, cons * Therefore, no `ServeProto::Serialize` functions can be used until * that field is set. */ -static void handshake(Machine::Connection & conn, unsigned int repeats) +static void handshake(::Machine::Connection & conn, unsigned int repeats) { conn.to << SERVE_MAGIC_1 << 0x206; conn.to.flush(); @@ -216,7 +216,7 @@ static BasicDerivation sendInputs( Step & step, Store & localStore, Store & destStore, - Machine::Connection & conn, + ::Machine::Connection & conn, unsigned int & overhead, counter & nrStepsWaiting, counter & nrStepsCopyingTo @@ -272,7 +272,7 @@ static BasicDerivation sendInputs( } static BuildResult performBuild( - Machine::Connection & conn, + ::Machine::Connection & conn, Store & localStore, StorePath drvPath, const BasicDerivation & drv, @@ -317,7 +317,7 @@ static BuildResult performBuild( } static std::map queryPathInfos( - Machine::Connection & conn, + ::Machine::Connection & conn, Store & localStore, StorePathSet & outputs, size_t & totalNarSize @@ -355,7 +355,7 @@ static std::map queryPathInfos( } static void copyPathFromRemote( - Machine::Connection & conn, + ::Machine::Connection & conn, NarMemberDatas & narMembers, Store & localStore, Store & destStore, @@ -385,7 +385,7 @@ static void copyPathFromRemote( } static void copyPathsFromRemote( - Machine::Connection & conn, + ::Machine::Connection & conn, NarMemberDatas & narMembers, Store & localStore, Store & destStore, @@ -462,7 +462,7 @@ void RemoteResult::updateWithBuildResult(const nix::BuildResult & buildResult) void State::buildRemote(ref destStore, - Machine::ptr machine, Step::ptr step, + ::Machine::ptr machine, Step::ptr step, const BuildOptions & buildOptions, RemoteResult & result, std::shared_ptr activeStep, std::function updateStep, @@ -503,7 +503,7 @@ void State::buildRemote(ref destStore, process. Meh. */ }); - Machine::Connection conn { + ::Machine::Connection conn { .from = child.out.get(), .to = child.in.get(), .machine = machine, diff --git a/src/hydra-queue-runner/builder.cc b/src/hydra-queue-runner/builder.cc index 307eee8e..cb343d77 100644 --- a/src/hydra-queue-runner/builder.cc +++ b/src/hydra-queue-runner/builder.cc @@ -400,7 +400,7 @@ void State::failStep( Step::ptr step, BuildID buildId, const RemoteResult & result, - Machine::ptr machine, + ::Machine::ptr machine, bool & stepFinished) { /* Register failure in the database for all Build objects that diff --git a/src/hydra-queue-runner/dispatcher.cc b/src/hydra-queue-runner/dispatcher.cc index af21dcbd..6d738ded 100644 --- a/src/hydra-queue-runner/dispatcher.cc +++ b/src/hydra-queue-runner/dispatcher.cc @@ -199,7 +199,7 @@ system_time State::doDispatch() filter out temporarily disabled machines. */ struct MachineInfo { - Machine::ptr machine; + ::Machine::ptr machine; unsigned long currentJobs; }; std::vector machinesSorted; @@ -435,7 +435,7 @@ void Jobset::pruneSteps() } -State::MachineReservation::MachineReservation(State & state, Step::ptr step, Machine::ptr machine) +State::MachineReservation::MachineReservation(State & state, Step::ptr step, ::Machine::ptr machine) : state(state), step(step), machine(machine) { machine->state->currentJobs++; diff --git a/src/hydra-queue-runner/hydra-queue-runner.cc b/src/hydra-queue-runner/hydra-queue-runner.cc index 1d54bb93..09d6fcb2 100644 --- a/src/hydra-queue-runner/hydra-queue-runner.cc +++ b/src/hydra-queue-runner/hydra-queue-runner.cc @@ -140,7 +140,7 @@ void State::parseMachines(const std::string & contents) if (tokens.size() < 3) continue; tokens.resize(8); - auto machine = std::make_shared(); + auto machine = std::make_shared<::Machine>(); machine->sshName = tokens[0]; machine->systemTypes = tokenizeString(tokens[1], ","); machine->sshKey = tokens[2] == "-" ? std::string("") : tokens[2]; @@ -166,7 +166,7 @@ void State::parseMachines(const std::string & contents) else printMsg(lvlChatty, "updating machine ‘%1%’", machine->sshName); machine->state = i == oldMachines.end() - ? std::make_shared() + ? std::make_shared<::Machine::State>() : i->second->state; newMachines[machine->sshName] = machine; } @@ -175,9 +175,9 @@ void State::parseMachines(const std::string & contents) if (newMachines.find(m.first) == newMachines.end()) { if (m.second->enabled) printInfo("removing machine ‘%1%’", m.first); - /* Add a disabled Machine object to make sure stats are + /* Add a disabled ::Machine object to make sure stats are maintained. */ - auto machine = std::make_shared(*(m.second)); + auto machine = std::make_shared<::Machine>(*(m.second)); machine->enabled = false; newMachines[m.first] = machine; } From 70e5469303b422bdb4b123be222bdea4d7f9611c Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 23 Jan 2024 12:08:05 -0500 Subject: [PATCH 086/313] Use Nix's `Machine` type in a mimimal way This is *just* using the fields from that type, and only where the types coincide. (There are two fields with different types, `speedFactor` most interestingly.) No code is reused, so we can be sure that no behavior is changed. Once the types are reconciled on the Nix side, then we can start carefully actually reusing code. Progress on #1164 --- src/hydra-queue-runner/dispatcher.cc | 6 +-- src/hydra-queue-runner/hydra-queue-runner.cc | 53 ++++++++++++++------ src/hydra-queue-runner/state.hh | 21 +++++--- 3 files changed, 53 insertions(+), 27 deletions(-) diff --git a/src/hydra-queue-runner/dispatcher.cc b/src/hydra-queue-runner/dispatcher.cc index 6d738ded..f5d9d3e3 100644 --- a/src/hydra-queue-runner/dispatcher.cc +++ b/src/hydra-queue-runner/dispatcher.cc @@ -231,11 +231,11 @@ system_time State::doDispatch() sort(machinesSorted.begin(), machinesSorted.end(), [](const MachineInfo & a, const MachineInfo & b) -> bool { - float ta = std::round(a.currentJobs / a.machine->speedFactor); - float tb = std::round(b.currentJobs / b.machine->speedFactor); + float ta = std::round(a.currentJobs / a.machine->speedFactorFloat); + float tb = std::round(b.currentJobs / b.machine->speedFactorFloat); return ta != tb ? ta < tb : - a.machine->speedFactor != b.machine->speedFactor ? a.machine->speedFactor > b.machine->speedFactor : + a.machine->speedFactorFloat != b.machine->speedFactorFloat ? a.machine->speedFactorFloat > b.machine->speedFactorFloat : a.currentJobs > b.currentJobs; }); diff --git a/src/hydra-queue-runner/hydra-queue-runner.cc b/src/hydra-queue-runner/hydra-queue-runner.cc index 09d6fcb2..66528302 100644 --- a/src/hydra-queue-runner/hydra-queue-runner.cc +++ b/src/hydra-queue-runner/hydra-queue-runner.cc @@ -1,6 +1,7 @@ #include #include #include +#include #include #include @@ -140,23 +141,43 @@ void State::parseMachines(const std::string & contents) if (tokens.size() < 3) continue; tokens.resize(8); - auto machine = std::make_shared<::Machine>(); - machine->sshName = tokens[0]; - machine->systemTypes = tokenizeString(tokens[1], ","); - machine->sshKey = tokens[2] == "-" ? std::string("") : tokens[2]; - if (tokens[3] != "") - machine->maxJobs = string2IntmaxJobs)>(tokens[3]).value(); - else - machine->maxJobs = 1; - machine->speedFactor = atof(tokens[4].c_str()); if (tokens[5] == "-") tokens[5] = ""; - machine->supportedFeatures = tokenizeString(tokens[5], ","); + auto supportedFeatures = tokenizeString(tokens[5], ","); + if (tokens[6] == "-") tokens[6] = ""; - machine->mandatoryFeatures = tokenizeString(tokens[6], ","); - for (auto & f : machine->mandatoryFeatures) - machine->supportedFeatures.insert(f); - if (tokens[7] != "" && tokens[7] != "-") - machine->sshPublicHostKey = base64Decode(tokens[7]); + auto mandatoryFeatures = tokenizeString(tokens[6], ","); + + for (auto & f : mandatoryFeatures) + supportedFeatures.insert(f); + + using MaxJobs = std::remove_const::type; + + auto machine = std::make_shared<::Machine>(nix::Machine { + // `storeUri`, not yet used + "", + // `systemTypes`, not yet used + {}, + // `sshKey` + tokens[2] == "-" ? "" : tokens[2], + // `maxJobs` + tokens[3] != "" + ? string2Int(tokens[3]).value() + : 1, + // `speedFactor`, not yet used + 1, + // `supportedFeatures` + std::move(supportedFeatures), + // `mandatoryFeatures` + std::move(mandatoryFeatures), + // `sshPublicHostKey` + tokens[7] != "" && tokens[7] != "-" + ? base64Decode(tokens[7]) + : "", + }); + + machine->sshName = tokens[0]; + machine->systemTypesSet = tokenizeString(tokens[1], ","); + machine->speedFactorFloat = atof(tokens[4].c_str()); /* Re-use the State object of the previous machine with the same name. */ @@ -596,7 +617,7 @@ void State::dumpStatus(Connection & conn) json machine = { {"enabled", m->enabled}, - {"systemTypes", m->systemTypes}, + {"systemTypes", m->systemTypesSet}, {"supportedFeatures", m->supportedFeatures}, {"mandatoryFeatures", m->mandatoryFeatures}, {"nrStepsDone", s->nrStepsDone.load()}, diff --git a/src/hydra-queue-runner/state.hh b/src/hydra-queue-runner/state.hh index 6359063a..b08e4e37 100644 --- a/src/hydra-queue-runner/state.hh +++ b/src/hydra-queue-runner/state.hh @@ -22,6 +22,7 @@ #include "sync.hh" #include "nar-extractor.hh" #include "serve-protocol.hh" +#include "machines.hh" typedef unsigned int BuildID; @@ -234,17 +235,21 @@ void getDependents(Step::ptr step, std::set & builds, std::set visitor, Step::ptr step); -struct Machine +struct Machine : nix::Machine { typedef std::shared_ptr ptr; - bool enabled{true}; + /* TODO Get rid of: `nix::Machine::storeUri` is normalized in a way + we are not yet used to, but once we are, we don't need this. */ + std::string sshName; - std::string sshName, sshKey; - std::set systemTypes, supportedFeatures, mandatoryFeatures; - unsigned int maxJobs = 1; - float speedFactor = 1.0; - std::string sshPublicHostKey; + /* TODO Get rid once `nix::Machine::systemTypes` is a set not + vector. */ + std::set systemTypesSet; + + /* TODO Get rid once `nix::Machine::systemTypes` is a `float` not + an `int`. */ + float speedFactorFloat = 1.0; struct State { typedef std::shared_ptr ptr; @@ -272,7 +277,7 @@ struct Machine { /* Check that this machine is of the type required by the step. */ - if (!systemTypes.count(step->drv->platform == "builtin" ? nix::settings.thisSystem : step->drv->platform)) + if (!systemTypesSet.count(step->drv->platform == "builtin" ? nix::settings.thisSystem : step->drv->platform)) return false; /* Check that the step requires all mandatory features of this From 2bdbf51d7d947c44af159dab391be2a0dfb930d7 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 24 Jan 2024 18:46:56 -0500 Subject: [PATCH 087/313] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nix': 'github:NixOS/nix/b6aee9a93f6646bbffd919d362a5c75c37bb9caa' (2024-01-23) → 'github:NixOS/nix/212ba69e6f995992f8b4e4c0656d19c0156c8714' (2024-01-24) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index c78d5203..bdbc4197 100644 --- a/flake.lock +++ b/flake.lock @@ -42,11 +42,11 @@ "nixpkgs-regression": "nixpkgs-regression" }, "locked": { - "lastModified": 1706016881, - "narHash": "sha256-XZz6tfTuZcrrX27vI+jAereNVWmnCrzFhBFhaKwEdoY=", + "lastModified": 1706100941, + "narHash": "sha256-YLYxZrQLQ6i22RLZG/rsB4sIfFzhep7XQYlodzCmKy0=", "owner": "NixOS", "repo": "nix", - "rev": "b6aee9a93f6646bbffd919d362a5c75c37bb9caa", + "rev": "212ba69e6f995992f8b4e4c0656d19c0156c8714", "type": "github" }, "original": { From 449eb2d873d3240ec4f6bc3e6679eeda7a5cadfd Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 24 Jan 2024 20:14:31 -0500 Subject: [PATCH 088/313] Use more `nix::Machine` fields The upstream fields were made to match Hydra, so we can get rid of the extra fields temporary added in 70e5469303b422bdb4b123be222bdea4d7f9611c. --- src/hydra-queue-runner/dispatcher.cc | 6 +++--- src/hydra-queue-runner/hydra-queue-runner.cc | 12 +++++------- src/hydra-queue-runner/state.hh | 10 +--------- 3 files changed, 9 insertions(+), 19 deletions(-) diff --git a/src/hydra-queue-runner/dispatcher.cc b/src/hydra-queue-runner/dispatcher.cc index f5d9d3e3..6d738ded 100644 --- a/src/hydra-queue-runner/dispatcher.cc +++ b/src/hydra-queue-runner/dispatcher.cc @@ -231,11 +231,11 @@ system_time State::doDispatch() sort(machinesSorted.begin(), machinesSorted.end(), [](const MachineInfo & a, const MachineInfo & b) -> bool { - float ta = std::round(a.currentJobs / a.machine->speedFactorFloat); - float tb = std::round(b.currentJobs / b.machine->speedFactorFloat); + float ta = std::round(a.currentJobs / a.machine->speedFactor); + float tb = std::round(b.currentJobs / b.machine->speedFactor); return ta != tb ? ta < tb : - a.machine->speedFactorFloat != b.machine->speedFactorFloat ? a.machine->speedFactorFloat > b.machine->speedFactorFloat : + a.machine->speedFactor != b.machine->speedFactor ? a.machine->speedFactor > b.machine->speedFactor : a.currentJobs > b.currentJobs; }); diff --git a/src/hydra-queue-runner/hydra-queue-runner.cc b/src/hydra-queue-runner/hydra-queue-runner.cc index 5fbcb641..3d48f3d7 100644 --- a/src/hydra-queue-runner/hydra-queue-runner.cc +++ b/src/hydra-queue-runner/hydra-queue-runner.cc @@ -155,16 +155,16 @@ void State::parseMachines(const std::string & contents) auto machine = std::make_shared<::Machine>(nix::Machine { // `storeUri`, not yet used "", - // `systemTypes`, not yet used - {}, + // `systemTypes` + tokenizeString(tokens[1], ","), // `sshKey` tokens[2] == "-" ? "" : tokens[2], // `maxJobs` tokens[3] != "" ? string2Int(tokens[3]).value() : 1, - // `speedFactor`, not yet used - 1, + // `speedFactor` + atof(tokens[4].c_str()), // `supportedFeatures` std::move(supportedFeatures), // `mandatoryFeatures` @@ -176,8 +176,6 @@ void State::parseMachines(const std::string & contents) }); machine->sshName = tokens[0]; - machine->systemTypesSet = tokenizeString(tokens[1], ","); - machine->speedFactorFloat = atof(tokens[4].c_str()); /* Re-use the State object of the previous machine with the same name. */ @@ -638,7 +636,7 @@ void State::dumpStatus(Connection & conn) json machine = { {"enabled", m->enabled}, - {"systemTypes", m->systemTypesSet}, + {"systemTypes", m->systemTypes}, {"supportedFeatures", m->supportedFeatures}, {"mandatoryFeatures", m->mandatoryFeatures}, {"nrStepsDone", s->nrStepsDone.load()}, diff --git a/src/hydra-queue-runner/state.hh b/src/hydra-queue-runner/state.hh index f6d9d5a9..cda238ae 100644 --- a/src/hydra-queue-runner/state.hh +++ b/src/hydra-queue-runner/state.hh @@ -244,14 +244,6 @@ struct Machine : nix::Machine we are not yet used to, but once we are, we don't need this. */ std::string sshName; - /* TODO Get rid once `nix::Machine::systemTypes` is a set not - vector. */ - std::set systemTypesSet; - - /* TODO Get rid once `nix::Machine::systemTypes` is a `float` not - an `int`. */ - float speedFactorFloat = 1.0; - struct State { typedef std::shared_ptr ptr; counter currentJobs{0}; @@ -278,7 +270,7 @@ struct Machine : nix::Machine { /* Check that this machine is of the type required by the step. */ - if (!systemTypesSet.count(step->drv->platform == "builtin" ? nix::settings.thisSystem : step->drv->platform)) + if (!systemTypes.count(step->drv->platform == "builtin" ? nix::settings.thisSystem : step->drv->platform)) return false; /* Check that the step requires all mandatory features of this From 07cb5d1b7c74fdc8423a93b14f4b56af1c49e4b3 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 24 Jan 2024 21:04:14 -0500 Subject: [PATCH 089/313] Use `nix::ParsedDerivation::getRequiredSystemFeatures()` A slight dedup, and also ensures that floating CA derivations require a `ca-derivations` experimental feature. This fixes the scheduling issue that @SuperSandro2000 found. --- src/hydra-queue-runner/queue-monitor.cc | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/hydra-queue-runner/queue-monitor.cc b/src/hydra-queue-runner/queue-monitor.cc index a3557316..513d399b 100644 --- a/src/hydra-queue-runner/queue-monitor.cc +++ b/src/hydra-queue-runner/queue-monitor.cc @@ -462,10 +462,7 @@ Step::ptr State::createStep(ref destStore, step->systemType = step->drv->platform; { - auto i = step->drv->env.find("requiredSystemFeatures"); - StringSet features; - if (i != step->drv->env.end()) - features = step->requiredSystemFeatures = tokenizeString>(i->second); + StringSet features = step->requiredSystemFeatures = step->parsedDrv->getRequiredSystemFeatures(); if (step->preferLocalBuild) features.insert("local"); if (!features.empty()) { From b1fa6b3aac4ce8225e506317119b32122cfc1edd Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 24 Jan 2024 21:37:13 -0500 Subject: [PATCH 090/313] Use `StoreConfig::getDefaultSystemFeatures` for default machine config We have to oddly make a `StoreConfig` subclass to get it, but https://github.com/NixOS/nix/pull/9848 will fix that. The purpose of this is to ensure that, absent an explicit config, `localhost` includes `ca-derivations` and `recursive-nix` if those experimental features are enabled. Very much the complement of #1342, the previous PR. --- src/hydra-queue-runner/hydra-queue-runner.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/hydra-queue-runner/hydra-queue-runner.cc b/src/hydra-queue-runner/hydra-queue-runner.cc index 5fbcb641..1cbcafea 100644 --- a/src/hydra-queue-runner/hydra-queue-runner.cc +++ b/src/hydra-queue-runner/hydra-queue-runner.cc @@ -15,6 +15,7 @@ #include "state.hh" #include "hydra-build-result.hh" #include "store-api.hh" +#include "local-store.hh" #include "remote-store.hh" #include "globals.hh" @@ -226,7 +227,7 @@ void State::monitorMachinesFile() parseMachines("localhost " + (settings.thisSystem == "x86_64-linux" ? "x86_64-linux,i686-linux" : settings.thisSystem.get()) + " - " + std::to_string(settings.maxBuildJobs) + " 1 " - + concatStringsSep(",", settings.systemFeatures.get())); + + concatStringsSep(",", (LocalStoreConfig { {} }).getDefaultSystemFeatures())); machinesReadyLock.unlock(); return; } From 6df06b089e1a3d6af59f3ee545c2b54e43356cd2 Mon Sep 17 00:00:00 2001 From: Pierre Bourdon Date: Thu, 25 Jan 2024 09:27:46 +0100 Subject: [PATCH 091/313] web: disable Sign in with Google popup --- src/root/topbar.tt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/root/topbar.tt b/src/root/topbar.tt index 1771222d..58dd94e3 100644 --- a/src/root/topbar.tt +++ b/src/root/topbar.tt @@ -134,7 +134,7 @@ [% WRAPPER makeSubMenu title="Sign in" id="sign-in-menu" align="right" %] [% IF c.config.enable_google_login %] -
+
From a876e46894c73973d155b581283d07ef90706c01 Mon Sep 17 00:00:00 2001 From: Sandro Date: Thu, 25 Jan 2024 17:12:40 +0100 Subject: [PATCH 092/313] Remove automake, libtool Those are already part of autoreconfHook --- flake.nix | 2 -- 1 file changed, 2 deletions(-) diff --git a/flake.nix b/flake.nix index 8280e076..eabe5518 100644 --- a/flake.nix +++ b/flake.nix @@ -146,8 +146,6 @@ with final.buildPackages; [ makeWrapper autoreconfHook - automake - libtool nukeReferences pkg-config mdbook From 1471aacadc8868fff6eab06d2c330abd90ec2867 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 25 Jan 2024 11:20:27 -0500 Subject: [PATCH 093/313] Split out a `package.nix` Just like we did with Nix. --- flake.nix | 196 +---------------------------------------- package.nix | 246 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 248 insertions(+), 194 deletions(-) create mode 100644 package.nix diff --git a/flake.nix b/flake.nix index eabe5518..f7975b27 100644 --- a/flake.nix +++ b/flake.nix @@ -7,8 +7,6 @@ outputs = { self, nixpkgs, nix }: let - version = "${builtins.readFile ./version.txt}.${builtins.substring 0 8 (self.lastModifiedDate or "19700101")}.${self.shortRev or "DIRTY"}"; - systems = [ "x86_64-linux" "aarch64-linux" ]; forEachSystem = nixpkgs.lib.genAttrs systems; @@ -61,198 +59,8 @@ }; - hydra = let - inherit (final) lib stdenv; - perlDeps = final.buildEnv { - name = "hydra-perl-deps"; - paths = with final.perlPackages; lib.closePropagation - [ - AuthenSASL - CatalystActionREST - CatalystAuthenticationStoreDBIxClass - CatalystAuthenticationStoreLDAP - CatalystDevel - CatalystPluginAccessLog - CatalystPluginAuthorizationRoles - CatalystPluginCaptcha - CatalystPluginPrometheusTiny - CatalystPluginSessionStateCookie - CatalystPluginSessionStoreFastMmap - CatalystPluginStackTrace - CatalystTraitForRequestProxyBase - CatalystViewDownload - CatalystViewJSON - CatalystViewTT - CatalystXRoleApplicator - CatalystXScriptServerStarman - CryptPassphrase - CryptPassphraseArgon2 - CryptRandPasswd - DataDump - DateTime - DBDPg - DBDSQLite - DigestSHA1 - EmailMIME - EmailSender - FileLibMagic - FileSlurper - FileWhich - final.nix.perl-bindings - final.git - IOCompress - IPCRun - IPCRun3 - JSON - JSONMaybeXS - JSONXS - ListSomeUtils - LWP - LWPProtocolHttps - ModulePluggable - NetAmazonS3 - NetPrometheus - NetStatsd - PadWalker - ParallelForkManager - PerlCriticCommunity - PrometheusTinyShared - ReadonlyX - SetScalar - SQLSplitStatement - Starman - StringCompareConstantTime - SysHostnameLong - TermSizeAny - TermReadKey - Test2Harness - TestPostgreSQL - TextDiff - TextTable - UUID4Tiny - YAML - XMLSimple - ]; - }; - - in - stdenv.mkDerivation { - - name = "hydra-${version}"; - + hydra = final.callPackage ./package.nix { src = self; - - nativeBuildInputs = - with final.buildPackages; [ - makeWrapper - autoreconfHook - nukeReferences - pkg-config - mdbook - ]; - - buildInputs = - with final; [ - unzip - libpqxx - top-git - mercurial - darcs - subversion - breezy - openssl - bzip2 - libxslt - final.nix - perlDeps - perl - pixz - boost - postgresql_13 - (if lib.versionAtLeast lib.version "20.03pre" - then nlohmann_json - else nlohmann_json.override { multipleHeaders = true; }) - prometheus-cpp - ]; - - checkInputs = with final; [ - cacert - foreman - glibcLocales - libressl.nc - openldap - python3 - ]; - - hydraPath = with final; lib.makeBinPath ( - [ - subversion - openssh - final.nix - coreutils - findutils - pixz - gzip - bzip2 - xz - gnutar - unzip - git - top-git - mercurial - darcs - gnused - breezy - ] ++ lib.optionals stdenv.isLinux [ rpm dpkg cdrkit ] - ); - - OPENLDAP_ROOT = final.openldap; - - shellHook = '' - pushd $(git rev-parse --show-toplevel) >/dev/null - - PATH=$(pwd)/src/hydra-evaluator:$(pwd)/src/script:$(pwd)/src/hydra-eval-jobs:$(pwd)/src/hydra-queue-runner:$PATH - PERL5LIB=$(pwd)/src/lib:$PERL5LIB - export HYDRA_HOME="$(pwd)/src/" - mkdir -p .hydra-data - export HYDRA_DATA="$(pwd)/.hydra-data" - export HYDRA_DBI='dbi:Pg:dbname=hydra;host=localhost;port=64444' - - popd >/dev/null - ''; - - NIX_LDFLAGS = [ "-lpthread" ]; - - enableParallelBuilding = true; - - doCheck = true; - - preCheck = '' - patchShebangs . - export LOGNAME=''${LOGNAME:-foo} - # set $HOME for bzr so it can create its trace file - export HOME=$(mktemp -d) - ''; - - postInstall = '' - mkdir -p $out/nix-support - - for i in $out/bin/*; do - read -n 4 chars < $i - if [[ $chars =~ ELF ]]; then continue; fi - wrapProgram $i \ - --prefix PERL5LIB ':' $out/libexec/hydra/lib:$PERL5LIB \ - --prefix PATH ':' $out/bin:$hydraPath \ - --set HYDRA_RELEASE ${version} \ - --set HYDRA_HOME $out/libexec/hydra \ - --set NIX_RELEASE ${final.nix.name or "unknown"} - done - ''; - - dontStrip = true; - - meta.description = "Build of Hydra on ${final.stdenv.system}"; - passthru = { inherit perlDeps; inherit (final) nix; }; }; }; @@ -262,7 +70,7 @@ manual = forEachSystem (system: let pkgs = pkgsBySystem.${system}; in - pkgs.runCommand "hydra-manual-${version}" { } + pkgs.runCommand "hydra-manual-${pkgs.hydra.version}" { } '' mkdir -p $out/share cp -prvd ${pkgs.hydra}/share/doc $out/share/ diff --git a/package.nix b/package.nix new file mode 100644 index 00000000..35e29be1 --- /dev/null +++ b/package.nix @@ -0,0 +1,246 @@ +{ stdenv +, lib + +, src + +, buildEnv + +, perlPackages + +, nix +, git + +, makeWrapper +, autoreconfHook +, nukeReferences +, pkg-config +, mdbook + +, unzip +, libpqxx +, top-git +, mercurial +, darcs +, subversion +, breezy +, openssl +, bzip2 +, libxslt +, perl +, pixz +, boost +, postgresql_13 +, nlohmann_json +, prometheus-cpp + +, cacert +, foreman +, glibcLocales +, libressl +, openldap +, python3 + +, openssh +, coreutils +, findutils +, gzip +, xz +, gnutar +, gnused + +, rpm +, dpkg +, cdrkit +}: + +let + perlDeps = buildEnv { + name = "hydra-perl-deps"; + paths = with perlPackages; lib.closePropagation + [ + AuthenSASL + CatalystActionREST + CatalystAuthenticationStoreDBIxClass + CatalystAuthenticationStoreLDAP + CatalystDevel + CatalystPluginAccessLog + CatalystPluginAuthorizationRoles + CatalystPluginCaptcha + CatalystPluginPrometheusTiny + CatalystPluginSessionStateCookie + CatalystPluginSessionStoreFastMmap + CatalystPluginStackTrace + CatalystTraitForRequestProxyBase + CatalystViewDownload + CatalystViewJSON + CatalystViewTT + CatalystXRoleApplicator + CatalystXScriptServerStarman + CryptPassphrase + CryptPassphraseArgon2 + CryptRandPasswd + DataDump + DateTime + DBDPg + DBDSQLite + DigestSHA1 + EmailMIME + EmailSender + FileLibMagic + FileSlurper + FileWhich + # Not Perl + nix.perl-bindings + git + # Perl again + IOCompress + IPCRun + IPCRun3 + JSON + JSONMaybeXS + JSONXS + ListSomeUtils + LWP + LWPProtocolHttps + ModulePluggable + NetAmazonS3 + NetPrometheus + NetStatsd + PadWalker + ParallelForkManager + PerlCriticCommunity + PrometheusTinyShared + ReadonlyX + SetScalar + SQLSplitStatement + Starman + StringCompareConstantTime + SysHostnameLong + TermSizeAny + TermReadKey + Test2Harness + TestPostgreSQL + TextDiff + TextTable + UUID4Tiny + YAML + XMLSimple + ]; + }; + + version = "${builtins.readFile ./version.txt}.${builtins.substring 0 8 (src.lastModifiedDate or "19700101")}.${src.shortRev or "DIRTY"}"; +in +stdenv.mkDerivation { + pname = "hydra"; + inherit version; + + inherit src; + + nativeBuildInputs = [ + makeWrapper + autoreconfHook + nukeReferences + pkg-config + mdbook + ]; + + buildInputs = [ + unzip + libpqxx + top-git + mercurial + darcs + subversion + breezy + openssl + bzip2 + libxslt + nix + perlDeps + perl + pixz + boost + postgresql_13 + nlohmann_json + prometheus-cpp + ]; + + checkInputs = [ + cacert + foreman + glibcLocales + libressl.nc + openldap + python3 + ]; + + hydraPath = lib.makeBinPath ( + [ + subversion + openssh + nix + coreutils + findutils + pixz + gzip + bzip2 + xz + gnutar + unzip + git + top-git + mercurial + darcs + gnused + breezy + ] ++ lib.optionals stdenv.isLinux [ rpm dpkg cdrkit ] + ); + + OPENLDAP_ROOT = openldap; + + shellHook = '' + pushd $(git rev-parse --show-toplevel) >/dev/null + + PATH=$(pwd)/src/hydra-evaluator:$(pwd)/src/script:$(pwd)/src/hydra-eval-jobs:$(pwd)/src/hydra-queue-runner:$PATH + PERL5LIB=$(pwd)/src/lib:$PERL5LIB + export HYDRA_HOME="$(pwd)/src/" + mkdir -p .hydra-data + export HYDRA_DATA="$(pwd)/.hydra-data" + export HYDRA_DBI='dbi:Pg:dbname=hydra;host=localhost;port=64444' + + popd >/dev/null + ''; + + NIX_LDFLAGS = [ "-lpthread" ]; + + enableParallelBuilding = true; + + doCheck = true; + + preCheck = '' + patchShebangs . + export LOGNAME=''${LOGNAME:-foo} + # set $HOME for bzr so it can create its trace file + export HOME=$(mktemp -d) + ''; + + postInstall = '' + mkdir -p $out/nix-support + + for i in $out/bin/*; do + read -n 4 chars < $i + if [[ $chars =~ ELF ]]; then continue; fi + wrapProgram $i \ + --prefix PERL5LIB ':' $out/libexec/hydra/lib:$PERL5LIB \ + --prefix PATH ':' $out/bin:$hydraPath \ + --set HYDRA_RELEASE ${version} \ + --set HYDRA_HOME $out/libexec/hydra \ + --set NIX_RELEASE ${nix.name or "unknown"} + done + ''; + + dontStrip = true; + + meta.description = "Build of Hydra on ${stdenv.system}"; + passthru = { inherit perlDeps nix; }; +} From 1bd195a5138b3c69c52110d713c706cb4908ba16 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 25 Jan 2024 11:32:00 -0500 Subject: [PATCH 094/313] Clean up deps - `strictDeps` - Ensure it builds with and without `doCheck` --- flake.nix | 6 ++++++ package.nix | 28 ++++++++++++++++++---------- 2 files changed, 24 insertions(+), 10 deletions(-) diff --git a/flake.nix b/flake.nix index f7975b27..a6b62cfa 100644 --- a/flake.nix +++ b/flake.nix @@ -68,6 +68,12 @@ build = forEachSystem (system: packages.${system}.hydra); + buildNoTests = forEachSystem (system: + packages.${system}.hydra.overrideAttrs (_: { + doCheck = false; + }) + ); + manual = forEachSystem (system: let pkgs = pkgsBySystem.${system}; in pkgs.runCommand "hydra-manual-${pkgs.hydra.version}" { } diff --git a/package.nix b/package.nix index 35e29be1..f91e7a24 100644 --- a/package.nix +++ b/package.nix @@ -136,41 +136,49 @@ stdenv.mkDerivation { inherit src; + strictDeps = true; + nativeBuildInputs = [ makeWrapper autoreconfHook nukeReferences pkg-config mdbook + nix + perlDeps + perl + unzip ]; buildInputs = [ - unzip libpqxx - top-git - mercurial - darcs - subversion - breezy openssl - bzip2 libxslt nix perlDeps perl - pixz boost - postgresql_13 nlohmann_json prometheus-cpp ]; + nativeCheckInputs = [ + bzip2 + darcs + top-git + mercurial + subversion + breezy + openldap + postgresql_13 + pixz + ]; + checkInputs = [ cacert foreman glibcLocales libressl.nc - openldap python3 ]; From d6d6d1b649ea3b83baf44267637e42c4eb1c92c0 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 25 Jan 2024 12:03:15 -0500 Subject: [PATCH 095/313] flake.nix: Temporarily add a second Nixpkgs for `lib.fileset` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nix': 'github:NixOS/nix/b38e5a665e9d0aa7975beb0ed12e42d13a392e74' (2023-12-13) → 'github:NixOS/nix/03e96b9dc011a16a0f6db9c7cb021ff93f8dcf88' (2024-01-19) • Added input 'nixpkgs-for-fileset': 'github:NixOS/nixpkgs/a77ab169a83a4175169d78684ddd2e54486ac651' (2024-01-24) --- flake.lock | 25 +++++++++++++++++++++---- flake.nix | 5 +++++ 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/flake.lock b/flake.lock index 66e30323..8231484c 100644 --- a/flake.lock +++ b/flake.lock @@ -42,11 +42,11 @@ "nixpkgs-regression": "nixpkgs-regression" }, "locked": { - "lastModified": 1702510710, - "narHash": "sha256-9K+w1mQgmUxCmEsPaSFkpYsj/cxjO2PSwTCPkNZ/NiU=", + "lastModified": 1705666051, + "narHash": "sha256-C3eht7uA7gZp9O5XpA9Mkqdi1WoTQaFH4FMGmM4DrFM=", "owner": "NixOS", "repo": "nix", - "rev": "b38e5a665e9d0aa7975beb0ed12e42d13a392e74", + "rev": "03e96b9dc011a16a0f6db9c7cb021ff93f8dcf88", "type": "github" }, "original": { @@ -72,6 +72,22 @@ "type": "github" } }, + "nixpkgs-for-fileset": { + "locked": { + "lastModified": 1706098335, + "narHash": "sha256-r3dWjT8P9/Ah5m5ul4WqIWD8muj5F+/gbCdjiNVBKmU=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "a77ab169a83a4175169d78684ddd2e54486ac651", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-23.11", + "repo": "nixpkgs", + "type": "github" + } + }, "nixpkgs-regression": { "locked": { "lastModified": 1643052045, @@ -91,7 +107,8 @@ "root": { "inputs": { "nix": "nix", - "nixpkgs": "nixpkgs" + "nixpkgs": "nixpkgs", + "nixpkgs-for-fileset": "nixpkgs-for-fileset" } } }, diff --git a/flake.nix b/flake.nix index a6b62cfa..0ab25018 100644 --- a/flake.nix +++ b/flake.nix @@ -5,6 +5,11 @@ inputs.nix.url = "github:NixOS/nix/2.19-maintenance"; inputs.nix.inputs.nixpkgs.follows = "nixpkgs"; + # TODO get rid of this once https://github.com/NixOS/nix/pull/9546 is + # mered and we upgrade or Nix, so the main `nixpkgs` input is at least + # 23.11 and has `lib.fileset`. + inputs.nixpkgs-for-fileset.url = "github:NixOS/nixpkgs/nixos-23.11"; + outputs = { self, nixpkgs, nix }: let systems = [ "x86_64-linux" "aarch64-linux" ]; From 4bbc7b8f75409aa8e99a0f3284bdc5f11f804cfb Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 25 Jan 2024 12:21:13 -0500 Subject: [PATCH 096/313] Use the Nixpkgs `fileset` library to filter source Now I can change Nix files without causing rebuilds. --- flake.nix | 5 +++-- package.nix | 26 +++++++++++++++++++++----- 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/flake.nix b/flake.nix index 0ab25018..71fd6c6d 100644 --- a/flake.nix +++ b/flake.nix @@ -10,7 +10,7 @@ # 23.11 and has `lib.fileset`. inputs.nixpkgs-for-fileset.url = "github:NixOS/nixpkgs/nixos-23.11"; - outputs = { self, nixpkgs, nix }: + outputs = { self, nixpkgs, nix, nixpkgs-for-fileset }: let systems = [ "x86_64-linux" "aarch64-linux" ]; forEachSystem = nixpkgs.lib.genAttrs systems; @@ -65,7 +65,8 @@ }; hydra = final.callPackage ./package.nix { - src = self; + inherit (nixpkgs-for-fileset.lib) fileset; + rawSrc = self; }; }; diff --git a/package.nix b/package.nix index f91e7a24..eff9ed37 100644 --- a/package.nix +++ b/package.nix @@ -1,7 +1,8 @@ { stdenv , lib +, fileset -, src +, rawSrc , buildEnv @@ -128,13 +129,28 @@ let ]; }; - version = "${builtins.readFile ./version.txt}.${builtins.substring 0 8 (src.lastModifiedDate or "19700101")}.${src.shortRev or "DIRTY"}"; + version = "${builtins.readFile ./version.txt}.${builtins.substring 0 8 (rawSrc.lastModifiedDate or "19700101")}.${rawSrc.shortRev or "DIRTY"}"; in -stdenv.mkDerivation { +stdenv.mkDerivation (finalAttrs: { pname = "hydra"; inherit version; - inherit src; + src = fileset.toSource { + root = ./.; + fileset = fileset.unions ([ + ./version.txt + ./configure.ac + ./Makefile.am + ./src + ./doc + ./hydra-module.nix + # TODO only when `doCheck` + ./t + ] ++ lib.optionals finalAttrs.doCheck [ + ./.perlcriticrc + ./.yath.rc + ]); + }; strictDeps = true; @@ -251,4 +267,4 @@ stdenv.mkDerivation { meta.description = "Build of Hydra on ${stdenv.system}"; passthru = { inherit perlDeps nix; }; -} +}) From 73b6c1fb11b7c5679725599c552f4e163bf668f0 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 25 Jan 2024 13:27:05 -0500 Subject: [PATCH 097/313] Filter out (mosts test) when `!doCheck` --- Makefile.am | 6 +++++- configure.ac | 20 ++++++++++++++------ package.nix | 7 +++++-- 3 files changed, 24 insertions(+), 9 deletions(-) diff --git a/Makefile.am b/Makefile.am index e744cc33..9e06ebdb 100644 --- a/Makefile.am +++ b/Makefile.am @@ -1,4 +1,8 @@ -SUBDIRS = src t doc +SUBDIRS = src doc +if CAN_DO_CHECK + SUBDIRS += t +endif + BOOTCLEAN_SUBDIRS = $(SUBDIRS) DIST_SUBDIRS = $(SUBDIRS) EXTRA_DIST = hydra-module.nix diff --git a/configure.ac b/configure.ac index eec647c3..e5c57d14 100644 --- a/configure.ac +++ b/configure.ac @@ -53,9 +53,6 @@ PKG_CHECK_MODULES([NIX], [nix-main nix-expr nix-store]) testPath="$(dirname $(type -p expr))" AC_SUBST(testPath) -jobsPath="$(realpath ./t/jobs)" -AC_SUBST(jobsPath) - CXXFLAGS+=" -include nix/config.h" AC_CONFIG_FILES([ @@ -71,11 +68,22 @@ AC_CONFIG_FILES([ src/lib/Makefile src/root/Makefile src/script/Makefile - t/Makefile - t/jobs/config.nix - t/jobs/declarative/project.json ]) +# Tests might be filtered out +AM_CONDITIONAL([CAN_DO_CHECK], [test -f "$srcdir/t/api-test.t"]) +AM_COND_IF( + [CAN_DO_CHECK], + [ + jobsPath="$(realpath ./t/jobs)" + AC_SUBST(jobsPath) + AC_CONFIG_FILES([ + t/Makefile + t/jobs/config.nix + t/jobs/declarative/project.json + ]) + ]) + AC_CONFIG_COMMANDS([executable-scripts], []) AC_CONFIG_HEADER([hydra-config.h]) diff --git a/package.nix b/package.nix index eff9ed37..73ef9b3f 100644 --- a/package.nix +++ b/package.nix @@ -144,9 +144,12 @@ stdenv.mkDerivation (finalAttrs: { ./src ./doc ./hydra-module.nix - # TODO only when `doCheck` - ./t + # These are always needed to appease Automake + ./t/Makefile.am + ./t/jobs/config.nix.in + ./t/jobs/declarative/project.json.in ] ++ lib.optionals finalAttrs.doCheck [ + ./t ./.perlcriticrc ./.yath.rc ]); From c5f37eca91bee5bacbe53f826abcd0c0d0f38ef1 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 25 Jan 2024 11:54:44 -0500 Subject: [PATCH 098/313] Reorganize hydra modules --- .gitignore | 1 + Makefile.am | 6 +-- flake.nix | 50 +++------------------ nixos-modules/default.nix | 49 ++++++++++++++++++++ hydra-module.nix => nixos-modules/hydra.nix | 0 package.nix | 2 +- 6 files changed, 59 insertions(+), 49 deletions(-) create mode 100644 nixos-modules/default.nix rename hydra-module.nix => nixos-modules/hydra.nix (100%) diff --git a/.gitignore b/.gitignore index 799db665..f8bf5718 100644 --- a/.gitignore +++ b/.gitignore @@ -38,6 +38,7 @@ t/jobs/declarative/project.json hydra-config.h hydra-config.h.in result +result-* outputs config stamp-h1 diff --git a/Makefile.am b/Makefile.am index 9e06ebdb..a28e3f33 100644 --- a/Makefile.am +++ b/Makefile.am @@ -5,8 +5,8 @@ endif BOOTCLEAN_SUBDIRS = $(SUBDIRS) DIST_SUBDIRS = $(SUBDIRS) -EXTRA_DIST = hydra-module.nix +EXTRA_DIST = nixos-modules/hydra.nix -install-data-local: hydra-module.nix +install-data-local: nixos-modules/hydra.nix $(INSTALL) -d $(DESTDIR)$(datadir)/nix - $(INSTALL_DATA) hydra-module.nix $(DESTDIR)$(datadir)/nix/ + $(INSTALL_DATA) nixos-modules/hydra.nix $(DESTDIR)$(datadir)/nix/hydra-module.nix diff --git a/flake.nix b/flake.nix index 71fd6c6d..30aa61c0 100644 --- a/flake.nix +++ b/flake.nix @@ -15,9 +15,11 @@ systems = [ "x86_64-linux" "aarch64-linux" ]; forEachSystem = nixpkgs.lib.genAttrs systems; + overlayList = [ self.overlays.default nix.overlays.default ]; + pkgsBySystem = forEachSystem (system: import nixpkgs { inherit system; - overlays = [ self.overlays.default nix.overlays.default ]; + overlays = overlayList; }); # NixOS configuration used for VM tests. @@ -386,50 +388,8 @@ default = pkgsBySystem.${system}.hydra; }); - nixosModules.hydra = { - imports = [ ./hydra-module.nix ]; - nixpkgs.overlays = [ self.overlays.default nix.overlays.default ]; - }; - - nixosModules.hydraTest = { pkgs, ... }: { - imports = [ self.nixosModules.hydra ]; - - services.hydra-dev.enable = true; - services.hydra-dev.hydraURL = "http://hydra.example.org"; - services.hydra-dev.notificationSender = "admin@hydra.example.org"; - - systemd.services.hydra-send-stats.enable = false; - - services.postgresql.enable = true; - services.postgresql.package = pkgs.postgresql_11; - - # The following is to work around the following error from hydra-server: - # [error] Caught exception in engine "Cannot determine local time zone" - time.timeZone = "UTC"; - - nix.extraOptions = '' - allowed-uris = https://github.com/ - ''; - }; - - nixosModules.hydraProxy = { - services.httpd = { - enable = true; - adminAddr = "hydra-admin@example.org"; - extraConfig = '' - - Order deny,allow - Allow from all - - - ProxyRequests Off - ProxyPreserveHost On - ProxyPass /apache-errors ! - ErrorDocument 503 /apache-errors/503.html - ProxyPass / http://127.0.0.1:3000/ retry=5 disablereuse=on - ProxyPassReverse / http://127.0.0.1:3000/ - ''; - }; + nixosModules = import ./nixos-modules { + overlays = overlayList; }; nixosConfigurations.container = nixpkgs.lib.nixosSystem { diff --git a/nixos-modules/default.nix b/nixos-modules/default.nix new file mode 100644 index 00000000..6fc19d31 --- /dev/null +++ b/nixos-modules/default.nix @@ -0,0 +1,49 @@ +{ overlays }: + +rec { + hydra = { + imports = [ ./hydra.nix ]; + nixpkgs = { inherit overlays; }; + }; + + hydraTest = { pkgs, ... }: { + imports = [ hydra ]; + + services.hydra-dev.enable = true; + services.hydra-dev.hydraURL = "http://hydra.example.org"; + services.hydra-dev.notificationSender = "admin@hydra.example.org"; + + systemd.services.hydra-send-stats.enable = false; + + services.postgresql.enable = true; + services.postgresql.package = pkgs.postgresql_11; + + # The following is to work around the following error from hydra-server: + # [error] Caught exception in engine "Cannot determine local time zone" + time.timeZone = "UTC"; + + nix.extraOptions = '' + allowed-uris = https://github.com/ + ''; + }; + + hydraProxy = { + services.httpd = { + enable = true; + adminAddr = "hydra-admin@example.org"; + extraConfig = '' + + Order deny,allow + Allow from all + + + ProxyRequests Off + ProxyPreserveHost On + ProxyPass /apache-errors ! + ErrorDocument 503 /apache-errors/503.html + ProxyPass / http://127.0.0.1:3000/ retry=5 disablereuse=on + ProxyPassReverse / http://127.0.0.1:3000/ + ''; + }; + }; +} diff --git a/hydra-module.nix b/nixos-modules/hydra.nix similarity index 100% rename from hydra-module.nix rename to nixos-modules/hydra.nix diff --git a/package.nix b/package.nix index 73ef9b3f..d5e6bb69 100644 --- a/package.nix +++ b/package.nix @@ -143,7 +143,7 @@ stdenv.mkDerivation (finalAttrs: { ./Makefile.am ./src ./doc - ./hydra-module.nix + ./nixos-modules/hydra.nix # These are always needed to appease Automake ./t/Makefile.am ./t/jobs/config.nix.in From b5ed0787f7eba4f0a7d900736976e335f443c19e Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 25 Jan 2024 14:25:12 -0500 Subject: [PATCH 099/313] Replace "not Perl" and "Perl again" with something more self-explanatory --- package.nix | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/package.nix b/package.nix index d5e6bb69..0f8f004d 100644 --- a/package.nix +++ b/package.nix @@ -57,8 +57,11 @@ let perlDeps = buildEnv { name = "hydra-perl-deps"; - paths = with perlPackages; lib.closePropagation - [ + paths = lib.closePropagation + ([ + nix.perl-bindings + git + ] ++ (with perlPackages; [ AuthenSASL CatalystActionREST CatalystAuthenticationStoreDBIxClass @@ -90,10 +93,6 @@ let FileLibMagic FileSlurper FileWhich - # Not Perl - nix.perl-bindings - git - # Perl again IOCompress IPCRun IPCRun3 @@ -126,7 +125,7 @@ let UUID4Tiny YAML XMLSimple - ]; + ])); }; version = "${builtins.readFile ./version.txt}.${builtins.substring 0 8 (rawSrc.lastModifiedDate or "19700101")}.${rawSrc.shortRev or "DIRTY"}"; From aed130cd17ae7da41d9f38f830af3826a85a5dc6 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 25 Jan 2024 15:57:39 -0500 Subject: [PATCH 100/313] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nix': 'github:NixOS/nix/03e96b9dc011a16a0f6db9c7cb021ff93f8dcf88' (2024-01-19) → 'github:NixOS/nix/2c4bb93ba5a97e7078896ebc36385ce172960e4e' (2024-01-25) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 8231484c..2ddc7c16 100644 --- a/flake.lock +++ b/flake.lock @@ -42,11 +42,11 @@ "nixpkgs-regression": "nixpkgs-regression" }, "locked": { - "lastModified": 1705666051, - "narHash": "sha256-C3eht7uA7gZp9O5XpA9Mkqdi1WoTQaFH4FMGmM4DrFM=", + "lastModified": 1706208340, + "narHash": "sha256-wNyHUEIiKKVs6UXrUzhP7RSJQv0A8jckgcuylzftl8k=", "owner": "NixOS", "repo": "nix", - "rev": "03e96b9dc011a16a0f6db9c7cb021ff93f8dcf88", + "rev": "2c4bb93ba5a97e7078896ebc36385ce172960e4e", "type": "github" }, "original": { From c64eed7d07ad432fa9c5ea979d1039a5207f4629 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 25 Jan 2024 14:48:53 -0500 Subject: [PATCH 101/313] Simplify `StoreConfig::getDefaultSystemFeatures` call That method is now static. --- src/hydra-queue-runner/hydra-queue-runner.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/hydra-queue-runner/hydra-queue-runner.cc b/src/hydra-queue-runner/hydra-queue-runner.cc index 1cbcafea..df79e1dc 100644 --- a/src/hydra-queue-runner/hydra-queue-runner.cc +++ b/src/hydra-queue-runner/hydra-queue-runner.cc @@ -15,7 +15,6 @@ #include "state.hh" #include "hydra-build-result.hh" #include "store-api.hh" -#include "local-store.hh" #include "remote-store.hh" #include "globals.hh" @@ -227,7 +226,7 @@ void State::monitorMachinesFile() parseMachines("localhost " + (settings.thisSystem == "x86_64-linux" ? "x86_64-linux,i686-linux" : settings.thisSystem.get()) + " - " + std::to_string(settings.maxBuildJobs) + " 1 " - + concatStringsSep(",", (LocalStoreConfig { {} }).getDefaultSystemFeatures())); + + concatStringsSep(",", StoreConfig::getDefaultSystemFeatures())); machinesReadyLock.unlock(); return; } From fcde5908d8e51f975b883329b34d24a9f30ea4b3 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 25 Jan 2024 21:32:22 -0500 Subject: [PATCH 102/313] More CA derivations prep Again, with care not to change the schema in any way. --- src/hydra-eval-jobs/hydra-eval-jobs.cc | 6 +----- src/hydra-queue-runner/hydra-queue-runner.cc | 5 ++++- src/hydra-queue-runner/queue-monitor.cc | 20 ++++++++++++-------- src/lib/Hydra/Controller/Build.pm | 6 ++++-- src/script/hydra-eval-jobset | 6 +++++- t/lib/HydraTestContext.pm | 6 +++++- 6 files changed, 31 insertions(+), 18 deletions(-) diff --git a/src/hydra-eval-jobs/hydra-eval-jobs.cc b/src/hydra-eval-jobs/hydra-eval-jobs.cc index 2794cc62..5cfc3a52 100644 --- a/src/hydra-eval-jobs/hydra-eval-jobs.cc +++ b/src/hydra-eval-jobs/hydra-eval-jobs.cc @@ -250,11 +250,7 @@ static void worker( // See the `queryOutputs` call above; we should // not encounter missing output paths otherwise. assert(experimentalFeatureSettings.isEnabled(Xp::CaDerivations)); - // TODO it would be better to set `null` than an - // empty string here, to force the consumer of - // this JSON to more explicitly handle this - // case. - out[outputName] = ""; + out[outputName] = nullptr; } } job["outputs"] = std::move(out); diff --git a/src/hydra-queue-runner/hydra-queue-runner.cc b/src/hydra-queue-runner/hydra-queue-runner.cc index df79e1dc..0f4e759c 100644 --- a/src/hydra-queue-runner/hydra-queue-runner.cc +++ b/src/hydra-queue-runner/hydra-queue-runner.cc @@ -336,7 +336,10 @@ unsigned int State::createBuildStep(pqxx::work & txn, time_t startTime, BuildID for (auto & [name, output] : getDestStore()->queryPartialDerivationOutputMap(step->drvPath, &*localStore)) txn.exec_params0 ("insert into BuildStepOutputs (build, stepnr, name, path) values ($1, $2, $3, $4)", - buildId, stepNr, name, output ? localStore->printStorePath(*output) : ""); + buildId, stepNr, name, + output + ? std::optional { localStore->printStorePath(*output)} + : std::nullopt); if (status == bsBusy) txn.exec(fmt("notify step_started, '%d\t%d'", buildId, stepNr)); diff --git a/src/hydra-queue-runner/queue-monitor.cc b/src/hydra-queue-runner/queue-monitor.cc index 513d399b..d7214c43 100644 --- a/src/hydra-queue-runner/queue-monitor.cc +++ b/src/hydra-queue-runner/queue-monitor.cc @@ -193,14 +193,18 @@ bool State::getQueuedBuilds(Connection & conn, if (!propagatedFrom) { for (auto & [outputName, optOutputPath] : destStore->queryPartialDerivationOutputMap(ex.step->drvPath, &*localStore)) { - // ca-derivations not actually supported yet - assert(optOutputPath); - auto res = txn.exec_params - ("select max(s.build) from BuildSteps s join BuildStepOutputs o on s.build = o.build where path = $1 and startTime != 0 and stopTime != 0 and status = 1", - localStore->printStorePath(*optOutputPath)); - if (!res[0][0].is_null()) { - propagatedFrom = res[0][0].as(); - break; + constexpr std::string_view common = "select max(s.build) from BuildSteps s join BuildStepOutputs o on s.build = o.build where startTime != 0 and stopTime != 0 and status = 1"; + auto res = optOutputPath + ? txn.exec_params( + std::string { common } + " and path = $1", + localStore->printStorePath(*optOutputPath)) + : txn.exec_params( + std::string { common } + " and drvPath = $1 and name = $2", + localStore->printStorePath(ex.step->drvPath), + outputName); + if (!res[0][0].is_null()) { + propagatedFrom = res[0][0].as(); + break; } } } diff --git a/src/lib/Hydra/Controller/Build.pm b/src/lib/Hydra/Controller/Build.pm index 2d74f86a..c3869838 100644 --- a/src/lib/Hydra/Controller/Build.pm +++ b/src/lib/Hydra/Controller/Build.pm @@ -78,14 +78,16 @@ sub build_GET { $c->stash->{template} = 'build.tt'; $c->stash->{isLocalStore} = isLocalStore(); + # XXX: If the derivation is content-addressed then this will always return + # false because `$_->path` will be empty $c->stash->{available} = $c->stash->{isLocalStore} - ? all { isValidPath($_->path) } $build->buildoutputs->all + ? all { $_->path && isValidPath($_->path) } $build->buildoutputs->all : 1; $c->stash->{drvAvailable} = isValidPath $build->drvpath; if ($build->finished && $build->iscachedbuild) { - my $path = ($build->buildoutputs)[0]->path or die; + my $path = ($build->buildoutputs)[0]->path or undef; my $cachedBuildStep = findBuildStepByOutPath($self, $c, $path); if (defined $cachedBuildStep) { $c->stash->{cachedBuild} = $cachedBuildStep->build; diff --git a/src/script/hydra-eval-jobset b/src/script/hydra-eval-jobset index c6f6c275..7ed7ebe8 100755 --- a/src/script/hydra-eval-jobset +++ b/src/script/hydra-eval-jobset @@ -438,13 +438,17 @@ sub checkBuild { # new build to be scheduled if the meta.maintainers field is # changed? if (defined $prevEval) { + my $pathOrDrvConstraint = defined $firstOutputPath + ? { path => $firstOutputPath } + : { drvPath => $drvPath }; + my ($prevBuild) = $prevEval->builds->search( # The "project" and "jobset" constraints are # semantically unnecessary (because they're implied by # the eval), but they give a factor 1000 speedup on # the Nixpkgs jobset with PostgreSQL. { jobset_id => $jobset->get_column('id'), job => $jobName, - name => $firstOutputName, path => $firstOutputPath }, + name => $firstOutputName, %$pathOrDrvConstraint }, { rows => 1, columns => ['id', 'finished'], join => ['buildoutputs'] }); if (defined $prevBuild) { #print STDERR " already scheduled/built as build ", $prevBuild->id, "\n"; diff --git a/t/lib/HydraTestContext.pm b/t/lib/HydraTestContext.pm index a22c3df1..e1a5b226 100644 --- a/t/lib/HydraTestContext.pm +++ b/t/lib/HydraTestContext.pm @@ -39,6 +39,8 @@ use Hydra::Helper::Exec; sub new { my ($class, %opts) = @_; + my $deststoredir; + # Cleanup will be managed by yath. By the default it will be cleaned # up, but can be kept to aid in debugging test failures. my $dir = File::Temp->newdir(CLEANUP => 0); @@ -55,6 +57,7 @@ sub new { my $hydra_config = $opts{'hydra_config'} || ""; $hydra_config = "queue_runner_metrics_address = 127.0.0.1:0\n" . $hydra_config; if ($opts{'use_external_destination_store'} // 1) { + $deststoredir = "$dir/nix/dest-store"; $hydra_config = "store_uri = file://$dir/nix/dest-store\n" . $hydra_config; } @@ -81,7 +84,8 @@ sub new { nix_state_dir => $nix_state_dir, nix_log_dir => $nix_log_dir, testdir => abs_path(dirname(__FILE__) . "/.."), - jobsdir => abs_path(dirname(__FILE__) . "/../jobs") + jobsdir => abs_path(dirname(__FILE__) . "/../jobs"), + deststoredir => $deststoredir, }, $class; if ($opts{'before_init'}) { From 323b556dc8c348f1f9d5bf5b8a35f2617bfe600c Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 25 Jan 2024 21:03:08 -0500 Subject: [PATCH 103/313] Minimal CA support This verison has a worse UI, but also chnages the schema less: One non-null constraint is removed, but no new columns are added. Co-Authored-By: Andrea Ciceri Co-Authored-By: regnat --- doc/manual/src/projects.md | 7 +++ src/lib/Hydra/Controller/Root.pm | 29 +++++++++ src/lib/Hydra/Schema/Result/BuildOutputs.pm | 8 +-- .../Hydra/Schema/Result/BuildStepOutputs.pm | 8 +-- src/sql/hydra.sql | 4 +- t/content-addressed/basic.t | 61 +++++++++++++++++++ .../without-experimental-feature.t | 28 +++++++++ t/jobs/config.nix.in | 5 ++ t/jobs/content-addressed.nix | 35 +++++++++++ t/jobs/dir-with-file-builder.sh | 7 +++ t/queue-runner/notifications.t | 2 +- 11 files changed, 183 insertions(+), 11 deletions(-) create mode 100644 t/content-addressed/basic.t create mode 100644 t/content-addressed/without-experimental-feature.t create mode 100644 t/jobs/content-addressed.nix create mode 100755 t/jobs/dir-with-file-builder.sh diff --git a/doc/manual/src/projects.md b/doc/manual/src/projects.md index a399406d..f7c4975f 100644 --- a/doc/manual/src/projects.md +++ b/doc/manual/src/projects.md @@ -404,3 +404,10 @@ analogous: | `String value` | `gitea_status_repo` | *Name of the `Git checkout` input* | | `String value` | `gitea_http_url` | *Public URL of `gitea`*, optional | +Content-addressed derivations +----------------------------- + +Hydra can to a certain extent use the [`ca-derivations` experimental Nix feature](https://github.com/NixOS/rfcs/pull/62). +To use it, make sure that the Nix version you use is at least as recent as the one used in hydra's flake. + +Be warned that this support is still highly experimental, and anything beyond the basic functionality might be broken at that point. diff --git a/src/lib/Hydra/Controller/Root.pm b/src/lib/Hydra/Controller/Root.pm index c6843d29..548cfac3 100644 --- a/src/lib/Hydra/Controller/Root.pm +++ b/src/lib/Hydra/Controller/Root.pm @@ -18,6 +18,8 @@ use Net::Prometheus; use Types::Standard qw/StrMatch/; use constant NARINFO_REGEX => qr{^([a-z0-9]{32})\.narinfo$}; +# e.g.: https://hydra.example.com/realisations/sha256:a62128132508a3a32eef651d6467695944763602f226ac630543e947d9feb140!out.doi +use constant REALISATIONS_REGEX => qr{^(sha256:[a-z0-9]{64}![a-z]+)\.doi$}; # Put this controller at top-level. __PACKAGE__->config->{namespace} = ''; @@ -355,6 +357,33 @@ sub nix_cache_info :Path('nix-cache-info') :Args(0) { } +sub realisations :Path('realisations') :Args(StrMatch[REALISATIONS_REGEX]) { + my ($self, $c, $realisation) = @_; + + if (!isLocalStore) { + notFound($c, "There is no binary cache here."); + } + + else { + my ($rawDrvOutput) = $realisation =~ REALISATIONS_REGEX; + my $rawRealisation = queryRawRealisation($rawDrvOutput); + + if (!$rawRealisation) { + $c->response->status(404); + $c->response->content_type('text/plain'); + $c->stash->{plain}->{data} = "does not exist\n"; + $c->forward('Hydra::View::Plain'); + setCacheHeaders($c, 60 * 60); + return; + } + + $c->response->content_type('text/plain'); + $c->stash->{plain}->{data} = $rawRealisation; + $c->forward('Hydra::View::Plain'); + } +} + + sub narinfo :Path :Args(StrMatch[NARINFO_REGEX]) { my ($self, $c, $narinfo) = @_; diff --git a/src/lib/Hydra/Schema/Result/BuildOutputs.pm b/src/lib/Hydra/Schema/Result/BuildOutputs.pm index 9fc4f7c7..3997b497 100644 --- a/src/lib/Hydra/Schema/Result/BuildOutputs.pm +++ b/src/lib/Hydra/Schema/Result/BuildOutputs.pm @@ -49,7 +49,7 @@ __PACKAGE__->table("buildoutputs"); =head2 path data_type: 'text' - is_nullable: 0 + is_nullable: 1 =cut @@ -59,7 +59,7 @@ __PACKAGE__->add_columns( "name", { data_type => "text", is_nullable => 0 }, "path", - { data_type => "text", is_nullable => 0 }, + { data_type => "text", is_nullable => 1 }, ); =head1 PRIMARY KEY @@ -94,8 +94,8 @@ __PACKAGE__->belongs_to( ); -# Created by DBIx::Class::Schema::Loader v0.07049 @ 2021-08-26 12:02:36 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:gU+kZ6A0ISKpaXGRGve8mg +# Created by DBIx::Class::Schema::Loader v0.07049 @ 2022-06-30 12:02:32 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:Jsabm3YTcI7YvCuNdKP5Ng my %hint = ( columns => [ diff --git a/src/lib/Hydra/Schema/Result/BuildStepOutputs.pm b/src/lib/Hydra/Schema/Result/BuildStepOutputs.pm index 016a35fe..6d997a8c 100644 --- a/src/lib/Hydra/Schema/Result/BuildStepOutputs.pm +++ b/src/lib/Hydra/Schema/Result/BuildStepOutputs.pm @@ -55,7 +55,7 @@ __PACKAGE__->table("buildstepoutputs"); =head2 path data_type: 'text' - is_nullable: 0 + is_nullable: 1 =cut @@ -67,7 +67,7 @@ __PACKAGE__->add_columns( "name", { data_type => "text", is_nullable => 0 }, "path", - { data_type => "text", is_nullable => 0 }, + { data_type => "text", is_nullable => 1 }, ); =head1 PRIMARY KEY @@ -119,8 +119,8 @@ __PACKAGE__->belongs_to( ); -# Created by DBIx::Class::Schema::Loader v0.07049 @ 2021-08-26 12:02:36 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:gxp8rOjpRVen4YbIjomHTw +# Created by DBIx::Class::Schema::Loader v0.07049 @ 2022-06-30 12:02:32 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:Bad70CRTt7zb2GGuRoQ++Q # You can replace this text with custom code or comments, and it will be preserved on regeneration diff --git a/src/sql/hydra.sql b/src/sql/hydra.sql index eaae6da3..e9457972 100644 --- a/src/sql/hydra.sql +++ b/src/sql/hydra.sql @@ -247,7 +247,7 @@ create trigger BuildBumped after update on Builds for each row create table BuildOutputs ( build integer not null, name text not null, - path text not null, + path text, primary key (build, name), foreign key (build) references Builds(id) on delete cascade ); @@ -303,7 +303,7 @@ create table BuildStepOutputs ( build integer not null, stepnr integer not null, name text not null, - path text not null, + path text, primary key (build, stepnr, name), foreign key (build) references Builds(id) on delete cascade, foreign key (build, stepnr) references BuildSteps(build, stepnr) on delete cascade diff --git a/t/content-addressed/basic.t b/t/content-addressed/basic.t new file mode 100644 index 00000000..6597e727 --- /dev/null +++ b/t/content-addressed/basic.t @@ -0,0 +1,61 @@ +use feature 'unicode_strings'; +use strict; +use warnings; +use Setup; + +my %ctx = test_init( + nix_config => qq| + experimental-features = ca-derivations + |, +); + +require Hydra::Schema; +require Hydra::Model::DB; + +use JSON::MaybeXS; + +use HTTP::Request::Common; +use Test2::V0; +require Catalyst::Test; +Catalyst::Test->import('Hydra'); + +my $db = Hydra::Model::DB->new; +hydra_setup($db); + +my $project = $db->resultset('Projects')->create({name => "tests", displayname => "", owner => "root"}); + +my $jobset = createBaseJobset("content-addressed", "content-addressed.nix", $ctx{jobsdir}); + +ok(evalSucceeds($jobset), "Evaluating jobs/content-addressed.nix should exit with return code 0"); +is(nrQueuedBuildsForJobset($jobset), 5, "Evaluating jobs/content-addressed.nix should result in 4 builds"); + +for my $build (queuedBuildsForJobset($jobset)) { + ok(runBuild($build), "Build '".$build->job."' from jobs/content-addressed.nix should exit with code 0"); + my $newbuild = $db->resultset('Builds')->find($build->id); + is($newbuild->finished, 1, "Build '".$build->job."' from jobs/content-addressed.nix should be finished."); + my $expected = $build->job eq "fails" ? 1 : $build->job =~ /with_failed/ ? 6 : 0; + is($newbuild->buildstatus, $expected, "Build '".$build->job."' from jobs/content-addressed.nix should have buildstatus $expected."); + + my $response = request("/build/".$build->id); + ok($response->is_success, "The 'build' page for build '".$build->job."' should load properly"); + + if ($newbuild->buildstatus == 0) { + my $buildOutputs = $newbuild->buildoutputs; + for my $output ($newbuild->buildoutputs) { + # XXX: This hardcodes /nix/store/. + # It's fine because in practice the nix store for the tests will be of + # the form `/some/thing/nix/store/`, but it would be cleaner if there + # was a way to query Nix for its store dir? + like( + $output->path, qr|/nix/store/|, + "Output '".$output->name."' of build '".$build->job."' should be a valid store path" + ); + } + } + +} + +isnt(<$ctx{deststoredir}/realisations/*>, "", "The destination store should have the realisations of the built derivations registered"); + +done_testing; + diff --git a/t/content-addressed/without-experimental-feature.t b/t/content-addressed/without-experimental-feature.t new file mode 100644 index 00000000..a37d138e --- /dev/null +++ b/t/content-addressed/without-experimental-feature.t @@ -0,0 +1,28 @@ +use feature 'unicode_strings'; +use strict; +use warnings; +use Setup; + +my %ctx = test_init(); + +require Hydra::Schema; +require Hydra::Model::DB; + +use JSON::MaybeXS; + +use HTTP::Request::Common; +use Test2::V0; +require Catalyst::Test; +Catalyst::Test->import('Hydra'); + +my $db = Hydra::Model::DB->new; +hydra_setup($db); + +my $project = $db->resultset('Projects')->create({name => "tests", displayname => "", owner => "root"}); + +my $jobset = createBaseJobset("content-addressed", "content-addressed.nix", $ctx{jobsdir}); + +ok(evalSucceeds($jobset), "Evaluating jobs/content-addressed.nix without the experimental feature should exit with return code 0"); +is(nrQueuedBuildsForJobset($jobset), 0, "Evaluating jobs/content-addressed.nix without the experimental Nix feature should result in 0 build"); + +done_testing; diff --git a/t/jobs/config.nix.in b/t/jobs/config.nix.in index 51b6c06f..41776341 100644 --- a/t/jobs/config.nix.in +++ b/t/jobs/config.nix.in @@ -6,4 +6,9 @@ rec { system = builtins.currentSystem; PATH = path; } // args); + mkContentAddressedDerivation = args: mkDerivation ({ + __contentAddressed = true; + outputHashMode = "recursive"; + outputHashAlgo = "sha256"; + } // args); } diff --git a/t/jobs/content-addressed.nix b/t/jobs/content-addressed.nix new file mode 100644 index 00000000..65496df5 --- /dev/null +++ b/t/jobs/content-addressed.nix @@ -0,0 +1,35 @@ +let cfg = import ./config.nix; in +rec { + empty_dir = + cfg.mkContentAddressedDerivation { + name = "empty-dir"; + builder = ./empty-dir-builder.sh; + }; + + fails = + cfg.mkContentAddressedDerivation { + name = "fails"; + builder = ./fail.sh; + }; + + succeed_with_failed = + cfg.mkContentAddressedDerivation { + name = "succeed-with-failed"; + builder = ./succeed-with-failed.sh; + }; + + caDependingOnCA = + cfg.mkContentAddressedDerivation { + name = "ca-depending-on-ca"; + builder = ./dir-with-file-builder.sh; + FOO = empty_dir; + }; + + nonCaDependingOnCA = + cfg.mkDerivation { + name = "non-ca-depending-on-ca"; + builder = ./dir-with-file-builder.sh; + FOO = empty_dir; + }; +} + diff --git a/t/jobs/dir-with-file-builder.sh b/t/jobs/dir-with-file-builder.sh new file mode 100755 index 00000000..e51c6558 --- /dev/null +++ b/t/jobs/dir-with-file-builder.sh @@ -0,0 +1,7 @@ +#! /bin/sh + +# Workaround for https://github.com/NixOS/nix/pull/6051 +echo "some output" + +mkdir $out +echo foo > $out/a-file diff --git a/t/queue-runner/notifications.t b/t/queue-runner/notifications.t index 1966cde1..d0e72409 100644 --- a/t/queue-runner/notifications.t +++ b/t/queue-runner/notifications.t @@ -8,7 +8,7 @@ my $binarycachedir = File::Temp->newdir(); my $ctx = test_context( nix_config => qq| - experimental-features = nix-command + experimental-features = nix-command ca-derivations substituters = file://${binarycachedir}?trusted=1 |, hydra_config => q| From 5ee0e443e4ea90888e74b90d2e4198494e8eb597 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 26 Jan 2024 01:08:11 -0500 Subject: [PATCH 104/313] Remove now-unneeded workaround --- t/jobs/empty-dir-builder.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/t/jobs/empty-dir-builder.sh b/t/jobs/empty-dir-builder.sh index 949216e0..addc7ef6 100755 --- a/t/jobs/empty-dir-builder.sh +++ b/t/jobs/empty-dir-builder.sh @@ -1,6 +1,3 @@ #! /bin/sh -# Workaround for https://github.com/NixOS/nix/pull/6051 -echo "some output" - mkdir $out From c62eaf248f1fe7653f37427e70ce4741cf9a1c2d Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 26 Jan 2024 01:20:07 -0500 Subject: [PATCH 105/313] Remove now-unneeded workaround --- t/jobs/dir-with-file-builder.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/t/jobs/dir-with-file-builder.sh b/t/jobs/dir-with-file-builder.sh index e51c6558..8592a1e8 100755 --- a/t/jobs/dir-with-file-builder.sh +++ b/t/jobs/dir-with-file-builder.sh @@ -1,7 +1,4 @@ #! /bin/sh -# Workaround for https://github.com/NixOS/nix/pull/6051 -echo "some output" - mkdir $out echo foo > $out/a-file From 8477009310c0cde111af3da9313d3c99902a48c7 Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Fri, 26 Jan 2024 17:28:07 +0100 Subject: [PATCH 106/313] doc/manual: fix instructions in contribution guidelines In 5db374cb500b687039ba4701b205ca7dfa67caba the `bootstrap` script was removed, however it's still referenced in the contribution guidelines. Change that to `autoreconfPhase` as intended by the commit. --- doc/manual/src/hacking.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/hacking.md b/doc/manual/src/hacking.md index 72a74c82..49c17395 100644 --- a/doc/manual/src/hacking.md +++ b/doc/manual/src/hacking.md @@ -18,7 +18,7 @@ $ nix-shell To build Hydra, you should then do: ```console -[nix-shell]$ ./bootstrap +[nix-shell]$ autoreconfPhase [nix-shell]$ configurePhase [nix-shell]$ make ``` From b4c91b5a6ab91177d2c52c5b68c880277b4fd35f Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Fri, 26 Jan 2024 17:30:07 +0100 Subject: [PATCH 107/313] package: move foreman to nativeCheckInputs In 1bd195a5138b3c69c52110d713c706cb4908ba16 strictDeps was set for the Hydra package. As a result, `checkInputs` aren't available anymore in the local dev-shell which is the sole purpose of foreman, to start services and a database for development. --- package.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.nix b/package.nix index 0f8f004d..f8b1849f 100644 --- a/package.nix +++ b/package.nix @@ -183,6 +183,7 @@ stdenv.mkDerivation (finalAttrs: { nativeCheckInputs = [ bzip2 darcs + foreman top-git mercurial subversion @@ -194,7 +195,6 @@ stdenv.mkDerivation (finalAttrs: { checkInputs = [ cacert - foreman glibcLocales libressl.nc python3 From b503280256bed6ced0c79d32ee17dda72827a381 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 26 Jan 2024 11:53:58 -0500 Subject: [PATCH 108/313] Add migration to drop non-null constraints --- src/sql/upgrade-84.sql | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 src/sql/upgrade-84.sql diff --git a/src/sql/upgrade-84.sql b/src/sql/upgrade-84.sql new file mode 100644 index 00000000..bf142b30 --- /dev/null +++ b/src/sql/upgrade-84.sql @@ -0,0 +1,4 @@ +-- CA derivations do not have statically known output paths. The values +-- are only filled in after the build runs. +ALTER TABLE BuildStepOutputs ALTER COLUMN path DROP NOT NULL; +ALTER TABLE BuildOutputs ALTER COLUMN path DROP NOT NULL; From 14aabc1cc94b1a51f67bbcb30f901785d47a9872 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 30 Jan 2024 13:31:56 -0500 Subject: [PATCH 109/313] Update to released Nix 2.20 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nix': 'github:NixOS/nix/8df68a213fc52a57b02a57005b0e06cc8de40ce3' (2024-01-25) → 'github:NixOS/nix/8f42912c80c0a03f62f6a3d28a3af05a9762565d' (2024-01-30) --- flake.lock | 7 ++++--- flake.nix | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/flake.lock b/flake.lock index e3ea8f61..d8b2e4a9 100644 --- a/flake.lock +++ b/flake.lock @@ -42,15 +42,16 @@ "nixpkgs-regression": "nixpkgs-regression" }, "locked": { - "lastModified": 1706195509, - "narHash": "sha256-1kwfk7H/MWZAcTKHnnWXo/+KlQeOTIRtOIzc4FX3QnE=", + "lastModified": 1706637536, + "narHash": "sha256-fjx+nCOzuSxGWfhwWWc8hCsLFZAjZLDDUcbBtldRqbk=", "owner": "NixOS", "repo": "nix", - "rev": "8df68a213fc52a57b02a57005b0e06cc8de40ce3", + "rev": "8f42912c80c0a03f62f6a3d28a3af05a9762565d", "type": "github" }, "original": { "owner": "NixOS", + "ref": "2.20-maintenance", "repo": "nix", "type": "github" } diff --git a/flake.nix b/flake.nix index 4dc7d25e..306ed292 100644 --- a/flake.nix +++ b/flake.nix @@ -2,7 +2,7 @@ description = "A Nix-based continuous build system"; inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.05-small"; - inputs.nix.url = "github:NixOS/nix"; + inputs.nix.url = "github:NixOS/nix/2.20-maintenance"; inputs.nix.inputs.nixpkgs.follows = "nixpkgs"; # TODO get rid of this once https://github.com/NixOS/nix/pull/9546 is From 878c0f240e950c4d7bf8ebc60299bc0db80e6686 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 30 Jan 2024 13:50:25 -0500 Subject: [PATCH 110/313] Switch (back) to Nix master MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Re-creating `nix-next` after using it in #1354. Flake lock file updates: • Updated input 'nix': 'github:NixOS/nix/8df68a213fc52a57b02a57005b0e06cc8de40ce3' (2024-01-25) → 'github:NixOS/nix/75ebb90a70f6320c1c7a1fca87a0a8adb0716143' (2024-01-30) --- flake.lock | 7 +++---- flake.nix | 2 +- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/flake.lock b/flake.lock index d8b2e4a9..f04d657b 100644 --- a/flake.lock +++ b/flake.lock @@ -42,16 +42,15 @@ "nixpkgs-regression": "nixpkgs-regression" }, "locked": { - "lastModified": 1706637536, - "narHash": "sha256-fjx+nCOzuSxGWfhwWWc8hCsLFZAjZLDDUcbBtldRqbk=", + "lastModified": 1706629374, + "narHash": "sha256-KyAiLGxJ39fSY0cuq8EWAZQ4vaDdqAItSRP4+vjYvq8=", "owner": "NixOS", "repo": "nix", - "rev": "8f42912c80c0a03f62f6a3d28a3af05a9762565d", + "rev": "75ebb90a70f6320c1c7a1fca87a0a8adb0716143", "type": "github" }, "original": { "owner": "NixOS", - "ref": "2.20-maintenance", "repo": "nix", "type": "github" } diff --git a/flake.nix b/flake.nix index 306ed292..4dc7d25e 100644 --- a/flake.nix +++ b/flake.nix @@ -2,7 +2,7 @@ description = "A Nix-based continuous build system"; inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.05-small"; - inputs.nix.url = "github:NixOS/nix/2.20-maintenance"; + inputs.nix.url = "github:NixOS/nix"; inputs.nix.inputs.nixpkgs.follows = "nixpkgs"; # TODO get rid of this once https://github.com/NixOS/nix/pull/9546 is From ceff5c5cfeaf211691f4d1156f358a940b5ef7b4 Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Mon, 12 Feb 2024 18:30:03 +0100 Subject: [PATCH 111/313] flake: fix gitea integration test This is an integration test that confirms that jobset definitions from git repositories are correctly built and status updates pushed to the gitea instance. The following things needed to be fixed: * We're still on 23.05 where gitea is marked as insecure. Not going to update nixpkgs right now, but going for the quick fix. * Since gitea 1.19 tokens have scopes that describe what's possible. Not specifying the scope in the DB appears to imply that no permissions are granted. * Apparently we have three status updates now (for three status hooks, queued/started/finished). No idea why that was broken before, but the behavior still looks correct. --- flake.nix | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/flake.nix b/flake.nix index 306ed292..a6dfb977 100644 --- a/flake.nix +++ b/flake.nix @@ -180,13 +180,9 @@ root=d7f16a3412e01a43a414535b16007c6931d3a9c7 ''; + nixpkgs.config.permittedInsecurePackages = [ "gitea-1.19.4" ]; nix = { - distributedBuilds = true; - buildMachines = [{ - hostName = "localhost"; - systems = [ system ]; - }]; - binaryCaches = [ ]; + settings.substituters = [ ]; }; services.gitea = { enable = true; @@ -202,7 +198,7 @@ testScript = let scripts.mktoken = pkgs.writeText "token.sql" '' - INSERT INTO access_token (id, uid, name, created_unix, updated_unix, token_hash, token_salt, token_last_eight) VALUES (1, 1, 'hydra', 1617107360, 1617107360, 'a930f319ca362d7b49a4040ac0af74521c3a3c3303a86f327b01994430672d33b6ec53e4ea774253208686c712495e12a486', 'XRjWE9YW0g', '31d3a9c7'); + INSERT INTO access_token (id, uid, name, created_unix, updated_unix, token_hash, token_salt, token_last_eight, scope) VALUES (1, 1, 'hydra', 1617107360, 1617107360, 'a930f319ca362d7b49a4040ac0af74521c3a3c3303a86f327b01994430672d33b6ec53e4ea774253208686c712495e12a486', 'XRjWE9YW0g', '31d3a9c7', 'all'); ''; scripts.git-setup = pkgs.writeShellScript "setup.sh" '' @@ -357,9 +353,10 @@ response = json.loads(data) - assert len(response) == 2, "Expected exactly two status updates for latest commit!" - assert response[0]['status'] == "success", "Expected latest status to be success!" - assert response[1]['status'] == "pending", "Expected first status to be pending!" + assert len(response) == 3, "Expected exactly three status updates for latest commit (queued, started, finished)!" + assert response[0]['status'] == "success", "Expected finished status to be success!" + assert response[1]['status'] == "pending", "Expected started status to be pending!" + assert response[2]['status'] == "pending", "Expected queued status to be pending!" machine.shutdown() ''; From e499509595859d670432c0d98162119ecd552666 Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Mon, 12 Feb 2024 17:12:49 +0100 Subject: [PATCH 112/313] Switch to new Nix bindings, update Nix for that Implements support for Nix's new Perl bindings[1]. The current state basically does `openStore()`, but always uses `auto` and doesn't support stores at other URIs. Even though the stores are cached inside the Perl implementation, I decided to instantiate those once in the Nix helper module. That way store openings aren't cluttered across the entire codebase. Also, there are two stores used later on - MACHINE_LOCAL_STORE for `auto`, BINARY_CACHE_STORE for the one from `store_uri` in `hydra.conf` - and using consistent names should make the intent clearer then. This doesn't contain any behavioral changes, i.e. the build product availability issue from #1352 isn't fixed. This patch only contains the migration to the new API. [1] https://github.com/NixOS/nix/pull/9863 --- flake.lock | 6 +++--- src/hydra-eval-jobs/hydra-eval-jobs.cc | 6 +++--- src/hydra-evaluator/hydra-evaluator.cc | 2 +- src/hydra-queue-runner/queue-monitor.cc | 2 +- src/lib/Hydra/Base/Controller/NixChannel.pm | 3 +-- src/lib/Hydra/Controller/Build.pm | 18 ++++++++---------- src/lib/Hydra/Controller/Root.pm | 2 +- src/lib/Hydra/Helper/Nix.pm | 5 ++++- src/lib/Hydra/Plugin/BazaarInput.pm | 7 +++---- src/lib/Hydra/Plugin/DarcsInput.pm | 7 +++---- src/lib/Hydra/Plugin/GitInput.pm | 6 +++--- src/lib/Hydra/Plugin/MercurialInput.pm | 7 +++---- src/lib/Hydra/Plugin/PathInput.pm | 5 ++--- src/lib/Hydra/Plugin/SubversionInput.pm | 9 ++++----- src/lib/Hydra/View/NARInfo.pm | 9 ++++----- src/script/hydra-eval-jobset | 8 ++++---- src/script/hydra-update-gc-roots | 5 ++--- t/s3-backup-test.pl | 1 - 18 files changed, 50 insertions(+), 58 deletions(-) diff --git a/flake.lock b/flake.lock index f04d657b..4b4f0076 100644 --- a/flake.lock +++ b/flake.lock @@ -42,11 +42,11 @@ "nixpkgs-regression": "nixpkgs-regression" }, "locked": { - "lastModified": 1706629374, - "narHash": "sha256-KyAiLGxJ39fSY0cuq8EWAZQ4vaDdqAItSRP4+vjYvq8=", + "lastModified": 1707750141, + "narHash": "sha256-9qSzGQs/Rjf2i3UQjyaZUznZzYDHkLYro/1FTT1easg=", "owner": "NixOS", "repo": "nix", - "rev": "75ebb90a70f6320c1c7a1fca87a0a8adb0716143", + "rev": "c4ebb82da4eade975e874da600dc50e9dec610cb", "type": "github" }, "original": { diff --git a/src/hydra-eval-jobs/hydra-eval-jobs.cc b/src/hydra-eval-jobs/hydra-eval-jobs.cc index 1d7de74b..d5619719 100644 --- a/src/hydra-eval-jobs/hydra-eval-jobs.cc +++ b/src/hydra-eval-jobs/hydra-eval-jobs.cc @@ -185,7 +185,7 @@ static void worker( !experimentalFeatureSettings.isEnabled(Xp::CaDerivations)); if (drv->querySystem() == "unknown") - throw EvalError("derivation must have a 'system' attribute"); + state.error("derivation must have a 'system' attribute").debugThrow(); auto drvPath = state.store->printStorePath(drv->requireDrvPath()); @@ -208,7 +208,7 @@ static void worker( if (a && state.forceBool(*a->value, a->pos, "while evaluating the `_hydraAggregate` attribute")) { auto a = v->attrs->get(state.symbols.create("constituents")); if (!a) - throw EvalError("derivation must have a ‘constituents’ attribute"); + state.error("derivation must have a ‘constituents’ attribute").debugThrow(); NixStringContext context; state.coerceToString(a->pos, *a->value, context, "while evaluating the `constituents` attribute", true, false); @@ -274,7 +274,7 @@ static void worker( else if (v->type() == nNull) ; - else throw TypeError("attribute '%s' is %s, which is not supported", attrPath, showType(*v)); + else state.error("attribute '%s' is %s, which is not supported", attrPath, showType(*v)).debugThrow(); } catch (EvalError & e) { auto msg = e.msg(); diff --git a/src/hydra-evaluator/hydra-evaluator.cc b/src/hydra-evaluator/hydra-evaluator.cc index 75506ff8..9312d085 100644 --- a/src/hydra-evaluator/hydra-evaluator.cc +++ b/src/hydra-evaluator/hydra-evaluator.cc @@ -38,7 +38,7 @@ class JobsetId { friend bool operator!= (const JobsetId & lhs, const JobsetName & rhs); std::string display() const { - return str(format("%1%:%2% (jobset#%3%)") % project % jobset % id); + return boost::str(boost::format("%1%:%2% (jobset#%3%)") % project % jobset % id); } }; bool operator==(const JobsetId & lhs, const JobsetId & rhs) diff --git a/src/hydra-queue-runner/queue-monitor.cc b/src/hydra-queue-runner/queue-monitor.cc index 203f9f1d..06b06512 100644 --- a/src/hydra-queue-runner/queue-monitor.cc +++ b/src/hydra-queue-runner/queue-monitor.cc @@ -294,7 +294,7 @@ bool State::getQueuedBuilds(Connection & conn, try { createBuild(build); } catch (Error & e) { - e.addTrace({}, hintfmt("while loading build %d: ", build->id)); + e.addTrace({}, HintFmt("while loading build %d: ", build->id)); throw; } diff --git a/src/lib/Hydra/Base/Controller/NixChannel.pm b/src/lib/Hydra/Base/Controller/NixChannel.pm index 3f8e9609..a5bc2784 100644 --- a/src/lib/Hydra/Base/Controller/NixChannel.pm +++ b/src/lib/Hydra/Base/Controller/NixChannel.pm @@ -4,7 +4,6 @@ use strict; use warnings; use base 'Hydra::Base::Controller::REST'; use List::SomeUtils qw(any); -use Nix::Store; use Hydra::Helper::Nix; use Hydra::Helper::CatalystUtils; @@ -30,7 +29,7 @@ sub getChannelData { my $outputs = {}; foreach my $output (@outputs) { my $outPath = $output->get_column("outpath"); - next if $checkValidity && !isValidPath($outPath); + next if $checkValidity && !$MACHINE_LOCAL_STORE->isValidPath($outPath); $outputs->{$output->get_column("outname")} = $outPath; push @storePaths, $outPath; # Put the system type in the manifest (for top-level diff --git a/src/lib/Hydra/Controller/Build.pm b/src/lib/Hydra/Controller/Build.pm index c3869838..4e249d14 100644 --- a/src/lib/Hydra/Controller/Build.pm +++ b/src/lib/Hydra/Controller/Build.pm @@ -10,8 +10,6 @@ use File::Basename; use File::LibMagic; use File::stat; use Data::Dump qw(dump); -use Nix::Store; -use Nix::Config; use List::SomeUtils qw(all); use Encode; use JSON::PP; @@ -82,9 +80,9 @@ sub build_GET { # false because `$_->path` will be empty $c->stash->{available} = $c->stash->{isLocalStore} - ? all { $_->path && isValidPath($_->path) } $build->buildoutputs->all + ? all { $_->path && $MACHINE_LOCAL_STORE->isValidPath($_->path) } $build->buildoutputs->all : 1; - $c->stash->{drvAvailable} = isValidPath $build->drvpath; + $c->stash->{drvAvailable} = $MACHINE_LOCAL_STORE->isValidPath($build->drvpath); if ($build->finished && $build->iscachedbuild) { my $path = ($build->buildoutputs)[0]->path or undef; @@ -308,7 +306,7 @@ sub output : Chained('buildChain') PathPart Args(1) { error($c, "This build is not finished yet.") unless $build->finished; my $output = $build->buildoutputs->find({name => $outputName}); notFound($c, "This build has no output named ‘$outputName’") unless defined $output; - gone($c, "Output is no longer available.") unless isValidPath $output->path; + gone($c, "Output is no longer available.") unless $MACHINE_LOCAL_STORE->isValidPath($output->path); $c->response->header('Content-Disposition', "attachment; filename=\"build-${\$build->id}-${\$outputName}.nar.bz2\""); $c->stash->{current_view} = 'NixNAR'; @@ -425,7 +423,7 @@ sub getDependencyGraph { }; $$done{$path} = $node; my @refs; - foreach my $ref (queryReferences($path)) { + foreach my $ref ($MACHINE_LOCAL_STORE->queryReferences($path)) { next if $ref eq $path; next unless $runtime || $ref =~ /\.drv$/; getDependencyGraph($self, $c, $runtime, $done, $ref); @@ -433,7 +431,7 @@ sub getDependencyGraph { } # Show in reverse topological order to flatten the graph. # Should probably do a proper BFS. - my @sorted = reverse topoSortPaths(@refs); + my @sorted = reverse $MACHINE_LOCAL_STORE->topoSortPaths(@refs); $node->{refs} = [map { $$done{$_} } @sorted]; } @@ -446,7 +444,7 @@ sub build_deps : Chained('buildChain') PathPart('build-deps') { my $build = $c->stash->{build}; my $drvPath = $build->drvpath; - error($c, "Derivation no longer available.") unless isValidPath $drvPath; + error($c, "Derivation no longer available.") unless $MACHINE_LOCAL_STORE->isValidPath($drvPath); $c->stash->{buildTimeGraph} = getDependencyGraph($self, $c, 0, {}, $drvPath); @@ -461,7 +459,7 @@ sub runtime_deps : Chained('buildChain') PathPart('runtime-deps') { requireLocalStore($c); - error($c, "Build outputs no longer available.") unless all { isValidPath($_) } @outPaths; + error($c, "Build outputs no longer available.") unless all { $MACHINE_LOCAL_STORE->isValidPath($_) } @outPaths; my $done = {}; $c->stash->{runtimeGraph} = [ map { getDependencyGraph($self, $c, 1, $done, $_) } @outPaths ]; @@ -481,7 +479,7 @@ sub nix : Chained('buildChain') PathPart('nix') CaptureArgs(0) { if (isLocalStore) { foreach my $out ($build->buildoutputs) { notFound($c, "Path " . $out->path . " is no longer available.") - unless isValidPath($out->path); + unless $MACHINE_LOCAL_STORE->isValidPath($out->path); } } diff --git a/src/lib/Hydra/Controller/Root.pm b/src/lib/Hydra/Controller/Root.pm index 548cfac3..162f347e 100644 --- a/src/lib/Hydra/Controller/Root.pm +++ b/src/lib/Hydra/Controller/Root.pm @@ -395,7 +395,7 @@ sub narinfo :Path :Args(StrMatch[NARINFO_REGEX]) { my ($hash) = $narinfo =~ NARINFO_REGEX; die("Hash length was not 32") if length($hash) != 32; - my $path = queryPathFromHashPart($hash); + my $path = $MACHINE_LOCAL_STORE->queryPathFromHashPart($hash); if (!$path) { $c->response->status(404); diff --git a/src/lib/Hydra/Helper/Nix.pm b/src/lib/Hydra/Helper/Nix.pm index 71a8a7d7..900514cb 100644 --- a/src/lib/Hydra/Helper/Nix.pm +++ b/src/lib/Hydra/Helper/Nix.pm @@ -40,8 +40,11 @@ our @EXPORT = qw( registerRoot restartBuilds run + $MACHINE_LOCAL_STORE ); +our $MACHINE_LOCAL_STORE = Nix::Store->new(); + sub getHydraHome { my $dir = $ENV{"HYDRA_HOME"} or die "The HYDRA_HOME directory does not exist!\n"; @@ -494,7 +497,7 @@ sub restartBuilds { $builds = $builds->search({ finished => 1 }); foreach my $build ($builds->search({}, { columns => ["drvpath"] })) { - next if !isValidPath($build->drvpath); + next if !$MACHINE_LOCAL_STORE->isValidPath($build->drvpath); registerRoot $build->drvpath; } diff --git a/src/lib/Hydra/Plugin/BazaarInput.pm b/src/lib/Hydra/Plugin/BazaarInput.pm index 230d108b..b35ed7c8 100644 --- a/src/lib/Hydra/Plugin/BazaarInput.pm +++ b/src/lib/Hydra/Plugin/BazaarInput.pm @@ -7,7 +7,6 @@ use Digest::SHA qw(sha256_hex); use File::Path; use Hydra::Helper::Exec; use Hydra::Helper::Nix; -use Nix::Store; sub supportedInputTypes { my ($self, $inputTypes) = @_; @@ -38,9 +37,9 @@ sub fetchInput { (my $cachedInput) = $self->{db}->resultset('CachedBazaarInputs')->search( {uri => $uri, revision => $revision}); - addTempRoot($cachedInput->storepath) if defined $cachedInput; + $MACHINE_LOCAL_STORE->addTempRoot($cachedInput->storepath) if defined $cachedInput; - if (defined $cachedInput && isValidPath($cachedInput->storepath)) { + if (defined $cachedInput && $MACHINE_LOCAL_STORE->isValidPath($cachedInput->storepath)) { $storePath = $cachedInput->storepath; $sha256 = $cachedInput->sha256hash; } else { @@ -58,7 +57,7 @@ sub fetchInput { ($sha256, $storePath) = split ' ', $stdout; # FIXME: time window between nix-prefetch-bzr and addTempRoot. - addTempRoot($storePath); + $MACHINE_LOCAL_STORE->addTempRoot($storePath); $self->{db}->txn_do(sub { $self->{db}->resultset('CachedBazaarInputs')->create( diff --git a/src/lib/Hydra/Plugin/DarcsInput.pm b/src/lib/Hydra/Plugin/DarcsInput.pm index b7f3db55..a8df6396 100644 --- a/src/lib/Hydra/Plugin/DarcsInput.pm +++ b/src/lib/Hydra/Plugin/DarcsInput.pm @@ -7,7 +7,6 @@ use Digest::SHA qw(sha256_hex); use File::Path; use Hydra::Helper::Exec; use Hydra::Helper::Nix; -use Nix::Store; sub supportedInputTypes { my ($self, $inputTypes) = @_; @@ -58,7 +57,7 @@ sub fetchInput { {uri => $uri, revision => $revision}, {rows => 1}); - if (defined $cachedInput && isValidPath($cachedInput->storepath)) { + if (defined $cachedInput && $MACHINE_LOCAL_STORE->isValidPath($cachedInput->storepath)) { $storePath = $cachedInput->storepath; $sha256 = $cachedInput->sha256hash; $revision = $cachedInput->revision; @@ -75,8 +74,8 @@ sub fetchInput { die "darcs changes --count failed" if $? != 0; system "rm", "-rf", "$tmpDir/export/_darcs"; - $storePath = addToStore("$tmpDir/export", 1, "sha256"); - $sha256 = queryPathHash($storePath); + $storePath = $MACHINE_LOCAL_STORE->addToStore("$tmpDir/export", 1, "sha256"); + $sha256 = $MACHINE_LOCAL_STORE->queryPathHash($storePath); $sha256 =~ s/sha256://; $self->{db}->txn_do(sub { diff --git a/src/lib/Hydra/Plugin/GitInput.pm b/src/lib/Hydra/Plugin/GitInput.pm index e5fc7de9..0de02128 100644 --- a/src/lib/Hydra/Plugin/GitInput.pm +++ b/src/lib/Hydra/Plugin/GitInput.pm @@ -186,9 +186,9 @@ sub fetchInput { {uri => $uri, branch => $branch, revision => $revision, isdeepclone => defined($deepClone) ? 1 : 0}, {rows => 1}); - addTempRoot($cachedInput->storepath) if defined $cachedInput; + $MACHINE_LOCAL_STORE->addTempRoot($cachedInput->storepath) if defined $cachedInput; - if (defined $cachedInput && isValidPath($cachedInput->storepath)) { + if (defined $cachedInput && $MACHINE_LOCAL_STORE->isValidPath($cachedInput->storepath)) { $storePath = $cachedInput->storepath; $sha256 = $cachedInput->sha256hash; $revision = $cachedInput->revision; @@ -217,7 +217,7 @@ sub fetchInput { ($sha256, $storePath) = split ' ', grab(cmd => ["nix-prefetch-git", $clonePath, $revision], chomp => 1); # FIXME: time window between nix-prefetch-git and addTempRoot. - addTempRoot($storePath); + $MACHINE_LOCAL_STORE->addTempRoot($storePath); $self->{db}->txn_do(sub { $self->{db}->resultset('CachedGitInputs')->update_or_create( diff --git a/src/lib/Hydra/Plugin/MercurialInput.pm b/src/lib/Hydra/Plugin/MercurialInput.pm index 921262ad..85bd2c70 100644 --- a/src/lib/Hydra/Plugin/MercurialInput.pm +++ b/src/lib/Hydra/Plugin/MercurialInput.pm @@ -7,7 +7,6 @@ use Digest::SHA qw(sha256_hex); use File::Path; use Hydra::Helper::Nix; use Hydra::Helper::Exec; -use Nix::Store; use Fcntl qw(:flock); sub supportedInputTypes { @@ -68,9 +67,9 @@ sub fetchInput { (my $cachedInput) = $self->{db}->resultset('CachedHgInputs')->search( {uri => $uri, branch => $branch, revision => $revision}); - addTempRoot($cachedInput->storepath) if defined $cachedInput; + $MACHINE_LOCAL_STORE->addTempRoot($cachedInput->storepath) if defined $cachedInput; - if (defined $cachedInput && isValidPath($cachedInput->storepath)) { + if (defined $cachedInput && $MACHINE_LOCAL_STORE->isValidPath($cachedInput->storepath)) { $storePath = $cachedInput->storepath; $sha256 = $cachedInput->sha256hash; } else { @@ -85,7 +84,7 @@ sub fetchInput { ($sha256, $storePath) = split ' ', $stdout; # FIXME: time window between nix-prefetch-hg and addTempRoot. - addTempRoot($storePath); + $MACHINE_LOCAL_STORE->addTempRoot($storePath); $self->{db}->txn_do(sub { $self->{db}->resultset('CachedHgInputs')->update_or_create( diff --git a/src/lib/Hydra/Plugin/PathInput.pm b/src/lib/Hydra/Plugin/PathInput.pm index d122ff57..c923a03c 100644 --- a/src/lib/Hydra/Plugin/PathInput.pm +++ b/src/lib/Hydra/Plugin/PathInput.pm @@ -5,7 +5,6 @@ use warnings; use parent 'Hydra::Plugin'; use POSIX qw(strftime); use Hydra::Helper::Nix; -use Nix::Store; sub supportedInputTypes { my ($self, $inputTypes) = @_; @@ -30,7 +29,7 @@ sub fetchInput { {srcpath => $uri, lastseen => {">", $timestamp - $timeout}}, {rows => 1, order_by => "lastseen DESC"}); - if (defined $cachedInput && isValidPath($cachedInput->storepath)) { + if (defined $cachedInput && $MACHINE_LOCAL_STORE->isValidPath($cachedInput->storepath)) { $storePath = $cachedInput->storepath; $sha256 = $cachedInput->sha256hash; $timestamp = $cachedInput->timestamp; @@ -46,7 +45,7 @@ sub fetchInput { } chomp $storePath; - $sha256 = (queryPathInfo($storePath, 0))[1] or die; + $sha256 = ($MACHINE_LOCAL_STORE->queryPathInfo($storePath, 0))[1] or die; ($cachedInput) = $self->{db}->resultset('CachedPathInputs')->search( {srcpath => $uri, sha256hash => $sha256}); diff --git a/src/lib/Hydra/Plugin/SubversionInput.pm b/src/lib/Hydra/Plugin/SubversionInput.pm index 456c6892..83c1f39d 100644 --- a/src/lib/Hydra/Plugin/SubversionInput.pm +++ b/src/lib/Hydra/Plugin/SubversionInput.pm @@ -7,7 +7,6 @@ use Digest::SHA qw(sha256_hex); use Hydra::Helper::Exec; use Hydra::Helper::Nix; use IPC::Run; -use Nix::Store; sub supportedInputTypes { my ($self, $inputTypes) = @_; @@ -45,7 +44,7 @@ sub fetchInput { (my $cachedInput) = $self->{db}->resultset('CachedSubversionInputs')->search( {uri => $uri, revision => $revision}); - addTempRoot($cachedInput->storepath) if defined $cachedInput; + $MACHINE_LOCAL_STORE->addTempRoot($cachedInput->storepath) if defined $cachedInput; if (defined $cachedInput && isValidPath($cachedInput->storepath)) { $storePath = $cachedInput->storepath; @@ -62,16 +61,16 @@ sub fetchInput { die "error checking out Subversion repo at `$uri':\n$stderr" if $res; if ($type eq "svn-checkout") { - $storePath = addToStore($wcPath, 1, "sha256"); + $storePath = $MACHINE_LOCAL_STORE->addToStore($wcPath, 1, "sha256"); } else { # Hm, if the Nix Perl bindings supported filters in # addToStore(), then we wouldn't need to make a copy here. my $tmpDir = File::Temp->newdir("hydra-svn-export.XXXXXX", CLEANUP => 1, TMPDIR => 1) or die; (system "svn", "export", $wcPath, "$tmpDir/source", "--quiet") == 0 or die "svn export failed"; - $storePath = addToStore("$tmpDir/source", 1, "sha256"); + $storePath = $MACHINE_LOCAL_STORE->addToStore("$tmpDir/source", 1, "sha256"); } - $sha256 = queryPathHash($storePath); $sha256 =~ s/sha256://; + $sha256 = $MACHINE_LOCAL_STORE->queryPathHash($storePath); $sha256 =~ s/sha256://; $self->{db}->txn_do(sub { $self->{db}->resultset('CachedSubversionInputs')->update_or_create( diff --git a/src/lib/Hydra/View/NARInfo.pm b/src/lib/Hydra/View/NARInfo.pm index 44db78b1..bf8711a4 100644 --- a/src/lib/Hydra/View/NARInfo.pm +++ b/src/lib/Hydra/View/NARInfo.pm @@ -6,8 +6,7 @@ use File::Basename; use Hydra::Helper::CatalystUtils; use MIME::Base64; use Nix::Manifest; -use Nix::Store; -use Nix::Utils; +use Hydra::Helper::Nix; use base qw/Catalyst::View/; sub process { @@ -17,7 +16,7 @@ sub process { $c->response->content_type('text/x-nix-narinfo'); # !!! check MIME type - my ($deriver, $narHash, $time, $narSize, $refs) = queryPathInfo($storePath, 1); + my ($deriver, $narHash, $time, $narSize, $refs) = $MACHINE_LOCAL_STORE->queryPathInfo($storePath, 1); my $info; $info .= "StorePath: $storePath\n"; @@ -28,8 +27,8 @@ sub process { $info .= "References: " . join(" ", map { basename $_ } @{$refs}) . "\n"; if (defined $deriver) { $info .= "Deriver: " . basename $deriver . "\n"; - if (isValidPath($deriver)) { - my $drv = derivationFromPath($deriver); + if ($MACHINE_LOCAL_STORE->isValidPath($deriver)) { + my $drv = $MACHINE_LOCAL_STORE->derivationFromPath($deriver); $info .= "System: $drv->{platform}\n"; } } diff --git a/src/script/hydra-eval-jobset b/src/script/hydra-eval-jobset index 7ed7ebe8..72a386f5 100755 --- a/src/script/hydra-eval-jobset +++ b/src/script/hydra-eval-jobset @@ -85,14 +85,14 @@ sub attrsToSQL { # Fetch a store path from 'eval_substituter' if not already present. sub getPath { my ($path) = @_; - return 1 if isValidPath($path); + return 1 if $MACHINE_LOCAL_STORE->isValidPath($path); my $substituter = $config->{eval_substituter}; system("nix", "--experimental-features", "nix-command", "copy", "--from", $substituter, "--", $path) if defined $substituter; - return isValidPath($path); + return $MACHINE_LOCAL_STORE->isValidPath($path); } @@ -143,7 +143,7 @@ sub fetchInputBuild { , version => $version , outputName => $mainOutput->name }; - if (isValidPath($prevBuild->drvpath)) { + if ($MACHINE_LOCAL_STORE->isValidPath($prevBuild->drvpath)) { $result->{drvPath} = $prevBuild->drvpath; } @@ -233,7 +233,7 @@ sub fetchInputEval { my $out = $build->buildoutputs->find({ name => "out" }); next unless defined $out; # FIXME: Should we fail if the path is not valid? - next unless isValidPath($out->path); + next unless $MACHINE_LOCAL_STORE->isValidPath($out->path); $jobs->{$build->get_column('job')} = $out->path; } diff --git a/src/script/hydra-update-gc-roots b/src/script/hydra-update-gc-roots index 11eba7a6..130cbb68 100755 --- a/src/script/hydra-update-gc-roots +++ b/src/script/hydra-update-gc-roots @@ -5,7 +5,6 @@ use warnings; use File::Path; use File::stat; use File::Basename; -use Nix::Store; use Hydra::Config; use Hydra::Schema; use Hydra::Helper::Nix; @@ -47,7 +46,7 @@ sub keepBuild { $build->finished && ($build->buildstatus == 0 || $build->buildstatus == 6)) { foreach my $path (split / /, $build->get_column('outpaths')) { - if (isValidPath($path)) { + if ($MACHINE_LOCAL_STORE->isValidPath($path)) { addRoot $path; } else { print STDERR " warning: output ", $path, " has disappeared\n" if $build->finished; @@ -55,7 +54,7 @@ sub keepBuild { } } if (!$build->finished || ($keepFailedDrvs && $build->buildstatus != 0)) { - if (isValidPath($build->drvpath)) { + if ($MACHINE_LOCAL_STORE->isValidPath($build->drvpath)) { addRoot $build->drvpath; } else { print STDERR " warning: derivation ", $build->drvpath, " has disappeared\n"; diff --git a/t/s3-backup-test.pl b/t/s3-backup-test.pl index 9332ab15..a367981f 100644 --- a/t/s3-backup-test.pl +++ b/t/s3-backup-test.pl @@ -3,7 +3,6 @@ use warnings; use File::Basename; use Hydra::Model::DB; use Hydra::Helper::Nix; -use Nix::Store; use Cwd; my $db = Hydra::Model::DB->new; From 9db5d0a88daee61b8b4507e517d53cab7377a70d Mon Sep 17 00:00:00 2001 From: K900 Date: Fri, 23 Feb 2024 14:02:36 +0300 Subject: [PATCH 113/313] urlencode drv names when fetching logs Otherwise names with special characters like + break things. --- src/lib/Hydra/Controller/Build.pm | 5 +++-- src/lib/Hydra/Controller/Root.pm | 3 ++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/lib/Hydra/Controller/Build.pm b/src/lib/Hydra/Controller/Build.pm index c3869838..6b25ff80 100644 --- a/src/lib/Hydra/Controller/Build.pm +++ b/src/lib/Hydra/Controller/Build.pm @@ -15,6 +15,7 @@ use Nix::Config; use List::SomeUtils qw(all); use Encode; use JSON::PP; +use WWW::Form::UrlEncoded::PP qw(); use feature 'state'; @@ -141,7 +142,7 @@ sub view_nixlog : Chained('buildChain') PathPart('nixlog') { $c->stash->{step} = $step; my $drvPath = $step->drvpath; - my $log_uri = $c->uri_for($c->controller('Root')->action_for("log"), [basename($drvPath)]); + my $log_uri = $c->uri_for($c->controller('Root')->action_for("log"), [WWW::Form::UrlEncoded::PP::url_encode(basename($drvPath))]); showLog($c, $mode, $log_uri); } @@ -150,7 +151,7 @@ sub view_log : Chained('buildChain') PathPart('log') { my ($self, $c, $mode) = @_; my $drvPath = $c->stash->{build}->drvpath; - my $log_uri = $c->uri_for($c->controller('Root')->action_for("log"), [basename($drvPath)]); + my $log_uri = $c->uri_for($c->controller('Root')->action_for("log"), [WWW::Form::UrlEncoded::PP::url_encode(basename($drvPath))]); showLog($c, $mode, $log_uri); } diff --git a/src/lib/Hydra/Controller/Root.pm b/src/lib/Hydra/Controller/Root.pm index 548cfac3..e6c3049f 100644 --- a/src/lib/Hydra/Controller/Root.pm +++ b/src/lib/Hydra/Controller/Root.pm @@ -16,6 +16,7 @@ use List::Util qw[min max]; use List::SomeUtils qw{any}; use Net::Prometheus; use Types::Standard qw/StrMatch/; +use WWW::Form::UrlEncoded::PP qw(); use constant NARINFO_REGEX => qr{^([a-z0-9]{32})\.narinfo$}; # e.g.: https://hydra.example.com/realisations/sha256:a62128132508a3a32eef651d6467695944763602f226ac630543e947d9feb140!out.doi @@ -553,7 +554,7 @@ sub log :Local :Args(1) { my $logPrefix = $c->config->{log_prefix}; if (defined $logPrefix) { - $c->res->redirect($logPrefix . "log/" . basename($drvPath)); + $c->res->redirect($logPrefix . "log/" . WWW::Form::UrlEncoded::PP::url_encode(basename($drvPath))); } else { notFound($c, "The build log of $drvPath is not available."); } From 669617ab54a667623fdbbc07dfd7354b5d66286b Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Thu, 7 Mar 2024 18:44:13 +0100 Subject: [PATCH 114/313] Use `submit` event in login form It's a pet peeve from me when logging into my personal Hydra that I always have to press the button rather than hitting Return after entering my password. Reason for that is that the form doesn't have a "submit" button, so far it was always listened to the "click" event. Submit does that and you can hit Return alternatively. --- src/root/auth.tt | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/root/auth.tt b/src/root/auth.tt index d1539765..d49ba5bd 100644 --- a/src/root/auth.tt +++ b/src/root/auth.tt @@ -33,7 +33,7 @@