diff --git a/flake.lock b/flake.lock index 2ddc7c16..3c4c0f75 100644 --- a/flake.lock +++ b/flake.lock @@ -16,58 +16,58 @@ "type": "github" } }, - "lowdown-src": { + "libgit2": { "flake": false, "locked": { - "lastModified": 1633514407, - "narHash": "sha256-Dw32tiMjdK9t3ETl5fzGrutQTzh2rufgZV4A/BbxuD4=", - "owner": "kristapsdz", - "repo": "lowdown", - "rev": "d2c2b44ff6c27b936ec27358a2653caaef8f73b8", + "lastModified": 1697646580, + "narHash": "sha256-oX4Z3S9WtJlwvj0uH9HlYcWv+x1hqp8mhXl7HsLu2f0=", + "owner": "libgit2", + "repo": "libgit2", + "rev": "45fd9ed7ae1a9b74b957ef4f337bc3c8b3df01b5", "type": "github" }, "original": { - "owner": "kristapsdz", - "repo": "lowdown", + "owner": "libgit2", + "repo": "libgit2", "type": "github" } }, "nix": { "inputs": { "flake-compat": "flake-compat", - "lowdown-src": "lowdown-src", + "libgit2": "libgit2", "nixpkgs": [ "nixpkgs" ], "nixpkgs-regression": "nixpkgs-regression" }, "locked": { - "lastModified": 1706208340, - "narHash": "sha256-wNyHUEIiKKVs6UXrUzhP7RSJQv0A8jckgcuylzftl8k=", + "lastModified": 1715805674, + "narHash": "sha256-0CIBMECsA3ISJZrJcOTzi6wa3QENTKGLtOpYIOoxwxo=", "owner": "NixOS", "repo": "nix", - "rev": "2c4bb93ba5a97e7078896ebc36385ce172960e4e", + "rev": "ab48ea416a203e9ccefb70aa634e27477e4c1ac4", "type": "github" }, "original": { "owner": "NixOS", - "ref": "2.19-maintenance", + "ref": "2.20-maintenance", "repo": "nix", "type": "github" } }, "nixpkgs": { "locked": { - "lastModified": 1701615100, - "narHash": "sha256-7VI84NGBvlCTduw2aHLVB62NvCiZUlALLqBe5v684Aw=", + "lastModified": 1705033721, + "narHash": "sha256-K5eJHmL1/kev6WuqyqqbS1cdNnSidIZ3jeqJ7GbrYnQ=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "e9f06adb793d1cca5384907b3b8a4071d5d7cb19", + "rev": "a1982c92d8980a0114372973cbdfe0a307f1bdea", "type": "github" }, "original": { "owner": "NixOS", - "ref": "nixos-23.05", + "ref": "nixos-23.05-small", "repo": "nixpkgs", "type": "github" } diff --git a/flake.nix b/flake.nix index 30aa61c0..31a38416 100644 --- a/flake.nix +++ b/flake.nix @@ -1,8 +1,8 @@ { description = "A Nix-based continuous build system"; - inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.05"; - inputs.nix.url = "github:NixOS/nix/2.19-maintenance"; + inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.05-small"; + inputs.nix.url = "github:NixOS/nix/2.20-maintenance"; inputs.nix.inputs.nixpkgs.follows = "nixpkgs"; # TODO get rid of this once https://github.com/NixOS/nix/pull/9546 is @@ -180,13 +180,9 @@ root=d7f16a3412e01a43a414535b16007c6931d3a9c7 ''; + nixpkgs.config.permittedInsecurePackages = [ "gitea-1.19.4" ]; nix = { - distributedBuilds = true; - buildMachines = [{ - hostName = "localhost"; - systems = [ system ]; - }]; - binaryCaches = [ ]; + settings.substituters = [ ]; }; services.gitea = { enable = true; @@ -202,7 +198,7 @@ testScript = let scripts.mktoken = pkgs.writeText "token.sql" '' - INSERT INTO access_token (id, uid, name, created_unix, updated_unix, token_hash, token_salt, token_last_eight) VALUES (1, 1, 'hydra', 1617107360, 1617107360, 'a930f319ca362d7b49a4040ac0af74521c3a3c3303a86f327b01994430672d33b6ec53e4ea774253208686c712495e12a486', 'XRjWE9YW0g', '31d3a9c7'); + INSERT INTO access_token (id, uid, name, created_unix, updated_unix, token_hash, token_salt, token_last_eight, scope) VALUES (1, 1, 'hydra', 1617107360, 1617107360, 'a930f319ca362d7b49a4040ac0af74521c3a3c3303a86f327b01994430672d33b6ec53e4ea774253208686c712495e12a486', 'XRjWE9YW0g', '31d3a9c7', 'all'); ''; scripts.git-setup = pkgs.writeShellScript "setup.sh" '' @@ -357,9 +353,9 @@ response = json.loads(data) - assert len(response) == 2, "Expected exactly two status updates for latest commit!" - assert response[0]['status'] == "success", "Expected latest status to be success!" - assert response[1]['status'] == "pending", "Expected first status to be pending!" + assert len(response) == 2, "Expected exactly three status updates for latest commit (queued, finished)!" + assert response[0]['status'] == "success", "Expected finished status to be success!" + assert response[1]['status'] == "pending", "Expected queued status to be pending!" machine.shutdown() ''; diff --git a/src/hydra-eval-jobs/hydra-eval-jobs.cc b/src/hydra-eval-jobs/hydra-eval-jobs.cc index 5cfc3a52..1d7de74b 100644 --- a/src/hydra-eval-jobs/hydra-eval-jobs.cc +++ b/src/hydra-eval-jobs/hydra-eval-jobs.cc @@ -89,7 +89,7 @@ struct MyArgs : MixEvalArgs, MixCommonArgs, RootArgs static MyArgs myArgs; -static std::string queryMetaStrings(EvalState & state, DrvInfo & drv, const std::string & name, const std::string & subAttribute) +static std::string queryMetaStrings(EvalState & state, PackageInfo & drv, const std::string & name, const std::string & subAttribute) { Strings res; std::function rec; @@ -181,7 +181,7 @@ static void worker( // CA derivations do not have static output paths, so we // have to defensively not query output paths in case we // encounter one. - DrvInfo::Outputs outputs = drv->queryOutputs( + PackageInfo::Outputs outputs = drv->queryOutputs( !experimentalFeatureSettings.isEnabled(Xp::CaDerivations)); if (drv->querySystem() == "unknown") diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc index 02b82d2b..e28ca795 100644 --- a/src/hydra-queue-runner/build-remote.cc +++ b/src/hydra-queue-runner/build-remote.cc @@ -8,6 +8,7 @@ #include "build-result.hh" #include "path.hh" #include "serve-protocol.hh" +#include "serve-protocol-impl.hh" #include "state.hh" #include "current-process.hh" #include "processes.hh" @@ -20,12 +21,6 @@ using namespace nix; - -static void append(Strings & dst, const Strings & src) -{ - dst.insert(dst.end(), src.begin(), src.end()); -} - namespace nix::build_remote { static Strings extraStoreArgs(std::string & machine) @@ -48,63 +43,31 @@ static Strings extraStoreArgs(std::string & machine) return result; } -static void openConnection(::Machine::ptr machine, Path tmpDir, int stderrFD, SSHMaster::Connection & child) +static std::unique_ptr openConnection( + ::Machine::ptr machine, SSHMaster & master) { - std::string pgmName; - Pipe to, from; - to.create(); - from.create(); - - Strings argv; + Strings command = {"nix-store", "--serve", "--write"}; if (machine->isLocalhost()) { - pgmName = "nix-store"; - argv = {"nix-store", "--builders", "", "--serve", "--write"}; + command.push_back("--builders"); + command.push_back(""); } else { - pgmName = "ssh"; - auto sshName = machine->sshName; - Strings extraArgs = extraStoreArgs(sshName); - argv = {"ssh", sshName}; - if (machine->sshKey != "") append(argv, {"-i", machine->sshKey}); - if (machine->sshPublicHostKey != "") { - Path fileName = tmpDir + "/host-key"; - auto p = machine->sshName.find("@"); - std::string host = p != std::string::npos ? std::string(machine->sshName, p + 1) : machine->sshName; - writeFile(fileName, host + " " + machine->sshPublicHostKey + "\n"); - append(argv, {"-oUserKnownHostsFile=" + fileName}); - } - append(argv, - { "-x", "-a", "-oBatchMode=yes", "-oConnectTimeout=60", "-oTCPKeepAlive=yes" - , "--", "nix-store", "--serve", "--write" }); - append(argv, extraArgs); + command.splice(command.end(), extraStoreArgs(machine->sshName)); } - child.sshPid = startProcess([&]() { - restoreProcessContext(); - - if (dup2(to.readSide.get(), STDIN_FILENO) == -1) - throw SysError("cannot dup input pipe to stdin"); - - if (dup2(from.writeSide.get(), STDOUT_FILENO) == -1) - throw SysError("cannot dup output pipe to stdout"); - - if (dup2(stderrFD, STDERR_FILENO) == -1) - throw SysError("cannot dup stderr"); - - execvp(argv.front().c_str(), (char * *) stringsToCharPtrs(argv).data()); // FIXME: remove cast - - throw SysError("cannot start %s", pgmName); + auto ret = master.startCommand(std::move(command), { + "-a", "-oBatchMode=yes", "-oConnectTimeout=60", "-oTCPKeepAlive=yes" }); - to.readSide = -1; - from.writeSide = -1; - - child.in = to.writeSide.release(); - child.out = from.readSide.release(); - // XXX: determine the actual max value we can use from /proc. + + // FIXME: Should this be upstreamed into `startCommand` in Nix? + int pipesize = 1024 * 1024; - fcntl(child.in.get(), F_SETPIPE_SZ, &pipesize); - fcntl(child.out.get(), F_SETPIPE_SZ, &pipesize); + + fcntl(ret->in.get(), F_SETPIPE_SZ, &pipesize); + fcntl(ret->out.get(), F_SETPIPE_SZ, &pipesize); + + return ret; } @@ -122,13 +85,10 @@ static void copyClosureTo( garbage-collect paths that are already there. Optionally, ask the remote host to substitute missing paths. */ // FIXME: substitute output pollutes our build log - conn.to << ServeProto::Command::QueryValidPaths << 1 << useSubstitutes; - ServeProto::write(destStore, conn, closure); - conn.to.flush(); - /* Get back the set of paths that are already valid on the remote host. */ - auto present = ServeProto::Serialise::read(destStore, conn); + auto present = conn.queryValidPaths( + destStore, true, closure, useSubstitutes); if (present.size() == closure.size()) return; @@ -153,7 +113,7 @@ static void copyClosureTo( // FIXME: use Store::topoSortPaths(). -static StorePaths reverseTopoSortPaths(const std::map & paths) +static StorePaths reverseTopoSortPaths(const std::map & paths) { StorePaths sorted; StorePathSet visited; @@ -194,28 +154,6 @@ static std::pair openLogFile(const std::string & logDir, cons return {std::move(logFile), std::move(logFD)}; } -/** - * @param conn is not fully initialized; it is this functions job to set - * the `remoteVersion` field after the handshake is completed. - * Therefore, no `ServeProto::Serialize` functions can be used until - * that field is set. - */ -static void handshake(::Machine::Connection & conn, unsigned int repeats) -{ - conn.to << SERVE_MAGIC_1 << 0x206; - conn.to.flush(); - - unsigned int magic = readInt(conn.from); - if (magic != SERVE_MAGIC_2) - throw Error("protocol mismatch with ‘nix-store --serve’ on ‘%1%’", conn.machine->sshName); - conn.remoteVersion = readInt(conn.from); - // Now `conn` is initialized. - if (GET_PROTOCOL_MAJOR(conn.remoteVersion) != 0x200) - throw Error("unsupported ‘nix-store --serve’ protocol version on ‘%1%’", conn.machine->sshName); - if (GET_PROTOCOL_MINOR(conn.remoteVersion) < 3 && repeats > 0) - throw Error("machine ‘%1%’ does not support repeating a build; please upgrade it to Nix 1.12", conn.machine->sshName); -} - static BasicDerivation sendInputs( State & state, Step & step, @@ -286,21 +224,11 @@ static BuildResult performBuild( Store & localStore, StorePath drvPath, const BasicDerivation & drv, - const State::BuildOptions & options, + const ServeProto::BuildOptions & options, counter & nrStepsBuilding ) { - conn.to << ServeProto::Command::BuildDerivation << localStore.printStorePath(drvPath); - writeDerivation(conn.to, localStore, drv); - conn.to << options.maxSilentTime << options.buildTimeout; - if (GET_PROTOCOL_MINOR(conn.remoteVersion) >= 2) - conn.to << options.maxLogSize; - if (GET_PROTOCOL_MINOR(conn.remoteVersion) >= 3) { - conn.to - << options.repeats // == build-repeat - << options.enforceDeterminism; - } - conn.to.flush(); + conn.putBuildDerivationRequest(localStore, drvPath, drv, options); BuildResult result; @@ -350,7 +278,7 @@ static BuildResult performBuild( return result; } -static std::map queryPathInfos( +static std::map queryPathInfos( ::Machine::Connection & conn, Store & localStore, StorePathSet & outputs, @@ -359,30 +287,18 @@ static std::map queryPathInfos( { /* Get info about each output path. */ - std::map infos; + std::map infos; conn.to << ServeProto::Command::QueryPathInfos; ServeProto::write(localStore, conn, outputs); conn.to.flush(); while (true) { auto storePathS = readString(conn.from); if (storePathS == "") break; - auto deriver = readString(conn.from); // deriver - auto references = ServeProto::Serialise::read(localStore, conn); - readLongLong(conn.from); // download size - auto narSize = readLongLong(conn.from); - auto narHash = Hash::parseAny(readString(conn.from), htSHA256); - auto ca = ContentAddress::parseOpt(readString(conn.from)); - readStrings(conn.from); // sigs - ValidPathInfo info(localStore.parseStorePath(storePathS), narHash); - assert(outputs.count(info.path)); - info.references = references; - info.narSize = narSize; + + auto storePath = localStore.parseStorePath(storePathS); + auto info = ServeProto::Serialise::read(localStore, conn); totalNarSize += info.narSize; - info.narHash = narHash; - info.ca = ca; - if (deriver != "") - info.deriver = localStore.parseStorePath(deriver); - infos.insert_or_assign(info.path, info); + infos.insert_or_assign(std::move(storePath), std::move(info)); } return infos; @@ -423,14 +339,16 @@ static void copyPathsFromRemote( NarMemberDatas & narMembers, Store & localStore, Store & destStore, - const std::map & infos + const std::map & infos ) { auto pathsSorted = reverseTopoSortPaths(infos); for (auto & path : pathsSorted) { auto & info = infos.find(path)->second; - copyPathFromRemote(conn, narMembers, localStore, destStore, info); + copyPathFromRemote( + conn, narMembers, localStore, destStore, + ValidPathInfo { path, info }); } } @@ -508,7 +426,7 @@ private: void State::buildRemote(ref destStore, MachineReservation::ptr & reservation, ::Machine::ptr machine, Step::ptr step, - const BuildOptions & buildOptions, + const ServeProto::BuildOptions & buildOptions, RemoteResult & result, std::shared_ptr activeStep, std::function updateStep, NarMemberDatas & narMembers) @@ -519,21 +437,26 @@ void State::buildRemote(ref destStore, AutoDelete logFileDel(logFile, false); result.logFile = logFile; - nix::Path tmpDir = createTempDir(); - AutoDelete tmpDirDel(tmpDir, true); - try { updateStep(ssConnecting); + SSHMaster master { + machine->sshName, + machine->sshKey, + machine->sshPublicHostKey, + false, // no SSH master yet + false, // no compression yet + logFD.get(), + }; + // FIXME: rewrite to use Store. - SSHMaster::Connection child; - build_remote::openConnection(machine, tmpDir, logFD.get(), child); + auto child = build_remote::openConnection(machine, master); { auto activeStepState(activeStep->state_.lock()); if (activeStepState->cancelled) throw Error("step cancelled"); - activeStepState->pid = child.sshPid; + activeStepState->pid = child->sshPid; } Finally clearPid([&]() { @@ -549,9 +472,13 @@ void State::buildRemote(ref destStore, }); ::Machine::Connection conn { - .from = child.out.get(), - .to = child.in.get(), - .machine = machine, + { + .to = child->in.get(), + .from = child->out.get(), + /* Handshake. */ + .remoteVersion = 0xdadbeef, // FIXME avoid dummy initialize + }, + /*.machine =*/ machine, }; Finally updateStats([&]() { @@ -559,14 +486,26 @@ void State::buildRemote(ref destStore, bytesSent += conn.to.written; }); + constexpr ServeProto::Version our_version = 0x206; + try { - build_remote::handshake(conn, buildOptions.repeats); + conn.remoteVersion = decltype(conn)::handshake( + conn.to, + conn.from, + our_version, + machine->sshName); } catch (EndOfFile & e) { - child.sshPid.wait(); + child->sshPid.wait(); std::string s = chomp(readFile(result.logFile)); throw Error("cannot connect to ‘%1%’: %2%", machine->sshName, s); } + // Do not attempt to speak a newer version of the protocol. + // + // Per https://github.com/NixOS/nix/issues/9584 should be handled as + // part of `handshake` in upstream nix. + conn.remoteVersion = std::min(conn.remoteVersion, our_version); + { auto info(machine->state->connectInfo.lock()); info->consecutiveFailures = 0; @@ -686,8 +625,8 @@ void State::buildRemote(ref destStore, } /* Shut down the connection. */ - child.in = -1; - child.sshPid.wait(); + child->in = -1; + child->sshPid.wait(); } catch (Error & e) { /* Disable this machine until a certain period of time has diff --git a/src/hydra-queue-runner/builder.cc b/src/hydra-queue-runner/builder.cc index eecf43c4..fade0fd5 100644 --- a/src/hydra-queue-runner/builder.cc +++ b/src/hydra-queue-runner/builder.cc @@ -100,10 +100,13 @@ State::StepResult State::doBuildStep(nix::ref destStore, it). */ BuildID buildId; std::optional buildDrvPath; - BuildOptions buildOptions; - buildOptions.repeats = step->isDeterministic ? 1 : 0; - buildOptions.maxLogSize = maxLogSize; - buildOptions.enforceDeterminism = step->isDeterministic; + // Other fields set below + nix::ServeProto::BuildOptions buildOptions { + .maxLogSize = maxLogSize, + .nrRepeats = step->isDeterministic ? 1u : 0u, + .enforceDeterminism = step->isDeterministic, + .keepFailed = false, + }; auto conn(dbPool.get()); @@ -138,7 +141,7 @@ State::StepResult State::doBuildStep(nix::ref destStore, { auto i = jobsetRepeats.find(std::make_pair(build2->projectName, build2->jobsetName)); if (i != jobsetRepeats.end()) - buildOptions.repeats = std::max(buildOptions.repeats, i->second); + buildOptions.nrRepeats = std::max(buildOptions.nrRepeats, i->second); } } if (!build) build = *dependents.begin(); @@ -149,7 +152,7 @@ State::StepResult State::doBuildStep(nix::ref destStore, buildOptions.buildTimeout = build->buildTimeout; printInfo("performing step ‘%s’ %d times on ‘%s’ (needed by build %d and %d others)", - localStore->printStorePath(step->drvPath), buildOptions.repeats + 1, machine->sshName, buildId, (dependents.size() - 1)); + localStore->printStorePath(step->drvPath), buildOptions.nrRepeats + 1, machine->sshName, buildId, (dependents.size() - 1)); } if (!buildOneDone) diff --git a/src/hydra-queue-runner/dispatcher.cc b/src/hydra-queue-runner/dispatcher.cc index 03b42433..e2c1608d 100644 --- a/src/hydra-queue-runner/dispatcher.cc +++ b/src/hydra-queue-runner/dispatcher.cc @@ -243,11 +243,11 @@ system_time State::doDispatch() sort(machinesSorted.begin(), machinesSorted.end(), [](const MachineInfo & a, const MachineInfo & b) -> bool { - float ta = std::round(a.currentJobs / a.machine->speedFactorFloat); - float tb = std::round(b.currentJobs / b.machine->speedFactorFloat); + float ta = std::round(a.currentJobs / a.machine->speedFactor); + float tb = std::round(b.currentJobs / b.machine->speedFactor); return ta != tb ? ta < tb : - a.machine->speedFactorFloat != b.machine->speedFactorFloat ? a.machine->speedFactorFloat > b.machine->speedFactorFloat : + a.machine->speedFactor != b.machine->speedFactor ? a.machine->speedFactor > b.machine->speedFactor : a.currentJobs > b.currentJobs; }); diff --git a/src/hydra-queue-runner/hydra-queue-runner.cc b/src/hydra-queue-runner/hydra-queue-runner.cc index b6eca0d1..073b387c 100644 --- a/src/hydra-queue-runner/hydra-queue-runner.cc +++ b/src/hydra-queue-runner/hydra-queue-runner.cc @@ -177,16 +177,16 @@ void State::parseMachines(const std::string & contents) auto machine = std::make_shared<::Machine>(nix::Machine { // `storeUri`, not yet used "", - // `systemTypes`, not yet used - {}, + // `systemTypes` + tokenizeString(tokens[1], ","), // `sshKey` tokens[2] == "-" ? "" : tokens[2], // `maxJobs` tokens[3] != "" ? string2Int(tokens[3]).value() : 1, - // `speedFactor`, not yet used - 1, + // `speedFactor` + atof(tokens[4].c_str()), // `supportedFeatures` std::move(supportedFeatures), // `mandatoryFeatures` @@ -198,8 +198,6 @@ void State::parseMachines(const std::string & contents) }); machine->sshName = tokens[0]; - machine->systemTypesSet = tokenizeString(tokens[1], ","); - machine->speedFactorFloat = atof(tokens[4].c_str()); /* Re-use the State object of the previous machine with the same name. */ @@ -663,7 +661,7 @@ void State::dumpStatus(Connection & conn) json machine = { {"enabled", m->enabled}, - {"systemTypes", m->systemTypesSet}, + {"systemTypes", m->systemTypes}, {"supportedFeatures", m->supportedFeatures}, {"mandatoryFeatures", m->mandatoryFeatures}, {"nrStepsDone", s->nrStepsDone.load()}, diff --git a/src/hydra-queue-runner/nar-extractor.cc b/src/hydra-queue-runner/nar-extractor.cc index 3c6857bf..61299ecd 100644 --- a/src/hydra-queue-runner/nar-extractor.cc +++ b/src/hydra-queue-runner/nar-extractor.cc @@ -6,7 +6,46 @@ using namespace nix; -struct Extractor : ParseSink + +struct NarMemberConstructor : CreateRegularFileSink +{ + NarMemberData & curMember; + + HashSink hashSink = HashSink { HashAlgorithm::SHA256 }; + + std::optional expectedSize; + + NarMemberConstructor(NarMemberData & curMember) + : curMember(curMember) + { } + + void isExecutable() override + { + } + + void preallocateContents(uint64_t size) override + { + expectedSize = size; + } + + void operator () (std::string_view data) override + { + assert(expectedSize); + *curMember.fileSize += data.size(); + hashSink(data); + if (curMember.contents) { + curMember.contents->append(data); + } + assert(curMember.fileSize <= expectedSize); + if (curMember.fileSize == expectedSize) { + auto [hash, len] = hashSink.finish(); + assert(curMember.fileSize == len); + curMember.sha256 = hash; + } + } +}; + +struct Extractor : FileSystemObjectSink { std::unordered_set filesToKeep { "/nix-support/hydra-build-products", @@ -15,7 +54,6 @@ struct Extractor : ParseSink }; NarMemberDatas & members; - NarMemberData * curMember = nullptr; Path prefix; Extractor(NarMemberDatas & members, const Path & prefix) @@ -27,53 +65,22 @@ struct Extractor : ParseSink members.insert_or_assign(prefix + path, NarMemberData { .type = SourceAccessor::Type::tDirectory }); } - void createRegularFile(const Path & path) override + void createRegularFile(const Path & path, std::function func) override { - curMember = &members.insert_or_assign(prefix + path, NarMemberData { - .type = SourceAccessor::Type::tRegular, - .fileSize = 0, - .contents = filesToKeep.count(path) ? std::optional("") : std::nullopt, - }).first->second; - } - - std::optional expectedSize; - std::unique_ptr hashSink; - - void preallocateContents(uint64_t size) override - { - expectedSize = size; - hashSink = std::make_unique(htSHA256); - } - - void receiveContents(std::string_view data) override - { - assert(expectedSize); - assert(curMember); - assert(hashSink); - *curMember->fileSize += data.size(); - (*hashSink)(data); - if (curMember->contents) { - curMember->contents->append(data); - } - assert(curMember->fileSize <= expectedSize); - if (curMember->fileSize == expectedSize) { - auto [hash, len] = hashSink->finish(); - assert(curMember->fileSize == len); - curMember->sha256 = hash; - hashSink.reset(); - } + NarMemberConstructor nmc { + members.insert_or_assign(prefix + path, NarMemberData { + .type = SourceAccessor::Type::tRegular, + .fileSize = 0, + .contents = filesToKeep.count(path) ? std::optional("") : std::nullopt, + }).first->second, + }; + func(nmc); } void createSymlink(const Path & path, const std::string & target) override { members.insert_or_assign(prefix + path, NarMemberData { .type = SourceAccessor::Type::tSymlink }); } - - void isExecutable() override - { } - - void closeRegularFile() override - { } }; diff --git a/src/hydra-queue-runner/queue-monitor.cc b/src/hydra-queue-runner/queue-monitor.cc index 4202bb6d..0e15f2fa 100644 --- a/src/hydra-queue-runner/queue-monitor.cc +++ b/src/hydra-queue-runner/queue-monitor.cc @@ -726,7 +726,7 @@ BuildOutput State::getBuildOutputCached(Connection & conn, nix::ref product.fileSize = row[2].as(); } if (!row[3].is_null()) - product.sha256hash = Hash::parseAny(row[3].as(), htSHA256); + product.sha256hash = Hash::parseAny(row[3].as(), HashAlgorithm::SHA256); if (!row[4].is_null()) product.path = row[4].as(); product.name = row[5].as(); diff --git a/src/hydra-queue-runner/state.hh b/src/hydra-queue-runner/state.hh index 49968558..6494bbc8 100644 --- a/src/hydra-queue-runner/state.hh +++ b/src/hydra-queue-runner/state.hh @@ -23,6 +23,7 @@ #include "sync.hh" #include "nar-extractor.hh" #include "serve-protocol.hh" +#include "serve-protocol-impl.hh" #include "machines.hh" @@ -245,14 +246,6 @@ struct Machine : nix::Machine we are not yet used to, but once we are, we don't need this. */ std::string sshName; - /* TODO Get rid once `nix::Machine::systemTypes` is a set not - vector. */ - std::set systemTypesSet; - - /* TODO Get rid once `nix::Machine::systemTypes` is a `float` not - an `int`. */ - float speedFactorFloat = 1.0; - struct State { typedef std::shared_ptr ptr; counter currentJobs{0}; @@ -279,7 +272,7 @@ struct Machine : nix::Machine { /* Check that this machine is of the type required by the step. */ - if (!systemTypesSet.count(step->drv->platform == "builtin" ? nix::settings.thisSystem : step->drv->platform)) + if (!systemTypes.count(step->drv->platform == "builtin" ? nix::settings.thisSystem : step->drv->platform)) return false; /* Check that the step requires all mandatory features of this @@ -309,29 +302,9 @@ struct Machine : nix::Machine } // A connection to a machine - struct Connection { - nix::FdSource from; - nix::FdSink to; - nix::ServeProto::Version remoteVersion; - + struct Connection : nix::ServeProto::BasicClientConnection { // Backpointer to the machine ptr machine; - - operator nix::ServeProto::ReadConn () - { - return { - .from = from, - .version = remoteVersion, - }; - } - - operator nix::ServeProto::WriteConn () - { - return { - .to = to, - .version = remoteVersion, - }; - } }; }; @@ -470,7 +443,7 @@ private: /* How often the build steps of a jobset should be repeated in order to detect non-determinism. */ - std::map, unsigned int> jobsetRepeats; + std::map, size_t> jobsetRepeats; bool uploadLogsToBinaryCache; @@ -504,12 +477,6 @@ private: public: State(std::optional metricsAddrOpt); - struct BuildOptions { - unsigned int maxSilentTime, buildTimeout, repeats; - size_t maxLogSize; - bool enforceDeterminism; - }; - private: nix::MaintainCount startDbUpdate(); @@ -600,7 +567,7 @@ private: void buildRemote(nix::ref destStore, MachineReservation::ptr & reservation, Machine::ptr machine, Step::ptr step, - const BuildOptions & buildOptions, + const nix::ServeProto::BuildOptions & buildOptions, RemoteResult & result, std::shared_ptr activeStep, std::function updateStep, NarMemberDatas & narMembers); diff --git a/src/lib/Hydra/Plugin/GiteaStatus.pm b/src/lib/Hydra/Plugin/GiteaStatus.pm index 426c93f5..f3498941 100644 --- a/src/lib/Hydra/Plugin/GiteaStatus.pm +++ b/src/lib/Hydra/Plugin/GiteaStatus.pm @@ -88,10 +88,6 @@ sub buildQueued { common(@_, [], 0); } -sub buildStarted { - common(@_, [], 1); -} - sub buildFinished { common(@_, 2); }