From 9ba4417940ffdd0fadea43f68c61ef948a4b8d39 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 4 Dec 2023 16:05:50 -0500 Subject: [PATCH 01/32] Prepare for CA derivation support with lower impact changes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is just C++ changes without any Perl / Frontend / SQL Schema changes. The idea is that it should be possible to redeploy Hydra with these chnages with (a) no schema migration and also (b) no regressions. We should be able to much more safely deploy these to a staging server and then production `hydra.nixos.org`. Extracted from #875 Co-Authored-By: ThΓ©ophane Hufschmitt Co-Authored-By: Alexander Sosedkin Co-Authored-By: Andrea Ciceri Co-Authored-By: Charlotte 🦝 Delenk Mlotte@chir.rs> Co-Authored-By: Sandro JΓ€ckel --- src/hydra-eval-jobs/hydra-eval-jobs.cc | 16 ++- src/hydra-queue-runner/build-remote.cc | 105 +++++++++++--- src/hydra-queue-runner/build-result.cc | 19 ++- src/hydra-queue-runner/builder.cc | 8 +- src/hydra-queue-runner/hydra-build-result.hh | 4 +- src/hydra-queue-runner/hydra-queue-runner.cc | 27 +++- src/hydra-queue-runner/queue-monitor.cc | 137 ++++++++++--------- src/hydra-queue-runner/state.hh | 4 +- 8 files changed, 213 insertions(+), 107 deletions(-) diff --git a/src/hydra-eval-jobs/hydra-eval-jobs.cc b/src/hydra-eval-jobs/hydra-eval-jobs.cc index 2fe2c80f..d007189d 100644 --- a/src/hydra-eval-jobs/hydra-eval-jobs.cc +++ b/src/hydra-eval-jobs/hydra-eval-jobs.cc @@ -178,7 +178,11 @@ static void worker( if (auto drv = getDerivation(state, *v, false)) { - DrvInfo::Outputs outputs = drv->queryOutputs(); + // CA derivations do not have static output paths, so we + // have to defensively not query output paths in case we + // encounter one. + DrvInfo::Outputs outputs = drv->queryOutputs( + !experimentalFeatureSettings.isEnabled(Xp::CaDerivations)); if (drv->querySystem() == "unknown") throw EvalError("derivation must have a 'system' attribute"); @@ -239,12 +243,12 @@ static void worker( } nlohmann::json out; - for (auto & j : outputs) - // FIXME: handle CA/impure builds. - if (j.second) - out[j.first] = state.store->printStorePath(*j.second); + for (auto & [outputName, optOutputPath] : outputs) + if (optOutputPath) + out[outputName] = state.store->printStorePath(*optOutputPath); + else + out[outputName] = ""; job["outputs"] = std::move(out); - reply["job"] = std::move(job); } diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc index 73f46a53..26f6d63f 100644 --- a/src/hydra-queue-runner/build-remote.cc +++ b/src/hydra-queue-runner/build-remote.cc @@ -182,6 +182,41 @@ static StorePaths reverseTopoSortPaths(const std::map return sorted; } +/** + * Replace the input derivations by their output paths to send a minimal closure + * to the builder. + * + * If we can afford it, resolve it, so that the newly generated derivation still + * has some sensible output paths. + */ +BasicDerivation inlineInputDerivations(Store & store, Derivation & drv, const StorePath & drvPath) +{ + BasicDerivation ret; + auto outputHashes = staticOutputHashes(store, drv); + if (!drv.type().hasKnownOutputPaths()) { + auto maybeBasicDrv = drv.tryResolve(store); + if (!maybeBasicDrv) + throw Error( + "the derivation '%s' can’t be resolved. It’s probably " + "missing some outputs", + store.printStorePath(drvPath)); + ret = *maybeBasicDrv; + } else { + // If the derivation is a real `InputAddressed` derivation, we must + // resolve it manually to keep the original output paths + ret = BasicDerivation(drv); + for (auto & [drvPath, node] : drv.inputDrvs.map) { + auto drv2 = store.readDerivation(drvPath); + auto drv2Outputs = drv2.outputsAndOptPaths(store); + for (auto & name : node.value) { + auto inputPath = drv2Outputs.at(name); + ret.inputSrcs.insert(*inputPath.second); + } + } + } + return ret; +} + static std::pair openLogFile(const std::string & logDir, const StorePath & drvPath) { std::string base(drvPath.to_string()); @@ -228,17 +263,7 @@ static BasicDerivation sendInputs( counter & nrStepsCopyingTo ) { - BasicDerivation basicDrv(*step.drv); - - for (const auto & [drvPath, node] : step.drv->inputDrvs.map) { - auto drv2 = localStore.readDerivation(drvPath); - for (auto & name : node.value) { - if (auto i = get(drv2.outputs, name)) { - auto outPath = i->path(localStore, drv2.name, name); - basicDrv.inputSrcs.insert(*outPath); - } - } - } + BasicDerivation basicDrv = inlineInputDerivations(localStore, *step.drv, step.drvPath); /* Ensure that the inputs exist in the destination store. This is a no-op for regular stores, but for the binary cache store, @@ -323,8 +348,33 @@ static BuildResult performBuild( result.stopTime = stop; } } + + // Get the newly built outputs, either from the remote if it + // supports it, or by introspecting the derivation if the remote is + // too old. if (GET_PROTOCOL_MINOR(conn.remoteVersion) >= 6) { - ServeProto::Serialise::read(localStore, conn); + auto builtOutputs = ServeProto::Serialise::read(localStore, conn); + for (auto && [output, realisation] : builtOutputs) + result.builtOutputs.insert_or_assign( + std::move(output.outputName), + std::move(realisation)); + } else { + // If the remote is too old to handle CA derivations, we can’t get this + // far anyways + assert(drv.type().hasKnownOutputPaths()); + DerivationOutputsAndOptPaths drvOutputs = drv.outputsAndOptPaths(localStore); + auto outputHashes = staticOutputHashes(localStore, drv); + for (auto & [outputName, output] : drvOutputs) { + auto outputPath = output.second; + // We’ve just asserted that the output paths of the derivation + // were known + assert(outputPath); + auto outputHash = outputHashes.at(outputName); + auto drvOutput = DrvOutput { outputHash, outputName }; + result.builtOutputs.insert_or_assign( + std::move(outputName), + Realisation { drvOutput, *outputPath }); + } } return result; @@ -376,6 +426,7 @@ static void copyPathFromRemote( const ValidPathInfo & info ) { + for (auto * store : {&destStore, &localStore}) { /* Receive the NAR from the remote and add it to the destination store. Meanwhile, extract all the info from the NAR that getBuildOutput() needs. */ @@ -395,7 +446,8 @@ static void copyPathFromRemote( extractNarData(tee, localStore.printStorePath(info.path), narMembers); }); - destStore.addToStore(info, *source2, NoRepair, NoCheckSigs); + store->addToStore(info, *source2, NoRepair, NoCheckSigs); + } } static void copyPathsFromRemote( @@ -592,6 +644,10 @@ void State::buildRemote(ref destStore, result.logFile = ""; } + StorePathSet outputs; + for (auto & [_, realisation] : buildResult.builtOutputs) + outputs.insert(realisation.outPath); + /* Copy the output paths. */ if (!machine->isLocalhost() || localStore != std::shared_ptr(destStore)) { updateStep(ssReceivingOutputs); @@ -600,12 +656,6 @@ void State::buildRemote(ref destStore, auto now1 = std::chrono::steady_clock::now(); - StorePathSet outputs; - for (auto & i : step->drv->outputsAndOptPaths(*localStore)) { - if (i.second.second) - outputs.insert(*i.second.second); - } - size_t totalNarSize = 0; auto infos = build_remote::queryPathInfos(conn, *localStore, outputs, totalNarSize); @@ -624,6 +674,23 @@ void State::buildRemote(ref destStore, result.overhead += std::chrono::duration_cast(now2 - now1).count(); } + /* Register the outputs of the newly built drv */ + if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { + auto outputHashes = staticOutputHashes(*localStore, *step->drv); + for (auto & [outputName, realisation] : buildResult.builtOutputs) { + // Register the resolved drv output + localStore->registerDrvOutput(realisation); + destStore->registerDrvOutput(realisation); + + // Also register the unresolved one + auto unresolvedRealisation = realisation; + unresolvedRealisation.signatures.clear(); + unresolvedRealisation.id.drvHash = outputHashes.at(outputName); + localStore->registerDrvOutput(unresolvedRealisation); + destStore->registerDrvOutput(unresolvedRealisation); + } + } + /* Shut down the connection. */ child.to = -1; child.pid.wait(); diff --git a/src/hydra-queue-runner/build-result.cc b/src/hydra-queue-runner/build-result.cc index 691c1f19..ffdc37b7 100644 --- a/src/hydra-queue-runner/build-result.cc +++ b/src/hydra-queue-runner/build-result.cc @@ -11,18 +11,18 @@ using namespace nix; BuildOutput getBuildOutput( nix::ref store, NarMemberDatas & narMembers, - const Derivation & drv) + const OutputPathMap derivationOutputs) { BuildOutput res; /* Compute the closure size. */ StorePathSet outputs; StorePathSet closure; - for (auto & i : drv.outputsAndOptPaths(*store)) - if (i.second.second) { - store->computeFSClosure(*i.second.second, closure); - outputs.insert(*i.second.second); - } + for (auto& [outputName, outputPath] : derivationOutputs) { + store->computeFSClosure(outputPath, closure); + outputs.insert(outputPath); + res.outputs.insert({outputName, outputPath}); + } for (auto & path : closure) { auto info = store->queryPathInfo(path); res.closureSize += info->narSize; @@ -107,13 +107,12 @@ BuildOutput getBuildOutput( /* If no build products were explicitly declared, then add all outputs as a product of type "nix-build". */ if (!explicitProducts) { - for (auto & [name, output] : drv.outputs) { + for (auto & [name, output] : derivationOutputs) { BuildProduct product; - auto outPath = output.path(*store, drv.name, name); - product.path = store->printStorePath(*outPath); + product.path = store->printStorePath(output); product.type = "nix-build"; product.subtype = name == "out" ? "" : name; - product.name = outPath->name(); + product.name = output.name(); auto file = narMembers.find(product.path); assert(file != narMembers.end()); diff --git a/src/hydra-queue-runner/builder.cc b/src/hydra-queue-runner/builder.cc index 307eee8e..42983941 100644 --- a/src/hydra-queue-runner/builder.cc +++ b/src/hydra-queue-runner/builder.cc @@ -223,7 +223,7 @@ State::StepResult State::doBuildStep(nix::ref destStore, if (result.stepStatus == bsSuccess) { updateStep(ssPostProcessing); - res = getBuildOutput(destStore, narMembers, *step->drv); + res = getBuildOutput(destStore, narMembers, localStore->queryDerivationOutputMap(step->drvPath)); } } @@ -277,9 +277,9 @@ State::StepResult State::doBuildStep(nix::ref destStore, assert(stepNr); - for (auto & i : step->drv->outputsAndOptPaths(*localStore)) { - if (i.second.second) - addRoot(*i.second.second); + for (auto & i : localStore->queryPartialDerivationOutputMap(step->drvPath)) { + if (i.second) + addRoot(*i.second); } /* Register success in the database for all Build objects that diff --git a/src/hydra-queue-runner/hydra-build-result.hh b/src/hydra-queue-runner/hydra-build-result.hh index a3f71ae9..7d47f67c 100644 --- a/src/hydra-queue-runner/hydra-build-result.hh +++ b/src/hydra-queue-runner/hydra-build-result.hh @@ -36,10 +36,12 @@ struct BuildOutput std::list products; + std::map outputs; + std::map metrics; }; BuildOutput getBuildOutput( nix::ref store, NarMemberDatas & narMembers, - const nix::Derivation & drv); + const nix::OutputPathMap derivationOutputs); diff --git a/src/hydra-queue-runner/hydra-queue-runner.cc b/src/hydra-queue-runner/hydra-queue-runner.cc index 1d54bb93..2f2c6091 100644 --- a/src/hydra-queue-runner/hydra-queue-runner.cc +++ b/src/hydra-queue-runner/hydra-queue-runner.cc @@ -312,10 +312,10 @@ unsigned int State::createBuildStep(pqxx::work & txn, time_t startTime, BuildID if (r.affected_rows() == 0) goto restart; - for (auto & [name, output] : step->drv->outputs) + for (auto & [name, output] : localStore->queryPartialDerivationOutputMap(step->drvPath)) txn.exec_params0 ("insert into BuildStepOutputs (build, stepnr, name, path) values ($1, $2, $3, $4)", - buildId, stepNr, name, localStore->printStorePath(*output.path(*localStore, step->drv->name, name))); + buildId, stepNr, name, output ? localStore->printStorePath(*output) : ""); if (status == bsBusy) txn.exec(fmt("notify step_started, '%d\t%d'", buildId, stepNr)); @@ -352,11 +352,23 @@ void State::finishBuildStep(pqxx::work & txn, const RemoteResult & result, assert(result.logFile.find('\t') == std::string::npos); txn.exec(fmt("notify step_finished, '%d\t%d\t%s'", buildId, stepNr, result.logFile)); + + if (result.stepStatus == bsSuccess) { + // Update the corresponding `BuildStepOutputs` row to add the output path + auto res = txn.exec_params1("select drvPath from BuildSteps where build = $1 and stepnr = $2", buildId, stepNr); + assert(res.size()); + StorePath drvPath = localStore->parseStorePath(res[0].as()); + // If we've finished building, all the paths should be known + for (auto & [name, output] : localStore->queryDerivationOutputMap(drvPath)) + txn.exec_params0 + ("update BuildStepOutputs set path = $4 where build = $1 and stepnr = $2 and name = $3", + buildId, stepNr, name, localStore->printStorePath(output)); + } } int State::createSubstitutionStep(pqxx::work & txn, time_t startTime, time_t stopTime, - Build::ptr build, const StorePath & drvPath, const std::string & outputName, const StorePath & storePath) + Build::ptr build, const StorePath & drvPath, const nix::Derivation drv, const std::string & outputName, const StorePath & storePath) { restart: auto stepNr = allocBuildStep(txn, build->id); @@ -457,6 +469,15 @@ void State::markSucceededBuild(pqxx::work & txn, Build::ptr build, res.releaseName != "" ? std::make_optional(res.releaseName) : std::nullopt, isCachedBuild ? 1 : 0); + for (auto & [outputName, outputPath] : res.outputs) { + txn.exec_params0 + ("update BuildOutputs set path = $3 where build = $1 and name = $2", + build->id, + outputName, + localStore->printStorePath(outputPath) + ); + } + txn.exec_params0("delete from BuildProducts where build = $1", build->id); unsigned int productNr = 1; diff --git a/src/hydra-queue-runner/queue-monitor.cc b/src/hydra-queue-runner/queue-monitor.cc index 6c339af6..cee5b6cc 100644 --- a/src/hydra-queue-runner/queue-monitor.cc +++ b/src/hydra-queue-runner/queue-monitor.cc @@ -192,15 +192,14 @@ bool State::getQueuedBuilds(Connection & conn, if (!res[0].is_null()) propagatedFrom = res[0].as(); if (!propagatedFrom) { - for (auto & i : ex.step->drv->outputsAndOptPaths(*localStore)) { - if (i.second.second) { - auto res = txn.exec_params - ("select max(s.build) from BuildSteps s join BuildStepOutputs o on s.build = o.build where path = $1 and startTime != 0 and stopTime != 0 and status = 1", - localStore->printStorePath(*i.second.second)); - if (!res[0][0].is_null()) { - propagatedFrom = res[0][0].as(); - break; - } + for (auto & i : localStore->queryPartialDerivationOutputMap(ex.step->drvPath)) { + auto res = txn.exec_params + ("select max(s.build) from BuildSteps s join BuildStepOutputs o on s.build = o.build where drvPath = $1 and name = $2 and startTime != 0 and stopTime != 0 and status = 1", + localStore->printStorePath(ex.step->drvPath), + i.first); + if (!res[0][0].is_null()) { + propagatedFrom = res[0][0].as(); + break; } } } @@ -236,12 +235,10 @@ bool State::getQueuedBuilds(Connection & conn, /* If we didn't get a step, it means the step's outputs are all valid. So we mark this as a finished, cached build. */ if (!step) { - auto drv = localStore->readDerivation(build->drvPath); - BuildOutput res = getBuildOutputCached(conn, destStore, drv); + BuildOutput res = getBuildOutputCached(conn, destStore, build->drvPath); - for (auto & i : drv.outputsAndOptPaths(*localStore)) - if (i.second.second) - addRoot(*i.second.second); + for (auto & i : localStore->queryDerivationOutputMap(build->drvPath)) + addRoot(i.second); { auto mc = startDbUpdate(); @@ -481,26 +478,39 @@ Step::ptr State::createStep(ref destStore, throw PreviousFailure{step}; /* Are all outputs valid? */ + auto outputHashes = staticOutputHashes(*localStore, *(step->drv)); bool valid = true; - DerivationOutputs missing; - for (auto & i : step->drv->outputs) - if (!destStore->isValidPath(*i.second.path(*localStore, step->drv->name, i.first))) { - valid = false; - missing.insert_or_assign(i.first, i.second); + std::map> missing; + for (auto &[outputName, maybeOutputPath] : step->drv->outputsAndOptPaths(*destStore)) { + auto outputHash = outputHashes.at(outputName); + if (maybeOutputPath.second) { + if (!destStore->isValidPath(*maybeOutputPath.second)) { + valid = false; + missing.insert({{outputHash, outputName}, maybeOutputPath.second}); + } + } else { + experimentalFeatureSettings.require(Xp::CaDerivations); + if (!destStore->queryRealisation(DrvOutput{outputHash, outputName})) { + valid = false; + missing.insert({{outputHash, outputName}, std::nullopt}); + } } + } /* Try to copy the missing paths from the local store or from substitutes. */ if (!missing.empty()) { size_t avail = 0; - for (auto & i : missing) { - auto path = i.second.path(*localStore, step->drv->name, i.first); - if (/* localStore != destStore && */ localStore->isValidPath(*path)) + for (auto & [i, maybePath] : missing) { + if ((maybePath && localStore->isValidPath(*maybePath))) avail++; - else if (useSubstitutes) { + else if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations) && localStore->queryRealisation(i)) { + maybePath = localStore->queryRealisation(i)->outPath; + avail++; + } else if (useSubstitutes && maybePath) { SubstitutablePathInfos infos; - localStore->querySubstitutablePathInfos({{*path, {}}}, infos); + localStore->querySubstitutablePathInfos({{*maybePath, {}}}, infos); if (infos.size() == 1) avail++; } @@ -508,44 +518,44 @@ Step::ptr State::createStep(ref destStore, if (missing.size() == avail) { valid = true; - for (auto & i : missing) { - auto path = i.second.path(*localStore, step->drv->name, i.first); + for (auto & [i, path] : missing) { + if (path) { + try { + time_t startTime = time(0); - try { - time_t startTime = time(0); + if (localStore->isValidPath(*path)) + printInfo("copying output β€˜%1%’ of β€˜%2%’ from local store", + localStore->printStorePath(*path), + localStore->printStorePath(drvPath)); + else { + printInfo("substituting output β€˜%1%’ of β€˜%2%’", + localStore->printStorePath(*path), + localStore->printStorePath(drvPath)); + localStore->ensurePath(*path); + // FIXME: should copy directly from substituter to destStore. + } - if (localStore->isValidPath(*path)) - printInfo("copying output β€˜%1%’ of β€˜%2%’ from local store", + StorePathSet closure; + localStore->computeFSClosure({*path}, closure); + copyPaths(*localStore, *destStore, closure, NoRepair, CheckSigs, NoSubstitute); + + time_t stopTime = time(0); + + { + auto mc = startDbUpdate(); + pqxx::work txn(conn); + createSubstitutionStep(txn, startTime, stopTime, build, drvPath, *(step->drv), "out", *path); + txn.commit(); + } + + } catch (Error & e) { + printError("while copying/substituting output β€˜%s’ of β€˜%s’: %s", localStore->printStorePath(*path), - localStore->printStorePath(drvPath)); - else { - printInfo("substituting output β€˜%1%’ of β€˜%2%’", - localStore->printStorePath(*path), - localStore->printStorePath(drvPath)); - localStore->ensurePath(*path); - // FIXME: should copy directly from substituter to destStore. + localStore->printStorePath(drvPath), + e.what()); + valid = false; + break; } - - copyClosure(*localStore, *destStore, - StorePathSet { *path }, - NoRepair, CheckSigs, NoSubstitute); - - time_t stopTime = time(0); - - { - auto mc = startDbUpdate(); - pqxx::work txn(conn); - createSubstitutionStep(txn, startTime, stopTime, build, drvPath, "out", *path); - txn.commit(); - } - - } catch (Error & e) { - printError("while copying/substituting output β€˜%s’ of β€˜%s’: %s", - localStore->printStorePath(*path), - localStore->printStorePath(drvPath), - e.what()); - valid = false; - break; } } } @@ -640,17 +650,20 @@ void State::processJobsetSharesChange(Connection & conn) } -BuildOutput State::getBuildOutputCached(Connection & conn, nix::ref destStore, const nix::Derivation & drv) +BuildOutput State::getBuildOutputCached(Connection & conn, nix::ref destStore, const nix::StorePath & drvPath) { + + auto derivationOutputs = localStore->queryDerivationOutputMap(drvPath); + { pqxx::work txn(conn); - for (auto & [name, output] : drv.outputsAndOptPaths(*localStore)) { + for (auto & [name, output] : derivationOutputs) { auto r = txn.exec_params ("select id, buildStatus, releaseName, closureSize, size from Builds b " "join BuildOutputs o on b.id = o.build " "where finished = 1 and (buildStatus = 0 or buildStatus = 6) and path = $1", - localStore->printStorePath(*output.second)); + localStore->printStorePath(output)); if (r.empty()) continue; BuildID id = r[0][0].as(); @@ -704,5 +717,5 @@ BuildOutput State::getBuildOutputCached(Connection & conn, nix::ref } NarMemberDatas narMembers; - return getBuildOutput(destStore, narMembers, drv); + return getBuildOutput(destStore, narMembers, derivationOutputs); } diff --git a/src/hydra-queue-runner/state.hh b/src/hydra-queue-runner/state.hh index 6359063a..5f01de1e 100644 --- a/src/hydra-queue-runner/state.hh +++ b/src/hydra-queue-runner/state.hh @@ -520,7 +520,7 @@ private: const std::string & machine); int createSubstitutionStep(pqxx::work & txn, time_t startTime, time_t stopTime, - Build::ptr build, const nix::StorePath & drvPath, const std::string & outputName, const nix::StorePath & storePath); + Build::ptr build, const nix::StorePath & drvPath, const nix::Derivation drv, const std::string & outputName, const nix::StorePath & storePath); void updateBuild(pqxx::work & txn, Build::ptr build, BuildStatus status); @@ -536,7 +536,7 @@ private: void processQueueChange(Connection & conn); BuildOutput getBuildOutputCached(Connection & conn, nix::ref destStore, - const nix::Derivation & drv); + const nix::StorePath & drvPath); Step::ptr createStep(nix::ref store, Connection & conn, Build::ptr build, const nix::StorePath & drvPath, From 8046ec266851d8b8d3f08900cf9ba905ed344ed8 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 4 Dec 2023 16:21:56 -0500 Subject: [PATCH 02/32] Remove unused `outputHashes` variable This looks like a stray copy paste. --- src/hydra-queue-runner/build-remote.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc index 26f6d63f..e26a8697 100644 --- a/src/hydra-queue-runner/build-remote.cc +++ b/src/hydra-queue-runner/build-remote.cc @@ -192,7 +192,6 @@ static StorePaths reverseTopoSortPaths(const std::map BasicDerivation inlineInputDerivations(Store & store, Derivation & drv, const StorePath & drvPath) { BasicDerivation ret; - auto outputHashes = staticOutputHashes(store, drv); if (!drv.type().hasKnownOutputPaths()) { auto maybeBasicDrv = drv.tryResolve(store); if (!maybeBasicDrv) From e3443cd22a3bc20c7d17f4e76d3868ea755b4b37 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 4 Dec 2023 17:41:11 -0500 Subject: [PATCH 03/32] Put back nicer `copyClosure` instead of manual closure + copy It looks like we accidentally got the old code back, probably after a merge conflict resolution. --- src/hydra-queue-runner/queue-monitor.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/hydra-queue-runner/queue-monitor.cc b/src/hydra-queue-runner/queue-monitor.cc index cee5b6cc..04917932 100644 --- a/src/hydra-queue-runner/queue-monitor.cc +++ b/src/hydra-queue-runner/queue-monitor.cc @@ -535,9 +535,9 @@ Step::ptr State::createStep(ref destStore, // FIXME: should copy directly from substituter to destStore. } - StorePathSet closure; - localStore->computeFSClosure({*path}, closure); - copyPaths(*localStore, *destStore, closure, NoRepair, CheckSigs, NoSubstitute); + copyClosure(*localStore, *destStore, + StorePathSet { *path }, + NoRepair, CheckSigs, NoSubstitute); time_t stopTime = time(0); From 069b7775c565f5999fe33e8c3f28c7b9306039ca Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 5 Dec 2023 11:26:26 -0500 Subject: [PATCH 04/32] hydra-eval-jobs: Ensure we have output path if ca-derivations is disabled Brought up by @thufschmitt in https://github.com/NixOS/hydra/pull/1316#discussion_r1415111329 . This makes this closer to what was originally there --- which just dispatched off the experimental feature rather than the presence/absense of the output, too. --- src/hydra-eval-jobs/hydra-eval-jobs.cc | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/hydra-eval-jobs/hydra-eval-jobs.cc b/src/hydra-eval-jobs/hydra-eval-jobs.cc index d007189d..b44ccbb3 100644 --- a/src/hydra-eval-jobs/hydra-eval-jobs.cc +++ b/src/hydra-eval-jobs/hydra-eval-jobs.cc @@ -243,11 +243,16 @@ static void worker( } nlohmann::json out; - for (auto & [outputName, optOutputPath] : outputs) - if (optOutputPath) + for (auto & [outputName, optOutputPath] : outputs) { + if (optOutputPath) { out[outputName] = state.store->printStorePath(*optOutputPath); - else + } else { + // See the `queryOutputs` call above; we should + // not encounter missing output paths otherwise. + assert(experimentalFeatureSettings.isEnabled(Xp::CaDerivations)); out[outputName] = ""; + } + } job["outputs"] = std::move(out); reply["job"] = std::move(job); } From 3df8feb3a2b30b733b141f6e6e43572998311524 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 5 Dec 2023 11:27:52 -0500 Subject: [PATCH 05/32] Add TODO about setting `null` instead of empty string in JSON An empty string is a sneaky way to avoid hard failures --- things that expect strings still get strings, but it does conversely open the door up to soft failures (spooky-action-at-a-distance ones because the string did not have the expected invariants). "Fail fast" with null will ultimately make the system more robust, but force us to fix more things up front, and I don't want to change this without also fixing those things up front, especially as this commit is for now just part of the the preparatory PR for which this is dead code. --- src/hydra-eval-jobs/hydra-eval-jobs.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/hydra-eval-jobs/hydra-eval-jobs.cc b/src/hydra-eval-jobs/hydra-eval-jobs.cc index b44ccbb3..2794cc62 100644 --- a/src/hydra-eval-jobs/hydra-eval-jobs.cc +++ b/src/hydra-eval-jobs/hydra-eval-jobs.cc @@ -250,6 +250,10 @@ static void worker( // See the `queryOutputs` call above; we should // not encounter missing output paths otherwise. assert(experimentalFeatureSettings.isEnabled(Xp::CaDerivations)); + // TODO it would be better to set `null` than an + // empty string here, to force the consumer of + // this JSON to more explicitly handle this + // case. out[outputName] = ""; } } From 11f8030b0f4c75ed7640563d68200cf3ec59edec Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 6 Dec 2023 17:59:25 -0500 Subject: [PATCH 06/32] Add comment from GitHub about adding to store as code comment --- src/hydra-queue-runner/build-remote.cc | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc index e26a8697..94a01676 100644 --- a/src/hydra-queue-runner/build-remote.cc +++ b/src/hydra-queue-runner/build-remote.cc @@ -425,6 +425,19 @@ static void copyPathFromRemote( const ValidPathInfo & info ) { + // Why both stores? @thufschmitt says: + // + // > I think it's an easy (and terribly inefficient 😬) way of + // making sure that `localStore.queryRealisations` will succeed + // (which we IIRC we need later to get back some metadata about the + // path to put it in the db). + // > + // > To be honest, we shouldn't do that but instead carry the needed + // metadata in memory until the point where we need it (but that can + // come later once we're confident that this is at least correct) + // + // TODO make the above change to avoid copying excess data back and + // forth. for (auto * store : {&destStore, &localStore}) { /* Receive the NAR from the remote and add it to the destination store. Meanwhile, extract all the info from the From 2ee0068fdca7d9fb8bef23b3671a86b8f0fcff07 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 7 Dec 2023 15:05:03 -0500 Subject: [PATCH 07/32] Do not copy for both stores for now It has a performance cost, and as the comment says we should be doing the better solution. We want to land this preparatory change on prod while the rest is still on staging, so we should just skip it for now. Skipping it will not affect regular fixed-output and input-addressed derivations, which are the only ones prod would deal with upon getting this code. The main CA derivations support branch will revert this commit so it still works. --- src/hydra-queue-runner/build-remote.cc | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc index 1d0bcc35..77484244 100644 --- a/src/hydra-queue-runner/build-remote.cc +++ b/src/hydra-queue-runner/build-remote.cc @@ -425,20 +425,6 @@ static void copyPathFromRemote( const ValidPathInfo & info ) { - // Why both stores? @thufschmitt says: - // - // > I think it's an easy (and terribly inefficient 😬) way of - // making sure that `localStore.queryRealisations` will succeed - // (which we IIRC we need later to get back some metadata about the - // path to put it in the db). - // > - // > To be honest, we shouldn't do that but instead carry the needed - // metadata in memory until the point where we need it (but that can - // come later once we're confident that this is at least correct) - // - // TODO make the above change to avoid copying excess data back and - // forth. - for (auto * store : {&destStore, &localStore}) { /* Receive the NAR from the remote and add it to the destination store. Meanwhile, extract all the info from the NAR that getBuildOutput() needs. */ @@ -458,8 +444,7 @@ static void copyPathFromRemote( extractNarData(tee, localStore.printStorePath(info.path), narMembers); }); - store->addToStore(info, *source2, NoRepair, NoCheckSigs); - } + destStore.addToStore(info, *source2, NoRepair, NoCheckSigs); } static void copyPathsFromRemote( From ebfefb9161046c83bbbebd26f4cb37ac69628b40 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 11 Dec 2023 12:46:36 -0500 Subject: [PATCH 08/32] Sync up with some changes done to the main CA branch --- flake.lock | 8 +- flake.nix | 2 +- src/hydra-queue-runner/build-remote.cc | 55 +++------ src/hydra-queue-runner/builder.cc | 11 +- src/hydra-queue-runner/hydra-queue-runner.cc | 4 +- src/hydra-queue-runner/queue-monitor.cc | 119 ++++++++++--------- 6 files changed, 94 insertions(+), 105 deletions(-) diff --git a/flake.lock b/flake.lock index 2871e70a..e7f9224c 100644 --- a/flake.lock +++ b/flake.lock @@ -42,16 +42,16 @@ "nixpkgs-regression": "nixpkgs-regression" }, "locked": { - "lastModified": 1701122567, - "narHash": "sha256-iA8DqS+W2fWTfR+nNJSvMHqQ+4NpYMRT3b+2zS6JTvE=", + "lastModified": 1702314838, + "narHash": "sha256-calxK+fZ4/tZy1fbph8qyx4ePUAf4ZdvIugpzWeFIGE=", "owner": "NixOS", "repo": "nix", - "rev": "50f8f1c8bc019a4c0fd098b9ac674b94cfc6af0d", + "rev": "ae451e2247b18be6bd36b9d85e41b632e774f40b", "type": "github" }, "original": { "owner": "NixOS", - "ref": "2.19.2", + "ref": "2.19-maintenance", "repo": "nix", "type": "github" } diff --git a/flake.nix b/flake.nix index 5a5aef55..8280e076 100644 --- a/flake.nix +++ b/flake.nix @@ -2,7 +2,7 @@ description = "A Nix-based continuous build system"; inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.05"; - inputs.nix.url = "github:NixOS/nix/2.19.2"; + inputs.nix.url = "github:NixOS/nix/2.19-maintenance"; inputs.nix.inputs.nixpkgs.follows = "nixpkgs"; outputs = { self, nixpkgs, nix }: diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc index 377bdb56..65f9f3fb 100644 --- a/src/hydra-queue-runner/build-remote.cc +++ b/src/hydra-queue-runner/build-remote.cc @@ -182,40 +182,6 @@ static StorePaths reverseTopoSortPaths(const std::map return sorted; } -/** - * Replace the input derivations by their output paths to send a minimal closure - * to the builder. - * - * If we can afford it, resolve it, so that the newly generated derivation still - * has some sensible output paths. - */ -BasicDerivation inlineInputDerivations(Store & store, Derivation & drv, const StorePath & drvPath) -{ - BasicDerivation ret; - if (!drv.type().hasKnownOutputPaths()) { - auto maybeBasicDrv = drv.tryResolve(store); - if (!maybeBasicDrv) - throw Error( - "the derivation '%s' can’t be resolved. It’s probably " - "missing some outputs", - store.printStorePath(drvPath)); - ret = *maybeBasicDrv; - } else { - // If the derivation is a real `InputAddressed` derivation, we must - // resolve it manually to keep the original output paths - ret = BasicDerivation(drv); - for (auto & [drvPath, node] : drv.inputDrvs.map) { - auto drv2 = store.readDerivation(drvPath); - auto drv2Outputs = drv2.outputsAndOptPaths(store); - for (auto & name : node.value) { - auto inputPath = drv2Outputs.at(name); - ret.inputSrcs.insert(*inputPath.second); - } - } - } - return ret; -} - static std::pair openLogFile(const std::string & logDir, const StorePath & drvPath) { std::string base(drvPath.to_string()); @@ -262,7 +228,22 @@ static BasicDerivation sendInputs( counter & nrStepsCopyingTo ) { - BasicDerivation basicDrv = inlineInputDerivations(localStore, *step.drv, step.drvPath); + /* Replace the input derivations by their output paths to send a + minimal closure to the builder. + + `tryResolve` currently does *not* rewrite input addresses, so it + is safe to do this in all cases. (It should probably have a mode + to do that, however, but we would not use it here.) + */ + BasicDerivation basicDrv = ({ + auto maybeBasicDrv = step.drv->tryResolve(destStore, &localStore); + if (!maybeBasicDrv) + throw Error( + "the derivation '%s' can’t be resolved. It’s probably " + "missing some outputs", + localStore.printStorePath(step.drvPath)); + *maybeBasicDrv; + }); /* Ensure that the inputs exist in the destination store. This is a no-op for regular stores, but for the binary cache store, @@ -351,6 +332,8 @@ static BuildResult performBuild( // far anyways assert(drv.type().hasKnownOutputPaths()); DerivationOutputsAndOptPaths drvOutputs = drv.outputsAndOptPaths(localStore); + // Since this a `BasicDerivation`, `staticOutputHashes` will not + // do any real work. auto outputHashes = staticOutputHashes(localStore, drv); for (auto & [outputName, output] : drvOutputs) { auto outputPath = output.second; @@ -665,14 +648,12 @@ void State::buildRemote(ref destStore, auto outputHashes = staticOutputHashes(*localStore, *step->drv); for (auto & [outputName, realisation] : buildResult.builtOutputs) { // Register the resolved drv output - localStore->registerDrvOutput(realisation); destStore->registerDrvOutput(realisation); // Also register the unresolved one auto unresolvedRealisation = realisation; unresolvedRealisation.signatures.clear(); unresolvedRealisation.id.drvHash = outputHashes.at(outputName); - localStore->registerDrvOutput(unresolvedRealisation); destStore->registerDrvOutput(unresolvedRealisation); } } diff --git a/src/hydra-queue-runner/builder.cc b/src/hydra-queue-runner/builder.cc index 42983941..b1b58c05 100644 --- a/src/hydra-queue-runner/builder.cc +++ b/src/hydra-queue-runner/builder.cc @@ -223,7 +223,7 @@ State::StepResult State::doBuildStep(nix::ref destStore, if (result.stepStatus == bsSuccess) { updateStep(ssPostProcessing); - res = getBuildOutput(destStore, narMembers, localStore->queryDerivationOutputMap(step->drvPath)); + res = getBuildOutput(destStore, narMembers, destStore->queryDerivationOutputMap(step->drvPath, &*localStore)); } } @@ -277,9 +277,12 @@ State::StepResult State::doBuildStep(nix::ref destStore, assert(stepNr); - for (auto & i : localStore->queryPartialDerivationOutputMap(step->drvPath)) { - if (i.second) - addRoot(*i.second); + for (auto & [outputName, optOutputPath] : destStore->queryPartialDerivationOutputMap(step->drvPath, &*localStore)) { + if (!optOutputPath) + throw Error( + "Missing output %s for derivation %d which was supposed to have succeeded", + outputName, localStore->printStorePath(step->drvPath)); + addRoot(*optOutputPath); } /* Register success in the database for all Build objects that diff --git a/src/hydra-queue-runner/hydra-queue-runner.cc b/src/hydra-queue-runner/hydra-queue-runner.cc index 2f2c6091..d514ecfa 100644 --- a/src/hydra-queue-runner/hydra-queue-runner.cc +++ b/src/hydra-queue-runner/hydra-queue-runner.cc @@ -312,7 +312,7 @@ unsigned int State::createBuildStep(pqxx::work & txn, time_t startTime, BuildID if (r.affected_rows() == 0) goto restart; - for (auto & [name, output] : localStore->queryPartialDerivationOutputMap(step->drvPath)) + for (auto & [name, output] : getDestStore()->queryPartialDerivationOutputMap(step->drvPath, &*localStore)) txn.exec_params0 ("insert into BuildStepOutputs (build, stepnr, name, path) values ($1, $2, $3, $4)", buildId, stepNr, name, output ? localStore->printStorePath(*output) : ""); @@ -359,7 +359,7 @@ void State::finishBuildStep(pqxx::work & txn, const RemoteResult & result, assert(res.size()); StorePath drvPath = localStore->parseStorePath(res[0].as()); // If we've finished building, all the paths should be known - for (auto & [name, output] : localStore->queryDerivationOutputMap(drvPath)) + for (auto & [name, output] : getDestStore()->queryDerivationOutputMap(drvPath, &*localStore)) txn.exec_params0 ("update BuildStepOutputs set path = $4 where build = $1 and stepnr = $2 and name = $3", buildId, stepNr, name, localStore->printStorePath(output)); diff --git a/src/hydra-queue-runner/queue-monitor.cc b/src/hydra-queue-runner/queue-monitor.cc index 04917932..a4d9b4dc 100644 --- a/src/hydra-queue-runner/queue-monitor.cc +++ b/src/hydra-queue-runner/queue-monitor.cc @@ -192,11 +192,11 @@ bool State::getQueuedBuilds(Connection & conn, if (!res[0].is_null()) propagatedFrom = res[0].as(); if (!propagatedFrom) { - for (auto & i : localStore->queryPartialDerivationOutputMap(ex.step->drvPath)) { + for (auto & [outputName, _] : destStore->queryPartialDerivationOutputMap(ex.step->drvPath, &*localStore)) { auto res = txn.exec_params ("select max(s.build) from BuildSteps s join BuildStepOutputs o on s.build = o.build where drvPath = $1 and name = $2 and startTime != 0 and stopTime != 0 and status = 1", localStore->printStorePath(ex.step->drvPath), - i.first); + outputName); if (!res[0][0].is_null()) { propagatedFrom = res[0][0].as(); break; @@ -237,7 +237,7 @@ bool State::getQueuedBuilds(Connection & conn, if (!step) { BuildOutput res = getBuildOutputCached(conn, destStore, build->drvPath); - for (auto & i : localStore->queryDerivationOutputMap(build->drvPath)) + for (auto & i : destStore->queryDerivationOutputMap(build->drvPath, &*localStore)) addRoot(i.second); { @@ -481,20 +481,12 @@ Step::ptr State::createStep(ref destStore, auto outputHashes = staticOutputHashes(*localStore, *(step->drv)); bool valid = true; std::map> missing; - for (auto &[outputName, maybeOutputPath] : step->drv->outputsAndOptPaths(*destStore)) { + for (auto & [outputName, maybeOutputPath] : destStore->queryPartialDerivationOutputMap(drvPath, &*localStore)) { auto outputHash = outputHashes.at(outputName); - if (maybeOutputPath.second) { - if (!destStore->isValidPath(*maybeOutputPath.second)) { - valid = false; - missing.insert({{outputHash, outputName}, maybeOutputPath.second}); - } - } else { - experimentalFeatureSettings.require(Xp::CaDerivations); - if (!destStore->queryRealisation(DrvOutput{outputHash, outputName})) { - valid = false; - missing.insert({{outputHash, outputName}, std::nullopt}); - } - } + if (maybeOutputPath && destStore->isValidPath(*maybeOutputPath)) + continue; + valid = false; + missing.insert({{outputHash, outputName}, maybeOutputPath}); } /* Try to copy the missing paths from the local store or from @@ -503,14 +495,24 @@ Step::ptr State::createStep(ref destStore, size_t avail = 0; for (auto & [i, maybePath] : missing) { - if ((maybePath && localStore->isValidPath(*maybePath))) + // If we don't know the output path from the destination + // store, see if the local store can tell us. + if (/* localStore != destStore && */ !maybePath && experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) + if (auto maybeRealisation = localStore->queryRealisation(i)) + maybePath = maybeRealisation->outPath; + + if (!maybePath) { + // No hope of getting the store object if we don't know + // the path. + continue; + } + auto & path = *maybePath; + + if (/* localStore != destStore && */ localStore->isValidPath(path)) avail++; - else if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations) && localStore->queryRealisation(i)) { - maybePath = localStore->queryRealisation(i)->outPath; - avail++; - } else if (useSubstitutes && maybePath) { + else if (useSubstitutes) { SubstitutablePathInfos infos; - localStore->querySubstitutablePathInfos({{*maybePath, {}}}, infos); + localStore->querySubstitutablePathInfos({{path, {}}}, infos); if (infos.size() == 1) avail++; } @@ -518,44 +520,47 @@ Step::ptr State::createStep(ref destStore, if (missing.size() == avail) { valid = true; - for (auto & [i, path] : missing) { - if (path) { - try { - time_t startTime = time(0); + for (auto & [i, maybePath] : missing) { + // If we found everything, then we should know the path + // to every missing store object now. + assert(maybePath); + auto & path = *maybePath; - if (localStore->isValidPath(*path)) - printInfo("copying output β€˜%1%’ of β€˜%2%’ from local store", - localStore->printStorePath(*path), - localStore->printStorePath(drvPath)); - else { - printInfo("substituting output β€˜%1%’ of β€˜%2%’", - localStore->printStorePath(*path), - localStore->printStorePath(drvPath)); - localStore->ensurePath(*path); - // FIXME: should copy directly from substituter to destStore. - } + try { + time_t startTime = time(0); - copyClosure(*localStore, *destStore, - StorePathSet { *path }, - NoRepair, CheckSigs, NoSubstitute); - - time_t stopTime = time(0); - - { - auto mc = startDbUpdate(); - pqxx::work txn(conn); - createSubstitutionStep(txn, startTime, stopTime, build, drvPath, *(step->drv), "out", *path); - txn.commit(); - } - - } catch (Error & e) { - printError("while copying/substituting output β€˜%s’ of β€˜%s’: %s", - localStore->printStorePath(*path), - localStore->printStorePath(drvPath), - e.what()); - valid = false; - break; + if (localStore->isValidPath(path)) + printInfo("copying output β€˜%1%’ of β€˜%2%’ from local store", + localStore->printStorePath(path), + localStore->printStorePath(drvPath)); + else { + printInfo("substituting output β€˜%1%’ of β€˜%2%’", + localStore->printStorePath(path), + localStore->printStorePath(drvPath)); + localStore->ensurePath(path); + // FIXME: should copy directly from substituter to destStore. } + + copyClosure(*localStore, *destStore, + StorePathSet { path }, + NoRepair, CheckSigs, NoSubstitute); + + time_t stopTime = time(0); + + { + auto mc = startDbUpdate(); + pqxx::work txn(conn); + createSubstitutionStep(txn, startTime, stopTime, build, drvPath, *(step->drv), "out", path); + txn.commit(); + } + + } catch (Error & e) { + printError("while copying/substituting output β€˜%s’ of β€˜%s’: %s", + localStore->printStorePath(path), + localStore->printStorePath(drvPath), + e.what()); + valid = false; + break; } } } From a6b6c5a5392a7e705825e8dac7cb818a1f9944fe Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 11 Dec 2023 12:55:57 -0500 Subject: [PATCH 09/32] Revert query -- those columns don't exist yet! --- src/hydra-queue-runner/queue-monitor.cc | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/hydra-queue-runner/queue-monitor.cc b/src/hydra-queue-runner/queue-monitor.cc index a4d9b4dc..fc344244 100644 --- a/src/hydra-queue-runner/queue-monitor.cc +++ b/src/hydra-queue-runner/queue-monitor.cc @@ -192,11 +192,12 @@ bool State::getQueuedBuilds(Connection & conn, if (!res[0].is_null()) propagatedFrom = res[0].as(); if (!propagatedFrom) { - for (auto & [outputName, _] : destStore->queryPartialDerivationOutputMap(ex.step->drvPath, &*localStore)) { + for (auto & [outputName, optOutputPath] : destStore->queryPartialDerivationOutputMap(ex.step->drvPath, &*localStore)) { + // ca-derivations not actually supported yet + assert(optOutputPath); auto res = txn.exec_params - ("select max(s.build) from BuildSteps s join BuildStepOutputs o on s.build = o.build where drvPath = $1 and name = $2 and startTime != 0 and stopTime != 0 and status = 1", - localStore->printStorePath(ex.step->drvPath), - outputName); + ("select max(s.build) from BuildSteps s join BuildStepOutputs o on s.build = o.build where path = $1 and startTime != 0 and stopTime != 0 and status = 1", + localStore->printStorePath(*optOutputPath)); if (!res[0][0].is_null()) { propagatedFrom = res[0][0].as(); break; From 6e67884ff164d9d1c115aef066e18f8de0c5fc44 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 11 Dec 2023 14:05:18 -0500 Subject: [PATCH 10/32] One more `queryDerivationOutputMap` should use the eval store param --- src/hydra-queue-runner/queue-monitor.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/hydra-queue-runner/queue-monitor.cc b/src/hydra-queue-runner/queue-monitor.cc index fc344244..81678fe1 100644 --- a/src/hydra-queue-runner/queue-monitor.cc +++ b/src/hydra-queue-runner/queue-monitor.cc @@ -658,8 +658,7 @@ void State::processJobsetSharesChange(Connection & conn) BuildOutput State::getBuildOutputCached(Connection & conn, nix::ref destStore, const nix::StorePath & drvPath) { - - auto derivationOutputs = localStore->queryDerivationOutputMap(drvPath); + auto derivationOutputs = destStore->queryDerivationOutputMap(drvPath, &*localStore); { pqxx::work txn(conn); From 2e6ee28f9bef00ef08e6b6125f04af82cfca9fd3 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 23 Jan 2024 11:03:19 -0500 Subject: [PATCH 11/32] `Machine` -> `::Machine` so we don't conflict with Nix's --- src/hydra-queue-runner/build-remote.cc | 20 ++++++++++---------- src/hydra-queue-runner/builder.cc | 2 +- src/hydra-queue-runner/dispatcher.cc | 4 ++-- src/hydra-queue-runner/hydra-queue-runner.cc | 8 ++++---- 4 files changed, 17 insertions(+), 17 deletions(-) diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc index a4f3a35c..5cc994fc 100644 --- a/src/hydra-queue-runner/build-remote.cc +++ b/src/hydra-queue-runner/build-remote.cc @@ -48,7 +48,7 @@ static Strings extraStoreArgs(std::string & machine) return result; } -static void openConnection(Machine::ptr machine, Path tmpDir, int stderrFD, SSHMaster::Connection & child) +static void openConnection(::Machine::ptr machine, Path tmpDir, int stderrFD, SSHMaster::Connection & child) { std::string pgmName; Pipe to, from; @@ -104,7 +104,7 @@ static void openConnection(Machine::ptr machine, Path tmpDir, int stderrFD, SSHM static void copyClosureTo( - Machine::Connection & conn, + ::Machine::Connection & conn, Store & destStore, const StorePathSet & paths, SubstituteFlag useSubstitutes = NoSubstitute) @@ -195,7 +195,7 @@ static std::pair openLogFile(const std::string & logDir, cons * Therefore, no `ServeProto::Serialize` functions can be used until * that field is set. */ -static void handshake(Machine::Connection & conn, unsigned int repeats) +static void handshake(::Machine::Connection & conn, unsigned int repeats) { conn.to << SERVE_MAGIC_1 << 0x206; conn.to.flush(); @@ -216,7 +216,7 @@ static BasicDerivation sendInputs( Step & step, Store & localStore, Store & destStore, - Machine::Connection & conn, + ::Machine::Connection & conn, unsigned int & overhead, counter & nrStepsWaiting, counter & nrStepsCopyingTo @@ -272,7 +272,7 @@ static BasicDerivation sendInputs( } static BuildResult performBuild( - Machine::Connection & conn, + ::Machine::Connection & conn, Store & localStore, StorePath drvPath, const BasicDerivation & drv, @@ -317,7 +317,7 @@ static BuildResult performBuild( } static std::map queryPathInfos( - Machine::Connection & conn, + ::Machine::Connection & conn, Store & localStore, StorePathSet & outputs, size_t & totalNarSize @@ -355,7 +355,7 @@ static std::map queryPathInfos( } static void copyPathFromRemote( - Machine::Connection & conn, + ::Machine::Connection & conn, NarMemberDatas & narMembers, Store & localStore, Store & destStore, @@ -385,7 +385,7 @@ static void copyPathFromRemote( } static void copyPathsFromRemote( - Machine::Connection & conn, + ::Machine::Connection & conn, NarMemberDatas & narMembers, Store & localStore, Store & destStore, @@ -462,7 +462,7 @@ void RemoteResult::updateWithBuildResult(const nix::BuildResult & buildResult) void State::buildRemote(ref destStore, - Machine::ptr machine, Step::ptr step, + ::Machine::ptr machine, Step::ptr step, const BuildOptions & buildOptions, RemoteResult & result, std::shared_ptr activeStep, std::function updateStep, @@ -503,7 +503,7 @@ void State::buildRemote(ref destStore, process. Meh. */ }); - Machine::Connection conn { + ::Machine::Connection conn { .from = child.out.get(), .to = child.in.get(), .machine = machine, diff --git a/src/hydra-queue-runner/builder.cc b/src/hydra-queue-runner/builder.cc index 307eee8e..cb343d77 100644 --- a/src/hydra-queue-runner/builder.cc +++ b/src/hydra-queue-runner/builder.cc @@ -400,7 +400,7 @@ void State::failStep( Step::ptr step, BuildID buildId, const RemoteResult & result, - Machine::ptr machine, + ::Machine::ptr machine, bool & stepFinished) { /* Register failure in the database for all Build objects that diff --git a/src/hydra-queue-runner/dispatcher.cc b/src/hydra-queue-runner/dispatcher.cc index af21dcbd..6d738ded 100644 --- a/src/hydra-queue-runner/dispatcher.cc +++ b/src/hydra-queue-runner/dispatcher.cc @@ -199,7 +199,7 @@ system_time State::doDispatch() filter out temporarily disabled machines. */ struct MachineInfo { - Machine::ptr machine; + ::Machine::ptr machine; unsigned long currentJobs; }; std::vector machinesSorted; @@ -435,7 +435,7 @@ void Jobset::pruneSteps() } -State::MachineReservation::MachineReservation(State & state, Step::ptr step, Machine::ptr machine) +State::MachineReservation::MachineReservation(State & state, Step::ptr step, ::Machine::ptr machine) : state(state), step(step), machine(machine) { machine->state->currentJobs++; diff --git a/src/hydra-queue-runner/hydra-queue-runner.cc b/src/hydra-queue-runner/hydra-queue-runner.cc index 1d54bb93..09d6fcb2 100644 --- a/src/hydra-queue-runner/hydra-queue-runner.cc +++ b/src/hydra-queue-runner/hydra-queue-runner.cc @@ -140,7 +140,7 @@ void State::parseMachines(const std::string & contents) if (tokens.size() < 3) continue; tokens.resize(8); - auto machine = std::make_shared(); + auto machine = std::make_shared<::Machine>(); machine->sshName = tokens[0]; machine->systemTypes = tokenizeString(tokens[1], ","); machine->sshKey = tokens[2] == "-" ? std::string("") : tokens[2]; @@ -166,7 +166,7 @@ void State::parseMachines(const std::string & contents) else printMsg(lvlChatty, "updating machine β€˜%1%’", machine->sshName); machine->state = i == oldMachines.end() - ? std::make_shared() + ? std::make_shared<::Machine::State>() : i->second->state; newMachines[machine->sshName] = machine; } @@ -175,9 +175,9 @@ void State::parseMachines(const std::string & contents) if (newMachines.find(m.first) == newMachines.end()) { if (m.second->enabled) printInfo("removing machine β€˜%1%’", m.first); - /* Add a disabled Machine object to make sure stats are + /* Add a disabled ::Machine object to make sure stats are maintained. */ - auto machine = std::make_shared(*(m.second)); + auto machine = std::make_shared<::Machine>(*(m.second)); machine->enabled = false; newMachines[m.first] = machine; } From 70e5469303b422bdb4b123be222bdea4d7f9611c Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 23 Jan 2024 12:08:05 -0500 Subject: [PATCH 12/32] Use Nix's `Machine` type in a mimimal way This is *just* using the fields from that type, and only where the types coincide. (There are two fields with different types, `speedFactor` most interestingly.) No code is reused, so we can be sure that no behavior is changed. Once the types are reconciled on the Nix side, then we can start carefully actually reusing code. Progress on #1164 --- src/hydra-queue-runner/dispatcher.cc | 6 +-- src/hydra-queue-runner/hydra-queue-runner.cc | 53 ++++++++++++++------ src/hydra-queue-runner/state.hh | 21 +++++--- 3 files changed, 53 insertions(+), 27 deletions(-) diff --git a/src/hydra-queue-runner/dispatcher.cc b/src/hydra-queue-runner/dispatcher.cc index 6d738ded..f5d9d3e3 100644 --- a/src/hydra-queue-runner/dispatcher.cc +++ b/src/hydra-queue-runner/dispatcher.cc @@ -231,11 +231,11 @@ system_time State::doDispatch() sort(machinesSorted.begin(), machinesSorted.end(), [](const MachineInfo & a, const MachineInfo & b) -> bool { - float ta = std::round(a.currentJobs / a.machine->speedFactor); - float tb = std::round(b.currentJobs / b.machine->speedFactor); + float ta = std::round(a.currentJobs / a.machine->speedFactorFloat); + float tb = std::round(b.currentJobs / b.machine->speedFactorFloat); return ta != tb ? ta < tb : - a.machine->speedFactor != b.machine->speedFactor ? a.machine->speedFactor > b.machine->speedFactor : + a.machine->speedFactorFloat != b.machine->speedFactorFloat ? a.machine->speedFactorFloat > b.machine->speedFactorFloat : a.currentJobs > b.currentJobs; }); diff --git a/src/hydra-queue-runner/hydra-queue-runner.cc b/src/hydra-queue-runner/hydra-queue-runner.cc index 09d6fcb2..66528302 100644 --- a/src/hydra-queue-runner/hydra-queue-runner.cc +++ b/src/hydra-queue-runner/hydra-queue-runner.cc @@ -1,6 +1,7 @@ #include #include #include +#include #include #include @@ -140,23 +141,43 @@ void State::parseMachines(const std::string & contents) if (tokens.size() < 3) continue; tokens.resize(8); - auto machine = std::make_shared<::Machine>(); - machine->sshName = tokens[0]; - machine->systemTypes = tokenizeString(tokens[1], ","); - machine->sshKey = tokens[2] == "-" ? std::string("") : tokens[2]; - if (tokens[3] != "") - machine->maxJobs = string2IntmaxJobs)>(tokens[3]).value(); - else - machine->maxJobs = 1; - machine->speedFactor = atof(tokens[4].c_str()); if (tokens[5] == "-") tokens[5] = ""; - machine->supportedFeatures = tokenizeString(tokens[5], ","); + auto supportedFeatures = tokenizeString(tokens[5], ","); + if (tokens[6] == "-") tokens[6] = ""; - machine->mandatoryFeatures = tokenizeString(tokens[6], ","); - for (auto & f : machine->mandatoryFeatures) - machine->supportedFeatures.insert(f); - if (tokens[7] != "" && tokens[7] != "-") - machine->sshPublicHostKey = base64Decode(tokens[7]); + auto mandatoryFeatures = tokenizeString(tokens[6], ","); + + for (auto & f : mandatoryFeatures) + supportedFeatures.insert(f); + + using MaxJobs = std::remove_const::type; + + auto machine = std::make_shared<::Machine>(nix::Machine { + // `storeUri`, not yet used + "", + // `systemTypes`, not yet used + {}, + // `sshKey` + tokens[2] == "-" ? "" : tokens[2], + // `maxJobs` + tokens[3] != "" + ? string2Int(tokens[3]).value() + : 1, + // `speedFactor`, not yet used + 1, + // `supportedFeatures` + std::move(supportedFeatures), + // `mandatoryFeatures` + std::move(mandatoryFeatures), + // `sshPublicHostKey` + tokens[7] != "" && tokens[7] != "-" + ? base64Decode(tokens[7]) + : "", + }); + + machine->sshName = tokens[0]; + machine->systemTypesSet = tokenizeString(tokens[1], ","); + machine->speedFactorFloat = atof(tokens[4].c_str()); /* Re-use the State object of the previous machine with the same name. */ @@ -596,7 +617,7 @@ void State::dumpStatus(Connection & conn) json machine = { {"enabled", m->enabled}, - {"systemTypes", m->systemTypes}, + {"systemTypes", m->systemTypesSet}, {"supportedFeatures", m->supportedFeatures}, {"mandatoryFeatures", m->mandatoryFeatures}, {"nrStepsDone", s->nrStepsDone.load()}, diff --git a/src/hydra-queue-runner/state.hh b/src/hydra-queue-runner/state.hh index 6359063a..b08e4e37 100644 --- a/src/hydra-queue-runner/state.hh +++ b/src/hydra-queue-runner/state.hh @@ -22,6 +22,7 @@ #include "sync.hh" #include "nar-extractor.hh" #include "serve-protocol.hh" +#include "machines.hh" typedef unsigned int BuildID; @@ -234,17 +235,21 @@ void getDependents(Step::ptr step, std::set & builds, std::set visitor, Step::ptr step); -struct Machine +struct Machine : nix::Machine { typedef std::shared_ptr ptr; - bool enabled{true}; + /* TODO Get rid of: `nix::Machine::storeUri` is normalized in a way + we are not yet used to, but once we are, we don't need this. */ + std::string sshName; - std::string sshName, sshKey; - std::set systemTypes, supportedFeatures, mandatoryFeatures; - unsigned int maxJobs = 1; - float speedFactor = 1.0; - std::string sshPublicHostKey; + /* TODO Get rid once `nix::Machine::systemTypes` is a set not + vector. */ + std::set systemTypesSet; + + /* TODO Get rid once `nix::Machine::systemTypes` is a `float` not + an `int`. */ + float speedFactorFloat = 1.0; struct State { typedef std::shared_ptr ptr; @@ -272,7 +277,7 @@ struct Machine { /* Check that this machine is of the type required by the step. */ - if (!systemTypes.count(step->drv->platform == "builtin" ? nix::settings.thisSystem : step->drv->platform)) + if (!systemTypesSet.count(step->drv->platform == "builtin" ? nix::settings.thisSystem : step->drv->platform)) return false; /* Check that the step requires all mandatory features of this From 07cb5d1b7c74fdc8423a93b14f4b56af1c49e4b3 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 24 Jan 2024 21:04:14 -0500 Subject: [PATCH 13/32] Use `nix::ParsedDerivation::getRequiredSystemFeatures()` A slight dedup, and also ensures that floating CA derivations require a `ca-derivations` experimental feature. This fixes the scheduling issue that @SuperSandro2000 found. --- src/hydra-queue-runner/queue-monitor.cc | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/hydra-queue-runner/queue-monitor.cc b/src/hydra-queue-runner/queue-monitor.cc index a3557316..513d399b 100644 --- a/src/hydra-queue-runner/queue-monitor.cc +++ b/src/hydra-queue-runner/queue-monitor.cc @@ -462,10 +462,7 @@ Step::ptr State::createStep(ref destStore, step->systemType = step->drv->platform; { - auto i = step->drv->env.find("requiredSystemFeatures"); - StringSet features; - if (i != step->drv->env.end()) - features = step->requiredSystemFeatures = tokenizeString>(i->second); + StringSet features = step->requiredSystemFeatures = step->parsedDrv->getRequiredSystemFeatures(); if (step->preferLocalBuild) features.insert("local"); if (!features.empty()) { From b1fa6b3aac4ce8225e506317119b32122cfc1edd Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 24 Jan 2024 21:37:13 -0500 Subject: [PATCH 14/32] Use `StoreConfig::getDefaultSystemFeatures` for default machine config We have to oddly make a `StoreConfig` subclass to get it, but https://github.com/NixOS/nix/pull/9848 will fix that. The purpose of this is to ensure that, absent an explicit config, `localhost` includes `ca-derivations` and `recursive-nix` if those experimental features are enabled. Very much the complement of #1342, the previous PR. --- src/hydra-queue-runner/hydra-queue-runner.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/hydra-queue-runner/hydra-queue-runner.cc b/src/hydra-queue-runner/hydra-queue-runner.cc index 5fbcb641..1cbcafea 100644 --- a/src/hydra-queue-runner/hydra-queue-runner.cc +++ b/src/hydra-queue-runner/hydra-queue-runner.cc @@ -15,6 +15,7 @@ #include "state.hh" #include "hydra-build-result.hh" #include "store-api.hh" +#include "local-store.hh" #include "remote-store.hh" #include "globals.hh" @@ -226,7 +227,7 @@ void State::monitorMachinesFile() parseMachines("localhost " + (settings.thisSystem == "x86_64-linux" ? "x86_64-linux,i686-linux" : settings.thisSystem.get()) + " - " + std::to_string(settings.maxBuildJobs) + " 1 " - + concatStringsSep(",", settings.systemFeatures.get())); + + concatStringsSep(",", (LocalStoreConfig { {} }).getDefaultSystemFeatures())); machinesReadyLock.unlock(); return; } From 6df06b089e1a3d6af59f3ee545c2b54e43356cd2 Mon Sep 17 00:00:00 2001 From: Pierre Bourdon Date: Thu, 25 Jan 2024 09:27:46 +0100 Subject: [PATCH 15/32] web: disable Sign in with Google popup --- src/root/topbar.tt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/root/topbar.tt b/src/root/topbar.tt index 1771222d..58dd94e3 100644 --- a/src/root/topbar.tt +++ b/src/root/topbar.tt @@ -134,7 +134,7 @@ [% WRAPPER makeSubMenu title="Sign in" id="sign-in-menu" align="right" %] [% IF c.config.enable_google_login %] -
+
From a876e46894c73973d155b581283d07ef90706c01 Mon Sep 17 00:00:00 2001 From: Sandro Date: Thu, 25 Jan 2024 17:12:40 +0100 Subject: [PATCH 16/32] Remove automake, libtool Those are already part of autoreconfHook --- flake.nix | 2 -- 1 file changed, 2 deletions(-) diff --git a/flake.nix b/flake.nix index 8280e076..eabe5518 100644 --- a/flake.nix +++ b/flake.nix @@ -146,8 +146,6 @@ with final.buildPackages; [ makeWrapper autoreconfHook - automake - libtool nukeReferences pkg-config mdbook From 1471aacadc8868fff6eab06d2c330abd90ec2867 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 25 Jan 2024 11:20:27 -0500 Subject: [PATCH 17/32] Split out a `package.nix` Just like we did with Nix. --- flake.nix | 196 +---------------------------------------- package.nix | 246 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 248 insertions(+), 194 deletions(-) create mode 100644 package.nix diff --git a/flake.nix b/flake.nix index eabe5518..f7975b27 100644 --- a/flake.nix +++ b/flake.nix @@ -7,8 +7,6 @@ outputs = { self, nixpkgs, nix }: let - version = "${builtins.readFile ./version.txt}.${builtins.substring 0 8 (self.lastModifiedDate or "19700101")}.${self.shortRev or "DIRTY"}"; - systems = [ "x86_64-linux" "aarch64-linux" ]; forEachSystem = nixpkgs.lib.genAttrs systems; @@ -61,198 +59,8 @@ }; - hydra = let - inherit (final) lib stdenv; - perlDeps = final.buildEnv { - name = "hydra-perl-deps"; - paths = with final.perlPackages; lib.closePropagation - [ - AuthenSASL - CatalystActionREST - CatalystAuthenticationStoreDBIxClass - CatalystAuthenticationStoreLDAP - CatalystDevel - CatalystPluginAccessLog - CatalystPluginAuthorizationRoles - CatalystPluginCaptcha - CatalystPluginPrometheusTiny - CatalystPluginSessionStateCookie - CatalystPluginSessionStoreFastMmap - CatalystPluginStackTrace - CatalystTraitForRequestProxyBase - CatalystViewDownload - CatalystViewJSON - CatalystViewTT - CatalystXRoleApplicator - CatalystXScriptServerStarman - CryptPassphrase - CryptPassphraseArgon2 - CryptRandPasswd - DataDump - DateTime - DBDPg - DBDSQLite - DigestSHA1 - EmailMIME - EmailSender - FileLibMagic - FileSlurper - FileWhich - final.nix.perl-bindings - final.git - IOCompress - IPCRun - IPCRun3 - JSON - JSONMaybeXS - JSONXS - ListSomeUtils - LWP - LWPProtocolHttps - ModulePluggable - NetAmazonS3 - NetPrometheus - NetStatsd - PadWalker - ParallelForkManager - PerlCriticCommunity - PrometheusTinyShared - ReadonlyX - SetScalar - SQLSplitStatement - Starman - StringCompareConstantTime - SysHostnameLong - TermSizeAny - TermReadKey - Test2Harness - TestPostgreSQL - TextDiff - TextTable - UUID4Tiny - YAML - XMLSimple - ]; - }; - - in - stdenv.mkDerivation { - - name = "hydra-${version}"; - + hydra = final.callPackage ./package.nix { src = self; - - nativeBuildInputs = - with final.buildPackages; [ - makeWrapper - autoreconfHook - nukeReferences - pkg-config - mdbook - ]; - - buildInputs = - with final; [ - unzip - libpqxx - top-git - mercurial - darcs - subversion - breezy - openssl - bzip2 - libxslt - final.nix - perlDeps - perl - pixz - boost - postgresql_13 - (if lib.versionAtLeast lib.version "20.03pre" - then nlohmann_json - else nlohmann_json.override { multipleHeaders = true; }) - prometheus-cpp - ]; - - checkInputs = with final; [ - cacert - foreman - glibcLocales - libressl.nc - openldap - python3 - ]; - - hydraPath = with final; lib.makeBinPath ( - [ - subversion - openssh - final.nix - coreutils - findutils - pixz - gzip - bzip2 - xz - gnutar - unzip - git - top-git - mercurial - darcs - gnused - breezy - ] ++ lib.optionals stdenv.isLinux [ rpm dpkg cdrkit ] - ); - - OPENLDAP_ROOT = final.openldap; - - shellHook = '' - pushd $(git rev-parse --show-toplevel) >/dev/null - - PATH=$(pwd)/src/hydra-evaluator:$(pwd)/src/script:$(pwd)/src/hydra-eval-jobs:$(pwd)/src/hydra-queue-runner:$PATH - PERL5LIB=$(pwd)/src/lib:$PERL5LIB - export HYDRA_HOME="$(pwd)/src/" - mkdir -p .hydra-data - export HYDRA_DATA="$(pwd)/.hydra-data" - export HYDRA_DBI='dbi:Pg:dbname=hydra;host=localhost;port=64444' - - popd >/dev/null - ''; - - NIX_LDFLAGS = [ "-lpthread" ]; - - enableParallelBuilding = true; - - doCheck = true; - - preCheck = '' - patchShebangs . - export LOGNAME=''${LOGNAME:-foo} - # set $HOME for bzr so it can create its trace file - export HOME=$(mktemp -d) - ''; - - postInstall = '' - mkdir -p $out/nix-support - - for i in $out/bin/*; do - read -n 4 chars < $i - if [[ $chars =~ ELF ]]; then continue; fi - wrapProgram $i \ - --prefix PERL5LIB ':' $out/libexec/hydra/lib:$PERL5LIB \ - --prefix PATH ':' $out/bin:$hydraPath \ - --set HYDRA_RELEASE ${version} \ - --set HYDRA_HOME $out/libexec/hydra \ - --set NIX_RELEASE ${final.nix.name or "unknown"} - done - ''; - - dontStrip = true; - - meta.description = "Build of Hydra on ${final.stdenv.system}"; - passthru = { inherit perlDeps; inherit (final) nix; }; }; }; @@ -262,7 +70,7 @@ manual = forEachSystem (system: let pkgs = pkgsBySystem.${system}; in - pkgs.runCommand "hydra-manual-${version}" { } + pkgs.runCommand "hydra-manual-${pkgs.hydra.version}" { } '' mkdir -p $out/share cp -prvd ${pkgs.hydra}/share/doc $out/share/ diff --git a/package.nix b/package.nix new file mode 100644 index 00000000..35e29be1 --- /dev/null +++ b/package.nix @@ -0,0 +1,246 @@ +{ stdenv +, lib + +, src + +, buildEnv + +, perlPackages + +, nix +, git + +, makeWrapper +, autoreconfHook +, nukeReferences +, pkg-config +, mdbook + +, unzip +, libpqxx +, top-git +, mercurial +, darcs +, subversion +, breezy +, openssl +, bzip2 +, libxslt +, perl +, pixz +, boost +, postgresql_13 +, nlohmann_json +, prometheus-cpp + +, cacert +, foreman +, glibcLocales +, libressl +, openldap +, python3 + +, openssh +, coreutils +, findutils +, gzip +, xz +, gnutar +, gnused + +, rpm +, dpkg +, cdrkit +}: + +let + perlDeps = buildEnv { + name = "hydra-perl-deps"; + paths = with perlPackages; lib.closePropagation + [ + AuthenSASL + CatalystActionREST + CatalystAuthenticationStoreDBIxClass + CatalystAuthenticationStoreLDAP + CatalystDevel + CatalystPluginAccessLog + CatalystPluginAuthorizationRoles + CatalystPluginCaptcha + CatalystPluginPrometheusTiny + CatalystPluginSessionStateCookie + CatalystPluginSessionStoreFastMmap + CatalystPluginStackTrace + CatalystTraitForRequestProxyBase + CatalystViewDownload + CatalystViewJSON + CatalystViewTT + CatalystXRoleApplicator + CatalystXScriptServerStarman + CryptPassphrase + CryptPassphraseArgon2 + CryptRandPasswd + DataDump + DateTime + DBDPg + DBDSQLite + DigestSHA1 + EmailMIME + EmailSender + FileLibMagic + FileSlurper + FileWhich + # Not Perl + nix.perl-bindings + git + # Perl again + IOCompress + IPCRun + IPCRun3 + JSON + JSONMaybeXS + JSONXS + ListSomeUtils + LWP + LWPProtocolHttps + ModulePluggable + NetAmazonS3 + NetPrometheus + NetStatsd + PadWalker + ParallelForkManager + PerlCriticCommunity + PrometheusTinyShared + ReadonlyX + SetScalar + SQLSplitStatement + Starman + StringCompareConstantTime + SysHostnameLong + TermSizeAny + TermReadKey + Test2Harness + TestPostgreSQL + TextDiff + TextTable + UUID4Tiny + YAML + XMLSimple + ]; + }; + + version = "${builtins.readFile ./version.txt}.${builtins.substring 0 8 (src.lastModifiedDate or "19700101")}.${src.shortRev or "DIRTY"}"; +in +stdenv.mkDerivation { + pname = "hydra"; + inherit version; + + inherit src; + + nativeBuildInputs = [ + makeWrapper + autoreconfHook + nukeReferences + pkg-config + mdbook + ]; + + buildInputs = [ + unzip + libpqxx + top-git + mercurial + darcs + subversion + breezy + openssl + bzip2 + libxslt + nix + perlDeps + perl + pixz + boost + postgresql_13 + nlohmann_json + prometheus-cpp + ]; + + checkInputs = [ + cacert + foreman + glibcLocales + libressl.nc + openldap + python3 + ]; + + hydraPath = lib.makeBinPath ( + [ + subversion + openssh + nix + coreutils + findutils + pixz + gzip + bzip2 + xz + gnutar + unzip + git + top-git + mercurial + darcs + gnused + breezy + ] ++ lib.optionals stdenv.isLinux [ rpm dpkg cdrkit ] + ); + + OPENLDAP_ROOT = openldap; + + shellHook = '' + pushd $(git rev-parse --show-toplevel) >/dev/null + + PATH=$(pwd)/src/hydra-evaluator:$(pwd)/src/script:$(pwd)/src/hydra-eval-jobs:$(pwd)/src/hydra-queue-runner:$PATH + PERL5LIB=$(pwd)/src/lib:$PERL5LIB + export HYDRA_HOME="$(pwd)/src/" + mkdir -p .hydra-data + export HYDRA_DATA="$(pwd)/.hydra-data" + export HYDRA_DBI='dbi:Pg:dbname=hydra;host=localhost;port=64444' + + popd >/dev/null + ''; + + NIX_LDFLAGS = [ "-lpthread" ]; + + enableParallelBuilding = true; + + doCheck = true; + + preCheck = '' + patchShebangs . + export LOGNAME=''${LOGNAME:-foo} + # set $HOME for bzr so it can create its trace file + export HOME=$(mktemp -d) + ''; + + postInstall = '' + mkdir -p $out/nix-support + + for i in $out/bin/*; do + read -n 4 chars < $i + if [[ $chars =~ ELF ]]; then continue; fi + wrapProgram $i \ + --prefix PERL5LIB ':' $out/libexec/hydra/lib:$PERL5LIB \ + --prefix PATH ':' $out/bin:$hydraPath \ + --set HYDRA_RELEASE ${version} \ + --set HYDRA_HOME $out/libexec/hydra \ + --set NIX_RELEASE ${nix.name or "unknown"} + done + ''; + + dontStrip = true; + + meta.description = "Build of Hydra on ${stdenv.system}"; + passthru = { inherit perlDeps nix; }; +} From 1bd195a5138b3c69c52110d713c706cb4908ba16 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 25 Jan 2024 11:32:00 -0500 Subject: [PATCH 18/32] Clean up deps - `strictDeps` - Ensure it builds with and without `doCheck` --- flake.nix | 6 ++++++ package.nix | 28 ++++++++++++++++++---------- 2 files changed, 24 insertions(+), 10 deletions(-) diff --git a/flake.nix b/flake.nix index f7975b27..a6b62cfa 100644 --- a/flake.nix +++ b/flake.nix @@ -68,6 +68,12 @@ build = forEachSystem (system: packages.${system}.hydra); + buildNoTests = forEachSystem (system: + packages.${system}.hydra.overrideAttrs (_: { + doCheck = false; + }) + ); + manual = forEachSystem (system: let pkgs = pkgsBySystem.${system}; in pkgs.runCommand "hydra-manual-${pkgs.hydra.version}" { } diff --git a/package.nix b/package.nix index 35e29be1..f91e7a24 100644 --- a/package.nix +++ b/package.nix @@ -136,41 +136,49 @@ stdenv.mkDerivation { inherit src; + strictDeps = true; + nativeBuildInputs = [ makeWrapper autoreconfHook nukeReferences pkg-config mdbook + nix + perlDeps + perl + unzip ]; buildInputs = [ - unzip libpqxx - top-git - mercurial - darcs - subversion - breezy openssl - bzip2 libxslt nix perlDeps perl - pixz boost - postgresql_13 nlohmann_json prometheus-cpp ]; + nativeCheckInputs = [ + bzip2 + darcs + top-git + mercurial + subversion + breezy + openldap + postgresql_13 + pixz + ]; + checkInputs = [ cacert foreman glibcLocales libressl.nc - openldap python3 ]; From d6d6d1b649ea3b83baf44267637e42c4eb1c92c0 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 25 Jan 2024 12:03:15 -0500 Subject: [PATCH 19/32] flake.nix: Temporarily add a second Nixpkgs for `lib.fileset` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: β€’ Updated input 'nix': 'github:NixOS/nix/b38e5a665e9d0aa7975beb0ed12e42d13a392e74' (2023-12-13) β†’ 'github:NixOS/nix/03e96b9dc011a16a0f6db9c7cb021ff93f8dcf88' (2024-01-19) β€’ Added input 'nixpkgs-for-fileset': 'github:NixOS/nixpkgs/a77ab169a83a4175169d78684ddd2e54486ac651' (2024-01-24) --- flake.lock | 25 +++++++++++++++++++++---- flake.nix | 5 +++++ 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/flake.lock b/flake.lock index 66e30323..8231484c 100644 --- a/flake.lock +++ b/flake.lock @@ -42,11 +42,11 @@ "nixpkgs-regression": "nixpkgs-regression" }, "locked": { - "lastModified": 1702510710, - "narHash": "sha256-9K+w1mQgmUxCmEsPaSFkpYsj/cxjO2PSwTCPkNZ/NiU=", + "lastModified": 1705666051, + "narHash": "sha256-C3eht7uA7gZp9O5XpA9Mkqdi1WoTQaFH4FMGmM4DrFM=", "owner": "NixOS", "repo": "nix", - "rev": "b38e5a665e9d0aa7975beb0ed12e42d13a392e74", + "rev": "03e96b9dc011a16a0f6db9c7cb021ff93f8dcf88", "type": "github" }, "original": { @@ -72,6 +72,22 @@ "type": "github" } }, + "nixpkgs-for-fileset": { + "locked": { + "lastModified": 1706098335, + "narHash": "sha256-r3dWjT8P9/Ah5m5ul4WqIWD8muj5F+/gbCdjiNVBKmU=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "a77ab169a83a4175169d78684ddd2e54486ac651", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-23.11", + "repo": "nixpkgs", + "type": "github" + } + }, "nixpkgs-regression": { "locked": { "lastModified": 1643052045, @@ -91,7 +107,8 @@ "root": { "inputs": { "nix": "nix", - "nixpkgs": "nixpkgs" + "nixpkgs": "nixpkgs", + "nixpkgs-for-fileset": "nixpkgs-for-fileset" } } }, diff --git a/flake.nix b/flake.nix index a6b62cfa..0ab25018 100644 --- a/flake.nix +++ b/flake.nix @@ -5,6 +5,11 @@ inputs.nix.url = "github:NixOS/nix/2.19-maintenance"; inputs.nix.inputs.nixpkgs.follows = "nixpkgs"; + # TODO get rid of this once https://github.com/NixOS/nix/pull/9546 is + # mered and we upgrade or Nix, so the main `nixpkgs` input is at least + # 23.11 and has `lib.fileset`. + inputs.nixpkgs-for-fileset.url = "github:NixOS/nixpkgs/nixos-23.11"; + outputs = { self, nixpkgs, nix }: let systems = [ "x86_64-linux" "aarch64-linux" ]; From 4bbc7b8f75409aa8e99a0f3284bdc5f11f804cfb Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 25 Jan 2024 12:21:13 -0500 Subject: [PATCH 20/32] Use the Nixpkgs `fileset` library to filter source Now I can change Nix files without causing rebuilds. --- flake.nix | 5 +++-- package.nix | 26 +++++++++++++++++++++----- 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/flake.nix b/flake.nix index 0ab25018..71fd6c6d 100644 --- a/flake.nix +++ b/flake.nix @@ -10,7 +10,7 @@ # 23.11 and has `lib.fileset`. inputs.nixpkgs-for-fileset.url = "github:NixOS/nixpkgs/nixos-23.11"; - outputs = { self, nixpkgs, nix }: + outputs = { self, nixpkgs, nix, nixpkgs-for-fileset }: let systems = [ "x86_64-linux" "aarch64-linux" ]; forEachSystem = nixpkgs.lib.genAttrs systems; @@ -65,7 +65,8 @@ }; hydra = final.callPackage ./package.nix { - src = self; + inherit (nixpkgs-for-fileset.lib) fileset; + rawSrc = self; }; }; diff --git a/package.nix b/package.nix index f91e7a24..eff9ed37 100644 --- a/package.nix +++ b/package.nix @@ -1,7 +1,8 @@ { stdenv , lib +, fileset -, src +, rawSrc , buildEnv @@ -128,13 +129,28 @@ let ]; }; - version = "${builtins.readFile ./version.txt}.${builtins.substring 0 8 (src.lastModifiedDate or "19700101")}.${src.shortRev or "DIRTY"}"; + version = "${builtins.readFile ./version.txt}.${builtins.substring 0 8 (rawSrc.lastModifiedDate or "19700101")}.${rawSrc.shortRev or "DIRTY"}"; in -stdenv.mkDerivation { +stdenv.mkDerivation (finalAttrs: { pname = "hydra"; inherit version; - inherit src; + src = fileset.toSource { + root = ./.; + fileset = fileset.unions ([ + ./version.txt + ./configure.ac + ./Makefile.am + ./src + ./doc + ./hydra-module.nix + # TODO only when `doCheck` + ./t + ] ++ lib.optionals finalAttrs.doCheck [ + ./.perlcriticrc + ./.yath.rc + ]); + }; strictDeps = true; @@ -251,4 +267,4 @@ stdenv.mkDerivation { meta.description = "Build of Hydra on ${stdenv.system}"; passthru = { inherit perlDeps nix; }; -} +}) From 73b6c1fb11b7c5679725599c552f4e163bf668f0 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 25 Jan 2024 13:27:05 -0500 Subject: [PATCH 21/32] Filter out (mosts test) when `!doCheck` --- Makefile.am | 6 +++++- configure.ac | 20 ++++++++++++++------ package.nix | 7 +++++-- 3 files changed, 24 insertions(+), 9 deletions(-) diff --git a/Makefile.am b/Makefile.am index e744cc33..9e06ebdb 100644 --- a/Makefile.am +++ b/Makefile.am @@ -1,4 +1,8 @@ -SUBDIRS = src t doc +SUBDIRS = src doc +if CAN_DO_CHECK + SUBDIRS += t +endif + BOOTCLEAN_SUBDIRS = $(SUBDIRS) DIST_SUBDIRS = $(SUBDIRS) EXTRA_DIST = hydra-module.nix diff --git a/configure.ac b/configure.ac index eec647c3..e5c57d14 100644 --- a/configure.ac +++ b/configure.ac @@ -53,9 +53,6 @@ PKG_CHECK_MODULES([NIX], [nix-main nix-expr nix-store]) testPath="$(dirname $(type -p expr))" AC_SUBST(testPath) -jobsPath="$(realpath ./t/jobs)" -AC_SUBST(jobsPath) - CXXFLAGS+=" -include nix/config.h" AC_CONFIG_FILES([ @@ -71,11 +68,22 @@ AC_CONFIG_FILES([ src/lib/Makefile src/root/Makefile src/script/Makefile - t/Makefile - t/jobs/config.nix - t/jobs/declarative/project.json ]) +# Tests might be filtered out +AM_CONDITIONAL([CAN_DO_CHECK], [test -f "$srcdir/t/api-test.t"]) +AM_COND_IF( + [CAN_DO_CHECK], + [ + jobsPath="$(realpath ./t/jobs)" + AC_SUBST(jobsPath) + AC_CONFIG_FILES([ + t/Makefile + t/jobs/config.nix + t/jobs/declarative/project.json + ]) + ]) + AC_CONFIG_COMMANDS([executable-scripts], []) AC_CONFIG_HEADER([hydra-config.h]) diff --git a/package.nix b/package.nix index eff9ed37..73ef9b3f 100644 --- a/package.nix +++ b/package.nix @@ -144,9 +144,12 @@ stdenv.mkDerivation (finalAttrs: { ./src ./doc ./hydra-module.nix - # TODO only when `doCheck` - ./t + # These are always needed to appease Automake + ./t/Makefile.am + ./t/jobs/config.nix.in + ./t/jobs/declarative/project.json.in ] ++ lib.optionals finalAttrs.doCheck [ + ./t ./.perlcriticrc ./.yath.rc ]); From c5f37eca91bee5bacbe53f826abcd0c0d0f38ef1 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 25 Jan 2024 11:54:44 -0500 Subject: [PATCH 22/32] Reorganize hydra modules --- .gitignore | 1 + Makefile.am | 6 +-- flake.nix | 50 +++------------------ nixos-modules/default.nix | 49 ++++++++++++++++++++ hydra-module.nix => nixos-modules/hydra.nix | 0 package.nix | 2 +- 6 files changed, 59 insertions(+), 49 deletions(-) create mode 100644 nixos-modules/default.nix rename hydra-module.nix => nixos-modules/hydra.nix (100%) diff --git a/.gitignore b/.gitignore index 799db665..f8bf5718 100644 --- a/.gitignore +++ b/.gitignore @@ -38,6 +38,7 @@ t/jobs/declarative/project.json hydra-config.h hydra-config.h.in result +result-* outputs config stamp-h1 diff --git a/Makefile.am b/Makefile.am index 9e06ebdb..a28e3f33 100644 --- a/Makefile.am +++ b/Makefile.am @@ -5,8 +5,8 @@ endif BOOTCLEAN_SUBDIRS = $(SUBDIRS) DIST_SUBDIRS = $(SUBDIRS) -EXTRA_DIST = hydra-module.nix +EXTRA_DIST = nixos-modules/hydra.nix -install-data-local: hydra-module.nix +install-data-local: nixos-modules/hydra.nix $(INSTALL) -d $(DESTDIR)$(datadir)/nix - $(INSTALL_DATA) hydra-module.nix $(DESTDIR)$(datadir)/nix/ + $(INSTALL_DATA) nixos-modules/hydra.nix $(DESTDIR)$(datadir)/nix/hydra-module.nix diff --git a/flake.nix b/flake.nix index 71fd6c6d..30aa61c0 100644 --- a/flake.nix +++ b/flake.nix @@ -15,9 +15,11 @@ systems = [ "x86_64-linux" "aarch64-linux" ]; forEachSystem = nixpkgs.lib.genAttrs systems; + overlayList = [ self.overlays.default nix.overlays.default ]; + pkgsBySystem = forEachSystem (system: import nixpkgs { inherit system; - overlays = [ self.overlays.default nix.overlays.default ]; + overlays = overlayList; }); # NixOS configuration used for VM tests. @@ -386,50 +388,8 @@ default = pkgsBySystem.${system}.hydra; }); - nixosModules.hydra = { - imports = [ ./hydra-module.nix ]; - nixpkgs.overlays = [ self.overlays.default nix.overlays.default ]; - }; - - nixosModules.hydraTest = { pkgs, ... }: { - imports = [ self.nixosModules.hydra ]; - - services.hydra-dev.enable = true; - services.hydra-dev.hydraURL = "http://hydra.example.org"; - services.hydra-dev.notificationSender = "admin@hydra.example.org"; - - systemd.services.hydra-send-stats.enable = false; - - services.postgresql.enable = true; - services.postgresql.package = pkgs.postgresql_11; - - # The following is to work around the following error from hydra-server: - # [error] Caught exception in engine "Cannot determine local time zone" - time.timeZone = "UTC"; - - nix.extraOptions = '' - allowed-uris = https://github.com/ - ''; - }; - - nixosModules.hydraProxy = { - services.httpd = { - enable = true; - adminAddr = "hydra-admin@example.org"; - extraConfig = '' - - Order deny,allow - Allow from all - - - ProxyRequests Off - ProxyPreserveHost On - ProxyPass /apache-errors ! - ErrorDocument 503 /apache-errors/503.html - ProxyPass / http://127.0.0.1:3000/ retry=5 disablereuse=on - ProxyPassReverse / http://127.0.0.1:3000/ - ''; - }; + nixosModules = import ./nixos-modules { + overlays = overlayList; }; nixosConfigurations.container = nixpkgs.lib.nixosSystem { diff --git a/nixos-modules/default.nix b/nixos-modules/default.nix new file mode 100644 index 00000000..6fc19d31 --- /dev/null +++ b/nixos-modules/default.nix @@ -0,0 +1,49 @@ +{ overlays }: + +rec { + hydra = { + imports = [ ./hydra.nix ]; + nixpkgs = { inherit overlays; }; + }; + + hydraTest = { pkgs, ... }: { + imports = [ hydra ]; + + services.hydra-dev.enable = true; + services.hydra-dev.hydraURL = "http://hydra.example.org"; + services.hydra-dev.notificationSender = "admin@hydra.example.org"; + + systemd.services.hydra-send-stats.enable = false; + + services.postgresql.enable = true; + services.postgresql.package = pkgs.postgresql_11; + + # The following is to work around the following error from hydra-server: + # [error] Caught exception in engine "Cannot determine local time zone" + time.timeZone = "UTC"; + + nix.extraOptions = '' + allowed-uris = https://github.com/ + ''; + }; + + hydraProxy = { + services.httpd = { + enable = true; + adminAddr = "hydra-admin@example.org"; + extraConfig = '' + + Order deny,allow + Allow from all + + + ProxyRequests Off + ProxyPreserveHost On + ProxyPass /apache-errors ! + ErrorDocument 503 /apache-errors/503.html + ProxyPass / http://127.0.0.1:3000/ retry=5 disablereuse=on + ProxyPassReverse / http://127.0.0.1:3000/ + ''; + }; + }; +} diff --git a/hydra-module.nix b/nixos-modules/hydra.nix similarity index 100% rename from hydra-module.nix rename to nixos-modules/hydra.nix diff --git a/package.nix b/package.nix index 73ef9b3f..d5e6bb69 100644 --- a/package.nix +++ b/package.nix @@ -143,7 +143,7 @@ stdenv.mkDerivation (finalAttrs: { ./Makefile.am ./src ./doc - ./hydra-module.nix + ./nixos-modules/hydra.nix # These are always needed to appease Automake ./t/Makefile.am ./t/jobs/config.nix.in From b5ed0787f7eba4f0a7d900736976e335f443c19e Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 25 Jan 2024 14:25:12 -0500 Subject: [PATCH 23/32] Replace "not Perl" and "Perl again" with something more self-explanatory --- package.nix | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/package.nix b/package.nix index d5e6bb69..0f8f004d 100644 --- a/package.nix +++ b/package.nix @@ -57,8 +57,11 @@ let perlDeps = buildEnv { name = "hydra-perl-deps"; - paths = with perlPackages; lib.closePropagation - [ + paths = lib.closePropagation + ([ + nix.perl-bindings + git + ] ++ (with perlPackages; [ AuthenSASL CatalystActionREST CatalystAuthenticationStoreDBIxClass @@ -90,10 +93,6 @@ let FileLibMagic FileSlurper FileWhich - # Not Perl - nix.perl-bindings - git - # Perl again IOCompress IPCRun IPCRun3 @@ -126,7 +125,7 @@ let UUID4Tiny YAML XMLSimple - ]; + ])); }; version = "${builtins.readFile ./version.txt}.${builtins.substring 0 8 (rawSrc.lastModifiedDate or "19700101")}.${rawSrc.shortRev or "DIRTY"}"; From aed130cd17ae7da41d9f38f830af3826a85a5dc6 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 25 Jan 2024 15:57:39 -0500 Subject: [PATCH 24/32] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: β€’ Updated input 'nix': 'github:NixOS/nix/03e96b9dc011a16a0f6db9c7cb021ff93f8dcf88' (2024-01-19) β†’ 'github:NixOS/nix/2c4bb93ba5a97e7078896ebc36385ce172960e4e' (2024-01-25) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 8231484c..2ddc7c16 100644 --- a/flake.lock +++ b/flake.lock @@ -42,11 +42,11 @@ "nixpkgs-regression": "nixpkgs-regression" }, "locked": { - "lastModified": 1705666051, - "narHash": "sha256-C3eht7uA7gZp9O5XpA9Mkqdi1WoTQaFH4FMGmM4DrFM=", + "lastModified": 1706208340, + "narHash": "sha256-wNyHUEIiKKVs6UXrUzhP7RSJQv0A8jckgcuylzftl8k=", "owner": "NixOS", "repo": "nix", - "rev": "03e96b9dc011a16a0f6db9c7cb021ff93f8dcf88", + "rev": "2c4bb93ba5a97e7078896ebc36385ce172960e4e", "type": "github" }, "original": { From c64eed7d07ad432fa9c5ea979d1039a5207f4629 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 25 Jan 2024 14:48:53 -0500 Subject: [PATCH 25/32] Simplify `StoreConfig::getDefaultSystemFeatures` call That method is now static. --- src/hydra-queue-runner/hydra-queue-runner.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/hydra-queue-runner/hydra-queue-runner.cc b/src/hydra-queue-runner/hydra-queue-runner.cc index 1cbcafea..df79e1dc 100644 --- a/src/hydra-queue-runner/hydra-queue-runner.cc +++ b/src/hydra-queue-runner/hydra-queue-runner.cc @@ -15,7 +15,6 @@ #include "state.hh" #include "hydra-build-result.hh" #include "store-api.hh" -#include "local-store.hh" #include "remote-store.hh" #include "globals.hh" @@ -227,7 +226,7 @@ void State::monitorMachinesFile() parseMachines("localhost " + (settings.thisSystem == "x86_64-linux" ? "x86_64-linux,i686-linux" : settings.thisSystem.get()) + " - " + std::to_string(settings.maxBuildJobs) + " 1 " - + concatStringsSep(",", (LocalStoreConfig { {} }).getDefaultSystemFeatures())); + + concatStringsSep(",", StoreConfig::getDefaultSystemFeatures())); machinesReadyLock.unlock(); return; } From fcde5908d8e51f975b883329b34d24a9f30ea4b3 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 25 Jan 2024 21:32:22 -0500 Subject: [PATCH 26/32] More CA derivations prep Again, with care not to change the schema in any way. --- src/hydra-eval-jobs/hydra-eval-jobs.cc | 6 +----- src/hydra-queue-runner/hydra-queue-runner.cc | 5 ++++- src/hydra-queue-runner/queue-monitor.cc | 20 ++++++++++++-------- src/lib/Hydra/Controller/Build.pm | 6 ++++-- src/script/hydra-eval-jobset | 6 +++++- t/lib/HydraTestContext.pm | 6 +++++- 6 files changed, 31 insertions(+), 18 deletions(-) diff --git a/src/hydra-eval-jobs/hydra-eval-jobs.cc b/src/hydra-eval-jobs/hydra-eval-jobs.cc index 2794cc62..5cfc3a52 100644 --- a/src/hydra-eval-jobs/hydra-eval-jobs.cc +++ b/src/hydra-eval-jobs/hydra-eval-jobs.cc @@ -250,11 +250,7 @@ static void worker( // See the `queryOutputs` call above; we should // not encounter missing output paths otherwise. assert(experimentalFeatureSettings.isEnabled(Xp::CaDerivations)); - // TODO it would be better to set `null` than an - // empty string here, to force the consumer of - // this JSON to more explicitly handle this - // case. - out[outputName] = ""; + out[outputName] = nullptr; } } job["outputs"] = std::move(out); diff --git a/src/hydra-queue-runner/hydra-queue-runner.cc b/src/hydra-queue-runner/hydra-queue-runner.cc index df79e1dc..0f4e759c 100644 --- a/src/hydra-queue-runner/hydra-queue-runner.cc +++ b/src/hydra-queue-runner/hydra-queue-runner.cc @@ -336,7 +336,10 @@ unsigned int State::createBuildStep(pqxx::work & txn, time_t startTime, BuildID for (auto & [name, output] : getDestStore()->queryPartialDerivationOutputMap(step->drvPath, &*localStore)) txn.exec_params0 ("insert into BuildStepOutputs (build, stepnr, name, path) values ($1, $2, $3, $4)", - buildId, stepNr, name, output ? localStore->printStorePath(*output) : ""); + buildId, stepNr, name, + output + ? std::optional { localStore->printStorePath(*output)} + : std::nullopt); if (status == bsBusy) txn.exec(fmt("notify step_started, '%d\t%d'", buildId, stepNr)); diff --git a/src/hydra-queue-runner/queue-monitor.cc b/src/hydra-queue-runner/queue-monitor.cc index 513d399b..d7214c43 100644 --- a/src/hydra-queue-runner/queue-monitor.cc +++ b/src/hydra-queue-runner/queue-monitor.cc @@ -193,14 +193,18 @@ bool State::getQueuedBuilds(Connection & conn, if (!propagatedFrom) { for (auto & [outputName, optOutputPath] : destStore->queryPartialDerivationOutputMap(ex.step->drvPath, &*localStore)) { - // ca-derivations not actually supported yet - assert(optOutputPath); - auto res = txn.exec_params - ("select max(s.build) from BuildSteps s join BuildStepOutputs o on s.build = o.build where path = $1 and startTime != 0 and stopTime != 0 and status = 1", - localStore->printStorePath(*optOutputPath)); - if (!res[0][0].is_null()) { - propagatedFrom = res[0][0].as(); - break; + constexpr std::string_view common = "select max(s.build) from BuildSteps s join BuildStepOutputs o on s.build = o.build where startTime != 0 and stopTime != 0 and status = 1"; + auto res = optOutputPath + ? txn.exec_params( + std::string { common } + " and path = $1", + localStore->printStorePath(*optOutputPath)) + : txn.exec_params( + std::string { common } + " and drvPath = $1 and name = $2", + localStore->printStorePath(ex.step->drvPath), + outputName); + if (!res[0][0].is_null()) { + propagatedFrom = res[0][0].as(); + break; } } } diff --git a/src/lib/Hydra/Controller/Build.pm b/src/lib/Hydra/Controller/Build.pm index 2d74f86a..c3869838 100644 --- a/src/lib/Hydra/Controller/Build.pm +++ b/src/lib/Hydra/Controller/Build.pm @@ -78,14 +78,16 @@ sub build_GET { $c->stash->{template} = 'build.tt'; $c->stash->{isLocalStore} = isLocalStore(); + # XXX: If the derivation is content-addressed then this will always return + # false because `$_->path` will be empty $c->stash->{available} = $c->stash->{isLocalStore} - ? all { isValidPath($_->path) } $build->buildoutputs->all + ? all { $_->path && isValidPath($_->path) } $build->buildoutputs->all : 1; $c->stash->{drvAvailable} = isValidPath $build->drvpath; if ($build->finished && $build->iscachedbuild) { - my $path = ($build->buildoutputs)[0]->path or die; + my $path = ($build->buildoutputs)[0]->path or undef; my $cachedBuildStep = findBuildStepByOutPath($self, $c, $path); if (defined $cachedBuildStep) { $c->stash->{cachedBuild} = $cachedBuildStep->build; diff --git a/src/script/hydra-eval-jobset b/src/script/hydra-eval-jobset index c6f6c275..7ed7ebe8 100755 --- a/src/script/hydra-eval-jobset +++ b/src/script/hydra-eval-jobset @@ -438,13 +438,17 @@ sub checkBuild { # new build to be scheduled if the meta.maintainers field is # changed? if (defined $prevEval) { + my $pathOrDrvConstraint = defined $firstOutputPath + ? { path => $firstOutputPath } + : { drvPath => $drvPath }; + my ($prevBuild) = $prevEval->builds->search( # The "project" and "jobset" constraints are # semantically unnecessary (because they're implied by # the eval), but they give a factor 1000 speedup on # the Nixpkgs jobset with PostgreSQL. { jobset_id => $jobset->get_column('id'), job => $jobName, - name => $firstOutputName, path => $firstOutputPath }, + name => $firstOutputName, %$pathOrDrvConstraint }, { rows => 1, columns => ['id', 'finished'], join => ['buildoutputs'] }); if (defined $prevBuild) { #print STDERR " already scheduled/built as build ", $prevBuild->id, "\n"; diff --git a/t/lib/HydraTestContext.pm b/t/lib/HydraTestContext.pm index a22c3df1..e1a5b226 100644 --- a/t/lib/HydraTestContext.pm +++ b/t/lib/HydraTestContext.pm @@ -39,6 +39,8 @@ use Hydra::Helper::Exec; sub new { my ($class, %opts) = @_; + my $deststoredir; + # Cleanup will be managed by yath. By the default it will be cleaned # up, but can be kept to aid in debugging test failures. my $dir = File::Temp->newdir(CLEANUP => 0); @@ -55,6 +57,7 @@ sub new { my $hydra_config = $opts{'hydra_config'} || ""; $hydra_config = "queue_runner_metrics_address = 127.0.0.1:0\n" . $hydra_config; if ($opts{'use_external_destination_store'} // 1) { + $deststoredir = "$dir/nix/dest-store"; $hydra_config = "store_uri = file://$dir/nix/dest-store\n" . $hydra_config; } @@ -81,7 +84,8 @@ sub new { nix_state_dir => $nix_state_dir, nix_log_dir => $nix_log_dir, testdir => abs_path(dirname(__FILE__) . "/.."), - jobsdir => abs_path(dirname(__FILE__) . "/../jobs") + jobsdir => abs_path(dirname(__FILE__) . "/../jobs"), + deststoredir => $deststoredir, }, $class; if ($opts{'before_init'}) { From 323b556dc8c348f1f9d5bf5b8a35f2617bfe600c Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 25 Jan 2024 21:03:08 -0500 Subject: [PATCH 27/32] Minimal CA support This verison has a worse UI, but also chnages the schema less: One non-null constraint is removed, but no new columns are added. Co-Authored-By: Andrea Ciceri Co-Authored-By: regnat --- doc/manual/src/projects.md | 7 +++ src/lib/Hydra/Controller/Root.pm | 29 +++++++++ src/lib/Hydra/Schema/Result/BuildOutputs.pm | 8 +-- .../Hydra/Schema/Result/BuildStepOutputs.pm | 8 +-- src/sql/hydra.sql | 4 +- t/content-addressed/basic.t | 61 +++++++++++++++++++ .../without-experimental-feature.t | 28 +++++++++ t/jobs/config.nix.in | 5 ++ t/jobs/content-addressed.nix | 35 +++++++++++ t/jobs/dir-with-file-builder.sh | 7 +++ t/queue-runner/notifications.t | 2 +- 11 files changed, 183 insertions(+), 11 deletions(-) create mode 100644 t/content-addressed/basic.t create mode 100644 t/content-addressed/without-experimental-feature.t create mode 100644 t/jobs/content-addressed.nix create mode 100755 t/jobs/dir-with-file-builder.sh diff --git a/doc/manual/src/projects.md b/doc/manual/src/projects.md index a399406d..f7c4975f 100644 --- a/doc/manual/src/projects.md +++ b/doc/manual/src/projects.md @@ -404,3 +404,10 @@ analogous: | `String value` | `gitea_status_repo` | *Name of the `Git checkout` input* | | `String value` | `gitea_http_url` | *Public URL of `gitea`*, optional | +Content-addressed derivations +----------------------------- + +Hydra can to a certain extent use the [`ca-derivations` experimental Nix feature](https://github.com/NixOS/rfcs/pull/62). +To use it, make sure that the Nix version you use is at least as recent as the one used in hydra's flake. + +Be warned that this support is still highly experimental, and anything beyond the basic functionality might be broken at that point. diff --git a/src/lib/Hydra/Controller/Root.pm b/src/lib/Hydra/Controller/Root.pm index c6843d29..548cfac3 100644 --- a/src/lib/Hydra/Controller/Root.pm +++ b/src/lib/Hydra/Controller/Root.pm @@ -18,6 +18,8 @@ use Net::Prometheus; use Types::Standard qw/StrMatch/; use constant NARINFO_REGEX => qr{^([a-z0-9]{32})\.narinfo$}; +# e.g.: https://hydra.example.com/realisations/sha256:a62128132508a3a32eef651d6467695944763602f226ac630543e947d9feb140!out.doi +use constant REALISATIONS_REGEX => qr{^(sha256:[a-z0-9]{64}![a-z]+)\.doi$}; # Put this controller at top-level. __PACKAGE__->config->{namespace} = ''; @@ -355,6 +357,33 @@ sub nix_cache_info :Path('nix-cache-info') :Args(0) { } +sub realisations :Path('realisations') :Args(StrMatch[REALISATIONS_REGEX]) { + my ($self, $c, $realisation) = @_; + + if (!isLocalStore) { + notFound($c, "There is no binary cache here."); + } + + else { + my ($rawDrvOutput) = $realisation =~ REALISATIONS_REGEX; + my $rawRealisation = queryRawRealisation($rawDrvOutput); + + if (!$rawRealisation) { + $c->response->status(404); + $c->response->content_type('text/plain'); + $c->stash->{plain}->{data} = "does not exist\n"; + $c->forward('Hydra::View::Plain'); + setCacheHeaders($c, 60 * 60); + return; + } + + $c->response->content_type('text/plain'); + $c->stash->{plain}->{data} = $rawRealisation; + $c->forward('Hydra::View::Plain'); + } +} + + sub narinfo :Path :Args(StrMatch[NARINFO_REGEX]) { my ($self, $c, $narinfo) = @_; diff --git a/src/lib/Hydra/Schema/Result/BuildOutputs.pm b/src/lib/Hydra/Schema/Result/BuildOutputs.pm index 9fc4f7c7..3997b497 100644 --- a/src/lib/Hydra/Schema/Result/BuildOutputs.pm +++ b/src/lib/Hydra/Schema/Result/BuildOutputs.pm @@ -49,7 +49,7 @@ __PACKAGE__->table("buildoutputs"); =head2 path data_type: 'text' - is_nullable: 0 + is_nullable: 1 =cut @@ -59,7 +59,7 @@ __PACKAGE__->add_columns( "name", { data_type => "text", is_nullable => 0 }, "path", - { data_type => "text", is_nullable => 0 }, + { data_type => "text", is_nullable => 1 }, ); =head1 PRIMARY KEY @@ -94,8 +94,8 @@ __PACKAGE__->belongs_to( ); -# Created by DBIx::Class::Schema::Loader v0.07049 @ 2021-08-26 12:02:36 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:gU+kZ6A0ISKpaXGRGve8mg +# Created by DBIx::Class::Schema::Loader v0.07049 @ 2022-06-30 12:02:32 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:Jsabm3YTcI7YvCuNdKP5Ng my %hint = ( columns => [ diff --git a/src/lib/Hydra/Schema/Result/BuildStepOutputs.pm b/src/lib/Hydra/Schema/Result/BuildStepOutputs.pm index 016a35fe..6d997a8c 100644 --- a/src/lib/Hydra/Schema/Result/BuildStepOutputs.pm +++ b/src/lib/Hydra/Schema/Result/BuildStepOutputs.pm @@ -55,7 +55,7 @@ __PACKAGE__->table("buildstepoutputs"); =head2 path data_type: 'text' - is_nullable: 0 + is_nullable: 1 =cut @@ -67,7 +67,7 @@ __PACKAGE__->add_columns( "name", { data_type => "text", is_nullable => 0 }, "path", - { data_type => "text", is_nullable => 0 }, + { data_type => "text", is_nullable => 1 }, ); =head1 PRIMARY KEY @@ -119,8 +119,8 @@ __PACKAGE__->belongs_to( ); -# Created by DBIx::Class::Schema::Loader v0.07049 @ 2021-08-26 12:02:36 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:gxp8rOjpRVen4YbIjomHTw +# Created by DBIx::Class::Schema::Loader v0.07049 @ 2022-06-30 12:02:32 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:Bad70CRTt7zb2GGuRoQ++Q # You can replace this text with custom code or comments, and it will be preserved on regeneration diff --git a/src/sql/hydra.sql b/src/sql/hydra.sql index eaae6da3..e9457972 100644 --- a/src/sql/hydra.sql +++ b/src/sql/hydra.sql @@ -247,7 +247,7 @@ create trigger BuildBumped after update on Builds for each row create table BuildOutputs ( build integer not null, name text not null, - path text not null, + path text, primary key (build, name), foreign key (build) references Builds(id) on delete cascade ); @@ -303,7 +303,7 @@ create table BuildStepOutputs ( build integer not null, stepnr integer not null, name text not null, - path text not null, + path text, primary key (build, stepnr, name), foreign key (build) references Builds(id) on delete cascade, foreign key (build, stepnr) references BuildSteps(build, stepnr) on delete cascade diff --git a/t/content-addressed/basic.t b/t/content-addressed/basic.t new file mode 100644 index 00000000..6597e727 --- /dev/null +++ b/t/content-addressed/basic.t @@ -0,0 +1,61 @@ +use feature 'unicode_strings'; +use strict; +use warnings; +use Setup; + +my %ctx = test_init( + nix_config => qq| + experimental-features = ca-derivations + |, +); + +require Hydra::Schema; +require Hydra::Model::DB; + +use JSON::MaybeXS; + +use HTTP::Request::Common; +use Test2::V0; +require Catalyst::Test; +Catalyst::Test->import('Hydra'); + +my $db = Hydra::Model::DB->new; +hydra_setup($db); + +my $project = $db->resultset('Projects')->create({name => "tests", displayname => "", owner => "root"}); + +my $jobset = createBaseJobset("content-addressed", "content-addressed.nix", $ctx{jobsdir}); + +ok(evalSucceeds($jobset), "Evaluating jobs/content-addressed.nix should exit with return code 0"); +is(nrQueuedBuildsForJobset($jobset), 5, "Evaluating jobs/content-addressed.nix should result in 4 builds"); + +for my $build (queuedBuildsForJobset($jobset)) { + ok(runBuild($build), "Build '".$build->job."' from jobs/content-addressed.nix should exit with code 0"); + my $newbuild = $db->resultset('Builds')->find($build->id); + is($newbuild->finished, 1, "Build '".$build->job."' from jobs/content-addressed.nix should be finished."); + my $expected = $build->job eq "fails" ? 1 : $build->job =~ /with_failed/ ? 6 : 0; + is($newbuild->buildstatus, $expected, "Build '".$build->job."' from jobs/content-addressed.nix should have buildstatus $expected."); + + my $response = request("/build/".$build->id); + ok($response->is_success, "The 'build' page for build '".$build->job."' should load properly"); + + if ($newbuild->buildstatus == 0) { + my $buildOutputs = $newbuild->buildoutputs; + for my $output ($newbuild->buildoutputs) { + # XXX: This hardcodes /nix/store/. + # It's fine because in practice the nix store for the tests will be of + # the form `/some/thing/nix/store/`, but it would be cleaner if there + # was a way to query Nix for its store dir? + like( + $output->path, qr|/nix/store/|, + "Output '".$output->name."' of build '".$build->job."' should be a valid store path" + ); + } + } + +} + +isnt(<$ctx{deststoredir}/realisations/*>, "", "The destination store should have the realisations of the built derivations registered"); + +done_testing; + diff --git a/t/content-addressed/without-experimental-feature.t b/t/content-addressed/without-experimental-feature.t new file mode 100644 index 00000000..a37d138e --- /dev/null +++ b/t/content-addressed/without-experimental-feature.t @@ -0,0 +1,28 @@ +use feature 'unicode_strings'; +use strict; +use warnings; +use Setup; + +my %ctx = test_init(); + +require Hydra::Schema; +require Hydra::Model::DB; + +use JSON::MaybeXS; + +use HTTP::Request::Common; +use Test2::V0; +require Catalyst::Test; +Catalyst::Test->import('Hydra'); + +my $db = Hydra::Model::DB->new; +hydra_setup($db); + +my $project = $db->resultset('Projects')->create({name => "tests", displayname => "", owner => "root"}); + +my $jobset = createBaseJobset("content-addressed", "content-addressed.nix", $ctx{jobsdir}); + +ok(evalSucceeds($jobset), "Evaluating jobs/content-addressed.nix without the experimental feature should exit with return code 0"); +is(nrQueuedBuildsForJobset($jobset), 0, "Evaluating jobs/content-addressed.nix without the experimental Nix feature should result in 0 build"); + +done_testing; diff --git a/t/jobs/config.nix.in b/t/jobs/config.nix.in index 51b6c06f..41776341 100644 --- a/t/jobs/config.nix.in +++ b/t/jobs/config.nix.in @@ -6,4 +6,9 @@ rec { system = builtins.currentSystem; PATH = path; } // args); + mkContentAddressedDerivation = args: mkDerivation ({ + __contentAddressed = true; + outputHashMode = "recursive"; + outputHashAlgo = "sha256"; + } // args); } diff --git a/t/jobs/content-addressed.nix b/t/jobs/content-addressed.nix new file mode 100644 index 00000000..65496df5 --- /dev/null +++ b/t/jobs/content-addressed.nix @@ -0,0 +1,35 @@ +let cfg = import ./config.nix; in +rec { + empty_dir = + cfg.mkContentAddressedDerivation { + name = "empty-dir"; + builder = ./empty-dir-builder.sh; + }; + + fails = + cfg.mkContentAddressedDerivation { + name = "fails"; + builder = ./fail.sh; + }; + + succeed_with_failed = + cfg.mkContentAddressedDerivation { + name = "succeed-with-failed"; + builder = ./succeed-with-failed.sh; + }; + + caDependingOnCA = + cfg.mkContentAddressedDerivation { + name = "ca-depending-on-ca"; + builder = ./dir-with-file-builder.sh; + FOO = empty_dir; + }; + + nonCaDependingOnCA = + cfg.mkDerivation { + name = "non-ca-depending-on-ca"; + builder = ./dir-with-file-builder.sh; + FOO = empty_dir; + }; +} + diff --git a/t/jobs/dir-with-file-builder.sh b/t/jobs/dir-with-file-builder.sh new file mode 100755 index 00000000..e51c6558 --- /dev/null +++ b/t/jobs/dir-with-file-builder.sh @@ -0,0 +1,7 @@ +#! /bin/sh + +# Workaround for https://github.com/NixOS/nix/pull/6051 +echo "some output" + +mkdir $out +echo foo > $out/a-file diff --git a/t/queue-runner/notifications.t b/t/queue-runner/notifications.t index 1966cde1..d0e72409 100644 --- a/t/queue-runner/notifications.t +++ b/t/queue-runner/notifications.t @@ -8,7 +8,7 @@ my $binarycachedir = File::Temp->newdir(); my $ctx = test_context( nix_config => qq| - experimental-features = nix-command + experimental-features = nix-command ca-derivations substituters = file://${binarycachedir}?trusted=1 |, hydra_config => q| From 5ee0e443e4ea90888e74b90d2e4198494e8eb597 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 26 Jan 2024 01:08:11 -0500 Subject: [PATCH 28/32] Remove now-unneeded workaround --- t/jobs/empty-dir-builder.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/t/jobs/empty-dir-builder.sh b/t/jobs/empty-dir-builder.sh index 949216e0..addc7ef6 100755 --- a/t/jobs/empty-dir-builder.sh +++ b/t/jobs/empty-dir-builder.sh @@ -1,6 +1,3 @@ #! /bin/sh -# Workaround for https://github.com/NixOS/nix/pull/6051 -echo "some output" - mkdir $out From c62eaf248f1fe7653f37427e70ce4741cf9a1c2d Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 26 Jan 2024 01:20:07 -0500 Subject: [PATCH 29/32] Remove now-unneeded workaround --- t/jobs/dir-with-file-builder.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/t/jobs/dir-with-file-builder.sh b/t/jobs/dir-with-file-builder.sh index e51c6558..8592a1e8 100755 --- a/t/jobs/dir-with-file-builder.sh +++ b/t/jobs/dir-with-file-builder.sh @@ -1,7 +1,4 @@ #! /bin/sh -# Workaround for https://github.com/NixOS/nix/pull/6051 -echo "some output" - mkdir $out echo foo > $out/a-file From 8477009310c0cde111af3da9313d3c99902a48c7 Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Fri, 26 Jan 2024 17:28:07 +0100 Subject: [PATCH 30/32] doc/manual: fix instructions in contribution guidelines In 5db374cb500b687039ba4701b205ca7dfa67caba the `bootstrap` script was removed, however it's still referenced in the contribution guidelines. Change that to `autoreconfPhase` as intended by the commit. --- doc/manual/src/hacking.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/hacking.md b/doc/manual/src/hacking.md index 72a74c82..49c17395 100644 --- a/doc/manual/src/hacking.md +++ b/doc/manual/src/hacking.md @@ -18,7 +18,7 @@ $ nix-shell To build Hydra, you should then do: ```console -[nix-shell]$ ./bootstrap +[nix-shell]$ autoreconfPhase [nix-shell]$ configurePhase [nix-shell]$ make ``` From b4c91b5a6ab91177d2c52c5b68c880277b4fd35f Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Fri, 26 Jan 2024 17:30:07 +0100 Subject: [PATCH 31/32] package: move foreman to nativeCheckInputs In 1bd195a5138b3c69c52110d713c706cb4908ba16 strictDeps was set for the Hydra package. As a result, `checkInputs` aren't available anymore in the local dev-shell which is the sole purpose of foreman, to start services and a database for development. --- package.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.nix b/package.nix index 0f8f004d..f8b1849f 100644 --- a/package.nix +++ b/package.nix @@ -183,6 +183,7 @@ stdenv.mkDerivation (finalAttrs: { nativeCheckInputs = [ bzip2 darcs + foreman top-git mercurial subversion @@ -194,7 +195,6 @@ stdenv.mkDerivation (finalAttrs: { checkInputs = [ cacert - foreman glibcLocales libressl.nc python3 From b503280256bed6ced0c79d32ee17dda72827a381 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 26 Jan 2024 11:53:58 -0500 Subject: [PATCH 32/32] Add migration to drop non-null constraints --- src/sql/upgrade-84.sql | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 src/sql/upgrade-84.sql diff --git a/src/sql/upgrade-84.sql b/src/sql/upgrade-84.sql new file mode 100644 index 00000000..bf142b30 --- /dev/null +++ b/src/sql/upgrade-84.sql @@ -0,0 +1,4 @@ +-- CA derivations do not have statically known output paths. The values +-- are only filled in after the build runs. +ALTER TABLE BuildStepOutputs ALTER COLUMN path DROP NOT NULL; +ALTER TABLE BuildOutputs ALTER COLUMN path DROP NOT NULL;