Merge remote-tracking branch 'upstream/nix-2.22' into use-store-api

2025-02-14 15:51:38 -05:00
parent a94c1aeac4 60dd7ec187
commit 9b0d74ed84
39 changed files with 599 additions and 389 deletions
--- a/src/hydra-eval-jobs/hydra-eval-jobs.cc
+++ b/src/hydra-eval-jobs/hydra-eval-jobs.cc
@@ -102,8 +102,8 @@ static std::string queryMetaStrings(EvalState & state, PackageInfo & drv, const
            for (unsigned int n = 0; n < v.listSize(); ++n)
                rec(*v.listElems()[n]);
        else if (v.type() == nAttrs) {
-            auto a = v.attrs->find(state.symbols.create(subAttribute));
-            if (a != v.attrs->end())
+            auto a = v.attrs()->find(state.symbols.create(subAttribute));
+            if (a != v.attrs()->end())
                res.push_back(std::string(state.forceString(*a->value, a->pos, "while evaluating meta attributes")));
        }
    };
@@ -138,12 +138,12 @@ static void worker(

        callFlake(state, lockedFlake, *vFlake);

-        auto vOutputs = vFlake->attrs->get(state.symbols.create("outputs"))->value;
+        auto vOutputs = vFlake->attrs()->get(state.symbols.create("outputs"))->value;
        state.forceValue(*vOutputs, noPos);

-        auto aHydraJobs = vOutputs->attrs->get(state.symbols.create("hydraJobs"));
+        auto aHydraJobs = vOutputs->attrs()->get(state.symbols.create("hydraJobs"));
        if (!aHydraJobs)
-            aHydraJobs = vOutputs->attrs->get(state.symbols.create("checks"));
+            aHydraJobs = vOutputs->attrs()->get(state.symbols.create("checks"));
        if (!aHydraJobs)
            throw Error("flake '%s' does not provide any Hydra jobs or checks", flakeRef);

@@ -204,9 +204,9 @@ static void worker(
                job["isChannel"] = drv->queryMetaBool("isHydraChannel", false);

                /* If this is an aggregate, then get its constituents. */
-                auto a = v->attrs->get(state.symbols.create("_hydraAggregate"));
+                auto a = v->attrs()->get(state.symbols.create("_hydraAggregate"));
                if (a && state.forceBool(*a->value, a->pos, "while evaluating the `_hydraAggregate` attribute")) {
-                    auto a = v->attrs->get(state.symbols.create("constituents"));
+                    auto a = v->attrs()->get(state.symbols.create("constituents"));
                    if (!a)
                        state.error<EvalError>("derivation must have a ‘constituents’ attribute").debugThrow();

@@ -260,7 +260,7 @@ static void worker(
            else if (v->type() == nAttrs) {
                auto attrs = nlohmann::json::array();
                StringSet ss;
-                for (auto & i : v->attrs->lexicographicOrder(state.symbols)) {
+                for (auto & i : v->attrs()->lexicographicOrder(state.symbols)) {
                    std::string name(state.symbols[i->name]);
                    if (name.find(' ') != std::string::npos) {
                        printError("skipping job with illegal name '%s'", name);
@@ -368,7 +368,7 @@ int main(int argc, char * * argv)
                            ]()
                            {
                                try {
-                                    EvalState state(myArgs.searchPath, openStore());
+                                    EvalState state(myArgs.lookupPath, openStore());
                                    Bindings & autoArgs = *myArgs.getAutoArgs(state);
                                    worker(state, autoArgs, *to, *from);
                                } catch (Error & e) {
--- a/src/hydra-queue-runner/hydra-queue-runner.cc
+++ b/src/hydra-queue-runner/hydra-queue-runner.cc
@@ -164,7 +164,7 @@ void State::parseMachines(const std::string & contents)
                ? string2Int<MaxJobs>(tokens[3]).value()
                : 1,
            // `speedFactor`
-            atof(tokens[4].c_str()),
+            std::stof(tokens[4].c_str()),
            // `supportedFeatures`
            std::move(supportedFeatures),
            // `mandatoryFeatures`
--- a/src/lib/Hydra/Config.pm
+++ b/src/lib/Hydra/Config.pm
@@ -95,6 +95,7 @@ sub get_legacy_ldap_config {
            "hydra_bump-to-front" => [ "bump-to-front" ],
            "hydra_cancel-build" => [ "cancel-build" ],
            "hydra_create-projects" => [ "create-projects" ],
+            "hydra_eval-jobset" => [ "eval-jobset" ],
            "hydra_restart-jobs" => [ "restart-jobs" ],
        },
    };
@@ -159,6 +160,7 @@ sub valid_roles {
        "bump-to-front",
        "cancel-build",
        "create-projects",
+        "eval-jobset",
        "restart-jobs",
    ];
 }
--- a/src/lib/Hydra/Controller/API.pm
+++ b/src/lib/Hydra/Controller/API.pm
@@ -239,6 +239,8 @@ sub triggerJobset {
 sub push : Chained('api') PathPart('push') Args(0) {
    my ($self, $c) = @_;

+    requirePost($c);
+
    $c->{stash}->{json}->{jobsetsTriggered} = [];

    my $force = exists $c->request->query_params->{force};
@@ -246,19 +248,24 @@ sub push : Chained('api') PathPart('push') Args(0) {
    foreach my $s (@jobsets) {
        my ($p, $j) = parseJobsetName($s);
        my $jobset = $c->model('DB::Jobsets')->find($p, $j);
+        requireEvalJobsetPrivileges($c, $jobset->project);
        next unless defined $jobset && ($force || ($jobset->project->enabled && $jobset->enabled));
        triggerJobset($self, $c, $jobset, $force);
    }

    my @repos = split /,/, ($c->request->query_params->{repos} // "");
    foreach my $r (@repos) {
-        triggerJobset($self, $c, $_, $force) foreach $c->model('DB::Jobsets')->search(
+        my @jobsets = $c->model('DB::Jobsets')->search(
            { 'project.enabled' => 1, 'me.enabled' => 1 },
            {
                join => 'project',
                where => \ [ 'exists (select 1 from JobsetInputAlts where project = me.project and jobset = me.name and value = ?)', [ 'value', $r ] ],
                order_by => 'me.id DESC'
            });
+        foreach my $jobset (@jobsets) {
+            requireEvalJobsetPrivileges($c, $jobset->project);
+            triggerJobset($self, $c, $jobset, $force)
+        }
    }

    $self->status_ok(
@@ -285,6 +292,23 @@ sub push_github : Chained('api') PathPart('push-github') Args(0) {
    $c->response->body("");
 }

+sub push_gitea : Chained('api') PathPart('push-gitea') Args(0) {
+    my ($self, $c) = @_;
+
+    $c->{stash}->{json}->{jobsetsTriggered} = [];
+
+    my $in = $c->request->{data};
+    my $url = $in->{repository}->{clone_url} or die;
+    $url =~ s/.git$//;
+    print STDERR "got push from Gitea repository $url\n";
+
+    triggerJobset($self, $c, $_, 0) foreach $c->model('DB::Jobsets')->search(
+        { 'project.enabled' => 1, 'me.enabled' => 1 },
+        { join => 'project'
+        , where => \ [ 'me.flake like ? or exists (select 1 from JobsetInputAlts where project = me.project and jobset = me.name and value like ?)', [ 'flake', "%$url%"], [ 'value', "%$url%" ] ]
+        });
+    $c->response->body("");
+}


 1;
--- a/src/lib/Hydra/Controller/Root.pm
+++ b/src/lib/Hydra/Controller/Root.pm
@@ -35,6 +35,7 @@ sub noLoginNeeded {

  return $whitelisted ||
         $c->request->path eq "api/push-github" ||
+         $c->request->path eq "api/push-gitea" ||
         $c->request->path eq "google-login" ||
         $c->request->path eq "github-redirect" ||
         $c->request->path eq "github-login" ||
@@ -80,7 +81,7 @@ sub begin :Private {
    $_->supportedInputTypes($c->stash->{inputTypes}) foreach @{$c->hydra_plugins};

    # XSRF protection: require POST requests to have the same origin.
-    if ($c->req->method eq "POST" && $c->req->path ne "api/push-github") {
+    if ($c->req->method eq "POST" && $c->req->path ne "api/push-github" && $c->req->path ne "api/push-gitea") {
        my $referer = $c->req->header('Referer');
        $referer //= $c->req->header('Origin');
        my $base = $c->req->base;
@@ -367,7 +368,7 @@ sub realisations :Path('realisations') :Args(StrMatch[REALISATIONS_REGEX]) {

    else {
        my ($rawDrvOutput) = $realisation =~ REALISATIONS_REGEX;
-        my $rawRealisation = queryRawRealisation($rawDrvOutput);
+        my $rawRealisation = $MACHINE_LOCAL_STORE->queryRawRealisation($rawDrvOutput);

        if (!$rawRealisation) {
            $c->response->status(404);
--- a/src/lib/Hydra/Helper/CatalystUtils.pm
+++ b/src/lib/Hydra/Helper/CatalystUtils.pm
@@ -15,6 +15,7 @@ our @EXPORT = qw(
    forceLogin requireUser requireProjectOwner requireRestartPrivileges requireAdmin requirePost isAdmin isProjectOwner
    requireBumpPrivileges
    requireCancelBuildPrivileges
+    requireEvalJobsetPrivileges
    trim
    getLatestFinishedEval getFirstEval
    paramToList
@@ -186,6 +187,27 @@ sub isProjectOwner {
         defined $c->model('DB::ProjectMembers')->find({ project => $project, userName => $c->user->username }));
 }

+sub hasEvalJobsetRole {
+    my ($c) = @_;
+    return $c->user_exists && $c->check_user_roles("eval-jobset");
+}
+
+sub mayEvalJobset {
+    my ($c, $project) = @_;
+    return
+        $c->user_exists &&
+        (isAdmin($c) ||
+         hasEvalJobsetRole($c) ||
+         isProjectOwner($c, $project));
+}
+
+sub requireEvalJobsetPrivileges {
+    my ($c, $project) = @_;
+    requireUser($c);
+    accessDenied($c, "Only the project members, administrators, and accounts with eval-jobset privileges can perform this operation.")
+        unless mayEvalJobset($c, $project);
+}
+
 sub hasCancelBuildRole {
    my ($c) = @_;
    return $c->user_exists && $c->check_user_roles('cancel-build');
@@ -272,7 +294,7 @@ sub requireAdmin {

 sub requirePost {
    my ($c) = @_;
-    error($c, "Request must be POSTed.") if $c->request->method ne "POST";
+    error($c, "Request must be POSTed.", 405) if $c->request->method ne "POST";
 }


--- a/src/lib/Hydra/Helper/Nix.pm
+++ b/src/lib/Hydra/Helper/Nix.pm
@@ -174,6 +174,9 @@ sub getDrvLogPath {
    for ($fn . $bucketed, $fn . $bucketed . ".bz2") {
        return $_ if -f $_;
    }
+    for ($fn . $bucketed, $fn . $bucketed . ".zst") {
+        return $_ if -f $_;
+    }
    return undef;
 }

--- a/src/lib/Hydra/Plugin/CompressLog.pm
+++ b/src/lib/Hydra/Plugin/CompressLog.pm
@@ -9,11 +9,24 @@ use Hydra::Helper::CatalystUtils;
 sub stepFinished {
    my ($self, $step, $logPath) = @_;

-    my $doCompress = $self->{config}->{'compress_build_logs'} // "1";
+    my $doCompress = $self->{config}->{'compress_build_logs'} // '1';
+    my $silent = $self->{config}->{'compress_build_logs_silent'} // '0';
+    my $compression = $self->{config}->{'compress_build_logs_compression'} // 'bzip2';

-    if ($doCompress eq "1" && -e $logPath) {
-        print STDERR "compressing ‘$logPath’...\n";
-        system("bzip2", "--force", $logPath);
+    if (not -e $logPath or $doCompress ne "1") {
+        return;
+    }
+
+    if ($silent ne '1') {
+        print STDERR "compressing '$logPath' with $compression...\n";
+    }
+
+    if ($compression eq 'bzip2') {
+        system('bzip2', '--force', $logPath);
+    } elsif ($compression eq 'zstd') {
+        system('zstd', '--rm', '--quiet', '-T0', $logPath);
+    } else {
+        print STDERR "unknown compression type '$compression'\n";
    }
 }

--- a/src/lib/Hydra/View/NixLog.pm
+++ b/src/lib/Hydra/View/NixLog.pm
@@ -16,7 +16,10 @@ sub process {

    my $tail = int($c->stash->{tail} // "0");

-    if ($logPath =~ /\.bz2$/) {
+    if ($logPath =~ /\.zst$/) {
+        my $doTail = $tail ? "| tail -n '$tail'" : "";
+        open($fh, "-|", "zstd -dc < '$logPath' $doTail") or die;
+    } elsif ($logPath =~ /\.bz2$/) {
        my $doTail = $tail ? "| tail -n '$tail'" : "";
        open($fh, "-|", "bzip2 -dc < '$logPath' $doTail") or die;
    } else {
--- a/src/root/common.tt
+++ b/src/root/common.tt
@@ -374,7 +374,7 @@ BLOCK renderInputDiff; %]
              [% ELSIF bi1.uri == bi2.uri && bi1.revision != bi2.revision %]
                [% IF bi1.type == "git" %]
                  <tr><td>
-                    <b>[% bi1.name %]</b></td><td><tt>[% INCLUDE renderDiffUri contents=(bi1.revision.substr(0, 8) _ ' to ' _ bi2.revision.substr(0, 8)) %]</tt>
+                    <b>[% bi1.name %]</b></td><td><tt>[% INCLUDE renderDiffUri contents=(bi1.revision.substr(0, 12) _ ' to ' _ bi2.revision.substr(0, 12)) %]</tt>
                  </td></tr>
                [% ELSE %]
                  <tr><td>
--- a/src/root/jobset.tt
+++ b/src/root/jobset.tt
@@ -205,6 +205,7 @@
        if (!c) return;
        requestJSON({
          url: "[% HTML.escape(c.uri_for('/api/push', { jobsets = project.name _ ':' _ jobset.name, force = "1" })) %]",
+          type: 'POST',
          success: function(data) {
            bootbox.alert("The jobset has been scheduled for evaluation.");
          }
--- a/src/root/user.tt
+++ b/src/root/user.tt
@@ -91,6 +91,7 @@
          [% INCLUDE roleoption mutable=mutable role="restart-jobs" %]
          [% INCLUDE roleoption mutable=mutable role="bump-to-front" %]
          [% INCLUDE roleoption mutable=mutable role="cancel-build" %]
+          [% INCLUDE roleoption mutable=mutable role="eval-jobset" %]
        </p>
      </div>
    </div>