Merge pull request #1532 from NixOS/proto-dedup

Deduplicate protocol code more with `ServeProto::BasicClientConnection`

flake.lock

@@ -3,16 +3,16 @@
     "nix": {
       "flake": false,
       "locked": {
-        "lastModified": 1758562014,
-        "narHash": "sha256-IazqNpt3jNldKy+rivmlGuo9pC1IczV0Xjk5+5EQEzQ=",
+        "lastModified": 1759956402,
+        "narHash": "sha256-CM27YK+KMi3HLRXqjPaJwkTabmKW+CDXOE3kMMtXH3s=",
         "owner": "NixOS",
         "repo": "nix",
-        "rev": "f2b45e014b909bb5e6a9f99a8a511deed3b3e2a4",
+        "rev": "3019db2c87006817b6201113ad4ceee0c53c3b62",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
-        "ref": "2.30-maintenance",
+        "ref": "2.31-maintenance",
         "repo": "nix",
         "type": "github"
       }
@@ -20,15 +20,16 @@
     "nix-eval-jobs": {
       "flake": false,
       "locked": {
-        "lastModified": 1752066681,
-        "narHash": "sha256-e10zYtdc5nDHs2iLf/h92+uW5WOUZGqohfaU919B/TI=",
+        "lastModified": 1757626891,
+        "narHash": "sha256-VrHPtHxVIboqgnw+tlCQepgtBOhBvU5hxbMHsPo8LAc=",
         "owner": "nix-community",
         "repo": "nix-eval-jobs",
-        "rev": "fae2b990f19c10b7d5718b6eff1df30188ca780e",
+        "rev": "c975efc5b2bec0c1ff93c67de4a03306af258ff7",
         "type": "github"
       },
       "original": {
         "owner": "nix-community",
+        "ref": "v2.31.0",
         "repo": "nix-eval-jobs",
         "type": "github"
       }

flake.nix

@@ -4,13 +4,13 @@
   inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-25.05-small";
 
   inputs.nix = {
-    url = "github:NixOS/nix/2.30-maintenance";
+    url = "github:NixOS/nix/2.31-maintenance";
     # We want to control the deps precisely
     flake = false;
   };
 
   inputs.nix-eval-jobs = {
-    url = "github:nix-community/nix-eval-jobs";
+    url = "github:nix-community/nix-eval-jobs/v2.31.0";
     # We want to control the deps precisely
     flake = false;
   };

meson.build

@@ -4,7 +4,7 @@ project('hydra', 'cpp',
   default_options: [
     'debug=true',
     'optimization=2',
-    'cpp_std=c++20',
+    'cpp_std=c++23',
   ],
 )
 

nixos-modules/hydra.nix

@@ -364,7 +364,7 @@ in
         requires = [ "hydra-init.service" ];
         restartTriggers = [ hydraConf ];
         after = [ "hydra-init.service" "network.target" ];
-        path = with pkgs; [ hostname-debian cfg.package jq ];
+        path = with pkgs; [ hostname-debian cfg.package ];
         environment = env // {
           HYDRA_DBI = "${env.HYDRA_DBI};application_name=hydra-evaluator";
         };

src/hydra-queue-runner/build-remote.cc

@@ -103,9 +103,9 @@ static void copyClosureTo(
     std::unique_lock<std::timed_mutex> sendLock(conn.machine->state->sendLock,
         std::chrono::seconds(600));
 
-    conn.to << ServeProto::Command::ImportPaths;
-    destStore.exportPaths(missing, conn.to);
-    conn.to.flush();
+    conn.importPaths(destStore, [&](Sink & sink) {
+        destStore.exportPaths(missing, sink);
+    });
 
     if (readInt(conn.from) != 1)
         throw Error("remote machine failed to import closure");
@@ -298,11 +298,10 @@ static void copyPathFromRemote(
               lambda function only gets executed if someone tries to read
               from source2, we will send the command from here rather
               than outside the lambda. */
-          conn.to << ServeProto::Command::DumpStorePath << localStore.printStorePath(info.path);
-          conn.to.flush();
-
-          TeeSource tee(conn.from, sink);
-          extractNarData(tee, localStore.printStorePath(info.path), narMembers);
+          conn.narFromPath(localStore, info.path, [&](Source & source) {
+              TeeSource tee(source, sink);
+              extractNarData(tee, localStore.printStorePath(info.path), narMembers);
+          });
       });
 
       destStore.addToStore(info, *source2, NoRepair, NoCheckSigs);

src/hydra-queue-runner/queue-monitor.cc

@@ -488,10 +488,11 @@ Step::ptr State::createStep(ref<Store> destStore,
        runnable while step->created == false. */
     step->drv = std::make_unique<Derivation>(localStore->readDerivation(drvPath));
     {
-        auto parsedOpt = StructuredAttrs::tryParse(step->drv->env);
         try {
             step->drvOptions = std::make_unique<DerivationOptions>(
-                DerivationOptions::fromStructuredAttrs(step->drv->env, parsedOpt ? &*parsedOpt : nullptr));
+                DerivationOptions::fromStructuredAttrs(
+                    step->drv->env,
+                    step->drv->structuredAttrs ? &*step->drv->structuredAttrs : nullptr));
         } catch (Error & e) {
             e.addTrace({}, "while parsing derivation '%s'", localStore->printStorePath(drvPath));
             throw;

src/hydra-queue-runner/state.hh

@@ -27,6 +27,7 @@
 #include <nix/store/serve-protocol-impl.hh>
 #include <nix/store/serve-protocol-connection.hh>
 #include <nix/store/machines.hh>
+#include <nix/store/globals.hh>
 
 
 typedef unsigned int BuildID;

src/lib/Hydra/Plugin/BitBucketPulls.pm

@@ -10,7 +10,6 @@ use Hydra::Helper::CatalystUtils;
 use Hydra::Helper::Nix;
 use File::Temp;
 use POSIX qw(strftime);
-use IPC::Run qw(run);
 
 sub supportedInputTypes {
     my ($self, $inputTypes) = @_;
@@ -45,12 +44,11 @@ sub fetchInput {
     my $ua = LWP::UserAgent->new();
     _iterate("https://api.bitbucket.com/2.0/repositories/$owner/$repo/pullrequests?state=OPEN", $auth, \%pulls, $ua);
     my $tempdir = File::Temp->newdir("bitbucket-pulls" . "XXXXX", TMPDIR => 1);
-    my $filename = "$tempdir/bitbucket-pulls.json";
+    my $filename = "$tempdir/bitbucket-pulls-sorted.json";
     open(my $fh, ">", $filename) or die "Cannot open $filename for writing: $!";
-    print $fh encode_json \%pulls;
+    print $fh JSON::MaybeXS->new(canonical => 1, pretty => 1)->encode(\%pulls);
     close $fh;
-    run(["jq", "-S", "."], '<', $filename, '>', "$tempdir/bitbucket-pulls-sorted.json") or die "jq command failed: $?";
-    my $storePath = addToStore("$tempdir/bitbucket-pulls-sorted.json");
+    my $storePath = addToStore($filename);
     my $timestamp = time;
     return { storePath => $storePath, revision => strftime "%Y%m%d%H%M%S", gmtime($timestamp) };
 }

src/lib/Hydra/Plugin/GithubRefs.pm

@@ -10,7 +10,6 @@ use Hydra::Helper::CatalystUtils;
 use Hydra::Helper::Nix;
 use File::Temp;
 use POSIX qw(strftime);
-use IPC::Run qw(run);
 
 =head1 NAME
 
@@ -112,12 +111,11 @@ sub fetchInput {
     my $ua = LWP::UserAgent->new();
     _iterate("$githubEndpoint/repos/$owner/$repo/git/matching-refs/$type/$prefix?per_page=100", $auth, \%refs, $ua);
     my $tempdir = File::Temp->newdir("github-refs" . "XXXXX", TMPDIR => 1);
-    my $filename = "$tempdir/github-refs.json";
+    my $filename = "$tempdir/github-refs-sorted.json";
     open(my $fh, ">", $filename) or die "Cannot open $filename for writing: $!";
-    print $fh encode_json \%refs;
+    print $fh JSON::MaybeXS->new(canonical => 1, pretty => 1)->encode(\%refs);
     close $fh;
-    run(["jq", "-S", "."], '<', $filename, '>', "$tempdir/github-refs-sorted.json") or die "jq command failed: $?";
-    my $storePath = addToStore("$tempdir/github-refs-sorted.json");
+    my $storePath = addToStore($filename);
     my $timestamp = time;
     return { storePath => $storePath, revision => strftime "%Y%m%d%H%M%S", gmtime($timestamp) };
 }

src/lib/Hydra/Plugin/GitlabPulls.pm

@@ -24,7 +24,6 @@ use Hydra::Helper::CatalystUtils;
 use Hydra::Helper::Nix;
 use File::Temp;
 use POSIX qw(strftime);
-use IPC::Run qw(run);
 
 sub supportedInputTypes {
     my ($self, $inputTypes) = @_;
@@ -83,12 +82,11 @@ sub fetchInput {
     _iterate($url, $baseUrl, \%pulls, $ua, $target_repo_url);
 
     my $tempdir = File::Temp->newdir("gitlab-pulls" . "XXXXX", TMPDIR => 1);
-    my $filename = "$tempdir/gitlab-pulls.json";
+    my $filename = "$tempdir/gitlab-pulls-sorted.json";
     open(my $fh, ">", $filename) or die "Cannot open $filename for writing: $!";
-    print $fh encode_json \%pulls;
+    print $fh JSON::MaybeXS->new(canonical => 1, pretty => 1)->encode(\%pulls);
     close $fh;
-    run(["jq", "-S", "."], '<', $filename, '>', "$tempdir/gitlab-pulls-sorted.json") or die "jq command failed: $?";
-    my $storePath = addToStore("$tempdir/gitlab-pulls-sorted.json");
+    my $storePath = addToStore($filename);
     my $timestamp = time;
     return { storePath => $storePath, revision => strftime "%Y%m%d%H%M%S", gmtime($timestamp) };
 }