Merge pull request #1458 from NixOS/meson

docs: fix contribution guide for new meson-based build

.gitignore

@@ -3,6 +3,7 @@
 /src/sql/hydra-postgresql.sql
 /src/sql/hydra-sqlite.sql
 /src/sql/tmp.sqlite
+.hydra-data
 result
 result-*
 outputs

README.md

@@ -72,17 +72,16 @@ Make sure **State** at the top of the page is set to "_Enabled_" and click on "_
 You can build Hydra via `nix-build` using the provided [default.nix](./default.nix):
 
 ```
-$ nix-build
+$ nix build
 ```
 
 ### Development Environment
 
 You can use the provided shell.nix to get a working development environment:
 ```
-$ nix-shell
-$ autoreconfPhase
-$ configurePhase # NOTE: not ./configure
-$ make
+$ nix develop
+$ mesonConfigurePhase
+$ ninja
 ```
 
 ### Executing Hydra During Development
@@ -91,9 +90,9 @@ When working on new features or bug fixes you need to be able to run Hydra from
 can be done using [foreman](https://github.com/ddollar/foreman):
 
 ```
-$ nix-shell
+$ nix develop
 $ # hack hack
-$ make
+$ ninja -C build
 $ foreman start
 ```
 
@@ -115,22 +114,24 @@ Start by following the steps in [Development Environment](#development-environme
 Then, you can run the tests and the perlcritic linter together with:
 
 ```console
-$ nix-shell
-$ make check
+$ nix develop
+$ ninja -C build test
 ```
 
 You can run a single test with:
 
 ```
-$ nix-shell
-$ yath test ./t/foo/bar.t
+$ nix develop
+$ cd build
+$ meson test --test-args=../t/Hydra/Event.t testsuite
 ```
 
 And you can run just perlcritic with:
 
 ```
-$ nix-shell
-$ make perlcritic
+$ nix develop
+$ cd build
+$ meson test perlcritic
 ```
 
 ### JSON API

doc/manual/src/hacking.md

@@ -11,12 +11,6 @@ $ cd hydra
 To enter a shell in which all environment variables (such as `PERL5LIB`)
 and dependencies can be found:
 
-```console
-$ nix-shell
-```
-
-of when flakes are enabled:
-
 ```console
 $ nix develop
 ```
@@ -24,15 +18,15 @@ $ nix develop
 To build Hydra, you should then do:
 
 ```console
-[nix-shell]$ autoreconfPhase
-[nix-shell]$ configurePhase
-[nix-shell]$ make -j$(nproc)
+$ mesonConfigurePhase
+$ ninja
 ```
 
 You start a local database, the webserver, and other components with
 foreman:
 
 ```console
+$ ninja -C build
 $ foreman start
 ```
 
@@ -47,18 +41,11 @@ $ ./src/script/hydra-server
 You can run Hydra's test suite with the following:
 
 ```console
-[nix-shell]$ make check
-[nix-shell]$ # to run as many tests as you have cores:
-[nix-shell]$ make check YATH_JOB_COUNT=$NIX_BUILD_CORES
-[nix-shell]$ # or run yath directly:
-[nix-shell]$ yath test
-[nix-shell]$ # to run as many tests as you have cores:
-[nix-shell]$ yath test -j $NIX_BUILD_CORES
+$ meson test
+# to run as many tests as you have cores:
+$ YATH_JOB_COUNT=$NIX_BUILD_CORES meson test
 ```
 
-When using `yath` instead of `make check`, ensure you have run `make`
-in the root of the repository at least once.
-
 **Warning**: Currently, the tests can fail
 if run with high parallelism [due to an issue in
 `Test::PostgreSQL`](https://github.com/TJC/Test-postgresql/issues/40)
@@ -75,7 +62,7 @@ will reload the page every time you save.
 To build Hydra and its dependencies:
 
 ```console
-$ nix-build release.nix -A build.x86_64-linux
+$ nix build .#packages.x86_64-linux.default
 ```
 
 ## Development Tasks

flake.lock

@@ -1,51 +1,10 @@
 {
   "nodes": {
-    "flake-parts": {
-      "inputs": {
-        "nixpkgs-lib": [
-          "nix-eval-jobs",
-          "nixpkgs"
-        ]
-      },
-      "locked": {
-        "lastModified": 1722555600,
-        "narHash": "sha256-XOQkdLafnb/p9ij77byFQjDf5m5QYl9b2REiVClC+x4=",
-        "owner": "hercules-ci",
-        "repo": "flake-parts",
-        "rev": "8471fe90ad337a8074e957b69ca4d0089218391d",
-        "type": "github"
-      },
-      "original": {
-        "owner": "hercules-ci",
-        "repo": "flake-parts",
-        "type": "github"
-      }
-    },
-    "libgit2": {
-      "flake": false,
-      "locked": {
-        "lastModified": 1715853528,
-        "narHash": "sha256-J2rCxTecyLbbDdsyBWn9w7r3pbKRMkI9E7RvRgAqBdY=",
-        "owner": "libgit2",
-        "repo": "libgit2",
-        "rev": "36f7e21ad757a3dacc58cf7944329da6bc1d6e96",
-        "type": "github"
-      },
-      "original": {
-        "owner": "libgit2",
-        "ref": "v1.8.1",
-        "repo": "libgit2",
-        "type": "github"
-      }
-    },
     "nix": {
       "inputs": {
         "flake-compat": [],
         "flake-parts": [],
         "git-hooks-nix": [],
-        "libgit2": [
-          "libgit2"
-        ],
         "nixpkgs": [
           "nixpkgs"
         ],
@@ -53,88 +12,58 @@
         "nixpkgs-regression": []
       },
       "locked": {
-        "lastModified": 1726787955,
-        "narHash": "sha256-XFznzb8L4SdUm9u+w3DPpMWJhffuv+/6+aiVl00slns=",
+        "lastModified": 1739899400,
+        "narHash": "sha256-q/RgA4bB7zWai4oPySq9mch7qH14IEeom2P64SXdqHs=",
         "owner": "NixOS",
         "repo": "nix",
-        "rev": "a7fdef6858dd45b9d7bda7c92324c63faee7f509",
+        "rev": "e310c19a1aeb1ce1ed4d41d5ab2d02db596e0918",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
-        "ref": "2.24-maintenance",
+        "ref": "2.26-maintenance",
         "repo": "nix",
         "type": "github"
       }
     },
     "nix-eval-jobs": {
-      "inputs": {
-        "flake-parts": "flake-parts",
-        "nix-github-actions": [],
-        "nixpkgs": [
-          "nixpkgs"
-        ],
-        "treefmt-nix": "treefmt-nix"
-      },
+      "flake": false,
       "locked": {
-        "lastModified": 1733814344,
-        "narHash": "sha256-3wwtKpS5tUBdjaGeSia7CotonbiRB6K5Kp0dsUt3nzU=",
+        "lastModified": 1743008255,
+        "narHash": "sha256-Lo4KFBNcY8tmBuCmEr2XV0IUZtxXHmbXPNLkov/QSU0=",
         "owner": "nix-community",
         "repo": "nix-eval-jobs",
-        "rev": "889ea1406736b53cf165b6c28398aae3969418d1",
+        "rev": "f7418fc1fa45b96d37baa95ff3c016dd5be3876b",
         "type": "github"
       },
       "original": {
         "owner": "nix-community",
-        "ref": "release-2.24",
         "repo": "nix-eval-jobs",
         "type": "github"
       }
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1726688310,
-        "narHash": "sha256-Xc9lEtentPCEtxc/F1e6jIZsd4MPDYv4Kugl9WtXlz0=",
+        "lastModified": 1739461644,
+        "narHash": "sha256-1o1qR0KYozYGRrnqytSpAhVBYLNBHX+Lv6I39zGRzKM=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "dbebdd67a6006bb145d98c8debf9140ac7e651d0",
+        "rev": "97a719c9f0a07923c957cf51b20b329f9fb9d43f",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
-        "ref": "nixos-24.05-small",
+        "ref": "nixos-24.11-small",
         "repo": "nixpkgs",
         "type": "github"
       }
     },
     "root": {
       "inputs": {
-        "libgit2": "libgit2",
         "nix": "nix",
         "nix-eval-jobs": "nix-eval-jobs",
         "nixpkgs": "nixpkgs"
       }
-    },
-    "treefmt-nix": {
-      "inputs": {
-        "nixpkgs": [
-          "nix-eval-jobs",
-          "nixpkgs"
-        ]
-      },
-      "locked": {
-        "lastModified": 1723303070,
-        "narHash": "sha256-krGNVA30yptyRonohQ+i9cnK+CfCpedg6z3qzqVJcTs=",
-        "owner": "numtide",
-        "repo": "treefmt-nix",
-        "rev": "14c092e0326de759e16b37535161b3cb9770cea3",
-        "type": "github"
-      },
-      "original": {
-        "owner": "numtide",
-        "repo": "treefmt-nix",
-        "type": "github"
-      }
     }
   },
   "root": "root",

flake.nix

@@ -1,25 +1,25 @@
 {
   description = "A Nix-based continuous build system";
 
-  inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-24.05-small";
+  inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-24.11-small";
 
-  inputs.libgit2 = { url = "github:libgit2/libgit2/v1.8.1"; flake = false; };
-  inputs.nix.url = "github:NixOS/nix/2.24-maintenance";
-  inputs.nix.inputs.nixpkgs.follows = "nixpkgs";
-  inputs.nix.inputs.libgit2.follows = "libgit2";
+  inputs.nix = {
+    url = "github:NixOS/nix/2.26-maintenance";
+    inputs.nixpkgs.follows = "nixpkgs";
 
-  inputs.nix-eval-jobs.url = "github:nix-community/nix-eval-jobs/release-2.24";
-  inputs.nix-eval-jobs.inputs.nixpkgs.follows = "nixpkgs";
+    # hide nix dev tooling from our lock file
+    inputs.flake-parts.follows = "";
+    inputs.git-hooks-nix.follows = "";
+    inputs.nixpkgs-regression.follows = "";
+    inputs.nixpkgs-23-11.follows = "";
+    inputs.flake-compat.follows = "";
+  };
 
-  # hide nix dev tooling from our lock file
-  inputs.nix.inputs.flake-parts.follows = "";
-  inputs.nix.inputs.git-hooks-nix.follows = "";
-  inputs.nix.inputs.nixpkgs-regression.follows = "";
-  inputs.nix.inputs.nixpkgs-23-11.follows = "";
-  inputs.nix.inputs.flake-compat.follows = "";
-
-  # hide nix-eval-jobs dev tooling from our lock file
-  inputs.nix-eval-jobs.inputs.nix-github-actions.follows = "";
+  inputs.nix-eval-jobs = {
+    url = "github:nix-community/nix-eval-jobs";
+    # We want to control the deps precisely
+    flake = false;
+  };
 
   outputs = { self, nixpkgs, nix, nix-eval-jobs, ... }:
     let
@@ -30,9 +30,9 @@
 
       # A Nixpkgs overlay that provides a 'hydra' package.
       overlays.default = final: prev: {
+        nix-eval-jobs = final.callPackage nix-eval-jobs {};
         hydra = final.callPackage ./package.nix {
           inherit (nixpkgs.lib) fileset;
-          nix-eval-jobs = nix-eval-jobs.packages.${final.system}.default;
           rawSrc = self;
           nix-perl-bindings = final.nixComponents.nix-perl-bindings;
         };
@@ -74,11 +74,19 @@
       });
 
       packages = forEachSystem (system: {
+        nix-eval-jobs = nixpkgs.legacyPackages.${system}.callPackage nix-eval-jobs {
+          nix = nix.packages.${system}.nix;
+        };
         hydra = nixpkgs.legacyPackages.${system}.callPackage ./package.nix {
           inherit (nixpkgs.lib) fileset;
-          nix-eval-jobs = nix-eval-jobs.packages.${system}.default;
+          inherit (self.packages.${system}) nix-eval-jobs;
           rawSrc = self;
-          nix = nix.packages.${system}.nix;
+          inherit (nix.packages.${system})
+            nix-util
+            nix-store
+            nix-main
+            nix-cli
+            ;
           nix-perl-bindings = nix.hydraJobs.perlBindings.${system};
         };
         default = self.packages.${system}.hydra;

meson.build

@@ -8,22 +8,22 @@ project('hydra', 'cpp',
   ],
 )
 
+nix_util_dep = dependency('nix-util', required: true)
 nix_store_dep = dependency('nix-store', required: true)
 nix_main_dep = dependency('nix-main', required: true)
-nix_expr_dep = dependency('nix-expr', required: true)
-nix_flake_dep = dependency('nix-flake', required: true)
-nix_cmd_dep = dependency('nix-cmd', required: true)
 
 # Nix need extra flags not provided in its pkg-config files.
 nix_dep = declare_dependency(
   dependencies: [
+    nix_util_dep,
     nix_store_dep,
     nix_main_dep,
-    nix_expr_dep,
-    nix_flake_dep,
-    nix_cmd_dep,
   ],
-  compile_args: ['-include', 'nix/config.h'],
+  compile_args: [
+    '-include', 'nix/config-util.hh',
+    '-include', 'nix/config-store.hh',
+    '-include', 'nix/config-main.hh',
+  ],
 )
 
 pqxx_dep = dependency('libpqxx', required: true)

nixos-modules/default.nix

@@ -15,7 +15,6 @@
     systemd.services.hydra-send-stats.enable = false;
 
     services.postgresql.enable = true;
-    services.postgresql.package = pkgs.postgresql_12;
 
     # The following is to work around the following error from hydra-server:
     #   [error] Caught exception in engine "Cannot determine local time zone"

nixos-modules/hydra.nix

@@ -468,7 +468,7 @@ in
             elif [[ $compression == zstd ]]; then
               compression="zstd --rm"
             fi
-            find ${baseDir}/build-logs -type f -name "*.drv" -mtime +3 -size +0c | xargs -r "$compression" --force --quiet
+            find ${baseDir}/build-logs -ignore_readdir_race -type f -name "*.drv" -mtime +3 -size +0c | xargs -r "$compression" --force --quiet
           '';
         startAt = "Sun 01:45";
       };

package.nix

@@ -8,7 +8,10 @@
 
 , perlPackages
 
-, nix
+, nix-util
+, nix-store
+, nix-main
+, nix-cli
 , nix-perl-bindings
 , git
 
@@ -162,7 +165,7 @@ stdenv.mkDerivation (finalAttrs: {
     nukeReferences
     pkg-config
     mdbook
-    nix
+    nix-cli
     perlDeps
     perl
     unzip
@@ -172,7 +175,9 @@ stdenv.mkDerivation (finalAttrs: {
     libpqxx
     openssl
     libxslt
-    nix
+    nix-util
+    nix-store
+    nix-main
     perlDeps
     perl
     boost
@@ -199,13 +204,14 @@ stdenv.mkDerivation (finalAttrs: {
     glibcLocales
     libressl.nc
     python3
+    nix-cli
   ];
 
   hydraPath = lib.makeBinPath (
     [
       subversion
       openssh
-      nix
+      nix-cli
       coreutils
       findutils
       pixz
@@ -235,7 +241,7 @@ stdenv.mkDerivation (finalAttrs: {
   shellHook = ''
     pushd $(git rev-parse --show-toplevel) >/dev/null
 
-    PATH=$(pwd)/src/hydra-evaluator:$(pwd)/src/script:$(pwd)/src/hydra-queue-runner:$PATH
+    PATH=$(pwd)/build/src/hydra-evaluator:$(pwd)/build/src/script:$(pwd)/build/src/hydra-queue-runner:$PATH
     PERL5LIB=$(pwd)/src/lib:$PERL5LIB
     export HYDRA_HOME="$(pwd)/src/"
     mkdir -p .hydra-data
@@ -266,7 +272,7 @@ stdenv.mkDerivation (finalAttrs: {
             --prefix PATH ':' $out/bin:$hydraPath \
             --set HYDRA_RELEASE ${version} \
             --set HYDRA_HOME $out/libexec/hydra \
-            --set NIX_RELEASE ${nix.name or "unknown"} \
+            --set NIX_RELEASE ${nix-cli.name or "unknown"} \
             --set NIX_EVAL_JOBS_RELEASE ${nix-eval-jobs.name or "unknown"}
     done
   '';
@@ -274,5 +280,5 @@ stdenv.mkDerivation (finalAttrs: {
   dontStrip = true;
 
   meta.description = "Build of Hydra on ${stdenv.system}";
-  passthru = { inherit perlDeps nix; };
+  passthru = { inherit perlDeps; };
 })

src/hydra-queue-runner/builder.cc

@@ -178,7 +178,7 @@ State::StepResult State::doBuildStep(nix::ref<Store> destStore,
                     unlink(result.logFile.c_str());
                 }
             } catch (...) {
-                ignoreException();
+                ignoreExceptionInDestructor();
             }
         }
     });

src/lib/Hydra/View/TT.pm

@@ -6,6 +6,7 @@ use base 'Catalyst::View::TT';
 use Template::Plugin::HTML;
 use Hydra::Helper::Nix;
 use Time::Seconds;
+use Digest::SHA qw(sha1_hex);
 
 __PACKAGE__->config(
     TEMPLATE_EXTENSION => '.tt',
@@ -25,8 +26,14 @@ __PACKAGE__->config(
     makeNameTextForJobset
     relativeDuration
     stripSSHUser
+    metricDivId
     /]);
 
+sub metricDivId {
+    my ($self, $c, $text) = @_;
+    return "metric-" . sha1_hex($text);
+}
+
 sub buildLogExists {
     my ($self, $c, $build) = @_;
     return 1 if defined $c->config->{log_prefix};

src/root/job-metrics-tab.tt

@@ -18,8 +18,7 @@
 
     <h3>Metric: <a [% HTML.attributes(href => c.uri_for('/job' project.name jobset.name job 'metric' metric.name)) %]><tt>[%HTML.escape(metric.name)%]</tt></a></h3>
 
-    [% id = "metric-" _ metric.name;
-       id = id.replace('\.', '_');
+    [% id = metricDivId(metric.name);
        INCLUDE createChart dataUrl=c.uri_for('/job' project.name jobset.name job 'metric' metric.name); %]
 
   [% END %]

src/script/hydra-eval-jobset

@@ -773,6 +773,9 @@ sub checkJobsetWrapped {
     my $jobsetChanged = 0;
     my %buildMap;
 
+    my @jobs;
+    push @jobs, $_ while defined($_ = $jobsIter->());
+
     $db->txn_do(sub {
         my $prevEval = getPrevJobsetEval($db, $jobset, 1);
 
@@ -796,7 +799,7 @@ sub checkJobsetWrapped {
 
         my @jobsWithConstituents;
 
-        while (defined(my $job = $jobsIter->())) {
+        foreach my $job (@jobs) {
             if ($jobsetsJobset) {
                 die "The .jobsets jobset must only have a single job named 'jobsets'"
                     unless $job->{attr} eq "jobsets";

t/evaluator/evaluate-constituents-broken.t

@@ -6,27 +6,55 @@ use Hydra::Helper::Exec;
 
 my $ctx = test_context();
 
-my $jobsetCtx = $ctx->makeJobset(
-    expression => 'constituents-broken.nix',
-);
-my $jobset = $jobsetCtx->{"jobset"};
+subtest "broken constituents expression" => sub {
+    my $jobsetCtx = $ctx->makeJobset(
+        expression => 'constituents-broken.nix',
+    );
+    my $jobset = $jobsetCtx->{"jobset"};
 
-my ($res, $stdout, $stderr) = captureStdoutStderr(60,
-    ("hydra-eval-jobset", $jobsetCtx->{"project"}->name, $jobset->name)
-);
-isnt($res, 0, "hydra-eval-jobset exits non-zero");
-ok(utf8::decode($stderr), "Stderr output is UTF8-clean");
-like(
-    $stderr,
-    qr/aggregate job ‘mixed_aggregate’ failed with the error: "constituentA": does not exist/,
-    "The stderr record includes a relevant error message"
-);
+    my ($res, $stdout, $stderr) = captureStdoutStderr(60,
+        ("hydra-eval-jobset", $jobsetCtx->{"project"}->name, $jobset->name)
+    );
+    isnt($res, 0, "hydra-eval-jobset exits non-zero");
+    ok(utf8::decode($stderr), "Stderr output is UTF8-clean");
+    like(
+        $stderr,
+        qr/aggregate job 'mixed_aggregate' references non-existent job 'constituentA'/,
+        "The stderr record includes a relevant error message"
+    );
 
-$jobset->discard_changes({ '+columns' => {'errormsg' => 'errormsg'} });  # refresh from DB
-like(
-    $jobset->errormsg,
-    qr/aggregate job ‘mixed_aggregate’ failed with the error: "constituentA": does not exist/,
-    "The jobset records a relevant error message"
-);
+    $jobset->discard_changes({ '+columns' => {'errormsg' => 'errormsg'} });  # refresh from DB
+    like(
+        $jobset->errormsg,
+        qr/aggregate job ‘mixed_aggregate’ failed with the error: constituentA: does not exist/,
+        "The jobset records a relevant error message"
+    );
+};
+
+subtest "no matches" => sub {
+    my $jobsetCtx = $ctx->makeJobset(
+        expression => 'constituents-no-matches.nix',
+    );
+    my $jobset = $jobsetCtx->{"jobset"};
+
+    my ($res, $stdout, $stderr) = captureStdoutStderr(60,
+        ("hydra-eval-jobset", $jobsetCtx->{"project"}->name, $jobset->name)
+    );
+    isnt($res, 0, "hydra-eval-jobset exits non-zero");
+    ok(utf8::decode($stderr), "Stderr output is UTF8-clean");
+    like(
+        $stderr,
+        qr/aggregate job 'non_match_aggregate' references constituent glob pattern 'tests\.\*' with no matches/,
+        "The stderr record includes a relevant error message"
+    );
+
+    $jobset->discard_changes({ '+columns' => {'errormsg' => 'errormsg'} });  # refresh from DB
+    like(
+        $jobset->errormsg,
+        qr/aggregate job ‘non_match_aggregate’ failed with the error: tests\.\*: constituent glob pattern had no matches/,
+        qr/in job ‘non_match_aggregate’:\ntests\.\*: constituent glob pattern had no matches/,
+        "The jobset records a relevant error message"
+    );
+};
 
 done_testing;

t/evaluator/evaluate-constituents-globbing.t

@@ -0,0 +1,138 @@
+use strict;
+use warnings;
+use Setup;
+use Test2::V0;
+use Hydra::Helper::Exec;
+use Data::Dumper;
+
+my $ctx = test_context();
+
+subtest "general glob testing" => sub {
+    my $jobsetCtx = $ctx->makeJobset(
+        expression => 'constituents-glob.nix',
+    );
+    my $jobset = $jobsetCtx->{"jobset"};
+
+    my ($res, $stdout, $stderr) = captureStdoutStderr(60,
+        ("hydra-eval-jobset", $jobsetCtx->{"project"}->name, $jobset->name)
+    );
+    is($res, 0, "hydra-eval-jobset exits zero");
+
+    my $builds = {};
+    for my $build ($jobset->builds) {
+        $builds->{$build->job} = $build;
+    }
+
+    subtest "basic globbing works" => sub {
+        ok(defined $builds->{"ok_aggregate"}, "'ok_aggregate' is part of the jobset evaluation");
+        my @constituents = $builds->{"ok_aggregate"}->constituents->all;
+        is(2, scalar @constituents, "'ok_aggregate' has two constituents");
+
+        my @sortedConstituentNames = sort (map { $_->nixname } @constituents);
+
+        is($sortedConstituentNames[0], "empty-dir-A", "first constituent of 'ok_aggregate' is 'empty-dir-A'");
+        is($sortedConstituentNames[1], "empty-dir-B", "second constituent of 'ok_aggregate' is 'empty-dir-B'");
+    };
+
+    subtest "transitivity is OK" => sub {
+        ok(defined $builds->{"indirect_aggregate"}, "'indirect_aggregate' is part of the jobset evaluation");
+        my @constituents = $builds->{"indirect_aggregate"}->constituents->all;
+        is(1, scalar @constituents, "'indirect_aggregate' has one constituent");
+        is($constituents[0]->nixname, "direct_aggregate", "'indirect_aggregate' has 'direct_aggregate' as single constituent");
+    };
+};
+
+subtest "* selects all except current aggregate" => sub {
+    my $jobsetCtx = $ctx->makeJobset(
+        expression => 'constituents-glob-all.nix',
+    );
+    my $jobset = $jobsetCtx->{"jobset"};
+
+    my ($res, $stdout, $stderr) = captureStdoutStderr(60,
+        ("hydra-eval-jobset", $jobsetCtx->{"project"}->name, $jobset->name)
+    );
+
+    subtest "no eval errors" => sub {
+        ok(utf8::decode($stderr), "Stderr output is UTF8-clean");
+        ok(
+            $stderr !~ "aggregate job ‘ok_aggregate’ has a constituent .* that doesn't correspond to a Hydra build",
+            "Catchall wildcard must not select itself as constituent"
+        );
+
+        $jobset->discard_changes;  # refresh from DB
+        is(
+            $jobset->errormsg,
+            "",
+            "eval-errors non-empty"
+        );
+    };
+
+    my $builds = {};
+    for my $build ($jobset->builds) {
+        $builds->{$build->job} = $build;
+    }
+
+    subtest "two constituents" => sub {
+        ok(defined $builds->{"ok_aggregate"}, "'ok_aggregate' is part of the jobset evaluation");
+        my @constituents = $builds->{"ok_aggregate"}->constituents->all;
+        is(2, scalar @constituents, "'ok_aggregate' has two constituents");
+
+        my @sortedConstituentNames = sort (map { $_->nixname } @constituents);
+
+        is($sortedConstituentNames[0], "empty-dir-A", "first constituent of 'ok_aggregate' is 'empty-dir-A'");
+        is($sortedConstituentNames[1], "empty-dir-B", "second constituent of 'ok_aggregate' is 'empty-dir-B'");
+    };
+};
+
+subtest "trivial cycle check" => sub {
+    my $jobsetCtx = $ctx->makeJobset(
+        expression => 'constituents-cycle.nix',
+    );
+    my $jobset = $jobsetCtx->{"jobset"};
+
+    my ($res, $stdout, $stderr) = captureStdoutStderr(60,
+        ("hydra-eval-jobset", $jobsetCtx->{"project"}->name, $jobset->name)
+    );
+
+    ok(
+        $stderr =~ "Found dependency cycle between jobs 'indirect_aggregate' and 'ok_aggregate'",
+        "Dependency cycle error is on stderr"
+    );
+
+    ok(utf8::decode($stderr), "Stderr output is UTF8-clean");
+
+    $jobset->discard_changes;  # refresh from DB
+    like(
+        $jobset->errormsg,
+        qr/Dependency cycle: indirect_aggregate <-> ok_aggregate/,
+        "eval-errors non-empty"
+    );
+
+    is(0, $jobset->builds->count, "No builds should be scheduled");
+};
+
+subtest "cycle check with globbing" => sub {
+    my $jobsetCtx = $ctx->makeJobset(
+        expression => 'constituents-cycle-glob.nix',
+    );
+    my $jobset = $jobsetCtx->{"jobset"};
+
+    my ($res, $stdout, $stderr) = captureStdoutStderr(60,
+        ("hydra-eval-jobset", $jobsetCtx->{"project"}->name, $jobset->name)
+    );
+
+    ok(utf8::decode($stderr), "Stderr output is UTF8-clean");
+
+    $jobset->discard_changes;  # refresh from DB
+    like(
+        $jobset->errormsg,
+        qr/aggregate job ‘indirect_aggregate’ failed with the error: Dependency cycle: indirect_aggregate <-> packages.constituentA/,
+        "packages.constituentA error missing"
+    );
+
+    # on this branch of Hydra, hydra-eval-jobset fails hard if an aggregate
+    # job is broken.
+    is(0, $jobset->builds->count, "Zero jobs are scheduled");
+};
+
+done_testing;

t/jobs/config.nix

@@ -0,0 +1,14 @@
+rec {
+  path = "/nix/store/l9mg93sgx50y88p5rr6x1vib6j1rjsds-coreutils-9.1/bin";
+
+  mkDerivation = args:
+    derivation ({
+      system = builtins.currentSystem;
+      PATH = path;
+    } // args);
+  mkContentAddressedDerivation = args: mkDerivation ({
+    __contentAddressed = true;
+    outputHashMode = "recursive";
+    outputHashAlgo = "sha256";
+  } // args);
+}

t/jobs/constituents-cycle-glob.nix

@@ -0,0 +1,34 @@
+with import ./config.nix;
+{
+  packages.constituentA = mkDerivation {
+    name = "empty-dir-A";
+    builder = ./empty-dir-builder.sh;
+    _hydraAggregate = true;
+    _hydraGlobConstituents = true;
+    constituents = [ "*_aggregate" ];
+  };
+
+  packages.constituentB = mkDerivation {
+    name = "empty-dir-B";
+    builder = ./empty-dir-builder.sh;
+  };
+
+  ok_aggregate = mkDerivation {
+    name = "direct_aggregate";
+    _hydraAggregate = true;
+    _hydraGlobConstituents = true;
+    constituents = [
+      "packages.*"
+    ];
+    builder = ./empty-dir-builder.sh;
+  };
+
+  indirect_aggregate = mkDerivation {
+    name = "indirect_aggregate";
+    _hydraAggregate = true;
+    constituents = [
+      "ok_aggregate"
+    ];
+    builder = ./empty-dir-builder.sh;
+  };
+}

t/jobs/constituents-cycle.nix

@@ -0,0 +1,21 @@
+with import ./config.nix;
+{
+  ok_aggregate = mkDerivation {
+    name = "direct_aggregate";
+    _hydraAggregate = true;
+    _hydraGlobConstituents = true;
+    constituents = [
+      "indirect_aggregate"
+    ];
+    builder = ./empty-dir-builder.sh;
+  };
+
+  indirect_aggregate = mkDerivation {
+    name = "indirect_aggregate";
+    _hydraAggregate = true;
+    constituents = [
+      "ok_aggregate"
+    ];
+    builder = ./empty-dir-builder.sh;
+  };
+}

t/jobs/constituents-glob-all.nix

@@ -0,0 +1,22 @@
+with import ./config.nix;
+{
+  packages.constituentA = mkDerivation {
+    name = "empty-dir-A";
+    builder = ./empty-dir-builder.sh;
+  };
+
+  packages.constituentB = mkDerivation {
+    name = "empty-dir-B";
+    builder = ./empty-dir-builder.sh;
+  };
+
+  ok_aggregate = mkDerivation {
+    name = "direct_aggregate";
+    _hydraAggregate = true;
+    _hydraGlobConstituents = true;
+    constituents = [
+      "*"
+    ];
+    builder = ./empty-dir-builder.sh;
+  };
+}

t/jobs/constituents-glob.nix

@@ -0,0 +1,31 @@
+with import ./config.nix;
+{
+  packages.constituentA = mkDerivation {
+    name = "empty-dir-A";
+    builder = ./empty-dir-builder.sh;
+  };
+
+  packages.constituentB = mkDerivation {
+    name = "empty-dir-B";
+    builder = ./empty-dir-builder.sh;
+  };
+
+  ok_aggregate = mkDerivation {
+    name = "direct_aggregate";
+    _hydraAggregate = true;
+    _hydraGlobConstituents = true;
+    constituents = [
+      "packages.*"
+    ];
+    builder = ./empty-dir-builder.sh;
+  };
+
+  indirect_aggregate = mkDerivation {
+    name = "indirect_aggregate";
+    _hydraAggregate = true;
+    constituents = [
+      "ok_aggregate"
+    ];
+    builder = ./empty-dir-builder.sh;
+  };
+}

t/jobs/constituents-no-matches.nix

@@ -0,0 +1,20 @@
+with import ./config.nix;
+{
+  non_match_aggregate = mkDerivation {
+    name = "mixed_aggregate";
+    _hydraAggregate = true;
+    _hydraGlobConstituents = true;
+    constituents = [
+      "tests.*"
+    ];
+    builder = ./empty-dir-builder.sh;
+  };
+
+  # Without a second job no jobset is attempted to be created
+  # (the only job would be broken)
+  # and thus the constituent validation is never reached.
+  dummy = mkDerivation {
+    name = "dummy";
+    builder = ./empty-dir-builder.sh;
+  };
+}

t/jobs/declarative/project.json

@@ -0,0 +1,24 @@
+{
+    "enabled": 1,
+    "hidden": false,
+    "description": "declarative-jobset-example",
+    "nixexprinput": "src",
+    "nixexprpath": "declarative/generator.nix",
+    "checkinterval": 300,
+    "schedulingshares": 100,
+    "enableemail": false,
+    "emailoverride": "",
+    "keepnr": 3,
+    "inputs": {
+        "src": {
+            "type": "path",
+            "value": "/home/ma27/Projects/hydra-cppnix/t/jobs",
+            "emailresponsible": false
+        },
+        "jobspath": {
+            "type": "string",
+            "value": "/home/ma27/Projects/hydra-cppnix/t/jobs",
+            "emailresponsible": false
+        }
+    }
+}

t/meson.build

@@ -27,6 +27,7 @@ testenv.prepend('PERL5LIB',
   separator: ':'
 )
 testenv.prepend('PATH',
+  fs.parent(find_program('nix').full_path()),
   fs.parent(hydra_evaluator.full_path()),
   fs.parent(hydra_queue_runner.full_path()),
   meson.project_source_root() / 'src/script',

t/queue-runner/constituents.t

@@ -22,11 +22,11 @@ is(nrQueuedBuildsForJobset($jobset), 0, "Evaluating jobs/broken-constituent.nix
 
 like(
     $jobset->errormsg,
-    qr/^"does-not-exist": does not exist$/m,
+    qr/^does-not-exist: does not exist$/m,
     "Evaluating jobs/broken-constituent.nix should log an error for does-not-exist");
 like(
     $jobset->errormsg,
-    qr/^"does-not-evaluate": "error: assertion 'false' failed/m,
+    qr/^does-not-evaluate: error: assertion 'false' failed/m,
     "Evaluating jobs/broken-constituent.nix should log an error for does-not-evaluate");
 
 done_testing;