From 4ea646130c5e637b883e9656ee5f6a71d1a728ab Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Wed, 8 Dec 2021 11:44:08 -0500
Subject: [PATCH 01/76] RunCommand: split out documentation, fixup the matcher
 syntax

---
 doc/manual/src/SUMMARY.md            |  1 +
 doc/manual/src/plugins/README.md     |  4 +++-
 doc/manual/src/plugins/RunCommand.md | 32 ++++++++++++++++++++++++++++
 3 files changed, 36 insertions(+), 1 deletion(-)
 create mode 100644 doc/manual/src/plugins/RunCommand.md

diff --git a/doc/manual/src/SUMMARY.md b/doc/manual/src/SUMMARY.md
index 80e73112..357e795a 100644
--- a/doc/manual/src/SUMMARY.md
+++ b/doc/manual/src/SUMMARY.md
@@ -7,6 +7,7 @@
 - [Hydra jobs](./jobs.md)
 - [Plugins](./plugins/README.md)
   - [Declarative Projects](./plugins/declarative-projects.md)
+  - [RunCommand](./plugins/RunCommand.md)
 - [Using the external API](api.md)
 - [Webhooks](webhooks.md)
 - [Monitoring Hydra](./monitoring/README.md)
diff --git a/doc/manual/src/plugins/README.md b/doc/manual/src/plugins/README.md
index b5486fdb..26ee2649 100644
--- a/doc/manual/src/plugins/README.md
+++ b/doc/manual/src/plugins/README.md
@@ -192,10 +192,12 @@ Writes InfluxDB events when a builds finished.
 - `influxdb.url`
 - `influxdb.db`
 
-## Run command
+## RunCommand
 
 Runs a shell command when the build is finished.
 
+See [The RunCommand Plugin](./RunCommand.md) for more information.
+
 ### Configuration options:
 
 - `runcommand.[].job`
diff --git a/doc/manual/src/plugins/RunCommand.md b/doc/manual/src/plugins/RunCommand.md
new file mode 100644
index 00000000..8b1818cc
--- /dev/null
+++ b/doc/manual/src/plugins/RunCommand.md
@@ -0,0 +1,32 @@
+## The RunCommand Plugin
+
+Hydra supports executing a program after certain builds finish.
+This behavior is disabled by default.
+
+Hydra executes these commands under the `hydra-notify` service.
+
+### Static Commands
+
+Configure specific commands to execute after the specified matching job finishes.
+
+#### Configuration
+
+- `runcommand.[].job`
+
+A matcher for jobs to match in the format `project:jobset:job`. Defaults to `*:*:*`.
+
+**Note:** This matcher format is not a regular expression.
+The `*` is a wildcard for that entire section, partial matches are not supported.
+
+- `runcommand.[].command`
+
+Command to run. Can use the `$HYDRA_JSON` environment variable to access information about the build.
+
+### Example
+
+```xml
+<runcommand>
+  job = myProject:*:*
+  command = cat $HYDRA_JSON > /tmp/hydra-output
+</runcommand>
+```

From 6ffc93c01a55c86b50d7a20dd5a474a9a5389be4 Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Wed, 8 Dec 2021 12:37:13 -0500
Subject: [PATCH 02/76] RunCommand: write documentation for dynamic commands

---
 doc/manual/src/plugins/RunCommand.md | 50 ++++++++++++++++++++++++++++
 1 file changed, 50 insertions(+)

diff --git a/doc/manual/src/plugins/RunCommand.md b/doc/manual/src/plugins/RunCommand.md
index 8b1818cc..b186be80 100644
--- a/doc/manual/src/plugins/RunCommand.md
+++ b/doc/manual/src/plugins/RunCommand.md
@@ -30,3 +30,53 @@ Command to run. Can use the `$HYDRA_JSON` environment variable to access informa
   command = cat $HYDRA_JSON > /tmp/hydra-output
 </runcommand>
 ```
+
+### Dynamic Commands
+
+Hydra can optionally run RunCommand hooks defined dynamically by the jobset.
+This must be turned on explicitly in the `hydra.conf` and per jobset.
+
+#### Behavior
+
+Hydra will execute any program defined under the `runCommandHook` attribute set. These jobs must have a single output named `out`, and that output must be an executable file located directly at `$out`.
+
+#### Security Properties
+
+Safely deploying dynamic commands requires careful design of your Hydra jobs. Allowing arbitrary users to define attributes in your top level attribute set will allow that user to execute code on your Hydra.
+
+If a jobset has dynamic commands enabled, you must ensure only trusted users can define top level attributes.
+
+
+#### Configuration
+
+- `dynamicruncommand.enable`
+
+Set to 1 to enable dynamic RunCommand program execution.
+
+#### Example
+
+In your Hydra configuration, specify:
+
+```xml
+<dynamicruncommand>
+  enable = 1
+</dynamicruncommand>
+```
+
+Then create a job named `runCommandHook.example` in your jobset:
+
+```
+{ pkgs, ... }: {
+    runCommandHook = {
+        recurseForDerivations = true;
+
+        example = pkgs.writeScript "run-me" ''
+          #!${pkgs.runtimeShell}
+
+          ${pkgs.jq}/bin/jq . "$HYDRA_JSON"
+        '';
+    };
+}
+```
+
+After the `runcommandHook.example` build finishes that script will execute.

From ea311a0eb4849cca49e328719e721aa33695e0b1 Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Wed, 8 Dec 2021 11:38:14 -0500
Subject: [PATCH 03/76] RunCommand: enable the plugin if dynamicruncommand is
 set

---
 src/lib/Hydra/Plugin/RunCommand.pm  |  24 +++++-
 t/Hydra/Plugin/RunCommand/matcher.t | 119 +++++++++++++++++++++++++++-
 2 files changed, 140 insertions(+), 3 deletions(-)

diff --git a/src/lib/Hydra/Plugin/RunCommand.pm b/src/lib/Hydra/Plugin/RunCommand.pm
index 401942c7..8404f280 100644
--- a/src/lib/Hydra/Plugin/RunCommand.pm
+++ b/src/lib/Hydra/Plugin/RunCommand.pm
@@ -12,7 +12,29 @@ use Try::Tiny;
 
 sub isEnabled {
     my ($self) = @_;
-    return defined $self->{config}->{runcommand};
+
+    return areStaticCommandsEnabled($self->{config}) || areDynamicCommandsEnabled($self->{config});
+}
+
+sub areStaticCommandsEnabled {
+    my ($config) = @_;
+
+    if (defined $config->{runcommand}) {
+        return 1;
+    }
+
+    return 0;
+}
+
+sub areDynamicCommandsEnabled {
+    my ($config) = @_;
+
+    if ((defined $config->{dynamicruncommand})
+        && $config->{dynamicruncommand}->{enable}) {
+        return 1;
+    }
+
+    return 0;
 }
 
 sub configSectionMatches {
diff --git a/t/Hydra/Plugin/RunCommand/matcher.t b/t/Hydra/Plugin/RunCommand/matcher.t
index bc40ba77..9797f7e1 100644
--- a/t/Hydra/Plugin/RunCommand/matcher.t
+++ b/t/Hydra/Plugin/RunCommand/matcher.t
@@ -7,13 +7,13 @@ use Hydra::Plugin::RunCommand;
 subtest "isEnabled" => sub {
     is(
         Hydra::Plugin::RunCommand::isEnabled({}),
-        "",
+        0,
         "Disabled by default."
     );
 
     is(
         Hydra::Plugin::RunCommand::isEnabled({ config => {}}),
-        "",
+        0,
         "Disabled by default."
     );
 
@@ -22,6 +22,121 @@ subtest "isEnabled" => sub {
         1,
         "Enabled if any runcommand blocks exist."
     );
+
+    is(
+        Hydra::Plugin::RunCommand::isEnabled({ config => { dynamicruncommand => {}}}),
+        0,
+        "Not enabled if an empty dynamicruncommand blocks exist."
+    );
+
+    is(
+        Hydra::Plugin::RunCommand::isEnabled({ config => { dynamicruncommand => { enable => 0 }}}),
+        0,
+        "Not enabled if a dynamicruncommand blocks exist without enable being set to 1."
+    );
+
+    is(
+        Hydra::Plugin::RunCommand::isEnabled({ config => { dynamicruncommand => { enable => 1 }}}),
+        1,
+        "Enabled if a dynamicruncommand blocks exist with enable being set to 1."
+    );
+
+    is(
+        Hydra::Plugin::RunCommand::isEnabled({ config => {
+            runcommand => {},
+            dynamicruncommand => { enable => 0 }
+        }}),
+        1,
+        "Enabled if a runcommand config block exists, even if a dynamicruncommand is explicitly disabled."
+    );
+};
+
+subtest "areStaticCommandsEnabled" => sub {
+    is(
+        Hydra::Plugin::RunCommand::areStaticCommandsEnabled({}),
+        0,
+        "Disabled by default."
+    );
+
+    is(
+        Hydra::Plugin::RunCommand::areStaticCommandsEnabled({}),
+        0,
+        "Disabled by default."
+    );
+
+    is(
+        Hydra::Plugin::RunCommand::areStaticCommandsEnabled({ runcommand => {}}),
+        1,
+        "Enabled if any runcommand blocks exist."
+    );
+
+    is(
+        Hydra::Plugin::RunCommand::areStaticCommandsEnabled({ dynamicruncommand => {}}),
+        0,
+        "Not enabled by dynamicruncommand blocks."
+    );
+
+    is(
+        Hydra::Plugin::RunCommand::areStaticCommandsEnabled({ dynamicruncommand => { enable => 0 }}),
+        0,
+        "Not enabled by dynamicruncommand blocks."
+    );
+
+    is(
+        Hydra::Plugin::RunCommand::areStaticCommandsEnabled({ dynamicruncommand => { enable => 1 }}),
+        0,
+        "Not enabled by dynamicruncommand blocks."
+    );
+
+    is(
+        Hydra::Plugin::RunCommand::areStaticCommandsEnabled({
+            runcommand => {},
+            dynamicruncommand => { enable => 0 }
+        }),
+        1,
+        "Enabled if a runcommand config block exists, even if a dynamicruncommand is explicitly disabled."
+    );
+};
+
+subtest "areDynamicCommandsEnabled" => sub {
+    is(
+        Hydra::Plugin::RunCommand::areDynamicCommandsEnabled({}),
+        0,
+        "Disabled by default."
+    );
+
+    is(
+        Hydra::Plugin::RunCommand::areDynamicCommandsEnabled({ runcommand => {}}),
+        0,
+        "Disabled even if any runcommand blocks exist."
+    );
+
+    is(
+        Hydra::Plugin::RunCommand::areDynamicCommandsEnabled({ dynamicruncommand => {}}),
+        0,
+        "Not enabled if an empty dynamicruncommand blocks exist."
+    );
+
+    is(
+        Hydra::Plugin::RunCommand::areDynamicCommandsEnabled({ dynamicruncommand => { enable => 0 }}),
+        0,
+        "Not enabled if a dynamicruncommand blocks exist without enable being set to 1."
+    );
+
+    is(
+        Hydra::Plugin::RunCommand::areDynamicCommandsEnabled({ dynamicruncommand => { enable => 1 }}),
+        1,
+        "Enabled if a dynamicruncommand blocks exist with enable being set to 1."
+    );
+
+    is(
+        Hydra::Plugin::RunCommand::areDynamicCommandsEnabled({
+            runcommand => {},
+            dynamicruncommand => { enable => 0 }
+        }),
+        0,
+        "Disabled if dynamicruncommand is explicitly disabled."
+    );
 };
 
 subtest "configSectionMatches" => sub {

From e56c49333f7d31654661f0f2f3b3d86a6ccd31cb Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Wed, 8 Dec 2021 16:03:43 -0500
Subject: [PATCH 04/76] RunCommand: Add a WIP execution of dynamic commands

This in-progress feature will run a dynamically generated set of
buildFinished hooks, which must be nested under the `runCommandHook.*`
attribute set. This implementation is not very good, with some to-dos:

1. Only run if the build succeeded
2. Verify the output is named $out and that it is an executable file
   (or a symlink to a file)
3. Require the jobset itself have a flag enabling the feature, since
   this feature can be a bit dangerous if various people of different
   trust levels can create the jobs.
---
 src/lib/Hydra/Plugin/RunCommand.pm  |  33 +++++++--
 t/Hydra/Plugin/RunCommand/fanout.t  | 108 ++++++++++++++++++++++++++++
 t/Hydra/Plugin/RunCommand/matcher.t |  40 -----------
 t/jobs/runcommand-dynamic.nix       |  27 +++++++
 4 files changed, 161 insertions(+), 47 deletions(-)
 create mode 100644 t/Hydra/Plugin/RunCommand/fanout.t
 create mode 100644 t/jobs/runcommand-dynamic.nix

diff --git a/src/lib/Hydra/Plugin/RunCommand.pm b/src/lib/Hydra/Plugin/RunCommand.pm
index 8404f280..92acc326 100644
--- a/src/lib/Hydra/Plugin/RunCommand.pm
+++ b/src/lib/Hydra/Plugin/RunCommand.pm
@@ -65,10 +65,11 @@ sub eventMatches {
 }
 
 sub fanoutToCommands {
-    my ($config, $event, $project, $jobset, $job) = @_;
+    my ($config, $event, $build) = @_;
 
     my @commands;
 
+    # Calculate all the statically defined commands to execute
     my $cfg = $config->{runcommand};
     my @config = defined $cfg ? ref $cfg eq "ARRAY" ? @$cfg : ($cfg) : ();
 
@@ -77,9 +78,10 @@ sub fanoutToCommands {
         next unless eventMatches($conf, $event);
         next unless configSectionMatches(
             $matcher,
-            $project,
-            $jobset,
-            $job);
+            $build->get_column('project'),
+            $build->get_column('jobset'),
+            $build->get_column('job')
+        );
 
         if (!defined($conf->{command})) {
             warn "<runcommand> section for '$matcher' lacks a 'command' option";
@@ -92,6 +94,25 @@ sub fanoutToCommands {
         })
     }
 
+    # Calculate all dynamically defined commands to execute
+    if (areDynamicCommandsEnabled($config)) {
+        # missing test cases:
+        #
+        # 1. is it enabled on the jobset?
+        # 2. what if the result is a directory?
+        # 3. what if the job doens't have an out?
+        # 4. what if the build failed?
+        my $job = $build->get_column('job');
+
+        if ($job =~ "^runCommandHook\.") {
+            my $out = $build->buildoutputs->find({name => "out"});
+            push(@commands, {
+                matcher => "DynamicRunCommand($job)",
+                command => $out->path
+            })
+        }
+    }
+
     return \@commands;
 }
 
@@ -160,9 +181,7 @@ sub buildFinished {
     my $commandsToRun = fanoutToCommands(
         $self->{config},
         $event,
-        $build->project->get_column('name'),
-        $build->jobset->get_column('name'),
-        $build->get_column('job')
+        $build
     );
 
     if (@$commandsToRun == 0) {
diff --git a/t/Hydra/Plugin/RunCommand/fanout.t b/t/Hydra/Plugin/RunCommand/fanout.t
new file mode 100644
index 00000000..d3a7b98a
--- /dev/null
+++ b/t/Hydra/Plugin/RunCommand/fanout.t
@@ -0,0 +1,108 @@
+use strict;
+use warnings;
+use Setup;
+
+my %ctx = test_init();
+
+use Test2::V0;
+use Hydra::Plugin::RunCommand;
+
+require Hydra::Schema;
+require Hydra::Model::DB;
+
+use Test2::V0;
+
+my $db = Hydra::Model::DB->new;
+hydra_setup($db);
+
+my $project = $db->resultset('Projects')->create({name => "tests", displayname => "", owner => "root"});
+
+my $jobset = createBaseJobset("basic", "runcommand-dynamic.nix", $ctx{jobsdir});
+
+ok(evalSucceeds($jobset), "Evaluating jobs/runcommand-dynamic.nix should exit with return code 0");
+is(nrQueuedBuildsForJobset($jobset), 1, "Evaluating jobs/runcommand-dynamic.nix should result in 1 build1");
+
+(my $build) = queuedBuildsForJobset($jobset);
+
+is($build->job, "runCommandHook.example", "The only job should be runCommandHook.example");
+ok(runBuild($build), "Build should exit with return code 0");
+my $newbuild = $db->resultset('Builds')->find($build->id);
+is($newbuild->finished, 1, "Build should be finished.");
+is($newbuild->buildstatus, 0, "Build should have buildstatus 0.");
+
+subtest "fanoutToCommands" => sub {
+    my $config = {
+        runcommand => [
+            {
+                job => "",
+                command => "foo"
+            },
+            {
+                job => "tests:*:*",
+                command => "bar"
+            },
+            {
+                job => "tests:basic:nomatch",
+                command => "baz"
+            }
+        ]
+    };
+
+    is(
+        Hydra::Plugin::RunCommand::fanoutToCommands(
+            $config,
+            "buildFinished",
+            $newbuild
+        ),
+        [
+            {
+                matcher => "",
+                command => "foo"
+            },
+            {
+                matcher => "tests:*:*",
+                command => "bar"
+            }
+        ],
+        "fanoutToCommands returns a command per matching job"
+    );
+};
+
+subtest "fanoutToCommandsWithDynamicRunCommandSupport" => sub {
+    like(
+        $build->buildoutputs->find({name => "out"})->path,
+        qr/my-build-product$/,
+        "The way we find the out path is reasonable"
+    );
+
+    my $config = {
+        dynamicruncommand => { enable => 1 },
+        runcommand => [
+            {
+                job => "tests:basic:*",
+                command => "baz"
+            }
+        ]
+    };
+
+    is(
+        Hydra::Plugin::RunCommand::fanoutToCommands(
+            $config,
+            "buildFinished",
+            $build
+        ),
+        [
+            {
+                matcher => "tests:basic:*",
+                command => "baz"
+            },
+            {
+                matcher => "DynamicRunCommand(runCommandHook.example)",
+                command => $build->buildoutputs->find({name => "out"})->path
+            }
+        ],
+        "fanoutToCommands returns a command per matching job"
+    );
+};
+
+done_testing;
diff --git a/t/Hydra/Plugin/RunCommand/matcher.t b/t/Hydra/Plugin/RunCommand/matcher.t
index 9797f7e1..ca74a84c 100644
--- a/t/Hydra/Plugin/RunCommand/matcher.t
+++ b/t/Hydra/Plugin/RunCommand/matcher.t
@@ -249,44 +249,4 @@ subtest "eventMatches" => sub {
     );
 };
 
-subtest "fanoutToCommands" => sub {
-    my $config = {
-        runcommand => [
-            {
-                job => "",
-                command => "foo"
-            },
-            {
-                job => "project:*:*",
-                command => "bar"
-            },
-            {
-                job => "project:jobset:nomatch",
-                command => "baz"
-            }
-        ]
-    };
-
-    is(
-        Hydra::Plugin::RunCommand::fanoutToCommands(
-            $config,
-            "buildFinished",
-            "project",
-            "jobset",
-            "job"
-        ),
-        [
-            {
-                matcher => "",
-                command => "foo"
-            },
-            {
-                matcher => "project:*:*",
-                command => "bar"
-            }
-        ],
-        "fanoutToCommands returns a command per matching job"
-    );
-};
-
 done_testing;
diff --git a/t/jobs/runcommand-dynamic.nix b/t/jobs/runcommand-dynamic.nix
new file mode 100644
index 00000000..cf231f8f
--- /dev/null
+++ b/t/jobs/runcommand-dynamic.nix
@@ -0,0 +1,27 @@
+with import ./config.nix;
+{
+  runCommandHook.example = mkDerivation
+    {
+      name = "my-build-product";
+      builder = "/bin/sh";
+      outputs = [ "out" "bin" ];
+      args = [
+        (
+          builtins.toFile "builder.sh" ''
+            #! /bin/sh
+
+            echo "$PATH"
+
+            mkdir $bin
+            echo "foo" > $bin/bar
+
+            metrics=$out/nix-support/hydra-metrics
+            mkdir -p "$(dirname "$metrics")"
+            echo "lineCoverage 18 %" >> "$metrics"
+            echo "maxResident 27 KiB" >> "$metrics"
+          ''
+        )
+      ];
+    };
+
+}

From e7f68045f445e4c32363d2e9f6cf9fa917fa67e1 Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Tue, 14 Dec 2021 16:31:19 -0500
Subject: [PATCH 05/76] DynamicRunCommand: pull out the function determining if
 a build is eligible for execution under dynamic run commands.

---
 src/lib/Hydra/Plugin/RunCommand.pm | 15 ++++++++++---
 t/Hydra/Plugin/RunCommand/fanout.t | 34 ++++++++++++++++++++++++++++++
 2 files changed, 46 insertions(+), 3 deletions(-)

diff --git a/src/lib/Hydra/Plugin/RunCommand.pm b/src/lib/Hydra/Plugin/RunCommand.pm
index 92acc326..8d3cf35a 100644
--- a/src/lib/Hydra/Plugin/RunCommand.pm
+++ b/src/lib/Hydra/Plugin/RunCommand.pm
@@ -37,6 +37,16 @@ sub areDynamicCommandsEnabled {
     return 0;
 }
 
+sub isBuildEligibleForDynamicRunCommand {
+    my ($build) = @_;
+
+    if ($build->get_column("job") =~ "^runCommandHook\..+") {
+        return 1;
+    }
+
+    return 0;
+}
+
 sub configSectionMatches {
     my ($name, $project, $jobset, $job) = @_;
 
@@ -102,9 +112,8 @@ sub fanoutToCommands {
         # 2. what if the result is a directory?
         # 3. what if the job doens't have an out?
         # 4. what if the build failed?
-        my $job = $build->get_column('job');
-
-        if ($job =~ "^runCommandHook\.") {
+        if (isBuildEligibleForDynamicRunCommand($build)) {
+            my $job = $build->get_column('job');
             my $out = $build->buildoutputs->find({name => "out"});
             push(@commands, {
                 matcher => "DynamicRunCommand($job)",
diff --git a/t/Hydra/Plugin/RunCommand/fanout.t b/t/Hydra/Plugin/RunCommand/fanout.t
index d3a7b98a..2edcb390 100644
--- a/t/Hydra/Plugin/RunCommand/fanout.t
+++ b/t/Hydra/Plugin/RunCommand/fanout.t
@@ -105,4 +105,38 @@ subtest "fanoutToCommandsWithDynamicRunCommandSupport" => sub {
     );
 };
 
+subtest "isBuildEligibleForDynamicRunCommand" => sub {
+    my $build = Hydra::Schema::Result::Builds->new({
+        "job" => "foo bar baz"
+    });
+
+    is(
+        Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($build),
+        0,
+        "The job name does not match"
+    );
+
+    $build->set_column("job", "runCommandHook");
+    is(
+        Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($build),
+        0,
+        "The job name does not match"
+    );
+
+    $build->set_column("job", "runCommandHook.");
+    is(
+        Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($build),
+        0,
+        "The job name does not match"
+    );
+
+    $build->set_column("job", "runCommandHook.a");
+    is(
+        Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($build),
+        1,
+        "The job name does match"
+    );
+};
+
+
 done_testing;

From c2be27e82b7d31ff585a5cb90b4d777d3b9c3bf2 Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Tue, 14 Dec 2021 21:39:13 -0500
Subject: [PATCH 06/76] fanout.t: switch to makeAndEvaluateJobset

---
 t/Hydra/Plugin/RunCommand/fanout.t | 39 ++++++++++--------------------
 1 file changed, 13 insertions(+), 26 deletions(-)

diff --git a/t/Hydra/Plugin/RunCommand/fanout.t b/t/Hydra/Plugin/RunCommand/fanout.t
index 2edcb390..d9f67f14 100644
--- a/t/Hydra/Plugin/RunCommand/fanout.t
+++ b/t/Hydra/Plugin/RunCommand/fanout.t
@@ -1,34 +1,21 @@
 use strict;
 use warnings;
 use Setup;
-
-my %ctx = test_init();
-
 use Test2::V0;
 use Hydra::Plugin::RunCommand;
 
-require Hydra::Schema;
-require Hydra::Model::DB;
+my $ctx = test_context();
 
-use Test2::V0;
+my $builds = $ctx->makeAndEvaluateJobset(
+    expression => "runcommand-dynamic.nix",
+    build => 1
+);
 
-my $db = Hydra::Model::DB->new;
-hydra_setup($db);
-
-my $project = $db->resultset('Projects')->create({name => "tests", displayname => "", owner => "root"});
-
-my $jobset = createBaseJobset("basic", "runcommand-dynamic.nix", $ctx{jobsdir});
-
-ok(evalSucceeds($jobset), "Evaluating jobs/runcommand-dynamic.nix should exit with return code 0");
-is(nrQueuedBuildsForJobset($jobset), 1, "Evaluating jobs/runcommand-dynamic.nix should result in 1 build1");
-
-(my $build) = queuedBuildsForJobset($jobset);
+my $build = $builds->{"runCommandHook.example"};
 
 is($build->job, "runCommandHook.example", "The only job should be runCommandHook.example");
-ok(runBuild($build), "Build should exit with return code 0");
-my $newbuild = $db->resultset('Builds')->find($build->id);
-is($newbuild->finished, 1, "Build should be finished.");
-is($newbuild->buildstatus, 0, "Build should have buildstatus 0.");
+is($build->finished, 1, "Build should be finished.");
+is($build->buildstatus, 0, "Build should have buildstatus 0.");
 
 subtest "fanoutToCommands" => sub {
     my $config = {
@@ -38,7 +25,7 @@ subtest "fanoutToCommands" => sub {
                 command => "foo"
             },
             {
-                job => "tests:*:*",
+                job => "*:*:*",
                 command => "bar"
             },
             {
@@ -52,7 +39,7 @@ subtest "fanoutToCommands" => sub {
         Hydra::Plugin::RunCommand::fanoutToCommands(
             $config,
             "buildFinished",
-            $newbuild
+            $build
         ),
         [
             {
@@ -60,7 +47,7 @@ subtest "fanoutToCommands" => sub {
                 command => "foo"
             },
             {
-                matcher => "tests:*:*",
+                matcher => "*:*:*",
                 command => "bar"
             }
         ],
@@ -79,7 +66,7 @@ subtest "fanoutToCommandsWithDynamicRunCommandSupport" => sub {
         dynamicruncommand => { enable => 1 },
         runcommand => [
             {
-                job => "tests:basic:*",
+                job => "*:*:*",
                 command => "baz"
             }
         ]
@@ -93,7 +80,7 @@ subtest "fanoutToCommandsWithDynamicRunCommandSupport" => sub {
         ),
         [
             {
-                matcher => "tests:basic:*",
+                matcher => "*:*:*",
                 command => "baz"
             },
             {

From 1a30a0c2f13ad9cc7a52480f918111f6278503cc Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Tue, 14 Dec 2021 22:07:15 -0500
Subject: [PATCH 07/76] Dynamic RunCommand: validate that the job's out exists,
 is a file (or points to a file) which is executable.

---
 src/lib/Hydra/Plugin/RunCommand.pm |  30 +++++-
 t/Hydra/Plugin/RunCommand/fanout.t |  88 +++++++++++------
 t/jobs/runcommand-dynamic.nix      | 145 ++++++++++++++++++++++++-----
 3 files changed, 213 insertions(+), 50 deletions(-)

diff --git a/src/lib/Hydra/Plugin/RunCommand.pm b/src/lib/Hydra/Plugin/RunCommand.pm
index 8d3cf35a..8998fc39 100644
--- a/src/lib/Hydra/Plugin/RunCommand.pm
+++ b/src/lib/Hydra/Plugin/RunCommand.pm
@@ -41,6 +41,32 @@ sub isBuildEligibleForDynamicRunCommand {
     my ($build) = @_;
 
     if ($build->get_column("job") =~ "^runCommandHook\..+") {
+        my $out = $build->buildoutputs->find({name => "out"});
+        if (!defined $out) {
+            warn "DynamicRunCommand hook on " . $build->job . " (" . $build->id . ") rejected: no output named 'out'.";
+            return 0;
+        }
+
+        my $path = $out->path;
+        if (-l $path) {
+            $path = readlink($path);
+        }
+
+        if (! -e $path) {
+            warn "DynamicRunCommand hook on " . $build->job . " (" . $build->id . ") rejected: The 'out' output doesn't exist locally. This is a bug.";
+            return 0;
+        }
+
+        if (! -x $path) {
+            warn "DynamicRunCommand hook on " . $build->job . " (" . $build->id . ") rejected: The 'out' output is not executable.";
+            return 0;
+        }
+
+        if (! -f $path) {
+            warn "DynamicRunCommand hook on " . $build->job . " (" . $build->id . ") rejected: The 'out' output is not a regular file or symlink.";
+            return 0;
+        }
+
         return 1;
     }
 
@@ -109,9 +135,7 @@ sub fanoutToCommands {
         # missing test cases:
         #
         # 1. is it enabled on the jobset?
-        # 2. what if the result is a directory?
-        # 3. what if the job doens't have an out?
-        # 4. what if the build failed?
+        # 2. what if the build failed?
         if (isBuildEligibleForDynamicRunCommand($build)) {
             my $job = $build->get_column('job');
             my $out = $build->buildoutputs->find({name => "out"});
diff --git a/t/Hydra/Plugin/RunCommand/fanout.t b/t/Hydra/Plugin/RunCommand/fanout.t
index d9f67f14..41236456 100644
--- a/t/Hydra/Plugin/RunCommand/fanout.t
+++ b/t/Hydra/Plugin/RunCommand/fanout.t
@@ -93,36 +93,72 @@ subtest "fanoutToCommandsWithDynamicRunCommandSupport" => sub {
 };
 
 subtest "isBuildEligibleForDynamicRunCommand" => sub {
-    my $build = Hydra::Schema::Result::Builds->new({
-        "job" => "foo bar baz"
-    });
+    subtest "Non-matches based on name alone ..." => sub {
+        my $build = $builds->{"foo-bar-baz"};
+        is(
+            Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($build),
+            0,
+            "The job name does not match"
+        );
 
-    is(
-        Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($build),
-        0,
-        "The job name does not match"
-    );
+        $build->set_column("job", "runCommandHook");
+        is(
+            Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($build),
+            0,
+            "The job name does not match"
+        );
 
-    $build->set_column("job", "runCommandHook");
-    is(
-        Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($build),
-        0,
-        "The job name does not match"
-    );
+        $build->set_column("job", "runCommandHook.");
+        is(
+            Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($build),
+            0,
+            "The job name does not match"
+        );
+    };
 
-    $build->set_column("job", "runCommandHook.");
-    is(
-        Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($build),
-        0,
-        "The job name does not match"
-    );
+    subtest "On outputs ..." => sub {
+        is(
+            Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($builds->{"runCommandHook.example"}),
+            1,
+            "out is an executable file"
+        );
 
-    $build->set_column("job", "runCommandHook.a");
-    is(
-        Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($build),
-        1,
-        "The job name does match"
-    );
+        is(
+            Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($builds->{"runCommandHook.symlink"}),
+            1,
+            "out is a symlink to an executable file"
+        );
+
+        is(
+            Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($builds->{"runCommandHook.no-out"}),
+            0,
+            "No output named out"
+        );
+
+        is(
+            Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($builds->{"runCommandHook.out-is-directory"}),
+            0,
+            "out is a directory"
+        );
+
+        is(
+            Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($builds->{"runCommandHook.out-is-not-executable-file"}),
+            0,
+            "out is a file which is not not executable"
+        );
+
+        is(
+            Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($builds->{"runCommandHook.symlink-non-executable"}),
+            0,
+            "out is a symlink to a non-executable file"
+        );
+
+        is(
+            Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($builds->{"runCommandHook.symlink-directory"}),
+            0,
+            "out is a symlink to a directory"
+        );
+    };
 };
 
 
diff --git a/t/jobs/runcommand-dynamic.nix b/t/jobs/runcommand-dynamic.nix
index cf231f8f..c0b005b7 100644
--- a/t/jobs/runcommand-dynamic.nix
+++ b/t/jobs/runcommand-dynamic.nix
@@ -1,27 +1,130 @@
 with import ./config.nix;
-{
-  runCommandHook.example = mkDerivation
-    {
-      name = "my-build-product";
-      builder = "/bin/sh";
-      outputs = [ "out" "bin" ];
-      args = [
-        (
-          builtins.toFile "builder.sh" ''
-            #! /bin/sh
+rec {
+  foo-bar-baz = mkDerivation {
+    name = "foo-bar-baz";
+    builder = "/bin/sh";
+    outputs = [ "out" ];
+    args = [
+      (
+        builtins.toFile "builder.sh" ''
+          #! /bin/sh
 
-            echo "$PATH"
+          touch $out
+        ''
+      )
+    ];
+  };
 
-            mkdir $bin
-            echo "foo" > $bin/bar
+  runCommandHook.example = mkDerivation {
+    name = "my-build-product";
+    builder = "/bin/sh";
+    outputs = [ "out" ];
+    args = [
+      (
+        builtins.toFile "builder.sh" ''
+          #! /bin/sh
 
-            metrics=$out/nix-support/hydra-metrics
-            mkdir -p "$(dirname "$metrics")"
-            echo "lineCoverage 18 %" >> "$metrics"
-            echo "maxResident 27 KiB" >> "$metrics"
-          ''
-        )
-      ];
-    };
+          touch $out
+          chmod +x $out
+          # ... dunno ...
+        ''
+      )
+    ];
+  };
+
+  runCommandHook.symlink = mkDerivation {
+    name = "symlink-out";
+    builder = "/bin/sh";
+    outputs = [ "out" ];
+    args = [
+      (
+        builtins.toFile "builder.sh" ''
+          #! /bin/sh
+
+          ln -s $1 $out
+        ''
+      )
+
+      runCommandHook.example
+    ];
+  };
+
+  runCommandHook.no-out = mkDerivation {
+    name = "no-out";
+    builder = "/bin/sh";
+    outputs = [ "bin" ];
+    args = [
+      (
+        builtins.toFile "builder.sh" ''
+          #! /bin/sh
+          mkdir $bin
+        ''
+      )
+    ];
+  };
+
+  runCommandHook.out-is-directory = mkDerivation {
+    name = "out-is-directory";
+    builder = "/bin/sh";
+    outputs = [ "out" ];
+    args = [
+      (
+        builtins.toFile "builder.sh" ''
+          #! /bin/sh
+
+          mkdir $out
+        ''
+      )
+    ];
+  };
+
+  runCommandHook.out-is-not-executable-file = mkDerivation {
+    name = "out-is-directory";
+    builder = "/bin/sh";
+    outputs = [ "out" ];
+    args = [
+      (
+        builtins.toFile "builder.sh" ''
+          #! /bin/sh
+
+          touch $out
+        ''
+      )
+    ];
+  };
+
+  runCommandHook.symlink-non-executable = mkDerivation {
+    name = "symlink-out";
+    builder = "/bin/sh";
+    outputs = [ "out" ];
+    args = [
+      (
+        builtins.toFile "builder.sh" ''
+          #! /bin/sh
+
+          ln -s $1 $out
+        ''
+      )
+
+      runCommandHook.out-is-not-executable-file
+    ];
+  };
+
+  runCommandHook.symlink-directory = mkDerivation {
+    name = "symlink-directory";
+    builder = "/bin/sh";
+    outputs = [ "out" ];
+    args = [
+      (
+        builtins.toFile "builder.sh" ''
+          #! /bin/sh
+
+          ln -s $1 $out
+        ''
+      )
+
+      runCommandHook.out-is-directory
+    ];
+  };
 
 }

From 216d8bee3532d3d5a2326aff3f7c8bd8cff6c007 Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Tue, 14 Dec 2021 22:10:02 -0500
Subject: [PATCH 08/76] DynamicRunCommand: don't run if the build failed

---
 src/lib/Hydra/Plugin/RunCommand.pm |  5 ++++-
 t/Hydra/Plugin/RunCommand/fanout.t |  8 ++++++++
 t/jobs/runcommand-dynamic.nix      | 18 ++++++++++++++++++
 3 files changed, 30 insertions(+), 1 deletion(-)

diff --git a/src/lib/Hydra/Plugin/RunCommand.pm b/src/lib/Hydra/Plugin/RunCommand.pm
index 8998fc39..2ab20274 100644
--- a/src/lib/Hydra/Plugin/RunCommand.pm
+++ b/src/lib/Hydra/Plugin/RunCommand.pm
@@ -40,6 +40,10 @@ sub areDynamicCommandsEnabled {
 sub isBuildEligibleForDynamicRunCommand {
     my ($build) = @_;
 
+    if ($build->get_column("buildstatus") != 0) {
+        return 0;
+    }
+
     if ($build->get_column("job") =~ "^runCommandHook\..+") {
         my $out = $build->buildoutputs->find({name => "out"});
         if (!defined $out) {
@@ -135,7 +139,6 @@ sub fanoutToCommands {
         # missing test cases:
         #
         # 1. is it enabled on the jobset?
-        # 2. what if the build failed?
         if (isBuildEligibleForDynamicRunCommand($build)) {
             my $job = $build->get_column('job');
             my $out = $build->buildoutputs->find({name => "out"});
diff --git a/t/Hydra/Plugin/RunCommand/fanout.t b/t/Hydra/Plugin/RunCommand/fanout.t
index 41236456..8d34e582 100644
--- a/t/Hydra/Plugin/RunCommand/fanout.t
+++ b/t/Hydra/Plugin/RunCommand/fanout.t
@@ -159,6 +159,14 @@ subtest "isBuildEligibleForDynamicRunCommand" => sub {
             "out is a symlink to a directory"
         );
     };
+
+    subtest "On build status ..." => sub {
+        is(
+            Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($builds->{"runCommandHook.failed"}),
+            0,
+            "Failed builds don't get run"
+        );
+    };
 };
 
 
diff --git a/t/jobs/runcommand-dynamic.nix b/t/jobs/runcommand-dynamic.nix
index c0b005b7..1971bb82 100644
--- a/t/jobs/runcommand-dynamic.nix
+++ b/t/jobs/runcommand-dynamic.nix
@@ -127,4 +127,22 @@ rec {
     ];
   };
 
+  runCommandHook.failed = mkDerivation {
+    name = "failed";
+    builder = "/bin/sh";
+    outputs = [ "out" ];
+    args = [
+      (
+        builtins.toFile "builder.sh" ''
+          #! /bin/sh
+
+          touch $out
+          chmod +x $out
+
+          exit 1
+        ''
+      )
+    ];
+  };
+
 }

From 97a1d2d1d488887c2f9a65a22197e0f1c46be58e Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Tue, 14 Dec 2021 22:12:03 -0500
Subject: [PATCH 09/76] Jobsets: add enable_dynamic_run_command

---
 src/lib/Hydra/Schema/Result/Jobsets.pm | 12 ++++++++++--
 src/sql/hydra.sql                      |  1 +
 2 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/src/lib/Hydra/Schema/Result/Jobsets.pm b/src/lib/Hydra/Schema/Result/Jobsets.pm
index 13ac09e4..bd4b7165 100644
--- a/src/lib/Hydra/Schema/Result/Jobsets.pm
+++ b/src/lib/Hydra/Schema/Result/Jobsets.pm
@@ -155,6 +155,12 @@ __PACKAGE__->table("jobsets");
   data_type: 'text'
   is_nullable: 1
 
+=head2 enable_dynamic_run_command
+
+  data_type: 'boolean'
+  default_value: false
+  is_nullable: 0
+
 =cut
 
 __PACKAGE__->add_columns(
@@ -207,6 +213,8 @@ __PACKAGE__->add_columns(
   { data_type => "integer", default_value => 0, is_nullable => 0 },
   "flake",
   { data_type => "text", is_nullable => 1 },
+  "enable_dynamic_run_command",
+  { data_type => "boolean", default_value => \"false", is_nullable => 0 },
 );
 
 =head1 PRIMARY KEY
@@ -354,8 +362,8 @@ __PACKAGE__->has_many(
 );
 
 
-# Created by DBIx::Class::Schema::Loader v0.07049 @ 2022-01-08 22:24:10
-# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:cQOnMitrWGMoJX6kZGNW+w
+# Created by DBIx::Class::Schema::Loader v0.07049 @ 2022-01-24 14:17:33
+# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:7wPE5ebeVTkenMCWG9Sgcg
 
 use JSON::MaybeXS;
 
diff --git a/src/sql/hydra.sql b/src/sql/hydra.sql
index 26617789..73802d75 100644
--- a/src/sql/hydra.sql
+++ b/src/sql/hydra.sql
@@ -88,6 +88,7 @@ create table Jobsets (
     startTime     integer, -- if jobset is currently running
     type          integer not null default 0, -- 0 == legacy, 1 == flake
     flake         text,
+    enable_dynamic_run_command boolean not null default false,
     constraint jobsets_schedulingshares_nonzero_check check (schedulingShares > 0),
     constraint jobsets_type_known_check   check (type = 0 or type = 1),
     -- If the type is 0, then nixExprInput and nixExprPath should be non-null and other type-specific fields should be null

From 3cce0c5ef6a928c7a7f7091c27691a25482dccc1 Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Tue, 14 Dec 2021 22:15:50 -0500
Subject: [PATCH 10/76] Only run dynamic runcommand hooks if the jobset enables
 them

---
 src/lib/Hydra/Plugin/RunCommand.pm |  7 +++----
 t/Hydra/Plugin/RunCommand/fanout.t | 13 +++++++++++++
 2 files changed, 16 insertions(+), 4 deletions(-)

diff --git a/src/lib/Hydra/Plugin/RunCommand.pm b/src/lib/Hydra/Plugin/RunCommand.pm
index 2ab20274..79bf72a8 100644
--- a/src/lib/Hydra/Plugin/RunCommand.pm
+++ b/src/lib/Hydra/Plugin/RunCommand.pm
@@ -71,7 +71,9 @@ sub isBuildEligibleForDynamicRunCommand {
             return 0;
         }
 
-        return 1;
+        if ($build->jobset->enable_dynamic_run_command) {
+            return 1;
+        }
     }
 
     return 0;
@@ -136,9 +138,6 @@ sub fanoutToCommands {
 
     # Calculate all dynamically defined commands to execute
     if (areDynamicCommandsEnabled($config)) {
-        # missing test cases:
-        #
-        # 1. is it enabled on the jobset?
         if (isBuildEligibleForDynamicRunCommand($build)) {
             my $job = $build->get_column('job');
             my $out = $build->buildoutputs->find({name => "out"});
diff --git a/t/Hydra/Plugin/RunCommand/fanout.t b/t/Hydra/Plugin/RunCommand/fanout.t
index 8d34e582..72d58b3c 100644
--- a/t/Hydra/Plugin/RunCommand/fanout.t
+++ b/t/Hydra/Plugin/RunCommand/fanout.t
@@ -13,6 +13,9 @@ my $builds = $ctx->makeAndEvaluateJobset(
 
 my $build = $builds->{"runCommandHook.example"};
 
+# Enable dynamic runcommand on the jobset
+$build->jobset->update({enable_dynamic_run_command => 1});
+
 is($build->job, "runCommandHook.example", "The only job should be runCommandHook.example");
 is($build->finished, 1, "Build should be finished.");
 is($build->buildstatus, 0, "Build should have buildstatus 0.");
@@ -167,6 +170,16 @@ subtest "isBuildEligibleForDynamicRunCommand" => sub {
             "Failed builds don't get run"
         );
     };
+
+    subtest "With dynamic runcommand disabled ..." => sub {
+        $build->jobset->update({enable_dynamic_run_command => 0});
+
+        is(
+            Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($builds->{"runCommandHook.example"}),
+            0,
+            "Builds don't run from a jobset with disabled dynamic runcommand"
+        );
+    };
 };
 
 

From a9bfabd6722bdfe8a1cd69d045a0bfabb0284e1d Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Wed, 15 Dec 2021 11:16:05 -0500
Subject: [PATCH 11/76] sql: add a migration for enable_dynamic_run_command

---
 src/sql/upgrade-82.sql | 2 ++
 1 file changed, 2 insertions(+)
 create mode 100644 src/sql/upgrade-82.sql

diff --git a/src/sql/upgrade-82.sql b/src/sql/upgrade-82.sql
new file mode 100644
index 00000000..eb012762
--- /dev/null
+++ b/src/sql/upgrade-82.sql
@@ -0,0 +1,2 @@
+ALTER TABLE Jobsets
+    ADD COLUMN enable_dynamic_run_command boolean not null default false;

From 85a53694c8a87544dc2192a1f8900dadca499a6f Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Wed, 15 Dec 2021 12:32:10 -0500
Subject: [PATCH 12/76] sql: add enable_dynamic_run_command to the Project as
 well

---
 src/sql/hydra.sql      | 1 +
 src/sql/upgrade-82.sql | 2 ++
 2 files changed, 3 insertions(+)

diff --git a/src/sql/hydra.sql b/src/sql/hydra.sql
index 73802d75..eaae6da3 100644
--- a/src/sql/hydra.sql
+++ b/src/sql/hydra.sql
@@ -49,6 +49,7 @@ create table Projects (
     declfile      text, -- File containing declarative jobset specification
     decltype      text, -- Type of the input containing declarative jobset specification
     declvalue     text, -- Value of the input containing declarative jobset specification
+    enable_dynamic_run_command boolean not null default false,
     foreign key   (owner) references Users(userName) on update cascade
 );
 
diff --git a/src/sql/upgrade-82.sql b/src/sql/upgrade-82.sql
index eb012762..a619caf3 100644
--- a/src/sql/upgrade-82.sql
+++ b/src/sql/upgrade-82.sql
@@ -1,2 +1,4 @@
 ALTER TABLE Jobsets
     ADD COLUMN enable_dynamic_run_command boolean not null default false;
+ALTER TABLE Projects
+    ADD COLUMN enable_dynamic_run_command boolean not null default false;

From 0c96172c2890d0043a6ba41bdbc8d4765c2d1ca4 Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Wed, 15 Dec 2021 12:33:16 -0500
Subject: [PATCH 13/76] RunCommand: only run dynamic runcommand hooks if the
 project AND jobset agree they should be enabled

---
 t/Hydra/Plugin/RunCommand/fanout.t | 40 ++++++++++++++++++++++++------
 1 file changed, 33 insertions(+), 7 deletions(-)

diff --git a/t/Hydra/Plugin/RunCommand/fanout.t b/t/Hydra/Plugin/RunCommand/fanout.t
index 72d58b3c..90bf4a6f 100644
--- a/t/Hydra/Plugin/RunCommand/fanout.t
+++ b/t/Hydra/Plugin/RunCommand/fanout.t
@@ -13,7 +13,8 @@ my $builds = $ctx->makeAndEvaluateJobset(
 
 my $build = $builds->{"runCommandHook.example"};
 
-# Enable dynamic runcommand on the jobset
+# Enable dynamic runcommand on the project and jobset
+$build->project->update({enable_dynamic_run_command => 1});
 $build->jobset->update({enable_dynamic_run_command => 1});
 
 is($build->job, "runCommandHook.example", "The only job should be runCommandHook.example");
@@ -172,13 +173,38 @@ subtest "isBuildEligibleForDynamicRunCommand" => sub {
     };
 
     subtest "With dynamic runcommand disabled ..." => sub {
-        $build->jobset->update({enable_dynamic_run_command => 0});
+        subtest "disabled on the project, enabled on the jobset" => {
+            $build->project->update({enable_dynamic_run_command => 0});
+            $build->jobset->update({enable_dynamic_run_command => 1});
 
-        is(
-            Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($builds->{"runCommandHook.example"}),
-            0,
-            "Builds don't run from a jobset with disabled dynamic runcommand"
-        );
+            is(
+                Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($builds->{"runCommandHook.example"}),
+                0,
+                "Builds don't run from a jobset with disabled dynamic runcommand"
+            );
+        };
+
+        subtest "enabled on the project, disabled on the jobset" => {
+            $build->project->update({enable_dynamic_run_command => 1});
+            $build->jobset->update({enable_dynamic_run_command => 0});
+
+            is(
+                Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($builds->{"runCommandHook.example"}),
+                0,
+                "Builds don't run from a jobset with disabled dynamic runcommand"
+            );
+        };
+
+        subtest "disabled on the project, disabled on the jobset" => {
+            $build->project->update({enable_dynamic_run_command => 0});
+            $build->jobset->update({enable_dynamic_run_command => 0});
+
+            is(
+                Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($builds->{"runCommandHook.example"}),
+                0,
+                "Builds don't run from a jobset with disabled dynamic runcommand"
+            );
+        };
     };
 };
 

From aef11685a0c75b8e332b1c3d515e3bbb46206888 Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Wed, 15 Dec 2021 12:34:29 -0500
Subject: [PATCH 14/76] regenerate schema files after adding the flag to the
 projects

---
 src/lib/Hydra/Schema/Result/Projects.pm | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/src/lib/Hydra/Schema/Result/Projects.pm b/src/lib/Hydra/Schema/Result/Projects.pm
index 35c3eeab..9e630b16 100644
--- a/src/lib/Hydra/Schema/Result/Projects.pm
+++ b/src/lib/Hydra/Schema/Result/Projects.pm
@@ -88,6 +88,12 @@ __PACKAGE__->table("projects");
   data_type: 'text'
   is_nullable: 1
 
+=head2 enable_dynamic_run_command
+
+  data_type: 'boolean'
+  default_value: false
+  is_nullable: 0
+
 =cut
 
 __PACKAGE__->add_columns(
@@ -111,6 +117,8 @@ __PACKAGE__->add_columns(
   { data_type => "text", is_nullable => 1 },
   "declvalue",
   { data_type => "text", is_nullable => 1 },
+  "enable_dynamic_run_command",
+  { data_type => "boolean", default_value => \"false", is_nullable => 0 },
 );
 
 =head1 PRIMARY KEY
@@ -228,8 +236,8 @@ Composing rels: L</projectmembers> -> username
 __PACKAGE__->many_to_many("usernames", "projectmembers", "username");
 
 
-# Created by DBIx::Class::Schema::Loader v0.07049 @ 2022-01-08 22:24:10
-# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:r/wbX3FAm5/OFrrwOQL5fA
+# Created by DBIx::Class::Schema::Loader v0.07049 @ 2022-01-24 14:20:32
+# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:PtXDyT8Pc7LYhhdEG39EKQ
 
 use JSON::MaybeXS;
 

From 0810f5debcaa4372f78b225453b578ec09db097b Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Wed, 15 Dec 2021 12:36:19 -0500
Subject: [PATCH 15/76] finish making the dynamic hooks only run on project &
 jobset agreement

---
 src/lib/Hydra/Plugin/RunCommand.pm | 10 ++++++++--
 t/Hydra/Plugin/RunCommand/fanout.t |  6 +++---
 2 files changed, 11 insertions(+), 5 deletions(-)

diff --git a/src/lib/Hydra/Plugin/RunCommand.pm b/src/lib/Hydra/Plugin/RunCommand.pm
index 79bf72a8..6d099142 100644
--- a/src/lib/Hydra/Plugin/RunCommand.pm
+++ b/src/lib/Hydra/Plugin/RunCommand.pm
@@ -71,9 +71,15 @@ sub isBuildEligibleForDynamicRunCommand {
             return 0;
         }
 
-        if ($build->jobset->enable_dynamic_run_command) {
-            return 1;
+        if (! $build->jobset->enable_dynamic_run_command) {
+            return 0;
         }
+
+        if (! $build->project->enable_dynamic_run_command) {
+            return 0;
+        }
+
+        return 1;
     }
 
     return 0;
diff --git a/t/Hydra/Plugin/RunCommand/fanout.t b/t/Hydra/Plugin/RunCommand/fanout.t
index 90bf4a6f..bd2502ec 100644
--- a/t/Hydra/Plugin/RunCommand/fanout.t
+++ b/t/Hydra/Plugin/RunCommand/fanout.t
@@ -173,7 +173,7 @@ subtest "isBuildEligibleForDynamicRunCommand" => sub {
     };
 
     subtest "With dynamic runcommand disabled ..." => sub {
-        subtest "disabled on the project, enabled on the jobset" => {
+        subtest "disabled on the project, enabled on the jobset" => sub {
             $build->project->update({enable_dynamic_run_command => 0});
             $build->jobset->update({enable_dynamic_run_command => 1});
 
@@ -184,7 +184,7 @@ subtest "isBuildEligibleForDynamicRunCommand" => sub {
             );
         };
 
-        subtest "enabled on the project, disabled on the jobset" => {
+        subtest "enabled on the project, disabled on the jobset" => sub {
             $build->project->update({enable_dynamic_run_command => 1});
             $build->jobset->update({enable_dynamic_run_command => 0});
 
@@ -195,7 +195,7 @@ subtest "isBuildEligibleForDynamicRunCommand" => sub {
             );
         };
 
-        subtest "disabled on the project, disabled on the jobset" => {
+        subtest "disabled on the project, disabled on the jobset" => sub {
             $build->project->update({enable_dynamic_run_command => 0});
             $build->jobset->update({enable_dynamic_run_command => 0});
 

From 1802bd011338f227b2100b5947431d9b52c4b4fa Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Wed, 15 Dec 2021 12:37:01 -0500
Subject: [PATCH 16/76] Declarative Jobs: add support for the
 enable_dynamic_run_command flag

---
 doc/manual/src/plugins/declarative-projects.md | 3 +++
 src/lib/Hydra/Helper/AddBuilds.pm              | 1 +
 2 files changed, 4 insertions(+)

diff --git a/doc/manual/src/plugins/declarative-projects.md b/doc/manual/src/plugins/declarative-projects.md
index 12dfed18..b72c6fd0 100644
--- a/doc/manual/src/plugins/declarative-projects.md
+++ b/doc/manual/src/plugins/declarative-projects.md
@@ -34,6 +34,7 @@ To configure a static declarative project, take the following steps:
         "checkinterval": 300,
         "schedulingshares": 100,
         "enableemail": false,
+        "enable_dynamic_run_command": false,
         "emailoverride": "",
         "keepnr": 3,
         "inputs": {
@@ -53,6 +54,7 @@ To configure a static declarative project, take the following steps:
         "checkinterval": 300,
         "schedulingshares": 100,
         "enableemail": false,
+        "enable_dynamic_run_command": false,
         "emailoverride": "",
         "keepnr": 3,
         "inputs": {
@@ -92,6 +94,7 @@ containing the configuration of the jobset, for example:
         "checkinterval": 300,
         "schedulingshares": 100,
         "enableemail": false,
+        "enable_dynamic_run_command": false,
         "emailoverride": "",
         "keepnr": 3,
         "inputs": {
diff --git a/src/lib/Hydra/Helper/AddBuilds.pm b/src/lib/Hydra/Helper/AddBuilds.pm
index 1e6d8944..f38737d3 100644
--- a/src/lib/Hydra/Helper/AddBuilds.pm
+++ b/src/lib/Hydra/Helper/AddBuilds.pm
@@ -39,6 +39,7 @@ sub updateDeclarativeJobset {
         checkinterval
         schedulingshares
         enableemail
+        enable_dynamic_run_command
         emailoverride
         keepnr
     );

From 726ea80e991aa540cf5ed5f1b66ca8a5c68efa00 Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Wed, 15 Dec 2021 12:37:35 -0500
Subject: [PATCH 17/76] HTTP/Jobset: support setting / reading
 enable_dynamic_run_command

---
 hydra-api.yaml                     | 3 +++
 src/lib/Hydra/Controller/Jobset.pm | 1 +
 src/root/edit-jobset.tt            | 7 +++++++
 src/root/jobset.tt                 | 4 ++++
 t/Hydra/Controller/Jobset/http.t   | 1 +
 5 files changed, 16 insertions(+)

diff --git a/hydra-api.yaml b/hydra-api.yaml
index 7857162e..0d203a41 100644
--- a/hydra-api.yaml
+++ b/hydra-api.yaml
@@ -689,6 +689,9 @@ components:
         enableemail:
           description: when true the jobset sends emails when previously-successful builds fail
           type: boolean
+        enable_dynamic_run_command:
+          description: when true the jobset supports executing dynamically defined RunCommand hooks. Requires the server and project's configuration to also enable dynamic RunCommand.
+          type: boolean
         visible:
           description: when true the jobset is visible in the web frontend
           type: boolean
diff --git a/src/lib/Hydra/Controller/Jobset.pm b/src/lib/Hydra/Controller/Jobset.pm
index b952031f..a2d48597 100644
--- a/src/lib/Hydra/Controller/Jobset.pm
+++ b/src/lib/Hydra/Controller/Jobset.pm
@@ -268,6 +268,7 @@ sub updateJobset {
         , nixexprinput => $nixExprInput
         , enabled => $enabled
         , enableemail => defined $c->stash->{params}->{enableemail} ? 1 : 0
+        , enable_dynamic_run_command => defined $c->stash->{params}->{enable_dynamic_run_command} ? 1 : 0
         , emailoverride => trim($c->stash->{params}->{emailoverride}) || ""
         , hidden => defined $c->stash->{params}->{visible} ? 0 : 1
         , keepnr => int(trim($c->stash->{params}->{keepnr} // "0"))
diff --git a/src/root/edit-jobset.tt b/src/root/edit-jobset.tt
index dbd26dcc..40da8f61 100644
--- a/src/root/edit-jobset.tt
+++ b/src/root/edit-jobset.tt
@@ -157,6 +157,13 @@
     </div>
   </div>
 
+  <div class="form-group row">
+    <label class="col-sm-3" for="editjobsetenable_dynamic_run_command">Enable Dynamic RunCommand Hooks</label>
+    <div class="col-sm-9">
+      <input type="checkbox" id="editjobsetenable_dynamic_run_command" name="enable_dynamic_run_command" [% IF jobset.enable_dynamic_run_command %]checked[% END %]/>
+    </div>
+  </div>
+
   <div class="form-group row">
     <label class="col-sm-3" for="editjobsetenableemail">Email notification</label>
     <div class="col-sm-9">
diff --git a/src/root/jobset.tt b/src/root/jobset.tt
index 4fb52517..3d6ca6ae 100644
--- a/src/root/jobset.tt
+++ b/src/root/jobset.tt
@@ -160,6 +160,10 @@
         <th>Scheduling shares:</th>
         <td>[% jobset.schedulingshares %] [% IF totalShares %] ([% f = format("%.2f"); f(jobset.schedulingshares / totalShares * 100) %]% out of [% totalShares %] shares)[% END %]</td>
       </tr>
+      <tr>
+        <th>Enable Dynamic RunCommand Hooks:</th>
+        <td>[% jobset.enable_dynamic_run_command ? "Yes" : "No" %]</td>
+      </tr>
       [% IF emailNotification %]
       <tr>
         <th>Enable email notification:</th>
diff --git a/t/Hydra/Controller/Jobset/http.t b/t/Hydra/Controller/Jobset/http.t
index 32b3a681..4bca7c15 100644
--- a/t/Hydra/Controller/Jobset/http.t
+++ b/t/Hydra/Controller/Jobset/http.t
@@ -73,6 +73,7 @@ subtest 'Read newly-created jobset "job"' => sub {
      emailoverride => "",
      enabled => 2,
      enableemail => JSON::MaybeXS::false,
+     enable_dynamic_run_command => JSON::MaybeXS::false,
      errortime => undef,
      errormsg => "",
      fetcherrormsg => "",

From 1affb1cfb198cb07b8f161d03a9696e308eddd90 Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Wed, 15 Dec 2021 13:55:54 -0500
Subject: [PATCH 18/76] jobset API: expose and check the
 enable_dynamic_run_command

---
 src/lib/Hydra/Schema/Result/Jobsets.pm | 1 +
 t/Hydra/Controller/Jobset/http.t       | 1 +
 2 files changed, 2 insertions(+)

diff --git a/src/lib/Hydra/Schema/Result/Jobsets.pm b/src/lib/Hydra/Schema/Result/Jobsets.pm
index bd4b7165..7b96c472 100644
--- a/src/lib/Hydra/Schema/Result/Jobsets.pm
+++ b/src/lib/Hydra/Schema/Result/Jobsets.pm
@@ -414,6 +414,7 @@ sub as_json {
 
         # boolean_columns
         "enableemail" => $self->get_column("enableemail") ? JSON::MaybeXS::true : JSON::MaybeXS::false,
+        "enable_dynamic_run_command" => $self->get_column("enable_dynamic_run_command") ? JSON::MaybeXS::true : JSON::MaybeXS::false,
         "visible" => $self->get_column("hidden") ? JSON::MaybeXS::false : JSON::MaybeXS::true,
 
         "inputs" => { map { $_->name => $_ } $self->jobsetinputs }
diff --git a/t/Hydra/Controller/Jobset/http.t b/t/Hydra/Controller/Jobset/http.t
index 4bca7c15..4e53949d 100644
--- a/t/Hydra/Controller/Jobset/http.t
+++ b/t/Hydra/Controller/Jobset/http.t
@@ -132,6 +132,7 @@ subtest 'Update jobset "job" to legacy type' => sub {
     emailoverride => "",
     enabled => 3,
     enableemail => JSON::MaybeXS::false,
+    enable_dynamic_run_command => JSON::MaybeXS::false,
     errortime => undef,
     errormsg => "",
     fetcherrormsg => "",

From 8a96f07f58bfd4c9e7da9c573718473e829bf104 Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Wed, 15 Dec 2021 15:32:49 -0500
Subject: [PATCH 19/76] Project: enable enabling dynamic runcommand per project

---
 src/lib/Hydra/Controller/Project.pm     | 1 +
 src/lib/Hydra/Schema/Result/Projects.pm | 1 +
 src/root/edit-project.tt                | 8 ++++++++
 src/root/project.tt                     | 4 ++++
 4 files changed, 14 insertions(+)

diff --git a/src/lib/Hydra/Controller/Project.pm b/src/lib/Hydra/Controller/Project.pm
index ed3c527c..98a8a6eb 100644
--- a/src/lib/Hydra/Controller/Project.pm
+++ b/src/lib/Hydra/Controller/Project.pm
@@ -157,6 +157,7 @@ sub updateProject {
         , enabled => defined $c->stash->{params}->{enabled} ? 1 : 0
         , hidden => defined $c->stash->{params}->{visible} ? 0 : 1
         , owner => $owner
+        , enable_dynamic_run_command => defined $c->stash->{params}->{enable_dynamic_run_command} ? 1 : 0
         , declfile => trim($c->stash->{params}->{declarative}->{file})
         , decltype => trim($c->stash->{params}->{declarative}->{type})
         , declvalue => trim($c->stash->{params}->{declarative}->{value})
diff --git a/src/lib/Hydra/Schema/Result/Projects.pm b/src/lib/Hydra/Schema/Result/Projects.pm
index 9e630b16..42ca22a4 100644
--- a/src/lib/Hydra/Schema/Result/Projects.pm
+++ b/src/lib/Hydra/Schema/Result/Projects.pm
@@ -259,6 +259,7 @@ sub as_json {
 
         # boolean_columns
         "enabled" => $self->get_column("enabled") ? JSON::MaybeXS::true : JSON::MaybeXS::false,
+        "enable_dynamic_run_command" => $self->get_column("enable_dynamic_run_command") ? JSON::MaybeXS::true : JSON::MaybeXS::false,
         "hidden" => $self->get_column("hidden") ? JSON::MaybeXS::true : JSON::MaybeXS::false,
 
         "jobsets" => [ map { $_->name } $self->jobsets ]
diff --git a/src/root/edit-project.tt b/src/root/edit-project.tt
index 6149ec1d..4b99f4ab 100644
--- a/src/root/edit-project.tt
+++ b/src/root/edit-project.tt
@@ -52,6 +52,14 @@
     </div>
   </div>
 
+
+  <div class="form-group row">
+    <label class="col-sm-3" for="editprojectenable_dynamic_run_command">Enable Dynamic RunCommand Hooks for Jobsets</label>
+    <div class="col-sm-9">
+      <input type="checkbox" id="editprojectenable_dynamic_run_command" name="enable_dynamic_run_command" [% IF jobset.enable_dynamic_run_command %]checked[% END %]/>
+    </div>
+  </div>
+
   <div class="form-group row">
     <label class="col-sm-3" for="editprojectdeclfile">
       Declarative spec file
diff --git a/src/root/project.tt b/src/root/project.tt
index 04661c82..f5a51e96 100644
--- a/src/root/project.tt
+++ b/src/root/project.tt
@@ -92,6 +92,10 @@
         <th>Enabled:</th>
         <td>[% project.enabled ? "Yes" : "No" %]</td>
       </tr>
+      <tr>
+        <th>Enable Dynamic RunCommand Hooks:</th>
+        <td>[% project.enable_dynamic_run_command ? "Yes" : "No" %]</td>
+      </tr>
     </table>
   </div>
 

From 2635607b6e926ad6ac570763dcaaf5c4be8bbe7c Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Wed, 15 Dec 2021 15:41:55 -0500
Subject: [PATCH 20/76] whoops: add a test on the enable_dynamic_run_command
 field

---
 hydra-api.yaml                | 3 +++
 t/Hydra/Controller/projects.t | 3 +++
 2 files changed, 6 insertions(+)

diff --git a/hydra-api.yaml b/hydra-api.yaml
index 0d203a41..0fe0a130 100644
--- a/hydra-api.yaml
+++ b/hydra-api.yaml
@@ -607,6 +607,9 @@ components:
         enabled:
           description: when set to true the project gets scheduled for evaluation
           type: boolean
+        enable_dynamic_run_command:
+          description: when true the project's jobsets support executing dynamically defined RunCommand hooks. Requires the server and project's configuration to also enable dynamic RunCommand.
+          type: boolean
         declarative:
           description: declarative input configured for this project
           type: object
diff --git a/t/Hydra/Controller/projects.t b/t/Hydra/Controller/projects.t
index df1290aa..130724cf 100644
--- a/t/Hydra/Controller/projects.t
+++ b/t/Hydra/Controller/projects.t
@@ -46,6 +46,7 @@ subtest "Read project 'tests'" => sub {
         description => "",
         displayname => "Tests",
         enabled => JSON::MaybeXS::true,
+        enable_dynamic_run_command => JSON::MaybeXS::false,
         hidden => JSON::MaybeXS::false,
         homepage => "",
         jobsets => [],
@@ -85,6 +86,7 @@ subtest "Transitioning from declarative project to normal" => sub {
             description => "",
             displayname => "Tests",
             enabled => JSON::MaybeXS::true,
+            enable_dynamic_run_command => JSON::MaybeXS::false,
             hidden => JSON::MaybeXS::false,
             homepage => "",
             jobsets => [".jobsets"],
@@ -128,6 +130,7 @@ subtest "Transitioning from declarative project to normal" => sub {
             description => "",
             displayname => "Tests",
             enabled => JSON::MaybeXS::true,
+            enable_dynamic_run_command => JSON::MaybeXS::false,
             hidden => JSON::MaybeXS::false,
             homepage => "",
             jobsets => [],

From bc1630bd27e6d489122e18b6c37a92ea18b83b0a Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Mon, 24 Jan 2022 15:55:18 -0500
Subject: [PATCH 21/76] fixup! RunCommand: Add a WIP execution of dynamic
 commands

---
 src/lib/Hydra/Plugin/RunCommand.pm | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/lib/Hydra/Plugin/RunCommand.pm b/src/lib/Hydra/Plugin/RunCommand.pm
index 6d099142..b55e96a9 100644
--- a/src/lib/Hydra/Plugin/RunCommand.pm
+++ b/src/lib/Hydra/Plugin/RunCommand.pm
@@ -126,8 +126,8 @@ sub fanoutToCommands {
         next unless eventMatches($conf, $event);
         next unless configSectionMatches(
             $matcher,
-            $build->get_column('project'),
-            $build->get_column('jobset'),
+            $build->jobset->get_column('project'),
+            $build->jobset->get_column('name'),
             $build->get_column('job')
         );
 

From 38514ae4940aaa9c0b499f0beed1ab8272dd8fa3 Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Mon, 24 Jan 2022 16:07:42 -0500
Subject: [PATCH 22/76] fanout tests: capture warnings and test their relevance

---
 t/Hydra/Plugin/RunCommand/fanout.t | 84 +++++++++++++++++-------------
 1 file changed, 49 insertions(+), 35 deletions(-)

diff --git a/t/Hydra/Plugin/RunCommand/fanout.t b/t/Hydra/Plugin/RunCommand/fanout.t
index bd2502ec..808f661c 100644
--- a/t/Hydra/Plugin/RunCommand/fanout.t
+++ b/t/Hydra/Plugin/RunCommand/fanout.t
@@ -121,47 +121,61 @@ subtest "isBuildEligibleForDynamicRunCommand" => sub {
     };
 
     subtest "On outputs ..." => sub {
-        is(
-            Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($builds->{"runCommandHook.example"}),
-            1,
-            "out is an executable file"
-        );
+        ok(!warns {
+            is(
+                Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($builds->{"runCommandHook.example"}),
+                1,
+                "out is an executable file"
+            );
+        }, "No warnings for an executable file.");
 
-        is(
-            Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($builds->{"runCommandHook.symlink"}),
-            1,
-            "out is a symlink to an executable file"
-        );
+        ok(!warns {
+            is(
+                Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($builds->{"runCommandHook.symlink"}),
+                1,
+                "out is a symlink to an executable file"
+            );
+        }, "No warnings for a symlink to an executable file.");
 
-        is(
-            Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($builds->{"runCommandHook.no-out"}),
-            0,
-            "No output named out"
-        );
+        like(warning {
+            is(
+                Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($builds->{"runCommandHook.no-out"}),
+                0,
+                "No output named out"
+            );
+        }, qr/rejected: no output named 'out'/, "A relevant warning is provided for a missing output");
 
-        is(
-            Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($builds->{"runCommandHook.out-is-directory"}),
-            0,
-            "out is a directory"
-        );
+        like(warning {
+            is(
+                Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($builds->{"runCommandHook.out-is-directory"}),
+                0,
+                "out is a directory"
+            );
+        }, qr/output is not a regular file or symlink/, "A relevant warning is provided for a directory output");
 
-        is(
-            Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($builds->{"runCommandHook.out-is-not-executable-file"}),
-            0,
-            "out is a file which is not not executable"
-        );
+        like(warning {
+            is(
+                Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($builds->{"runCommandHook.out-is-not-executable-file"}),
+                0,
+                "out is a file which is not a regular file or symlink"
+            );
+        }, qr/output is not executable/, "A relevant warning is provided if the file isn't executable");
 
-        is(
-            Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($builds->{"runCommandHook.symlink-non-executable"}),
-            0,
-            "out is a symlink to a non-executable file"
-        );
+        like(warning {
+            is(
+                Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($builds->{"runCommandHook.symlink-non-executable"}),
+                0,
+                "out is a symlink to a non-executable file"
+            );
+        }, qr/output is not executable/, "A relevant warning is provided for symlinks to non-executables");
 
-        is(
-            Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($builds->{"runCommandHook.symlink-directory"}),
-            0,
-            "out is a symlink to a directory"
-        );
+        like(warning {
+            is(
+                Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($builds->{"runCommandHook.symlink-directory"}),
+                0,
+                "out is a symlink to a directory"
+            );
+        }, qr/output is not a regular file or symlink/, "A relevant warning is provided for symlinks to directories");
     };
 
     subtest "On build status ..." => sub {

From daa6864a58e0f29509c2718d6dc91dc84198808b Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Mon, 24 Jan 2022 16:09:45 -0500
Subject: [PATCH 23/76] Project result: add a supportsDynamicRunCommand helper

---
 src/lib/Hydra/Plugin/RunCommand.pm      | 2 +-
 src/lib/Hydra/Schema/Result/Projects.pm | 6 ++++++
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/src/lib/Hydra/Plugin/RunCommand.pm b/src/lib/Hydra/Plugin/RunCommand.pm
index b55e96a9..9e173278 100644
--- a/src/lib/Hydra/Plugin/RunCommand.pm
+++ b/src/lib/Hydra/Plugin/RunCommand.pm
@@ -75,7 +75,7 @@ sub isBuildEligibleForDynamicRunCommand {
             return 0;
         }
 
-        if (! $build->project->enable_dynamic_run_command) {
+        if (! $build->project->supportsDynamicRunCommand()) {
             return 0;
         }
 
diff --git a/src/lib/Hydra/Schema/Result/Projects.pm b/src/lib/Hydra/Schema/Result/Projects.pm
index 42ca22a4..d6e66bf7 100644
--- a/src/lib/Hydra/Schema/Result/Projects.pm
+++ b/src/lib/Hydra/Schema/Result/Projects.pm
@@ -246,6 +246,12 @@ sub builds {
   return $self->jobsets->related_resultset('builds');
 };
 
+sub supportsDynamicRunCommand {
+  my ($self) = @_;
+
+  return $self->get_column('enable_dynamic_run_command') == 1;
+}
+
 sub as_json {
     my $self = shift;
 

From 3aa239309181d04dc6831b22c8abb0bbf85cb657 Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Mon, 24 Jan 2022 16:11:52 -0500
Subject: [PATCH 24/76] Jobsets: add a supportsDynamicRunCommand which also
 checks the project's dynamic runcommand support

---
 src/lib/Hydra/Plugin/RunCommand.pm     | 6 +-----
 src/lib/Hydra/Schema/Result/Jobsets.pm | 7 +++++++
 2 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/src/lib/Hydra/Plugin/RunCommand.pm b/src/lib/Hydra/Plugin/RunCommand.pm
index 9e173278..2b3bb6f4 100644
--- a/src/lib/Hydra/Plugin/RunCommand.pm
+++ b/src/lib/Hydra/Plugin/RunCommand.pm
@@ -71,11 +71,7 @@ sub isBuildEligibleForDynamicRunCommand {
             return 0;
         }
 
-        if (! $build->jobset->enable_dynamic_run_command) {
-            return 0;
-        }
-
-        if (! $build->project->supportsDynamicRunCommand()) {
+        if (! $build->jobset->supportsDynamicRunCommand()) {
             return 0;
         }
 
diff --git a/src/lib/Hydra/Schema/Result/Jobsets.pm b/src/lib/Hydra/Schema/Result/Jobsets.pm
index 7b96c472..cd704ac8 100644
--- a/src/lib/Hydra/Schema/Result/Jobsets.pm
+++ b/src/lib/Hydra/Schema/Result/Jobsets.pm
@@ -386,6 +386,13 @@ __PACKAGE__->add_column(
     "+id" => { retrieve_on_insert => 1 }
 );
 
+sub supportsDynamicRunCommand {
+  my ($self) = @_;
+
+  return $self->get_column('enable_dynamic_run_command') == 1
+    && $self->project->supportsDynamicRunCommand();
+}
+
 sub as_json {
     my $self = shift;
 

From d8b56f022d1acd2b9f08106ab502c0a51393e6a7 Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Mon, 24 Jan 2022 16:16:58 -0500
Subject: [PATCH 25/76] RunCommand: print a warning if the hook isn't run
 because the project / jobset doens't have it enabled

---
 src/lib/Hydra/Plugin/RunCommand.pm |  1 +
 t/Hydra/Plugin/RunCommand/fanout.t | 37 ++++++++++++++++++------------
 2 files changed, 23 insertions(+), 15 deletions(-)

diff --git a/src/lib/Hydra/Plugin/RunCommand.pm b/src/lib/Hydra/Plugin/RunCommand.pm
index 2b3bb6f4..43163764 100644
--- a/src/lib/Hydra/Plugin/RunCommand.pm
+++ b/src/lib/Hydra/Plugin/RunCommand.pm
@@ -72,6 +72,7 @@ sub isBuildEligibleForDynamicRunCommand {
         }
 
         if (! $build->jobset->supportsDynamicRunCommand()) {
+            warn "DynamicRunCommand hook on " . $build->job . " (" . $build->id . ") rejected: The project or jobset don't have dynamic runcommand enabled.";
             return 0;
         }
 
diff --git a/t/Hydra/Plugin/RunCommand/fanout.t b/t/Hydra/Plugin/RunCommand/fanout.t
index 808f661c..328824f9 100644
--- a/t/Hydra/Plugin/RunCommand/fanout.t
+++ b/t/Hydra/Plugin/RunCommand/fanout.t
@@ -191,33 +191,40 @@ subtest "isBuildEligibleForDynamicRunCommand" => sub {
             $build->project->update({enable_dynamic_run_command => 0});
             $build->jobset->update({enable_dynamic_run_command => 1});
 
-            is(
-                Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($builds->{"runCommandHook.example"}),
-                0,
-                "Builds don't run from a jobset with disabled dynamic runcommand"
-            );
+
+            like(warning {
+                is(
+                    Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($builds->{"runCommandHook.example"}),
+                    0,
+                    "Builds don't run from a jobset with disabled dynamic runcommand"
+                );
+            }, qr/project or jobset don't have dynamic runcommand enabled./, "A relevant warning is provided for a disabled runcommand support")
         };
 
         subtest "enabled on the project, disabled on the jobset" => sub {
             $build->project->update({enable_dynamic_run_command => 1});
             $build->jobset->update({enable_dynamic_run_command => 0});
 
-            is(
-                Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($builds->{"runCommandHook.example"}),
-                0,
-                "Builds don't run from a jobset with disabled dynamic runcommand"
-            );
+            like(warning {
+                is(
+                    Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($builds->{"runCommandHook.example"}),
+                    0,
+                    "Builds don't run from a jobset with disabled dynamic runcommand"
+                );
+            }, qr/project or jobset don't have dynamic runcommand enabled./, "A relevant warning is provided for a disabled runcommand support")
         };
 
         subtest "disabled on the project, disabled on the jobset" => sub {
             $build->project->update({enable_dynamic_run_command => 0});
             $build->jobset->update({enable_dynamic_run_command => 0});
 
-            is(
-                Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($builds->{"runCommandHook.example"}),
-                0,
-                "Builds don't run from a jobset with disabled dynamic runcommand"
-            );
+            like(warning {
+                is(
+                    Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($builds->{"runCommandHook.example"}),
+                    0,
+                    "Builds don't run from a jobset with disabled dynamic runcommand"
+                );
+            }, qr/project or jobset don't have dynamic runcommand enabled./, "A relevant warning is provided for a disabled runcommand support")
         };
     };
 };

From 3b895aec54ba07d03feef7c151ccf980967d29cd Mon Sep 17 00:00:00 2001
From: Cole Helbling <cole.helbling@determinate.systems>
Date: Fri, 17 Dec 2021 10:29:47 -0800
Subject: [PATCH 26/76] DynamicRunCommand: needs to be enabled by server,
 project, and jobset

---
 doc/manual/src/plugins/RunCommand.md | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/doc/manual/src/plugins/RunCommand.md b/doc/manual/src/plugins/RunCommand.md
index b186be80..652a171e 100644
--- a/doc/manual/src/plugins/RunCommand.md
+++ b/doc/manual/src/plugins/RunCommand.md
@@ -33,8 +33,9 @@ Command to run. Can use the `$HYDRA_JSON` environment variable to access informa
 
 ### Dynamic Commands
 
-Hydra can optionally run RunCommand hooks defined dynamically by the jobset.
-This must be turned on explicitly in the `hydra.conf` and per jobset.
+Hydra can optionally run RunCommand hooks defined dynamically by the jobset. In
+order to enable dynamic commands, you must enable this feature in your
+`hydra.conf`, *as well as* in the parent project and jobset configuration.
 
 #### Behavior
 

From 3f4f1837928add779759da21522b617b93748a14 Mon Sep 17 00:00:00 2001
From: Cole Helbling <cole.helbling@determinate.systems>
Date: Fri, 17 Dec 2021 10:31:03 -0800
Subject: [PATCH 27/76] jobset.tt: more info on why Dynamic RunCommand is
 disabled

---
 src/root/jobset.tt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/root/jobset.tt b/src/root/jobset.tt
index 3d6ca6ae..56abdb50 100644
--- a/src/root/jobset.tt
+++ b/src/root/jobset.tt
@@ -162,7 +162,7 @@
       </tr>
       <tr>
         <th>Enable Dynamic RunCommand Hooks:</th>
-        <td>[% jobset.enable_dynamic_run_command ? "Yes" : "No" %]</td>
+        <td>[% c.config.dynamicruncommand.enable ? project.enable_dynamic_run_command ? jobset.enable_dynamic_run_command ? "Yes" : "No (not enabled by jobset)" : "No (not enabled by project)" : "No (not enabled by server)" %]</td>
       </tr>
       [% IF emailNotification %]
       <tr>

From dfd3a67424c0cd834d2a009d061336f3b61814ff Mon Sep 17 00:00:00 2001
From: Cole Helbling <cole.helbling@determinate.systems>
Date: Fri, 17 Dec 2021 10:31:46 -0800
Subject: [PATCH 28/76] project.tt: more info on why Dynamic RunCommand is
 disabled

---
 src/root/project.tt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/root/project.tt b/src/root/project.tt
index f5a51e96..5e8ec0c8 100644
--- a/src/root/project.tt
+++ b/src/root/project.tt
@@ -94,7 +94,7 @@
       </tr>
       <tr>
         <th>Enable Dynamic RunCommand Hooks:</th>
-        <td>[% project.enable_dynamic_run_command ? "Yes" : "No" %]</td>
+        <td>[% c.config.dynamicruncommand.enable ? project.enable_dynamic_run_command ? "Yes" : "No (not enabled by project)" : "No (not enabled by server)" %]</td>
       </tr>
     </table>
   </div>

From 6053e5fd4b2e23aeb69b632417a4e42ca88fe168 Mon Sep 17 00:00:00 2001
From: Cole Helbling <cole.helbling@determinate.systems>
Date: Fri, 17 Dec 2021 11:02:59 -0800
Subject: [PATCH 29/76] edit-jobset.tt: disable when disabled by project and
 server

Also add a tooltip describing why it's disabled, to make it easier to
chase down.
---
 src/root/edit-jobset.tt | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/src/root/edit-jobset.tt b/src/root/edit-jobset.tt
index 40da8f61..61e3636f 100644
--- a/src/root/edit-jobset.tt
+++ b/src/root/edit-jobset.tt
@@ -160,7 +160,15 @@
   <div class="form-group row">
     <label class="col-sm-3" for="editjobsetenable_dynamic_run_command">Enable Dynamic RunCommand Hooks</label>
     <div class="col-sm-9">
-      <input type="checkbox" id="editjobsetenable_dynamic_run_command" name="enable_dynamic_run_command" [% IF jobset.enable_dynamic_run_command %]checked[% END %]/>
+      <input type="checkbox" id="editjobsetenable_dynamic_run_command" name="enable_dynamic_run_command"
+        [% IF !c.config.dynamicruncommand.enable %]
+          title="The server has not enabled dynamic RunCommands" disabled
+        [% ELSIF !project.enable_dynamic_run_command %]
+          title="The parent project has not enabled dynamic RunCommands" disabled
+        [% ELSIF jobset.enable_dynamic_run_command %]
+          checked
+        [% END %]
+      />
     </div>
   </div>
 

From d680c209feffb3fe1087cd39e62545cd03cb22f8 Mon Sep 17 00:00:00 2001
From: Cole Helbling <cole.helbling@determinate.systems>
Date: Fri, 17 Dec 2021 11:03:55 -0800
Subject: [PATCH 30/76] edit-project.tt: disable when disabled by server

Also add a tooltip describing why it's disabled, to make it easier to
chase down.
---
 src/root/edit-project.tt | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/src/root/edit-project.tt b/src/root/edit-project.tt
index 4b99f4ab..bb850e5c 100644
--- a/src/root/edit-project.tt
+++ b/src/root/edit-project.tt
@@ -56,7 +56,13 @@
   <div class="form-group row">
     <label class="col-sm-3" for="editprojectenable_dynamic_run_command">Enable Dynamic RunCommand Hooks for Jobsets</label>
     <div class="col-sm-9">
-      <input type="checkbox" id="editprojectenable_dynamic_run_command" name="enable_dynamic_run_command" [% IF jobset.enable_dynamic_run_command %]checked[% END %]/>
+      <input type="checkbox" id="editprojectenable_dynamic_run_command" name="enable_dynamic_run_command"
+        [% IF !c.config.dynamicruncommand.enable %]
+          title="The server has not enabled dynamic RunCommands" disabled
+        [% ELSIF project.enable_dynamic_run_command %]
+          checked
+        [% END %]
+      />
     </div>
   </div>
 

From 928ba9e854e8fb7ee88f352a32848f114004e70c Mon Sep 17 00:00:00 2001
From: Cole Helbling <cole.helbling@determinate.systems>
Date: Fri, 17 Dec 2021 12:34:19 -0800
Subject: [PATCH 31/76] Controller/{Jobset,Project}: error when enabling
 dynamic runcommand but it's disabled elsewhere

---
 src/lib/Hydra/Controller/Jobset.pm           |  10 +-
 src/lib/Hydra/Controller/Project.pm          |   7 +-
 t/Hydra/Plugin/RunCommand/dynamic-disabled.t | 110 +++++++++++++++++++
 t/Hydra/Plugin/RunCommand/dynamic-enabled.t  | 106 ++++++++++++++++++
 4 files changed, 231 insertions(+), 2 deletions(-)
 create mode 100644 t/Hydra/Plugin/RunCommand/dynamic-disabled.t
 create mode 100644 t/Hydra/Plugin/RunCommand/dynamic-enabled.t

diff --git a/src/lib/Hydra/Controller/Jobset.pm b/src/lib/Hydra/Controller/Jobset.pm
index a2d48597..eeb4232a 100644
--- a/src/lib/Hydra/Controller/Jobset.pm
+++ b/src/lib/Hydra/Controller/Jobset.pm
@@ -261,6 +261,14 @@ sub updateJobset {
 
     my $checkinterval = int(trim($c->stash->{params}->{checkinterval}));
 
+    my $enable_dynamic_run_command = defined $c->stash->{params}->{enable_dynamic_run_command} ? 1 : 0;
+    if ($enable_dynamic_run_command
+        && !($c->config->{dynamicruncommand}->{enable}
+            && $jobset->project->enable_dynamic_run_command))
+    {
+        badRequest($c, "Dynamic RunCommand is not enabled by the server or the parent project.");
+    }
+
     $jobset->update(
         { name => $jobsetName
         , description => trim($c->stash->{params}->{"description"})
@@ -268,7 +276,7 @@ sub updateJobset {
         , nixexprinput => $nixExprInput
         , enabled => $enabled
         , enableemail => defined $c->stash->{params}->{enableemail} ? 1 : 0
-        , enable_dynamic_run_command => defined $c->stash->{params}->{enable_dynamic_run_command} ? 1 : 0
+        , enable_dynamic_run_command => $enable_dynamic_run_command
         , emailoverride => trim($c->stash->{params}->{emailoverride}) || ""
         , hidden => defined $c->stash->{params}->{visible} ? 0 : 1
         , keepnr => int(trim($c->stash->{params}->{keepnr} // "0"))
diff --git a/src/lib/Hydra/Controller/Project.pm b/src/lib/Hydra/Controller/Project.pm
index 98a8a6eb..1141de4a 100644
--- a/src/lib/Hydra/Controller/Project.pm
+++ b/src/lib/Hydra/Controller/Project.pm
@@ -149,6 +149,11 @@ sub updateProject {
     my $displayName = trim $c->stash->{params}->{displayname};
     error($c, "You must specify a display name.") if $displayName eq "";
 
+    my $enable_dynamic_run_command = defined $c->stash->{params}->{enable_dynamic_run_command} ? 1 : 0;
+    if ($enable_dynamic_run_command && !$c->config->{dynamicruncommand}->{enable}) {
+        badRequest($c, "Dynamic RunCommand is not enabled by the server.");
+    }
+
     $project->update(
         { name => $projectName
         , displayname => $displayName
@@ -157,7 +162,7 @@ sub updateProject {
         , enabled => defined $c->stash->{params}->{enabled} ? 1 : 0
         , hidden => defined $c->stash->{params}->{visible} ? 0 : 1
         , owner => $owner
-        , enable_dynamic_run_command => defined $c->stash->{params}->{enable_dynamic_run_command} ? 1 : 0
+        , enable_dynamic_run_command => $enable_dynamic_run_command
         , declfile => trim($c->stash->{params}->{declarative}->{file})
         , decltype => trim($c->stash->{params}->{declarative}->{type})
         , declvalue => trim($c->stash->{params}->{declarative}->{value})
diff --git a/t/Hydra/Plugin/RunCommand/dynamic-disabled.t b/t/Hydra/Plugin/RunCommand/dynamic-disabled.t
new file mode 100644
index 00000000..ad2e9a4b
--- /dev/null
+++ b/t/Hydra/Plugin/RunCommand/dynamic-disabled.t
@@ -0,0 +1,110 @@
+use strict;
+use warnings;
+use Setup;
+use Test2::V0;
+
+require Catalyst::Test;
+use HTTP::Request::Common qw(POST PUT GET DELETE);
+use JSON::MaybeXS qw(decode_json encode_json);
+
+my $ctx = test_context();
+Catalyst::Test->import('Hydra');
+
+# Create a user to log in to
+my $user = $ctx->db->resultset('Users')->create({ username => 'alice', emailaddress => 'root@invalid.org', password => '!' });
+$user->setPassword('foobar');
+$user->userroles->update_or_create({ role => 'admin' });
+
+subtest "can't enable dynamic RunCommand when disabled by server" => sub {
+    my $builds = $ctx->makeAndEvaluateJobset(
+        expression => "runcommand-dynamic.nix",
+        build => 1
+    );
+
+    my $build = $builds->{"runCommandHook.example"};
+    my $project = $build->project;
+    my $project_name = $project->name;
+    my $jobset = $build->jobset;
+    my $jobset_name = $jobset->name;
+
+    is($project->enable_dynamic_run_command, 0, "dynamic RunCommand is disabled on projects by default");
+    is($jobset->enable_dynamic_run_command, 0, "dynamic RunCommand is disabled on jobsets by default");
+
+    my $req = request(POST '/login',
+        Referer => 'http://localhost/',
+        Content => {
+            username => 'alice',
+            password => 'foobar'
+        }
+    );
+    is($req->code, 302, "logged in successfully");
+    my $cookie = $req->header("set-cookie");
+
+    subtest "can't enable dynamic RunCommand on project" => sub {
+        my $projectresponse = request(GET "/project/$project_name",
+            Accept => 'application/json',
+            Content_Type => 'application/json',
+            Cookie => $cookie,
+        );
+
+        my $projectjson = decode_json($projectresponse->content);
+        $projectjson->{enable_dynamic_run_command} = 1;
+
+        my $projectupdate = request(PUT "/project/$project_name",
+            Accept => 'application/json',
+            Content_Type => 'application/json',
+            Cookie => $cookie,
+            Content => encode_json($projectjson)
+        );
+
+        $projectresponse = request(GET "/project/$project_name",
+            Accept => 'application/json',
+            Content_Type => 'application/json',
+            Cookie => $cookie,
+        );
+        $projectjson = decode_json($projectresponse->content);
+
+        is($projectupdate->code, 400);
+        like(
+            $projectupdate->content,
+            qr/Dynamic RunCommand is not/,
+            "failed to change enable_dynamic_run_command, not any other error"
+        );
+        is($projectjson->{enable_dynamic_run_command}, JSON::MaybeXS::false);
+    };
+
+    subtest "can't enable dynamic RunCommand on jobset" => sub {
+        my $jobsetresponse = request(GET "/jobset/$project_name/$jobset_name",
+            Accept => 'application/json',
+            Content_Type => 'application/json',
+            Cookie => $cookie,
+        );
+
+        my $jobsetjson = decode_json($jobsetresponse->content);
+        $jobsetjson->{enable_dynamic_run_command} = 1;
+
+        my $jobsetupdate = request(PUT "/jobset/$project_name/$jobset_name",
+            Accept => 'application/json',
+            Content_Type => 'application/json',
+            Cookie => $cookie,
+            Content => encode_json($jobsetjson)
+        );
+
+        $jobsetresponse = request(GET "/jobset/$project_name/$jobset_name",
+            Accept => 'application/json',
+            Content_Type => 'application/json',
+            Cookie => $cookie,
+        );
+        $jobsetjson = decode_json($jobsetresponse->content);
+
+        is($jobsetupdate->code, 400);
+        like(
+            $jobsetupdate->content,
+            qr/Dynamic RunCommand is not/,
+            "failed to change enable_dynamic_run_command, not any other error"
+        );
+        is($jobsetjson->{enable_dynamic_run_command}, JSON::MaybeXS::false);
+    };
+};
+
+done_testing;
diff --git a/t/Hydra/Plugin/RunCommand/dynamic-enabled.t b/t/Hydra/Plugin/RunCommand/dynamic-enabled.t
new file mode 100644
index 00000000..68c6d593
--- /dev/null
+++ b/t/Hydra/Plugin/RunCommand/dynamic-enabled.t
@@ -0,0 +1,106 @@
+use strict;
+use warnings;
+use Setup;
+use Test2::V0;
+
+require Catalyst::Test;
+use HTTP::Request::Common qw(POST PUT GET DELETE);
+use JSON::MaybeXS qw(decode_json encode_json);
+
+my $ctx = test_context(
+    hydra_config => q|
+    <dynamicruncommand>
+    enable = 1
+    </dynamicruncommand>
+    |
+);
+Catalyst::Test->import('Hydra');
+
+# Create a user to log in to
+my $user = $ctx->db->resultset('Users')->create({ username => 'alice', emailaddress => 'root@invalid.org', password => '!' });
+$user->setPassword('foobar');
+$user->userroles->update_or_create({ role => 'admin' });
+
+subtest "can enable dynamic RunCommand when enabled by server" => sub {
+    my $builds = $ctx->makeAndEvaluateJobset(
+        expression => "runcommand-dynamic.nix",
+        build => 1
+    );
+
+    my $build = $builds->{"runCommandHook.example"};
+    my $project = $build->project;
+    my $project_name = $project->name;
+    my $jobset = $build->jobset;
+    my $jobset_name = $jobset->name;
+
+    is($project->enable_dynamic_run_command, 0, "dynamic RunCommand is disabled on projects by default");
+    is($jobset->enable_dynamic_run_command, 0, "dynamic RunCommand is disabled on jobsets by default");
+
+    my $req = request(POST '/login',
+        Referer => 'http://localhost/',
+        Content => {
+            username => 'alice',
+            password => 'foobar'
+        }
+    );
+    is($req->code, 302, "logged in successfully");
+    my $cookie = $req->header("set-cookie");
+
+    subtest "can enable dynamic RunCommand on project" => sub {
+        my $projectresponse = request(GET "/project/$project_name",
+            Accept => 'application/json',
+            Content_Type => 'application/json',
+            Cookie => $cookie,
+        );
+
+        my $projectjson = decode_json($projectresponse->content);
+        $projectjson->{enable_dynamic_run_command} = 1;
+
+        my $projectupdate = request(PUT "/project/$project_name",
+            Accept => 'application/json',
+            Content_Type => 'application/json',
+            Cookie => $cookie,
+            Content => encode_json($projectjson)
+        );
+
+        $projectresponse = request(GET "/project/$project_name",
+            Accept => 'application/json',
+            Content_Type => 'application/json',
+            Cookie => $cookie,
+        );
+        $projectjson = decode_json($projectresponse->content);
+
+        is($projectupdate->code, 200);
+        is($projectjson->{enable_dynamic_run_command}, JSON::MaybeXS::true);
+    };
+
+    subtest "can enable dynamic RunCommand on jobset" => sub {
+        my $jobsetresponse = request(GET "/jobset/$project_name/$jobset_name",
+            Accept => 'application/json',
+            Content_Type => 'application/json',
+            Cookie => $cookie,
+        );
+
+        my $jobsetjson = decode_json($jobsetresponse->content);
+        $jobsetjson->{enable_dynamic_run_command} = 1;
+
+        my $jobsetupdate = request(PUT "/jobset/$project_name/$jobset_name",
+            Accept => 'application/json',
+            Content_Type => 'application/json',
+            Cookie => $cookie,
+            Content => encode_json($jobsetjson)
+        );
+
+        $jobsetresponse = request(GET "/jobset/$project_name/$jobset_name",
+            Accept => 'application/json',
+            Content_Type => 'application/json',
+            Cookie => $cookie,
+        );
+        $jobsetjson = decode_json($jobsetresponse->content);
+
+        is($jobsetupdate->code, 200);
+        is($jobsetjson->{enable_dynamic_run_command}, JSON::MaybeXS::true);
+    };
+};
+
+done_testing;

From a22a8fa62d777950dca470b9791e5ac8cda2ddbf Mon Sep 17 00:00:00 2001
From: Cole Helbling <cole.helbling@determinate.systems>
Date: Mon, 20 Dec 2021 09:37:14 -0800
Subject: [PATCH 32/76] AddBuilds: reject declarative jobsets with dynamic
 runcommand enabled if disabled elsewhere

---
 src/lib/Hydra/Helper/AddBuilds.pm     | 44 +++++++++++---
 src/script/hydra-eval-jobset          |  2 +-
 t/Helper/AddBuilds/dynamic-disabled.t | 85 ++++++++++++++++++++++++++
 t/Helper/AddBuilds/dynamic-enabled.t  | 88 +++++++++++++++++++++++++++
 4 files changed, 209 insertions(+), 10 deletions(-)
 create mode 100644 t/Helper/AddBuilds/dynamic-disabled.t
 create mode 100644 t/Helper/AddBuilds/dynamic-enabled.t

diff --git a/src/lib/Hydra/Helper/AddBuilds.pm b/src/lib/Hydra/Helper/AddBuilds.pm
index f38737d3..9e3ddfd2 100644
--- a/src/lib/Hydra/Helper/AddBuilds.pm
+++ b/src/lib/Hydra/Helper/AddBuilds.pm
@@ -19,14 +19,16 @@ use Hydra::Helper::CatalystUtils;
 
 our @ISA = qw(Exporter);
 our @EXPORT = qw(
+    validateDeclarativeJobset
+    createJobsetInputsRowAndData
     updateDeclarativeJobset
     handleDeclarativeJobsetBuild
     handleDeclarativeJobsetJson
 );
 
 
-sub updateDeclarativeJobset {
-    my ($db, $project, $jobsetName, $declSpec) = @_;
+sub validateDeclarativeJobset {
+    my ($config, $project, $jobsetName, $declSpec) = @_;
 
     my @allowed_keys = qw(
         enabled
@@ -62,16 +64,39 @@ sub updateDeclarativeJobset {
         }
     }
 
+    my $enable_dynamic_run_command = defined $update{enable_dynamic_run_command} ? 1 : 0;
+    if ($enable_dynamic_run_command
+        && !($config->{dynamicruncommand}->{enable}
+            && $project->{enable_dynamic_run_command}))
+    {
+        die "Dynamic RunCommand is not enabled by the server or the parent project.";
+    }
+
+    return %update;
+}
+
+sub createJobsetInputsRowAndData {
+    my ($name, $declSpec) = @_;
+    my $data = $declSpec->{"inputs"}->{$name};
+    my $row = {
+        name => $name,
+        type => $data->{type}
+    };
+    $row->{emailresponsible} = $data->{emailresponsible} // 0;
+
+    return ($row, $data);
+}
+
+sub updateDeclarativeJobset {
+    my ($config, $db, $project, $jobsetName, $declSpec) = @_;
+
+    my %update = validateDeclarativeJobset($config, $project, $jobsetName, $declSpec);
+
     $db->txn_do(sub {
         my $jobset = $project->jobsets->update_or_create(\%update);
         $jobset->jobsetinputs->delete;
         foreach my $name (keys %{$declSpec->{"inputs"}}) {
-            my $data = $declSpec->{"inputs"}->{$name};
-            my $row = {
-                name => $name,
-                type => $data->{type}
-            };
-            $row->{emailresponsible} = $data->{emailresponsible} // 0;
+            my ($row, $data) = createJobsetInputsRowAndData($name, $declSpec);
             my $input = $jobset->jobsetinputs->create($row);
             $input->jobsetinputalts->create({altnr => 0, value => $data->{value}});
         }
@@ -82,6 +107,7 @@ sub updateDeclarativeJobset {
 
 sub handleDeclarativeJobsetJson {
     my ($db, $project, $declSpec) = @_;
+    my $config = getHydraConfig();
     $db->txn_do(sub {
             my @kept = keys %$declSpec;
             push @kept, ".jobsets";
@@ -89,7 +115,7 @@ sub handleDeclarativeJobsetJson {
             foreach my $jobsetName (keys %$declSpec) {
                 my $spec = $declSpec->{$jobsetName};
                 eval {
-                    updateDeclarativeJobset($db, $project, $jobsetName, $spec);
+                    updateDeclarativeJobset($config, $db, $project, $jobsetName, $spec);
                     1;
                 } or do {
                     print STDERR "ERROR: failed to process declarative jobset ", $project->name, ":${jobsetName}, ", $@, "\n";
diff --git a/src/script/hydra-eval-jobset b/src/script/hydra-eval-jobset
index de437ecd..a9bd7355 100755
--- a/src/script/hydra-eval-jobset
+++ b/src/script/hydra-eval-jobset
@@ -617,7 +617,7 @@ sub checkJobsetWrapped {
             } else {
                 # Update the jobset with the spec's inputs, and the continue
                 # evaluating the .jobsets jobset.
-                updateDeclarativeJobset($db, $project, ".jobsets", $declSpec);
+                updateDeclarativeJobset($config, $db, $project, ".jobsets", $declSpec);
                 $jobset->discard_changes;
                 $inputInfo->{"declInput"} = [ $declInput ];
                 $inputInfo->{"projectName"} = [ fetchInput($plugins, $db, $project, $jobset, "projectName", "string", $project->name, 0) ];
diff --git a/t/Helper/AddBuilds/dynamic-disabled.t b/t/Helper/AddBuilds/dynamic-disabled.t
new file mode 100644
index 00000000..0507b03e
--- /dev/null
+++ b/t/Helper/AddBuilds/dynamic-disabled.t
@@ -0,0 +1,85 @@
+use strict;
+use warnings;
+use Setup;
+use Test2::V0;
+
+require Catalyst::Test;
+use HTTP::Request::Common qw(POST PUT GET DELETE);
+use JSON::MaybeXS qw(decode_json encode_json);
+use Hydra::Helper::AddBuilds qw(validateDeclarativeJobset);
+use Hydra::Helper::Nix qw(getHydraConfig);
+
+my $ctx = test_context();
+
+sub makeJobsetSpec {
+    my ($dynamic) = @_;
+
+    return {
+        enabled => 2,
+        enable_dynamic_run_command => $dynamic ? JSON::MaybeXS::true : undef,
+        visible => JSON::MaybeXS::true,
+        name => "job",
+        type => 1,
+        description => "test jobset",
+        flake => "github:nixos/nix",
+        checkinterval => 0,
+        schedulingshares => 100,
+        keepnr => 3
+    };
+};
+
+subtest "validate declarative jobset with dynamic RunCommand disabled by server" => sub {
+    my $config = getHydraConfig();
+
+    subtest "project enabled dynamic runcommand, declarative jobset enabled dynamic runcommand" => sub {
+        like(
+            dies {
+              validateDeclarativeJobset(
+                  $config,
+                  { enable_dynamic_run_command => 1 },
+                  "test-jobset",
+                  makeJobsetSpec(JSON::MaybeXS::true),
+              ),
+            },
+            qr/Dynamic RunCommand is not enabled/,
+        );
+    };
+
+    subtest "project enabled dynamic runcommand, declarative jobset disabled dynamic runcommand" => sub {
+        ok(
+            validateDeclarativeJobset(
+                $config,
+                { enable_dynamic_run_command => 1 },
+                "test-jobset",
+                makeJobsetSpec(JSON::MaybeXS::false)
+            ),
+        );
+    };
+
+    subtest "project disabled dynamic runcommand, declarative jobset enabled dynamic runcommand" => sub {
+        like(
+            dies {
+              validateDeclarativeJobset(
+                  $config,
+                  { enable_dynamic_run_command => 0 },
+                  "test-jobset",
+                  makeJobsetSpec(JSON::MaybeXS::true),
+              ),
+            },
+            qr/Dynamic RunCommand is not enabled/,
+        );
+    };
+
+    subtest "project disabled dynamic runcommand, declarative jobset disabled dynamic runcommand" => sub {
+        ok(
+            validateDeclarativeJobset(
+                $config,
+                { enable_dynamic_run_command => 0 },
+                "test-jobset",
+                makeJobsetSpec(JSON::MaybeXS::false)
+            ),
+        );
+    };
+};
+
+done_testing;
diff --git a/t/Helper/AddBuilds/dynamic-enabled.t b/t/Helper/AddBuilds/dynamic-enabled.t
new file mode 100644
index 00000000..d2f5a386
--- /dev/null
+++ b/t/Helper/AddBuilds/dynamic-enabled.t
@@ -0,0 +1,88 @@
+use strict;
+use warnings;
+use Setup;
+use Test2::V0;
+
+require Catalyst::Test;
+use HTTP::Request::Common qw(POST PUT GET DELETE);
+use JSON::MaybeXS qw(decode_json encode_json);
+use Hydra::Helper::AddBuilds qw(validateDeclarativeJobset);
+use Hydra::Helper::Nix qw(getHydraConfig);
+
+my $ctx = test_context(
+    hydra_config => q|
+    <dynamicruncommand>
+    enable = 1
+    </dynamicruncommand>
+    |
+);
+
+sub makeJobsetSpec {
+    my ($dynamic) = @_;
+
+    return {
+        enabled => 2,
+        enable_dynamic_run_command => $dynamic ? JSON::MaybeXS::true : undef,
+        visible => JSON::MaybeXS::true,
+        name => "job",
+        type => 1,
+        description => "test jobset",
+        flake => "github:nixos/nix",
+        checkinterval => 0,
+        schedulingshares => 100,
+        keepnr => 3
+    };
+};
+
+subtest "validate declarative jobset with dynamic RunCommand enabled by server" => sub {
+    my $config = getHydraConfig();
+
+    subtest "project enabled dynamic runcommand, declarative jobset enabled dynamic runcommand" => sub {
+        ok(
+            validateDeclarativeJobset(
+                $config,
+                { enable_dynamic_run_command => 1 },
+                "test-jobset",
+                makeJobsetSpec(JSON::MaybeXS::true)
+            ),
+        );
+    };
+
+    subtest "project enabled dynamic runcommand, declarative jobset disabled dynamic runcommand" => sub {
+        ok(
+            validateDeclarativeJobset(
+                $config,
+                { enable_dynamic_run_command => 1 },
+                "test-jobset",
+                makeJobsetSpec(JSON::MaybeXS::false)
+            ),
+        );
+    };
+
+    subtest "project disabled dynamic runcommand, declarative jobset enabled dynamic runcommand" => sub {
+        like(
+            dies {
+              validateDeclarativeJobset(
+                  $config,
+                  { enable_dynamic_run_command => 0 },
+                  "test-jobset",
+                  makeJobsetSpec(JSON::MaybeXS::true),
+              ),
+            },
+            qr/Dynamic RunCommand is not enabled/,
+        );
+    };
+
+    subtest "project disabled dynamic runcommand, declarative jobset disabled dynamic runcommand" => sub {
+        ok(
+            validateDeclarativeJobset(
+                $config,
+                { enable_dynamic_run_command => 0 },
+                "test-jobset",
+                makeJobsetSpec(JSON::MaybeXS::false)
+            ),
+        );
+    };
+};
+
+done_testing;

From 8c3122cacd82d86467b1e69b58db14f058a7afe5 Mon Sep 17 00:00:00 2001
From: Cole Helbling <cole.helbling@determinate.systems>
Date: Mon, 20 Dec 2021 11:20:17 -0800
Subject: [PATCH 33/76] hydra-api: add enable_dynamic_run_command to Project
 PUT

---
 hydra-api.yaml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/hydra-api.yaml b/hydra-api.yaml
index 0fe0a130..ce7e0f9a 100644
--- a/hydra-api.yaml
+++ b/hydra-api.yaml
@@ -178,6 +178,9 @@ paths:
                 enabled:
                   description: when set to true the project gets scheduled for evaluation
                   type: boolean
+                enable_dynamic_run_command:
+                  description: when true the project's jobsets support executing dynamically defined RunCommand hooks. Requires the server and project's configuration to also enable dynamic RunCommand.
+                  type: boolean
                 visible:
                   description: when set to true the project is displayed in the web interface
                   type: boolean

From 27ddde1e9eb95b694884dce5dfb24b67c02aefbd Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Fri, 11 Feb 2022 15:03:09 -0500
Subject: [PATCH 34/76] dynamic runcommand: print a notice on the build page if
 it is disabled

---
 src/lib/Hydra/Controller/Build.pm | 11 +++++++++++
 src/root/build.tt                 | 15 ++++++++++++++-
 2 files changed, 25 insertions(+), 1 deletion(-)

diff --git a/src/lib/Hydra/Controller/Build.pm b/src/lib/Hydra/Controller/Build.pm
index af648109..552f31af 100644
--- a/src/lib/Hydra/Controller/Build.pm
+++ b/src/lib/Hydra/Controller/Build.pm
@@ -38,6 +38,17 @@ sub buildChain :Chained('/') :PathPart('build') :CaptureArgs(1) {
     $c->stash->{jobset} = $c->stash->{build}->jobset;
     $c->stash->{job} = $c->stash->{build}->job;
     $c->stash->{runcommandlogs} = [$c->stash->{build}->runcommandlogs->search({}, {order_by => ["id DESC"]})];
+
+    $c->stash->{runcommandlogProblem} = undef;
+    if ($c->stash->{job} =~ qr/^runCommandHook\..*/) {
+        if (!$c->config->{dynamicruncommand}->{enable}) {
+            $c->stash->{runcommandlogProblem} = "disabled-server";
+        } elsif (!$c->stash->{project}->enable_dynamic_run_command) {
+            $c->stash->{runcommandlogProblem} = "disabled-project";
+        } elsif (!$c->stash->{jobset}->enable_dynamic_run_command) {
+            $c->stash->{runcommandlogProblem} = "disabled-jobset";
+        }
+    }
 }
 
 
diff --git a/src/root/build.tt b/src/root/build.tt
index 0848da4a..027ce3e4 100644
--- a/src/root/build.tt
+++ b/src/root/build.tt
@@ -149,7 +149,7 @@ END;
   [% IF build.dependents %]<li class="nav-item"><a class="nav-link" href="#tabs-usedby" data-toggle="tab">Used By</a></li>[% END%]
   [% IF drvAvailable %]<li class="nav-item"><a class="nav-link" href="#tabs-build-deps" data-toggle="tab">Build Dependencies</a></li>[% END %]
   [% IF localStore && available %]<li class="nav-item"><a class="nav-link" href="#tabs-runtime-deps" data-toggle="tab">Runtime Dependencies</a></li>[% END %]
-  [% IF runcommandlogs.size() > 0 %]<li class="nav-item"><a class="nav-link" href="#tabs-runcommandlogs" data-toggle="tab">RunCommand Logs</a></li>[% END %]
+  [% IF runcommandlogProblem || runcommandlogs.size() > 0 %]<li class="nav-item"><a class="nav-link" href="#tabs-runcommandlogs" data-toggle="tab">RunCommand Logs[% IF runcommandlogProblem %] <span class="badge badge-warning">Disabled</span>[% END %]</a></li>[% END %]
 </ul>
 
 <div id="generic-tabs" class="tab-content">
@@ -489,6 +489,19 @@ END;
   [% END %]
 
   <div id="tabs-runcommandlogs" class="tab-pane">
+    [% IF runcommandlogProblem %]
+      <div class="alert alert-warning" role="alert">
+        [% IF runcommandlogProblem == "disabled-server" %]
+          This server does not enable Dynamic RunCommand support.
+        [% ELSIF runcommandlogProblem == "disabled-project" %]
+          This project does not enable Dynamic RunCommand support.
+        [% ELSIF runcommandlogProblem == "disabled-jobset" %]
+          This jobset does not enable Dynamic RunCommand support.
+        [% ELSE %]
+          Dynamic RunCommand is not enabled: [% runcommandlogProblem %].
+        [% END %]
+      </div>
+    [% END %]
     <div class="d-flex flex-column">
     [% FOREACH runcommandlog IN runcommandlogs %]
       <div class="p-2 border-bottom">

From 1c846765273018ff9b97edbfcaf9e5884fd9e140 Mon Sep 17 00:00:00 2001
From: ajs124 <git@ajs124.de>
Date: Mon, 10 May 2021 14:32:24 +0200
Subject: [PATCH 35/76] Fit more content on screen

---
 src/root/static/css/hydra.css | 22 +++++++++++++++++++++-
 1 file changed, 21 insertions(+), 1 deletion(-)

diff --git a/src/root/static/css/hydra.css b/src/root/static/css/hydra.css
index 475d61c2..53304b19 100644
--- a/src/root/static/css/hydra.css
+++ b/src/root/static/css/hydra.css
@@ -1,5 +1,5 @@
 div.skip-topbar {
-    padding-top: 40px;
+    padding-top: 20px;
     margin-bottom: 1.5em;
 }
 
@@ -146,6 +146,26 @@ td.step-status span.warn {
     padding-top: 1.5rem;
 }
 
+.container {
+    max-width: 80%;
+}
+
+.tab-content {
+    margin-right: 0 !important;
+}
+
+body {
+    line-height: 1;
+}
+
+.navbar-nav {
+    line-height: 1.5;
+}
+
+.dropdown-item {
+    line-height: 1.5;
+}
+
 @media (prefers-color-scheme: dark) {
     /* Prevent some flickering */
     html {

From f1f2fc742700c6f80f87758687afedefd9d08ac6 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 7 Mar 2022 19:07:06 +0000
Subject: [PATCH 36/76] build(deps): bump actions/checkout from 2 to 3

Bumps [actions/checkout](https://github.com/actions/checkout) from 2 to 3.
- [Release notes](https://github.com/actions/checkout/releases)
- [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md)
- [Commits](https://github.com/actions/checkout/compare/v2...v3)

---
updated-dependencies:
- dependency-name: actions/checkout
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 .github/workflows/test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 7d98e3ff..8d32f581 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -6,7 +6,7 @@ jobs:
   tests:
     runs-on: ubuntu-18.04
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
       with:
         fetch-depth: 0
     - uses: cachix/install-nix-action@v12

From a179f0be610f37635c981de441c73d246b88f1fb Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 11 Apr 2022 15:26:03 +0000
Subject: [PATCH 37/76] build(deps): bump cachix/install-nix-action from 16 to
 17

Bumps [cachix/install-nix-action](https://github.com/cachix/install-nix-action) from 16 to 17.
- [Release notes](https://github.com/cachix/install-nix-action/releases)
- [Commits](https://github.com/cachix/install-nix-action/compare/v16...v17)

---
updated-dependencies:
- dependency-name: cachix/install-nix-action
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 .github/workflows/test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 3ba4aba6..0f5f43da 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -9,6 +9,6 @@ jobs:
     - uses: actions/checkout@v3
       with:
         fetch-depth: 0
-    - uses: cachix/install-nix-action@v16
+    - uses: cachix/install-nix-action@v17
     #- run: nix flake check
     - run: nix-build -A checks.x86_64-linux.build -A checks.x86_64-linux.validate-openapi

From cb4fa0000ff4f884bb0064243d99ba4133ee0ae7 Mon Sep 17 00:00:00 2001
From: Kayla Firestack <dev@kaylafire.me>
Date: Thu, 14 Apr 2022 11:03:10 -0400
Subject: [PATCH 38/76] fix(hydra-eval-jobs.cc): add function to report pid
 status

---
 src/hydra-eval-jobs/hydra-eval-jobs.cc | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/src/hydra-eval-jobs/hydra-eval-jobs.cc b/src/hydra-eval-jobs/hydra-eval-jobs.cc
index 7485b297..bbc55a2b 100644
--- a/src/hydra-eval-jobs/hydra-eval-jobs.cc
+++ b/src/hydra-eval-jobs/hydra-eval-jobs.cc
@@ -25,6 +25,28 @@
 
 #include <nlohmann/json.hpp>
 
+void check_pid_status_quick(pid_t check_pid) {
+    // Only check 'initialized' and known PID's
+    if (check_pid <= 0) { return; }
+
+    int wstatus = 0;
+    pid_t pid = waitpid(check_pid, &wstatus, WNOHANG);
+    // -1 = failiure, WNOHANG: 0 = no change
+    if (pid <= 0) { return; }
+
+    std::cerr << "child process (" << pid << ") ";
+
+    if (WIFEXITED(wstatus)) {
+        std::cerr << "exited with status=" << WEXITSTATUS(wstatus) << std::endl;
+    } else if (WIFSIGNALED(wstatus)) {
+        std::cerr << "killed by signal=" << WTERMSIG(wstatus) << std::endl;
+    } else if (WIFSTOPPED(wstatus)) {
+        std::cerr << "stopped by signal=" << WSTOPSIG(wstatus) << std::endl;
+    } else if (WIFCONTINUED(wstatus)) {
+        std::cerr << "continued" << std::endl;
+    }
+}
+
 using namespace nix;
 
 static Path gcRootsDir;

From 62cdbc41389757322ddd0255038de2d6c57d197d Mon Sep 17 00:00:00 2001
From: Kayla Firestack <dev@kaylafire.me>
Date: Thu, 14 Apr 2022 11:18:29 -0400
Subject: [PATCH 39/76] feat(hydra-eval-jobs.cc): add
 check_pid_status_nonblocking to catch handler

---
 src/hydra-eval-jobs/hydra-eval-jobs.cc | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/hydra-eval-jobs/hydra-eval-jobs.cc b/src/hydra-eval-jobs/hydra-eval-jobs.cc
index bbc55a2b..f1cc1434 100644
--- a/src/hydra-eval-jobs/hydra-eval-jobs.cc
+++ b/src/hydra-eval-jobs/hydra-eval-jobs.cc
@@ -25,7 +25,7 @@
 
 #include <nlohmann/json.hpp>
 
-void check_pid_status_quick(pid_t check_pid) {
+void check_pid_status_nonblocking(pid_t check_pid) {
     // Only check 'initialized' and known PID's
     if (check_pid <= 0) { return; }
 
@@ -333,8 +333,8 @@ int main(int argc, char * * argv)
         /* Start a handler thread per worker process. */
         auto handler = [&]()
         {
+            pid_t pid = -1;
             try {
-                pid_t pid = -1;
                 AutoCloseFD from, to;
 
                 while (true) {
@@ -436,6 +436,7 @@ int main(int argc, char * * argv)
                     }
                 }
             } catch (...) {
+                check_pid_status_nonblocking(pid);
                 auto state(state_.lock());
                 state->exc = std::current_exception();
                 wakeup.notify_all();

From 2cdd7974de5e8ed2360caabe586510f0b512feec Mon Sep 17 00:00:00 2001
From: Kayla Firestack <dev@kaylafire.me>
Date: Fri, 29 Apr 2022 13:06:16 -0400
Subject: [PATCH 40/76] fix(hydra-eval-jobs): fix typo

---
 src/hydra-eval-jobs/hydra-eval-jobs.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/hydra-eval-jobs/hydra-eval-jobs.cc b/src/hydra-eval-jobs/hydra-eval-jobs.cc
index f1cc1434..918bd451 100644
--- a/src/hydra-eval-jobs/hydra-eval-jobs.cc
+++ b/src/hydra-eval-jobs/hydra-eval-jobs.cc
@@ -31,7 +31,7 @@ void check_pid_status_nonblocking(pid_t check_pid) {
 
     int wstatus = 0;
     pid_t pid = waitpid(check_pid, &wstatus, WNOHANG);
-    // -1 = failiure, WNOHANG: 0 = no change
+    // -1 = failure, WNOHANG: 0 = no change
     if (pid <= 0) { return; }
 
     std::cerr << "child process (" << pid << ") ";

From 90769ab5adcb6191e149c65aec72643c89b2a233 Mon Sep 17 00:00:00 2001
From: Kayla Firestack <dev@kaylafire.me>
Date: Mon, 2 May 2022 13:49:32 -0400
Subject: [PATCH 41/76] feat(t/jobs): add test job to cause an OOM

---
 t/jobs/oom.nix | 3 +++
 1 file changed, 3 insertions(+)
 create mode 100644 t/jobs/oom.nix

diff --git a/t/jobs/oom.nix b/t/jobs/oom.nix
new file mode 100644
index 00000000..abbd0c0d
--- /dev/null
+++ b/t/jobs/oom.nix
@@ -0,0 +1,3 @@
+{
+  oom = builtins.readFile "/dev/zero";
+}

From 2c909c038fad7cd6107706915e819a12e88fe425 Mon Sep 17 00:00:00 2001
From: Kayla Firestack <dev@kaylafire.me>
Date: Mon, 2 May 2022 13:50:57 -0400
Subject: [PATCH 42/76] feat(t/evaluator/hydra-eval-jobs): add basic evaluation
 test for hydra-eval-jobs

---
 t/evaluator/evaluate-oom-job.t | 25 +++++++++++++++++++++++++
 1 file changed, 25 insertions(+)
 create mode 100644 t/evaluator/evaluate-oom-job.t

diff --git a/t/evaluator/evaluate-oom-job.t b/t/evaluator/evaluate-oom-job.t
new file mode 100644
index 00000000..dd494c03
--- /dev/null
+++ b/t/evaluator/evaluate-oom-job.t
@@ -0,0 +1,25 @@
+use strict;
+use warnings;
+use Setup;
+use Test2::V0;
+use Hydra::Helper::Exec;
+
+my ($res, $stdout, $stderr) = captureStdoutStderr(60,
+    (
+       "systemd-run", "--user", "--collect", "--scope", "--property", "MemoryMax=25M", "--",
+       "hydra-eval-jobs",
+         "-I", "/dev/zero",
+         "-I", "./t/jobs",
+         "./t/jobs/oom.nix"
+   )
+);
+
+isnt($res, 0, "hydra-eval-jobs exits non-zero");
+ok(utf8::decode($stderr), "Stderr output is UTF8-clean");
+like(
+    $stderr,
+    qr/^child process \(\d+?\) killed by signal=9$/m,
+    "The stderr record includes a relevant error message"
+);
+
+done_testing;

From 01ec004108177c0077f83ac824f92672fd871678 Mon Sep 17 00:00:00 2001
From: Kayla Firestack <dev@kaylafire.me>
Date: Mon, 2 May 2022 14:08:50 -0400
Subject: [PATCH 43/76] feat(t/evaluator/evaluate-oom-job):  skip test if
 systemd-run is not present

---
 t/evaluator/evaluate-oom-job.t | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/t/evaluator/evaluate-oom-job.t b/t/evaluator/evaluate-oom-job.t
index dd494c03..bf8f214f 100644
--- a/t/evaluator/evaluate-oom-job.t
+++ b/t/evaluator/evaluate-oom-job.t
@@ -4,6 +4,14 @@ use Setup;
 use Test2::V0;
 use Hydra::Helper::Exec;
 
+my ($systemdrRes) = captureStdoutStderr(3, (
+   "systemd-run", "--user", "--collect", "--scope", "--property", "MemoryMax=25M", "--",
+   "true"
+));
+
+skip_all("systemd-run does not work in this environment") if($systemdrRes != 0);
+
+
 my ($res, $stdout, $stderr) = captureStdoutStderr(60,
     (
        "systemd-run", "--user", "--collect", "--scope", "--property", "MemoryMax=25M", "--",

From e917d9e54662da32d5b4d9a1ff950a05cb7da5c1 Mon Sep 17 00:00:00 2001
From: Kayla Firestack <dev@kaylafire.me>
Date: Mon, 2 May 2022 14:40:13 -0400
Subject: [PATCH 44/76] fix(t/evaluator/evaluate-oom): convert systemd-run
 presence check to eval, fix indentaion, show relationships between flags and
 commands with indentation

---
 t/evaluator/evaluate-oom-job.t | 49 +++++++++++++++++++++-------------
 1 file changed, 30 insertions(+), 19 deletions(-)

diff --git a/t/evaluator/evaluate-oom-job.t b/t/evaluator/evaluate-oom-job.t
index bf8f214f..8d2264c0 100644
--- a/t/evaluator/evaluate-oom-job.t
+++ b/t/evaluator/evaluate-oom-job.t
@@ -4,30 +4,41 @@ use Setup;
 use Test2::V0;
 use Hydra::Helper::Exec;
 
-my ($systemdrRes) = captureStdoutStderr(3, (
-   "systemd-run", "--user", "--collect", "--scope", "--property", "MemoryMax=25M", "--",
-   "true"
+eval {
+  captureStdoutStderr(3, (
+    "systemd-run",
+      "--user",
+      "--collect",
+      "--scope",
+      "--property",
+        "MemoryMax=25M",
+    "--",
+    "true"
+  ));
+} or do {
+  skip_all("systemd-run does not work in this environment");
+};
+
+my ($res, $stdout, $stderr) = captureStdoutStderr(60, (
+  "systemd-run",
+    "--user",
+    "--collect",
+    "--scope",
+    "--property",
+      "MemoryMax=25M",
+  "--",
+  "hydra-eval-jobs",
+    "-I", "/dev/zero",
+    "-I", "./t/jobs",
+    "./t/jobs/oom.nix"
 ));
 
-skip_all("systemd-run does not work in this environment") if($systemdrRes != 0);
-
-
-my ($res, $stdout, $stderr) = captureStdoutStderr(60,
-    (
-       "systemd-run", "--user", "--collect", "--scope", "--property", "MemoryMax=25M", "--",
-       "hydra-eval-jobs",
-         "-I", "/dev/zero",
-         "-I", "./t/jobs",
-         "./t/jobs/oom.nix"
-   )
-);
-
 isnt($res, 0, "hydra-eval-jobs exits non-zero");
 ok(utf8::decode($stderr), "Stderr output is UTF8-clean");
 like(
-    $stderr,
-    qr/^child process \(\d+?\) killed by signal=9$/m,
-    "The stderr record includes a relevant error message"
+  $stderr,
+  qr/^child process \(\d+?\) killed by signal=9$/m,
+  "The stderr record includes a relevant error message"
 );
 
 done_testing;

From 013a1dcabc7d4bf06340ab23d6dbfd14a783fdcb Mon Sep 17 00:00:00 2001
From: Kayla Firestack <dev@kaylafire.me>
Date: Mon, 2 May 2022 15:13:59 -0400
Subject: [PATCH 45/76] fix(t/evaluator/evaluate-oom): check that the exit
 value of the `systemd-run` check is zero. Rework skip messages

---
 t/evaluator/evaluate-oom-job.t | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/t/evaluator/evaluate-oom-job.t b/t/evaluator/evaluate-oom-job.t
index 8d2264c0..7a527825 100644
--- a/t/evaluator/evaluate-oom-job.t
+++ b/t/evaluator/evaluate-oom-job.t
@@ -4,8 +4,9 @@ use Setup;
 use Test2::V0;
 use Hydra::Helper::Exec;
 
+my $sd_res;
 eval {
-  captureStdoutStderr(3, (
+  ($sd_res) = captureStdoutStderr(3, (
     "systemd-run",
       "--user",
       "--collect",
@@ -16,8 +17,9 @@ eval {
     "true"
   ));
 } or do {
-  skip_all("systemd-run does not work in this environment");
+  skip_all("`systemd-run` failed when invoked in this environment");
 };
+if ($sd_res != 0) { skip_all("`systemd-run` returned non-zero when executing `true` (expected 0)"); }
 
 my ($res, $stdout, $stderr) = captureStdoutStderr(60, (
   "systemd-run",

From 87f610e7c18b85c77098b1530b1d015b935710ab Mon Sep 17 00:00:00 2001
From: Kayla Firestack <dev@kaylafire.me>
Date: Mon, 2 May 2022 15:14:46 -0400
Subject: [PATCH 46/76] fix(t/evaluator/evaluate-oom): use `test_context` to
 get path to ./t/jobs instead of relative paths

---
 t/evaluator/evaluate-oom-job.t | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/t/evaluator/evaluate-oom-job.t b/t/evaluator/evaluate-oom-job.t
index 7a527825..8c8c5f60 100644
--- a/t/evaluator/evaluate-oom-job.t
+++ b/t/evaluator/evaluate-oom-job.t
@@ -21,6 +21,8 @@ eval {
 };
 if ($sd_res != 0) { skip_all("`systemd-run` returned non-zero when executing `true` (expected 0)"); }
 
+my $ctx = test_context();
+
 my ($res, $stdout, $stderr) = captureStdoutStderr(60, (
   "systemd-run",
     "--user",
@@ -31,11 +33,11 @@ my ($res, $stdout, $stderr) = captureStdoutStderr(60, (
   "--",
   "hydra-eval-jobs",
     "-I", "/dev/zero",
-    "-I", "./t/jobs",
-    "./t/jobs/oom.nix"
+    "-I", $ctx->jobsdir,
+    ($ctx->jobsdir . "/oom.nix")
 ));
 
-isnt($res, 0, "hydra-eval-jobs exits non-zero");
+isnt($res, 0, "`hydra-eval-jobs` exits non-zero");
 ok(utf8::decode($stderr), "Stderr output is UTF8-clean");
 like(
   $stderr,

From 065039beba4aa8fc998762b145aa6176daa44522 Mon Sep 17 00:00:00 2001
From: Kayla Firestack <dev@kaylafire.me>
Date: Mon, 2 May 2022 15:26:26 -0400
Subject: [PATCH 47/76] feat(t/evaluator/evaluate-oom): comment intentions

---
 t/evaluator/evaluate-oom-job.t | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/t/evaluator/evaluate-oom-job.t b/t/evaluator/evaluate-oom-job.t
index 8c8c5f60..6c17d4e4 100644
--- a/t/evaluator/evaluate-oom-job.t
+++ b/t/evaluator/evaluate-oom-job.t
@@ -4,6 +4,10 @@ use Setup;
 use Test2::V0;
 use Hydra::Helper::Exec;
 
+# Ensure that `systemd-run` is
+# - Available in the PATH/envionment
+# - Accessable to the user executing it
+# - Capable of using the command switches we use in our test
 my $sd_res;
 eval {
   ($sd_res) = captureStdoutStderr(3, (
@@ -17,12 +21,21 @@ eval {
     "true"
   ));
 } or do {
+  # The command failed to execute, likely because `systemd-run` is not present
+  # in `PATH`
   skip_all("`systemd-run` failed when invoked in this environment");
 };
-if ($sd_res != 0) { skip_all("`systemd-run` returned non-zero when executing `true` (expected 0)"); }
+if ($sd_res != 0) {
+  # `systemd-run` executed but `sytemd-run` failed to call `true` and return
+  # successfully
+  skip_all("`systemd-run` returned non-zero when executing `true` (expected 0)");
+}
 
 my $ctx = test_context();
 
+# Contain the memory usage to 25 MegaBytes using `systemd-run`
+# Run `hydra-eval-jobs` on test job that will purposefully consume all memory
+# available
 my ($res, $stdout, $stderr) = captureStdoutStderr(60, (
   "systemd-run",
     "--user",
@@ -41,6 +54,8 @@ isnt($res, 0, "`hydra-eval-jobs` exits non-zero");
 ok(utf8::decode($stderr), "Stderr output is UTF8-clean");
 like(
   $stderr,
+  # Assert error log contains messages added in PR
+  # https://github.com/NixOS/hydra/pull/1203
   qr/^child process \(\d+?\) killed by signal=9$/m,
   "The stderr record includes a relevant error message"
 );

From 3c71be5b5b9b8fd4739c925ab46642f6121ec218 Mon Sep 17 00:00:00 2001
From: Ulrik Strid <ulrik.strid@outlook.com>
Date: Wed, 18 May 2022 08:14:00 +0200
Subject: [PATCH 48/76] GithubPulls: Don't fail on missing `Link`

---
 src/lib/Hydra/Plugin/GithubPulls.pm | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/lib/Hydra/Plugin/GithubPulls.pm b/src/lib/Hydra/Plugin/GithubPulls.pm
index db0e8d25..9d8412c3 100644
--- a/src/lib/Hydra/Plugin/GithubPulls.pm
+++ b/src/lib/Hydra/Plugin/GithubPulls.pm
@@ -30,7 +30,7 @@ sub _iterate {
         $pulls->{$pull->{number}} = $pull;
     }
     # TODO Make Link header parsing more robust!!!
-    my @links = split ',', $res->header("Link");
+    my @links = split ',', ($res->header("Link") // "");
     my $next = "";
     foreach my $link (@links) {
         my ($url, $rel) = split ";", $link;

From a8b590014b4b3f59aaf8eec3465f600b5899b2fd Mon Sep 17 00:00:00 2001
From: Maximilian Bosch <maximilian@mbosch.me>
Date: Sun, 22 May 2022 14:14:14 +0200
Subject: [PATCH 49/76] Fix email notifications for jobsets w/git-inputs

I started to wonder quite recently why Hydra doesn't send email
notifications anymore to me. I saw the following issue in the log of
`hydra-notify.service`:

    May 22 11:57:29 hydra 9bik0bxyxbrklhx6lqwifd6af8kj84va-hydra-notify[1887289]: fatal: unsafe repository ('/var/lib/hydra/scm/git/3e70c16c266ef70dc4198705a688acccf71e932878f178277c9ac47d133cc663' is owned by someone else)
    May 22 11:57:29 hydra 9bik0bxyxbrklhx6lqwifd6af8kj84va-hydra-notify[1887289]: To add an exception for this directory, call:
    May 22 11:57:29 hydra 9bik0bxyxbrklhx6lqwifd6af8kj84va-hydra-notify[1887289]:         git config --global --add safe.directory /var/lib/hydra/scm/git/3e70c16c266ef70dc4198705a688acccf71e932878f178277c9ac47d133cc663
    May 22 11:57:29 hydra 9bik0bxyxbrklhx6lqwifd6af8kj84va-hydra-notify[1886654]: error running build_finished hooks: command `git log --pretty=format:%H%x09%an%x09%ae%x09%at b0c30a7557685d25a8ab3f34fdb775e66db0bc4c..eaf28389fcebc2beca13a802f79b2cca6e9ca309 --git-dir=.git' failed with e>

This is also a problem because of Git's fix for CVE-2022-24765[1], so I
applied the same fix as for Nix[2], by using `--git-dir` which skips the
code-path for the ownership-check[3].

[1] https://lore.kernel.org/git/xmqqv8veb5i6.fsf@gitster.g/
[2] https://github.com/NixOS/nix/pull/6440
[3] To quote `git(1)`:
    > Specifying the location of the ".git" directory using this option
    > (or GIT_DIR environment variable) turns off the repository
    > discovery that tries to find a directory with ".git" subdirectory
---
 src/lib/Hydra/Plugin/GitInput.pm | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/lib/Hydra/Plugin/GitInput.pm b/src/lib/Hydra/Plugin/GitInput.pm
index aca35c30..e5fc7de9 100644
--- a/src/lib/Hydra/Plugin/GitInput.pm
+++ b/src/lib/Hydra/Plugin/GitInput.pm
@@ -261,7 +261,7 @@ sub getCommits {
 
     my $clonePath = getSCMCacheDir . "/git/" . sha256_hex($uri);
 
-    my $out = grab(cmd => ["git", "log", "--pretty=format:%H%x09%an%x09%ae%x09%at", "$rev1..$rev2"], dir => $clonePath);
+    my $out = grab(cmd => ["git", "--git-dir=.git", "log", "--pretty=format:%H%x09%an%x09%ae%x09%at", "$rev1..$rev2"], dir => $clonePath);
 
     my $res = [];
     foreach my $line (split /\n/, $out) {

From 5c01800fbe49939c53566457047283eb5c93f51b Mon Sep 17 00:00:00 2001
From: Maximilian Bosch <maximilian@mbosch.me>
Date: Thu, 16 Jun 2022 14:54:57 +0200
Subject: [PATCH 50/76] flake: Update Nix to 2.9.1

NOTE: I'm well-aware that we have to be careful with this to avoid new
regressions on hydra.nixos.org, so this should only be merged after
extensive testing from more people.

Motivation: I updated Nix in my deployment to 2.9.1 and decided to also
update Hydra in one go (and compile it against the newer Nix). Given
that this also updates the C++ code in `hydra-{queue-runner,eval-jobs}`
this patch might become useful in the future though.
---
 flake.lock                              | 12 +++++++-----
 flake.nix                               |  2 +-
 src/hydra-eval-jobs/Makefile.am         |  2 +-
 src/hydra-eval-jobs/hydra-eval-jobs.cc  | 10 +++++-----
 src/hydra-queue-runner/queue-monitor.cc |  2 +-
 5 files changed, 15 insertions(+), 13 deletions(-)

diff --git a/flake.lock b/flake.lock
index 5c726a24..1310c53d 100644
--- a/flake.lock
+++ b/flake.lock
@@ -39,16 +39,18 @@
         "nixpkgs-regression": "nixpkgs-regression"
       },
       "locked": {
-        "lastModified": 1649172203,
-        "narHash": "sha256-Q3nYaXqbseDOvZrlePKeIrx0/KzqyrtNpxHIUbtFHuI=",
+        "lastModified": 1654014617,
+        "narHash": "sha256-qNL3lQPBsnStkru3j1ajN/H+knXI+X3dku8/dBfSw3g=",
         "owner": "NixOS",
         "repo": "nix",
-        "rev": "5fe4fe823c193cbb7bfa05a468de91eeab09058d",
+        "rev": "624e38aa43f304fbb78b4779172809add042b513",
         "type": "github"
       },
       "original": {
-        "id": "nix",
-        "type": "indirect"
+        "owner": "NixOS",
+        "ref": "2.9.1",
+        "repo": "nix",
+        "type": "github"
       }
     },
     "nixpkgs": {
diff --git a/flake.nix b/flake.nix
index 01b0c988..794555ec 100644
--- a/flake.nix
+++ b/flake.nix
@@ -5,7 +5,7 @@
   # even 2.7.0's Nixpkgs pin).
   inputs.newNixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable-small";
   inputs.nixpkgs.follows = "nix/nixpkgs";
-  #inputs.nix.url = github:NixOS/nix/2.7.0;
+  inputs.nix.url = github:NixOS/nix/2.9.1;
 
   outputs = { self, newNixpkgs, nixpkgs, nix }:
     let
diff --git a/src/hydra-eval-jobs/Makefile.am b/src/hydra-eval-jobs/Makefile.am
index 7a4e9c91..90742a30 100644
--- a/src/hydra-eval-jobs/Makefile.am
+++ b/src/hydra-eval-jobs/Makefile.am
@@ -1,5 +1,5 @@
 bin_PROGRAMS = hydra-eval-jobs
 
 hydra_eval_jobs_SOURCES = hydra-eval-jobs.cc
-hydra_eval_jobs_LDADD = $(NIX_LIBS)
+hydra_eval_jobs_LDADD = $(NIX_LIBS) -lnixcmd
 hydra_eval_jobs_CXXFLAGS = $(NIX_CFLAGS) -I ../libhydra
diff --git a/src/hydra-eval-jobs/hydra-eval-jobs.cc b/src/hydra-eval-jobs/hydra-eval-jobs.cc
index 918bd451..18d39620 100644
--- a/src/hydra-eval-jobs/hydra-eval-jobs.cc
+++ b/src/hydra-eval-jobs/hydra-eval-jobs.cc
@@ -197,21 +197,21 @@ static void worker(
 
                 /* If this is an aggregate, then get its constituents. */
                 auto a = v->attrs->get(state.symbols.create("_hydraAggregate"));
-                if (a && state.forceBool(*a->value, *a->pos)) {
+                if (a && state.forceBool(*a->value, a->pos)) {
                     auto a = v->attrs->get(state.symbols.create("constituents"));
                     if (!a)
                         throw EvalError("derivation must have a ‘constituents’ attribute");
 
 
                     PathSet context;
-                    state.coerceToString(*a->pos, *a->value, context, true, false);
+                    state.coerceToString(a->pos, *a->value, context, true, false);
                     for (auto & i : context)
                         if (i.at(0) == '!') {
                             size_t index = i.find("!", 1);
                             job["constituents"].push_back(std::string(i, index + 1));
                         }
 
-                    state.forceList(*a->value, *a->pos);
+                    state.forceList(*a->value, a->pos);
                     for (unsigned int n = 0; n < a->value->listSize(); ++n) {
                         auto v = a->value->listElems()[n];
                         state.forceValue(*v, noPos);
@@ -243,8 +243,8 @@ static void worker(
             else if (v->type() == nAttrs) {
                 auto attrs = nlohmann::json::array();
                 StringSet ss;
-                for (auto & i : v->attrs->lexicographicOrder()) {
-                    std::string name(i->name);
+                for (auto & i : v->attrs->lexicographicOrder(state.symbols)) {
+                    std::string name(state.symbols[i->name]);
                     if (name.find('.') != std::string::npos || name.find(' ') != std::string::npos) {
                         printError("skipping job with illegal name '%s'", name);
                         continue;
diff --git a/src/hydra-queue-runner/queue-monitor.cc b/src/hydra-queue-runner/queue-monitor.cc
index 3bde0d99..12d55b79 100644
--- a/src/hydra-queue-runner/queue-monitor.cc
+++ b/src/hydra-queue-runner/queue-monitor.cc
@@ -460,7 +460,7 @@ Step::ptr State::createStep(ref<Store> destStore,
     step->parsedDrv = std::make_unique<ParsedDerivation>(drvPath, *step->drv);
 
     step->preferLocalBuild = step->parsedDrv->willBuildLocally(*localStore);
-    step->isDeterministic = get(step->drv->env, "isDetermistic").value_or("0") == "1";
+    step->isDeterministic = getOr(step->drv->env, "isDetermistic", "0") == "1";
 
     step->systemType = step->drv->platform;
     {

From 183f58ca9f10a0a6a18e42c75168145a1de864df Mon Sep 17 00:00:00 2001
From: Sandro <sandro.jaeckel@gmail.com>
Date: Thu, 16 Jun 2022 22:48:39 +0200
Subject: [PATCH 51/76] Remove url literal

---
 flake.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/flake.nix b/flake.nix
index 794555ec..2e891364 100644
--- a/flake.nix
+++ b/flake.nix
@@ -5,7 +5,7 @@
   # even 2.7.0's Nixpkgs pin).
   inputs.newNixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable-small";
   inputs.nixpkgs.follows = "nix/nixpkgs";
-  inputs.nix.url = github:NixOS/nix/2.9.1;
+  inputs.nix.url = "github:NixOS/nix/2.9.1";
 
   outputs = { self, newNixpkgs, nixpkgs, nix }:
     let

From bab671124df0bc43a521d708bf86eece908d33af Mon Sep 17 00:00:00 2001
From: ajs124 <git@ajs124.de>
Date: Thu, 30 Jun 2022 00:24:09 +0200
Subject: [PATCH 52/76] replace nix cat-store with nix store cat

the former was deprecated in favor of the latter
---
 src/lib/Hydra/Controller/Build.pm | 4 ++--
 src/lib/Hydra/Helper/Nix.pm       | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/lib/Hydra/Controller/Build.pm b/src/lib/Hydra/Controller/Build.pm
index 552f31af..c7811c62 100644
--- a/src/lib/Hydra/Controller/Build.pm
+++ b/src/lib/Hydra/Controller/Build.pm
@@ -234,7 +234,7 @@ sub serveFile {
     elsif ($ls->{type} eq "regular") {
 
         $c->stash->{'plain'} = { data => grab(cmd => ["nix", "--experimental-features", "nix-command",
-                                                      "cat-store", "--store", getStoreUri(), "$path"]) };
+                                                      "store", "cat", "--store", getStoreUri(), "$path"]) };
 
         # Detect MIME type. Borrowed from Catalyst::Plugin::Static::Simple.
         my $type = "text/plain";
@@ -366,7 +366,7 @@ sub contents : Chained('buildChain') PathPart Args(1) {
 
     # FIXME: don't use shell invocations below.
 
-    # FIXME: use nix cat-store
+    # FIXME: use nix store cat
 
     my $res;
 
diff --git a/src/lib/Hydra/Helper/Nix.pm b/src/lib/Hydra/Helper/Nix.pm
index 514fb439..71a8a7d7 100644
--- a/src/lib/Hydra/Helper/Nix.pm
+++ b/src/lib/Hydra/Helper/Nix.pm
@@ -537,7 +537,7 @@ sub getStoreUri {
 sub readNixFile {
     my ($path) = @_;
     return grab(cmd => ["nix", "--experimental-features", "nix-command",
-                        "cat-store", "--store", getStoreUri(), "$path"]);
+                        "store", "cat", "--store", getStoreUri(), "$path"]);
 }
 
 

From bb1f04ed8669cc012fc552a40a376f3b93228833 Mon Sep 17 00:00:00 2001
From: ajs124 <git@ajs124.de>
Date: Thu, 30 Jun 2022 00:32:31 +0200
Subject: [PATCH 53/76] AddBuilds: fix declarative jobsets with dynamic
 runcommand enabled

$project->{enable_dynamic_run_command} is undefined
---
 src/lib/Hydra/Helper/AddBuilds.pm     |  2 +-
 t/Helper/AddBuilds/dynamic-disabled.t | 36 +++++++++++++++++++++------
 t/Helper/AddBuilds/dynamic-enabled.t  | 36 +++++++++++++++++++++------
 3 files changed, 59 insertions(+), 15 deletions(-)

diff --git a/src/lib/Hydra/Helper/AddBuilds.pm b/src/lib/Hydra/Helper/AddBuilds.pm
index 9e3ddfd2..a6373be5 100644
--- a/src/lib/Hydra/Helper/AddBuilds.pm
+++ b/src/lib/Hydra/Helper/AddBuilds.pm
@@ -67,7 +67,7 @@ sub validateDeclarativeJobset {
     my $enable_dynamic_run_command = defined $update{enable_dynamic_run_command} ? 1 : 0;
     if ($enable_dynamic_run_command
         && !($config->{dynamicruncommand}->{enable}
-            && $project->{enable_dynamic_run_command}))
+            && $project->enable_dynamic_run_command))
     {
         die "Dynamic RunCommand is not enabled by the server or the parent project.";
     }
diff --git a/t/Helper/AddBuilds/dynamic-disabled.t b/t/Helper/AddBuilds/dynamic-disabled.t
index 0507b03e..0c91f382 100644
--- a/t/Helper/AddBuilds/dynamic-disabled.t
+++ b/t/Helper/AddBuilds/dynamic-disabled.t
@@ -6,11 +6,31 @@ use Test2::V0;
 require Catalyst::Test;
 use HTTP::Request::Common qw(POST PUT GET DELETE);
 use JSON::MaybeXS qw(decode_json encode_json);
-use Hydra::Helper::AddBuilds qw(validateDeclarativeJobset);
-use Hydra::Helper::Nix qw(getHydraConfig);
 
 my $ctx = test_context();
 
+Catalyst::Test->import('Hydra');
+
+my $db = Hydra::Model::DB->new;
+hydra_setup($db);
+
+my $user = $db->resultset('Users')->create({ username => 'alice', emailaddress => 'root@invalid.org', password => '!' });
+$user->setPassword('foobar');
+$user->userroles->update_or_create({ role => 'admin' });
+
+my $project_with_dynamic_run_command = $db->resultset('Projects')->create({
+    name => 'tests_with_dynamic_runcommand',
+    displayname => 'Tests with dynamic runcommand',
+    owner => 'alice',
+    enable_dynamic_run_command => 1,
+});
+my $project_without_dynamic_run_command = $db->resultset('Projects')->create({
+    name => 'tests_without_dynamic_runcommand',
+    displayname => 'Tests without dynamic runcommand',
+    owner => 'alice',
+    enable_dynamic_run_command => 0,
+});
+
 sub makeJobsetSpec {
     my ($dynamic) = @_;
 
@@ -29,14 +49,16 @@ sub makeJobsetSpec {
 };
 
 subtest "validate declarative jobset with dynamic RunCommand disabled by server" => sub {
-    my $config = getHydraConfig();
+    my $config = Hydra::Helper::Nix->getHydraConfig();
+    require Hydra::Helper::AddBuilds;
+    Hydra::Helper::AddBuilds->import( qw(validateDeclarativeJobset) );
 
     subtest "project enabled dynamic runcommand, declarative jobset enabled dynamic runcommand" => sub {
         like(
             dies {
               validateDeclarativeJobset(
                   $config,
-                  { enable_dynamic_run_command => 1 },
+                  $project_with_dynamic_run_command,
                   "test-jobset",
                   makeJobsetSpec(JSON::MaybeXS::true),
               ),
@@ -49,7 +71,7 @@ subtest "validate declarative jobset with dynamic RunCommand disabled by server"
         ok(
             validateDeclarativeJobset(
                 $config,
-                { enable_dynamic_run_command => 1 },
+                $project_with_dynamic_run_command,
                 "test-jobset",
                 makeJobsetSpec(JSON::MaybeXS::false)
             ),
@@ -61,7 +83,7 @@ subtest "validate declarative jobset with dynamic RunCommand disabled by server"
             dies {
               validateDeclarativeJobset(
                   $config,
-                  { enable_dynamic_run_command => 0 },
+                  $project_without_dynamic_run_command,
                   "test-jobset",
                   makeJobsetSpec(JSON::MaybeXS::true),
               ),
@@ -74,7 +96,7 @@ subtest "validate declarative jobset with dynamic RunCommand disabled by server"
         ok(
             validateDeclarativeJobset(
                 $config,
-                { enable_dynamic_run_command => 0 },
+                $project_without_dynamic_run_command,
                 "test-jobset",
                 makeJobsetSpec(JSON::MaybeXS::false)
             ),
diff --git a/t/Helper/AddBuilds/dynamic-enabled.t b/t/Helper/AddBuilds/dynamic-enabled.t
index d2f5a386..46497bed 100644
--- a/t/Helper/AddBuilds/dynamic-enabled.t
+++ b/t/Helper/AddBuilds/dynamic-enabled.t
@@ -6,8 +6,6 @@ use Test2::V0;
 require Catalyst::Test;
 use HTTP::Request::Common qw(POST PUT GET DELETE);
 use JSON::MaybeXS qw(decode_json encode_json);
-use Hydra::Helper::AddBuilds qw(validateDeclarativeJobset);
-use Hydra::Helper::Nix qw(getHydraConfig);
 
 my $ctx = test_context(
     hydra_config => q|
@@ -17,6 +15,28 @@ my $ctx = test_context(
     |
 );
 
+Catalyst::Test->import('Hydra');
+
+my $db = Hydra::Model::DB->new;
+hydra_setup($db);
+
+my $user = $db->resultset('Users')->create({ username => 'alice', emailaddress => 'root@invalid.org', password => '!' });
+$user->setPassword('foobar');
+$user->userroles->update_or_create({ role => 'admin' });
+
+my $project_with_dynamic_run_command = $db->resultset('Projects')->create({
+    name => 'tests_with_dynamic_runcommand',
+    displayname => 'Tests with dynamic runcommand',
+    owner => 'alice',
+    enable_dynamic_run_command => 1,
+});
+my $project_without_dynamic_run_command = $db->resultset('Projects')->create({
+    name => 'tests_without_dynamic_runcommand',
+    displayname => 'Tests without dynamic runcommand',
+    owner => 'alice',
+    enable_dynamic_run_command => 0,
+});
+
 sub makeJobsetSpec {
     my ($dynamic) = @_;
 
@@ -35,13 +55,15 @@ sub makeJobsetSpec {
 };
 
 subtest "validate declarative jobset with dynamic RunCommand enabled by server" => sub {
-    my $config = getHydraConfig();
+    my $config = Hydra::Helper::Nix->getHydraConfig();
+    require Hydra::Helper::AddBuilds;
+    Hydra::Helper::AddBuilds->import( qw(validateDeclarativeJobset) );
 
     subtest "project enabled dynamic runcommand, declarative jobset enabled dynamic runcommand" => sub {
         ok(
             validateDeclarativeJobset(
                 $config,
-                { enable_dynamic_run_command => 1 },
+                $project_with_dynamic_run_command,
                 "test-jobset",
                 makeJobsetSpec(JSON::MaybeXS::true)
             ),
@@ -52,7 +74,7 @@ subtest "validate declarative jobset with dynamic RunCommand enabled by server"
         ok(
             validateDeclarativeJobset(
                 $config,
-                { enable_dynamic_run_command => 1 },
+                $project_with_dynamic_run_command,
                 "test-jobset",
                 makeJobsetSpec(JSON::MaybeXS::false)
             ),
@@ -64,7 +86,7 @@ subtest "validate declarative jobset with dynamic RunCommand enabled by server"
             dies {
               validateDeclarativeJobset(
                   $config,
-                  { enable_dynamic_run_command => 0 },
+                  $project_without_dynamic_run_command,
                   "test-jobset",
                   makeJobsetSpec(JSON::MaybeXS::true),
               ),
@@ -77,7 +99,7 @@ subtest "validate declarative jobset with dynamic RunCommand enabled by server"
         ok(
             validateDeclarativeJobset(
                 $config,
-                { enable_dynamic_run_command => 0 },
+                $project_without_dynamic_run_command,
                 "test-jobset",
                 makeJobsetSpec(JSON::MaybeXS::false)
             ),

From 38e033e7ceb8fecea9fe3d8c621e77d1af884948 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Janne=20He=C3=9F?= <janne@hess.ooo>
Date: Sun, 10 Jul 2022 13:31:21 +0200
Subject: [PATCH 54/76] Remove yet another URL literal

---
 shell.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/shell.nix b/shell.nix
index 9e967032..1ad58f49 100644
--- a/shell.nix
+++ b/shell.nix
@@ -1,6 +1,6 @@
 # The `default.nix` in flake-compat reads `flake.nix` and `flake.lock` from `src` and
 # returns an attribute set of the shape `{ defaultNix, shellNix }`
 
-(import (fetchTarball https://github.com/edolstra/flake-compat/archive/master.tar.gz) {
+(import (fetchTarball "https://github.com/edolstra/flake-compat/archive/master.tar.gz") {
   src = ./.;
 }).shellNix

From 8dd1daac8a9bd1a5b4d8f37969c344d5ef593c6e Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 12 Jul 2022 12:14:50 +0200
Subject: [PATCH 55/76] Update to Nix 2.10

---
 flake.nix        | 28 +++++++++++++---------------
 hydra-module.nix |  2 +-
 2 files changed, 14 insertions(+), 16 deletions(-)

diff --git a/flake.nix b/flake.nix
index 2e891364..36863913 100644
--- a/flake.nix
+++ b/flake.nix
@@ -5,7 +5,7 @@
   # even 2.7.0's Nixpkgs pin).
   inputs.newNixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable-small";
   inputs.nixpkgs.follows = "nix/nixpkgs";
-  inputs.nix.url = "github:NixOS/nix/2.9.1";
+  inputs.nix.url = "github:NixOS/nix/2.10.0";
 
   outputs = { self, newNixpkgs, nixpkgs, nix }:
     let
@@ -14,7 +14,7 @@
 
       pkgs = import nixpkgs {
         system = "x86_64-linux";
-        overlays = [ self.overlay nix.overlay ];
+        overlays = [ self.overlay nix.overlays.default ];
       };
 
       # NixOS configuration used for VM tests.
@@ -332,7 +332,7 @@
               url = "mirror://cpan/authors/id/A/AA/AAR/Net-LDAP-Server-0.43.tar.gz";
               sha256 = "0qmh3cri3fpccmwz6bhwp78yskrb3qmalzvqn0a23hqbsfs4qv6x";
             };
-            propagatedBuildInputs = with final.perlPackages; [ NetLDAP ConvertASN1 ];
+            propagatedBuildInputs = with final.perlPackages; [ perlldap ConvertASN1 ];
             meta = {
               description = "LDAP server side protocol handling";
               license = with final.lib.licenses; [ artistic1 ];
@@ -359,7 +359,7 @@
               url = "mirror://cpan/authors/id/K/KA/KARMAN/Net-LDAP-Server-Test-0.22.tar.gz";
               sha256 = "13idip7jky92v4adw60jn2gcc3zf339gsdqlnc9nnvqzbxxp285i";
             };
-            propagatedBuildInputs = with final.perlPackages; [ NetLDAP NetLDAPServer TestMore DataDump NetLDAPSID ];
+            propagatedBuildInputs = with final.perlPackages; [ perlldap NetLDAPServer DataDump NetLDAPSID ];
             meta = {
               description = "test Net::LDAP code";
               license = with final.lib.licenses; [ artistic1 ];
@@ -373,8 +373,8 @@
               url = "mirror://cpan/authors/id/I/IL/ILMARI/Catalyst-Authentication-Store-LDAP-1.016.tar.gz";
               sha256 = "0cm399vxqqf05cjgs1j5v3sk4qc6nmws5nfhf52qvpbwc4m82mq8";
             };
-            propagatedBuildInputs = with final.perlPackages; [ NetLDAP CatalystPluginAuthentication ClassAccessorFast ];
-            buildInputs = with final.perlPackages; [ TestMore TestMockObject TestException NetLDAPServerTest ];
+            propagatedBuildInputs = with final.perlPackages; [ perlldap CatalystPluginAuthentication ClassAccessor ];
+            buildInputs = with final.perlPackages; [ TestMockObject TestException NetLDAPServerTest ];
             meta = {
               description = "Authentication from an LDAP Directory";
               license = with final.lib.licenses; [ artistic1 ];
@@ -486,7 +486,6 @@
                 CatalystPluginSessionStateCookie
                 CatalystPluginSessionStoreFastMmap
                 CatalystPluginStackTrace
-                CatalystPluginUnicodeEncoding
                 CatalystTraitForRequestProxyBase
                 CatalystViewDownload
                 CatalystViewJSON
@@ -533,7 +532,6 @@
                 TermSizeAny
                 TermReadKey
                 Test2Harness
-                TestMore
                 TestPostgreSQL
                 TextDiff
                 TextTable
@@ -558,9 +556,9 @@
               libtool
               unzip
               nukeReferences
-              pkgconfig
+              pkg-config
               libpqxx
-              gitAndTools.topGit
+              top-git
               mercurial
               darcs
               subversion
@@ -585,7 +583,7 @@
             cacert
             # FIXME: foreman is broken on all nix/nixpkgs pin, up to and
             # including 2.7.0
-            newNixpkgs.legacyPackages.${final.system}.foreman
+            newNixpkgs.legacyPackages.${final.stdenv.system}.foreman
             glibcLocales
             libressl.nc
             openldap
@@ -602,11 +600,11 @@
               pixz
               gzip
               bzip2
-              lzma
+              xz
               gnutar
               unzip
               git
-              gitAndTools.topGit
+              top-git
               mercurial
               darcs
               gnused
@@ -661,7 +659,7 @@
 
           dontStrip = true;
 
-          meta.description = "Build of Hydra on ${system}";
+          meta.description = "Build of Hydra on ${final.stdenv.system}";
           passthru = { inherit perlDeps; inherit (final) nix; };
         };
       };
@@ -966,7 +964,7 @@
 
       nixosModules.hydra = {
         imports = [ ./hydra-module.nix ];
-        nixpkgs.overlays = [ self.overlay nix.overlay ];
+        nixpkgs.overlays = [ self.overlay nix.overlays.default ];
       };
 
       nixosModules.hydraTest = {
diff --git a/hydra-module.nix b/hydra-module.nix
index 0df5e690..f826ef36 100644
--- a/hydra-module.nix
+++ b/hydra-module.nix
@@ -268,7 +268,7 @@ in
         environment = env // {
           HYDRA_DBI = "${env.HYDRA_DBI};application_name=hydra-init";
         };
-        path = [ pkgs.utillinux ];
+        path = [ pkgs.util-linux ];
         preStart = ''
           ln -sf ${hydraConf} ${baseDir}/hydra.conf
 

From e06c480fd67d71fe1ab175c3f14e694f08a0cae0 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 12 Jul 2022 11:58:38 +0200
Subject: [PATCH 56/76] flake.lock: Update
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Flake lock file updates:

• Updated input 'newNixpkgs':
    'github:NixOS/nixpkgs/6e3ee8957637a60f5072e33d78e05c0f65c54366' (2022-03-15)
  → 'github:NixOS/nixpkgs/de5b3dd17034e6106e75746e81618e5bd408de8a' (2022-07-10)
• Updated input 'nix':
    'github:NixOS/nix/624e38aa43f304fbb78b4779172809add042b513' (2022-05-31)
  → 'github:NixOS/nix/b9cf655150b52d071c85a337cb5db96e735fa64a' (2022-07-11)
• Updated input 'nix/nixpkgs':
    'github:NixOS/nixpkgs/530a53dcbc9437363471167a5e4762c5fcfa34a1' (2022-02-19)
  → 'github:NixOS/nixpkgs/2fa57ed190fd6c7c746319444f34b5917666e5c1' (2022-05-31)
---
 flake.lock | 32 +++++++++++++++++---------------
 1 file changed, 17 insertions(+), 15 deletions(-)

diff --git a/flake.lock b/flake.lock
index 1310c53d..befc8e5b 100644
--- a/flake.lock
+++ b/flake.lock
@@ -18,11 +18,11 @@
     },
     "newNixpkgs": {
       "locked": {
-        "lastModified": 1647380550,
-        "narHash": "sha256-909TI9poX7CIUiFx203WL29YON6m/I6k0ExbZvR7bLM=",
+        "lastModified": 1657425264,
+        "narHash": "sha256-3aHvoI2e8vJKw3hvnHECaBpSsL5mxVsVtaLCnTdNcH8=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "6e3ee8957637a60f5072e33d78e05c0f65c54366",
+        "rev": "de5b3dd17034e6106e75746e81618e5bd408de8a",
         "type": "github"
       },
       "original": {
@@ -39,33 +39,34 @@
         "nixpkgs-regression": "nixpkgs-regression"
       },
       "locked": {
-        "lastModified": 1654014617,
-        "narHash": "sha256-qNL3lQPBsnStkru3j1ajN/H+knXI+X3dku8/dBfSw3g=",
+        "lastModified": 1657569404,
+        "narHash": "sha256-zJONRtGALmYifrWKzcH6MMfSKLxeuW2iqG13500OrY4=",
         "owner": "NixOS",
         "repo": "nix",
-        "rev": "624e38aa43f304fbb78b4779172809add042b513",
+        "rev": "b9cf655150b52d071c85a337cb5db96e735fa64a",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
-        "ref": "2.9.1",
+        "ref": "2.10.0",
         "repo": "nix",
         "type": "github"
       }
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1645296114,
-        "narHash": "sha256-y53N7TyIkXsjMpOG7RhvqJFGDacLs9HlyHeSTBioqYU=",
+        "lastModified": 1653988320,
+        "narHash": "sha256-ZaqFFsSDipZ6KVqriwM34T739+KLYJvNmCWzErjAg7c=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "530a53dcbc9437363471167a5e4762c5fcfa34a1",
+        "rev": "2fa57ed190fd6c7c746319444f34b5917666e5c1",
         "type": "github"
       },
       "original": {
-        "id": "nixpkgs",
-        "ref": "nixos-21.05-small",
-        "type": "indirect"
+        "owner": "NixOS",
+        "ref": "nixos-22.05-small",
+        "repo": "nixpkgs",
+        "type": "github"
       }
     },
     "nixpkgs-regression": {
@@ -78,9 +79,10 @@
         "type": "github"
       },
       "original": {
-        "id": "nixpkgs",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
         "rev": "215d4d0fd80ca5163643b03a33fde804a29cc1e2",
-        "type": "indirect"
+        "type": "github"
       }
     },
     "root": {

From c72bed5cb4f24853ebb7daaf26bb67d3cb4397f7 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 12 Jul 2022 14:28:55 +0200
Subject: [PATCH 57/76] Fix tests

Use $NIX_REMOTE instead of the legacy environment variables.
---
 t/lib/HydraTestContext.pm      | 16 ++++++++++------
 t/queue-runner/notifications.t |  4 +---
 2 files changed, 11 insertions(+), 9 deletions(-)

diff --git a/t/lib/HydraTestContext.pm b/t/lib/HydraTestContext.pm
index badb3728..53eaa0f7 100644
--- a/t/lib/HydraTestContext.pm
+++ b/t/lib/HydraTestContext.pm
@@ -53,16 +53,19 @@ sub new {
     my $hydra_config = $opts{'hydra_config'} || "";
     $hydra_config = "queue_runner_metrics_address = 127.0.0.1:0\n" . $hydra_config;
     if ($opts{'use_external_destination_store'} // 1) {
-        $hydra_config = "store_uri = file:$dir/nix/dest-store\n" . $hydra_config;
+        $hydra_config = "store_uri = file://$dir/nix/dest-store\n" . $hydra_config;
     }
 
     write_file($ENV{'HYDRA_CONFIG'}, $hydra_config);
 
-    $ENV{'NIX_LOG_DIR'} = "$dir/nix/var/log/nix";
+    my $nix_store_dir = "$dir/nix/store";
+    my $nix_state_dir = "$dir/nix/var/nix";
+    my $nix_log_dir = "$dir/nix/var/log/nix";
+
     $ENV{'NIX_REMOTE_SYSTEMS'} = '';
-    $ENV{'NIX_REMOTE'} = '';
-    $ENV{'NIX_STATE_DIR'} = "$dir/nix/var/nix";
-    $ENV{'NIX_STORE_DIR'} = "$dir/nix/store";
+    $ENV{'NIX_REMOTE'} = "local?store=$nix_store_dir&state=$nix_state_dir&log=$nix_log_dir";
+    $ENV{'NIX_STATE_DIR'} = $nix_state_dir; # FIXME: remove
+    $ENV{'NIX_STORE_DIR'} = $nix_store_dir; # FIXME: remove
 
     my $pgsql = Test::PostgreSQL->new(
         extra_initdb_args => "--locale C.UTF-8"
@@ -73,7 +76,8 @@ sub new {
         _db => undef,
         db_handle => $pgsql,
         tmpdir => $dir,
-        nix_state_dir => "$dir/nix/var/nix",
+        nix_state_dir => $nix_state_dir,
+        nix_log_dir => $nix_log_dir,
         testdir => abs_path(dirname(__FILE__) . "/.."),
         jobsdir => abs_path(dirname(__FILE__) . "/../jobs")
     }, $class;
diff --git a/t/queue-runner/notifications.t b/t/queue-runner/notifications.t
index b35b2b2f..1966cde1 100644
--- a/t/queue-runner/notifications.t
+++ b/t/queue-runner/notifications.t
@@ -33,9 +33,6 @@ my $ctx = test_context(
 # the build locally.
 
 subtest "Pre-build the job, upload to the cache, and then delete locally" => sub {
-    my $scratchlogdir = File::Temp->newdir();
-    $ENV{'NIX_LOG_DIR'} = "$scratchlogdir";
-
     my $outlink = $ctx->tmpdir . "/basic-canbesubstituted";
     is(system('nix-build', $ctx->jobsdir . '/notifications.nix', '-A', 'canbesubstituted', '--out-link', $outlink), 0, "Building notifications.nix succeeded");
     is(system('nix', 'copy', '--to', "file://${binarycachedir}", $outlink), 0, "Copying the closure to the binary cache succeeded");
@@ -46,6 +43,7 @@ subtest "Pre-build the job, upload to the cache, and then delete locally" => sub
     is(system('nix', 'log', $outpath), 0, "Reading the output's log succeeds");
     is(system('nix-store', '--delete', $outpath), 0, "Deleting the notifications.nix output succeeded");
     is(system("nix-collect-garbage"), 0, "Delete all the system's garbage");
+    File::Path::rmtree($ctx->{nix_log_dir});
 };
 
 subtest "Ensure substituting the job works, but reading the log fails" => sub {

From d5ba1bba50a7fa2f343d9ad284e293e13bb13f85 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 12 Jul 2022 14:46:48 +0200
Subject: [PATCH 58/76] Fix deprecation warning

---
 flake.nix | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/flake.nix b/flake.nix
index 36863913..db4eb854 100644
--- a/flake.nix
+++ b/flake.nix
@@ -681,7 +681,7 @@
         tests.install.x86_64-linux =
           with import (nixpkgs + "/nixos/lib/testing-python.nix") { system = "x86_64-linux"; };
           simpleTest {
-            machine = hydraServer;
+            nodes.machine = hydraServer;
             testScript =
               ''
                 machine.wait_for_job("hydra-init")
@@ -696,7 +696,7 @@
         tests.notifications.x86_64-linux =
           with import (nixpkgs + "/nixos/lib/testing-python.nix") { system = "x86_64-linux"; };
           simpleTest {
-            machine = { pkgs, ... }: {
+            nodes.machine = { pkgs, ... }: {
               imports = [ hydraServer ];
               services.hydra-dev.extraConfig = ''
                 <influxdb>
@@ -753,7 +753,7 @@
         tests.gitea.x86_64-linux =
           with import (nixpkgs + "/nixos/lib/testing-python.nix") { system = "x86_64-linux"; };
           makeTest {
-            machine = { pkgs, ... }: {
+            nodes.machine = { pkgs, ... }: {
               imports = [ hydraServer ];
               services.hydra-dev.extraConfig = ''
                 <gitea_authorization>

From 3e001a8f05cbd0f7f5a8c0fc2d8da8f839d7b3f8 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 12 Jul 2022 14:58:25 +0200
Subject: [PATCH 59/76] Remove newNixpkgs and a lot of packages that are in
 Nixpkgs 22.05

---
 flake.lock |  17 ---
 flake.nix  | 419 +----------------------------------------------------
 2 files changed, 3 insertions(+), 433 deletions(-)

diff --git a/flake.lock b/flake.lock
index befc8e5b..012d2407 100644
--- a/flake.lock
+++ b/flake.lock
@@ -16,22 +16,6 @@
         "type": "github"
       }
     },
-    "newNixpkgs": {
-      "locked": {
-        "lastModified": 1657425264,
-        "narHash": "sha256-3aHvoI2e8vJKw3hvnHECaBpSsL5mxVsVtaLCnTdNcH8=",
-        "owner": "NixOS",
-        "repo": "nixpkgs",
-        "rev": "de5b3dd17034e6106e75746e81618e5bd408de8a",
-        "type": "github"
-      },
-      "original": {
-        "owner": "NixOS",
-        "ref": "nixos-unstable-small",
-        "repo": "nixpkgs",
-        "type": "github"
-      }
-    },
     "nix": {
       "inputs": {
         "lowdown-src": "lowdown-src",
@@ -87,7 +71,6 @@
     },
     "root": {
       "inputs": {
-        "newNixpkgs": "newNixpkgs",
         "nix": "nix",
         "nixpkgs": [
           "nix",
diff --git a/flake.nix b/flake.nix
index db4eb854..cb1d9aea 100644
--- a/flake.nix
+++ b/flake.nix
@@ -1,13 +1,10 @@
 {
   description = "A Nix-based continuous build system";
 
-  # FIXME: All the pinned versions of nix/nixpkgs have a broken foreman (yes,
-  # even 2.7.0's Nixpkgs pin).
-  inputs.newNixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable-small";
   inputs.nixpkgs.follows = "nix/nixpkgs";
   inputs.nix.url = "github:NixOS/nix/2.10.0";
 
-  outputs = { self, newNixpkgs, nixpkgs, nix }:
+  outputs = { self, nixpkgs, nix }:
     let
 
       version = "${builtins.readFile ./version.txt}.${builtins.substring 0 8 (self.lastModifiedDate or "19700101")}.${self.shortRev or "DIRTY"}";
@@ -41,153 +38,8 @@
       # A Nixpkgs overlay that provides a 'hydra' package.
       overlay = final: prev: {
 
-        # Overlay these packages to use dependencies from the Nixpkgs everything
-        # else uses, to side-step the version difference: glibc is 2.32 in the
-        # nix-pinned Nixpkgs, but 2.33 in the newNixpkgs commit.
-        civetweb = (final.callPackage "${newNixpkgs}/pkgs/development/libraries/civetweb" { }).overrideAttrs
-          # Can be dropped once newNixpkgs points to a revision containing
-          # https://github.com/NixOS/nixpkgs/pull/167751
-          ({ cmakeFlags ? [ ], ... }: {
-            cmakeFlags = cmakeFlags ++ [
-              "-DCIVETWEB_ENABLE_IPV6=1"
-            ];
-          });
-        prometheus-cpp = final.callPackage "${newNixpkgs}/pkgs/development/libraries/prometheus-cpp" { };
-
         # Add LDAP dependencies that aren't currently found within nixpkgs.
         perlPackages = prev.perlPackages // {
-          TestPostgreSQL = final.perlPackages.buildPerlModule {
-            pname = "Test-PostgreSQL";
-            version = "1.28-1";
-            src = final.fetchFromGitHub {
-              owner = "grahamc";
-              repo = "Test-postgresql";
-              rev = "release-1.28-1";
-              hash = "sha256-SFC1C3q3dbcBos18CYd/s0TIcfJW4g04ld0+XQXVToQ=";
-            };
-            buildInputs = with final.perlPackages; [ ModuleBuildTiny TestSharedFork pkgs.postgresql ];
-            propagatedBuildInputs = with final.perlPackages; [ DBDPg DBI FileWhich FunctionParameters Moo TieHashMethod TryTiny TypeTiny ];
-
-            makeMakerFlags = "POSTGRES_HOME=${final.postgresql}";
-
-            meta = {
-              homepage = "https://github.com/grahamc/Test-postgresql/releases/tag/release-1.28-1";
-              description = "PostgreSQL runner for tests";
-              license = with final.lib.licenses; [ artistic2 ];
-            };
-          };
-
-          FunctionParameters = final.perlPackages.buildPerlPackage {
-            pname = "Function-Parameters";
-            version = "2.001003";
-            src = final.fetchurl {
-              url = "mirror://cpan/authors/id/M/MA/MAUKE/Function-Parameters-2.001003.tar.gz";
-              sha256 = "eaa22c6b43c02499ec7db0758c2dd218a3b2ab47a714b2bdf8010b5ee113c242";
-            };
-            buildInputs = with final.perlPackages; [ DirSelf TestFatal ];
-            meta = {
-              description = "Define functions and methods with parameter lists (\"subroutine signatures\")";
-              license = with final.lib.licenses; [ artistic1 gpl1Plus ];
-            };
-          };
-
-          CatalystPluginPrometheusTiny = final.perlPackages.buildPerlPackage {
-            pname = "Catalyst-Plugin-PrometheusTiny";
-            version = "0.005";
-            src = final.fetchurl {
-              url = "mirror://cpan/authors/id/S/SY/SYSPETE/Catalyst-Plugin-PrometheusTiny-0.005.tar.gz";
-              sha256 = "a42ef09efdc3053899ae007c41220d3ed7207582cc86e491b4f534539c992c5a";
-            };
-            buildInputs = with final.perlPackages; [ HTTPMessage Plack SubOverride TestDeep ];
-            propagatedBuildInputs = with final.perlPackages; [ CatalystRuntime Moose PrometheusTiny PrometheusTinyShared ];
-            meta = {
-              description = "Prometheus metrics for Catalyst";
-              license = with final.lib.licenses; [ artistic1 gpl1Plus ];
-            };
-          };
-
-          CryptArgon2 = final.perlPackages.buildPerlModule {
-            pname = "Crypt-Argon2";
-            version = "0.010";
-            src = final.fetchurl {
-              url = "mirror://cpan/authors/id/L/LE/LEONT/Crypt-Argon2-0.010.tar.gz";
-              sha256 = "3ea1c006f10ef66fd417e502a569df15c4cc1c776b084e35639751c41ce6671a";
-            };
-            nativeBuildInputs = [ pkgs.ld-is-cc-hook ];
-            meta = {
-              description = "Perl interface to the Argon2 key derivation functions";
-              license = final.lib.licenses.cc0;
-            };
-          };
-
-          CryptPassphrase = final.perlPackages.buildPerlPackage {
-            pname = "Crypt-Passphrase";
-            version = "0.003";
-            src = final.fetchurl {
-              url = "mirror://cpan/authors/id/L/LE/LEONT/Crypt-Passphrase-0.003.tar.gz";
-              sha256 = "685aa090f8179a86d6896212ccf8ccfde7a79cce857199bb14e2277a10d240ad";
-            };
-            meta = {
-              description = "A module for managing passwords in a cryptographically agile manner";
-              license = with final.lib.licenses; [ artistic1 gpl1Plus ];
-            };
-          };
-
-          CryptPassphraseArgon2 = final.perlPackages.buildPerlPackage {
-            pname = "Crypt-Passphrase-Argon2";
-            version = "0.002";
-            src = final.fetchurl {
-              url = "mirror://cpan/authors/id/L/LE/LEONT/Crypt-Passphrase-Argon2-0.002.tar.gz";
-              sha256 = "3906ff81697d13804ee21bd5ab78ffb1c4408b4822ce020e92ecf4737ba1f3a8";
-            };
-            propagatedBuildInputs = with final.perlPackages; [ CryptArgon2 CryptPassphrase ];
-            meta = {
-              description = "An Argon2 encoder for Crypt::Passphrase";
-              license = with final.lib.licenses; [ artistic1 gpl1Plus ];
-            };
-          };
-
-          DataRandom = final.perlPackages.buildPerlPackage {
-            pname = "Data-Random";
-            version = "0.13";
-            src = final.fetchurl {
-              url = "mirror://cpan/authors/id/B/BA/BAREFOOT/Data-Random-0.13.tar.gz";
-              sha256 = "eb590184a8db28a7e49eab09e25f8650c33f1f668b6a472829de74a53256bfc0";
-            };
-            buildInputs = with final.perlPackages; [ FileShareDirInstall TestMockTime ];
-            meta = {
-              description = "Perl module to generate random data";
-              license = with final.lib.licenses; [ artistic1 gpl1Plus ];
-            };
-          };
-
-          DirSelf = final.perlPackages.buildPerlPackage {
-            pname = "Dir-Self";
-            version = "0.11";
-            src = final.fetchurl {
-              url = "mirror://cpan/authors/id/M/MA/MAUKE/Dir-Self-0.11.tar.gz";
-              sha256 = "e251a51abc7d9ba3e708f73c2aa208e09d47a0c528d6254710fa78cc8d6885b5";
-            };
-            meta = {
-              homepage = "https://github.com/mauke/Dir-Self";
-              description = "A __DIR__ constant for the directory your source file is in";
-              license = with final.lib.licenses; [ artistic1 gpl1Plus ];
-            };
-          };
-
-          HashSharedMem = final.perlPackages.buildPerlModule {
-            pname = "Hash-SharedMem";
-            version = "0.005";
-            src = final.fetchurl {
-              url = "mirror://cpan/authors/id/Z/ZE/ZEFRAM/Hash-SharedMem-0.005.tar.gz";
-              sha256 = "324776808602f7bdc44adaa937895365454029a926fa611f321c9bf6b940bb5e";
-            };
-            buildInputs = with final.perlPackages; [ ScalarString ];
-            meta = {
-              description = "Efficient shared mutable hash";
-              license = with final.lib.licenses; [ artistic1 gpl1Plus ];
-            };
-          };
 
           PrometheusTiny = final.perlPackages.buildPerlPackage {
             pname = "Prometheus-Tiny";
@@ -204,269 +56,6 @@
             };
           };
 
-          PrometheusTinyShared = final.perlPackages.buildPerlPackage {
-            pname = "Prometheus-Tiny-Shared";
-            version = "0.023";
-            src = final.fetchurl {
-              url = "mirror://cpan/authors/id/R/RO/ROBN/Prometheus-Tiny-Shared-0.023.tar.gz";
-              sha256 = "7c2c72397be5d8e4839d1bf4033c1800f467f2509689673c6419df48794f2abe";
-            };
-            buildInputs = with final.perlPackages; [ DataRandom HTTPMessage Plack TestDifferences TestException ];
-            propagatedBuildInputs = with final.perlPackages; [ HashSharedMem JSONXS PrometheusTiny ];
-            meta = {
-              homepage = "https://github.com/robn/Prometheus-Tiny-Shared";
-              description = "A tiny Prometheus client with a shared database behind it";
-              license = with final.lib.licenses; [ artistic1 gpl1Plus ];
-            };
-          };
-
-          ReadonlyX = final.perlPackages.buildPerlModule {
-            pname = "ReadonlyX";
-            version = "1.04";
-            src = final.fetchurl {
-              url = "mirror://cpan/authors/id/S/SA/SANKO/ReadonlyX-1.04.tar.gz";
-              sha256 = "81bb97dba93ac6b5ccbce04a42c3590eb04557d75018773ee18d5a30fcf48188";
-            };
-            buildInputs = with final.perlPackages; [ ModuleBuildTiny TestFatal ];
-            meta = {
-              homepage = "https://github.com/sanko/readonly";
-              description = "Faster facility for creating read-only scalars, arrays, hashes";
-              license = final.lib.licenses.artistic2;
-            };
-          };
-
-          TieHashMethod = final.perlPackages.buildPerlPackage {
-            pname = "Tie-Hash-Method";
-            version = "0.02";
-            src = final.fetchurl {
-              url = "mirror://cpan/authors/id/Y/YV/YVES/Tie-Hash-Method-0.02.tar.gz";
-              sha256 = "d513fbb51413f7ca1e64a1bdce6194df7ec6076dea55066d67b950191eec32a9";
-            };
-            meta = {
-              description = "Tied hash with specific methods overriden by callbacks";
-              license = with final.lib.licenses; [ artistic1 ];
-            };
-          };
-
-          Test2Harness = final.perlPackages.buildPerlPackage {
-            pname = "Test2-Harness";
-            version = "1.000042";
-            src = final.fetchurl {
-              url = "mirror://cpan/authors/id/E/EX/EXODIST/Test2-Harness-1.000042.tar.gz";
-              sha256 = "aaf231a68af1a6ffd6a11188875fcf572e373e43c8285945227b9d687b43db2d";
-            };
-
-            checkPhase = ''
-              patchShebangs ./t ./scripts/yath
-              ./scripts/yath test -j $NIX_BUILD_CORES
-            '';
-
-            propagatedBuildInputs = with final.perlPackages; [ DataUUID Importer LongJump ScopeGuard TermTable Test2PluginMemUsage Test2PluginUUID Test2Suite gotofile ];
-            meta = {
-              description = "A new and improved test harness with better Test2 integration";
-              license = with final.lib.licenses; [ artistic1 gpl1Plus ];
-            };
-          };
-
-          Test2PluginMemUsage = prev.perlPackages.buildPerlPackage {
-            pname = "Test2-Plugin-MemUsage";
-            version = "0.002003";
-            src = final.fetchurl {
-              url = "mirror://cpan/authors/id/E/EX/EXODIST/Test2-Plugin-MemUsage-0.002003.tar.gz";
-              sha256 = "5e0662d5a823ae081641f5ce82843111eec1831cd31f883a6c6de54afdf87c25";
-            };
-            buildInputs = with final.perlPackages; [ Test2Suite ];
-            meta = {
-              description = "Collect and display memory usage information";
-              license = with final.lib.licenses; [ artistic1 gpl1Plus ];
-            };
-          };
-
-          Test2PluginUUID = prev.perlPackages.buildPerlPackage {
-            pname = "Test2-Plugin-UUID";
-            version = "0.002001";
-            src = final.fetchurl {
-              url = "mirror://cpan/authors/id/E/EX/EXODIST/Test2-Plugin-UUID-0.002001.tar.gz";
-              sha256 = "4c6c8d484d7153d8779dc155a992b203095b5c5aa1cfb1ee8bcedcd0601878c9";
-            };
-            buildInputs = with final.perlPackages;[ Test2Suite ];
-            propagatedBuildInputs = with final.perlPackages; [ DataUUID ];
-            meta = {
-              description = "Use REAL UUIDs in Test2";
-              license = with final.lib.licenses; [ artistic1 gpl1Plus ];
-            };
-          };
-
-          LongJump = final.perlPackages.buildPerlPackage {
-            pname = "Long-Jump";
-            version = "0.000001";
-            src = final.fetchurl {
-              url = "mirror://cpan/authors/id/E/EX/EXODIST/Long-Jump-0.000001.tar.gz";
-              sha256 = "d5d6456d86992b559d8f66fc90960f919292cd3803c13403faac575762c77af4";
-            };
-            buildInputs = with final.perlPackages; [ Test2Suite ];
-            meta = {
-              description = "Mechanism for returning to a specific point from a deeply nested stack";
-              license = with final.lib.licenses; [ artistic1 gpl1Plus ];
-            };
-          };
-
-          gotofile = final.perlPackages.buildPerlPackage {
-            pname = "goto-file";
-            version = "0.005";
-            src = final.fetchurl {
-              url = "mirror://cpan/authors/id/E/EX/EXODIST/goto-file-0.005.tar.gz";
-              sha256 = "c6cdd5ee4a6cdcbdbf314d92a4f9985dbcdf9e4258048cae76125c052aa31f77";
-            };
-            buildInputs = with final.perlPackages; [ Test2Suite ];
-            meta = {
-              description = "Stop parsing the current file and move on to a different one";
-              license = with final.lib.licenses; [ artistic1 gpl1Plus ];
-            };
-          };
-
-          NetLDAPServer = prev.perlPackages.buildPerlPackage {
-            pname = "Net-LDAP-Server";
-            version = "0.43";
-            src = final.fetchurl {
-              url = "mirror://cpan/authors/id/A/AA/AAR/Net-LDAP-Server-0.43.tar.gz";
-              sha256 = "0qmh3cri3fpccmwz6bhwp78yskrb3qmalzvqn0a23hqbsfs4qv6x";
-            };
-            propagatedBuildInputs = with final.perlPackages; [ perlldap ConvertASN1 ];
-            meta = {
-              description = "LDAP server side protocol handling";
-              license = with final.lib.licenses; [ artistic1 ];
-            };
-          };
-
-          NetLDAPSID = prev.perlPackages.buildPerlPackage {
-            pname = "Net-LDAP-SID";
-            version = "0.0001";
-            src = final.fetchurl {
-              url = "mirror://cpan/authors/id/K/KA/KARMAN/Net-LDAP-SID-0.001.tar.gz";
-              sha256 = "1mnnpkmj8kpb7qw50sm8h4sd8py37ssy2xi5hhxzr5whcx0cvhm8";
-            };
-            meta = {
-              description = "Active Directory Security Identifier manipulation";
-              license = with final.lib.licenses; [ artistic2 ];
-            };
-          };
-
-          NetLDAPServerTest = prev.perlPackages.buildPerlPackage {
-            pname = "Net-LDAP-Server-Test";
-            version = "0.22";
-            src = final.fetchurl {
-              url = "mirror://cpan/authors/id/K/KA/KARMAN/Net-LDAP-Server-Test-0.22.tar.gz";
-              sha256 = "13idip7jky92v4adw60jn2gcc3zf339gsdqlnc9nnvqzbxxp285i";
-            };
-            propagatedBuildInputs = with final.perlPackages; [ perlldap NetLDAPServer DataDump NetLDAPSID ];
-            meta = {
-              description = "test Net::LDAP code";
-              license = with final.lib.licenses; [ artistic1 ];
-            };
-          };
-
-          CatalystAuthenticationStoreLDAP = prev.perlPackages.buildPerlPackage {
-            pname = "Catalyst-Authentication-Store-LDAP";
-            version = "1.016";
-            src = final.fetchurl {
-              url = "mirror://cpan/authors/id/I/IL/ILMARI/Catalyst-Authentication-Store-LDAP-1.016.tar.gz";
-              sha256 = "0cm399vxqqf05cjgs1j5v3sk4qc6nmws5nfhf52qvpbwc4m82mq8";
-            };
-            propagatedBuildInputs = with final.perlPackages; [ perlldap CatalystPluginAuthentication ClassAccessor ];
-            buildInputs = with final.perlPackages; [ TestMockObject TestException NetLDAPServerTest ];
-            meta = {
-              description = "Authentication from an LDAP Directory";
-              license = with final.lib.licenses; [ artistic1 ];
-            };
-          };
-
-          PerlCriticCommunity = prev.perlPackages.buildPerlModule {
-            pname = "Perl-Critic-Community";
-            version = "1.0.0";
-            src = final.fetchurl {
-              url = "mirror://cpan/authors/id/D/DB/DBOOK/Perl-Critic-Community-v1.0.0.tar.gz";
-              sha256 = "311b775da4193e9de94cf5225e993cc54dd096ae1e7ef60738cdae1d9b8854e7";
-            };
-            buildInputs = with final.perlPackages; [ ModuleBuildTiny ];
-            propagatedBuildInputs = with final.perlPackages; [ PPI PathTiny PerlCritic PerlCriticPolicyVariablesProhibitLoopOnHash PerlCriticPulp ];
-            meta = {
-              homepage = "https://github.com/Grinnz/Perl-Critic-Freenode";
-              description = "Community-inspired Perl::Critic policies";
-              license = final.lib.licenses.artistic2;
-            };
-          };
-
-          PerlCriticPolicyVariablesProhibitLoopOnHash = prev.perlPackages.buildPerlPackage {
-            pname = "Perl-Critic-Policy-Variables-ProhibitLoopOnHash";
-            version = "0.008";
-            src = final.fetchurl {
-              url = "mirror://cpan/authors/id/X/XS/XSAWYERX/Perl-Critic-Policy-Variables-ProhibitLoopOnHash-0.008.tar.gz";
-              sha256 = "12f5f0be96ea1bdc7828058577bd1c5c63ca23c17fac9c3709452b3dff5b84e0";
-            };
-            propagatedBuildInputs = with final.perlPackages; [ PerlCritic ];
-            meta = {
-              description = "Don't write loops on hashes, only on keys and values of hashes";
-              license = with final.lib.licenses; [ artistic1 gpl1Plus ];
-            };
-          };
-
-          PerlCriticPulp = prev.perlPackages.buildPerlPackage {
-            pname = "Perl-Critic-Pulp";
-            version = "99";
-            src = final.fetchurl {
-              url = "mirror://cpan/authors/id/K/KR/KRYDE/Perl-Critic-Pulp-99.tar.gz";
-              sha256 = "b8fda842fcbed74d210257c0a284b6dc7b1d0554a47a3de5d97e7d542e23e7fe";
-            };
-            propagatedBuildInputs = with final.perlPackages; [ IOString ListMoreUtils PPI PerlCritic PodMinimumVersion ];
-            meta = {
-              homepage = "http://user42.tuxfamily.org/perl-critic-pulp/index.html";
-              description = "Some add-on policies for Perl::Critic";
-              license = final.lib.licenses.gpl3Plus;
-            };
-          };
-
-          PodMinimumVersion = prev.perlPackages.buildPerlPackage {
-            pname = "Pod-MinimumVersion";
-            version = "50";
-            src = final.fetchurl {
-              url = "mirror://cpan/authors/id/K/KR/KRYDE/Pod-MinimumVersion-50.tar.gz";
-              sha256 = "0bd2812d9aacbd99bb71fa103a4bb129e955c138ba7598734207dc9fb67b5a6f";
-            };
-            propagatedBuildInputs = with final.perlPackages; [ IOString PodParser ];
-            meta = {
-              homepage = "http://user42.tuxfamily.org/pod-minimumversion/index.html";
-              description = "Determine minimum Perl version of POD directives";
-              license = final.lib.licenses.free;
-            };
-          };
-
-          StringCompareConstantTime = final.perlPackages.buildPerlPackage {
-            pname = "String-Compare-ConstantTime";
-            version = "0.321";
-            src = final.fetchurl {
-              url = "mirror://cpan/authors/id/F/FR/FRACTAL/String-Compare-ConstantTime-0.321.tar.gz";
-              sha256 = "0b26ba2b121d8004425d4485d1d46f59001c83763aa26624dff6220d7735d7f7";
-            };
-            meta = {
-              description = "Timing side-channel protected string compare";
-              license = with final.lib.licenses; [ artistic1 gpl1Plus ];
-            };
-          };
-
-          UUID4Tiny = final.perlPackages.buildPerlPackage {
-            pname = "UUID4-Tiny";
-            version = "0.002";
-            src = final.fetchurl {
-              url = "mirror://cpan/authors/id/C/CV/CVLIBRARY/UUID4-Tiny-0.002.tar.gz";
-              sha256 = "e7535b31e386d432dec7adde214348389e1d5cf753e7ed07f1ae04c4360840cf";
-            };
-            meta = {
-              description = "Cryptographically secure v4 UUIDs for Linux x64";
-              license = with final.lib.licenses; [ artistic1 gpl1Plus ];
-            };
-          };
-
         };
 
         hydra = with final; let
@@ -581,9 +170,7 @@
 
           checkInputs = [
             cacert
-            # FIXME: foreman is broken on all nix/nixpkgs pin, up to and
-            # including 2.7.0
-            newNixpkgs.legacyPackages.${final.stdenv.system}.foreman
+            foreman
             glibcLocales
             libressl.nc
             openldap
@@ -1015,7 +602,7 @@
             self.nixosModules.hydraTest
             self.nixosModules.hydraProxy
             {
-              system.configurationRevision = self.rev;
+              system.configurationRevision = self.lastModifiedDate;
 
               boot.isContainer = true;
               networking.useDHCP = false;

From 481ca71d6e7a391b7c93da7149f50ada02e9303f Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 12 Jul 2022 15:03:27 +0200
Subject: [PATCH 60/76] Use new flake output naming convention

---
 flake.nix | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/flake.nix b/flake.nix
index cb1d9aea..c5d4b547 100644
--- a/flake.nix
+++ b/flake.nix
@@ -11,7 +11,7 @@
 
       pkgs = import nixpkgs {
         system = "x86_64-linux";
-        overlays = [ self.overlay nix.overlays.default ];
+        overlays = [ self.overlays.default nix.overlays.default ];
       };
 
       # NixOS configuration used for VM tests.
@@ -36,7 +36,7 @@
     rec {
 
       # A Nixpkgs overlay that provides a 'hydra' package.
-      overlay = final: prev: {
+      overlays.default = final: prev: {
 
         # Add LDAP dependencies that aren't currently found within nixpkgs.
         perlPackages = prev.perlPackages // {
@@ -547,11 +547,11 @@
       checks.x86_64-linux.validate-openapi = hydraJobs.tests.validate-openapi;
 
       packages.x86_64-linux.hydra = pkgs.hydra;
-      defaultPackage.x86_64-linux = pkgs.hydra;
+      packages.x86_64-linux.default = pkgs.hydra;
 
       nixosModules.hydra = {
         imports = [ ./hydra-module.nix ];
-        nixpkgs.overlays = [ self.overlay nix.overlays.default ];
+        nixpkgs.overlays = [ self.overlays.default nix.overlays.default ];
       };
 
       nixosModules.hydraTest = {

From 9656f16509665a2d49ff7de2905d7115b5fe4440 Mon Sep 17 00:00:00 2001
From: Amanda Cameron <git@amanda.darkdna.net>
Date: Fri, 22 Jul 2022 10:35:38 -0400
Subject: [PATCH 61/76] Update hydra-module.nix to use newer nixos options

nix.trustedUsers is deprecated as of 22.05, and since the nix.extraOptions config is just doing something similar, I moved that to the new nix.settings as well
---
 hydra-module.nix | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/hydra-module.nix b/hydra-module.nix
index 0df5e690..1f46a3ca 100644
--- a/hydra-module.nix
+++ b/hydra-module.nix
@@ -228,8 +228,12 @@ in
         useDefaultShell = true;
       };
 
-    nix.trustedUsers = [ "hydra-queue-runner" ];
-
+    nix.settings = {
+      trusted-users = [ "hydra-queue-runner" ];
+      gc-keep-outputs = true;
+      gc-keep-derivations = true;
+    };
+    
     services.hydra-dev.extraConfig =
       ''
         using_frontend_proxy = 1
@@ -256,11 +260,6 @@ in
 
     environment.variables = hydraEnv;
 
-    nix.extraOptions = ''
-      gc-keep-outputs = true
-      gc-keep-derivations = true
-    '';
-
     systemd.services.hydra-init =
       { wantedBy = [ "multi-user.target" ];
         requires = optional haveLocalDB "postgresql.service";

From a58e2f1a642233672f9941a77c0f4bd5c3597878 Mon Sep 17 00:00:00 2001
From: Marco Rebhan <me@dblsaiko.net>
Date: Thu, 4 Aug 2022 21:18:55 +0200
Subject: [PATCH 62/76] Use libmagic for better output MIME detection

---
 flake.nix                         |  1 +
 src/lib/Hydra/Controller/Build.pm | 39 ++++++-------------------------
 2 files changed, 8 insertions(+), 32 deletions(-)

diff --git a/flake.nix b/flake.nix
index 2e891364..93186ee0 100644
--- a/flake.nix
+++ b/flake.nix
@@ -503,6 +503,7 @@
                 DigestSHA1
                 EmailMIME
                 EmailSender
+                FileLibMagic
                 FileSlurper
                 FileWhich
                 final.nix.perl-bindings
diff --git a/src/lib/Hydra/Controller/Build.pm b/src/lib/Hydra/Controller/Build.pm
index c7811c62..18a0eba3 100644
--- a/src/lib/Hydra/Controller/Build.pm
+++ b/src/lib/Hydra/Controller/Build.pm
@@ -7,15 +7,16 @@ use base 'Hydra::Base::Controller::NixChannel';
 use Hydra::Helper::Nix;
 use Hydra::Helper::CatalystUtils;
 use File::Basename;
+use File::LibMagic;
 use File::stat;
 use Data::Dump qw(dump);
 use Nix::Store;
 use Nix::Config;
 use List::SomeUtils qw(all);
 use Encode;
-use MIME::Types;
 use JSON::PP;
 
+use feature 'state';
 
 sub buildChain :Chained('/') :PathPart('build') :CaptureArgs(1) {
     my ($self, $c, $id) = @_;
@@ -236,14 +237,10 @@ sub serveFile {
         $c->stash->{'plain'} = { data => grab(cmd => ["nix", "--experimental-features", "nix-command",
                                                       "store", "cat", "--store", getStoreUri(), "$path"]) };
 
-        # Detect MIME type. Borrowed from Catalyst::Plugin::Static::Simple.
-        my $type = "text/plain";
-        if ($path =~ /.*\.(\S{1,})$/xms) {
-            my $ext = $1;
-            my $mimeTypes = MIME::Types->new(only_complete => 1);
-            my $t = $mimeTypes->mimeTypeOf($ext);
-            $type = ref $t ? $t->type : $t if $t;
-        }
+        # Detect MIME type.
+        state $magic = File::LibMagic->new(follow_symlinks => 1);
+        my $info = $magic->info_from_filename($path);
+        my $type = $info->{mime_with_encoding};
         $c->response->content_type($type);
         $c->forward('Hydra::View::Plain');
     }
@@ -288,29 +285,7 @@ sub download : Chained('buildChain') PathPart {
     my $path = $product->path;
     $path .= "/" . join("/", @path) if scalar @path > 0;
 
-    if (isLocalStore) {
-
-        notFound($c, "File '" . $product->path . "' does not exist.") unless -e $product->path;
-
-        # Make sure the file is in the Nix store.
-        $path = checkPath($self, $c, $path);
-
-        # If this is a directory but no "/" is attached, then redirect.
-        if (-d $path && substr($c->request->uri, -1) ne "/") {
-            return $c->res->redirect($c->request->uri . "/");
-        }
-
-        $path = "$path/index.html" if -d $path && -e "$path/index.html";
-
-        notFound($c, "File '$path' does not exist.") if !-e $path;
-
-        notFound($c, "Path '$path' is a directory.") if -d $path;
-
-        $c->serve_static_file($path);
-
-    } else {
-        serveFile($c, $path);
-    }
+    serveFile($c, $path);
 
     $c->response->headers->last_modified($c->stash->{build}->stoptime);
 }

From f6d45b0f0cdedf940df67a935f4eb337b43300e7 Mon Sep 17 00:00:00 2001
From: Maximilian Bosch <maximilian@mbosch.me>
Date: Mon, 8 Aug 2022 13:35:56 +0200
Subject: [PATCH 63/76] doc/configuration: fix ldap role mapping example

To group is called `cancel-build`, not `cancel-builds` (note the
trailing `s`).
---
 doc/manual/src/configuration.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/manual/src/configuration.md b/doc/manual/src/configuration.md
index 2700625d..ab68df43 100644
--- a/doc/manual/src/configuration.md
+++ b/doc/manual/src/configuration.md
@@ -185,7 +185,7 @@ Example configuration:
     hydra_admin = admin
     # Allow all users in the dev group to restart jobs and cancel builds
     dev = restart-jobs
-    dev = cancel-builds
+    dev = cancel-build
   </role_mapping>
 </ldap>
 ```

From 74caaa696e5732313a621ada9a0d9283a57fa764 Mon Sep 17 00:00:00 2001
From: K900 <me@0upti.me>
Date: Thu, 11 Aug 2022 13:30:19 +0300
Subject: [PATCH 64/76] Run the JS code to make build trees collapsible at the
 right time

---
 src/root/build.tt            |  4 ++--
 src/root/common.tt           |  6 +++++-
 src/root/static/js/common.js | 27 ++++++++++++++++-----------
 3 files changed, 23 insertions(+), 14 deletions(-)

diff --git a/src/root/build.tt b/src/root/build.tt
index 027ce3e4..93a02e0f 100644
--- a/src/root/build.tt
+++ b/src/root/build.tt
@@ -481,11 +481,11 @@ END;
 [% END %]
 
   [% IF drvAvailable %]
-    [% INCLUDE makeLazyTab tabName="tabs-build-deps" uri=c.uri_for('/build' build.id 'build-deps') %]
+    [% INCLUDE makeLazyTab tabName="tabs-build-deps" uri=c.uri_for('/build' build.id 'build-deps') callback="makeTreeCollapsible" %]
   [% END %]
 
   [% IF available %]
-    [% INCLUDE makeLazyTab tabName="tabs-runtime-deps" uri=c.uri_for('/build' build.id 'runtime-deps') %]
+    [% INCLUDE makeLazyTab tabName="tabs-runtime-deps" uri=c.uri_for('/build' build.id 'runtime-deps') callback="makeTreeCollapsible" %]
   [% END %]
 
   <div id="tabs-runcommandlogs" class="tab-pane">
diff --git a/src/root/common.tt b/src/root/common.tt
index 6b4e406e..32d6c8cc 100644
--- a/src/root/common.tt
+++ b/src/root/common.tt
@@ -520,7 +520,11 @@ BLOCK makeLazyTab %]
     <center><span class="spinner-border spinner-border-sm"/></center>
   </div>
   <script>
-    $(function() { makeLazyTab("[% tabName %]", "[% uri %]"); });
+    [% IF callback.defined %]
+    $(function() { makeLazyTab("[% tabName %]", "[% uri %]", [% callback %] ); });
+    [% ELSE %]
+    $(function() { makeLazyTab("[% tabName %]", "[% uri %]", null ); });
+    [% END %]
   </script>
 [% END;
 
diff --git a/src/root/static/js/common.js b/src/root/static/js/common.js
index 433c06d7..c51f769a 100644
--- a/src/root/static/js/common.js
+++ b/src/root/static/js/common.js
@@ -1,10 +1,9 @@
-$(document).ready(function() {
-
+function makeTreeCollapsible(tab) {
     /*** Tree toggles in logfiles. ***/
 
     /* Set the appearance of the toggle depending on whether the
        corresponding subtree is initially shown or hidden. */
-    $(".tree-toggle").map(function() {
+    tab.find(".tree-toggle").map(function() {
         if ($(this).siblings("ul:hidden").length == 0) {
             $(this).text("-");
         } else {
@@ -13,7 +12,7 @@ $(document).ready(function() {
     });
 
     /* When a toggle is clicked, show or hide the subtree. */
-    $(".tree-toggle").click(function() {
+    tab.find(".tree-toggle").click(function() {
         if ($(this).siblings("ul:hidden").length != 0) {
             $(this).siblings("ul").show();
             $(this).text("-");
@@ -24,21 +23,23 @@ $(document).ready(function() {
     });
 
     /* Implementation of the expand all link. */
-    $(".tree-expand-all").click(function() {
-        $(".tree-toggle", $(this).parent().siblings(".tree")).map(function() {
+    tab.find(".tree-expand-all").click(function() {
+        tab.find(".tree-toggle", $(this).parent().siblings(".tree")).map(function() {
             $(this).siblings("ul").show();
             $(this).text("-");
         });
     });
 
     /* Implementation of the collapse all link. */
-    $(".tree-collapse-all").click(function() {
-        $(".tree-toggle", $(this).parent().siblings(".tree")).map(function() {
+    tab.find(".tree-collapse-all").click(function() {
+        tab.find(".tree-toggle", $(this).parent().siblings(".tree")).map(function() {
             $(this).siblings("ul").hide();
             $(this).text("+");
         });
     });
+}
 
+$(document).ready(function() {
     $("table.clickable-rows").click(function(event) {
         if ($(event.target).closest("a").length) return;
         link = $(event.target).parents("tr").find("a.row-link");
@@ -132,7 +133,7 @@ $(document).ready(function() {
 
 var tabsLoaded = {};
 
-function makeLazyTab(tabName, uri) {
+function makeLazyTab(tabName, uri, callback) {
     $('.nav-tabs').bind('show.bs.tab', function(e) {
         var pattern = /#.+/gi;
         var id = e.target.toString().match(pattern)[0];
@@ -140,11 +141,15 @@ function makeLazyTab(tabName, uri) {
           tabsLoaded[id] = 1;
           $('#' + tabName).load(uri, function(response, status, xhr) {
             var lazy = xhr.getResponseHeader("X-Hydra-Lazy") === "Yes";
+            var tab = $('#' + tabName);
             if (status == "error" && !lazy) {
-              $('#' + tabName).html("<div class='alert alert-error'>Error loading tab: " + xhr.status + " " + xhr.statusText + "</div>");
+              tab.html("<div class='alert alert-error'>Error loading tab: " + xhr.status + " " + xhr.statusText + "</div>");
             }
             else {
-              $('#' + tabName).html(response);
+              tab.html(response);
+              if (callback) {
+                callback(tab);
+              }
             }
           });
         }

From 93bbd6925b8c08d8080d0a317b5f30694da1e67c Mon Sep 17 00:00:00 2001
From: K900 <me@0upti.me>
Date: Fri, 12 Aug 2022 09:46:17 +0300
Subject: [PATCH 65/76] Also restore the "expand all" and "collapse all"
 buttons

---
 src/root/build-deps.tt        | 2 +-
 src/root/deps.tt              | 9 ++++++++-
 src/root/static/css/hydra.css | 5 +++++
 3 files changed, 14 insertions(+), 2 deletions(-)

diff --git a/src/root/build-deps.tt b/src/root/build-deps.tt
index 157b113e..f15342fa 100644
--- a/src/root/build-deps.tt
+++ b/src/root/build-deps.tt
@@ -4,6 +4,6 @@
 
 <div class="dep-tree">
   <ul class="tree">
-    [% INCLUDE renderNode node=buildTimeGraph %]
+    [% INCLUDE renderNode node=buildTimeGraph isRoot=1 %]
   </ul>
 </div>
diff --git a/src/root/deps.tt b/src/root/deps.tt
index a2c1fbba..6daa9725 100644
--- a/src/root/deps.tt
+++ b/src/root/deps.tt
@@ -19,9 +19,16 @@
           <tt>[% node.name %]</tt> (<em>no info</em>)
         [% END %]
       </span></span>
+      [% IF isRoot %]
+      <span class="dep-tree-buttons">
+        (<a href="#" class="tree-collapse-all">collapse all</a>
+        &ndash;
+        <a href="#" class="tree-expand-all">expand all</a>)
+      </span>
+      [% END %]
       [% IF node.refs.size > 0 %]
         <ul class="subtree">
-          [% FOREACH ref IN node.refs; INCLUDE renderNode node=ref; END %]
+          [% FOREACH ref IN node.refs; INCLUDE renderNode node=ref isRoot=0; END %]
         </ul>
       [% END %]
     [% END %]
diff --git a/src/root/static/css/hydra.css b/src/root/static/css/hydra.css
index 3b50d246..aac3f1ea 100644
--- a/src/root/static/css/hydra.css
+++ b/src/root/static/css/hydra.css
@@ -33,6 +33,11 @@ span:target > span.dep-tree-line {
     font-weight: bold;
 }
 
+span.dep-tree-buttons {
+    font-style: italic;
+    padding-left: 10px;
+}
+
 span.disabled-project, span.disabled-jobset, span.disabled-job {
     text-decoration: line-through;
 }

From 2b8a8fdd9cd7c6ea7d2d892fa9c93acdfef0c25d Mon Sep 17 00:00:00 2001
From: K900 <me@0upti.me>
Date: Fri, 12 Aug 2022 09:46:32 +0300
Subject: [PATCH 66/76] Make the tree a little less dense

---
 src/root/static/css/tree.css | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/root/static/css/tree.css b/src/root/static/css/tree.css
index 71df2f12..e80f6871 100644
--- a/src/root/static/css/tree.css
+++ b/src/root/static/css/tree.css
@@ -9,6 +9,7 @@ ul.tree, ul.subtree {
 ul.subtree > li {
     position: relative;
     padding-left: 2.0em;
+    line-height: 140%;
     border-left: 0.1em solid #6185a0;
 }
 

From 9addaeb17fc9663f706b58b33e0de8ec57f4bf2b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Janne=20He=C3=9F?= <janne@hess.ooo>
Date: Fri, 8 Jul 2022 22:23:35 +0200
Subject: [PATCH 67/76] Add a squiggly line to the Hydra link on hover

The effect is the same as the one on links in mail bodys on
https://lists.apache.org/
---
 src/root/layout.tt            |  2 +-
 src/root/static/css/hydra.css | 10 ++++++++++
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/src/root/layout.tt b/src/root/layout.tt
index 8eb1f119..d67ff1b8 100644
--- a/src/root/layout.tt
+++ b/src/root/layout.tt
@@ -93,7 +93,7 @@
       <footer class="navbar">
         <hr />
         <small>
-          <em><a href="http://nixos.org/hydra" target="_blank">Hydra</a> [% HTML.escape(version) %] (using [% HTML.escape(nixVersion) %]).</em>
+          <em><a href="http://nixos.org/hydra" target="_blank" class="squiggle">Hydra</a> [% HTML.escape(version) %] (using [% HTML.escape(nixVersion) %]).</em>
           [% IF c.user_exists %]
           You are signed in as <tt>[% HTML.escape(c.user.username) %]</tt>
             [%- IF c.user.type == 'google' %] via Google[% END %].
diff --git a/src/root/static/css/hydra.css b/src/root/static/css/hydra.css
index 3b50d246..251e0e75 100644
--- a/src/root/static/css/hydra.css
+++ b/src/root/static/css/hydra.css
@@ -166,6 +166,16 @@ body {
     line-height: 1.5;
 }
 
+a.squiggle:hover {
+    background-image: url("data:image/svg+xml;charset=utf8,%3Csvg id='squiggle-link' xmlns='http://www.w3.org/2000/svg' xmlns:xlink='http://www.w3.org/1999/xlink' xmlns:ev='http://www.w3.org/2001/xml-events' viewBox='0 0 10 18'%3E%3Cstyle type='text/css'%3E.squiggle{animation:shift .5s linear infinite;}@keyframes shift {from {transform:translateX(-10px);}to {transform:translateX(0);}}%3C/style%3E%3Cpath fill='none' stroke='%230056b3' stroke-width='0.65' class='squiggle' d='M0,17.5 c 2.5,0,2.5,-1.5,5,-1.5 s 2.5,1.5,5,1.5 c 2.5,0,2.5,-1.5,5,-1.5 s 2.5,1.5,5,1.5' /%3E%3C/svg%3E");
+    background-position: 0 100%;
+    background-size: auto 24px;
+    background-repeat: repeat;
+    text-decoration: none;
+    border-bottom: none;
+    padding-bottom: 1px;
+}
+
 @media (prefers-color-scheme: dark) {
     /* Prevent some flickering */
     html {

From 371402c3c1d76a7d4e2e1691fda3d90397c8929f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Janne=20He=C3=9F?= <janne@hess.ooo>
Date: Wed, 17 Aug 2022 11:41:37 +0200
Subject: [PATCH 68/76] Drop the HipChat plugin

https://en.wikipedia.org/wiki/HipChat says:
> Following this, HipChat and Stride customers were migrated to the
> Slack group collaboration platform in a transition that was completed by
> February 2019.
---
 doc/manual/src/plugins/README.md            | 11 ---
 src/lib/Hydra/Plugin/HipChatNotification.pm | 89 ---------------------
 2 files changed, 100 deletions(-)
 delete mode 100644 src/lib/Hydra/Plugin/HipChatNotification.pm

diff --git a/doc/manual/src/plugins/README.md b/doc/manual/src/plugins/README.md
index 26ee2649..6e46224f 100644
--- a/doc/manual/src/plugins/README.md
+++ b/doc/manual/src/plugins/README.md
@@ -172,17 +172,6 @@ Sets Gitlab CI status.
 
 - `gitlab_authorization.<projectId>`
 
-## HipChat notification
-
-Sends hipchat chat notifications when a build finish.
-
-### Configuration options
-
-- `hipchat.[].jobs`
-- `hipchat.[].builds`
-- `hipchat.[].token`
-- `hipchat.[].notify`
-
 ## InfluxDB notification
 
 Writes InfluxDB events when a builds finished.
diff --git a/src/lib/Hydra/Plugin/HipChatNotification.pm b/src/lib/Hydra/Plugin/HipChatNotification.pm
deleted file mode 100644
index 61cea57a..00000000
--- a/src/lib/Hydra/Plugin/HipChatNotification.pm
+++ /dev/null
@@ -1,89 +0,0 @@
-package Hydra::Plugin::HipChatNotification;
-
-use strict;
-use warnings;
-use parent 'Hydra::Plugin';
-use LWP::UserAgent;
-use Hydra::Helper::CatalystUtils;
-
-sub isEnabled {
-    my ($self) = @_;
-    return defined $self->{config}->{hipchat};
-}
-
-sub buildFinished {
-    my ($self, $topbuild, $dependents) = @_;
-
-    my $cfg = $self->{config}->{hipchat};
-    my @config = defined $cfg ? ref $cfg eq "ARRAY" ? @$cfg : ($cfg) : ();
-
-    my $baseurl = $self->{config}->{'base_uri'} || "http://localhost:3000";
-
-    # Figure out to which rooms to send notification.  For each email
-    # room, we send one aggregate message.
-    my %rooms;
-    foreach my $build ($topbuild, @{$dependents}) {
-        my $prevBuild = getPreviousBuild($build);
-        my $jobName = showJobName $build;
-
-        foreach my $room (@config) {
-            my $force = $room->{force};
-            next unless $jobName =~ /^$room->{jobs}$/;
-
-            # If build is cancelled or aborted, do not send email.
-            next if ! $force && ($build->buildstatus == 4 || $build->buildstatus == 3);
-
-            # If there is a previous (that is not cancelled or aborted) build
-            # with same buildstatus, do not send email.
-            next if ! $force && defined $prevBuild && ($build->buildstatus == $prevBuild->buildstatus);
-
-            $rooms{$room->{room}} //= { room => $room, builds => [] };
-            push @{$rooms{$room->{room}}->{builds}}, $build;
-        }
-    }
-
-    return if scalar keys %rooms == 0;
-
-    my ($authors, $nrCommits) = getResponsibleAuthors($topbuild, $self->{plugins});
-
-    # Send a message to each room.
-    foreach my $roomId (keys %rooms) {
-        my $room = $rooms{$roomId};
-        my @deps = grep { $_->id != $topbuild->id } @{$room->{builds}};
-
-        my $img =
-            $topbuild->buildstatus == 0 ? "$baseurl/static/images/checkmark_16.png" :
-            $topbuild->buildstatus == 2 ? "$baseurl/static/images/dependency_16.png" :
-            $topbuild->buildstatus == 4 ? "$baseurl/static/images/cancelled_16.png" :
-            "$baseurl/static/images/error_16.png";
-
-        my $msg = "";
-        $msg .= "<img src='$img'/> ";
-        $msg .= "Job <a href='$baseurl/job/${\$topbuild->jobset->get_column('project')}/${\$topbuild->jobset->get_column('name')}/${\$topbuild->get_column('job')}'>${\showJobName($topbuild)}</a>";
-        $msg .= " (and ${\scalar @deps} others)" if scalar @deps > 0;
-        $msg .= ": <a href='$baseurl/build/${\$topbuild->id}'>" . showStatus($topbuild) . "</a>";
-
-        if (scalar keys %{$authors} > 0) {
-            # FIXME: HTML escaping
-            my @x = map { "<a href='mailto:$authors->{$_}'>$_</a>" } (sort keys %{$authors});
-            $msg .= ", likely due to ";
-            $msg .= "$nrCommits commits by " if $nrCommits > 1;
-            $msg .= join(" or ", scalar @x > 1 ? join(", ", @x[0..scalar @x - 2]) : (), $x[-1]);
-        }
-
-        print STDERR "sending hipchat notification to room $roomId: $msg\n";
-
-        my $ua = LWP::UserAgent->new();
-        my $resp = $ua->post('https://api.hipchat.com/v1/rooms/message?format=json&auth_token=' . $room->{room}->{token}, {
-            room_id => $roomId,
-            from => 'Hydra',
-            message => $msg,
-            message_format => 'html',
-            notify => $room->{room}->{notify} || 0,
-            color => $topbuild->buildstatus == 0 ? 'green' : 'red' });
-
-        print STDERR $resp->status_line, ": ", $resp->decoded_content,"\n" if !$resp->is_success;
-    }
-}
-
-1;

From 1dbc100837dcc705c6d2451603da17f14e72b606 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 6 Sep 2022 16:18:52 +0200
Subject: [PATCH 69/76] flake.lock: Update
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Flake lock file updates:

• Updated input 'nix':
    'github:NixOS/nix/b9cf655150b52d071c85a337cb5db96e735fa64a' (2022-07-11)
  → 'github:NixOS/nix/11e45768b34fdafdcf019ddbd337afa16127ff0f' (2022-08-27)
• Updated input 'nix/nixpkgs':
    'github:NixOS/nixpkgs/2fa57ed190fd6c7c746319444f34b5917666e5c1' (2022-05-31)
  → 'github:NixOS/nixpkgs/365e1b3a859281cf11b94f87231adeabbdd878a2' (2022-07-13)
---
 flake.lock | 14 +++++++-------
 flake.nix  |  2 +-
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/flake.lock b/flake.lock
index 012d2407..b41b843a 100644
--- a/flake.lock
+++ b/flake.lock
@@ -23,27 +23,27 @@
         "nixpkgs-regression": "nixpkgs-regression"
       },
       "locked": {
-        "lastModified": 1657569404,
-        "narHash": "sha256-zJONRtGALmYifrWKzcH6MMfSKLxeuW2iqG13500OrY4=",
+        "lastModified": 1661606874,
+        "narHash": "sha256-9+rpYzI+SmxJn+EbYxjGv68Ucp22bdFUSy/4LkHkkDQ=",
         "owner": "NixOS",
         "repo": "nix",
-        "rev": "b9cf655150b52d071c85a337cb5db96e735fa64a",
+        "rev": "11e45768b34fdafdcf019ddbd337afa16127ff0f",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
-        "ref": "2.10.0",
+        "ref": "2.11.0",
         "repo": "nix",
         "type": "github"
       }
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1653988320,
-        "narHash": "sha256-ZaqFFsSDipZ6KVqriwM34T739+KLYJvNmCWzErjAg7c=",
+        "lastModified": 1657693803,
+        "narHash": "sha256-G++2CJ9u0E7NNTAi9n5G8TdDmGJXcIjkJ3NF8cetQB8=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "2fa57ed190fd6c7c746319444f34b5917666e5c1",
+        "rev": "365e1b3a859281cf11b94f87231adeabbdd878a2",
         "type": "github"
       },
       "original": {
diff --git a/flake.nix b/flake.nix
index b731e684..2e413277 100644
--- a/flake.nix
+++ b/flake.nix
@@ -2,7 +2,7 @@
   description = "A Nix-based continuous build system";
 
   inputs.nixpkgs.follows = "nix/nixpkgs";
-  inputs.nix.url = "github:NixOS/nix/2.10.0";
+  inputs.nix.url = "github:NixOS/nix/2.11.0";
 
   outputs = { self, nixpkgs, nix }:
     let

From b16470c54434966b227de93c5229811c9c2afab4 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 6 Sep 2022 19:22:20 +0200
Subject: [PATCH 70/76] nix flake info -> nix flake metadata

This gets rid of a deprecation warning.
---
 src/script/hydra-eval-jobset | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/script/hydra-eval-jobset b/src/script/hydra-eval-jobset
index 8804b2bb..c6f6c275 100755
--- a/src/script/hydra-eval-jobset
+++ b/src/script/hydra-eval-jobset
@@ -640,8 +640,8 @@ sub checkJobsetWrapped {
     my $flakeRef = $jobset->flake;
     if (defined $flakeRef) {
         (my $res, my $json, my $stderr) = captureStdoutStderr(
-            600, "nix", "flake", "info", "--tarball-ttl", 0, "--json", "--", $flakeRef);
-        die "'nix flake info' returned " . ($res & 127 ? "signal $res" : "exit code " . ($res >> 8))
+            600, "nix", "flake", "metadata", "--refresh", "--json", "--", $flakeRef);
+        die "'nix flake metadata' returned " . ($res & 127 ? "signal $res" : "exit code " . ($res >> 8))
             . ":\n" . ($stderr ? decode("utf-8", $stderr) : "(no output)\n")
             if $res;
         $flakeRef = decode_json($json)->{'url'};

From 44e1efff7f51cd884f3924fdb086f3268b753972 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 8 Sep 2022 11:51:13 +0200
Subject: [PATCH 71/76] Send the right nix-serve client version

We were using protocol version 6 but requesting version 4. The only
reason that this worked was because of a broken version check in
'nix-store --serve'. That was fixed in
https://github.com/NixOS/nix/commit/c2d74569269d9642448a701aa6efe08ec0fe6f00,
which had the side-effect of breaking hydra-queue-runner.
---
 src/hydra-queue-runner/build-remote.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc
index 57a5f0df..21a6c331 100644
--- a/src/hydra-queue-runner/build-remote.cc
+++ b/src/hydra-queue-runner/build-remote.cc
@@ -234,7 +234,7 @@ void State::buildRemote(ref<Store> destStore,
         unsigned int remoteVersion;
 
         try {
-            to << SERVE_MAGIC_1 << 0x204;
+            to << SERVE_MAGIC_1 << 0x206;
             to.flush();
 
             unsigned int magic = readInt(from);

From d3fe4ffbf645b8fbc1110a30614a6871a23f0f23 Mon Sep 17 00:00:00 2001
From: Maximilian Bosch <maximilian@mbosch.me>
Date: Fri, 9 Sep 2022 00:38:46 +0200
Subject: [PATCH 72/76] Job: expose `closuresize` and `size` (output size in
 the UI) as prometheus metrics

---
 src/lib/Hydra/Controller/Job.pm | 22 +++++++++++++++++++++-
 t/Hydra/Controller/Job/builds.t |  6 ++++++
 2 files changed, 27 insertions(+), 1 deletion(-)

diff --git a/src/lib/Hydra/Controller/Job.pm b/src/lib/Hydra/Controller/Job.pm
index 378887ea..b392e8e1 100644
--- a/src/lib/Hydra/Controller/Job.pm
+++ b/src/lib/Hydra/Controller/Job.pm
@@ -69,7 +69,7 @@ sub prometheus : Chained('job') PathPart('prometheus') Args(0) {
 
     my $lastBuild = $c->stash->{jobset}->builds->find(
         { job => $c->stash->{job}, finished => 1 },
-        { order_by => 'id DESC', rows => 1, columns => [@buildListColumns] }
+        { order_by => 'id DESC', rows => 1, columns => ["stoptime", "buildstatus", "closuresize", "size"] }
     );
 
     $prometheus->new_counter(
@@ -92,6 +92,26 @@ sub prometheus : Chained('job') PathPart('prometheus') Args(0) {
         $c->stash->{job},
     )->inc($lastBuild->buildstatus > 0);
 
+    $prometheus->new_gauge(
+        name => "hydra_build_closure_size",
+        help => "Closure size of the last job's build in bytes",
+        labels => [ "project", "jobset", "job" ]
+    )->labels(
+        $c->stash->{project}->name,
+        $c->stash->{jobset}->name,
+        $c->stash->{job},
+    )->inc($lastBuild->closuresize);
+
+    $prometheus->new_gauge(
+        name => "hydra_build_output_size",
+        help => "Output size of the last job's build in bytes",
+        labels => [ "project", "jobset", "job" ]
+    )->labels(
+        $c->stash->{project}->name,
+        $c->stash->{jobset}->name,
+        $c->stash->{job},
+    )->inc($lastBuild->size);
+
     $c->stash->{'plain'} = { data => $prometheus->render };
     $c->forward('Hydra::View::Plain');
 }
diff --git a/t/Hydra/Controller/Job/builds.t b/t/Hydra/Controller/Job/builds.t
index e1097c3e..d933fbb8 100644
--- a/t/Hydra/Controller/Job/builds.t
+++ b/t/Hydra/Controller/Job/builds.t
@@ -55,6 +55,12 @@ subtest "/job/PROJECT/JOBSET/JOB/shield" => sub {
 subtest "/job/PROJECT/JOBSET/JOB/prometheus" => sub {
     my $response = request(GET '/job/' . $project->name . '/' . $jobset->name . '/' . $build->job . '/prometheus');
     ok($response->is_success, "The page showing the job's prometheus data returns 200.");
+    my $metrics = $response->content;
+
+    ok($metrics =~ m/hydra_job_failed\{.*\} 0/);
+    ok($metrics =~ m/hydra_job_completion_time\{.*\} [\d]+/);
+    ok($metrics =~ m/hydra_build_closure_size\{.*\} 96/);
+    ok($metrics =~ m/hydra_build_output_size\{.*\} 96/);
 };
 
 done_testing;

From 2ff67cbaccb8dcad239c7a92e7dc3d80e208e752 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= <joerg@thalheim.io>
Date: Tue, 27 Sep 2022 22:44:27 +0200
Subject: [PATCH 73/76] docs/hacking: update nixos module option for nix-daemon

---
 doc/manual/src/hacking.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/manual/src/hacking.md b/doc/manual/src/hacking.md
index 9d98b00c..72a74c82 100644
--- a/doc/manual/src/hacking.md
+++ b/doc/manual/src/hacking.md
@@ -92,7 +92,7 @@ On NixOS:
 
 ```nix
 {
-  nix.trustedUsers = [ "YOURUSER" ];
+  nix.settings.trusted-users = [ "YOURUSER" ];
 }
 ```
 

From 94d19e19725905c3d10acd28cce78268a53d5ff6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= <joerg@thalheim.io>
Date: Thu, 29 Sep 2022 20:42:57 +0200
Subject: [PATCH 74/76] hydra: fix localhost detection when protocol prefix are
 used

---
 src/hydra-queue-runner/state.hh | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/hydra-queue-runner/state.hh b/src/hydra-queue-runner/state.hh
index 47e74f55..55c99afc 100644
--- a/src/hydra-queue-runner/state.hh
+++ b/src/hydra-queue-runner/state.hh
@@ -6,6 +6,7 @@
 #include <map>
 #include <memory>
 #include <queue>
+#include <regex>
 
 #include <prometheus/counter.h>
 #include <prometheus/gauge.h>
@@ -293,7 +294,8 @@ struct Machine
 
     bool isLocalhost()
     {
-        return sshName == "localhost";
+        std::regex r("^(ssh://|ssh-ng://)?localhost$");
+        return std::regex_search(sshName, r);
     }
 };
 

From 940c0d7d978ce35c13d7d78c5fb0e9b55f17e397 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= <joerg@thalheim.io>
Date: Thu, 29 Sep 2022 21:22:01 +0200
Subject: [PATCH 75/76] fix deprecated chown syntax

---
 hydra-module.nix | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/hydra-module.nix b/hydra-module.nix
index b58be922..8e02dcbb 100644
--- a/hydra-module.nix
+++ b/hydra-module.nix
@@ -272,12 +272,12 @@ in
           ln -sf ${hydraConf} ${baseDir}/hydra.conf
 
           mkdir -m 0700 -p ${baseDir}/www
-          chown hydra-www.hydra ${baseDir}/www
+          chown hydra-www:hydra ${baseDir}/www
 
           mkdir -m 0700 -p ${baseDir}/queue-runner
           mkdir -m 0750 -p ${baseDir}/build-logs
           mkdir -m 0750 -p ${baseDir}/runcommand-logs
-          chown hydra-queue-runner.hydra \
+          chown hydra-queue-runner:hydra \
             ${baseDir}/queue-runner \
             ${baseDir}/build-logs \
             ${baseDir}/runcommand-logs
@@ -308,7 +308,7 @@ in
             rmdir /nix/var/nix/gcroots/per-user/hydra-www/hydra-roots
           fi
 
-          chown hydra.hydra ${cfg.gcRootsDir}
+          chown hydra:hydra ${cfg.gcRootsDir}
           chmod 2775 ${cfg.gcRootsDir}
         '';
         serviceConfig.ExecStart = "${cfg.package}/bin/hydra-init";

From c737bed42f73f3ee1be576f5f739d4fc3d713f40 Mon Sep 17 00:00:00 2001
From: MaxHearnden <MaxHearnden@users.noreply.github.com>
Date: Wed, 19 Oct 2022 16:24:01 +0100
Subject: [PATCH 76/76] Fix insecure advice in the manual

Bring the manual in line with the configuration documentation
---
 doc/manual/src/projects.md | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/doc/manual/src/projects.md b/doc/manual/src/projects.md
index 95174f1b..a399406d 100644
--- a/doc/manual/src/projects.md
+++ b/doc/manual/src/projects.md
@@ -378,13 +378,18 @@ This section describes how it can be implemented for `gitea`, but the approach f
 analogous:
 
 * [Obtain an API token for your user](https://docs.gitea.io/en-us/api-usage/#authentication)
-* Add it to your `hydra.conf` like this:
+* Add it to a file which only users in the hydra group can read like this: see [including files](configuration.md#including-files) for more information
+  ```
+  <gitea_authorization>
+    your_username=your_token
+  </gitea_authorization>
+  ```
+
+* Include the file in your `hydra.conf` like this:
   ``` nix
   {
     services.hydra-dev.extraConfig = ''
-      <gitea_authorization>
-      your_username=your_token
-      </gitea_authorization>
+      Include /path/to/secret/file
     '';
   }
   ```