tests: use like for testing regexes

This gives us better diagnostics when the test fails.

README.md

@@ -140,7 +140,7 @@ You can also interface with Hydra through a JSON API. The API is defined in [hyd
 ## Additional Resources
 
 - [Hydra User's Guide](https://nixos.org/hydra/manual/)
-- [Hydra on the NixOS Wiki](https://nixos.wiki/wiki/Hydra)
+- [Hydra on the NixOS Wiki](https://wiki.nixos.org/wiki/Hydra)
 - [hydra-cli](https://github.com/nlewo/hydra-cli)
 - [Peter Simons - Hydra: Setting up your own build farm (NixOS)](https://www.youtube.com/watch?v=RXV0Y5Bn-QQ)
 

doc/manual/src/hacking.md

@@ -30,6 +30,8 @@ foreman:
 $ foreman start
 ```
 
+The Hydra interface will be available on port 63333, with an admin user named "alice" with password "foobar"
+
 You can run just the Hydra web server in your source tree as follows:
 
 ```console

doc/manual/src/webhooks.md

@@ -1,9 +1,12 @@
 # Webhooks
 
-Hydra can be notified by github's webhook to trigger a new evaluation when a
+Hydra can be notified by github or gitea with webhooks to trigger a new evaluation when a
 jobset has a github repo in its input.
-To set up a github webhook go to `https://github.com/<yourhandle>/<yourrepo>/settings` and in the `Webhooks` tab
-click on `Add webhook`.
+
+## GitHub
+
+To set up a webhook for a GitHub repository go to `https://github.com/<yourhandle>/<yourrepo>/settings`
+and in the `Webhooks` tab click on `Add webhook`.
 
 - In `Payload URL` fill in `https://<your-hydra-domain>/api/push-github`.
 - In `Content type` switch to `application/json`.
@@ -11,3 +14,14 @@ click on `Add webhook`.
 - For `Which events would you like to trigger this webhook?` keep the default option for events on `Just the push event.`.
 
 Then add the hook with `Add webhook`.
+
+## Gitea
+
+To set up a webhook for a Gitea repository go to the settings of the repository in your Gitea instance
+and in the `Webhooks` tab click on `Add Webhook` and choose `Gitea` in the drop down.
+
+- In `Target URL` fill in `https://<your-hydra-domain>/api/push-gitea`.
+- Keep HTTP method `POST`, POST Content Type `application/json` and Trigger On `Push Events`.
+- Change the branch filter to match the git branch hydra builds.
+
+Then add the hook with `Add webhook`.

flake.lock

@@ -16,6 +16,42 @@
         "type": "github"
       }
     },
+    "flake-parts": {
+      "inputs": {
+        "nixpkgs-lib": [
+          "nix",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1712014858,
+        "narHash": "sha256-sB4SWl2lX95bExY2gMFG5HIzvva5AVMJd4Igm+GpZNw=",
+        "owner": "hercules-ci",
+        "repo": "flake-parts",
+        "rev": "9126214d0a59633752a136528f5f3b9aa8565b7d",
+        "type": "github"
+      },
+      "original": {
+        "owner": "hercules-ci",
+        "repo": "flake-parts",
+        "type": "github"
+      }
+    },
+    "flake-utils": {
+      "locked": {
+        "lastModified": 1667395993,
+        "narHash": "sha256-nuEHfE/LcWyuSWnS8t12N1wc105Qtau+/OdUAjtQ0rA=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "5aed5285a952e0b949eb3ba02c12fa4fcfef535f",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
     "libgit2": {
       "flake": false,
       "locked": {
@@ -35,55 +71,41 @@
     "nix": {
       "inputs": {
         "flake-compat": "flake-compat",
+        "flake-parts": "flake-parts",
         "libgit2": "libgit2",
         "nixpkgs": [
           "nixpkgs"
         ],
-        "nixpkgs-regression": "nixpkgs-regression"
+        "nixpkgs-regression": "nixpkgs-regression",
+        "pre-commit-hooks": "pre-commit-hooks"
       },
       "locked": {
-        "lastModified": 1715805674,
-        "narHash": "sha256-0CIBMECsA3ISJZrJcOTzi6wa3QENTKGLtOpYIOoxwxo=",
+        "lastModified": 1713874370,
+        "narHash": "sha256-gW1mO/CvsQQ5gvgiwzxsGhPFI/tx30NING+qgF5Do0s=",
         "owner": "NixOS",
         "repo": "nix",
-        "rev": "ab48ea416a203e9ccefb70aa634e27477e4c1ac4",
+        "rev": "1c8150ac312b5f9ba1b3f6768ff43b09867e5883",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
-        "ref": "2.20-maintenance",
+        "ref": "2.22-maintenance",
         "repo": "nix",
         "type": "github"
       }
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1705033721,
-        "narHash": "sha256-K5eJHmL1/kev6WuqyqqbS1cdNnSidIZ3jeqJ7GbrYnQ=",
+        "lastModified": 1712848736,
+        "narHash": "sha256-CzZwhqyLlebljv1zFS2KWVH/3byHND0LfaO1jKsGuVo=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "a1982c92d8980a0114372973cbdfe0a307f1bdea",
+        "rev": "1d6a23f11e44d0fb64b3237569b87658a9eb5643",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
-        "ref": "nixos-23.05-small",
-        "repo": "nixpkgs",
-        "type": "github"
-      }
-    },
-    "nixpkgs-for-fileset": {
-      "locked": {
-        "lastModified": 1706098335,
-        "narHash": "sha256-r3dWjT8P9/Ah5m5ul4WqIWD8muj5F+/gbCdjiNVBKmU=",
-        "owner": "NixOS",
-        "repo": "nixpkgs",
-        "rev": "a77ab169a83a4175169d78684ddd2e54486ac651",
-        "type": "github"
-      },
-      "original": {
-        "owner": "NixOS",
-        "ref": "nixos-23.11",
+        "ref": "nixos-23.11-small",
         "repo": "nixpkgs",
         "type": "github"
       }
@@ -104,11 +126,42 @@
         "type": "github"
       }
     },
+    "pre-commit-hooks": {
+      "inputs": {
+        "flake-compat": [
+          "nix"
+        ],
+        "flake-utils": "flake-utils",
+        "gitignore": [
+          "nix"
+        ],
+        "nixpkgs": [
+          "nix",
+          "nixpkgs"
+        ],
+        "nixpkgs-stable": [
+          "nix",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1712897695,
+        "narHash": "sha256-nMirxrGteNAl9sWiOhoN5tIHyjBbVi5e2tgZUgZlK3Y=",
+        "owner": "cachix",
+        "repo": "pre-commit-hooks.nix",
+        "rev": "40e6053ecb65fcbf12863338a6dcefb3f55f1bf8",
+        "type": "github"
+      },
+      "original": {
+        "owner": "cachix",
+        "repo": "pre-commit-hooks.nix",
+        "type": "github"
+      }
+    },
     "root": {
       "inputs": {
         "nix": "nix",
-        "nixpkgs": "nixpkgs",
-        "nixpkgs-for-fileset": "nixpkgs-for-fileset"
+        "nixpkgs": "nixpkgs"
       }
     }
   },

flake.nix

@@ -1,16 +1,11 @@
 {
   description = "A Nix-based continuous build system";
 
-  inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.05-small";
-  inputs.nix.url = "github:NixOS/nix/2.20-maintenance";
+  inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.11-small";
+  inputs.nix.url = "github:NixOS/nix/2.22-maintenance";
   inputs.nix.inputs.nixpkgs.follows = "nixpkgs";
 
-  # TODO get rid of this once https://github.com/NixOS/nix/pull/9546 is
-  # mered and we upgrade or Nix, so the main `nixpkgs` input is at least
-  # 23.11 and has `lib.fileset`.
-  inputs.nixpkgs-for-fileset.url = "github:NixOS/nixpkgs/nixos-23.11";
-
-  outputs = { self, nixpkgs, nix, nixpkgs-for-fileset }:
+  outputs = { self, nixpkgs, nix }:
     let
       systems = [ "x86_64-linux" "aarch64-linux" ];
       forEachSystem = nixpkgs.lib.genAttrs systems;
@@ -22,52 +17,13 @@
         overlays = overlayList;
       });
 
-      # NixOS configuration used for VM tests.
-      hydraServer =
-        { config, pkgs, ... }:
-        {
-          imports = [ self.nixosModules.hydraTest ];
-
-          virtualisation.memorySize = 1024;
-          virtualisation.writableStore = true;
-
-          environment.systemPackages = [ pkgs.perlPackages.LWP pkgs.perlPackages.JSON ];
-
-          nix = {
-            # Without this nix tries to fetch packages from the default
-            # cache.nixos.org which is not reachable from this sandboxed NixOS test.
-            binaryCaches = [ ];
-          };
-        };
-
     in
     rec {
 
       # A Nixpkgs overlay that provides a 'hydra' package.
       overlays.default = final: prev: {
-
-        # Add LDAP dependencies that aren't currently found within nixpkgs.
-        perlPackages = prev.perlPackages // {
-
-          PrometheusTiny = final.perlPackages.buildPerlPackage {
-            pname = "Prometheus-Tiny";
-            version = "0.007";
-            src = final.fetchurl {
-              url = "mirror://cpan/authors/id/R/RO/ROBN/Prometheus-Tiny-0.007.tar.gz";
-              sha256 = "0ef8b226a2025cdde4df80129dd319aa29e884e653c17dc96f4823d985c028ec";
-            };
-            buildInputs = with final.perlPackages; [ HTTPMessage Plack TestException ];
-            meta = {
-              homepage = "https://github.com/robn/Prometheus-Tiny";
-              description = "A tiny Prometheus client";
-              license = with final.lib.licenses; [ artistic1 gpl1Plus ];
-            };
-          };
-
-        };
-
         hydra = final.callPackage ./package.nix {
-          inherit (nixpkgs-for-fileset.lib) fileset;
+          inherit (nixpkgs.lib) fileset;
           rawSrc = self;
         };
       };
@@ -93,282 +49,9 @@
               echo "doc manual $out/share/doc/hydra" >> $out/nix-support/hydra-build-products
             '');
 
-        tests.install = forEachSystem (system:
-          with import (nixpkgs + "/nixos/lib/testing-python.nix") { inherit system; };
-          simpleTest {
-            name = "hydra-install";
-            nodes.machine = hydraServer;
-            testScript =
-              ''
-                machine.wait_for_job("hydra-init")
-                machine.wait_for_job("hydra-server")
-                machine.wait_for_job("hydra-evaluator")
-                machine.wait_for_job("hydra-queue-runner")
-                machine.wait_for_open_port(3000)
-                machine.succeed("curl --fail http://localhost:3000/")
-              '';
-          });
-
-        tests.notifications = forEachSystem (system:
-          let pkgs = pkgsBySystem.${system}; in
-          with import (nixpkgs + "/nixos/lib/testing-python.nix") { inherit system; };
-          simpleTest {
-            name = "hydra-notifications";
-            nodes.machine = { pkgs, ... }: {
-              imports = [ hydraServer ];
-              services.hydra-dev.extraConfig = ''
-                <influxdb>
-                  url = http://127.0.0.1:8086
-                  db = hydra
-                </influxdb>
-              '';
-              services.influxdb.enable = true;
-            };
-            testScript = ''
-              machine.wait_for_job("hydra-init")
-
-              # Create an admin account and some other state.
-              machine.succeed(
-                  """
-                      su - hydra -c "hydra-create-user root --email-address 'alice@example.org' --password foobar --role admin"
-                      mkdir /run/jobset
-                      chmod 755 /run/jobset
-                      cp ${./t/jobs/api-test.nix} /run/jobset/default.nix
-                      chmod 644 /run/jobset/default.nix
-                      chown -R hydra /run/jobset
-              """
-              )
-
-              # Wait until InfluxDB can receive web requests
-              machine.wait_for_job("influxdb")
-              machine.wait_for_open_port(8086)
-
-              # Create an InfluxDB database where hydra will write to
-              machine.succeed(
-                  "curl -XPOST 'http://127.0.0.1:8086/query' "
-                  + "--data-urlencode 'q=CREATE DATABASE hydra'"
-              )
-
-              # Wait until hydra-server can receive HTTP requests
-              machine.wait_for_job("hydra-server")
-              machine.wait_for_open_port(3000)
-
-              # Setup the project and jobset
-              machine.succeed(
-                  "su - hydra -c 'perl -I ${pkgs.hydra.perlDeps}/lib/perl5/site_perl ${./t/setup-notifications-jobset.pl}' >&2"
-              )
-
-              # Wait until hydra has build the job and
-              # the InfluxDBNotification plugin uploaded its notification to InfluxDB
-              machine.wait_until_succeeds(
-                  "curl -s -H 'Accept: application/csv' "
-                  + "-G 'http://127.0.0.1:8086/query?db=hydra' "
-                  + "--data-urlencode 'q=SELECT * FROM hydra_build_status' | grep success"
-              )
-            '';
-          });
-
-        tests.gitea = forEachSystem (system:
-          let pkgs = pkgsBySystem.${system}; in
-          with import (nixpkgs + "/nixos/lib/testing-python.nix") { inherit system; };
-          makeTest {
-            name = "hydra-gitea";
-            nodes.machine = { pkgs, ... }: {
-              imports = [ hydraServer ];
-              services.hydra-dev.extraConfig = ''
-                <gitea_authorization>
-                root=d7f16a3412e01a43a414535b16007c6931d3a9c7
-                </gitea_authorization>
-              '';
-              nixpkgs.config.permittedInsecurePackages = [ "gitea-1.19.4" ];
-              nix = {
-                settings.substituters = [ ];
-              };
-              services.gitea = {
-                enable = true;
-                database.type = "postgres";
-                disableRegistration = true;
-                httpPort = 3001;
-              };
-              services.openssh.enable = true;
-              environment.systemPackages = with pkgs; [ gitea git jq gawk ];
-              networking.firewall.allowedTCPPorts = [ 3000 ];
-            };
-            skipLint = true;
-            testScript =
-              let
-                scripts.mktoken = pkgs.writeText "token.sql" ''
-                  INSERT INTO access_token (id, uid, name, created_unix, updated_unix, token_hash, token_salt, token_last_eight, scope) VALUES (1, 1, 'hydra', 1617107360, 1617107360, 'a930f319ca362d7b49a4040ac0af74521c3a3c3303a86f327b01994430672d33b6ec53e4ea774253208686c712495e12a486', 'XRjWE9YW0g', '31d3a9c7', 'all');
-                '';
-
-                scripts.git-setup = pkgs.writeShellScript "setup.sh" ''
-                  set -x
-                  mkdir -p /tmp/repo $HOME/.ssh
-                  cat ${snakeoilKeypair.privkey} > $HOME/.ssh/privk
-                  chmod 0400 $HOME/.ssh/privk
-                  git -C /tmp/repo init
-                  cp ${smallDrv} /tmp/repo/jobset.nix
-                  git -C /tmp/repo add .
-                  git config --global user.email test@localhost
-                  git config --global user.name test
-                  git -C /tmp/repo commit -m 'Initial import'
-                  git -C /tmp/repo remote add origin gitea@machine:root/repo
-                  GIT_SSH_COMMAND='ssh -i $HOME/.ssh/privk -o StrictHostKeyChecking=no' \
-                    git -C /tmp/repo push origin master
-                  git -C /tmp/repo log >&2
-                '';
-
-                scripts.hydra-setup = pkgs.writeShellScript "hydra.sh" ''
-                  set -x
-                  su -l hydra -c "hydra-create-user root --email-address \
-                    'alice@example.org' --password foobar --role admin"
-
-                  URL=http://localhost:3000
-                  USERNAME="root"
-                  PASSWORD="foobar"
-                  PROJECT_NAME="trivial"
-                  JOBSET_NAME="trivial"
-                  mycurl() {
-                    curl --referer $URL -H "Accept: application/json" \
-                      -H "Content-Type: application/json" $@
-                  }
-
-                  cat >data.json <<EOF
-                  { "username": "$USERNAME", "password": "$PASSWORD" }
-                  EOF
-                  mycurl -X POST -d '@data.json' $URL/login -c hydra-cookie.txt
-
-                  cat >data.json <<EOF
-                  {
-                    "displayname":"Trivial",
-                    "enabled":"1",
-                    "visible":"1"
-                  }
-                  EOF
-                  mycurl --silent -X PUT $URL/project/$PROJECT_NAME \
-                    -d @data.json -b hydra-cookie.txt
-
-                  cat >data.json <<EOF
-                  {
-                    "description": "Trivial",
-                    "checkinterval": "60",
-                    "enabled": "1",
-                    "visible": "1",
-                    "keepnr": "1",
-                    "enableemail": true,
-                    "emailoverride": "hydra@localhost",
-                    "type": 0,
-                    "nixexprinput": "git",
-                    "nixexprpath": "jobset.nix",
-                    "inputs": {
-                      "git": {"value": "http://localhost:3001/root/repo.git", "type": "git"},
-                      "gitea_repo_name": {"value": "repo", "type": "string"},
-                      "gitea_repo_owner": {"value": "root", "type": "string"},
-                      "gitea_status_repo": {"value": "git", "type": "string"},
-                      "gitea_http_url": {"value": "http://localhost:3001", "type": "string"}
-                    }
-                  }
-                  EOF
-
-                  mycurl --silent -X PUT $URL/jobset/$PROJECT_NAME/$JOBSET_NAME \
-                    -d @data.json -b hydra-cookie.txt
-                '';
-
-                api_token = "d7f16a3412e01a43a414535b16007c6931d3a9c7";
-
-                snakeoilKeypair = {
-                  privkey = pkgs.writeText "privkey.snakeoil" ''
-                    -----BEGIN EC PRIVATE KEY-----
-                    MHcCAQEEIHQf/khLvYrQ8IOika5yqtWvI0oquHlpRLTZiJy5dRJmoAoGCCqGSM49
-                    AwEHoUQDQgAEKF0DYGbBwbj06tA3fd/+yP44cvmwmHBWXZCKbS+RQlAKvLXMWkpN
-                    r1lwMyJZoSGgBHoUahoYjTh9/sJL7XLJtA==
-                    -----END EC PRIVATE KEY-----
-                  '';
-
-                  pubkey = pkgs.lib.concatStrings [
-                    "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHA"
-                    "yNTYAAABBBChdA2BmwcG49OrQN33f/sj+OHL5sJhwVl2Qim0vkUJQCry1zFpKTa"
-                    "9ZcDMiWaEhoAR6FGoaGI04ff7CS+1yybQ= sakeoil"
-                  ];
-                };
-
-                smallDrv = pkgs.writeText "jobset.nix" ''
-                  { trivial = builtins.derivation {
-                      name = "trivial";
-                      system = "${system}";
-                      builder = "/bin/sh";
-                      allowSubstitutes = false;
-                      preferLocalBuild = true;
-                      args = ["-c" "echo success > $out; exit 0"];
-                    };
-                   }
-                '';
-              in
-              ''
-                import json
-
-                machine.start()
-                machine.wait_for_unit("multi-user.target")
-                machine.wait_for_open_port(3000)
-                machine.wait_for_open_port(3001)
-
-                machine.succeed(
-                    "su -l gitea -c 'GITEA_WORK_DIR=/var/lib/gitea gitea admin user create "
-                    + "--username root --password root --email test@localhost'"
-                )
-                machine.succeed("su -l postgres -c 'psql gitea < ${scripts.mktoken}'")
-
-                machine.succeed(
-                    "curl --fail -X POST http://localhost:3001/api/v1/user/repos "
-                    + "-H 'Accept: application/json' -H 'Content-Type: application/json' "
-                    + f"-H 'Authorization: token ${api_token}'"
-                    + ' -d \'{"auto_init":false, "description":"string", "license":"mit", "name":"repo", "private":false}\'''
-                )
-
-                machine.succeed(
-                    "curl --fail -X POST http://localhost:3001/api/v1/user/keys "
-                    + "-H 'Accept: application/json' -H 'Content-Type: application/json' "
-                    + f"-H 'Authorization: token ${api_token}'"
-                    + ' -d \'{"key":"${snakeoilKeypair.pubkey}","read_only":true,"title":"SSH"}\'''
-                )
-
-                machine.succeed(
-                    "${scripts.git-setup}"
-                )
-
-                machine.succeed(
-                    "${scripts.hydra-setup}"
-                )
-
-                machine.wait_until_succeeds(
-                    'curl -Lf -s http://localhost:3000/build/1 -H "Accept: application/json" '
-                    + '|  jq .buildstatus | xargs test 0 -eq'
-                )
-
-                data = machine.succeed(
-                    'curl -Lf -s "http://localhost:3001/api/v1/repos/root/repo/statuses/$(cd /tmp/repo && git show | head -n1 | awk "{print \\$2}")" '
-                    + "-H 'Accept: application/json' -H 'Content-Type: application/json' "
-                    + f"-H 'Authorization: token ${api_token}'"
-                )
-
-                response = json.loads(data)
-
-                assert len(response) == 2, "Expected exactly three status updates for latest commit (queued, finished)!"
-                assert response[0]['status'] == "success", "Expected finished status to be success!"
-                assert response[1]['status'] == "pending", "Expected queued status to be pending!"
-
-                machine.shutdown()
-              '';
-          });
-
-        tests.validate-openapi = forEachSystem (system:
-          let pkgs = pkgsBySystem.${system}; in
-          pkgs.runCommand "validate-openapi"
-          { buildInputs = [ pkgs.openapi-generator-cli ]; }
-          ''
-            openapi-generator-cli validate -i ${./hydra-api.yaml}
-            touch $out
-          '');
+        tests = import ./nixos-tests.nix {
+          inherit forEachSystem nixpkgs pkgsBySystem nixosModules;
+        };
 
         container = nixosConfigurations.container.config.system.build.toplevel;
       };
@@ -392,6 +75,8 @@
         system = "x86_64-linux";
         modules =
           [
+            self.nixosModules.hydra
+            self.nixosModules.overlayNixpkgsForThisHydra
             self.nixosModules.hydraTest
             self.nixosModules.hydraProxy
             {

nixos-modules/default.nix

@@ -1,14 +1,14 @@
 { overlays }:
 
-rec {
-  hydra = {
-    imports = [ ./hydra.nix ];
+{
+  hydra = import ./hydra.nix;
+
+  overlayNixpkgsForThisHydra = { pkgs, ... }: {
     nixpkgs = { inherit overlays; };
+    services.hydra.package = pkgs.hydra;
   };
 
   hydraTest = { pkgs, ... }: {
-    imports = [ hydra ];
-
     services.hydra-dev.enable = true;
     services.hydra-dev.hydraURL = "http://hydra.example.org";
     services.hydra-dev.notificationSender = "admin@hydra.example.org";
@@ -16,7 +16,7 @@ rec {
     systemd.services.hydra-send-stats.enable = false;
 
     services.postgresql.enable = true;
-    services.postgresql.package = pkgs.postgresql_11;
+    services.postgresql.package = pkgs.postgresql_12;
 
     # The following is to work around the following error from hydra-server:
     #   [error] Caught exception in engine "Cannot determine local time zone"

nixos-modules/hydra.nix

@@ -68,7 +68,7 @@ in
 
       package = mkOption {
         type = types.path;
-        default = pkgs.hydra;
+        default = pkgs.hydra_unstable;
         defaultText = literalExpression "pkgs.hydra";
         description = "The Hydra package.";
       };
@@ -233,7 +233,7 @@ in
       gc-keep-outputs = true;
       gc-keep-derivations = true;
     };
-    
+
     services.hydra-dev.extraConfig =
       ''
         using_frontend_proxy = 1

nixos-tests.nix

@@ -0,0 +1,309 @@
+{ forEachSystem, nixpkgs, pkgsBySystem, nixosModules }:
+
+let
+  # NixOS configuration used for VM tests.
+  hydraServer =
+    { config, pkgs, ... }:
+    {
+      imports = [
+        nixosModules.hydra
+        nixosModules.overlayNixpkgsForThisHydra
+        nixosModules.hydraTest
+      ];
+
+      virtualisation.memorySize = 1024;
+      virtualisation.writableStore = true;
+
+      environment.systemPackages = [ pkgs.perlPackages.LWP pkgs.perlPackages.JSON ];
+
+      nix = {
+        # Without this nix tries to fetch packages from the default
+        # cache.nixos.org which is not reachable from this sandboxed NixOS test.
+        settings.substituters = [ ];
+      };
+    };
+
+in
+
+{
+
+  install = forEachSystem (system:
+    with import (nixpkgs + "/nixos/lib/testing-python.nix") { inherit system; };
+    simpleTest {
+      name = "hydra-install";
+      nodes.machine = hydraServer;
+      testScript =
+        ''
+          machine.wait_for_job("hydra-init")
+          machine.wait_for_job("hydra-server")
+          machine.wait_for_job("hydra-evaluator")
+          machine.wait_for_job("hydra-queue-runner")
+          machine.wait_for_open_port(3000)
+          machine.succeed("curl --fail http://localhost:3000/")
+        '';
+    });
+
+  notifications = forEachSystem (system:
+    let pkgs = pkgsBySystem.${system}; in
+    with import (nixpkgs + "/nixos/lib/testing-python.nix") { inherit system; };
+    simpleTest {
+      name = "hydra-notifications";
+      nodes.machine = { pkgs, ... }: {
+        imports = [ hydraServer ];
+        services.hydra-dev.extraConfig = ''
+          <influxdb>
+            url = http://127.0.0.1:8086
+            db = hydra
+          </influxdb>
+        '';
+        services.influxdb.enable = true;
+      };
+      testScript = ''
+        machine.wait_for_job("hydra-init")
+
+        # Create an admin account and some other state.
+        machine.succeed(
+            """
+                su - hydra -c "hydra-create-user root --email-address 'alice@example.org' --password foobar --role admin"
+                mkdir /run/jobset
+                chmod 755 /run/jobset
+                cp ${./t/jobs/api-test.nix} /run/jobset/default.nix
+                chmod 644 /run/jobset/default.nix
+                chown -R hydra /run/jobset
+        """
+        )
+
+        # Wait until InfluxDB can receive web requests
+        machine.wait_for_job("influxdb")
+        machine.wait_for_open_port(8086)
+
+        # Create an InfluxDB database where hydra will write to
+        machine.succeed(
+            "curl -XPOST 'http://127.0.0.1:8086/query' "
+            + "--data-urlencode 'q=CREATE DATABASE hydra'"
+        )
+
+        # Wait until hydra-server can receive HTTP requests
+        machine.wait_for_job("hydra-server")
+        machine.wait_for_open_port(3000)
+
+        # Setup the project and jobset
+        machine.succeed(
+            "su - hydra -c 'perl -I ${pkgs.hydra.perlDeps}/lib/perl5/site_perl ${./t/setup-notifications-jobset.pl}' >&2"
+        )
+
+        # Wait until hydra has build the job and
+        # the InfluxDBNotification plugin uploaded its notification to InfluxDB
+        machine.wait_until_succeeds(
+            "curl -s -H 'Accept: application/csv' "
+            + "-G 'http://127.0.0.1:8086/query?db=hydra' "
+            + "--data-urlencode 'q=SELECT * FROM hydra_build_status' | grep success"
+        )
+      '';
+    });
+
+  gitea = forEachSystem (system:
+    let pkgs = pkgsBySystem.${system}; in
+    with import (nixpkgs + "/nixos/lib/testing-python.nix") { inherit system; };
+    makeTest {
+      name = "hydra-gitea";
+      nodes.machine = { pkgs, ... }: {
+        imports = [ hydraServer ];
+        services.hydra-dev.extraConfig = ''
+          <gitea_authorization>
+          root=d7f16a3412e01a43a414535b16007c6931d3a9c7
+          </gitea_authorization>
+        '';
+        nixpkgs.config.permittedInsecurePackages = [ "gitea-1.19.4" ];
+        nix = {
+          settings.substituters = [ ];
+        };
+        services.gitea = {
+          enable = true;
+          database.type = "postgres";
+          settings = {
+            service.DISABLE_REGISTRATION = true;
+            server.HTTP_PORT = 3001;
+          };
+        };
+        services.openssh.enable = true;
+        environment.systemPackages = with pkgs; [ gitea git jq gawk ];
+        networking.firewall.allowedTCPPorts = [ 3000 ];
+      };
+      skipLint = true;
+      testScript =
+        let
+          scripts.mktoken = pkgs.writeText "token.sql" ''
+            INSERT INTO access_token (id, uid, name, created_unix, updated_unix, token_hash, token_salt, token_last_eight, scope) VALUES (1, 1, 'hydra', 1617107360, 1617107360, 'a930f319ca362d7b49a4040ac0af74521c3a3c3303a86f327b01994430672d33b6ec53e4ea774253208686c712495e12a486', 'XRjWE9YW0g', '31d3a9c7', 'all');
+          '';
+
+          scripts.git-setup = pkgs.writeShellScript "setup.sh" ''
+            set -x
+            mkdir -p /tmp/repo $HOME/.ssh
+            cat ${snakeoilKeypair.privkey} > $HOME/.ssh/privk
+            chmod 0400 $HOME/.ssh/privk
+            git -C /tmp/repo init
+            cp ${smallDrv} /tmp/repo/jobset.nix
+            git -C /tmp/repo add .
+            git config --global user.email test@localhost
+            git config --global user.name test
+            git -C /tmp/repo commit -m 'Initial import'
+            git -C /tmp/repo remote add origin gitea@machine:root/repo
+            GIT_SSH_COMMAND='ssh -i $HOME/.ssh/privk -o StrictHostKeyChecking=no' \
+              git -C /tmp/repo push origin master
+            git -C /tmp/repo log >&2
+          '';
+
+          scripts.hydra-setup = pkgs.writeShellScript "hydra.sh" ''
+            set -x
+            su -l hydra -c "hydra-create-user root --email-address \
+              'alice@example.org' --password foobar --role admin"
+
+            URL=http://localhost:3000
+            USERNAME="root"
+            PASSWORD="foobar"
+            PROJECT_NAME="trivial"
+            JOBSET_NAME="trivial"
+            mycurl() {
+              curl --referer $URL -H "Accept: application/json" \
+                -H "Content-Type: application/json" $@
+            }
+
+            cat >data.json <<EOF
+            { "username": "$USERNAME", "password": "$PASSWORD" }
+            EOF
+            mycurl -X POST -d '@data.json' $URL/login -c hydra-cookie.txt
+
+            cat >data.json <<EOF
+            {
+              "displayname":"Trivial",
+              "enabled":"1",
+              "visible":"1"
+            }
+            EOF
+            mycurl --silent -X PUT $URL/project/$PROJECT_NAME \
+              -d @data.json -b hydra-cookie.txt
+
+            cat >data.json <<EOF
+            {
+              "description": "Trivial",
+              "checkinterval": "60",
+              "enabled": "1",
+              "visible": "1",
+              "keepnr": "1",
+              "enableemail": true,
+              "emailoverride": "hydra@localhost",
+              "type": 0,
+              "nixexprinput": "git",
+              "nixexprpath": "jobset.nix",
+              "inputs": {
+                "git": {"value": "http://localhost:3001/root/repo.git", "type": "git"},
+                "gitea_repo_name": {"value": "repo", "type": "string"},
+                "gitea_repo_owner": {"value": "root", "type": "string"},
+                "gitea_status_repo": {"value": "git", "type": "string"},
+                "gitea_http_url": {"value": "http://localhost:3001", "type": "string"}
+              }
+            }
+            EOF
+
+            mycurl --silent -X PUT $URL/jobset/$PROJECT_NAME/$JOBSET_NAME \
+              -d @data.json -b hydra-cookie.txt
+          '';
+
+          api_token = "d7f16a3412e01a43a414535b16007c6931d3a9c7";
+
+          snakeoilKeypair = {
+            privkey = pkgs.writeText "privkey.snakeoil" ''
+              -----BEGIN EC PRIVATE KEY-----
+              MHcCAQEEIHQf/khLvYrQ8IOika5yqtWvI0oquHlpRLTZiJy5dRJmoAoGCCqGSM49
+              AwEHoUQDQgAEKF0DYGbBwbj06tA3fd/+yP44cvmwmHBWXZCKbS+RQlAKvLXMWkpN
+              r1lwMyJZoSGgBHoUahoYjTh9/sJL7XLJtA==
+              -----END EC PRIVATE KEY-----
+            '';
+
+            pubkey = pkgs.lib.concatStrings [
+              "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHA"
+              "yNTYAAABBBChdA2BmwcG49OrQN33f/sj+OHL5sJhwVl2Qim0vkUJQCry1zFpKTa"
+              "9ZcDMiWaEhoAR6FGoaGI04ff7CS+1yybQ= sakeoil"
+            ];
+          };
+
+          smallDrv = pkgs.writeText "jobset.nix" ''
+            { trivial = builtins.derivation {
+                name = "trivial";
+                system = "${system}";
+                builder = "/bin/sh";
+                allowSubstitutes = false;
+                preferLocalBuild = true;
+                args = ["-c" "echo success > $out; exit 0"];
+              };
+             }
+          '';
+        in
+        ''
+          import json
+
+          machine.start()
+          machine.wait_for_unit("multi-user.target")
+          machine.wait_for_open_port(3000)
+          machine.wait_for_open_port(3001)
+
+          machine.succeed(
+              "su -l gitea -c 'GITEA_WORK_DIR=/var/lib/gitea gitea admin user create "
+              + "--username root --password root --email test@localhost'"
+          )
+          machine.succeed("su -l postgres -c 'psql gitea < ${scripts.mktoken}'")
+
+          machine.succeed(
+              "curl --fail -X POST http://localhost:3001/api/v1/user/repos "
+              + "-H 'Accept: application/json' -H 'Content-Type: application/json' "
+              + f"-H 'Authorization: token ${api_token}'"
+              + ' -d \'{"auto_init":false, "description":"string", "license":"mit", "name":"repo", "private":false}\'''
+          )
+
+          machine.succeed(
+              "curl --fail -X POST http://localhost:3001/api/v1/user/keys "
+              + "-H 'Accept: application/json' -H 'Content-Type: application/json' "
+              + f"-H 'Authorization: token ${api_token}'"
+              + ' -d \'{"key":"${snakeoilKeypair.pubkey}","read_only":true,"title":"SSH"}\'''
+          )
+
+          machine.succeed(
+              "${scripts.git-setup}"
+          )
+
+          machine.succeed(
+              "${scripts.hydra-setup}"
+          )
+
+          machine.wait_until_succeeds(
+              'curl -Lf -s http://localhost:3000/build/1 -H "Accept: application/json" '
+              + '|  jq .buildstatus | xargs test 0 -eq'
+          )
+
+          data = machine.succeed(
+              'curl -Lf -s "http://localhost:3001/api/v1/repos/root/repo/statuses/$(cd /tmp/repo && git show | head -n1 | awk "{print \\$2}")" '
+              + "-H 'Accept: application/json' -H 'Content-Type: application/json' "
+              + f"-H 'Authorization: token ${api_token}'"
+          )
+
+          response = json.loads(data)
+
+          assert len(response) == 2, "Expected exactly three status updates for latest commit (queued, finished)!"
+          assert response[0]['status'] == "success", "Expected finished status to be success!"
+          assert response[1]['status'] == "pending", "Expected queued status to be pending!"
+
+          machine.shutdown()
+        '';
+    });
+
+  validate-openapi = forEachSystem (system:
+    let pkgs = pkgsBySystem.${system}; in
+    pkgs.runCommand "validate-openapi"
+    { buildInputs = [ pkgs.openapi-generator-cli ]; }
+    ''
+      openapi-generator-cli validate -i ${./hydra-api.yaml}
+      touch $out
+    '');
+
+}

src/hydra-eval-jobs/hydra-eval-jobs.cc

@@ -102,8 +102,8 @@ static std::string queryMetaStrings(EvalState & state, PackageInfo & drv, const
             for (unsigned int n = 0; n < v.listSize(); ++n)
                 rec(*v.listElems()[n]);
         else if (v.type() == nAttrs) {
-            auto a = v.attrs->find(state.symbols.create(subAttribute));
-            if (a != v.attrs->end())
+            auto a = v.attrs()->find(state.symbols.create(subAttribute));
+            if (a != v.attrs()->end())
                 res.push_back(std::string(state.forceString(*a->value, a->pos, "while evaluating meta attributes")));
         }
     };
@@ -138,12 +138,12 @@ static void worker(
 
         callFlake(state, lockedFlake, *vFlake);
 
-        auto vOutputs = vFlake->attrs->get(state.symbols.create("outputs"))->value;
+        auto vOutputs = vFlake->attrs()->get(state.symbols.create("outputs"))->value;
         state.forceValue(*vOutputs, noPos);
 
-        auto aHydraJobs = vOutputs->attrs->get(state.symbols.create("hydraJobs"));
+        auto aHydraJobs = vOutputs->attrs()->get(state.symbols.create("hydraJobs"));
         if (!aHydraJobs)
-            aHydraJobs = vOutputs->attrs->get(state.symbols.create("checks"));
+            aHydraJobs = vOutputs->attrs()->get(state.symbols.create("checks"));
         if (!aHydraJobs)
             throw Error("flake '%s' does not provide any Hydra jobs or checks", flakeRef);
 
@@ -185,7 +185,7 @@ static void worker(
                     !experimentalFeatureSettings.isEnabled(Xp::CaDerivations));
 
                 if (drv->querySystem() == "unknown")
-                    throw EvalError("derivation must have a 'system' attribute");
+                    state.error<EvalError>("derivation must have a 'system' attribute").debugThrow();
 
                 auto drvPath = state.store->printStorePath(drv->requireDrvPath());
 
@@ -204,11 +204,11 @@ static void worker(
                 job["isChannel"] = drv->queryMetaBool("isHydraChannel", false);
 
                 /* If this is an aggregate, then get its constituents. */
-                auto a = v->attrs->get(state.symbols.create("_hydraAggregate"));
+                auto a = v->attrs()->get(state.symbols.create("_hydraAggregate"));
                 if (a && state.forceBool(*a->value, a->pos, "while evaluating the `_hydraAggregate` attribute")) {
-                    auto a = v->attrs->get(state.symbols.create("constituents"));
+                    auto a = v->attrs()->get(state.symbols.create("constituents"));
                     if (!a)
-                        throw EvalError("derivation must have a ‘constituents’ attribute");
+                        state.error<EvalError>("derivation must have a ‘constituents’ attribute").debugThrow();
 
                     NixStringContext context;
                     state.coerceToString(a->pos, *a->value, context, "while evaluating the `constituents` attribute", true, false);
@@ -260,7 +260,7 @@ static void worker(
             else if (v->type() == nAttrs) {
                 auto attrs = nlohmann::json::array();
                 StringSet ss;
-                for (auto & i : v->attrs->lexicographicOrder(state.symbols)) {
+                for (auto & i : v->attrs()->lexicographicOrder(state.symbols)) {
                     std::string name(state.symbols[i->name]);
                     if (name.find(' ') != std::string::npos) {
                         printError("skipping job with illegal name '%s'", name);
@@ -274,7 +274,7 @@ static void worker(
             else if (v->type() == nNull)
                 ;
 
-            else throw TypeError("attribute '%s' is %s, which is not supported", attrPath, showType(*v));
+            else state.error<TypeError>("attribute '%s' is %s, which is not supported", attrPath, showType(*v)).debugThrow();
 
         } catch (EvalError & e) {
             auto msg = e.msg();
@@ -368,7 +368,7 @@ int main(int argc, char * * argv)
                             ]()
                             {
                                 try {
-                                    EvalState state(myArgs.searchPath, openStore());
+                                    EvalState state(myArgs.lookupPath, openStore());
                                     Bindings & autoArgs = *myArgs.getAutoArgs(state);
                                     worker(state, autoArgs, *to, *from);
                                 } catch (Error & e) {

src/hydra-evaluator/hydra-evaluator.cc

@@ -38,7 +38,7 @@ class JobsetId {
     friend bool operator!= (const JobsetId & lhs, const JobsetName & rhs);
 
     std::string display() const {
-        return str(format("%1%:%2% (jobset#%3%)") % project % jobset % id);
+        return boost::str(boost::format("%1%:%2% (jobset#%3%)") % project % jobset % id);
     }
 };
 bool operator==(const JobsetId & lhs, const JobsetId & rhs)

src/hydra-queue-runner/build-remote.cc

@@ -54,9 +54,20 @@ static std::unique_ptr<SSHMaster::Connection> openConnection(
         command.splice(command.end(), extraStoreArgs(machine->sshName));
     }
 
-    return master.startCommand(std::move(command), {
+    auto ret = master.startCommand(std::move(command), {
         "-a", "-oBatchMode=yes", "-oConnectTimeout=60", "-oTCPKeepAlive=yes"
     });
+
+    // XXX: determine the actual max value we can use from /proc.
+
+    // FIXME: Should this be upstreamed into `startCommand` in Nix?
+
+    int pipesize = 1024 * 1024;
+
+    fcntl(ret->in.get(), F_SETPIPE_SZ, &pipesize);
+    fcntl(ret->out.get(), F_SETPIPE_SZ, &pipesize);
+
+    return ret;
 }
 
 

src/hydra-queue-runner/hydra-queue-runner.cc

@@ -171,7 +171,7 @@ void State::parseMachines(const std::string & contents)
             std::move(mandatoryFeatures),
             // `sshPublicHostKey`
             tokens[7] != "" && tokens[7] != "-"
-                ? tokens[7]
+                ? base64Decode(tokens[7])
                 : "",
         });
 

src/hydra-queue-runner/queue-monitor.cc

@@ -298,7 +298,7 @@ bool State::getQueuedBuilds(Connection & conn,
         try {
             createBuild(build);
         } catch (Error & e) {
-            e.addTrace({}, hintfmt("while loading build %d: ", build->id));
+            e.addTrace({}, HintFmt("while loading build %d: ", build->id));
             throw;
         }
 

src/lib/Hydra/Base/Controller/NixChannel.pm

@@ -4,7 +4,6 @@ use strict;
 use warnings;
 use base 'Hydra::Base::Controller::REST';
 use List::SomeUtils qw(any);
-use Nix::Store;
 use Hydra::Helper::Nix;
 use Hydra::Helper::CatalystUtils;
 
@@ -30,7 +29,7 @@ sub getChannelData {
         my $outputs = {};
         foreach my $output (@outputs) {
             my $outPath = $output->get_column("outpath");
-            next if $checkValidity && !isValidPath($outPath);
+            next if $checkValidity && !$MACHINE_LOCAL_STORE->isValidPath($outPath);
             $outputs->{$output->get_column("outname")} = $outPath;
             push @storePaths, $outPath;
             # Put the system type in the manifest (for top-level

src/lib/Hydra/Controller/API.pm

@@ -285,6 +285,23 @@ sub push_github : Chained('api') PathPart('push-github') Args(0) {
     $c->response->body("");
 }
 
+sub push_gitea : Chained('api') PathPart('push-gitea') Args(0) {
+    my ($self, $c) = @_;
+
+    $c->{stash}->{json}->{jobsetsTriggered} = [];
+
+    my $in = $c->request->{data};
+    my $url = $in->{repository}->{clone_url} or die;
+    $url =~ s/.git$//;
+    print STDERR "got push from Gitea repository $url\n";
+
+    triggerJobset($self, $c, $_, 0) foreach $c->model('DB::Jobsets')->search(
+        { 'project.enabled' => 1, 'me.enabled' => 1 },
+        { join => 'project'
+        , where => \ [ 'me.flake like ? or exists (select 1 from JobsetInputAlts where project = me.project and jobset = me.name and value like ?)', [ 'flake', "%$url%"], [ 'value', "%$url%" ] ]
+        });
+    $c->response->body("");
+}
 
 
 1;

src/lib/Hydra/Controller/Build.pm

@@ -10,8 +10,6 @@ use File::Basename;
 use File::LibMagic;
 use File::stat;
 use Data::Dump qw(dump);
-use Nix::Store;
-use Nix::Config;
 use List::SomeUtils qw(all);
 use Encode;
 use JSON::PP;
@@ -83,9 +81,9 @@ sub build_GET {
     # false because `$_->path` will be empty
     $c->stash->{available} =
         $c->stash->{isLocalStore}
-        ? all { $_->path && isValidPath($_->path) } $build->buildoutputs->all
+        ? all { $_->path && $MACHINE_LOCAL_STORE->isValidPath($_->path) } $build->buildoutputs->all
         : 1;
-    $c->stash->{drvAvailable} = isValidPath $build->drvpath;
+    $c->stash->{drvAvailable} = $MACHINE_LOCAL_STORE->isValidPath($build->drvpath);
 
     if ($build->finished && $build->iscachedbuild) {
         my $path = ($build->buildoutputs)[0]->path or undef;
@@ -236,6 +234,9 @@ sub serveFile {
     }
 
     elsif ($ls->{type} eq "regular") {
+        # Have the hosted data considered its own origin to avoid being a giant
+        # XSS hole.
+        $c->response->header('Content-Security-Policy' => 'sandbox allow-scripts');
 
         $c->stash->{'plain'} = { data => grab(cmd => ["nix", "--experimental-features", "nix-command",
                                                       "store", "cat", "--store", getStoreUri(), "$path"]) };
@@ -309,7 +310,7 @@ sub output : Chained('buildChain') PathPart Args(1) {
     error($c, "This build is not finished yet.") unless $build->finished;
     my $output = $build->buildoutputs->find({name => $outputName});
     notFound($c, "This build has no output named ‘$outputName’") unless defined $output;
-    gone($c, "Output is no longer available.") unless isValidPath $output->path;
+    gone($c, "Output is no longer available.") unless $MACHINE_LOCAL_STORE->isValidPath($output->path);
 
     $c->response->header('Content-Disposition', "attachment; filename=\"build-${\$build->id}-${\$outputName}.nar.bz2\"");
     $c->stash->{current_view} = 'NixNAR';
@@ -426,7 +427,7 @@ sub getDependencyGraph {
             };
         $$done{$path} = $node;
         my @refs;
-        foreach my $ref (queryReferences($path)) {
+        foreach my $ref ($MACHINE_LOCAL_STORE->queryReferences($path)) {
             next if $ref eq $path;
             next unless $runtime || $ref =~ /\.drv$/;
             getDependencyGraph($self, $c, $runtime, $done, $ref);
@@ -434,7 +435,7 @@ sub getDependencyGraph {
         }
         # Show in reverse topological order to flatten the graph.
         # Should probably do a proper BFS.
-        my @sorted = reverse topoSortPaths(@refs);
+        my @sorted = reverse $MACHINE_LOCAL_STORE->topoSortPaths(@refs);
         $node->{refs} = [map { $$done{$_} } @sorted];
     }
 
@@ -447,7 +448,7 @@ sub build_deps : Chained('buildChain') PathPart('build-deps') {
     my $build = $c->stash->{build};
     my $drvPath = $build->drvpath;
 
-    error($c, "Derivation no longer available.") unless isValidPath $drvPath;
+    error($c, "Derivation no longer available.") unless $MACHINE_LOCAL_STORE->isValidPath($drvPath);
 
     $c->stash->{buildTimeGraph} = getDependencyGraph($self, $c, 0, {}, $drvPath);
 
@@ -462,7 +463,7 @@ sub runtime_deps : Chained('buildChain') PathPart('runtime-deps') {
 
     requireLocalStore($c);
 
-    error($c, "Build outputs no longer available.") unless all { isValidPath($_) } @outPaths;
+    error($c, "Build outputs no longer available.") unless all { $MACHINE_LOCAL_STORE->isValidPath($_) } @outPaths;
 
     my $done = {};
     $c->stash->{runtimeGraph} = [ map { getDependencyGraph($self, $c, 1, $done, $_) } @outPaths ];
@@ -482,7 +483,7 @@ sub nix : Chained('buildChain') PathPart('nix') CaptureArgs(0) {
     if (isLocalStore) {
         foreach my $out ($build->buildoutputs) {
             notFound($c, "Path " . $out->path . " is no longer available.")
-                unless isValidPath($out->path);
+                unless $MACHINE_LOCAL_STORE->isValidPath($out->path);
         }
     }
 

src/lib/Hydra/Controller/Root.pm

@@ -35,6 +35,7 @@ sub noLoginNeeded {
 
   return $whitelisted ||
          $c->request->path eq "api/push-github" ||
+         $c->request->path eq "api/push-gitea" ||
          $c->request->path eq "google-login" ||
          $c->request->path eq "github-redirect" ||
          $c->request->path eq "github-login" ||
@@ -80,7 +81,7 @@ sub begin :Private {
     $_->supportedInputTypes($c->stash->{inputTypes}) foreach @{$c->hydra_plugins};
 
     # XSRF protection: require POST requests to have the same origin.
-    if ($c->req->method eq "POST" && $c->req->path ne "api/push-github") {
+    if ($c->req->method eq "POST" && $c->req->path ne "api/push-github" && $c->req->path ne "api/push-gitea") {
         my $referer = $c->req->header('Referer');
         $referer //= $c->req->header('Origin');
         my $base = $c->req->base;
@@ -367,7 +368,7 @@ sub realisations :Path('realisations') :Args(StrMatch[REALISATIONS_REGEX]) {
 
     else {
         my ($rawDrvOutput) = $realisation =~ REALISATIONS_REGEX;
-        my $rawRealisation = queryRawRealisation($rawDrvOutput);
+        my $rawRealisation = $MACHINE_LOCAL_STORE->queryRawRealisation($rawDrvOutput);
 
         if (!$rawRealisation) {
             $c->response->status(404);
@@ -396,7 +397,7 @@ sub narinfo :Path :Args(StrMatch[NARINFO_REGEX]) {
         my ($hash) = $narinfo =~ NARINFO_REGEX;
 
         die("Hash length was not 32") if length($hash) != 32;
-        my $path = queryPathFromHashPart($hash);
+        my $path = $MACHINE_LOCAL_STORE->queryPathFromHashPart($hash);
 
         if (!$path) {
             $c->response->status(404);

src/lib/Hydra/Helper/Nix.pm

@@ -40,8 +40,11 @@ our @EXPORT = qw(
     registerRoot
     restartBuilds
     run
+    $MACHINE_LOCAL_STORE
     );
 
+our $MACHINE_LOCAL_STORE = Nix::Store->new();
+
 
 sub getHydraHome {
     my $dir = $ENV{"HYDRA_HOME"} or die "The HYDRA_HOME directory does not exist!\n";
@@ -187,6 +190,10 @@ sub findLog {
 
     return undef if scalar @outPaths == 0;
 
+    # Filter out any NULLs. Content-addressed derivations
+    # that haven't built yet or failed to build may have a NULL outPath.
+    @outPaths = grep {defined} @outPaths;
+
     my @steps = $c->model('DB::BuildSteps')->search(
         { path => { -in => [@outPaths] } },
         { select => ["drvpath"]
@@ -494,7 +501,7 @@ sub restartBuilds {
     $builds = $builds->search({ finished => 1 });
 
     foreach my $build ($builds->search({}, { columns => ["drvpath"] })) {
-        next if !isValidPath($build->drvpath);
+        next if !$MACHINE_LOCAL_STORE->isValidPath($build->drvpath);
         registerRoot $build->drvpath;
     }
 

src/lib/Hydra/Plugin/BazaarInput.pm

@@ -7,7 +7,6 @@ use Digest::SHA qw(sha256_hex);
 use File::Path;
 use Hydra::Helper::Exec;
 use Hydra::Helper::Nix;
-use Nix::Store;
 
 sub supportedInputTypes {
     my ($self, $inputTypes) = @_;
@@ -38,9 +37,9 @@ sub fetchInput {
     (my $cachedInput) = $self->{db}->resultset('CachedBazaarInputs')->search(
         {uri => $uri, revision => $revision});
 
-    addTempRoot($cachedInput->storepath) if defined $cachedInput;
+    $MACHINE_LOCAL_STORE->addTempRoot($cachedInput->storepath) if defined $cachedInput;
 
-    if (defined $cachedInput && isValidPath($cachedInput->storepath)) {
+    if (defined $cachedInput && $MACHINE_LOCAL_STORE->isValidPath($cachedInput->storepath)) {
         $storePath = $cachedInput->storepath;
         $sha256 = $cachedInput->sha256hash;
     } else {
@@ -58,7 +57,7 @@ sub fetchInput {
         ($sha256, $storePath) = split ' ', $stdout;
 
         # FIXME: time window between nix-prefetch-bzr and addTempRoot.
-        addTempRoot($storePath);
+        $MACHINE_LOCAL_STORE->addTempRoot($storePath);
 
         $self->{db}->txn_do(sub {
             $self->{db}->resultset('CachedBazaarInputs')->create(

src/lib/Hydra/Plugin/DarcsInput.pm

@@ -7,7 +7,6 @@ use Digest::SHA qw(sha256_hex);
 use File::Path;
 use Hydra::Helper::Exec;
 use Hydra::Helper::Nix;
-use Nix::Store;
 
 sub supportedInputTypes {
     my ($self, $inputTypes) = @_;
@@ -58,7 +57,7 @@ sub fetchInput {
         {uri => $uri, revision => $revision},
         {rows => 1});
 
-    if (defined $cachedInput && isValidPath($cachedInput->storepath)) {
+    if (defined $cachedInput && $MACHINE_LOCAL_STORE->isValidPath($cachedInput->storepath)) {
         $storePath = $cachedInput->storepath;
         $sha256 = $cachedInput->sha256hash;
         $revision = $cachedInput->revision;
@@ -75,8 +74,8 @@ sub fetchInput {
         die "darcs changes --count failed" if $? != 0;
 
         system "rm", "-rf", "$tmpDir/export/_darcs";
-        $storePath = addToStore("$tmpDir/export", 1, "sha256");
-        $sha256 = queryPathHash($storePath);
+        $storePath = $MACHINE_LOCAL_STORE->addToStore("$tmpDir/export", 1, "sha256");
+        $sha256 = $MACHINE_LOCAL_STORE->queryPathHash($storePath);
         $sha256 =~ s/sha256://;
 
         $self->{db}->txn_do(sub {

src/lib/Hydra/Plugin/GitInput.pm

@@ -186,9 +186,9 @@ sub fetchInput {
         {uri => $uri, branch => $branch, revision => $revision, isdeepclone => defined($deepClone) ? 1 : 0},
         {rows => 1});
 
-    addTempRoot($cachedInput->storepath) if defined $cachedInput;
+    $MACHINE_LOCAL_STORE->addTempRoot($cachedInput->storepath) if defined $cachedInput;
 
-    if (defined $cachedInput && isValidPath($cachedInput->storepath)) {
+    if (defined $cachedInput && $MACHINE_LOCAL_STORE->isValidPath($cachedInput->storepath)) {
         $storePath = $cachedInput->storepath;
         $sha256 = $cachedInput->sha256hash;
         $revision = $cachedInput->revision;
@@ -217,7 +217,7 @@ sub fetchInput {
         ($sha256, $storePath) = split ' ', grab(cmd => ["nix-prefetch-git", $clonePath, $revision], chomp => 1);
 
         # FIXME: time window between nix-prefetch-git and addTempRoot.
-        addTempRoot($storePath);
+        $MACHINE_LOCAL_STORE->addTempRoot($storePath);
 
         $self->{db}->txn_do(sub {
             $self->{db}->resultset('CachedGitInputs')->update_or_create(

src/lib/Hydra/Plugin/MercurialInput.pm

@@ -7,7 +7,6 @@ use Digest::SHA qw(sha256_hex);
 use File::Path;
 use Hydra::Helper::Nix;
 use Hydra::Helper::Exec;
-use Nix::Store;
 use Fcntl qw(:flock);
 
 sub supportedInputTypes {
@@ -68,9 +67,9 @@ sub fetchInput {
     (my $cachedInput) = $self->{db}->resultset('CachedHgInputs')->search(
         {uri => $uri, branch => $branch, revision => $revision});
 
-    addTempRoot($cachedInput->storepath) if defined $cachedInput;
+    $MACHINE_LOCAL_STORE->addTempRoot($cachedInput->storepath) if defined $cachedInput;
 
-    if (defined $cachedInput && isValidPath($cachedInput->storepath)) {
+    if (defined $cachedInput && $MACHINE_LOCAL_STORE->isValidPath($cachedInput->storepath)) {
         $storePath = $cachedInput->storepath;
         $sha256 = $cachedInput->sha256hash;
     } else {
@@ -85,7 +84,7 @@ sub fetchInput {
         ($sha256, $storePath) = split ' ', $stdout;
 
         # FIXME: time window between nix-prefetch-hg and addTempRoot.
-        addTempRoot($storePath);
+        $MACHINE_LOCAL_STORE->addTempRoot($storePath);
 
         $self->{db}->txn_do(sub {
             $self->{db}->resultset('CachedHgInputs')->update_or_create(

src/lib/Hydra/Plugin/PathInput.pm

@@ -5,7 +5,6 @@ use warnings;
 use parent 'Hydra::Plugin';
 use POSIX qw(strftime);
 use Hydra::Helper::Nix;
-use Nix::Store;
 
 sub supportedInputTypes {
     my ($self, $inputTypes) = @_;
@@ -30,7 +29,7 @@ sub fetchInput {
         {srcpath => $uri, lastseen => {">", $timestamp - $timeout}},
         {rows => 1, order_by => "lastseen DESC"});
 
-    if (defined $cachedInput && isValidPath($cachedInput->storepath)) {
+    if (defined $cachedInput && $MACHINE_LOCAL_STORE->isValidPath($cachedInput->storepath)) {
         $storePath = $cachedInput->storepath;
         $sha256 = $cachedInput->sha256hash;
         $timestamp = $cachedInput->timestamp;
@@ -46,7 +45,7 @@ sub fetchInput {
         }
         chomp $storePath;
 
-        $sha256 = (queryPathInfo($storePath, 0))[1] or die;
+        $sha256 = ($MACHINE_LOCAL_STORE->queryPathInfo($storePath, 0))[1] or die;
 
         ($cachedInput) = $self->{db}->resultset('CachedPathInputs')->search(
             {srcpath => $uri, sha256hash => $sha256});

src/lib/Hydra/Plugin/SubversionInput.pm

@@ -7,7 +7,6 @@ use Digest::SHA qw(sha256_hex);
 use Hydra::Helper::Exec;
 use Hydra::Helper::Nix;
 use IPC::Run;
-use Nix::Store;
 
 sub supportedInputTypes {
     my ($self, $inputTypes) = @_;
@@ -45,7 +44,7 @@ sub fetchInput {
     (my $cachedInput) = $self->{db}->resultset('CachedSubversionInputs')->search(
         {uri => $uri, revision => $revision});
 
-    addTempRoot($cachedInput->storepath) if defined $cachedInput;
+    $MACHINE_LOCAL_STORE->addTempRoot($cachedInput->storepath) if defined $cachedInput;
 
     if (defined $cachedInput && isValidPath($cachedInput->storepath)) {
         $storePath = $cachedInput->storepath;
@@ -62,16 +61,16 @@ sub fetchInput {
         die "error checking out Subversion repo at `$uri':\n$stderr" if $res;
 
         if ($type eq "svn-checkout") {
-            $storePath = addToStore($wcPath, 1, "sha256");
+            $storePath = $MACHINE_LOCAL_STORE->addToStore($wcPath, 1, "sha256");
         } else {
             # Hm, if the Nix Perl bindings supported filters in
             # addToStore(), then we wouldn't need to make a copy here.
             my $tmpDir = File::Temp->newdir("hydra-svn-export.XXXXXX", CLEANUP => 1, TMPDIR => 1) or die;
             (system "svn", "export", $wcPath, "$tmpDir/source", "--quiet") == 0 or die "svn export failed";
-            $storePath = addToStore("$tmpDir/source", 1, "sha256");
+            $storePath = $MACHINE_LOCAL_STORE->addToStore("$tmpDir/source", 1, "sha256");
         }
 
-        $sha256 = queryPathHash($storePath); $sha256 =~ s/sha256://;
+        $sha256 = $MACHINE_LOCAL_STORE->queryPathHash($storePath); $sha256 =~ s/sha256://;
 
         $self->{db}->txn_do(sub {
             $self->{db}->resultset('CachedSubversionInputs')->update_or_create(

src/lib/Hydra/View/NARInfo.pm

@@ -6,8 +6,7 @@ use File::Basename;
 use Hydra::Helper::CatalystUtils;
 use MIME::Base64;
 use Nix::Manifest;
-use Nix::Store;
-use Nix::Utils;
+use Hydra::Helper::Nix;
 use base qw/Catalyst::View/;
 
 sub process {
@@ -17,7 +16,7 @@ sub process {
 
     $c->response->content_type('text/x-nix-narinfo'); # !!! check MIME type
 
-    my ($deriver, $narHash, $time, $narSize, $refs) = queryPathInfo($storePath, 1);
+    my ($deriver, $narHash, $time, $narSize, $refs) = $MACHINE_LOCAL_STORE->queryPathInfo($storePath, 1);
 
     my $info;
     $info .= "StorePath: $storePath\n";
@@ -28,8 +27,8 @@ sub process {
     $info .= "References: " . join(" ", map { basename $_ } @{$refs}) . "\n";
     if (defined $deriver) {
         $info .= "Deriver: " . basename $deriver . "\n";
-        if (isValidPath($deriver)) {
-            my $drv = derivationFromPath($deriver);
+        if ($MACHINE_LOCAL_STORE->isValidPath($deriver)) {
+            my $drv = $MACHINE_LOCAL_STORE->derivationFromPath($deriver);
             $info .= "System: $drv->{platform}\n";
         }
     }

src/script/hydra-eval-jobset

@@ -85,14 +85,14 @@ sub attrsToSQL {
 # Fetch a store path from 'eval_substituter' if not already present.
 sub getPath {
     my ($path) = @_;
-    return 1 if isValidPath($path);
+    return 1 if $MACHINE_LOCAL_STORE->isValidPath($path);
 
     my $substituter = $config->{eval_substituter};
 
     system("nix", "--experimental-features", "nix-command", "copy", "--from", $substituter, "--", $path)
         if defined $substituter;
 
-    return isValidPath($path);
+    return $MACHINE_LOCAL_STORE->isValidPath($path);
 }
 
 
@@ -143,7 +143,7 @@ sub fetchInputBuild {
         , version => $version
         , outputName => $mainOutput->name
         };
-    if (isValidPath($prevBuild->drvpath)) {
+    if ($MACHINE_LOCAL_STORE->isValidPath($prevBuild->drvpath)) {
         $result->{drvPath} = $prevBuild->drvpath;
     }
 
@@ -233,7 +233,7 @@ sub fetchInputEval {
         my $out = $build->buildoutputs->find({ name => "out" });
         next unless defined $out;
         # FIXME: Should we fail if the path is not valid?
-        next unless isValidPath($out->path);
+        next unless $MACHINE_LOCAL_STORE->isValidPath($out->path);
         $jobs->{$build->get_column('job')} = $out->path;
     }
 

src/script/hydra-update-gc-roots

@@ -5,7 +5,6 @@ use warnings;
 use File::Path;
 use File::stat;
 use File::Basename;
-use Nix::Store;
 use Hydra::Config;
 use Hydra::Schema;
 use Hydra::Helper::Nix;
@@ -47,7 +46,7 @@ sub keepBuild {
         $build->finished && ($build->buildstatus == 0 || $build->buildstatus == 6))
     {
         foreach my $path (split / /, $build->get_column('outpaths')) {
-            if (isValidPath($path)) {
+            if ($MACHINE_LOCAL_STORE->isValidPath($path)) {
                 addRoot $path;
             } else {
                 print STDERR "    warning: output ", $path, " has disappeared\n" if $build->finished;
@@ -55,7 +54,7 @@ sub keepBuild {
         }
     }
     if (!$build->finished || ($keepFailedDrvs && $build->buildstatus != 0)) {
-        if (isValidPath($build->drvpath)) {
+        if ($MACHINE_LOCAL_STORE->isValidPath($build->drvpath)) {
             addRoot $build->drvpath;
         } else {
             print STDERR "    warning: derivation ", $build->drvpath, " has disappeared\n";

t/Hydra/Controller/Job/builds.t

@@ -54,13 +54,14 @@ subtest "/job/PROJECT/JOBSET/JOB/shield" => sub {
 
 subtest "/job/PROJECT/JOBSET/JOB/prometheus" => sub {
     my $response = request(GET '/job/' . $project->name . '/' . $jobset->name . '/' . $build->job . '/prometheus');
-    ok($response->is_success, "The page showing the job's prometheus data returns 200.");
-    my $metrics = $response->content;
 
-    ok($metrics =~ m/hydra_job_failed\{.*\} 0/);
-    ok($metrics =~ m/hydra_job_completion_time\{.*\} [\d]+/);
-    ok($metrics =~ m/hydra_build_closure_size\{.*\} 96/);
-    ok($metrics =~ m/hydra_build_output_size\{.*\} 96/);
+    ok($response->is_success, "The page showing the job's prometheus data returns 200.");
+
+    my $metrics = $response->content;
+    like($metrics, qr/hydra_job_failed\{.*\} 0/);
+    like($metrics, qr/hydra_job_completion_time\{.*\} [\d]+/);
+    like($metrics, qr/hydra_build_closure_size\{.*\} 96/);
+    like($metrics, qr/hydra_build_output_size\{.*\} 96/);
 };
 
 done_testing;

t/Hydra/Controller/Jobset/http.t

@@ -186,7 +186,7 @@ subtest 'Update jobset "job" to have an invalid input type' => sub {
       })
   );
   ok(!$jobsetupdate->is_success);
-  ok($jobsetupdate->content =~ m/Invalid input type.*valid types:/);
+  like($jobsetupdate->content, qr/Invalid input type.*valid types:/);
 };
 
 

t/Hydra/Controller/User/dashboard.t

@@ -24,7 +24,7 @@ my $cookie = $login->header("set-cookie");
 my $my_jobs = request(GET '/dashboard/alice/my-jobs-tab', Accept => 'application/json', Cookie => $cookie);
 ok($my_jobs->is_success);
 my $content = $my_jobs->content();
-ok($content =~ /empty_dir/);
+like($content, qr/empty_dir/);
 ok(!($content =~ /fails/));
 ok(!($content =~ /succeed_with_failed/));
 done_testing;

t/api-test.t

@@ -115,7 +115,7 @@ subtest "evaluation" => sub {
         my $build = decode_json(request_json({ uri => "/build/" . $evals->[0]->{builds}->[0] })->content());
         is($build->{job}, "job", "The build's job name is job");
         is($build->{finished}, 0, "The build isn't finished yet");
-        ok($build->{buildoutputs}->{out}->{path} =~ /\/nix\/store\/[a-zA-Z0-9]{32}-job$/, "The build's outpath is in the Nix store and named 'job'");
+        like($build->{buildoutputs}->{out}->{path}, qr/\/nix\/store\/[a-zA-Z0-9]{32}-job$/, "The build's outpath is in the Nix store and named 'job'");
 
         subtest "search" => sub {
             my $search_project = decode_json(request_json({ uri => "/search/?query=sample" })->content());

t/content-addressed/basic.t

@@ -27,13 +27,13 @@ my $project = $db->resultset('Projects')->create({name => "tests", displayname =
 my $jobset = createBaseJobset("content-addressed", "content-addressed.nix", $ctx{jobsdir});
 
 ok(evalSucceeds($jobset), "Evaluating jobs/content-addressed.nix should exit with return code 0");
-is(nrQueuedBuildsForJobset($jobset), 5, "Evaluating jobs/content-addressed.nix should result in 4 builds");
+is(nrQueuedBuildsForJobset($jobset), 6, "Evaluating jobs/content-addressed.nix should result in 6 builds");
 
 for my $build (queuedBuildsForJobset($jobset)) {
     ok(runBuild($build), "Build '".$build->job."' from jobs/content-addressed.nix should exit with code 0");
     my $newbuild = $db->resultset('Builds')->find($build->id);
     is($newbuild->finished, 1, "Build '".$build->job."' from jobs/content-addressed.nix should be finished.");
-    my $expected = $build->job eq "fails" ? 1 : $build->job =~ /with_failed/ ? 6 : 0;
+    my $expected = $build->job eq "fails" ? 1 : $build->job =~ /with_failed/ ? 6 : $build->job =~ /FailingCA/ ? 2 : 0;
     is($newbuild->buildstatus, $expected, "Build '".$build->job."' from jobs/content-addressed.nix should have buildstatus $expected.");
 
     my $response = request("/build/".$build->id);
@@ -55,6 +55,8 @@ for my $build (queuedBuildsForJobset($jobset)) {
 
 }
 
+# XXX: deststoredir is undefined: Use of uninitialized value $ctx{"deststoredir"} in concatenation (.) or string at t/content-addressed/basic.t line 58.
+# XXX: This test seems to not do what it seems to be doing. See documentation: https://metacpan.org/pod/Test2::V0#isnt($got,-$do_not_want,-$name)
 isnt(<$ctx{deststoredir}/realisations/*>, "", "The destination store should have the realisations of the built derivations registered");
 
 done_testing;

t/jobs/content-addressed.nix

@@ -25,6 +25,13 @@ rec {
       FOO = empty_dir;
     };
 
+  caDependingOnFailingCA =
+    cfg.mkContentAddressedDerivation {
+      name = "ca-depending-on-failing-ca";
+      builder = ./dir-with-file-builder.sh;
+      FOO = fails;
+    };
+
   nonCaDependingOnCA =
     cfg.mkDerivation {
       name = "non-ca-depending-on-ca";

t/s3-backup-test.pl

@@ -3,7 +3,6 @@ use warnings;
 use File::Basename;
 use Hydra::Model::DB;
 use Hydra::Helper::Nix;
-use Nix::Store;
 use Cwd;
 
 my $db = Hydra::Model::DB->new;