Merge branch 'master' into use-store-api

Use `LegacySSHStore`

.gitignore

@@ -1,47 +1,8 @@
-/.pls_cache
-*.o
 *~
-Makefile
+.test_info.*
-Makefile.in
-.deps
-.hydra-data
-/config.guess
-/config.log
-/config.status
-/config.sub
-/configure
-/depcomp
-/libtool
-/ltmain.sh
-/autom4te.cache
-/aclocal.m4
-/missing
-/install-sh
 /src/sql/hydra-postgresql.sql
 /src/sql/hydra-sqlite.sql
 /src/sql/tmp.sqlite
-/src/hydra-eval-jobs/hydra-eval-jobs
-/src/root/static/bootstrap
-/src/root/static/js/flot
-/tests
-/doc/manual/images
-/doc/manual/manual.html
-/doc/manual/manual.pdf
-/t/.bzr*
-/t/.git*
-/t/.hg*
-/t/nix
-/t/data
-/t/jobs/config.nix
-t/jobs/declarative/project.json
-/inst
-hydra-config.h
-hydra-config.h.in
 result
+result-*
 outputs
-config
-stamp-h1
-src/hydra-evaluator/hydra-evaluator
-src/hydra-queue-runner/hydra-queue-runner
-src/root/static/fontawesome/
-src/root/static/bootstrap*/

.yath.rc

@@ -1,2 +0,0 @@
-[test]
--I=rel(t/lib)

Makefile.am

@@ -1,8 +0,0 @@
-SUBDIRS = src t doc
-BOOTCLEAN_SUBDIRS = $(SUBDIRS)
-DIST_SUBDIRS      = $(SUBDIRS)
-EXTRA_DIST        = hydra-module.nix
-
-install-data-local: hydra-module.nix
-	$(INSTALL) -d $(DESTDIR)$(datadir)/nix
-	$(INSTALL_DATA) hydra-module.nix $(DESTDIR)$(datadir)/nix/

README.md

@@ -39,16 +39,16 @@ In order to evaluate and build anything you need to create _projects_ that conta
 #### Creating A Project
 Log in as administrator, click "_Admin_" and select "_Create project_". Fill the form as follows:
 
-- **Identifier**: `hello`
+- **Identifier**: `hello-project`
 - **Display name**: `hello`
 - **Description**: `hello project`
 
 Click "_Create project_".
 
 #### Creating A Jobset
-After creating a project you are forwarded to the project page. Click "_Actions_" and choose "_Create jobset_". Fill the form with the following values:
+After creating a project you are forwarded to the project page. Click "_Actions_" and choose "_Create jobset_". Change **Type** to Legacy for the example below. Fill the form with the following values:
 
-- **Identifier**: `hello`
+- **Identifier**: `hello-project`
 - **Nix expression**: `examples/hello.nix` in `hydra`
 - **Check interval**: 60
 - **Scheduling shares**: 1
@@ -57,7 +57,7 @@ We have to add two inputs for this jobset. One for _nixpkgs_ and one for _hydra_
 
 - **Input name**: `nixpkgs`
 - **Type**: `Git checkout`
-- **Value**: `https://github.com/nixos/nixpkgs-channels nixos-20.03`
+- **Value**: `https://github.com/NixOS/nixpkgs nixos-24.05`
 
 - **Input name**: `hydra`
 - **Type**: `Git checkout`
@@ -80,7 +80,7 @@ $ nix-build
 You can use the provided shell.nix to get a working development environment:
 ```
 $ nix-shell
-$ ./bootstrap
+$ autoreconfPhase
 $ configurePhase # NOTE: not ./configure
 $ make
 ```
@@ -140,7 +140,7 @@ You can also interface with Hydra through a JSON API. The API is defined in [hyd
 ## Additional Resources
 
 - [Hydra User's Guide](https://nixos.org/hydra/manual/)
-- [Hydra on the NixOS Wiki](https://nixos.wiki/wiki/Hydra)
+- [Hydra on the NixOS Wiki](https://wiki.nixos.org/wiki/Hydra)
 - [hydra-cli](https://github.com/nlewo/hydra-cli)
 - [Peter Simons - Hydra: Setting up your own build farm (NixOS)](https://www.youtube.com/watch?v=RXV0Y5Bn-QQ)
 

bootstrap

@@ -1,2 +0,0 @@
-#! /bin/sh -e
-exec autoreconf -vfi

configure.ac

@@ -1,83 +0,0 @@
-AC_INIT([Hydra], [m4_esyscmd([echo -n $(cat ./version.txt)$VERSION_SUFFIX])])
-AC_CONFIG_AUX_DIR(config)
-AM_INIT_AUTOMAKE([foreign serial-tests])
-
-AC_LANG([C++])
-
-AC_PROG_CC
-AC_PROG_INSTALL
-AC_PROG_LN_S
-AC_PROG_LIBTOOL
-AC_PROG_CXX
-
-AC_PATH_PROG([XSLTPROC], [xsltproc])
-
-AC_ARG_WITH([docbook-xsl],
-  [AS_HELP_STRING([--with-docbook-xsl=PATH],
-    [path of the DocBook XSL stylesheets])],
-  [docbookxsl="$withval"],
-  [docbookxsl="/docbook-xsl-missing"])
-AC_SUBST([docbookxsl])
-
-
-AC_DEFUN([NEED_PROG],
-[
-AC_PATH_PROG($1, $2)
-if test -z "$$1"; then
-    AC_MSG_ERROR([$2 is required])
-fi
-])
-
-NEED_PROG(perl, perl)
-
-NEED_PROG([NIX_STORE_PROGRAM], [nix-store])
-
-AC_MSG_CHECKING([whether $NIX_STORE_PROGRAM is recent enough])
-if test -n "$NIX_STORE" -a -n "$TMPDIR"
-then
-    # This may be executed from within a build chroot, so pacify
-    # `nix-store' instead of letting it choke while trying to mkdir
-    # /nix/var.
-    NIX_STATE_DIR="$TMPDIR"
-    export NIX_STATE_DIR
-fi
-if NIX_REMOTE=daemon PAGER=cat "$NIX_STORE_PROGRAM" --timeout 123 -q; then
-    AC_MSG_RESULT([yes])
-else
-    AC_MSG_RESULT([no])
-    AC_MSG_ERROR([`$NIX_STORE_PROGRAM' doesn't support `--timeout'; please use a newer version.])
-fi
-
-PKG_CHECK_MODULES([NIX], [nix-main nix-expr nix-store])
-
-testPath="$(dirname $(type -p expr))"
-AC_SUBST(testPath)
-
-jobsPath="$(realpath ./t/jobs)"
-AC_SUBST(jobsPath)
-
-CXXFLAGS+=" -include nix/config.h"
-
-AC_CONFIG_FILES([
-  Makefile
-  doc/Makefile
-  doc/manual/Makefile
-  src/Makefile
-  src/hydra-evaluator/Makefile
-  src/hydra-eval-jobs/Makefile
-  src/hydra-queue-runner/Makefile
-  src/sql/Makefile
-  src/ttf/Makefile
-  src/lib/Makefile
-  src/root/Makefile
-  src/script/Makefile
-  t/Makefile
-  t/jobs/config.nix
-  t/jobs/declarative/project.json
-])
-
-AC_CONFIG_COMMANDS([executable-scripts], [])
-
-AC_CONFIG_HEADER([hydra-config.h])
-
-AC_OUTPUT

default.nix

@@ -1,6 +1,6 @@
 # The `default.nix` in flake-compat reads `flake.nix` and `flake.lock` from `src` and
 # returns an attribute set of the shape `{ defaultNix, shellNix }`
 
-(import (fetchTarball https://github.com/edolstra/flake-compat/archive/master.tar.gz) {
+(import (fetchTarball "https://github.com/edolstra/flake-compat/archive/master.tar.gz") {
   src = ./.;
 }).defaultNix

doc/Makefile.am

@@ -1,4 +0,0 @@
-SUBDIRS = manual
-BOOTCLEAN_SUBDIRS = $(SUBDIRS)
-DIST_SUBDIRS      = $(SUBDIRS)
-

doc/manual/Makefile.am

@@ -1,6 +0,0 @@
-MD_FILES = src/*.md
-
-EXTRA_DIST = $(MD_FILES)
-
-install: $(MD_FILES)
-	mdbook build . -d $(docdir)

doc/manual/meson.build

@@ -0,0 +1,36 @@
+srcs = files(
+  'src/SUMMARY.md',
+  'src/about.md',
+  'src/api.md',
+  'src/configuration.md',
+  'src/hacking.md',
+  'src/installation.md',
+  'src/introduction.md',
+  'src/jobs.md',
+  'src/monitoring/README.md',
+  'src/notifications.md',
+  'src/plugins/README.md',
+  'src/plugins/RunCommand.md',
+  'src/plugins/declarative-projects.md',
+  'src/projects.md',
+  'src/webhooks.md',
+)
+
+manual = custom_target(
+  'manual',
+  command: [
+    mdbook,
+    'build',
+    '@SOURCE_ROOT@/doc/manual',
+    '-d', meson.current_build_dir() / 'html'
+  ],
+  depend_files: srcs,
+  output: ['html'],
+  build_by_default: true,
+)
+
+install_subdir(
+  manual.full_path(),
+  install_dir: get_option('datadir') / 'doc/hydra',
+  strip_directory: true,
+)

doc/manual/src/configuration.md

@@ -74,6 +74,30 @@ following:
         }
     }
 
+Populating a Cache
+------------------
+
+A common use for Hydra is to pre-build and cache derivations which
+take a long time to build. While it is possible to direcly access the
+Hydra server's store over SSH, a more scalable option is to upload
+built derivations to a remote store like an [S3-compatible object
+store](https://nixos.org/manual/nix/stable/command-ref/new-cli/nix3-help-stores.html#s3-binary-cache-store). Setting
+the `store_uri` parameter will cause Hydra to sign and upload
+derivations as they are built:
+
+```
+store_uri = s3://cache-bucket-name?compression=zstd&parallel-compression=true&write-nar-listing=1&ls-compression=br&log-compression=br&secret-key=/path/to/cache/private/key
+```
+
+This example uses [Zstandard](https://github.com/facebook/zstd)
+compression on derivations to reduce CPU usage on the server, but
+[Brotli](https://brotli.org/) compression for derivation listings and
+build logs because it has better browser support.
+
+See [`nix help
+stores`](https://nixos.org/manual/nix/stable/command-ref/new-cli/nix3-help-stores.html)
+for a description of the store URI format.
+
 Statsd Configuration
 --------------------
 
@@ -184,7 +208,8 @@ Example configuration:
   <role_mapping>
     # Make all users in the hydra_admin group Hydra admins
     hydra_admin = admin
-    # Allow all users in the dev group to restart jobs and cancel builds
+    # Allow all users in the dev group to eval jobsets, restart jobs and cancel builds
+    dev = eval-jobset
     dev = restart-jobs
     dev = cancel-build
   </role_mapping>

doc/manual/src/hacking.md

@@ -15,12 +15,18 @@ and dependencies can be found:
 $ nix-shell
 ```
 
+of when flakes are enabled:
+
+```console
+$ nix develop
+```
+
 To build Hydra, you should then do:
 
 ```console
-[nix-shell]$ ./bootstrap
+[nix-shell]$ autoreconfPhase
 [nix-shell]$ configurePhase
-[nix-shell]$ make
+[nix-shell]$ make -j$(nproc)
 ```
 
 You start a local database, the webserver, and other components with
@@ -30,6 +36,8 @@ foreman:
 $ foreman start
 ```
 
+The Hydra interface will be available on port 63333, with an admin user named "alice" with password "foobar"
+
 You can run just the Hydra web server in your source tree as follows:
 
 ```console

doc/manual/src/plugins/README.md

@@ -42,7 +42,7 @@ Sets CircleCI status.
 
 ## Compress build logs
 
-Compresses build logs after a build with bzip2.
+Compresses build logs after a build with bzip2 or zstd.
 
 ### Configuration options
 
@@ -50,6 +50,14 @@ Compresses build logs after a build with bzip2.
 
 Enable log compression
 
+- `compress_build_logs_compression`
+
+Which compression format to use. Valid values are bzip2 (default) and zstd.
+
+- `compress_build_logs_silent`
+
+Whether to compress logs silently.
+
 ### Example
 
 ```xml

doc/manual/src/projects.md

@@ -404,3 +404,10 @@ analogous:
   | `String value` | `gitea_status_repo` | *Name of the `Git checkout` input* |
   | `String value` | `gitea_http_url`    | *Public URL of `gitea`*, optional  |
 
+Content-addressed derivations
+-----------------------------
+
+Hydra can to a certain extent use the [`ca-derivations` experimental Nix feature](https://github.com/NixOS/rfcs/pull/62).
+To use it, make sure that the Nix version you use is at least as recent as the one used in hydra's flake.
+
+Be warned that this support is still highly experimental, and anything beyond the basic functionality might be broken at that point.

doc/manual/src/webhooks.md

@@ -1,9 +1,12 @@
 # Webhooks
 
-Hydra can be notified by github's webhook to trigger a new evaluation when a
+Hydra can be notified by github or gitea with webhooks to trigger a new evaluation when a
 jobset has a github repo in its input.
-To set up a github webhook go to `https://github.com/<yourhandle>/<yourrepo>/settings` and in the `Webhooks` tab
+
-click on `Add webhook`.
+## GitHub
+
+To set up a webhook for a GitHub repository go to `https://github.com/<yourhandle>/<yourrepo>/settings`
+and in the `Webhooks` tab click on `Add webhook`.
 
 - In `Payload URL` fill in `https://<your-hydra-domain>/api/push-github`.
 - In `Content type` switch to `application/json`.
@@ -11,3 +14,14 @@ click on `Add webhook`.
 - For `Which events would you like to trigger this webhook?` keep the default option for events on `Just the push event.`.
 
 Then add the hook with `Add webhook`.
+
+## Gitea
+
+To set up a webhook for a Gitea repository go to the settings of the repository in your Gitea instance
+and in the `Webhooks` tab click on `Add Webhook` and choose `Gitea` in the drop down.
+
+- In `Target URL` fill in `https://<your-hydra-domain>/api/push-gitea`.
+- Keep HTTP method `POST`, POST Content Type `application/json` and Trigger On `Push Events`.
+- Change the branch filter to match the git branch hydra builds.
+
+Then add the hook with `Add webhook`.

examples/hello.nix

@@ -1,5 +1,5 @@
 #
-# jobset example file. This file canbe referenced as Nix expression
+# jobset example file. This file can be referenced as Nix expression
 # in a jobset configuration along with inputs for nixpkgs and the
 # repository containing this file.
 #

flake.lock

@@ -1,96 +1,67 @@
 {
   "nodes": {
-    "flake-compat": {
-      "flake": false,
-      "locked": {
-        "lastModified": 1673956053,
-        "narHash": "sha256-4gtG9iQuiKITOjNQQeQIpoIB6b16fm+504Ch3sNKLd8=",
-        "owner": "edolstra",
-        "repo": "flake-compat",
-        "rev": "35bb57c0c8d8b62bbfd284272c928ceb64ddbde9",
-        "type": "github"
-      },
-      "original": {
-        "owner": "edolstra",
-        "repo": "flake-compat",
-        "type": "github"
-      }
-    },
-    "lowdown-src": {
-      "flake": false,
-      "locked": {
-        "lastModified": 1633514407,
-        "narHash": "sha256-Dw32tiMjdK9t3ETl5fzGrutQTzh2rufgZV4A/BbxuD4=",
-        "owner": "kristapsdz",
-        "repo": "lowdown",
-        "rev": "d2c2b44ff6c27b936ec27358a2653caaef8f73b8",
-        "type": "github"
-      },
-      "original": {
-        "owner": "kristapsdz",
-        "repo": "lowdown",
-        "type": "github"
-      }
-    },
     "nix": {
       "inputs": {
-        "flake-compat": "flake-compat",
+        "flake-compat": [],
-        "lowdown-src": "lowdown-src",
+        "flake-parts": [],
+        "git-hooks-nix": [],
         "nixpkgs": [
           "nixpkgs"
         ],
-        "nixpkgs-regression": "nixpkgs-regression"
+        "nixpkgs-23-11": [],
+        "nixpkgs-regression": []
       },
       "locked": {
-        "lastModified": 1696259154,
+        "lastModified": 1739899400,
-        "narHash": "sha256-WNmifcTsN9aG1ONkv+l2BC4sHZZxtNKy0keqBHXXQ7w=",
+        "narHash": "sha256-q/RgA4bB7zWai4oPySq9mch7qH14IEeom2P64SXdqHs=",
         "owner": "NixOS",
         "repo": "nix",
-        "rev": "f5f4de6a550327b4b1a06123c2e450f1b92c73b6",
+        "rev": "e310c19a1aeb1ce1ed4d41d5ab2d02db596e0918",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
-        "ref": "2.18.1",
+        "ref": "2.26-maintenance",
         "repo": "nix",
         "type": "github"
       }
     },
+    "nix-eval-jobs": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1739500569,
+        "narHash": "sha256-3wIReAqdTALv39gkWXLMZQvHyBOc3yPkWT2ZsItxedY=",
+        "owner": "nix-community",
+        "repo": "nix-eval-jobs",
+        "rev": "4b392b284877d203ae262e16af269f702df036bc",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-community",
+        "repo": "nix-eval-jobs",
+        "type": "github"
+      }
+    },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1687379288,
+        "lastModified": 1739461644,
-        "narHash": "sha256-cSuwfiqYfeVyqzCRkU9AvLTysmEuSal8nh6CYr+xWog=",
+        "narHash": "sha256-1o1qR0KYozYGRrnqytSpAhVBYLNBHX+Lv6I39zGRzKM=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "ef0bc3976340dab9a4e087a0bcff661a8b2e87f3",
+        "rev": "97a719c9f0a07923c957cf51b20b329f9fb9d43f",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
-        "ref": "nixos-23.05",
+        "ref": "nixos-24.11-small",
         "repo": "nixpkgs",
         "type": "github"
       }
     },
-    "nixpkgs-regression": {
-      "locked": {
-        "lastModified": 1643052045,
-        "narHash": "sha256-uGJ0VXIhWKGXxkeNnq4TvV3CIOkUJ3PAoLZ3HMzNVMw=",
-        "owner": "NixOS",
-        "repo": "nixpkgs",
-        "rev": "215d4d0fd80ca5163643b03a33fde804a29cc1e2",
-        "type": "github"
-      },
-      "original": {
-        "owner": "NixOS",
-        "repo": "nixpkgs",
-        "rev": "215d4d0fd80ca5163643b03a33fde804a29cc1e2",
-        "type": "github"
-      }
-    },
     "root": {
       "inputs": {
         "nix": "nix",
+        "nix-eval-jobs": "nix-eval-jobs",
         "nixpkgs": "nixpkgs"
       }
     }

flake.nix

@@ -1,555 +1,68 @@
 {
   description = "A Nix-based continuous build system";
 
-  inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.05";
+  inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-24.11-small";
-  inputs.nix.url = "github:NixOS/nix/2.18.1";
-  inputs.nix.inputs.nixpkgs.follows = "nixpkgs";
 
-  outputs = { self, nixpkgs, nix }:
+  inputs.nix = {
+    url = "github:NixOS/nix/2.26-maintenance";
+    inputs.nixpkgs.follows = "nixpkgs";
+
+    # hide nix dev tooling from our lock file
+    inputs.flake-parts.follows = "";
+    inputs.git-hooks-nix.follows = "";
+    inputs.nixpkgs-regression.follows = "";
+    inputs.nixpkgs-23-11.follows = "";
+    inputs.flake-compat.follows = "";
+  };
+
+  inputs.nix-eval-jobs = {
+    url = "github:nix-community/nix-eval-jobs";
+    # We want to control the deps precisely
+    flake = false;
+  };
+
+  outputs = { self, nixpkgs, nix, nix-eval-jobs, ... }:
     let
-      version = "${builtins.readFile ./version.txt}.${builtins.substring 0 8 (self.lastModifiedDate or "19700101")}.${self.shortRev or "DIRTY"}";
-
       systems = [ "x86_64-linux" "aarch64-linux" ];
       forEachSystem = nixpkgs.lib.genAttrs systems;
-
-      pkgsBySystem = forEachSystem (system: import nixpkgs {
-        inherit system;
-        overlays = [ self.overlays.default nix.overlays.default ];
-      });
-
-      # NixOS configuration used for VM tests.
-      hydraServer =
-        { config, pkgs, ... }:
-        {
-          imports = [ self.nixosModules.hydraTest ];
-
-          virtualisation.memorySize = 1024;
-          virtualisation.writableStore = true;
-
-          environment.systemPackages = [ pkgs.perlPackages.LWP pkgs.perlPackages.JSON ];
-
-          nix = {
-            # Without this nix tries to fetch packages from the default
-            # cache.nixos.org which is not reachable from this sandboxed NixOS test.
-            binaryCaches = [ ];
-          };
-        };
-
     in
     rec {
 
       # A Nixpkgs overlay that provides a 'hydra' package.
       overlays.default = final: prev: {
-
+        nix-eval-jobs = final.callPackage nix-eval-jobs {};
-        # Add LDAP dependencies that aren't currently found within nixpkgs.
+        hydra = final.callPackage ./package.nix {
-        perlPackages = prev.perlPackages // {
+          inherit (nixpkgs.lib) fileset;
-
+          rawSrc = self;
-          PrometheusTiny = final.perlPackages.buildPerlPackage {
+          nix-perl-bindings = final.nixComponents.nix-perl-bindings;
-            pname = "Prometheus-Tiny";
-            version = "0.007";
-            src = final.fetchurl {
-              url = "mirror://cpan/authors/id/R/RO/ROBN/Prometheus-Tiny-0.007.tar.gz";
-              sha256 = "0ef8b226a2025cdde4df80129dd319aa29e884e653c17dc96f4823d985c028ec";
-            };
-            buildInputs = with final.perlPackages; [ HTTPMessage Plack TestException ];
-            meta = {
-              homepage = "https://github.com/robn/Prometheus-Tiny";
-              description = "A tiny Prometheus client";
-              license = with final.lib.licenses; [ artistic1 gpl1Plus ];
-            };
-          };
-
-        };
-
-        hydra = with final; let
-          perlDeps = buildEnv {
-            name = "hydra-perl-deps";
-            paths = with perlPackages; lib.closePropagation
-              [
-                AuthenSASL
-                CatalystActionREST
-                CatalystAuthenticationStoreDBIxClass
-                CatalystAuthenticationStoreLDAP
-                CatalystDevel
-                CatalystPluginAccessLog
-                CatalystPluginAuthorizationRoles
-                CatalystPluginCaptcha
-                CatalystPluginPrometheusTiny
-                CatalystPluginSessionStateCookie
-                CatalystPluginSessionStoreFastMmap
-                CatalystPluginStackTrace
-                CatalystTraitForRequestProxyBase
-                CatalystViewDownload
-                CatalystViewJSON
-                CatalystViewTT
-                CatalystXRoleApplicator
-                CatalystXScriptServerStarman
-                CryptPassphrase
-                CryptPassphraseArgon2
-                CryptRandPasswd
-                DataDump
-                DateTime
-                DBDPg
-                DBDSQLite
-                DigestSHA1
-                EmailMIME
-                EmailSender
-                FileLibMagic
-                FileSlurper
-                FileWhich
-                final.nix.perl-bindings
-                git
-                IOCompress
-                IPCRun
-                IPCRun3
-                JSON
-                JSONMaybeXS
-                JSONXS
-                ListSomeUtils
-                LWP
-                LWPProtocolHttps
-                ModulePluggable
-                NetAmazonS3
-                NetPrometheus
-                NetStatsd
-                PadWalker
-                ParallelForkManager
-                PerlCriticCommunity
-                PrometheusTinyShared
-                ReadonlyX
-                SetScalar
-                SQLSplitStatement
-                Starman
-                StringCompareConstantTime
-                SysHostnameLong
-                TermSizeAny
-                TermReadKey
-                Test2Harness
-                TestPostgreSQL
-                TextDiff
-                TextTable
-                UUID4Tiny
-                YAML
-                XMLSimple
-              ];
-          };
-
-        in
-        stdenv.mkDerivation {
-
-          name = "hydra-${version}";
-
-          src = self;
-
-          buildInputs =
-            [
-              makeWrapper
-              autoconf
-              automake
-              libtool
-              unzip
-              nukeReferences
-              pkg-config
-              libpqxx
-              top-git
-              mercurial
-              darcs
-              subversion
-              breezy
-              openssl
-              bzip2
-              libxslt
-              final.nix
-              perlDeps
-              perl
-              mdbook
-              pixz
-              boost
-              postgresql_13
-              (if lib.versionAtLeast lib.version "20.03pre"
-              then nlohmann_json
-              else nlohmann_json.override { multipleHeaders = true; })
-              prometheus-cpp
-            ];
-
-          checkInputs = [
-            cacert
-            foreman
-            glibcLocales
-            libressl.nc
-            openldap
-            python3
-          ];
-
-          hydraPath = lib.makeBinPath (
-            [
-              subversion
-              openssh
-              final.nix
-              coreutils
-              findutils
-              pixz
-              gzip
-              bzip2
-              xz
-              gnutar
-              unzip
-              git
-              top-git
-              mercurial
-              darcs
-              gnused
-              breezy
-            ] ++ lib.optionals stdenv.isLinux [ rpm dpkg cdrkit ]
-          );
-
-          OPENLDAP_ROOT = openldap;
-
-          shellHook = ''
-            pushd $(git rev-parse --show-toplevel) >/dev/null
-
-            PATH=$(pwd)/src/hydra-evaluator:$(pwd)/src/script:$(pwd)/src/hydra-eval-jobs:$(pwd)/src/hydra-queue-runner:$PATH
-            PERL5LIB=$(pwd)/src/lib:$PERL5LIB
-            export HYDRA_HOME="$(pwd)/src/"
-            mkdir -p .hydra-data
-            export HYDRA_DATA="$(pwd)/.hydra-data"
-            export HYDRA_DBI='dbi:Pg:dbname=hydra;host=localhost;port=64444'
-
-            popd >/dev/null
-          '';
-
-          preConfigure = "autoreconf -vfi";
-
-          NIX_LDFLAGS = [ "-lpthread" ];
-
-          enableParallelBuilding = true;
-
-          doCheck = true;
-
-          preCheck = ''
-            patchShebangs .
-            export LOGNAME=''${LOGNAME:-foo}
-            # set $HOME for bzr so it can create its trace file
-            export HOME=$(mktemp -d)
-          '';
-
-          postInstall = ''
-            mkdir -p $out/nix-support
-
-            for i in $out/bin/*; do
-                read -n 4 chars < $i
-                if [[ $chars =~ ELF ]]; then continue; fi
-                wrapProgram $i \
-                    --prefix PERL5LIB ':' $out/libexec/hydra/lib:$PERL5LIB \
-                    --prefix PATH ':' $out/bin:$hydraPath \
-                    --set HYDRA_RELEASE ${version} \
-                    --set HYDRA_HOME $out/libexec/hydra \
-                    --set NIX_RELEASE ${final.nix.name or "unknown"}
-            done
-          '';
-
-          dontStrip = true;
-
-          meta.description = "Build of Hydra on ${final.stdenv.system}";
-          passthru = { inherit perlDeps; inherit (final) nix; };
         };
       };
 
       hydraJobs = {
-
         build = forEachSystem (system: packages.${system}.hydra);
 
-        manual = forEachSystem (system:
+        buildNoTests = forEachSystem (system:
-          let pkgs = pkgsBySystem.${system}; in
+          packages.${system}.hydra.overrideAttrs (_: {
-          pkgs.runCommand "hydra-manual-${version}" { }
+            doCheck = false;
+          })
+        );
+
+        manual = forEachSystem (system: let
+          pkgs = nixpkgs.legacyPackages.${system};
+          hydra = self.packages.${pkgs.hostPlatform.system}.hydra;
+        in
+          pkgs.runCommand "hydra-manual-${hydra.version}" { }
             ''
               mkdir -p $out/share
-              cp -prvd ${pkgs.hydra}/share/doc $out/share/
+              cp -prvd ${hydra.doc}/share/doc $out/share/
 
               mkdir $out/nix-support
               echo "doc manual $out/share/doc/hydra" >> $out/nix-support/hydra-build-products
             '');
 
-        tests.install = forEachSystem (system:
+        tests = import ./nixos-tests.nix {
-          with import (nixpkgs + "/nixos/lib/testing-python.nix") { inherit system; };
+          inherit forEachSystem nixpkgs nixosModules;
-          simpleTest {
+        };
-            name = "hydra-install";
-            nodes.machine = hydraServer;
-            testScript =
-              ''
-                machine.wait_for_job("hydra-init")
-                machine.wait_for_job("hydra-server")
-                machine.wait_for_job("hydra-evaluator")
-                machine.wait_for_job("hydra-queue-runner")
-                machine.wait_for_open_port(3000)
-                machine.succeed("curl --fail http://localhost:3000/")
-              '';
-          });
-
-        tests.notifications = forEachSystem (system:
-          let pkgs = pkgsBySystem.${system}; in
-          with import (nixpkgs + "/nixos/lib/testing-python.nix") { inherit system; };
-          simpleTest {
-            name = "hydra-notifications";
-            nodes.machine = { pkgs, ... }: {
-              imports = [ hydraServer ];
-              services.hydra-dev.extraConfig = ''
-                <influxdb>
-                  url = http://127.0.0.1:8086
-                  db = hydra
-                </influxdb>
-              '';
-              services.influxdb.enable = true;
-            };
-            testScript = ''
-              machine.wait_for_job("hydra-init")
-
-              # Create an admin account and some other state.
-              machine.succeed(
-                  """
-                      su - hydra -c "hydra-create-user root --email-address 'alice@example.org' --password foobar --role admin"
-                      mkdir /run/jobset
-                      chmod 755 /run/jobset
-                      cp ${./t/jobs/api-test.nix} /run/jobset/default.nix
-                      chmod 644 /run/jobset/default.nix
-                      chown -R hydra /run/jobset
-              """
-              )
-
-              # Wait until InfluxDB can receive web requests
-              machine.wait_for_job("influxdb")
-              machine.wait_for_open_port(8086)
-
-              # Create an InfluxDB database where hydra will write to
-              machine.succeed(
-                  "curl -XPOST 'http://127.0.0.1:8086/query' "
-                  + "--data-urlencode 'q=CREATE DATABASE hydra'"
-              )
-
-              # Wait until hydra-server can receive HTTP requests
-              machine.wait_for_job("hydra-server")
-              machine.wait_for_open_port(3000)
-
-              # Setup the project and jobset
-              machine.succeed(
-                  "su - hydra -c 'perl -I ${pkgs.hydra.perlDeps}/lib/perl5/site_perl ${./t/setup-notifications-jobset.pl}' >&2"
-              )
-
-              # Wait until hydra has build the job and
-              # the InfluxDBNotification plugin uploaded its notification to InfluxDB
-              machine.wait_until_succeeds(
-                  "curl -s -H 'Accept: application/csv' "
-                  + "-G 'http://127.0.0.1:8086/query?db=hydra' "
-                  + "--data-urlencode 'q=SELECT * FROM hydra_build_status' | grep success"
-              )
-            '';
-          });
-
-        tests.gitea = forEachSystem (system:
-          let pkgs = pkgsBySystem.${system}; in
-          with import (nixpkgs + "/nixos/lib/testing-python.nix") { inherit system; };
-          makeTest {
-            name = "hydra-gitea";
-            nodes.machine = { pkgs, ... }: {
-              imports = [ hydraServer ];
-              services.hydra-dev.extraConfig = ''
-                <gitea_authorization>
-                root=d7f16a3412e01a43a414535b16007c6931d3a9c7
-                </gitea_authorization>
-              '';
-              nix = {
-                distributedBuilds = true;
-                buildMachines = [{
-                  hostName = "localhost";
-                  systems = [ system ];
-                }];
-                binaryCaches = [ ];
-              };
-              services.gitea = {
-                enable = true;
-                database.type = "postgres";
-                disableRegistration = true;
-                httpPort = 3001;
-              };
-              services.openssh.enable = true;
-              environment.systemPackages = with pkgs; [ gitea git jq gawk ];
-              networking.firewall.allowedTCPPorts = [ 3000 ];
-            };
-            skipLint = true;
-            testScript =
-              let
-                scripts.mktoken = pkgs.writeText "token.sql" ''
-                  INSERT INTO access_token (id, uid, name, created_unix, updated_unix, token_hash, token_salt, token_last_eight) VALUES (1, 1, 'hydra', 1617107360, 1617107360, 'a930f319ca362d7b49a4040ac0af74521c3a3c3303a86f327b01994430672d33b6ec53e4ea774253208686c712495e12a486', 'XRjWE9YW0g', '31d3a9c7');
-                '';
-
-                scripts.git-setup = pkgs.writeShellScript "setup.sh" ''
-                  set -x
-                  mkdir -p /tmp/repo $HOME/.ssh
-                  cat ${snakeoilKeypair.privkey} > $HOME/.ssh/privk
-                  chmod 0400 $HOME/.ssh/privk
-                  git -C /tmp/repo init
-                  cp ${smallDrv} /tmp/repo/jobset.nix
-                  git -C /tmp/repo add .
-                  git config --global user.email test@localhost
-                  git config --global user.name test
-                  git -C /tmp/repo commit -m 'Initial import'
-                  git -C /tmp/repo remote add origin gitea@machine:root/repo
-                  GIT_SSH_COMMAND='ssh -i $HOME/.ssh/privk -o StrictHostKeyChecking=no' \
-                    git -C /tmp/repo push origin master
-                  git -C /tmp/repo log >&2
-                '';
-
-                scripts.hydra-setup = pkgs.writeShellScript "hydra.sh" ''
-                  set -x
-                  su -l hydra -c "hydra-create-user root --email-address \
-                    'alice@example.org' --password foobar --role admin"
-
-                  URL=http://localhost:3000
-                  USERNAME="root"
-                  PASSWORD="foobar"
-                  PROJECT_NAME="trivial"
-                  JOBSET_NAME="trivial"
-                  mycurl() {
-                    curl --referer $URL -H "Accept: application/json" \
-                      -H "Content-Type: application/json" $@
-                  }
-
-                  cat >data.json <<EOF
-                  { "username": "$USERNAME", "password": "$PASSWORD" }
-                  EOF
-                  mycurl -X POST -d '@data.json' $URL/login -c hydra-cookie.txt
-
-                  cat >data.json <<EOF
-                  {
-                    "displayname":"Trivial",
-                    "enabled":"1",
-                    "visible":"1"
-                  }
-                  EOF
-                  mycurl --silent -X PUT $URL/project/$PROJECT_NAME \
-                    -d @data.json -b hydra-cookie.txt
-
-                  cat >data.json <<EOF
-                  {
-                    "description": "Trivial",
-                    "checkinterval": "60",
-                    "enabled": "1",
-                    "visible": "1",
-                    "keepnr": "1",
-                    "enableemail": true,
-                    "emailoverride": "hydra@localhost",
-                    "type": 0,
-                    "nixexprinput": "git",
-                    "nixexprpath": "jobset.nix",
-                    "inputs": {
-                      "git": {"value": "http://localhost:3001/root/repo.git", "type": "git"},
-                      "gitea_repo_name": {"value": "repo", "type": "string"},
-                      "gitea_repo_owner": {"value": "root", "type": "string"},
-                      "gitea_status_repo": {"value": "git", "type": "string"},
-                      "gitea_http_url": {"value": "http://localhost:3001", "type": "string"}
-                    }
-                  }
-                  EOF
-
-                  mycurl --silent -X PUT $URL/jobset/$PROJECT_NAME/$JOBSET_NAME \
-                    -d @data.json -b hydra-cookie.txt
-                '';
-
-                api_token = "d7f16a3412e01a43a414535b16007c6931d3a9c7";
-
-                snakeoilKeypair = {
-                  privkey = pkgs.writeText "privkey.snakeoil" ''
-                    -----BEGIN EC PRIVATE KEY-----
-                    MHcCAQEEIHQf/khLvYrQ8IOika5yqtWvI0oquHlpRLTZiJy5dRJmoAoGCCqGSM49
-                    AwEHoUQDQgAEKF0DYGbBwbj06tA3fd/+yP44cvmwmHBWXZCKbS+RQlAKvLXMWkpN
-                    r1lwMyJZoSGgBHoUahoYjTh9/sJL7XLJtA==
-                    -----END EC PRIVATE KEY-----
-                  '';
-
-                  pubkey = pkgs.lib.concatStrings [
-                    "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHA"
-                    "yNTYAAABBBChdA2BmwcG49OrQN33f/sj+OHL5sJhwVl2Qim0vkUJQCry1zFpKTa"
-                    "9ZcDMiWaEhoAR6FGoaGI04ff7CS+1yybQ= sakeoil"
-                  ];
-                };
-
-                smallDrv = pkgs.writeText "jobset.nix" ''
-                  { trivial = builtins.derivation {
-                      name = "trivial";
-                      system = "${system}";
-                      builder = "/bin/sh";
-                      allowSubstitutes = false;
-                      preferLocalBuild = true;
-                      args = ["-c" "echo success > $out; exit 0"];
-                    };
-                   }
-                '';
-              in
-              ''
-                import json
-
-                machine.start()
-                machine.wait_for_unit("multi-user.target")
-                machine.wait_for_open_port(3000)
-                machine.wait_for_open_port(3001)
-
-                machine.succeed(
-                    "su -l gitea -c 'GITEA_WORK_DIR=/var/lib/gitea gitea admin user create "
-                    + "--username root --password root --email test@localhost'"
-                )
-                machine.succeed("su -l postgres -c 'psql gitea < ${scripts.mktoken}'")
-
-                machine.succeed(
-                    "curl --fail -X POST http://localhost:3001/api/v1/user/repos "
-                    + "-H 'Accept: application/json' -H 'Content-Type: application/json' "
-                    + f"-H 'Authorization: token ${api_token}'"
-                    + ' -d \'{"auto_init":false, "description":"string", "license":"mit", "name":"repo", "private":false}\'''
-                )
-
-                machine.succeed(
-                    "curl --fail -X POST http://localhost:3001/api/v1/user/keys "
-                    + "-H 'Accept: application/json' -H 'Content-Type: application/json' "
-                    + f"-H 'Authorization: token ${api_token}'"
-                    + ' -d \'{"key":"${snakeoilKeypair.pubkey}","read_only":true,"title":"SSH"}\'''
-                )
-
-                machine.succeed(
-                    "${scripts.git-setup}"
-                )
-
-                machine.succeed(
-                    "${scripts.hydra-setup}"
-                )
-
-                machine.wait_until_succeeds(
-                    'curl -Lf -s http://localhost:3000/build/1 -H "Accept: application/json" '
-                    + '|  jq .buildstatus | xargs test 0 -eq'
-                )
-
-                data = machine.succeed(
-                    'curl -Lf -s "http://localhost:3001/api/v1/repos/root/repo/statuses/$(cd /tmp/repo && git show | head -n1 | awk "{print \\$2}")" '
-                    + "-H 'Accept: application/json' -H 'Content-Type: application/json' "
-                    + f"-H 'Authorization: token ${api_token}'"
-                )
-
-                response = json.loads(data)
-
-                assert len(response) == 2, "Expected exactly two status updates for latest commit!"
-                assert response[0]['status'] == "success", "Expected latest status to be success!"
-                assert response[1]['status'] == "pending", "Expected first status to be pending!"
-
-                machine.shutdown()
-              '';
-          });
-
-        tests.validate-openapi = forEachSystem (system:
-          let pkgs = pkgsBySystem.${system}; in
-          pkgs.runCommand "validate-openapi"
-          { buildInputs = [ pkgs.openapi-generator-cli ]; }
-          ''
-            openapi-generator-cli validate -i ${./hydra-api.yaml}
-            touch $out
-          '');
 
         container = nixosConfigurations.container.config.system.build.toplevel;
       };
@@ -561,60 +74,34 @@
       });
 
       packages = forEachSystem (system: {
-        hydra = pkgsBySystem.${system}.hydra;
+        nix-eval-jobs = nixpkgs.legacyPackages.${system}.callPackage nix-eval-jobs {
-        default = pkgsBySystem.${system}.hydra;
+          nix = nix.packages.${system}.nix;
+        };
+        hydra = nixpkgs.legacyPackages.${system}.callPackage ./package.nix {
+          inherit (nixpkgs.lib) fileset;
+          inherit (self.packages.${system}) nix-eval-jobs;
+          rawSrc = self;
+          inherit (nix.packages.${system})
+            nix-util
+            nix-store
+            nix-main
+            nix-cmd
+            nix-cli
+            ;
+          nix-perl-bindings = nix.hydraJobs.perlBindings.${system};
+        };
+        default = self.packages.${system}.hydra;
       });
 
-      nixosModules.hydra = {
+      nixosModules = import ./nixos-modules {
-        imports = [ ./hydra-module.nix ];
+        inherit self;
-        nixpkgs.overlays = [ self.overlays.default nix.overlays.default ];
-      };
-
-      nixosModules.hydraTest = { pkgs, ... }: {
-        imports = [ self.nixosModules.hydra ];
-
-        services.hydra-dev.enable = true;
-        services.hydra-dev.hydraURL = "http://hydra.example.org";
-        services.hydra-dev.notificationSender = "admin@hydra.example.org";
-
-        systemd.services.hydra-send-stats.enable = false;
-
-        services.postgresql.enable = true;
-        services.postgresql.package = pkgs.postgresql_11;
-
-        # The following is to work around the following error from hydra-server:
-        #   [error] Caught exception in engine "Cannot determine local time zone"
-        time.timeZone = "UTC";
-
-        nix.extraOptions = ''
-          allowed-uris = https://github.com/
-        '';
-      };
-
-      nixosModules.hydraProxy = {
-        services.httpd = {
-          enable = true;
-          adminAddr = "hydra-admin@example.org";
-          extraConfig = ''
-            <Proxy *>
-              Order deny,allow
-              Allow from all
-            </Proxy>
-
-            ProxyRequests     Off
-            ProxyPreserveHost On
-            ProxyPass         /apache-errors !
-            ErrorDocument 503 /apache-errors/503.html
-            ProxyPass         /       http://127.0.0.1:3000/ retry=5 disablereuse=on
-            ProxyPassReverse  /       http://127.0.0.1:3000/
-          '';
-        };
       };
 
       nixosConfigurations.container = nixpkgs.lib.nixosSystem {
         system = "x86_64-linux";
         modules =
           [
+            self.nixosModules.hydra
             self.nixosModules.hydraTest
             self.nixosModules.hydraProxy
             {

foreman/start-manual.sh

@@ -1,6 +1,6 @@
 #!/bin/sh
 
-mdbook serve \
+exec mdbook serve \
   --port 63332 \
   --dest-dir ./.hydra-data/manual \
   ./doc/manual/

hydra-api.yaml

@@ -70,7 +70,7 @@ paths:
                   $ref: '#/components/examples/projects-success'
 
   /api/push:
-    put:
+    post:
       summary: trigger jobsets
       parameters:
         - in: query

meson.build

@@ -0,0 +1,40 @@
+project('hydra', 'cpp',
+  version: files('version.txt'),
+  license: 'GPL-3.0',
+  default_options: [
+    'debug=true',
+    'optimization=2',
+    'cpp_std=c++20',
+  ],
+)
+
+nix_util_dep = dependency('nix-util', required: true)
+nix_store_dep = dependency('nix-store', required: true)
+nix_main_dep = dependency('nix-main', required: true)
+
+# Nix need extra flags not provided in its pkg-config files.
+nix_dep = declare_dependency(
+  dependencies: [
+    nix_util_dep,
+    nix_store_dep,
+    nix_main_dep,
+  ],
+  compile_args: [
+    '-include', 'nix/config-util.hh',
+    '-include', 'nix/config-store.hh',
+    '-include', 'nix/config-main.hh',
+  ],
+)
+
+pqxx_dep = dependency('libpqxx', required: true)
+
+prom_cpp_core_dep = dependency('prometheus-cpp-core', required: true)
+prom_cpp_pull_dep = dependency('prometheus-cpp-pull', required: true)
+
+mdbook = find_program('mdbook', native: true)
+perl = find_program('perl', native: true)
+
+subdir('doc/manual')
+subdir('nixos-modules')
+subdir('src')
+subdir('t')

nixos-modules/default.nix

@@ -0,0 +1,48 @@
+{ self }:
+
+{
+  hydra = { pkgs, lib,... }: {
+    _file = ./default.nix;
+    imports = [ ./hydra.nix ];
+    services.hydra-dev.package = lib.mkDefault self.packages.${pkgs.hostPlatform.system}.hydra;
+  };
+
+  hydraTest = { pkgs, ... }: {
+    services.hydra-dev.enable = true;
+    services.hydra-dev.hydraURL = "http://hydra.example.org";
+    services.hydra-dev.notificationSender = "admin@hydra.example.org";
+
+    systemd.services.hydra-send-stats.enable = false;
+
+    services.postgresql.enable = true;
+    services.postgresql.package = pkgs.postgresql_12;
+
+    # The following is to work around the following error from hydra-server:
+    #   [error] Caught exception in engine "Cannot determine local time zone"
+    time.timeZone = "UTC";
+
+    nix.extraOptions = ''
+      allowed-uris = https://github.com/
+    '';
+  };
+
+  hydraProxy = {
+    services.httpd = {
+      enable = true;
+      adminAddr = "hydra-admin@example.org";
+      extraConfig = ''
+        <Proxy *>
+          Order deny,allow
+          Allow from all
+        </Proxy>
+
+        ProxyRequests     Off
+        ProxyPreserveHost On
+        ProxyPass         /apache-errors !
+        ErrorDocument 503 /apache-errors/503.html
+        ProxyPass         /       http://127.0.0.1:3000/ retry=5 disablereuse=on
+        ProxyPassReverse  /       http://127.0.0.1:3000/
+      '';
+    };
+  };
+}

nixos-modules/hydra.nix

@@ -68,8 +68,6 @@ in
 
       package = mkOption {
         type = types.path;
-        default = pkgs.hydra;
-        defaultText = literalExpression "pkgs.hydra";
         description = "The Hydra package.";
       };
 
@@ -233,7 +231,7 @@ in
       gc-keep-outputs = true;
       gc-keep-derivations = true;
     };
-    
+
     services.hydra-dev.extraConfig =
       ''
         using_frontend_proxy = 1
@@ -340,6 +338,7 @@ in
     systemd.services.hydra-queue-runner =
       { wantedBy = [ "multi-user.target" ];
         requires = [ "hydra-init.service" ];
+        wants = [ "network-online.target" ];
         after = [ "hydra-init.service" "network.target" "network-online.target" ];
         path = [ cfg.package pkgs.nettools pkgs.openssh pkgs.bzip2 config.nix.package ];
         restartTriggers = [ hydraConf ];
@@ -408,6 +407,7 @@ in
         requires = [ "hydra-init.service" ];
         after = [ "hydra-init.service" ];
         restartTriggers = [ hydraConf ];
+        path = [ pkgs.zstd ];
         environment = env // {
           PGPASSFILE = "${baseDir}/pgpass-queue-runner"; # grrr
           HYDRA_DBI = "${env.HYDRA_DBI};application_name=hydra-notify";
@@ -458,10 +458,17 @@ in
     # logs automatically after a step finishes, but this doesn't work
     # if the queue runner is stopped prematurely.
     systemd.services.hydra-compress-logs =
-      { path = [ pkgs.bzip2 ];
+      { path = [ pkgs.bzip2 pkgs.zstd ];
         script =
           ''
-            find ${baseDir}/build-logs -type f -name "*.drv" -mtime +3 -size +0c | xargs -r bzip2 -v -f
+            set -eou pipefail
+            compression=$(sed -nr 's/compress_build_logs_compression = ()/\1/p' ${baseDir}/hydra.conf)
+            if [[ $compression == "" ]]; then
+              compression="bzip2"
+            elif [[ $compression == zstd ]]; then
+              compression="zstd --rm"
+            fi
+            find ${baseDir}/build-logs -type f -name "*.drv" -mtime +3 -size +0c | xargs -r "$compression" --force --quiet
           '';
         startAt = "Sun 01:45";
       };

nixos-modules/meson.build

@@ -0,0 +1,4 @@
+install_data('hydra.nix',
+  install_dir: get_option('datadir') / 'nix',
+  rename: ['hydra-module.nix'],
+)

nixos-tests.nix

@@ -0,0 +1,307 @@
+{ forEachSystem, nixpkgs, nixosModules }:
+
+let
+  # NixOS configuration used for VM tests.
+  hydraServer =
+    { pkgs, ... }:
+    {
+      imports = [
+        nixosModules.hydra
+        nixosModules.hydraTest
+      ];
+
+      virtualisation.memorySize = 1024;
+      virtualisation.writableStore = true;
+
+      environment.systemPackages = [ pkgs.perlPackages.LWP pkgs.perlPackages.JSON ];
+
+      nix = {
+        # Without this nix tries to fetch packages from the default
+        # cache.nixos.org which is not reachable from this sandboxed NixOS test.
+        settings.substituters = [ ];
+      };
+    };
+
+in
+
+{
+
+  install = forEachSystem (system:
+    with import (nixpkgs + "/nixos/lib/testing-python.nix") { inherit system; };
+    simpleTest {
+      name = "hydra-install";
+      nodes.machine = hydraServer;
+      testScript =
+        ''
+          machine.wait_for_job("hydra-init")
+          machine.wait_for_job("hydra-server")
+          machine.wait_for_job("hydra-evaluator")
+          machine.wait_for_job("hydra-queue-runner")
+          machine.wait_for_open_port(3000)
+          machine.succeed("curl --fail http://localhost:3000/")
+        '';
+    });
+
+  notifications = forEachSystem (system:
+    with import (nixpkgs + "/nixos/lib/testing-python.nix") { inherit system; };
+    simpleTest {
+      name = "hydra-notifications";
+      nodes.machine = {
+        imports = [ hydraServer ];
+        services.hydra-dev.extraConfig = ''
+          <influxdb>
+            url = http://127.0.0.1:8086
+            db = hydra
+          </influxdb>
+        '';
+        services.influxdb.enable = true;
+      };
+      testScript = ''
+        machine.wait_for_job("hydra-init")
+
+        # Create an admin account and some other state.
+        machine.succeed(
+            """
+                su - hydra -c "hydra-create-user root --email-address 'alice@example.org' --password foobar --role admin"
+                mkdir /run/jobset
+                chmod 755 /run/jobset
+                cp ${./t/jobs/api-test.nix} /run/jobset/default.nix
+                chmod 644 /run/jobset/default.nix
+                chown -R hydra /run/jobset
+        """
+        )
+
+        # Wait until InfluxDB can receive web requests
+        machine.wait_for_job("influxdb")
+        machine.wait_for_open_port(8086)
+
+        # Create an InfluxDB database where hydra will write to
+        machine.succeed(
+            "curl -XPOST 'http://127.0.0.1:8086/query' "
+            + "--data-urlencode 'q=CREATE DATABASE hydra'"
+        )
+
+        # Wait until hydra-server can receive HTTP requests
+        machine.wait_for_job("hydra-server")
+        machine.wait_for_open_port(3000)
+
+        # Setup the project and jobset
+        machine.succeed(
+            "su - hydra -c 'perl -I ${config.services.hydra-dev.package.perlDeps}/lib/perl5/site_perl ${./t/setup-notifications-jobset.pl}' >&2"
+        )
+
+        # Wait until hydra has build the job and
+        # the InfluxDBNotification plugin uploaded its notification to InfluxDB
+        machine.wait_until_succeeds(
+            "curl -s -H 'Accept: application/csv' "
+            + "-G 'http://127.0.0.1:8086/query?db=hydra' "
+            + "--data-urlencode 'q=SELECT * FROM hydra_build_status' | grep success"
+        )
+      '';
+    });
+
+  gitea = forEachSystem (system:
+    let pkgs = nixpkgs.legacyPackages.${system}; in
+    with import (nixpkgs + "/nixos/lib/testing-python.nix") { inherit system; };
+    makeTest {
+      name = "hydra-gitea";
+      nodes.machine = { pkgs, ... }: {
+        imports = [ hydraServer ];
+        services.hydra-dev.extraConfig = ''
+          <gitea_authorization>
+          root=d7f16a3412e01a43a414535b16007c6931d3a9c7
+          </gitea_authorization>
+        '';
+        nixpkgs.config.permittedInsecurePackages = [ "gitea-1.19.4" ];
+        nix = {
+          settings.substituters = [ ];
+        };
+        services.gitea = {
+          enable = true;
+          database.type = "postgres";
+          settings = {
+            service.DISABLE_REGISTRATION = true;
+            server.HTTP_PORT = 3001;
+          };
+        };
+        services.openssh.enable = true;
+        environment.systemPackages = with pkgs; [ gitea git jq gawk ];
+        networking.firewall.allowedTCPPorts = [ 3000 ];
+      };
+      skipLint = true;
+      testScript =
+        let
+          scripts.mktoken = pkgs.writeText "token.sql" ''
+            INSERT INTO access_token (id, uid, name, created_unix, updated_unix, token_hash, token_salt, token_last_eight, scope) VALUES (1, 1, 'hydra', 1617107360, 1617107360, 'a930f319ca362d7b49a4040ac0af74521c3a3c3303a86f327b01994430672d33b6ec53e4ea774253208686c712495e12a486', 'XRjWE9YW0g', '31d3a9c7', 'all');
+          '';
+
+          scripts.git-setup = pkgs.writeShellScript "setup.sh" ''
+            set -x
+            mkdir -p /tmp/repo $HOME/.ssh
+            cat ${snakeoilKeypair.privkey} > $HOME/.ssh/privk
+            chmod 0400 $HOME/.ssh/privk
+            git -C /tmp/repo init
+            cp ${smallDrv} /tmp/repo/jobset.nix
+            git -C /tmp/repo add .
+            git config --global user.email test@localhost
+            git config --global user.name test
+            git -C /tmp/repo commit -m 'Initial import'
+            git -C /tmp/repo remote add origin gitea@machine:root/repo
+            GIT_SSH_COMMAND='ssh -i $HOME/.ssh/privk -o StrictHostKeyChecking=no' \
+              git -C /tmp/repo push origin master
+            git -C /tmp/repo log >&2
+          '';
+
+          scripts.hydra-setup = pkgs.writeShellScript "hydra.sh" ''
+            set -x
+            su -l hydra -c "hydra-create-user root --email-address \
+              'alice@example.org' --password foobar --role admin"
+
+            URL=http://localhost:3000
+            USERNAME="root"
+            PASSWORD="foobar"
+            PROJECT_NAME="trivial"
+            JOBSET_NAME="trivial"
+            mycurl() {
+              curl --referer $URL -H "Accept: application/json" \
+                -H "Content-Type: application/json" $@
+            }
+
+            cat >data.json <<EOF
+            { "username": "$USERNAME", "password": "$PASSWORD" }
+            EOF
+            mycurl -X POST -d '@data.json' $URL/login -c hydra-cookie.txt
+
+            cat >data.json <<EOF
+            {
+              "displayname":"Trivial",
+              "enabled":"1",
+              "visible":"1"
+            }
+            EOF
+            mycurl --silent -X PUT $URL/project/$PROJECT_NAME \
+              -d @data.json -b hydra-cookie.txt
+
+            cat >data.json <<EOF
+            {
+              "description": "Trivial",
+              "checkinterval": "60",
+              "enabled": "1",
+              "visible": "1",
+              "keepnr": "1",
+              "enableemail": true,
+              "emailoverride": "hydra@localhost",
+              "type": 0,
+              "nixexprinput": "git",
+              "nixexprpath": "jobset.nix",
+              "inputs": {
+                "git": {"value": "http://localhost:3001/root/repo.git", "type": "git"},
+                "gitea_repo_name": {"value": "repo", "type": "string"},
+                "gitea_repo_owner": {"value": "root", "type": "string"},
+                "gitea_status_repo": {"value": "git", "type": "string"},
+                "gitea_http_url": {"value": "http://localhost:3001", "type": "string"}
+              }
+            }
+            EOF
+
+            mycurl --silent -X PUT $URL/jobset/$PROJECT_NAME/$JOBSET_NAME \
+              -d @data.json -b hydra-cookie.txt
+          '';
+
+          api_token = "d7f16a3412e01a43a414535b16007c6931d3a9c7";
+
+          snakeoilKeypair = {
+            privkey = pkgs.writeText "privkey.snakeoil" ''
+              -----BEGIN EC PRIVATE KEY-----
+              MHcCAQEEIHQf/khLvYrQ8IOika5yqtWvI0oquHlpRLTZiJy5dRJmoAoGCCqGSM49
+              AwEHoUQDQgAEKF0DYGbBwbj06tA3fd/+yP44cvmwmHBWXZCKbS+RQlAKvLXMWkpN
+              r1lwMyJZoSGgBHoUahoYjTh9/sJL7XLJtA==
+              -----END EC PRIVATE KEY-----
+            '';
+
+            pubkey = pkgs.lib.concatStrings [
+              "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHA"
+              "yNTYAAABBBChdA2BmwcG49OrQN33f/sj+OHL5sJhwVl2Qim0vkUJQCry1zFpKTa"
+              "9ZcDMiWaEhoAR6FGoaGI04ff7CS+1yybQ= sakeoil"
+            ];
+          };
+
+          smallDrv = pkgs.writeText "jobset.nix" ''
+            { trivial = builtins.derivation {
+                name = "trivial";
+                system = "${system}";
+                builder = "/bin/sh";
+                allowSubstitutes = false;
+                preferLocalBuild = true;
+                args = ["-c" "echo success > $out; exit 0"];
+              };
+             }
+          '';
+        in
+        ''
+          import json
+
+          machine.start()
+          machine.wait_for_unit("multi-user.target")
+          machine.wait_for_open_port(3000)
+          machine.wait_for_open_port(3001)
+
+          machine.succeed(
+              "su -l gitea -c 'GITEA_WORK_DIR=/var/lib/gitea gitea admin user create "
+              + "--username root --password root --email test@localhost'"
+          )
+          machine.succeed("su -l postgres -c 'psql gitea < ${scripts.mktoken}'")
+
+          machine.succeed(
+              "curl --fail -X POST http://localhost:3001/api/v1/user/repos "
+              + "-H 'Accept: application/json' -H 'Content-Type: application/json' "
+              + f"-H 'Authorization: token ${api_token}'"
+              + ' -d \'{"auto_init":false, "description":"string", "license":"mit", "name":"repo", "private":false}\'''
+          )
+
+          machine.succeed(
+              "curl --fail -X POST http://localhost:3001/api/v1/user/keys "
+              + "-H 'Accept: application/json' -H 'Content-Type: application/json' "
+              + f"-H 'Authorization: token ${api_token}'"
+              + ' -d \'{"key":"${snakeoilKeypair.pubkey}","read_only":true,"title":"SSH"}\'''
+          )
+
+          machine.succeed(
+              "${scripts.git-setup}"
+          )
+
+          machine.succeed(
+              "${scripts.hydra-setup}"
+          )
+
+          machine.wait_until_succeeds(
+              'curl -Lf -s http://localhost:3000/build/1 -H "Accept: application/json" '
+              + '|  jq .buildstatus | xargs test 0 -eq'
+          )
+
+          data = machine.succeed(
+              'curl -Lf -s "http://localhost:3001/api/v1/repos/root/repo/statuses/$(cd /tmp/repo && git show | head -n1 | awk "{print \\$2}")" '
+              + "-H 'Accept: application/json' -H 'Content-Type: application/json' "
+              + f"-H 'Authorization: token ${api_token}'"
+          )
+
+          response = json.loads(data)
+
+          assert len(response) == 2, "Expected exactly three status updates for latest commit (queued, finished)!"
+          assert response[0]['status'] == "success", "Expected finished status to be success!"
+          assert response[1]['status'] == "pending", "Expected queued status to be pending!"
+
+          machine.shutdown()
+        '';
+    });
+
+  validate-openapi = forEachSystem (system:
+    let pkgs = nixpkgs.legacyPackages.${system}; in
+    pkgs.runCommand "validate-openapi"
+    { buildInputs = [ pkgs.openapi-generator-cli ]; }
+    ''
+      openapi-generator-cli validate -i ${./hydra-api.yaml}
+      touch $out
+    '');
+
+}

package.nix

@@ -0,0 +1,286 @@
+{ stdenv
+, lib
+, fileset
+
+, rawSrc
+
+, buildEnv
+
+, perlPackages
+
+, nix-util
+, nix-store
+, nix-main
+, nix-cmd
+, nix-cli
+, nix-perl-bindings
+, git
+
+, makeWrapper
+, meson
+, ninja
+, nukeReferences
+, pkg-config
+, mdbook
+
+, unzip
+, libpqxx
+, top-git
+, mercurial
+, darcs
+, subversion
+, breezy
+, openssl
+, bzip2
+, libxslt
+, perl
+, pixz
+, boost
+, postgresql_13
+, nlohmann_json
+, prometheus-cpp
+
+, cacert
+, foreman
+, glibcLocales
+, libressl
+, openldap
+, python3
+
+, openssh
+, coreutils
+, findutils
+, gzip
+, xz
+, gnutar
+, gnused
+, nix-eval-jobs
+
+, rpm
+, dpkg
+, cdrkit
+}:
+
+let
+  perlDeps = buildEnv {
+    name = "hydra-perl-deps";
+    paths = lib.closePropagation
+      ([
+        nix-perl-bindings
+        git
+      ] ++ (with perlPackages; [
+        AuthenSASL
+        CatalystActionREST
+        CatalystAuthenticationStoreDBIxClass
+        CatalystAuthenticationStoreLDAP
+        CatalystDevel
+        CatalystPluginAccessLog
+        CatalystPluginAuthorizationRoles
+        CatalystPluginCaptcha
+        CatalystPluginPrometheusTiny
+        CatalystPluginSessionStateCookie
+        CatalystPluginSessionStoreFastMmap
+        CatalystPluginStackTrace
+        CatalystTraitForRequestProxyBase
+        CatalystViewDownload
+        CatalystViewJSON
+        CatalystViewTT
+        CatalystXRoleApplicator
+        CatalystXScriptServerStarman
+        CryptPassphrase
+        CryptPassphraseArgon2
+        CryptRandPasswd
+        DataDump
+        DateTime
+        DBDPg
+        DBDSQLite
+        DigestSHA1
+        EmailMIME
+        EmailSender
+        FileCopyRecursive
+        FileLibMagic
+        FileSlurper
+        FileWhich
+        IOCompress
+        IPCRun
+        IPCRun3
+        JSON
+        JSONMaybeXS
+        JSONXS
+        ListSomeUtils
+        LWP
+        LWPProtocolHttps
+        ModulePluggable
+        NetAmazonS3
+        NetPrometheus
+        NetStatsd
+        PadWalker
+        ParallelForkManager
+        PerlCriticCommunity
+        PrometheusTinyShared
+        ReadonlyX
+        SetScalar
+        SQLSplitStatement
+        Starman
+        StringCompareConstantTime
+        SysHostnameLong
+        TermSizeAny
+        TermReadKey
+        Test2Harness
+        TestPostgreSQL
+        TextDiff
+        TextTable
+        UUID4Tiny
+        YAML
+        XMLSimple
+      ]));
+  };
+
+  version = "${builtins.readFile ./version.txt}.${builtins.substring 0 8 (rawSrc.lastModifiedDate or "19700101")}.${rawSrc.shortRev or "DIRTY"}";
+in
+stdenv.mkDerivation (finalAttrs: {
+  pname = "hydra";
+  inherit version;
+
+  src = fileset.toSource {
+    root = ./.;
+    fileset = fileset.unions ([
+      ./doc
+      ./meson.build
+      ./nixos-modules
+      ./src
+      ./t
+      ./version.txt
+      ./.perlcriticrc
+    ]);
+  };
+
+  outputs = [ "out" "doc" ];
+
+  strictDeps = true;
+
+  nativeBuildInputs = [
+    makeWrapper
+    meson
+    ninja
+    nukeReferences
+    pkg-config
+    mdbook
+    nix-cli
+    perlDeps
+    perl
+    unzip
+  ];
+
+  buildInputs = [
+    libpqxx
+    openssl
+    libxslt
+    nix-util
+    nix-store
+    nix-main
+    nix-cmd
+    perlDeps
+    perl
+    boost
+    nlohmann_json
+    prometheus-cpp
+  ];
+
+  nativeCheckInputs = [
+    bzip2
+    darcs
+    foreman
+    top-git
+    mercurial
+    subversion
+    breezy
+    openldap
+    postgresql_13
+    pixz
+    nix-eval-jobs
+  ];
+
+  checkInputs = [
+    cacert
+    glibcLocales
+    libressl.nc
+    python3
+    nix-cli
+  ];
+
+  hydraPath = lib.makeBinPath (
+    [
+      subversion
+      openssh
+      nix-cli
+      coreutils
+      findutils
+      pixz
+      gzip
+      bzip2
+      xz
+      gnutar
+      unzip
+      git
+      top-git
+      mercurial
+      darcs
+      gnused
+      breezy
+      nix-eval-jobs
+    ] ++ lib.optionals stdenv.isLinux [ rpm dpkg cdrkit ]
+  );
+
+  OPENLDAP_ROOT = openldap;
+
+  mesonBuildType = "release";
+
+  postPatch = ''
+    patchShebangs .
+  '';
+
+  shellHook = ''
+    pushd $(git rev-parse --show-toplevel) >/dev/null
+
+    PATH=$(pwd)/src/hydra-evaluator:$(pwd)/src/script:$(pwd)/src/hydra-queue-runner:$PATH
+    PERL5LIB=$(pwd)/src/lib:$PERL5LIB
+    export HYDRA_HOME="$(pwd)/src/"
+    mkdir -p .hydra-data
+    export HYDRA_DATA="$(pwd)/.hydra-data"
+    export HYDRA_DBI='dbi:Pg:dbname=hydra;host=localhost;port=64444'
+
+    popd >/dev/null
+  '';
+
+  doCheck = true;
+
+  mesonCheckFlags = [ "--verbose" ];
+
+  preCheck = ''
+    export LOGNAME=''${LOGNAME:-foo}
+    # set $HOME for bzr so it can create its trace file
+    export HOME=$(mktemp -d)
+  '';
+
+  postInstall = ''
+    mkdir -p $out/nix-support
+
+    for i in $out/bin/*; do
+        read -n 4 chars < $i
+        if [[ $chars =~ ELF ]]; then continue; fi
+        wrapProgram $i \
+            --prefix PERL5LIB ':' $out/libexec/hydra/lib:$PERL5LIB \
+            --prefix PATH ':' $out/bin:$hydraPath \
+            --set HYDRA_RELEASE ${version} \
+            --set HYDRA_HOME $out/libexec/hydra \
+            --set NIX_RELEASE ${nix-cli.name or "unknown"} \
+            --set NIX_EVAL_JOBS_RELEASE ${nix-eval-jobs.name or "unknown"}
+    done
+  '';
+
+  dontStrip = true;
+
+  meta.description = "Build of Hydra on ${stdenv.system}";
+  passthru = { inherit perlDeps; };
+})

src/Makefile.am

@@ -1,3 +0,0 @@
-SUBDIRS = hydra-evaluator hydra-eval-jobs hydra-queue-runner sql script lib root ttf
-BOOTCLEAN_SUBDIRS = $(SUBDIRS)
-DIST_SUBDIRS      = $(SUBDIRS)

src/hydra-build-step/hydra-build-step.cc

@@ -0,0 +1,213 @@
+/* This is a helper program that performs a build step, i.e. a single
+   derivation. In addition to a derivation path, it takes three store
+   URLs as arguments:
+
+   * --store: The store that will hold the resulting store paths
+       (typically a binary cache).
+
+   * --eval-store: The store that holds the .drv files, as produced by
+       hydra-evaluator.
+
+   * --build-store: The store that performs the build (often a
+       SSHStore for remote builds).
+
+   The build log is written to the path indicated by --log-file.
+*/
+
+#include "util.hh"
+#include "shared.hh"
+#include "common-eval-args.hh"
+#include "store-api.hh"
+#include "build-result.hh"
+#include "derivations.hh"
+#include "worker-protocol.hh"
+
+#include <chrono>
+
+using namespace nix;
+
+// FIXME: cut&paste
+static std::string_view getS(const std::vector<Logger::Field> & fields, size_t n)
+{
+    assert(n < fields.size());
+    assert(fields[n].type == Logger::Field::tString);
+    return fields[n].s;
+}
+
+void mainWrapped(std::list<std::string> args)
+{
+    verbosity = lvlError;
+
+    struct MyArgs : MixEvalArgs, MixCommonArgs, RootArgs
+    {
+        Path drvPath;
+        std::optional<std::string> buildStoreUrl;
+        std::optional<Path> logPath;
+        std::optional<uint64_t> maxOutputSize;
+
+        MyArgs() : MixCommonArgs("hydra-build-step")
+        {
+            expectArg("drv-path", &drvPath);
+
+            addFlag({
+                .longName = "build-store",
+                .description = "The Nix store to use for building the derivation.",
+                //.category = category,
+                .labels = {"store-url"},
+                .handler = {&buildStoreUrl},
+            });
+
+            addFlag({
+                .longName = "log-file",
+                .description = "The path to the build log.",
+                .labels = {"path"},
+                .handler = {&logPath},
+            });
+
+            addFlag({
+                .longName = "max-output-size",
+                .description = "Maximum size of the outputs.",
+                .labels = {"bytes"},
+                .handler = {&maxOutputSize},
+            });
+        }
+    };
+
+    /* A logger that intercepts all build log lines and writes them to
+       the log file. */
+    MyArgs myArgs;
+    myArgs.parseCmdline(args);
+
+    struct MyLogger : public Logger
+    {
+        Logger & prev;
+        AutoCloseFD logFile;
+
+        MyLogger(Logger & prev, Path logPath) : prev(prev)
+        {
+            logFile = open(logPath.c_str(), O_CREAT | O_TRUNC | O_WRONLY, 0666);
+            if (!logFile)
+                throw SysError("creating log file '%s'", logPath);
+        }
+
+        void log(Verbosity lvl, std::string_view s) override
+        { prev.log(lvl, s); }
+
+        void logEI(const ErrorInfo & ei) override
+        { prev.logEI(ei); }
+
+        void writeToStdout(std::string_view s) override
+        { prev.writeToStdout(s); }
+
+        void result(ActivityId act, ResultType type, const Fields & fields) override
+        {
+            if (type == resBuildLogLine)
+                writeLine(logFile.get(), std::string(getS(fields, 0)));
+            else
+                prev.result(act, type, fields);
+        }
+    };
+
+    auto destStore = openStore();
+    auto evalStore = myArgs.evalStoreUrl ? openStore(*myArgs.evalStoreUrl) : destStore;
+    auto buildStore = myArgs.buildStoreUrl ? openStore(*myArgs.buildStoreUrl) : destStore;
+
+    auto drvPath = evalStore->parseStorePath(myArgs.drvPath);
+
+    auto drv = evalStore->readDerivation(drvPath);
+    BasicDerivation basicDrv(drv);
+
+    uint64_t overhead = 0;
+
+    /* Gather the inputs. */
+    StorePathSet inputs;
+
+    for (auto & p : drv.inputSrcs)
+        inputs.insert(p);
+
+    for (auto & [drvPath, node] : drv.inputDrvs.map) {
+        auto drv2 = evalStore->readDerivation(drvPath);
+        for (auto & name : node.value) {
+            if (auto i = get(drv2.outputs, name)) {
+                auto outPath = i->path(*evalStore, drv2.name, name);
+                inputs.insert(*outPath);
+                basicDrv.inputSrcs.insert(*outPath);
+            }
+        }
+    }
+
+    /* Ensure that the inputs exist in the destination store (so that
+       the builder can substitute them from the destination
+       store). This is a no-op for regular stores, but for the binary
+       cache store, this will copy the inputs to the binary cache from
+       the local store. */
+    {
+        auto now1 = std::chrono::steady_clock::now();
+
+        debug("sending closure of '%s' to '%s'",
+            evalStore->printStorePath(drvPath), destStore->getUri());
+
+        if (evalStore != destStore)
+            copyClosure(*evalStore, *destStore, drv.inputSrcs, NoRepair, NoCheckSigs);
+
+        copyClosure(*destStore, *buildStore, inputs, NoRepair, NoCheckSigs, Substitute);
+
+        auto now2 = std::chrono::steady_clock::now();
+
+        overhead += std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count();
+    }
+
+    /* Perform the build. */
+    if (myArgs.logPath)
+        logger = new MyLogger(*logger, *myArgs.logPath);
+
+    auto buildResult = buildStore->buildDerivation(drvPath, basicDrv);
+
+    /* Copy the output paths from the build store to the destination
+       store. */
+    size_t totalNarSize = 0;
+
+    if (buildResult.success()) {
+
+        std::map<StorePath, ValidPathInfo> infos;
+        StorePathSet outputs;
+        for (auto & [output, realisation] : buildResult.builtOutputs) {
+            auto info = buildStore->queryPathInfo(realisation.outPath);
+            totalNarSize += info->narSize;
+            infos.insert_or_assign(info->path, *info);
+            outputs.insert(info->path);
+        }
+
+        if ((!myArgs.maxOutputSize || totalNarSize <= *myArgs.maxOutputSize)
+            && buildStore != destStore)
+        {
+            debug("copying outputs of '%s' from '%s' (%d bytes)",
+                buildStore->printStorePath(drvPath), buildStore->getUri(), totalNarSize);
+
+            auto now1 = std::chrono::steady_clock::now();
+
+            copyPaths(*buildStore, *destStore, outputs, NoRepair, NoCheckSigs);
+
+            auto now2 = std::chrono::steady_clock::now();
+
+            overhead += std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count();
+        }
+    }
+
+    FdSink to { STDOUT_FILENO };
+    WorkerProto::WriteConn wconn {
+        .to = to,
+        // Hardcode latest version because we are deploying hydra
+        // itself atomically
+        .version = PROTOCOL_VERSION,
+    };
+    WorkerProto::write(*evalStore, wconn, buildResult);
+}
+
+int main(int argc, char * * argv)
+{
+    return handleExceptions(argv[0], [&]() {
+        initNix();
+        mainWrapped(argvToStrings(argc, argv));
+    });
+}

src/hydra-build-step/meson.build

@@ -0,0 +1,14 @@
+srcs = files(
+  'hydra-build-step.cc',
+)
+
+hydra_build_step = executable('hydra-build-step',
+  'hydra-build-step.cc',
+  srcs,
+  dependencies: [
+    libhydra_dep,
+    nix_dep,
+    dependency('nix-cmd', required: true)
+  ],
+  install: true,
+)

src/hydra-eval-jobs/Makefile.am

@@ -1,5 +0,0 @@
-bin_PROGRAMS = hydra-eval-jobs
-
-hydra_eval_jobs_SOURCES = hydra-eval-jobs.cc
-hydra_eval_jobs_LDADD = $(NIX_LIBS) -lnixcmd
-hydra_eval_jobs_CXXFLAGS = $(NIX_CFLAGS) -I ../libhydra

src/hydra-eval-jobs/hydra-eval-jobs.cc

@@ -1,568 +0,0 @@
-#include <iostream>
-#include <thread>
-#include <optional>
-#include <unordered_map>
-
-#include "shared.hh"
-#include "store-api.hh"
-#include "eval.hh"
-#include "eval-inline.hh"
-#include "eval-settings.hh"
-#include "util.hh"
-#include "get-drvs.hh"
-#include "globals.hh"
-#include "common-eval-args.hh"
-#include "flake/flakeref.hh"
-#include "flake/flake.hh"
-#include "attr-path.hh"
-#include "derivations.hh"
-#include "local-fs-store.hh"
-
-#include "hydra-config.hh"
-
-#include <sys/types.h>
-#include <sys/wait.h>
-#include <sys/resource.h>
-
-#include <nlohmann/json.hpp>
-
-void check_pid_status_nonblocking(pid_t check_pid)
-{
-    // Only check 'initialized' and known PID's
-    if (check_pid <= 0) { return; }
-
-    int wstatus = 0;
-    pid_t pid = waitpid(check_pid, &wstatus, WNOHANG);
-    // -1 = failure, WNOHANG: 0 = no change
-    if (pid <= 0) { return; }
-
-    std::cerr << "child process (" << pid << ") ";
-
-    if (WIFEXITED(wstatus)) {
-        std::cerr << "exited with status=" << WEXITSTATUS(wstatus) << std::endl;
-    } else if (WIFSIGNALED(wstatus)) {
-        std::cerr << "killed by signal=" << WTERMSIG(wstatus) << std::endl;
-    } else if (WIFSTOPPED(wstatus)) {
-        std::cerr << "stopped by signal=" << WSTOPSIG(wstatus) << std::endl;
-    } else if (WIFCONTINUED(wstatus)) {
-        std::cerr << "continued" << std::endl;
-    }
-}
-
-using namespace nix;
-
-static Path gcRootsDir;
-static size_t maxMemorySize;
-
-struct MyArgs : MixEvalArgs, MixCommonArgs
-{
-    Path releaseExpr;
-    bool flake = false;
-    bool dryRun = false;
-
-    MyArgs() : MixCommonArgs("hydra-eval-jobs")
-    {
-        addFlag({
-            .longName = "gc-roots-dir",
-            .description = "garbage collector roots directory",
-            .labels = {"path"},
-            .handler = {&gcRootsDir}
-        });
-
-        addFlag({
-            .longName = "dry-run",
-            .description = "don't create store derivations",
-            .handler = {&dryRun, true}
-        });
-
-        addFlag({
-            .longName = "flake",
-            .description = "build a flake",
-            .handler = {&flake, true}
-        });
-
-        expectArg("expr", &releaseExpr);
-    }
-};
-
-static MyArgs myArgs;
-
-static std::string queryMetaStrings(EvalState & state, DrvInfo & drv, const std::string & name, const std::string & subAttribute)
-{
-    Strings res;
-    std::function<void(Value & v)> rec;
-
-    rec = [&](Value & v) {
-        state.forceValue(v, noPos);
-        if (v.type() == nString)
-            res.push_back(v.string.s);
-        else if (v.isList())
-            for (unsigned int n = 0; n < v.listSize(); ++n)
-                rec(*v.listElems()[n]);
-        else if (v.type() == nAttrs) {
-            auto a = v.attrs->find(state.symbols.create(subAttribute));
-            if (a != v.attrs->end())
-                res.push_back(std::string(state.forceString(*a->value, a->pos, "while evaluating meta attributes")));
-        }
-    };
-
-    Value * v = drv.queryMeta(name);
-    if (v) rec(*v);
-
-    return concatStringsSep(", ", res);
-}
-
-static void worker(
-    EvalState & state,
-    Bindings & autoArgs,
-    AutoCloseFD & to,
-    AutoCloseFD & from)
-{
-    Value vTop;
-
-    if (myArgs.flake) {
-        using namespace flake;
-
-        auto flakeRef = parseFlakeRef(myArgs.releaseExpr);
-
-        auto vFlake = state.allocValue();
-
-        auto lockedFlake = lockFlake(state, flakeRef,
-            LockFlags {
-                .updateLockFile = false,
-                .useRegistries = false,
-                .allowUnlocked = false,
-            });
-
-        callFlake(state, lockedFlake, *vFlake);
-
-        auto vOutputs = vFlake->attrs->get(state.symbols.create("outputs"))->value;
-        state.forceValue(*vOutputs, noPos);
-
-        auto aHydraJobs = vOutputs->attrs->get(state.symbols.create("hydraJobs"));
-        if (!aHydraJobs)
-            aHydraJobs = vOutputs->attrs->get(state.symbols.create("checks"));
-        if (!aHydraJobs)
-            throw Error("flake '%s' does not provide any Hydra jobs or checks", flakeRef);
-
-        vTop = *aHydraJobs->value;
-
-    } else {
-        state.evalFile(lookupFileArg(state, myArgs.releaseExpr), vTop);
-    }
-
-    auto vRoot = state.allocValue();
-    state.autoCallFunction(autoArgs, vTop, *vRoot);
-
-    while (true) {
-        /* Wait for the master to send us a job name. */
-        writeLine(to.get(), "next");
-
-        auto s = readLine(from.get());
-        if (s == "exit") break;
-        if (!hasPrefix(s, "do ")) abort();
-        std::string attrPath(s, 3);
-
-        debug("worker process %d at '%s'", getpid(), attrPath);
-
-        /* Evaluate it and send info back to the master. */
-        nlohmann::json reply;
-
-        try {
-            auto vTmp = findAlongAttrPath(state, attrPath, autoArgs, *vRoot).first;
-
-            auto v = state.allocValue();
-            state.autoCallFunction(autoArgs, *vTmp, *v);
-
-            if (auto drv = getDerivation(state, *v, false)) {
-
-                DrvInfo::Outputs outputs = drv->queryOutputs();
-
-                if (drv->querySystem() == "unknown")
-                    throw EvalError("derivation must have a 'system' attribute");
-
-                auto drvPath = state.store->printStorePath(drv->requireDrvPath());
-
-                nlohmann::json job;
-
-                job["nixName"] = drv->queryName();
-                job["system"] =drv->querySystem();
-                job["drvPath"] = drvPath;
-                job["description"] = drv->queryMetaString("description");
-                job["license"] = queryMetaStrings(state, *drv, "license", "shortName");
-                job["homepage"] = drv->queryMetaString("homepage");
-                job["maintainers"] = queryMetaStrings(state, *drv, "maintainers", "email");
-                job["schedulingPriority"] = drv->queryMetaInt("schedulingPriority", 100);
-                job["timeout"] = drv->queryMetaInt("timeout", 36000);
-                job["maxSilent"] = drv->queryMetaInt("maxSilent", 7200);
-                job["isChannel"] = drv->queryMetaBool("isHydraChannel", false);
-
-                /* If this is an aggregate, then get its constituents. */
-                auto a = v->attrs->get(state.symbols.create("_hydraAggregate"));
-                if (a && state.forceBool(*a->value, a->pos, "while evaluating the `_hydraAggregate` attribute")) {
-                    auto a = v->attrs->get(state.symbols.create("constituents"));
-                    if (!a)
-                        throw EvalError("derivation must have a ‘constituents’ attribute");
-
-                    NixStringContext context;
-                    state.coerceToString(a->pos, *a->value, context, "while evaluating the `constituents` attribute", true, false);
-                    for (auto & c : context)
-                        std::visit(overloaded {
-                            [&](const NixStringContextElem::Built & b) {
-                                job["constituents"].push_back(b.drvPath->to_string(*state.store));
-                            },
-                            [&](const NixStringContextElem::Opaque & o) {
-                            },
-                            [&](const NixStringContextElem::DrvDeep & d) {
-                            },
-                        }, c.raw);
-
-                    state.forceList(*a->value, a->pos, "while evaluating the `constituents` attribute");
-                    for (unsigned int n = 0; n < a->value->listSize(); ++n) {
-                        auto v = a->value->listElems()[n];
-                        state.forceValue(*v, noPos);
-                        if (v->type() == nString)
-                            job["namedConstituents"].push_back(v->str());
-                    }
-                }
-
-                /* Register the derivation as a GC root.  !!! This
-                   registers roots for jobs that we may have already
-                   done. */
-                auto localStore = state.store.dynamic_pointer_cast<LocalFSStore>();
-                if (gcRootsDir != "" && localStore) {
-                    Path root = gcRootsDir + "/" + std::string(baseNameOf(drvPath));
-                    if (!pathExists(root))
-                        localStore->addPermRoot(localStore->parseStorePath(drvPath), root);
-                }
-
-                nlohmann::json out;
-                for (auto & j : outputs)
-                    // FIXME: handle CA/impure builds.
-                    if (j.second)
-                        out[j.first] = state.store->printStorePath(*j.second);
-                job["outputs"] = std::move(out);
-
-                reply["job"] = std::move(job);
-            }
-
-            else if (v->type() == nAttrs) {
-                auto attrs = nlohmann::json::array();
-                StringSet ss;
-                for (auto & i : v->attrs->lexicographicOrder(state.symbols)) {
-                    std::string name(state.symbols[i->name]);
-                    if (name.find(' ') != std::string::npos) {
-                        printError("skipping job with illegal name '%s'", name);
-                        continue;
-                    }
-                    attrs.push_back(name);
-                }
-                reply["attrs"] = std::move(attrs);
-            }
-
-            else if (v->type() == nNull)
-                ;
-
-            else throw TypeError("attribute '%s' is %s, which is not supported", attrPath, showType(*v));
-
-        } catch (EvalError & e) {
-            auto msg = e.msg();
-            // Transmits the error we got from the previous evaluation
-            // in the JSON output.
-            reply["error"] = filterANSIEscapes(msg, true);
-            // Don't forget to print it into the STDERR log, this is
-            // what's shown in the Hydra UI.
-            printError(msg);
-        }
-
-        writeLine(to.get(), reply.dump());
-
-        /* If our RSS exceeds the maximum, exit. The master will
-           start a new process. */
-        struct rusage r;
-        getrusage(RUSAGE_SELF, &r);
-        if ((size_t) r.ru_maxrss > maxMemorySize * 1024) break;
-    }
-
-    writeLine(to.get(), "restart");
-}
-
-int main(int argc, char * * argv)
-{
-    /* Prevent undeclared dependencies in the evaluation via
-       $NIX_PATH. */
-    unsetenv("NIX_PATH");
-
-    return handleExceptions(argv[0], [&]() {
-
-        auto config = std::make_unique<HydraConfig>();
-
-        auto nrWorkers = config->getIntOption("evaluator_workers", 1);
-        maxMemorySize = config->getIntOption("evaluator_max_memory_size", 4096);
-
-        initNix();
-        initGC();
-
-        myArgs.parseCmdline(argvToStrings(argc, argv));
-
-        auto pureEval = config->getBoolOption("evaluator_pure_eval", myArgs.flake);
-
-        /* FIXME: The build hook in conjunction with import-from-derivation is causing "unexpected EOF" during eval */
-        settings.builders = "";
-
-        /* Prevent access to paths outside of the Nix search path and
-           to the environment. */
-        evalSettings.restrictEval = true;
-
-        /* When building a flake, use pure evaluation (no access to
-           'getEnv', 'currentSystem' etc. */
-        evalSettings.pureEval = pureEval;
-
-        if (myArgs.dryRun) settings.readOnlyMode = true;
-
-        if (myArgs.releaseExpr == "") throw UsageError("no expression specified");
-
-        if (gcRootsDir == "") printMsg(lvlError, "warning: `--gc-roots-dir' not specified");
-
-        struct State
-        {
-            std::set<std::string> todo{""};
-            std::set<std::string> active;
-            nlohmann::json jobs;
-            std::exception_ptr exc;
-        };
-
-        std::condition_variable wakeup;
-
-        Sync<State> state_;
-
-        /* Start a handler thread per worker process. */
-        auto handler = [&]()
-        {
-            pid_t pid = -1;
-            try {
-                AutoCloseFD from, to;
-
-                while (true) {
-
-                    /* Start a new worker process if necessary. */
-                    if (pid == -1) {
-                        Pipe toPipe, fromPipe;
-                        toPipe.create();
-                        fromPipe.create();
-                        pid = startProcess(
-                            [&,
-                             to{std::make_shared<AutoCloseFD>(std::move(fromPipe.writeSide))},
-                             from{std::make_shared<AutoCloseFD>(std::move(toPipe.readSide))}
-                            ]()
-                            {
-                                try {
-                                    EvalState state(myArgs.searchPath, openStore());
-                                    Bindings & autoArgs = *myArgs.getAutoArgs(state);
-                                    worker(state, autoArgs, *to, *from);
-                                } catch (Error & e) {
-                                    nlohmann::json err;
-                                    auto msg = e.msg();
-                                    err["error"] = filterANSIEscapes(msg, true);
-                                    printError(msg);
-                                    writeLine(to->get(), err.dump());
-                                    // Don't forget to print it into the STDERR log, this is
-                                    // what's shown in the Hydra UI.
-                                    writeLine(to->get(), "restart");
-                                }
-                            },
-                            ProcessOptions { .allowVfork = false });
-                        from = std::move(fromPipe.readSide);
-                        to = std::move(toPipe.writeSide);
-                        debug("created worker process %d", pid);
-                    }
-
-                    /* Check whether the existing worker process is still there. */
-                    auto s = readLine(from.get());
-                    if (s == "restart") {
-                        pid = -1;
-                        continue;
-                    } else if (s != "next") {
-                        auto json = nlohmann::json::parse(s);
-                        throw Error("worker error: %s", (std::string) json["error"]);
-                    }
-
-                    /* Wait for a job name to become available. */
-                    std::string attrPath;
-
-                    while (true) {
-                        checkInterrupt();
-                        auto state(state_.lock());
-                        if ((state->todo.empty() && state->active.empty()) || state->exc) {
-                            writeLine(to.get(), "exit");
-                            return;
-                        }
-                        if (!state->todo.empty()) {
-                            attrPath = *state->todo.begin();
-                            state->todo.erase(state->todo.begin());
-                            state->active.insert(attrPath);
-                            break;
-                        } else
-                            state.wait(wakeup);
-                    }
-
-                    /* Tell the worker to evaluate it. */
-                    writeLine(to.get(), "do " + attrPath);
-
-                    /* Wait for the response. */
-                    auto response = nlohmann::json::parse(readLine(from.get()));
-
-                    /* Handle the response. */
-                    StringSet newAttrs;
-
-                    if (response.find("job") != response.end()) {
-                        auto state(state_.lock());
-                        state->jobs[attrPath] = response["job"];
-                    }
-
-                    if (response.find("attrs") != response.end()) {
-                        for (auto & i : response["attrs"]) {
-                            std::string path = i;
-                            if (path.find(".") != std::string::npos){
-                                path = "\"" + path  + "\"";
-                            }
-                            auto s = (attrPath.empty() ? "" : attrPath + ".") + (std::string) path;
-                            newAttrs.insert(s);
-                        }
-                    }
-
-                    if (response.find("error") != response.end()) {
-                        auto state(state_.lock());
-                        state->jobs[attrPath]["error"] = response["error"];
-                    }
-
-                    /* Add newly discovered job names to the queue. */
-                    {
-                        auto state(state_.lock());
-                        state->active.erase(attrPath);
-                        for (auto & s : newAttrs)
-                            state->todo.insert(s);
-                        wakeup.notify_all();
-                    }
-                }
-            } catch (...) {
-                check_pid_status_nonblocking(pid);
-                auto state(state_.lock());
-                state->exc = std::current_exception();
-                wakeup.notify_all();
-            }
-        };
-
-        std::vector<std::thread> threads;
-        for (size_t i = 0; i < nrWorkers; i++)
-            threads.emplace_back(std::thread(handler));
-
-        for (auto & thread : threads)
-            thread.join();
-
-        auto state(state_.lock());
-
-        if (state->exc)
-            std::rethrow_exception(state->exc);
-
-        /* For aggregate jobs that have named consistuents
-           (i.e. constituents that are a job name rather than a
-           derivation), look up the referenced job and add it to the
-           dependencies of the aggregate derivation. */
-        auto store = openStore();
-
-        for (auto i = state->jobs.begin(); i != state->jobs.end(); ++i) {
-            auto jobName = i.key();
-            auto & job = i.value();
-
-            auto named = job.find("namedConstituents");
-            if (named == job.end()) continue;
-
-            std::unordered_map<std::string, std::string> brokenJobs;
-            auto getNonBrokenJobOrRecordError = [&brokenJobs, &jobName, &state](
-                    const std::string & childJobName) -> std::optional<nlohmann::json> {
-                auto childJob = state->jobs.find(childJobName);
-                if (childJob == state->jobs.end()) {
-                    printError("aggregate job '%s' references non-existent job '%s'", jobName, childJobName);
-                    brokenJobs[childJobName] = "does not exist";
-                    return std::nullopt;
-                }
-                if (childJob->find("error") != childJob->end()) {
-                    std::string error = (*childJob)["error"];
-                    printError("aggregate job '%s' references broken job '%s': %s", jobName, childJobName, error);
-                    brokenJobs[childJobName] = error;
-                    return std::nullopt;
-                }
-                return *childJob;
-            };
-
-            if (myArgs.dryRun) {
-                for (std::string jobName2 : *named) {
-                    auto job2 = getNonBrokenJobOrRecordError(jobName2);
-                    if (!job2) {
-                        continue;
-                    }
-                    std::string drvPath2 = (*job2)["drvPath"];
-                    job["constituents"].push_back(drvPath2);
-                }
-            } else {
-                auto drvPath = store->parseStorePath((std::string) job["drvPath"]);
-                auto drv = store->readDerivation(drvPath);
-
-                for (std::string jobName2 : *named) {
-                    auto job2 = getNonBrokenJobOrRecordError(jobName2);
-                    if (!job2) {
-                        continue;
-                    }
-                    auto drvPath2 = store->parseStorePath((std::string) (*job2)["drvPath"]);
-                    auto drv2 = store->readDerivation(drvPath2);
-                    job["constituents"].push_back(store->printStorePath(drvPath2));
-                    drv.inputDrvs.map[drvPath2].value = {drv2.outputs.begin()->first};
-                }
-
-                if (brokenJobs.empty()) {
-                    std::string drvName(drvPath.name());
-                    assert(hasSuffix(drvName, drvExtension));
-                    drvName.resize(drvName.size() - drvExtension.size());
-
-                    auto hashModulo = hashDerivationModulo(*store, drv, true);
-                    if (hashModulo.kind != DrvHash::Kind::Regular) continue;
-                    auto h = hashModulo.hashes.find("out");
-                    if (h == hashModulo.hashes.end()) continue;
-                    auto outPath = store->makeOutputPath("out", h->second, drvName);
-                    drv.env["out"] = store->printStorePath(outPath);
-                    drv.outputs.insert_or_assign("out", DerivationOutput::InputAddressed { .path = outPath });
-                    auto newDrvPath = store->printStorePath(writeDerivation(*store, drv));
-
-                    debug("rewrote aggregate derivation %s -> %s", store->printStorePath(drvPath), newDrvPath);
-
-                    job["drvPath"] = newDrvPath;
-                    job["outputs"]["out"] = store->printStorePath(outPath);
-                }
-            }
-
-            job.erase("namedConstituents");
-
-            /* Register the derivation as a GC root.  !!! This
-                registers roots for jobs that we may have already
-                done. */
-            auto localStore = store.dynamic_pointer_cast<LocalFSStore>();
-            if (gcRootsDir != "" && localStore) {
-                auto drvPath = job["drvPath"].get<std::string>();
-                Path root = gcRootsDir + "/" + std::string(baseNameOf(drvPath));
-                if (!pathExists(root))
-                    localStore->addPermRoot(localStore->parseStorePath(drvPath), root);
-            }
-
-            if (!brokenJobs.empty()) {
-                std::stringstream ss;
-                for (const auto& [jobName, error] : brokenJobs) {
-                    ss << jobName << ": " << error << "\n";
-                }
-                job["error"] = ss.str();
-            }
-        }
-
-        std::cout << state->jobs.dump(2) << "\n";
-    });
-}

src/hydra-evaluator/Makefile.am

@@ -1,5 +0,0 @@
-bin_PROGRAMS = hydra-evaluator
-
-hydra_evaluator_SOURCES = hydra-evaluator.cc
-hydra_evaluator_LDADD = $(NIX_LIBS) -lpqxx
-hydra_evaluator_CXXFLAGS = $(NIX_CFLAGS) -Wall -I ../libhydra -Wno-deprecated-declarations

src/hydra-evaluator/hydra-evaluator.cc

@@ -2,6 +2,7 @@
 #include "hydra-config.hh"
 #include "pool.hh"
 #include "shared.hh"
+#include "signals.hh"
 
 #include <algorithm>
 #include <thread>
@@ -37,7 +38,7 @@ class JobsetId {
     friend bool operator!= (const JobsetId & lhs, const JobsetName & rhs);
 
     std::string display() const {
-        return str(format("%1%:%2% (jobset#%3%)") % project % jobset % id);
+        return boost::str(boost::format("%1%:%2% (jobset#%3%)") % project % jobset % id);
     }
 };
 bool operator==(const JobsetId & lhs, const JobsetId & rhs)

src/hydra-evaluator/meson.build

@@ -0,0 +1,9 @@
+hydra_evaluator = executable('hydra-evaluator',
+  'hydra-evaluator.cc',
+  dependencies: [
+    libhydra_dep,
+    nix_dep,
+    pqxx_dep,
+  ],
+  install: true,
+)

src/hydra-queue-runner/Makefile.am

@@ -1,8 +0,0 @@
-bin_PROGRAMS = hydra-queue-runner
-
-hydra_queue_runner_SOURCES = hydra-queue-runner.cc queue-monitor.cc dispatcher.cc \
- builder.cc build-result.cc build-remote.cc \
- hydra-build-result.hh counter.hh state.hh db.hh \
- nar-extractor.cc nar-extractor.hh
-hydra_queue_runner_LDADD = $(NIX_LIBS) -lpqxx -lprometheus-cpp-pull -lprometheus-cpp-core
-hydra_queue_runner_CXXFLAGS = $(NIX_CFLAGS) -Wall -I ../libhydra -Wno-deprecated-declarations

src/hydra-queue-runner/build-remote.cc

@@ -1,217 +1,139 @@
-#include <algorithm>
+#include <math.h>
-#include <cmath>
-
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <fcntl.h>
 
 #include "build-result.hh"
-#include "path.hh"
 #include "serve-protocol.hh"
 #include "state.hh"
+#include "current-process.hh"
+#include "processes.hh"
 #include "util.hh"
-#include "worker-protocol.hh"
-#include "worker-protocol-impl.hh"
 #include "finally.hh"
 #include "url.hh"
+#include "worker-protocol.hh"
 
 using namespace nix;
 
+namespace nix::build_remote {
 
-struct Child
+static Path createLogFileDir(const std::string & logDir, const StorePath & drvPath)
 {
-    Pid pid;
+    std::string base(drvPath.to_string());
-    AutoCloseFD to, from;
+    auto logFile = logDir + "/" + std::string(base, 0, 2) + "/" + std::string(base, 2);
-};
 
+    createDirs(dirOf(logFile));
 
-static void append(Strings & dst, const Strings & src)
+    return logFile;
-{
-    dst.insert(dst.end(), src.begin(), src.end());
 }
 
-static Strings extraStoreArgs(std::string & machine)
+}
+
+/* using namespace nix::build_remote; */
+
+void RemoteResult::updateWithBuildResult(const nix::BuildResult & buildResult)
 {
-    Strings result;
+    // FIXME: make RemoteResult inherit BuildResult.
-    try {
+    timesBuilt = buildResult.timesBuilt;
-        auto parsed = parseURL(machine);
+    errorMsg = buildResult.errorMsg;
-        if (parsed.scheme != "ssh") {
+    isNonDeterministic = buildResult.isNonDeterministic;
-            throw SysError("Currently, only (legacy-)ssh stores are supported!");
+    if (buildResult.startTime && buildResult.stopTime) {
-        }
+        startTime = buildResult.startTime;
-        machine = parsed.authority.value_or("");
+        stopTime = buildResult.stopTime;
-        auto remoteStore = parsed.query.find("remote-store");
-        if (remoteStore != parsed.query.end()) {
-            result = {"--store", shellEscape(remoteStore->second)};
-        }
-    } catch (BadURL &) {
-        // We just try to continue with `machine->sshName` here for backwards compat.
     }
 
-    return result;
+    switch ((BuildResult::Status) buildResult.status) {
-}
+        case BuildResult::Built:
-
+            stepStatus = bsSuccess;
-static void openConnection(Machine::ptr machine, Path tmpDir, int stderrFD, Child & child)
+            break;
-{
+        case BuildResult::Substituted:
-    std::string pgmName;
+        case BuildResult::AlreadyValid:
-    Pipe to, from;
+            stepStatus = bsSuccess;
-    to.create();
+            isCached = true;
-    from.create();
+            break;
-
+        case BuildResult::PermanentFailure:
-    Strings argv;
+            stepStatus = bsFailed;
-    if (machine->isLocalhost()) {
+            canCache = true;
-        pgmName = "nix-store";
+            errorMsg = "";
-        argv = {"nix-store", "--builders", "", "--serve", "--write"};
+            break;
-    } else {
+        case BuildResult::InputRejected:
-        pgmName = "ssh";
+        case BuildResult::OutputRejected:
-        auto sshName = machine->sshName;
+            stepStatus = bsFailed;
-        Strings extraArgs = extraStoreArgs(sshName);
+            canCache = true;
-        argv = {"ssh", sshName};
+            break;
-        if (machine->sshKey != "") append(argv, {"-i", machine->sshKey});
+        case BuildResult::TransientFailure:
-        if (machine->sshPublicHostKey != "") {
+            stepStatus = bsFailed;
-            Path fileName = tmpDir + "/host-key";
+            canRetry = true;
-            auto p = machine->sshName.find("@");
+            errorMsg = "";
-            std::string host = p != std::string::npos ? std::string(machine->sshName, p + 1) : machine->sshName;
+            break;
-            writeFile(fileName, host + " " + machine->sshPublicHostKey + "\n");
+        case BuildResult::TimedOut:
-            append(argv, {"-oUserKnownHostsFile=" + fileName});
+            stepStatus = bsTimedOut;
-        }
+            errorMsg = "";
-        append(argv,
+            break;
-            { "-x", "-a", "-oBatchMode=yes", "-oConnectTimeout=60", "-oTCPKeepAlive=yes"
+        case BuildResult::MiscFailure:
-            , "--", "nix-store", "--serve", "--write" });
+            stepStatus = bsAborted;
-        append(argv, extraArgs);
+            canRetry = true;
+            break;
+        case BuildResult::LogLimitExceeded:
+            stepStatus = bsLogLimitExceeded;
+            break;
+        case BuildResult::NotDeterministic:
+            stepStatus = bsNotDeterministic;
+            canRetry = false;
+            canCache = true;
+            break;
+        default:
+            stepStatus = bsAborted;
+            break;
     }
 
-    child.pid = startProcess([&]() {
-        restoreProcessContext();
-
-        if (dup2(to.readSide.get(), STDIN_FILENO) == -1)
-            throw SysError("cannot dup input pipe to stdin");
-
-        if (dup2(from.writeSide.get(), STDOUT_FILENO) == -1)
-            throw SysError("cannot dup output pipe to stdout");
-
-        if (dup2(stderrFD, STDERR_FILENO) == -1)
-            throw SysError("cannot dup stderr");
-
-        execvp(argv.front().c_str(), (char * *) stringsToCharPtrs(argv).data()); // FIXME: remove cast
-
-        throw SysError("cannot start %s", pgmName);
-    });
-
-    to.readSide = -1;
-    from.writeSide = -1;
-
-    child.to = to.writeSide.release();
-    child.from = from.readSide.release();
-}
-
-
-static void copyClosureTo(std::timed_mutex & sendMutex, Store & destStore,
-    FdSource & from, FdSink & to, const StorePathSet & paths,
-    bool useSubstitutes = false)
-{
-    StorePathSet closure;
-    destStore.computeFSClosure(paths, closure);
-
-    WorkerProto::WriteConn wconn { .to = to };
-    WorkerProto::ReadConn rconn { .from = from };
-    /* Send the "query valid paths" command with the "lock" option
-       enabled. This prevents a race where the remote host
-       garbage-collect paths that are already there. Optionally, ask
-       the remote host to substitute missing paths. */
-    // FIXME: substitute output pollutes our build log
-    to << ServeProto::Command::QueryValidPaths << 1 << useSubstitutes;
-    WorkerProto::write(destStore, wconn, closure);
-    to.flush();
-
-    /* Get back the set of paths that are already valid on the remote
-       host. */
-    auto present = WorkerProto::Serialise<StorePathSet>::read(destStore, rconn);
-
-    if (present.size() == closure.size()) return;
-
-    auto sorted = destStore.topoSortPaths(closure);
-
-    StorePathSet missing;
-    for (auto i = sorted.rbegin(); i != sorted.rend(); ++i)
-        if (!present.count(*i)) missing.insert(*i);
-
-    printMsg(lvlDebug, "sending %d missing paths", missing.size());
-
-    std::unique_lock<std::timed_mutex> sendLock(sendMutex,
-        std::chrono::seconds(600));
-
-    to << ServeProto::Command::ImportPaths;
-    destStore.exportPaths(missing, to);
-    to.flush();
-
-    if (readInt(from) != 1)
-        throw Error("remote machine failed to import closure");
-}
-
-
-// FIXME: use Store::topoSortPaths().
-StorePaths reverseTopoSortPaths(const std::map<StorePath, ValidPathInfo> & paths)
-{
-    StorePaths sorted;
-    StorePathSet visited;
-
-    std::function<void(const StorePath & path)> dfsVisit;
-
-    dfsVisit = [&](const StorePath & path) {
-        if (!visited.insert(path).second) return;
-
-        auto info = paths.find(path);
-        auto references = info == paths.end() ? StorePathSet() : info->second.references;
-
-        for (auto & i : references)
-            /* Don't traverse into paths that don't exist.  That can
-               happen due to substitutes for non-existent paths. */
-            if (i != path && paths.count(i))
-                dfsVisit(i);
-
-        sorted.push_back(path);
-    };
-
-    for (auto & i : paths)
-        dfsVisit(i.first);
-
-    return sorted;
 }
 
 
 void State::buildRemote(ref<Store> destStore,
-    Machine::ptr machine, Step::ptr step,
+    ::Machine::ptr machine, Step::ptr step,
-    unsigned int maxSilentTime, unsigned int buildTimeout, unsigned int repeats,
+    const ServeProto::BuildOptions & buildOptions,
     RemoteResult & result, std::shared_ptr<ActiveStep> activeStep,
     std::function<void(StepState)> updateStep,
     NarMemberDatas & narMembers)
 {
     assert(BuildResult::TimedOut == 8);
 
-    std::string base(step->drvPath.to_string());
+    result.logFile = build_remote::createLogFileDir(logDir, step->drvPath);
-    result.logFile = logDir + "/" + std::string(base, 0, 2) + "/" + std::string(base, 2);
-    AutoDelete autoDelete(result.logFile, false);
-
-    createDirs(dirOf(result.logFile));
-
-    AutoCloseFD logFD = open(result.logFile.c_str(), O_CREAT | O_TRUNC | O_WRONLY, 0666);
-    if (!logFD) throw SysError("creating log file ‘%s’", result.logFile);
-
-    nix::Path tmpDir = createTempDir();
-    AutoDelete tmpDirDel(tmpDir, true);
 
     try {
 
-        updateStep(ssConnecting);
+        updateStep(ssBuilding);
+        result.startTime = time(0);
 
-        // FIXME: rewrite to use Store.
+        auto buildStoreUrl = machine->completeStoreReference().render();
-        Child child;
-        openConnection(machine, tmpDir, logFD.get(), child);
 
+        Strings args = {
+            localStore->printStorePath(step->drvPath),
+            "--store", destStore->getUri(),
+            "--eval-store", localStore->getUri(),
+            "--build-store", buildStoreUrl,
+            "--max-silent-time", std::to_string(buildOptions.maxSilentTime),
+            "--timeout", std::to_string(buildOptions.buildTimeout),
+            "--max-build-log-size", std::to_string(buildOptions.maxLogSize),
+            "--max-output-size", std::to_string(maxOutputSize),
+            "--repeat", std::to_string(buildOptions.nrRepeats),
+            "--log-file", result.logFile,
+            // FIXME: step->isDeterministic
+        };
+
+        // FIXME: set pid for cancellation
+
+        auto [status, childStdout] = [&]() {
+            MaintainCount<counter> mc(nrStepsBuilding);
+            return runProgram({
+                .program = "hydra-build-step",
+                .args = std::move(args),
+            });
+        }();
+
+        #if 0
         {
             auto activeStepState(activeStep->state_.lock());
             if (activeStepState->cancelled) throw Error("step cancelled");
-            activeStepState->pid = child.pid;
+            activeStepState->pid = conn.store->getConnectionPid();
         }
 
         Finally clearPid([&]() {
@@ -225,300 +147,55 @@ void State::buildRemote(ref<Store> destStore,
                possibility that we end up killing another
                process. Meh. */
         });
+        #endif
 
-        FdSource from(child.from.get());
+        result.stopTime = time(0);
-        WorkerProto::ReadConn rconn { .from = from };
-        FdSink to(child.to.get());
-        WorkerProto::WriteConn wconn { .to = to };
 
-        Finally updateStats([&]() {
+        if (!statusOk(status))
-            bytesReceived += from.read;
+            throw ExecError(status, fmt("hydra-build-step %s with output:\n%s", statusToString(status), stdout));
-            bytesSent += to.written;
-        });
-
-        /* Handshake. */
-        unsigned int remoteVersion;
-
-        try {
-            to << SERVE_MAGIC_1 << 0x206;
-            to.flush();
-
-            unsigned int magic = readInt(from);
-            if (magic != SERVE_MAGIC_2)
-                throw Error("protocol mismatch with ‘nix-store --serve’ on ‘%1%’", machine->sshName);
-            remoteVersion = readInt(from);
-            if (GET_PROTOCOL_MAJOR(remoteVersion) != 0x200)
-                throw Error("unsupported ‘nix-store --serve’ protocol version on ‘%1%’", machine->sshName);
-            if (GET_PROTOCOL_MINOR(remoteVersion) < 3 && repeats > 0)
-                throw Error("machine ‘%1%’ does not support repeating a build; please upgrade it to Nix 1.12", machine->sshName);
-
-        } catch (EndOfFile & e) {
-            child.pid.wait();
-            std::string s = chomp(readFile(result.logFile));
-            throw Error("cannot connect to ‘%1%’: %2%", machine->sshName, s);
-        }
 
+        /* The build was executed successfully, so clear the failure
+           count for this machine. */
         {
             auto info(machine->state->connectInfo.lock());
             info->consecutiveFailures = 0;
         }
 
-        /* Gather the inputs. If the remote side is Nix <= 1.9, we have to
+        StringSource from { childStdout };
-           copy the entire closure of ‘drvPath’, as well as the required
+        /* Read the BuildResult from the child. */
-           outputs of the input derivations. On Nix > 1.9, we only need to
+        WorkerProto::ReadConn rconn {
-           copy the immediate sources of the derivation and the required
+            .from = from,
-           outputs of the input derivations. */
+            // Hardcode latest version because we are deploying hydra
-        updateStep(ssSendingInputs);
+            // itself atomically
+            .version = PROTOCOL_VERSION,
+        };
+        result.overhead += readNum<uint64_t>(rconn.from);
+        auto totalNarSize = readNum<uint64_t>(rconn.from);
+        auto buildResult = WorkerProto::Serialise<BuildResult>::read(*localStore, rconn);
 
-        StorePathSet inputs;
-        BasicDerivation basicDrv(*step->drv);
 
-        for (auto & p : step->drv->inputSrcs)
+        result.updateWithBuildResult(buildResult);
-            inputs.insert(p);
 
-        for (auto & [drvPath, node] : step->drv->inputDrvs.map) {
-            auto drv2 = localStore->readDerivation(drvPath);
-            for (auto & name : node.value) {
-                if (auto i = get(drv2.outputs, name)) {
-                    auto outPath = i->path(*localStore, drv2.name, name);
-                    inputs.insert(*outPath);
-                    basicDrv.inputSrcs.insert(*outPath);
-                }
-            }
-        }
-
-        /* Ensure that the inputs exist in the destination store. This is
-           a no-op for regular stores, but for the binary cache store,
-           this will copy the inputs to the binary cache from the local
-           store. */
-        if (localStore != std::shared_ptr<Store>(destStore)) {
-            copyClosure(*localStore, *destStore,
-                step->drv->inputSrcs,
-                NoRepair, NoCheckSigs, NoSubstitute);
-        }
-
-        {
-            auto mc1 = std::make_shared<MaintainCount<counter>>(nrStepsWaiting);
-            mc1.reset();
-            MaintainCount<counter> mc2(nrStepsCopyingTo);
-
-            printMsg(lvlDebug, "sending closure of ‘%s’ to ‘%s’",
-                localStore->printStorePath(step->drvPath), machine->sshName);
-
-            auto now1 = std::chrono::steady_clock::now();
-
-            /* Copy the input closure. */
-            if (machine->isLocalhost()) {
-                StorePathSet closure;
-                destStore->computeFSClosure(inputs, closure);
-                copyPaths(*destStore, *localStore, closure, NoRepair, NoCheckSigs, NoSubstitute);
-            } else {
-                copyClosureTo(machine->state->sendLock, *destStore, from, to, inputs, true);
-            }
-
-            auto now2 = std::chrono::steady_clock::now();
-
-            result.overhead += std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count();
-        }
-
-        autoDelete.cancel();
-
-        /* Truncate the log to get rid of messages about substitutions
-           etc. on the remote system. */
-        if (lseek(logFD.get(), SEEK_SET, 0) != 0)
-            throw SysError("seeking to the start of log file ‘%s’", result.logFile);
-
-        if (ftruncate(logFD.get(), 0) == -1)
-            throw SysError("truncating log file ‘%s’", result.logFile);
-
-        logFD = -1;
-
-        /* Do the build. */
-        printMsg(lvlDebug, "building ‘%s’ on ‘%s’",
-            localStore->printStorePath(step->drvPath),
-            machine->sshName);
-
-        updateStep(ssBuilding);
-
-        to << ServeProto::Command::BuildDerivation << localStore->printStorePath(step->drvPath);
-        writeDerivation(to, *localStore, basicDrv);
-        to << maxSilentTime << buildTimeout;
-        if (GET_PROTOCOL_MINOR(remoteVersion) >= 2)
-            to << maxLogSize;
-        if (GET_PROTOCOL_MINOR(remoteVersion) >= 3) {
-            to << repeats // == build-repeat
-               << step->isDeterministic; // == enforce-determinism
-        }
-        to.flush();
-
-        result.startTime = time(0);
-        int res;
-        {
-            MaintainCount<counter> mc(nrStepsBuilding);
-            res = readInt(from);
-        }
-        result.stopTime = time(0);
-
-        result.errorMsg = readString(from);
-        if (GET_PROTOCOL_MINOR(remoteVersion) >= 3) {
-            result.timesBuilt = readInt(from);
-            result.isNonDeterministic = readInt(from);
-            auto start = readInt(from);
-            auto stop = readInt(from);
-            if (start && start) {
-                /* Note: this represents the duration of a single
-                    round, rather than all rounds. */
-                result.startTime = start;
-                result.stopTime = stop;
-            }
-        }
-        if (GET_PROTOCOL_MINOR(remoteVersion) >= 6) {
-            WorkerProto::Serialise<DrvOutputs>::read(*localStore, rconn);
-        }
-        switch ((BuildResult::Status) res) {
-            case BuildResult::Built:
-                result.stepStatus = bsSuccess;
-                break;
-            case BuildResult::Substituted:
-            case BuildResult::AlreadyValid:
-                result.stepStatus = bsSuccess;
-                result.isCached = true;
-                break;
-            case BuildResult::PermanentFailure:
-                result.stepStatus = bsFailed;
-                result.canCache = true;
-                result.errorMsg = "";
-                break;
-            case BuildResult::InputRejected:
-            case BuildResult::OutputRejected:
-                result.stepStatus = bsFailed;
-                result.canCache = true;
-                break;
-            case BuildResult::TransientFailure:
-                result.stepStatus = bsFailed;
-                result.canRetry = true;
-                result.errorMsg = "";
-                break;
-            case BuildResult::TimedOut:
-                result.stepStatus = bsTimedOut;
-                result.errorMsg = "";
-                break;
-            case BuildResult::MiscFailure:
-                result.stepStatus = bsAborted;
-                result.canRetry = true;
-                break;
-            case BuildResult::LogLimitExceeded:
-                result.stepStatus = bsLogLimitExceeded;
-                break;
-            case BuildResult::NotDeterministic:
-                result.stepStatus = bsNotDeterministic;
-                result.canRetry = false;
-                result.canCache = true;
-                break;
-            default:
-                result.stepStatus = bsAborted;
-                break;
-        }
         if (result.stepStatus != bsSuccess) return;
 
         result.errorMsg = "";
 
+        /* If the NAR size limit was exceeded, then hydra-build-step
+           will not have copied the output paths. */
+        if (totalNarSize > maxOutputSize) {
+            result.stepStatus = bsNarSizeLimitExceeded;
+            return;
+        }
+
         /* If the path was substituted or already valid, then we didn't
            get a build log. */
         if (result.isCached) {
             printMsg(lvlInfo, "outputs of ‘%s’ substituted or already valid on ‘%s’",
-                localStore->printStorePath(step->drvPath), machine->sshName);
+                localStore->printStorePath(step->drvPath), machine->storeUri.render());
             unlink(result.logFile.c_str());
             result.logFile = "";
         }
 
-        /* Copy the output paths. */
-        if (!machine->isLocalhost() || localStore != std::shared_ptr<Store>(destStore)) {
-            updateStep(ssReceivingOutputs);
-
-            MaintainCount<counter> mc(nrStepsCopyingFrom);
-
-            auto now1 = std::chrono::steady_clock::now();
-
-            StorePathSet outputs;
-            for (auto & i : step->drv->outputsAndOptPaths(*localStore)) {
-                if (i.second.second)
-                   outputs.insert(*i.second.second);
-            }
-
-            /* Get info about each output path. */
-            std::map<StorePath, ValidPathInfo> infos;
-            size_t totalNarSize = 0;
-            to << ServeProto::Command::QueryPathInfos;
-            WorkerProto::write(*localStore, wconn, outputs);
-            to.flush();
-            while (true) {
-                auto storePathS = readString(from);
-                if (storePathS == "") break;
-                auto deriver = readString(from); // deriver
-                auto references = WorkerProto::Serialise<StorePathSet>::read(*localStore, rconn);
-                readLongLong(from); // download size
-                auto narSize = readLongLong(from);
-                auto narHash = Hash::parseAny(readString(from), htSHA256);
-                auto ca = ContentAddress::parseOpt(readString(from));
-                readStrings<StringSet>(from); // sigs
-                ValidPathInfo info(localStore->parseStorePath(storePathS), narHash);
-                assert(outputs.count(info.path));
-                info.references = references;
-                info.narSize = narSize;
-                totalNarSize += info.narSize;
-                info.narHash = narHash;
-                info.ca = ca;
-                if (deriver != "")
-                    info.deriver = localStore->parseStorePath(deriver);
-                infos.insert_or_assign(info.path, info);
-            }
-
-            if (totalNarSize > maxOutputSize) {
-                result.stepStatus = bsNarSizeLimitExceeded;
-                return;
-            }
-
-            /* Copy each path. */
-            printMsg(lvlDebug, "copying outputs of ‘%s’ from ‘%s’ (%d bytes)",
-                localStore->printStorePath(step->drvPath), machine->sshName, totalNarSize);
-
-            auto pathsSorted = reverseTopoSortPaths(infos);
-
-            for (auto & path : pathsSorted) {
-                auto & info = infos.find(path)->second;
-
-                /* Receive the NAR from the remote and add it to the
-                   destination store. Meanwhile, extract all the info from the
-                   NAR that getBuildOutput() needs. */
-                auto source2 = sinkToSource([&](Sink & sink)
-                {
-                    /* Note: we should only send the command to dump the store
-                       path to the remote if the NAR is actually going to get read
-                       by the destination store, which won't happen if this path
-                       is already valid on the destination store. Since this
-                       lambda function only gets executed if someone tries to read
-                       from source2, we will send the command from here rather
-                       than outside the lambda. */
-                    to << ServeProto::Command::DumpStorePath << localStore->printStorePath(path);
-                    to.flush();
-
-                    TeeSource tee(from, sink);
-                    extractNarData(tee, localStore->printStorePath(path), narMembers);
-                });
-
-                destStore->addToStore(info, *source2, NoRepair, NoCheckSigs);
-            }
-
-            auto now2 = std::chrono::steady_clock::now();
-
-            result.overhead += std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count();
-        }
-
-        /* Shut down the connection. */
-        child.to = -1;
-        child.pid.wait();
-
     } catch (Error & e) {
         /* Disable this machine until a certain period of time has
            passed. This period increases on every consecutive
@@ -531,7 +208,7 @@ void State::buildRemote(ref<Store> destStore,
             info->consecutiveFailures = std::min(info->consecutiveFailures + 1, (unsigned int) 4);
             info->lastFailure = now;
             int delta = retryInterval * std::pow(retryBackoff, info->consecutiveFailures - 1) + (rand() % 30);
-            printMsg(lvlInfo, "will disable machine ‘%1%’ for %2%s", machine->sshName, delta);
+            printMsg(lvlInfo, "will disable machine ‘%1%’ for %2%s", machine->storeUri.render(), delta);
             info->disabledUntil = now + std::chrono::seconds(delta);
         }
         throw;

src/hydra-queue-runner/build-result.cc

@@ -1,7 +1,7 @@
 #include "hydra-build-result.hh"
 #include "store-api.hh"
 #include "util.hh"
-#include "fs-accessor.hh"
+#include "source-accessor.hh"
 
 #include <regex>
 
@@ -11,18 +11,18 @@ using namespace nix;
 BuildOutput getBuildOutput(
     nix::ref<Store> store,
     NarMemberDatas & narMembers,
-    const Derivation & drv)
+    const OutputPathMap derivationOutputs)
 {
     BuildOutput res;
 
     /* Compute the closure size. */
     StorePathSet outputs;
     StorePathSet closure;
-    for (auto & i : drv.outputsAndOptPaths(*store))
+    for (auto& [outputName, outputPath] : derivationOutputs) {
-        if (i.second.second) {
+        store->computeFSClosure(outputPath, closure);
-            store->computeFSClosure(*i.second.second, closure);
+        outputs.insert(outputPath);
-            outputs.insert(*i.second.second);
+        res.outputs.insert({outputName, outputPath});
-        }
+    }
     for (auto & path : closure) {
         auto info = store->queryPathInfo(path);
         res.closureSize += info->narSize;
@@ -63,7 +63,7 @@ BuildOutput getBuildOutput(
 
         auto productsFile = narMembers.find(outputS + "/nix-support/hydra-build-products");
         if (productsFile == narMembers.end() ||
-            productsFile->second.type != FSAccessor::Type::tRegular)
+            productsFile->second.type != SourceAccessor::Type::tRegular)
             continue;
         assert(productsFile->second.contents);
 
@@ -94,7 +94,7 @@ BuildOutput getBuildOutput(
 
             product.name = product.path == store->printStorePath(output) ? "" : baseNameOf(product.path);
 
-            if (file->second.type == FSAccessor::Type::tRegular) {
+            if (file->second.type == SourceAccessor::Type::tRegular) {
                 product.isRegular = true;
                 product.fileSize = file->second.fileSize.value();
                 product.sha256hash = file->second.sha256.value();
@@ -107,17 +107,16 @@ BuildOutput getBuildOutput(
     /* If no build products were explicitly declared, then add all
        outputs as a product of type "nix-build". */
     if (!explicitProducts) {
-        for (auto & [name, output] : drv.outputs) {
+        for (auto & [name, output] : derivationOutputs) {
             BuildProduct product;
-            auto outPath = output.path(*store, drv.name, name);
+            product.path = store->printStorePath(output);
-            product.path = store->printStorePath(*outPath);
             product.type = "nix-build";
             product.subtype = name == "out" ? "" : name;
-            product.name = outPath->name();
+            product.name = output.name();
 
             auto file = narMembers.find(product.path);
             assert(file != narMembers.end());
-            if (file->second.type == FSAccessor::Type::tDirectory)
+            if (file->second.type == SourceAccessor::Type::tDirectory)
                 res.products.push_back(product);
         }
     }
@@ -126,7 +125,7 @@ BuildOutput getBuildOutput(
     for (auto & output : outputs) {
         auto file = narMembers.find(store->printStorePath(output) + "/nix-support/hydra-release-name");
         if (file == narMembers.end() ||
-            file->second.type != FSAccessor::Type::tRegular)
+            file->second.type != SourceAccessor::Type::tRegular)
             continue;
         res.releaseName = trim(file->second.contents.value());
         // FIXME: validate release name
@@ -136,7 +135,7 @@ BuildOutput getBuildOutput(
     for (auto & output : outputs) {
         auto file = narMembers.find(store->printStorePath(output) + "/nix-support/hydra-metrics");
         if (file == narMembers.end() ||
-            file->second.type != FSAccessor::Type::tRegular)
+            file->second.type != SourceAccessor::Type::tRegular)
             continue;
         for (auto & line : tokenizeString<Strings>(file->second.contents.value(), "\n")) {
             auto fields = tokenizeString<std::vector<std::string>>(line);

src/hydra-queue-runner/builder.cc

@@ -3,6 +3,7 @@
 #include "state.hh"
 #include "hydra-build-result.hh"
 #include "finally.hh"
+#include "terminal.hh"
 #include "binary-cache-store.hh"
 
 using namespace nix;
@@ -41,7 +42,7 @@ void State::builder(MachineReservation::ptr reservation)
         } catch (std::exception & e) {
             printMsg(lvlError, "uncaught exception building ‘%s’ on ‘%s’: %s",
                 localStore->printStorePath(reservation->step->drvPath),
-                reservation->machine->sshName,
+                reservation->machine->storeUri.render(),
                 e.what());
         }
     }
@@ -98,8 +99,13 @@ State::StepResult State::doBuildStep(nix::ref<Store> destStore,
        it). */
     BuildID buildId;
     std::optional<StorePath> buildDrvPath;
-    unsigned int maxSilentTime, buildTimeout;
+    // Other fields set below
-    unsigned int repeats = step->isDeterministic ? 1 : 0;
+    nix::ServeProto::BuildOptions buildOptions {
+        .maxLogSize = maxLogSize,
+        .nrRepeats = step->isDeterministic ? 1u : 0u,
+        .enforceDeterminism = step->isDeterministic,
+        .keepFailed = false,
+    };
 
     auto conn(dbPool.get());
 
@@ -134,18 +140,18 @@ State::StepResult State::doBuildStep(nix::ref<Store> destStore,
             {
                 auto i = jobsetRepeats.find(std::make_pair(build2->projectName, build2->jobsetName));
                 if (i != jobsetRepeats.end())
-                    repeats = std::max(repeats, i->second);
+                    buildOptions.nrRepeats = std::max(buildOptions.nrRepeats, i->second);
             }
         }
         if (!build) build = *dependents.begin();
 
         buildId = build->id;
         buildDrvPath = build->drvPath;
-        maxSilentTime = build->maxSilentTime;
+        buildOptions.maxSilentTime = build->maxSilentTime;
-        buildTimeout = build->buildTimeout;
+        buildOptions.buildTimeout = build->buildTimeout;
 
         printInfo("performing step ‘%s’ %d times on ‘%s’ (needed by build %d and %d others)",
-            localStore->printStorePath(step->drvPath), repeats + 1, machine->sshName, buildId, (dependents.size() - 1));
+            localStore->printStorePath(step->drvPath), buildOptions.nrRepeats + 1, machine->storeUri.render(), buildId, (dependents.size() - 1));
     }
 
     if (!buildOneDone)
@@ -173,7 +179,7 @@ State::StepResult State::doBuildStep(nix::ref<Store> destStore,
                     unlink(result.logFile.c_str());
                 }
             } catch (...) {
-                ignoreException();
+                ignoreExceptionInDestructor();
             }
         }
     });
@@ -191,7 +197,7 @@ State::StepResult State::doBuildStep(nix::ref<Store> destStore,
         {
             auto mc = startDbUpdate();
             pqxx::work txn(*conn);
-            stepNr = createBuildStep(txn, result.startTime, buildId, step, machine->sshName, bsBusy);
+            stepNr = createBuildStep(txn, result.startTime, buildId, step, machine->storeUri.render(), bsBusy);
             txn.commit();
         }
 
@@ -206,7 +212,7 @@ State::StepResult State::doBuildStep(nix::ref<Store> destStore,
 
         try {
             /* FIXME: referring builds may have conflicting timeouts. */
-            buildRemote(destStore, machine, step, maxSilentTime, buildTimeout, repeats, result, activeStep, updateStep, narMembers);
+            buildRemote(destStore, machine, step, buildOptions, result, activeStep, updateStep, narMembers);
         } catch (Error & e) {
             if (activeStep->state_.lock()->cancelled) {
                 printInfo("marking step %d of build %d as cancelled", stepNr, buildId);
@@ -214,14 +220,14 @@ State::StepResult State::doBuildStep(nix::ref<Store> destStore,
                 result.canRetry = false;
             } else {
                 result.stepStatus = bsAborted;
-                result.errorMsg = e.msg();
+                result.errorMsg = filterANSIEscapes(e.msg(), true);
                 result.canRetry = true;
             }
         }
 
         if (result.stepStatus == bsSuccess) {
             updateStep(ssPostProcessing);
-            res = getBuildOutput(destStore, narMembers, *step->drv);
+            res = getBuildOutput(destStore, narMembers, destStore->queryDerivationOutputMap(step->drvPath, &*localStore));
         }
     }
 
@@ -248,7 +254,7 @@ State::StepResult State::doBuildStep(nix::ref<Store> destStore,
     /* Finish the step in the database. */
     if (stepNr) {
         pqxx::work txn(*conn);
-        finishBuildStep(txn, result, buildId, stepNr, machine->sshName);
+        finishBuildStep(txn, result, buildId, stepNr, machine->storeUri.render());
         txn.commit();
     }
 
@@ -256,7 +262,7 @@ State::StepResult State::doBuildStep(nix::ref<Store> destStore,
        issue). Retry a number of times. */
     if (result.canRetry) {
         printMsg(lvlError, "possibly transient failure building ‘%s’ on ‘%s’: %s",
-            localStore->printStorePath(step->drvPath), machine->sshName, result.errorMsg);
+            localStore->printStorePath(step->drvPath), machine->storeUri.render(), result.errorMsg);
         assert(stepNr);
         bool retry;
         {
@@ -275,9 +281,12 @@ State::StepResult State::doBuildStep(nix::ref<Store> destStore,
 
         assert(stepNr);
 
-        for (auto & i : step->drv->outputsAndOptPaths(*localStore)) {
+        for (auto & [outputName, optOutputPath] : destStore->queryPartialDerivationOutputMap(step->drvPath, &*localStore)) {
-            if (i.second.second)
+            if (!optOutputPath)
-               addRoot(*i.second.second);
+                throw Error(
+                    "Missing output %s for derivation %d which was supposed to have succeeded",
+                    outputName, localStore->printStorePath(step->drvPath));
+            addRoot(*optOutputPath);
         }
 
         /* Register success in the database for all Build objects that
@@ -398,7 +407,7 @@ void State::failStep(
     Step::ptr step,
     BuildID buildId,
     const RemoteResult & result,
-    Machine::ptr machine,
+    ::Machine::ptr machine,
     bool & stepFinished)
 {
     /* Register failure in the database for all Build objects that
@@ -444,7 +453,7 @@ void State::failStep(
                     build->finishedInDB)
                     continue;
                 createBuildStep(txn,
-                    0, build->id, step, machine ? machine->sshName : "",
+                    0, build->id, step, machine ? machine->storeUri.render() : "",
                     result.stepStatus, result.errorMsg, buildId == build->id ? 0 : buildId);
             }
 

src/hydra-queue-runner/dispatcher.cc

@@ -2,6 +2,7 @@
 #include <cmath>
 #include <thread>
 #include <unordered_map>
+#include <unordered_set>
 
 #include "state.hh"
 
@@ -85,12 +86,113 @@ system_time State::doDispatch()
         }
     }
 
+    system_time now = std::chrono::system_clock::now();
+
     /* Start steps until we're out of steps or slots. */
     auto sleepUntil = system_time::max();
     bool keepGoing;
 
+    /* Sort the runnable steps by priority. Priority is establised
+       as follows (in order of precedence):
+
+       - The global priority of the builds that depend on the
+         step. This allows admins to bump a build to the front of
+         the queue.
+
+       - The lowest used scheduling share of the jobsets depending
+         on the step.
+
+       - The local priority of the build, as set via the build's
+         meta.schedulingPriority field. Note that this is not
+         quite correct: the local priority should only be used to
+         establish priority between builds in the same jobset, but
+         here it's used between steps in different jobsets if they
+         happen to have the same lowest used scheduling share. But
+         that's not very likely.
+
+       - The lowest ID of the builds depending on the step;
+         i.e. older builds take priority over new ones.
+
+       FIXME: O(n lg n); obviously, it would be better to keep a
+       runnable queue sorted by priority. */
+    struct StepInfo
+    {
+        Step::ptr step;
+        bool alreadyScheduled = false;
+
+        /* The lowest share used of any jobset depending on this
+           step. */
+        double lowestShareUsed = 1e9;
+
+        /* Info copied from step->state to ensure that the
+           comparator is a partial ordering (see MachineInfo). */
+        int highestGlobalPriority;
+        int highestLocalPriority;
+        BuildID lowestBuildID;
+
+        StepInfo(Step::ptr step, Step::State & step_) : step(step)
+        {
+            for (auto & jobset : step_.jobsets)
+                lowestShareUsed = std::min(lowestShareUsed, jobset->shareUsed());
+            highestGlobalPriority = step_.highestGlobalPriority;
+            highestLocalPriority = step_.highestLocalPriority;
+            lowestBuildID = step_.lowestBuildID;
+        }
+    };
+
+    std::vector<StepInfo> runnableSorted;
+
+    struct RunnablePerType
+    {
+        unsigned int count{0};
+        std::chrono::seconds waitTime{0};
+    };
+
+    std::unordered_map<std::string, RunnablePerType> runnablePerType;
+
+    {
+        auto runnable_(runnable.lock());
+        runnableSorted.reserve(runnable_->size());
+        for (auto i = runnable_->begin(); i != runnable_->end(); ) {
+            auto step = i->lock();
+
+            /* Remove dead steps. */
+            if (!step) {
+                i = runnable_->erase(i);
+                continue;
+            }
+
+            ++i;
+
+            auto & r = runnablePerType[step->systemType];
+            r.count++;
+
+            /* Skip previously failed steps that aren't ready
+               to be retried. */
+            auto step_(step->state.lock());
+            r.waitTime += std::chrono::duration_cast<std::chrono::seconds>(now - step_->runnableSince);
+            if (step_->tries > 0 && step_->after > now) {
+                if (step_->after < sleepUntil)
+                    sleepUntil = step_->after;
+                continue;
+            }
+
+            runnableSorted.emplace_back(step, *step_);
+        }
+    }
+
+    sort(runnableSorted.begin(), runnableSorted.end(),
+        [](const StepInfo & a, const StepInfo & b)
+        {
+            return
+                a.highestGlobalPriority != b.highestGlobalPriority ? a.highestGlobalPriority > b.highestGlobalPriority :
+                a.lowestShareUsed != b.lowestShareUsed ? a.lowestShareUsed < b.lowestShareUsed :
+                a.highestLocalPriority != b.highestLocalPriority ? a.highestLocalPriority > b.highestLocalPriority :
+                a.lowestBuildID < b.lowestBuildID;
+        });
+
     do {
-        system_time now = std::chrono::system_clock::now();
+        now = std::chrono::system_clock::now();
 
         /* Copy the currentJobs field of each machine. This is
            necessary to ensure that the sort comparator below is
@@ -98,7 +200,7 @@ system_time State::doDispatch()
            filter out temporarily disabled machines. */
         struct MachineInfo
         {
-            Machine::ptr machine;
+            ::Machine::ptr machine;
             unsigned long currentJobs;
         };
         std::vector<MachineInfo> machinesSorted;
@@ -138,104 +240,6 @@ system_time State::doDispatch()
                     a.currentJobs > b.currentJobs;
             });
 
-        /* Sort the runnable steps by priority. Priority is establised
-           as follows (in order of precedence):
-
-           - The global priority of the builds that depend on the
-             step. This allows admins to bump a build to the front of
-             the queue.
-
-           - The lowest used scheduling share of the jobsets depending
-             on the step.
-
-           - The local priority of the build, as set via the build's
-             meta.schedulingPriority field. Note that this is not
-             quite correct: the local priority should only be used to
-             establish priority between builds in the same jobset, but
-             here it's used between steps in different jobsets if they
-             happen to have the same lowest used scheduling share. But
-             that's not very likely.
-
-           - The lowest ID of the builds depending on the step;
-             i.e. older builds take priority over new ones.
-
-           FIXME: O(n lg n); obviously, it would be better to keep a
-           runnable queue sorted by priority. */
-        struct StepInfo
-        {
-            Step::ptr step;
-
-            /* The lowest share used of any jobset depending on this
-               step. */
-            double lowestShareUsed = 1e9;
-
-            /* Info copied from step->state to ensure that the
-               comparator is a partial ordering (see MachineInfo). */
-            int highestGlobalPriority;
-            int highestLocalPriority;
-            BuildID lowestBuildID;
-
-            StepInfo(Step::ptr step, Step::State & step_) : step(step)
-            {
-                for (auto & jobset : step_.jobsets)
-                    lowestShareUsed = std::min(lowestShareUsed, jobset->shareUsed());
-                highestGlobalPriority = step_.highestGlobalPriority;
-                highestLocalPriority = step_.highestLocalPriority;
-                lowestBuildID = step_.lowestBuildID;
-            }
-        };
-
-        std::vector<StepInfo> runnableSorted;
-
-        struct RunnablePerType
-        {
-            unsigned int count{0};
-            std::chrono::seconds waitTime{0};
-        };
-
-        std::unordered_map<std::string, RunnablePerType> runnablePerType;
-
-        {
-            auto runnable_(runnable.lock());
-            runnableSorted.reserve(runnable_->size());
-            for (auto i = runnable_->begin(); i != runnable_->end(); ) {
-                auto step = i->lock();
-
-                /* Remove dead steps. */
-                if (!step) {
-                    i = runnable_->erase(i);
-                    continue;
-                }
-
-                ++i;
-
-                auto & r = runnablePerType[step->systemType];
-                r.count++;
-
-                /* Skip previously failed steps that aren't ready
-                   to be retried. */
-                auto step_(step->state.lock());
-                r.waitTime += std::chrono::duration_cast<std::chrono::seconds>(now - step_->runnableSince);
-                if (step_->tries > 0 && step_->after > now) {
-                    if (step_->after < sleepUntil)
-                        sleepUntil = step_->after;
-                    continue;
-                }
-
-                runnableSorted.emplace_back(step, *step_);
-            }
-        }
-
-        sort(runnableSorted.begin(), runnableSorted.end(),
-            [](const StepInfo & a, const StepInfo & b)
-            {
-                return
-                    a.highestGlobalPriority != b.highestGlobalPriority ? a.highestGlobalPriority > b.highestGlobalPriority :
-                    a.lowestShareUsed != b.lowestShareUsed ? a.lowestShareUsed < b.lowestShareUsed :
-                    a.highestLocalPriority != b.highestLocalPriority ? a.highestLocalPriority > b.highestLocalPriority :
-                    a.lowestBuildID < b.lowestBuildID;
-            });
-
         /* Find a machine with a free slot and find a step to run
            on it. Once we find such a pair, we restart the outer
            loop because the machine sorting will have changed. */
@@ -245,12 +249,14 @@ system_time State::doDispatch()
             if (mi.machine->state->currentJobs >= mi.machine->maxJobs) continue;
 
             for (auto & stepInfo : runnableSorted) {
+                if (stepInfo.alreadyScheduled) continue;
+
                 auto & step(stepInfo.step);
 
                 /* Can this machine do this step? */
                 if (!mi.machine->supportsStep(step)) {
                     debug("machine '%s' does not support step '%s' (system type '%s')",
-                        mi.machine->sshName, localStore->printStorePath(step->drvPath), step->drv->platform);
+                        mi.machine->storeUri.render(), localStore->printStorePath(step->drvPath), step->drv->platform);
                     continue;
                 }
 
@@ -271,6 +277,8 @@ system_time State::doDispatch()
                     r.count--;
                 }
 
+                stepInfo.alreadyScheduled = true;
+
                 /* Make a slot reservation and start a thread to
                    do the build. */
                 auto builderThread = std::thread(&State::builder, this,
@@ -428,7 +436,7 @@ void Jobset::pruneSteps()
 }
 
 
-State::MachineReservation::MachineReservation(State & state, Step::ptr step, Machine::ptr machine)
+State::MachineReservation::MachineReservation(State & state, Step::ptr step, ::Machine::ptr machine)
     : state(state), step(step), machine(machine)
 {
     machine->state->currentJobs++;

src/hydra-queue-runner/hydra-build-result.hh

@@ -36,10 +36,12 @@ struct BuildOutput
 
     std::list<BuildProduct> products;
 
+    std::map<std::string, nix::StorePath> outputs;
+
     std::map<std::string, BuildMetric> metrics;
 };
 
 BuildOutput getBuildOutput(
     nix::ref<nix::Store> store,
     NarMemberDatas & narMembers,
-    const nix::Derivation & drv);
+    const nix::OutputPathMap derivationOutputs);

src/hydra-queue-runner/hydra-queue-runner.cc

@@ -1,6 +1,7 @@
 #include <iostream>
 #include <thread>
 #include <optional>
+#include <type_traits>
 
 #include <sys/types.h>
 #include <sys/stat.h>
@@ -10,6 +11,7 @@
 
 #include <nlohmann/json.hpp>
 
+#include "signals.hh"
 #include "state.hh"
 #include "hydra-build-result.hh"
 #include "store-api.hh"
@@ -133,50 +135,29 @@ void State::parseMachines(const std::string & contents)
         oldMachines = *machines_;
     }
 
-    for (auto line : tokenizeString<Strings>(contents, "\n")) {
+    for (auto && machine_ : nix::Machine::parseConfig({}, contents)) {
-        line = trim(std::string(line, 0, line.find('#')));
+        auto machine = std::make_shared<::Machine>(std::move(machine_));
-        auto tokens = tokenizeString<std::vector<std::string>>(line);
-        if (tokens.size() < 3) continue;
-        tokens.resize(8);
-
-        auto machine = std::make_shared<Machine>();
-        machine->sshName = tokens[0];
-        machine->systemTypes = tokenizeString<StringSet>(tokens[1], ",");
-        machine->sshKey = tokens[2] == "-" ? std::string("") : tokens[2];
-        if (tokens[3] != "")
-            machine->maxJobs = string2Int<decltype(machine->maxJobs)>(tokens[3]).value();
-        else
-            machine->maxJobs = 1;
-        machine->speedFactor = atof(tokens[4].c_str());
-        if (tokens[5] == "-") tokens[5] = "";
-        machine->supportedFeatures = tokenizeString<StringSet>(tokens[5], ",");
-        if (tokens[6] == "-") tokens[6] = "";
-        machine->mandatoryFeatures = tokenizeString<StringSet>(tokens[6], ",");
-        for (auto & f : machine->mandatoryFeatures)
-            machine->supportedFeatures.insert(f);
-        if (tokens[7] != "" && tokens[7] != "-")
-            machine->sshPublicHostKey = base64Decode(tokens[7]);
 
         /* Re-use the State object of the previous machine with the
            same name. */
-        auto i = oldMachines.find(machine->sshName);
+        auto i = oldMachines.find(machine->storeUri.variant);
         if (i == oldMachines.end())
-            printMsg(lvlChatty, "adding new machine ‘%1%’", machine->sshName);
+            printMsg(lvlChatty, "adding new machine ‘%1%’", machine->storeUri.render());
         else
-            printMsg(lvlChatty, "updating machine ‘%1%’", machine->sshName);
+            printMsg(lvlChatty, "updating machine ‘%1%’", machine->storeUri.render());
         machine->state = i == oldMachines.end()
-            ? std::make_shared<Machine::State>()
+            ? std::make_shared<::Machine::State>()
             : i->second->state;
-        newMachines[machine->sshName] = machine;
+        newMachines[machine->storeUri.variant] = machine;
     }
 
     for (auto & m : oldMachines)
         if (newMachines.find(m.first) == newMachines.end()) {
             if (m.second->enabled)
-                printInfo("removing machine ‘%1%’", m.first);
+                printInfo("removing machine ‘%1%’", m.second->storeUri.render());
-            /* Add a disabled Machine object to make sure stats are
+            /* Add a disabled ::Machine object to make sure stats are
                maintained. */
-            auto machine = std::make_shared<Machine>(*(m.second));
+            auto machine = std::make_shared<::Machine>(*(m.second));
             machine->enabled = false;
             newMachines[m.first] = machine;
         }
@@ -204,7 +185,7 @@ void State::monitorMachinesFile()
         parseMachines("localhost " +
             (settings.thisSystem == "x86_64-linux" ? "x86_64-linux,i686-linux" : settings.thisSystem.get())
             + " - " + std::to_string(settings.maxBuildJobs) + " 1 "
-            + concatStringsSep(",", settings.systemFeatures.get()));
+            + concatStringsSep(",", StoreConfig::getDefaultSystemFeatures()));
         machinesReadyLock.unlock();
         return;
     }
@@ -311,10 +292,13 @@ unsigned int State::createBuildStep(pqxx::work & txn, time_t startTime, BuildID
 
     if (r.affected_rows() == 0) goto restart;
 
-    for (auto & [name, output] : step->drv->outputs)
+    for (auto & [name, output] : getDestStore()->queryPartialDerivationOutputMap(step->drvPath, &*localStore))
         txn.exec_params0
             ("insert into BuildStepOutputs (build, stepnr, name, path) values ($1, $2, $3, $4)",
-            buildId, stepNr, name, localStore->printStorePath(*output.path(*localStore, step->drv->name, name)));
+            buildId, stepNr, name,
+            output
+                ? std::optional { localStore->printStorePath(*output)}
+                : std::nullopt);
 
     if (status == bsBusy)
         txn.exec(fmt("notify step_started, '%d\t%d'", buildId, stepNr));
@@ -351,11 +335,23 @@ void State::finishBuildStep(pqxx::work & txn, const RemoteResult & result,
     assert(result.logFile.find('\t') == std::string::npos);
     txn.exec(fmt("notify step_finished, '%d\t%d\t%s'",
             buildId, stepNr, result.logFile));
+
+    if (result.stepStatus == bsSuccess) {
+        // Update the corresponding `BuildStepOutputs` row to add the output path
+        auto res = txn.exec_params1("select drvPath from BuildSteps where build = $1 and stepnr = $2", buildId, stepNr);
+        assert(res.size());
+        StorePath drvPath = localStore->parseStorePath(res[0].as<std::string>());
+        // If we've finished building, all the paths should be known
+        for (auto & [name, output] : getDestStore()->queryDerivationOutputMap(drvPath, &*localStore))
+            txn.exec_params0
+                ("update BuildStepOutputs set path = $4 where build = $1 and stepnr = $2 and name = $3",
+                  buildId, stepNr, name, localStore->printStorePath(output));
+    }
 }
 
 
 int State::createSubstitutionStep(pqxx::work & txn, time_t startTime, time_t stopTime,
-    Build::ptr build, const StorePath & drvPath, const std::string & outputName, const StorePath & storePath)
+    Build::ptr build, const StorePath & drvPath, const nix::Derivation drv, const std::string & outputName, const StorePath & storePath)
 {
  restart:
     auto stepNr = allocBuildStep(txn, build->id);
@@ -456,6 +452,15 @@ void State::markSucceededBuild(pqxx::work & txn, Build::ptr build,
          res.releaseName != "" ? std::make_optional(res.releaseName) : std::nullopt,
          isCachedBuild ? 1 : 0);
 
+    for (auto & [outputName, outputPath] : res.outputs) {
+        txn.exec_params0
+            ("update BuildOutputs set path = $3 where build = $1 and name = $2",
+             build->id,
+             outputName,
+             localStore->printStorePath(outputPath)
+            );
+    }
+
     txn.exec_params0("delete from BuildProducts where build = $1", build->id);
 
     unsigned int productNr = 1;
@@ -467,7 +472,7 @@ void State::markSucceededBuild(pqxx::work & txn, Build::ptr build,
              product.type,
              product.subtype,
              product.fileSize ? std::make_optional(*product.fileSize) : std::nullopt,
-             product.sha256hash ? std::make_optional(product.sha256hash->to_string(Base16, false)) : std::nullopt,
+             product.sha256hash ? std::make_optional(product.sha256hash->to_string(HashFormat::Base16, false)) : std::nullopt,
              product.path,
              product.name,
              product.defaultPath);
@@ -545,12 +550,11 @@ void State::dumpStatus(Connection & conn)
         {"nrQueuedBuilds", builds.lock()->size()},
         {"nrActiveSteps", activeSteps_.lock()->size()},
         {"nrStepsBuilding", nrStepsBuilding.load()},
+        #if 0
         {"nrStepsCopyingTo", nrStepsCopyingTo.load()},
         {"nrStepsCopyingFrom", nrStepsCopyingFrom.load()},
-        {"nrStepsWaiting", nrStepsWaiting.load()},
+        #endif
         {"nrUnsupportedSteps", nrUnsupportedSteps.load()},
-        {"bytesSent", bytesSent.load()},
-        {"bytesReceived", bytesReceived.load()},
         {"nrBuildsRead", nrBuildsRead.load()},
         {"buildReadTimeMs", buildReadTimeMs.load()},
         {"buildReadTimeAvgMs", nrBuildsRead == 0 ? 0.0 : (float) buildReadTimeMs / nrBuildsRead},
@@ -613,7 +617,7 @@ void State::dumpStatus(Connection & conn)
                     machine["avgStepTime"] = (float) s->totalStepTime / s->nrStepsDone;
                     machine["avgStepBuildTime"] = (float) s->totalStepBuildTime / s->nrStepsDone;
                 }
-                statusJson["machines"][m->sshName] = machine;
+                statusJson["machines"][m->storeUri.render()] = machine;
             }
         }
 
@@ -882,10 +886,17 @@ void State::run(BuildID buildOne)
     while (true) {
         try {
             auto conn(dbPool.get());
-            receiver dumpStatus_(*conn, "dump_status");
+            try {
-            while (true) {
+                receiver dumpStatus_(*conn, "dump_status");
-                conn->await_notification();
+                while (true) {
-                dumpStatus(*conn);
+                    conn->await_notification();
+                    dumpStatus(*conn);
+                }
+            } catch (pqxx::broken_connection & connEx) {
+                printMsg(lvlError, "main thread: %s", connEx.what());
+                printMsg(lvlError, "main thread: Reconnecting in 10s");
+                conn.markBad();
+                sleep(10);
             }
         } catch (std::exception & e) {
             printMsg(lvlError, "main thread: %s", e.what());

src/hydra-queue-runner/meson.build

@@ -0,0 +1,22 @@
+srcs = files(
+  'builder.cc',
+  'build-remote.cc',
+  'build-result.cc',
+  'dispatcher.cc',
+  'hydra-queue-runner.cc',
+  'nar-extractor.cc',
+  'queue-monitor.cc',
+)
+
+hydra_queue_runner = executable('hydra-queue-runner',
+  'hydra-queue-runner.cc',
+  srcs,
+  dependencies: [
+    libhydra_dep,
+    nix_dep,
+    pqxx_dep,
+    prom_cpp_core_dep,
+    prom_cpp_pull_dep,
+  ],
+  install: true,
+)

src/hydra-queue-runner/nar-extractor.cc

@@ -6,7 +6,46 @@
 
 using namespace nix;
 
-struct Extractor : ParseSink
+
+struct NarMemberConstructor : CreateRegularFileSink
+{
+    NarMemberData & curMember;
+
+    HashSink hashSink = HashSink { HashAlgorithm::SHA256 };
+
+    std::optional<uint64_t> expectedSize;
+
+    NarMemberConstructor(NarMemberData & curMember)
+        : curMember(curMember)
+    { }
+
+    void isExecutable() override
+    {
+    }
+
+    void preallocateContents(uint64_t size) override
+    {
+        expectedSize = size;
+    }
+
+    void operator () (std::string_view data) override
+    {
+        assert(expectedSize);
+        *curMember.fileSize += data.size();
+        hashSink(data);
+        if (curMember.contents) {
+            curMember.contents->append(data);
+        }
+        assert(curMember.fileSize <= expectedSize);
+        if (curMember.fileSize == expectedSize) {
+            auto [hash, len] = hashSink.finish();
+            assert(curMember.fileSize == len);
+            curMember.sha256 = hash;
+        }
+    }
+};
+
+struct Extractor : FileSystemObjectSink
 {
     std::unordered_set<Path> filesToKeep {
         "/nix-support/hydra-build-products",
@@ -15,58 +54,40 @@ struct Extractor : ParseSink
     };
 
     NarMemberDatas & members;
-    NarMemberData * curMember = nullptr;
+    std::filesystem::path prefix;
-    Path prefix;
+
+    Path toKey(const CanonPath & path)
+    {
+        std::filesystem::path p = prefix;
+        // Conditional to avoid trailing slash
+        if (!path.isRoot()) p /= path.rel();
+        return p;
+    }
 
     Extractor(NarMemberDatas & members, const Path & prefix)
         : members(members), prefix(prefix)
     { }
 
-    void createDirectory(const Path & path) override
+    void createDirectory(const CanonPath & path) override
     {
-        members.insert_or_assign(prefix + path, NarMemberData { .type = FSAccessor::Type::tDirectory });
+        members.insert_or_assign(toKey(path), NarMemberData { .type = SourceAccessor::Type::tDirectory });
     }
 
-    void createRegularFile(const Path & path) override
+    void createRegularFile(const CanonPath & path, std::function<void(CreateRegularFileSink &)> func) override
     {
-        curMember = &members.insert_or_assign(prefix + path, NarMemberData {
+        NarMemberConstructor nmc {
-            .type = FSAccessor::Type::tRegular,
+            members.insert_or_assign(toKey(path), NarMemberData {
-            .fileSize = 0,
+                .type = SourceAccessor::Type::tRegular,
-            .contents = filesToKeep.count(path) ? std::optional("") : std::nullopt,
+                .fileSize = 0,
-        }).first->second;
+                .contents = filesToKeep.count(path.abs()) ? std::optional("") : std::nullopt,
+            }).first->second,
+        };
+        func(nmc);
     }
 
-    std::optional<uint64_t> expectedSize;
+    void createSymlink(const CanonPath & path, const std::string & target) override
-    std::unique_ptr<HashSink> hashSink;
-
-    void preallocateContents(uint64_t size) override
     {
-        expectedSize = size;
+        members.insert_or_assign(toKey(path), NarMemberData { .type = SourceAccessor::Type::tSymlink });
-        hashSink = std::make_unique<HashSink>(htSHA256);
-    }
-
-    void receiveContents(std::string_view data) override
-    {
-        assert(expectedSize);
-        assert(curMember);
-        assert(hashSink);
-        *curMember->fileSize += data.size();
-        (*hashSink)(data);
-        if (curMember->contents) {
-            curMember->contents->append(data);
-        }
-        assert(curMember->fileSize <= expectedSize);
-        if (curMember->fileSize == expectedSize) {
-            auto [hash, len] = hashSink->finish();
-            assert(curMember->fileSize == len);
-            curMember->sha256 = hash;
-            hashSink.reset();
-        }
-    }
-
-    void createSymlink(const Path & path, const std::string & target) override
-    {
-        members.insert_or_assign(prefix + path, NarMemberData { .type = FSAccessor::Type::tSymlink });
     }
 };
 

src/hydra-queue-runner/nar-extractor.hh

@@ -1,13 +1,13 @@
 #pragma once
 
-#include "fs-accessor.hh"
+#include "source-accessor.hh"
 #include "types.hh"
 #include "serialise.hh"
 #include "hash.hh"
 
 struct NarMemberData
 {
-    nix::FSAccessor::Type type;
+    nix::SourceAccessor::Type type;
     std::optional<uint64_t> fileSize;
     std::optional<std::string> contents;
     std::optional<nix::Hash> sha256;

src/hydra-queue-runner/queue-monitor.cc

@@ -10,8 +10,14 @@ using namespace nix;
 void State::queueMonitor()
 {
     while (true) {
+        auto conn(dbPool.get());
         try {
-            queueMonitorLoop();
+            queueMonitorLoop(*conn);
+        } catch (pqxx::broken_connection & e) {
+            printMsg(lvlError, "queue monitor: %s", e.what());
+            printMsg(lvlError, "queue monitor: Reconnecting in 10s");
+            conn.markBad();
+            sleep(10);
         } catch (std::exception & e) {
             printError("queue monitor: %s", e.what());
             sleep(10); // probably a DB problem, so don't retry right away
@@ -20,16 +26,14 @@ void State::queueMonitor()
 }
 
 
-void State::queueMonitorLoop()
+void State::queueMonitorLoop(Connection & conn)
 {
-    auto conn(dbPool.get());
+    receiver buildsAdded(conn, "builds_added");
-
+    receiver buildsRestarted(conn, "builds_restarted");
-    receiver buildsAdded(*conn, "builds_added");
+    receiver buildsCancelled(conn, "builds_cancelled");
-    receiver buildsRestarted(*conn, "builds_restarted");
+    receiver buildsDeleted(conn, "builds_deleted");
-    receiver buildsCancelled(*conn, "builds_cancelled");
+    receiver buildsBumped(conn, "builds_bumped");
-    receiver buildsDeleted(*conn, "builds_deleted");
+    receiver jobsetSharesChanged(conn, "jobset_shares_changed");
-    receiver buildsBumped(*conn, "builds_bumped");
-    receiver jobsetSharesChanged(*conn, "jobset_shares_changed");
 
     auto destStore = getDestStore();
 
@@ -39,17 +43,17 @@ void State::queueMonitorLoop()
     while (!quit) {
         localStore->clearPathInfoCache();
 
-        bool done = getQueuedBuilds(*conn, destStore, lastBuildId);
+        bool done = getQueuedBuilds(conn, destStore, lastBuildId);
 
         if (buildOne && buildOneDone) quit = true;
 
         /* Sleep until we get notification from the database about an
            event. */
         if (done && !quit) {
-            conn->await_notification();
+            conn.await_notification();
             nrQueueWakeups++;
         } else
-            conn->get_notifs();
+            conn.get_notifs();
 
         if (auto lowestId = buildsAdded.get()) {
             lastBuildId = std::min(lastBuildId, static_cast<unsigned>(std::stoul(*lowestId) - 1));
@@ -61,11 +65,11 @@ void State::queueMonitorLoop()
         }
         if (buildsCancelled.get() || buildsDeleted.get() || buildsBumped.get()) {
             printMsg(lvlTalkative, "got notification: builds cancelled or bumped");
-            processQueueChange(*conn);
+            processQueueChange(conn);
         }
         if (jobsetSharesChanged.get()) {
             printMsg(lvlTalkative, "got notification: jobset shares changed");
-            processJobsetSharesChange(*conn);
+            processJobsetSharesChange(conn);
         }
     }
 
@@ -192,15 +196,19 @@ bool State::getQueuedBuilds(Connection & conn,
                 if (!res[0].is_null()) propagatedFrom = res[0].as<BuildID>();
 
                 if (!propagatedFrom) {
-                    for (auto & i : ex.step->drv->outputsAndOptPaths(*localStore)) {
+                    for (auto & [outputName, optOutputPath] : destStore->queryPartialDerivationOutputMap(ex.step->drvPath, &*localStore)) {
-                        if (i.second.second) {
+                        constexpr std::string_view common = "select max(s.build) from BuildSteps s join BuildStepOutputs o on s.build = o.build where startTime != 0 and stopTime != 0 and status = 1";
-                            auto res = txn.exec_params
+                        auto res = optOutputPath
-                                ("select max(s.build) from BuildSteps s join BuildStepOutputs o on s.build = o.build where path = $1 and startTime != 0 and stopTime != 0 and status = 1",
+                            ? txn.exec_params(
-                                 localStore->printStorePath(*i.second.second));
+                                std::string { common } + " and path = $1",
-                            if (!res[0][0].is_null()) {
+                                localStore->printStorePath(*optOutputPath))
-                                propagatedFrom = res[0][0].as<BuildID>();
+                            : txn.exec_params(
-                                break;
+                                std::string { common } + " and drvPath = $1 and name = $2",
-                            }
+                                localStore->printStorePath(ex.step->drvPath),
+                                outputName);
+                        if (!res[0][0].is_null()) {
+                            propagatedFrom = res[0][0].as<BuildID>();
+                            break;
                         }
                     }
                 }
@@ -236,12 +244,10 @@ bool State::getQueuedBuilds(Connection & conn,
         /* If we didn't get a step, it means the step's outputs are
            all valid. So we mark this as a finished, cached build. */
         if (!step) {
-            auto drv = localStore->readDerivation(build->drvPath);
+            BuildOutput res = getBuildOutputCached(conn, destStore, build->drvPath);
-            BuildOutput res = getBuildOutputCached(conn, destStore, drv);
 
-            for (auto & i : drv.outputsAndOptPaths(*localStore))
+            for (auto & i : destStore->queryDerivationOutputMap(build->drvPath, &*localStore))
-                if (i.second.second)
+                addRoot(i.second);
-                    addRoot(*i.second.second);
 
             {
             auto mc = startDbUpdate();
@@ -292,7 +298,7 @@ bool State::getQueuedBuilds(Connection & conn,
         try {
             createBuild(build);
         } catch (Error & e) {
-            e.addTrace({}, hintfmt("while loading build %d: ", build->id));
+            e.addTrace({}, HintFmt("while loading build %d: ", build->id));
             throw;
         }
 
@@ -464,10 +470,7 @@ Step::ptr State::createStep(ref<Store> destStore,
 
     step->systemType = step->drv->platform;
     {
-        auto i = step->drv->env.find("requiredSystemFeatures");
+        StringSet features = step->requiredSystemFeatures = step->parsedDrv->getRequiredSystemFeatures();
-        StringSet features;
-        if (i != step->drv->env.end())
-            features = step->requiredSystemFeatures = tokenizeString<std::set<std::string>>(i->second);
         if (step->preferLocalBuild)
             features.insert("local");
         if (!features.empty()) {
@@ -481,26 +484,41 @@ Step::ptr State::createStep(ref<Store> destStore,
         throw PreviousFailure{step};
 
     /* Are all outputs valid? */
+    auto outputHashes = staticOutputHashes(*localStore, *(step->drv));
     bool valid = true;
-    DerivationOutputs missing;
+    std::map<DrvOutput, std::optional<StorePath>> missing;
-    for (auto & i : step->drv->outputs)
+    for (auto & [outputName, maybeOutputPath] : destStore->queryPartialDerivationOutputMap(drvPath, &*localStore)) {
-        if (!destStore->isValidPath(*i.second.path(*localStore, step->drv->name, i.first))) {
+        auto outputHash = outputHashes.at(outputName);
-            valid = false;
+        if (maybeOutputPath && destStore->isValidPath(*maybeOutputPath))
-            missing.insert_or_assign(i.first, i.second);
+            continue;
-        }
+        valid = false;
+        missing.insert({{outputHash, outputName}, maybeOutputPath});
+    }
 
     /* Try to copy the missing paths from the local store or from
        substitutes. */
     if (!missing.empty()) {
 
         size_t avail = 0;
-        for (auto & i : missing) {
+        for (auto & [i, pathOpt] : missing) {
-            auto path = i.second.path(*localStore, step->drv->name, i.first);
+            // If we don't know the output path from the destination
-            if (/* localStore != destStore && */ localStore->isValidPath(*path))
+            // store, see if the local store can tell us.
+            if (/* localStore != destStore && */ !pathOpt && experimentalFeatureSettings.isEnabled(Xp::CaDerivations))
+                if (auto maybeRealisation = localStore->queryRealisation(i))
+                    pathOpt = maybeRealisation->outPath;
+
+            if (!pathOpt) {
+                // No hope of getting the store object if we don't know
+                // the path.
+                continue;
+            }
+            auto & path = *pathOpt;
+
+            if (/* localStore != destStore && */ localStore->isValidPath(path))
                 avail++;
             else if (useSubstitutes) {
                 SubstitutablePathInfos infos;
-                localStore->querySubstitutablePathInfos({{*path, {}}}, infos);
+                localStore->querySubstitutablePathInfos({{path, {}}}, infos);
                 if (infos.size() == 1)
                     avail++;
             }
@@ -508,26 +526,29 @@ Step::ptr State::createStep(ref<Store> destStore,
 
         if (missing.size() == avail) {
             valid = true;
-            for (auto & i : missing) {
+            for (auto & [i, pathOpt] : missing) {
-                auto path = i.second.path(*localStore, step->drv->name, i.first);
+                // If we found everything, then we should know the path
+                // to every missing store object now.
+                assert(pathOpt);
+                auto & path = *pathOpt;
 
                 try {
                     time_t startTime = time(0);
 
-                    if (localStore->isValidPath(*path))
+                    if (localStore->isValidPath(path))
                         printInfo("copying output ‘%1%’ of ‘%2%’ from local store",
-                            localStore->printStorePath(*path),
+                            localStore->printStorePath(path),
                             localStore->printStorePath(drvPath));
                     else {
                         printInfo("substituting output ‘%1%’ of ‘%2%’",
-                            localStore->printStorePath(*path),
+                            localStore->printStorePath(path),
                             localStore->printStorePath(drvPath));
-                        localStore->ensurePath(*path);
+                        localStore->ensurePath(path);
                         // FIXME: should copy directly from substituter to destStore.
                     }
 
                     copyClosure(*localStore, *destStore,
-                        StorePathSet { *path },
+                        StorePathSet { path },
                         NoRepair, CheckSigs, NoSubstitute);
 
                     time_t stopTime = time(0);
@@ -535,13 +556,13 @@ Step::ptr State::createStep(ref<Store> destStore,
                     {
                         auto mc = startDbUpdate();
                         pqxx::work txn(conn);
-                        createSubstitutionStep(txn, startTime, stopTime, build, drvPath, "out", *path);
+                        createSubstitutionStep(txn, startTime, stopTime, build, drvPath, *(step->drv), "out", path);
                         txn.commit();
                     }
 
                 } catch (Error & e) {
                     printError("while copying/substituting output ‘%s’ of ‘%s’: %s",
-                        localStore->printStorePath(*path),
+                        localStore->printStorePath(path),
                         localStore->printStorePath(drvPath),
                         e.what());
                     valid = false;
@@ -640,17 +661,19 @@ void State::processJobsetSharesChange(Connection & conn)
 }
 
 
-BuildOutput State::getBuildOutputCached(Connection & conn, nix::ref<nix::Store> destStore, const nix::Derivation & drv)
+BuildOutput State::getBuildOutputCached(Connection & conn, nix::ref<nix::Store> destStore, const nix::StorePath & drvPath)
 {
+    auto derivationOutputs = destStore->queryDerivationOutputMap(drvPath, &*localStore);
+
     {
     pqxx::work txn(conn);
 
-    for (auto & [name, output] : drv.outputsAndOptPaths(*localStore)) {
+    for (auto & [name, output] : derivationOutputs) {
         auto r = txn.exec_params
             ("select id, buildStatus, releaseName, closureSize, size from Builds b "
              "join BuildOutputs o on b.id = o.build "
              "where finished = 1 and (buildStatus = 0 or buildStatus = 6) and path = $1",
-             localStore->printStorePath(*output.second));
+             localStore->printStorePath(output));
         if (r.empty()) continue;
         BuildID id = r[0][0].as<BuildID>();
 
@@ -677,7 +700,7 @@ BuildOutput State::getBuildOutputCached(Connection & conn, nix::ref<nix::Store>
                 product.fileSize = row[2].as<off_t>();
             }
             if (!row[3].is_null())
-                product.sha256hash = Hash::parseAny(row[3].as<std::string>(), htSHA256);
+                product.sha256hash = Hash::parseAny(row[3].as<std::string>(), HashAlgorithm::SHA256);
             if (!row[4].is_null())
                 product.path = row[4].as<std::string>();
             product.name = row[5].as<std::string>();
@@ -704,5 +727,5 @@ BuildOutput State::getBuildOutputCached(Connection & conn, nix::ref<nix::Store>
     }
 
     NarMemberDatas narMembers;
-    return getBuildOutput(destStore, narMembers, drv);
+    return getBuildOutput(destStore, narMembers, derivationOutputs);
 }

src/hydra-queue-runner/state.hh

@@ -6,7 +6,6 @@
 #include <map>
 #include <memory>
 #include <queue>
-#include <regex>
 
 #include <prometheus/counter.h>
 #include <prometheus/gauge.h>
@@ -21,6 +20,8 @@
 #include "store-api.hh"
 #include "sync.hh"
 #include "nar-extractor.hh"
+#include "serve-protocol.hh"
+#include "machines.hh"
 
 
 typedef unsigned int BuildID;
@@ -78,6 +79,8 @@ struct RemoteResult
     {
         return stepStatus == bsCachedFailure ? bsFailed : stepStatus;
     }
+
+    void updateWithBuildResult(const nix::BuildResult &);
 };
 
 
@@ -231,18 +234,10 @@ void getDependents(Step::ptr step, std::set<Build::ptr> & builds, std::set<Step:
 void visitDependencies(std::function<void(Step::ptr)> visitor, Step::ptr step);
 
 
-struct Machine
+struct Machine : nix::Machine
 {
     typedef std::shared_ptr<Machine> ptr;
 
-    bool enabled{true};
-
-    std::string sshName, sshKey;
-    std::set<std::string> systemTypes, supportedFeatures, mandatoryFeatures;
-    unsigned int maxJobs = 1;
-    float speedFactor = 1.0;
-    std::string sshPublicHostKey;
-
     struct State {
         typedef std::shared_ptr<State> ptr;
         counter currentJobs{0};
@@ -292,11 +287,7 @@ struct Machine
         return true;
     }
 
-    bool isLocalhost()
+    bool isLocalhost() const;
-    {
-        std::regex r("^(ssh://|ssh-ng://)?localhost$");
-        return std::regex_search(sshName, r);
-    }
 };
 
 
@@ -350,7 +341,7 @@ private:
 
     /* The build machines. */
     std::mutex machinesReadyLock;
-    typedef std::map<std::string, Machine::ptr> Machines;
+    typedef std::map<nix::StoreReference::Variant, Machine::ptr> Machines;
     nix::Sync<Machines> machines; // FIXME: use atomic_shared_ptr
 
     /* Various stats. */
@@ -361,9 +352,10 @@ private:
     counter nrStepsStarted{0};
     counter nrStepsDone{0};
     counter nrStepsBuilding{0};
+    #if 0
     counter nrStepsCopyingTo{0};
     counter nrStepsCopyingFrom{0};
-    counter nrStepsWaiting{0};
+    #endif
     counter nrUnsupportedSteps{0};
     counter nrRetries{0};
     counter maxNrRetries{0};
@@ -372,8 +364,6 @@ private:
     counter nrQueueWakeups{0};
     counter nrDispatcherWakeups{0};
     counter dispatchTimeMs{0};
-    counter bytesSent{0};
-    counter bytesReceived{0};
     counter nrActiveDbUpdates{0};
 
     /* Specific build to do for --build-one (testing only). */
@@ -430,7 +420,7 @@ private:
 
     /* How often the build steps of a jobset should be repeated in
        order to detect non-determinism. */
-    std::map<std::pair<std::string, std::string>, unsigned int> jobsetRepeats;
+    std::map<std::pair<std::string, std::string>, size_t> jobsetRepeats;
 
     bool uploadLogsToBinaryCache;
 
@@ -485,13 +475,13 @@ private:
         const std::string & machine);
 
     int createSubstitutionStep(pqxx::work & txn, time_t startTime, time_t stopTime,
-        Build::ptr build, const nix::StorePath & drvPath, const std::string & outputName, const nix::StorePath & storePath);
+        Build::ptr build, const nix::StorePath & drvPath, const nix::Derivation drv, const std::string & outputName, const nix::StorePath & storePath);
 
     void updateBuild(pqxx::work & txn, Build::ptr build, BuildStatus status);
 
     void queueMonitor();
 
-    void queueMonitorLoop();
+    void queueMonitorLoop(Connection & conn);
 
     /* Check the queue for new builds. */
     bool getQueuedBuilds(Connection & conn,
@@ -501,7 +491,7 @@ private:
     void processQueueChange(Connection & conn);
 
     BuildOutput getBuildOutputCached(Connection & conn, nix::ref<nix::Store> destStore,
-        const nix::Derivation & drv);
+        const nix::StorePath & drvPath);
 
     Step::ptr createStep(nix::ref<nix::Store> store,
         Connection & conn, Build::ptr build, const nix::StorePath & drvPath,
@@ -543,8 +533,7 @@ private:
 
     void buildRemote(nix::ref<nix::Store> destStore,
         Machine::ptr machine, Step::ptr step,
-        unsigned int maxSilentTime, unsigned int buildTimeout,
+        const nix::ServeProto::BuildOptions & buildOptions,
-        unsigned int repeats,
         RemoteResult & result, std::shared_ptr<ActiveStep> activeStep,
         std::function<void(StepState)> updateStep,
         NarMemberDatas & narMembers);

src/lib/Hydra/Base/Controller/NixChannel.pm

@@ -4,7 +4,6 @@ use strict;
 use warnings;
 use base 'Hydra::Base::Controller::REST';
 use List::SomeUtils qw(any);
-use Nix::Store;
 use Hydra::Helper::Nix;
 use Hydra::Helper::CatalystUtils;
 
@@ -30,7 +29,7 @@ sub getChannelData {
         my $outputs = {};
         foreach my $output (@outputs) {
             my $outPath = $output->get_column("outpath");
-            next if $checkValidity && !isValidPath($outPath);
+            next if $checkValidity && !$MACHINE_LOCAL_STORE->isValidPath($outPath);
             $outputs->{$output->get_column("outname")} = $outPath;
             push @storePaths, $outPath;
             # Put the system type in the manifest (for top-level

src/lib/Hydra/Config.pm

@@ -95,6 +95,7 @@ sub get_legacy_ldap_config {
             "hydra_bump-to-front" => [ "bump-to-front" ],
             "hydra_cancel-build" => [ "cancel-build" ],
             "hydra_create-projects" => [ "create-projects" ],
+            "hydra_eval-jobset" => [ "eval-jobset" ],
             "hydra_restart-jobs" => [ "restart-jobs" ],
         },
     };
@@ -159,6 +160,7 @@ sub valid_roles {
         "bump-to-front",
         "cancel-build",
         "create-projects",
+        "eval-jobset",
         "restart-jobs",
     ];
 }

src/lib/Hydra/Controller/API.pm

@@ -239,6 +239,8 @@ sub triggerJobset {
 sub push : Chained('api') PathPart('push') Args(0) {
     my ($self, $c) = @_;
 
+    requirePost($c);
+
     $c->{stash}->{json}->{jobsetsTriggered} = [];
 
     my $force = exists $c->request->query_params->{force};
@@ -246,19 +248,24 @@ sub push : Chained('api') PathPart('push') Args(0) {
     foreach my $s (@jobsets) {
         my ($p, $j) = parseJobsetName($s);
         my $jobset = $c->model('DB::Jobsets')->find($p, $j);
+        requireEvalJobsetPrivileges($c, $jobset->project);
         next unless defined $jobset && ($force || ($jobset->project->enabled && $jobset->enabled));
         triggerJobset($self, $c, $jobset, $force);
     }
 
     my @repos = split /,/, ($c->request->query_params->{repos} // "");
     foreach my $r (@repos) {
-        triggerJobset($self, $c, $_, $force) foreach $c->model('DB::Jobsets')->search(
+        my @jobsets = $c->model('DB::Jobsets')->search(
             { 'project.enabled' => 1, 'me.enabled' => 1 },
             {
                 join => 'project',
                 where => \ [ 'exists (select 1 from JobsetInputAlts where project = me.project and jobset = me.name and value = ?)', [ 'value', $r ] ],
                 order_by => 'me.id DESC'
             });
+        foreach my $jobset (@jobsets) {
+            requireEvalJobsetPrivileges($c, $jobset->project);
+            triggerJobset($self, $c, $jobset, $force)
+        }
     }
 
     $self->status_ok(
@@ -285,6 +292,23 @@ sub push_github : Chained('api') PathPart('push-github') Args(0) {
     $c->response->body("");
 }
 
+sub push_gitea : Chained('api') PathPart('push-gitea') Args(0) {
+    my ($self, $c) = @_;
+
+    $c->{stash}->{json}->{jobsetsTriggered} = [];
+
+    my $in = $c->request->{data};
+    my $url = $in->{repository}->{clone_url} or die;
+    $url =~ s/.git$//;
+    print STDERR "got push from Gitea repository $url\n";
+
+    triggerJobset($self, $c, $_, 0) foreach $c->model('DB::Jobsets')->search(
+        { 'project.enabled' => 1, 'me.enabled' => 1 },
+        { join => 'project'
+        , where => \ [ 'me.flake like ? or exists (select 1 from JobsetInputAlts where project = me.project and jobset = me.name and value like ?)', [ 'flake', "%$url%"], [ 'value', "%$url%" ] ]
+        });
+    $c->response->body("");
+}
 
 
 1;

src/lib/Hydra/Controller/Build.pm

@@ -10,11 +10,10 @@ use File::Basename;
 use File::LibMagic;
 use File::stat;
 use Data::Dump qw(dump);
-use Nix::Store;
-use Nix::Config;
 use List::SomeUtils qw(all);
 use Encode;
 use JSON::PP;
+use WWW::Form::UrlEncoded::PP qw();
 
 use feature 'state';
 
@@ -78,14 +77,16 @@ sub build_GET {
 
     $c->stash->{template} = 'build.tt';
     $c->stash->{isLocalStore} = isLocalStore();
+    # XXX: If the derivation is content-addressed then this will always return
+    # false because `$_->path` will be empty
     $c->stash->{available} =
         $c->stash->{isLocalStore}
-        ? all { isValidPath($_->path) } $build->buildoutputs->all
+        ? all { $_->path && $MACHINE_LOCAL_STORE->isValidPath($_->path) } $build->buildoutputs->all
         : 1;
-    $c->stash->{drvAvailable} = isValidPath $build->drvpath;
+    $c->stash->{drvAvailable} = $MACHINE_LOCAL_STORE->isValidPath($build->drvpath);
 
     if ($build->finished && $build->iscachedbuild) {
-        my $path = ($build->buildoutputs)[0]->path or die;
+        my $path = ($build->buildoutputs)[0]->path or undef;
         my $cachedBuildStep = findBuildStepByOutPath($self, $c, $path);
         if (defined $cachedBuildStep) {
             $c->stash->{cachedBuild} = $cachedBuildStep->build;
@@ -139,7 +140,7 @@ sub view_nixlog : Chained('buildChain') PathPart('nixlog') {
     $c->stash->{step} = $step;
 
     my $drvPath = $step->drvpath;
-    my $log_uri = $c->uri_for($c->controller('Root')->action_for("log"), [basename($drvPath)]);
+    my $log_uri = $c->uri_for($c->controller('Root')->action_for("log"), [WWW::Form::UrlEncoded::PP::url_encode(basename($drvPath))]);
     showLog($c, $mode, $log_uri);
 }
 
@@ -148,7 +149,7 @@ sub view_log : Chained('buildChain') PathPart('log') {
     my ($self, $c, $mode) = @_;
 
     my $drvPath = $c->stash->{build}->drvpath;
-    my $log_uri = $c->uri_for($c->controller('Root')->action_for("log"), [basename($drvPath)]);
+    my $log_uri = $c->uri_for($c->controller('Root')->action_for("log"), [WWW::Form::UrlEncoded::PP::url_encode(basename($drvPath))]);
     showLog($c, $mode, $log_uri);
 }
 
@@ -233,6 +234,9 @@ sub serveFile {
     }
 
     elsif ($ls->{type} eq "regular") {
+        # Have the hosted data considered its own origin to avoid being a giant
+        # XSS hole.
+        $c->response->header('Content-Security-Policy' => 'sandbox allow-scripts');
 
         $c->stash->{'plain'} = { data => grab(cmd => ["nix", "--experimental-features", "nix-command",
                                                       "store", "cat", "--store", getStoreUri(), "$path"]) };
@@ -306,7 +310,7 @@ sub output : Chained('buildChain') PathPart Args(1) {
     error($c, "This build is not finished yet.") unless $build->finished;
     my $output = $build->buildoutputs->find({name => $outputName});
     notFound($c, "This build has no output named ‘$outputName’") unless defined $output;
-    gone($c, "Output is no longer available.") unless isValidPath $output->path;
+    gone($c, "Output is no longer available.") unless $MACHINE_LOCAL_STORE->isValidPath($output->path);
 
     $c->response->header('Content-Disposition', "attachment; filename=\"build-${\$build->id}-${\$outputName}.nar.bz2\"");
     $c->stash->{current_view} = 'NixNAR';
@@ -423,7 +427,7 @@ sub getDependencyGraph {
             };
         $$done{$path} = $node;
         my @refs;
-        foreach my $ref (queryReferences($path)) {
+        foreach my $ref ($MACHINE_LOCAL_STORE->queryReferences($path)) {
             next if $ref eq $path;
             next unless $runtime || $ref =~ /\.drv$/;
             getDependencyGraph($self, $c, $runtime, $done, $ref);
@@ -431,7 +435,7 @@ sub getDependencyGraph {
         }
         # Show in reverse topological order to flatten the graph.
         # Should probably do a proper BFS.
-        my @sorted = reverse topoSortPaths(@refs);
+        my @sorted = reverse $MACHINE_LOCAL_STORE->topoSortPaths(@refs);
         $node->{refs} = [map { $$done{$_} } @sorted];
     }
 
@@ -444,7 +448,7 @@ sub build_deps : Chained('buildChain') PathPart('build-deps') {
     my $build = $c->stash->{build};
     my $drvPath = $build->drvpath;
 
-    error($c, "Derivation no longer available.") unless isValidPath $drvPath;
+    error($c, "Derivation no longer available.") unless $MACHINE_LOCAL_STORE->isValidPath($drvPath);
 
     $c->stash->{buildTimeGraph} = getDependencyGraph($self, $c, 0, {}, $drvPath);
 
@@ -459,7 +463,7 @@ sub runtime_deps : Chained('buildChain') PathPart('runtime-deps') {
 
     requireLocalStore($c);
 
-    error($c, "Build outputs no longer available.") unless all { isValidPath($_) } @outPaths;
+    error($c, "Build outputs no longer available.") unless all { $MACHINE_LOCAL_STORE->isValidPath($_) } @outPaths;
 
     my $done = {};
     $c->stash->{runtimeGraph} = [ map { getDependencyGraph($self, $c, 1, $done, $_) } @outPaths ];
@@ -479,7 +483,7 @@ sub nix : Chained('buildChain') PathPart('nix') CaptureArgs(0) {
     if (isLocalStore) {
         foreach my $out ($build->buildoutputs) {
             notFound($c, "Path " . $out->path . " is no longer available.")
-                unless isValidPath($out->path);
+                unless $MACHINE_LOCAL_STORE->isValidPath($out->path);
         }
     }
 

src/lib/Hydra/Controller/Root.pm

@@ -16,8 +16,11 @@ use List::Util qw[min max];
 use List::SomeUtils qw{any};
 use Net::Prometheus;
 use Types::Standard qw/StrMatch/;
+use WWW::Form::UrlEncoded::PP qw();
 
 use constant NARINFO_REGEX => qr{^([a-z0-9]{32})\.narinfo$};
+# e.g.: https://hydra.example.com/realisations/sha256:a62128132508a3a32eef651d6467695944763602f226ac630543e947d9feb140!out.doi
+use constant REALISATIONS_REGEX => qr{^(sha256:[a-z0-9]{64}![a-z]+)\.doi$};
 
 # Put this controller at top-level.
 __PACKAGE__->config->{namespace} = '';
@@ -32,6 +35,7 @@ sub noLoginNeeded {
 
   return $whitelisted ||
          $c->request->path eq "api/push-github" ||
+         $c->request->path eq "api/push-gitea" ||
          $c->request->path eq "google-login" ||
          $c->request->path eq "github-redirect" ||
          $c->request->path eq "github-login" ||
@@ -47,6 +51,7 @@ sub begin :Private {
     $c->stash->{curUri} = $c->request->uri;
     $c->stash->{version} = $ENV{"HYDRA_RELEASE"} || "<devel>";
     $c->stash->{nixVersion} = $ENV{"NIX_RELEASE"} || "<devel>";
+    $c->stash->{nixEvalJobsVersion} = $ENV{"NIX_EVAL_JOBS_RELEASE"} || "<devel>";
     $c->stash->{curTime} = time;
     $c->stash->{logo} = defined $c->config->{hydra_logo} ? "/logo" : "";
     $c->stash->{tracker} = defined $c->config->{tracker} ? $c->config->{tracker} : "";
@@ -77,7 +82,7 @@ sub begin :Private {
     $_->supportedInputTypes($c->stash->{inputTypes}) foreach @{$c->hydra_plugins};
 
     # XSRF protection: require POST requests to have the same origin.
-    if ($c->req->method eq "POST" && $c->req->path ne "api/push-github") {
+    if ($c->req->method eq "POST" && $c->req->path ne "api/push-github" && $c->req->path ne "api/push-gitea") {
         my $referer = $c->req->header('Referer');
         $referer //= $c->req->header('Origin');
         my $base = $c->req->base;
@@ -326,7 +331,7 @@ sub nar :Local :Args(1) {
     else {
         $path = $Nix::Config::storeDir . "/$path";
 
-        gone($c, "Path " . $path . " is no longer available.") unless isValidPath($path);
+        gone($c, "Path " . $path . " is no longer available.") unless $MACHINE_LOCAL_STORE->isValidPath($path);
 
         $c->stash->{current_view} = 'NixNAR';
         $c->stash->{storePath} = $path;
@@ -355,6 +360,33 @@ sub nix_cache_info :Path('nix-cache-info') :Args(0) {
 }
 
 
+sub realisations :Path('realisations') :Args(StrMatch[REALISATIONS_REGEX]) {
+    my ($self, $c, $realisation) = @_;
+
+    if (!isLocalStore) {
+        notFound($c, "There is no binary cache here.");
+    }
+
+    else {
+        my ($rawDrvOutput) = $realisation =~ REALISATIONS_REGEX;
+        my $rawRealisation = $MACHINE_LOCAL_STORE->queryRawRealisation($rawDrvOutput);
+
+        if (!$rawRealisation) {
+            $c->response->status(404);
+            $c->response->content_type('text/plain');
+            $c->stash->{plain}->{data} = "does not exist\n";
+            $c->forward('Hydra::View::Plain');
+            setCacheHeaders($c, 60 * 60);
+            return;
+        }
+
+        $c->response->content_type('text/plain');
+        $c->stash->{plain}->{data} = $rawRealisation;
+        $c->forward('Hydra::View::Plain');
+    }
+}
+
+
 sub narinfo :Path :Args(StrMatch[NARINFO_REGEX]) {
     my ($self, $c, $narinfo) = @_;
 
@@ -366,7 +398,7 @@ sub narinfo :Path :Args(StrMatch[NARINFO_REGEX]) {
         my ($hash) = $narinfo =~ NARINFO_REGEX;
 
         die("Hash length was not 32") if length($hash) != 32;
-        my $path = queryPathFromHashPart($hash);
+        my $path = $MACHINE_LOCAL_STORE->queryPathFromHashPart($hash);
 
         if (!$path) {
             $c->response->status(404);
@@ -524,7 +556,7 @@ sub log :Local :Args(1) {
     my $logPrefix = $c->config->{log_prefix};
 
     if (defined $logPrefix) {
-        $c->res->redirect($logPrefix . "log/" . basename($drvPath));
+        $c->res->redirect($logPrefix . "log/" . WWW::Form::UrlEncoded::PP::url_encode(basename($drvPath)));
     } else {
         notFound($c, "The build log of $drvPath is not available.");
     }

src/lib/Hydra/Helper/CatalystUtils.pm

@@ -15,6 +15,7 @@ our @EXPORT = qw(
     forceLogin requireUser requireProjectOwner requireRestartPrivileges requireAdmin requirePost isAdmin isProjectOwner
     requireBumpPrivileges
     requireCancelBuildPrivileges
+    requireEvalJobsetPrivileges
     trim
     getLatestFinishedEval getFirstEval
     paramToList
@@ -186,6 +187,27 @@ sub isProjectOwner {
          defined $c->model('DB::ProjectMembers')->find({ project => $project, userName => $c->user->username }));
 }
 
+sub hasEvalJobsetRole {
+    my ($c) = @_;
+    return $c->user_exists && $c->check_user_roles("eval-jobset");
+}
+
+sub mayEvalJobset {
+    my ($c, $project) = @_;
+    return
+        $c->user_exists &&
+        (isAdmin($c) ||
+         hasEvalJobsetRole($c) ||
+         isProjectOwner($c, $project));
+}
+
+sub requireEvalJobsetPrivileges {
+    my ($c, $project) = @_;
+    requireUser($c);
+    accessDenied($c, "Only the project members, administrators, and accounts with eval-jobset privileges can perform this operation.")
+        unless mayEvalJobset($c, $project);
+}
+
 sub hasCancelBuildRole {
     my ($c) = @_;
     return $c->user_exists && $c->check_user_roles('cancel-build');
@@ -272,7 +294,7 @@ sub requireAdmin {
 
 sub requirePost {
     my ($c) = @_;
-    error($c, "Request must be POSTed.") if $c->request->method ne "POST";
+    error($c, "Request must be POSTed.", 405) if $c->request->method ne "POST";
 }
 
 

src/lib/Hydra/Helper/Nix.pm

@@ -40,8 +40,11 @@ our @EXPORT = qw(
     registerRoot
     restartBuilds
     run
+    $MACHINE_LOCAL_STORE
     );
 
+our $MACHINE_LOCAL_STORE = Nix::Store->new();
+
 
 sub getHydraHome {
     my $dir = $ENV{"HYDRA_HOME"} or die "The HYDRA_HOME directory does not exist!\n";
@@ -171,6 +174,9 @@ sub getDrvLogPath {
     for ($fn . $bucketed, $fn . $bucketed . ".bz2") {
         return $_ if -f $_;
     }
+    for ($fn . $bucketed, $fn . $bucketed . ".zst") {
+        return $_ if -f $_;
+    }
     return undef;
 }
 
@@ -187,6 +193,10 @@ sub findLog {
 
     return undef if scalar @outPaths == 0;
 
+    # Filter out any NULLs. Content-addressed derivations
+    # that haven't built yet or failed to build may have a NULL outPath.
+    @outPaths = grep {defined} @outPaths;
+
     my @steps = $c->model('DB::BuildSteps')->search(
         { path => { -in => [@outPaths] } },
         { select => ["drvpath"]
@@ -494,7 +504,7 @@ sub restartBuilds {
     $builds = $builds->search({ finished => 1 });
 
     foreach my $build ($builds->search({}, { columns => ["drvpath"] })) {
-        next if !isValidPath($build->drvpath);
+        next if !$MACHINE_LOCAL_STORE->isValidPath($build->drvpath);
         registerRoot $build->drvpath;
     }
 

src/lib/Hydra/Plugin/BazaarInput.pm

@@ -7,7 +7,6 @@ use Digest::SHA qw(sha256_hex);
 use File::Path;
 use Hydra::Helper::Exec;
 use Hydra::Helper::Nix;
-use Nix::Store;
 
 sub supportedInputTypes {
     my ($self, $inputTypes) = @_;
@@ -38,9 +37,9 @@ sub fetchInput {
     (my $cachedInput) = $self->{db}->resultset('CachedBazaarInputs')->search(
         {uri => $uri, revision => $revision});
 
-    addTempRoot($cachedInput->storepath) if defined $cachedInput;
+    $MACHINE_LOCAL_STORE->addTempRoot($cachedInput->storepath) if defined $cachedInput;
 
-    if (defined $cachedInput && isValidPath($cachedInput->storepath)) {
+    if (defined $cachedInput && $MACHINE_LOCAL_STORE->isValidPath($cachedInput->storepath)) {
         $storePath = $cachedInput->storepath;
         $sha256 = $cachedInput->sha256hash;
     } else {
@@ -58,7 +57,7 @@ sub fetchInput {
         ($sha256, $storePath) = split ' ', $stdout;
 
         # FIXME: time window between nix-prefetch-bzr and addTempRoot.
-        addTempRoot($storePath);
+        $MACHINE_LOCAL_STORE->addTempRoot($storePath);
 
         $self->{db}->txn_do(sub {
             $self->{db}->resultset('CachedBazaarInputs')->create(

src/lib/Hydra/Plugin/CompressLog.pm

@@ -9,11 +9,24 @@ use Hydra::Helper::CatalystUtils;
 sub stepFinished {
     my ($self, $step, $logPath) = @_;
 
-    my $doCompress = $self->{config}->{'compress_build_logs'} // "1";
+    my $doCompress = $self->{config}->{'compress_build_logs'} // '1';
+    my $silent = $self->{config}->{'compress_build_logs_silent'} // '0';
+    my $compression = $self->{config}->{'compress_build_logs_compression'} // 'bzip2';
 
-    if ($doCompress eq "1" && -e $logPath) {
+    if (not -e $logPath or $doCompress ne "1") {
-        print STDERR "compressing ‘$logPath’...\n";
+        return;
-        system("bzip2", "--force", $logPath);
+    }
+
+    if ($silent ne '1') {
+        print STDERR "compressing '$logPath' with $compression...\n";
+    }
+
+    if ($compression eq 'bzip2') {
+        system('bzip2', '--force', $logPath);
+    } elsif ($compression eq 'zstd') {
+        system('zstd', '--rm', '--quiet', '-T0', $logPath);
+    } else {
+        print STDERR "unknown compression type '$compression'\n";
     }
 }
 

src/lib/Hydra/Plugin/DarcsInput.pm

@@ -7,7 +7,6 @@ use Digest::SHA qw(sha256_hex);
 use File::Path;
 use Hydra::Helper::Exec;
 use Hydra::Helper::Nix;
-use Nix::Store;
 
 sub supportedInputTypes {
     my ($self, $inputTypes) = @_;
@@ -58,7 +57,7 @@ sub fetchInput {
         {uri => $uri, revision => $revision},
         {rows => 1});
 
-    if (defined $cachedInput && isValidPath($cachedInput->storepath)) {
+    if (defined $cachedInput && $MACHINE_LOCAL_STORE->isValidPath($cachedInput->storepath)) {
         $storePath = $cachedInput->storepath;
         $sha256 = $cachedInput->sha256hash;
         $revision = $cachedInput->revision;
@@ -75,8 +74,8 @@ sub fetchInput {
         die "darcs changes --count failed" if $? != 0;
 
         system "rm", "-rf", "$tmpDir/export/_darcs";
-        $storePath = addToStore("$tmpDir/export", 1, "sha256");
+        $storePath = $MACHINE_LOCAL_STORE->addToStore("$tmpDir/export", 1, "sha256");
-        $sha256 = queryPathHash($storePath);
+        $sha256 = $MACHINE_LOCAL_STORE->queryPathHash($storePath);
         $sha256 =~ s/sha256://;
 
         $self->{db}->txn_do(sub {

src/lib/Hydra/Plugin/GitInput.pm

@@ -186,9 +186,9 @@ sub fetchInput {
         {uri => $uri, branch => $branch, revision => $revision, isdeepclone => defined($deepClone) ? 1 : 0},
         {rows => 1});
 
-    addTempRoot($cachedInput->storepath) if defined $cachedInput;
+    $MACHINE_LOCAL_STORE->addTempRoot($cachedInput->storepath) if defined $cachedInput;
 
-    if (defined $cachedInput && isValidPath($cachedInput->storepath)) {
+    if (defined $cachedInput && $MACHINE_LOCAL_STORE->isValidPath($cachedInput->storepath)) {
         $storePath = $cachedInput->storepath;
         $sha256 = $cachedInput->sha256hash;
         $revision = $cachedInput->revision;
@@ -217,7 +217,7 @@ sub fetchInput {
         ($sha256, $storePath) = split ' ', grab(cmd => ["nix-prefetch-git", $clonePath, $revision], chomp => 1);
 
         # FIXME: time window between nix-prefetch-git and addTempRoot.
-        addTempRoot($storePath);
+        $MACHINE_LOCAL_STORE->addTempRoot($storePath);
 
         $self->{db}->txn_do(sub {
             $self->{db}->resultset('CachedGitInputs')->update_or_create(

src/lib/Hydra/Plugin/GiteaStatus.pm

@@ -88,10 +88,6 @@ sub buildQueued {
     common(@_, [], 0);
 }
 
-sub buildStarted {
-    common(@_, [], 1);
-}
-
 sub buildFinished {
     common(@_, 2);
 }

src/lib/Hydra/Plugin/MercurialInput.pm

@@ -7,7 +7,6 @@ use Digest::SHA qw(sha256_hex);
 use File::Path;
 use Hydra::Helper::Nix;
 use Hydra::Helper::Exec;
-use Nix::Store;
 use Fcntl qw(:flock);
 
 sub supportedInputTypes {
@@ -68,9 +67,9 @@ sub fetchInput {
     (my $cachedInput) = $self->{db}->resultset('CachedHgInputs')->search(
         {uri => $uri, branch => $branch, revision => $revision});
 
-    addTempRoot($cachedInput->storepath) if defined $cachedInput;
+    $MACHINE_LOCAL_STORE->addTempRoot($cachedInput->storepath) if defined $cachedInput;
 
-    if (defined $cachedInput && isValidPath($cachedInput->storepath)) {
+    if (defined $cachedInput && $MACHINE_LOCAL_STORE->isValidPath($cachedInput->storepath)) {
         $storePath = $cachedInput->storepath;
         $sha256 = $cachedInput->sha256hash;
     } else {
@@ -85,7 +84,7 @@ sub fetchInput {
         ($sha256, $storePath) = split ' ', $stdout;
 
         # FIXME: time window between nix-prefetch-hg and addTempRoot.
-        addTempRoot($storePath);
+        $MACHINE_LOCAL_STORE->addTempRoot($storePath);
 
         $self->{db}->txn_do(sub {
             $self->{db}->resultset('CachedHgInputs')->update_or_create(

src/lib/Hydra/Plugin/PathInput.pm

@@ -5,7 +5,6 @@ use warnings;
 use parent 'Hydra::Plugin';
 use POSIX qw(strftime);
 use Hydra::Helper::Nix;
-use Nix::Store;
 
 sub supportedInputTypes {
     my ($self, $inputTypes) = @_;
@@ -30,7 +29,7 @@ sub fetchInput {
         {srcpath => $uri, lastseen => {">", $timestamp - $timeout}},
         {rows => 1, order_by => "lastseen DESC"});
 
-    if (defined $cachedInput && isValidPath($cachedInput->storepath)) {
+    if (defined $cachedInput && $MACHINE_LOCAL_STORE->isValidPath($cachedInput->storepath)) {
         $storePath = $cachedInput->storepath;
         $sha256 = $cachedInput->sha256hash;
         $timestamp = $cachedInput->timestamp;
@@ -46,7 +45,7 @@ sub fetchInput {
         }
         chomp $storePath;
 
-        $sha256 = (queryPathInfo($storePath, 0))[1] or die;
+        $sha256 = ($MACHINE_LOCAL_STORE->queryPathInfo($storePath, 0))[1] or die;
 
         ($cachedInput) = $self->{db}->resultset('CachedPathInputs')->search(
             {srcpath => $uri, sha256hash => $sha256});

src/lib/Hydra/Plugin/S3Backup.pm

@@ -14,6 +14,7 @@ use Nix::Config;
 use Nix::Store;
 use Hydra::Model::DB;
 use Hydra::Helper::CatalystUtils;
+use Hydra::Helper::Nix;
 
 sub isEnabled {
     my ($self) = @_;
@@ -92,7 +93,7 @@ sub buildFinished {
         my $hash = substr basename($path), 0, 32;
         my ($deriver, $narHash, $time, $narSize, $refs) = queryPathInfo($path, 0);
         my $system;
-        if (defined $deriver and isValidPath($deriver)) {
+        if (defined $deriver and $MACHINE_LOCAL_STORE->isValidPath($deriver)) {
             $system = derivationFromPath($deriver)->{platform};
         }
         foreach my $reference (@{$refs}) {

src/lib/Hydra/Plugin/SubversionInput.pm

@@ -7,7 +7,6 @@ use Digest::SHA qw(sha256_hex);
 use Hydra::Helper::Exec;
 use Hydra::Helper::Nix;
 use IPC::Run;
-use Nix::Store;
 
 sub supportedInputTypes {
     my ($self, $inputTypes) = @_;
@@ -45,9 +44,9 @@ sub fetchInput {
     (my $cachedInput) = $self->{db}->resultset('CachedSubversionInputs')->search(
         {uri => $uri, revision => $revision});
 
-    addTempRoot($cachedInput->storepath) if defined $cachedInput;
+    $MACHINE_LOCAL_STORE->addTempRoot($cachedInput->storepath) if defined $cachedInput;
 
-    if (defined $cachedInput && isValidPath($cachedInput->storepath)) {
+    if (defined $cachedInput && $MACHINE_LOCAL_STORE->isValidPath($cachedInput->storepath)) {
         $storePath = $cachedInput->storepath;
         $sha256 = $cachedInput->sha256hash;
     } else {
@@ -62,16 +61,16 @@ sub fetchInput {
         die "error checking out Subversion repo at `$uri':\n$stderr" if $res;
 
         if ($type eq "svn-checkout") {
-            $storePath = addToStore($wcPath, 1, "sha256");
+            $storePath = $MACHINE_LOCAL_STORE->addToStore($wcPath, 1, "sha256");
         } else {
             # Hm, if the Nix Perl bindings supported filters in
             # addToStore(), then we wouldn't need to make a copy here.
             my $tmpDir = File::Temp->newdir("hydra-svn-export.XXXXXX", CLEANUP => 1, TMPDIR => 1) or die;
             (system "svn", "export", $wcPath, "$tmpDir/source", "--quiet") == 0 or die "svn export failed";
-            $storePath = addToStore("$tmpDir/source", 1, "sha256");
+            $storePath = $MACHINE_LOCAL_STORE->addToStore("$tmpDir/source", 1, "sha256");
         }
 
-        $sha256 = queryPathHash($storePath); $sha256 =~ s/sha256://;
+        $sha256 = $MACHINE_LOCAL_STORE->queryPathHash($storePath); $sha256 =~ s/sha256://;
 
         $self->{db}->txn_do(sub {
             $self->{db}->resultset('CachedSubversionInputs')->update_or_create(

src/lib/Hydra/Schema/Result/BuildOutputs.pm

@@ -49,7 +49,7 @@ __PACKAGE__->table("buildoutputs");
 =head2 path
 
   data_type: 'text'
-  is_nullable: 0
+  is_nullable: 1
 
 =cut
 
@@ -59,7 +59,7 @@ __PACKAGE__->add_columns(
   "name",
   { data_type => "text", is_nullable => 0 },
   "path",
-  { data_type => "text", is_nullable => 0 },
+  { data_type => "text", is_nullable => 1 },
 );
 
 =head1 PRIMARY KEY
@@ -94,8 +94,8 @@ __PACKAGE__->belongs_to(
 );
 
 
-# Created by DBIx::Class::Schema::Loader v0.07049 @ 2021-08-26 12:02:36
+# Created by DBIx::Class::Schema::Loader v0.07049 @ 2022-06-30 12:02:32
-# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:gU+kZ6A0ISKpaXGRGve8mg
+# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:Jsabm3YTcI7YvCuNdKP5Ng
 
 my %hint = (
     columns => [

src/lib/Hydra/Schema/Result/BuildStepOutputs.pm

@@ -55,7 +55,7 @@ __PACKAGE__->table("buildstepoutputs");
 =head2 path
 
   data_type: 'text'
-  is_nullable: 0
+  is_nullable: 1
 
 =cut
 
@@ -67,7 +67,7 @@ __PACKAGE__->add_columns(
   "name",
   { data_type => "text", is_nullable => 0 },
   "path",
-  { data_type => "text", is_nullable => 0 },
+  { data_type => "text", is_nullable => 1 },
 );
 
 =head1 PRIMARY KEY
@@ -119,8 +119,8 @@ __PACKAGE__->belongs_to(
 );
 
 
-# Created by DBIx::Class::Schema::Loader v0.07049 @ 2021-08-26 12:02:36
+# Created by DBIx::Class::Schema::Loader v0.07049 @ 2022-06-30 12:02:32
-# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:gxp8rOjpRVen4YbIjomHTw
+# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:Bad70CRTt7zb2GGuRoQ++Q
 
 
 # You can replace this text with custom code or comments, and it will be preserved on regeneration

src/lib/Hydra/View/NARInfo.pm

@@ -8,6 +8,7 @@ use MIME::Base64;
 use Nix::Manifest;
 use Nix::Store;
 use Nix::Utils;
+use Hydra::Helper::Nix;
 use base qw/Catalyst::View/;
 
 sub process {
@@ -17,7 +18,7 @@ sub process {
 
     $c->response->content_type('text/x-nix-narinfo'); # !!! check MIME type
 
-    my ($deriver, $narHash, $time, $narSize, $refs) = queryPathInfo($storePath, 1);
+    my ($deriver, $narHash, $time, $narSize, $refs) = $MACHINE_LOCAL_STORE->queryPathInfo($storePath, 1);
 
     my $info;
     $info .= "StorePath: $storePath\n";
@@ -28,8 +29,8 @@ sub process {
     $info .= "References: " . join(" ", map { basename $_ } @{$refs}) . "\n";
     if (defined $deriver) {
         $info .= "Deriver: " . basename $deriver . "\n";
-        if (isValidPath($deriver)) {
+        if ($MACHINE_LOCAL_STORE->isValidPath($deriver)) {
-            my $drv = derivationFromPath($deriver);
+            my $drv = $MACHINE_LOCAL_STORE->derivationFromPath($deriver);
             $info .= "System: $drv->{platform}\n";
         }
     }

src/lib/Hydra/View/NixLog.pm

@@ -16,7 +16,10 @@ sub process {
 
     my $tail = int($c->stash->{tail} // "0");
 
-    if ($logPath =~ /\.bz2$/) {
+    if ($logPath =~ /\.zst$/) {
+        my $doTail = $tail ? "| tail -n '$tail'" : "";
+        open($fh, "-|", "zstd -dc < '$logPath' $doTail") or die;
+    } elsif ($logPath =~ /\.bz2$/) {
         my $doTail = $tail ? "| tail -n '$tail'" : "";
         open($fh, "-|", "bzip2 -dc < '$logPath' $doTail") or die;
     } else {

src/lib/Makefile.am

@@ -1,22 +0,0 @@
-PERL_MODULES =					\
-  $(wildcard *.pm)				\
-  $(wildcard Hydra/*.pm)			\
-  $(wildcard Hydra/Helper/*.pm)			\
-  $(wildcard Hydra/Model/*.pm)			\
-  $(wildcard Hydra/View/*.pm)			\
-  $(wildcard Hydra/Schema/*.pm)			\
-  $(wildcard Hydra/Schema/Result/*.pm)			\
-  $(wildcard Hydra/Schema/ResultSet/*.pm)			\
-  $(wildcard Hydra/Controller/*.pm)		\
-  $(wildcard Hydra/Base/*.pm)			\
-  $(wildcard Hydra/Base/Controller/*.pm) 	\
-  $(wildcard Hydra/Script/*.pm)			\
-  $(wildcard Hydra/Component/*.pm)		\
-  $(wildcard Hydra/Event/*.pm)		\
-  $(wildcard Hydra/Plugin/*.pm)
-
-EXTRA_DIST = \
-  $(PERL_MODULES)
-
-hydradir = $(libexecdir)/hydra/lib
-nobase_hydra_DATA = $(PERL_MODULES)

src/libhydra/db.hh

@@ -2,6 +2,7 @@
 
 #include <pqxx/pqxx>
 
+#include "environment-variables.hh"
 #include "util.hh"
 
 

src/libhydra/hydra-config.hh

@@ -2,6 +2,7 @@
 
 #include <map>
 
+#include "file-system.hh"
 #include "util.hh"
 
 struct HydraConfig

src/libhydra/meson.build

@@ -0,0 +1,5 @@
+libhydra_inc = include_directories('.')
+
+libhydra_dep = declare_dependency(
+  include_directories: [libhydra_inc],
+)

src/meson.build

@@ -0,0 +1,86 @@
+# Native code
+subdir('libhydra')
+subdir('hydra-build-step')
+subdir('hydra-evaluator')
+subdir('hydra-queue-runner')
+
+hydra_libexecdir = get_option('libexecdir') / 'hydra'
+
+# Data and interpreted
+foreach dir : ['lib', 'root']
+  install_subdir(dir,
+    install_dir: hydra_libexecdir,
+  )
+endforeach
+subdir('sql')
+subdir('ttf')
+
+# Static files for website
+
+hydra_libexecdir_static = hydra_libexecdir / 'root' / 'static'
+
+## Bootstrap
+
+bootstrap_name = 'bootstrap-4.3.1-dist'
+bootstrap = custom_target(
+  'extract-bootstrap',
+  input: 'root' / (bootstrap_name + '.zip'),
+  output: bootstrap_name,
+  command: ['unzip', '-u', '-d', '@OUTDIR@', '@INPUT@'],
+)
+custom_target(
+  'name-bootstrap',
+  input: bootstrap,
+  output: 'bootstrap',
+  command: ['cp', '-r', '@INPUT@' , '@OUTPUT@'],
+  install: true,
+  install_dir: hydra_libexecdir_static,
+)
+
+## Flot
+
+custom_target(
+  'extract-flot',
+  input: 'root' / 'flot-0.8.3.zip',
+  output: 'flot',
+  command: ['unzip', '-u', '-d', '@OUTDIR@', '@INPUT@'],
+  install: true,
+  install_dir: hydra_libexecdir_static / 'js',
+)
+
+## Fontawesome
+
+fontawesome_name = 'fontawesome-free-5.10.2-web'
+fontawesome = custom_target(
+  'extract-fontawesome',
+  input: 'root' / (fontawesome_name + '.zip'),
+  output: fontawesome_name,
+  command: ['unzip', '-u', '-d', '@OUTDIR@', '@INPUT@'],
+)
+custom_target(
+  'name-fontawesome-css',
+  input: fontawesome,
+  output: 'css',
+  command: ['cp', '-r', '@INPUT@/css', '@OUTPUT@'],
+  install: true,
+  install_dir: hydra_libexecdir_static / 'fontawesome',
+)
+custom_target(
+  'name-fontawesome-webfonts',
+  input: fontawesome,
+  output: 'webfonts',
+  command: ['cp', '-r', '@INPUT@/webfonts', '@OUTPUT@'],
+  install: true,
+  install_dir: hydra_libexecdir_static / 'fontawesome',
+)
+
+# Scripts
+
+install_subdir('script',
+  install_dir: get_option('bindir'),
+  exclude_files: [
+    'hydra-dev-server',
+  ],
+  install_mode: 'rwxr-xr-x',
+  strip_directory: true,
+)

src/root/Makefile.am

@@ -1,39 +0,0 @@
-TEMPLATES = $(wildcard *.tt)
-STATIC = \
-  $(wildcard static/images/*) \
-  $(wildcard static/css/*) \
-  static/js/bootbox.min.js \
-  static/js/popper.min.js \
-  static/js/common.js \
-  static/js/jquery/jquery-3.4.1.min.js \
-  static/js/jquery/jquery-ui-1.10.4.min.js
-
-FLOT = flot-0.8.3.zip
-BOOTSTRAP = bootstrap-4.3.1-dist.zip
-FONTAWESOME = fontawesome-free-5.10.2-web.zip
-
-ZIPS = $(FLOT) $(BOOTSTRAP) $(FONTAWESOME)
-
-EXTRA_DIST = $(TEMPLATES) $(STATIC) $(ZIPS)
-
-hydradir = $(libexecdir)/hydra/root
-nobase_hydra_DATA = $(EXTRA_DIST)
-
-all:
-	mkdir -p $(srcdir)/static/js
-	unzip -u -d $(srcdir)/static $(BOOTSTRAP)
-	rm -rf $(srcdir)/static/bootstrap
-	mv $(srcdir)/static/$(basename $(BOOTSTRAP)) $(srcdir)/static/bootstrap
-	unzip -u -d $(srcdir)/static/js $(FLOT)
-	unzip -u -d $(srcdir)/static $(FONTAWESOME)
-	rm -rf $(srcdir)/static/fontawesome
-	mv $(srcdir)/static/$(basename $(FONTAWESOME)) $(srcdir)/static/fontawesome
-
-install-data-local: $(ZIPS)
-	mkdir -p $(hydradir)/static/js
-	cp -prvd $(srcdir)/static/js/* $(hydradir)/static/js
-	mkdir -p $(hydradir)/static/bootstrap
-	cp -prvd $(srcdir)/static/bootstrap/* $(hydradir)/static/bootstrap
-	mkdir -p $(hydradir)/static/fontawesome/{css,webfonts}
-	cp -prvd $(srcdir)/static/fontawesome/css/* $(hydradir)/static/fontawesome/css
-	cp -prvd $(srcdir)/static/fontawesome/webfonts/* $(hydradir)/static/fontawesome/webfonts

src/root/auth.tt

@@ -33,7 +33,7 @@
   <div id="hydra-signin" class="modal hide fade" tabindex="-1" role="dialog" aria-hidden="true">
     <div class="modal-dialog" role="document">
       <div class="modal-content">
-        <form>
+        <form id="signin-form">
           <div class="modal-body">
             <div class="form-group">
               <label for="username" class="col-form-label">User name</label>
@@ -45,7 +45,7 @@
             </div>
           </div>
           <div class="modal-footer">
-            <button id="do-signin" type="button" class="btn btn-primary">Sign in</button>
+            <button type="submit" class="btn btn-primary">Sign in</button>
             <button type="button" class="btn btn-secondary" data-dismiss="modal">Cancel</button>
           </div>
         </form>
@@ -57,10 +57,11 @@
 
     function finishSignOut() { }
 
-    $("#do-signin").click(function() {
+    $("#signin-form").submit(function(e) {
+      e.preventDefault();
       requestJSON({
         url: "[% c.uri_for('/login') %]",
-        data: $(this).parents("form").serialize(),
+        data: $(this).serialize(),
         type: 'POST',
         success: function(data) {
           window.location.reload();

src/root/common.tt

@@ -374,7 +374,7 @@ BLOCK renderInputDiff; %]
               [% ELSIF bi1.uri == bi2.uri && bi1.revision != bi2.revision %]
                 [% IF bi1.type == "git" %]
                   <tr><td>
-                    <b>[% bi1.name %]</b></td><td><tt>[% INCLUDE renderDiffUri contents=(bi1.revision.substr(0, 8) _ ' to ' _ bi2.revision.substr(0, 8)) %]</tt>
+                    <b>[% bi1.name %]</b></td><td><tt>[% INCLUDE renderDiffUri contents=(bi1.revision.substr(0, 12) _ ' to ' _ bi2.revision.substr(0, 12)) %]</tt>
                   </td></tr>
                 [% ELSE %]
                   <tr><td>

src/root/jobset.tt

@@ -205,6 +205,7 @@
         if (!c) return;
         requestJSON({
           url: "[% HTML.escape(c.uri_for('/api/push', { jobsets = project.name _ ':' _ jobset.name, force = "1" })) %]",
+          type: 'POST',
           success: function(data) {
             bootbox.alert("The jobset has been scheduled for evaluation.");
           }

src/root/layout.tt

@@ -93,7 +93,7 @@
       <footer class="navbar">
         <hr />
         <small>
-          <em><a href="http://nixos.org/hydra" target="_blank" class="squiggle">Hydra</a> [% HTML.escape(version) %] (using [% HTML.escape(nixVersion) %]).</em>
+          <em><a href="http://nixos.org/hydra" target="_blank" class="squiggle">Hydra</a> [% HTML.escape(version) %] (using [% HTML.escape(nixVersion) %] and [% HTML.escape(nixEvalJobsVersion) %]).</em>
           [% IF c.user_exists %]
           You are signed in as <tt>[% HTML.escape(c.user.username) %]</tt>
             [%- IF c.user.type == 'google' %] via Google[% END %].

src/root/reproduce.tt

@@ -7,7 +7,7 @@ main() {
 
 set -e
 
-tmpDir=${TMPDIR:-/tmp}/build-[% build.id +%]
+tmpDir=$(realpath "${TMPDIR:-/tmp}")/build-[% build.id +%]
 declare -a args extraArgs
 
 

src/root/topbar.tt

@@ -134,7 +134,7 @@
     [% WRAPPER makeSubMenu title="Sign in" id="sign-in-menu" align="right" %]
       [% IF c.config.enable_google_login %]
         <script src="https://accounts.google.com/gsi/client" async defer></script>
-        <div id="g_id_onload" data-client_id="[% c.config.google_client_id %]" data-callback="onGoogleSignIn">
+        <div id="g_id_onload" data-client_id="[% c.config.google_client_id %]" data-auto_prompt="false" data-callback="onGoogleSignIn">
         </div>
         <div class="g_id_signin" data-type="standard"></div>
         <div class="dropdown-divider"></div>

src/root/user.tt

@@ -91,6 +91,7 @@
           [% INCLUDE roleoption mutable=mutable role="restart-jobs" %]
           [% INCLUDE roleoption mutable=mutable role="bump-to-front" %]
           [% INCLUDE roleoption mutable=mutable role="cancel-build" %]
+          [% INCLUDE roleoption mutable=mutable role="eval-jobset" %]
         </p>
       </div>
     </div>

src/script/Makefile.am

@@ -1,19 +0,0 @@
-EXTRA_DIST =					\
-  $(distributable_scripts)
-
-distributable_scripts =				\
-  hydra-backfill-ids				\
-  hydra-init					\
-  hydra-eval-jobset				\
-  hydra-server					\
-  hydra-update-gc-roots				\
-  hydra-s3-backup-collect-garbage		\
-  hydra-create-user				\
-  hydra-notify					\
-  hydra-send-stats				\
-  nix-prefetch-git				\
-  nix-prefetch-bzr				\
-  nix-prefetch-hg
-
-bin_SCRIPTS =					\
-  $(distributable_scripts)

src/script/hydra-eval-jobset

@@ -17,6 +17,7 @@ use Hydra::Helper::Nix;
 use Hydra::Model::DB;
 use Hydra::Plugin;
 use Hydra::Schema;
+use IPC::Run;
 use JSON::MaybeXS;
 use Net::Statsd;
 use Nix::Store;
@@ -85,14 +86,14 @@ sub attrsToSQL {
 # Fetch a store path from 'eval_substituter' if not already present.
 sub getPath {
     my ($path) = @_;
-    return 1 if isValidPath($path);
+    return 1 if $MACHINE_LOCAL_STORE->isValidPath($path);
 
     my $substituter = $config->{eval_substituter};
 
     system("nix", "--experimental-features", "nix-command", "copy", "--from", $substituter, "--", $path)
         if defined $substituter;
 
-    return isValidPath($path);
+    return $MACHINE_LOCAL_STORE->isValidPath($path);
 }
 
 
@@ -143,7 +144,7 @@ sub fetchInputBuild {
         , version => $version
         , outputName => $mainOutput->name
         };
-    if (isValidPath($prevBuild->drvpath)) {
+    if ($MACHINE_LOCAL_STORE->isValidPath($prevBuild->drvpath)) {
         $result->{drvPath} = $prevBuild->drvpath;
     }
 
@@ -233,7 +234,7 @@ sub fetchInputEval {
         my $out = $build->buildoutputs->find({ name => "out" });
         next unless defined $out;
         # FIXME: Should we fail if the path is not valid?
-        next unless isValidPath($out->path);
+        next unless $MACHINE_LOCAL_STORE->isValidPath($out->path);
         $jobs->{$build->get_column('job')} = $out->path;
     }
 
@@ -357,22 +358,32 @@ sub evalJobs {
     my @cmd;
 
     if (defined $flakeRef) {
-        @cmd = ("hydra-eval-jobs",
+        my $nix_expr =
-                "--flake", $flakeRef,
+            "let " .
-                "--gc-roots-dir", getGCRootsDir,
+            "flake = builtins.getFlake (toString \"$flakeRef\"); " .
-                "--max-jobs", 1);
+            "in " .
+            "flake.hydraJobs " .
+            "or flake.checks " .
+            "or (throw \"flake '$flakeRef' does not provide any Hydra jobs or checks\")";
+
+        @cmd = ("nix-eval-jobs", "--expr", $nix_expr);
     } else {
         my $nixExprInput = $inputInfo->{$nixExprInputName}->[0]
             or die "cannot find the input containing the job expression\n";
 
-        @cmd = ("hydra-eval-jobs",
+        @cmd = ("nix-eval-jobs",
                 "<" . $nixExprInputName . "/" . $nixExprPath . ">",
-                "--gc-roots-dir", getGCRootsDir,
-                "--max-jobs", 1,
                 inputsToArgs($inputInfo));
     }
 
-    push @cmd, "--no-allow-import-from-derivation" if $config->{allow_import_from_derivation} // "true" ne "true";
+    push @cmd, ("--gc-roots-dir", getGCRootsDir);
+    push @cmd, ("--max-jobs", 1);
+    push @cmd, "--meta";
+    push @cmd, "--constituents";
+    push @cmd, "--force-recurse";
+    push @cmd, ("--option", "allow-import-from-derivation", "false") if $config->{allow_import_from_derivation} // "true" ne "true";
+    push @cmd, ("--workers", $config->{evaluator_workers} // 1);
+    push @cmd, ("--max-memory-size", $config->{evaluator_max_memory_size} // 4096);
 
     if (defined $ENV{'HYDRA_DEBUG'}) {
         sub escape {
@@ -384,14 +395,40 @@ sub evalJobs {
         print STDERR "evaluator: @escaped\n";
     }
 
-    (my $res, my $jobsJSON, my $stderr) = captureStdoutStderr(21600, @cmd);
+    my $evalProc = IPC::Run::start \@cmd,
-    die "hydra-eval-jobs returned " . ($res & 127 ? "signal $res" : "exit code " . ($res >> 8))
+            '>', IPC::Run::new_chunker, \my $out,
-        . ":\n" . ($stderr ? decode("utf-8", $stderr) : "(no output)\n")
+            '2>', \my $err;
-        if $res;
 
-    print STDERR "$stderr";
+    return sub {
+        while (1) {
+            $evalProc->pump;
+            if (!defined $out && !defined $err) {
+                $evalProc->finish;
+                if ($?) {
+                    die "nix-eval-jobs returned " . ($? & 127 ? "signal $?" : "exit code " . ($? >> 8)) . "\n";
+                }
+                return;
+            }
 
-    return decode_json($jobsJSON);
+            if (defined $err) {
+                print STDERR "$err";
+                undef $err;
+            }
+
+            if (defined $out && $out ne '') {
+                my $job;
+                try {
+                    $job = decode_json($out);
+                } catch {
+                    warn "nix-eval-jobs sent invalid JSON.\n  parse error: $_\n  invalid json: $out\n";
+                };
+                undef $out;
+                if (defined $job) {
+                    return $job;
+                }
+            }
+        }
+    };
 }
 
 
@@ -420,7 +457,7 @@ sub checkBuild {
     my $firstOutputName = $outputNames[0];
     my $firstOutputPath = $buildInfo->{outputs}->{$firstOutputName};
 
-    my $jobName = $buildInfo->{jobName} or die;
+    my $jobName = $buildInfo->{attr} or die;
     my $drvPath = $buildInfo->{drvPath} or die;
 
     my $build;
@@ -438,13 +475,17 @@ sub checkBuild {
         # new build to be scheduled if the meta.maintainers field is
         # changed?
         if (defined $prevEval) {
+            my $pathOrDrvConstraint = defined $firstOutputPath
+                ? { path => $firstOutputPath }
+                : { drvPath => $drvPath };
+
             my ($prevBuild) = $prevEval->builds->search(
                 # The "project" and "jobset" constraints are
                 # semantically unnecessary (because they're implied by
                 # the eval), but they give a factor 1000 speedup on
                 # the Nixpkgs jobset with PostgreSQL.
                 { jobset_id => $jobset->get_column('id'), job => $jobName,
-                  name => $firstOutputName, path => $firstOutputPath },
+                  name => $firstOutputName, %$pathOrDrvConstraint },
                 { rows => 1, columns => ['id', 'finished'], join => ['buildoutputs'] });
             if (defined $prevBuild) {
                 #print STDERR "    already scheduled/built as build ", $prevBuild->id, "\n";
@@ -470,9 +511,30 @@ sub checkBuild {
 
         my $time = time();
 
-        sub null {
+        sub getMeta {
-            my ($s) = @_;
+            my ($s, $def) = @_;
-            return $s eq "" ? undef : $s;
+            return ($s || "") eq "" ? $def : $s;
+        }
+
+        sub getMetaStrings {
+            my ($v, $k, $acc) = @_;
+            my $t = ref $v;
+
+            if ($t eq 'HASH') {
+                push @$acc, $v->{$k} if exists $v->{$k};
+            } elsif ($t eq 'ARRAY') {
+                getMetaStrings($_, $k, $acc) foreach @$v;
+            } elsif (defined $v) {
+                push @$acc, $v;
+            }
+        }
+
+        sub getMetaConcatStrings {
+            my ($v, $k) = @_;
+
+            my @strings;
+            getMetaStrings($v, $k, \@strings);
+            return join(", ", @strings) || undef;
         }
 
         # Add the build to the database.
@@ -480,19 +542,19 @@ sub checkBuild {
             { timestamp => $time
             , jobset_id => $jobset->id
             , job => $jobName
-            , description => null($buildInfo->{description})
+            , description => getMeta($buildInfo->{meta}->{description}, undef)
-            , license => null($buildInfo->{license})
+            , license => getMetaConcatStrings($buildInfo->{meta}->{license}, "shortName")
-            , homepage => null($buildInfo->{homepage})
+            , homepage => getMeta($buildInfo->{meta}->{homepage}, undef)
-            , maintainers => null($buildInfo->{maintainers})
+            , maintainers => getMetaConcatStrings($buildInfo->{meta}->{maintainers}, "email")
-            , maxsilent => $buildInfo->{maxSilent}
+            , maxsilent => getMeta($buildInfo->{meta}->{maxSilent}, 7200)
-            , timeout => $buildInfo->{timeout}
+            , timeout => getMeta($buildInfo->{meta}->{timeout}, 36000)
-            , nixname => $buildInfo->{nixName}
+            , nixname => $buildInfo->{name}
             , drvpath => $drvPath
             , system => $buildInfo->{system}
-            , priority => $buildInfo->{schedulingPriority}
+            , priority => getMeta($buildInfo->{meta}->{schedulingPriority}, 100)
             , finished => 0
             , iscurrent => 1
-            , ischannel => $buildInfo->{isChannel}
+            , ischannel => getMeta($buildInfo->{meta}->{isChannel}, 0)
             });
 
         $build->buildoutputs->create({ name => $_, path => $buildInfo->{outputs}->{$_} })
@@ -661,7 +723,7 @@ sub checkJobsetWrapped {
         return;
     }
 
-    # Hash the arguments to hydra-eval-jobs and check the
+    # Hash the arguments to nix-eval-jobs and check the
     # JobsetInputHashes to see if the previous evaluation had the same
     # inputs.  If so, bail out.
     my @args = ($jobset->nixexprinput // "", $jobset->nixexprpath // "", inputsToArgs($inputInfo));
@@ -683,19 +745,12 @@ sub checkJobsetWrapped {
 
     # Evaluate the job expression.
     my $evalStart = clock_gettime(CLOCK_MONOTONIC);
-    my $jobs = evalJobs($project->name . ":" . $jobset->name, $inputInfo, $jobset->nixexprinput, $jobset->nixexprpath, $flakeRef);
+    my $evalStop;
-    my $evalStop = clock_gettime(CLOCK_MONOTONIC);
+    my $jobsIter = evalJobs($project->name . ":" . $jobset->name, $inputInfo, $jobset->nixexprinput, $jobset->nixexprpath, $flakeRef);
-
-    if ($jobsetsJobset) {
-        my @keys = keys %$jobs;
-        die "The .jobsets jobset must only have a single job named 'jobsets'"
-            unless (scalar @keys) == 1 && $keys[0] eq "jobsets";
-    }
-    Net::Statsd::timing("hydra.evaluator.eval_time", int(($evalStop - $evalStart) * 1000));
 
     if ($dryRun) {
-        foreach my $name (keys %{$jobs}) {
+        while (defined(my $job = $jobsIter->())) {
-            my $job = $jobs->{$name};
+            my $name = $job->{attr};
             if (defined $job->{drvPath}) {
                 print STDERR "good job $name: $job->{drvPath}\n";
             } else {
@@ -705,36 +760,20 @@ sub checkJobsetWrapped {
         return;
     }
 
-    die "Jobset contains a job with an empty name. Make sure the jobset evaluates to an attrset of jobs.\n"
-        if defined $jobs->{""};
-
-    $jobs->{$_}->{jobName} = $_ for keys %{$jobs};
-
-    my $jobOutPathMap = {};
-    my $jobsetChanged = 0;
-    my $dbStart = clock_gettime(CLOCK_MONOTONIC);
-
-
     # Store the error messages for jobs that failed to evaluate.
     my $evaluationErrorTime = time;
     my $evaluationErrorMsg = "";
-    foreach my $job (values %{$jobs}) {
-        next unless defined $job->{error};
-        $evaluationErrorMsg .=
-            ($job->{jobName} ne "" ? "in job ‘$job->{jobName}’" : "at top-level") .
-            ":\n" . $job->{error} . "\n\n";
-    }
-    setJobsetError($jobset, $evaluationErrorMsg, $evaluationErrorTime);
-
     my $evaluationErrorRecord = $db->resultset('EvaluationErrors')->create(
         { errormsg => $evaluationErrorMsg
         , errortime => $evaluationErrorTime
         }
     );
 
+    my $jobOutPathMap = {};
+    my $jobsetChanged = 0;
     my %buildMap;
-    $db->txn_do(sub {
 
+    $db->txn_do(sub {
         my $prevEval = getPrevJobsetEval($db, $jobset, 1);
 
         # Clear the "current" flag on all builds.  Since we're in a
@@ -747,7 +786,7 @@ sub checkJobsetWrapped {
             , evaluationerror => $evaluationErrorRecord
             , timestamp => time
             , checkouttime => abs(int($checkoutStop - $checkoutStart))
-            , evaltime => abs(int($evalStop - $evalStart))
+            , evaltime => 0
             , hasnewbuilds => 0
             , nrbuilds => 0
             , flake => $flakeRef
@@ -755,11 +794,24 @@ sub checkJobsetWrapped {
             , nixexprpath => $jobset->nixexprpath
             });
 
-        # Schedule each successfully evaluated job.
+        my @jobsWithConstituents;
-        foreach my $job (permute(values %{$jobs})) {
+
-            next if defined $job->{error};
+        while (defined(my $job = $jobsIter->())) {
-            #print STDERR "considering job " . $project->name, ":", $jobset->name, ":", $job->{jobName} . "\n";
+            if ($jobsetsJobset) {
-            checkBuild($db, $jobset, $ev, $inputInfo, $job, \%buildMap, $prevEval, $jobOutPathMap, $plugins);
+                die "The .jobsets jobset must only have a single job named 'jobsets'"
+                    unless $job->{attr} eq "jobsets";
+            }
+
+            $evaluationErrorMsg .=
+                ($job->{attr} ne "" ? "in job ‘$job->{attr}’" : "at top-level") .
+                ":\n" . $job->{error} . "\n\n" if defined $job->{error};
+
+            checkBuild($db, $jobset, $ev, $inputInfo, $job, \%buildMap, $prevEval, $jobOutPathMap, $plugins)
+                unless defined $job->{error};
+
+            if (defined $job->{constituents}) {
+                push @jobsWithConstituents, $job;
+            }
         }
 
         # Have any builds been added or removed since last time?
@@ -797,21 +849,20 @@ sub checkJobsetWrapped {
                 $drvPathToId{$x->{drvPath}} = $x;
             }
 
-            foreach my $job (values %{$jobs}) {
+            foreach my $job (values @jobsWithConstituents) {
-                next unless $job->{constituents};
+                next unless defined $job->{constituents};
-
                 if (defined $job->{error}) {
-                    die "aggregate job ‘$job->{jobName}’ failed with the error: $job->{error}\n";
+                    die "aggregate job ‘$job->{attr}’ failed with the error: $job->{error}\n";
                 }
 
                 my $x = $drvPathToId{$job->{drvPath}} or
-                    die "aggregate job ‘$job->{jobName}’ has no corresponding build record.\n";
+                    die "aggregate job ‘$job->{attr}’ has no corresponding build record.\n";
                 foreach my $drvPath (@{$job->{constituents}}) {
                     my $constituent = $drvPathToId{$drvPath};
                     if (defined $constituent) {
                         $db->resultset('AggregateConstituents')->update_or_create({aggregate => $x->{id}, constituent => $constituent->{id}});
                     } else {
-                        warn "aggregate job ‘$job->{jobName}’ has a constituent ‘$drvPath’ that doesn't correspond to a Hydra build\n";
+                        warn "aggregate job ‘$job->{attr}’ has a constituent ‘$drvPath’ that doesn't correspond to a Hydra build\n";
                     }
                 }
             }
@@ -853,11 +904,15 @@ sub checkJobsetWrapped {
         $jobset->update({ enabled => 0 }) if $jobset->enabled == 2;
 
         $jobset->update({ lastcheckedtime => time, forceeval => undef });
+
+        $evaluationErrorRecord->update({ errormsg => $evaluationErrorMsg });
+        setJobsetError($jobset, $evaluationErrorMsg, $evaluationErrorTime);
+
+        $evalStop = clock_gettime(CLOCK_MONOTONIC);
+        $ev->update({ evaltime => abs(int($evalStop - $evalStart)) });
     });
 
-    my $dbStop = clock_gettime(CLOCK_MONOTONIC);
+    Net::Statsd::timing("hydra.evaluator.eval_time", int(($evalStop - $evalStart) * 1000));
-
-    Net::Statsd::timing("hydra.evaluator.db_time", int(($dbStop - $dbStart) * 1000));
     Net::Statsd::increment("hydra.evaluator.evals");
     Net::Statsd::increment("hydra.evaluator.cached_evals") unless $jobsetChanged;
 }

src/script/hydra-update-gc-roots

@@ -5,7 +5,6 @@ use warnings;
 use File::Path;
 use File::stat;
 use File::Basename;
-use Nix::Store;
 use Hydra::Config;
 use Hydra::Schema;
 use Hydra::Helper::Nix;
@@ -47,7 +46,7 @@ sub keepBuild {
         $build->finished && ($build->buildstatus == 0 || $build->buildstatus == 6))
     {
         foreach my $path (split / /, $build->get_column('outpaths')) {
-            if (isValidPath($path)) {
+            if ($MACHINE_LOCAL_STORE->isValidPath($path)) {
                 addRoot $path;
             } else {
                 print STDERR "    warning: output ", $path, " has disappeared\n" if $build->finished;
@@ -55,7 +54,7 @@ sub keepBuild {
         }
     }
     if (!$build->finished || ($keepFailedDrvs && $build->buildstatus != 0)) {
-        if (isValidPath($build->drvpath)) {
+        if ($MACHINE_LOCAL_STORE->isValidPath($build->drvpath)) {
             addRoot $build->drvpath;
         } else {
             print STDERR "    warning: derivation ", $build->drvpath, " has disappeared\n";

src/script/nix-prefetch-git

@@ -78,7 +78,7 @@ fi
 
 init_remote(){
     local url=$1;
-    git init;
+    git init --initial-branch=trunk;
     git remote add origin $url;
 }
 

src/sql/Makefile.am

@@ -1,9 +0,0 @@
-sqldir = $(libexecdir)/hydra/sql
-nobase_dist_sql_DATA =				\
-  hydra.sql					\
-  test.sql					\
-  upgrade-*.sql					\
-  update-dbix.pl
-
-update-dbix: hydra.sql
-	./update-dbix-harness.sh

src/sql/hydra.sql

@@ -247,7 +247,7 @@ create trigger BuildBumped after update on Builds for each row
 create table BuildOutputs (
     build         integer not null,
     name          text not null,
-    path          text not null,
+    path          text,
     primary key   (build, name),
     foreign key   (build) references Builds(id) on delete cascade
 );
@@ -303,7 +303,7 @@ create table BuildStepOutputs (
     build         integer not null,
     stepnr        integer not null,
     name          text not null,
-    path          text not null,
+    path          text,
     primary key   (build, stepnr, name),
     foreign key   (build) references Builds(id) on delete cascade,
     foreign key   (build, stepnr) references BuildSteps(build, stepnr) on delete cascade

src/sql/meson.build

@@ -0,0 +1,90 @@
+sql_files = files(
+  'hydra.sql',
+  'test.sql',
+  'update-dbix.pl',
+  'upgrade-2.sql',
+  'upgrade-3.sql',
+  'upgrade-4.sql',
+  'upgrade-5.sql',
+  'upgrade-6.sql',
+  'upgrade-7.sql',
+  'upgrade-8.sql',
+  'upgrade-9.sql',
+  'upgrade-10.sql',
+  'upgrade-11.sql',
+  'upgrade-12.sql',
+  'upgrade-13.sql',
+  'upgrade-14.sql',
+  'upgrade-15.sql',
+  'upgrade-16.sql',
+  'upgrade-17.sql',
+  'upgrade-18.sql',
+  'upgrade-19.sql',
+  'upgrade-20.sql',
+  'upgrade-21.sql',
+  'upgrade-22.sql',
+  'upgrade-23.sql',
+  'upgrade-24.sql',
+  'upgrade-25.sql',
+  'upgrade-26.sql',
+  'upgrade-27.sql',
+  'upgrade-28.sql',
+  'upgrade-29.sql',
+  'upgrade-30.sql',
+  'upgrade-31.sql',
+  'upgrade-32.sql',
+  'upgrade-33.sql',
+  'upgrade-34.sql',
+  'upgrade-35.sql',
+  'upgrade-36.sql',
+  'upgrade-37.sql',
+  'upgrade-38.sql',
+  'upgrade-39.sql',
+  'upgrade-40.sql',
+  'upgrade-41.sql',
+  'upgrade-42.sql',
+  'upgrade-43.sql',
+  'upgrade-44.sql',
+  'upgrade-45.sql',
+  'upgrade-46.sql',
+  'upgrade-47.sql',
+  'upgrade-48.sql',
+  'upgrade-49.sql',
+  'upgrade-50.sql',
+  'upgrade-51.sql',
+  'upgrade-52.sql',
+  'upgrade-53.sql',
+  'upgrade-54.sql',
+  'upgrade-55.sql',
+  'upgrade-56.sql',
+  'upgrade-57.sql',
+  'upgrade-58.sql',
+  'upgrade-59.sql',
+  'upgrade-60.sql',
+  'upgrade-61.sql',
+  'upgrade-62.sql',
+  'upgrade-63.sql',
+  'upgrade-64.sql',
+  'upgrade-65.sql',
+  'upgrade-66.sql',
+  'upgrade-67.sql',
+  'upgrade-68.sql',
+  'upgrade-69.sql',
+  'upgrade-70.sql',
+  'upgrade-71.sql',
+  'upgrade-72.sql',
+  'upgrade-73.sql',
+  'upgrade-74.sql',
+  'upgrade-75.sql',
+  'upgrade-76.sql',
+  'upgrade-77.sql',
+  'upgrade-78.sql',
+  'upgrade-79.sql',
+  'upgrade-80.sql',
+  'upgrade-81.sql',
+  'upgrade-82.sql',
+  'upgrade-83.sql',
+  'upgrade-84.sql',
+)
+
+install_data(sql_files, install_dir: hydra_libexecdir / 'sql')

src/sql/upgrade-84.sql

@@ -0,0 +1,4 @@
+-- CA derivations do not have statically known output paths. The values
+-- are only filled in after the build runs.
+ALTER TABLE BuildStepOutputs ALTER COLUMN path DROP NOT NULL;
+ALTER TABLE BuildOutputs ALTER COLUMN path DROP NOT NULL;

src/ttf/Makefile.am

@@ -1,4 +0,0 @@
-EXTRA_DIST = COPYING.LIB StayPuft.ttf
-
-ttfdir = $(libexecdir)/hydra/ttf
-nobase_ttf_DATA = $(EXTRA_DIST)

src/ttf/meson.build

@@ -0,0 +1,5 @@
+data_files = files(
+  'StayPuft.ttf',
+  'COPYING.LIB',
+)
+install_data(data_files, install_dir: hydra_libexecdir / 'ttf')

t/Hydra/Config/ldap_role_map.t

@@ -57,6 +57,7 @@ subtest "getLDAPConfig" => sub {
                             "hydra_cancel-build" => [ "cancel-build" ],
                             "hydra_create-projects" => [ "create-projects" ],
                             "hydra_restart-jobs" => [ "restart-jobs" ],
+                            "hydra_eval-jobset" => [ "eval-jobset" ],
                         }
                     },
                     "The empty file and set env var make legacy mode active."
@@ -177,6 +178,7 @@ subtest "get_legacy_ldap_config" => sub {
                 "hydra_cancel-build" => [ "cancel-build" ],
                 "hydra_create-projects" => [ "create-projects" ],
                 "hydra_restart-jobs" => [ "restart-jobs" ],
+                "hydra_eval-jobset" => [ "eval-jobset" ],
             }
         },
         "Legacy, default role maps are applied."

t/Hydra/Controller/API/checks.t

@@ -22,9 +22,24 @@ sub is_json {
 }
 
 my $ctx = test_context();
-
 Catalyst::Test->import('Hydra');
 
+# Create a user to log in to
+my $user = $ctx->db->resultset('Users')->create({ username => 'alice', emailaddress => 'alice@example.com', password => '!' });
+$user->setPassword('foobar');
+$user->userroles->update_or_create({ role => 'admin' });
+
+# Login and save cookie for future requests
+my $req = request(POST '/login',
+    Referer => 'http://localhost/',
+    Content => {
+        username => 'alice',
+        password => 'foobar'
+    }
+);
+is($req->code, 302, "The login redirects");
+my $cookie = $req->header("set-cookie");
+
 my $finishedBuilds = $ctx->makeAndEvaluateJobset(
     expression => "one-job.nix",
     build => 1
@@ -109,7 +124,10 @@ subtest "/api/push" => sub {
         my $jobsetName = $jobset->name;
         is($jobset->forceeval, undef, "The existing jobset is not set to be forced to eval");
 
-        my $response = request(GET "/api/push?jobsets=$projectName:$jobsetName&force=1");
+        my $response = request(POST "/api/push?jobsets=$projectName:$jobsetName&force=1",
+            Cookie => $cookie,
+            Referer => 'http://localhost/',
+        );
         ok($response->is_success, "The API enpdoint for triggering jobsets returns 200.");
 
         my $data = is_json($response);
@@ -128,7 +146,10 @@ subtest "/api/push" => sub {
 
         print STDERR $repo;
 
-        my $response = request(GET "/api/push?repos=$repo&force=1");
+        my $response = request(POST "/api/push?repos=$repo&force=1",
+            Cookie => $cookie,
+            Referer => 'http://localhost/',
+        );
         ok($response->is_success, "The API enpdoint for triggering jobsets returns 200.");
 
         my $data = is_json($response);

t/Hydra/Controller/Job/builds.t

@@ -54,13 +54,14 @@ subtest "/job/PROJECT/JOBSET/JOB/shield" => sub {
 
 subtest "/job/PROJECT/JOBSET/JOB/prometheus" => sub {
     my $response = request(GET '/job/' . $project->name . '/' . $jobset->name . '/' . $build->job . '/prometheus');
-    ok($response->is_success, "The page showing the job's prometheus data returns 200.");
-    my $metrics = $response->content;
 
-    ok($metrics =~ m/hydra_job_failed\{.*\} 0/);
+    ok($response->is_success, "The page showing the job's prometheus data returns 200.");
-    ok($metrics =~ m/hydra_job_completion_time\{.*\} [\d]+/);
+
-    ok($metrics =~ m/hydra_build_closure_size\{.*\} 96/);
+    my $metrics = $response->content;
-    ok($metrics =~ m/hydra_build_output_size\{.*\} 96/);
+    like($metrics, qr/hydra_job_failed\{.*\} 0/);
+    like($metrics, qr/hydra_job_completion_time\{.*\} [\d]+/);
+    like($metrics, qr/hydra_build_closure_size\{.*\} 96/);
+    like($metrics, qr/hydra_build_output_size\{.*\} 96/);
 };
 
 done_testing;

t/Hydra/Controller/Jobset/http.t

@@ -186,7 +186,7 @@ subtest 'Update jobset "job" to have an invalid input type' => sub {
       })
   );
   ok(!$jobsetupdate->is_success);
-  ok($jobsetupdate->content =~ m/Invalid input type.*valid types:/);
+  like($jobsetupdate->content, qr/Invalid input type.*valid types:/);
 };
 
 

t/Hydra/Controller/User/dashboard.t

@@ -24,7 +24,7 @@ my $cookie = $login->header("set-cookie");
 my $my_jobs = request(GET '/dashboard/alice/my-jobs-tab', Accept => 'application/json', Cookie => $cookie);
 ok($my_jobs->is_success);
 my $content = $my_jobs->content();
-ok($content =~ /empty_dir/);
+like($content, qr/empty_dir/);
 ok(!($content =~ /fails/));
 ok(!($content =~ /succeed_with_failed/));
 done_testing;

t/Hydra/Controller/User/ldap-legacy.t

@@ -24,6 +24,7 @@ $ldap->add_group("hydra_create-projects", $users->{"many_roles"}->{"username"});
 $ldap->add_group("hydra_restart-jobs", $users->{"many_roles"}->{"username"});
 $ldap->add_group("hydra_bump-to-front", $users->{"many_roles"}->{"username"});
 $ldap->add_group("hydra_cancel-build", $users->{"many_roles"}->{"username"});
+$ldap->add_group("hydra_eval-jobset", $users->{"many_roles"}->{"username"});
 
 my $hydra_ldap_config = "${\$ldap->tmpdir()}/hydra_ldap_config.yaml";
 LDAPContext::write_file($hydra_ldap_config, <<YAML);
@@ -68,7 +69,7 @@ subtest "Valid login attempts" => sub {
         unrelated => [],
          admin => ["admin"],
          not_admin => [],
-         many_roles => [ "create-projects", "restart-jobs", "bump-to-front", "cancel-build" ],
+         many_roles => [ "create-projects", "restart-jobs", "bump-to-front", "cancel-build", "eval-jobset" ],
     );
     for my $username (keys %users_to_roles) {
         my $user = $users->{$username};

t/Hydra/Controller/User/ldap.t

@@ -24,6 +24,7 @@ $ldap->add_group("hydra_create-projects", $users->{"many_roles"}->{"username"});
 $ldap->add_group("hydra_restart-jobs", $users->{"many_roles"}->{"username"});
 $ldap->add_group("hydra_bump-to-front", $users->{"many_roles"}->{"username"});
 $ldap->add_group("hydra_cancel-build", $users->{"many_roles"}->{"username"});
+$ldap->add_group("hydra_eval-jobset", $users->{"many_roles"}->{"username"});
 
 
 my $ctx = test_context(
@@ -76,10 +77,12 @@ my $ctx = test_context(
                 hydra_cancel-build = cancel-build
                 hydra_bump-to-front = bump-to-front
                 hydra_restart-jobs = restart-jobs
+                hydra_eval-jobset = eval-jobset
 
                 hydra_one_group_many_roles = create-projects
                 hydra_one_group_many_roles = cancel-build
                 hydra_one_group_many_roles = bump-to-front
+                hydra_one_group_many-roles = eval-jobset
             </role_mapping>
         </ldap>
 CFG
@@ -92,7 +95,7 @@ subtest "Valid login attempts" => sub {
         unrelated => [],
         admin => ["admin"],
         not_admin => [],
-        many_roles => [ "create-projects", "restart-jobs", "bump-to-front", "cancel-build" ],
+        many_roles => [ "create-projects", "restart-jobs", "bump-to-front", "cancel-build", "eval-jobset" ],
         many_roles_one_group => [ "create-projects", "bump-to-front", "cancel-build" ],
     );
     for my $username (keys %users_to_roles) {

t/Makefile.am

@@ -1,39 +0,0 @@
-TESTS_ENVIRONMENT = \
-  BZR_HOME="$(abs_builddir)/data"			\
-  HYDRA_DBI="dbi:Pg:dbname=hydra-test-suite;port=6433"	\
-  HYDRA_DATA="$(abs_builddir)/data"			\
-  HYDRA_HOME="$(top_srcdir)/src"			\
-  HYDRA_CONFIG=						\
-  NIX_REMOTE=						\
-  NIX_REMOTE_SYSTEMS=					\
-  NIX_CONF_DIR="$(abs_builddir)/nix/etc/nix"		\
-  NIX_STATE_DIR="$(abs_builddir)/nix/var/nix"		\
-  NIX_STORE_DIR="$(abs_builddir)/nix/store"			\
-  NIX_LOG_DIR="$(abs_builddir)/nix/var/log/nix"		\
-  PGHOST=/tmp	\
-  PERL5LIB="$(srcdir):$(abs_top_srcdir)/src/lib:$$PERL5LIB"	\
-  PYTHONPATH= \
-  PATH=$(abs_top_srcdir)/src/hydra-evaluator:$(abs_top_srcdir)/src/script:$(abs_top_srcdir)/src/hydra-eval-jobs:$(abs_top_srcdir)/src/hydra-queue-runner:$$PATH \
-  perl -w
-
-EXTRA_DIST = \
-  $(wildcard *.pm) \
-  $(wildcard jobs/*.nix) \
-  $(wildcard jobs/*.sh) \
-  $(TESTS)
-
-TESTS = \
-  perlcritic.pl \
-  test.pl
-
-check_SCRIPTS = repos
-
-repos: dirs
-
-dirs:
-	mkdir -p data
-	touch data/hydra.conf
-	mkdir -p nix
-	mkdir -p nix/etc/nix
-	mkdir -p nix/store
-	mkdir -p nix/var

t/api-test.t

@@ -115,7 +115,7 @@ subtest "evaluation" => sub {
         my $build = decode_json(request_json({ uri => "/build/" . $evals->[0]->{builds}->[0] })->content());
         is($build->{job}, "job", "The build's job name is job");
         is($build->{finished}, 0, "The build isn't finished yet");
-        ok($build->{buildoutputs}->{out}->{path} =~ /\/nix\/store\/[a-zA-Z0-9]{32}-job$/, "The build's outpath is in the Nix store and named 'job'");
+        like($build->{buildoutputs}->{out}->{path}, qr/\/nix\/store\/[a-zA-Z0-9]{32}-job$/, "The build's outpath is in the Nix store and named 'job'");
 
         subtest "search" => sub {
             my $search_project = decode_json(request_json({ uri => "/search/?query=sample" })->content());

t/content-addressed/basic.t

@@ -0,0 +1,63 @@
+use feature 'unicode_strings';
+use strict;
+use warnings;
+use Setup;
+
+my %ctx = test_init(
+    nix_config => qq|
+    experimental-features = ca-derivations
+    |,
+);
+
+require Hydra::Schema;
+require Hydra::Model::DB;
+
+use JSON::MaybeXS;
+
+use HTTP::Request::Common;
+use Test2::V0;
+require Catalyst::Test;
+Catalyst::Test->import('Hydra');
+
+my $db = Hydra::Model::DB->new;
+hydra_setup($db);
+
+my $project = $db->resultset('Projects')->create({name => "tests", displayname => "", owner => "root"});
+
+my $jobset = createBaseJobset("content-addressed", "content-addressed.nix", $ctx{jobsdir});
+
+ok(evalSucceeds($jobset), "Evaluating jobs/content-addressed.nix should exit with return code 0");
+is(nrQueuedBuildsForJobset($jobset), 6, "Evaluating jobs/content-addressed.nix should result in 6 builds");
+
+for my $build (queuedBuildsForJobset($jobset)) {
+    ok(runBuild($build), "Build '".$build->job."' from jobs/content-addressed.nix should exit with code 0");
+    my $newbuild = $db->resultset('Builds')->find($build->id);
+    is($newbuild->finished, 1, "Build '".$build->job."' from jobs/content-addressed.nix should be finished.");
+    my $expected = $build->job eq "fails" ? 1 : $build->job =~ /with_failed/ ? 6 : $build->job =~ /FailingCA/ ? 2 : 0;
+    is($newbuild->buildstatus, $expected, "Build '".$build->job."' from jobs/content-addressed.nix should have buildstatus $expected.");
+
+    my $response = request("/build/".$build->id);
+    ok($response->is_success, "The 'build' page for build '".$build->job."' should load properly");
+
+    if ($newbuild->buildstatus == 0) {
+      my $buildOutputs = $newbuild->buildoutputs;
+      for my $output ($newbuild->buildoutputs) {
+        # XXX: This hardcodes /nix/store/.
+        # It's fine because in practice the nix store for the tests will be of
+        # the form `/some/thing/nix/store/`, but it would be cleaner if there
+        # was a way to query Nix for its store dir?
+        like(
+          $output->path, qr|/nix/store/|,
+          "Output '".$output->name."' of build '".$build->job."' should be a valid store path"
+        );
+      }
+    }
+
+}
+
+# XXX: deststoredir is undefined: Use of uninitialized value $ctx{"deststoredir"} in concatenation (.) or string at t/content-addressed/basic.t line 58.
+# XXX: This test seems to not do what it seems to be doing. See documentation: https://metacpan.org/pod/Test2::V0#isnt($got,-$do_not_want,-$name)
+isnt(<$ctx{deststoredir}/realisations/*>, "", "The destination store should have the realisations of the built derivations registered");
+
+done_testing;
+

t/content-addressed/without-experimental-feature.t

@@ -0,0 +1,28 @@
+use feature 'unicode_strings';
+use strict;
+use warnings;
+use Setup;
+
+my %ctx = test_init();
+
+require Hydra::Schema;
+require Hydra::Model::DB;
+
+use JSON::MaybeXS;
+
+use HTTP::Request::Common;
+use Test2::V0;
+require Catalyst::Test;
+Catalyst::Test->import('Hydra');
+
+my $db = Hydra::Model::DB->new;
+hydra_setup($db);
+
+my $project = $db->resultset('Projects')->create({name => "tests", displayname => "", owner => "root"});
+
+my $jobset = createBaseJobset("content-addressed", "content-addressed.nix", $ctx{jobsdir});
+
+ok(evalSucceeds($jobset),                  "Evaluating jobs/content-addressed.nix without the experimental feature should exit with return code 0");
+is(nrQueuedBuildsForJobset($jobset), 0, "Evaluating jobs/content-addressed.nix without the experimental Nix feature should result in 0 build");
+
+done_testing;