Compare commits
1 Commits
internal-b
...
queued-job
| Author | SHA1 | Date | |
|---|---|---|---|
|
3e24c2eaf7 |
3
.github/workflows/test.yml
vendored
3
.github/workflows/test.yml
vendored
@@ -1,10 +1,7 @@
|
||||
name: "Test"
|
||||
on:
|
||||
pull_request:
|
||||
merge_group:
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
jobs:
|
||||
tests:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
1
.gitignore
vendored
1
.gitignore
vendored
@@ -3,7 +3,6 @@
|
||||
/src/sql/hydra-postgresql.sql
|
||||
/src/sql/hydra-sqlite.sql
|
||||
/src/sql/tmp.sqlite
|
||||
.hydra-data
|
||||
result
|
||||
result-*
|
||||
outputs
|
||||
|
||||
27
README.md
27
README.md
@@ -72,16 +72,17 @@ Make sure **State** at the top of the page is set to "_Enabled_" and click on "_
|
||||
You can build Hydra via `nix-build` using the provided [default.nix](./default.nix):
|
||||
|
||||
```
|
||||
$ nix build
|
||||
$ nix-build
|
||||
```
|
||||
|
||||
### Development Environment
|
||||
|
||||
You can use the provided shell.nix to get a working development environment:
|
||||
```
|
||||
$ nix develop
|
||||
$ mesonConfigurePhase
|
||||
$ ninja
|
||||
$ nix-shell
|
||||
$ autoreconfPhase
|
||||
$ configurePhase # NOTE: not ./configure
|
||||
$ make
|
||||
```
|
||||
|
||||
### Executing Hydra During Development
|
||||
@@ -90,9 +91,9 @@ When working on new features or bug fixes you need to be able to run Hydra from
|
||||
can be done using [foreman](https://github.com/ddollar/foreman):
|
||||
|
||||
```
|
||||
$ nix develop
|
||||
$ nix-shell
|
||||
$ # hack hack
|
||||
$ ninja -C build
|
||||
$ make
|
||||
$ foreman start
|
||||
```
|
||||
|
||||
@@ -114,24 +115,22 @@ Start by following the steps in [Development Environment](#development-environme
|
||||
Then, you can run the tests and the perlcritic linter together with:
|
||||
|
||||
```console
|
||||
$ nix develop
|
||||
$ ninja -C build test
|
||||
$ nix-shell
|
||||
$ make check
|
||||
```
|
||||
|
||||
You can run a single test with:
|
||||
|
||||
```
|
||||
$ nix develop
|
||||
$ cd build
|
||||
$ meson test --test-args=../t/Hydra/Event.t testsuite
|
||||
$ nix-shell
|
||||
$ yath test ./t/foo/bar.t
|
||||
```
|
||||
|
||||
And you can run just perlcritic with:
|
||||
|
||||
```
|
||||
$ nix develop
|
||||
$ cd build
|
||||
$ meson test perlcritic
|
||||
$ nix-shell
|
||||
$ make perlcritic
|
||||
```
|
||||
|
||||
### JSON API
|
||||
|
||||
@@ -11,6 +11,12 @@ $ cd hydra
|
||||
To enter a shell in which all environment variables (such as `PERL5LIB`)
|
||||
and dependencies can be found:
|
||||
|
||||
```console
|
||||
$ nix-shell
|
||||
```
|
||||
|
||||
of when flakes are enabled:
|
||||
|
||||
```console
|
||||
$ nix develop
|
||||
```
|
||||
@@ -18,15 +24,15 @@ $ nix develop
|
||||
To build Hydra, you should then do:
|
||||
|
||||
```console
|
||||
$ mesonConfigurePhase
|
||||
$ ninja
|
||||
[nix-shell]$ autoreconfPhase
|
||||
[nix-shell]$ configurePhase
|
||||
[nix-shell]$ make -j$(nproc)
|
||||
```
|
||||
|
||||
You start a local database, the webserver, and other components with
|
||||
foreman:
|
||||
|
||||
```console
|
||||
$ ninja -C build
|
||||
$ foreman start
|
||||
```
|
||||
|
||||
@@ -41,11 +47,18 @@ $ ./src/script/hydra-server
|
||||
You can run Hydra's test suite with the following:
|
||||
|
||||
```console
|
||||
$ meson test
|
||||
# to run as many tests as you have cores:
|
||||
$ YATH_JOB_COUNT=$NIX_BUILD_CORES meson test
|
||||
[nix-shell]$ make check
|
||||
[nix-shell]$ # to run as many tests as you have cores:
|
||||
[nix-shell]$ make check YATH_JOB_COUNT=$NIX_BUILD_CORES
|
||||
[nix-shell]$ # or run yath directly:
|
||||
[nix-shell]$ yath test
|
||||
[nix-shell]$ # to run as many tests as you have cores:
|
||||
[nix-shell]$ yath test -j $NIX_BUILD_CORES
|
||||
```
|
||||
|
||||
When using `yath` instead of `make check`, ensure you have run `make`
|
||||
in the root of the repository at least once.
|
||||
|
||||
**Warning**: Currently, the tests can fail
|
||||
if run with high parallelism [due to an issue in
|
||||
`Test::PostgreSQL`](https://github.com/TJC/Test-postgresql/issues/40)
|
||||
@@ -62,7 +75,7 @@ will reload the page every time you save.
|
||||
To build Hydra and its dependencies:
|
||||
|
||||
```console
|
||||
$ nix build .#packages.x86_64-linux.default
|
||||
$ nix-build release.nix -A build.x86_64-linux
|
||||
```
|
||||
|
||||
## Development Tasks
|
||||
|
||||
54
flake.lock
generated
54
flake.lock
generated
@@ -1,10 +1,30 @@
|
||||
{
|
||||
"nodes": {
|
||||
"libgit2": {
|
||||
"flake": false,
|
||||
"locked": {
|
||||
"lastModified": 1715853528,
|
||||
"narHash": "sha256-J2rCxTecyLbbDdsyBWn9w7r3pbKRMkI9E7RvRgAqBdY=",
|
||||
"owner": "libgit2",
|
||||
"repo": "libgit2",
|
||||
"rev": "36f7e21ad757a3dacc58cf7944329da6bc1d6e96",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "libgit2",
|
||||
"ref": "v1.8.1",
|
||||
"repo": "libgit2",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"nix": {
|
||||
"inputs": {
|
||||
"flake-compat": [],
|
||||
"flake-parts": [],
|
||||
"git-hooks-nix": [],
|
||||
"libgit2": [
|
||||
"libgit2"
|
||||
],
|
||||
"nixpkgs": [
|
||||
"nixpkgs"
|
||||
],
|
||||
@@ -12,56 +32,40 @@
|
||||
"nixpkgs-regression": []
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1744030329,
|
||||
"narHash": "sha256-r+psCOW77vTSTNbxTVrYHeh6OgB0QukbnyUVDwg8s4I=",
|
||||
"lastModified": 1726787955,
|
||||
"narHash": "sha256-XFznzb8L4SdUm9u+w3DPpMWJhffuv+/6+aiVl00slns=",
|
||||
"owner": "NixOS",
|
||||
"repo": "nix",
|
||||
"rev": "a4962f73b5fc874d4b16baef47921daf349addfc",
|
||||
"rev": "a7fdef6858dd45b9d7bda7c92324c63faee7f509",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "NixOS",
|
||||
"ref": "2.28-maintenance",
|
||||
"ref": "2.24-maintenance",
|
||||
"repo": "nix",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"nix-eval-jobs": {
|
||||
"flake": false,
|
||||
"locked": {
|
||||
"lastModified": 1744018595,
|
||||
"narHash": "sha256-v5n6t49X7MOpqS9j0FtI6TWOXvxuZMmGsp2OfUK5QfA=",
|
||||
"owner": "nix-community",
|
||||
"repo": "nix-eval-jobs",
|
||||
"rev": "cba718bafe5dc1607c2b6761ecf53c641a6f3b21",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "nix-community",
|
||||
"repo": "nix-eval-jobs",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"nixpkgs": {
|
||||
"locked": {
|
||||
"lastModified": 1743987495,
|
||||
"narHash": "sha256-46T2vMZ4/AfCK0Y2OjlFzJPxmdpP8GtsuEqSSJv3oe4=",
|
||||
"lastModified": 1726688310,
|
||||
"narHash": "sha256-Xc9lEtentPCEtxc/F1e6jIZsd4MPDYv4Kugl9WtXlz0=",
|
||||
"owner": "NixOS",
|
||||
"repo": "nixpkgs",
|
||||
"rev": "db8f4fe18ce772a9c8f3adf321416981c8fe9371",
|
||||
"rev": "dbebdd67a6006bb145d98c8debf9140ac7e651d0",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "NixOS",
|
||||
"ref": "nixos-24.11-small",
|
||||
"ref": "nixos-24.05-small",
|
||||
"repo": "nixpkgs",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"root": {
|
||||
"inputs": {
|
||||
"libgit2": "libgit2",
|
||||
"nix": "nix",
|
||||
"nix-eval-jobs": "nix-eval-jobs",
|
||||
"nixpkgs": "nixpkgs"
|
||||
}
|
||||
}
|
||||
|
||||
55
flake.nix
55
flake.nix
@@ -1,27 +1,21 @@
|
||||
{
|
||||
description = "A Nix-based continuous build system";
|
||||
|
||||
inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-24.11-small";
|
||||
inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-24.05-small";
|
||||
|
||||
inputs.nix = {
|
||||
url = "github:NixOS/nix/2.28-maintenance";
|
||||
inputs.nixpkgs.follows = "nixpkgs";
|
||||
inputs.libgit2 = { url = "github:libgit2/libgit2/v1.8.1"; flake = false; };
|
||||
inputs.nix.url = "github:NixOS/nix/2.24-maintenance";
|
||||
inputs.nix.inputs.nixpkgs.follows = "nixpkgs";
|
||||
inputs.nix.inputs.libgit2.follows = "libgit2";
|
||||
|
||||
# hide nix dev tooling from our lock file
|
||||
inputs.flake-parts.follows = "";
|
||||
inputs.git-hooks-nix.follows = "";
|
||||
inputs.nixpkgs-regression.follows = "";
|
||||
inputs.nixpkgs-23-11.follows = "";
|
||||
inputs.flake-compat.follows = "";
|
||||
};
|
||||
# hide nix dev tooling from our lock file
|
||||
inputs.nix.inputs.flake-parts.follows = "";
|
||||
inputs.nix.inputs.git-hooks-nix.follows = "";
|
||||
inputs.nix.inputs.nixpkgs-regression.follows = "";
|
||||
inputs.nix.inputs.nixpkgs-23-11.follows = "";
|
||||
inputs.nix.inputs.flake-compat.follows = "";
|
||||
|
||||
inputs.nix-eval-jobs = {
|
||||
url = "github:nix-community/nix-eval-jobs";
|
||||
# We want to control the deps precisely
|
||||
flake = false;
|
||||
};
|
||||
|
||||
outputs = { self, nixpkgs, nix, nix-eval-jobs, ... }:
|
||||
outputs = { self, nixpkgs, nix, ... }:
|
||||
let
|
||||
systems = [ "x86_64-linux" "aarch64-linux" ];
|
||||
forEachSystem = nixpkgs.lib.genAttrs systems;
|
||||
@@ -30,10 +24,10 @@
|
||||
|
||||
# A Nixpkgs overlay that provides a 'hydra' package.
|
||||
overlays.default = final: prev: {
|
||||
nix-eval-jobs = final.callPackage nix-eval-jobs {};
|
||||
hydra = final.callPackage ./package.nix {
|
||||
inherit (nixpkgs.lib) fileset;
|
||||
rawSrc = self;
|
||||
nix-perl-bindings = final.nixComponents.nix-perl-bindings;
|
||||
};
|
||||
};
|
||||
|
||||
@@ -72,29 +66,12 @@
|
||||
validate-openapi = hydraJobs.tests.validate-openapi.${system};
|
||||
});
|
||||
|
||||
packages = forEachSystem (system: let
|
||||
nixComponents = {
|
||||
inherit (nix.packages.${system})
|
||||
nix-util
|
||||
nix-store
|
||||
nix-expr
|
||||
nix-fetchers
|
||||
nix-flake
|
||||
nix-main
|
||||
nix-cmd
|
||||
nix-cli
|
||||
nix-perl-bindings
|
||||
;
|
||||
};
|
||||
in {
|
||||
nix-eval-jobs = nixpkgs.legacyPackages.${system}.callPackage nix-eval-jobs {
|
||||
inherit nixComponents;
|
||||
};
|
||||
packages = forEachSystem (system: {
|
||||
hydra = nixpkgs.legacyPackages.${system}.callPackage ./package.nix {
|
||||
inherit (nixpkgs.lib) fileset;
|
||||
inherit nixComponents;
|
||||
inherit (self.packages.${system}) nix-eval-jobs;
|
||||
rawSrc = self;
|
||||
nix = nix.packages.${system}.nix;
|
||||
nix-perl-bindings = nix.hydraJobs.perlBindings.${system};
|
||||
};
|
||||
default = self.packages.${system}.hydra;
|
||||
});
|
||||
|
||||
16
meson.build
16
meson.build
@@ -8,9 +8,23 @@ project('hydra', 'cpp',
|
||||
],
|
||||
)
|
||||
|
||||
nix_util_dep = dependency('nix-util', required: true)
|
||||
nix_store_dep = dependency('nix-store', required: true)
|
||||
nix_main_dep = dependency('nix-main', required: true)
|
||||
nix_expr_dep = dependency('nix-expr', required: true)
|
||||
nix_flake_dep = dependency('nix-flake', required: true)
|
||||
nix_cmd_dep = dependency('nix-cmd', required: true)
|
||||
|
||||
# Nix need extra flags not provided in its pkg-config files.
|
||||
nix_dep = declare_dependency(
|
||||
dependencies: [
|
||||
nix_store_dep,
|
||||
nix_main_dep,
|
||||
nix_expr_dep,
|
||||
nix_flake_dep,
|
||||
nix_cmd_dep,
|
||||
],
|
||||
compile_args: ['-include', 'nix/config.h'],
|
||||
)
|
||||
|
||||
pqxx_dep = dependency('libpqxx', required: true)
|
||||
|
||||
|
||||
@@ -15,6 +15,7 @@
|
||||
systemd.services.hydra-send-stats.enable = false;
|
||||
|
||||
services.postgresql.enable = true;
|
||||
services.postgresql.package = pkgs.postgresql_12;
|
||||
|
||||
# The following is to work around the following error from hydra-server:
|
||||
# [error] Caught exception in engine "Cannot determine local time zone"
|
||||
|
||||
@@ -468,7 +468,7 @@ in
|
||||
elif [[ $compression == zstd ]]; then
|
||||
compression="zstd --rm"
|
||||
fi
|
||||
find ${baseDir}/build-logs -ignore_readdir_race -type f -name "*.drv" -mtime +3 -size +0c | xargs -r "$compression" --force --quiet
|
||||
find ${baseDir}/build-logs -type f -name "*.drv" -mtime +3 -size +0c | xargs -r "$compression" --force --quiet
|
||||
'';
|
||||
startAt = "Sun 01:45";
|
||||
};
|
||||
|
||||
@@ -27,7 +27,8 @@ in
|
||||
{
|
||||
|
||||
install = forEachSystem (system:
|
||||
(import (nixpkgs + "/nixos/lib/testing-python.nix") { inherit system; }).simpleTest {
|
||||
with import (nixpkgs + "/nixos/lib/testing-python.nix") { inherit system; };
|
||||
simpleTest {
|
||||
name = "hydra-install";
|
||||
nodes.machine = hydraServer;
|
||||
testScript =
|
||||
@@ -42,7 +43,8 @@ in
|
||||
});
|
||||
|
||||
notifications = forEachSystem (system:
|
||||
(import (nixpkgs + "/nixos/lib/testing-python.nix") { inherit system; }).simpleTest {
|
||||
with import (nixpkgs + "/nixos/lib/testing-python.nix") { inherit system; };
|
||||
simpleTest {
|
||||
name = "hydra-notifications";
|
||||
nodes.machine = {
|
||||
imports = [ hydraServer ];
|
||||
@@ -54,7 +56,7 @@ in
|
||||
'';
|
||||
services.influxdb.enable = true;
|
||||
};
|
||||
testScript = { nodes, ... }: ''
|
||||
testScript = ''
|
||||
machine.wait_for_job("hydra-init")
|
||||
|
||||
# Create an admin account and some other state.
|
||||
@@ -85,7 +87,7 @@ in
|
||||
|
||||
# Setup the project and jobset
|
||||
machine.succeed(
|
||||
"su - hydra -c 'perl -I ${nodes.machine.services.hydra-dev.package.perlDeps}/lib/perl5/site_perl ${./t/setup-notifications-jobset.pl}' >&2"
|
||||
"su - hydra -c 'perl -I ${config.services.hydra-dev.package.perlDeps}/lib/perl5/site_perl ${./t/setup-notifications-jobset.pl}' >&2"
|
||||
)
|
||||
|
||||
# Wait until hydra has build the job and
|
||||
@@ -99,10 +101,9 @@ in
|
||||
});
|
||||
|
||||
gitea = forEachSystem (system:
|
||||
let
|
||||
pkgs = nixpkgs.legacyPackages.${system};
|
||||
in
|
||||
(import (nixpkgs + "/nixos/lib/testing-python.nix") { inherit system; }).makeTest {
|
||||
let pkgs = nixpkgs.legacyPackages.${system}; in
|
||||
with import (nixpkgs + "/nixos/lib/testing-python.nix") { inherit system; };
|
||||
makeTest {
|
||||
name = "hydra-gitea";
|
||||
nodes.machine = { pkgs, ... }: {
|
||||
imports = [ hydraServer ];
|
||||
|
||||
25
package.nix
25
package.nix
@@ -8,7 +8,8 @@
|
||||
|
||||
, perlPackages
|
||||
|
||||
, nixComponents
|
||||
, nix
|
||||
, nix-perl-bindings
|
||||
, git
|
||||
|
||||
, makeWrapper
|
||||
@@ -49,7 +50,6 @@
|
||||
, xz
|
||||
, gnutar
|
||||
, gnused
|
||||
, nix-eval-jobs
|
||||
|
||||
, rpm
|
||||
, dpkg
|
||||
@@ -61,7 +61,7 @@ let
|
||||
name = "hydra-perl-deps";
|
||||
paths = lib.closePropagation
|
||||
([
|
||||
nixComponents.nix-perl-bindings
|
||||
nix-perl-bindings
|
||||
git
|
||||
] ++ (with perlPackages; [
|
||||
AuthenSASL
|
||||
@@ -89,7 +89,6 @@ let
|
||||
DateTime
|
||||
DBDPg
|
||||
DBDSQLite
|
||||
DBIxClassHelpers
|
||||
DigestSHA1
|
||||
EmailMIME
|
||||
EmailSender
|
||||
@@ -162,7 +161,7 @@ stdenv.mkDerivation (finalAttrs: {
|
||||
nukeReferences
|
||||
pkg-config
|
||||
mdbook
|
||||
nixComponents.nix-cli
|
||||
nix
|
||||
perlDeps
|
||||
perl
|
||||
unzip
|
||||
@@ -172,9 +171,7 @@ stdenv.mkDerivation (finalAttrs: {
|
||||
libpqxx
|
||||
openssl
|
||||
libxslt
|
||||
nixComponents.nix-util
|
||||
nixComponents.nix-store
|
||||
nixComponents.nix-main
|
||||
nix
|
||||
perlDeps
|
||||
perl
|
||||
boost
|
||||
@@ -193,7 +190,6 @@ stdenv.mkDerivation (finalAttrs: {
|
||||
openldap
|
||||
postgresql_13
|
||||
pixz
|
||||
nix-eval-jobs
|
||||
];
|
||||
|
||||
checkInputs = [
|
||||
@@ -201,14 +197,13 @@ stdenv.mkDerivation (finalAttrs: {
|
||||
glibcLocales
|
||||
libressl.nc
|
||||
python3
|
||||
nixComponents.nix-cli
|
||||
];
|
||||
|
||||
hydraPath = lib.makeBinPath (
|
||||
[
|
||||
subversion
|
||||
openssh
|
||||
nixComponents.nix-cli
|
||||
nix
|
||||
coreutils
|
||||
findutils
|
||||
pixz
|
||||
@@ -223,7 +218,6 @@ stdenv.mkDerivation (finalAttrs: {
|
||||
darcs
|
||||
gnused
|
||||
breezy
|
||||
nix-eval-jobs
|
||||
] ++ lib.optionals stdenv.isLinux [ rpm dpkg cdrkit ]
|
||||
);
|
||||
|
||||
@@ -238,7 +232,7 @@ stdenv.mkDerivation (finalAttrs: {
|
||||
shellHook = ''
|
||||
pushd $(git rev-parse --show-toplevel) >/dev/null
|
||||
|
||||
PATH=$(pwd)/build/src/hydra-evaluator:$(pwd)/build/src/script:$(pwd)/build/src/hydra-queue-runner:$PATH
|
||||
PATH=$(pwd)/src/hydra-evaluator:$(pwd)/src/script:$(pwd)/src/hydra-eval-jobs:$(pwd)/src/hydra-queue-runner:$PATH
|
||||
PERL5LIB=$(pwd)/src/lib:$PERL5LIB
|
||||
export HYDRA_HOME="$(pwd)/src/"
|
||||
mkdir -p .hydra-data
|
||||
@@ -269,13 +263,12 @@ stdenv.mkDerivation (finalAttrs: {
|
||||
--prefix PATH ':' $out/bin:$hydraPath \
|
||||
--set HYDRA_RELEASE ${version} \
|
||||
--set HYDRA_HOME $out/libexec/hydra \
|
||||
--set NIX_RELEASE ${nixComponents.nix-cli.name or "unknown"} \
|
||||
--set NIX_EVAL_JOBS_RELEASE ${nix-eval-jobs.name or "unknown"}
|
||||
--set NIX_RELEASE ${nix.name or "unknown"}
|
||||
done
|
||||
'';
|
||||
|
||||
dontStrip = true;
|
||||
|
||||
meta.description = "Build of Hydra on ${stdenv.system}";
|
||||
passthru = { inherit perlDeps; };
|
||||
passthru = { inherit perlDeps nix; };
|
||||
})
|
||||
|
||||
587
src/hydra-eval-jobs/hydra-eval-jobs.cc
Normal file
587
src/hydra-eval-jobs/hydra-eval-jobs.cc
Normal file
@@ -0,0 +1,587 @@
|
||||
#include <iostream>
|
||||
#include <thread>
|
||||
#include <optional>
|
||||
#include <unordered_map>
|
||||
|
||||
#include "shared.hh"
|
||||
#include "store-api.hh"
|
||||
#include "eval.hh"
|
||||
#include "eval-gc.hh"
|
||||
#include "eval-inline.hh"
|
||||
#include "eval-settings.hh"
|
||||
#include "signals.hh"
|
||||
#include "terminal.hh"
|
||||
#include "util.hh"
|
||||
#include "get-drvs.hh"
|
||||
#include "globals.hh"
|
||||
#include "common-eval-args.hh"
|
||||
#include "flake/flakeref.hh"
|
||||
#include "flake/flake.hh"
|
||||
#include "attr-path.hh"
|
||||
#include "derivations.hh"
|
||||
#include "local-fs-store.hh"
|
||||
|
||||
#include "hydra-config.hh"
|
||||
|
||||
#include <sys/types.h>
|
||||
#include <sys/wait.h>
|
||||
#include <sys/resource.h>
|
||||
|
||||
#include <nlohmann/json.hpp>
|
||||
|
||||
void check_pid_status_nonblocking(pid_t check_pid)
|
||||
{
|
||||
// Only check 'initialized' and known PID's
|
||||
if (check_pid <= 0) { return; }
|
||||
|
||||
int wstatus = 0;
|
||||
pid_t pid = waitpid(check_pid, &wstatus, WNOHANG);
|
||||
// -1 = failure, WNOHANG: 0 = no change
|
||||
if (pid <= 0) { return; }
|
||||
|
||||
std::cerr << "child process (" << pid << ") ";
|
||||
|
||||
if (WIFEXITED(wstatus)) {
|
||||
std::cerr << "exited with status=" << WEXITSTATUS(wstatus) << std::endl;
|
||||
} else if (WIFSIGNALED(wstatus)) {
|
||||
std::cerr << "killed by signal=" << WTERMSIG(wstatus) << std::endl;
|
||||
} else if (WIFSTOPPED(wstatus)) {
|
||||
std::cerr << "stopped by signal=" << WSTOPSIG(wstatus) << std::endl;
|
||||
} else if (WIFCONTINUED(wstatus)) {
|
||||
std::cerr << "continued" << std::endl;
|
||||
}
|
||||
}
|
||||
|
||||
using namespace nix;
|
||||
|
||||
static Path gcRootsDir;
|
||||
static size_t maxMemorySize;
|
||||
|
||||
struct MyArgs : MixEvalArgs, MixCommonArgs, RootArgs
|
||||
{
|
||||
Path releaseExpr;
|
||||
bool flake = false;
|
||||
bool dryRun = false;
|
||||
|
||||
MyArgs() : MixCommonArgs("hydra-eval-jobs")
|
||||
{
|
||||
addFlag({
|
||||
.longName = "gc-roots-dir",
|
||||
.description = "garbage collector roots directory",
|
||||
.labels = {"path"},
|
||||
.handler = {&gcRootsDir}
|
||||
});
|
||||
|
||||
addFlag({
|
||||
.longName = "dry-run",
|
||||
.description = "don't create store derivations",
|
||||
.handler = {&dryRun, true}
|
||||
});
|
||||
|
||||
addFlag({
|
||||
.longName = "flake",
|
||||
.description = "build a flake",
|
||||
.handler = {&flake, true}
|
||||
});
|
||||
|
||||
expectArg("expr", &releaseExpr);
|
||||
}
|
||||
};
|
||||
|
||||
static MyArgs myArgs;
|
||||
|
||||
static std::string queryMetaStrings(EvalState & state, PackageInfo & drv, const std::string & name, const std::string & subAttribute)
|
||||
{
|
||||
Strings res;
|
||||
std::function<void(Value & v)> rec;
|
||||
|
||||
rec = [&](Value & v) {
|
||||
state.forceValue(v, noPos);
|
||||
if (v.type() == nString)
|
||||
res.emplace_back(v.string_view());
|
||||
else if (v.isList())
|
||||
for (unsigned int n = 0; n < v.listSize(); ++n)
|
||||
rec(*v.listElems()[n]);
|
||||
else if (v.type() == nAttrs) {
|
||||
auto a = v.attrs()->find(state.symbols.create(subAttribute));
|
||||
if (a != v.attrs()->end())
|
||||
res.push_back(std::string(state.forceString(*a->value, a->pos, "while evaluating meta attributes")));
|
||||
}
|
||||
};
|
||||
|
||||
Value * v = drv.queryMeta(name);
|
||||
if (v) rec(*v);
|
||||
|
||||
return concatStringsSep(", ", res);
|
||||
}
|
||||
|
||||
static void worker(
|
||||
EvalState & state,
|
||||
Bindings & autoArgs,
|
||||
AutoCloseFD & to,
|
||||
AutoCloseFD & from)
|
||||
{
|
||||
Value vTop;
|
||||
|
||||
if (myArgs.flake) {
|
||||
using namespace flake;
|
||||
|
||||
auto [flakeRef, fragment, outputSpec] = parseFlakeRefWithFragmentAndExtendedOutputsSpec(fetchSettings, myArgs.releaseExpr, absPath("."));
|
||||
|
||||
auto vFlake = state.allocValue();
|
||||
|
||||
auto lockedFlake = lockFlake(
|
||||
flakeSettings,
|
||||
state,
|
||||
flakeRef,
|
||||
LockFlags {
|
||||
.updateLockFile = false,
|
||||
.useRegistries = false,
|
||||
.allowUnlocked = false,
|
||||
});
|
||||
|
||||
callFlake(state, lockedFlake, *vFlake);
|
||||
|
||||
auto vOutputs = vFlake->attrs()->get(state.symbols.create("outputs"))->value;
|
||||
state.forceValue(*vOutputs, noPos);
|
||||
|
||||
auto aHydraJobs = vOutputs->attrs()->get(state.symbols.create("hydraJobs"));
|
||||
if (!aHydraJobs)
|
||||
aHydraJobs = vOutputs->attrs()->get(state.symbols.create("checks"));
|
||||
if (!aHydraJobs)
|
||||
throw Error("flake '%s' does not provide any Hydra jobs or checks", flakeRef);
|
||||
|
||||
vTop = *aHydraJobs->value;
|
||||
|
||||
} else {
|
||||
state.evalFile(lookupFileArg(state, myArgs.releaseExpr), vTop);
|
||||
}
|
||||
|
||||
auto vRoot = state.allocValue();
|
||||
state.autoCallFunction(autoArgs, vTop, *vRoot);
|
||||
|
||||
while (true) {
|
||||
/* Wait for the master to send us a job name. */
|
||||
writeLine(to.get(), "next");
|
||||
|
||||
auto s = readLine(from.get());
|
||||
if (s == "exit") break;
|
||||
if (!hasPrefix(s, "do ")) abort();
|
||||
std::string attrPath(s, 3);
|
||||
|
||||
debug("worker process %d at '%s'", getpid(), attrPath);
|
||||
|
||||
/* Evaluate it and send info back to the master. */
|
||||
nlohmann::json reply;
|
||||
|
||||
try {
|
||||
auto vTmp = findAlongAttrPath(state, attrPath, autoArgs, *vRoot).first;
|
||||
|
||||
auto v = state.allocValue();
|
||||
state.autoCallFunction(autoArgs, *vTmp, *v);
|
||||
|
||||
if (auto drv = getDerivation(state, *v, false)) {
|
||||
|
||||
// CA derivations do not have static output paths, so we
|
||||
// have to defensively not query output paths in case we
|
||||
// encounter one.
|
||||
PackageInfo::Outputs outputs = drv->queryOutputs(
|
||||
!experimentalFeatureSettings.isEnabled(Xp::CaDerivations));
|
||||
|
||||
if (drv->querySystem() == "unknown")
|
||||
state.error<EvalError>("derivation must have a 'system' attribute").debugThrow();
|
||||
|
||||
auto drvPath = state.store->printStorePath(drv->requireDrvPath());
|
||||
|
||||
nlohmann::json job;
|
||||
|
||||
job["nixName"] = drv->queryName();
|
||||
job["system"] =drv->querySystem();
|
||||
job["drvPath"] = drvPath;
|
||||
job["description"] = drv->queryMetaString("description");
|
||||
job["license"] = queryMetaStrings(state, *drv, "license", "shortName");
|
||||
job["homepage"] = drv->queryMetaString("homepage");
|
||||
job["maintainers"] = queryMetaStrings(state, *drv, "maintainers", "email");
|
||||
job["schedulingPriority"] = drv->queryMetaInt("schedulingPriority", 100);
|
||||
job["timeout"] = drv->queryMetaInt("timeout", 36000);
|
||||
job["maxSilent"] = drv->queryMetaInt("maxSilent", 7200);
|
||||
job["isChannel"] = drv->queryMetaBool("isHydraChannel", false);
|
||||
|
||||
/* If this is an aggregate, then get its constituents. */
|
||||
auto a = v->attrs()->get(state.symbols.create("_hydraAggregate"));
|
||||
if (a && state.forceBool(*a->value, a->pos, "while evaluating the `_hydraAggregate` attribute")) {
|
||||
auto a = v->attrs()->get(state.symbols.create("constituents"));
|
||||
if (!a)
|
||||
state.error<EvalError>("derivation must have a ‘constituents’ attribute").debugThrow();
|
||||
|
||||
NixStringContext context;
|
||||
state.coerceToString(a->pos, *a->value, context, "while evaluating the `constituents` attribute", true, false);
|
||||
for (auto & c : context)
|
||||
std::visit(overloaded {
|
||||
[&](const NixStringContextElem::Built & b) {
|
||||
job["constituents"].push_back(b.drvPath->to_string(*state.store));
|
||||
},
|
||||
[&](const NixStringContextElem::Opaque & o) {
|
||||
},
|
||||
[&](const NixStringContextElem::DrvDeep & d) {
|
||||
},
|
||||
}, c.raw);
|
||||
|
||||
state.forceList(*a->value, a->pos, "while evaluating the `constituents` attribute");
|
||||
for (unsigned int n = 0; n < a->value->listSize(); ++n) {
|
||||
auto v = a->value->listElems()[n];
|
||||
state.forceValue(*v, noPos);
|
||||
if (v->type() == nString)
|
||||
job["namedConstituents"].push_back(v->string_view());
|
||||
}
|
||||
}
|
||||
|
||||
/* Register the derivation as a GC root. !!! This
|
||||
registers roots for jobs that we may have already
|
||||
done. */
|
||||
auto localStore = state.store.dynamic_pointer_cast<LocalFSStore>();
|
||||
if (gcRootsDir != "" && localStore) {
|
||||
Path root = gcRootsDir + "/" + std::string(baseNameOf(drvPath));
|
||||
if (!pathExists(root))
|
||||
localStore->addPermRoot(localStore->parseStorePath(drvPath), root);
|
||||
}
|
||||
|
||||
nlohmann::json out;
|
||||
for (auto & [outputName, optOutputPath] : outputs) {
|
||||
if (optOutputPath) {
|
||||
out[outputName] = state.store->printStorePath(*optOutputPath);
|
||||
} else {
|
||||
// See the `queryOutputs` call above; we should
|
||||
// not encounter missing output paths otherwise.
|
||||
assert(experimentalFeatureSettings.isEnabled(Xp::CaDerivations));
|
||||
out[outputName] = nullptr;
|
||||
}
|
||||
}
|
||||
job["outputs"] = std::move(out);
|
||||
reply["job"] = std::move(job);
|
||||
}
|
||||
|
||||
else if (v->type() == nAttrs) {
|
||||
auto attrs = nlohmann::json::array();
|
||||
StringSet ss;
|
||||
for (auto & i : v->attrs()->lexicographicOrder(state.symbols)) {
|
||||
std::string name(state.symbols[i->name]);
|
||||
if (name.find(' ') != std::string::npos) {
|
||||
printError("skipping job with illegal name '%s'", name);
|
||||
continue;
|
||||
}
|
||||
attrs.push_back(name);
|
||||
}
|
||||
reply["attrs"] = std::move(attrs);
|
||||
}
|
||||
|
||||
else if (v->type() == nNull)
|
||||
;
|
||||
|
||||
else state.error<TypeError>("attribute '%s' is %s, which is not supported", attrPath, showType(*v)).debugThrow();
|
||||
|
||||
} catch (EvalError & e) {
|
||||
auto msg = e.msg();
|
||||
// Transmits the error we got from the previous evaluation
|
||||
// in the JSON output.
|
||||
reply["error"] = filterANSIEscapes(msg, true);
|
||||
// Don't forget to print it into the STDERR log, this is
|
||||
// what's shown in the Hydra UI.
|
||||
printError(msg);
|
||||
}
|
||||
|
||||
writeLine(to.get(), reply.dump());
|
||||
|
||||
/* If our RSS exceeds the maximum, exit. The master will
|
||||
start a new process. */
|
||||
struct rusage r;
|
||||
getrusage(RUSAGE_SELF, &r);
|
||||
if ((size_t) r.ru_maxrss > maxMemorySize * 1024) break;
|
||||
}
|
||||
|
||||
writeLine(to.get(), "restart");
|
||||
}
|
||||
|
||||
int main(int argc, char * * argv)
|
||||
{
|
||||
/* Prevent undeclared dependencies in the evaluation via
|
||||
$NIX_PATH. */
|
||||
unsetenv("NIX_PATH");
|
||||
|
||||
return handleExceptions(argv[0], [&]() {
|
||||
|
||||
auto config = std::make_unique<HydraConfig>();
|
||||
|
||||
auto nrWorkers = config->getIntOption("evaluator_workers", 1);
|
||||
maxMemorySize = config->getIntOption("evaluator_max_memory_size", 4096);
|
||||
|
||||
initNix();
|
||||
initGC();
|
||||
|
||||
myArgs.parseCmdline(argvToStrings(argc, argv));
|
||||
|
||||
auto pureEval = config->getBoolOption("evaluator_pure_eval", myArgs.flake);
|
||||
|
||||
/* FIXME: The build hook in conjunction with import-from-derivation is causing "unexpected EOF" during eval */
|
||||
settings.builders = "";
|
||||
|
||||
/* Prevent access to paths outside of the Nix search path and
|
||||
to the environment. */
|
||||
evalSettings.restrictEval = true;
|
||||
|
||||
/* When building a flake, use pure evaluation (no access to
|
||||
'getEnv', 'currentSystem' etc. */
|
||||
evalSettings.pureEval = pureEval;
|
||||
|
||||
if (myArgs.dryRun) settings.readOnlyMode = true;
|
||||
|
||||
if (myArgs.releaseExpr == "") throw UsageError("no expression specified");
|
||||
|
||||
if (gcRootsDir == "") printMsg(lvlError, "warning: `--gc-roots-dir' not specified");
|
||||
|
||||
struct State
|
||||
{
|
||||
std::set<std::string> todo{""};
|
||||
std::set<std::string> active;
|
||||
nlohmann::json jobs;
|
||||
std::exception_ptr exc;
|
||||
};
|
||||
|
||||
std::condition_variable wakeup;
|
||||
|
||||
Sync<State> state_;
|
||||
|
||||
/* Start a handler thread per worker process. */
|
||||
auto handler = [&]()
|
||||
{
|
||||
pid_t pid = -1;
|
||||
try {
|
||||
AutoCloseFD from, to;
|
||||
|
||||
while (true) {
|
||||
|
||||
/* Start a new worker process if necessary. */
|
||||
if (pid == -1) {
|
||||
Pipe toPipe, fromPipe;
|
||||
toPipe.create();
|
||||
fromPipe.create();
|
||||
pid = startProcess(
|
||||
[&,
|
||||
to{std::make_shared<AutoCloseFD>(std::move(fromPipe.writeSide))},
|
||||
from{std::make_shared<AutoCloseFD>(std::move(toPipe.readSide))}
|
||||
]()
|
||||
{
|
||||
try {
|
||||
auto evalStore = myArgs.evalStoreUrl
|
||||
? openStore(*myArgs.evalStoreUrl)
|
||||
: openStore();
|
||||
EvalState state(myArgs.lookupPath,
|
||||
evalStore, fetchSettings, evalSettings);
|
||||
Bindings & autoArgs = *myArgs.getAutoArgs(state);
|
||||
worker(state, autoArgs, *to, *from);
|
||||
} catch (Error & e) {
|
||||
nlohmann::json err;
|
||||
auto msg = e.msg();
|
||||
err["error"] = filterANSIEscapes(msg, true);
|
||||
printError(msg);
|
||||
writeLine(to->get(), err.dump());
|
||||
// Don't forget to print it into the STDERR log, this is
|
||||
// what's shown in the Hydra UI.
|
||||
writeLine(to->get(), "restart");
|
||||
}
|
||||
},
|
||||
ProcessOptions { .allowVfork = false });
|
||||
from = std::move(fromPipe.readSide);
|
||||
to = std::move(toPipe.writeSide);
|
||||
debug("created worker process %d", pid);
|
||||
}
|
||||
|
||||
/* Check whether the existing worker process is still there. */
|
||||
auto s = readLine(from.get());
|
||||
if (s == "restart") {
|
||||
pid = -1;
|
||||
continue;
|
||||
} else if (s != "next") {
|
||||
auto json = nlohmann::json::parse(s);
|
||||
throw Error("worker error: %s", (std::string) json["error"]);
|
||||
}
|
||||
|
||||
/* Wait for a job name to become available. */
|
||||
std::string attrPath;
|
||||
|
||||
while (true) {
|
||||
checkInterrupt();
|
||||
auto state(state_.lock());
|
||||
if ((state->todo.empty() && state->active.empty()) || state->exc) {
|
||||
writeLine(to.get(), "exit");
|
||||
return;
|
||||
}
|
||||
if (!state->todo.empty()) {
|
||||
attrPath = *state->todo.begin();
|
||||
state->todo.erase(state->todo.begin());
|
||||
state->active.insert(attrPath);
|
||||
break;
|
||||
} else
|
||||
state.wait(wakeup);
|
||||
}
|
||||
|
||||
/* Tell the worker to evaluate it. */
|
||||
writeLine(to.get(), "do " + attrPath);
|
||||
|
||||
/* Wait for the response. */
|
||||
auto response = nlohmann::json::parse(readLine(from.get()));
|
||||
|
||||
/* Handle the response. */
|
||||
StringSet newAttrs;
|
||||
|
||||
if (response.find("job") != response.end()) {
|
||||
auto state(state_.lock());
|
||||
state->jobs[attrPath] = response["job"];
|
||||
}
|
||||
|
||||
if (response.find("attrs") != response.end()) {
|
||||
for (auto & i : response["attrs"]) {
|
||||
std::string path = i;
|
||||
if (path.find(".") != std::string::npos){
|
||||
path = "\"" + path + "\"";
|
||||
}
|
||||
auto s = (attrPath.empty() ? "" : attrPath + ".") + (std::string) path;
|
||||
newAttrs.insert(s);
|
||||
}
|
||||
}
|
||||
|
||||
if (response.find("error") != response.end()) {
|
||||
auto state(state_.lock());
|
||||
state->jobs[attrPath]["error"] = response["error"];
|
||||
}
|
||||
|
||||
/* Add newly discovered job names to the queue. */
|
||||
{
|
||||
auto state(state_.lock());
|
||||
state->active.erase(attrPath);
|
||||
for (auto & s : newAttrs)
|
||||
state->todo.insert(s);
|
||||
wakeup.notify_all();
|
||||
}
|
||||
}
|
||||
} catch (...) {
|
||||
check_pid_status_nonblocking(pid);
|
||||
auto state(state_.lock());
|
||||
state->exc = std::current_exception();
|
||||
wakeup.notify_all();
|
||||
}
|
||||
};
|
||||
|
||||
std::vector<std::thread> threads;
|
||||
for (size_t i = 0; i < nrWorkers; i++)
|
||||
threads.emplace_back(std::thread(handler));
|
||||
|
||||
for (auto & thread : threads)
|
||||
thread.join();
|
||||
|
||||
auto state(state_.lock());
|
||||
|
||||
if (state->exc)
|
||||
std::rethrow_exception(state->exc);
|
||||
|
||||
/* For aggregate jobs that have named consistuents
|
||||
(i.e. constituents that are a job name rather than a
|
||||
derivation), look up the referenced job and add it to the
|
||||
dependencies of the aggregate derivation. */
|
||||
auto store = openStore();
|
||||
|
||||
for (auto i = state->jobs.begin(); i != state->jobs.end(); ++i) {
|
||||
auto jobName = i.key();
|
||||
auto & job = i.value();
|
||||
|
||||
auto named = job.find("namedConstituents");
|
||||
if (named == job.end()) continue;
|
||||
|
||||
std::unordered_map<std::string, std::string> brokenJobs;
|
||||
auto getNonBrokenJobOrRecordError = [&brokenJobs, &jobName, &state](
|
||||
const std::string & childJobName) -> std::optional<nlohmann::json> {
|
||||
auto childJob = state->jobs.find(childJobName);
|
||||
if (childJob == state->jobs.end()) {
|
||||
printError("aggregate job '%s' references non-existent job '%s'", jobName, childJobName);
|
||||
brokenJobs[childJobName] = "does not exist";
|
||||
return std::nullopt;
|
||||
}
|
||||
if (childJob->find("error") != childJob->end()) {
|
||||
std::string error = (*childJob)["error"];
|
||||
printError("aggregate job '%s' references broken job '%s': %s", jobName, childJobName, error);
|
||||
brokenJobs[childJobName] = error;
|
||||
return std::nullopt;
|
||||
}
|
||||
return *childJob;
|
||||
};
|
||||
|
||||
if (myArgs.dryRun) {
|
||||
for (std::string jobName2 : *named) {
|
||||
auto job2 = getNonBrokenJobOrRecordError(jobName2);
|
||||
if (!job2) {
|
||||
continue;
|
||||
}
|
||||
std::string drvPath2 = (*job2)["drvPath"];
|
||||
job["constituents"].push_back(drvPath2);
|
||||
}
|
||||
} else {
|
||||
auto drvPath = store->parseStorePath((std::string) job["drvPath"]);
|
||||
auto drv = store->readDerivation(drvPath);
|
||||
|
||||
for (std::string jobName2 : *named) {
|
||||
auto job2 = getNonBrokenJobOrRecordError(jobName2);
|
||||
if (!job2) {
|
||||
continue;
|
||||
}
|
||||
auto drvPath2 = store->parseStorePath((std::string) (*job2)["drvPath"]);
|
||||
auto drv2 = store->readDerivation(drvPath2);
|
||||
job["constituents"].push_back(store->printStorePath(drvPath2));
|
||||
drv.inputDrvs.map[drvPath2].value = {drv2.outputs.begin()->first};
|
||||
}
|
||||
|
||||
if (brokenJobs.empty()) {
|
||||
std::string drvName(drvPath.name());
|
||||
assert(hasSuffix(drvName, drvExtension));
|
||||
drvName.resize(drvName.size() - drvExtension.size());
|
||||
|
||||
auto hashModulo = hashDerivationModulo(*store, drv, true);
|
||||
if (hashModulo.kind != DrvHash::Kind::Regular) continue;
|
||||
auto h = hashModulo.hashes.find("out");
|
||||
if (h == hashModulo.hashes.end()) continue;
|
||||
auto outPath = store->makeOutputPath("out", h->second, drvName);
|
||||
drv.env["out"] = store->printStorePath(outPath);
|
||||
drv.outputs.insert_or_assign("out", DerivationOutput::InputAddressed { .path = outPath });
|
||||
auto newDrvPath = store->printStorePath(writeDerivation(*store, drv));
|
||||
|
||||
debug("rewrote aggregate derivation %s -> %s", store->printStorePath(drvPath), newDrvPath);
|
||||
|
||||
job["drvPath"] = newDrvPath;
|
||||
job["outputs"]["out"] = store->printStorePath(outPath);
|
||||
}
|
||||
}
|
||||
|
||||
job.erase("namedConstituents");
|
||||
|
||||
/* Register the derivation as a GC root. !!! This
|
||||
registers roots for jobs that we may have already
|
||||
done. */
|
||||
auto localStore = store.dynamic_pointer_cast<LocalFSStore>();
|
||||
if (gcRootsDir != "" && localStore) {
|
||||
auto drvPath = job["drvPath"].get<std::string>();
|
||||
Path root = gcRootsDir + "/" + std::string(baseNameOf(drvPath));
|
||||
if (!pathExists(root))
|
||||
localStore->addPermRoot(localStore->parseStorePath(drvPath), root);
|
||||
}
|
||||
|
||||
if (!brokenJobs.empty()) {
|
||||
std::stringstream ss;
|
||||
for (const auto& [jobName, error] : brokenJobs) {
|
||||
ss << jobName << ": " << error << "\n";
|
||||
}
|
||||
job["error"] = ss.str();
|
||||
}
|
||||
}
|
||||
|
||||
std::cout << state->jobs.dump(2) << "\n";
|
||||
});
|
||||
}
|
||||
8
src/hydra-eval-jobs/meson.build
Normal file
8
src/hydra-eval-jobs/meson.build
Normal file
@@ -0,0 +1,8 @@
|
||||
hydra_eval_jobs = executable('hydra-eval-jobs',
|
||||
'hydra-eval-jobs.cc',
|
||||
dependencies: [
|
||||
libhydra_dep,
|
||||
nix_dep,
|
||||
],
|
||||
install: true,
|
||||
)
|
||||
@@ -1,8 +1,8 @@
|
||||
#include "db.hh"
|
||||
#include "hydra-config.hh"
|
||||
#include <nix/util/pool.hh>
|
||||
#include <nix/main/shared.hh>
|
||||
#include <nix/util/signals.hh>
|
||||
#include "pool.hh"
|
||||
#include "shared.hh"
|
||||
#include "signals.hh"
|
||||
|
||||
#include <algorithm>
|
||||
#include <thread>
|
||||
|
||||
@@ -2,8 +2,7 @@ hydra_evaluator = executable('hydra-evaluator',
|
||||
'hydra-evaluator.cc',
|
||||
dependencies: [
|
||||
libhydra_dep,
|
||||
nix_util_dep,
|
||||
nix_main_dep,
|
||||
nix_dep,
|
||||
pqxx_dep,
|
||||
],
|
||||
install: true,
|
||||
|
||||
@@ -5,40 +5,44 @@
|
||||
#include <sys/stat.h>
|
||||
#include <fcntl.h>
|
||||
|
||||
#include <nix/store/build-result.hh>
|
||||
#include <nix/store/path.hh>
|
||||
#include <nix/store/legacy-ssh-store.hh>
|
||||
#include <nix/store/serve-protocol.hh>
|
||||
#include <nix/store/serve-protocol-impl.hh>
|
||||
#include "build-result.hh"
|
||||
#include "path.hh"
|
||||
#include "serve-protocol.hh"
|
||||
#include "serve-protocol-impl.hh"
|
||||
#include "state.hh"
|
||||
#include <nix/util/current-process.hh>
|
||||
#include <nix/util/processes.hh>
|
||||
#include <nix/util/util.hh>
|
||||
#include <nix/store/serve-protocol.hh>
|
||||
#include <nix/store/serve-protocol-impl.hh>
|
||||
#include <nix/store/ssh.hh>
|
||||
#include <nix/util/finally.hh>
|
||||
#include <nix/util/url.hh>
|
||||
#include "current-process.hh"
|
||||
#include "processes.hh"
|
||||
#include "util.hh"
|
||||
#include "serve-protocol.hh"
|
||||
#include "serve-protocol-impl.hh"
|
||||
#include "ssh.hh"
|
||||
#include "finally.hh"
|
||||
#include "url.hh"
|
||||
|
||||
using namespace nix;
|
||||
|
||||
bool ::Machine::isLocalhost() const
|
||||
{
|
||||
return storeUri.params.empty() && std::visit(overloaded {
|
||||
[](const StoreReference::Auto &) {
|
||||
return true;
|
||||
},
|
||||
[](const StoreReference::Specified & s) {
|
||||
return
|
||||
(s.scheme == "local" || s.scheme == "unix") ||
|
||||
((s.scheme == "ssh" || s.scheme == "ssh-ng") &&
|
||||
s.authority == "localhost");
|
||||
},
|
||||
}, storeUri.variant);
|
||||
}
|
||||
|
||||
namespace nix::build_remote {
|
||||
|
||||
static Strings extraStoreArgs(std::string & machine)
|
||||
{
|
||||
Strings result;
|
||||
try {
|
||||
auto parsed = parseURL(machine);
|
||||
if (parsed.scheme != "ssh") {
|
||||
throw SysError("Currently, only (legacy-)ssh stores are supported!");
|
||||
}
|
||||
machine = parsed.authority.value_or("");
|
||||
auto remoteStore = parsed.query.find("remote-store");
|
||||
if (remoteStore != parsed.query.end()) {
|
||||
result = {"--store", shellEscape(remoteStore->second)};
|
||||
}
|
||||
} catch (BadURL &) {
|
||||
// We just try to continue with `machine->sshName` here for backwards compat.
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
static std::unique_ptr<SSHMaster::Connection> openConnection(
|
||||
::Machine::ptr machine, SSHMaster & master)
|
||||
{
|
||||
@@ -47,11 +51,7 @@ static std::unique_ptr<SSHMaster::Connection> openConnection(
|
||||
command.push_back("--builders");
|
||||
command.push_back("");
|
||||
} else {
|
||||
auto remoteStore = machine->storeUri.params.find("remote-store");
|
||||
if (remoteStore != machine->storeUri.params.end()) {
|
||||
command.push_back("--store");
|
||||
command.push_back(shellEscape(remoteStore->second));
|
||||
}
|
||||
command.splice(command.end(), extraStoreArgs(machine->sshName));
|
||||
}
|
||||
|
||||
auto ret = master.startCommand(std::move(command), {
|
||||
@@ -198,7 +198,7 @@ static BasicDerivation sendInputs(
|
||||
MaintainCount<counter> mc2(nrStepsCopyingTo);
|
||||
|
||||
printMsg(lvlDebug, "sending closure of ‘%s’ to ‘%s’",
|
||||
localStore.printStorePath(step.drvPath), conn.machine->storeUri.render());
|
||||
localStore.printStorePath(step.drvPath), conn.machine->sshName);
|
||||
|
||||
auto now1 = std::chrono::steady_clock::now();
|
||||
|
||||
@@ -278,6 +278,32 @@ static BuildResult performBuild(
|
||||
return result;
|
||||
}
|
||||
|
||||
static std::map<StorePath, UnkeyedValidPathInfo> queryPathInfos(
|
||||
::Machine::Connection & conn,
|
||||
Store & localStore,
|
||||
StorePathSet & outputs,
|
||||
size_t & totalNarSize
|
||||
)
|
||||
{
|
||||
|
||||
/* Get info about each output path. */
|
||||
std::map<StorePath, UnkeyedValidPathInfo> infos;
|
||||
conn.to << ServeProto::Command::QueryPathInfos;
|
||||
ServeProto::write(localStore, conn, outputs);
|
||||
conn.to.flush();
|
||||
while (true) {
|
||||
auto storePathS = readString(conn.from);
|
||||
if (storePathS == "") break;
|
||||
|
||||
auto storePath = localStore.parseStorePath(storePathS);
|
||||
auto info = ServeProto::Serialise<UnkeyedValidPathInfo>::read(localStore, conn);
|
||||
totalNarSize += info.narSize;
|
||||
infos.insert_or_assign(std::move(storePath), std::move(info));
|
||||
}
|
||||
|
||||
return infos;
|
||||
}
|
||||
|
||||
static void copyPathFromRemote(
|
||||
::Machine::Connection & conn,
|
||||
NarMemberDatas & narMembers,
|
||||
@@ -386,19 +412,8 @@ void RemoteResult::updateWithBuildResult(const nix::BuildResult & buildResult)
|
||||
|
||||
}
|
||||
|
||||
/* Utility guard object to auto-release a semaphore on destruction. */
|
||||
template <typename T>
|
||||
class SemaphoreReleaser {
|
||||
public:
|
||||
SemaphoreReleaser(T* s) : sem(s) {}
|
||||
~SemaphoreReleaser() { sem->release(); }
|
||||
|
||||
private:
|
||||
T* sem;
|
||||
};
|
||||
|
||||
void State::buildRemote(ref<Store> destStore,
|
||||
std::unique_ptr<MachineReservation> reservation,
|
||||
::Machine::ptr machine, Step::ptr step,
|
||||
const ServeProto::BuildOptions & buildOptions,
|
||||
RemoteResult & result, std::shared_ptr<ActiveStep> activeStep,
|
||||
@@ -415,22 +430,14 @@ void State::buildRemote(ref<Store> destStore,
|
||||
|
||||
updateStep(ssConnecting);
|
||||
|
||||
auto storeRef = machine->completeStoreReference();
|
||||
|
||||
auto * pSpecified = std::get_if<StoreReference::Specified>(&storeRef.variant);
|
||||
if (!pSpecified || pSpecified->scheme != "ssh") {
|
||||
throw Error("Currently, only (legacy-)ssh stores are supported!");
|
||||
}
|
||||
|
||||
LegacySSHStoreConfig storeConfig {
|
||||
pSpecified->scheme,
|
||||
pSpecified->authority,
|
||||
storeRef.params
|
||||
};
|
||||
|
||||
auto master = storeConfig.createSSHMaster(
|
||||
SSHMaster master {
|
||||
machine->sshName,
|
||||
machine->sshKey,
|
||||
machine->sshPublicHostKey,
|
||||
false, // no SSH master yet
|
||||
logFD.get());
|
||||
false, // no compression yet
|
||||
logFD.get(),
|
||||
};
|
||||
|
||||
// FIXME: rewrite to use Store.
|
||||
auto child = build_remote::openConnection(machine, master);
|
||||
@@ -475,13 +482,19 @@ void State::buildRemote(ref<Store> destStore,
|
||||
conn.to,
|
||||
conn.from,
|
||||
our_version,
|
||||
machine->storeUri.render());
|
||||
machine->sshName);
|
||||
} catch (EndOfFile & e) {
|
||||
child->sshPid.wait();
|
||||
std::string s = chomp(readFile(result.logFile));
|
||||
throw Error("cannot connect to ‘%1%’: %2%", machine->storeUri.render(), s);
|
||||
throw Error("cannot connect to ‘%1%’: %2%", machine->sshName, s);
|
||||
}
|
||||
|
||||
// Do not attempt to speak a newer version of the protocol.
|
||||
//
|
||||
// Per https://github.com/NixOS/nix/issues/9584 should be handled as
|
||||
// part of `handshake` in upstream nix.
|
||||
conn.remoteVersion = std::min(conn.remoteVersion, our_version);
|
||||
|
||||
{
|
||||
auto info(machine->state->connectInfo.lock());
|
||||
info->consecutiveFailures = 0;
|
||||
@@ -510,7 +523,7 @@ void State::buildRemote(ref<Store> destStore,
|
||||
/* Do the build. */
|
||||
printMsg(lvlDebug, "building ‘%s’ on ‘%s’",
|
||||
localStore->printStorePath(step->drvPath),
|
||||
machine->storeUri.render());
|
||||
machine->sshName);
|
||||
|
||||
updateStep(ssBuilding);
|
||||
|
||||
@@ -533,28 +546,11 @@ void State::buildRemote(ref<Store> destStore,
|
||||
get a build log. */
|
||||
if (result.isCached) {
|
||||
printMsg(lvlInfo, "outputs of ‘%s’ substituted or already valid on ‘%s’",
|
||||
localStore->printStorePath(step->drvPath), machine->storeUri.render());
|
||||
localStore->printStorePath(step->drvPath), machine->sshName);
|
||||
unlink(result.logFile.c_str());
|
||||
result.logFile = "";
|
||||
}
|
||||
|
||||
/* Throttle CPU-bound work. Opportunistically skip updating the current
|
||||
* step, since this requires a DB roundtrip. */
|
||||
if (!localWorkThrottler.try_acquire()) {
|
||||
MaintainCount<counter> mc(nrStepsWaitingForDownloadSlot);
|
||||
updateStep(ssWaitingForLocalSlot);
|
||||
localWorkThrottler.acquire();
|
||||
}
|
||||
SemaphoreReleaser releaser(&localWorkThrottler);
|
||||
|
||||
/* Once we've started copying outputs, release the machine reservation
|
||||
* so further builds can happen. We do not release the machine earlier
|
||||
* to avoid situations where the queue runner is bottlenecked on
|
||||
* copying outputs and we end up building too many things that we
|
||||
* haven't been able to allow copy slots for. */
|
||||
reservation.reset();
|
||||
wakeDispatcher();
|
||||
|
||||
StorePathSet outputs;
|
||||
for (auto & [_, realisation] : buildResult.builtOutputs)
|
||||
outputs.insert(realisation.outPath);
|
||||
@@ -567,10 +563,8 @@ void State::buildRemote(ref<Store> destStore,
|
||||
|
||||
auto now1 = std::chrono::steady_clock::now();
|
||||
|
||||
auto infos = conn.queryPathInfos(*localStore, outputs);
|
||||
|
||||
size_t totalNarSize = 0;
|
||||
for (auto & [_, info] : infos) totalNarSize += info.narSize;
|
||||
auto infos = build_remote::queryPathInfos(conn, *localStore, outputs, totalNarSize);
|
||||
|
||||
if (totalNarSize > maxOutputSize) {
|
||||
result.stepStatus = bsNarSizeLimitExceeded;
|
||||
@@ -579,7 +573,7 @@ void State::buildRemote(ref<Store> destStore,
|
||||
|
||||
/* Copy each path. */
|
||||
printMsg(lvlDebug, "copying outputs of ‘%s’ from ‘%s’ (%d bytes)",
|
||||
localStore->printStorePath(step->drvPath), machine->storeUri.render(), totalNarSize);
|
||||
localStore->printStorePath(step->drvPath), machine->sshName, totalNarSize);
|
||||
|
||||
build_remote::copyPathsFromRemote(conn, narMembers, *localStore, *destStore, infos);
|
||||
auto now2 = std::chrono::steady_clock::now();
|
||||
@@ -618,7 +612,7 @@ void State::buildRemote(ref<Store> destStore,
|
||||
info->consecutiveFailures = std::min(info->consecutiveFailures + 1, (unsigned int) 4);
|
||||
info->lastFailure = now;
|
||||
int delta = retryInterval * std::pow(retryBackoff, info->consecutiveFailures - 1) + (rand() % 30);
|
||||
printMsg(lvlInfo, "will disable machine ‘%1%’ for %2%s", machine->storeUri.render(), delta);
|
||||
printMsg(lvlInfo, "will disable machine ‘%1%’ for %2%s", machine->sshName, delta);
|
||||
info->disabledUntil = now + std::chrono::seconds(delta);
|
||||
}
|
||||
throw;
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
#include "hydra-build-result.hh"
|
||||
#include <nix/store/store-api.hh>
|
||||
#include <nix/util/util.hh>
|
||||
#include <nix/util/source-accessor.hh>
|
||||
#include "store-api.hh"
|
||||
#include "util.hh"
|
||||
#include "source-accessor.hh"
|
||||
|
||||
#include <regex>
|
||||
|
||||
|
||||
@@ -2,8 +2,8 @@
|
||||
|
||||
#include "state.hh"
|
||||
#include "hydra-build-result.hh"
|
||||
#include <nix/util/finally.hh>
|
||||
#include <nix/store/binary-cache-store.hh>
|
||||
#include "finally.hh"
|
||||
#include "binary-cache-store.hh"
|
||||
|
||||
using namespace nix;
|
||||
|
||||
@@ -16,7 +16,7 @@ void setThreadName(const std::string & name)
|
||||
}
|
||||
|
||||
|
||||
void State::builder(std::unique_ptr<MachineReservation> reservation)
|
||||
void State::builder(MachineReservation::ptr reservation)
|
||||
{
|
||||
setThreadName("bld~" + std::string(reservation->step->drvPath.to_string()));
|
||||
|
||||
@@ -35,20 +35,22 @@ void State::builder(std::unique_ptr<MachineReservation> reservation)
|
||||
activeSteps_.lock()->erase(activeStep);
|
||||
});
|
||||
|
||||
std::string machine = reservation->machine->storeUri.render();
|
||||
|
||||
try {
|
||||
auto destStore = getDestStore();
|
||||
// Might release the reservation.
|
||||
res = doBuildStep(destStore, std::move(reservation), activeStep);
|
||||
res = doBuildStep(destStore, reservation, activeStep);
|
||||
} catch (std::exception & e) {
|
||||
printMsg(lvlError, "uncaught exception building ‘%s’ on ‘%s’: %s",
|
||||
localStore->printStorePath(activeStep->step->drvPath),
|
||||
machine,
|
||||
localStore->printStorePath(reservation->step->drvPath),
|
||||
reservation->machine->sshName,
|
||||
e.what());
|
||||
}
|
||||
}
|
||||
|
||||
/* Release the machine and wake up the dispatcher. */
|
||||
assert(reservation.unique());
|
||||
reservation = 0;
|
||||
wakeDispatcher();
|
||||
|
||||
/* If there was a temporary failure, retry the step after an
|
||||
exponentially increasing interval. */
|
||||
Step::ptr step = wstep.lock();
|
||||
@@ -70,11 +72,11 @@ void State::builder(std::unique_ptr<MachineReservation> reservation)
|
||||
|
||||
|
||||
State::StepResult State::doBuildStep(nix::ref<Store> destStore,
|
||||
std::unique_ptr<MachineReservation> reservation,
|
||||
MachineReservation::ptr reservation,
|
||||
std::shared_ptr<ActiveStep> activeStep)
|
||||
{
|
||||
auto step(reservation->step);
|
||||
auto machine(reservation->machine);
|
||||
auto & step(reservation->step);
|
||||
auto & machine(reservation->machine);
|
||||
|
||||
{
|
||||
auto step_(step->state.lock());
|
||||
@@ -148,7 +150,7 @@ State::StepResult State::doBuildStep(nix::ref<Store> destStore,
|
||||
buildOptions.buildTimeout = build->buildTimeout;
|
||||
|
||||
printInfo("performing step ‘%s’ %d times on ‘%s’ (needed by build %d and %d others)",
|
||||
localStore->printStorePath(step->drvPath), buildOptions.nrRepeats + 1, machine->storeUri.render(), buildId, (dependents.size() - 1));
|
||||
localStore->printStorePath(step->drvPath), buildOptions.nrRepeats + 1, machine->sshName, buildId, (dependents.size() - 1));
|
||||
}
|
||||
|
||||
if (!buildOneDone)
|
||||
@@ -176,7 +178,7 @@ State::StepResult State::doBuildStep(nix::ref<Store> destStore,
|
||||
unlink(result.logFile.c_str());
|
||||
}
|
||||
} catch (...) {
|
||||
ignoreExceptionInDestructor();
|
||||
ignoreException();
|
||||
}
|
||||
}
|
||||
});
|
||||
@@ -194,7 +196,7 @@ State::StepResult State::doBuildStep(nix::ref<Store> destStore,
|
||||
{
|
||||
auto mc = startDbUpdate();
|
||||
pqxx::work txn(*conn);
|
||||
stepNr = createBuildStep(txn, result.startTime, buildId, step, machine->storeUri.render(), bsBusy);
|
||||
stepNr = createBuildStep(txn, result.startTime, buildId, step, machine->sshName, bsBusy);
|
||||
txn.commit();
|
||||
}
|
||||
|
||||
@@ -209,7 +211,7 @@ State::StepResult State::doBuildStep(nix::ref<Store> destStore,
|
||||
|
||||
try {
|
||||
/* FIXME: referring builds may have conflicting timeouts. */
|
||||
buildRemote(destStore, std::move(reservation), machine, step, buildOptions, result, activeStep, updateStep, narMembers);
|
||||
buildRemote(destStore, machine, step, buildOptions, result, activeStep, updateStep, narMembers);
|
||||
} catch (Error & e) {
|
||||
if (activeStep->state_.lock()->cancelled) {
|
||||
printInfo("marking step %d of build %d as cancelled", stepNr, buildId);
|
||||
@@ -251,7 +253,7 @@ State::StepResult State::doBuildStep(nix::ref<Store> destStore,
|
||||
/* Finish the step in the database. */
|
||||
if (stepNr) {
|
||||
pqxx::work txn(*conn);
|
||||
finishBuildStep(txn, result, buildId, stepNr, machine->storeUri.render());
|
||||
finishBuildStep(txn, result, buildId, stepNr, machine->sshName);
|
||||
txn.commit();
|
||||
}
|
||||
|
||||
@@ -259,7 +261,7 @@ State::StepResult State::doBuildStep(nix::ref<Store> destStore,
|
||||
issue). Retry a number of times. */
|
||||
if (result.canRetry) {
|
||||
printMsg(lvlError, "possibly transient failure building ‘%s’ on ‘%s’: %s",
|
||||
localStore->printStorePath(step->drvPath), machine->storeUri.render(), result.errorMsg);
|
||||
localStore->printStorePath(step->drvPath), machine->sshName, result.errorMsg);
|
||||
assert(stepNr);
|
||||
bool retry;
|
||||
{
|
||||
@@ -450,7 +452,7 @@ void State::failStep(
|
||||
build->finishedInDB)
|
||||
continue;
|
||||
createBuildStep(txn,
|
||||
0, build->id, step, machine ? machine->storeUri.render() : "",
|
||||
0, build->id, step, machine ? machine->sshName : "",
|
||||
result.stepStatus, result.errorMsg, buildId == build->id ? 0 : buildId);
|
||||
}
|
||||
|
||||
|
||||
@@ -40,15 +40,13 @@ void State::dispatcher()
|
||||
printMsg(lvlDebug, "dispatcher woken up");
|
||||
nrDispatcherWakeups++;
|
||||
|
||||
auto t_before_work = std::chrono::steady_clock::now();
|
||||
auto now1 = std::chrono::steady_clock::now();
|
||||
|
||||
auto sleepUntil = doDispatch();
|
||||
|
||||
auto t_after_work = std::chrono::steady_clock::now();
|
||||
auto now2 = std::chrono::steady_clock::now();
|
||||
|
||||
prom.dispatcher_time_spent_running.Increment(
|
||||
std::chrono::duration_cast<std::chrono::microseconds>(t_after_work - t_before_work).count());
|
||||
dispatchTimeMs += std::chrono::duration_cast<std::chrono::milliseconds>(t_after_work - t_before_work).count();
|
||||
dispatchTimeMs += std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count();
|
||||
|
||||
/* Sleep until we're woken up (either because a runnable build
|
||||
is added, or because a build finishes). */
|
||||
@@ -62,10 +60,6 @@ void State::dispatcher()
|
||||
*dispatcherWakeup_ = false;
|
||||
}
|
||||
|
||||
auto t_after_sleep = std::chrono::steady_clock::now();
|
||||
prom.dispatcher_time_spent_waiting.Increment(
|
||||
std::chrono::duration_cast<std::chrono::microseconds>(t_after_sleep - t_after_work).count());
|
||||
|
||||
} catch (std::exception & e) {
|
||||
printError("dispatcher: %s", e.what());
|
||||
sleep(1);
|
||||
@@ -262,7 +256,7 @@ system_time State::doDispatch()
|
||||
/* Can this machine do this step? */
|
||||
if (!mi.machine->supportsStep(step)) {
|
||||
debug("machine '%s' does not support step '%s' (system type '%s')",
|
||||
mi.machine->storeUri.render(), localStore->printStorePath(step->drvPath), step->drv->platform);
|
||||
mi.machine->sshName, localStore->printStorePath(step->drvPath), step->drv->platform);
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -288,7 +282,7 @@ system_time State::doDispatch()
|
||||
/* Make a slot reservation and start a thread to
|
||||
do the build. */
|
||||
auto builderThread = std::thread(&State::builder, this,
|
||||
std::make_unique<MachineReservation>(*this, step, mi.machine));
|
||||
std::make_shared<MachineReservation>(*this, step, mi.machine));
|
||||
builderThread.detach(); // FIXME?
|
||||
|
||||
keepGoing = true;
|
||||
|
||||
@@ -2,9 +2,9 @@
|
||||
|
||||
#include <memory>
|
||||
|
||||
#include <nix/util/hash.hh>
|
||||
#include <nix/store/derivations.hh>
|
||||
#include <nix/store/store-api.hh>
|
||||
#include "hash.hh"
|
||||
#include "derivations.hh"
|
||||
#include "store-api.hh"
|
||||
#include "nar-extractor.hh"
|
||||
|
||||
struct BuildProduct
|
||||
|
||||
@@ -11,16 +11,16 @@
|
||||
|
||||
#include <nlohmann/json.hpp>
|
||||
|
||||
#include <nix/util/signals.hh>
|
||||
#include "signals.hh"
|
||||
#include "state.hh"
|
||||
#include "hydra-build-result.hh"
|
||||
#include <nix/store/store-api.hh>
|
||||
#include <nix/store/remote-store.hh>
|
||||
#include "store-api.hh"
|
||||
#include "remote-store.hh"
|
||||
|
||||
#include <nix/store/globals.hh>
|
||||
#include "globals.hh"
|
||||
#include "hydra-config.hh"
|
||||
#include <nix/store/s3-binary-cache-store.hh>
|
||||
#include <nix/main/shared.hh>
|
||||
#include "s3-binary-cache-store.hh"
|
||||
#include "shared.hh"
|
||||
|
||||
using namespace nix;
|
||||
using nlohmann::json;
|
||||
@@ -70,59 +70,10 @@ State::PromMetrics::PromMetrics()
|
||||
.Register(*registry)
|
||||
.Add({})
|
||||
)
|
||||
, dispatcher_time_spent_running(
|
||||
prometheus::BuildCounter()
|
||||
.Name("hydraqueuerunner_dispatcher_time_spent_running")
|
||||
.Help("Time (in micros) spent running the dispatcher")
|
||||
.Register(*registry)
|
||||
.Add({})
|
||||
)
|
||||
, dispatcher_time_spent_waiting(
|
||||
prometheus::BuildCounter()
|
||||
.Name("hydraqueuerunner_dispatcher_time_spent_waiting")
|
||||
.Help("Time (in micros) spent waiting for the dispatcher to obtain work")
|
||||
.Register(*registry)
|
||||
.Add({})
|
||||
)
|
||||
, queue_monitor_time_spent_running(
|
||||
prometheus::BuildCounter()
|
||||
.Name("hydraqueuerunner_queue_monitor_time_spent_running")
|
||||
.Help("Time (in micros) spent running the queue monitor")
|
||||
.Register(*registry)
|
||||
.Add({})
|
||||
)
|
||||
, queue_monitor_time_spent_waiting(
|
||||
prometheus::BuildCounter()
|
||||
.Name("hydraqueuerunner_queue_monitor_time_spent_waiting")
|
||||
.Help("Time (in micros) spent waiting for the queue monitor to obtain work")
|
||||
.Register(*registry)
|
||||
.Add({})
|
||||
)
|
||||
, dispatcher_time_spent_running(
|
||||
prometheus::BuildCounter()
|
||||
.Name("hydraqueuerunner_dispatcher_time_spent_running")
|
||||
.Help("Time (in micros) spent running the dispatcher")
|
||||
.Register(*registry)
|
||||
.Add({})
|
||||
)
|
||||
, dispatcher_time_spent_waiting(
|
||||
prometheus::BuildCounter()
|
||||
.Name("hydraqueuerunner_dispatcher_time_spent_waiting")
|
||||
.Help("Time (in micros) spent waiting for the dispatcher to obtain work")
|
||||
.Register(*registry)
|
||||
.Add({})
|
||||
)
|
||||
, queue_monitor_time_spent_running(
|
||||
prometheus::BuildCounter()
|
||||
.Name("hydraqueuerunner_queue_monitor_time_spent_running")
|
||||
.Help("Time (in micros) spent running the queue monitor")
|
||||
.Register(*registry)
|
||||
.Add({})
|
||||
)
|
||||
, queue_monitor_time_spent_waiting(
|
||||
prometheus::BuildCounter()
|
||||
.Name("hydraqueuerunner_queue_monitor_time_spent_waiting")
|
||||
.Help("Time (in micros) spent waiting for the queue monitor to obtain work")
|
||||
, queue_max_id(
|
||||
prometheus::BuildGauge()
|
||||
.Name("hydraqueuerunner_queue_max_build_id_info")
|
||||
.Help("Maximum build record ID in the queue")
|
||||
.Register(*registry)
|
||||
.Add({})
|
||||
)
|
||||
@@ -134,7 +85,6 @@ State::State(std::optional<std::string> metricsAddrOpt)
|
||||
: config(std::make_unique<HydraConfig>())
|
||||
, maxUnsupportedTime(config->getIntOption("max_unsupported_time", 0))
|
||||
, dbPool(config->getIntOption("max_db_connections", 128))
|
||||
, localWorkThrottler(config->getIntOption("max_local_worker_threads", std::min(maxSupportedLocalWorkers, std::max(4u, std::thread::hardware_concurrency()) - 2)))
|
||||
, maxOutputSize(config->getIntOption("max_output_size", 2ULL << 30))
|
||||
, maxLogSize(config->getIntOption("max_log_size", 64ULL << 20))
|
||||
, uploadLogsToBinaryCache(config->getBoolOption("upload_logs_to_binary_cache", false))
|
||||
@@ -185,26 +135,65 @@ void State::parseMachines(const std::string & contents)
|
||||
oldMachines = *machines_;
|
||||
}
|
||||
|
||||
for (auto && machine_ : nix::Machine::parseConfig({}, contents)) {
|
||||
auto machine = std::make_shared<::Machine>(std::move(machine_));
|
||||
for (auto line : tokenizeString<Strings>(contents, "\n")) {
|
||||
line = trim(std::string(line, 0, line.find('#')));
|
||||
auto tokens = tokenizeString<std::vector<std::string>>(line);
|
||||
if (tokens.size() < 3) continue;
|
||||
tokens.resize(8);
|
||||
|
||||
if (tokens[5] == "-") tokens[5] = "";
|
||||
auto supportedFeatures = tokenizeString<StringSet>(tokens[5], ",");
|
||||
|
||||
if (tokens[6] == "-") tokens[6] = "";
|
||||
auto mandatoryFeatures = tokenizeString<StringSet>(tokens[6], ",");
|
||||
|
||||
for (auto & f : mandatoryFeatures)
|
||||
supportedFeatures.insert(f);
|
||||
|
||||
using MaxJobs = std::remove_const<decltype(nix::Machine::maxJobs)>::type;
|
||||
|
||||
auto machine = std::make_shared<::Machine>(nix::Machine {
|
||||
// `storeUri`, not yet used
|
||||
"",
|
||||
// `systemTypes`
|
||||
tokenizeString<StringSet>(tokens[1], ","),
|
||||
// `sshKey`
|
||||
tokens[2] == "-" ? "" : tokens[2],
|
||||
// `maxJobs`
|
||||
tokens[3] != ""
|
||||
? string2Int<MaxJobs>(tokens[3]).value()
|
||||
: 1,
|
||||
// `speedFactor`
|
||||
std::stof(tokens[4].c_str()),
|
||||
// `supportedFeatures`
|
||||
std::move(supportedFeatures),
|
||||
// `mandatoryFeatures`
|
||||
std::move(mandatoryFeatures),
|
||||
// `sshPublicHostKey`
|
||||
tokens[7] != "" && tokens[7] != "-"
|
||||
? tokens[7]
|
||||
: "",
|
||||
});
|
||||
|
||||
machine->sshName = tokens[0];
|
||||
|
||||
/* Re-use the State object of the previous machine with the
|
||||
same name. */
|
||||
auto i = oldMachines.find(machine->storeUri.variant);
|
||||
auto i = oldMachines.find(machine->sshName);
|
||||
if (i == oldMachines.end())
|
||||
printMsg(lvlChatty, "adding new machine ‘%1%’", machine->storeUri.render());
|
||||
printMsg(lvlChatty, "adding new machine ‘%1%’", machine->sshName);
|
||||
else
|
||||
printMsg(lvlChatty, "updating machine ‘%1%’", machine->storeUri.render());
|
||||
printMsg(lvlChatty, "updating machine ‘%1%’", machine->sshName);
|
||||
machine->state = i == oldMachines.end()
|
||||
? std::make_shared<::Machine::State>()
|
||||
: i->second->state;
|
||||
newMachines[machine->storeUri.variant] = machine;
|
||||
newMachines[machine->sshName] = machine;
|
||||
}
|
||||
|
||||
for (auto & m : oldMachines)
|
||||
if (newMachines.find(m.first) == newMachines.end()) {
|
||||
if (m.second->enabled)
|
||||
printInfo("removing machine ‘%1%’", m.second->storeUri.render());
|
||||
printInfo("removing machine ‘%1%’", m.first);
|
||||
/* Add a disabled ::Machine object to make sure stats are
|
||||
maintained. */
|
||||
auto machine = std::make_shared<::Machine>(*(m.second));
|
||||
@@ -601,7 +590,6 @@ void State::dumpStatus(Connection & conn)
|
||||
{"nrActiveSteps", activeSteps_.lock()->size()},
|
||||
{"nrStepsBuilding", nrStepsBuilding.load()},
|
||||
{"nrStepsCopyingTo", nrStepsCopyingTo.load()},
|
||||
{"nrStepsWaitingForDownloadSlot", nrStepsWaitingForDownloadSlot.load()},
|
||||
{"nrStepsCopyingFrom", nrStepsCopyingFrom.load()},
|
||||
{"nrStepsWaiting", nrStepsWaiting.load()},
|
||||
{"nrUnsupportedSteps", nrUnsupportedSteps.load()},
|
||||
@@ -643,7 +631,6 @@ void State::dumpStatus(Connection & conn)
|
||||
}
|
||||
|
||||
{
|
||||
auto machines_json = json::object();
|
||||
auto machines_(machines.lock());
|
||||
for (auto & i : *machines_) {
|
||||
auto & m(i.second);
|
||||
@@ -670,9 +657,8 @@ void State::dumpStatus(Connection & conn)
|
||||
machine["avgStepTime"] = (float) s->totalStepTime / s->nrStepsDone;
|
||||
machine["avgStepBuildTime"] = (float) s->totalStepBuildTime / s->nrStepsDone;
|
||||
}
|
||||
machines_json[m->storeUri.render()] = machine;
|
||||
statusJson["machines"][m->sshName] = machine;
|
||||
}
|
||||
statusJson["machines"] = machines_json;
|
||||
}
|
||||
|
||||
{
|
||||
@@ -731,7 +717,6 @@ void State::dumpStatus(Connection & conn)
|
||||
: 0.0},
|
||||
};
|
||||
|
||||
#if NIX_WITH_S3_SUPPORT
|
||||
auto s3Store = dynamic_cast<S3BinaryCacheStore *>(&*store);
|
||||
if (s3Store) {
|
||||
auto & s3Stats = s3Store->getS3Stats();
|
||||
@@ -757,7 +742,6 @@ void State::dumpStatus(Connection & conn)
|
||||
+ s3Stats.getBytes / (1024.0 * 1024.0 * 1024.0) * 0.09},
|
||||
};
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
{
|
||||
|
||||
@@ -13,9 +13,7 @@ hydra_queue_runner = executable('hydra-queue-runner',
|
||||
srcs,
|
||||
dependencies: [
|
||||
libhydra_dep,
|
||||
nix_util_dep,
|
||||
nix_store_dep,
|
||||
nix_main_dep,
|
||||
nix_dep,
|
||||
pqxx_dep,
|
||||
prom_cpp_core_dep,
|
||||
prom_cpp_pull_dep,
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
#include "nar-extractor.hh"
|
||||
|
||||
#include <nix/util/archive.hh>
|
||||
#include "archive.hh"
|
||||
|
||||
#include <unordered_set>
|
||||
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
#pragma once
|
||||
|
||||
#include <nix/util/source-accessor.hh>
|
||||
#include <nix/util/types.hh>
|
||||
#include <nix/util/serialise.hh>
|
||||
#include <nix/util/hash.hh>
|
||||
#include "source-accessor.hh"
|
||||
#include "types.hh"
|
||||
#include "serialise.hh"
|
||||
#include "hash.hh"
|
||||
|
||||
struct NarMemberData
|
||||
{
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
#include "state.hh"
|
||||
#include "hydra-build-result.hh"
|
||||
#include <nix/store/globals.hh>
|
||||
#include <nix/store/parsed-derivations.hh>
|
||||
#include <nix/util/thread-pool.hh>
|
||||
#include "globals.hh"
|
||||
|
||||
#include <cstring>
|
||||
|
||||
@@ -39,21 +37,16 @@ void State::queueMonitorLoop(Connection & conn)
|
||||
|
||||
auto destStore = getDestStore();
|
||||
|
||||
unsigned int lastBuildId = 0;
|
||||
|
||||
bool quit = false;
|
||||
while (!quit) {
|
||||
auto t_before_work = std::chrono::steady_clock::now();
|
||||
|
||||
localStore->clearPathInfoCache();
|
||||
|
||||
bool done = getQueuedBuilds(conn, destStore);
|
||||
bool done = getQueuedBuilds(conn, destStore, lastBuildId);
|
||||
|
||||
if (buildOne && buildOneDone) quit = true;
|
||||
|
||||
auto t_after_work = std::chrono::steady_clock::now();
|
||||
|
||||
prom.queue_monitor_time_spent_running.Increment(
|
||||
std::chrono::duration_cast<std::chrono::microseconds>(t_after_work - t_before_work).count());
|
||||
|
||||
/* Sleep until we get notification from the database about an
|
||||
event. */
|
||||
if (done && !quit) {
|
||||
@@ -63,10 +56,12 @@ void State::queueMonitorLoop(Connection & conn)
|
||||
conn.get_notifs();
|
||||
|
||||
if (auto lowestId = buildsAdded.get()) {
|
||||
lastBuildId = std::min(lastBuildId, static_cast<unsigned>(std::stoul(*lowestId) - 1));
|
||||
printMsg(lvlTalkative, "got notification: new builds added to the queue");
|
||||
}
|
||||
if (buildsRestarted.get()) {
|
||||
printMsg(lvlTalkative, "got notification: builds restarted");
|
||||
lastBuildId = 0; // check all builds
|
||||
}
|
||||
if (buildsCancelled.get() || buildsDeleted.get() || buildsBumped.get()) {
|
||||
printMsg(lvlTalkative, "got notification: builds cancelled or bumped");
|
||||
@@ -76,10 +71,6 @@ void State::queueMonitorLoop(Connection & conn)
|
||||
printMsg(lvlTalkative, "got notification: jobset shares changed");
|
||||
processJobsetSharesChange(conn);
|
||||
}
|
||||
|
||||
auto t_after_sleep = std::chrono::steady_clock::now();
|
||||
prom.queue_monitor_time_spent_waiting.Increment(
|
||||
std::chrono::duration_cast<std::chrono::microseconds>(t_after_sleep - t_after_work).count());
|
||||
}
|
||||
|
||||
exit(0);
|
||||
@@ -93,18 +84,20 @@ struct PreviousFailure : public std::exception {
|
||||
|
||||
|
||||
bool State::getQueuedBuilds(Connection & conn,
|
||||
ref<Store> destStore)
|
||||
ref<Store> destStore, unsigned int & lastBuildId)
|
||||
{
|
||||
prom.queue_checks_started.Increment();
|
||||
|
||||
printInfo("checking the queue for builds...");
|
||||
printInfo("checking the queue for builds > %d...", lastBuildId);
|
||||
|
||||
/* Grab the queued builds from the database, but don't process
|
||||
them yet (since we don't want a long-running transaction). */
|
||||
std::vector<BuildID> newIDs;
|
||||
std::unordered_map<BuildID, Build::ptr> newBuildsByID;
|
||||
std::map<BuildID, Build::ptr> newBuildsByID;
|
||||
std::multimap<StorePath, BuildID> newBuildsByPath;
|
||||
|
||||
unsigned int newLastBuildId = lastBuildId;
|
||||
|
||||
{
|
||||
pqxx::work txn(conn);
|
||||
|
||||
@@ -113,12 +106,17 @@ bool State::getQueuedBuilds(Connection & conn,
|
||||
"jobsets.name as jobset, job, drvPath, maxsilent, timeout, timestamp, "
|
||||
"globalPriority, priority from Builds "
|
||||
"inner join jobsets on builds.jobset_id = jobsets.id "
|
||||
"where finished = 0 order by globalPriority desc, random()");
|
||||
"where builds.id > $1 and finished = 0 order by globalPriority desc, builds.id",
|
||||
lastBuildId);
|
||||
|
||||
for (auto const & row : res) {
|
||||
auto builds_(builds.lock());
|
||||
BuildID id = row["id"].as<BuildID>();
|
||||
if (buildOne && id != buildOne) continue;
|
||||
if (id > newLastBuildId) {
|
||||
newLastBuildId = id;
|
||||
prom.queue_max_id.Set(id);
|
||||
}
|
||||
if (builds_->count(id)) continue;
|
||||
|
||||
auto build = std::make_shared<Build>(
|
||||
@@ -320,13 +318,15 @@ bool State::getQueuedBuilds(Connection & conn,
|
||||
|
||||
/* Stop after a certain time to allow priority bumps to be
|
||||
processed. */
|
||||
if (std::chrono::system_clock::now() > start + std::chrono::seconds(60)) {
|
||||
if (std::chrono::system_clock::now() > start + std::chrono::seconds(600)) {
|
||||
prom.queue_checks_early_exits.Increment();
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
prom.queue_checks_finished.Increment();
|
||||
|
||||
lastBuildId = newBuildsByID.empty() ? newLastBuildId : newBuildsByID.begin()->first - 1;
|
||||
return newBuildsByID.empty();
|
||||
}
|
||||
|
||||
@@ -405,34 +405,6 @@ void State::processQueueChange(Connection & conn)
|
||||
}
|
||||
|
||||
|
||||
std::map<DrvOutput, std::optional<StorePath>> State::getMissingRemotePaths(
|
||||
ref<Store> destStore,
|
||||
const std::map<DrvOutput, std::optional<StorePath>> & paths)
|
||||
{
|
||||
Sync<std::map<DrvOutput, std::optional<StorePath>>> missing_;
|
||||
ThreadPool tp;
|
||||
|
||||
for (auto & [output, maybeOutputPath] : paths) {
|
||||
if (!maybeOutputPath) {
|
||||
auto missing(missing_.lock());
|
||||
missing->insert({output, maybeOutputPath});
|
||||
} else {
|
||||
tp.enqueue([&] {
|
||||
if (!destStore->isValidPath(*maybeOutputPath)) {
|
||||
auto missing(missing_.lock());
|
||||
missing->insert({output, maybeOutputPath});
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
tp.process();
|
||||
|
||||
auto missing(missing_.lock());
|
||||
return *missing;
|
||||
}
|
||||
|
||||
|
||||
Step::ptr State::createStep(ref<Store> destStore,
|
||||
Connection & conn, Build::ptr build, const StorePath & drvPath,
|
||||
Build::ptr referringBuild, Step::ptr referringStep, std::set<StorePath> & finishedDrvs,
|
||||
@@ -491,17 +463,14 @@ Step::ptr State::createStep(ref<Store> destStore,
|
||||
it's not runnable yet, and other threads won't make it
|
||||
runnable while step->created == false. */
|
||||
step->drv = std::make_unique<Derivation>(localStore->readDerivation(drvPath));
|
||||
{
|
||||
auto parsedDrv = ParsedDerivation{drvPath, *step->drv};
|
||||
step->drvOptions = std::make_unique<DerivationOptions>(DerivationOptions::fromParsedDerivation(parsedDrv));
|
||||
}
|
||||
step->parsedDrv = std::make_unique<ParsedDerivation>(drvPath, *step->drv);
|
||||
|
||||
step->preferLocalBuild = step->drvOptions->willBuildLocally(*localStore, *step->drv);
|
||||
step->preferLocalBuild = step->parsedDrv->willBuildLocally(*localStore);
|
||||
step->isDeterministic = getOr(step->drv->env, "isDetermistic", "0") == "1";
|
||||
|
||||
step->systemType = step->drv->platform;
|
||||
{
|
||||
StringSet features = step->requiredSystemFeatures = step->drvOptions->getRequiredSystemFeatures(*step->drv);
|
||||
StringSet features = step->requiredSystemFeatures = step->parsedDrv->getRequiredSystemFeatures();
|
||||
if (step->preferLocalBuild)
|
||||
features.insert("local");
|
||||
if (!features.empty()) {
|
||||
@@ -516,15 +485,16 @@ Step::ptr State::createStep(ref<Store> destStore,
|
||||
|
||||
/* Are all outputs valid? */
|
||||
auto outputHashes = staticOutputHashes(*localStore, *(step->drv));
|
||||
std::map<DrvOutput, std::optional<StorePath>> paths;
|
||||
bool valid = true;
|
||||
std::map<DrvOutput, std::optional<StorePath>> missing;
|
||||
for (auto & [outputName, maybeOutputPath] : destStore->queryPartialDerivationOutputMap(drvPath, &*localStore)) {
|
||||
auto outputHash = outputHashes.at(outputName);
|
||||
paths.insert({{outputHash, outputName}, maybeOutputPath});
|
||||
if (maybeOutputPath && destStore->isValidPath(*maybeOutputPath))
|
||||
continue;
|
||||
valid = false;
|
||||
missing.insert({{outputHash, outputName}, maybeOutputPath});
|
||||
}
|
||||
|
||||
auto missing = getMissingRemotePaths(destStore, paths);
|
||||
bool valid = missing.empty();
|
||||
|
||||
/* Try to copy the missing paths from the local store or from
|
||||
substitutes. */
|
||||
if (!missing.empty()) {
|
||||
|
||||
@@ -7,7 +7,6 @@
|
||||
#include <memory>
|
||||
#include <queue>
|
||||
#include <regex>
|
||||
#include <semaphore>
|
||||
|
||||
#include <prometheus/counter.h>
|
||||
#include <prometheus/gauge.h>
|
||||
@@ -15,18 +14,17 @@
|
||||
|
||||
#include "db.hh"
|
||||
|
||||
#include <nix/store/derivations.hh>
|
||||
#include <nix/store/derivation-options.hh>
|
||||
#include <nix/store/pathlocks.hh>
|
||||
#include <nix/util/pool.hh>
|
||||
#include <nix/store/build-result.hh>
|
||||
#include <nix/store/store-api.hh>
|
||||
#include <nix/util/sync.hh>
|
||||
#include "parsed-derivations.hh"
|
||||
#include "pathlocks.hh"
|
||||
#include "pool.hh"
|
||||
#include "build-result.hh"
|
||||
#include "store-api.hh"
|
||||
#include "sync.hh"
|
||||
#include "nar-extractor.hh"
|
||||
#include <nix/store/serve-protocol.hh>
|
||||
#include <nix/store/serve-protocol-impl.hh>
|
||||
#include <nix/store/serve-protocol-connection.hh>
|
||||
#include <nix/store/machines.hh>
|
||||
#include "serve-protocol.hh"
|
||||
#include "serve-protocol-impl.hh"
|
||||
#include "serve-protocol-connection.hh"
|
||||
#include "machines.hh"
|
||||
|
||||
|
||||
typedef unsigned int BuildID;
|
||||
@@ -60,7 +58,6 @@ typedef enum {
|
||||
ssConnecting = 10,
|
||||
ssSendingInputs = 20,
|
||||
ssBuilding = 30,
|
||||
ssWaitingForLocalSlot = 35,
|
||||
ssReceivingOutputs = 40,
|
||||
ssPostProcessing = 50,
|
||||
} StepState;
|
||||
@@ -171,7 +168,7 @@ struct Step
|
||||
|
||||
nix::StorePath drvPath;
|
||||
std::unique_ptr<nix::Derivation> drv;
|
||||
std::unique_ptr<nix::DerivationOptions> drvOptions;
|
||||
std::unique_ptr<nix::ParsedDerivation> parsedDrv;
|
||||
std::set<std::string> requiredSystemFeatures;
|
||||
bool preferLocalBuild;
|
||||
bool isDeterministic;
|
||||
@@ -244,6 +241,10 @@ struct Machine : nix::Machine
|
||||
{
|
||||
typedef std::shared_ptr<Machine> ptr;
|
||||
|
||||
/* TODO Get rid of: `nix::Machine::storeUri` is normalized in a way
|
||||
we are not yet used to, but once we are, we don't need this. */
|
||||
std::string sshName;
|
||||
|
||||
struct State {
|
||||
typedef std::shared_ptr<State> ptr;
|
||||
counter currentJobs{0};
|
||||
@@ -293,7 +294,11 @@ struct Machine : nix::Machine
|
||||
return true;
|
||||
}
|
||||
|
||||
bool isLocalhost() const;
|
||||
bool isLocalhost()
|
||||
{
|
||||
std::regex r("^(ssh://|ssh-ng://)?localhost$");
|
||||
return std::regex_search(sshName, r);
|
||||
}
|
||||
|
||||
// A connection to a machine
|
||||
struct Connection : nix::ServeProto::BasicClientConnection {
|
||||
@@ -353,13 +358,9 @@ private:
|
||||
|
||||
/* The build machines. */
|
||||
std::mutex machinesReadyLock;
|
||||
typedef std::map<nix::StoreReference::Variant, Machine::ptr> Machines;
|
||||
typedef std::map<std::string, Machine::ptr> Machines;
|
||||
nix::Sync<Machines> machines; // FIXME: use atomic_shared_ptr
|
||||
|
||||
/* Throttler for CPU-bound local work. */
|
||||
static constexpr unsigned int maxSupportedLocalWorkers = 1024;
|
||||
std::counting_semaphore<maxSupportedLocalWorkers> localWorkThrottler;
|
||||
|
||||
/* Various stats. */
|
||||
time_t startedAt;
|
||||
counter nrBuildsRead{0};
|
||||
@@ -369,7 +370,6 @@ private:
|
||||
counter nrStepsDone{0};
|
||||
counter nrStepsBuilding{0};
|
||||
counter nrStepsCopyingTo{0};
|
||||
counter nrStepsWaitingForDownloadSlot{0};
|
||||
counter nrStepsCopyingFrom{0};
|
||||
counter nrStepsWaiting{0};
|
||||
counter nrUnsupportedSteps{0};
|
||||
@@ -400,6 +400,7 @@ private:
|
||||
|
||||
struct MachineReservation
|
||||
{
|
||||
typedef std::shared_ptr<MachineReservation> ptr;
|
||||
State & state;
|
||||
Step::ptr step;
|
||||
Machine::ptr machine;
|
||||
@@ -457,18 +458,7 @@ private:
|
||||
prometheus::Counter& queue_steps_created;
|
||||
prometheus::Counter& queue_checks_early_exits;
|
||||
prometheus::Counter& queue_checks_finished;
|
||||
|
||||
prometheus::Counter& dispatcher_time_spent_running;
|
||||
prometheus::Counter& dispatcher_time_spent_waiting;
|
||||
|
||||
prometheus::Counter& queue_monitor_time_spent_running;
|
||||
prometheus::Counter& queue_monitor_time_spent_waiting;
|
||||
|
||||
prometheus::Counter& dispatcher_time_spent_running;
|
||||
prometheus::Counter& dispatcher_time_spent_waiting;
|
||||
|
||||
prometheus::Counter& queue_monitor_time_spent_running;
|
||||
prometheus::Counter& queue_monitor_time_spent_waiting;
|
||||
prometheus::Gauge& queue_max_id;
|
||||
|
||||
PromMetrics();
|
||||
};
|
||||
@@ -512,7 +502,8 @@ private:
|
||||
void queueMonitorLoop(Connection & conn);
|
||||
|
||||
/* Check the queue for new builds. */
|
||||
bool getQueuedBuilds(Connection & conn, nix::ref<nix::Store> destStore);
|
||||
bool getQueuedBuilds(Connection & conn,
|
||||
nix::ref<nix::Store> destStore, unsigned int & lastBuildId);
|
||||
|
||||
/* Handle cancellation, deletion and priority bumps. */
|
||||
void processQueueChange(Connection & conn);
|
||||
@@ -520,12 +511,6 @@ private:
|
||||
BuildOutput getBuildOutputCached(Connection & conn, nix::ref<nix::Store> destStore,
|
||||
const nix::StorePath & drvPath);
|
||||
|
||||
/* Returns paths missing from the remote store. Paths are processed in
|
||||
* parallel to work around the possible latency of remote stores. */
|
||||
std::map<nix::DrvOutput, std::optional<nix::StorePath>> getMissingRemotePaths(
|
||||
nix::ref<nix::Store> destStore,
|
||||
const std::map<nix::DrvOutput, std::optional<nix::StorePath>> & paths);
|
||||
|
||||
Step::ptr createStep(nix::ref<nix::Store> store,
|
||||
Connection & conn, Build::ptr build, const nix::StorePath & drvPath,
|
||||
Build::ptr referringBuild, Step::ptr referringStep, std::set<nix::StorePath> & finishedDrvs,
|
||||
@@ -555,17 +540,16 @@ private:
|
||||
|
||||
void abortUnsupported();
|
||||
|
||||
void builder(std::unique_ptr<MachineReservation> reservation);
|
||||
void builder(MachineReservation::ptr reservation);
|
||||
|
||||
/* Perform the given build step. Return true if the step is to be
|
||||
retried. */
|
||||
enum StepResult { sDone, sRetry, sMaybeCancelled };
|
||||
StepResult doBuildStep(nix::ref<nix::Store> destStore,
|
||||
std::unique_ptr<MachineReservation> reservation,
|
||||
MachineReservation::ptr reservation,
|
||||
std::shared_ptr<ActiveStep> activeStep);
|
||||
|
||||
void buildRemote(nix::ref<nix::Store> destStore,
|
||||
std::unique_ptr<MachineReservation> reservation,
|
||||
Machine::ptr machine, Step::ptr step,
|
||||
const nix::ServeProto::BuildOptions & buildOptions,
|
||||
RemoteResult & result, std::shared_ptr<ActiveStep> activeStep,
|
||||
|
||||
@@ -238,7 +238,7 @@ sub serveFile {
|
||||
# XSS hole.
|
||||
$c->response->header('Content-Security-Policy' => 'sandbox allow-scripts');
|
||||
|
||||
$c->stash->{'plain'} = { data => readIntoSocket(cmd => ["nix", "--experimental-features", "nix-command",
|
||||
$c->stash->{'plain'} = { data => grab(cmd => ["nix", "--experimental-features", "nix-command",
|
||||
"store", "cat", "--store", getStoreUri(), "$path"]) };
|
||||
|
||||
# Detect MIME type.
|
||||
|
||||
@@ -364,21 +364,6 @@ sub evals_GET {
|
||||
);
|
||||
}
|
||||
|
||||
sub errors :Chained('jobsetChain') :PathPart('errors') :Args(0) :ActionClass('REST') { }
|
||||
|
||||
sub errors_GET {
|
||||
my ($self, $c) = @_;
|
||||
|
||||
$c->stash->{template} = 'eval-error.tt';
|
||||
|
||||
my $jobsetName = $c->stash->{params}->{name};
|
||||
$c->stash->{jobset} = $c->stash->{project}->jobsets->find(
|
||||
{ name => $jobsetName },
|
||||
{ '+columns' => { 'errormsg' => 'errormsg' } }
|
||||
);
|
||||
|
||||
$self->status_ok($c, entity => $c->stash->{jobset});
|
||||
}
|
||||
|
||||
# Redirect to the latest finished evaluation of this jobset.
|
||||
sub latest_eval : Chained('jobsetChain') PathPart('latest-eval') {
|
||||
|
||||
@@ -86,17 +86,6 @@ sub view_GET {
|
||||
);
|
||||
}
|
||||
|
||||
sub errors :Chained('evalChain') :PathPart('errors') :Args(0) :ActionClass('REST') { }
|
||||
|
||||
sub errors_GET {
|
||||
my ($self, $c) = @_;
|
||||
|
||||
$c->stash->{template} = 'eval-error.tt';
|
||||
|
||||
$c->stash->{eval} = $c->model('DB::JobsetEvals')->find($c->stash->{eval}->id, { prefetch => 'evaluationerror' });
|
||||
|
||||
$self->status_ok($c, entity => $c->stash->{eval});
|
||||
}
|
||||
|
||||
sub create_jobset : Chained('evalChain') PathPart('create-jobset') Args(0) {
|
||||
my ($self, $c) = @_;
|
||||
|
||||
@@ -51,7 +51,6 @@ sub begin :Private {
|
||||
$c->stash->{curUri} = $c->request->uri;
|
||||
$c->stash->{version} = $ENV{"HYDRA_RELEASE"} || "<devel>";
|
||||
$c->stash->{nixVersion} = $ENV{"NIX_RELEASE"} || "<devel>";
|
||||
$c->stash->{nixEvalJobsVersion} = $ENV{"NIX_EVAL_JOBS_RELEASE"} || "<devel>";
|
||||
$c->stash->{curTime} = time;
|
||||
$c->stash->{logo} = defined $c->config->{hydra_logo} ? "/logo" : "";
|
||||
$c->stash->{tracker} = defined $c->config->{tracker} ? $c->config->{tracker} : "";
|
||||
@@ -162,7 +161,7 @@ sub status_GET {
|
||||
{ "buildsteps.busy" => { '!=', 0 } },
|
||||
{ order_by => ["globalpriority DESC", "id"],
|
||||
join => "buildsteps",
|
||||
columns => [@buildListColumns, 'buildsteps.drvpath', 'buildsteps.type']
|
||||
columns => [@buildListColumns]
|
||||
})]
|
||||
);
|
||||
}
|
||||
|
||||
@@ -37,16 +37,7 @@ sub buildDiff {
|
||||
|
||||
my $n = 0;
|
||||
foreach my $build (@{$builds}) {
|
||||
my $aborted = $build->finished != 0 && (
|
||||
# aborted
|
||||
$build->buildstatus == 3
|
||||
# cancelled
|
||||
|| $build->buildstatus == 4
|
||||
# timeout
|
||||
|| $build->buildstatus == 7
|
||||
# log limit exceeded
|
||||
|| $build->buildstatus == 10
|
||||
);
|
||||
my $aborted = $build->finished != 0 && ($build->buildstatus == 3 || $build->buildstatus == 4);
|
||||
my $d;
|
||||
my $found = 0;
|
||||
while ($n < scalar(@{$builds2})) {
|
||||
@@ -88,4 +79,4 @@ sub buildDiff {
|
||||
return $ret;
|
||||
}
|
||||
|
||||
1;
|
||||
1;
|
||||
@@ -36,7 +36,6 @@ our @EXPORT = qw(
|
||||
jobsetOverview
|
||||
jobsetOverview_
|
||||
pathIsInsidePrefix
|
||||
readIntoSocket
|
||||
readNixFile
|
||||
registerRoot
|
||||
restartBuilds
|
||||
@@ -297,7 +296,8 @@ sub getEvals {
|
||||
|
||||
my @evals = $evals_result_set->search(
|
||||
{ hasnewbuilds => 1 },
|
||||
{ order_by => "$me.id DESC", rows => $rows, offset => $offset });
|
||||
{ order_by => "$me.id DESC", rows => $rows, offset => $offset
|
||||
, prefetch => { evaluationerror => [ ] } });
|
||||
my @res = ();
|
||||
my $cache = {};
|
||||
|
||||
@@ -417,16 +417,6 @@ sub pathIsInsidePrefix {
|
||||
return $cur;
|
||||
}
|
||||
|
||||
sub readIntoSocket{
|
||||
my (%args) = @_;
|
||||
my $sock;
|
||||
|
||||
eval {
|
||||
open($sock, "-|", @{$args{cmd}}) or die q(failed to open socket from command:\n $x);
|
||||
};
|
||||
|
||||
return $sock;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -105,6 +105,4 @@ __PACKAGE__->add_column(
|
||||
"+id" => { retrieve_on_insert => 1 }
|
||||
);
|
||||
|
||||
__PACKAGE__->mk_group_accessors('column' => 'has_error');
|
||||
|
||||
1;
|
||||
|
||||
@@ -386,8 +386,6 @@ __PACKAGE__->add_column(
|
||||
"+id" => { retrieve_on_insert => 1 }
|
||||
);
|
||||
|
||||
__PACKAGE__->mk_group_accessors('column' => 'has_error');
|
||||
|
||||
sub supportsDynamicRunCommand {
|
||||
my ($self) = @_;
|
||||
|
||||
|
||||
@@ -1,30 +0,0 @@
|
||||
package Hydra::Schema::ResultSet::EvaluationErrors;
|
||||
|
||||
use strict;
|
||||
use utf8;
|
||||
use warnings;
|
||||
|
||||
use parent 'DBIx::Class::ResultSet';
|
||||
|
||||
use Storable qw(dclone);
|
||||
|
||||
__PACKAGE__->load_components('Helper::ResultSet::RemoveColumns');
|
||||
|
||||
# Exclude expensive error message values unless explicitly requested, and
|
||||
# replace them with a summary field describing their presence/absence.
|
||||
sub search_rs {
|
||||
my ( $class, $query, $attrs ) = @_;
|
||||
|
||||
if ($attrs) {
|
||||
$attrs = dclone($attrs);
|
||||
}
|
||||
|
||||
unless (exists $attrs->{'select'} || exists $attrs->{'columns'}) {
|
||||
$attrs->{'+columns'}->{'has_error'} = "errormsg != ''";
|
||||
}
|
||||
unless (exists $attrs->{'+columns'}->{'errormsg'}) {
|
||||
push @{ $attrs->{'remove_columns'} }, 'errormsg';
|
||||
}
|
||||
|
||||
return $class->next::method($query, $attrs);
|
||||
}
|
||||
@@ -1,30 +0,0 @@
|
||||
package Hydra::Schema::ResultSet::Jobsets;
|
||||
|
||||
use strict;
|
||||
use utf8;
|
||||
use warnings;
|
||||
|
||||
use parent 'DBIx::Class::ResultSet';
|
||||
|
||||
use Storable qw(dclone);
|
||||
|
||||
__PACKAGE__->load_components('Helper::ResultSet::RemoveColumns');
|
||||
|
||||
# Exclude expensive error message values unless explicitly requested, and
|
||||
# replace them with a summary field describing their presence/absence.
|
||||
sub search_rs {
|
||||
my ( $class, $query, $attrs ) = @_;
|
||||
|
||||
if ($attrs) {
|
||||
$attrs = dclone($attrs);
|
||||
}
|
||||
|
||||
unless (exists $attrs->{'select'} || exists $attrs->{'columns'}) {
|
||||
$attrs->{'+columns'}->{'has_error'} = "errormsg != ''";
|
||||
}
|
||||
unless (exists $attrs->{'+columns'}->{'errormsg'}) {
|
||||
push @{ $attrs->{'remove_columns'} }, 'errormsg';
|
||||
}
|
||||
|
||||
return $class->next::method($query, $attrs);
|
||||
}
|
||||
@@ -6,7 +6,6 @@ use base 'Catalyst::View::TT';
|
||||
use Template::Plugin::HTML;
|
||||
use Hydra::Helper::Nix;
|
||||
use Time::Seconds;
|
||||
use Digest::SHA qw(sha1_hex);
|
||||
|
||||
__PACKAGE__->config(
|
||||
TEMPLATE_EXTENSION => '.tt',
|
||||
@@ -26,14 +25,8 @@ __PACKAGE__->config(
|
||||
makeNameTextForJobset
|
||||
relativeDuration
|
||||
stripSSHUser
|
||||
metricDivId
|
||||
/]);
|
||||
|
||||
sub metricDivId {
|
||||
my ($self, $c, $text) = @_;
|
||||
return "metric-" . sha1_hex($text);
|
||||
}
|
||||
|
||||
sub buildLogExists {
|
||||
my ($self, $c, $build) = @_;
|
||||
return 1 if defined $c->config->{log_prefix};
|
||||
|
||||
@@ -2,8 +2,8 @@
|
||||
|
||||
#include <pqxx/pqxx>
|
||||
|
||||
#include <nix/util/environment-variables.hh>
|
||||
#include <nix/util/util.hh>
|
||||
#include "environment-variables.hh"
|
||||
#include "util.hh"
|
||||
|
||||
|
||||
struct Connection : pqxx::connection
|
||||
|
||||
@@ -2,8 +2,8 @@
|
||||
|
||||
#include <map>
|
||||
|
||||
#include <nix/util/file-system.hh>
|
||||
#include <nix/util/util.hh>
|
||||
#include "file-system.hh"
|
||||
#include "util.hh"
|
||||
|
||||
struct HydraConfig
|
||||
{
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
# Native code
|
||||
subdir('libhydra')
|
||||
subdir('hydra-eval-jobs')
|
||||
subdir('hydra-evaluator')
|
||||
subdir('hydra-queue-runner')
|
||||
|
||||
|
||||
@@ -61,7 +61,21 @@ END;
|
||||
<td>[% IF step.busy != 0 || ((step.machine || step.starttime) && (step.status == 0 || step.status == 1 || step.status == 3 || step.status == 4 || step.status == 7)); INCLUDE renderMachineName machine=step.machine; ELSE; "<em>n/a</em>"; END %]</td>
|
||||
<td class="step-status">
|
||||
[% IF step.busy != 0 %]
|
||||
[% INCLUDE renderBusyStatus %]
|
||||
[% IF step.busy == 1 %]
|
||||
<strong>Preparing</strong>
|
||||
[% ELSIF step.busy == 10 %]
|
||||
<strong>Connecting</strong>
|
||||
[% ELSIF step.busy == 20 %]
|
||||
<strong>Sending inputs</strong>
|
||||
[% ELSIF step.busy == 30 %]
|
||||
<strong>Building</strong>
|
||||
[% ELSIF step.busy == 40 %]
|
||||
<strong>Receiving outputs</strong>
|
||||
[% ELSIF step.busy == 50 %]
|
||||
<strong>Post-processing</strong>
|
||||
[% ELSE %]
|
||||
<strong>Unknown state</strong>
|
||||
[% END %]
|
||||
[% ELSIF step.status == 0 %]
|
||||
[% IF step.isnondeterministic %]
|
||||
<span class="warn">Succeeded with non-determistic result</span>
|
||||
|
||||
@@ -91,17 +91,6 @@ BLOCK renderDuration;
|
||||
duration % 60 %]s[%
|
||||
END;
|
||||
|
||||
BLOCK renderDrvInfo;
|
||||
drvname = step.drvpath
|
||||
.substr(11) # strip `/nix/store/`
|
||||
.split('-').slice(1).join("-") # strip hash part
|
||||
.substr(0, -4); # strip `.drv`
|
||||
IF drvname != releasename;
|
||||
IF step.type == 0; action = "Build"; ELSE; action = "Substitution"; END;
|
||||
IF drvname; %]<em> ([% action %] of [% drvname %])</em>[% END;
|
||||
END;
|
||||
END;
|
||||
|
||||
|
||||
BLOCK renderBuildListHeader %]
|
||||
<table class="table table-striped table-condensed clickable-rows">
|
||||
@@ -142,12 +131,7 @@ BLOCK renderBuildListBody;
|
||||
[% END %]
|
||||
<td><a class="row-link" href="[% link %]">[% build.id %]</a></td>
|
||||
[% IF !hideJobName %]
|
||||
<td>
|
||||
<a href="[%link%]">[% IF !hideJobsetName %][%build.jobset.get_column("project")%]:[%build.jobset.get_column("name")%]:[% END %][%build.get_column("job")%]</a>
|
||||
[% IF showStepName %]
|
||||
[% INCLUDE renderDrvInfo step=build.buildsteps releasename=build.nixname %]
|
||||
[% END %]
|
||||
</td>
|
||||
<td><a href="[%link%]">[% IF !hideJobsetName %][%build.jobset.get_column("project")%]:[%build.jobset.get_column("name")%]:[% END %][%build.get_column("job")%]</td>
|
||||
[% END %]
|
||||
<td class="nowrap">[% t = showSchedulingInfo ? build.timestamp : build.stoptime; IF t; INCLUDE renderRelativeDate timestamp=(showSchedulingInfo ? build.timestamp : build.stoptime); ELSE; "-"; END %]</td>
|
||||
<td>[% !showSchedulingInfo and build.get_column('releasename') ? build.get_column('releasename') : build.nixname %]</td>
|
||||
@@ -261,27 +245,6 @@ BLOCK renderBuildStatusIcon;
|
||||
END;
|
||||
|
||||
|
||||
BLOCK renderBusyStatus;
|
||||
IF step.busy == 1 %]
|
||||
<strong>Preparing</strong>
|
||||
[% ELSIF step.busy == 10 %]
|
||||
<strong>Connecting</strong>
|
||||
[% ELSIF step.busy == 20 %]
|
||||
<strong>Sending inputs</strong>
|
||||
[% ELSIF step.busy == 30 %]
|
||||
<strong>Building</strong>
|
||||
[% ELSIF step.busy == 35 %]
|
||||
<strong>Waiting to receive outputs</strong>
|
||||
[% ELSIF step.busy == 40 %]
|
||||
<strong>Receiving outputs</strong>
|
||||
[% ELSIF step.busy == 50 %]
|
||||
<strong>Post-processing</strong>
|
||||
[% ELSE %]
|
||||
<strong>Unknown state</strong>
|
||||
[% END;
|
||||
END;
|
||||
|
||||
|
||||
BLOCK renderStatus;
|
||||
IF build.finished;
|
||||
buildstatus = build.buildstatus;
|
||||
@@ -513,7 +476,7 @@ BLOCK renderEvals %]
|
||||
ELSE %]
|
||||
-
|
||||
[% END %]
|
||||
[% IF eval.evaluationerror.has_error %]
|
||||
[% IF eval.evaluationerror.errormsg %]
|
||||
<span class="badge badge-warning">Eval Errors</span>
|
||||
[% END %]
|
||||
</td>
|
||||
@@ -639,7 +602,7 @@ BLOCK renderJobsetOverview %]
|
||||
<td>[% HTML.escape(j.description) %]</td>
|
||||
<td>[% IF j.lastcheckedtime;
|
||||
INCLUDE renderDateTime timestamp = j.lastcheckedtime;
|
||||
IF j.has_error || j.fetcherrormsg; %] <span class = 'badge badge-warning'>Error</span>[% END;
|
||||
IF j.errormsg || j.fetcherrormsg; %] <span class = 'badge badge-warning'>Error</span>[% END;
|
||||
ELSE; "-";
|
||||
END %]</td>
|
||||
[% IF j.get_column('nrtotal') > 0 %]
|
||||
|
||||
@@ -1,26 +0,0 @@
|
||||
[% PROCESS common.tt %]
|
||||
<!DOCTYPE html>
|
||||
|
||||
<html lang="en">
|
||||
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta http-equiv="X-UA-Compatible" content="IE=Edge" />
|
||||
[% INCLUDE style.tt %]
|
||||
</head>
|
||||
|
||||
<body>
|
||||
|
||||
<div class="tab-content tab-pane">
|
||||
<div id="tabs-errors" class="">
|
||||
[% IF eval %]
|
||||
<p>Errors occurred at [% INCLUDE renderDateTime timestamp=(eval.evaluationerror.errortime || eval.timestamp) %].</p>
|
||||
<div class="card bg-light"><div class="card-body"><pre>[% HTML.escape(eval.evaluationerror.errormsg) %]</pre></div></div>
|
||||
[% ELSIF jobset %]
|
||||
<p>Errors occurred at [% INCLUDE renderDateTime timestamp=(jobset.errortime || jobset.lastcheckedtime) %].</p>
|
||||
<div class="card bg-light"><div class="card-body"><pre>[% HTML.escape(jobset.fetcherrormsg || jobset.errormsg) %]</pre></div></div>
|
||||
[% END %]
|
||||
</div>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
@@ -18,7 +18,8 @@
|
||||
|
||||
<h3>Metric: <a [% HTML.attributes(href => c.uri_for('/job' project.name jobset.name job 'metric' metric.name)) %]><tt>[%HTML.escape(metric.name)%]</tt></a></h3>
|
||||
|
||||
[% id = metricDivId(metric.name);
|
||||
[% id = "metric-" _ metric.name;
|
||||
id = id.replace('\.', '_');
|
||||
INCLUDE createChart dataUrl=c.uri_for('/job' project.name jobset.name job 'metric' metric.name); %]
|
||||
|
||||
[% END %]
|
||||
|
||||
@@ -27,15 +27,15 @@ removed or had an evaluation error.</div>
|
||||
<div id="generic-tabs" class="tab-content">
|
||||
|
||||
<div id="tabs-status" class="tab-pane active">
|
||||
[% IF queuedBuilds.size != 0 %]
|
||||
<h3>Queued builds</h3>
|
||||
[% INCLUDE renderBuildList builds=queuedBuilds showSchedulingInfo=1 hideResultInfo=1 %]
|
||||
[% END %]
|
||||
[% IF lastBuilds.size != 0 %]
|
||||
<h3>Latest builds</h3>
|
||||
[% INCLUDE renderBuildList builds=lastBuilds
|
||||
linkToAll=c.uri_for('/job' project.name jobset.name job 'all') %]
|
||||
[% END %]
|
||||
[% IF queuedBuilds.size != 0 %]
|
||||
<h3>Queued builds</h3>
|
||||
[% INCLUDE renderBuildList builds=queuedBuilds showSchedulingInfo=1 hideResultInfo=1 %]
|
||||
[% END %]
|
||||
</div>
|
||||
|
||||
[% IF constituentJobs.size > 0 %]
|
||||
|
||||
@@ -65,7 +65,7 @@ c.uri_for(c.controller('JobsetEval').action_for('view'),
|
||||
[% END %]
|
||||
|
||||
[% IF aborted.size > 0 %]
|
||||
<li class="nav-item"><a class="nav-link" href="#tabs-aborted" data-toggle="tab"><span class="text-warning">Aborted / Timed out Jobs ([% aborted.size %])</span></a></li>
|
||||
<li class="nav-item"><a class="nav-link" href="#tabs-aborted" data-toggle="tab"><span class="text-warning">Aborted Jobs ([% aborted.size %])</span></a></li>
|
||||
[% END %]
|
||||
[% IF nowFail.size > 0 %]
|
||||
<li class="nav-item"><a class="nav-link" href="#tabs-now-fail" data-toggle="tab"><span class="text-warning">Newly Failing Jobs ([% nowFail.size %])</span></a></li>
|
||||
@@ -90,7 +90,7 @@ c.uri_for(c.controller('JobsetEval').action_for('view'),
|
||||
[% END %]
|
||||
<li class="nav-item"><a class="nav-link" href="#tabs-inputs" data-toggle="tab">Inputs</a></li>
|
||||
|
||||
[% IF eval.evaluationerror.has_error %]
|
||||
[% IF eval.evaluationerror.errormsg %]
|
||||
<li class="nav-item"><a class="nav-link" href="#tabs-errors" data-toggle="tab"><span class="text-warning">Evaluation Errors</span></a></li>
|
||||
[% END %]
|
||||
</ul>
|
||||
@@ -108,6 +108,13 @@ c.uri_for(c.controller('JobsetEval').action_for('view'),
|
||||
|
||||
<div class="tab-content">
|
||||
|
||||
[% IF eval.evaluationerror.errormsg %]
|
||||
<div id="tabs-errors" class="tab-pane">
|
||||
<p>Errors occurred at [% INCLUDE renderDateTime timestamp=(eval.evaluationerror.errortime || eval.timestamp) %].</p>
|
||||
<div class="card bg-light"><div class="card-body"><pre>[% HTML.escape(eval.evaluationerror.errormsg) %]</pre></div></div>
|
||||
</div>
|
||||
[% END %]
|
||||
|
||||
<div id="tabs-aborted" class="tab-pane">
|
||||
[% INCLUDE renderSome builds=aborted tabname="#tabs-aborted" %]
|
||||
</div>
|
||||
@@ -165,9 +172,10 @@ c.uri_for(c.controller('JobsetEval').action_for('view'),
|
||||
[% END %]
|
||||
</div>
|
||||
|
||||
[% IF eval.evaluationerror.has_error %]
|
||||
[% IF eval.evaluationerror.errormsg %]
|
||||
<div id="tabs-errors" class="tab-pane">
|
||||
<iframe src="[% c.uri_for(c.controller('JobsetEval').action_for('errors'), [eval.id], params) %]" loading="lazy" frameBorder="0" width="100%"></iframe>
|
||||
<p>Errors occurred at [% INCLUDE renderDateTime timestamp=(eval.evaluationerror.errortime || eval.timestamp) %].</p>
|
||||
<div class="card bg-light"><div class="card-body"><pre>[% HTML.escape(eval.evaluationerror.errormsg) %]</pre></div></div>
|
||||
</div>
|
||||
[% END %]
|
||||
</div>
|
||||
|
||||
@@ -61,7 +61,7 @@
|
||||
[% END %]
|
||||
|
||||
<li class="nav-item"><a class="nav-link active" href="#tabs-evaluations" data-toggle="tab">Evaluations</a></li>
|
||||
[% IF jobset.has_error || jobset.fetcherrormsg %]
|
||||
[% IF jobset.errormsg || jobset.fetcherrormsg %]
|
||||
<li class="nav-item"><a class="nav-link" href="#tabs-errors" data-toggle="tab"><span class="text-warning">Evaluation Errors</span></a></li>
|
||||
[% END %]
|
||||
<li class="nav-item"><a class="nav-link" href="#tabs-jobs" data-toggle="tab">Jobs</a></li>
|
||||
@@ -79,7 +79,7 @@
|
||||
<th>Last checked:</th>
|
||||
<td>
|
||||
[% IF jobset.lastcheckedtime %]
|
||||
[% INCLUDE renderDateTime timestamp = jobset.lastcheckedtime %], [% IF jobset.has_error || jobset.fetcherrormsg %]<em class="text-warning">with errors!</em>[% ELSE %]<em>no errors</em>[% END %]
|
||||
[% INCLUDE renderDateTime timestamp = jobset.lastcheckedtime %], [% IF jobset.errormsg || jobset.fetcherrormsg %]<em class="text-warning">with errors!</em>[% ELSE %]<em>no errors</em>[% END %]
|
||||
[% ELSE %]
|
||||
<em>never</em>
|
||||
[% END %]
|
||||
@@ -117,9 +117,10 @@
|
||||
|
||||
</div>
|
||||
|
||||
[% IF jobset.has_error || jobset.fetcherrormsg %]
|
||||
[% IF jobset.errormsg || jobset.fetcherrormsg %]
|
||||
<div id="tabs-errors" class="tab-pane">
|
||||
<iframe src="[% c.uri_for('/jobset' project.name jobset.name "errors") %]" loading="lazy" frameBorder="0" width="100%"></iframe>
|
||||
<p>Errors occurred at [% INCLUDE renderDateTime timestamp=(jobset.errortime || jobset.lastcheckedtime) %].</p>
|
||||
<div class="card bg-light"><div class="card-body"><pre>[% HTML.escape(jobset.fetcherrormsg || jobset.errormsg) %]</pre></div></div>
|
||||
</div>
|
||||
[% END %]
|
||||
|
||||
|
||||
@@ -10,7 +10,31 @@
|
||||
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta http-equiv="X-UA-Compatible" content="IE=Edge" />
|
||||
[% INCLUDE style.tt %]
|
||||
|
||||
<script type="text/javascript" src="[% c.uri_for("/static/js/jquery/jquery-3.4.1.min.js") %]"></script>
|
||||
<script type="text/javascript" src="[% c.uri_for("/static/js/jquery/jquery-ui-1.10.4.min.js") %]"></script>
|
||||
<script type="text/javascript" src="[% c.uri_for("/static/js/moment/moment-2.24.0.min.js") %]"></script>
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
|
||||
<link href="[% c.uri_for("/static/fontawesome/css/all.css") %]" rel="stylesheet" />
|
||||
<script type="text/javascript" src="[% c.uri_for("/static/js/popper.min.js") %]"></script>
|
||||
<script type="text/javascript" src="[% c.uri_for("/static/bootstrap/js/bootstrap.min.js") %]"></script>
|
||||
<link href="[% c.uri_for("/static/bootstrap/css/bootstrap.min.css") %]" rel="stylesheet" />
|
||||
|
||||
<!-- hydra.css may need to be moved to before boostrap to make the @media rule work. -->
|
||||
<link rel="stylesheet" href="[% c.uri_for("/static/css/hydra.css") %]" type="text/css" />
|
||||
<link rel="stylesheet" href="[% c.uri_for("/static/css/rotated-th.css") %]" type="text/css" />
|
||||
|
||||
<style>
|
||||
.popover { max-width: 40%; }
|
||||
</style>
|
||||
|
||||
<script type="text/javascript" src="[% c.uri_for("/static/js/bootbox.min.js") %]"></script>
|
||||
|
||||
<link rel="stylesheet" href="[% c.uri_for("/static/css/tree.css") %]" type="text/css" />
|
||||
|
||||
<script type="text/javascript" src="[% c.uri_for("/static/js/common.js") %]"></script>
|
||||
|
||||
[% IF c.config.enable_google_login %]
|
||||
<meta name="google-signin-client_id" content="[% c.config.google_client_id %]">
|
||||
@@ -69,7 +93,7 @@
|
||||
<footer class="navbar">
|
||||
<hr />
|
||||
<small>
|
||||
<em><a href="http://nixos.org/hydra" target="_blank" class="squiggle">Hydra</a> [% HTML.escape(version) %] (using [% HTML.escape(nixVersion) %] and [% HTML.escape(nixEvalJobsVersion) %]).</em>
|
||||
<em><a href="http://nixos.org/hydra" target="_blank" class="squiggle">Hydra</a> [% HTML.escape(version) %] (using [% HTML.escape(nixVersion) %]).</em>
|
||||
[% IF c.user_exists %]
|
||||
You are signed in as <tt>[% HTML.escape(c.user.username) %]</tt>
|
||||
[%- IF c.user.type == 'google' %] via Google[% END %].
|
||||
|
||||
@@ -6,10 +6,10 @@
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Job</th>
|
||||
<th>System</th>
|
||||
<th>Build</th>
|
||||
<th>Step</th>
|
||||
<th>What</th>
|
||||
<th>Status</th>
|
||||
<th>Since</th>
|
||||
</tr>
|
||||
</thead>
|
||||
@@ -40,10 +40,10 @@
|
||||
[% idle = 0 %]
|
||||
<tr>
|
||||
<td><tt>[% INCLUDE renderFullJobName project=step.project jobset=step.jobset job=step.job %]</tt></td>
|
||||
<td><tt>[% step.system %]</tt></td>
|
||||
<td><a href="[% c.uri_for('/build' step.build) %]">[% step.build %]</a></td>
|
||||
<td>[% IF step.busy >= 30 %]<a class="row-link" href="[% c.uri_for('/build' step.build 'nixlog' step.stepnr 'tail') %]">[% step.stepnr %]</a>[% ELSE; step.stepnr; END %]</td>
|
||||
<td><tt>[% step.drvpath.match('-(.*)').0 %]</tt></td>
|
||||
<td>[% INCLUDE renderBusyStatus %]</td>
|
||||
<td style="width: 10em">[% INCLUDE renderDuration duration = curTime - step.starttime %] </td>
|
||||
</tr>
|
||||
[% END %]
|
||||
|
||||
@@ -129,12 +129,6 @@ $(document).ready(function() {
|
||||
el.addClass("is-local");
|
||||
}
|
||||
});
|
||||
|
||||
[...document.getElementsByTagName("iframe")].forEach((element) => {
|
||||
element.contentWindow.addEventListener("DOMContentLoaded", (_) => {
|
||||
element.style.height = element.contentWindow.document.body.scrollHeight + 'px';
|
||||
})
|
||||
})
|
||||
});
|
||||
|
||||
var tabsLoaded = {};
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
|
||||
[% ELSE %]
|
||||
|
||||
[% INCLUDE renderBuildList builds=resource showSchedulingInfo=1 hideResultInfo=1 busy=1 showStepName=1 %]
|
||||
[% INCLUDE renderBuildList builds=resource showSchedulingInfo=1 hideResultInfo=1 busy=1 %]
|
||||
|
||||
[% END %]
|
||||
|
||||
|
||||
@@ -1,24 +0,0 @@
|
||||
<script type="text/javascript" src="[% c.uri_for("/static/js/jquery/jquery-3.4.1.min.js") %]"></script>
|
||||
<script type="text/javascript" src="[% c.uri_for("/static/js/jquery/jquery-ui-1.10.4.min.js") %]"></script>
|
||||
<script type="text/javascript" src="[% c.uri_for("/static/js/moment/moment-2.24.0.min.js") %]"></script>
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
|
||||
<link href="[% c.uri_for("/static/fontawesome/css/all.css") %]" rel="stylesheet" />
|
||||
<script type="text/javascript" src="[% c.uri_for("/static/js/popper.min.js") %]"></script>
|
||||
<script type="text/javascript" src="[% c.uri_for("/static/bootstrap/js/bootstrap.min.js") %]"></script>
|
||||
<link href="[% c.uri_for("/static/bootstrap/css/bootstrap.min.css") %]" rel="stylesheet" />
|
||||
|
||||
<!-- hydra.css may need to be moved to before boostrap to make the @media rule work. -->
|
||||
<link rel="stylesheet" href="[% c.uri_for("/static/css/hydra.css") %]" type="text/css" />
|
||||
<link rel="stylesheet" href="[% c.uri_for("/static/css/rotated-th.css") %]" type="text/css" />
|
||||
|
||||
<style>
|
||||
.popover { max-width: 40%; }
|
||||
</style>
|
||||
|
||||
<script type="text/javascript" src="[% c.uri_for("/static/js/bootbox.min.js") %]"></script>
|
||||
|
||||
<link rel="stylesheet" href="[% c.uri_for("/static/css/tree.css") %]" type="text/css" />
|
||||
|
||||
<script type="text/javascript" src="[% c.uri_for("/static/js/common.js") %]"></script>
|
||||
@@ -17,7 +17,6 @@ use Hydra::Helper::Nix;
|
||||
use Hydra::Model::DB;
|
||||
use Hydra::Plugin;
|
||||
use Hydra::Schema;
|
||||
use IPC::Run;
|
||||
use JSON::MaybeXS;
|
||||
use Net::Statsd;
|
||||
use Nix::Store;
|
||||
@@ -358,32 +357,22 @@ sub evalJobs {
|
||||
my @cmd;
|
||||
|
||||
if (defined $flakeRef) {
|
||||
my $nix_expr =
|
||||
"let " .
|
||||
"flake = builtins.getFlake (toString \"$flakeRef\"); " .
|
||||
"in " .
|
||||
"flake.hydraJobs " .
|
||||
"or flake.checks " .
|
||||
"or (throw \"flake '$flakeRef' does not provide any Hydra jobs or checks\")";
|
||||
|
||||
@cmd = ("nix-eval-jobs", "--expr", $nix_expr);
|
||||
@cmd = ("hydra-eval-jobs",
|
||||
"--flake", $flakeRef,
|
||||
"--gc-roots-dir", getGCRootsDir,
|
||||
"--max-jobs", 1);
|
||||
} else {
|
||||
my $nixExprInput = $inputInfo->{$nixExprInputName}->[0]
|
||||
or die "cannot find the input containing the job expression\n";
|
||||
|
||||
@cmd = ("nix-eval-jobs",
|
||||
@cmd = ("hydra-eval-jobs",
|
||||
"<" . $nixExprInputName . "/" . $nixExprPath . ">",
|
||||
"--gc-roots-dir", getGCRootsDir,
|
||||
"--max-jobs", 1,
|
||||
inputsToArgs($inputInfo));
|
||||
}
|
||||
|
||||
push @cmd, ("--gc-roots-dir", getGCRootsDir);
|
||||
push @cmd, ("--max-jobs", 1);
|
||||
push @cmd, "--meta";
|
||||
push @cmd, "--constituents";
|
||||
push @cmd, "--force-recurse";
|
||||
push @cmd, ("--option", "allow-import-from-derivation", "false") if $config->{allow_import_from_derivation} // "true" ne "true";
|
||||
push @cmd, ("--workers", $config->{evaluator_workers} // 1);
|
||||
push @cmd, ("--max-memory-size", $config->{evaluator_max_memory_size} // 4096);
|
||||
push @cmd, "--no-allow-import-from-derivation" if $config->{allow_import_from_derivation} // "true" ne "true";
|
||||
|
||||
if (defined $ENV{'HYDRA_DEBUG'}) {
|
||||
sub escape {
|
||||
@@ -395,40 +384,14 @@ sub evalJobs {
|
||||
print STDERR "evaluator: @escaped\n";
|
||||
}
|
||||
|
||||
my $evalProc = IPC::Run::start \@cmd,
|
||||
'>', IPC::Run::new_chunker, \my $out,
|
||||
'2>', \my $err;
|
||||
(my $res, my $jobsJSON, my $stderr) = captureStdoutStderr(21600, @cmd);
|
||||
die "hydra-eval-jobs returned " . ($res & 127 ? "signal $res" : "exit code " . ($res >> 8))
|
||||
. ":\n" . ($stderr ? decode("utf-8", $stderr) : "(no output)\n")
|
||||
if $res;
|
||||
|
||||
return sub {
|
||||
while (1) {
|
||||
$evalProc->pump;
|
||||
if (!defined $out && !defined $err) {
|
||||
$evalProc->finish;
|
||||
if ($?) {
|
||||
die "nix-eval-jobs returned " . ($? & 127 ? "signal $?" : "exit code " . ($? >> 8)) . "\n";
|
||||
}
|
||||
return;
|
||||
}
|
||||
print STDERR "$stderr";
|
||||
|
||||
if (defined $err) {
|
||||
print STDERR "$err";
|
||||
undef $err;
|
||||
}
|
||||
|
||||
if (defined $out && $out ne '') {
|
||||
my $job;
|
||||
try {
|
||||
$job = decode_json($out);
|
||||
} catch {
|
||||
warn "nix-eval-jobs sent invalid JSON.\n parse error: $_\n invalid json: $out\n";
|
||||
};
|
||||
undef $out;
|
||||
if (defined $job) {
|
||||
return $job;
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
return decode_json($jobsJSON);
|
||||
}
|
||||
|
||||
|
||||
@@ -457,7 +420,7 @@ sub checkBuild {
|
||||
my $firstOutputName = $outputNames[0];
|
||||
my $firstOutputPath = $buildInfo->{outputs}->{$firstOutputName};
|
||||
|
||||
my $jobName = $buildInfo->{attr} or die;
|
||||
my $jobName = $buildInfo->{jobName} or die;
|
||||
my $drvPath = $buildInfo->{drvPath} or die;
|
||||
|
||||
my $build;
|
||||
@@ -511,30 +474,9 @@ sub checkBuild {
|
||||
|
||||
my $time = time();
|
||||
|
||||
sub getMeta {
|
||||
my ($s, $def) = @_;
|
||||
return ($s || "") eq "" ? $def : $s;
|
||||
}
|
||||
|
||||
sub getMetaStrings {
|
||||
my ($v, $k, $acc) = @_;
|
||||
my $t = ref $v;
|
||||
|
||||
if ($t eq 'HASH') {
|
||||
push @$acc, $v->{$k} if exists $v->{$k};
|
||||
} elsif ($t eq 'ARRAY') {
|
||||
getMetaStrings($_, $k, $acc) foreach @$v;
|
||||
} elsif (defined $v) {
|
||||
push @$acc, $v;
|
||||
}
|
||||
}
|
||||
|
||||
sub getMetaConcatStrings {
|
||||
my ($v, $k) = @_;
|
||||
|
||||
my @strings;
|
||||
getMetaStrings($v, $k, \@strings);
|
||||
return join(", ", @strings) || undef;
|
||||
sub null {
|
||||
my ($s) = @_;
|
||||
return $s eq "" ? undef : $s;
|
||||
}
|
||||
|
||||
# Add the build to the database.
|
||||
@@ -542,19 +484,19 @@ sub checkBuild {
|
||||
{ timestamp => $time
|
||||
, jobset_id => $jobset->id
|
||||
, job => $jobName
|
||||
, description => getMeta($buildInfo->{meta}->{description}, undef)
|
||||
, license => getMetaConcatStrings($buildInfo->{meta}->{license}, "shortName")
|
||||
, homepage => getMeta($buildInfo->{meta}->{homepage}, undef)
|
||||
, maintainers => getMetaConcatStrings($buildInfo->{meta}->{maintainers}, "email")
|
||||
, maxsilent => getMeta($buildInfo->{meta}->{maxSilent}, 7200)
|
||||
, timeout => getMeta($buildInfo->{meta}->{timeout}, 36000)
|
||||
, nixname => $buildInfo->{name}
|
||||
, description => null($buildInfo->{description})
|
||||
, license => null($buildInfo->{license})
|
||||
, homepage => null($buildInfo->{homepage})
|
||||
, maintainers => null($buildInfo->{maintainers})
|
||||
, maxsilent => $buildInfo->{maxSilent}
|
||||
, timeout => $buildInfo->{timeout}
|
||||
, nixname => $buildInfo->{nixName}
|
||||
, drvpath => $drvPath
|
||||
, system => $buildInfo->{system}
|
||||
, priority => getMeta($buildInfo->{meta}->{schedulingPriority}, 100)
|
||||
, priority => $buildInfo->{schedulingPriority}
|
||||
, finished => 0
|
||||
, iscurrent => 1
|
||||
, ischannel => getMeta($buildInfo->{meta}->{isChannel}, 0)
|
||||
, ischannel => $buildInfo->{isChannel}
|
||||
});
|
||||
|
||||
$build->buildoutputs->create({ name => $_, path => $buildInfo->{outputs}->{$_} })
|
||||
@@ -723,7 +665,7 @@ sub checkJobsetWrapped {
|
||||
return;
|
||||
}
|
||||
|
||||
# Hash the arguments to nix-eval-jobs and check the
|
||||
# Hash the arguments to hydra-eval-jobs and check the
|
||||
# JobsetInputHashes to see if the previous evaluation had the same
|
||||
# inputs. If so, bail out.
|
||||
my @args = ($jobset->nixexprinput // "", $jobset->nixexprpath // "", inputsToArgs($inputInfo));
|
||||
@@ -745,12 +687,19 @@ sub checkJobsetWrapped {
|
||||
|
||||
# Evaluate the job expression.
|
||||
my $evalStart = clock_gettime(CLOCK_MONOTONIC);
|
||||
my $evalStop;
|
||||
my $jobsIter = evalJobs($project->name . ":" . $jobset->name, $inputInfo, $jobset->nixexprinput, $jobset->nixexprpath, $flakeRef);
|
||||
my $jobs = evalJobs($project->name . ":" . $jobset->name, $inputInfo, $jobset->nixexprinput, $jobset->nixexprpath, $flakeRef);
|
||||
my $evalStop = clock_gettime(CLOCK_MONOTONIC);
|
||||
|
||||
if ($jobsetsJobset) {
|
||||
my @keys = keys %$jobs;
|
||||
die "The .jobsets jobset must only have a single job named 'jobsets'"
|
||||
unless (scalar @keys) == 1 && $keys[0] eq "jobsets";
|
||||
}
|
||||
Net::Statsd::timing("hydra.evaluator.eval_time", int(($evalStop - $evalStart) * 1000));
|
||||
|
||||
if ($dryRun) {
|
||||
while (defined(my $job = $jobsIter->())) {
|
||||
my $name = $job->{attr};
|
||||
foreach my $name (keys %{$jobs}) {
|
||||
my $job = $jobs->{$name};
|
||||
if (defined $job->{drvPath}) {
|
||||
print STDERR "good job $name: $job->{drvPath}\n";
|
||||
} else {
|
||||
@@ -760,23 +709,36 @@ sub checkJobsetWrapped {
|
||||
return;
|
||||
}
|
||||
|
||||
die "Jobset contains a job with an empty name. Make sure the jobset evaluates to an attrset of jobs.\n"
|
||||
if defined $jobs->{""};
|
||||
|
||||
$jobs->{$_}->{jobName} = $_ for keys %{$jobs};
|
||||
|
||||
my $jobOutPathMap = {};
|
||||
my $jobsetChanged = 0;
|
||||
my $dbStart = clock_gettime(CLOCK_MONOTONIC);
|
||||
|
||||
|
||||
# Store the error messages for jobs that failed to evaluate.
|
||||
my $evaluationErrorTime = time;
|
||||
my $evaluationErrorMsg = "";
|
||||
foreach my $job (values %{$jobs}) {
|
||||
next unless defined $job->{error};
|
||||
$evaluationErrorMsg .=
|
||||
($job->{jobName} ne "" ? "in job ‘$job->{jobName}’" : "at top-level") .
|
||||
":\n" . $job->{error} . "\n\n";
|
||||
}
|
||||
setJobsetError($jobset, $evaluationErrorMsg, $evaluationErrorTime);
|
||||
|
||||
my $evaluationErrorRecord = $db->resultset('EvaluationErrors')->create(
|
||||
{ errormsg => $evaluationErrorMsg
|
||||
, errortime => $evaluationErrorTime
|
||||
}
|
||||
);
|
||||
|
||||
my $jobOutPathMap = {};
|
||||
my $jobsetChanged = 0;
|
||||
my %buildMap;
|
||||
|
||||
my @jobs;
|
||||
push @jobs, $_ while defined($_ = $jobsIter->());
|
||||
|
||||
$db->txn_do(sub {
|
||||
|
||||
my $prevEval = getPrevJobsetEval($db, $jobset, 1);
|
||||
|
||||
# Clear the "current" flag on all builds. Since we're in a
|
||||
@@ -789,7 +751,7 @@ sub checkJobsetWrapped {
|
||||
, evaluationerror => $evaluationErrorRecord
|
||||
, timestamp => time
|
||||
, checkouttime => abs(int($checkoutStop - $checkoutStart))
|
||||
, evaltime => 0
|
||||
, evaltime => abs(int($evalStop - $evalStart))
|
||||
, hasnewbuilds => 0
|
||||
, nrbuilds => 0
|
||||
, flake => $flakeRef
|
||||
@@ -797,24 +759,11 @@ sub checkJobsetWrapped {
|
||||
, nixexprpath => $jobset->nixexprpath
|
||||
});
|
||||
|
||||
my @jobsWithConstituents;
|
||||
|
||||
foreach my $job (@jobs) {
|
||||
if ($jobsetsJobset) {
|
||||
die "The .jobsets jobset must only have a single job named 'jobsets'"
|
||||
unless $job->{attr} eq "jobsets";
|
||||
}
|
||||
|
||||
$evaluationErrorMsg .=
|
||||
($job->{attr} ne "" ? "in job ‘$job->{attr}’" : "at top-level") .
|
||||
":\n" . $job->{error} . "\n\n" if defined $job->{error};
|
||||
|
||||
checkBuild($db, $jobset, $ev, $inputInfo, $job, \%buildMap, $prevEval, $jobOutPathMap, $plugins)
|
||||
unless defined $job->{error};
|
||||
|
||||
if (defined $job->{constituents}) {
|
||||
push @jobsWithConstituents, $job;
|
||||
}
|
||||
# Schedule each successfully evaluated job.
|
||||
foreach my $job (permute(values %{$jobs})) {
|
||||
next if defined $job->{error};
|
||||
#print STDERR "considering job " . $project->name, ":", $jobset->name, ":", $job->{jobName} . "\n";
|
||||
checkBuild($db, $jobset, $ev, $inputInfo, $job, \%buildMap, $prevEval, $jobOutPathMap, $plugins);
|
||||
}
|
||||
|
||||
# Have any builds been added or removed since last time?
|
||||
@@ -852,20 +801,21 @@ sub checkJobsetWrapped {
|
||||
$drvPathToId{$x->{drvPath}} = $x;
|
||||
}
|
||||
|
||||
foreach my $job (values @jobsWithConstituents) {
|
||||
next unless defined $job->{constituents};
|
||||
foreach my $job (values %{$jobs}) {
|
||||
next unless $job->{constituents};
|
||||
|
||||
if (defined $job->{error}) {
|
||||
die "aggregate job ‘$job->{attr}’ failed with the error: $job->{error}\n";
|
||||
die "aggregate job ‘$job->{jobName}’ failed with the error: $job->{error}\n";
|
||||
}
|
||||
|
||||
my $x = $drvPathToId{$job->{drvPath}} or
|
||||
die "aggregate job ‘$job->{attr}’ has no corresponding build record.\n";
|
||||
die "aggregate job ‘$job->{jobName}’ has no corresponding build record.\n";
|
||||
foreach my $drvPath (@{$job->{constituents}}) {
|
||||
my $constituent = $drvPathToId{$drvPath};
|
||||
if (defined $constituent) {
|
||||
$db->resultset('AggregateConstituents')->update_or_create({aggregate => $x->{id}, constituent => $constituent->{id}});
|
||||
} else {
|
||||
warn "aggregate job ‘$job->{attr}’ has a constituent ‘$drvPath’ that doesn't correspond to a Hydra build\n";
|
||||
warn "aggregate job ‘$job->{jobName}’ has a constituent ‘$drvPath’ that doesn't correspond to a Hydra build\n";
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -907,15 +857,11 @@ sub checkJobsetWrapped {
|
||||
$jobset->update({ enabled => 0 }) if $jobset->enabled == 2;
|
||||
|
||||
$jobset->update({ lastcheckedtime => time, forceeval => undef });
|
||||
|
||||
$evaluationErrorRecord->update({ errormsg => $evaluationErrorMsg });
|
||||
setJobsetError($jobset, $evaluationErrorMsg, $evaluationErrorTime);
|
||||
|
||||
$evalStop = clock_gettime(CLOCK_MONOTONIC);
|
||||
$ev->update({ evaltime => abs(int($evalStop - $evalStart)) });
|
||||
});
|
||||
|
||||
Net::Statsd::timing("hydra.evaluator.eval_time", int(($evalStop - $evalStart) * 1000));
|
||||
my $dbStop = clock_gettime(CLOCK_MONOTONIC);
|
||||
|
||||
Net::Statsd::timing("hydra.evaluator.db_time", int(($dbStop - $dbStart) * 1000));
|
||||
Net::Statsd::increment("hydra.evaluator.evals");
|
||||
Net::Statsd::increment("hydra.evaluator.cached_evals") unless $jobsetChanged;
|
||||
}
|
||||
|
||||
@@ -32,9 +32,4 @@ subtest "/jobset/PROJECT/JOBSET/evals" => sub {
|
||||
ok($jobsetevals->is_success, "The page showing the jobset evals returns 200.");
|
||||
};
|
||||
|
||||
subtest "/jobset/PROJECT/JOBSET/errors" => sub {
|
||||
my $jobsetevals = request(GET '/jobset/' . $project->name . '/' . $jobset->name . '/errors');
|
||||
ok($jobsetevals->is_success, "The page showing the jobset eval errors returns 200.");
|
||||
};
|
||||
|
||||
done_testing;
|
||||
|
||||
@@ -35,10 +35,6 @@ subtest "Fetching the eval's overview" => sub {
|
||||
is($fetch->code, 200, "channel page is 200");
|
||||
};
|
||||
|
||||
subtest "Fetching the eval's overview" => sub {
|
||||
my $fetch = request(GET '/eval/' . $eval->id, '/errors');
|
||||
is($fetch->code, 200, "errors page is 200");
|
||||
};
|
||||
|
||||
|
||||
done_testing;
|
||||
|
||||
@@ -6,55 +6,27 @@ use Hydra::Helper::Exec;
|
||||
|
||||
my $ctx = test_context();
|
||||
|
||||
subtest "broken constituents expression" => sub {
|
||||
my $jobsetCtx = $ctx->makeJobset(
|
||||
expression => 'constituents-broken.nix',
|
||||
);
|
||||
my $jobset = $jobsetCtx->{"jobset"};
|
||||
my $jobsetCtx = $ctx->makeJobset(
|
||||
expression => 'constituents-broken.nix',
|
||||
);
|
||||
my $jobset = $jobsetCtx->{"jobset"};
|
||||
|
||||
my ($res, $stdout, $stderr) = captureStdoutStderr(60,
|
||||
("hydra-eval-jobset", $jobsetCtx->{"project"}->name, $jobset->name)
|
||||
);
|
||||
isnt($res, 0, "hydra-eval-jobset exits non-zero");
|
||||
ok(utf8::decode($stderr), "Stderr output is UTF8-clean");
|
||||
like(
|
||||
$stderr,
|
||||
qr/aggregate job 'mixed_aggregate' references non-existent job 'constituentA'/,
|
||||
"The stderr record includes a relevant error message"
|
||||
);
|
||||
my ($res, $stdout, $stderr) = captureStdoutStderr(60,
|
||||
("hydra-eval-jobset", $jobsetCtx->{"project"}->name, $jobset->name)
|
||||
);
|
||||
isnt($res, 0, "hydra-eval-jobset exits non-zero");
|
||||
ok(utf8::decode($stderr), "Stderr output is UTF8-clean");
|
||||
like(
|
||||
$stderr,
|
||||
qr/aggregate job ‘mixed_aggregate’ failed with the error: constituentA: does not exist/,
|
||||
"The stderr record includes a relevant error message"
|
||||
);
|
||||
|
||||
$jobset->discard_changes({ '+columns' => {'errormsg' => 'errormsg'} }); # refresh from DB
|
||||
like(
|
||||
$jobset->errormsg,
|
||||
qr/aggregate job ‘mixed_aggregate’ failed with the error: constituentA: does not exist/,
|
||||
"The jobset records a relevant error message"
|
||||
);
|
||||
};
|
||||
|
||||
subtest "no matches" => sub {
|
||||
my $jobsetCtx = $ctx->makeJobset(
|
||||
expression => 'constituents-no-matches.nix',
|
||||
);
|
||||
my $jobset = $jobsetCtx->{"jobset"};
|
||||
|
||||
my ($res, $stdout, $stderr) = captureStdoutStderr(60,
|
||||
("hydra-eval-jobset", $jobsetCtx->{"project"}->name, $jobset->name)
|
||||
);
|
||||
isnt($res, 0, "hydra-eval-jobset exits non-zero");
|
||||
ok(utf8::decode($stderr), "Stderr output is UTF8-clean");
|
||||
like(
|
||||
$stderr,
|
||||
qr/aggregate job 'non_match_aggregate' references constituent glob pattern 'tests\.\*' with no matches/,
|
||||
"The stderr record includes a relevant error message"
|
||||
);
|
||||
|
||||
$jobset->discard_changes({ '+columns' => {'errormsg' => 'errormsg'} }); # refresh from DB
|
||||
like(
|
||||
$jobset->errormsg,
|
||||
qr/aggregate job ‘non_match_aggregate’ failed with the error: tests\.\*: constituent glob pattern had no matches/,
|
||||
qr/in job ‘non_match_aggregate’:\ntests\.\*: constituent glob pattern had no matches/,
|
||||
"The jobset records a relevant error message"
|
||||
);
|
||||
};
|
||||
$jobset->discard_changes; # refresh from DB
|
||||
like(
|
||||
$jobset->errormsg,
|
||||
qr/aggregate job ‘mixed_aggregate’ failed with the error: constituentA: does not exist/,
|
||||
"The jobset records a relevant error message"
|
||||
);
|
||||
|
||||
done_testing;
|
||||
|
||||
@@ -5,58 +5,13 @@ use Test2::V0;
|
||||
|
||||
my $ctx = test_context();
|
||||
|
||||
my $expression = 'constituents.nix';
|
||||
my $jobsetCtx = $ctx->makeJobset(
|
||||
expression => $expression,
|
||||
);
|
||||
my $builds = $ctx->evaluateJobset(
|
||||
jobset => $jobsetCtx->{"jobset"},
|
||||
expression => $expression,
|
||||
build => 0,
|
||||
my $builds = $ctx->makeAndEvaluateJobset(
|
||||
expression => 'constituents.nix',
|
||||
);
|
||||
|
||||
my $constituentA = $builds->{"constituentA"};
|
||||
my $directAggregate = $builds->{"direct_aggregate"};
|
||||
my $indirectAggregate = $builds->{"indirect_aggregate"};
|
||||
my $mixedAggregate = $builds->{"mixed_aggregate"};
|
||||
|
||||
# Ensure that we get exactly the aggregates we expect
|
||||
my %expected_constituents = (
|
||||
'direct_aggregate' => {
|
||||
'constituentA' => 1,
|
||||
},
|
||||
'indirect_aggregate' => {
|
||||
'constituentA' => 1,
|
||||
},
|
||||
'mixed_aggregate' => {
|
||||
# Note that `constituentA_alias` becomes `constituentA`, because
|
||||
# the shorter name is preferred
|
||||
'constituentA' => 1,
|
||||
'constituentB' => 1,
|
||||
},
|
||||
);
|
||||
|
||||
my $rs = $ctx->db->resultset('AggregateConstituents')->search(
|
||||
{},
|
||||
{
|
||||
join => [ 'aggregate', 'constituent' ], # Use correct relationship names
|
||||
columns => [],
|
||||
'+select' => [ 'aggregate.job', 'constituent.job' ],
|
||||
'+as' => [ 'aggregate_job', 'constituent_job' ],
|
||||
}
|
||||
);
|
||||
|
||||
my %actual_constituents;
|
||||
while (my $row = $rs->next) {
|
||||
my $aggregate_job = $row->get_column('aggregate_job');
|
||||
my $constituent_job = $row->get_column('constituent_job');
|
||||
$actual_constituents{$aggregate_job} //= {};
|
||||
$actual_constituents{$aggregate_job}{$constituent_job} = 1;
|
||||
}
|
||||
|
||||
is(\%actual_constituents, \%expected_constituents, "Exact aggregate constituents as expected");
|
||||
|
||||
# Check that deletion also doesn't work accordingly
|
||||
|
||||
is(system('nix-store', '--delete', $constituentA->drvpath), 256, "Deleting a constituent derivation fails");
|
||||
is(system('nix-store', '--delete', $directAggregate->drvpath), 256, "Deleting the direct aggregate derivation fails");
|
||||
|
||||
@@ -1,138 +0,0 @@
|
||||
use strict;
|
||||
use warnings;
|
||||
use Setup;
|
||||
use Test2::V0;
|
||||
use Hydra::Helper::Exec;
|
||||
use Data::Dumper;
|
||||
|
||||
my $ctx = test_context();
|
||||
|
||||
subtest "general glob testing" => sub {
|
||||
my $jobsetCtx = $ctx->makeJobset(
|
||||
expression => 'constituents-glob.nix',
|
||||
);
|
||||
my $jobset = $jobsetCtx->{"jobset"};
|
||||
|
||||
my ($res, $stdout, $stderr) = captureStdoutStderr(60,
|
||||
("hydra-eval-jobset", $jobsetCtx->{"project"}->name, $jobset->name)
|
||||
);
|
||||
is($res, 0, "hydra-eval-jobset exits zero");
|
||||
|
||||
my $builds = {};
|
||||
for my $build ($jobset->builds) {
|
||||
$builds->{$build->job} = $build;
|
||||
}
|
||||
|
||||
subtest "basic globbing works" => sub {
|
||||
ok(defined $builds->{"ok_aggregate"}, "'ok_aggregate' is part of the jobset evaluation");
|
||||
my @constituents = $builds->{"ok_aggregate"}->constituents->all;
|
||||
is(2, scalar @constituents, "'ok_aggregate' has two constituents");
|
||||
|
||||
my @sortedConstituentNames = sort (map { $_->nixname } @constituents);
|
||||
|
||||
is($sortedConstituentNames[0], "empty-dir-A", "first constituent of 'ok_aggregate' is 'empty-dir-A'");
|
||||
is($sortedConstituentNames[1], "empty-dir-B", "second constituent of 'ok_aggregate' is 'empty-dir-B'");
|
||||
};
|
||||
|
||||
subtest "transitivity is OK" => sub {
|
||||
ok(defined $builds->{"indirect_aggregate"}, "'indirect_aggregate' is part of the jobset evaluation");
|
||||
my @constituents = $builds->{"indirect_aggregate"}->constituents->all;
|
||||
is(1, scalar @constituents, "'indirect_aggregate' has one constituent");
|
||||
is($constituents[0]->nixname, "direct_aggregate", "'indirect_aggregate' has 'direct_aggregate' as single constituent");
|
||||
};
|
||||
};
|
||||
|
||||
subtest "* selects all except current aggregate" => sub {
|
||||
my $jobsetCtx = $ctx->makeJobset(
|
||||
expression => 'constituents-glob-all.nix',
|
||||
);
|
||||
my $jobset = $jobsetCtx->{"jobset"};
|
||||
|
||||
my ($res, $stdout, $stderr) = captureStdoutStderr(60,
|
||||
("hydra-eval-jobset", $jobsetCtx->{"project"}->name, $jobset->name)
|
||||
);
|
||||
|
||||
subtest "no eval errors" => sub {
|
||||
ok(utf8::decode($stderr), "Stderr output is UTF8-clean");
|
||||
ok(
|
||||
$stderr !~ "aggregate job ‘ok_aggregate’ has a constituent .* that doesn't correspond to a Hydra build",
|
||||
"Catchall wildcard must not select itself as constituent"
|
||||
);
|
||||
|
||||
$jobset->discard_changes; # refresh from DB
|
||||
is(
|
||||
$jobset->has_error,
|
||||
0,
|
||||
"eval-errors non-empty"
|
||||
);
|
||||
};
|
||||
|
||||
my $builds = {};
|
||||
for my $build ($jobset->builds) {
|
||||
$builds->{$build->job} = $build;
|
||||
}
|
||||
|
||||
subtest "two constituents" => sub {
|
||||
ok(defined $builds->{"ok_aggregate"}, "'ok_aggregate' is part of the jobset evaluation");
|
||||
my @constituents = $builds->{"ok_aggregate"}->constituents->all;
|
||||
is(2, scalar @constituents, "'ok_aggregate' has two constituents");
|
||||
|
||||
my @sortedConstituentNames = sort (map { $_->nixname } @constituents);
|
||||
|
||||
is($sortedConstituentNames[0], "empty-dir-A", "first constituent of 'ok_aggregate' is 'empty-dir-A'");
|
||||
is($sortedConstituentNames[1], "empty-dir-B", "second constituent of 'ok_aggregate' is 'empty-dir-B'");
|
||||
};
|
||||
};
|
||||
|
||||
subtest "trivial cycle check" => sub {
|
||||
my $jobsetCtx = $ctx->makeJobset(
|
||||
expression => 'constituents-cycle.nix',
|
||||
);
|
||||
my $jobset = $jobsetCtx->{"jobset"};
|
||||
|
||||
my ($res, $stdout, $stderr) = captureStdoutStderr(60,
|
||||
("hydra-eval-jobset", $jobsetCtx->{"project"}->name, $jobset->name)
|
||||
);
|
||||
|
||||
ok(
|
||||
$stderr =~ "Found dependency cycle between jobs 'indirect_aggregate' and 'ok_aggregate'",
|
||||
"Dependency cycle error is on stderr"
|
||||
);
|
||||
|
||||
ok(utf8::decode($stderr), "Stderr output is UTF8-clean");
|
||||
|
||||
$jobset->discard_changes({ '+columns' => {'errormsg' => 'errormsg'} }); # refresh from DB
|
||||
like(
|
||||
$jobset->errormsg,
|
||||
qr/Dependency cycle: indirect_aggregate <-> ok_aggregate/,
|
||||
"eval-errors non-empty"
|
||||
);
|
||||
|
||||
is(0, $jobset->builds->count, "No builds should be scheduled");
|
||||
};
|
||||
|
||||
subtest "cycle check with globbing" => sub {
|
||||
my $jobsetCtx = $ctx->makeJobset(
|
||||
expression => 'constituents-cycle-glob.nix',
|
||||
);
|
||||
my $jobset = $jobsetCtx->{"jobset"};
|
||||
|
||||
my ($res, $stdout, $stderr) = captureStdoutStderr(60,
|
||||
("hydra-eval-jobset", $jobsetCtx->{"project"}->name, $jobset->name)
|
||||
);
|
||||
|
||||
ok(utf8::decode($stderr), "Stderr output is UTF8-clean");
|
||||
|
||||
$jobset->discard_changes({ '+columns' => {'errormsg' => 'errormsg'} }); # refresh from DB
|
||||
like(
|
||||
$jobset->errormsg,
|
||||
qr/aggregate job ‘indirect_aggregate’ failed with the error: Dependency cycle: indirect_aggregate <-> packages.constituentA/,
|
||||
"packages.constituentA error missing"
|
||||
);
|
||||
|
||||
# on this branch of Hydra, hydra-eval-jobset fails hard if an aggregate
|
||||
# job is broken.
|
||||
is(0, $jobset->builds->count, "Zero jobs are scheduled");
|
||||
};
|
||||
|
||||
done_testing;
|
||||
@@ -1,67 +0,0 @@
|
||||
use feature 'unicode_strings';
|
||||
use strict;
|
||||
use warnings;
|
||||
use Setup;
|
||||
use Test2::V0;
|
||||
use File::Copy qw(cp);
|
||||
|
||||
my $ctx = test_context(
|
||||
nix_config => qq|
|
||||
experimental-features = nix-command flakes
|
||||
|,
|
||||
hydra_config => q|
|
||||
<runcommand>
|
||||
evaluator_pure_eval = false
|
||||
</runcommand>
|
||||
|
|
||||
);
|
||||
|
||||
sub checkFlake {
|
||||
my ($flake) = @_;
|
||||
|
||||
cp($ctx->jobsdir . "/basic.nix", $ctx->jobsdir . "/" . $flake);
|
||||
cp($ctx->jobsdir . "/config.nix", $ctx->jobsdir . "/" . $flake);
|
||||
cp($ctx->jobsdir . "/empty-dir-builder.sh", $ctx->jobsdir . "/" . $flake);
|
||||
cp($ctx->jobsdir . "/fail.sh", $ctx->jobsdir . "/" . $flake);
|
||||
cp($ctx->jobsdir . "/succeed-with-failed.sh", $ctx->jobsdir . "/" . $flake);
|
||||
|
||||
chmod 0755, $ctx->jobsdir . "/" . $flake . "/empty-dir-builder.sh";
|
||||
chmod 0755, $ctx->jobsdir . "/" . $flake . "/fail.sh";
|
||||
chmod 0755, $ctx->jobsdir . "/" . $flake . "/succeed-with-failed.sh";
|
||||
|
||||
my $builds = $ctx->makeAndEvaluateJobset(
|
||||
flake => 'path:' . $ctx->jobsdir . "/" . $flake,
|
||||
build => 1
|
||||
);
|
||||
|
||||
subtest "Build: succeed_with_failed" => sub {
|
||||
my $build = $builds->{"succeed_with_failed"};
|
||||
|
||||
is($build->finished, 1, "Build should be finished.");
|
||||
is($build->buildstatus, 6, "succeeeded-but-failed should have buildstatus 6.");
|
||||
};
|
||||
|
||||
subtest "Build: empty_dir" => sub {
|
||||
my $build = $builds->{"empty_dir"};
|
||||
|
||||
is($build->finished, 1, "Build should be finished.");
|
||||
is($build->buildstatus, 0, "Should have succeeded.");
|
||||
};
|
||||
|
||||
subtest "Build: fails" => sub {
|
||||
my $build = $builds->{"fails"};
|
||||
|
||||
is($build->finished, 1, "Build should be finished.");
|
||||
is($build->buildstatus, 1, "Should have failed.");
|
||||
};
|
||||
}
|
||||
|
||||
subtest "Flake using `checks`" => sub {
|
||||
checkFlake 'flake-checks'
|
||||
};
|
||||
|
||||
subtest "Flake using `hydraJobs`" => sub {
|
||||
checkFlake 'flake-hydraJobs'
|
||||
};
|
||||
|
||||
done_testing;
|
||||
@@ -1,22 +0,0 @@
|
||||
use feature 'unicode_strings';
|
||||
use strict;
|
||||
use warnings;
|
||||
use Setup;
|
||||
use Test2::V0;
|
||||
|
||||
my $ctx = test_context();
|
||||
|
||||
my $builds = $ctx->makeAndEvaluateJobset(
|
||||
expression => "meta.nix",
|
||||
build => 1
|
||||
);
|
||||
|
||||
my $build = $builds->{"full-of-meta"};
|
||||
|
||||
is($build->finished, 1, "Build should be finished.");
|
||||
is($build->description, "This is the description of the job.", "Wrong description extracted from the build.");
|
||||
is($build->license, "MIT, BSD", "Wrong licenses extracted from the build.");
|
||||
is($build->homepage, "https://example.com/", "Wrong homepage extracted from the build.");
|
||||
is($build->maintainers, 'alice@example.com, bob@not.found', "Wrong maintainers extracted from the build.");
|
||||
|
||||
done_testing;
|
||||
@@ -1,14 +0,0 @@
|
||||
rec {
|
||||
path = "/nix/store/l9mg93sgx50y88p5rr6x1vib6j1rjsds-coreutils-9.1/bin";
|
||||
|
||||
mkDerivation = args:
|
||||
derivation ({
|
||||
system = builtins.currentSystem;
|
||||
PATH = path;
|
||||
} // args);
|
||||
mkContentAddressedDerivation = args: mkDerivation ({
|
||||
__contentAddressed = true;
|
||||
outputHashMode = "recursive";
|
||||
outputHashAlgo = "sha256";
|
||||
} // args);
|
||||
}
|
||||
@@ -1,34 +0,0 @@
|
||||
with import ./config.nix;
|
||||
{
|
||||
packages.constituentA = mkDerivation {
|
||||
name = "empty-dir-A";
|
||||
builder = ./empty-dir-builder.sh;
|
||||
_hydraAggregate = true;
|
||||
_hydraGlobConstituents = true;
|
||||
constituents = [ "*_aggregate" ];
|
||||
};
|
||||
|
||||
packages.constituentB = mkDerivation {
|
||||
name = "empty-dir-B";
|
||||
builder = ./empty-dir-builder.sh;
|
||||
};
|
||||
|
||||
ok_aggregate = mkDerivation {
|
||||
name = "direct_aggregate";
|
||||
_hydraAggregate = true;
|
||||
_hydraGlobConstituents = true;
|
||||
constituents = [
|
||||
"packages.*"
|
||||
];
|
||||
builder = ./empty-dir-builder.sh;
|
||||
};
|
||||
|
||||
indirect_aggregate = mkDerivation {
|
||||
name = "indirect_aggregate";
|
||||
_hydraAggregate = true;
|
||||
constituents = [
|
||||
"ok_aggregate"
|
||||
];
|
||||
builder = ./empty-dir-builder.sh;
|
||||
};
|
||||
}
|
||||
@@ -1,21 +0,0 @@
|
||||
with import ./config.nix;
|
||||
{
|
||||
ok_aggregate = mkDerivation {
|
||||
name = "direct_aggregate";
|
||||
_hydraAggregate = true;
|
||||
_hydraGlobConstituents = true;
|
||||
constituents = [
|
||||
"indirect_aggregate"
|
||||
];
|
||||
builder = ./empty-dir-builder.sh;
|
||||
};
|
||||
|
||||
indirect_aggregate = mkDerivation {
|
||||
name = "indirect_aggregate";
|
||||
_hydraAggregate = true;
|
||||
constituents = [
|
||||
"ok_aggregate"
|
||||
];
|
||||
builder = ./empty-dir-builder.sh;
|
||||
};
|
||||
}
|
||||
@@ -1,22 +0,0 @@
|
||||
with import ./config.nix;
|
||||
{
|
||||
packages.constituentA = mkDerivation {
|
||||
name = "empty-dir-A";
|
||||
builder = ./empty-dir-builder.sh;
|
||||
};
|
||||
|
||||
packages.constituentB = mkDerivation {
|
||||
name = "empty-dir-B";
|
||||
builder = ./empty-dir-builder.sh;
|
||||
};
|
||||
|
||||
ok_aggregate = mkDerivation {
|
||||
name = "direct_aggregate";
|
||||
_hydraAggregate = true;
|
||||
_hydraGlobConstituents = true;
|
||||
constituents = [
|
||||
"*"
|
||||
];
|
||||
builder = ./empty-dir-builder.sh;
|
||||
};
|
||||
}
|
||||
@@ -1,31 +0,0 @@
|
||||
with import ./config.nix;
|
||||
{
|
||||
packages.constituentA = mkDerivation {
|
||||
name = "empty-dir-A";
|
||||
builder = ./empty-dir-builder.sh;
|
||||
};
|
||||
|
||||
packages.constituentB = mkDerivation {
|
||||
name = "empty-dir-B";
|
||||
builder = ./empty-dir-builder.sh;
|
||||
};
|
||||
|
||||
ok_aggregate = mkDerivation {
|
||||
name = "direct_aggregate";
|
||||
_hydraAggregate = true;
|
||||
_hydraGlobConstituents = true;
|
||||
constituents = [
|
||||
"packages.*"
|
||||
];
|
||||
builder = ./empty-dir-builder.sh;
|
||||
};
|
||||
|
||||
indirect_aggregate = mkDerivation {
|
||||
name = "indirect_aggregate";
|
||||
_hydraAggregate = true;
|
||||
constituents = [
|
||||
"ok_aggregate"
|
||||
];
|
||||
builder = ./empty-dir-builder.sh;
|
||||
};
|
||||
}
|
||||
@@ -1,20 +0,0 @@
|
||||
with import ./config.nix;
|
||||
{
|
||||
non_match_aggregate = mkDerivation {
|
||||
name = "mixed_aggregate";
|
||||
_hydraAggregate = true;
|
||||
_hydraGlobConstituents = true;
|
||||
constituents = [
|
||||
"tests.*"
|
||||
];
|
||||
builder = ./empty-dir-builder.sh;
|
||||
};
|
||||
|
||||
# Without a second job no jobset is attempted to be created
|
||||
# (the only job would be broken)
|
||||
# and thus the constituent validation is never reached.
|
||||
dummy = mkDerivation {
|
||||
name = "dummy";
|
||||
builder = ./empty-dir-builder.sh;
|
||||
};
|
||||
}
|
||||
@@ -5,8 +5,6 @@ rec {
|
||||
builder = ./empty-dir-builder.sh;
|
||||
};
|
||||
|
||||
constituentA_alias = constituentA;
|
||||
|
||||
constituentB = mkDerivation {
|
||||
name = "empty-dir-B";
|
||||
builder = ./empty-dir-builder.sh;
|
||||
@@ -34,7 +32,7 @@ rec {
|
||||
name = "mixed_aggregate";
|
||||
_hydraAggregate = true;
|
||||
constituents = [
|
||||
"constituentA_alias"
|
||||
"constituentA"
|
||||
constituentB
|
||||
];
|
||||
builder = ./empty-dir-builder.sh;
|
||||
|
||||
@@ -1,24 +0,0 @@
|
||||
{
|
||||
"enabled": 1,
|
||||
"hidden": false,
|
||||
"description": "declarative-jobset-example",
|
||||
"nixexprinput": "src",
|
||||
"nixexprpath": "declarative/generator.nix",
|
||||
"checkinterval": 300,
|
||||
"schedulingshares": 100,
|
||||
"enableemail": false,
|
||||
"emailoverride": "",
|
||||
"keepnr": 3,
|
||||
"inputs": {
|
||||
"src": {
|
||||
"type": "path",
|
||||
"value": "/home/ma27/Projects/hydra-cppnix/t/jobs",
|
||||
"emailresponsible": false
|
||||
},
|
||||
"jobspath": {
|
||||
"type": "string",
|
||||
"value": "/home/ma27/Projects/hydra-cppnix/t/jobs",
|
||||
"emailresponsible": false
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,6 +0,0 @@
|
||||
{
|
||||
outputs = { ... }: {
|
||||
checks =
|
||||
import ./basic.nix;
|
||||
};
|
||||
}
|
||||
@@ -1,6 +0,0 @@
|
||||
{
|
||||
outputs = { ... }: {
|
||||
hydraJobs =
|
||||
import ./basic.nix;
|
||||
};
|
||||
}
|
||||
@@ -1,17 +0,0 @@
|
||||
with import ./config.nix;
|
||||
{
|
||||
full-of-meta =
|
||||
mkDerivation {
|
||||
name = "full-of-meta";
|
||||
builder = ./empty-dir-builder.sh;
|
||||
|
||||
meta = {
|
||||
description = "This is the description of the job.";
|
||||
license = [ { shortName = "MIT"; } "BSD" ];
|
||||
homepage = "https://example.com/";
|
||||
maintainers = [ "alice@example.com" { email = "bob@not.found"; } ];
|
||||
|
||||
outPath = "${placeholder "out"}";
|
||||
};
|
||||
};
|
||||
}
|
||||
@@ -14,7 +14,7 @@ our @EXPORT = qw(
|
||||
sub evalSucceeds {
|
||||
my ($jobset) = @_;
|
||||
my ($res, $stdout, $stderr) = captureStdoutStderr(60, ("hydra-eval-jobset", $jobset->project->name, $jobset->name));
|
||||
$jobset->discard_changes({ '+columns' => {'errormsg' => 'errormsg'} }); # refresh from DB
|
||||
$jobset->discard_changes; # refresh from DB
|
||||
if ($res) {
|
||||
chomp $stdout; chomp $stderr;
|
||||
utf8::decode($stdout) or die "Invalid unicode in stdout.";
|
||||
@@ -29,7 +29,7 @@ sub evalSucceeds {
|
||||
sub evalFails {
|
||||
my ($jobset) = @_;
|
||||
my ($res, $stdout, $stderr) = captureStdoutStderr(60, ("hydra-eval-jobset", $jobset->project->name, $jobset->name));
|
||||
$jobset->discard_changes({ '+columns' => {'errormsg' => 'errormsg'} }); # refresh from DB
|
||||
$jobset->discard_changes; # refresh from DB
|
||||
if (!$res) {
|
||||
chomp $stdout; chomp $stderr;
|
||||
utf8::decode($stdout) or die "Invalid unicode in stdout.";
|
||||
|
||||
@@ -165,46 +165,20 @@ sub nix_state_dir {
|
||||
sub makeAndEvaluateJobset {
|
||||
my ($self, %opts) = @_;
|
||||
|
||||
my $expression = $opts{'expression'};
|
||||
my $flake = $opts{'flake'};
|
||||
if (not $expression and not $flake) {
|
||||
die "One of 'expression' or 'flake' must be passed to makeEvaluateJobset.\n";
|
||||
}
|
||||
|
||||
my $expression = $opts{'expression'} || die "Mandatory 'expression' option not passed to makeAndEvaluateJobset.\n";
|
||||
my $jobsdir = $opts{'jobsdir'} // $self->jobsdir;
|
||||
my $should_build = $opts{'build'} // 0;
|
||||
|
||||
my %args = (
|
||||
my $jobsetCtx = $self->makeJobset(
|
||||
expression => $expression,
|
||||
jobsdir => $jobsdir,
|
||||
);
|
||||
if ($expression) {
|
||||
$args{expression} = $expression;
|
||||
}
|
||||
if ($flake) {
|
||||
$args{flake} = $flake;
|
||||
}
|
||||
my $jobsetCtx = $self->makeJobset(%args);
|
||||
|
||||
return $self->evaluateJobset(
|
||||
jobset => $jobsetCtx->{"jobset"},
|
||||
expression => $expression,
|
||||
flake => $flake,
|
||||
build => $opts{"build"} // 0,
|
||||
)
|
||||
}
|
||||
|
||||
sub evaluateJobset {
|
||||
my ($self, %opts) = @_;
|
||||
|
||||
my $jobset = $opts{'jobset'};
|
||||
|
||||
my $expression = $opts{'expression'} // $opts{'flake'};
|
||||
my $jobset = $jobsetCtx->{"jobset"};
|
||||
|
||||
evalSucceeds($jobset) or die "Evaluating jobs/$expression should exit with return code 0.\n";
|
||||
|
||||
my $builds = {};
|
||||
|
||||
my $should_build = $opts{'build'};
|
||||
|
||||
for my $build ($jobset->builds) {
|
||||
if ($should_build) {
|
||||
runBuild($build) or die "Build '".$build->job."' from jobs/$expression should exit with return code 0.\n";
|
||||
@@ -221,7 +195,7 @@ sub evaluateJobset {
|
||||
#
|
||||
# In return, you get a hash of the user, project, and jobset records.
|
||||
#
|
||||
# This always uses an `expression` or `flake` from the `jobsdir` directory.
|
||||
# This always uses an `expression` from the `jobsdir` directory.
|
||||
#
|
||||
# Hash Parameters:
|
||||
#
|
||||
@@ -230,12 +204,7 @@ sub evaluateJobset {
|
||||
sub makeJobset {
|
||||
my ($self, %opts) = @_;
|
||||
|
||||
my $expression = $opts{'expression'};
|
||||
my $flake = $opts{'flake'};
|
||||
if (not $expression and not $flake) {
|
||||
die "One of 'expression' or 'flake' must be passed to makeJobset.\n";
|
||||
}
|
||||
|
||||
my $expression = $opts{'expression'} || die "Mandatory 'expression' option not passed to makeJobset.\n";
|
||||
my $jobsdir = $opts{'jobsdir'} // $self->jobsdir;
|
||||
|
||||
# Create a new user for this test
|
||||
@@ -253,20 +222,12 @@ sub makeJobset {
|
||||
});
|
||||
|
||||
# Create a new jobset for this test and set up the inputs
|
||||
my %args = (
|
||||
my $jobset = $project->jobsets->create({
|
||||
name => rand_chars(),
|
||||
nixexprinput => "jobs",
|
||||
nixexprpath => $expression,
|
||||
emailoverride => ""
|
||||
);
|
||||
if ($expression) {
|
||||
$args{type} = 0;
|
||||
$args{nixexprinput} = "jobs";
|
||||
$args{nixexprpath} = $expression;
|
||||
}
|
||||
if ($flake) {
|
||||
$args{type} = 1;
|
||||
$args{flake} = $flake;
|
||||
}
|
||||
my $jobset = $project->jobsets->create(\%args);
|
||||
});
|
||||
my $jobsetinput = $jobset->jobsetinputs->create({name => "jobs", type => "path"});
|
||||
$jobsetinput->jobsetinputalts->create({altnr => 0, value => $jobsdir});
|
||||
|
||||
|
||||
@@ -27,7 +27,7 @@ testenv.prepend('PERL5LIB',
|
||||
separator: ':'
|
||||
)
|
||||
testenv.prepend('PATH',
|
||||
fs.parent(find_program('nix').full_path()),
|
||||
fs.parent(hydra_eval_jobs.full_path()),
|
||||
fs.parent(hydra_evaluator.full_path()),
|
||||
fs.parent(hydra_queue_runner.full_path()),
|
||||
meson.project_source_root() / 'src/script',
|
||||
|
||||
@@ -26,7 +26,7 @@ like(
|
||||
"Evaluating jobs/broken-constituent.nix should log an error for does-not-exist");
|
||||
like(
|
||||
$jobset->errormsg,
|
||||
qr/^does-not-evaluate: error: assertion 'false' failed/m,
|
||||
qr/^does-not-evaluate: error: assertion 'false' failed$/m,
|
||||
"Evaluating jobs/broken-constituent.nix should log an error for does-not-evaluate");
|
||||
|
||||
done_testing;
|
||||
|
||||
@@ -13,7 +13,7 @@ my $constituentBuildA = $builds->{"constituentA"};
|
||||
my $constituentBuildB = $builds->{"constituentB"};
|
||||
|
||||
my $eval = $constituentBuildA->jobsetevals->first();
|
||||
is($eval->evaluationerror->has_error, 0);
|
||||
is($eval->evaluationerror->errormsg, "");
|
||||
|
||||
subtest "Verifying the direct aggregate" => sub {
|
||||
my $aggBuild = $builds->{"direct_aggregate"};
|
||||
|
||||
Reference in New Issue
Block a user