diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3ba4aba6..0f5f43da 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -9,6 +9,6 @@ jobs: - uses: actions/checkout@v3 with: fetch-depth: 0 - - uses: cachix/install-nix-action@v16 + - uses: cachix/install-nix-action@v17 #- run: nix flake check - run: nix-build -A checks.x86_64-linux.build -A checks.x86_64-linux.validate-openapi diff --git a/doc/manual/src/SUMMARY.md b/doc/manual/src/SUMMARY.md index 80e73112..357e795a 100644 --- a/doc/manual/src/SUMMARY.md +++ b/doc/manual/src/SUMMARY.md @@ -7,6 +7,7 @@ - [Hydra jobs](./jobs.md) - [Plugins](./plugins/README.md) - [Declarative Projects](./plugins/declarative-projects.md) + - [RunCommand](./plugins/RunCommand.md) - [Using the external API](api.md) - [Webhooks](webhooks.md) - [Monitoring Hydra](./monitoring/README.md) diff --git a/doc/manual/src/configuration.md b/doc/manual/src/configuration.md index 2700625d..ab68df43 100644 --- a/doc/manual/src/configuration.md +++ b/doc/manual/src/configuration.md @@ -185,7 +185,7 @@ Example configuration: hydra_admin = admin # Allow all users in the dev group to restart jobs and cancel builds dev = restart-jobs - dev = cancel-builds + dev = cancel-build ``` diff --git a/doc/manual/src/hacking.md b/doc/manual/src/hacking.md index 9d98b00c..72a74c82 100644 --- a/doc/manual/src/hacking.md +++ b/doc/manual/src/hacking.md @@ -92,7 +92,7 @@ On NixOS: ```nix { - nix.trustedUsers = [ "YOURUSER" ]; + nix.settings.trusted-users = [ "YOURUSER" ]; } ``` diff --git a/doc/manual/src/plugins/README.md b/doc/manual/src/plugins/README.md index b5486fdb..6e46224f 100644 --- a/doc/manual/src/plugins/README.md +++ b/doc/manual/src/plugins/README.md @@ -172,17 +172,6 @@ Sets Gitlab CI status. - `gitlab_authorization.` -## HipChat notification - -Sends hipchat chat notifications when a build finish. - -### Configuration options - -- `hipchat.[].jobs` -- `hipchat.[].builds` -- `hipchat.[].token` -- `hipchat.[].notify` - ## InfluxDB notification Writes InfluxDB events when a builds finished. @@ -192,10 +181,12 @@ Writes InfluxDB events when a builds finished. - `influxdb.url` - `influxdb.db` -## Run command +## RunCommand Runs a shell command when the build is finished. +See [The RunCommand Plugin](./RunCommand.md) for more information. + ### Configuration options: - `runcommand.[].job` diff --git a/doc/manual/src/plugins/RunCommand.md b/doc/manual/src/plugins/RunCommand.md new file mode 100644 index 00000000..652a171e --- /dev/null +++ b/doc/manual/src/plugins/RunCommand.md @@ -0,0 +1,83 @@ +## The RunCommand Plugin + +Hydra supports executing a program after certain builds finish. +This behavior is disabled by default. + +Hydra executes these commands under the `hydra-notify` service. + +### Static Commands + +Configure specific commands to execute after the specified matching job finishes. + +#### Configuration + +- `runcommand.[].job` + +A matcher for jobs to match in the format `project:jobset:job`. Defaults to `*:*:*`. + +**Note:** This matcher format is not a regular expression. +The `*` is a wildcard for that entire section, partial matches are not supported. + +- `runcommand.[].command` + +Command to run. Can use the `$HYDRA_JSON` environment variable to access information about the build. + +### Example + +```xml + + job = myProject:*:* + command = cat $HYDRA_JSON > /tmp/hydra-output + +``` + +### Dynamic Commands + +Hydra can optionally run RunCommand hooks defined dynamically by the jobset. In +order to enable dynamic commands, you must enable this feature in your +`hydra.conf`, *as well as* in the parent project and jobset configuration. + +#### Behavior + +Hydra will execute any program defined under the `runCommandHook` attribute set. These jobs must have a single output named `out`, and that output must be an executable file located directly at `$out`. + +#### Security Properties + +Safely deploying dynamic commands requires careful design of your Hydra jobs. Allowing arbitrary users to define attributes in your top level attribute set will allow that user to execute code on your Hydra. + +If a jobset has dynamic commands enabled, you must ensure only trusted users can define top level attributes. + + +#### Configuration + +- `dynamicruncommand.enable` + +Set to 1 to enable dynamic RunCommand program execution. + +#### Example + +In your Hydra configuration, specify: + +```xml + + enable = 1 + +``` + +Then create a job named `runCommandHook.example` in your jobset: + +``` +{ pkgs, ... }: { + runCommandHook = { + recurseForDerivations = true; + + example = pkgs.writeScript "run-me" '' + #!${pkgs.runtimeShell} + + ${pkgs.jq}/bin/jq . "$HYDRA_JSON" + ''; + }; +} +``` + +After the `runcommandHook.example` build finishes that script will execute. diff --git a/doc/manual/src/plugins/declarative-projects.md b/doc/manual/src/plugins/declarative-projects.md index 12dfed18..b72c6fd0 100644 --- a/doc/manual/src/plugins/declarative-projects.md +++ b/doc/manual/src/plugins/declarative-projects.md @@ -34,6 +34,7 @@ To configure a static declarative project, take the following steps: "checkinterval": 300, "schedulingshares": 100, "enableemail": false, + "enable_dynamic_run_command": false, "emailoverride": "", "keepnr": 3, "inputs": { @@ -53,6 +54,7 @@ To configure a static declarative project, take the following steps: "checkinterval": 300, "schedulingshares": 100, "enableemail": false, + "enable_dynamic_run_command": false, "emailoverride": "", "keepnr": 3, "inputs": { @@ -92,6 +94,7 @@ containing the configuration of the jobset, for example: "checkinterval": 300, "schedulingshares": 100, "enableemail": false, + "enable_dynamic_run_command": false, "emailoverride": "", "keepnr": 3, "inputs": { diff --git a/doc/manual/src/projects.md b/doc/manual/src/projects.md index 95174f1b..a399406d 100644 --- a/doc/manual/src/projects.md +++ b/doc/manual/src/projects.md @@ -378,13 +378,18 @@ This section describes how it can be implemented for `gitea`, but the approach f analogous: * [Obtain an API token for your user](https://docs.gitea.io/en-us/api-usage/#authentication) -* Add it to your `hydra.conf` like this: +* Add it to a file which only users in the hydra group can read like this: see [including files](configuration.md#including-files) for more information + ``` + + your_username=your_token + + ``` + +* Include the file in your `hydra.conf` like this: ``` nix { services.hydra-dev.extraConfig = '' - - your_username=your_token - + Include /path/to/secret/file ''; } ``` diff --git a/flake.lock b/flake.lock index 5c726a24..b41b843a 100644 --- a/flake.lock +++ b/flake.lock @@ -16,22 +16,6 @@ "type": "github" } }, - "newNixpkgs": { - "locked": { - "lastModified": 1647380550, - "narHash": "sha256-909TI9poX7CIUiFx203WL29YON6m/I6k0ExbZvR7bLM=", - "owner": "NixOS", - "repo": "nixpkgs", - "rev": "6e3ee8957637a60f5072e33d78e05c0f65c54366", - "type": "github" - }, - "original": { - "owner": "NixOS", - "ref": "nixos-unstable-small", - "repo": "nixpkgs", - "type": "github" - } - }, "nix": { "inputs": { "lowdown-src": "lowdown-src", @@ -39,31 +23,34 @@ "nixpkgs-regression": "nixpkgs-regression" }, "locked": { - "lastModified": 1649172203, - "narHash": "sha256-Q3nYaXqbseDOvZrlePKeIrx0/KzqyrtNpxHIUbtFHuI=", + "lastModified": 1661606874, + "narHash": "sha256-9+rpYzI+SmxJn+EbYxjGv68Ucp22bdFUSy/4LkHkkDQ=", "owner": "NixOS", "repo": "nix", - "rev": "5fe4fe823c193cbb7bfa05a468de91eeab09058d", + "rev": "11e45768b34fdafdcf019ddbd337afa16127ff0f", "type": "github" }, "original": { - "id": "nix", - "type": "indirect" + "owner": "NixOS", + "ref": "2.11.0", + "repo": "nix", + "type": "github" } }, "nixpkgs": { "locked": { - "lastModified": 1645296114, - "narHash": "sha256-y53N7TyIkXsjMpOG7RhvqJFGDacLs9HlyHeSTBioqYU=", + "lastModified": 1657693803, + "narHash": "sha256-G++2CJ9u0E7NNTAi9n5G8TdDmGJXcIjkJ3NF8cetQB8=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "530a53dcbc9437363471167a5e4762c5fcfa34a1", + "rev": "365e1b3a859281cf11b94f87231adeabbdd878a2", "type": "github" }, "original": { - "id": "nixpkgs", - "ref": "nixos-21.05-small", - "type": "indirect" + "owner": "NixOS", + "ref": "nixos-22.05-small", + "repo": "nixpkgs", + "type": "github" } }, "nixpkgs-regression": { @@ -76,14 +63,14 @@ "type": "github" }, "original": { - "id": "nixpkgs", + "owner": "NixOS", + "repo": "nixpkgs", "rev": "215d4d0fd80ca5163643b03a33fde804a29cc1e2", - "type": "indirect" + "type": "github" } }, "root": { "inputs": { - "newNixpkgs": "newNixpkgs", "nix": "nix", "nixpkgs": [ "nix", diff --git a/flake.nix b/flake.nix index a4c051aa..1cd0f3e7 100644 --- a/flake.nix +++ b/flake.nix @@ -1,20 +1,17 @@ { description = "A Nix-based continuous build system"; - # FIXME: All the pinned versions of nix/nixpkgs have a broken foreman (yes, - # even 2.7.0's Nixpkgs pin). - inputs.newNixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable-small"; inputs.nixpkgs.follows = "nix/nixpkgs"; - #inputs.nix.url = github:NixOS/nix/2.7.0; + inputs.nix.url = "github:NixOS/nix/2.11.0"; - outputs = { self, newNixpkgs, nixpkgs, nix }: + outputs = { self, nixpkgs, nix }: let version = "${builtins.readFile ./version.txt}.${builtins.substring 0 8 (self.lastModifiedDate or "19700101")}.${self.shortRev or "DIRTY"}"; pkgs = import nixpkgs { system = "x86_64-linux"; - overlays = [ self.overlay nix.overlay ]; + overlays = [ self.overlays.default nix.overlays.default ]; }; # NixOS configuration used for VM tests. @@ -39,155 +36,10 @@ rec { # A Nixpkgs overlay that provides a 'hydra' package. - overlay = final: prev: { - - # Overlay these packages to use dependencies from the Nixpkgs everything - # else uses, to side-step the version difference: glibc is 2.32 in the - # nix-pinned Nixpkgs, but 2.33 in the newNixpkgs commit. - civetweb = (final.callPackage "${newNixpkgs}/pkgs/development/libraries/civetweb" { }).overrideAttrs - # Can be dropped once newNixpkgs points to a revision containing - # https://github.com/NixOS/nixpkgs/pull/167751 - ({ cmakeFlags ? [ ], ... }: { - cmakeFlags = cmakeFlags ++ [ - "-DCIVETWEB_ENABLE_IPV6=1" - ]; - }); - prometheus-cpp = final.callPackage "${newNixpkgs}/pkgs/development/libraries/prometheus-cpp" { }; + overlays.default = final: prev: { # Add LDAP dependencies that aren't currently found within nixpkgs. perlPackages = prev.perlPackages // { - TestPostgreSQL = final.perlPackages.buildPerlModule { - pname = "Test-PostgreSQL"; - version = "1.28-1"; - src = final.fetchFromGitHub { - owner = "grahamc"; - repo = "Test-postgresql"; - rev = "release-1.28-1"; - hash = "sha256-SFC1C3q3dbcBos18CYd/s0TIcfJW4g04ld0+XQXVToQ="; - }; - buildInputs = with final.perlPackages; [ ModuleBuildTiny TestSharedFork pkgs.postgresql ]; - propagatedBuildInputs = with final.perlPackages; [ DBDPg DBI FileWhich FunctionParameters Moo TieHashMethod TryTiny TypeTiny ]; - - makeMakerFlags = "POSTGRES_HOME=${final.postgresql}"; - - meta = { - homepage = "https://github.com/grahamc/Test-postgresql/releases/tag/release-1.28-1"; - description = "PostgreSQL runner for tests"; - license = with final.lib.licenses; [ artistic2 ]; - }; - }; - - FunctionParameters = final.perlPackages.buildPerlPackage { - pname = "Function-Parameters"; - version = "2.001003"; - src = final.fetchurl { - url = "mirror://cpan/authors/id/M/MA/MAUKE/Function-Parameters-2.001003.tar.gz"; - sha256 = "eaa22c6b43c02499ec7db0758c2dd218a3b2ab47a714b2bdf8010b5ee113c242"; - }; - buildInputs = with final.perlPackages; [ DirSelf TestFatal ]; - meta = { - description = "Define functions and methods with parameter lists (\"subroutine signatures\")"; - license = with final.lib.licenses; [ artistic1 gpl1Plus ]; - }; - }; - - CatalystPluginPrometheusTiny = final.perlPackages.buildPerlPackage { - pname = "Catalyst-Plugin-PrometheusTiny"; - version = "0.005"; - src = final.fetchurl { - url = "mirror://cpan/authors/id/S/SY/SYSPETE/Catalyst-Plugin-PrometheusTiny-0.005.tar.gz"; - sha256 = "a42ef09efdc3053899ae007c41220d3ed7207582cc86e491b4f534539c992c5a"; - }; - buildInputs = with final.perlPackages; [ HTTPMessage Plack SubOverride TestDeep ]; - propagatedBuildInputs = with final.perlPackages; [ CatalystRuntime Moose PrometheusTiny PrometheusTinyShared ]; - meta = { - description = "Prometheus metrics for Catalyst"; - license = with final.lib.licenses; [ artistic1 gpl1Plus ]; - }; - }; - - CryptArgon2 = final.perlPackages.buildPerlModule { - pname = "Crypt-Argon2"; - version = "0.010"; - src = final.fetchurl { - url = "mirror://cpan/authors/id/L/LE/LEONT/Crypt-Argon2-0.010.tar.gz"; - sha256 = "3ea1c006f10ef66fd417e502a569df15c4cc1c776b084e35639751c41ce6671a"; - }; - nativeBuildInputs = [ pkgs.ld-is-cc-hook ]; - meta = { - description = "Perl interface to the Argon2 key derivation functions"; - license = final.lib.licenses.cc0; - }; - }; - - CryptPassphrase = final.perlPackages.buildPerlPackage { - pname = "Crypt-Passphrase"; - version = "0.003"; - src = final.fetchurl { - url = "mirror://cpan/authors/id/L/LE/LEONT/Crypt-Passphrase-0.003.tar.gz"; - sha256 = "685aa090f8179a86d6896212ccf8ccfde7a79cce857199bb14e2277a10d240ad"; - }; - meta = { - description = "A module for managing passwords in a cryptographically agile manner"; - license = with final.lib.licenses; [ artistic1 gpl1Plus ]; - }; - }; - - CryptPassphraseArgon2 = final.perlPackages.buildPerlPackage { - pname = "Crypt-Passphrase-Argon2"; - version = "0.002"; - src = final.fetchurl { - url = "mirror://cpan/authors/id/L/LE/LEONT/Crypt-Passphrase-Argon2-0.002.tar.gz"; - sha256 = "3906ff81697d13804ee21bd5ab78ffb1c4408b4822ce020e92ecf4737ba1f3a8"; - }; - propagatedBuildInputs = with final.perlPackages; [ CryptArgon2 CryptPassphrase ]; - meta = { - description = "An Argon2 encoder for Crypt::Passphrase"; - license = with final.lib.licenses; [ artistic1 gpl1Plus ]; - }; - }; - - DataRandom = final.perlPackages.buildPerlPackage { - pname = "Data-Random"; - version = "0.13"; - src = final.fetchurl { - url = "mirror://cpan/authors/id/B/BA/BAREFOOT/Data-Random-0.13.tar.gz"; - sha256 = "eb590184a8db28a7e49eab09e25f8650c33f1f668b6a472829de74a53256bfc0"; - }; - buildInputs = with final.perlPackages; [ FileShareDirInstall TestMockTime ]; - meta = { - description = "Perl module to generate random data"; - license = with final.lib.licenses; [ artistic1 gpl1Plus ]; - }; - }; - - DirSelf = final.perlPackages.buildPerlPackage { - pname = "Dir-Self"; - version = "0.11"; - src = final.fetchurl { - url = "mirror://cpan/authors/id/M/MA/MAUKE/Dir-Self-0.11.tar.gz"; - sha256 = "e251a51abc7d9ba3e708f73c2aa208e09d47a0c528d6254710fa78cc8d6885b5"; - }; - meta = { - homepage = "https://github.com/mauke/Dir-Self"; - description = "A __DIR__ constant for the directory your source file is in"; - license = with final.lib.licenses; [ artistic1 gpl1Plus ]; - }; - }; - - HashSharedMem = final.perlPackages.buildPerlModule { - pname = "Hash-SharedMem"; - version = "0.005"; - src = final.fetchurl { - url = "mirror://cpan/authors/id/Z/ZE/ZEFRAM/Hash-SharedMem-0.005.tar.gz"; - sha256 = "324776808602f7bdc44adaa937895365454029a926fa611f321c9bf6b940bb5e"; - }; - buildInputs = with final.perlPackages; [ ScalarString ]; - meta = { - description = "Efficient shared mutable hash"; - license = with final.lib.licenses; [ artistic1 gpl1Plus ]; - }; - }; PrometheusTiny = final.perlPackages.buildPerlPackage { pname = "Prometheus-Tiny"; @@ -204,269 +56,6 @@ }; }; - PrometheusTinyShared = final.perlPackages.buildPerlPackage { - pname = "Prometheus-Tiny-Shared"; - version = "0.023"; - src = final.fetchurl { - url = "mirror://cpan/authors/id/R/RO/ROBN/Prometheus-Tiny-Shared-0.023.tar.gz"; - sha256 = "7c2c72397be5d8e4839d1bf4033c1800f467f2509689673c6419df48794f2abe"; - }; - buildInputs = with final.perlPackages; [ DataRandom HTTPMessage Plack TestDifferences TestException ]; - propagatedBuildInputs = with final.perlPackages; [ HashSharedMem JSONXS PrometheusTiny ]; - meta = { - homepage = "https://github.com/robn/Prometheus-Tiny-Shared"; - description = "A tiny Prometheus client with a shared database behind it"; - license = with final.lib.licenses; [ artistic1 gpl1Plus ]; - }; - }; - - ReadonlyX = final.perlPackages.buildPerlModule { - pname = "ReadonlyX"; - version = "1.04"; - src = final.fetchurl { - url = "mirror://cpan/authors/id/S/SA/SANKO/ReadonlyX-1.04.tar.gz"; - sha256 = "81bb97dba93ac6b5ccbce04a42c3590eb04557d75018773ee18d5a30fcf48188"; - }; - buildInputs = with final.perlPackages; [ ModuleBuildTiny TestFatal ]; - meta = { - homepage = "https://github.com/sanko/readonly"; - description = "Faster facility for creating read-only scalars, arrays, hashes"; - license = final.lib.licenses.artistic2; - }; - }; - - TieHashMethod = final.perlPackages.buildPerlPackage { - pname = "Tie-Hash-Method"; - version = "0.02"; - src = final.fetchurl { - url = "mirror://cpan/authors/id/Y/YV/YVES/Tie-Hash-Method-0.02.tar.gz"; - sha256 = "d513fbb51413f7ca1e64a1bdce6194df7ec6076dea55066d67b950191eec32a9"; - }; - meta = { - description = "Tied hash with specific methods overriden by callbacks"; - license = with final.lib.licenses; [ artistic1 ]; - }; - }; - - Test2Harness = final.perlPackages.buildPerlPackage { - pname = "Test2-Harness"; - version = "1.000042"; - src = final.fetchurl { - url = "mirror://cpan/authors/id/E/EX/EXODIST/Test2-Harness-1.000042.tar.gz"; - sha256 = "aaf231a68af1a6ffd6a11188875fcf572e373e43c8285945227b9d687b43db2d"; - }; - - checkPhase = '' - patchShebangs ./t ./scripts/yath - ./scripts/yath test -j $NIX_BUILD_CORES - ''; - - propagatedBuildInputs = with final.perlPackages; [ DataUUID Importer LongJump ScopeGuard TermTable Test2PluginMemUsage Test2PluginUUID Test2Suite gotofile ]; - meta = { - description = "A new and improved test harness with better Test2 integration"; - license = with final.lib.licenses; [ artistic1 gpl1Plus ]; - }; - }; - - Test2PluginMemUsage = prev.perlPackages.buildPerlPackage { - pname = "Test2-Plugin-MemUsage"; - version = "0.002003"; - src = final.fetchurl { - url = "mirror://cpan/authors/id/E/EX/EXODIST/Test2-Plugin-MemUsage-0.002003.tar.gz"; - sha256 = "5e0662d5a823ae081641f5ce82843111eec1831cd31f883a6c6de54afdf87c25"; - }; - buildInputs = with final.perlPackages; [ Test2Suite ]; - meta = { - description = "Collect and display memory usage information"; - license = with final.lib.licenses; [ artistic1 gpl1Plus ]; - }; - }; - - Test2PluginUUID = prev.perlPackages.buildPerlPackage { - pname = "Test2-Plugin-UUID"; - version = "0.002001"; - src = final.fetchurl { - url = "mirror://cpan/authors/id/E/EX/EXODIST/Test2-Plugin-UUID-0.002001.tar.gz"; - sha256 = "4c6c8d484d7153d8779dc155a992b203095b5c5aa1cfb1ee8bcedcd0601878c9"; - }; - buildInputs = with final.perlPackages;[ Test2Suite ]; - propagatedBuildInputs = with final.perlPackages; [ DataUUID ]; - meta = { - description = "Use REAL UUIDs in Test2"; - license = with final.lib.licenses; [ artistic1 gpl1Plus ]; - }; - }; - - LongJump = final.perlPackages.buildPerlPackage { - pname = "Long-Jump"; - version = "0.000001"; - src = final.fetchurl { - url = "mirror://cpan/authors/id/E/EX/EXODIST/Long-Jump-0.000001.tar.gz"; - sha256 = "d5d6456d86992b559d8f66fc90960f919292cd3803c13403faac575762c77af4"; - }; - buildInputs = with final.perlPackages; [ Test2Suite ]; - meta = { - description = "Mechanism for returning to a specific point from a deeply nested stack"; - license = with final.lib.licenses; [ artistic1 gpl1Plus ]; - }; - }; - - gotofile = final.perlPackages.buildPerlPackage { - pname = "goto-file"; - version = "0.005"; - src = final.fetchurl { - url = "mirror://cpan/authors/id/E/EX/EXODIST/goto-file-0.005.tar.gz"; - sha256 = "c6cdd5ee4a6cdcbdbf314d92a4f9985dbcdf9e4258048cae76125c052aa31f77"; - }; - buildInputs = with final.perlPackages; [ Test2Suite ]; - meta = { - description = "Stop parsing the current file and move on to a different one"; - license = with final.lib.licenses; [ artistic1 gpl1Plus ]; - }; - }; - - NetLDAPServer = prev.perlPackages.buildPerlPackage { - pname = "Net-LDAP-Server"; - version = "0.43"; - src = final.fetchurl { - url = "mirror://cpan/authors/id/A/AA/AAR/Net-LDAP-Server-0.43.tar.gz"; - sha256 = "0qmh3cri3fpccmwz6bhwp78yskrb3qmalzvqn0a23hqbsfs4qv6x"; - }; - propagatedBuildInputs = with final.perlPackages; [ NetLDAP ConvertASN1 ]; - meta = { - description = "LDAP server side protocol handling"; - license = with final.lib.licenses; [ artistic1 ]; - }; - }; - - NetLDAPSID = prev.perlPackages.buildPerlPackage { - pname = "Net-LDAP-SID"; - version = "0.0001"; - src = final.fetchurl { - url = "mirror://cpan/authors/id/K/KA/KARMAN/Net-LDAP-SID-0.001.tar.gz"; - sha256 = "1mnnpkmj8kpb7qw50sm8h4sd8py37ssy2xi5hhxzr5whcx0cvhm8"; - }; - meta = { - description = "Active Directory Security Identifier manipulation"; - license = with final.lib.licenses; [ artistic2 ]; - }; - }; - - NetLDAPServerTest = prev.perlPackages.buildPerlPackage { - pname = "Net-LDAP-Server-Test"; - version = "0.22"; - src = final.fetchurl { - url = "mirror://cpan/authors/id/K/KA/KARMAN/Net-LDAP-Server-Test-0.22.tar.gz"; - sha256 = "13idip7jky92v4adw60jn2gcc3zf339gsdqlnc9nnvqzbxxp285i"; - }; - propagatedBuildInputs = with final.perlPackages; [ NetLDAP NetLDAPServer TestMore DataDump NetLDAPSID ]; - meta = { - description = "test Net::LDAP code"; - license = with final.lib.licenses; [ artistic1 ]; - }; - }; - - CatalystAuthenticationStoreLDAP = prev.perlPackages.buildPerlPackage { - pname = "Catalyst-Authentication-Store-LDAP"; - version = "1.016"; - src = final.fetchurl { - url = "mirror://cpan/authors/id/I/IL/ILMARI/Catalyst-Authentication-Store-LDAP-1.016.tar.gz"; - sha256 = "0cm399vxqqf05cjgs1j5v3sk4qc6nmws5nfhf52qvpbwc4m82mq8"; - }; - propagatedBuildInputs = with final.perlPackages; [ NetLDAP CatalystPluginAuthentication ClassAccessorFast ]; - buildInputs = with final.perlPackages; [ TestMore TestMockObject TestException NetLDAPServerTest ]; - meta = { - description = "Authentication from an LDAP Directory"; - license = with final.lib.licenses; [ artistic1 ]; - }; - }; - - PerlCriticCommunity = prev.perlPackages.buildPerlModule { - pname = "Perl-Critic-Community"; - version = "1.0.0"; - src = final.fetchurl { - url = "mirror://cpan/authors/id/D/DB/DBOOK/Perl-Critic-Community-v1.0.0.tar.gz"; - sha256 = "311b775da4193e9de94cf5225e993cc54dd096ae1e7ef60738cdae1d9b8854e7"; - }; - buildInputs = with final.perlPackages; [ ModuleBuildTiny ]; - propagatedBuildInputs = with final.perlPackages; [ PPI PathTiny PerlCritic PerlCriticPolicyVariablesProhibitLoopOnHash PerlCriticPulp ]; - meta = { - homepage = "https://github.com/Grinnz/Perl-Critic-Freenode"; - description = "Community-inspired Perl::Critic policies"; - license = final.lib.licenses.artistic2; - }; - }; - - PerlCriticPolicyVariablesProhibitLoopOnHash = prev.perlPackages.buildPerlPackage { - pname = "Perl-Critic-Policy-Variables-ProhibitLoopOnHash"; - version = "0.008"; - src = final.fetchurl { - url = "mirror://cpan/authors/id/X/XS/XSAWYERX/Perl-Critic-Policy-Variables-ProhibitLoopOnHash-0.008.tar.gz"; - sha256 = "12f5f0be96ea1bdc7828058577bd1c5c63ca23c17fac9c3709452b3dff5b84e0"; - }; - propagatedBuildInputs = with final.perlPackages; [ PerlCritic ]; - meta = { - description = "Don't write loops on hashes, only on keys and values of hashes"; - license = with final.lib.licenses; [ artistic1 gpl1Plus ]; - }; - }; - - PerlCriticPulp = prev.perlPackages.buildPerlPackage { - pname = "Perl-Critic-Pulp"; - version = "99"; - src = final.fetchurl { - url = "mirror://cpan/authors/id/K/KR/KRYDE/Perl-Critic-Pulp-99.tar.gz"; - sha256 = "b8fda842fcbed74d210257c0a284b6dc7b1d0554a47a3de5d97e7d542e23e7fe"; - }; - propagatedBuildInputs = with final.perlPackages; [ IOString ListMoreUtils PPI PerlCritic PodMinimumVersion ]; - meta = { - homepage = "http://user42.tuxfamily.org/perl-critic-pulp/index.html"; - description = "Some add-on policies for Perl::Critic"; - license = final.lib.licenses.gpl3Plus; - }; - }; - - PodMinimumVersion = prev.perlPackages.buildPerlPackage { - pname = "Pod-MinimumVersion"; - version = "50"; - src = final.fetchurl { - url = "mirror://cpan/authors/id/K/KR/KRYDE/Pod-MinimumVersion-50.tar.gz"; - sha256 = "0bd2812d9aacbd99bb71fa103a4bb129e955c138ba7598734207dc9fb67b5a6f"; - }; - propagatedBuildInputs = with final.perlPackages; [ IOString PodParser ]; - meta = { - homepage = "http://user42.tuxfamily.org/pod-minimumversion/index.html"; - description = "Determine minimum Perl version of POD directives"; - license = final.lib.licenses.free; - }; - }; - - StringCompareConstantTime = final.perlPackages.buildPerlPackage { - pname = "String-Compare-ConstantTime"; - version = "0.321"; - src = final.fetchurl { - url = "mirror://cpan/authors/id/F/FR/FRACTAL/String-Compare-ConstantTime-0.321.tar.gz"; - sha256 = "0b26ba2b121d8004425d4485d1d46f59001c83763aa26624dff6220d7735d7f7"; - }; - meta = { - description = "Timing side-channel protected string compare"; - license = with final.lib.licenses; [ artistic1 gpl1Plus ]; - }; - }; - - UUID4Tiny = final.perlPackages.buildPerlPackage { - pname = "UUID4-Tiny"; - version = "0.002"; - src = final.fetchurl { - url = "mirror://cpan/authors/id/C/CV/CVLIBRARY/UUID4-Tiny-0.002.tar.gz"; - sha256 = "e7535b31e386d432dec7adde214348389e1d5cf753e7ed07f1ae04c4360840cf"; - }; - meta = { - description = "Cryptographically secure v4 UUIDs for Linux x64"; - license = with final.lib.licenses; [ artistic1 gpl1Plus ]; - }; - }; - }; hydra = with final; let @@ -486,7 +75,6 @@ CatalystPluginSessionStateCookie CatalystPluginSessionStoreFastMmap CatalystPluginStackTrace - CatalystPluginUnicodeEncoding CatalystTraitForRequestProxyBase CatalystViewDownload CatalystViewJSON @@ -503,6 +91,7 @@ DigestSHA1 EmailMIME EmailSender + FileLibMagic FileSlurper FileWhich final.nix.perl-bindings @@ -533,7 +122,6 @@ TermSizeAny TermReadKey Test2Harness - TestMore TestPostgreSQL TextDiff TextTable @@ -558,9 +146,9 @@ libtool unzip nukeReferences - pkgconfig + pkg-config libpqxx - gitAndTools.topGit + top-git mercurial darcs subversion @@ -583,9 +171,7 @@ checkInputs = [ cacert - # FIXME: foreman is broken on all nix/nixpkgs pin, up to and - # including 2.7.0 - newNixpkgs.legacyPackages.${final.system}.foreman + foreman glibcLocales libressl.nc openldap @@ -602,11 +188,11 @@ pixz gzip bzip2 - lzma + xz gnutar unzip git - gitAndTools.topGit + top-git mercurial darcs gnused @@ -661,7 +247,7 @@ dontStrip = true; - meta.description = "Build of Hydra on ${system}"; + meta.description = "Build of Hydra on ${final.stdenv.system}"; passthru = { inherit perlDeps; inherit (final) nix; }; }; }; @@ -683,7 +269,7 @@ tests.install.x86_64-linux = with import (nixpkgs + "/nixos/lib/testing-python.nix") { system = "x86_64-linux"; }; simpleTest { - machine = hydraServer; + nodes.machine = hydraServer; testScript = '' machine.wait_for_job("hydra-init") @@ -698,7 +284,7 @@ tests.notifications.x86_64-linux = with import (nixpkgs + "/nixos/lib/testing-python.nix") { system = "x86_64-linux"; }; simpleTest { - machine = { pkgs, ... }: { + nodes.machine = { pkgs, ... }: { imports = [ hydraServer ]; services.hydra-dev.extraConfig = '' @@ -755,7 +341,7 @@ tests.gitea.x86_64-linux = with import (nixpkgs + "/nixos/lib/testing-python.nix") { system = "x86_64-linux"; }; makeTest { - machine = { pkgs, ... }: { + nodes.machine = { pkgs, ... }: { imports = [ hydraServer ]; services.hydra-dev.extraConfig = '' @@ -962,11 +548,11 @@ checks.x86_64-linux.validate-openapi = hydraJobs.tests.validate-openapi; packages.x86_64-linux.hydra = pkgs.hydra; - defaultPackage.x86_64-linux = pkgs.hydra; + packages.x86_64-linux.default = pkgs.hydra; nixosModules.hydra = { imports = [ ./hydra-module.nix ]; - nixpkgs.overlays = [ self.overlay nix.overlay ]; + nixpkgs.overlays = [ self.overlays.default nix.overlays.default ]; }; nixosModules.hydraTest = { @@ -1017,7 +603,7 @@ self.nixosModules.hydraTest self.nixosModules.hydraProxy { - system.configurationRevision = self.rev; + system.configurationRevision = self.lastModifiedDate; boot.isContainer = true; networking.useDHCP = false; diff --git a/hydra-api.yaml b/hydra-api.yaml index 7857162e..ce7e0f9a 100644 --- a/hydra-api.yaml +++ b/hydra-api.yaml @@ -178,6 +178,9 @@ paths: enabled: description: when set to true the project gets scheduled for evaluation type: boolean + enable_dynamic_run_command: + description: when true the project's jobsets support executing dynamically defined RunCommand hooks. Requires the server and project's configuration to also enable dynamic RunCommand. + type: boolean visible: description: when set to true the project is displayed in the web interface type: boolean @@ -607,6 +610,9 @@ components: enabled: description: when set to true the project gets scheduled for evaluation type: boolean + enable_dynamic_run_command: + description: when true the project's jobsets support executing dynamically defined RunCommand hooks. Requires the server and project's configuration to also enable dynamic RunCommand. + type: boolean declarative: description: declarative input configured for this project type: object @@ -689,6 +695,9 @@ components: enableemail: description: when true the jobset sends emails when previously-successful builds fail type: boolean + enable_dynamic_run_command: + description: when true the jobset supports executing dynamically defined RunCommand hooks. Requires the server and project's configuration to also enable dynamic RunCommand. + type: boolean visible: description: when true the jobset is visible in the web frontend type: boolean diff --git a/hydra-module.nix b/hydra-module.nix index 0df5e690..8e02dcbb 100644 --- a/hydra-module.nix +++ b/hydra-module.nix @@ -228,8 +228,12 @@ in useDefaultShell = true; }; - nix.trustedUsers = [ "hydra-queue-runner" ]; - + nix.settings = { + trusted-users = [ "hydra-queue-runner" ]; + gc-keep-outputs = true; + gc-keep-derivations = true; + }; + services.hydra-dev.extraConfig = '' using_frontend_proxy = 1 @@ -256,11 +260,6 @@ in environment.variables = hydraEnv; - nix.extraOptions = '' - gc-keep-outputs = true - gc-keep-derivations = true - ''; - systemd.services.hydra-init = { wantedBy = [ "multi-user.target" ]; requires = optional haveLocalDB "postgresql.service"; @@ -268,17 +267,17 @@ in environment = env // { HYDRA_DBI = "${env.HYDRA_DBI};application_name=hydra-init"; }; - path = [ pkgs.utillinux ]; + path = [ pkgs.util-linux ]; preStart = '' ln -sf ${hydraConf} ${baseDir}/hydra.conf mkdir -m 0700 -p ${baseDir}/www - chown hydra-www.hydra ${baseDir}/www + chown hydra-www:hydra ${baseDir}/www mkdir -m 0700 -p ${baseDir}/queue-runner mkdir -m 0750 -p ${baseDir}/build-logs mkdir -m 0750 -p ${baseDir}/runcommand-logs - chown hydra-queue-runner.hydra \ + chown hydra-queue-runner:hydra \ ${baseDir}/queue-runner \ ${baseDir}/build-logs \ ${baseDir}/runcommand-logs @@ -309,7 +308,7 @@ in rmdir /nix/var/nix/gcroots/per-user/hydra-www/hydra-roots fi - chown hydra.hydra ${cfg.gcRootsDir} + chown hydra:hydra ${cfg.gcRootsDir} chmod 2775 ${cfg.gcRootsDir} ''; serviceConfig.ExecStart = "${cfg.package}/bin/hydra-init"; diff --git a/shell.nix b/shell.nix index 9e967032..1ad58f49 100644 --- a/shell.nix +++ b/shell.nix @@ -1,6 +1,6 @@ # The `default.nix` in flake-compat reads `flake.nix` and `flake.lock` from `src` and # returns an attribute set of the shape `{ defaultNix, shellNix }` -(import (fetchTarball https://github.com/edolstra/flake-compat/archive/master.tar.gz) { +(import (fetchTarball "https://github.com/edolstra/flake-compat/archive/master.tar.gz") { src = ./.; }).shellNix diff --git a/src/hydra-eval-jobs/Makefile.am b/src/hydra-eval-jobs/Makefile.am index 7a4e9c91..90742a30 100644 --- a/src/hydra-eval-jobs/Makefile.am +++ b/src/hydra-eval-jobs/Makefile.am @@ -1,5 +1,5 @@ bin_PROGRAMS = hydra-eval-jobs hydra_eval_jobs_SOURCES = hydra-eval-jobs.cc -hydra_eval_jobs_LDADD = $(NIX_LIBS) +hydra_eval_jobs_LDADD = $(NIX_LIBS) -lnixcmd hydra_eval_jobs_CXXFLAGS = $(NIX_CFLAGS) -I ../libhydra diff --git a/src/hydra-eval-jobs/hydra-eval-jobs.cc b/src/hydra-eval-jobs/hydra-eval-jobs.cc index 7485b297..18d39620 100644 --- a/src/hydra-eval-jobs/hydra-eval-jobs.cc +++ b/src/hydra-eval-jobs/hydra-eval-jobs.cc @@ -25,6 +25,28 @@ #include +void check_pid_status_nonblocking(pid_t check_pid) { + // Only check 'initialized' and known PID's + if (check_pid <= 0) { return; } + + int wstatus = 0; + pid_t pid = waitpid(check_pid, &wstatus, WNOHANG); + // -1 = failure, WNOHANG: 0 = no change + if (pid <= 0) { return; } + + std::cerr << "child process (" << pid << ") "; + + if (WIFEXITED(wstatus)) { + std::cerr << "exited with status=" << WEXITSTATUS(wstatus) << std::endl; + } else if (WIFSIGNALED(wstatus)) { + std::cerr << "killed by signal=" << WTERMSIG(wstatus) << std::endl; + } else if (WIFSTOPPED(wstatus)) { + std::cerr << "stopped by signal=" << WSTOPSIG(wstatus) << std::endl; + } else if (WIFCONTINUED(wstatus)) { + std::cerr << "continued" << std::endl; + } +} + using namespace nix; static Path gcRootsDir; @@ -175,21 +197,21 @@ static void worker( /* If this is an aggregate, then get its constituents. */ auto a = v->attrs->get(state.symbols.create("_hydraAggregate")); - if (a && state.forceBool(*a->value, *a->pos)) { + if (a && state.forceBool(*a->value, a->pos)) { auto a = v->attrs->get(state.symbols.create("constituents")); if (!a) throw EvalError("derivation must have a ‘constituents’ attribute"); PathSet context; - state.coerceToString(*a->pos, *a->value, context, true, false); + state.coerceToString(a->pos, *a->value, context, true, false); for (auto & i : context) if (i.at(0) == '!') { size_t index = i.find("!", 1); job["constituents"].push_back(std::string(i, index + 1)); } - state.forceList(*a->value, *a->pos); + state.forceList(*a->value, a->pos); for (unsigned int n = 0; n < a->value->listSize(); ++n) { auto v = a->value->listElems()[n]; state.forceValue(*v, noPos); @@ -221,8 +243,8 @@ static void worker( else if (v->type() == nAttrs) { auto attrs = nlohmann::json::array(); StringSet ss; - for (auto & i : v->attrs->lexicographicOrder()) { - std::string name(i->name); + for (auto & i : v->attrs->lexicographicOrder(state.symbols)) { + std::string name(state.symbols[i->name]); if (name.find('.') != std::string::npos || name.find(' ') != std::string::npos) { printError("skipping job with illegal name '%s'", name); continue; @@ -311,8 +333,8 @@ int main(int argc, char * * argv) /* Start a handler thread per worker process. */ auto handler = [&]() { + pid_t pid = -1; try { - pid_t pid = -1; AutoCloseFD from, to; while (true) { @@ -414,6 +436,7 @@ int main(int argc, char * * argv) } } } catch (...) { + check_pid_status_nonblocking(pid); auto state(state_.lock()); state->exc = std::current_exception(); wakeup.notify_all(); diff --git a/src/hydra-queue-runner/queue-monitor.cc b/src/hydra-queue-runner/queue-monitor.cc index 3bde0d99..12d55b79 100644 --- a/src/hydra-queue-runner/queue-monitor.cc +++ b/src/hydra-queue-runner/queue-monitor.cc @@ -460,7 +460,7 @@ Step::ptr State::createStep(ref destStore, step->parsedDrv = std::make_unique(drvPath, *step->drv); step->preferLocalBuild = step->parsedDrv->willBuildLocally(*localStore); - step->isDeterministic = get(step->drv->env, "isDetermistic").value_or("0") == "1"; + step->isDeterministic = getOr(step->drv->env, "isDetermistic", "0") == "1"; step->systemType = step->drv->platform; { diff --git a/src/hydra-queue-runner/state.hh b/src/hydra-queue-runner/state.hh index 651aaf97..8beaa54d 100644 --- a/src/hydra-queue-runner/state.hh +++ b/src/hydra-queue-runner/state.hh @@ -6,6 +6,7 @@ #include #include #include +#include #include #include @@ -293,7 +294,8 @@ struct Machine bool isLocalhost() { - return sshName == "localhost"; + std::regex r("^(ssh://|ssh-ng://)?localhost$"); + return std::regex_search(sshName, r); } }; diff --git a/src/lib/Hydra/Controller/Build.pm b/src/lib/Hydra/Controller/Build.pm index af648109..18a0eba3 100644 --- a/src/lib/Hydra/Controller/Build.pm +++ b/src/lib/Hydra/Controller/Build.pm @@ -7,15 +7,16 @@ use base 'Hydra::Base::Controller::NixChannel'; use Hydra::Helper::Nix; use Hydra::Helper::CatalystUtils; use File::Basename; +use File::LibMagic; use File::stat; use Data::Dump qw(dump); use Nix::Store; use Nix::Config; use List::SomeUtils qw(all); use Encode; -use MIME::Types; use JSON::PP; +use feature 'state'; sub buildChain :Chained('/') :PathPart('build') :CaptureArgs(1) { my ($self, $c, $id) = @_; @@ -38,6 +39,17 @@ sub buildChain :Chained('/') :PathPart('build') :CaptureArgs(1) { $c->stash->{jobset} = $c->stash->{build}->jobset; $c->stash->{job} = $c->stash->{build}->job; $c->stash->{runcommandlogs} = [$c->stash->{build}->runcommandlogs->search({}, {order_by => ["id DESC"]})]; + + $c->stash->{runcommandlogProblem} = undef; + if ($c->stash->{job} =~ qr/^runCommandHook\..*/) { + if (!$c->config->{dynamicruncommand}->{enable}) { + $c->stash->{runcommandlogProblem} = "disabled-server"; + } elsif (!$c->stash->{project}->enable_dynamic_run_command) { + $c->stash->{runcommandlogProblem} = "disabled-project"; + } elsif (!$c->stash->{jobset}->enable_dynamic_run_command) { + $c->stash->{runcommandlogProblem} = "disabled-jobset"; + } + } } @@ -223,16 +235,12 @@ sub serveFile { elsif ($ls->{type} eq "regular") { $c->stash->{'plain'} = { data => grab(cmd => ["nix", "--experimental-features", "nix-command", - "cat-store", "--store", getStoreUri(), "$path"]) }; + "store", "cat", "--store", getStoreUri(), "$path"]) }; - # Detect MIME type. Borrowed from Catalyst::Plugin::Static::Simple. - my $type = "text/plain"; - if ($path =~ /.*\.(\S{1,})$/xms) { - my $ext = $1; - my $mimeTypes = MIME::Types->new(only_complete => 1); - my $t = $mimeTypes->mimeTypeOf($ext); - $type = ref $t ? $t->type : $t if $t; - } + # Detect MIME type. + state $magic = File::LibMagic->new(follow_symlinks => 1); + my $info = $magic->info_from_filename($path); + my $type = $info->{mime_with_encoding}; $c->response->content_type($type); $c->forward('Hydra::View::Plain'); } @@ -277,29 +285,7 @@ sub download : Chained('buildChain') PathPart { my $path = $product->path; $path .= "/" . join("/", @path) if scalar @path > 0; - if (isLocalStore) { - - notFound($c, "File '" . $product->path . "' does not exist.") unless -e $product->path; - - # Make sure the file is in the Nix store. - $path = checkPath($self, $c, $path); - - # If this is a directory but no "/" is attached, then redirect. - if (-d $path && substr($c->request->uri, -1) ne "/") { - return $c->res->redirect($c->request->uri . "/"); - } - - $path = "$path/index.html" if -d $path && -e "$path/index.html"; - - notFound($c, "File '$path' does not exist.") if !-e $path; - - notFound($c, "Path '$path' is a directory.") if -d $path; - - $c->serve_static_file($path); - - } else { - serveFile($c, $path); - } + serveFile($c, $path); $c->response->headers->last_modified($c->stash->{build}->stoptime); } @@ -355,7 +341,7 @@ sub contents : Chained('buildChain') PathPart Args(1) { # FIXME: don't use shell invocations below. - # FIXME: use nix cat-store + # FIXME: use nix store cat my $res; diff --git a/src/lib/Hydra/Controller/Job.pm b/src/lib/Hydra/Controller/Job.pm index 378887ea..b392e8e1 100644 --- a/src/lib/Hydra/Controller/Job.pm +++ b/src/lib/Hydra/Controller/Job.pm @@ -69,7 +69,7 @@ sub prometheus : Chained('job') PathPart('prometheus') Args(0) { my $lastBuild = $c->stash->{jobset}->builds->find( { job => $c->stash->{job}, finished => 1 }, - { order_by => 'id DESC', rows => 1, columns => [@buildListColumns] } + { order_by => 'id DESC', rows => 1, columns => ["stoptime", "buildstatus", "closuresize", "size"] } ); $prometheus->new_counter( @@ -92,6 +92,26 @@ sub prometheus : Chained('job') PathPart('prometheus') Args(0) { $c->stash->{job}, )->inc($lastBuild->buildstatus > 0); + $prometheus->new_gauge( + name => "hydra_build_closure_size", + help => "Closure size of the last job's build in bytes", + labels => [ "project", "jobset", "job" ] + )->labels( + $c->stash->{project}->name, + $c->stash->{jobset}->name, + $c->stash->{job}, + )->inc($lastBuild->closuresize); + + $prometheus->new_gauge( + name => "hydra_build_output_size", + help => "Output size of the last job's build in bytes", + labels => [ "project", "jobset", "job" ] + )->labels( + $c->stash->{project}->name, + $c->stash->{jobset}->name, + $c->stash->{job}, + )->inc($lastBuild->size); + $c->stash->{'plain'} = { data => $prometheus->render }; $c->forward('Hydra::View::Plain'); } diff --git a/src/lib/Hydra/Controller/Jobset.pm b/src/lib/Hydra/Controller/Jobset.pm index b952031f..eeb4232a 100644 --- a/src/lib/Hydra/Controller/Jobset.pm +++ b/src/lib/Hydra/Controller/Jobset.pm @@ -261,6 +261,14 @@ sub updateJobset { my $checkinterval = int(trim($c->stash->{params}->{checkinterval})); + my $enable_dynamic_run_command = defined $c->stash->{params}->{enable_dynamic_run_command} ? 1 : 0; + if ($enable_dynamic_run_command + && !($c->config->{dynamicruncommand}->{enable} + && $jobset->project->enable_dynamic_run_command)) + { + badRequest($c, "Dynamic RunCommand is not enabled by the server or the parent project."); + } + $jobset->update( { name => $jobsetName , description => trim($c->stash->{params}->{"description"}) @@ -268,6 +276,7 @@ sub updateJobset { , nixexprinput => $nixExprInput , enabled => $enabled , enableemail => defined $c->stash->{params}->{enableemail} ? 1 : 0 + , enable_dynamic_run_command => $enable_dynamic_run_command , emailoverride => trim($c->stash->{params}->{emailoverride}) || "" , hidden => defined $c->stash->{params}->{visible} ? 0 : 1 , keepnr => int(trim($c->stash->{params}->{keepnr} // "0")) diff --git a/src/lib/Hydra/Controller/Project.pm b/src/lib/Hydra/Controller/Project.pm index ed3c527c..1141de4a 100644 --- a/src/lib/Hydra/Controller/Project.pm +++ b/src/lib/Hydra/Controller/Project.pm @@ -149,6 +149,11 @@ sub updateProject { my $displayName = trim $c->stash->{params}->{displayname}; error($c, "You must specify a display name.") if $displayName eq ""; + my $enable_dynamic_run_command = defined $c->stash->{params}->{enable_dynamic_run_command} ? 1 : 0; + if ($enable_dynamic_run_command && !$c->config->{dynamicruncommand}->{enable}) { + badRequest($c, "Dynamic RunCommand is not enabled by the server."); + } + $project->update( { name => $projectName , displayname => $displayName @@ -157,6 +162,7 @@ sub updateProject { , enabled => defined $c->stash->{params}->{enabled} ? 1 : 0 , hidden => defined $c->stash->{params}->{visible} ? 0 : 1 , owner => $owner + , enable_dynamic_run_command => $enable_dynamic_run_command , declfile => trim($c->stash->{params}->{declarative}->{file}) , decltype => trim($c->stash->{params}->{declarative}->{type}) , declvalue => trim($c->stash->{params}->{declarative}->{value}) diff --git a/src/lib/Hydra/Helper/AddBuilds.pm b/src/lib/Hydra/Helper/AddBuilds.pm index 1e6d8944..a6373be5 100644 --- a/src/lib/Hydra/Helper/AddBuilds.pm +++ b/src/lib/Hydra/Helper/AddBuilds.pm @@ -19,14 +19,16 @@ use Hydra::Helper::CatalystUtils; our @ISA = qw(Exporter); our @EXPORT = qw( + validateDeclarativeJobset + createJobsetInputsRowAndData updateDeclarativeJobset handleDeclarativeJobsetBuild handleDeclarativeJobsetJson ); -sub updateDeclarativeJobset { - my ($db, $project, $jobsetName, $declSpec) = @_; +sub validateDeclarativeJobset { + my ($config, $project, $jobsetName, $declSpec) = @_; my @allowed_keys = qw( enabled @@ -39,6 +41,7 @@ sub updateDeclarativeJobset { checkinterval schedulingshares enableemail + enable_dynamic_run_command emailoverride keepnr ); @@ -61,16 +64,39 @@ sub updateDeclarativeJobset { } } + my $enable_dynamic_run_command = defined $update{enable_dynamic_run_command} ? 1 : 0; + if ($enable_dynamic_run_command + && !($config->{dynamicruncommand}->{enable} + && $project->enable_dynamic_run_command)) + { + die "Dynamic RunCommand is not enabled by the server or the parent project."; + } + + return %update; +} + +sub createJobsetInputsRowAndData { + my ($name, $declSpec) = @_; + my $data = $declSpec->{"inputs"}->{$name}; + my $row = { + name => $name, + type => $data->{type} + }; + $row->{emailresponsible} = $data->{emailresponsible} // 0; + + return ($row, $data); +} + +sub updateDeclarativeJobset { + my ($config, $db, $project, $jobsetName, $declSpec) = @_; + + my %update = validateDeclarativeJobset($config, $project, $jobsetName, $declSpec); + $db->txn_do(sub { my $jobset = $project->jobsets->update_or_create(\%update); $jobset->jobsetinputs->delete; foreach my $name (keys %{$declSpec->{"inputs"}}) { - my $data = $declSpec->{"inputs"}->{$name}; - my $row = { - name => $name, - type => $data->{type} - }; - $row->{emailresponsible} = $data->{emailresponsible} // 0; + my ($row, $data) = createJobsetInputsRowAndData($name, $declSpec); my $input = $jobset->jobsetinputs->create($row); $input->jobsetinputalts->create({altnr => 0, value => $data->{value}}); } @@ -81,6 +107,7 @@ sub updateDeclarativeJobset { sub handleDeclarativeJobsetJson { my ($db, $project, $declSpec) = @_; + my $config = getHydraConfig(); $db->txn_do(sub { my @kept = keys %$declSpec; push @kept, ".jobsets"; @@ -88,7 +115,7 @@ sub handleDeclarativeJobsetJson { foreach my $jobsetName (keys %$declSpec) { my $spec = $declSpec->{$jobsetName}; eval { - updateDeclarativeJobset($db, $project, $jobsetName, $spec); + updateDeclarativeJobset($config, $db, $project, $jobsetName, $spec); 1; } or do { print STDERR "ERROR: failed to process declarative jobset ", $project->name, ":${jobsetName}, ", $@, "\n"; diff --git a/src/lib/Hydra/Helper/Nix.pm b/src/lib/Hydra/Helper/Nix.pm index 514fb439..71a8a7d7 100644 --- a/src/lib/Hydra/Helper/Nix.pm +++ b/src/lib/Hydra/Helper/Nix.pm @@ -537,7 +537,7 @@ sub getStoreUri { sub readNixFile { my ($path) = @_; return grab(cmd => ["nix", "--experimental-features", "nix-command", - "cat-store", "--store", getStoreUri(), "$path"]); + "store", "cat", "--store", getStoreUri(), "$path"]); } diff --git a/src/lib/Hydra/Plugin/GitInput.pm b/src/lib/Hydra/Plugin/GitInput.pm index aca35c30..e5fc7de9 100644 --- a/src/lib/Hydra/Plugin/GitInput.pm +++ b/src/lib/Hydra/Plugin/GitInput.pm @@ -261,7 +261,7 @@ sub getCommits { my $clonePath = getSCMCacheDir . "/git/" . sha256_hex($uri); - my $out = grab(cmd => ["git", "log", "--pretty=format:%H%x09%an%x09%ae%x09%at", "$rev1..$rev2"], dir => $clonePath); + my $out = grab(cmd => ["git", "--git-dir=.git", "log", "--pretty=format:%H%x09%an%x09%ae%x09%at", "$rev1..$rev2"], dir => $clonePath); my $res = []; foreach my $line (split /\n/, $out) { diff --git a/src/lib/Hydra/Plugin/GithubPulls.pm b/src/lib/Hydra/Plugin/GithubPulls.pm index db0e8d25..9d8412c3 100644 --- a/src/lib/Hydra/Plugin/GithubPulls.pm +++ b/src/lib/Hydra/Plugin/GithubPulls.pm @@ -30,7 +30,7 @@ sub _iterate { $pulls->{$pull->{number}} = $pull; } # TODO Make Link header parsing more robust!!! - my @links = split ',', $res->header("Link"); + my @links = split ',', ($res->header("Link") // ""); my $next = ""; foreach my $link (@links) { my ($url, $rel) = split ";", $link; diff --git a/src/lib/Hydra/Plugin/HipChatNotification.pm b/src/lib/Hydra/Plugin/HipChatNotification.pm deleted file mode 100644 index 61cea57a..00000000 --- a/src/lib/Hydra/Plugin/HipChatNotification.pm +++ /dev/null @@ -1,89 +0,0 @@ -package Hydra::Plugin::HipChatNotification; - -use strict; -use warnings; -use parent 'Hydra::Plugin'; -use LWP::UserAgent; -use Hydra::Helper::CatalystUtils; - -sub isEnabled { - my ($self) = @_; - return defined $self->{config}->{hipchat}; -} - -sub buildFinished { - my ($self, $topbuild, $dependents) = @_; - - my $cfg = $self->{config}->{hipchat}; - my @config = defined $cfg ? ref $cfg eq "ARRAY" ? @$cfg : ($cfg) : (); - - my $baseurl = $self->{config}->{'base_uri'} || "http://localhost:3000"; - - # Figure out to which rooms to send notification. For each email - # room, we send one aggregate message. - my %rooms; - foreach my $build ($topbuild, @{$dependents}) { - my $prevBuild = getPreviousBuild($build); - my $jobName = showJobName $build; - - foreach my $room (@config) { - my $force = $room->{force}; - next unless $jobName =~ /^$room->{jobs}$/; - - # If build is cancelled or aborted, do not send email. - next if ! $force && ($build->buildstatus == 4 || $build->buildstatus == 3); - - # If there is a previous (that is not cancelled or aborted) build - # with same buildstatus, do not send email. - next if ! $force && defined $prevBuild && ($build->buildstatus == $prevBuild->buildstatus); - - $rooms{$room->{room}} //= { room => $room, builds => [] }; - push @{$rooms{$room->{room}}->{builds}}, $build; - } - } - - return if scalar keys %rooms == 0; - - my ($authors, $nrCommits) = getResponsibleAuthors($topbuild, $self->{plugins}); - - # Send a message to each room. - foreach my $roomId (keys %rooms) { - my $room = $rooms{$roomId}; - my @deps = grep { $_->id != $topbuild->id } @{$room->{builds}}; - - my $img = - $topbuild->buildstatus == 0 ? "$baseurl/static/images/checkmark_16.png" : - $topbuild->buildstatus == 2 ? "$baseurl/static/images/dependency_16.png" : - $topbuild->buildstatus == 4 ? "$baseurl/static/images/cancelled_16.png" : - "$baseurl/static/images/error_16.png"; - - my $msg = ""; - $msg .= " "; - $msg .= "Job jobset->get_column('name')}/${\$topbuild->get_column('job')}'>${\showJobName($topbuild)}"; - $msg .= " (and ${\scalar @deps} others)" if scalar @deps > 0; - $msg .= ": " . showStatus($topbuild) . ""; - - if (scalar keys %{$authors} > 0) { - # FIXME: HTML escaping - my @x = map { "$_" } (sort keys %{$authors}); - $msg .= ", likely due to "; - $msg .= "$nrCommits commits by " if $nrCommits > 1; - $msg .= join(" or ", scalar @x > 1 ? join(", ", @x[0..scalar @x - 2]) : (), $x[-1]); - } - - print STDERR "sending hipchat notification to room $roomId: $msg\n"; - - my $ua = LWP::UserAgent->new(); - my $resp = $ua->post('https://api.hipchat.com/v1/rooms/message?format=json&auth_token=' . $room->{room}->{token}, { - room_id => $roomId, - from => 'Hydra', - message => $msg, - message_format => 'html', - notify => $room->{room}->{notify} || 0, - color => $topbuild->buildstatus == 0 ? 'green' : 'red' }); - - print STDERR $resp->status_line, ": ", $resp->decoded_content,"\n" if !$resp->is_success; - } -} - -1; diff --git a/src/lib/Hydra/Plugin/RunCommand.pm b/src/lib/Hydra/Plugin/RunCommand.pm index 401942c7..43163764 100644 --- a/src/lib/Hydra/Plugin/RunCommand.pm +++ b/src/lib/Hydra/Plugin/RunCommand.pm @@ -12,7 +12,74 @@ use Try::Tiny; sub isEnabled { my ($self) = @_; - return defined $self->{config}->{runcommand}; + + return areStaticCommandsEnabled($self->{config}) || areDynamicCommandsEnabled($self->{config}); +} + +sub areStaticCommandsEnabled { + my ($config) = @_; + + if (defined $config->{runcommand}) { + return 1; + } + + return 0; +} + +sub areDynamicCommandsEnabled { + my ($config) = @_; + + if ((defined $config->{dynamicruncommand}) + && $config->{dynamicruncommand}->{enable}) { + return 1; + } + + return 0; +} + +sub isBuildEligibleForDynamicRunCommand { + my ($build) = @_; + + if ($build->get_column("buildstatus") != 0) { + return 0; + } + + if ($build->get_column("job") =~ "^runCommandHook\..+") { + my $out = $build->buildoutputs->find({name => "out"}); + if (!defined $out) { + warn "DynamicRunCommand hook on " . $build->job . " (" . $build->id . ") rejected: no output named 'out'."; + return 0; + } + + my $path = $out->path; + if (-l $path) { + $path = readlink($path); + } + + if (! -e $path) { + warn "DynamicRunCommand hook on " . $build->job . " (" . $build->id . ") rejected: The 'out' output doesn't exist locally. This is a bug."; + return 0; + } + + if (! -x $path) { + warn "DynamicRunCommand hook on " . $build->job . " (" . $build->id . ") rejected: The 'out' output is not executable."; + return 0; + } + + if (! -f $path) { + warn "DynamicRunCommand hook on " . $build->job . " (" . $build->id . ") rejected: The 'out' output is not a regular file or symlink."; + return 0; + } + + if (! $build->jobset->supportsDynamicRunCommand()) { + warn "DynamicRunCommand hook on " . $build->job . " (" . $build->id . ") rejected: The project or jobset don't have dynamic runcommand enabled."; + return 0; + } + + return 1; + } + + return 0; } sub configSectionMatches { @@ -43,10 +110,11 @@ sub eventMatches { } sub fanoutToCommands { - my ($config, $event, $project, $jobset, $job) = @_; + my ($config, $event, $build) = @_; my @commands; + # Calculate all the statically defined commands to execute my $cfg = $config->{runcommand}; my @config = defined $cfg ? ref $cfg eq "ARRAY" ? @$cfg : ($cfg) : (); @@ -55,9 +123,10 @@ sub fanoutToCommands { next unless eventMatches($conf, $event); next unless configSectionMatches( $matcher, - $project, - $jobset, - $job); + $build->jobset->get_column('project'), + $build->jobset->get_column('name'), + $build->get_column('job') + ); if (!defined($conf->{command})) { warn " section for '$matcher' lacks a 'command' option"; @@ -70,6 +139,18 @@ sub fanoutToCommands { }) } + # Calculate all dynamically defined commands to execute + if (areDynamicCommandsEnabled($config)) { + if (isBuildEligibleForDynamicRunCommand($build)) { + my $job = $build->get_column('job'); + my $out = $build->buildoutputs->find({name => "out"}); + push(@commands, { + matcher => "DynamicRunCommand($job)", + command => $out->path + }) + } + } + return \@commands; } @@ -138,9 +219,7 @@ sub buildFinished { my $commandsToRun = fanoutToCommands( $self->{config}, $event, - $build->project->get_column('name'), - $build->jobset->get_column('name'), - $build->get_column('job') + $build ); if (@$commandsToRun == 0) { diff --git a/src/lib/Hydra/Schema/Result/Jobsets.pm b/src/lib/Hydra/Schema/Result/Jobsets.pm index 13ac09e4..cd704ac8 100644 --- a/src/lib/Hydra/Schema/Result/Jobsets.pm +++ b/src/lib/Hydra/Schema/Result/Jobsets.pm @@ -155,6 +155,12 @@ __PACKAGE__->table("jobsets"); data_type: 'text' is_nullable: 1 +=head2 enable_dynamic_run_command + + data_type: 'boolean' + default_value: false + is_nullable: 0 + =cut __PACKAGE__->add_columns( @@ -207,6 +213,8 @@ __PACKAGE__->add_columns( { data_type => "integer", default_value => 0, is_nullable => 0 }, "flake", { data_type => "text", is_nullable => 1 }, + "enable_dynamic_run_command", + { data_type => "boolean", default_value => \"false", is_nullable => 0 }, ); =head1 PRIMARY KEY @@ -354,8 +362,8 @@ __PACKAGE__->has_many( ); -# Created by DBIx::Class::Schema::Loader v0.07049 @ 2022-01-08 22:24:10 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:cQOnMitrWGMoJX6kZGNW+w +# Created by DBIx::Class::Schema::Loader v0.07049 @ 2022-01-24 14:17:33 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:7wPE5ebeVTkenMCWG9Sgcg use JSON::MaybeXS; @@ -378,6 +386,13 @@ __PACKAGE__->add_column( "+id" => { retrieve_on_insert => 1 } ); +sub supportsDynamicRunCommand { + my ($self) = @_; + + return $self->get_column('enable_dynamic_run_command') == 1 + && $self->project->supportsDynamicRunCommand(); +} + sub as_json { my $self = shift; @@ -406,6 +421,7 @@ sub as_json { # boolean_columns "enableemail" => $self->get_column("enableemail") ? JSON::MaybeXS::true : JSON::MaybeXS::false, + "enable_dynamic_run_command" => $self->get_column("enable_dynamic_run_command") ? JSON::MaybeXS::true : JSON::MaybeXS::false, "visible" => $self->get_column("hidden") ? JSON::MaybeXS::false : JSON::MaybeXS::true, "inputs" => { map { $_->name => $_ } $self->jobsetinputs } diff --git a/src/lib/Hydra/Schema/Result/Projects.pm b/src/lib/Hydra/Schema/Result/Projects.pm index 35c3eeab..d6e66bf7 100644 --- a/src/lib/Hydra/Schema/Result/Projects.pm +++ b/src/lib/Hydra/Schema/Result/Projects.pm @@ -88,6 +88,12 @@ __PACKAGE__->table("projects"); data_type: 'text' is_nullable: 1 +=head2 enable_dynamic_run_command + + data_type: 'boolean' + default_value: false + is_nullable: 0 + =cut __PACKAGE__->add_columns( @@ -111,6 +117,8 @@ __PACKAGE__->add_columns( { data_type => "text", is_nullable => 1 }, "declvalue", { data_type => "text", is_nullable => 1 }, + "enable_dynamic_run_command", + { data_type => "boolean", default_value => \"false", is_nullable => 0 }, ); =head1 PRIMARY KEY @@ -228,8 +236,8 @@ Composing rels: L -> username __PACKAGE__->many_to_many("usernames", "projectmembers", "username"); -# Created by DBIx::Class::Schema::Loader v0.07049 @ 2022-01-08 22:24:10 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:r/wbX3FAm5/OFrrwOQL5fA +# Created by DBIx::Class::Schema::Loader v0.07049 @ 2022-01-24 14:20:32 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:PtXDyT8Pc7LYhhdEG39EKQ use JSON::MaybeXS; @@ -238,6 +246,12 @@ sub builds { return $self->jobsets->related_resultset('builds'); }; +sub supportsDynamicRunCommand { + my ($self) = @_; + + return $self->get_column('enable_dynamic_run_command') == 1; +} + sub as_json { my $self = shift; @@ -251,6 +265,7 @@ sub as_json { # boolean_columns "enabled" => $self->get_column("enabled") ? JSON::MaybeXS::true : JSON::MaybeXS::false, + "enable_dynamic_run_command" => $self->get_column("enable_dynamic_run_command") ? JSON::MaybeXS::true : JSON::MaybeXS::false, "hidden" => $self->get_column("hidden") ? JSON::MaybeXS::true : JSON::MaybeXS::false, "jobsets" => [ map { $_->name } $self->jobsets ] diff --git a/src/root/build-deps.tt b/src/root/build-deps.tt index 157b113e..f15342fa 100644 --- a/src/root/build-deps.tt +++ b/src/root/build-deps.tt @@ -4,6 +4,6 @@
    - [% INCLUDE renderNode node=buildTimeGraph %] + [% INCLUDE renderNode node=buildTimeGraph isRoot=1 %]
diff --git a/src/root/build.tt b/src/root/build.tt index 0848da4a..93a02e0f 100644 --- a/src/root/build.tt +++ b/src/root/build.tt @@ -149,7 +149,7 @@ END; [% IF build.dependents %]
  • Used By
    • [% END%] [% IF drvAvailable %]
  • Build Dependencies
    • [% END %] [% IF localStore && available %]
  • Runtime Dependencies
    • [% END %] - [% IF runcommandlogs.size() > 0 %]
  • RunCommand Logs
    • [% END %] + [% IF runcommandlogProblem || runcommandlogs.size() > 0 %]
  • RunCommand Logs[% IF runcommandlogProblem %] Disabled[% END %]
    • [% END %]
    @@ -481,14 +481,27 @@ END; [% END %] [% IF drvAvailable %] - [% INCLUDE makeLazyTab tabName="tabs-build-deps" uri=c.uri_for('/build' build.id 'build-deps') %] + [% INCLUDE makeLazyTab tabName="tabs-build-deps" uri=c.uri_for('/build' build.id 'build-deps') callback="makeTreeCollapsible" %] [% END %] [% IF available %] - [% INCLUDE makeLazyTab tabName="tabs-runtime-deps" uri=c.uri_for('/build' build.id 'runtime-deps') %] + [% INCLUDE makeLazyTab tabName="tabs-runtime-deps" uri=c.uri_for('/build' build.id 'runtime-deps') callback="makeTreeCollapsible" %] [% END %]
    + [% IF runcommandlogProblem %] + + [% END %]
    [% FOREACH runcommandlog IN runcommandlogs %]
    diff --git a/src/root/common.tt b/src/root/common.tt index 6b4e406e..32d6c8cc 100644 --- a/src/root/common.tt +++ b/src/root/common.tt @@ -520,7 +520,11 @@ BLOCK makeLazyTab %]
    [% END; diff --git a/src/root/deps.tt b/src/root/deps.tt index a2c1fbba..6daa9725 100644 --- a/src/root/deps.tt +++ b/src/root/deps.tt @@ -19,9 +19,16 @@ [% node.name %] (no info) [% END %] + [% IF isRoot %] + + (collapse all + – + expand all) + + [% END %] [% IF node.refs.size > 0 %]
      - [% FOREACH ref IN node.refs; INCLUDE renderNode node=ref; END %] + [% FOREACH ref IN node.refs; INCLUDE renderNode node=ref isRoot=0; END %]
    [% END %] [% END %] diff --git a/src/root/edit-jobset.tt b/src/root/edit-jobset.tt index dbd26dcc..61e3636f 100644 --- a/src/root/edit-jobset.tt +++ b/src/root/edit-jobset.tt @@ -157,6 +157,21 @@
    +
    + +
    + +
    +
    +
    diff --git a/src/root/edit-project.tt b/src/root/edit-project.tt index 6149ec1d..bb850e5c 100644 --- a/src/root/edit-project.tt +++ b/src/root/edit-project.tt @@ -52,6 +52,20 @@
    + +
    + +
    + +
    +
    +
    diff --git a/src/root/static/css/hydra.css b/src/root/static/css/hydra.css index bbe0f1dd..ec7e4b64 100644 --- a/src/root/static/css/hydra.css +++ b/src/root/static/css/hydra.css @@ -1,5 +1,5 @@ div.skip-topbar { - padding-top: 40px; + padding-top: 20px; margin-bottom: 1.5em; } @@ -33,6 +33,11 @@ span:target > span.dep-tree-line { font-weight: bold; } +span.dep-tree-buttons { + font-style: italic; + padding-left: 10px; +} + span.disabled-project, span.disabled-jobset, span.disabled-job { text-decoration: line-through; } @@ -146,6 +151,36 @@ td.step-status span.warn { padding-top: 1.5rem; } +.container { + max-width: 80%; +} + +.tab-content { + margin-right: 0 !important; +} + +body { + line-height: 1; +} + +.navbar-nav { + line-height: 1.5; +} + +.dropdown-item { + line-height: 1.5; +} + +a.squiggle:hover { + background-image: url("data:image/svg+xml;charset=utf8,%3Csvg id='squiggle-link' xmlns='http://www.w3.org/2000/svg' xmlns:xlink='http://www.w3.org/1999/xlink' xmlns:ev='http://www.w3.org/2001/xml-events' viewBox='0 0 10 18'%3E%3Cstyle type='text/css'%3E.squiggle{animation:shift .5s linear infinite;}@keyframes shift {from {transform:translateX(-10px);}to {transform:translateX(0);}}%3C/style%3E%3Cpath fill='none' stroke='%230056b3' stroke-width='0.65' class='squiggle' d='M0,17.5 c 2.5,0,2.5,-1.5,5,-1.5 s 2.5,1.5,5,1.5 c 2.5,0,2.5,-1.5,5,-1.5 s 2.5,1.5,5,1.5' /%3E%3C/svg%3E"); + background-position: 0 100%; + background-size: auto 24px; + background-repeat: repeat; + text-decoration: none; + border-bottom: none; + padding-bottom: 1px; +} + @media (prefers-color-scheme: dark) { /* Prevent some flickering */ html { diff --git a/src/root/static/css/tree.css b/src/root/static/css/tree.css index 71df2f12..e80f6871 100644 --- a/src/root/static/css/tree.css +++ b/src/root/static/css/tree.css @@ -9,6 +9,7 @@ ul.tree, ul.subtree { ul.subtree > li { position: relative; padding-left: 2.0em; + line-height: 140%; border-left: 0.1em solid #6185a0; } diff --git a/src/root/static/js/common.js b/src/root/static/js/common.js index 433c06d7..c51f769a 100644 --- a/src/root/static/js/common.js +++ b/src/root/static/js/common.js @@ -1,10 +1,9 @@ -$(document).ready(function() { - +function makeTreeCollapsible(tab) { /*** Tree toggles in logfiles. ***/ /* Set the appearance of the toggle depending on whether the corresponding subtree is initially shown or hidden. */ - $(".tree-toggle").map(function() { + tab.find(".tree-toggle").map(function() { if ($(this).siblings("ul:hidden").length == 0) { $(this).text("-"); } else { @@ -13,7 +12,7 @@ $(document).ready(function() { }); /* When a toggle is clicked, show or hide the subtree. */ - $(".tree-toggle").click(function() { + tab.find(".tree-toggle").click(function() { if ($(this).siblings("ul:hidden").length != 0) { $(this).siblings("ul").show(); $(this).text("-"); @@ -24,21 +23,23 @@ $(document).ready(function() { }); /* Implementation of the expand all link. */ - $(".tree-expand-all").click(function() { - $(".tree-toggle", $(this).parent().siblings(".tree")).map(function() { + tab.find(".tree-expand-all").click(function() { + tab.find(".tree-toggle", $(this).parent().siblings(".tree")).map(function() { $(this).siblings("ul").show(); $(this).text("-"); }); }); /* Implementation of the collapse all link. */ - $(".tree-collapse-all").click(function() { - $(".tree-toggle", $(this).parent().siblings(".tree")).map(function() { + tab.find(".tree-collapse-all").click(function() { + tab.find(".tree-toggle", $(this).parent().siblings(".tree")).map(function() { $(this).siblings("ul").hide(); $(this).text("+"); }); }); +} +$(document).ready(function() { $("table.clickable-rows").click(function(event) { if ($(event.target).closest("a").length) return; link = $(event.target).parents("tr").find("a.row-link"); @@ -132,7 +133,7 @@ $(document).ready(function() { var tabsLoaded = {}; -function makeLazyTab(tabName, uri) { +function makeLazyTab(tabName, uri, callback) { $('.nav-tabs').bind('show.bs.tab', function(e) { var pattern = /#.+/gi; var id = e.target.toString().match(pattern)[0]; @@ -140,11 +141,15 @@ function makeLazyTab(tabName, uri) { tabsLoaded[id] = 1; $('#' + tabName).load(uri, function(response, status, xhr) { var lazy = xhr.getResponseHeader("X-Hydra-Lazy") === "Yes"; + var tab = $('#' + tabName); if (status == "error" && !lazy) { - $('#' + tabName).html("
    Error loading tab: " + xhr.status + " " + xhr.statusText + "
    "); + tab.html("
    Error loading tab: " + xhr.status + " " + xhr.statusText + "
    "); } else { - $('#' + tabName).html(response); + tab.html(response); + if (callback) { + callback(tab); + } } }); } diff --git a/src/script/hydra-eval-jobset b/src/script/hydra-eval-jobset index 8bafe07c..c6f6c275 100755 --- a/src/script/hydra-eval-jobset +++ b/src/script/hydra-eval-jobset @@ -619,7 +619,7 @@ sub checkJobsetWrapped { } else { # Update the jobset with the spec's inputs, and the continue # evaluating the .jobsets jobset. - updateDeclarativeJobset($db, $project, ".jobsets", $declSpec); + updateDeclarativeJobset($config, $db, $project, ".jobsets", $declSpec); $jobset->discard_changes; $inputInfo->{"declInput"} = [ $declInput ]; $inputInfo->{"projectName"} = [ fetchInput($plugins, $db, $project, $jobset, "projectName", "string", $project->name, 0) ]; @@ -640,8 +640,8 @@ sub checkJobsetWrapped { my $flakeRef = $jobset->flake; if (defined $flakeRef) { (my $res, my $json, my $stderr) = captureStdoutStderr( - 600, "nix", "flake", "info", "--tarball-ttl", 0, "--json", "--", $flakeRef); - die "'nix flake info' returned " . ($res & 127 ? "signal $res" : "exit code " . ($res >> 8)) + 600, "nix", "flake", "metadata", "--refresh", "--json", "--", $flakeRef); + die "'nix flake metadata' returned " . ($res & 127 ? "signal $res" : "exit code " . ($res >> 8)) . ":\n" . ($stderr ? decode("utf-8", $stderr) : "(no output)\n") if $res; $flakeRef = decode_json($json)->{'url'}; diff --git a/src/sql/hydra.sql b/src/sql/hydra.sql index 26617789..eaae6da3 100644 --- a/src/sql/hydra.sql +++ b/src/sql/hydra.sql @@ -49,6 +49,7 @@ create table Projects ( declfile text, -- File containing declarative jobset specification decltype text, -- Type of the input containing declarative jobset specification declvalue text, -- Value of the input containing declarative jobset specification + enable_dynamic_run_command boolean not null default false, foreign key (owner) references Users(userName) on update cascade ); @@ -88,6 +89,7 @@ create table Jobsets ( startTime integer, -- if jobset is currently running type integer not null default 0, -- 0 == legacy, 1 == flake flake text, + enable_dynamic_run_command boolean not null default false, constraint jobsets_schedulingshares_nonzero_check check (schedulingShares > 0), constraint jobsets_type_known_check check (type = 0 or type = 1), -- If the type is 0, then nixExprInput and nixExprPath should be non-null and other type-specific fields should be null diff --git a/src/sql/upgrade-82.sql b/src/sql/upgrade-82.sql new file mode 100644 index 00000000..a619caf3 --- /dev/null +++ b/src/sql/upgrade-82.sql @@ -0,0 +1,4 @@ +ALTER TABLE Jobsets + ADD COLUMN enable_dynamic_run_command boolean not null default false; +ALTER TABLE Projects + ADD COLUMN enable_dynamic_run_command boolean not null default false; diff --git a/t/Helper/AddBuilds/dynamic-disabled.t b/t/Helper/AddBuilds/dynamic-disabled.t new file mode 100644 index 00000000..0c91f382 --- /dev/null +++ b/t/Helper/AddBuilds/dynamic-disabled.t @@ -0,0 +1,107 @@ +use strict; +use warnings; +use Setup; +use Test2::V0; + +require Catalyst::Test; +use HTTP::Request::Common qw(POST PUT GET DELETE); +use JSON::MaybeXS qw(decode_json encode_json); + +my $ctx = test_context(); + +Catalyst::Test->import('Hydra'); + +my $db = Hydra::Model::DB->new; +hydra_setup($db); + +my $user = $db->resultset('Users')->create({ username => 'alice', emailaddress => 'root@invalid.org', password => '!' }); +$user->setPassword('foobar'); +$user->userroles->update_or_create({ role => 'admin' }); + +my $project_with_dynamic_run_command = $db->resultset('Projects')->create({ + name => 'tests_with_dynamic_runcommand', + displayname => 'Tests with dynamic runcommand', + owner => 'alice', + enable_dynamic_run_command => 1, +}); +my $project_without_dynamic_run_command = $db->resultset('Projects')->create({ + name => 'tests_without_dynamic_runcommand', + displayname => 'Tests without dynamic runcommand', + owner => 'alice', + enable_dynamic_run_command => 0, +}); + +sub makeJobsetSpec { + my ($dynamic) = @_; + + return { + enabled => 2, + enable_dynamic_run_command => $dynamic ? JSON::MaybeXS::true : undef, + visible => JSON::MaybeXS::true, + name => "job", + type => 1, + description => "test jobset", + flake => "github:nixos/nix", + checkinterval => 0, + schedulingshares => 100, + keepnr => 3 + }; +}; + +subtest "validate declarative jobset with dynamic RunCommand disabled by server" => sub { + my $config = Hydra::Helper::Nix->getHydraConfig(); + require Hydra::Helper::AddBuilds; + Hydra::Helper::AddBuilds->import( qw(validateDeclarativeJobset) ); + + subtest "project enabled dynamic runcommand, declarative jobset enabled dynamic runcommand" => sub { + like( + dies { + validateDeclarativeJobset( + $config, + $project_with_dynamic_run_command, + "test-jobset", + makeJobsetSpec(JSON::MaybeXS::true), + ), + }, + qr/Dynamic RunCommand is not enabled/, + ); + }; + + subtest "project enabled dynamic runcommand, declarative jobset disabled dynamic runcommand" => sub { + ok( + validateDeclarativeJobset( + $config, + $project_with_dynamic_run_command, + "test-jobset", + makeJobsetSpec(JSON::MaybeXS::false) + ), + ); + }; + + subtest "project disabled dynamic runcommand, declarative jobset enabled dynamic runcommand" => sub { + like( + dies { + validateDeclarativeJobset( + $config, + $project_without_dynamic_run_command, + "test-jobset", + makeJobsetSpec(JSON::MaybeXS::true), + ), + }, + qr/Dynamic RunCommand is not enabled/, + ); + }; + + subtest "project disabled dynamic runcommand, declarative jobset disabled dynamic runcommand" => sub { + ok( + validateDeclarativeJobset( + $config, + $project_without_dynamic_run_command, + "test-jobset", + makeJobsetSpec(JSON::MaybeXS::false) + ), + ); + }; +}; + +done_testing; diff --git a/t/Helper/AddBuilds/dynamic-enabled.t b/t/Helper/AddBuilds/dynamic-enabled.t new file mode 100644 index 00000000..46497bed --- /dev/null +++ b/t/Helper/AddBuilds/dynamic-enabled.t @@ -0,0 +1,110 @@ +use strict; +use warnings; +use Setup; +use Test2::V0; + +require Catalyst::Test; +use HTTP::Request::Common qw(POST PUT GET DELETE); +use JSON::MaybeXS qw(decode_json encode_json); + +my $ctx = test_context( + hydra_config => q| + + enable = 1 + + | +); + +Catalyst::Test->import('Hydra'); + +my $db = Hydra::Model::DB->new; +hydra_setup($db); + +my $user = $db->resultset('Users')->create({ username => 'alice', emailaddress => 'root@invalid.org', password => '!' }); +$user->setPassword('foobar'); +$user->userroles->update_or_create({ role => 'admin' }); + +my $project_with_dynamic_run_command = $db->resultset('Projects')->create({ + name => 'tests_with_dynamic_runcommand', + displayname => 'Tests with dynamic runcommand', + owner => 'alice', + enable_dynamic_run_command => 1, +}); +my $project_without_dynamic_run_command = $db->resultset('Projects')->create({ + name => 'tests_without_dynamic_runcommand', + displayname => 'Tests without dynamic runcommand', + owner => 'alice', + enable_dynamic_run_command => 0, +}); + +sub makeJobsetSpec { + my ($dynamic) = @_; + + return { + enabled => 2, + enable_dynamic_run_command => $dynamic ? JSON::MaybeXS::true : undef, + visible => JSON::MaybeXS::true, + name => "job", + type => 1, + description => "test jobset", + flake => "github:nixos/nix", + checkinterval => 0, + schedulingshares => 100, + keepnr => 3 + }; +}; + +subtest "validate declarative jobset with dynamic RunCommand enabled by server" => sub { + my $config = Hydra::Helper::Nix->getHydraConfig(); + require Hydra::Helper::AddBuilds; + Hydra::Helper::AddBuilds->import( qw(validateDeclarativeJobset) ); + + subtest "project enabled dynamic runcommand, declarative jobset enabled dynamic runcommand" => sub { + ok( + validateDeclarativeJobset( + $config, + $project_with_dynamic_run_command, + "test-jobset", + makeJobsetSpec(JSON::MaybeXS::true) + ), + ); + }; + + subtest "project enabled dynamic runcommand, declarative jobset disabled dynamic runcommand" => sub { + ok( + validateDeclarativeJobset( + $config, + $project_with_dynamic_run_command, + "test-jobset", + makeJobsetSpec(JSON::MaybeXS::false) + ), + ); + }; + + subtest "project disabled dynamic runcommand, declarative jobset enabled dynamic runcommand" => sub { + like( + dies { + validateDeclarativeJobset( + $config, + $project_without_dynamic_run_command, + "test-jobset", + makeJobsetSpec(JSON::MaybeXS::true), + ), + }, + qr/Dynamic RunCommand is not enabled/, + ); + }; + + subtest "project disabled dynamic runcommand, declarative jobset disabled dynamic runcommand" => sub { + ok( + validateDeclarativeJobset( + $config, + $project_without_dynamic_run_command, + "test-jobset", + makeJobsetSpec(JSON::MaybeXS::false) + ), + ); + }; +}; + +done_testing; diff --git a/t/Hydra/Controller/Job/builds.t b/t/Hydra/Controller/Job/builds.t index e1097c3e..d933fbb8 100644 --- a/t/Hydra/Controller/Job/builds.t +++ b/t/Hydra/Controller/Job/builds.t @@ -55,6 +55,12 @@ subtest "/job/PROJECT/JOBSET/JOB/shield" => sub { subtest "/job/PROJECT/JOBSET/JOB/prometheus" => sub { my $response = request(GET '/job/' . $project->name . '/' . $jobset->name . '/' . $build->job . '/prometheus'); ok($response->is_success, "The page showing the job's prometheus data returns 200."); + my $metrics = $response->content; + + ok($metrics =~ m/hydra_job_failed\{.*\} 0/); + ok($metrics =~ m/hydra_job_completion_time\{.*\} [\d]+/); + ok($metrics =~ m/hydra_build_closure_size\{.*\} 96/); + ok($metrics =~ m/hydra_build_output_size\{.*\} 96/); }; done_testing; diff --git a/t/Hydra/Controller/Jobset/http.t b/t/Hydra/Controller/Jobset/http.t index 32b3a681..4e53949d 100644 --- a/t/Hydra/Controller/Jobset/http.t +++ b/t/Hydra/Controller/Jobset/http.t @@ -73,6 +73,7 @@ subtest 'Read newly-created jobset "job"' => sub { emailoverride => "", enabled => 2, enableemail => JSON::MaybeXS::false, + enable_dynamic_run_command => JSON::MaybeXS::false, errortime => undef, errormsg => "", fetcherrormsg => "", @@ -131,6 +132,7 @@ subtest 'Update jobset "job" to legacy type' => sub { emailoverride => "", enabled => 3, enableemail => JSON::MaybeXS::false, + enable_dynamic_run_command => JSON::MaybeXS::false, errortime => undef, errormsg => "", fetcherrormsg => "", diff --git a/t/Hydra/Controller/projects.t b/t/Hydra/Controller/projects.t index df1290aa..130724cf 100644 --- a/t/Hydra/Controller/projects.t +++ b/t/Hydra/Controller/projects.t @@ -46,6 +46,7 @@ subtest "Read project 'tests'" => sub { description => "", displayname => "Tests", enabled => JSON::MaybeXS::true, + enable_dynamic_run_command => JSON::MaybeXS::false, hidden => JSON::MaybeXS::false, homepage => "", jobsets => [], @@ -85,6 +86,7 @@ subtest "Transitioning from declarative project to normal" => sub { description => "", displayname => "Tests", enabled => JSON::MaybeXS::true, + enable_dynamic_run_command => JSON::MaybeXS::false, hidden => JSON::MaybeXS::false, homepage => "", jobsets => [".jobsets"], @@ -128,6 +130,7 @@ subtest "Transitioning from declarative project to normal" => sub { description => "", displayname => "Tests", enabled => JSON::MaybeXS::true, + enable_dynamic_run_command => JSON::MaybeXS::false, hidden => JSON::MaybeXS::false, homepage => "", jobsets => [], diff --git a/t/Hydra/Plugin/RunCommand/dynamic-disabled.t b/t/Hydra/Plugin/RunCommand/dynamic-disabled.t new file mode 100644 index 00000000..ad2e9a4b --- /dev/null +++ b/t/Hydra/Plugin/RunCommand/dynamic-disabled.t @@ -0,0 +1,110 @@ +use strict; +use warnings; +use Setup; +use Test2::V0; + +require Catalyst::Test; +use HTTP::Request::Common qw(POST PUT GET DELETE); +use JSON::MaybeXS qw(decode_json encode_json); + +my $ctx = test_context(); +Catalyst::Test->import('Hydra'); + +# Create a user to log in to +my $user = $ctx->db->resultset('Users')->create({ username => 'alice', emailaddress => 'root@invalid.org', password => '!' }); +$user->setPassword('foobar'); +$user->userroles->update_or_create({ role => 'admin' }); + +subtest "can't enable dynamic RunCommand when disabled by server" => sub { + my $builds = $ctx->makeAndEvaluateJobset( + expression => "runcommand-dynamic.nix", + build => 1 + ); + + my $build = $builds->{"runCommandHook.example"}; + my $project = $build->project; + my $project_name = $project->name; + my $jobset = $build->jobset; + my $jobset_name = $jobset->name; + + is($project->enable_dynamic_run_command, 0, "dynamic RunCommand is disabled on projects by default"); + is($jobset->enable_dynamic_run_command, 0, "dynamic RunCommand is disabled on jobsets by default"); + + my $req = request(POST '/login', + Referer => 'http://localhost/', + Content => { + username => 'alice', + password => 'foobar' + } + ); + is($req->code, 302, "logged in successfully"); + my $cookie = $req->header("set-cookie"); + + subtest "can't enable dynamic RunCommand on project" => sub { + my $projectresponse = request(GET "/project/$project_name", + Accept => 'application/json', + Content_Type => 'application/json', + Cookie => $cookie, + ); + + my $projectjson = decode_json($projectresponse->content); + $projectjson->{enable_dynamic_run_command} = 1; + + my $projectupdate = request(PUT "/project/$project_name", + Accept => 'application/json', + Content_Type => 'application/json', + Cookie => $cookie, + Content => encode_json($projectjson) + ); + + $projectresponse = request(GET "/project/$project_name", + Accept => 'application/json', + Content_Type => 'application/json', + Cookie => $cookie, + ); + $projectjson = decode_json($projectresponse->content); + + is($projectupdate->code, 400); + like( + $projectupdate->content, + qr/Dynamic RunCommand is not/, + "failed to change enable_dynamic_run_command, not any other error" + ); + is($projectjson->{enable_dynamic_run_command}, JSON::MaybeXS::false); + }; + + subtest "can't enable dynamic RunCommand on jobset" => sub { + my $jobsetresponse = request(GET "/jobset/$project_name/$jobset_name", + Accept => 'application/json', + Content_Type => 'application/json', + Cookie => $cookie, + ); + + my $jobsetjson = decode_json($jobsetresponse->content); + $jobsetjson->{enable_dynamic_run_command} = 1; + + my $jobsetupdate = request(PUT "/jobset/$project_name/$jobset_name", + Accept => 'application/json', + Content_Type => 'application/json', + Cookie => $cookie, + Content => encode_json($jobsetjson) + ); + + $jobsetresponse = request(GET "/jobset/$project_name/$jobset_name", + Accept => 'application/json', + Content_Type => 'application/json', + Cookie => $cookie, + ); + $jobsetjson = decode_json($jobsetresponse->content); + + is($jobsetupdate->code, 400); + like( + $jobsetupdate->content, + qr/Dynamic RunCommand is not/, + "failed to change enable_dynamic_run_command, not any other error" + ); + is($jobsetjson->{enable_dynamic_run_command}, JSON::MaybeXS::false); + }; +}; + +done_testing; diff --git a/t/Hydra/Plugin/RunCommand/dynamic-enabled.t b/t/Hydra/Plugin/RunCommand/dynamic-enabled.t new file mode 100644 index 00000000..68c6d593 --- /dev/null +++ b/t/Hydra/Plugin/RunCommand/dynamic-enabled.t @@ -0,0 +1,106 @@ +use strict; +use warnings; +use Setup; +use Test2::V0; + +require Catalyst::Test; +use HTTP::Request::Common qw(POST PUT GET DELETE); +use JSON::MaybeXS qw(decode_json encode_json); + +my $ctx = test_context( + hydra_config => q| + + enable = 1 + + | +); +Catalyst::Test->import('Hydra'); + +# Create a user to log in to +my $user = $ctx->db->resultset('Users')->create({ username => 'alice', emailaddress => 'root@invalid.org', password => '!' }); +$user->setPassword('foobar'); +$user->userroles->update_or_create({ role => 'admin' }); + +subtest "can enable dynamic RunCommand when enabled by server" => sub { + my $builds = $ctx->makeAndEvaluateJobset( + expression => "runcommand-dynamic.nix", + build => 1 + ); + + my $build = $builds->{"runCommandHook.example"}; + my $project = $build->project; + my $project_name = $project->name; + my $jobset = $build->jobset; + my $jobset_name = $jobset->name; + + is($project->enable_dynamic_run_command, 0, "dynamic RunCommand is disabled on projects by default"); + is($jobset->enable_dynamic_run_command, 0, "dynamic RunCommand is disabled on jobsets by default"); + + my $req = request(POST '/login', + Referer => 'http://localhost/', + Content => { + username => 'alice', + password => 'foobar' + } + ); + is($req->code, 302, "logged in successfully"); + my $cookie = $req->header("set-cookie"); + + subtest "can enable dynamic RunCommand on project" => sub { + my $projectresponse = request(GET "/project/$project_name", + Accept => 'application/json', + Content_Type => 'application/json', + Cookie => $cookie, + ); + + my $projectjson = decode_json($projectresponse->content); + $projectjson->{enable_dynamic_run_command} = 1; + + my $projectupdate = request(PUT "/project/$project_name", + Accept => 'application/json', + Content_Type => 'application/json', + Cookie => $cookie, + Content => encode_json($projectjson) + ); + + $projectresponse = request(GET "/project/$project_name", + Accept => 'application/json', + Content_Type => 'application/json', + Cookie => $cookie, + ); + $projectjson = decode_json($projectresponse->content); + + is($projectupdate->code, 200); + is($projectjson->{enable_dynamic_run_command}, JSON::MaybeXS::true); + }; + + subtest "can enable dynamic RunCommand on jobset" => sub { + my $jobsetresponse = request(GET "/jobset/$project_name/$jobset_name", + Accept => 'application/json', + Content_Type => 'application/json', + Cookie => $cookie, + ); + + my $jobsetjson = decode_json($jobsetresponse->content); + $jobsetjson->{enable_dynamic_run_command} = 1; + + my $jobsetupdate = request(PUT "/jobset/$project_name/$jobset_name", + Accept => 'application/json', + Content_Type => 'application/json', + Cookie => $cookie, + Content => encode_json($jobsetjson) + ); + + $jobsetresponse = request(GET "/jobset/$project_name/$jobset_name", + Accept => 'application/json', + Content_Type => 'application/json', + Cookie => $cookie, + ); + $jobsetjson = decode_json($jobsetresponse->content); + + is($jobsetupdate->code, 200); + is($jobsetjson->{enable_dynamic_run_command}, JSON::MaybeXS::true); + }; +}; + +done_testing; diff --git a/t/Hydra/Plugin/RunCommand/fanout.t b/t/Hydra/Plugin/RunCommand/fanout.t new file mode 100644 index 00000000..328824f9 --- /dev/null +++ b/t/Hydra/Plugin/RunCommand/fanout.t @@ -0,0 +1,233 @@ +use strict; +use warnings; +use Setup; +use Test2::V0; +use Hydra::Plugin::RunCommand; + +my $ctx = test_context(); + +my $builds = $ctx->makeAndEvaluateJobset( + expression => "runcommand-dynamic.nix", + build => 1 +); + +my $build = $builds->{"runCommandHook.example"}; + +# Enable dynamic runcommand on the project and jobset +$build->project->update({enable_dynamic_run_command => 1}); +$build->jobset->update({enable_dynamic_run_command => 1}); + +is($build->job, "runCommandHook.example", "The only job should be runCommandHook.example"); +is($build->finished, 1, "Build should be finished."); +is($build->buildstatus, 0, "Build should have buildstatus 0."); + +subtest "fanoutToCommands" => sub { + my $config = { + runcommand => [ + { + job => "", + command => "foo" + }, + { + job => "*:*:*", + command => "bar" + }, + { + job => "tests:basic:nomatch", + command => "baz" + } + ] + }; + + is( + Hydra::Plugin::RunCommand::fanoutToCommands( + $config, + "buildFinished", + $build + ), + [ + { + matcher => "", + command => "foo" + }, + { + matcher => "*:*:*", + command => "bar" + } + ], + "fanoutToCommands returns a command per matching job" + ); +}; + +subtest "fanoutToCommandsWithDynamicRunCommandSupport" => sub { + like( + $build->buildoutputs->find({name => "out"})->path, + qr/my-build-product$/, + "The way we find the out path is reasonable" + ); + + my $config = { + dynamicruncommand => { enable => 1 }, + runcommand => [ + { + job => "*:*:*", + command => "baz" + } + ] + }; + + is( + Hydra::Plugin::RunCommand::fanoutToCommands( + $config, + "buildFinished", + $build + ), + [ + { + matcher => "*:*:*", + command => "baz" + }, + { + matcher => "DynamicRunCommand(runCommandHook.example)", + command => $build->buildoutputs->find({name => "out"})->path + } + ], + "fanoutToCommands returns a command per matching job" + ); +}; + +subtest "isBuildEligibleForDynamicRunCommand" => sub { + subtest "Non-matches based on name alone ..." => sub { + my $build = $builds->{"foo-bar-baz"}; + is( + Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($build), + 0, + "The job name does not match" + ); + + $build->set_column("job", "runCommandHook"); + is( + Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($build), + 0, + "The job name does not match" + ); + + $build->set_column("job", "runCommandHook."); + is( + Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($build), + 0, + "The job name does not match" + ); + }; + + subtest "On outputs ..." => sub { + ok(!warns { + is( + Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($builds->{"runCommandHook.example"}), + 1, + "out is an executable file" + ); + }, "No warnings for an executable file."); + + ok(!warns { + is( + Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($builds->{"runCommandHook.symlink"}), + 1, + "out is a symlink to an executable file" + ); + }, "No warnings for a symlink to an executable file."); + + like(warning { + is( + Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($builds->{"runCommandHook.no-out"}), + 0, + "No output named out" + ); + }, qr/rejected: no output named 'out'/, "A relevant warning is provided for a missing output"); + + like(warning { + is( + Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($builds->{"runCommandHook.out-is-directory"}), + 0, + "out is a directory" + ); + }, qr/output is not a regular file or symlink/, "A relevant warning is provided for a directory output"); + + like(warning { + is( + Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($builds->{"runCommandHook.out-is-not-executable-file"}), + 0, + "out is a file which is not a regular file or symlink" + ); + }, qr/output is not executable/, "A relevant warning is provided if the file isn't executable"); + + like(warning { + is( + Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($builds->{"runCommandHook.symlink-non-executable"}), + 0, + "out is a symlink to a non-executable file" + ); + }, qr/output is not executable/, "A relevant warning is provided for symlinks to non-executables"); + + like(warning { + is( + Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($builds->{"runCommandHook.symlink-directory"}), + 0, + "out is a symlink to a directory" + ); + }, qr/output is not a regular file or symlink/, "A relevant warning is provided for symlinks to directories"); + }; + + subtest "On build status ..." => sub { + is( + Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($builds->{"runCommandHook.failed"}), + 0, + "Failed builds don't get run" + ); + }; + + subtest "With dynamic runcommand disabled ..." => sub { + subtest "disabled on the project, enabled on the jobset" => sub { + $build->project->update({enable_dynamic_run_command => 0}); + $build->jobset->update({enable_dynamic_run_command => 1}); + + + like(warning { + is( + Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($builds->{"runCommandHook.example"}), + 0, + "Builds don't run from a jobset with disabled dynamic runcommand" + ); + }, qr/project or jobset don't have dynamic runcommand enabled./, "A relevant warning is provided for a disabled runcommand support") + }; + + subtest "enabled on the project, disabled on the jobset" => sub { + $build->project->update({enable_dynamic_run_command => 1}); + $build->jobset->update({enable_dynamic_run_command => 0}); + + like(warning { + is( + Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($builds->{"runCommandHook.example"}), + 0, + "Builds don't run from a jobset with disabled dynamic runcommand" + ); + }, qr/project or jobset don't have dynamic runcommand enabled./, "A relevant warning is provided for a disabled runcommand support") + }; + + subtest "disabled on the project, disabled on the jobset" => sub { + $build->project->update({enable_dynamic_run_command => 0}); + $build->jobset->update({enable_dynamic_run_command => 0}); + + like(warning { + is( + Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($builds->{"runCommandHook.example"}), + 0, + "Builds don't run from a jobset with disabled dynamic runcommand" + ); + }, qr/project or jobset don't have dynamic runcommand enabled./, "A relevant warning is provided for a disabled runcommand support") + }; + }; +}; + + +done_testing; diff --git a/t/Hydra/Plugin/RunCommand/matcher.t b/t/Hydra/Plugin/RunCommand/matcher.t index bc40ba77..ca74a84c 100644 --- a/t/Hydra/Plugin/RunCommand/matcher.t +++ b/t/Hydra/Plugin/RunCommand/matcher.t @@ -7,13 +7,13 @@ use Hydra::Plugin::RunCommand; subtest "isEnabled" => sub { is( Hydra::Plugin::RunCommand::isEnabled({}), - "", + 0, "Disabled by default." ); is( Hydra::Plugin::RunCommand::isEnabled({ config => {}}), - "", + 0, "Disabled by default." ); @@ -22,6 +22,121 @@ subtest "isEnabled" => sub { 1, "Enabled if any runcommand blocks exist." ); + + is( + Hydra::Plugin::RunCommand::isEnabled({ config => { dynamicruncommand => {}}}), + 0, + "Not enabled if an empty dynamicruncommand blocks exist." + ); + + is( + Hydra::Plugin::RunCommand::isEnabled({ config => { dynamicruncommand => { enable => 0 }}}), + 0, + "Not enabled if a dynamicruncommand blocks exist without enable being set to 1." + ); + + is( + Hydra::Plugin::RunCommand::isEnabled({ config => { dynamicruncommand => { enable => 1 }}}), + 1, + "Enabled if a dynamicruncommand blocks exist with enable being set to 1." + ); + + is( + Hydra::Plugin::RunCommand::isEnabled({ config => { + runcommand => {}, + dynamicruncommand => { enable => 0 } + }}), + 1, + "Enabled if a runcommand config block exists, even if a dynamicruncommand is explicitly disabled." + ); +}; + +subtest "areStaticCommandsEnabled" => sub { + is( + Hydra::Plugin::RunCommand::areStaticCommandsEnabled({}), + 0, + "Disabled by default." + ); + + is( + Hydra::Plugin::RunCommand::areStaticCommandsEnabled({}), + 0, + "Disabled by default." + ); + + is( + Hydra::Plugin::RunCommand::areStaticCommandsEnabled({ runcommand => {}}), + 1, + "Enabled if any runcommand blocks exist." + ); + + is( + Hydra::Plugin::RunCommand::areStaticCommandsEnabled({ dynamicruncommand => {}}), + 0, + "Not enabled by dynamicruncommand blocks." + ); + + is( + Hydra::Plugin::RunCommand::areStaticCommandsEnabled({ dynamicruncommand => { enable => 0 }}), + 0, + "Not enabled by dynamicruncommand blocks." + ); + + is( + Hydra::Plugin::RunCommand::areStaticCommandsEnabled({ dynamicruncommand => { enable => 1 }}), + 0, + "Not enabled by dynamicruncommand blocks." + ); + + is( + Hydra::Plugin::RunCommand::areStaticCommandsEnabled({ + runcommand => {}, + dynamicruncommand => { enable => 0 } + }), + 1, + "Enabled if a runcommand config block exists, even if a dynamicruncommand is explicitly disabled." + ); +}; + +subtest "areDynamicCommandsEnabled" => sub { + is( + Hydra::Plugin::RunCommand::areDynamicCommandsEnabled({}), + 0, + "Disabled by default." + ); + + is( + Hydra::Plugin::RunCommand::areDynamicCommandsEnabled({ runcommand => {}}), + 0, + "Disabled even if any runcommand blocks exist." + ); + + is( + Hydra::Plugin::RunCommand::areDynamicCommandsEnabled({ dynamicruncommand => {}}), + 0, + "Not enabled if an empty dynamicruncommand blocks exist." + ); + + is( + Hydra::Plugin::RunCommand::areDynamicCommandsEnabled({ dynamicruncommand => { enable => 0 }}), + 0, + "Not enabled if a dynamicruncommand blocks exist without enable being set to 1." + ); + + is( + Hydra::Plugin::RunCommand::areDynamicCommandsEnabled({ dynamicruncommand => { enable => 1 }}), + 1, + "Enabled if a dynamicruncommand blocks exist with enable being set to 1." + ); + + is( + Hydra::Plugin::RunCommand::areDynamicCommandsEnabled({ + runcommand => {}, + dynamicruncommand => { enable => 0 } + }), + 0, + "Disabled if dynamicruncommand is explicitly disabled." + ); }; subtest "configSectionMatches" => sub { @@ -134,44 +249,4 @@ subtest "eventMatches" => sub { ); }; -subtest "fanoutToCommands" => sub { - my $config = { - runcommand => [ - { - job => "", - command => "foo" - }, - { - job => "project:*:*", - command => "bar" - }, - { - job => "project:jobset:nomatch", - command => "baz" - } - ] - }; - - is( - Hydra::Plugin::RunCommand::fanoutToCommands( - $config, - "buildFinished", - "project", - "jobset", - "job" - ), - [ - { - matcher => "", - command => "foo" - }, - { - matcher => "project:*:*", - command => "bar" - } - ], - "fanoutToCommands returns a command per matching job" - ); -}; - done_testing; diff --git a/t/evaluator/evaluate-oom-job.t b/t/evaluator/evaluate-oom-job.t new file mode 100644 index 00000000..6c17d4e4 --- /dev/null +++ b/t/evaluator/evaluate-oom-job.t @@ -0,0 +1,63 @@ +use strict; +use warnings; +use Setup; +use Test2::V0; +use Hydra::Helper::Exec; + +# Ensure that `systemd-run` is +# - Available in the PATH/envionment +# - Accessable to the user executing it +# - Capable of using the command switches we use in our test +my $sd_res; +eval { + ($sd_res) = captureStdoutStderr(3, ( + "systemd-run", + "--user", + "--collect", + "--scope", + "--property", + "MemoryMax=25M", + "--", + "true" + )); +} or do { + # The command failed to execute, likely because `systemd-run` is not present + # in `PATH` + skip_all("`systemd-run` failed when invoked in this environment"); +}; +if ($sd_res != 0) { + # `systemd-run` executed but `sytemd-run` failed to call `true` and return + # successfully + skip_all("`systemd-run` returned non-zero when executing `true` (expected 0)"); +} + +my $ctx = test_context(); + +# Contain the memory usage to 25 MegaBytes using `systemd-run` +# Run `hydra-eval-jobs` on test job that will purposefully consume all memory +# available +my ($res, $stdout, $stderr) = captureStdoutStderr(60, ( + "systemd-run", + "--user", + "--collect", + "--scope", + "--property", + "MemoryMax=25M", + "--", + "hydra-eval-jobs", + "-I", "/dev/zero", + "-I", $ctx->jobsdir, + ($ctx->jobsdir . "/oom.nix") +)); + +isnt($res, 0, "`hydra-eval-jobs` exits non-zero"); +ok(utf8::decode($stderr), "Stderr output is UTF8-clean"); +like( + $stderr, + # Assert error log contains messages added in PR + # https://github.com/NixOS/hydra/pull/1203 + qr/^child process \(\d+?\) killed by signal=9$/m, + "The stderr record includes a relevant error message" +); + +done_testing; diff --git a/t/jobs/oom.nix b/t/jobs/oom.nix new file mode 100644 index 00000000..abbd0c0d --- /dev/null +++ b/t/jobs/oom.nix @@ -0,0 +1,3 @@ +{ + oom = builtins.readFile "/dev/zero"; +} diff --git a/t/jobs/runcommand-dynamic.nix b/t/jobs/runcommand-dynamic.nix new file mode 100644 index 00000000..1971bb82 --- /dev/null +++ b/t/jobs/runcommand-dynamic.nix @@ -0,0 +1,148 @@ +with import ./config.nix; +rec { + foo-bar-baz = mkDerivation { + name = "foo-bar-baz"; + builder = "/bin/sh"; + outputs = [ "out" ]; + args = [ + ( + builtins.toFile "builder.sh" '' + #! /bin/sh + + touch $out + '' + ) + ]; + }; + + runCommandHook.example = mkDerivation { + name = "my-build-product"; + builder = "/bin/sh"; + outputs = [ "out" ]; + args = [ + ( + builtins.toFile "builder.sh" '' + #! /bin/sh + + touch $out + chmod +x $out + # ... dunno ... + '' + ) + ]; + }; + + runCommandHook.symlink = mkDerivation { + name = "symlink-out"; + builder = "/bin/sh"; + outputs = [ "out" ]; + args = [ + ( + builtins.toFile "builder.sh" '' + #! /bin/sh + + ln -s $1 $out + '' + ) + + runCommandHook.example + ]; + }; + + runCommandHook.no-out = mkDerivation { + name = "no-out"; + builder = "/bin/sh"; + outputs = [ "bin" ]; + args = [ + ( + builtins.toFile "builder.sh" '' + #! /bin/sh + mkdir $bin + '' + ) + ]; + }; + + runCommandHook.out-is-directory = mkDerivation { + name = "out-is-directory"; + builder = "/bin/sh"; + outputs = [ "out" ]; + args = [ + ( + builtins.toFile "builder.sh" '' + #! /bin/sh + + mkdir $out + '' + ) + ]; + }; + + runCommandHook.out-is-not-executable-file = mkDerivation { + name = "out-is-directory"; + builder = "/bin/sh"; + outputs = [ "out" ]; + args = [ + ( + builtins.toFile "builder.sh" '' + #! /bin/sh + + touch $out + '' + ) + ]; + }; + + runCommandHook.symlink-non-executable = mkDerivation { + name = "symlink-out"; + builder = "/bin/sh"; + outputs = [ "out" ]; + args = [ + ( + builtins.toFile "builder.sh" '' + #! /bin/sh + + ln -s $1 $out + '' + ) + + runCommandHook.out-is-not-executable-file + ]; + }; + + runCommandHook.symlink-directory = mkDerivation { + name = "symlink-directory"; + builder = "/bin/sh"; + outputs = [ "out" ]; + args = [ + ( + builtins.toFile "builder.sh" '' + #! /bin/sh + + ln -s $1 $out + '' + ) + + runCommandHook.out-is-directory + ]; + }; + + runCommandHook.failed = mkDerivation { + name = "failed"; + builder = "/bin/sh"; + outputs = [ "out" ]; + args = [ + ( + builtins.toFile "builder.sh" '' + #! /bin/sh + + touch $out + chmod +x $out + + exit 1 + '' + ) + ]; + }; + +} diff --git a/t/lib/HydraTestContext.pm b/t/lib/HydraTestContext.pm index badb3728..53eaa0f7 100644 --- a/t/lib/HydraTestContext.pm +++ b/t/lib/HydraTestContext.pm @@ -53,16 +53,19 @@ sub new { my $hydra_config = $opts{'hydra_config'} || ""; $hydra_config = "queue_runner_metrics_address = 127.0.0.1:0\n" . $hydra_config; if ($opts{'use_external_destination_store'} // 1) { - $hydra_config = "store_uri = file:$dir/nix/dest-store\n" . $hydra_config; + $hydra_config = "store_uri = file://$dir/nix/dest-store\n" . $hydra_config; } write_file($ENV{'HYDRA_CONFIG'}, $hydra_config); - $ENV{'NIX_LOG_DIR'} = "$dir/nix/var/log/nix"; + my $nix_store_dir = "$dir/nix/store"; + my $nix_state_dir = "$dir/nix/var/nix"; + my $nix_log_dir = "$dir/nix/var/log/nix"; + $ENV{'NIX_REMOTE_SYSTEMS'} = ''; - $ENV{'NIX_REMOTE'} = ''; - $ENV{'NIX_STATE_DIR'} = "$dir/nix/var/nix"; - $ENV{'NIX_STORE_DIR'} = "$dir/nix/store"; + $ENV{'NIX_REMOTE'} = "local?store=$nix_store_dir&state=$nix_state_dir&log=$nix_log_dir"; + $ENV{'NIX_STATE_DIR'} = $nix_state_dir; # FIXME: remove + $ENV{'NIX_STORE_DIR'} = $nix_store_dir; # FIXME: remove my $pgsql = Test::PostgreSQL->new( extra_initdb_args => "--locale C.UTF-8" @@ -73,7 +76,8 @@ sub new { _db => undef, db_handle => $pgsql, tmpdir => $dir, - nix_state_dir => "$dir/nix/var/nix", + nix_state_dir => $nix_state_dir, + nix_log_dir => $nix_log_dir, testdir => abs_path(dirname(__FILE__) . "/.."), jobsdir => abs_path(dirname(__FILE__) . "/../jobs") }, $class; diff --git a/t/queue-runner/notifications.t b/t/queue-runner/notifications.t index b35b2b2f..1966cde1 100644 --- a/t/queue-runner/notifications.t +++ b/t/queue-runner/notifications.t @@ -33,9 +33,6 @@ my $ctx = test_context( # the build locally. subtest "Pre-build the job, upload to the cache, and then delete locally" => sub { - my $scratchlogdir = File::Temp->newdir(); - $ENV{'NIX_LOG_DIR'} = "$scratchlogdir"; - my $outlink = $ctx->tmpdir . "/basic-canbesubstituted"; is(system('nix-build', $ctx->jobsdir . '/notifications.nix', '-A', 'canbesubstituted', '--out-link', $outlink), 0, "Building notifications.nix succeeded"); is(system('nix', 'copy', '--to', "file://${binarycachedir}", $outlink), 0, "Copying the closure to the binary cache succeeded"); @@ -46,6 +43,7 @@ subtest "Pre-build the job, upload to the cache, and then delete locally" => sub is(system('nix', 'log', $outpath), 0, "Reading the output's log succeeds"); is(system('nix-store', '--delete', $outpath), 0, "Deleting the notifications.nix output succeeded"); is(system("nix-collect-garbage"), 0, "Delete all the system's garbage"); + File::Path::rmtree($ctx->{nix_log_dir}); }; subtest "Ensure substituting the job works, but reading the log fails" => sub {