Merge branch 'nix-2.20' into nix-2.21

queue-runner: don't decode base64 hostkey in hydra
Nix expects a base64 encoded hostkey in SSHMaster, so make sure we don't decode this prematurely in hydra. Reported-By: Puck Meerburg <puck@puck.moe>
2024-10-21 11:23:20 -04:00 · 2024-10-21 11:22:44 -04:00 · 2024-09-20 13:24:47 +02:00 · 2024-09-20 08:04:23 -03:00 · 2024-09-20 04:08:22 -03:00 · 2024-05-21 18:23:35 -04:00
20 changed files with 359 additions and 472 deletions
--- a/README.md
+++ b/README.md
@@ -140,7 +140,7 @@ You can also interface with Hydra through a JSON API. The API is defined in [hyd
 ## Additional Resources

 - [Hydra User's Guide](https://nixos.org/hydra/manual/)
- [Hydra on the NixOS Wiki](https://wiki.nixos.org/wiki/Hydra)
+- [Hydra on the NixOS Wiki](https://nixos.wiki/wiki/Hydra)
 - [hydra-cli](https://github.com/nlewo/hydra-cli)
 - [Peter Simons - Hydra: Setting up your own build farm (NixOS)](https://www.youtube.com/watch?v=RXV0Y5Bn-QQ)

--- a/doc/manual/src/hacking.md
+++ b/doc/manual/src/hacking.md
@@ -30,8 +30,6 @@ foreman:
 $ foreman start
 ```

-The Hydra interface will be available on port 63333, with an admin user named "alice" with password "foobar"
-
 You can run just the Hydra web server in your source tree as follows:

 ```console
--- a/doc/manual/src/webhooks.md
+++ b/doc/manual/src/webhooks.md
@@ -1,12 +1,9 @@
 # Webhooks

-Hydra can be notified by github or gitea with webhooks to trigger a new evaluation when a
+Hydra can be notified by github's webhook to trigger a new evaluation when a
 jobset has a github repo in its input.
-
-## GitHub
-
-To set up a webhook for a GitHub repository go to `https://github.com/<yourhandle>/<yourrepo>/settings`
-and in the `Webhooks` tab click on `Add webhook`.
+To set up a github webhook go to `https://github.com/<yourhandle>/<yourrepo>/settings` and in the `Webhooks` tab
+click on `Add webhook`.

 - In `Payload URL` fill in `https://<your-hydra-domain>/api/push-github`.
 - In `Content type` switch to `application/json`.
@@ -14,14 +11,3 @@ and in the `Webhooks` tab click on `Add webhook`.
 - For `Which events would you like to trigger this webhook?` keep the default option for events on `Just the push event.`.

 Then add the hook with `Add webhook`.
-
-## Gitea
-
-To set up a webhook for a Gitea repository go to the settings of the repository in your Gitea instance
-and in the `Webhooks` tab click on `Add Webhook` and choose `Gitea` in the drop down.
-
- In `Target URL` fill in `https://<your-hydra-domain>/api/push-gitea`.
- Keep HTTP method `POST`, POST Content Type `application/json` and Trigger On `Push Events`.
- Change the branch filter to match the git branch hydra builds.
-
-Then add the hook with `Add webhook`.
--- a/flake.lock
+++ b/flake.lock
@@ -16,42 +16,6 @@
        "type": "github"
      }
    },
-    "flake-parts": {
-      "inputs": {
-        "nixpkgs-lib": [
-          "nix",
-          "nixpkgs"
-        ]
-      },
-      "locked": {
-        "lastModified": 1712014858,
-        "narHash": "sha256-sB4SWl2lX95bExY2gMFG5HIzvva5AVMJd4Igm+GpZNw=",
-        "owner": "hercules-ci",
-        "repo": "flake-parts",
-        "rev": "9126214d0a59633752a136528f5f3b9aa8565b7d",
-        "type": "github"
-      },
-      "original": {
-        "owner": "hercules-ci",
-        "repo": "flake-parts",
-        "type": "github"
-      }
-    },
-    "flake-utils": {
-      "locked": {
-        "lastModified": 1667395993,
-        "narHash": "sha256-nuEHfE/LcWyuSWnS8t12N1wc105Qtau+/OdUAjtQ0rA=",
-        "owner": "numtide",
-        "repo": "flake-utils",
-        "rev": "5aed5285a952e0b949eb3ba02c12fa4fcfef535f",
-        "type": "github"
-      },
-      "original": {
-        "owner": "numtide",
-        "repo": "flake-utils",
-        "type": "github"
-      }
-    },
    "libgit2": {
      "flake": false,
      "locked": {
@@ -71,25 +35,23 @@
    "nix": {
      "inputs": {
        "flake-compat": "flake-compat",
-        "flake-parts": "flake-parts",
        "libgit2": "libgit2",
        "nixpkgs": [
          "nixpkgs"
        ],
-        "nixpkgs-regression": "nixpkgs-regression",
-        "pre-commit-hooks": "pre-commit-hooks"
+        "nixpkgs-regression": "nixpkgs-regression"
      },
      "locked": {
-        "lastModified": 1713874370,
-        "narHash": "sha256-gW1mO/CvsQQ5gvgiwzxsGhPFI/tx30NING+qgF5Do0s=",
+        "lastModified": 1715845907,
+        "narHash": "sha256-1OigUcZGDInTVZJBTioo9vwRt70yvcfAkSRUeAD/mfg=",
        "owner": "NixOS",
        "repo": "nix",
-        "rev": "1c8150ac312b5f9ba1b3f6768ff43b09867e5883",
+        "rev": "1ebc34e9c54b740ea4f4466443047d709dccf5b2",
        "type": "github"
      },
      "original": {
        "owner": "NixOS",
-        "ref": "2.22-maintenance",
+        "ref": "2.21-maintenance",
        "repo": "nix",
        "type": "github"
      }
@@ -126,38 +88,6 @@
        "type": "github"
      }
    },
-    "pre-commit-hooks": {
-      "inputs": {
-        "flake-compat": [
-          "nix"
-        ],
-        "flake-utils": "flake-utils",
-        "gitignore": [
-          "nix"
-        ],
-        "nixpkgs": [
-          "nix",
-          "nixpkgs"
-        ],
-        "nixpkgs-stable": [
-          "nix",
-          "nixpkgs"
-        ]
-      },
-      "locked": {
-        "lastModified": 1712897695,
-        "narHash": "sha256-nMirxrGteNAl9sWiOhoN5tIHyjBbVi5e2tgZUgZlK3Y=",
-        "owner": "cachix",
-        "repo": "pre-commit-hooks.nix",
-        "rev": "40e6053ecb65fcbf12863338a6dcefb3f55f1bf8",
-        "type": "github"
-      },
-      "original": {
-        "owner": "cachix",
-        "repo": "pre-commit-hooks.nix",
-        "type": "github"
-      }
-    },
    "root": {
      "inputs": {
        "nix": "nix",
--- a/flake.nix
+++ b/flake.nix
@@ -2,7 +2,7 @@
  description = "A Nix-based continuous build system";

  inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.11-small";
-  inputs.nix.url = "github:NixOS/nix/2.22-maintenance";
+  inputs.nix.url = "github:NixOS/nix/2.21-maintenance";
  inputs.nix.inputs.nixpkgs.follows = "nixpkgs";

  outputs = { self, nixpkgs, nix }:
@@ -17,11 +17,50 @@
        overlays = overlayList;
      });

+      # NixOS configuration used for VM tests.
+      hydraServer =
+        { config, pkgs, ... }:
+        {
+          imports = [ self.nixosModules.hydraTest ];
+
+          virtualisation.memorySize = 1024;
+          virtualisation.writableStore = true;
+
+          environment.systemPackages = [ pkgs.perlPackages.LWP pkgs.perlPackages.JSON ];
+
+          nix = {
+            # Without this nix tries to fetch packages from the default
+            # cache.nixos.org which is not reachable from this sandboxed NixOS test.
+            binaryCaches = [ ];
+          };
+        };
+
    in
    rec {

      # A Nixpkgs overlay that provides a 'hydra' package.
      overlays.default = final: prev: {
+
+        # Add LDAP dependencies that aren't currently found within nixpkgs.
+        perlPackages = prev.perlPackages // {
+
+          PrometheusTiny = final.perlPackages.buildPerlPackage {
+            pname = "Prometheus-Tiny";
+            version = "0.007";
+            src = final.fetchurl {
+              url = "mirror://cpan/authors/id/R/RO/ROBN/Prometheus-Tiny-0.007.tar.gz";
+              sha256 = "0ef8b226a2025cdde4df80129dd319aa29e884e653c17dc96f4823d985c028ec";
+            };
+            buildInputs = with final.perlPackages; [ HTTPMessage Plack TestException ];
+            meta = {
+              homepage = "https://github.com/robn/Prometheus-Tiny";
+              description = "A tiny Prometheus client";
+              license = with final.lib.licenses; [ artistic1 gpl1Plus ];
+            };
+          };
+
+        };
+
        hydra = final.callPackage ./package.nix {
          inherit (nixpkgs.lib) fileset;
          rawSrc = self;
@@ -49,9 +88,282 @@
              echo "doc manual $out/share/doc/hydra" >> $out/nix-support/hydra-build-products
            '');

-        tests = import ./nixos-tests.nix {
-          inherit forEachSystem nixpkgs pkgsBySystem nixosModules;
-        };
+        tests.install = forEachSystem (system:
+          with import (nixpkgs + "/nixos/lib/testing-python.nix") { inherit system; };
+          simpleTest {
+            name = "hydra-install";
+            nodes.machine = hydraServer;
+            testScript =
+              ''
+                machine.wait_for_job("hydra-init")
+                machine.wait_for_job("hydra-server")
+                machine.wait_for_job("hydra-evaluator")
+                machine.wait_for_job("hydra-queue-runner")
+                machine.wait_for_open_port(3000)
+                machine.succeed("curl --fail http://localhost:3000/")
+              '';
+          });
+
+        tests.notifications = forEachSystem (system:
+          let pkgs = pkgsBySystem.${system}; in
+          with import (nixpkgs + "/nixos/lib/testing-python.nix") { inherit system; };
+          simpleTest {
+            name = "hydra-notifications";
+            nodes.machine = { pkgs, ... }: {
+              imports = [ hydraServer ];
+              services.hydra-dev.extraConfig = ''
+                <influxdb>
+                  url = http://127.0.0.1:8086
+                  db = hydra
+                </influxdb>
+              '';
+              services.influxdb.enable = true;
+            };
+            testScript = ''
+              machine.wait_for_job("hydra-init")
+
+              # Create an admin account and some other state.
+              machine.succeed(
+                  """
+                      su - hydra -c "hydra-create-user root --email-address 'alice@example.org' --password foobar --role admin"
+                      mkdir /run/jobset
+                      chmod 755 /run/jobset
+                      cp ${./t/jobs/api-test.nix} /run/jobset/default.nix
+                      chmod 644 /run/jobset/default.nix
+                      chown -R hydra /run/jobset
+              """
+              )
+
+              # Wait until InfluxDB can receive web requests
+              machine.wait_for_job("influxdb")
+              machine.wait_for_open_port(8086)
+
+              # Create an InfluxDB database where hydra will write to
+              machine.succeed(
+                  "curl -XPOST 'http://127.0.0.1:8086/query' "
+                  + "--data-urlencode 'q=CREATE DATABASE hydra'"
+              )
+
+              # Wait until hydra-server can receive HTTP requests
+              machine.wait_for_job("hydra-server")
+              machine.wait_for_open_port(3000)
+
+              # Setup the project and jobset
+              machine.succeed(
+                  "su - hydra -c 'perl -I ${pkgs.hydra.perlDeps}/lib/perl5/site_perl ${./t/setup-notifications-jobset.pl}' >&2"
+              )
+
+              # Wait until hydra has build the job and
+              # the InfluxDBNotification plugin uploaded its notification to InfluxDB
+              machine.wait_until_succeeds(
+                  "curl -s -H 'Accept: application/csv' "
+                  + "-G 'http://127.0.0.1:8086/query?db=hydra' "
+                  + "--data-urlencode 'q=SELECT * FROM hydra_build_status' | grep success"
+              )
+            '';
+          });
+
+        tests.gitea = forEachSystem (system:
+          let pkgs = pkgsBySystem.${system}; in
+          with import (nixpkgs + "/nixos/lib/testing-python.nix") { inherit system; };
+          makeTest {
+            name = "hydra-gitea";
+            nodes.machine = { pkgs, ... }: {
+              imports = [ hydraServer ];
+              services.hydra-dev.extraConfig = ''
+                <gitea_authorization>
+                root=d7f16a3412e01a43a414535b16007c6931d3a9c7
+                </gitea_authorization>
+              '';
+              nixpkgs.config.permittedInsecurePackages = [ "gitea-1.19.4" ];
+              nix = {
+                settings.substituters = [ ];
+              };
+              services.gitea = {
+                enable = true;
+                database.type = "postgres";
+                disableRegistration = true;
+                httpPort = 3001;
+              };
+              services.openssh.enable = true;
+              environment.systemPackages = with pkgs; [ gitea git jq gawk ];
+              networking.firewall.allowedTCPPorts = [ 3000 ];
+            };
+            skipLint = true;
+            testScript =
+              let
+                scripts.mktoken = pkgs.writeText "token.sql" ''
+                  INSERT INTO access_token (id, uid, name, created_unix, updated_unix, token_hash, token_salt, token_last_eight, scope) VALUES (1, 1, 'hydra', 1617107360, 1617107360, 'a930f319ca362d7b49a4040ac0af74521c3a3c3303a86f327b01994430672d33b6ec53e4ea774253208686c712495e12a486', 'XRjWE9YW0g', '31d3a9c7', 'all');
+                '';
+
+                scripts.git-setup = pkgs.writeShellScript "setup.sh" ''
+                  set -x
+                  mkdir -p /tmp/repo $HOME/.ssh
+                  cat ${snakeoilKeypair.privkey} > $HOME/.ssh/privk
+                  chmod 0400 $HOME/.ssh/privk
+                  git -C /tmp/repo init
+                  cp ${smallDrv} /tmp/repo/jobset.nix
+                  git -C /tmp/repo add .
+                  git config --global user.email test@localhost
+                  git config --global user.name test
+                  git -C /tmp/repo commit -m 'Initial import'
+                  git -C /tmp/repo remote add origin gitea@machine:root/repo
+                  GIT_SSH_COMMAND='ssh -i $HOME/.ssh/privk -o StrictHostKeyChecking=no' \
+                    git -C /tmp/repo push origin master
+                  git -C /tmp/repo log >&2
+                '';
+
+                scripts.hydra-setup = pkgs.writeShellScript "hydra.sh" ''
+                  set -x
+                  su -l hydra -c "hydra-create-user root --email-address \
+                    'alice@example.org' --password foobar --role admin"
+
+                  URL=http://localhost:3000
+                  USERNAME="root"
+                  PASSWORD="foobar"
+                  PROJECT_NAME="trivial"
+                  JOBSET_NAME="trivial"
+                  mycurl() {
+                    curl --referer $URL -H "Accept: application/json" \
+                      -H "Content-Type: application/json" $@
+                  }
+
+                  cat >data.json <<EOF
+                  { "username": "$USERNAME", "password": "$PASSWORD" }
+                  EOF
+                  mycurl -X POST -d '@data.json' $URL/login -c hydra-cookie.txt
+
+                  cat >data.json <<EOF
+                  {
+                    "displayname":"Trivial",
+                    "enabled":"1",
+                    "visible":"1"
+                  }
+                  EOF
+                  mycurl --silent -X PUT $URL/project/$PROJECT_NAME \
+                    -d @data.json -b hydra-cookie.txt
+
+                  cat >data.json <<EOF
+                  {
+                    "description": "Trivial",
+                    "checkinterval": "60",
+                    "enabled": "1",
+                    "visible": "1",
+                    "keepnr": "1",
+                    "enableemail": true,
+                    "emailoverride": "hydra@localhost",
+                    "type": 0,
+                    "nixexprinput": "git",
+                    "nixexprpath": "jobset.nix",
+                    "inputs": {
+                      "git": {"value": "http://localhost:3001/root/repo.git", "type": "git"},
+                      "gitea_repo_name": {"value": "repo", "type": "string"},
+                      "gitea_repo_owner": {"value": "root", "type": "string"},
+                      "gitea_status_repo": {"value": "git", "type": "string"},
+                      "gitea_http_url": {"value": "http://localhost:3001", "type": "string"}
+                    }
+                  }
+                  EOF
+
+                  mycurl --silent -X PUT $URL/jobset/$PROJECT_NAME/$JOBSET_NAME \
+                    -d @data.json -b hydra-cookie.txt
+                '';
+
+                api_token = "d7f16a3412e01a43a414535b16007c6931d3a9c7";
+
+                snakeoilKeypair = {
+                  privkey = pkgs.writeText "privkey.snakeoil" ''
+                    -----BEGIN EC PRIVATE KEY-----
+                    MHcCAQEEIHQf/khLvYrQ8IOika5yqtWvI0oquHlpRLTZiJy5dRJmoAoGCCqGSM49
+                    AwEHoUQDQgAEKF0DYGbBwbj06tA3fd/+yP44cvmwmHBWXZCKbS+RQlAKvLXMWkpN
+                    r1lwMyJZoSGgBHoUahoYjTh9/sJL7XLJtA==
+                    -----END EC PRIVATE KEY-----
+                  '';
+
+                  pubkey = pkgs.lib.concatStrings [
+                    "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHA"
+                    "yNTYAAABBBChdA2BmwcG49OrQN33f/sj+OHL5sJhwVl2Qim0vkUJQCry1zFpKTa"
+                    "9ZcDMiWaEhoAR6FGoaGI04ff7CS+1yybQ= sakeoil"
+                  ];
+                };
+
+                smallDrv = pkgs.writeText "jobset.nix" ''
+                  { trivial = builtins.derivation {
+                      name = "trivial";
+                      system = "${system}";
+                      builder = "/bin/sh";
+                      allowSubstitutes = false;
+                      preferLocalBuild = true;
+                      args = ["-c" "echo success > $out; exit 0"];
+                    };
+                   }
+                '';
+              in
+              ''
+                import json
+
+                machine.start()
+                machine.wait_for_unit("multi-user.target")
+                machine.wait_for_open_port(3000)
+                machine.wait_for_open_port(3001)
+
+                machine.succeed(
+                    "su -l gitea -c 'GITEA_WORK_DIR=/var/lib/gitea gitea admin user create "
+                    + "--username root --password root --email test@localhost'"
+                )
+                machine.succeed("su -l postgres -c 'psql gitea < ${scripts.mktoken}'")
+
+                machine.succeed(
+                    "curl --fail -X POST http://localhost:3001/api/v1/user/repos "
+                    + "-H 'Accept: application/json' -H 'Content-Type: application/json' "
+                    + f"-H 'Authorization: token ${api_token}'"
+                    + ' -d \'{"auto_init":false, "description":"string", "license":"mit", "name":"repo", "private":false}\'''
+                )
+
+                machine.succeed(
+                    "curl --fail -X POST http://localhost:3001/api/v1/user/keys "
+                    + "-H 'Accept: application/json' -H 'Content-Type: application/json' "
+                    + f"-H 'Authorization: token ${api_token}'"
+                    + ' -d \'{"key":"${snakeoilKeypair.pubkey}","read_only":true,"title":"SSH"}\'''
+                )
+
+                machine.succeed(
+                    "${scripts.git-setup}"
+                )
+
+                machine.succeed(
+                    "${scripts.hydra-setup}"
+                )
+
+                machine.wait_until_succeeds(
+                    'curl -Lf -s http://localhost:3000/build/1 -H "Accept: application/json" '
+                    + '|  jq .buildstatus | xargs test 0 -eq'
+                )
+
+                data = machine.succeed(
+                    'curl -Lf -s "http://localhost:3001/api/v1/repos/root/repo/statuses/$(cd /tmp/repo && git show | head -n1 | awk "{print \\$2}")" '
+                    + "-H 'Accept: application/json' -H 'Content-Type: application/json' "
+                    + f"-H 'Authorization: token ${api_token}'"
+                )
+
+                response = json.loads(data)
+
+                assert len(response) == 2, "Expected exactly three status updates for latest commit (queued, finished)!"
+                assert response[0]['status'] == "success", "Expected finished status to be success!"
+                assert response[1]['status'] == "pending", "Expected queued status to be pending!"
+
+                machine.shutdown()
+              '';
+          });
+
+        tests.validate-openapi = forEachSystem (system:
+          let pkgs = pkgsBySystem.${system}; in
+          pkgs.runCommand "validate-openapi"
+          { buildInputs = [ pkgs.openapi-generator-cli ]; }
+          ''
+            openapi-generator-cli validate -i ${./hydra-api.yaml}
+            touch $out
+          '');

        container = nixosConfigurations.container.config.system.build.toplevel;
      };
@@ -75,8 +387,6 @@
        system = "x86_64-linux";
        modules =
          [
-            self.nixosModules.hydra
-            self.nixosModules.overlayNixpkgsForThisHydra
            self.nixosModules.hydraTest
            self.nixosModules.hydraProxy
            {
--- a/nixos-modules/default.nix
+++ b/nixos-modules/default.nix
@@ -1,14 +1,14 @@
 { overlays }:

-{
-  hydra = import ./hydra.nix;
-
-  overlayNixpkgsForThisHydra = { pkgs, ... }: {
+rec {
+  hydra = {
+    imports = [ ./hydra.nix ];
    nixpkgs = { inherit overlays; };
-    services.hydra.package = pkgs.hydra;
  };

  hydraTest = { pkgs, ... }: {
+    imports = [ hydra ];
+
    services.hydra-dev.enable = true;
    services.hydra-dev.hydraURL = "http://hydra.example.org";
    services.hydra-dev.notificationSender = "admin@hydra.example.org";
@@ -16,7 +16,7 @@
    systemd.services.hydra-send-stats.enable = false;

    services.postgresql.enable = true;
-    services.postgresql.package = pkgs.postgresql_12;
+    services.postgresql.package = pkgs.postgresql_11;

    # The following is to work around the following error from hydra-server:
    #   [error] Caught exception in engine "Cannot determine local time zone"
--- a/nixos-modules/hydra.nix
+++ b/nixos-modules/hydra.nix
@@ -68,7 +68,7 @@ in

      package = mkOption {
        type = types.path;
-        default = pkgs.hydra_unstable;
+        default = pkgs.hydra;
        defaultText = literalExpression "pkgs.hydra";
        description = "The Hydra package.";
      };
@@ -233,7 +233,7 @@ in
      gc-keep-outputs = true;
      gc-keep-derivations = true;
    };
-
+    
    services.hydra-dev.extraConfig =
      ''
        using_frontend_proxy = 1
--- a/nixos-tests.nix
+++ b/nixos-tests.nix
@@ -1,309 +0,0 @@
-{ forEachSystem, nixpkgs, pkgsBySystem, nixosModules }:
-
-let
-  # NixOS configuration used for VM tests.
-  hydraServer =
-    { config, pkgs, ... }:
-    {
-      imports = [
-        nixosModules.hydra
-        nixosModules.overlayNixpkgsForThisHydra
-        nixosModules.hydraTest
-      ];
-
-      virtualisation.memorySize = 1024;
-      virtualisation.writableStore = true;
-
-      environment.systemPackages = [ pkgs.perlPackages.LWP pkgs.perlPackages.JSON ];
-
-      nix = {
-        # Without this nix tries to fetch packages from the default
-        # cache.nixos.org which is not reachable from this sandboxed NixOS test.
-        settings.substituters = [ ];
-      };
-    };
-
-in
-
-{
-
-  install = forEachSystem (system:
-    with import (nixpkgs + "/nixos/lib/testing-python.nix") { inherit system; };
-    simpleTest {
-      name = "hydra-install";
-      nodes.machine = hydraServer;
-      testScript =
-        ''
-          machine.wait_for_job("hydra-init")
-          machine.wait_for_job("hydra-server")
-          machine.wait_for_job("hydra-evaluator")
-          machine.wait_for_job("hydra-queue-runner")
-          machine.wait_for_open_port(3000)
-          machine.succeed("curl --fail http://localhost:3000/")
-        '';
-    });
-
-  notifications = forEachSystem (system:
-    let pkgs = pkgsBySystem.${system}; in
-    with import (nixpkgs + "/nixos/lib/testing-python.nix") { inherit system; };
-    simpleTest {
-      name = "hydra-notifications";
-      nodes.machine = { pkgs, ... }: {
-        imports = [ hydraServer ];
-        services.hydra-dev.extraConfig = ''
-          <influxdb>
-            url = http://127.0.0.1:8086
-            db = hydra
-          </influxdb>
-        '';
-        services.influxdb.enable = true;
-      };
-      testScript = ''
-        machine.wait_for_job("hydra-init")
-
-        # Create an admin account and some other state.
-        machine.succeed(
-            """
-                su - hydra -c "hydra-create-user root --email-address 'alice@example.org' --password foobar --role admin"
-                mkdir /run/jobset
-                chmod 755 /run/jobset
-                cp ${./t/jobs/api-test.nix} /run/jobset/default.nix
-                chmod 644 /run/jobset/default.nix
-                chown -R hydra /run/jobset
-        """
-        )
-
-        # Wait until InfluxDB can receive web requests
-        machine.wait_for_job("influxdb")
-        machine.wait_for_open_port(8086)
-
-        # Create an InfluxDB database where hydra will write to
-        machine.succeed(
-            "curl -XPOST 'http://127.0.0.1:8086/query' "
-            + "--data-urlencode 'q=CREATE DATABASE hydra'"
-        )
-
-        # Wait until hydra-server can receive HTTP requests
-        machine.wait_for_job("hydra-server")
-        machine.wait_for_open_port(3000)
-
-        # Setup the project and jobset
-        machine.succeed(
-            "su - hydra -c 'perl -I ${pkgs.hydra.perlDeps}/lib/perl5/site_perl ${./t/setup-notifications-jobset.pl}' >&2"
-        )
-
-        # Wait until hydra has build the job and
-        # the InfluxDBNotification plugin uploaded its notification to InfluxDB
-        machine.wait_until_succeeds(
-            "curl -s -H 'Accept: application/csv' "
-            + "-G 'http://127.0.0.1:8086/query?db=hydra' "
-            + "--data-urlencode 'q=SELECT * FROM hydra_build_status' | grep success"
-        )
-      '';
-    });
-
-  gitea = forEachSystem (system:
-    let pkgs = pkgsBySystem.${system}; in
-    with import (nixpkgs + "/nixos/lib/testing-python.nix") { inherit system; };
-    makeTest {
-      name = "hydra-gitea";
-      nodes.machine = { pkgs, ... }: {
-        imports = [ hydraServer ];
-        services.hydra-dev.extraConfig = ''
-          <gitea_authorization>
-          root=d7f16a3412e01a43a414535b16007c6931d3a9c7
-          </gitea_authorization>
-        '';
-        nixpkgs.config.permittedInsecurePackages = [ "gitea-1.19.4" ];
-        nix = {
-          settings.substituters = [ ];
-        };
-        services.gitea = {
-          enable = true;
-          database.type = "postgres";
-          settings = {
-            service.DISABLE_REGISTRATION = true;
-            server.HTTP_PORT = 3001;
-          };
-        };
-        services.openssh.enable = true;
-        environment.systemPackages = with pkgs; [ gitea git jq gawk ];
-        networking.firewall.allowedTCPPorts = [ 3000 ];
-      };
-      skipLint = true;
-      testScript =
-        let
-          scripts.mktoken = pkgs.writeText "token.sql" ''
-            INSERT INTO access_token (id, uid, name, created_unix, updated_unix, token_hash, token_salt, token_last_eight, scope) VALUES (1, 1, 'hydra', 1617107360, 1617107360, 'a930f319ca362d7b49a4040ac0af74521c3a3c3303a86f327b01994430672d33b6ec53e4ea774253208686c712495e12a486', 'XRjWE9YW0g', '31d3a9c7', 'all');
-          '';
-
-          scripts.git-setup = pkgs.writeShellScript "setup.sh" ''
-            set -x
-            mkdir -p /tmp/repo $HOME/.ssh
-            cat ${snakeoilKeypair.privkey} > $HOME/.ssh/privk
-            chmod 0400 $HOME/.ssh/privk
-            git -C /tmp/repo init
-            cp ${smallDrv} /tmp/repo/jobset.nix
-            git -C /tmp/repo add .
-            git config --global user.email test@localhost
-            git config --global user.name test
-            git -C /tmp/repo commit -m 'Initial import'
-            git -C /tmp/repo remote add origin gitea@machine:root/repo
-            GIT_SSH_COMMAND='ssh -i $HOME/.ssh/privk -o StrictHostKeyChecking=no' \
-              git -C /tmp/repo push origin master
-            git -C /tmp/repo log >&2
-          '';
-
-          scripts.hydra-setup = pkgs.writeShellScript "hydra.sh" ''
-            set -x
-            su -l hydra -c "hydra-create-user root --email-address \
-              'alice@example.org' --password foobar --role admin"
-
-            URL=http://localhost:3000
-            USERNAME="root"
-            PASSWORD="foobar"
-            PROJECT_NAME="trivial"
-            JOBSET_NAME="trivial"
-            mycurl() {
-              curl --referer $URL -H "Accept: application/json" \
-                -H "Content-Type: application/json" $@
-            }
-
-            cat >data.json <<EOF
-            { "username": "$USERNAME", "password": "$PASSWORD" }
-            EOF
-            mycurl -X POST -d '@data.json' $URL/login -c hydra-cookie.txt
-
-            cat >data.json <<EOF
-            {
-              "displayname":"Trivial",
-              "enabled":"1",
-              "visible":"1"
-            }
-            EOF
-            mycurl --silent -X PUT $URL/project/$PROJECT_NAME \
-              -d @data.json -b hydra-cookie.txt
-
-            cat >data.json <<EOF
-            {
-              "description": "Trivial",
-              "checkinterval": "60",
-              "enabled": "1",
-              "visible": "1",
-              "keepnr": "1",
-              "enableemail": true,
-              "emailoverride": "hydra@localhost",
-              "type": 0,
-              "nixexprinput": "git",
-              "nixexprpath": "jobset.nix",
-              "inputs": {
-                "git": {"value": "http://localhost:3001/root/repo.git", "type": "git"},
-                "gitea_repo_name": {"value": "repo", "type": "string"},
-                "gitea_repo_owner": {"value": "root", "type": "string"},
-                "gitea_status_repo": {"value": "git", "type": "string"},
-                "gitea_http_url": {"value": "http://localhost:3001", "type": "string"}
-              }
-            }
-            EOF
-
-            mycurl --silent -X PUT $URL/jobset/$PROJECT_NAME/$JOBSET_NAME \
-              -d @data.json -b hydra-cookie.txt
-          '';
-
-          api_token = "d7f16a3412e01a43a414535b16007c6931d3a9c7";
-
-          snakeoilKeypair = {
-            privkey = pkgs.writeText "privkey.snakeoil" ''
-              -----BEGIN EC PRIVATE KEY-----
-              MHcCAQEEIHQf/khLvYrQ8IOika5yqtWvI0oquHlpRLTZiJy5dRJmoAoGCCqGSM49
-              AwEHoUQDQgAEKF0DYGbBwbj06tA3fd/+yP44cvmwmHBWXZCKbS+RQlAKvLXMWkpN
-              r1lwMyJZoSGgBHoUahoYjTh9/sJL7XLJtA==
-              -----END EC PRIVATE KEY-----
-            '';
-
-            pubkey = pkgs.lib.concatStrings [
-              "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHA"
-              "yNTYAAABBBChdA2BmwcG49OrQN33f/sj+OHL5sJhwVl2Qim0vkUJQCry1zFpKTa"
-              "9ZcDMiWaEhoAR6FGoaGI04ff7CS+1yybQ= sakeoil"
-            ];
-          };
-
-          smallDrv = pkgs.writeText "jobset.nix" ''
-            { trivial = builtins.derivation {
-                name = "trivial";
-                system = "${system}";
-                builder = "/bin/sh";
-                allowSubstitutes = false;
-                preferLocalBuild = true;
-                args = ["-c" "echo success > $out; exit 0"];
-              };
-             }
-          '';
-        in
-        ''
-          import json
-
-          machine.start()
-          machine.wait_for_unit("multi-user.target")
-          machine.wait_for_open_port(3000)
-          machine.wait_for_open_port(3001)
-
-          machine.succeed(
-              "su -l gitea -c 'GITEA_WORK_DIR=/var/lib/gitea gitea admin user create "
-              + "--username root --password root --email test@localhost'"
-          )
-          machine.succeed("su -l postgres -c 'psql gitea < ${scripts.mktoken}'")
-
-          machine.succeed(
-              "curl --fail -X POST http://localhost:3001/api/v1/user/repos "
-              + "-H 'Accept: application/json' -H 'Content-Type: application/json' "
-              + f"-H 'Authorization: token ${api_token}'"
-              + ' -d \'{"auto_init":false, "description":"string", "license":"mit", "name":"repo", "private":false}\'''
-          )
-
-          machine.succeed(
-              "curl --fail -X POST http://localhost:3001/api/v1/user/keys "
-              + "-H 'Accept: application/json' -H 'Content-Type: application/json' "
-              + f"-H 'Authorization: token ${api_token}'"
-              + ' -d \'{"key":"${snakeoilKeypair.pubkey}","read_only":true,"title":"SSH"}\'''
-          )
-
-          machine.succeed(
-              "${scripts.git-setup}"
-          )
-
-          machine.succeed(
-              "${scripts.hydra-setup}"
-          )
-
-          machine.wait_until_succeeds(
-              'curl -Lf -s http://localhost:3000/build/1 -H "Accept: application/json" '
-              + '|  jq .buildstatus | xargs test 0 -eq'
-          )
-
-          data = machine.succeed(
-              'curl -Lf -s "http://localhost:3001/api/v1/repos/root/repo/statuses/$(cd /tmp/repo && git show | head -n1 | awk "{print \\$2}")" '
-              + "-H 'Accept: application/json' -H 'Content-Type: application/json' "
-              + f"-H 'Authorization: token ${api_token}'"
-          )
-
-          response = json.loads(data)
-
-          assert len(response) == 2, "Expected exactly three status updates for latest commit (queued, finished)!"
-          assert response[0]['status'] == "success", "Expected finished status to be success!"
-          assert response[1]['status'] == "pending", "Expected queued status to be pending!"
-
-          machine.shutdown()
-        '';
-    });
-
-  validate-openapi = forEachSystem (system:
-    let pkgs = pkgsBySystem.${system}; in
-    pkgs.runCommand "validate-openapi"
-    { buildInputs = [ pkgs.openapi-generator-cli ]; }
-    ''
-      openapi-generator-cli validate -i ${./hydra-api.yaml}
-      touch $out
-    '');
-
-}
--- a/src/hydra-eval-jobs/hydra-eval-jobs.cc
+++ b/src/hydra-eval-jobs/hydra-eval-jobs.cc
@@ -102,8 +102,8 @@ static std::string queryMetaStrings(EvalState & state, PackageInfo & drv, const
            for (unsigned int n = 0; n < v.listSize(); ++n)
                rec(*v.listElems()[n]);
        else if (v.type() == nAttrs) {
-            auto a = v.attrs()->find(state.symbols.create(subAttribute));
-            if (a != v.attrs()->end())
+            auto a = v.attrs->find(state.symbols.create(subAttribute));
+            if (a != v.attrs->end())
                res.push_back(std::string(state.forceString(*a->value, a->pos, "while evaluating meta attributes")));
        }
    };
@@ -138,12 +138,12 @@ static void worker(

        callFlake(state, lockedFlake, *vFlake);

-        auto vOutputs = vFlake->attrs()->get(state.symbols.create("outputs"))->value;
+        auto vOutputs = vFlake->attrs->get(state.symbols.create("outputs"))->value;
        state.forceValue(*vOutputs, noPos);

-        auto aHydraJobs = vOutputs->attrs()->get(state.symbols.create("hydraJobs"));
+        auto aHydraJobs = vOutputs->attrs->get(state.symbols.create("hydraJobs"));
        if (!aHydraJobs)
-            aHydraJobs = vOutputs->attrs()->get(state.symbols.create("checks"));
+            aHydraJobs = vOutputs->attrs->get(state.symbols.create("checks"));
        if (!aHydraJobs)
            throw Error("flake '%s' does not provide any Hydra jobs or checks", flakeRef);

@@ -204,9 +204,9 @@ static void worker(
                job["isChannel"] = drv->queryMetaBool("isHydraChannel", false);

                /* If this is an aggregate, then get its constituents. */
-                auto a = v->attrs()->get(state.symbols.create("_hydraAggregate"));
+                auto a = v->attrs->get(state.symbols.create("_hydraAggregate"));
                if (a && state.forceBool(*a->value, a->pos, "while evaluating the `_hydraAggregate` attribute")) {
-                    auto a = v->attrs()->get(state.symbols.create("constituents"));
+                    auto a = v->attrs->get(state.symbols.create("constituents"));
                    if (!a)
                        state.error<EvalError>("derivation must have a ‘constituents’ attribute").debugThrow();

@@ -260,7 +260,7 @@ static void worker(
            else if (v->type() == nAttrs) {
                auto attrs = nlohmann::json::array();
                StringSet ss;
-                for (auto & i : v->attrs()->lexicographicOrder(state.symbols)) {
+                for (auto & i : v->attrs->lexicographicOrder(state.symbols)) {
                    std::string name(state.symbols[i->name]);
                    if (name.find(' ') != std::string::npos) {
                        printError("skipping job with illegal name '%s'", name);
@@ -368,7 +368,7 @@ int main(int argc, char * * argv)
                            ]()
                            {
                                try {
-                                    EvalState state(myArgs.lookupPath, openStore());
+                                    EvalState state(myArgs.searchPath, openStore());
                                    Bindings & autoArgs = *myArgs.getAutoArgs(state);
                                    worker(state, autoArgs, *to, *from);
                                } catch (Error & e) {
--- a/src/hydra-queue-runner/build-remote.cc
+++ b/src/hydra-queue-runner/build-remote.cc
@@ -54,20 +54,9 @@ static std::unique_ptr<SSHMaster::Connection> openConnection(
        command.splice(command.end(), extraStoreArgs(machine->sshName));
    }

-    auto ret = master.startCommand(std::move(command), {
+    return master.startCommand(std::move(command), {
        "-a", "-oBatchMode=yes", "-oConnectTimeout=60", "-oTCPKeepAlive=yes"
    });
-
-    // XXX: determine the actual max value we can use from /proc.
-
-    // FIXME: Should this be upstreamed into `startCommand` in Nix?
-
-    int pipesize = 1024 * 1024;
-
-    fcntl(ret->in.get(), F_SETPIPE_SZ, &pipesize);
-    fcntl(ret->out.get(), F_SETPIPE_SZ, &pipesize);
-
-    return ret;
 }


--- a/src/hydra-queue-runner/hydra-queue-runner.cc
+++ b/src/hydra-queue-runner/hydra-queue-runner.cc
@@ -171,7 +171,7 @@ void State::parseMachines(const std::string & contents)
            std::move(mandatoryFeatures),
            // `sshPublicHostKey`
            tokens[7] != "" && tokens[7] != "-"
-                ? base64Decode(tokens[7])
+                ? tokens[7]
                : "",
        });

--- a/src/lib/Hydra/Controller/API.pm
+++ b/src/lib/Hydra/Controller/API.pm
@@ -285,23 +285,6 @@ sub push_github : Chained('api') PathPart('push-github') Args(0) {
    $c->response->body("");
 }

-sub push_gitea : Chained('api') PathPart('push-gitea') Args(0) {
-    my ($self, $c) = @_;
-
-    $c->{stash}->{json}->{jobsetsTriggered} = [];
-
-    my $in = $c->request->{data};
-    my $url = $in->{repository}->{clone_url} or die;
-    $url =~ s/.git$//;
-    print STDERR "got push from Gitea repository $url\n";
-
-    triggerJobset($self, $c, $_, 0) foreach $c->model('DB::Jobsets')->search(
-        { 'project.enabled' => 1, 'me.enabled' => 1 },
-        { join => 'project'
-        , where => \ [ 'me.flake like ? or exists (select 1 from JobsetInputAlts where project = me.project and jobset = me.name and value like ?)', [ 'flake', "%$url%"], [ 'value', "%$url%" ] ]
-        });
-    $c->response->body("");
-}


 1;
--- a/src/lib/Hydra/Controller/Root.pm
+++ b/src/lib/Hydra/Controller/Root.pm
@@ -35,7 +35,6 @@ sub noLoginNeeded {

  return $whitelisted ||
         $c->request->path eq "api/push-github" ||
-         $c->request->path eq "api/push-gitea" ||
         $c->request->path eq "google-login" ||
         $c->request->path eq "github-redirect" ||
         $c->request->path eq "github-login" ||
@@ -81,7 +80,7 @@ sub begin :Private {
    $_->supportedInputTypes($c->stash->{inputTypes}) foreach @{$c->hydra_plugins};

    # XSRF protection: require POST requests to have the same origin.
-    if ($c->req->method eq "POST" && $c->req->path ne "api/push-github" && $c->req->path ne "api/push-gitea") {
+    if ($c->req->method eq "POST" && $c->req->path ne "api/push-github") {
        my $referer = $c->req->header('Referer');
        $referer //= $c->req->header('Origin');
        my $base = $c->req->base;
@@ -330,7 +329,7 @@ sub nar :Local :Args(1) {
    else {
        $path = $Nix::Config::storeDir . "/$path";

-        gone($c, "Path " . $path . " is no longer available.") unless isValidPath($path);
+        gone($c, "Path " . $path . " is no longer available.") unless $MACHINE_LOCAL_STORE->isValidPath($path);

        $c->stash->{current_view} = 'NixNAR';
        $c->stash->{storePath} = $path;
@@ -368,7 +367,7 @@ sub realisations :Path('realisations') :Args(StrMatch[REALISATIONS_REGEX]) {

    else {
        my ($rawDrvOutput) = $realisation =~ REALISATIONS_REGEX;
-        my $rawRealisation = $MACHINE_LOCAL_STORE->queryRawRealisation($rawDrvOutput);
+        my $rawRealisation = queryRawRealisation($rawDrvOutput);

        if (!$rawRealisation) {
            $c->response->status(404);
--- a/src/lib/Hydra/Plugin/S3Backup.pm
+++ b/src/lib/Hydra/Plugin/S3Backup.pm
@@ -92,7 +92,7 @@ sub buildFinished {
        my $hash = substr basename($path), 0, 32;
        my ($deriver, $narHash, $time, $narSize, $refs) = queryPathInfo($path, 0);
        my $system;
-        if (defined $deriver and isValidPath($deriver)) {
+        if (defined $deriver and $MACHINE_LOCAL_STORE->isValidPath($deriver)) {
            $system = derivationFromPath($deriver)->{platform};
        }
        foreach my $reference (@{$refs}) {
--- a/src/lib/Hydra/Plugin/SubversionInput.pm
+++ b/src/lib/Hydra/Plugin/SubversionInput.pm
@@ -46,7 +46,7 @@ sub fetchInput {

    $MACHINE_LOCAL_STORE->addTempRoot($cachedInput->storepath) if defined $cachedInput;

-    if (defined $cachedInput && isValidPath($cachedInput->storepath)) {
+    if (defined $cachedInput && $MACHINE_LOCAL_STORE->isValidPath($cachedInput->storepath)) {
        $storePath = $cachedInput->storepath;
        $sha256 = $cachedInput->sha256hash;
    } else {
--- a/src/lib/Hydra/View/NARInfo.pm
+++ b/src/lib/Hydra/View/NARInfo.pm
@@ -6,6 +6,8 @@ use File::Basename;
 use Hydra::Helper::CatalystUtils;
 use MIME::Base64;
 use Nix::Manifest;
+use Nix::Store;
+use Nix::Utils;
 use Hydra::Helper::Nix;
 use base qw/Catalyst::View/;

--- a/t/Hydra/Controller/Job/builds.t
+++ b/t/Hydra/Controller/Job/builds.t
@@ -54,14 +54,13 @@ subtest "/job/PROJECT/JOBSET/JOB/shield" => sub {

 subtest "/job/PROJECT/JOBSET/JOB/prometheus" => sub {
    my $response = request(GET '/job/' . $project->name . '/' . $jobset->name . '/' . $build->job . '/prometheus');
-
    ok($response->is_success, "The page showing the job's prometheus data returns 200.");
-
    my $metrics = $response->content;
-    like($metrics, qr/hydra_job_failed\{.*\} 0/);
-    like($metrics, qr/hydra_job_completion_time\{.*\} [\d]+/);
-    like($metrics, qr/hydra_build_closure_size\{.*\} 96/);
-    like($metrics, qr/hydra_build_output_size\{.*\} 96/);
+
+    ok($metrics =~ m/hydra_job_failed\{.*\} 0/);
+    ok($metrics =~ m/hydra_job_completion_time\{.*\} [\d]+/);
+    ok($metrics =~ m/hydra_build_closure_size\{.*\} 96/);
+    ok($metrics =~ m/hydra_build_output_size\{.*\} 96/);
 };

 done_testing;
--- a/t/Hydra/Controller/Jobset/http.t
+++ b/t/Hydra/Controller/Jobset/http.t
@@ -186,7 +186,7 @@ subtest 'Update jobset "job" to have an invalid input type' => sub {
      })
  );
  ok(!$jobsetupdate->is_success);
-  like($jobsetupdate->content, qr/Invalid input type.*valid types:/);
+  ok($jobsetupdate->content =~ m/Invalid input type.*valid types:/);
 };


--- a/t/Hydra/Controller/User/dashboard.t
+++ b/t/Hydra/Controller/User/dashboard.t
@@ -24,7 +24,7 @@ my $cookie = $login->header("set-cookie");
 my $my_jobs = request(GET '/dashboard/alice/my-jobs-tab', Accept => 'application/json', Cookie => $cookie);
 ok($my_jobs->is_success);
 my $content = $my_jobs->content();
-like($content, qr/empty_dir/);
+ok($content =~ /empty_dir/);
 ok(!($content =~ /fails/));
 ok(!($content =~ /succeed_with_failed/));
 done_testing;
--- a/t/api-test.t
+++ b/t/api-test.t
@@ -115,7 +115,7 @@ subtest "evaluation" => sub {
        my $build = decode_json(request_json({ uri => "/build/" . $evals->[0]->{builds}->[0] })->content());
        is($build->{job}, "job", "The build's job name is job");
        is($build->{finished}, 0, "The build isn't finished yet");
-        like($build->{buildoutputs}->{out}->{path}, qr/\/nix\/store\/[a-zA-Z0-9]{32}-job$/, "The build's outpath is in the Nix store and named 'job'");
+        ok($build->{buildoutputs}->{out}->{path} =~ /\/nix\/store\/[a-zA-Z0-9]{32}-job$/, "The build's outpath is in the Nix store and named 'job'");

        subtest "search" => sub {
            my $search_project = decode_json(request_json({ uri => "/search/?query=sample" })->content());
Author	SHA1	Message	Date
John Ericson	53b04ddf74	Merge branch 'nix-2.20' into nix-2.21	2024-10-21 11:23:20 -04:00
Martin Weinelt	4e2c06ec2c	queue-runner: don't decode base64 hostkey in hydra Nix expects a base64 encoded hostkey in SSHMaster, so make sure we don't decode this prematurely in hydra. Reported-By: Puck Meerburg <puck@puck.moe>	2024-10-21 11:22:44 -04:00
Jörg Thalheim	73e51b94b1	Merge pull request #1414 from cleverca22/fix-binary-cache fix nar endpoint for binary cache hosting	2024-09-20 13:24:47 +02:00
hacker1024	1ef6b5e7b4	Use Nix::Store and Nix::Utils in NARInfo.pm These are required for the `signString` and `readFile` subroutines used when signing NARs. (cherry picked from commit `b94a7b6d5c`)	2024-09-20 08:04:23 -03:00
Michael Bishop	cc1b6d394e	fix the binary cache calling isValidPath, and 2 other files with the same issue	2024-09-20 04:08:22 -03:00
John Ericson	879ceb5cdc	flake.lock: Update Flake lock file updates: • Updated input 'nix': 'github:NixOS/nix/60824fa97c588a0faf68ea61260a47e388b0a4e5' (2024-04-11) → 'github:NixOS/nix/1ebc34e9c54b740ea4f4466443047d709dccf5b2' (2024-05-16)	2024-05-21 18:23:35 -04:00
John Ericson	898ca2f600	Merge branch 'nix-2.20' into nix-2.21	2024-05-21 18:23:04 -04:00
John Ericson	21044bc4d9	flake.lock: Update Flake lock file updates: • Updated input 'nix': 'github:NixOS/nix/8f42912c80c0a03f62f6a3d28a3af05a9762565d' (2024-01-30) → 'github:NixOS/nix/ab48ea416a203e9ccefb70aa634e27477e4c1ac4' (2024-05-15)	2024-05-21 18:01:30 -04:00