From dd25c7224b832cb21a1aa3e0261e11f926002219 Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Fri, 21 Feb 2025 22:36:48 +0100 Subject: [PATCH 1/2] subprojects/nix-eval-jobs: build tests in Hydra jobs This exposes the `pytest` suite as a Nix build. Previously, this didn't exist and upstream just runs this in GHA. As a result, the test was actually broken since the assumptions about the directory structure have changed. To not require network access, the nixpkgs used by the Lix flake is injected into the test. The evaluation will be done in a chroot store since the `openStore` call would otherwise need write access to /nix/store. Change-Id: Ib4ff5f0319ce3aabb3b7243a076ea169e27e3533 --- flake.nix | 2 + subprojects/nix-eval-jobs/default.nix | 118 +++++++++++------- subprojects/nix-eval-jobs/tests/assets/ci.nix | 2 +- .../nix-eval-jobs/tests/assets/flake.nix | 2 +- subprojects/nix-eval-jobs/tests/test_eval.py | 106 +++++++++------- 5 files changed, 138 insertions(+), 92 deletions(-) diff --git a/flake.nix b/flake.nix index f7f4f4ef0..5a89c13dd 100644 --- a/flake.nix +++ b/flake.nix @@ -294,6 +294,8 @@ # System tests. tests = import ./tests/nixos { inherit lib nixpkgs nixpkgsFor; } // { + nix-eval-jobs = forAllSystems (system: self.packages.${system}.nix-eval-jobs.tests.nix-eval-jobs); + # This is x86_64-linux only, just because we have significantly # cheaper x86_64-linux compute in CI. # It is clangStdenv because clang's sanitizers are nicer. diff --git a/subprojects/nix-eval-jobs/default.nix b/subprojects/nix-eval-jobs/default.nix index 5eaeb5e26..92dee1f9b 100644 --- a/subprojects/nix-eval-jobs/default.nix +++ b/subprojects/nix-eval-jobs/default.nix @@ -3,49 +3,83 @@ lib, nix, pkgs, - srcDir ? null, + srcDir ? ./., + callPackage, }: let - filterMesonBuild = builtins.filterSource ( - path: type: type != "directory" || baseNameOf path != "build" - ); -in -stdenv.mkDerivation { - pname = "nix-eval-jobs"; - version = "2.93.0-dev"; - src = if srcDir == null then filterMesonBuild ./. else srcDir; - buildInputs = with pkgs; [ - nlohmann_json - nix - boost - ]; - nativeBuildInputs = - with pkgs; - [ - meson - pkg-config - ninja - # nlohmann_json can be only discovered via cmake files - cmake - # XXX: ew - nix.passthru.capnproto-lix - ] - ++ (lib.optional stdenv.cc.isClang [ pkgs.clang-tools ]); - - # nix-fast-build wants the nix attr on nix-eval-jobs - passthru = { - inherit nix; - }; - - meta = { - description = "Hydra's builtin hydra-eval-jobs as a standalone"; - homepage = "https://github.com/nix-community/nix-eval-jobs"; - license = lib.licenses.gpl3; - maintainers = with lib.maintainers; [ - adisbladis - mic92 + package = stdenv.mkDerivation (finalAttrs: { + pname = "nix-eval-jobs"; + version = "2.93.0-dev"; + src = lib.fileset.toSource { + root = if srcDir == null then ./. else srcDir; + fileset = lib.fileset.unions [ + ./meson.build + ./src + ]; + }; + buildInputs = with pkgs; [ + nlohmann_json + nix + boost ]; - platforms = lib.platforms.unix; - }; -} + nativeBuildInputs = + with pkgs; + [ + meson + pkg-config + ninja + # nlohmann_json can be only discovered via cmake files + cmake + # XXX: ew + nix.passthru.capnproto-lix + ] + ++ (lib.optional stdenv.cc.isClang [ pkgs.clang-tools ]); + + passthru = { + inherit nix; + tests.nix-eval-jobs = callPackage ( + { + stdenv, + python3Packages, + path, + }: + let + nejAttrs = finalAttrs; + in + stdenv.mkDerivation (finalAttrs: { + pname = "nix-eval-jobs-tests"; + inherit (nejAttrs) version; + src = ./tests; + + nativeBuildInputs = [ + python3Packages.pytest + package + ]; + + env.NEJ_NIXPKGS_PATH = "${path}"; + + dontConfigure = true; + buildPhase = '' + export NIX_REMOTE="$NIX_BUILD_TOP" + export HOME="$(mktemp -d)" + pytest + ''; + installPhase = "touch $out"; + }) + ) { }; + }; + + meta = { + description = "Hydra's builtin hydra-eval-jobs as a standalone"; + homepage = "https://github.com/nix-community/nix-eval-jobs"; + license = lib.licenses.gpl3; + maintainers = with lib.maintainers; [ + adisbladis + mic92 + ]; + platforms = lib.platforms.unix; + }; + }); +in +package diff --git a/subprojects/nix-eval-jobs/tests/assets/ci.nix b/subprojects/nix-eval-jobs/tests/assets/ci.nix index e16369219..55bed43ca 100644 --- a/subprojects/nix-eval-jobs/tests/assets/ci.nix +++ b/subprojects/nix-eval-jobs/tests/assets/ci.nix @@ -1,6 +1,6 @@ { pkgs ? import (builtins.getFlake (toString ./.)).inputs.nixpkgs { }, - system ? pkgs.system, + system ? pkgs.stdenv.hostPlatform.system, }: { diff --git a/subprojects/nix-eval-jobs/tests/assets/flake.nix b/subprojects/nix-eval-jobs/tests/assets/flake.nix index 7f6cc61fa..abd5faa08 100644 --- a/subprojects/nix-eval-jobs/tests/assets/flake.nix +++ b/subprojects/nix-eval-jobs/tests/assets/flake.nix @@ -19,7 +19,7 @@ recursion = [ recursion ]; in derivation { - inherit (pkgs) system; + inherit (pkgs.stdenv.hostPlatform) system; name = "drvB"; recursiveAttr = recursion; builder = ":"; diff --git a/subprojects/nix-eval-jobs/tests/test_eval.py b/subprojects/nix-eval-jobs/tests/test_eval.py index 8fa41cb72..c452b79bf 100755 --- a/subprojects/nix-eval-jobs/tests/test_eval.py +++ b/subprojects/nix-eval-jobs/tests/test_eval.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 import json +import os import subprocess import pytest from pathlib import Path @@ -8,22 +9,48 @@ from tempfile import TemporaryDirectory from typing import Any, Dict, List TEST_ROOT = Path(__file__).parent.resolve() -PROJECT_ROOT = TEST_ROOT.parent -BIN = PROJECT_ROOT.joinpath("build", "src", "nix-eval-jobs") +# subprojects/nix-eval-jobs/tests +# PROJECT_ROOT = TEST_ROOT.parent.parent.parent +# BIN = PROJECT_ROOT.joinpath("outputs", "out", "bin", "nix-eval-jobs") +BIN = "nix-eval-jobs" + + +def evaluate( + tempdir: TemporaryDirectory, + expected_statuscode: int = 0, + extra_args: List[str] = [], +) -> tuple[Dict[str, Dict[str, Any]], str]: + if nixpkgs_path := os.getenv("NEJ_NIXPKGS_PATH"): + if "--flake" in extra_args: + extra_args.extend(["--override-input", "nixpkgs", f"path:{nixpkgs_path}"]) + else: + extra_args.extend(["--arg", "pkgs", f"import {nixpkgs_path} {{}}"]) + + cmd = [ + str(BIN), + "--gc-roots-dir", + tempdir, + "--meta", + "--extra-experimental-features", + "flakes", + ] + extra_args + res = subprocess.run( + cmd, + cwd=TEST_ROOT.joinpath("assets"), + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + + assert res.returncode == expected_statuscode + print(res.stdout) + print(res.stderr) + return [json.loads(r) for r in res.stdout.split("\n") if r], res.stderr def common_test(extra_args: List[str]) -> List[Dict[str, Any]]: with TemporaryDirectory() as tempdir: - cmd = [str(BIN), "--gc-roots-dir", tempdir, "--meta"] + extra_args - res = subprocess.run( - cmd, - cwd=TEST_ROOT.joinpath("assets"), - text=True, - check=True, - stdout=subprocess.PIPE, - ) - - results = [json.loads(r) for r in res.stdout.split("\n") if r] + results, _ = evaluate(tempdir, 0, extra_args) assert len(results) == 4 built_job = results[0] @@ -73,48 +100,31 @@ def test_expression() -> None: def test_eval_error() -> None: with TemporaryDirectory() as tempdir: - cmd = [ - str(BIN), - "--gc-roots-dir", + results, _ = evaluate( tempdir, - "--meta", - "--workers", - "1", - "--flake", - ".#legacyPackages.x86_64-linux.brokenPkgs", - ] - res = subprocess.run( - cmd, - cwd=TEST_ROOT.joinpath("assets"), - text=True, - stdout=subprocess.PIPE, + 0, + ["--workers", "1", "--flake", ".#legacyPackages.x86_64-linux.brokenPkgs"], ) - print(res.stdout) - attrs = json.loads(res.stdout) - assert attrs["attr"] == "brokenPackage" - assert "this is an evaluation error" in attrs["error"] + assert len(results) == 1 + + attr = results[0] + assert attr["attr"] == "brokenPackage" + assert "this is an evaluation error" in attr["error"] @pytest.mark.infiniterecursion def test_recursion_error() -> None: with TemporaryDirectory() as tempdir: - cmd = [ - str(BIN), - "--gc-roots-dir", + results, stderr = evaluate( tempdir, - "--meta", - "--workers", - "1", - "--flake", - ".#legacyPackages.x86_64-linux.infiniteRecursionPkgs", - ] - res = subprocess.run( - cmd, - cwd=TEST_ROOT.joinpath("assets"), - text=True, - stderr=subprocess.PIPE, + 1, + [ + "--workers", + "1", + "--flake", + ".#legacyPackages.x86_64-linux.infiniteRecursionPkgs", + ], ) - assert res.returncode == 1 - print(res.stderr) - assert "packageWithInfiniteRecursion" in res.stderr - assert "possible infinite recursion" in res.stderr + print(stderr) + assert "packageWithInfiniteRecursion" in stderr + assert "possible infinite recursion" in stderr From 5002a4a2b9ef6534c563318805ef15dd362d2657 Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Fri, 21 Feb 2025 22:33:28 +0100 Subject: [PATCH 2/2] subprojects/nix-eval-jobs: fix gcroots There's an `aio.blockOn` missing to resolve the promise. Also added a test to verify that the GC roots are created as expected. Change-Id: I88816cc1161372674ded15374b13323f41846453 --- subprojects/nix-eval-jobs/default.nix | 11 +++++++++-- subprojects/nix-eval-jobs/src/worker.cc | 2 +- subprojects/nix-eval-jobs/tests/test_eval.py | 12 ++++++++++++ 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/subprojects/nix-eval-jobs/default.nix b/subprojects/nix-eval-jobs/default.nix index 92dee1f9b..a0ec6369b 100644 --- a/subprojects/nix-eval-jobs/default.nix +++ b/subprojects/nix-eval-jobs/default.nix @@ -8,11 +8,12 @@ }: let + sourceBase = if srcDir == null then ./. else srcDir; package = stdenv.mkDerivation (finalAttrs: { pname = "nix-eval-jobs"; version = "2.93.0-dev"; src = lib.fileset.toSource { - root = if srcDir == null then ./. else srcDir; + root = sourceBase; fileset = lib.fileset.unions [ ./meson.build ./src @@ -50,7 +51,13 @@ let stdenv.mkDerivation (finalAttrs: { pname = "nix-eval-jobs-tests"; inherit (nejAttrs) version; - src = ./tests; + src = lib.fileset.toSource { + root = sourceBase; + fileset = lib.fileset.unions [ + ./pyproject.toml + ./tests + ]; + }; nativeBuildInputs = [ python3Packages.pytest diff --git a/subprojects/nix-eval-jobs/src/worker.cc b/subprojects/nix-eval-jobs/src/worker.cc index 934119d7e..35c176407 100644 --- a/subprojects/nix-eval-jobs/src/worker.cc +++ b/subprojects/nix-eval-jobs/src/worker.cc @@ -144,7 +144,7 @@ void worker(nix::ref evaluator, .dynamic_pointer_cast(); auto storePath = localStore->parseStorePath(drv.drvPath); - localStore->addPermRoot(storePath, root); + aio.blockOn(localStore->addPermRoot(storePath, root)); } } } else { diff --git a/subprojects/nix-eval-jobs/tests/test_eval.py b/subprojects/nix-eval-jobs/tests/test_eval.py index c452b79bf..166dc8029 100755 --- a/subprojects/nix-eval-jobs/tests/test_eval.py +++ b/subprojects/nix-eval-jobs/tests/test_eval.py @@ -15,6 +15,15 @@ TEST_ROOT = Path(__file__).parent.resolve() BIN = "nix-eval-jobs" +def check_gc_root(gcRootDir: str, drvPath: str): + """ + Make sure the expected GC root exists in the given dir + """ + link_name = os.path.basename(drvPath) + symlink_path = os.path.join(gcRootDir, link_name) + assert os.path.islink(symlink_path) and drvPath == os.readlink(symlink_path) + + def evaluate( tempdir: TemporaryDirectory, expected_statuscode: int = 0, @@ -59,14 +68,17 @@ def common_test(extra_args: List[str]) -> List[Dict[str, Any]]: assert built_job["outputs"]["out"].startswith("/nix/store") assert built_job["drvPath"].endswith(".drv") assert built_job["meta"]["broken"] is False + check_gc_root(tempdir, built_job['drvPath']) dotted_job = results[1] assert dotted_job["attr"] == '"dotted.attr"' assert dotted_job["attrPath"] == ["dotted.attr"] + check_gc_root(tempdir, dotted_job['drvPath']) recurse_drv = results[2] assert recurse_drv["attr"] == "recurse.drvB" assert recurse_drv["name"] == "drvB" + check_gc_root(tempdir, recurse_drv['drvPath']) substituted_job = results[3] assert substituted_job["attr"] == "substitutedJob"