From 2d58d8a513bc6c02bd81697e87d985cad1ddcd94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sun, 21 Mar 2021 18:35:16 +0100 Subject: [PATCH 1/6] gitignore python --- .gitignore | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.gitignore b/.gitignore index 4ec0ffddd..250b5c443 100644 --- a/.gitignore +++ b/.gitignore @@ -42,3 +42,13 @@ tmp/ /build # nix-build /result + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json From b48e9667dcce3127f8cbdb703b6491c72eb8272e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sun, 21 Mar 2021 18:37:15 +0100 Subject: [PATCH 2/6] make gcRootsDir local just to avoid unecessary globals --- src/hydra-eval-jobs.cc | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/hydra-eval-jobs.cc b/src/hydra-eval-jobs.cc index 944476e31..3582f9f6f 100644 --- a/src/hydra-eval-jobs.cc +++ b/src/hydra-eval-jobs.cc @@ -25,11 +25,11 @@ using namespace nix; -static Path gcRootsDir; struct MyArgs : MixEvalArgs, MixCommonArgs { Path releaseExpr; + Path gcRootsDir; bool flake = false; bool dryRun = false; size_t nrWorkers = 1; @@ -123,7 +123,8 @@ static void worker( EvalState & state, Bindings & autoArgs, AutoCloseFD & to, - AutoCloseFD & from) + AutoCloseFD & from, + const Path &gcRootsDir) { Value vTop; @@ -315,7 +316,7 @@ int main(int argc, char * * argv) if (myArgs.releaseExpr == "") throw UsageError("no expression specified"); - if (gcRootsDir == "") printMsg(lvlError, "warning: `--gc-roots-dir' not specified"); + if (myArgs.gcRootsDir == "") printMsg(lvlError, "warning: `--gc-roots-dir' not specified"); struct State { @@ -352,7 +353,7 @@ int main(int argc, char * * argv) try { EvalState state(myArgs.searchPath, openStore()); Bindings & autoArgs = *myArgs.getAutoArgs(state); - worker(state, autoArgs, *to, *from); + worker(state, autoArgs, *to, *from, myArgs.gcRootsDir); } catch (Error & e) { nlohmann::json err; auto msg = e.msg(); From f00607a7c6cf5b523ccb300a92f3924ae3c9e139 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sun, 21 Mar 2021 18:38:56 +0100 Subject: [PATCH 3/6] exit hydra-eval-jobs after printing help --- src/hydra-eval-jobs.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/hydra-eval-jobs.cc b/src/hydra-eval-jobs.cc index 3582f9f6f..2f964736e 100644 --- a/src/hydra-eval-jobs.cc +++ b/src/hydra-eval-jobs.cc @@ -48,6 +48,7 @@ struct MyArgs : MixEvalArgs, MixCommonArgs } printf(" --%-20s %s\n", name.c_str(), flag->description.c_str()); } + ::exit(0); }}, }); From 617d4ee15138840c468d7ac4fefe21bef740d8bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sun, 21 Mar 2021 18:39:32 +0100 Subject: [PATCH 4/6] add --impure flag for flakes --- src/hydra-eval-jobs.cc | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/hydra-eval-jobs.cc b/src/hydra-eval-jobs.cc index 2f964736e..5a0979090 100644 --- a/src/hydra-eval-jobs.cc +++ b/src/hydra-eval-jobs.cc @@ -25,6 +25,7 @@ using namespace nix; +typedef enum { evalAuto, evalImpure, evalPure } pureEval; struct MyArgs : MixEvalArgs, MixCommonArgs { @@ -34,6 +35,7 @@ struct MyArgs : MixEvalArgs, MixCommonArgs bool dryRun = false; size_t nrWorkers = 1; size_t maxMemorySize = 4096; + pureEval evalMode = evalAuto; MyArgs() : MixCommonArgs("hydra-eval-jobs") { @@ -52,6 +54,13 @@ struct MyArgs : MixEvalArgs, MixCommonArgs }}, }); + addFlag({ + .longName = "impure", + .description = "set evaluation mode", + .handler = {[&]() { + evalMode = evalImpure; + }}, + }); addFlag({ .longName = "gc-roots-dir", .description = "garbage collector roots directory", @@ -311,7 +320,7 @@ int main(int argc, char * * argv) /* When building a flake, use pure evaluation (no access to 'getEnv', 'currentSystem' etc. */ - evalSettings.pureEval = myArgs.flake; + evalSettings.pureEval = myArgs.evalMode == evalAuto ? myArgs.flake : myArgs.evalMode == evalPure; if (myArgs.dryRun) settings.readOnlyMode = true; From 71cbe4eab47cf5790b7ca9b9eb3fbc180c23866f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sun, 21 Mar 2021 19:05:11 +0100 Subject: [PATCH 5/6] output missing/substitutable derivations in eval output --- flake.nix | 21 +++++--- src/hydra-eval-jobs.cc | 105 ++++++++++++++++++++++------------------ tests/assets/ci.nix | 5 ++ tests/assets/flake.lock | 26 ++++++++++ tests/assets/flake.nix | 12 +++++ tests/test_eval.py | 34 +++++++++++++ 6 files changed, 151 insertions(+), 52 deletions(-) create mode 100644 tests/assets/ci.nix create mode 100644 tests/assets/flake.lock create mode 100644 tests/assets/flake.nix create mode 100644 tests/test_eval.py diff --git a/flake.nix b/flake.nix index 10b15c4f2..998772dbe 100644 --- a/flake.nix +++ b/flake.nix @@ -5,10 +5,19 @@ inputs.flake-utils.url = "github:numtide/flake-utils"; outputs = { self, nixpkgs, flake-utils }: - flake-utils.lib.eachDefaultSystem (system: { - packages.hydra-eval-jobs = nixpkgs.legacyPackages.${system}.callPackage ./hydra.nix { - srcDir = self; - }; - defaultPackage = self.packages.${system}.hydra-eval-jobs; - }); + flake-utils.lib.eachDefaultSystem (system: + let + pkgs = nixpkgs.legacyPackages.${system}; + in + rec { + packages.hydra-eval-jobs = pkgs.callPackage ./hydra.nix { + srcDir = self; + }; + defaultPackage = self.packages.${system}.hydra-eval-jobs; + devShell = defaultPackage.overrideAttrs (old: { + nativeBuildInputs = old.nativeBuildInputs ++ [ + pkgs.python3.pkgs.pytest + ]; + }); + }); } diff --git a/src/hydra-eval-jobs.cc b/src/hydra-eval-jobs.cc index 5a0979090..bcfecb75e 100644 --- a/src/hydra-eval-jobs.cc +++ b/src/hydra-eval-jobs.cc @@ -32,7 +32,6 @@ struct MyArgs : MixEvalArgs, MixCommonArgs Path releaseExpr; Path gcRootsDir; bool flake = false; - bool dryRun = false; size_t nrWorkers = 1; size_t maxMemorySize = 4096; pureEval evalMode = evalAuto; @@ -61,6 +60,7 @@ struct MyArgs : MixEvalArgs, MixCommonArgs evalMode = evalImpure; }}, }); + addFlag({ .longName = "gc-roots-dir", .description = "garbage collector roots directory", @@ -86,12 +86,6 @@ struct MyArgs : MixEvalArgs, MixCommonArgs }} }); - addFlag({ - .longName = "dry-run", - .description = "don't create store derivations", - .handler = {&dryRun, true} - }); - addFlag({ .longName = "flake", .description = "build a flake", @@ -129,6 +123,14 @@ static std::string queryMetaStrings(EvalState & state, DrvInfo & drv, const stri return concatStringsSep(", ", res); } +static nlohmann::json serializeStorePathSet(StorePathSet &paths, LocalFSStore &store) { + auto array = nlohmann::json::array(); + for (auto & p : paths) { + array.push_back(store.printStorePath(p)); + } + return array; +} + static void worker( EvalState & state, Bindings & autoArgs, @@ -244,17 +246,40 @@ static void worker( registers roots for jobs that we may have already done. */ auto localStore = state.store.dynamic_pointer_cast(); + auto storePath = localStore->parseStorePath(drvPath); if (gcRootsDir != "" && localStore) { Path root = gcRootsDir + "/" + std::string(baseNameOf(drvPath)); if (!pathExists(root)) - localStore->addPermRoot(localStore->parseStorePath(drvPath), root); + localStore->addPermRoot(storePath, root); } + uint64_t downloadSize, narSize; + StorePathSet willBuild, willSubstitute, unknown; + std::vector paths; + StringSet outputNames; + + for (auto & output : outputs) { + outputNames.insert(output.first); + } + paths.push_back({storePath, outputNames}); + + localStore->queryMissing(paths, + willBuild, + willSubstitute, + unknown, + downloadSize, + narSize); + nlohmann::json out; - for (auto & j : outputs) - out[j.first] = j.second; + for (auto & p : outputs) { + out[p.first] = p.second; + } job["outputs"] = std::move(out); + job["builds"] = serializeStorePathSet(willBuild, *localStore); + job["substitutes"] = serializeStorePathSet(willSubstitute, *localStore); + job["unknown"] = serializeStorePathSet(unknown, *localStore); + reply["job"] = std::move(job); } @@ -322,8 +347,6 @@ int main(int argc, char * * argv) 'getEnv', 'currentSystem' etc. */ evalSettings.pureEval = myArgs.evalMode == evalAuto ? myArgs.flake : myArgs.evalMode == evalPure; - if (myArgs.dryRun) settings.readOnlyMode = true; - if (myArgs.releaseExpr == "") throw UsageError("no expression specified"); if (myArgs.gcRootsDir == "") printMsg(lvlError, "warning: `--gc-roots-dir' not specified"); @@ -477,43 +500,33 @@ int main(int argc, char * * argv) auto named = job.find("namedConstituents"); if (named == job.end()) continue; - if (myArgs.dryRun) { - for (std::string jobName2 : *named) { - auto job2 = state->jobs.find(jobName2); - if (job2 == state->jobs.end()) - throw Error("aggregate job '%s' references non-existent job '%s'", jobName, jobName2); - std::string drvPath2 = (*job2)["drvPath"]; - job["constituents"].push_back(drvPath2); - } - } else { - auto drvPath = store->parseStorePath((std::string) job["drvPath"]); - auto drv = store->readDerivation(drvPath); + auto drvPath = store->parseStorePath((std::string) job["drvPath"]); + auto drv = store->readDerivation(drvPath); - for (std::string jobName2 : *named) { - auto job2 = state->jobs.find(jobName2); - if (job2 == state->jobs.end()) - throw Error("aggregate job '%s' references non-existent job '%s'", jobName, jobName2); - auto drvPath2 = store->parseStorePath((std::string) (*job2)["drvPath"]); - auto drv2 = store->readDerivation(drvPath2); - job["constituents"].push_back(store->printStorePath(drvPath2)); - drv.inputDrvs[drvPath2] = {drv2.outputs.begin()->first}; - } - - std::string drvName(drvPath.name()); - assert(hasSuffix(drvName, drvExtension)); - drvName.resize(drvName.size() - drvExtension.size()); - auto h = std::get(hashDerivationModulo(*store, drv, true)); - auto outPath = store->makeOutputPath("out", h, drvName); - drv.env["out"] = store->printStorePath(outPath); - drv.outputs.insert_or_assign("out", DerivationOutput { .output = DerivationOutputInputAddressed { .path = outPath } }); - auto newDrvPath = store->printStorePath(writeDerivation(*store, drv)); - - debug("rewrote aggregate derivation %s -> %s", store->printStorePath(drvPath), newDrvPath); - - job["drvPath"] = newDrvPath; - job["outputs"]["out"] = store->printStorePath(outPath); + for (std::string jobName2 : *named) { + auto job2 = state->jobs.find(jobName2); + if (job2 == state->jobs.end()) + throw Error("aggregate job '%s' references non-existent job '%s'", jobName, jobName2); + auto drvPath2 = store->parseStorePath((std::string) (*job2)["drvPath"]); + auto drv2 = store->readDerivation(drvPath2); + job["constituents"].push_back(store->printStorePath(drvPath2)); + drv.inputDrvs[drvPath2] = {drv2.outputs.begin()->first}; } + std::string drvName(drvPath.name()); + assert(hasSuffix(drvName, drvExtension)); + drvName.resize(drvName.size() - drvExtension.size()); + auto h = std::get(hashDerivationModulo(*store, drv, true)); + auto outPath = store->makeOutputPath("out", h, drvName); + drv.env["out"] = store->printStorePath(outPath); + drv.outputs.insert_or_assign("out", DerivationOutput { .output = DerivationOutputInputAddressed { .path = outPath } }); + auto newDrvPath = store->printStorePath(writeDerivation(*store, drv)); + + debug("rewrote aggregate derivation %s -> %s", store->printStorePath(drvPath), newDrvPath); + + job["drvPath"] = newDrvPath; + job["outputs"]["out"] = store->printStorePath(outPath); + job.erase("namedConstituents"); } diff --git a/tests/assets/ci.nix b/tests/assets/ci.nix new file mode 100644 index 000000000..5e80362e4 --- /dev/null +++ b/tests/assets/ci.nix @@ -0,0 +1,5 @@ +with import {}; +{ + builtJob = pkgs.writeText "job1" "job1"; + substitutedJob = pkgs.hello; +} diff --git a/tests/assets/flake.lock b/tests/assets/flake.lock new file mode 100644 index 000000000..a1a80bf7c --- /dev/null +++ b/tests/assets/flake.lock @@ -0,0 +1,26 @@ +{ + "nodes": { + "nixpkgs": { + "locked": { + "lastModified": 1616345250, + "narHash": "sha256-WLbLFIJyKCklGyEMGwh9XDTzafafyO95s4+rJHOc/Ag=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "5e4a4e0c32f0ca0a5bd4ebbbf17aedd347de7f3e", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "nixpkgs": "nixpkgs" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/tests/assets/flake.nix b/tests/assets/flake.nix new file mode 100644 index 000000000..bbd81a6bf --- /dev/null +++ b/tests/assets/flake.nix @@ -0,0 +1,12 @@ +{ + inputs.nixpkgs.url = "github:NixOS/nixpkgs"; + + outputs = { self, nixpkgs }: let + pkgs = nixpkgs.legacyPackages.x86_64-linux; + in { + hydraJobs = { + builtJob = pkgs.writeText "job1" "job1"; + substitutedJob = pkgs.hello; + }; + }; +} diff --git a/tests/test_eval.py b/tests/test_eval.py new file mode 100644 index 000000000..02eced9df --- /dev/null +++ b/tests/test_eval.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python3 + +import subprocess +import json +from tempfile import TemporaryDirectory +from pathlib import Path +from typing import List + +TEST_ROOT = Path(__file__).parent.resolve() +PROJECT_ROOT = TEST_ROOT.parent +BIN = PROJECT_ROOT.joinpath("build", "src", "hydra-eval-jobs") + + +def common_test(extra_args: List[str]) -> None: + with TemporaryDirectory() as tempdir: + cmd = [str(BIN), "--gc-roots-dir", tempdir] + extra_args + res = subprocess.run( + cmd, + cwd=TEST_ROOT.joinpath("assets"), + text=True, + check=True, + stdout=subprocess.PIPE, + ) + data = json.loads(res.stdout) + assert len(data["builtJob"]["builds"]) == 1 + assert len(data["substitutedJob"]["substitutes"]) >= 1 + + +def test_flake() -> None: + common_test(["--flake", ".#"]) + + +def test_expression() -> None: + common_test(["ci.nix"]) From 76c3e68ccd369c0a788d6a779012d75248ffe96c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sun, 21 Mar 2021 19:18:03 +0100 Subject: [PATCH 6/6] run tests in CI --- .github/workflows/test-flakes.yml | 6 +++++- tests/assets/ci.nix | 4 +++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-flakes.yml b/.github/workflows/test-flakes.yml index a241baef4..b400fc060 100644 --- a/.github/workflows/test-flakes.yml +++ b/.github/workflows/test-flakes.yml @@ -26,4 +26,8 @@ jobs: - name: List flake structure run: nix flake show - name: Build - run: nix build + run: nix build --out-link result + - name: Run tests + run: | + nix develop -c install -D ./result/bin/hydra-eval-jobs ./build/src/hydra-eval-jobs + nix develop -c pytest ./tests diff --git a/tests/assets/ci.nix b/tests/assets/ci.nix index 5e80362e4..48bf1774f 100644 --- a/tests/assets/ci.nix +++ b/tests/assets/ci.nix @@ -1,4 +1,6 @@ -with import {}; +let + pkgs = import (builtins.getFlake (toString ./.)).inputs.nixpkgs {}; +in { builtJob = pkgs.writeText "job1" "job1"; substitutedJob = pkgs.hello;