From 8cb2306138089f6d9deb49cb3cedec29af7e29dd Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Wed, 19 Feb 2025 12:51:30 +0100 Subject: [PATCH] subprojects/nix-eval-jobs: bring back constituents Hydra used to support aggregate jobs that only succeeded when their constituents succeed. This is still used by e.g. nixpkgs[1]. Prior art: * https://git.lix.systems/lix-project/nix-eval-jobs/pulls/17: got ported into the CppNix implementation[2] * https://github.com/nix-community/nix-eval-jobs/pull/349: implements glob expressions for constituents - something we needed at work. This also restructures the code a bit which is what I re-used here. The globbing is not part of this patch. Essentially, the following things happen here (assuming `--constituents` is set): * Derivations with `_hydraAggregate = true;` are considered aggregates. These are not written to stdout when received by a worker, but stored until the end. * Constituents can be drv paths or strings (that must be the `attr` of another job). In that case, the derivation of the aggregate job is rewritten so that it depends on the drv of the constituent job. * At the very end the aggregate jobs are also written to stdout. Additionally, this fixes one bug, the old `hydra-eval-jobs` implementation had (and we actually hit at work): Given the leaf jobs `packages.foo` & `packages.bar`, an aggregate job `aggregate0` with _hydraAggregate = true; constituents = [ "packages.bar" "packages.foo" ]; and an aggregate job `aggregate1` with constituents = [ "aggregate0" ]; then it may happen depending on the order of evaluation that `aggregate1` depends on the old derivation of `aggregate0` (i.e. the one without rewritten constituents) and doesn't depend on `packages.foo` and `packages.bar` because it was rewritten before `aggregate0` was rewritten. This is done in here correctly, but topologically sorting the aggregate jobs before rewriting those. [1] https://github.com/NixOS/nixpkgs/blob/bba6b37c9d0898867a7d9c38a1b5b77efcfb07b9/nixos/release-combined.nix#L69 [2] https://github.com/nix-community/nix-eval-jobs/pull/340 Change-Id: I5baad5e57336b4985ef8595e903814de83eb01c1 --- subprojects/nix-eval-jobs/README.md | 51 +++++ subprojects/nix-eval-jobs/src/constituents.cc | 184 ++++++++++++++++++ subprojects/nix-eval-jobs/src/constituents.hh | 45 +++++ subprojects/nix-eval-jobs/src/drv.cc | 25 ++- subprojects/nix-eval-jobs/src/drv.hh | 18 +- subprojects/nix-eval-jobs/src/eval-args.cc | 6 + subprojects/nix-eval-jobs/src/eval-args.hh | 1 + subprojects/nix-eval-jobs/src/meson.build | 1 + .../nix-eval-jobs/src/nix-eval-jobs.cc | 41 +++- subprojects/nix-eval-jobs/src/worker.cc | 69 +++++-- .../nix-eval-jobs/tests/assets/flake.nix | 46 ++++- subprojects/nix-eval-jobs/tests/test_eval.py | 130 +++++++++++++ 12 files changed, 598 insertions(+), 19 deletions(-) create mode 100644 subprojects/nix-eval-jobs/src/constituents.cc create mode 100644 subprojects/nix-eval-jobs/src/constituents.hh diff --git a/subprojects/nix-eval-jobs/README.md b/subprojects/nix-eval-jobs/README.md index e2c8bcc18..cea04ee10 100644 --- a/subprojects/nix-eval-jobs/README.md +++ b/subprojects/nix-eval-jobs/README.md @@ -90,6 +90,57 @@ single large log file. In the [wiki](https://github.com/nix-community/nix-eval-jobs/wiki#ci-example-configurations) we collect example ci configuration for various CIs. +## Aggregate jobs + +`nix-eval-jobs` supports the [Hydra's aggregate job feature](https://determinate.systems/posts/hydra-deployment-source-of-truth/#aggregate-jobs). The behavior is turned off by default and must be activated by passing `--constituents`. + +When evaluating +```nix +stdenv.mkDerivation { + pname = "aggregate-job"; + + _hydraAggregate = true; + constituents = [ + /* drvs */ + ]; +} +``` + +a JSON object is returned with a field `constituents` containing +the drv paths of each constituent. + +It's generally recommended to pass strings into `constituents` given +it makes the evaluation way cheaper at the expense of `nix-eval-jobs` +producing a different derivation in contrast to other Nix tools. In that +case, the strings must correspond to the `attr` key of other jobs from +the same evaluation. The aggregate job will be rewritten such that +it has its constituents as input derivations. + +For example, + +```nix +{ + jobs.constituent = mkDerivation { + name = "foo"; + }; + aggregate = mkDerivation { + name = "aggregate"; + _hydraAggregate = true; + constituents = [ "jobs.constituent" ]; + }; +} +``` + +results in two JSON objects returned from `nix-eval-jobs` where + +* `jobs.constituent` isn't changed. +* `aggregate` is a modified variant of the derivation in the expression + above which also depends on the derivation of `jobs.constituent`. + The `constituents` field doesn't contain the string `jobs.constituent`, + but the corresponding drv path. + +Cycles in aggregate jobs are not allowed and cause an error. + ## Organisation of this repository `main` follows Lix HEAD, and is updated alongside the Lix NixOS module. When we diff --git a/subprojects/nix-eval-jobs/src/constituents.cc b/subprojects/nix-eval-jobs/src/constituents.cc new file mode 100644 index 000000000..5b55efa92 --- /dev/null +++ b/subprojects/nix-eval-jobs/src/constituents.cc @@ -0,0 +1,184 @@ +#include +#include +#include +#include +#include + +#include "constituents.hh" +#include "lix/libutil/async.hh" +#include "drv.hh" + +#include + +namespace { +// This is copied from `libutil/topo-sort.hh` in CppNix and slightly modified. +// However, I needed a way to use strings as identifiers to sort, but still be +// able to put AggregateJob objects into this function since I'd rather not have +// to transform back and forth between a list of strings and AggregateJobs in +// resolveNamedConstituents. +auto topoSort(const std::set &items) + -> std::vector { + std::vector sorted; + std::set visited; + std::set parents; + + std::map dictIdentToObject; + for (const auto &it : items) { + dictIdentToObject.insert({it.name, it}); + } + + std::function + dfsVisit; + + dfsVisit = [&](const std::string &path, const std::string *parent) { + if (parents.contains(path)) { + dictIdentToObject.erase(path); + dictIdentToObject.erase(*parent); + std::set remaining; + for (auto &[k, _] : dictIdentToObject) { + remaining.insert(k); + } + throw DependencyCycle(path, *parent, remaining); + } + + if (!visited.insert(path).second) { + return; + } + parents.insert(path); + + std::set references = dictIdentToObject[path].dependencies; + + for (const auto &i : references) { + /* Don't traverse into items that don't exist in our starting set. + */ + if (i != path && + dictIdentToObject.find(i) != dictIdentToObject.end()) { + dfsVisit(i, &path); + } + } + + sorted.push_back(dictIdentToObject[path]); + parents.erase(path); + }; + + for (auto &[i, _] : dictIdentToObject) { + dfsVisit(i, nullptr); + } + + return sorted; +} +} // namespace + +auto resolveNamedConstituents(const std::map &jobs) + -> std::variant, DependencyCycle> { + std::set aggregateJobs; + for (auto const &[jobName, job] : jobs) { + auto named = job.find("namedConstituents"); + if (named != job.end() && !named->empty()) { + std::unordered_map brokenJobs; + std::set results; + + auto isBroken = [&brokenJobs, + &jobName](const std::string &childJobName, + const nlohmann::json &job) -> bool { + if (job.find("error") != job.end()) { + std::string error = job["error"]; + nix::logger->log( + nix::lvlError, + nix::fmt( + "aggregate job '%s' references broken job '%s': %s", + jobName, childJobName, error)); + brokenJobs[childJobName] = error; + return true; + } + return false; + }; + + for (const std::string childJobName : *named) { + auto childJobIter = jobs.find(childJobName); + if (childJobIter == jobs.end()) { + nix::logger->log(nix::lvlError, + nix::fmt("aggregate job '%s' references " + "non-existent job '%s'", + jobName, childJobName)); + brokenJobs[childJobName] = "does not exist"; + } else if (!isBroken(childJobName, childJobIter->second)) { + results.insert(childJobName); + } + } + + aggregateJobs.insert(AggregateJob(jobName, results, brokenJobs)); + } + } + + try { + return topoSort(aggregateJobs); + } catch (DependencyCycle &e) { + return e; + } +} + +void rewriteAggregates(std::map &jobs, + const std::vector &aggregateJobs, + nix::ref &store, nix::Path &gcRootsDir, + nix::AsyncIoRoot &aio) { + for (const auto &aggregateJob : aggregateJobs) { + auto &job = jobs.find(aggregateJob.name)->second; + auto drvPath = store->parseStorePath(std::string(job["drvPath"])); + auto drv = store->readDerivation(drvPath); + + if (aggregateJob.brokenJobs.empty()) { + for (const auto &childJobName : aggregateJob.dependencies) { + auto childDrvPath = store->parseStorePath( + std::string(jobs.find(childJobName)->second["drvPath"])); + auto childDrv = store->readDerivation(childDrvPath); + job["constituents"].push_back( + store->printStorePath(childDrvPath)); + drv.inputDrvs.map[childDrvPath].value = { + childDrv.outputs.begin()->first}; + } + + std::string drvName(drvPath.name()); + assert(drvName.ends_with(nix::drvExtension)); + drvName.resize(drvName.size() - nix::drvExtension.size()); + + auto hashModulo = hashDerivationModulo(*store, drv, true); + if (hashModulo.kind != nix::DrvHash::Kind::Regular) { + continue; + } + auto h = hashModulo.hashes.find("out"); + if (h == hashModulo.hashes.end()) { + continue; + } + auto outPath = store->makeOutputPath("out", h->second, drvName); + drv.env["out"] = store->printStorePath(outPath); + drv.outputs.insert_or_assign( + "out", nix::DerivationOutput::InputAddressed{.path = outPath}); + + auto newDrvPath = aio.blockOn(nix::writeDerivation(*store, drv)); + auto newDrvPathS = store->printStorePath(newDrvPath); + + register_gc_root(gcRootsDir, newDrvPathS, store, aio); + + nix::logger->log(nix::lvlDebug, + nix::fmt("rewrote aggregate derivation %s -> %s", + store->printStorePath(drvPath), + newDrvPathS)); + + job["drvPath"] = newDrvPathS; + job["outputs"]["out"] = store->printStorePath(outPath); + } + + job.erase("namedConstituents"); + + if (!aggregateJob.brokenJobs.empty()) { + std::stringstream ss; + for (const auto &[jobName, error] : aggregateJob.brokenJobs) { + ss << jobName << ": " << error << "\n"; + } + job["error"] = ss.str(); + } + + std::cout << job.dump() << "\n" << std::flush; + } +} diff --git a/subprojects/nix-eval-jobs/src/constituents.hh b/subprojects/nix-eval-jobs/src/constituents.hh new file mode 100644 index 000000000..94c2c89ca --- /dev/null +++ b/subprojects/nix-eval-jobs/src/constituents.hh @@ -0,0 +1,45 @@ +#pragma once + +#include "lix/libutil/fmt.hh" +#include +#include +#include +#include +#include +#include + +#include +#include + +struct DependencyCycle : public std::exception { + std::string a; + std::string b; + std::set remainingAggregates; + + DependencyCycle(std::string a, std::string b, + const std::set &remainingAggregates) + : a(std::move(a)), b(std::move(b)), + remainingAggregates(remainingAggregates) {} + + [[nodiscard]] auto message() const -> std::string { + return nix::fmt("Dependency cycle: %s <-> %s", a, b); + } +}; + +struct AggregateJob { + std::string name; + std::set dependencies; + std::unordered_map brokenJobs; + + auto operator<(const AggregateJob &b) const -> bool { + return name < b.name; + } +}; + +auto resolveNamedConstituents(const std::map &jobs) + -> std::variant, DependencyCycle>; + +void rewriteAggregates(std::map &jobs, + const std::vector &aggregateJobs, + nix::ref &store, nix::Path &gcRootsDir, + nix::AsyncIoRoot &aio); diff --git a/subprojects/nix-eval-jobs/src/drv.cc b/subprojects/nix-eval-jobs/src/drv.cc index bacbf2a1c..00a732aee 100644 --- a/subprojects/nix-eval-jobs/src/drv.cc +++ b/subprojects/nix-eval-jobs/src/drv.cc @@ -43,7 +43,8 @@ queryIsCached(nix::Store &store, /* The fields of a derivation that are printed in json form */ Drv::Drv(std::string &attrPath, nix::EvalState &state, nix::DrvInfo &drvInfo, - MyArgs &args) { + MyArgs &args, std::optional constituents) + : constituents(constituents) { auto localStore = state.ctx.store.dynamic_pointer_cast(); @@ -128,7 +129,29 @@ void to_json(nlohmann::json &json, const Drv &drv) { json["meta"] = drv.meta.value(); } + if (auto constituents = drv.constituents) { + json["constituents"] = constituents->constituents; + json["namedConstituents"] = constituents->namedConstituents; + } + if (drv.cacheStatus != Drv::CacheStatus::Unknown) { json["isCached"] = drv.cacheStatus == Drv::CacheStatus::Cached; } } + +void register_gc_root(nix::Path &gcRootsDir, std::string &drvPath, const nix::ref &store, + nix::AsyncIoRoot &aio) { + if (!gcRootsDir.empty()) { + nix::Path root = + gcRootsDir + "/" + + std::string(nix::baseNameOf(drvPath)); + if (!nix::pathExists(root)) { + auto localStore = + store + .dynamic_pointer_cast(); + auto storePath = + localStore->parseStorePath(drvPath); + aio.blockOn(localStore->addPermRoot(storePath, root)); + } + } +} diff --git a/subprojects/nix-eval-jobs/src/drv.hh b/subprojects/nix-eval-jobs/src/drv.hh index 4cfc6a0bb..ef2bd3532 100644 --- a/subprojects/nix-eval-jobs/src/drv.hh +++ b/subprojects/nix-eval-jobs/src/drv.hh @@ -9,13 +9,22 @@ #include #include "eval-args.hh" +#include "lix/libstore/store-api.hh" class MyArgs; namespace nix { class EvalState; struct DrvInfo; -} // namespace nix +} // namespace nix + +struct Constituents { + std::vector constituents; + std::vector namedConstituents; + Constituents(std::vector constituents, + std::vector namedConstituents) + : constituents(constituents), namedConstituents(namedConstituents) {}; +}; /* The fields of a derivation that are printed in json form */ struct Drv { @@ -27,7 +36,12 @@ struct Drv { std::map> outputs; std::map> inputDrvs; std::optional meta; + std::optional constituents; - Drv(std::string &attrPath, nix::EvalState &state, nix::DrvInfo &drvInfo, MyArgs &args); + Drv(std::string &attrPath, nix::EvalState &state, nix::DrvInfo &drvInfo, + MyArgs &args, std::optional constituents); }; void to_json(nlohmann::json &json, const Drv &drv); + +void register_gc_root(nix::Path &gcRootsDir, std::string &drvPath, + const nix::ref &store, nix::AsyncIoRoot &aio); diff --git a/subprojects/nix-eval-jobs/src/eval-args.cc b/subprojects/nix-eval-jobs/src/eval-args.cc index 57b839c28..5f6c1677d 100644 --- a/subprojects/nix-eval-jobs/src/eval-args.cc +++ b/subprojects/nix-eval-jobs/src/eval-args.cc @@ -37,6 +37,12 @@ MyArgs::MyArgs(nix::AsyncIoRoot & aio) : MixCommonArgs("nix-eval-jobs"), aio_(ai .description = "force recursion (don't respect recurseIntoAttrs)", .handler = {&forceRecurse, true}}); + addFlag( + {.longName = "constituents", + .description = + "whether to evaluate constituents for Hydra's aggregate feature", + .handler = {&constituents, true}}); + addFlag({.longName = "gc-roots-dir", .description = "garbage collector roots directory", .labels = {"path"}, diff --git a/subprojects/nix-eval-jobs/src/eval-args.hh b/subprojects/nix-eval-jobs/src/eval-args.hh index 1316c154f..e46ab1ad3 100644 --- a/subprojects/nix-eval-jobs/src/eval-args.hh +++ b/subprojects/nix-eval-jobs/src/eval-args.hh @@ -28,6 +28,7 @@ class MyArgs : virtual public nix::MixEvalArgs, bool impure = false; bool forceRecurse = false; bool checkCacheStatus = false; + bool constituents = false; size_t nrWorkers = 1; size_t maxMemorySize = 4096; diff --git a/subprojects/nix-eval-jobs/src/meson.build b/subprojects/nix-eval-jobs/src/meson.build index fe068955d..f9288c767 100644 --- a/subprojects/nix-eval-jobs/src/meson.build +++ b/subprojects/nix-eval-jobs/src/meson.build @@ -2,6 +2,7 @@ src = files( 'nix-eval-jobs.cc', 'eval-args.cc', 'drv.cc', + 'constituents.cc', 'buffered-io.cc', 'worker.cc', ) diff --git a/subprojects/nix-eval-jobs/src/nix-eval-jobs.cc b/subprojects/nix-eval-jobs/src/nix-eval-jobs.cc index 97c3aba9b..19b0c0221 100644 --- a/subprojects/nix-eval-jobs/src/nix-eval-jobs.cc +++ b/subprojects/nix-eval-jobs/src/nix-eval-jobs.cc @@ -38,6 +38,7 @@ #include #include +#include "constituents.hh" #include "eval-args.hh" #include "buffered-io.hh" #include "worker.hh" @@ -153,6 +154,7 @@ struct State { std::set todo = json::array({json::array()}); std::set active; std::exception_ptr exc; + std::map jobs; }; void handleBrokenWorkerPipe(Proc &proc, std::string_view msg) { @@ -313,7 +315,12 @@ void collector(MyArgs &myArgs, Sync &state_, } } else { auto state(state_.lock()); - std::cout << respString << "\n" << std::flush; + state->jobs.insert_or_assign(response["attr"], response); + auto named = response.find("namedConstituents"); + if (named == response.end() || named->empty()) { + response.erase("namedConstituents"); + std::cout << response.dump() << "\n" << std::flush; + } } proc_ = std::move(proc); @@ -392,5 +399,37 @@ int main(int argc, char **argv) { if (state->exc) std::rethrow_exception(state->exc); + + if (myArgs.constituents) { + auto store = aio.blockOn(myArgs.evalStoreUrl + ? nix::openStore(*myArgs.evalStoreUrl) + : nix::openStore()); + std::visit( + nix::overloaded{ + [&](const std::vector &namedConstituents) { + rewriteAggregates(state->jobs, namedConstituents, store, + myArgs.gcRootsDir, aio); + }, + [&](const DependencyCycle &e) { + nix::logger->log(nix::lvlError, + nix::fmt("Found dependency cycle " + "between jobs '%s' and '%s'", + e.a, e.b)); + state->jobs[e.a]["error"] = e.message(); + state->jobs[e.b]["error"] = e.message(); + + std::cout << state->jobs[e.a].dump() << "\n" + << state->jobs[e.b].dump() << "\n"; + + for (const auto &jobName : e.remainingAggregates) { + state->jobs[jobName]["error"] = + "Skipping aggregate because of a dependency " + "cycle"; + std::cout << state->jobs[jobName].dump() << "\n"; + } + }, + }, + resolveNamedConstituents(state->jobs)); + } }); } diff --git a/subprojects/nix-eval-jobs/src/worker.cc b/subprojects/nix-eval-jobs/src/worker.cc index 35c176407..34ca87649 100644 --- a/subprojects/nix-eval-jobs/src/worker.cc +++ b/subprojects/nix-eval-jobs/src/worker.cc @@ -74,6 +74,53 @@ static std::string attrPathJoin(nlohmann::json input) { }); } +static std::optional +readConstituents(const nix::Value *v, nix::box_ptr &state, + nix::ref &evaluator) { + auto a = v->attrs->get(state->ctx.symbols.create("_hydraAggregate")); + if (a && state->forceBool(*a->value, a->pos, + "while evaluating the " + "`_hydraAggregate` attribute")) { + std::vector constituents; + std::vector namedConstituents; + auto a = v->attrs->get(state->ctx.symbols.create("constituents")); + if (!a) + state->ctx.errors + .make("derivation must have a ‘constituents’ " + "attribute") + .debugThrow(); + + nix::NixStringContext context; + state->coerceToString(a->pos, *a->value, context, + "while evaluating the `constituents` attribute", + true, false); + for (auto &c : context) + std::visit(nix::overloaded{ + [&](const nix::NixStringContextElem::Built &b) { + constituents.push_back( + b.drvPath->to_string(*evaluator->store)); + }, + [&](const nix::NixStringContextElem::Opaque &) {}, + [&](const nix::NixStringContextElem::DrvDeep &) {}, + }, + c.raw); + + state->forceList(*a->value, a->pos, + "while evaluating the " + "`constituents` attribute"); + for (unsigned int n = 0; n < a->value->listSize(); ++n) { + auto v = a->value->listElems()[n]; + state->forceValue(*v, nix::noPos); + if (v->type() == nix::nString) + namedConstituents.push_back(v->string.s); + } + + return Constituents(constituents, namedConstituents); + } + + return std::nullopt; +} + void worker(nix::ref evaluator, nix::Bindings &autoArgs, nix::AutoCloseFD &to, nix::AutoCloseFD &from, MyArgs &args, nix::AsyncIoRoot &aio) { @@ -128,25 +175,19 @@ void worker(nix::ref evaluator, if (v->type() == nix::nAttrs) { if (auto drvInfo = nix::getDerivation(*state, *v, false)) { - auto drv = Drv(attrPathS, *state, *drvInfo, args); + std::optional maybeConstituents; + if (args.constituents) { + maybeConstituents = + readConstituents(v, state, evaluator); + } + auto drv = Drv(attrPathS, *state, *drvInfo, args, + maybeConstituents); reply.update(drv); /* Register the derivation as a GC root. !!! This registers roots for jobs that we may have already done. */ - if (args.gcRootsDir != "") { - nix::Path root = - args.gcRootsDir + "/" + - std::string(nix::baseNameOf(drv.drvPath)); - if (!nix::pathExists(root)) { - auto localStore = - evaluator->store - .dynamic_pointer_cast(); - auto storePath = - localStore->parseStorePath(drv.drvPath); - aio.blockOn(localStore->addPermRoot(storePath, root)); - } - } + register_gc_root(args.gcRootsDir, drv.drvPath, evaluator->store, aio); } else { auto attrs = nlohmann::json::array(); bool recurse = diff --git a/subprojects/nix-eval-jobs/tests/assets/flake.nix b/subprojects/nix-eval-jobs/tests/assets/flake.nix index abd5faa08..29aedc10b 100644 --- a/subprojects/nix-eval-jobs/tests/assets/flake.nix +++ b/subprojects/nix-eval-jobs/tests/assets/flake.nix @@ -2,9 +2,17 @@ inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable"; outputs = - { nixpkgs, ... }: + { self, nixpkgs, ... }: let + pkgs = nixpkgs.legacyPackages.x86_64-linux; + + mkDrvWithConstituents = + name: constituents: + pkgs.runCommand name { + _hydraAggregate = true; + inherit constituents; + } "touch $out"; in { hydraJobs = import ./ci.nix { inherit pkgs; }; @@ -25,6 +33,42 @@ builder = ":"; }; }; + constituents = { + success = { + indirect_aggregate = mkDrvWithConstituents "indirect_aggregate" [ + "anotherone" + ]; + direct_aggregate = mkDrvWithConstituents "direct_aggregate" [ + self.hydraJobs.builtJob + ]; + mixed_aggregate = mkDrvWithConstituents "mixed_aggregate" [ + self.hydraJobs.builtJob + "anotherone" + ]; + anotherone = pkgs.writeText "constituent" "text"; + }; + failures = { + aggregate = mkDrvWithConstituents "aggregate" [ + "doesntexist" + "doesnteval" + ]; + doesnteval = pkgs.writeText "constituent" (toString { }); + }; + cycle = { + aggregate0 = mkDrvWithConstituents "aggregate0" [ + "aggregate1" + ]; + aggregate1 = mkDrvWithConstituents "aggregate1" [ + "aggregate0" + ]; + }; + transitive = { + constituent = pkgs.hello; + # also flip the order to make sure the toposort works as intended. + aggregate1 = mkDrvWithConstituents "aggregate1" [ "constituent" ]; + aggregate0 = mkDrvWithConstituents "aggregate0" [ "aggregate1" ]; + }; + }; }; }; } diff --git a/subprojects/nix-eval-jobs/tests/test_eval.py b/subprojects/nix-eval-jobs/tests/test_eval.py index 166dc8029..b5dc9d3c6 100755 --- a/subprojects/nix-eval-jobs/tests/test_eval.py +++ b/subprojects/nix-eval-jobs/tests/test_eval.py @@ -140,3 +140,133 @@ def test_recursion_error() -> None: print(stderr) assert "packageWithInfiniteRecursion" in stderr assert "possible infinite recursion" in stderr + + +def test_constituents() -> None: + with TemporaryDirectory() as tempdir: + results, _ = evaluate( + tempdir, + 0, + [ + "--workers", + "1", + "--flake", + ".#legacyPackages.x86_64-linux.constituents.success", + "--constituents", + ], + ) + assert len(results) == 4 + + child = results[0] + assert child["attr"] == "anotherone" + assert "constituents" not in child + assert "namedConstituents" not in child + + direct = results[1] + assert direct["attr"] == "direct_aggregate" + assert "constituents" in direct + assert "namedConstituents" not in direct + + indirect = results[2] + assert indirect["attr"] == "indirect_aggregate" + assert "constituents" in indirect + assert "namedConstituents" not in indirect + + mixed = results[3] + assert mixed["attr"] == "mixed_aggregate" + + def absent_or_empty(f: str, d: dict) -> bool: + return f not in d or len(d[f]) == 0 + + assert absent_or_empty("namedConstituents", direct) + assert absent_or_empty("namedConstituents", indirect) + assert absent_or_empty("namedConstituents", mixed) + + assert direct["constituents"][0].endswith("-job1.drv") + + assert indirect["constituents"][0] == child["drvPath"] + + assert mixed["constituents"][0].endswith("-job1.drv") + assert mixed["constituents"][1] == child["drvPath"] + + assert "error" not in direct + assert "error" not in indirect + assert "error" not in mixed + + check_gc_root(tempdir, direct["drvPath"]) + check_gc_root(tempdir, indirect["drvPath"]) + check_gc_root(tempdir, mixed["drvPath"]) + + +def test_constituents_cycle() -> None: + with TemporaryDirectory() as tempdir: + results, _ = evaluate( + tempdir, + 0, + [ + "--workers", + "1", + "--flake", + ".#legacyPackages.x86_64-linux.constituents.cycle", + "--constituents", + ], + ) + assert len(results) == 2 + + assert list(map(lambda x: x["name"], results)) == ["aggregate0", "aggregate1"] + for i in results: + assert i["error"] == "Dependency cycle: aggregate0 <-> aggregate1" + + +def test_constituents_error() -> None: + with TemporaryDirectory() as tempdir: + results, _ = evaluate( + tempdir, + 0, + [ + "--workers", + "1", + "--flake", + ".#legacyPackages.x86_64-linux.constituents.failures", + "--constituents", + ], + ) + assert len(results) == 2 + + child = results[0] + assert child["attr"] == "doesnteval" + assert "error" in child + + aggregate = results[1] + assert aggregate["attr"] == "aggregate" + assert "namedConstituents" not in aggregate + assert "doesntexist: does not exist\n" in aggregate["error"] + assert "constituents" in aggregate + + +def test_transitivity() -> None: + with TemporaryDirectory() as tempdir: + results, _ = evaluate( + tempdir, + 0, + [ + "--workers", + "1", + "--flake", + ".#legacyPackages.x86_64-linux.constituents.transitive", + "--constituents", + ], + ) + assert len(results) == 3 + + job = results[0] + assert job["attr"] == "constituent" + assert "constituents" not in job + + aggregate1 = results[1] + assert aggregate1["attr"] == "aggregate1" + + aggregate0 = results[2] + assert aggregate0["attr"] == "aggregate0" + + assert aggregate1["drvPath"] == aggregate0["constituents"][0]