From 5ed547605807f4aefb7075680b7db357b5dd5135 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sun, 29 Nov 2020 15:33:55 +0100 Subject: [PATCH 001/419] add project --- default.nix | 31 +++ flake.lock | 42 ++++ flake.nix | 14 ++ meson.build | 13 ++ src/hydra-eval-jobs.cc | 503 +++++++++++++++++++++++++++++++++++++++++ src/meson.build | 15 ++ 6 files changed, 618 insertions(+) create mode 100644 default.nix create mode 100644 flake.lock create mode 100644 flake.nix create mode 100644 meson.build create mode 100644 src/hydra-eval-jobs.cc create mode 100644 src/meson.build diff --git a/default.nix b/default.nix new file mode 100644 index 000000000..05d4c8daf --- /dev/null +++ b/default.nix @@ -0,0 +1,31 @@ +{ stdenv +, nixFlakes +, meson +, cmake +, ninja +, pkg-config +, boost +, nlohmann_json +, srcDir ? ./. +}: + +stdenv.mkDerivation rec { + pname = "hydra-eval-jobs"; + version = "0.0.1"; + src = srcDir; + buildInputs = [ + nlohmann_json nixFlakes boost + ]; + nativeBuildInputs = [ + meson pkg-config ninja + # nlohmann_json can be only discovered via cmake files + cmake + ]; + meta = with stdenv.lib; { + description = "Hydra's builtin hydra-eval-jobs as a standalone"; + homepage = "https://github.com/Mic92/hydra-eval-jobs"; + license = licenses.mit; + maintainers = with maintainers; [ mic92 ]; + platforms = platforms.unix; + }; +} diff --git a/flake.lock b/flake.lock new file mode 100644 index 000000000..234ae528e --- /dev/null +++ b/flake.lock @@ -0,0 +1,42 @@ +{ + "nodes": { + "flake-utils": { + "locked": { + "lastModified": 1605370193, + "narHash": "sha256-YyMTf3URDL/otKdKgtoMChu4vfVL3vCMkRqpGifhUn0=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "5021eac20303a61fafe17224c087f5519baed54d", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1606657618, + "narHash": "sha256-I/sA0wtjqy1JVqHX2HnqdJNqulap+mj8hp/kRreW36o=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "a86b1f48bf373706e5ef50547ceeaeaec9ee7d34", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "flake-utils": "flake-utils", + "nixpkgs": "nixpkgs" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 000000000..10016947a --- /dev/null +++ b/flake.nix @@ -0,0 +1,14 @@ +{ + description = "Hydra's builtin hydra-eval-jobs as a standalone"; + + inputs.nixpkgs.url = "github:NixOS/nixpkgs"; + inputs.flake-utils.url = "github:numtide/flake-utils"; + + outputs = { self, nixpkgs, flake-utils }: + flake-utils.lib.eachDefaultSystem (system: { + packages.hydra-eval-jobs = nixpkgs.legacyPackages.${system}.callPackage ./. { + srcDir = self; + }; + defaultPackage = self.packages.${system}.hydra-eval-jobs; + }); +} diff --git a/meson.build b/meson.build new file mode 100644 index 000000000..c27792529 --- /dev/null +++ b/meson.build @@ -0,0 +1,13 @@ +project('hydra-eval-jobs', 'cpp', + version : '0.1.6', + license : 'GPL-3.0', +) + +nix_main_dep = dependency('nix-main', required: true) +nix_store_dep = dependency('nix-store', required: true) +nix_expr_dep = dependency('nix-expr', required: true) +threads_dep = dependency('threads', required: true) +nlohmann_json_dep = dependency('nlohmann_json', required: true) +boost_dep = dependency('boost', required: true) + +subdir('src') diff --git a/src/hydra-eval-jobs.cc b/src/hydra-eval-jobs.cc new file mode 100644 index 000000000..77233029a --- /dev/null +++ b/src/hydra-eval-jobs.cc @@ -0,0 +1,503 @@ +#include +#include +#include + +#include +#include "shared.hh" +#include "store-api.hh" +#include "eval.hh" +#include "eval-inline.hh" +#include "util.hh" +#include "get-drvs.hh" +#include "globals.hh" +#include "common-eval-args.hh" +#include "flake/flakeref.hh" +#include "flake/flake.hh" +#include "attr-path.hh" +#include "derivations.hh" +#include "local-fs-store.hh" + +#include +#include +#include + +#include + +using namespace nix; + +static Path gcRootsDir; + +struct MyArgs : MixEvalArgs, MixCommonArgs +{ + Path releaseExpr; + bool flake = false; + bool dryRun = false; + size_t nrWorkers = 1; + size_t maxMemorySize = 4096; + + MyArgs() : MixCommonArgs("hydra-eval-jobs") + { + addFlag({ + .longName = "help", + .description = "show usage information", + .handler = {[&]() { + printHelp(programName, std::cout); + throw Exit(); + }} + }); + + addFlag({ + .longName = "gc-roots-dir", + .description = "garbage collector roots directory", + .labels = {"path"}, + .handler = {&gcRootsDir} + }); + + addFlag({ + .longName = "workers", + .description = "number of evaluate workers", + .labels = {"workers"}, + .handler = {[=](std::string s) { + nrWorkers = std::stoi(s); + }} + }); + + addFlag({ + .longName = "max-memory-size", + .description = "maximum evaluation memory size", + .labels = {"size"}, + .handler = {[=](std::string s) { + maxMemorySize = std::stoi(s); + }} + }); + + addFlag({ + .longName = "dry-run", + .description = "don't create store derivations", + .handler = {&dryRun, true} + }); + + addFlag({ + .longName = "flake", + .description = "build a flake", + .handler = {&flake, true} + }); + + expectArg("expr", &releaseExpr); + } +}; + +static MyArgs myArgs; + +static std::string queryMetaStrings(EvalState & state, DrvInfo & drv, const string & name, const string & subAttribute) +{ + Strings res; + std::function rec; + + rec = [&](Value & v) { + state.forceValue(v); + if (v.type == tString) + res.push_back(v.string.s); + else if (v.isList()) + for (unsigned int n = 0; n < v.listSize(); ++n) + rec(*v.listElems()[n]); + else if (v.type == tAttrs) { + auto a = v.attrs->find(state.symbols.create(subAttribute)); + if (a != v.attrs->end()) + res.push_back(state.forceString(*a->value)); + } + }; + + Value * v = drv.queryMeta(name); + if (v) rec(*v); + + return concatStringsSep(", ", res); +} + +static void worker( + EvalState & state, + Bindings & autoArgs, + AutoCloseFD & to, + AutoCloseFD & from) +{ + Value vTop; + + if (myArgs.flake) { + using namespace flake; + + auto flakeRef = parseFlakeRef(myArgs.releaseExpr); + + auto vFlake = state.allocValue(); + + auto lockedFlake = lockFlake(state, flakeRef, + LockFlags { + .updateLockFile = false, + .useRegistries = false, + .allowMutable = false, + }); + + callFlake(state, lockedFlake, *vFlake); + + auto vOutputs = vFlake->attrs->get(state.symbols.create("outputs"))->value; + state.forceValue(*vOutputs); + + auto aHydraJobs = vOutputs->attrs->get(state.symbols.create("hydraJobs")); + if (!aHydraJobs) + aHydraJobs = vOutputs->attrs->get(state.symbols.create("checks")); + if (!aHydraJobs) + throw Error("flake '%s' does not provide any Hydra jobs or checks", flakeRef); + + vTop = *aHydraJobs->value; + + } else { + state.evalFile(lookupFileArg(state, myArgs.releaseExpr), vTop); + } + + auto vRoot = state.allocValue(); + state.autoCallFunction(autoArgs, vTop, *vRoot); + + while (true) { + /* Wait for the master to send us a job name. */ + writeLine(to.get(), "next"); + + auto s = readLine(from.get()); + if (s == "exit") break; + if (!hasPrefix(s, "do ")) abort(); + std::string attrPath(s, 3); + + debug("worker process %d at '%s'", getpid(), attrPath); + + /* Evaluate it and send info back to the master. */ + nlohmann::json reply; + + try { + auto vTmp = findAlongAttrPath(state, attrPath, autoArgs, *vRoot).first; + + auto v = state.allocValue(); + state.autoCallFunction(autoArgs, *vTmp, *v); + + if (auto drv = getDerivation(state, *v, false)) { + + DrvInfo::Outputs outputs = drv->queryOutputs(); + + if (drv->querySystem() == "unknown") + throw EvalError("derivation must have a 'system' attribute"); + + auto drvPath = drv->queryDrvPath(); + + nlohmann::json job; + + job["nixName"] = drv->queryName(); + job["system"] =drv->querySystem(); + job["drvPath"] = drvPath; + job["description"] = drv->queryMetaString("description"); + job["license"] = queryMetaStrings(state, *drv, "license", "shortName"); + job["homepage"] = drv->queryMetaString("homepage"); + job["maintainers"] = queryMetaStrings(state, *drv, "maintainers", "email"); + job["schedulingPriority"] = drv->queryMetaInt("schedulingPriority", 100); + job["timeout"] = drv->queryMetaInt("timeout", 36000); + job["maxSilent"] = drv->queryMetaInt("maxSilent", 7200); + job["isChannel"] = drv->queryMetaBool("isHydraChannel", false); + + /* If this is an aggregate, then get its constituents. */ + auto a = v->attrs->get(state.symbols.create("_hydraAggregate")); + if (a && state.forceBool(*a->value, *a->pos)) { + auto a = v->attrs->get(state.symbols.create("constituents")); + if (!a) + throw EvalError("derivation must have a ‘constituents’ attribute"); + + + PathSet context; + state.coerceToString(*a->pos, *a->value, context, true, false); + for (auto & i : context) + if (i.at(0) == '!') { + size_t index = i.find("!", 1); + job["constituents"].push_back(string(i, index + 1)); + } + + state.forceList(*a->value, *a->pos); + for (unsigned int n = 0; n < a->value->listSize(); ++n) { + auto v = a->value->listElems()[n]; + state.forceValue(*v); + if (v->type == tString) + job["namedConstituents"].push_back(state.forceStringNoCtx(*v)); + } + } + + /* Register the derivation as a GC root. !!! This + registers roots for jobs that we may have already + done. */ + auto localStore = state.store.dynamic_pointer_cast(); + if (gcRootsDir != "" && localStore) { + Path root = gcRootsDir + "/" + std::string(baseNameOf(drvPath)); + if (!pathExists(root)) + localStore->addPermRoot(localStore->parseStorePath(drvPath), root); + } + + nlohmann::json out; + for (auto & j : outputs) + out[j.first] = j.second; + job["outputs"] = std::move(out); + + reply["job"] = std::move(job); + } + + else if (v->type == tAttrs) { + auto attrs = nlohmann::json::array(); + StringSet ss; + for (auto & i : v->attrs->lexicographicOrder()) { + std::string name(i->name); + if (name.find('.') != std::string::npos || name.find(' ') != std::string::npos) { + printError("skipping job with illegal name '%s'", name); + continue; + } + attrs.push_back(name); + } + reply["attrs"] = std::move(attrs); + } + + else if (v->type == tNull) + ; + + else throw TypeError("attribute '%s' is %s, which is not supported", attrPath, showType(*v)); + + } catch (EvalError & e) { + // Transmits the error we got from the previous evaluation + // in the JSON output. + reply["error"] = filterANSIEscapes(e.msg(), true); + // Don't forget to print it into the STDERR log, this is + // what's shown in the Hydra UI. + printError("error: %s", reply["error"]); + } + + writeLine(to.get(), reply.dump()); + + /* If our RSS exceeds the maximum, exit. The master will + start a new process. */ + struct rusage r; + getrusage(RUSAGE_SELF, &r); + if ((size_t) r.ru_maxrss > myArgs.maxMemorySize * 1024) break; + } + + writeLine(to.get(), "restart"); +} + +int main(int argc, char * * argv) +{ + /* Prevent undeclared dependencies in the evaluation via + $NIX_PATH. */ + unsetenv("NIX_PATH"); + + return handleExceptions(argv[0], [&]() { + initNix(); + initGC(); + + myArgs.parseCmdline(argvToStrings(argc, argv)); + + /* FIXME: The build hook in conjunction with import-from-derivation is causing "unexpected EOF" during eval */ + settings.builders = ""; + + /* Prevent access to paths outside of the Nix search path and + to the environment. */ + evalSettings.restrictEval = true; + + /* When building a flake, use pure evaluation (no access to + 'getEnv', 'currentSystem' etc. */ + evalSettings.pureEval = myArgs.flake; + + if (myArgs.dryRun) settings.readOnlyMode = true; + + if (myArgs.releaseExpr == "") throw UsageError("no expression specified"); + + if (gcRootsDir == "") printMsg(lvlError, "warning: `--gc-roots-dir' not specified"); + + struct State + { + std::set todo{""}; + std::set active; + nlohmann::json jobs; + std::exception_ptr exc; + }; + + std::condition_variable wakeup; + + Sync state_; + + /* Start a handler thread per worker process. */ + auto handler = [&]() + { + try { + pid_t pid = -1; + AutoCloseFD from, to; + + while (true) { + + /* Start a new worker process if necessary. */ + if (pid == -1) { + Pipe toPipe, fromPipe; + toPipe.create(); + fromPipe.create(); + pid = startProcess( + [&, + to{std::make_shared(std::move(fromPipe.writeSide))}, + from{std::make_shared(std::move(toPipe.readSide))} + ]() + { + try { + EvalState state(myArgs.searchPath, openStore()); + Bindings & autoArgs = *myArgs.getAutoArgs(state); + worker(state, autoArgs, *to, *from); + } catch (std::exception & e) { + nlohmann::json err; + err["error"] = e.what(); + writeLine(to->get(), err.dump()); + // Don't forget to print it into the STDERR log, this is + // what's shown in the Hydra UI. + printError("error: %s", err["error"]); + } + }, + ProcessOptions { .allowVfork = false }); + from = std::move(fromPipe.readSide); + to = std::move(toPipe.writeSide); + debug("created worker process %d", pid); + } + + /* Check whether the existing worker process is still there. */ + auto s = readLine(from.get()); + if (s == "restart") { + pid = -1; + continue; + } else if (s != "next") { + auto json = nlohmann::json::parse(s); + throw Error("worker error: %s", (std::string) json["error"]); + } + + /* Wait for a job name to become available. */ + std::string attrPath; + + while (true) { + checkInterrupt(); + auto state(state_.lock()); + if ((state->todo.empty() && state->active.empty()) || state->exc) { + writeLine(to.get(), "exit"); + return; + } + if (!state->todo.empty()) { + attrPath = *state->todo.begin(); + state->todo.erase(state->todo.begin()); + state->active.insert(attrPath); + break; + } else + state.wait(wakeup); + } + + /* Tell the worker to evaluate it. */ + writeLine(to.get(), "do " + attrPath); + + /* Wait for the response. */ + auto response = nlohmann::json::parse(readLine(from.get())); + + /* Handle the response. */ + StringSet newAttrs; + + if (response.find("job") != response.end()) { + auto state(state_.lock()); + state->jobs[attrPath] = response["job"]; + } + + if (response.find("attrs") != response.end()) { + for (auto & i : response["attrs"]) { + auto s = (attrPath.empty() ? "" : attrPath + ".") + (std::string) i; + newAttrs.insert(s); + } + } + + if (response.find("error") != response.end()) { + auto state(state_.lock()); + state->jobs[attrPath]["error"] = response["error"]; + } + + /* Add newly discovered job names to the queue. */ + { + auto state(state_.lock()); + state->active.erase(attrPath); + for (auto & s : newAttrs) + state->todo.insert(s); + wakeup.notify_all(); + } + } + } catch (...) { + auto state(state_.lock()); + state->exc = std::current_exception(); + wakeup.notify_all(); + } + }; + + std::vector threads; + for (size_t i = 0; i < myArgs.nrWorkers; i++) + threads.emplace_back(std::thread(handler)); + + for (auto & thread : threads) + thread.join(); + + auto state(state_.lock()); + + if (state->exc) + std::rethrow_exception(state->exc); + + /* For aggregate jobs that have named consistuents + (i.e. constituents that are a job name rather than a + derivation), look up the referenced job and add it to the + dependencies of the aggregate derivation. */ + auto store = openStore(); + + for (auto i = state->jobs.begin(); i != state->jobs.end(); ++i) { + auto jobName = i.key(); + auto & job = i.value(); + + auto named = job.find("namedConstituents"); + if (named == job.end()) continue; + + if (myArgs.dryRun) { + for (std::string jobName2 : *named) { + auto job2 = state->jobs.find(jobName2); + if (job2 == state->jobs.end()) + throw Error("aggregate job '%s' references non-existent job '%s'", jobName, jobName2); + std::string drvPath2 = (*job2)["drvPath"]; + job["constituents"].push_back(drvPath2); + } + } else { + auto drvPath = store->parseStorePath((std::string) job["drvPath"]); + auto drv = store->readDerivation(drvPath); + + for (std::string jobName2 : *named) { + auto job2 = state->jobs.find(jobName2); + if (job2 == state->jobs.end()) + throw Error("aggregate job '%s' references non-existent job '%s'", jobName, jobName2); + auto drvPath2 = store->parseStorePath((std::string) (*job2)["drvPath"]); + auto drv2 = store->readDerivation(drvPath2); + job["constituents"].push_back(store->printStorePath(drvPath2)); + drv.inputDrvs[drvPath2] = {drv2.outputs.begin()->first}; + } + + std::string drvName(drvPath.name()); + assert(hasSuffix(drvName, drvExtension)); + drvName.resize(drvName.size() - drvExtension.size()); + auto h = std::get(hashDerivationModulo(*store, drv, true)); + auto outPath = store->makeOutputPath("out", h, drvName); + drv.env["out"] = store->printStorePath(outPath); + drv.outputs.insert_or_assign("out", DerivationOutput { .output = DerivationOutputInputAddressed { .path = outPath } }); + auto newDrvPath = store->printStorePath(writeDerivation(*store, drv)); + + debug("rewrote aggregate derivation %s -> %s", store->printStorePath(drvPath), newDrvPath); + + job["drvPath"] = newDrvPath; + job["outputs"]["out"] = store->printStorePath(outPath); + } + + job.erase("namedConstituents"); + } + + std::cout << state->jobs.dump(2) << "\n"; + }); +} diff --git a/src/meson.build b/src/meson.build new file mode 100644 index 000000000..50603b417 --- /dev/null +++ b/src/meson.build @@ -0,0 +1,15 @@ +src = [ + 'hydra-eval-jobs.cc', +] + +executable('hydra-eval-jobs', src, + dependencies : [ + nix_main_dep, + nix_store_dep, + nix_expr_dep, + boost_dep, + nlohmann_json_dep, + threads_dep + ], + install: true, + cpp_args: ['-std=c++17', '-fvisibility=hidden']) From e2a3c951791c4b9b3efe732e12881e2e5f0df178 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sun, 29 Nov 2020 21:32:04 +0100 Subject: [PATCH 002/419] prefix header files with nix/ --- src/hydra-eval-jobs.cc | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/hydra-eval-jobs.cc b/src/hydra-eval-jobs.cc index 77233029a..a6d032f76 100644 --- a/src/hydra-eval-jobs.cc +++ b/src/hydra-eval-jobs.cc @@ -3,19 +3,19 @@ #include #include -#include "shared.hh" -#include "store-api.hh" -#include "eval.hh" -#include "eval-inline.hh" -#include "util.hh" -#include "get-drvs.hh" -#include "globals.hh" -#include "common-eval-args.hh" -#include "flake/flakeref.hh" -#include "flake/flake.hh" -#include "attr-path.hh" -#include "derivations.hh" -#include "local-fs-store.hh" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include #include From 9205107bd6a30c8a6f0fc06c5d8c1ed121b50e33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sun, 14 Mar 2021 23:16:11 +0100 Subject: [PATCH 003/419] add gitignore --- .gitignore | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..a6f420b16 --- /dev/null +++ b/.gitignore @@ -0,0 +1,42 @@ +.DS_Store +.idea +*.log + +tmp/ + + +# Prerequisites +*.d + +# Compiled Object files +*.slo +*.lo +*.o +*.obj + +# Precompiled Headers +*.gch +*.pch + +# Compiled Dynamic libraries +*.so +*.dylib +*.dll + +# Fortran module files +*.mod +*.smod + +# Compiled Static libraries +*.lai +*.la +*.a +*.lib + +# Executables +*.exe +*.out +*.app + +# build directory +/build From 0f22a976e529e554564057fa9e45e228eb59d402 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sun, 14 Mar 2021 23:16:46 +0100 Subject: [PATCH 004/419] update nixpkgs --- flake.lock | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/flake.lock b/flake.lock index 234ae528e..633cd6300 100644 --- a/flake.lock +++ b/flake.lock @@ -2,11 +2,11 @@ "nodes": { "flake-utils": { "locked": { - "lastModified": 1605370193, - "narHash": "sha256-YyMTf3URDL/otKdKgtoMChu4vfVL3vCMkRqpGifhUn0=", + "lastModified": 1614513358, + "narHash": "sha256-LakhOx3S1dRjnh0b5Dg3mbZyH0ToC9I8Y2wKSkBaTzU=", "owner": "numtide", "repo": "flake-utils", - "rev": "5021eac20303a61fafe17224c087f5519baed54d", + "rev": "5466c5bbece17adaab2d82fae80b46e807611bf3", "type": "github" }, "original": { @@ -17,11 +17,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1606657618, - "narHash": "sha256-I/sA0wtjqy1JVqHX2HnqdJNqulap+mj8hp/kRreW36o=", + "lastModified": 1615756511, + "narHash": "sha256-rcuG5eYBtDlN0yTJdFw8GktML7Og1hFctp44wE3MHp4=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "a86b1f48bf373706e5ef50547ceeaeaec9ee7d34", + "rev": "87acb6d60bfe9469475dbaa80df90971aaa21997", "type": "github" }, "original": { From ac566dea95bb3fdcb0d034bfbc96471ba1845f70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sun, 14 Mar 2021 23:16:57 +0100 Subject: [PATCH 005/419] update from upstream --- src/hydra-eval-jobs.cc | 37 ++++++++++++++++++++++++------------- 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/src/hydra-eval-jobs.cc b/src/hydra-eval-jobs.cc index a6d032f76..e88531fab 100644 --- a/src/hydra-eval-jobs.cc +++ b/src/hydra-eval-jobs.cc @@ -1,6 +1,8 @@ #include #include +#include #include +#include #include #include @@ -41,9 +43,15 @@ struct MyArgs : MixEvalArgs, MixCommonArgs .longName = "help", .description = "show usage information", .handler = {[&]() { - printHelp(programName, std::cout); - throw Exit(); - }} + std::cout << "Usage:" << std::endl; + std::cout << " " << "hydra-eval-jobs [options] expr" << std::endl; + for (const auto & [name, flag] : longFlags) { + if (hiddenCategories.count(flag->category)) { + continue; + } + printf(" --%-20s %s\n", name.c_str(), flag->description.c_str()); + } + }}, }); addFlag({ @@ -96,12 +104,12 @@ static std::string queryMetaStrings(EvalState & state, DrvInfo & drv, const stri rec = [&](Value & v) { state.forceValue(v); - if (v.type == tString) + if (v.type() == nString) res.push_back(v.string.s); else if (v.isList()) for (unsigned int n = 0; n < v.listSize(); ++n) rec(*v.listElems()[n]); - else if (v.type == tAttrs) { + else if (v.type() == nAttrs) { auto a = v.attrs->find(state.symbols.create(subAttribute)); if (a != v.attrs->end()) res.push_back(state.forceString(*a->value)); @@ -219,7 +227,7 @@ static void worker( for (unsigned int n = 0; n < a->value->listSize(); ++n) { auto v = a->value->listElems()[n]; state.forceValue(*v); - if (v->type == tString) + if (v->type() == nString) job["namedConstituents"].push_back(state.forceStringNoCtx(*v)); } } @@ -242,7 +250,7 @@ static void worker( reply["job"] = std::move(job); } - else if (v->type == tAttrs) { + else if (v->type() == nAttrs) { auto attrs = nlohmann::json::array(); StringSet ss; for (auto & i : v->attrs->lexicographicOrder()) { @@ -256,18 +264,19 @@ static void worker( reply["attrs"] = std::move(attrs); } - else if (v->type == tNull) + else if (v->type() == nNull) ; else throw TypeError("attribute '%s' is %s, which is not supported", attrPath, showType(*v)); } catch (EvalError & e) { + auto msg = e.msg(); // Transmits the error we got from the previous evaluation // in the JSON output. - reply["error"] = filterANSIEscapes(e.msg(), true); + reply["error"] = filterANSIEscapes(msg, true); // Don't forget to print it into the STDERR log, this is // what's shown in the Hydra UI. - printError("error: %s", reply["error"]); + printError(msg); } writeLine(to.get(), reply.dump()); @@ -347,13 +356,15 @@ int main(int argc, char * * argv) EvalState state(myArgs.searchPath, openStore()); Bindings & autoArgs = *myArgs.getAutoArgs(state); worker(state, autoArgs, *to, *from); - } catch (std::exception & e) { + } catch (Error & e) { nlohmann::json err; - err["error"] = e.what(); + auto msg = e.msg(); + err["error"] = filterANSIEscapes(msg, true); + printError(msg); writeLine(to->get(), err.dump()); // Don't forget to print it into the STDERR log, this is // what's shown in the Hydra UI. - printError("error: %s", err["error"]); + writeLine(to->get(), "restart"); } }, ProcessOptions { .allowVfork = false }); From 034f7804213c5d271f5b47b766a59ddf95ebe623 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sun, 14 Mar 2021 23:19:18 +0100 Subject: [PATCH 006/419] remove unused headers --- src/hydra-eval-jobs.cc | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/hydra-eval-jobs.cc b/src/hydra-eval-jobs.cc index e88531fab..5fec1d7b4 100644 --- a/src/hydra-eval-jobs.cc +++ b/src/hydra-eval-jobs.cc @@ -1,8 +1,6 @@ #include #include -#include #include -#include #include #include From 0e1be9fdaf0d85dc61c2fb9f414871c54fee29af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sun, 14 Mar 2021 23:29:49 +0100 Subject: [PATCH 007/419] hydra-eval-jobs: smaller visual improvements --- src/hydra-eval-jobs.cc | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/hydra-eval-jobs.cc b/src/hydra-eval-jobs.cc index 5fec1d7b4..86d7ac4cc 100644 --- a/src/hydra-eval-jobs.cc +++ b/src/hydra-eval-jobs.cc @@ -41,13 +41,12 @@ struct MyArgs : MixEvalArgs, MixCommonArgs .longName = "help", .description = "show usage information", .handler = {[&]() { - std::cout << "Usage:" << std::endl; - std::cout << " " << "hydra-eval-jobs [options] expr" << std::endl; + printf("USAGE: hydra-eval-jobs [options] expr\n\n"); for (const auto & [name, flag] : longFlags) { if (hiddenCategories.count(flag->category)) { continue; } - printf(" --%-20s %s\n", name.c_str(), flag->description.c_str()); + printf(" --%-20s %s\n", name.c_str(), flag->description.c_str()); } }}, }); From 7e41287c1bf3259d02430c9216918836286d828f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sun, 14 Mar 2021 23:32:36 +0100 Subject: [PATCH 008/419] disable restrictEval again --- src/hydra-eval-jobs.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hydra-eval-jobs.cc b/src/hydra-eval-jobs.cc index 86d7ac4cc..e4bb8c42b 100644 --- a/src/hydra-eval-jobs.cc +++ b/src/hydra-eval-jobs.cc @@ -305,7 +305,7 @@ int main(int argc, char * * argv) /* Prevent access to paths outside of the Nix search path and to the environment. */ - evalSettings.restrictEval = true; + evalSettings.restrictEval = false; /* When building a flake, use pure evaluation (no access to 'getEnv', 'currentSystem' etc. */ From ad8bab8694c669f1f930cb6f4b5090b8b60b68cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Mon, 15 Mar 2021 04:55:07 +0000 Subject: [PATCH 009/419] Create README.md --- README.md | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 000000000..51ee96f35 --- /dev/null +++ b/README.md @@ -0,0 +1,2 @@ +# hydra-eval-jobs +Eval nix expressions from flakes (extracted from hydra) From 6afe31d79b3ea155bb144e796e08f7effb3003d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Mon, 15 Mar 2021 05:57:05 +0100 Subject: [PATCH 010/419] add LICENSE --- LICENSE.md | 597 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 597 insertions(+) create mode 100644 LICENSE.md diff --git a/LICENSE.md b/LICENSE.md new file mode 100644 index 000000000..def709e12 --- /dev/null +++ b/LICENSE.md @@ -0,0 +1,597 @@ +GNU General Public License +========================== + +_Version 3, 29 June 2007_ +_Copyright © 2007 Free Software Foundation, Inc. <>_ + +Everyone is permitted to copy and distribute verbatim copies of this license +document, but changing it is not allowed. + +## Preamble + +The GNU General Public License is a free, copyleft license for software and other +kinds of works. + +The licenses for most software and other practical works are designed to take away +your freedom to share and change the works. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change all versions of a +program--to make sure it remains free software for all its users. We, the Free +Software Foundation, use the GNU General Public License for most of our software; it +applies also to any other work released this way by its authors. You can apply it to +your programs, too. + +When we speak of free software, we are referring to freedom, not price. Our General +Public Licenses are designed to make sure that you have the freedom to distribute +copies of free software (and charge for them if you wish), that you receive source +code or can get it if you want it, that you can change the software or use pieces of +it in new free programs, and that you know you can do these things. + +To protect your rights, we need to prevent others from denying you these rights or +asking you to surrender the rights. Therefore, you have certain responsibilities if +you distribute copies of the software, or if you modify it: responsibilities to +respect the freedom of others. + +For example, if you distribute copies of such a program, whether gratis or for a fee, +you must pass on to the recipients the same freedoms that you received. You must make +sure that they, too, receive or can get the source code. And you must show them these +terms so they know their rights. + +Developers that use the GNU GPL protect your rights with two steps: **(1)** assert +copyright on the software, and **(2)** offer you this License giving you legal permission +to copy, distribute and/or modify it. + +For the developers' and authors' protection, the GPL clearly explains that there is +no warranty for this free software. For both users' and authors' sake, the GPL +requires that modified versions be marked as changed, so that their problems will not +be attributed erroneously to authors of previous versions. + +Some devices are designed to deny users access to install or run modified versions of +the software inside them, although the manufacturer can do so. This is fundamentally +incompatible with the aim of protecting users' freedom to change the software. The +systematic pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we have designed +this version of the GPL to prohibit the practice for those products. If such problems +arise substantially in other domains, we stand ready to extend this provision to +those domains in future versions of the GPL, as needed to protect the freedom of +users. + +Finally, every program is threatened constantly by software patents. States should +not allow patents to restrict development and use of software on general-purpose +computers, but in those that do, we wish to avoid the special danger that patents +applied to a free program could make it effectively proprietary. To prevent this, the +GPL assures that patents cannot be used to render the program non-free. + +The precise terms and conditions for copying, distribution and modification follow. + +## TERMS AND CONDITIONS + +### 0. Definitions + +“This License” refers to version 3 of the GNU General Public License. + +“Copyright” also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + +“The Program” refers to any copyrightable work licensed under this +License. Each licensee is addressed as “you”. “Licensees” and +“recipients” may be individuals or organizations. + +To “modify” a work means to copy from or adapt all or part of the work in +a fashion requiring copyright permission, other than the making of an exact copy. The +resulting work is called a “modified version” of the earlier work or a +work “based on” the earlier work. + +A “covered work” means either the unmodified Program or a work based on +the Program. + +To “propagate” a work means to do anything with it that, without +permission, would make you directly or secondarily liable for infringement under +applicable copyright law, except executing it on a computer or modifying a private +copy. Propagation includes copying, distribution (with or without modification), +making available to the public, and in some countries other activities as well. + +To “convey” a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through a computer +network, with no transfer of a copy, is not conveying. + +An interactive user interface displays “Appropriate Legal Notices” to the +extent that it includes a convenient and prominently visible feature that **(1)** +displays an appropriate copyright notice, and **(2)** tells the user that there is no +warranty for the work (except to the extent that warranties are provided), that +licensees may convey the work under this License, and how to view a copy of this +License. If the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + +### 1. Source Code + +The “source code” for a work means the preferred form of the work for +making modifications to it. “Object code” means any non-source form of a +work. + +A “Standard Interface” means an interface that either is an official +standard defined by a recognized standards body, or, in the case of interfaces +specified for a particular programming language, one that is widely used among +developers working in that language. + +The “System Libraries” of an executable work include anything, other than +the work as a whole, that **(a)** is included in the normal form of packaging a Major +Component, but which is not part of that Major Component, and **(b)** serves only to +enable use of the work with that Major Component, or to implement a Standard +Interface for which an implementation is available to the public in source code form. +A “Major Component”, in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system (if any) on which +the executable work runs, or a compiler used to produce the work, or an object code +interpreter used to run it. + +The “Corresponding Source” for a work in object code form means all the +source code needed to generate, install, and (for an executable work) run the object +code and to modify the work, including scripts to control those activities. However, +it does not include the work's System Libraries, or general-purpose tools or +generally available free programs which are used unmodified in performing those +activities but which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for the work, and +the source code for shared libraries and dynamically linked subprograms that the work +is specifically designed to require, such as by intimate data communication or +control flow between those subprograms and other parts of the work. + +The Corresponding Source need not include anything that users can regenerate +automatically from other parts of the Corresponding Source. + +The Corresponding Source for a work in source code form is that same work. + +### 2. Basic Permissions + +All rights granted under this License are granted for the term of copyright on the +Program, and are irrevocable provided the stated conditions are met. This License +explicitly affirms your unlimited permission to run the unmodified Program. The +output from running a covered work is covered by this License only if the output, +given its content, constitutes a covered work. This License acknowledges your rights +of fair use or other equivalent, as provided by copyright law. + +You may make, run and propagate covered works that you do not convey, without +conditions so long as your license otherwise remains in force. You may convey covered +works to others for the sole purpose of having them make modifications exclusively +for you, or provide you with facilities for running those works, provided that you +comply with the terms of this License in conveying all material for which you do not +control copyright. Those thus making or running the covered works for you must do so +exclusively on your behalf, under your direction and control, on terms that prohibit +them from making any copies of your copyrighted material outside their relationship +with you. + +Conveying under any other circumstances is permitted solely under the conditions +stated below. Sublicensing is not allowed; section 10 makes it unnecessary. + +### 3. Protecting Users' Legal Rights From Anti-Circumvention Law + +No covered work shall be deemed part of an effective technological measure under any +applicable law fulfilling obligations under article 11 of the WIPO copyright treaty +adopted on 20 December 1996, or similar laws prohibiting or restricting circumvention +of such measures. + +When you convey a covered work, you waive any legal power to forbid circumvention of +technological measures to the extent such circumvention is effected by exercising +rights under this License with respect to the covered work, and you disclaim any +intention to limit operation or modification of the work as a means of enforcing, +against the work's users, your or third parties' legal rights to forbid circumvention +of technological measures. + +### 4. Conveying Verbatim Copies + +You may convey verbatim copies of the Program's source code as you receive it, in any +medium, provided that you conspicuously and appropriately publish on each copy an +appropriate copyright notice; keep intact all notices stating that this License and +any non-permissive terms added in accord with section 7 apply to the code; keep +intact all notices of the absence of any warranty; and give all recipients a copy of +this License along with the Program. + +You may charge any price or no price for each copy that you convey, and you may offer +support or warranty protection for a fee. + +### 5. Conveying Modified Source Versions + +You may convey a work based on the Program, or the modifications to produce it from +the Program, in the form of source code under the terms of section 4, provided that +you also meet all of these conditions: + +* **a)** The work must carry prominent notices stating that you modified it, and giving a +relevant date. +* **b)** The work must carry prominent notices stating that it is released under this +License and any conditions added under section 7. This requirement modifies the +requirement in section 4 to “keep intact all notices”. +* **c)** You must license the entire work, as a whole, under this License to anyone who +comes into possession of a copy. This License will therefore apply, along with any +applicable section 7 additional terms, to the whole of the work, and all its parts, +regardless of how they are packaged. This License gives no permission to license the +work in any other way, but it does not invalidate such permission if you have +separately received it. +* **d)** If the work has interactive user interfaces, each must display Appropriate Legal +Notices; however, if the Program has interactive interfaces that do not display +Appropriate Legal Notices, your work need not make them do so. + +A compilation of a covered work with other separate and independent works, which are +not by their nature extensions of the covered work, and which are not combined with +it such as to form a larger program, in or on a volume of a storage or distribution +medium, is called an “aggregate” if the compilation and its resulting +copyright are not used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work in an aggregate +does not cause this License to apply to the other parts of the aggregate. + +### 6. Conveying Non-Source Forms + +You may convey a covered work in object code form under the terms of sections 4 and +5, provided that you also convey the machine-readable Corresponding Source under the +terms of this License, in one of these ways: + +* **a)** Convey the object code in, or embodied in, a physical product (including a +physical distribution medium), accompanied by the Corresponding Source fixed on a +durable physical medium customarily used for software interchange. +* **b)** Convey the object code in, or embodied in, a physical product (including a +physical distribution medium), accompanied by a written offer, valid for at least +three years and valid for as long as you offer spare parts or customer support for +that product model, to give anyone who possesses the object code either **(1)** a copy of +the Corresponding Source for all the software in the product that is covered by this +License, on a durable physical medium customarily used for software interchange, for +a price no more than your reasonable cost of physically performing this conveying of +source, or **(2)** access to copy the Corresponding Source from a network server at no +charge. +* **c)** Convey individual copies of the object code with a copy of the written offer to +provide the Corresponding Source. This alternative is allowed only occasionally and +noncommercially, and only if you received the object code with such an offer, in +accord with subsection 6b. +* **d)** Convey the object code by offering access from a designated place (gratis or for +a charge), and offer equivalent access to the Corresponding Source in the same way +through the same place at no further charge. You need not require recipients to copy +the Corresponding Source along with the object code. If the place to copy the object +code is a network server, the Corresponding Source may be on a different server +(operated by you or a third party) that supports equivalent copying facilities, +provided you maintain clear directions next to the object code saying where to find +the Corresponding Source. Regardless of what server hosts the Corresponding Source, +you remain obligated to ensure that it is available for as long as needed to satisfy +these requirements. +* **e)** Convey the object code using peer-to-peer transmission, provided you inform +other peers where the object code and Corresponding Source of the work are being +offered to the general public at no charge under subsection 6d. + +A separable portion of the object code, whose source code is excluded from the +Corresponding Source as a System Library, need not be included in conveying the +object code work. + +A “User Product” is either **(1)** a “consumer product”, which +means any tangible personal property which is normally used for personal, family, or +household purposes, or **(2)** anything designed or sold for incorporation into a +dwelling. In determining whether a product is a consumer product, doubtful cases +shall be resolved in favor of coverage. For a particular product received by a +particular user, “normally used” refers to a typical or common use of +that class of product, regardless of the status of the particular user or of the way +in which the particular user actually uses, or expects or is expected to use, the +product. A product is a consumer product regardless of whether the product has +substantial commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + +“Installation Information” for a User Product means any methods, +procedures, authorization keys, or other information required to install and execute +modified versions of a covered work in that User Product from a modified version of +its Corresponding Source. The information must suffice to ensure that the continued +functioning of the modified object code is in no case prevented or interfered with +solely because modification has been made. + +If you convey an object code work under this section in, or with, or specifically for +use in, a User Product, and the conveying occurs as part of a transaction in which +the right of possession and use of the User Product is transferred to the recipient +in perpetuity or for a fixed term (regardless of how the transaction is +characterized), the Corresponding Source conveyed under this section must be +accompanied by the Installation Information. But this requirement does not apply if +neither you nor any third party retains the ability to install modified object code +on the User Product (for example, the work has been installed in ROM). + +The requirement to provide Installation Information does not include a requirement to +continue to provide support service, warranty, or updates for a work that has been +modified or installed by the recipient, or for the User Product in which it has been +modified or installed. Access to a network may be denied when the modification itself +materially and adversely affects the operation of the network or violates the rules +and protocols for communication across the network. + +Corresponding Source conveyed, and Installation Information provided, in accord with +this section must be in a format that is publicly documented (and with an +implementation available to the public in source code form), and must require no +special password or key for unpacking, reading or copying. + +### 7. Additional Terms + +“Additional permissions” are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. Additional +permissions that are applicable to the entire Program shall be treated as though they +were included in this License, to the extent that they are valid under applicable +law. If additional permissions apply only to part of the Program, that part may be +used separately under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + +When you convey a copy of a covered work, you may at your option remove any +additional permissions from that copy, or from any part of it. (Additional +permissions may be written to require their own removal in certain cases when you +modify the work.) You may place additional permissions on material, added by you to a +covered work, for which you have or can give appropriate copyright permission. + +Notwithstanding any other provision of this License, for material you add to a +covered work, you may (if authorized by the copyright holders of that material) +supplement the terms of this License with terms: + +* **a)** Disclaiming warranty or limiting liability differently from the terms of +sections 15 and 16 of this License; or +* **b)** Requiring preservation of specified reasonable legal notices or author +attributions in that material or in the Appropriate Legal Notices displayed by works +containing it; or +* **c)** Prohibiting misrepresentation of the origin of that material, or requiring that +modified versions of such material be marked in reasonable ways as different from the +original version; or +* **d)** Limiting the use for publicity purposes of names of licensors or authors of the +material; or +* **e)** Declining to grant rights under trademark law for use of some trade names, +trademarks, or service marks; or +* **f)** Requiring indemnification of licensors and authors of that material by anyone +who conveys the material (or modified versions of it) with contractual assumptions of +liability to the recipient, for any liability that these contractual assumptions +directly impose on those licensors and authors. + +All other non-permissive additional terms are considered “further +restrictions” within the meaning of section 10. If the Program as you received +it, or any part of it, contains a notice stating that it is governed by this License +along with a term that is a further restriction, you may remove that term. If a +license document contains a further restriction but permits relicensing or conveying +under this License, you may add to a covered work material governed by the terms of +that license document, provided that the further restriction does not survive such +relicensing or conveying. + +If you add terms to a covered work in accord with this section, you must place, in +the relevant source files, a statement of the additional terms that apply to those +files, or a notice indicating where to find the applicable terms. + +Additional terms, permissive or non-permissive, may be stated in the form of a +separately written license, or stated as exceptions; the above requirements apply +either way. + +### 8. Termination + +You may not propagate or modify a covered work except as expressly provided under +this License. Any attempt otherwise to propagate or modify it is void, and will +automatically terminate your rights under this License (including any patent licenses +granted under the third paragraph of section 11). + +However, if you cease all violation of this License, then your license from a +particular copyright holder is reinstated **(a)** provisionally, unless and until the +copyright holder explicitly and finally terminates your license, and **(b)** permanently, +if the copyright holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + +Moreover, your license from a particular copyright holder is reinstated permanently +if the copyright holder notifies you of the violation by some reasonable means, this +is the first time you have received notice of violation of this License (for any +work) from that copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + +Termination of your rights under this section does not terminate the licenses of +parties who have received copies or rights from you under this License. If your +rights have been terminated and not permanently reinstated, you do not qualify to +receive new licenses for the same material under section 10. + +### 9. Acceptance Not Required for Having Copies + +You are not required to accept this License in order to receive or run a copy of the +Program. Ancillary propagation of a covered work occurring solely as a consequence of +using peer-to-peer transmission to receive a copy likewise does not require +acceptance. However, nothing other than this License grants you permission to +propagate or modify any covered work. These actions infringe copyright if you do not +accept this License. Therefore, by modifying or propagating a covered work, you +indicate your acceptance of this License to do so. + +### 10. Automatic Licensing of Downstream Recipients + +Each time you convey a covered work, the recipient automatically receives a license +from the original licensors, to run, modify and propagate that work, subject to this +License. You are not responsible for enforcing compliance by third parties with this +License. + +An “entity transaction” is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an organization, or +merging organizations. If propagation of a covered work results from an entity +transaction, each party to that transaction who receives a copy of the work also +receives whatever licenses to the work the party's predecessor in interest had or +could give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if the predecessor +has it or can get it with reasonable efforts. + +You may not impose any further restrictions on the exercise of the rights granted or +affirmed under this License. For example, you may not impose a license fee, royalty, +or other charge for exercise of rights granted under this License, and you may not +initiate litigation (including a cross-claim or counterclaim in a lawsuit) alleging +that any patent claim is infringed by making, using, selling, offering for sale, or +importing the Program or any portion of it. + +### 11. Patents + +A “contributor” is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The work thus +licensed is called the contributor's “contributor version”. + +A contributor's “essential patent claims” are all patent claims owned or +controlled by the contributor, whether already acquired or hereafter acquired, that +would be infringed by some manner, permitted by this License, of making, using, or +selling its contributor version, but do not include claims that would be infringed +only as a consequence of further modification of the contributor version. For +purposes of this definition, “control” includes the right to grant patent +sublicenses in a manner consistent with the requirements of this License. + +Each contributor grants you a non-exclusive, worldwide, royalty-free patent license +under the contributor's essential patent claims, to make, use, sell, offer for sale, +import and otherwise run, modify and propagate the contents of its contributor +version. + +In the following three paragraphs, a “patent license” is any express +agreement or commitment, however denominated, not to enforce a patent (such as an +express permission to practice a patent or covenant not to sue for patent +infringement). To “grant” such a patent license to a party means to make +such an agreement or commitment not to enforce a patent against the party. + +If you convey a covered work, knowingly relying on a patent license, and the +Corresponding Source of the work is not available for anyone to copy, free of charge +and under the terms of this License, through a publicly available network server or +other readily accessible means, then you must either **(1)** cause the Corresponding +Source to be so available, or **(2)** arrange to deprive yourself of the benefit of the +patent license for this particular work, or **(3)** arrange, in a manner consistent with +the requirements of this License, to extend the patent license to downstream +recipients. “Knowingly relying” means you have actual knowledge that, but +for the patent license, your conveying the covered work in a country, or your +recipient's use of the covered work in a country, would infringe one or more +identifiable patents in that country that you have reason to believe are valid. + +If, pursuant to or in connection with a single transaction or arrangement, you +convey, or propagate by procuring conveyance of, a covered work, and grant a patent +license to some of the parties receiving the covered work authorizing them to use, +propagate, modify or convey a specific copy of the covered work, then the patent +license you grant is automatically extended to all recipients of the covered work and +works based on it. + +A patent license is “discriminatory” if it does not include within the +scope of its coverage, prohibits the exercise of, or is conditioned on the +non-exercise of one or more of the rights that are specifically granted under this +License. You may not convey a covered work if you are a party to an arrangement with +a third party that is in the business of distributing software, under which you make +payment to the third party based on the extent of your activity of conveying the +work, and under which the third party grants, to any of the parties who would receive +the covered work from you, a discriminatory patent license **(a)** in connection with +copies of the covered work conveyed by you (or copies made from those copies), or **(b)** +primarily for and in connection with specific products or compilations that contain +the covered work, unless you entered into that arrangement, or that patent license +was granted, prior to 28 March 2007. + +Nothing in this License shall be construed as excluding or limiting any implied +license or other defenses to infringement that may otherwise be available to you +under applicable patent law. + +### 12. No Surrender of Others' Freedom + +If conditions are imposed on you (whether by court order, agreement or otherwise) +that contradict the conditions of this License, they do not excuse you from the +conditions of this License. If you cannot convey a covered work so as to satisfy +simultaneously your obligations under this License and any other pertinent +obligations, then as a consequence you may not convey it at all. For example, if you +agree to terms that obligate you to collect a royalty for further conveying from +those to whom you convey the Program, the only way you could satisfy both those terms +and this License would be to refrain entirely from conveying the Program. + +### 13. Use with the GNU Affero General Public License + +Notwithstanding any other provision of this License, you have permission to link or +combine any covered work with a work licensed under version 3 of the GNU Affero +General Public License into a single combined work, and to convey the resulting work. +The terms of this License will continue to apply to the part which is the covered +work, but the special requirements of the GNU Affero General Public License, section +13, concerning interaction through a network will apply to the combination as such. + +### 14. Revised Versions of this License + +The Free Software Foundation may publish revised and/or new versions of the GNU +General Public License from time to time. Such new versions will be similar in spirit +to the present version, but may differ in detail to address new problems or concerns. + +Each version is given a distinguishing version number. If the Program specifies that +a certain numbered version of the GNU General Public License “or any later +version” applies to it, you have the option of following the terms and +conditions either of that numbered version or of any later version published by the +Free Software Foundation. If the Program does not specify a version number of the GNU +General Public License, you may choose any version ever published by the Free +Software Foundation. + +If the Program specifies that a proxy can decide which future versions of the GNU +General Public License can be used, that proxy's public statement of acceptance of a +version permanently authorizes you to choose that version for the Program. + +Later license versions may give you additional or different permissions. However, no +additional obligations are imposed on any author or copyright holder as a result of +your choosing to follow a later version. + +### 15. Disclaimer of Warranty + +THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. +EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM “AS IS” WITHOUT WARRANTY OF ANY KIND, EITHER +EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE +QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE +DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + +### 16. Limitation of Liability + +IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL ANY +COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS THE PROGRAM AS +PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, +INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE +PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE +OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE +WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + +### 17. Interpretation of Sections 15 and 16 + +If the disclaimer of warranty and limitation of liability provided above cannot be +given local legal effect according to their terms, reviewing courts shall apply local +law that most closely approximates an absolute waiver of all civil liability in +connection with the Program, unless a warranty or assumption of liability accompanies +a copy of the Program in return for a fee. + +_END OF TERMS AND CONDITIONS_ + +## How to Apply These Terms to Your New Programs + +If you develop a new program, and you want it to be of the greatest possible use to +the public, the best way to achieve this is to make it free software which everyone +can redistribute and change under these terms. + +To do so, attach the following notices to the program. It is safest to attach them +to the start of each source file to most effectively state the exclusion of warranty; +and each file should have at least the “copyright” line and a pointer to +where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + +If the program does terminal interaction, make it output a short notice like this +when it starts in an interactive mode: + + Copyright (C) + This program comes with ABSOLUTELY NO WARRANTY; for details type 'show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type 'show c' for details. + +The hypothetical commands `show w` and `show c` should show the appropriate parts of +the General Public License. Of course, your program's commands might be different; +for a GUI interface, you would use an “about box”. + +You should also get your employer (if you work as a programmer) or school, if any, to +sign a “copyright disclaimer” for the program, if necessary. For more +information on this, and how to apply and follow the GNU GPL, see +<>. + +The GNU General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may consider it +more useful to permit linking proprietary applications with the library. If this is +what you want to do, use the GNU Lesser General Public License instead of this +License. But first, please read +<>. + + From 25c46dc1b0f167347200e2e9dd1c683f5c1cb0e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Mon, 15 Mar 2021 06:02:14 +0100 Subject: [PATCH 011/419] fix evaluating relative flake urls --- src/hydra-eval-jobs.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hydra-eval-jobs.cc b/src/hydra-eval-jobs.cc index e4bb8c42b..944476e31 100644 --- a/src/hydra-eval-jobs.cc +++ b/src/hydra-eval-jobs.cc @@ -130,7 +130,7 @@ static void worker( if (myArgs.flake) { using namespace flake; - auto flakeRef = parseFlakeRef(myArgs.releaseExpr); + auto flakeRef = parseFlakeRef(myArgs.releaseExpr, absPath(".")); auto vFlake = state.allocValue(); From 32cb862f987a61c87cc6a32d1437031941875144 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Mon, 15 Mar 2021 06:09:41 +0100 Subject: [PATCH 012/419] add ci configuration --- .github/dependabot.yml | 6 ++++++ .github/workflows/test.yml | 20 ++++++++++++++++++++ default.nix | 33 ++------------------------------- hydra.nix | 35 +++++++++++++++++++++++++++++++++++ 4 files changed, 63 insertions(+), 31 deletions(-) create mode 100644 .github/dependabot.yml create mode 100644 .github/workflows/test.yml create mode 100644 hydra.nix diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 000000000..5ace4600a --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,6 @@ +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 000000000..2549b24d0 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,20 @@ +name: "Test" +on: + pull_request: + push: + schedule: + - cron: '51 2 * * *' +jobs: + tests: + strategy: + matrix: + nixPath: + - nixpkgs=https://github.com/NixOS/nixpkgs/archive/nixos-20.09.tar.gz + - nixpkgs=https://github.com/NixOS/nixpkgs/archive/nixpkgs-unstable.tar.gz + os: [ ubuntu-latest, macos-latest ] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v2 + - uses: cachix/install-nix-action@v12 + - name: build + run: NIX_PATH=nixpkgs=https://github.com/NixOS/nixpkgs/archive/nixpkgs-unstable.tar.gz nix-build diff --git a/default.nix b/default.nix index 05d4c8daf..b2286a660 100644 --- a/default.nix +++ b/default.nix @@ -1,31 +1,2 @@ -{ stdenv -, nixFlakes -, meson -, cmake -, ninja -, pkg-config -, boost -, nlohmann_json -, srcDir ? ./. -}: - -stdenv.mkDerivation rec { - pname = "hydra-eval-jobs"; - version = "0.0.1"; - src = srcDir; - buildInputs = [ - nlohmann_json nixFlakes boost - ]; - nativeBuildInputs = [ - meson pkg-config ninja - # nlohmann_json can be only discovered via cmake files - cmake - ]; - meta = with stdenv.lib; { - description = "Hydra's builtin hydra-eval-jobs as a standalone"; - homepage = "https://github.com/Mic92/hydra-eval-jobs"; - license = licenses.mit; - maintainers = with maintainers; [ mic92 ]; - platforms = platforms.unix; - }; -} +{ pkgs ? import {} }: +pkgs.callPackage ./hydra.nix {} diff --git a/hydra.nix b/hydra.nix new file mode 100644 index 000000000..36c3228da --- /dev/null +++ b/hydra.nix @@ -0,0 +1,35 @@ +{ stdenv +, nixFlakes +, meson +, cmake +, ninja +, pkg-config +, boost +, nlohmann_json +, srcDir ? ./. +}: + +let + filterMesonBuild = dir: builtins.filterSource + (path: type: type != "directory" || baseNameOf path != "build") dir; +in +stdenv.mkDerivation rec { + pname = "hydra-eval-jobs"; + version = "0.0.1"; + src = filterMesonBuild srcDir; + buildInputs = [ + nlohmann_json nixFlakes boost + ]; + nativeBuildInputs = [ + meson pkg-config ninja + # nlohmann_json can be only discovered via cmake files + cmake + ]; + meta = with stdenv.lib; { + description = "Hydra's builtin hydra-eval-jobs as a standalone"; + homepage = "https://github.com/Mic92/hydra-eval-jobs"; + license = licenses.mit; + maintainers = with maintainers; [ mic92 ]; + platforms = platforms.unix; + }; +} From 7745ffbab0456c46a49acb8dd2750af6051c9bf8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Mon, 15 Mar 2021 06:09:41 +0100 Subject: [PATCH 013/419] add ci configuration --- .github/workflows/test.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2549b24d0..31f709840 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -2,6 +2,8 @@ name: "Test" on: pull_request: push: + branches: + - main schedule: - cron: '51 2 * * *' jobs: From 620300b1fb5c6a8b0bf2a1e03995e8b66dd9610e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Mon, 15 Mar 2021 06:19:54 +0100 Subject: [PATCH 014/419] add flake to ci --- .github/workflows/test-flakes.yml | 29 +++++++++++++++++++++++++++++ .gitignore | 2 ++ flake.nix | 2 +- hydra.nix | 4 ++-- 4 files changed, 34 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/test-flakes.yml diff --git a/.github/workflows/test-flakes.yml b/.github/workflows/test-flakes.yml new file mode 100644 index 000000000..a241baef4 --- /dev/null +++ b/.github/workflows/test-flakes.yml @@ -0,0 +1,29 @@ +name: "Flake test" +on: + pull_request: + push: + branches: + - main + schedule: + - cron: '51 2 * * *' +jobs: + tests: + strategy: + matrix: + os: [ ubuntu-latest, macos-latest ] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v2 + with: + # Nix Flakes doesn't work on shallow clones + fetch-depth: 0 + - uses: cachix/install-nix-action@v12 + with: + install_url: https://github.com/numtide/nix-flakes-installer/releases/download/nix-2.4pre20210207_fd6eaa1/install + extra_nix_config: | + experimental-features = nix-command flakes + system-features = nixos-test benchmark big-parallel kvm + - name: List flake structure + run: nix flake show + - name: Build + run: nix build diff --git a/.gitignore b/.gitignore index a6f420b16..4ec0ffddd 100644 --- a/.gitignore +++ b/.gitignore @@ -40,3 +40,5 @@ tmp/ # build directory /build +# nix-build +/result diff --git a/flake.nix b/flake.nix index 10016947a..10b15c4f2 100644 --- a/flake.nix +++ b/flake.nix @@ -6,7 +6,7 @@ outputs = { self, nixpkgs, flake-utils }: flake-utils.lib.eachDefaultSystem (system: { - packages.hydra-eval-jobs = nixpkgs.legacyPackages.${system}.callPackage ./. { + packages.hydra-eval-jobs = nixpkgs.legacyPackages.${system}.callPackage ./hydra.nix { srcDir = self; }; defaultPackage = self.packages.${system}.hydra-eval-jobs; diff --git a/hydra.nix b/hydra.nix index 36c3228da..dc94dddd9 100644 --- a/hydra.nix +++ b/hydra.nix @@ -6,7 +6,7 @@ , pkg-config , boost , nlohmann_json -, srcDir ? ./. +, srcDir ? null }: let @@ -16,7 +16,7 @@ in stdenv.mkDerivation rec { pname = "hydra-eval-jobs"; version = "0.0.1"; - src = filterMesonBuild srcDir; + src = if srcDir == null then filterMesonBuild ./. else srcDir; buildInputs = [ nlohmann_json nixFlakes boost ]; From 2d58d8a513bc6c02bd81697e87d985cad1ddcd94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sun, 21 Mar 2021 18:35:16 +0100 Subject: [PATCH 015/419] gitignore python --- .gitignore | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.gitignore b/.gitignore index 4ec0ffddd..250b5c443 100644 --- a/.gitignore +++ b/.gitignore @@ -42,3 +42,13 @@ tmp/ /build # nix-build /result + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json From b48e9667dcce3127f8cbdb703b6491c72eb8272e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sun, 21 Mar 2021 18:37:15 +0100 Subject: [PATCH 016/419] make gcRootsDir local just to avoid unecessary globals --- src/hydra-eval-jobs.cc | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/hydra-eval-jobs.cc b/src/hydra-eval-jobs.cc index 944476e31..3582f9f6f 100644 --- a/src/hydra-eval-jobs.cc +++ b/src/hydra-eval-jobs.cc @@ -25,11 +25,11 @@ using namespace nix; -static Path gcRootsDir; struct MyArgs : MixEvalArgs, MixCommonArgs { Path releaseExpr; + Path gcRootsDir; bool flake = false; bool dryRun = false; size_t nrWorkers = 1; @@ -123,7 +123,8 @@ static void worker( EvalState & state, Bindings & autoArgs, AutoCloseFD & to, - AutoCloseFD & from) + AutoCloseFD & from, + const Path &gcRootsDir) { Value vTop; @@ -315,7 +316,7 @@ int main(int argc, char * * argv) if (myArgs.releaseExpr == "") throw UsageError("no expression specified"); - if (gcRootsDir == "") printMsg(lvlError, "warning: `--gc-roots-dir' not specified"); + if (myArgs.gcRootsDir == "") printMsg(lvlError, "warning: `--gc-roots-dir' not specified"); struct State { @@ -352,7 +353,7 @@ int main(int argc, char * * argv) try { EvalState state(myArgs.searchPath, openStore()); Bindings & autoArgs = *myArgs.getAutoArgs(state); - worker(state, autoArgs, *to, *from); + worker(state, autoArgs, *to, *from, myArgs.gcRootsDir); } catch (Error & e) { nlohmann::json err; auto msg = e.msg(); From f00607a7c6cf5b523ccb300a92f3924ae3c9e139 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sun, 21 Mar 2021 18:38:56 +0100 Subject: [PATCH 017/419] exit hydra-eval-jobs after printing help --- src/hydra-eval-jobs.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/hydra-eval-jobs.cc b/src/hydra-eval-jobs.cc index 3582f9f6f..2f964736e 100644 --- a/src/hydra-eval-jobs.cc +++ b/src/hydra-eval-jobs.cc @@ -48,6 +48,7 @@ struct MyArgs : MixEvalArgs, MixCommonArgs } printf(" --%-20s %s\n", name.c_str(), flag->description.c_str()); } + ::exit(0); }}, }); From 617d4ee15138840c468d7ac4fefe21bef740d8bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sun, 21 Mar 2021 18:39:32 +0100 Subject: [PATCH 018/419] add --impure flag for flakes --- src/hydra-eval-jobs.cc | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/hydra-eval-jobs.cc b/src/hydra-eval-jobs.cc index 2f964736e..5a0979090 100644 --- a/src/hydra-eval-jobs.cc +++ b/src/hydra-eval-jobs.cc @@ -25,6 +25,7 @@ using namespace nix; +typedef enum { evalAuto, evalImpure, evalPure } pureEval; struct MyArgs : MixEvalArgs, MixCommonArgs { @@ -34,6 +35,7 @@ struct MyArgs : MixEvalArgs, MixCommonArgs bool dryRun = false; size_t nrWorkers = 1; size_t maxMemorySize = 4096; + pureEval evalMode = evalAuto; MyArgs() : MixCommonArgs("hydra-eval-jobs") { @@ -52,6 +54,13 @@ struct MyArgs : MixEvalArgs, MixCommonArgs }}, }); + addFlag({ + .longName = "impure", + .description = "set evaluation mode", + .handler = {[&]() { + evalMode = evalImpure; + }}, + }); addFlag({ .longName = "gc-roots-dir", .description = "garbage collector roots directory", @@ -311,7 +320,7 @@ int main(int argc, char * * argv) /* When building a flake, use pure evaluation (no access to 'getEnv', 'currentSystem' etc. */ - evalSettings.pureEval = myArgs.flake; + evalSettings.pureEval = myArgs.evalMode == evalAuto ? myArgs.flake : myArgs.evalMode == evalPure; if (myArgs.dryRun) settings.readOnlyMode = true; From 71cbe4eab47cf5790b7ca9b9eb3fbc180c23866f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sun, 21 Mar 2021 19:05:11 +0100 Subject: [PATCH 019/419] output missing/substitutable derivations in eval output --- flake.nix | 21 +++++--- src/hydra-eval-jobs.cc | 105 ++++++++++++++++++++++------------------ tests/assets/ci.nix | 5 ++ tests/assets/flake.lock | 26 ++++++++++ tests/assets/flake.nix | 12 +++++ tests/test_eval.py | 34 +++++++++++++ 6 files changed, 151 insertions(+), 52 deletions(-) create mode 100644 tests/assets/ci.nix create mode 100644 tests/assets/flake.lock create mode 100644 tests/assets/flake.nix create mode 100644 tests/test_eval.py diff --git a/flake.nix b/flake.nix index 10b15c4f2..998772dbe 100644 --- a/flake.nix +++ b/flake.nix @@ -5,10 +5,19 @@ inputs.flake-utils.url = "github:numtide/flake-utils"; outputs = { self, nixpkgs, flake-utils }: - flake-utils.lib.eachDefaultSystem (system: { - packages.hydra-eval-jobs = nixpkgs.legacyPackages.${system}.callPackage ./hydra.nix { - srcDir = self; - }; - defaultPackage = self.packages.${system}.hydra-eval-jobs; - }); + flake-utils.lib.eachDefaultSystem (system: + let + pkgs = nixpkgs.legacyPackages.${system}; + in + rec { + packages.hydra-eval-jobs = pkgs.callPackage ./hydra.nix { + srcDir = self; + }; + defaultPackage = self.packages.${system}.hydra-eval-jobs; + devShell = defaultPackage.overrideAttrs (old: { + nativeBuildInputs = old.nativeBuildInputs ++ [ + pkgs.python3.pkgs.pytest + ]; + }); + }); } diff --git a/src/hydra-eval-jobs.cc b/src/hydra-eval-jobs.cc index 5a0979090..bcfecb75e 100644 --- a/src/hydra-eval-jobs.cc +++ b/src/hydra-eval-jobs.cc @@ -32,7 +32,6 @@ struct MyArgs : MixEvalArgs, MixCommonArgs Path releaseExpr; Path gcRootsDir; bool flake = false; - bool dryRun = false; size_t nrWorkers = 1; size_t maxMemorySize = 4096; pureEval evalMode = evalAuto; @@ -61,6 +60,7 @@ struct MyArgs : MixEvalArgs, MixCommonArgs evalMode = evalImpure; }}, }); + addFlag({ .longName = "gc-roots-dir", .description = "garbage collector roots directory", @@ -86,12 +86,6 @@ struct MyArgs : MixEvalArgs, MixCommonArgs }} }); - addFlag({ - .longName = "dry-run", - .description = "don't create store derivations", - .handler = {&dryRun, true} - }); - addFlag({ .longName = "flake", .description = "build a flake", @@ -129,6 +123,14 @@ static std::string queryMetaStrings(EvalState & state, DrvInfo & drv, const stri return concatStringsSep(", ", res); } +static nlohmann::json serializeStorePathSet(StorePathSet &paths, LocalFSStore &store) { + auto array = nlohmann::json::array(); + for (auto & p : paths) { + array.push_back(store.printStorePath(p)); + } + return array; +} + static void worker( EvalState & state, Bindings & autoArgs, @@ -244,17 +246,40 @@ static void worker( registers roots for jobs that we may have already done. */ auto localStore = state.store.dynamic_pointer_cast(); + auto storePath = localStore->parseStorePath(drvPath); if (gcRootsDir != "" && localStore) { Path root = gcRootsDir + "/" + std::string(baseNameOf(drvPath)); if (!pathExists(root)) - localStore->addPermRoot(localStore->parseStorePath(drvPath), root); + localStore->addPermRoot(storePath, root); } + uint64_t downloadSize, narSize; + StorePathSet willBuild, willSubstitute, unknown; + std::vector paths; + StringSet outputNames; + + for (auto & output : outputs) { + outputNames.insert(output.first); + } + paths.push_back({storePath, outputNames}); + + localStore->queryMissing(paths, + willBuild, + willSubstitute, + unknown, + downloadSize, + narSize); + nlohmann::json out; - for (auto & j : outputs) - out[j.first] = j.second; + for (auto & p : outputs) { + out[p.first] = p.second; + } job["outputs"] = std::move(out); + job["builds"] = serializeStorePathSet(willBuild, *localStore); + job["substitutes"] = serializeStorePathSet(willSubstitute, *localStore); + job["unknown"] = serializeStorePathSet(unknown, *localStore); + reply["job"] = std::move(job); } @@ -322,8 +347,6 @@ int main(int argc, char * * argv) 'getEnv', 'currentSystem' etc. */ evalSettings.pureEval = myArgs.evalMode == evalAuto ? myArgs.flake : myArgs.evalMode == evalPure; - if (myArgs.dryRun) settings.readOnlyMode = true; - if (myArgs.releaseExpr == "") throw UsageError("no expression specified"); if (myArgs.gcRootsDir == "") printMsg(lvlError, "warning: `--gc-roots-dir' not specified"); @@ -477,43 +500,33 @@ int main(int argc, char * * argv) auto named = job.find("namedConstituents"); if (named == job.end()) continue; - if (myArgs.dryRun) { - for (std::string jobName2 : *named) { - auto job2 = state->jobs.find(jobName2); - if (job2 == state->jobs.end()) - throw Error("aggregate job '%s' references non-existent job '%s'", jobName, jobName2); - std::string drvPath2 = (*job2)["drvPath"]; - job["constituents"].push_back(drvPath2); - } - } else { - auto drvPath = store->parseStorePath((std::string) job["drvPath"]); - auto drv = store->readDerivation(drvPath); + auto drvPath = store->parseStorePath((std::string) job["drvPath"]); + auto drv = store->readDerivation(drvPath); - for (std::string jobName2 : *named) { - auto job2 = state->jobs.find(jobName2); - if (job2 == state->jobs.end()) - throw Error("aggregate job '%s' references non-existent job '%s'", jobName, jobName2); - auto drvPath2 = store->parseStorePath((std::string) (*job2)["drvPath"]); - auto drv2 = store->readDerivation(drvPath2); - job["constituents"].push_back(store->printStorePath(drvPath2)); - drv.inputDrvs[drvPath2] = {drv2.outputs.begin()->first}; - } - - std::string drvName(drvPath.name()); - assert(hasSuffix(drvName, drvExtension)); - drvName.resize(drvName.size() - drvExtension.size()); - auto h = std::get(hashDerivationModulo(*store, drv, true)); - auto outPath = store->makeOutputPath("out", h, drvName); - drv.env["out"] = store->printStorePath(outPath); - drv.outputs.insert_or_assign("out", DerivationOutput { .output = DerivationOutputInputAddressed { .path = outPath } }); - auto newDrvPath = store->printStorePath(writeDerivation(*store, drv)); - - debug("rewrote aggregate derivation %s -> %s", store->printStorePath(drvPath), newDrvPath); - - job["drvPath"] = newDrvPath; - job["outputs"]["out"] = store->printStorePath(outPath); + for (std::string jobName2 : *named) { + auto job2 = state->jobs.find(jobName2); + if (job2 == state->jobs.end()) + throw Error("aggregate job '%s' references non-existent job '%s'", jobName, jobName2); + auto drvPath2 = store->parseStorePath((std::string) (*job2)["drvPath"]); + auto drv2 = store->readDerivation(drvPath2); + job["constituents"].push_back(store->printStorePath(drvPath2)); + drv.inputDrvs[drvPath2] = {drv2.outputs.begin()->first}; } + std::string drvName(drvPath.name()); + assert(hasSuffix(drvName, drvExtension)); + drvName.resize(drvName.size() - drvExtension.size()); + auto h = std::get(hashDerivationModulo(*store, drv, true)); + auto outPath = store->makeOutputPath("out", h, drvName); + drv.env["out"] = store->printStorePath(outPath); + drv.outputs.insert_or_assign("out", DerivationOutput { .output = DerivationOutputInputAddressed { .path = outPath } }); + auto newDrvPath = store->printStorePath(writeDerivation(*store, drv)); + + debug("rewrote aggregate derivation %s -> %s", store->printStorePath(drvPath), newDrvPath); + + job["drvPath"] = newDrvPath; + job["outputs"]["out"] = store->printStorePath(outPath); + job.erase("namedConstituents"); } diff --git a/tests/assets/ci.nix b/tests/assets/ci.nix new file mode 100644 index 000000000..5e80362e4 --- /dev/null +++ b/tests/assets/ci.nix @@ -0,0 +1,5 @@ +with import {}; +{ + builtJob = pkgs.writeText "job1" "job1"; + substitutedJob = pkgs.hello; +} diff --git a/tests/assets/flake.lock b/tests/assets/flake.lock new file mode 100644 index 000000000..a1a80bf7c --- /dev/null +++ b/tests/assets/flake.lock @@ -0,0 +1,26 @@ +{ + "nodes": { + "nixpkgs": { + "locked": { + "lastModified": 1616345250, + "narHash": "sha256-WLbLFIJyKCklGyEMGwh9XDTzafafyO95s4+rJHOc/Ag=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "5e4a4e0c32f0ca0a5bd4ebbbf17aedd347de7f3e", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "nixpkgs": "nixpkgs" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/tests/assets/flake.nix b/tests/assets/flake.nix new file mode 100644 index 000000000..bbd81a6bf --- /dev/null +++ b/tests/assets/flake.nix @@ -0,0 +1,12 @@ +{ + inputs.nixpkgs.url = "github:NixOS/nixpkgs"; + + outputs = { self, nixpkgs }: let + pkgs = nixpkgs.legacyPackages.x86_64-linux; + in { + hydraJobs = { + builtJob = pkgs.writeText "job1" "job1"; + substitutedJob = pkgs.hello; + }; + }; +} diff --git a/tests/test_eval.py b/tests/test_eval.py new file mode 100644 index 000000000..02eced9df --- /dev/null +++ b/tests/test_eval.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python3 + +import subprocess +import json +from tempfile import TemporaryDirectory +from pathlib import Path +from typing import List + +TEST_ROOT = Path(__file__).parent.resolve() +PROJECT_ROOT = TEST_ROOT.parent +BIN = PROJECT_ROOT.joinpath("build", "src", "hydra-eval-jobs") + + +def common_test(extra_args: List[str]) -> None: + with TemporaryDirectory() as tempdir: + cmd = [str(BIN), "--gc-roots-dir", tempdir] + extra_args + res = subprocess.run( + cmd, + cwd=TEST_ROOT.joinpath("assets"), + text=True, + check=True, + stdout=subprocess.PIPE, + ) + data = json.loads(res.stdout) + assert len(data["builtJob"]["builds"]) == 1 + assert len(data["substitutedJob"]["substitutes"]) >= 1 + + +def test_flake() -> None: + common_test(["--flake", ".#"]) + + +def test_expression() -> None: + common_test(["ci.nix"]) From 76c3e68ccd369c0a788d6a779012d75248ffe96c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sun, 21 Mar 2021 19:18:03 +0100 Subject: [PATCH 020/419] run tests in CI --- .github/workflows/test-flakes.yml | 6 +++++- tests/assets/ci.nix | 4 +++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-flakes.yml b/.github/workflows/test-flakes.yml index a241baef4..b400fc060 100644 --- a/.github/workflows/test-flakes.yml +++ b/.github/workflows/test-flakes.yml @@ -26,4 +26,8 @@ jobs: - name: List flake structure run: nix flake show - name: Build - run: nix build + run: nix build --out-link result + - name: Run tests + run: | + nix develop -c install -D ./result/bin/hydra-eval-jobs ./build/src/hydra-eval-jobs + nix develop -c pytest ./tests diff --git a/tests/assets/ci.nix b/tests/assets/ci.nix index 5e80362e4..48bf1774f 100644 --- a/tests/assets/ci.nix +++ b/tests/assets/ci.nix @@ -1,4 +1,6 @@ -with import {}; +let + pkgs = import (builtins.getFlake (toString ./.)).inputs.nixpkgs {}; +in { builtJob = pkgs.writeText "job1" "job1"; substitutedJob = pkgs.hello; From 4af1e548e972ba56aa45b5623561486631edaddb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 5 Apr 2021 07:49:05 +0000 Subject: [PATCH 021/419] Bump cachix/install-nix-action from v12 to v13 Bumps [cachix/install-nix-action](https://github.com/cachix/install-nix-action) from v12 to v13. - [Release notes](https://github.com/cachix/install-nix-action/releases) - [Commits](https://github.com/cachix/install-nix-action/compare/v12...8d6d5e949675fbadb765c6b1a975047fa5f09b27) Signed-off-by: dependabot[bot] --- .github/workflows/test-flakes.yml | 2 +- .github/workflows/test.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-flakes.yml b/.github/workflows/test-flakes.yml index b400fc060..067a97e7d 100644 --- a/.github/workflows/test-flakes.yml +++ b/.github/workflows/test-flakes.yml @@ -17,7 +17,7 @@ jobs: with: # Nix Flakes doesn't work on shallow clones fetch-depth: 0 - - uses: cachix/install-nix-action@v12 + - uses: cachix/install-nix-action@v13 with: install_url: https://github.com/numtide/nix-flakes-installer/releases/download/nix-2.4pre20210207_fd6eaa1/install extra_nix_config: | diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 31f709840..3ce90cbc0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -17,6 +17,6 @@ jobs: runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v2 - - uses: cachix/install-nix-action@v12 + - uses: cachix/install-nix-action@v13 - name: build run: NIX_PATH=nixpkgs=https://github.com/NixOS/nixpkgs/archive/nixpkgs-unstable.tar.gz nix-build From b92dfbba572ca433276fc9319fb1319111cc5ae3 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Mon, 23 Aug 2021 12:20:16 -0500 Subject: [PATCH 022/419] Use python3.withPackages to pull in pytest Using the other method leaks into PYTHONPATH. --- flake.nix | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index 998772dbe..bf87bd66b 100644 --- a/flake.nix +++ b/flake.nix @@ -16,7 +16,9 @@ defaultPackage = self.packages.${system}.hydra-eval-jobs; devShell = defaultPackage.overrideAttrs (old: { nativeBuildInputs = old.nativeBuildInputs ++ [ - pkgs.python3.pkgs.pytest + (pkgs.python3.withPackages(ps: [ + ps.pytest + ])) ]; }); }); From 19823c899d42a6c26b89770e01768edcacf3329e Mon Sep 17 00:00:00 2001 From: adisbladis Date: Mon, 23 Aug 2021 18:29:04 -0500 Subject: [PATCH 023/419] Add full meta to output While the current output may be sufficient for Hydra it's not enough for a more generically useful evaluator. --- src/hydra-eval-jobs.cc | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/hydra-eval-jobs.cc b/src/hydra-eval-jobs.cc index bcfecb75e..c2fdc23fc 100644 --- a/src/hydra-eval-jobs.cc +++ b/src/hydra-eval-jobs.cc @@ -17,6 +17,8 @@ #include #include +#include + #include #include #include @@ -217,6 +219,16 @@ static void worker( job["maxSilent"] = drv->queryMetaInt("maxSilent", 7200); job["isChannel"] = drv->queryMetaBool("isHydraChannel", false); + nlohmann::json meta; + for (auto & name : drv->queryMetaNames()) { + PathSet context; + std::stringstream ss; + printValueAsJSON(state, true, *drv->queryMeta(name), ss, context); + nlohmann::json field = nlohmann::json::parse(ss.str()); + meta[name] = field; + } + job["meta"] = meta; + /* If this is an aggregate, then get its constituents. */ auto a = v->attrs->get(state.symbols.create("_hydraAggregate")); if (a && state.forceBool(*a->value, *a->pos)) { From 1774f874ee82e184eefc73c470108fe3a37eafc2 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Mon, 23 Aug 2021 18:31:24 -0500 Subject: [PATCH 024/419] Remove superflous meta fields from output Since meta is now included in it's entirety we no longer need to output them separately. --- src/hydra-eval-jobs.cc | 33 --------------------------------- 1 file changed, 33 deletions(-) diff --git a/src/hydra-eval-jobs.cc b/src/hydra-eval-jobs.cc index c2fdc23fc..8d509ff11 100644 --- a/src/hydra-eval-jobs.cc +++ b/src/hydra-eval-jobs.cc @@ -100,31 +100,6 @@ struct MyArgs : MixEvalArgs, MixCommonArgs static MyArgs myArgs; -static std::string queryMetaStrings(EvalState & state, DrvInfo & drv, const string & name, const string & subAttribute) -{ - Strings res; - std::function rec; - - rec = [&](Value & v) { - state.forceValue(v); - if (v.type() == nString) - res.push_back(v.string.s); - else if (v.isList()) - for (unsigned int n = 0; n < v.listSize(); ++n) - rec(*v.listElems()[n]); - else if (v.type() == nAttrs) { - auto a = v.attrs->find(state.symbols.create(subAttribute)); - if (a != v.attrs->end()) - res.push_back(state.forceString(*a->value)); - } - }; - - Value * v = drv.queryMeta(name); - if (v) rec(*v); - - return concatStringsSep(", ", res); -} - static nlohmann::json serializeStorePathSet(StorePathSet &paths, LocalFSStore &store) { auto array = nlohmann::json::array(); for (auto & p : paths) { @@ -210,14 +185,6 @@ static void worker( job["nixName"] = drv->queryName(); job["system"] =drv->querySystem(); job["drvPath"] = drvPath; - job["description"] = drv->queryMetaString("description"); - job["license"] = queryMetaStrings(state, *drv, "license", "shortName"); - job["homepage"] = drv->queryMetaString("homepage"); - job["maintainers"] = queryMetaStrings(state, *drv, "maintainers", "email"); - job["schedulingPriority"] = drv->queryMetaInt("schedulingPriority", 100); - job["timeout"] = drv->queryMetaInt("timeout", 36000); - job["maxSilent"] = drv->queryMetaInt("maxSilent", 7200); - job["isChannel"] = drv->queryMetaBool("isHydraChannel", false); nlohmann::json meta; for (auto & name : drv->queryMetaNames()) { From 174e73518373998c48d6480c97f9a27381c8d8a6 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Tue, 24 Aug 2021 13:13:04 -0500 Subject: [PATCH 025/419] Rename project to nix-eval-jobs We are getting rid of Hydra specific hacks and making this a more generically useful component for any projects that need to run large evaluations. --- README.md | 2 +- flake.nix | 4 ++-- hydra.nix | 4 ++-- meson.build | 2 +- src/meson.build | 4 ++-- src/{hydra-eval-jobs.cc => nix-eval-jobs.cc} | 4 ++-- tests/test_eval.py | 2 +- 7 files changed, 11 insertions(+), 11 deletions(-) rename src/{hydra-eval-jobs.cc => nix-eval-jobs.cc} (99%) diff --git a/README.md b/README.md index 51ee96f35..3bb702869 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,2 @@ -# hydra-eval-jobs +# nix-eval-jobs Eval nix expressions from flakes (extracted from hydra) diff --git a/flake.nix b/flake.nix index bf87bd66b..10b44270c 100644 --- a/flake.nix +++ b/flake.nix @@ -10,10 +10,10 @@ pkgs = nixpkgs.legacyPackages.${system}; in rec { - packages.hydra-eval-jobs = pkgs.callPackage ./hydra.nix { + packages.nix-eval-jobs = pkgs.callPackage ./hydra.nix { srcDir = self; }; - defaultPackage = self.packages.${system}.hydra-eval-jobs; + defaultPackage = self.packages.${system}.nix-eval-jobs; devShell = defaultPackage.overrideAttrs (old: { nativeBuildInputs = old.nativeBuildInputs ++ [ (pkgs.python3.withPackages(ps: [ diff --git a/hydra.nix b/hydra.nix index dc94dddd9..886be0bb5 100644 --- a/hydra.nix +++ b/hydra.nix @@ -14,7 +14,7 @@ let (path: type: type != "directory" || baseNameOf path != "build") dir; in stdenv.mkDerivation rec { - pname = "hydra-eval-jobs"; + pname = "nix-eval-jobs"; version = "0.0.1"; src = if srcDir == null then filterMesonBuild ./. else srcDir; buildInputs = [ @@ -27,7 +27,7 @@ stdenv.mkDerivation rec { ]; meta = with stdenv.lib; { description = "Hydra's builtin hydra-eval-jobs as a standalone"; - homepage = "https://github.com/Mic92/hydra-eval-jobs"; + homepage = "https://github.com/nix-community/nix-eval-jobs"; license = licenses.mit; maintainers = with maintainers; [ mic92 ]; platforms = platforms.unix; diff --git a/meson.build b/meson.build index c27792529..1767e6380 100644 --- a/meson.build +++ b/meson.build @@ -1,4 +1,4 @@ -project('hydra-eval-jobs', 'cpp', +project('nix-eval-jobs', 'cpp', version : '0.1.6', license : 'GPL-3.0', ) diff --git a/src/meson.build b/src/meson.build index 50603b417..56c480d54 100644 --- a/src/meson.build +++ b/src/meson.build @@ -1,8 +1,8 @@ src = [ - 'hydra-eval-jobs.cc', + 'nix-eval-jobs.cc', ] -executable('hydra-eval-jobs', src, +executable('nix-eval-jobs', src, dependencies : [ nix_main_dep, nix_store_dep, diff --git a/src/hydra-eval-jobs.cc b/src/nix-eval-jobs.cc similarity index 99% rename from src/hydra-eval-jobs.cc rename to src/nix-eval-jobs.cc index 8d509ff11..31eee9c81 100644 --- a/src/hydra-eval-jobs.cc +++ b/src/nix-eval-jobs.cc @@ -38,13 +38,13 @@ struct MyArgs : MixEvalArgs, MixCommonArgs size_t maxMemorySize = 4096; pureEval evalMode = evalAuto; - MyArgs() : MixCommonArgs("hydra-eval-jobs") + MyArgs() : MixCommonArgs("nix-eval-jobs") { addFlag({ .longName = "help", .description = "show usage information", .handler = {[&]() { - printf("USAGE: hydra-eval-jobs [options] expr\n\n"); + printf("USAGE: nix-eval-jobs [options] expr\n\n"); for (const auto & [name, flag] : longFlags) { if (hiddenCategories.count(flag->category)) { continue; diff --git a/tests/test_eval.py b/tests/test_eval.py index 02eced9df..fe5373ad1 100644 --- a/tests/test_eval.py +++ b/tests/test_eval.py @@ -8,7 +8,7 @@ from typing import List TEST_ROOT = Path(__file__).parent.resolve() PROJECT_ROOT = TEST_ROOT.parent -BIN = PROJECT_ROOT.joinpath("build", "src", "hydra-eval-jobs") +BIN = PROJECT_ROOT.joinpath("build", "src", "nix-eval-jobs") def common_test(extra_args: List[str]) -> None: From a4441fe1c8949665a6128ef9224def7a04dbd211 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Tue, 24 Aug 2021 13:47:30 -0500 Subject: [PATCH 026/419] Remove support for Hydra aggregate jobs This might be a somewhat useful feature for Hydra, but not for a generic Nix evaluator. --- src/nix-eval-jobs.cc | 68 -------------------------------------------- 1 file changed, 68 deletions(-) diff --git a/src/nix-eval-jobs.cc b/src/nix-eval-jobs.cc index 31eee9c81..d6fb4ee4a 100644 --- a/src/nix-eval-jobs.cc +++ b/src/nix-eval-jobs.cc @@ -196,31 +196,6 @@ static void worker( } job["meta"] = meta; - /* If this is an aggregate, then get its constituents. */ - auto a = v->attrs->get(state.symbols.create("_hydraAggregate")); - if (a && state.forceBool(*a->value, *a->pos)) { - auto a = v->attrs->get(state.symbols.create("constituents")); - if (!a) - throw EvalError("derivation must have a ‘constituents’ attribute"); - - - PathSet context; - state.coerceToString(*a->pos, *a->value, context, true, false); - for (auto & i : context) - if (i.at(0) == '!') { - size_t index = i.find("!", 1); - job["constituents"].push_back(string(i, index + 1)); - } - - state.forceList(*a->value, *a->pos); - for (unsigned int n = 0; n < a->value->listSize(); ++n) { - auto v = a->value->listElems()[n]; - state.forceValue(*v); - if (v->type() == nString) - job["namedConstituents"].push_back(state.forceStringNoCtx(*v)); - } - } - /* Register the derivation as a GC root. !!! This registers roots for jobs that we may have already done. */ @@ -466,49 +441,6 @@ int main(int argc, char * * argv) if (state->exc) std::rethrow_exception(state->exc); - /* For aggregate jobs that have named consistuents - (i.e. constituents that are a job name rather than a - derivation), look up the referenced job and add it to the - dependencies of the aggregate derivation. */ - auto store = openStore(); - - for (auto i = state->jobs.begin(); i != state->jobs.end(); ++i) { - auto jobName = i.key(); - auto & job = i.value(); - - auto named = job.find("namedConstituents"); - if (named == job.end()) continue; - - auto drvPath = store->parseStorePath((std::string) job["drvPath"]); - auto drv = store->readDerivation(drvPath); - - for (std::string jobName2 : *named) { - auto job2 = state->jobs.find(jobName2); - if (job2 == state->jobs.end()) - throw Error("aggregate job '%s' references non-existent job '%s'", jobName, jobName2); - auto drvPath2 = store->parseStorePath((std::string) (*job2)["drvPath"]); - auto drv2 = store->readDerivation(drvPath2); - job["constituents"].push_back(store->printStorePath(drvPath2)); - drv.inputDrvs[drvPath2] = {drv2.outputs.begin()->first}; - } - - std::string drvName(drvPath.name()); - assert(hasSuffix(drvName, drvExtension)); - drvName.resize(drvName.size() - drvExtension.size()); - auto h = std::get(hashDerivationModulo(*store, drv, true)); - auto outPath = store->makeOutputPath("out", h, drvName); - drv.env["out"] = store->printStorePath(outPath); - drv.outputs.insert_or_assign("out", DerivationOutput { .output = DerivationOutputInputAddressed { .path = outPath } }); - auto newDrvPath = store->printStorePath(writeDerivation(*store, drv)); - - debug("rewrote aggregate derivation %s -> %s", store->printStorePath(drvPath), newDrvPath); - - job["drvPath"] = newDrvPath; - job["outputs"]["out"] = store->printStorePath(outPath); - - job.erase("namedConstituents"); - } - std::cout << state->jobs.dump(2) << "\n"; }); } From a80fc92bc15a0faa3d8bd92f2a0d2c37a46d9a28 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Tue, 24 Aug 2021 14:07:11 -0500 Subject: [PATCH 027/419] Remove deprecated use of stdenv.lib --- hydra.nix | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hydra.nix b/hydra.nix index 886be0bb5..ab5ce02e0 100644 --- a/hydra.nix +++ b/hydra.nix @@ -1,4 +1,5 @@ { stdenv +, lib , nixFlakes , meson , cmake @@ -25,7 +26,7 @@ stdenv.mkDerivation rec { # nlohmann_json can be only discovered via cmake files cmake ]; - meta = with stdenv.lib; { + meta = with lib; { description = "Hydra's builtin hydra-eval-jobs as a standalone"; homepage = "https://github.com/nix-community/nix-eval-jobs"; license = licenses.mit; From 8f2e588dbd2cac95abc588f3f6237c52b3cd40e3 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Tue, 24 Aug 2021 18:23:28 -0500 Subject: [PATCH 028/419] Set correct license in Nix expression Hydra upstream was always GPL3 licensed, so by extension this repo is also GPL3. --- hydra.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hydra.nix b/hydra.nix index ab5ce02e0..cd5af6ec3 100644 --- a/hydra.nix +++ b/hydra.nix @@ -29,7 +29,7 @@ stdenv.mkDerivation rec { meta = with lib; { description = "Hydra's builtin hydra-eval-jobs as a standalone"; homepage = "https://github.com/nix-community/nix-eval-jobs"; - license = licenses.mit; + license = licenses.gpl3; maintainers = with maintainers; [ mic92 ]; platforms = platforms.unix; }; From 08499559fc2a3b5fe2fc77224a1ebcdb2a66e015 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Tue, 24 Aug 2021 19:30:55 -0500 Subject: [PATCH 029/419] Bump nixpkgs / flake.lock --- flake.lock | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/flake.lock b/flake.lock index 633cd6300..39eba216e 100644 --- a/flake.lock +++ b/flake.lock @@ -2,11 +2,11 @@ "nodes": { "flake-utils": { "locked": { - "lastModified": 1614513358, - "narHash": "sha256-LakhOx3S1dRjnh0b5Dg3mbZyH0ToC9I8Y2wKSkBaTzU=", + "lastModified": 1629481132, + "narHash": "sha256-JHgasjPR0/J1J3DRm4KxM4zTyAj4IOJY8vIl75v/kPI=", "owner": "numtide", "repo": "flake-utils", - "rev": "5466c5bbece17adaab2d82fae80b46e807611bf3", + "rev": "997f7efcb746a9c140ce1f13c72263189225f482", "type": "github" }, "original": { @@ -17,11 +17,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1615756511, - "narHash": "sha256-rcuG5eYBtDlN0yTJdFw8GktML7Og1hFctp44wE3MHp4=", + "lastModified": 1629846896, + "narHash": "sha256-3uVhnCgWHymKQKtVNspxNJDLXCY4uiGlCPcnib5w1PY=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "87acb6d60bfe9469475dbaa80df90971aaa21997", + "rev": "1c9124b63bf1c12f420b023836ea770e05adeafa", "type": "github" }, "original": { From f8dad73ac8e33820bde3e5bae86cee4a421c75ca Mon Sep 17 00:00:00 2001 From: adisbladis Date: Wed, 25 Aug 2021 03:11:16 -0500 Subject: [PATCH 030/419] Move outputs definition closer to use --- src/nix-eval-jobs.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/nix-eval-jobs.cc b/src/nix-eval-jobs.cc index d6fb4ee4a..a9bf44b0d 100644 --- a/src/nix-eval-jobs.cc +++ b/src/nix-eval-jobs.cc @@ -173,8 +173,6 @@ static void worker( if (auto drv = getDerivation(state, *v, false)) { - DrvInfo::Outputs outputs = drv->queryOutputs(); - if (drv->querySystem() == "unknown") throw EvalError("derivation must have a 'system' attribute"); @@ -224,6 +222,7 @@ static void worker( downloadSize, narSize); + DrvInfo::Outputs outputs = drv->queryOutputs(); nlohmann::json out; for (auto & p : outputs) { out[p.first] = p.second; From be59cd8bfbc4965604204a8a423742e74def1d31 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Wed, 25 Aug 2021 03:12:03 -0500 Subject: [PATCH 031/419] Remove unused output --- src/nix-eval-jobs.cc | 29 ----------------------------- 1 file changed, 29 deletions(-) diff --git a/src/nix-eval-jobs.cc b/src/nix-eval-jobs.cc index a9bf44b0d..a13e99f7f 100644 --- a/src/nix-eval-jobs.cc +++ b/src/nix-eval-jobs.cc @@ -100,14 +100,6 @@ struct MyArgs : MixEvalArgs, MixCommonArgs static MyArgs myArgs; -static nlohmann::json serializeStorePathSet(StorePathSet &paths, LocalFSStore &store) { - auto array = nlohmann::json::array(); - for (auto & p : paths) { - array.push_back(store.printStorePath(p)); - } - return array; -} - static void worker( EvalState & state, Bindings & autoArgs, @@ -205,23 +197,6 @@ static void worker( localStore->addPermRoot(storePath, root); } - uint64_t downloadSize, narSize; - StorePathSet willBuild, willSubstitute, unknown; - std::vector paths; - StringSet outputNames; - - for (auto & output : outputs) { - outputNames.insert(output.first); - } - paths.push_back({storePath, outputNames}); - - localStore->queryMissing(paths, - willBuild, - willSubstitute, - unknown, - downloadSize, - narSize); - DrvInfo::Outputs outputs = drv->queryOutputs(); nlohmann::json out; for (auto & p : outputs) { @@ -229,10 +204,6 @@ static void worker( } job["outputs"] = std::move(out); - job["builds"] = serializeStorePathSet(willBuild, *localStore); - job["substitutes"] = serializeStorePathSet(willSubstitute, *localStore); - job["unknown"] = serializeStorePathSet(unknown, *localStore); - reply["job"] = std::move(job); } From 4c28ae88a6545c3a5c92a44557a7b67d74753181 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Wed, 25 Aug 2021 03:12:19 -0500 Subject: [PATCH 032/419] Skip non-serialisable *Value in output Most notably this includes derivations which should be fixed and serialised to store paths. --- src/nix-eval-jobs.cc | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/nix-eval-jobs.cc b/src/nix-eval-jobs.cc index a13e99f7f..8e35234c1 100644 --- a/src/nix-eval-jobs.cc +++ b/src/nix-eval-jobs.cc @@ -180,7 +180,15 @@ static void worker( for (auto & name : drv->queryMetaNames()) { PathSet context; std::stringstream ss; - printValueAsJSON(state, true, *drv->queryMeta(name), ss, context); + + auto metaValue = drv->queryMeta(name); + // Skip non-serialisable types + // TODO: Fix serialisation of derivations to store paths + if (metaValue == 0) { + continue; + } + + printValueAsJSON(state, true, *metaValue, ss, context); nlohmann::json field = nlohmann::json::parse(ss.str()); meta[name] = field; } From 9bfc3762eb1487d2b5c3fe2c0f7d1c26f10a8545 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Wed, 25 Aug 2021 03:29:10 -0500 Subject: [PATCH 033/419] Update github actions workflow with new naming scheme --- .github/workflows/test-flakes.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-flakes.yml b/.github/workflows/test-flakes.yml index 067a97e7d..5f5387581 100644 --- a/.github/workflows/test-flakes.yml +++ b/.github/workflows/test-flakes.yml @@ -29,5 +29,5 @@ jobs: run: nix build --out-link result - name: Run tests run: | - nix develop -c install -D ./result/bin/hydra-eval-jobs ./build/src/hydra-eval-jobs + nix develop -c install -D ./result/bin/nix-eval-jobs ./build/src/nix-eval-jobs nix develop -c pytest ./tests From 0c93e00635a2ffa6b589756137439453320bf4a5 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Wed, 25 Aug 2021 03:36:32 -0500 Subject: [PATCH 034/419] Fix pytest tests --- tests/test_eval.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/test_eval.py b/tests/test_eval.py index fe5373ad1..fe5a82606 100644 --- a/tests/test_eval.py +++ b/tests/test_eval.py @@ -22,8 +22,11 @@ def common_test(extra_args: List[str]) -> None: stdout=subprocess.PIPE, ) data = json.loads(res.stdout) - assert len(data["builtJob"]["builds"]) == 1 - assert len(data["substitutedJob"]["substitutes"]) >= 1 + + assert data["builtJob"]["nixName"] == "job1" + assert "out" in data["builtJob"]["outputs"] + + assert data["substitutedJob"]["nixName"].startswith("hello-") def test_flake() -> None: From a6171c9a620ee042ac25337346a59e9f768f6c00 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Wed, 25 Aug 2021 12:43:35 -0500 Subject: [PATCH 035/419] Switch to streaming output (using line delimited JSON) --- src/nix-eval-jobs.cc | 21 +++++++-------------- tests/test_eval.py | 14 ++++++++++---- 2 files changed, 17 insertions(+), 18 deletions(-) diff --git a/src/nix-eval-jobs.cc b/src/nix-eval-jobs.cc index 8e35234c1..2840194dc 100644 --- a/src/nix-eval-jobs.cc +++ b/src/nix-eval-jobs.cc @@ -156,6 +156,7 @@ static void worker( /* Evaluate it and send info back to the master. */ nlohmann::json reply; + reply["attr"] = attrPath; try { auto vTmp = findAlongAttrPath(state, attrPath, autoArgs, *vRoot).first; @@ -215,7 +216,8 @@ static void worker( reply["job"] = std::move(job); } - else if (v->type() == nAttrs) { + else if (v->type() == nAttrs) + { auto attrs = nlohmann::json::array(); StringSet ss; for (auto & i : v->attrs->lexicographicOrder()) { @@ -287,7 +289,6 @@ int main(int argc, char * * argv) { std::set todo{""}; std::set active; - nlohmann::json jobs; std::exception_ptr exc; }; @@ -369,26 +370,19 @@ int main(int argc, char * * argv) writeLine(to.get(), "do " + attrPath); /* Wait for the response. */ - auto response = nlohmann::json::parse(readLine(from.get())); + auto respString = readLine(from.get()); + auto response = nlohmann::json::parse(respString); /* Handle the response. */ StringSet newAttrs; - - if (response.find("job") != response.end()) { - auto state(state_.lock()); - state->jobs[attrPath] = response["job"]; - } - if (response.find("attrs") != response.end()) { for (auto & i : response["attrs"]) { auto s = (attrPath.empty() ? "" : attrPath + ".") + (std::string) i; newAttrs.insert(s); } - } - - if (response.find("error") != response.end()) { + } else { auto state(state_.lock()); - state->jobs[attrPath]["error"] = response["error"]; + std::cout << respString << "\n"; } /* Add newly discovered job names to the queue. */ @@ -419,6 +413,5 @@ int main(int argc, char * * argv) if (state->exc) std::rethrow_exception(state->exc); - std::cout << state->jobs.dump(2) << "\n"; }); } diff --git a/tests/test_eval.py b/tests/test_eval.py index fe5a82606..7f5e2bf25 100644 --- a/tests/test_eval.py +++ b/tests/test_eval.py @@ -21,12 +21,18 @@ def common_test(extra_args: List[str]) -> None: check=True, stdout=subprocess.PIPE, ) - data = json.loads(res.stdout) - assert data["builtJob"]["nixName"] == "job1" - assert "out" in data["builtJob"]["outputs"] + results = [json.loads(r) for r in res.stdout.split("\n") if r] + assert len(results) == 2 - assert data["substitutedJob"]["nixName"].startswith("hello-") + built_job = results[0] + assert built_job["attr"] == "builtJob" + assert built_job["job"]["nixName"] == "job1" + + + substituted_job = results[1] + assert substituted_job["attr"] == "substitutedJob" + assert substituted_job["job"]["nixName"].startswith("hello-") def test_flake() -> None: From aa0b6fd4a824d29e49393ddd53c546638465d9e6 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Wed, 25 Aug 2021 12:49:23 -0500 Subject: [PATCH 036/419] Add editorconfig file --- .editorconfig | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 .editorconfig diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 000000000..1ccb157ca --- /dev/null +++ b/.editorconfig @@ -0,0 +1,24 @@ +# top-most EditorConfig file +root = true + +# Unix-style newlines with a newline ending every file +[*] +charset = utf-8 +end_of_line = lf +insert_final_newline = true +trim_trailing_whitespace = true + +[*.{cc,hh,hpp,pl,pm,sh,t}] +indent_style = space +intend_size = 4 + +[Makefile] +indent_style = tab + +[*.nix] +indent_style = space +indent_size = 2 + +# Match diffs, avoid to trim trailing whitespace +[*.{diff,patch}] +trim_trailing_whitespace = false From ebe49463b89f83d2173cb890aecbbd6cc5d3a495 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Wed, 25 Aug 2021 12:50:06 -0500 Subject: [PATCH 037/419] Add editorconfig-checker --- .gitignore | 3 +++ LICENSE.md | 4 +--- flake.nix | 19 +++++++++++++++++++ 3 files changed, 23 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 250b5c443..c6f9d8c6f 100644 --- a/.gitignore +++ b/.gitignore @@ -52,3 +52,6 @@ __pycache__/ .mypy_cache/ .dmypy.json dmypy.json + +# nix-direnv +.direnv diff --git a/LICENSE.md b/LICENSE.md index def709e12..1110e8987 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -1,7 +1,7 @@ GNU General Public License ========================== -_Version 3, 29 June 2007_ +_Version 3, 29 June 2007_ _Copyright © 2007 Free Software Foundation, Inc. <>_ Everyone is permitted to copy and distribute verbatim copies of this license @@ -593,5 +593,3 @@ more useful to permit linking proprietary applications with the library. If this what you want to do, use the GNU Lesser General Public License instead of this License. But first, please read <>. - - diff --git a/flake.nix b/flake.nix index 10b44270c..a69e3d58a 100644 --- a/flake.nix +++ b/flake.nix @@ -13,12 +13,31 @@ packages.nix-eval-jobs = pkgs.callPackage ./hydra.nix { srcDir = self; }; + + checks = { + + editorconfig = pkgs.runCommand "editorconfig-checks" { + nativeBuildInputs = [ + pkgs.editorconfig-checker + ]; + } '' + editorconfig-checker ${self} + touch $out + ''; + + }; + defaultPackage = self.packages.${system}.nix-eval-jobs; devShell = defaultPackage.overrideAttrs (old: { + nativeBuildInputs = old.nativeBuildInputs ++ [ + + pkgs.editorconfig-checker + (pkgs.python3.withPackages(ps: [ ps.pytest ])) + ]; }); }); From 0e3df9f5d53f279858d1f991e46343fdf4911b46 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Wed, 25 Aug 2021 14:01:17 -0500 Subject: [PATCH 038/419] Update CI stable channel to 21.05 --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3ce90cbc0..e8bacccab 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -11,7 +11,7 @@ jobs: strategy: matrix: nixPath: - - nixpkgs=https://github.com/NixOS/nixpkgs/archive/nixos-20.09.tar.gz + - nixpkgs=https://github.com/NixOS/nixpkgs/archive/nixos-21.05.tar.gz - nixpkgs=https://github.com/NixOS/nixpkgs/archive/nixpkgs-unstable.tar.gz os: [ ubuntu-latest, macos-latest ] runs-on: ${{ matrix.os }} From 8ccdf6497c65ac1f5bd1080efec1fab94f8dca08 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Wed, 25 Aug 2021 14:06:20 -0500 Subject: [PATCH 039/419] Add build to flake checks --- flake.nix | 2 ++ 1 file changed, 2 insertions(+) diff --git a/flake.nix b/flake.nix index a69e3d58a..b58e5066c 100644 --- a/flake.nix +++ b/flake.nix @@ -25,6 +25,8 @@ touch $out ''; + build = packages.nix-eval-jobs; + }; defaultPackage = self.packages.${system}.nix-eval-jobs; From 434376f8e1ef653e79b4c2b5a38cba45109152e4 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Thu, 26 Aug 2021 16:56:23 -0500 Subject: [PATCH 040/419] Remove outputs from JSON output The output of the evaluator should only include either the full derivation (not yet implemented) or fields not directly accessible from the drv such as meta. Right now the output is a fairly arbitrary selection of fields. --- src/nix-eval-jobs.cc | 17 +++-------------- tests/test_eval.py | 4 ++-- 2 files changed, 5 insertions(+), 16 deletions(-) diff --git a/src/nix-eval-jobs.cc b/src/nix-eval-jobs.cc index 2840194dc..ce7462ab8 100644 --- a/src/nix-eval-jobs.cc +++ b/src/nix-eval-jobs.cc @@ -171,11 +171,9 @@ static void worker( auto drvPath = drv->queryDrvPath(); - nlohmann::json job; - - job["nixName"] = drv->queryName(); - job["system"] =drv->querySystem(); - job["drvPath"] = drvPath; + reply["name"] = drv->queryName(); + reply["system"] = drv->querySystem(); + reply["drvPath"] = drvPath; nlohmann::json meta; for (auto & name : drv->queryMetaNames()) { @@ -193,7 +191,6 @@ static void worker( nlohmann::json field = nlohmann::json::parse(ss.str()); meta[name] = field; } - job["meta"] = meta; /* Register the derivation as a GC root. !!! This registers roots for jobs that we may have already @@ -206,14 +203,6 @@ static void worker( localStore->addPermRoot(storePath, root); } - DrvInfo::Outputs outputs = drv->queryOutputs(); - nlohmann::json out; - for (auto & p : outputs) { - out[p.first] = p.second; - } - job["outputs"] = std::move(out); - - reply["job"] = std::move(job); } else if (v->type() == nAttrs) diff --git a/tests/test_eval.py b/tests/test_eval.py index 7f5e2bf25..7e32d7a54 100644 --- a/tests/test_eval.py +++ b/tests/test_eval.py @@ -27,12 +27,12 @@ def common_test(extra_args: List[str]) -> None: built_job = results[0] assert built_job["attr"] == "builtJob" - assert built_job["job"]["nixName"] == "job1" + assert built_job["name"] == "job1" substituted_job = results[1] assert substituted_job["attr"] == "substitutedJob" - assert substituted_job["job"]["nixName"].startswith("hello-") + assert substituted_job["name"].startswith("hello-") def test_flake() -> None: From 0648bc0cb83c1e4c132db5648b3761ce4c0350ea Mon Sep 17 00:00:00 2001 From: adisbladis Date: Thu, 26 Aug 2021 18:48:11 -0500 Subject: [PATCH 041/419] Select flake output by the flake fragment --- src/nix-eval-jobs.cc | 17 +++++++++-------- tests/test_eval.py | 2 +- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/src/nix-eval-jobs.cc b/src/nix-eval-jobs.cc index ce7462ab8..c07ca9b40 100644 --- a/src/nix-eval-jobs.cc +++ b/src/nix-eval-jobs.cc @@ -112,7 +112,7 @@ static void worker( if (myArgs.flake) { using namespace flake; - auto flakeRef = parseFlakeRef(myArgs.releaseExpr, absPath(".")); + auto [flakeRef, fragment] = parseFlakeRefWithFragment(myArgs.releaseExpr, absPath(".")); auto vFlake = state.allocValue(); @@ -127,14 +127,15 @@ static void worker( auto vOutputs = vFlake->attrs->get(state.symbols.create("outputs"))->value; state.forceValue(*vOutputs); + vTop = *vOutputs; - auto aHydraJobs = vOutputs->attrs->get(state.symbols.create("hydraJobs")); - if (!aHydraJobs) - aHydraJobs = vOutputs->attrs->get(state.symbols.create("checks")); - if (!aHydraJobs) - throw Error("flake '%s' does not provide any Hydra jobs or checks", flakeRef); - - vTop = *aHydraJobs->value; + if (fragment.length() > 0) { + Bindings & bindings(*state.allocBindings(0)); + auto [nTop, pos] = findAlongAttrPath(state, fragment, bindings, vTop); + if (!nTop) + throw Error("error: attribute '%s' missing", nTop); + vTop = *nTop; + } } else { state.evalFile(lookupFileArg(state, myArgs.releaseExpr), vTop); diff --git a/tests/test_eval.py b/tests/test_eval.py index 7e32d7a54..32a31e77c 100644 --- a/tests/test_eval.py +++ b/tests/test_eval.py @@ -36,7 +36,7 @@ def common_test(extra_args: List[str]) -> None: def test_flake() -> None: - common_test(["--flake", ".#"]) + common_test(["--flake", ".#hydraJobs"]) def test_expression() -> None: From 2d169571e456cbf5e58570adb2ecc6140d395500 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Sep 2021 23:07:08 +0000 Subject: [PATCH 042/419] Bump cachix/install-nix-action from 13 to 14 Bumps [cachix/install-nix-action](https://github.com/cachix/install-nix-action) from 13 to 14. - [Release notes](https://github.com/cachix/install-nix-action/releases) - [Commits](https://github.com/cachix/install-nix-action/compare/v13...v14) --- updated-dependencies: - dependency-name: cachix/install-nix-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/test-flakes.yml | 2 +- .github/workflows/test.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-flakes.yml b/.github/workflows/test-flakes.yml index 5f5387581..75e8e2fad 100644 --- a/.github/workflows/test-flakes.yml +++ b/.github/workflows/test-flakes.yml @@ -17,7 +17,7 @@ jobs: with: # Nix Flakes doesn't work on shallow clones fetch-depth: 0 - - uses: cachix/install-nix-action@v13 + - uses: cachix/install-nix-action@v14 with: install_url: https://github.com/numtide/nix-flakes-installer/releases/download/nix-2.4pre20210207_fd6eaa1/install extra_nix_config: | diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e8bacccab..9b2ebf67a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -17,6 +17,6 @@ jobs: runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v2 - - uses: cachix/install-nix-action@v13 + - uses: cachix/install-nix-action@v14 - name: build run: NIX_PATH=nixpkgs=https://github.com/NixOS/nixpkgs/archive/nixpkgs-unstable.tar.gz nix-build From 7550bb8cc12b734501a0908a820d34ae65edba15 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Fri, 24 Sep 2021 22:02:34 -0500 Subject: [PATCH 043/419] Move hydra.nix to default.nix So it's easier to use as a classic (non-flake) Nix expression. --- default.nix | 39 +++++++++++++++++++++++++++++++++++++-- flake.nix | 2 +- hydra.nix | 36 ------------------------------------ 3 files changed, 38 insertions(+), 39 deletions(-) delete mode 100644 hydra.nix diff --git a/default.nix b/default.nix index b2286a660..811bdc724 100644 --- a/default.nix +++ b/default.nix @@ -1,2 +1,37 @@ -{ pkgs ? import {} }: -pkgs.callPackage ./hydra.nix {} +{ stdenv +, lib +, nixFlakes +, meson +, cmake +, ninja +, pkg-config +, boost +, nlohmann_json +, srcDir ? null +}: + +let + filterMesonBuild = dir: builtins.filterSource + (path: type: type != "directory" || baseNameOf path != "build") dir; +in +stdenv.mkDerivation rec { + pname = "nix-eval-jobs"; + version = "0.0.1"; + src = if srcDir == null then filterMesonBuild ./. else srcDir; + buildInputs = [ + nlohmann_json nixFlakes boost + ]; + nativeBuildInputs = [ + meson pkg-config ninja + # nlohmann_json can be only discovered via cmake files + cmake + ]; + + meta = { + description = "Hydra's builtin hydra-eval-jobs as a standalone"; + homepage = "https://github.com/nix-community/nix-eval-jobs"; + license = lib.licenses.gpl3; + maintainers = with lib.maintainers; [ adisbladis mic92 ]; + platforms = lib.platforms.unix; + }; +} diff --git a/flake.nix b/flake.nix index b58e5066c..8c290d8a7 100644 --- a/flake.nix +++ b/flake.nix @@ -10,7 +10,7 @@ pkgs = nixpkgs.legacyPackages.${system}; in rec { - packages.nix-eval-jobs = pkgs.callPackage ./hydra.nix { + packages.nix-eval-jobs = pkgs.callPackage ./default.nix { srcDir = self; }; diff --git a/hydra.nix b/hydra.nix deleted file mode 100644 index cd5af6ec3..000000000 --- a/hydra.nix +++ /dev/null @@ -1,36 +0,0 @@ -{ stdenv -, lib -, nixFlakes -, meson -, cmake -, ninja -, pkg-config -, boost -, nlohmann_json -, srcDir ? null -}: - -let - filterMesonBuild = dir: builtins.filterSource - (path: type: type != "directory" || baseNameOf path != "build") dir; -in -stdenv.mkDerivation rec { - pname = "nix-eval-jobs"; - version = "0.0.1"; - src = if srcDir == null then filterMesonBuild ./. else srcDir; - buildInputs = [ - nlohmann_json nixFlakes boost - ]; - nativeBuildInputs = [ - meson pkg-config ninja - # nlohmann_json can be only discovered via cmake files - cmake - ]; - meta = with lib; { - description = "Hydra's builtin hydra-eval-jobs as a standalone"; - homepage = "https://github.com/nix-community/nix-eval-jobs"; - license = licenses.gpl3; - maintainers = with maintainers; [ mic92 ]; - platforms = platforms.unix; - }; -} From 9030f0f8c50259b09e15e1ce0e68db325aeb621d Mon Sep 17 00:00:00 2001 From: adisbladis Date: Fri, 24 Sep 2021 22:12:56 -0500 Subject: [PATCH 044/419] Move devShell from flake.nix to separate shell.nix Again, to facilitate non-flake development. --- flake.nix | 17 +++-------------- shell.nix | 17 +++++++++++++++++ 2 files changed, 20 insertions(+), 14 deletions(-) create mode 100644 shell.nix diff --git a/flake.nix b/flake.nix index 8c290d8a7..5ba387724 100644 --- a/flake.nix +++ b/flake.nix @@ -8,11 +8,10 @@ flake-utils.lib.eachDefaultSystem (system: let pkgs = nixpkgs.legacyPackages.${system}; + drvArgs = { srcDir = self; }; in rec { - packages.nix-eval-jobs = pkgs.callPackage ./default.nix { - srcDir = self; - }; + packages.nix-eval-jobs = pkgs.callPackage ./default.nix drvArgs; checks = { @@ -30,17 +29,7 @@ }; defaultPackage = self.packages.${system}.nix-eval-jobs; - devShell = defaultPackage.overrideAttrs (old: { + devShell = pkgs.callPackage ./shell.nix drvArgs; - nativeBuildInputs = old.nativeBuildInputs ++ [ - - pkgs.editorconfig-checker - - (pkgs.python3.withPackages(ps: [ - ps.pytest - ])) - - ]; - }); }); } diff --git a/shell.nix b/shell.nix new file mode 100644 index 000000000..736a1c346 --- /dev/null +++ b/shell.nix @@ -0,0 +1,17 @@ +{ pkgs ? import { } +, srcDir ? null +}: + +(pkgs.callPackage ./default.nix { inherit srcDir; }).overrideAttrs(old: { + + nativeBuildInputs = old.nativeBuildInputs ++ [ + + pkgs.editorconfig-checker + + (pkgs.python3.withPackages(ps: [ + ps.pytest + ])) + + ]; + +}) From 0d3e7c2240890a9e304bd5cf3af6cd0ffa457077 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Fri, 24 Sep 2021 22:17:06 -0500 Subject: [PATCH 045/419] Add nixpkgs-fmt + associated checks --- default.nix | 11 ++++++++--- flake.nix | 24 ++++++++++++++++++------ shell.nix | 6 ++++-- tests/assets/ci.nix | 2 +- tests/assets/flake.nix | 16 +++++++++------- 5 files changed, 40 insertions(+), 19 deletions(-) diff --git a/default.nix b/default.nix index 811bdc724..abfcda46a 100644 --- a/default.nix +++ b/default.nix @@ -12,17 +12,22 @@ let filterMesonBuild = dir: builtins.filterSource - (path: type: type != "directory" || baseNameOf path != "build") dir; + (path: type: type != "directory" || baseNameOf path != "build") + dir; in stdenv.mkDerivation rec { pname = "nix-eval-jobs"; version = "0.0.1"; src = if srcDir == null then filterMesonBuild ./. else srcDir; buildInputs = [ - nlohmann_json nixFlakes boost + nlohmann_json + nixFlakes + boost ]; nativeBuildInputs = [ - meson pkg-config ninja + meson + pkg-config + ninja # nlohmann_json can be only discovered via cmake files cmake ]; diff --git a/flake.nix b/flake.nix index 5ba387724..6e4dd50a8 100644 --- a/flake.nix +++ b/flake.nix @@ -15,15 +15,26 @@ checks = { - editorconfig = pkgs.runCommand "editorconfig-checks" { - nativeBuildInputs = [ - pkgs.editorconfig-checker - ]; - } '' + editorconfig = pkgs.runCommand "editorconfig-check" + { + nativeBuildInputs = [ + pkgs.editorconfig-checker + ]; + } '' editorconfig-checker ${self} touch $out ''; + nixpkgs-fmt = pkgs.runCommand "fmt-check" + { + nativeBuildInputs = [ + pkgs.nixpkgs-fmt + ]; + } '' + nixpkgs-fmt --check . + touch $out + ''; + build = packages.nix-eval-jobs; }; @@ -31,5 +42,6 @@ defaultPackage = self.packages.${system}.nix-eval-jobs; devShell = pkgs.callPackage ./shell.nix drvArgs; - }); + } + ); } diff --git a/shell.nix b/shell.nix index 736a1c346..7b97e75ea 100644 --- a/shell.nix +++ b/shell.nix @@ -2,13 +2,15 @@ , srcDir ? null }: -(pkgs.callPackage ./default.nix { inherit srcDir; }).overrideAttrs(old: { +(pkgs.callPackage ./default.nix { inherit srcDir; }).overrideAttrs (old: { nativeBuildInputs = old.nativeBuildInputs ++ [ pkgs.editorconfig-checker - (pkgs.python3.withPackages(ps: [ + pkgs.nixpkgs-fmt + + (pkgs.python3.withPackages (ps: [ ps.pytest ])) diff --git a/tests/assets/ci.nix b/tests/assets/ci.nix index 48bf1774f..3aa2eca91 100644 --- a/tests/assets/ci.nix +++ b/tests/assets/ci.nix @@ -1,5 +1,5 @@ let - pkgs = import (builtins.getFlake (toString ./.)).inputs.nixpkgs {}; + pkgs = import (builtins.getFlake (toString ./.)).inputs.nixpkgs { }; in { builtJob = pkgs.writeText "job1" "job1"; diff --git a/tests/assets/flake.nix b/tests/assets/flake.nix index bbd81a6bf..c3835109d 100644 --- a/tests/assets/flake.nix +++ b/tests/assets/flake.nix @@ -1,12 +1,14 @@ { inputs.nixpkgs.url = "github:NixOS/nixpkgs"; - outputs = { self, nixpkgs }: let - pkgs = nixpkgs.legacyPackages.x86_64-linux; - in { - hydraJobs = { - builtJob = pkgs.writeText "job1" "job1"; - substitutedJob = pkgs.hello; + outputs = { self, nixpkgs }: + let + pkgs = nixpkgs.legacyPackages.x86_64-linux; + in + { + hydraJobs = { + builtJob = pkgs.writeText "job1" "job1"; + substitutedJob = pkgs.hello; + }; }; - }; } From a6975281430d25ec69ba1136bb1e7b1e0d3417f0 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Fri, 24 Sep 2021 22:34:46 -0500 Subject: [PATCH 046/419] Attempt to fix clasical Nix tests --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9b2ebf67a..2518d15e7 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -19,4 +19,4 @@ jobs: - uses: actions/checkout@v2 - uses: cachix/install-nix-action@v14 - name: build - run: NIX_PATH=nixpkgs=https://github.com/NixOS/nixpkgs/archive/nixpkgs-unstable.tar.gz nix-build + run: NIX_PATH=nixpkgs=https://github.com/NixOS/nixpkgs/archive/nixpkgs-unstable.tar.gz nix-build --expr '(import { }).callPackage ./. { }' From 20751faaffdbb8643e07122683c5c67a5fcf1383 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sat, 9 Oct 2021 12:46:01 +0200 Subject: [PATCH 047/419] bump nixpkgs --- flake.lock | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/flake.lock b/flake.lock index 39eba216e..cd313f517 100644 --- a/flake.lock +++ b/flake.lock @@ -2,11 +2,11 @@ "nodes": { "flake-utils": { "locked": { - "lastModified": 1629481132, - "narHash": "sha256-JHgasjPR0/J1J3DRm4KxM4zTyAj4IOJY8vIl75v/kPI=", + "lastModified": 1631561581, + "narHash": "sha256-3VQMV5zvxaVLvqqUrNz3iJelLw30mIVSfZmAaauM3dA=", "owner": "numtide", "repo": "flake-utils", - "rev": "997f7efcb746a9c140ce1f13c72263189225f482", + "rev": "7e5bf3925f6fbdfaf50a2a7ca0be2879c4261d19", "type": "github" }, "original": { @@ -17,11 +17,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1629846896, - "narHash": "sha256-3uVhnCgWHymKQKtVNspxNJDLXCY4uiGlCPcnib5w1PY=", + "lastModified": 1633770157, + "narHash": "sha256-XARYW5Txxdu2DDFPEJNh2Mds3tp3/UgZ2YvNZZc4c+o=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "1c9124b63bf1c12f420b023836ea770e05adeafa", + "rev": "dd713915de0133cd8132e2117e8f13970f16ebd2", "type": "github" }, "original": { From f4026e6bf36e0777ecd24a699454be95ad9e89cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sat, 9 Oct 2021 12:46:50 +0200 Subject: [PATCH 048/419] drop nixPath from matrix --- .github/workflows/test.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2518d15e7..f554a219b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,9 +10,6 @@ jobs: tests: strategy: matrix: - nixPath: - - nixpkgs=https://github.com/NixOS/nixpkgs/archive/nixos-21.05.tar.gz - - nixpkgs=https://github.com/NixOS/nixpkgs/archive/nixpkgs-unstable.tar.gz os: [ ubuntu-latest, macos-latest ] runs-on: ${{ matrix.os }} steps: From a22fa0b97b7f2f89e8b35a28c579348e3ea356fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sat, 9 Oct 2021 12:50:39 +0200 Subject: [PATCH 049/419] bump nix-unstable --- .github/workflows/test-flakes.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-flakes.yml b/.github/workflows/test-flakes.yml index 75e8e2fad..959bfae1a 100644 --- a/.github/workflows/test-flakes.yml +++ b/.github/workflows/test-flakes.yml @@ -19,7 +19,7 @@ jobs: fetch-depth: 0 - uses: cachix/install-nix-action@v14 with: - install_url: https://github.com/numtide/nix-flakes-installer/releases/download/nix-2.4pre20210207_fd6eaa1/install + install_url: https://github.com/numtide/nix-unstable-installer/releases/download/nix-2.4pre20210823_af94b54/install extra_nix_config: | experimental-features = nix-command flakes system-features = nixos-test benchmark big-parallel kvm From 28ff04751e65a03830a4f0ed239e61a149a78023 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sat, 9 Oct 2021 13:20:05 +0200 Subject: [PATCH 050/419] add a real README --- README.md | 74 ++++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 73 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 3bb702869..cbaf702df 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,74 @@ # nix-eval-jobs -Eval nix expressions from flakes (extracted from hydra) + +This project evaluates nix attributes sets in parallel with a streamable json output. +This is useful for time and memory-intensive evaluations such as nixos machines i.e. in a CI context. +Evaluation happens with a controlable number of threads that are restarted if +their memory consumption grows beyond a threshold. + +For ease of integration nix-eval-jobs creates garbage collection roots for each +evaluated derivation (drv file not the build) inside the supplied attribute. +This prevent race conditions between nix garbage collection service and nix +builds processes started by the user. + +## Why using nix-eval-jobs? + +- Faster evaluation due the use of threads +- Memory used for evaluation is reclaimed after nix-eval-jobs is finished so that the build can use it. +- Evaluation of jobs can fail individually + +## Example + +In the following example we evaluate the hydraJobs attribute of the [patchelf](https://github.com/NixOS/patchelf) flake: + +```console +$ nix-eval-jobs --gc-roots-dir $(pwd)/gcroot --flake 'github:NixOS/patchelf#hydraJobs' +{"attr":"build-sanitized-clang.aarch64-linux","drvPath":"/nix/store/361mr6bzzwcv65sp0bhbakaa21fj4p1b-patchelf-0.13.20210926.18a389b.drv","name":"patchelf-0.13.20210926.18a389b","system":"x86_64-linux"} +{"attr":"build-sanitized-clang.i686-linux","drvPath":"/nix/store/ial7z46jy8kivmq5dz6f9vqr0b70jqkd-patchelf-0.13.20210926.18a389b.drv","name":"patchelf-0.13.20210926.18a389b","system":"x86_64-linux"} +{"attr":"build-sanitized-clang.x86_64-linux","drvPath":"/nix/store/h2m3k085m21gd3cxc4n1wzhcjv3iap9m-patchelf-0.13.20210926.18a389b.drv","name":"patchelf-0.13.20210926.18a389b","system":"x86_64-linux"} +{"attr":"build-sanitized.aarch64-linux","drvPath":"/nix/store/m9jl25lcwvdk8rz79ibzd55wqfaxhdxx-patchelf-0.13.20210926.18a389b.drv","name":"patchelf-0.13.20210926.18a389b","system":"aarch64-linux"} +{"attr":"build-sanitized.i686-linux","drvPath":"/nix/store/0njjscgha4smzd9qsi4839pbsyqs18zl-patchelf-0.13.20210926.18a389b.drv","name":"patchelf-0.13.20210926.18a389b","system":"i686-linux"} +{"attr":"build-sanitized.x86_64-linux","drvPath":"/nix/store/cp8z7idqzf2cvfj9lzyr3xqll26bbz76-patchelf-0.13.20210926.18a389b.drv","name":"patchelf-0.13.20210926.18a389b","system":"x86_64-linux"} +{"attr":"build.aarch64-linux","drvPath":"/nix/store/rsgwdq3503ibln8hwilbl8ifjhrlb9mv-patchelf-0.13.20210926.18a389b.drv","name":"patchelf-0.13.20210926.18a389b","system":"aarch64-linux"} +{"attr":"build.i686-linux","drvPath":"/nix/store/l5k6ma3lrb2rmbw50s8s8x4c4wvj35s7-patchelf-0.13.20210926.18a389b.drv","name":"patchelf-0.13.20210926.18a389b","system":"i686-linux"} +{"attr":"build.x86_64-linux","drvPath":"/nix/store/lmhpwvj4y9ypz5rgp0y1jbw2vqryc80l-patchelf-0.13.20210926.18a389b.drv","name":"patchelf-0.13.20210926.18a389b","system":"x86_64-linux"} +{"attr":"coverage","drvPath":"/nix/store/hlh7x41c2nnklbnhrc41wm2rir0l3zq3-patchelf-coverage-0.13.20210926.18a389b.drv","name":"patchelf-coverage-0.13.20210926.18a389b","system":"x86_64-linux"} +{"attr":"release","drvPath":"/nix/store/b1jfn3pjdhq1ds4d52sj8k2z33lmb3jk-patchelf-0.13.20210926.18a389b.drv","name":"patchelf-0.13.20210926.18a389b","system":"x86_64-linux"} +{"attr":"tarball","drvPath":"/nix/store/jcharij3ylh36hvszb48j2pzjas9hmx1-patchelf-tarball-0.13.20210926.18a389b.drv","name":"patchelf-tarball-0.13.20210926.18a389b","system":"x86_64-linux"} +``` + +The output here newline-seperated json according to https://jsonlines.org/ + +The code is derived from [hydra's](https://github.com/nixos/hydra) eval-jobs executable. + +## Further options + +``` console +$ nix-eval-jobs --help +USAGE: nix-eval-jobs [options] expr + + --arg Pass the value *expr* as the argument *name* to Nix functions. + --argstr Pass the string *string* as the argument *name* to Nix functions. + --debug Set the logging verbosity level to 'debug'. + --eval-store The Nix store to use for evaluations. + --flake build a flake + --gc-roots-dir garbage collector roots directory + --help show usage information + --impure set evaluation mode + --include Add *path* to the list of locations used to look up `<...>` file names. + --log-format Set the format of log output; one of `raw`, `internal-json`, `bar` or `bar-with-logs`. + --max-memory-size maximum evaluation memory size + --option Set the Nix configuration setting *name* to *value* (overriding `nix.conf`). + --override-flake Override the flake registries, redirecting *original-ref* to *resolved-ref*. + --quiet Decrease the logging verbosity level. + --verbose Increase the logging verbosity level. + --workers number of evaluate workers +``` + + +## Potential use-cases for the tool + +**Faster evaluator in deployment tools.** When evaluating nixos machines evaluation can take several minutes when performed on a single core. +This limits the usuability of current deployment tools such as [NixOps](https://github.com/NixOS/nixops). +**Faster evaluator in CI.** In addition to evaluation speed for CIs it is also useful if evaluation of individual jobs can fail in CIs in contrast to failing the whole jobset. +Furthermore for CIs that allow to create dynamic build steps, one can leverage the fact that nix-eval-jobs outputs derivation path seperatly. +This allows to have seperate logs and success status per job rather than one big log file. From b196a08ee3aa1d0f258b12d0c0f28037816d00fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sat, 9 Oct 2021 12:26:22 +0100 Subject: [PATCH 051/419] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index cbaf702df..79daacdc0 100644 --- a/README.md +++ b/README.md @@ -68,7 +68,7 @@ USAGE: nix-eval-jobs [options] expr ## Potential use-cases for the tool **Faster evaluator in deployment tools.** When evaluating nixos machines evaluation can take several minutes when performed on a single core. -This limits the usuability of current deployment tools such as [NixOps](https://github.com/NixOS/nixops). +This limits the scalability for large deployment with deployment tools such as [NixOps](https://github.com/NixOS/nixops). **Faster evaluator in CI.** In addition to evaluation speed for CIs it is also useful if evaluation of individual jobs can fail in CIs in contrast to failing the whole jobset. Furthermore for CIs that allow to create dynamic build steps, one can leverage the fact that nix-eval-jobs outputs derivation path seperatly. This allows to have seperate logs and success status per job rather than one big log file. From aa86d0c567fcc1c144812b1a012f3d1463baeec8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 15 Nov 2021 23:11:14 +0000 Subject: [PATCH 052/419] Bump cachix/install-nix-action from 14 to 15 Bumps [cachix/install-nix-action](https://github.com/cachix/install-nix-action) from 14 to 15. - [Release notes](https://github.com/cachix/install-nix-action/releases) - [Commits](https://github.com/cachix/install-nix-action/compare/v14...v15) --- updated-dependencies: - dependency-name: cachix/install-nix-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/test-flakes.yml | 2 +- .github/workflows/test.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-flakes.yml b/.github/workflows/test-flakes.yml index 959bfae1a..bb920e660 100644 --- a/.github/workflows/test-flakes.yml +++ b/.github/workflows/test-flakes.yml @@ -17,7 +17,7 @@ jobs: with: # Nix Flakes doesn't work on shallow clones fetch-depth: 0 - - uses: cachix/install-nix-action@v14 + - uses: cachix/install-nix-action@v15 with: install_url: https://github.com/numtide/nix-unstable-installer/releases/download/nix-2.4pre20210823_af94b54/install extra_nix_config: | diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f554a219b..3cc8f030a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -14,6 +14,6 @@ jobs: runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v2 - - uses: cachix/install-nix-action@v14 + - uses: cachix/install-nix-action@v15 - name: build run: NIX_PATH=nixpkgs=https://github.com/NixOS/nixpkgs/archive/nixpkgs-unstable.tar.gz nix-build --expr '(import { }).callPackage ./. { }' From b7230f5e4b4f52ffc4e58721751bcb0eb7aa41d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Tue, 16 Nov 2021 08:48:04 +0100 Subject: [PATCH 053/419] ci: drop hard-coded flake installer --- .github/workflows/test-flakes.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/test-flakes.yml b/.github/workflows/test-flakes.yml index bb920e660..51f5c5451 100644 --- a/.github/workflows/test-flakes.yml +++ b/.github/workflows/test-flakes.yml @@ -18,11 +18,6 @@ jobs: # Nix Flakes doesn't work on shallow clones fetch-depth: 0 - uses: cachix/install-nix-action@v15 - with: - install_url: https://github.com/numtide/nix-unstable-installer/releases/download/nix-2.4pre20210823_af94b54/install - extra_nix_config: | - experimental-features = nix-command flakes - system-features = nixos-test benchmark big-parallel kvm - name: List flake structure run: nix flake show - name: Build From 42ea2ec12e5b45999f8b2361c9b1c9a383cbf5b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Tue, 16 Nov 2021 08:48:08 +0100 Subject: [PATCH 054/419] update flakes --- flake.lock | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/flake.lock b/flake.lock index cd313f517..ea8caf834 100644 --- a/flake.lock +++ b/flake.lock @@ -2,11 +2,11 @@ "nodes": { "flake-utils": { "locked": { - "lastModified": 1631561581, - "narHash": "sha256-3VQMV5zvxaVLvqqUrNz3iJelLw30mIVSfZmAaauM3dA=", + "lastModified": 1637014545, + "narHash": "sha256-26IZAc5yzlD9FlDT54io1oqG/bBoyka+FJk5guaX4x4=", "owner": "numtide", "repo": "flake-utils", - "rev": "7e5bf3925f6fbdfaf50a2a7ca0be2879c4261d19", + "rev": "bba5dcc8e0b20ab664967ad83d24d64cb64ec4f4", "type": "github" }, "original": { @@ -17,11 +17,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1633770157, - "narHash": "sha256-XARYW5Txxdu2DDFPEJNh2Mds3tp3/UgZ2YvNZZc4c+o=", + "lastModified": 1637048323, + "narHash": "sha256-h2m2LuUEbV5pvNjXoWbk2rAaBAQAsf3cvJXS8DGi+Bo=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "dd713915de0133cd8132e2117e8f13970f16ebd2", + "rev": "6b68d74f036def3d48a23969edfd0fa47f34af3e", "type": "github" }, "original": { From c3ed61b0752ad3a9f923de2aac58effbabd849fc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 22 Nov 2021 23:09:53 +0000 Subject: [PATCH 055/419] Bump cachix/install-nix-action from 15 to 16 Bumps [cachix/install-nix-action](https://github.com/cachix/install-nix-action) from 15 to 16. - [Release notes](https://github.com/cachix/install-nix-action/releases) - [Commits](https://github.com/cachix/install-nix-action/compare/v15...v16) --- updated-dependencies: - dependency-name: cachix/install-nix-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/test-flakes.yml | 2 +- .github/workflows/test.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-flakes.yml b/.github/workflows/test-flakes.yml index 51f5c5451..8b8d29fdd 100644 --- a/.github/workflows/test-flakes.yml +++ b/.github/workflows/test-flakes.yml @@ -17,7 +17,7 @@ jobs: with: # Nix Flakes doesn't work on shallow clones fetch-depth: 0 - - uses: cachix/install-nix-action@v15 + - uses: cachix/install-nix-action@v16 - name: List flake structure run: nix flake show - name: Build diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3cc8f030a..ce115e5d8 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -14,6 +14,6 @@ jobs: runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v2 - - uses: cachix/install-nix-action@v15 + - uses: cachix/install-nix-action@v16 - name: build run: NIX_PATH=nixpkgs=https://github.com/NixOS/nixpkgs/archive/nixpkgs-unstable.tar.gz nix-build --expr '(import { }).callPackage ./. { }' From c297bd9564e84c3c14999b269bab14eeeef0a246 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Wed, 15 Dec 2021 14:20:38 +0100 Subject: [PATCH 056/419] fix build against nixUnstable --- default.nix | 4 ++-- flake.lock | 6 +++--- src/nix-eval-jobs.cc | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/default.nix b/default.nix index abfcda46a..b05231781 100644 --- a/default.nix +++ b/default.nix @@ -1,6 +1,6 @@ { stdenv , lib -, nixFlakes +, nixUnstable , meson , cmake , ninja @@ -21,7 +21,7 @@ stdenv.mkDerivation rec { src = if srcDir == null then filterMesonBuild ./. else srcDir; buildInputs = [ nlohmann_json - nixFlakes + nixUnstable boost ]; nativeBuildInputs = [ diff --git a/flake.lock b/flake.lock index ea8caf834..6063da830 100644 --- a/flake.lock +++ b/flake.lock @@ -17,11 +17,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1637048323, - "narHash": "sha256-h2m2LuUEbV5pvNjXoWbk2rAaBAQAsf3cvJXS8DGi+Bo=", + "lastModified": 1639573498, + "narHash": "sha256-YAoywqjyjOJYuEZpA8sln84jk99GG9hYbPEYXwGEGGM=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "6b68d74f036def3d48a23969edfd0fa47f34af3e", + "rev": "aef12c8678fd5e927edba764bd04f3ba2930ae15", "type": "github" }, "original": { diff --git a/src/nix-eval-jobs.cc b/src/nix-eval-jobs.cc index c07ca9b40..b6614b0da 100644 --- a/src/nix-eval-jobs.cc +++ b/src/nix-eval-jobs.cc @@ -188,7 +188,7 @@ static void worker( continue; } - printValueAsJSON(state, true, *metaValue, ss, context); + printValueAsJSON(state, true, *metaValue, noPos, ss, context); nlohmann::json field = nlohmann::json::parse(ss.str()); meta[name] = field; } From 26948992b3d64309e7946014301778b3c7dbbdb3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Fri, 31 Dec 2021 12:00:04 +0100 Subject: [PATCH 057/419] also expose store path in json --- src/nix-eval-jobs.cc | 7 ++++--- tests/test_eval.py | 2 ++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/nix-eval-jobs.cc b/src/nix-eval-jobs.cc index b6614b0da..8ee192a08 100644 --- a/src/nix-eval-jobs.cc +++ b/src/nix-eval-jobs.cc @@ -171,10 +171,13 @@ static void worker( throw EvalError("derivation must have a 'system' attribute"); auto drvPath = drv->queryDrvPath(); + auto localStore = state.store.dynamic_pointer_cast(); + auto storePath = localStore->parseStorePath(drvPath); reply["name"] = drv->queryName(); reply["system"] = drv->querySystem(); reply["drvPath"] = drvPath; + reply["storePath"] = localStore->printStorePath(storePath); nlohmann::json meta; for (auto & name : drv->queryMetaNames()) { @@ -196,9 +199,7 @@ static void worker( /* Register the derivation as a GC root. !!! This registers roots for jobs that we may have already done. */ - auto localStore = state.store.dynamic_pointer_cast(); - auto storePath = localStore->parseStorePath(drvPath); - if (gcRootsDir != "" && localStore) { + if (gcRootsDir != "") { Path root = gcRootsDir + "/" + std::string(baseNameOf(drvPath)); if (!pathExists(root)) localStore->addPermRoot(storePath, root); diff --git a/tests/test_eval.py b/tests/test_eval.py index 32a31e77c..404985c5e 100644 --- a/tests/test_eval.py +++ b/tests/test_eval.py @@ -28,6 +28,8 @@ def common_test(extra_args: List[str]) -> None: built_job = results[0] assert built_job["attr"] == "builtJob" assert built_job["name"] == "job1" + assert built_job["storePath"].startswith("/nix/store") + assert built_job["drvPath"].endswith(".drv") substituted_job = results[1] From d36d77c8737f88bb801a2799c5c81282a04987a3 Mon Sep 17 00:00:00 2001 From: Tom Bereknyei Date: Tue, 4 Jan 2022 03:43:06 -0500 Subject: [PATCH 058/419] use outPath --- src/nix-eval-jobs.cc | 5 ++++- tests/test_eval.py | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/nix-eval-jobs.cc b/src/nix-eval-jobs.cc index 8ee192a08..996d5f209 100644 --- a/src/nix-eval-jobs.cc +++ b/src/nix-eval-jobs.cc @@ -173,11 +173,14 @@ static void worker( auto drvPath = drv->queryDrvPath(); auto localStore = state.store.dynamic_pointer_cast(); auto storePath = localStore->parseStorePath(drvPath); + auto outputs = drv->queryOutputs(false); reply["name"] = drv->queryName(); reply["system"] = drv->querySystem(); reply["drvPath"] = drvPath; - reply["storePath"] = localStore->printStorePath(storePath); + for (auto out : outputs){ + reply["outputs"][out.first] = out.second; + } nlohmann::json meta; for (auto & name : drv->queryMetaNames()) { diff --git a/tests/test_eval.py b/tests/test_eval.py index 404985c5e..7df722a6c 100644 --- a/tests/test_eval.py +++ b/tests/test_eval.py @@ -28,7 +28,7 @@ def common_test(extra_args: List[str]) -> None: built_job = results[0] assert built_job["attr"] == "builtJob" assert built_job["name"] == "job1" - assert built_job["storePath"].startswith("/nix/store") + assert built_job["outputs"]["out"].startswith("/nix/store") assert built_job["drvPath"].endswith(".drv") From 3268f3a6affd6a904bb5e4ac1266a5ea57c71afe Mon Sep 17 00:00:00 2001 From: adisbladis Date: Thu, 6 Jan 2022 13:19:15 +1300 Subject: [PATCH 059/419] Add flag to enable meta I removed meta from the output in https://github.com/nix-community/nix-eval-jobs/commit/434376f8e1ef653e79b4c2b5a38cba45109152e4 with the intention of adding it back gated by a flag, but that never happened. Adding meta is quite a substantial increase in output size and has some non-trivial performance impact at scale, so it's best to leave it as opt-in. --- src/nix-eval-jobs.cc | 36 +++++++++++++++++++++++------------- tests/test_eval.py | 5 +++-- 2 files changed, 26 insertions(+), 15 deletions(-) diff --git a/src/nix-eval-jobs.cc b/src/nix-eval-jobs.cc index 996d5f209..47f202b91 100644 --- a/src/nix-eval-jobs.cc +++ b/src/nix-eval-jobs.cc @@ -34,6 +34,7 @@ struct MyArgs : MixEvalArgs, MixCommonArgs Path releaseExpr; Path gcRootsDir; bool flake = false; + bool meta = false; size_t nrWorkers = 1; size_t maxMemorySize = 4096; pureEval evalMode = evalAuto; @@ -94,6 +95,12 @@ struct MyArgs : MixEvalArgs, MixCommonArgs .handler = {&flake, true} }); + addFlag({ + .longName = "meta", + .description = "include derivation meta field in output", + .handler = {&meta, true} + }); + expectArg("expr", &releaseExpr); } }; @@ -182,21 +189,24 @@ static void worker( reply["outputs"][out.first] = out.second; } - nlohmann::json meta; - for (auto & name : drv->queryMetaNames()) { - PathSet context; - std::stringstream ss; + if (myArgs.meta) { + nlohmann::json meta; + for (auto & name : drv->queryMetaNames()) { + PathSet context; + std::stringstream ss; - auto metaValue = drv->queryMeta(name); - // Skip non-serialisable types - // TODO: Fix serialisation of derivations to store paths - if (metaValue == 0) { - continue; - } + auto metaValue = drv->queryMeta(name); + // Skip non-serialisable types + // TODO: Fix serialisation of derivations to store paths + if (metaValue == 0) { + continue; + } - printValueAsJSON(state, true, *metaValue, noPos, ss, context); - nlohmann::json field = nlohmann::json::parse(ss.str()); - meta[name] = field; + printValueAsJSON(state, true, *metaValue, noPos, ss, context); + nlohmann::json field = nlohmann::json::parse(ss.str()); + meta[name] = field; + } + reply["meta"] = meta; } /* Register the derivation as a GC root. !!! This diff --git a/tests/test_eval.py b/tests/test_eval.py index 7df722a6c..db93f9745 100644 --- a/tests/test_eval.py +++ b/tests/test_eval.py @@ -13,7 +13,7 @@ BIN = PROJECT_ROOT.joinpath("build", "src", "nix-eval-jobs") def common_test(extra_args: List[str]) -> None: with TemporaryDirectory() as tempdir: - cmd = [str(BIN), "--gc-roots-dir", tempdir] + extra_args + cmd = [str(BIN), "--gc-roots-dir", tempdir, "--meta"] + extra_args res = subprocess.run( cmd, cwd=TEST_ROOT.joinpath("assets"), @@ -30,11 +30,12 @@ def common_test(extra_args: List[str]) -> None: assert built_job["name"] == "job1" assert built_job["outputs"]["out"].startswith("/nix/store") assert built_job["drvPath"].endswith(".drv") - + assert built_job["meta"]['broken'] is False substituted_job = results[1] assert substituted_job["attr"] == "substitutedJob" assert substituted_job["name"].startswith("hello-") + assert substituted_job["meta"]['broken'] is False def test_flake() -> None: From 2cc698ef4e5880eb3a571a11b70292da679f7ed6 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Thu, 6 Jan 2022 13:51:00 +1300 Subject: [PATCH 060/419] Add meta field to README help section --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 79daacdc0..6aaf25d5a 100644 --- a/README.md +++ b/README.md @@ -57,6 +57,7 @@ USAGE: nix-eval-jobs [options] expr --include Add *path* to the list of locations used to look up `<...>` file names. --log-format Set the format of log output; one of `raw`, `internal-json`, `bar` or `bar-with-logs`. --max-memory-size maximum evaluation memory size + --meta include derivation meta field in output --option Set the Nix configuration setting *name* to *value* (overriding `nix.conf`). --override-flake Override the flake registries, redirecting *original-ref* to *resolved-ref*. --quiet Decrease the logging verbosity level. From 1e0f309fefc9b2d597f8475a74c82ce29c189152 Mon Sep 17 00:00:00 2001 From: Zhaofeng Li Date: Thu, 6 Jan 2022 23:20:41 -0800 Subject: [PATCH 061/419] Add flag to enable trace output --- src/nix-eval-jobs.cc | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/src/nix-eval-jobs.cc b/src/nix-eval-jobs.cc index 47f202b91..f9ca986dc 100644 --- a/src/nix-eval-jobs.cc +++ b/src/nix-eval-jobs.cc @@ -16,6 +16,8 @@ #include #include #include +#include +#include #include @@ -35,6 +37,7 @@ struct MyArgs : MixEvalArgs, MixCommonArgs Path gcRootsDir; bool flake = false; bool meta = false; + bool showTrace = false; size_t nrWorkers = 1; size_t maxMemorySize = 4096; pureEval evalMode = evalAuto; @@ -101,6 +104,12 @@ struct MyArgs : MixEvalArgs, MixCommonArgs .handler = {&meta, true} }); + addFlag({ + .longName = "show-trace", + .description = "print out a stack trace in case of evaluation errors", + .handler = {&showTrace, true} + }); + expectArg("expr", &releaseExpr); } }; @@ -241,13 +250,18 @@ static void worker( else throw TypeError("attribute '%s' is %s, which is not supported", attrPath, showType(*v)); } catch (EvalError & e) { - auto msg = e.msg(); + auto err = e.info(); + + std::ostringstream oss; + showErrorInfo(oss, err, loggerSettings.showTrace.get()); + auto msg = oss.str(); + // Transmits the error we got from the previous evaluation // in the JSON output. reply["error"] = filterANSIEscapes(msg, true); // Don't forget to print it into the STDERR log, this is // what's shown in the Hydra UI. - printError(msg); + printError(e.msg()); } writeLine(to.get(), reply.dump()); @@ -289,6 +303,10 @@ int main(int argc, char * * argv) if (myArgs.gcRootsDir == "") printMsg(lvlError, "warning: `--gc-roots-dir' not specified"); + if (myArgs.showTrace) { + loggerSettings.showTrace.assign(true); + } + struct State { std::set todo{""}; From 985096e89d146e3814f5ead9e448a3a9fcfc6b1e Mon Sep 17 00:00:00 2001 From: adisbladis Date: Sat, 8 Jan 2022 08:41:07 +1300 Subject: [PATCH 062/419] Bump flake inputs --- flake.lock | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/flake.lock b/flake.lock index 6063da830..3f2e2f553 100644 --- a/flake.lock +++ b/flake.lock @@ -2,11 +2,11 @@ "nodes": { "flake-utils": { "locked": { - "lastModified": 1637014545, - "narHash": "sha256-26IZAc5yzlD9FlDT54io1oqG/bBoyka+FJk5guaX4x4=", + "lastModified": 1638122382, + "narHash": "sha256-sQzZzAbvKEqN9s0bzWuYmRaA03v40gaJ4+iL1LXjaeI=", "owner": "numtide", "repo": "flake-utils", - "rev": "bba5dcc8e0b20ab664967ad83d24d64cb64ec4f4", + "rev": "74f7e4319258e287b0f9cb95426c9853b282730b", "type": "github" }, "original": { @@ -17,11 +17,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1639573498, - "narHash": "sha256-YAoywqjyjOJYuEZpA8sln84jk99GG9hYbPEYXwGEGGM=", + "lastModified": 1641577433, + "narHash": "sha256-T7lS8vpbC3dgtrkb2ueC9HWaX4RYUwdP7IEttnvKQ8Y=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "aef12c8678fd5e927edba764bd04f3ba2930ae15", + "rev": "568e0bc498ee51fdd88e1e94089de05f2fdbd18b", "type": "github" }, "original": { From 44655761c5e524814be760e19b6f024d379445b9 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Sat, 8 Jan 2022 08:53:26 +1300 Subject: [PATCH 063/419] Use locked nixpkgs version for nix-shell --- shell.nix | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/shell.nix b/shell.nix index 7b97e75ea..1def6dbad 100644 --- a/shell.nix +++ b/shell.nix @@ -1,4 +1,15 @@ -{ pkgs ? import { } +{ pkgs ? ( + let + inherit (builtins) fromJSON readFile; + flakeLock = fromJSON (readFile ./flake.lock); + locked = flakeLock.nodes.nixpkgs.locked; + nixpkgs = assert locked.type == "github"; builtins.fetchTarball { + url = "https://github.com/${locked.owner}/${locked.repo}/archive/${locked.rev}.tar.gz"; + sha256 = locked.narHash; + }; + in + import nixpkgs { } + ) , srcDir ? null }: From a0bd67f9cbdd1ad0c3938b0c626bcf2994731e3a Mon Sep 17 00:00:00 2001 From: adisbladis Date: Sat, 8 Jan 2022 08:53:59 +1300 Subject: [PATCH 064/419] Use `nix` as the argument name for `nix` in default.nix And pass in `nix = nixUnstable` from the development shell. --- default.nix | 4 ++-- shell.nix | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/default.nix b/default.nix index b05231781..1178d04fb 100644 --- a/default.nix +++ b/default.nix @@ -1,6 +1,6 @@ { stdenv , lib -, nixUnstable +, nix , meson , cmake , ninja @@ -21,7 +21,7 @@ stdenv.mkDerivation rec { src = if srcDir == null then filterMesonBuild ./. else srcDir; buildInputs = [ nlohmann_json - nixUnstable + nix boost ]; nativeBuildInputs = [ diff --git a/shell.nix b/shell.nix index 1def6dbad..70b4e37ab 100644 --- a/shell.nix +++ b/shell.nix @@ -13,7 +13,10 @@ , srcDir ? null }: -(pkgs.callPackage ./default.nix { inherit srcDir; }).overrideAttrs (old: { +(pkgs.callPackage ./default.nix { + inherit srcDir; + nix = pkgs.nixUnstable; +}).overrideAttrs (old: { nativeBuildInputs = old.nativeBuildInputs ++ [ From 9bcb1bb3fc95de1f29cba9999c284db6285f300a Mon Sep 17 00:00:00 2001 From: adisbladis Date: Sat, 8 Jan 2022 09:06:12 +1300 Subject: [PATCH 065/419] Run build against both stable nix and nixUnstable in CI --- flake.nix | 51 +++++++++++++++++++++++++++++---------------------- 1 file changed, 29 insertions(+), 22 deletions(-) diff --git a/flake.nix b/flake.nix index 6e4dd50a8..963055a0a 100644 --- a/flake.nix +++ b/flake.nix @@ -13,31 +13,38 @@ rec { packages.nix-eval-jobs = pkgs.callPackage ./default.nix drvArgs; - checks = { + checks = + let + mkVariant = nix: packages.nix-eval-jobs.overrideAttrs (_: { + name = "nix-eval-jobs-${nix.version}"; + inherit (nix) version; + }); + in + { - editorconfig = pkgs.runCommand "editorconfig-check" - { - nativeBuildInputs = [ - pkgs.editorconfig-checker - ]; - } '' - editorconfig-checker ${self} - touch $out - ''; + editorconfig = pkgs.runCommand "editorconfig-check" + { + nativeBuildInputs = [ + pkgs.editorconfig-checker + ]; + } '' + editorconfig-checker ${self} + touch $out + ''; - nixpkgs-fmt = pkgs.runCommand "fmt-check" - { - nativeBuildInputs = [ - pkgs.nixpkgs-fmt - ]; - } '' - nixpkgs-fmt --check . - touch $out - ''; + nixpkgs-fmt = pkgs.runCommand "fmt-check" + { + nativeBuildInputs = [ + pkgs.nixpkgs-fmt + ]; + } '' + nixpkgs-fmt --check . + touch $out + ''; - build = packages.nix-eval-jobs; - - }; + build = mkVariant pkgs.nix; + build-unstable = mkVariant pkgs.nixUnstable; + }; defaultPackage = self.packages.${system}.nix-eval-jobs; devShell = pkgs.callPackage ./shell.nix drvArgs; From f343083a49c7bba6db24929bd28304757bd60739 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Sat, 8 Jan 2022 09:17:15 +1300 Subject: [PATCH 066/419] Add github action to auto-update flake.lock --- .github/workflows/update-flake-lock.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 .github/workflows/update-flake-lock.yml diff --git a/.github/workflows/update-flake-lock.yml b/.github/workflows/update-flake-lock.yml new file mode 100644 index 000000000..e53ba9315 --- /dev/null +++ b/.github/workflows/update-flake-lock.yml @@ -0,0 +1,19 @@ +name: update-flake-lock +on: + workflow_dispatch: # allows manual triggering + schedule: + - cron: '0 0 * * 0' # runs weekly on Sunday at 00:00 + +jobs: + lockfile: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v2 + - name: Install Nix + uses: cachix/install-nix-action@v16 + # with: + # extra_nix_config: | + # access-tokens = github.com=${{ secrets.GITHUB_TOKEN }} + - name: Update flake.lock + uses: DeterminateSystems/update-flake-lock@v3 From 0f6ac00feaf6da743b5f023964f4fbbeb04724b9 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Sat, 8 Jan 2022 09:32:53 +1300 Subject: [PATCH 067/419] Use the NixOS unstable channel --- flake.lock | 7 ++++--- flake.nix | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/flake.lock b/flake.lock index 3f2e2f553..0dfc6918b 100644 --- a/flake.lock +++ b/flake.lock @@ -17,15 +17,16 @@ }, "nixpkgs": { "locked": { - "lastModified": 1641577433, - "narHash": "sha256-T7lS8vpbC3dgtrkb2ueC9HWaX4RYUwdP7IEttnvKQ8Y=", + "lastModified": 1641528457, + "narHash": "sha256-FyU9E63n1W7Ql4pMnhW2/rO9OftWZ37pLppn/c1aisY=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "568e0bc498ee51fdd88e1e94089de05f2fdbd18b", + "rev": "ff377a78794d412a35245e05428c8f95fef3951f", "type": "github" }, "original": { "owner": "NixOS", + "ref": "nixos-unstable", "repo": "nixpkgs", "type": "github" } diff --git a/flake.nix b/flake.nix index 963055a0a..ec269a0ba 100644 --- a/flake.nix +++ b/flake.nix @@ -1,7 +1,7 @@ { description = "Hydra's builtin hydra-eval-jobs as a standalone"; - inputs.nixpkgs.url = "github:NixOS/nixpkgs"; + inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; inputs.flake-utils.url = "github:numtide/flake-utils"; outputs = { self, nixpkgs, flake-utils }: From a0966f6d2ee5b3a77ac0bd7227b57580886c7f2f Mon Sep 17 00:00:00 2001 From: adisbladis Date: Sat, 8 Jan 2022 09:36:33 +1300 Subject: [PATCH 068/419] Update flake auto-update cron schedule to run twice a week --- .github/workflows/update-flake-lock.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/update-flake-lock.yml b/.github/workflows/update-flake-lock.yml index e53ba9315..5a3d2326f 100644 --- a/.github/workflows/update-flake-lock.yml +++ b/.github/workflows/update-flake-lock.yml @@ -2,7 +2,7 @@ name: update-flake-lock on: workflow_dispatch: # allows manual triggering schedule: - - cron: '0 0 * * 0' # runs weekly on Sunday at 00:00 + - cron: '0 0 * * 1,4' # Run twice a week jobs: lockfile: From 1c6887b5bf8b9156bc4531d03bd13dbcd4b8414f Mon Sep 17 00:00:00 2001 From: adisbladis Date: Sat, 8 Jan 2022 21:14:05 +1300 Subject: [PATCH 069/419] ci: Remove cron schedules We're using a pinned auto-updater workflow instead which is deterministic. --- .github/workflows/test-flakes.yml | 2 -- .github/workflows/test.yml | 2 -- 2 files changed, 4 deletions(-) diff --git a/.github/workflows/test-flakes.yml b/.github/workflows/test-flakes.yml index 8b8d29fdd..ea6de7880 100644 --- a/.github/workflows/test-flakes.yml +++ b/.github/workflows/test-flakes.yml @@ -4,8 +4,6 @@ on: push: branches: - main - schedule: - - cron: '51 2 * * *' jobs: tests: strategy: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ce115e5d8..44c648895 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -4,8 +4,6 @@ on: push: branches: - main - schedule: - - cron: '51 2 * * *' jobs: tests: strategy: From 5b9621cd6ce89cb5451376e2d0aa44d8c48b657e Mon Sep 17 00:00:00 2001 From: adisbladis Date: Sat, 8 Jan 2022 21:22:01 +1300 Subject: [PATCH 070/419] Remove old tests and add development workflow tests --- .../workflows/{test.yml => test-develop-classic.yml} | 10 +++++++--- .../{test-flakes.yml => test-develop-flakes.yml} | 10 +++------- 2 files changed, 10 insertions(+), 10 deletions(-) rename .github/workflows/{test.yml => test-develop-classic.yml} (50%) rename .github/workflows/{test-flakes.yml => test-develop-flakes.yml} (60%) diff --git a/.github/workflows/test.yml b/.github/workflows/test-develop-classic.yml similarity index 50% rename from .github/workflows/test.yml rename to .github/workflows/test-develop-classic.yml index 44c648895..0455e3006 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test-develop-classic.yml @@ -1,4 +1,4 @@ -name: "Test" +name: "Development workflow - nix-shell" on: pull_request: push: @@ -13,5 +13,9 @@ jobs: steps: - uses: actions/checkout@v2 - uses: cachix/install-nix-action@v16 - - name: build - run: NIX_PATH=nixpkgs=https://github.com/NixOS/nixpkgs/archive/nixpkgs-unstable.tar.gz nix-build --expr '(import { }).callPackage ./. { }' + with: + nix_path: nixpkgs=channel:nixos-unstable + - name: Build + run: nix-shell --run 'meson build && cd build && ninja' + - name: Run tests + run: nix-shell --run 'pytest ./tests' diff --git a/.github/workflows/test-flakes.yml b/.github/workflows/test-develop-flakes.yml similarity index 60% rename from .github/workflows/test-flakes.yml rename to .github/workflows/test-develop-flakes.yml index ea6de7880..b4c0afaf7 100644 --- a/.github/workflows/test-flakes.yml +++ b/.github/workflows/test-develop-flakes.yml @@ -1,4 +1,4 @@ -name: "Flake test" +name: "Development workflow - flakes" on: pull_request: push: @@ -16,11 +16,7 @@ jobs: # Nix Flakes doesn't work on shallow clones fetch-depth: 0 - uses: cachix/install-nix-action@v16 - - name: List flake structure - run: nix flake show - name: Build - run: nix build --out-link result + run: nix develop -c bash -c 'meson build && cd build && ninja' - name: Run tests - run: | - nix develop -c install -D ./result/bin/nix-eval-jobs ./build/src/nix-eval-jobs - nix develop -c pytest ./tests + run: nix develop -c pytest ./tests From 49020f2dc855621d249da0b41f78d862cc66d72f Mon Sep 17 00:00:00 2001 From: adisbladis Date: Sat, 8 Jan 2022 21:42:19 +1300 Subject: [PATCH 071/419] ci: Use a dynamic matrix generated from flake checks for github actions --- .github/workflows/flake-check.yml | 45 +++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 .github/workflows/flake-check.yml diff --git a/.github/workflows/flake-check.yml b/.github/workflows/flake-check.yml new file mode 100644 index 000000000..839ad0201 --- /dev/null +++ b/.github/workflows/flake-check.yml @@ -0,0 +1,45 @@ +name: "Flake checks" +on: + pull_request: + push: + branches: + - main +jobs: + + flake-checks: + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + steps: + - uses: actions/checkout@v2.4.0 + with: + # Nix Flakes doesn't work on shallow clones + fetch-depth: 0 + - uses: cachix/install-nix-action@v16 + with: + nix_path: nixpkgs=channel:nixos-unstable + - id: set-matrix + run: | + set -euo pipefail + + matrix="$(nix flake show --json | jq '.checks."x86_64-linux" | keys' | jq -rcM '{attr: ., os: ["ubuntu-latest", "macos-latest"]}')" + echo "::set-output name=matrix::$matrix" + + builds: + needs: flake-checks + runs-on: ${{ matrix.os }} + strategy: + matrix: ${{fromJSON(needs.flake-checks.outputs.matrix)}} + steps: + - uses: actions/checkout@v2.4.0 + with: + # Nix Flakes doesn't work on shallow clones + fetch-depth: 0 + - uses: cachix/install-nix-action@v16 + with: + nix_path: nixpkgs=channel:nixos-unstable + - run: | + set -euo pipefail + + system=$(nix-instantiate --eval --expr builtins.currentSystem | jq -r) + nix build -L .#checks.$system.${{ matrix.attr }} From 30e17128278edcce437ccdf3ec5d5d2621a51d86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sat, 8 Jan 2022 10:45:55 +0100 Subject: [PATCH 072/419] README: update output --- README.md | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 6aaf25d5a..6afc86105 100644 --- a/README.md +++ b/README.md @@ -22,18 +22,17 @@ In the following example we evaluate the hydraJobs attribute of the [patchelf](h ```console $ nix-eval-jobs --gc-roots-dir $(pwd)/gcroot --flake 'github:NixOS/patchelf#hydraJobs' -{"attr":"build-sanitized-clang.aarch64-linux","drvPath":"/nix/store/361mr6bzzwcv65sp0bhbakaa21fj4p1b-patchelf-0.13.20210926.18a389b.drv","name":"patchelf-0.13.20210926.18a389b","system":"x86_64-linux"} -{"attr":"build-sanitized-clang.i686-linux","drvPath":"/nix/store/ial7z46jy8kivmq5dz6f9vqr0b70jqkd-patchelf-0.13.20210926.18a389b.drv","name":"patchelf-0.13.20210926.18a389b","system":"x86_64-linux"} -{"attr":"build-sanitized-clang.x86_64-linux","drvPath":"/nix/store/h2m3k085m21gd3cxc4n1wzhcjv3iap9m-patchelf-0.13.20210926.18a389b.drv","name":"patchelf-0.13.20210926.18a389b","system":"x86_64-linux"} -{"attr":"build-sanitized.aarch64-linux","drvPath":"/nix/store/m9jl25lcwvdk8rz79ibzd55wqfaxhdxx-patchelf-0.13.20210926.18a389b.drv","name":"patchelf-0.13.20210926.18a389b","system":"aarch64-linux"} -{"attr":"build-sanitized.i686-linux","drvPath":"/nix/store/0njjscgha4smzd9qsi4839pbsyqs18zl-patchelf-0.13.20210926.18a389b.drv","name":"patchelf-0.13.20210926.18a389b","system":"i686-linux"} -{"attr":"build-sanitized.x86_64-linux","drvPath":"/nix/store/cp8z7idqzf2cvfj9lzyr3xqll26bbz76-patchelf-0.13.20210926.18a389b.drv","name":"patchelf-0.13.20210926.18a389b","system":"x86_64-linux"} -{"attr":"build.aarch64-linux","drvPath":"/nix/store/rsgwdq3503ibln8hwilbl8ifjhrlb9mv-patchelf-0.13.20210926.18a389b.drv","name":"patchelf-0.13.20210926.18a389b","system":"aarch64-linux"} -{"attr":"build.i686-linux","drvPath":"/nix/store/l5k6ma3lrb2rmbw50s8s8x4c4wvj35s7-patchelf-0.13.20210926.18a389b.drv","name":"patchelf-0.13.20210926.18a389b","system":"i686-linux"} -{"attr":"build.x86_64-linux","drvPath":"/nix/store/lmhpwvj4y9ypz5rgp0y1jbw2vqryc80l-patchelf-0.13.20210926.18a389b.drv","name":"patchelf-0.13.20210926.18a389b","system":"x86_64-linux"} -{"attr":"coverage","drvPath":"/nix/store/hlh7x41c2nnklbnhrc41wm2rir0l3zq3-patchelf-coverage-0.13.20210926.18a389b.drv","name":"patchelf-coverage-0.13.20210926.18a389b","system":"x86_64-linux"} -{"attr":"release","drvPath":"/nix/store/b1jfn3pjdhq1ds4d52sj8k2z33lmb3jk-patchelf-0.13.20210926.18a389b.drv","name":"patchelf-0.13.20210926.18a389b","system":"x86_64-linux"} -{"attr":"tarball","drvPath":"/nix/store/jcharij3ylh36hvszb48j2pzjas9hmx1-patchelf-tarball-0.13.20210926.18a389b.drv","name":"patchelf-tarball-0.13.20210926.18a389b","system":"x86_64-linux"} +{"attr":"build-sanitized-clang.x86_64-linux","drvPath":"/nix/store/igmkq61cwys8nj34yqvnpdg921h0i0mp-patchelf-0.14.3.drv","name":"patchelf-0.14.3","outputs":{"out":"/nix/store/nwwgff1fwkws4wxv7k7cfvvin8ab9gbh-patchelf-0.14.3"},"system":"x86_64-linux"} +{"attr":"build-sanitized.aarch64-linux","drvPath":"/nix/store/d8ma8d7gjwx6ix4ibs910z9fkm3hwdvz-patchelf-0.14.3.drv","name":"patchelf-0.14.3","outputs":{"out":"/nix/store/6j26m4sznwdyfk4sbmnls3sk0lxm38ih-patchelf-0.14.3"},"system":"aarch64-linux"} +{"attr":"build-sanitized.i686-linux","drvPath":"/nix/store/87rwijvfqqs7dw9lbmckmz4nbryvjaq3-patchelf-0.14.3.drv","name":"patchelf-0.14.3","outputs":{"out":"/nix/store/za5w0gzf97na44fza9sdys15qnjqayd7-patchelf-0.14.3"},"system":"i686-linux"} +{"attr":"build-sanitized.x86_64-linux","drvPath":"/nix/store/nmx50wly2qvd00svx0vqsjfh0jv7q3kl-patchelf-0.14.3.drv","name":"patchelf-0.14.3","outputs":{"out":"/nix/store/38d6bhz3a5jq48gm1diji0rjfcm5vi9n-patchelf-0.14.3"},"system":"x86_64-linux"} +{"attr":"build.aarch64-linux","drvPath":"/nix/store/yjz9msbr6pl8mj7im5kiyhk7wwkvxywa-patchelf-0.14.3.drv","name":"patchelf-0.14.3","outputs":{"out":"/nix/store/as9xhcfwnhfy5x30kxh7lfgla1qrk182-patchelf-0.14.3"},"system":"aarch64-linux"} +{"attr":"build.i686-linux","drvPath":"/nix/store/nwcmdcimnaci0knri5ga019lgbvc4am4-patchelf-0.14.3.drv","name":"patchelf-0.14.3","outputs":{"out":"/nix/store/64x12dmbscnnl42r4y2av52y55ksphhk-patchelf-0.14.3"},"system":"i686-linux"} +{"attr":"build.x86_64-linux","drvPath":"/nix/store/k6p4qnjryr2l1lz31pf085ay9bd7j8gj-patchelf-0.14.3.drv","name":"patchelf-0.14.3","outputs":{"out":"/nix/store/h9a779ghpibfqkkdchx6s08bb3v3i8vy-patchelf-0.14.3"},"system":"x86_64-linux"} +{"attr":"coverage","drvPath":"/nix/store/lsrg05dx3hyi5b6ak99pn9g1rn8xwx39-patchelf-coverage-0.14.3.drv","name":"patchelf-coverage-0.14.3","outputs":{"out":"/nix/store/6h4l5axy5lvxzq662yw47y9r60mxw3zz-patchelf-coverage-0.14.3"},"system":"x86_64-linux"} +{"attr":"release","drvPath":"/nix/store/dgn5gy64pjskfnv7vqh0s86nb998f8sq-patchelf-0.14.3.drv","name":"patchelf-0.14.3","outputs":{"out":"/nix/store/nn05yaznr5af8g8mpgd82yx16pvfzjcy-patchelf-0.14.3"},"system":"x86_64-linux"} +{"attr":"tarball","drvPath":"/nix/store/5ajrgfd5nx29ykgg942k154mcaqfbhxd-patchelf-tarball-0.14.3.drv","name":"patchelf-tarball-0.14.3","outputs":{"out":"/nix/store/5cli6rh0h32yhfcgjkgbplcc73cqvplv-patchelf-tarball-0.14.3"},"system":"x86_64-linux"} + ``` The output here newline-seperated json according to https://jsonlines.org/ From bdc67cf4a660e17bf900b11619c87ab7b09e7c2e Mon Sep 17 00:00:00 2001 From: adisbladis Date: Sat, 8 Jan 2022 22:48:32 +1300 Subject: [PATCH 073/419] Enable github token in auto-update workflow --- .github/workflows/update-flake-lock.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/update-flake-lock.yml b/.github/workflows/update-flake-lock.yml index 5a3d2326f..5c9d23361 100644 --- a/.github/workflows/update-flake-lock.yml +++ b/.github/workflows/update-flake-lock.yml @@ -12,8 +12,8 @@ jobs: uses: actions/checkout@v2 - name: Install Nix uses: cachix/install-nix-action@v16 - # with: - # extra_nix_config: | - # access-tokens = github.com=${{ secrets.GITHUB_TOKEN }} + with: + extra_nix_config: | + access-tokens = github.com=${{ secrets.GITHUB_TOKEN }} - name: Update flake.lock uses: DeterminateSystems/update-flake-lock@v3 From dfe69db146012bdb4541c2e4f5408551a7dbb63f Mon Sep 17 00:00:00 2001 From: adisbladis Date: Sat, 8 Jan 2022 22:58:22 +1300 Subject: [PATCH 074/419] Revert "Enable github token in auto-update workflow" This reverts commit bdc67cf4a660e17bf900b11619c87ab7b09e7c2e. --- .github/workflows/update-flake-lock.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/update-flake-lock.yml b/.github/workflows/update-flake-lock.yml index 5c9d23361..5a3d2326f 100644 --- a/.github/workflows/update-flake-lock.yml +++ b/.github/workflows/update-flake-lock.yml @@ -12,8 +12,8 @@ jobs: uses: actions/checkout@v2 - name: Install Nix uses: cachix/install-nix-action@v16 - with: - extra_nix_config: | - access-tokens = github.com=${{ secrets.GITHUB_TOKEN }} + # with: + # extra_nix_config: | + # access-tokens = github.com=${{ secrets.GITHUB_TOKEN }} - name: Update flake.lock uses: DeterminateSystems/update-flake-lock@v3 From be77b8ea1709b10f7747c9e1aa5b970921dac09d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sat, 8 Jan 2022 10:58:47 +0100 Subject: [PATCH 075/419] README: fix spelling/style --- README.md | 39 +++++++++++++++++++++++---------------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 6afc86105..9dcbf9a40 100644 --- a/README.md +++ b/README.md @@ -1,19 +1,20 @@ # nix-eval-jobs -This project evaluates nix attributes sets in parallel with a streamable json output. -This is useful for time and memory-intensive evaluations such as nixos machines i.e. in a CI context. -Evaluation happens with a controlable number of threads that are restarted if -their memory consumption grows beyond a threshold. +This project evaluates nix attributes sets in parallel with streamable json +output. This is useful for time and memory intensive evaluations such as NixOS +machines, i.e. in a CI context. The evaluation is done with a controllable +number of threads that are restarted when their memory consumption exceeds a +certain threshold. -For ease of integration nix-eval-jobs creates garbage collection roots for each -evaluated derivation (drv file not the build) inside the supplied attribute. -This prevent race conditions between nix garbage collection service and nix -builds processes started by the user. +To facilitate integration, nix-eval-jobs creates garbage collection roots for +each evaluated derivation (drv file, not the build) within the provided +attribute. This prevents race conditions between the nix garbage collection +service and user-started nix builds processes. ## Why using nix-eval-jobs? -- Faster evaluation due the use of threads -- Memory used for evaluation is reclaimed after nix-eval-jobs is finished so that the build can use it. +- Faster evaluation by using threads +- Memory used for evaluation is reclaimed after nix-eval-jobs finish, so that the build can use it. - Evaluation of jobs can fail individually ## Example @@ -35,7 +36,7 @@ $ nix-eval-jobs --gc-roots-dir $(pwd)/gcroot --flake 'github:NixOS/patchelf#hydr ``` -The output here newline-seperated json according to https://jsonlines.org/ +The output here is newline-seperated json according to https://jsonlines.org. The code is derived from [hydra's](https://github.com/nixos/hydra) eval-jobs executable. @@ -67,8 +68,14 @@ USAGE: nix-eval-jobs [options] expr ## Potential use-cases for the tool -**Faster evaluator in deployment tools.** When evaluating nixos machines evaluation can take several minutes when performed on a single core. -This limits the scalability for large deployment with deployment tools such as [NixOps](https://github.com/NixOS/nixops). -**Faster evaluator in CI.** In addition to evaluation speed for CIs it is also useful if evaluation of individual jobs can fail in CIs in contrast to failing the whole jobset. -Furthermore for CIs that allow to create dynamic build steps, one can leverage the fact that nix-eval-jobs outputs derivation path seperatly. -This allows to have seperate logs and success status per job rather than one big log file. +**Faster evaluator in deployment tools.** When evaluating NixOS machines, +evaluation can take several minutes when run on a single core. This limits +scalability for large deployments with deployment tools such as +[NixOps](https://github.com/NixOS/nixops). + +**Faster evaluator in CIs.** In addition to evaluation speed for CIs, it is also +useful if evaluation of individual jobs in CIs can fail, as opposed to failing +the entire jobset. For CIs that allow dynamic build steps to be created, one +can also take advantage of the fact that nix-eval-jobs outputs the derivation +path separately. This allows separate logs and success status per job instead +of a single large log file. From 6296952c9f657712e4e632a2e9f620a3609b28a6 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Sun, 9 Jan 2022 11:02:04 +1300 Subject: [PATCH 076/419] Fix build input not taken into account in mkVariant --- flake.nix | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index ec269a0ba..3b0b7ff58 100644 --- a/flake.nix +++ b/flake.nix @@ -15,7 +15,9 @@ checks = let - mkVariant = nix: packages.nix-eval-jobs.overrideAttrs (_: { + mkVariant = nix: (packages.nix-eval-jobs.override { + inherit nix; + }).overrideAttrs (_: { name = "nix-eval-jobs-${nix.version}"; inherit (nix) version; }); From 0fa3b51289de98a37fc29457eff96249d8757af1 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Sun, 9 Jan 2022 11:05:43 +1300 Subject: [PATCH 077/419] Set src to null in shell.nix To avoid the contents being copied to the store during development --- shell.nix | 2 ++ 1 file changed, 2 insertions(+) diff --git a/shell.nix b/shell.nix index 70b4e37ab..4d785cb43 100644 --- a/shell.nix +++ b/shell.nix @@ -18,6 +18,8 @@ nix = pkgs.nixUnstable; }).overrideAttrs (old: { + src = null; + nativeBuildInputs = old.nativeBuildInputs ++ [ pkgs.editorconfig-checker From 00ced01200f05f9a2d00a9e12c00346edbb99541 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 Jan 2022 23:08:20 +0000 Subject: [PATCH 078/419] Bump DeterminateSystems/update-flake-lock from 3 to 6 Bumps [DeterminateSystems/update-flake-lock](https://github.com/DeterminateSystems/update-flake-lock) from 3 to 6. - [Release notes](https://github.com/DeterminateSystems/update-flake-lock/releases) - [Commits](https://github.com/DeterminateSystems/update-flake-lock/compare/v3...v6) --- updated-dependencies: - dependency-name: DeterminateSystems/update-flake-lock dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/update-flake-lock.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/update-flake-lock.yml b/.github/workflows/update-flake-lock.yml index 5a3d2326f..38fe06179 100644 --- a/.github/workflows/update-flake-lock.yml +++ b/.github/workflows/update-flake-lock.yml @@ -16,4 +16,4 @@ jobs: # extra_nix_config: | # access-tokens = github.com=${{ secrets.GITHUB_TOKEN }} - name: Update flake.lock - uses: DeterminateSystems/update-flake-lock@v3 + uses: DeterminateSystems/update-flake-lock@v6 From 1677bab8fa6d987eef7602a3d4d3a9447d14c2d9 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 13 Jan 2022 00:33:32 +0000 Subject: [PATCH 079/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file changes: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/ff377a78794d412a35245e05428c8f95fef3951f' (2022-01-07) → 'github:NixOS/nixpkgs/b2737d4980a17cc2b7d600d7d0b32fd7333aca88' (2022-01-11) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 0dfc6918b..83726154c 100644 --- a/flake.lock +++ b/flake.lock @@ -17,11 +17,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1641528457, - "narHash": "sha256-FyU9E63n1W7Ql4pMnhW2/rO9OftWZ37pLppn/c1aisY=", + "lastModified": 1641887635, + "narHash": "sha256-kDGpufwzVaiGe5e1sBUBPo9f1YN+nYHJlYqCaVpZTQQ=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "ff377a78794d412a35245e05428c8f95fef3951f", + "rev": "b2737d4980a17cc2b7d600d7d0b32fd7333aca88", "type": "github" }, "original": { From 2e1cef0dca1f94460b1378e61c8638d27e660807 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Thu, 13 Jan 2022 08:47:26 +0100 Subject: [PATCH 080/419] switch to update-flake-lock fork for testing this one hard codes a different github personal access token so we can trigger CI/CD. --- .github/workflows/update-flake-lock.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/update-flake-lock.yml b/.github/workflows/update-flake-lock.yml index 38fe06179..029a8621f 100644 --- a/.github/workflows/update-flake-lock.yml +++ b/.github/workflows/update-flake-lock.yml @@ -16,4 +16,4 @@ jobs: # extra_nix_config: | # access-tokens = github.com=${{ secrets.GITHUB_TOKEN }} - name: Update flake.lock - uses: DeterminateSystems/update-flake-lock@v6 + uses: Mic92/update-flake-lock@main From ff6bede4b6b99ba9b0ab80c5a6ebff89c6705039 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Thu, 13 Jan 2022 09:00:12 +0100 Subject: [PATCH 081/419] update-flake-lock: set bot github token This token hopefully triggers ci builds --- .github/workflows/update-flake-lock.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/update-flake-lock.yml b/.github/workflows/update-flake-lock.yml index 029a8621f..9d97c2cd1 100644 --- a/.github/workflows/update-flake-lock.yml +++ b/.github/workflows/update-flake-lock.yml @@ -17,3 +17,5 @@ jobs: # access-tokens = github.com=${{ secrets.GITHUB_TOKEN }} - name: Update flake.lock uses: Mic92/update-flake-lock@main + with: + token: ${{ secrets.GH_TOKEN_FOR_UPDATES }} From d2f12e992fd6ebf170003ea45d605dda640fea61 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sun, 16 Jan 2022 09:16:23 +0000 Subject: [PATCH 082/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file changes: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/b2737d4980a17cc2b7d600d7d0b32fd7333aca88' (2022-01-11) → 'github:NixOS/nixpkgs/5aaed40d22f0d9376330b6fa413223435ad6fee5' (2022-01-13) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 83726154c..ac634de67 100644 --- a/flake.lock +++ b/flake.lock @@ -17,11 +17,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1641887635, - "narHash": "sha256-kDGpufwzVaiGe5e1sBUBPo9f1YN+nYHJlYqCaVpZTQQ=", + "lastModified": 1642104392, + "narHash": "sha256-m71b7MgMh9FDv4MnI5sg9MiBVW6DhE1zq+d/KlLWSC8=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "b2737d4980a17cc2b7d600d7d0b32fd7333aca88", + "rev": "5aaed40d22f0d9376330b6fa413223435ad6fee5", "type": "github" }, "original": { From 6581e33aeb0238b03c47c142264580efdbf2563c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sun, 16 Jan 2022 10:38:36 +0100 Subject: [PATCH 083/419] add mergify configuration --- .mergify.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 .mergify.yml diff --git a/.mergify.yml b/.mergify.yml new file mode 100644 index 000000000..766725a82 --- /dev/null +++ b/.mergify.yml @@ -0,0 +1,17 @@ +pull_request_rules: + - name: automatic merge on CI success + conditions: + - check-success=flake-checks + - check-success=builds (build, ubuntu-latest) + - check-success=builds (build, macos-latest) + - check-success=builds (build-unstable, ubuntu-latest) + - check-success=builds (build-unstable, macos-latest) + - check-success=builds (editorconfig, ubuntu-latest) + - check-success=builds (editorconfig, macos-latest) + - check-success=builds (nixpkgs-fmt, ubuntu-latest) + - check-success=builds (nixpkgs-fmt, macos-latest) + - author=nix-eval-jobs-bot + actions: + merge: + method: merge + delete_head_branch: {} From 3f8095ce80765d8126fdb5f6d4941e7bde7f9bb7 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 20 Jan 2022 01:06:33 +0000 Subject: [PATCH 084/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file changes: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/5aaed40d22f0d9376330b6fa413223435ad6fee5' (2022-01-13) → 'github:NixOS/nixpkgs/d5dae6569ea9952f1ae4e727946d93a71c507821' (2022-01-15) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index ac634de67..a89cb9f91 100644 --- a/flake.lock +++ b/flake.lock @@ -17,11 +17,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1642104392, - "narHash": "sha256-m71b7MgMh9FDv4MnI5sg9MiBVW6DhE1zq+d/KlLWSC8=", + "lastModified": 1642281915, + "narHash": "sha256-jcMsXmmO1knyf99o242A+2cy1A0eKa9afly0cwBknPA=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "5aaed40d22f0d9376330b6fa413223435ad6fee5", + "rev": "d5dae6569ea9952f1ae4e727946d93a71c507821", "type": "github" }, "original": { From 1a2c6cb15cbb83b55607ade216639a633ba2a301 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 24 Jan 2022 00:55:39 +0000 Subject: [PATCH 085/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file changes: • Updated input 'flake-utils': 'github:numtide/flake-utils/74f7e4319258e287b0f9cb95426c9853b282730b' (2021-11-28) → 'github:numtide/flake-utils/846b2ae0fc4cc943637d3d1def4454213e203cba' (2022-01-20) • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/d5dae6569ea9952f1ae4e727946d93a71c507821' (2022-01-15) → 'github:NixOS/nixpkgs/689b76bcf36055afdeb2e9852f5ecdd2bf483f87' (2022-01-23) --- flake.lock | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/flake.lock b/flake.lock index a89cb9f91..327152fe3 100644 --- a/flake.lock +++ b/flake.lock @@ -2,11 +2,11 @@ "nodes": { "flake-utils": { "locked": { - "lastModified": 1638122382, - "narHash": "sha256-sQzZzAbvKEqN9s0bzWuYmRaA03v40gaJ4+iL1LXjaeI=", + "lastModified": 1642700792, + "narHash": "sha256-XqHrk7hFb+zBvRg6Ghl+AZDq03ov6OshJLiSWOoX5es=", "owner": "numtide", "repo": "flake-utils", - "rev": "74f7e4319258e287b0f9cb95426c9853b282730b", + "rev": "846b2ae0fc4cc943637d3d1def4454213e203cba", "type": "github" }, "original": { @@ -17,11 +17,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1642281915, - "narHash": "sha256-jcMsXmmO1knyf99o242A+2cy1A0eKa9afly0cwBknPA=", + "lastModified": 1642903813, + "narHash": "sha256-0lNfGW8sNfyTrixoQhVG00Drl/ECaf5GbfKAQ1ZDoyE=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "d5dae6569ea9952f1ae4e727946d93a71c507821", + "rev": "689b76bcf36055afdeb2e9852f5ecdd2bf483f87", "type": "github" }, "original": { From b7b7c875c42665b869617802496ba806158c4f22 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 27 Jan 2022 00:56:58 +0000 Subject: [PATCH 086/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/689b76bcf36055afdeb2e9852f5ecdd2bf483f87' (2022-01-23) → 'github:NixOS/nixpkgs/c07b471b52be8fbc49a7dc194e9b37a6e19ee04d' (2022-01-25) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 327152fe3..256393768 100644 --- a/flake.lock +++ b/flake.lock @@ -17,11 +17,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1642903813, - "narHash": "sha256-0lNfGW8sNfyTrixoQhVG00Drl/ECaf5GbfKAQ1ZDoyE=", + "lastModified": 1643080866, + "narHash": "sha256-iO3Z6jw0HEiie8UnXVpq1SxphprDYBXrVzubEa5D4eE=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "689b76bcf36055afdeb2e9852f5ecdd2bf483f87", + "rev": "c07b471b52be8fbc49a7dc194e9b37a6e19ee04d", "type": "github" }, "original": { From aa8182412509a6ef7e85c02f2eb69dced64f21fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Fri, 4 Feb 2022 16:37:45 +0100 Subject: [PATCH 087/419] ci/update-flake-lock: switch back to upstream --- .github/workflows/update-flake-lock.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/update-flake-lock.yml b/.github/workflows/update-flake-lock.yml index 9d97c2cd1..de95f6443 100644 --- a/.github/workflows/update-flake-lock.yml +++ b/.github/workflows/update-flake-lock.yml @@ -16,6 +16,6 @@ jobs: # extra_nix_config: | # access-tokens = github.com=${{ secrets.GITHUB_TOKEN }} - name: Update flake.lock - uses: Mic92/update-flake-lock@main + uses: DeterminateSystems/update-flake-lock@v7 with: token: ${{ secrets.GH_TOKEN_FOR_UPDATES }} From 2c42a8702020df9e7568411bc019350cd7a9becf Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 7 Feb 2022 00:57:26 +0000 Subject: [PATCH 088/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/c07b471b52be8fbc49a7dc194e9b37a6e19ee04d' (2022-01-25) → 'github:NixOS/nixpkgs/76e3df7c0687d5b9ff31431fd4ee4d4cd07a4b2f' (2022-02-03) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 256393768..9b626cf4c 100644 --- a/flake.lock +++ b/flake.lock @@ -17,11 +17,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1643080866, - "narHash": "sha256-iO3Z6jw0HEiie8UnXVpq1SxphprDYBXrVzubEa5D4eE=", + "lastModified": 1643852787, + "narHash": "sha256-RHrvroV61jcfTXfW0Rwc0qu/8lHmfYxSy49adyXkB0I=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "c07b471b52be8fbc49a7dc194e9b37a6e19ee04d", + "rev": "76e3df7c0687d5b9ff31431fd4ee4d4cd07a4b2f", "type": "github" }, "original": { From 404534202e1c9b36e09c4118b3c955df9806de23 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 7 Feb 2022 23:12:17 +0000 Subject: [PATCH 089/419] Bump DeterminateSystems/update-flake-lock from 7 to 8 Bumps [DeterminateSystems/update-flake-lock](https://github.com/DeterminateSystems/update-flake-lock) from 7 to 8. - [Release notes](https://github.com/DeterminateSystems/update-flake-lock/releases) - [Commits](https://github.com/DeterminateSystems/update-flake-lock/compare/v7...v8) --- updated-dependencies: - dependency-name: DeterminateSystems/update-flake-lock dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/update-flake-lock.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/update-flake-lock.yml b/.github/workflows/update-flake-lock.yml index de95f6443..4f097788c 100644 --- a/.github/workflows/update-flake-lock.yml +++ b/.github/workflows/update-flake-lock.yml @@ -16,6 +16,6 @@ jobs: # extra_nix_config: | # access-tokens = github.com=${{ secrets.GITHUB_TOKEN }} - name: Update flake.lock - uses: DeterminateSystems/update-flake-lock@v7 + uses: DeterminateSystems/update-flake-lock@v8 with: token: ${{ secrets.GH_TOKEN_FOR_UPDATES }} From 4c70bbf3c3960e9cc9e6e029bc5ce38f10a45dec Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 10 Feb 2022 00:59:32 +0000 Subject: [PATCH 090/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'flake-utils': 'github:numtide/flake-utils/846b2ae0fc4cc943637d3d1def4454213e203cba' (2022-01-20) → 'github:numtide/flake-utils/3cecb5b042f7f209c56ffd8371b2711a290ec797' (2022-02-07) • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/76e3df7c0687d5b9ff31431fd4ee4d4cd07a4b2f' (2022-02-03) → 'github:NixOS/nixpkgs/c5051e2b5fe9fab43a64f0e0d06b62c81a890b90' (2022-02-08) --- flake.lock | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/flake.lock b/flake.lock index 9b626cf4c..8a2e1f016 100644 --- a/flake.lock +++ b/flake.lock @@ -2,11 +2,11 @@ "nodes": { "flake-utils": { "locked": { - "lastModified": 1642700792, - "narHash": "sha256-XqHrk7hFb+zBvRg6Ghl+AZDq03ov6OshJLiSWOoX5es=", + "lastModified": 1644229661, + "narHash": "sha256-1YdnJAsNy69bpcjuoKdOYQX0YxZBiCYZo4Twxerqv7k=", "owner": "numtide", "repo": "flake-utils", - "rev": "846b2ae0fc4cc943637d3d1def4454213e203cba", + "rev": "3cecb5b042f7f209c56ffd8371b2711a290ec797", "type": "github" }, "original": { @@ -17,11 +17,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1643852787, - "narHash": "sha256-RHrvroV61jcfTXfW0Rwc0qu/8lHmfYxSy49adyXkB0I=", + "lastModified": 1644359234, + "narHash": "sha256-u/sBnRgrFrn9W8gZMS6vN3ZnJsoTvbws968TpqwlDJQ=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "76e3df7c0687d5b9ff31431fd4ee4d4cd07a4b2f", + "rev": "c5051e2b5fe9fab43a64f0e0d06b62c81a890b90", "type": "github" }, "original": { From 7e6e8083b47b1dd9b327f6f792b4b5b6e61d99a2 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 14 Feb 2022 00:56:31 +0000 Subject: [PATCH 091/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/c5051e2b5fe9fab43a64f0e0d06b62c81a890b90' (2022-02-08) → 'github:NixOS/nixpkgs/48d63e924a2666baf37f4f14a18f19347fbd54a2' (2022-02-10) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 8a2e1f016..b88dfad8d 100644 --- a/flake.lock +++ b/flake.lock @@ -17,11 +17,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1644359234, - "narHash": "sha256-u/sBnRgrFrn9W8gZMS6vN3ZnJsoTvbws968TpqwlDJQ=", + "lastModified": 1644525281, + "narHash": "sha256-D3VuWLdnLmAXIkooWAtbTGSQI9Fc1lkvAr94wTxhnTU=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "c5051e2b5fe9fab43a64f0e0d06b62c81a890b90", + "rev": "48d63e924a2666baf37f4f14a18f19347fbd54a2", "type": "github" }, "original": { From 6d61193286aedd4e514fd8f375b2000b95fff4fb Mon Sep 17 00:00:00 2001 From: Zhaofeng Li Date: Wed, 16 Feb 2022 00:46:57 -0800 Subject: [PATCH 092/419] Flush cout after each output line The default buffering behavior depends on whether the output is connected to an interactive device. This causes output lines to be buffered in an undesirable way when stdout is piped, which is how nix-eval-jobs is normally used. Let's fix it by flushing stdout explicitly. --- src/nix-eval-jobs.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix-eval-jobs.cc b/src/nix-eval-jobs.cc index f9ca986dc..842040c95 100644 --- a/src/nix-eval-jobs.cc +++ b/src/nix-eval-jobs.cc @@ -404,7 +404,7 @@ int main(int argc, char * * argv) } } else { auto state(state_.lock()); - std::cout << respString << "\n"; + std::cout << respString << "\n" << std::flush; } /* Add newly discovered job names to the queue. */ From f3bb3a1dd7c35811422228ee5b1f587e0e00d825 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Wed, 16 Feb 2022 18:57:35 +0100 Subject: [PATCH 093/419] README: mention projects using nix-eval-jobs --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index 9dcbf9a40..cbd5fb4a0 100644 --- a/README.md +++ b/README.md @@ -79,3 +79,9 @@ the entire jobset. For CIs that allow dynamic build steps to be created, one can also take advantage of the fact that nix-eval-jobs outputs the derivation path separately. This allows separate logs and success status per job instead of a single large log file. + + +## Projects using nix-eval-jobs + +- [colmena](https://github.com/zhaofengli/colmena) - A simple, stateless NixOS deployment tool +- [robotnix](https://github.com/danielfullmer/robotnix) - Build Android (AOSP) using Nix, used in their [CI](https://github.com/danielfullmer/robotnix/blob/38b80700ee4265c306dcfdcce45056e32ab2973f/.github/workflows/instantiate.yml#L18) From 9a5fd838d114f5673d5763a373ceef66eac612eb Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 21 Feb 2022 00:58:37 +0000 Subject: [PATCH 094/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/48d63e924a2666baf37f4f14a18f19347fbd54a2' (2022-02-10) → 'github:NixOS/nixpkgs/23d785aa6f853e6cf3430119811c334025bbef55' (2022-02-11) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index b88dfad8d..ee1a234e5 100644 --- a/flake.lock +++ b/flake.lock @@ -17,11 +17,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1644525281, - "narHash": "sha256-D3VuWLdnLmAXIkooWAtbTGSQI9Fc1lkvAr94wTxhnTU=", + "lastModified": 1644613700, + "narHash": "sha256-wLRPJclMH8vsHuFtyI78aF09lw5mbi3lMB6uiK5S2wE=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "48d63e924a2666baf37f4f14a18f19347fbd54a2", + "rev": "23d785aa6f853e6cf3430119811c334025bbef55", "type": "github" }, "original": { From 7a9d31fbd8211c3562fdf4857fc575beadc508ef Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 24 Feb 2022 01:05:34 +0000 Subject: [PATCH 095/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/23d785aa6f853e6cf3430119811c334025bbef55' (2022-02-11) → 'github:NixOS/nixpkgs/7f9b6e2babf232412682c09e57ed666d8f84ac2d' (2022-02-21) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index ee1a234e5..3afec6cde 100644 --- a/flake.lock +++ b/flake.lock @@ -17,11 +17,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1644613700, - "narHash": "sha256-wLRPJclMH8vsHuFtyI78aF09lw5mbi3lMB6uiK5S2wE=", + "lastModified": 1645433236, + "narHash": "sha256-4va4MvJ076XyPp5h8sm5eMQvCrJ6yZAbBmyw95dGyw4=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "23d785aa6f853e6cf3430119811c334025bbef55", + "rev": "7f9b6e2babf232412682c09e57ed666d8f84ac2d", "type": "github" }, "original": { From 4a08b3d7aa79984e39b1be44b1461218eede38f6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 7 Mar 2022 23:12:51 +0000 Subject: [PATCH 096/419] Bump actions/checkout from 2.4.0 to 3 Bumps [actions/checkout](https://github.com/actions/checkout) from 2.4.0 to 3. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v2.4.0...v3) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/flake-check.yml | 4 ++-- .github/workflows/test-develop-classic.yml | 2 +- .github/workflows/test-develop-flakes.yml | 2 +- .github/workflows/update-flake-lock.yml | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/flake-check.yml b/.github/workflows/flake-check.yml index 839ad0201..7be090667 100644 --- a/.github/workflows/flake-check.yml +++ b/.github/workflows/flake-check.yml @@ -11,7 +11,7 @@ jobs: outputs: matrix: ${{ steps.set-matrix.outputs.matrix }} steps: - - uses: actions/checkout@v2.4.0 + - uses: actions/checkout@v3 with: # Nix Flakes doesn't work on shallow clones fetch-depth: 0 @@ -31,7 +31,7 @@ jobs: strategy: matrix: ${{fromJSON(needs.flake-checks.outputs.matrix)}} steps: - - uses: actions/checkout@v2.4.0 + - uses: actions/checkout@v3 with: # Nix Flakes doesn't work on shallow clones fetch-depth: 0 diff --git a/.github/workflows/test-develop-classic.yml b/.github/workflows/test-develop-classic.yml index 0455e3006..226afe675 100644 --- a/.github/workflows/test-develop-classic.yml +++ b/.github/workflows/test-develop-classic.yml @@ -11,7 +11,7 @@ jobs: os: [ ubuntu-latest, macos-latest ] runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - uses: cachix/install-nix-action@v16 with: nix_path: nixpkgs=channel:nixos-unstable diff --git a/.github/workflows/test-develop-flakes.yml b/.github/workflows/test-develop-flakes.yml index b4c0afaf7..86a82e93a 100644 --- a/.github/workflows/test-develop-flakes.yml +++ b/.github/workflows/test-develop-flakes.yml @@ -11,7 +11,7 @@ jobs: os: [ ubuntu-latest, macos-latest ] runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: # Nix Flakes doesn't work on shallow clones fetch-depth: 0 diff --git a/.github/workflows/update-flake-lock.yml b/.github/workflows/update-flake-lock.yml index 4f097788c..c64294c40 100644 --- a/.github/workflows/update-flake-lock.yml +++ b/.github/workflows/update-flake-lock.yml @@ -9,7 +9,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Install Nix uses: cachix/install-nix-action@v16 # with: From d592323c6b759ba1324b425f486ef6d8efcd9ff7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sun, 13 Mar 2022 17:16:53 +0100 Subject: [PATCH 097/419] fix build with nix 2.7 --- flake.lock | 12 ++++++------ flake.nix | 4 +++- src/nix-eval-jobs.cc | 6 +++--- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/flake.lock b/flake.lock index 3afec6cde..c679e3c08 100644 --- a/flake.lock +++ b/flake.lock @@ -17,16 +17,16 @@ }, "nixpkgs": { "locked": { - "lastModified": 1645433236, - "narHash": "sha256-4va4MvJ076XyPp5h8sm5eMQvCrJ6yZAbBmyw95dGyw4=", - "owner": "NixOS", + "lastModified": 1647186833, + "narHash": "sha256-62CMNrhICLcI/5jNSQDAQpUUvtvTXBsYWSU6ZlhPIzs=", + "owner": "Mic92", "repo": "nixpkgs", - "rev": "7f9b6e2babf232412682c09e57ed666d8f84ac2d", + "rev": "de173cbbb66e9b454e0e836cec1b807856934539", "type": "github" }, "original": { - "owner": "NixOS", - "ref": "nixos-unstable", + "owner": "Mic92", + "ref": "nix-unstable", "repo": "nixpkgs", "type": "github" } diff --git a/flake.nix b/flake.nix index 3b0b7ff58..5ab579ea4 100644 --- a/flake.nix +++ b/flake.nix @@ -1,7 +1,9 @@ { description = "Hydra's builtin hydra-eval-jobs as a standalone"; - inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; + # switch back when https://github.com/NixOS/nixpkgs/pull/164012 is merged + #inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; + inputs.nixpkgs.url = "github:Mic92/nixpkgs/nix-unstable"; inputs.flake-utils.url = "github:numtide/flake-utils"; outputs = { self, nixpkgs, flake-utils }: diff --git a/src/nix-eval-jobs.cc b/src/nix-eval-jobs.cc index 842040c95..51fb27e8a 100644 --- a/src/nix-eval-jobs.cc +++ b/src/nix-eval-jobs.cc @@ -142,7 +142,7 @@ static void worker( callFlake(state, lockedFlake, *vFlake); auto vOutputs = vFlake->attrs->get(state.symbols.create("outputs"))->value; - state.forceValue(*vOutputs); + state.forceValue(*vOutputs, noPos); vTop = *vOutputs; if (fragment.length() > 0) { @@ -186,8 +186,8 @@ static void worker( if (drv->querySystem() == "unknown") throw EvalError("derivation must have a 'system' attribute"); - auto drvPath = drv->queryDrvPath(); auto localStore = state.store.dynamic_pointer_cast(); + auto drvPath = localStore->printStorePath(drv->requireDrvPath()); auto storePath = localStore->parseStorePath(drvPath); auto outputs = drv->queryOutputs(false); @@ -195,7 +195,7 @@ static void worker( reply["system"] = drv->querySystem(); reply["drvPath"] = drvPath; for (auto out : outputs){ - reply["outputs"][out.first] = out.second; + reply["outputs"][out.first] = localStore->printStorePath(out.second); } if (myArgs.meta) { From ed136e13960f8e9dc61800133279ad674ed5d2b2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Apr 2022 23:22:37 +0000 Subject: [PATCH 098/419] Bump cachix/install-nix-action from 16 to 17 Bumps [cachix/install-nix-action](https://github.com/cachix/install-nix-action) from 16 to 17. - [Release notes](https://github.com/cachix/install-nix-action/releases) - [Commits](https://github.com/cachix/install-nix-action/compare/v16...v17) --- updated-dependencies: - dependency-name: cachix/install-nix-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/flake-check.yml | 4 ++-- .github/workflows/test-develop-classic.yml | 2 +- .github/workflows/test-develop-flakes.yml | 2 +- .github/workflows/update-flake-lock.yml | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/flake-check.yml b/.github/workflows/flake-check.yml index 7be090667..603916100 100644 --- a/.github/workflows/flake-check.yml +++ b/.github/workflows/flake-check.yml @@ -15,7 +15,7 @@ jobs: with: # Nix Flakes doesn't work on shallow clones fetch-depth: 0 - - uses: cachix/install-nix-action@v16 + - uses: cachix/install-nix-action@v17 with: nix_path: nixpkgs=channel:nixos-unstable - id: set-matrix @@ -35,7 +35,7 @@ jobs: with: # Nix Flakes doesn't work on shallow clones fetch-depth: 0 - - uses: cachix/install-nix-action@v16 + - uses: cachix/install-nix-action@v17 with: nix_path: nixpkgs=channel:nixos-unstable - run: | diff --git a/.github/workflows/test-develop-classic.yml b/.github/workflows/test-develop-classic.yml index 226afe675..22b2f632f 100644 --- a/.github/workflows/test-develop-classic.yml +++ b/.github/workflows/test-develop-classic.yml @@ -12,7 +12,7 @@ jobs: runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v3 - - uses: cachix/install-nix-action@v16 + - uses: cachix/install-nix-action@v17 with: nix_path: nixpkgs=channel:nixos-unstable - name: Build diff --git a/.github/workflows/test-develop-flakes.yml b/.github/workflows/test-develop-flakes.yml index 86a82e93a..243eb244f 100644 --- a/.github/workflows/test-develop-flakes.yml +++ b/.github/workflows/test-develop-flakes.yml @@ -15,7 +15,7 @@ jobs: with: # Nix Flakes doesn't work on shallow clones fetch-depth: 0 - - uses: cachix/install-nix-action@v16 + - uses: cachix/install-nix-action@v17 - name: Build run: nix develop -c bash -c 'meson build && cd build && ninja' - name: Run tests diff --git a/.github/workflows/update-flake-lock.yml b/.github/workflows/update-flake-lock.yml index c64294c40..b06d20f05 100644 --- a/.github/workflows/update-flake-lock.yml +++ b/.github/workflows/update-flake-lock.yml @@ -11,7 +11,7 @@ jobs: - name: Checkout repository uses: actions/checkout@v3 - name: Install Nix - uses: cachix/install-nix-action@v16 + uses: cachix/install-nix-action@v17 # with: # extra_nix_config: | # access-tokens = github.com=${{ secrets.GITHUB_TOKEN }} From b062ac705df9d5b5040ce35c8c01cda394cd8a2a Mon Sep 17 00:00:00 2001 From: John Soo Date: Thu, 21 Apr 2022 09:31:53 -0700 Subject: [PATCH 099/419] Set GC_DONT_GC=1. To avoid `Collecting from unknown thread'. --- src/nix-eval-jobs.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/nix-eval-jobs.cc b/src/nix-eval-jobs.cc index 51fb27e8a..347e88991 100644 --- a/src/nix-eval-jobs.cc +++ b/src/nix-eval-jobs.cc @@ -282,6 +282,9 @@ int main(int argc, char * * argv) $NIX_PATH. */ unsetenv("NIX_PATH"); + /* We are doing the garbage collection by killing forks */ + setenv("GC_DONT_GC", "1", 1); + return handleExceptions(argv[0], [&]() { initNix(); initGC(); From 4a3d2e00085722005323a08122d8babfee5b7240 Mon Sep 17 00:00:00 2001 From: John Soo Date: Thu, 21 Apr 2022 11:34:49 -0700 Subject: [PATCH 100/419] Make function to get top-level value from releaseExpr. --- src/nix-eval-jobs.cc | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/nix-eval-jobs.cc b/src/nix-eval-jobs.cc index 51fb27e8a..2615f306e 100644 --- a/src/nix-eval-jobs.cc +++ b/src/nix-eval-jobs.cc @@ -116,6 +116,18 @@ struct MyArgs : MixEvalArgs, MixCommonArgs static MyArgs myArgs; +static Value* releaseExprTopLevelValue(EvalState & state, Bindings & autoArgs) { + Value vTop; + + state.evalFile(lookupFileArg(state, myArgs.releaseExpr), vTop); + + auto vRoot = state.allocValue(); + + state.autoCallFunction(autoArgs, vTop, *vRoot); + + return vRoot; +} + static void worker( EvalState & state, Bindings & autoArgs, From e05e6254112abbec259e100d055075af87b1359f Mon Sep 17 00:00:00 2001 From: John Soo Date: Thu, 21 Apr 2022 09:15:20 -0700 Subject: [PATCH 101/419] Make function to get top-level value from a flake. --- src/nix-eval-jobs.cc | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/src/nix-eval-jobs.cc b/src/nix-eval-jobs.cc index 2615f306e..59d15dcf2 100644 --- a/src/nix-eval-jobs.cc +++ b/src/nix-eval-jobs.cc @@ -128,6 +128,40 @@ static Value* releaseExprTopLevelValue(EvalState & state, Bindings & autoArgs) { return vRoot; } +static Value* flakeTopLevelValue(EvalState & state, Bindings & autoArgs) { + using namespace flake; + + auto [flakeRef, fragment] = parseFlakeRefWithFragment(myArgs.releaseExpr, absPath(".")); + + auto vFlake = state.allocValue(); + + auto lockedFlake = lockFlake(state, flakeRef, + LockFlags { + .updateLockFile = false, + .useRegistries = false, + .allowMutable = false, + }); + + callFlake(state, lockedFlake, *vFlake); + + auto vOutputs = vFlake->attrs->get(state.symbols.create("outputs"))->value; + state.forceValue(*vOutputs, noPos); + auto vTop = *vOutputs; + + if (fragment.length() > 0) { + Bindings & bindings(*state.allocBindings(0)); + auto [nTop, pos] = findAlongAttrPath(state, fragment, bindings, vTop); + if (!nTop) + throw Error("error: attribute '%s' missing", nTop); + vTop = *nTop; + } + + auto vRoot = state.allocValue(); + state.autoCallFunction(autoArgs, vTop, *vRoot); + + return vRoot; +} + static void worker( EvalState & state, Bindings & autoArgs, From dd8a2e89a0e4b0a1638826065a4b78022e5b60e6 Mon Sep 17 00:00:00 2001 From: John Soo Date: Thu, 21 Apr 2022 11:41:31 -0700 Subject: [PATCH 102/419] Simplify top-level value fetching. --- src/nix-eval-jobs.cc | 43 +++++++------------------------------------ 1 file changed, 7 insertions(+), 36 deletions(-) diff --git a/src/nix-eval-jobs.cc b/src/nix-eval-jobs.cc index 59d15dcf2..8ef43b22a 100644 --- a/src/nix-eval-jobs.cc +++ b/src/nix-eval-jobs.cc @@ -162,6 +162,12 @@ static Value* flakeTopLevelValue(EvalState & state, Bindings & autoArgs) { return vRoot; } +Value * topLevelValue(EvalState & state, Bindings & autoArgs) { + return myArgs.flake + ? flakeTopLevelValue(state, autoArgs) + : releaseExprTopLevelValue(state, autoArgs); +} + static void worker( EvalState & state, Bindings & autoArgs, @@ -169,42 +175,7 @@ static void worker( AutoCloseFD & from, const Path &gcRootsDir) { - Value vTop; - - if (myArgs.flake) { - using namespace flake; - - auto [flakeRef, fragment] = parseFlakeRefWithFragment(myArgs.releaseExpr, absPath(".")); - - auto vFlake = state.allocValue(); - - auto lockedFlake = lockFlake(state, flakeRef, - LockFlags { - .updateLockFile = false, - .useRegistries = false, - .allowMutable = false, - }); - - callFlake(state, lockedFlake, *vFlake); - - auto vOutputs = vFlake->attrs->get(state.symbols.create("outputs"))->value; - state.forceValue(*vOutputs, noPos); - vTop = *vOutputs; - - if (fragment.length() > 0) { - Bindings & bindings(*state.allocBindings(0)); - auto [nTop, pos] = findAlongAttrPath(state, fragment, bindings, vTop); - if (!nTop) - throw Error("error: attribute '%s' missing", nTop); - vTop = *nTop; - } - - } else { - state.evalFile(lookupFileArg(state, myArgs.releaseExpr), vTop); - } - - auto vRoot = state.allocValue(); - state.autoCallFunction(autoArgs, vTop, *vRoot); + auto vRoot = topLevelValue(state, autoArgs); while (true) { /* Wait for the master to send us a job name. */ From 0e0c6c44d0c3a4bcdf4eb88d1a8fa539f339505f Mon Sep 17 00:00:00 2001 From: John Soo Date: Thu, 21 Apr 2022 10:02:05 -0700 Subject: [PATCH 103/419] Ignore -Wnon-virtual-dtor for static struct. --- src/nix-eval-jobs.cc | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/nix-eval-jobs.cc b/src/nix-eval-jobs.cc index 51fb27e8a..32d49ef4f 100644 --- a/src/nix-eval-jobs.cc +++ b/src/nix-eval-jobs.cc @@ -31,6 +31,9 @@ using namespace nix; typedef enum { evalAuto, evalImpure, evalPure } pureEval; +// Safe to ignore - the args will be static. +#pragma GCC diagnostic ignored "-Wnon-virtual-dtor" +#pragma clang diagnostic ignored "-Wnon-virtual-dtor" struct MyArgs : MixEvalArgs, MixCommonArgs { Path releaseExpr; @@ -113,6 +116,8 @@ struct MyArgs : MixEvalArgs, MixCommonArgs expectArg("expr", &releaseExpr); } }; +#pragma GCC diagnostic warning "-Wnon-virtual-dtor" +#pragma clang diagnostic warning "-Wnon-virtual-dtor" static MyArgs myArgs; From 0aa88f2f492302d5c785ecce79d6b1e6efa7ee0d Mon Sep 17 00:00:00 2001 From: John Soo Date: Thu, 21 Apr 2022 10:12:22 -0700 Subject: [PATCH 104/419] Add debug symbols to nix-shell. --- shell.nix | 3 +++ 1 file changed, 3 insertions(+) diff --git a/shell.nix b/shell.nix index 4d785cb43..bbaa60d05 100644 --- a/shell.nix +++ b/shell.nix @@ -32,4 +32,7 @@ ]; + shellHook = '' + export NIX_DEBUG_INFO_DIRS="${pkgs.curl.debug}/lib/debug:${pkgs.nixUnstable.debug}/lib/debug''${NIX_DEBUG_INFO_DIRS:+:$NIX_DEBUG_INFO_DIRS}" + ''; }) From 6ef1b314d7c23024ef37df7f22c9f6208645f77c Mon Sep 17 00:00:00 2001 From: adisbladis Date: Fri, 22 Apr 2022 18:42:17 +1200 Subject: [PATCH 105/419] Only add debug symbols to shell on Linux --- shell.nix | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/shell.nix b/shell.nix index bbaa60d05..451727833 100644 --- a/shell.nix +++ b/shell.nix @@ -13,9 +13,13 @@ , srcDir ? null }: -(pkgs.callPackage ./default.nix { - inherit srcDir; +let + inherit (pkgs) lib stdenv; nix = pkgs.nixUnstable; + +in +(pkgs.callPackage ./default.nix { + inherit nix srcDir; }).overrideAttrs (old: { src = null; @@ -32,7 +36,7 @@ ]; - shellHook = '' - export NIX_DEBUG_INFO_DIRS="${pkgs.curl.debug}/lib/debug:${pkgs.nixUnstable.debug}/lib/debug''${NIX_DEBUG_INFO_DIRS:+:$NIX_DEBUG_INFO_DIRS}" + shellHook = lib.optionalString stdenv.isLinux '' + export NIX_DEBUG_INFO_DIRS="${pkgs.curl.debug}/lib/debug:${nix.debug}/lib/debug''${NIX_DEBUG_INFO_DIRS:+:$NIX_DEBUG_INFO_DIRS}" ''; }) From 3094e12732c7308fc529c61deb6c3ccf1b7669c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Mon, 18 Apr 2022 14:31:47 +0200 Subject: [PATCH 106/419] flake: switch back to original nixpkgs --- flake.lock | 12 ++++++------ flake.nix | 4 +--- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/flake.lock b/flake.lock index c679e3c08..0f133c751 100644 --- a/flake.lock +++ b/flake.lock @@ -17,16 +17,16 @@ }, "nixpkgs": { "locked": { - "lastModified": 1647186833, - "narHash": "sha256-62CMNrhICLcI/5jNSQDAQpUUvtvTXBsYWSU6ZlhPIzs=", - "owner": "Mic92", + "lastModified": 1650161686, + "narHash": "sha256-70ZWAlOQ9nAZ08OU6WY7n4Ij2kOO199dLfNlvO/+pf8=", + "owner": "NixOS", "repo": "nixpkgs", - "rev": "de173cbbb66e9b454e0e836cec1b807856934539", + "rev": "1ffba9f2f683063c2b14c9f4d12c55ad5f4ed887", "type": "github" }, "original": { - "owner": "Mic92", - "ref": "nix-unstable", + "owner": "NixOS", + "ref": "nixos-unstable", "repo": "nixpkgs", "type": "github" } diff --git a/flake.nix b/flake.nix index 5ab579ea4..3b0b7ff58 100644 --- a/flake.nix +++ b/flake.nix @@ -1,9 +1,7 @@ { description = "Hydra's builtin hydra-eval-jobs as a standalone"; - # switch back when https://github.com/NixOS/nixpkgs/pull/164012 is merged - #inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; - inputs.nixpkgs.url = "github:Mic92/nixpkgs/nix-unstable"; + inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; inputs.flake-utils.url = "github:numtide/flake-utils"; outputs = { self, nixpkgs, flake-utils }: From c1b86245a1b556b79b330378b88fafb827287deb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Mon, 18 Apr 2022 14:45:49 +0200 Subject: [PATCH 107/419] fix for nixUnstable --- flake.nix | 8 ++++++-- src/nix-eval-jobs.cc | 4 +++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/flake.nix b/flake.nix index 3b0b7ff58..8b5bc6c9d 100644 --- a/flake.nix +++ b/flake.nix @@ -16,7 +16,9 @@ checks = let mkVariant = nix: (packages.nix-eval-jobs.override { - inherit nix; + # TODO: fix to stable after next nix release + nix = pkgs.nixUnstable; + #inherit nix; }).overrideAttrs (_: { name = "nix-eval-jobs-${nix.version}"; inherit (nix) version; @@ -44,7 +46,9 @@ touch $out ''; - build = mkVariant pkgs.nix; + # TODO fix to unstable in next release + build = mkVariant pkgs.nixUnstable; + #build = mkVariant pkgs.nix; build-unstable = mkVariant pkgs.nixUnstable; }; diff --git a/src/nix-eval-jobs.cc b/src/nix-eval-jobs.cc index fd333200f..50c99d5ca 100644 --- a/src/nix-eval-jobs.cc +++ b/src/nix-eval-jobs.cc @@ -217,7 +217,9 @@ static void worker( reply["system"] = drv->querySystem(); reply["drvPath"] = drvPath; for (auto out : outputs){ - reply["outputs"][out.first] = localStore->printStorePath(out.second); + if (out.second) { + reply["outputs"][out.first] = localStore->printStorePath(*out.second); + } } if (myArgs.meta) { From 353724df901a4a3284f53f81f3c860940970ecc9 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Fri, 22 Apr 2022 19:06:23 +1200 Subject: [PATCH 108/419] Fix querying output paths The upstream function signature changed, we no longer have to pass onlyOutputsToInstall. --- src/nix-eval-jobs.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix-eval-jobs.cc b/src/nix-eval-jobs.cc index 50c99d5ca..628a24586 100644 --- a/src/nix-eval-jobs.cc +++ b/src/nix-eval-jobs.cc @@ -211,7 +211,7 @@ static void worker( auto localStore = state.store.dynamic_pointer_cast(); auto drvPath = localStore->printStorePath(drv->requireDrvPath()); auto storePath = localStore->parseStorePath(drvPath); - auto outputs = drv->queryOutputs(false); + auto outputs = drv->queryOutputs(); reply["name"] = drv->queryName(); reply["system"] = drv->querySystem(); From 9d4c256fa05468fab900b778473d1b380915a030 Mon Sep 17 00:00:00 2001 From: John Soo Date: Thu, 14 Apr 2022 23:28:19 -0700 Subject: [PATCH 109/419] Make a Proc struct for running processes. --- src/nix-eval-jobs.cc | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/src/nix-eval-jobs.cc b/src/nix-eval-jobs.cc index fd333200f..045577ba2 100644 --- a/src/nix-eval-jobs.cc +++ b/src/nix-eval-jobs.cc @@ -298,6 +298,50 @@ static void worker( writeLine(to.get(), "restart"); } +typedef std::function + Processor; + +/* Auto-cleanup of fork's process and fds. */ +struct Proc { + AutoCloseFD to, from; + Pid pid; + + Proc(const Processor & proc) { + Pipe toPipe, fromPipe; + toPipe.create(); + fromPipe.create(); + auto p = startProcess( + [&, + to{std::make_shared(std::move(fromPipe.writeSide))}, + from{std::make_shared(std::move(toPipe.readSide))} + ]() + { + debug("created worker process %d", getpid()); + try { + EvalState state(myArgs.searchPath, openStore()); + Bindings & autoArgs = *myArgs.getAutoArgs(state); + proc(state, autoArgs, *to, *from); + } catch (Error & e) { + nlohmann::json err; + auto msg = e.msg(); + err["error"] = filterANSIEscapes(msg, true); + printError(msg); + writeLine(to->get(), err.dump()); + // Don't forget to print it into the STDERR log, this is + // what's shown in the Hydra UI. + writeLine(to->get(), "restart"); + } + }, + ProcessOptions { .allowVfork = false }); + + to = std::move(toPipe.writeSide); + from = std::move(fromPipe.readSide); + pid = p; + } + + ~Proc() { } +}; + int main(int argc, char * * argv) { /* Prevent undeclared dependencies in the evaluation via From a27faabd0a3fc0a693433fa5b277587127c14fb3 Mon Sep 17 00:00:00 2001 From: John Soo Date: Thu, 21 Apr 2022 10:46:38 -0700 Subject: [PATCH 110/419] Use Proc struct to manage forked processes. Cleans up zombie processes. --- src/nix-eval-jobs.cc | 56 +++++++++++--------------------------------- 1 file changed, 14 insertions(+), 42 deletions(-) diff --git a/src/nix-eval-jobs.cc b/src/nix-eval-jobs.cc index 045577ba2..d71cf8da5 100644 --- a/src/nix-eval-jobs.cc +++ b/src/nix-eval-jobs.cc @@ -177,8 +177,7 @@ static void worker( EvalState & state, Bindings & autoArgs, AutoCloseFD & to, - AutoCloseFD & from, - const Path &gcRootsDir) + AutoCloseFD & from) { auto vRoot = topLevelValue(state, autoArgs); @@ -243,8 +242,8 @@ static void worker( /* Register the derivation as a GC root. !!! This registers roots for jobs that we may have already done. */ - if (gcRootsDir != "") { - Path root = gcRootsDir + "/" + std::string(baseNameOf(drvPath)); + if (myArgs.gcRootsDir != "") { + Path root = myArgs.gcRootsDir + "/" + std::string(baseNameOf(drvPath)); if (!pathExists(root)) localStore->addPermRoot(storePath, root); } @@ -391,47 +390,18 @@ int main(int argc, char * * argv) auto handler = [&]() { try { - pid_t pid = -1; - AutoCloseFD from, to; + std::optional> proc_; while (true) { - /* Start a new worker process if necessary. */ - if (pid == -1) { - Pipe toPipe, fromPipe; - toPipe.create(); - fromPipe.create(); - pid = startProcess( - [&, - to{std::make_shared(std::move(fromPipe.writeSide))}, - from{std::make_shared(std::move(toPipe.readSide))} - ]() - { - try { - EvalState state(myArgs.searchPath, openStore()); - Bindings & autoArgs = *myArgs.getAutoArgs(state); - worker(state, autoArgs, *to, *from, myArgs.gcRootsDir); - } catch (Error & e) { - nlohmann::json err; - auto msg = e.msg(); - err["error"] = filterANSIEscapes(msg, true); - printError(msg); - writeLine(to->get(), err.dump()); - // Don't forget to print it into the STDERR log, this is - // what's shown in the Hydra UI. - writeLine(to->get(), "restart"); - } - }, - ProcessOptions { .allowVfork = false }); - from = std::move(fromPipe.readSide); - to = std::move(toPipe.writeSide); - debug("created worker process %d", pid); - } + auto proc = proc_.has_value() + ? std::move(proc_.value()) + : std::make_unique(worker); /* Check whether the existing worker process is still there. */ - auto s = readLine(from.get()); + auto s = readLine(proc->from.get()); if (s == "restart") { - pid = -1; + proc_ = std::nullopt; continue; } else if (s != "next") { auto json = nlohmann::json::parse(s); @@ -445,7 +415,7 @@ int main(int argc, char * * argv) checkInterrupt(); auto state(state_.lock()); if ((state->todo.empty() && state->active.empty()) || state->exc) { - writeLine(to.get(), "exit"); + writeLine(proc->to.get(), "exit"); return; } if (!state->todo.empty()) { @@ -458,10 +428,10 @@ int main(int argc, char * * argv) } /* Tell the worker to evaluate it. */ - writeLine(to.get(), "do " + attrPath); + writeLine(proc->to.get(), "do " + attrPath); /* Wait for the response. */ - auto respString = readLine(from.get()); + auto respString = readLine(proc->from.get()); auto response = nlohmann::json::parse(respString); /* Handle the response. */ @@ -476,6 +446,8 @@ int main(int argc, char * * argv) std::cout << respString << "\n" << std::flush; } + proc_ = std::move(proc); + /* Add newly discovered job names to the queue. */ { auto state(state_.lock()); From b9a87464a06c60598d0948e974ee6538ab024f60 Mon Sep 17 00:00:00 2001 From: John Soo Date: Wed, 20 Apr 2022 21:01:02 -0700 Subject: [PATCH 111/419] Move collecting handler to separate function. s/master/collector/g --- src/nix-eval-jobs.cc | 182 +++++++++++++++++++++---------------------- 1 file changed, 91 insertions(+), 91 deletions(-) diff --git a/src/nix-eval-jobs.cc b/src/nix-eval-jobs.cc index d71cf8da5..0e5d78ae1 100644 --- a/src/nix-eval-jobs.cc +++ b/src/nix-eval-jobs.cc @@ -182,7 +182,7 @@ static void worker( auto vRoot = topLevelValue(state, autoArgs); while (true) { - /* Wait for the master to send us a job name. */ + /* Wait for the collector to send us a job name. */ writeLine(to.get(), "next"); auto s = readLine(from.get()); @@ -192,7 +192,7 @@ static void worker( debug("worker process %d at '%s'", getpid(), attrPath); - /* Evaluate it and send info back to the master. */ + /* Evaluate it and send info back to the collector. */ nlohmann::json reply; reply["attr"] = attrPath; @@ -287,7 +287,7 @@ static void worker( writeLine(to.get(), reply.dump()); - /* If our RSS exceeds the maximum, exit. The master will + /* If our RSS exceeds the maximum, exit. The collector will start a new process. */ struct rusage r; getrusage(RUSAGE_SELF, &r); @@ -341,6 +341,91 @@ struct Proc { ~Proc() { } }; +struct State +{ + std::set todo{""}; + std::set active; + std::exception_ptr exc; +}; + +std::function collector(Sync & state_, std::condition_variable & wakeup) { + return [&]() { + try { + std::optional> proc_; + + while (true) { + + auto proc = proc_.has_value() + ? std::move(proc_.value()) + : std::make_unique(worker); + + /* Check whether the existing worker process is still there. */ + auto s = readLine(proc->from.get()); + if (s == "restart") { + proc_ = std::nullopt; + continue; + } else if (s != "next") { + auto json = nlohmann::json::parse(s); + throw Error("worker error: %s", (std::string) json["error"]); + } + + /* Wait for a job name to become available. */ + std::string attrPath; + + while (true) { + checkInterrupt(); + auto state(state_.lock()); + if ((state->todo.empty() && state->active.empty()) || state->exc) { + writeLine(proc->to.get(), "exit"); + return; + } + if (!state->todo.empty()) { + attrPath = *state->todo.begin(); + state->todo.erase(state->todo.begin()); + state->active.insert(attrPath); + break; + } else + state.wait(wakeup); + } + + /* Tell the worker to evaluate it. */ + writeLine(proc->to.get(), "do " + attrPath); + + /* Wait for the response. */ + auto respString = readLine(proc->from.get()); + auto response = nlohmann::json::parse(respString); + + /* Handle the response. */ + StringSet newAttrs; + if (response.find("attrs") != response.end()) { + for (auto & i : response["attrs"]) { + auto s = (attrPath.empty() ? "" : attrPath + ".") + (std::string) i; + newAttrs.insert(s); + } + } else { + auto state(state_.lock()); + std::cout << respString << "\n" << std::flush; + } + + proc_ = std::move(proc); + + /* Add newly discovered job names to the queue. */ + { + auto state(state_.lock()); + state->active.erase(attrPath); + for (auto & s : newAttrs) + state->todo.insert(s); + wakeup.notify_all(); + } + } + } catch (...) { + auto state(state_.lock()); + state->exc = std::current_exception(); + wakeup.notify_all(); + } + }; +} + int main(int argc, char * * argv) { /* Prevent undeclared dependencies in the evaluation via @@ -375,98 +460,13 @@ int main(int argc, char * * argv) loggerSettings.showTrace.assign(true); } - struct State - { - std::set todo{""}; - std::set active; - std::exception_ptr exc; - }; - - std::condition_variable wakeup; - Sync state_; - /* Start a handler thread per worker process. */ - auto handler = [&]() - { - try { - std::optional> proc_; - - while (true) { - - auto proc = proc_.has_value() - ? std::move(proc_.value()) - : std::make_unique(worker); - - /* Check whether the existing worker process is still there. */ - auto s = readLine(proc->from.get()); - if (s == "restart") { - proc_ = std::nullopt; - continue; - } else if (s != "next") { - auto json = nlohmann::json::parse(s); - throw Error("worker error: %s", (std::string) json["error"]); - } - - /* Wait for a job name to become available. */ - std::string attrPath; - - while (true) { - checkInterrupt(); - auto state(state_.lock()); - if ((state->todo.empty() && state->active.empty()) || state->exc) { - writeLine(proc->to.get(), "exit"); - return; - } - if (!state->todo.empty()) { - attrPath = *state->todo.begin(); - state->todo.erase(state->todo.begin()); - state->active.insert(attrPath); - break; - } else - state.wait(wakeup); - } - - /* Tell the worker to evaluate it. */ - writeLine(proc->to.get(), "do " + attrPath); - - /* Wait for the response. */ - auto respString = readLine(proc->from.get()); - auto response = nlohmann::json::parse(respString); - - /* Handle the response. */ - StringSet newAttrs; - if (response.find("attrs") != response.end()) { - for (auto & i : response["attrs"]) { - auto s = (attrPath.empty() ? "" : attrPath + ".") + (std::string) i; - newAttrs.insert(s); - } - } else { - auto state(state_.lock()); - std::cout << respString << "\n" << std::flush; - } - - proc_ = std::move(proc); - - /* Add newly discovered job names to the queue. */ - { - auto state(state_.lock()); - state->active.erase(attrPath); - for (auto & s : newAttrs) - state->todo.insert(s); - wakeup.notify_all(); - } - } - } catch (...) { - auto state(state_.lock()); - state->exc = std::current_exception(); - wakeup.notify_all(); - } - }; - + /* Start a collector thread per worker process. */ std::vector threads; + std::condition_variable wakeup; for (size_t i = 0; i < myArgs.nrWorkers; i++) - threads.emplace_back(std::thread(handler)); + threads.emplace_back(std::thread(collector(state_, wakeup))); for (auto & thread : threads) thread.join(); From 8bb83e4c699201c98f04f10093712fef2f436986 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Fri, 22 Apr 2022 19:14:19 +1200 Subject: [PATCH 112/419] Disable nix-shell based checks We are essentially testing the same thing in the Flake check, the difference is in the nix-shell vs `nix develop` invocation. I will admit (even though this is my making) that it's a bit silly to essentially test the same thing twice. --- .github/workflows/test-develop-classic.yml | 21 --------------------- .github/workflows/test-develop-flakes.yml | 2 +- 2 files changed, 1 insertion(+), 22 deletions(-) delete mode 100644 .github/workflows/test-develop-classic.yml diff --git a/.github/workflows/test-develop-classic.yml b/.github/workflows/test-develop-classic.yml deleted file mode 100644 index 22b2f632f..000000000 --- a/.github/workflows/test-develop-classic.yml +++ /dev/null @@ -1,21 +0,0 @@ -name: "Development workflow - nix-shell" -on: - pull_request: - push: - branches: - - main -jobs: - tests: - strategy: - matrix: - os: [ ubuntu-latest, macos-latest ] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v3 - - uses: cachix/install-nix-action@v17 - with: - nix_path: nixpkgs=channel:nixos-unstable - - name: Build - run: nix-shell --run 'meson build && cd build && ninja' - - name: Run tests - run: nix-shell --run 'pytest ./tests' diff --git a/.github/workflows/test-develop-flakes.yml b/.github/workflows/test-develop-flakes.yml index 243eb244f..6a8d714b8 100644 --- a/.github/workflows/test-develop-flakes.yml +++ b/.github/workflows/test-develop-flakes.yml @@ -1,4 +1,4 @@ -name: "Development workflow - flakes" +name: "Development workflow" on: pull_request: push: From 8dd6426d15cc4787c9c6766fcf9771e8698d7e0b Mon Sep 17 00:00:00 2001 From: John Soo Date: Wed, 13 Apr 2022 15:41:51 -0700 Subject: [PATCH 113/419] Make Drv struct. To facilitate collecting and jsonifying derivations in different contexts --- src/nix-eval-jobs.cc | 101 ++++++++++++++++++++++++++++--------------- 1 file changed, 65 insertions(+), 36 deletions(-) diff --git a/src/nix-eval-jobs.cc b/src/nix-eval-jobs.cc index 7c0ff5007..6eb275151 100644 --- a/src/nix-eval-jobs.cc +++ b/src/nix-eval-jobs.cc @@ -173,6 +173,65 @@ Value * topLevelValue(EvalState & state, Bindings & autoArgs) { : releaseExprTopLevelValue(state, autoArgs); } +/* The fields of a derivation that are printed in json form */ +struct Drv { + std::string name; + std::string system; + std::string drvPath; + std::map outputs; + std::optional meta; + + Drv (EvalState & state, DrvInfo & drvInfo) { + if (drvInfo.querySystem() == "unknown") + throw EvalError("derivation must have a 'system' attribute"); + + auto localStore = state.store.dynamic_pointer_cast(); + + for (auto out : drvInfo.queryOutputs(true)) { + if (out.second) + outputs[out.first] = localStore->printStorePath(*out.second); + + } + + if (myArgs.meta) { + nlohmann::json meta_; + for (auto & name : drvInfo.queryMetaNames()) { + PathSet context; + std::stringstream ss; + + auto metaValue = drvInfo.queryMeta(name); + // Skip non-serialisable types + // TODO: Fix serialisation of derivations to store paths + if (metaValue == 0) { + continue; + } + + printValueAsJSON(state, true, *metaValue, noPos, ss, context); + + meta_[name] = nlohmann::json::parse(ss.str()); + } + meta = meta_; + } + + name = drvInfo.queryName(); + system = drvInfo.querySystem(); + drvPath = localStore->printStorePath(drvInfo.requireDrvPath()); + } +}; + +static void to_json(nlohmann::json & json, const Drv & drv) { + json = nlohmann::json{ + { "name", drv.name }, + { "system", drv.system }, + { "drvPath", drv.drvPath }, + { "outputs", drv.outputs }, + }; + + if (drv.meta.has_value()) + json["meta"] = drv.meta.value(); + +} + static void worker( EvalState & state, Bindings & autoArgs, @@ -202,50 +261,20 @@ static void worker( auto v = state.allocValue(); state.autoCallFunction(autoArgs, *vTmp, *v); - if (auto drv = getDerivation(state, *v, false)) { - - if (drv->querySystem() == "unknown") - throw EvalError("derivation must have a 'system' attribute"); + if (auto drvInfo = getDerivation(state, *v, false)) { + auto drv = Drv(state, *drvInfo); auto localStore = state.store.dynamic_pointer_cast(); - auto drvPath = localStore->printStorePath(drv->requireDrvPath()); - auto storePath = localStore->parseStorePath(drvPath); - auto outputs = drv->queryOutputs(); + auto storePath = localStore->parseStorePath(drv.drvPath); - reply["name"] = drv->queryName(); - reply["system"] = drv->querySystem(); - reply["drvPath"] = drvPath; - for (auto out : outputs){ - if (out.second) { - reply["outputs"][out.first] = localStore->printStorePath(*out.second); - } - } - - if (myArgs.meta) { - nlohmann::json meta; - for (auto & name : drv->queryMetaNames()) { - PathSet context; - std::stringstream ss; - - auto metaValue = drv->queryMeta(name); - // Skip non-serialisable types - // TODO: Fix serialisation of derivations to store paths - if (metaValue == 0) { - continue; - } - - printValueAsJSON(state, true, *metaValue, noPos, ss, context); - nlohmann::json field = nlohmann::json::parse(ss.str()); - meta[name] = field; - } - reply["meta"] = meta; - } + reply = drv; + reply["attr"] = attrPath; /* Register the derivation as a GC root. !!! This registers roots for jobs that we may have already done. */ if (myArgs.gcRootsDir != "") { - Path root = myArgs.gcRootsDir + "/" + std::string(baseNameOf(drvPath)); + Path root = myArgs.gcRootsDir + "/" + std::string(baseNameOf(drv.drvPath)); if (!pathExists(root)) localStore->addPermRoot(storePath, root); } From 7aa9835adb0ff1713fd4a6bf2fff7dcd90e3371e Mon Sep 17 00:00:00 2001 From: John Soo Date: Thu, 21 Apr 2022 11:22:29 -0700 Subject: [PATCH 114/419] Cleanup gcroots creation. --- src/nix-eval-jobs.cc | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/nix-eval-jobs.cc b/src/nix-eval-jobs.cc index 6eb275151..3f43d84f2 100644 --- a/src/nix-eval-jobs.cc +++ b/src/nix-eval-jobs.cc @@ -264,8 +264,6 @@ static void worker( if (auto drvInfo = getDerivation(state, *v, false)) { auto drv = Drv(state, *drvInfo); - auto localStore = state.store.dynamic_pointer_cast(); - auto storePath = localStore->parseStorePath(drv.drvPath); reply = drv; reply["attr"] = attrPath; @@ -275,8 +273,11 @@ static void worker( done. */ if (myArgs.gcRootsDir != "") { Path root = myArgs.gcRootsDir + "/" + std::string(baseNameOf(drv.drvPath)); - if (!pathExists(root)) + if (!pathExists(root)) { + auto localStore = state.store.dynamic_pointer_cast(); + auto storePath = localStore->parseStorePath(drv.drvPath); localStore->addPermRoot(storePath, root); + } } } From f98da2e00b59b9f124002251e7f1b452402c512a Mon Sep 17 00:00:00 2001 From: John Soo Date: Thu, 21 Apr 2022 11:25:40 -0700 Subject: [PATCH 115/419] Cleanup worker json handling. --- src/nix-eval-jobs.cc | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/nix-eval-jobs.cc b/src/nix-eval-jobs.cc index 3f43d84f2..3031c6955 100644 --- a/src/nix-eval-jobs.cc +++ b/src/nix-eval-jobs.cc @@ -252,9 +252,7 @@ static void worker( debug("worker process %d at '%s'", getpid(), attrPath); /* Evaluate it and send info back to the collector. */ - nlohmann::json reply; - reply["attr"] = attrPath; - + nlohmann::json reply = nlohmann::json{ { "attr", attrPath } }; try { auto vTmp = findAlongAttrPath(state, attrPath, autoArgs, *vRoot).first; @@ -264,9 +262,7 @@ static void worker( if (auto drvInfo = getDerivation(state, *v, false)) { auto drv = Drv(state, *drvInfo); - - reply = drv; - reply["attr"] = attrPath; + reply.update(drv); /* Register the derivation as a GC root. !!! This registers roots for jobs that we may have already From 813c5d7aae9d63ceadabe478ab45fb91e62d4202 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Fri, 22 Apr 2022 20:36:52 +1200 Subject: [PATCH 116/419] Add test for a job with a nested attrset --- tests/assets/ci.nix | 8 +++++--- tests/assets/flake.nix | 5 +---- tests/test_eval.py | 8 ++++++-- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/tests/assets/ci.nix b/tests/assets/ci.nix index 3aa2eca91..27b46040e 100644 --- a/tests/assets/ci.nix +++ b/tests/assets/ci.nix @@ -1,7 +1,9 @@ -let - pkgs = import (builtins.getFlake (toString ./.)).inputs.nixpkgs { }; -in +{ pkgs ? import (builtins.getFlake (toString ./.)).inputs.nixpkgs { } }: + { builtJob = pkgs.writeText "job1" "job1"; substitutedJob = pkgs.hello; + nested = { + job = pkgs.hello; + }; } diff --git a/tests/assets/flake.nix b/tests/assets/flake.nix index c3835109d..bedd2257f 100644 --- a/tests/assets/flake.nix +++ b/tests/assets/flake.nix @@ -6,9 +6,6 @@ pkgs = nixpkgs.legacyPackages.x86_64-linux; in { - hydraJobs = { - builtJob = pkgs.writeText "job1" "job1"; - substitutedJob = pkgs.hello; - }; + hydraJobs = import ./ci.nix { inherit pkgs; }; }; } diff --git a/tests/test_eval.py b/tests/test_eval.py index db93f9745..a334d519f 100644 --- a/tests/test_eval.py +++ b/tests/test_eval.py @@ -23,7 +23,7 @@ def common_test(extra_args: List[str]) -> None: ) results = [json.loads(r) for r in res.stdout.split("\n") if r] - assert len(results) == 2 + assert len(results) == 3 built_job = results[0] assert built_job["attr"] == "builtJob" @@ -32,7 +32,11 @@ def common_test(extra_args: List[str]) -> None: assert built_job["drvPath"].endswith(".drv") assert built_job["meta"]['broken'] is False - substituted_job = results[1] + nested_job = results[1] + assert nested_job["attr"] == "nested.job" + assert nested_job["name"].startswith("hello-") + + substituted_job = results[2] assert substituted_job["attr"] == "substitutedJob" assert substituted_job["name"].startswith("hello-") assert substituted_job["meta"]['broken'] is False From 1ea8948eb718782265054c2cb9a289a1ee03a3fb Mon Sep 17 00:00:00 2001 From: adisbladis Date: Sun, 24 Apr 2022 14:07:25 +1200 Subject: [PATCH 117/419] Fix build with stable Nix --- flake.lock | 12 ++++++------ flake.nix | 6 ++---- meson.build | 1 + src/meson.build | 1 + 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/flake.lock b/flake.lock index 0f133c751..b832c0ff1 100644 --- a/flake.lock +++ b/flake.lock @@ -2,11 +2,11 @@ "nodes": { "flake-utils": { "locked": { - "lastModified": 1644229661, - "narHash": "sha256-1YdnJAsNy69bpcjuoKdOYQX0YxZBiCYZo4Twxerqv7k=", + "lastModified": 1649676176, + "narHash": "sha256-OWKJratjt2RW151VUlJPRALb7OU2S5s+f0vLj4o1bHM=", "owner": "numtide", "repo": "flake-utils", - "rev": "3cecb5b042f7f209c56ffd8371b2711a290ec797", + "rev": "a4b154ebbdc88c8498a5c7b01589addc9e9cb678", "type": "github" }, "original": { @@ -17,11 +17,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1650161686, - "narHash": "sha256-70ZWAlOQ9nAZ08OU6WY7n4Ij2kOO199dLfNlvO/+pf8=", + "lastModified": 1650701402, + "narHash": "sha256-XKfstdtqDg+O+gNBx1yGVKWIhLgfEDg/e2lvJSsp9vU=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "1ffba9f2f683063c2b14c9f4d12c55ad5f4ed887", + "rev": "bc41b01dd7a9fdffd32d9b03806798797532a5fe", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index 8b5bc6c9d..ecd3fc805 100644 --- a/flake.nix +++ b/flake.nix @@ -17,7 +17,7 @@ let mkVariant = nix: (packages.nix-eval-jobs.override { # TODO: fix to stable after next nix release - nix = pkgs.nixUnstable; + nix = pkgs.nix; #inherit nix; }).overrideAttrs (_: { name = "nix-eval-jobs-${nix.version}"; @@ -46,9 +46,7 @@ touch $out ''; - # TODO fix to unstable in next release - build = mkVariant pkgs.nixUnstable; - #build = mkVariant pkgs.nix; + build = mkVariant pkgs.nix; build-unstable = mkVariant pkgs.nixUnstable; }; diff --git a/meson.build b/meson.build index 1767e6380..00086fdf7 100644 --- a/meson.build +++ b/meson.build @@ -6,6 +6,7 @@ project('nix-eval-jobs', 'cpp', nix_main_dep = dependency('nix-main', required: true) nix_store_dep = dependency('nix-store', required: true) nix_expr_dep = dependency('nix-expr', required: true) +nix_cmd_dep = dependency('nix-cmd', required: true) threads_dep = dependency('threads', required: true) nlohmann_json_dep = dependency('nlohmann_json', required: true) boost_dep = dependency('boost', required: true) diff --git a/src/meson.build b/src/meson.build index 56c480d54..ef01a7d93 100644 --- a/src/meson.build +++ b/src/meson.build @@ -7,6 +7,7 @@ executable('nix-eval-jobs', src, nix_main_dep, nix_store_dep, nix_expr_dep, + nix_cmd_dep, boost_dep, nlohmann_json_dep, threads_dep From c1bbb11c5d41b2a0596de5e16d7a2f990e8d2b16 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Mon, 25 Apr 2022 21:23:06 +1200 Subject: [PATCH 118/419] Add support for recurseForDerivations This will respect `recurseForDerivations` when iterating over attrsets. Example expression: ``` nix { system ? builtins.currentSystem }: { recurseForDerivations = true; # This should build as it's in the top-level attrset drvA = derivation { inherit system; name = "drvA"; builder = ":"; }; dontRecurse = { # This shouldn't build as `recurseForDerivations = true;` is not set # recurseForDerivations = true; # This should not build drvB = derivation { inherit system; name = "drvA"; builder = ":"; }; }; recurse = { # This should build recurseForDerivations = true; # This should not build drvC = derivation { inherit system; name = "drvC"; builder = ":"; }; }; } ``` --- src/nix-eval-jobs.cc | 13 +++++++++++-- tests/assets/ci.nix | 31 ++++++++++++++++++++++++++++--- tests/test_eval.py | 13 ++++++++----- 3 files changed, 47 insertions(+), 10 deletions(-) diff --git a/src/nix-eval-jobs.cc b/src/nix-eval-jobs.cc index 3031c6955..8bd2479a9 100644 --- a/src/nix-eval-jobs.cc +++ b/src/nix-eval-jobs.cc @@ -281,7 +281,8 @@ static void worker( else if (v->type() == nAttrs) { auto attrs = nlohmann::json::array(); - StringSet ss; + bool recurse = attrPath == ""; // Dont require `recurseForDerivations = true;` for top-level attrset + for (auto & i : v->attrs->lexicographicOrder()) { std::string name(i->name); if (name.find('.') != std::string::npos || name.find(' ') != std::string::npos) { @@ -289,8 +290,16 @@ static void worker( continue; } attrs.push_back(name); + + if (name == "recurseForDerivations") { + auto attrv = v->attrs->get(state.sRecurseForDerivations); + recurse = state.forceBool(*attrv->value, *attrv->pos); + } } - reply["attrs"] = std::move(attrs); + if (recurse) + reply["attrs"] = std::move(attrs); + else + reply["attrs"] = nlohmann::json::array(); } else if (v->type() == nNull) diff --git a/tests/assets/ci.nix b/tests/assets/ci.nix index 27b46040e..a80ad8804 100644 --- a/tests/assets/ci.nix +++ b/tests/assets/ci.nix @@ -1,9 +1,34 @@ -{ pkgs ? import (builtins.getFlake (toString ./.)).inputs.nixpkgs { } }: +{ + pkgs ? import (builtins.getFlake (toString ./.)).inputs.nixpkgs { } + , system ? pkgs.system +}: { builtJob = pkgs.writeText "job1" "job1"; substitutedJob = pkgs.hello; - nested = { - job = pkgs.hello; + + dontRecurse = { + # This shouldn't build as `recurseForDerivations = true;` is not set + # recurseForDerivations = true; + + # This should not build + drvB = derivation { + inherit system; + name = "drvA"; + builder = ":"; + }; }; + + recurse = { + # This should build + recurseForDerivations = true; + + # This should not build + drvB = derivation { + inherit system; + name = "drvB"; + builder = ":"; + }; + }; + } diff --git a/tests/test_eval.py b/tests/test_eval.py index a334d519f..0c865240c 100644 --- a/tests/test_eval.py +++ b/tests/test_eval.py @@ -23,7 +23,7 @@ def common_test(extra_args: List[str]) -> None: ) results = [json.loads(r) for r in res.stdout.split("\n") if r] - assert len(results) == 3 + assert len(results) == 4 built_job = results[0] assert built_job["attr"] == "builtJob" @@ -32,11 +32,14 @@ def common_test(extra_args: List[str]) -> None: assert built_job["drvPath"].endswith(".drv") assert built_job["meta"]['broken'] is False - nested_job = results[1] - assert nested_job["attr"] == "nested.job" - assert nested_job["name"].startswith("hello-") + recurse_drv = results[1] + assert recurse_drv["attr"] == "recurse.drvB" + assert recurse_drv["name"] == "drvB" - substituted_job = results[2] + recurse_recurse_bool = results[2] + assert "error" in recurse_recurse_bool + + substituted_job = results[3] assert substituted_job["attr"] == "substitutedJob" assert substituted_job["name"].startswith("hello-") assert substituted_job["meta"]['broken'] is False From 859db1330b9a11a5d6b737889e7d6fbae1017038 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 25 Apr 2022 23:11:18 +0000 Subject: [PATCH 119/419] Bump DeterminateSystems/update-flake-lock from 8 to 9 Bumps [DeterminateSystems/update-flake-lock](https://github.com/DeterminateSystems/update-flake-lock) from 8 to 9. - [Release notes](https://github.com/DeterminateSystems/update-flake-lock/releases) - [Commits](https://github.com/DeterminateSystems/update-flake-lock/compare/v8...v9) --- updated-dependencies: - dependency-name: DeterminateSystems/update-flake-lock dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/update-flake-lock.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/update-flake-lock.yml b/.github/workflows/update-flake-lock.yml index b06d20f05..b89af8801 100644 --- a/.github/workflows/update-flake-lock.yml +++ b/.github/workflows/update-flake-lock.yml @@ -16,6 +16,6 @@ jobs: # extra_nix_config: | # access-tokens = github.com=${{ secrets.GITHUB_TOKEN }} - name: Update flake.lock - uses: DeterminateSystems/update-flake-lock@v8 + uses: DeterminateSystems/update-flake-lock@v9 with: token: ${{ secrets.GH_TOKEN_FOR_UPDATES }} From 61c9f4cfcca405c439e4ba373e647ab87300e28a Mon Sep 17 00:00:00 2001 From: adisbladis Date: Tue, 26 Apr 2022 17:11:17 +1200 Subject: [PATCH 120/419] Replace internal attr string representation with an array This ensures correct handling of attrnames with a dot in them and will not throw errors about illegal attrnames. Additionally this escapes any attributes containing dots in the JSON output and adds another field called `attrPath` which contains the attribute path as a list. Example output: ``` { "attr": "hello", "attrPath": [ "hello" ], "drvPath": "/nix/store/n204jib73z55cp9s0rmw1c5v5q528j7v-hello-2.12.drv", "name": "hello-2.12", "outputs": { "out": "/nix/store/h59dfk7dwrn7d2csykh9z9xm2miqmrnz-hello-2.12" }, "system": "x86_64-linux" } ``` --- src/nix-eval-jobs.cc | 55 ++++++++++++++++++++++++++------------------ tests/assets/ci.nix | 2 ++ tests/test_eval.py | 12 ++++++---- 3 files changed, 43 insertions(+), 26 deletions(-) diff --git a/src/nix-eval-jobs.cc b/src/nix-eval-jobs.cc index 8bd2479a9..68e5d1259 100644 --- a/src/nix-eval-jobs.cc +++ b/src/nix-eval-jobs.cc @@ -28,6 +28,7 @@ #include using namespace nix; +using namespace nlohmann; typedef enum { evalAuto, evalImpure, evalPure } pureEval; @@ -232,6 +233,17 @@ static void to_json(nlohmann::json & json, const Drv & drv) { } +std::string attrPathJoin(json input) { + return std::accumulate(input.begin(), input.end(), std::string(), + [](std::string ss, std::string s) { + // Escape token if containing dots + if (s.find(".") != std::string::npos) { + s = "\"" + s + "\""; + } + return ss.empty() ? s : ss + "." + s; + }); +} + static void worker( EvalState & state, Bindings & autoArgs, @@ -247,14 +259,15 @@ static void worker( auto s = readLine(from.get()); if (s == "exit") break; if (!hasPrefix(s, "do ")) abort(); - std::string attrPath(s, 3); + auto path = json::parse(s.substr(3)); + auto attrPathS = attrPathJoin(path); - debug("worker process %d at '%s'", getpid(), attrPath); + debug("worker process %d at '%s'", getpid(), path); /* Evaluate it and send info back to the collector. */ - nlohmann::json reply = nlohmann::json{ { "attr", attrPath } }; + json reply = json{ {"attr", attrPathS }, {"attrPath", path} }; try { - auto vTmp = findAlongAttrPath(state, attrPath, autoArgs, *vRoot).first; + auto vTmp = findAlongAttrPath(state, attrPathS, autoArgs, *vRoot).first; auto v = state.allocValue(); state.autoCallFunction(autoArgs, *vTmp, *v); @@ -281,14 +294,10 @@ static void worker( else if (v->type() == nAttrs) { auto attrs = nlohmann::json::array(); - bool recurse = attrPath == ""; // Dont require `recurseForDerivations = true;` for top-level attrset + bool recurse = path.size() == 0; // Dont require `recurseForDerivations = true;` for top-level attrset for (auto & i : v->attrs->lexicographicOrder()) { std::string name(i->name); - if (name.find('.') != std::string::npos || name.find(' ') != std::string::npos) { - printError("skipping job with illegal name '%s'", name); - continue; - } attrs.push_back(name); if (name == "recurseForDerivations") { @@ -305,7 +314,7 @@ static void worker( else if (v->type() == nNull) ; - else throw TypeError("attribute '%s' is %s, which is not supported", attrPath, showType(*v)); + else throw TypeError("attribute '%s' is %s, which is not supported", path, showType(*v)); } catch (EvalError & e) { auto err = e.info(); @@ -380,9 +389,9 @@ struct Proc { struct State { - std::set todo{""}; - std::set active; - std::exception_ptr exc; + std::set todo = json::array({ json::array() }); + std::set active; + std::exception_ptr exc; }; std::function collector(Sync & state_, std::condition_variable & wakeup) { @@ -402,12 +411,12 @@ std::function collector(Sync & state_, std::condition_variable & proc_ = std::nullopt; continue; } else if (s != "next") { - auto json = nlohmann::json::parse(s); + auto json = json::parse(s); throw Error("worker error: %s", (std::string) json["error"]); } /* Wait for a job name to become available. */ - std::string attrPath; + json attrPath; while (true) { checkInterrupt(); @@ -426,18 +435,19 @@ std::function collector(Sync & state_, std::condition_variable & } /* Tell the worker to evaluate it. */ - writeLine(proc->to.get(), "do " + attrPath); + writeLine(proc->to.get(), "do " + attrPath.dump()); /* Wait for the response. */ auto respString = readLine(proc->from.get()); - auto response = nlohmann::json::parse(respString); + auto response = json::parse(respString); /* Handle the response. */ - StringSet newAttrs; + std::vector newAttrs; if (response.find("attrs") != response.end()) { for (auto & i : response["attrs"]) { - auto s = (attrPath.empty() ? "" : attrPath + ".") + (std::string) i; - newAttrs.insert(s); + json newAttr = json(response["attrPath"]); + newAttr.emplace_back(i); + newAttrs.push_back(newAttr); } } else { auto state(state_.lock()); @@ -450,8 +460,9 @@ std::function collector(Sync & state_, std::condition_variable & { auto state(state_.lock()); state->active.erase(attrPath); - for (auto & s : newAttrs) - state->todo.insert(s); + for (auto p : newAttrs) { + state->todo.insert(p); + } wakeup.notify_all(); } } diff --git a/tests/assets/ci.nix b/tests/assets/ci.nix index a80ad8804..150f718a3 100644 --- a/tests/assets/ci.nix +++ b/tests/assets/ci.nix @@ -31,4 +31,6 @@ }; }; + "dotted.attr" = pkgs.hello; + } diff --git a/tests/test_eval.py b/tests/test_eval.py index 0c865240c..48bba7cc9 100644 --- a/tests/test_eval.py +++ b/tests/test_eval.py @@ -23,7 +23,7 @@ def common_test(extra_args: List[str]) -> None: ) results = [json.loads(r) for r in res.stdout.split("\n") if r] - assert len(results) == 4 + assert len(results) == 5 built_job = results[0] assert built_job["attr"] == "builtJob" @@ -32,14 +32,18 @@ def common_test(extra_args: List[str]) -> None: assert built_job["drvPath"].endswith(".drv") assert built_job["meta"]['broken'] is False - recurse_drv = results[1] + dotted_job = results[1] + assert dotted_job["attr"] == "\"dotted.attr\"" + assert dotted_job["attrPath"] == [ "dotted.attr" ] + + recurse_drv = results[2] assert recurse_drv["attr"] == "recurse.drvB" assert recurse_drv["name"] == "drvB" - recurse_recurse_bool = results[2] + recurse_recurse_bool = results[3] assert "error" in recurse_recurse_bool - substituted_job = results[3] + substituted_job = results[4] assert substituted_job["attr"] == "substitutedJob" assert substituted_job["name"].startswith("hello-") assert substituted_job["meta"]['broken'] is False From 516fdc8f6c649dbccfe15d92b401bf2fa5aaf0d9 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Tue, 26 Apr 2022 20:48:39 +1200 Subject: [PATCH 121/419] Add treefmt And use it in place of editorconfig/nixpkgs-fmt checks --- .editorconfig | 24 ------------------------ flake.nix | 30 ++++++++++++------------------ shell.nix | 3 +-- tests/assets/ci.nix | 5 ++--- treefmt.toml | 3 +++ 5 files changed, 18 insertions(+), 47 deletions(-) delete mode 100644 .editorconfig create mode 100644 treefmt.toml diff --git a/.editorconfig b/.editorconfig deleted file mode 100644 index 1ccb157ca..000000000 --- a/.editorconfig +++ /dev/null @@ -1,24 +0,0 @@ -# top-most EditorConfig file -root = true - -# Unix-style newlines with a newline ending every file -[*] -charset = utf-8 -end_of_line = lf -insert_final_newline = true -trim_trailing_whitespace = true - -[*.{cc,hh,hpp,pl,pm,sh,t}] -indent_style = space -intend_size = 4 - -[Makefile] -indent_style = tab - -[*.nix] -indent_style = space -indent_size = 2 - -# Match diffs, avoid to trim trailing whitespace -[*.{diff,patch}] -trim_trailing_whitespace = false diff --git a/flake.nix b/flake.nix index ecd3fc805..c15587efc 100644 --- a/flake.nix +++ b/flake.nix @@ -8,6 +8,7 @@ flake-utils.lib.eachDefaultSystem (system: let pkgs = nixpkgs.legacyPackages.${system}; + inherit (pkgs) stdenv; drvArgs = { srcDir = self; }; in rec { @@ -26,25 +27,18 @@ in { - editorconfig = pkgs.runCommand "editorconfig-check" - { - nativeBuildInputs = [ - pkgs.editorconfig-checker - ]; - } '' - editorconfig-checker ${self} - touch $out - ''; + treefmt = stdenv.mkDerivation { + name = "treefmt-check"; + src = self; + nativeBuildInputs = devShells.default.nativeBuildInputs; + dontConfigure = true; - nixpkgs-fmt = pkgs.runCommand "fmt-check" - { - nativeBuildInputs = [ - pkgs.nixpkgs-fmt - ]; - } '' - nixpkgs-fmt --check . - touch $out - ''; + buildPhase = '' + env HOME=$(mktemp -d) treefmt --fail-on-change + ''; + + installPhase = "touch $out"; + }; build = mkVariant pkgs.nix; build-unstable = mkVariant pkgs.nixUnstable; diff --git a/shell.nix b/shell.nix index 451727833..159cd4248 100644 --- a/shell.nix +++ b/shell.nix @@ -26,8 +26,7 @@ in nativeBuildInputs = old.nativeBuildInputs ++ [ - pkgs.editorconfig-checker - + pkgs.treefmt pkgs.nixpkgs-fmt (pkgs.python3.withPackages (ps: [ diff --git a/tests/assets/ci.nix b/tests/assets/ci.nix index 150f718a3..5354be14b 100644 --- a/tests/assets/ci.nix +++ b/tests/assets/ci.nix @@ -1,6 +1,5 @@ -{ - pkgs ? import (builtins.getFlake (toString ./.)).inputs.nixpkgs { } - , system ? pkgs.system +{ pkgs ? import (builtins.getFlake (toString ./.)).inputs.nixpkgs { } +, system ? pkgs.system }: { diff --git a/treefmt.toml b/treefmt.toml new file mode 100644 index 000000000..8309d20c8 --- /dev/null +++ b/treefmt.toml @@ -0,0 +1,3 @@ +[formatter.nix] +command = "nixpkgs-fmt" +includes = ["*.nix"] From 05218ff30ddd941e1f515290b59142a5c462f020 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Tue, 26 Apr 2022 21:03:08 +1200 Subject: [PATCH 122/419] Fix nix flake check warnings about deprecated attr names --- flake.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flake.nix b/flake.nix index c15587efc..f041e9404 100644 --- a/flake.nix +++ b/flake.nix @@ -44,8 +44,8 @@ build-unstable = mkVariant pkgs.nixUnstable; }; - defaultPackage = self.packages.${system}.nix-eval-jobs; - devShell = pkgs.callPackage ./shell.nix drvArgs; + packages.default = self.packages.${system}.nix-eval-jobs; + devShells.default = pkgs.callPackage ./shell.nix drvArgs; } ); From 76f68a8605aca83bcb4cd264be0440d45b84a109 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Tue, 26 Apr 2022 21:04:10 +1200 Subject: [PATCH 123/419] Fix using erroneous Nix derivation in mkVariant based tests --- flake.nix | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/flake.nix b/flake.nix index f041e9404..1f8605efc 100644 --- a/flake.nix +++ b/flake.nix @@ -17,9 +17,7 @@ checks = let mkVariant = nix: (packages.nix-eval-jobs.override { - # TODO: fix to stable after next nix release - nix = pkgs.nix; - #inherit nix; + inherit nix; }).overrideAttrs (_: { name = "nix-eval-jobs-${nix.version}"; inherit (nix) version; From 8b8f456765943bfdb4587ec85a881246e6a580ef Mon Sep 17 00:00:00 2001 From: adisbladis Date: Tue, 26 Apr 2022 21:04:54 +1200 Subject: [PATCH 124/419] Add clang-format --- shell.nix | 1 + treefmt.toml | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/shell.nix b/shell.nix index 159cd4248..d920a6f32 100644 --- a/shell.nix +++ b/shell.nix @@ -27,6 +27,7 @@ in nativeBuildInputs = old.nativeBuildInputs ++ [ pkgs.treefmt + pkgs.llvmPackages.clang # clang-format pkgs.nixpkgs-fmt (pkgs.python3.withPackages (ps: [ diff --git a/treefmt.toml b/treefmt.toml index 8309d20c8..aacc3bb10 100644 --- a/treefmt.toml +++ b/treefmt.toml @@ -1,3 +1,8 @@ +[formatter."c++"] +command = "clang-format" +options = [ "-i", "-style", "{BasedOnStyle: llvm, IndentWidth: 4, SortIncludes: false}" ] +includes = [ "*.c", "*.cpp", "*.cc", "*.h", "*.hpp" ] + [formatter.nix] command = "nixpkgs-fmt" includes = ["*.nix"] From fec10f2cfcb2e1304d29f789b4dbac7560a468af Mon Sep 17 00:00:00 2001 From: adisbladis Date: Tue, 26 Apr 2022 21:15:56 +1200 Subject: [PATCH 125/419] src/nix-eval-jobs.cc: Reformat with clang-format --- src/nix-eval-jobs.cc | 240 +++++++++++++++++++++---------------------- 1 file changed, 116 insertions(+), 124 deletions(-) diff --git a/src/nix-eval-jobs.cc b/src/nix-eval-jobs.cc index 68e5d1259..b03fa9354 100644 --- a/src/nix-eval-jobs.cc +++ b/src/nix-eval-jobs.cc @@ -35,8 +35,7 @@ typedef enum { evalAuto, evalImpure, evalPure } pureEval; // Safe to ignore - the args will be static. #pragma GCC diagnostic ignored "-Wnon-virtual-dtor" #pragma clang diagnostic ignored "-Wnon-virtual-dtor" -struct MyArgs : MixEvalArgs, MixCommonArgs -{ +struct MyArgs : MixEvalArgs, MixCommonArgs { Path releaseExpr; Path gcRootsDir; bool flake = false; @@ -46,18 +45,18 @@ struct MyArgs : MixEvalArgs, MixCommonArgs size_t maxMemorySize = 4096; pureEval evalMode = evalAuto; - MyArgs() : MixCommonArgs("nix-eval-jobs") - { + MyArgs() : MixCommonArgs("nix-eval-jobs") { addFlag({ .longName = "help", .description = "show usage information", .handler = {[&]() { printf("USAGE: nix-eval-jobs [options] expr\n\n"); - for (const auto & [name, flag] : longFlags) { + for (const auto &[name, flag] : longFlags) { if (hiddenCategories.count(flag->category)) { continue; } - printf(" --%-20s %s\n", name.c_str(), flag->description.c_str()); + printf(" --%-20s %s\n", name.c_str(), + flag->description.c_str()); } ::exit(0); }}, @@ -66,53 +65,38 @@ struct MyArgs : MixEvalArgs, MixCommonArgs addFlag({ .longName = "impure", .description = "set evaluation mode", - .handler = {[&]() { - evalMode = evalImpure; - }}, + .handler = {[&]() { evalMode = evalImpure; }}, }); - addFlag({ - .longName = "gc-roots-dir", - .description = "garbage collector roots directory", - .labels = {"path"}, - .handler = {&gcRootsDir} - }); + addFlag({.longName = "gc-roots-dir", + .description = "garbage collector roots directory", + .labels = {"path"}, + .handler = {&gcRootsDir}}); - addFlag({ - .longName = "workers", - .description = "number of evaluate workers", - .labels = {"workers"}, - .handler = {[=](std::string s) { - nrWorkers = std::stoi(s); - }} - }); + addFlag( + {.longName = "workers", + .description = "number of evaluate workers", + .labels = {"workers"}, + .handler = {[=](std::string s) { nrWorkers = std::stoi(s); }}}); - addFlag({ - .longName = "max-memory-size", - .description = "maximum evaluation memory size", - .labels = {"size"}, - .handler = {[=](std::string s) { - maxMemorySize = std::stoi(s); - }} - }); + addFlag({.longName = "max-memory-size", + .description = "maximum evaluation memory size", + .labels = {"size"}, + .handler = { + [=](std::string s) { maxMemorySize = std::stoi(s); }}}); - addFlag({ - .longName = "flake", - .description = "build a flake", - .handler = {&flake, true} - }); + addFlag({.longName = "flake", + .description = "build a flake", + .handler = {&flake, true}}); - addFlag({ - .longName = "meta", - .description = "include derivation meta field in output", - .handler = {&meta, true} - }); + addFlag({.longName = "meta", + .description = "include derivation meta field in output", + .handler = {&meta, true}}); - addFlag({ - .longName = "show-trace", - .description = "print out a stack trace in case of evaluation errors", - .handler = {&showTrace, true} - }); + addFlag({.longName = "show-trace", + .description = + "print out a stack trace in case of evaluation errors", + .handler = {&showTrace, true}}); expectArg("expr", &releaseExpr); } @@ -122,7 +106,7 @@ struct MyArgs : MixEvalArgs, MixCommonArgs static MyArgs myArgs; -static Value* releaseExprTopLevelValue(EvalState & state, Bindings & autoArgs) { +static Value *releaseExprTopLevelValue(EvalState &state, Bindings &autoArgs) { Value vTop; state.evalFile(lookupFileArg(state, myArgs.releaseExpr), vTop); @@ -134,19 +118,20 @@ static Value* releaseExprTopLevelValue(EvalState & state, Bindings & autoArgs) { return vRoot; } -static Value* flakeTopLevelValue(EvalState & state, Bindings & autoArgs) { +static Value *flakeTopLevelValue(EvalState &state, Bindings &autoArgs) { using namespace flake; - auto [flakeRef, fragment] = parseFlakeRefWithFragment(myArgs.releaseExpr, absPath(".")); + auto [flakeRef, fragment] = + parseFlakeRefWithFragment(myArgs.releaseExpr, absPath(".")); auto vFlake = state.allocValue(); auto lockedFlake = lockFlake(state, flakeRef, - LockFlags { - .updateLockFile = false, - .useRegistries = false, - .allowMutable = false, - }); + LockFlags{ + .updateLockFile = false, + .useRegistries = false, + .allowMutable = false, + }); callFlake(state, lockedFlake, *vFlake); @@ -155,7 +140,7 @@ static Value* flakeTopLevelValue(EvalState & state, Bindings & autoArgs) { auto vTop = *vOutputs; if (fragment.length() > 0) { - Bindings & bindings(*state.allocBindings(0)); + Bindings &bindings(*state.allocBindings(0)); auto [nTop, pos] = findAlongAttrPath(state, fragment, bindings, vTop); if (!nTop) throw Error("error: attribute '%s' missing", nTop); @@ -168,10 +153,9 @@ static Value* flakeTopLevelValue(EvalState & state, Bindings & autoArgs) { return vRoot; } -Value * topLevelValue(EvalState & state, Bindings & autoArgs) { - return myArgs.flake - ? flakeTopLevelValue(state, autoArgs) - : releaseExprTopLevelValue(state, autoArgs); +Value *topLevelValue(EvalState &state, Bindings &autoArgs) { + return myArgs.flake ? flakeTopLevelValue(state, autoArgs) + : releaseExprTopLevelValue(state, autoArgs); } /* The fields of a derivation that are printed in json form */ @@ -182,7 +166,7 @@ struct Drv { std::map outputs; std::optional meta; - Drv (EvalState & state, DrvInfo & drvInfo) { + Drv(EvalState &state, DrvInfo &drvInfo) { if (drvInfo.querySystem() == "unknown") throw EvalError("derivation must have a 'system' attribute"); @@ -191,12 +175,11 @@ struct Drv { for (auto out : drvInfo.queryOutputs(true)) { if (out.second) outputs[out.first] = localStore->printStorePath(*out.second); - } if (myArgs.meta) { nlohmann::json meta_; - for (auto & name : drvInfo.queryMetaNames()) { + for (auto &name : drvInfo.queryMetaNames()) { PathSet context; std::stringstream ss; @@ -220,17 +203,16 @@ struct Drv { } }; -static void to_json(nlohmann::json & json, const Drv & drv) { +static void to_json(nlohmann::json &json, const Drv &drv) { json = nlohmann::json{ - { "name", drv.name }, - { "system", drv.system }, - { "drvPath", drv.drvPath }, - { "outputs", drv.outputs }, + {"name", drv.name}, + {"system", drv.system}, + {"drvPath", drv.drvPath}, + {"outputs", drv.outputs}, }; if (drv.meta.has_value()) json["meta"] = drv.meta.value(); - } std::string attrPathJoin(json input) { @@ -244,12 +226,8 @@ std::string attrPathJoin(json input) { }); } -static void worker( - EvalState & state, - Bindings & autoArgs, - AutoCloseFD & to, - AutoCloseFD & from) -{ +static void worker(EvalState &state, Bindings &autoArgs, AutoCloseFD &to, + AutoCloseFD &from) { auto vRoot = topLevelValue(state, autoArgs); while (true) { @@ -257,17 +235,20 @@ static void worker( writeLine(to.get(), "next"); auto s = readLine(from.get()); - if (s == "exit") break; - if (!hasPrefix(s, "do ")) abort(); + if (s == "exit") + break; + if (!hasPrefix(s, "do ")) + abort(); auto path = json::parse(s.substr(3)); auto attrPathS = attrPathJoin(path); debug("worker process %d at '%s'", getpid(), path); /* Evaluate it and send info back to the collector. */ - json reply = json{ {"attr", attrPathS }, {"attrPath", path} }; + json reply = json{{"attr", attrPathS}, {"attrPath", path}}; try { - auto vTmp = findAlongAttrPath(state, attrPathS, autoArgs, *vRoot).first; + auto vTmp = + findAlongAttrPath(state, attrPathS, autoArgs, *vRoot).first; auto v = state.allocValue(); state.autoCallFunction(autoArgs, *vTmp, *v); @@ -281,42 +262,49 @@ static void worker( registers roots for jobs that we may have already done. */ if (myArgs.gcRootsDir != "") { - Path root = myArgs.gcRootsDir + "/" + std::string(baseNameOf(drv.drvPath)); + Path root = myArgs.gcRootsDir + "/" + + std::string(baseNameOf(drv.drvPath)); if (!pathExists(root)) { - auto localStore = state.store.dynamic_pointer_cast(); - auto storePath = localStore->parseStorePath(drv.drvPath); + auto localStore = + state.store.dynamic_pointer_cast(); + auto storePath = + localStore->parseStorePath(drv.drvPath); localStore->addPermRoot(storePath, root); } } } - else if (v->type() == nAttrs) - { + else if (v->type() == nAttrs) { auto attrs = nlohmann::json::array(); - bool recurse = path.size() == 0; // Dont require `recurseForDerivations = true;` for top-level attrset + bool recurse = + path.size() == 0; // Dont require `recurseForDerivations = + // true;` for top-level attrset - for (auto & i : v->attrs->lexicographicOrder()) { + for (auto &i : v->attrs->lexicographicOrder()) { std::string name(i->name); attrs.push_back(name); if (name == "recurseForDerivations") { - auto attrv = v->attrs->get(state.sRecurseForDerivations); - recurse = state.forceBool(*attrv->value, *attrv->pos); + auto attrv = + v->attrs->get(state.sRecurseForDerivations); + recurse = state.forceBool(*attrv->value, *attrv->pos); } } if (recurse) - reply["attrs"] = std::move(attrs); + reply["attrs"] = std::move(attrs); else - reply["attrs"] = nlohmann::json::array(); + reply["attrs"] = nlohmann::json::array(); } else if (v->type() == nNull) ; - else throw TypeError("attribute '%s' is %s, which is not supported", path, showType(*v)); + else + throw TypeError("attribute '%s' is %s, which is not supported", + path, showType(*v)); - } catch (EvalError & e) { + } catch (EvalError &e) { auto err = e.info(); std::ostringstream oss; @@ -337,13 +325,15 @@ static void worker( start a new process. */ struct rusage r; getrusage(RUSAGE_SELF, &r); - if ((size_t) r.ru_maxrss > myArgs.maxMemorySize * 1024) break; + if ((size_t)r.ru_maxrss > myArgs.maxMemorySize * 1024) + break; } writeLine(to.get(), "restart"); } -typedef std::function +typedef std::function Processor; /* Auto-cleanup of fork's process and fds. */ @@ -351,22 +341,21 @@ struct Proc { AutoCloseFD to, from; Pid pid; - Proc(const Processor & proc) { + Proc(const Processor &proc) { Pipe toPipe, fromPipe; toPipe.create(); fromPipe.create(); auto p = startProcess( [&, to{std::make_shared(std::move(fromPipe.writeSide))}, - from{std::make_shared(std::move(toPipe.readSide))} - ]() - { + from{ + std::make_shared(std::move(toPipe.readSide))}]() { debug("created worker process %d", getpid()); try { EvalState state(myArgs.searchPath, openStore()); - Bindings & autoArgs = *myArgs.getAutoArgs(state); + Bindings &autoArgs = *myArgs.getAutoArgs(state); proc(state, autoArgs, *to, *from); - } catch (Error & e) { + } catch (Error &e) { nlohmann::json err; auto msg = e.msg(); err["error"] = filterANSIEscapes(msg, true); @@ -377,33 +366,32 @@ struct Proc { writeLine(to->get(), "restart"); } }, - ProcessOptions { .allowVfork = false }); + ProcessOptions{.allowVfork = false}); to = std::move(toPipe.writeSide); from = std::move(fromPipe.readSide); pid = p; } - ~Proc() { } + ~Proc() {} }; -struct State -{ - std::set todo = json::array({ json::array() }); - std::set active; - std::exception_ptr exc; +struct State { + std::set todo = json::array({json::array()}); + std::set active; + std::exception_ptr exc; }; -std::function collector(Sync & state_, std::condition_variable & wakeup) { +std::function collector(Sync &state_, + std::condition_variable &wakeup) { return [&]() { try { std::optional> proc_; while (true) { - auto proc = proc_.has_value() - ? std::move(proc_.value()) - : std::make_unique(worker); + auto proc = proc_.has_value() ? std::move(proc_.value()) + : std::make_unique(worker); /* Check whether the existing worker process is still there. */ auto s = readLine(proc->from.get()); @@ -412,7 +400,7 @@ std::function collector(Sync & state_, std::condition_variable & continue; } else if (s != "next") { auto json = json::parse(s); - throw Error("worker error: %s", (std::string) json["error"]); + throw Error("worker error: %s", (std::string)json["error"]); } /* Wait for a job name to become available. */ @@ -421,7 +409,8 @@ std::function collector(Sync & state_, std::condition_variable & while (true) { checkInterrupt(); auto state(state_.lock()); - if ((state->todo.empty() && state->active.empty()) || state->exc) { + if ((state->todo.empty() && state->active.empty()) || + state->exc) { writeLine(proc->to.get(), "exit"); return; } @@ -444,10 +433,10 @@ std::function collector(Sync & state_, std::condition_variable & /* Handle the response. */ std::vector newAttrs; if (response.find("attrs") != response.end()) { - for (auto & i : response["attrs"]) { - json newAttr = json(response["attrPath"]); - newAttr.emplace_back(i); - newAttrs.push_back(newAttr); + for (auto &i : response["attrs"]) { + json newAttr = json(response["attrPath"]); + newAttr.emplace_back(i); + newAttrs.push_back(newAttr); } } else { auto state(state_.lock()); @@ -474,8 +463,7 @@ std::function collector(Sync & state_, std::condition_variable & }; } -int main(int argc, char * * argv) -{ +int main(int argc, char **argv) { /* Prevent undeclared dependencies in the evaluation via $NIX_PATH. */ unsetenv("NIX_PATH"); @@ -489,7 +477,8 @@ int main(int argc, char * * argv) myArgs.parseCmdline(argvToStrings(argc, argv)); - /* FIXME: The build hook in conjunction with import-from-derivation is causing "unexpected EOF" during eval */ + /* FIXME: The build hook in conjunction with import-from-derivation is + * causing "unexpected EOF" during eval */ settings.builders = ""; /* Prevent access to paths outside of the Nix search path and @@ -498,11 +487,15 @@ int main(int argc, char * * argv) /* When building a flake, use pure evaluation (no access to 'getEnv', 'currentSystem' etc. */ - evalSettings.pureEval = myArgs.evalMode == evalAuto ? myArgs.flake : myArgs.evalMode == evalPure; + evalSettings.pureEval = myArgs.evalMode == evalAuto + ? myArgs.flake + : myArgs.evalMode == evalPure; - if (myArgs.releaseExpr == "") throw UsageError("no expression specified"); + if (myArgs.releaseExpr == "") + throw UsageError("no expression specified"); - if (myArgs.gcRootsDir == "") printMsg(lvlError, "warning: `--gc-roots-dir' not specified"); + if (myArgs.gcRootsDir == "") + printMsg(lvlError, "warning: `--gc-roots-dir' not specified"); if (myArgs.showTrace) { loggerSettings.showTrace.assign(true); @@ -516,13 +509,12 @@ int main(int argc, char * * argv) for (size_t i = 0; i < myArgs.nrWorkers; i++) threads.emplace_back(std::thread(collector(state_, wakeup))); - for (auto & thread : threads) + for (auto &thread : threads) thread.join(); auto state(state_.lock()); if (state->exc) std::rethrow_exception(state->exc); - }); } From 2a26070ea49295d5ee3bd9e9e8418c56c41fd944 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Tue, 26 Apr 2022 21:34:51 +1200 Subject: [PATCH 126/419] Add TOML formatter (prettier) --- .prettierrc.js | 3 +++ flake.nix | 26 ++++++++++++++++---------- shell.nix | 3 +++ treefmt.toml | 13 +++++++++++-- 4 files changed, 33 insertions(+), 12 deletions(-) create mode 100644 .prettierrc.js diff --git a/.prettierrc.js b/.prettierrc.js new file mode 100644 index 000000000..dc1e020bf --- /dev/null +++ b/.prettierrc.js @@ -0,0 +1,3 @@ +module.exports = { + "plugins": ["prettier-plugin-toml"], +}; diff --git a/flake.nix b/flake.nix index 1f8605efc..11567726e 100644 --- a/flake.nix +++ b/flake.nix @@ -25,18 +25,24 @@ in { - treefmt = stdenv.mkDerivation { - name = "treefmt-check"; - src = self; - nativeBuildInputs = devShells.default.nativeBuildInputs; - dontConfigure = true; + treefmt = + let + devShell = devShells.default; + in + stdenv.mkDerivation { + name = "treefmt-check"; + src = self; + nativeBuildInputs = devShell.nativeBuildInputs; + dontConfigure = true; - buildPhase = '' - env HOME=$(mktemp -d) treefmt --fail-on-change - ''; + inherit (devShell) NODE_PATH; - installPhase = "touch $out"; - }; + buildPhase = '' + env HOME=$(mktemp -d) treefmt --fail-on-change + ''; + + installPhase = "touch $out"; + }; build = mkVariant pkgs.nix; build-unstable = mkVariant pkgs.nixUnstable; diff --git a/shell.nix b/shell.nix index d920a6f32..5c0cd0db4 100644 --- a/shell.nix +++ b/shell.nix @@ -29,6 +29,7 @@ in pkgs.treefmt pkgs.llvmPackages.clang # clang-format pkgs.nixpkgs-fmt + pkgs.nodePackages.prettier (pkgs.python3.withPackages (ps: [ ps.pytest @@ -36,6 +37,8 @@ in ]; + NODE_PATH = "${pkgs.nodePackages.prettier-plugin-toml}/lib/node_modules"; + shellHook = lib.optionalString stdenv.isLinux '' export NIX_DEBUG_INFO_DIRS="${pkgs.curl.debug}/lib/debug:${nix.debug}/lib/debug''${NIX_DEBUG_INFO_DIRS:+:$NIX_DEBUG_INFO_DIRS}" ''; diff --git a/treefmt.toml b/treefmt.toml index aacc3bb10..d1280df60 100644 --- a/treefmt.toml +++ b/treefmt.toml @@ -1,8 +1,17 @@ [formatter."c++"] command = "clang-format" -options = [ "-i", "-style", "{BasedOnStyle: llvm, IndentWidth: 4, SortIncludes: false}" ] -includes = [ "*.c", "*.cpp", "*.cc", "*.h", "*.hpp" ] +options = [ + "-i", + "-style", + "{BasedOnStyle: llvm, IndentWidth: 4, SortIncludes: false}" +] +includes = ["*.c", "*.cpp", "*.cc", "*.h", "*.hpp"] [formatter.nix] command = "nixpkgs-fmt" includes = ["*.nix"] + +[formatter.toml] +command = "prettier" +options = ["--write"] +includes = ["*.toml"] From 83773704be0f70e8730573a6007ddd0da2868ba6 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Tue, 26 Apr 2022 21:36:48 +1200 Subject: [PATCH 127/419] Add Python Black formatter --- shell.nix | 1 + tests/test_eval.py | 8 ++++---- treefmt.toml | 4 ++++ 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/shell.nix b/shell.nix index 5c0cd0db4..ec82a6c20 100644 --- a/shell.nix +++ b/shell.nix @@ -33,6 +33,7 @@ in (pkgs.python3.withPackages (ps: [ ps.pytest + ps.black ])) ]; diff --git a/tests/test_eval.py b/tests/test_eval.py index 48bba7cc9..68dca7957 100644 --- a/tests/test_eval.py +++ b/tests/test_eval.py @@ -30,11 +30,11 @@ def common_test(extra_args: List[str]) -> None: assert built_job["name"] == "job1" assert built_job["outputs"]["out"].startswith("/nix/store") assert built_job["drvPath"].endswith(".drv") - assert built_job["meta"]['broken'] is False + assert built_job["meta"]["broken"] is False dotted_job = results[1] - assert dotted_job["attr"] == "\"dotted.attr\"" - assert dotted_job["attrPath"] == [ "dotted.attr" ] + assert dotted_job["attr"] == '"dotted.attr"' + assert dotted_job["attrPath"] == ["dotted.attr"] recurse_drv = results[2] assert recurse_drv["attr"] == "recurse.drvB" @@ -46,7 +46,7 @@ def common_test(extra_args: List[str]) -> None: substituted_job = results[4] assert substituted_job["attr"] == "substitutedJob" assert substituted_job["name"].startswith("hello-") - assert substituted_job["meta"]['broken'] is False + assert substituted_job["meta"]["broken"] is False def test_flake() -> None: diff --git a/treefmt.toml b/treefmt.toml index d1280df60..19a28ed6d 100644 --- a/treefmt.toml +++ b/treefmt.toml @@ -15,3 +15,7 @@ includes = ["*.nix"] command = "prettier" options = ["--write"] includes = ["*.toml"] + +[formatter.python] +command = "black" +includes = ["*.py"] From efca7105c7576cf090cf0b3460d47bb703e368e2 Mon Sep 17 00:00:00 2001 From: John Soo Date: Mon, 25 Apr 2022 08:53:17 -0700 Subject: [PATCH 128/419] Silence more harmless warnings. --- src/nix-eval-jobs.cc | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/nix-eval-jobs.cc b/src/nix-eval-jobs.cc index b03fa9354..8ffd03671 100644 --- a/src/nix-eval-jobs.cc +++ b/src/nix-eval-jobs.cc @@ -33,8 +33,11 @@ using namespace nlohmann; typedef enum { evalAuto, evalImpure, evalPure } pureEval; // Safe to ignore - the args will be static. +#ifdef __GNUC__ #pragma GCC diagnostic ignored "-Wnon-virtual-dtor" +#elif __clang__ #pragma clang diagnostic ignored "-Wnon-virtual-dtor" +#endif struct MyArgs : MixEvalArgs, MixCommonArgs { Path releaseExpr; Path gcRootsDir; @@ -101,8 +104,11 @@ struct MyArgs : MixEvalArgs, MixCommonArgs { expectArg("expr", &releaseExpr); } }; -#pragma GCC diagnostic warning "-Wnon-virtual-dtor" -#pragma clang diagnostic warning "-Wnon-virtual-dtor" +#ifdef __GNUC__ +#pragma GCC diagnostic ignored "-Wnon-virtual-dtor" +#elif __clang__ +#pragma clang diagnostic ignored "-Wnon-virtual-dtor" +#endif static MyArgs myArgs; From c8ca4075c9c0bda3f3ad44f66cea0d3872031697 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 28 Apr 2022 02:07:38 +0000 Subject: [PATCH 129/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/bc41b01dd7a9fdffd32d9b03806798797532a5fe' (2022-04-23) → 'github:NixOS/nixpkgs/e10da1c7f542515b609f8dfbcf788f3d85b14936' (2022-04-26) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index b832c0ff1..389a35c90 100644 --- a/flake.lock +++ b/flake.lock @@ -17,11 +17,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1650701402, - "narHash": "sha256-XKfstdtqDg+O+gNBx1yGVKWIhLgfEDg/e2lvJSsp9vU=", + "lastModified": 1651007983, + "narHash": "sha256-GNay7yDPtLcRcKCNHldug85AhAvBpTtPEJWSSDYBw8U=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "bc41b01dd7a9fdffd32d9b03806798797532a5fe", + "rev": "e10da1c7f542515b609f8dfbcf788f3d85b14936", "type": "github" }, "original": { From 5088049295015e0ba303b67cb122b797fbeb6e8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Mon, 9 May 2022 09:20:56 +0200 Subject: [PATCH 130/419] only target single nix version in repo --- .nix-version | 1 + flake.nix | 60 ++++++++++++++++++++++------------------------------ shell.nix | 4 ++-- 3 files changed, 28 insertions(+), 37 deletions(-) create mode 100644 .nix-version diff --git a/.nix-version b/.nix-version new file mode 100644 index 000000000..6842dbdf3 --- /dev/null +++ b/.nix-version @@ -0,0 +1 @@ +unstable diff --git a/flake.nix b/flake.nix index 11567726e..971dd6e38 100644 --- a/flake.nix +++ b/flake.nix @@ -4,53 +4,43 @@ inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; inputs.flake-utils.url = "github:numtide/flake-utils"; - outputs = { self, nixpkgs, flake-utils }: - flake-utils.lib.eachDefaultSystem (system: + outputs = + { self + , nixpkgs + , flake-utils + }: + flake-utils.lib.eachDefaultSystem ( + system: let + nixVersion = nixpkgs.lib.fileContents ./.nix-version; pkgs = nixpkgs.legacyPackages.${system}; inherit (pkgs) stdenv; - drvArgs = { srcDir = self; }; + devShell = self.devShells.${system}.default; + drvArgs = { + srcDir = self; + nix = if nixVersion == "unstable" then pkgs.nixUnstable else pkgs.nixVersions."nix_${nixVersion}"; + }; in - rec { + { packages.nix-eval-jobs = pkgs.callPackage ./default.nix drvArgs; - checks = - let - mkVariant = nix: (packages.nix-eval-jobs.override { - inherit nix; - }).overrideAttrs (_: { - name = "nix-eval-jobs-${nix.version}"; - inherit (nix) version; - }); - in - { + checks.treefmt = stdenv.mkDerivation { + name = "treefmt-check"; + src = self; + nativeBuildInputs = devShell.nativeBuildInputs; + dontConfigure = true; - treefmt = - let - devShell = devShells.default; - in - stdenv.mkDerivation { - name = "treefmt-check"; - src = self; - nativeBuildInputs = devShell.nativeBuildInputs; - dontConfigure = true; + inherit (devShell) NODE_PATH; - inherit (devShell) NODE_PATH; + buildPhase = '' + env HOME=$(mktemp -d) treefmt --fail-on-change + ''; - buildPhase = '' - env HOME=$(mktemp -d) treefmt --fail-on-change - ''; - - installPhase = "touch $out"; - }; - - build = mkVariant pkgs.nix; - build-unstable = mkVariant pkgs.nixUnstable; - }; + installPhase = "touch $out"; + }; packages.default = self.packages.${system}.nix-eval-jobs; devShells.default = pkgs.callPackage ./shell.nix drvArgs; - } ); } diff --git a/shell.nix b/shell.nix index ec82a6c20..f64215ea4 100644 --- a/shell.nix +++ b/shell.nix @@ -11,15 +11,15 @@ import nixpkgs { } ) , srcDir ? null +, nix }: let inherit (pkgs) lib stdenv; - nix = pkgs.nixUnstable; in (pkgs.callPackage ./default.nix { - inherit nix srcDir; + inherit srcDir nix; }).overrideAttrs (old: { src = null; From 13209fb02420f6f70e2d9d97c242893ae38abaa8 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 9 May 2022 01:46:54 +0000 Subject: [PATCH 131/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/e10da1c7f542515b609f8dfbcf788f3d85b14936' (2022-04-26) → 'github:NixOS/nixpkgs/c777cdf5c564015d5f63b09cc93bef4178b19b01' (2022-05-05) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 389a35c90..a6d08634b 100644 --- a/flake.lock +++ b/flake.lock @@ -17,11 +17,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1651007983, - "narHash": "sha256-GNay7yDPtLcRcKCNHldug85AhAvBpTtPEJWSSDYBw8U=", + "lastModified": 1651726670, + "narHash": "sha256-dSGdzB49SEvdOJvrQWfQYkAefewXraHIV08Vz6iDXWQ=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "e10da1c7f542515b609f8dfbcf788f3d85b14936", + "rev": "c777cdf5c564015d5f63b09cc93bef4178b19b01", "type": "github" }, "original": { From 882d2376cff749d83ca36debca5172da1920c757 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sat, 7 May 2022 09:01:41 +0200 Subject: [PATCH 132/419] fixes for nixUnstable --- src/nix-eval-jobs.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/nix-eval-jobs.cc b/src/nix-eval-jobs.cc index 8ffd03671..eb97d0b4e 100644 --- a/src/nix-eval-jobs.cc +++ b/src/nix-eval-jobs.cc @@ -287,14 +287,14 @@ static void worker(EvalState &state, Bindings &autoArgs, AutoCloseFD &to, path.size() == 0; // Dont require `recurseForDerivations = // true;` for top-level attrset - for (auto &i : v->attrs->lexicographicOrder()) { - std::string name(i->name); + for (auto &i : v->attrs->lexicographicOrder(state.symbols)) { + const std::string &name = state.symbols[i->name]; attrs.push_back(name); if (name == "recurseForDerivations") { auto attrv = v->attrs->get(state.sRecurseForDerivations); - recurse = state.forceBool(*attrv->value, *attrv->pos); + recurse = state.forceBool(*attrv->value, attrv->pos); } } if (recurse) From e67f2f80907d0f9fb82ed38dd423eacbc653b551 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Mon, 9 May 2022 09:53:59 +0200 Subject: [PATCH 133/419] README: clarify branches --- README.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/README.md b/README.md index cbd5fb4a0..0a55d0a68 100644 --- a/README.md +++ b/README.md @@ -81,6 +81,15 @@ path separately. This allows separate logs and success status per job instead of a single large log file. +## Organisation of this repository + +On the `main` branch we target nixUnstable. When a release of nix happens, we +fork for a release branch i.e. `release-2.8` and change the nix version in +`.nix-version`. Changes and improvements made in `main` also may be backported +to these release branches. At the time of writing we only intent to support the +latest release branch. + + ## Projects using nix-eval-jobs - [colmena](https://github.com/zhaofengli/colmena) - A simple, stateless NixOS deployment tool From 7f00569c2bc8e02855176784c3e8d5577d3b8f59 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 12 May 2022 01:44:59 +0000 Subject: [PATCH 134/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/c777cdf5c564015d5f63b09cc93bef4178b19b01' (2022-05-05) → 'github:NixOS/nixpkgs/41ff747f882914c1f8c233207ce280ac9d0c867f' (2022-05-11) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index a6d08634b..30bcab581 100644 --- a/flake.lock +++ b/flake.lock @@ -17,11 +17,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1651726670, - "narHash": "sha256-dSGdzB49SEvdOJvrQWfQYkAefewXraHIV08Vz6iDXWQ=", + "lastModified": 1652231724, + "narHash": "sha256-MjalcXFZgcgchp4QqnF05JTkFBBGad5hbksA1EKoP98=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "c777cdf5c564015d5f63b09cc93bef4178b19b01", + "rev": "41ff747f882914c1f8c233207ce280ac9d0c867f", "type": "github" }, "original": { From d656c30f8c2aece25b70be37addf6849178fdbea Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 19 May 2022 01:48:11 +0000 Subject: [PATCH 135/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'flake-utils': 'github:numtide/flake-utils/a4b154ebbdc88c8498a5c7b01589addc9e9cb678' (2022-04-11) → 'github:numtide/flake-utils/04c1b180862888302ddfb2e3ad9eaa63afc60cf8' (2022-05-17) • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/41ff747f882914c1f8c233207ce280ac9d0c867f' (2022-05-11) → 'github:NixOS/nixpkgs/1d7db1b9e4cf1ee075a9f52e5c36f7b9f4207502' (2022-05-16) --- flake.lock | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/flake.lock b/flake.lock index 30bcab581..b5eb45fcd 100644 --- a/flake.lock +++ b/flake.lock @@ -2,11 +2,11 @@ "nodes": { "flake-utils": { "locked": { - "lastModified": 1649676176, - "narHash": "sha256-OWKJratjt2RW151VUlJPRALb7OU2S5s+f0vLj4o1bHM=", + "lastModified": 1652776076, + "narHash": "sha256-gzTw/v1vj4dOVbpBSJX4J0DwUR6LIyXo7/SuuTJp1kM=", "owner": "numtide", "repo": "flake-utils", - "rev": "a4b154ebbdc88c8498a5c7b01589addc9e9cb678", + "rev": "04c1b180862888302ddfb2e3ad9eaa63afc60cf8", "type": "github" }, "original": { @@ -17,11 +17,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1652231724, - "narHash": "sha256-MjalcXFZgcgchp4QqnF05JTkFBBGad5hbksA1EKoP98=", + "lastModified": 1652659998, + "narHash": "sha256-FqNrXC1EE6U2RACwXBlsAvg1lqQGLYpuYb6+W3DL9vA=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "41ff747f882914c1f8c233207ce280ac9d0c867f", + "rev": "1d7db1b9e4cf1ee075a9f52e5c36f7b9f4207502", "type": "github" }, "original": { From f293656be1add74be1c5b868f4110dcc7621cf41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Thu, 19 May 2022 08:04:36 +0200 Subject: [PATCH 136/419] update mergify rules --- .mergify.yml | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/.mergify.yml b/.mergify.yml index 766725a82..b50b3b33c 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -2,14 +2,10 @@ pull_request_rules: - name: automatic merge on CI success conditions: - check-success=flake-checks - - check-success=builds (build, ubuntu-latest) - - check-success=builds (build, macos-latest) - - check-success=builds (build-unstable, ubuntu-latest) - - check-success=builds (build-unstable, macos-latest) - - check-success=builds (editorconfig, ubuntu-latest) - - check-success=builds (editorconfig, macos-latest) - - check-success=builds (nixpkgs-fmt, ubuntu-latest) - - check-success=builds (nixpkgs-fmt, macos-latest) + - check-success=builds (treefmt, ubuntu-latest) + - check-success=builds (treefmt, macos-latest) + - check-success=tests (ubuntu-latest) + - check-success=tests (macos-latest) - author=nix-eval-jobs-bot actions: merge: From 4d2e2149edd0092e56e354dd64155124407772db Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 23 May 2022 01:27:11 +0000 Subject: [PATCH 137/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/1d7db1b9e4cf1ee075a9f52e5c36f7b9f4207502' (2022-05-16) → 'github:NixOS/nixpkgs/dfd82985c273aac6eced03625f454b334daae2e8' (2022-05-20) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index b5eb45fcd..f968d582f 100644 --- a/flake.lock +++ b/flake.lock @@ -17,11 +17,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1652659998, - "narHash": "sha256-FqNrXC1EE6U2RACwXBlsAvg1lqQGLYpuYb6+W3DL9vA=", + "lastModified": 1653060744, + "narHash": "sha256-kfRusllRumpt33J1hPV+CeCCylCXEU7e0gn2/cIM7cY=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "1d7db1b9e4cf1ee075a9f52e5c36f7b9f4207502", + "rev": "dfd82985c273aac6eced03625f454b334daae2e8", "type": "github" }, "original": { From 2a5a405700514db799bc9fc161fef217040faffd Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 26 May 2022 01:45:51 +0000 Subject: [PATCH 138/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/dfd82985c273aac6eced03625f454b334daae2e8' (2022-05-20) → 'github:NixOS/nixpkgs/5ce6597eca7d7b518c03ecda57d45f9404b5e060' (2022-05-24) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index f968d582f..f8f85e560 100644 --- a/flake.lock +++ b/flake.lock @@ -17,11 +17,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1653060744, - "narHash": "sha256-kfRusllRumpt33J1hPV+CeCCylCXEU7e0gn2/cIM7cY=", + "lastModified": 1653407748, + "narHash": "sha256-g9puJaILRTb9ttlLQ7IehpV7Wcy0n+vs8LOFu6ylQcM=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "dfd82985c273aac6eced03625f454b334daae2e8", + "rev": "5ce6597eca7d7b518c03ecda57d45f9404b5e060", "type": "github" }, "original": { From f6fe06fbbe04959e51908df5189a6f51e66ed739 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 30 May 2022 01:43:24 +0000 Subject: [PATCH 139/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/5ce6597eca7d7b518c03ecda57d45f9404b5e060' (2022-05-24) → 'github:NixOS/nixpkgs/83658b28fe638a170a19b8933aa008b30640fbd1' (2022-05-26) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index f8f85e560..b1234b657 100644 --- a/flake.lock +++ b/flake.lock @@ -17,11 +17,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1653407748, - "narHash": "sha256-g9puJaILRTb9ttlLQ7IehpV7Wcy0n+vs8LOFu6ylQcM=", + "lastModified": 1653581809, + "narHash": "sha256-Uvka0V5MTGbeOfWte25+tfRL3moECDh1VwokWSZUdoY=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "5ce6597eca7d7b518c03ecda57d45f9404b5e060", + "rev": "83658b28fe638a170a19b8933aa008b30640fbd1", "type": "github" }, "original": { From a6ed4475b67f2636c5c6fabcb830d1146ec2733b Mon Sep 17 00:00:00 2001 From: adisbladis Date: Wed, 1 Jun 2022 03:40:56 +0800 Subject: [PATCH 140/419] flake.lock: Bump flake inputs --- flake.lock | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/flake.lock b/flake.lock index b1234b657..f7e41b27f 100644 --- a/flake.lock +++ b/flake.lock @@ -2,11 +2,11 @@ "nodes": { "flake-utils": { "locked": { - "lastModified": 1652776076, - "narHash": "sha256-gzTw/v1vj4dOVbpBSJX4J0DwUR6LIyXo7/SuuTJp1kM=", + "lastModified": 1653893745, + "narHash": "sha256-0jntwV3Z8//YwuOjzhV2sgJJPt+HY6KhU7VZUL0fKZQ=", "owner": "numtide", "repo": "flake-utils", - "rev": "04c1b180862888302ddfb2e3ad9eaa63afc60cf8", + "rev": "1ed9fb1935d260de5fe1c2f7ee0ebaae17ed2fa1", "type": "github" }, "original": { @@ -17,11 +17,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1653581809, - "narHash": "sha256-Uvka0V5MTGbeOfWte25+tfRL3moECDh1VwokWSZUdoY=", + "lastModified": 1653931853, + "narHash": "sha256-O3wncIouj9x7gBPntzHeK/Hkmm9M1SGlYq7JI7saTAE=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "83658b28fe638a170a19b8933aa008b30640fbd1", + "rev": "f1c167688a6f81f4a51ab542e5f476c8c595e457", "type": "github" }, "original": { From 464cb34a7c1cf1524e84dae7c6242e9b72cd2110 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 6 Jun 2022 01:42:10 +0000 Subject: [PATCH 141/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/f1c167688a6f81f4a51ab542e5f476c8c595e457' (2022-05-30) → 'github:NixOS/nixpkgs/236cc2971ac72acd90f0ae3a797f9f83098b17ec' (2022-06-03) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index f7e41b27f..3eef24e7c 100644 --- a/flake.lock +++ b/flake.lock @@ -17,11 +17,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1653931853, - "narHash": "sha256-O3wncIouj9x7gBPntzHeK/Hkmm9M1SGlYq7JI7saTAE=", + "lastModified": 1654230545, + "narHash": "sha256-8Vlwf0x8ow6pPOK2a04bT+pxIeRnM1+O0Xv9/CuDzRs=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "f1c167688a6f81f4a51ab542e5f476c8c595e457", + "rev": "236cc2971ac72acd90f0ae3a797f9f83098b17ec", "type": "github" }, "original": { From 5b45b27f18104413f392362a0389e064575d16e4 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 9 Jun 2022 01:37:14 +0000 Subject: [PATCH 142/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/236cc2971ac72acd90f0ae3a797f9f83098b17ec' (2022-06-03) → 'github:NixOS/nixpkgs/033bd4fa9a8fbe0c68a88e925d9a884161044b25' (2022-06-07) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 3eef24e7c..acfc6d2f2 100644 --- a/flake.lock +++ b/flake.lock @@ -17,11 +17,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1654230545, - "narHash": "sha256-8Vlwf0x8ow6pPOK2a04bT+pxIeRnM1+O0Xv9/CuDzRs=", + "lastModified": 1654593855, + "narHash": "sha256-c+SyXvj7THre87OyIdZfRVR+HhI/g1ZDrQ3VUtTuHkU=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "236cc2971ac72acd90f0ae3a797f9f83098b17ec", + "rev": "033bd4fa9a8fbe0c68a88e925d9a884161044b25", "type": "github" }, "original": { From 4cbc088820109ccab2a592e27779202a5a034f81 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 13 Jun 2022 01:49:37 +0000 Subject: [PATCH 143/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/033bd4fa9a8fbe0c68a88e925d9a884161044b25' (2022-06-07) → 'github:NixOS/nixpkgs/90cd5459a1fd707819b9a3fb9c852beaaac3b79a' (2022-06-11) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index acfc6d2f2..1fb28e0ee 100644 --- a/flake.lock +++ b/flake.lock @@ -17,11 +17,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1654593855, - "narHash": "sha256-c+SyXvj7THre87OyIdZfRVR+HhI/g1ZDrQ3VUtTuHkU=", + "lastModified": 1654953433, + "narHash": "sha256-TwEeh4r50NdWHFAHQSyjCk2cZxgwUfcCCAJOhPdXB28=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "033bd4fa9a8fbe0c68a88e925d9a884161044b25", + "rev": "90cd5459a1fd707819b9a3fb9c852beaaac3b79a", "type": "github" }, "original": { From a38ad66767a4c1df50a09a5567cfa415645afa12 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 16 Jun 2022 01:41:32 +0000 Subject: [PATCH 144/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/90cd5459a1fd707819b9a3fb9c852beaaac3b79a' (2022-06-11) → 'github:NixOS/nixpkgs/6616de389ed55fba6eeba60377fc04732d5a207c' (2022-06-14) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 1fb28e0ee..db1a4bf18 100644 --- a/flake.lock +++ b/flake.lock @@ -17,11 +17,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1654953433, - "narHash": "sha256-TwEeh4r50NdWHFAHQSyjCk2cZxgwUfcCCAJOhPdXB28=", + "lastModified": 1655221618, + "narHash": "sha256-ht8HRFthDKzYt+il+sGgkBwrv+Ex2l8jdGVpsrPfFME=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "90cd5459a1fd707819b9a3fb9c852beaaac3b79a", + "rev": "6616de389ed55fba6eeba60377fc04732d5a207c", "type": "github" }, "original": { From 463e8febf4aa615551d9f4ef104ca2ad354599a0 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 20 Jun 2022 01:42:59 +0000 Subject: [PATCH 145/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/6616de389ed55fba6eeba60377fc04732d5a207c' (2022-06-14) → 'github:NixOS/nixpkgs/e0a42267f73ea52adc061a64650fddc59906fc99' (2022-06-18) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index db1a4bf18..113414592 100644 --- a/flake.lock +++ b/flake.lock @@ -17,11 +17,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1655221618, - "narHash": "sha256-ht8HRFthDKzYt+il+sGgkBwrv+Ex2l8jdGVpsrPfFME=", + "lastModified": 1655567057, + "narHash": "sha256-Cc5hQSMsTzOHmZnYm8OSJ5RNUp22bd5NADWLHorULWQ=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "6616de389ed55fba6eeba60377fc04732d5a207c", + "rev": "e0a42267f73ea52adc061a64650fddc59906fc99", "type": "github" }, "original": { From 9dd5159bc003161bab531f07515f82f5206a682b Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 23 Jun 2022 01:42:45 +0000 Subject: [PATCH 146/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/e0a42267f73ea52adc061a64650fddc59906fc99' (2022-06-18) → 'github:NixOS/nixpkgs/e1e08fe28bf0588a41cd556eac40b98d2793da99' (2022-06-22) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 113414592..7a191b66d 100644 --- a/flake.lock +++ b/flake.lock @@ -17,11 +17,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1655567057, - "narHash": "sha256-Cc5hQSMsTzOHmZnYm8OSJ5RNUp22bd5NADWLHorULWQ=", + "lastModified": 1655895887, + "narHash": "sha256-AbihSVmPKFcQmaVgzNSLQ8sLAaVTTELPSzZ+/gRVfyk=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "e0a42267f73ea52adc061a64650fddc59906fc99", + "rev": "e1e08fe28bf0588a41cd556eac40b98d2793da99", "type": "github" }, "original": { From 933fd7ecd2bf8663ad92e3986b5f701ec2d41c5b Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 27 Jun 2022 01:52:25 +0000 Subject: [PATCH 147/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'flake-utils': 'github:numtide/flake-utils/1ed9fb1935d260de5fe1c2f7ee0ebaae17ed2fa1' (2022-05-30) → 'github:numtide/flake-utils/bee6a7250dd1b01844a2de7e02e4df7d8a0a206c' (2022-06-24) • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/e1e08fe28bf0588a41cd556eac40b98d2793da99' (2022-06-22) → 'github:NixOS/nixpkgs/f2537a505d45c31fe5d9c27ea9829b6f4c4e6ac5' (2022-06-26) --- flake.lock | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/flake.lock b/flake.lock index 7a191b66d..7a5087ded 100644 --- a/flake.lock +++ b/flake.lock @@ -2,11 +2,11 @@ "nodes": { "flake-utils": { "locked": { - "lastModified": 1653893745, - "narHash": "sha256-0jntwV3Z8//YwuOjzhV2sgJJPt+HY6KhU7VZUL0fKZQ=", + "lastModified": 1656065134, + "narHash": "sha256-oc6E6ByIw3oJaIyc67maaFcnjYOz1mMcOtHxbEf9NwQ=", "owner": "numtide", "repo": "flake-utils", - "rev": "1ed9fb1935d260de5fe1c2f7ee0ebaae17ed2fa1", + "rev": "bee6a7250dd1b01844a2de7e02e4df7d8a0a206c", "type": "github" }, "original": { @@ -17,11 +17,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1655895887, - "narHash": "sha256-AbihSVmPKFcQmaVgzNSLQ8sLAaVTTELPSzZ+/gRVfyk=", + "lastModified": 1656239181, + "narHash": "sha256-wW1xRFBn376yGloXZ4QzBE4hjipMawpV18Lshd9QSPw=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "e1e08fe28bf0588a41cd556eac40b98d2793da99", + "rev": "f2537a505d45c31fe5d9c27ea9829b6f4c4e6ac5", "type": "github" }, "original": { From 14c417dc7e550b5b27c70fb77386c9d8e641b60c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Jun 2022 23:13:21 +0000 Subject: [PATCH 148/419] Bump DeterminateSystems/update-flake-lock from 9 to 10 Bumps [DeterminateSystems/update-flake-lock](https://github.com/DeterminateSystems/update-flake-lock) from 9 to 10. - [Release notes](https://github.com/DeterminateSystems/update-flake-lock/releases) - [Commits](https://github.com/DeterminateSystems/update-flake-lock/compare/v9...v10) --- updated-dependencies: - dependency-name: DeterminateSystems/update-flake-lock dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/update-flake-lock.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/update-flake-lock.yml b/.github/workflows/update-flake-lock.yml index b89af8801..a0cbaf20e 100644 --- a/.github/workflows/update-flake-lock.yml +++ b/.github/workflows/update-flake-lock.yml @@ -16,6 +16,6 @@ jobs: # extra_nix_config: | # access-tokens = github.com=${{ secrets.GITHUB_TOKEN }} - name: Update flake.lock - uses: DeterminateSystems/update-flake-lock@v9 + uses: DeterminateSystems/update-flake-lock@v10 with: token: ${{ secrets.GH_TOKEN_FOR_UPDATES }} From 1fadf56d2a2cba8a729d0d890e206a7f38dfc70c Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 30 Jun 2022 01:48:39 +0000 Subject: [PATCH 149/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/f2537a505d45c31fe5d9c27ea9829b6f4c4e6ac5' (2022-06-26) → 'github:NixOS/nixpkgs/020c74014b9e2fa905bb4059c979965816cd9118' (2022-06-27) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 7a5087ded..530eade98 100644 --- a/flake.lock +++ b/flake.lock @@ -17,11 +17,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1656239181, - "narHash": "sha256-wW1xRFBn376yGloXZ4QzBE4hjipMawpV18Lshd9QSPw=", + "lastModified": 1656372800, + "narHash": "sha256-1u9SDLXvKix/QejNb2sY2J2QZXnbe/14MnLtn+ln9j0=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "f2537a505d45c31fe5d9c27ea9829b6f4c4e6ac5", + "rev": "020c74014b9e2fa905bb4059c979965816cd9118", "type": "github" }, "original": { From fa317d9f429ff0fae5039aafe699aa42adc737d7 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 4 Jul 2022 01:58:06 +0000 Subject: [PATCH 150/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/020c74014b9e2fa905bb4059c979965816cd9118' (2022-06-27) → 'github:NixOS/nixpkgs/0ea7a8f1b939d74e5df8af9a8f7342097cdf69eb' (2022-07-02) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 530eade98..8518fd2d8 100644 --- a/flake.lock +++ b/flake.lock @@ -17,11 +17,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1656372800, - "narHash": "sha256-1u9SDLXvKix/QejNb2sY2J2QZXnbe/14MnLtn+ln9j0=", + "lastModified": 1656753965, + "narHash": "sha256-BCrB3l0qpJokOnIVc3g2lHiGhnjUi0MoXiw6t1o8H1E=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "020c74014b9e2fa905bb4059c979965816cd9118", + "rev": "0ea7a8f1b939d74e5df8af9a8f7342097cdf69eb", "type": "github" }, "original": { From 52ad7f6af98248ffe0548f54b7e01a02fc681846 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 7 Jul 2022 01:54:49 +0000 Subject: [PATCH 151/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'flake-utils': 'github:numtide/flake-utils/bee6a7250dd1b01844a2de7e02e4df7d8a0a206c' (2022-06-24) → 'github:numtide/flake-utils/7e2a3b3dfd9af950a856d66b0a7d01e3c18aa249' (2022-07-04) • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/0ea7a8f1b939d74e5df8af9a8f7342097cdf69eb' (2022-07-02) → 'github:NixOS/nixpkgs/71a4f0dc3d80ba76f437c888c1c3d59f1df98163' (2022-07-05) --- flake.lock | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/flake.lock b/flake.lock index 8518fd2d8..68212b62f 100644 --- a/flake.lock +++ b/flake.lock @@ -2,11 +2,11 @@ "nodes": { "flake-utils": { "locked": { - "lastModified": 1656065134, - "narHash": "sha256-oc6E6ByIw3oJaIyc67maaFcnjYOz1mMcOtHxbEf9NwQ=", + "lastModified": 1656928814, + "narHash": "sha256-RIFfgBuKz6Hp89yRr7+NR5tzIAbn52h8vT6vXkYjZoM=", "owner": "numtide", "repo": "flake-utils", - "rev": "bee6a7250dd1b01844a2de7e02e4df7d8a0a206c", + "rev": "7e2a3b3dfd9af950a856d66b0a7d01e3c18aa249", "type": "github" }, "original": { @@ -17,11 +17,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1656753965, - "narHash": "sha256-BCrB3l0qpJokOnIVc3g2lHiGhnjUi0MoXiw6t1o8H1E=", + "lastModified": 1657020478, + "narHash": "sha256-sU5hXEGcOcvz2xoPAuNLBQJLXjwvPpTkoddyXE8gw20=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "0ea7a8f1b939d74e5df8af9a8f7342097cdf69eb", + "rev": "71a4f0dc3d80ba76f437c888c1c3d59f1df98163", "type": "github" }, "original": { From 8acc159af0bfc797d7c9cd0e5d605337e76cd34e Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 11 Jul 2022 01:50:25 +0000 Subject: [PATCH 152/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/71a4f0dc3d80ba76f437c888c1c3d59f1df98163' (2022-07-05) → 'github:NixOS/nixpkgs/87e7965bbcdbac3d103e3ed14ff04f719a4f7a58' (2022-07-09) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 68212b62f..bf9cb6eb3 100644 --- a/flake.lock +++ b/flake.lock @@ -17,11 +17,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1657020478, - "narHash": "sha256-sU5hXEGcOcvz2xoPAuNLBQJLXjwvPpTkoddyXE8gw20=", + "lastModified": 1657356697, + "narHash": "sha256-sT38tcx7m0Quz+Uj6jzx+yRa2+EVW2C3cE0FkROXUzQ=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "71a4f0dc3d80ba76f437c888c1c3d59f1df98163", + "rev": "87e7965bbcdbac3d103e3ed14ff04f719a4f7a58", "type": "github" }, "original": { From 3071792e912e5f221bc6a39ebf80de0d4f7eee7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Mon, 11 Jul 2022 13:38:35 +0200 Subject: [PATCH 153/419] README: link to ci configurations --- README.md | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 0a55d0a68..98433c629 100644 --- a/README.md +++ b/README.md @@ -75,10 +75,13 @@ scalability for large deployments with deployment tools such as **Faster evaluator in CIs.** In addition to evaluation speed for CIs, it is also useful if evaluation of individual jobs in CIs can fail, as opposed to failing -the entire jobset. For CIs that allow dynamic build steps to be created, one -can also take advantage of the fact that nix-eval-jobs outputs the derivation -path separately. This allows separate logs and success status per job instead -of a single large log file. +the entire jobset. For CIs that allow dynamic build steps to be created, one can +also take advantage of the fact that nix-eval-jobs outputs the derivation path +separately. This allows separate logs and success status per job instead of a +single large log file. In the +[wiki](https://github.com/nix-community/nix-eval-jobs/wiki#ci-example-configurations) +we collect example ci configuration for various CIs. + ## Organisation of this repository From 363e0fe4fb8fc8f9e1026abeaaf181acb177b0fd Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 14 Jul 2022 01:54:48 +0000 Subject: [PATCH 154/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/87e7965bbcdbac3d103e3ed14ff04f719a4f7a58' (2022-07-09) → 'github:NixOS/nixpkgs/38860c9e91cb00f4d8cd19c7b4e36c45680c89b5' (2022-07-11) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index bf9cb6eb3..d2e89aafe 100644 --- a/flake.lock +++ b/flake.lock @@ -17,11 +17,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1657356697, - "narHash": "sha256-sT38tcx7m0Quz+Uj6jzx+yRa2+EVW2C3cE0FkROXUzQ=", + "lastModified": 1657533762, + "narHash": "sha256-/cxTFSMmpAb8tBp1yVga1fj+i8LB9aAxnMjYFpRMuVs=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "87e7965bbcdbac3d103e3ed14ff04f719a4f7a58", + "rev": "38860c9e91cb00f4d8cd19c7b4e36c45680c89b5", "type": "github" }, "original": { From 518fd64ceff1a807e5ce80678f54d8ed7ce8aba7 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 18 Jul 2022 01:52:22 +0000 Subject: [PATCH 155/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/38860c9e91cb00f4d8cd19c7b4e36c45680c89b5' (2022-07-11) → 'github:NixOS/nixpkgs/8f485713f5e6b6883a9b6959afa98688360a3ecb' (2022-07-16) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index d2e89aafe..10b57e4dc 100644 --- a/flake.lock +++ b/flake.lock @@ -17,11 +17,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1657533762, - "narHash": "sha256-/cxTFSMmpAb8tBp1yVga1fj+i8LB9aAxnMjYFpRMuVs=", + "lastModified": 1658015103, + "narHash": "sha256-mO+23f3SO+fBzEvbxRe6GkSB5Xp43CT2sV8Rs8MYdz8=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "38860c9e91cb00f4d8cd19c7b4e36c45680c89b5", + "rev": "8f485713f5e6b6883a9b6959afa98688360a3ecb", "type": "github" }, "original": { From 76285edc5385ff754dd94f70b8ea627e019d275c Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 21 Jul 2022 01:51:04 +0000 Subject: [PATCH 156/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/8f485713f5e6b6883a9b6959afa98688360a3ecb' (2022-07-16) → 'github:NixOS/nixpkgs/614a842b74b7a1497e8cfca7c61bec38f51911b3' (2022-07-20) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 10b57e4dc..971d0f66e 100644 --- a/flake.lock +++ b/flake.lock @@ -17,11 +17,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1658015103, - "narHash": "sha256-mO+23f3SO+fBzEvbxRe6GkSB5Xp43CT2sV8Rs8MYdz8=", + "lastModified": 1658290795, + "narHash": "sha256-t9hCidaSxPmzUimcSn51bjqcjjGsoQi6JLrAzJq0dz4=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "8f485713f5e6b6883a9b6959afa98688360a3ecb", + "rev": "614a842b74b7a1497e8cfca7c61bec38f51911b3", "type": "github" }, "original": { From 14347964296482ca02d2b8691ab7d6ef07cda185 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 25 Jul 2022 01:50:38 +0000 Subject: [PATCH 157/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/614a842b74b7a1497e8cfca7c61bec38f51911b3' (2022-07-20) → 'github:NixOS/nixpkgs/e494a908e8895b9cba18e21d5fc83362f64b3f6a' (2022-07-24) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 971d0f66e..358e304a4 100644 --- a/flake.lock +++ b/flake.lock @@ -17,11 +17,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1658290795, - "narHash": "sha256-t9hCidaSxPmzUimcSn51bjqcjjGsoQi6JLrAzJq0dz4=", + "lastModified": 1658648081, + "narHash": "sha256-RL5nr4Xhp0zQeEGG/I3t3FmqaI9QrBg5PH31NF+7A/A=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "614a842b74b7a1497e8cfca7c61bec38f51911b3", + "rev": "e494a908e8895b9cba18e21d5fc83362f64b3f6a", "type": "github" }, "original": { From 36a57a6658af04da86e02bf2575d1fa4429353c8 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 28 Jul 2022 01:49:12 +0000 Subject: [PATCH 158/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/e494a908e8895b9cba18e21d5fc83362f64b3f6a' (2022-07-24) → 'github:NixOS/nixpkgs/ce49cb7792a7ffd65ef352dda1110a4e4a204eac' (2022-07-26) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 358e304a4..d16979cc7 100644 --- a/flake.lock +++ b/flake.lock @@ -17,11 +17,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1658648081, - "narHash": "sha256-RL5nr4Xhp0zQeEGG/I3t3FmqaI9QrBg5PH31NF+7A/A=", + "lastModified": 1658826464, + "narHash": "sha256-94ZTF0uIX/iZdiD4RJ5f933ak/OM4XLl7hF+gCa4Iuk=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "e494a908e8895b9cba18e21d5fc83362f64b3f6a", + "rev": "ce49cb7792a7ffd65ef352dda1110a4e4a204eac", "type": "github" }, "original": { From 9c779cc520936209686e3caf5641dbb939db9115 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 1 Aug 2022 02:01:16 +0000 Subject: [PATCH 159/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/ce49cb7792a7ffd65ef352dda1110a4e4a204eac' (2022-07-26) → 'github:NixOS/nixpkgs/7b9be38c7250b22d829ab6effdee90d5e40c6e5c' (2022-07-30) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index d16979cc7..66251bec9 100644 --- a/flake.lock +++ b/flake.lock @@ -17,11 +17,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1658826464, - "narHash": "sha256-94ZTF0uIX/iZdiD4RJ5f933ak/OM4XLl7hF+gCa4Iuk=", + "lastModified": 1659219666, + "narHash": "sha256-pzYr5fokQPHv7CmUXioOhhzDy/XyWOIXP4LZvv/T7Mk=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "ce49cb7792a7ffd65ef352dda1110a4e4a204eac", + "rev": "7b9be38c7250b22d829ab6effdee90d5e40c6e5c", "type": "github" }, "original": { From b949dc3606585f109d04bfed56592054537a659e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Aug 2022 23:11:32 +0000 Subject: [PATCH 160/419] Bump DeterminateSystems/update-flake-lock from 10 to 12 Bumps [DeterminateSystems/update-flake-lock](https://github.com/DeterminateSystems/update-flake-lock) from 10 to 12. - [Release notes](https://github.com/DeterminateSystems/update-flake-lock/releases) - [Commits](https://github.com/DeterminateSystems/update-flake-lock/compare/v10...v12) --- updated-dependencies: - dependency-name: DeterminateSystems/update-flake-lock dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/update-flake-lock.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/update-flake-lock.yml b/.github/workflows/update-flake-lock.yml index a0cbaf20e..da877d051 100644 --- a/.github/workflows/update-flake-lock.yml +++ b/.github/workflows/update-flake-lock.yml @@ -16,6 +16,6 @@ jobs: # extra_nix_config: | # access-tokens = github.com=${{ secrets.GITHUB_TOKEN }} - name: Update flake.lock - uses: DeterminateSystems/update-flake-lock@v10 + uses: DeterminateSystems/update-flake-lock@v12 with: token: ${{ secrets.GH_TOKEN_FOR_UPDATES }} From 52ee427bf6e535e27b20fd4f8fc9f3679227dd35 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 4 Aug 2022 01:47:43 +0000 Subject: [PATCH 161/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/7b9be38c7250b22d829ab6effdee90d5e40c6e5c' (2022-07-30) → 'github:NixOS/nixpkgs/12363fb6d89859a37cd7e27f85288599f13e49d9' (2022-08-03) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 66251bec9..d88a8451f 100644 --- a/flake.lock +++ b/flake.lock @@ -17,11 +17,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1659219666, - "narHash": "sha256-pzYr5fokQPHv7CmUXioOhhzDy/XyWOIXP4LZvv/T7Mk=", + "lastModified": 1659487974, + "narHash": "sha256-CVGOtR/Wyq3TVCjf8/kdnYD5G2JwUKUQVtd+5WIDTuY=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "7b9be38c7250b22d829ab6effdee90d5e40c6e5c", + "rev": "12363fb6d89859a37cd7e27f85288599f13e49d9", "type": "github" }, "original": { From 21167c299b7b9e07410d7b1a09e6d9f2b96cd75e Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 8 Aug 2022 01:44:52 +0000 Subject: [PATCH 162/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'flake-utils': 'github:numtide/flake-utils/7e2a3b3dfd9af950a856d66b0a7d01e3c18aa249' (2022-07-04) → 'github:numtide/flake-utils/c0e246b9b83f637f4681389ecabcb2681b4f3af0' (2022-08-07) • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/12363fb6d89859a37cd7e27f85288599f13e49d9' (2022-08-03) → 'github:NixOS/nixpkgs/f44884060cb94240efbe55620f38a8ec8d9af601' (2022-08-06) --- flake.lock | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/flake.lock b/flake.lock index d88a8451f..f6e0ad161 100644 --- a/flake.lock +++ b/flake.lock @@ -2,11 +2,11 @@ "nodes": { "flake-utils": { "locked": { - "lastModified": 1656928814, - "narHash": "sha256-RIFfgBuKz6Hp89yRr7+NR5tzIAbn52h8vT6vXkYjZoM=", + "lastModified": 1659877975, + "narHash": "sha256-zllb8aq3YO3h8B/U0/J1WBgAL8EX5yWf5pMj3G0NAmc=", "owner": "numtide", "repo": "flake-utils", - "rev": "7e2a3b3dfd9af950a856d66b0a7d01e3c18aa249", + "rev": "c0e246b9b83f637f4681389ecabcb2681b4f3af0", "type": "github" }, "original": { @@ -17,11 +17,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1659487974, - "narHash": "sha256-CVGOtR/Wyq3TVCjf8/kdnYD5G2JwUKUQVtd+5WIDTuY=", + "lastModified": 1659803779, + "narHash": "sha256-+5zkHlbcbFyN5f3buO1RAZ9pH1wXLxCesUJ0vFmLr9Y=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "12363fb6d89859a37cd7e27f85288599f13e49d9", + "rev": "f44884060cb94240efbe55620f38a8ec8d9af601", "type": "github" }, "original": { From da171c915b31f65aaf6914701d3e8ebc00be5c16 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 11 Aug 2022 01:42:47 +0000 Subject: [PATCH 163/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/f44884060cb94240efbe55620f38a8ec8d9af601' (2022-08-06) → 'github:NixOS/nixpkgs/36cc29d837e7232e3176e4651e8e117a6f231793' (2022-08-09) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index f6e0ad161..b830a83d1 100644 --- a/flake.lock +++ b/flake.lock @@ -17,11 +17,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1659803779, - "narHash": "sha256-+5zkHlbcbFyN5f3buO1RAZ9pH1wXLxCesUJ0vFmLr9Y=", + "lastModified": 1660071133, + "narHash": "sha256-XX6T9wcvEZIVWY4TO5O1d2MgFyFrF2v4TpCFs7fjdn8=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "f44884060cb94240efbe55620f38a8ec8d9af601", + "rev": "36cc29d837e7232e3176e4651e8e117a6f231793", "type": "github" }, "original": { From 8fae18783d3e70be27f5dc22e1581dc4e8e8dfc0 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 15 Aug 2022 01:57:08 +0000 Subject: [PATCH 164/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/36cc29d837e7232e3176e4651e8e117a6f231793' (2022-08-09) → 'github:NixOS/nixpkgs/e105167e98817ba9fe079c6c3c544c6ef188e276' (2022-08-13) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index b830a83d1..6fef03ae3 100644 --- a/flake.lock +++ b/flake.lock @@ -17,11 +17,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1660071133, - "narHash": "sha256-XX6T9wcvEZIVWY4TO5O1d2MgFyFrF2v4TpCFs7fjdn8=", + "lastModified": 1660396586, + "narHash": "sha256-ePuWn7z/J5p2lO7YokOG1o01M0pDDVL3VrStaPpS5Ig=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "36cc29d837e7232e3176e4651e8e117a6f231793", + "rev": "e105167e98817ba9fe079c6c3c544c6ef188e276", "type": "github" }, "original": { From 9f6461221530d0336d159bf5e8509c31b798ea68 Mon Sep 17 00:00:00 2001 From: Zhaofeng Li Date: Wed, 17 Aug 2022 00:38:37 -0600 Subject: [PATCH 165/419] Don't override evalSettings.pureEval unless necessary Other eval flags like `--pure-eval` may activate `evalSettings.pureEval`, so let's avoid touching it if we don't have to. --- src/nix-eval-jobs.cc | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/src/nix-eval-jobs.cc b/src/nix-eval-jobs.cc index eb97d0b4e..dac8532ec 100644 --- a/src/nix-eval-jobs.cc +++ b/src/nix-eval-jobs.cc @@ -30,8 +30,6 @@ using namespace nix; using namespace nlohmann; -typedef enum { evalAuto, evalImpure, evalPure } pureEval; - // Safe to ignore - the args will be static. #ifdef __GNUC__ #pragma GCC diagnostic ignored "-Wnon-virtual-dtor" @@ -44,9 +42,9 @@ struct MyArgs : MixEvalArgs, MixCommonArgs { bool flake = false; bool meta = false; bool showTrace = false; + bool impure = false; size_t nrWorkers = 1; size_t maxMemorySize = 4096; - pureEval evalMode = evalAuto; MyArgs() : MixCommonArgs("nix-eval-jobs") { addFlag({ @@ -65,11 +63,9 @@ struct MyArgs : MixEvalArgs, MixCommonArgs { }}, }); - addFlag({ - .longName = "impure", - .description = "set evaluation mode", - .handler = {[&]() { evalMode = evalImpure; }}, - }); + addFlag({.longName = "impure", + .description = "allow impure expressions", + .handler = {&impure, true}}); addFlag({.longName = "gc-roots-dir", .description = "garbage collector roots directory", @@ -493,9 +489,11 @@ int main(int argc, char **argv) { /* When building a flake, use pure evaluation (no access to 'getEnv', 'currentSystem' etc. */ - evalSettings.pureEval = myArgs.evalMode == evalAuto - ? myArgs.flake - : myArgs.evalMode == evalPure; + if (myArgs.impure) { + evalSettings.pureEval = false; + } else if (myArgs.flake) { + evalSettings.pureEval = true; + } if (myArgs.releaseExpr == "") throw UsageError("no expression specified"); From 23f4dfdc24b06fa8d653c4f374b4a897641f0e8f Mon Sep 17 00:00:00 2001 From: Zhaofeng Li Date: Wed, 17 Aug 2022 00:38:37 -0600 Subject: [PATCH 166/419] Add support for passing a Nix expression on the command line Fixes #38. --- src/nix-eval-jobs.cc | 15 +++++++++++++-- tests/test_eval.py | 3 +++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/src/nix-eval-jobs.cc b/src/nix-eval-jobs.cc index dac8532ec..5a335a03c 100644 --- a/src/nix-eval-jobs.cc +++ b/src/nix-eval-jobs.cc @@ -37,9 +37,10 @@ using namespace nlohmann; #pragma clang diagnostic ignored "-Wnon-virtual-dtor" #endif struct MyArgs : MixEvalArgs, MixCommonArgs { - Path releaseExpr; + std::string releaseExpr; Path gcRootsDir; bool flake = false; + bool fromArgs = false; bool meta = false; bool showTrace = false; bool impure = false; @@ -97,6 +98,11 @@ struct MyArgs : MixEvalArgs, MixCommonArgs { "print out a stack trace in case of evaluation errors", .handler = {&showTrace, true}}); + addFlag({.longName = "expr", + .shortName = 'E', + .description = "treat the argument as a Nix expression", + .handler = {&fromArgs, true}}); + expectArg("expr", &releaseExpr); } }; @@ -111,7 +117,12 @@ static MyArgs myArgs; static Value *releaseExprTopLevelValue(EvalState &state, Bindings &autoArgs) { Value vTop; - state.evalFile(lookupFileArg(state, myArgs.releaseExpr), vTop); + if (myArgs.fromArgs) { + Expr *e = state.parseExprFromString(myArgs.releaseExpr, absPath(".")); + state.eval(e, vTop); + } else { + state.evalFile(lookupFileArg(state, myArgs.releaseExpr), vTop); + } auto vRoot = state.allocValue(); diff --git a/tests/test_eval.py b/tests/test_eval.py index 68dca7957..8ab26b7ce 100644 --- a/tests/test_eval.py +++ b/tests/test_eval.py @@ -55,3 +55,6 @@ def test_flake() -> None: def test_expression() -> None: common_test(["ci.nix"]) + + with open(TEST_ROOT.joinpath("assets/ci.nix"), "r") as ci_nix: + common_test(["-E", ci_nix.read()]) From 20685e0b23209816b5e5b07922c4f064cb0c096e Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 18 Aug 2022 02:01:23 +0000 Subject: [PATCH 167/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/e105167e98817ba9fe079c6c3c544c6ef188e276' (2022-08-13) → 'github:NixOS/nixpkgs/762b003329510ea855b4097a37511eb19c7077f0' (2022-08-16) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 6fef03ae3..ac034c5cd 100644 --- a/flake.lock +++ b/flake.lock @@ -17,11 +17,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1660396586, - "narHash": "sha256-ePuWn7z/J5p2lO7YokOG1o01M0pDDVL3VrStaPpS5Ig=", + "lastModified": 1660646295, + "narHash": "sha256-V4G+egGRc3elXPTr7QLJ7r7yrYed0areIKDiIAlMLC8=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "e105167e98817ba9fe079c6c3c544c6ef188e276", + "rev": "762b003329510ea855b4097a37511eb19c7077f0", "type": "github" }, "original": { From 2da08e05e28740241a513f039044913eaa3eaf57 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 22 Aug 2022 01:54:36 +0000 Subject: [PATCH 168/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/762b003329510ea855b4097a37511eb19c7077f0' (2022-08-16) → 'github:NixOS/nixpkgs/13711c9ab9f5a160a44affb7a6221be53318a873' (2022-08-20) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index ac034c5cd..85f8db323 100644 --- a/flake.lock +++ b/flake.lock @@ -17,11 +17,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1660646295, - "narHash": "sha256-V4G+egGRc3elXPTr7QLJ7r7yrYed0areIKDiIAlMLC8=", + "lastModified": 1660998696, + "narHash": "sha256-N5eDv9THZz5pFn7NR1swaFrAJYByfrA5gU5L7JONItA=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "762b003329510ea855b4097a37511eb19c7077f0", + "rev": "13711c9ab9f5a160a44affb7a6221be53318a873", "type": "github" }, "original": { From 086df547b2dadf3e163edd1a958f56e532fde91c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 22 Aug 2022 23:19:21 +0000 Subject: [PATCH 169/419] Bump DeterminateSystems/update-flake-lock from 12 to 13 Bumps [DeterminateSystems/update-flake-lock](https://github.com/DeterminateSystems/update-flake-lock) from 12 to 13. - [Release notes](https://github.com/DeterminateSystems/update-flake-lock/releases) - [Commits](https://github.com/DeterminateSystems/update-flake-lock/compare/v12...v13) --- updated-dependencies: - dependency-name: DeterminateSystems/update-flake-lock dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/update-flake-lock.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/update-flake-lock.yml b/.github/workflows/update-flake-lock.yml index da877d051..0932927bf 100644 --- a/.github/workflows/update-flake-lock.yml +++ b/.github/workflows/update-flake-lock.yml @@ -16,6 +16,6 @@ jobs: # extra_nix_config: | # access-tokens = github.com=${{ secrets.GITHUB_TOKEN }} - name: Update flake.lock - uses: DeterminateSystems/update-flake-lock@v12 + uses: DeterminateSystems/update-flake-lock@v13 with: token: ${{ secrets.GH_TOKEN_FOR_UPDATES }} From c7c39f25b6bb2c49684bde3d89c8ac2355b7b588 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 25 Aug 2022 02:00:25 +0000 Subject: [PATCH 170/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/13711c9ab9f5a160a44affb7a6221be53318a873' (2022-08-20) → 'github:NixOS/nixpkgs/f034b5693a26625f56068af983ed7727a60b5f8b' (2022-08-24) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 85f8db323..7645c2ff0 100644 --- a/flake.lock +++ b/flake.lock @@ -17,11 +17,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1660998696, - "narHash": "sha256-N5eDv9THZz5pFn7NR1swaFrAJYByfrA5gU5L7JONItA=", + "lastModified": 1661328374, + "narHash": "sha256-GGMupfk/lGzPBQ/dRrcQEhiFZ0F5KPg0j5Q4Fb5coxc=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "13711c9ab9f5a160a44affb7a6221be53318a873", + "rev": "f034b5693a26625f56068af983ed7727a60b5f8b", "type": "github" }, "original": { From 636bbca5d34dbf8ab797d434ef63d3f7195367f8 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 29 Aug 2022 02:07:51 +0000 Subject: [PATCH 171/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/f034b5693a26625f56068af983ed7727a60b5f8b' (2022-08-24) → 'github:NixOS/nixpkgs/324c8aaf25b2f2027af7798e5582ce3040a793b6' (2022-08-27) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 7645c2ff0..d2797570f 100644 --- a/flake.lock +++ b/flake.lock @@ -17,11 +17,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1661328374, - "narHash": "sha256-GGMupfk/lGzPBQ/dRrcQEhiFZ0F5KPg0j5Q4Fb5coxc=", + "lastModified": 1661628722, + "narHash": "sha256-oR/7NhG7pPkACToUtaaT6hH+rONE2z5/4NzjoUwEZt8=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "f034b5693a26625f56068af983ed7727a60b5f8b", + "rev": "324c8aaf25b2f2027af7798e5582ce3040a793b6", "type": "github" }, "original": { From c232716bae721bdb51e9c0bbd211a0d17f1ac3f5 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 1 Sep 2022 01:57:31 +0000 Subject: [PATCH 172/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/324c8aaf25b2f2027af7798e5582ce3040a793b6' (2022-08-27) → 'github:NixOS/nixpkgs/97747d3209efde533f7b1b28f1be11619f556a06' (2022-08-31) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index d2797570f..ab0f416a6 100644 --- a/flake.lock +++ b/flake.lock @@ -17,11 +17,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1661628722, - "narHash": "sha256-oR/7NhG7pPkACToUtaaT6hH+rONE2z5/4NzjoUwEZt8=", + "lastModified": 1661931183, + "narHash": "sha256-0+2KzcexiJCB3Il5t7cZAM2RXNRfm5/gMCwhcZJxLuQ=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "324c8aaf25b2f2027af7798e5582ce3040a793b6", + "rev": "97747d3209efde533f7b1b28f1be11619f556a06", "type": "github" }, "original": { From 86f857519117d03ea4a71e8b38bc013b0591c214 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 5 Sep 2022 02:08:43 +0000 Subject: [PATCH 173/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/97747d3209efde533f7b1b28f1be11619f556a06' (2022-08-31) → 'github:NixOS/nixpkgs/2da64a81275b68fdad38af669afeda43d401e94b' (2022-09-01) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index ab0f416a6..45b4c6d70 100644 --- a/flake.lock +++ b/flake.lock @@ -17,11 +17,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1661931183, - "narHash": "sha256-0+2KzcexiJCB3Il5t7cZAM2RXNRfm5/gMCwhcZJxLuQ=", + "lastModified": 1662019588, + "narHash": "sha256-oPEjHKGGVbBXqwwL+UjsveJzghWiWV0n9ogo1X6l4cw=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "97747d3209efde533f7b1b28f1be11619f556a06", + "rev": "2da64a81275b68fdad38af669afeda43d401e94b", "type": "github" }, "original": { From 7c38c184a33fbeedc4716f007c6ece3c866d052e Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 12 Sep 2022 02:16:38 +0000 Subject: [PATCH 174/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/2da64a81275b68fdad38af669afeda43d401e94b' (2022-09-01) → 'github:NixOS/nixpkgs/93a0067a9c85c17764f7755947e6ecf52dc47d8a' (2022-09-10) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 45b4c6d70..061fe19b0 100644 --- a/flake.lock +++ b/flake.lock @@ -17,11 +17,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1662019588, - "narHash": "sha256-oPEjHKGGVbBXqwwL+UjsveJzghWiWV0n9ogo1X6l4cw=", + "lastModified": 1662821606, + "narHash": "sha256-Z9z9iSH+tgJ0iyRcBfEQRwELgjnhpVXsktiWiFe3SuY=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "2da64a81275b68fdad38af669afeda43d401e94b", + "rev": "93a0067a9c85c17764f7755947e6ecf52dc47d8a", "type": "github" }, "original": { From fc3180440284725d07e1a5d443c28c7b3d945486 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 15 Sep 2022 02:13:42 +0000 Subject: [PATCH 175/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/93a0067a9c85c17764f7755947e6ecf52dc47d8a' (2022-09-10) → 'github:NixOS/nixpkgs/9608ace7009ce5bc3aeb940095e01553e635cbc7' (2022-09-13) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 061fe19b0..a16b090fd 100644 --- a/flake.lock +++ b/flake.lock @@ -17,11 +17,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1662821606, - "narHash": "sha256-Z9z9iSH+tgJ0iyRcBfEQRwELgjnhpVXsktiWiFe3SuY=", + "lastModified": 1663087123, + "narHash": "sha256-cNIRkF/J4mRxDtNYw+9/fBNq/NOA2nCuPOa3EdIyeDs=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "93a0067a9c85c17764f7755947e6ecf52dc47d8a", + "rev": "9608ace7009ce5bc3aeb940095e01553e635cbc7", "type": "github" }, "original": { From f6890b93e868db9dfe94499550ec5e91e9a0de0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Fri, 9 Sep 2022 11:24:53 +0000 Subject: [PATCH 176/419] add check-cache-status option to query wether a binary cache has the build --- src/nix-eval-jobs.cc | 45 +++++++++++++++++++++++++++++++++++++------- tests/test_eval.py | 20 ++++++++++++++++---- 2 files changed, 54 insertions(+), 11 deletions(-) diff --git a/src/nix-eval-jobs.cc b/src/nix-eval-jobs.cc index 5a335a03c..8544b15da 100644 --- a/src/nix-eval-jobs.cc +++ b/src/nix-eval-jobs.cc @@ -44,6 +44,7 @@ struct MyArgs : MixEvalArgs, MixCommonArgs { bool meta = false; bool showTrace = false; bool impure = false; + bool checkCacheStatus = false; size_t nrWorkers = 1; size_t maxMemorySize = 4096; @@ -93,6 +94,15 @@ struct MyArgs : MixEvalArgs, MixCommonArgs { .description = "include derivation meta field in output", .handler = {&meta, true}}); + addFlag( + {.longName = "check-cache-status", + .description = + "Check if the derivations are present locally or in " + "any configured substituters (i.e. binary cache). The " + "information " + "will be exposed in the `isCached` field of the JSON output.", + .handler = {&checkCacheStatus, true}}); + addFlag({.longName = "show-trace", .description = "print out a stack trace in case of evaluation errors", @@ -171,11 +181,26 @@ Value *topLevelValue(EvalState &state, Bindings &autoArgs) { : releaseExprTopLevelValue(state, autoArgs); } +bool queryIsCached(Store &store, std::map &outputs) { + uint64_t downloadSize, narSize; + StorePathSet willBuild, willSubstitute, unknown; + + std::vector paths; + for (auto const &[key, val] : outputs) { + paths.push_back(followLinksToStorePathWithOutputs(store, val)); + } + + store.queryMissing(toDerivedPaths(paths), willBuild, willSubstitute, + unknown, downloadSize, narSize); + return willBuild.empty() && unknown.empty(); +} + /* The fields of a derivation that are printed in json form */ struct Drv { std::string name; std::string system; std::string drvPath; + bool isCached; std::map outputs; std::optional meta; @@ -209,6 +234,9 @@ struct Drv { } meta = meta_; } + if (myArgs.checkCacheStatus) { + isCached = queryIsCached(*localStore, outputs); + } name = drvInfo.queryName(); system = drvInfo.querySystem(); @@ -217,15 +245,18 @@ struct Drv { }; static void to_json(nlohmann::json &json, const Drv &drv) { - json = nlohmann::json{ - {"name", drv.name}, - {"system", drv.system}, - {"drvPath", drv.drvPath}, - {"outputs", drv.outputs}, - }; + json = nlohmann::json{{"name", drv.name}, + {"system", drv.system}, + {"drvPath", drv.drvPath}, + {"outputs", drv.outputs}}; - if (drv.meta.has_value()) + if (drv.meta.has_value()) { json["meta"] = drv.meta.value(); + } + + if (myArgs.checkCacheStatus) { + json["isCached"] = drv.isCached; + } } std::string attrPathJoin(json input) { diff --git a/tests/test_eval.py b/tests/test_eval.py index 8ab26b7ce..5cdad3b03 100644 --- a/tests/test_eval.py +++ b/tests/test_eval.py @@ -4,14 +4,14 @@ import subprocess import json from tempfile import TemporaryDirectory from pathlib import Path -from typing import List +from typing import List, Dict, Any TEST_ROOT = Path(__file__).parent.resolve() PROJECT_ROOT = TEST_ROOT.parent BIN = PROJECT_ROOT.joinpath("build", "src", "nix-eval-jobs") -def common_test(extra_args: List[str]) -> None: +def common_test(extra_args: List[str]) -> List[Dict[str, Any]]: with TemporaryDirectory() as tempdir: cmd = [str(BIN), "--gc-roots-dir", tempdir, "--meta"] + extra_args res = subprocess.run( @@ -47,14 +47,26 @@ def common_test(extra_args: List[str]) -> None: assert substituted_job["attr"] == "substitutedJob" assert substituted_job["name"].startswith("hello-") assert substituted_job["meta"]["broken"] is False + return results def test_flake() -> None: - common_test(["--flake", ".#hydraJobs"]) + results = common_test(["--flake", ".#hydraJobs"]) + for result in results: + assert "isCached" not in result + + +def test_query_cache_status() -> None: + results = common_test(["--flake", ".#hydraJobs", "--check-cache-status"]) + # FIXME in the nix sandbox we cannot query binary caches, this would need some local one + for result in results: + assert "isCached" in result def test_expression() -> None: - common_test(["ci.nix"]) + results = common_test(["ci.nix"]) + for result in results: + assert "isCached" not in result with open(TEST_ROOT.joinpath("assets/ci.nix"), "r") as ci_nix: common_test(["-E", ci_nix.read()]) From cb8126538f1a7ee37eeae895a683d81d37305447 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Fri, 9 Sep 2022 11:26:15 +0000 Subject: [PATCH 177/419] fix recurseIntoAttrs handling --- src/nix-eval-jobs.cc | 85 +++++++++++++++++++++----------------------- tests/test_eval.py | 7 ++-- 2 files changed, 42 insertions(+), 50 deletions(-) diff --git a/src/nix-eval-jobs.cc b/src/nix-eval-jobs.cc index 8544b15da..68b641682 100644 --- a/src/nix-eval-jobs.cc +++ b/src/nix-eval-jobs.cc @@ -297,60 +297,55 @@ static void worker(EvalState &state, Bindings &autoArgs, AutoCloseFD &to, auto v = state.allocValue(); state.autoCallFunction(autoArgs, *vTmp, *v); - if (auto drvInfo = getDerivation(state, *v, false)) { + if (v->type() == nAttrs) { + if (auto drvInfo = getDerivation(state, *v, false)) { + auto drv = Drv(state, *drvInfo); + reply.update(drv); - auto drv = Drv(state, *drvInfo); - reply.update(drv); - - /* Register the derivation as a GC root. !!! This - registers roots for jobs that we may have already - done. */ - if (myArgs.gcRootsDir != "") { - Path root = myArgs.gcRootsDir + "/" + - std::string(baseNameOf(drv.drvPath)); - if (!pathExists(root)) { - auto localStore = - state.store.dynamic_pointer_cast(); - auto storePath = - localStore->parseStorePath(drv.drvPath); - localStore->addPermRoot(storePath, root); + /* Register the derivation as a GC root. !!! This + registers roots for jobs that we may have already + done. */ + if (myArgs.gcRootsDir != "") { + Path root = myArgs.gcRootsDir + "/" + + std::string(baseNameOf(drv.drvPath)); + if (!pathExists(root)) { + auto localStore = + state.store + .dynamic_pointer_cast(); + auto storePath = + localStore->parseStorePath(drv.drvPath); + localStore->addPermRoot(storePath, root); + } } - } + } else { + auto attrs = nlohmann::json::array(); + bool recurse = + path.size() == 0; // Dont require `recurseForDerivations + // = true;` for top-level attrset - } + for (auto &i : + v->attrs->lexicographicOrder(state.symbols)) { + const std::string &name = state.symbols[i->name]; + attrs.push_back(name); - else if (v->type() == nAttrs) { - auto attrs = nlohmann::json::array(); - bool recurse = - path.size() == 0; // Dont require `recurseForDerivations = - // true;` for top-level attrset - - for (auto &i : v->attrs->lexicographicOrder(state.symbols)) { - const std::string &name = state.symbols[i->name]; - attrs.push_back(name); - - if (name == "recurseForDerivations") { - auto attrv = - v->attrs->get(state.sRecurseForDerivations); - recurse = state.forceBool(*attrv->value, attrv->pos); + if (name == "recurseForDerivations") { + auto attrv = + v->attrs->get(state.sRecurseForDerivations); + recurse = + state.forceBool(*attrv->value, attrv->pos); + } } + if (recurse) + reply["attrs"] = std::move(attrs); + else + reply["attrs"] = nlohmann::json::array(); } - if (recurse) - reply["attrs"] = std::move(attrs); - else - reply["attrs"] = nlohmann::json::array(); + } else { + // We ignore everything that cannot be build + reply["attrs"] = nlohmann::json::array(); } - - else if (v->type() == nNull) - ; - - else - throw TypeError("attribute '%s' is %s, which is not supported", - path, showType(*v)); - } catch (EvalError &e) { auto err = e.info(); - std::ostringstream oss; showErrorInfo(oss, err, loggerSettings.showTrace.get()); auto msg = oss.str(); diff --git a/tests/test_eval.py b/tests/test_eval.py index 5cdad3b03..a08332e60 100644 --- a/tests/test_eval.py +++ b/tests/test_eval.py @@ -23,7 +23,7 @@ def common_test(extra_args: List[str]) -> List[Dict[str, Any]]: ) results = [json.loads(r) for r in res.stdout.split("\n") if r] - assert len(results) == 5 + assert len(results) == 4 built_job = results[0] assert built_job["attr"] == "builtJob" @@ -40,10 +40,7 @@ def common_test(extra_args: List[str]) -> List[Dict[str, Any]]: assert recurse_drv["attr"] == "recurse.drvB" assert recurse_drv["name"] == "drvB" - recurse_recurse_bool = results[3] - assert "error" in recurse_recurse_bool - - substituted_job = results[4] + substituted_job = results[3] assert substituted_job["attr"] == "substitutedJob" assert substituted_job["name"].startswith("hello-") assert substituted_job["meta"]["broken"] is False From f8fa3ea069bf0f2547b57abf8c1aaa848def7e0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Fri, 9 Sep 2022 12:32:22 +0000 Subject: [PATCH 178/419] README: update available options --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 98433c629..569b885d3 100644 --- a/README.md +++ b/README.md @@ -48,12 +48,14 @@ USAGE: nix-eval-jobs [options] expr --arg Pass the value *expr* as the argument *name* to Nix functions. --argstr Pass the string *string* as the argument *name* to Nix functions. + --check-cache-status Check if the derivations are present locally or in any configured substituters (i.e. binary cache). The information will be exposed in the `isCached` field of the JSON output. --debug Set the logging verbosity level to 'debug'. --eval-store The Nix store to use for evaluations. + --expr treat the argument as a Nix expression --flake build a flake --gc-roots-dir garbage collector roots directory --help show usage information - --impure set evaluation mode + --impure allow impure expressions --include Add *path* to the list of locations used to look up `<...>` file names. --log-format Set the format of log output; one of `raw`, `internal-json`, `bar` or `bar-with-logs`. --max-memory-size maximum evaluation memory size @@ -61,6 +63,7 @@ USAGE: nix-eval-jobs [options] expr --option Set the Nix configuration setting *name* to *value* (overriding `nix.conf`). --override-flake Override the flake registries, redirecting *original-ref* to *resolved-ref*. --quiet Decrease the logging verbosity level. + --show-trace print out a stack trace in case of evaluation errors --verbose Increase the logging verbosity level. --workers number of evaluate workers ``` @@ -83,7 +86,6 @@ single large log file. In the we collect example ci configuration for various CIs. - ## Organisation of this repository On the `main` branch we target nixUnstable. When a release of nix happens, we From 70afaeebca9f27a7f85fa930c5c954f680cb5f87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sat, 17 Sep 2022 10:48:25 +0200 Subject: [PATCH 179/419] avoid shadowing metaName --- src/nix-eval-jobs.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/nix-eval-jobs.cc b/src/nix-eval-jobs.cc index 68b641682..a8d56ab46 100644 --- a/src/nix-eval-jobs.cc +++ b/src/nix-eval-jobs.cc @@ -217,11 +217,11 @@ struct Drv { if (myArgs.meta) { nlohmann::json meta_; - for (auto &name : drvInfo.queryMetaNames()) { + for (auto &metaName : drvInfo.queryMetaNames()) { PathSet context; std::stringstream ss; - auto metaValue = drvInfo.queryMeta(name); + auto metaValue = drvInfo.queryMeta(metaName); // Skip non-serialisable types // TODO: Fix serialisation of derivations to store paths if (metaValue == 0) { @@ -230,7 +230,7 @@ struct Drv { printValueAsJSON(state, true, *metaValue, noPos, ss, context); - meta_[name] = nlohmann::json::parse(ss.str()); + meta_[metaName] = nlohmann::json::parse(ss.str()); } meta = meta_; } From 4238ece39f6cb7b29f396bb1a3b46ad3c116c310 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 19 Sep 2022 02:13:02 +0000 Subject: [PATCH 180/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/9608ace7009ce5bc3aeb940095e01553e635cbc7' (2022-09-13) → 'github:NixOS/nixpkgs/da6a05816e7fa5226c3f61e285ef8d9dfc868f3c' (2022-09-16) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index a16b090fd..91d9df48b 100644 --- a/flake.lock +++ b/flake.lock @@ -17,11 +17,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1663087123, - "narHash": "sha256-cNIRkF/J4mRxDtNYw+9/fBNq/NOA2nCuPOa3EdIyeDs=", + "lastModified": 1663357389, + "narHash": "sha256-oYA2nVRSi6yhCBqS5Vz465Hw+3BQOVFEhfbfy//3vTs=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "9608ace7009ce5bc3aeb940095e01553e635cbc7", + "rev": "da6a05816e7fa5226c3f61e285ef8d9dfc868f3c", "type": "github" }, "original": { From 183f7593db3d6142793858056f6889753f2923db Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 Sep 2022 23:10:09 +0000 Subject: [PATCH 181/419] Bump DeterminateSystems/update-flake-lock from 13 to 14 Bumps [DeterminateSystems/update-flake-lock](https://github.com/DeterminateSystems/update-flake-lock) from 13 to 14. - [Release notes](https://github.com/DeterminateSystems/update-flake-lock/releases) - [Commits](https://github.com/DeterminateSystems/update-flake-lock/compare/v13...v14) --- updated-dependencies: - dependency-name: DeterminateSystems/update-flake-lock dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/update-flake-lock.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/update-flake-lock.yml b/.github/workflows/update-flake-lock.yml index 0932927bf..c60af9a9f 100644 --- a/.github/workflows/update-flake-lock.yml +++ b/.github/workflows/update-flake-lock.yml @@ -16,6 +16,6 @@ jobs: # extra_nix_config: | # access-tokens = github.com=${{ secrets.GITHUB_TOKEN }} - name: Update flake.lock - uses: DeterminateSystems/update-flake-lock@v13 + uses: DeterminateSystems/update-flake-lock@v14 with: token: ${{ secrets.GH_TOKEN_FOR_UPDATES }} From 47a74fed83c4ea0985b50ee45bd8fe722c3c18f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Tue, 20 Sep 2022 20:12:55 +0200 Subject: [PATCH 182/419] also accept relative gc root directories --- src/nix-eval-jobs.cc | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/nix-eval-jobs.cc b/src/nix-eval-jobs.cc index a8d56ab46..e20ed5c5a 100644 --- a/src/nix-eval-jobs.cc +++ b/src/nix-eval-jobs.cc @@ -1,6 +1,7 @@ #include #include #include +#include #include #include @@ -535,8 +536,11 @@ int main(int argc, char **argv) { if (myArgs.releaseExpr == "") throw UsageError("no expression specified"); - if (myArgs.gcRootsDir == "") + if (myArgs.gcRootsDir == "") { printMsg(lvlError, "warning: `--gc-roots-dir' not specified"); + } else { + myArgs.gcRootsDir = std::filesystem::absolute(myArgs.gcRootsDir); + } if (myArgs.showTrace) { loggerSettings.showTrace.assign(true); From 1d170192d3f44c548a8066dc465d0d0f2f8dc91d Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 22 Sep 2022 02:04:54 +0000 Subject: [PATCH 183/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/da6a05816e7fa5226c3f61e285ef8d9dfc868f3c' (2022-09-16) → 'github:NixOS/nixpkgs/f677051b8dc0b5e2a9348941c99eea8c4b0ff28f' (2022-09-18) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 91d9df48b..6366ed320 100644 --- a/flake.lock +++ b/flake.lock @@ -17,11 +17,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1663357389, - "narHash": "sha256-oYA2nVRSi6yhCBqS5Vz465Hw+3BQOVFEhfbfy//3vTs=", + "lastModified": 1663494472, + "narHash": "sha256-fSowlaoXXWcAM8m9wA6u+eTJJtvruYHMA+Lb/tFi/qM=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "da6a05816e7fa5226c3f61e285ef8d9dfc868f3c", + "rev": "f677051b8dc0b5e2a9348941c99eea8c4b0ff28f", "type": "github" }, "original": { From 765cbd89bc434cddc93a1e0ce6929f8663ae6f8c Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 26 Sep 2022 02:10:08 +0000 Subject: [PATCH 184/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/f677051b8dc0b5e2a9348941c99eea8c4b0ff28f' (2022-09-18) → 'github:NixOS/nixpkgs/fde244a8c7655bc28616864e2290ad9c95409c2c' (2022-09-24) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 6366ed320..ed6feded3 100644 --- a/flake.lock +++ b/flake.lock @@ -17,11 +17,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1663494472, - "narHash": "sha256-fSowlaoXXWcAM8m9wA6u+eTJJtvruYHMA+Lb/tFi/qM=", + "lastModified": 1664017330, + "narHash": "sha256-919WZKBTxFdTkzIK6uJXE7hwSPQb7e/ekybxxWaotR4=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "f677051b8dc0b5e2a9348941c99eea8c4b0ff28f", + "rev": "fde244a8c7655bc28616864e2290ad9c95409c2c", "type": "github" }, "original": { From 74abfca3010607c89677d8faa9a1e101c5a6e13a Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 29 Sep 2022 02:19:15 +0000 Subject: [PATCH 185/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/fde244a8c7655bc28616864e2290ad9c95409c2c' (2022-09-24) → 'github:NixOS/nixpkgs/7e52b35fe98481a279d89f9c145f8076d049d2b9' (2022-09-27) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index ed6feded3..0bd8d08e7 100644 --- a/flake.lock +++ b/flake.lock @@ -17,11 +17,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1664017330, - "narHash": "sha256-919WZKBTxFdTkzIK6uJXE7hwSPQb7e/ekybxxWaotR4=", + "lastModified": 1664281702, + "narHash": "sha256-haixZ4TJLu1Dciow54wrHrHvlGDVr5sW6MTeAV/ZLuI=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "fde244a8c7655bc28616864e2290ad9c95409c2c", + "rev": "7e52b35fe98481a279d89f9c145f8076d049d2b9", "type": "github" }, "original": { From 8b67134e5524bf64b5e125d75984560a6d1aaff6 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 3 Oct 2022 01:50:55 +0000 Subject: [PATCH 186/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/7e52b35fe98481a279d89f9c145f8076d049d2b9' (2022-09-27) → 'github:NixOS/nixpkgs/59d2991d4256cdca1c0cda45d876c80a0fe45c31' (2022-10-02) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 0bd8d08e7..c46f316c4 100644 --- a/flake.lock +++ b/flake.lock @@ -17,11 +17,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1664281702, - "narHash": "sha256-haixZ4TJLu1Dciow54wrHrHvlGDVr5sW6MTeAV/ZLuI=", + "lastModified": 1664687381, + "narHash": "sha256-9czSuDzS+OGGwq2kC4KXBLXWfYaup+oLB+AA1Md25U4=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "7e52b35fe98481a279d89f9c145f8076d049d2b9", + "rev": "59d2991d4256cdca1c0cda45d876c80a0fe45c31", "type": "github" }, "original": { From e011cbd4336b5658d81877be8671f9323cb47bdf Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 6 Oct 2022 01:51:39 +0000 Subject: [PATCH 187/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/59d2991d4256cdca1c0cda45d876c80a0fe45c31' (2022-10-02) → 'github:NixOS/nixpkgs/b7a6fde153d9470afdb6aa1da51c4117f03b84ed' (2022-10-04) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index c46f316c4..2c10a3b9a 100644 --- a/flake.lock +++ b/flake.lock @@ -17,11 +17,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1664687381, - "narHash": "sha256-9czSuDzS+OGGwq2kC4KXBLXWfYaup+oLB+AA1Md25U4=", + "lastModified": 1664871473, + "narHash": "sha256-1LzbW6G6Uz8akWiOdlIi435GAm1ct5jF5tovw/9to0o=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "59d2991d4256cdca1c0cda45d876c80a0fe45c31", + "rev": "b7a6fde153d9470afdb6aa1da51c4117f03b84ed", "type": "github" }, "original": { From b445d2c1ad0787529f9c91632f62304cd9efd89f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Thu, 6 Oct 2022 22:06:14 +0200 Subject: [PATCH 188/419] README: update patchelf example --- README.md | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 569b885d3..bf6570063 100644 --- a/README.md +++ b/README.md @@ -22,18 +22,10 @@ service and user-started nix builds processes. In the following example we evaluate the hydraJobs attribute of the [patchelf](https://github.com/NixOS/patchelf) flake: ```console -$ nix-eval-jobs --gc-roots-dir $(pwd)/gcroot --flake 'github:NixOS/patchelf#hydraJobs' -{"attr":"build-sanitized-clang.x86_64-linux","drvPath":"/nix/store/igmkq61cwys8nj34yqvnpdg921h0i0mp-patchelf-0.14.3.drv","name":"patchelf-0.14.3","outputs":{"out":"/nix/store/nwwgff1fwkws4wxv7k7cfvvin8ab9gbh-patchelf-0.14.3"},"system":"x86_64-linux"} -{"attr":"build-sanitized.aarch64-linux","drvPath":"/nix/store/d8ma8d7gjwx6ix4ibs910z9fkm3hwdvz-patchelf-0.14.3.drv","name":"patchelf-0.14.3","outputs":{"out":"/nix/store/6j26m4sznwdyfk4sbmnls3sk0lxm38ih-patchelf-0.14.3"},"system":"aarch64-linux"} -{"attr":"build-sanitized.i686-linux","drvPath":"/nix/store/87rwijvfqqs7dw9lbmckmz4nbryvjaq3-patchelf-0.14.3.drv","name":"patchelf-0.14.3","outputs":{"out":"/nix/store/za5w0gzf97na44fza9sdys15qnjqayd7-patchelf-0.14.3"},"system":"i686-linux"} -{"attr":"build-sanitized.x86_64-linux","drvPath":"/nix/store/nmx50wly2qvd00svx0vqsjfh0jv7q3kl-patchelf-0.14.3.drv","name":"patchelf-0.14.3","outputs":{"out":"/nix/store/38d6bhz3a5jq48gm1diji0rjfcm5vi9n-patchelf-0.14.3"},"system":"x86_64-linux"} -{"attr":"build.aarch64-linux","drvPath":"/nix/store/yjz9msbr6pl8mj7im5kiyhk7wwkvxywa-patchelf-0.14.3.drv","name":"patchelf-0.14.3","outputs":{"out":"/nix/store/as9xhcfwnhfy5x30kxh7lfgla1qrk182-patchelf-0.14.3"},"system":"aarch64-linux"} -{"attr":"build.i686-linux","drvPath":"/nix/store/nwcmdcimnaci0knri5ga019lgbvc4am4-patchelf-0.14.3.drv","name":"patchelf-0.14.3","outputs":{"out":"/nix/store/64x12dmbscnnl42r4y2av52y55ksphhk-patchelf-0.14.3"},"system":"i686-linux"} -{"attr":"build.x86_64-linux","drvPath":"/nix/store/k6p4qnjryr2l1lz31pf085ay9bd7j8gj-patchelf-0.14.3.drv","name":"patchelf-0.14.3","outputs":{"out":"/nix/store/h9a779ghpibfqkkdchx6s08bb3v3i8vy-patchelf-0.14.3"},"system":"x86_64-linux"} -{"attr":"coverage","drvPath":"/nix/store/lsrg05dx3hyi5b6ak99pn9g1rn8xwx39-patchelf-coverage-0.14.3.drv","name":"patchelf-coverage-0.14.3","outputs":{"out":"/nix/store/6h4l5axy5lvxzq662yw47y9r60mxw3zz-patchelf-coverage-0.14.3"},"system":"x86_64-linux"} -{"attr":"release","drvPath":"/nix/store/dgn5gy64pjskfnv7vqh0s86nb998f8sq-patchelf-0.14.3.drv","name":"patchelf-0.14.3","outputs":{"out":"/nix/store/nn05yaznr5af8g8mpgd82yx16pvfzjcy-patchelf-0.14.3"},"system":"x86_64-linux"} -{"attr":"tarball","drvPath":"/nix/store/5ajrgfd5nx29ykgg942k154mcaqfbhxd-patchelf-tarball-0.14.3.drv","name":"patchelf-tarball-0.14.3","outputs":{"out":"/nix/store/5cli6rh0h32yhfcgjkgbplcc73cqvplv-patchelf-tarball-0.14.3"},"system":"x86_64-linux"} - +$ nix-eval-jobs --gc-roots-dir gcroot --flake 'github:NixOS/patchelf#hydraJobs' +{"attr":"coverage","attrPath":["coverage"],"drvPath":"/nix/store/8hq9f09xa5s6g9m02lw0sw59kkkvj57c-patchelf-coverage-0.15.0.drv","name":"patchelf-coverage-0.15.0","outputs":{"out":"/nix/store/dwf255bdbfvvbiqak941r83zlvxyipcs-patchelf-coverage-0.15.0"},"system":"x86_64-linux"} +{"attr":"release","attrPath":["release"],"drvPath":"/nix/store/ip9dy4vlyha5a7kq4bnf4pxk0sfwjfda-patchelf-0.15.0.drv","name":"patchelf-0.15.0","outputs":{"out":"/nix/store/5z9ynn29asakf1b5736im2glcqpf6s2f-patchelf-0.15.0"},"system":"x86_64-linux"} +{"attr":"tarball","attrPath":["tarball"],"drvPath":"/nix/store/g1alnfi3mrkcb9blclr77fpyp35mpsdd-patchelf-tarball-0.15.0.drv","name":"patchelf-tarball-0.15.0","outputs":{"out":"/nix/store/iy0w42pffhjg6wy0w46r4cjc1yjk410y-patchelf-tarball-0.15.0"},"system":"x86_64-linux"} ``` The output here is newline-seperated json according to https://jsonlines.org. From d2cac01995db0e5767bffcac97c7c5fd8266becf Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 10 Oct 2022 02:07:09 +0000 Subject: [PATCH 189/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/b7a6fde153d9470afdb6aa1da51c4117f03b84ed' (2022-10-04) → 'github:NixOS/nixpkgs/c5924154f000e6306030300592f4282949b2db6c' (2022-10-08) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 2c10a3b9a..d3b6bd9f2 100644 --- a/flake.lock +++ b/flake.lock @@ -17,11 +17,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1664871473, - "narHash": "sha256-1LzbW6G6Uz8akWiOdlIi435GAm1ct5jF5tovw/9to0o=", + "lastModified": 1665259268, + "narHash": "sha256-ONFhHBLv5nZKhwV/F2GOH16197PbvpyWhoO0AOyktkU=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "b7a6fde153d9470afdb6aa1da51c4117f03b84ed", + "rev": "c5924154f000e6306030300592f4282949b2db6c", "type": "github" }, "original": { From 633830a6a2a0cda47271f0d62a2d4ee64f047489 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 13 Oct 2022 02:04:49 +0000 Subject: [PATCH 190/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/c5924154f000e6306030300592f4282949b2db6c' (2022-10-08) → 'github:NixOS/nixpkgs/285e77efe87df64105ec14b204de6636fb0a7a27' (2022-10-11) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index d3b6bd9f2..889cd9d0e 100644 --- a/flake.lock +++ b/flake.lock @@ -17,11 +17,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1665259268, - "narHash": "sha256-ONFhHBLv5nZKhwV/F2GOH16197PbvpyWhoO0AOyktkU=", + "lastModified": 1665449268, + "narHash": "sha256-cw4xrQIAZUyJGj58Dp5VLICI0rscd+uap83afiFzlcA=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "c5924154f000e6306030300592f4282949b2db6c", + "rev": "285e77efe87df64105ec14b204de6636fb0a7a27", "type": "github" }, "original": { From a0a9e23248c3e2f88946027b1cac947479cf6453 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 17 Oct 2022 02:24:47 +0000 Subject: [PATCH 191/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/285e77efe87df64105ec14b204de6636fb0a7a27' (2022-10-11) → 'github:NixOS/nixpkgs/83b198a2083774844962c854f811538323f9f7b1' (2022-10-15) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 889cd9d0e..795cc10d7 100644 --- a/flake.lock +++ b/flake.lock @@ -17,11 +17,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1665449268, - "narHash": "sha256-cw4xrQIAZUyJGj58Dp5VLICI0rscd+uap83afiFzlcA=", + "lastModified": 1665848363, + "narHash": "sha256-3Jow1YxzPtQnck1bAAvbVxgRH4gNnkIdw871Vm6UtAU=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "285e77efe87df64105ec14b204de6636fb0a7a27", + "rev": "83b198a2083774844962c854f811538323f9f7b1", "type": "github" }, "original": { From 4f0d7650e1b5a324995846723c4ae2fa993437b5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 17 Oct 2022 23:09:30 +0000 Subject: [PATCH 192/419] Bump cachix/install-nix-action from 17 to 18 Bumps [cachix/install-nix-action](https://github.com/cachix/install-nix-action) from 17 to 18. - [Release notes](https://github.com/cachix/install-nix-action/releases) - [Commits](https://github.com/cachix/install-nix-action/compare/v17...v18) --- updated-dependencies: - dependency-name: cachix/install-nix-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/flake-check.yml | 4 ++-- .github/workflows/test-develop-flakes.yml | 2 +- .github/workflows/update-flake-lock.yml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/flake-check.yml b/.github/workflows/flake-check.yml index 603916100..98303a577 100644 --- a/.github/workflows/flake-check.yml +++ b/.github/workflows/flake-check.yml @@ -15,7 +15,7 @@ jobs: with: # Nix Flakes doesn't work on shallow clones fetch-depth: 0 - - uses: cachix/install-nix-action@v17 + - uses: cachix/install-nix-action@v18 with: nix_path: nixpkgs=channel:nixos-unstable - id: set-matrix @@ -35,7 +35,7 @@ jobs: with: # Nix Flakes doesn't work on shallow clones fetch-depth: 0 - - uses: cachix/install-nix-action@v17 + - uses: cachix/install-nix-action@v18 with: nix_path: nixpkgs=channel:nixos-unstable - run: | diff --git a/.github/workflows/test-develop-flakes.yml b/.github/workflows/test-develop-flakes.yml index 6a8d714b8..cc33cfcea 100644 --- a/.github/workflows/test-develop-flakes.yml +++ b/.github/workflows/test-develop-flakes.yml @@ -15,7 +15,7 @@ jobs: with: # Nix Flakes doesn't work on shallow clones fetch-depth: 0 - - uses: cachix/install-nix-action@v17 + - uses: cachix/install-nix-action@v18 - name: Build run: nix develop -c bash -c 'meson build && cd build && ninja' - name: Run tests diff --git a/.github/workflows/update-flake-lock.yml b/.github/workflows/update-flake-lock.yml index c60af9a9f..f7ab7b1e8 100644 --- a/.github/workflows/update-flake-lock.yml +++ b/.github/workflows/update-flake-lock.yml @@ -11,7 +11,7 @@ jobs: - name: Checkout repository uses: actions/checkout@v3 - name: Install Nix - uses: cachix/install-nix-action@v17 + uses: cachix/install-nix-action@v18 # with: # extra_nix_config: | # access-tokens = github.com=${{ secrets.GITHUB_TOKEN }} From b9836f2e0693e403109ff7bb1ba70a723c555834 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Tue, 18 Oct 2022 13:56:45 +1300 Subject: [PATCH 193/419] Add parameter to force recursion (don't honour recurseIntoAttrs) This can be useful when you are not in control over your expressions. My use case is evaluating a Hydra jobset. --- src/nix-eval-jobs.cc | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/nix-eval-jobs.cc b/src/nix-eval-jobs.cc index e20ed5c5a..b4322f581 100644 --- a/src/nix-eval-jobs.cc +++ b/src/nix-eval-jobs.cc @@ -45,6 +45,7 @@ struct MyArgs : MixEvalArgs, MixCommonArgs { bool meta = false; bool showTrace = false; bool impure = false; + bool forceRecurse = false; bool checkCacheStatus = false; size_t nrWorkers = 1; size_t maxMemorySize = 4096; @@ -70,6 +71,11 @@ struct MyArgs : MixEvalArgs, MixCommonArgs { .description = "allow impure expressions", .handler = {&impure, true}}); + addFlag( + {.longName = "force-recurse", + .description = "force recursion (don't respect recurseIntoAttrs)", + .handler = {&forceRecurse, true}}); + addFlag({.longName = "gc-roots-dir", .description = "garbage collector roots directory", .labels = {"path"}, @@ -321,6 +327,7 @@ static void worker(EvalState &state, Bindings &autoArgs, AutoCloseFD &to, } else { auto attrs = nlohmann::json::array(); bool recurse = + myArgs.forceRecurse || path.size() == 0; // Dont require `recurseForDerivations // = true;` for top-level attrset From e0c27dc9840de943065b5f3d8535151a8d0d1233 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Tue, 18 Oct 2022 14:42:02 +1300 Subject: [PATCH 194/419] Respect passed eval store --- src/nix-eval-jobs.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/nix-eval-jobs.cc b/src/nix-eval-jobs.cc index e20ed5c5a..41308ff8f 100644 --- a/src/nix-eval-jobs.cc +++ b/src/nix-eval-jobs.cc @@ -392,7 +392,8 @@ struct Proc { std::make_shared(std::move(toPipe.readSide))}]() { debug("created worker process %d", getpid()); try { - EvalState state(myArgs.searchPath, openStore()); + EvalState state(myArgs.searchPath, + openStore(*myArgs.evalStoreUrl)); Bindings &autoArgs = *myArgs.getAutoArgs(state); proc(state, autoArgs, *to, *from); } catch (Error &e) { From ea18d590e5ee391bb67eb2df41b2ac3a275d1ac5 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 20 Oct 2022 02:07:07 +0000 Subject: [PATCH 195/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/83b198a2083774844962c854f811538323f9f7b1' (2022-10-15) → 'github:NixOS/nixpkgs/32096899af23d49010bd8cf6a91695888d9d9e73' (2022-10-18) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 795cc10d7..4a3a4aa9f 100644 --- a/flake.lock +++ b/flake.lock @@ -17,11 +17,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1665848363, - "narHash": "sha256-3Jow1YxzPtQnck1bAAvbVxgRH4gNnkIdw871Vm6UtAU=", + "lastModified": 1666109165, + "narHash": "sha256-BMLyNVkr0oONuq3lKlFCRVuYqF75CO68Z8EoCh81Zdk=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "83b198a2083774844962c854f811538323f9f7b1", + "rev": "32096899af23d49010bd8cf6a91695888d9d9e73", "type": "github" }, "original": { From d09b1e7593acbe4095e7fcc385f69e94f63a0e55 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 24 Oct 2022 02:24:50 +0000 Subject: [PATCH 196/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/32096899af23d49010bd8cf6a91695888d9d9e73' (2022-10-18) → 'github:NixOS/nixpkgs/95aeaf83c247b8f5aa561684317ecd860476fcd6' (2022-10-22) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 4a3a4aa9f..42757a384 100644 --- a/flake.lock +++ b/flake.lock @@ -17,11 +17,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1666109165, - "narHash": "sha256-BMLyNVkr0oONuq3lKlFCRVuYqF75CO68Z8EoCh81Zdk=", + "lastModified": 1666447894, + "narHash": "sha256-i9WHX4w/et4qPMzEXd9POmnO0/bthjr7R4cblKNHGms=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "32096899af23d49010bd8cf6a91695888d9d9e73", + "rev": "95aeaf83c247b8f5aa561684317ecd860476fcd6", "type": "github" }, "original": { From 11f1ddc40573f4cff6bb60d59976df2f2d2a306a Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 27 Oct 2022 01:52:17 +0000 Subject: [PATCH 197/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/95aeaf83c247b8f5aa561684317ecd860476fcd6' (2022-10-22) → 'github:NixOS/nixpkgs/f994293d1eb8812f032e8919e10a594567cf6ef7' (2022-10-25) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 42757a384..ed7e51a8e 100644 --- a/flake.lock +++ b/flake.lock @@ -17,11 +17,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1666447894, - "narHash": "sha256-i9WHX4w/et4qPMzEXd9POmnO0/bthjr7R4cblKNHGms=", + "lastModified": 1666703756, + "narHash": "sha256-GwpMJ1hT+z1fMAUkaGtvbvofJQwdVFDEGVhfE82+AUk=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "95aeaf83c247b8f5aa561684317ecd860476fcd6", + "rev": "f994293d1eb8812f032e8919e10a594567cf6ef7", "type": "github" }, "original": { From f8b3aa4449d943f7b0e4e69e5b9efa6b660ae5b0 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 31 Oct 2022 02:05:04 +0000 Subject: [PATCH 198/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'flake-utils': 'github:numtide/flake-utils/c0e246b9b83f637f4681389ecabcb2681b4f3af0' (2022-08-07) → 'github:numtide/flake-utils/6ee9ebb6b1ee695d2cacc4faa053a7b9baa76817' (2022-10-29) • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/f994293d1eb8812f032e8919e10a594567cf6ef7' (2022-10-25) → 'github:NixOS/nixpkgs/fdebb81f45a1ba2c4afca5fd9f526e1653ad0949' (2022-10-29) --- flake.lock | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/flake.lock b/flake.lock index ed7e51a8e..6575f1523 100644 --- a/flake.lock +++ b/flake.lock @@ -2,11 +2,11 @@ "nodes": { "flake-utils": { "locked": { - "lastModified": 1659877975, - "narHash": "sha256-zllb8aq3YO3h8B/U0/J1WBgAL8EX5yWf5pMj3G0NAmc=", + "lastModified": 1667077288, + "narHash": "sha256-bdC8sFNDpT0HK74u9fUkpbf1MEzVYJ+ka7NXCdgBoaA=", "owner": "numtide", "repo": "flake-utils", - "rev": "c0e246b9b83f637f4681389ecabcb2681b4f3af0", + "rev": "6ee9ebb6b1ee695d2cacc4faa053a7b9baa76817", "type": "github" }, "original": { @@ -17,11 +17,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1666703756, - "narHash": "sha256-GwpMJ1hT+z1fMAUkaGtvbvofJQwdVFDEGVhfE82+AUk=", + "lastModified": 1667050928, + "narHash": "sha256-xOn0ZgjImIyeecEsrjxuvlW7IW5genTwvvnDQRFncB8=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "f994293d1eb8812f032e8919e10a594567cf6ef7", + "rev": "fdebb81f45a1ba2c4afca5fd9f526e1653ad0949", "type": "github" }, "original": { From ad9c0e0b7071b806d279e3f29382d73ab1067df3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Mon, 31 Oct 2022 07:09:44 +0100 Subject: [PATCH 199/419] flake.nix: bump version --- default.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/default.nix b/default.nix index 1178d04fb..a60ed77d2 100644 --- a/default.nix +++ b/default.nix @@ -17,7 +17,7 @@ let in stdenv.mkDerivation rec { pname = "nix-eval-jobs"; - version = "0.0.1"; + version = "2.11.0"; src = if srcDir == null then filterMesonBuild ./. else srcDir; buildInputs = [ nlohmann_json From 724c8df41911832bd6f9dc50a13a68637a1ec55c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Mon, 31 Oct 2022 07:14:24 +0100 Subject: [PATCH 200/419] also trigger ci on release branches --- .github/workflows/flake-check.yml | 1 + .github/workflows/test-develop-flakes.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/flake-check.yml b/.github/workflows/flake-check.yml index 98303a577..3194ccbd0 100644 --- a/.github/workflows/flake-check.yml +++ b/.github/workflows/flake-check.yml @@ -4,6 +4,7 @@ on: push: branches: - main + - release-* jobs: flake-checks: diff --git a/.github/workflows/test-develop-flakes.yml b/.github/workflows/test-develop-flakes.yml index cc33cfcea..080f0a03e 100644 --- a/.github/workflows/test-develop-flakes.yml +++ b/.github/workflows/test-develop-flakes.yml @@ -4,6 +4,7 @@ on: push: branches: - main + - release-* jobs: tests: strategy: From be849ae99f9c771707e6f60fc36257752cfe72f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Mon, 31 Oct 2022 07:22:17 +0100 Subject: [PATCH 201/419] create bors.toml --- bors.toml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 bors.toml diff --git a/bors.toml b/bors.toml new file mode 100644 index 000000000..4a9f23eb0 --- /dev/null +++ b/bors.toml @@ -0,0 +1,6 @@ +cut_body_after = "" # don't include text from the PR body in the merge commit message +status = [ + "flake-checks", + "builds (treefmt, ubuntu-latest)", + "builds (treefmt, macos-latest)" +] From 6a066bc2b5afd0576053d5828d2fe081314d7e11 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 3 Nov 2022 01:47:46 +0000 Subject: [PATCH 202/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'flake-utils': 'github:numtide/flake-utils/6ee9ebb6b1ee695d2cacc4faa053a7b9baa76817' (2022-10-29) → 'github:numtide/flake-utils/5aed5285a952e0b949eb3ba02c12fa4fcfef535f' (2022-11-02) • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/fdebb81f45a1ba2c4afca5fd9f526e1653ad0949' (2022-10-29) → 'github:NixOS/nixpkgs/d40fea9aeb8840fea0d377baa4b38e39b9582458' (2022-10-31) --- flake.lock | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/flake.lock b/flake.lock index 6575f1523..d877efefe 100644 --- a/flake.lock +++ b/flake.lock @@ -2,11 +2,11 @@ "nodes": { "flake-utils": { "locked": { - "lastModified": 1667077288, - "narHash": "sha256-bdC8sFNDpT0HK74u9fUkpbf1MEzVYJ+ka7NXCdgBoaA=", + "lastModified": 1667395993, + "narHash": "sha256-nuEHfE/LcWyuSWnS8t12N1wc105Qtau+/OdUAjtQ0rA=", "owner": "numtide", "repo": "flake-utils", - "rev": "6ee9ebb6b1ee695d2cacc4faa053a7b9baa76817", + "rev": "5aed5285a952e0b949eb3ba02c12fa4fcfef535f", "type": "github" }, "original": { @@ -17,11 +17,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1667050928, - "narHash": "sha256-xOn0ZgjImIyeecEsrjxuvlW7IW5genTwvvnDQRFncB8=", + "lastModified": 1667231093, + "narHash": "sha256-RERXruzBEBuf0c7OfZeX1hxEKB+PTCUNxWeB6C1jd8Y=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "fdebb81f45a1ba2c4afca5fd9f526e1653ad0949", + "rev": "d40fea9aeb8840fea0d377baa4b38e39b9582458", "type": "github" }, "original": { From c8b52f58363b1e399cfc225a435c2445e92718d5 Mon Sep 17 00:00:00 2001 From: Timothy DeHerrera Date: Fri, 4 Nov 2022 09:10:54 -0600 Subject: [PATCH 203/419] fix: use `InstallableFlake` type & methods Fixes #134 Use the `InstallableFlake` type in order to make use of it's `toValue` method. This fixes the functor auto-call by including the work from nixos/nix#6404. Future work may make use of this object and its methods to employ the flake based eval cache. --- src/nix-eval-jobs.cc | 62 ++++++++++++++++---------------------------- 1 file changed, 23 insertions(+), 39 deletions(-) diff --git a/src/nix-eval-jobs.cc b/src/nix-eval-jobs.cc index 12ad152a1..f6dacb50c 100644 --- a/src/nix-eval-jobs.cc +++ b/src/nix-eval-jobs.cc @@ -19,6 +19,7 @@ #include #include #include +#include #include @@ -148,44 +149,10 @@ static Value *releaseExprTopLevelValue(EvalState &state, Bindings &autoArgs) { return vRoot; } -static Value *flakeTopLevelValue(EvalState &state, Bindings &autoArgs) { - using namespace flake; - - auto [flakeRef, fragment] = - parseFlakeRefWithFragment(myArgs.releaseExpr, absPath(".")); - - auto vFlake = state.allocValue(); - - auto lockedFlake = lockFlake(state, flakeRef, - LockFlags{ - .updateLockFile = false, - .useRegistries = false, - .allowMutable = false, - }); - - callFlake(state, lockedFlake, *vFlake); - - auto vOutputs = vFlake->attrs->get(state.symbols.create("outputs"))->value; - state.forceValue(*vOutputs, noPos); - auto vTop = *vOutputs; - - if (fragment.length() > 0) { - Bindings &bindings(*state.allocBindings(0)); - auto [nTop, pos] = findAlongAttrPath(state, fragment, bindings, vTop); - if (!nTop) - throw Error("error: attribute '%s' missing", nTop); - vTop = *nTop; - } - - auto vRoot = state.allocValue(); - state.autoCallFunction(autoArgs, vTop, *vRoot); - - return vRoot; -} - -Value *topLevelValue(EvalState &state, Bindings &autoArgs) { - return myArgs.flake ? flakeTopLevelValue(state, autoArgs) - : releaseExprTopLevelValue(state, autoArgs); +Value *topLevelValue(EvalState &state, Bindings &autoArgs, + std::optional flake) { + return flake.has_value() ? flake.value().toValue(state).first + : releaseExprTopLevelValue(state, autoArgs); } bool queryIsCached(Store &store, std::map &outputs) { @@ -279,7 +246,24 @@ std::string attrPathJoin(json input) { static void worker(EvalState &state, Bindings &autoArgs, AutoCloseFD &to, AutoCloseFD &from) { - auto vRoot = topLevelValue(state, autoArgs); + + std::optional flake; + if (myArgs.flake) { + auto [flakeRef, fragment, outputSpec] = + parseFlakeRefWithFragmentAndOutputsSpec(myArgs.releaseExpr, + absPath(".")); + + flake.emplace(InstallableFlake({}, ref(&state), + std::move(flakeRef), fragment, + outputSpec, {}, {}, + flake::LockFlags{ + .updateLockFile = false, + .useRegistries = false, + .allowMutable = false, + })); + }; + + auto vRoot = topLevelValue(state, autoArgs, flake); while (true) { /* Wait for the collector to send us a job name. */ From 69815f48be89f92c683fd1f37a9ccbb9cb2f2373 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 7 Nov 2022 01:43:55 +0000 Subject: [PATCH 204/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/d40fea9aeb8840fea0d377baa4b38e39b9582458' (2022-10-31) → 'github:NixOS/nixpkgs/3bacde6273b09a21a8ccfba15586fb165078fb62' (2022-11-05) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index d877efefe..4f2ce58ef 100644 --- a/flake.lock +++ b/flake.lock @@ -17,11 +17,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1667231093, - "narHash": "sha256-RERXruzBEBuf0c7OfZeX1hxEKB+PTCUNxWeB6C1jd8Y=", + "lastModified": 1667629849, + "narHash": "sha256-P+v+nDOFWicM4wziFK9S/ajF2lc0N2Rg9p6Y35uMoZI=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "d40fea9aeb8840fea0d377baa4b38e39b9582458", + "rev": "3bacde6273b09a21a8ccfba15586fb165078fb62", "type": "github" }, "original": { From 2b3844be65f13bd7fa782717b9ffa936b527b512 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 10 Nov 2022 01:47:33 +0000 Subject: [PATCH 205/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/3bacde6273b09a21a8ccfba15586fb165078fb62' (2022-11-05) → 'github:NixOS/nixpkgs/093268502280540a7f5bf1e2a6330a598ba3b7d0' (2022-11-08) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 4f2ce58ef..c0cfd35ab 100644 --- a/flake.lock +++ b/flake.lock @@ -17,11 +17,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1667629849, - "narHash": "sha256-P+v+nDOFWicM4wziFK9S/ajF2lc0N2Rg9p6Y35uMoZI=", + "lastModified": 1667901915, + "narHash": "sha256-IkSou5ox/yZ2YUhGpk8vxd2TNU2pwRlYtir5k55NaxE=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "3bacde6273b09a21a8ccfba15586fb165078fb62", + "rev": "093268502280540a7f5bf1e2a6330a598ba3b7d0", "type": "github" }, "original": { From b5b6c8cf266941641c55c5d77d38dd3c23adfff3 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 14 Nov 2022 01:42:28 +0000 Subject: [PATCH 206/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/093268502280540a7f5bf1e2a6330a598ba3b7d0' (2022-11-08) → 'github:NixOS/nixpkgs/5f588eb4a958f1a526ed8da02d6ea1bea0047b9f' (2022-11-10) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index c0cfd35ab..433811796 100644 --- a/flake.lock +++ b/flake.lock @@ -17,11 +17,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1667901915, - "narHash": "sha256-IkSou5ox/yZ2YUhGpk8vxd2TNU2pwRlYtir5k55NaxE=", + "lastModified": 1668087632, + "narHash": "sha256-T/cUx44aYDuLMFfaiVpMdTjL4kpG7bh0VkN6JEM78/E=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "093268502280540a7f5bf1e2a6330a598ba3b7d0", + "rev": "5f588eb4a958f1a526ed8da02d6ea1bea0047b9f", "type": "github" }, "original": { From a7bff0a0d66b70b9d27c9e07a0e1f3afaba3c7a9 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 17 Nov 2022 01:39:58 +0000 Subject: [PATCH 207/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/5f588eb4a958f1a526ed8da02d6ea1bea0047b9f' (2022-11-10) → 'github:NixOS/nixpkgs/85d6b3990def7eef45f4502a82496de02a02b6e8' (2022-11-15) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 433811796..864f0d88b 100644 --- a/flake.lock +++ b/flake.lock @@ -17,11 +17,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1668087632, - "narHash": "sha256-T/cUx44aYDuLMFfaiVpMdTjL4kpG7bh0VkN6JEM78/E=", + "lastModified": 1668505710, + "narHash": "sha256-DulcfsGjpSXL9Ma0iQIsb3HRbARCDcA+CNH67pPyMQ0=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "5f588eb4a958f1a526ed8da02d6ea1bea0047b9f", + "rev": "85d6b3990def7eef45f4502a82496de02a02b6e8", "type": "github" }, "original": { From 63775e6f7a1f2633b2d204750e717689d1188698 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 21 Nov 2022 01:40:57 +0000 Subject: [PATCH 208/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/85d6b3990def7eef45f4502a82496de02a02b6e8' (2022-11-15) → 'github:NixOS/nixpkgs/690ffff026b4e635b46f69002c0f4e81c65dfc2e' (2022-11-20) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 864f0d88b..b57ea8946 100644 --- a/flake.lock +++ b/flake.lock @@ -17,11 +17,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1668505710, - "narHash": "sha256-DulcfsGjpSXL9Ma0iQIsb3HRbARCDcA+CNH67pPyMQ0=", + "lastModified": 1668905981, + "narHash": "sha256-RBQa/+9Uk1eFTqIOXBSBezlEbA3v5OkgP+qptQs1OxY=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "85d6b3990def7eef45f4502a82496de02a02b6e8", + "rev": "690ffff026b4e635b46f69002c0f4e81c65dfc2e", "type": "github" }, "original": { From 0eb77bb9c6c3db0951860d00bb43e5bcf207ae62 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 24 Nov 2022 01:35:02 +0000 Subject: [PATCH 209/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/690ffff026b4e635b46f69002c0f4e81c65dfc2e' (2022-11-20) → 'github:NixOS/nixpkgs/2788904d26dda6cfa1921c5abb7a2466ffe3cb8c' (2022-11-22) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index b57ea8946..ec3c2152a 100644 --- a/flake.lock +++ b/flake.lock @@ -17,11 +17,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1668905981, - "narHash": "sha256-RBQa/+9Uk1eFTqIOXBSBezlEbA3v5OkgP+qptQs1OxY=", + "lastModified": 1669140675, + "narHash": "sha256-npzfyfLECsJWgzK/M4gWhykP2DNAJTYjgY2BWkz/oEQ=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "690ffff026b4e635b46f69002c0f4e81c65dfc2e", + "rev": "2788904d26dda6cfa1921c5abb7a2466ffe3cb8c", "type": "github" }, "original": { From 7f98fdf6526ad87478aa61d063b602de7b4f5e21 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 28 Nov 2022 01:21:40 +0000 Subject: [PATCH 210/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/2788904d26dda6cfa1921c5abb7a2466ffe3cb8c' (2022-11-22) → 'github:NixOS/nixpkgs/5dc7114b7b256d217fe7752f1614be2514e61bb8' (2022-11-25) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index ec3c2152a..309604f23 100644 --- a/flake.lock +++ b/flake.lock @@ -17,11 +17,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1669140675, - "narHash": "sha256-npzfyfLECsJWgzK/M4gWhykP2DNAJTYjgY2BWkz/oEQ=", + "lastModified": 1669411043, + "narHash": "sha256-LfPd3+EY+jaIHTRIEOUtHXuanxm59YKgUacmSzaqMLc=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "2788904d26dda6cfa1921c5abb7a2466ffe3cb8c", + "rev": "5dc7114b7b256d217fe7752f1614be2514e61bb8", "type": "github" }, "original": { From 1f36338caf79ac2dd45ce824971dabe9666393e8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 28 Nov 2022 23:01:47 +0000 Subject: [PATCH 211/419] Bump DeterminateSystems/update-flake-lock from 14 to 15 Bumps [DeterminateSystems/update-flake-lock](https://github.com/DeterminateSystems/update-flake-lock) from 14 to 15. - [Release notes](https://github.com/DeterminateSystems/update-flake-lock/releases) - [Commits](https://github.com/DeterminateSystems/update-flake-lock/compare/v14...v15) --- updated-dependencies: - dependency-name: DeterminateSystems/update-flake-lock dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/update-flake-lock.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/update-flake-lock.yml b/.github/workflows/update-flake-lock.yml index f7ab7b1e8..d2c6bf249 100644 --- a/.github/workflows/update-flake-lock.yml +++ b/.github/workflows/update-flake-lock.yml @@ -16,6 +16,6 @@ jobs: # extra_nix_config: | # access-tokens = github.com=${{ secrets.GITHUB_TOKEN }} - name: Update flake.lock - uses: DeterminateSystems/update-flake-lock@v14 + uses: DeterminateSystems/update-flake-lock@v15 with: token: ${{ secrets.GH_TOKEN_FOR_UPDATES }} From a965933bf655785ae273ac89a2a9f359f7b332b2 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 1 Dec 2022 01:41:02 +0000 Subject: [PATCH 212/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/5dc7114b7b256d217fe7752f1614be2514e61bb8' (2022-11-25) → 'github:NixOS/nixpkgs/a115bb9bd56831941be3776c8a94005867f316a7' (2022-11-27) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 309604f23..44903ae82 100644 --- a/flake.lock +++ b/flake.lock @@ -17,11 +17,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1669411043, - "narHash": "sha256-LfPd3+EY+jaIHTRIEOUtHXuanxm59YKgUacmSzaqMLc=", + "lastModified": 1669542132, + "narHash": "sha256-DRlg++NJAwPh8io3ExBJdNW7Djs3plVI5jgYQ+iXAZQ=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "5dc7114b7b256d217fe7752f1614be2514e61bb8", + "rev": "a115bb9bd56831941be3776c8a94005867f316a7", "type": "github" }, "original": { From 2b806bfaef2d36e44867c30f247913bd2a685340 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 5 Dec 2022 01:17:13 +0000 Subject: [PATCH 213/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/a115bb9bd56831941be3776c8a94005867f316a7' (2022-11-27) → 'github:NixOS/nixpkgs/61a8a98e6d557e6dd7ed0cdb54c3a3e3bbc5e25c' (2022-12-03) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 44903ae82..904b3f7fb 100644 --- a/flake.lock +++ b/flake.lock @@ -17,11 +17,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1669542132, - "narHash": "sha256-DRlg++NJAwPh8io3ExBJdNW7Djs3plVI5jgYQ+iXAZQ=", + "lastModified": 1670064435, + "narHash": "sha256-+ELoY30UN+Pl3Yn7RWRPabykwebsVK/kYE9JsIsUMxQ=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "a115bb9bd56831941be3776c8a94005867f316a7", + "rev": "61a8a98e6d557e6dd7ed0cdb54c3a3e3bbc5e25c", "type": "github" }, "original": { From f4eda9f598d89a4291f93a8f3b8636be93ca316a Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 8 Dec 2022 01:18:19 +0000 Subject: [PATCH 214/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/61a8a98e6d557e6dd7ed0cdb54c3a3e3bbc5e25c' (2022-12-03) → 'github:NixOS/nixpkgs/6e51c97f1c849efdfd4f3b78a4870e6aa2da4198' (2022-12-05) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 904b3f7fb..695e33ef4 100644 --- a/flake.lock +++ b/flake.lock @@ -17,11 +17,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1670064435, - "narHash": "sha256-+ELoY30UN+Pl3Yn7RWRPabykwebsVK/kYE9JsIsUMxQ=", + "lastModified": 1670242877, + "narHash": "sha256-jBLh7dRHnbfvPPA9znOC6oQfKrCPJ0El8Zoe0BqnCjQ=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "61a8a98e6d557e6dd7ed0cdb54c3a3e3bbc5e25c", + "rev": "6e51c97f1c849efdfd4f3b78a4870e6aa2da4198", "type": "github" }, "original": { From 68c4f681e12dfd227a3ba7ff4a1fa9b9fdacee32 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 12 Dec 2022 01:19:52 +0000 Subject: [PATCH 215/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/6e51c97f1c849efdfd4f3b78a4870e6aa2da4198' (2022-12-05) → 'github:NixOS/nixpkgs/2dea0f4c2d6e4603f54b2c56c22367e77869490c' (2022-12-09) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 695e33ef4..40c76b719 100644 --- a/flake.lock +++ b/flake.lock @@ -17,11 +17,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1670242877, - "narHash": "sha256-jBLh7dRHnbfvPPA9znOC6oQfKrCPJ0El8Zoe0BqnCjQ=", + "lastModified": 1670597555, + "narHash": "sha256-/k939P2S2246G6K5fyvC0U2IWvULhb4ZJg9K7ZxsX+k=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "6e51c97f1c849efdfd4f3b78a4870e6aa2da4198", + "rev": "2dea0f4c2d6e4603f54b2c56c22367e77869490c", "type": "github" }, "original": { From 908da989418dfdebf0f50dd3aa6688f4033751e0 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 15 Dec 2022 01:20:24 +0000 Subject: [PATCH 216/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/2dea0f4c2d6e4603f54b2c56c22367e77869490c' (2022-12-09) → 'github:NixOS/nixpkgs/1710ed1f6f8ceb75cf7d1cf55ee0cc21760e1c7a' (2022-12-13) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 40c76b719..89eac9e8d 100644 --- a/flake.lock +++ b/flake.lock @@ -17,11 +17,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1670597555, - "narHash": "sha256-/k939P2S2246G6K5fyvC0U2IWvULhb4ZJg9K7ZxsX+k=", + "lastModified": 1670929434, + "narHash": "sha256-n5UBO6XBV4h3TB7FYu2yAuNQMEYOrQyKeODUwKe06ow=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "2dea0f4c2d6e4603f54b2c56c22367e77869490c", + "rev": "1710ed1f6f8ceb75cf7d1cf55ee0cc21760e1c7a", "type": "github" }, "original": { From f5231548a74c5b10c5ed099a31d356418ca19589 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 19 Dec 2022 01:12:20 +0000 Subject: [PATCH 217/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/1710ed1f6f8ceb75cf7d1cf55ee0cc21760e1c7a' (2022-12-13) → 'github:NixOS/nixpkgs/40f79f003b6377bd2f4ed4027dde1f8f922995dd' (2022-12-17) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 89eac9e8d..748e94c48 100644 --- a/flake.lock +++ b/flake.lock @@ -17,11 +17,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1670929434, - "narHash": "sha256-n5UBO6XBV4h3TB7FYu2yAuNQMEYOrQyKeODUwKe06ow=", + "lastModified": 1671271357, + "narHash": "sha256-xRJdLbWK4v2SewmSStYrcLa0YGJpleufl44A19XSW8k=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "1710ed1f6f8ceb75cf7d1cf55ee0cc21760e1c7a", + "rev": "40f79f003b6377bd2f4ed4027dde1f8f922995dd", "type": "github" }, "original": { From 1aecf35f5acfd401383c87f5f67411bd9f5859d1 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 22 Dec 2022 01:14:55 +0000 Subject: [PATCH 218/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/40f79f003b6377bd2f4ed4027dde1f8f922995dd' (2022-12-17) → 'github:NixOS/nixpkgs/04f574a1c0fde90b51bf68198e2297ca4e7cccf4' (2022-12-18) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 748e94c48..248cf0b65 100644 --- a/flake.lock +++ b/flake.lock @@ -17,11 +17,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1671271357, - "narHash": "sha256-xRJdLbWK4v2SewmSStYrcLa0YGJpleufl44A19XSW8k=", + "lastModified": 1671359686, + "narHash": "sha256-3MpC6yZo+Xn9cPordGz2/ii6IJpP2n8LE8e/ebUXLrs=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "40f79f003b6377bd2f4ed4027dde1f8f922995dd", + "rev": "04f574a1c0fde90b51bf68198e2297ca4e7cccf4", "type": "github" }, "original": { From df1fa78289c0cf673a84525acc89086a906f1b50 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 29 Dec 2022 01:16:02 +0000 Subject: [PATCH 219/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/04f574a1c0fde90b51bf68198e2297ca4e7cccf4' (2022-12-18) → 'github:NixOS/nixpkgs/1eb875e811dd59e21e77f6337f2c1592889b48b3' (2022-12-26) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 248cf0b65..b5ecf00fc 100644 --- a/flake.lock +++ b/flake.lock @@ -17,11 +17,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1671359686, - "narHash": "sha256-3MpC6yZo+Xn9cPordGz2/ii6IJpP2n8LE8e/ebUXLrs=", + "lastModified": 1672080458, + "narHash": "sha256-Ukjn8YUwZevxDPaVUmTx2sf9bCcIJSasmLz+xjGBKrs=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "04f574a1c0fde90b51bf68198e2297ca4e7cccf4", + "rev": "1eb875e811dd59e21e77f6337f2c1592889b48b3", "type": "github" }, "original": { From 7110a1a6c74f11cb3205e0834a2b2f1ae52f648d Mon Sep 17 00:00:00 2001 From: Tom Bereknyei Date: Sun, 1 Jan 2023 23:20:13 -0500 Subject: [PATCH 220/419] fix: catch errors for invalid derivations --- src/nix-eval-jobs.cc | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/nix-eval-jobs.cc b/src/nix-eval-jobs.cc index f6dacb50c..b3e00b175 100644 --- a/src/nix-eval-jobs.cc +++ b/src/nix-eval-jobs.cc @@ -184,9 +184,14 @@ struct Drv { auto localStore = state.store.dynamic_pointer_cast(); - for (auto out : drvInfo.queryOutputs(true)) { - if (out.second) - outputs[out.first] = localStore->printStorePath(*out.second); + try { + for (auto out : drvInfo.queryOutputs(true)) { + if (out.second) + outputs[out.first] = + localStore->printStorePath(*out.second); + } + } catch (const std::exception &e) { + throw EvalError("derivation must have valid outputs: %s", e.what()); } if (myArgs.meta) { From e6392759be222d8bb69f46ca6817d7c306aab177 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Mon, 2 Jan 2023 11:09:55 +0100 Subject: [PATCH 221/419] ci/update-flake-lock: provide access to github token --- .github/workflows/update-flake-lock.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/update-flake-lock.yml b/.github/workflows/update-flake-lock.yml index d2c6bf249..6aefd749a 100644 --- a/.github/workflows/update-flake-lock.yml +++ b/.github/workflows/update-flake-lock.yml @@ -12,9 +12,9 @@ jobs: uses: actions/checkout@v3 - name: Install Nix uses: cachix/install-nix-action@v18 - # with: - # extra_nix_config: | - # access-tokens = github.com=${{ secrets.GITHUB_TOKEN }} + with: + extra_nix_config: | + access-tokens = github.com=${{ secrets.GITHUB_TOKEN }} - name: Update flake.lock uses: DeterminateSystems/update-flake-lock@v15 with: From c1a77354cb895db3649ce4fe63c7c2b4c9013b48 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 2 Jan 2023 10:10:54 +0000 Subject: [PATCH 222/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/1eb875e811dd59e21e77f6337f2c1592889b48b3' (2022-12-26) → 'github:NixOS/nixpkgs/677ed08a50931e38382dbef01cba08a8f7eac8f6' (2022-12-29) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index b5ecf00fc..3cbf13e59 100644 --- a/flake.lock +++ b/flake.lock @@ -17,11 +17,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1672080458, - "narHash": "sha256-Ukjn8YUwZevxDPaVUmTx2sf9bCcIJSasmLz+xjGBKrs=", + "lastModified": 1672350804, + "narHash": "sha256-jo6zkiCabUBn3ObuKXHGqqORUMH27gYDIFFfLq5P4wg=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "1eb875e811dd59e21e77f6337f2c1592889b48b3", + "rev": "677ed08a50931e38382dbef01cba08a8f7eac8f6", "type": "github" }, "original": { From 9d1abc0bc8f342de08eb4092da3ef5e482d20a33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Mon, 2 Jan 2023 11:19:32 +0100 Subject: [PATCH 223/419] switch to flake-parts from flake-utils --- flake.lock | 23 +++++++++++------- flake.nix | 69 +++++++++++++++++++++++++++--------------------------- 2 files changed, 49 insertions(+), 43 deletions(-) diff --git a/flake.lock b/flake.lock index 3cbf13e59..a2240e7c0 100644 --- a/flake.lock +++ b/flake.lock @@ -1,17 +1,22 @@ { "nodes": { - "flake-utils": { + "flake-parts": { + "inputs": { + "nixpkgs-lib": [ + "nixpkgs" + ] + }, "locked": { - "lastModified": 1667395993, - "narHash": "sha256-nuEHfE/LcWyuSWnS8t12N1wc105Qtau+/OdUAjtQ0rA=", - "owner": "numtide", - "repo": "flake-utils", - "rev": "5aed5285a952e0b949eb3ba02c12fa4fcfef535f", + "lastModified": 1672616755, + "narHash": "sha256-dvwU2ORLpiP6ZMXL3CJ/qrqmtLBLF6VAc+Fois7Qfew=", + "owner": "hercules-ci", + "repo": "flake-parts", + "rev": "87673d7c13a799d95ce25ff5dc7b9e15f01af2ea", "type": "github" }, "original": { - "owner": "numtide", - "repo": "flake-utils", + "owner": "hercules-ci", + "repo": "flake-parts", "type": "github" } }, @@ -33,7 +38,7 @@ }, "root": { "inputs": { - "flake-utils": "flake-utils", + "flake-parts": "flake-parts", "nixpkgs": "nixpkgs" } } diff --git a/flake.nix b/flake.nix index 971dd6e38..c934680a7 100644 --- a/flake.nix +++ b/flake.nix @@ -2,45 +2,46 @@ description = "Hydra's builtin hydra-eval-jobs as a standalone"; inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; - inputs.flake-utils.url = "github:numtide/flake-utils"; + inputs.flake-parts.url = "github:hercules-ci/flake-parts"; + inputs.flake-parts.inputs.nixpkgs-lib.follows = "nixpkgs"; - outputs = - { self - , nixpkgs - , flake-utils - }: - flake-utils.lib.eachDefaultSystem ( - system: - let - nixVersion = nixpkgs.lib.fileContents ./.nix-version; - pkgs = nixpkgs.legacyPackages.${system}; - inherit (pkgs) stdenv; - devShell = self.devShells.${system}.default; - drvArgs = { - srcDir = self; - nix = if nixVersion == "unstable" then pkgs.nixUnstable else pkgs.nixVersions."nix_${nixVersion}"; - }; - in + outputs = inputs @ { flake-parts, ... }: + let + inherit (inputs.nixpkgs) lib; + inherit (inputs) self; + nixVersion = lib.fileContents ./.nix-version; + in + flake-parts.lib.mkFlake { inherit inputs; } { - packages.nix-eval-jobs = pkgs.callPackage ./default.nix drvArgs; + systems = inputs.nixpkgs.lib.systems.flakeExposed; + perSystem = { pkgs, self', ... }: + let + devShell = self'.devShells.default; + drvArgs = { + srcDir = self; + nix = if nixVersion == "unstable" then pkgs.nixUnstable else pkgs.nixVersions."nix_${nixVersion}"; + }; + in + { + packages.nix-eval-jobs = pkgs.callPackage ./default.nix drvArgs; - checks.treefmt = stdenv.mkDerivation { - name = "treefmt-check"; - src = self; - nativeBuildInputs = devShell.nativeBuildInputs; - dontConfigure = true; + checks.treefmt = pkgs.stdenv.mkDerivation { + name = "treefmt-check"; + src = self; + nativeBuildInputs = devShell.nativeBuildInputs; + dontConfigure = true; - inherit (devShell) NODE_PATH; + inherit (devShell) NODE_PATH; - buildPhase = '' - env HOME=$(mktemp -d) treefmt --fail-on-change - ''; + buildPhase = '' + env HOME=$(mktemp -d) treefmt --fail-on-change + ''; - installPhase = "touch $out"; - }; + installPhase = "touch $out"; + }; - packages.default = self.packages.${system}.nix-eval-jobs; - devShells.default = pkgs.callPackage ./shell.nix drvArgs; - } - ); + packages.default = self'.packages.nix-eval-jobs; + devShells.default = pkgs.callPackage ./shell.nix drvArgs; + }; + }; } From 0d5f97d6171d882749fc2f86095675c4f5ebd023 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Mon, 2 Jan 2023 11:31:28 +0100 Subject: [PATCH 224/419] switch to garnix --- .github/workflows/flake-check.yml | 46 ------------------- .../{test-develop-flakes.yml => tests.yml} | 6 ++- flake.nix | 7 +++ 3 files changed, 12 insertions(+), 47 deletions(-) delete mode 100644 .github/workflows/flake-check.yml rename .github/workflows/{test-develop-flakes.yml => tests.yml} (77%) diff --git a/.github/workflows/flake-check.yml b/.github/workflows/flake-check.yml deleted file mode 100644 index 3194ccbd0..000000000 --- a/.github/workflows/flake-check.yml +++ /dev/null @@ -1,46 +0,0 @@ -name: "Flake checks" -on: - pull_request: - push: - branches: - - main - - release-* -jobs: - - flake-checks: - runs-on: ubuntu-latest - outputs: - matrix: ${{ steps.set-matrix.outputs.matrix }} - steps: - - uses: actions/checkout@v3 - with: - # Nix Flakes doesn't work on shallow clones - fetch-depth: 0 - - uses: cachix/install-nix-action@v18 - with: - nix_path: nixpkgs=channel:nixos-unstable - - id: set-matrix - run: | - set -euo pipefail - - matrix="$(nix flake show --json | jq '.checks."x86_64-linux" | keys' | jq -rcM '{attr: ., os: ["ubuntu-latest", "macos-latest"]}')" - echo "::set-output name=matrix::$matrix" - - builds: - needs: flake-checks - runs-on: ${{ matrix.os }} - strategy: - matrix: ${{fromJSON(needs.flake-checks.outputs.matrix)}} - steps: - - uses: actions/checkout@v3 - with: - # Nix Flakes doesn't work on shallow clones - fetch-depth: 0 - - uses: cachix/install-nix-action@v18 - with: - nix_path: nixpkgs=channel:nixos-unstable - - run: | - set -euo pipefail - - system=$(nix-instantiate --eval --expr builtins.currentSystem | jq -r) - nix build -L .#checks.$system.${{ matrix.attr }} diff --git a/.github/workflows/test-develop-flakes.yml b/.github/workflows/tests.yml similarity index 77% rename from .github/workflows/test-develop-flakes.yml rename to .github/workflows/tests.yml index 080f0a03e..6702da197 100644 --- a/.github/workflows/test-develop-flakes.yml +++ b/.github/workflows/tests.yml @@ -1,4 +1,4 @@ -name: "Development workflow" +name: "Tests" on: pull_request: push: @@ -17,6 +17,10 @@ jobs: # Nix Flakes doesn't work on shallow clones fetch-depth: 0 - uses: cachix/install-nix-action@v18 + with: + github_access_token: ${{ secrets.GITHUB_TOKEN }} + extra_nix_config: | + accept-flake-config = true - name: Build run: nix develop -c bash -c 'meson build && cd build && ninja' - name: Run tests diff --git a/flake.nix b/flake.nix index c934680a7..9259fabb1 100644 --- a/flake.nix +++ b/flake.nix @@ -5,6 +5,13 @@ inputs.flake-parts.url = "github:hercules-ci/flake-parts"; inputs.flake-parts.inputs.nixpkgs-lib.follows = "nixpkgs"; + nixConfig.extra-substituters = [ + "https://cache.garnix.io" + ]; + nixConfig.extra-trusted-public-keys = [ + "cache.garnix.io:CTFPyKSLcx5RMJKfLo5EEPUObbA78b0YQ2DTCJXqr9g=" + ]; + outputs = inputs @ { flake-parts, ... }: let inherit (inputs.nixpkgs) lib; From 01369215dcecfdc41bebc4d54dff217573abebf5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Mon, 2 Jan 2023 11:21:55 +0100 Subject: [PATCH 225/419] bors: update toml --- bors.toml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/bors.toml b/bors.toml index 4a9f23eb0..c9665f943 100644 --- a/bors.toml +++ b/bors.toml @@ -1,6 +1,2 @@ cut_body_after = "" # don't include text from the PR body in the merge commit message -status = [ - "flake-checks", - "builds (treefmt, ubuntu-latest)", - "builds (treefmt, macos-latest)" -] +status = ["tests (ubuntu-latest)", "tests (macos-latest)"] From d2c00b8c849eb93f103ba07b4a44034bb4d0c933 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Mon, 2 Jan 2023 11:35:39 +0100 Subject: [PATCH 226/419] fix build of devShell --- shell.nix | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/shell.nix b/shell.nix index f64215ea4..a4bda273f 100644 --- a/shell.nix +++ b/shell.nix @@ -16,16 +16,13 @@ let inherit (pkgs) lib stdenv; - + nix-eval-jobs = pkgs.callPackage ./default.nix { + inherit srcDir nix; + }; in -(pkgs.callPackage ./default.nix { - inherit srcDir nix; -}).overrideAttrs (old: { - - src = null; - - nativeBuildInputs = old.nativeBuildInputs ++ [ - +pkgs.mkShell { + inherit (nix-eval-jobs) buildInputs; + nativeBuildInputs = nix-eval-jobs.nativeBuildInputs ++ [ pkgs.treefmt pkgs.llvmPackages.clang # clang-format pkgs.nixpkgs-fmt @@ -37,10 +34,9 @@ in ])) ]; - NODE_PATH = "${pkgs.nodePackages.prettier-plugin-toml}/lib/node_modules"; shellHook = lib.optionalString stdenv.isLinux '' export NIX_DEBUG_INFO_DIRS="${pkgs.curl.debug}/lib/debug:${nix.debug}/lib/debug''${NIX_DEBUG_INFO_DIRS:+:$NIX_DEBUG_INFO_DIRS}" ''; -}) +} From 24db95cbac615ebc84d57520e8d2b3821936fa85 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Mon, 2 Jan 2023 11:42:56 +0100 Subject: [PATCH 227/419] ci: also build github actions in staging --- .github/workflows/tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 6702da197..37a00b020 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -4,6 +4,7 @@ on: push: branches: - main + - staging - release-* jobs: tests: From be12f80cfa54b4021138e9470c8c177c975a67c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Mon, 2 Jan 2023 11:49:45 +0100 Subject: [PATCH 228/419] bors: include garnix in tests --- bors.toml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/bors.toml b/bors.toml index c9665f943..826bb373e 100644 --- a/bors.toml +++ b/bors.toml @@ -1,2 +1,12 @@ cut_body_after = "" # don't include text from the PR body in the merge commit message -status = ["tests (ubuntu-latest)", "tests (macos-latest)"] +status = [ + # garnix + "Evaluate flake.nix", + "package nix-eval-jobs [x86_64-linux]", + "devShell default [x86_64-linux]", + "check treefmt [x86_64-linux]", + "package default [x86_64-linux]", + # github actions + "tests (ubuntu-latest)", + "tests (macos-latest)" +] From a42845708ebaca35d1770b0f906f0e7df884316e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Mon, 2 Jan 2023 11:51:28 +0100 Subject: [PATCH 229/419] add mergify rules for garnix --- .mergify.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.mergify.yml b/.mergify.yml index b50b3b33c..ea18fb2d1 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -1,9 +1,11 @@ pull_request_rules: - name: automatic merge on CI success conditions: - - check-success=flake-checks - - check-success=builds (treefmt, ubuntu-latest) - - check-success=builds (treefmt, macos-latest) + - check-success=Evaluate flake.nix + - check-success=package nix-eval-jobs [x86_64-linux] + - check-success=devShell default [x86_64-linux] + - check-success=check treefmt [x86_64-linux] + - check-success=package default [x86_64-linux] - check-success=tests (ubuntu-latest) - check-success=tests (macos-latest) - author=nix-eval-jobs-bot From 8652ca8fe17e9ab74317235f908a7685c8781448 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 5 Jan 2023 01:18:41 +0000 Subject: [PATCH 230/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'flake-parts': 'github:hercules-ci/flake-parts/87673d7c13a799d95ce25ff5dc7b9e15f01af2ea' (2023-01-01) → 'github:hercules-ci/flake-parts/7930f5b1c356270cec420d4f4cb43f4907206640' (2023-01-05) • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/677ed08a50931e38382dbef01cba08a8f7eac8f6' (2022-12-29) → 'github:NixOS/nixpkgs/0fc9fca9c8d43edd79d33fea0dd8409d7c4580f4' (2023-01-02) --- flake.lock | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/flake.lock b/flake.lock index a2240e7c0..0290f52d2 100644 --- a/flake.lock +++ b/flake.lock @@ -7,11 +7,11 @@ ] }, "locked": { - "lastModified": 1672616755, - "narHash": "sha256-dvwU2ORLpiP6ZMXL3CJ/qrqmtLBLF6VAc+Fois7Qfew=", + "lastModified": 1672877861, + "narHash": "sha256-ROnSmsk5grROL6gnHBnSdqlPPBrBJMApCeB7xzY567M=", "owner": "hercules-ci", "repo": "flake-parts", - "rev": "87673d7c13a799d95ce25ff5dc7b9e15f01af2ea", + "rev": "7930f5b1c356270cec420d4f4cb43f4907206640", "type": "github" }, "original": { @@ -22,11 +22,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1672350804, - "narHash": "sha256-jo6zkiCabUBn3ObuKXHGqqORUMH27gYDIFFfLq5P4wg=", + "lastModified": 1672617983, + "narHash": "sha256-68WDiCBs631mbDDk4UAKdGURKcsfW6hjb7wgudTAe5o=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "677ed08a50931e38382dbef01cba08a8f7eac8f6", + "rev": "0fc9fca9c8d43edd79d33fea0dd8409d7c4580f4", "type": "github" }, "original": { From 2a376f9d2ea99b1c425ecd4a8773ee202cd04d18 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 9 Jan 2023 01:18:10 +0000 Subject: [PATCH 231/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'flake-parts': 'github:hercules-ci/flake-parts/7930f5b1c356270cec420d4f4cb43f4907206640' (2023-01-05) → 'github:hercules-ci/flake-parts/aa1f6ca773b6e740037ebfb35f7010e0c3960638' (2023-01-06) • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/0fc9fca9c8d43edd79d33fea0dd8409d7c4580f4' (2023-01-02) → 'github:NixOS/nixpkgs/a518c77148585023ff56022f09c4b2c418a51ef5' (2023-01-05) --- flake.lock | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/flake.lock b/flake.lock index 0290f52d2..8b0ec04e0 100644 --- a/flake.lock +++ b/flake.lock @@ -7,11 +7,11 @@ ] }, "locked": { - "lastModified": 1672877861, - "narHash": "sha256-ROnSmsk5grROL6gnHBnSdqlPPBrBJMApCeB7xzY567M=", + "lastModified": 1673047662, + "narHash": "sha256-dXYxH/0Ea5oQSkGAWWNy7HzmFutguycDGn2dt6lTYRQ=", "owner": "hercules-ci", "repo": "flake-parts", - "rev": "7930f5b1c356270cec420d4f4cb43f4907206640", + "rev": "aa1f6ca773b6e740037ebfb35f7010e0c3960638", "type": "github" }, "original": { @@ -22,11 +22,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1672617983, - "narHash": "sha256-68WDiCBs631mbDDk4UAKdGURKcsfW6hjb7wgudTAe5o=", + "lastModified": 1672953546, + "narHash": "sha256-oz757DnJ1ITvwyTovuwG3l9cX6j9j6/DH9eH+cXFJmc=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "0fc9fca9c8d43edd79d33fea0dd8409d7c4580f4", + "rev": "a518c77148585023ff56022f09c4b2c418a51ef5", "type": "github" }, "original": { From 5586926cede3e056258c91703b12fd2c76b116a6 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 12 Jan 2023 01:18:30 +0000 Subject: [PATCH 232/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'flake-parts': 'github:hercules-ci/flake-parts/aa1f6ca773b6e740037ebfb35f7010e0c3960638' (2023-01-06) → 'github:hercules-ci/flake-parts/82c16f1682cf50c01cb0280b38a1eed202b3fe9f' (2023-01-10) • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/a518c77148585023ff56022f09c4b2c418a51ef5' (2023-01-05) → 'github:NixOS/nixpkgs/c07552f6f7d4eead7806645ec03f7f1eb71ba6bd' (2023-01-10) --- flake.lock | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/flake.lock b/flake.lock index 8b0ec04e0..85ba318a4 100644 --- a/flake.lock +++ b/flake.lock @@ -7,11 +7,11 @@ ] }, "locked": { - "lastModified": 1673047662, - "narHash": "sha256-dXYxH/0Ea5oQSkGAWWNy7HzmFutguycDGn2dt6lTYRQ=", + "lastModified": 1673362319, + "narHash": "sha256-Pjp45Vnj7S/b3BRpZEVfdu8sqqA6nvVjvYu59okhOyI=", "owner": "hercules-ci", "repo": "flake-parts", - "rev": "aa1f6ca773b6e740037ebfb35f7010e0c3960638", + "rev": "82c16f1682cf50c01cb0280b38a1eed202b3fe9f", "type": "github" }, "original": { @@ -22,11 +22,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1672953546, - "narHash": "sha256-oz757DnJ1ITvwyTovuwG3l9cX6j9j6/DH9eH+cXFJmc=", + "lastModified": 1673315479, + "narHash": "sha256-GNCFRtDHjTygXGJp/H+f2XQPMGxpYSmNiibIqYzihtM=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "a518c77148585023ff56022f09c4b2c418a51ef5", + "rev": "c07552f6f7d4eead7806645ec03f7f1eb71ba6bd", "type": "github" }, "original": { From 230cab57cda93baa6f496b89403701f004d87d6a Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 16 Jan 2023 01:16:55 +0000 Subject: [PATCH 233/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/c07552f6f7d4eead7806645ec03f7f1eb71ba6bd' (2023-01-10) → 'github:NixOS/nixpkgs/befc83905c965adfd33e5cae49acb0351f6e0404' (2023-01-13) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 85ba318a4..66769e6ba 100644 --- a/flake.lock +++ b/flake.lock @@ -22,11 +22,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1673315479, - "narHash": "sha256-GNCFRtDHjTygXGJp/H+f2XQPMGxpYSmNiibIqYzihtM=", + "lastModified": 1673631141, + "narHash": "sha256-AprpYQ5JvLS4wQG/ghm2UriZ9QZXvAwh1HlgA/6ZEVQ=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "c07552f6f7d4eead7806645ec03f7f1eb71ba6bd", + "rev": "befc83905c965adfd33e5cae49acb0351f6e0404", "type": "github" }, "original": { From 9d516a72d7a1fa4e17e0edcaafbf16e10427d490 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sun, 22 Jan 2023 10:28:47 +0100 Subject: [PATCH 234/419] fix double-free caused by usage of shared pointer --- src/nix-eval-jobs.cc | 37 ++++++++++++++++++------------------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/src/nix-eval-jobs.cc b/src/nix-eval-jobs.cc index b3e00b175..4d6c2cd89 100644 --- a/src/nix-eval-jobs.cc +++ b/src/nix-eval-jobs.cc @@ -249,7 +249,7 @@ std::string attrPathJoin(json input) { }); } -static void worker(EvalState &state, Bindings &autoArgs, AutoCloseFD &to, +static void worker(ref state, Bindings &autoArgs, AutoCloseFD &to, AutoCloseFD &from) { std::optional flake; @@ -258,8 +258,7 @@ static void worker(EvalState &state, Bindings &autoArgs, AutoCloseFD &to, parseFlakeRefWithFragmentAndOutputsSpec(myArgs.releaseExpr, absPath(".")); - flake.emplace(InstallableFlake({}, ref(&state), - std::move(flakeRef), fragment, + flake.emplace(InstallableFlake({}, state, std::move(flakeRef), fragment, outputSpec, {}, {}, flake::LockFlags{ .updateLockFile = false, @@ -268,7 +267,7 @@ static void worker(EvalState &state, Bindings &autoArgs, AutoCloseFD &to, })); }; - auto vRoot = topLevelValue(state, autoArgs, flake); + auto vRoot = topLevelValue(*state, autoArgs, flake); while (true) { /* Wait for the collector to send us a job name. */ @@ -288,14 +287,14 @@ static void worker(EvalState &state, Bindings &autoArgs, AutoCloseFD &to, json reply = json{{"attr", attrPathS}, {"attrPath", path}}; try { auto vTmp = - findAlongAttrPath(state, attrPathS, autoArgs, *vRoot).first; + findAlongAttrPath(*state, attrPathS, autoArgs, *vRoot).first; - auto v = state.allocValue(); - state.autoCallFunction(autoArgs, *vTmp, *v); + auto v = state->allocValue(); + state->autoCallFunction(autoArgs, *vTmp, *v); if (v->type() == nAttrs) { - if (auto drvInfo = getDerivation(state, *v, false)) { - auto drv = Drv(state, *drvInfo); + if (auto drvInfo = getDerivation(*state, *v, false)) { + auto drv = Drv(*state, *drvInfo); reply.update(drv); /* Register the derivation as a GC root. !!! This @@ -306,7 +305,7 @@ static void worker(EvalState &state, Bindings &autoArgs, AutoCloseFD &to, std::string(baseNameOf(drv.drvPath)); if (!pathExists(root)) { auto localStore = - state.store + state->store .dynamic_pointer_cast(); auto storePath = localStore->parseStorePath(drv.drvPath); @@ -321,15 +320,15 @@ static void worker(EvalState &state, Bindings &autoArgs, AutoCloseFD &to, // = true;` for top-level attrset for (auto &i : - v->attrs->lexicographicOrder(state.symbols)) { - const std::string &name = state.symbols[i->name]; + v->attrs->lexicographicOrder(state->symbols)) { + const std::string &name = state->symbols[i->name]; attrs.push_back(name); if (name == "recurseForDerivations") { auto attrv = - v->attrs->get(state.sRecurseForDerivations); + v->attrs->get(state->sRecurseForDerivations); recurse = - state.forceBool(*attrv->value, attrv->pos); + state->forceBool(*attrv->value, attrv->pos); } } if (recurse) @@ -368,7 +367,7 @@ static void worker(EvalState &state, Bindings &autoArgs, AutoCloseFD &to, writeLine(to.get(), "restart"); } -typedef std::function state, Bindings &autoArgs, AutoCloseFD &to, AutoCloseFD &from)> Processor; @@ -388,10 +387,10 @@ struct Proc { std::make_shared(std::move(toPipe.readSide))}]() { debug("created worker process %d", getpid()); try { - EvalState state(myArgs.searchPath, - openStore(*myArgs.evalStoreUrl)); - Bindings &autoArgs = *myArgs.getAutoArgs(state); - proc(state, autoArgs, *to, *from); + auto state = std::make_shared( + myArgs.searchPath, openStore(*myArgs.evalStoreUrl)); + Bindings &autoArgs = *myArgs.getAutoArgs(*state); + proc(ref(state), autoArgs, *to, *from); } catch (Error &e) { nlohmann::json err; auto msg = e.msg(); From eaa84c7e97a101265e955dcfcbc3350bca233f1c Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sun, 22 Jan 2023 09:47:53 +0000 Subject: [PATCH 235/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/befc83905c965adfd33e5cae49acb0351f6e0404' (2023-01-13) → 'github:NixOS/nixpkgs/5ed481943351e9fd354aeb557679624224de38d5' (2023-01-20) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 66769e6ba..975744e18 100644 --- a/flake.lock +++ b/flake.lock @@ -22,11 +22,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1673631141, - "narHash": "sha256-AprpYQ5JvLS4wQG/ghm2UriZ9QZXvAwh1HlgA/6ZEVQ=", + "lastModified": 1674211260, + "narHash": "sha256-xU6Rv9sgnwaWK7tgCPadV6HhI2Y/fl4lKxJoG2+m9qs=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "befc83905c965adfd33e5cae49acb0351f6e0404", + "rev": "5ed481943351e9fd354aeb557679624224de38d5", "type": "github" }, "original": { From 396a0d3045bbf0a5252cce5771be6607be05c4f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Wed, 18 Jan 2023 16:38:32 +0100 Subject: [PATCH 236/419] bump nixpkgs and set version to 2_13 --- .nix-version | 2 +- flake.lock | 8 ++++---- flake.nix | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.nix-version b/.nix-version index 6842dbdf3..8b980bc7d 100644 --- a/.nix-version +++ b/.nix-version @@ -1 +1 @@ -unstable +2_13 diff --git a/flake.lock b/flake.lock index 975744e18..4803c32c0 100644 --- a/flake.lock +++ b/flake.lock @@ -22,16 +22,16 @@ }, "nixpkgs": { "locked": { - "lastModified": 1674211260, - "narHash": "sha256-xU6Rv9sgnwaWK7tgCPadV6HhI2Y/fl4lKxJoG2+m9qs=", + "lastModified": 1674380517, + "narHash": "sha256-+wjehzo+LlHb34fTlSK0OW8N9Us9+6mzydCVQyIhE9k=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "5ed481943351e9fd354aeb557679624224de38d5", + "rev": "4a91562abad9ef3dd581da561e80408ea54e8ab6", "type": "github" }, "original": { "owner": "NixOS", - "ref": "nixos-unstable", + "ref": "master", "repo": "nixpkgs", "type": "github" } diff --git a/flake.nix b/flake.nix index 9259fabb1..24efad70d 100644 --- a/flake.nix +++ b/flake.nix @@ -1,7 +1,7 @@ { description = "Hydra's builtin hydra-eval-jobs as a standalone"; - inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; + inputs.nixpkgs.url = "github:NixOS/nixpkgs/master"; inputs.flake-parts.url = "github:hercules-ci/flake-parts"; inputs.flake-parts.inputs.nixpkgs-lib.follows = "nixpkgs"; From 205f05ac02e322b1f5f8be38d188fed578e8bc43 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Wed, 18 Jan 2023 16:38:38 +0100 Subject: [PATCH 237/419] fix build with 2.13 --- src/nix-eval-jobs.cc | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/nix-eval-jobs.cc b/src/nix-eval-jobs.cc index 4d6c2cd89..7b22fc4ad 100644 --- a/src/nix-eval-jobs.cc +++ b/src/nix-eval-jobs.cc @@ -20,6 +20,7 @@ #include #include #include +#include #include @@ -255,15 +256,15 @@ static void worker(ref state, Bindings &autoArgs, AutoCloseFD &to, std::optional flake; if (myArgs.flake) { auto [flakeRef, fragment, outputSpec] = - parseFlakeRefWithFragmentAndOutputsSpec(myArgs.releaseExpr, - absPath(".")); + parseFlakeRefWithFragmentAndExtendedOutputsSpec(myArgs.releaseExpr, + absPath(".")); flake.emplace(InstallableFlake({}, state, std::move(flakeRef), fragment, outputSpec, {}, {}, flake::LockFlags{ .updateLockFile = false, .useRegistries = false, - .allowMutable = false, + .allowUnlocked = false, })); }; From cdd53c2782546f171fa9310ab305bfb421c74754 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Wed, 18 Jan 2023 16:44:36 +0100 Subject: [PATCH 238/419] bump nix-eval-jobs version --- default.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/default.nix b/default.nix index a60ed77d2..982c5e044 100644 --- a/default.nix +++ b/default.nix @@ -17,7 +17,7 @@ let in stdenv.mkDerivation rec { pname = "nix-eval-jobs"; - version = "2.11.0"; + version = "2.13.0"; src = if srcDir == null then filterMesonBuild ./. else srcDir; buildInputs = [ nlohmann_json From 33691ffc8f63e0266b18f9a0e3ad1142694f73cd Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 23 Jan 2023 01:16:08 +0000 Subject: [PATCH 239/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/4a91562abad9ef3dd581da561e80408ea54e8ab6' (2023-01-22) → 'github:NixOS/nixpkgs/6992a4c3a69ab0d61d1c929d80d15b0cf4bf1404' (2023-01-23) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 4803c32c0..a40dc2fe5 100644 --- a/flake.lock +++ b/flake.lock @@ -22,11 +22,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1674380517, - "narHash": "sha256-+wjehzo+LlHb34fTlSK0OW8N9Us9+6mzydCVQyIhE9k=", + "lastModified": 1674436393, + "narHash": "sha256-xVMIdvyA3R/stDifjRjJr+xFu6NpFn8IQ3kDSpd9nAk=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "4a91562abad9ef3dd581da561e80408ea54e8ab6", + "rev": "6992a4c3a69ab0d61d1c929d80d15b0cf4bf1404", "type": "github" }, "original": { From 4cf669e1c6f6c54d88288a01f5b71cbf885e2010 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 Jan 2023 23:02:11 +0000 Subject: [PATCH 240/419] build(deps): bump DeterminateSystems/update-flake-lock from 15 to 16 Bumps [DeterminateSystems/update-flake-lock](https://github.com/DeterminateSystems/update-flake-lock) from 15 to 16. - [Release notes](https://github.com/DeterminateSystems/update-flake-lock/releases) - [Commits](https://github.com/DeterminateSystems/update-flake-lock/compare/v15...v16) --- updated-dependencies: - dependency-name: DeterminateSystems/update-flake-lock dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/update-flake-lock.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/update-flake-lock.yml b/.github/workflows/update-flake-lock.yml index 6aefd749a..2575203f0 100644 --- a/.github/workflows/update-flake-lock.yml +++ b/.github/workflows/update-flake-lock.yml @@ -16,6 +16,6 @@ jobs: extra_nix_config: | access-tokens = github.com=${{ secrets.GITHUB_TOKEN }} - name: Update flake.lock - uses: DeterminateSystems/update-flake-lock@v15 + uses: DeterminateSystems/update-flake-lock@v16 with: token: ${{ secrets.GH_TOKEN_FOR_UPDATES }} From 7d090705207e83a5f7b08abb4e6cbcb1e0e99887 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 26 Jan 2023 01:17:01 +0000 Subject: [PATCH 241/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/6992a4c3a69ab0d61d1c929d80d15b0cf4bf1404' (2023-01-23) → 'github:NixOS/nixpkgs/15b19586b6e05692ca1c160ecdff095089310549' (2023-01-26) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index a40dc2fe5..44c63889e 100644 --- a/flake.lock +++ b/flake.lock @@ -22,11 +22,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1674436393, - "narHash": "sha256-xVMIdvyA3R/stDifjRjJr+xFu6NpFn8IQ3kDSpd9nAk=", + "lastModified": 1674695221, + "narHash": "sha256-6Sn/m4aGaZbDxBRAdSVgmcJUMWiI8cyqzQICDZmISdk=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "6992a4c3a69ab0d61d1c929d80d15b0cf4bf1404", + "rev": "15b19586b6e05692ca1c160ecdff095089310549", "type": "github" }, "original": { From 1b14a6705a59054eacfbbeb13fbbd87e0c4522e9 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 30 Jan 2023 01:14:16 +0000 Subject: [PATCH 242/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'flake-parts': 'github:hercules-ci/flake-parts/82c16f1682cf50c01cb0280b38a1eed202b3fe9f' (2023-01-10) → 'github:hercules-ci/flake-parts/7c7a8bce3dffe71203dcd4276504d1cb49dfe05f' (2023-01-26) • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/15b19586b6e05692ca1c160ecdff095089310549' (2023-01-26) → 'github:NixOS/nixpkgs/99f5676ba0a0c2d7605b63b2dd1b146c384f42dd' (2023-01-30) --- flake.lock | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/flake.lock b/flake.lock index 44c63889e..c96a145d5 100644 --- a/flake.lock +++ b/flake.lock @@ -7,11 +7,11 @@ ] }, "locked": { - "lastModified": 1673362319, - "narHash": "sha256-Pjp45Vnj7S/b3BRpZEVfdu8sqqA6nvVjvYu59okhOyI=", + "lastModified": 1674771137, + "narHash": "sha256-Zpk1GbEsYrqKmuIZkx+f+8pU0qcCYJoSUwNz1Zk+R00=", "owner": "hercules-ci", "repo": "flake-parts", - "rev": "82c16f1682cf50c01cb0280b38a1eed202b3fe9f", + "rev": "7c7a8bce3dffe71203dcd4276504d1cb49dfe05f", "type": "github" }, "original": { @@ -22,11 +22,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1674695221, - "narHash": "sha256-6Sn/m4aGaZbDxBRAdSVgmcJUMWiI8cyqzQICDZmISdk=", + "lastModified": 1675040521, + "narHash": "sha256-+YhT+lQT95qHcj5SNXdRFqIV/SvAezT90T8GzqQ94lE=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "15b19586b6e05692ca1c160ecdff095089310549", + "rev": "99f5676ba0a0c2d7605b63b2dd1b146c384f42dd", "type": "github" }, "original": { From 2f6aa8717e533af7519c317de2c0ba1d53d66c1d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Feb 2023 23:01:27 +0000 Subject: [PATCH 243/419] build(deps): bump cachix/install-nix-action from 18 to 19 Bumps [cachix/install-nix-action](https://github.com/cachix/install-nix-action) from 18 to 19. - [Release notes](https://github.com/cachix/install-nix-action/releases) - [Commits](https://github.com/cachix/install-nix-action/compare/v18...v19) --- updated-dependencies: - dependency-name: cachix/install-nix-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/tests.yml | 2 +- .github/workflows/update-flake-lock.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 37a00b020..868bb5d92 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -17,7 +17,7 @@ jobs: with: # Nix Flakes doesn't work on shallow clones fetch-depth: 0 - - uses: cachix/install-nix-action@v18 + - uses: cachix/install-nix-action@v19 with: github_access_token: ${{ secrets.GITHUB_TOKEN }} extra_nix_config: | diff --git a/.github/workflows/update-flake-lock.yml b/.github/workflows/update-flake-lock.yml index 2575203f0..56f361aae 100644 --- a/.github/workflows/update-flake-lock.yml +++ b/.github/workflows/update-flake-lock.yml @@ -11,7 +11,7 @@ jobs: - name: Checkout repository uses: actions/checkout@v3 - name: Install Nix - uses: cachix/install-nix-action@v18 + uses: cachix/install-nix-action@v19 with: extra_nix_config: | access-tokens = github.com=${{ secrets.GITHUB_TOKEN }} From 148ebb91c7eed05088c75b85b9dbdec14d02a964 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 16 Feb 2023 01:21:17 +0000 Subject: [PATCH 244/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'flake-parts': 'github:hercules-ci/flake-parts/7c7a8bce3dffe71203dcd4276504d1cb49dfe05f' (2023-01-26) → 'github:hercules-ci/flake-parts/47478a4a003e745402acf63be7f9a092d51b83d7' (2023-02-09) • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/99f5676ba0a0c2d7605b63b2dd1b146c384f42dd' (2023-01-30) → 'github:NixOS/nixpkgs/cd1364e35b503d0add3f4bc57006ebb02070ae14' (2023-02-16) --- flake.lock | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/flake.lock b/flake.lock index c96a145d5..736a48f64 100644 --- a/flake.lock +++ b/flake.lock @@ -7,11 +7,11 @@ ] }, "locked": { - "lastModified": 1674771137, - "narHash": "sha256-Zpk1GbEsYrqKmuIZkx+f+8pU0qcCYJoSUwNz1Zk+R00=", + "lastModified": 1675933616, + "narHash": "sha256-/rczJkJHtx16IFxMmAWu5nNYcSXNg1YYXTHoGjLrLUA=", "owner": "hercules-ci", "repo": "flake-parts", - "rev": "7c7a8bce3dffe71203dcd4276504d1cb49dfe05f", + "rev": "47478a4a003e745402acf63be7f9a092d51b83d7", "type": "github" }, "original": { @@ -22,11 +22,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1675040521, - "narHash": "sha256-+YhT+lQT95qHcj5SNXdRFqIV/SvAezT90T8GzqQ94lE=", + "lastModified": 1676510113, + "narHash": "sha256-TBB/1Fv1/S7si9/Dy/DwYCoJBBZSwJu2zzQzagW8P48=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "99f5676ba0a0c2d7605b63b2dd1b146c384f42dd", + "rev": "cd1364e35b503d0add3f4bc57006ebb02070ae14", "type": "github" }, "original": { From d034f8fd2ba673a8508db90262d2fedfc1daacfe Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 27 Feb 2023 01:21:03 +0000 Subject: [PATCH 245/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/cd1364e35b503d0add3f4bc57006ebb02070ae14' (2023-02-16) → 'github:NixOS/nixpkgs/5d447a9e7009a116ab1e62dd5599b9272206067f' (2023-02-27) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 736a48f64..fcfad16b9 100644 --- a/flake.lock +++ b/flake.lock @@ -22,11 +22,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1676510113, - "narHash": "sha256-TBB/1Fv1/S7si9/Dy/DwYCoJBBZSwJu2zzQzagW8P48=", + "lastModified": 1677460846, + "narHash": "sha256-zvz9zYoELpyR947BiSf+jL6MvBrjXNy3DH+rqeymoWU=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "cd1364e35b503d0add3f4bc57006ebb02070ae14", + "rev": "5d447a9e7009a116ab1e62dd5599b9272206067f", "type": "github" }, "original": { From 72ca2d9735449a5a1741935777ec4ff09d3d71bf Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 2 Mar 2023 01:25:24 +0000 Subject: [PATCH 246/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'flake-parts': 'github:hercules-ci/flake-parts/47478a4a003e745402acf63be7f9a092d51b83d7' (2023-02-09) → 'github:hercules-ci/flake-parts/dc531e3a9ce757041e1afaff8ee932725ca60002' (2023-03-01) • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/5d447a9e7009a116ab1e62dd5599b9272206067f' (2023-02-27) → 'github:NixOS/nixpkgs/00ebdb7b9cf257fc26bdc49749d1204ef3667ddc' (2023-03-02) --- flake.lock | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/flake.lock b/flake.lock index fcfad16b9..ee6a7c8aa 100644 --- a/flake.lock +++ b/flake.lock @@ -7,11 +7,11 @@ ] }, "locked": { - "lastModified": 1675933616, - "narHash": "sha256-/rczJkJHtx16IFxMmAWu5nNYcSXNg1YYXTHoGjLrLUA=", + "lastModified": 1677714448, + "narHash": "sha256-Hq8qLs8xFu28aDjytfxjdC96bZ6pds21Yy09mSC156I=", "owner": "hercules-ci", "repo": "flake-parts", - "rev": "47478a4a003e745402acf63be7f9a092d51b83d7", + "rev": "dc531e3a9ce757041e1afaff8ee932725ca60002", "type": "github" }, "original": { @@ -22,11 +22,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1677460846, - "narHash": "sha256-zvz9zYoELpyR947BiSf+jL6MvBrjXNy3DH+rqeymoWU=", + "lastModified": 1677719885, + "narHash": "sha256-d9Lc2bZNgelX9I2gxN/TAwIE+RcEDthH+zysYWPlQ1k=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "5d447a9e7009a116ab1e62dd5599b9272206067f", + "rev": "00ebdb7b9cf257fc26bdc49749d1204ef3667ddc", "type": "github" }, "original": { From f658392ffc241b129ae3eab21f39ec0e5c33f67f Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 6 Mar 2023 01:21:47 +0000 Subject: [PATCH 247/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/00ebdb7b9cf257fc26bdc49749d1204ef3667ddc' (2023-03-02) → 'github:NixOS/nixpkgs/d154f809e9c3c47fee72186aa3ff6479403435d4' (2023-03-06) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index ee6a7c8aa..d71f3facd 100644 --- a/flake.lock +++ b/flake.lock @@ -22,11 +22,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1677719885, - "narHash": "sha256-d9Lc2bZNgelX9I2gxN/TAwIE+RcEDthH+zysYWPlQ1k=", + "lastModified": 1678062977, + "narHash": "sha256-i+wOuZ8arDBldx/5VdhSbv3XoOZsrleaJ/ydP74szIc=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "00ebdb7b9cf257fc26bdc49749d1204ef3667ddc", + "rev": "d154f809e9c3c47fee72186aa3ff6479403435d4", "type": "github" }, "original": { From 2168237dea9ae0ec90e88b7597fb0c4c29531d88 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Mar 2023 23:57:46 +0000 Subject: [PATCH 248/419] build(deps): bump cachix/install-nix-action from 19 to 20 Bumps [cachix/install-nix-action](https://github.com/cachix/install-nix-action) from 19 to 20. - [Release notes](https://github.com/cachix/install-nix-action/releases) - [Commits](https://github.com/cachix/install-nix-action/compare/v19...v20) --- updated-dependencies: - dependency-name: cachix/install-nix-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/tests.yml | 2 +- .github/workflows/update-flake-lock.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 868bb5d92..4c2b8d775 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -17,7 +17,7 @@ jobs: with: # Nix Flakes doesn't work on shallow clones fetch-depth: 0 - - uses: cachix/install-nix-action@v19 + - uses: cachix/install-nix-action@v20 with: github_access_token: ${{ secrets.GITHUB_TOKEN }} extra_nix_config: | diff --git a/.github/workflows/update-flake-lock.yml b/.github/workflows/update-flake-lock.yml index 56f361aae..25eb806e5 100644 --- a/.github/workflows/update-flake-lock.yml +++ b/.github/workflows/update-flake-lock.yml @@ -11,7 +11,7 @@ jobs: - name: Checkout repository uses: actions/checkout@v3 - name: Install Nix - uses: cachix/install-nix-action@v19 + uses: cachix/install-nix-action@v20 with: extra_nix_config: | access-tokens = github.com=${{ secrets.GITHUB_TOKEN }} From d66f3aaa14337691dcea0b1f4781d2c382987770 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Mar 2023 23:57:54 +0000 Subject: [PATCH 249/419] build(deps): bump DeterminateSystems/update-flake-lock from 16 to 17 Bumps [DeterminateSystems/update-flake-lock](https://github.com/DeterminateSystems/update-flake-lock) from 16 to 17. - [Release notes](https://github.com/DeterminateSystems/update-flake-lock/releases) - [Commits](https://github.com/DeterminateSystems/update-flake-lock/compare/v16...v17) --- updated-dependencies: - dependency-name: DeterminateSystems/update-flake-lock dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/update-flake-lock.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/update-flake-lock.yml b/.github/workflows/update-flake-lock.yml index 56f361aae..83ef51180 100644 --- a/.github/workflows/update-flake-lock.yml +++ b/.github/workflows/update-flake-lock.yml @@ -16,6 +16,6 @@ jobs: extra_nix_config: | access-tokens = github.com=${{ secrets.GITHUB_TOKEN }} - name: Update flake.lock - uses: DeterminateSystems/update-flake-lock@v16 + uses: DeterminateSystems/update-flake-lock@v17 with: token: ${{ secrets.GH_TOKEN_FOR_UPDATES }} From 9fe7efbd102e4e630acd836393347d514c91808f Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 9 Mar 2023 01:25:46 +0000 Subject: [PATCH 250/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/d154f809e9c3c47fee72186aa3ff6479403435d4' (2023-03-06) → 'github:NixOS/nixpkgs/75c8abce0657f9981be937f2bc7b88125d98c03f' (2023-03-09) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index d71f3facd..d2fb4df0e 100644 --- a/flake.lock +++ b/flake.lock @@ -22,11 +22,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1678062977, - "narHash": "sha256-i+wOuZ8arDBldx/5VdhSbv3XoOZsrleaJ/ydP74szIc=", + "lastModified": 1678321859, + "narHash": "sha256-WnAM2zrtDJDfSEmiiklSAHfep+XMIjfpU/8aTXwHZ5A=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "d154f809e9c3c47fee72186aa3ff6479403435d4", + "rev": "75c8abce0657f9981be937f2bc7b88125d98c03f", "type": "github" }, "original": { From 2f518e70ecf4468fbeae598e6f80158cdfabcf2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Thu, 9 Mar 2023 20:57:30 +0100 Subject: [PATCH 251/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'flake-parts': 'github:hercules-ci/flake-parts/dc531e3a9ce757041e1afaff8ee932725ca60002' (2023-03-01) → 'github:hercules-ci/flake-parts/c13d60b89adea3dc20704c045ec4d50dd964d447' (2023-03-09) • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/75c8abce0657f9981be937f2bc7b88125d98c03f' (2023-03-09) → 'github:NixOS/nixpkgs/8d8f5ede919dc60a86ac37311a63092411c72e90' (2023-03-09) --- flake.lock | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/flake.lock b/flake.lock index d2fb4df0e..9e533d9e2 100644 --- a/flake.lock +++ b/flake.lock @@ -7,11 +7,11 @@ ] }, "locked": { - "lastModified": 1677714448, - "narHash": "sha256-Hq8qLs8xFu28aDjytfxjdC96bZ6pds21Yy09mSC156I=", + "lastModified": 1678379998, + "narHash": "sha256-TZdfNqftHhDuIFwBcN9MUThx5sQXCTeZk9je5byPKRw=", "owner": "hercules-ci", "repo": "flake-parts", - "rev": "dc531e3a9ce757041e1afaff8ee932725ca60002", + "rev": "c13d60b89adea3dc20704c045ec4d50dd964d447", "type": "github" }, "original": { @@ -22,11 +22,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1678321859, - "narHash": "sha256-WnAM2zrtDJDfSEmiiklSAHfep+XMIjfpU/8aTXwHZ5A=", + "lastModified": 1678391601, + "narHash": "sha256-KpP4agiaNf2PO67HlvToZKuWS1jvDrORz09bJXkE8bQ=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "75c8abce0657f9981be937f2bc7b88125d98c03f", + "rev": "8d8f5ede919dc60a86ac37311a63092411c72e90", "type": "github" }, "original": { From 94b3a3a43dff7722b6503c2e9309e6183bbafe08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Thu, 9 Mar 2023 21:01:15 +0100 Subject: [PATCH 252/419] update nix version --- .nix-version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.nix-version b/.nix-version index 8b980bc7d..d28682992 100644 --- a/.nix-version +++ b/.nix-version @@ -1 +1 @@ -2_13 +2_14 From 54a9b264313c4aecc5a0861da418ca0d3975199a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Thu, 9 Mar 2023 22:18:00 +0100 Subject: [PATCH 253/419] fix build with nix 2.14 --- src/nix-eval-jobs.cc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/nix-eval-jobs.cc b/src/nix-eval-jobs.cc index 7b22fc4ad..b9c5a1960 100644 --- a/src/nix-eval-jobs.cc +++ b/src/nix-eval-jobs.cc @@ -21,6 +21,7 @@ #include #include #include +#include #include @@ -328,8 +329,9 @@ static void worker(ref state, Bindings &autoArgs, AutoCloseFD &to, if (name == "recurseForDerivations") { auto attrv = v->attrs->get(state->sRecurseForDerivations); - recurse = - state->forceBool(*attrv->value, attrv->pos); + recurse = state->forceBool( + *attrv->value, attrv->pos, + "while evaluating recurseForDerivations"); } } if (recurse) From 5a7dc07ad152518f04e26e805c365a4dab93a2d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Fri, 10 Mar 2023 09:08:07 +0100 Subject: [PATCH 254/419] bump version --- .nix-version | 2 +- default.nix | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.nix-version b/.nix-version index d28682992..6842dbdf3 100644 --- a/.nix-version +++ b/.nix-version @@ -1 +1 @@ -2_14 +unstable diff --git a/default.nix b/default.nix index 982c5e044..4167e69c8 100644 --- a/default.nix +++ b/default.nix @@ -17,7 +17,7 @@ let in stdenv.mkDerivation rec { pname = "nix-eval-jobs"; - version = "2.13.0"; + version = "2.14.0"; src = if srcDir == null then filterMesonBuild ./. else srcDir; buildInputs = [ nlohmann_json From 4983b1512197c9cec73f652ab2ccc3ecdc890da4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Fri, 10 Mar 2023 09:08:07 +0100 Subject: [PATCH 255/419] bump version --- .nix-version | 2 +- default.nix | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.nix-version b/.nix-version index d28682992..6842dbdf3 100644 --- a/.nix-version +++ b/.nix-version @@ -1 +1 @@ -2_14 +unstable diff --git a/default.nix b/default.nix index 982c5e044..4167e69c8 100644 --- a/default.nix +++ b/default.nix @@ -17,7 +17,7 @@ let in stdenv.mkDerivation rec { pname = "nix-eval-jobs"; - version = "2.13.0"; + version = "2.14.0"; src = if srcDir == null then filterMesonBuild ./. else srcDir; buildInputs = [ nlohmann_json From 3170433774c55f41d6ce3e7b0302a62a66a1f9f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sat, 11 Mar 2023 10:25:59 +0100 Subject: [PATCH 256/419] switch to nix 2_14 --- .nix-version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.nix-version b/.nix-version index 6842dbdf3..d28682992 100644 --- a/.nix-version +++ b/.nix-version @@ -1 +1 @@ -unstable +2_14 From 6e5df498addc3148af7c0ae1517f4357a7d9e2b3 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 13 Mar 2023 01:17:55 +0000 Subject: [PATCH 257/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/8d8f5ede919dc60a86ac37311a63092411c72e90' (2023-03-09) → 'github:NixOS/nixpkgs/7c84ea61abbe8038e273fbef035e0364a35be17d' (2023-03-13) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 9e533d9e2..5dc4a0e45 100644 --- a/flake.lock +++ b/flake.lock @@ -22,11 +22,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1678391601, - "narHash": "sha256-KpP4agiaNf2PO67HlvToZKuWS1jvDrORz09bJXkE8bQ=", + "lastModified": 1678670110, + "narHash": "sha256-Yi84/EUqMl+S3Uafzoskta3eW0/HId/fH0rSkaw5nk8=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "8d8f5ede919dc60a86ac37311a63092411c72e90", + "rev": "7c84ea61abbe8038e273fbef035e0364a35be17d", "type": "github" }, "original": { From 4c6c2e5c11e6ad2e01285127d2733ef2768725d0 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 16 Mar 2023 01:18:51 +0000 Subject: [PATCH 258/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/7c84ea61abbe8038e273fbef035e0364a35be17d' (2023-03-13) → 'github:NixOS/nixpkgs/8083b23ad5b179ffc539204b20f8b7b50610cf29' (2023-03-16) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 5dc4a0e45..4faaa115b 100644 --- a/flake.lock +++ b/flake.lock @@ -22,11 +22,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1678670110, - "narHash": "sha256-Yi84/EUqMl+S3Uafzoskta3eW0/HId/fH0rSkaw5nk8=", + "lastModified": 1678928838, + "narHash": "sha256-sF7k+PpiqgSVkwyPOsuLFQi4KVeu5AEfjWMleahbovc=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "7c84ea61abbe8038e273fbef035e0364a35be17d", + "rev": "8083b23ad5b179ffc539204b20f8b7b50610cf29", "type": "github" }, "original": { From ac66fb34ef28d19f19d483c3cafc68d0197d47be Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 20 Mar 2023 01:19:43 +0000 Subject: [PATCH 259/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/8083b23ad5b179ffc539204b20f8b7b50610cf29' (2023-03-16) → 'github:NixOS/nixpkgs/18b17c58dc248a66469df89e3d334b305d8235ec' (2023-03-20) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 4faaa115b..13276e166 100644 --- a/flake.lock +++ b/flake.lock @@ -22,11 +22,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1678928838, - "narHash": "sha256-sF7k+PpiqgSVkwyPOsuLFQi4KVeu5AEfjWMleahbovc=", + "lastModified": 1679271670, + "narHash": "sha256-QKC6m81hD6JcBzGhRvI+HDEktEzf78vr5bhM3WCIeQU=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "8083b23ad5b179ffc539204b20f8b7b50610cf29", + "rev": "18b17c58dc248a66469df89e3d334b305d8235ec", "type": "github" }, "original": { From 9c491732dc80573526222e3d919f01b5a7a1cf3f Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 27 Mar 2023 19:09:00 +0000 Subject: [PATCH 260/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'flake-parts': 'github:hercules-ci/flake-parts/c13d60b89adea3dc20704c045ec4d50dd964d447' (2023-03-09) → 'github:hercules-ci/flake-parts/3502ee99d6dade045bdeaf7b0cd8ec703484c25c' (2023-03-25) • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/18b17c58dc248a66469df89e3d334b305d8235ec' (2023-03-20) → 'github:NixOS/nixpkgs/2749074b2e2960fc97aaeca251f3152d6e6effb7' (2023-03-27) --- flake.lock | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/flake.lock b/flake.lock index 13276e166..40fa2caf3 100644 --- a/flake.lock +++ b/flake.lock @@ -7,11 +7,11 @@ ] }, "locked": { - "lastModified": 1678379998, - "narHash": "sha256-TZdfNqftHhDuIFwBcN9MUThx5sQXCTeZk9je5byPKRw=", + "lastModified": 1679737941, + "narHash": "sha256-srSD9CwsVPnUMsIZ7Kt/UegkKUEBcTyU1Rev7mO45S0=", "owner": "hercules-ci", "repo": "flake-parts", - "rev": "c13d60b89adea3dc20704c045ec4d50dd964d447", + "rev": "3502ee99d6dade045bdeaf7b0cd8ec703484c25c", "type": "github" }, "original": { @@ -22,11 +22,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1679271670, - "narHash": "sha256-QKC6m81hD6JcBzGhRvI+HDEktEzf78vr5bhM3WCIeQU=", + "lastModified": 1679943299, + "narHash": "sha256-RsGh4KrY4rnhTI7vTv0eVS6P5/MbakSKkGFXOnuN3q4=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "18b17c58dc248a66469df89e3d334b305d8235ec", + "rev": "2749074b2e2960fc97aaeca251f3152d6e6effb7", "type": "github" }, "original": { From 2d507977311c1e850fcbe5bd7a6c4e7cdcd6b7b0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Mar 2023 23:57:19 +0000 Subject: [PATCH 261/419] build(deps): bump DeterminateSystems/update-flake-lock from 17 to 18 Bumps [DeterminateSystems/update-flake-lock](https://github.com/DeterminateSystems/update-flake-lock) from 17 to 18. - [Release notes](https://github.com/DeterminateSystems/update-flake-lock/releases) - [Commits](https://github.com/DeterminateSystems/update-flake-lock/compare/v17...v18) --- updated-dependencies: - dependency-name: DeterminateSystems/update-flake-lock dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/update-flake-lock.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/update-flake-lock.yml b/.github/workflows/update-flake-lock.yml index 12b69a397..cb7e992bc 100644 --- a/.github/workflows/update-flake-lock.yml +++ b/.github/workflows/update-flake-lock.yml @@ -16,6 +16,6 @@ jobs: extra_nix_config: | access-tokens = github.com=${{ secrets.GITHUB_TOKEN }} - name: Update flake.lock - uses: DeterminateSystems/update-flake-lock@v17 + uses: DeterminateSystems/update-flake-lock@v18 with: token: ${{ secrets.GH_TOKEN_FOR_UPDATES }} From e9f3037c1e5c61ec47bab0f8f283a394fad6be95 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 30 Mar 2023 01:16:07 +0000 Subject: [PATCH 262/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/2749074b2e2960fc97aaeca251f3152d6e6effb7' (2023-03-27) → 'github:NixOS/nixpkgs/4e416a8e847057c49e73be37ae8dc4fcdfe9eff8' (2023-03-30) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 40fa2caf3..2801633e1 100644 --- a/flake.lock +++ b/flake.lock @@ -22,11 +22,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1679943299, - "narHash": "sha256-RsGh4KrY4rnhTI7vTv0eVS6P5/MbakSKkGFXOnuN3q4=", + "lastModified": 1680138801, + "narHash": "sha256-TSO0F7fvX2+M6lpVYaGEOciq4Iawy/su2jZ8Mfi9nxc=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "2749074b2e2960fc97aaeca251f3152d6e6effb7", + "rev": "4e416a8e847057c49e73be37ae8dc4fcdfe9eff8", "type": "github" }, "original": { From 4bf3bee93d057fc2577b032b583e1aca156769ff Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 3 Apr 2023 01:09:34 +0000 Subject: [PATCH 263/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'flake-parts': 'github:hercules-ci/flake-parts/3502ee99d6dade045bdeaf7b0cd8ec703484c25c' (2023-03-25) → 'github:hercules-ci/flake-parts/dcc36e45d054d7bb554c9cdab69093debd91a0b5' (2023-04-01) • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/4e416a8e847057c49e73be37ae8dc4fcdfe9eff8' (2023-03-30) → 'github:NixOS/nixpkgs/ed9eb3ac00e8442504f8baa5d234d0f9fed93657' (2023-04-03) --- flake.lock | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/flake.lock b/flake.lock index 2801633e1..94e035272 100644 --- a/flake.lock +++ b/flake.lock @@ -7,11 +7,11 @@ ] }, "locked": { - "lastModified": 1679737941, - "narHash": "sha256-srSD9CwsVPnUMsIZ7Kt/UegkKUEBcTyU1Rev7mO45S0=", + "lastModified": 1680392223, + "narHash": "sha256-n3g7QFr85lDODKt250rkZj2IFS3i4/8HBU2yKHO3tqw=", "owner": "hercules-ci", "repo": "flake-parts", - "rev": "3502ee99d6dade045bdeaf7b0cd8ec703484c25c", + "rev": "dcc36e45d054d7bb554c9cdab69093debd91a0b5", "type": "github" }, "original": { @@ -22,11 +22,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1680138801, - "narHash": "sha256-TSO0F7fvX2+M6lpVYaGEOciq4Iawy/su2jZ8Mfi9nxc=", + "lastModified": 1680481868, + "narHash": "sha256-/3GyhIx1PZErPgPoA6botQ+yp5MGUB5uCh9z/6Shj9s=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "4e416a8e847057c49e73be37ae8dc4fcdfe9eff8", + "rev": "ed9eb3ac00e8442504f8baa5d234d0f9fed93657", "type": "github" }, "original": { From c72f7d35f2ecc5105dd015ba9fd761d2292e71f3 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 6 Apr 2023 01:09:02 +0000 Subject: [PATCH 264/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/ed9eb3ac00e8442504f8baa5d234d0f9fed93657' (2023-04-03) → 'github:NixOS/nixpkgs/0e950a1ec78f8ef1f7bcce6b999c03a62bd3e796' (2023-04-06) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 94e035272..6751e6cd6 100644 --- a/flake.lock +++ b/flake.lock @@ -22,11 +22,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1680481868, - "narHash": "sha256-/3GyhIx1PZErPgPoA6botQ+yp5MGUB5uCh9z/6Shj9s=", + "lastModified": 1680742364, + "narHash": "sha256-3195lvlV1NF+GeX8apkbYPjHjNvUNlsPHFkFnoJz3i4=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "ed9eb3ac00e8442504f8baa5d234d0f9fed93657", + "rev": "0e950a1ec78f8ef1f7bcce6b999c03a62bd3e796", "type": "github" }, "original": { From 7d03aa86aaa4f3488151101727348822ebd91cdc Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 10 Apr 2023 01:09:59 +0000 Subject: [PATCH 265/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/0e950a1ec78f8ef1f7bcce6b999c03a62bd3e796' (2023-04-06) → 'github:NixOS/nixpkgs/619ca2064f709582ef4710be2c18433241adfdd0' (2023-04-09) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 6751e6cd6..416e70b86 100644 --- a/flake.lock +++ b/flake.lock @@ -22,11 +22,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1680742364, - "narHash": "sha256-3195lvlV1NF+GeX8apkbYPjHjNvUNlsPHFkFnoJz3i4=", + "lastModified": 1681082853, + "narHash": "sha256-fpmxJNpxwg8tjQKPMEH8QnXgG+2h/5qc2CC1VmSStx8=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "0e950a1ec78f8ef1f7bcce6b999c03a62bd3e796", + "rev": "619ca2064f709582ef4710be2c18433241adfdd0", "type": "github" }, "original": { From 78230da06ba40e58566d4e4b3cbbf5a620c09891 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 Apr 2023 23:57:27 +0000 Subject: [PATCH 266/419] build(deps): bump DeterminateSystems/update-flake-lock from 18 to 19 Bumps [DeterminateSystems/update-flake-lock](https://github.com/DeterminateSystems/update-flake-lock) from 18 to 19. - [Release notes](https://github.com/DeterminateSystems/update-flake-lock/releases) - [Commits](https://github.com/DeterminateSystems/update-flake-lock/compare/v18...v19) --- updated-dependencies: - dependency-name: DeterminateSystems/update-flake-lock dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/update-flake-lock.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/update-flake-lock.yml b/.github/workflows/update-flake-lock.yml index cb7e992bc..d0ec57654 100644 --- a/.github/workflows/update-flake-lock.yml +++ b/.github/workflows/update-flake-lock.yml @@ -16,6 +16,6 @@ jobs: extra_nix_config: | access-tokens = github.com=${{ secrets.GITHUB_TOKEN }} - name: Update flake.lock - uses: DeterminateSystems/update-flake-lock@v18 + uses: DeterminateSystems/update-flake-lock@v19 with: token: ${{ secrets.GH_TOKEN_FOR_UPDATES }} From 71f88d56668e988f7d7151a94f6f7719e07974c2 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 13 Apr 2023 01:08:42 +0000 Subject: [PATCH 267/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/619ca2064f709582ef4710be2c18433241adfdd0' (2023-04-09) → 'github:NixOS/nixpkgs/1a9d9175ecc48ecd033062fa09b1834d13ae9c69' (2023-04-13) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 416e70b86..42977f3bb 100644 --- a/flake.lock +++ b/flake.lock @@ -22,11 +22,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1681082853, - "narHash": "sha256-fpmxJNpxwg8tjQKPMEH8QnXgG+2h/5qc2CC1VmSStx8=", + "lastModified": 1681347147, + "narHash": "sha256-B+hTioRc3Jdf4SJyeCiO0fW5ShIznJk2OTiW2vOV+mc=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "619ca2064f709582ef4710be2c18433241adfdd0", + "rev": "1a9d9175ecc48ecd033062fa09b1834d13ae9c69", "type": "github" }, "original": { From e9c301bcbe42bd5c34d781b79b4c233eaf811587 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Thu, 13 Apr 2023 19:29:18 +0200 Subject: [PATCH 268/419] document the worker/memory tradeoff --- README.md | 29 +++++++++++++++++++++++++++++ src/nix-eval-jobs.cc | 12 +++++++----- 2 files changed, 36 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index bf6570063..17e521fed 100644 --- a/README.md +++ b/README.md @@ -91,3 +91,32 @@ latest release branch. - [colmena](https://github.com/zhaofengli/colmena) - A simple, stateless NixOS deployment tool - [robotnix](https://github.com/danielfullmer/robotnix) - Build Android (AOSP) using Nix, used in their [CI](https://github.com/danielfullmer/robotnix/blob/38b80700ee4265c306dcfdcce45056e32ab2973f/.github/workflows/instantiate.yml#L18) + +## FAQ + +### nix-eval-jobs consumes too much memory / is too slow + +By default, nix-eval-jobs spawns as many worker processes as there are +hardware threads in the system and limits the memory usage for each worker to +4GB. + +However, keep in mind that each worker process may need to re-evaluate shared +dependencies of the attributes, which can introduce some overhead for each +evaluation or cause workers to exceed their memory limit. If you encounter +these situations, you can tune the following options: + +`--workers`: This option allows you to set the number of evaluation workers that +nix-eval-jobs should spawn. You can increase or decrease this number to +optimize the evaluation speed and memory usage. For example, if you have a +system with many CPU cores but limited memory, you may want to reduce the +number of workers to avoid exceeding the memory limit. + +`--max-memory-size`: This option allows you to adjust the memory limit for each +worker process. By default, it's set to 4GiB, but you can increase or decrease +this value as needed. For example, if you have a system with a lot of memory +and want to speed up the evaluation, you may want to increase the memory limit +to allow workers to cache more data in memory before getting restarted by +nix-eval-jobs. + +Overall, tuning these options can help you optimize the performance and memory +usage of nix-eval-jobs to better fit your system and evaluation needs. diff --git a/src/nix-eval-jobs.cc b/src/nix-eval-jobs.cc index b9c5a1960..49aca0580 100644 --- a/src/nix-eval-jobs.cc +++ b/src/nix-eval-jobs.cc @@ -90,11 +90,13 @@ struct MyArgs : MixEvalArgs, MixCommonArgs { .labels = {"workers"}, .handler = {[=](std::string s) { nrWorkers = std::stoi(s); }}}); - addFlag({.longName = "max-memory-size", - .description = "maximum evaluation memory size", - .labels = {"size"}, - .handler = { - [=](std::string s) { maxMemorySize = std::stoi(s); }}}); + addFlag( + {.longName = "max-memory-size", + .description = + "maximum evaluation memory size (4GiB per worker by default)", + .labels = {"size"}, + .handler = { + [=](std::string s) { maxMemorySize = std::stoi(s); }}}); addFlag({.longName = "flake", .description = "build a flake", From 4d674c352e1292bbf5f28f50b6c8eac86ea8bdff Mon Sep 17 00:00:00 2001 From: Julien Malka Date: Mon, 24 Apr 2023 17:08:57 +0200 Subject: [PATCH 269/419] fix recurseForDerivations evaluation in force-recurse mode --- src/nix-eval-jobs.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/nix-eval-jobs.cc b/src/nix-eval-jobs.cc index 49aca0580..d07d3d24e 100644 --- a/src/nix-eval-jobs.cc +++ b/src/nix-eval-jobs.cc @@ -328,7 +328,8 @@ static void worker(ref state, Bindings &autoArgs, AutoCloseFD &to, const std::string &name = state->symbols[i->name]; attrs.push_back(name); - if (name == "recurseForDerivations") { + if (name == "recurseForDerivations" && + !myArgs.forceRecurse) { auto attrv = v->attrs->get(state->sRecurseForDerivations); recurse = state->forceBool( From 469a68d5c1efb1605f7afd5647c71f6643fa9246 Mon Sep 17 00:00:00 2001 From: Raito Bezarius Date: Mon, 24 Apr 2023 19:33:02 +0200 Subject: [PATCH 270/419] feat: add inputDrvs to the JSON Currently, not a lot of things expose inputDrvs (except `show-derivation`), which is a showstopper whenever you want to compute popularity ranking based on the dependency relation. Having `inputsDrvs` in the reply enable downstream users to perform such computations in an efficient way. --- src/nix-eval-jobs.cc | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/nix-eval-jobs.cc b/src/nix-eval-jobs.cc index 49aca0580..4f1c6e59b 100644 --- a/src/nix-eval-jobs.cc +++ b/src/nix-eval-jobs.cc @@ -180,6 +180,7 @@ struct Drv { std::string drvPath; bool isCached; std::map outputs; + std::map> inputDrvs; std::optional meta; Drv(EvalState &state, DrvInfo &drvInfo) { @@ -224,6 +225,11 @@ struct Drv { name = drvInfo.queryName(); system = drvInfo.querySystem(); drvPath = localStore->printStorePath(drvInfo.requireDrvPath()); + + auto drv = localStore->readDerivation(drvInfo.requireDrvPath()); + for (auto &input : drv.inputDrvs) { + inputDrvs[localStore->printStorePath(input.first)] = input.second; + } } }; @@ -231,7 +237,8 @@ static void to_json(nlohmann::json &json, const Drv &drv) { json = nlohmann::json{{"name", drv.name}, {"system", drv.system}, {"drvPath", drv.drvPath}, - {"outputs", drv.outputs}}; + {"outputs", drv.outputs}, + {"inputDrvs", drv.inputDrvs}}; if (drv.meta.has_value()) { json["meta"] = drv.meta.value(); From fa4de3759ca537600fec109a82fcce9725c1e627 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 May 2023 23:57:07 +0000 Subject: [PATCH 271/419] build(deps): bump cachix/install-nix-action from 20 to 21 Bumps [cachix/install-nix-action](https://github.com/cachix/install-nix-action) from 20 to 21. - [Release notes](https://github.com/cachix/install-nix-action/releases) - [Commits](https://github.com/cachix/install-nix-action/compare/v20...v21) --- updated-dependencies: - dependency-name: cachix/install-nix-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/tests.yml | 2 +- .github/workflows/update-flake-lock.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 4c2b8d775..1e1df2723 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -17,7 +17,7 @@ jobs: with: # Nix Flakes doesn't work on shallow clones fetch-depth: 0 - - uses: cachix/install-nix-action@v20 + - uses: cachix/install-nix-action@v21 with: github_access_token: ${{ secrets.GITHUB_TOKEN }} extra_nix_config: | diff --git a/.github/workflows/update-flake-lock.yml b/.github/workflows/update-flake-lock.yml index d0ec57654..785d291a5 100644 --- a/.github/workflows/update-flake-lock.yml +++ b/.github/workflows/update-flake-lock.yml @@ -11,7 +11,7 @@ jobs: - name: Checkout repository uses: actions/checkout@v3 - name: Install Nix - uses: cachix/install-nix-action@v20 + uses: cachix/install-nix-action@v21 with: extra_nix_config: | access-tokens = github.com=${{ secrets.GITHUB_TOKEN }} From 69ae132fc50e70c3e7af01cfd5870d09b3664e9f Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 8 Jun 2023 01:24:53 +0000 Subject: [PATCH 272/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'flake-parts': 'github:hercules-ci/flake-parts/dcc36e45d054d7bb554c9cdab69093debd91a0b5' (2023-04-01) → 'github:hercules-ci/flake-parts/71fb97f0d875fd4de4994dfb849f2c75e17eb6c3' (2023-06-01) • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/1a9d9175ecc48ecd033062fa09b1834d13ae9c69' (2023-04-13) → 'github:NixOS/nixpkgs/5715d6b452b97d12c6f9077321d202a0cb50b8fc' (2023-06-08) --- flake.lock | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/flake.lock b/flake.lock index 42977f3bb..f4e0bf943 100644 --- a/flake.lock +++ b/flake.lock @@ -7,11 +7,11 @@ ] }, "locked": { - "lastModified": 1680392223, - "narHash": "sha256-n3g7QFr85lDODKt250rkZj2IFS3i4/8HBU2yKHO3tqw=", + "lastModified": 1685662779, + "narHash": "sha256-cKDDciXGpMEjP1n6HlzKinN0H+oLmNpgeCTzYnsA2po=", "owner": "hercules-ci", "repo": "flake-parts", - "rev": "dcc36e45d054d7bb554c9cdab69093debd91a0b5", + "rev": "71fb97f0d875fd4de4994dfb849f2c75e17eb6c3", "type": "github" }, "original": { @@ -22,11 +22,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1681347147, - "narHash": "sha256-B+hTioRc3Jdf4SJyeCiO0fW5ShIznJk2OTiW2vOV+mc=", + "lastModified": 1686186815, + "narHash": "sha256-h6OTafX2eLgFQpu5yJf65JHDKrEVoMLLtTn8ksu1+qE=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "1a9d9175ecc48ecd033062fa09b1834d13ae9c69", + "rev": "5715d6b452b97d12c6f9077321d202a0cb50b8fc", "type": "github" }, "original": { From 7130e84e9e5c718984114bc086e957bbae1174b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sat, 10 Jun 2023 13:28:57 +0200 Subject: [PATCH 273/419] fix build --- src/nix-eval-jobs.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/nix-eval-jobs.cc b/src/nix-eval-jobs.cc index 62f14fe85..0d8c97cac 100644 --- a/src/nix-eval-jobs.cc +++ b/src/nix-eval-jobs.cc @@ -140,7 +140,8 @@ static Value *releaseExprTopLevelValue(EvalState &state, Bindings &autoArgs) { Value vTop; if (myArgs.fromArgs) { - Expr *e = state.parseExprFromString(myArgs.releaseExpr, absPath(".")); + Expr *e = state.parseExprFromString( + myArgs.releaseExpr, state.rootPath(CanonPath::fromCwd())); state.eval(e, vTop); } else { state.evalFile(lookupFileArg(state, myArgs.releaseExpr), vTop); @@ -202,7 +203,7 @@ struct Drv { if (myArgs.meta) { nlohmann::json meta_; for (auto &metaName : drvInfo.queryMetaNames()) { - PathSet context; + NixStringContext context; std::stringstream ss; auto metaValue = drvInfo.queryMeta(metaName); From cace652bff0df578119bdd6eee0152c7f284c361 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sat, 10 Jun 2023 13:50:23 +0200 Subject: [PATCH 274/419] fix build on darwin --- src/meson.build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/meson.build b/src/meson.build index ef01a7d93..f8fb52881 100644 --- a/src/meson.build +++ b/src/meson.build @@ -13,4 +13,4 @@ executable('nix-eval-jobs', src, threads_dep ], install: true, - cpp_args: ['-std=c++17', '-fvisibility=hidden']) + cpp_args: ['-std=c++2a', '-fvisibility=hidden']) From 9e8320dbd64d84ca2f008f7b4f9f2451130734a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sat, 10 Jun 2023 13:51:59 +0200 Subject: [PATCH 275/419] fix c++20 warnings --- src/nix-eval-jobs.cc | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/nix-eval-jobs.cc b/src/nix-eval-jobs.cc index 0d8c97cac..2fce82cac 100644 --- a/src/nix-eval-jobs.cc +++ b/src/nix-eval-jobs.cc @@ -84,11 +84,11 @@ struct MyArgs : MixEvalArgs, MixCommonArgs { .labels = {"path"}, .handler = {&gcRootsDir}}); - addFlag( - {.longName = "workers", - .description = "number of evaluate workers", - .labels = {"workers"}, - .handler = {[=](std::string s) { nrWorkers = std::stoi(s); }}}); + addFlag({.longName = "workers", + .description = "number of evaluate workers", + .labels = {"workers"}, + .handler = { + [=, this](std::string s) { nrWorkers = std::stoi(s); }}}); addFlag( {.longName = "max-memory-size", @@ -96,7 +96,7 @@ struct MyArgs : MixEvalArgs, MixCommonArgs { "maximum evaluation memory size (4GiB per worker by default)", .labels = {"size"}, .handler = { - [=](std::string s) { maxMemorySize = std::stoi(s); }}}); + [=, this](std::string s) { maxMemorySize = std::stoi(s); }}}); addFlag({.longName = "flake", .description = "build a flake", From 3af31068771684c4040769cad5b31b902a9558c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sat, 10 Jun 2023 14:03:57 +0200 Subject: [PATCH 276/419] bump version to 2.16.0 --- default.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/default.nix b/default.nix index 4167e69c8..6a75bd59e 100644 --- a/default.nix +++ b/default.nix @@ -17,7 +17,7 @@ let in stdenv.mkDerivation rec { pname = "nix-eval-jobs"; - version = "2.14.0"; + version = "2.16.0"; src = if srcDir == null then filterMesonBuild ./. else srcDir; buildInputs = [ nlohmann_json From 9de6075c0278129c1954c1c3d3d469cee1c87b73 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 12 Jun 2023 01:26:12 +0000 Subject: [PATCH 277/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/5715d6b452b97d12c6f9077321d202a0cb50b8fc' (2023-06-08) → 'github:NixOS/nixpkgs/e1fe78d916aea75ebd68aba8782853b1fb1371f3' (2023-06-12) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index f4e0bf943..c1ed52fbd 100644 --- a/flake.lock +++ b/flake.lock @@ -22,11 +22,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1686186815, - "narHash": "sha256-h6OTafX2eLgFQpu5yJf65JHDKrEVoMLLtTn8ksu1+qE=", + "lastModified": 1686530445, + "narHash": "sha256-zpsGXHty9PsrQEZp2hr1jwPwi13n1jPFEqtPmD7L4rA=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "5715d6b452b97d12c6f9077321d202a0cb50b8fc", + "rev": "e1fe78d916aea75ebd68aba8782853b1fb1371f3", "type": "github" }, "original": { From 239f8d26c9716ed10a4fac4ca17cda10f4575343 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Mon, 12 Jun 2023 06:41:43 +0200 Subject: [PATCH 278/419] mergify: replace bors's merge queue --- .github/workflows/update-flake-lock.yml | 2 ++ .mergify.yml | 26 +++++++++++++++---------- bors.toml | 12 ------------ 3 files changed, 18 insertions(+), 22 deletions(-) delete mode 100644 bors.toml diff --git a/.github/workflows/update-flake-lock.yml b/.github/workflows/update-flake-lock.yml index 785d291a5..bb4f8d77c 100644 --- a/.github/workflows/update-flake-lock.yml +++ b/.github/workflows/update-flake-lock.yml @@ -19,3 +19,5 @@ jobs: uses: DeterminateSystems/update-flake-lock@v19 with: token: ${{ secrets.GH_TOKEN_FOR_UPDATES }} + pr-labels: | # Labels to be set on the PR + merge-queue diff --git a/.mergify.yml b/.mergify.yml index ea18fb2d1..0507bba39 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -1,15 +1,21 @@ -pull_request_rules: - - name: automatic merge on CI success - conditions: +queue_rules: + - name: default + merge_conditions: - check-success=Evaluate flake.nix - - check-success=package nix-eval-jobs [x86_64-linux] - - check-success=devShell default [x86_64-linux] - check-success=check treefmt [x86_64-linux] + - check-success=devShell default [x86_64-linux] - check-success=package default [x86_64-linux] - - check-success=tests (ubuntu-latest) + - check-success=package nix-eval-jobs [x86_64-linux] - check-success=tests (macos-latest) - - author=nix-eval-jobs-bot + - check-success=tests (ubuntu-latest) +defaults: + actions: + queue: + allow_merging_configuration_change: true +pull_request_rules: + - name: merge using the merge queue + conditions: + - base=main + - label=merge-queue actions: - merge: - method: merge - delete_head_branch: {} + queue: {} diff --git a/bors.toml b/bors.toml deleted file mode 100644 index 826bb373e..000000000 --- a/bors.toml +++ /dev/null @@ -1,12 +0,0 @@ -cut_body_after = "" # don't include text from the PR body in the merge commit message -status = [ - # garnix - "Evaluate flake.nix", - "package nix-eval-jobs [x86_64-linux]", - "devShell default [x86_64-linux]", - "check treefmt [x86_64-linux]", - "package default [x86_64-linux]", - # github actions - "tests (ubuntu-latest)", - "tests (macos-latest)" -] From 2d1d238318a7a3b1b37dac9314c1e8f32be31576 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 12 Jun 2023 04:43:52 +0000 Subject: [PATCH 279/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/e1fe78d916aea75ebd68aba8782853b1fb1371f3' (2023-06-12) → 'github:NixOS/nixpkgs/c1944ee51b8d6885aaa5470fbb010b86c01d6470' (2023-06-12) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index c1ed52fbd..4ac82b232 100644 --- a/flake.lock +++ b/flake.lock @@ -22,11 +22,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1686530445, - "narHash": "sha256-zpsGXHty9PsrQEZp2hr1jwPwi13n1jPFEqtPmD7L4rA=", + "lastModified": 1686544600, + "narHash": "sha256-QRSZuGex5W+41zqm7NHXcokkgev8WULS8xGyQEpMXtI=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "e1fe78d916aea75ebd68aba8782853b1fb1371f3", + "rev": "c1944ee51b8d6885aaa5470fbb010b86c01d6470", "type": "github" }, "original": { From 06e9c9ee7c898d95300a2ca8c47a90597d080285 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 15 Jun 2023 09:20:10 +0000 Subject: [PATCH 280/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/c1944ee51b8d6885aaa5470fbb010b86c01d6470' (2023-06-12) → 'github:NixOS/nixpkgs/66e3d3b8d9cf3ebf0d2326b8267f71fe38e7f1b8' (2023-06-15) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 4ac82b232..3b9d0e54e 100644 --- a/flake.lock +++ b/flake.lock @@ -22,11 +22,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1686544600, - "narHash": "sha256-QRSZuGex5W+41zqm7NHXcokkgev8WULS8xGyQEpMXtI=", + "lastModified": 1686819257, + "narHash": "sha256-RYrWGRd7XfaZDYy8lnrHQjSky+OaChI9UOMLmGkUgmY=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "c1944ee51b8d6885aaa5470fbb010b86c01d6470", + "rev": "66e3d3b8d9cf3ebf0d2326b8267f71fe38e7f1b8", "type": "github" }, "original": { From fa8b6db8f10a610e3ce2630538990f1666b5e38c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 Jun 2023 23:57:22 +0000 Subject: [PATCH 281/419] build(deps): bump cachix/install-nix-action from 21 to 22 Bumps [cachix/install-nix-action](https://github.com/cachix/install-nix-action) from 21 to 22. - [Release notes](https://github.com/cachix/install-nix-action/releases) - [Commits](https://github.com/cachix/install-nix-action/compare/v21...v22) --- updated-dependencies: - dependency-name: cachix/install-nix-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/tests.yml | 2 +- .github/workflows/update-flake-lock.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 1e1df2723..aacb0ae5b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -17,7 +17,7 @@ jobs: with: # Nix Flakes doesn't work on shallow clones fetch-depth: 0 - - uses: cachix/install-nix-action@v21 + - uses: cachix/install-nix-action@v22 with: github_access_token: ${{ secrets.GITHUB_TOKEN }} extra_nix_config: | diff --git a/.github/workflows/update-flake-lock.yml b/.github/workflows/update-flake-lock.yml index bb4f8d77c..324376eb8 100644 --- a/.github/workflows/update-flake-lock.yml +++ b/.github/workflows/update-flake-lock.yml @@ -11,7 +11,7 @@ jobs: - name: Checkout repository uses: actions/checkout@v3 - name: Install Nix - uses: cachix/install-nix-action@v21 + uses: cachix/install-nix-action@v22 with: extra_nix_config: | access-tokens = github.com=${{ secrets.GITHUB_TOKEN }} From 21c6125232830336e002709fc3a446beee8eab6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Tue, 20 Jun 2023 11:09:56 +0700 Subject: [PATCH 282/419] update mergify-config --- .mergify.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.mergify.yml b/.mergify.yml index 0507bba39..f4ade59ad 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -12,10 +12,11 @@ defaults: actions: queue: allow_merging_configuration_change: true + method: rebase pull_request_rules: - name: merge using the merge queue conditions: - base=main - - label=merge-queue + - label~=merge-queue|dependencies actions: queue: {} From 92739a2bbc46de55c7920a1ffba724ec9e46b0ea Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 19 Jun 2023 01:21:01 +0000 Subject: [PATCH 283/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/66e3d3b8d9cf3ebf0d2326b8267f71fe38e7f1b8' (2023-06-15) → 'github:NixOS/nixpkgs/0d1c0755a8bce318ae64b95cdbd28377aa2edf29' (2023-06-19) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 3b9d0e54e..4860f79b4 100644 --- a/flake.lock +++ b/flake.lock @@ -22,11 +22,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1686819257, - "narHash": "sha256-RYrWGRd7XfaZDYy8lnrHQjSky+OaChI9UOMLmGkUgmY=", + "lastModified": 1687136808, + "narHash": "sha256-A3lphGpoCleIr6aoABaP/VkaEArh6/Q2U7Ox18UAJEA=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "66e3d3b8d9cf3ebf0d2326b8267f71fe38e7f1b8", + "rev": "0d1c0755a8bce318ae64b95cdbd28377aa2edf29", "type": "github" }, "original": { From d9396a82ce903c5b3ad80f641042fb9cc7384812 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 22 Jun 2023 01:21:18 +0000 Subject: [PATCH 284/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/0d1c0755a8bce318ae64b95cdbd28377aa2edf29' (2023-06-19) → 'github:NixOS/nixpkgs/3fb3ce0b6b84d3b4e7b49e142da9c5764b563058' (2023-06-22) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 4860f79b4..07cde8c9b 100644 --- a/flake.lock +++ b/flake.lock @@ -22,11 +22,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1687136808, - "narHash": "sha256-A3lphGpoCleIr6aoABaP/VkaEArh6/Q2U7Ox18UAJEA=", + "lastModified": 1687392660, + "narHash": "sha256-E4bsKvHGFsKYegkfJ/FwR64OMtpjTWHM4CvCyWSTlnM=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "0d1c0755a8bce318ae64b95cdbd28377aa2edf29", + "rev": "3fb3ce0b6b84d3b4e7b49e142da9c5764b563058", "type": "github" }, "original": { From 3df60c26212d2048264ffab2a83089979cbfdaf5 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Mon, 26 Jun 2023 12:10:51 +1200 Subject: [PATCH 285/419] Add note about momentarily exceeding memory limits --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 17e521fed..ad657b4d9 100644 --- a/README.md +++ b/README.md @@ -117,6 +117,8 @@ this value as needed. For example, if you have a system with a lot of memory and want to speed up the evaluation, you may want to increase the memory limit to allow workers to cache more data in memory before getting restarted by nix-eval-jobs. +Note that this is not a hard limit and memory usage may rise above the limit momentarily +before the worker process exits. Overall, tuning these options can help you optimize the performance and memory usage of nix-eval-jobs to better fit your system and evaluation needs. From a74d3c57242f34a767d920b4b31820047c59ff92 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 26 Jun 2023 01:42:17 +0000 Subject: [PATCH 286/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/3fb3ce0b6b84d3b4e7b49e142da9c5764b563058' (2023-06-22) → 'github:NixOS/nixpkgs/4137dbc2737384c4cc9e2b5d3c4cfc6ba6e5f0cc' (2023-06-26) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 07cde8c9b..63765a405 100644 --- a/flake.lock +++ b/flake.lock @@ -22,11 +22,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1687392660, - "narHash": "sha256-E4bsKvHGFsKYegkfJ/FwR64OMtpjTWHM4CvCyWSTlnM=", + "lastModified": 1687740418, + "narHash": "sha256-hZ51wbEaMFEo5MAOR9o+h7LNVxMIOCYABT5OwFesfCU=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "3fb3ce0b6b84d3b4e7b49e142da9c5764b563058", + "rev": "4137dbc2737384c4cc9e2b5d3c4cfc6ba6e5f0cc", "type": "github" }, "original": { From db318eee754563269536c5e3513abbb9b130481a Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 29 Jun 2023 01:27:34 +0000 Subject: [PATCH 287/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'flake-parts': 'github:hercules-ci/flake-parts/71fb97f0d875fd4de4994dfb849f2c75e17eb6c3' (2023-06-01) → 'github:hercules-ci/flake-parts/37dd7bb15791c86d55c5121740a1887ab55ee836' (2023-06-26) • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/4137dbc2737384c4cc9e2b5d3c4cfc6ba6e5f0cc' (2023-06-26) → 'github:NixOS/nixpkgs/2c8591ad6a6f9d679817a94f847c59b0d1e3289e' (2023-06-29) --- flake.lock | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/flake.lock b/flake.lock index 63765a405..588713bc0 100644 --- a/flake.lock +++ b/flake.lock @@ -7,11 +7,11 @@ ] }, "locked": { - "lastModified": 1685662779, - "narHash": "sha256-cKDDciXGpMEjP1n6HlzKinN0H+oLmNpgeCTzYnsA2po=", + "lastModified": 1687762428, + "narHash": "sha256-DIf7mi45PKo+s8dOYF+UlXHzE0Wl/+k3tXUyAoAnoGE=", "owner": "hercules-ci", "repo": "flake-parts", - "rev": "71fb97f0d875fd4de4994dfb849f2c75e17eb6c3", + "rev": "37dd7bb15791c86d55c5121740a1887ab55ee836", "type": "github" }, "original": { @@ -22,11 +22,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1687740418, - "narHash": "sha256-hZ51wbEaMFEo5MAOR9o+h7LNVxMIOCYABT5OwFesfCU=", + "lastModified": 1688001024, + "narHash": "sha256-Zf88j+DUj6rDgveWfdEyUo4fL1KZTowzPAN6gpeqzKg=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "4137dbc2737384c4cc9e2b5d3c4cfc6ba6e5f0cc", + "rev": "2c8591ad6a6f9d679817a94f847c59b0d1e3289e", "type": "github" }, "original": { From 410b14bb39b73154e12b4e50ae2eb5f7c1a4967b Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 3 Jul 2023 01:38:57 +0000 Subject: [PATCH 288/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'flake-parts': 'github:hercules-ci/flake-parts/37dd7bb15791c86d55c5121740a1887ab55ee836' (2023-06-26) → 'github:hercules-ci/flake-parts/267149c58a14d15f7f81b4d737308421de9d7152' (2023-07-01) • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/2c8591ad6a6f9d679817a94f847c59b0d1e3289e' (2023-06-29) → 'github:NixOS/nixpkgs/ee5cc38432031b66e7fe395b14235eeb4b2b0d6e' (2023-07-03) --- flake.lock | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/flake.lock b/flake.lock index 588713bc0..b78300ee9 100644 --- a/flake.lock +++ b/flake.lock @@ -7,11 +7,11 @@ ] }, "locked": { - "lastModified": 1687762428, - "narHash": "sha256-DIf7mi45PKo+s8dOYF+UlXHzE0Wl/+k3tXUyAoAnoGE=", + "lastModified": 1688254665, + "narHash": "sha256-8FHEgBrr7gYNiS/NzCxIO3m4hvtLRW9YY1nYo1ivm3o=", "owner": "hercules-ci", "repo": "flake-parts", - "rev": "37dd7bb15791c86d55c5121740a1887ab55ee836", + "rev": "267149c58a14d15f7f81b4d737308421de9d7152", "type": "github" }, "original": { @@ -22,11 +22,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1688001024, - "narHash": "sha256-Zf88j+DUj6rDgveWfdEyUo4fL1KZTowzPAN6gpeqzKg=", + "lastModified": 1688346760, + "narHash": "sha256-w6JFZsZ+qEJNaBrYUmqKAbA8+qXWm5pwMQHhbNfpAYE=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "2c8591ad6a6f9d679817a94f847c59b0d1e3289e", + "rev": "ee5cc38432031b66e7fe395b14235eeb4b2b0d6e", "type": "github" }, "original": { From 477d7196a493dd011f05704fc7b42cbe95f5b30d Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 6 Jul 2023 01:40:30 +0000 Subject: [PATCH 289/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'flake-parts': 'github:hercules-ci/flake-parts/267149c58a14d15f7f81b4d737308421de9d7152' (2023-07-01) → 'github:hercules-ci/flake-parts/8e8d955c22df93dbe24f19ea04f47a74adbdc5ec' (2023-07-04) • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/ee5cc38432031b66e7fe395b14235eeb4b2b0d6e' (2023-07-03) → 'github:NixOS/nixpkgs/ff81c24d1dd4dc3698aeb27d2cc3991124e627e6' (2023-07-06) --- flake.lock | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/flake.lock b/flake.lock index b78300ee9..c89fb4127 100644 --- a/flake.lock +++ b/flake.lock @@ -7,11 +7,11 @@ ] }, "locked": { - "lastModified": 1688254665, - "narHash": "sha256-8FHEgBrr7gYNiS/NzCxIO3m4hvtLRW9YY1nYo1ivm3o=", + "lastModified": 1688466019, + "narHash": "sha256-VeM2akYrBYMsb4W/MmBo1zmaMfgbL4cH3Pu8PGyIwJ0=", "owner": "hercules-ci", "repo": "flake-parts", - "rev": "267149c58a14d15f7f81b4d737308421de9d7152", + "rev": "8e8d955c22df93dbe24f19ea04f47a74adbdc5ec", "type": "github" }, "original": { @@ -22,11 +22,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1688346760, - "narHash": "sha256-w6JFZsZ+qEJNaBrYUmqKAbA8+qXWm5pwMQHhbNfpAYE=", + "lastModified": 1688607075, + "narHash": "sha256-KDWpwZ4xl4au5R+A+Ka+uVbyiwMDVczjwRTSqBOyqWM=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "ee5cc38432031b66e7fe395b14235eeb4b2b0d6e", + "rev": "ff81c24d1dd4dc3698aeb27d2cc3991124e627e6", "type": "github" }, "original": { From 4006da54d54e1243da26ad4d75d6b4c9f7a456ba Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 10 Jul 2023 01:38:29 +0000 Subject: [PATCH 290/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/ff81c24d1dd4dc3698aeb27d2cc3991124e627e6' (2023-07-06) → 'github:NixOS/nixpkgs/2a5f6cac357616d2596167d0631b4ca729e9a3ea' (2023-07-10) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index c89fb4127..8b2437f9b 100644 --- a/flake.lock +++ b/flake.lock @@ -22,11 +22,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1688607075, - "narHash": "sha256-KDWpwZ4xl4au5R+A+Ka+uVbyiwMDVczjwRTSqBOyqWM=", + "lastModified": 1688951312, + "narHash": "sha256-0oG4uv60m5+oOMqgYYQ3ao3OK3YP3n3t7nWFtuyR/uQ=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "ff81c24d1dd4dc3698aeb27d2cc3991124e627e6", + "rev": "2a5f6cac357616d2596167d0631b4ca729e9a3ea", "type": "github" }, "original": { From cc9fa47406449cab4bb9a79aa969e715f4d461e9 Mon Sep 17 00:00:00 2001 From: Andrea Bedini Date: Thu, 13 Jul 2023 17:45:06 +0800 Subject: [PATCH 291/419] Add clangStdenv --- default.nix | 21 ++++++++------------- flake.nix | 1 + 2 files changed, 9 insertions(+), 13 deletions(-) diff --git a/default.nix b/default.nix index 6a75bd59e..2c41d21c7 100644 --- a/default.nix +++ b/default.nix @@ -1,36 +1,31 @@ { stdenv , lib , nix -, meson -, cmake -, ninja -, pkg-config -, boost -, nlohmann_json +, pkgs , srcDir ? null }: let - filterMesonBuild = dir: builtins.filterSource - (path: type: type != "directory" || baseNameOf path != "build") - dir; + filterMesonBuild = builtins.filterSource + (path: type: type != "directory" || baseNameOf path != "build"); in -stdenv.mkDerivation rec { +stdenv.mkDerivation { pname = "nix-eval-jobs"; version = "2.16.0"; src = if srcDir == null then filterMesonBuild ./. else srcDir; - buildInputs = [ + buildInputs = with pkgs; [ nlohmann_json nix boost ]; - nativeBuildInputs = [ + nativeBuildInputs = with pkgs; [ + bear meson pkg-config ninja # nlohmann_json can be only discovered via cmake files cmake - ]; + ] ++ (lib.optional stdenv.cc.isClang [ pkgs.bear pkgs.clang-tools ]); meta = { description = "Hydra's builtin hydra-eval-jobs as a standalone"; diff --git a/flake.nix b/flake.nix index 24efad70d..097cffb76 100644 --- a/flake.nix +++ b/flake.nix @@ -31,6 +31,7 @@ in { packages.nix-eval-jobs = pkgs.callPackage ./default.nix drvArgs; + packages.clangStdenv-nix-eval-jobs = pkgs.callPackage ./default.nix (drvArgs // { stdenv = pkgs.clangStdenv; }); checks.treefmt = pkgs.stdenv.mkDerivation { name = "treefmt-check"; From ab07651f7409782187806cddb9e79830d19f5971 Mon Sep 17 00:00:00 2001 From: Andrea Bedini Date: Fri, 14 Jul 2023 12:34:20 +0800 Subject: [PATCH 292/419] Small refactor to avoid optional values --- src/nix-eval-jobs.cc | 39 ++++++++++++++++++--------------------- 1 file changed, 18 insertions(+), 21 deletions(-) diff --git a/src/nix-eval-jobs.cc b/src/nix-eval-jobs.cc index 2fce82cac..72f784db4 100644 --- a/src/nix-eval-jobs.cc +++ b/src/nix-eval-jobs.cc @@ -154,12 +154,6 @@ static Value *releaseExprTopLevelValue(EvalState &state, Bindings &autoArgs) { return vRoot; } -Value *topLevelValue(EvalState &state, Bindings &autoArgs, - std::optional flake) { - return flake.has_value() ? flake.value().toValue(state).first - : releaseExprTopLevelValue(state, autoArgs); -} - bool queryIsCached(Store &store, std::map &outputs) { uint64_t downloadSize, narSize; StorePathSet willBuild, willSubstitute, unknown; @@ -264,22 +258,25 @@ std::string attrPathJoin(json input) { static void worker(ref state, Bindings &autoArgs, AutoCloseFD &to, AutoCloseFD &from) { - std::optional flake; - if (myArgs.flake) { - auto [flakeRef, fragment, outputSpec] = - parseFlakeRefWithFragmentAndExtendedOutputsSpec(myArgs.releaseExpr, - absPath(".")); + nix::Value *vRoot = [&]() { + if (myArgs.flake) { + auto [flakeRef, fragment, outputSpec] = + parseFlakeRefWithFragmentAndExtendedOutputsSpec(myArgs.releaseExpr, + absPath(".")); + InstallableFlake flake { + {}, state, std::move(flakeRef), fragment, + outputSpec, {}, {}, + flake::LockFlags{ + .updateLockFile = false, + .useRegistries = false, + .allowUnlocked = false, + }}; - flake.emplace(InstallableFlake({}, state, std::move(flakeRef), fragment, - outputSpec, {}, {}, - flake::LockFlags{ - .updateLockFile = false, - .useRegistries = false, - .allowUnlocked = false, - })); - }; - - auto vRoot = topLevelValue(*state, autoArgs, flake); + return flake.toValue(*state).first; + } else { + return releaseExprTopLevelValue(*state, autoArgs); + } + }(); while (true) { /* Wait for the collector to send us a job name. */ From 15e5f5f7c8d7fd054813ed112ea5d6b267b860bc Mon Sep 17 00:00:00 2001 From: Andrea Bedini Date: Fri, 14 Jul 2023 13:03:41 +0800 Subject: [PATCH 293/419] Add --override-input --- src/nix-eval-jobs.cc | 31 +++++++++++++++++++++++++------ 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/src/nix-eval-jobs.cc b/src/nix-eval-jobs.cc index 72f784db4..8f8b00ad1 100644 --- a/src/nix-eval-jobs.cc +++ b/src/nix-eval-jobs.cc @@ -53,6 +53,14 @@ struct MyArgs : MixEvalArgs, MixCommonArgs { size_t nrWorkers = 1; size_t maxMemorySize = 4096; + // usually in MixFlakeOptions + flake::LockFlags lockFlags = { + .updateLockFile = false, + .writeLockFile = false, + .useRegistries = false, + .allowUnlocked = false + }; + MyArgs() : MixCommonArgs("nix-eval-jobs") { addFlag({ .longName = "help", @@ -125,6 +133,21 @@ struct MyArgs : MixEvalArgs, MixCommonArgs { .description = "treat the argument as a Nix expression", .handler = {&fromArgs, true}}); + // usually in MixFlakeOptions + addFlag({ + .longName = "override-input", + .description = "Override a specific flake input (e.g. `dwarffs/nixpkgs`).", + .category = category, + .labels = {"input-path", "flake-url"}, + .handler = {[&](std::string inputPath, std::string flakeRef) { + // overriden inputs are unlocked + lockFlags.allowUnlocked = true; + lockFlags.inputOverrides.insert_or_assign( + flake::parseInputPath(inputPath), + parseFlakeRef(flakeRef, absPath("."), true)); + }}, + }); + expectArg("expr", &releaseExpr); } }; @@ -265,12 +288,8 @@ static void worker(ref state, Bindings &autoArgs, AutoCloseFD &to, absPath(".")); InstallableFlake flake { {}, state, std::move(flakeRef), fragment, - outputSpec, {}, {}, - flake::LockFlags{ - .updateLockFile = false, - .useRegistries = false, - .allowUnlocked = false, - }}; + outputSpec, {}, {}, myArgs.lockFlags + }; return flake.toValue(*state).first; } else { From 22a2008a683736c766288d8dd627f0eaec707d1b Mon Sep 17 00:00:00 2001 From: Andrea Bedini Date: Fri, 14 Jul 2023 14:46:10 +0800 Subject: [PATCH 294/419] Move clang-format config to .clang-format --- .clang-format | 3 +++ treefmt.toml | 6 +----- 2 files changed, 4 insertions(+), 5 deletions(-) create mode 100644 .clang-format diff --git a/.clang-format b/.clang-format new file mode 100644 index 000000000..b389fd02b --- /dev/null +++ b/.clang-format @@ -0,0 +1,3 @@ +BasedOnStyle: llvm +IndentWidth: 4 +SortIncludes: false diff --git a/treefmt.toml b/treefmt.toml index 19a28ed6d..647cb784a 100644 --- a/treefmt.toml +++ b/treefmt.toml @@ -1,10 +1,6 @@ [formatter."c++"] command = "clang-format" -options = [ - "-i", - "-style", - "{BasedOnStyle: llvm, IndentWidth: 4, SortIncludes: false}" -] +options = ["-i"] includes = ["*.c", "*.cpp", "*.cc", "*.h", "*.hpp"] [formatter.nix] From f88571cfc9132e8f2768aa41d57f5f471941d4b6 Mon Sep 17 00:00:00 2001 From: Andrea Bedini Date: Fri, 14 Jul 2023 14:50:35 +0800 Subject: [PATCH 295/419] Fix formatting --- src/nix-eval-jobs.cc | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/src/nix-eval-jobs.cc b/src/nix-eval-jobs.cc index 8f8b00ad1..75cda1468 100644 --- a/src/nix-eval-jobs.cc +++ b/src/nix-eval-jobs.cc @@ -54,12 +54,10 @@ struct MyArgs : MixEvalArgs, MixCommonArgs { size_t maxMemorySize = 4096; // usually in MixFlakeOptions - flake::LockFlags lockFlags = { - .updateLockFile = false, - .writeLockFile = false, - .useRegistries = false, - .allowUnlocked = false - }; + flake::LockFlags lockFlags = {.updateLockFile = false, + .writeLockFile = false, + .useRegistries = false, + .allowUnlocked = false}; MyArgs() : MixCommonArgs("nix-eval-jobs") { addFlag({ @@ -136,7 +134,8 @@ struct MyArgs : MixEvalArgs, MixCommonArgs { // usually in MixFlakeOptions addFlag({ .longName = "override-input", - .description = "Override a specific flake input (e.g. `dwarffs/nixpkgs`).", + .description = + "Override a specific flake input (e.g. `dwarffs/nixpkgs`).", .category = category, .labels = {"input-path", "flake-url"}, .handler = {[&](std::string inputPath, std::string flakeRef) { @@ -284,12 +283,11 @@ static void worker(ref state, Bindings &autoArgs, AutoCloseFD &to, nix::Value *vRoot = [&]() { if (myArgs.flake) { auto [flakeRef, fragment, outputSpec] = - parseFlakeRefWithFragmentAndExtendedOutputsSpec(myArgs.releaseExpr, - absPath(".")); - InstallableFlake flake { - {}, state, std::move(flakeRef), fragment, - outputSpec, {}, {}, myArgs.lockFlags - }; + parseFlakeRefWithFragmentAndExtendedOutputsSpec( + myArgs.releaseExpr, absPath(".")); + InstallableFlake flake{ + {}, state, std::move(flakeRef), fragment, outputSpec, + {}, {}, myArgs.lockFlags}; return flake.toValue(*state).first; } else { From a1673cdc91ea51bc720cd55db99659b0d05b55bf Mon Sep 17 00:00:00 2001 From: adisbladis Date: Fri, 21 Jul 2023 13:23:16 +1200 Subject: [PATCH 296/419] Use treefmt-nix --- dev/treefmt.nix | 43 +++++++++++++++++++++++++++++++++++++++++++ flake.lock | 23 ++++++++++++++++++++++- flake.nix | 20 ++++---------------- shell.nix | 7 ------- treefmt.toml | 17 ----------------- 5 files changed, 69 insertions(+), 41 deletions(-) create mode 100644 dev/treefmt.nix delete mode 100644 treefmt.toml diff --git a/dev/treefmt.nix b/dev/treefmt.nix new file mode 100644 index 000000000..6fb3b1ade --- /dev/null +++ b/dev/treefmt.nix @@ -0,0 +1,43 @@ +{ pkgs, lib, ... }: { + # Used to find the project root + projectRootFile = "flake.lock"; + + programs.prettier.enable = true; + programs.prettier.package = pkgs.writeShellScriptBin "prettier" '' + export NODE_PATH=${pkgs.nodePackages.prettier-plugin-toml}/lib/node_modules + exec ${lib.getExe pkgs.nodePackages.prettier} "$@" + ''; + + programs.clang-format.enable = true; + + settings.formatter = { + nix = { + command = "sh"; + options = [ + "-eucx" + '' + ${pkgs.lib.getExe pkgs.nixpkgs-fmt} "$@" + '' + "--" + ]; + includes = [ "*.nix" ]; + excludes = [ ]; + }; + + clang-format = { }; + + prettier.includes = lib.mkForce [ "*.toml" ]; + + python = { + command = "sh"; + options = [ + "-eucx" + '' + ${pkgs.lib.getExe pkgs.python3.pkgs.black} "$@" + '' + "--" # this argument is ignored by bash + ]; + includes = [ "*.py" ]; + }; + }; +} diff --git a/flake.lock b/flake.lock index 8b2437f9b..1556e0c15 100644 --- a/flake.lock +++ b/flake.lock @@ -39,7 +39,28 @@ "root": { "inputs": { "flake-parts": "flake-parts", - "nixpkgs": "nixpkgs" + "nixpkgs": "nixpkgs", + "treefmt-nix": "treefmt-nix" + } + }, + "treefmt-nix": { + "inputs": { + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1689620039, + "narHash": "sha256-BtNwghr05z7k5YMdq+6nbue+nEalvDepuA7qdQMAKoQ=", + "owner": "numtide", + "repo": "treefmt-nix", + "rev": "719c2977f958c41fa60a928e2fbc50af14844114", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "treefmt-nix", + "type": "github" } } }, diff --git a/flake.nix b/flake.nix index 097cffb76..fadf545b1 100644 --- a/flake.nix +++ b/flake.nix @@ -4,6 +4,8 @@ inputs.nixpkgs.url = "github:NixOS/nixpkgs/master"; inputs.flake-parts.url = "github:hercules-ci/flake-parts"; inputs.flake-parts.inputs.nixpkgs-lib.follows = "nixpkgs"; + inputs.treefmt-nix.url = "github:numtide/treefmt-nix"; + inputs.treefmt-nix.inputs.nixpkgs.follows = "nixpkgs"; nixConfig.extra-substituters = [ "https://cache.garnix.io" @@ -21,6 +23,7 @@ flake-parts.lib.mkFlake { inherit inputs; } { systems = inputs.nixpkgs.lib.systems.flakeExposed; + imports = [ inputs.treefmt-nix.flakeModule ]; perSystem = { pkgs, self', ... }: let devShell = self'.devShells.default; @@ -30,24 +33,9 @@ }; in { + treefmt.imports = [ ./dev/treefmt.nix ]; packages.nix-eval-jobs = pkgs.callPackage ./default.nix drvArgs; packages.clangStdenv-nix-eval-jobs = pkgs.callPackage ./default.nix (drvArgs // { stdenv = pkgs.clangStdenv; }); - - checks.treefmt = pkgs.stdenv.mkDerivation { - name = "treefmt-check"; - src = self; - nativeBuildInputs = devShell.nativeBuildInputs; - dontConfigure = true; - - inherit (devShell) NODE_PATH; - - buildPhase = '' - env HOME=$(mktemp -d) treefmt --fail-on-change - ''; - - installPhase = "touch $out"; - }; - packages.default = self'.packages.nix-eval-jobs; devShells.default = pkgs.callPackage ./shell.nix drvArgs; }; diff --git a/shell.nix b/shell.nix index a4bda273f..c24738f7b 100644 --- a/shell.nix +++ b/shell.nix @@ -23,18 +23,11 @@ in pkgs.mkShell { inherit (nix-eval-jobs) buildInputs; nativeBuildInputs = nix-eval-jobs.nativeBuildInputs ++ [ - pkgs.treefmt - pkgs.llvmPackages.clang # clang-format - pkgs.nixpkgs-fmt - pkgs.nodePackages.prettier - (pkgs.python3.withPackages (ps: [ ps.pytest - ps.black ])) ]; - NODE_PATH = "${pkgs.nodePackages.prettier-plugin-toml}/lib/node_modules"; shellHook = lib.optionalString stdenv.isLinux '' export NIX_DEBUG_INFO_DIRS="${pkgs.curl.debug}/lib/debug:${nix.debug}/lib/debug''${NIX_DEBUG_INFO_DIRS:+:$NIX_DEBUG_INFO_DIRS}" diff --git a/treefmt.toml b/treefmt.toml deleted file mode 100644 index 647cb784a..000000000 --- a/treefmt.toml +++ /dev/null @@ -1,17 +0,0 @@ -[formatter."c++"] -command = "clang-format" -options = ["-i"] -includes = ["*.c", "*.cpp", "*.cc", "*.h", "*.hpp"] - -[formatter.nix] -command = "nixpkgs-fmt" -includes = ["*.nix"] - -[formatter.toml] -command = "prettier" -options = ["--write"] -includes = ["*.toml"] - -[formatter.python] -command = "black" -includes = ["*.py"] From 276b68bade14a749d530a949fc94613754a57ca8 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Fri, 21 Jul 2023 13:24:29 +1200 Subject: [PATCH 297/419] Apply statix/deadnix --- dev/treefmt.nix | 6 ++++++ flake.nix | 1 - shell.nix | 2 +- tests/assets/flake.nix | 2 +- 4 files changed, 8 insertions(+), 3 deletions(-) diff --git a/dev/treefmt.nix b/dev/treefmt.nix index 6fb3b1ade..2372dfff2 100644 --- a/dev/treefmt.nix +++ b/dev/treefmt.nix @@ -16,6 +16,12 @@ options = [ "-eucx" '' + ${pkgs.lib.getExe pkgs.deadnix} --edit "$@" + + for i in "$@"; do + ${pkgs.lib.getExe pkgs.statix} fix "$i" + done + ${pkgs.lib.getExe pkgs.nixpkgs-fmt} "$@" '' "--" diff --git a/flake.nix b/flake.nix index fadf545b1..1acfcaf4f 100644 --- a/flake.nix +++ b/flake.nix @@ -26,7 +26,6 @@ imports = [ inputs.treefmt-nix.flakeModule ]; perSystem = { pkgs, self', ... }: let - devShell = self'.devShells.default; drvArgs = { srcDir = self; nix = if nixVersion == "unstable" then pkgs.nixUnstable else pkgs.nixVersions."nix_${nixVersion}"; diff --git a/shell.nix b/shell.nix index c24738f7b..a94d8814f 100644 --- a/shell.nix +++ b/shell.nix @@ -2,7 +2,7 @@ let inherit (builtins) fromJSON readFile; flakeLock = fromJSON (readFile ./flake.lock); - locked = flakeLock.nodes.nixpkgs.locked; + inherit (flakeLock.nodes.nixpkgs) locked; nixpkgs = assert locked.type == "github"; builtins.fetchTarball { url = "https://github.com/${locked.owner}/${locked.repo}/archive/${locked.rev}.tar.gz"; sha256 = locked.narHash; diff --git a/tests/assets/flake.nix b/tests/assets/flake.nix index bedd2257f..aa28425f2 100644 --- a/tests/assets/flake.nix +++ b/tests/assets/flake.nix @@ -1,7 +1,7 @@ { inputs.nixpkgs.url = "github:NixOS/nixpkgs"; - outputs = { self, nixpkgs }: + outputs = { nixpkgs, ... }: let pkgs = nixpkgs.legacyPackages.x86_64-linux; in From fcaf7773e3a6713b78dacfd442339a96cfab405f Mon Sep 17 00:00:00 2001 From: adisbladis Date: Fri, 21 Jul 2023 13:25:44 +1200 Subject: [PATCH 298/419] Apply ruff --- dev/treefmt.nix | 1 + tests/test_eval.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/dev/treefmt.nix b/dev/treefmt.nix index 2372dfff2..6aa768b5b 100644 --- a/dev/treefmt.nix +++ b/dev/treefmt.nix @@ -40,6 +40,7 @@ "-eucx" '' ${pkgs.lib.getExe pkgs.python3.pkgs.black} "$@" + ${pkgs.lib.getExe pkgs.ruff} --fix "$@" '' "--" # this argument is ignored by bash ]; diff --git a/tests/test_eval.py b/tests/test_eval.py index a08332e60..c64c8b1af 100644 --- a/tests/test_eval.py +++ b/tests/test_eval.py @@ -55,7 +55,8 @@ def test_flake() -> None: def test_query_cache_status() -> None: results = common_test(["--flake", ".#hydraJobs", "--check-cache-status"]) - # FIXME in the nix sandbox we cannot query binary caches, this would need some local one + # FIXME in the nix sandbox we cannot query binary caches + # this would need some local one for result in results: assert "isCached" in result From f360004d785937aa0a9ed19bbeec864dec37737b Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 27 Jul 2023 01:08:53 +0000 Subject: [PATCH 299/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/2a5f6cac357616d2596167d0631b4ca729e9a3ea' (2023-07-10) → 'github:NixOS/nixpkgs/6d8c4bd21e6f9fb6c67bb599459f35cf9486e18b' (2023-07-27) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 1556e0c15..15beeb785 100644 --- a/flake.lock +++ b/flake.lock @@ -22,11 +22,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1688951312, - "narHash": "sha256-0oG4uv60m5+oOMqgYYQ3ao3OK3YP3n3t7nWFtuyR/uQ=", + "lastModified": 1690420098, + "narHash": "sha256-47kn22jGYlCeHQxdmqoZeR2jHhZpJjxYFgCXdClVadA=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "2a5f6cac357616d2596167d0631b4ca729e9a3ea", + "rev": "6d8c4bd21e6f9fb6c67bb599459f35cf9486e18b", "type": "github" }, "original": { From 6a57b81709e4b78ec022076bec2cf7dc75eb2c48 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 31 Jul 2023 01:13:37 +0000 Subject: [PATCH 300/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/6d8c4bd21e6f9fb6c67bb599459f35cf9486e18b' (2023-07-27) → 'github:NixOS/nixpkgs/7efa5777e2924792bac419dd6d6960d924925c3c' (2023-07-31) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 15beeb785..9b3eb3117 100644 --- a/flake.lock +++ b/flake.lock @@ -22,11 +22,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1690420098, - "narHash": "sha256-47kn22jGYlCeHQxdmqoZeR2jHhZpJjxYFgCXdClVadA=", + "lastModified": 1690764218, + "narHash": "sha256-SAKu5hE5QN7l2qn8fm7mR60kswaII/Cp2xCQKbYNPjY=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "6d8c4bd21e6f9fb6c67bb599459f35cf9486e18b", + "rev": "7efa5777e2924792bac419dd6d6960d924925c3c", "type": "github" }, "original": { From a749e7c748ca8981a71b7fc36cfdb810a8834f8d Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 3 Aug 2023 01:11:03 +0000 Subject: [PATCH 301/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'flake-parts': 'github:hercules-ci/flake-parts/8e8d955c22df93dbe24f19ea04f47a74adbdc5ec' (2023-07-04) → 'github:hercules-ci/flake-parts/59cf3f1447cfc75087e7273b04b31e689a8599fb' (2023-08-01) • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/7efa5777e2924792bac419dd6d6960d924925c3c' (2023-07-31) → 'github:NixOS/nixpkgs/f78cfdd5ae4f4543cf0c27bfcc017232f5ed8905' (2023-08-03) • Updated input 'treefmt-nix': 'github:numtide/treefmt-nix/719c2977f958c41fa60a928e2fbc50af14844114' (2023-07-17) → 'github:numtide/treefmt-nix/fab56c8ce88f593300cd8c7351c9f97d10c333c5' (2023-08-01) --- flake.lock | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/flake.lock b/flake.lock index 9b3eb3117..fa7cda944 100644 --- a/flake.lock +++ b/flake.lock @@ -7,11 +7,11 @@ ] }, "locked": { - "lastModified": 1688466019, - "narHash": "sha256-VeM2akYrBYMsb4W/MmBo1zmaMfgbL4cH3Pu8PGyIwJ0=", + "lastModified": 1690933134, + "narHash": "sha256-ab989mN63fQZBFrkk4Q8bYxQCktuHmBIBqUG1jl6/FQ=", "owner": "hercules-ci", "repo": "flake-parts", - "rev": "8e8d955c22df93dbe24f19ea04f47a74adbdc5ec", + "rev": "59cf3f1447cfc75087e7273b04b31e689a8599fb", "type": "github" }, "original": { @@ -22,11 +22,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1690764218, - "narHash": "sha256-SAKu5hE5QN7l2qn8fm7mR60kswaII/Cp2xCQKbYNPjY=", + "lastModified": 1691024687, + "narHash": "sha256-NlMDQSweO5xC+EE0eGsBAAuFQ0J73FMzopBmWesonok=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "7efa5777e2924792bac419dd6d6960d924925c3c", + "rev": "f78cfdd5ae4f4543cf0c27bfcc017232f5ed8905", "type": "github" }, "original": { @@ -50,11 +50,11 @@ ] }, "locked": { - "lastModified": 1689620039, - "narHash": "sha256-BtNwghr05z7k5YMdq+6nbue+nEalvDepuA7qdQMAKoQ=", + "lastModified": 1690874496, + "narHash": "sha256-qYZJVAfilFbUL6U+euMjKLXUADueMNQBqwihpNzTbDU=", "owner": "numtide", "repo": "treefmt-nix", - "rev": "719c2977f958c41fa60a928e2fbc50af14844114", + "rev": "fab56c8ce88f593300cd8c7351c9f97d10c333c5", "type": "github" }, "original": { From 68c4ce65a0dfbb1974298533eb5c6e0ba001ffb7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sun, 6 Aug 2023 18:49:32 +0200 Subject: [PATCH 302/419] treefmt: fix eval warnings --- dev/treefmt.nix | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/dev/treefmt.nix b/dev/treefmt.nix index 6aa768b5b..3f4c03566 100644 --- a/dev/treefmt.nix +++ b/dev/treefmt.nix @@ -5,7 +5,7 @@ programs.prettier.enable = true; programs.prettier.package = pkgs.writeShellScriptBin "prettier" '' export NODE_PATH=${pkgs.nodePackages.prettier-plugin-toml}/lib/node_modules - exec ${lib.getExe pkgs.nodePackages.prettier} "$@" + exec ${pkgs.nodePackages.prettier}/bin/prettier "$@" ''; programs.clang-format.enable = true; @@ -16,13 +16,13 @@ options = [ "-eucx" '' - ${pkgs.lib.getExe pkgs.deadnix} --edit "$@" + ${pkgs.deadnix}/bin/deadnix --edit "$@" for i in "$@"; do - ${pkgs.lib.getExe pkgs.statix} fix "$i" + ${pkgs.statix}/bin/statix fix "$i" done - ${pkgs.lib.getExe pkgs.nixpkgs-fmt} "$@" + ${pkgs.nixpkgs-fmt}/bin/nixpkgs-fmt "$@" '' "--" ]; @@ -39,8 +39,8 @@ options = [ "-eucx" '' - ${pkgs.lib.getExe pkgs.python3.pkgs.black} "$@" - ${pkgs.lib.getExe pkgs.ruff} --fix "$@" + ${pkgs.python3.pkgs.black}/bin/black "$@" + ${pkgs.ruff}/bin/ruff --fix "$@" '' "--" # this argument is ignored by bash ]; From b02b4e287fddc969fc490478b5666603f4ab0d3c Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 7 Aug 2023 01:12:03 +0000 Subject: [PATCH 303/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/f78cfdd5ae4f4543cf0c27bfcc017232f5ed8905' (2023-08-03) → 'github:NixOS/nixpkgs/b51660a128c09baf31c614284b500eb53772496f' (2023-08-07) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index fa7cda944..098e8aeb9 100644 --- a/flake.lock +++ b/flake.lock @@ -22,11 +22,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1691024687, - "narHash": "sha256-NlMDQSweO5xC+EE0eGsBAAuFQ0J73FMzopBmWesonok=", + "lastModified": 1691370583, + "narHash": "sha256-LnKMx9NQ0Qx0DTYQVewkcRr+7uW5NY7xU9kjh+Lxnb0=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "f78cfdd5ae4f4543cf0c27bfcc017232f5ed8905", + "rev": "b51660a128c09baf31c614284b500eb53772496f", "type": "github" }, "original": { From d20f2a3c0d962e7e5ef868759d36d4ebbd0a707c Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 10 Aug 2023 01:13:31 +0000 Subject: [PATCH 304/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/b51660a128c09baf31c614284b500eb53772496f' (2023-08-07) → 'github:NixOS/nixpkgs/cd165f66c1551c602e6473b094708920f6d26fbd' (2023-08-10) • Updated input 'treefmt-nix': 'github:numtide/treefmt-nix/fab56c8ce88f593300cd8c7351c9f97d10c333c5' (2023-08-01) → 'github:numtide/treefmt-nix/7b380d3cce8271b37394790b521ec2f7a6b248ad' (2023-08-08) --- flake.lock | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/flake.lock b/flake.lock index 098e8aeb9..91e16512e 100644 --- a/flake.lock +++ b/flake.lock @@ -22,11 +22,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1691370583, - "narHash": "sha256-LnKMx9NQ0Qx0DTYQVewkcRr+7uW5NY7xU9kjh+Lxnb0=", + "lastModified": 1691629382, + "narHash": "sha256-6bil2OX12qy2CD6dLDxSTKRu6aUKRZfT/Qw3pg1050Q=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "b51660a128c09baf31c614284b500eb53772496f", + "rev": "cd165f66c1551c602e6473b094708920f6d26fbd", "type": "github" }, "original": { @@ -50,11 +50,11 @@ ] }, "locked": { - "lastModified": 1690874496, - "narHash": "sha256-qYZJVAfilFbUL6U+euMjKLXUADueMNQBqwihpNzTbDU=", + "lastModified": 1691522377, + "narHash": "sha256-1LafgFJaSk53ccsTlI2gWSmIyxRJfFVyoaGJg0c3LjM=", "owner": "numtide", "repo": "treefmt-nix", - "rev": "fab56c8ce88f593300cd8c7351c9f97d10c333c5", + "rev": "7b380d3cce8271b37394790b521ec2f7a6b248ad", "type": "github" }, "original": { From 9c42d241950e9e24b8dec1b04e36d2a9e61e9078 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 14 Aug 2023 01:00:02 +0000 Subject: [PATCH 305/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/cd165f66c1551c602e6473b094708920f6d26fbd' (2023-08-10) → 'github:NixOS/nixpkgs/a1fa57b4ba546ef8869237ae1341f393cb9b5806' (2023-08-14) • Updated input 'treefmt-nix': 'github:numtide/treefmt-nix/7b380d3cce8271b37394790b521ec2f7a6b248ad' (2023-08-08) → 'github:numtide/treefmt-nix/19dee4bf6001849006a63f3435247316b0488e99' (2023-08-12) --- flake.lock | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/flake.lock b/flake.lock index 91e16512e..344b28972 100644 --- a/flake.lock +++ b/flake.lock @@ -22,11 +22,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1691629382, - "narHash": "sha256-6bil2OX12qy2CD6dLDxSTKRu6aUKRZfT/Qw3pg1050Q=", + "lastModified": 1691974608, + "narHash": "sha256-S268S0UKxY/24m2peKN8v/HUO1Hi6CB4cfxZDUZf5do=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "cd165f66c1551c602e6473b094708920f6d26fbd", + "rev": "a1fa57b4ba546ef8869237ae1341f393cb9b5806", "type": "github" }, "original": { @@ -50,11 +50,11 @@ ] }, "locked": { - "lastModified": 1691522377, - "narHash": "sha256-1LafgFJaSk53ccsTlI2gWSmIyxRJfFVyoaGJg0c3LjM=", + "lastModified": 1691833704, + "narHash": "sha256-ASGhgGduEgcD3gQZhGr8xtmZ3PlVY+m2HuPnIZDbu78=", "owner": "numtide", "repo": "treefmt-nix", - "rev": "7b380d3cce8271b37394790b521ec2f7a6b248ad", + "rev": "19dee4bf6001849006a63f3435247316b0488e99", "type": "github" }, "original": { From d98c3253176892bba3cfcf240528ffda19490b82 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 17 Aug 2023 00:58:22 +0000 Subject: [PATCH 306/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/a1fa57b4ba546ef8869237ae1341f393cb9b5806' (2023-08-14) → 'github:NixOS/nixpkgs/ecf0aba4c9e096196f67a862606ba521c67a3e42' (2023-08-17) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 344b28972..bbf8f86b0 100644 --- a/flake.lock +++ b/flake.lock @@ -22,11 +22,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1691974608, - "narHash": "sha256-S268S0UKxY/24m2peKN8v/HUO1Hi6CB4cfxZDUZf5do=", + "lastModified": 1692233877, + "narHash": "sha256-gtYcf70quK6YBuWsDER3ayz4Tc0h1jbDDrsvQqiMXms=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "a1fa57b4ba546ef8869237ae1341f393cb9b5806", + "rev": "ecf0aba4c9e096196f67a862606ba521c67a3e42", "type": "github" }, "original": { From a830f68be9ea5632b0b7ed92ad86eb0ba417d846 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Tue, 22 Aug 2023 21:34:30 +0200 Subject: [PATCH 307/419] drop optional bear --- default.nix | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/default.nix b/default.nix index 2c41d21c7..f858e9f0f 100644 --- a/default.nix +++ b/default.nix @@ -19,13 +19,12 @@ stdenv.mkDerivation { boost ]; nativeBuildInputs = with pkgs; [ - bear meson pkg-config ninja # nlohmann_json can be only discovered via cmake files cmake - ] ++ (lib.optional stdenv.cc.isClang [ pkgs.bear pkgs.clang-tools ]); + ] ++ (lib.optional stdenv.cc.isClang [ pkgs.clang-tools ]); meta = { description = "Hydra's builtin hydra-eval-jobs as a standalone"; From 68851ed4d0d0c52df831320e1b6a3fdac5cc4684 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 21 Aug 2023 00:59:25 +0000 Subject: [PATCH 308/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/ecf0aba4c9e096196f67a862606ba521c67a3e42' (2023-08-17) → 'github:NixOS/nixpkgs/36ec59c0ff9bd8fd40e355b87c67a91a6cb02309' (2023-08-21) • Updated input 'treefmt-nix': 'github:numtide/treefmt-nix/19dee4bf6001849006a63f3435247316b0488e99' (2023-08-12) → 'github:numtide/treefmt-nix/e2761d701581d8dcc4e0e88aecfde317ddf6f0cd' (2023-08-20) --- flake.lock | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/flake.lock b/flake.lock index bbf8f86b0..36dbff0b0 100644 --- a/flake.lock +++ b/flake.lock @@ -22,11 +22,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1692233877, - "narHash": "sha256-gtYcf70quK6YBuWsDER3ayz4Tc0h1jbDDrsvQqiMXms=", + "lastModified": 1692579448, + "narHash": "sha256-vTZnXtxFYxglBxp2egE7UweHrdDT3Ojzeb9SbrqbgrE=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "ecf0aba4c9e096196f67a862606ba521c67a3e42", + "rev": "36ec59c0ff9bd8fd40e355b87c67a91a6cb02309", "type": "github" }, "original": { @@ -50,11 +50,11 @@ ] }, "locked": { - "lastModified": 1691833704, - "narHash": "sha256-ASGhgGduEgcD3gQZhGr8xtmZ3PlVY+m2HuPnIZDbu78=", + "lastModified": 1692524468, + "narHash": "sha256-wJffwu1deOgc3c/cBIZQ52dfWfPWBzjOamYBX121hcw=", "owner": "numtide", "repo": "treefmt-nix", - "rev": "19dee4bf6001849006a63f3435247316b0488e99", + "rev": "e2761d701581d8dcc4e0e88aecfde317ddf6f0cd", "type": "github" }, "original": { From 3681d5930d1479898758b752f2d77be0c8e0f90f Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 24 Aug 2023 00:58:59 +0000 Subject: [PATCH 309/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/36ec59c0ff9bd8fd40e355b87c67a91a6cb02309' (2023-08-21) → 'github:NixOS/nixpkgs/6408fedbfacd0d323edc2512f6033dbce818672a' (2023-08-24) • Updated input 'treefmt-nix': 'github:numtide/treefmt-nix/e2761d701581d8dcc4e0e88aecfde317ddf6f0cd' (2023-08-20) → 'github:numtide/treefmt-nix/b070c28bf9d7d3ef93084aa47c01b4b6c16cdce4' (2023-08-23) --- flake.lock | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/flake.lock b/flake.lock index 36dbff0b0..7da994dbc 100644 --- a/flake.lock +++ b/flake.lock @@ -22,11 +22,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1692579448, - "narHash": "sha256-vTZnXtxFYxglBxp2egE7UweHrdDT3Ojzeb9SbrqbgrE=", + "lastModified": 1692838642, + "narHash": "sha256-Y47k7ckDN2nlH+A2hdfgNulXqKkTX8WdWCfd6l0ys0Y=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "36ec59c0ff9bd8fd40e355b87c67a91a6cb02309", + "rev": "6408fedbfacd0d323edc2512f6033dbce818672a", "type": "github" }, "original": { @@ -50,11 +50,11 @@ ] }, "locked": { - "lastModified": 1692524468, - "narHash": "sha256-wJffwu1deOgc3c/cBIZQ52dfWfPWBzjOamYBX121hcw=", + "lastModified": 1692792358, + "narHash": "sha256-yqKPLUvl9lFTy43+GvVRwT39k1qu7Yd0HNktZjRbUP4=", "owner": "numtide", "repo": "treefmt-nix", - "rev": "e2761d701581d8dcc4e0e88aecfde317ddf6f0cd", + "rev": "b070c28bf9d7d3ef93084aa47c01b4b6c16cdce4", "type": "github" }, "original": { From 7eb201cdfcc7dfde07935134c5ddb3b5d721865a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 28 Aug 2023 23:09:11 +0000 Subject: [PATCH 310/419] build(deps): bump DeterminateSystems/update-flake-lock from 19 to 20 Bumps [DeterminateSystems/update-flake-lock](https://github.com/determinatesystems/update-flake-lock) from 19 to 20. - [Release notes](https://github.com/determinatesystems/update-flake-lock/releases) - [Commits](https://github.com/determinatesystems/update-flake-lock/compare/v19...v20) --- updated-dependencies: - dependency-name: DeterminateSystems/update-flake-lock dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/update-flake-lock.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/update-flake-lock.yml b/.github/workflows/update-flake-lock.yml index 324376eb8..d60edb97a 100644 --- a/.github/workflows/update-flake-lock.yml +++ b/.github/workflows/update-flake-lock.yml @@ -16,7 +16,7 @@ jobs: extra_nix_config: | access-tokens = github.com=${{ secrets.GITHUB_TOKEN }} - name: Update flake.lock - uses: DeterminateSystems/update-flake-lock@v19 + uses: DeterminateSystems/update-flake-lock@v20 with: token: ${{ secrets.GH_TOKEN_FOR_UPDATES }} pr-labels: | # Labels to be set on the PR From 76994681653503b6691af1cacd3aca07d76665bc Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 31 Aug 2023 01:00:37 +0000 Subject: [PATCH 311/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/6408fedbfacd0d323edc2512f6033dbce818672a' (2023-08-24) → 'github:NixOS/nixpkgs/e35f1fd9763f2b11b3f723b88287949cc2afd37c' (2023-08-31) • Updated input 'treefmt-nix': 'github:numtide/treefmt-nix/b070c28bf9d7d3ef93084aa47c01b4b6c16cdce4' (2023-08-23) → 'github:numtide/treefmt-nix/6befd3b6b8544952e0261f054cf16769294bacba' (2023-08-28) --- flake.lock | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/flake.lock b/flake.lock index 7da994dbc..20ac8c4be 100644 --- a/flake.lock +++ b/flake.lock @@ -22,11 +22,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1692838642, - "narHash": "sha256-Y47k7ckDN2nlH+A2hdfgNulXqKkTX8WdWCfd6l0ys0Y=", + "lastModified": 1693443141, + "narHash": "sha256-uvXHx0ysNeEPC4UjPiwNgX/8z9ItBS4GLkcM7ency+w=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "6408fedbfacd0d323edc2512f6033dbce818672a", + "rev": "e35f1fd9763f2b11b3f723b88287949cc2afd37c", "type": "github" }, "original": { @@ -50,11 +50,11 @@ ] }, "locked": { - "lastModified": 1692792358, - "narHash": "sha256-yqKPLUvl9lFTy43+GvVRwT39k1qu7Yd0HNktZjRbUP4=", + "lastModified": 1693247164, + "narHash": "sha256-M6qZo8H8fBFnipCy6q6RlpSXF3sDvfTEtyFwdAP7juM=", "owner": "numtide", "repo": "treefmt-nix", - "rev": "b070c28bf9d7d3ef93084aa47c01b4b6c16cdce4", + "rev": "6befd3b6b8544952e0261f054cf16769294bacba", "type": "github" }, "original": { From 3b6256633b7ead3275ffab509a0595e7abc76e08 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 4 Sep 2023 01:01:34 +0000 Subject: [PATCH 312/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'flake-parts': 'github:hercules-ci/flake-parts/59cf3f1447cfc75087e7273b04b31e689a8599fb' (2023-08-01) → 'github:hercules-ci/flake-parts/7f53fdb7bdc5bb237da7fefef12d099e4fd611ca' (2023-09-01) • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/e35f1fd9763f2b11b3f723b88287949cc2afd37c' (2023-08-31) → 'github:NixOS/nixpkgs/f53ec4f6d815f80f7ee6a490a946b1b2b4f9cb09' (2023-09-04) • Updated input 'treefmt-nix': 'github:numtide/treefmt-nix/6befd3b6b8544952e0261f054cf16769294bacba' (2023-08-28) → 'github:numtide/treefmt-nix/e3e0f9f6d47f8fc68aff15150eda1224fb46f4d4' (2023-09-02) --- flake.lock | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/flake.lock b/flake.lock index 20ac8c4be..c29effadd 100644 --- a/flake.lock +++ b/flake.lock @@ -7,11 +7,11 @@ ] }, "locked": { - "lastModified": 1690933134, - "narHash": "sha256-ab989mN63fQZBFrkk4Q8bYxQCktuHmBIBqUG1jl6/FQ=", + "lastModified": 1693611461, + "narHash": "sha256-aPODl8vAgGQ0ZYFIRisxYG5MOGSkIczvu2Cd8Gb9+1Y=", "owner": "hercules-ci", "repo": "flake-parts", - "rev": "59cf3f1447cfc75087e7273b04b31e689a8599fb", + "rev": "7f53fdb7bdc5bb237da7fefef12d099e4fd611ca", "type": "github" }, "original": { @@ -22,11 +22,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1693443141, - "narHash": "sha256-uvXHx0ysNeEPC4UjPiwNgX/8z9ItBS4GLkcM7ency+w=", + "lastModified": 1693785888, + "narHash": "sha256-RTJnFrFaLsQGzg0VwiGfR+aNbhcIaP267C92YC2i3mE=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "e35f1fd9763f2b11b3f723b88287949cc2afd37c", + "rev": "f53ec4f6d815f80f7ee6a490a946b1b2b4f9cb09", "type": "github" }, "original": { @@ -50,11 +50,11 @@ ] }, "locked": { - "lastModified": 1693247164, - "narHash": "sha256-M6qZo8H8fBFnipCy6q6RlpSXF3sDvfTEtyFwdAP7juM=", + "lastModified": 1693689099, + "narHash": "sha256-NuilTRYMH+DDR/uBWQjDbX5mWCA05lwo2Sg9iTkkEs4=", "owner": "numtide", "repo": "treefmt-nix", - "rev": "6befd3b6b8544952e0261f054cf16769294bacba", + "rev": "e3e0f9f6d47f8fc68aff15150eda1224fb46f4d4", "type": "github" }, "original": { From 0fdb0a8519c365b2f8c2f6ee33b953d8beaca957 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Sep 2023 23:06:31 +0000 Subject: [PATCH 313/419] build(deps): bump cachix/install-nix-action from 22 to 23 Bumps [cachix/install-nix-action](https://github.com/cachix/install-nix-action) from 22 to 23. - [Release notes](https://github.com/cachix/install-nix-action/releases) - [Commits](https://github.com/cachix/install-nix-action/compare/v22...v23) --- updated-dependencies: - dependency-name: cachix/install-nix-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/tests.yml | 2 +- .github/workflows/update-flake-lock.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index aacb0ae5b..60807b9fc 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -17,7 +17,7 @@ jobs: with: # Nix Flakes doesn't work on shallow clones fetch-depth: 0 - - uses: cachix/install-nix-action@v22 + - uses: cachix/install-nix-action@v23 with: github_access_token: ${{ secrets.GITHUB_TOKEN }} extra_nix_config: | diff --git a/.github/workflows/update-flake-lock.yml b/.github/workflows/update-flake-lock.yml index d60edb97a..050a66d6d 100644 --- a/.github/workflows/update-flake-lock.yml +++ b/.github/workflows/update-flake-lock.yml @@ -11,7 +11,7 @@ jobs: - name: Checkout repository uses: actions/checkout@v3 - name: Install Nix - uses: cachix/install-nix-action@v22 + uses: cachix/install-nix-action@v23 with: extra_nix_config: | access-tokens = github.com=${{ secrets.GITHUB_TOKEN }} From ff16969e651bc8ccf3d21c16f249368bc63817df Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Sep 2023 23:16:06 +0000 Subject: [PATCH 314/419] build(deps): bump actions/checkout from 3 to 4 Bumps [actions/checkout](https://github.com/actions/checkout) from 3 to 4. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v3...v4) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/tests.yml | 2 +- .github/workflows/update-flake-lock.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 60807b9fc..b26284cd9 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -13,7 +13,7 @@ jobs: os: [ ubuntu-latest, macos-latest ] runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: # Nix Flakes doesn't work on shallow clones fetch-depth: 0 diff --git a/.github/workflows/update-flake-lock.yml b/.github/workflows/update-flake-lock.yml index 050a66d6d..c7c53fae9 100644 --- a/.github/workflows/update-flake-lock.yml +++ b/.github/workflows/update-flake-lock.yml @@ -9,7 +9,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Install Nix uses: cachix/install-nix-action@v23 with: From 9cc7944c28c2bc5de3c96c253b3fd814c1cb85dd Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 7 Sep 2023 01:00:12 +0000 Subject: [PATCH 315/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/f53ec4f6d815f80f7ee6a490a946b1b2b4f9cb09' (2023-09-04) → 'github:NixOS/nixpkgs/308e5f73e17dc2fe43ba95ec83697999b5dd544d' (2023-09-07) • Updated input 'treefmt-nix': 'github:numtide/treefmt-nix/e3e0f9f6d47f8fc68aff15150eda1224fb46f4d4' (2023-09-02) → 'github:numtide/treefmt-nix/b8d3a059f5487d6767d07c3716386753e3132d9f' (2023-09-04) --- flake.lock | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/flake.lock b/flake.lock index c29effadd..2c87e2a27 100644 --- a/flake.lock +++ b/flake.lock @@ -22,11 +22,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1693785888, - "narHash": "sha256-RTJnFrFaLsQGzg0VwiGfR+aNbhcIaP267C92YC2i3mE=", + "lastModified": 1694048283, + "narHash": "sha256-QexXMDukc4fmXq5SJsDg8WRA6+FiEOt+PB3hx+fbc8o=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "f53ec4f6d815f80f7ee6a490a946b1b2b4f9cb09", + "rev": "308e5f73e17dc2fe43ba95ec83697999b5dd544d", "type": "github" }, "original": { @@ -50,11 +50,11 @@ ] }, "locked": { - "lastModified": 1693689099, - "narHash": "sha256-NuilTRYMH+DDR/uBWQjDbX5mWCA05lwo2Sg9iTkkEs4=", + "lastModified": 1693817438, + "narHash": "sha256-fg3+n4Ky1gCzDtPm0MomMTFw0YkH05Y8ojy5t7bkfHg=", "owner": "numtide", "repo": "treefmt-nix", - "rev": "e3e0f9f6d47f8fc68aff15150eda1224fb46f4d4", + "rev": "b8d3a059f5487d6767d07c3716386753e3132d9f", "type": "github" }, "original": { From 4af42f97d330cd1b83d851cb7aa5c235eb2e85c9 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 11 Sep 2023 01:01:15 +0000 Subject: [PATCH 316/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/308e5f73e17dc2fe43ba95ec83697999b5dd544d' (2023-09-07) → 'github:NixOS/nixpkgs/ca40349951374b558bc49465f92f1ff8856f095d' (2023-09-11) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 2c87e2a27..98eb39d35 100644 --- a/flake.lock +++ b/flake.lock @@ -22,11 +22,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1694048283, - "narHash": "sha256-QexXMDukc4fmXq5SJsDg8WRA6+FiEOt+PB3hx+fbc8o=", + "lastModified": 1694393089, + "narHash": "sha256-jUJs+1e7eTcXvG3+Muoytq8kVBmGak0Ylo3yn8sVYBg=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "308e5f73e17dc2fe43ba95ec83697999b5dd544d", + "rev": "ca40349951374b558bc49465f92f1ff8856f095d", "type": "github" }, "original": { From 15ec2c466356b3267abe0fd993b5d8992c73381f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Tue, 12 Sep 2023 09:01:51 +0200 Subject: [PATCH 317/419] print derivation name if system attribute is missing --- src/nix-eval-jobs.cc | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/src/nix-eval-jobs.cc b/src/nix-eval-jobs.cc index 75cda1468..8acdc64d1 100644 --- a/src/nix-eval-jobs.cc +++ b/src/nix-eval-jobs.cc @@ -96,13 +96,13 @@ struct MyArgs : MixEvalArgs, MixCommonArgs { .handler = { [=, this](std::string s) { nrWorkers = std::stoi(s); }}}); - addFlag( - {.longName = "max-memory-size", - .description = - "maximum evaluation memory size (4GiB per worker by default)", - .labels = {"size"}, - .handler = { - [=, this](std::string s) { maxMemorySize = std::stoi(s); }}}); + addFlag({.longName = "max-memory-size", + .description = "maximum evaluation memory size in megabyte " + "(4GiB per worker by default)", + .labels = {"size"}, + .handler = {[=, this](std::string s) { + maxMemorySize = std::stoi(s); + }}}); addFlag({.longName = "flake", .description = "build a flake", @@ -201,8 +201,11 @@ struct Drv { std::optional meta; Drv(EvalState &state, DrvInfo &drvInfo) { - if (drvInfo.querySystem() == "unknown") - throw EvalError("derivation must have a 'system' attribute"); + name = drvInfo.queryName(); + system = drvInfo.querySystem(); + if (system == "unknown") + throw EvalError("derivation '" + name + + "' must have a 'system' attribute"); auto localStore = state.store.dynamic_pointer_cast(); @@ -239,8 +242,6 @@ struct Drv { isCached = queryIsCached(*localStore, outputs); } - name = drvInfo.queryName(); - system = drvInfo.querySystem(); drvPath = localStore->printStorePath(drvInfo.requireDrvPath()); auto drv = localStore->readDerivation(drvInfo.requireDrvPath()); From 3e635f33fb31b39305ff378ed66149a4b3715985 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 14 Sep 2023 01:00:25 +0000 Subject: [PATCH 318/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/ca40349951374b558bc49465f92f1ff8856f095d' (2023-09-11) → 'github:NixOS/nixpkgs/46ea94edba83944a236850bbc0bfd92785736b00' (2023-09-14) • Updated input 'treefmt-nix': 'github:numtide/treefmt-nix/b8d3a059f5487d6767d07c3716386753e3132d9f' (2023-09-04) → 'github:numtide/treefmt-nix/7a49c388d7a6b63bb551b1ddedfa4efab8f400d8' (2023-09-12) --- flake.lock | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/flake.lock b/flake.lock index 98eb39d35..2ee73a74e 100644 --- a/flake.lock +++ b/flake.lock @@ -22,11 +22,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1694393089, - "narHash": "sha256-jUJs+1e7eTcXvG3+Muoytq8kVBmGak0Ylo3yn8sVYBg=", + "lastModified": 1694651847, + "narHash": "sha256-W+2eI96glLiEwLnX/kWn5HDO7WfKKkF0lKW9yyNLEbY=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "ca40349951374b558bc49465f92f1ff8856f095d", + "rev": "46ea94edba83944a236850bbc0bfd92785736b00", "type": "github" }, "original": { @@ -50,11 +50,11 @@ ] }, "locked": { - "lastModified": 1693817438, - "narHash": "sha256-fg3+n4Ky1gCzDtPm0MomMTFw0YkH05Y8ojy5t7bkfHg=", + "lastModified": 1694528738, + "narHash": "sha256-aWMEjib5oTqEzF9f3WXffC1cwICo6v/4dYKjwNktV8k=", "owner": "numtide", "repo": "treefmt-nix", - "rev": "b8d3a059f5487d6767d07c3716386753e3132d9f", + "rev": "7a49c388d7a6b63bb551b1ddedfa4efab8f400d8", "type": "github" }, "original": { From a91f3595b22037f561912cd3a9ca549933e4544d Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 18 Sep 2023 01:01:22 +0000 Subject: [PATCH 319/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/46ea94edba83944a236850bbc0bfd92785736b00' (2023-09-14) → 'github:NixOS/nixpkgs/5b859eef2e5dd7aacfd229e819f426942eed25fc' (2023-09-18) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 2ee73a74e..7a1ca4e67 100644 --- a/flake.lock +++ b/flake.lock @@ -22,11 +22,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1694651847, - "narHash": "sha256-W+2eI96glLiEwLnX/kWn5HDO7WfKKkF0lKW9yyNLEbY=", + "lastModified": 1694998849, + "narHash": "sha256-A23ROwLGc+lbgUZOkHMhsJ+3IMC+5MmRXXl61iEuhhQ=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "46ea94edba83944a236850bbc0bfd92785736b00", + "rev": "5b859eef2e5dd7aacfd229e819f426942eed25fc", "type": "github" }, "original": { From 39657d146828157ef51c4f2d8bebb96a77075fc6 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 21 Sep 2023 01:00:47 +0000 Subject: [PATCH 320/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/5b859eef2e5dd7aacfd229e819f426942eed25fc' (2023-09-18) → 'github:NixOS/nixpkgs/ff7daa56614b083d3a87e2872917b676e9ba62a6' (2023-09-21) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 7a1ca4e67..9d35e7894 100644 --- a/flake.lock +++ b/flake.lock @@ -22,11 +22,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1694998849, - "narHash": "sha256-A23ROwLGc+lbgUZOkHMhsJ+3IMC+5MmRXXl61iEuhhQ=", + "lastModified": 1695256509, + "narHash": "sha256-Je+ZId+dYrx0NOZ8J6le7CwZZdVZAAP5dddxK9kZNfA=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "5b859eef2e5dd7aacfd229e819f426942eed25fc", + "rev": "ff7daa56614b083d3a87e2872917b676e9ba62a6", "type": "github" }, "original": { From 82cede4edd01989095040b55d0212d61a65fc5fd Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 2 Oct 2023 01:02:26 +0000 Subject: [PATCH 321/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'flake-parts': 'github:hercules-ci/flake-parts/7f53fdb7bdc5bb237da7fefef12d099e4fd611ca' (2023-09-01) → 'github:hercules-ci/flake-parts/21928e6758af0a258002647d14363d5ffc85545b' (2023-10-01) • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/ff7daa56614b083d3a87e2872917b676e9ba62a6' (2023-09-21) → 'github:NixOS/nixpkgs/fe0b3b663e98c85db7f08ab3a4ac318c523c0684' (2023-10-02) • Updated input 'treefmt-nix': 'github:numtide/treefmt-nix/7a49c388d7a6b63bb551b1ddedfa4efab8f400d8' (2023-09-12) → 'github:numtide/treefmt-nix/720bd006d855b08e60664e4683ccddb7a9ff614a' (2023-09-27) --- flake.lock | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/flake.lock b/flake.lock index 9d35e7894..59039a2b6 100644 --- a/flake.lock +++ b/flake.lock @@ -7,11 +7,11 @@ ] }, "locked": { - "lastModified": 1693611461, - "narHash": "sha256-aPODl8vAgGQ0ZYFIRisxYG5MOGSkIczvu2Cd8Gb9+1Y=", + "lastModified": 1696203690, + "narHash": "sha256-774XMEL7VHSTLDYVkqrbl5GCdmkVKsjMs+KLM4N4t7k=", "owner": "hercules-ci", "repo": "flake-parts", - "rev": "7f53fdb7bdc5bb237da7fefef12d099e4fd611ca", + "rev": "21928e6758af0a258002647d14363d5ffc85545b", "type": "github" }, "original": { @@ -22,11 +22,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1695256509, - "narHash": "sha256-Je+ZId+dYrx0NOZ8J6le7CwZZdVZAAP5dddxK9kZNfA=", + "lastModified": 1696207572, + "narHash": "sha256-w24NTSMrc7bMIQP5Y8BFsKbpYjbRh/+ptf/9gCEFrKo=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "ff7daa56614b083d3a87e2872917b676e9ba62a6", + "rev": "fe0b3b663e98c85db7f08ab3a4ac318c523c0684", "type": "github" }, "original": { @@ -50,11 +50,11 @@ ] }, "locked": { - "lastModified": 1694528738, - "narHash": "sha256-aWMEjib5oTqEzF9f3WXffC1cwICo6v/4dYKjwNktV8k=", + "lastModified": 1695822946, + "narHash": "sha256-IQU3fYo0H+oGlqX5YrgZU3VRhbt2Oqe6KmslQKUO4II=", "owner": "numtide", "repo": "treefmt-nix", - "rev": "7a49c388d7a6b63bb551b1ddedfa4efab8f400d8", + "rev": "720bd006d855b08e60664e4683ccddb7a9ff614a", "type": "github" }, "original": { From 6841d05ad796d57ecb34e8f5a3910f8fe5211b84 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 5 Oct 2023 01:02:06 +0000 Subject: [PATCH 322/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'flake-parts': 'github:hercules-ci/flake-parts/21928e6758af0a258002647d14363d5ffc85545b' (2023-10-01) → 'github:hercules-ci/flake-parts/c9afaba3dfa4085dbd2ccb38dfade5141e33d9d4' (2023-10-03) • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/fe0b3b663e98c85db7f08ab3a4ac318c523c0684' (2023-10-02) → 'github:NixOS/nixpkgs/c52af267ad0c11b55f89cf6c70adb10694ad938e' (2023-10-05) --- flake.lock | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/flake.lock b/flake.lock index 59039a2b6..0a57867b2 100644 --- a/flake.lock +++ b/flake.lock @@ -7,11 +7,11 @@ ] }, "locked": { - "lastModified": 1696203690, - "narHash": "sha256-774XMEL7VHSTLDYVkqrbl5GCdmkVKsjMs+KLM4N4t7k=", + "lastModified": 1696343447, + "narHash": "sha256-B2xAZKLkkeRFG5XcHHSXXcP7To9Xzr59KXeZiRf4vdQ=", "owner": "hercules-ci", "repo": "flake-parts", - "rev": "21928e6758af0a258002647d14363d5ffc85545b", + "rev": "c9afaba3dfa4085dbd2ccb38dfade5141e33d9d4", "type": "github" }, "original": { @@ -22,11 +22,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1696207572, - "narHash": "sha256-w24NTSMrc7bMIQP5Y8BFsKbpYjbRh/+ptf/9gCEFrKo=", + "lastModified": 1696466515, + "narHash": "sha256-SQJyUBoLXmPGueYTLj1yDVHolg2pnB+rUR4Z6p5AKpA=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "fe0b3b663e98c85db7f08ab3a4ac318c523c0684", + "rev": "c52af267ad0c11b55f89cf6c70adb10694ad938e", "type": "github" }, "original": { From 56f0464288aa390a8b0c57e35a282709fee409fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Tue, 12 Sep 2023 09:01:51 +0200 Subject: [PATCH 323/419] use platform instead of querySystem() nix-build uses "system" from the derivation rather than the derivation attributes --- src/nix-eval-jobs.cc | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/nix-eval-jobs.cc b/src/nix-eval-jobs.cc index 8acdc64d1..39f410cfd 100644 --- a/src/nix-eval-jobs.cc +++ b/src/nix-eval-jobs.cc @@ -201,11 +201,6 @@ struct Drv { std::optional meta; Drv(EvalState &state, DrvInfo &drvInfo) { - name = drvInfo.queryName(); - system = drvInfo.querySystem(); - if (system == "unknown") - throw EvalError("derivation '" + name + - "' must have a 'system' attribute"); auto localStore = state.store.dynamic_pointer_cast(); @@ -248,6 +243,8 @@ struct Drv { for (auto &input : drv.inputDrvs) { inputDrvs[localStore->printStorePath(input.first)] = input.second; } + name = drvInfo.queryName(); + system = drv.platform; } }; From 7cdbfd5ffe59fe54fd5c44be96f58c45e25d5b62 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 9 Oct 2023 01:02:05 +0000 Subject: [PATCH 324/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/c52af267ad0c11b55f89cf6c70adb10694ad938e' (2023-10-05) → 'github:NixOS/nixpkgs/35c640b19a189ce3a86698ce2fdcd87d085a339b' (2023-10-09) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 0a57867b2..d8420d16e 100644 --- a/flake.lock +++ b/flake.lock @@ -22,11 +22,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1696466515, - "narHash": "sha256-SQJyUBoLXmPGueYTLj1yDVHolg2pnB+rUR4Z6p5AKpA=", + "lastModified": 1696810678, + "narHash": "sha256-XAw8D1ZEbdqwhSvn8RsgeeNrDktx4YSikTb5V4ArsrA=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "c52af267ad0c11b55f89cf6c70adb10694ad938e", + "rev": "35c640b19a189ce3a86698ce2fdcd87d085a339b", "type": "github" }, "original": { From bdf17c44b19325b5476703400cbafe64f7553fa6 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 16 Oct 2023 01:03:36 +0000 Subject: [PATCH 325/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/35c640b19a189ce3a86698ce2fdcd87d085a339b' (2023-10-09) → 'github:NixOS/nixpkgs/21f56f3209c0272852be7a704d9b21f2601c72e3' (2023-10-16) • Updated input 'treefmt-nix': 'github:numtide/treefmt-nix/720bd006d855b08e60664e4683ccddb7a9ff614a' (2023-09-27) → 'github:numtide/treefmt-nix/aae39f64f5ecbe89792d05eacea5cb241891292a' (2023-10-15) --- flake.lock | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/flake.lock b/flake.lock index d8420d16e..499d2446e 100644 --- a/flake.lock +++ b/flake.lock @@ -22,11 +22,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1696810678, - "narHash": "sha256-XAw8D1ZEbdqwhSvn8RsgeeNrDktx4YSikTb5V4ArsrA=", + "lastModified": 1697417052, + "narHash": "sha256-QyFpNZ28H0IoWhbGxD4j2h3aYwap2l2rSWyoFue95sM=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "35c640b19a189ce3a86698ce2fdcd87d085a339b", + "rev": "21f56f3209c0272852be7a704d9b21f2601c72e3", "type": "github" }, "original": { @@ -50,11 +50,11 @@ ] }, "locked": { - "lastModified": 1695822946, - "narHash": "sha256-IQU3fYo0H+oGlqX5YrgZU3VRhbt2Oqe6KmslQKUO4II=", + "lastModified": 1697388351, + "narHash": "sha256-63N2eBpKaziIy4R44vjpUu8Nz5fCJY7okKrkixvDQmY=", "owner": "numtide", "repo": "treefmt-nix", - "rev": "720bd006d855b08e60664e4683ccddb7a9ff614a", + "rev": "aae39f64f5ecbe89792d05eacea5cb241891292a", "type": "github" }, "original": { From 01a606e119963957eefaf1b22ef92b69b90f5b85 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 19 Oct 2023 01:02:15 +0000 Subject: [PATCH 326/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/21f56f3209c0272852be7a704d9b21f2601c72e3' (2023-10-16) → 'github:NixOS/nixpkgs/18e505d654892d057f308c817d220faf962dbf23' (2023-10-19) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 499d2446e..76030c8cf 100644 --- a/flake.lock +++ b/flake.lock @@ -22,11 +22,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1697417052, - "narHash": "sha256-QyFpNZ28H0IoWhbGxD4j2h3aYwap2l2rSWyoFue95sM=", + "lastModified": 1697677194, + "narHash": "sha256-lN2eJCsOzjhxrvTQsNcW7r0E9hMJ7ABrKDQWpmYFRkM=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "21f56f3209c0272852be7a704d9b21f2601c72e3", + "rev": "18e505d654892d057f308c817d220faf962dbf23", "type": "github" }, "original": { From 783357db13b4c3da84fbaf5ee2266b8d2b2768f2 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 2 Nov 2023 01:02:17 +0000 Subject: [PATCH 327/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'flake-parts': 'github:hercules-ci/flake-parts/c9afaba3dfa4085dbd2ccb38dfade5141e33d9d4' (2023-10-03) → 'github:hercules-ci/flake-parts/8c9fa2545007b49a5db5f650ae91f227672c3877' (2023-11-01) • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/18e505d654892d057f308c817d220faf962dbf23' (2023-10-19) → 'github:NixOS/nixpkgs/2c732a9b5a5a60d91c685c92b87db5b8f5cf5812' (2023-11-01) • Updated input 'treefmt-nix': 'github:numtide/treefmt-nix/aae39f64f5ecbe89792d05eacea5cb241891292a' (2023-10-15) → 'github:numtide/treefmt-nix/5deb8dc125a9f83b65ca86cf0c8167c46593e0b1' (2023-10-27) --- flake.lock | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/flake.lock b/flake.lock index 76030c8cf..09a9eff89 100644 --- a/flake.lock +++ b/flake.lock @@ -7,11 +7,11 @@ ] }, "locked": { - "lastModified": 1696343447, - "narHash": "sha256-B2xAZKLkkeRFG5XcHHSXXcP7To9Xzr59KXeZiRf4vdQ=", + "lastModified": 1698882062, + "narHash": "sha256-HkhafUayIqxXyHH1X8d9RDl1M2CkFgZLjKD3MzabiEo=", "owner": "hercules-ci", "repo": "flake-parts", - "rev": "c9afaba3dfa4085dbd2ccb38dfade5141e33d9d4", + "rev": "8c9fa2545007b49a5db5f650ae91f227672c3877", "type": "github" }, "original": { @@ -22,11 +22,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1697677194, - "narHash": "sha256-lN2eJCsOzjhxrvTQsNcW7r0E9hMJ7ABrKDQWpmYFRkM=", + "lastModified": 1698877057, + "narHash": "sha256-8fZ72oKQxJvzJjsm8kikcwLNFs96mZZaWYJV1NFnK6c=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "18e505d654892d057f308c817d220faf962dbf23", + "rev": "2c732a9b5a5a60d91c685c92b87db5b8f5cf5812", "type": "github" }, "original": { @@ -50,11 +50,11 @@ ] }, "locked": { - "lastModified": 1697388351, - "narHash": "sha256-63N2eBpKaziIy4R44vjpUu8Nz5fCJY7okKrkixvDQmY=", + "lastModified": 1698438538, + "narHash": "sha256-AWxaKTDL3MtxaVTVU5lYBvSnlspOS0Fjt8GxBgnU0Do=", "owner": "numtide", "repo": "treefmt-nix", - "rev": "aae39f64f5ecbe89792d05eacea5cb241891292a", + "rev": "5deb8dc125a9f83b65ca86cf0c8167c46593e0b1", "type": "github" }, "original": { From dd8affe264b133a39e5771bb316b568742ce9fd4 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Fri, 3 Nov 2023 15:40:43 +1300 Subject: [PATCH 328/419] Add missing include for eval-settings.hh --- src/nix-eval-jobs.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/nix-eval-jobs.cc b/src/nix-eval-jobs.cc index 39f410cfd..f7e290387 100644 --- a/src/nix-eval-jobs.cc +++ b/src/nix-eval-jobs.cc @@ -3,6 +3,7 @@ #include #include +#include #include #include #include From 5ac2dd6281bc6510646a2e22fd2e5147487964f0 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Fri, 3 Nov 2023 15:43:20 +1300 Subject: [PATCH 329/419] Bump nixpkgs --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 09a9eff89..3983f18bd 100644 --- a/flake.lock +++ b/flake.lock @@ -22,11 +22,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1698877057, - "narHash": "sha256-8fZ72oKQxJvzJjsm8kikcwLNFs96mZZaWYJV1NFnK6c=", + "lastModified": 1698977350, + "narHash": "sha256-OUDOHWrX3EjX/MlOoCHEb3JMONklbpu4Wa+Xf5s/U+s=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "2c732a9b5a5a60d91c685c92b87db5b8f5cf5812", + "rev": "4285a2a67daf39e63d9564a47773a1c2081c36a8", "type": "github" }, "original": { From cb9c7cac6bb527904db44ffc75bd8049a6503be5 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Fri, 3 Nov 2023 16:00:52 +1300 Subject: [PATCH 330/419] Fix iterating over input derivation outputs --- src/nix-eval-jobs.cc | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/nix-eval-jobs.cc b/src/nix-eval-jobs.cc index f7e290387..4dfcdb8f0 100644 --- a/src/nix-eval-jobs.cc +++ b/src/nix-eval-jobs.cc @@ -241,8 +241,13 @@ struct Drv { drvPath = localStore->printStorePath(drvInfo.requireDrvPath()); auto drv = localStore->readDerivation(drvInfo.requireDrvPath()); - for (auto &input : drv.inputDrvs) { - inputDrvs[localStore->printStorePath(input.first)] = input.second; + for (const auto &[inputDrvPath, inputNode] : drv.inputDrvs.map) { + std::set inputDrvOutputs; + for (auto &outputName : inputNode.value) { + inputDrvOutputs.insert(outputName); + } + inputDrvs[localStore->printStorePath(inputDrvPath)] = + inputDrvOutputs; } name = drvInfo.queryName(); system = drv.platform; From a64814310ae7b9560c5619e391e05f0907300914 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Sat, 4 Nov 2023 12:44:04 +1300 Subject: [PATCH 331/419] update-flack-lock action -> renovate --- .github/workflows/update-flake-lock.yml | 23 ----------------------- .mergify.yml | 9 ++------- renovate.json | 15 +++++++++++++++ 3 files changed, 17 insertions(+), 30 deletions(-) delete mode 100644 .github/workflows/update-flake-lock.yml create mode 100644 renovate.json diff --git a/.github/workflows/update-flake-lock.yml b/.github/workflows/update-flake-lock.yml deleted file mode 100644 index c7c53fae9..000000000 --- a/.github/workflows/update-flake-lock.yml +++ /dev/null @@ -1,23 +0,0 @@ -name: update-flake-lock -on: - workflow_dispatch: # allows manual triggering - schedule: - - cron: '0 0 * * 1,4' # Run twice a week - -jobs: - lockfile: - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - name: Install Nix - uses: cachix/install-nix-action@v23 - with: - extra_nix_config: | - access-tokens = github.com=${{ secrets.GITHUB_TOKEN }} - - name: Update flake.lock - uses: DeterminateSystems/update-flake-lock@v20 - with: - token: ${{ secrets.GH_TOKEN_FOR_UPDATES }} - pr-labels: | # Labels to be set on the PR - merge-queue diff --git a/.mergify.yml b/.mergify.yml index f4ade59ad..a8cf23afc 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -1,13 +1,8 @@ queue_rules: - name: default merge_conditions: - - check-success=Evaluate flake.nix - - check-success=check treefmt [x86_64-linux] - - check-success=devShell default [x86_64-linux] - - check-success=package default [x86_64-linux] - - check-success=package nix-eval-jobs [x86_64-linux] - - check-success=tests (macos-latest) - - check-success=tests (ubuntu-latest) + - author=renovate[bot] + - check-success=collect defaults: actions: queue: diff --git a/renovate.json b/renovate.json new file mode 100644 index 000000000..03086d773 --- /dev/null +++ b/renovate.json @@ -0,0 +1,15 @@ +{ + "$schema": "https://docs.renovatebot.com/renovate-schema.json", + "extends": [ + "config:base" + ], + "lockFileMaintenance": { + "enabled": true, + "extends": [ + "schedule:weekly" + ] + }, + "nix": { + "enabled": true + } +} From 209f590412eb7cd8812fae98368b439134527dcd Mon Sep 17 00:00:00 2001 From: adisbladis Date: Sat, 4 Nov 2023 12:48:38 +1300 Subject: [PATCH 332/419] Garnix -> Github Actions It's much easier for forks to use GHA than it is to sign up for another proprietary service. I was annoyed by this myself when I forked nix-eval-jobs to nix-unit. --- .github/workflows/nix-github-actions.yml | 60 ++++++++++++++++++++++++ .github/workflows/tests.yml | 28 ----------- .mergify.yml | 2 + flake.lock | 21 +++++++++ flake.nix | 21 ++++++--- 5 files changed, 97 insertions(+), 35 deletions(-) create mode 100644 .github/workflows/nix-github-actions.yml delete mode 100644 .github/workflows/tests.yml diff --git a/.github/workflows/nix-github-actions.yml b/.github/workflows/nix-github-actions.yml new file mode 100644 index 000000000..e992ffe3d --- /dev/null +++ b/.github/workflows/nix-github-actions.yml @@ -0,0 +1,60 @@ +name: Nix actions + +on: + pull_request: + push: + branches: + - main + - staging + - release-* + +jobs: + nix-matrix: + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + steps: + - uses: actions/checkout@v4 + - uses: cachix/install-nix-action@v23 + - id: set-matrix + name: Generate Nix Matrix + run: | + set -Eeu + echo "matrix=$(nix eval --json '.#githubActions.matrix')" >> "$GITHUB_OUTPUT" + + nix-build: + needs: nix-matrix + runs-on: ${{ matrix.os }} + strategy: + matrix: ${{fromJSON(needs.nix-matrix.outputs.matrix)}} + steps: + - uses: actions/checkout@v4 + - uses: cachix/install-nix-action@v23 + - run: nix build -L ".#${{ matrix.attr }}" + tests: + strategy: + matrix: + os: [ ubuntu-latest, macos-latest ] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + with: + # Nix Flakes doesn't work on shallow clones + fetch-depth: 0 + - uses: cachix/install-nix-action@v23 + with: + github_access_token: ${{ secrets.GITHUB_TOKEN }} + extra_nix_config: | + accept-flake-config = true + - name: Build + run: nix develop -c bash -c 'meson build && cd build && ninja' + - name: Run tests + run: nix develop -c pytest ./tests + + collect: + runs-on: ubuntu-latest + needs: + - nix-build + - tests + steps: + - run: true diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml deleted file mode 100644 index b26284cd9..000000000 --- a/.github/workflows/tests.yml +++ /dev/null @@ -1,28 +0,0 @@ -name: "Tests" -on: - pull_request: - push: - branches: - - main - - staging - - release-* -jobs: - tests: - strategy: - matrix: - os: [ ubuntu-latest, macos-latest ] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v4 - with: - # Nix Flakes doesn't work on shallow clones - fetch-depth: 0 - - uses: cachix/install-nix-action@v23 - with: - github_access_token: ${{ secrets.GITHUB_TOKEN }} - extra_nix_config: | - accept-flake-config = true - - name: Build - run: nix develop -c bash -c 'meson build && cd build && ninja' - - name: Run tests - run: nix develop -c pytest ./tests diff --git a/.mergify.yml b/.mergify.yml index a8cf23afc..de8921669 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -3,11 +3,13 @@ queue_rules: merge_conditions: - author=renovate[bot] - check-success=collect + defaults: actions: queue: allow_merging_configuration_change: true method: rebase + pull_request_rules: - name: merge using the merge queue conditions: diff --git a/flake.lock b/flake.lock index 3983f18bd..c071e0383 100644 --- a/flake.lock +++ b/flake.lock @@ -20,6 +20,26 @@ "type": "github" } }, + "nix-github-actions": { + "inputs": { + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1698974481, + "narHash": "sha256-yPncV9Ohdz1zPZxYHQf47S8S0VrnhV7nNhCawY46hDA=", + "owner": "nix-community", + "repo": "nix-github-actions", + "rev": "4bb5e752616262457bc7ca5882192a564c0472d2", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "nix-github-actions", + "type": "github" + } + }, "nixpkgs": { "locked": { "lastModified": 1698977350, @@ -39,6 +59,7 @@ "root": { "inputs": { "flake-parts": "flake-parts", + "nix-github-actions": "nix-github-actions", "nixpkgs": "nixpkgs", "treefmt-nix": "treefmt-nix" } diff --git a/flake.nix b/flake.nix index 1acfcaf4f..4b6a65b74 100644 --- a/flake.nix +++ b/flake.nix @@ -6,13 +6,8 @@ inputs.flake-parts.inputs.nixpkgs-lib.follows = "nixpkgs"; inputs.treefmt-nix.url = "github:numtide/treefmt-nix"; inputs.treefmt-nix.inputs.nixpkgs.follows = "nixpkgs"; - - nixConfig.extra-substituters = [ - "https://cache.garnix.io" - ]; - nixConfig.extra-trusted-public-keys = [ - "cache.garnix.io:CTFPyKSLcx5RMJKfLo5EEPUObbA78b0YQ2DTCJXqr9g=" - ]; + inputs.nix-github-actions.url = "github:nix-community/nix-github-actions"; + inputs.nix-github-actions.inputs.nixpkgs.follows = "nixpkgs"; outputs = inputs @ { flake-parts, ... }: let @@ -24,6 +19,14 @@ { systems = inputs.nixpkgs.lib.systems.flakeExposed; imports = [ inputs.treefmt-nix.flakeModule ]; + + flake.githubActions = inputs.nix-github-actions.lib.mkGithubMatrix { + checks = { + inherit (self.checks) x86_64-linux; + x86_64-darwin = builtins.removeAttrs self.checks.x86_64-darwin [ "treefmt" ]; + }; + }; + perSystem = { pkgs, self', ... }: let drvArgs = { @@ -37,6 +40,10 @@ packages.clangStdenv-nix-eval-jobs = pkgs.callPackage ./default.nix (drvArgs // { stdenv = pkgs.clangStdenv; }); packages.default = self'.packages.nix-eval-jobs; devShells.default = pkgs.callPackage ./shell.nix drvArgs; + + checks = builtins.removeAttrs self'.packages [ "default" ] // { + shell = self'.devShells.default; + }; }; }; } From 3fdf27e5dfecfc821df5e953e414a76784d6b339 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 6 Nov 2023 02:33:20 +0000 Subject: [PATCH 333/419] Lock file maintenance --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index c071e0383..42aec3a2f 100644 --- a/flake.lock +++ b/flake.lock @@ -42,11 +42,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1698977350, - "narHash": "sha256-OUDOHWrX3EjX/MlOoCHEb3JMONklbpu4Wa+Xf5s/U+s=", + "lastModified": 1699236715, + "narHash": "sha256-oel+a6B5mBO7vA1A/I9A9VTK2jW5shnYAuu08RYhmxQ=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "4285a2a67daf39e63d9564a47773a1c2081c36a8", + "rev": "0d93ec62e06faec6c52331a8a87bd5721b38ce14", "type": "github" }, "original": { From cdbfafc2e6c86991bc9465d259a67c1d4a1983d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sun, 12 Nov 2023 14:11:00 +0100 Subject: [PATCH 334/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/0d93ec62e06faec6c52331a8a87bd5721b38ce14' (2023-11-06) → 'github:NixOS/nixpkgs/44cf4801c0937b76cc6f416a0b160b5d1b3286af' (2023-11-12) • Updated input 'treefmt-nix': 'github:numtide/treefmt-nix/5deb8dc125a9f83b65ca86cf0c8167c46593e0b1' (2023-10-27) → 'github:numtide/treefmt-nix/e82f32aa7f06bbbd56d7b12186d555223dc399d1' (2023-11-12) --- flake.lock | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/flake.lock b/flake.lock index 42aec3a2f..a67ac6872 100644 --- a/flake.lock +++ b/flake.lock @@ -42,11 +42,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1699236715, - "narHash": "sha256-oel+a6B5mBO7vA1A/I9A9VTK2jW5shnYAuu08RYhmxQ=", + "lastModified": 1699794571, + "narHash": "sha256-a9fa+AYCSiL1w9GBz4bIoj/rw4hZIntpaCdhXqHSLfM=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "0d93ec62e06faec6c52331a8a87bd5721b38ce14", + "rev": "44cf4801c0937b76cc6f416a0b160b5d1b3286af", "type": "github" }, "original": { @@ -71,11 +71,11 @@ ] }, "locked": { - "lastModified": 1698438538, - "narHash": "sha256-AWxaKTDL3MtxaVTVU5lYBvSnlspOS0Fjt8GxBgnU0Do=", + "lastModified": 1699786194, + "narHash": "sha256-3h3EH1FXQkIeAuzaWB+nK0XK54uSD46pp+dMD3gAcB4=", "owner": "numtide", "repo": "treefmt-nix", - "rev": "5deb8dc125a9f83b65ca86cf0c8167c46593e0b1", + "rev": "e82f32aa7f06bbbd56d7b12186d555223dc399d1", "type": "github" }, "original": { From a53aaefbccc60ef4a540c67e197c2dabbab0ce94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sun, 12 Nov 2023 14:17:19 +0100 Subject: [PATCH 335/419] publish to flakestry --- .github/workflows/publish.yml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 .github/workflows/publish.yml diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 000000000..58660afa5 --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,22 @@ +name: "Publish a flake to flakestry" +on: + push: + tags: + - "v?[0-9]+.[0-9]+.[0-9]+" + - "v?[0-9]+.[0-9]+" + workflow_dispatch: + inputs: + tag: + description: "The existing tag to publish" + type: "string" + required: true +jobs: + publish-flake: + runs-on: ubuntu-latest + permissions: + id-token: "write" + contents: "read" + steps: + - uses: flakestry/flakestry-publish@main + with: + version: "${{ inputs.tag || github.ref_name }}" From 6ac0adcbb04924ed356acfde98d7f04f842312cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sun, 12 Nov 2023 14:17:43 +0100 Subject: [PATCH 336/419] update version: 2.16.0 -> 2.18.0 --- default.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/default.nix b/default.nix index f858e9f0f..70fe2736a 100644 --- a/default.nix +++ b/default.nix @@ -11,7 +11,7 @@ let in stdenv.mkDerivation { pname = "nix-eval-jobs"; - version = "2.16.0"; + version = "2.18.0"; src = if srcDir == null then filterMesonBuild ./. else srcDir; buildInputs = with pkgs; [ nlohmann_json From 67b35acd0e4290701e26335db93d52d3f281ba0d Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 13 Nov 2023 01:58:28 +0000 Subject: [PATCH 337/419] Lock file maintenance --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index a67ac6872..fd0cf6fd4 100644 --- a/flake.lock +++ b/flake.lock @@ -42,11 +42,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1699794571, - "narHash": "sha256-a9fa+AYCSiL1w9GBz4bIoj/rw4hZIntpaCdhXqHSLfM=", + "lastModified": 1699839047, + "narHash": "sha256-FAoWKSDZ9vpd8sLeJYeVGUnSlOCqkSochTEvOA7+qeM=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "44cf4801c0937b76cc6f416a0b160b5d1b3286af", + "rev": "8423b2dff7b10463eb97f9242bd62a1ff8d2ee3e", "type": "github" }, "original": { From 9a456e0e723ebfa44bcc78762fb240cbe6ad6eca Mon Sep 17 00:00:00 2001 From: "Shahar \"Dawn\" Or" Date: Tue, 14 Nov 2023 12:14:58 +0700 Subject: [PATCH 338/419] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ad657b4d9..5bb9aa261 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # nix-eval-jobs -This project evaluates nix attributes sets in parallel with streamable json +This project evaluates nix attribute sets in parallel with streamable json output. This is useful for time and memory intensive evaluations such as NixOS machines, i.e. in a CI context. The evaluation is done with a controllable number of threads that are restarted when their memory consumption exceeds a From 0cdb4f4c63e4c13aaf71c9868e89255c8242f9ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Tue, 14 Nov 2023 09:49:42 +0100 Subject: [PATCH 339/419] renovate: add dependencies label --- renovate.json | 1 + 1 file changed, 1 insertion(+) diff --git a/renovate.json b/renovate.json index 03086d773..da5dfa5ed 100644 --- a/renovate.json +++ b/renovate.json @@ -9,6 +9,7 @@ "schedule:weekly" ] }, + "labels": ["dependencies"], "nix": { "enabled": true } From 333af7cb0f3dc54e893d2032e4032821bc90e145 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 13 Nov 2023 01:58:28 +0000 Subject: [PATCH 340/419] Lock file maintenance --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index a67ac6872..fd0cf6fd4 100644 --- a/flake.lock +++ b/flake.lock @@ -42,11 +42,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1699794571, - "narHash": "sha256-a9fa+AYCSiL1w9GBz4bIoj/rw4hZIntpaCdhXqHSLfM=", + "lastModified": 1699839047, + "narHash": "sha256-FAoWKSDZ9vpd8sLeJYeVGUnSlOCqkSochTEvOA7+qeM=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "44cf4801c0937b76cc6f416a0b160b5d1b3286af", + "rev": "8423b2dff7b10463eb97f9242bd62a1ff8d2ee3e", "type": "github" }, "original": { From 7ee77ecd804b935d242933e9d8320811f68a2d36 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 20 Nov 2023 02:20:50 +0000 Subject: [PATCH 341/419] chore(deps): lock file maintenance --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index fd0cf6fd4..783b55e9d 100644 --- a/flake.lock +++ b/flake.lock @@ -42,11 +42,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1699839047, - "narHash": "sha256-FAoWKSDZ9vpd8sLeJYeVGUnSlOCqkSochTEvOA7+qeM=", + "lastModified": 1700444282, + "narHash": "sha256-s/+tgT+Iz0LZO+nBvSms+xsMqvHt2LqYniG9r+CYyJc=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "8423b2dff7b10463eb97f9242bd62a1ff8d2ee3e", + "rev": "3f21a22b5aafefa1845dec6f4a378a8f53d8681c", "type": "github" }, "original": { From 354fd8b93d5b88922701df499b9a1282dc19fb70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sun, 19 Nov 2023 13:49:38 +0100 Subject: [PATCH 342/419] README: mention nix-fast-build and buildbot-nix --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 5bb9aa261..afd9e7161 100644 --- a/README.md +++ b/README.md @@ -89,6 +89,8 @@ latest release branch. ## Projects using nix-eval-jobs +- [nix-fast-build](https://github.com/Mic92/nix-fast-build) - Combine the power of nix-eval-jobs with nix-output-monitor to speed-up your evaluation and building process +- [buildbot-nix](https://github.com/Mic92/buildbot-nix) - A nixos module to make buildbot a proper Nix-CI - [colmena](https://github.com/zhaofengli/colmena) - A simple, stateless NixOS deployment tool - [robotnix](https://github.com/danielfullmer/robotnix) - Build Android (AOSP) using Nix, used in their [CI](https://github.com/danielfullmer/robotnix/blob/38b80700ee4265c306dcfdcce45056e32ab2973f/.github/workflows/instantiate.yml#L18) From 70786f5ff43bc75a4ec34deed5e53b2772b132f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Mon, 20 Nov 2023 06:50:56 +0100 Subject: [PATCH 343/419] fix mergify for normal non-bot merges --- .mergify.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.mergify.yml b/.mergify.yml index de8921669..21dea9be7 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -1,7 +1,6 @@ queue_rules: - name: default merge_conditions: - - author=renovate[bot] - check-success=collect defaults: From e543721cfff2b79d82c8b0932ec82281a9950677 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 27 Nov 2023 00:50:20 +0000 Subject: [PATCH 344/419] chore(deps): lock file maintenance --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 783b55e9d..4cc2a6411 100644 --- a/flake.lock +++ b/flake.lock @@ -42,11 +42,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1700444282, - "narHash": "sha256-s/+tgT+Iz0LZO+nBvSms+xsMqvHt2LqYniG9r+CYyJc=", + "lastModified": 1701045352, + "narHash": "sha256-iWsDbWzBP4gotkRfg/lH2A3O9wFoJc+yVO8CDuHLRe8=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "3f21a22b5aafefa1845dec6f4a378a8f53d8681c", + "rev": "5171694860f185961daff3b1b413dabcab421300", "type": "github" }, "original": { From 608089054e7cf4e86b218375aaecbb2ca80c9e67 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 28 Nov 2023 18:05:34 +0000 Subject: [PATCH 345/419] chore(deps): update cachix/install-nix-action action to v24 --- .github/workflows/nix-github-actions.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/nix-github-actions.yml b/.github/workflows/nix-github-actions.yml index e992ffe3d..f6eede4fa 100644 --- a/.github/workflows/nix-github-actions.yml +++ b/.github/workflows/nix-github-actions.yml @@ -15,7 +15,7 @@ jobs: matrix: ${{ steps.set-matrix.outputs.matrix }} steps: - uses: actions/checkout@v4 - - uses: cachix/install-nix-action@v23 + - uses: cachix/install-nix-action@v24 - id: set-matrix name: Generate Nix Matrix run: | @@ -29,7 +29,7 @@ jobs: matrix: ${{fromJSON(needs.nix-matrix.outputs.matrix)}} steps: - uses: actions/checkout@v4 - - uses: cachix/install-nix-action@v23 + - uses: cachix/install-nix-action@v24 - run: nix build -L ".#${{ matrix.attr }}" tests: strategy: @@ -41,7 +41,7 @@ jobs: with: # Nix Flakes doesn't work on shallow clones fetch-depth: 0 - - uses: cachix/install-nix-action@v23 + - uses: cachix/install-nix-action@v24 with: github_access_token: ${{ secrets.GITHUB_TOKEN }} extra_nix_config: | From 34110992a84eb5175340226c58eda19dd8e470cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Wed, 6 Dec 2023 09:05:22 +0100 Subject: [PATCH 346/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'flake-parts': 'github:hercules-ci/flake-parts/8c9fa2545007b49a5db5f650ae91f227672c3877' (2023-11-01) → 'github:hercules-ci/flake-parts/34fed993f1674c8d06d58b37ce1e0fe5eebcb9f5' (2023-12-01) • Updated input 'nix-github-actions': 'github:nix-community/nix-github-actions/4bb5e752616262457bc7ca5882192a564c0472d2' (2023-11-03) → 'github:nix-community/nix-github-actions/93e39cc1a087d65bcf7a132e75a650c44dd2b734' (2023-11-28) • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/5171694860f185961daff3b1b413dabcab421300' (2023-11-27) → 'github:NixOS/nixpkgs/9ed8ade77aef706a03d8cc3a5ad4f60848ac59a7' (2023-12-06) • Updated input 'treefmt-nix': 'github:numtide/treefmt-nix/e82f32aa7f06bbbd56d7b12186d555223dc399d1' (2023-11-12) → 'github:numtide/treefmt-nix/affe7fc3f5790e1d0b5ba51bcff0f7ebe465e92d' (2023-12-04) --- flake.lock | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/flake.lock b/flake.lock index 4cc2a6411..cc936179e 100644 --- a/flake.lock +++ b/flake.lock @@ -7,11 +7,11 @@ ] }, "locked": { - "lastModified": 1698882062, - "narHash": "sha256-HkhafUayIqxXyHH1X8d9RDl1M2CkFgZLjKD3MzabiEo=", + "lastModified": 1701473968, + "narHash": "sha256-YcVE5emp1qQ8ieHUnxt1wCZCC3ZfAS+SRRWZ2TMda7E=", "owner": "hercules-ci", "repo": "flake-parts", - "rev": "8c9fa2545007b49a5db5f650ae91f227672c3877", + "rev": "34fed993f1674c8d06d58b37ce1e0fe5eebcb9f5", "type": "github" }, "original": { @@ -27,11 +27,11 @@ ] }, "locked": { - "lastModified": 1698974481, - "narHash": "sha256-yPncV9Ohdz1zPZxYHQf47S8S0VrnhV7nNhCawY46hDA=", + "lastModified": 1701208414, + "narHash": "sha256-xrQ0FyhwTZK6BwKhahIkUVZhMNk21IEI1nUcWSONtpo=", "owner": "nix-community", "repo": "nix-github-actions", - "rev": "4bb5e752616262457bc7ca5882192a564c0472d2", + "rev": "93e39cc1a087d65bcf7a132e75a650c44dd2b734", "type": "github" }, "original": { @@ -42,11 +42,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1701045352, - "narHash": "sha256-iWsDbWzBP4gotkRfg/lH2A3O9wFoJc+yVO8CDuHLRe8=", + "lastModified": 1701847270, + "narHash": "sha256-ttPWHy1NZwJzSzY7OmofFNyrm9kWc+RFFHpJGeQ4kWw=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "5171694860f185961daff3b1b413dabcab421300", + "rev": "9ed8ade77aef706a03d8cc3a5ad4f60848ac59a7", "type": "github" }, "original": { @@ -71,11 +71,11 @@ ] }, "locked": { - "lastModified": 1699786194, - "narHash": "sha256-3h3EH1FXQkIeAuzaWB+nK0XK54uSD46pp+dMD3gAcB4=", + "lastModified": 1701682826, + "narHash": "sha256-2lxeTUGs8Jzz/wjLgWYmZoXn60BYNRMzwHFtxNFUDLU=", "owner": "numtide", "repo": "treefmt-nix", - "rev": "e82f32aa7f06bbbd56d7b12186d555223dc399d1", + "rev": "affe7fc3f5790e1d0b5ba51bcff0f7ebe465e92d", "type": "github" }, "original": { From c240e61481ee65ae66485c455d1135f3e6483540 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Wed, 6 Dec 2023 09:14:07 +0100 Subject: [PATCH 347/419] fix missing includes for nix 2.19 --- src/nix-eval-jobs.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/nix-eval-jobs.cc b/src/nix-eval-jobs.cc index 4dfcdb8f0..f1ff27811 100644 --- a/src/nix-eval-jobs.cc +++ b/src/nix-eval-jobs.cc @@ -21,6 +21,8 @@ #include #include #include +#include +#include #include #include From d4eeecf6b2652ababb6b7b17f1aed4e8c346a743 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Wed, 6 Dec 2023 09:14:20 +0100 Subject: [PATCH 348/419] fix commandline parsing for nix 2.19 --- src/nix-eval-jobs.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/nix-eval-jobs.cc b/src/nix-eval-jobs.cc index f1ff27811..ec5ff3488 100644 --- a/src/nix-eval-jobs.cc +++ b/src/nix-eval-jobs.cc @@ -43,7 +43,7 @@ using namespace nlohmann; #elif __clang__ #pragma clang diagnostic ignored "-Wnon-virtual-dtor" #endif -struct MyArgs : MixEvalArgs, MixCommonArgs { +struct MyArgs : virtual MixEvalArgs, virtual MixCommonArgs, virtual RootArgs { std::string releaseExpr; Path gcRootsDir; bool flake = false; @@ -545,7 +545,7 @@ int main(int argc, char **argv) { initNix(); initGC(); - myArgs.parseCmdline(argvToStrings(argc, argv)); + myArgs.parseCmdline(argvToStrings(argc, argv), 0); /* FIXME: The build hook in conjunction with import-from-derivation is * causing "unexpected EOF" during eval */ From 6f9a8d2cda78abad1d48e116741fe823427156f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Wed, 6 Dec 2023 09:40:57 +0100 Subject: [PATCH 349/419] bump to 2.19.0 --- default.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/default.nix b/default.nix index 70fe2736a..bb2ddf3c1 100644 --- a/default.nix +++ b/default.nix @@ -11,7 +11,7 @@ let in stdenv.mkDerivation { pname = "nix-eval-jobs"; - version = "2.18.0"; + version = "2.19.0"; src = if srcDir == null then filterMesonBuild ./. else srcDir; buildInputs = with pkgs; [ nlohmann_json From fad244725fe00fc8e4c05b35ae03316e4a2bca5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sun, 10 Dec 2023 09:17:51 +0100 Subject: [PATCH 350/419] print which derivation failed to evaluate --- src/nix-eval-jobs.cc | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/nix-eval-jobs.cc b/src/nix-eval-jobs.cc index ec5ff3488..56b6c49ec 100644 --- a/src/nix-eval-jobs.cc +++ b/src/nix-eval-jobs.cc @@ -203,7 +203,7 @@ struct Drv { std::map> inputDrvs; std::optional meta; - Drv(EvalState &state, DrvInfo &drvInfo) { + Drv(std::string &attrPath, EvalState &state, DrvInfo &drvInfo) { auto localStore = state.store.dynamic_pointer_cast(); @@ -214,7 +214,8 @@ struct Drv { localStore->printStorePath(*out.second); } } catch (const std::exception &e) { - throw EvalError("derivation must have valid outputs: %s", e.what()); + throw EvalError("derivation '%s' does not have valid outputs: %s", + attrPath, e.what()); } if (myArgs.meta) { @@ -326,7 +327,7 @@ static void worker(ref state, Bindings &autoArgs, AutoCloseFD &to, if (v->type() == nAttrs) { if (auto drvInfo = getDerivation(*state, *v, false)) { - auto drv = Drv(*state, *drvInfo); + auto drv = Drv(attrPathS, *state, *drvInfo); reply.update(drv); /* Register the derivation as a GC root. !!! This From 40ad82808858c5a7137bfa860afabcafa505a94d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sun, 10 Dec 2023 09:22:38 +0100 Subject: [PATCH 351/419] update flakes to fix tests on macos --- flake.lock | 12 ++++++------ tests/assets/flake.lock | 7 ++++--- tests/assets/flake.nix | 2 +- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/flake.lock b/flake.lock index cc936179e..f885e6e8f 100644 --- a/flake.lock +++ b/flake.lock @@ -42,11 +42,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1701847270, - "narHash": "sha256-ttPWHy1NZwJzSzY7OmofFNyrm9kWc+RFFHpJGeQ4kWw=", + "lastModified": 1702192996, + "narHash": "sha256-taRtgPtpYl7KofdDC9sDHe1urV3+pP2JFwuAyVlccYI=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "9ed8ade77aef706a03d8cc3a5ad4f60848ac59a7", + "rev": "db3bd72ed27a5b2b96c3e6cca463117d27bb052b", "type": "github" }, "original": { @@ -71,11 +71,11 @@ ] }, "locked": { - "lastModified": 1701682826, - "narHash": "sha256-2lxeTUGs8Jzz/wjLgWYmZoXn60BYNRMzwHFtxNFUDLU=", + "lastModified": 1701958734, + "narHash": "sha256-3h3EH1FXQkIeAuzaWB+nK0XK54uSD46pp+dMD3gAcB4=", "owner": "numtide", "repo": "treefmt-nix", - "rev": "affe7fc3f5790e1d0b5ba51bcff0f7ebe465e92d", + "rev": "e8cea581dd2b7c9998c1e6662db2c1dc30e7fdb0", "type": "github" }, "original": { diff --git a/tests/assets/flake.lock b/tests/assets/flake.lock index a1a80bf7c..551565bcb 100644 --- a/tests/assets/flake.lock +++ b/tests/assets/flake.lock @@ -2,15 +2,16 @@ "nodes": { "nixpkgs": { "locked": { - "lastModified": 1616345250, - "narHash": "sha256-WLbLFIJyKCklGyEMGwh9XDTzafafyO95s4+rJHOc/Ag=", + "lastModified": 1702172177, + "narHash": "sha256-2T1DjuXz0bVxy5g8oF9FYioHOLWkXw5EdW687NDQakE=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "5e4a4e0c32f0ca0a5bd4ebbbf17aedd347de7f3e", + "rev": "2873a73123077953f3e6f34964466018876d87c4", "type": "github" }, "original": { "owner": "NixOS", + "ref": "nixpkgs-unstable", "repo": "nixpkgs", "type": "github" } diff --git a/tests/assets/flake.nix b/tests/assets/flake.nix index aa28425f2..90cb732bf 100644 --- a/tests/assets/flake.nix +++ b/tests/assets/flake.nix @@ -1,5 +1,5 @@ { - inputs.nixpkgs.url = "github:NixOS/nixpkgs"; + inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable"; outputs = { nixpkgs, ... }: let From 5ad4e7266f9dcd1ed242586e89e74040317cce29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sat, 9 Dec 2023 17:17:01 +0100 Subject: [PATCH 352/419] handle broken evaluation worker pipes more gracefully writeLine will throw a SysError exception, which obfuscates out-of-memory events where the eval worker is killed by the OS. readLine is suffering from the same problem and will be handled in a subsequent commit. --- src/nix-eval-jobs.cc | 65 ++++++++++++++++++++++++++++++++++++++------ 1 file changed, 57 insertions(+), 8 deletions(-) diff --git a/src/nix-eval-jobs.cc b/src/nix-eval-jobs.cc index 56b6c49ec..b766dee5d 100644 --- a/src/nix-eval-jobs.cc +++ b/src/nix-eval-jobs.cc @@ -2,7 +2,6 @@ #include #include #include - #include #include #include @@ -284,6 +283,22 @@ std::string attrPathJoin(json input) { }); } +[[nodiscard]] static int tryWriteLine(int fd, std::string s) { + s += "\n"; + std::string_view sv{s}; + while (!sv.empty()) { + checkInterrupt(); + ssize_t res = write(fd, sv.data(), sv.size()); + if (res == -1 && errno != EINTR) { + return -errno; + } + if (res > 0) { + sv.remove_prefix(res); + } + } + return 0; +} + static void worker(ref state, Bindings &autoArgs, AutoCloseFD &to, AutoCloseFD &from) { @@ -304,7 +319,9 @@ static void worker(ref state, Bindings &autoArgs, AutoCloseFD &to, while (true) { /* Wait for the collector to send us a job name. */ - writeLine(to.get(), "next"); + if (tryWriteLine(to.get(), "next") < 0) { + return; // main process died + } auto s = readLine(from.get()); if (s == "exit") @@ -389,7 +406,9 @@ static void worker(ref state, Bindings &autoArgs, AutoCloseFD &to, printError(e.msg()); } - writeLine(to.get(), reply.dump()); + if (tryWriteLine(to.get(), reply.dump()) < 0) { + return; // main process died + } /* If our RSS exceeds the maximum, exit. The collector will start a new process. */ @@ -399,7 +418,9 @@ static void worker(ref state, Bindings &autoArgs, AutoCloseFD &to, break; } - writeLine(to.get(), "restart"); + if (tryWriteLine(to.get(), "restart") < 0) { + return; // main process died + }; } typedef std::function state, Bindings &autoArgs, @@ -431,10 +452,14 @@ struct Proc { auto msg = e.msg(); err["error"] = filterANSIEscapes(msg, true); printError(msg); - writeLine(to->get(), err.dump()); + if (tryWriteLine(to->get(), err.dump()) < 0) { + return; // main process died + }; // Don't forget to print it into the STDERR log, this is // what's shown in the Hydra UI. - writeLine(to->get(), "restart"); + if (tryWriteLine(to->get(), "restart") < 0) { + return; // main process died + } } }, ProcessOptions{.allowVfork = false}); @@ -453,6 +478,26 @@ struct State { std::exception_ptr exc; }; +void handleBrokenWorkerPipe(pid_t child) { + while (1) { + int rc = waitpid(child, nullptr, WNOHANG); + if (rc == 0) { + throw Error("BUG: worker pipe closed but worker still running?"); + } else if (rc == -1) { + throw Error("BUG: waitpid waiting for worker failed: %s", + strerror(errno)); + } else { + if (WIFEXITED(rc)) { + throw Error("evaluation worker exited with %d", + WEXITSTATUS(rc)); + } else if (WIFSIGNALED(rc)) { + throw Error("evaluation worker killed by signal %d", + WTERMSIG(rc)); + } // else ignore WIFSTOPPED and WIFCONTINUED + } + } +} + std::function collector(Sync &state_, std::condition_variable &wakeup) { return [&]() { @@ -482,7 +527,9 @@ std::function collector(Sync &state_, auto state(state_.lock()); if ((state->todo.empty() && state->active.empty()) || state->exc) { - writeLine(proc->to.get(), "exit"); + if (tryWriteLine(proc->to.get(), "exit") < 0) { + handleBrokenWorkerPipe(proc->pid); + } return; } if (!state->todo.empty()) { @@ -495,7 +542,9 @@ std::function collector(Sync &state_, } /* Tell the worker to evaluate it. */ - writeLine(proc->to.get(), "do " + attrPath.dump()); + if (tryWriteLine(proc->to.get(), "do " + attrPath.dump()) < 0) { + handleBrokenWorkerPipe(proc->pid); + } /* Wait for the response. */ auto respString = readLine(proc->from.get()); From 36483b325c6a3395c95d2b8b6c4a127111921c4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sat, 9 Dec 2023 17:34:26 +0100 Subject: [PATCH 353/419] treefmt: replace prettier with deno we don't have toml files and deno has a smaller install closure. --- LICENSE.md | 821 +++++++++++++++++++++++++----------------------- README.md | 64 ++-- dev/treefmt.nix | 10 +- 3 files changed, 456 insertions(+), 439 deletions(-) diff --git a/LICENSE.md b/LICENSE.md index 1110e8987..1bbb815ce 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -1,67 +1,69 @@ -GNU General Public License -========================== +# GNU General Public License -_Version 3, 29 June 2007_ -_Copyright © 2007 Free Software Foundation, Inc. <>_ +_Version 3, 29 June 2007_ _Copyright © 2007 Free Software Foundation, Inc. +<>_ Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed. ## Preamble -The GNU General Public License is a free, copyleft license for software and other -kinds of works. +The GNU General Public License is a free, copyleft license for software and +other kinds of works. -The licenses for most software and other practical works are designed to take away -your freedom to share and change the works. By contrast, the GNU General Public -License is intended to guarantee your freedom to share and change all versions of a -program--to make sure it remains free software for all its users. We, the Free -Software Foundation, use the GNU General Public License for most of our software; it -applies also to any other work released this way by its authors. You can apply it to -your programs, too. +The licenses for most software and other practical works are designed to take +away your freedom to share and change the works. By contrast, the GNU General +Public License is intended to guarantee your freedom to share and change all +versions of a program--to make sure it remains free software for all its users. +We, the Free Software Foundation, use the GNU General Public License for most of +our software; it applies also to any other work released this way by its +authors. You can apply it to your programs, too. -When we speak of free software, we are referring to freedom, not price. Our General -Public Licenses are designed to make sure that you have the freedom to distribute -copies of free software (and charge for them if you wish), that you receive source -code or can get it if you want it, that you can change the software or use pieces of -it in new free programs, and that you know you can do these things. +When we speak of free software, we are referring to freedom, not price. Our +General Public Licenses are designed to make sure that you have the freedom to +distribute copies of free software (and charge for them if you wish), that you +receive source code or can get it if you want it, that you can change the +software or use pieces of it in new free programs, and that you know you can do +these things. -To protect your rights, we need to prevent others from denying you these rights or -asking you to surrender the rights. Therefore, you have certain responsibilities if -you distribute copies of the software, or if you modify it: responsibilities to -respect the freedom of others. +To protect your rights, we need to prevent others from denying you these rights +or asking you to surrender the rights. Therefore, you have certain +responsibilities if you distribute copies of the software, or if you modify it: +responsibilities to respect the freedom of others. -For example, if you distribute copies of such a program, whether gratis or for a fee, -you must pass on to the recipients the same freedoms that you received. You must make -sure that they, too, receive or can get the source code. And you must show them these -terms so they know their rights. +For example, if you distribute copies of such a program, whether gratis or for a +fee, you must pass on to the recipients the same freedoms that you received. You +must make sure that they, too, receive or can get the source code. And you must +show them these terms so they know their rights. -Developers that use the GNU GPL protect your rights with two steps: **(1)** assert -copyright on the software, and **(2)** offer you this License giving you legal permission -to copy, distribute and/or modify it. +Developers that use the GNU GPL protect your rights with two steps: **(1)** +assert copyright on the software, and **(2)** offer you this License giving you +legal permission to copy, distribute and/or modify it. -For the developers' and authors' protection, the GPL clearly explains that there is -no warranty for this free software. For both users' and authors' sake, the GPL -requires that modified versions be marked as changed, so that their problems will not -be attributed erroneously to authors of previous versions. +For the developers' and authors' protection, the GPL clearly explains that there +is no warranty for this free software. For both users' and authors' sake, the +GPL requires that modified versions be marked as changed, so that their problems +will not be attributed erroneously to authors of previous versions. -Some devices are designed to deny users access to install or run modified versions of -the software inside them, although the manufacturer can do so. This is fundamentally -incompatible with the aim of protecting users' freedom to change the software. The -systematic pattern of such abuse occurs in the area of products for individuals to -use, which is precisely where it is most unacceptable. Therefore, we have designed -this version of the GPL to prohibit the practice for those products. If such problems -arise substantially in other domains, we stand ready to extend this provision to -those domains in future versions of the GPL, as needed to protect the freedom of -users. +Some devices are designed to deny users access to install or run modified +versions of the software inside them, although the manufacturer can do so. This +is fundamentally incompatible with the aim of protecting users' freedom to +change the software. The systematic pattern of such abuse occurs in the area of +products for individuals to use, which is precisely where it is most +unacceptable. Therefore, we have designed this version of the GPL to prohibit +the practice for those products. If such problems arise substantially in other +domains, we stand ready to extend this provision to those domains in future +versions of the GPL, as needed to protect the freedom of users. -Finally, every program is threatened constantly by software patents. States should -not allow patents to restrict development and use of software on general-purpose -computers, but in those that do, we wish to avoid the special danger that patents -applied to a free program could make it effectively proprietary. To prevent this, the -GPL assures that patents cannot be used to render the program non-free. +Finally, every program is threatened constantly by software patents. States +should not allow patents to restrict development and use of software on +general-purpose computers, but in those that do, we wish to avoid the special +danger that patents applied to a free program could make it effectively +proprietary. To prevent this, the GPL assures that patents cannot be used to +render the program non-free. -The precise terms and conditions for copying, distribution and modification follow. +The precise terms and conditions for copying, distribution and modification +follow. ## TERMS AND CONDITIONS @@ -69,70 +71,70 @@ The precise terms and conditions for copying, distribution and modification foll “This License” refers to version 3 of the GNU General Public License. -“Copyright” also means copyright-like laws that apply to other kinds of -works, such as semiconductor masks. +“Copyright” also means copyright-like laws that apply to other kinds of works, +such as semiconductor masks. -“The Program” refers to any copyrightable work licensed under this -License. Each licensee is addressed as “you”. “Licensees” and -“recipients” may be individuals or organizations. +“The Program” refers to any copyrightable work licensed under this License. Each +licensee is addressed as “you”. “Licensees” and “recipients” may be individuals +or organizations. -To “modify” a work means to copy from or adapt all or part of the work in -a fashion requiring copyright permission, other than the making of an exact copy. The -resulting work is called a “modified version” of the earlier work or a -work “based on” the earlier work. +To “modify” a work means to copy from or adapt all or part of the work in a +fashion requiring copyright permission, other than the making of an exact copy. +The resulting work is called a “modified version” of the earlier work or a work +“based on” the earlier work. -A “covered work” means either the unmodified Program or a work based on -the Program. +A “covered work” means either the unmodified Program or a work based on the +Program. -To “propagate” a work means to do anything with it that, without -permission, would make you directly or secondarily liable for infringement under -applicable copyright law, except executing it on a computer or modifying a private -copy. Propagation includes copying, distribution (with or without modification), +To “propagate” a work means to do anything with it that, without permission, +would make you directly or secondarily liable for infringement under applicable +copyright law, except executing it on a computer or modifying a private copy. +Propagation includes copying, distribution (with or without modification), making available to the public, and in some countries other activities as well. -To “convey” a work means any kind of propagation that enables other -parties to make or receive copies. Mere interaction with a user through a computer -network, with no transfer of a copy, is not conveying. +To “convey” a work means any kind of propagation that enables other parties to +make or receive copies. Mere interaction with a user through a computer network, +with no transfer of a copy, is not conveying. -An interactive user interface displays “Appropriate Legal Notices” to the -extent that it includes a convenient and prominently visible feature that **(1)** -displays an appropriate copyright notice, and **(2)** tells the user that there is no -warranty for the work (except to the extent that warranties are provided), that -licensees may convey the work under this License, and how to view a copy of this -License. If the interface presents a list of user commands or options, such as a -menu, a prominent item in the list meets this criterion. +An interactive user interface displays “Appropriate Legal Notices” to the extent +that it includes a convenient and prominently visible feature that **(1)** +displays an appropriate copyright notice, and **(2)** tells the user that there +is no warranty for the work (except to the extent that warranties are provided), +that licensees may convey the work under this License, and how to view a copy of +this License. If the interface presents a list of user commands or options, such +as a menu, a prominent item in the list meets this criterion. ### 1. Source Code -The “source code” for a work means the preferred form of the work for -making modifications to it. “Object code” means any non-source form of a -work. +The “source code” for a work means the preferred form of the work for making +modifications to it. “Object code” means any non-source form of a work. -A “Standard Interface” means an interface that either is an official -standard defined by a recognized standards body, or, in the case of interfaces -specified for a particular programming language, one that is widely used among -developers working in that language. +A “Standard Interface” means an interface that either is an official standard +defined by a recognized standards body, or, in the case of interfaces specified +for a particular programming language, one that is widely used among developers +working in that language. -The “System Libraries” of an executable work include anything, other than -the work as a whole, that **(a)** is included in the normal form of packaging a Major -Component, but which is not part of that Major Component, and **(b)** serves only to -enable use of the work with that Major Component, or to implement a Standard -Interface for which an implementation is available to the public in source code form. -A “Major Component”, in this context, means a major essential component -(kernel, window system, and so on) of the specific operating system (if any) on which -the executable work runs, or a compiler used to produce the work, or an object code -interpreter used to run it. +The “System Libraries” of an executable work include anything, other than the +work as a whole, that **(a)** is included in the normal form of packaging a +Major Component, but which is not part of that Major Component, and **(b)** +serves only to enable use of the work with that Major Component, or to implement +a Standard Interface for which an implementation is available to the public in +source code form. A “Major Component”, in this context, means a major essential +component (kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to produce the +work, or an object code interpreter used to run it. -The “Corresponding Source” for a work in object code form means all the -source code needed to generate, install, and (for an executable work) run the object -code and to modify the work, including scripts to control those activities. However, -it does not include the work's System Libraries, or general-purpose tools or -generally available free programs which are used unmodified in performing those -activities but which are not part of the work. For example, Corresponding Source -includes interface definition files associated with source files for the work, and -the source code for shared libraries and dynamically linked subprograms that the work -is specifically designed to require, such as by intimate data communication or -control flow between those subprograms and other parts of the work. +The “Corresponding Source” for a work in object code form means all the source +code needed to generate, install, and (for an executable work) run the object +code and to modify the work, including scripts to control those activities. +However, it does not include the work's System Libraries, or general-purpose +tools or generally available free programs which are used unmodified in +performing those activities but which are not part of the work. For example, +Corresponding Source includes interface definition files associated with source +files for the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, such as by +intimate data communication or control flow between those subprograms and other +parts of the work. The Corresponding Source need not include anything that users can regenerate automatically from other parts of the Corresponding Source. @@ -141,328 +143,341 @@ The Corresponding Source for a work in source code form is that same work. ### 2. Basic Permissions -All rights granted under this License are granted for the term of copyright on the -Program, and are irrevocable provided the stated conditions are met. This License -explicitly affirms your unlimited permission to run the unmodified Program. The -output from running a covered work is covered by this License only if the output, -given its content, constitutes a covered work. This License acknowledges your rights -of fair use or other equivalent, as provided by copyright law. +All rights granted under this License are granted for the term of copyright on +the Program, and are irrevocable provided the stated conditions are met. This +License explicitly affirms your unlimited permission to run the unmodified +Program. The output from running a covered work is covered by this License only +if the output, given its content, constitutes a covered work. This License +acknowledges your rights of fair use or other equivalent, as provided by +copyright law. You may make, run and propagate covered works that you do not convey, without -conditions so long as your license otherwise remains in force. You may convey covered -works to others for the sole purpose of having them make modifications exclusively -for you, or provide you with facilities for running those works, provided that you -comply with the terms of this License in conveying all material for which you do not -control copyright. Those thus making or running the covered works for you must do so -exclusively on your behalf, under your direction and control, on terms that prohibit -them from making any copies of your copyrighted material outside their relationship -with you. +conditions so long as your license otherwise remains in force. You may convey +covered works to others for the sole purpose of having them make modifications +exclusively for you, or provide you with facilities for running those works, +provided that you comply with the terms of this License in conveying all +material for which you do not control copyright. Those thus making or running +the covered works for you must do so exclusively on your behalf, under your +direction and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. Conveying under any other circumstances is permitted solely under the conditions stated below. Sublicensing is not allowed; section 10 makes it unnecessary. ### 3. Protecting Users' Legal Rights From Anti-Circumvention Law -No covered work shall be deemed part of an effective technological measure under any -applicable law fulfilling obligations under article 11 of the WIPO copyright treaty -adopted on 20 December 1996, or similar laws prohibiting or restricting circumvention -of such measures. +No covered work shall be deemed part of an effective technological measure under +any applicable law fulfilling obligations under article 11 of the WIPO copyright +treaty adopted on 20 December 1996, or similar laws prohibiting or restricting +circumvention of such measures. -When you convey a covered work, you waive any legal power to forbid circumvention of -technological measures to the extent such circumvention is effected by exercising -rights under this License with respect to the covered work, and you disclaim any -intention to limit operation or modification of the work as a means of enforcing, -against the work's users, your or third parties' legal rights to forbid circumvention -of technological measures. +When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention is +effected by exercising rights under this License with respect to the covered +work, and you disclaim any intention to limit operation or modification of the +work as a means of enforcing, against the work's users, your or third parties' +legal rights to forbid circumvention of technological measures. ### 4. Conveying Verbatim Copies -You may convey verbatim copies of the Program's source code as you receive it, in any -medium, provided that you conspicuously and appropriately publish on each copy an -appropriate copyright notice; keep intact all notices stating that this License and -any non-permissive terms added in accord with section 7 apply to the code; keep -intact all notices of the absence of any warranty; and give all recipients a copy of -this License along with the Program. +You may convey verbatim copies of the Program's source code as you receive it, +in any medium, provided that you conspicuously and appropriately publish on each +copy an appropriate copyright notice; keep intact all notices stating that this +License and any non-permissive terms added in accord with section 7 apply to the +code; keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. -You may charge any price or no price for each copy that you convey, and you may offer -support or warranty protection for a fee. +You may charge any price or no price for each copy that you convey, and you may +offer support or warranty protection for a fee. ### 5. Conveying Modified Source Versions -You may convey a work based on the Program, or the modifications to produce it from -the Program, in the form of source code under the terms of section 4, provided that -you also meet all of these conditions: +You may convey a work based on the Program, or the modifications to produce it +from the Program, in the form of source code under the terms of section 4, +provided that you also meet all of these conditions: -* **a)** The work must carry prominent notices stating that you modified it, and giving a -relevant date. -* **b)** The work must carry prominent notices stating that it is released under this -License and any conditions added under section 7. This requirement modifies the -requirement in section 4 to “keep intact all notices”. -* **c)** You must license the entire work, as a whole, under this License to anyone who -comes into possession of a copy. This License will therefore apply, along with any -applicable section 7 additional terms, to the whole of the work, and all its parts, -regardless of how they are packaged. This License gives no permission to license the -work in any other way, but it does not invalidate such permission if you have -separately received it. -* **d)** If the work has interactive user interfaces, each must display Appropriate Legal -Notices; however, if the Program has interactive interfaces that do not display -Appropriate Legal Notices, your work need not make them do so. +- **a)** The work must carry prominent notices stating that you modified it, and + giving a relevant date. +- **b)** The work must carry prominent notices stating that it is released under + this License and any conditions added under section 7. This requirement + modifies the requirement in section 4 to “keep intact all notices”. +- **c)** You must license the entire work, as a whole, under this License to + anyone who comes into possession of a copy. This License will therefore apply, + along with any applicable section 7 additional terms, to the whole of the + work, and all its parts, regardless of how they are packaged. This License + gives no permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. +- **d)** If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive interfaces + that do not display Appropriate Legal Notices, your work need not make them do + so. -A compilation of a covered work with other separate and independent works, which are -not by their nature extensions of the covered work, and which are not combined with -it such as to form a larger program, in or on a volume of a storage or distribution -medium, is called an “aggregate” if the compilation and its resulting -copyright are not used to limit the access or legal rights of the compilation's users -beyond what the individual works permit. Inclusion of a covered work in an aggregate -does not cause this License to apply to the other parts of the aggregate. +A compilation of a covered work with other separate and independent works, which +are not by their nature extensions of the covered work, and which are not +combined with it such as to form a larger program, in or on a volume of a +storage or distribution medium, is called an “aggregate” if the compilation and +its resulting copyright are not used to limit the access or legal rights of the +compilation's users beyond what the individual works permit. Inclusion of a +covered work in an aggregate does not cause this License to apply to the other +parts of the aggregate. ### 6. Conveying Non-Source Forms -You may convey a covered work in object code form under the terms of sections 4 and -5, provided that you also convey the machine-readable Corresponding Source under the -terms of this License, in one of these ways: +You may convey a covered work in object code form under the terms of sections 4 +and 5, provided that you also convey the machine-readable Corresponding Source +under the terms of this License, in one of these ways: -* **a)** Convey the object code in, or embodied in, a physical product (including a -physical distribution medium), accompanied by the Corresponding Source fixed on a -durable physical medium customarily used for software interchange. -* **b)** Convey the object code in, or embodied in, a physical product (including a -physical distribution medium), accompanied by a written offer, valid for at least -three years and valid for as long as you offer spare parts or customer support for -that product model, to give anyone who possesses the object code either **(1)** a copy of -the Corresponding Source for all the software in the product that is covered by this -License, on a durable physical medium customarily used for software interchange, for -a price no more than your reasonable cost of physically performing this conveying of -source, or **(2)** access to copy the Corresponding Source from a network server at no -charge. -* **c)** Convey individual copies of the object code with a copy of the written offer to -provide the Corresponding Source. This alternative is allowed only occasionally and -noncommercially, and only if you received the object code with such an offer, in -accord with subsection 6b. -* **d)** Convey the object code by offering access from a designated place (gratis or for -a charge), and offer equivalent access to the Corresponding Source in the same way -through the same place at no further charge. You need not require recipients to copy -the Corresponding Source along with the object code. If the place to copy the object -code is a network server, the Corresponding Source may be on a different server -(operated by you or a third party) that supports equivalent copying facilities, -provided you maintain clear directions next to the object code saying where to find -the Corresponding Source. Regardless of what server hosts the Corresponding Source, -you remain obligated to ensure that it is available for as long as needed to satisfy -these requirements. -* **e)** Convey the object code using peer-to-peer transmission, provided you inform -other peers where the object code and Corresponding Source of the work are being -offered to the general public at no charge under subsection 6d. +- **a)** Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the Corresponding + Source fixed on a durable physical medium customarily used for software + interchange. +- **b)** Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a written offer, + valid for at least three years and valid for as long as you offer spare parts + or customer support for that product model, to give anyone who possesses the + object code either **(1)** a copy of the Corresponding Source for all the + software in the product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no more than + your reasonable cost of physically performing this conveying of source, or + **(2)** access to copy the Corresponding Source from a network server at no + charge. +- **c)** Convey individual copies of the object code with a copy of the written + offer to provide the Corresponding Source. This alternative is allowed only + occasionally and noncommercially, and only if you received the object code + with such an offer, in accord with subsection 6b. +- **d)** Convey the object code by offering access from a designated place + (gratis or for a charge), and offer equivalent access to the Corresponding + Source in the same way through the same place at no further charge. You need + not require recipients to copy the Corresponding Source along with the object + code. If the place to copy the object code is a network server, the + Corresponding Source may be on a different server (operated by you or a third + party) that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the Corresponding + Source, you remain obligated to ensure that it is available for as long as + needed to satisfy these requirements. +- **e)** Convey the object code using peer-to-peer transmission, provided you + inform other peers where the object code and Corresponding Source of the work + are being offered to the general public at no charge under subsection 6d. A separable portion of the object code, whose source code is excluded from the Corresponding Source as a System Library, need not be included in conveying the object code work. -A “User Product” is either **(1)** a “consumer product”, which -means any tangible personal property which is normally used for personal, family, or -household purposes, or **(2)** anything designed or sold for incorporation into a -dwelling. In determining whether a product is a consumer product, doubtful cases -shall be resolved in favor of coverage. For a particular product received by a -particular user, “normally used” refers to a typical or common use of -that class of product, regardless of the status of the particular user or of the way -in which the particular user actually uses, or expects or is expected to use, the -product. A product is a consumer product regardless of whether the product has -substantial commercial, industrial or non-consumer uses, unless such uses represent -the only significant mode of use of the product. +A “User Product” is either **(1)** a “consumer product”, which means any +tangible personal property which is normally used for personal, family, or +household purposes, or **(2)** anything designed or sold for incorporation into +a dwelling. In determining whether a product is a consumer product, doubtful +cases shall be resolved in favor of coverage. For a particular product received +by a particular user, “normally used” refers to a typical or common use of that +class of product, regardless of the status of the particular user or of the way +in which the particular user actually uses, or expects or is expected to use, +the product. A product is a consumer product regardless of whether the product +has substantial commercial, industrial or non-consumer uses, unless such uses +represent the only significant mode of use of the product. -“Installation Information” for a User Product means any methods, -procedures, authorization keys, or other information required to install and execute -modified versions of a covered work in that User Product from a modified version of -its Corresponding Source. The information must suffice to ensure that the continued -functioning of the modified object code is in no case prevented or interfered with -solely because modification has been made. +“Installation Information” for a User Product means any methods, procedures, +authorization keys, or other information required to install and execute +modified versions of a covered work in that User Product from a modified version +of its Corresponding Source. The information must suffice to ensure that the +continued functioning of the modified object code is in no case prevented or +interfered with solely because modification has been made. -If you convey an object code work under this section in, or with, or specifically for -use in, a User Product, and the conveying occurs as part of a transaction in which -the right of possession and use of the User Product is transferred to the recipient -in perpetuity or for a fixed term (regardless of how the transaction is -characterized), the Corresponding Source conveyed under this section must be -accompanied by the Installation Information. But this requirement does not apply if -neither you nor any third party retains the ability to install modified object code -on the User Product (for example, the work has been installed in ROM). +If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as part of a +transaction in which the right of possession and use of the User Product is +transferred to the recipient in perpetuity or for a fixed term (regardless of +how the transaction is characterized), the Corresponding Source conveyed under +this section must be accompanied by the Installation Information. But this +requirement does not apply if neither you nor any third party retains the +ability to install modified object code on the User Product (for example, the +work has been installed in ROM). -The requirement to provide Installation Information does not include a requirement to -continue to provide support service, warranty, or updates for a work that has been -modified or installed by the recipient, or for the User Product in which it has been -modified or installed. Access to a network may be denied when the modification itself -materially and adversely affects the operation of the network or violates the rules -and protocols for communication across the network. +The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates for a +work that has been modified or installed by the recipient, or for the User +Product in which it has been modified or installed. Access to a network may be +denied when the modification itself materially and adversely affects the +operation of the network or violates the rules and protocols for communication +across the network. -Corresponding Source conveyed, and Installation Information provided, in accord with -this section must be in a format that is publicly documented (and with an +Corresponding Source conveyed, and Installation Information provided, in accord +with this section must be in a format that is publicly documented (and with an implementation available to the public in source code form), and must require no special password or key for unpacking, reading or copying. ### 7. Additional Terms -“Additional permissions” are terms that supplement the terms of this -License by making exceptions from one or more of its conditions. Additional -permissions that are applicable to the entire Program shall be treated as though they -were included in this License, to the extent that they are valid under applicable -law. If additional permissions apply only to part of the Program, that part may be -used separately under those permissions, but the entire Program remains governed by -this License without regard to the additional permissions. +“Additional permissions” are terms that supplement the terms of this License by +making exceptions from one or more of its conditions. Additional permissions +that are applicable to the entire Program shall be treated as though they were +included in this License, to the extent that they are valid under applicable +law. If additional permissions apply only to part of the Program, that part may +be used separately under those permissions, but the entire Program remains +governed by this License without regard to the additional permissions. When you convey a copy of a covered work, you may at your option remove any additional permissions from that copy, or from any part of it. (Additional -permissions may be written to require their own removal in certain cases when you -modify the work.) You may place additional permissions on material, added by you to a -covered work, for which you have or can give appropriate copyright permission. +permissions may be written to require their own removal in certain cases when +you modify the work.) You may place additional permissions on material, added by +you to a covered work, for which you have or can give appropriate copyright +permission. Notwithstanding any other provision of this License, for material you add to a covered work, you may (if authorized by the copyright holders of that material) supplement the terms of this License with terms: -* **a)** Disclaiming warranty or limiting liability differently from the terms of -sections 15 and 16 of this License; or -* **b)** Requiring preservation of specified reasonable legal notices or author -attributions in that material or in the Appropriate Legal Notices displayed by works -containing it; or -* **c)** Prohibiting misrepresentation of the origin of that material, or requiring that -modified versions of such material be marked in reasonable ways as different from the -original version; or -* **d)** Limiting the use for publicity purposes of names of licensors or authors of the -material; or -* **e)** Declining to grant rights under trademark law for use of some trade names, -trademarks, or service marks; or -* **f)** Requiring indemnification of licensors and authors of that material by anyone -who conveys the material (or modified versions of it) with contractual assumptions of -liability to the recipient, for any liability that these contractual assumptions -directly impose on those licensors and authors. +- **a)** Disclaiming warranty or limiting liability differently from the terms + of sections 15 and 16 of this License; or +- **b)** Requiring preservation of specified reasonable legal notices or author + attributions in that material or in the Appropriate Legal Notices displayed by + works containing it; or +- **c)** Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in reasonable ways + as different from the original version; or +- **d)** Limiting the use for publicity purposes of names of licensors or + authors of the material; or +- **e)** Declining to grant rights under trademark law for use of some trade + names, trademarks, or service marks; or +- **f)** Requiring indemnification of licensors and authors of that material by + anyone who conveys the material (or modified versions of it) with contractual + assumptions of liability to the recipient, for any liability that these + contractual assumptions directly impose on those licensors and authors. -All other non-permissive additional terms are considered “further -restrictions” within the meaning of section 10. If the Program as you received -it, or any part of it, contains a notice stating that it is governed by this License -along with a term that is a further restriction, you may remove that term. If a -license document contains a further restriction but permits relicensing or conveying -under this License, you may add to a covered work material governed by the terms of -that license document, provided that the further restriction does not survive such -relicensing or conveying. +All other non-permissive additional terms are considered “further restrictions” +within the meaning of section 10. If the Program as you received it, or any part +of it, contains a notice stating that it is governed by this License along with +a term that is a further restriction, you may remove that term. If a license +document contains a further restriction but permits relicensing or conveying +under this License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does not survive +such relicensing or conveying. -If you add terms to a covered work in accord with this section, you must place, in -the relevant source files, a statement of the additional terms that apply to those -files, or a notice indicating where to find the applicable terms. +If you add terms to a covered work in accord with this section, you must place, +in the relevant source files, a statement of the additional terms that apply to +those files, or a notice indicating where to find the applicable terms. Additional terms, permissive or non-permissive, may be stated in the form of a -separately written license, or stated as exceptions; the above requirements apply -either way. +separately written license, or stated as exceptions; the above requirements +apply either way. ### 8. Termination -You may not propagate or modify a covered work except as expressly provided under -this License. Any attempt otherwise to propagate or modify it is void, and will -automatically terminate your rights under this License (including any patent licenses -granted under the third paragraph of section 11). +You may not propagate or modify a covered work except as expressly provided +under this License. Any attempt otherwise to propagate or modify it is void, and +will automatically terminate your rights under this License (including any +patent licenses granted under the third paragraph of section 11). However, if you cease all violation of this License, then your license from a -particular copyright holder is reinstated **(a)** provisionally, unless and until the -copyright holder explicitly and finally terminates your license, and **(b)** permanently, -if the copyright holder fails to notify you of the violation by some reasonable means -prior to 60 days after the cessation. +particular copyright holder is reinstated **(a)** provisionally, unless and +until the copyright holder explicitly and finally terminates your license, and +**(b)** permanently, if the copyright holder fails to notify you of the +violation by some reasonable means prior to 60 days after the cessation. -Moreover, your license from a particular copyright holder is reinstated permanently -if the copyright holder notifies you of the violation by some reasonable means, this -is the first time you have received notice of violation of this License (for any -work) from that copyright holder, and you cure the violation prior to 30 days after -your receipt of the notice. +Moreover, your license from a particular copyright holder is reinstated +permanently if the copyright holder notifies you of the violation by some +reasonable means, this is the first time you have received notice of violation +of this License (for any work) from that copyright holder, and you cure the +violation prior to 30 days after your receipt of the notice. Termination of your rights under this section does not terminate the licenses of parties who have received copies or rights from you under this License. If your -rights have been terminated and not permanently reinstated, you do not qualify to -receive new licenses for the same material under section 10. +rights have been terminated and not permanently reinstated, you do not qualify +to receive new licenses for the same material under section 10. ### 9. Acceptance Not Required for Having Copies -You are not required to accept this License in order to receive or run a copy of the -Program. Ancillary propagation of a covered work occurring solely as a consequence of -using peer-to-peer transmission to receive a copy likewise does not require -acceptance. However, nothing other than this License grants you permission to -propagate or modify any covered work. These actions infringe copyright if you do not -accept this License. Therefore, by modifying or propagating a covered work, you -indicate your acceptance of this License to do so. +You are not required to accept this License in order to receive or run a copy of +the Program. Ancillary propagation of a covered work occurring solely as a +consequence of using peer-to-peer transmission to receive a copy likewise does +not require acceptance. However, nothing other than this License grants you +permission to propagate or modify any covered work. These actions infringe +copyright if you do not accept this License. Therefore, by modifying or +propagating a covered work, you indicate your acceptance of this License to do +so. ### 10. Automatic Licensing of Downstream Recipients -Each time you convey a covered work, the recipient automatically receives a license -from the original licensors, to run, modify and propagate that work, subject to this -License. You are not responsible for enforcing compliance by third parties with this -License. +Each time you convey a covered work, the recipient automatically receives a +license from the original licensors, to run, modify and propagate that work, +subject to this License. You are not responsible for enforcing compliance by +third parties with this License. An “entity transaction” is a transaction transferring control of an -organization, or substantially all assets of one, or subdividing an organization, or -merging organizations. If propagation of a covered work results from an entity -transaction, each party to that transaction who receives a copy of the work also -receives whatever licenses to the work the party's predecessor in interest had or -could give under the previous paragraph, plus a right to possession of the -Corresponding Source of the work from the predecessor in interest, if the predecessor -has it or can get it with reasonable efforts. +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered work results +from an entity transaction, each party to that transaction who receives a copy +of the work also receives whatever licenses to the work the party's predecessor +in interest had or could give under the previous paragraph, plus a right to +possession of the Corresponding Source of the work from the predecessor in +interest, if the predecessor has it or can get it with reasonable efforts. -You may not impose any further restrictions on the exercise of the rights granted or -affirmed under this License. For example, you may not impose a license fee, royalty, -or other charge for exercise of rights granted under this License, and you may not -initiate litigation (including a cross-claim or counterclaim in a lawsuit) alleging -that any patent claim is infringed by making, using, selling, offering for sale, or -importing the Program or any portion of it. +You may not impose any further restrictions on the exercise of the rights +granted or affirmed under this License. For example, you may not impose a +license fee, royalty, or other charge for exercise of rights granted under this +License, and you may not initiate litigation (including a cross-claim or +counterclaim in a lawsuit) alleging that any patent claim is infringed by +making, using, selling, offering for sale, or importing the Program or any +portion of it. ### 11. Patents -A “contributor” is a copyright holder who authorizes use under this -License of the Program or a work on which the Program is based. The work thus -licensed is called the contributor's “contributor version”. +A “contributor” is a copyright holder who authorizes use under this License of +the Program or a work on which the Program is based. The work thus licensed is +called the contributor's “contributor version”. A contributor's “essential patent claims” are all patent claims owned or -controlled by the contributor, whether already acquired or hereafter acquired, that -would be infringed by some manner, permitted by this License, of making, using, or -selling its contributor version, but do not include claims that would be infringed -only as a consequence of further modification of the contributor version. For -purposes of this definition, “control” includes the right to grant patent -sublicenses in a manner consistent with the requirements of this License. +controlled by the contributor, whether already acquired or hereafter acquired, +that would be infringed by some manner, permitted by this License, of making, +using, or selling its contributor version, but do not include claims that would +be infringed only as a consequence of further modification of the contributor +version. For purposes of this definition, “control” includes the right to grant +patent sublicenses in a manner consistent with the requirements of this License. -Each contributor grants you a non-exclusive, worldwide, royalty-free patent license -under the contributor's essential patent claims, to make, use, sell, offer for sale, -import and otherwise run, modify and propagate the contents of its contributor -version. +Each contributor grants you a non-exclusive, worldwide, royalty-free patent +license under the contributor's essential patent claims, to make, use, sell, +offer for sale, import and otherwise run, modify and propagate the contents of +its contributor version. -In the following three paragraphs, a “patent license” is any express -agreement or commitment, however denominated, not to enforce a patent (such as an -express permission to practice a patent or covenant not to sue for patent -infringement). To “grant” such a patent license to a party means to make -such an agreement or commitment not to enforce a patent against the party. +In the following three paragraphs, a “patent license” is any express agreement +or commitment, however denominated, not to enforce a patent (such as an express +permission to practice a patent or covenant not to sue for patent infringement). +To “grant” such a patent license to a party means to make such an agreement or +commitment not to enforce a patent against the party. If you convey a covered work, knowingly relying on a patent license, and the -Corresponding Source of the work is not available for anyone to copy, free of charge -and under the terms of this License, through a publicly available network server or -other readily accessible means, then you must either **(1)** cause the Corresponding -Source to be so available, or **(2)** arrange to deprive yourself of the benefit of the -patent license for this particular work, or **(3)** arrange, in a manner consistent with -the requirements of this License, to extend the patent license to downstream -recipients. “Knowingly relying” means you have actual knowledge that, but -for the patent license, your conveying the covered work in a country, or your -recipient's use of the covered work in a country, would infringe one or more -identifiable patents in that country that you have reason to believe are valid. +Corresponding Source of the work is not available for anyone to copy, free of +charge and under the terms of this License, through a publicly available network +server or other readily accessible means, then you must either **(1)** cause the +Corresponding Source to be so available, or **(2)** arrange to deprive yourself +of the benefit of the patent license for this particular work, or **(3)** +arrange, in a manner consistent with the requirements of this License, to extend +the patent license to downstream recipients. “Knowingly relying” means you have +actual knowledge that, but for the patent license, your conveying the covered +work in a country, or your recipient's use of the covered work in a country, +would infringe one or more identifiable patents in that country that you have +reason to believe are valid. If, pursuant to or in connection with a single transaction or arrangement, you -convey, or propagate by procuring conveyance of, a covered work, and grant a patent -license to some of the parties receiving the covered work authorizing them to use, -propagate, modify or convey a specific copy of the covered work, then the patent -license you grant is automatically extended to all recipients of the covered work and -works based on it. +convey, or propagate by procuring conveyance of, a covered work, and grant a +patent license to some of the parties receiving the covered work authorizing +them to use, propagate, modify or convey a specific copy of the covered work, +then the patent license you grant is automatically extended to all recipients of +the covered work and works based on it. -A patent license is “discriminatory” if it does not include within the -scope of its coverage, prohibits the exercise of, or is conditioned on the -non-exercise of one or more of the rights that are specifically granted under this -License. You may not convey a covered work if you are a party to an arrangement with -a third party that is in the business of distributing software, under which you make -payment to the third party based on the extent of your activity of conveying the -work, and under which the third party grants, to any of the parties who would receive -the covered work from you, a discriminatory patent license **(a)** in connection with -copies of the covered work conveyed by you (or copies made from those copies), or **(b)** -primarily for and in connection with specific products or compilations that contain -the covered work, unless you entered into that arrangement, or that patent license -was granted, prior to 28 March 2007. +A patent license is “discriminatory” if it does not include within the scope of +its coverage, prohibits the exercise of, or is conditioned on the non-exercise +of one or more of the rights that are specifically granted under this License. +You may not convey a covered work if you are a party to an arrangement with a +third party that is in the business of distributing software, under which you +make payment to the third party based on the extent of your activity of +conveying the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory patent +license **(a)** in connection with copies of the covered work conveyed by you +(or copies made from those copies), or **(b)** primarily for and in connection +with specific products or compilations that contain the covered work, unless you +entered into that arrangement, or that patent license was granted, prior to 28 +March 2007. Nothing in this License shall be construed as excluding or limiting any implied license or other defenses to infringement that may otherwise be available to you @@ -470,51 +485,55 @@ under applicable patent law. ### 12. No Surrender of Others' Freedom -If conditions are imposed on you (whether by court order, agreement or otherwise) -that contradict the conditions of this License, they do not excuse you from the -conditions of this License. If you cannot convey a covered work so as to satisfy -simultaneously your obligations under this License and any other pertinent -obligations, then as a consequence you may not convey it at all. For example, if you -agree to terms that obligate you to collect a royalty for further conveying from -those to whom you convey the Program, the only way you could satisfy both those terms -and this License would be to refrain entirely from conveying the Program. +If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not excuse +you from the conditions of this License. If you cannot convey a covered work so +as to satisfy simultaneously your obligations under this License and any other +pertinent obligations, then as a consequence you may not convey it at all. For +example, if you agree to terms that obligate you to collect a royalty for +further conveying from those to whom you convey the Program, the only way you +could satisfy both those terms and this License would be to refrain entirely +from conveying the Program. ### 13. Use with the GNU Affero General Public License -Notwithstanding any other provision of this License, you have permission to link or -combine any covered work with a work licensed under version 3 of the GNU Affero -General Public License into a single combined work, and to convey the resulting work. -The terms of this License will continue to apply to the part which is the covered -work, but the special requirements of the GNU Affero General Public License, section -13, concerning interaction through a network will apply to the combination as such. +Notwithstanding any other provision of this License, you have permission to link +or combine any covered work with a work licensed under version 3 of the GNU +Affero General Public License into a single combined work, and to convey the +resulting work. The terms of this License will continue to apply to the part +which is the covered work, but the special requirements of the GNU Affero +General Public License, section 13, concerning interaction through a network +will apply to the combination as such. ### 14. Revised Versions of this License The Free Software Foundation may publish revised and/or new versions of the GNU -General Public License from time to time. Such new versions will be similar in spirit -to the present version, but may differ in detail to address new problems or concerns. +General Public License from time to time. Such new versions will be similar in +spirit to the present version, but may differ in detail to address new problems +or concerns. -Each version is given a distinguishing version number. If the Program specifies that -a certain numbered version of the GNU General Public License “or any later +Each version is given a distinguishing version number. If the Program specifies +that a certain numbered version of the GNU General Public License “or any later version” applies to it, you have the option of following the terms and -conditions either of that numbered version or of any later version published by the -Free Software Foundation. If the Program does not specify a version number of the GNU -General Public License, you may choose any version ever published by the Free -Software Foundation. +conditions either of that numbered version or of any later version published by +the Free Software Foundation. If the Program does not specify a version number +of the GNU General Public License, you may choose any version ever published by +the Free Software Foundation. -If the Program specifies that a proxy can decide which future versions of the GNU -General Public License can be used, that proxy's public statement of acceptance of a -version permanently authorizes you to choose that version for the Program. +If the Program specifies that a proxy can decide which future versions of the +GNU General Public License can be used, that proxy's public statement of +acceptance of a version permanently authorizes you to choose that version for +the Program. -Later license versions may give you additional or different permissions. However, no -additional obligations are imposed on any author or copyright holder as a result of -your choosing to follow a later version. +Later license versions may give you additional or different permissions. +However, no additional obligations are imposed on any author or copyright holder +as a result of your choosing to follow a later version. ### 15. Disclaimer of Warranty THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. -EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES -PROVIDE THE PROGRAM “AS IS” WITHOUT WARRANTY OF ANY KIND, EITHER +EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER +PARTIES PROVIDE THE PROGRAM “AS IS” WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE @@ -525,32 +544,32 @@ DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, -INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE -PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE -OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE -WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE -POSSIBILITY OF SUCH DAMAGES. +INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE +THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED +INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE +PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY +HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. ### 17. Interpretation of Sections 15 and 16 -If the disclaimer of warranty and limitation of liability provided above cannot be -given local legal effect according to their terms, reviewing courts shall apply local -law that most closely approximates an absolute waiver of all civil liability in -connection with the Program, unless a warranty or assumption of liability accompanies -a copy of the Program in return for a fee. +If the disclaimer of warranty and limitation of liability provided above cannot +be given local legal effect according to their terms, reviewing courts shall +apply local law that most closely approximates an absolute waiver of all civil +liability in connection with the Program, unless a warranty or assumption of +liability accompanies a copy of the Program in return for a fee. _END OF TERMS AND CONDITIONS_ ## How to Apply These Terms to Your New Programs -If you develop a new program, and you want it to be of the greatest possible use to -the public, the best way to achieve this is to make it free software which everyone -can redistribute and change under these terms. +If you develop a new program, and you want it to be of the greatest possible use +to the public, the best way to achieve this is to make it free software which +everyone can redistribute and change under these terms. -To do so, attach the following notices to the program. It is safest to attach them -to the start of each source file to most effectively state the exclusion of warranty; -and each file should have at least the “copyright” line and a pointer to -where the full notice is found. +To do so, attach the following notices to the program. It is safest to attach +them to the start of each source file to most effectively state the exclusion of +warranty; and each file should have at least the “copyright” line and a pointer +to where the full notice is found. Copyright (C) @@ -570,26 +589,26 @@ where the full notice is found. Also add information on how to contact you by electronic and paper mail. -If the program does terminal interaction, make it output a short notice like this -when it starts in an interactive mode: +If the program does terminal interaction, make it output a short notice like +this when it starts in an interactive mode: Copyright (C) This program comes with ABSOLUTELY NO WARRANTY; for details type 'show w'. This is free software, and you are welcome to redistribute it under certain conditions; type 'show c' for details. -The hypothetical commands `show w` and `show c` should show the appropriate parts of -the General Public License. Of course, your program's commands might be different; -for a GUI interface, you would use an “about box”. +The hypothetical commands `show w` and `show c` should show the appropriate +parts of the General Public License. Of course, your program's commands might be +different; for a GUI interface, you would use an “about box”. -You should also get your employer (if you work as a programmer) or school, if any, to -sign a “copyright disclaimer” for the program, if necessary. For more +You should also get your employer (if you work as a programmer) or school, if +any, to sign a “copyright disclaimer” for the program, if necessary. For more information on this, and how to apply and follow the GNU GPL, see <>. The GNU General Public License does not permit incorporating your program into -proprietary programs. If your program is a subroutine library, you may consider it -more useful to permit linking proprietary applications with the library. If this is -what you want to do, use the GNU Lesser General Public License instead of this -License. But first, please read +proprietary programs. If your program is a subroutine library, you may consider +it more useful to permit linking proprietary applications with the library. If +this is what you want to do, use the GNU Lesser General Public License instead +of this License. But first, please read <>. diff --git a/README.md b/README.md index afd9e7161..8f89b4d26 100644 --- a/README.md +++ b/README.md @@ -1,25 +1,27 @@ # nix-eval-jobs This project evaluates nix attribute sets in parallel with streamable json -output. This is useful for time and memory intensive evaluations such as NixOS -machines, i.e. in a CI context. The evaluation is done with a controllable +output. This is useful for time and memory intensive evaluations such as NixOS +machines, i.e. in a CI context. The evaluation is done with a controllable number of threads that are restarted when their memory consumption exceeds a certain threshold. To facilitate integration, nix-eval-jobs creates garbage collection roots for each evaluated derivation (drv file, not the build) within the provided -attribute. This prevents race conditions between the nix garbage collection +attribute. This prevents race conditions between the nix garbage collection service and user-started nix builds processes. ## Why using nix-eval-jobs? - Faster evaluation by using threads -- Memory used for evaluation is reclaimed after nix-eval-jobs finish, so that the build can use it. +- Memory used for evaluation is reclaimed after nix-eval-jobs finish, so that + the build can use it. - Evaluation of jobs can fail individually ## Example -In the following example we evaluate the hydraJobs attribute of the [patchelf](https://github.com/NixOS/patchelf) flake: +In the following example we evaluate the hydraJobs attribute of the +[patchelf](https://github.com/NixOS/patchelf) flake: ```console $ nix-eval-jobs --gc-roots-dir gcroot --flake 'github:NixOS/patchelf#hydraJobs' @@ -30,11 +32,12 @@ $ nix-eval-jobs --gc-roots-dir gcroot --flake 'github:NixOS/patchelf#hydraJobs' The output here is newline-seperated json according to https://jsonlines.org. -The code is derived from [hydra's](https://github.com/nixos/hydra) eval-jobs executable. +The code is derived from [hydra's](https://github.com/nixos/hydra) eval-jobs +executable. ## Further options -``` console +```console $ nix-eval-jobs --help USAGE: nix-eval-jobs [options] expr @@ -60,11 +63,10 @@ USAGE: nix-eval-jobs [options] expr --workers number of evaluate workers ``` - ## Potential use-cases for the tool **Faster evaluator in deployment tools.** When evaluating NixOS machines, -evaluation can take several minutes when run on a single core. This limits +evaluation can take several minutes when run on a single core. This limits scalability for large deployments with deployment tools such as [NixOps](https://github.com/NixOS/nixops). @@ -77,7 +79,6 @@ single large log file. In the [wiki](https://github.com/nix-community/nix-eval-jobs/wiki#ci-example-configurations) we collect example ci configuration for various CIs. - ## Organisation of this repository On the `main` branch we target nixUnstable. When a release of nix happens, we @@ -86,41 +87,44 @@ fork for a release branch i.e. `release-2.8` and change the nix version in to these release branches. At the time of writing we only intent to support the latest release branch. - ## Projects using nix-eval-jobs -- [nix-fast-build](https://github.com/Mic92/nix-fast-build) - Combine the power of nix-eval-jobs with nix-output-monitor to speed-up your evaluation and building process -- [buildbot-nix](https://github.com/Mic92/buildbot-nix) - A nixos module to make buildbot a proper Nix-CI -- [colmena](https://github.com/zhaofengli/colmena) - A simple, stateless NixOS deployment tool -- [robotnix](https://github.com/danielfullmer/robotnix) - Build Android (AOSP) using Nix, used in their [CI](https://github.com/danielfullmer/robotnix/blob/38b80700ee4265c306dcfdcce45056e32ab2973f/.github/workflows/instantiate.yml#L18) +- [nix-fast-build](https://github.com/Mic92/nix-fast-build) - Combine the power + of nix-eval-jobs with nix-output-monitor to speed-up your evaluation and + building process +- [buildbot-nix](https://github.com/Mic92/buildbot-nix) - A nixos module to make + buildbot a proper Nix-CI +- [colmena](https://github.com/zhaofengli/colmena) - A simple, stateless NixOS + deployment tool +- [robotnix](https://github.com/danielfullmer/robotnix) - Build Android (AOSP) + using Nix, used in their + [CI](https://github.com/danielfullmer/robotnix/blob/38b80700ee4265c306dcfdcce45056e32ab2973f/.github/workflows/instantiate.yml#L18) ## FAQ ### nix-eval-jobs consumes too much memory / is too slow -By default, nix-eval-jobs spawns as many worker processes as there are -hardware threads in the system and limits the memory usage for each worker to -4GB. +By default, nix-eval-jobs spawns as many worker processes as there are hardware +threads in the system and limits the memory usage for each worker to 4GB. However, keep in mind that each worker process may need to re-evaluate shared dependencies of the attributes, which can introduce some overhead for each -evaluation or cause workers to exceed their memory limit. If you encounter -these situations, you can tune the following options: +evaluation or cause workers to exceed their memory limit. If you encounter these +situations, you can tune the following options: `--workers`: This option allows you to set the number of evaluation workers that -nix-eval-jobs should spawn. You can increase or decrease this number to -optimize the evaluation speed and memory usage. For example, if you have a -system with many CPU cores but limited memory, you may want to reduce the -number of workers to avoid exceeding the memory limit. +nix-eval-jobs should spawn. You can increase or decrease this number to optimize +the evaluation speed and memory usage. For example, if you have a system with +many CPU cores but limited memory, you may want to reduce the number of workers +to avoid exceeding the memory limit. `--max-memory-size`: This option allows you to adjust the memory limit for each worker process. By default, it's set to 4GiB, but you can increase or decrease -this value as needed. For example, if you have a system with a lot of memory -and want to speed up the evaluation, you may want to increase the memory limit -to allow workers to cache more data in memory before getting restarted by -nix-eval-jobs. -Note that this is not a hard limit and memory usage may rise above the limit momentarily -before the worker process exits. +this value as needed. For example, if you have a system with a lot of memory and +want to speed up the evaluation, you may want to increase the memory limit to +allow workers to cache more data in memory before getting restarted by +nix-eval-jobs. Note that this is not a hard limit and memory usage may rise +above the limit momentarily before the worker process exits. Overall, tuning these options can help you optimize the performance and memory usage of nix-eval-jobs to better fit your system and evaluation needs. diff --git a/dev/treefmt.nix b/dev/treefmt.nix index 3f4c03566..f505413f2 100644 --- a/dev/treefmt.nix +++ b/dev/treefmt.nix @@ -1,12 +1,8 @@ -{ pkgs, lib, ... }: { +{ pkgs, ... }: { # Used to find the project root projectRootFile = "flake.lock"; - programs.prettier.enable = true; - programs.prettier.package = pkgs.writeShellScriptBin "prettier" '' - export NODE_PATH=${pkgs.nodePackages.prettier-plugin-toml}/lib/node_modules - exec ${pkgs.nodePackages.prettier}/bin/prettier "$@" - ''; + programs.deno.enable = true; programs.clang-format.enable = true; @@ -32,8 +28,6 @@ clang-format = { }; - prettier.includes = lib.mkForce [ "*.toml" ]; - python = { command = "sh"; options = [ From 93972c0c1887766c8bc4419f29eb42c5caa398b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sun, 10 Dec 2023 08:16:54 +0100 Subject: [PATCH 354/419] handle broken evaluation worker pipes on write --- src/nix-eval-jobs.cc | 108 +++++++++++++++++++++++++++++++++++++------ 1 file changed, 94 insertions(+), 14 deletions(-) diff --git a/src/nix-eval-jobs.cc b/src/nix-eval-jobs.cc index b766dee5d..e2e78d7e8 100644 --- a/src/nix-eval-jobs.cc +++ b/src/nix-eval-jobs.cc @@ -299,6 +299,46 @@ std::string attrPathJoin(json input) { return 0; } +class LineReader { + public: + LineReader(int fd) { + stream = fdopen(fd, "r"); + if (!stream) { + throw Error("fdopen failed: %s", strerror(errno)); + } + } + + ~LineReader() { + fclose(stream); + free(buffer); + } + + LineReader(LineReader &&other) { + stream = other.stream; + other.stream = nullptr; + buffer = other.buffer; + other.buffer = nullptr; + len = other.len; + other.len = 0; + } + + [[nodiscard]] std::string_view readLine() { + ssize_t read = getline(&buffer, &len, stream); + + if (read == -1) { + return {}; // Return an empty string_view in case of error + } + + // Remove trailing newline + return std::string_view(buffer, read - 1); + } + + private: + FILE *stream = nullptr; + char *buffer = nullptr; + size_t len = 0; +}; + static void worker(ref state, Bindings &autoArgs, AutoCloseFD &to, AutoCloseFD &from) { @@ -317,13 +357,15 @@ static void worker(ref state, Bindings &autoArgs, AutoCloseFD &to, } }(); + LineReader fromReader(from.release()); + while (true) { /* Wait for the collector to send us a job name. */ if (tryWriteLine(to.get(), "next") < 0) { return; // main process died } - auto s = readLine(from.get()); + auto s = fromReader.readLine(); if (s == "exit") break; if (!hasPrefix(s, "do ")) @@ -478,19 +520,28 @@ struct State { std::exception_ptr exc; }; -void handleBrokenWorkerPipe(pid_t child) { +void handleBrokenWorkerPipe(Proc &proc) { while (1) { - int rc = waitpid(child, nullptr, WNOHANG); + int rc = waitpid(proc.pid, nullptr, WNOHANG); if (rc == 0) { + proc.pid = -1; // we already took the process status from Proc, no + // need to wait for it again to avoid error messages throw Error("BUG: worker pipe closed but worker still running?"); } else if (rc == -1) { + proc.pid = -1; throw Error("BUG: waitpid waiting for worker failed: %s", strerror(errno)); } else { if (WIFEXITED(rc)) { + proc.pid = -1; throw Error("evaluation worker exited with %d", WEXITSTATUS(rc)); } else if (WIFSIGNALED(rc)) { + proc.pid = -1; + if (WTERMSIG(rc) == SIGKILL) { + throw Error("evaluation worker killed by SIGKILL, maybe " + "memory limit reached?"); + } throw Error("evaluation worker killed by signal %d", WTERMSIG(rc)); } // else ignore WIFSTOPPED and WIFCONTINUED @@ -503,20 +554,35 @@ std::function collector(Sync &state_, return [&]() { try { std::optional> proc_; + std::optional> fromReader_; while (true) { - - auto proc = proc_.has_value() ? std::move(proc_.value()) - : std::make_unique(worker); + if (!proc_.has_value()) { + proc_ = std::make_unique(worker); + fromReader_ = std::make_unique( + proc_.value()->from.release()); + } + auto proc = std::move(proc_.value()); + auto fromReader = std::move(fromReader_.value()); /* Check whether the existing worker process is still there. */ - auto s = readLine(proc->from.get()); - if (s == "restart") { + auto s = fromReader->readLine(); + if (s == "") { + handleBrokenWorkerPipe(*proc.get()); + } else if (s == "restart") { proc_ = std::nullopt; + fromReader_ = std::nullopt; continue; } else if (s != "next") { - auto json = json::parse(s); - throw Error("worker error: %s", (std::string)json["error"]); + try { + auto json = json::parse(s); + throw Error("worker error: %s", + (std::string)json["error"]); + } catch (const json::exception &e) { + throw Error( + "Received invalid JSON from worker: %s '%s'", + e.what(), s); + } } /* Wait for a job name to become available. */ @@ -528,7 +594,7 @@ std::function collector(Sync &state_, if ((state->todo.empty() && state->active.empty()) || state->exc) { if (tryWriteLine(proc->to.get(), "exit") < 0) { - handleBrokenWorkerPipe(proc->pid); + handleBrokenWorkerPipe(*proc.get()); } return; } @@ -543,12 +609,25 @@ std::function collector(Sync &state_, /* Tell the worker to evaluate it. */ if (tryWriteLine(proc->to.get(), "do " + attrPath.dump()) < 0) { - handleBrokenWorkerPipe(proc->pid); + handleBrokenWorkerPipe(*proc.get()); } /* Wait for the response. */ - auto respString = readLine(proc->from.get()); - auto response = json::parse(respString); + auto respString = fromReader->readLine(); + if (respString == "") { + handleBrokenWorkerPipe(*proc.get()); + } + json response; + try { + response = json::parse(respString); + if (response.find("error") != response.end()) { + throw Error("worker error: %s", + (std::string)response["error"]); + } + } catch (const json::exception &e) { + throw Error("Received invalid JSON from worker: %s '%s'", + e.what(), respString); + } /* Handle the response. */ std::vector newAttrs; @@ -564,6 +643,7 @@ std::function collector(Sync &state_, } proc_ = std::move(proc); + fromReader_ = std::move(fromReader); /* Add newly discovered job names to the queue. */ { From 00d3f014e729df1ccc21ee3ee07156516e34482d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sun, 10 Dec 2023 08:55:56 +0100 Subject: [PATCH 355/419] print error if worker receives invalid command --- src/nix-eval-jobs.cc | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/nix-eval-jobs.cc b/src/nix-eval-jobs.cc index e2e78d7e8..27be870b4 100644 --- a/src/nix-eval-jobs.cc +++ b/src/nix-eval-jobs.cc @@ -366,10 +366,14 @@ static void worker(ref state, Bindings &autoArgs, AutoCloseFD &to, } auto s = fromReader.readLine(); - if (s == "exit") + if (s == "exit") { break; - if (!hasPrefix(s, "do ")) + } + if (!hasPrefix(s, "do ")) { + fprintf(stderr, "worker error: received invalid command '%s'\n", + s.data()); abort(); + } auto path = json::parse(s.substr(3)); auto attrPathS = attrPathJoin(path); From f49cb8796379514c98b7acff21462470f4525fc8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sun, 10 Dec 2023 11:03:00 +0100 Subject: [PATCH 356/419] enable asan/ubsan in ci --- .github/workflows/nix-github-actions.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/nix-github-actions.yml b/.github/workflows/nix-github-actions.yml index f6eede4fa..405515166 100644 --- a/.github/workflows/nix-github-actions.yml +++ b/.github/workflows/nix-github-actions.yml @@ -47,7 +47,7 @@ jobs: extra_nix_config: | accept-flake-config = true - name: Build - run: nix develop -c bash -c 'meson build && cd build && ninja' + run: nix develop -c bash -c 'meson setup -Db_sanitize=address,undefined build && ninja -C build' - name: Run tests run: nix develop -c pytest ./tests From 6f4bee53f6b4a44f5f70ced82703ef9313128119 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sun, 10 Dec 2023 11:12:28 +0100 Subject: [PATCH 357/419] nix-github-action: fix value beeing interpreted as bool --- .github/workflows/nix-github-actions.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/nix-github-actions.yml b/.github/workflows/nix-github-actions.yml index 405515166..241dfa1a8 100644 --- a/.github/workflows/nix-github-actions.yml +++ b/.github/workflows/nix-github-actions.yml @@ -57,4 +57,4 @@ jobs: - nix-build - tests steps: - - run: true + - run: "true" From db3099bc8f607d26d3f7128a215446474aef1ce5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sun, 10 Dec 2023 14:22:32 +0100 Subject: [PATCH 358/419] move argument parsing to new file --- src/eval-args.cc | 93 ++++++++++++++++++++++++++ src/eval-args.hh | 32 +++++++++ src/meson.build | 1 + src/nix-eval-jobs.cc | 155 +++++-------------------------------------- 4 files changed, 142 insertions(+), 139 deletions(-) create mode 100644 src/eval-args.cc create mode 100644 src/eval-args.hh diff --git a/src/eval-args.cc b/src/eval-args.cc new file mode 100644 index 000000000..c0a8723ef --- /dev/null +++ b/src/eval-args.cc @@ -0,0 +1,93 @@ +#include "eval-args.hh" + +MyArgs::MyArgs() : MixCommonArgs("nix-eval-jobs") { + addFlag({ + .longName = "help", + .description = "show usage information", + .handler = {[&]() { + printf("USAGE: nix-eval-jobs [options] expr\n\n"); + for (const auto &[name, flag] : longFlags) { + if (hiddenCategories.count(flag->category)) { + continue; + } + printf(" --%-20s %s\n", name.c_str(), + flag->description.c_str()); + } + ::exit(0); + }}, + }); + + addFlag({.longName = "impure", + .description = "allow impure expressions", + .handler = {&impure, true}}); + + addFlag({.longName = "force-recurse", + .description = "force recursion (don't respect recurseIntoAttrs)", + .handler = {&forceRecurse, true}}); + + addFlag({.longName = "gc-roots-dir", + .description = "garbage collector roots directory", + .labels = {"path"}, + .handler = {&gcRootsDir}}); + + addFlag( + {.longName = "workers", + .description = "number of evaluate workers", + .labels = {"workers"}, + .handler = {[=, this](std::string s) { nrWorkers = std::stoi(s); }}}); + + addFlag({.longName = "max-memory-size", + .description = "maximum evaluation memory size in megabyte " + "(4GiB per worker by default)", + .labels = {"size"}, + .handler = { + [=, this](std::string s) { maxMemorySize = std::stoi(s); }}}); + + addFlag({.longName = "flake", + .description = "build a flake", + .handler = {&flake, true}}); + + addFlag({.longName = "meta", + .description = "include derivation meta field in output", + .handler = {&meta, true}}); + + addFlag({.longName = "check-cache-status", + .description = + "Check if the derivations are present locally or in " + "any configured substituters (i.e. binary cache). The " + "information " + "will be exposed in the `isCached` field of the JSON output.", + .handler = {&checkCacheStatus, true}}); + + addFlag( + {.longName = "show-trace", + .description = "print out a stack trace in case of evaluation errors", + .handler = {&showTrace, true}}); + + addFlag({.longName = "expr", + .shortName = 'E', + .description = "treat the argument as a Nix expression", + .handler = {&fromArgs, true}}); + + // usually in MixFlakeOptions + addFlag({ + .longName = "override-input", + .description = + "Override a specific flake input (e.g. `dwarffs/nixpkgs`).", + .category = category, + .labels = {"input-path", "flake-url"}, + .handler = {[&](std::string inputPath, std::string flakeRef) { + // overriden inputs are unlocked + lockFlags.allowUnlocked = true; + lockFlags.inputOverrides.insert_or_assign( + nix::flake::parseInputPath(inputPath), + nix::parseFlakeRef(flakeRef, nix::absPath("."), true)); + }}, + }); + + expectArg("expr", &releaseExpr); +} + +void MyArgs::parseArgs(char** argv, int argc) { + parseCmdline(nix::argvToStrings(argc, argv), 0); +} diff --git a/src/eval-args.hh b/src/eval-args.hh new file mode 100644 index 000000000..cd5a9aacc --- /dev/null +++ b/src/eval-args.hh @@ -0,0 +1,32 @@ +#pragma once + +#include +#include +#include +#include + +class MyArgs : virtual public nix::MixEvalArgs, + virtual public nix::MixCommonArgs, + virtual nix::RootArgs { + public: + std::string releaseExpr; + nix::Path gcRootsDir; + bool flake = false; + bool fromArgs = false; + bool meta = false; + bool showTrace = false; + bool impure = false; + bool forceRecurse = false; + bool checkCacheStatus = false; + size_t nrWorkers = 1; + size_t maxMemorySize = 4096; + + // usually in MixFlakeOptions + nix::flake::LockFlags lockFlags = {.updateLockFile = false, + .writeLockFile = false, + .useRegistries = false, + .allowUnlocked = false}; + MyArgs(); + + void parseArgs(char** argv, int argc); +}; diff --git a/src/meson.build b/src/meson.build index f8fb52881..d2a145c6e 100644 --- a/src/meson.build +++ b/src/meson.build @@ -1,5 +1,6 @@ src = [ 'nix-eval-jobs.cc', + 'eval-args.cc' ] executable('nix-eval-jobs', src, diff --git a/src/nix-eval-jobs.cc b/src/nix-eval-jobs.cc index 27be870b4..271a16ee5 100644 --- a/src/nix-eval-jobs.cc +++ b/src/nix-eval-jobs.cc @@ -1,36 +1,29 @@ + #include -#include #include -#include -#include +#include #include -#include -#include -#include -#include -#include -#include -#include +#include #include -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include + +#include +#include #include +#include #include +#include #include - +#include +#include #include - -#include +#include +#include #include #include +#include "eval-args.hh" + #include using namespace nix; @@ -42,122 +35,6 @@ using namespace nlohmann; #elif __clang__ #pragma clang diagnostic ignored "-Wnon-virtual-dtor" #endif -struct MyArgs : virtual MixEvalArgs, virtual MixCommonArgs, virtual RootArgs { - std::string releaseExpr; - Path gcRootsDir; - bool flake = false; - bool fromArgs = false; - bool meta = false; - bool showTrace = false; - bool impure = false; - bool forceRecurse = false; - bool checkCacheStatus = false; - size_t nrWorkers = 1; - size_t maxMemorySize = 4096; - - // usually in MixFlakeOptions - flake::LockFlags lockFlags = {.updateLockFile = false, - .writeLockFile = false, - .useRegistries = false, - .allowUnlocked = false}; - - MyArgs() : MixCommonArgs("nix-eval-jobs") { - addFlag({ - .longName = "help", - .description = "show usage information", - .handler = {[&]() { - printf("USAGE: nix-eval-jobs [options] expr\n\n"); - for (const auto &[name, flag] : longFlags) { - if (hiddenCategories.count(flag->category)) { - continue; - } - printf(" --%-20s %s\n", name.c_str(), - flag->description.c_str()); - } - ::exit(0); - }}, - }); - - addFlag({.longName = "impure", - .description = "allow impure expressions", - .handler = {&impure, true}}); - - addFlag( - {.longName = "force-recurse", - .description = "force recursion (don't respect recurseIntoAttrs)", - .handler = {&forceRecurse, true}}); - - addFlag({.longName = "gc-roots-dir", - .description = "garbage collector roots directory", - .labels = {"path"}, - .handler = {&gcRootsDir}}); - - addFlag({.longName = "workers", - .description = "number of evaluate workers", - .labels = {"workers"}, - .handler = { - [=, this](std::string s) { nrWorkers = std::stoi(s); }}}); - - addFlag({.longName = "max-memory-size", - .description = "maximum evaluation memory size in megabyte " - "(4GiB per worker by default)", - .labels = {"size"}, - .handler = {[=, this](std::string s) { - maxMemorySize = std::stoi(s); - }}}); - - addFlag({.longName = "flake", - .description = "build a flake", - .handler = {&flake, true}}); - - addFlag({.longName = "meta", - .description = "include derivation meta field in output", - .handler = {&meta, true}}); - - addFlag( - {.longName = "check-cache-status", - .description = - "Check if the derivations are present locally or in " - "any configured substituters (i.e. binary cache). The " - "information " - "will be exposed in the `isCached` field of the JSON output.", - .handler = {&checkCacheStatus, true}}); - - addFlag({.longName = "show-trace", - .description = - "print out a stack trace in case of evaluation errors", - .handler = {&showTrace, true}}); - - addFlag({.longName = "expr", - .shortName = 'E', - .description = "treat the argument as a Nix expression", - .handler = {&fromArgs, true}}); - - // usually in MixFlakeOptions - addFlag({ - .longName = "override-input", - .description = - "Override a specific flake input (e.g. `dwarffs/nixpkgs`).", - .category = category, - .labels = {"input-path", "flake-url"}, - .handler = {[&](std::string inputPath, std::string flakeRef) { - // overriden inputs are unlocked - lockFlags.allowUnlocked = true; - lockFlags.inputOverrides.insert_or_assign( - flake::parseInputPath(inputPath), - parseFlakeRef(flakeRef, absPath("."), true)); - }}, - }); - - expectArg("expr", &releaseExpr); - } -}; -#ifdef __GNUC__ -#pragma GCC diagnostic ignored "-Wnon-virtual-dtor" -#elif __clang__ -#pragma clang diagnostic ignored "-Wnon-virtual-dtor" -#endif - static MyArgs myArgs; static Value *releaseExprTopLevelValue(EvalState &state, Bindings &autoArgs) { @@ -679,7 +556,7 @@ int main(int argc, char **argv) { initNix(); initGC(); - myArgs.parseCmdline(argvToStrings(argc, argv), 0); + myArgs.parseArgs(argv, argc); /* FIXME: The build hook in conjunction with import-from-derivation is * causing "unexpected EOF" during eval */ From 880c66a7d10d680faf8ba41e4536dd9bdd045348 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sun, 10 Dec 2023 15:07:38 +0100 Subject: [PATCH 359/419] move drvs ot its own class --- src/drv.cc | 95 ++++++++++++++++++++++++++++++++++++++++++ src/drv.hh | 27 ++++++++++++ src/meson.build | 3 +- src/nix-eval-jobs.cc | 98 +------------------------------------------- 4 files changed, 126 insertions(+), 97 deletions(-) create mode 100644 src/drv.cc create mode 100644 src/drv.hh diff --git a/src/drv.cc b/src/drv.cc new file mode 100644 index 000000000..dc91ddfce --- /dev/null +++ b/src/drv.cc @@ -0,0 +1,95 @@ +#include "drvs.hh" +#include +#include +#include +#include +#include +#include + + +static bool queryIsCached(nix::Store &store, + std::map &outputs) { + uint64_t downloadSize, narSize; + nix::StorePathSet willBuild, willSubstitute, unknown; + + std::vector paths; + for (auto const &[key, val] : outputs) { + paths.push_back(followLinksToStorePathWithOutputs(store, val)); + } + + store.queryMissing(toDerivedPaths(paths), willBuild, willSubstitute, + unknown, downloadSize, narSize); + return willBuild.empty() && unknown.empty(); +} + +/* The fields of a derivation that are printed in json form */ +Drv::Drv(std::string &attrPath, nix::EvalState &state, nix::DrvInfo &drvInfo, MyArgs &args) { + + auto localStore = state.store.dynamic_pointer_cast(); + + try { + for (auto out : drvInfo.queryOutputs(true)) { + if (out.second) + outputs[out.first] = localStore->printStorePath(*out.second); + } + } catch (const std::exception &e) { + throw nix::EvalError("derivation '%s' does not have valid outputs: %s", + attrPath, e.what()); + } + + if (args.meta) { + nlohmann::json meta_; + for (auto &metaName : drvInfo.queryMetaNames()) { + nix::NixStringContext context; + std::stringstream ss; + + auto metaValue = drvInfo.queryMeta(metaName); + // Skip non-serialisable types + // TODO: Fix serialisation of derivations to store paths + if (metaValue == 0) { + continue; + } + + nix::printValueAsJSON(state, true, *metaValue, nix::noPos, ss, + context); + + meta_[metaName] = nlohmann::json::parse(ss.str()); + } + meta = meta_; + } + if (args.checkCacheStatus) { + cacheStatus = queryIsCached(*localStore, outputs) ? Drv::CacheStatus::Cached + : Drv::CacheStatus::Uncached; + } else { + cacheStatus = Drv::CacheStatus::Unknown; + } + + drvPath = localStore->printStorePath(drvInfo.requireDrvPath()); + + auto drv = localStore->readDerivation(drvInfo.requireDrvPath()); + for (const auto &[inputDrvPath, inputNode] : drv.inputDrvs.map) { + std::set inputDrvOutputs; + for (auto &outputName : inputNode.value) { + inputDrvOutputs.insert(outputName); + } + inputDrvs[localStore->printStorePath(inputDrvPath)] = inputDrvOutputs; + } + name = drvInfo.queryName(); + system = drv.platform; +} + +void to_json(nlohmann::json &json, const Drv &drv) { + json = nlohmann::json{{"name", drv.name}, + {"system", drv.system}, + {"drvPath", drv.drvPath}, + {"outputs", drv.outputs}, + {"inputDrvs", drv.inputDrvs}}; + + if (drv.meta.has_value()) { + json["meta"] = drv.meta.value(); + } + + if (drv.cacheStatus != Drv::CacheStatus::Unknown) { + json["isCached"] = drv.cacheStatus == Drv::CacheStatus::Cached; + } +} diff --git a/src/drv.hh b/src/drv.hh new file mode 100644 index 000000000..4f1fbd043 --- /dev/null +++ b/src/drv.hh @@ -0,0 +1,27 @@ +#include +#include +#include +#include +#include + +#include +#include + +#include + +#include "eval-args.hh" + +/* The fields of a derivation that are printed in json form */ +struct Drv { + std::string name; + std::string system; + std::string drvPath; + + enum class CacheStatus { Cached, Uncached, Unknown } cacheStatus; + std::map outputs; + std::map> inputDrvs; + std::optional meta; + + Drv(std::string &attrPath, nix::EvalState &state, nix::DrvInfo &drvInfo, MyArgs &args); +}; +void to_json(nlohmann::json &json, const Drv &drv); diff --git a/src/meson.build b/src/meson.build index d2a145c6e..6b8d6dc77 100644 --- a/src/meson.build +++ b/src/meson.build @@ -1,6 +1,7 @@ src = [ 'nix-eval-jobs.cc', - 'eval-args.cc' + 'eval-args.cc', + 'drv.cc' ] executable('nix-eval-jobs', src, diff --git a/src/nix-eval-jobs.cc b/src/nix-eval-jobs.cc index 271a16ee5..670bd4816 100644 --- a/src/nix-eval-jobs.cc +++ b/src/nix-eval-jobs.cc @@ -11,18 +11,17 @@ #include #include #include -#include #include #include #include #include #include #include -#include #include #include #include "eval-args.hh" +#include "drv.hh" #include @@ -55,99 +54,6 @@ static Value *releaseExprTopLevelValue(EvalState &state, Bindings &autoArgs) { return vRoot; } -bool queryIsCached(Store &store, std::map &outputs) { - uint64_t downloadSize, narSize; - StorePathSet willBuild, willSubstitute, unknown; - - std::vector paths; - for (auto const &[key, val] : outputs) { - paths.push_back(followLinksToStorePathWithOutputs(store, val)); - } - - store.queryMissing(toDerivedPaths(paths), willBuild, willSubstitute, - unknown, downloadSize, narSize); - return willBuild.empty() && unknown.empty(); -} - -/* The fields of a derivation that are printed in json form */ -struct Drv { - std::string name; - std::string system; - std::string drvPath; - bool isCached; - std::map outputs; - std::map> inputDrvs; - std::optional meta; - - Drv(std::string &attrPath, EvalState &state, DrvInfo &drvInfo) { - - auto localStore = state.store.dynamic_pointer_cast(); - - try { - for (auto out : drvInfo.queryOutputs(true)) { - if (out.second) - outputs[out.first] = - localStore->printStorePath(*out.second); - } - } catch (const std::exception &e) { - throw EvalError("derivation '%s' does not have valid outputs: %s", - attrPath, e.what()); - } - - if (myArgs.meta) { - nlohmann::json meta_; - for (auto &metaName : drvInfo.queryMetaNames()) { - NixStringContext context; - std::stringstream ss; - - auto metaValue = drvInfo.queryMeta(metaName); - // Skip non-serialisable types - // TODO: Fix serialisation of derivations to store paths - if (metaValue == 0) { - continue; - } - - printValueAsJSON(state, true, *metaValue, noPos, ss, context); - - meta_[metaName] = nlohmann::json::parse(ss.str()); - } - meta = meta_; - } - if (myArgs.checkCacheStatus) { - isCached = queryIsCached(*localStore, outputs); - } - - drvPath = localStore->printStorePath(drvInfo.requireDrvPath()); - - auto drv = localStore->readDerivation(drvInfo.requireDrvPath()); - for (const auto &[inputDrvPath, inputNode] : drv.inputDrvs.map) { - std::set inputDrvOutputs; - for (auto &outputName : inputNode.value) { - inputDrvOutputs.insert(outputName); - } - inputDrvs[localStore->printStorePath(inputDrvPath)] = - inputDrvOutputs; - } - name = drvInfo.queryName(); - system = drv.platform; - } -}; - -static void to_json(nlohmann::json &json, const Drv &drv) { - json = nlohmann::json{{"name", drv.name}, - {"system", drv.system}, - {"drvPath", drv.drvPath}, - {"outputs", drv.outputs}, - {"inputDrvs", drv.inputDrvs}}; - - if (drv.meta.has_value()) { - json["meta"] = drv.meta.value(); - } - - if (myArgs.checkCacheStatus) { - json["isCached"] = drv.isCached; - } -} std::string attrPathJoin(json input) { return std::accumulate(input.begin(), input.end(), std::string(), @@ -267,7 +173,7 @@ static void worker(ref state, Bindings &autoArgs, AutoCloseFD &to, if (v->type() == nAttrs) { if (auto drvInfo = getDerivation(*state, *v, false)) { - auto drv = Drv(attrPathS, *state, *drvInfo); + auto drv = Drv(attrPathS, *state, *drvInfo, myArgs); reply.update(drv); /* Register the derivation as a GC root. !!! This From a03f039a562f0b51bbbd9e5032a5b1042768d7cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sun, 10 Dec 2023 15:41:51 +0100 Subject: [PATCH 360/419] split nix-eval-jobs further into smaller files --- src/buffered-io.cc | 52 ++++++++++ src/buffered-io.hh | 20 ++++ src/drv.cc | 2 +- src/meson.build | 4 +- src/nix-eval-jobs.cc | 231 ++----------------------------------------- src/worker.cc | 173 ++++++++++++++++++++++++++++++++ src/worker.hh | 9 ++ 7 files changed, 265 insertions(+), 226 deletions(-) create mode 100644 src/buffered-io.cc create mode 100644 src/buffered-io.hh create mode 100644 src/worker.cc create mode 100644 src/worker.hh diff --git a/src/buffered-io.cc b/src/buffered-io.cc new file mode 100644 index 000000000..6d3ab6db9 --- /dev/null +++ b/src/buffered-io.cc @@ -0,0 +1,52 @@ +#include "buffered-io.hh" +#include +#include +#include + +[[nodiscard]] int tryWriteLine(int fd, std::string s) { + s += "\n"; + std::string_view sv{s}; + while (!sv.empty()) { + nix::checkInterrupt(); + ssize_t res = write(fd, sv.data(), sv.size()); + if (res == -1 && errno != EINTR) { + return -errno; + } + if (res > 0) { + sv.remove_prefix(res); + } + } + return 0; +} + +LineReader::LineReader(int fd) { + stream = fdopen(fd, "r"); + if (!stream) { + throw nix::Error("fdopen failed: %s", strerror(errno)); + } +} + +LineReader::~LineReader() { + fclose(stream); + free(buffer); +} + +LineReader::LineReader(LineReader &&other) { + stream = other.stream; + other.stream = nullptr; + buffer = other.buffer; + other.buffer = nullptr; + len = other.len; + other.len = 0; +} + +[[nodiscard]] std::string_view LineReader::readLine() { + ssize_t read = getline(&buffer, &len, stream); + + if (read == -1) { + return {}; // Return an empty string_view in case of error + } + + // Remove trailing newline + return std::string_view(buffer, read - 1); +} diff --git a/src/buffered-io.hh b/src/buffered-io.hh new file mode 100644 index 000000000..e1f068500 --- /dev/null +++ b/src/buffered-io.hh @@ -0,0 +1,20 @@ +#pragma once +#include +#include +#include + +[[nodiscard]] int tryWriteLine(int fd, std::string s); + +class LineReader { + public: + LineReader(int fd); + ~LineReader(); + + LineReader(LineReader &&other); + [[nodiscard]] std::string_view readLine(); + + private: + FILE *stream = nullptr; + char *buffer = nullptr; + size_t len = 0; +}; diff --git a/src/drv.cc b/src/drv.cc index dc91ddfce..43578f8ba 100644 --- a/src/drv.cc +++ b/src/drv.cc @@ -1,4 +1,4 @@ -#include "drvs.hh" +#include "drv.hh" #include #include #include diff --git a/src/meson.build b/src/meson.build index 6b8d6dc77..cf470971c 100644 --- a/src/meson.build +++ b/src/meson.build @@ -1,7 +1,9 @@ src = [ 'nix-eval-jobs.cc', 'eval-args.cc', - 'drv.cc' + 'drv.cc', + 'buffered-io.cc', + 'worker.cc' ] executable('nix-eval-jobs', src, diff --git a/src/nix-eval-jobs.cc b/src/nix-eval-jobs.cc index 670bd4816..5a25f3605 100644 --- a/src/nix-eval-jobs.cc +++ b/src/nix-eval-jobs.cc @@ -9,19 +9,18 @@ #include #include -#include #include -#include -#include #include -#include #include +#include #include +#include #include -#include #include "eval-args.hh" #include "drv.hh" +#include "buffered-io.hh" +#include "worker.hh" #include @@ -36,224 +35,8 @@ using namespace nlohmann; #endif static MyArgs myArgs; -static Value *releaseExprTopLevelValue(EvalState &state, Bindings &autoArgs) { - Value vTop; - - if (myArgs.fromArgs) { - Expr *e = state.parseExprFromString( - myArgs.releaseExpr, state.rootPath(CanonPath::fromCwd())); - state.eval(e, vTop); - } else { - state.evalFile(lookupFileArg(state, myArgs.releaseExpr), vTop); - } - - auto vRoot = state.allocValue(); - - state.autoCallFunction(autoArgs, vTop, *vRoot); - - return vRoot; -} - - -std::string attrPathJoin(json input) { - return std::accumulate(input.begin(), input.end(), std::string(), - [](std::string ss, std::string s) { - // Escape token if containing dots - if (s.find(".") != std::string::npos) { - s = "\"" + s + "\""; - } - return ss.empty() ? s : ss + "." + s; - }); -} - -[[nodiscard]] static int tryWriteLine(int fd, std::string s) { - s += "\n"; - std::string_view sv{s}; - while (!sv.empty()) { - checkInterrupt(); - ssize_t res = write(fd, sv.data(), sv.size()); - if (res == -1 && errno != EINTR) { - return -errno; - } - if (res > 0) { - sv.remove_prefix(res); - } - } - return 0; -} - -class LineReader { - public: - LineReader(int fd) { - stream = fdopen(fd, "r"); - if (!stream) { - throw Error("fdopen failed: %s", strerror(errno)); - } - } - - ~LineReader() { - fclose(stream); - free(buffer); - } - - LineReader(LineReader &&other) { - stream = other.stream; - other.stream = nullptr; - buffer = other.buffer; - other.buffer = nullptr; - len = other.len; - other.len = 0; - } - - [[nodiscard]] std::string_view readLine() { - ssize_t read = getline(&buffer, &len, stream); - - if (read == -1) { - return {}; // Return an empty string_view in case of error - } - - // Remove trailing newline - return std::string_view(buffer, read - 1); - } - - private: - FILE *stream = nullptr; - char *buffer = nullptr; - size_t len = 0; -}; - -static void worker(ref state, Bindings &autoArgs, AutoCloseFD &to, - AutoCloseFD &from) { - - nix::Value *vRoot = [&]() { - if (myArgs.flake) { - auto [flakeRef, fragment, outputSpec] = - parseFlakeRefWithFragmentAndExtendedOutputsSpec( - myArgs.releaseExpr, absPath(".")); - InstallableFlake flake{ - {}, state, std::move(flakeRef), fragment, outputSpec, - {}, {}, myArgs.lockFlags}; - - return flake.toValue(*state).first; - } else { - return releaseExprTopLevelValue(*state, autoArgs); - } - }(); - - LineReader fromReader(from.release()); - - while (true) { - /* Wait for the collector to send us a job name. */ - if (tryWriteLine(to.get(), "next") < 0) { - return; // main process died - } - - auto s = fromReader.readLine(); - if (s == "exit") { - break; - } - if (!hasPrefix(s, "do ")) { - fprintf(stderr, "worker error: received invalid command '%s'\n", - s.data()); - abort(); - } - auto path = json::parse(s.substr(3)); - auto attrPathS = attrPathJoin(path); - - debug("worker process %d at '%s'", getpid(), path); - - /* Evaluate it and send info back to the collector. */ - json reply = json{{"attr", attrPathS}, {"attrPath", path}}; - try { - auto vTmp = - findAlongAttrPath(*state, attrPathS, autoArgs, *vRoot).first; - - auto v = state->allocValue(); - state->autoCallFunction(autoArgs, *vTmp, *v); - - if (v->type() == nAttrs) { - if (auto drvInfo = getDerivation(*state, *v, false)) { - auto drv = Drv(attrPathS, *state, *drvInfo, myArgs); - reply.update(drv); - - /* Register the derivation as a GC root. !!! This - registers roots for jobs that we may have already - done. */ - if (myArgs.gcRootsDir != "") { - Path root = myArgs.gcRootsDir + "/" + - std::string(baseNameOf(drv.drvPath)); - if (!pathExists(root)) { - auto localStore = - state->store - .dynamic_pointer_cast(); - auto storePath = - localStore->parseStorePath(drv.drvPath); - localStore->addPermRoot(storePath, root); - } - } - } else { - auto attrs = nlohmann::json::array(); - bool recurse = - myArgs.forceRecurse || - path.size() == 0; // Dont require `recurseForDerivations - // = true;` for top-level attrset - - for (auto &i : - v->attrs->lexicographicOrder(state->symbols)) { - const std::string &name = state->symbols[i->name]; - attrs.push_back(name); - - if (name == "recurseForDerivations" && - !myArgs.forceRecurse) { - auto attrv = - v->attrs->get(state->sRecurseForDerivations); - recurse = state->forceBool( - *attrv->value, attrv->pos, - "while evaluating recurseForDerivations"); - } - } - if (recurse) - reply["attrs"] = std::move(attrs); - else - reply["attrs"] = nlohmann::json::array(); - } - } else { - // We ignore everything that cannot be build - reply["attrs"] = nlohmann::json::array(); - } - } catch (EvalError &e) { - auto err = e.info(); - std::ostringstream oss; - showErrorInfo(oss, err, loggerSettings.showTrace.get()); - auto msg = oss.str(); - - // Transmits the error we got from the previous evaluation - // in the JSON output. - reply["error"] = filterANSIEscapes(msg, true); - // Don't forget to print it into the STDERR log, this is - // what's shown in the Hydra UI. - printError(e.msg()); - } - - if (tryWriteLine(to.get(), reply.dump()) < 0) { - return; // main process died - } - - /* If our RSS exceeds the maximum, exit. The collector will - start a new process. */ - struct rusage r; - getrusage(RUSAGE_SELF, &r); - if ((size_t)r.ru_maxrss > myArgs.maxMemorySize * 1024) - break; - } - - if (tryWriteLine(to.get(), "restart") < 0) { - return; // main process died - }; -} - typedef std::function state, Bindings &autoArgs, - AutoCloseFD &to, AutoCloseFD &from)> + AutoCloseFD &to, AutoCloseFD &from, MyArgs &args)> Processor; /* Auto-cleanup of fork's process and fds. */ @@ -275,11 +58,11 @@ struct Proc { auto state = std::make_shared( myArgs.searchPath, openStore(*myArgs.evalStoreUrl)); Bindings &autoArgs = *myArgs.getAutoArgs(*state); - proc(ref(state), autoArgs, *to, *from); + proc(ref(state), autoArgs, *to, *from, myArgs); } catch (Error &e) { nlohmann::json err; auto msg = e.msg(); - err["error"] = filterANSIEscapes(msg, true); + err["error"] = nix::filterANSIEscapes(msg, true); printError(msg); if (tryWriteLine(to->get(), err.dump()) < 0) { return; // main process died diff --git a/src/worker.cc b/src/worker.cc new file mode 100644 index 000000000..a1aa293cd --- /dev/null +++ b/src/worker.cc @@ -0,0 +1,173 @@ +#include "worker.hh" +#include "drv.hh" +#include "buffered-io.hh" + +#include +#include +#include +#include + +#include +#include + +static nix::Value *releaseExprTopLevelValue(nix::EvalState &state, + nix::Bindings &autoArgs, + MyArgs &args) { + nix::Value vTop; + + if (args.fromArgs) { + nix::Expr *e = state.parseExprFromString( + args.releaseExpr, state.rootPath(nix::CanonPath::fromCwd())); + state.eval(e, vTop); + } else { + state.evalFile(lookupFileArg(state, args.releaseExpr), vTop); + } + + auto vRoot = state.allocValue(); + + state.autoCallFunction(autoArgs, vTop, *vRoot); + + return vRoot; +} + +static std::string attrPathJoin(nlohmann::json input) { + return std::accumulate(input.begin(), input.end(), std::string(), + [](std::string ss, std::string s) { + // Escape token if containing dots + if (s.find(".") != std::string::npos) { + s = "\"" + s + "\""; + } + return ss.empty() ? s : ss + "." + s; + }); +} + +void worker(nix::ref state, nix::Bindings &autoArgs, + nix::AutoCloseFD &to, nix::AutoCloseFD &from, MyArgs &args) { + + nix::Value *vRoot = [&]() { + if (args.flake) { + auto [flakeRef, fragment, outputSpec] = + nix::parseFlakeRefWithFragmentAndExtendedOutputsSpec( + args.releaseExpr, nix::absPath(".")); + nix::InstallableFlake flake{ + {}, state, std::move(flakeRef), fragment, outputSpec, + {}, {}, args.lockFlags}; + + return flake.toValue(*state).first; + } else { + return releaseExprTopLevelValue(*state, autoArgs, args); + } + }(); + + LineReader fromReader(from.release()); + + while (true) { + /* Wait for the collector to send us a job name. */ + if (tryWriteLine(to.get(), "next") < 0) { + return; // main process died + } + + auto s = fromReader.readLine(); + if (s == "exit") { + break; + } + if (!nix::hasPrefix(s, "do ")) { + fprintf(stderr, "worker error: received invalid command '%s'\n", + s.data()); + abort(); + } + auto path = nlohmann::json::parse(s.substr(3)); + auto attrPathS = attrPathJoin(path); + + /* Evaluate it and send info back to the collector. */ + nlohmann::json reply = + nlohmann::json{{"attr", attrPathS}, {"attrPath", path}}; + try { + auto vTmp = + nix::findAlongAttrPath(*state, attrPathS, autoArgs, *vRoot) + .first; + + auto v = state->allocValue(); + state->autoCallFunction(autoArgs, *vTmp, *v); + + if (v->type() == nix::nAttrs) { + if (auto drvInfo = nix::getDerivation(*state, *v, false)) { + auto drv = Drv(attrPathS, *state, *drvInfo, args); + reply.update(drv); + + /* Register the derivation as a GC root. !!! This + registers roots for jobs that we may have already + done. */ + if (args.gcRootsDir != "") { + nix::Path root = + args.gcRootsDir + "/" + + std::string(nix::baseNameOf(drv.drvPath)); + if (!nix::pathExists(root)) { + auto localStore = + state->store + .dynamic_pointer_cast(); + auto storePath = + localStore->parseStorePath(drv.drvPath); + localStore->addPermRoot(storePath, root); + } + } + } else { + auto attrs = nlohmann::json::array(); + bool recurse = + args.forceRecurse || + path.size() == 0; // Dont require `recurseForDerivations + // = true;` for top-level attrset + + for (auto &i : + v->attrs->lexicographicOrder(state->symbols)) { + const std::string &name = state->symbols[i->name]; + attrs.push_back(name); + + if (name == "recurseForDerivations" && + !args.forceRecurse) { + auto attrv = + v->attrs->get(state->sRecurseForDerivations); + recurse = state->forceBool( + *attrv->value, attrv->pos, + "while evaluating recurseForDerivations"); + } + } + if (recurse) + reply["attrs"] = std::move(attrs); + else + reply["attrs"] = nlohmann::json::array(); + } + } else { + // We ignore everything that cannot be build + reply["attrs"] = nlohmann::json::array(); + } + } catch (nix::EvalError &e) { + auto err = e.info(); + std::ostringstream oss; + nix::showErrorInfo(oss, err, nix::loggerSettings.showTrace.get()); + auto msg = oss.str(); + + // Transmits the error we got from the previous evaluation + // in the JSON output. + reply["error"] = nix::filterANSIEscapes(msg, true); + // Don't forget to print it into the STDERR log, this is + // what's shown in the Hydra UI. + fprintf(stderr, "%s\n", msg.c_str()); + } + + if (tryWriteLine(to.get(), reply.dump()) < 0) { + return; // main process died + } + + /* If our RSS exceeds the maximum, exit. The collector will + start a new process. */ + struct rusage r; + getrusage(RUSAGE_SELF, &r); + if ((size_t)r.ru_maxrss > args.maxMemorySize * 1024) + break; + } + + if (tryWriteLine(to.get(), "restart") < 0) { + return; // main process died + }; +} diff --git a/src/worker.hh b/src/worker.hh new file mode 100644 index 000000000..0f7d73599 --- /dev/null +++ b/src/worker.hh @@ -0,0 +1,9 @@ +#pragma once +#include +#include +#include + +#include "eval-args.hh" + +void worker(nix::ref state, nix::Bindings &autoArgs, + nix::AutoCloseFD &to, nix::AutoCloseFD &from, MyArgs &args); From 3be8c48f54c74dfb991e8909e1e2201b9b34399e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sun, 10 Dec 2023 15:44:15 +0100 Subject: [PATCH 361/419] fix format --- src/drv.cc | 11 ++++++----- src/eval-args.cc | 2 +- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/drv.cc b/src/drv.cc index 43578f8ba..2eca9b09f 100644 --- a/src/drv.cc +++ b/src/drv.cc @@ -6,9 +6,8 @@ #include #include - static bool queryIsCached(nix::Store &store, - std::map &outputs) { + std::map &outputs) { uint64_t downloadSize, narSize; nix::StorePathSet willBuild, willSubstitute, unknown; @@ -23,7 +22,8 @@ static bool queryIsCached(nix::Store &store, } /* The fields of a derivation that are printed in json form */ -Drv::Drv(std::string &attrPath, nix::EvalState &state, nix::DrvInfo &drvInfo, MyArgs &args) { +Drv::Drv(std::string &attrPath, nix::EvalState &state, nix::DrvInfo &drvInfo, + MyArgs &args) { auto localStore = state.store.dynamic_pointer_cast(); @@ -58,8 +58,9 @@ Drv::Drv(std::string &attrPath, nix::EvalState &state, nix::DrvInfo &drvInfo, My meta = meta_; } if (args.checkCacheStatus) { - cacheStatus = queryIsCached(*localStore, outputs) ? Drv::CacheStatus::Cached - : Drv::CacheStatus::Uncached; + cacheStatus = queryIsCached(*localStore, outputs) + ? Drv::CacheStatus::Cached + : Drv::CacheStatus::Uncached; } else { cacheStatus = Drv::CacheStatus::Unknown; } diff --git a/src/eval-args.cc b/src/eval-args.cc index c0a8723ef..b25ecc7b0 100644 --- a/src/eval-args.cc +++ b/src/eval-args.cc @@ -88,6 +88,6 @@ MyArgs::MyArgs() : MixCommonArgs("nix-eval-jobs") { expectArg("expr", &releaseExpr); } -void MyArgs::parseArgs(char** argv, int argc) { +void MyArgs::parseArgs(char **argv, int argc) { parseCmdline(nix::argvToStrings(argc, argv), 0); } From 36a5368dfc198a4dc9aad791632e6c14d8c206e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sun, 10 Dec 2023 15:47:56 +0100 Subject: [PATCH 362/419] include missing filesystem header --- src/nix-eval-jobs.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/nix-eval-jobs.cc b/src/nix-eval-jobs.cc index 5a25f3605..95cf67317 100644 --- a/src/nix-eval-jobs.cc +++ b/src/nix-eval-jobs.cc @@ -1,12 +1,12 @@ - #include #include #include +#include + #include #include #include #include - #include #include #include From e7c30b306dacb18588f5cef36cf1bdadd6e6be69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sun, 10 Dec 2023 16:50:13 +0100 Subject: [PATCH 363/419] don't fail nix-eval-jobs on eval errors This error handling was copied by accident in https://github.com/nix-community/nix-eval-jobs/pull/277/commits/5c764d4a67feceae1848414fd9434e79120884e6#diff-a79ded172fd76747492a417a39848b6c25c14238e65971e6a05fe81706d5048fR622 --- src/nix-eval-jobs.cc | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/nix-eval-jobs.cc b/src/nix-eval-jobs.cc index 95cf67317..87eb456b4 100644 --- a/src/nix-eval-jobs.cc +++ b/src/nix-eval-jobs.cc @@ -190,10 +190,6 @@ std::function collector(Sync &state_, json response; try { response = json::parse(respString); - if (response.find("error") != response.end()) { - throw Error("worker error: %s", - (std::string)response["error"]); - } } catch (const json::exception &e) { throw Error("Received invalid JSON from worker: %s '%s'", e.what(), respString); From f970ec352483ad411b1ea090dc46a92ca0faebab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sun, 10 Dec 2023 16:58:22 +0100 Subject: [PATCH 364/419] add test for evaluation errors --- tests/assets/flake.nix | 2 ++ tests/test_eval.py | 13 +++++++++++++ 2 files changed, 15 insertions(+) diff --git a/tests/assets/flake.nix b/tests/assets/flake.nix index 90cb732bf..fd0a07f19 100644 --- a/tests/assets/flake.nix +++ b/tests/assets/flake.nix @@ -7,5 +7,7 @@ in { hydraJobs = import ./ci.nix { inherit pkgs; }; + + legacyPackages.x86_64-linux.brokenPackage = throw "this is an evaluation error"; }; } diff --git a/tests/test_eval.py b/tests/test_eval.py index c64c8b1af..eb7e78684 100644 --- a/tests/test_eval.py +++ b/tests/test_eval.py @@ -68,3 +68,16 @@ def test_expression() -> None: with open(TEST_ROOT.joinpath("assets/ci.nix"), "r") as ci_nix: common_test(["-E", ci_nix.read()]) + +def test_eval_error() -> None: + with TemporaryDirectory() as tempdir: + cmd = [str(BIN), "--gc-roots-dir", tempdir, "--meta", "--flake", ".#legacyPackages.x86_64-linux"] + res = subprocess.run( + cmd, + cwd=TEST_ROOT.joinpath("assets"), + text=True, + stdout=subprocess.PIPE, + ) + attrs = json.loads(res.stdout) + assert attrs["attr"] == "brokenPackage" + assert "this is an evaluation error" in attrs["error"] From b267eb917d752fe615b7fb318ffaabe9dec967d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sun, 10 Dec 2023 17:00:31 +0100 Subject: [PATCH 365/419] apply treefmt --- tests/test_eval.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/tests/test_eval.py b/tests/test_eval.py index eb7e78684..88e15de40 100644 --- a/tests/test_eval.py +++ b/tests/test_eval.py @@ -69,9 +69,17 @@ def test_expression() -> None: with open(TEST_ROOT.joinpath("assets/ci.nix"), "r") as ci_nix: common_test(["-E", ci_nix.read()]) + def test_eval_error() -> None: with TemporaryDirectory() as tempdir: - cmd = [str(BIN), "--gc-roots-dir", tempdir, "--meta", "--flake", ".#legacyPackages.x86_64-linux"] + cmd = [ + str(BIN), + "--gc-roots-dir", + tempdir, + "--meta", + "--flake", + ".#legacyPackages.x86_64-linux", + ] res = subprocess.run( cmd, cwd=TEST_ROOT.joinpath("assets"), @@ -80,4 +88,4 @@ def test_eval_error() -> None: ) attrs = json.loads(res.stdout) assert attrs["attr"] == "brokenPackage" - assert "this is an evaluation error" in attrs["error"] + assert "this is an evaluation error" in attrs["error"] From b24c03e2dec7420576b1f27d1d843a46ec75136d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sun, 10 Dec 2023 17:19:47 +0100 Subject: [PATCH 366/419] simplify collector function looks like the lambda doesn't buy us anything here. --- src/nix-eval-jobs.cc | 201 +++++++++++++++++++++---------------------- 1 file changed, 99 insertions(+), 102 deletions(-) diff --git a/src/nix-eval-jobs.cc b/src/nix-eval-jobs.cc index 87eb456b4..042952974 100644 --- a/src/nix-eval-jobs.cc +++ b/src/nix-eval-jobs.cc @@ -119,117 +119,113 @@ void handleBrokenWorkerPipe(Proc &proc) { } } -std::function collector(Sync &state_, - std::condition_variable &wakeup) { - return [&]() { - try { - std::optional> proc_; - std::optional> fromReader_; +void collector(Sync &state_, std::condition_variable &wakeup) { + try { + std::optional> proc_; + std::optional> fromReader_; - while (true) { - if (!proc_.has_value()) { - proc_ = std::make_unique(worker); - fromReader_ = std::make_unique( - proc_.value()->from.release()); - } - auto proc = std::move(proc_.value()); - auto fromReader = std::move(fromReader_.value()); + while (true) { + if (!proc_.has_value()) { + proc_ = std::make_unique(worker); + fromReader_ = + std::make_unique(proc_.value()->from.release()); + } + auto proc = std::move(proc_.value()); + auto fromReader = std::move(fromReader_.value()); - /* Check whether the existing worker process is still there. */ - auto s = fromReader->readLine(); - if (s == "") { - handleBrokenWorkerPipe(*proc.get()); - } else if (s == "restart") { - proc_ = std::nullopt; - fromReader_ = std::nullopt; - continue; - } else if (s != "next") { - try { - auto json = json::parse(s); - throw Error("worker error: %s", - (std::string)json["error"]); - } catch (const json::exception &e) { - throw Error( - "Received invalid JSON from worker: %s '%s'", - e.what(), s); - } - } - - /* Wait for a job name to become available. */ - json attrPath; - - while (true) { - checkInterrupt(); - auto state(state_.lock()); - if ((state->todo.empty() && state->active.empty()) || - state->exc) { - if (tryWriteLine(proc->to.get(), "exit") < 0) { - handleBrokenWorkerPipe(*proc.get()); - } - return; - } - if (!state->todo.empty()) { - attrPath = *state->todo.begin(); - state->todo.erase(state->todo.begin()); - state->active.insert(attrPath); - break; - } else - state.wait(wakeup); - } - - /* Tell the worker to evaluate it. */ - if (tryWriteLine(proc->to.get(), "do " + attrPath.dump()) < 0) { - handleBrokenWorkerPipe(*proc.get()); - } - - /* Wait for the response. */ - auto respString = fromReader->readLine(); - if (respString == "") { - handleBrokenWorkerPipe(*proc.get()); - } - json response; + /* Check whether the existing worker process is still there. */ + auto s = fromReader->readLine(); + if (s == "") { + handleBrokenWorkerPipe(*proc.get()); + } else if (s == "restart") { + proc_ = std::nullopt; + fromReader_ = std::nullopt; + continue; + } else if (s != "next") { try { - response = json::parse(respString); + auto json = json::parse(s); + throw Error("worker error: %s", (std::string)json["error"]); } catch (const json::exception &e) { throw Error("Received invalid JSON from worker: %s '%s'", - e.what(), respString); - } - - /* Handle the response. */ - std::vector newAttrs; - if (response.find("attrs") != response.end()) { - for (auto &i : response["attrs"]) { - json newAttr = json(response["attrPath"]); - newAttr.emplace_back(i); - newAttrs.push_back(newAttr); - } - } else { - auto state(state_.lock()); - std::cout << respString << "\n" << std::flush; - } - - proc_ = std::move(proc); - fromReader_ = std::move(fromReader); - - /* Add newly discovered job names to the queue. */ - { - auto state(state_.lock()); - state->active.erase(attrPath); - for (auto p : newAttrs) { - state->todo.insert(p); - } - wakeup.notify_all(); + e.what(), s); } } - } catch (...) { - auto state(state_.lock()); - state->exc = std::current_exception(); - wakeup.notify_all(); + + /* Wait for a job name to become available. */ + json attrPath; + + while (true) { + checkInterrupt(); + auto state(state_.lock()); + if ((state->todo.empty() && state->active.empty()) || + state->exc) { + if (tryWriteLine(proc->to.get(), "exit") < 0) { + handleBrokenWorkerPipe(*proc.get()); + } + return; + } + if (!state->todo.empty()) { + attrPath = *state->todo.begin(); + state->todo.erase(state->todo.begin()); + state->active.insert(attrPath); + break; + } else + state.wait(wakeup); + } + + /* Tell the worker to evaluate it. */ + if (tryWriteLine(proc->to.get(), "do " + attrPath.dump()) < 0) { + handleBrokenWorkerPipe(*proc.get()); + } + + /* Wait for the response. */ + auto respString = fromReader->readLine(); + if (respString == "") { + handleBrokenWorkerPipe(*proc.get()); + } + json response; + try { + response = json::parse(respString); + } catch (const json::exception &e) { + throw Error("Received invalid JSON from worker: %s '%s'", + e.what(), respString); + } + + /* Handle the response. */ + std::vector newAttrs; + if (response.find("attrs") != response.end()) { + for (auto &i : response["attrs"]) { + json newAttr = json(response["attrPath"]); + newAttr.emplace_back(i); + newAttrs.push_back(newAttr); + } + } else { + auto state(state_.lock()); + std::cout << respString << "\n" << std::flush; + } + + proc_ = std::move(proc); + fromReader_ = std::move(fromReader); + + /* Add newly discovered job names to the queue. */ + { + auto state(state_.lock()); + state->active.erase(attrPath); + for (auto p : newAttrs) { + state->todo.insert(p); + } + wakeup.notify_all(); + } } - }; + } catch (...) { + auto state(state_.lock()); + state->exc = std::current_exception(); + wakeup.notify_all(); + } } int main(int argc, char **argv) { + /* Prevent undeclared dependencies in the evaluation via $NIX_PATH. */ unsetenv("NIX_PATH"); @@ -277,8 +273,9 @@ int main(int argc, char **argv) { /* Start a collector thread per worker process. */ std::vector threads; std::condition_variable wakeup; - for (size_t i = 0; i < myArgs.nrWorkers; i++) - threads.emplace_back(std::thread(collector(state_, wakeup))); + for (size_t i = 0; i < myArgs.nrWorkers; i++) { + threads.emplace_back(collector, std::ref(state_), std::ref(wakeup)); + } for (auto &thread : threads) thread.join(); From b6ec7d2ecfce49d265487f196489741936880c17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sun, 10 Dec 2023 20:20:17 +0100 Subject: [PATCH 367/419] make MyArgs non-copyable this seems to have weird effects --- src/eval-args.hh | 1 + 1 file changed, 1 insertion(+) diff --git a/src/eval-args.hh b/src/eval-args.hh index cd5a9aacc..e0ce2d22e 100644 --- a/src/eval-args.hh +++ b/src/eval-args.hh @@ -27,6 +27,7 @@ class MyArgs : virtual public nix::MixEvalArgs, .useRegistries = false, .allowUnlocked = false}; MyArgs(); + MyArgs(const MyArgs&) = delete; void parseArgs(char** argv, int argc); }; From c00fcbba8de26558a60a7408bf9e61022c635805 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sun, 10 Dec 2023 20:19:44 +0100 Subject: [PATCH 368/419] drop obsolete compiler warning suppression --- src/nix-eval-jobs.cc | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/nix-eval-jobs.cc b/src/nix-eval-jobs.cc index 042952974..d1dd7a6cf 100644 --- a/src/nix-eval-jobs.cc +++ b/src/nix-eval-jobs.cc @@ -27,12 +27,6 @@ using namespace nix; using namespace nlohmann; -// Safe to ignore - the args will be static. -#ifdef __GNUC__ -#pragma GCC diagnostic ignored "-Wnon-virtual-dtor" -#elif __clang__ -#pragma clang diagnostic ignored "-Wnon-virtual-dtor" -#endif static MyArgs myArgs; typedef std::function state, Bindings &autoArgs, From d48cfadb3d8f4ce667e03c7df53866a7f5f43a65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sun, 10 Dec 2023 21:17:27 +0100 Subject: [PATCH 369/419] fix catching eval errors on macOS --- src/worker.cc | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/worker.cc b/src/worker.cc index a1aa293cd..42d4d73f2 100644 --- a/src/worker.cc +++ b/src/worker.cc @@ -153,6 +153,12 @@ void worker(nix::ref state, nix::Bindings &autoArgs, // Don't forget to print it into the STDERR log, this is // what's shown in the Hydra UI. fprintf(stderr, "%s\n", msg.c_str()); + } catch ( + const std::exception &e) { // FIXME: for some reason the catch block + // above, doesn't trigger on macOS (?) + auto msg = e.what(); + reply["error"] = nix::filterANSIEscapes(msg, true); + fprintf(stderr, "%s\n", msg); } if (tryWriteLine(to.get(), reply.dump()) < 0) { From e1ad62cef18a9f7d977ac93d13136f002f0ba185 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sun, 10 Dec 2023 21:18:04 +0100 Subject: [PATCH 370/419] include orignal json in worker error message --- src/nix-eval-jobs.cc | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/nix-eval-jobs.cc b/src/nix-eval-jobs.cc index d1dd7a6cf..fce0aa679 100644 --- a/src/nix-eval-jobs.cc +++ b/src/nix-eval-jobs.cc @@ -129,7 +129,7 @@ void collector(Sync &state_, std::condition_variable &wakeup) { /* Check whether the existing worker process is still there. */ auto s = fromReader->readLine(); - if (s == "") { + if (s.empty()) { handleBrokenWorkerPipe(*proc.get()); } else if (s == "restart") { proc_ = std::nullopt; @@ -140,8 +140,9 @@ void collector(Sync &state_, std::condition_variable &wakeup) { auto json = json::parse(s); throw Error("worker error: %s", (std::string)json["error"]); } catch (const json::exception &e) { - throw Error("Received invalid JSON from worker: %s '%s'", - e.what(), s); + throw Error( + "Received invalid JSON from worker: %s\n json: '%s'", + e.what(), s); } } @@ -174,15 +175,16 @@ void collector(Sync &state_, std::condition_variable &wakeup) { /* Wait for the response. */ auto respString = fromReader->readLine(); - if (respString == "") { + if (respString.empty()) { handleBrokenWorkerPipe(*proc.get()); } json response; try { response = json::parse(respString); } catch (const json::exception &e) { - throw Error("Received invalid JSON from worker: %s '%s'", - e.what(), respString); + throw Error( + "Received invalid JSON from worker: %s\n json: '%s'", + e.what(), respString); } /* Handle the response. */ From 521380076d5f10dc7dfbdf89690e46ac0da0acf0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sun, 10 Dec 2023 21:20:45 +0100 Subject: [PATCH 371/419] release proc.pid properly before calling waitpid() --- src/nix-eval-jobs.cc | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/nix-eval-jobs.cc b/src/nix-eval-jobs.cc index fce0aa679..0fe5a399a 100644 --- a/src/nix-eval-jobs.cc +++ b/src/nix-eval-jobs.cc @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -85,23 +86,23 @@ struct State { }; void handleBrokenWorkerPipe(Proc &proc) { + // we already took the process status from Proc, no + // need to wait for it again to avoid error messages + pid_t pid = proc.pid.release(); while (1) { - int rc = waitpid(proc.pid, nullptr, WNOHANG); + int rc = waitpid(pid, nullptr, WNOHANG); if (rc == 0) { - proc.pid = -1; // we already took the process status from Proc, no - // need to wait for it again to avoid error messages + kill(pid, SIGKILL); throw Error("BUG: worker pipe closed but worker still running?"); } else if (rc == -1) { - proc.pid = -1; + kill(pid, SIGKILL); throw Error("BUG: waitpid waiting for worker failed: %s", strerror(errno)); } else { if (WIFEXITED(rc)) { - proc.pid = -1; throw Error("evaluation worker exited with %d", WEXITSTATUS(rc)); } else if (WIFSIGNALED(rc)) { - proc.pid = -1; if (WTERMSIG(rc) == SIGKILL) { throw Error("evaluation worker killed by SIGKILL, maybe " "memory limit reached?"); From 843dc25cfe93eb620f8dc8825b1de001e5b33e9f Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 11 Dec 2023 00:16:02 +0000 Subject: [PATCH 372/419] chore(deps): lock file maintenance --- flake.lock | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/flake.lock b/flake.lock index f885e6e8f..2d9ab658e 100644 --- a/flake.lock +++ b/flake.lock @@ -42,11 +42,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1702192996, - "narHash": "sha256-taRtgPtpYl7KofdDC9sDHe1urV3+pP2JFwuAyVlccYI=", + "lastModified": 1702249933, + "narHash": "sha256-OlqCC8A6OC5n0n4meCklk73V64JcyoJQ+EGa5arhvsI=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "db3bd72ed27a5b2b96c3e6cca463117d27bb052b", + "rev": "0838486e9183ce39398f01b221766e68b50f405c", "type": "github" }, "original": { @@ -71,11 +71,11 @@ ] }, "locked": { - "lastModified": 1701958734, - "narHash": "sha256-3h3EH1FXQkIeAuzaWB+nK0XK54uSD46pp+dMD3gAcB4=", + "lastModified": 1702212301, + "narHash": "sha256-Rvl8BD7mnHZC1Qz6TxT120UyjsGUEdcsthHbEY+1vnU=", "owner": "numtide", "repo": "treefmt-nix", - "rev": "e8cea581dd2b7c9998c1e6662db2c1dc30e7fdb0", + "rev": "afdd5e48a0869b389027307652a658051c0d2f96", "type": "github" }, "original": { From 1f4bbded618d405199b3e758bf57d567b7446056 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Mon, 11 Dec 2023 22:05:02 +0100 Subject: [PATCH 373/419] optimize imports with "include-what-you-use" --- src/buffered-io.cc | 6 +++++- src/drv.cc | 21 ++++++++++++++++++-- src/drv.hh | 16 ++++++++++----- src/eval-args.cc | 12 +++++++++++ src/eval-args.hh | 7 ++++++- src/nix-eval-jobs.cc | 47 +++++++++++++++++++++++++++++++------------- src/worker.cc | 46 +++++++++++++++++++++++++++++++++++++++---- src/worker.hh | 10 +++++++++- 8 files changed, 137 insertions(+), 28 deletions(-) diff --git a/src/buffered-io.cc b/src/buffered-io.cc index 6d3ab6db9..4354740bd 100644 --- a/src/buffered-io.cc +++ b/src/buffered-io.cc @@ -1,7 +1,11 @@ -#include "buffered-io.hh" #include #include #include +#include +#include +#include + +#include "buffered-io.hh" [[nodiscard]] int tryWriteLine(int fd, std::string s) { s += "\n"; diff --git a/src/drv.cc b/src/drv.cc index 2eca9b09f..14fd35276 100644 --- a/src/drv.cc +++ b/src/drv.cc @@ -1,10 +1,27 @@ -#include "drv.hh" -#include +#include // IWYU pragma: keep + #include #include #include #include #include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "drv.hh" +#include "eval-args.hh" static bool queryIsCached(nix::Store &store, std::map &outputs) { diff --git a/src/drv.hh b/src/drv.hh index 4f1fbd043..a5de64a9b 100644 --- a/src/drv.hh +++ b/src/drv.hh @@ -1,16 +1,22 @@ +#include +#include +#include +#include #include #include #include #include #include -#include -#include - -#include - #include "eval-args.hh" +class MyArgs; + +namespace nix { +class EvalState; +struct DrvInfo; +} // namespace nix + /* The fields of a derivation that are printed in json form */ struct Drv { std::string name; diff --git a/src/eval-args.cc b/src/eval-args.cc index b25ecc7b0..5ccf732b2 100644 --- a/src/eval-args.cc +++ b/src/eval-args.cc @@ -1,3 +1,15 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + #include "eval-args.hh" MyArgs::MyArgs() : MixCommonArgs("nix-eval-jobs") { diff --git a/src/eval-args.hh b/src/eval-args.hh index e0ce2d22e..fe505834f 100644 --- a/src/eval-args.hh +++ b/src/eval-args.hh @@ -1,9 +1,14 @@ #pragma once -#include #include #include #include +#include +#include +#include +#include +#include +#include class MyArgs : virtual public nix::MixEvalArgs, virtual public nix::MixCommonArgs, diff --git a/src/nix-eval-jobs.cc b/src/nix-eval-jobs.cc index 0fe5a399a..704afe14b 100644 --- a/src/nix-eval-jobs.cc +++ b/src/nix-eval-jobs.cc @@ -1,30 +1,49 @@ -#include -#include -#include -#include -#include +#include // IWYU pragma: keep -#include #include -#include -#include #include #include #include -#include -#include -#include #include #include #include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include "eval-args.hh" -#include "drv.hh" #include "buffered-io.hh" #include "worker.hh" -#include - using namespace nix; using namespace nlohmann; diff --git a/src/worker.cc b/src/worker.cc index 42d4d73f2..cb0a7d3a8 100644 --- a/src/worker.cc +++ b/src/worker.cc @@ -1,14 +1,52 @@ -#include "worker.hh" -#include "drv.hh" -#include "buffered-io.hh" +#include // IWYU pragma: keep + +// doesn't exist on macOS +// IWYU pragma: no_include #include #include #include #include - #include #include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "worker.hh" +#include "drv.hh" +#include "buffered-io.hh" +#include "eval-args.hh" static nix::Value *releaseExprTopLevelValue(nix::EvalState &state, nix::Bindings &autoArgs, diff --git a/src/worker.hh b/src/worker.hh index 0f7d73599..45e9032dd 100644 --- a/src/worker.hh +++ b/src/worker.hh @@ -1,9 +1,17 @@ #pragma once -#include #include #include #include "eval-args.hh" +class MyArgs; + +namespace nix { +class AutoCloseFD; +class Bindings; +class EvalState; +template class ref; +} // namespace nix + void worker(nix::ref state, nix::Bindings &autoArgs, nix::AutoCloseFD &to, nix::AutoCloseFD &from, MyArgs &args); From 4b6d34214d02227db7bba31c996f09e90d507e83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Fri, 15 Dec 2023 21:50:33 +0100 Subject: [PATCH 374/419] add include-what-you-use to devshell --- shell.nix | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/shell.nix b/shell.nix index a94d8814f..13a5d3430 100644 --- a/shell.nix +++ b/shell.nix @@ -26,8 +26,14 @@ pkgs.mkShell { (pkgs.python3.withPackages (ps: [ ps.pytest ])) - - ]; + ] ++ lib.optional stdenv.isLinux # broken on darwin + (pkgs.writeShellScriptBin "update-include-what-you-use" '' + #!${pkgs.stdenv.shell} + export PATH=${pkgs.include-what-you-use}/bin:$PATH + find src -type f -name '*.cpp' -o -name '*.hh' -print0 | \ + xargs -n1 --null include-what-you-use -std=c++20 -isystem ${lib.getDev nix}/include/nix 2>&1 | \ + fix_includes.py + ''); shellHook = lib.optionalString stdenv.isLinux '' export NIX_DEBUG_INFO_DIRS="${pkgs.curl.debug}/lib/debug:${nix.debug}/lib/debug''${NIX_DEBUG_INFO_DIRS:+:$NIX_DEBUG_INFO_DIRS}" From 8daea338f3ab90b7862ed123d9d9769bf33903cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Fri, 15 Dec 2023 21:50:53 +0100 Subject: [PATCH 375/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/0838486e9183ce39398f01b221766e68b50f405c' (2023-12-10) → 'github:NixOS/nixpkgs/203ecda835bcf69633df7183459283543dd4a874' (2023-12-15) • Updated input 'treefmt-nix': 'github:numtide/treefmt-nix/afdd5e48a0869b389027307652a658051c0d2f96' (2023-12-10) → 'github:numtide/treefmt-nix/d06b70e5163a903f19009c3f97770014787a080f' (2023-12-13) --- flake.lock | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/flake.lock b/flake.lock index 2d9ab658e..d0d8b0499 100644 --- a/flake.lock +++ b/flake.lock @@ -42,11 +42,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1702249933, - "narHash": "sha256-OlqCC8A6OC5n0n4meCklk73V64JcyoJQ+EGa5arhvsI=", + "lastModified": 1702667777, + "narHash": "sha256-qpgZVpFrOEgW0DimJ24UXeFh63TI9fQFXxc58DPtG8Q=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "0838486e9183ce39398f01b221766e68b50f405c", + "rev": "203ecda835bcf69633df7183459283543dd4a874", "type": "github" }, "original": { @@ -71,11 +71,11 @@ ] }, "locked": { - "lastModified": 1702212301, - "narHash": "sha256-Rvl8BD7mnHZC1Qz6TxT120UyjsGUEdcsthHbEY+1vnU=", + "lastModified": 1702461037, + "narHash": "sha256-ssyGxfGHRuuLHuMex+vV6RMOt7nAo07nwufg9L5GkLg=", "owner": "numtide", "repo": "treefmt-nix", - "rev": "afdd5e48a0869b389027307652a658051c0d2f96", + "rev": "d06b70e5163a903f19009c3f97770014787a080f", "type": "github" }, "original": { From 89927c434c31187603f7f4f328d8a1a3a7dba033 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sat, 16 Dec 2023 09:18:04 +0100 Subject: [PATCH 376/419] make sure we also define HAVE_STRUCT_DIRENT_D_TYPE outside of autotools --- src/autotools-config.h.in | 1 + src/meson.build | 16 +++++++++++++++- 2 files changed, 16 insertions(+), 1 deletion(-) create mode 100644 src/autotools-config.h.in diff --git a/src/autotools-config.h.in b/src/autotools-config.h.in new file mode 100644 index 000000000..4cee198b7 --- /dev/null +++ b/src/autotools-config.h.in @@ -0,0 +1 @@ +#mesondefine HAVE_STRUCT_DIRENT_D_TYPE diff --git a/src/meson.build b/src/meson.build index cf470971c..dfbcd8754 100644 --- a/src/meson.build +++ b/src/meson.build @@ -6,6 +6,20 @@ src = [ 'worker.cc' ] +cc = meson.get_compiler('cpp') + +autotool_config = configuration_data() +# nix defines this with autotools +if cc.has_member('struct dirent', 'd_type', prefix: '#include ') + autotool_config.set('HAVE_STRUCT_DIRENT_D_TYPE', 1) +endif + +configure_file( + input: 'autotools-config.h.in', + output: 'autotools-config.h', + configuration: autotool_config +) + executable('nix-eval-jobs', src, dependencies : [ nix_main_dep, @@ -17,4 +31,4 @@ executable('nix-eval-jobs', src, threads_dep ], install: true, - cpp_args: ['-std=c++2a', '-fvisibility=hidden']) + cpp_args: ['-std=c++2a', '-fvisibility=hidden', '--include', 'autotools-config.h']) From 83df9d4e2402bbe2ea82ee783d1318d66f171c93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sat, 16 Dec 2023 09:48:33 +0100 Subject: [PATCH 377/419] fix exit status reporting when evaluation fails --- src/nix-eval-jobs.cc | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/nix-eval-jobs.cc b/src/nix-eval-jobs.cc index 704afe14b..7e9cf3874 100644 --- a/src/nix-eval-jobs.cc +++ b/src/nix-eval-jobs.cc @@ -109,7 +109,8 @@ void handleBrokenWorkerPipe(Proc &proc) { // need to wait for it again to avoid error messages pid_t pid = proc.pid.release(); while (1) { - int rc = waitpid(pid, nullptr, WNOHANG); + int status; + int rc = waitpid(pid, &status, WNOHANG); if (rc == 0) { kill(pid, SIGKILL); throw Error("BUG: worker pipe closed but worker still running?"); @@ -118,16 +119,16 @@ void handleBrokenWorkerPipe(Proc &proc) { throw Error("BUG: waitpid waiting for worker failed: %s", strerror(errno)); } else { - if (WIFEXITED(rc)) { + if (WIFEXITED(status)) { throw Error("evaluation worker exited with %d", - WEXITSTATUS(rc)); - } else if (WIFSIGNALED(rc)) { - if (WTERMSIG(rc) == SIGKILL) { + WEXITSTATUS(status)); + } else if (WIFSIGNALED(status)) { + if (WTERMSIG(status) == SIGKILL) { throw Error("evaluation worker killed by SIGKILL, maybe " "memory limit reached?"); } - throw Error("evaluation worker killed by signal %d", - WTERMSIG(rc)); + throw Error("evaluation worker killed by signal %d (%s)", + WTERMSIG(status), strsignal(WTERMSIG(status))); } // else ignore WIFSTOPPED and WIFCONTINUED } } From 4d97e5a38688110a8e4b0c559b020ead9f1688a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sat, 16 Dec 2023 10:10:49 +0100 Subject: [PATCH 378/419] improve infinite recursion errors --- src/nix-eval-jobs.cc | 52 ++++++++++++++++++++++++++++++------------ tests/assets/flake.nix | 18 ++++++++++++++- tests/test_eval.py | 35 ++++++++++++++++++++++++---- 3 files changed, 86 insertions(+), 19 deletions(-) diff --git a/src/nix-eval-jobs.cc b/src/nix-eval-jobs.cc index 7e9cf3874..44b88e700 100644 --- a/src/nix-eval-jobs.cc +++ b/src/nix-eval-jobs.cc @@ -104,7 +104,7 @@ struct State { std::exception_ptr exc; }; -void handleBrokenWorkerPipe(Proc &proc) { +void handleBrokenWorkerPipe(Proc &proc, std::string_view msg) { // we already took the process status from Proc, no // need to wait for it again to avoid error messages pid_t pid = proc.pid.release(); @@ -113,27 +113,49 @@ void handleBrokenWorkerPipe(Proc &proc) { int rc = waitpid(pid, &status, WNOHANG); if (rc == 0) { kill(pid, SIGKILL); - throw Error("BUG: worker pipe closed but worker still running?"); + throw Error( + "BUG: while %s, worker pipe got closed but evaluation worker still running?", + msg); } else if (rc == -1) { kill(pid, SIGKILL); - throw Error("BUG: waitpid waiting for worker failed: %s", - strerror(errno)); + throw Error("BUG: while %s, waitpid for evaluation worker failed: %s", + msg, strerror(errno)); } else { if (WIFEXITED(status)) { - throw Error("evaluation worker exited with %d", - WEXITSTATUS(status)); + if (WEXITSTATUS(status) == 1) { + throw Error( + "while %s, evaluation worker exited with exit code 1, " + "(possibly an infinite recursion)", + msg); + } + throw Error("while %s, evaluation worker exited with %d", + msg, WEXITSTATUS(status)); } else if (WIFSIGNALED(status)) { if (WTERMSIG(status) == SIGKILL) { - throw Error("evaluation worker killed by SIGKILL, maybe " - "memory limit reached?"); + throw Error( + "while %s, evaluation worker got killed by SIGKILL, maybe " + "memory limit reached?", + msg); } - throw Error("evaluation worker killed by signal %d (%s)", - WTERMSIG(status), strsignal(WTERMSIG(status))); + throw Error( + "while %s, evaluation worker got killed by signal %d (%s)", + msg, WTERMSIG(status), strsignal(WTERMSIG(status))); } // else ignore WIFSTOPPED and WIFCONTINUED } } } +std::string joinAttrPath(json &attrPath) { + std::string joined; + for (auto& element : attrPath) { + if (!joined.empty()) { + joined += '.'; + } + joined += element.get(); + } + return joined; +} + void collector(Sync &state_, std::condition_variable &wakeup) { try { std::optional> proc_; @@ -151,7 +173,7 @@ void collector(Sync &state_, std::condition_variable &wakeup) { /* Check whether the existing worker process is still there. */ auto s = fromReader->readLine(); if (s.empty()) { - handleBrokenWorkerPipe(*proc.get()); + handleBrokenWorkerPipe(*proc.get(), "checking worker process"); } else if (s == "restart") { proc_ = std::nullopt; fromReader_ = std::nullopt; @@ -176,7 +198,7 @@ void collector(Sync &state_, std::condition_variable &wakeup) { if ((state->todo.empty() && state->active.empty()) || state->exc) { if (tryWriteLine(proc->to.get(), "exit") < 0) { - handleBrokenWorkerPipe(*proc.get()); + handleBrokenWorkerPipe(*proc.get(), "sending exit"); } return; } @@ -191,13 +213,15 @@ void collector(Sync &state_, std::condition_variable &wakeup) { /* Tell the worker to evaluate it. */ if (tryWriteLine(proc->to.get(), "do " + attrPath.dump()) < 0) { - handleBrokenWorkerPipe(*proc.get()); + auto msg = "sending attrPath '" + joinAttrPath(attrPath) + "'"; + handleBrokenWorkerPipe(*proc.get(), msg); } /* Wait for the response. */ auto respString = fromReader->readLine(); if (respString.empty()) { - handleBrokenWorkerPipe(*proc.get()); + auto msg = "reading result for attrPath '" + joinAttrPath(attrPath) + "'"; + handleBrokenWorkerPipe(*proc.get(), msg); } json response; try { diff --git a/tests/assets/flake.nix b/tests/assets/flake.nix index fd0a07f19..5a322776b 100644 --- a/tests/assets/flake.nix +++ b/tests/assets/flake.nix @@ -8,6 +8,22 @@ { hydraJobs = import ./ci.nix { inherit pkgs; }; - legacyPackages.x86_64-linux.brokenPackage = throw "this is an evaluation error"; + legacyPackages.x86_64-linux = { + brokenPkgs = { + brokenPackage = throw "this is an evaluation error"; + }; + infiniteRecursionPkgs = { + packageWithInfiniteRecursion = + let + recursion = [ recursion ]; + in + derivation { + inherit (pkgs) system; + name = "drvB"; + recursiveAttr = recursion; + builder = ":"; + }; + }; + }; }; } diff --git a/tests/test_eval.py b/tests/test_eval.py index 88e15de40..2bc60df01 100644 --- a/tests/test_eval.py +++ b/tests/test_eval.py @@ -1,10 +1,10 @@ #!/usr/bin/env python3 -import subprocess import json -from tempfile import TemporaryDirectory +import subprocess from pathlib import Path -from typing import List, Dict, Any +from tempfile import TemporaryDirectory +from typing import Any, Dict, List TEST_ROOT = Path(__file__).parent.resolve() PROJECT_ROOT = TEST_ROOT.parent @@ -77,8 +77,10 @@ def test_eval_error() -> None: "--gc-roots-dir", tempdir, "--meta", + "--workers", + "1", "--flake", - ".#legacyPackages.x86_64-linux", + ".#legacyPackages.x86_64-linux.brokenPkgs", ] res = subprocess.run( cmd, @@ -86,6 +88,31 @@ def test_eval_error() -> None: text=True, stdout=subprocess.PIPE, ) + print(res.stdout) attrs = json.loads(res.stdout) assert attrs["attr"] == "brokenPackage" assert "this is an evaluation error" in attrs["error"] + + +def test_recursion_error() -> None: + with TemporaryDirectory() as tempdir: + cmd = [ + str(BIN), + "--gc-roots-dir", + tempdir, + "--meta", + "--workers", + "1", + "--flake", + ".#legacyPackages.x86_64-linux.infiniteRecursionPkgs", + ] + res = subprocess.run( + cmd, + cwd=TEST_ROOT.joinpath("assets"), + text=True, + stderr=subprocess.PIPE, + ) + assert res.returncode == 1 + print(res.stderr) + assert 'packageWithInfiniteRecursion' in res.stderr + assert "possible infinite recursion" in res.stderr From b4f7eb3c77bd53a3411fbe4084d70c47538e3260 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sat, 16 Dec 2023 11:17:40 +0100 Subject: [PATCH 379/419] fix ci for infinite recursion case --- .github/workflows/nix-github-actions.yml | 8 +++++++- pyproject.toml | 4 ++++ tests/test_eval.py | 2 ++ 3 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 pyproject.toml diff --git a/.github/workflows/nix-github-actions.yml b/.github/workflows/nix-github-actions.yml index 241dfa1a8..313274ac3 100644 --- a/.github/workflows/nix-github-actions.yml +++ b/.github/workflows/nix-github-actions.yml @@ -49,7 +49,13 @@ jobs: - name: Build run: nix develop -c bash -c 'meson setup -Db_sanitize=address,undefined build && ninja -C build' - name: Run tests - run: nix develop -c pytest ./tests + run: nix develop -c pytest ./tests -m 'not infiniterecursion' + + # address sanitizer will lead to out-of-memory in the infinite recursion case + - name: Build without sanitizer + run: nix develop -c bash -c 'rm -rf build && meson setup build && ninja -C build' + - name: Run tests + run: nix develop -c pytest ./tests -m 'infiniterecursion' collect: runs-on: ubuntu-latest diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000..0b0d0d11a --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,4 @@ +[tool.pytest.ini_options] +markers = [ + "infiniterecursion: mark test as infinite recursion", +] diff --git a/tests/test_eval.py b/tests/test_eval.py index 2bc60df01..93a17ac81 100644 --- a/tests/test_eval.py +++ b/tests/test_eval.py @@ -2,6 +2,7 @@ import json import subprocess +import pytest from pathlib import Path from tempfile import TemporaryDirectory from typing import Any, Dict, List @@ -94,6 +95,7 @@ def test_eval_error() -> None: assert "this is an evaluation error" in attrs["error"] +@pytest.mark.infiniterecursion def test_recursion_error() -> None: with TemporaryDirectory() as tempdir: cmd = [ From 093b8ce5cccd7aac3996bea5a293f0e0172fad95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sat, 16 Dec 2023 11:44:02 +0100 Subject: [PATCH 380/419] apply treefmt --- src/nix-eval-jobs.cc | 28 +++++++++++++++------------- tests/test_eval.py | 2 +- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/src/nix-eval-jobs.cc b/src/nix-eval-jobs.cc index 44b88e700..3c7280df3 100644 --- a/src/nix-eval-jobs.cc +++ b/src/nix-eval-jobs.cc @@ -113,13 +113,14 @@ void handleBrokenWorkerPipe(Proc &proc, std::string_view msg) { int rc = waitpid(pid, &status, WNOHANG); if (rc == 0) { kill(pid, SIGKILL); - throw Error( - "BUG: while %s, worker pipe got closed but evaluation worker still running?", - msg); + throw Error("BUG: while %s, worker pipe got closed but evaluation " + "worker still running?", + msg); } else if (rc == -1) { kill(pid, SIGKILL); - throw Error("BUG: while %s, waitpid for evaluation worker failed: %s", - msg, strerror(errno)); + throw Error( + "BUG: while %s, waitpid for evaluation worker failed: %s", msg, + strerror(errno)); } else { if (WIFEXITED(status)) { if (WEXITSTATUS(status) == 1) { @@ -128,14 +129,14 @@ void handleBrokenWorkerPipe(Proc &proc, std::string_view msg) { "(possibly an infinite recursion)", msg); } - throw Error("while %s, evaluation worker exited with %d", - msg, WEXITSTATUS(status)); + throw Error("while %s, evaluation worker exited with %d", msg, + WEXITSTATUS(status)); } else if (WIFSIGNALED(status)) { if (WTERMSIG(status) == SIGKILL) { - throw Error( - "while %s, evaluation worker got killed by SIGKILL, maybe " - "memory limit reached?", - msg); + throw Error("while %s, evaluation worker got killed by " + "SIGKILL, maybe " + "memory limit reached?", + msg); } throw Error( "while %s, evaluation worker got killed by signal %d (%s)", @@ -147,7 +148,7 @@ void handleBrokenWorkerPipe(Proc &proc, std::string_view msg) { std::string joinAttrPath(json &attrPath) { std::string joined; - for (auto& element : attrPath) { + for (auto &element : attrPath) { if (!joined.empty()) { joined += '.'; } @@ -220,7 +221,8 @@ void collector(Sync &state_, std::condition_variable &wakeup) { /* Wait for the response. */ auto respString = fromReader->readLine(); if (respString.empty()) { - auto msg = "reading result for attrPath '" + joinAttrPath(attrPath) + "'"; + auto msg = "reading result for attrPath '" + + joinAttrPath(attrPath) + "'"; handleBrokenWorkerPipe(*proc.get(), msg); } json response; diff --git a/tests/test_eval.py b/tests/test_eval.py index 93a17ac81..8fa41cb72 100644 --- a/tests/test_eval.py +++ b/tests/test_eval.py @@ -116,5 +116,5 @@ def test_recursion_error() -> None: ) assert res.returncode == 1 print(res.stderr) - assert 'packageWithInfiniteRecursion' in res.stderr + assert "packageWithInfiniteRecursion" in res.stderr assert "possible infinite recursion" in res.stderr From b73f7ceff479a96fad9d79ffdbfda346ec0bd567 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sat, 16 Dec 2023 11:53:10 +0100 Subject: [PATCH 381/419] classify SIGSEV/SIGBUS as infinite recursion errors --- src/nix-eval-jobs.cc | 29 +++++++++++++++++++++++------ 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/src/nix-eval-jobs.cc b/src/nix-eval-jobs.cc index 3c7280df3..187c95da8 100644 --- a/src/nix-eval-jobs.cc +++ b/src/nix-eval-jobs.cc @@ -126,17 +126,34 @@ void handleBrokenWorkerPipe(Proc &proc, std::string_view msg) { if (WEXITSTATUS(status) == 1) { throw Error( "while %s, evaluation worker exited with exit code 1, " - "(possibly an infinite recursion)", + "(possible infinite recursion)", msg); } throw Error("while %s, evaluation worker exited with %d", msg, WEXITSTATUS(status)); } else if (WIFSIGNALED(status)) { - if (WTERMSIG(status) == SIGKILL) { - throw Error("while %s, evaluation worker got killed by " - "SIGKILL, maybe " - "memory limit reached?", - msg); + switch (WTERMSIG(status)) { + case SIGKILL: + throw Error( + "while %s, evaluation worker got killed by SIGKILL, " + "maybe " + "memory limit reached?", + msg); + break; +#ifdef __APPLE__ + case SIGBUS: + throw Error( + "while %s, evaluation worker got killed by SIGBUS, " + "(possible infinite recursion)", + msg); + break; +#else + case SIGSEGV: + throw Error( + "while %s, evaluation worker got killed by SIGSEGV, " + "(possible infinite recursion)", + msg); +#endif } throw Error( "while %s, evaluation worker got killed by signal %d (%s)", From e3d71921c336b6760d4079674d87a7e56afc31a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sun, 17 Dec 2023 12:44:23 +0100 Subject: [PATCH 382/419] buffered-io: also check for interrupts in readline This allows us to more reliable interrupt nix-eval-jobs --- src/buffered-io.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/buffered-io.cc b/src/buffered-io.cc index 4354740bd..c74d65eaa 100644 --- a/src/buffered-io.cc +++ b/src/buffered-io.cc @@ -51,6 +51,8 @@ LineReader::LineReader(LineReader &&other) { return {}; // Return an empty string_view in case of error } + nix::checkInterrupt(); + // Remove trailing newline return std::string_view(buffer, read - 1); } From 280de12085b3986ffda8c7810433beec62afb72f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sun, 17 Dec 2023 12:45:38 +0100 Subject: [PATCH 383/419] README: update json and help output --- README.md | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 8f89b4d26..c812b1668 100644 --- a/README.md +++ b/README.md @@ -25,9 +25,11 @@ In the following example we evaluate the hydraJobs attribute of the ```console $ nix-eval-jobs --gc-roots-dir gcroot --flake 'github:NixOS/patchelf#hydraJobs' -{"attr":"coverage","attrPath":["coverage"],"drvPath":"/nix/store/8hq9f09xa5s6g9m02lw0sw59kkkvj57c-patchelf-coverage-0.15.0.drv","name":"patchelf-coverage-0.15.0","outputs":{"out":"/nix/store/dwf255bdbfvvbiqak941r83zlvxyipcs-patchelf-coverage-0.15.0"},"system":"x86_64-linux"} -{"attr":"release","attrPath":["release"],"drvPath":"/nix/store/ip9dy4vlyha5a7kq4bnf4pxk0sfwjfda-patchelf-0.15.0.drv","name":"patchelf-0.15.0","outputs":{"out":"/nix/store/5z9ynn29asakf1b5736im2glcqpf6s2f-patchelf-0.15.0"},"system":"x86_64-linux"} -{"attr":"tarball","attrPath":["tarball"],"drvPath":"/nix/store/g1alnfi3mrkcb9blclr77fpyp35mpsdd-patchelf-tarball-0.15.0.drv","name":"patchelf-tarball-0.15.0","outputs":{"out":"/nix/store/iy0w42pffhjg6wy0w46r4cjc1yjk410y-patchelf-tarball-0.15.0"},"system":"x86_64-linux"} +{"attr":"coverage","attrPath":["coverage"],"drvPath":"/nix/store/fmbqzaq8mim1423879lhn9whs6imx5w4-patchelf-coverage-0.18.0.drv","inputDrvs":{"/nix/store/23632hx2c98lbbjld279dx0w08lxn6kp-hook.drv":["out"],"/nix/store/6z1jfnqqgyqr221zgbpm30v91yfj3r45-bash-5.1-p16.drv":["out"],"/nix/store/ap9g09fxbicj836zm88d56dn3ff4clxl-stdenv-linux.drv":["out"],"/nix/store/c0gg7lj101xhd8v2b3cjl5dwwkpxfc0q-patchelf-tarball-0.18.0.drv":["out"],"/nix/store/vslywm6kbazi37q1vbq8y7bi884yc6yx-lcov-1.16.drv":["out"],"/nix/store/y964yq4vz1gsn7azd44vyg65gnr4gpvi-hook.drv":["out"]},"name":"patchelf-coverage-0.18.0","outputs":{"out":"/nix/store/gfni9sbhhwhxxfqziq1fs3n82bvw962l-patchelf-coverage-0.18.0"},"system":"x86_64-linux"} +{"attr":"patchelf-win32","attrPath":["patchelf-win32"],"drvPath":"/nix/store/s38l0fg5ja6j8qpws7slw2ws0c6v0qcf-patchelf-i686-w64-mingw32-0.18.0.drv","inputDrvs":{"/nix/store/6z1jfnqqgyqr221zgbpm30v91yfj3r45-bash-5.1-p16.drv":["out"],"/nix/store/b2p151ilwqpd47fbmzz50a5cmj12ixbf-hook.drv":["out"],"/nix/store/fbnhh18m4jh6cwa92am2sv3aqzjnzpdd-stdenv-linux.drv":["out"]},"name":"patchelf-i686-w64-mingw32-0.18.0","outputs":{"out":"/nix/store/w8r4h1xk71fryb99df8aszp83kfhw3bc-patchelf-i686-w64-mingw32-0.18.0"},"system":"x86_64-linux"} +{"attr":"patchelf-win64","attrPath":["patchelf-win64"],"drvPath":"/nix/store/wxpym6d3dxr1w9syhinp7f058gwxfmd3-patchelf-x86_64-w64-mingw32-0.18.0.drv","inputDrvs":{"/nix/store/6z1jfnqqgyqr221zgbpm30v91yfj3r45-bash-5.1-p16.drv":["out"],"/nix/store/71lv5lsr1y59bv1b91jc9gg0n85kf1sq-stdenv-linux.drv":["out"],"/nix/store/b2p151ilwqpd47fbmzz50a5cmj12ixbf-hook.drv":["out"]},"name":"patchelf-x86_64-w64-mingw32-0.18.0","outputs":{"out":"/nix/store/fkq5428l2xsb84yj0cc6q1lkvsrga7sv-patchelf-x86_64-w64-mingw32-0.18.0"},"system":"x86_64-linux"} +{"attr":"release","attrPath":["release"],"drvPath":"/nix/store/3xpwg8f623dpkh6cblv2fzcq5n99xl0j-patchelf-0.18.0.drv","inputDrvs":{"/nix/store/6z1jfnqqgyqr221zgbpm30v91yfj3r45-bash-5.1-p16.drv":["out"],"/nix/store/9rmihrl9ys0sap6827xyns0y73vqafjx-patchelf-0.18.0.drv":["out"],"/nix/store/am2zqx3pyc1i14f888jna785h0f841sg-patchelf-0.18.0.drv":["out"],"/nix/store/c0gg7lj101xhd8v2b3cjl5dwwkpxfc0q-patchelf-tarball-0.18.0.drv":["out"],"/nix/store/csjiccxbwpfv55m8kqs2xwrkkha14dnq-patchelf-0.18.0.drv":["out"],"/nix/store/jsrnpxdx5vmpnakd9bkb3sk3lgh0k8hm-patchelf-0.18.0.drv":["out"],"/nix/store/k8a51ax83554c67g98xf3y751vjgjs7m-patchelf-0.18.0.drv":["out"],"/nix/store/wq3ncl207isqqkqmsa5ql4fg19jbrhxg-stdenv-linux.drv":["out"]},"name":"patchelf-0.18.0","outputs":{"out":"/nix/store/d0mzprvv3vhasj23r1a6qn8qip0srbc4-patchelf-0.18.0"},"system":"x86_64-linux"} +{"attr":"tarball","attrPath":["tarball"],"drvPath":"/nix/store/c0gg7lj101xhd8v2b3cjl5dwwkpxfc0q-patchelf-tarball-0.18.0.drv","inputDrvs":{"/nix/store/6z1jfnqqgyqr221zgbpm30v91yfj3r45-bash-5.1-p16.drv":["out"],"/nix/store/9d754glmsvpjm5kxvgsjslvgv356kbmn-libtool-2.4.7.drv":["out"],"/nix/store/ap9g09fxbicj836zm88d56dn3ff4clxl-stdenv-linux.drv":["out"],"/nix/store/f1ksgsyplvb0sli4pls6k6vsfvmv519d-autoconf-2.71.drv":["out"],"/nix/store/jf58lcnch1bmpbi2188c59w5zr1cqrx2-automake-1.16.5.drv":["out"]},"name":"patchelf-tarball-0.18.0","outputs":{"out":"/nix/store/72pz5awc7gpwdqxrdsy8j0bvg2n7z78q-patchelf-tarball-0.18.0"},"system":"x86_64-linux"} ``` The output here is newline-seperated json according to https://jsonlines.org. @@ -45,19 +47,25 @@ USAGE: nix-eval-jobs [options] expr --argstr Pass the string *string* as the argument *name* to Nix functions. --check-cache-status Check if the derivations are present locally or in any configured substituters (i.e. binary cache). The information will be exposed in the `isCached` field of the JSON output. --debug Set the logging verbosity level to 'debug'. - --eval-store The Nix store to use for evaluations. + --eval-store + The [URL of the Nix store](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format) + to use for evaluation, i.e. to store derivations (`.drv` files) and inputs referenced by them. + --expr treat the argument as a Nix expression --flake build a flake + --force-recurse force recursion (don't respect recurseIntoAttrs) --gc-roots-dir garbage collector roots directory --help show usage information --impure allow impure expressions - --include Add *path* to the list of locations used to look up `<...>` file names. + --log-format Set the format of log output; one of `raw`, `internal-json`, `bar` or `bar-with-logs`. - --max-memory-size maximum evaluation memory size + --max-memory-size maximum evaluation memory size in megabyte (4GiB per worker by default) --meta include derivation meta field in output --option Set the Nix configuration setting *name* to *value* (overriding `nix.conf`). --override-flake Override the flake registries, redirecting *original-ref* to *resolved-ref*. + --override-input Override a specific flake input (e.g. `dwarffs/nixpkgs`). --quiet Decrease the logging verbosity level. + --repair During evaluation, rewrite missing or corrupted files in the Nix store. During building, rebuild missing or corrupted store paths. --show-trace print out a stack trace in case of evaluation errors --verbose Increase the logging verbosity level. --workers number of evaluate workers From 8ef37192bdd381a07e496d73368bac8c95d4ca4c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sun, 17 Dec 2023 12:46:08 +0100 Subject: [PATCH 384/419] README: add check-cache-status and nixpkgs evaluation examples --- README.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/README.md b/README.md index c812b1668..72df09101 100644 --- a/README.md +++ b/README.md @@ -110,6 +110,20 @@ latest release branch. ## FAQ +### How can I check if my package already have been uploaded in the binary cache? + +If you provide the `--check-cache-status`, the json will contain a `"isCached"` +key in its json, that is true or false based on the status. + +### How can I evaluate nixpkgs? + +If you want to evaluate nixpkgs in the same way +[hydra](https://hydra.nixos.org/) does it, use this snippet: + +```console +$ nix-eval-jobs --force-recurse pkgs/top-level/release.nix +``` + ### nix-eval-jobs consumes too much memory / is too slow By default, nix-eval-jobs spawns as many worker processes as there are hardware From 3c6e1234af3aa26fc60d0969619cf6806ec51639 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sun, 17 Dec 2023 13:00:32 +0100 Subject: [PATCH 385/419] flake.nix: return to nixpkgs-unstable --- flake.lock | 8 ++++---- flake.nix | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/flake.lock b/flake.lock index d0d8b0499..9385af682 100644 --- a/flake.lock +++ b/flake.lock @@ -42,16 +42,16 @@ }, "nixpkgs": { "locked": { - "lastModified": 1702667777, - "narHash": "sha256-qpgZVpFrOEgW0DimJ24UXeFh63TI9fQFXxc58DPtG8Q=", + "lastModified": 1702539185, + "narHash": "sha256-KnIRG5NMdLIpEkZTnN5zovNYc0hhXjAgv6pfd5Z4c7U=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "203ecda835bcf69633df7183459283543dd4a874", + "rev": "aa9d4729cbc99dabacb50e3994dcefb3ea0f7447", "type": "github" }, "original": { "owner": "NixOS", - "ref": "master", + "ref": "nixpkgs-unstable", "repo": "nixpkgs", "type": "github" } diff --git a/flake.nix b/flake.nix index 4b6a65b74..07126d884 100644 --- a/flake.nix +++ b/flake.nix @@ -1,7 +1,7 @@ { description = "Hydra's builtin hydra-eval-jobs as a standalone"; - inputs.nixpkgs.url = "github:NixOS/nixpkgs/master"; + inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable"; inputs.flake-parts.url = "github:hercules-ci/flake-parts"; inputs.flake-parts.inputs.nixpkgs-lib.follows = "nixpkgs"; inputs.treefmt-nix.url = "github:numtide/treefmt-nix"; From 1a9b928a5ff112558fb97da6aefd46c69e391e1c Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 25 Dec 2023 01:02:06 +0000 Subject: [PATCH 386/419] Lock file maintenance --- flake.lock | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/flake.lock b/flake.lock index 9385af682..545fb2587 100644 --- a/flake.lock +++ b/flake.lock @@ -42,11 +42,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1702539185, - "narHash": "sha256-KnIRG5NMdLIpEkZTnN5zovNYc0hhXjAgv6pfd5Z4c7U=", + "lastModified": 1703134684, + "narHash": "sha256-SQmng1EnBFLzS7WSRyPM9HgmZP2kLJcPAz+Ug/nug6o=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "aa9d4729cbc99dabacb50e3994dcefb3ea0f7447", + "rev": "d6863cbcbbb80e71cecfc03356db1cda38919523", "type": "github" }, "original": { @@ -71,11 +71,11 @@ ] }, "locked": { - "lastModified": 1702461037, - "narHash": "sha256-ssyGxfGHRuuLHuMex+vV6RMOt7nAo07nwufg9L5GkLg=", + "lastModified": 1702979157, + "narHash": "sha256-RnFBbLbpqtn4AoJGXKevQMCGhra4h6G2MPcuTSZZQ+g=", "owner": "numtide", "repo": "treefmt-nix", - "rev": "d06b70e5163a903f19009c3f97770014787a080f", + "rev": "2961375283668d867e64129c22af532de8e77734", "type": "github" }, "original": { From 64104a3c55593c903af78af86a4c9d2e5487a2d7 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 25 Dec 2023 01:02:06 +0000 Subject: [PATCH 387/419] Lock file maintenance --- flake.lock | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/flake.lock b/flake.lock index 9385af682..545fb2587 100644 --- a/flake.lock +++ b/flake.lock @@ -42,11 +42,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1702539185, - "narHash": "sha256-KnIRG5NMdLIpEkZTnN5zovNYc0hhXjAgv6pfd5Z4c7U=", + "lastModified": 1703134684, + "narHash": "sha256-SQmng1EnBFLzS7WSRyPM9HgmZP2kLJcPAz+Ug/nug6o=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "aa9d4729cbc99dabacb50e3994dcefb3ea0f7447", + "rev": "d6863cbcbbb80e71cecfc03356db1cda38919523", "type": "github" }, "original": { @@ -71,11 +71,11 @@ ] }, "locked": { - "lastModified": 1702461037, - "narHash": "sha256-ssyGxfGHRuuLHuMex+vV6RMOt7nAo07nwufg9L5GkLg=", + "lastModified": 1702979157, + "narHash": "sha256-RnFBbLbpqtn4AoJGXKevQMCGhra4h6G2MPcuTSZZQ+g=", "owner": "numtide", "repo": "treefmt-nix", - "rev": "d06b70e5163a903f19009c3f97770014787a080f", + "rev": "2961375283668d867e64129c22af532de8e77734", "type": "github" }, "original": { From 69371f7bae49d5d55bcee9fd829585148215bedb Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 12 Jan 2024 17:28:25 +0000 Subject: [PATCH 388/419] Update cachix/install-nix-action action to v25 --- .github/workflows/nix-github-actions.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/nix-github-actions.yml b/.github/workflows/nix-github-actions.yml index 313274ac3..cafc26ccc 100644 --- a/.github/workflows/nix-github-actions.yml +++ b/.github/workflows/nix-github-actions.yml @@ -15,7 +15,7 @@ jobs: matrix: ${{ steps.set-matrix.outputs.matrix }} steps: - uses: actions/checkout@v4 - - uses: cachix/install-nix-action@v24 + - uses: cachix/install-nix-action@v25 - id: set-matrix name: Generate Nix Matrix run: | @@ -29,7 +29,7 @@ jobs: matrix: ${{fromJSON(needs.nix-matrix.outputs.matrix)}} steps: - uses: actions/checkout@v4 - - uses: cachix/install-nix-action@v24 + - uses: cachix/install-nix-action@v25 - run: nix build -L ".#${{ matrix.attr }}" tests: strategy: @@ -41,7 +41,7 @@ jobs: with: # Nix Flakes doesn't work on shallow clones fetch-depth: 0 - - uses: cachix/install-nix-action@v24 + - uses: cachix/install-nix-action@v25 with: github_access_token: ${{ secrets.GITHUB_TOKEN }} extra_nix_config: | From 733f3051b15448e491644e3737de0887b2aa7df6 Mon Sep 17 00:00:00 2001 From: Linus Heckemann Date: Mon, 15 Apr 2024 19:20:21 +0200 Subject: [PATCH 389/419] Fix include paths for Nix headers While the previous style seems more sensible to me (the header names are quite generic), the pkg-config definitions both from upstream Nix and from Lix specify -I${includedir}/nix in CFLAGS rather than -I${includedir}. This may be worth changing, but for now I want nix-eval-jobs to work and this does what I want. --- src/eval-args.cc | 7 +++---- src/nix-eval-jobs.cc | 27 ++++++++++++--------------- src/worker.cc | 43 ++++++++++++++++++++----------------------- 3 files changed, 35 insertions(+), 42 deletions(-) diff --git a/src/eval-args.cc b/src/eval-args.cc index 5ccf732b2..0417c3197 100644 --- a/src/eval-args.cc +++ b/src/eval-args.cc @@ -1,9 +1,8 @@ #include #include -#include -#include -#include -#include +#include +#include +#include #include #include #include diff --git a/src/nix-eval-jobs.cc b/src/nix-eval-jobs.cc index 187c95da8..b1b3bd548 100644 --- a/src/nix-eval-jobs.cc +++ b/src/nix-eval-jobs.cc @@ -1,11 +1,10 @@ #include // IWYU pragma: keep -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include #include #include #include @@ -13,18 +12,16 @@ #include #include #include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include #include #include #include -#include -#include -#include +#include +#include #include #include #include diff --git a/src/worker.cc b/src/worker.cc index cb0a7d3a8..9e8ea2b7e 100644 --- a/src/worker.cc +++ b/src/worker.cc @@ -3,35 +3,32 @@ // doesn't exist on macOS // IWYU pragma: no_include -#include -#include -#include -#include +#include +#include +#include #include #include #include #include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include #include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include #include #include #include From a2bd94525953ffab62348c6c5eed7e69c154125c Mon Sep 17 00:00:00 2001 From: Linus Heckemann Date: Mon, 15 Apr 2024 19:28:36 +0200 Subject: [PATCH 390/419] hasPrefix -> starts_with hasPrefix was removed in lix commit 61e21b25576f7f3491f6a837bf59d8b44c6897a0 --- src/worker.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/worker.cc b/src/worker.cc index 9e8ea2b7e..ef7b100a5 100644 --- a/src/worker.cc +++ b/src/worker.cc @@ -106,7 +106,7 @@ void worker(nix::ref state, nix::Bindings &autoArgs, if (s == "exit") { break; } - if (!nix::hasPrefix(s, "do ")) { + if (!s.starts_with("do ")) { fprintf(stderr, "worker error: received invalid command '%s'\n", s.data()); abort(); From 14e4308346c24c611c74c4b4d7334409a9f98081 Mon Sep 17 00:00:00 2001 From: Linus Heckemann Date: Mon, 15 Apr 2024 19:30:32 +0200 Subject: [PATCH 391/419] parseCmdline: no allowShebang arg This arg was only introduced in Nix 2.19 (commit ffd414eb756dcb3c64348551d5dbaf674c0d4900) --- src/eval-args.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/eval-args.cc b/src/eval-args.cc index 0417c3197..871c6b04a 100644 --- a/src/eval-args.cc +++ b/src/eval-args.cc @@ -100,5 +100,5 @@ MyArgs::MyArgs() : MixCommonArgs("nix-eval-jobs") { } void MyArgs::parseArgs(char **argv, int argc) { - parseCmdline(nix::argvToStrings(argc, argv), 0); + parseCmdline(nix::argvToStrings(argc, argv)); } From c1ee00bf7ce056274b38cc2ff490e57416eeb4b6 Mon Sep 17 00:00:00 2001 From: Linus Heckemann Date: Mon, 15 Apr 2024 19:30:56 +0200 Subject: [PATCH 392/419] ProcessOptions: remove allowVfork field This was removed in Lix 1f8b85786eed623319e5c71a5341b15e3006f870 --- src/nix-eval-jobs.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix-eval-jobs.cc b/src/nix-eval-jobs.cc index b1b3bd548..f7ce554ad 100644 --- a/src/nix-eval-jobs.cc +++ b/src/nix-eval-jobs.cc @@ -85,7 +85,7 @@ struct Proc { } } }, - ProcessOptions{.allowVfork = false}); + ProcessOptions{}); to = std::move(toPipe.writeSide); from = std::move(fromPipe.readSide); From c3d8ca19b39f4cc5a2df1061baf649d1fe20517e Mon Sep 17 00:00:00 2001 From: Linus Heckemann Date: Mon, 15 Apr 2024 19:35:05 +0200 Subject: [PATCH 393/419] EvalError needs to reference an eval state now --- src/drv.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/drv.cc b/src/drv.cc index 14fd35276..7d1170b02 100644 --- a/src/drv.cc +++ b/src/drv.cc @@ -50,7 +50,7 @@ Drv::Drv(std::string &attrPath, nix::EvalState &state, nix::DrvInfo &drvInfo, outputs[out.first] = localStore->printStorePath(*out.second); } } catch (const std::exception &e) { - throw nix::EvalError("derivation '%s' does not have valid outputs: %s", + throw state.error("derivation '%s' does not have valid outputs: %s", attrPath, e.what()); } From 793841a9b7b689e37c9a7902710aab2bd6a833d5 Mon Sep 17 00:00:00 2001 From: eldritch horrors Date: Thu, 18 Apr 2024 18:05:33 +0200 Subject: [PATCH 394/419] fix drv.cc invalid output throw EvalState::error does not return an exception instance in lix, but an exception *builder*. throwing this thing will not trigger any catches, which then causes the worker process to die without reporting an error to the collector. this confuses the collector and causes *it* to exit, effectively breaking nix-eval-jobs for anything that has broken attrs. --- src/drv.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/drv.cc b/src/drv.cc index 7d1170b02..c5e133993 100644 --- a/src/drv.cc +++ b/src/drv.cc @@ -50,8 +50,9 @@ Drv::Drv(std::string &attrPath, nix::EvalState &state, nix::DrvInfo &drvInfo, outputs[out.first] = localStore->printStorePath(*out.second); } } catch (const std::exception &e) { - throw state.error("derivation '%s' does not have valid outputs: %s", - attrPath, e.what()); + throw nix::EvalError(state, + "derivation '%s' does not have valid outputs: %s", + attrPath, e.what()); } if (args.meta) { From 2dbcbe4179a13c7d04659f9d0c6fdef23f71526b Mon Sep 17 00:00:00 2001 From: Linus Heckemann Date: Thu, 9 May 2024 13:11:03 +0200 Subject: [PATCH 395/419] flake: actually use lix --- flake.lock | 73 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ flake.nix | 8 ++++-- 2 files changed, 79 insertions(+), 2 deletions(-) diff --git a/flake.lock b/flake.lock index 545fb2587..4d9f0466c 100644 --- a/flake.lock +++ b/flake.lock @@ -1,5 +1,21 @@ { "nodes": { + "flake-compat": { + "flake": false, + "locked": { + "lastModified": 1696426674, + "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=", + "owner": "edolstra", + "repo": "flake-compat", + "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33", + "type": "github" + }, + "original": { + "owner": "edolstra", + "repo": "flake-compat", + "type": "github" + } + }, "flake-parts": { "inputs": { "nixpkgs-lib": [ @@ -20,6 +36,30 @@ "type": "github" } }, + "lix": { + "inputs": { + "flake-compat": "flake-compat", + "nixpkgs": [ + "nixpkgs" + ], + "nixpkgs-regression": "nixpkgs-regression", + "pre-commit-hooks": "pre-commit-hooks" + }, + "locked": { + "lastModified": 1714955862, + "narHash": "sha256-REWlo2RYHfJkxnmZTEJu3Cd/2VM+wjjpPy7Xi4BdDTQ=", + "ref": "refs/tags/2.90-beta.1", + "rev": "b6799ab0374a8e1907a48915d3187e07da41d88c", + "revCount": 15501, + "type": "git", + "url": "https://git@git.lix.systems/lix-project/lix" + }, + "original": { + "ref": "refs/tags/2.90-beta.1", + "type": "git", + "url": "https://git@git.lix.systems/lix-project/lix" + } + }, "nix-github-actions": { "inputs": { "nixpkgs": [ @@ -56,9 +96,42 @@ "type": "github" } }, + "nixpkgs-regression": { + "locked": { + "lastModified": 1643052045, + "narHash": "sha256-uGJ0VXIhWKGXxkeNnq4TvV3CIOkUJ3PAoLZ3HMzNVMw=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "215d4d0fd80ca5163643b03a33fde804a29cc1e2", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "215d4d0fd80ca5163643b03a33fde804a29cc1e2", + "type": "github" + } + }, + "pre-commit-hooks": { + "flake": false, + "locked": { + "lastModified": 1712055707, + "narHash": "sha256-4XLvuSIDZJGS17xEwSrNuJLL7UjDYKGJSbK1WWX2AK8=", + "owner": "cachix", + "repo": "git-hooks.nix", + "rev": "e35aed5fda3cc79f88ed7f1795021e559582093a", + "type": "github" + }, + "original": { + "owner": "cachix", + "repo": "git-hooks.nix", + "type": "github" + } + }, "root": { "inputs": { "flake-parts": "flake-parts", + "lix": "lix", "nix-github-actions": "nix-github-actions", "nixpkgs": "nixpkgs", "treefmt-nix": "treefmt-nix" diff --git a/flake.nix b/flake.nix index 07126d884..f764e666b 100644 --- a/flake.nix +++ b/flake.nix @@ -8,6 +8,10 @@ inputs.treefmt-nix.inputs.nixpkgs.follows = "nixpkgs"; inputs.nix-github-actions.url = "github:nix-community/nix-github-actions"; inputs.nix-github-actions.inputs.nixpkgs.follows = "nixpkgs"; + inputs.lix = { + url = "git+https://git@git.lix.systems/lix-project/lix?ref=refs/tags/2.90-beta.1"; + inputs.nixpkgs.follows = "nixpkgs"; + }; outputs = inputs @ { flake-parts, ... }: let @@ -27,11 +31,11 @@ }; }; - perSystem = { pkgs, self', ... }: + perSystem = { pkgs, self', inputs', ... }: let drvArgs = { srcDir = self; - nix = if nixVersion == "unstable" then pkgs.nixUnstable else pkgs.nixVersions."nix_${nixVersion}"; + nix = inputs'.lix.packages.default; }; in { From 52e96bd421e027052a3dd0f40124f0547c9bea47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Tue, 23 Apr 2024 09:07:53 +0200 Subject: [PATCH 396/419] fix crash in worker when opening the store --- src/nix-eval-jobs.cc | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/nix-eval-jobs.cc b/src/nix-eval-jobs.cc index f7ce554ad..855fa983a 100644 --- a/src/nix-eval-jobs.cc +++ b/src/nix-eval-jobs.cc @@ -66,8 +66,11 @@ struct Proc { std::make_shared(std::move(toPipe.readSide))}]() { debug("created worker process %d", getpid()); try { - auto state = std::make_shared( - myArgs.searchPath, openStore(*myArgs.evalStoreUrl)); + auto evalStore = myArgs.evalStoreUrl + ? openStore(*myArgs.evalStoreUrl) + : openStore(); + auto state = std::make_shared(myArgs.searchPath, + evalStore); Bindings &autoArgs = *myArgs.getAutoArgs(*state); proc(ref(state), autoArgs, *to, *from, myArgs); } catch (Error &e) { From a94f80e5128847184b29f283738ee62dd4240c21 Mon Sep 17 00:00:00 2001 From: Jade Lovelace Date: Thu, 16 May 2024 16:47:20 -0700 Subject: [PATCH 397/419] nixexpr -> lixexpr --- meson.build | 8 ++++---- src/buffered-io.cc | 4 ++-- src/drv.cc | 26 +++++++++++++------------- src/drv.hh | 4 ++-- src/eval-args.cc | 6 +++--- src/eval-args.hh | 12 ++++++------ src/nix-eval-jobs.cc | 26 +++++++++++++------------- src/worker.cc | 42 +++++++++++++++++++++--------------------- src/worker.hh | 4 ++-- 9 files changed, 66 insertions(+), 66 deletions(-) diff --git a/meson.build b/meson.build index 00086fdf7..60dfef734 100644 --- a/meson.build +++ b/meson.build @@ -3,10 +3,10 @@ project('nix-eval-jobs', 'cpp', license : 'GPL-3.0', ) -nix_main_dep = dependency('nix-main', required: true) -nix_store_dep = dependency('nix-store', required: true) -nix_expr_dep = dependency('nix-expr', required: true) -nix_cmd_dep = dependency('nix-cmd', required: true) +nix_main_dep = dependency('lix-main', required: true) +nix_store_dep = dependency('lix-store', required: true) +nix_expr_dep = dependency('lix-expr', required: true) +nix_cmd_dep = dependency('lix-cmd', required: true) threads_dep = dependency('threads', required: true) nlohmann_json_dep = dependency('nlohmann_json', required: true) boost_dep = dependency('boost', required: true) diff --git a/src/buffered-io.cc b/src/buffered-io.cc index c74d65eaa..469dbe23f 100644 --- a/src/buffered-io.cc +++ b/src/buffered-io.cc @@ -1,9 +1,9 @@ #include #include -#include +#include #include #include -#include +#include #include "buffered-io.hh" diff --git a/src/drv.cc b/src/drv.cc index c5e133993..5af2dfc57 100644 --- a/src/drv.cc +++ b/src/drv.cc @@ -1,19 +1,19 @@ -#include // IWYU pragma: keep +#include // IWYU pragma: keep -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include #include -#include -#include -#include -#include +#include +#include +#include +#include #include -#include -#include -#include +#include +#include +#include #include #include #include diff --git a/src/drv.hh b/src/drv.hh index a5de64a9b..ab817c27e 100644 --- a/src/drv.hh +++ b/src/drv.hh @@ -1,5 +1,5 @@ -#include -#include +#include +#include #include #include #include diff --git a/src/eval-args.cc b/src/eval-args.cc index 871c6b04a..b12763892 100644 --- a/src/eval-args.cc +++ b/src/eval-args.cc @@ -1,8 +1,8 @@ #include #include -#include -#include -#include +#include +#include +#include #include #include #include diff --git a/src/eval-args.hh b/src/eval-args.hh index fe505834f..84f2ba195 100644 --- a/src/eval-args.hh +++ b/src/eval-args.hh @@ -1,12 +1,12 @@ #pragma once -#include -#include -#include +#include +#include +#include #include -#include -#include -#include +#include +#include +#include #include #include diff --git a/src/nix-eval-jobs.cc b/src/nix-eval-jobs.cc index 855fa983a..ee082f7a4 100644 --- a/src/nix-eval-jobs.cc +++ b/src/nix-eval-jobs.cc @@ -1,10 +1,10 @@ -#include // IWYU pragma: keep +#include // IWYU pragma: keep -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include #include #include #include @@ -12,16 +12,16 @@ #include #include #include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include #include #include #include -#include -#include +#include +#include #include #include #include diff --git a/src/worker.cc b/src/worker.cc index ef7b100a5..656cb075a 100644 --- a/src/worker.cc +++ b/src/worker.cc @@ -1,34 +1,34 @@ -#include // IWYU pragma: keep +#include // IWYU pragma: keep // doesn't exist on macOS // IWYU pragma: no_include -#include -#include -#include +#include +#include +#include #include #include #include #include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include #include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include #include #include #include diff --git a/src/worker.hh b/src/worker.hh index 45e9032dd..caf4200b8 100644 --- a/src/worker.hh +++ b/src/worker.hh @@ -1,6 +1,6 @@ #pragma once -#include -#include +#include +#include #include "eval-args.hh" From 7e22a3daab4766800f2973a195f7fb7c810aaef7 Mon Sep 17 00:00:00 2001 From: Jade Lovelace Date: Mon, 27 May 2024 12:54:51 -0600 Subject: [PATCH 398/419] Fixup readme and version to say what this is --- .nix-version | 1 - README.md | 14 +++++++++----- default.nix | 2 +- flake.nix | 1 - 4 files changed, 10 insertions(+), 8 deletions(-) delete mode 100644 .nix-version diff --git a/.nix-version b/.nix-version deleted file mode 100644 index 6842dbdf3..000000000 --- a/.nix-version +++ /dev/null @@ -1 +0,0 @@ -unstable diff --git a/README.md b/README.md index 72df09101..e2c8bcc18 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,8 @@ # nix-eval-jobs +> [!NOTE] +> This is a fork of nix-eval-jobs that works with Lix. + This project evaluates nix attribute sets in parallel with streamable json output. This is useful for time and memory intensive evaluations such as NixOS machines, i.e. in a CI context. The evaluation is done with a controllable @@ -89,11 +92,12 @@ we collect example ci configuration for various CIs. ## Organisation of this repository -On the `main` branch we target nixUnstable. When a release of nix happens, we -fork for a release branch i.e. `release-2.8` and change the nix version in -`.nix-version`. Changes and improvements made in `main` also may be backported -to these release branches. At the time of writing we only intent to support the -latest release branch. +`main` follows Lix HEAD, and is updated alongside the Lix NixOS module. When we +release we will make a `release-2.90` etc branch, which receives backports. + +The version of nix-eval-jobs follows the major version of Lix and minor +versions of nix-eval-jobs are released as necessary when changes are made in +n-e-j itself. ## Projects using nix-eval-jobs diff --git a/default.nix b/default.nix index bb2ddf3c1..9ef66afd4 100644 --- a/default.nix +++ b/default.nix @@ -11,7 +11,7 @@ let in stdenv.mkDerivation { pname = "nix-eval-jobs"; - version = "2.19.0"; + version = "2.90.0-unstable"; src = if srcDir == null then filterMesonBuild ./. else srcDir; buildInputs = with pkgs; [ nlohmann_json diff --git a/flake.nix b/flake.nix index f764e666b..1219d5c95 100644 --- a/flake.nix +++ b/flake.nix @@ -17,7 +17,6 @@ let inherit (inputs.nixpkgs) lib; inherit (inputs) self; - nixVersion = lib.fileContents ./.nix-version; in flake-parts.lib.mkFlake { inherit inputs; } { From 30cf61fd26894d397c3f24eed888e84acf9c9218 Mon Sep 17 00:00:00 2001 From: Jade Lovelace Date: Mon, 27 May 2024 18:23:13 -0600 Subject: [PATCH 399/419] fix: don't crash on startup on macOS This is caused, through several layers of absurdity, by runtime type information of Nix things being invisible due to -fvisibility=hidden inside n-e-j. We don't have any idea why n-e-j has -fvisibility=hidden, since blame says it's from the initial commit. It is plausible that it was some ill-advised optimization but it's not sound. The crash is caused by dynamic_cast(MyArgs *) failing, which is in turn caused by the RTTI being invisible. See: https://www.qt.io/blog/quality-assurance/one-way-dynamic_cast-across-library-boundaries-can-fail-and-how-to-fix-it Fixes: https://git.lix.systems/lix-project/nix-eval-jobs/issues/2 --- src/eval-args.hh | 2 +- src/meson.build | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/eval-args.hh b/src/eval-args.hh index 84f2ba195..b0932fb8c 100644 --- a/src/eval-args.hh +++ b/src/eval-args.hh @@ -12,7 +12,7 @@ class MyArgs : virtual public nix::MixEvalArgs, virtual public nix::MixCommonArgs, - virtual nix::RootArgs { + virtual public nix::RootArgs { public: std::string releaseExpr; nix::Path gcRootsDir; diff --git a/src/meson.build b/src/meson.build index dfbcd8754..745e0365e 100644 --- a/src/meson.build +++ b/src/meson.build @@ -31,4 +31,4 @@ executable('nix-eval-jobs', src, threads_dep ], install: true, - cpp_args: ['-std=c++2a', '-fvisibility=hidden', '--include', 'autotools-config.h']) + cpp_args: ['-std=c++2a', '--include', 'autotools-config.h']) From b67c46d32084b8bf49cb9d71918c52022eb63d2e Mon Sep 17 00:00:00 2001 From: Jade Lovelace Date: Wed, 29 May 2024 19:17:54 -0700 Subject: [PATCH 400/419] lix: deal with util.hh removal --- src/nix-eval-jobs.cc | 1 + src/worker.cc | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/nix-eval-jobs.cc b/src/nix-eval-jobs.cc index ee082f7a4..e2b204bfc 100644 --- a/src/nix-eval-jobs.cc +++ b/src/nix-eval-jobs.cc @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include diff --git a/src/worker.cc b/src/worker.cc index 656cb075a..817d81fa4 100644 --- a/src/worker.cc +++ b/src/worker.cc @@ -27,8 +27,8 @@ #include #include #include -#include #include +#include #include #include #include From ed7a959ae19a785dd4ce38e81a77fd8b3888604a Mon Sep 17 00:00:00 2001 From: Jade Lovelace Date: Thu, 30 May 2024 12:50:13 -0700 Subject: [PATCH 401/419] update lix pin to actually work --- flake.lock | 11 +++++------ flake.nix | 2 +- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/flake.lock b/flake.lock index 4d9f0466c..e9e6be0dc 100644 --- a/flake.lock +++ b/flake.lock @@ -46,16 +46,15 @@ "pre-commit-hooks": "pre-commit-hooks" }, "locked": { - "lastModified": 1714955862, - "narHash": "sha256-REWlo2RYHfJkxnmZTEJu3Cd/2VM+wjjpPy7Xi4BdDTQ=", - "ref": "refs/tags/2.90-beta.1", - "rev": "b6799ab0374a8e1907a48915d3187e07da41d88c", - "revCount": 15501, + "lastModified": 1717081103, + "narHash": "sha256-4hrY8lIK6boX0xe6LN+OFpsmOAITl0Iam17FC8Kjslk=", + "ref": "refs/heads/main", + "rev": "c161687b5fa6e7604e99ee5df2e73388952baafb", + "revCount": 15698, "type": "git", "url": "https://git@git.lix.systems/lix-project/lix" }, "original": { - "ref": "refs/tags/2.90-beta.1", "type": "git", "url": "https://git@git.lix.systems/lix-project/lix" } diff --git a/flake.nix b/flake.nix index f764e666b..630163c16 100644 --- a/flake.nix +++ b/flake.nix @@ -9,7 +9,7 @@ inputs.nix-github-actions.url = "github:nix-community/nix-github-actions"; inputs.nix-github-actions.inputs.nixpkgs.follows = "nixpkgs"; inputs.lix = { - url = "git+https://git@git.lix.systems/lix-project/lix?ref=refs/tags/2.90-beta.1"; + url = "git+https://git@git.lix.systems/lix-project/lix"; inputs.nixpkgs.follows = "nixpkgs"; }; From 040db2fe26f6c1b2c6268be994a136a9bbf5dbde Mon Sep 17 00:00:00 2001 From: Puck Meerburg Date: Wed, 12 Jun 2024 22:39:29 +0000 Subject: [PATCH 402/419] flake.nix: nixpkgs-unstable -> nixos-23.11-unstable; flake update This makes nix-eval-jobs build again. --- flake.lock | 51 ++++++++++++++++++++++++++++++++++----------------- flake.nix | 2 +- 2 files changed, 35 insertions(+), 18 deletions(-) diff --git a/flake.lock b/flake.lock index e9e6be0dc..d5f09956b 100644 --- a/flake.lock +++ b/flake.lock @@ -23,11 +23,11 @@ ] }, "locked": { - "lastModified": 1701473968, - "narHash": "sha256-YcVE5emp1qQ8ieHUnxt1wCZCC3ZfAS+SRRWZ2TMda7E=", + "lastModified": 1717285511, + "narHash": "sha256-iKzJcpdXih14qYVcZ9QC9XuZYnPc6T8YImb6dX166kw=", "owner": "hercules-ci", "repo": "flake-parts", - "rev": "34fed993f1674c8d06d58b37ce1e0fe5eebcb9f5", + "rev": "2a55567fcf15b1b1c7ed712a2c6fadaec7412ea8", "type": "github" }, "original": { @@ -39,6 +39,7 @@ "lix": { "inputs": { "flake-compat": "flake-compat", + "nix2container": "nix2container", "nixpkgs": [ "nixpkgs" ], @@ -46,11 +47,11 @@ "pre-commit-hooks": "pre-commit-hooks" }, "locked": { - "lastModified": 1717081103, - "narHash": "sha256-4hrY8lIK6boX0xe6LN+OFpsmOAITl0Iam17FC8Kjslk=", + "lastModified": 1718228457, + "narHash": "sha256-vGumESUGu/jo2Lm5bha/xBsJKVlb1wuclXlL9xudRp4=", "ref": "refs/heads/main", - "rev": "c161687b5fa6e7604e99ee5df2e73388952baafb", - "revCount": 15698, + "rev": "f46194faa2fc9c78250702c8eb7a4b756e0bd944", + "revCount": 15757, "type": "git", "url": "https://git@git.lix.systems/lix-project/lix" }, @@ -66,11 +67,11 @@ ] }, "locked": { - "lastModified": 1701208414, - "narHash": "sha256-xrQ0FyhwTZK6BwKhahIkUVZhMNk21IEI1nUcWSONtpo=", + "lastModified": 1703863825, + "narHash": "sha256-rXwqjtwiGKJheXB43ybM8NwWB8rO2dSRrEqes0S7F5Y=", "owner": "nix-community", "repo": "nix-github-actions", - "rev": "93e39cc1a087d65bcf7a132e75a650c44dd2b734", + "rev": "5163432afc817cf8bd1f031418d1869e4c9d5547", "type": "github" }, "original": { @@ -79,18 +80,34 @@ "type": "github" } }, + "nix2container": { + "flake": false, + "locked": { + "lastModified": 1712990762, + "narHash": "sha256-hO9W3w7NcnYeX8u8cleHiSpK2YJo7ecarFTUlbybl7k=", + "owner": "nlewo", + "repo": "nix2container", + "rev": "20aad300c925639d5d6cbe30013c8357ce9f2a2e", + "type": "github" + }, + "original": { + "owner": "nlewo", + "repo": "nix2container", + "type": "github" + } + }, "nixpkgs": { "locked": { - "lastModified": 1703134684, - "narHash": "sha256-SQmng1EnBFLzS7WSRyPM9HgmZP2kLJcPAz+Ug/nug6o=", + "lastModified": 1718132686, + "narHash": "sha256-JRinkq+FeAkYnrrK8+Bh+jtLHJBN5jDzSimk1ye00EE=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "d6863cbcbbb80e71cecfc03356db1cda38919523", + "rev": "96b3dae4f8753c1f5ce0d06b57fe250fb5d9b0e0", "type": "github" }, "original": { "owner": "NixOS", - "ref": "nixpkgs-unstable", + "ref": "nixos-23.11-small", "repo": "nixpkgs", "type": "github" } @@ -143,11 +160,11 @@ ] }, "locked": { - "lastModified": 1702979157, - "narHash": "sha256-RnFBbLbpqtn4AoJGXKevQMCGhra4h6G2MPcuTSZZQ+g=", + "lastModified": 1718139168, + "narHash": "sha256-1TZQcdETNdJMcfwwoshVeCjwWfrPtkSQ8y8wFX3it7k=", "owner": "numtide", "repo": "treefmt-nix", - "rev": "2961375283668d867e64129c22af532de8e77734", + "rev": "1cb529bffa880746a1d0ec4e0f5076876af931f1", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index b91a1c7f8..e3a589b58 100644 --- a/flake.nix +++ b/flake.nix @@ -1,7 +1,7 @@ { description = "Hydra's builtin hydra-eval-jobs as a standalone"; - inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable"; + inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.11-small"; inputs.flake-parts.url = "github:hercules-ci/flake-parts"; inputs.flake-parts.inputs.nixpkgs-lib.follows = "nixpkgs"; inputs.treefmt-nix.url = "github:numtide/treefmt-nix"; From 11d467fecdc325b8d8ff75ce4c9b0ef1f0a5e011 Mon Sep 17 00:00:00 2001 From: Puck Meerburg Date: Sun, 9 Jun 2024 13:20:20 +0000 Subject: [PATCH 403/419] Use our own Thread struct instead of std::thread We'd highly prefer using std::thread here; but this won't let us configure the stack size. macOS uses 512KiB size stacks for non-main threads, and musl defaults to 128k. While Nix configures a 64MiB size for the main thread, this doesn't propagate to the threads we launch here. It turns out, running the evaluator under an anemic stack of 0.5MiB has it overflow way too quickly. Hence, we have our own custom Thread struct. --- src/nix-eval-jobs.cc | 54 +++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 51 insertions(+), 3 deletions(-) diff --git a/src/nix-eval-jobs.cc b/src/nix-eval-jobs.cc index e2b204bfc..ca935af95 100644 --- a/src/nix-eval-jobs.cc +++ b/src/nix-eval-jobs.cc @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -24,7 +25,6 @@ #include #include #include -#include #include #include #include @@ -99,6 +99,54 @@ struct Proc { ~Proc() {} }; +// We'd highly prefer using std::thread here; but this won't let us configure the stack +// size. macOS uses 512KiB size stacks for non-main threads, and musl defaults to 128k. +// While Nix configures a 64MiB size for the main thread, this doesn't propagate to the +// threads we launch here. It turns out, running the evaluator under an anemic stack of +// 0.5MiB has it overflow way too quickly. Hence, we have our own custom Thread struct. +struct Thread { + pthread_t thread; + + Thread(const Thread &) = delete; + Thread(Thread &&) noexcept = default; + + Thread(std::function f) { + int s; + pthread_attr_t attr; + + auto func = std::make_unique>(std::move(f)); + + if ((s = pthread_attr_init(&attr)) != 0) { + throw SysError(s, "calling pthread_attr_init"); + } + if ((s = pthread_attr_setstacksize(&attr, 64 * 1024 * 1024)) != 0) { + throw SysError(s, "calling pthread_attr_setstacksize"); + } + if ((s = pthread_create(&thread, &attr, Thread::init, func.release())) != 0) { + throw SysError(s, "calling pthread_launch"); + } + if ((s = pthread_attr_destroy(&attr)) != 0) { + throw SysError(s, "calling pthread_attr_destroy"); + } + } + + void join() { + int s; + s = pthread_join(thread, nullptr); + if (s != 0) { + throw SysError(s, "calling pthread_join"); + } + } +private: + static void *init(void *ptr) { + std::unique_ptr> func; + func.reset(static_cast *>(ptr)); + + (*func)(); + return 0; + } +}; + struct State { std::set todo = json::array({json::array()}); std::set active; @@ -332,10 +380,10 @@ int main(int argc, char **argv) { Sync state_; /* Start a collector thread per worker process. */ - std::vector threads; + std::vector threads; std::condition_variable wakeup; for (size_t i = 0; i < myArgs.nrWorkers; i++) { - threads.emplace_back(collector, std::ref(state_), std::ref(wakeup)); + threads.emplace_back(std::bind(collector, std::ref(state_), std::ref(wakeup))); } for (auto &thread : threads) From 9c23772cf25e0d891bef70b7bcb7df36239672a5 Mon Sep 17 00:00:00 2001 From: Jade Lovelace Date: Tue, 18 Jun 2024 22:29:20 -0700 Subject: [PATCH 404/419] Tidy and make it work on release-2.90 --- flake.lock | 24 +++++++++++------------- flake.nix | 4 ++-- meson.build | 5 +++++ src/meson.build | 8 ++++---- 4 files changed, 22 insertions(+), 19 deletions(-) diff --git a/flake.lock b/flake.lock index d5f09956b..2270df41a 100644 --- a/flake.lock +++ b/flake.lock @@ -47,17 +47,15 @@ "pre-commit-hooks": "pre-commit-hooks" }, "locked": { - "lastModified": 1718228457, - "narHash": "sha256-vGumESUGu/jo2Lm5bha/xBsJKVlb1wuclXlL9xudRp4=", - "ref": "refs/heads/main", - "rev": "f46194faa2fc9c78250702c8eb7a4b756e0bd944", - "revCount": 15757, - "type": "git", - "url": "https://git@git.lix.systems/lix-project/lix" + "lastModified": 1718590005, + "narHash": "sha256-fiWc1ZyMlTXXSjcmoEQ+NHhIgtcImPHszbOu5c515cU=", + "rev": "98d0249d5c7f5dcc1d2436c4829f073fca668f80", + "type": "tarball", + "url": "https://git.lix.systems/api/v1/repos/lix-project/lix/archive/98d0249d5c7f5dcc1d2436c4829f073fca668f80.tar.gz" }, "original": { - "type": "git", - "url": "https://git@git.lix.systems/lix-project/lix" + "type": "tarball", + "url": "https://git.lix.systems/lix-project/lix/archive/release-2.90.tar.gz" } }, "nix-github-actions": { @@ -98,16 +96,16 @@ }, "nixpkgs": { "locked": { - "lastModified": 1718132686, - "narHash": "sha256-JRinkq+FeAkYnrrK8+Bh+jtLHJBN5jDzSimk1ye00EE=", + "lastModified": 1718676691, + "narHash": "sha256-DWKbARWtRpT1yiLLm+5vMijx65YB7NwGZMV6NXOtrJ8=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "96b3dae4f8753c1f5ce0d06b57fe250fb5d9b0e0", + "rev": "d9e18354acbf59c625505b7315c85508e9831bf4", "type": "github" }, "original": { "owner": "NixOS", - "ref": "nixos-23.11-small", + "ref": "nixos-24.05-small", "repo": "nixpkgs", "type": "github" } diff --git a/flake.nix b/flake.nix index e3a589b58..5fdcfb445 100644 --- a/flake.nix +++ b/flake.nix @@ -1,7 +1,7 @@ { description = "Hydra's builtin hydra-eval-jobs as a standalone"; - inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.11-small"; + inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-24.05-small"; inputs.flake-parts.url = "github:hercules-ci/flake-parts"; inputs.flake-parts.inputs.nixpkgs-lib.follows = "nixpkgs"; inputs.treefmt-nix.url = "github:numtide/treefmt-nix"; @@ -9,7 +9,7 @@ inputs.nix-github-actions.url = "github:nix-community/nix-github-actions"; inputs.nix-github-actions.inputs.nixpkgs.follows = "nixpkgs"; inputs.lix = { - url = "git+https://git@git.lix.systems/lix-project/lix"; + url = "https://git.lix.systems/lix-project/lix/archive/release-2.90.tar.gz"; inputs.nixpkgs.follows = "nixpkgs"; }; diff --git a/meson.build b/meson.build index 60dfef734..15708a675 100644 --- a/meson.build +++ b/meson.build @@ -1,6 +1,11 @@ project('nix-eval-jobs', 'cpp', version : '0.1.6', license : 'GPL-3.0', + default_options : [ + 'debug=true', + 'optimization=2', + 'cpp_std=c++20', + ], ) nix_main_dep = dependency('lix-main', required: true) diff --git a/src/meson.build b/src/meson.build index 745e0365e..4ece5923b 100644 --- a/src/meson.build +++ b/src/meson.build @@ -1,10 +1,10 @@ -src = [ +src = files( 'nix-eval-jobs.cc', 'eval-args.cc', 'drv.cc', 'buffered-io.cc', - 'worker.cc' -] + 'worker.cc', +) cc = meson.get_compiler('cpp') @@ -31,4 +31,4 @@ executable('nix-eval-jobs', src, threads_dep ], install: true, - cpp_args: ['-std=c++2a', '--include', 'autotools-config.h']) + cpp_args: ['--include', 'autotools-config.h']) From fd86a1a0687d27e8d65ca6d06912f8f97ae9dc52 Mon Sep 17 00:00:00 2001 From: Jade Lovelace Date: Tue, 18 Jun 2024 23:08:04 -0700 Subject: [PATCH 405/419] Fix on latest lix Fixes: https://git.lix.systems/lix-project/nix-eval-jobs/issues/9 --- flake.lock | 10 +++++----- flake.nix | 2 +- shell.nix | 1 + src/worker.cc | 5 +---- 4 files changed, 8 insertions(+), 10 deletions(-) diff --git a/flake.lock b/flake.lock index 2270df41a..b864e068b 100644 --- a/flake.lock +++ b/flake.lock @@ -47,15 +47,15 @@ "pre-commit-hooks": "pre-commit-hooks" }, "locked": { - "lastModified": 1718590005, - "narHash": "sha256-fiWc1ZyMlTXXSjcmoEQ+NHhIgtcImPHszbOu5c515cU=", - "rev": "98d0249d5c7f5dcc1d2436c4829f073fca668f80", + "lastModified": 1718767907, + "narHash": "sha256-gpd+mGQxqVHw2kO6rSPQel8TkChHh9UpqxjsmQi0QJM=", + "rev": "85f282ef572577899b3d80ba8def1b920a386218", "type": "tarball", - "url": "https://git.lix.systems/api/v1/repos/lix-project/lix/archive/98d0249d5c7f5dcc1d2436c4829f073fca668f80.tar.gz" + "url": "https://git.lix.systems/api/v1/repos/lix-project/lix/archive/85f282ef572577899b3d80ba8def1b920a386218.tar.gz" }, "original": { "type": "tarball", - "url": "https://git.lix.systems/lix-project/lix/archive/release-2.90.tar.gz" + "url": "https://git.lix.systems/lix-project/lix/archive/main.tar.gz" } }, "nix-github-actions": { diff --git a/flake.nix b/flake.nix index 5fdcfb445..a2cc4798d 100644 --- a/flake.nix +++ b/flake.nix @@ -9,7 +9,7 @@ inputs.nix-github-actions.url = "github:nix-community/nix-github-actions"; inputs.nix-github-actions.inputs.nixpkgs.follows = "nixpkgs"; inputs.lix = { - url = "https://git.lix.systems/lix-project/lix/archive/release-2.90.tar.gz"; + url = "https://git.lix.systems/lix-project/lix/archive/main.tar.gz"; inputs.nixpkgs.follows = "nixpkgs"; }; diff --git a/shell.nix b/shell.nix index 13a5d3430..7035ed9e1 100644 --- a/shell.nix +++ b/shell.nix @@ -23,6 +23,7 @@ in pkgs.mkShell { inherit (nix-eval-jobs) buildInputs; nativeBuildInputs = nix-eval-jobs.nativeBuildInputs ++ [ + pkgs.clang-tools (pkgs.python3.withPackages (ps: [ ps.pytest ])) diff --git a/src/worker.cc b/src/worker.cc index 817d81fa4..5e5ff9fad 100644 --- a/src/worker.cc +++ b/src/worker.cc @@ -30,15 +30,12 @@ #include #include #include -#include -#include #include #include #include #include #include #include -#include #include "worker.hh" #include "drv.hh" @@ -51,7 +48,7 @@ static nix::Value *releaseExprTopLevelValue(nix::EvalState &state, nix::Value vTop; if (args.fromArgs) { - nix::Expr *e = state.parseExprFromString( + nix::Expr &e = state.parseExprFromString( args.releaseExpr, state.rootPath(nix::CanonPath::fromCwd())); state.eval(e, vTop); } else { From f8869bdcca7c1d5aaf37de3da3a4176811279a57 Mon Sep 17 00:00:00 2001 From: eldritch horrors Date: Tue, 25 Jun 2024 23:57:51 +0200 Subject: [PATCH 406/419] update for lix 2.91-dev the api of nix::Pid changed, causing a build failure. --- flake.lock | 8 ++++---- src/nix-eval-jobs.cc | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/flake.lock b/flake.lock index b864e068b..a5f6e775f 100644 --- a/flake.lock +++ b/flake.lock @@ -47,11 +47,11 @@ "pre-commit-hooks": "pre-commit-hooks" }, "locked": { - "lastModified": 1718767907, - "narHash": "sha256-gpd+mGQxqVHw2kO6rSPQel8TkChHh9UpqxjsmQi0QJM=", - "rev": "85f282ef572577899b3d80ba8def1b920a386218", + "lastModified": 1719348166, + "narHash": "sha256-GK6PusfbMgkg+qdgChmrw78KTNQkm7SDoJ6+lJKY6vg=", + "rev": "f170870ae7b18c8ee13fb42fe19b5aa05ddf56c0", "type": "tarball", - "url": "https://git.lix.systems/api/v1/repos/lix-project/lix/archive/85f282ef572577899b3d80ba8def1b920a386218.tar.gz" + "url": "https://git.lix.systems/api/v1/repos/lix-project/lix/archive/f170870ae7b18c8ee13fb42fe19b5aa05ddf56c0.tar.gz" }, "original": { "type": "tarball", diff --git a/src/nix-eval-jobs.cc b/src/nix-eval-jobs.cc index ca935af95..ea45a41d5 100644 --- a/src/nix-eval-jobs.cc +++ b/src/nix-eval-jobs.cc @@ -93,7 +93,7 @@ struct Proc { to = std::move(toPipe.writeSide); from = std::move(fromPipe.readSide); - pid = p; + pid = std::move(p); } ~Proc() {} From d9a46559a4a4de199c68c9b369c50ca5664ce564 Mon Sep 17 00:00:00 2001 From: Pierre Bourdon Date: Tue, 16 Jul 2024 09:11:01 +0200 Subject: [PATCH 407/419] drv: backport CA derivations support changes from hydra-eval-jobs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It is not possible to query output paths for CA derivations since they're not static / known at eval time. Instead, return JSON nulls for outputs paths. This is a partial port of the following Hydra commits: - 9ba4417940ffdd0fadea43f68c61ef948a4b8d39 - 069b7775c565f5999fe33e8c3f28c7b9306039ca - fcde5908d8e51f975b883329b34d24a9f30ea4b3 By the following authors: Co-Authored-By: John Ericson Co-Authored-By: Théophane Hufschmitt Co-Authored-By: Alexander Sosedkin Co-Authored-By: Andrea Ciceri Co-Authored-By: Charlotte 🦝 Delenk Mlotte@chir.rs> Co-Authored-By: Sandro Jäckel --- src/drv.cc | 33 ++++++++++++++++++++++++++------- src/drv.hh | 2 +- 2 files changed, 27 insertions(+), 8 deletions(-) diff --git a/src/drv.cc b/src/drv.cc index 5af2dfc57..dca58de9d 100644 --- a/src/drv.cc +++ b/src/drv.cc @@ -23,14 +23,17 @@ #include "drv.hh" #include "eval-args.hh" -static bool queryIsCached(nix::Store &store, - std::map &outputs) { +static bool +queryIsCached(nix::Store &store, + std::map> &outputs) { uint64_t downloadSize, narSize; nix::StorePathSet willBuild, willSubstitute, unknown; std::vector paths; for (auto const &[key, val] : outputs) { - paths.push_back(followLinksToStorePathWithOutputs(store, val)); + if (val) { + paths.push_back(followLinksToStorePathWithOutputs(store, *val)); + } } store.queryMissing(toDerivedPaths(paths), willBuild, willSubstitute, @@ -45,9 +48,19 @@ Drv::Drv(std::string &attrPath, nix::EvalState &state, nix::DrvInfo &drvInfo, auto localStore = state.store.dynamic_pointer_cast(); try { - for (auto out : drvInfo.queryOutputs(true)) { - if (out.second) - outputs[out.first] = localStore->printStorePath(*out.second); + // CA derivations do not have static output paths, so we have to + // defensively not query output paths in case we encounter one. + for (auto &[outputName, optOutputPath] : + drvInfo.queryOutputs(!nix::experimentalFeatureSettings.isEnabled( + nix::Xp::CaDerivations))) { + if (optOutputPath) { + outputs[outputName] = + localStore->printStorePath(*optOutputPath); + } else { + assert(nix::experimentalFeatureSettings.isEnabled( + nix::Xp::CaDerivations)); + outputs[outputName] = std::nullopt; + } } } catch (const std::exception &e) { throw nix::EvalError(state, @@ -98,10 +111,16 @@ Drv::Drv(std::string &attrPath, nix::EvalState &state, nix::DrvInfo &drvInfo, } void to_json(nlohmann::json &json, const Drv &drv) { + std::map outputsJson; + for (auto &[name, optPath] : drv.outputs) { + outputsJson[name] = + optPath ? nlohmann::json(*optPath) : nlohmann::json(nullptr); + } + json = nlohmann::json{{"name", drv.name}, {"system", drv.system}, {"drvPath", drv.drvPath}, - {"outputs", drv.outputs}, + {"outputs", outputsJson}, {"inputDrvs", drv.inputDrvs}}; if (drv.meta.has_value()) { diff --git a/src/drv.hh b/src/drv.hh index ab817c27e..4cfc6a0bb 100644 --- a/src/drv.hh +++ b/src/drv.hh @@ -24,7 +24,7 @@ struct Drv { std::string drvPath; enum class CacheStatus { Cached, Uncached, Unknown } cacheStatus; - std::map outputs; + std::map> outputs; std::map> inputDrvs; std::optional meta; From c057494450f2d1420726ddb0bab145a5ff4ddfdd Mon Sep 17 00:00:00 2001 From: Pierre Bourdon Date: Wed, 17 Jul 2024 07:57:52 +0200 Subject: [PATCH 408/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'flake-parts': 'github:hercules-ci/flake-parts/2a55567fcf15b1b1c7ed712a2c6fadaec7412ea8' (2024-06-01) → 'github:hercules-ci/flake-parts/9227223f6d922fee3c7b190b2cc238a99527bbb7' (2024-07-03) • Updated input 'lix': 'https://git.lix.systems/api/v1/repos/lix-project/lix/archive/f170870ae7b18c8ee13fb42fe19b5aa05ddf56c0.tar.gz?narHash=sha256-GK6PusfbMgkg%2BqdgChmrw78KTNQkm7SDoJ6%2BlJKY6vg%3D' (2024-06-25) → 'https://git.lix.systems/api/v1/repos/lix-project/lix/archive/ef0de7c79f3b32f66db447220d26eae7e7c07b19.tar.gz?narHash=sha256-GUH5%2BB1JztzDNSN1D7KbndrYSq0LWvVIJnuWKHlpN3Q%3D' (2024-07-16) • Updated input 'nix-github-actions': 'github:nix-community/nix-github-actions/5163432afc817cf8bd1f031418d1869e4c9d5547' (2023-12-29) → 'github:nix-community/nix-github-actions/622f829f5fe69310a866c8a6cd07e747c44ef820' (2024-07-04) • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/d9e18354acbf59c625505b7315c85508e9831bf4' (2024-06-18) → 'github:NixOS/nixpkgs/732b4f3a3afdfe6a6c4fcb2511e529588d4e5ccd' (2024-07-15) • Updated input 'treefmt-nix': 'github:numtide/treefmt-nix/1cb529bffa880746a1d0ec4e0f5076876af931f1' (2024-06-11) → 'github:numtide/treefmt-nix/0fb28f237f83295b4dd05e342f333b447c097398' (2024-07-15) --- flake.lock | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/flake.lock b/flake.lock index a5f6e775f..fa32b775c 100644 --- a/flake.lock +++ b/flake.lock @@ -23,11 +23,11 @@ ] }, "locked": { - "lastModified": 1717285511, - "narHash": "sha256-iKzJcpdXih14qYVcZ9QC9XuZYnPc6T8YImb6dX166kw=", + "lastModified": 1719994518, + "narHash": "sha256-pQMhCCHyQGRzdfAkdJ4cIWiw+JNuWsTX7f0ZYSyz0VY=", "owner": "hercules-ci", "repo": "flake-parts", - "rev": "2a55567fcf15b1b1c7ed712a2c6fadaec7412ea8", + "rev": "9227223f6d922fee3c7b190b2cc238a99527bbb7", "type": "github" }, "original": { @@ -47,11 +47,11 @@ "pre-commit-hooks": "pre-commit-hooks" }, "locked": { - "lastModified": 1719348166, - "narHash": "sha256-GK6PusfbMgkg+qdgChmrw78KTNQkm7SDoJ6+lJKY6vg=", - "rev": "f170870ae7b18c8ee13fb42fe19b5aa05ddf56c0", + "lastModified": 1721094616, + "narHash": "sha256-GUH5+B1JztzDNSN1D7KbndrYSq0LWvVIJnuWKHlpN3Q=", + "rev": "ef0de7c79f3b32f66db447220d26eae7e7c07b19", "type": "tarball", - "url": "https://git.lix.systems/api/v1/repos/lix-project/lix/archive/f170870ae7b18c8ee13fb42fe19b5aa05ddf56c0.tar.gz" + "url": "https://git.lix.systems/api/v1/repos/lix-project/lix/archive/ef0de7c79f3b32f66db447220d26eae7e7c07b19.tar.gz" }, "original": { "type": "tarball", @@ -65,11 +65,11 @@ ] }, "locked": { - "lastModified": 1703863825, - "narHash": "sha256-rXwqjtwiGKJheXB43ybM8NwWB8rO2dSRrEqes0S7F5Y=", + "lastModified": 1720066371, + "narHash": "sha256-uPlLYH2S0ACj0IcgaK9Lsf4spmJoGejR9DotXiXSBZQ=", "owner": "nix-community", "repo": "nix-github-actions", - "rev": "5163432afc817cf8bd1f031418d1869e4c9d5547", + "rev": "622f829f5fe69310a866c8a6cd07e747c44ef820", "type": "github" }, "original": { @@ -96,11 +96,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1718676691, - "narHash": "sha256-DWKbARWtRpT1yiLLm+5vMijx65YB7NwGZMV6NXOtrJ8=", + "lastModified": 1721079475, + "narHash": "sha256-wZ62hFCMTUG68u3hSUSJOCP/ltuE32Yb4dy7FfPCpso=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "d9e18354acbf59c625505b7315c85508e9831bf4", + "rev": "732b4f3a3afdfe6a6c4fcb2511e529588d4e5ccd", "type": "github" }, "original": { @@ -158,11 +158,11 @@ ] }, "locked": { - "lastModified": 1718139168, - "narHash": "sha256-1TZQcdETNdJMcfwwoshVeCjwWfrPtkSQ8y8wFX3it7k=", + "lastModified": 1721059077, + "narHash": "sha256-gCICMMX7VMSKKt99giDDtRLkHJ0cwSgBtDijJAqTlto=", "owner": "numtide", "repo": "treefmt-nix", - "rev": "1cb529bffa880746a1d0ec4e0f5076876af931f1", + "rev": "0fb28f237f83295b4dd05e342f333b447c097398", "type": "github" }, "original": { From 42a160bce2fd9ffebc3809746bc80cc7208f9b08 Mon Sep 17 00:00:00 2001 From: Yureka Date: Tue, 13 Aug 2024 22:00:51 +0200 Subject: [PATCH 409/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'flake-parts': 'github:hercules-ci/flake-parts/9227223f6d922fee3c7b190b2cc238a99527bbb7' (2024-07-03) → 'github:hercules-ci/flake-parts/8471fe90ad337a8074e957b69ca4d0089218391d' (2024-08-01) • Updated input 'lix': 'https://git.lix.systems/api/v1/repos/lix-project/lix/archive/ef0de7c79f3b32f66db447220d26eae7e7c07b19.tar.gz?narHash=sha256-GUH5%2BB1JztzDNSN1D7KbndrYSq0LWvVIJnuWKHlpN3Q%3D' (2024-07-16) → 'https://git.lix.systems/api/v1/repos/lix-project/lix/archive/b016eb0895bb6714a4f6530d9a2bb6577ac6c3cf.tar.gz?narHash=sha256-kOpGI9WPmte1L4QWHviuXsr8jxmGn27zwi82jtzYObM%3D&rev=b016eb0895bb6714a4f6530d9a2bb6577ac6c3cf' (2024-08-13) • Updated input 'lix/nix2container': 'github:nlewo/nix2container/20aad300c925639d5d6cbe30013c8357ce9f2a2e' (2024-04-13) → 'github:nlewo/nix2container/3853e5caf9ad24103b13aa6e0e8bcebb47649fe4' (2024-07-10) • Updated input 'lix/pre-commit-hooks': 'github:cachix/git-hooks.nix/e35aed5fda3cc79f88ed7f1795021e559582093a' (2024-04-02) → 'github:cachix/git-hooks.nix/f451c19376071a90d8c58ab1a953c6e9840527fd' (2024-07-15) • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/732b4f3a3afdfe6a6c4fcb2511e529588d4e5ccd' (2024-07-15) → 'github:NixOS/nixpkgs/fb81cec9eda2a6b5365ad723995f0329d9e356fd' (2024-08-13) • Updated input 'treefmt-nix': 'github:numtide/treefmt-nix/0fb28f237f83295b4dd05e342f333b447c097398' (2024-07-15) → 'github:numtide/treefmt-nix/349de7bc435bdff37785c2466f054ed1766173be' (2024-08-12) --- flake.lock | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/flake.lock b/flake.lock index fa32b775c..3efca8906 100644 --- a/flake.lock +++ b/flake.lock @@ -23,11 +23,11 @@ ] }, "locked": { - "lastModified": 1719994518, - "narHash": "sha256-pQMhCCHyQGRzdfAkdJ4cIWiw+JNuWsTX7f0ZYSyz0VY=", + "lastModified": 1722555600, + "narHash": "sha256-XOQkdLafnb/p9ij77byFQjDf5m5QYl9b2REiVClC+x4=", "owner": "hercules-ci", "repo": "flake-parts", - "rev": "9227223f6d922fee3c7b190b2cc238a99527bbb7", + "rev": "8471fe90ad337a8074e957b69ca4d0089218391d", "type": "github" }, "original": { @@ -47,11 +47,11 @@ "pre-commit-hooks": "pre-commit-hooks" }, "locked": { - "lastModified": 1721094616, - "narHash": "sha256-GUH5+B1JztzDNSN1D7KbndrYSq0LWvVIJnuWKHlpN3Q=", - "rev": "ef0de7c79f3b32f66db447220d26eae7e7c07b19", + "lastModified": 1723577950, + "narHash": "sha256-kOpGI9WPmte1L4QWHviuXsr8jxmGn27zwi82jtzYObM=", + "rev": "b016eb0895bb6714a4f6530d9a2bb6577ac6c3cf", "type": "tarball", - "url": "https://git.lix.systems/api/v1/repos/lix-project/lix/archive/ef0de7c79f3b32f66db447220d26eae7e7c07b19.tar.gz" + "url": "https://git.lix.systems/api/v1/repos/lix-project/lix/archive/b016eb0895bb6714a4f6530d9a2bb6577ac6c3cf.tar.gz?rev=b016eb0895bb6714a4f6530d9a2bb6577ac6c3cf" }, "original": { "type": "tarball", @@ -81,11 +81,11 @@ "nix2container": { "flake": false, "locked": { - "lastModified": 1712990762, - "narHash": "sha256-hO9W3w7NcnYeX8u8cleHiSpK2YJo7ecarFTUlbybl7k=", + "lastModified": 1720642556, + "narHash": "sha256-qsnqk13UmREKmRT7c8hEnz26X3GFFyIQrqx4EaRc1Is=", "owner": "nlewo", "repo": "nix2container", - "rev": "20aad300c925639d5d6cbe30013c8357ce9f2a2e", + "rev": "3853e5caf9ad24103b13aa6e0e8bcebb47649fe4", "type": "github" }, "original": { @@ -96,11 +96,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1721079475, - "narHash": "sha256-wZ62hFCMTUG68u3hSUSJOCP/ltuE32Yb4dy7FfPCpso=", + "lastModified": 1723540975, + "narHash": "sha256-rxpxOz2VSqgmwI7g7FGVAoye5bxwO1MSpnELY5bsITw=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "732b4f3a3afdfe6a6c4fcb2511e529588d4e5ccd", + "rev": "fb81cec9eda2a6b5365ad723995f0329d9e356fd", "type": "github" }, "original": { @@ -129,11 +129,11 @@ "pre-commit-hooks": { "flake": false, "locked": { - "lastModified": 1712055707, - "narHash": "sha256-4XLvuSIDZJGS17xEwSrNuJLL7UjDYKGJSbK1WWX2AK8=", + "lastModified": 1721042469, + "narHash": "sha256-6FPUl7HVtvRHCCBQne7Ylp4p+dpP3P/OYuzjztZ4s70=", "owner": "cachix", "repo": "git-hooks.nix", - "rev": "e35aed5fda3cc79f88ed7f1795021e559582093a", + "rev": "f451c19376071a90d8c58ab1a953c6e9840527fd", "type": "github" }, "original": { @@ -158,11 +158,11 @@ ] }, "locked": { - "lastModified": 1721059077, - "narHash": "sha256-gCICMMX7VMSKKt99giDDtRLkHJ0cwSgBtDijJAqTlto=", + "lastModified": 1723454642, + "narHash": "sha256-S0Gvsenh0II7EAaoc9158ZB4vYyuycvMGKGxIbERNAM=", "owner": "numtide", "repo": "treefmt-nix", - "rev": "0fb28f237f83295b4dd05e342f333b447c097398", + "rev": "349de7bc435bdff37785c2466f054ed1766173be", "type": "github" }, "original": { From 50a1455953eff57a5cda2e9fc37d47f08c983c57 Mon Sep 17 00:00:00 2001 From: Qyriad Date: Fri, 25 Oct 2024 12:48:08 -0600 Subject: [PATCH 410/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'flake-parts': 'github:hercules-ci/flake-parts/8471fe90ad337a8074e957b69ca4d0089218391d' (2024-08-01) → 'github:hercules-ci/flake-parts/3d04084d54bedc3d6b8b736c70ef449225c361b1' (2024-10-01) • Updated input 'lix': 'https://git.lix.systems/api/v1/repos/lix-project/lix/archive/b016eb0895bb6714a4f6530d9a2bb6577ac6c3cf.tar.gz?narHash=sha256-kOpGI9WPmte1L4QWHviuXsr8jxmGn27zwi82jtzYObM%3D&rev=b016eb0895bb6714a4f6530d9a2bb6577ac6c3cf' (2024-08-13) → 'https://git.lix.systems/api/v1/repos/lix-project/lix/archive/2734a9cf94debc6baef4e7d4d9fa28cc28f5b31d.tar.gz?narHash=sha256-XME7TzBvjK6GEmZqPLK%2B2%2BWk0qnwc7DCwYH434hMcOM%3D&rev=2734a9cf94debc6baef4e7d4d9fa28cc28f5b31d' (2024-10-23) • Updated input 'lix/nix2container': 'github:nlewo/nix2container/3853e5caf9ad24103b13aa6e0e8bcebb47649fe4' (2024-07-10) → 'github:nlewo/nix2container/fa6bb0a1159f55d071ba99331355955ae30b3401' (2024-08-30) • Updated input 'lix/pre-commit-hooks': 'github:cachix/git-hooks.nix/f451c19376071a90d8c58ab1a953c6e9840527fd' (2024-07-15) → 'github:cachix/git-hooks.nix/4e743a6920eab45e8ba0fbe49dc459f1423a4b74' (2024-09-19) • Updated input 'nix-github-actions': 'github:nix-community/nix-github-actions/622f829f5fe69310a866c8a6cd07e747c44ef820' (2024-07-04) → 'github:nix-community/nix-github-actions/e04df33f62cdcf93d73e9a04142464753a16db67' (2024-10-24) • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/fb81cec9eda2a6b5365ad723995f0329d9e356fd' (2024-08-13) → 'github:NixOS/nixpkgs/45e5197248e59e92e88956c5aa12553a7f62337f' (2024-10-25) • Updated input 'treefmt-nix': 'github:numtide/treefmt-nix/349de7bc435bdff37785c2466f054ed1766173be' (2024-08-12) → 'github:numtide/treefmt-nix/aac86347fb5063960eccb19493e0cadcdb4205ca' (2024-10-22) --- flake.lock | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/flake.lock b/flake.lock index 3efca8906..d7efaad33 100644 --- a/flake.lock +++ b/flake.lock @@ -23,11 +23,11 @@ ] }, "locked": { - "lastModified": 1722555600, - "narHash": "sha256-XOQkdLafnb/p9ij77byFQjDf5m5QYl9b2REiVClC+x4=", + "lastModified": 1727826117, + "narHash": "sha256-K5ZLCyfO/Zj9mPFldf3iwS6oZStJcU4tSpiXTMYaaL0=", "owner": "hercules-ci", "repo": "flake-parts", - "rev": "8471fe90ad337a8074e957b69ca4d0089218391d", + "rev": "3d04084d54bedc3d6b8b736c70ef449225c361b1", "type": "github" }, "original": { @@ -47,11 +47,11 @@ "pre-commit-hooks": "pre-commit-hooks" }, "locked": { - "lastModified": 1723577950, - "narHash": "sha256-kOpGI9WPmte1L4QWHviuXsr8jxmGn27zwi82jtzYObM=", - "rev": "b016eb0895bb6714a4f6530d9a2bb6577ac6c3cf", + "lastModified": 1729696851, + "narHash": "sha256-XME7TzBvjK6GEmZqPLK+2+Wk0qnwc7DCwYH434hMcOM=", + "rev": "2734a9cf94debc6baef4e7d4d9fa28cc28f5b31d", "type": "tarball", - "url": "https://git.lix.systems/api/v1/repos/lix-project/lix/archive/b016eb0895bb6714a4f6530d9a2bb6577ac6c3cf.tar.gz?rev=b016eb0895bb6714a4f6530d9a2bb6577ac6c3cf" + "url": "https://git.lix.systems/api/v1/repos/lix-project/lix/archive/2734a9cf94debc6baef4e7d4d9fa28cc28f5b31d.tar.gz?rev=2734a9cf94debc6baef4e7d4d9fa28cc28f5b31d" }, "original": { "type": "tarball", @@ -65,11 +65,11 @@ ] }, "locked": { - "lastModified": 1720066371, - "narHash": "sha256-uPlLYH2S0ACj0IcgaK9Lsf4spmJoGejR9DotXiXSBZQ=", + "lastModified": 1729742964, + "narHash": "sha256-B4mzTcQ0FZHdpeWcpDYPERtyjJd/NIuaQ9+BV1h+MpA=", "owner": "nix-community", "repo": "nix-github-actions", - "rev": "622f829f5fe69310a866c8a6cd07e747c44ef820", + "rev": "e04df33f62cdcf93d73e9a04142464753a16db67", "type": "github" }, "original": { @@ -81,11 +81,11 @@ "nix2container": { "flake": false, "locked": { - "lastModified": 1720642556, - "narHash": "sha256-qsnqk13UmREKmRT7c8hEnz26X3GFFyIQrqx4EaRc1Is=", + "lastModified": 1724996935, + "narHash": "sha256-njRK9vvZ1JJsP8oV2OgkBrpJhgQezI03S7gzskCcHos=", "owner": "nlewo", "repo": "nix2container", - "rev": "3853e5caf9ad24103b13aa6e0e8bcebb47649fe4", + "rev": "fa6bb0a1159f55d071ba99331355955ae30b3401", "type": "github" }, "original": { @@ -96,11 +96,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1723540975, - "narHash": "sha256-rxpxOz2VSqgmwI7g7FGVAoye5bxwO1MSpnELY5bsITw=", + "lastModified": 1729851744, + "narHash": "sha256-c3ZSmkQdcdmNAzHud2b0fANXNa8TcySH0ZAqND5zEi0=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "fb81cec9eda2a6b5365ad723995f0329d9e356fd", + "rev": "45e5197248e59e92e88956c5aa12553a7f62337f", "type": "github" }, "original": { @@ -129,11 +129,11 @@ "pre-commit-hooks": { "flake": false, "locked": { - "lastModified": 1721042469, - "narHash": "sha256-6FPUl7HVtvRHCCBQne7Ylp4p+dpP3P/OYuzjztZ4s70=", + "lastModified": 1726745158, + "narHash": "sha256-D5AegvGoEjt4rkKedmxlSEmC+nNLMBPWFxvmYnVLhjk=", "owner": "cachix", "repo": "git-hooks.nix", - "rev": "f451c19376071a90d8c58ab1a953c6e9840527fd", + "rev": "4e743a6920eab45e8ba0fbe49dc459f1423a4b74", "type": "github" }, "original": { @@ -158,11 +158,11 @@ ] }, "locked": { - "lastModified": 1723454642, - "narHash": "sha256-S0Gvsenh0II7EAaoc9158ZB4vYyuycvMGKGxIbERNAM=", + "lastModified": 1729613947, + "narHash": "sha256-XGOvuIPW1XRfPgHtGYXd5MAmJzZtOuwlfKDgxX5KT3s=", "owner": "numtide", "repo": "treefmt-nix", - "rev": "349de7bc435bdff37785c2466f054ed1766173be", + "rev": "aac86347fb5063960eccb19493e0cadcdb4205ca", "type": "github" }, "original": { From 43aaa943bfdbc56277e86ff430a761d38d194ad0 Mon Sep 17 00:00:00 2001 From: Qyriad Date: Fri, 25 Oct 2024 14:29:57 -0600 Subject: [PATCH 411/419] fix build with latest Lix Lix commit 4dbbd721e[1] changed the way settings are changed, removing operator= in the process. This commit changes the places where we use operator= to using either setDefault(), or override(). I *believe* I have used the correct ones for each changed setting. Fixes #13. [1]: 4dbbd721eb9db75d4968a624b8cb9e75e979a144 --- src/nix-eval-jobs.cc | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/nix-eval-jobs.cc b/src/nix-eval-jobs.cc index ea45a41d5..1e58a4471 100644 --- a/src/nix-eval-jobs.cc +++ b/src/nix-eval-jobs.cc @@ -350,18 +350,18 @@ int main(int argc, char **argv) { /* FIXME: The build hook in conjunction with import-from-derivation is * causing "unexpected EOF" during eval */ - settings.builders = ""; + settings.builders.setDefault(""); /* Prevent access to paths outside of the Nix search path and to the environment. */ - evalSettings.restrictEval = false; + evalSettings.restrictEval.setDefault(false); /* When building a flake, use pure evaluation (no access to 'getEnv', 'currentSystem' etc. */ if (myArgs.impure) { - evalSettings.pureEval = false; + evalSettings.pureEval.setDefault(false); } else if (myArgs.flake) { - evalSettings.pureEval = true; + evalSettings.pureEval.setDefault(true); } if (myArgs.releaseExpr == "") @@ -374,7 +374,7 @@ int main(int argc, char **argv) { } if (myArgs.showTrace) { - loggerSettings.showTrace.assign(true); + loggerSettings.showTrace.override(true); } Sync state_; From 9160d40f804f642fb36e29f718b8a05cf515a67d Mon Sep 17 00:00:00 2001 From: Jade Lovelace Date: Sat, 9 Nov 2024 12:46:21 -0800 Subject: [PATCH 412/419] Address feedback from alois31 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Remove restrict-eval stuff that did nothing - Remove builders stuff that appears unnecessary: /* FIXME: The build hook in conjunction with import-from-derivation is * causing "unexpected EOF" during eval */ settings.builders.setDefault(""); We removed that line and then observed that it works, so idk: ifdtest.nix: let ifd = builtins.derivation { name = "wat2"; builder = "/bin/sh"; args = [ "-c" "echo meow > $out" ]; system = "aarch64-linux"; }; in builtins.readFile ifd » NIX_CONFIG="builders = @/etc/nix/machines" build/src/nix-eval-jobs ifdtest.nix warning: unknown setting 'trusted-users' warning: `--gc-roots-dir' not specified building '/nix/store/xxnd5rb49n3anyla5v71lgdk0wmhmijp-wat2.drv' on 'ssh-ng://root@voracle.jade.fyi'... copying 0 paths... building '/nix/store/xxnd5rb49n3anyla5v71lgdk0wmhmijp-wat2.drv'... copying 1 paths... copying path '/nix/store/h2yxq9lb7l0nd9plgqrcgf7nvsg67gl7-wat2' from 'ssh-ng://root@voracle.jade.fyi'... - Changed the impure/flake code to override the pureEval setting, which it was definitely *supposed* to be doing in the first place. --- src/nix-eval-jobs.cc | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/src/nix-eval-jobs.cc b/src/nix-eval-jobs.cc index 1e58a4471..bd7c207ba 100644 --- a/src/nix-eval-jobs.cc +++ b/src/nix-eval-jobs.cc @@ -348,20 +348,12 @@ int main(int argc, char **argv) { myArgs.parseArgs(argv, argc); - /* FIXME: The build hook in conjunction with import-from-derivation is - * causing "unexpected EOF" during eval */ - settings.builders.setDefault(""); - - /* Prevent access to paths outside of the Nix search path and - to the environment. */ - evalSettings.restrictEval.setDefault(false); - /* When building a flake, use pure evaluation (no access to 'getEnv', 'currentSystem' etc. */ if (myArgs.impure) { - evalSettings.pureEval.setDefault(false); + evalSettings.pureEval.override(false); } else if (myArgs.flake) { - evalSettings.pureEval.setDefault(true); + evalSettings.pureEval.override(true); } if (myArgs.releaseExpr == "") From b9e0abe9e6e49f50aaf394cbec2fad918594bfe6 Mon Sep 17 00:00:00 2001 From: Alois Wohlschlager Date: Sat, 16 Nov 2024 19:56:09 +0100 Subject: [PATCH 413/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'flake-parts': 'github:hercules-ci/flake-parts/3d04084d54bedc3d6b8b736c70ef449225c361b1' (2024-10-01) → 'github:hercules-ci/flake-parts/506278e768c2a08bec68eb62932193e341f55c90' (2024-11-01) • Updated input 'lix': 'https://git.lix.systems/api/v1/repos/lix-project/lix/archive/2734a9cf94debc6baef4e7d4d9fa28cc28f5b31d.tar.gz?narHash=sha256-XME7TzBvjK6GEmZqPLK%2B2%2BWk0qnwc7DCwYH434hMcOM%3D&rev=2734a9cf94debc6baef4e7d4d9fa28cc28f5b31d' (2024-10-23) → 'https://git.lix.systems/api/v1/repos/lix-project/lix/archive/c859d03013712b349d82ee6223948d6d03e63a8d.tar.gz?narHash=sha256-bq21I1EjXJa/s5Rra9J9ot2NkPCnI0F5uNPurwYLdpE%3D&rev=c859d03013712b349d82ee6223948d6d03e63a8d' (2024-11-15) • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/45e5197248e59e92e88956c5aa12553a7f62337f' (2024-10-25) → 'github:NixOS/nixpkgs/035d434d48f4375ac5d3a620954cf5fda7dd7c36' (2024-11-15) • Updated input 'treefmt-nix': 'github:numtide/treefmt-nix/aac86347fb5063960eccb19493e0cadcdb4205ca' (2024-10-22) → 'github:numtide/treefmt-nix/746901bb8dba96d154b66492a29f5db0693dbfcc' (2024-10-30) --- flake.lock | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/flake.lock b/flake.lock index d7efaad33..a0d91a967 100644 --- a/flake.lock +++ b/flake.lock @@ -23,11 +23,11 @@ ] }, "locked": { - "lastModified": 1727826117, - "narHash": "sha256-K5ZLCyfO/Zj9mPFldf3iwS6oZStJcU4tSpiXTMYaaL0=", + "lastModified": 1730504689, + "narHash": "sha256-hgmguH29K2fvs9szpq2r3pz2/8cJd2LPS+b4tfNFCwE=", "owner": "hercules-ci", "repo": "flake-parts", - "rev": "3d04084d54bedc3d6b8b736c70ef449225c361b1", + "rev": "506278e768c2a08bec68eb62932193e341f55c90", "type": "github" }, "original": { @@ -47,11 +47,11 @@ "pre-commit-hooks": "pre-commit-hooks" }, "locked": { - "lastModified": 1729696851, - "narHash": "sha256-XME7TzBvjK6GEmZqPLK+2+Wk0qnwc7DCwYH434hMcOM=", - "rev": "2734a9cf94debc6baef4e7d4d9fa28cc28f5b31d", + "lastModified": 1731683711, + "narHash": "sha256-bq21I1EjXJa/s5Rra9J9ot2NkPCnI0F5uNPurwYLdpE=", + "rev": "c859d03013712b349d82ee6223948d6d03e63a8d", "type": "tarball", - "url": "https://git.lix.systems/api/v1/repos/lix-project/lix/archive/2734a9cf94debc6baef4e7d4d9fa28cc28f5b31d.tar.gz?rev=2734a9cf94debc6baef4e7d4d9fa28cc28f5b31d" + "url": "https://git.lix.systems/api/v1/repos/lix-project/lix/archive/c859d03013712b349d82ee6223948d6d03e63a8d.tar.gz?rev=c859d03013712b349d82ee6223948d6d03e63a8d" }, "original": { "type": "tarball", @@ -96,11 +96,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1729851744, - "narHash": "sha256-c3ZSmkQdcdmNAzHud2b0fANXNa8TcySH0ZAqND5zEi0=", + "lastModified": 1731663789, + "narHash": "sha256-x07g4NcqGP6mQn6AISXJaks9sQYDjZmTMBlKIvajvyc=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "45e5197248e59e92e88956c5aa12553a7f62337f", + "rev": "035d434d48f4375ac5d3a620954cf5fda7dd7c36", "type": "github" }, "original": { @@ -158,11 +158,11 @@ ] }, "locked": { - "lastModified": 1729613947, - "narHash": "sha256-XGOvuIPW1XRfPgHtGYXd5MAmJzZtOuwlfKDgxX5KT3s=", + "lastModified": 1730321837, + "narHash": "sha256-vK+a09qq19QNu2MlLcvN4qcRctJbqWkX7ahgPZ/+maI=", "owner": "numtide", "repo": "treefmt-nix", - "rev": "aac86347fb5063960eccb19493e0cadcdb4205ca", + "rev": "746901bb8dba96d154b66492a29f5db0693dbfcc", "type": "github" }, "original": { From bbbebaf380461a81684d0ba83d794f8a4219fda3 Mon Sep 17 00:00:00 2001 From: Alois Wohlschlager Date: Sat, 16 Nov 2024 20:14:39 +0100 Subject: [PATCH 414/419] fix build with latest Lix Commit 8088927b90ff84dd37c342e1ef0a91bc8feca6ec renamed initGC to initLibExpr. Use the new name so that the build works again. --- src/nix-eval-jobs.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix-eval-jobs.cc b/src/nix-eval-jobs.cc index bd7c207ba..2b4885ee0 100644 --- a/src/nix-eval-jobs.cc +++ b/src/nix-eval-jobs.cc @@ -344,7 +344,7 @@ int main(int argc, char **argv) { return handleExceptions(argv[0], [&]() { initNix(); - initGC(); + initLibExpr(); myArgs.parseArgs(argv, argc); From dfc286ca3dc49118c30d8d6205d6d6af76c62b7a Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Sat, 23 Nov 2024 09:16:17 +0100 Subject: [PATCH 415/419] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'lix': 'https://git.lix.systems/api/v1/repos/lix-project/lix/archive/c859d03013712b349d82ee6223948d6d03e63a8d.tar.gz?narHash=sha256-bq21I1EjXJa/s5Rra9J9ot2NkPCnI0F5uNPurwYLdpE%3D&rev=c859d03013712b349d82ee6223948d6d03e63a8d' (2024-11-15) → 'https://git.lix.systems/api/v1/repos/lix-project/lix/archive/66f6dbda32959dd5cf3a9aaba15af72d037ab7ff.tar.gz?narHash=sha256-H7GN4%2B%2Ba4vE49SUNojZx%2BFSk4mmpb2ifJUtJMJHProI%3D&rev=66f6dbda32959dd5cf3a9aaba15af72d037ab7ff' (2024-11-20) • Updated input 'nix-github-actions': 'github:nix-community/nix-github-actions/e04df33f62cdcf93d73e9a04142464753a16db67' (2024-10-24) → 'github:nix-community/nix-github-actions/7b5f051df789b6b20d259924d349a9ba3319b226' (2024-11-18) • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/035d434d48f4375ac5d3a620954cf5fda7dd7c36' (2024-11-15) → 'github:NixOS/nixpkgs/df94f897ffe1af1bcd60cb68697c5d8e6431346e' (2024-11-22) • Updated input 'treefmt-nix': 'github:numtide/treefmt-nix/746901bb8dba96d154b66492a29f5db0693dbfcc' (2024-10-30) → 'github:numtide/treefmt-nix/705df92694af7093dfbb27109ce16d828a79155f' (2024-11-22) --- flake.lock | 26 +++++++++++++------------- src/eval-args.cc | 2 +- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/flake.lock b/flake.lock index a0d91a967..cf03d2a86 100644 --- a/flake.lock +++ b/flake.lock @@ -47,11 +47,11 @@ "pre-commit-hooks": "pre-commit-hooks" }, "locked": { - "lastModified": 1731683711, - "narHash": "sha256-bq21I1EjXJa/s5Rra9J9ot2NkPCnI0F5uNPurwYLdpE=", - "rev": "c859d03013712b349d82ee6223948d6d03e63a8d", + "lastModified": 1732112222, + "narHash": "sha256-H7GN4++a4vE49SUNojZx+FSk4mmpb2ifJUtJMJHProI=", + "rev": "66f6dbda32959dd5cf3a9aaba15af72d037ab7ff", "type": "tarball", - "url": "https://git.lix.systems/api/v1/repos/lix-project/lix/archive/c859d03013712b349d82ee6223948d6d03e63a8d.tar.gz?rev=c859d03013712b349d82ee6223948d6d03e63a8d" + "url": "https://git.lix.systems/api/v1/repos/lix-project/lix/archive/66f6dbda32959dd5cf3a9aaba15af72d037ab7ff.tar.gz?rev=66f6dbda32959dd5cf3a9aaba15af72d037ab7ff" }, "original": { "type": "tarball", @@ -65,11 +65,11 @@ ] }, "locked": { - "lastModified": 1729742964, - "narHash": "sha256-B4mzTcQ0FZHdpeWcpDYPERtyjJd/NIuaQ9+BV1h+MpA=", + "lastModified": 1731952509, + "narHash": "sha256-p4gB3Rhw8R6Ak4eMl8pqjCPOLCZRqaehZxdZ/mbFClM=", "owner": "nix-community", "repo": "nix-github-actions", - "rev": "e04df33f62cdcf93d73e9a04142464753a16db67", + "rev": "7b5f051df789b6b20d259924d349a9ba3319b226", "type": "github" }, "original": { @@ -96,11 +96,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1731663789, - "narHash": "sha256-x07g4NcqGP6mQn6AISXJaks9sQYDjZmTMBlKIvajvyc=", + "lastModified": 1732244845, + "narHash": "sha256-aspop5sCDNpDMS23BplGFtQDadwkSb/sOxpuC3lafvo=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "035d434d48f4375ac5d3a620954cf5fda7dd7c36", + "rev": "df94f897ffe1af1bcd60cb68697c5d8e6431346e", "type": "github" }, "original": { @@ -158,11 +158,11 @@ ] }, "locked": { - "lastModified": 1730321837, - "narHash": "sha256-vK+a09qq19QNu2MlLcvN4qcRctJbqWkX7ahgPZ/+maI=", + "lastModified": 1732292307, + "narHash": "sha256-5WSng844vXt8uytT5djmqBCkopyle6ciFgteuA9bJpw=", "owner": "numtide", "repo": "treefmt-nix", - "rev": "746901bb8dba96d154b66492a29f5db0693dbfcc", + "rev": "705df92694af7093dfbb27109ce16d828a79155f", "type": "github" }, "original": { diff --git a/src/eval-args.cc b/src/eval-args.cc index b12763892..b7c526afb 100644 --- a/src/eval-args.cc +++ b/src/eval-args.cc @@ -100,5 +100,5 @@ MyArgs::MyArgs() : MixCommonArgs("nix-eval-jobs") { } void MyArgs::parseArgs(char **argv, int argc) { - parseCmdline(nix::argvToStrings(argc, argv)); + parseCmdline(nix::Strings(argv + 1, argv + argc)); } From bdbe3e13040029c2333a1d10c4704d0a63197a20 Mon Sep 17 00:00:00 2001 From: Jade Lovelace Date: Fri, 6 Dec 2024 13:14:46 -0800 Subject: [PATCH 416/419] Fix compilation for latest Lix main --- flake.lock | 14 +++++++------- src/drv.cc | 19 ++++++++++--------- src/nix-eval-jobs.cc | 12 +++++------- src/worker.cc | 25 ++++++++++++++----------- src/worker.hh | 3 ++- 5 files changed, 38 insertions(+), 35 deletions(-) diff --git a/flake.lock b/flake.lock index cf03d2a86..7602a77bf 100644 --- a/flake.lock +++ b/flake.lock @@ -47,11 +47,11 @@ "pre-commit-hooks": "pre-commit-hooks" }, "locked": { - "lastModified": 1732112222, - "narHash": "sha256-H7GN4++a4vE49SUNojZx+FSk4mmpb2ifJUtJMJHProI=", - "rev": "66f6dbda32959dd5cf3a9aaba15af72d037ab7ff", + "lastModified": 1733448312, + "narHash": "sha256-id5U81bzXk/Lg900nGLM4CQb0wmTdzIvQz7CZk2OcTM=", + "rev": "2a9e560570982a91937d199af3e7b7a8f3cbe14b", "type": "tarball", - "url": "https://git.lix.systems/api/v1/repos/lix-project/lix/archive/66f6dbda32959dd5cf3a9aaba15af72d037ab7ff.tar.gz?rev=66f6dbda32959dd5cf3a9aaba15af72d037ab7ff" + "url": "https://git.lix.systems/api/v1/repos/lix-project/lix/archive/2a9e560570982a91937d199af3e7b7a8f3cbe14b.tar.gz?rev=2a9e560570982a91937d199af3e7b7a8f3cbe14b" }, "original": { "type": "tarball", @@ -96,11 +96,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1732244845, - "narHash": "sha256-aspop5sCDNpDMS23BplGFtQDadwkSb/sOxpuC3lafvo=", + "lastModified": 1733408989, + "narHash": "sha256-VCQpCQy+6ik+oYKWUCvq0WM2V7UtEKldqdsEzCNEOLc=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "df94f897ffe1af1bcd60cb68697c5d8e6431346e", + "rev": "d916387b68a74040a3873ad2a08a559c60cedb5e", "type": "github" }, "original": { diff --git a/src/drv.cc b/src/drv.cc index dca58de9d..bacbf2a1c 100644 --- a/src/drv.cc +++ b/src/drv.cc @@ -45,13 +45,13 @@ queryIsCached(nix::Store &store, Drv::Drv(std::string &attrPath, nix::EvalState &state, nix::DrvInfo &drvInfo, MyArgs &args) { - auto localStore = state.store.dynamic_pointer_cast(); + auto localStore = state.ctx.store.dynamic_pointer_cast(); try { // CA derivations do not have static output paths, so we have to // defensively not query output paths in case we encounter one. for (auto &[outputName, optOutputPath] : - drvInfo.queryOutputs(!nix::experimentalFeatureSettings.isEnabled( + drvInfo.queryOutputs(state, !nix::experimentalFeatureSettings.isEnabled( nix::Xp::CaDerivations))) { if (optOutputPath) { outputs[outputName] = @@ -63,18 +63,19 @@ Drv::Drv(std::string &attrPath, nix::EvalState &state, nix::DrvInfo &drvInfo, } } } catch (const std::exception &e) { - throw nix::EvalError(state, + state.ctx.errors.make( "derivation '%s' does not have valid outputs: %s", - attrPath, e.what()); + attrPath, e.what() + ).debugThrow(); } if (args.meta) { nlohmann::json meta_; - for (auto &metaName : drvInfo.queryMetaNames()) { + for (auto &metaName : drvInfo.queryMetaNames(state)) { nix::NixStringContext context; std::stringstream ss; - auto metaValue = drvInfo.queryMeta(metaName); + auto metaValue = drvInfo.queryMeta(state, metaName); // Skip non-serialisable types // TODO: Fix serialisation of derivations to store paths if (metaValue == 0) { @@ -96,9 +97,9 @@ Drv::Drv(std::string &attrPath, nix::EvalState &state, nix::DrvInfo &drvInfo, cacheStatus = Drv::CacheStatus::Unknown; } - drvPath = localStore->printStorePath(drvInfo.requireDrvPath()); + drvPath = localStore->printStorePath(drvInfo.requireDrvPath(state)); - auto drv = localStore->readDerivation(drvInfo.requireDrvPath()); + auto drv = localStore->readDerivation(drvInfo.requireDrvPath(state)); for (const auto &[inputDrvPath, inputNode] : drv.inputDrvs.map) { std::set inputDrvOutputs; for (auto &outputName : inputNode.value) { @@ -106,7 +107,7 @@ Drv::Drv(std::string &attrPath, nix::EvalState &state, nix::DrvInfo &drvInfo, } inputDrvs[localStore->printStorePath(inputDrvPath)] = inputDrvOutputs; } - name = drvInfo.queryName(); + name = drvInfo.queryName(state); system = drv.platform; } diff --git a/src/nix-eval-jobs.cc b/src/nix-eval-jobs.cc index 2b4885ee0..8cba1c24d 100644 --- a/src/nix-eval-jobs.cc +++ b/src/nix-eval-jobs.cc @@ -24,7 +24,6 @@ #include #include #include -#include #include #include #include @@ -47,9 +46,8 @@ using namespace nlohmann; static MyArgs myArgs; -typedef std::function state, Bindings &autoArgs, - AutoCloseFD &to, AutoCloseFD &from, MyArgs &args)> - Processor; +using Processor = std::function state, Bindings &autoArgs, + AutoCloseFD &to, AutoCloseFD &from, MyArgs &args)>; /* Auto-cleanup of fork's process and fds. */ struct Proc { @@ -70,10 +68,10 @@ struct Proc { auto evalStore = myArgs.evalStoreUrl ? openStore(*myArgs.evalStoreUrl) : openStore(); - auto state = std::make_shared(myArgs.searchPath, + auto evaluator = nix::make_ref(myArgs.searchPath, evalStore); - Bindings &autoArgs = *myArgs.getAutoArgs(*state); - proc(ref(state), autoArgs, *to, *from, myArgs); + Bindings &autoArgs = *myArgs.getAutoArgs(*evaluator); + proc(evaluator, autoArgs, *to, *from, myArgs); } catch (Error &e) { nlohmann::json err; auto msg = e.msg(); diff --git a/src/worker.cc b/src/worker.cc index 5e5ff9fad..9c8caa275 100644 --- a/src/worker.cc +++ b/src/worker.cc @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -48,14 +49,14 @@ static nix::Value *releaseExprTopLevelValue(nix::EvalState &state, nix::Value vTop; if (args.fromArgs) { - nix::Expr &e = state.parseExprFromString( - args.releaseExpr, state.rootPath(nix::CanonPath::fromCwd())); + nix::Expr &e = state.ctx.parseExprFromString( + args.releaseExpr, nix::CanonPath::fromCwd()); state.eval(e, vTop); } else { - state.evalFile(lookupFileArg(state, args.releaseExpr), vTop); + state.evalFile(nix::lookupFileArg(state.ctx, args.releaseExpr), vTop); } - auto vRoot = state.allocValue(); + auto vRoot = state.ctx.mem.allocValue(); state.autoCallFunction(autoArgs, vTop, *vRoot); @@ -73,16 +74,17 @@ static std::string attrPathJoin(nlohmann::json input) { }); } -void worker(nix::ref state, nix::Bindings &autoArgs, +void worker(nix::ref evaluator, nix::Bindings &autoArgs, nix::AutoCloseFD &to, nix::AutoCloseFD &from, MyArgs &args) { nix::Value *vRoot = [&]() { + auto state = evaluator->begin(); if (args.flake) { auto [flakeRef, fragment, outputSpec] = nix::parseFlakeRefWithFragmentAndExtendedOutputsSpec( args.releaseExpr, nix::absPath(".")); nix::InstallableFlake flake{ - {}, state, std::move(flakeRef), fragment, outputSpec, + {}, evaluator, std::move(flakeRef), fragment, outputSpec, {}, {}, args.lockFlags}; return flake.toValue(*state).first; @@ -92,6 +94,7 @@ void worker(nix::ref state, nix::Bindings &autoArgs, }(); LineReader fromReader(from.release()); + auto state = evaluator->begin(); while (true) { /* Wait for the collector to send us a job name. */ @@ -119,7 +122,7 @@ void worker(nix::ref state, nix::Bindings &autoArgs, nix::findAlongAttrPath(*state, attrPathS, autoArgs, *vRoot) .first; - auto v = state->allocValue(); + auto v = evaluator->mem.allocValue(); state->autoCallFunction(autoArgs, *vTmp, *v); if (v->type() == nix::nAttrs) { @@ -136,7 +139,7 @@ void worker(nix::ref state, nix::Bindings &autoArgs, std::string(nix::baseNameOf(drv.drvPath)); if (!nix::pathExists(root)) { auto localStore = - state->store + evaluator->store .dynamic_pointer_cast(); auto storePath = localStore->parseStorePath(drv.drvPath); @@ -151,14 +154,14 @@ void worker(nix::ref state, nix::Bindings &autoArgs, // = true;` for top-level attrset for (auto &i : - v->attrs->lexicographicOrder(state->symbols)) { - const std::string &name = state->symbols[i->name]; + v->attrs->lexicographicOrder(evaluator->symbols)) { + const std::string &name = evaluator->symbols[i->name]; attrs.push_back(name); if (name == "recurseForDerivations" && !args.forceRecurse) { auto attrv = - v->attrs->get(state->sRecurseForDerivations); + v->attrs->get(evaluator->s.recurseForDerivations); recurse = state->forceBool( *attrv->value, attrv->pos, "while evaluating recurseForDerivations"); diff --git a/src/worker.hh b/src/worker.hh index caf4200b8..013e64ada 100644 --- a/src/worker.hh +++ b/src/worker.hh @@ -1,6 +1,7 @@ #pragma once #include #include +#include #include "eval-args.hh" @@ -13,5 +14,5 @@ class EvalState; template class ref; } // namespace nix -void worker(nix::ref state, nix::Bindings &autoArgs, +void worker(nix::ref evaluator, nix::Bindings &autoArgs, nix::AutoCloseFD &to, nix::AutoCloseFD &from, MyArgs &args); From 08182f4ed7b04fcda7069edfe8ef807f4c4ce78a Mon Sep 17 00:00:00 2001 From: Jade Lovelace Date: Sat, 18 Jan 2025 14:01:48 -0800 Subject: [PATCH 417/419] flake: update nixpkgs, lix --- flake.lock | 22 +++++++++++----------- flake.nix | 2 +- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/flake.lock b/flake.lock index 7602a77bf..fa18ef32b 100644 --- a/flake.lock +++ b/flake.lock @@ -47,11 +47,11 @@ "pre-commit-hooks": "pre-commit-hooks" }, "locked": { - "lastModified": 1733448312, - "narHash": "sha256-id5U81bzXk/Lg900nGLM4CQb0wmTdzIvQz7CZk2OcTM=", - "rev": "2a9e560570982a91937d199af3e7b7a8f3cbe14b", + "lastModified": 1737234286, + "narHash": "sha256-pgDJZjj4jpzkFxsqBTI/9Yb0n3gW+DvDtuv9SwQZZcs=", + "rev": "079528098f5998ba13c88821a2eca1005c1695de", "type": "tarball", - "url": "https://git.lix.systems/api/v1/repos/lix-project/lix/archive/2a9e560570982a91937d199af3e7b7a8f3cbe14b.tar.gz?rev=2a9e560570982a91937d199af3e7b7a8f3cbe14b" + "url": "https://git.lix.systems/api/v1/repos/lix-project/lix/archive/079528098f5998ba13c88821a2eca1005c1695de.tar.gz?rev=079528098f5998ba13c88821a2eca1005c1695de" }, "original": { "type": "tarball", @@ -96,16 +96,16 @@ }, "nixpkgs": { "locked": { - "lastModified": 1733408989, - "narHash": "sha256-VCQpCQy+6ik+oYKWUCvq0WM2V7UtEKldqdsEzCNEOLc=", + "lastModified": 1737226685, + "narHash": "sha256-34x0t/x5SkClo04gaG+KPBwN9JtXjFSQGTl//Yry4Gc=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "d916387b68a74040a3873ad2a08a559c60cedb5e", + "rev": "bf68d76e54ac5da0d77f82656a37552996195e80", "type": "github" }, "original": { "owner": "NixOS", - "ref": "nixos-24.05-small", + "ref": "nixos-24.11-small", "repo": "nixpkgs", "type": "github" } @@ -129,11 +129,11 @@ "pre-commit-hooks": { "flake": false, "locked": { - "lastModified": 1726745158, - "narHash": "sha256-D5AegvGoEjt4rkKedmxlSEmC+nNLMBPWFxvmYnVLhjk=", + "lastModified": 1733318908, + "narHash": "sha256-SVQVsbafSM1dJ4fpgyBqLZ+Lft+jcQuMtEL3lQWx2Sk=", "owner": "cachix", "repo": "git-hooks.nix", - "rev": "4e743a6920eab45e8ba0fbe49dc459f1423a4b74", + "rev": "6f4e2a2112050951a314d2733a994fbab94864c6", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index a2cc4798d..cc805f22d 100644 --- a/flake.nix +++ b/flake.nix @@ -1,7 +1,7 @@ { description = "Hydra's builtin hydra-eval-jobs as a standalone"; - inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-24.05-small"; + inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-24.11-small"; inputs.flake-parts.url = "github:hercules-ci/flake-parts"; inputs.flake-parts.inputs.nixpkgs-lib.follows = "nixpkgs"; inputs.treefmt-nix.url = "github:numtide/treefmt-nix"; From 6482bee40b89ab2c28d2e988848aa9ae0429d78c Mon Sep 17 00:00:00 2001 From: Jade Lovelace Date: Sat, 18 Jan 2025 14:05:17 -0800 Subject: [PATCH 418/419] Update version to 2.93, I guess --- default.nix | 2 +- meson.build | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/default.nix b/default.nix index 9ef66afd4..8fb058a46 100644 --- a/default.nix +++ b/default.nix @@ -11,7 +11,7 @@ let in stdenv.mkDerivation { pname = "nix-eval-jobs"; - version = "2.90.0-unstable"; + version = "2.93.0-dev"; src = if srcDir == null then filterMesonBuild ./. else srcDir; buildInputs = with pkgs; [ nlohmann_json diff --git a/meson.build b/meson.build index 15708a675..6cbdce64c 100644 --- a/meson.build +++ b/meson.build @@ -1,5 +1,5 @@ project('nix-eval-jobs', 'cpp', - version : '0.1.6', + version : '2.93.0-dev', license : 'GPL-3.0', default_options : [ 'debug=true', From 53ce7ebc33b9c09e0798da582ddddede7febe653 Mon Sep 17 00:00:00 2001 From: Jade Lovelace Date: Fri, 24 Jan 2025 11:24:51 -0800 Subject: [PATCH 419/419] fix for asyncized lix --- default.nix | 2 ++ flake.lock | 14 +++++++------- meson.build | 1 + src/eval-args.cc | 3 ++- src/eval-args.hh | 8 +++++++- src/nix-eval-jobs.cc | 35 ++++++++++++++++++++++------------- src/worker.cc | 19 ++++++++++--------- src/worker.hh | 8 +++++--- 8 files changed, 56 insertions(+), 34 deletions(-) diff --git a/default.nix b/default.nix index 8fb058a46..9b398d61d 100644 --- a/default.nix +++ b/default.nix @@ -24,6 +24,8 @@ stdenv.mkDerivation { ninja # nlohmann_json can be only discovered via cmake files cmake + # XXX: ew + nix.passthru.capnproto-lix ] ++ (lib.optional stdenv.cc.isClang [ pkgs.clang-tools ]); meta = { diff --git a/flake.lock b/flake.lock index fa18ef32b..656b2cfc5 100644 --- a/flake.lock +++ b/flake.lock @@ -47,11 +47,11 @@ "pre-commit-hooks": "pre-commit-hooks" }, "locked": { - "lastModified": 1737234286, - "narHash": "sha256-pgDJZjj4jpzkFxsqBTI/9Yb0n3gW+DvDtuv9SwQZZcs=", - "rev": "079528098f5998ba13c88821a2eca1005c1695de", + "lastModified": 1737857294, + "narHash": "sha256-bzC+anLF/NlgolaMoB4uTFgSejLJlTzPcNF1Kbq/BP0=", + "rev": "4af6b5ed9f8f2412bef5331b8e3b93f3ad305ea1", "type": "tarball", - "url": "https://git.lix.systems/api/v1/repos/lix-project/lix/archive/079528098f5998ba13c88821a2eca1005c1695de.tar.gz?rev=079528098f5998ba13c88821a2eca1005c1695de" + "url": "https://git.lix.systems/api/v1/repos/lix-project/lix/archive/4af6b5ed9f8f2412bef5331b8e3b93f3ad305ea1.tar.gz?rev=4af6b5ed9f8f2412bef5331b8e3b93f3ad305ea1" }, "original": { "type": "tarball", @@ -96,11 +96,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1737226685, - "narHash": "sha256-34x0t/x5SkClo04gaG+KPBwN9JtXjFSQGTl//Yry4Gc=", + "lastModified": 1737672001, + "narHash": "sha256-YnHJJ19wqmibLQdUeq9xzE6CjrMA568KN/lFPuSVs4I=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "bf68d76e54ac5da0d77f82656a37552996195e80", + "rev": "035f8c0853c2977b24ffc4d0a42c74f00b182cd8", "type": "github" }, "original": { diff --git a/meson.build b/meson.build index 6cbdce64c..a4eede748 100644 --- a/meson.build +++ b/meson.build @@ -15,5 +15,6 @@ nix_cmd_dep = dependency('lix-cmd', required: true) threads_dep = dependency('threads', required: true) nlohmann_json_dep = dependency('nlohmann_json', required: true) boost_dep = dependency('boost', required: true) +kj_async_dep = dependency('kj-async', required: true) subdir('src') diff --git a/src/eval-args.cc b/src/eval-args.cc index b7c526afb..57b839c28 100644 --- a/src/eval-args.cc +++ b/src/eval-args.cc @@ -1,3 +1,4 @@ +#include #include #include #include @@ -11,7 +12,7 @@ #include "eval-args.hh" -MyArgs::MyArgs() : MixCommonArgs("nix-eval-jobs") { +MyArgs::MyArgs(nix::AsyncIoRoot & aio) : MixCommonArgs("nix-eval-jobs"), aio_(aio) { addFlag({ .longName = "help", .description = "show usage information", diff --git a/src/eval-args.hh b/src/eval-args.hh index b0932fb8c..1316c154f 100644 --- a/src/eval-args.hh +++ b/src/eval-args.hh @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -13,6 +14,10 @@ class MyArgs : virtual public nix::MixEvalArgs, virtual public nix::MixCommonArgs, virtual public nix::RootArgs { + // intentionally hidden in this subclass because it's mondo dangerous + // in n-e-j due to all the forking we do for worker process creation. + nix::AsyncIoRoot & aio_; + nix::AsyncIoRoot & aio() override { return aio_; } public: std::string releaseExpr; nix::Path gcRootsDir; @@ -31,7 +36,8 @@ class MyArgs : virtual public nix::MixEvalArgs, .writeLockFile = false, .useRegistries = false, .allowUnlocked = false}; - MyArgs(); + + MyArgs(nix::AsyncIoRoot & aio); MyArgs(const MyArgs&) = delete; void parseArgs(char** argv, int argc); diff --git a/src/nix-eval-jobs.cc b/src/nix-eval-jobs.cc index 8cba1c24d..8990398f5 100644 --- a/src/nix-eval-jobs.cc +++ b/src/nix-eval-jobs.cc @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -44,17 +45,16 @@ using namespace nix; using namespace nlohmann; -static MyArgs myArgs; - -using Processor = std::function state, Bindings &autoArgs, - AutoCloseFD &to, AutoCloseFD &from, MyArgs &args)>; +using Processor = std::function state, Bindings &autoArgs, + AutoCloseFD &to, AutoCloseFD &from, MyArgs &args, AsyncIoRoot &aio)>; /* Auto-cleanup of fork's process and fds. */ struct Proc { AutoCloseFD to, from; Pid pid; - Proc(const Processor &proc) { + Proc(MyArgs &myArgs, const Processor &proc) { Pipe toPipe, fromPipe; toPipe.create(); fromPipe.create(); @@ -65,13 +65,15 @@ struct Proc { std::make_shared(std::move(toPipe.readSide))}]() { debug("created worker process %d", getpid()); try { + AsyncIoRoot aio; auto evalStore = myArgs.evalStoreUrl ? openStore(*myArgs.evalStoreUrl) : openStore(); - auto evaluator = nix::make_ref(myArgs.searchPath, - evalStore); + auto evaluator = + nix::make_ref( + aio, myArgs.searchPath, evalStore); Bindings &autoArgs = *myArgs.getAutoArgs(*evaluator); - proc(evaluator, autoArgs, *to, *from, myArgs); + proc(evaluator, autoArgs, *to, *from, myArgs, aio); } catch (Error &e) { nlohmann::json err; auto msg = e.msg(); @@ -120,7 +122,8 @@ struct Thread { if ((s = pthread_attr_setstacksize(&attr, 64 * 1024 * 1024)) != 0) { throw SysError(s, "calling pthread_attr_setstacksize"); } - if ((s = pthread_create(&thread, &attr, Thread::init, func.release())) != 0) { + if ((s = pthread_create(&thread, &attr, Thread::init, + func.release())) != 0) { throw SysError(s, "calling pthread_launch"); } if ((s = pthread_attr_destroy(&attr)) != 0) { @@ -135,7 +138,8 @@ struct Thread { throw SysError(s, "calling pthread_join"); } } -private: + + private: static void *init(void *ptr) { std::unique_ptr> func; func.reset(static_cast *>(ptr)); @@ -221,14 +225,15 @@ std::string joinAttrPath(json &attrPath) { return joined; } -void collector(Sync &state_, std::condition_variable &wakeup) { +void collector(MyArgs &myArgs, Sync &state_, + std::condition_variable &wakeup) { try { std::optional> proc_; std::optional> fromReader_; while (true) { if (!proc_.has_value()) { - proc_ = std::make_unique(worker); + proc_ = std::make_unique(myArgs, worker); fromReader_ = std::make_unique(proc_.value()->from.release()); } @@ -344,6 +349,9 @@ int main(int argc, char **argv) { initNix(); initLibExpr(); + nix::AsyncIoRoot aio; + MyArgs myArgs(aio); + myArgs.parseArgs(argv, argc); /* When building a flake, use pure evaluation (no access to @@ -373,7 +381,8 @@ int main(int argc, char **argv) { std::vector threads; std::condition_variable wakeup; for (size_t i = 0; i < myArgs.nrWorkers; i++) { - threads.emplace_back(std::bind(collector, std::ref(state_), std::ref(wakeup))); + threads.emplace_back(std::bind(collector, std::ref(myArgs), + std::ref(state_), std::ref(wakeup))); } for (auto &thread : threads) diff --git a/src/worker.cc b/src/worker.cc index 9c8caa275..a11696809 100644 --- a/src/worker.cc +++ b/src/worker.cc @@ -49,8 +49,8 @@ static nix::Value *releaseExprTopLevelValue(nix::EvalState &state, nix::Value vTop; if (args.fromArgs) { - nix::Expr &e = state.ctx.parseExprFromString( - args.releaseExpr, nix::CanonPath::fromCwd()); + nix::Expr &e = state.ctx.parseExprFromString(args.releaseExpr, + nix::CanonPath::fromCwd()); state.eval(e, vTop); } else { state.evalFile(nix::lookupFileArg(state.ctx, args.releaseExpr), vTop); @@ -74,18 +74,19 @@ static std::string attrPathJoin(nlohmann::json input) { }); } -void worker(nix::ref evaluator, nix::Bindings &autoArgs, - nix::AutoCloseFD &to, nix::AutoCloseFD &from, MyArgs &args) { +void worker(nix::ref evaluator, + nix::Bindings &autoArgs, nix::AutoCloseFD &to, + nix::AutoCloseFD &from, MyArgs &args, nix::AsyncIoRoot &aio) { nix::Value *vRoot = [&]() { - auto state = evaluator->begin(); + auto state = evaluator->begin(aio); if (args.flake) { auto [flakeRef, fragment, outputSpec] = nix::parseFlakeRefWithFragmentAndExtendedOutputsSpec( args.releaseExpr, nix::absPath(".")); nix::InstallableFlake flake{ {}, evaluator, std::move(flakeRef), fragment, outputSpec, - {}, {}, args.lockFlags}; + {}, {}, args.lockFlags}; return flake.toValue(*state).first; } else { @@ -94,7 +95,7 @@ void worker(nix::ref evaluator, nix::Bindings }(); LineReader fromReader(from.release()); - auto state = evaluator->begin(); + auto state = evaluator->begin(aio); while (true) { /* Wait for the collector to send us a job name. */ @@ -160,8 +161,8 @@ void worker(nix::ref evaluator, nix::Bindings if (name == "recurseForDerivations" && !args.forceRecurse) { - auto attrv = - v->attrs->get(evaluator->s.recurseForDerivations); + auto attrv = v->attrs->get( + evaluator->s.recurseForDerivations); recurse = state->forceBool( *attrv->value, attrv->pos, "while evaluating recurseForDerivations"); diff --git a/src/worker.hh b/src/worker.hh index 013e64ada..d4c41caff 100644 --- a/src/worker.hh +++ b/src/worker.hh @@ -2,6 +2,7 @@ #include #include #include +#include #include "eval-args.hh" @@ -12,7 +13,8 @@ class AutoCloseFD; class Bindings; class EvalState; template class ref; -} // namespace nix +} // namespace nix -void worker(nix::ref evaluator, nix::Bindings &autoArgs, - nix::AutoCloseFD &to, nix::AutoCloseFD &from, MyArgs &args); +void worker(nix::ref evaluator, + nix::Bindings &autoArgs, nix::AutoCloseFD &to, + nix::AutoCloseFD &from, MyArgs &args, nix::AsyncIoRoot &aio);