From 5e88d3b08972e3bf1ae89a3852861b6efb05dc67 Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Sat, 25 Jan 2025 12:42:16 +0100 Subject: [PATCH 1/3] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'lix': 'git+https://git.lix.systems/lix-project/lix?ref=refs/heads/main&rev=4cacb5412fedc9f37bc7a502540374f774d923cf' (2025-01-18) → 'git+https://git.lix.systems/lix-project/lix?ref=refs/heads/main&rev=5a41803f744822377587f784ded0e6a061b39cd4' (2025-01-25) • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/6a3ae7a5a12fb8cac2d59d7df7cbd95f9b2f0566' (2025-01-18) → 'github:NixOS/nixpkgs/035f8c0853c2977b24ffc4d0a42c74f00b182cd8' (2025-01-23) --- flake.lock | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/flake.lock b/flake.lock index bdb5cfe6d..8f108495d 100644 --- a/flake.lock +++ b/flake.lock @@ -48,11 +48,11 @@ "pre-commit-hooks": "pre-commit-hooks" }, "locked": { - "lastModified": 1737238254, - "narHash": "sha256-GoOwW81pV5T0YhD3yuLZG0+/Qr/Ld4qC71eM0mD3pJA=", + "lastModified": 1737770806, + "narHash": "sha256-mdtBGOkNySRbQ1zkroCdyOHFqMOZSOJCPxjXTpCaWvc=", "ref": "refs/heads/main", - "rev": "4cacb5412fedc9f37bc7a502540374f774d923cf", - "revCount": 16633, + "rev": "5a41803f744822377587f784ded0e6a061b39cd4", + "revCount": 16688, "type": "git", "url": "https://git.lix.systems/lix-project/lix" }, @@ -126,11 +126,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1737165118, - "narHash": "sha256-s40Kk/OulP3J/1JvC3VT16U4r/Xw6Qdi7SRw3LYkPWs=", + "lastModified": 1737672001, + "narHash": "sha256-YnHJJ19wqmibLQdUeq9xzE6CjrMA568KN/lFPuSVs4I=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "6a3ae7a5a12fb8cac2d59d7df7cbd95f9b2f0566", + "rev": "035f8c0853c2977b24ffc4d0a42c74f00b182cd8", "type": "github" }, "original": { From 44a472af6791250b451d4d105ee0aa98861a7376 Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Sun, 26 Jan 2025 10:52:06 +0100 Subject: [PATCH 2/3] Update flake; fix build with latest Lix / nix-eval-jobs Most notably, this fixes the API change of `Store::narFromPath`[1] that now returns a `box_ptr` instead of a Generator. I decided to do the most minimal invasive thing here, i.e. dereferencing the source out of the box in the NAR extraction code to fix the build again given that the API will change even more in the future and thus the implementation in here will require more work anyways. To quote the upstream commit: > [...] and would need to be changed much as they are now once async > streams come around either way [1] See Lix commit c82cc16754cbd583fd22da068ea1b98d33a22fb5 --- flake.lock | 8 ++++---- package.nix | 1 + src/hydra-evaluator/hydra-evaluator.cc | 4 +++- src/hydra-queue-runner/build-result.cc | 2 +- src/hydra-queue-runner/hydra-queue-runner.cc | 4 +++- src/hydra-queue-runner/nar-extractor.cc | 4 ++-- src/hydra-queue-runner/nar-extractor.hh | 2 +- 7 files changed, 15 insertions(+), 10 deletions(-) diff --git a/flake.lock b/flake.lock index 8f108495d..f35fc8feb 100644 --- a/flake.lock +++ b/flake.lock @@ -74,11 +74,11 @@ "treefmt-nix": "treefmt-nix" }, "locked": { - "lastModified": 1737237917, - "narHash": "sha256-xd3/Wr4aPc2MY61qyQ6qQ8AGoG3H/DDcJWziP4EhhXY=", + "lastModified": 1738008985, + "narHash": "sha256-EYmllgk4J7jZPfyf9ps7JWJFiNv9wLncpND23d+7dVo=", "ref": "refs/heads/main", - "rev": "6482bee40b89ab2c28d2e988848aa9ae0429d78c", - "revCount": 621, + "rev": "e489d2190a9e7b6a369d3457b49ba80c93f80eae", + "revCount": 623, "type": "git", "url": "https://git.lix.systems/lix-project/nix-eval-jobs" }, diff --git a/package.nix b/package.nix index 8789e058d..60ef1fd0d 100644 --- a/package.nix +++ b/package.nix @@ -177,6 +177,7 @@ stdenv.mkDerivation (finalAttrs: { boost nlohmann_json prometheus-cpp + nix.passthru.capnproto-lix ]; nativeCheckInputs = [ diff --git a/src/hydra-evaluator/hydra-evaluator.cc b/src/hydra-evaluator/hydra-evaluator.cc index e13ee75a6..b03ba8bfe 100644 --- a/src/hydra-evaluator/hydra-evaluator.cc +++ b/src/hydra-evaluator/hydra-evaluator.cc @@ -4,6 +4,7 @@ #include "lix/libutil/logging.hh" #include "lix/libutil/pool.hh" #include "lix/libutil/signals.hh" +#include "lix/libutil/async.hh" #include #include @@ -513,7 +514,8 @@ int main(int argc, char * * argv) std::vector args; - LegacyArgs(argv[0], [&](Strings::iterator & arg, const Strings::iterator & end) { + nix::AsyncIoRoot aio; + LegacyArgs(aio, argv[0], [&](Strings::iterator & arg, const Strings::iterator & end) { if (*arg == "--unlock") unlock = true; else if (arg->starts_with("-")) diff --git a/src/hydra-queue-runner/build-result.cc b/src/hydra-queue-runner/build-result.cc index fa605319f..2672c4df3 100644 --- a/src/hydra-queue-runner/build-result.cc +++ b/src/hydra-queue-runner/build-result.cc @@ -35,7 +35,7 @@ BuildOutput getBuildOutput( auto outputS = store->printStorePath(output); if (!narMembers.count(outputS)) { printInfo("fetching NAR contents of '%s'...", outputS); - GeneratorSource source{store->narFromPath(output)}; + auto source = store->narFromPath(output); extractNarData(source, outputS, narMembers); } } diff --git a/src/hydra-queue-runner/hydra-queue-runner.cc b/src/hydra-queue-runner/hydra-queue-runner.cc index f49579f17..c09be0108 100644 --- a/src/hydra-queue-runner/hydra-queue-runner.cc +++ b/src/hydra-queue-runner/hydra-queue-runner.cc @@ -21,6 +21,7 @@ #include "lix/libstore/globals.hh" #include "lix/libstore/s3-binary-cache-store.hh" #include "lix/libutil/args.hh" +#include "lix/libutil/async.hh" using namespace nix; using nlohmann::json; @@ -974,7 +975,8 @@ int main(int argc, char * * argv) BuildID buildOne = 0; std::optional metricsAddrOpt = std::nullopt; - LegacyArgs(argv[0], [&](Strings::iterator & arg, const Strings::iterator & end) { + nix::AsyncIoRoot aio; + LegacyArgs(aio, argv[0], [&](Strings::iterator & arg, const Strings::iterator & end) { if (*arg == "--unlock") unlock = true; else if (*arg == "--status") diff --git a/src/hydra-queue-runner/nar-extractor.cc b/src/hydra-queue-runner/nar-extractor.cc index d96ca87c2..0378a9b81 100644 --- a/src/hydra-queue-runner/nar-extractor.cc +++ b/src/hydra-queue-runner/nar-extractor.cc @@ -75,11 +75,11 @@ struct Extractor : NARParseVisitor void extractNarData( - Source & source, + box_ptr & source, const Path & prefix, NarMemberDatas & members) { - auto parser = extractNarDataFilter(source, prefix, members); + auto parser = extractNarDataFilter(*source, prefix, members); while (parser.next()) { // ignore raw data } diff --git a/src/hydra-queue-runner/nar-extractor.hh b/src/hydra-queue-runner/nar-extractor.hh index dc9d00f46..f05cbe9de 100644 --- a/src/hydra-queue-runner/nar-extractor.hh +++ b/src/hydra-queue-runner/nar-extractor.hh @@ -18,7 +18,7 @@ using NarMemberDatas = std::map; /* Read a NAR from a source and get to some info about every file inside the NAR. */ void extractNarData( - nix::Source & source, + nix::box_ptr & source, const nix::Path & prefix, NarMemberDatas & members); From 67fc8213df9b418eb16ab9f05a6644e83381542e Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Tue, 28 Jan 2025 17:56:10 +0100 Subject: [PATCH 3/3] Update flake; start aio event loop in queue monitor --- flake.lock | 8 ++++---- src/hydra-queue-runner/queue-monitor.cc | 15 +++++++++------ src/hydra-queue-runner/state.hh | 5 +++-- 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/flake.lock b/flake.lock index f35fc8feb..f045bf7d4 100644 --- a/flake.lock +++ b/flake.lock @@ -48,11 +48,11 @@ "pre-commit-hooks": "pre-commit-hooks" }, "locked": { - "lastModified": 1737770806, - "narHash": "sha256-mdtBGOkNySRbQ1zkroCdyOHFqMOZSOJCPxjXTpCaWvc=", + "lastModified": 1738053421, + "narHash": "sha256-B9HigdAgTs/FxUVcGYIRIxJVPQ+vGp3/G6WRuVKIGds=", "ref": "refs/heads/main", - "rev": "5a41803f744822377587f784ded0e6a061b39cd4", - "revCount": 16688, + "rev": "f8a592713d14bf0374950c8d3575eb3795e63987", + "revCount": 16698, "type": "git", "url": "https://git.lix.systems/lix-project/lix" }, diff --git a/src/hydra-queue-runner/queue-monitor.cc b/src/hydra-queue-runner/queue-monitor.cc index b66becb0f..6231449bc 100644 --- a/src/hydra-queue-runner/queue-monitor.cc +++ b/src/hydra-queue-runner/queue-monitor.cc @@ -1,5 +1,6 @@ #include "hydra-build-result.hh" #include "lix/libstore/globals.hh" +#include "lix/libutil/async.hh" #include "lix/libutil/thread-pool.hh" #include "state.hh" @@ -40,13 +41,15 @@ void State::queueMonitorLoop(Connection & conn) auto destStore = getDestStore(); + AsyncIoRoot aio; + bool quit = false; while (!quit) { auto t_before_work = std::chrono::steady_clock::now(); localStore->clearPathInfoCache(); - bool done = getQueuedBuilds(conn, destStore); + bool done = getQueuedBuilds(aio, conn, destStore); if (buildOne && buildOneDone) quit = true; @@ -93,7 +96,7 @@ struct PreviousFailure : public std::exception { }; -bool State::getQueuedBuilds(Connection & conn, +bool State::getQueuedBuilds(AsyncIoRoot & aio, Connection & conn, ref destStore) { prom.queue_checks_started.Increment(); @@ -176,7 +179,7 @@ bool State::getQueuedBuilds(Connection & conn, /* Create steps for this derivation and its dependencies. */ try { - step = createStep(destStore, conn, build, build->drvPath, + step = createStep(aio, destStore, conn, build, build->drvPath, build, nullptr, finishedDrvs, newSteps, newRunnable); } catch (PreviousFailure & ex) { @@ -434,7 +437,7 @@ std::map> State::getMissingRemotePaths( } -Step::ptr State::createStep(ref destStore, +Step::ptr State::createStep(AsyncIoRoot & aio, ref destStore, Connection & conn, Build::ptr build, const StorePath & drvPath, Build::ptr referringBuild, Step::ptr referringStep, std::set & finishedDrvs, std::set & newSteps, std::set & newRunnable) @@ -571,7 +574,7 @@ Step::ptr State::createStep(ref destStore, printInfo("substituting output ‘%1%’ of ‘%2%’", localStore->printStorePath(path), localStore->printStorePath(drvPath)); - localStore->ensurePath(path); + aio.blockOn(localStore->ensurePath(path)); // FIXME: should copy directly from substituter to destStore. } @@ -611,7 +614,7 @@ Step::ptr State::createStep(ref destStore, /* Create steps for the dependencies. */ for (auto & i : step->drv->inputDrvs.map) { - auto dep = createStep(destStore, conn, build, i.first, nullptr, step, finishedDrvs, newSteps, newRunnable); + auto dep = createStep(aio, destStore, conn, build, i.first, nullptr, step, finishedDrvs, newSteps, newRunnable); if (dep) { auto step_(step->state.lock()); step_->deps.insert(dep); diff --git a/src/hydra-queue-runner/state.hh b/src/hydra-queue-runner/state.hh index 88fe262f4..9e5f21b6f 100644 --- a/src/hydra-queue-runner/state.hh +++ b/src/hydra-queue-runner/state.hh @@ -22,6 +22,7 @@ #include "lix/libstore/pathlocks.hh" #include "lix/libstore/serve-protocol.hh" #include "lix/libstore/store-api.hh" +#include "lix/libutil/async.hh" #include "lix/libutil/pool.hh" #include "lix/libutil/sync.hh" #include "nar-extractor.hh" @@ -541,7 +542,7 @@ private: void queueMonitorLoop(Connection & conn); /* Check the queue for new builds. */ - bool getQueuedBuilds(Connection & conn, nix::ref destStore); + bool getQueuedBuilds(nix::AsyncIoRoot & aio, Connection & conn, nix::ref destStore); /* Handle cancellation, deletion and priority bumps. */ void processQueueChange(Connection & conn); @@ -555,7 +556,7 @@ private: nix::ref destStore, const std::map> & paths); - Step::ptr createStep(nix::ref store, + Step::ptr createStep(nix::AsyncIoRoot & aio, nix::ref store, Connection & conn, Build::ptr build, const nix::StorePath & drvPath, Build::ptr referringBuild, Step::ptr referringStep, std::set & finishedDrvs, std::set & newSteps, std::set & newRunnable);