diff --git a/flake.lock b/flake.lock index bdb5cfe6d..f045bf7d4 100644 --- a/flake.lock +++ b/flake.lock @@ -48,11 +48,11 @@ "pre-commit-hooks": "pre-commit-hooks" }, "locked": { - "lastModified": 1737238254, - "narHash": "sha256-GoOwW81pV5T0YhD3yuLZG0+/Qr/Ld4qC71eM0mD3pJA=", + "lastModified": 1738053421, + "narHash": "sha256-B9HigdAgTs/FxUVcGYIRIxJVPQ+vGp3/G6WRuVKIGds=", "ref": "refs/heads/main", - "rev": "4cacb5412fedc9f37bc7a502540374f774d923cf", - "revCount": 16633, + "rev": "f8a592713d14bf0374950c8d3575eb3795e63987", + "revCount": 16698, "type": "git", "url": "https://git.lix.systems/lix-project/lix" }, @@ -74,11 +74,11 @@ "treefmt-nix": "treefmt-nix" }, "locked": { - "lastModified": 1737237917, - "narHash": "sha256-xd3/Wr4aPc2MY61qyQ6qQ8AGoG3H/DDcJWziP4EhhXY=", + "lastModified": 1738008985, + "narHash": "sha256-EYmllgk4J7jZPfyf9ps7JWJFiNv9wLncpND23d+7dVo=", "ref": "refs/heads/main", - "rev": "6482bee40b89ab2c28d2e988848aa9ae0429d78c", - "revCount": 621, + "rev": "e489d2190a9e7b6a369d3457b49ba80c93f80eae", + "revCount": 623, "type": "git", "url": "https://git.lix.systems/lix-project/nix-eval-jobs" }, @@ -126,11 +126,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1737165118, - "narHash": "sha256-s40Kk/OulP3J/1JvC3VT16U4r/Xw6Qdi7SRw3LYkPWs=", + "lastModified": 1737672001, + "narHash": "sha256-YnHJJ19wqmibLQdUeq9xzE6CjrMA568KN/lFPuSVs4I=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "6a3ae7a5a12fb8cac2d59d7df7cbd95f9b2f0566", + "rev": "035f8c0853c2977b24ffc4d0a42c74f00b182cd8", "type": "github" }, "original": { diff --git a/package.nix b/package.nix index 8789e058d..60ef1fd0d 100644 --- a/package.nix +++ b/package.nix @@ -177,6 +177,7 @@ stdenv.mkDerivation (finalAttrs: { boost nlohmann_json prometheus-cpp + nix.passthru.capnproto-lix ]; nativeCheckInputs = [ diff --git a/src/hydra-evaluator/hydra-evaluator.cc b/src/hydra-evaluator/hydra-evaluator.cc index e13ee75a6..b03ba8bfe 100644 --- a/src/hydra-evaluator/hydra-evaluator.cc +++ b/src/hydra-evaluator/hydra-evaluator.cc @@ -4,6 +4,7 @@ #include "lix/libutil/logging.hh" #include "lix/libutil/pool.hh" #include "lix/libutil/signals.hh" +#include "lix/libutil/async.hh" #include #include @@ -513,7 +514,8 @@ int main(int argc, char * * argv) std::vector args; - LegacyArgs(argv[0], [&](Strings::iterator & arg, const Strings::iterator & end) { + nix::AsyncIoRoot aio; + LegacyArgs(aio, argv[0], [&](Strings::iterator & arg, const Strings::iterator & end) { if (*arg == "--unlock") unlock = true; else if (arg->starts_with("-")) diff --git a/src/hydra-queue-runner/build-result.cc b/src/hydra-queue-runner/build-result.cc index fa605319f..2672c4df3 100644 --- a/src/hydra-queue-runner/build-result.cc +++ b/src/hydra-queue-runner/build-result.cc @@ -35,7 +35,7 @@ BuildOutput getBuildOutput( auto outputS = store->printStorePath(output); if (!narMembers.count(outputS)) { printInfo("fetching NAR contents of '%s'...", outputS); - GeneratorSource source{store->narFromPath(output)}; + auto source = store->narFromPath(output); extractNarData(source, outputS, narMembers); } } diff --git a/src/hydra-queue-runner/hydra-queue-runner.cc b/src/hydra-queue-runner/hydra-queue-runner.cc index f49579f17..c09be0108 100644 --- a/src/hydra-queue-runner/hydra-queue-runner.cc +++ b/src/hydra-queue-runner/hydra-queue-runner.cc @@ -21,6 +21,7 @@ #include "lix/libstore/globals.hh" #include "lix/libstore/s3-binary-cache-store.hh" #include "lix/libutil/args.hh" +#include "lix/libutil/async.hh" using namespace nix; using nlohmann::json; @@ -974,7 +975,8 @@ int main(int argc, char * * argv) BuildID buildOne = 0; std::optional metricsAddrOpt = std::nullopt; - LegacyArgs(argv[0], [&](Strings::iterator & arg, const Strings::iterator & end) { + nix::AsyncIoRoot aio; + LegacyArgs(aio, argv[0], [&](Strings::iterator & arg, const Strings::iterator & end) { if (*arg == "--unlock") unlock = true; else if (*arg == "--status") diff --git a/src/hydra-queue-runner/nar-extractor.cc b/src/hydra-queue-runner/nar-extractor.cc index d96ca87c2..0378a9b81 100644 --- a/src/hydra-queue-runner/nar-extractor.cc +++ b/src/hydra-queue-runner/nar-extractor.cc @@ -75,11 +75,11 @@ struct Extractor : NARParseVisitor void extractNarData( - Source & source, + box_ptr & source, const Path & prefix, NarMemberDatas & members) { - auto parser = extractNarDataFilter(source, prefix, members); + auto parser = extractNarDataFilter(*source, prefix, members); while (parser.next()) { // ignore raw data } diff --git a/src/hydra-queue-runner/nar-extractor.hh b/src/hydra-queue-runner/nar-extractor.hh index dc9d00f46..f05cbe9de 100644 --- a/src/hydra-queue-runner/nar-extractor.hh +++ b/src/hydra-queue-runner/nar-extractor.hh @@ -18,7 +18,7 @@ using NarMemberDatas = std::map; /* Read a NAR from a source and get to some info about every file inside the NAR. */ void extractNarData( - nix::Source & source, + nix::box_ptr & source, const nix::Path & prefix, NarMemberDatas & members); diff --git a/src/hydra-queue-runner/queue-monitor.cc b/src/hydra-queue-runner/queue-monitor.cc index b66becb0f..6231449bc 100644 --- a/src/hydra-queue-runner/queue-monitor.cc +++ b/src/hydra-queue-runner/queue-monitor.cc @@ -1,5 +1,6 @@ #include "hydra-build-result.hh" #include "lix/libstore/globals.hh" +#include "lix/libutil/async.hh" #include "lix/libutil/thread-pool.hh" #include "state.hh" @@ -40,13 +41,15 @@ void State::queueMonitorLoop(Connection & conn) auto destStore = getDestStore(); + AsyncIoRoot aio; + bool quit = false; while (!quit) { auto t_before_work = std::chrono::steady_clock::now(); localStore->clearPathInfoCache(); - bool done = getQueuedBuilds(conn, destStore); + bool done = getQueuedBuilds(aio, conn, destStore); if (buildOne && buildOneDone) quit = true; @@ -93,7 +96,7 @@ struct PreviousFailure : public std::exception { }; -bool State::getQueuedBuilds(Connection & conn, +bool State::getQueuedBuilds(AsyncIoRoot & aio, Connection & conn, ref destStore) { prom.queue_checks_started.Increment(); @@ -176,7 +179,7 @@ bool State::getQueuedBuilds(Connection & conn, /* Create steps for this derivation and its dependencies. */ try { - step = createStep(destStore, conn, build, build->drvPath, + step = createStep(aio, destStore, conn, build, build->drvPath, build, nullptr, finishedDrvs, newSteps, newRunnable); } catch (PreviousFailure & ex) { @@ -434,7 +437,7 @@ std::map> State::getMissingRemotePaths( } -Step::ptr State::createStep(ref destStore, +Step::ptr State::createStep(AsyncIoRoot & aio, ref destStore, Connection & conn, Build::ptr build, const StorePath & drvPath, Build::ptr referringBuild, Step::ptr referringStep, std::set & finishedDrvs, std::set & newSteps, std::set & newRunnable) @@ -571,7 +574,7 @@ Step::ptr State::createStep(ref destStore, printInfo("substituting output ‘%1%’ of ‘%2%’", localStore->printStorePath(path), localStore->printStorePath(drvPath)); - localStore->ensurePath(path); + aio.blockOn(localStore->ensurePath(path)); // FIXME: should copy directly from substituter to destStore. } @@ -611,7 +614,7 @@ Step::ptr State::createStep(ref destStore, /* Create steps for the dependencies. */ for (auto & i : step->drv->inputDrvs.map) { - auto dep = createStep(destStore, conn, build, i.first, nullptr, step, finishedDrvs, newSteps, newRunnable); + auto dep = createStep(aio, destStore, conn, build, i.first, nullptr, step, finishedDrvs, newSteps, newRunnable); if (dep) { auto step_(step->state.lock()); step_->deps.insert(dep); diff --git a/src/hydra-queue-runner/state.hh b/src/hydra-queue-runner/state.hh index 88fe262f4..9e5f21b6f 100644 --- a/src/hydra-queue-runner/state.hh +++ b/src/hydra-queue-runner/state.hh @@ -22,6 +22,7 @@ #include "lix/libstore/pathlocks.hh" #include "lix/libstore/serve-protocol.hh" #include "lix/libstore/store-api.hh" +#include "lix/libutil/async.hh" #include "lix/libutil/pool.hh" #include "lix/libutil/sync.hh" #include "nar-extractor.hh" @@ -541,7 +542,7 @@ private: void queueMonitorLoop(Connection & conn); /* Check the queue for new builds. */ - bool getQueuedBuilds(Connection & conn, nix::ref destStore); + bool getQueuedBuilds(nix::AsyncIoRoot & aio, Connection & conn, nix::ref destStore); /* Handle cancellation, deletion and priority bumps. */ void processQueueChange(Connection & conn); @@ -555,7 +556,7 @@ private: nix::ref destStore, const std::map> & paths); - Step::ptr createStep(nix::ref store, + Step::ptr createStep(nix::AsyncIoRoot & aio, nix::ref store, Connection & conn, Build::ptr build, const nix::StorePath & drvPath, Build::ptr referringBuild, Step::ptr referringStep, std::set & finishedDrvs, std::set & newSteps, std::set & newRunnable);