From 19b8502cedbceea2704bd8836a932f12cb7ac32f Mon Sep 17 00:00:00 2001 From: eldritch horrors Date: Tue, 4 Mar 2025 23:08:35 +0100 Subject: [PATCH] libstore: asyncify Store::read{,Invalid}Derivation Change-Id: Id1af47e5f35ea81ede9c47dba8f50045a2b9bde2 --- lix/legacy/build-remote.cc | 2 +- lix/libcmd/built-path.cc | 5 +++-- lix/libcmd/repl.cc | 2 +- lix/libexpr/eval.cc | 2 +- lix/libexpr/primops.cc | 5 +++-- lix/libstore/build/derivation-goal.cc | 4 ++-- lix/libstore/daemon.cc | 2 +- lix/libstore/derivations.cc | 2 +- lix/libstore/local-store.cc | 6 ++++-- lix/libstore/misc.cc | 5 +++-- lix/libstore/remote-store.cc | 2 +- lix/libstore/store-api.cc | 21 +++++++++++-------- lix/libstore/store-api.hh | 4 ++-- lix/nix/derivation-show.cc | 2 +- subprojects/nix-eval-jobs/src/constituents.cc | 4 ++-- subprojects/nix-eval-jobs/src/drv.cc | 2 +- 16 files changed, 39 insertions(+), 31 deletions(-) diff --git a/lix/legacy/build-remote.cc b/lix/legacy/build-remote.cc index dd994fdb7..d0140d66f 100644 --- a/lix/legacy/build-remote.cc +++ b/lix/legacy/build-remote.cc @@ -297,7 +297,7 @@ connected: uploadLock.reset(); - auto drv = store->readDerivation(*drvPath); + auto drv = aio.blockOn(store->readDerivation(*drvPath)); std::optional optResult; diff --git a/lix/libcmd/built-path.cc b/lix/libcmd/built-path.cc index 2e4caa804..d8ccf4b03 100644 --- a/lix/libcmd/built-path.cc +++ b/lix/libcmd/built-path.cc @@ -142,8 +142,9 @@ try { // NOLINTNEXTLINE(cppcoreguidelines-avoid-capturing-lambda-coroutines) [&](const BuiltPath::Built & p) -> kj::Promise> { try { - auto drvHashes = - TRY_AWAIT(staticOutputHashes(store, store.readDerivation(p.drvPath->outPath()))); + auto drvHashes = TRY_AWAIT( + staticOutputHashes(store, TRY_AWAIT(store.readDerivation(p.drvPath->outPath()))) + ); for (auto& [outputName, outputPath] : p.outputs) { if (experimentalFeatureSettings.isEnabled( Xp::CaDerivations)) { diff --git a/lix/libcmd/repl.cc b/lix/libcmd/repl.cc index 8383d1ab8..5353bb16e 100644 --- a/lix/libcmd/repl.cc +++ b/lix/libcmd/repl.cc @@ -776,7 +776,7 @@ ProcessLineResult NixRepl::processLine(std::string line) .outputs = OutputsSpec::All { }, }, })); - auto drv = evaluator.store->readDerivation(drvPath); + auto drv = state.aio.blockOn(evaluator.store->readDerivation(drvPath)); logger->cout("\nThis derivation produced the following outputs:"); for (auto & [outputName, outputPath] : state.aio.blockOn(evaluator.store->queryDerivationOutputMap(drvPath))) diff --git a/lix/libexpr/eval.cc b/lix/libexpr/eval.cc index 312464721..e11ac712a 100644 --- a/lix/libexpr/eval.cc +++ b/lix/libexpr/eval.cc @@ -921,7 +921,7 @@ std::string EvalState::mkSingleDerivedPathStringRaw( [&](const SingleDerivedPath::Built & b) { auto optStaticOutputPath = std::visit(overloaded { [&](const SingleDerivedPath::Opaque & o) { - auto drv = ctx.store->readDerivation(o.path); + auto drv = aio.blockOn(ctx.store->readDerivation(o.path)); auto i = drv.outputs.find(b.output); if (i == drv.outputs.end()) throw Error("derivation '%s' does not have output '%s'", b.drvPath->to_string(*ctx.store), b.output); diff --git a/lix/libexpr/primops.cc b/lix/libexpr/primops.cc index 5da6f2453..9121d27fd 100644 --- a/lix/libexpr/primops.cc +++ b/lix/libexpr/primops.cc @@ -190,7 +190,7 @@ static void import(EvalState & state, const PosIdx pos, Value & vPath, Value * v }; if (auto storePath = isValidDerivationInStore()) { - Derivation drv = state.ctx.store->readDerivation(*storePath); + Derivation drv = state.aio.blockOn(state.ctx.store->readDerivation(*storePath)); auto attrs = state.ctx.buildBindings(3 + drv.outputs.size()); attrs.alloc(state.ctx.s.drvPath).mkString(path2, { NixStringContextElem::DrvDeep { .drvPath = *storePath }, @@ -982,7 +982,8 @@ drvName, Bindings * attrs, Value & v) for (auto & j : refs) { drv.inputSrcs.insert(j); if (j.isDerivation()) { - drv.inputDrvs.map[j].value = state.ctx.store->readDerivation(j).outputNames(); + drv.inputDrvs.map[j].value = + state.aio.blockOn(state.ctx.store->readDerivation(j)).outputNames(); } } }, diff --git a/lix/libstore/build/derivation-goal.cc b/lix/libstore/build/derivation-goal.cc index c0ce5d50b..4c7b210ea 100644 --- a/lix/libstore/build/derivation-goal.cc +++ b/lix/libstore/build/derivation-goal.cc @@ -206,7 +206,7 @@ try { */ for (auto * drvStore : { &worker.evalStore, &worker.store }) { if (drvStore->isValidPath(drvPath)) { - drv = std::make_unique(drvStore->readDerivation(drvPath)); + drv = std::make_unique(TRY_AWAIT(drvStore->readDerivation(drvPath))); break; } } @@ -412,7 +412,7 @@ try { /* Ensure that pure, non-fixed-output derivations don't depend on impure derivations. */ if (experimentalFeatureSettings.isEnabled(Xp::ImpureDerivations) && drv->type().isPure() && !drv->type().isFixed()) { - auto inputDrv = worker.evalStore.readDerivation(inputDrvPath); + auto inputDrv = TRY_AWAIT(worker.evalStore.readDerivation(inputDrvPath)); if (!inputDrv.type().isPure()) throw Error("pure derivation '%s' depends on impure derivation '%s'", worker.store.printStorePath(drvPath), diff --git a/lix/libstore/daemon.cc b/lix/libstore/daemon.cc index fbdae0b7e..fe053c233 100644 --- a/lix/libstore/daemon.cc +++ b/lix/libstore/daemon.cc @@ -378,7 +378,7 @@ static void performOp(AsyncIoRoot & aio, TunnelLogger * logger, ref store REMOVE_AFTER_DROPPING_PROTO_MINOR(31); auto path = store->parseStorePath(readString(from)); logger->startWork(); - auto names = store->readDerivation(path).outputNames(); + auto names = aio.blockOn(store->readDerivation(path)).outputNames(); logger->stopWork(); to << names; break; diff --git a/lix/libstore/derivations.cc b/lix/libstore/derivations.cc index 5463d7f48..29dd6ee40 100644 --- a/lix/libstore/derivations.cc +++ b/lix/libstore/derivations.cc @@ -794,7 +794,7 @@ try { } auto h = TRY_AWAIT(hashDerivationModulo( store, - store.readInvalidDerivation(drvPath), + TRY_AWAIT(store.readInvalidDerivation(drvPath)), false)); // Cache it drvHashes.lock()->insert_or_assign(drvPath, h); diff --git a/lix/libstore/local-store.cc b/lix/libstore/local-store.cc index 07eaa295b..8f52b51de 100644 --- a/lix/libstore/local-store.cc +++ b/lix/libstore/local-store.cc @@ -887,7 +887,7 @@ try { efficiently query whether a path is an output of some derivation. */ if (info.path.isDerivation()) { - auto drv = readInvalidDerivation(info.path); + auto drv = TRY_AWAIT(readInvalidDerivation(info.path)); /* Verify that the output paths in the derivation are correct (i.e., follow the scheme for computing output paths from @@ -1228,7 +1228,9 @@ try { for (auto & [_, i] : infos) if (i.path.isDerivation()) { // FIXME: inefficient; we already loaded the derivation in addValidPath(). - TRY_AWAIT(readInvalidDerivation(i.path).checkInvariants(*this, i.path)); + TRY_AWAIT( + TRY_AWAIT(readInvalidDerivation(i.path)).checkInvariants(*this, i.path) + ); } /* Do a topological sort of the paths. This will throw an diff --git a/lix/libstore/misc.cc b/lix/libstore/misc.cc index 0dbac91dc..acd2d79ea 100644 --- a/lix/libstore/misc.cc +++ b/lix/libstore/misc.cc @@ -424,8 +424,9 @@ try { ) -> kj::Promise> { try { if (!inputNode.value.empty()) { - auto outputHashes = - TRY_AWAIT(staticOutputHashes(evalStore, evalStore.readDerivation(inputDrv))); + auto outputHashes = TRY_AWAIT( + staticOutputHashes(evalStore, TRY_AWAIT(evalStore.readDerivation(inputDrv))) + ); for (const auto & outputName : inputNode.value) { auto outputHash = get(outputHashes, outputName); if (!outputHash) diff --git a/lix/libstore/remote-store.cc b/lix/libstore/remote-store.cc index 13ea7e16e..95eda7c56 100644 --- a/lix/libstore/remote-store.cc +++ b/lix/libstore/remote-store.cc @@ -757,7 +757,7 @@ try { OutputPathMap outputs; auto drvPath = TRY_AWAIT(resolveDerivedPath(*evalStore, *bfd.drvPath)); - auto drv = evalStore->readDerivation(drvPath); + auto drv = TRY_AWAIT(evalStore->readDerivation(drvPath)); const auto outputHashes = TRY_AWAIT(staticOutputHashes(*evalStore, drv)); // FIXME: expensive auto built = TRY_AWAIT(resolveDerivedPath(*this, bfd, &*evalStore)); diff --git a/lix/libstore/store-api.cc b/lix/libstore/store-api.cc index d906857d1..c70e9ddaa 100644 --- a/lix/libstore/store-api.cc +++ b/lix/libstore/store-api.cc @@ -510,7 +510,7 @@ kj::Promise>>> Store::queryStaticPartialDerivationOutputMap(const StorePath & path) try { std::map> outputs; - auto drv = readInvalidDerivation(path); + auto drv = TRY_AWAIT(readInvalidDerivation(path)); for (auto & [outputName, output] : drv.outputsAndOptPaths(*this)) { outputs.emplace(outputName, output.second); } @@ -529,7 +529,7 @@ try { if (!experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) co_return outputs; - auto drv = evalStore.readInvalidDerivation(path); + auto drv = TRY_AWAIT(evalStore.readInvalidDerivation(path)); auto drvHashes = TRY_AWAIT(staticOutputHashes(*this, drv)); for (auto & [outputName, hash] : drvHashes) { auto realisation = TRY_AWAIT(queryRealisation(DrvOutput{hash, outputName})); @@ -1382,21 +1382,24 @@ std::string showPaths(const PathSet & paths) kj::Promise> Store::derivationFromPath(const StorePath & drvPath) try { TRY_AWAIT(ensurePath(drvPath)); - co_return readDerivation(drvPath); + co_return TRY_AWAIT(readDerivation(drvPath)); } catch (...) { co_return result::current_exception(); } -Derivation readDerivationCommon(Store& store, const StorePath& drvPath, bool requireValidPath) -{ +kj::Promise> +readDerivationCommon(Store& store, const StorePath& drvPath, bool requireValidPath) +try { auto accessor = store.getFSAccessor(); try { - return parseDerivation(store, + co_return parseDerivation(store, accessor->readFile(store.printStorePath(drvPath), requireValidPath), Derivation::nameFromPath(drvPath)); } catch (FormatError & e) { throw Error("error parsing derivation '%s': %s", store.printStorePath(drvPath), e.msg()); } +} catch (...) { + co_return result::current_exception(); } kj::Promise>> Store::getBuildDerivationPath(const StorePath & path) @@ -1414,7 +1417,7 @@ try { if (!experimentalFeatureSettings.isEnabled(Xp::CaDerivations) || !isValidPath(path)) co_return path; - auto drv = readDerivation(path); + auto drv = TRY_AWAIT(readDerivation(path)); if (!drv.type().hasKnownOutputPaths()) { // The build log is actually attached to the corresponding // resolved derivation, so we need to get it first @@ -1428,10 +1431,10 @@ try { co_return result::current_exception(); } -Derivation Store::readDerivation(const StorePath & drvPath) +kj::Promise> Store::readDerivation(const StorePath & drvPath) { return readDerivationCommon(*this, drvPath, true); } -Derivation Store::readInvalidDerivation(const StorePath & drvPath) +kj::Promise> Store::readInvalidDerivation(const StorePath & drvPath) { return readDerivationCommon(*this, drvPath, false); } } diff --git a/lix/libstore/store-api.hh b/lix/libstore/store-api.hh index 24ed0d69b..2448005eb 100644 --- a/lix/libstore/store-api.hh +++ b/lix/libstore/store-api.hh @@ -770,12 +770,12 @@ public: /** * Read a derivation (which must already be valid). */ - Derivation readDerivation(const StorePath & drvPath); + kj::Promise> readDerivation(const StorePath & drvPath); /** * Read a derivation from a potentially invalid path. */ - Derivation readInvalidDerivation(const StorePath & drvPath); + kj::Promise> readInvalidDerivation(const StorePath & drvPath); /** * @param [out] out Place in here the set of all store paths in the diff --git a/lix/nix/derivation-show.cc b/lix/nix/derivation-show.cc index 49d415479..b6405e75f 100644 --- a/lix/nix/derivation-show.cc +++ b/lix/nix/derivation-show.cc @@ -56,7 +56,7 @@ struct CmdShowDerivation : InstallablesCommand if (!drvPath.isDerivation()) continue; jsonRoot[store->printStorePath(drvPath)] = - store->readDerivation(drvPath).toJSON(*store); + aio().blockOn(store->readDerivation(drvPath)).toJSON(*store); } logger->cout(jsonRoot.dump(2)); } diff --git a/subprojects/nix-eval-jobs/src/constituents.cc b/subprojects/nix-eval-jobs/src/constituents.cc index a731a6ca5..8dabf0c29 100644 --- a/subprojects/nix-eval-jobs/src/constituents.cc +++ b/subprojects/nix-eval-jobs/src/constituents.cc @@ -125,13 +125,13 @@ void rewriteAggregates(std::map &jobs, for (const auto &aggregateJob : aggregateJobs) { auto &job = jobs.find(aggregateJob.name)->second; auto drvPath = store->parseStorePath(std::string(job["drvPath"])); - auto drv = store->readDerivation(drvPath); + auto drv = aio.blockOn(store->readDerivation(drvPath)); if (aggregateJob.brokenJobs.empty()) { for (const auto &childJobName : aggregateJob.dependencies) { auto childDrvPath = store->parseStorePath( std::string(jobs.find(childJobName)->second["drvPath"])); - auto childDrv = store->readDerivation(childDrvPath); + auto childDrv = aio.blockOn(store->readDerivation(childDrvPath)); job["constituents"].push_back( store->printStorePath(childDrvPath)); drv.inputDrvs.map[childDrvPath].value = { diff --git a/subprojects/nix-eval-jobs/src/drv.cc b/subprojects/nix-eval-jobs/src/drv.cc index 00a732aee..240465123 100644 --- a/subprojects/nix-eval-jobs/src/drv.cc +++ b/subprojects/nix-eval-jobs/src/drv.cc @@ -100,7 +100,7 @@ Drv::Drv(std::string &attrPath, nix::EvalState &state, nix::DrvInfo &drvInfo, drvPath = localStore->printStorePath(drvInfo.requireDrvPath(state)); - auto drv = localStore->readDerivation(drvInfo.requireDrvPath(state)); + auto drv = state.aio.blockOn(localStore->readDerivation(drvInfo.requireDrvPath(state))); for (const auto &[inputDrvPath, inputNode] : drv.inputDrvs.map) { std::set inputDrvOutputs; for (auto &outputName : inputNode.value) {