diff --git a/lix/legacy/build-remote.cc b/lix/legacy/build-remote.cc index 234e58ecd..aa05899b1 100644 --- a/lix/legacy/build-remote.cc +++ b/lix/legacy/build-remote.cc @@ -325,18 +325,18 @@ connected: // output ids, which break CA derivations if (!drv.inputDrvs.map.empty()) drv.inputSrcs = store->parseStorePathSet(inputs); - optResult = sshStore->buildDerivation(*drvPath, (const BasicDerivation &) drv); + optResult = aio.blockOn(sshStore->buildDerivation(*drvPath, (const BasicDerivation &) drv)); auto & result = *optResult; if (!result.success()) throw Error("build of '%s' on '%s' failed: %s", store->printStorePath(*drvPath), storeUri, result.errorMsg); } else { copyClosure(*store, *sshStore, StorePathSet {*drvPath}, NoRepair, NoCheckSigs, substitute); - auto res = sshStore->buildPathsWithResults({ + auto res = aio.blockOn(sshStore->buildPathsWithResults({ DerivedPath::Built { .drvPath = makeConstantStorePathRef(*drvPath), .outputs = OutputsSpec::All {}, } - }); + })); // One path to build should produce exactly one build result assert(res.size() == 1); optResult = std::move(res[0]); diff --git a/lix/legacy/nix-build.cc b/lix/legacy/nix-build.cc index 07d488b3f..8b7f312b6 100644 --- a/lix/legacy/nix-build.cc +++ b/lix/legacy/nix-build.cc @@ -319,7 +319,7 @@ static void main_nix_build(AsyncIoRoot & aio, std::string programName, Strings a printMissing(ref(store), willBuild, willSubstitute, unknown, downloadSize, narSize); if (!dryRun) - store->buildPaths(paths, buildMode, evalStore); + aio.blockOn(store->buildPaths(paths, buildMode, evalStore)); }; if (runEnv) { diff --git a/lix/legacy/nix-env.cc b/lix/legacy/nix-env.cc index 466223b17..3cbd98c78 100644 --- a/lix/legacy/nix-env.cc +++ b/lix/legacy/nix-env.cc @@ -790,7 +790,9 @@ static void opSet(Globals & globals, Strings opFlags, Strings opArgs) }; printMissing(globals.state->store, paths); if (globals.dryRun) return; - globals.state->store->buildPaths(paths, globals.state->repair ? bmRepair : bmNormal); + globals.aio.blockOn( + globals.state->store->buildPaths(paths, globals.state->repair ? bmRepair : bmNormal) + ); debug("switching to new user environment"); Path generation = createGeneration( diff --git a/lix/legacy/nix-store.cc b/lix/legacy/nix-store.cc index 7eaaca5a1..71018f3b7 100644 --- a/lix/legacy/nix-store.cc +++ b/lix/legacy/nix-store.cc @@ -1,6 +1,7 @@ #include "lix/libutil/archive.hh" #include "lix/libstore/derivations.hh" #include "dotgraph.hh" +#include "lix/libutil/async.hh" #include "lix/libutil/exit.hh" #include "lix/libstore/globals.hh" #include "lix/libstore/build-result.hh" @@ -66,7 +67,7 @@ try { auto store2 = std::dynamic_pointer_cast(store); if (path.path.isDerivation()) { - if (build) store->buildPaths({path.toDerivedPath()}); + if (build) TRY_AWAIT(store->buildPaths({path.toDerivedPath()})); auto outputPaths = store->queryDerivationOutputMap(path.path); Derivation drv = store->derivationFromPath(path.path); rootNr++; @@ -100,7 +101,7 @@ try { } else { - if (build) store->ensurePath(path.path); + if (build) TRY_AWAIT(store->ensurePath(path.path)); else if (!store->isValidPath(path.path)) throw Error("path '%s' does not exist and cannot be created", store->printStorePath(path.path)); if (store2) { @@ -159,7 +160,7 @@ static void opRealise(AsyncIoRoot & aio, Strings opFlags, Strings opArgs) if (dryRun) return; /* Build all paths at the same time to exploit parallelism. */ - store->buildPaths(toDerivedPaths(paths), buildMode); + aio.blockOn(store->buildPaths(toDerivedPaths(paths), buildMode)); if (!ignoreUnknown) for (auto & i : paths) { @@ -790,7 +791,7 @@ static void opRepairPath(AsyncIoRoot & aio, Strings opFlags, Strings opArgs) throw UsageError("no flags expected"); for (auto & i : opArgs) - store->repairPath(store->followLinksToStorePath(i)); + aio.blockOn(store->repairPath(store->followLinksToStorePath(i))); } /* Optimise the disk space usage of the Nix store by hard-linking @@ -932,7 +933,7 @@ static void opServe(AsyncIoRoot & aio, Strings opFlags, Strings opArgs) try { MonitorFdHup monitor(in.fd); - store->buildPaths(toDerivedPaths(paths)); + aio.blockOn(store->buildPaths(toDerivedPaths(paths))); out << 0; } catch (Error & e) { assert(e.info().status); @@ -952,7 +953,7 @@ static void opServe(AsyncIoRoot & aio, Strings opFlags, Strings opArgs) getBuildSettings(); MonitorFdHup monitor(in.fd); - auto status = store->buildDerivation(drvPath, drv); + auto status = aio.blockOn(store->buildDerivation(drvPath, drv)); out << ServeProto::write(*store, wconn, status); break; diff --git a/lix/legacy/user-env.cc b/lix/legacy/user-env.cc index 4e0fa9442..fb94bbb1b 100644 --- a/lix/legacy/user-env.cc +++ b/lix/legacy/user-env.cc @@ -28,9 +28,9 @@ bool createUserEnv(EvalState & state, DrvInfos & elems, drvsToBuild.push_back({*drvPath}); debug("building user environment dependencies"); - state.ctx.store->buildPaths( + state.aio.blockOn(state.ctx.store->buildPaths( toDerivedPaths(drvsToBuild), - state.ctx.repair ? bmRepair : bmNormal); + state.ctx.repair ? bmRepair : bmNormal)); /* Construct the whole top level derivation. */ StorePathSet references; @@ -67,7 +67,7 @@ bool createUserEnv(EvalState & state, DrvInfos & elems, /* This is only necessary when installing store paths, e.g., `nix-env -i /nix/store/abcd...-foo'. */ state.ctx.store->addTempRoot(*j.second); - state.ctx.store->ensurePath(*j.second); + state.aio.blockOn(state.ctx.store->ensurePath(*j.second)); references.insert(*j.second); } @@ -125,9 +125,9 @@ bool createUserEnv(EvalState & state, DrvInfos & elems, debug("building user environment"); std::vector topLevelDrvs; topLevelDrvs.push_back({topLevelDrv}); - state.ctx.store->buildPaths( + state.aio.blockOn(state.ctx.store->buildPaths( toDerivedPaths(topLevelDrvs), - state.ctx.repair ? bmRepair : bmNormal); + state.ctx.repair ? bmRepair : bmNormal)); /* Switch the current user environment to the output path. */ auto store2 = state.ctx.store.dynamic_pointer_cast(); diff --git a/lix/libcmd/installables.cc b/lix/libcmd/installables.cc index c4454f716..33fd3529e 100644 --- a/lix/libcmd/installables.cc +++ b/lix/libcmd/installables.cc @@ -652,7 +652,8 @@ std::vector, BuiltPathWithResult>> Installable::build if (settings.printMissing) printMissing(store, pathsToBuild, lvlInfo); - auto buildResults = store->buildPathsWithResults(pathsToBuild, bMode, evalStore); + auto buildResults = + state.aio.blockOn(store->buildPathsWithResults(pathsToBuild, bMode, evalStore)); throwBuildErrors(buildResults, *store); for (auto & buildResult : buildResults) { for (auto & aux : backmap[buildResult.path]) { diff --git a/lix/libcmd/repl.cc b/lix/libcmd/repl.cc index 86feec53f..a03bbe28a 100644 --- a/lix/libcmd/repl.cc +++ b/lix/libcmd/repl.cc @@ -756,12 +756,12 @@ ProcessLineResult NixRepl::processLine(std::string line) logger->pause(); }); - evaluator.store->buildPaths({ + state.aio.blockOn(evaluator.store->buildPaths({ DerivedPath::Built { .drvPath = makeConstantStorePathRef(drvPath), .outputs = OutputsSpec::All { }, }, - }); + })); auto drv = evaluator.store->readDerivation(drvPath); logger->cout("\nThis derivation produced the following outputs:"); for (auto & [outputName, outputPath] : evaluator.store->queryDerivationOutputMap(drvPath)) { diff --git a/lix/libexpr/primops.cc b/lix/libexpr/primops.cc index f9e363e34..2c1ef5fe3 100644 --- a/lix/libexpr/primops.cc +++ b/lix/libexpr/primops.cc @@ -10,6 +10,7 @@ #include "lix/libexpr/json-to-value.hh" #include "lix/libstore/names.hh" #include "lix/libstore/path-references.hh" +#include "lix/libutil/async.hh" #include "lix/libutil/processes.hh" #include "lix/libstore/store-api.hh" #include "lix/libexpr/value-to-json.hh" @@ -82,7 +83,7 @@ try { /* Build/substitute the context. */ std::vector buildReqs; for (auto & d : drvs) buildReqs.emplace_back(DerivedPath { d }); - buildStore->buildPaths(buildReqs, bmNormal, store); + TRY_AWAIT(buildStore->buildPaths(buildReqs, bmNormal, store)); StorePathSet outputsToCopyAndAllow; @@ -1184,7 +1185,7 @@ static void prim_storePath(EvalState & state, const PosIdx pos, Value * * args, .atPos(pos).debugThrow(); auto path2 = state.ctx.store->toStorePath(path.abs()).first; if (!settings.readOnlyMode) - state.ctx.store->ensurePath(path2); + state.aio.blockOn(state.ctx.store->ensurePath(path2)); context.insert(NixStringContextElem::Opaque { .path = path2 }); v.mkString(path.abs(), context); } diff --git a/lix/libexpr/primops/context.cc b/lix/libexpr/primops/context.cc index 7dfce3214..3f2a42932 100644 --- a/lix/libexpr/primops/context.cc +++ b/lix/libexpr/primops/context.cc @@ -182,7 +182,7 @@ static void prim_appendContext(EvalState & state, const PosIdx pos, Value * * ar ).atPos(i.pos).debugThrow(); auto namePath = state.ctx.store->parseStorePath(name); if (!settings.readOnlyMode) - state.ctx.store->ensurePath(namePath); + state.aio.blockOn(state.ctx.store->ensurePath(namePath)); state.forceAttrs(*i.value, i.pos, "while evaluating the value of a string context"); auto iter = i.value->attrs->find(state.ctx.s.path); if (iter != i.value->attrs->end()) { diff --git a/lix/libfetchers/fetchers.cc b/lix/libfetchers/fetchers.cc index 9434dfb6f..866794f9c 100644 --- a/lix/libfetchers/fetchers.cc +++ b/lix/libfetchers/fetchers.cc @@ -1,6 +1,7 @@ #include "lix/libfetchers/fetchers.hh" #include "lix/libfetchers/builtin-fetchers.hh" #include "lix/libstore/store-api.hh" +#include "lix/libutil/async.hh" #include "lix/libutil/source-path.hh" #include "lix/libfetchers/fetch-to-store.hh" @@ -134,7 +135,7 @@ std::pair Input::fetch(ref store) const try { auto storePath = computeStorePath(*store); - store->ensurePath(storePath); + RUN_ASYNC_IN_NEW_THREAD(store->ensurePath(storePath)); debug("using substituted/cached input '%s' in '%s'", to_string(), store->printStorePath(storePath)); diff --git a/lix/libstore/build/entry-points.cc b/lix/libstore/build/entry-points.cc index db4c3f891..4dd471e40 100644 --- a/lix/libstore/build/entry-points.cc +++ b/lix/libstore/build/entry-points.cc @@ -3,18 +3,21 @@ #include "lix/libstore/build/derivation-goal.hh" #include "lix/libstore/local-store.hh" #include "lix/libutil/async.hh" +#include "lix/libutil/result.hh" #include "lix/libutil/strings.hh" namespace nix { -void Store::buildPaths(const std::vector & reqs, BuildMode buildMode, std::shared_ptr evalStore) -{ - auto results = RUN_ASYNC_IN_NEW_THREAD(processGoals(*this, evalStore ? *evalStore : *this, [&](GoalFactory & gf) { - Worker::Targets goals; - for (auto & br : reqs) - goals.emplace_back(gf.makeGoal(br, buildMode)); - return goals; - })); +kj::Promise> Store::buildPaths(const std::vector & reqs, BuildMode buildMode, std::shared_ptr evalStore) +try { + auto results = + TRY_AWAIT(processGoals(*this, evalStore ? *evalStore : *this, [&](GoalFactory & gf) { + Worker::Targets goals; + for (auto & br : reqs) { + goals.emplace_back(gf.makeGoal(br, buildMode)); + } + return goals; + })); StringSet failed; std::shared_ptr ex; @@ -38,58 +41,69 @@ void Store::buildPaths(const std::vector & reqs, BuildMode buildMod if (ex) logError(ex->info()); throw Error(results.failingExitStatus, "build of %s failed", concatStringsSep(", ", quoteStrings(failed))); } + + co_return result::success(); +} catch (...) { + co_return result::current_exception(); } -std::vector Store::buildPathsWithResults( +kj::Promise>> Store::buildPathsWithResults( const std::vector & reqs, BuildMode buildMode, std::shared_ptr evalStore) -{ - auto goals = RUN_ASYNC_IN_NEW_THREAD(processGoals(*this, evalStore ? *evalStore : *this, [&](GoalFactory & gf) { - Worker::Targets goals; - for (const auto & req : reqs) { - goals.emplace_back(gf.makeGoal(req, buildMode)); - } - return goals; - })).goals; +try { + auto goals = + TRY_AWAIT(processGoals(*this, evalStore ? *evalStore : *this, [&](GoalFactory & gf) { + Worker::Targets goals; + for (const auto & req : reqs) { + goals.emplace_back(gf.makeGoal(req, buildMode)); + } + return goals; + })).goals; std::vector results; for (auto && [goalIdx, req] : enumerate(reqs)) results.emplace_back(goals[goalIdx].result.restrictTo(req)); - return results; + co_return results; +} catch (...) { + co_return result::current_exception(); } -BuildResult Store::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, +kj::Promise> Store::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode) -{ +try { try { - auto results = RUN_ASYNC_IN_NEW_THREAD(processGoals(*this, *this, [&](GoalFactory & gf) { + auto results = TRY_AWAIT(processGoals(*this, *this, [&](GoalFactory & gf) { Worker::Targets goals; - goals.emplace_back(gf.makeBasicDerivationGoal(drvPath, drv, OutputsSpec::All{}, buildMode)); + goals.emplace_back( + gf.makeBasicDerivationGoal(drvPath, drv, OutputsSpec::All{}, buildMode) + ); return goals; })); auto & result = results.goals.begin()->second; - return result.result.restrictTo(DerivedPath::Built { + co_return result.result.restrictTo(DerivedPath::Built { .drvPath = makeConstantStorePathRef(drvPath), .outputs = OutputsSpec::All {}, }); } catch (Error & e) { - return BuildResult { + co_return BuildResult { .status = BuildResult::MiscFailure, .errorMsg = e.msg(), }; }; +} catch (...) { + co_return result::current_exception(); } -void Store::ensurePath(const StorePath & path) -{ +kj::Promise> Store::ensurePath(const StorePath & path) +try { /* If the path is already valid, we're done. */ - if (isValidPath(path)) return; + if (isValidPath(path)) co_return result::success(); - auto results = RUN_ASYNC_IN_NEW_THREAD(processGoals(*this, *this, [&](GoalFactory & gf) { + auto results = TRY_AWAIT(processGoals(*this, *this, [&](GoalFactory & gf) { Worker::Targets goals; goals.emplace_back(gf.makePathSubstitutionGoal(path)); return goals; @@ -103,12 +117,16 @@ void Store::ensurePath(const StorePath & path) } else throw Error(results.failingExitStatus, "path '%s' does not exist and cannot be created", printStorePath(path)); } + + co_return result::success(); +} catch (...) { + co_return result::current_exception(); } -void Store::repairPath(const StorePath & path) -{ - auto results = RUN_ASYNC_IN_NEW_THREAD(processGoals(*this, *this, [&](GoalFactory & gf) { +kj::Promise> Store::repairPath(const StorePath & path) +try { + auto results = TRY_AWAIT(processGoals(*this, *this, [&](GoalFactory & gf) { Worker::Targets goals; goals.emplace_back(gf.makePathSubstitutionGoal(path, Repair)); return goals; @@ -120,7 +138,7 @@ void Store::repairPath(const StorePath & path) deriver, then rebuild the deriver. */ auto info = queryPathInfo(path); if (info->deriver && isValidPath(*info->deriver)) { - RUN_ASYNC_IN_NEW_THREAD(processGoals(*this, *this, [&](GoalFactory & gf) { + TRY_AWAIT(processGoals(*this, *this, [&](GoalFactory & gf) { Worker::Targets goals; goals.emplace_back(gf.makeGoal( DerivedPath::Built{ @@ -135,6 +153,10 @@ void Store::repairPath(const StorePath & path) } else throw Error(results.failingExitStatus, "cannot repair path '%s'", printStorePath(path)); } + + co_return result::success(); +} catch (...) { + co_return result::current_exception(); } } diff --git a/lix/libstore/build/local-derivation-goal.cc b/lix/libstore/build/local-derivation-goal.cc index 7e5f97f8d..f3f5d7549 100644 --- a/lix/libstore/build/local-derivation-goal.cc +++ b/lix/libstore/build/local-derivation-goal.cc @@ -1092,11 +1092,14 @@ struct RestrictedStore : public virtual IndirectRootStore, public virtual GcStor return LocalFSStore::narFromPath(path); } - void ensurePath(const StorePath & path) override - { + kj::Promise> ensurePath(const StorePath & path) override + try { if (!goal.isAllowed(path)) throw InvalidPath("cannot substitute unknown path '%s' in recursive Nix", printStorePath(path)); /* Nothing to be done; 'path' must already be valid. */ + return {result::success()}; + } catch (...) { + return {result::current_exception()}; } void registerDrvOutput(const Realisation & info) override @@ -1113,18 +1116,25 @@ struct RestrictedStore : public virtual IndirectRootStore, public virtual GcStor return next->queryRealisation(id); } - void buildPaths(const std::vector & paths, BuildMode buildMode, std::shared_ptr evalStore) override - { - for (auto & result : buildPathsWithResults(paths, buildMode, evalStore)) + kj::Promise> buildPaths( + const std::vector & paths, + BuildMode buildMode, + std::shared_ptr evalStore + ) override + try { + for (auto & result : TRY_AWAIT(buildPathsWithResults(paths, buildMode, evalStore))) if (!result.success()) result.rethrow(); + co_return result::success(); + } catch (...) { + co_return result::current_exception(); } - std::vector buildPathsWithResults( + kj::Promise>> buildPathsWithResults( const std::vector & paths, BuildMode buildMode = bmNormal, std::shared_ptr evalStore = nullptr) override - { + try { assert(!evalStore); if (buildMode != bmNormal) throw Error("unsupported build mode"); @@ -1137,7 +1147,7 @@ struct RestrictedStore : public virtual IndirectRootStore, public virtual GcStor throw InvalidPath("cannot build '%s' in recursive Nix because path is unknown", req.to_string(*next)); } - auto results = next->buildPathsWithResults(paths, buildMode); + auto results = TRY_AWAIT(next->buildPathsWithResults(paths, buildMode)); for (auto & result : results) { for (auto & [outputName, output] : result.builtOutputs) { @@ -1153,12 +1163,15 @@ struct RestrictedStore : public virtual IndirectRootStore, public virtual GcStor for (auto & real : Realisation::closure(*next, newRealisations)) goal.addedDrvOutputs.insert(real.id); - return results; + co_return results; + } catch (...) { + co_return result::current_exception(); } - BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, - BuildMode buildMode = bmNormal) override - { unsupported("buildDerivation"); } + kj::Promise> buildDerivation( + const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode = bmNormal + ) override + try { unsupported("buildDerivation"); } catch (...) { return {result::current_exception()}; } void addTempRoot(const StorePath & path) override { } diff --git a/lix/libstore/daemon.cc b/lix/libstore/daemon.cc index 713873872..3b8d1e2df 100644 --- a/lix/libstore/daemon.cc +++ b/lix/libstore/daemon.cc @@ -550,7 +550,7 @@ static void performOp(AsyncIoRoot & aio, TunnelLogger * logger, ref store if (mode == bmRepair && !trusted) throw Error("repairing is not allowed because you are not in 'trusted-users'"); logger->startWork(); - store->buildPaths(drvs, mode); + aio.blockOn(store->buildPaths(drvs, mode)); logger->stopWork(); to << 1; break; @@ -569,7 +569,7 @@ static void performOp(AsyncIoRoot & aio, TunnelLogger * logger, ref store throw Error("repairing is not allowed because you are not in 'trusted-users'"); logger->startWork(); - auto results = store->buildPathsWithResults(drvs, mode); + auto results = aio.blockOn(store->buildPathsWithResults(drvs, mode)); logger->stopWork(); to << WorkerProto::write(*store, wconn, results); @@ -648,7 +648,7 @@ static void performOp(AsyncIoRoot & aio, TunnelLogger * logger, ref store drvPath = writeDerivation(*store, Derivation { drv2 }); } - auto res = store->buildDerivation(drvPath, drv, buildMode); + auto res = aio.blockOn(store->buildDerivation(drvPath, drv, buildMode)); logger->stopWork(); to << WorkerProto::write(*store, wconn, res); break; @@ -657,7 +657,7 @@ static void performOp(AsyncIoRoot & aio, TunnelLogger * logger, ref store case WorkerProto::Op::EnsurePath: { auto path = store->parseStorePath(readString(from)); logger->startWork(); - store->ensurePath(path); + aio.blockOn(store->ensurePath(path)); logger->stopWork(); to << 1; break; diff --git a/lix/libstore/legacy-ssh-store.cc b/lix/libstore/legacy-ssh-store.cc index f6a79dbbd..0f4be8064 100644 --- a/lix/libstore/legacy-ssh-store.cc +++ b/lix/libstore/legacy-ssh-store.cc @@ -294,9 +294,10 @@ private: public: - BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, - BuildMode buildMode) override - { + kj::Promise> buildDerivation( + const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode + ) override + try { auto conn(connections->get()); conn->to @@ -308,11 +309,17 @@ public: conn->to.flush(); - return ServeProto::Serialise::read(*this, *conn); + return {ServeProto::Serialise::read(*this, *conn)}; + } catch (...) { + return {result::current_exception()}; } - void buildPaths(const std::vector & drvPaths, BuildMode buildMode, std::shared_ptr evalStore) override - { + kj ::Promise> buildPaths( + const std::vector & drvPaths, + BuildMode buildMode, + std::shared_ptr evalStore + ) override + try { if (evalStore && evalStore.get() != this) throw Error("building on an SSH store is incompatible with '--eval-store'"); @@ -347,10 +354,14 @@ public: conn->from >> result.errorMsg; throw Error(result.status, result.errorMsg); } + + co_return result::success(); + } catch (...) { + co_return result::current_exception(); } - void ensurePath(const StorePath & path) override - { unsupported("ensurePath"); } + kj::Promise> ensurePath(const StorePath & path) override + try { unsupported("ensurePath"); } catch (...) { return {result::current_exception()}; } virtual ref getFSAccessor() override { unsupported("getFSAccessor"); } @@ -363,8 +374,8 @@ public: * We make this fail for now so we can add implement this properly later * without it being a breaking change. */ - void repairPath(const StorePath & path) override - { unsupported("repairPath"); } + kj::Promise> repairPath(const StorePath & path) override + try { unsupported("repairPath"); } catch (...) { return {result::current_exception()}; } void computeFSClosure(const StorePathSet & paths, StorePathSet & out, bool flipDirection = false, diff --git a/lix/libstore/local-store.cc b/lix/libstore/local-store.cc index 6124f15e0..73955c065 100644 --- a/lix/libstore/local-store.cc +++ b/lix/libstore/local-store.cc @@ -6,6 +6,7 @@ #include "lix/libstore/worker-protocol.hh" #include "lix/libstore/derivations.hh" #include "lix/libstore/nar-info.hh" +#include "lix/libutil/async.hh" #include "lix/libutil/references.hh" #include "lix/libutil/topo-sort.hh" #include "lix/libutil/signals.hh" @@ -1596,7 +1597,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) if (info->narHash != nullHash && info->narHash != current.first) { printError("path '%s' was modified! expected hash '%s', got '%s'", printStorePath(i), info->narHash.to_string(Base::Base32, true), current.first.to_string(Base::Base32, true)); - if (repair) repairPath(i); else errors = true; + if (repair) RUN_ASYNC_IN_NEW_THREAD(repairPath(i)); else errors = true; } else { bool update = false; @@ -1667,7 +1668,7 @@ void LocalStore::verifyPath(const StorePath & path, const StorePathSet & storePa printError("path '%s' disappeared, but it still has valid referrers!", pathS); if (repair) try { - repairPath(path); + RUN_ASYNC_IN_NEW_THREAD(repairPath(path)); } catch (Error & e) { logWarning(e.info()); errors = true; diff --git a/lix/libstore/remote-store.cc b/lix/libstore/remote-store.cc index 46a3a2da7..3c93bd26c 100644 --- a/lix/libstore/remote-store.cc +++ b/lix/libstore/remote-store.cc @@ -1,4 +1,6 @@ +#include "lix/libutil/async.hh" #include "lix/libutil/error.hh" +#include "lix/libutil/result.hh" #include "lix/libutil/serialise.hh" #include "lix/libutil/signals.hh" #include "lix/libstore/path-with-outputs.hh" @@ -604,8 +606,10 @@ void RemoteStore::copyDrvsFromEvalStore( } } -void RemoteStore::buildPaths(const std::vector & drvPaths, BuildMode buildMode, std::shared_ptr evalStore) -{ +kj ::Promise> RemoteStore::buildPaths( + const std::vector & drvPaths, BuildMode buildMode, std::shared_ptr evalStore +) +try { copyDrvsFromEvalStore(drvPaths, evalStore); auto conn(getConnection()); @@ -614,13 +618,16 @@ void RemoteStore::buildPaths(const std::vector & drvPaths, BuildMod conn->to << buildMode; conn.processStderr(); readInt(conn->from); + return {result::success()}; +} catch (...) { + return {result::current_exception()}; } -std::vector RemoteStore::buildPathsWithResults( +kj::Promise>> RemoteStore::buildPathsWithResults( const std::vector & paths, BuildMode buildMode, std::shared_ptr evalStore) -{ +try { copyDrvsFromEvalStore(paths, evalStore); std::optional conn_(getConnection()); @@ -631,7 +638,7 @@ std::vector RemoteStore::buildPathsWithResults( conn->to << WorkerProto::write(*this, *conn, paths); conn->to << buildMode; conn.processStderr(); - return WorkerProto::Serialise>::read(*this, *conn); + co_return WorkerProto::Serialise>::read(*this, *conn); } else { REMOVE_AFTER_DROPPING_PROTO_MINOR(33); // Avoid deadlock. @@ -639,7 +646,7 @@ std::vector RemoteStore::buildPathsWithResults( // Note: this throws an exception if a build/substitution // fails, but meh. - buildPaths(paths, buildMode, evalStore); + TRY_AWAIT(buildPaths(paths, buildMode, evalStore)); std::vector results; @@ -696,29 +703,36 @@ std::vector RemoteStore::buildPathsWithResults( path.raw()); } - return results; + co_return results; } +} catch (...) { + co_return result::current_exception(); } - -BuildResult RemoteStore::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, - BuildMode buildMode) -{ +kj ::Promise> RemoteStore::buildDerivation( + const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode +) +try { auto conn(getConnection()); conn->to << WorkerProto::Op::BuildDerivation << printStorePath(drvPath); writeDerivation(conn->to, *this, drv); conn->to << buildMode; conn.processStderr(); - return WorkerProto::Serialise::read(*this, *conn); + return {WorkerProto::Serialise::read(*this, *conn)}; +} catch (...) { + return {result::current_exception()}; } -void RemoteStore::ensurePath(const StorePath & path) -{ +kj::Promise> RemoteStore::ensurePath(const StorePath & path) +try { auto conn(getConnection()); conn->to << WorkerProto::Op::EnsurePath << printStorePath(path); conn.processStderr(); readInt(conn->from); + return {result::success()}; +} catch (...) { + return {result::current_exception()}; } diff --git a/lix/libstore/remote-store.hh b/lix/libstore/remote-store.hh index 7e72ea089..f878504a5 100644 --- a/lix/libstore/remote-store.hh +++ b/lix/libstore/remote-store.hh @@ -112,17 +112,22 @@ public: std::shared_ptr queryRealisationUncached(const DrvOutput &) override; - void buildPaths(const std::vector & paths, BuildMode buildMode, std::shared_ptr evalStore) override; + kj ::Promise> buildPaths( + const std::vector & paths, + BuildMode buildMode, + std::shared_ptr evalStore + ) override; - std::vector buildPathsWithResults( + kj::Promise>> buildPathsWithResults( const std::vector & paths, BuildMode buildMode, std::shared_ptr evalStore) override; - BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, - BuildMode buildMode) override; + kj ::Promise> buildDerivation( + const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode + ) override; - void ensurePath(const StorePath & path) override; + kj::Promise> ensurePath(const StorePath & path) override; void addTempRoot(const StorePath & path) override; @@ -142,8 +147,8 @@ public: * We make this fail for now so we can add implement this properly later * without it being a breaking change. */ - void repairPath(const StorePath & path) override - { unsupported("repairPath"); } + kj::Promise> repairPath(const StorePath & path) override + try { unsupported("repairPath"); } catch (...) { return {result::current_exception()}; } void addSignatures(const StorePath & storePath, const StringSet & sigs) override; diff --git a/lix/libstore/store-api.cc b/lix/libstore/store-api.cc index d4cba2105..a937c2eb0 100644 --- a/lix/libstore/store-api.cc +++ b/lix/libstore/store-api.cc @@ -3,6 +3,7 @@ #include "lix/libstore/derivations.hh" #include "lix/libstore/store-api.hh" #include "lix/libstore/nar-info-disk-cache.hh" +#include "lix/libutil/async.hh" #include "lix/libutil/thread-pool.hh" #include "lix/libutil/url.hh" #include "lix/libutil/archive.hh" @@ -815,7 +816,7 @@ void Store::substitutePaths(const StorePathSet & paths) try { std::vector subs; for (auto & p : willSubstitute) subs.emplace_back(DerivedPath::Opaque{p}); - buildPaths(subs); + RUN_ASYNC_IN_NEW_THREAD(buildPaths(subs)); } catch (Error & e) { logWarning(e.info()); } @@ -1355,7 +1356,7 @@ std::string showPaths(const PathSet & paths) Derivation Store::derivationFromPath(const StorePath & drvPath) { - ensurePath(drvPath); + RUN_ASYNC_IN_NEW_THREAD(ensurePath(drvPath)); return readDerivation(drvPath); } diff --git a/lix/libstore/store-api.hh b/lix/libstore/store-api.hh index 7bd3300c8..ff53e8ffc 100644 --- a/lix/libstore/store-api.hh +++ b/lix/libstore/store-api.hh @@ -8,6 +8,7 @@ #include "lix/libstore/derived-path.hh" #include "lix/libutil/hash.hh" #include "lix/libstore/content-address.hh" +#include "lix/libutil/result.hh" #include "lix/libutil/serialise.hh" #include "lix/libutil/lru-cache.hh" #include "lix/libutil/sync.hh" @@ -17,6 +18,7 @@ #include "lix/libutil/repair-flag.hh" #include "lix/libutil/source-path.hh" +#include #include #include #include @@ -592,7 +594,7 @@ public: * recursively building any sub-derivations. For inputs that are * not derivations, substitute them. */ - virtual void buildPaths( + virtual kj::Promise> buildPaths( const std::vector & paths, BuildMode buildMode = bmNormal, std::shared_ptr evalStore = nullptr); @@ -603,7 +605,7 @@ public: * case of a build/substitution error, this function won't throw an * exception, but return a BuildResult containing an error message. */ - virtual std::vector buildPathsWithResults( + virtual kj::Promise>> buildPathsWithResults( const std::vector & paths, BuildMode buildMode = bmNormal, std::shared_ptr evalStore = nullptr); @@ -643,15 +645,18 @@ public: * up with multiple different versions of dependencies without * explicitly choosing to allow it). */ - virtual BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, - BuildMode buildMode = bmNormal); + virtual kj::Promise> buildDerivation( + const StorePath & drvPath, + const BasicDerivation & drv, + BuildMode buildMode = bmNormal + ); /** * Ensure that a path is valid. If it is not currently valid, it * may be made valid by running a substitute (if defined for the * path). */ - virtual void ensurePath(const StorePath & path); + virtual kj::Promise> ensurePath(const StorePath & path); /** * Add a store path as a temporary root of the garbage collector. @@ -712,7 +717,7 @@ public: * Repair the contents of the given path by redownloading it using * a substituter (if available). */ - virtual void repairPath(const StorePath & path); + virtual kj::Promise> repairPath(const StorePath & path); /** * Add signatures to the specified store path. The signatures are diff --git a/lix/nix/bundle.cc b/lix/nix/bundle.cc index 13e80c1ed..94628992f 100644 --- a/lix/nix/bundle.cc +++ b/lix/nix/bundle.cc @@ -110,12 +110,12 @@ struct CmdBundle : InstallableCommand auto outPath = evalState->coerceToStorePath(attr2->pos, *attr2->value, context2, ""); - store->buildPaths({ + aio().blockOn(store->buildPaths({ DerivedPath::Built { .drvPath = makeConstantStorePathRef(drvPath), .outputs = OutputsSpec::All { }, }, - }); + })); if (!outLink) { auto * attr = vRes->attrs->get(evaluator->s.name); diff --git a/lix/nix/daemon.cc b/lix/nix/daemon.cc index 3301695be..4f1437288 100644 --- a/lix/nix/daemon.cc +++ b/lix/nix/daemon.cc @@ -252,7 +252,8 @@ static std::pair authPeer(const PeerInfo & peer) * the client. Otherwise, decide based on the authentication settings * and user credentials (from the unix domain socket). */ -static void daemonLoop(AsyncIoRoot & aio, std::optional forceTrustClientOpt) +static void daemonLoop(AsyncIoRoot & aio, std::optional forceTrustClientOpt); +static void daemonLoopImpl(std::optional forceTrustClientOpt) { if (chdir("/") == -1) throw SysError("cannot change current directory"); @@ -324,6 +325,8 @@ static void daemonLoop(AsyncIoRoot & aio, std::optional forceTrustC if (setsid() == -1) throw SysError("creating a new session"); + AsyncIoRoot aio; + // Restart the signal handler thread since it met its untimely // demise at fork time. startSignalHandlerThread(DoSignalSave::DontSaveBecauseAdvancedProcess); @@ -355,6 +358,20 @@ static void daemonLoop(AsyncIoRoot & aio, std::optional forceTrustC } } } +static void daemonLoop(AsyncIoRoot & aio, std::optional forceTrustClientOpt) +{ + // we can't reuse the external async io root since it'd be shared with the + // children we will create, potentially trashing state, but the *previous* + // root is still alive as far as kj is concerned. we cannot recreate it in + // the child easily because darwin closes kqueues after fork, and since kj + // asserts after the kqueue close returns EBADF we'll die. the least awful + // way around this is to run the daemon loop in its own thread, without an + // async io root, and thus not have any shared state after we have forked. + std::async(std::launch::async, [&] { + ReceiveInterrupts ri; + return daemonLoopImpl(forceTrustClientOpt); + }).get(); +} /** * Forward a standard IO connection to the given remote store. diff --git a/lix/nix/develop.cc b/lix/nix/develop.cc index d6e9d1a05..054e76302 100644 --- a/lix/nix/develop.cc +++ b/lix/nix/develop.cc @@ -6,6 +6,7 @@ #include "lix/libstore/store-api.hh" #include "lix/libstore/outputs-spec.hh" #include "lix/libstore/derivations.hh" +#include "lix/libutil/async.hh" #include "run.hh" #include "lix/libstore/temporary-dir.hh" @@ -258,12 +259,12 @@ try { auto shellDrvPath = writeDerivation(*evalStore, drv); /* Build the derivation. */ - store->buildPaths( + TRY_AWAIT(store->buildPaths( { DerivedPath::Built { .drvPath = makeConstantStorePathRef(shellDrvPath), .outputs = OutputsSpec::All { }, }}, - bmNormal, evalStore); + bmNormal, evalStore)); for (auto & [_0, optPath] : evalStore->queryPartialDerivationOutputMap(shellDrvPath)) { assert(optPath); diff --git a/lix/nix/flake.cc b/lix/nix/flake.cc index 410ededa3..7c65ffb16 100644 --- a/lix/nix/flake.cc +++ b/lix/nix/flake.cc @@ -808,7 +808,7 @@ struct CmdFlakeCheck : FlakeCommand if (build && !drvPaths.empty()) { Activity act(*logger, lvlInfo, actUnknown, fmt("running %d flake checks", drvPaths.size())); - store->buildPaths(drvPaths); + aio().blockOn(store->buildPaths(drvPaths)); } if (hasErrors) throw Error("some errors were encountered during the evaluation"); diff --git a/lix/nix/store-repair.cc b/lix/nix/store-repair.cc index 29bc84e22..c080cb866 100644 --- a/lix/nix/store-repair.cc +++ b/lix/nix/store-repair.cc @@ -20,7 +20,7 @@ struct CmdStoreRepair : StorePathsCommand void run(ref store, StorePaths && storePaths) override { for (auto & path : storePaths) - store->repairPath(path); + aio().blockOn(store->repairPath(path)); } }; diff --git a/lix/nix/upgrade-nix.cc b/lix/nix/upgrade-nix.cc index 33e9f1483..ae01a50db 100644 --- a/lix/nix/upgrade-nix.cc +++ b/lix/nix/upgrade-nix.cc @@ -94,7 +94,7 @@ struct CmdUpgradeNix : MixDryRun, EvalCommand { Activity act(*logger, lvlInfo, actUnknown, fmt("downloading '%s'...", store->printStorePath(storePath))); - store->ensurePath(storePath); + aio().blockOn(store->ensurePath(storePath)); } // {profileDir}/bin/nix-env is a symlink to {profileDir}/bin/nix, which *then* diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index c25a8336d..faa368ad5 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -1,5 +1,10 @@ #include "lix/config.h" +// perl defines _ as a function-like macro for some reason. +// boost outcome uses _ as a name an internal storage type. +// can i make it any more obvious? +#include "lix/libutil/result.hh" + #include "EXTERN.h" #include "perl.h" #include "XSUB.h" diff --git a/perl/lib/Nix/meson.build b/perl/lib/Nix/meson.build index 1d235b74c..8551a6c1f 100644 --- a/perl/lib/Nix/meson.build +++ b/perl/lib/Nix/meson.build @@ -24,6 +24,7 @@ perl_libstore = shared_module( libstore, sodium, perl_include, + kj, ], link_args : [ # Nix doesn't provide a pkg-config file for libutil. diff --git a/perl/meson.build b/perl/meson.build index 6f078f8e6..55bf4561a 100644 --- a/perl/meson.build +++ b/perl/meson.build @@ -66,5 +66,6 @@ endif libstore = dependency('lixstore', 'lix-store', required : true) libutil = dependency('lixutil', 'lix-util', required : true) +kj = dependency('kj-async', required : true, include_type : 'system') subdir('lib/Nix') diff --git a/tests/functional/test-libstoreconsumer/main.cc b/tests/functional/test-libstoreconsumer/main.cc index 9029453d0..5821d3899 100644 --- a/tests/functional/test-libstoreconsumer/main.cc +++ b/tests/functional/test-libstoreconsumer/main.cc @@ -1,6 +1,7 @@ #include "lix/libstore/globals.hh" #include "lix/libstore/store-api.hh" #include "lix/libstore/build-result.hh" +#include "lix/libutil/async.hh" #include using namespace nix; @@ -13,6 +14,7 @@ int main (int argc, char **argv) return 1; } + AsyncIoRoot aio; std::string drvPath = argv[1]; initLibStore(); @@ -28,7 +30,7 @@ int main (int argc, char **argv) } }; - const auto results = store->buildPathsWithResults(paths, bmNormal, store); + const auto results = aio.blockOn(store->buildPathsWithResults(paths, bmNormal, store)); for (const auto & result : results) { for (const auto & [outputName, realisation] : result.builtOutputs) {