From fb9e3f6cb5256f7a5258da1e3595a23396f778e2 Mon Sep 17 00:00:00 2001 From: eldritch horrors Date: Sun, 2 Feb 2025 18:19:53 +0100 Subject: [PATCH] libstore: asyncify copyClosure Change-Id: I18cc324d80b502ac87dda3fba2166d4e25f53664 --- lix/legacy/build-remote.cc | 4 +++- lix/legacy/nix-copy-closure.cc | 2 +- lix/libexpr/primops.cc | 2 +- lix/libexpr/primops/fetchClosure.cc | 4 ++-- lix/libstore/build/derivation-goal.cc | 2 +- lix/libstore/remote-store.cc | 17 ++++++++++------- lix/libstore/remote-store.hh | 2 +- lix/libstore/store-api.cc | 18 ++++++++++++------ lix/libstore/store-api.hh | 4 ++-- 9 files changed, 33 insertions(+), 22 deletions(-) diff --git a/lix/legacy/build-remote.cc b/lix/legacy/build-remote.cc index 7cbcce146..2a1642829 100644 --- a/lix/legacy/build-remote.cc +++ b/lix/legacy/build-remote.cc @@ -324,7 +324,9 @@ connected: if (!result.success()) throw Error("build of '%s' on '%s' failed: %s", store->printStorePath(*drvPath), storeUri, result.errorMsg); } else { - copyClosure(*store, *sshStore, StorePathSet {*drvPath}, NoRepair, NoCheckSigs, substitute); + aio.blockOn(copyClosure( + *store, *sshStore, StorePathSet{*drvPath}, NoRepair, NoCheckSigs, substitute + )); auto res = aio.blockOn(sshStore->buildPathsWithResults({ DerivedPath::Built { .drvPath = makeConstantStorePathRef(*drvPath), diff --git a/lix/legacy/nix-copy-closure.cc b/lix/legacy/nix-copy-closure.cc index 45911932d..811485cd7 100644 --- a/lix/legacy/nix-copy-closure.cc +++ b/lix/legacy/nix-copy-closure.cc @@ -55,7 +55,7 @@ static int main_nix_copy_closure(AsyncIoRoot & aio, std::string programName, Str for (auto & path : storePaths) storePaths2.insert(from->followLinksToStorePath(path)); - copyClosure(*from, *to, storePaths2, NoRepair, NoCheckSigs, useSubstitutes); + aio.blockOn(copyClosure(*from, *to, storePaths2, NoRepair, NoCheckSigs, useSubstitutes)); return 0; } diff --git a/lix/libexpr/primops.cc b/lix/libexpr/primops.cc index 2ef457c76..1d597a9c8 100644 --- a/lix/libexpr/primops.cc +++ b/lix/libexpr/primops.cc @@ -106,7 +106,7 @@ try { } } - if (store != buildStore) copyClosure(*buildStore, *store, outputsToCopyAndAllow); + if (store != buildStore) TRY_AWAIT(copyClosure(*buildStore, *store, outputsToCopyAndAllow)); if (allowedPaths) { for (auto & outputPath : outputsToCopyAndAllow) { /* Add the output of this derivations to the allowed diff --git a/lix/libexpr/primops/fetchClosure.cc b/lix/libexpr/primops/fetchClosure.cc index 2b9bdf569..a5843d9cb 100644 --- a/lix/libexpr/primops/fetchClosure.cc +++ b/lix/libexpr/primops/fetchClosure.cc @@ -67,7 +67,7 @@ static void runFetchClosureWithRewrite(EvalState & state, const PosIdx pos, Stor static void runFetchClosureWithContentAddressedPath(EvalState & state, const PosIdx pos, Store & fromStore, const StorePath & fromPath, Value & v) { if (!state.ctx.store->isValidPath(fromPath)) - copyClosure(fromStore, *state.ctx.store, RealisedPath::Set { fromPath }); + state.aio.blockOn(copyClosure(fromStore, *state.ctx.store, RealisedPath::Set { fromPath })); auto info = state.ctx.store->queryPathInfo(fromPath); @@ -93,7 +93,7 @@ static void runFetchClosureWithContentAddressedPath(EvalState & state, const Pos static void runFetchClosureWithInputAddressedPath(EvalState & state, const PosIdx pos, Store & fromStore, const StorePath & fromPath, Value & v) { if (!state.ctx.store->isValidPath(fromPath)) - copyClosure(fromStore, *state.ctx.store, RealisedPath::Set { fromPath }); + state.aio.blockOn(copyClosure(fromStore, *state.ctx.store, RealisedPath::Set { fromPath })); auto info = state.ctx.store->queryPathInfo(fromPath); diff --git a/lix/libstore/build/derivation-goal.cc b/lix/libstore/build/derivation-goal.cc index e2980a18e..ecd9bfadc 100644 --- a/lix/libstore/build/derivation-goal.cc +++ b/lix/libstore/build/derivation-goal.cc @@ -436,7 +436,7 @@ try { for (auto & i : drv->inputSrcs) if (worker.evalStore.isValidPath(i)) inputSrcs.insert(i); - copyClosure(worker.evalStore, worker.store, inputSrcs); + TRY_AWAIT(copyClosure(worker.evalStore, worker.store, inputSrcs)); } for (auto & i : drv->inputSrcs) { diff --git a/lix/libstore/remote-store.cc b/lix/libstore/remote-store.cc index ba4b747ed..a89f7e460 100644 --- a/lix/libstore/remote-store.cc +++ b/lix/libstore/remote-store.cc @@ -590,10 +590,10 @@ std::shared_ptr RemoteStore::queryRealisationUncached(const D } } -void RemoteStore::copyDrvsFromEvalStore( +kj::Promise> RemoteStore::copyDrvsFromEvalStore( const std::vector & paths, std::shared_ptr evalStore) -{ +try { if (evalStore && evalStore.get() != this) { /* The remote doesn't have a way to access evalStore, so copy the .drvs. */ @@ -608,15 +608,18 @@ void RemoteStore::copyDrvsFromEvalStore( }, }, i.raw()); } - copyClosure(*evalStore, *this, drvPaths2); + TRY_AWAIT(copyClosure(*evalStore, *this, drvPaths2)); } + co_return result::success(); +} catch (...) { + co_return result::current_exception(); } kj ::Promise> RemoteStore::buildPaths( const std::vector & drvPaths, BuildMode buildMode, std::shared_ptr evalStore ) try { - copyDrvsFromEvalStore(drvPaths, evalStore); + TRY_AWAIT(copyDrvsFromEvalStore(drvPaths, evalStore)); auto conn(getConnection()); conn->to << WorkerProto::Op::BuildPaths; @@ -624,9 +627,9 @@ try { conn->to << buildMode; conn.processStderr(); readInt(conn->from); - return {result::success()}; + co_return result::success(); } catch (...) { - return {result::current_exception()}; + co_return result::current_exception(); } kj::Promise>> RemoteStore::buildPathsWithResults( @@ -634,7 +637,7 @@ kj::Promise>> RemoteStore::buildPathsWithRe BuildMode buildMode, std::shared_ptr evalStore) try { - copyDrvsFromEvalStore(paths, evalStore); + TRY_AWAIT(copyDrvsFromEvalStore(paths, evalStore)); std::optional conn_(getConnection()); auto & conn = *conn_; diff --git a/lix/libstore/remote-store.hh b/lix/libstore/remote-store.hh index 8921c1ca7..d1a9aef9f 100644 --- a/lix/libstore/remote-store.hh +++ b/lix/libstore/remote-store.hh @@ -196,7 +196,7 @@ private: std::atomic_bool failed{false}; - void copyDrvsFromEvalStore( + kj::Promise> copyDrvsFromEvalStore( const std::vector & paths, std::shared_ptr evalStore); }; diff --git a/lix/libstore/store-api.cc b/lix/libstore/store-api.cc index 6ad729b9f..181ac3f49 100644 --- a/lix/libstore/store-api.cc +++ b/lix/libstore/store-api.cc @@ -1288,35 +1288,41 @@ std::map copyPaths( return pathsMap; } -void copyClosure( +kj::Promise> copyClosure( Store & srcStore, Store & dstStore, const RealisedPath::Set & paths, RepairFlag repair, CheckSigsFlag checkSigs, SubstituteFlag substitute) -{ - if (&srcStore == &dstStore) return; +try { + if (&srcStore == &dstStore) return {result::success()}; RealisedPath::Set closure; RealisedPath::closure(srcStore, paths, closure); copyPaths(srcStore, dstStore, closure, repair, checkSigs, substitute); + return {result::success()}; +} catch (...) { + return {result::current_exception()}; } -void copyClosure( +kj::Promise> copyClosure( Store & srcStore, Store & dstStore, const StorePathSet & storePaths, RepairFlag repair, CheckSigsFlag checkSigs, SubstituteFlag substitute) -{ - if (&srcStore == &dstStore) return; +try { + if (&srcStore == &dstStore) return {result::success()}; StorePathSet closure; srcStore.computeFSClosure(storePaths, closure); copyPaths(srcStore, dstStore, closure, repair, checkSigs, substitute); + return {result::success()}; +} catch (...) { + return {result::current_exception()}; } std::optional decodeValidPathInfo(const Store & store, std::istream & str, std::optional hashGiven) diff --git a/lix/libstore/store-api.hh b/lix/libstore/store-api.hh index 383e59f12..f4e802de3 100644 --- a/lix/libstore/store-api.hh +++ b/lix/libstore/store-api.hh @@ -938,14 +938,14 @@ std::map copyPaths( /** * Copy the closure of `paths` from `srcStore` to `dstStore`. */ -void copyClosure( +kj::Promise> copyClosure( Store & srcStore, Store & dstStore, const RealisedPath::Set & paths, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs, SubstituteFlag substitute = NoSubstitute); -void copyClosure( +kj::Promise> copyClosure( Store & srcStore, Store & dstStore, const StorePathSet & paths, RepairFlag repair = NoRepair,