From c82cc16754cbd583fd22da068ea1b98d33a22fb5 Mon Sep 17 00:00:00 2001 From: eldritch horrors Date: Sun, 19 Jan 2025 16:40:26 +0100 Subject: [PATCH] libstore: return sources from Store::narFromPath this will make it easier to return async streams instead of sources at some point in the future. the primary benefactors of the current state are not greatly inconvenienced by the api change, and would need to be changed much as they are now once async streams come around either way Change-Id: I4db9ea8b186f358c239f7863ac8140c500986c2d --- lix/legacy/nix-store.cc | 4 +-- lix/libstore/binary-cache-store.cc | 6 ++-- lix/libstore/binary-cache-store.hh | 2 +- lix/libstore/build/local-derivation-goal.cc | 2 +- lix/libstore/dummy-store.cc | 2 +- lix/libstore/export-import.cc | 2 +- lix/libstore/legacy-ssh-store.cc | 6 ++-- lix/libstore/local-fs-store.cc | 4 +-- lix/libstore/local-fs-store.hh | 2 +- lix/libstore/make-content-addressed.cc | 2 +- lix/libstore/remote-fs-accessor.cc | 2 +- lix/libstore/remote-store.cc | 6 ++-- lix/libstore/remote-store.hh | 2 +- lix/libstore/store-api.cc | 32 +++++++++++++++------ lix/libstore/store-api.hh | 2 +- lix/libstore/uds-remote-store.hh | 2 +- lix/nix/dump-path.cc | 2 +- lix/nix/verify.cc | 2 +- 18 files changed, 49 insertions(+), 33 deletions(-) diff --git a/lix/legacy/nix-store.cc b/lix/legacy/nix-store.cc index d5c3d8fda..20dfc7412 100644 --- a/lix/legacy/nix-store.cc +++ b/lix/legacy/nix-store.cc @@ -763,7 +763,7 @@ static void opVerifyPath(Strings opFlags, Strings opArgs) printMsg(lvlTalkative, "checking path '%s'...", store->printStorePath(path)); auto info = store->queryPathInfo(path); HashSink sink(info->narHash.type); - sink << store->narFromPath(path); + store->narFromPath(path)->drainInto(sink); auto current = sink.finish(); if (current.first != info->narHash) { printError("path '%s' was modified! expected hash '%s', got '%s'", @@ -900,7 +900,7 @@ static void opServe(Strings opFlags, Strings opArgs) } case ServeProto::Command::DumpStorePath: - out << store->narFromPath(store->parseStorePath(readString(in))); + store->narFromPath(store->parseStorePath(readString(in)))->drainInto(out); break; case ServeProto::Command::ImportPaths: { diff --git a/lix/libstore/binary-cache-store.cc b/lix/libstore/binary-cache-store.cc index 609dc962a..a9301df3c 100644 --- a/lix/libstore/binary-cache-store.cc +++ b/lix/libstore/binary-cache-store.cc @@ -317,13 +317,13 @@ std::optional BinaryCacheStore::queryPathFromHashPart(const std::stri } } -WireFormatGenerator BinaryCacheStore::narFromPath(const StorePath & storePath) +box_ptr BinaryCacheStore::narFromPath(const StorePath & storePath) { auto info = queryPathInfo(storePath).cast(); try { auto file = getFile(info->url); - return [](auto info, auto file, auto & stats) -> WireFormatGenerator { + return make_box_ptr([](auto info, auto file, auto & stats) -> WireFormatGenerator { constexpr size_t buflen = 65536; auto buf = std::make_unique(buflen); size_t total = 0; @@ -340,7 +340,7 @@ WireFormatGenerator BinaryCacheStore::narFromPath(const StorePath & storePath) stats.narRead++; //stats.narReadCompressedBytes += nar->size(); // FIXME stats.narReadBytes += total; - }(std::move(info), std::move(file), stats); + }(std::move(info), std::move(file), stats)); } catch (NoSuchBinaryCacheFile & e) { throw SubstituteGone(std::move(e.info())); } diff --git a/lix/libstore/binary-cache-store.hh b/lix/libstore/binary-cache-store.hh index a8a393133..c251ca108 100644 --- a/lix/libstore/binary-cache-store.hh +++ b/lix/libstore/binary-cache-store.hh @@ -138,7 +138,7 @@ public: std::shared_ptr queryRealisationUncached(const DrvOutput &) override; - WireFormatGenerator narFromPath(const StorePath & path) override; + box_ptr narFromPath(const StorePath & path) override; ref getFSAccessor() override; diff --git a/lix/libstore/build/local-derivation-goal.cc b/lix/libstore/build/local-derivation-goal.cc index 060d4acca..ddf628077 100644 --- a/lix/libstore/build/local-derivation-goal.cc +++ b/lix/libstore/build/local-derivation-goal.cc @@ -1076,7 +1076,7 @@ struct RestrictedStore : public virtual IndirectRootStore, public virtual GcStor return path; } - WireFormatGenerator narFromPath(const StorePath & path) override + box_ptr narFromPath(const StorePath & path) override { if (!goal.isAllowed(path)) throw InvalidPath("cannot dump unknown path '%s' in recursive Nix", printStorePath(path)); diff --git a/lix/libstore/dummy-store.cc b/lix/libstore/dummy-store.cc index caf333d80..38da8110e 100644 --- a/lix/libstore/dummy-store.cc +++ b/lix/libstore/dummy-store.cc @@ -65,7 +65,7 @@ struct DummyStore final : public Store RepairFlag repair) override { unsupported("addTextToStore"); } - WireFormatGenerator narFromPath(const StorePath & path) override + box_ptr narFromPath(const StorePath & path) override { unsupported("narFromPath"); } std::shared_ptr queryRealisationUncached(const DrvOutput &) override diff --git a/lix/libstore/export-import.cc b/lix/libstore/export-import.cc index a79c1300a..a0caeff2e 100644 --- a/lix/libstore/export-import.cc +++ b/lix/libstore/export-import.cc @@ -28,7 +28,7 @@ void Store::exportPath(const StorePath & path, Sink & sink) HashSink hashSink(HashType::SHA256); TeeSink teeSink(sink, hashSink); - teeSink << narFromPath(path); + narFromPath(path)->drainInto(teeSink); /* Refuse to export paths that have changed. This prevents filesystem corruption from spreading to other machines. diff --git a/lix/libstore/legacy-ssh-store.cc b/lix/libstore/legacy-ssh-store.cc index 3c36fefe6..f6a79dbbd 100644 --- a/lix/libstore/legacy-ssh-store.cc +++ b/lix/libstore/legacy-ssh-store.cc @@ -241,15 +241,15 @@ struct LegacySSHStore final : public Store throw Error("failed to add path '%s' to remote host '%s'", printStorePath(info.path), host); } - WireFormatGenerator narFromPath(const StorePath & path) override + box_ptr narFromPath(const StorePath & path) override { auto conn(connections->get()); conn->to << ServeProto::Command::DumpStorePath << printStorePath(path); conn->to.flush(); - return [] (auto conn) -> WireFormatGenerator { + return make_box_ptr([] (auto conn) -> WireFormatGenerator { co_yield copyNAR(conn->from); - }(std::move(conn)); + }(std::move(conn))); } std::optional queryPathFromHashPart(const std::string & hashPart) override diff --git a/lix/libstore/local-fs-store.cc b/lix/libstore/local-fs-store.cc index 02481cef3..c385bbf3d 100644 --- a/lix/libstore/local-fs-store.cc +++ b/lix/libstore/local-fs-store.cc @@ -73,11 +73,11 @@ ref LocalFSStore::getFSAccessor() std::dynamic_pointer_cast(shared_from_this()))); } -WireFormatGenerator LocalFSStore::narFromPath(const StorePath & path) +box_ptr LocalFSStore::narFromPath(const StorePath & path) { if (!isValidPath(path)) throw Error("path '%s' does not exist in store", printStorePath(path)); - return dumpPath(getRealStoreDir() + std::string(printStorePath(path), config().storeDir.size())); + return make_box_ptr(dumpPath(getRealStoreDir() + std::string(printStorePath(path), config().storeDir.size()))); } const std::string LocalFSStore::drvsLogDir = "drvs"; diff --git a/lix/libstore/local-fs-store.hh b/lix/libstore/local-fs-store.hh index 31eba2e66..b9ad4480b 100644 --- a/lix/libstore/local-fs-store.hh +++ b/lix/libstore/local-fs-store.hh @@ -42,7 +42,7 @@ public: LocalFSStoreConfig & config() override = 0; const LocalFSStoreConfig & config() const override = 0; - WireFormatGenerator narFromPath(const StorePath & path) override; + box_ptr narFromPath(const StorePath & path) override; ref getFSAccessor() override; /** diff --git a/lix/libstore/make-content-addressed.cc b/lix/libstore/make-content-addressed.cc index cdc3ad122..7c4535af5 100644 --- a/lix/libstore/make-content-addressed.cc +++ b/lix/libstore/make-content-addressed.cc @@ -24,7 +24,7 @@ std::map makeContentAddressed( std::string oldHashPart(path.hashPart()); StringSink sink; - sink << srcStore.narFromPath(path); + srcStore.narFromPath(path)->drainInto(sink); StringMap rewrites; diff --git a/lix/libstore/remote-fs-accessor.cc b/lix/libstore/remote-fs-accessor.cc index ce62653c4..af6d0a4a3 100644 --- a/lix/libstore/remote-fs-accessor.cc +++ b/lix/libstore/remote-fs-accessor.cc @@ -97,7 +97,7 @@ std::pair, Path> RemoteFSAccessor::fetch(const Path & path_, boo } StringSink sink; - sink << store->narFromPath(storePath); + store->narFromPath(storePath)->drainInto(sink); return {addToCache(storePath.hashPart(), std::move(sink.s)), restPath}; } diff --git a/lix/libstore/remote-store.cc b/lix/libstore/remote-store.cc index 909e85ca7..46a3a2da7 100644 --- a/lix/libstore/remote-store.cc +++ b/lix/libstore/remote-store.cc @@ -861,14 +861,14 @@ RemoteStore::Connection::~Connection() } } -WireFormatGenerator RemoteStore::narFromPath(const StorePath & path) +box_ptr RemoteStore::narFromPath(const StorePath & path) { auto conn(connections->get()); conn->to << WorkerProto::Op::NarFromPath << printStorePath(path); conn->processStderr(); - return [](auto conn) -> WireFormatGenerator { + return make_box_ptr([](auto conn) -> WireFormatGenerator { co_yield copyNAR(conn->from); - }(std::move(conn)); + }(std::move(conn))); } ref RemoteStore::getFSAccessor() diff --git a/lix/libstore/remote-store.hh b/lix/libstore/remote-store.hh index df8d4f294..7e72ea089 100644 --- a/lix/libstore/remote-store.hh +++ b/lix/libstore/remote-store.hh @@ -185,7 +185,7 @@ protected: virtual ref getFSAccessor() override; - virtual WireFormatGenerator narFromPath(const StorePath & path) override; + virtual box_ptr narFromPath(const StorePath & path) override; private: diff --git a/lix/libstore/store-api.cc b/lix/libstore/store-api.cc index aca310fec..d4cba2105 100644 --- a/lix/libstore/store-api.cc +++ b/lix/libstore/store-api.cc @@ -1061,6 +1061,10 @@ static std::string makeCopyPathMessage( } +// buffer size for path copy progress reporting. should be large enough to not cause excessive +// overhead during copies, but small enough to provide reasonably quick copy progress updates. +static constexpr unsigned PATH_COPY_BUFSIZE = 65536; + void copyStorePath( Store & srcStore, Store & dstStore, @@ -1103,11 +1107,17 @@ void copyStorePath( GeneratorSource source{ [](auto & act, auto & info, auto & srcStore, auto & storePath) -> WireFormatGenerator { auto nar = srcStore.narFromPath(storePath); + auto buf = std::make_unique(PATH_COPY_BUFSIZE); uint64_t total = 0; - while (auto data = nar.next()) { - total += data->size(); - act.progress(total, info->narSize); - co_yield *data; + while (true) { + try { + auto got = nar->read(buf.get(), PATH_COPY_BUFSIZE); + total += got; + act.progress(total, info->narSize); + co_yield std::span{buf.get(), got}; + } catch (EndOfFile &) { + break; + } } }(act, info, srcStore, storePath) }; @@ -1239,11 +1249,17 @@ std::map copyPaths( PushActivity pact(act.id); auto nar = srcStore.narFromPath(missingPath); + auto buf = std::make_unique(PATH_COPY_BUFSIZE); uint64_t total = 0; - while (auto data = nar.next()) { - total += data->size(); - act.progress(total, info->narSize); - co_yield *data; + while (true) { + try { + auto got = nar->read(buf.get(), PATH_COPY_BUFSIZE); + total += got; + act.progress(total, info->narSize); + co_yield std::span{buf.get(), got}; + } catch (EndOfFile &) { + break; + } } }; pathsToCopy.push_back(std::pair{ diff --git a/lix/libstore/store-api.hh b/lix/libstore/store-api.hh index 21ab07f70..7bd3300c8 100644 --- a/lix/libstore/store-api.hh +++ b/lix/libstore/store-api.hh @@ -580,7 +580,7 @@ public: /** * Generate a NAR dump of a store path. */ - virtual WireFormatGenerator narFromPath(const StorePath & path) = 0; + virtual box_ptr narFromPath(const StorePath & path) = 0; /** * For each path, if it's a derivation, build it. Building a diff --git a/lix/libstore/uds-remote-store.hh b/lix/libstore/uds-remote-store.hh index fbc8fa2e1..e39506c48 100644 --- a/lix/libstore/uds-remote-store.hh +++ b/lix/libstore/uds-remote-store.hh @@ -42,7 +42,7 @@ public: ref getFSAccessor() override { return LocalFSStore::getFSAccessor(); } - WireFormatGenerator narFromPath(const StorePath & path) override + box_ptr narFromPath(const StorePath & path) override { return LocalFSStore::narFromPath(path); } /** diff --git a/lix/nix/dump-path.cc b/lix/nix/dump-path.cc index be2bbcc7b..de3e8f302 100644 --- a/lix/nix/dump-path.cc +++ b/lix/nix/dump-path.cc @@ -22,7 +22,7 @@ struct CmdDumpPath : StorePathCommand { logger->pause(); FdSink sink(STDOUT_FILENO); - sink << store->narFromPath(storePath); + store->narFromPath(storePath)->drainInto(sink); sink.flush(); } }; diff --git a/lix/nix/verify.cc b/lix/nix/verify.cc index f51f00888..64baae297 100644 --- a/lix/nix/verify.cc +++ b/lix/nix/verify.cc @@ -99,7 +99,7 @@ struct CmdVerify : StorePathsCommand auto hashSink = HashSink(info->narHash.type); - hashSink << store->narFromPath(info->path); + store->narFromPath(info->path)->drainInto(hashSink); auto hash = hashSink.finish();