diff --git a/lix/legacy/nix-store.cc b/lix/legacy/nix-store.cc index d5c3d8fda..20dfc7412 100644 --- a/lix/legacy/nix-store.cc +++ b/lix/legacy/nix-store.cc @@ -763,7 +763,7 @@ static void opVerifyPath(Strings opFlags, Strings opArgs) printMsg(lvlTalkative, "checking path '%s'...", store->printStorePath(path)); auto info = store->queryPathInfo(path); HashSink sink(info->narHash.type); - sink << store->narFromPath(path); + store->narFromPath(path)->drainInto(sink); auto current = sink.finish(); if (current.first != info->narHash) { printError("path '%s' was modified! expected hash '%s', got '%s'", @@ -900,7 +900,7 @@ static void opServe(Strings opFlags, Strings opArgs) } case ServeProto::Command::DumpStorePath: - out << store->narFromPath(store->parseStorePath(readString(in))); + store->narFromPath(store->parseStorePath(readString(in)))->drainInto(out); break; case ServeProto::Command::ImportPaths: { diff --git a/lix/libstore/binary-cache-store.cc b/lix/libstore/binary-cache-store.cc index 609dc962a..a9301df3c 100644 --- a/lix/libstore/binary-cache-store.cc +++ b/lix/libstore/binary-cache-store.cc @@ -317,13 +317,13 @@ std::optional BinaryCacheStore::queryPathFromHashPart(const std::stri } } -WireFormatGenerator BinaryCacheStore::narFromPath(const StorePath & storePath) +box_ptr BinaryCacheStore::narFromPath(const StorePath & storePath) { auto info = queryPathInfo(storePath).cast(); try { auto file = getFile(info->url); - return [](auto info, auto file, auto & stats) -> WireFormatGenerator { + return make_box_ptr([](auto info, auto file, auto & stats) -> WireFormatGenerator { constexpr size_t buflen = 65536; auto buf = std::make_unique(buflen); size_t total = 0; @@ -340,7 +340,7 @@ WireFormatGenerator BinaryCacheStore::narFromPath(const StorePath & storePath) stats.narRead++; //stats.narReadCompressedBytes += nar->size(); // FIXME stats.narReadBytes += total; - }(std::move(info), std::move(file), stats); + }(std::move(info), std::move(file), stats)); } catch (NoSuchBinaryCacheFile & e) { throw SubstituteGone(std::move(e.info())); } diff --git a/lix/libstore/binary-cache-store.hh b/lix/libstore/binary-cache-store.hh index a8a393133..c251ca108 100644 --- a/lix/libstore/binary-cache-store.hh +++ b/lix/libstore/binary-cache-store.hh @@ -138,7 +138,7 @@ public: std::shared_ptr queryRealisationUncached(const DrvOutput &) override; - WireFormatGenerator narFromPath(const StorePath & path) override; + box_ptr narFromPath(const StorePath & path) override; ref getFSAccessor() override; diff --git a/lix/libstore/build/local-derivation-goal.cc b/lix/libstore/build/local-derivation-goal.cc index 060d4acca..ddf628077 100644 --- a/lix/libstore/build/local-derivation-goal.cc +++ b/lix/libstore/build/local-derivation-goal.cc @@ -1076,7 +1076,7 @@ struct RestrictedStore : public virtual IndirectRootStore, public virtual GcStor return path; } - WireFormatGenerator narFromPath(const StorePath & path) override + box_ptr narFromPath(const StorePath & path) override { if (!goal.isAllowed(path)) throw InvalidPath("cannot dump unknown path '%s' in recursive Nix", printStorePath(path)); diff --git a/lix/libstore/dummy-store.cc b/lix/libstore/dummy-store.cc index caf333d80..38da8110e 100644 --- a/lix/libstore/dummy-store.cc +++ b/lix/libstore/dummy-store.cc @@ -65,7 +65,7 @@ struct DummyStore final : public Store RepairFlag repair) override { unsupported("addTextToStore"); } - WireFormatGenerator narFromPath(const StorePath & path) override + box_ptr narFromPath(const StorePath & path) override { unsupported("narFromPath"); } std::shared_ptr queryRealisationUncached(const DrvOutput &) override diff --git a/lix/libstore/export-import.cc b/lix/libstore/export-import.cc index a79c1300a..a0caeff2e 100644 --- a/lix/libstore/export-import.cc +++ b/lix/libstore/export-import.cc @@ -28,7 +28,7 @@ void Store::exportPath(const StorePath & path, Sink & sink) HashSink hashSink(HashType::SHA256); TeeSink teeSink(sink, hashSink); - teeSink << narFromPath(path); + narFromPath(path)->drainInto(teeSink); /* Refuse to export paths that have changed. This prevents filesystem corruption from spreading to other machines. diff --git a/lix/libstore/legacy-ssh-store.cc b/lix/libstore/legacy-ssh-store.cc index 3c36fefe6..f6a79dbbd 100644 --- a/lix/libstore/legacy-ssh-store.cc +++ b/lix/libstore/legacy-ssh-store.cc @@ -241,15 +241,15 @@ struct LegacySSHStore final : public Store throw Error("failed to add path '%s' to remote host '%s'", printStorePath(info.path), host); } - WireFormatGenerator narFromPath(const StorePath & path) override + box_ptr narFromPath(const StorePath & path) override { auto conn(connections->get()); conn->to << ServeProto::Command::DumpStorePath << printStorePath(path); conn->to.flush(); - return [] (auto conn) -> WireFormatGenerator { + return make_box_ptr([] (auto conn) -> WireFormatGenerator { co_yield copyNAR(conn->from); - }(std::move(conn)); + }(std::move(conn))); } std::optional queryPathFromHashPart(const std::string & hashPart) override diff --git a/lix/libstore/local-fs-store.cc b/lix/libstore/local-fs-store.cc index 02481cef3..c385bbf3d 100644 --- a/lix/libstore/local-fs-store.cc +++ b/lix/libstore/local-fs-store.cc @@ -73,11 +73,11 @@ ref LocalFSStore::getFSAccessor() std::dynamic_pointer_cast(shared_from_this()))); } -WireFormatGenerator LocalFSStore::narFromPath(const StorePath & path) +box_ptr LocalFSStore::narFromPath(const StorePath & path) { if (!isValidPath(path)) throw Error("path '%s' does not exist in store", printStorePath(path)); - return dumpPath(getRealStoreDir() + std::string(printStorePath(path), config().storeDir.size())); + return make_box_ptr(dumpPath(getRealStoreDir() + std::string(printStorePath(path), config().storeDir.size()))); } const std::string LocalFSStore::drvsLogDir = "drvs"; diff --git a/lix/libstore/local-fs-store.hh b/lix/libstore/local-fs-store.hh index 31eba2e66..b9ad4480b 100644 --- a/lix/libstore/local-fs-store.hh +++ b/lix/libstore/local-fs-store.hh @@ -42,7 +42,7 @@ public: LocalFSStoreConfig & config() override = 0; const LocalFSStoreConfig & config() const override = 0; - WireFormatGenerator narFromPath(const StorePath & path) override; + box_ptr narFromPath(const StorePath & path) override; ref getFSAccessor() override; /** diff --git a/lix/libstore/make-content-addressed.cc b/lix/libstore/make-content-addressed.cc index cdc3ad122..7c4535af5 100644 --- a/lix/libstore/make-content-addressed.cc +++ b/lix/libstore/make-content-addressed.cc @@ -24,7 +24,7 @@ std::map makeContentAddressed( std::string oldHashPart(path.hashPart()); StringSink sink; - sink << srcStore.narFromPath(path); + srcStore.narFromPath(path)->drainInto(sink); StringMap rewrites; diff --git a/lix/libstore/remote-fs-accessor.cc b/lix/libstore/remote-fs-accessor.cc index ce62653c4..af6d0a4a3 100644 --- a/lix/libstore/remote-fs-accessor.cc +++ b/lix/libstore/remote-fs-accessor.cc @@ -97,7 +97,7 @@ std::pair, Path> RemoteFSAccessor::fetch(const Path & path_, boo } StringSink sink; - sink << store->narFromPath(storePath); + store->narFromPath(storePath)->drainInto(sink); return {addToCache(storePath.hashPart(), std::move(sink.s)), restPath}; } diff --git a/lix/libstore/remote-store.cc b/lix/libstore/remote-store.cc index 909e85ca7..46a3a2da7 100644 --- a/lix/libstore/remote-store.cc +++ b/lix/libstore/remote-store.cc @@ -861,14 +861,14 @@ RemoteStore::Connection::~Connection() } } -WireFormatGenerator RemoteStore::narFromPath(const StorePath & path) +box_ptr RemoteStore::narFromPath(const StorePath & path) { auto conn(connections->get()); conn->to << WorkerProto::Op::NarFromPath << printStorePath(path); conn->processStderr(); - return [](auto conn) -> WireFormatGenerator { + return make_box_ptr([](auto conn) -> WireFormatGenerator { co_yield copyNAR(conn->from); - }(std::move(conn)); + }(std::move(conn))); } ref RemoteStore::getFSAccessor() diff --git a/lix/libstore/remote-store.hh b/lix/libstore/remote-store.hh index df8d4f294..7e72ea089 100644 --- a/lix/libstore/remote-store.hh +++ b/lix/libstore/remote-store.hh @@ -185,7 +185,7 @@ protected: virtual ref getFSAccessor() override; - virtual WireFormatGenerator narFromPath(const StorePath & path) override; + virtual box_ptr narFromPath(const StorePath & path) override; private: diff --git a/lix/libstore/store-api.cc b/lix/libstore/store-api.cc index aca310fec..d4cba2105 100644 --- a/lix/libstore/store-api.cc +++ b/lix/libstore/store-api.cc @@ -1061,6 +1061,10 @@ static std::string makeCopyPathMessage( } +// buffer size for path copy progress reporting. should be large enough to not cause excessive +// overhead during copies, but small enough to provide reasonably quick copy progress updates. +static constexpr unsigned PATH_COPY_BUFSIZE = 65536; + void copyStorePath( Store & srcStore, Store & dstStore, @@ -1103,11 +1107,17 @@ void copyStorePath( GeneratorSource source{ [](auto & act, auto & info, auto & srcStore, auto & storePath) -> WireFormatGenerator { auto nar = srcStore.narFromPath(storePath); + auto buf = std::make_unique(PATH_COPY_BUFSIZE); uint64_t total = 0; - while (auto data = nar.next()) { - total += data->size(); - act.progress(total, info->narSize); - co_yield *data; + while (true) { + try { + auto got = nar->read(buf.get(), PATH_COPY_BUFSIZE); + total += got; + act.progress(total, info->narSize); + co_yield std::span{buf.get(), got}; + } catch (EndOfFile &) { + break; + } } }(act, info, srcStore, storePath) }; @@ -1239,11 +1249,17 @@ std::map copyPaths( PushActivity pact(act.id); auto nar = srcStore.narFromPath(missingPath); + auto buf = std::make_unique(PATH_COPY_BUFSIZE); uint64_t total = 0; - while (auto data = nar.next()) { - total += data->size(); - act.progress(total, info->narSize); - co_yield *data; + while (true) { + try { + auto got = nar->read(buf.get(), PATH_COPY_BUFSIZE); + total += got; + act.progress(total, info->narSize); + co_yield std::span{buf.get(), got}; + } catch (EndOfFile &) { + break; + } } }; pathsToCopy.push_back(std::pair{ diff --git a/lix/libstore/store-api.hh b/lix/libstore/store-api.hh index 21ab07f70..7bd3300c8 100644 --- a/lix/libstore/store-api.hh +++ b/lix/libstore/store-api.hh @@ -580,7 +580,7 @@ public: /** * Generate a NAR dump of a store path. */ - virtual WireFormatGenerator narFromPath(const StorePath & path) = 0; + virtual box_ptr narFromPath(const StorePath & path) = 0; /** * For each path, if it's a derivation, build it. Building a diff --git a/lix/libstore/uds-remote-store.hh b/lix/libstore/uds-remote-store.hh index fbc8fa2e1..e39506c48 100644 --- a/lix/libstore/uds-remote-store.hh +++ b/lix/libstore/uds-remote-store.hh @@ -42,7 +42,7 @@ public: ref getFSAccessor() override { return LocalFSStore::getFSAccessor(); } - WireFormatGenerator narFromPath(const StorePath & path) override + box_ptr narFromPath(const StorePath & path) override { return LocalFSStore::narFromPath(path); } /** diff --git a/lix/nix/dump-path.cc b/lix/nix/dump-path.cc index be2bbcc7b..de3e8f302 100644 --- a/lix/nix/dump-path.cc +++ b/lix/nix/dump-path.cc @@ -22,7 +22,7 @@ struct CmdDumpPath : StorePathCommand { logger->pause(); FdSink sink(STDOUT_FILENO); - sink << store->narFromPath(storePath); + store->narFromPath(storePath)->drainInto(sink); sink.flush(); } }; diff --git a/lix/nix/verify.cc b/lix/nix/verify.cc index f51f00888..64baae297 100644 --- a/lix/nix/verify.cc +++ b/lix/nix/verify.cc @@ -99,7 +99,7 @@ struct CmdVerify : StorePathsCommand auto hashSink = HashSink(info->narHash.type); - hashSink << store->narFromPath(info->path); + store->narFromPath(info->path)->drainInto(hashSink); auto hash = hashSink.finish();