From 446e22323e48ba6a9dab2c1da4e824731bf9b25a Mon Sep 17 00:00:00 2001 From: eldritch horrors Date: Sat, 1 Mar 2025 23:55:42 +0100 Subject: [PATCH] libstore: pass around prepared dumps, not paths and filters this fully decouples the possibly-never-async bits of dumping from the generation of dump bitstream. having the two separate will allow us to change store import methods to use async streams, not our sync sources Change-Id: I9dbd5e30ad3ee380c244b4a3760c11e37db3895f --- lix/legacy/nix-store.cc | 6 +++--- lix/libcmd/installable-value.cc | 8 +++++--- lix/libexpr/eval.cc | 4 ++-- lix/libexpr/primops.cc | 5 ++++- lix/libfetchers/fetch-to-store.cc | 14 ++++++-------- lix/libfetchers/fetch-to-store.hh | 4 ++-- lix/libfetchers/git.cc | 8 +++++--- lix/libfetchers/mercurial.cc | 5 +++-- lix/libfetchers/tarball.cc | 2 +- lix/libstore/binary-cache-store.cc | 7 +++---- lix/libstore/binary-cache-store.hh | 4 ++-- lix/libstore/build/local-derivation-goal.cc | 3 +-- lix/libstore/legacy-ssh-store.cc | 3 +-- lix/libstore/store-api.cc | 10 ++++------ lix/libstore/store-api.hh | 9 ++++----- lix/libutil/hash.cc | 5 ++--- lix/libutil/hash.hh | 8 ++++++-- perl/lib/Nix/Store.xs | 2 +- 18 files changed, 55 insertions(+), 52 deletions(-) diff --git a/lix/legacy/nix-store.cc b/lix/legacy/nix-store.cc index f2ca4e38e..a93e9899b 100644 --- a/lix/legacy/nix-store.cc +++ b/lix/legacy/nix-store.cc @@ -183,9 +183,9 @@ static void opAdd(AsyncIoRoot & aio, Strings opFlags, Strings opArgs) for (auto & i : opArgs) { cout << fmt( "%s\n", - store->printStorePath( - aio.blockOn(store->addToStoreRecursive(std::string(baseNameOf(i)), i)) - ) + store->printStorePath(aio.blockOn( + store->addToStoreRecursive(std::string(baseNameOf(i)), *prepareDump(i)) + )) ); } } diff --git a/lix/libcmd/installable-value.cc b/lix/libcmd/installable-value.cc index 601be415c..9b3b6884b 100644 --- a/lix/libcmd/installable-value.cc +++ b/lix/libcmd/installable-value.cc @@ -1,6 +1,7 @@ #include "lix/libcmd/installable-value.hh" #include "lix/libexpr/eval-cache.hh" #include "lix/libfetchers/fetch-to-store.hh" +#include "lix/libutil/archive.hh" namespace nix { @@ -47,9 +48,10 @@ std::optional InstallableValue::trySinglePathToDerivedPaths ) { if (v.type() == nPath) { - auto storePath = state.aio.blockOn( - fetchToStoreRecursive(*evaluator->store, state.ctx.paths.checkSourcePath(v.path())) - ); + auto storePath = state.aio.blockOn(fetchToStoreRecursive( + *evaluator->store, + *prepareDump(state.ctx.paths.checkSourcePath(v.path()).canonical().abs()) + )); return {{ .path = DerivedPath::Opaque { .path = std::move(storePath), diff --git a/lix/libexpr/eval.cc b/lix/libexpr/eval.cc index 92006dee7..530f25d3e 100644 --- a/lix/libexpr/eval.cc +++ b/lix/libexpr/eval.cc @@ -1,5 +1,6 @@ #include "lix/libexpr/eval.hh" #include "lix/libexpr/eval-settings.hh" +#include "lix/libutil/archive.hh" #include "lix/libutil/async.hh" #include "lix/libutil/hash.hh" #include "lix/libexpr/primops.hh" @@ -2385,9 +2386,8 @@ try { : ({ auto dstPath = TRY_AWAIT(fetchToStoreRecursive( *store, - checkSourcePath(path), + *prepareDump(checkSourcePath(path).canonical().abs()), path.baseName(), - nullptr, repair )); allowPath(dstPath); diff --git a/lix/libexpr/primops.cc b/lix/libexpr/primops.cc index 003703231..af99230d3 100644 --- a/lix/libexpr/primops.cc +++ b/lix/libexpr/primops.cc @@ -1569,7 +1569,10 @@ static void addPath( method == FileIngestionMethod::Flat ? fetchToStoreFlat(*state.ctx.store, checkedPath, name, state.ctx.repair) : fetchToStoreRecursive( - *state.ctx.store, checkedPath, name, &filter, state.ctx.repair + *state.ctx.store, + *prepareDump(checkedPath.canonical().abs(), filter), + name, + state.ctx.repair ) ); if (expectedHash && expectedStorePath != dstPath) diff --git a/lix/libfetchers/fetch-to-store.cc b/lix/libfetchers/fetch-to-store.cc index 835bd77e0..b86effc8b 100644 --- a/lix/libfetchers/fetch-to-store.cc +++ b/lix/libfetchers/fetch-to-store.cc @@ -22,19 +22,17 @@ try { kj::Promise> fetchToStoreRecursive( Store & store, - const CheckedSourcePath & path, + const PreparedDump & contents, std::string_view name, - PathFilter * filter, RepairFlag repair) try { - Activity act(*logger, lvlChatty, actUnknown, fmt("copying '%s' to the store", path)); - - auto filter2 = filter ? *filter : defaultPathFilter; - auto physicalPath = path.canonical().abs(); + Activity act( + *logger, lvlChatty, actUnknown, fmt("copying '%s' to the store", contents.rootPath) + ); co_return settings.readOnlyMode - ? store.computeStorePathForPathRecursive(name, physicalPath, filter2) - : TRY_AWAIT(store.addToStoreRecursive(name, physicalPath, HashType::SHA256, filter2, repair)); + ? store.computeStorePathForPathRecursive(name, contents) + : TRY_AWAIT(store.addToStoreRecursive(name, contents, HashType::SHA256, repair)); } catch (...) { co_return result::current_exception(); } diff --git a/lix/libfetchers/fetch-to-store.hh b/lix/libfetchers/fetch-to-store.hh index 848a49ef8..72acac629 100644 --- a/lix/libfetchers/fetch-to-store.hh +++ b/lix/libfetchers/fetch-to-store.hh @@ -1,6 +1,7 @@ #pragma once ///@file +#include "lix/libutil/archive.hh" #include "lix/libutil/source-path.hh" #include "lix/libstore/store-api.hh" #include "lix/libutil/repair-flag.hh" @@ -18,9 +19,8 @@ kj::Promise> fetchToStoreFlat( RepairFlag repair = NoRepair); kj::Promise> fetchToStoreRecursive( Store & store, - const CheckedSourcePath & path, + const PreparedDump & contents, std::string_view name = "source", - PathFilter * filter = nullptr, RepairFlag repair = NoRepair); } diff --git a/lix/libfetchers/git.cc b/lix/libfetchers/git.cc index 8d590aa8d..21fa1904d 100644 --- a/lix/libfetchers/git.cc +++ b/lix/libfetchers/git.cc @@ -1,3 +1,4 @@ +#include "lix/libutil/archive.hh" #include "lix/libutil/async.hh" #include "lix/libutil/error.hh" #include "lix/libfetchers/fetchers.hh" @@ -243,7 +244,7 @@ try { }; auto storePath = TRY_AWAIT(store->addToStoreRecursive( - input.getName(), actualPath, HashType::SHA256, filter + input.getName(), *prepareDump(actualPath, filter), HashType::SHA256 )); // FIXME: maybe we should use the timestamp of the last @@ -770,8 +771,9 @@ struct GitInputScheme : InputScheme unpackTarfile(*proc.getStdout(), tmpDir); } - auto storePath = - TRY_AWAIT(store->addToStoreRecursive(name, tmpDir, HashType::SHA256, filter)); + auto storePath = TRY_AWAIT( + store->addToStoreRecursive(name, *prepareDump(tmpDir, filter), HashType::SHA256) + ); auto lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "--git-dir", gitDir, "log", "-1", "--format=%ct", "--no-show-signature", input.getRev()->gitRev() })); diff --git a/lix/libfetchers/mercurial.cc b/lix/libfetchers/mercurial.cc index 179ab67c1..0cd007d8c 100644 --- a/lix/libfetchers/mercurial.cc +++ b/lix/libfetchers/mercurial.cc @@ -1,6 +1,7 @@ #include "lix/libfetchers/fetchers.hh" #include "lix/libfetchers/cache.hh" #include "lix/libfetchers/builtin-fetchers.hh" +#include "lix/libutil/archive.hh" #include "lix/libutil/async.hh" #include "lix/libutil/processes.hh" #include "lix/libstore/store-api.hh" @@ -203,7 +204,7 @@ struct MercurialInputScheme : InputScheme }; auto storePath = TRY_AWAIT(store->addToStoreRecursive( - input.getName(), actualPath, HashType::SHA256, filter + input.getName(), *prepareDump(actualPath, filter), HashType::SHA256 )); co_return {std::move(storePath), input}; @@ -315,7 +316,7 @@ struct MercurialInputScheme : InputScheme deletePath(tmpDir + "/.hg_archival.txt"); - auto storePath = TRY_AWAIT(store->addToStoreRecursive(name, tmpDir)); + auto storePath = TRY_AWAIT(store->addToStoreRecursive(name, *prepareDump(tmpDir))); Attrs infoAttrs({ {"rev", input.getRev()->gitRev()}, diff --git a/lix/libfetchers/tarball.cc b/lix/libfetchers/tarball.cc index 513189bb0..fe9f331c1 100644 --- a/lix/libfetchers/tarball.cc +++ b/lix/libfetchers/tarball.cc @@ -162,7 +162,7 @@ try { auto topDir = tmpDir + "/" + members.begin()->name; lastModified = lstat(topDir).st_mtime; unpackedStorePath = TRY_AWAIT( - store->addToStoreRecursive(name, topDir, HashType::SHA256, defaultPathFilter, NoRepair) + store->addToStoreRecursive(name, *prepareDump(topDir), HashType::SHA256, NoRepair) ); } diff --git a/lix/libstore/binary-cache-store.cc b/lix/libstore/binary-cache-store.cc index 030335cd7..cd6d96bba 100644 --- a/lix/libstore/binary-cache-store.cc +++ b/lix/libstore/binary-cache-store.cc @@ -400,9 +400,8 @@ static ValidPathInfo makeAddToStoreInfo( kj::Promise> BinaryCacheStore::addToStoreRecursive( std::string_view name, - const Path & srcPath, + const PreparedDump & _source, HashType hashAlgo, - PathFilter & filter, RepairFlag repair) try { /* FIXME: Make BinaryCacheStore::addToStoreCommon support @@ -410,10 +409,10 @@ try { implementation of this method in terms of addToStoreFromDump. */ HashSink sink { hashAlgo }; - sink << dumpPath(srcPath, filter); + sink << _source.dump(); auto h = sink.finish().first; - auto source = GeneratorSource{dumpPath(srcPath, filter)}; + auto source = GeneratorSource{_source.dump()}; co_return TRY_AWAIT(addToStoreCommon(source, repair, CheckSigs, [&](HashResult nar) { return makeAddToStoreInfo(nar, *this, FileIngestionMethod::Recursive, name, h); }))->path; diff --git a/lix/libstore/binary-cache-store.hh b/lix/libstore/binary-cache-store.hh index 2b4a492a5..8d236ea12 100644 --- a/lix/libstore/binary-cache-store.hh +++ b/lix/libstore/binary-cache-store.hh @@ -5,6 +5,7 @@ #include "lix/libstore/store-api.hh" #include "lix/libstore/log-store.hh" +#include "lix/libutil/archive.hh" #include "lix/libutil/pool.hh" #include @@ -121,9 +122,8 @@ public: kj::Promise> addToStoreRecursive( std::string_view name, - const Path & srcPath, + const PreparedDump & source, HashType hashAlgo, - PathFilter & filter, RepairFlag repair) override; kj::Promise> addToStoreFlat( std::string_view name, diff --git a/lix/libstore/build/local-derivation-goal.cc b/lix/libstore/build/local-derivation-goal.cc index d1bc0a265..3740d8b8a 100644 --- a/lix/libstore/build/local-derivation-goal.cc +++ b/lix/libstore/build/local-derivation-goal.cc @@ -1083,9 +1083,8 @@ struct RestrictedStore : public virtual IndirectRootStore, public virtual GcStor kj::Promise> addToStoreRecursive( std::string_view name, - const Path & srcPath, + const PreparedDump & source, HashType hashAlgo, - PathFilter & filter, RepairFlag repair) override try { throw Error("addToStoreRecursive"); } catch (...) { return {result::current_exception()}; } diff --git a/lix/libstore/legacy-ssh-store.cc b/lix/libstore/legacy-ssh-store.cc index 654d62447..a990c17dd 100644 --- a/lix/libstore/legacy-ssh-store.cc +++ b/lix/libstore/legacy-ssh-store.cc @@ -264,9 +264,8 @@ struct LegacySSHStore final : public Store kj::Promise> addToStoreRecursive( std::string_view name, - const Path & srcPath, + const PreparedDump & source, HashType hashAlgo, - PathFilter & filter, RepairFlag repair) override try { throw Error("addToStoreRecursive"); } catch (...) { return {result::current_exception()}; } diff --git a/lix/libstore/store-api.cc b/lix/libstore/store-api.cc index 3fad40da7..18a2a0458 100644 --- a/lix/libstore/store-api.cc +++ b/lix/libstore/store-api.cc @@ -247,11 +247,11 @@ StorePath Store::makeFixedOutputPathFromCA(std::string_view name, const ContentA StorePath Store::computeStorePathForPathRecursive(std::string_view name, - const Path & srcPath, PathFilter & filter) const + const PreparedDump & source) const { FixedOutputInfo caInfo { .method = FileIngestionMethod::Recursive, - .hash = hashPath(HashType::SHA256, srcPath, filter).first, + .hash = hashPath(HashType::SHA256, source).first, .references = {}, }; return makeFixedOutputPath(name, caInfo); @@ -282,13 +282,11 @@ StorePath Store::computeStorePathForText( kj::Promise> Store::addToStoreRecursive( std::string_view name, - const Path & _srcPath, + const PreparedDump & _source, HashType hashAlgo, - PathFilter & filter, RepairFlag repair) try { - Path srcPath(absPath(_srcPath)); - auto source = GeneratorSource{dumpPath(srcPath, filter)}; + auto source = GeneratorSource{_source.dump()}; co_return TRY_AWAIT( addToStoreFromDump(source, name, FileIngestionMethod::Recursive, hashAlgo, repair, {}) ); diff --git a/lix/libstore/store-api.hh b/lix/libstore/store-api.hh index cc81ee38a..c25f63e1f 100644 --- a/lix/libstore/store-api.hh +++ b/lix/libstore/store-api.hh @@ -1,6 +1,7 @@ #pragma once ///@file +#include "lix/libutil/archive.hh" #include "lix/libutil/async.hh" #include "lix/libutil/logging.hh" #include "lix/libstore/nar-info.hh" @@ -315,9 +316,8 @@ public: * * @return the store path to which srcPath is to be copied. */ - StorePath computeStorePathForPathRecursive( - std::string_view name, const Path & srcPath, PathFilter & filter = defaultPathFilter - ) const; + StorePath + computeStorePathForPathRecursive(std::string_view name, const PreparedDump & source) const; StorePath computeStorePathForPathFlat(std::string_view name, const Path & srcPath) const; /** @@ -528,9 +528,8 @@ public: */ virtual kj::Promise> addToStoreRecursive( std::string_view name, - const Path & srcPath, + const PreparedDump & source, HashType hashAlgo = HashType::SHA256, - PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair); virtual kj::Promise> addToStoreFlat( std::string_view name, diff --git a/lix/libutil/hash.cc b/lix/libutil/hash.cc index d8952e013..bd8be18b1 100644 --- a/lix/libutil/hash.cc +++ b/lix/libutil/hash.cc @@ -367,11 +367,10 @@ HashResult HashSink::currentHash() } -HashResult hashPath( - HashType ht, const Path & path, PathFilter & filter) +HashResult hashPath(HashType ht, const PreparedDump & path) { HashSink sink(ht); - sink << dumpPath(path, filter); + sink << path.dump(); return sink.finish(); } diff --git a/lix/libutil/hash.hh b/lix/libutil/hash.hh index c0f50c744..c347cb1e5 100644 --- a/lix/libutil/hash.hh +++ b/lix/libutil/hash.hh @@ -1,6 +1,7 @@ #pragma once ///@file +#include "lix/libutil/archive.hh" #include "lix/libutil/types.hh" #include "lix/libutil/serialise.hh" #include "lix/libutil/file-system.hh" @@ -155,8 +156,11 @@ Hash hashFile(HashType ht, const Path & path); * (essentially) hashString(ht, dumpPath(path)). */ typedef std::pair HashResult; -HashResult hashPath(HashType ht, const Path & path, - PathFilter & filter = defaultPathFilter); +HashResult hashPath(HashType ht, const PreparedDump & path); +inline HashResult hashPath(HashType ht, Path path) +{ + return hashPath(ht, *prepareDump(std::move(path))); +} /** * Compress a hash to the specified number of bytes by cyclically diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index 1ad3d45cd..be9979a55 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -295,7 +295,7 @@ SV * addToStore(char * srcPath, int recursive, char * algo) try { auto hash = parseHashType(algo); auto path = aio().blockOn(recursive - ? store()->addToStoreRecursive(std::string(baseNameOf(srcPath)), srcPath, hash) + ? store()->addToStoreRecursive(std::string(baseNameOf(srcPath)), *prepareDump(srcPath), hash) : store()->addToStoreFlat(std::string(baseNameOf(srcPath)), srcPath, hash)); XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(path).c_str(), 0))); } catch (Error & e) {