libstore: pass around prepared dumps, not paths and filters

this fully decouples the possibly-never-async bits of dumping from the
generation of dump bitstream. having the two separate will allow us to
change store import methods to use async streams, not our sync sources

Change-Id: I9dbd5e30ad3ee380c244b4a3760c11e37db3895f

lix/legacy/nix-store.cc

@@ -183,9 +183,9 @@ static void opAdd(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
     for (auto & i : opArgs) {
         cout << fmt(
             "%s\n",
-            store->printStorePath(
-                aio.blockOn(store->addToStoreRecursive(std::string(baseNameOf(i)), i))
-            )
+            store->printStorePath(aio.blockOn(
+                store->addToStoreRecursive(std::string(baseNameOf(i)), *prepareDump(i))
+            ))
         );
     }
 }

lix/libcmd/installable-value.cc

@@ -1,6 +1,7 @@
 #include "lix/libcmd/installable-value.hh"
 #include "lix/libexpr/eval-cache.hh"
 #include "lix/libfetchers/fetch-to-store.hh"
+#include "lix/libutil/archive.hh"
 
 namespace nix {
 
@@ -47,9 +48,10 @@ std::optional<DerivedPathWithInfo> InstallableValue::trySinglePathToDerivedPaths
 )
 {
     if (v.type() == nPath) {
-        auto storePath = state.aio.blockOn(
-            fetchToStoreRecursive(*evaluator->store, state.ctx.paths.checkSourcePath(v.path()))
-        );
+        auto storePath = state.aio.blockOn(fetchToStoreRecursive(
+            *evaluator->store,
+            *prepareDump(state.ctx.paths.checkSourcePath(v.path()).canonical().abs())
+        ));
         return {{
             .path = DerivedPath::Opaque {
                 .path = std::move(storePath),

lix/libexpr/eval.cc

@@ -1,5 +1,6 @@
 #include "lix/libexpr/eval.hh"
 #include "lix/libexpr/eval-settings.hh"
+#include "lix/libutil/archive.hh"
 #include "lix/libutil/async.hh"
 #include "lix/libutil/hash.hh"
 #include "lix/libexpr/primops.hh"
@@ -2385,9 +2386,8 @@ try {
         : ({
             auto dstPath = TRY_AWAIT(fetchToStoreRecursive(
                 *store,
-                checkSourcePath(path),
+                *prepareDump(checkSourcePath(path).canonical().abs()),
                 path.baseName(),
-                nullptr,
                 repair
             ));
             allowPath(dstPath);

lix/libexpr/primops.cc

@@ -1569,7 +1569,10 @@ static void addPath(
                 method == FileIngestionMethod::Flat
                     ? fetchToStoreFlat(*state.ctx.store, checkedPath, name, state.ctx.repair)
                     : fetchToStoreRecursive(
-                          *state.ctx.store, checkedPath, name, &filter, state.ctx.repair
+                          *state.ctx.store,
+                          *prepareDump(checkedPath.canonical().abs(), filter),
+                          name,
+                          state.ctx.repair
                       )
             );
             if (expectedHash && expectedStorePath != dstPath)

lix/libfetchers/fetch-to-store.cc

@@ -22,19 +22,17 @@ try {
 
 kj::Promise<Result<StorePath>> fetchToStoreRecursive(
     Store & store,
-    const CheckedSourcePath & path,
+    const PreparedDump & contents,
     std::string_view name,
-    PathFilter * filter,
     RepairFlag repair)
 try {
-    Activity act(*logger, lvlChatty, actUnknown, fmt("copying '%s' to the store", path));
-
-    auto filter2 = filter ? *filter : defaultPathFilter;
-    auto physicalPath = path.canonical().abs();
+    Activity act(
+        *logger, lvlChatty, actUnknown, fmt("copying '%s' to the store", contents.rootPath)
+    );
 
     co_return settings.readOnlyMode
-        ? store.computeStorePathForPathRecursive(name, physicalPath, filter2)
-        : TRY_AWAIT(store.addToStoreRecursive(name, physicalPath, HashType::SHA256, filter2, repair));
+        ? store.computeStorePathForPathRecursive(name, contents)
+        : TRY_AWAIT(store.addToStoreRecursive(name, contents, HashType::SHA256, repair));
 } catch (...) {
     co_return result::current_exception();
 }

lix/libfetchers/fetch-to-store.hh

@@ -1,6 +1,7 @@
 #pragma once
 ///@file
 
+#include "lix/libutil/archive.hh"
 #include "lix/libutil/source-path.hh"
 #include "lix/libstore/store-api.hh"
 #include "lix/libutil/repair-flag.hh"
@@ -18,9 +19,8 @@ kj::Promise<Result<StorePath>> fetchToStoreFlat(
     RepairFlag repair = NoRepair);
 kj::Promise<Result<StorePath>> fetchToStoreRecursive(
     Store & store,
-    const CheckedSourcePath & path,
+    const PreparedDump & contents,
     std::string_view name = "source",
-    PathFilter * filter = nullptr,
     RepairFlag repair = NoRepair);
 
 }

lix/libfetchers/git.cc

@@ -1,3 +1,4 @@
+#include "lix/libutil/archive.hh"
 #include "lix/libutil/async.hh"
 #include "lix/libutil/error.hh"
 #include "lix/libfetchers/fetchers.hh"
@@ -243,7 +244,7 @@ try {
     };
 
     auto storePath = TRY_AWAIT(store->addToStoreRecursive(
-        input.getName(), actualPath, HashType::SHA256, filter
+        input.getName(), *prepareDump(actualPath, filter), HashType::SHA256
     ));
 
     // FIXME: maybe we should use the timestamp of the last
@@ -770,8 +771,9 @@ struct GitInputScheme : InputScheme
             unpackTarfile(*proc.getStdout(), tmpDir);
         }
 
-        auto storePath =
-            TRY_AWAIT(store->addToStoreRecursive(name, tmpDir, HashType::SHA256, filter));
+        auto storePath = TRY_AWAIT(
+            store->addToStoreRecursive(name, *prepareDump(tmpDir, filter), HashType::SHA256)
+        );
 
         auto lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "--git-dir", gitDir, "log", "-1", "--format=%ct", "--no-show-signature", input.getRev()->gitRev() }));
 

lix/libfetchers/mercurial.cc

@@ -1,6 +1,7 @@
 #include "lix/libfetchers/fetchers.hh"
 #include "lix/libfetchers/cache.hh"
 #include "lix/libfetchers/builtin-fetchers.hh"
+#include "lix/libutil/archive.hh"
 #include "lix/libutil/async.hh"
 #include "lix/libutil/processes.hh"
 #include "lix/libstore/store-api.hh"
@@ -203,7 +204,7 @@ struct MercurialInputScheme : InputScheme
                 };
 
                 auto storePath = TRY_AWAIT(store->addToStoreRecursive(
-                    input.getName(), actualPath, HashType::SHA256, filter
+                    input.getName(), *prepareDump(actualPath, filter), HashType::SHA256
                 ));
 
                 co_return {std::move(storePath), input};
@@ -315,7 +316,7 @@ struct MercurialInputScheme : InputScheme
 
         deletePath(tmpDir + "/.hg_archival.txt");
 
-        auto storePath = TRY_AWAIT(store->addToStoreRecursive(name, tmpDir));
+        auto storePath = TRY_AWAIT(store->addToStoreRecursive(name, *prepareDump(tmpDir)));
 
         Attrs infoAttrs({
             {"rev", input.getRev()->gitRev()},

lix/libfetchers/tarball.cc

@@ -162,7 +162,7 @@ try {
         auto topDir = tmpDir + "/" + members.begin()->name;
         lastModified = lstat(topDir).st_mtime;
         unpackedStorePath = TRY_AWAIT(
-            store->addToStoreRecursive(name, topDir, HashType::SHA256, defaultPathFilter, NoRepair)
+            store->addToStoreRecursive(name, *prepareDump(topDir), HashType::SHA256, NoRepair)
         );
     }
 

lix/libstore/binary-cache-store.cc

@@ -400,9 +400,8 @@ static ValidPathInfo makeAddToStoreInfo(
 
 kj::Promise<Result<StorePath>> BinaryCacheStore::addToStoreRecursive(
     std::string_view name,
-    const Path & srcPath,
+    const PreparedDump & _source,
     HashType hashAlgo,
-    PathFilter & filter,
     RepairFlag repair)
 try {
     /* FIXME: Make BinaryCacheStore::addToStoreCommon support
@@ -410,10 +409,10 @@ try {
        implementation of this method in terms of addToStoreFromDump. */
 
     HashSink sink { hashAlgo };
-    sink << dumpPath(srcPath, filter);
+    sink << _source.dump();
     auto h = sink.finish().first;
 
-    auto source = GeneratorSource{dumpPath(srcPath, filter)};
+    auto source = GeneratorSource{_source.dump()};
     co_return TRY_AWAIT(addToStoreCommon(source, repair, CheckSigs, [&](HashResult nar) {
         return makeAddToStoreInfo(nar, *this, FileIngestionMethod::Recursive, name, h);
     }))->path;

lix/libstore/binary-cache-store.hh

@@ -5,6 +5,7 @@
 #include "lix/libstore/store-api.hh"
 #include "lix/libstore/log-store.hh"
 
+#include "lix/libutil/archive.hh"
 #include "lix/libutil/pool.hh"
 
 #include <atomic>
@@ -121,9 +122,8 @@ public:
 
     kj::Promise<Result<StorePath>> addToStoreRecursive(
         std::string_view name,
-        const Path & srcPath,
+        const PreparedDump & source,
         HashType hashAlgo,
-        PathFilter & filter,
         RepairFlag repair) override;
     kj::Promise<Result<StorePath>> addToStoreFlat(
         std::string_view name,

lix/libstore/build/local-derivation-goal.cc

@@ -1083,9 +1083,8 @@ struct RestrictedStore : public virtual IndirectRootStore, public virtual GcStor
 
     kj::Promise<Result<StorePath>> addToStoreRecursive(
         std::string_view name,
-        const Path & srcPath,
+        const PreparedDump & source,
         HashType hashAlgo,
-        PathFilter & filter,
         RepairFlag repair) override
     try { throw Error("addToStoreRecursive"); } catch (...) { return {result::current_exception()}; }
 

lix/libstore/legacy-ssh-store.cc

@@ -264,9 +264,8 @@ struct LegacySSHStore final : public Store
 
     kj::Promise<Result<StorePath>> addToStoreRecursive(
         std::string_view name,
-        const Path & srcPath,
+        const PreparedDump & source,
         HashType hashAlgo,
-        PathFilter & filter,
         RepairFlag repair) override
     try { throw Error("addToStoreRecursive"); } catch (...) { return {result::current_exception()}; }
 

lix/libstore/store-api.cc

@@ -247,11 +247,11 @@ StorePath Store::makeFixedOutputPathFromCA(std::string_view name, const ContentA
 
 
 StorePath Store::computeStorePathForPathRecursive(std::string_view name,
-    const Path & srcPath, PathFilter & filter) const
+    const PreparedDump & source) const
 {
     FixedOutputInfo caInfo {
         .method = FileIngestionMethod::Recursive,
-        .hash = hashPath(HashType::SHA256, srcPath, filter).first,
+        .hash = hashPath(HashType::SHA256, source).first,
         .references = {},
     };
     return makeFixedOutputPath(name, caInfo);
@@ -282,13 +282,11 @@ StorePath Store::computeStorePathForText(
 
 kj::Promise<Result<StorePath>> Store::addToStoreRecursive(
     std::string_view name,
-    const Path & _srcPath,
+    const PreparedDump & _source,
     HashType hashAlgo,
-    PathFilter & filter,
     RepairFlag repair)
 try {
-    Path srcPath(absPath(_srcPath));
-    auto source = GeneratorSource{dumpPath(srcPath, filter)};
+    auto source = GeneratorSource{_source.dump()};
     co_return TRY_AWAIT(
         addToStoreFromDump(source, name, FileIngestionMethod::Recursive, hashAlgo, repair, {})
     );

lix/libstore/store-api.hh

@@ -1,6 +1,7 @@
 #pragma once
 ///@file
 
+#include "lix/libutil/archive.hh"
 #include "lix/libutil/async.hh"
 #include "lix/libutil/logging.hh"
 #include "lix/libstore/nar-info.hh"
@@ -315,9 +316,8 @@ public:
      *
      * @return the store path to which srcPath is to be copied.
      */
-    StorePath computeStorePathForPathRecursive(
-        std::string_view name, const Path & srcPath, PathFilter & filter = defaultPathFilter
-    ) const;
+    StorePath
+    computeStorePathForPathRecursive(std::string_view name, const PreparedDump & source) const;
     StorePath computeStorePathForPathFlat(std::string_view name, const Path & srcPath) const;
 
     /**
@@ -528,9 +528,8 @@ public:
      */
     virtual kj::Promise<Result<StorePath>> addToStoreRecursive(
         std::string_view name,
-        const Path & srcPath,
+        const PreparedDump & source,
         HashType hashAlgo = HashType::SHA256,
-        PathFilter & filter = defaultPathFilter,
         RepairFlag repair = NoRepair);
     virtual kj::Promise<Result<StorePath>> addToStoreFlat(
         std::string_view name,

lix/libutil/hash.cc

@@ -367,11 +367,10 @@ HashResult HashSink::currentHash()
 }
 
 
-HashResult hashPath(
-    HashType ht, const Path & path, PathFilter & filter)
+HashResult hashPath(HashType ht, const PreparedDump & path)
 {
     HashSink sink(ht);
-    sink << dumpPath(path, filter);
+    sink << path.dump();
     return sink.finish();
 }
 

lix/libutil/hash.hh

@@ -1,6 +1,7 @@
 #pragma once
 ///@file
 
+#include "lix/libutil/archive.hh"
 #include "lix/libutil/types.hh"
 #include "lix/libutil/serialise.hh"
 #include "lix/libutil/file-system.hh"
@@ -155,8 +156,11 @@ Hash hashFile(HashType ht, const Path & path);
  * (essentially) hashString(ht, dumpPath(path)).
  */
 typedef std::pair<Hash, uint64_t> HashResult;
-HashResult hashPath(HashType ht, const Path & path,
-    PathFilter & filter = defaultPathFilter);
+HashResult hashPath(HashType ht, const PreparedDump & path);
+inline HashResult hashPath(HashType ht, Path path)
+{
+    return hashPath(ht, *prepareDump(std::move(path)));
+}
 
 /**
  * Compress a hash to the specified number of bytes by cyclically

perl/lib/Nix/Store.xs

@@ -295,7 +295,7 @@ SV * addToStore(char * srcPath, int recursive, char * algo)
         try {
             auto hash = parseHashType(algo);
             auto path = aio().blockOn(recursive
-                ? store()->addToStoreRecursive(std::string(baseNameOf(srcPath)), srcPath, hash)
+                ? store()->addToStoreRecursive(std::string(baseNameOf(srcPath)), *prepareDump(srcPath), hash)
                 : store()->addToStoreFlat(std::string(baseNameOf(srcPath)), srcPath, hash));
             XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(path).c_str(), 0)));
         } catch (Error & e) {