diff --git a/lix/legacy/nix-store.cc b/lix/legacy/nix-store.cc index 3137a34de..f2ca4e38e 100644 --- a/lix/legacy/nix-store.cc +++ b/lix/legacy/nix-store.cc @@ -183,7 +183,9 @@ static void opAdd(AsyncIoRoot & aio, Strings opFlags, Strings opArgs) for (auto & i : opArgs) { cout << fmt( "%s\n", - store->printStorePath(aio.blockOn(store->addToStore(std::string(baseNameOf(i)), i))) + store->printStorePath( + aio.blockOn(store->addToStoreRecursive(std::string(baseNameOf(i)), i)) + ) ); } } diff --git a/lix/libfetchers/fetch-to-store.cc b/lix/libfetchers/fetch-to-store.cc index da9857fe0..f6cc21a7c 100644 --- a/lix/libfetchers/fetch-to-store.cc +++ b/lix/libfetchers/fetch-to-store.cc @@ -25,9 +25,14 @@ try { co_return store.computeStorePathForPathFlat(name, physicalPath); } } else { - co_return TRY_AWAIT( - store.addToStore(name, physicalPath, method, HashType::SHA256, filter2, repair) - ); + switch (method) { + case FileIngestionMethod::Recursive: + co_return TRY_AWAIT( + store.addToStoreRecursive(name, physicalPath, HashType::SHA256, filter2, repair) + ); + case FileIngestionMethod::Flat: + co_return TRY_AWAIT(store.addToStoreFlat(name, physicalPath, HashType::SHA256, repair)); + } } } catch (...) { co_return result::current_exception(); diff --git a/lix/libfetchers/git.cc b/lix/libfetchers/git.cc index 1c1d54674..8d590aa8d 100644 --- a/lix/libfetchers/git.cc +++ b/lix/libfetchers/git.cc @@ -242,8 +242,8 @@ try { return files.count(file); }; - auto storePath = TRY_AWAIT(store->addToStore( - input.getName(), actualPath, FileIngestionMethod::Recursive, HashType::SHA256, filter + auto storePath = TRY_AWAIT(store->addToStoreRecursive( + input.getName(), actualPath, HashType::SHA256, filter )); // FIXME: maybe we should use the timestamp of the last @@ -770,9 +770,8 @@ struct GitInputScheme : InputScheme unpackTarfile(*proc.getStdout(), tmpDir); } - auto storePath = TRY_AWAIT(store->addToStore( - name, tmpDir, FileIngestionMethod::Recursive, HashType::SHA256, filter - )); + auto storePath = + TRY_AWAIT(store->addToStoreRecursive(name, tmpDir, HashType::SHA256, filter)); auto lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "--git-dir", gitDir, "log", "-1", "--format=%ct", "--no-show-signature", input.getRev()->gitRev() })); diff --git a/lix/libfetchers/mercurial.cc b/lix/libfetchers/mercurial.cc index d70a075e5..179ab67c1 100644 --- a/lix/libfetchers/mercurial.cc +++ b/lix/libfetchers/mercurial.cc @@ -202,12 +202,8 @@ struct MercurialInputScheme : InputScheme return files.count(file); }; - auto storePath = TRY_AWAIT(store->addToStore( - input.getName(), - actualPath, - FileIngestionMethod::Recursive, - HashType::SHA256, - filter + auto storePath = TRY_AWAIT(store->addToStoreRecursive( + input.getName(), actualPath, HashType::SHA256, filter )); co_return {std::move(storePath), input}; @@ -319,7 +315,7 @@ struct MercurialInputScheme : InputScheme deletePath(tmpDir + "/.hg_archival.txt"); - auto storePath = TRY_AWAIT(store->addToStore(name, tmpDir)); + auto storePath = TRY_AWAIT(store->addToStoreRecursive(name, tmpDir)); Attrs infoAttrs({ {"rev", input.getRev()->gitRev()}, diff --git a/lix/libfetchers/tarball.cc b/lix/libfetchers/tarball.cc index c9afa6cd8..513189bb0 100644 --- a/lix/libfetchers/tarball.cc +++ b/lix/libfetchers/tarball.cc @@ -161,14 +161,9 @@ try { throw nix::Error("tarball '%s' contains an unexpected number of top-level files", url); auto topDir = tmpDir + "/" + members.begin()->name; lastModified = lstat(topDir).st_mtime; - unpackedStorePath = TRY_AWAIT(store->addToStore( - name, - topDir, - FileIngestionMethod::Recursive, - HashType::SHA256, - defaultPathFilter, - NoRepair - )); + unpackedStorePath = TRY_AWAIT( + store->addToStoreRecursive(name, topDir, HashType::SHA256, defaultPathFilter, NoRepair) + ); } Attrs infoAttrs({ diff --git a/lix/libstore/binary-cache-store.cc b/lix/libstore/binary-cache-store.cc index 2d18e368c..030335cd7 100644 --- a/lix/libstore/binary-cache-store.cc +++ b/lix/libstore/binary-cache-store.cc @@ -376,10 +376,31 @@ std::shared_ptr BinaryCacheStore::queryPathInfoUncached(con return std::make_shared(*this, *data, narInfoFile); } -kj::Promise> BinaryCacheStore::addToStore( +static ValidPathInfo makeAddToStoreInfo( + HashResult nar, Store & store, FileIngestionMethod method, std::string_view name, Hash h +) +{ + ValidPathInfo info{ + store, + name, + FixedOutputInfo { + .method = method, + .hash = h, + .references = { + .others = {}, + // caller is not capable of creating a self-reference, because this is content-addressed without modulus + .self = false, + }, + }, + nar.first, + }; + info.narSize = nar.second; + return info; +} + +kj::Promise> BinaryCacheStore::addToStoreRecursive( std::string_view name, const Path & srcPath, - FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair) @@ -389,31 +410,34 @@ try { implementation of this method in terms of addToStoreFromDump. */ HashSink sink { hashAlgo }; - if (method == FileIngestionMethod::Recursive) { - sink << dumpPath(srcPath, filter); - } else { - sink << readFileSource(srcPath); - } + sink << dumpPath(srcPath, filter); auto h = sink.finish().first; auto source = GeneratorSource{dumpPath(srcPath, filter)}; co_return TRY_AWAIT(addToStoreCommon(source, repair, CheckSigs, [&](HashResult nar) { - ValidPathInfo info { - *this, - name, - FixedOutputInfo { - .method = method, - .hash = h, - .references = { - .others = {}, - // caller is not capable of creating a self-reference, because this is content-addressed without modulus - .self = false, - }, - }, - nar.first, - }; - info.narSize = nar.second; - return info; + return makeAddToStoreInfo(nar, *this, FileIngestionMethod::Recursive, name, h); + }))->path; +} catch (...) { + co_return result::current_exception(); +} + +kj::Promise> BinaryCacheStore::addToStoreFlat( + std::string_view name, + const Path & srcPath, + HashType hashAlgo, + RepairFlag repair) +try { + /* FIXME: Make BinaryCacheStore::addToStoreCommon support + non-recursive+sha256 so we can just use the default + implementation of this method in terms of addToStoreFromDump. */ + + HashSink sink { hashAlgo }; + sink << readFileSource(srcPath); + auto h = sink.finish().first; + + auto source = GeneratorSource{dumpPath(srcPath)}; + co_return TRY_AWAIT(addToStoreCommon(source, repair, CheckSigs, [&](HashResult nar) { + return makeAddToStoreInfo(nar, *this, FileIngestionMethod::Flat, name, h); }))->path; } catch (...) { co_return result::current_exception(); diff --git a/lix/libstore/binary-cache-store.hh b/lix/libstore/binary-cache-store.hh index 356d5636f..2b4a492a5 100644 --- a/lix/libstore/binary-cache-store.hh +++ b/lix/libstore/binary-cache-store.hh @@ -119,13 +119,17 @@ public: kj::Promise> addToStoreFromDump(Source & dump, std::string_view name, FileIngestionMethod method, HashType hashAlgo, RepairFlag repair, const StorePathSet & references) override; - kj::Promise> addToStore( + kj::Promise> addToStoreRecursive( std::string_view name, const Path & srcPath, - FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair) override; + kj::Promise> addToStoreFlat( + std::string_view name, + const Path & srcPath, + HashType hashAlgo, + RepairFlag repair) override; kj::Promise> addTextToStore( std::string_view name, diff --git a/lix/libstore/build/local-derivation-goal.cc b/lix/libstore/build/local-derivation-goal.cc index 804c99084..d1bc0a265 100644 --- a/lix/libstore/build/local-derivation-goal.cc +++ b/lix/libstore/build/local-derivation-goal.cc @@ -1081,14 +1081,20 @@ struct RestrictedStore : public virtual IndirectRootStore, public virtual GcStor std::optional queryPathFromHashPart(const std::string & hashPart) override { throw Error("queryPathFromHashPart"); } - kj::Promise> addToStore( + kj::Promise> addToStoreRecursive( std::string_view name, const Path & srcPath, - FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair) override - try { throw Error("addToStore"); } catch (...) { return {result::current_exception()}; } + try { throw Error("addToStoreRecursive"); } catch (...) { return {result::current_exception()}; } + + kj::Promise> addToStoreFlat( + std::string_view name, + const Path & srcPath, + HashType hashAlgo, + RepairFlag repair) override + try { throw Error("addToStoreFlat"); } catch (...) { return {result::current_exception()}; } kj::Promise> addToStore(const ValidPathInfo & info, Source & narSource, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs) override diff --git a/lix/libstore/legacy-ssh-store.cc b/lix/libstore/legacy-ssh-store.cc index 030676e34..654d62447 100644 --- a/lix/libstore/legacy-ssh-store.cc +++ b/lix/libstore/legacy-ssh-store.cc @@ -262,14 +262,20 @@ struct LegacySSHStore final : public Store std::optional queryPathFromHashPart(const std::string & hashPart) override { unsupported("queryPathFromHashPart"); } - kj::Promise> addToStore( + kj::Promise> addToStoreRecursive( std::string_view name, const Path & srcPath, - FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair) override - try { unsupported("addToStore"); } catch (...) { return {result::current_exception()}; } + try { throw Error("addToStoreRecursive"); } catch (...) { return {result::current_exception()}; } + + kj::Promise> addToStoreFlat( + std::string_view name, + const Path & srcPath, + HashType hashAlgo, + RepairFlag repair) override + try { throw Error("addToStoreFlat"); } catch (...) { return {result::current_exception()}; } kj::Promise> addTextToStore( std::string_view name, diff --git a/lix/libstore/store-api.cc b/lix/libstore/store-api.cc index 07d7a240d..3fad40da7 100644 --- a/lix/libstore/store-api.cc +++ b/lix/libstore/store-api.cc @@ -280,20 +280,33 @@ StorePath Store::computeStorePathForText( } -kj::Promise> Store::addToStore( +kj::Promise> Store::addToStoreRecursive( std::string_view name, const Path & _srcPath, - FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair) try { Path srcPath(absPath(_srcPath)); - auto source = GeneratorSource{ - method == FileIngestionMethod::Recursive ? dumpPath(srcPath, filter).decay() - : readFileSource(srcPath) - }; - co_return TRY_AWAIT(addToStoreFromDump(source, name, method, hashAlgo, repair, {})); + auto source = GeneratorSource{dumpPath(srcPath, filter)}; + co_return TRY_AWAIT( + addToStoreFromDump(source, name, FileIngestionMethod::Recursive, hashAlgo, repair, {}) + ); +} catch (...) { + co_return result::current_exception(); +} + +kj::Promise> Store::addToStoreFlat( + std::string_view name, + const Path & _srcPath, + HashType hashAlgo, + RepairFlag repair) +try { + Path srcPath(absPath(_srcPath)); + auto source = GeneratorSource{readFileSource(srcPath)}; + co_return TRY_AWAIT( + addToStoreFromDump(source, name, FileIngestionMethod::Flat, hashAlgo, repair, {}) + ); } catch (...) { co_return result::current_exception(); } diff --git a/lix/libstore/store-api.hh b/lix/libstore/store-api.hh index 8ccec21e6..cc81ee38a 100644 --- a/lix/libstore/store-api.hh +++ b/lix/libstore/store-api.hh @@ -526,13 +526,17 @@ public: * @param filter This function can be used to exclude files (see * libutil/archive.hh). */ - virtual kj::Promise> addToStore( + virtual kj::Promise> addToStoreRecursive( std::string_view name, const Path & srcPath, - FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = HashType::SHA256, PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair); + virtual kj::Promise> addToStoreFlat( + std::string_view name, + const Path & srcPath, + HashType hashAlgo = HashType::SHA256, + RepairFlag repair = NoRepair); /** * Copy the contents of a path to the store and register the diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index de6884ca0..1ad3d45cd 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -293,8 +293,10 @@ int checkSignature(SV * publicKey_, SV * sig_, char * msg) SV * addToStore(char * srcPath, int recursive, char * algo) PPCODE: try { - auto method = recursive ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; - auto path = aio().blockOn(store()->addToStore(std::string(baseNameOf(srcPath)), srcPath, method, parseHashType(algo))); + auto hash = parseHashType(algo); + auto path = aio().blockOn(recursive + ? store()->addToStoreRecursive(std::string(baseNameOf(srcPath)), srcPath, hash) + : store()->addToStoreFlat(std::string(baseNameOf(srcPath)), srcPath, hash)); XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(path).c_str(), 0))); } catch (Error & e) { croak("%s", e.what());