guiwf/lix
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

libstore: asyncify Store::queryPathInfo{,Uncached}

Browse Source 
Change-Id: I182c396ec0bfdfc8febd6752774e668b3d97dcc1
This commit is contained in:
eldritch horrors
2025-03-05 18:49:45 +01:00
parent 09727e15c5
commit 81d5528646
34 changed files with 290 additions and 217 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lix/legacy/dotgraph.cc
+2 -1
View File
@@ -1,5 +1,6 @@
#include "dotgraph.hh"
#include "lix/libstore/store-api.hh"
#include "lix/libutil/async.hh"
#include "lix/libutil/result.hh"
#include <iostream>
@@ -56,7 +57,7 @@ try {
cout << makeNode(std::string(path.to_string()), path.name(), "#ff0000");
for (auto & p : store->queryPathInfo(path)->references) {
for (auto & p : TRY_AWAIT(store->queryPathInfo(path))->references) {
if (p != path) {
workList.insert(p);
cout << makeEdge(std::string(p.to_string()), std::string(path.to_string()));
lix/legacy/graphml.cc
+2 -1
View File
@@ -1,6 +1,7 @@
#include "graphml.hh"
#include "lix/libstore/store-api.hh"
#include "lix/libstore/derivations.hh"
#include "lix/libutil/async.hh"
#include "lix/libutil/result.hh"
#include <iostream>
@@ -68,7 +69,7 @@ try {
ret = doneSet.insert(path);
if (ret.second == false) continue;
auto info = store->queryPathInfo(path);
auto info = TRY_AWAIT(store->queryPathInfo(path));
cout << makeNode(*info);
for (auto & p : info->references) {
lix/legacy/nix-store.cc
+14 -12
View File
@@ -51,13 +51,15 @@ ref<LocalStore> ensureLocalStore()
}
static StorePath useDeriver(const StorePath & path)
{
if (path.isDerivation()) return path;
auto info = store->queryPathInfo(path);
static kj::Promise<Result<StorePath>> useDeriver(const StorePath & path)
try {
if (path.isDerivation()) co_return path;
auto info = TRY_AWAIT(store->queryPathInfo(path));
if (!info->deriver)
throw Error("deriver of path '%s' is not known", store->printStorePath(path));
return *info->deriver;
co_return *info->deriver;
} catch (...) {
co_return result::current_exception();
}
@@ -278,7 +280,7 @@ static void printTree(AsyncIoRoot & aio, const StorePath & path,
cout << fmt("%s%s\n", firstPad, store->printStorePath(path));
auto info = store->queryPathInfo(path);
auto info = aio.blockOn(store->queryPathInfo(path));
/* Topologically sort under the relation A < B iff A \in
closure(B). That is, if derivation A is an (possibly indirect)
@@ -368,7 +370,7 @@ static void opQuery(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
aio.blockOn(store->computeFSClosure(j, paths, false, includeOutputs));
}
else if (query == qReferences) {
for (auto & p : store->queryPathInfo(j)->references)
for (auto & p : aio.blockOn(store->queryPathInfo(j))->references)
paths.insert(p);
}
else if (query == qReferrers) {
@@ -390,7 +392,7 @@ static void opQuery(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
case qDeriver:
for (auto & i : opArgs) {
auto info = store->queryPathInfo(store->followLinksToStorePath(i));
auto info = aio.blockOn(store->queryPathInfo(store->followLinksToStorePath(i)));
cout << fmt("%s\n", info->deriver ? store->printStorePath(*info->deriver) : "unknown-deriver");
}
break;
@@ -413,7 +415,7 @@ static void opQuery(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
case qBinding:
for (auto & i : opArgs) {
auto path = useDeriver(store->followLinksToStorePath(i));
auto path = aio.blockOn(useDeriver(store->followLinksToStorePath(i)));
Derivation drv = aio.blockOn(store->derivationFromPath(path));
StringPairs::iterator j = drv.env.find(bindingName);
if (j == drv.env.end())
@@ -427,7 +429,7 @@ static void opQuery(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
case qSize:
for (auto & i : opArgs) {
for (auto & j : aio.blockOn(maybeUseOutputs(store->followLinksToStorePath(i), useOutput, forceRealise))) {
auto info = store->queryPathInfo(j);
auto info = aio.blockOn(store->queryPathInfo(j));
if (query == qHash) {
assert(info->narHash.type == HashType::SHA256);
cout << fmt("%s\n", info->narHash.to_string(Base::Base32, true));
@@ -789,7 +791,7 @@ static void opVerifyPath(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
for (auto & i : opArgs) {
auto path = store->followLinksToStorePath(i);
printMsg(lvlTalkative, "checking path '%s'...", store->printStorePath(path));
auto info = store->queryPathInfo(path);
auto info = aio.blockOn(store->queryPathInfo(path));
HashSink sink(info->narHash.type);
aio.blockOn(store->narFromPath(path))->drainInto(sink);
auto current = sink.finish();
@@ -917,7 +919,7 @@ static void opServe(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
// !!! Maybe we want a queryPathInfos?
for (auto & i : paths) {
try {
auto info = store->queryPathInfo(i);
auto info = aio.blockOn(store->queryPathInfo(i));
out << store->printStorePath(info->path);
out << ServeProto::write(*store, wconn, static_cast<const UnkeyedValidPathInfo &>(*info));
} catch (InvalidPath &) {
lix/libexpr/primops.cc
+6 -2
View File
@@ -1284,7 +1284,11 @@ static void prim_readFile(EvalState & state, const PosIdx pos, Value * * args, V
StorePathSet refs;
if (state.ctx.store->isInStore(path.canonical().abs())) {
try {
refs = state.ctx.store->queryPathInfo(state.ctx.store->toStorePath(path.canonical().abs()).first)->references;
refs = state.aio
.blockOn(state.ctx.store->queryPathInfo(
state.ctx.store->toStorePath(path.canonical().abs()).first
))
->references;
} catch (Error &) { // FIXME: should be InvalidPathError
}
// Re-scan references to filter down to just the ones that actually occur in the file.
@@ -1530,7 +1534,7 @@ static void addPath(
try {
auto [storePath, subPath] = state.ctx.store->toStorePath(path);
// FIXME: we should scanForReferences on the path before adding it
refs = state.ctx.store->queryPathInfo(storePath)->references;
refs = state.aio.blockOn(state.ctx.store->queryPathInfo(storePath))->references;
realPath = state.ctx.store->toRealPath(path);
} catch (Error &) { // FIXME: should be InvalidPathError
}
lix/libexpr/primops/fetchClosure.cc
+3 -3
View File
@@ -46,7 +46,7 @@ static void runFetchClosureWithRewrite(EvalState & state, const PosIdx pos, Stor
// check and return
auto resultInfo = state.ctx.store->queryPathInfo(toPath);
auto resultInfo = state.aio.blockOn(state.ctx.store->queryPathInfo(toPath));
if (!resultInfo->isContentAddressed(*state.ctx.store)) {
// We don't perform the rewriting when outPath already exists, as an optimisation.
@@ -71,7 +71,7 @@ static void runFetchClosureWithContentAddressedPath(EvalState & state, const Pos
if (!state.aio.blockOn(state.ctx.store->isValidPath(fromPath)))
state.aio.blockOn(copyClosure(fromStore, *state.ctx.store, RealisedPath::Set { fromPath }));
auto info = state.ctx.store->queryPathInfo(fromPath);
auto info = state.aio.blockOn(state.ctx.store->queryPathInfo(fromPath));
if (!info->isContentAddressed(*state.ctx.store)) {
throw Error({
@@ -97,7 +97,7 @@ static void runFetchClosureWithInputAddressedPath(EvalState & state, const PosId
if (!state.aio.blockOn(state.ctx.store->isValidPath(fromPath)))
state.aio.blockOn(copyClosure(fromStore, *state.ctx.store, RealisedPath::Set { fromPath }));
auto info = state.ctx.store->queryPathInfo(fromPath);
auto info = state.aio.blockOn(state.ctx.store->queryPathInfo(fromPath));
if (info->isContentAddressed(*state.ctx.store)) {
throw Error({
lix/libexpr/primops/fetchTree.cc
+1 -1
View File
@@ -280,7 +280,7 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v
if (expectedHash) {
auto hash = unpack
? state.ctx.store->queryPathInfo(storePath)->narHash
? state.aio.blockOn(state.ctx.store->queryPathInfo(storePath))->narHash
: hashFile(HashType::SHA256, state.ctx.store->toRealPath(storePath));
if (hash != *expectedHash) {
state.ctx.errors.make<EvalError>(
lix/libfetchers/fetchers.cc
+1 -1
View File
@@ -176,7 +176,7 @@ try {
.storePath = storePath,
};
auto narHash = store->queryPathInfo(tree.storePath)->narHash;
auto narHash = TRY_AWAIT(store->queryPathInfo(tree.storePath))->narHash;
input.attrs.insert_or_assign("narHash", narHash.to_string(Base::SRI, true));
if (auto prevNarHash = getNarHash()) {
lix/libstore/binary-cache-store.cc
+12 -8
View File
@@ -166,7 +166,7 @@ try {
for (auto & ref : info.references)
try {
if (ref != info.path)
queryPathInfo(ref);
TRY_AWAIT(queryPathInfo(ref));
} catch (InvalidPath &) {
throw Error("cannot add '%s' to the binary cache because the reference '%s' does not exist",
printStorePath(info.path), printStorePath(ref));
@@ -343,7 +343,7 @@ BinaryCacheStore::queryPathFromHashPart(const std::string & hashPart)
try {
auto pseudoPath = StorePath(hashPart + "-" + MissingName);
try {
auto info = queryPathInfo(pseudoPath);
auto info = TRY_AWAIT(queryPathInfo(pseudoPath));
co_return info->path;
} catch (InvalidPath &) {
co_return std::nullopt;
@@ -354,7 +354,7 @@ try {
kj::Promise<Result<box_ptr<Source>>> BinaryCacheStore::narFromPath(const StorePath & storePath)
try {
auto info = queryPathInfo(storePath).cast<const NarInfo>();
auto info = TRY_AWAIT(queryPathInfo(storePath)).cast<const NarInfo>();
try {
auto file = getFile(info->url);
@@ -385,8 +385,9 @@ try {
co_return result::current_exception();
}
std::shared_ptr<const ValidPathInfo> BinaryCacheStore::queryPathInfoUncached(const StorePath & storePath)
{
kj::Promise<Result<std::shared_ptr<const ValidPathInfo>>>
BinaryCacheStore::queryPathInfoUncached(const StorePath & storePath)
try {
auto uri = getUri();
auto storePathS = printStorePath(storePath);
auto act = std::make_shared<Activity>(*logger, lvlTalkative, actQueryPathInfo,
@@ -397,11 +398,13 @@ std::shared_ptr<const ValidPathInfo> BinaryCacheStore::queryPathInfoUncached(con
auto data = getFileContents(narInfoFile);
if (!data) return nullptr;
if (!data) co_return result::success(nullptr);
stats.narInfoRead++;
return std::make_shared<NarInfo>(*this, *data, narInfoFile);
co_return std::make_shared<NarInfo>(*this, *data, narInfoFile);
} catch (...) {
co_return result::current_exception();
}
static ValidPathInfo makeAddToStoreInfo(
@@ -542,7 +545,8 @@ try {
S3 might return an outdated cached version. */
// downcast: BinaryCacheStore always returns NarInfo from queryPathInfoUncached, making it sound
auto narInfo = make_ref<NarInfo>(dynamic_cast<NarInfo const &>(*queryPathInfo(storePath)));
auto narInfo =
make_ref<NarInfo>(dynamic_cast<NarInfo const &>(*TRY_AWAIT(queryPathInfo(storePath))));
narInfo->sigs.insert(sigs.begin(), sigs.end());
lix/libstore/binary-cache-store.hh
+2 -1
View File
@@ -111,7 +111,8 @@ public:
kj::Promise<Result<bool>> isValidPathUncached(const StorePath & path) override;
std::shared_ptr<const ValidPathInfo> queryPathInfoUncached(const StorePath & path) override;
kj::Promise<Result<std::shared_ptr<const ValidPathInfo>>>
queryPathInfoUncached(const StorePath & path) override;
kj::Promise<Result<std::optional<StorePath>>>
queryPathFromHashPart(const std::string & hashPart) override;
lix/libstore/build/entry-points.cc
+1 -1
View File
@@ -145,7 +145,7 @@ try {
if (result.exitCode != Goal::ecSuccess) {
/* Since substituting the path didn't work, if we have a valid
deriver, then rebuild the deriver. */
auto info = queryPathInfo(path);
auto info = TRY_AWAIT(queryPathInfo(path));
if (info->deriver && TRY_AWAIT(isValidPath(*info->deriver))) {
TRY_AWAIT(processGoals(*this, *this, [&](GoalFactory & gf) {
Worker::Targets goals;
lix/libstore/build/local-derivation-goal.cc
+127 -107
View File
@@ -1050,23 +1050,26 @@ struct RestrictedStore : public virtual IndirectRootStore, public virtual GcStor
co_return result::current_exception();
}
std::shared_ptr<const ValidPathInfo> queryPathInfoUncached(const StorePath & path) override
{
kj::Promise<Result<std::shared_ptr<const ValidPathInfo>>>
queryPathInfoUncached(const StorePath & path) override
try {
if (goal.isAllowed(path)) {
try {
/* Censor impure information. */
auto info = std::make_shared<ValidPathInfo>(*next->queryPathInfo(path));
auto info = std::make_shared<ValidPathInfo>(*TRY_AWAIT(next->queryPathInfo(path)));
info->deriver.reset();
info->registrationTime = 0;
info->ultimate = false;
info->sigs.clear();
return info;
co_return info;
} catch (InvalidPath &) {
return nullptr;
co_return result::success(nullptr);
}
} else
return nullptr;
};
co_return result::success(nullptr);
} catch (...) {
co_return result::current_exception();
}
kj::Promise<Result<void>>
queryReferrers(const StorePath & path, StorePathSet & referrers) override
@@ -2403,7 +2406,7 @@ try {
if (buildMode == bmCheck) {
if (!TRY_AWAIT(worker.store.isValidPath(newInfo.path))) continue;
ValidPathInfo oldInfo(*worker.store.queryPathInfo(newInfo.path));
ValidPathInfo oldInfo(*TRY_AWAIT(worker.store.queryPathInfo(newInfo.path)));
if (newInfo.narHash != oldInfo.narHash) {
anyCheckMismatchSeen = true;
if (settings.runDiffHook || settings.keepFailed) {
@@ -2558,115 +2561,132 @@ try {
/* Compute the closure and closure size of some output. This
is slightly tricky because some of its references (namely
other outputs) may not be valid yet. */
auto getClosure = [&](const StorePath & path)
{
uint64_t closureSize = 0;
StorePathSet pathsDone;
std::queue<StorePath> pathsLeft;
pathsLeft.push(path);
// NOLINTNEXTLINE(cppcoreguidelines-avoid-capturing-lambda-coroutines)
auto getClosure = [&](const StorePath & path
) -> kj::Promise<Result<std::pair<StorePathSet, uint64_t>>> {
try {
uint64_t closureSize = 0;
StorePathSet pathsDone;
std::queue<StorePath> pathsLeft;
pathsLeft.push(path);
while (!pathsLeft.empty()) {
auto path = pathsLeft.front();
pathsLeft.pop();
if (!pathsDone.insert(path).second) continue;
while (!pathsLeft.empty()) {
auto path = pathsLeft.front();
pathsLeft.pop();
if (!pathsDone.insert(path).second) continue;
auto i = outputsByPath.find(worker.store.printStorePath(path));
if (i != outputsByPath.end()) {
closureSize += i->second.narSize;
for (auto & ref : i->second.references)
pathsLeft.push(ref);
} else {
auto info = worker.store.queryPathInfo(path);
closureSize += info->narSize;
for (auto & ref : info->references)
pathsLeft.push(ref);
auto i = outputsByPath.find(worker.store.printStorePath(path));
if (i != outputsByPath.end()) {
closureSize += i->second.narSize;
for (auto & ref : i->second.references)
pathsLeft.push(ref);
} else {
auto info = TRY_AWAIT(worker.store.queryPathInfo(path));
closureSize += info->narSize;
for (auto & ref : info->references)
pathsLeft.push(ref);
}
}
}
return std::make_pair(std::move(pathsDone), closureSize);
co_return std::make_pair(std::move(pathsDone), closureSize);
} catch (...) {
co_return result::current_exception();
}
};
auto applyChecks = [&](const Checks & checks)
{
if (checks.maxSize && info.narSize > *checks.maxSize)
throw BuildError("path '%s' is too large at %d bytes; limit is %d bytes",
worker.store.printStorePath(info.path), info.narSize, *checks.maxSize);
// NOLINTNEXTLINE(cppcoreguidelines-avoid-capturing-lambda-coroutines)
auto applyChecks = [&](const Checks & checks) -> kj::Promise<Result<void>> {
try {
if (checks.maxSize && info.narSize > *checks.maxSize)
throw BuildError("path '%s' is too large at %d bytes; limit is %d bytes",
worker.store.printStorePath(info.path), info.narSize, *checks.maxSize);
if (checks.maxClosureSize) {
uint64_t closureSize = getClosure(info.path).second;
if (closureSize > *checks.maxClosureSize)
throw BuildError("closure of path '%s' is too large at %d bytes; limit is %d bytes",
worker.store.printStorePath(info.path), closureSize, *checks.maxClosureSize);
}
if (checks.maxClosureSize) {
uint64_t closureSize = TRY_AWAIT(getClosure(info.path)).second;
if (closureSize > *checks.maxClosureSize)
throw BuildError("closure of path '%s' is too large at %d bytes; limit is %d bytes",
worker.store.printStorePath(info.path), closureSize, *checks.maxClosureSize);
}
auto checkRefs = [&](const std::optional<Strings> & value, bool allowed, bool recursive)
{
if (!value) return;
// NOLINTNEXTLINE(cppcoreguidelines-avoid-capturing-lambda-coroutines)
auto checkRefs = [&](const std::optional<Strings> & value,
bool allowed,
bool recursive) -> kj::Promise<Result<void>> {
try {
if (!value) co_return result::success();
/* Parse a list of reference specifiers. Each element must
either be a store path, or the symbolic name of the output
of the derivation (such as `out'). */
StorePathSet spec;
for (auto & i : *value) {
if (worker.store.isStorePath(i))
spec.insert(worker.store.parseStorePath(i));
else if (auto output = get(newlyBuiltOutputs, i))
spec.insert(output->path);
else if (auto storePath = get(alreadyRegisteredOutputs, i))
spec.insert(*storePath);
else {
std::string outputsListing = concatMapStringsSep(
", ",
newlyBuiltOutputs,
[](auto & o) { return o.first; }
);
if (!alreadyRegisteredOutputs.empty()) {
outputsListing.append(outputsListing.empty() ? "" : ", ");
outputsListing.append(concatMapStringsSep(
", ",
alreadyRegisteredOutputs,
[](auto & o) { return o.first; })
);
/* Parse a list of reference specifiers. Each element must
either be a store path, or the symbolic name of the output
of the derivation (such as `out'). */
StorePathSet spec;
for (auto & i : *value) {
if (worker.store.isStorePath(i))
spec.insert(worker.store.parseStorePath(i));
else if (auto output = get(newlyBuiltOutputs, i))
spec.insert(output->path);
else if (auto storePath = get(alreadyRegisteredOutputs, i))
spec.insert(*storePath);
else {
std::string outputsListing = concatMapStringsSep(
", ",
newlyBuiltOutputs,
[](auto & o) { return o.first; }
);
if (!alreadyRegisteredOutputs.empty()) {
outputsListing.append(outputsListing.empty() ? "" : ", ");
outputsListing.append(concatMapStringsSep(
", ",
alreadyRegisteredOutputs,
[](auto & o) { return o.first; })
);
}
throw BuildError("derivation '%s' output check for '%s' contains an illegal reference specifier '%s',"
" expected store path or output name (one of [%s])",
worker.store.printStorePath(drvPath), outputName, i, outputsListing);
}
}
throw BuildError("derivation '%s' output check for '%s' contains an illegal reference specifier '%s',"
" expected store path or output name (one of [%s])",
worker.store.printStorePath(drvPath), outputName, i, outputsListing);
auto used = recursive
? TRY_AWAIT(getClosure(info.path)).first
: info.references;
if (recursive && checks.ignoreSelfRefs)
used.erase(info.path);
StorePathSet badPaths;
for (auto & i : used)
if (allowed) {
if (!spec.count(i))
badPaths.insert(i);
} else {
if (spec.count(i))
badPaths.insert(i);
}
if (!badPaths.empty()) {
std::string badPathsStr;
for (auto & i : badPaths) {
badPathsStr += "\n ";
badPathsStr += worker.store.printStorePath(i);
}
throw BuildError("output '%s' is not allowed to refer to the following paths:%s",
worker.store.printStorePath(info.path), badPathsStr);
}
co_return result::success();
} catch (...) {
co_return result::current_exception();
}
}
};
auto used = recursive
? getClosure(info.path).first
: info.references;
if (recursive && checks.ignoreSelfRefs)
used.erase(info.path);
StorePathSet badPaths;
for (auto & i : used)
if (allowed) {
if (!spec.count(i))
badPaths.insert(i);
} else {
if (spec.count(i))
badPaths.insert(i);
}
if (!badPaths.empty()) {
std::string badPathsStr;
for (auto & i : badPaths) {
badPathsStr += "\n ";
badPathsStr += worker.store.printStorePath(i);
}
throw BuildError("output '%s' is not allowed to refer to the following paths:%s",
worker.store.printStorePath(info.path), badPathsStr);
}
};
checkRefs(checks.allowedReferences, true, false);
checkRefs(checks.allowedRequisites, true, true);
checkRefs(checks.disallowedReferences, false, false);
checkRefs(checks.disallowedRequisites, false, true);
TRY_AWAIT(checkRefs(checks.allowedReferences, true, false));
TRY_AWAIT(checkRefs(checks.allowedRequisites, true, true));
TRY_AWAIT(checkRefs(checks.disallowedReferences, false, false));
TRY_AWAIT(checkRefs(checks.disallowedRequisites, false, true));
co_return result::success();
} catch (...) {
co_return result::current_exception();
}
};
if (auto structuredAttrs = parsedDrv->getStructuredAttrs()) {
@@ -2717,7 +2737,7 @@ try {
checks.disallowedReferences = get_("disallowedReferences");
checks.disallowedRequisites = get_("disallowedRequisites");
applyChecks(checks);
TRY_AWAIT(applyChecks(checks));
}
}
} else {
@@ -2728,7 +2748,7 @@ try {
checks.allowedRequisites = parsedDrv->getStringsAttr("allowedRequisites");
checks.disallowedReferences = parsedDrv->getStringsAttr("disallowedReferences");
checks.disallowedRequisites = parsedDrv->getStringsAttr("disallowedRequisites");
applyChecks(checks);
TRY_AWAIT(applyChecks(checks));
}
}
lix/libstore/build/substitution-goal.cc
+1 -2
View File
@@ -109,8 +109,7 @@ try {
do {
try {
// FIXME: make async
info = sub->queryPathInfo(subPath ? *subPath : storePath);
info = TRY_AWAIT(sub->queryPathInfo(subPath ? *subPath : storePath));
break;
} catch (InvalidPath &) {
} catch (SubstituterDisabled &) {
lix/libstore/build/worker.cc
+2 -1
View File
@@ -1,5 +1,6 @@
#include "build/derivation-goal.hh"
#include "lix/libutil/async-collect.hh"
#include "lix/libutil/async.hh"
#include "lix/libutil/charptr-cast.hh"
#include "lix/libstore/build/worker.hh"
#include "lix/libutil/finally.hh"
@@ -326,7 +327,7 @@ try {
auto i = pathContentsGoodCache.find(path);
if (i != pathContentsGoodCache.end()) co_return i->second;
printInfo("checking path '%s'...", store.printStorePath(path));
auto info = store.queryPathInfo(path);
auto info = TRY_AWAIT(store.queryPathInfo(path));
bool res;
if (!pathExists(store.printStorePath(path)))
res = false;
lix/libstore/daemon.cc
+3 -3
View File
@@ -426,14 +426,14 @@ static void performOp(AsyncIoRoot & aio, TunnelLogger * logger, ref<Store> store
// We could stream this by changing Store
std::string contents = source.drain();
auto path = aio.blockOn(store->addTextToStore(name, contents, refs, repair));
return store->queryPathInfo(path);
return aio.blockOn(store->queryPathInfo(path));
},
[&](const FileIngestionMethod & fim) {
AsyncSourceInputStream stream{source};
auto path = aio.blockOn(
store->addToStoreFromDump(stream, name, fim, hashType, repair, refs)
);
return store->queryPathInfo(path);
return aio.blockOn(store->queryPathInfo(path));
},
}, contentAddressMethod.raw);
}();
@@ -837,7 +837,7 @@ static void performOp(AsyncIoRoot & aio, TunnelLogger * logger, ref<Store> store
std::shared_ptr<const ValidPathInfo> info;
logger->startWork();
try {
info = store->queryPathInfo(path);
info = aio.blockOn(store->queryPathInfo(path));
} catch (InvalidPath &) {
// The path being invalid isn't fatal here since it will just be
// sent as not present.
lix/libstore/dummy-store.cc
+3 -2
View File
@@ -34,9 +34,10 @@ struct DummyStore final : public Store
return *uriSchemes().begin();
}
std::shared_ptr<const ValidPathInfo> queryPathInfoUncached(const StorePath & path) override
kj::Promise<Result<std::shared_ptr<const ValidPathInfo>>>
queryPathInfoUncached(const StorePath & path) override
{
return nullptr;
return {result::success(nullptr)};
}
/**
lix/libstore/export-import.cc
+1 -1
View File
@@ -27,7 +27,7 @@ try {
kj::Promise<Result<void>> Store::exportPath(const StorePath & path, Sink & sink)
try {
auto info = queryPathInfo(path);
auto info = TRY_AWAIT(queryPathInfo(path));
HashSink hashSink(HashType::SHA256);
TeeSink teeSink(sink, hashSink);
lix/libstore/gc.cc
+1 -1
View File
@@ -780,7 +780,7 @@ try {
{
if (maybeOutPath &&
TRY_AWAIT(isValidPath(*maybeOutPath)) &&
queryPathInfo(*maybeOutPath)->deriver == *path)
TRY_AWAIT(queryPathInfo(*maybeOutPath))->deriver == *path)
enqueue(*maybeOutPath);
}
}
lix/libstore/legacy-ssh-store.cc
+7 -4
View File
@@ -160,8 +160,9 @@ struct LegacySSHStore final : public Store
return *uriSchemes().begin() + "://" + host;
}
std::shared_ptr<const ValidPathInfo> queryPathInfoUncached(const StorePath & path) override
{
kj::Promise<Result<std::shared_ptr<const ValidPathInfo>>>
queryPathInfoUncached(const StorePath & path) override
try {
auto conn(connections->get());
/* No longer support missing NAR hash */
@@ -173,7 +174,7 @@ struct LegacySSHStore final : public Store
conn->to.flush();
auto p = readString(conn->from);
if (p.empty()) return nullptr;
if (p.empty()) co_return result::success(nullptr);
auto path2 = parseStorePath(p);
assert(path == path2);
auto info = std::make_shared<ValidPathInfo>(
@@ -186,7 +187,9 @@ struct LegacySSHStore final : public Store
auto s = readString(conn->from);
assert(s == "");
return info;
co_return info;
} catch (...) {
co_return result::current_exception();
}
kj::Promise<Result<void>> addToStore(const ValidPathInfo & info, AsyncInputStream & source,
lix/libstore/local-store.cc
+19 -7
View File
@@ -916,12 +916,22 @@ try {
}
std::shared_ptr<const ValidPathInfo> LocalStore::queryPathInfoUncached(const StorePath & path)
{
return retrySQLite([&]() {
auto state = dbPool.get();
return queryPathInfoInternal(*state, path);
}, always_progresses);
kj::Promise<Result<std::shared_ptr<const ValidPathInfo>>>
LocalStore::queryPathInfoUncached(const StorePath & path)
try {
co_return TRY_AWAIT(
// NOLINTNEXTLINE(cppcoreguidelines-avoid-capturing-lambda-coroutines)
retrySQLite([&]() -> kj::Promise<Result<std::shared_ptr<const ValidPathInfo>>> {
try {
auto state = dbPool.get();
co_return queryPathInfoInternal(*state, path);
} catch (...) {
co_return result::current_exception();
}
})
);
} catch (...) {
co_return result::current_exception();
}
@@ -1734,7 +1744,9 @@ try {
for (auto & i : validPaths) {
std::optional<Error> caught;
try {
auto info = std::const_pointer_cast<ValidPathInfo>(std::shared_ptr<const ValidPathInfo>(queryPathInfo(i)));
auto info = std::const_pointer_cast<ValidPathInfo>(
std::shared_ptr<const ValidPathInfo>(TRY_AWAIT(queryPathInfo(i)))
);
/* Check the content hash (optionally - slow). */
printMsg(lvlTalkative, "checking contents of '%s'", printStorePath(i));
lix/libstore/local-store.hh
+2 -1
View File
@@ -196,7 +196,8 @@ public:
kj::Promise<Result<StorePathSet>> queryAllValidPaths() override;
std::shared_ptr<const ValidPathInfo> queryPathInfoUncached(const StorePath & path) override;
kj::Promise<Result<std::shared_ptr<const ValidPathInfo>>>
queryPathInfoUncached(const StorePath & path) override;
kj::Promise<Result<void>>
queryReferrers(const StorePath & path, StorePathSet & referrers) override;
lix/libstore/make-content-addressed.cc
+1 -1
View File
@@ -22,7 +22,7 @@ try {
for (auto & path : paths) {
auto pathS = srcStore.printStorePath(path);
auto oldInfo = srcStore.queryPathInfo(path);
auto oldInfo = TRY_AWAIT(srcStore.queryPathInfo(path));
std::string oldHashPart(path.hashPart());
StringSink sink;
lix/libstore/misc.cc
+17 -8
View File
@@ -9,6 +9,7 @@
#include "lix/libutil/closure.hh"
#include "lix/libstore/filetransfer.hh"
#include "lix/libutil/strings.hh"
#include <kj/async.h>
#include <kj/common.h>
namespace nix {
@@ -70,8 +71,13 @@ try {
paths_.merge(TRY_AWAIT(computeClosureAsync<StorePath>(
startPaths,
[&](const StorePath& path) -> kj::Promise<Result<std::set<StorePath>>> {
return queryDeps(path, queryPathInfo(path));
// NOLINTNEXTLINE(cppcoreguidelines-avoid-capturing-lambda-coroutines)
[&](const StorePath& path) -> kj::Promise<Result<StorePathSet>> {
try {
co_return TRY_AWAIT(queryDeps(path, TRY_AWAIT(queryPathInfo(path))));
} catch (...) {
co_return result::current_exception();
}
})));
co_return result::success();
} catch (...) {
@@ -369,12 +375,15 @@ try {
kj::Promise<Result<StorePaths>> Store::topoSortPaths(const StorePathSet & paths)
try {
co_return topoSort(paths,
{[&](const StorePath & path) {
co_return TRY_AWAIT(topoSortAsync(paths,
// NOLINTNEXTLINE(cppcoreguidelines-avoid-capturing-lambda-coroutines)
{[&](const StorePath & path) -> kj::Promise<Result<StorePathSet>> {
try {
return queryPathInfo(path)->references;
co_return TRY_AWAIT(queryPathInfo(path))->references;
} catch (InvalidPath &) {
return StorePathSet();
co_return StorePathSet();
} catch (...) {
co_return result::current_exception();
}
}},
{[&](const StorePath & path, const StorePath & parent) {
@@ -382,7 +391,7 @@ try {
"cycle detected in the references of '%s' from '%s'",
printStorePath(path),
printStorePath(parent));
}});
}}));
} catch (...) {
co_return result::current_exception();
}
@@ -461,7 +470,7 @@ try {
for (const auto & [inputDrv, inputNode] : drv.inputDrvs.map)
TRY_AWAIT(accumRealisations(inputDrv, inputNode));
auto info = store.queryPathInfo(outputPath);
auto info = TRY_AWAIT(store.queryPathInfo(outputPath));
co_return TRY_AWAIT(drvOutputReferences(
TRY_AWAIT(Realisation::closure(store, inputRealisations)), info->references
lix/libstore/remote-store.cc
+9 -6
View File
@@ -286,8 +286,9 @@ try {
}
std::shared_ptr<const ValidPathInfo> RemoteStore::queryPathInfoUncached(const StorePath & path)
{
kj::Promise<Result<std::shared_ptr<const ValidPathInfo>>>
RemoteStore::queryPathInfoUncached(const StorePath & path)
try {
auto conn(getConnection());
conn->to << WorkerProto::Op::QueryPathInfo << printStorePath(path);
try {
@@ -295,16 +296,18 @@ std::shared_ptr<const ValidPathInfo> RemoteStore::queryPathInfoUncached(const St
} catch (Error & e) {
// Ugly backwards compatibility hack. TODO(fj#325): remove.
if (e.msg().find("is not valid") != std::string::npos)
return nullptr;
co_return result::success(nullptr);
throw;
}
bool valid; conn->from >> valid;
if (!valid) return nullptr;
if (!valid) co_return result::success(nullptr);
return std::make_shared<ValidPathInfo>(
co_return std::make_shared<ValidPathInfo>(
StorePath{path},
WorkerProto::Serialise<UnkeyedValidPathInfo>::read(*this, *conn));
} catch (...) {
co_return result::current_exception();
}
@@ -497,7 +500,7 @@ try {
auto path = parseStorePath(readString(conn->from));
// Release our connection to prevent a deadlock in queryPathInfo().
conn_.reset();
co_return queryPathInfo(path);
co_return TRY_AWAIT(queryPathInfo(path));
}
} catch (...) {
co_return result::current_exception();
lix/libstore/remote-store.hh
+2 -1
View File
@@ -57,7 +57,8 @@ public:
kj::Promise<Result<StorePathSet>> queryAllValidPaths() override;
std::shared_ptr<const ValidPathInfo> queryPathInfoUncached(const StorePath & path) override;
kj::Promise<Result<std::shared_ptr<const ValidPathInfo>>>
queryPathInfoUncached(const StorePath & path) override;
kj::Promise<Result<void>>
queryReferrers(const StorePath & path, StorePathSet & referrers) override;
lix/libstore/s3-binary-cache-store.cc
+1 -1
View File
@@ -315,7 +315,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
a GET is unlikely to be slower than HEAD. */
kj::Promise<Result<bool>> isValidPathUncached(const StorePath & storePath) override
try {
queryPathInfo(storePath);
TRY_AWAIT(queryPathInfo(storePath));
co_return true;
} catch (InvalidPath & e) {
co_return false;
lix/libstore/store-api.cc
+20 -17
View File
@@ -21,6 +21,7 @@
#include "lix/libstore/worker-protocol.hh"
#include "lix/libutil/users.hh"
#include <functional>
#include <kj/async.h>
#include <mutex>
#include <nlohmann/json.hpp>
@@ -600,7 +601,7 @@ try {
debug("checking substituter '%s' for path '%s'", sub->getUri(), sub->printStorePath(subPath));
try {
auto info = sub->queryPathInfo(subPath);
auto info = TRY_AWAIT(sub->queryPathInfo(subPath));
if (sub->config().storeDir != config().storeDir
&& !(info->isContentAddressed(*sub) && info->references.empty()))
@@ -671,7 +672,7 @@ try {
queryPathInfoUncached(). */
kj::Promise<Result<bool>> Store::isValidPathUncached(const StorePath & path)
try {
queryPathInfo(path);
TRY_AWAIT(queryPathInfo(path));
co_return true;
} catch (InvalidPath &) {
co_return false;
@@ -696,8 +697,8 @@ static void ensureGoodStorePath(Store * store, const StorePath & expected, const
}
ref<const ValidPathInfo> Store::queryPathInfo(const StorePath & storePath)
{
kj::Promise<Result<ref<const ValidPathInfo>>> Store::queryPathInfo(const StorePath & storePath)
try {
auto hashPart = std::string(storePath.hashPart());
{
@@ -706,7 +707,7 @@ ref<const ValidPathInfo> Store::queryPathInfo(const StorePath & storePath)
stats.narInfoReadAverted++;
if (!res->didExist())
throw InvalidPath("path '%s' does not exist in the store", printStorePath(storePath));
return ref<const ValidPathInfo>(res->value);
co_return ref<const ValidPathInfo>(res->value);
}
}
@@ -721,11 +722,11 @@ ref<const ValidPathInfo> Store::queryPathInfo(const StorePath & storePath)
if (res.first == NarInfoDiskCache::oInvalid)
throw InvalidPath("path '%s' does not exist in the store", printStorePath(storePath));
}
return ref<const ValidPathInfo>(res.second);
co_return ref<const ValidPathInfo>(res.second);
}
}
auto info = queryPathInfoUncached(storePath);
auto info = TRY_AWAIT(queryPathInfoUncached(storePath));
if (info) {
// first, before we cache anything, check that the store gave us valid data.
ensureGoodStorePath(this, storePath, info->path);
@@ -745,7 +746,9 @@ ref<const ValidPathInfo> Store::queryPathInfo(const StorePath & storePath)
throw InvalidPath("path '%s' does not exist in the store", printStorePath(storePath));
}
return ref<const ValidPathInfo>(info);
co_return ref<const ValidPathInfo>(info);
} catch (...) {
co_return result::current_exception();
}
kj::Promise<Result<std::shared_ptr<const Realisation>>> Store::queryRealisation(const DrvOutput & id)
@@ -823,14 +826,14 @@ try {
std::condition_variable wakeup;
ThreadPool pool{"queryValidPaths pool"};
auto doQuery = [&](const StorePath & path) {
auto doQuery = [&](AsyncIoRoot & aio, const StorePath & path) {
checkInterrupt();
bool exists = false;
std::exception_ptr newExc{};
try {
queryPathInfo(path);
aio.blockOn(queryPathInfo(path));
exists = true;
} catch (InvalidPath &) {
} catch (...) {
@@ -853,7 +856,7 @@ try {
};
for (auto & path : paths)
pool.enqueue(std::bind(doQuery, path));
pool.enqueueWithAio(std::bind(doQuery, std::placeholders::_1, path));
TRY_AWAIT(pool.processAsync());
@@ -881,7 +884,7 @@ try {
for (auto & i : paths) {
s += printStorePath(i) + "\n";
auto info = queryPathInfo(i);
auto info = TRY_AWAIT(queryPathInfo(i));
if (showHash) {
s += info->narHash.to_string(Base::Base16, false) + "\n";
@@ -952,7 +955,7 @@ try {
auto& jsonPath = jsonList.emplace_back(json::object());
try {
auto info = queryPathInfo(storePath);
auto info = TRY_AWAIT(queryPathInfo(storePath));
jsonPath["path"] = printStorePath(info->path);
jsonPath["valid"] = true;
@@ -1024,7 +1027,7 @@ try {
StorePathSet closure;
TRY_AWAIT(computeFSClosure(storePath, closure, false, false));
for (auto & p : closure) {
auto info = queryPathInfo(p);
auto info = TRY_AWAIT(queryPathInfo(p));
totalNarSize += info->narSize;
auto narInfo = std::dynamic_pointer_cast<const NarInfo>(
std::shared_ptr<const ValidPathInfo>(info));
@@ -1106,7 +1109,7 @@ try {
{storePathS, srcUri, dstUri});
PushActivity pact(act.id);
auto info = srcStore.queryPathInfo(storePath);
auto info = TRY_AWAIT(srcStore.queryPathInfo(storePath));
// recompute store path on the chance dstStore does it differently
if (info->ca && info->references.empty()) {
@@ -1239,7 +1242,7 @@ try {
};
for (auto & missingPath : sortedMissing) {
auto info = srcStore.queryPathInfo(missingPath);
auto info = TRY_AWAIT(srcStore.queryPathInfo(missingPath));
auto storePathForDst = computeStorePathForDst(*info);
pathsMap.insert_or_assign(missingPath, storePathForDst);
@@ -1411,7 +1414,7 @@ try {
if (!path.isDerivation()) {
try {
auto info = queryPathInfo(path);
auto info = TRY_AWAIT(queryPathInfo(path));
co_return info->deriver;
} catch (InvalidPath &) {
co_return std::nullopt;
lix/libstore/store-api.hh
+3 -2
View File
@@ -385,7 +385,7 @@ public:
* Query information about a valid path. It is permitted to omit
* the name part of the store path.
*/
ref<const ValidPathInfo> queryPathInfo(const StorePath & path);
kj::Promise<Result<ref<const ValidPathInfo>>> queryPathInfo(const StorePath & path);
/**
* Query the information about a realisation.
@@ -420,7 +420,8 @@ protected:
* Queries the path info without caching.
* Note to implementors: should return `nullptr` when the path is not found.
*/
virtual std::shared_ptr<const ValidPathInfo> queryPathInfoUncached(const StorePath & path) = 0;
virtual kj::Promise<Result<std::shared_ptr<const ValidPathInfo>>>
queryPathInfoUncached(const StorePath & path) = 0;
virtual kj::Promise<Result<std::shared_ptr<const Realisation>>>
queryRealisationUncached(const DrvOutput &) = 0;
lix/nix/diff-closures.cc
+14 -9
View File
@@ -67,17 +67,22 @@ try {
auto & beforeVersions = beforeClosure[name];
auto & afterVersions = afterClosure[name];
auto totalSize = [&](const std::map<std::string, std::map<StorePath, Info>> & versions)
{
uint64_t sum = 0;
for (auto & [_, paths] : versions)
for (auto & [path, _] : paths)
sum += store->queryPathInfo(path)->narSize;
return sum;
// NOLINTNEXTLINE(cppcoreguidelines-avoid-capturing-lambda-coroutines)
auto totalSize = [&](const std::map<std::string, std::map<StorePath, Info>> & versions
) -> kj::Promise<Result<uint64_t>> {
try {
uint64_t sum = 0;
for (auto & [_, paths] : versions)
for (auto & [path, _] : paths)
sum += TRY_AWAIT(store->queryPathInfo(path))->narSize;
co_return sum;
} catch (...) {
co_return result::current_exception();
}
};
auto beforeSize = totalSize(beforeVersions);
auto afterSize = totalSize(afterVersions);
auto beforeSize = TRY_AWAIT(totalSize(beforeVersions));
auto afterSize = TRY_AWAIT(totalSize(afterVersions));
auto sizeDelta = (int64_t) afterSize - (int64_t) beforeSize;
auto showDelta = std::abs(sizeDelta) >= 8 * 1024;
lix/nix/flake.cc
+1 -1
View File
@@ -1439,7 +1439,7 @@ struct CmdFlakePrefetch : FlakeCommand, MixJSON
auto originalRef = getFlakeRef();
auto resolvedRef = aio().blockOn(originalRef.resolve(store));
auto [tree, lockedRef] = aio().blockOn(resolvedRef.fetchTree(store));
auto hash = store->queryPathInfo(tree.storePath)->narHash;
auto hash = aio().blockOn(store->queryPathInfo(tree.storePath))->narHash;
if (json) {
auto res = nlohmann::json::object();
lix/nix/path-info.cc
+2 -1
View File
@@ -2,6 +2,7 @@
#include "lix/libmain/shared.hh"
#include "lix/libstore/store-api.hh"
#include "lix/libmain/common-args.hh"
#include "lix/libutil/async.hh"
#include <algorithm>
#include <array>
@@ -100,7 +101,7 @@ struct CmdPathInfo : StorePathsCommand, MixJSON
else {
for (auto & storePath : storePaths) {
auto info = store->queryPathInfo(storePath);
auto info = aio().blockOn(store->queryPathInfo(storePath));
auto storePathS = store->printStorePath(info->path);
std::cout << storePathS;
lix/nix/sigs.cc
+3 -3
View File
@@ -50,13 +50,13 @@ struct CmdCopySigs : StorePathsCommand
auto storePath = store->parseStorePath(storePathS);
auto info = store->queryPathInfo(storePath);
auto info = aio.blockOn(store->queryPathInfo(storePath));
StringSet newSigs;
for (auto & store2 : substituters) {
try {
auto info2 = store2->queryPathInfo(info->path);
auto info2 = aio.blockOn(store2->queryPathInfo(info->path));
/* Don't import signatures that don't match this
binary. */
@@ -130,7 +130,7 @@ struct CmdSign : StorePathsCommand
auto storePath = store->parseStorePath(storePathS);
auto info = store->queryPathInfo(storePath);
auto info = aio.blockOn(store->queryPathInfo(storePath));
auto info2(*info);
info2.sigs.clear();
lix/nix/verify.cc
+2 -2
View File
@@ -90,7 +90,7 @@ struct CmdVerify : StorePathsCommand
MaintainCount<std::atomic<size_t>> mcActive(active);
update();
auto info = store->queryPathInfo(storePath);
auto info = aio.blockOn(store->queryPathInfo(storePath));
// Note: info->path can be different from storePath
// for binary cache stores when using --all (since we
@@ -143,7 +143,7 @@ struct CmdVerify : StorePathsCommand
for (auto & store2 : substituters) {
if (validSigs >= actualSigsNeeded) break;
try {
auto info2 = store2->queryPathInfo(info->path);
auto info2 = aio.blockOn(store2->queryPathInfo(info->path));
if (info2->isContentAddressed(*store)) validSigs = ValidPathInfo::maxSigs;
doSigs(info2->sigs);
} catch (InvalidPath &) {
lix/nix/why-depends.cc
+1 -1
View File
@@ -132,7 +132,7 @@ struct CmdWhyDepends : SourceExprCommand, MixOperateOnOptions
for (auto & path : closure)
graph.emplace(path, Node {
.path = path,
.refs = store->queryPathInfo(path)->references,
.refs = aio().blockOn(store->queryPathInfo(path))->references,
.dist = path == dependencyPath ? 0 : inf
});
perl/lib/Nix/Store.xs
+4 -4
View File
@@ -79,7 +79,7 @@ int isValidPath(char * path)
SV * queryReferences(char * path)
PPCODE:
try {
for (auto & i : store()->queryPathInfo(store()->parseStorePath(path))->references)
for (auto & i : aio().blockOn(store()->queryPathInfo(store()->parseStorePath(path)))->references)
XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(i).c_str(), 0)));
} catch (Error & e) {
croak("%s", e.what());
@@ -89,7 +89,7 @@ SV * queryReferences(char * path)
SV * queryPathHash(char * path)
PPCODE:
try {
auto s = store()->queryPathInfo(store()->parseStorePath(path))->narHash.to_string(Base::Base32, true);
auto s = aio().blockOn(store()->queryPathInfo(store()->parseStorePath(path)))->narHash.to_string(Base::Base32, true);
XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
} catch (Error & e) {
croak("%s", e.what());
@@ -99,7 +99,7 @@ SV * queryPathHash(char * path)
SV * queryDeriver(char * path)
PPCODE:
try {
auto info = store()->queryPathInfo(store()->parseStorePath(path));
auto info = aio().blockOn(store()->queryPathInfo(store()->parseStorePath(path)));
if (!info->deriver) XSRETURN_UNDEF;
XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(*info->deriver).c_str(), 0)));
} catch (Error & e) {
@@ -110,7 +110,7 @@ SV * queryDeriver(char * path)
SV * queryPathInfo(char * path, int base32)
PPCODE:
try {
auto info = store()->queryPathInfo(store()->parseStorePath(path));
auto info = aio().blockOn(store()->queryPathInfo(store()->parseStorePath(path)));
if (!info->deriver)
XPUSHs(&PL_sv_undef);
else