guiwf/lix
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

libstore: asyncify Store::query{,Partial}DerivationOutputMap

Browse Source 
Change-Id: I1383f7986d454963409096501df47ddd4aa33602
This commit is contained in:
eldritch horrors
2025-02-25 02:09:22 +00:00
parent 446af4c6fe
commit c566a69e79
16 changed files with 142 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lix/legacy/nix-build.cc
+6 -3
View File
@@ -410,7 +410,8 @@ static void main_nix_build(AsyncIoRoot & aio, std::string programName, Strings a
if (dryRun) return;
if (shellDrv) {
auto shellDrvOutputs = store->queryPartialDerivationOutputMap(shellDrv.value(), &*evalStore);
auto shellDrvOutputs =
aio.blockOn(store->queryPartialDerivationOutputMap(shellDrv.value(), &*evalStore));
shell = store->printStorePath(shellDrvOutputs.at("out").value()) + "/bin/bash";
}
@@ -461,7 +462,8 @@ static void main_nix_build(AsyncIoRoot & aio, std::string programName, Strings a
std::function<void(const StorePath &, const DerivedPathMap<StringSet>::ChildNode &)> accumInputClosure;
accumInputClosure = [&](const StorePath & inputDrv, const DerivedPathMap<StringSet>::ChildNode & inputNode) {
auto outputs = store->queryPartialDerivationOutputMap(inputDrv, &*evalStore);
auto outputs =
aio.blockOn(store->queryPartialDerivationOutputMap(inputDrv, &*evalStore));
for (auto & i : inputNode.value) {
auto o = outputs.at(i);
aio.blockOn(store->computeFSClosure(*o, inputs));
@@ -608,7 +610,8 @@ static void main_nix_build(AsyncIoRoot & aio, std::string programName, Strings a
if (counter)
drvPrefix += fmt("-%d", counter + 1);
auto builtOutputs = store->queryPartialDerivationOutputMap(drvPath, &*evalStore);
auto builtOutputs =
aio.blockOn(store->queryPartialDerivationOutputMap(drvPath, &*evalStore));
auto maybeOutputPath = builtOutputs.at(outputName);
assert(maybeOutputPath);
lix/legacy/nix-env.cc
+2 -1
View File
@@ -448,7 +448,8 @@ static void queryInstSources(EvalState & state,
if (path.isDerivation()) {
elem.setDrvPath(path);
auto outputs = state.ctx.store->queryDerivationOutputMap(path);
auto outputs =
state.aio.blockOn(state.ctx.store->queryDerivationOutputMap(path));
elem.setOutPath(outputs.at("out"));
if (name.size() >= drvExtension.size() &&
std::string(name, name.size() - drvExtension.size()) == drvExtension)
lix/legacy/nix-store.cc
+1 -1
View File
@@ -68,7 +68,7 @@ try {
if (path.path.isDerivation()) {
if (build) TRY_AWAIT(store->buildPaths({path.toDerivedPath()}));
auto outputPaths = store->queryDerivationOutputMap(path.path);
auto outputPaths = TRY_AWAIT(store->queryDerivationOutputMap(path.path));
Derivation drv = TRY_AWAIT(store->derivationFromPath(path.path));
rootNr++;
lix/libcmd/repl.cc
+3 -1
View File
@@ -778,7 +778,9 @@ ProcessLineResult NixRepl::processLine(std::string line)
}));
auto drv = evaluator.store->readDerivation(drvPath);
logger->cout("\nThis derivation produced the following outputs:");
for (auto & [outputName, outputPath] : evaluator.store->queryDerivationOutputMap(drvPath)) {
for (auto & [outputName, outputPath] :
state.aio.blockOn(evaluator.store->queryDerivationOutputMap(drvPath)))
{
auto localStore = evaluator.store.dynamic_pointer_cast<LocalFSStore>();
if (localStore && command == ":bl") {
std::string symlink = "repl-result-" + outputName;
lix/libstore/build/derivation-goal.cc
+42 -28
View File
@@ -482,7 +482,8 @@ try {
std::map<StorePath, StorePath> outputsToDrv;
for (auto & i : inputClosure)
if (i.isDerivation()) {
auto depOutputs = worker.store.queryPartialDerivationOutputMap(i, &worker.evalStore);
auto depOutputs =
TRY_AWAIT(worker.store.queryPartialDerivationOutputMap(i, &worker.evalStore));
for (auto & j : depOutputs)
if (j.second)
outputsToDrv.insert_or_assign(*j.second, i);
@@ -628,7 +629,9 @@ try {
/* Add the relevant output closures of the input derivation
`i' as input paths. Only add the closures of output paths
that are specified as inputs. */
auto getOutput = [&](const std::string & outputName) {
// NOLINTNEXTLINE(cppcoreguidelines-avoid-capturing-lambda-coroutines)
auto getOutput = [&](const std::string & outputName
) -> kj::Promise<Result<StorePath>> {
/* TODO (impure derivations-induced tech debt):
Tracking input derivation outputs statefully through the
goals is error prone and has led to bugs.
@@ -640,32 +643,38 @@ try {
a representation in the store, which is a usability problem
in itself. When implementing this logic entirely with lookups
make sure that they're cached. */
if (auto outPath = get(inputDrvOutputs, { depDrvPath, outputName })) {
return *outPath;
}
else {
auto outMap = [&]{
for (auto * drvStore : { &worker.evalStore, &worker.store })
if (drvStore->isValidPath(depDrvPath))
return worker.store.queryDerivationOutputMap(depDrvPath, drvStore);
assert(false);
}();
auto outMapPath = outMap.find(outputName);
if (outMapPath == outMap.end()) {
throw Error(
"derivation '%s' requires non-existent output '%s' from input derivation '%s'",
worker.store.printStorePath(drvPath), outputName, worker.store.printStorePath(depDrvPath));
try {
if (auto outPath = get(inputDrvOutputs, { depDrvPath, outputName })) {
co_return *outPath;
}
return outMapPath->second;
else {
auto outMap = worker.evalStore.isValidPath(depDrvPath)
? TRY_AWAIT(worker.store.queryDerivationOutputMap(depDrvPath, &worker.evalStore))
: worker.store.isValidPath(depDrvPath)
? TRY_AWAIT(worker.store.queryDerivationOutputMap(depDrvPath, &worker.store))
: (assert(false), OutputPathMap{});
auto outMapPath = outMap.find(outputName);
if (outMapPath == outMap.end()) {
throw Error(
"derivation '%s' requires non-existent output '%s' from input derivation '%s'",
worker.store.printStorePath(drvPath), outputName, worker.store.printStorePath(depDrvPath));
}
co_return outMapPath->second;
}
} catch (...) {
co_return result::current_exception();
}
};
for (auto & outputName : inputNode.value)
TRY_AWAIT(worker.store.computeFSClosure(getOutput(outputName), inputPaths));
for (auto & outputName : inputNode.value) {
TRY_AWAIT(
worker.store.computeFSClosure(TRY_AWAIT(getOutput(outputName)), inputPaths)
);
}
for (auto & [outputName, childNode] : inputNode.childMap)
TRY_AWAIT(accumInputPaths(getOutput(outputName), childNode));
TRY_AWAIT(accumInputPaths(TRY_AWAIT(getOutput(outputName)), childNode));
co_return result::success();
} catch (...) {
co_return result::current_exception();
@@ -1602,9 +1611,12 @@ try {
res.insert_or_assign(name, output.path(worker.store, drv->name, name));
co_return res;
} else {
for (auto * drvStore : { &worker.evalStore, &worker.store })
if (drvStore->isValidPath(drvPath))
co_return worker.store.queryPartialDerivationOutputMap(drvPath, drvStore);
for (auto * drvStore : {&worker.evalStore, &worker.store}) {
if (drvStore->isValidPath(drvPath)) {
co_return TRY_AWAIT(worker.store.queryPartialDerivationOutputMap(drvPath, drvStore)
);
}
}
assert(false);
}
} catch (...) {
@@ -1620,9 +1632,11 @@ try {
res.insert_or_assign(name, *output.second);
co_return res;
} else {
for (auto * drvStore : { &worker.evalStore, &worker.store })
if (drvStore->isValidPath(drvPath))
co_return worker.store.queryDerivationOutputMap(drvPath, drvStore);
for (auto * drvStore : {&worker.evalStore, &worker.store}) {
if (drvStore->isValidPath(drvPath)) {
co_return TRY_AWAIT(worker.store.queryDerivationOutputMap(drvPath, drvStore));
}
}
assert(false);
}
} catch (...) {
lix/libstore/build/local-derivation-goal.cc
+13 -6
View File
@@ -483,7 +483,13 @@ try {
/* Create a temporary directory where the build will take
place. */
tmpDir = createTempDir(settings.buildDir.get().value_or(""), "nix-build-" + std::string(drvPath.name()), false, false, 0700);
tmpDir = createTempDir(
settings.buildDir.get().value_or(""),
"nix-build-" + std::string(drvPath.name()),
false,
false,
0700
);
chownToBuilder(tmpDir);
@@ -1062,13 +1068,14 @@ struct RestrictedStore : public virtual IndirectRootStore, public virtual GcStor
void queryReferrers(const StorePath & path, StorePathSet & referrers) override
{ }
std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap(
const StorePath & path,
Store * evalStore = nullptr) override
{
kj::Promise<Result<std::map<std::string, std::optional<StorePath>>>>
queryPartialDerivationOutputMap(const StorePath & path, Store * evalStore = nullptr) override
try {
if (!goal.isAllowed(path))
throw InvalidPath("cannot query output map for unknown path '%s' in recursive Nix", printStorePath(path));
return next->queryPartialDerivationOutputMap(path, evalStore);
co_return TRY_AWAIT(next->queryPartialDerivationOutputMap(path, evalStore));
} catch (...) {
co_return result::current_exception();
}
std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override
lix/libstore/daemon.cc
+1 -1
View File
@@ -386,7 +386,7 @@ static void performOp(AsyncIoRoot & aio, TunnelLogger * logger, ref<Store> store
case WorkerProto::Op::QueryDerivationOutputMap: {
auto path = store->parseStorePath(readString(from));
logger->startWork();
auto outputs = store->queryPartialDerivationOutputMap(path);
auto outputs = aio.blockOn(store->queryPartialDerivationOutputMap(path));
logger->stopWork();
to << WorkerProto::write(*store, wconn, outputs);
break;
lix/libstore/derivations.cc
+19 -8
View File
@@ -2,6 +2,7 @@
#include "lix/libstore/downstream-placeholder.hh"
#include "lix/libstore/store-api.hh"
#include "lix/libstore/globals.hh"
#include "lix/libutil/result.hh"
#include "lix/libutil/types.hh"
#include "lix/libstore/common-protocol.hh"
#include "lix/libstore/common-protocol-impl.hh"
@@ -1059,19 +1060,29 @@ Derivation::tryResolve(Store & store, Store * evalStore) const
try {
std::map<std::pair<StorePath, std::string>, StorePath> inputDrvOutputs;
std::function<void(const StorePath &, const DerivedPathMap<StringSet>::ChildNode &)> accum;
accum = [&](auto & inputDrv, auto & node) {
for (auto & [outputName, outputPath] : store.queryPartialDerivationOutputMap(inputDrv, evalStore)) {
if (outputPath) {
inputDrvOutputs.insert_or_assign({inputDrv, outputName}, *outputPath);
if (auto p = get(node.childMap, outputName))
accum(*outputPath, *p);
std::function<
kj::Promise<Result<void>>(const StorePath &, const DerivedPathMap<StringSet>::ChildNode &)>
accum;
// NOLINTNEXTLINE(cppcoreguidelines-avoid-capturing-lambda-coroutines)
accum = [&](auto & inputDrv, auto & node) -> kj::Promise<Result<void>> {
try {
for (auto & [outputName, outputPath] :
TRY_AWAIT(store.queryPartialDerivationOutputMap(inputDrv, evalStore)))
{
if (outputPath) {
inputDrvOutputs.insert_or_assign({inputDrv, outputName}, *outputPath);
if (auto p = get(node.childMap, outputName))
TRY_AWAIT(accum(*outputPath, *p));
}
}
co_return result::success();
} catch (...) {
co_return result::current_exception();
}
};
for (auto & [inputDrv, node] : inputDrvs.map)
accum(inputDrv, node);
TRY_AWAIT(accum(inputDrv, node));
co_return TRY_AWAIT(tryResolve(store, inputDrvOutputs));
} catch (...) {
lix/libstore/derived-path.cc
+6 -4
View File
@@ -42,8 +42,9 @@ try {
// Fallback for the input-addressed derivation case: We expect to always be
// able to print the output paths, so let’s do it
// FIXME try-resolve on drvPath
const auto outputMap =
store.queryPartialDerivationOutputMap(TRY_AWAIT(resolveDerivedPath(store, *drvPath)));
const auto outputMap = TRY_AWAIT(
store.queryPartialDerivationOutputMap(TRY_AWAIT(resolveDerivedPath(store, *drvPath)))
);
res["output"] = output;
auto outputPathIter = outputMap.find(output);
if (outputPathIter == outputMap.end())
@@ -64,8 +65,9 @@ try {
// Fallback for the input-addressed derivation case: We expect to always be
// able to print the output paths, so let’s do it
// FIXME try-resolve on drvPath
const auto outputMap =
store.queryPartialDerivationOutputMap(TRY_AWAIT(resolveDerivedPath(store, *drvPath)));
const auto outputMap = TRY_AWAIT(
store.queryPartialDerivationOutputMap(TRY_AWAIT(resolveDerivedPath(store, *drvPath)))
);
for (const auto & [output, outputPathOpt] : outputMap) {
if (!outputs.contains(output)) continue;
if (outputPathOpt)
lix/libstore/gc.cc
+4 -1
View File
@@ -767,11 +767,14 @@ try {
/* If keep-derivations is set and this is a
derivation, then visit the derivation outputs. */
if (gcKeepDerivations && path->isDerivation()) {
for (auto & [name, maybeOutPath] : queryPartialDerivationOutputMap(*path))
for (auto & [name, maybeOutPath] :
TRY_AWAIT(queryPartialDerivationOutputMap(*path)))
{
if (maybeOutPath &&
isValidPath(*maybeOutPath) &&
queryPathInfo(*maybeOutPath)->deriver == *path)
enqueue(*maybeOutPath);
}
}
/* If keep-outputs is set, then visit the derivers. */
lix/libstore/misc.cc
+9 -6
View File
@@ -36,7 +36,7 @@ try {
res.insert(i);
if (includeDerivers && path.isDerivation())
for (auto& [_, maybeOutPath] : queryPartialDerivationOutputMap(path))
for (auto& [_, maybeOutPath] : TRY_AWAIT(queryPartialDerivationOutputMap(path)))
if (maybeOutPath && isValidPath(*maybeOutPath))
res.insert(*maybeOutPath);
co_return res;
@@ -55,7 +55,7 @@ try {
res.insert(ref);
if (includeOutputs && path.isDerivation())
for (auto& [_, maybeOutPath] : queryPartialDerivationOutputMap(path))
for (auto& [_, maybeOutPath] : TRY_AWAIT(queryPartialDerivationOutputMap(path)))
if (maybeOutPath && isValidPath(*maybeOutPath))
res.insert(*maybeOutPath);
@@ -247,7 +247,9 @@ struct QueryMissingContext
/* true for regular derivations, and CA derivations for which we
have a trust mapping for all wanted outputs. */
auto knownOutputPaths = true;
for (auto & [outputName, pathOpt] : store.queryPartialDerivationOutputMap(drvPath)) {
for (auto & [outputName, pathOpt] :
aio.blockOn(store.queryPartialDerivationOutputMap(drvPath)))
{
if (!pathOpt) {
knownOutputPaths = false;
break;
@@ -463,7 +465,7 @@ resolveDerivedPath(Store & store, const DerivedPath::Built & bfd, Store * evalSt
try {
auto drvPath = TRY_AWAIT(resolveDerivedPath(store, *bfd.drvPath, evalStore_));
auto outputsOpt_ = store.queryPartialDerivationOutputMap(drvPath, evalStore_);
auto outputsOpt_ = TRY_AWAIT(store.queryPartialDerivationOutputMap(drvPath, evalStore_));
auto outputsOpt = std::visit(overloaded {
[&](const OutputsSpec::All &) {
@@ -511,7 +513,8 @@ try {
[&](const SingleDerivedPath::Built & bfd) -> kj::Promise<Result<StorePath>> {
try {
auto drvPath = TRY_AWAIT(resolveDerivedPath(store, *bfd.drvPath, evalStore_));
auto outputPaths = evalStore.queryPartialDerivationOutputMap(drvPath, evalStore_);
auto outputPaths =
TRY_AWAIT(evalStore.queryPartialDerivationOutputMap(drvPath, evalStore_));
if (outputPaths.count(bfd.output) == 0)
throw Error("derivation '%s' does not have an output named '%s'",
store.printStorePath(drvPath), bfd.output);
@@ -534,7 +537,7 @@ kj::Promise<Result<OutputPathMap>>
resolveDerivedPath(Store & store, const DerivedPath::Built & bfd)
try {
auto drvPath = TRY_AWAIT(resolveDerivedPath(store, *bfd.drvPath));
auto outputMap = store.queryDerivationOutputMap(drvPath);
auto outputMap = TRY_AWAIT(store.queryDerivationOutputMap(drvPath));
auto outputsLeft = std::visit(overloaded {
[&](const OutputsSpec::All &) {
return StringSet {};
lix/libstore/remote-store.cc
+13 -6
View File
@@ -334,26 +334,31 @@ try {
}
std::map<std::string, std::optional<StorePath>> RemoteStore::queryPartialDerivationOutputMap(const StorePath & path, Store * evalStore_)
{
kj ::Promise<Result<std::map<std::string, std::optional<StorePath>>>>
RemoteStore::queryPartialDerivationOutputMap(const StorePath & path, Store * evalStore_)
try {
if (GET_PROTOCOL_MINOR(getProtocol()) >= 22) {
if (!evalStore_) {
auto conn(getConnection());
conn->to << WorkerProto::Op::QueryDerivationOutputMap << printStorePath(path);
conn.processStderr();
return WorkerProto::Serialise<std::map<std::string, std::optional<StorePath>>>::read(*this, *conn);
co_return WorkerProto::Serialise<std::map<std::string, std::optional<StorePath>>>::read(
*this, *conn
);
} else {
auto & evalStore = *evalStore_;
auto outputs = evalStore.queryStaticPartialDerivationOutputMap(path);
// union with the first branch overriding the statically-known ones
// when non-`std::nullopt`.
for (auto && [outputName, optPath] : queryPartialDerivationOutputMap(path, nullptr)) {
for (auto && [outputName, optPath] :
TRY_AWAIT(queryPartialDerivationOutputMap(path, nullptr)))
{
if (optPath)
outputs.insert_or_assign(std::move(outputName), std::move(optPath));
else
outputs.insert({std::move(outputName), std::nullopt});
}
return outputs;
co_return outputs;
}
} else {
REMOVE_AFTER_DROPPING_PROTO_MINOR(21);
@@ -364,8 +369,10 @@ std::map<std::string, std::optional<StorePath>> RemoteStore::queryPartialDerivat
// from the derivation itself (and not the ones that are known because
// the have been built), but as old stores don't handle floating-CA
// derivations this shouldn't matter
return evalStore.queryStaticPartialDerivationOutputMap(path);
co_return evalStore.queryStaticPartialDerivationOutputMap(path);
}
} catch (...) {
co_return result::current_exception();
}
std::optional<StorePath> RemoteStore::queryPathFromHashPart(const std::string & hashPart)
lix/libstore/remote-store.hh
+2 -1
View File
@@ -64,7 +64,8 @@ public:
kj::Promise<Result<StorePathSet>> queryDerivationOutputs(const StorePath & path) override;
std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap(const StorePath & path, Store * evalStore = nullptr) override;
kj::Promise<Result<std::map<std::string, std::optional<StorePath>>>>
queryPartialDerivationOutputMap(const StorePath & path, Store * evalStore = nullptr) override;
std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override;
kj::Promise<Result<StorePathSet>> querySubstitutablePaths(const StorePathSet & paths) override;
lix/libstore/store-api.cc
+15 -10
View File
@@ -566,16 +566,15 @@ std::map<std::string, std::optional<StorePath>> Store::queryStaticPartialDerivat
return outputs;
}
std::map<std::string, std::optional<StorePath>> Store::queryPartialDerivationOutputMap(
const StorePath & path,
Store * evalStore_)
{
kj::Promise<Result<std::map<std::string, std::optional<StorePath>>>>
Store::queryPartialDerivationOutputMap(const StorePath & path, Store * evalStore_)
try {
auto & evalStore = evalStore_ ? *evalStore_ : *this;
auto outputs = evalStore.queryStaticPartialDerivationOutputMap(path);
if (!experimentalFeatureSettings.isEnabled(Xp::CaDerivations))
return outputs;
co_return outputs;
auto drv = evalStore.readInvalidDerivation(path);
auto drvHashes = staticOutputHashes(*this, drv);
@@ -591,23 +590,29 @@ std::map<std::string, std::optional<StorePath>> Store::queryPartialDerivationOut
}
}
return outputs;
co_return outputs;
} catch (...) {
co_return result::current_exception();
}
OutputPathMap Store::queryDerivationOutputMap(const StorePath & path, Store * evalStore) {
auto resp = queryPartialDerivationOutputMap(path, evalStore);
kj::Promise<Result<OutputPathMap>>
Store::queryDerivationOutputMap(const StorePath & path, Store * evalStore)
try {
auto resp = TRY_AWAIT(queryPartialDerivationOutputMap(path, evalStore));
OutputPathMap result;
for (auto & [outName, optOutPath] : resp) {
if (!optOutPath)
throw MissingRealisation(printStorePath(path), outName);
result.insert_or_assign(outName, *optOutPath);
}
return result;
co_return result;
} catch (...) {
co_return result::current_exception();
}
kj::Promise<Result<StorePathSet>> Store::queryDerivationOutputs(const StorePath & path)
try {
auto outputMap = this->queryDerivationOutputMap(path);
auto outputMap = TRY_AWAIT(this->queryDerivationOutputMap(path));
StorePathSet outputPaths;
for (auto & i: outputMap) {
outputPaths.emplace(std::move(i.second));
lix/libstore/store-api.hh
+4 -4
View File
@@ -451,9 +451,8 @@ public:
* derivation. All outputs are mentioned so ones mising the mapping
* are mapped to `std::nullopt`.
*/
virtual std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap(
const StorePath & path,
Store * evalStore = nullptr);
virtual kj::Promise<Result<std::map<std::string, std::optional<StorePath>>>>
queryPartialDerivationOutputMap(const StorePath & path, Store * evalStore = nullptr);
/**
* Like `queryPartialDerivationOutputMap` but only considers
@@ -470,7 +469,8 @@ public:
* Query the mapping outputName=>outputPath for the given derivation.
* Assume every output has a mapping and throw an exception otherwise.
*/
OutputPathMap queryDerivationOutputMap(const StorePath & path, Store * evalStore = nullptr);
kj::Promise<Result<OutputPathMap>>
queryDerivationOutputMap(const StorePath & path, Store * evalStore = nullptr);
/**
* Query the full store path given the hash part of a valid store
lix/nix/develop.cc
+2 -1
View File
@@ -266,7 +266,8 @@ try {
}},
bmNormal, evalStore));
for (auto & [_0, optPath] : evalStore->queryPartialDerivationOutputMap(shellDrvPath)) {
for (auto & [_0, optPath] : TRY_AWAIT(evalStore->queryPartialDerivationOutputMap(shellDrvPath)))
{
assert(optPath);
auto & outPath = *optPath;
assert(store->isValidPath(outPath));