guiwf/lix
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

libstore: deoptionalize queryPartialDerivationOutputMap

Browse Source 
derivation outpaths are now statically known at all times. the one snag
here is that the wires encode even statically known paths as optionals,
forcing us to check for this any time we receive an output map. remotes
answering with nullopt paths for derivations we still support now would
be a protocol error on its own though, so we do not diagnose it deeply.

Change-Id: Ib7080b2a0c45c3506233e87c8ef6842576f61050
This commit is contained in:
eldritch horrors
2025-05-20 17:43:46 +00:00
parent ca7f6ff96b
commit 6b5f82e78b
12 changed files with 51 additions and 81 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lix/legacy/nix-build.cc
+3 -5
View File
@@ -402,7 +402,7 @@ static void main_nix_build(AsyncIoRoot & aio, std::string programName, Strings a
if (shellDrv) {
auto shellDrvOutputs =
aio.blockOn(store->queryPartialDerivationOutputMap(shellDrv.value(), &*evalStore));
shell = store->printStorePath(shellDrvOutputs.at("out").value()) + "/bin/bash";
shell = store->printStorePath(shellDrvOutputs.at("out")) + "/bin/bash";
}
// Set the environment.
@@ -448,7 +448,7 @@ static void main_nix_build(AsyncIoRoot & aio, std::string programName, Strings a
aio.blockOn(store->queryPartialDerivationOutputMap(inputDrv, &*evalStore));
for (auto & i : inputNode) {
auto o = outputs.at(i);
aio.blockOn(store->computeFSClosure(*o, inputs));
aio.blockOn(store->computeFSClosure(o, inputs));
}
};
@@ -593,9 +593,7 @@ static void main_nix_build(AsyncIoRoot & aio, std::string programName, Strings a
auto builtOutputs =
aio.blockOn(store->queryPartialDerivationOutputMap(drvPath, &*evalStore));
auto maybeOutputPath = builtOutputs.at(outputName);
assert(maybeOutputPath);
auto outputPath = *maybeOutputPath;
auto outputPath = builtOutputs.at(outputName);
if (auto store2 = store.try_cast_shared<LocalFSStore>()) {
std::string symlink = drvPrefix;
lix/libstore/build/derivation-goal.cc
+1 -2
View File
@@ -440,8 +440,7 @@ try {
auto depOutputs =
TRY_AWAIT(worker.store.queryPartialDerivationOutputMap(i, &worker.evalStore));
for (auto & j : depOutputs)
if (j.second)
outputsToDrv.insert_or_assign(*j.second, i);
outputsToDrv.insert_or_assign(j.second, i);
}
/* Check each path (slow!). */
lix/libstore/derived-path.cc
+1 -4
View File
@@ -43,10 +43,7 @@ try {
const auto outputMap = TRY_AWAIT(store.queryPartialDerivationOutputMap(drvPath.path));
for (const auto & [output, outputPathOpt] : outputMap) {
if (!outputs.contains(output)) continue;
if (outputPathOpt)
res["outputs"][output] = store.printStorePath(*outputPathOpt);
else
res["outputs"][output] = nullptr;
res["outputs"][output] = store.printStorePath(outputPathOpt);
}
co_return res;
} catch (...) {
lix/libstore/gc.cc
+4 -5
View File
@@ -774,13 +774,12 @@ try {
/* If keep-derivations is set and this is a
derivation, then visit the derivation outputs. */
if (gcKeepDerivations && path->isDerivation()) {
for (auto & [name, maybeOutPath] :
for (auto & [name, outPath] :
TRY_AWAIT(queryPartialDerivationOutputMap(*path)))
{
if (maybeOutPath &&
TRY_AWAIT(isValidPath(*maybeOutPath)) &&
TRY_AWAIT(queryPathInfo(*maybeOutPath))->deriver == *path)
enqueue(*maybeOutPath);
if (TRY_AWAIT(isValidPath(outPath)) &&
TRY_AWAIT(queryPathInfo(outPath))->deriver == *path)
enqueue(outPath);
}
}
lix/libstore/local-store.cc
+3 -3
View File
@@ -1026,15 +1026,15 @@ try {
}
kj::Promise<Result<std::map<std::string, std::optional<StorePath>>>>
kj::Promise<Result<std::map<std::string, StorePath>>>
LocalStore::queryStaticPartialDerivationOutputMap(const StorePath & path)
try {
co_return TRY_AWAIT(
// NOLINTNEXTLINE(cppcoreguidelines-avoid-capturing-lambda-coroutines)
retrySQLite([&]() -> kj::Promise<Result<std::map<std::string, std::optional<StorePath>>>> {
retrySQLite([&]() -> kj::Promise<Result<std::map<std::string, StorePath>>> {
try {
auto state = co_await _dbState.lock();
std::map<std::string, std::optional<StorePath>> outputs;
std::map<std::string, StorePath> outputs;
uint64_t drvId;
drvId = queryValidPathId(*state, path);
auto use(state->stmts->QueryDerivationOutputs.use()(drvId));
lix/libstore/local-store.hh
+1 -1
View File
@@ -201,7 +201,7 @@ public:
kj::Promise<Result<StorePathSet>> queryValidDerivers(const StorePath & path) override;
kj::Promise<Result<std::map<std::string, std::optional<StorePath>>>>
kj::Promise<Result<std::map<std::string, StorePath>>>
queryStaticPartialDerivationOutputMap(const StorePath & path) override;
kj::Promise<Result<std::optional<StorePath>>>
lix/libstore/misc.cc
+16 -36
View File
@@ -37,9 +37,9 @@ try {
res.insert(i);
if (includeDerivers && path.isDerivation())
for (auto& [_, maybeOutPath] : TRY_AWAIT(queryPartialDerivationOutputMap(path)))
if (maybeOutPath && TRY_AWAIT(isValidPath(*maybeOutPath)))
res.insert(*maybeOutPath);
for (auto& [_, outPath] : TRY_AWAIT(queryPartialDerivationOutputMap(path)))
if (TRY_AWAIT(isValidPath(outPath)))
res.insert(outPath);
co_return res;
} catch (...) {
co_return result::current_exception();
@@ -56,9 +56,9 @@ try {
res.insert(ref);
if (includeOutputs && path.isDerivation())
for (auto& [_, maybeOutPath] : TRY_AWAIT(queryPartialDerivationOutputMap(path)))
if (maybeOutPath && TRY_AWAIT(isValidPath(*maybeOutPath)))
res.insert(*maybeOutPath);
for (auto& [_, outPath] : TRY_AWAIT(queryPartialDerivationOutputMap(path)))
if (TRY_AWAIT(isValidPath(outPath)))
res.insert(outPath);
if (includeDerivers && info->deriver && TRY_AWAIT(isValidPath(*info->deriver)))
res.insert(*info->deriver);
@@ -240,29 +240,18 @@ struct QueryMissingContext
}
StorePathSet invalid;
/* true for regular derivations, and CA derivations for which we
have a trust mapping for all wanted outputs. */
auto knownOutputPaths = true;
for (auto & [outputName, pathOpt] :
for (auto & [outputName, path] :
aio.blockOn(store.queryPartialDerivationOutputMap(drvPath)))
{
if (!pathOpt) {
knownOutputPaths = false;
break;
}
if (bfd.outputs.contains(outputName) && !aio.blockOn(store.isValidPath(*pathOpt)))
invalid.insert(*pathOpt);
if (bfd.outputs.contains(outputName) && !aio.blockOn(store.isValidPath(path)))
invalid.insert(path);
}
if (knownOutputPaths && invalid.empty()) return;
if (invalid.empty()) return;
auto drv = make_ref<Derivation>(aio.blockOn(store.derivationFromPath(drvPath)));
ParsedDerivation parsedDrv(StorePath(drvPath), *drv);
if (!knownOutputPaths && settings.useSubstitutes && parsedDrv.substitutesAllowed()) {
throw UnimplementedError("ca derivations are not supported");
}
if (knownOutputPaths && settings.useSubstitutes && parsedDrv.substitutesAllowed()) {
if (settings.useSubstitutes && parsedDrv.substitutesAllowed()) {
auto drvState = make_ref<Sync<DrvState>>(DrvState(invalid.size()));
for (auto & output : invalid) {
pool.enqueueWithAio([=, this](AsyncIoRoot & aio) {
@@ -365,18 +354,18 @@ resolveDerivedPath(Store & store, const DerivedPath::Built & bfd, Store * evalSt
try {
auto drvPath = bfd.drvPath.path;
auto outputsOpt_ = TRY_AWAIT(store.queryPartialDerivationOutputMap(drvPath, evalStore_));
auto outputs_ = TRY_AWAIT(store.queryPartialDerivationOutputMap(drvPath, evalStore_));
auto outputsOpt = std::visit(overloaded {
co_return std::visit(overloaded {
[&](const OutputsSpec::All &) {
// Keep all outputs
return std::move(outputsOpt_);
return std::move(outputs_);
},
[&](const OutputsSpec::Names & names) {
// Get just those mentioned by name
std::map<std::string, std::optional<StorePath>> outputsOpt;
std::map<std::string, StorePath> outputsOpt;
for (auto & output : names) {
auto * pOutputPathOpt = get(outputsOpt_, output);
auto * pOutputPathOpt = get(outputs_, output);
if (!pOutputPathOpt)
throw Error(
"the derivation '%s' doesn't have an output named '%s'",
@@ -386,15 +375,6 @@ try {
return outputsOpt;
},
}, bfd.outputs.raw);
OutputPathMap outputs;
for (auto & [outputName, outputPathOpt] : outputsOpt) {
if (!outputPathOpt)
throw MissingRealisation(bfd.drvPath.to_string(store), outputName);
auto & outputPath = *outputPathOpt;
outputs.insert_or_assign(outputName, outputPath);
}
co_return outputs;
} catch (...) {
co_return result::current_exception();
}
lix/libstore/remote-store.cc
+13 -6
View File
@@ -338,7 +338,7 @@ try {
}
kj ::Promise<Result<std::map<std::string, std::optional<StorePath>>>>
kj ::Promise<Result<std::map<std::string, StorePath>>>
RemoteStore::queryPartialDerivationOutputMap(const StorePath & path, Store * evalStore_)
try {
if (GET_PROTOCOL_MINOR(TRY_AWAIT(getProtocol())) >= 22) {
@@ -346,9 +346,19 @@ try {
auto conn(TRY_AWAIT(getConnection()));
conn->to << WorkerProto::Op::QueryDerivationOutputMap << printStorePath(path);
conn.processStderr();
co_return WorkerProto::Serialise<std::map<std::string, std::optional<StorePath>>>::read(
auto tmp = WorkerProto::Serialise<std::map<std::string, std::optional<StorePath>>>::read(
*this, *conn
);
std::map<std::string, StorePath> result;
for (auto & [name, outPath] : tmp) {
if (!outPath) {
throw Error(
"remote responded with unknown outpath for %s^%s", path.to_string(), name
);
}
result.emplace(std::move(name), std::move(*outPath));
}
co_return result;
} else {
auto & evalStore = *evalStore_;
auto outputs = TRY_AWAIT(evalStore.queryStaticPartialDerivationOutputMap(path));
@@ -357,10 +367,7 @@ try {
for (auto && [outputName, optPath] :
TRY_AWAIT(queryPartialDerivationOutputMap(path, nullptr)))
{
if (optPath)
outputs.insert_or_assign(std::move(outputName), std::move(optPath));
else
outputs.insert({std::move(outputName), std::nullopt});
outputs.insert_or_assign(std::move(outputName), std::move(optPath));
}
co_return outputs;
}
lix/libstore/remote-store.hh
+1 -1
View File
@@ -69,7 +69,7 @@ public:
kj::Promise<Result<StorePathSet>> queryDerivationOutputs(const StorePath & path) override;
kj::Promise<Result<std::map<std::string, std::optional<StorePath>>>>
kj::Promise<Result<std::map<std::string, StorePath>>>
queryPartialDerivationOutputMap(const StorePath & path, Store * evalStore = nullptr) override;
kj::Promise<Result<std::optional<StorePath>>>
queryPathFromHashPart(const std::string & hashPart) override;
lix/libstore/store-api.cc
+4 -11
View File
@@ -508,10 +508,10 @@ bool Store::PathInfoCacheValue::isKnownNow()
return std::chrono::steady_clock::now() < time_point + ttl;
}
kj::Promise<Result<std::map<std::string, std::optional<StorePath>>>>
kj::Promise<Result<std::map<std::string, StorePath>>>
Store::queryStaticPartialDerivationOutputMap(const StorePath & path)
try {
std::map<std::string, std::optional<StorePath>> outputs;
std::map<std::string, StorePath> outputs;
auto drv = TRY_AWAIT(readInvalidDerivation(path));
for (auto & [outputName, output] : drv.outputsAndPaths(*this)) {
outputs.emplace(outputName, output.second);
@@ -521,7 +521,7 @@ try {
co_return result::current_exception();
}
kj::Promise<Result<std::map<std::string, std::optional<StorePath>>>>
kj::Promise<Result<std::map<std::string, StorePath>>>
Store::queryPartialDerivationOutputMap(const StorePath & path, Store * evalStore_)
try {
auto & evalStore = evalStore_ ? *evalStore_ : *this;
@@ -534,14 +534,7 @@ try {
kj::Promise<Result<OutputPathMap>>
Store::queryDerivationOutputMap(const StorePath & path, Store * evalStore)
try {
auto resp = TRY_AWAIT(queryPartialDerivationOutputMap(path, evalStore));
OutputPathMap result;
for (auto & [outName, optOutPath] : resp) {
if (!optOutPath)
throw MissingRealisation(printStorePath(path), outName);
result.insert_or_assign(outName, *optOutPath);
}
co_return result;
co_return TRY_AWAIT(queryPartialDerivationOutputMap(path, evalStore));
} catch (...) {
co_return result::current_exception();
}
lix/libstore/store-api.hh
+3 -4
View File
@@ -448,10 +448,9 @@ public:
/**
* Query the mapping outputName => outputPath for the given
* derivation. All outputs are mentioned so ones mising the mapping
* are mapped to `std::nullopt`.
* derivation. Naming is a historical accident.
*/
virtual kj::Promise<Result<std::map<std::string, std::optional<StorePath>>>>
virtual kj::Promise<Result<std::map<std::string, StorePath>>>
queryPartialDerivationOutputMap(const StorePath & path, Store * evalStore = nullptr);
/**
@@ -462,7 +461,7 @@ public:
* Just a helper function for implementing
* `queryPartialDerivationOutputMap`.
*/
virtual kj::Promise<Result<std::map<std::string, std::optional<StorePath>>>>
virtual kj::Promise<Result<std::map<std::string, StorePath>>>
queryStaticPartialDerivationOutputMap(const StorePath & path);
/**
lix/nix/develop.cc
+1 -3
View File
@@ -260,10 +260,8 @@ try {
}},
bmNormal, evalStore));
for (auto & [_0, optPath] : TRY_AWAIT(evalStore->queryPartialDerivationOutputMap(shellDrvPath)))
for (auto & [_0, outPath] : TRY_AWAIT(evalStore->queryPartialDerivationOutputMap(shellDrvPath)))
{
assert(optPath);
auto & outPath = *optPath;
assert(TRY_AWAIT(store->isValidPath(outPath)));
auto outPathS = store->toRealPath(outPath);
if (lstat(outPathS).st_size)