guiwf/lix
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

libstore: asyncify Store worker entry points

Browse Source 
Change-Id: Iafa0c4093064095d196c738cc7e9ba1d8e906b8c
This commit is contained in:
eldritch horrors
2025-01-25 13:01:21 +00:00
parent 5a41803f74
commit 0ad79775b6
29 changed files with 226 additions and 121 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lix/legacy/build-remote.cc
+3 -3
View File
@@ -325,18 +325,18 @@ connected:
// output ids, which break CA derivations
if (!drv.inputDrvs.map.empty())
drv.inputSrcs = store->parseStorePathSet(inputs);
optResult = sshStore->buildDerivation(*drvPath, (const BasicDerivation &) drv);
optResult = aio.blockOn(sshStore->buildDerivation(*drvPath, (const BasicDerivation &) drv));
auto & result = *optResult;
if (!result.success())
throw Error("build of '%s' on '%s' failed: %s", store->printStorePath(*drvPath), storeUri, result.errorMsg);
} else {
copyClosure(*store, *sshStore, StorePathSet {*drvPath}, NoRepair, NoCheckSigs, substitute);
auto res = sshStore->buildPathsWithResults({
auto res = aio.blockOn(sshStore->buildPathsWithResults({
DerivedPath::Built {
.drvPath = makeConstantStorePathRef(*drvPath),
.outputs = OutputsSpec::All {},
}
});
}));
// One path to build should produce exactly one build result
assert(res.size() == 1);
optResult = std::move(res[0]);
lix/legacy/nix-build.cc
+1 -1
View File
@@ -319,7 +319,7 @@ static void main_nix_build(AsyncIoRoot & aio, std::string programName, Strings a
printMissing(ref<Store>(store), willBuild, willSubstitute, unknown, downloadSize, narSize);
if (!dryRun)
store->buildPaths(paths, buildMode, evalStore);
aio.blockOn(store->buildPaths(paths, buildMode, evalStore));
};
if (runEnv) {
lix/legacy/nix-env.cc
+3 -1
View File
@@ -790,7 +790,9 @@ static void opSet(Globals & globals, Strings opFlags, Strings opArgs)
};
printMissing(globals.state->store, paths);
if (globals.dryRun) return;
globals.state->store->buildPaths(paths, globals.state->repair ? bmRepair : bmNormal);
globals.aio.blockOn(
globals.state->store->buildPaths(paths, globals.state->repair ? bmRepair : bmNormal)
);
debug("switching to new user environment");
Path generation = createGeneration(
lix/legacy/nix-store.cc
+7 -6
View File
@@ -1,6 +1,7 @@
#include "lix/libutil/archive.hh"
#include "lix/libstore/derivations.hh"
#include "dotgraph.hh"
#include "lix/libutil/async.hh"
#include "lix/libutil/exit.hh"
#include "lix/libstore/globals.hh"
#include "lix/libstore/build-result.hh"
@@ -66,7 +67,7 @@ try {
auto store2 = std::dynamic_pointer_cast<LocalFSStore>(store);
if (path.path.isDerivation()) {
if (build) store->buildPaths({path.toDerivedPath()});
if (build) TRY_AWAIT(store->buildPaths({path.toDerivedPath()}));
auto outputPaths = store->queryDerivationOutputMap(path.path);
Derivation drv = store->derivationFromPath(path.path);
rootNr++;
@@ -100,7 +101,7 @@ try {
}
else {
if (build) store->ensurePath(path.path);
if (build) TRY_AWAIT(store->ensurePath(path.path));
else if (!store->isValidPath(path.path))
throw Error("path '%s' does not exist and cannot be created", store->printStorePath(path.path));
if (store2) {
@@ -159,7 +160,7 @@ static void opRealise(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
if (dryRun) return;
/* Build all paths at the same time to exploit parallelism. */
store->buildPaths(toDerivedPaths(paths), buildMode);
aio.blockOn(store->buildPaths(toDerivedPaths(paths), buildMode));
if (!ignoreUnknown)
for (auto & i : paths) {
@@ -790,7 +791,7 @@ static void opRepairPath(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
throw UsageError("no flags expected");
for (auto & i : opArgs)
store->repairPath(store->followLinksToStorePath(i));
aio.blockOn(store->repairPath(store->followLinksToStorePath(i)));
}
/* Optimise the disk space usage of the Nix store by hard-linking
@@ -932,7 +933,7 @@ static void opServe(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
try {
MonitorFdHup monitor(in.fd);
store->buildPaths(toDerivedPaths(paths));
aio.blockOn(store->buildPaths(toDerivedPaths(paths)));
out << 0;
} catch (Error & e) {
assert(e.info().status);
@@ -952,7 +953,7 @@ static void opServe(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
getBuildSettings();
MonitorFdHup monitor(in.fd);
auto status = store->buildDerivation(drvPath, drv);
auto status = aio.blockOn(store->buildDerivation(drvPath, drv));
out << ServeProto::write(*store, wconn, status);
break;
lix/legacy/user-env.cc
+5 -5
View File
@@ -28,9 +28,9 @@ bool createUserEnv(EvalState & state, DrvInfos & elems,
drvsToBuild.push_back({*drvPath});
debug("building user environment dependencies");
state.ctx.store->buildPaths(
state.aio.blockOn(state.ctx.store->buildPaths(
toDerivedPaths(drvsToBuild),
state.ctx.repair ? bmRepair : bmNormal);
state.ctx.repair ? bmRepair : bmNormal));
/* Construct the whole top level derivation. */
StorePathSet references;
@@ -67,7 +67,7 @@ bool createUserEnv(EvalState & state, DrvInfos & elems,
/* This is only necessary when installing store paths, e.g.,
`nix-env -i /nix/store/abcd...-foo'. */
state.ctx.store->addTempRoot(*j.second);
state.ctx.store->ensurePath(*j.second);
state.aio.blockOn(state.ctx.store->ensurePath(*j.second));
references.insert(*j.second);
}
@@ -125,9 +125,9 @@ bool createUserEnv(EvalState & state, DrvInfos & elems,
debug("building user environment");
std::vector<StorePathWithOutputs> topLevelDrvs;
topLevelDrvs.push_back({topLevelDrv});
state.ctx.store->buildPaths(
state.aio.blockOn(state.ctx.store->buildPaths(
toDerivedPaths(topLevelDrvs),
state.ctx.repair ? bmRepair : bmNormal);
state.ctx.repair ? bmRepair : bmNormal));
/* Switch the current user environment to the output path. */
auto store2 = state.ctx.store.dynamic_pointer_cast<LocalFSStore>();
lix/libcmd/installables.cc
+2 -1
View File
@@ -652,7 +652,8 @@ std::vector<std::pair<ref<Installable>, BuiltPathWithResult>> Installable::build
if (settings.printMissing)
printMissing(store, pathsToBuild, lvlInfo);
auto buildResults = store->buildPathsWithResults(pathsToBuild, bMode, evalStore);
auto buildResults =
state.aio.blockOn(store->buildPathsWithResults(pathsToBuild, bMode, evalStore));
throwBuildErrors(buildResults, *store);
for (auto & buildResult : buildResults) {
for (auto & aux : backmap[buildResult.path]) {
lix/libcmd/repl.cc
+2 -2
View File
@@ -756,12 +756,12 @@ ProcessLineResult NixRepl::processLine(std::string line)
logger->pause();
});
evaluator.store->buildPaths({
state.aio.blockOn(evaluator.store->buildPaths({
DerivedPath::Built {
.drvPath = makeConstantStorePathRef(drvPath),
.outputs = OutputsSpec::All { },
},
});
}));
auto drv = evaluator.store->readDerivation(drvPath);
logger->cout("\nThis derivation produced the following outputs:");
for (auto & [outputName, outputPath] : evaluator.store->queryDerivationOutputMap(drvPath)) {
lix/libexpr/primops.cc
+3 -2
View File
@@ -10,6 +10,7 @@
#include "lix/libexpr/json-to-value.hh"
#include "lix/libstore/names.hh"
#include "lix/libstore/path-references.hh"
#include "lix/libutil/async.hh"
#include "lix/libutil/processes.hh"
#include "lix/libstore/store-api.hh"
#include "lix/libexpr/value-to-json.hh"
@@ -82,7 +83,7 @@ try {
/* Build/substitute the context. */
std::vector<DerivedPath> buildReqs;
for (auto & d : drvs) buildReqs.emplace_back(DerivedPath { d });
buildStore->buildPaths(buildReqs, bmNormal, store);
TRY_AWAIT(buildStore->buildPaths(buildReqs, bmNormal, store));
StorePathSet outputsToCopyAndAllow;
@@ -1184,7 +1185,7 @@ static void prim_storePath(EvalState & state, const PosIdx pos, Value * * args,
.atPos(pos).debugThrow();
auto path2 = state.ctx.store->toStorePath(path.abs()).first;
if (!settings.readOnlyMode)
state.ctx.store->ensurePath(path2);
state.aio.blockOn(state.ctx.store->ensurePath(path2));
context.insert(NixStringContextElem::Opaque { .path = path2 });
v.mkString(path.abs(), context);
}
lix/libexpr/primops/context.cc
+1 -1
View File
@@ -182,7 +182,7 @@ static void prim_appendContext(EvalState & state, const PosIdx pos, Value * * ar
).atPos(i.pos).debugThrow();
auto namePath = state.ctx.store->parseStorePath(name);
if (!settings.readOnlyMode)
state.ctx.store->ensurePath(namePath);
state.aio.blockOn(state.ctx.store->ensurePath(namePath));
state.forceAttrs(*i.value, i.pos, "while evaluating the value of a string context");
auto iter = i.value->attrs->find(state.ctx.s.path);
if (iter != i.value->attrs->end()) {
lix/libfetchers/fetchers.cc
+2 -1
View File
@@ -1,6 +1,7 @@
#include "lix/libfetchers/fetchers.hh"
#include "lix/libfetchers/builtin-fetchers.hh"
#include "lix/libstore/store-api.hh"
#include "lix/libutil/async.hh"
#include "lix/libutil/source-path.hh"
#include "lix/libfetchers/fetch-to-store.hh"
@@ -134,7 +135,7 @@ std::pair<Tree, Input> Input::fetch(ref<Store> store) const
try {
auto storePath = computeStorePath(*store);
store->ensurePath(storePath);
RUN_ASYNC_IN_NEW_THREAD(store->ensurePath(storePath));
debug("using substituted/cached input '%s' in '%s'",
to_string(), store->printStorePath(storePath));
lix/libstore/build/entry-points.cc
+54 -32
View File
@@ -3,18 +3,21 @@
#include "lix/libstore/build/derivation-goal.hh"
#include "lix/libstore/local-store.hh"
#include "lix/libutil/async.hh"
#include "lix/libutil/result.hh"
#include "lix/libutil/strings.hh"
namespace nix {
void Store::buildPaths(const std::vector<DerivedPath> & reqs, BuildMode buildMode, std::shared_ptr<Store> evalStore)
{
auto results = RUN_ASYNC_IN_NEW_THREAD(processGoals(*this, evalStore ? *evalStore : *this, [&](GoalFactory & gf) {
Worker::Targets goals;
for (auto & br : reqs)
goals.emplace_back(gf.makeGoal(br, buildMode));
return goals;
}));
kj::Promise<Result<void>> Store::buildPaths(const std::vector<DerivedPath> & reqs, BuildMode buildMode, std::shared_ptr<Store> evalStore)
try {
auto results =
TRY_AWAIT(processGoals(*this, evalStore ? *evalStore : *this, [&](GoalFactory & gf) {
Worker::Targets goals;
for (auto & br : reqs) {
goals.emplace_back(gf.makeGoal(br, buildMode));
}
return goals;
}));
StringSet failed;
std::shared_ptr<Error> ex;
@@ -38,58 +41,69 @@ void Store::buildPaths(const std::vector<DerivedPath> & reqs, BuildMode buildMod
if (ex) logError(ex->info());
throw Error(results.failingExitStatus, "build of %s failed", concatStringsSep(", ", quoteStrings(failed)));
}
co_return result::success();
} catch (...) {
co_return result::current_exception();
}
std::vector<KeyedBuildResult> Store::buildPathsWithResults(
kj::Promise<Result<std::vector<KeyedBuildResult>>> Store::buildPathsWithResults(
const std::vector<DerivedPath> & reqs,
BuildMode buildMode,
std::shared_ptr<Store> evalStore)
{
auto goals = RUN_ASYNC_IN_NEW_THREAD(processGoals(*this, evalStore ? *evalStore : *this, [&](GoalFactory & gf) {
Worker::Targets goals;
for (const auto & req : reqs) {
goals.emplace_back(gf.makeGoal(req, buildMode));
}
return goals;
})).goals;
try {
auto goals =
TRY_AWAIT(processGoals(*this, evalStore ? *evalStore : *this, [&](GoalFactory & gf) {
Worker::Targets goals;
for (const auto & req : reqs) {
goals.emplace_back(gf.makeGoal(req, buildMode));
}
return goals;
})).goals;
std::vector<KeyedBuildResult> results;
for (auto && [goalIdx, req] : enumerate(reqs))
results.emplace_back(goals[goalIdx].result.restrictTo(req));
return results;
co_return results;
} catch (...) {
co_return result::current_exception();
}
BuildResult Store::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
kj::Promise<Result<BuildResult>> Store::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
BuildMode buildMode)
{
try {
try {
auto results = RUN_ASYNC_IN_NEW_THREAD(processGoals(*this, *this, [&](GoalFactory & gf) {
auto results = TRY_AWAIT(processGoals(*this, *this, [&](GoalFactory & gf) {
Worker::Targets goals;
goals.emplace_back(gf.makeBasicDerivationGoal(drvPath, drv, OutputsSpec::All{}, buildMode));
goals.emplace_back(
gf.makeBasicDerivationGoal(drvPath, drv, OutputsSpec::All{}, buildMode)
);
return goals;
}));
auto & result = results.goals.begin()->second;
return result.result.restrictTo(DerivedPath::Built {
co_return result.result.restrictTo(DerivedPath::Built {
.drvPath = makeConstantStorePathRef(drvPath),
.outputs = OutputsSpec::All {},
});
} catch (Error & e) {
return BuildResult {
co_return BuildResult {
.status = BuildResult::MiscFailure,
.errorMsg = e.msg(),
};
};
} catch (...) {
co_return result::current_exception();
}
void Store::ensurePath(const StorePath & path)
{
kj::Promise<Result<void>> Store::ensurePath(const StorePath & path)
try {
/* If the path is already valid, we're done. */
if (isValidPath(path)) return;
if (isValidPath(path)) co_return result::success();
auto results = RUN_ASYNC_IN_NEW_THREAD(processGoals(*this, *this, [&](GoalFactory & gf) {
auto results = TRY_AWAIT(processGoals(*this, *this, [&](GoalFactory & gf) {
Worker::Targets goals;
goals.emplace_back(gf.makePathSubstitutionGoal(path));
return goals;
@@ -103,12 +117,16 @@ void Store::ensurePath(const StorePath & path)
} else
throw Error(results.failingExitStatus, "path '%s' does not exist and cannot be created", printStorePath(path));
}
co_return result::success();
} catch (...) {
co_return result::current_exception();
}
void Store::repairPath(const StorePath & path)
{
auto results = RUN_ASYNC_IN_NEW_THREAD(processGoals(*this, *this, [&](GoalFactory & gf) {
kj::Promise<Result<void>> Store::repairPath(const StorePath & path)
try {
auto results = TRY_AWAIT(processGoals(*this, *this, [&](GoalFactory & gf) {
Worker::Targets goals;
goals.emplace_back(gf.makePathSubstitutionGoal(path, Repair));
return goals;
@@ -120,7 +138,7 @@ void Store::repairPath(const StorePath & path)
deriver, then rebuild the deriver. */
auto info = queryPathInfo(path);
if (info->deriver && isValidPath(*info->deriver)) {
RUN_ASYNC_IN_NEW_THREAD(processGoals(*this, *this, [&](GoalFactory & gf) {
TRY_AWAIT(processGoals(*this, *this, [&](GoalFactory & gf) {
Worker::Targets goals;
goals.emplace_back(gf.makeGoal(
DerivedPath::Built{
@@ -135,6 +153,10 @@ void Store::repairPath(const StorePath & path)
} else
throw Error(results.failingExitStatus, "cannot repair path '%s'", printStorePath(path));
}
co_return result::success();
} catch (...) {
co_return result::current_exception();
}
}
lix/libstore/build/local-derivation-goal.cc
+25 -12
View File
@@ -1092,11 +1092,14 @@ struct RestrictedStore : public virtual IndirectRootStore, public virtual GcStor
return LocalFSStore::narFromPath(path);
}
void ensurePath(const StorePath & path) override
{
kj::Promise<Result<void>> ensurePath(const StorePath & path) override
try {
if (!goal.isAllowed(path))
throw InvalidPath("cannot substitute unknown path '%s' in recursive Nix", printStorePath(path));
/* Nothing to be done; 'path' must already be valid. */
return {result::success()};
} catch (...) {
return {result::current_exception()};
}
void registerDrvOutput(const Realisation & info) override
@@ -1113,18 +1116,25 @@ struct RestrictedStore : public virtual IndirectRootStore, public virtual GcStor
return next->queryRealisation(id);
}
void buildPaths(const std::vector<DerivedPath> & paths, BuildMode buildMode, std::shared_ptr<Store> evalStore) override
{
for (auto & result : buildPathsWithResults(paths, buildMode, evalStore))
kj::Promise<Result<void>> buildPaths(
const std::vector<DerivedPath> & paths,
BuildMode buildMode,
std::shared_ptr<Store> evalStore
) override
try {
for (auto & result : TRY_AWAIT(buildPathsWithResults(paths, buildMode, evalStore)))
if (!result.success())
result.rethrow();
co_return result::success();
} catch (...) {
co_return result::current_exception();
}
std::vector<KeyedBuildResult> buildPathsWithResults(
kj::Promise<Result<std::vector<KeyedBuildResult>>> buildPathsWithResults(
const std::vector<DerivedPath> & paths,
BuildMode buildMode = bmNormal,
std::shared_ptr<Store> evalStore = nullptr) override
{
try {
assert(!evalStore);
if (buildMode != bmNormal) throw Error("unsupported build mode");
@@ -1137,7 +1147,7 @@ struct RestrictedStore : public virtual IndirectRootStore, public virtual GcStor
throw InvalidPath("cannot build '%s' in recursive Nix because path is unknown", req.to_string(*next));
}
auto results = next->buildPathsWithResults(paths, buildMode);
auto results = TRY_AWAIT(next->buildPathsWithResults(paths, buildMode));
for (auto & result : results) {
for (auto & [outputName, output] : result.builtOutputs) {
@@ -1153,12 +1163,15 @@ struct RestrictedStore : public virtual IndirectRootStore, public virtual GcStor
for (auto & real : Realisation::closure(*next, newRealisations))
goal.addedDrvOutputs.insert(real.id);
return results;
co_return results;
} catch (...) {
co_return result::current_exception();
}
BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
BuildMode buildMode = bmNormal) override
{ unsupported("buildDerivation"); }
kj::Promise<Result<BuildResult>> buildDerivation(
const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode = bmNormal
) override
try { unsupported("buildDerivation"); } catch (...) { return {result::current_exception()}; }
void addTempRoot(const StorePath & path) override
{ }
lix/libstore/daemon.cc
+4 -4
View File
@@ -550,7 +550,7 @@ static void performOp(AsyncIoRoot & aio, TunnelLogger * logger, ref<Store> store
if (mode == bmRepair && !trusted)
throw Error("repairing is not allowed because you are not in 'trusted-users'");
logger->startWork();
store->buildPaths(drvs, mode);
aio.blockOn(store->buildPaths(drvs, mode));
logger->stopWork();
to << 1;
break;
@@ -569,7 +569,7 @@ static void performOp(AsyncIoRoot & aio, TunnelLogger * logger, ref<Store> store
throw Error("repairing is not allowed because you are not in 'trusted-users'");
logger->startWork();
auto results = store->buildPathsWithResults(drvs, mode);
auto results = aio.blockOn(store->buildPathsWithResults(drvs, mode));
logger->stopWork();
to << WorkerProto::write(*store, wconn, results);
@@ -648,7 +648,7 @@ static void performOp(AsyncIoRoot & aio, TunnelLogger * logger, ref<Store> store
drvPath = writeDerivation(*store, Derivation { drv2 });
}
auto res = store->buildDerivation(drvPath, drv, buildMode);
auto res = aio.blockOn(store->buildDerivation(drvPath, drv, buildMode));
logger->stopWork();
to << WorkerProto::write(*store, wconn, res);
break;
@@ -657,7 +657,7 @@ static void performOp(AsyncIoRoot & aio, TunnelLogger * logger, ref<Store> store
case WorkerProto::Op::EnsurePath: {
auto path = store->parseStorePath(readString(from));
logger->startWork();
store->ensurePath(path);
aio.blockOn(store->ensurePath(path));
logger->stopWork();
to << 1;
break;
lix/libstore/legacy-ssh-store.cc
+21 -10
View File
@@ -294,9 +294,10 @@ private:
public:
BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
BuildMode buildMode) override
{
kj::Promise<Result<BuildResult>> buildDerivation(
const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode
) override
try {
auto conn(connections->get());
conn->to
@@ -308,11 +309,17 @@ public:
conn->to.flush();
return ServeProto::Serialise<BuildResult>::read(*this, *conn);
return {ServeProto::Serialise<BuildResult>::read(*this, *conn)};
} catch (...) {
return {result::current_exception()};
}
void buildPaths(const std::vector<DerivedPath> & drvPaths, BuildMode buildMode, std::shared_ptr<Store> evalStore) override
{
kj ::Promise<Result<void>> buildPaths(
const std::vector<DerivedPath> & drvPaths,
BuildMode buildMode,
std::shared_ptr<Store> evalStore
) override
try {
if (evalStore && evalStore.get() != this)
throw Error("building on an SSH store is incompatible with '--eval-store'");
@@ -347,10 +354,14 @@ public:
conn->from >> result.errorMsg;
throw Error(result.status, result.errorMsg);
}
co_return result::success();
} catch (...) {
co_return result::current_exception();
}
void ensurePath(const StorePath & path) override
{ unsupported("ensurePath"); }
kj::Promise<Result<void>> ensurePath(const StorePath & path) override
try { unsupported("ensurePath"); } catch (...) { return {result::current_exception()}; }
virtual ref<FSAccessor> getFSAccessor() override
{ unsupported("getFSAccessor"); }
@@ -363,8 +374,8 @@ public:
* We make this fail for now so we can add implement this properly later
* without it being a breaking change.
*/
void repairPath(const StorePath & path) override
{ unsupported("repairPath"); }
kj::Promise<Result<void>> repairPath(const StorePath & path) override
try { unsupported("repairPath"); } catch (...) { return {result::current_exception()}; }
void computeFSClosure(const StorePathSet & paths,
StorePathSet & out, bool flipDirection = false,
lix/libstore/local-store.cc
+3 -2
View File
@@ -6,6 +6,7 @@
#include "lix/libstore/worker-protocol.hh"
#include "lix/libstore/derivations.hh"
#include "lix/libstore/nar-info.hh"
#include "lix/libutil/async.hh"
#include "lix/libutil/references.hh"
#include "lix/libutil/topo-sort.hh"
#include "lix/libutil/signals.hh"
@@ -1596,7 +1597,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
if (info->narHash != nullHash && info->narHash != current.first) {
printError("path '%s' was modified! expected hash '%s', got '%s'",
printStorePath(i), info->narHash.to_string(Base::Base32, true), current.first.to_string(Base::Base32, true));
if (repair) repairPath(i); else errors = true;
if (repair) RUN_ASYNC_IN_NEW_THREAD(repairPath(i)); else errors = true;
} else {
bool update = false;
@@ -1667,7 +1668,7 @@ void LocalStore::verifyPath(const StorePath & path, const StorePathSet & storePa
printError("path '%s' disappeared, but it still has valid referrers!", pathS);
if (repair)
try {
repairPath(path);
RUN_ASYNC_IN_NEW_THREAD(repairPath(path));
} catch (Error & e) {
logWarning(e.info());
errors = true;
lix/libstore/remote-store.cc
+28 -14
View File
@@ -1,4 +1,6 @@
#include "lix/libutil/async.hh"
#include "lix/libutil/error.hh"
#include "lix/libutil/result.hh"
#include "lix/libutil/serialise.hh"
#include "lix/libutil/signals.hh"
#include "lix/libstore/path-with-outputs.hh"
@@ -604,8 +606,10 @@ void RemoteStore::copyDrvsFromEvalStore(
}
}
void RemoteStore::buildPaths(const std::vector<DerivedPath> & drvPaths, BuildMode buildMode, std::shared_ptr<Store> evalStore)
{
kj ::Promise<Result<void>> RemoteStore::buildPaths(
const std::vector<DerivedPath> & drvPaths, BuildMode buildMode, std::shared_ptr<Store> evalStore
)
try {
copyDrvsFromEvalStore(drvPaths, evalStore);
auto conn(getConnection());
@@ -614,13 +618,16 @@ void RemoteStore::buildPaths(const std::vector<DerivedPath> & drvPaths, BuildMod
conn->to << buildMode;
conn.processStderr();
readInt(conn->from);
return {result::success()};
} catch (...) {
return {result::current_exception()};
}
std::vector<KeyedBuildResult> RemoteStore::buildPathsWithResults(
kj::Promise<Result<std::vector<KeyedBuildResult>>> RemoteStore::buildPathsWithResults(
const std::vector<DerivedPath> & paths,
BuildMode buildMode,
std::shared_ptr<Store> evalStore)
{
try {
copyDrvsFromEvalStore(paths, evalStore);
std::optional<ConnectionHandle> conn_(getConnection());
@@ -631,7 +638,7 @@ std::vector<KeyedBuildResult> RemoteStore::buildPathsWithResults(
conn->to << WorkerProto::write(*this, *conn, paths);
conn->to << buildMode;
conn.processStderr();
return WorkerProto::Serialise<std::vector<KeyedBuildResult>>::read(*this, *conn);
co_return WorkerProto::Serialise<std::vector<KeyedBuildResult>>::read(*this, *conn);
} else {
REMOVE_AFTER_DROPPING_PROTO_MINOR(33);
// Avoid deadlock.
@@ -639,7 +646,7 @@ std::vector<KeyedBuildResult> RemoteStore::buildPathsWithResults(
// Note: this throws an exception if a build/substitution
// fails, but meh.
buildPaths(paths, buildMode, evalStore);
TRY_AWAIT(buildPaths(paths, buildMode, evalStore));
std::vector<KeyedBuildResult> results;
@@ -696,29 +703,36 @@ std::vector<KeyedBuildResult> RemoteStore::buildPathsWithResults(
path.raw());
}
return results;
co_return results;
}
} catch (...) {
co_return result::current_exception();
}
BuildResult RemoteStore::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
BuildMode buildMode)
{
kj ::Promise<Result<BuildResult>> RemoteStore::buildDerivation(
const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode
)
try {
auto conn(getConnection());
conn->to << WorkerProto::Op::BuildDerivation << printStorePath(drvPath);
writeDerivation(conn->to, *this, drv);
conn->to << buildMode;
conn.processStderr();
return WorkerProto::Serialise<BuildResult>::read(*this, *conn);
return {WorkerProto::Serialise<BuildResult>::read(*this, *conn)};
} catch (...) {
return {result::current_exception()};
}
void RemoteStore::ensurePath(const StorePath & path)
{
kj::Promise<Result<void>> RemoteStore::ensurePath(const StorePath & path)
try {
auto conn(getConnection());
conn->to << WorkerProto::Op::EnsurePath << printStorePath(path);
conn.processStderr();
readInt(conn->from);
return {result::success()};
} catch (...) {
return {result::current_exception()};
}
lix/libstore/remote-store.hh
+12 -7
View File
@@ -112,17 +112,22 @@ public:
std::shared_ptr<const Realisation> queryRealisationUncached(const DrvOutput &) override;
void buildPaths(const std::vector<DerivedPath> & paths, BuildMode buildMode, std::shared_ptr<Store> evalStore) override;
kj ::Promise<Result<void>> buildPaths(
const std::vector<DerivedPath> & paths,
BuildMode buildMode,
std::shared_ptr<Store> evalStore
) override;
std::vector<KeyedBuildResult> buildPathsWithResults(
kj::Promise<Result<std::vector<KeyedBuildResult>>> buildPathsWithResults(
const std::vector<DerivedPath> & paths,
BuildMode buildMode,
std::shared_ptr<Store> evalStore) override;
BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
BuildMode buildMode) override;
kj ::Promise<Result<BuildResult>> buildDerivation(
const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode
) override;
void ensurePath(const StorePath & path) override;
kj::Promise<Result<void>> ensurePath(const StorePath & path) override;
void addTempRoot(const StorePath & path) override;
@@ -142,8 +147,8 @@ public:
* We make this fail for now so we can add implement this properly later
* without it being a breaking change.
*/
void repairPath(const StorePath & path) override
{ unsupported("repairPath"); }
kj::Promise<Result<void>> repairPath(const StorePath & path) override
try { unsupported("repairPath"); } catch (...) { return {result::current_exception()}; }
void addSignatures(const StorePath & storePath, const StringSet & sigs) override;
lix/libstore/store-api.cc
+3 -2
View File
@@ -3,6 +3,7 @@
#include "lix/libstore/derivations.hh"
#include "lix/libstore/store-api.hh"
#include "lix/libstore/nar-info-disk-cache.hh"
#include "lix/libutil/async.hh"
#include "lix/libutil/thread-pool.hh"
#include "lix/libutil/url.hh"
#include "lix/libutil/archive.hh"
@@ -815,7 +816,7 @@ void Store::substitutePaths(const StorePathSet & paths)
try {
std::vector<DerivedPath> subs;
for (auto & p : willSubstitute) subs.emplace_back(DerivedPath::Opaque{p});
buildPaths(subs);
RUN_ASYNC_IN_NEW_THREAD(buildPaths(subs));
} catch (Error & e) {
logWarning(e.info());
}
@@ -1355,7 +1356,7 @@ std::string showPaths(const PathSet & paths)
Derivation Store::derivationFromPath(const StorePath & drvPath)
{
ensurePath(drvPath);
RUN_ASYNC_IN_NEW_THREAD(ensurePath(drvPath));
return readDerivation(drvPath);
}
lix/libstore/store-api.hh
+11 -6
View File
@@ -8,6 +8,7 @@
#include "lix/libstore/derived-path.hh"
#include "lix/libutil/hash.hh"
#include "lix/libstore/content-address.hh"
#include "lix/libutil/result.hh"
#include "lix/libutil/serialise.hh"
#include "lix/libutil/lru-cache.hh"
#include "lix/libutil/sync.hh"
@@ -17,6 +18,7 @@
#include "lix/libutil/repair-flag.hh"
#include "lix/libutil/source-path.hh"
#include <kj/async.h>
#include <nlohmann/json_fwd.hpp>
#include <atomic>
#include <limits>
@@ -592,7 +594,7 @@ public:
* recursively building any sub-derivations. For inputs that are
* not derivations, substitute them.
*/
virtual void buildPaths(
virtual kj::Promise<Result<void>> buildPaths(
const std::vector<DerivedPath> & paths,
BuildMode buildMode = bmNormal,
std::shared_ptr<Store> evalStore = nullptr);
@@ -603,7 +605,7 @@ public:
* case of a build/substitution error, this function won't throw an
* exception, but return a BuildResult containing an error message.
*/
virtual std::vector<KeyedBuildResult> buildPathsWithResults(
virtual kj::Promise<Result<std::vector<KeyedBuildResult>>> buildPathsWithResults(
const std::vector<DerivedPath> & paths,
BuildMode buildMode = bmNormal,
std::shared_ptr<Store> evalStore = nullptr);
@@ -643,15 +645,18 @@ public:
* up with multiple different versions of dependencies without
* explicitly choosing to allow it).
*/
virtual BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
BuildMode buildMode = bmNormal);
virtual kj::Promise<Result<BuildResult>> buildDerivation(
const StorePath & drvPath,
const BasicDerivation & drv,
BuildMode buildMode = bmNormal
);
/**
* Ensure that a path is valid. If it is not currently valid, it
* may be made valid by running a substitute (if defined for the
* path).
*/
virtual void ensurePath(const StorePath & path);
virtual kj::Promise<Result<void>> ensurePath(const StorePath & path);
/**
* Add a store path as a temporary root of the garbage collector.
@@ -712,7 +717,7 @@ public:
* Repair the contents of the given path by redownloading it using
* a substituter (if available).
*/
virtual void repairPath(const StorePath & path);
virtual kj::Promise<Result<void>> repairPath(const StorePath & path);
/**
* Add signatures to the specified store path. The signatures are
lix/nix/bundle.cc
+2 -2
View File
@@ -110,12 +110,12 @@ struct CmdBundle : InstallableCommand
auto outPath = evalState->coerceToStorePath(attr2->pos, *attr2->value, context2, "");
store->buildPaths({
aio().blockOn(store->buildPaths({
DerivedPath::Built {
.drvPath = makeConstantStorePathRef(drvPath),
.outputs = OutputsSpec::All { },
},
});
}));
if (!outLink) {
auto * attr = vRes->attrs->get(evaluator->s.name);
lix/nix/daemon.cc
+18 -1
View File
@@ -252,7 +252,8 @@ static std::pair<TrustedFlag, std::string> authPeer(const PeerInfo & peer)
* the client. Otherwise, decide based on the authentication settings
* and user credentials (from the unix domain socket).
*/
static void daemonLoop(AsyncIoRoot & aio, std::optional<TrustedFlag> forceTrustClientOpt)
static void daemonLoop(AsyncIoRoot & aio, std::optional<TrustedFlag> forceTrustClientOpt);
static void daemonLoopImpl(std::optional<TrustedFlag> forceTrustClientOpt)
{
if (chdir("/") == -1)
throw SysError("cannot change current directory");
@@ -324,6 +325,8 @@ static void daemonLoop(AsyncIoRoot & aio, std::optional<TrustedFlag> forceTrustC
if (setsid() == -1)
throw SysError("creating a new session");
AsyncIoRoot aio;
// Restart the signal handler thread since it met its untimely
// demise at fork time.
startSignalHandlerThread(DoSignalSave::DontSaveBecauseAdvancedProcess);
@@ -355,6 +358,20 @@ static void daemonLoop(AsyncIoRoot & aio, std::optional<TrustedFlag> forceTrustC
}
}
}
static void daemonLoop(AsyncIoRoot & aio, std::optional<TrustedFlag> forceTrustClientOpt)
{
// we can't reuse the external async io root since it'd be shared with the
// children we will create, potentially trashing state, but the *previous*
// root is still alive as far as kj is concerned. we cannot recreate it in
// the child easily because darwin closes kqueues after fork, and since kj
// asserts after the kqueue close returns EBADF we'll die. the least awful
// way around this is to run the daemon loop in its own thread, without an
// async io root, and thus not have any shared state after we have forked.
std::async(std::launch::async, [&] {
ReceiveInterrupts ri;
return daemonLoopImpl(forceTrustClientOpt);
}).get();
}
/**
* Forward a standard IO connection to the given remote store.
lix/nix/develop.cc
+3 -2
View File
@@ -6,6 +6,7 @@
#include "lix/libstore/store-api.hh"
#include "lix/libstore/outputs-spec.hh"
#include "lix/libstore/derivations.hh"
#include "lix/libutil/async.hh"
#include "run.hh"
#include "lix/libstore/temporary-dir.hh"
@@ -258,12 +259,12 @@ try {
auto shellDrvPath = writeDerivation(*evalStore, drv);
/* Build the derivation. */
store->buildPaths(
TRY_AWAIT(store->buildPaths(
{ DerivedPath::Built {
.drvPath = makeConstantStorePathRef(shellDrvPath),
.outputs = OutputsSpec::All { },
}},
bmNormal, evalStore);
bmNormal, evalStore));
for (auto & [_0, optPath] : evalStore->queryPartialDerivationOutputMap(shellDrvPath)) {
assert(optPath);
lix/nix/flake.cc
+1 -1
View File
@@ -808,7 +808,7 @@ struct CmdFlakeCheck : FlakeCommand
if (build && !drvPaths.empty()) {
Activity act(*logger, lvlInfo, actUnknown,
fmt("running %d flake checks", drvPaths.size()));
store->buildPaths(drvPaths);
aio().blockOn(store->buildPaths(drvPaths));
}
if (hasErrors)
throw Error("some errors were encountered during the evaluation");
lix/nix/store-repair.cc
+1 -1
View File
@@ -20,7 +20,7 @@ struct CmdStoreRepair : StorePathsCommand
void run(ref<Store> store, StorePaths && storePaths) override
{
for (auto & path : storePaths)
store->repairPath(path);
aio().blockOn(store->repairPath(path));
}
};
lix/nix/upgrade-nix.cc
+1 -1
View File
@@ -94,7 +94,7 @@ struct CmdUpgradeNix : MixDryRun, EvalCommand
{
Activity act(*logger, lvlInfo, actUnknown, fmt("downloading '%s'...", store->printStorePath(storePath)));
store->ensurePath(storePath);
aio().blockOn(store->ensurePath(storePath));
}
// {profileDir}/bin/nix-env is a symlink to {profileDir}/bin/nix, which *then*
perl/lib/Nix/Store.xs
+5
View File
@@ -1,5 +1,10 @@
#include "lix/config.h"
// perl defines _ as a function-like macro for some reason.
// boost outcome uses _ as a name an internal storage type.
// can i make it any more obvious?
#include "lix/libutil/result.hh"
#include "EXTERN.h"
#include "perl.h"
#include "XSUB.h"
perl/lib/Nix/meson.build
+1
View File
@@ -24,6 +24,7 @@ perl_libstore = shared_module(
libstore,
sodium,
perl_include,
kj,
],
link_args : [
# Nix doesn't provide a pkg-config file for libutil.
perl/meson.build
+1
View File
@@ -66,5 +66,6 @@ endif
libstore = dependency('lixstore', 'lix-store', required : true)
libutil = dependency('lixutil', 'lix-util', required : true)
kj = dependency('kj-async', required : true, include_type : 'system')
subdir('lib/Nix')
tests/functional/test-libstoreconsumer/main.cc
+3 -1
View File
@@ -1,6 +1,7 @@
#include "lix/libstore/globals.hh"
#include "lix/libstore/store-api.hh"
#include "lix/libstore/build-result.hh"
#include "lix/libutil/async.hh"
#include <iostream>
using namespace nix;
@@ -13,6 +14,7 @@ int main (int argc, char **argv)
return 1;
}
AsyncIoRoot aio;
std::string drvPath = argv[1];
initLibStore();
@@ -28,7 +30,7 @@ int main (int argc, char **argv)
}
};
const auto results = store->buildPathsWithResults(paths, bmNormal, store);
const auto results = aio.blockOn(store->buildPathsWithResults(paths, bmNormal, store));
for (const auto & result : results) {
for (const auto & [outputName, realisation] : result.builtOutputs) {