guiwf/lix
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

libstore: asyncify build child setup completion wait

Browse Source 
Change-Id: Ica70af2a1205830f1ca4bb48f42d09923cff2e58
This commit is contained in:
eldritch horrors
2026-02-03 14:11:59 +00:00
parent ac64c727b5
commit aa896041e0
5 changed files with 78 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files
doc/manual/rl-next/linux-sandbox-launch-overhead.md
+2 -2
View File
@@ -1,6 +1,6 @@
---
synopsis: "Linux sandbox launch overhead greatly reduced"
cls: [5030, 5073]
cls: [5030, 5073, 5074]
category: "Improvements"
credits: [horrors]
---
@@ -9,4 +9,4 @@ Sandboxed builds are now much cheaper to launch on Linux, with constant manageme
overhead. This will mostly be noticeable when building derivation trees containing
many small derivations like nixpkgs' `writeFile` or `runCommand` with scripts that
very quickly. In synthetic tests we have seen build times of 3000 small runCommand
drop from 80 seconds to 24 seconds, which is the most optimistic case in practice.
drop from 80 seconds to 14 seconds, which is the most optimistic case in practice.
lix/libexec/launch-builder.cc
+14 -13
View File
@@ -143,15 +143,6 @@ int main(int argc, char * argv[])
throw SysError("cannot dup stderr into stdout");
}
/* Reroute stdin to /dev/null. */
kj::AutoCloseFd fdDevNull{open("/dev/null", O_RDWR)};
if (fdDevNull == nullptr) {
throw SysError("cannot open /dev/null");
}
if (dup2(fdDevNull.get(), STDIN_FILENO) == -1) {
throw SysError("cannot dup null device into stdin");
}
const bool setUser = prepareChildSetup(request);
// NOLINTNEXTLINE(lix-unsafe-c-calls): we trust the parent here
@@ -199,18 +190,28 @@ int main(int argc, char * argv[])
finishChildSetup(request);
/* Indicate that we managed to set up the build environment. */
writeFull(STDERR_FILENO, std::string("\2\n"));
/* Close all other file descriptors. */
closeExtraFDs();
// Reroute stdin to /dev/null. closing the setup socket fd also signals
// successful setup of the builder, all other errors must go to stderr.
kj::AutoCloseFd fdDevNull{open("/dev/null", O_RDWR | O_CLOEXEC)};
if (fdDevNull == nullptr) {
throw SysError("cannot open /dev/null");
}
if (dup2(fdDevNull.get(), STDIN_FILENO) == -1) {
throw SysError("cannot dup null device into stdin");
}
sendException = false;
execBuilder(request);
} catch (std::exception & e) { // NOLINT(lix-foreign-exceptions)
if (sendException) {
writeFull(STDERR_FILENO, std::format("\1{}\n", e.what()));
capnp::MallocMessageBuilder builder;
auto error = builder.getRoot<build::SetupResponse>();
RPC_FILL(error, setFatalError, e.what());
capnp::writeMessageToFd(STDIN_FILENO, builder);
} else {
writeFull(STDERR_FILENO, e.what());
}
lix/libexec/launch-builder.hh
+9 -2
View File
@@ -2,7 +2,10 @@
///@file
#include "lix/libstore/build/request.capnp.h"
#include "lix/libutil/rpc.hh"
#include <boost/format.hpp>
#include <capnp/message.h>
#include <capnp/serialize.h>
#include <cstring>
#include <exception>
#include <memory>
@@ -60,13 +63,17 @@ inline void printDebugLog(auto fmt, const auto &... args)
{
auto format = boost::format(fmt);
((format % args), ...);
writeFull(STDERR_FILENO, format.str());
capnp::MallocMessageBuilder builder;
auto log = builder.getRoot<build::SetupResponse>();
RPC_FILL(log, setLogLine, format.str());
capnp::writeMessageToFd(STDIN_FILENO, builder);
}
#define debug(msg, ...) \
do { \
if (::nix::printDebugLogs) { \
printDebugLog(msg "\n", __VA_ARGS__); \
printDebugLog(msg, __VA_ARGS__); \
} \
} while (0)
}
lix/libstore/build/local-derivation-goal.cc
+37 -30
View File
@@ -1,5 +1,6 @@
#include "lix/libstore/build/local-derivation-goal.hh"
#include "derivation-goal.hh"
#include "libutil/async-collect.hh"
#include "libutil/logging.hh"
#include "lix/libutil/async-io.hh"
#include "lix/libutil/async.hh"
@@ -38,6 +39,7 @@
#include "request.capnp.h"
#include <capnp/message.h>
#include <capnp/serialize-async.h>
#include <capnp/serialize.h>
#include <cstddef>
#include <cstdio>
@@ -950,45 +952,50 @@ try {
fillBuilderConfig(request);
auto setupFD = sys::openat(tmpDirFd.get(), "build-request", O_RDWR | O_CREAT | O_CLOEXEC, 0400);
if (!setupFD) {
throw SysError("creating builder setup file");
}
if (sys::unlinkat(tmpDirFd.get(), "build-request", 0)) {
throw SysError("unlinking builder setup file");
}
capnp::writeMessageToFd(setupFD.get(), requestBuilder);
if (lseek(setupFD.get(), 0, SEEK_SET) == -1) {
throw SysError("seeking builder setup file");
}
auto [setupParentFD, setupChildFD] = SocketPair::stream();
auto setupParent = AIO().lowLevelProvider.wrapSocketFd(setupParentFD.get());
/* Fork a child to build the package. */
pg = ProcessGroup{startChild(std::move(setupFD), std::move(builderOut))};
pg = ProcessGroup{startChild(std::move(setupChildFD), std::move(builderOut))};
/* Check if setting up the build environment failed. */
std::vector<std::string> msgs;
while (true) {
std::string msg = [&]() {
// NOLINTNEXTLINE(cppcoreguidelines-avoid-capturing-lambda-coroutines)
auto sendSetup = [&] -> kj::Promise<Result<void>> {
try {
return readLine(builderOutPTY.get());
} catch (Error & e) {
auto status = pg.wait();
e.addTrace({}, "while waiting for the build environment for '%s' to initialize (%s, previous messages: %s)",
worker.store.printStorePath(drvPath),
statusToString(status),
concatStringsSep("|", msgs));
throw;
co_await capnp::writeMessage(*setupParent, requestBuilder);
co_return result::success();
} catch (...) {
co_return {result::current_exception()};
}
}();
if (msg.substr(0, 1) == "\2") break;
if (msg.substr(0, 1) == "\1") {
Error ex("%s", Uncolored(msg.substr(1)));
};
// NOLINTNEXTLINE(cppcoreguidelines-avoid-capturing-lambda-coroutines)
auto waitForChildSetup = [&] -> kj::Promise<Result<void>> {
try {
while (true) {
KJ_IF_MAYBE (msg, co_await capnp::tryReadMessage(*setupParent)) {
auto response = (*msg)->getRoot<build::SetupResponse>();
if (response.isLogLine()) {
debug(
"sandbox setup: %1%", Uncolored(rpc::to<std::string_view>(response.getLogLine()))
);
} else if (response.isFatalError()) {
Error ex("%s", Uncolored(rpc::to<std::string_view>(response.getFatalError())));
ex.addTrace({}, "while setting up the build environment");
throw ex;
} else {
throw Error("unexpected setup response");
}
debug("sandbox setup: %1%", Uncolored(msg));
msgs.push_back(std::move(msg));
} else {
co_return result::success();
}
}
} catch (...) {
co_return result::current_exception();
}
};
/* Check if setting up the build environment failed. */
TRY_AWAIT(asyncJoin(sendSetup(), waitForChildSetup()));
co_return result::success();
} catch (...) {
lix/libstore/build/request.capnp
+7
View File
@@ -3,6 +3,13 @@
using Cxx = import "/capnp/c++.capnp";
$Cxx.namespace("nix::build");
struct SetupResponse {
union {
logLine @0 :Data;
fatalError @1 :Data;
}
}
struct Request {
struct Credentials {
uid @0 :UInt32;