diff --git a/doc/manual/rl-next/linux-sandbox-launch-overhead.md b/doc/manual/rl-next/linux-sandbox-launch-overhead.md index 36d381859..9f24d8d93 100644 --- a/doc/manual/rl-next/linux-sandbox-launch-overhead.md +++ b/doc/manual/rl-next/linux-sandbox-launch-overhead.md @@ -1,6 +1,6 @@ --- synopsis: "Linux sandbox launch overhead greatly reduced" -cls: [5030, 5073] +cls: [5030, 5073, 5074] category: "Improvements" credits: [horrors] --- @@ -9,4 +9,4 @@ Sandboxed builds are now much cheaper to launch on Linux, with constant manageme overhead. This will mostly be noticeable when building derivation trees containing many small derivations like nixpkgs' `writeFile` or `runCommand` with scripts that very quickly. In synthetic tests we have seen build times of 3000 small runCommand -drop from 80 seconds to 24 seconds, which is the most optimistic case in practice. +drop from 80 seconds to 14 seconds, which is the most optimistic case in practice. diff --git a/lix/libexec/launch-builder.cc b/lix/libexec/launch-builder.cc index d3509b5fb..88fa312d8 100644 --- a/lix/libexec/launch-builder.cc +++ b/lix/libexec/launch-builder.cc @@ -143,15 +143,6 @@ int main(int argc, char * argv[]) throw SysError("cannot dup stderr into stdout"); } - /* Reroute stdin to /dev/null. */ - kj::AutoCloseFd fdDevNull{open("/dev/null", O_RDWR)}; - if (fdDevNull == nullptr) { - throw SysError("cannot open /dev/null"); - } - if (dup2(fdDevNull.get(), STDIN_FILENO) == -1) { - throw SysError("cannot dup null device into stdin"); - } - const bool setUser = prepareChildSetup(request); // NOLINTNEXTLINE(lix-unsafe-c-calls): we trust the parent here @@ -199,18 +190,28 @@ int main(int argc, char * argv[]) finishChildSetup(request); - /* Indicate that we managed to set up the build environment. */ - writeFull(STDERR_FILENO, std::string("\2\n")); - /* Close all other file descriptors. */ closeExtraFDs(); + // Reroute stdin to /dev/null. closing the setup socket fd also signals + // successful setup of the builder, all other errors must go to stderr. + kj::AutoCloseFd fdDevNull{open("/dev/null", O_RDWR | O_CLOEXEC)}; + if (fdDevNull == nullptr) { + throw SysError("cannot open /dev/null"); + } + if (dup2(fdDevNull.get(), STDIN_FILENO) == -1) { + throw SysError("cannot dup null device into stdin"); + } + sendException = false; execBuilder(request); } catch (std::exception & e) { // NOLINT(lix-foreign-exceptions) if (sendException) { - writeFull(STDERR_FILENO, std::format("\1{}\n", e.what())); + capnp::MallocMessageBuilder builder; + auto error = builder.getRoot(); + RPC_FILL(error, setFatalError, e.what()); + capnp::writeMessageToFd(STDIN_FILENO, builder); } else { writeFull(STDERR_FILENO, e.what()); } diff --git a/lix/libexec/launch-builder.hh b/lix/libexec/launch-builder.hh index 682a5695c..52a7e791b 100644 --- a/lix/libexec/launch-builder.hh +++ b/lix/libexec/launch-builder.hh @@ -2,7 +2,10 @@ ///@file #include "lix/libstore/build/request.capnp.h" +#include "lix/libutil/rpc.hh" #include +#include +#include #include #include #include @@ -60,13 +63,17 @@ inline void printDebugLog(auto fmt, const auto &... args) { auto format = boost::format(fmt); ((format % args), ...); - writeFull(STDERR_FILENO, format.str()); + + capnp::MallocMessageBuilder builder; + auto log = builder.getRoot(); + RPC_FILL(log, setLogLine, format.str()); + capnp::writeMessageToFd(STDIN_FILENO, builder); } -#define debug(msg, ...) \ - do { \ - if (::nix::printDebugLogs) { \ - printDebugLog(msg "\n", __VA_ARGS__); \ - } \ +#define debug(msg, ...) \ + do { \ + if (::nix::printDebugLogs) { \ + printDebugLog(msg, __VA_ARGS__); \ + } \ } while (0) } diff --git a/lix/libstore/build/local-derivation-goal.cc b/lix/libstore/build/local-derivation-goal.cc index a77f01dce..6b3985c0f 100644 --- a/lix/libstore/build/local-derivation-goal.cc +++ b/lix/libstore/build/local-derivation-goal.cc @@ -1,5 +1,6 @@ #include "lix/libstore/build/local-derivation-goal.hh" #include "derivation-goal.hh" +#include "libutil/async-collect.hh" #include "libutil/logging.hh" #include "lix/libutil/async-io.hh" #include "lix/libutil/async.hh" @@ -38,6 +39,7 @@ #include "request.capnp.h" #include +#include #include #include #include @@ -950,45 +952,50 @@ try { fillBuilderConfig(request); - auto setupFD = sys::openat(tmpDirFd.get(), "build-request", O_RDWR | O_CREAT | O_CLOEXEC, 0400); - if (!setupFD) { - throw SysError("creating builder setup file"); - } - if (sys::unlinkat(tmpDirFd.get(), "build-request", 0)) { - throw SysError("unlinking builder setup file"); - } - capnp::writeMessageToFd(setupFD.get(), requestBuilder); - if (lseek(setupFD.get(), 0, SEEK_SET) == -1) { - throw SysError("seeking builder setup file"); - } + auto [setupParentFD, setupChildFD] = SocketPair::stream(); + auto setupParent = AIO().lowLevelProvider.wrapSocketFd(setupParentFD.get()); /* Fork a child to build the package. */ - pg = ProcessGroup{startChild(std::move(setupFD), std::move(builderOut))}; + pg = ProcessGroup{startChild(std::move(setupChildFD), std::move(builderOut))}; + + // NOLINTNEXTLINE(cppcoreguidelines-avoid-capturing-lambda-coroutines) + auto sendSetup = [&] -> kj::Promise> { + try { + co_await capnp::writeMessage(*setupParent, requestBuilder); + co_return result::success(); + } catch (...) { + co_return {result::current_exception()}; + } + }; + + // NOLINTNEXTLINE(cppcoreguidelines-avoid-capturing-lambda-coroutines) + auto waitForChildSetup = [&] -> kj::Promise> { + try { + while (true) { + KJ_IF_MAYBE (msg, co_await capnp::tryReadMessage(*setupParent)) { + auto response = (*msg)->getRoot(); + if (response.isLogLine()) { + debug( + "sandbox setup: %1%", Uncolored(rpc::to(response.getLogLine())) + ); + } else if (response.isFatalError()) { + Error ex("%s", Uncolored(rpc::to(response.getFatalError()))); + ex.addTrace({}, "while setting up the build environment"); + throw ex; + } else { + throw Error("unexpected setup response"); + } + } else { + co_return result::success(); + } + } + } catch (...) { + co_return result::current_exception(); + } + }; /* Check if setting up the build environment failed. */ - std::vector msgs; - while (true) { - std::string msg = [&]() { - try { - return readLine(builderOutPTY.get()); - } catch (Error & e) { - auto status = pg.wait(); - e.addTrace({}, "while waiting for the build environment for '%s' to initialize (%s, previous messages: %s)", - worker.store.printStorePath(drvPath), - statusToString(status), - concatStringsSep("|", msgs)); - throw; - } - }(); - if (msg.substr(0, 1) == "\2") break; - if (msg.substr(0, 1) == "\1") { - Error ex("%s", Uncolored(msg.substr(1))); - ex.addTrace({}, "while setting up the build environment"); - throw ex; - } - debug("sandbox setup: %1%", Uncolored(msg)); - msgs.push_back(std::move(msg)); - } + TRY_AWAIT(asyncJoin(sendSetup(), waitForChildSetup())); co_return result::success(); } catch (...) { diff --git a/lix/libstore/build/request.capnp b/lix/libstore/build/request.capnp index 3952bb0c8..aa18c1433 100644 --- a/lix/libstore/build/request.capnp +++ b/lix/libstore/build/request.capnp @@ -3,6 +3,13 @@ using Cxx = import "/capnp/c++.capnp"; $Cxx.namespace("nix::build"); +struct SetupResponse { + union { + logLine @0 :Data; + fatalError @1 :Data; + } +} + struct Request { struct Credentials { uid @0 :UInt32;