libstore: asyncify build child setup completion wait

Change-Id: Ica70af2a1205830f1ca4bb48f42d09923cff2e58

doc/manual/rl-next/linux-sandbox-launch-overhead.md

@@ -1,6 +1,6 @@
 ---
 synopsis: "Linux sandbox launch overhead greatly reduced"
-cls: [5030, 5073]
+cls: [5030, 5073, 5074]
 category: "Improvements"
 credits: [horrors]
 ---
@@ -9,4 +9,4 @@ Sandboxed builds are now much cheaper to launch on Linux, with constant manageme
 overhead. This will mostly be noticeable when building derivation trees containing
 many small derivations like nixpkgs' `writeFile` or `runCommand` with scripts that
 very quickly. In synthetic tests we have seen build times of 3000 small runCommand
-drop from 80 seconds to 24 seconds, which is the most optimistic case in practice.
+drop from 80 seconds to 14 seconds, which is the most optimistic case in practice.

lix/libexec/launch-builder.cc

@@ -143,15 +143,6 @@ int main(int argc, char * argv[])
             throw SysError("cannot dup stderr into stdout");
         }
 
-        /* Reroute stdin to /dev/null. */
-        kj::AutoCloseFd fdDevNull{open("/dev/null", O_RDWR)};
-        if (fdDevNull == nullptr) {
-            throw SysError("cannot open /dev/null");
-        }
-        if (dup2(fdDevNull.get(), STDIN_FILENO) == -1) {
-            throw SysError("cannot dup null device into stdin");
-        }
-
         const bool setUser = prepareChildSetup(request);
 
         // NOLINTNEXTLINE(lix-unsafe-c-calls): we trust the parent here
@@ -199,18 +190,28 @@ int main(int argc, char * argv[])
 
         finishChildSetup(request);
 
-        /* Indicate that we managed to set up the build environment. */
-        writeFull(STDERR_FILENO, std::string("\2\n"));
-
         /* Close all other file descriptors. */
         closeExtraFDs();
 
+        // Reroute stdin to /dev/null. closing the setup socket fd also signals
+        // successful setup of the builder, all other errors must go to stderr.
+        kj::AutoCloseFd fdDevNull{open("/dev/null", O_RDWR | O_CLOEXEC)};
+        if (fdDevNull == nullptr) {
+            throw SysError("cannot open /dev/null");
+        }
+        if (dup2(fdDevNull.get(), STDIN_FILENO) == -1) {
+            throw SysError("cannot dup null device into stdin");
+        }
+
         sendException = false;
 
         execBuilder(request);
     } catch (std::exception & e) { // NOLINT(lix-foreign-exceptions)
         if (sendException) {
-            writeFull(STDERR_FILENO, std::format("\1{}\n", e.what()));
+            capnp::MallocMessageBuilder builder;
+            auto error = builder.getRoot<build::SetupResponse>();
+            RPC_FILL(error, setFatalError, e.what());
+            capnp::writeMessageToFd(STDIN_FILENO, builder);
         } else {
             writeFull(STDERR_FILENO, e.what());
         }

lix/libexec/launch-builder.hh

@@ -2,7 +2,10 @@
 ///@file
 
 #include "lix/libstore/build/request.capnp.h"
+#include "lix/libutil/rpc.hh"
 #include <boost/format.hpp>
+#include <capnp/message.h>
+#include <capnp/serialize.h>
 #include <cstring>
 #include <exception>
 #include <memory>
@@ -60,13 +63,17 @@ inline void printDebugLog(auto fmt, const auto &... args)
 {
     auto format = boost::format(fmt);
     ((format % args), ...);
-    writeFull(STDERR_FILENO, format.str());
+
+    capnp::MallocMessageBuilder builder;
+    auto log = builder.getRoot<build::SetupResponse>();
+    RPC_FILL(log, setLogLine, format.str());
+    capnp::writeMessageToFd(STDIN_FILENO, builder);
 }
 
 #define debug(msg, ...)                      \
     do {                                     \
         if (::nix::printDebugLogs) {         \
-            printDebugLog(msg "\n", __VA_ARGS__); \
+            printDebugLog(msg, __VA_ARGS__); \
         }                                    \
     } while (0)
 }

lix/libstore/build/local-derivation-goal.cc

@@ -1,5 +1,6 @@
 #include "lix/libstore/build/local-derivation-goal.hh"
 #include "derivation-goal.hh"
+#include "libutil/async-collect.hh"
 #include "libutil/logging.hh"
 #include "lix/libutil/async-io.hh"
 #include "lix/libutil/async.hh"
@@ -38,6 +39,7 @@
 #include "request.capnp.h"
 
 #include <capnp/message.h>
+#include <capnp/serialize-async.h>
 #include <capnp/serialize.h>
 #include <cstddef>
 #include <cstdio>
@@ -950,45 +952,50 @@ try {
 
     fillBuilderConfig(request);
 
-    auto setupFD = sys::openat(tmpDirFd.get(), "build-request", O_RDWR | O_CREAT | O_CLOEXEC, 0400);
+    auto [setupParentFD, setupChildFD] = SocketPair::stream();
-    if (!setupFD) {
+    auto setupParent = AIO().lowLevelProvider.wrapSocketFd(setupParentFD.get());
-        throw SysError("creating builder setup file");
-    }
-    if (sys::unlinkat(tmpDirFd.get(), "build-request", 0)) {
-        throw SysError("unlinking builder setup file");
-    }
-    capnp::writeMessageToFd(setupFD.get(), requestBuilder);
-    if (lseek(setupFD.get(), 0, SEEK_SET) == -1) {
-        throw SysError("seeking builder setup file");
-    }
 
     /* Fork a child to build the package. */
-    pg = ProcessGroup{startChild(std::move(setupFD), std::move(builderOut))};
+    pg = ProcessGroup{startChild(std::move(setupChildFD), std::move(builderOut))};
 
-    /* Check if setting up the build environment failed. */
+    // NOLINTNEXTLINE(cppcoreguidelines-avoid-capturing-lambda-coroutines)
-    std::vector<std::string> msgs;
+    auto sendSetup = [&] -> kj::Promise<Result<void>> {
-    while (true) {
-        std::string msg = [&]() {
         try {
-                return readLine(builderOutPTY.get());
+            co_await capnp::writeMessage(*setupParent, requestBuilder);
-            } catch (Error & e) {
+            co_return result::success();
-                auto status = pg.wait();
+        } catch (...) {
-                e.addTrace({}, "while waiting for the build environment for '%s' to initialize (%s, previous messages: %s)",
+            co_return {result::current_exception()};
-                    worker.store.printStorePath(drvPath),
-                    statusToString(status),
-                    concatStringsSep("|", msgs));
-                throw;
         }
-        }();
+    };
-        if (msg.substr(0, 1) == "\2") break;
+
-        if (msg.substr(0, 1) == "\1") {
+    // NOLINTNEXTLINE(cppcoreguidelines-avoid-capturing-lambda-coroutines)
-            Error ex("%s", Uncolored(msg.substr(1)));
+    auto waitForChildSetup = [&] -> kj::Promise<Result<void>> {
+        try {
+            while (true) {
+                KJ_IF_MAYBE (msg, co_await capnp::tryReadMessage(*setupParent)) {
+                    auto response = (*msg)->getRoot<build::SetupResponse>();
+                    if (response.isLogLine()) {
+                        debug(
+                            "sandbox setup: %1%", Uncolored(rpc::to<std::string_view>(response.getLogLine()))
+                        );
+                    } else if (response.isFatalError()) {
+                        Error ex("%s", Uncolored(rpc::to<std::string_view>(response.getFatalError())));
                         ex.addTrace({}, "while setting up the build environment");
                         throw ex;
+                    } else {
+                        throw Error("unexpected setup response");
                     }
-        debug("sandbox setup: %1%", Uncolored(msg));
+                } else {
-        msgs.push_back(std::move(msg));
+                    co_return result::success();
                 }
+            }
+        } catch (...) {
+            co_return result::current_exception();
+        }
+    };
+
+    /* Check if setting up the build environment failed. */
+    TRY_AWAIT(asyncJoin(sendSetup(), waitForChildSetup()));
 
     co_return result::success();
 } catch (...) {

lix/libstore/build/request.capnp

@@ -3,6 +3,13 @@
 using Cxx = import "/capnp/c++.capnp";
 $Cxx.namespace("nix::build");
 
+struct SetupResponse {
+  union {
+    logLine @0 :Data;
+    fatalError @1 :Data;
+  }
+}
+
 struct Request {
   struct Credentials {
     uid @0 :UInt32;