diff --git a/doc/manual/rl-next/linux-sandbox-launch-overhead.md b/doc/manual/rl-next/linux-sandbox-launch-overhead.md index 41aca49ae..36d381859 100644 --- a/doc/manual/rl-next/linux-sandbox-launch-overhead.md +++ b/doc/manual/rl-next/linux-sandbox-launch-overhead.md @@ -1,12 +1,12 @@ --- synopsis: "Linux sandbox launch overhead greatly reduced" -cls: [5030] +cls: [5030, 5073] category: "Improvements" credits: [horrors] --- -Sandboxed builds are now much cheaper to launch on Linux with 50% lower management +Sandboxed builds are now much cheaper to launch on Linux, with constant management overhead. This will mostly be noticeable when building derivation trees containing many small derivations like nixpkgs' `writeFile` or `runCommand` with scripts that very quickly. In synthetic tests we have seen build times of 3000 small runCommand -drop from 80 seconds to 44 seconds, which is the most optimistic case in practice. +drop from 80 seconds to 24 seconds, which is the most optimistic case in practice. diff --git a/lix/libexec/launch-builder-darwin.cc b/lix/libexec/launch-builder-darwin.cc new file mode 100644 index 000000000..3eb38ed42 --- /dev/null +++ b/lix/libexec/launch-builder-darwin.cc @@ -0,0 +1,90 @@ +#include "launch-builder.hh" +#include "lix/libstore/build/request.capnp.h" +#include "lix/libutil/rpc.hh" +#include +#include +#include +#include +#include + +namespace nix { + +/* This definition is undocumented but depended upon by all major browsers. */ +extern "C" int sandbox_init_with_parameters( + const char * profile, uint64_t flags, const char * const parameters[], char ** errorbuf +); + +bool prepareChildSetup(build::Request::Reader request) +{ + return true; +} + +void finishChildSetup(build::Request::Reader request) +{ + const auto config = request.getPlatform().getDarwin(); + + /* The tmpDir in scope points at the temporary build directory for our derivation. Some packages try + different mechanisms to find temporary directories, so we want to open up a broader place for them + to put their files, if needed. */ + auto globalTmpDir = rpc::to(config.getGlobalTempDir()); + + /* They don't like trailing slashes on subpath directives */ + if (globalTmpDir.back() == '/') { + globalTmpDir.pop_back(); + } + + if (auto env = getenv("_NIX_TEST_NO_SANDBOX"); env && env != std::string_view("1")) { + std::vector sandboxArgs; + sandboxArgs.push_back("_NIX_BUILD_TOP"); + sandboxArgs.push_back(config.getTempDir().cStr()); + sandboxArgs.push_back("_GLOBAL_TMP_DIR"); + sandboxArgs.push_back(globalTmpDir.c_str()); + if (config.getAllowLocalNetworking()) { + sandboxArgs.push_back("_ALLOW_LOCAL_NETWORKING"); + sandboxArgs.push_back("1"); + } + sandboxArgs.push_back(nullptr); + // NOLINTNEXTLINE(lix-unsafe-c-calls): all of these are env names or paths + if (sandbox_init_with_parameters(config.getSandboxProfile().cStr(), 0, sandboxArgs.data(), nullptr)) { + writeFull(STDERR_FILENO, "failed to configure sandbox\n"); + _exit(1); + } + } +} + +[[noreturn]] +void execBuilder(build::Request::Reader request) +{ + const auto config = request.getPlatform().getDarwin(); + + posix_spawnattr_t attrp; + + if (posix_spawnattr_init(&attrp)) { + throw SysError("failed to initialize builder"); + } + + if (posix_spawnattr_setflags(&attrp, POSIX_SPAWN_SETEXEC)) { + throw SysError("failed to initialize builder"); + } + + const auto platform = rpc::to(config.getPlatform()); + + if (platform == "aarch64-darwin") { + // Unset kern.curproc_arch_affinity so we can escape Rosetta + int affinity = 0; + sysctlbyname("kern.curproc_arch_affinity", nullptr, nullptr, &affinity, sizeof(affinity)); + + cpu_type_t cpu = CPU_TYPE_ARM64; + posix_spawnattr_setbinpref_np(&attrp, 1, &cpu, nullptr); + } else if (platform == "x86_64-darwin") { + cpu_type_t cpu = CPU_TYPE_X86_64; + posix_spawnattr_setbinpref_np(&attrp, 1, &cpu, nullptr); + } + + ExecRequest req{request}; + + posix_spawn(nullptr, req.builder.c_str(), nullptr, &attrp, req.args.data(), req.envs.data()); + throw SysError(errno, std::format("running {}", req.builder)); +} + +} diff --git a/lix/libexec/launch-builder-fallback.cc b/lix/libexec/launch-builder-fallback.cc new file mode 100644 index 000000000..2d803292c --- /dev/null +++ b/lix/libexec/launch-builder-fallback.cc @@ -0,0 +1,24 @@ +#include "launch-builder.hh" +#include "lix/libstore/build/request.capnp.h" +#include +#include +#include + +namespace nix { + +bool prepareChildSetup(build::Request::Reader config) +{ + return true; +} + +void finishChildSetup(build::Request::Reader config) {} + +void execBuilder(build::Request::Reader config) +{ + ExecRequest req{config}; + + execve(req.builder.data(), req.args.data(), req.envs.data()); + throw SysError("running %s", req.builder); +} + +} diff --git a/lix/libexec/launch-builder-linux.cc b/lix/libexec/launch-builder-linux.cc new file mode 100644 index 000000000..1413d79e8 --- /dev/null +++ b/lix/libexec/launch-builder-linux.cc @@ -0,0 +1,446 @@ +#include "launch-builder.hh" +#include "lix/libstore/build/request.capnp.h" +#include "lix/libutil/rpc.hh" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if HAVE_SECCOMP +#include +#include +#include +#endif + +namespace fs = std::filesystem; + +namespace nix { + +// TODO dedup with libutil +static void setPersonality(std::string_view system) +{ + /* Change the personality to 32-bit if we're doing an + i686-linux build on an x86_64-linux machine. */ + struct utsname utsbuf; + uname(&utsbuf); + if ((system == "i686-linux" + && (std::string_view(SYSTEM) == "x86_64-linux" + || (!strcmp(utsbuf.sysname, "Linux") && !strcmp(utsbuf.machine, "x86_64")))) + || system == "armv7l-linux" || system == "armv6l-linux" || system == "armv5tel-linux") + { + if (personality(PER_LINUX32) == -1) { + throw SysError("cannot set 32-bit personality"); + } + } + + /* Disable address space randomization for improved + determinism. */ + int cur = personality(0xffffffff); + if (cur != -1) { + personality(cur | ADDR_NO_RANDOMIZE); + } +} + +bool pathExists(const fs::path & path) +{ + return fs::exists(fs::symlink_status(path)); +} + +void bindPath(const fs::path & source, const fs::path & target, bool optional = false) +{ + debug("bind mounting %1% to %2%", source, target); + + auto bindMount = [&]() { + if (mount(source.c_str(), target.c_str(), "", MS_BIND | MS_REC, 0) == -1) { + throw SysError("bind mount from %1% to %2% failed", source, target); + } + }; + + auto st = fs::symlink_status(source); + if (st.type() == fs::file_type::not_found) { + if (optional) { + return; + } else { + throw SysError("getting attributes of path %1%", source); + } + } + + if (st.type() == fs::file_type::directory) { + fs::create_directories(target); + bindMount(); + } else if (st.type() == fs::file_type::symlink) { + // Symlinks can (apparently) not be bind-mounted, so just copy it + fs::create_directories(target.parent_path()); + fs::copy_symlink(source, target); + } else { + fs::create_directories(target.parent_path()); + if (kj::AutoCloseFd file{open(target.c_str(), O_RDWR | O_CREAT, 0644)}; file == nullptr) { + throw SysError("could not create %s", target); + } + bindMount(); + } +} + +bool prepareChildSetup(build::Request::Reader request) +{ + auto config = request.getPlatform().getLinux(); + + // Set the NO_NEW_PRIVS prctl flag. + // This both makes loading seccomp filters work for unprivileged users, + // and is an additional security measure in its own right. + if (prctl(PR_SET_NO_NEW_PRIVS, 1L, 0L, 0L, 0L) == -1) { + throw SysError("PR_SET_NO_NEW_PRIVS failed"); + } +#if HAVE_SECCOMP + if (config.hasSeccompFilters()) { + const auto seccompBPF = config.getSeccompFilters(); + const auto entries = seccompBPF.size() / sizeof(struct sock_filter); + assert(entries <= std::numeric_limits::max()); + struct sock_fprog fprog = { + .len = static_cast(entries), + // the kernel does not actually write to the filter, and doesn't care about alignment + .filter = const_cast( + reinterpret_cast(seccompBPF.begin()) + ), + }; + if (syscall(SYS_seccomp, SECCOMP_SET_MODE_FILTER, 0, &fprog) != 0) { + throw SysError("unable to load seccomp BPF program"); + } + } +#endif + + KJ_DEFER(setPersonality(rpc::to(config.getPlatform()))); + + if (!config.hasSandbox()) { + return true; + } + + auto sandbox = config.getSandbox(); + + // NOLINTBEGIN(lix-unsafe-c-calls): we trust the parent that all sandbox config is correct. + // no strings in the linux sandbox config can be set by normal users or derivation authors, + // except (in single-user instances) storeDir and chrootRootDir, which must be valid paths. + // + // NOLINTBEGIN(lix-foreign-exceptions): they're all properly caught by the builder main fn. + + const fs::path chrootRootDir{rpc::to(sandbox.getChrootRootDir())}; + + if (sandbox.getPrivateNetwork()) { + /* Initialise the loopback interface. */ + kj::AutoCloseFd fd(socket(PF_INET, SOCK_DGRAM, IPPROTO_IP)); + if (fd == nullptr) { + throw SysError("cannot open IP socket"); + } + + struct ifreq ifr; + strcpy(ifr.ifr_name, "lo"); + ifr.ifr_flags = IFF_UP | IFF_LOOPBACK | IFF_RUNNING; + if (ioctl(fd.get(), SIOCSIFFLAGS, &ifr) == -1) { + throw SysError("cannot set loopback interface flags"); + } + } + + /* Set the hostname etc. to fixed values. */ + char hostname[] = "localhost"; + if (sethostname(hostname, sizeof(hostname)) == -1) { + throw SysError("cannot set host name"); + } + char domainname[] = "(none)"; // kernel default + if (setdomainname(domainname, sizeof(domainname)) == -1) { + throw SysError("cannot set domain name"); + } + + /* Make all filesystems private. This is necessary + because subtrees may have been mounted as "shared" + (MS_SHARED). (Systemd does this, for instance.) Even + though we have a private mount namespace, mounting + filesystems on top of a shared subtree still propagates + outside of the namespace. Making a subtree private is + local to the namespace, though, so setting MS_PRIVATE + does not affect the outside world. */ + const fs::path storeDir{rpc::to(sandbox.getStoreDir())}; + const auto chrootStoreDir = chrootRootDir / storeDir.relative_path(); + + if (mount(0, "/", 0, MS_PRIVATE | MS_REC, 0) == -1) { + throw SysError("unable to make '/' private"); + } + + /* Bind-mount chroot directory to itself, to treat it as a + different filesystem from /, as needed for pivot_root. */ + if (mount(chrootRootDir.c_str(), chrootRootDir.c_str(), "", MS_BIND, 0) == -1) { + throw SysError("unable to bind mount %1%", chrootRootDir); + } + + /* Bind-mount the sandbox's Nix store onto itself so that + we can mark it as a "shared" subtree, allowing bind + mounts made in *this* mount namespace to be propagated + into the child namespace created by the + unshare(CLONE_NEWNS) call below. + + Marking chrootRootDir as MS_SHARED causes pivot_root() + to fail with EINVAL. Don't know why. */ + if (mount(chrootStoreDir.c_str(), chrootStoreDir.c_str(), "", MS_BIND, 0) == -1) { + throw SysError("unable to bind mount the Nix store"); + } + + if (mount("", chrootStoreDir.c_str(), "", MS_SHARED, 0) == -1) { + throw SysError("unable to make %s shared", chrootStoreDir); + } + + bool devMounted = false; + bool devPtsMounted = false; + + /* Bind-mount all the directories from the "host" + filesystem that we want in the chroot + environment. */ + for (auto path : sandbox.getPaths()) { + const fs::path source{rpc::to(path.getSource())}; + const fs::path target{rpc::to(path.getTarget())}; + devMounted |= target == "/dev"; + devPtsMounted |= target == "/dev/pts"; + if (source == "/proc") { + continue; // backwards compatibility + } + +#if HAVE_EMBEDDED_SANDBOX_SHELL + if (source == "__embedded_sandbox_shell__") { + static unsigned char sh[] = { +#include "embedded-sandbox-shell.gen.hh" + }; + const fs::path dst = chrootRootDir / target.relative_path(); + fs::create_directories(dst.parent_path()); + writeFile(dst, std::string_view((const char *) sh, sizeof(sh))); + fs::permissions(dst, fs::perms(0555)); + } else +#endif + bindPath(source, chrootRootDir / target.relative_path(), path.getOptional()); + } + + /* Set up a nearly empty /dev, unless the user asked to + bind-mount the host /dev. */ + if (!devMounted) { + const auto bind = [&](fs::path item) { bindPath(item, chrootRootDir / item.relative_path()); }; + + fs::create_directories(chrootRootDir / "dev/shm"); + fs::create_directories(chrootRootDir / "dev/pts"); + bind("/dev/full"); + if (sandbox.getWantsKvm() && pathExists("/dev/kvm")) { + bind("/dev/kvm"); + } + bind("/dev/null"); + bind("/dev/random"); + bind("/dev/tty"); + bind("/dev/urandom"); + bind("/dev/zero"); + fs::create_symlink("/proc/self/fd", chrootRootDir / "dev/fd"); + fs::create_symlink("/proc/self/fd/0", chrootRootDir / "dev/stdin"); + fs::create_symlink("/proc/self/fd/1", chrootRootDir / "dev/stdout"); + fs::create_symlink("/proc/self/fd/2", chrootRootDir / "dev/stderr"); + } + + /* Bind a new instance of procfs on /proc. */ + fs::create_directories(chrootRootDir / "proc"); + if (mount("none", (chrootRootDir / "proc").c_str(), "proc", 0, 0) == -1) { + throw SysError("mounting /proc"); + } + + /* Mount sysfs on /sys. */ + if (request.hasCredentials() && request.getCredentials().getUidCount() != 1) { + fs::create_directories(chrootRootDir / "sys"); + if (mount("none", (chrootRootDir / "sys").c_str(), "sysfs", 0, 0) == -1) { + throw SysError("mounting /sys"); + } + } + + /* Mount a new tmpfs on /dev/shm to ensure that whatever + the builder puts in /dev/shm is cleaned up automatically. */ + if (pathExists("/dev/shm") + && mount("none", (chrootRootDir / "dev/shm").c_str(), "tmpfs", 0, sandbox.getSandboxShmFlags().cStr()) + == -1) + { + throw SysError("mounting /dev/shm"); + } + + /* Mount a new devpts on /dev/pts. Note that this + requires the kernel to be compiled with + CONFIG_DEVPTS_MULTIPLE_INSTANCES=y (which is the case + if /dev/ptx/ptmx exists). */ + if (pathExists("/dev/pts/ptmx") && !pathExists(chrootRootDir / "dev/ptmx") && !devPtsMounted) { + if (mount("none", (chrootRootDir / "dev/pts").c_str(), "devpts", 0, "newinstance,mode=0620") == 0) { + fs::create_symlink("/dev/pts/ptmx", chrootRootDir / "dev/ptmx"); + + /* Make sure /dev/pts/ptmx is world-writable. With some + Linux versions, it is created with permissions 0. */ + fs::permissions(chrootRootDir / "dev/pts/ptmx", fs::perms(0666)); + } else { + if (errno != EINVAL) { + throw SysError("mounting /dev/pts"); + } + bindPath("/dev/pts", chrootRootDir / "dev/pts"); + bindPath("/dev/ptmx", chrootRootDir / "dev/ptmx"); + } + } + + /* Make /etc unwritable */ + if (!sandbox.getUseUidRange()) { + fs::permissions(chrootRootDir / "etc", fs::perms(0555)); + } + + /* The comment below is now outdated. Recursive Nix has been removed. + * So there's no need to make path appear in the sandbox. + * TODO(Raito): cleanup before a merge. + */ + /* Unshare this mount namespace. This is necessary because + pivot_root() below changes the root of the mount + namespace. This means that the call to setns() in + addDependency() would hide the host's filesystem, + making it impossible to bind-mount paths from the host + Nix store into the sandbox. Therefore, we save the + pre-pivot_root namespace in + sandboxMountNamespace. Since we made /nix/store a + shared subtree above, this allows addDependency() to + make paths appear in the sandbox. */ + if (unshare(CLONE_NEWNS) == -1) { + throw SysError("unsharing mount namespace"); + } + + /* Creating a new cgroup namespace is independent of whether we enabled the cgroup experimental feature. + * We always create a new cgroup namespace from a sandboxing perspective. */ + /* Unshare the cgroup namespace. This means + /proc/self/cgroup will show the child's cgroup as '/' + rather than whatever it is in the parent. */ + if (unshare(CLONE_NEWCGROUP) == -1) { + throw SysError("unsharing cgroup namespace"); + } + + /* Do the chroot(). */ + if (chdir(chrootRootDir.c_str()) == -1) { + throw SysError("cannot change directory to %1%", chrootRootDir); + } + + if (mkdir("real-root", 0) == -1) { + throw SysError("cannot create real-root directory"); + } + + if (syscall(SYS_pivot_root, ".", "real-root") == -1) { + throw SysError("cannot pivot old root directory onto %1%", chrootRootDir / "real-root"); + } + + if (chroot(".") == -1) { + throw SysError("cannot change root directory to %1%", chrootRootDir); + } + + if (umount2("real-root", MNT_DETACH) == -1) { + throw SysError("cannot unmount real root filesystem"); + } + + if (rmdir("real-root") == -1) { + throw SysError("cannot remove real-root directory"); + } + + /* Switch to the sandbox uid/gid in the user namespace, + which corresponds to the build user or calling user in + the parent namespace. */ + if (setgid(sandbox.getGid()) == -1) { + throw SysError("setgid failed"); + } + if (setuid(sandbox.getUid()) == -1) { + throw SysError("setuid failed"); + } + + if (sandbox.hasWaitForInterface()) { + // wait for the pasta interface to appear. pasta can't signal us when + // it's done setting up the namespace, so we have to wait for a while + kj::AutoCloseFd fd(socket(PF_INET, SOCK_DGRAM, IPPROTO_IP)); + if (fd == nullptr) { + throw SysError("cannot open IP socket"); + } + + struct ifreq ifr; + strncpy(ifr.ifr_name, sandbox.getWaitForInterface().cStr(), sizeof(ifr.ifr_name)); + // wait two minutes for the interface to appear. if it does not do so + // we are either grossly overloaded, or pasta startup failed somehow. + static constexpr int SINGLE_WAIT_US = 1000; + static constexpr int TOTAL_WAIT_US = 120'000'000; + for (unsigned tries = 0;; tries++) { + if (tries > TOTAL_WAIT_US / SINGLE_WAIT_US) { + throw std::runtime_error( + "sandbox network setup timed out, please check daemon logs for possible error output." + ); + } else if (ioctl(fd.get(), SIOCGIFFLAGS, &ifr) == 0) { + if ((ifr.ifr_ifru.ifru_flags & IFF_UP) != 0) { + break; + } + } else if (errno == ENODEV) { + usleep(SINGLE_WAIT_US); + } else { + throw SysError("cannot get loopback interface flags"); + } + } + } + + // NOLINTEND(lix-foreign-exceptions) + // NOLINTEND(lix-unsafe-c-calls) + + return false; +} + +void finishChildSetup(build::Request::Reader request) +{ + // clear all capabilities when not running as root in the sandbox. + // we always clear ambient capabilities because they survive exec. + if (prctl(PR_CAP_AMBIENT, PR_CAP_AMBIENT_CLEAR_ALL, 0L, 0L, 0L) == -1) { + throw SysError("clearing ambient caps"); + } + if (!request.getPlatform().getLinux().getSandbox().getUseUidRange()) { + static constexpr uint32_t LINUX_CAPABILITY_VERSION_3 = 0x20080522; + static constexpr uint32_t LINUX_CAPABILITY_U32S_3 = 2; + struct user_cap_header_struct + { + uint32_t version; + int pid; + } hdr = {LINUX_CAPABILITY_VERSION_3, 0}; + struct user_cap_data_struct + { + uint32_t effective; + uint32_t permitted; + uint32_t inheritable; + } data[LINUX_CAPABILITY_U32S_3] = {}; + if (syscall(SYS_capset, &hdr, data)) { + throw SysError("couldn't set capabilities"); + } + } + + if (prctl(PR_SET_PDEATHSIG, SIGKILL) == -1) { + throw SysError("setting death signal"); + } + if (getppid() != request.getPlatform().getLinux().getParentPid()) { + raise(SIGKILL); + } +} + +[[noreturn]] +void execBuilder(build::Request::Reader request) +{ + ExecRequest req{request}; + + execve(req.builder.data(), req.args.data(), req.envs.data()); + throw SysError("running %s", req.builder); +} + +} diff --git a/lix/libexec/launch-builder.cc b/lix/libexec/launch-builder.cc new file mode 100644 index 000000000..d3509b5fb --- /dev/null +++ b/lix/libexec/launch-builder.cc @@ -0,0 +1,219 @@ +#include "launch-builder.hh" +#include "lix/libstore/build/request.capnp.h" +#include "lix/libutil/rpc.hh" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace nix { +bool printDebugLogs = false; + +static void requireCString(const char * context, const std::string & s) +{ + if (s.contains('\0')) { + std::string p{s}; + for (auto pos = p.find('\0'); pos != p.npos; pos = p.find('\0')) { + p.replace(pos, 1, "␀"); + } + // NOLINTNEXTLINE(lix-foreign-exceptions) + throw std::runtime_error(std::format("derivation {} {} contains NUL bytes", context, p)); + } +} + +ExecRequest::ExecRequest(build::Request::Reader request) +{ + const auto fill = [](auto context, auto & strings, auto & pointers, auto from) { + strings.reserve(from.size()); + for (auto arg : from) { + strings.push_back(rpc::to(arg)); + requireCString(context, strings.back()); + pointers.push_back(strings.back().data()); + } + pointers.push_back(nullptr); + }; + + builder = rpc::to(request.getBuilder()); + requireCString("derivation builder", builder); + + fill("derivation argument", argsStorage, args, request.getArgs()); + fill("derivation environment entry", envsStorage, envs, request.getEnvironment()); +} + +void writeFull(int fd, std::string_view data) +{ + while (!data.empty()) { + const auto wrote = ::write(fd, data.data(), data.size()); + if (wrote < 0) { + throw SysError("write()"); + } else { + data.remove_prefix(size_t(wrote)); + } + } +} + +static void closeExtraFDs() +{ + constexpr int MAX_KEPT_FD = 2; + static_assert(std::max({STDIN_FILENO, STDOUT_FILENO, STDERR_FILENO}) == MAX_KEPT_FD); + + // Both Linux and FreeBSD support close_range. +#if __linux__ || __FreeBSD__ + auto closeRange = [](unsigned int first, unsigned int last, int flags) -> int { + // musl does not have close_range as of 2024-08-10 + // patch: https://www.openwall.com/lists/musl/2024/08/01/9 +#if HAVE_CLOSE_RANGE + return close_range(first, last, flags); +#else + return syscall(SYS_close_range, first, last, flags); +#endif + }; + // first try to close_range everything we don't care about. if this + // returns an error with these parameters we're running on a kernel + // that does not implement close_range (i.e. pre 5.9) and fall back + // to the old method. we should remove that though, in some future. + if (closeRange(3, ~0U, 0) == 0) { + return; + } +#endif + +#if __linux__ + try { + for (auto & s : std::filesystem::directory_iterator("/proc/self/fd")) { + auto fd = std::stoi(s.path().filename().c_str()); + if (fd > MAX_KEPT_FD) { + debug("closing leaked FD %d", fd); + close(fd); + } + } + return; + } catch (std::exception &) { // NOLINT(lix-foreign-exceptions): that's what std::filesystem throws + } +#endif + + int maxFD = 0; + maxFD = sysconf(_SC_OPEN_MAX); + for (int fd = MAX_KEPT_FD + 1; fd < maxFD; ++fd) { + close(fd); /* ignore result */ + } +} +} + +int main(int argc, char * argv[]) +{ + using namespace nix; + + if (argc < 1) { + return 255; + } + + bool sendException = true; + + try { + capnp::StreamFdMessageReader reader(STDIN_FILENO); + + auto request = reader.getRoot(); + + printDebugLogs = request.getDebug(); + + { + sigset_t set; + sigemptyset(&set); + if (sigprocmask(SIG_SETMASK, &set, nullptr)) { + throw SysError("failed to unmask signals"); + } + } + + /* Put the child in a separate session (and thus a separate + process group) so that it has no controlling terminal (meaning + that e.g. ssh cannot open /dev/tty) and it doesn't receive + terminal signals. */ + if (setsid() == -1) { + throw SysError("creating a new session"); + } + + /* Dup stderr to stdout. */ + if (dup2(STDERR_FILENO, STDOUT_FILENO) == -1) { + throw SysError("cannot dup stderr into stdout"); + } + + /* Reroute stdin to /dev/null. */ + kj::AutoCloseFd fdDevNull{open("/dev/null", O_RDWR)}; + if (fdDevNull == nullptr) { + throw SysError("cannot open /dev/null"); + } + if (dup2(fdDevNull.get(), STDIN_FILENO) == -1) { + throw SysError("cannot dup null device into stdin"); + } + + const bool setUser = prepareChildSetup(request); + + // NOLINTNEXTLINE(lix-unsafe-c-calls): we trust the parent here + if (chdir(rpc::to(request.getWorkingDir()).c_str()) == -1) { + throw SysError("changing into %s", rpc::to(request.getWorkingDir())); + } + + /* Disable core dumps by default. */ + struct rlimit limit = {0, RLIM_INFINITY}; + if (request.getEnableCoreDumps()) { + limit.rlim_cur = RLIM_INFINITY; + } + setrlimit(RLIMIT_CORE, &limit); + + // FIXME: set other limits to deterministic values? + + /* If we are running in `build-users' mode, then switch to the + user we allocated above. Make sure that we drop all root + privileges. Note that above we have closed all file + descriptors except std*, so that's safe. Also note that + setuid() when run as root sets the real, effective and + saved UIDs. */ + if (setUser && request.hasCredentials()) { + auto creds = request.getCredentials(); + /* Preserve supplementary groups of the build user, to allow + admins to specify groups such as "kvm". */ + std::vector gids; + std::copy( + creds.getSupplementaryGroups().begin(), + creds.getSupplementaryGroups().end(), + std::back_inserter(gids) + ); + if (setgroups(gids.size(), gids.data()) == -1) { + throw SysError("cannot set supplementary groups of build user"); + } + + if (setgid(creds.getGid()) == -1 || getgid() != creds.getGid() || getegid() != creds.getGid()) { + throw SysError("setgid failed"); + } + + if (setuid(creds.getUid()) == -1 || getuid() != creds.getUid() || geteuid() != creds.getUid()) { + throw SysError("setuid failed"); + } + } + + finishChildSetup(request); + + /* Indicate that we managed to set up the build environment. */ + writeFull(STDERR_FILENO, std::string("\2\n")); + + /* Close all other file descriptors. */ + closeExtraFDs(); + + sendException = false; + + execBuilder(request); + } catch (std::exception & e) { // NOLINT(lix-foreign-exceptions) + if (sendException) { + writeFull(STDERR_FILENO, std::format("\1{}\n", e.what())); + } else { + writeFull(STDERR_FILENO, e.what()); + } + return 1; + } +} diff --git a/lix/libexec/launch-builder.hh b/lix/libexec/launch-builder.hh new file mode 100644 index 000000000..682a5695c --- /dev/null +++ b/lix/libexec/launch-builder.hh @@ -0,0 +1,72 @@ +#pragma once +///@file + +#include "lix/libstore/build/request.capnp.h" +#include +#include +#include +#include +#include +#include +#include +#include + +namespace nix { +bool prepareChildSetup(nix::build::Request::Reader request); +void finishChildSetup(nix::build::Request::Reader request); + +[[noreturn]] +void execBuilder(nix::build::Request::Reader request); + +// silence the foreign exception lint for this helper +class BaseException : public std::exception +{}; + +class SysError : public BaseException +{ +private: + std::shared_ptr msg; + +public: + explicit SysError(auto fmt, const auto &... args) : SysError(errno, fmt, args...) {} + SysError(int error, auto fmt, const auto &... args) + { + const auto errstr = strerror(error); + auto format = boost::format(fmt); + ((format % args), ...); + msg = std::make_shared(format.str() + ": " + errstr); + } + + const char * what() const noexcept override + { + return msg->c_str(); + } +}; + +struct ExecRequest +{ + std::string builder; + std::vector argsStorage, envsStorage; + std::vector args, envs; + + ExecRequest(nix::build::Request::Reader request); +}; + +void writeFull(int fd, std::string_view data); + +extern bool printDebugLogs; + +inline void printDebugLog(auto fmt, const auto &... args) +{ + auto format = boost::format(fmt); + ((format % args), ...); + writeFull(STDERR_FILENO, format.str()); +} + +#define debug(msg, ...) \ + do { \ + if (::nix::printDebugLogs) { \ + printDebugLog(msg "\n", __VA_ARGS__); \ + } \ + } while (0) +} diff --git a/lix/libexec/meson.build b/lix/libexec/meson.build index add540602..5136aaed7 100644 --- a/lix/libexec/meson.build +++ b/lix/libexec/meson.build @@ -14,6 +14,29 @@ kill_user = executable( install_dir : libexecdir / 'lix', ) +if is_linux + launch_builder_impl = 'linux' +elif is_darwin + launch_builder_impl = 'darwin' +else + launch_builder_impl = 'fallback' +endif + +launch_builder = executable( + 'launch-builder', + files( + 'launch-builder.cc', + f'launch-builder-@launch_builder_impl@.cc', + ), + liblix_generated_headers, + include_directories : [ '../..' ], + dependencies : [ + capnp, + ], + install : true, + install_dir : libexecdir / 'lix', +) + run_build_hook = executable( 'run-build-hook', files('run-build-hook.cc'), diff --git a/lix/libstore/build/local-derivation-goal.cc b/lix/libstore/build/local-derivation-goal.cc index 5c2e5fde7..a77f01dce 100644 --- a/lix/libstore/build/local-derivation-goal.cc +++ b/lix/libstore/build/local-derivation-goal.cc @@ -1,5 +1,6 @@ #include "lix/libstore/build/local-derivation-goal.hh" #include "derivation-goal.hh" +#include "libutil/logging.hh" #include "lix/libutil/async-io.hh" #include "lix/libutil/async.hh" #include "lix/libutil/current-process.hh" @@ -37,7 +38,9 @@ #include "request.capnp.h" #include +#include #include +#include #include #include #include @@ -943,11 +946,24 @@ try { auto groups = buildUser->getSupplementaryGIDs(); creds.setSupplementaryGroups({groups.data(), groups.size()}); } + request.setDebug(verbosity >= lvlDebug); fillBuilderConfig(request); + auto setupFD = sys::openat(tmpDirFd.get(), "build-request", O_RDWR | O_CREAT | O_CLOEXEC, 0400); + if (!setupFD) { + throw SysError("creating builder setup file"); + } + if (sys::unlinkat(tmpDirFd.get(), "build-request", 0)) { + throw SysError("unlinking builder setup file"); + } + capnp::writeMessageToFd(setupFD.get(), requestBuilder); + if (lseek(setupFD.get(), 0, SEEK_SET) == -1) { + throw SysError("seeking builder setup file"); + } + /* Fork a child to build the package. */ - pg = ProcessGroup{startChild(request.asReader(), std::move(builderOut))}; + pg = ProcessGroup{startChild(std::move(setupFD), std::move(builderOut))}; /* Check if setting up the build environment failed. */ std::vector msgs; @@ -979,15 +995,19 @@ try { co_return result::current_exception(); } -Pid LocalDerivationGoal::startChild(build::Request::Reader request, AutoCloseFD logPTY) +Pid LocalDerivationGoal::startChild(AutoCloseFD setupFD, AutoCloseFD logPTY) { - return startProcess([&]() { - if (dup2(logPTY.get(), STDERR_FILENO) == -1) { - throw SysError("failed to redirect build output to log file"); - } - closeOnExec(STDERR_FILENO, false); - runChild(request); + auto child = runProgram2({ + .program = LIX_LIBEXEC_DIR "/launch-builder", + .searchPath = false, + .redirections = + { + {.dup = STDIN_FILENO, .from = setupFD.get()}, + {.dup = STDERR_FILENO, .from = logPTY.get()}, + }, + .keepContext = true, }); + return std::get<0>(child.release()); } void LocalDerivationGoal::initTmpDir() { @@ -1221,173 +1241,6 @@ void LocalDerivationGoal::chownToBuilder(const AutoCloseFD & fd) throw SysError("cannot change ownership of file '%1%'", fd.guessOrInventPath()); } -static void closeExtraFDs() -{ - constexpr int MAX_KEPT_FD = 2; - static_assert(std::max({STDIN_FILENO, STDOUT_FILENO, STDERR_FILENO}) == MAX_KEPT_FD); - - // Both Linux and FreeBSD support close_range. -#if __linux__ || __FreeBSD__ - auto closeRange = [](unsigned int first, unsigned int last, int flags) -> int { - // musl does not have close_range as of 2024-08-10 - // patch: https://www.openwall.com/lists/musl/2024/08/01/9 -#if HAVE_CLOSE_RANGE - return close_range(first, last, flags); -#else - return syscall(SYS_close_range, first, last, flags); -#endif - }; - // first try to close_range everything we don't care about. if this - // returns an error with these parameters we're running on a kernel - // that does not implement close_range (i.e. pre 5.9) and fall back - // to the old method. we should remove that though, in some future. - if (closeRange(3, ~0U, 0) == 0) { - return; - } -#endif - -#if __linux__ - try { - for (auto & s : std::filesystem::directory_iterator("/proc/self/fd")) { - auto fd = std::stoi(s.path().filename().c_str()); - if (fd > MAX_KEPT_FD) { - debug("closing leaked FD %d", fd); - close(fd); - } - } - return; - } catch (std::exception &) { // NOLINT(lix-foreign-exceptions): that's what std::filesystem throws - } -#endif - - int maxFD = 0; - maxFD = sysconf(_SC_OPEN_MAX); - for (int fd = MAX_KEPT_FD + 1; fd < maxFD; ++fd) { - close(fd); /* ignore result */ - } -} - -void LocalDerivationGoal::runChild(build::Request::Reader request) -{ - /* Warning: in the child we should absolutely not make any SQLite - calls! */ - - bool sendException = true; - - try { /* child */ - - logger = makeSimpleLogger(); - - { - sigset_t set; - sigemptyset(&set); - if (sigprocmask(SIG_SETMASK, &set, nullptr)) { - throw SysError("failed to unmask signals"); - } - } - - /* Put the child in a separate session (and thus a separate - process group) so that it has no controlling terminal (meaning - that e.g. ssh cannot open /dev/tty) and it doesn't receive - terminal signals. */ - if (setsid() == -1) { - throw SysError("creating a new session"); - } - - /* Dup stderr to stdout. */ - if (dup2(STDERR_FILENO, STDOUT_FILENO) == -1) { - throw SysError("cannot dup stderr into stdout"); - } - - /* Reroute stdin to /dev/null. */ - kj::AutoCloseFd fdDevNull{open("/dev/null", O_RDWR)}; - if (fdDevNull == nullptr) { - throw SysError("cannot open '%1%'", "/dev/null"); - } - if (dup2(fdDevNull.get(), STDIN_FILENO) == -1) { - throw SysError("cannot dup null device into stdin"); - } - - const bool setUser = prepareChildSetup(request); - - // NOLINTNEXTLINE(lix-unsafe-c-calls): we trust the parent here - if (chdir(rpc::to(request.getWorkingDir()).c_str()) == -1) { - throw SysError("changing into '%1%'", rpc::to(request.getWorkingDir())); - } - - /* Close all other file descriptors. */ - closeExtraFDs(); - - /* Disable core dumps by default. */ - struct rlimit limit = { 0, RLIM_INFINITY }; - if (request.getEnableCoreDumps()) { - limit.rlim_cur = RLIM_INFINITY; - } - setrlimit(RLIMIT_CORE, &limit); - - // FIXME: set other limits to deterministic values? - - /* If we are running in `build-users' mode, then switch to the - user we allocated above. Make sure that we drop all root - privileges. Note that above we have closed all file - descriptors except std*, so that's safe. Also note that - setuid() when run as root sets the real, effective and - saved UIDs. */ - if (setUser && request.hasCredentials()) { - auto creds = request.getCredentials(); - /* Preserve supplementary groups of the build user, to allow - admins to specify groups such as "kvm". */ - std::vector gids; - std::copy( - creds.getSupplementaryGroups().begin(), - creds.getSupplementaryGroups().end(), - std::back_inserter(gids) - ); - if (setgroups(gids.size(), gids.data()) == -1) { - throw SysError("cannot set supplementary groups of build user"); - } - - if (setgid(creds.getGid()) == -1 || getgid() != creds.getGid() || getegid() != creds.getGid()) { - throw SysError("setgid failed"); - } - - if (setuid(creds.getUid()) == -1 || getuid() != creds.getUid() || geteuid() != creds.getUid()) { - throw SysError("setuid failed"); - } - } - - finishChildSetup(request); - - /* Indicate that we managed to set up the build environment. */ - writeFull(STDERR_FILENO, std::string("\2\n")); - - sendException = false; - - /* Execute the program. This should not return. */ - execBuilder(request); - - throw SysError("executing '%1%'", drv->builder); - - } catch (std::exception & e) { // NOLINT(lix-foreign-exceptions) - if (sendException) { - writeFull(STDERR_FILENO, std::format("\1{}\n", e.what())); - } else { - writeFull(STDERR_FILENO, e.what()); - } - _exit(1); - } -} - -void LocalDerivationGoal::execBuilder(build::Request::Reader request) -{ - sys::execve( - rpc::to(request.getBuilder()), - rpc::to(request.getArgs()), - rpc::to(request.getEnvironment()) - ); -} - - kj::Promise> LocalDerivationGoal::registerOutputs() try { /* When using a build hook, the build hook can register the output diff --git a/lix/libstore/build/local-derivation-goal.hh b/lix/libstore/build/local-derivation-goal.hh index adbb2de80..9afd43d3f 100644 --- a/lix/libstore/build/local-derivation-goal.hh +++ b/lix/libstore/build/local-derivation-goal.hh @@ -7,6 +7,7 @@ #include "lix/libutil/error.hh" #include "lix/libutil/processes.hh" #include "lix/libutil/cgroup.hh" +#include namespace nix { @@ -207,11 +208,6 @@ struct LocalDerivationGoal : public DerivationGoal int getChildStatus() override; - /** - * Run the builder's process. - */ - void runChild(build::Request::Reader request); - /** * Check that the derivation outputs all exist and register them * as valid. @@ -292,30 +288,13 @@ protected: * Create a new process that runs `openSlave` and `runChild` * On some platforms this process is created with sandboxing flags. */ - virtual Pid startChild(build::Request::Reader request, AutoCloseFD logPTY); + virtual Pid startChild(AutoCloseFD setupFD, AutoCloseFD logPTY); kj::Promise> handleRawChild() noexcept; kj::Promise>> handleRawChildStream() noexcept; virtual void fillBuilderConfig(build::Request::Builder request) {} - /** - * Prepare the sandbox. Currently only used on linux to build the sandbox namespace, - * write configuration files inside it, and to set up networking with pasta enabled. - * Returns `true` if sandbox is running under the same credentials as the daemon, or - * `false` if this step has changed our credentials to the build user/group already. - */ - [[nodiscard]] - virtual bool prepareChildSetup(build::Request::Reader request) - { - return true; - } - - /** - * Finish sandbox setup and prepare for actually executing the builder processes. - */ - virtual void finishChildSetup(build::Request::Reader request) {} - /** * Create a special accessor that can access paths that were built within the sandbox's * chroot. @@ -325,12 +304,6 @@ protected: return std::nullopt; }; - /** - * Execute the builder, replacing the current process. - * Generally this means an `execve` call. - */ - virtual void execBuilder(build::Request::Reader request); - /** * Whether derivation can be built on current platform with `uid-range` feature */ diff --git a/lix/libstore/build/request.capnp b/lix/libstore/build/request.capnp index 5d1363514..3952bb0c8 100644 --- a/lix/libstore/build/request.capnp +++ b/lix/libstore/build/request.capnp @@ -51,6 +51,7 @@ struct Request { workingDir @3 :Data; enableCoreDumps @4 :Bool; credentials @5 :Credentials; + debug @8 :Bool; platform :union { linux @6 :LinuxPlatform; diff --git a/lix/libstore/platform/darwin.cc b/lix/libstore/platform/darwin.cc index 15e5c3af7..04c470e74 100644 --- a/lix/libstore/platform/darwin.cc +++ b/lix/libstore/platform/darwin.cc @@ -16,11 +16,6 @@ #include #include -/* This definition is undocumented but depended upon by all major browsers. */ -extern "C" int sandbox_init_with_parameters( - const char * profile, uint64_t flags, const char * const parameters[], char ** errorbuf -); - namespace nix { kj::Promise> DarwinLocalStore::findPlatformRoots(UncheckedRoots & unchecked) @@ -380,73 +375,6 @@ void DarwinLocalDerivationGoal::fillBuilderConfig(build::Request::Builder reques RPC_FILL(config, setGlobalTempDir, canonPath(defaultTempDir(), true)); } -void DarwinLocalDerivationGoal::finishChildSetup(build::Request::Reader request) -{ - auto config = request.getPlatform().getDarwin(); - - /* The tmpDir in scope points at the temporary build directory for our derivation. Some packages try - different mechanisms to find temporary directories, so we want to open up a broader place for them - to put their files, if needed. */ - auto globalTmpDir = rpc::to(config.getGlobalTempDir()); - - /* They don't like trailing slashes on subpath directives */ - if (globalTmpDir.back() == '/') { - globalTmpDir.pop_back(); - } - - if (auto env = getenv("_NIX_TEST_NO_SANDBOX"); env && env != std::string_view("1")) { - std::vector sandboxArgs; - sandboxArgs.push_back("_NIX_BUILD_TOP"); - sandboxArgs.push_back(config.getTempDir().cStr()); - sandboxArgs.push_back("_GLOBAL_TMP_DIR"); - sandboxArgs.push_back(globalTmpDir.c_str()); - if (config.getAllowLocalNetworking()) { - sandboxArgs.push_back("_ALLOW_LOCAL_NETWORKING"); - sandboxArgs.push_back("1"); - } - sandboxArgs.push_back(nullptr); - // NOLINTNEXTLINE(lix-unsafe-c-calls): all of these are env names or paths - if (sandbox_init_with_parameters(config.getSandboxProfile().cStr(), 0, sandboxArgs.data(), nullptr)) { - writeFull(STDERR_FILENO, "failed to configure sandbox\n"); - _exit(1); - } - } -} - -void DarwinLocalDerivationGoal::execBuilder(build::Request::Reader request) -{ - auto config = request.getPlatform().getDarwin(); - - posix_spawnattr_t attrp; - - if (posix_spawnattr_init(&attrp)) { - throw SysError("failed to initialize builder"); - } - - if (posix_spawnattr_setflags(&attrp, POSIX_SPAWN_SETEXEC)) { - throw SysError("failed to initialize builder"); - } - - const auto platform = rpc::to(config.getPlatform()); - - if (platform == "aarch64-darwin") { - // Unset kern.curproc_arch_affinity so we can escape Rosetta - int affinity = 0; - sysctlbyname("kern.curproc_arch_affinity", nullptr, nullptr, &affinity, sizeof(affinity)); - - cpu_type_t cpu = CPU_TYPE_ARM64; - posix_spawnattr_setbinpref_np(&attrp, 1, &cpu, nullptr); - } else if (platform == "x86_64-darwin") { - cpu_type_t cpu = CPU_TYPE_X86_64; - posix_spawnattr_setbinpref_np(&attrp, 1, &cpu, nullptr); - } - - auto builder = rpc::to(request.getBuilder()); - auto args = rpc::to(request.getArgs()); - auto envStrs = rpc::to(request.getEnvironment()); - posix_spawn(nullptr, builder.c_str(), nullptr, &attrp, stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data()); -} - void registerLocalStore() { StoreImplementations::add(); } diff --git a/lix/libstore/platform/darwin.hh b/lix/libstore/platform/darwin.hh index 16a411ba7..80ddd056c 100644 --- a/lix/libstore/platform/darwin.hh +++ b/lix/libstore/platform/darwin.hh @@ -45,13 +45,6 @@ private: void fillBuilderConfig(build::Request::Builder config) override; - void finishChildSetup(build::Request::Reader request) override; - - /** - * Set process flags to enter or leave rosetta, then execute the builder - */ - void execBuilder(build::Request::Reader request) override; - /** * Whether we need to rewrite output hashes. * Always true on Darwin since Darwin requires hash rewriting diff --git a/lix/libstore/platform/linux.cc b/lix/libstore/platform/linux.cc index 4f1281aab..bcecb4c97 100644 --- a/lix/libstore/platform/linux.cc +++ b/lix/libstore/platform/linux.cc @@ -1,3 +1,4 @@ +#include "config.h" #include "lix/libstore/build/personality.hh" #include "lix/libstore/build/request.capnp.h" #include "lix/libstore/build/worker.hh" @@ -21,8 +22,10 @@ #include #include +#include #include #include +#include #include #include #include @@ -31,6 +34,7 @@ #include #include #include +#include #include #include #include @@ -1242,355 +1246,6 @@ std::string LinuxLocalDerivationGoal::rewriteResolvConf(std::string fromHost) return std::regex_replace(fromHost, lineRegex, "") + nsInSandbox; } -static void bindPath(const fs::path & source, const fs::path & target, bool optional = false) -{ - debug("bind mounting '%1%' to '%2%'", source, target); - - auto bindMount = [&]() { - if (mount(source.c_str(), target.c_str(), "", MS_BIND | MS_REC, 0) == -1) { - throw SysError("bind mount from %1% to %2% failed", source, target); - } - }; - - auto st = fs::symlink_status(source); - if (st.type() == fs::file_type::not_found) { - if (optional) { - return; - } else { - throw SysError("getting attributes of path %1%", source); - } - } - - if (st.type() == fs::file_type::directory) { - fs::create_directories(target); - bindMount(); - } else if (st.type() == fs::file_type::symlink) { - // Symlinks can (apparently) not be bind-mounted, so just copy it - fs::create_directories(target.parent_path()); - fs::copy_symlink(source, target); - } else { - fs::create_directories(target.parent_path()); - if (kj::AutoCloseFd file{open(target.c_str(), O_RDWR | O_CREAT, 0644)}; file == nullptr) { - throw SysError("could not create %s", target); - } - bindMount(); - } -} - -static bool prepareChildSetup_(build::Request::Reader request) -{ - auto config = request.getPlatform().getLinux(); - - // Set the NO_NEW_PRIVS prctl flag. - // This both makes loading seccomp filters work for unprivileged users, - // and is an additional security measure in its own right. - if (prctl(PR_SET_NO_NEW_PRIVS, 1L, 0L, 0L, 0L) == -1) { - throw SysError("PR_SET_NO_NEW_PRIVS failed"); - } -#if HAVE_SECCOMP - if (config.hasSeccompFilters()) { - const auto seccompBPF = config.getSeccompFilters(); - const auto entries = seccompBPF.size() / sizeof(struct sock_filter); - assert(entries <= std::numeric_limits::max()); - struct sock_fprog fprog = { - .len = static_cast(entries), - // the kernel does not actually write to the filter, and doesn't care about alignment - .filter = const_cast( - reinterpret_cast(seccompBPF.begin()) - ), - }; - if (syscall(SYS_seccomp, SECCOMP_SET_MODE_FILTER, 0, &fprog) != 0) { - throw SysError("unable to load seccomp BPF program"); - } - } -#endif - - KJ_DEFER(setPersonality(rpc::to(config.getPlatform()))); - - if (!config.hasSandbox()) { - return true; - } - - auto sandbox = config.getSandbox(); - - // NOLINTBEGIN(lix-unsafe-c-calls): we trust the parent that all sandbox config is correct. - // no strings in the linux sandbox config can be set by normal users or derivation authors, - // except (in single-user instances) storeDir and chrootRootDir, which must be valid paths. - // - // NOLINTBEGIN(lix-foreign-exceptions): they're all properly caught by the builder main fn. - - const fs::path chrootRootDir{rpc::to(sandbox.getChrootRootDir())}; - - if (sandbox.getPrivateNetwork()) { - /* Initialise the loopback interface. */ - kj::AutoCloseFd fd(socket(PF_INET, SOCK_DGRAM, IPPROTO_IP)); - if (fd == nullptr) { - throw SysError("cannot open IP socket"); - } - - struct ifreq ifr; - strcpy(ifr.ifr_name, "lo"); - ifr.ifr_flags = IFF_UP | IFF_LOOPBACK | IFF_RUNNING; - if (ioctl(fd.get(), SIOCSIFFLAGS, &ifr) == -1) { - throw SysError("cannot set loopback interface flags"); - } - } - - /* Set the hostname etc. to fixed values. */ - char hostname[] = "localhost"; - if (sethostname(hostname, sizeof(hostname)) == -1) { - throw SysError("cannot set host name"); - } - char domainname[] = "(none)"; // kernel default - if (setdomainname(domainname, sizeof(domainname)) == -1) { - throw SysError("cannot set domain name"); - } - - /* Make all filesystems private. This is necessary - because subtrees may have been mounted as "shared" - (MS_SHARED). (Systemd does this, for instance.) Even - though we have a private mount namespace, mounting - filesystems on top of a shared subtree still propagates - outside of the namespace. Making a subtree private is - local to the namespace, though, so setting MS_PRIVATE - does not affect the outside world. */ - const fs::path storeDir{rpc::to(sandbox.getStoreDir())}; - const auto chrootStoreDir = chrootRootDir / storeDir.relative_path(); - - if (mount(0, "/", 0, MS_PRIVATE | MS_REC, 0) == -1) { - throw SysError("unable to make '/' private"); - } - - /* Bind-mount chroot directory to itself, to treat it as a - different filesystem from /, as needed for pivot_root. */ - if (mount(chrootRootDir.c_str(), chrootRootDir.c_str(), "", MS_BIND, 0) == -1) { - throw SysError("unable to bind mount %1%", chrootRootDir); - } - - /* Bind-mount the sandbox's Nix store onto itself so that - we can mark it as a "shared" subtree, allowing bind - mounts made in *this* mount namespace to be propagated - into the child namespace created by the - unshare(CLONE_NEWNS) call below. - - Marking chrootRootDir as MS_SHARED causes pivot_root() - to fail with EINVAL. Don't know why. */ - if (mount(chrootStoreDir.c_str(), chrootStoreDir.c_str(), "", MS_BIND, 0) == -1) { - throw SysError("unable to bind mount the Nix store"); - } - - if (mount("", chrootStoreDir.c_str(), "", MS_SHARED, 0) == -1) { - throw SysError("unable to make %s shared", chrootStoreDir); - } - - bool devMounted = false; - bool devPtsMounted = false; - - /* Bind-mount all the directories from the "host" - filesystem that we want in the chroot - environment. */ - for (auto path : sandbox.getPaths()) { - const fs::path source{rpc::to(path.getSource())}; - const fs::path target{rpc::to(path.getTarget())}; - devMounted |= target == "/dev"; - devPtsMounted |= target == "/dev/pts"; - if (source == "/proc") { - continue; // backwards compatibility - } - -#if HAVE_EMBEDDED_SANDBOX_SHELL - if (source == "__embedded_sandbox_shell__") { - static unsigned char sh[] = { -#include "embedded-sandbox-shell.gen.hh" - }; - const fs::path dst = chrootRootDir / target.relative_path(); - fs::create_directories(dst.parent_path()); - writeFile(dst, std::string_view((const char *) sh, sizeof(sh))); - fs::permissions(dst, fs::perms(0555)); - } else -#endif - bindPath(source, chrootRootDir / target.relative_path(), path.getOptional()); - } - - /* Set up a nearly empty /dev, unless the user asked to - bind-mount the host /dev. */ - if (!devMounted) { - const auto bind = [&](fs::path item) { bindPath(item, chrootRootDir / item.relative_path()); }; - - fs::create_directories(chrootRootDir / "dev/shm"); - fs::create_directories(chrootRootDir / "dev/pts"); - bind("/dev/full"); - if (sandbox.getWantsKvm() && pathExists("/dev/kvm")) { - bind("/dev/kvm"); - } - bind("/dev/null"); - bind("/dev/random"); - bind("/dev/tty"); - bind("/dev/urandom"); - bind("/dev/zero"); - fs::create_symlink("/proc/self/fd", chrootRootDir / "dev/fd"); - fs::create_symlink("/proc/self/fd/0", chrootRootDir / "dev/stdin"); - fs::create_symlink("/proc/self/fd/1", chrootRootDir / "dev/stdout"); - fs::create_symlink("/proc/self/fd/2", chrootRootDir / "dev/stderr"); - } - - /* Bind a new instance of procfs on /proc. */ - fs::create_directories(chrootRootDir / "proc"); - if (mount("none", (chrootRootDir / "proc").c_str(), "proc", 0, 0) == -1) { - throw SysError("mounting /proc"); - } - - /* Mount sysfs on /sys. */ - if (request.hasCredentials() && request.getCredentials().getUidCount() != 1) { - fs::create_directories(chrootRootDir / "sys"); - if (mount("none", (chrootRootDir / "sys").c_str(), "sysfs", 0, 0) == -1) { - throw SysError("mounting /sys"); - } - } - - /* Mount a new tmpfs on /dev/shm to ensure that whatever - the builder puts in /dev/shm is cleaned up automatically. */ - if (pathExists("/dev/shm") - && mount("none", (chrootRootDir / "dev/shm").c_str(), "tmpfs", 0, sandbox.getSandboxShmFlags().cStr()) - == -1) - { - throw SysError("mounting /dev/shm"); - } - - /* Mount a new devpts on /dev/pts. Note that this - requires the kernel to be compiled with - CONFIG_DEVPTS_MULTIPLE_INSTANCES=y (which is the case - if /dev/ptx/ptmx exists). */ - if (pathExists("/dev/pts/ptmx") && !pathExists(chrootRootDir / "dev/ptmx") && !devPtsMounted) { - if (mount("none", (chrootRootDir / "dev/pts").c_str(), "devpts", 0, "newinstance,mode=0620") == 0) { - fs::create_symlink("/dev/pts/ptmx", chrootRootDir / "dev/ptmx"); - - /* Make sure /dev/pts/ptmx is world-writable. With some - Linux versions, it is created with permissions 0. */ - fs::permissions(chrootRootDir / "dev/pts/ptmx", fs::perms(0666)); - } else { - if (errno != EINVAL) { - throw SysError("mounting /dev/pts"); - } - bindPath("/dev/pts", chrootRootDir / "dev/pts"); - bindPath("/dev/ptmx", chrootRootDir / "dev/ptmx"); - } - } - - /* Make /etc unwritable */ - if (!sandbox.getUseUidRange()) { - fs::permissions(chrootRootDir / "etc", fs::perms(0555)); - } - - /* The comment below is now outdated. Recursive Nix has been removed. - * So there's no need to make path appear in the sandbox. - * TODO(Raito): cleanup before a merge. - */ - /* Unshare this mount namespace. This is necessary because - pivot_root() below changes the root of the mount - namespace. This means that the call to setns() in - addDependency() would hide the host's filesystem, - making it impossible to bind-mount paths from the host - Nix store into the sandbox. Therefore, we save the - pre-pivot_root namespace in - sandboxMountNamespace. Since we made /nix/store a - shared subtree above, this allows addDependency() to - make paths appear in the sandbox. */ - if (unshare(CLONE_NEWNS) == -1) { - throw SysError("unsharing mount namespace"); - } - - /* Creating a new cgroup namespace is independent of whether we enabled the cgroup experimental feature. - * We always create a new cgroup namespace from a sandboxing perspective. */ - /* Unshare the cgroup namespace. This means - /proc/self/cgroup will show the child's cgroup as '/' - rather than whatever it is in the parent. */ - if (unshare(CLONE_NEWCGROUP) == -1) { - throw SysError("unsharing cgroup namespace"); - } - - /* Do the chroot(). */ - if (chdir(chrootRootDir.c_str()) == -1) { - throw SysError("cannot change directory to %1%", chrootRootDir); - } - - if (mkdir("real-root", 0) == -1) { - throw SysError("cannot create real-root directory"); - } - - if (syscall(SYS_pivot_root, ".", "real-root") == -1) { - throw SysError("cannot pivot old root directory onto %1%", chrootRootDir / "real-root"); - } - - if (chroot(".") == -1) { - throw SysError("cannot change root directory to %1%", chrootRootDir); - } - - if (umount2("real-root", MNT_DETACH) == -1) { - throw SysError("cannot unmount real root filesystem"); - } - - if (rmdir("real-root") == -1) { - throw SysError("cannot remove real-root directory"); - } - - /* Switch to the sandbox uid/gid in the user namespace, - which corresponds to the build user or calling user in - the parent namespace. */ - if (setgid(sandbox.getGid()) == -1) { - throw SysError("setgid failed"); - } - if (setuid(sandbox.getUid()) == -1) { - throw SysError("setuid failed"); - } - - if (sandbox.hasWaitForInterface()) { - // wait for the pasta interface to appear. pasta can't signal us when - // it's done setting up the namespace, so we have to wait for a while - kj::AutoCloseFd fd(socket(PF_INET, SOCK_DGRAM, IPPROTO_IP)); - if (fd == nullptr) { - throw SysError("cannot open IP socket"); - } - - struct ifreq ifr; - strncpy(ifr.ifr_name, sandbox.getWaitForInterface().cStr(), sizeof(ifr.ifr_name)); - // wait two minutes for the interface to appear. if it does not do so - // we are either grossly overloaded, or pasta startup failed somehow. - static constexpr int SINGLE_WAIT_US = 1000; - static constexpr int TOTAL_WAIT_US = 120'000'000; - for (unsigned tries = 0;; tries++) { - if (tries > TOTAL_WAIT_US / SINGLE_WAIT_US) { - throw std::runtime_error( - "sandbox network setup timed out, please check daemon logs for possible error output." - ); - } else if (ioctl(fd.get(), SIOCGIFFLAGS, &ifr) == 0) { - if ((ifr.ifr_ifru.ifru_flags & IFF_UP) != 0) { - break; - } - } else if (errno == ENODEV) { - usleep(SINGLE_WAIT_US); - } else { - throw SysError("cannot get loopback interface flags"); - } - } - } - - // NOLINTEND(lix-foreign-exceptions) - // NOLINTEND(lix-unsafe-c-calls) - - return false; -} - -void LinuxLocalDerivationGoal::finishChildSetup(build::Request::Reader request) -{ - if (prctl(PR_SET_PDEATHSIG, SIGKILL) == -1) { - throw SysError("setting death signal"); - } - if (getppid() != request.getPlatform().getLinux().getParentPid()) { - raise(SIGKILL); - } -} - void LinuxLocalDerivationGoal::fillBuilderConfig(build::Request::Builder request) { auto config = request.getPlatform().getLinux(); @@ -1631,16 +1286,11 @@ void LinuxLocalDerivationGoal::fillBuilderConfig(build::Request::Builder request config.setParentPid(getpid()); } -bool LinuxLocalDerivationGoal::prepareChildSetup(build::Request::Reader request) -{ - return prepareChildSetup_(request); -} - -Pid LinuxLocalDerivationGoal::startChild(build::Request::Reader request, AutoCloseFD logPTY) +Pid LinuxLocalDerivationGoal::startChild(AutoCloseFD setupFD, AutoCloseFD logPTY) { // If we're not sandboxing no need to faff about, use the fallback if (!useChroot) { - return LocalDerivationGoal::startChild(request, std::move(logPTY)); + return LocalDerivationGoal::startChild(std::move(setupFD), std::move(logPTY)); } /* Set up private namespaces for the build: @@ -1776,6 +1426,10 @@ Pid LinuxLocalDerivationGoal::startChild(build::Request::Reader request, AutoClo } return inVFork(/* flags*/ 0, [&]() { + if (dup2(setupFD.get(), STDIN_FILENO) == -1) { + throw SysError("failed to hook up setup fd"); + } + closeOnExec(STDIN_FILENO, false); if (dup2(logPTY.get(), STDERR_FILENO) == -1) { throw SysError("failed to redirect build output to log file"); } @@ -1806,11 +1460,22 @@ Pid LinuxLocalDerivationGoal::startChild(build::Request::Reader request, AutoClo throw SysError("setns(netNS)"); } - ProcessOptions options; - options.cloneFlags = - CLONE_NEWPID | CLONE_NEWNS | CLONE_NEWIPC | CLONE_NEWUTS | CLONE_PARENT | SIGCHLD; + CloneStack stack; + return inClone( + stack, + CLONE_NEWPID | CLONE_NEWNS | CLONE_NEWIPC | CLONE_NEWUTS | CLONE_PARENT | CLONE_VM | CLONE_FILES + | CLONE_VFORK, + [&]() -> int { + if (userns) { + std::array caps; + std::iota(caps.begin(), caps.end(), 0); + raiseAmbientCaps(caps); + } - return startProcess([&]() { runChild(request); }, options); + execl(LIX_LIBEXEC_DIR "/launch-builder", "launch-builder", nullptr); + throw SysError("exec failed"); + } + ); }); } diff --git a/lix/libstore/platform/linux.hh b/lix/libstore/platform/linux.hh index 5acea7eaf..6550bf3b1 100644 --- a/lix/libstore/platform/linux.hh +++ b/lix/libstore/platform/linux.hh @@ -76,7 +76,7 @@ private: * Start child process in new namespaces, * create /etc/passwd and /etc/group based on discovered uid/gid */ - Pid startChild(build::Request::Reader request, AutoCloseFD logPTY) override; + Pid startChild(AutoCloseFD setupFD, AutoCloseFD logPTY) override; /** * Kill all processes by build user. @@ -88,10 +88,6 @@ private: return true; } - bool prepareChildSetup(build::Request::Reader request) override; - - void finishChildSetup(build::Request::Reader request) override; - std::string rewriteResolvConf(std::string fromHost); /** diff --git a/lix/libutil/processes.cc b/lix/libutil/processes.cc index 595bb2a59..f66b98997 100644 --- a/lix/libutil/processes.cc +++ b/lix/libutil/processes.cc @@ -380,7 +380,9 @@ RunningProgram runProgram2(const RunOptions & options) Strings args_(options.args); args_.push_front(options.argv0.value_or(options.program)); - restoreProcessContext(); + if (!options.keepContext) { + restoreProcessContext(); + } if (options.searchPath) { sys::execvp(options.program, args_); diff --git a/lix/libutil/processes.hh b/lix/libutil/processes.hh index b37b07d06..a4808cd06 100644 --- a/lix/libutil/processes.hh +++ b/lix/libutil/processes.hh @@ -110,6 +110,7 @@ struct RunOptions std::optional> environment = {}; bool captureStdout = false; std::vector redirections; + bool keepContext = false; }; struct [[nodiscard("you must call RunningProgram::wait()")]] RunningProgram