lix/libstore/linux: rename cgroup with drvHash

Change-Id: I238d0568a3e4b1ff3057781c0639528d666b4d37
Signed-off-by: Raito Bezarius <raito@lix.systems>

doc/manual/rl-next/socket-activation.md

@@ -0,0 +1,15 @@
+---
+synopsis: "Lix daemons are now fully socket-activated on systemd setups"
+cls: []
+issues: [1030]
+category: "Miscellany"
+credits: [horrors]
+---
+
+When launched by systemd, Lix no longer uses a persistent daemon process and uses systemd socket
+activation instead. This is necessary to support the `cgroups` and `auto-allocate-uids` features
+and may improve observability of daemon behavior with common systemd-based monitoring solutions.
+
+The old behavior with a single persistent daemon is still available, but disabled by default. It
+is not possible to enable both a persistent daemon and socket activation, starting one stops the
+other automatically. Existing installations should not require any changes when they're updated.

lix/libcmd/command.hh

@@ -15,8 +15,6 @@ namespace nix {
 
 extern std::string programPath;
 
-extern char * * savedArgv;
-
 class EvalState;
 struct Pos;
 class Store;

lix/libcmd/legacy.hh

@@ -9,14 +9,27 @@
 
 namespace nix {
 
-typedef std::function<int(AsyncIoRoot &, std::string, std::list<std::string>)> MainFunction;
-
 struct LegacyCommandRegistry
 {
-    using LegacyCommandMap = std::map<std::string, MainFunction>;
+    typedef std::function<int(AsyncIoRoot &, std::string, std::list<std::string>)> MainFunction;
+    typedef std::function<
+        int(AsyncIoRoot &, std::string, std::list<std::string>, std::span<char *>)>
+        RawMainFunction;
+
+    using LegacyCommandMap = std::map<std::string, RawMainFunction>;
     static LegacyCommandMap * commands;
 
     static void add(const std::string & name, MainFunction fun)
+    {
+        addWithRaw(
+            name,
+            [fun](AsyncIoRoot & aio, std::string name, std::list<std::string> args, std::span<char *>) {
+                return fun(aio, name, args);
+            }
+        );
+    }
+
+    static void addWithRaw(const std::string & name, RawMainFunction fun)
     {
         if (!commands) commands = new LegacyCommandMap;
         (*commands)[name] = fun;

lix/libmain/shared.cc

@@ -33,8 +33,6 @@
 
 namespace nix {
 
-char * * savedArgv;
-
 static bool gcWarning = true;
 
 void printGCWarning()

lix/libstore/platform/linux.cc

@@ -858,7 +858,7 @@ void LinuxLocalDerivationGoal::prepareSandbox()
     if (buildUser && (buildUser->getUIDCount() != 1 || settings.useCgroups)) {
         context.cgroup.emplace(
             settings.nixStateDir + "/cgroups",
-            fmt("nix-build-uid-%d", buildUser->getUID()),
+            fmt("nix-build-drvHash-%s", drvPath.hashPart()),
             buildUser->getUID(),
             buildUser->getGID()
         );

lix/nix/daemon.cc

@@ -55,7 +55,6 @@
 #endif
 
 static constexpr int SUBDAEMON_CONNECTION_FD = 0;
-static constexpr int SUBDAEMON_SETTINGS_FD = 3;
 
 namespace nix {
 
@@ -346,9 +345,6 @@ try {
                 peer.pidKnown ? fmt("pid %1%", peer.pid) : "unknown peer"
             );
 
-            Pipe settings;
-            settings.create();
-
             // Fork a child to handle the connection. make sure it's called with
             // argv0 `nix-daemon` so we don't try to run `nix --for` when called
             // from more modern scripts that assume nix-command being available.
@@ -363,11 +359,7 @@ try {
                         fmt("%1%", int(verbosity)),
                     },
                 .dieWithParent = false,
-                .redirections =
+                .redirections = {{.dup = SUBDAEMON_CONNECTION_FD, .from = remote.get()}}
-                    {
-                        {.dup = SUBDAEMON_CONNECTION_FD, .from = remote.get()},
-                        {.dup = SUBDAEMON_SETTINGS_FD, .from = settings.readSide.get()},
-                    }
             };
             if (forceTrustClientOpt) {
                 options.args.push_back(
@@ -375,15 +367,6 @@ try {
                 );
             }
             runProgram2(options).release();
-
-            FdSink sink(settings.writeSide.get());
-            std::map<std::string, Config::SettingInfo> overriddenSettings;
-            globalConfig.getSettings(overriddenSettings, true);
-            for (auto & setting : overriddenSettings) {
-                sink << 1 << setting.first << setting.second.value;
-            }
-            sink << 0;
-            sink.flush();
         } catch (Error & error) {
             auto ei = error.info();
             // FIXME: add to trace?
@@ -397,12 +380,37 @@ try {
     co_return result::current_exception();
 }
 
-static void daemonInstance(AsyncIoRoot & aio, std::optional<TrustedFlag> forceTrustClientOpt)
+static void
+daemonInstance(AsyncIoRoot & aio, std::optional<TrustedFlag> forceTrustClientOpt, char * peerPidArg)
 {
-    PeerInfo peer = getPeerInfo(SUBDAEMON_CONNECTION_FD);
+    //  Handle socket-based activation by systemd.
+    const auto [launchedByManager, connectionFd] = []() -> std::pair<bool, int> {
+        auto listenFds = getEnv("LISTEN_FDS");
+        if (listenFds) {
+            if (getEnv("LISTEN_PID") != std::to_string(getpid()) || listenFds != "1") {
+                throw Error("unexpected systemd environment variables");
+            }
+            closeOnExec(SD_LISTEN_FDS_START);
+            return {true, SD_LISTEN_FDS_START};
+        } else {
+            return {false, SUBDAEMON_CONNECTION_FD};
+        }
+    }();
+
+    PeerInfo peer = getPeerInfo(connectionFd);
     TrustedFlag trusted;
     std::string user;
 
+    // replace peerPidArg contents with the peer pid if possible. the forking daemon does
+    // this as a debugging aid and it is easy enough to do it here also, so we just do it
+    if (peerPidArg && peer.pidKnown) {
+        auto pidForArgv = std::to_string(peer.pid);
+        if (pidForArgv.size() <= strlen(peerPidArg)) {
+            memset(peerPidArg, ' ', strlen(peerPidArg));
+            strcpy(peerPidArg, pidForArgv.c_str());
+        }
+    }
+
     if (forceTrustClientOpt) {
         trusted = *forceTrustClientOpt;
     } else {
@@ -417,23 +425,8 @@ static void daemonInstance(AsyncIoRoot & aio, std::optional<TrustedFlag> forceTr
         forceTrustClientOpt ? " by override" : ""
     );
 
-    {
-        FdSource source(SUBDAEMON_SETTINGS_FD);
-
-        /* Read the parent's settings. */
-        while (readNum<unsigned>(source)) {
-            auto name = readString(source);
-            auto value = readString(source);
-            settings.set(name, value);
-        }
-
-        if (close(SUBDAEMON_SETTINGS_FD) < 0) {
-            throw SysError("preparing subdaemon connection");
-        }
-    }
-
     //  Background the daemon.
-    if (setsid() == -1) {
+    if (!launchedByManager && setsid() == -1) {
         throw SysError("creating a new session");
     }
 
@@ -446,8 +439,8 @@ static void daemonInstance(AsyncIoRoot & aio, std::optional<TrustedFlag> forceTr
     }
 
     //  Handle the connection.
-    FdSource from(SUBDAEMON_CONNECTION_FD);
+    FdSource from(connectionFd);
-    FdSink to(SUBDAEMON_CONNECTION_FD);
+    FdSink to(connectionFd);
     processConnection(aio, store, from, to, trusted);
 }
 
@@ -525,12 +518,14 @@ runDaemon(AsyncIoRoot & aio, bool stdio, std::optional<TrustedFlag> forceTrustCl
     }
 }
 
-static int main_nix_daemon(AsyncIoRoot & aio, std::string programName, Strings argv)
+static int
+main_nix_daemon(AsyncIoRoot & aio, std::string programName, Strings argv, std::span<char *> rawArgv)
 {
     {
         auto stdio = false;
         std::optional<TrustedFlag> isTrustedOpt = std::nullopt;
         bool isInstance = false;
+        char * peerPidArg = nullptr;
         Verbosity subdaemonLogLevel = lvlInfo;
 
         LegacyArgs(aio, programName, [&](Strings::iterator & arg, const Strings::iterator & end) {
@@ -554,6 +549,18 @@ static int main_nix_daemon(AsyncIoRoot & aio, std::string programName, Strings a
             } else if (*arg == "--for") {
                 isInstance = true;
                 getArg(*arg, arg, end);
+            } else if (*arg == "--for-socket-activation") {
+                isInstance = true;
+                // HACK: too many copies and rewrites happen by the time we get here to
+                // be able to calculate a rawArgv offset. instead we will search for an
+                // exact match and blindly assume that it's the one we want to rewrite.
+                for (auto [i, rawArg] : enumerate(rawArgv)) {
+                    if (rawArg == *arg) {
+                        peerPidArg = rawArg + strlen("--for-");
+                        peerPidArg[-1] = ' ';
+                        break;
+                    }
+                }
             } else if (*arg == "--log-level") {
                 if (auto level = string2Int<int>(getArg(*arg, arg, end)); level) {
                     subdaemonLogLevel = static_cast<Verbosity>(std::min<int>(lvlVomit, *level));
@@ -568,7 +575,7 @@ static int main_nix_daemon(AsyncIoRoot & aio, std::string programName, Strings a
 
         if (isInstance) {
             verbosity = Verbosity(std::min<uint64_t>(subdaemonLogLevel, lvlVomit));
-            daemonInstance(aio, isTrustedOpt);
+            daemonInstance(aio, isTrustedOpt, peerPidArg);
         } else {
             runDaemon(aio, stdio, isTrustedOpt);
         }
@@ -578,7 +585,7 @@ static int main_nix_daemon(AsyncIoRoot & aio, std::string programName, Strings a
 }
 
 void registerLegacyNixDaemon() {
-    LegacyCommandRegistry::add("nix-daemon", main_nix_daemon);
+    LegacyCommandRegistry::addWithRaw("nix-daemon", main_nix_daemon);
 }
 
 struct CmdDaemon : StoreCommand

lix/nix/main.cc

@@ -455,8 +455,6 @@ void registerNixHelp()
 
 int mainWrapped(AsyncIoRoot & aio, int argc, char ** argv)
 {
-    savedArgv = argv;
-
     /* The chroot helper needs to be run before any threads have been
        started. */
     if (argc > 0 && argv[0] == chrootHelperName) {
@@ -493,7 +491,12 @@ int mainWrapped(AsyncIoRoot & aio, int argc, char ** argv)
         registerLegacyCommands();
         auto legacy = (*LegacyCommandRegistry::commands)[programName];
         if (legacy) {
-            return legacy(aio, std::string(baseNameOf(argv[0])), Strings(argv + 1, argv + argc));
+            return legacy(
+                aio,
+                std::string(baseNameOf(argv[0])),
+                Strings(argv + 1, argv + argc),
+                {argv + 1, argv + argc}
+            );
         }
     }
 
@@ -645,7 +648,6 @@ int mainWrapped(AsyncIoRoot & aio, int argc, char ** argv)
 
     return 0;
 }
-
 }
 
 int main(int argc, char * * argv)

misc/systemd/meson.build

@@ -1,4 +1,4 @@
-foreach config : [ 'nix-daemon.socket', 'nix-daemon.service' ]
+foreach config : [ 'nix-daemon.socket', 'nix-daemon.service', 'nix-daemon@.service' ]
   configure_file(
     input : config + '.in',
     output : config,

misc/systemd/nix-daemon.service.in

@@ -1,6 +1,7 @@
 [Unit]
 Description=Nix Daemon
 Documentation=man:nix-daemon https://docs.lix.systems/manual/lix/stable
+Conflicts=nix-daemon.socket
 RequiresMountsFor=@storedir@
 RequiresMountsFor=@localstatedir@
 RequiresMountsFor=@localstatedir@/nix/db
@@ -14,6 +15,3 @@ LimitNOFILE=1048576
 TasksMax=1048576
 Delegate=yes
 DelegateSubgroup=supervisor
-
-[Install]
-WantedBy=multi-user.target

misc/systemd/nix-daemon.socket.in

@@ -1,11 +1,13 @@
 [Unit]
 Description=Nix Daemon Socket
 Before=multi-user.target
+Conflicts=nix-daemon.service
 RequiresMountsFor=@storedir@
 ConditionPathIsReadWrite=@localstatedir@/nix/daemon-socket
 
 [Socket]
 ListenStream=@localstatedir@/nix/daemon-socket/socket
+Accept=yes
 
 [Install]
 WantedBy=sockets.target

misc/systemd/nix-daemon@.service.in

@@ -0,0 +1,15 @@
+[Unit]
+Description=Nix Daemon instance
+Documentation=man:nix-daemon https://docs.lix.systems/manual/lix/stable
+CollectMode=inactive-or-failed
+RequiresMountsFor=@storedir@
+RequiresMountsFor=@localstatedir@
+RequiresMountsFor=@localstatedir@/nix/db
+
+[Service]
+ExecStart=@@bindir@/nix-daemon nix-daemon --for-socket-activation
+CacheDirectory=nix
+LimitNOFILE=1048576
+TasksMax=1048576
+Delegate=yes
+DelegateSubgroup=supervisor

tests/nixos/default.nix

@@ -9,11 +9,13 @@ let
     (nixos-lib.runTest {
       imports = [ test ];
       hostPkgs = nixpkgsFor.${system}.native;
-      defaults = {
+      defaults = { config, ... }: {
         nixpkgs.pkgs = nixpkgsFor.${system}.native;
         nix.checkAllErrors = false;
         # nixos-option fails to build with lix and no tests use any of the tools
         system.disableInstallerTools = true;
+        # FIXME: remove this once the nixos module sets these overrides
+        systemd.services."nix-daemon@".path = config.systemd.services.nix-daemon.path;
       };
       _module.args.nixpkgs = nixpkgs;
       _module.args.system = system;