libstore: allow for multiple daemon sockets with preference

this will let us configure more than one socket to connect/bind to,
which in turn lets us use posix acls on sockets for access control.
we will also need something like this for the final rpc transition.

Change-Id: I9c39f14906e9bf809055ab5c94bf687745b4f69e

doc/manual/src/contributing/testing.md

@@ -383,7 +383,10 @@ I grepped `lix/` for `get[eE]nv\("` to find the mentions in Lix code.
   Overrides compile-time configuration of various locations used by Lix. See `lix/libstore/globals.cc`.
 
   **Expected value**: a directory
-- `NIX_DAEMON_SOCKET_PATH` (optional) - Overrides the daemon socket path from `$NIX_STATE_DIR/daemon-socket/socket`.
+- `LIX_DAEMON_SOCKET_DIR` (optional) - Overrides the daemon socket directory from `$NIX_STATE_DIR/daemon-socket`.
+
+  **Expected value**: a directory
+- `NIX_DAEMON_SOCKET_PATH` (optional) - Overrides the daemon socket path from `$NIX_STATE_DIR/daemon-socket/socket`. Ignored if `LIX_DAEMON_SOCKET_DIR` is set.
 
   **Expected value**: path to a socket
 - `NIX_LOG_FD` (output) - An FD number for logs in `internal-json` format to be sent to.

lix/libstore/globals.cc

@@ -49,13 +49,16 @@ namespace nix {
    Nix daemon by setting the mode/ownership of the directory
    appropriately.  (This wouldn't work on the socket itself since it
    must be deleted and recreated on startup.) */
-#define DEFAULT_SOCKET_PATH "/daemon-socket/socket"
+#define DEFAULT_SOCKET_DIR "/daemon-socket"
+#define LEGACY_SOCKET "/socket"
 
 Settings settings;
 
 Settings::Settings()
     : nixPrefix(NIX_PREFIX)
-    , nixStore(canonPath(getEnvNonEmpty("NIX_STORE_DIR").value_or(getEnvNonEmpty("NIX_STORE").value_or(NIX_STORE_DIR))))
+    , nixStore(canonPath(
+          getEnvNonEmpty("NIX_STORE_DIR").value_or(getEnvNonEmpty("NIX_STORE").value_or(NIX_STORE_DIR))
+      ))
     , nixDataDir(canonPath(getEnvNonEmpty("NIX_DATA_DIR").value_or(NIX_DATA_DIR)))
     , nixLogDir(canonPath(getEnvNonEmpty("NIX_LOG_DIR").value_or(NIX_LOG_DIR)))
     , nixStateDir(canonPath(getEnvNonEmpty("NIX_STATE_DIR").value_or(NIX_STATE_DIR)))
@@ -63,8 +66,19 @@ Settings::Settings()
     , nixUserConfFiles(getUserConfigFiles())
     , nixBinDir(canonPath(getEnvNonEmpty("NIX_BIN_DIR").value_or(NIX_BIN_DIR)))
     , nixManDir(canonPath(NIX_MAN_DIR))
-    , nixDaemonSocketFile(canonPath(getEnvNonEmpty("NIX_DAEMON_SOCKET_PATH").value_or(nixStateDir + DEFAULT_SOCKET_PATH)))
 {
+    if (auto socketDirFromEnv = getEnvNonEmpty("LIX_DAEMON_SOCKET_DIR")) {
+        nixDaemonSockets_ = {{canonPath(*socketDirFromEnv + LEGACY_SOCKET)}};
+    } else if (auto socketPathFromEnv = getEnvNonEmpty("NIX_DAEMON_SOCKET_PATH")) {
+        nixDaemonSockets_ = {{canonPath(*socketPathFromEnv)}};
+    } else {
+        auto baseDir = nixStateDir + DEFAULT_SOCKET_DIR;
+        // this should always match the list of sockets created by daemonLoop and the socket units
+        nixDaemonSockets_ = {
+            {canonPath(baseDir + LEGACY_SOCKET)},
+        };
+    }
+
     buildUsersGroup.setDefault(getuid() == 0 ? "nixbld" : "");
     allowSymlinkedStore.setDefault(getEnv("NIX_IGNORE_SYMLINK_STORE") == "1");
 

lix/libstore/globals.hh

@@ -61,7 +61,16 @@ const uint32_t maxIdsPerBuild =
     #endif
     ;
 
-class Settings : public Config {
+class Settings : public Config
+{
+public:
+    struct DaemonSocketPath
+    {
+        Path path;
+    };
+
+private:
+    std::list<DaemonSocketPath> nixDaemonSockets_;
 
     unsigned int getDefaultCores();
 
@@ -117,9 +126,12 @@ public:
     Path nixManDir;
 
     /**
-     * File name of the socket the daemon listens to.
+     * Socket paths a client should connect to, in order of decreasing preference.
      */
-    Path nixDaemonSocketFile;
+    const std::list<DaemonSocketPath> & nixDaemonSockets() const
+    {
+        return nixDaemonSockets_;
+    }
 
     /**
      * Whether to show build log output in real time.
@@ -135,7 +147,6 @@ public:
 #include "lix/libstore/libstore-settings.gen.inc"
 };
 
-
 // FIXME: don't use a global variable.
 extern Settings settings;
 

lix/libstore/store-api.cc

@@ -27,6 +27,7 @@
 #include "lix/libstore/worker-protocol.hh"
 #include "lix/libutil/users.hh"
 
+#include <algorithm>
 #include <functional>
 #include <kj/async.h>
 #include <memory>
@@ -1422,7 +1423,11 @@ openFromNonUri(const std::string & uri, const StoreConfig::Params & params, Allo
 {
     if (uri == "" || uri == "auto") {
         auto stateDir = getOr(params, "state", settings.nixStateDir);
-        if (allowDaemon == AllowDaemon::Allow && pathExists(settings.nixDaemonSocketFile)) {
+        if (allowDaemon == AllowDaemon::Allow
+            && std::ranges::any_of(
+                settings.nixDaemonSockets(), [](auto & socket) { return pathExists(socket.path); }
+            ))
+        {
             return make_ref<UDSRemoteStore>(params);
         } else if (sys::access(stateDir, R_OK | W_OK) == 0) {
             return LocalStore::makeLocalStore(params);
@@ -1453,7 +1458,7 @@ openFromNonUri(const std::string & uri, const StoreConfig::Params & params, Allo
             // FIXME? this ignores *all* store parameters passed to this function?
             return LocalStore::makeLocalStore(chrootStoreParams);
         }
-        #endif
+#endif
         else
             return LocalStore::makeLocalStore(params);
     } else if (uri == "daemon") {

lix/libstore/uds-remote-store.cc

@@ -1,9 +1,15 @@
 #include "lix/libstore/uds-remote-store.hh"
+#include "globals.hh"
 #include "lix/libutil/async.hh"
+#include "lix/libutil/error.hh"
+#include "lix/libutil/file-descriptor.hh"
 #include "lix/libutil/result.hh"
+#include "lix/libutil/strings.hh"
 #include "lix/libutil/unix-domain-socket.hh"
 #include "lix/libstore/worker-protocol.hh"
 
+#include <cerrno>
+#include <ranges>
 #include <sys/types.h>
 #include <sys/stat.h>
 #include <sys/socket.h>
@@ -51,6 +57,23 @@ std::string UDSRemoteStore::getUri()
     }
 }
 
+static void connectToFirstAvailableSocket(AutoCloseFD & sockFD, const std::list<Path> & paths)
+{
+    for (const auto & socket : paths) {
+        try {
+            nix::connect(sockFD.get(), socket);
+            return;
+        } catch (SysError & e) {
+            if (e.errNo == EACCES || e.errNo == EPERM || e.errNo == ECONNREFUSED || e.errNo == ENOENT) {
+                debug("skipping socket %s: %s", socket, strerror(e.errNo));
+            } else {
+                throw;
+            }
+        }
+    }
+    throw Error("could not connect to any lix socket (tried %s)", concatStringsSep(", ", paths));
+}
+
 ref<RemoteStore::Connection> UDSRemoteStore::openConnection()
 {
     auto conn = make_ref<Connection>();
@@ -58,7 +81,17 @@ ref<RemoteStore::Connection> UDSRemoteStore::openConnection()
     /* Connect to a daemon that does the privileged work for us. */
     conn->fd = createUnixDomainSocket();
 
-    nix::connect(conn->fd.get(), path ? *path : settings.nixDaemonSocketFile);
+    std::list<Path> candidates;
+
+    if (path) {
+        candidates.emplace_back(*path);
+    } else {
+        candidates = settings.nixDaemonSockets()
+            | std::views::transform([](auto & socket) { return socket.path; })
+            | std::ranges::to<std::list<Path>>();
+    }
+
+    connectToFirstAvailableSocket(conn->fd, candidates);
 
     conn->startTime = std::chrono::steady_clock::now();
 

lix/nix/daemon.cc

@@ -6,6 +6,7 @@
 #include "lix/libstore/remote-store.hh"
 #include "lix/libstore/remote-store-connection.hh"
 #include "lix/libstore/store-api.hh"
+#include "lix/libutil/async-collect.hh"
 #include "lix/libutil/async-io.hh"
 #include "lix/libutil/async.hh"
 #include "lix/libutil/current-process.hh"
@@ -33,6 +34,7 @@
 #include <cstdint>
 #include <cstring>
 
+#include <kj/async.h>
 #include <string>
 #include <unistd.h>
 #include <signal.h>
@@ -276,34 +278,13 @@ static std::pair<TrustedFlag, std::string> authPeer(const PeerInfo & peer)
     return { trusted, std::move(user) };
 }
 
-
-/**
- * Run a server. The loop opens a socket and accepts new connections from that
- * socket.
- *
- * @param forceTrustClientOpt If present, force trusting or not trusted
- * the client. Otherwise, decide based on the authentication settings
- * and user credentials (from the unix domain socket).
- */
-static kj::Promise<Result<void>> daemonLoop(std::optional<TrustedFlag> forceTrustClientOpt)
+static kj::Promise<Result<void>> daemonLoopForSocket(
+    const Path & self,
+    const Settings::DaemonSocketPath & socket,
+    AutoCloseFD & fdSocket,
+    std::optional<TrustedFlag> forceTrustClientOpt
+)
 try {
-    if (chdir("/") == -1)
-        throw SysError("cannot change current directory");
-
-    createDirs(dirOf(settings.nixDaemonSocketFile));
-    auto fdSocket = createUnixDomainSocket(settings.nixDaemonSocketFile, 0666);
-
-    //  Get rid of children automatically; don't let them become zombies.
-    setSigChldAction(true);
-
-    const auto self = [] {
-        auto tmp = getSelfExe();
-        if (!tmp) {
-            throw Error("can't locate the daemon binary!");
-        }
-        return *tmp;
-    }();
-
     makeNonBlocking(fdSocket.get());
     auto observer = kj::UnixEventPort::FdObserver{
         AIO().unixEventPort, fdSocket.get(), kj::UnixEventPort::FdObserver::OBSERVE_READ
@@ -377,6 +358,46 @@ try {
     co_return result::current_exception();
 }
 
+/**
+ * Run a server. The loop opens a socket and accepts new connections from that
+ * socket.
+ *
+ * @param forceTrustClientOpt If present, force trusting or not trusted
+ * the client. Otherwise, decide based on the authentication settings
+ * and user credentials (from the unix domain socket).
+ */
+static kj::Promise<Result<void>> daemonLoop(std::optional<TrustedFlag> forceTrustClientOpt)
+try {
+    if (chdir("/") == -1) {
+        throw SysError("cannot change current directory");
+    }
+
+    const auto self = [] {
+        auto tmp = getSelfExe();
+        if (!tmp) {
+            throw Error("can't locate the daemon binary!");
+        }
+        return *tmp;
+    }();
+
+    std::list<std::pair<Settings::DaemonSocketPath, AutoCloseFD>> sockets;
+    for (auto & socket : settings.nixDaemonSockets()) {
+        createDirs(dirOf(socket.path));
+        sockets.emplace_back(socket, createUnixDomainSocket(socket.path, 0666));
+    }
+
+    //  Get rid of children automatically; don't let them become zombies.
+    setSigChldAction(true);
+
+    TRY_AWAIT(asyncSpread(sockets, [&](auto & socket) {
+        return daemonLoopForSocket(self, socket.first, socket.second, forceTrustClientOpt);
+    }));
+
+    co_return result::success();
+} catch (...) {
+    co_return result::current_exception();
+}
+
 static void
 daemonInstance(AsyncIoRoot & aio, std::optional<TrustedFlag> forceTrustClientOpt, char * peerPidArg)
 {
@@ -651,5 +672,4 @@ void registerNixDaemon()
 {
     registerCommand2<CmdDaemon>({"daemon"});
 }
-
 }

misc/systemd/meson.build

@@ -1,4 +1,41 @@
-foreach config : [ 'nix-daemon.socket', 'nix-daemon.service', 'nix-daemon@.service' ]
+lix_daemon_socket_units = []
+
+# legacy sockets are handled specially for graceful fallback if nixos module
+# updates are not applied in lockstep with the socket split updates for lix.
+lix_daemon_socket_units += [ 'nix-daemon.socket' ]
+configure_file(
+  input : 'daemon.socket.in',
+  output : lix_daemon_socket_units[-1],
+  install : true,
+  install_dir : prefix / 'lib/systemd/system',
+  install_mode : 'rw-r--r--',
+  configuration : {
+    'storedir' : store_dir,
+    'localstatedir' : state_dir,
+    'bindir' : bindir,
+    'protocol' : 'legacy',
+    'kind' : 'combined',
+    'path' : 'socket',
+    'dirmode' : '0755',
+  },
+)
+configure_file(
+  input : 'daemon@.service.in',
+  output : 'nix-daemon@.service',
+  install : true,
+  install_dir : prefix / 'lib/systemd/system',
+  install_mode : 'rw-r--r--',
+  configuration : {
+    'storedir' : store_dir,
+    'localstatedir' : state_dir,
+    'bindir' : bindir,
+    'conflicts' : ' '.join(lix_daemon_socket_units),
+    'protocol' : 'legacy',
+    'kind' : 'combined',
+  },
+)
+
+foreach config : [ 'nix-daemon.service' ]
   configure_file(
     input : config + '.in',
     output : config,
@@ -9,6 +46,7 @@ foreach config : [ 'nix-daemon.socket', 'nix-daemon.service', 'nix-daemon@.servi
       'storedir' : store_dir,
       'localstatedir' : state_dir,
       'bindir' : bindir,
+      'conflicts' : ' '.join(lix_daemon_socket_units),
     },
   )
 endforeach

misc/systemd/nix-daemon.service.in

@@ -1,7 +1,7 @@
 [Unit]
 Description=Nix Daemon
 Documentation=man:nix-daemon https://docs.lix.systems/manual/lix/stable
-Conflicts=nix-daemon.socket
+Conflicts=@conflicts@
 RequiresMountsFor=@storedir@
 RequiresMountsFor=@localstatedir@
 RequiresMountsFor=@localstatedir@/nix/db

tests/functional2/testlib/fixtures/nix.py

@@ -179,7 +179,8 @@ class Nix:
         daemon.settings.store = f"local?root={self.env.dirs.test_root}"
         daemon.settings.other_settings |= settings
 
-        sockets = [Path(daemon.env.dirs.nix_state_dir) / "daemon-socket/socket"]
+        sockets_dir = Path(daemon.env.dirs.nix_state_dir) / "daemon-socket"
+        sockets = [sockets_dir / "socket"]
         for p in sockets:
             p.unlink(missing_ok=True)
 
@@ -194,7 +195,9 @@ class Nix:
                 self.logger.error("daemon exited unexpectedly")
 
         # wait for daemon to come up. this may take a while under load.
-        while not all(p.exists() for p in sockets):
+        # we only test the *last* socket in the list because that's the
+        # last one the daemon creates, once it's there the daemon is up
+        while not sockets[-1].exists():
             if status := proc.wait(0.01):
                 log_daemon_result(status, logging.ERROR)
                 raise RuntimeError("daemon exited during startup")