[WIP] remove boost coroutine uses

we no longer need these since sinkToSource and sourceToSink are gone.

Change-Id: Ibbf440e2cf71bf3e9f3b833af2d78a21fb1b3193

.envrc

@@ -1,9 +1,5 @@
 # shellcheck shell=bash
 source_env_if_exists .envrc.local
-# TODO: `use flake .#native-clangStdenvPackages` on macOS?
-use flake ".#${LIX_SHELL_VARIANT:-default}" "${LIX_SHELL_EXTRA_ARGS[@]}"
-export MAKEFLAGS="$MAKEFLAGS -e"
-if [[ -n "$NIX_BUILD_CORES" ]]; then
-    export MAKEFLAGS="$MAKEFLAGS -j $NIX_BUILD_CORES"
-fi
+# Use native-clangStdenvPackages to get clangd by default.
+use flake ".#${LIX_SHELL_VARIANT:-native-clangStdenvPackages}" "${LIX_SHELL_EXTRA_ARGS[@]}"
 export GTEST_BRIEF=1

.this-is-lix

@@ -0,0 +1 @@
+This is a file used by the dev shell shellHook in package.nix to check that this is actually a Lix repo before installing git hooks. Its contents have no meaning.

boehmgc-coroutine-sp-fallback.diff

@@ -1,54 +0,0 @@
-diff --git a/pthread_stop_world.c b/pthread_stop_world.c
-index 2b45489..0e6d8ef 100644
---- a/pthread_stop_world.c
-+++ b/pthread_stop_world.c
-@@ -776,6 +776,8 @@ STATIC void GC_restart_handler(int sig)
- /* world is stopped.  Should not fail if it isn't.                      */
- GC_INNER void GC_push_all_stacks(void)
- {
-+    size_t stack_limit;
-+    pthread_attr_t pattr;
-     GC_bool found_me = FALSE;
-     size_t nthreads = 0;
-     int i;
-@@ -868,6 +870,40 @@ GC_INNER void GC_push_all_stacks(void)
-             hi = p->altstack + p->altstack_size;
- #         endif
-           /* FIXME: Need to scan the normal stack too, but how ? */
-+        } else {
-+          #ifdef HAVE_PTHREAD_ATTR_GET_NP
-+          if (pthread_attr_init(&pattr) != 0) {
-+            ABORT("GC_push_all_stacks: pthread_attr_init failed!");
-+          }
-+          if (pthread_attr_get_np(p->id, &pattr) != 0) {
-+            ABORT("GC_push_all_stacks: pthread_attr_get_np failed!");
-+          }
-+          #else
-+          if (pthread_getattr_np(p->id, &pattr)) {
-+            ABORT("GC_push_all_stacks: pthread_getattr_np failed!");
-+          }
-+          #endif
-+          if (pthread_attr_getstacksize(&pattr, &stack_limit)) {
-+            ABORT("GC_push_all_stacks: pthread_attr_getstacksize failed!");
-+          }
-+          if (pthread_attr_destroy(&pattr)) {
-+            ABORT("GC_push_all_stacks: pthread_attr_destroy failed!");
-+          }
-+          // When a thread goes into a coroutine, we lose its original sp until
-+          // control flow returns to the thread.
-+          // While in the coroutine, the sp points outside the thread stack,
-+          // so we can detect this and push the entire thread stack instead,
-+          // as an approximation.
-+          // We assume that the coroutine has similarly added its entire stack.
-+          // This could be made accurate by cooperating with the application
-+          // via new functions and/or callbacks.
-+          #ifndef STACK_GROWS_UP
-+            if (lo >= hi || lo < hi - stack_limit) { // sp outside stack
-+              lo = hi - stack_limit;
-+            }
-+          #else
-+          #error "STACK_GROWS_UP not supported in boost_coroutine2 (as of june 2021), so we don't support it in Nix."
-+          #endif
-         }
- #       ifdef STACKPTR_CORRECTOR_AVAILABLE
-           if (GC_sp_corrector != 0)

doc/manual/change-authors.yml

@@ -3,6 +3,10 @@
 #
 # It's used for crediting people accurately in release notes. The release notes
 # script will link to forgejo, then to GitHub if forgejo is not present.
+#
+# When adding someone from outside the Lix project, you generally want to simply link their GitHub profile without adding a display name unless they are well-known in the community by that display name.
+#
+# See doc/manual/src/contributing/hacking.md for more documentation on this file's format and typical usage.
 9999years:
   display_name: wiggles
   forgejo: rbt
@@ -60,6 +64,10 @@ jade:
   forgejo: jade
   github: lf-
 
+kloenk:
+  forgejo: kloenk
+  github: kloenk
+
 lovesegfault:
   github: lovesegfault
 

doc/manual/process-includes.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/usr/bin/env bash
 
 set -euo pipefail
 

doc/manual/render-manpage.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/usr/bin/env bash
 
 set -euo pipefail
 

doc/manual/rl-next/build-dir.md

@@ -0,0 +1,12 @@
+---
+synopsis: "Add a `build-dir` setting to set the backing directory for builds"
+cls: 1514
+credits: [roberth, tomberek]
+category: Improvements
+---
+
+`build-dir` can now be set in the Nix configuration to choose the backing directory for the build sandbox.
+This can be useful on systems with `/tmp` on tmpfs, or simply to relocate large builds to another disk.
+
+Also, `XDG_RUNTIME_DIR` is no longer considered when selecting the default temporary directory,
+as it's not intended to be used for large amounts of data.

doc/manual/rl-next/multiline-log-format.md

@@ -0,0 +1,14 @@
+---
+synopsis: Add log formats `multiline` and `multiline-with-logs`
+cls: [1369]
+credits: [kloenk]
+category: Improvements
+---
+
+Added two new log formats (`multiline` and `multiline-with-logs`) that display
+current activities below each other for better visibility.
+
+These formats attempt to use the maximum available lines
+(defaulting to 25 if unable to determine) and print up to that many lines.
+The status bar is displayed as the first line, with each subsequent
+activity on its own line.

doc/manual/rl-next/nix-copy-is-fast.md

@@ -0,0 +1,12 @@
+---
+synopsis: "`nix copy` is now several times faster at `querying info about /nix/store/...`"
+cls: [1462]
+issues: [fj#366]
+credits: [jade]
+category: Fixes
+---
+
+We fixed a locking bug that serialized `querying info about /nix/store/...`
+onto just one thread such that it was eating `O(paths to copy * latency)` time
+while setting up to copy paths to s3 and other stores. It is now `nproc` times
+faster.

doc/manual/rl-next/old-protocol-removal.md

@@ -0,0 +1,21 @@
+---
+synopsis: "Lix no longer speaks the Nix remote-build worker protocol to clients or servers older than CppNix 2.3"
+cls: [1207, 1208, 1206, 1205, 1204, 1203, 1479]
+issues: [fj#325]
+credits: [jade]
+category: Breaking Changes
+---
+
+CppNix 2.3 was released in 2019, and is the new oldest supported version. We
+will increase our support baseline in the future up to a final version of CppNix
+2.18 (which may happen soon given that it is the only still-packaged and thus
+still-tested >2.3 version), but this step already removes a significant amount
+of dead, untested, code paths.
+
+Lix speaks the same version of the protocol as CppNix 2.18 and that fact will
+never change in the future; the Lix plans to replace the protocol for evolution
+will entail a complete incompatible replacement that will be supported in
+parallel with the old protocol. Lix will thus retain remote build compatibility
+with CppNix as long as CppNix maintains protocol compatibility with 2.18, and
+as long as Lix retains legacy protocol support (which will likely be a long
+time given that we plan to convert it to a frozen-in-time shim).

doc/manual/rl-next/sanitizers.md

@@ -0,0 +1,10 @@
+---
+synopsis: "Lix now supports building with UndefinedBehaviorSanitizer"
+cls: [1483]
+credits: [jade]
+category: Development
+---
+
+You can now build Lix with the configuration option `-Db_sanitize=undefined` and it will both work and pass tests. AddressSanitizer support is also coming soon.
+
+For a list of undefined behaviour fixed by sanitizer usage, see [the gerrit topic "undefined-behaviour"](https://gerrit.lix.systems/q/topic:%22undefined-behaviour%22).

doc/manual/src/command-ref/opt-common.md

@@ -78,6 +78,16 @@ Most commands in Lix accept the following command-line options:
 
     Display the raw logs, with the progress bar at the bottom.
 
+  - `multiline`
+
+    Display a progress bar during the builds and in the lines below that one line per activity.
+
+
+  - `multiline-with-logs`
+
+    Displayes the raw logs, with a progress bar and activities each in a new line at the bottom.
+
+
 - <span id="opt-no-build-output">[`--no-build-output`](#opt-no-build-output)</span> / `-Q`
 
   By default, output written by builders to standard output and standard error is echoed to the Lix command's standard error.

doc/manual/src/contributing/hacking.md

@@ -39,17 +39,19 @@ $ nix-shell -A native-clangStdenvPackages
 
 ### Building from the development shell
 
-As always you may run [stdenv's phases by name](https://nixos.org/manual/nixpkgs/unstable/#sec-building-stdenv-package-in-nix-shell), e.g.:
+You can build and test Lix with just:
 
 ```bash
-$ configurePhase
-$ buildPhase
-$ checkPhase
-$ installPhase
-$ installCheckPhase
+$ just setup
+$ just build
+$ just test --suite=check
+$ just install
+$ just test --suite=installcheck
 ```
 
-To build manually, however, use the following:
+(Check and installcheck may both be done after install, allowing you to omit the --suite argument entirely, but this is the order package.nix runs them in.)
+
+You can also build Lix manually:
 
 ```bash
 $ meson setup ./build "--prefix=$out" $mesonFlags
@@ -64,9 +66,7 @@ $ meson install -C build
 $ meson test -C build --suite=installcheck
 ```
 
-(Check and installcheck may both be done after install, allowing you to omit the --suite argument entirely, but this is the order package.nix runs them in.)
-
-This will install Lix to `$PWD/outputs`, the `/bin` of which is prepended to PATH in the development shells.
+In both cases, Lix will be installed to `$PWD/outputs`, the `/bin` of which is prepended to PATH in the development shells.
 
 If the tests fail and Meson helpfully has no output for why, use the `--print-error-logs` option to `meson test`.
 
@@ -168,8 +168,26 @@ or for Nix with the [`flakes`] and [`nix-command`] experimental features enabled
 $ nix build .#packages.aarch64-linux.default
 ```
 
-Cross-compiled builds are available for ARMv6 (`armv6l-linux`) and ARMv7 (`armv7l-linux`).
-Add more [system types](#system-type) to `crossSystems` in `flake.nix` to bootstrap Nix on unsupported platforms.
+### Cross compiling using the Lix flake
+
+Lix can also be easily cross compiled to the following arbitrarily-chosen system doubles, which can be useful for bootstrapping Lix on new platforms.
+These are specified in `crossSystems` in `flake.nix`; feel free to submit changes to add new ones if they are useful to you.
+
+- `armv6l-linux`
+- `armv7l-linux`
+- `riscv64-linux`
+
+For example, to cross-compile Lix for `armv6l-linux` from another Linux, use the following:
+
+```console
+$ nix build .#nix-armv6l-linux
+```
+
+It's also possible to cross-compile a tarball of binaries suitable for the Lix installer, for example, for `riscv64-linux`:
+
+```console
+$ nix build .#nix-riscv64-linux.passthru.binaryTarball
+```
 
 ### Building for multiple platforms at once
 
@@ -282,9 +300,8 @@ Regular markdown files used for the manual have a base path of their own and the
 
 ## API documentation
 
-Doxygen API documentation is [available
-online](https://hydra.nixos.org/job/nix/master/internal-api-docs/latest/download-by-type/doc/internal-api-docs). You
-can also build and view it yourself:
+Doxygen API documentation will be available online in the future ([tracking issue](https://git.lix.systems/lix-project/lix/issues/422)).
+You can also build and view it yourself:
 
 ```console
 # nix build .#hydraJobs.internal-api-docs
@@ -294,44 +311,50 @@ can also build and view it yourself:
 or inside a `nix develop` shell by running:
 
 ```bash
+$ meson configure build -Dinternal-api-docs=enabled
 $ meson compile -C build internal-api-docs
 $ xdg-open ./outputs/doc/share/doc/nix/internal-api/html/index.html
 ```
 
 ## Coverage analysis
 
-A coverage analysis report is [available
-online](https://hydra.nixos.org/job/nix/master/coverage/latest/download-by-type/report/coverage). You
-can build it yourself:
+A coverage analysis report will be available online in the future (FIXME(lix-hydra)).
+You can build it yourself:
 
 ```
 # nix build .#hydraJobs.coverage
 # xdg-open ./result/coverage/index.html
 ```
 
-Metrics about the change in line/function coverage over time are also
-[available](https://hydra.nixos.org/job/nix/master/coverage#tabs-charts).
+Metrics about the change in line/function coverage over time will be available in the future (FIXME(lix-hydra)).
 
 ## Add a release note
 
 `doc/manual/rl-next` contains release notes entries for all unreleased changes.
 
 User-visible changes should come with a release note.
+Developer-facing changes should have a release note in the Development category if they are significant and if developers should know about them.
 
 ### Add an entry
 
-Here's what a complete entry looks like. The file name is not incorporated in the document.
+Here's what a complete entry looks like.
+The file name is not incorporated in the final document, and is generally a super brief summary of the change synopsis.
 
-```
+```markdown
 ---
 synopsis: Basically a title
 # 1234 or gh#1234 will refer to CppNix GitHub, fj#1234 will refer to a Lix forgejo issue.
 issues: [1234, fj#1234]
-# Use this *only* if there is a CppNix pull request associated with this change
+# Use this *only* if there is a CppNix pull request associated with this change.
 prs: 1238
-# List of Lix Gerrit changelist numbers; if there is an associated Lix GitHub
-# PR, just put in the Gerrit CL number.
+# List of Lix Gerrit changelist numbers.
+# If there is an associated Lix GitHub PR, just put in the Gerrit CL number.
 cls: [123]
+# Heading that this release note will appear under.
+category: Breaking Changes
+# Add a credit mention in the bottom of the release note.
+# your-name is used as a key into doc/manual/change-authors.yml for metadata
+credits: [your-name]
 ---
 
 Here's one or more paragraphs that describe the change.
@@ -346,6 +369,31 @@ Significant changes should add the following header, which moves them to the top
 significance: significant
 ```
 
+The following categories of release notes are supported (see `maintainers/build-release-notes.py`):
+- Breaking Changes
+- Features
+- Improvements
+- Fixes
+- Packaging
+- Development
+- Miscellany
+
+The `credits` field, if present, gives credit to the author of the patch in the release notes with a message like "Many thanks to (your-name) for this" and linking to GitHub or Forgejo profiles if listed.
+
+If you are forward-porting a change from CppNix, please credit the original author, and optionally credit yourself.
+When adding credits metadata for people external to the project and deciding whether to put in a `display_name`, consider what they are generally known as in the community; even if you know their full name (e.g. from their GitHub profile), we suggest only adding it as a display name if that is what they go by in the community.
+There are multiple reasons we follow this practice, but it boils down to privacy and consent: we would rather not capture full names that are not widely used in the community without the consent of the parties involved, even if they are publicly available.
+As of this writing, the entries with full names as `display_name` are either members of the CppNix team or people who added them themselves.
+
+The names specified in `credits` are used as keys to look up the authorship info in `doc/manual/change-authors.yml`.
+The only mandatory part is that every key appearing in `credits` has an entry present in `change-authors.yml`.
+All of the following properties are optional; you can specify `{}` as the metadata if you want a simple non-hyperlinked mention.
+The following properties are supported:
+
+- `display_name`: display name used in place of the key when showing names, if present.
+- `forgejo`: Forgejo username. The name in the release notes will be a link to this, if present.
+- `github`: GitHub username, used if `forgejo` is not set, again making a link.
+
 ### Build process
 
 Releases have a precomputed `rl-MAJOR.MINOR.md`, and no `rl-next.md`.

flake.nix

@@ -84,6 +84,8 @@
       ];
       systems = linuxSystems ++ darwinSystems;
 
+      # If you add something here, please update the list in doc/manual/src/contributing/hacking.md.
+      # Thanks~
       crossSystems = [
         "armv6l-linux"
         "armv7l-linux"
@@ -97,7 +99,8 @@
       stdenvs = [
         "gccStdenv"
         "clangStdenv"
-        "stdenv"
+        # FIXME: gcc 12 (default in 23.11) has bugs that break the nar parser.
+        "gcc13Stdenv" # "stdenv"
         "libcxxStdenv"
         "ccacheStdenv"
       ];
@@ -143,7 +146,7 @@
               ];
             };
           stdenvs = forAllStdenvs (make-pkgs null);
-          native = stdenvs.stdenvPackages;
+          native = stdenvs.gcc13StdenvPackages;
         in
         {
           inherit stdenvs native;
@@ -195,14 +198,19 @@
             busybox-sandbox-shell = final.busybox-sandbox-shell or final.default-busybox-sandbox-shell;
           };
 
+          pegtl = final.nix.passthru.pegtl;
+
           # Export the patched version of boehmgc that Lix uses into the overlay
           # for consumers of this flake.
-          boehmgc-nix = final.nix.boehmgc-nix;
+          boehmgc-nix = final.nix.passthru.boehmgc-nix;
           # And same thing for our build-release-notes package.
-          build-release-notes = final.nix.build-release-notes;
+          build-release-notes = final.nix.passthru.build-release-notes;
         };
     in
     {
+      # for repl debugging
+      inherit self;
+
       # A Nixpkgs overlay that overrides the 'nix' and
       # 'nix.perl-bindings' packages.
       overlays.default = overlayFor (p: p.stdenv);
@@ -385,7 +393,7 @@
               nix = pkgs.callPackage ./package.nix {
                 inherit stdenv officialRelease versionSuffix;
                 busybox-sandbox-shell = pkgs.busybox-sandbox-shell or pkgs.default-busybox-sandbox;
-                internalApiDocs = true;
+                internalApiDocs = false;
               };
               pre-commit = self.hydraJobs.pre-commit.${pkgs.system} or { };
             in
@@ -413,7 +421,7 @@
             makeShell pkgs pkgs.stdenv
           ))
           // {
-            default = self.devShells.${system}.native-stdenvPackages;
+            default = self.devShells.${system}.native-gcc13StdenvPackages;
           }
         );
     };

justfile

@@ -9,8 +9,8 @@ clean:
     rm -rf build
 
 # Prepare meson for building
-setup:
-    meson setup build --prefix="$PWD/outputs/out"
+setup *OPTIONS:
+    meson setup build --prefix="$PWD/outputs/out" $mesonFlags {{ OPTIONS }}
 
 # Build lix
 build *OPTIONS:

maintainers/build-release-notes.py

@@ -20,6 +20,8 @@ SIGNIFICANCECES = {
 
 # This is just hardcoded for better validation. If you think there should be
 # more of them, feel free to add more.
+#
+# Please update doc/manual/src/contributing/hacking.md if you do. Thanks~
 CATEGORIES = [
     'Breaking Changes',
     'Features',

maintainers/buildtime_report.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/usr/bin/env bash
 
 # Generates a report of build time based on a meson build using -ftime-trace in
 # Clang.

meson.build

@@ -142,6 +142,16 @@ else
   cpp_pch = []
 endif
 
+# gcc 12 is known to miscompile some coroutine-based code quite horribly,
+# causing (among other things) copies of move-only objects and the double
+# frees one would expect when the objects are unique_ptrs. these problems
+# often show up as memory corruption when nesting generators (since we do
+# treat generators like owned memory) and will cause inexplicable crashs.
+assert(
+  cxx.get_id() != 'gcc' or cxx.version().version_compare('>=13'),
+  'GCC 12 and earlier are known to miscompile lix coroutines, use GCC 13 or clang.'
+)
+
 
 # Translate some historical and Mesony CPU names to Lixy CPU names.
 # FIXME(Qyriad): the 32-bit x86 code is not tested right now, because cross compilation for Lix
@@ -194,7 +204,7 @@ configdata += {
   'HAVE_BOEHMGC': boehm.found().to_int(),
 }
 
-boost = dependency('boost', required : true, modules : ['context', 'coroutine', 'container'])
+boost = dependency('boost', required : true, modules : ['container'])
 
 # cpuid only makes sense on x86_64
 cpuid_required = is_x64 ? get_option('cpuid') : false
@@ -287,6 +297,14 @@ gtest = [
 
 toml11 = dependency('toml11', version : '>=3.7.0', required : true, method : 'cmake')
 
+pegtl = dependency(
+  'pegtl',
+  version : '>=3.2.7',
+  required : true,
+  method : 'cmake',
+  modules : [ 'taocpp::pegtl' ],
+)
+
 nlohmann_json = dependency('nlohmann_json', required : true)
 
 # lix-doc is a Rust project provided via buildInputs and unfortunately doesn't have any way to be detected.
@@ -335,8 +353,6 @@ endif
 # that busybox sh won't run busybox applets as builtins (which would break our sandbox).
 
 lsof = find_program('lsof', native : true)
-bison = find_program('bison', native : true)
-flex = find_program('flex', native : true)
 
 # This is how Nix does generated headers...
 # other instances of header generation use a very similar command.
@@ -426,6 +442,7 @@ add_project_arguments(
   '-Wimplicit-fallthrough',
   '-Werror=switch',
   '-Werror=switch-enum',
+  '-Werror=unused-result',
   '-Wdeprecated-copy',
   '-Wignored-qualifiers',
   # Enable assertions in libstdc++ by default. Harmless on libc++. Benchmarked

meson/cleanup-install.bash

@@ -1,50 +0,0 @@
-#!/usr/bin/env bash
-# Meson will call this with an absolute path to Bash.
-# The shebang is just for convenience.
-
-# The parser and lexer tab are generated via custom Meson targets in src/libexpr/meson.build,
-# but Meson doesn't support marking only part of a target for install. The generation creates
-# both headers (parser-tab.hh, lexer-tab.hh) and source files (parser-tab.cc, lexer-tab.cc),
-# and we definitely want the former installed, but not the latter. This script is added to
-# Meson's install steps to correct this, as the logic for it is just complex enough to
-# warrant separate and careful handling, because both Meson's configured include directory
-# may or may not be an absolute path, and DESTDIR may or may not be set at all, but can't be
-# manipulated in Meson logic.
-
-set -euo pipefail
-
-echo "cleanup-install: removing Meson-placed C++ sources from dest includedir"
-
-if [[ "${1/--help/}" != "$1" ]]; then
-	echo "cleanup-install: this script should only be called from the Meson build system"
-	exit 1
-fi
-
-# Ensure the includedir was passed as the first argument
-# (set -u will make this fail otherwise).
-includedir="$1"
-# And then ensure that first argument is a directory that exists.
-if ! [[ -d "$1" ]]; then
-	echo "cleanup-install: this script should only be called from the Meson build system"
-	echo "argv[1] (${1@Q}) is not a directory"
-	exit 2
-fi
-
-# If DESTDIR environment variable is set, prepend it to the include dir.
-# Unfortunately, we cannot do this on the Meson side. We do have an environment variable
-# `MESON_INSTALL_DESTDIR_PREFIX`, but that will not refer to the include directory if
-# includedir has been set separately, which Lix's split-output derivation does.
-# We also cannot simply do an inline bash conditional like "${DESTDIR:=}" or similar,
-# because we need to specifically *join* DESTDIR and includedir with a slash, and *not*
-# have a slash if DESTDIR isn't set at all, since $includedir could be a relative directory.
-# Finally, DESTDIR is only available to us as an environment variable in these install scripts,
-# not in Meson logic.
-# Therefore, our best option is to have Meson pass this script the configured includedir,
-# and perform this dance with it and $DESTDIR.
-if [[ -n "${DESTDIR:-}" ]]; then
-	includedir="$DESTDIR/$includedir"
-fi
-
-# Intentionally not using -f.
-# If these files don't exist then our assumptions have been violated and we should fail.
-rm -v "$includedir/lix/libexpr/parser-tab.cc" "$includedir/lix/libexpr/lexer-tab.cc"

misc/pegtl.nix

@@ -0,0 +1,23 @@
+{
+  stdenv,
+  cmake,
+  ninja,
+  fetchFromGitHub,
+}:
+
+stdenv.mkDerivation {
+  pname = "pegtl";
+  version = "3.2.7";
+
+  src = fetchFromGitHub {
+    repo = "PEGTL";
+    owner = "taocpp";
+    rev = "refs/tags/3.2.7";
+    hash = "sha256-IV5YNGE4EWVrmg2Sia/rcU8jCuiBynQGJM6n3DCWTQU=";
+  };
+
+  nativeBuildInputs = [
+    cmake
+    ninja
+  ];
+}

package.nix

@@ -10,17 +10,16 @@
   boehmgc-nix ? __forDefaults.boehmgc-nix,
   boehmgc,
   nlohmann_json,
-  bison,
   build-release-notes ? __forDefaults.build-release-notes,
   boost,
   brotli,
   bzip2,
+  callPackage,
   cmake,
   curl,
   doxygen,
   editline-lix ? __forDefaults.editline-lix,
   editline,
-  flex,
   git,
   gtest,
   jq,
@@ -36,6 +35,7 @@
   meson,
   ninja,
   openssl,
+  pegtl ? __forDefaults.pegtl,
   pkg-config,
   python3,
   rapidcheck,
@@ -62,22 +62,16 @@
   __forDefaults ? {
     canRunInstalled = stdenv.buildPlatform.canExecute stdenv.hostPlatform;
 
-    boehmgc-nix = (boehmgc.override { enableLargeConfig = true; }).overrideAttrs {
-      patches = [
-        # We do *not* include prev.patches (which doesn't exist in normal pkgs.boehmgc anyway)
-        # because if the caller of this package passed a patched boehm as `boehmgc` instead of
-        # `boehmgc-nix` then this will almost certainly have duplicate patches, which means
-        # the patches won't apply and we'll get a build failure.
-        ./boehmgc-coroutine-sp-fallback.diff
-      ];
-    };
+    boehmgc-nix = boehmgc.override { enableLargeConfig = true; };
 
     editline-lix = editline.overrideAttrs (prev: {
       configureFlags = prev.configureFlags or [ ] ++ [ (lib.enableFeature true "sigstop") ];
     });
 
-    lix-doc = pkgs.callPackage ./lix-doc/package.nix { };
-    build-release-notes = pkgs.callPackage ./maintainers/build-release-notes.nix { };
+    lix-doc = callPackage ./lix-doc/package.nix { };
+    build-release-notes = callPackage ./maintainers/build-release-notes.nix { };
+
+    pegtl = callPackage ./misc/pegtl.nix { };
   },
 }:
 let
@@ -165,7 +159,6 @@ stdenv.mkDerivation (finalAttrs: {
           functionalTestFiles
         ]
         ++ lib.optionals (!finalAttrs.dontBuild || internalApiDocs) [
-          ./boehmgc-coroutine-sp-fallback.diff
           ./doc
           ./misc
           ./src
@@ -210,8 +203,6 @@ stdenv.mkDerivation (finalAttrs: {
 
   nativeBuildInputs =
     [
-      bison
-      flex
       python3
       meson
       ninja
@@ -250,6 +241,7 @@ stdenv.mkDerivation (finalAttrs: {
       libsodium
       toml11
       lix-doc
+      pegtl
     ]
     ++ lib.optionals hostPlatform.isLinux [
       libseccomp
@@ -304,6 +296,9 @@ stdenv.mkDerivation (finalAttrs: {
       else
         appendToVar configureFlags "--disable-tests"
       fi
+
+      # Fix up /usr/bin/env shebangs relied on by the build
+      patchShebangs --build tests/ doc/manual/
     '';
 
   mesonBuildType = "debugoptimized";
@@ -379,7 +374,12 @@ stdenv.mkDerivation (finalAttrs: {
   # Export the patched version of boehmgc.
   # flake.nix exports that into its overlay.
   passthru = {
-    inherit (__forDefaults) boehmgc-nix editline-lix build-release-notes;
+    inherit (__forDefaults)
+      boehmgc-nix
+      editline-lix
+      build-release-notes
+      pegtl
+      ;
 
     inherit officialRelease;
 
@@ -392,6 +392,7 @@ stdenv.mkDerivation (finalAttrs: {
         bashInteractive,
         clang-tools,
         clangbuildanalyzer,
+        doxygen,
         glibcLocales,
         just,
         llvmPackages,
@@ -457,6 +458,10 @@ stdenv.mkDerivation (finalAttrs: {
               skopeo
               just
               nixfmt
+              # Included above when internalApiDocs is true, but we set that to
+              # false intentionally to save dev build time.
+              # To build them in a dev shell, you can set -Dinternal-api-docs=enabled when configuring.
+              doxygen
               # Load-bearing order. Must come before clang-unwrapped below, but after clang_tools above.
               stdenv.cc
             ]
@@ -476,7 +481,7 @@ stdenv.mkDerivation (finalAttrs: {
               # https://git.lix.systems/lix-project/lix/src/commit/7575db522e9008685c4009423398f6900a16bcce/src/nix/develop.cc#L240-L241
               # this is, of course, absurd.
               if [[ $name != lix-shell-env && $name != lix-shell-env-env ]]; then
-                return;
+                return
               fi
 
               PATH=$prefix/bin:$PATH
@@ -486,6 +491,11 @@ stdenv.mkDerivation (finalAttrs: {
               # Make bash completion work.
               XDG_DATA_DIRS+=:$out/share
 
+              if [[ ! -f ./.this-is-lix ]]; then
+                echo "Dev shell not started from inside a Lix repo, skipping repo setup" >&2
+                return
+              fi
+
               ${lib.optionalString (pre-commit-checks ? shellHook) pre-commit-checks.shellHook}
               # Allow `touch .nocontribmsg` to turn this notice off.
               if ! [[ -f .nocontribmsg ]]; then

perl/meson.build

@@ -65,5 +65,6 @@ if cxx.get_linker_id() in ['ld.bfd', 'ld.gold']
 endif
 
 libstore = dependency('lixstore', 'lix-store', required : true)
+libutil = dependency('lixutil', 'lix-util', required : true)
 
 subdir('lib/Nix')

src/libcmd/cmd-profiles.cc

@@ -244,7 +244,7 @@ StorePath ProfileManifest::build(ref<Store> store)
 
     /* Add the symlink tree to the store. */
     StringSink sink;
-    dumpPath(tempDir, sink);
+    sink << dumpPath(tempDir);
 
     auto narHash = hashString(htSHA256, sink.s);
 

src/libcmd/repl.cc

@@ -1,3 +1,4 @@
+#include <cstdio>
 #include <editline.h>
 #include <iostream>
 #include <cstdlib>
@@ -242,8 +243,7 @@ NixRepl::NixRepl(const SearchPath & searchPath, nix::ref<Store> store, ref<EvalS
 {
 }
 
-void runNix(Path program, const Strings & args,
-    const std::optional<std::string> & input = {})
+void runNix(Path program, const Strings & args)
 {
     auto subprocessEnv = getEnv();
     subprocessEnv["NIX_CONFIG"] = globalConfig.toKeyValue();
@@ -252,8 +252,7 @@ void runNix(Path program, const Strings & args,
         .program = settings.nixBinDir+ "/" + program,
         .args = args,
         .environment = subprocessEnv,
-        .input = input,
-    });
+    }).wait();
 
     return;
 }
@@ -652,7 +651,7 @@ ProcessLineResult NixRepl::processLine(std::string line)
 
         // runProgram redirects stdout to a StringSink,
         // using runProgram2 to allow editors to display their UI
-        runProgram2(RunOptions { .program = editor, .searchPath = true, .args = args });
+        runProgram2(RunOptions { .program = editor, .searchPath = true, .args = args }).wait();
 
         // Reload right after exiting the editor
         state->resetFileCache();

src/libexpr/eval.cc

@@ -4,6 +4,7 @@
 #include "primops.hh"
 #include "print-options.hh"
 #include "shared.hh"
+#include "suggestions.hh"
 #include "types.hh"
 #include "store-api.hh"
 #include "derivations.hh"
@@ -17,7 +18,6 @@
 #include "gc-small-vector.hh"
 #include "fetch-to-store.hh"
 #include "flake/flakeref.hh"
-#include "parser-tab.hh"
 
 #include <algorithm>
 #include <chrono>
@@ -42,10 +42,6 @@
 #include <gc/gc.h>
 #include <gc/gc_cpp.h>
 
-#include <boost/coroutine2/coroutine.hpp>
-#include <boost/coroutine2/protected_fixedsize_stack.hpp>
-#include <boost/context/stack_context.hpp>
-
 #endif
 
 using json = nlohmann::json;
@@ -205,42 +201,6 @@ static void * oomHandler(size_t requested)
     throw std::bad_alloc();
 }
 
-class BoehmGCStackAllocator : public StackAllocator {
-    boost::coroutines2::protected_fixedsize_stack stack {
-        // We allocate 8 MB, the default max stack size on NixOS.
-        // A smaller stack might be quicker to allocate but reduces the stack
-        // depth available for source filter expressions etc.
-        std::max(boost::context::stack_traits::default_size(), static_cast<std::size_t>(8 * 1024 * 1024))
-    };
-
-    // This is specific to boost::coroutines2::protected_fixedsize_stack.
-    // The stack protection page is included in sctx.size, so we have to
-    // subtract one page size from the stack size.
-    std::size_t pfss_usable_stack_size(boost::context::stack_context &sctx) {
-        return sctx.size - boost::context::stack_traits::page_size();
-    }
-
-  public:
-    boost::context::stack_context allocate() override {
-        auto sctx = stack.allocate();
-
-        // Stacks generally start at a high address and grow to lower addresses.
-        // Architectures that do the opposite are rare; in fact so rare that
-        // boost_routine does not implement it.
-        // So we subtract the stack size.
-        GC_add_roots(static_cast<char *>(sctx.sp) - pfss_usable_stack_size(sctx), sctx.sp);
-        return sctx;
-    }
-
-    void deallocate(boost::context::stack_context sctx) override {
-        GC_remove_roots(static_cast<char *>(sctx.sp) - pfss_usable_stack_size(sctx), sctx.sp);
-        stack.deallocate(sctx);
-    }
-
-};
-
-static BoehmGCStackAllocator boehmGCStackAllocator;
-
 #endif
 
 
@@ -256,23 +216,6 @@ static Symbol getName(const AttrName & name, EvalState & state, Env & env)
     }
 }
 
-#if HAVE_BOEHMGC
-/* Disable GC while this object lives. Used by CoroutineContext.
- *
- * Boehm keeps a count of GC_disable() and GC_enable() calls,
- * and only enables GC when the count matches.
- */
-class BoehmDisableGC {
-public:
-    BoehmDisableGC() {
-        GC_disable();
-    };
-    ~BoehmDisableGC() {
-        GC_enable();
-    };
-};
-#endif
-
 static bool gcInitialised = false;
 
 void initGC()
@@ -294,17 +237,6 @@ void initGC()
 
     GC_set_oom_fn(oomHandler);
 
-    StackAllocator::defaultAllocator = &boehmGCStackAllocator;
-
-
-#if NIX_BOEHM_PATCH_VERSION != 1
-    printTalkative("Unpatched BoehmGC, disabling GC inside coroutines");
-    /* Used to disable GC when entering coroutines on macOS */
-    create_coro_gc_hook = []() -> std::shared_ptr<void> {
-        return std::make_shared<BoehmDisableGC>();
-    };
-#endif
-
     /* Set the initial heap size to something fairly big (25% of
        physical RAM, up to a maximum of 384 MiB) so that in most cases
        we don't need to garbage collect at all.  (Collection has a
@@ -1426,11 +1358,13 @@ static std::string showAttrPath(EvalState & state, Env & env, const AttrPath & a
 
 void ExprSelect::eval(EvalState & state, Env & env, Value & v)
 {
-    Value vTmp;
-    PosIdx pos2;
-    Value * vAttrs = &vTmp;
+    Value vFirst;
 
-    e->eval(state, env, vTmp);
+    // Pointer to the current attrset Value in this select chain.
+    Value * vCurrent = &vFirst;
+    // Position for the current attrset Value in this select chain.
+    PosIdx posCurrent;
+    e->eval(state, env, vFirst);
 
     try {
         auto dts = state.debugRepl
@@ -1443,48 +1377,75 @@ void ExprSelect::eval(EvalState & state, Env & env, Value & v)
                 showAttrPath(state, env, attrPath))
             : nullptr;
 
-        for (auto & i : attrPath) {
+        for (auto const & currentAttrName : attrPath) {
             state.nrLookups++;
-            Bindings::iterator j;
-            auto name = getName(i, state, env);
-            if (def) {
-                state.forceValue(*vAttrs, pos);
-                if (vAttrs->type() != nAttrs ||
-                    (j = vAttrs->attrs->find(name)) == vAttrs->attrs->end())
-                {
-                    def->eval(state, env, v);
+
+            Symbol const name = getName(currentAttrName, state, env);
+
+            state.forceValue(*vCurrent, pos);
+
+            if (vCurrent->type() != nAttrs) {
+
+                // If we have an `or` provided default,
+                // then this is allowed to not be an attrset.
+                if (def != nullptr) {
+                    this->def->eval(state, env, v);
                     return;
                 }
-            } else {
-                state.forceAttrs(*vAttrs, pos, "while selecting an attribute");
-                if ((j = vAttrs->attrs->find(name)) == vAttrs->attrs->end()) {
-                    std::set<std::string> allAttrNames;
-                    for (auto & attr : *vAttrs->attrs)
-                        allAttrNames.insert(state.symbols[attr.name]);
-                    auto suggestions = Suggestions::bestMatches(allAttrNames, state.symbols[name]);
-                    state.error<EvalError>("attribute '%1%' missing", state.symbols[name])
-                        .atPos(pos).withSuggestions(suggestions).withFrame(env, *this).debugThrow();
-                }
+
+                // Otherwise, we must type error.
+                state.error<TypeError>(
+                    "expected a set but found %s: %s",
+                    showType(*vCurrent),
+                    ValuePrinter(state, *vCurrent, errorPrintOptions)
+                ).withTrace(pos, "while selecting an attribute").debugThrow();
             }
-            vAttrs = j->value;
-            pos2 = j->pos;
-            if (state.countCalls) state.attrSelects[pos2]++;
+
+            // Now that we know this is actually an attrset, try to find an attr
+            // with the selected name.
+            Bindings::iterator attrIt = vCurrent->attrs->find(name);
+            if (attrIt == vCurrent->attrs->end()) {
+
+                // If we have an `or` provided default, then we'll use that.
+                if (def != nullptr) {
+                    this->def->eval(state, env, v);
+                    return;
+                }
+
+                // Otherwise, missing attr error.
+                std::set<std::string> allAttrNames;
+                for (auto const & attr : *vCurrent->attrs) {
+                    allAttrNames.insert(state.symbols[attr.name]);
+                }
+                auto suggestions = Suggestions::bestMatches(allAttrNames, state.symbols[name]);
+                state.error<EvalError>("attribute '%s' missing", state.symbols[name])
+                    .atPos(pos)
+                    .withSuggestions(suggestions)
+                    .withFrame(env, *this)
+                    .debugThrow();
+            }
+
+            // If we're here, then we successfully found the attribute.
+            // Set our currently operated-on attrset to this one, and keep going.
+            vCurrent = attrIt->value;
+            posCurrent = attrIt->pos;
+            if (state.countCalls) state.attrSelects[posCurrent]++;
         }
 
-        state.forceValue(*vAttrs, (pos2 ? pos2 : this->pos ) );
+        state.forceValue(*vCurrent, (posCurrent ? posCurrent : this->pos));
 
     } catch (Error & e) {
-        if (pos2) {
-            auto pos2r = state.positions[pos2];
+        if (posCurrent) {
+            auto pos2r = state.positions[posCurrent];
             auto origin = std::get_if<SourcePath>(&pos2r.origin);
             if (!(origin && *origin == state.derivationInternal))
-                state.addErrorTrace(e, pos2, "while evaluating the attribute '%1%'",
+                state.addErrorTrace(e, posCurrent, "while evaluating the attribute '%1%'",
                     showAttrPath(state, env, attrPath));
         }
         throw;
     }
 
-    v = *vAttrs;
+    v = *vCurrent;
 }
 
 
@@ -2928,21 +2889,6 @@ std::optional<std::string> EvalState::resolveSearchPathPath(const SearchPath::Pa
 }
 
 
-Expr * EvalState::parse(
-    char * text,
-    size_t length,
-    Pos::Origin origin,
-    const SourcePath & basePath,
-    std::shared_ptr<StaticEnv> & staticEnv)
-{
-    auto result = parseExprFromBuf(text, length, origin, basePath, symbols, positions, exprSymbols);
-
-    result->bindVars(*this, staticEnv);
-
-    return result;
-}
-
-
 std::string ExternalValueBase::coerceToString(EvalState & state, const PosIdx & pos, NixStringContext & context, bool copyMore, bool copyToStore) const
 {
     state.error<TypeError>(

src/libexpr/lexer.l

@@ -1,302 +0,0 @@
-%option reentrant bison-bridge bison-locations
-%option align
-%option noyywrap
-%option never-interactive
-%option stack
-%option nodefault
-%option nounput noyy_top_state
-
-
-%s DEFAULT
-%x STRING
-%x IND_STRING
-%x INPATH
-%x INPATH_SLASH
-%x PATH_START
-
-
-%{
-#ifdef __clang__
-#pragma clang diagnostic ignored "-Wunneeded-internal-declaration"
-#endif
-
-// yacc generates code that uses unannotated fallthrough.
-#pragma GCC diagnostic ignored "-Wimplicit-fallthrough"
-#ifdef __clang__
-#pragma clang diagnostic ignored "-Wimplicit-fallthrough"
-#endif
-
-#include "nixexpr.hh"
-#include "parser-tab.hh"
-#include "strings.hh"
-
-using namespace nix;
-
-#define THROW(...)                             \
-  do {                                         \
-    state->error.reset(new auto(__VA_ARGS__)); \
-    return YYerror;                            \
-  } while (0)
-
-namespace nix {
-
-#define CUR_POS state->at(*yylloc)
-
-static void initLoc(YYLTYPE * loc)
-{
-    loc->first_line = loc->last_line = 0;
-    loc->first_column = loc->last_column = 0;
-}
-
-static void adjustLoc(YYLTYPE * loc, const char * s, size_t len)
-{
-    loc->stash();
-
-    loc->first_column = loc->last_column;
-    loc->last_column += len;
-}
-
-
-// we make use of the fact that the parser receives a private copy of the input
-// string and can munge around in it.
-static StringToken unescapeStr(SymbolTable & symbols, char * s, size_t length)
-{
-    char * result = s;
-    char * t = s;
-    char c;
-    // the input string is terminated with *two* NULs, so we can safely take
-    // *one* character after the one being checked against.
-    while ((c = *s++)) {
-        if (c == '\\') {
-            c = *s++;
-            if (c == 'n') *t = '\n';
-            else if (c == 'r') *t = '\r';
-            else if (c == 't') *t = '\t';
-            else *t = c;
-        }
-        else if (c == '\r') {
-            /* Normalise CR and CR/LF into LF. */
-            *t = '\n';
-            if (*s == '\n') s++; /* cr/lf */
-        }
-        else *t = c;
-        t++;
-    }
-    return {result, size_t(t - result)};
-}
-
-
-}
-
-#define YY_USER_INIT initLoc(yylloc)
-#define YY_USER_ACTION adjustLoc(yylloc, yytext, yyleng);
-
-#define PUSH_STATE(state) yy_push_state(state, yyscanner)
-#define POP_STATE() yy_pop_state(yyscanner)
-
-%}
-
-
-ANY         .|\n
-ID          [a-zA-Z\_][a-zA-Z0-9\_\'\-]*
-INT         [0-9]+
-FLOAT       (([1-9][0-9]*\.[0-9]*)|(0?\.[0-9]+))([Ee][+-]?[0-9]+)?
-PATH_CHAR   [a-zA-Z0-9\.\_\-\+]
-PATH        {PATH_CHAR}*(\/{PATH_CHAR}+)+\/?
-PATH_SEG    {PATH_CHAR}*\/
-HPATH       \~(\/{PATH_CHAR}+)+\/?
-HPATH_START \~\/
-SPATH       \<{PATH_CHAR}+(\/{PATH_CHAR}+)*\>
-URI         [a-zA-Z][a-zA-Z0-9\+\-\.]*\:[a-zA-Z0-9\%\/\?\:\@\&\=\+\$\,\-\_\.\!\~\*\']+
-
-
-%%
-
-
-if          { return IF; }
-then        { return THEN; }
-else        { return ELSE; }
-assert      { return ASSERT; }
-with        { return WITH; }
-let         { return LET; }
-in          { return IN; }
-rec         { return REC; }
-inherit     { return INHERIT; }
-or          { return OR_KW; }
-\.\.\.      { return ELLIPSIS; }
-
-\=\=        { return EQ; }
-\!\=        { return NEQ; }
-\<\=        { return LEQ; }
-\>\=        { return GEQ; }
-\&\&        { return AND; }
-\|\|        { return OR; }
-\-\>        { return IMPL; }
-\/\/        { return UPDATE; }
-\+\+        { return CONCAT; }
-
-{ID}        { yylval->id = {yytext, (size_t) yyleng}; return ID; }
-{INT}       { errno = 0;
-              std::optional<int64_t> numMay = string2Int<int64_t>(yytext);
-              if (numMay.has_value()) {
-                  yylval->n = *numMay;
-              } else {
-                  THROW(ParseError(ErrorInfo{
-                      .msg = HintFmt("invalid integer '%1%'", yytext),
-                      .pos = state->positions[CUR_POS],
-                  }));
-              }
-              return INT;
-            }
-{FLOAT}     { errno = 0;
-              yylval->nf = strtod(yytext, 0);
-              if (errno != 0)
-                  THROW(ParseError(ErrorInfo{
-                      .msg = HintFmt("invalid float '%1%'", yytext),
-                      .pos = state->positions[CUR_POS],
-                  }));
-              return FLOAT;
-            }
-
-\$\{        { PUSH_STATE(DEFAULT); return DOLLAR_CURLY; }
-
-\}          { /* State INITIAL only exists at the bottom of the stack and is
-                 used as a marker. DEFAULT replaces it everywhere else.
-                 Popping when in INITIAL state causes an empty stack exception,
-                 so don't */
-              if (YYSTATE != INITIAL)
-                POP_STATE();
-              return '}';
-            }
-\{          { PUSH_STATE(DEFAULT); return '{'; }
-
-\"          { PUSH_STATE(STRING); return '"'; }
-<STRING>([^\$\"\\]|\$[^\{\"\\]|\\{ANY}|\$\\{ANY})*\$/\" |
-<STRING>([^\$\"\\]|\$[^\{\"\\]|\\{ANY}|\$\\{ANY})+ {
-                /* It is impossible to match strings ending with '$' with one
-                   regex because trailing contexts are only valid at the end
-                   of a rule. (A sane but undocumented limitation.) */
-                yylval->str = unescapeStr(state->symbols, yytext, yyleng);
-                return STR;
-              }
-<STRING>\$\{  { PUSH_STATE(DEFAULT); return DOLLAR_CURLY; }
-<STRING>\"    { POP_STATE(); return '"'; }
-<STRING>\$|\\|\$\\ {
-                /* This can only occur when we reach EOF, otherwise the above
-                   (...|\$[^\{\"\\]|\\.|\$\\.)+ would have triggered.
-                   This is technically invalid, but we leave the problem to the
-                   parser who fails with exact location. */
-                return EOF;
-              }
-
-\'\'(\ *\n)?     { PUSH_STATE(IND_STRING); return IND_STRING_OPEN; }
-<IND_STRING>([^\$\']|\$[^\{\']|\'[^\'\$])+ {
-                   yylval->str = {yytext, (size_t) yyleng, true};
-                   return IND_STR;
-                 }
-<IND_STRING>\'\'\$ |
-<IND_STRING>\$   {
-                   yylval->str = {"$", 1};
-                   return IND_STR;
-                 }
-<IND_STRING>\'\'\' {
-                   yylval->str = {"''", 2};
-                   return IND_STR;
-                 }
-<IND_STRING>\'\'\\{ANY} {
-                   yylval->str = unescapeStr(state->symbols, yytext + 2, yyleng - 2);
-                   return IND_STR;
-                 }
-<IND_STRING>\$\{ { PUSH_STATE(DEFAULT); return DOLLAR_CURLY; }
-<IND_STRING>\'\' { POP_STATE(); return IND_STRING_CLOSE; }
-<IND_STRING>\'   {
-                   yylval->str = {"'", 1};
-                   return IND_STR;
-                 }
-
-{PATH_SEG}\$\{ |
-{HPATH_START}\$\{ {
-  PUSH_STATE(PATH_START);
-  yyless(0);
-  yylloc->unstash();
-}
-
-<PATH_START>{PATH_SEG} {
-  POP_STATE();
-  PUSH_STATE(INPATH_SLASH);
-  yylval->path = {yytext, (size_t) yyleng};
-  return PATH;
-}
-
-<PATH_START>{HPATH_START} {
-  POP_STATE();
-  PUSH_STATE(INPATH_SLASH);
-  yylval->path = {yytext, (size_t) yyleng};
-  return HPATH;
-}
-
-{PATH} {
-  if (yytext[yyleng-1] == '/')
-    PUSH_STATE(INPATH_SLASH);
-  else
-    PUSH_STATE(INPATH);
-  yylval->path = {yytext, (size_t) yyleng};
-  return PATH;
-}
-{HPATH} {
-  if (yytext[yyleng-1] == '/')
-    PUSH_STATE(INPATH_SLASH);
-  else
-    PUSH_STATE(INPATH);
-  yylval->path = {yytext, (size_t) yyleng};
-  return HPATH;
-}
-
-<INPATH,INPATH_SLASH>\$\{ {
-  POP_STATE();
-  PUSH_STATE(INPATH);
-  PUSH_STATE(DEFAULT);
-  return DOLLAR_CURLY;
-}
-<INPATH,INPATH_SLASH>{PATH}|{PATH_SEG}|{PATH_CHAR}+ {
-  POP_STATE();
-  if (yytext[yyleng-1] == '/')
-      PUSH_STATE(INPATH_SLASH);
-  else
-      PUSH_STATE(INPATH);
-  yylval->str = {yytext, (size_t) yyleng};
-  return STR;
-}
-<INPATH>{ANY} |
-<INPATH><<EOF>> {
-  /* if we encounter a non-path character we inform the parser that the path has
-     ended with a PATH_END token and re-parse this character in the default
-     context (it may be ')', ';', or something of that sort) */
-  POP_STATE();
-  yyless(0);
-  yylloc->unstash();
-  return PATH_END;
-}
-
-<INPATH_SLASH>{ANY} |
-<INPATH_SLASH><<EOF>> {
-  THROW(ParseError(ErrorInfo{
-      .msg = HintFmt("path has a trailing slash"),
-      .pos = state->positions[CUR_POS],
-  }));
-}
-
-{SPATH}     { yylval->path = {yytext, (size_t) yyleng}; return SPATH; }
-{URI}       { yylval->uri = {yytext, (size_t) yyleng}; return URI; }
-
-[ \t\r\n]+    /* eat up whitespace */
-\#[^\r\n]*    /* single-line comments */
-\/\*([^*]|\*+[^*/])*\*+\/  /* long comments */
-
-{ANY}       {
-              /* Don't return a negative number, as this will cause
-                 Bison to stop parsing without an error. */
-              return (unsigned char) yytext[0];
-            }
-
-%%

src/libexpr/meson.build

@@ -1,54 +1,3 @@
-parser_tab = custom_target(
-  input : 'parser.y',
-  output : [
-    'parser-tab.cc',
-    'parser-tab.hh',
-  ],
-  command : [
-    'bison',
-    '-v',
-    '-o',
-    '@OUTPUT0@',
-    '@INPUT@',
-    '-d',
-  ],
-  # NOTE(Qyriad): Meson doesn't support installing only part of a custom target, so we add
-  # an install script below which removes parser-tab.cc.
-  install : true,
-  install_dir : includedir / 'lix/libexpr',
-)
-
-lexer_tab = custom_target(
-  input : [
-    'lexer.l',
-    parser_tab,
-  ],
-  output : [
-    'lexer-tab.cc',
-    'lexer-tab.hh',
-  ],
-  command : [
-    'flex',
-    '--outfile',
-    '@OUTPUT0@',
-    '--header-file=' + '@OUTPUT1@',
-    '@INPUT0@',
-  ],
-  # NOTE(Qyriad): Meson doesn't support installing only part of a custom target, so we add
-  # an install script below which removes lexer-tab.cc.
-  install : true,
-  install_dir : includedir / 'lix/libexpr',
-)
-
-# TODO(Qyriad): When the parser and lexer are rewritten this should be removed.
-# NOTE(Qyriad): We do this this way instead of an inline bash or rm command
-# due to subtleties in Meson. Check the comments in cleanup-install.bash for details.
-meson.add_install_script(
-  bash,
-  meson.project_source_root() / 'meson/cleanup-install.bash',
-  '@0@'.format(includedir),
-)
-
 libexpr_generated_headers = [
   gen_header.process('primops/derivation.nix', preserve_path_from : meson.current_source_dir()),
 ]
@@ -75,6 +24,7 @@ libexpr_sources = files(
   'get-drvs.cc',
   'json-to-value.cc',
   'nixexpr.cc',
+  'parser/parser.cc',
   'paths.cc',
   'primops.cc',
   'print-ambiguous.cc',
@@ -110,7 +60,9 @@ libexpr_headers = files(
   'get-drvs.hh',
   'json-to-value.hh',
   'nixexpr.hh',
-  'parser-state.hh',
+  'parser/change_head.hh',
+  'parser/grammar.hh',
+  'parser/state.hh',
   'pos-idx.hh',
   'pos-table.hh',
   'primops.hh',
@@ -129,8 +81,6 @@ libexpr_headers = files(
 libexpr = library(
   'lixexpr',
   libexpr_sources,
-  parser_tab,
-  lexer_tab,
   libexpr_generated_headers,
   dependencies : [
     liblixutil,

src/libexpr/nixexpr.hh

@@ -157,8 +157,18 @@ struct ExprInheritFrom : ExprVar
 struct ExprSelect : Expr
 {
     PosIdx pos;
-    std::unique_ptr<Expr> e, def;
+
+    /** The expression attributes are being selected on. e.g. `foo` in `foo.bar.baz`. */
+    std::unique_ptr<Expr> e;
+
+    /** A default value specified with `or`, if the selected attr doesn't exist.
+     * e.g. `bix` in `foo.bar.baz or bix`
+     */
+    std::unique_ptr<Expr> def;
+
+    /** The path of attributes being selected. e.g. `bar.baz` in `foo.bar.baz.` */
     AttrPath attrPath;
+
     ExprSelect(const PosIdx & pos, std::unique_ptr<Expr> e, AttrPath attrPath, std::unique_ptr<Expr> def) : pos(pos), e(std::move(e)), def(std::move(def)), attrPath(std::move(attrPath)) { };
     ExprSelect(const PosIdx & pos, std::unique_ptr<Expr> e, Symbol name) : pos(pos), e(std::move(e)) { attrPath.push_back(AttrName(name)); };
     PosIdx getPos() const override { return pos; }

src/libexpr/parser.y

@@ -1,503 +0,0 @@
-%glr-parser
-%define api.pure
-%locations
-%define parse.error verbose
-%defines
-/* %no-lines */
-%parse-param { void * scanner }
-%parse-param { nix::ParserState * state }
-%lex-param { void * scanner }
-%lex-param { nix::ParserState * state }
-%expect 1
-%expect-rr 1
-
-%code requires {
-
-#ifndef BISON_HEADER
-#define BISON_HEADER
-
-#include <variant>
-
-#include "finally.hh"
-#include "users.hh"
-
-#include "nixexpr.hh"
-#include "eval.hh"
-#include "eval-settings.hh"
-#include "globals.hh"
-#include "parser-state.hh"
-
-#define YYLTYPE ::nix::ParserLocation
-#define YY_DECL int yylex \
-    (YYSTYPE * yylval_param, YYLTYPE * yylloc_param, yyscan_t yyscanner, nix::ParserState * state)
-
-namespace nix {
-
-Expr * parseExprFromBuf(
-    char * text,
-    size_t length,
-    Pos::Origin origin,
-    const SourcePath & basePath,
-    SymbolTable & symbols,
-    PosTable & positions,
-    const Expr::AstSymbols & astSymbols);
-
-}
-
-#endif
-
-}
-
-%{
-
-#include "parser-tab.hh"
-#include "lexer-tab.hh"
-
-YY_DECL;
-
-using namespace nix;
-
-#define CUR_POS state->at(*yylocp)
-
-// otherwise destructors cause compiler errors
-#pragma GCC diagnostic ignored "-Wswitch-enum"
-
-#define THROW(err, ...)                              \
-  do {                                               \
-    state->error.reset(new auto(err));               \
-    [](auto... d) { (delete d, ...); }(__VA_ARGS__); \
-    YYABORT;                                         \
-  } while (0)
-
-void yyerror(YYLTYPE * loc, yyscan_t scanner, ParserState * state, const char * error)
-{
-    if (std::string_view(error).starts_with("syntax error, unexpected end of file")) {
-        loc->first_column = loc->last_column;
-        loc->first_line = loc->last_line;
-    }
-    throw ParseError({
-        .msg = HintFmt(error),
-        .pos = state->positions[state->at(*loc)]
-    });
-}
-
-template<typename T>
-static std::unique_ptr<T> unp(T * e)
-{
-  return std::unique_ptr<T>(e);
-}
-
-template<typename T = std::unique_ptr<nix::Expr>, typename... Args>
-static std::vector<T> vec(Args && ... args)
-{
-  std::vector<T> result;
-  result.reserve(sizeof...(Args));
-  (result.emplace_back(std::forward<Args>(args)), ...);
-  return result;
-}
-
-
-%}
-
-%union {
-  // !!! We're probably leaking stuff here.
-  nix::Expr * e;
-  nix::ExprList * list;
-  nix::ExprAttrs * attrs;
-  nix::Formals * formals;
-  nix::Formal * formal;
-  nix::NixInt n;
-  nix::NixFloat nf;
-  nix::StringToken id; // !!! -> Symbol
-  nix::StringToken path;
-  nix::StringToken uri;
-  nix::StringToken str;
-  std::vector<nix::AttrName> * attrNames;
-  std::vector<std::pair<nix::AttrName, nix::PosIdx>> * inheritAttrs;
-  std::vector<std::pair<nix::PosIdx, std::unique_ptr<nix::Expr>>> * string_parts;
-  std::vector<std::pair<nix::PosIdx, std::variant<std::unique_ptr<nix::Expr>, nix::StringToken>>> * ind_string_parts;
-}
-
-%destructor { delete $$; } <e>
-%destructor { delete $$; } <list>
-%destructor { delete $$; } <attrs>
-%destructor { delete $$; } <formals>
-%destructor { delete $$; } <formal>
-%destructor { delete $$; } <attrNames>
-%destructor { delete $$; } <inheritAttrs>
-%destructor { delete $$; } <string_parts>
-%destructor { delete $$; } <ind_string_parts>
-
-%type <e> start
-%type <e> expr expr_function expr_if expr_op
-%type <e> expr_select expr_simple expr_app
-%type <list> expr_list
-%type <attrs> binds
-%type <formals> formals
-%type <formal> formal
-%type <attrNames> attrpath
-%type <inheritAttrs> attrs
-%type <string_parts> string_parts_interpolated
-%type <ind_string_parts> ind_string_parts
-%type <e> path_start string_parts string_attr
-%type <id> attr
-%token <id> ID
-%token <str> STR IND_STR
-%token <n> INT
-%token <nf> FLOAT
-%token <path> PATH HPATH SPATH PATH_END
-%token <uri> URI
-%token IF THEN ELSE ASSERT WITH LET IN REC INHERIT EQ NEQ AND OR IMPL OR_KW
-%token DOLLAR_CURLY /* == ${ */
-%token IND_STRING_OPEN IND_STRING_CLOSE
-%token ELLIPSIS
-
-%right IMPL
-%left OR
-%left AND
-%nonassoc EQ NEQ
-%nonassoc '<' '>' LEQ GEQ
-%right UPDATE
-%left NOT
-%left '+' '-'
-%left '*' '/'
-%right CONCAT
-%nonassoc '?'
-%nonassoc NEGATE
-
-%%
-
-start: expr { state->result = $1; $$ = 0; };
-
-expr: expr_function;
-
-expr_function
-  : ID ':' expr_function
-    { $$ = new ExprLambda(CUR_POS, state->symbols.create($1), nullptr, unp($3)); }
-  | '{' formals '}' ':' expr_function
-    { if (auto e = state->validateFormals($2)) THROW(*e);
-      $$ = new ExprLambda(CUR_POS, unp($2), unp($5));
-    }
-  | '{' formals '}' '@' ID ':' expr_function
-    {
-      auto arg = state->symbols.create($5);
-      if (auto e = state->validateFormals($2, CUR_POS, arg)) THROW(*e, $2, $7);
-      $$ = new ExprLambda(CUR_POS, arg, unp($2), unp($7));
-    }
-  | ID '@' '{' formals '}' ':' expr_function
-    {
-      auto arg = state->symbols.create($1);
-      if (auto e = state->validateFormals($4, CUR_POS, arg)) THROW(*e, $4, $7);
-      $$ = new ExprLambda(CUR_POS, arg, unp($4), unp($7));
-    }
-  | ASSERT expr ';' expr_function
-    { $$ = new ExprAssert(CUR_POS, unp($2), unp($4)); }
-  | WITH expr ';' expr_function
-    { $$ = new ExprWith(CUR_POS, unp($2), unp($4)); }
-  | LET binds IN expr_function
-    { if (!$2->dynamicAttrs.empty())
-        THROW(ParseError({
-            .msg = HintFmt("dynamic attributes not allowed in let"),
-            .pos = state->positions[CUR_POS]
-        }), $2, $4);
-      $$ = new ExprLet(unp($2), unp($4));
-    }
-  | expr_if
-  ;
-
-expr_if
-  : IF expr THEN expr ELSE expr { $$ = new ExprIf(CUR_POS, unp($2), unp($4), unp($6)); }
-  | expr_op
-  ;
-
-expr_op
-  : '!' expr_op %prec NOT { $$ = new ExprOpNot(unp($2)); }
-  | '-' expr_op %prec NEGATE { $$ = new ExprCall(CUR_POS, std::make_unique<ExprVar>(state->s.sub), vec(std::make_unique<ExprInt>(0), unp($2))); }
-  | expr_op EQ expr_op { $$ = new ExprOpEq(unp($1), unp($3)); }
-  | expr_op NEQ expr_op { $$ = new ExprOpNEq(unp($1), unp($3)); }
-  | expr_op '<' expr_op { $$ = new ExprCall(state->at(@2), std::make_unique<ExprVar>(state->s.lessThan), vec($1, $3)); }
-  | expr_op LEQ expr_op { $$ = new ExprOpNot(std::make_unique<ExprCall>(state->at(@2), std::make_unique<ExprVar>(state->s.lessThan), vec($3, $1))); }
-  | expr_op '>' expr_op { $$ = new ExprCall(state->at(@2), std::make_unique<ExprVar>(state->s.lessThan), vec($3, $1)); }
-  | expr_op GEQ expr_op { $$ = new ExprOpNot(std::make_unique<ExprCall>(state->at(@2), std::make_unique<ExprVar>(state->s.lessThan), vec($1, $3))); }
-  | expr_op AND expr_op { $$ = new ExprOpAnd(state->at(@2), unp($1), unp($3)); }
-  | expr_op OR expr_op { $$ = new ExprOpOr(state->at(@2), unp($1), unp($3)); }
-  | expr_op IMPL expr_op { $$ = new ExprOpImpl(state->at(@2), unp($1), unp($3)); }
-  | expr_op UPDATE expr_op { $$ = new ExprOpUpdate(state->at(@2), unp($1), unp($3)); }
-  | expr_op '?' attrpath { $$ = new ExprOpHasAttr(unp($1), std::move(*$3)); delete $3; }
-  | expr_op '+' expr_op
-    { $$ = new ExprConcatStrings(state->at(@2), false, vec<std::pair<PosIdx, std::unique_ptr<Expr>>>(std::pair(state->at(@1), unp($1)), std::pair(state->at(@3), unp($3)))); }
-  | expr_op '-' expr_op { $$ = new ExprCall(state->at(@2), std::make_unique<ExprVar>(state->s.sub), vec($1, $3)); }
-  | expr_op '*' expr_op { $$ = new ExprCall(state->at(@2), std::make_unique<ExprVar>(state->s.mul), vec($1, $3)); }
-  | expr_op '/' expr_op { $$ = new ExprCall(state->at(@2), std::make_unique<ExprVar>(state->s.div), vec($1, $3)); }
-  | expr_op CONCAT expr_op { $$ = new ExprOpConcatLists(state->at(@2), unp($1), unp($3)); }
-  | expr_app
-  ;
-
-expr_app
-  : expr_app expr_select {
-      if (auto e2 = dynamic_cast<ExprCall *>($1)) {
-          e2->args.emplace_back($2);
-          $$ = $1;
-      } else
-          $$ = new ExprCall(CUR_POS, unp($1), vec(unp($2)));
-  }
-  | expr_select
-  ;
-
-expr_select
-  : expr_simple '.' attrpath
-    { $$ = new ExprSelect(CUR_POS, unp($1), std::move(*$3), nullptr); delete $3; }
-  | expr_simple '.' attrpath OR_KW expr_select
-    { $$ = new ExprSelect(CUR_POS, unp($1), std::move(*$3), unp($5)); delete $3; }
-  | /* Backwards compatibility: because Nixpkgs has a rarely used
-       function named ‘or’, allow stuff like ‘map or [...]’. */
-    expr_simple OR_KW
-    { $$ = new ExprCall(CUR_POS, unp($1), vec(std::make_unique<ExprVar>(CUR_POS, state->s.or_))); }
-  | expr_simple
-  ;
-
-expr_simple
-  : ID {
-      std::string_view s = "__curPos";
-      if ($1.l == s.size() && strncmp($1.p, s.data(), s.size()) == 0)
-          $$ = new ExprPos(CUR_POS);
-      else
-          $$ = new ExprVar(CUR_POS, state->symbols.create($1));
-  }
-  | INT { $$ = new ExprInt($1); }
-  | FLOAT { $$ = new ExprFloat($1); }
-  | '"' string_parts '"' { $$ = $2; }
-  | IND_STRING_OPEN ind_string_parts IND_STRING_CLOSE {
-      $$ = state->stripIndentation(CUR_POS, std::move(*$2)).release();
-      delete $2;
-  }
-  | path_start PATH_END
-  | path_start string_parts_interpolated PATH_END {
-      $2->emplace($2->begin(), state->at(@1), $1);
-      $$ = new ExprConcatStrings(CUR_POS, false, std::move(*$2));
-      delete $2;
-  }
-  | SPATH {
-      std::string path($1.p + 1, $1.l - 2);
-      $$ = new ExprCall(CUR_POS,
-          std::make_unique<ExprVar>(state->s.findFile),
-          vec(std::make_unique<ExprVar>(state->s.nixPath),
-              std::make_unique<ExprString>(std::move(path))));
-  }
-  | URI {
-      static bool noURLLiterals = experimentalFeatureSettings.isEnabled(Xp::NoUrlLiterals);
-      if (noURLLiterals)
-          THROW(ParseError({
-              .msg = HintFmt("URL literals are disabled"),
-              .pos = state->positions[CUR_POS]
-          }));
-      $$ = new ExprString(std::string($1));
-  }
-  | '(' expr ')' { $$ = $2; }
-  /* Let expressions `let {..., body = ...}' are just desugared
-     into `(rec {..., body = ...}).body'. */
-  | LET '{' binds '}'
-    { $3->recursive = true; $$ = new ExprSelect(noPos, unp($3), state->s.body); }
-  | REC '{' binds '}'
-    { $3->recursive = true; $$ = $3; }
-  | '{' binds '}'
-    { $$ = $2; }
-  | '[' expr_list ']' { $$ = $2; }
-  ;
-
-string_parts
-  : STR { $$ = new ExprString(std::string($1)); }
-  | string_parts_interpolated
-    { $$ = new ExprConcatStrings(CUR_POS, true, std::move(*$1));
-      delete $1;
-    }
-  | { $$ = new ExprString(""); }
-  ;
-
-string_parts_interpolated
-  : string_parts_interpolated STR
-  { $$ = $1; $1->emplace_back(state->at(@2), new ExprString(std::string($2))); }
-  | string_parts_interpolated DOLLAR_CURLY expr '}' { $$ = $1; $1->emplace_back(state->at(@2), $3); }
-  | DOLLAR_CURLY expr '}' { $$ = new std::vector<std::pair<PosIdx, std::unique_ptr<Expr>>>; $$->emplace_back(state->at(@1), $2); }
-  | STR DOLLAR_CURLY expr '}' {
-      $$ = new std::vector<std::pair<PosIdx, std::unique_ptr<Expr>>>;
-      $$->emplace_back(state->at(@1), new ExprString(std::string($1)));
-      $$->emplace_back(state->at(@2), $3);
-    }
-  ;
-
-path_start
-  : PATH {
-    Path path(absPath({$1.p, $1.l}, state->basePath.path.abs()));
-    /* add back in the trailing '/' to the first segment */
-    if ($1.p[$1.l-1] == '/' && $1.l > 1)
-      path += "/";
-    $$ = new ExprPath(path);
-  }
-  | HPATH {
-    if (evalSettings.pureEval) {
-        THROW(Error(
-            "the path '%s' can not be resolved in pure mode",
-            std::string_view($1.p, $1.l)
-        ));
-    }
-    Path path(getHome() + std::string($1.p + 1, $1.l - 1));
-    $$ = new ExprPath(path);
-  }
-  ;
-
-ind_string_parts
-  : ind_string_parts IND_STR { $$ = $1; $1->emplace_back(state->at(@2), $2); }
-  | ind_string_parts DOLLAR_CURLY expr '}' { $$ = $1; $1->emplace_back(state->at(@2), unp($3)); }
-  | { $$ = new std::vector<std::pair<PosIdx, std::variant<std::unique_ptr<Expr>, StringToken>>>; }
-  ;
-
-binds
-  : binds attrpath '=' expr ';'
-    { $$ = $1;
-      if (auto e = state->addAttr($$, std::move(*$2), unp($4), state->at(@2))) THROW(*e, $1, $2);
-      delete $2;
-    }
-  | binds INHERIT attrs ';'
-    { $$ = $1;
-      for (auto & [i, iPos] : *$3) {
-          if ($$->attrs.find(i.symbol) != $$->attrs.end())
-              THROW(state->dupAttr(i.symbol, iPos, $$->attrs[i.symbol].pos), $1);
-          $$->attrs.emplace(
-              i.symbol,
-              ExprAttrs::AttrDef(std::make_unique<ExprVar>(iPos, i.symbol), iPos, ExprAttrs::AttrDef::Kind::Inherited));
-      }
-      delete $3;
-    }
-  | binds INHERIT '(' expr ')' attrs ';'
-    { $$ = $1;
-      if (!$$->inheritFromExprs)
-          $$->inheritFromExprs = std::make_unique<std::vector<std::unique_ptr<Expr>>>();
-      $$->inheritFromExprs->push_back(unp($4));
-      for (auto & [i, iPos] : *$6) {
-          if ($$->attrs.find(i.symbol) != $$->attrs.end())
-              THROW(state->dupAttr(i.symbol, iPos, $$->attrs[i.symbol].pos), $1);
-          auto from = std::make_unique<nix::ExprInheritFrom>(state->at(@4), $$->inheritFromExprs->size() - 1);
-          $$->attrs.emplace(
-              i.symbol,
-              ExprAttrs::AttrDef(
-                  std::make_unique<ExprSelect>(iPos, std::move(from), i.symbol),
-                  iPos,
-                  ExprAttrs::AttrDef::Kind::InheritedFrom));
-      }
-      delete $6;
-    }
-  | { $$ = new ExprAttrs(state->at(@0)); }
-  ;
-
-attrs
-  : attrs attr { $$ = $1; $1->emplace_back(AttrName(state->symbols.create($2)), state->at(@2)); }
-  | attrs string_attr
-    { $$ = $1;
-      ExprString * str = dynamic_cast<ExprString *>($2);
-      if (str) {
-          $$->emplace_back(AttrName(state->symbols.create(str->s)), state->at(@2));
-          delete str;
-      } else
-          THROW(ParseError({
-              .msg = HintFmt("dynamic attributes not allowed in inherit"),
-              .pos = state->positions[state->at(@2)]
-          }), $1, $2);
-    }
-  | { $$ = new std::vector<std::pair<AttrName, PosIdx>>; }
-  ;
-
-attrpath
-  : attrpath '.' attr { $$ = $1; $1->push_back(AttrName(state->symbols.create($3))); }
-  | attrpath '.' string_attr
-    { $$ = $1;
-      ExprString * str = dynamic_cast<ExprString *>($3);
-      if (str) {
-          $$->push_back(AttrName(state->symbols.create(str->s)));
-          delete str;
-      } else
-          $$->emplace_back(unp($3));
-    }
-  | attr { $$ = new std::vector<AttrName>; $$->push_back(AttrName(state->symbols.create($1))); }
-  | string_attr
-    { $$ = new std::vector<AttrName>;
-      ExprString *str = dynamic_cast<ExprString *>($1);
-      if (str) {
-          $$->push_back(AttrName(state->symbols.create(str->s)));
-          delete str;
-      } else
-          $$->emplace_back(unp($1));
-    }
-  ;
-
-attr
-  : ID
-  | OR_KW { $$ = {"or", 2}; }
-  ;
-
-string_attr
-  : '"' string_parts '"' { $$ = $2; }
-  | DOLLAR_CURLY expr '}' { $$ = $2; }
-  ;
-
-expr_list
-  : expr_list expr_select { $$ = $1; $1->elems.emplace_back($2); /* !!! dangerous */ }
-  | { $$ = new ExprList; }
-  ;
-
-formals
-  : formal ',' formals
-    { $$ = $3; $$->formals.emplace_back(std::move(*$1)); delete $1; }
-  | formal
-    { $$ = new Formals; $$->formals.emplace_back(std::move(*$1)); $$->ellipsis = false; delete $1; }
-  |
-    { $$ = new Formals; $$->ellipsis = false; }
-  | ELLIPSIS
-    { $$ = new Formals; $$->ellipsis = true; }
-  ;
-
-formal
-  : ID { $$ = new Formal{CUR_POS, state->symbols.create($1), nullptr}; }
-  | ID '?' expr { $$ = new Formal{CUR_POS, state->symbols.create($1), unp($3)}; }
-  ;
-
-%%
-
-#include "eval.hh"
-
-
-namespace nix {
-
-Expr * parseExprFromBuf(
-    char * text,
-    size_t length,
-    Pos::Origin origin,
-    const SourcePath & basePath,
-    SymbolTable & symbols,
-    PosTable & positions,
-    const Expr::AstSymbols & astSymbols)
-{
-    yyscan_t scanner;
-    ParserState state {
-        .symbols = symbols,
-        .positions = positions,
-        .basePath = basePath,
-        .origin = positions.addOrigin(origin, length),
-        .s = astSymbols,
-    };
-
-    yylex_init(&scanner);
-    Finally _destroy([&] { yylex_destroy(scanner); });
-
-    yy_scan_buffer(text, length, scanner);
-    yyparse(scanner, &state);
-    if (state.error) {
-      delete state.result;
-      throw *state.error;
-    }
-
-    return state.result;
-}
-
-
-}

src/libexpr/parser/change_head.hh

@@ -0,0 +1,66 @@
+#pragma once
+///@file
+
+#include <tao/pegtl.hpp>
+
+namespace nix::parser {
+
+// modified copy of change_state, as the manual suggest for more involved
+// state manipulation. we want to change only the first state parameter,
+// and we care about the *initial* position of a rule application (not the
+// past-the-end position as pegtl change_state provides)
+template<typename NewState>
+struct change_head : tao::pegtl::maybe_nothing
+{
+    template<
+        typename Rule,
+        tao::pegtl::apply_mode A,
+        tao::pegtl::rewind_mode M,
+        template<typename...> class Action,
+        template<typename...> class Control,
+        typename ParseInput,
+        typename State,
+        typename... States
+    >
+    [[nodiscard]] static bool match(ParseInput & in, State && st, States &&... sts)
+    {
+        const auto begin = in.iterator();
+
+        if constexpr (std::is_constructible_v<NewState, State, States...>) {
+            NewState s(st, sts...);
+            if (tao::pegtl::match<Rule, A, M, Action, Control>(in, s, sts...)) {
+                if constexpr (A == tao::pegtl::apply_mode::action) {
+                    _success<Action<Rule>>(0, begin, in, s, st, sts...);
+                }
+                return true;
+            }
+            return false;
+        } else if constexpr (std::is_default_constructible_v<NewState>) {
+            NewState s;
+            if (tao::pegtl::match<Rule, A, M, Action, Control>(in, s, sts...)) {
+                if constexpr (A == tao::pegtl::apply_mode::action) {
+                    _success<Action<Rule>>(0, begin, in, s, st, sts...);
+                }
+                return true;
+            }
+            return false;
+        } else {
+            static_assert(decltype(sizeof(NewState))(), "unable to instantiate new state");
+        }
+    }
+
+    template<typename Target, typename ParseInput, typename... S>
+    static void _success(void *, auto & begin, ParseInput & in, S & ... sts)
+    {
+        const typename ParseInput::action_t at(begin, in);
+        Target::success(at, sts...);
+    }
+
+    template<typename Target, typename... S>
+    static void _success(decltype(Target::success0(std::declval<S &>()...), 0), auto &, auto &, S & ... sts)
+    {
+        Target::success0(sts...);
+    }
+};
+
+}

src/libexpr/parser/grammar.hh

@@ -0,0 +1,707 @@
+#pragma once
+///@file
+
+#include "tao/pegtl.hpp"
+#include <type_traits>
+#include <variant>
+
+#include <boost/container/small_vector.hpp>
+
+// NOTE
+// nix line endings are \n, \r\n, \r. the grammar does not use eol or
+// eolf rules in favor of reproducing the old flex lexer as faithfully as
+// possible, and deferring calculation of positions to downstream users.
+
+namespace nix::parser::grammar {
+
+using namespace tao::pegtl;
+namespace p = tao::pegtl;
+
+// character classes
+namespace c {
+
+struct path : sor<
+    ranges<'a', 'z', 'A', 'Z', '0', '9'>,
+    one<'.', '_', '-', '+'>
+> {};
+struct path_sep : one<'/'> {};
+
+struct id_first : ranges<'a', 'z', 'A', 'Z', '_'> {};
+struct id_rest : sor<
+    ranges<'a', 'z', 'A', 'Z', '0', '9'>,
+    one<'_', '\'', '-'>
+> {};
+
+struct uri_scheme_first : ranges<'a', 'z', 'A', 'Z'> {};
+struct uri_scheme_rest : sor<
+    ranges<'a', 'z', 'A', 'Z', '0', '9'>,
+    one<'+', '-', '.'>
+> {};
+struct uri_sep : one<':'> {};
+struct uri_rest : sor<
+    ranges<'a', 'z', 'A', 'Z', '0', '9'>,
+    one<'%', '/', '?', ':', '@', '&', '=', '+', '$', ',', '-', '_', '.', '!', '~', '*', '\''>
+> {};
+
+}
+
+// "tokens". PEGs don't really care about tokens, we merely use them as a convenient
+// way of writing down keywords and a couple complicated syntax rules.
+namespace t {
+
+struct _extend_as_path : seq<
+    star<c::path>,
+    not_at<TAO_PEGTL_STRING("/*")>,
+    not_at<TAO_PEGTL_STRING("//")>,
+    c::path_sep,
+    sor<c::path, TAO_PEGTL_STRING("${")>
+> {};
+struct _extend_as_uri : seq<
+    star<c::uri_scheme_rest>,
+    c::uri_sep,
+    c::uri_rest
+> {};
+
+// keywords might be extended to identifiers, paths, or uris.
+// NOTE this assumes that keywords are a-zA-Z only, otherwise uri schemes would never
+// match correctly.
+// NOTE not a simple seq<...> because this would report incorrect positions for
+// keywords used inside must<> if a prefix of the keyword matches.
+template<typename S>
+struct _keyword : sor<
+    seq<
+        S,
+        not_at<c::id_rest>,
+        not_at<_extend_as_path>,
+        not_at<_extend_as_uri>
+    >,
+    failure
+> {};
+
+struct kw_if      : _keyword<TAO_PEGTL_STRING("if")> {};
+struct kw_then    : _keyword<TAO_PEGTL_STRING("then")> {};
+struct kw_else    : _keyword<TAO_PEGTL_STRING("else")> {};
+struct kw_assert  : _keyword<TAO_PEGTL_STRING("assert")> {};
+struct kw_with    : _keyword<TAO_PEGTL_STRING("with")> {};
+struct kw_let     : _keyword<TAO_PEGTL_STRING("let")> {};
+struct kw_in      : _keyword<TAO_PEGTL_STRING("in")> {};
+struct kw_rec     : _keyword<TAO_PEGTL_STRING("rec")> {};
+struct kw_inherit : _keyword<TAO_PEGTL_STRING("inherit")> {};
+struct kw_or      : _keyword<TAO_PEGTL_STRING("or")> {};
+
+// `-` can be a unary prefix op, a binary infix op, or the first character
+// of a path or -> (ex 1->1--1)
+// `/` can be a path leader or an operator (ex a?a /a)
+struct op_minus : seq<one<'-'>, not_at<one<'>'>>, not_at<_extend_as_path>> {};
+struct op_div   : seq<one<'/'>, not_at<c::path>> {};
+
+// match a rule, making sure we are not matching it where a keyword would match.
+// using minus like this is a lot faster than flipping the order and using seq.
+template<typename... Rules>
+struct _not_at_any_keyword : minus<
+    seq<Rules...>,
+    sor<
+        TAO_PEGTL_STRING("inherit"),
+        TAO_PEGTL_STRING("assert"),
+        TAO_PEGTL_STRING("else"),
+        TAO_PEGTL_STRING("then"),
+        TAO_PEGTL_STRING("with"),
+        TAO_PEGTL_STRING("let"),
+        TAO_PEGTL_STRING("rec"),
+        TAO_PEGTL_STRING("if"),
+        TAO_PEGTL_STRING("in"),
+        TAO_PEGTL_STRING("or")
+    >
+> {};
+
+// identifiers are kind of horrid:
+//
+//   - uri_scheme_first ⊂ id_first
+//   - uri_scheme_first ⊂ uri_scheme_rest ⊂ path
+//   - id_first ⊂ id_rest ∖ { ' } ⊂ path
+//   - id_first ∩ (path ∖ uri_scheme_first) = { _ }
+//   - uri_sep ∉ ⋃ { id_first, id_rest, uri_scheme_first, uri_scheme_rest, path }
+//   - path_sep ∉ ⋃ { id_first, id_rest, uri_scheme_first, uri_scheme_rest }
+//
+// and we want, without reading the input more than once, a string that
+// matches (id_first id_rest*) and is not followed by any number of
+// characters such that the extended string matches path or uri rules.
+//
+// since the first character must be either _ or a uri scheme character
+// we can ignore path-like bits at the beginning. uri_sep cannot appear anywhere
+// in an identifier, so it's only needed in lookahead checks at the uri-like
+// prefix. likewise path_sep cannot appear anywhere in the idenfier, so it's
+// only needed in lookahead checks in the path-like prefix.
+//
+// in total that gives us a decomposition of
+//
+//     (uri-scheme-like? (?! continues-as-uri) | _)
+//     (path-segment-like? (?! continues-as-path))
+//     id_rest*
+struct identifier : _not_at_any_keyword<
+    // we don't use (at<id_rest>, ...) matches here because identifiers are
+    // a really hot path and rewinding as needed by at<> isn't entirely free.
+    sor<
+        seq<
+            c::uri_scheme_first,
+            star<ranges<'a', 'z', 'A', 'Z', '0', '9', '-'>>,
+            not_at<_extend_as_uri>
+        >,
+        one<'_'>
+    >,
+    star<sor<ranges<'a', 'z', 'A', 'Z', '0', '9'>, one<'_', '-'>>>,
+    not_at<_extend_as_path>,
+    star<c::id_rest>
+> {};
+
+// floats may extend ints, thus these rules are very similar.
+struct integer : seq<
+    sor<
+        seq<range<'1', '9'>, star<digit>, not_at<one<'.'>>>,
+        seq<one<'0'>, not_at<one<'.'>, digit>, star<digit>>
+    >,
+    not_at<_extend_as_path>
+> {};
+
+struct floating : seq<
+    sor<
+        seq<range<'1', '9'>, star<digit>, one<'.'>, star<digit>>,
+        seq<opt<one<'0'>>, one<'.'>, plus<digit>>
+    >,
+    opt<one<'E', 'e'>, opt<one<'+', '-'>>, plus<digit>>,
+    not_at<_extend_as_path>
+> {};
+
+struct uri : seq<
+    c::uri_scheme_first,
+    star<c::uri_scheme_rest>,
+    c::uri_sep,
+    plus<c::uri_rest>
+> {};
+
+struct sep : sor<
+    plus<one<' ', '\t', '\r', '\n'>>,
+    seq<one<'#'>, star<not_one<'\r', '\n'>>>,
+    seq<string<'/', '*'>, until<string<'*', '/'>>>
+> {};
+
+}
+
+
+
+using seps = star<t::sep>;
+
+
+// marker for semantic rules. not handling one of these in an action that cares about
+// semantics is probably an error.
+struct semantic {};
+
+
+struct expr;
+
+struct _string {
+    template<typename... Inner>
+    struct literal : semantic, seq<Inner...> {};
+    struct cr_lf : semantic, seq<one<'\r'>, opt<one<'\n'>>> {};
+    struct interpolation : semantic, seq<
+        p::string<'$', '{'>, seps,
+        must<expr>, seps,
+        must<one<'}'>>
+    > {};
+    struct escape : semantic, must<any> {};
+};
+struct string : _string, seq<
+    one<'"'>,
+    star<
+        sor<
+            _string::literal<plus<not_one<'$', '"', '\\', '\r'>>>,
+            _string::cr_lf,
+            _string::interpolation,
+            _string::literal<one<'$'>, opt<one<'$'>>>,
+            seq<one<'\\'>, _string::escape>
+        >
+    >,
+    must<one<'"'>>
+> {};
+
+struct _ind_string {
+    template<bool Indented, typename... Inner>
+    struct literal : semantic, seq<Inner...> {};
+    struct interpolation : semantic, seq<
+        p::string<'$', '{'>, seps,
+        must<expr>, seps,
+        must<one<'}'>>
+    > {};
+    struct escape : semantic, must<any> {};
+};
+struct ind_string : _ind_string, seq<
+    TAO_PEGTL_STRING("''"),
+    opt<star<one<' '>>, one<'\n'>>,
+    star<
+        sor<
+            _ind_string::literal<
+                true,
+                plus<
+                    sor<
+                        not_one<'$', '\''>,
+                        seq<one<'$'>, not_one<'{', '\''>>,
+                        seq<one<'\''>, not_one<'\'', '$'>>
+                    >
+                >
+            >,
+            _ind_string::interpolation,
+            _ind_string::literal<false, one<'$'>>,
+            _ind_string::literal<false, one<'\''>, not_at<one<'\''>>>,
+            seq<one<'\''>, _ind_string::literal<false, p::string<'\'', '\''>>>,
+            seq<
+                p::string<'\'', '\''>,
+                sor<
+                    _ind_string::literal<false, one<'$'>>,
+                    seq<one<'\\'>, _ind_string::escape>
+                >
+            >
+        >
+    >,
+    must<TAO_PEGTL_STRING("''")>
+> {};
+
+struct _path {
+    // legacy lexer rules. extra l_ to avoid reserved c++ identifiers.
+    struct _l_PATH : seq<star<c::path>, plus<c::path_sep, plus<c::path>>, opt<c::path_sep>> {};
+    struct _l_PATH_SEG : seq<star<c::path>, c::path_sep> {};
+    struct _l_HPATH : seq<one<'~'>, plus<c::path_sep, plus<c::path>>, opt<c::path_sep>> {};
+    struct _l_HPATH_START : TAO_PEGTL_STRING("~/") {};
+    struct _path_str : sor<_l_PATH, _l_PATH_SEG, plus<c::path>> {};
+    // modern rules
+    template<typename... Inner>
+    struct literal : semantic, seq<Inner...> {};
+    struct interpolation : semantic, seq<
+        p::string<'$', '{'>, seps,
+        must<expr>, seps,
+        must<one<'}'>>
+    > {};
+    struct anchor : semantic, sor<
+        _l_PATH,
+        seq<_l_PATH_SEG, at<TAO_PEGTL_STRING("${")>>
+    > {};
+    struct home_anchor : semantic, sor<
+        _l_HPATH,
+        seq<_l_HPATH_START, at<TAO_PEGTL_STRING("${")>>
+    > {};
+    struct searched_path : semantic, list<plus<c::path>, c::path_sep> {};
+    struct forbid_prefix_triple_slash : sor<not_at<c::path_sep>, failure> {};
+    struct forbid_prefix_double_slash_no_interp : sor<
+        not_at<c::path_sep, star<c::path>, not_at<TAO_PEGTL_STRING("${")>>,
+        failure
+    > {};
+    // legacy parser rules
+    struct _str_rest : seq<
+        must<forbid_prefix_double_slash_no_interp>,
+        opt<literal<_path_str>>,
+        must<forbid_prefix_triple_slash>,
+        star<
+            sor<
+                literal<_path_str>,
+                interpolation
+            >
+        >
+    > {};
+};
+struct path : _path, sor<
+    seq<
+        sor<_path::anchor, _path::home_anchor>,
+        _path::_str_rest
+    >,
+    seq<one<'<'>, _path::searched_path, one<'>'>>
+> {};
+
+struct _formal {
+    struct name : semantic, t::identifier {};
+    struct default_value : semantic, must<expr> {};
+};
+struct formal : semantic, _formal, seq<
+    _formal::name,
+    opt<seps, one<'?'>, seps, _formal::default_value>
+> {};
+
+struct _formals {
+    struct ellipsis : semantic, p::ellipsis {};
+};
+struct formals : semantic, _formals, seq<
+    one<'{'>, seps,
+    // formals and attrsets share a two-token head sequence ('{' <id>).
+    // this rule unrolls the formals list a bit to provide better error messages than
+    // "expected '='" at the first ',' if formals are incorrect.
+    sor<
+        one<'}'>,
+        seq<_formals::ellipsis, seps, must<one<'}'>>>,
+        seq<
+            formal, seps,
+            if_then_else<
+                at<one<','>>,
+                seq<
+                    star<one<','>, seps, formal, seps>,
+                    opt<one<','>, seps, opt<_formals::ellipsis, seps>>,
+                    must<one<'}'>>
+                >,
+                one<'}'>
+            >
+        >
+    >
+> {};
+
+struct _attr {
+    struct simple : semantic, sor<t::identifier, t::kw_or> {};
+    struct string : semantic, seq<grammar::string> {};
+    struct expr : semantic, seq<
+        TAO_PEGTL_STRING("${"), seps,
+        must<grammar::expr>, seps,
+        must<one<'}'>>
+    > {};
+};
+struct attr : _attr, sor<
+    _attr::simple,
+    _attr::string,
+    _attr::expr
+> {};
+
+struct attrpath : list<attr, one<'.'>, t::sep> {};
+
+struct _inherit {
+    struct from : semantic, must<expr> {};
+    struct attrs : list<attr, seps> {};
+};
+struct inherit : _inherit, seq<
+    t::kw_inherit, seps,
+    opt<one<'('>, seps, _inherit::from, seps, must<one<')'>>, seps>,
+    opt<_inherit::attrs, seps>,
+    must<one<';'>>
+> {};
+
+struct _binding {
+    struct path : semantic, attrpath {};
+    struct equal : one<'='> {};
+    struct value : semantic, must<expr> {};
+};
+struct binding : _binding, seq<
+    _binding::path, seps,
+    must<_binding::equal>, seps,
+    _binding::value, seps,
+    must<one<';'>>
+> {};
+
+struct bindings : opt<list<sor<inherit, binding>, seps>> {};
+
+struct op {
+    enum class kind {
+        // NOTE non-associativity is *NOT* handled in the grammar structure.
+        // handling it in the grammar itself instead of in semantic actions
+        // slows down the parser significantly and makes the rules *much*
+        // harder to read. maybe this will be different at some point when
+        // ! does not sit between two binary precedence levels.
+        nonAssoc,
+        leftAssoc,
+        rightAssoc,
+        unary,
+    };
+    template<typename Rule, unsigned Precedence, kind Kind = kind::leftAssoc>
+    struct _op : Rule {
+        static constexpr unsigned precedence = Precedence;
+        static constexpr op::kind kind = Kind;
+    };
+
+    struct unary_minus : _op<t::op_minus,           3, kind::unary> {};
+
+    // treating this like a unary postfix operator is sketchy, but that's
+    // the most reasonable way to implement the operator precedence set forth
+    // by the language way back. it'd be much better if `.` and `?` had the same
+    // precedence, but alas.
+    struct has_attr   : _op<seq<one<'?'>, seps, must<attrpath>>, 4> {};
+
+    struct concat     : _op<TAO_PEGTL_STRING("++"),  5, kind::rightAssoc> {};
+    struct mul        : _op<one<'*'>,                6> {};
+    struct div        : _op<t::op_div,               6> {};
+    struct plus       : _op<one<'+'>,                7> {};
+    struct minus      : _op<t::op_minus,             7> {};
+    struct not_       : _op<one<'!'>,                8, kind::unary> {};
+    struct update     : _op<TAO_PEGTL_STRING("//"),  9, kind::rightAssoc> {};
+    struct less_eq    : _op<TAO_PEGTL_STRING("<="), 10, kind::nonAssoc> {};
+    struct greater_eq : _op<TAO_PEGTL_STRING(">="), 10, kind::nonAssoc> {};
+    struct less       : _op<one<'<'>,               10, kind::nonAssoc> {};
+    struct greater    : _op<one<'>'>,               10, kind::nonAssoc> {};
+    struct equals     : _op<TAO_PEGTL_STRING("=="), 11, kind::nonAssoc> {};
+    struct not_equals : _op<TAO_PEGTL_STRING("!="), 11, kind::nonAssoc> {};
+    struct and_       : _op<TAO_PEGTL_STRING("&&"), 12> {};
+    struct or_        : _op<TAO_PEGTL_STRING("||"), 13> {};
+    struct implies    : _op<TAO_PEGTL_STRING("->"), 14, kind::rightAssoc> {};
+};
+
+struct _expr {
+    template<template<typename...> class OpenMod = seq, typename... Init>
+    struct _attrset : seq<
+        Init...,
+        OpenMod<one<'{'>>, seps,
+        bindings, seps,
+        must<one<'}'>>
+    > {};
+
+    struct select;
+
+    struct id : semantic, t::identifier {};
+    struct int_ : semantic, t::integer {};
+    struct float_ : semantic, t::floating {};
+    struct string : semantic, seq<grammar::string> {};
+    struct ind_string : semantic, seq<grammar::ind_string> {};
+    struct path : semantic, seq<grammar::path> {};
+    struct uri : semantic, t::uri {};
+    struct ancient_let : semantic, _attrset<must, t::kw_let, seps> {};
+    struct rec_set : semantic, _attrset<must, t::kw_rec, seps> {};
+    struct set : semantic, _attrset<> {};
+
+    struct _list {
+        struct entry : semantic, seq<select> {};
+    };
+    struct list : semantic, _list, seq<
+        one<'['>, seps,
+        opt<p::list<_list::entry, seps>, seps>,
+        must<one<']'>>
+    > {};
+
+    struct _simple : sor<
+        id,
+        int_,
+        float_,
+        string,
+        ind_string,
+        path,
+        uri,
+        seq<one<'('>, seps, must<expr>, seps, must<one<')'>>>,
+        ancient_let,
+        rec_set,
+        set,
+        list
+    > {};
+
+    struct _select {
+        struct head : _simple {};
+        struct attr : semantic, seq<attrpath> {};
+        struct attr_or : semantic, must<select> {};
+        struct as_app_or : semantic, t::kw_or {};
+    };
+    struct _app {
+        struct first_arg : semantic, seq<select> {};
+        struct another_arg : semantic, seq<select> {};
+        // can be used to stash a position of the application head node
+        struct select_or_fn : seq<select> {};
+    };
+
+    struct select : _select, seq<
+        _select::head, seps,
+        opt<
+            sor<
+                seq<
+                    one<'.'>, seps, _select::attr,
+                    opt<seps, t::kw_or, seps, _select::attr_or>
+                >,
+                _select::as_app_or
+            >
+        >
+    > {};
+
+    struct app : _app, seq<
+        _app::select_or_fn,
+        opt<seps, _app::first_arg, star<seps, _app::another_arg>>
+    > {};
+
+    template<typename Op>
+    struct operator_ : semantic, Op {};
+
+    struct unary : seq<
+        star<sor<operator_<op::not_>, operator_<op::unary_minus>>, seps>,
+        app
+    > {};
+
+    struct _binary_operator : sor<
+        operator_<op::implies>,
+        operator_<op::update>,
+        operator_<op::concat>,
+        operator_<op::plus>,
+        operator_<op::minus>,
+        operator_<op::mul>,
+        operator_<op::div>,
+        operator_<op::less_eq>,
+        operator_<op::greater_eq>,
+        operator_<op::less>,
+        operator_<op::greater>,
+        operator_<op::equals>,
+        operator_<op::not_equals>,
+        operator_<op::or_>,
+        operator_<op::and_>
+    > {};
+
+    struct _binop : seq<
+        unary,
+        star<
+            seps,
+            sor<
+                seq<_binary_operator, seps, must<unary>>,
+                operator_<op::has_attr>
+            >
+        >
+    > {};
+
+    struct _lambda {
+        struct arg : semantic, t::identifier {};
+    };
+    struct lambda : semantic, _lambda, sor<
+        seq<
+            _lambda::arg, seps,
+            sor<
+                seq<one<':'>, seps, must<expr>>,
+                seq<one<'@'>, seps, must<formals, seps, one<':'>, seps, expr>>
+            >
+        >,
+        seq<
+            formals, seps,
+            sor<
+                seq<one<':'>, seps, must<expr>>,
+                seq<one<'@'>, seps, must<_lambda::arg, seps, one<':'>, seps, expr>>
+            >
+        >
+    > {};
+
+    struct assert_ : semantic, seq<
+        t::kw_assert, seps,
+        must<expr>, seps,
+        must<one<';'>>, seps,
+        must<expr>
+    > {};
+    struct with : semantic, seq<
+        t::kw_with, seps,
+        must<expr>, seps,
+        must<one<';'>>, seps,
+        must<expr>
+    > {};
+    struct let : seq<
+        t::kw_let, seps,
+        not_at<one<'{'>>, // exclude ancient_let so we can must<kw_in>
+        bindings, seps,
+        must<t::kw_in>, seps,
+        must<expr>
+    > {};
+    struct if_ : semantic, seq<
+        t::kw_if, seps,
+        must<expr>, seps,
+        must<t::kw_then>, seps,
+        must<expr>, seps,
+        must<t::kw_else>, seps,
+        must<expr>
+    > {};
+};
+struct expr : semantic, _expr, sor<
+    _expr::lambda,
+    _expr::assert_,
+    _expr::with,
+    _expr::let,
+    _expr::if_,
+    _expr::_binop
+> {};
+
+// legacy support: \0 terminates input if passed from flex to bison as a token
+struct eof : sor<p::eof, one<0>> {};
+
+struct root : must<seps, expr, seps, eof> {};
+
+
+
+template<typename Rule>
+struct nothing : p::nothing<Rule> {
+    static_assert(!std::is_base_of_v<semantic, Rule>);
+};
+
+
+
+template<typename Self, typename OpCtx, typename AttrPathT, typename ExprT>
+struct operator_semantics {
+    struct has_attr : grammar::op::has_attr {
+        AttrPathT path;
+    };
+
+    struct OpEntry {
+        OpCtx ctx;
+        uint8_t prec;
+        grammar::op::kind assoc;
+        std::variant<
+            grammar::op::not_,
+            grammar::op::unary_minus,
+            grammar::op::implies,
+            grammar::op::or_,
+            grammar::op::and_,
+            grammar::op::equals,
+            grammar::op::not_equals,
+            grammar::op::less_eq,
+            grammar::op::greater_eq,
+            grammar::op::update,
+            grammar::op::concat,
+            grammar::op::less,
+            grammar::op::greater,
+            grammar::op::plus,
+            grammar::op::minus,
+            grammar::op::mul,
+            grammar::op::div,
+            has_attr
+        > op;
+    };
+
+    // statistics here are taken from nixpkgs commit de502c4d0ba96261e5de803e4d1d1925afd3e22f.
+    // over 99.9% of contexts in nixpkgs need at most 4 slots, ~85% need only 1
+    boost::container::small_vector<ExprT, 4> exprs;
+    // over 99.9% of contexts in nixpkgs need at most 2 slots, ~85% need only 1
+    boost::container::small_vector<OpEntry, 2> ops;
+
+    // derived class is expected to define members:
+    //
+    // ExprT applyOp(OpCtx & pos, auto & op, auto &... args);
+    // [[noreturn]] static void badOperator(OpCtx & pos, auto &... args);
+
+    void reduce(uint8_t toPrecedence, auto &... args) {
+        while (!ops.empty()) {
+            auto & [ctx, precedence, kind, op] = ops.back();
+            // NOTE this relies on associativity not being mixed within a precedence level.
+            if ((precedence > toPrecedence)
+                || (kind != grammar::op::kind::leftAssoc && precedence == toPrecedence))
+                break;
+            std::visit([&, ctx=std::move(ctx)] (auto & op) {
+                exprs.push_back(static_cast<Self &>(*this).applyOp(ctx, op, args...));
+            }, op);
+            ops.pop_back();
+        }
+    }
+
+    ExprT popExpr()
+    {
+        auto r = std::move(exprs.back());
+        exprs.pop_back();
+        return r;
+    }
+
+    void pushOp(OpCtx ctx, auto o, auto &... args)
+    {
+        if (o.kind != grammar::op::kind::unary)
+            reduce(o.precedence, args...);
+        if (!ops.empty() && o.kind == grammar::op::kind::nonAssoc) {
+            auto & [_pos, _prec, _kind, _o] = ops.back();
+            if (_kind == o.kind && _prec == o.precedence)
+                Self::badOperator(ctx, args...);
+        }
+        ops.emplace_back(ctx, o.precedence, o.kind, std::move(o));
+    }
+
+    ExprT finish(auto &... args)
+    {
+        reduce(255, args...);
+        return popExpr();
+    }
+};
+
+}

src/libexpr/parser/parser.cc

@@ -0,0 +1,862 @@
+#include "attr-set.hh"
+#include "error.hh"
+#include "eval-settings.hh"
+#include "eval.hh"
+#include "finally.hh"
+#include "nixexpr.hh"
+#include "symbol-table.hh"
+#include "users.hh"
+
+#include "change_head.hh"
+#include "grammar.hh"
+#include "state.hh"
+
+#include <charconv>
+#include <clocale>
+#include <memory>
+
+// flip this define when doing parser development to enable some g checks.
+#if 0
+#include <tao/pegtl/contrib/analyze.hpp>
+#define ANALYZE_GRAMMAR \
+    ([] { \
+        const std::size_t issues = tao::pegtl::analyze<grammar::root>(); \
+        assert(issues == 0); \
+    })()
+#else
+#define ANALYZE_GRAMMAR ((void) 0)
+#endif
+
+namespace p = tao::pegtl;
+
+namespace nix::parser {
+namespace {
+
+template<typename>
+inline constexpr const char * error_message = nullptr;
+
+#define error_message_for(...) \
+    template<> inline constexpr auto error_message<__VA_ARGS__>
+
+error_message_for(p::one<'{'>) = "expecting '{'";
+error_message_for(p::one<'}'>) = "expecting '}'";
+error_message_for(p::one<'"'>) = "expecting '\"'";
+error_message_for(p::one<';'>) = "expecting ';'";
+error_message_for(p::one<')'>) = "expecting ')'";
+error_message_for(p::one<'='>) = "expecting '='";
+error_message_for(p::one<']'>) = "expecting ']'";
+error_message_for(p::one<':'>) = "expecting ':'";
+error_message_for(p::string<'\'', '\''>) = "expecting \"''\"";
+error_message_for(p::any) = "expecting any character";
+error_message_for(grammar::eof) = "expecting end of file";
+error_message_for(grammar::seps) = "expecting separators";
+error_message_for(grammar::path::forbid_prefix_triple_slash) = "too many slashes in path";
+error_message_for(grammar::path::forbid_prefix_double_slash_no_interp) = "path has a trailing slash";
+error_message_for(grammar::expr) = "expecting expression";
+error_message_for(grammar::expr::unary) = "expecting expression";
+error_message_for(grammar::binding::equal) = "expecting '='";
+error_message_for(grammar::expr::lambda::arg) = "expecting identifier";
+error_message_for(grammar::formals) = "expecting formals";
+error_message_for(grammar::attrpath) = "expecting attribute path";
+error_message_for(grammar::expr::select) = "expecting selection expression";
+error_message_for(grammar::t::kw_then) = "expecting 'then'";
+error_message_for(grammar::t::kw_else) = "expecting 'else'";
+error_message_for(grammar::t::kw_in) = "expecting 'in'";
+
+struct SyntaxErrors
+{
+    template<typename Rule>
+    static constexpr auto message = error_message<Rule>;
+
+    template<typename Rule>
+    static constexpr bool raise_on_failure = false;
+};
+
+template<typename Rule>
+struct Control : p::must_if<SyntaxErrors>::control<Rule>
+{
+    template<typename ParseInput, typename... States>
+    [[noreturn]] static void raise(const ParseInput & in, States &&... st)
+    {
+        if (in.empty()) {
+            std::string expected;
+            if constexpr (constexpr auto msg = error_message<Rule>)
+                expected = fmt(", %s", msg);
+            throw p::parse_error("unexpected end of file" + expected, in);
+        }
+        p::must_if<SyntaxErrors>::control<Rule>::raise(in, st...);
+    }
+};
+
+struct ExprState
+    : grammar::
+          operator_semantics<ExprState, PosIdx, AttrPath, std::pair<PosIdx, std::unique_ptr<Expr>>>
+{
+    std::unique_ptr<Expr> popExprOnly() {
+        return std::move(popExpr().second);
+    }
+
+    template<typename Op, typename... Args>
+    std::unique_ptr<Expr> applyUnary(Args &&... args) {
+        return std::make_unique<Op>(popExprOnly(), std::forward<Args>(args)...);
+    }
+
+    template<typename Op>
+    std::unique_ptr<Expr> applyBinary(PosIdx pos) {
+        auto right = popExprOnly(), left = popExprOnly();
+        return std::make_unique<Op>(pos, std::move(left), std::move(right));
+    }
+
+    std::unique_ptr<Expr> call(PosIdx pos, Symbol fn, bool flip = false)
+    {
+        std::vector<std::unique_ptr<Expr>> args(2);
+        args[flip ? 0 : 1] = popExprOnly();
+        args[flip ? 1 : 0] = popExprOnly();
+        return std::make_unique<ExprCall>(pos, std::make_unique<ExprVar>(fn), std::move(args));
+    }
+
+    std::unique_ptr<Expr> order(PosIdx pos, bool less, State & state)
+    {
+        return call(pos, state.s.lessThan, !less);
+    }
+
+    std::unique_ptr<Expr> concatStrings(PosIdx pos)
+    {
+        std::vector<std::pair<PosIdx, std::unique_ptr<Expr>>> args(2);
+        args[1] = popExpr();
+        args[0] = popExpr();
+        return std::make_unique<ExprConcatStrings>(pos, false, std::move(args));
+    }
+
+    std::unique_ptr<Expr> negate(PosIdx pos, State & state)
+    {
+        std::vector<std::unique_ptr<Expr>> args(2);
+        args[0] = std::make_unique<ExprInt>(0);
+        args[1] = popExprOnly();
+        return std::make_unique<ExprCall>(pos, std::make_unique<ExprVar>(state.s.sub), std::move(args));
+    }
+
+    std::pair<PosIdx, std::unique_ptr<Expr>> applyOp(PosIdx pos, auto & op, State & state) {
+        using Op = grammar::op;
+
+        auto not_ = [] (auto e) {
+            return std::make_unique<ExprOpNot>(std::move(e));
+        };
+
+        return {
+            pos,
+            (overloaded {
+                [&] (Op::implies)     { return applyBinary<ExprOpImpl>(pos); },
+                [&] (Op::or_)         { return applyBinary<ExprOpOr>(pos); },
+                [&] (Op::and_)        { return applyBinary<ExprOpAnd>(pos); },
+                [&] (Op::equals)      { return applyBinary<ExprOpEq>(pos); },
+                [&] (Op::not_equals)  { return applyBinary<ExprOpNEq>(pos); },
+                [&] (Op::less)        { return order(pos, true, state); },
+                [&] (Op::greater_eq)  { return not_(order(pos, true, state)); },
+                [&] (Op::greater)     { return order(pos, false, state); },
+                [&] (Op::less_eq)     { return not_(order(pos, false, state)); },
+                [&] (Op::update)      { return applyBinary<ExprOpUpdate>(pos); },
+                [&] (Op::not_)        { return applyUnary<ExprOpNot>(); },
+                [&] (Op::plus)        { return concatStrings(pos); },
+                [&] (Op::minus)       { return call(pos, state.s.sub); },
+                [&] (Op::mul)         { return call(pos, state.s.mul); },
+                [&] (Op::div)         { return call(pos, state.s.div); },
+                [&] (Op::concat)      { return applyBinary<ExprOpConcatLists>(pos); },
+                [&] (has_attr & a)    { return applyUnary<ExprOpHasAttr>(std::move(a.path)); },
+                [&] (Op::unary_minus) { return negate(pos, state); },
+            })(op)
+        };
+    }
+
+    // always_inline is needed, otherwise pushOp slows down considerably
+    [[noreturn, gnu::always_inline]]
+    static void badOperator(PosIdx pos, State & state)
+    {
+        throw ParseError({
+            .msg = HintFmt("syntax error, unexpected operator"),
+            .pos = state.positions[pos]
+        });
+    }
+
+    template<typename Expr, typename... Args>
+    Expr & pushExpr(PosIdx pos, Args && ... args)
+    {
+        auto p = std::make_unique<Expr>(std::forward<Args>(args)...);
+        auto & result = *p;
+        exprs.emplace_back(pos, std::move(p));
+        return result;
+    }
+};
+
+struct SubexprState {
+private:
+    ExprState * up;
+
+public:
+    explicit SubexprState(ExprState & up, auto &...) : up(&up) {}
+    operator ExprState &() { return *up; }
+    ExprState * operator->() { return up; }
+};
+
+
+
+template<typename Rule>
+struct BuildAST : grammar::nothing<Rule> {};
+
+struct LambdaState : SubexprState {
+    using SubexprState::SubexprState;
+
+    Symbol arg;
+    std::unique_ptr<Formals> formals;
+};
+
+struct FormalsState : SubexprState {
+    using SubexprState::SubexprState;
+
+    Formals formals{};
+    Formal formal{};
+};
+
+template<> struct BuildAST<grammar::formal::name> {
+    static void apply(const auto & in, FormalsState & s, State & ps) {
+        s.formal = {
+            .pos = ps.at(in),
+            .name = ps.symbols.create(in.string_view()),
+        };
+    }
+};
+
+template<> struct BuildAST<grammar::formal> {
+    static void apply0(FormalsState & s, State &) {
+        s.formals.formals.emplace_back(std::move(s.formal));
+    }
+};
+
+template<> struct BuildAST<grammar::formal::default_value> {
+    static void apply0(FormalsState & s, State & ps) {
+        s.formal.def = s->popExprOnly();
+    }
+};
+
+template<> struct BuildAST<grammar::formals::ellipsis> {
+    static void apply0(FormalsState & s, State &) {
+        s.formals.ellipsis = true;
+    }
+};
+
+template<> struct BuildAST<grammar::formals> : change_head<FormalsState> {
+    static void success0(FormalsState & f, LambdaState & s, State &) {
+        s.formals = std::make_unique<Formals>(std::move(f.formals));
+    }
+};
+
+struct AttrState : SubexprState {
+    using SubexprState::SubexprState;
+
+    std::vector<AttrName> attrs;
+
+    void pushAttr(auto && attr, PosIdx) { attrs.emplace_back(std::move(attr)); }
+};
+
+template<> struct BuildAST<grammar::attr::simple> {
+    static void apply(const auto & in, auto & s, State & ps) {
+        s.pushAttr(ps.symbols.create(in.string_view()), ps.at(in));
+    }
+};
+
+template<> struct BuildAST<grammar::attr::string> {
+    static void apply(const auto & in, auto & s, State & ps) {
+        auto e = s->popExprOnly();
+        if (auto str = dynamic_cast<ExprString *>(e.get()))
+            s.pushAttr(ps.symbols.create(str->s), ps.at(in));
+        else
+            s.pushAttr(std::move(e), ps.at(in));
+    }
+};
+
+template<> struct BuildAST<grammar::attr::expr> : BuildAST<grammar::attr::string> {};
+
+struct BindingsState : SubexprState {
+    using SubexprState::SubexprState;
+
+    ExprAttrs attrs;
+    AttrPath path;
+    std::unique_ptr<Expr> value;
+};
+
+struct InheritState : SubexprState {
+    using SubexprState::SubexprState;
+
+    std::vector<std::pair<AttrName, PosIdx>> attrs;
+    std::unique_ptr<Expr> from;
+    PosIdx fromPos;
+
+    void pushAttr(auto && attr, PosIdx pos) { attrs.emplace_back(std::move(attr), pos); }
+};
+
+template<> struct BuildAST<grammar::inherit::from> {
+    static void apply(const auto & in, InheritState & s, State & ps) {
+        s.from = s->popExprOnly();
+        s.fromPos = ps.at(in);
+    }
+};
+
+template<> struct BuildAST<grammar::inherit> : change_head<InheritState> {
+    static void success0(InheritState & s, BindingsState & b, State & ps) {
+        auto & attrs = b.attrs.attrs;
+        // TODO this should not reuse generic attrpath rules.
+        for (auto & [i, iPos] : s.attrs) {
+            if (i.symbol)
+                continue;
+            if (auto str = dynamic_cast<ExprString *>(i.expr.get()))
+                i = AttrName(ps.symbols.create(str->s));
+            else {
+                throw ParseError({
+                    .msg = HintFmt("dynamic attributes not allowed in inherit"),
+                    .pos = ps.positions[iPos]
+                });
+            }
+        }
+        if (auto fromE = std::move(s.from)) {
+            if (!b.attrs.inheritFromExprs)
+                b.attrs.inheritFromExprs = std::make_unique<std::vector<std::unique_ptr<Expr>>>();
+            b.attrs.inheritFromExprs->push_back(std::move(fromE));
+            for (auto & [i, iPos] : s.attrs) {
+                if (attrs.find(i.symbol) != attrs.end())
+                    ps.dupAttr(i.symbol, iPos, attrs[i.symbol].pos);
+                auto from = std::make_unique<ExprInheritFrom>(s.fromPos, b.attrs.inheritFromExprs->size() - 1);
+                attrs.emplace(
+                    i.symbol,
+                    ExprAttrs::AttrDef(
+                        std::make_unique<ExprSelect>(iPos, std::move(from), i.symbol),
+                        iPos,
+                        ExprAttrs::AttrDef::Kind::InheritedFrom));
+            }
+        } else {
+            for (auto & [i, iPos] : s.attrs) {
+                if (attrs.find(i.symbol) != attrs.end())
+                    ps.dupAttr(i.symbol, iPos, attrs[i.symbol].pos);
+                attrs.emplace(
+                    i.symbol,
+                    ExprAttrs::AttrDef(
+                        std::make_unique<ExprVar>(iPos, i.symbol),
+                        iPos,
+                        ExprAttrs::AttrDef::Kind::Inherited));
+            }
+        }
+    }
+};
+
+template<> struct BuildAST<grammar::binding::path> : change_head<AttrState> {
+    static void success0(AttrState & a, BindingsState & s, State & ps) {
+        s.path = std::move(a.attrs);
+    }
+};
+
+template<> struct BuildAST<grammar::binding::value> {
+    static void apply0(BindingsState & s, State & ps) {
+        s.value = s->popExprOnly();
+    }
+};
+
+template<> struct BuildAST<grammar::binding> {
+    static void apply(const auto & in, BindingsState & s, State & ps) {
+        ps.addAttr(&s.attrs, std::move(s.path), std::move(s.value), ps.at(in));
+    }
+};
+
+template<> struct BuildAST<grammar::expr::id> {
+    static void apply(const auto & in, ExprState & s, State & ps) {
+        if (in.string_view() == "__curPos")
+            s.pushExpr<ExprPos>(ps.at(in), ps.at(in));
+        else
+            s.pushExpr<ExprVar>(ps.at(in), ps.at(in), ps.symbols.create(in.string_view()));
+    }
+};
+
+template<> struct BuildAST<grammar::expr::int_> {
+    static void apply(const auto & in, ExprState & s, State & ps) {
+        int64_t v;
+        if (std::from_chars(in.begin(), in.end(), v).ec != std::errc{}) {
+            throw ParseError({
+                .msg = HintFmt("invalid integer '%1%'", in.string_view()),
+                .pos = ps.positions[ps.at(in)],
+            });
+        }
+        s.pushExpr<ExprInt>(noPos, v);
+    }
+};
+
+template<> struct BuildAST<grammar::expr::float_> {
+    static void apply(const auto & in, ExprState & s, State & ps) {
+        // copy the input into a temporary string so we can call stod.
+        // can't use from_chars because libc++ (thus darwin) does not have it,
+        // and floats are not performance-sensitive anyway. if they were you'd
+        // be in much bigger trouble than this.
+        //
+        // we also get to do a locale-save dance because stod is locale-aware and
+        // something (a plugin?) may have called setlocale or uselocale.
+        static struct locale_hack {
+            locale_t posix;
+            locale_hack(): posix(newlocale(LC_ALL_MASK, "POSIX", 0))
+            {
+                if (posix == 0)
+                    throw SysError("could not get POSIX locale");
+            }
+        } locale;
+
+        auto tmp = in.string();
+        double v = [&] {
+            auto oldLocale = uselocale(locale.posix);
+            Finally resetLocale([=] { uselocale(oldLocale); });
+            try {
+                return std::stod(tmp);
+            } catch (...) {
+                throw ParseError({
+                    .msg = HintFmt("invalid float '%1%'", in.string_view()),
+                    .pos = ps.positions[ps.at(in)],
+                });
+            }
+        }();
+        s.pushExpr<ExprFloat>(noPos, v);
+    }
+};
+
+struct StringState : SubexprState {
+    using SubexprState::SubexprState;
+
+    std::string currentLiteral;
+    PosIdx currentPos;
+    std::vector<std::pair<nix::PosIdx, std::unique_ptr<Expr>>> parts;
+
+    void append(PosIdx pos, std::string_view s)
+    {
+        if (currentLiteral.empty())
+            currentPos = pos;
+        currentLiteral += s;
+    }
+
+    // FIXME this truncates strings on NUL for compat with the old parser. ideally
+    // we should use the decomposition the g gives us instead of iterating over
+    // the entire string again.
+    static void unescapeStr(std::string & str)
+    {
+        char * s = str.data();
+        char * t = s;
+        char c;
+        while ((c = *s++)) {
+            if (c == '\\') {
+                c = *s++;
+                if (c == 'n') *t = '\n';
+                else if (c == 'r') *t = '\r';
+                else if (c == 't') *t = '\t';
+                else *t = c;
+            }
+            else if (c == '\r') {
+                /* Normalise CR and CR/LF into LF. */
+                *t = '\n';
+                if (*s == '\n') s++; /* cr/lf */
+            }
+            else *t = c;
+            t++;
+        }
+        str.resize(t - str.data());
+    }
+
+    void endLiteral()
+    {
+        if (!currentLiteral.empty()) {
+            unescapeStr(currentLiteral);
+            parts.emplace_back(currentPos, std::make_unique<ExprString>(std::move(currentLiteral)));
+        }
+    }
+
+    std::unique_ptr<Expr> finish()
+    {
+        if (parts.empty()) {
+            unescapeStr(currentLiteral);
+            return std::make_unique<ExprString>(std::move(currentLiteral));
+        } else {
+            endLiteral();
+            auto pos = parts[0].first;
+            return std::make_unique<ExprConcatStrings>(pos, true, std::move(parts));
+        }
+    }
+};
+
+template<typename... Content> struct BuildAST<grammar::string::literal<Content...>> {
+    static void apply(const auto & in, StringState & s, State & ps) {
+        s.append(ps.at(in), in.string_view());
+    }
+};
+
+template<> struct BuildAST<grammar::string::cr_lf> {
+    static void apply(const auto & in, StringState & s, State & ps) {
+        s.append(ps.at(in), in.string_view()); // FIXME compat with old parser
+    }
+};
+
+template<> struct BuildAST<grammar::string::interpolation> {
+    static void apply(const auto & in, StringState & s, State & ps) {
+        s.endLiteral();
+        s.parts.emplace_back(ps.at(in), s->popExprOnly());
+    }
+};
+
+template<> struct BuildAST<grammar::string::escape> {
+    static void apply(const auto & in, StringState & s, State & ps) {
+        s.append(ps.at(in), "\\"); // FIXME compat with old parser
+        s.append(ps.at(in), in.string_view());
+    }
+};
+
+template<> struct BuildAST<grammar::string> : change_head<StringState> {
+    static void success0(StringState & s, ExprState & e, State &) {
+        e.exprs.emplace_back(noPos, s.finish());
+    }
+};
+
+struct IndStringState : SubexprState {
+    using SubexprState::SubexprState;
+
+    std::vector<std::pair<PosIdx, std::variant<std::unique_ptr<Expr>, StringToken>>> parts;
+};
+
+template<bool Indented, typename... Content>
+struct BuildAST<grammar::ind_string::literal<Indented, Content...>> {
+    static void apply(const auto & in, IndStringState & s, State & ps) {
+        s.parts.emplace_back(ps.at(in), StringToken{in.string_view(), Indented});
+    }
+};
+
+template<> struct BuildAST<grammar::ind_string::interpolation> {
+    static void apply(const auto & in, IndStringState & s, State & ps) {
+        s.parts.emplace_back(ps.at(in), s->popExprOnly());
+    }
+};
+
+template<> struct BuildAST<grammar::ind_string::escape> {
+    static void apply(const auto & in, IndStringState & s, State & ps) {
+        switch (*in.begin()) {
+        case 'n': s.parts.emplace_back(ps.at(in), StringToken{"\n"}); break;
+        case 'r': s.parts.emplace_back(ps.at(in), StringToken{"\r"}); break;
+        case 't': s.parts.emplace_back(ps.at(in), StringToken{"\t"}); break;
+        default:  s.parts.emplace_back(ps.at(in), StringToken{in.string_view()}); break;
+        }
+    }
+};
+
+template<> struct BuildAST<grammar::ind_string> : change_head<IndStringState> {
+    static void success(const auto & in, IndStringState & s, ExprState & e, State & ps) {
+        e.exprs.emplace_back(noPos, ps.stripIndentation(ps.at(in), std::move(s.parts)));
+    }
+};
+
+template<typename... Content> struct BuildAST<grammar::path::literal<Content...>> {
+    static void apply(const auto & in, StringState & s, State & ps) {
+        s.append(ps.at(in), in.string_view());
+        s.endLiteral();
+    }
+};
+
+template<> struct BuildAST<grammar::path::interpolation> : BuildAST<grammar::string::interpolation> {};
+
+template<> struct BuildAST<grammar::path::anchor> {
+    static void apply(const auto & in, StringState & s, State & ps) {
+        Path path(absPath(in.string(), ps.basePath.path.abs()));
+        /* add back in the trailing '/' to the first segment */
+        if (in.string_view().ends_with('/') && in.size() > 1)
+            path += "/";
+        s.parts.emplace_back(ps.at(in), new ExprPath(std::move(path)));
+    }
+};
+
+template<> struct BuildAST<grammar::path::home_anchor> {
+    static void apply(const auto & in, StringState & s, State & ps) {
+        if (evalSettings.pureEval)
+            throw Error("the path '%s' can not be resolved in pure mode", in.string_view());
+        Path path(getHome() + in.string_view().substr(1));
+        s.parts.emplace_back(ps.at(in), new ExprPath(std::move(path)));
+    }
+};
+
+template<> struct BuildAST<grammar::path::searched_path> {
+    static void apply(const auto & in, StringState & s, State & ps) {
+        std::vector<std::unique_ptr<Expr>> args{2};
+        args[0] = std::make_unique<ExprVar>(ps.s.nixPath);
+        args[1] = std::make_unique<ExprString>(in.string());
+        s.parts.emplace_back(
+            ps.at(in),
+            std::make_unique<ExprCall>(
+                ps.at(in),
+                std::make_unique<ExprVar>(ps.s.findFile),
+                std::move(args)));
+    }
+};
+
+template<> struct BuildAST<grammar::path> : change_head<StringState> {
+    template<typename E>
+    static void check_slash(PosIdx end, StringState & s, State & ps) {
+        auto e = dynamic_cast<E *>(s.parts.back().second.get());
+        if (!e || !e->s.ends_with('/'))
+            return;
+        if (s.parts.size() > 1 || e->s != "/")
+            throw ParseError({
+                .msg = HintFmt("path has a trailing slash"),
+                .pos = ps.positions[end],
+            });
+    }
+
+    static void success(const auto & in, StringState & s, ExprState & e, State & ps) {
+        s.endLiteral();
+        check_slash<ExprPath>(ps.atEnd(in), s, ps);
+        check_slash<ExprString>(ps.atEnd(in), s, ps);
+        if (s.parts.size() == 1) {
+            e.exprs.emplace_back(noPos, std::move(s.parts.back().second));
+        } else {
+            e.pushExpr<ExprConcatStrings>(ps.at(in), ps.at(in), false, std::move(s.parts));
+        }
+    }
+};
+
+// strings and paths sare handled fully by the grammar-level rule for now
+template<> struct BuildAST<grammar::expr::string> : p::maybe_nothing {};
+template<> struct BuildAST<grammar::expr::ind_string> : p::maybe_nothing {};
+template<> struct BuildAST<grammar::expr::path> : p::maybe_nothing {};
+
+template<> struct BuildAST<grammar::expr::uri> {
+    static void apply(const auto & in, ExprState & s, State & ps) {
+       static bool noURLLiterals = experimentalFeatureSettings.isEnabled(Xp::NoUrlLiterals);
+       if (noURLLiterals)
+           throw ParseError({
+               .msg = HintFmt("URL literals are disabled"),
+               .pos = ps.positions[ps.at(in)]
+           });
+       s.pushExpr<ExprString>(ps.at(in), in.string());
+    }
+};
+
+template<> struct BuildAST<grammar::expr::ancient_let> : change_head<BindingsState> {
+    static void success(const auto & in, BindingsState & b, ExprState & s, State & ps) {
+        b.attrs.pos = ps.at(in);
+        b.attrs.recursive = true;
+        s.pushExpr<ExprSelect>(b.attrs.pos, b.attrs.pos, std::make_unique<ExprAttrs>(std::move(b.attrs)), ps.s.body);
+    }
+};
+
+template<> struct BuildAST<grammar::expr::rec_set> : change_head<BindingsState> {
+    static void success(const auto & in, BindingsState & b, ExprState & s, State & ps) {
+        b.attrs.pos = ps.at(in);
+        b.attrs.recursive = true;
+        s.pushExpr<ExprAttrs>(b.attrs.pos, std::move(b.attrs));
+    }
+};
+
+template<> struct BuildAST<grammar::expr::set> : change_head<BindingsState> {
+    static void success(const auto & in, BindingsState & b, ExprState & s, State & ps) {
+        b.attrs.pos = ps.at(in);
+        s.pushExpr<ExprAttrs>(b.attrs.pos, std::move(b.attrs));
+    }
+};
+
+using ListState = std::vector<std::unique_ptr<Expr>>;
+
+template<> struct BuildAST<grammar::expr::list> : change_head<ListState> {
+    static void success(const auto & in, ListState & ls, ExprState & s, State & ps) {
+        auto e = std::make_unique<ExprList>();
+        e->elems = std::move(ls);
+        s.exprs.emplace_back(ps.at(in), std::move(e));
+    }
+};
+
+template<> struct BuildAST<grammar::expr::list::entry> : change_head<ExprState> {
+    static void success0(ExprState & e, ListState & s, State & ps) {
+        s.emplace_back(e.finish(ps).second);
+    }
+};
+
+struct SelectState : SubexprState {
+    using SubexprState::SubexprState;
+
+    PosIdx pos;
+    ExprSelect * e = nullptr;
+};
+
+template<> struct BuildAST<grammar::expr::select::head> {
+    static void apply(const auto & in, SelectState & s, State & ps) {
+        s.pos = ps.at(in);
+    }
+};
+
+template<> struct BuildAST<grammar::expr::select::attr> : change_head<AttrState> {
+    static void success0(AttrState & a, SelectState & s, State &) {
+        s.e = &s->pushExpr<ExprSelect>(s.pos, s.pos, s->popExprOnly(), std::move(a.attrs), nullptr);
+    }
+};
+
+template<> struct BuildAST<grammar::expr::select::attr_or> {
+    static void apply0(SelectState & s, State &) {
+        s.e->def = s->popExprOnly();
+    }
+};
+
+template<> struct BuildAST<grammar::expr::select::as_app_or> {
+    static void apply(const auto & in, SelectState & s, State & ps) {
+        std::vector<std::unique_ptr<Expr>> args(1);
+        args[0] = std::make_unique<ExprVar>(ps.at(in), ps.s.or_);
+        s->pushExpr<ExprCall>(s.pos, s.pos, s->popExprOnly(), std::move(args));
+    }
+};
+
+template<> struct BuildAST<grammar::expr::select> : change_head<SelectState> {
+    static void success0(const auto &...) {}
+};
+
+struct AppState : SubexprState {
+    using SubexprState::SubexprState;
+
+    PosIdx pos;
+    ExprCall * e = nullptr;
+};
+
+template<> struct BuildAST<grammar::expr::app::select_or_fn> {
+    static void apply(const auto & in, AppState & s, State & ps) {
+        s.pos = ps.at(in);
+    }
+};
+
+template<> struct BuildAST<grammar::expr::app::first_arg> {
+    static void apply(auto & in, AppState & s, State & ps) {
+        auto arg = s->popExprOnly(), fn = s->popExprOnly();
+        if ((s.e = dynamic_cast<ExprCall *>(fn.get()))) {
+            // TODO remove.
+            // AST compat with old parser, semantics are the same.
+            // this can happen on occasions such as `<p> <p>` or `a or b or`,
+            // neither of which are super worth optimizing.
+            s.e->args.push_back(std::move(arg));
+            s->exprs.emplace_back(noPos, std::move(fn));
+        } else {
+            std::vector<std::unique_ptr<Expr>> args{1};
+            args[0] = std::move(arg);
+            s.e = &s->pushExpr<ExprCall>(s.pos, s.pos, std::move(fn), std::move(args));
+        }
+    }
+};
+
+template<> struct BuildAST<grammar::expr::app::another_arg> {
+    static void apply0(AppState & s, State & ps) {
+        s.e->args.push_back(s->popExprOnly());
+    }
+};
+
+template<> struct BuildAST<grammar::expr::app> : change_head<AppState> {
+    static void success0(const auto &...) {}
+};
+
+template<typename Op> struct BuildAST<grammar::expr::operator_<Op>> {
+    static void apply(const auto & in, ExprState & s, State & ps) {
+        s.pushOp(ps.at(in), Op{}, ps);
+    }
+};
+template<> struct BuildAST<grammar::expr::operator_<grammar::op::has_attr>> : change_head<AttrState> {
+    static void success(const auto & in, AttrState & a, ExprState & s, State & ps) {
+        s.pushOp(ps.at(in), ExprState::has_attr{{}, std::move(a.attrs)}, ps);
+    }
+};
+
+template<> struct BuildAST<grammar::expr::lambda::arg> {
+    static void apply(const auto & in, LambdaState & s, State & ps) {
+        s.arg = ps.symbols.create(in.string_view());
+    }
+};
+
+template<> struct BuildAST<grammar::expr::lambda> : change_head<LambdaState> {
+    static void success(const auto & in, LambdaState & l, ExprState & s, State & ps) {
+        if (l.formals)
+            l.formals = ps.validateFormals(std::move(l.formals), ps.at(in), l.arg);
+        s.pushExpr<ExprLambda>(ps.at(in), ps.at(in), l.arg, std::move(l.formals), l->popExprOnly());
+    }
+};
+
+template<> struct BuildAST<grammar::expr::assert_> {
+    static void apply(const auto & in, ExprState & s, State & ps) {
+        auto body = s.popExprOnly(), cond = s.popExprOnly();
+        s.pushExpr<ExprAssert>(ps.at(in), ps.at(in), std::move(cond), std::move(body));
+    }
+};
+
+template<> struct BuildAST<grammar::expr::with> {
+    static void apply(const auto & in, ExprState & s, State & ps) {
+        auto body = s.popExprOnly(), scope = s.popExprOnly();
+        s.pushExpr<ExprWith>(ps.at(in), ps.at(in), std::move(scope), std::move(body));
+    }
+};
+
+template<> struct BuildAST<grammar::expr::let> : change_head<BindingsState> {
+    static void success(const auto & in, BindingsState & b, ExprState & s, State & ps) {
+        if (!b.attrs.dynamicAttrs.empty())
+            throw ParseError({
+                .msg = HintFmt("dynamic attributes not allowed in let"),
+                .pos = ps.positions[ps.at(in)]
+            });
+
+        s.pushExpr<ExprLet>(ps.at(in), std::make_unique<ExprAttrs>(std::move(b.attrs)), b->popExprOnly());
+    }
+};
+
+template<> struct BuildAST<grammar::expr::if_> {
+    static void apply(const auto & in, ExprState & s, State & ps) {
+        auto else_ = s.popExprOnly(), then = s.popExprOnly(), cond = s.popExprOnly();
+        s.pushExpr<ExprIf>(ps.at(in), ps.at(in), std::move(cond), std::move(then), std::move(else_));
+    }
+};
+
+template<> struct BuildAST<grammar::expr> : change_head<ExprState> {
+    static void success0(ExprState & inner, ExprState & outer, State & ps) {
+        outer.exprs.push_back(inner.finish(ps));
+    }
+};
+
+}
+}
+
+namespace nix {
+
+Expr * EvalState::parse(
+    char * text,
+    size_t length,
+    Pos::Origin origin,
+    const SourcePath & basePath,
+    std::shared_ptr<StaticEnv> & staticEnv)
+{
+    parser::State s = {
+        symbols,
+        positions,
+        basePath,
+        positions.addOrigin(origin, length),
+        exprSymbols,
+    };
+    parser::ExprState x;
+
+    assert(length >= 2);
+    assert(text[length - 1] == 0);
+    assert(text[length - 2] == 0);
+    length -= 2;
+
+    p::string_input<p::tracking_mode::lazy> inp{std::string_view{text, length}, "input"};
+    try {
+        p::parse<parser::grammar::root, parser::BuildAST, parser::Control>(inp, x, s);
+    } catch (p::parse_error & e) {
+        auto pos = e.positions().back();
+        throw ParseError({
+            .msg = HintFmt("syntax error, %s", e.message()),
+            .pos = positions[s.positions.add(s.origin, pos.byte)]
+        });
+    }
+
+    auto [_pos, result] = x.finish(s);
+    result->bindVars(*this, staticEnv);
+    return result.release();
+}
+
+}

src/libexpr/parser/state.hh

@@ -3,77 +3,61 @@
 
 #include "eval.hh"
 
-namespace nix {
+namespace nix::parser {
 
-/**
- * @note Storing a C-style `char *` and `size_t` allows us to avoid
- * having to define the special members that using string_view here
- * would implicitly delete.
- */
 struct StringToken
 {
-    const char * p;
-    size_t l;
+    std::string_view s;
     bool hasIndentation;
-    operator std::string_view() const { return {p, l}; }
+    operator std::string_view() const { return s; }
 };
 
-struct ParserLocation
-{
-    int first_line, first_column;
-    int last_line, last_column;
-
-    // backup to recover from yyless(0)
-    int stashed_first_column, stashed_last_column;
-
-    void stash() {
-        stashed_first_column = first_column;
-        stashed_last_column = last_column;
-    }
-
-    void unstash() {
-        first_column = stashed_first_column;
-        last_column = stashed_last_column;
-    }
-};
-
-struct ParserState
+struct State
 {
     SymbolTable & symbols;
     PosTable & positions;
-    Expr * result;
     SourcePath basePath;
     PosTable::Origin origin;
     const Expr::AstSymbols & s;
-    std::unique_ptr<Error> error;
 
-    [[nodiscard]] ParseError dupAttr(const AttrPath & attrPath, const PosIdx pos, const PosIdx prevPos);
-    [[nodiscard]] ParseError dupAttr(Symbol attr, const PosIdx pos, const PosIdx prevPos);
-    [[nodiscard]] std::optional<ParseError> addAttr(ExprAttrs * attrs, AttrPath && attrPath, std::unique_ptr<Expr> e, const PosIdx pos);
-    [[nodiscard]] std::optional<ParseError> validateFormals(Formals * formals, PosIdx pos = noPos, Symbol arg = {});
+    void dupAttr(const AttrPath & attrPath, const PosIdx pos, const PosIdx prevPos);
+    void dupAttr(Symbol attr, const PosIdx pos, const PosIdx prevPos);
+    void addAttr(ExprAttrs * attrs, AttrPath && attrPath, std::unique_ptr<Expr> e, const PosIdx pos);
+    std::unique_ptr<Formals> validateFormals(std::unique_ptr<Formals> formals, PosIdx pos = noPos, Symbol arg = {});
     std::unique_ptr<Expr> stripIndentation(const PosIdx pos,
         std::vector<std::pair<PosIdx, std::variant<std::unique_ptr<Expr>, StringToken>>> && es);
-    PosIdx at(const ParserLocation & loc);
+
+    // lazy positioning means we don't get byte offsets directly, in.position() would work
+    // but also requires line and column (which is expensive)
+    PosIdx at(const auto & in)
+    {
+        return positions.add(origin, in.begin() - in.input().begin());
+    }
+
+    PosIdx atEnd(const auto & in)
+    {
+        return positions.add(origin, in.end() - in.input().begin());
+    }
 };
 
-inline ParseError ParserState::dupAttr(const AttrPath & attrPath, const PosIdx pos, const PosIdx prevPos)
+inline void State::dupAttr(const AttrPath & attrPath, const PosIdx pos, const PosIdx prevPos)
 {
-    return ParseError({
+    throw ParseError({
          .msg = HintFmt("attribute '%1%' already defined at %2%",
              showAttrPath(symbols, attrPath), positions[prevPos]),
          .pos = positions[pos]
     });
 }
 
-inline ParseError ParserState::dupAttr(Symbol attr, const PosIdx pos, const PosIdx prevPos)
+inline void State::dupAttr(Symbol attr, const PosIdx pos, const PosIdx prevPos)
 {
-    return ParseError({
+    throw ParseError({
         .msg = HintFmt("attribute '%1%' already defined at %2%", symbols[attr], positions[prevPos]),
         .pos = positions[pos]
     });
 }
 
-inline std::optional<ParseError> ParserState::addAttr(ExprAttrs * attrs, AttrPath && attrPath, std::unique_ptr<Expr> e, const PosIdx pos)
+inline void State::addAttr(ExprAttrs * attrs, AttrPath && attrPath, std::unique_ptr<Expr> e, const PosIdx pos)
 {
     AttrPath::iterator i;
     // All attrpaths have at least one attr
@@ -88,12 +72,12 @@ inline std::optional<ParseError> ParserState::addAttr(ExprAttrs * attrs, AttrPat
                     ExprAttrs * attrs2 = dynamic_cast<ExprAttrs *>(j->second.e.get());
                     if (!attrs2) {
                         attrPath.erase(i + 1, attrPath.end());
-                        return dupAttr(attrPath, pos, j->second.pos);
+                        dupAttr(attrPath, pos, j->second.pos);
                     }
                     attrs = attrs2;
                 } else {
                     attrPath.erase(i + 1, attrPath.end());
-                    return dupAttr(attrPath, pos, j->second.pos);
+                    dupAttr(attrPath, pos, j->second.pos);
                 }
             } else {
                 auto next = attrs->attrs.emplace(std::piecewise_construct,
@@ -135,7 +119,7 @@ inline std::optional<ParseError> ParserState::addAttr(ExprAttrs * attrs, AttrPat
                 if (ae->inheritFromExprs)
                     std::ranges::move(*ae->inheritFromExprs, std::back_inserter(*jAttrs->inheritFromExprs));
             } else {
-                return dupAttr(attrPath, pos, j->second.pos);
+                dupAttr(attrPath, pos, j->second.pos);
             }
         } else {
             // This attr path is not defined. Let's create it.
@@ -147,11 +131,9 @@ inline std::optional<ParseError> ParserState::addAttr(ExprAttrs * attrs, AttrPat
     } else {
         attrs->dynamicAttrs.emplace_back(std::move(i->expr), std::move(e), pos);
     }
-
-    return {};
 }
 
-inline std::optional<ParseError> ParserState::validateFormals(Formals * formals, PosIdx pos, Symbol arg)
+inline std::unique_ptr<Formals> State::validateFormals(std::unique_ptr<Formals> formals, PosIdx pos, Symbol arg)
 {
     std::sort(formals->formals.begin(), formals->formals.end(),
         [] (const auto & a, const auto & b) {
@@ -166,21 +148,21 @@ inline std::optional<ParseError> ParserState::validateFormals(Formals * formals,
         duplicate = std::min(thisDup, duplicate.value_or(thisDup));
     }
     if (duplicate)
-        return ParseError({
+        throw ParseError({
             .msg = HintFmt("duplicate formal function argument '%1%'", symbols[duplicate->first]),
             .pos = positions[duplicate->second]
         });
 
     if (arg && formals->has(arg))
-        return ParseError({
+        throw ParseError({
             .msg = HintFmt("duplicate formal function argument '%1%'", symbols[arg]),
             .pos = positions[pos]
         });
 
-    return {};
+    return formals;
 }
 
-inline std::unique_ptr<Expr> ParserState::stripIndentation(const PosIdx pos,
+inline std::unique_ptr<Expr> State::stripIndentation(const PosIdx pos,
     std::vector<std::pair<PosIdx, std::variant<std::unique_ptr<Expr>, StringToken>>> && es)
 {
     if (es.empty()) return std::make_unique<ExprString>("");
@@ -201,11 +183,11 @@ inline std::unique_ptr<Expr> ParserState::stripIndentation(const PosIdx pos,
             }
             continue;
         }
-        for (size_t j = 0; j < str->l; ++j) {
+        for (size_t j = 0; j < str->s.size(); ++j) {
             if (atStartOfLine) {
-                if (str->p[j] == ' ')
+                if (str->s[j] == ' ')
                     curIndent++;
-                else if (str->p[j] == '\n') {
+                else if (str->s[j] == '\n') {
                     /* Empty line, doesn't influence minimum
                        indentation. */
                     curIndent = 0;
@@ -213,7 +195,7 @@ inline std::unique_ptr<Expr> ParserState::stripIndentation(const PosIdx pos,
                     atStartOfLine = false;
                     if (curIndent < minIndent) minIndent = curIndent;
                 }
-            } else if (str->p[j] == '\n') {
+            } else if (str->s[j] == '\n') {
                 atStartOfLine = true;
                 curIndent = 0;
             }
@@ -233,23 +215,23 @@ inline std::unique_ptr<Expr> ParserState::stripIndentation(const PosIdx pos,
     };
     const auto trimString = [&] (const StringToken t) {
         std::string s2;
-        for (size_t j = 0; j < t.l; ++j) {
+        for (size_t j = 0; j < t.s.size(); ++j) {
             if (atStartOfLine) {
-                if (t.p[j] == ' ') {
+                if (t.s[j] == ' ') {
                     if (curDropped++ >= minIndent)
-                        s2 += t.p[j];
+                        s2 += t.s[j];
                 }
-                else if (t.p[j] == '\n') {
+                else if (t.s[j] == '\n') {
                     curDropped = 0;
-                    s2 += t.p[j];
+                    s2 += t.s[j];
                 } else {
                     atStartOfLine = false;
                     curDropped = 0;
-                    s2 += t.p[j];
+                    s2 += t.s[j];
                 }
             } else {
-                s2 += t.p[j];
-                if (t.p[j] == '\n') atStartOfLine = true;
+                s2 += t.s[j];
+                if (t.s[j] == '\n') atStartOfLine = true;
             }
         }
 
@@ -274,9 +256,4 @@ inline std::unique_ptr<Expr> ParserState::stripIndentation(const PosIdx pos,
     return std::make_unique<ExprConcatStrings>(pos, true, std::move(es2));
 }
 
-inline PosIdx ParserState::at(const ParserLocation & loc)
-{
-    return positions.add(origin, loc.first_column);
-}
-
 }

src/libfetchers/git.cc

@@ -361,7 +361,7 @@ struct GitInputScheme : InputScheme
 
         args.push_back(destDir);
 
-        runProgram("git", true, args, {}, true);
+        runProgram("git", true, args, true);
     }
 
     std::optional<Path> getSourcePath(const Input & input) const override
@@ -399,12 +399,15 @@ struct GitInputScheme : InputScheme
 
 
             if (commitMsg) {
+                auto [_fd, msgPath] = createTempFile("nix-msg");
+                AutoDelete const _delete{msgPath};
+                writeFile(msgPath, *commitMsg);
+
                 // Pause the logger to allow for user input (such as a gpg passphrase) in `git commit`
                 logger->pause();
                 Finally restoreLogger([]() { logger->resume(); });
                 runProgram("git", true,
-                    { "-C", *root, "--git-dir", gitDir, "commit", std::string(path.rel()), "-F", "-" },
-                    *commitMsg);
+                    { "-C", *root, "--git-dir", gitDir, "commit", std::string(path.rel()), "-F", msgPath });
             }
         }
     }
@@ -589,7 +592,7 @@ struct GitInputScheme : InputScheme
                             : ref == "HEAD"
                                 ? *ref
                                 : "refs/heads/" + *ref;
-                    runProgram("git", true, { "-C", repoDir, "--git-dir", gitDir, "fetch", "--quiet", "--force", "--", actualUrl, fmt("%s:%s", fetchRef, fetchRef) }, {}, true);
+                    runProgram("git", true, { "-C", repoDir, "--git-dir", gitDir, "fetch", "--quiet", "--force", "--", actualUrl, fmt("%s:%s", fetchRef, fetchRef) }, true);
                 } catch (Error & e) {
                     if (!pathExists(localRefFile)) throw;
                     warn("could not update local clone of Git repository '%s'; continuing with the most recent version", actualUrl);
@@ -656,7 +659,7 @@ struct GitInputScheme : InputScheme
                 // everything to ensure we get the rev.
                 Activity act(*logger, lvlTalkative, actUnknown, fmt("making temporary clone of '%s'", repoDir));
                 runProgram("git", true, { "-C", tmpDir, "fetch", "--quiet", "--force",
-                        "--update-head-ok", "--", repoDir, "refs/*:refs/*" }, {}, true);
+                        "--update-head-ok", "--", repoDir, "refs/*:refs/*" }, true);
             }
 
             runProgram("git", true, { "-C", tmpDir, "checkout", "--quiet", input.getRev()->gitRev() });
@@ -683,22 +686,19 @@ struct GitInputScheme : InputScheme
 
             {
                 Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching submodules of '%s'", actualUrl));
-                runProgram("git", true, { "-C", tmpDir, "submodule", "--quiet", "update", "--init", "--recursive" }, {}, true);
+                runProgram("git", true, { "-C", tmpDir, "submodule", "--quiet", "update", "--init", "--recursive" }, true);
             }
 
             filter = isNotDotGitDirectory;
         } else {
-            // FIXME: should pipe this, or find some better way to extract a
-            // revision.
-            auto source = sinkToSource([&](Sink & sink) {
-                runProgram2({
-                    .program = "git",
-                    .args = { "-C", repoDir, "--git-dir", gitDir, "archive", input.getRev()->gitRev() },
-                    .standardOut = &sink
-                });
+            auto proc = runProgram2({
+                .program = "git",
+                .args = { "-C", repoDir, "--git-dir", gitDir, "archive", input.getRev()->gitRev() },
+                .captureStdout = true,
             });
+            Finally const _wait([&] { proc.wait(); });
 
-            unpackTarfile(*source, tmpDir);
+            unpackTarfile(*proc.stdout(), tmpDir);
         }
 
         auto storePath = store->addToStore(name, tmpDir, FileIngestionMethod::Recursive, htSHA256, filter);

src/libfetchers/mercurial.cc

@@ -28,10 +28,9 @@ static RunOptions hgOptions(const Strings & args)
 }
 
 // runProgram wrapper that uses hgOptions instead of stock RunOptions.
-static std::string runHg(const Strings & args, const std::optional<std::string> & input = {})
+static std::string runHg(const Strings & args)
 {
     RunOptions opts = hgOptions(args);
-    opts.input = input;
 
     auto res = runProgram(std::move(opts));
 

src/libfetchers/path.cc

@@ -130,10 +130,8 @@ struct PathInputScheme : InputScheme
         time_t mtime = 0;
         if (!storePath || storePath->name() != "source" || !store->isValidPath(*storePath)) {
             // FIXME: try to substitute storePath.
-            auto src = sinkToSource([&](Sink & sink) {
-                mtime = dumpPathAndGetMtime(absPath, sink, defaultPathFilter);
-            });
-            storePath = store->addToStoreFromDump(*src, "source");
+            auto src = GeneratorSource{dumpPathAndGetMtime(absPath, mtime, defaultPathFilter)};
+            storePath = store->addToStoreFromDump(src, "source");
         }
         input.attrs.insert_or_assign("lastModified", uint64_t(mtime));
 

src/libfetchers/tarball.cc

@@ -71,7 +71,7 @@ DownloadFileResult downloadFile(
         storePath = std::move(cached->storePath);
     } else {
         StringSink sink;
-        dumpString(res.data, sink);
+        sink << dumpString(res.data);
         auto hash = hashString(htSHA256, res.data);
         ValidPathInfo info {
             *store,

src/libmain/common-args.cc

@@ -58,7 +58,7 @@ MixCommonArgs::MixCommonArgs(const std::string & programName)
 
     addFlag({
         .longName = "log-format",
-        .description = "Set the format of log output; one of `raw`, `internal-json`, `bar` or `bar-with-logs`.",
+        .description = "Set the format of log output; one of `raw`, `internal-json`, `bar`, `bar-with-logs`, `multiline` or `multiline-with-logs`.",
         .category = loggingCategory,
         .labels = {"format"},
         .handler = {[](std::string format) { setLogFormat(format); }},

src/libmain/loggers.cc

@@ -17,6 +17,10 @@ LogFormat parseLogFormat(const std::string & logFormatStr) {
         return LogFormat::bar;
     else if (logFormatStr == "bar-with-logs")
         return LogFormat::barWithLogs;
+    else if (logFormatStr == "multiline")
+        return LogFormat::multiline;
+    else if (logFormatStr == "multiline-with-logs")
+        return LogFormat::multilineWithLogs;
     throw Error("option 'log-format' has an invalid value '%s'", logFormatStr);
 }
 
@@ -35,6 +39,17 @@ Logger * makeDefaultLogger() {
         logger->setPrintBuildLogs(true);
         return logger;
     }
+    case LogFormat::multiline: {
+        auto logger = makeProgressBar();
+        logger->setPrintMultiline(true);
+        return logger;
+    }
+    case LogFormat::multilineWithLogs: {
+        auto logger = makeProgressBar();
+        logger->setPrintMultiline(true);
+        logger->setPrintBuildLogs(true);
+        return logger;
+    }
     default:
         abort();
     }

src/libmain/loggers.hh

@@ -11,6 +11,8 @@ enum class LogFormat {
   internalJSON,
   bar,
   barWithLogs,
+  multiline,
+  multilineWithLogs,
 };
 
 void setLogFormat(const std::string & logFormatStr);

src/libmain/progress-bar.cc

@@ -13,6 +13,11 @@
 
 namespace nix {
 
+// 100 years ought to be enough for anyone (yet sufficiently smaller than max() to not cause signed integer overflow).
+constexpr const auto A_LONG_TIME = std::chrono::duration_cast<std::chrono::milliseconds>(
+    100 * 365 * std::chrono::seconds(86400)
+);
+
 using namespace std::literals::chrono_literals;
 
 static std::string_view getS(const std::vector<Logger::Field> & fields, size_t n)
@@ -36,6 +41,21 @@ static std::string_view storePathToName(std::string_view path)
     return i == std::string::npos ? base.substr(0, 0) : base.substr(i + 1);
 }
 
+ProgressBar::ProgressBar(bool isTTY)
+    : isTTY(isTTY)
+{
+    state_.lock()->active = isTTY;
+    updateThread = std::thread([&]() {
+        auto state(state_.lock());
+        auto nextWakeup = A_LONG_TIME;
+        while (state->active) {
+            if (!state->haveUpdate)
+                state.wait_for(updateCV, nextWakeup);
+            nextWakeup = draw(*state, {});
+            state.wait_for(quitCV, std::chrono::milliseconds(50));
+        }
+    });
+}
 
 ProgressBar::~ProgressBar()
 {
@@ -95,8 +115,7 @@ void ProgressBar::logEI(const ErrorInfo & ei)
 void ProgressBar::log(State & state, Verbosity lvl, std::string_view s)
 {
     if (state.active) {
-        writeToStderr("\r\e[K" + filterANSIEscapes(s, !isTTY) + ANSI_NORMAL "\n");
-        draw(state);
+        draw(state, s);
     } else {
         auto s2 = s + ANSI_NORMAL "\n";
         if (!isTTY) s2 = filterANSIEscapes(s2, true);
@@ -234,10 +253,14 @@ void ProgressBar::result(ActivityId act, ResultType type, const std::vector<Fiel
                 }
                 log(*state, lvlInfo, ANSI_FAINT + info.name.value_or("unnamed") + suffix + ANSI_NORMAL + lastLine);
             } else {
-                state->activities.erase(i->second);
-                info.lastLine = lastLine;
-                state->activities.emplace_back(info);
-                i->second = std::prev(state->activities.end());
+                if (!printMultiline) {
+                    state->activities.erase(i->second);
+                    info.lastLine = lastLine;
+                    state->activities.emplace_back(info);
+                    i->second = std::prev(state->activities.end());
+                } else {
+                    i->second->lastLine = lastLine;
+                }
                 update(*state);
             }
         }
@@ -290,60 +313,93 @@ void ProgressBar::update(State & state)
     updateCV.notify_one();
 }
 
-std::chrono::milliseconds ProgressBar::draw(State & state)
+std::chrono::milliseconds ProgressBar::draw(State & state, const std::optional<std::string_view> & s)
 {
     auto nextWakeup = A_LONG_TIME;
 
     state.haveUpdate = false;
     if (state.paused || !state.active) return nextWakeup;
 
-    std::string line;
+    auto windowSize = getWindowSize();
+    auto width = windowSize.second;
+    if (width <= 0) {
+        width = std::numeric_limits<decltype(width)>::max();
+    }
 
+    if (printMultiline && (state.lastLines >= 1)) {
+        // FIXME: make sure this works on windows
+        writeToStderr(fmt("\e[G\e[%dF\e[J", state.lastLines));
+    }
+
+    state.lastLines = 0;
+
+    if (s != std::nullopt)
+        writeToStderr("\r\e[K" + filterANSIEscapes(s.value(), !isTTY) + ANSI_NORMAL "\n");
+
+    std::string line;
     std::string status = getStatus(state);
     if (!status.empty()) {
         line += '[';
         line += status;
         line += "]";
     }
+    if (printMultiline && !line.empty()) {
+        writeToStderr(filterANSIEscapes(line, false, width) + "\n");
+        state.lastLines++;
+    }
 
+     auto height = windowSize.first > 0 ? windowSize.first : 25;
+     auto moreActivities = 0;
     auto now = std::chrono::steady_clock::now();
 
+    std::string activity_line;
     if (!state.activities.empty()) {
-        if (!status.empty()) line += " ";
-        auto i = state.activities.rbegin();
-
-        while (i != state.activities.rend()) {
-            if (i->visible && (!i->s.empty() || !i->lastLine.empty())) {
-                /* Don't show activities until some time has
-                   passed, to avoid displaying very short
-                   activities. */
-                auto delay = std::chrono::milliseconds(10);
-                if (i->startTime + delay < now)
-                    break;
-                else
-                    nextWakeup = std::min(nextWakeup, std::chrono::duration_cast<std::chrono::milliseconds>(delay - (now - i->startTime)));
+        for (auto i = state.activities.begin(); i != state.activities.end(); ++i) {
+            if (!(i->visible && (!i->s.empty() || !i->lastLine.empty()))) {
+                continue;
+            }
+            /* Don't show activities until some time has
+               passed, to avoid displaying very short
+               activities. */
+            auto delay = std::chrono::milliseconds(10);
+            if (i->startTime + delay >= now) {
+                nextWakeup = std::min(
+                    nextWakeup,
+                    std::chrono::duration_cast<std::chrono::milliseconds>(
+                        delay - (now - i->startTime)
+                        )
+                    );
             }
-            ++i;
-        }
 
-        if (i != state.activities.rend()) {
-            line += i->s;
+            activity_line = i->s;
+
             if (!i->phase.empty()) {
-                line += " (";
-                line += i->phase;
-                line += ")";
+                activity_line += " (";
+                activity_line += i->phase;
+                activity_line += ")";
             }
             if (!i->lastLine.empty()) {
-                if (!i->s.empty()) line += ": ";
-                line += i->lastLine;
+                if (!i->s.empty())
+                    activity_line += ": ";
+                activity_line += i->lastLine;
+            }
+
+            if (printMultiline) {
+                if (state.lastLines < (height -1)) {
+                    writeToStderr(filterANSIEscapes(activity_line, false, width) + "\n");
+                    state.lastLines++;
+                } else moreActivities++;
             }
         }
     }
 
-    auto width = getWindowSize().second;
-    if (width <= 0) width = std::numeric_limits<decltype(width)>::max();
+    if (printMultiline && moreActivities)
+        writeToStderr(fmt("And %d more...", moreActivities));
 
-    writeToStderr("\r" + filterANSIEscapes(line, false, width) + ANSI_NORMAL + "\e[K");
+    if (!printMultiline && !line.empty()) {
+        line += " " + activity_line;
+         writeToStderr("\r" + filterANSIEscapes(line, false, width) + ANSI_NORMAL + "\e[K");
+    }
 
     return nextWakeup;
 }
@@ -366,31 +422,59 @@ std::string ProgressBar::getStatus(State & state)
 
         expected = std::max(expected, act.expected);
 
-        std::string s;
+        std::string rendered;
 
         if (running || done || expected || failed) {
-            if (running)
-                if (expected != 0)
-                    s = fmt(ANSI_BLUE + numberFmt + ANSI_NORMAL "/" ANSI_GREEN + numberFmt + ANSI_NORMAL "/" + numberFmt,
-                        running / unit, done / unit, expected / unit);
-                else
-                    s = fmt(ANSI_BLUE + numberFmt + ANSI_NORMAL "/" ANSI_GREEN + numberFmt + ANSI_NORMAL,
-                        running / unit, done / unit);
-            else if (expected != done)
-                if (expected != 0)
-                    s = fmt(ANSI_GREEN + numberFmt + ANSI_NORMAL "/" + numberFmt,
-                        done / unit, expected / unit);
-                else
-                    s = fmt(ANSI_GREEN + numberFmt + ANSI_NORMAL, done / unit);
-            else
-                s = fmt(done ? ANSI_GREEN + numberFmt + ANSI_NORMAL : numberFmt, done / unit);
-            s = fmt(itemFmt, s);
+            if (running) {
+                if (expected != 0) {
+                    auto const runningPart = fmt(numberFmt, running / unit);
+                    auto const donePart = fmt(numberFmt, done / unit);
+                    auto const expectedPart = fmt(numberFmt, expected / unit);
+                    rendered = fmt(
+                        ANSI_BLUE "%s" ANSI_NORMAL "/" ANSI_GREEN "%s" ANSI_NORMAL "/%s",
+                        runningPart,
+                        donePart,
+                        expectedPart
+                    );
+                } else {
+                    auto const runningPart = fmt(numberFmt, running / unit);
+                    auto const donePart = fmt(numberFmt, done / unit);
+                    rendered = fmt(
+                        ANSI_BLUE "%s" ANSI_NORMAL "/" ANSI_GREEN "%s" ANSI_NORMAL,
+                        runningPart,
+                        donePart
+                    );
+                }
+            } else if (expected != done) {
+                if (expected != 0) {
+                    auto const donePart = fmt(numberFmt, done / unit);
+                    auto const expectedPart = fmt(numberFmt, expected / unit);
+                    rendered = fmt(
+                        ANSI_GREEN "%s" ANSI_NORMAL "/%s",
+                        donePart,
+                        expectedPart
+                    );
+                } else {
+                    auto const donePart = fmt(numberFmt, done / unit);
+                    rendered = fmt(ANSI_GREEN "%s" ANSI_NORMAL, donePart);
+                }
+            } else {
+                auto const donePart = fmt(numberFmt, done / unit);
+
+                // We only color if `done` is non-zero.
+                if (done) {
+                    rendered = concatStrings(ANSI_GREEN, donePart, ANSI_NORMAL);
+                } else {
+                    rendered = donePart;
+                }
+            }
+            rendered = fmt(itemFmt, rendered);
 
             if (failed)
-                s += fmt(" (" ANSI_RED "%d failed" ANSI_NORMAL ")", failed / unit);
+                rendered += fmt(" (" ANSI_RED "%d failed" ANSI_NORMAL ")", failed / unit);
         }
 
-        return s;
+        return rendered;
     };
 
     auto showActivity = [&](ActivityType type, const std::string & itemFmt, const std::string & numberFmt = "%d", double unit = 1) {
@@ -442,9 +526,8 @@ void ProgressBar::writeToStdout(std::string_view s)
 {
     auto state(state_.lock());
     if (state->active) {
-        std::cerr << "\r\e[K";
         Logger::writeToStdout(s);
-        draw(*state);
+        draw(*state, {});
     } else {
         Logger::writeToStdout(s);
     }
@@ -457,7 +540,7 @@ std::optional<char> ProgressBar::ask(std::string_view msg)
     std::cerr << fmt("\r\e[K%s ", msg);
     auto s = trim(readLine(STDIN_FILENO));
     if (s.size() != 1) return {};
-    draw(*state);
+    draw(*state, {});
     return s[0];
 }
 
@@ -466,6 +549,11 @@ void ProgressBar::setPrintBuildLogs(bool printBuildLogs)
     this->printBuildLogs = printBuildLogs;
 }
 
+void ProgressBar::setPrintMultiline(bool printMultiline)
+{
+    this->printMultiline = printMultiline;
+}
+
 Logger * makeProgressBar()
 {
     return new ProgressBar(shouldANSI());

src/libmain/progress-bar.hh

@@ -8,11 +8,6 @@
 
 namespace nix {
 
-// 100 years ought to be enough for anyone (yet sufficiently smaller than max() to not cause signed integer overflow).
-constexpr const auto A_LONG_TIME = std::chrono::duration_cast<std::chrono::milliseconds>(
-    100 * 365 * std::chrono::seconds(86400)
-);
-
 struct ProgressBar : public Logger
 {
     struct ActInfo
@@ -47,6 +42,8 @@ struct ProgressBar : public Logger
 
         std::map<ActivityType, ActivitiesByType> activitiesByType;
 
+        int lastLines = 0;
+
         uint64_t filesLinked = 0, bytesLinked = 0;
 
         uint64_t corruptedPaths = 0, untrustedPaths = 0;
@@ -63,23 +60,10 @@ struct ProgressBar : public Logger
     std::condition_variable quitCV, updateCV;
 
     bool printBuildLogs = false;
+    bool printMultiline = false;
     bool isTTY;
 
-    ProgressBar(bool isTTY)
-        : isTTY(isTTY)
-    {
-        state_.lock()->active = isTTY;
-        updateThread = std::thread([&]() {
-            auto state(state_.lock());
-            auto nextWakeup = A_LONG_TIME;
-            while (state->active) {
-                if (!state->haveUpdate)
-                    state.wait_for(updateCV, nextWakeup);
-                nextWakeup = draw(*state);
-                state.wait_for(quitCV, std::chrono::milliseconds(50));
-            }
-        });
-    }
+    ProgressBar(bool isTTY);
 
     ~ProgressBar();
 
@@ -114,7 +98,7 @@ struct ProgressBar : public Logger
 
     void update(State & state);
 
-    std::chrono::milliseconds draw(State & state);
+    std::chrono::milliseconds draw(State & state, const std::optional<std::string_view> & s);
 
     std::string getStatus(State & state);
 
@@ -123,6 +107,8 @@ struct ProgressBar : public Logger
     std::optional<char> ask(std::string_view msg) override;
 
     void setPrintBuildLogs(bool printBuildLogs) override;
+
+    void setPrintMultiline(bool printMultiline) override;
 };
 
 Logger * makeProgressBar();

src/libmain/shared.cc

@@ -378,7 +378,7 @@ RunPager::RunPager()
 RunPager::~RunPager()
 {
     try {
-        if (pid != -1) {
+        if (pid) {
             std::cout.flush();
             dup2(std_out, STDOUT_FILENO);
             pid.wait();

src/libstore/binary-cache-store.cc

@@ -67,16 +67,18 @@ void BinaryCacheStore::upsertFile(const std::string & path,
     upsertFile(path, std::make_shared<std::stringstream>(std::move(data)), mimeType);
 }
 
-void BinaryCacheStore::getFile(const std::string & path, Sink & sink)
+box_ptr<Source> BinaryCacheStore::getFile(const std::string & path)
 {
-    sink(*getFileContents(path));
+    return make_box_ptr<GeneratorSource>([](std::string data) -> Generator<Bytes> {
+        co_yield std::span{data.data(), data.size()};
+    }(std::move(*getFileContents(path))));
 }
 
 std::optional<std::string> BinaryCacheStore::getFileContents(const std::string & path)
 {
     StringSink sink;
     try {
-        getFile(path, sink);
+        return getFile(path)->drain();
     } catch (NoSuchBinaryCacheFile &) {
         return std::nullopt;
     }
@@ -327,26 +329,32 @@ std::optional<StorePath> BinaryCacheStore::queryPathFromHashPart(const std::stri
     }
 }
 
-void BinaryCacheStore::narFromPath(const StorePath & storePath, Sink & sink)
+WireFormatGenerator BinaryCacheStore::narFromPath(const StorePath & storePath)
 {
     auto info = queryPathInfo(storePath).cast<const NarInfo>();
 
-    LengthSink narSize;
-    TeeSink tee { sink, narSize };
-
-    auto decompressor = makeDecompressionSink(info->compression, tee);
-
     try {
-        getFile(info->url, *decompressor);
+        auto file = getFile(info->url);
+        return [](auto info, auto file, auto & stats) -> WireFormatGenerator {
+            char buf[65536];
+            size_t total = 0;
+            auto decompressor = makeDecompressionSource(info->compression, *file);
+            try {
+                while (true) {
+                    const auto len = decompressor->read(buf, sizeof(buf));
+                    co_yield std::span{buf, len};
+                    total += len;
+                }
+            } catch (EndOfFile &) {
+            }
+
+            stats.narRead++;
+            //stats.narReadCompressedBytes += nar->size(); // FIXME
+            stats.narReadBytes += total;
+        }(std::move(info), std::move(file), stats);
     } catch (NoSuchBinaryCacheFile & e) {
         throw SubstituteGone(std::move(e.info()));
     }
-
-    decompressor->finish();
-
-    stats.narRead++;
-    //stats.narReadCompressedBytes += nar->size(); // FIXME
-    stats.narReadBytes += narSize.length;
 }
 
 std::shared_ptr<const ValidPathInfo> BinaryCacheStore::queryPathInfoUncached(const StorePath & storePath)
@@ -383,16 +391,14 @@ StorePath BinaryCacheStore::addToStore(
 
     HashSink sink { hashAlgo };
     if (method == FileIngestionMethod::Recursive) {
-        dumpPath(srcPath, sink, filter);
+        sink << dumpPath(srcPath, filter);
     } else {
-        readFileSource(srcPath)->drainInto(sink);
+        sink << readFileSource(srcPath);
     }
     auto h = sink.finish().first;
 
-    auto source = sinkToSource([&](Sink & sink) {
-        dumpPath(srcPath, sink, filter);
-    });
-    return addToStoreCommon(*source, repair, CheckSigs, [&](HashResult nar) {
+    auto source = GeneratorSource{dumpPath(srcPath, filter)};
+    return addToStoreCommon(source, repair, CheckSigs, [&](HashResult nar) {
         ValidPathInfo info {
             *this,
             name,
@@ -425,7 +431,7 @@ StorePath BinaryCacheStore::addTextToStore(
         return path;
 
     StringSink sink;
-    dumpString(s, sink);
+    sink << dumpString(s);
     StringSource source(sink.s);
     return addToStoreCommon(source, repair, CheckSigs, [&](HashResult nar) {
         ValidPathInfo info {

src/libstore/binary-cache-store.hh

@@ -83,7 +83,7 @@ public:
     /**
      * Dump the contents of the specified file to a sink.
      */
-    virtual void getFile(const std::string & path, Sink & sink);
+    virtual box_ptr<Source> getFile(const std::string & path);
 
     virtual std::optional<std::string> getFileContents(const std::string & path);
 
@@ -136,7 +136,7 @@ public:
 
     std::shared_ptr<const Realisation> queryRealisationUncached(const DrvOutput &) override;
 
-    void narFromPath(const StorePath & path, Sink & sink) override;
+    WireFormatGenerator narFromPath(const StorePath & path) override;
 
     ref<FSAccessor> getFSAccessor() override;
 

src/libstore/build/derivation-goal.cc

@@ -760,7 +760,7 @@ void DerivationGoal::tryToBuild()
                 /* Not now; wait until at least one child finishes or
                    the wake-up timeout expires. */
                 if (!actLock)
-                    actLock = std::make_unique<Activity>(*logger, lvlWarn, actBuildWaiting,
+                    actLock = std::make_unique<Activity>(*logger, lvlTalkative, actBuildWaiting,
                         fmt("waiting for a machine to build '%s'", Magenta(worker.store.printStorePath(drvPath))));
                 worker.waitForAWhile(shared_from_this());
                 outputLocks.unlock();
@@ -923,12 +923,16 @@ void runPostBuildHook(
     };
     LogSink sink(act);
 
-    runProgram2({
+    auto proc = runProgram2({
         .program = settings.postBuildHook,
         .environment = hookEnvironment,
-        .standardOut = &sink,
+        .captureStdout = true,
         .mergeStderrToStdout = true,
     });
+    Finally const _wait([&] { proc.wait(); });
+
+    // FIXME just process the data, without a wrapper sink class
+    proc.stdout()->drainInto(sink);
 }
 
 void DerivationGoal::buildDone()
@@ -1204,7 +1208,7 @@ HookReply DerivationGoal::tryBuildHook()
 
     /* Tell the hook all the inputs that have to be copied to the
        remote system. */
-    CommonProto::write(worker.store, conn, inputPaths);
+    conn.to << CommonProto::write(worker.store, conn, inputPaths);
 
     /* Tell the hooks the missing outputs that have to be copied back
        from the remote system. */
@@ -1215,7 +1219,7 @@ HookReply DerivationGoal::tryBuildHook()
             if (buildMode != bmCheck && status.known && status.known->isValid()) continue;
             missingOutputs.insert(outputName);
         }
-        CommonProto::write(worker.store, conn, missingOutputs);
+        conn.to << CommonProto::write(worker.store, conn, missingOutputs);
     }
 
     hook->sink = FdSink();

src/libstore/build/hook-instance.cc

@@ -79,7 +79,7 @@ HookInstance::~HookInstance()
 {
     try {
         toHook.writeSide.reset();
-        if (pid != -1) pid.kill();
+        if (pid) pid.kill();
     } catch (...) {
         ignoreException();
     }

src/libstore/build/local-derivation-goal.cc

@@ -51,9 +51,6 @@
 #endif
 
 #if __APPLE__
-#include <spawn.h>
-#include <sys/sysctl.h>
-
 /* This definition is undocumented but depended upon by all major browsers. */
 extern "C" int sandbox_init_with_parameters(const char *profile, uint64_t flags, const char *const parameters[], char **errorbuf);
 #endif
@@ -140,7 +137,7 @@ LocalStore & LocalDerivationGoal::getLocalStore()
 
 void LocalDerivationGoal::killChild()
 {
-    if (pid != -1) {
+    if (pid) {
         worker.childTerminated(this);
 
         /* If we're using a build user, then there is a tricky race
@@ -148,7 +145,7 @@ void LocalDerivationGoal::killChild()
            done its setuid() to the build user uid, then it won't be
            killed, and we'll potentially lock up in pid.wait().  So
            also send a conventional kill to the child. */
-        ::kill(-pid, SIGKILL); /* ignore the result */
+        ::kill(-pid.get(), SIGKILL); /* ignore the result */
 
         killSandbox(true);
 
@@ -161,19 +158,7 @@ void LocalDerivationGoal::killChild()
 
 void LocalDerivationGoal::killSandbox(bool getStats)
 {
-    if (cgroup) {
-        #if __linux__
-        auto stats = destroyCgroup(*cgroup);
-        if (getStats) {
-            buildResult.cpuUser = stats.cpuUser;
-            buildResult.cpuSystem = stats.cpuSystem;
-        }
-        #else
-        abort();
-        #endif
-    }
-
-    else if (buildUser) {
+    if (buildUser) {
         auto uid = buildUser->getUID();
         assert(uid != 0);
         killUser(uid);
@@ -510,7 +495,7 @@ void LocalDerivationGoal::startBuilder()
 
     /* Create a temporary directory where the build will take
        place. */
-    tmpDir = createTempDir("", "nix-build-" + std::string(drvPath.name()), false, false, 0700);
+    tmpDir = createTempDir(settings.buildDir.get().value_or(""), "nix-build-" + std::string(drvPath.name()), false, false, 0700);
 
     chownToBuilder(tmpDir);
 
@@ -945,7 +930,7 @@ void LocalDerivationGoal::startBuilder()
             if (usingUserNamespace)
                 options.cloneFlags |= CLONE_NEWUSER;
 
-            pid_t child = startProcess([&]() { runChild(); }, options);
+            pid_t child = startProcess([&]() { runChild(); }, options).release();
 
             writeFull(sendPid.writeSide.get(), fmt("%d\n", child));
             _exit(0);
@@ -966,7 +951,7 @@ void LocalDerivationGoal::startBuilder()
 
         auto ss = tokenizeString<std::vector<std::string>>(readLine(sendPid.readSide.get()));
         assert(ss.size() == 1);
-        pid = string2Int<pid_t>(ss[0]).value();
+        pid = Pid{string2Int<pid_t>(ss[0]).value()};
 
         if (usingUserNamespace) {
             /* Set the UID/GID mapping of the builder's user namespace
@@ -976,13 +961,13 @@ void LocalDerivationGoal::startBuilder()
             uid_t hostGid = buildUser ? buildUser->getGID() : getgid();
             uid_t nrIds = buildUser ? buildUser->getUIDCount() : 1;
 
-            writeFile("/proc/" + std::to_string(pid) + "/uid_map",
+            writeFile("/proc/" + std::to_string(pid.get()) + "/uid_map",
                 fmt("%d %d %d", sandboxUid(), hostUid, nrIds));
 
             if (!buildUser || buildUser->getUIDCount() == 1)
-                writeFile("/proc/" + std::to_string(pid) + "/setgroups", "deny");
+                writeFile("/proc/" + std::to_string(pid.get()) + "/setgroups", "deny");
 
-            writeFile("/proc/" + std::to_string(pid) + "/gid_map",
+            writeFile("/proc/" + std::to_string(pid.get()) + "/gid_map",
                 fmt("%d %d %d", sandboxGid(), hostGid, nrIds));
         } else {
             debug("note: not using a user namespace");
@@ -1005,19 +990,19 @@ void LocalDerivationGoal::startBuilder()
 
         /* Save the mount- and user namespace of the child. We have to do this
            *before* the child does a chroot. */
-        sandboxMountNamespace = AutoCloseFD{open(fmt("/proc/%d/ns/mnt", (pid_t) pid).c_str(), O_RDONLY)};
+        sandboxMountNamespace = AutoCloseFD{open(fmt("/proc/%d/ns/mnt", pid.get()).c_str(), O_RDONLY)};
         if (sandboxMountNamespace.get() == -1)
             throw SysError("getting sandbox mount namespace");
 
         if (usingUserNamespace) {
-            sandboxUserNamespace = AutoCloseFD{open(fmt("/proc/%d/ns/user", (pid_t) pid).c_str(), O_RDONLY)};
+            sandboxUserNamespace = AutoCloseFD{open(fmt("/proc/%d/ns/user", pid.get()).c_str(), O_RDONLY)};
             if (sandboxUserNamespace.get() == -1)
                 throw SysError("getting sandbox user namespace");
         }
 
         /* Move the child into its own cgroup. */
         if (cgroup)
-            writeFile(*cgroup + "/cgroup.procs", fmt("%d", (pid_t) pid));
+            writeFile(*cgroup + "/cgroup.procs", fmt("%d", pid.get()));
 
         /* Signal the builder that we've updated its user namespace. */
         writeFull(userNamespaceSync.writeSide.get(), "1");
@@ -1339,11 +1324,11 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual In
         return path;
     }
 
-    void narFromPath(const StorePath & path, Sink & sink) override
+    WireFormatGenerator narFromPath(const StorePath & path) override
     {
         if (!goal.isAllowed(path))
             throw InvalidPath("cannot dump unknown path '%s' in recursive Nix", printStorePath(path));
-        LocalFSStore::narFromPath(path, sink);
+        return LocalFSStore::narFromPath(path);
     }
 
     void ensurePath(const StorePath & path) override
@@ -1585,7 +1570,7 @@ void LocalDerivationGoal::addDependency(const StorePath & path)
                entering its mount namespace, which is not possible
                in multithreaded programs. So we do this in a
                child process.*/
-            Pid child(startProcess([&]() {
+            Pid child = startProcess([&]() {
 
                 if (usingUserNamespace && (setns(sandboxUserNamespace.get(), 0) == -1))
                     throw SysError("entering sandbox user namespace");
@@ -1596,7 +1581,7 @@ void LocalDerivationGoal::addDependency(const StorePath & path)
                 doBind(source, target);
 
                 _exit(0);
-            }));
+            });
 
             int status = child.wait();
             if (status != 0)
@@ -2122,8 +2107,8 @@ void LocalDerivationGoal::runChild()
         bool allowLocalNetworking = parsedDrv->getBoolAttr("__darwinAllowLocalNetworking");
 
         /* The tmpDir in scope points at the temporary build directory for our derivation. Some packages try different mechanisms
-           to find temporary directories, so we want to open up a broader place for them to dump their files, if needed. */
-        Path globalTmpDir = canonPath(getEnvNonEmpty("TMPDIR").value_or("/tmp"), true);
+           to find temporary directories, so we want to open up a broader place for them to put their files, if needed. */
+        Path globalTmpDir = canonPath(defaultTempDir(), true);
 
         /* They don't like trailing slashes on subpath directives */
         if (globalTmpDir.back() == '/') globalTmpDir.pop_back();
@@ -2177,31 +2162,8 @@ void LocalDerivationGoal::runChild()
             }
         }
 
-#if __APPLE__
-        posix_spawnattr_t attrp;
-
-        if (posix_spawnattr_init(&attrp))
-            throw SysError("failed to initialize builder");
-
-        if (posix_spawnattr_setflags(&attrp, POSIX_SPAWN_SETEXEC))
-            throw SysError("failed to initialize builder");
-
-        if (drv->platform == "aarch64-darwin") {
-            // Unset kern.curproc_arch_affinity so we can escape Rosetta
-            int affinity = 0;
-            sysctlbyname("kern.curproc_arch_affinity", NULL, NULL, &affinity, sizeof(affinity));
-
-            cpu_type_t cpu = CPU_TYPE_ARM64;
-            posix_spawnattr_setbinpref_np(&attrp, 1, &cpu, NULL);
-        } else if (drv->platform == "x86_64-darwin") {
-            cpu_type_t cpu = CPU_TYPE_X86_64;
-            posix_spawnattr_setbinpref_np(&attrp, 1, &cpu, NULL);
-        }
-
-        posix_spawn(NULL, drv->builder.c_str(), NULL, &attrp, stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data());
-#else
-        execve(drv->builder.c_str(), stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data());
-#endif
+        execBuilder(drv->builder, args, envStrs);
+        // execBuilder should not return
 
         throw SysError("executing '%1%'", drv->builder);
 
@@ -2217,6 +2179,11 @@ void LocalDerivationGoal::runChild()
     }
 }
 
+void LocalDerivationGoal::execBuilder(std::string builder, Strings args, Strings envStrs)
+{
+    execve(builder.c_str(), stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data());
+}
+
 
 SingleDrvOutputs LocalDerivationGoal::registerOutputs()
 {
@@ -2422,14 +2389,10 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
             if (!rewrites.empty()) {
                 debug("rewriting hashes in '%1%'; cross fingers", actualPath);
 
-                /* FIXME: Is this actually streaming? */
-                auto source = sinkToSource([&](Sink & nextSink) {
-                    RewritingSink rsink(rewrites, nextSink);
-                    dumpPath(actualPath, rsink);
-                    rsink.flush();
-                });
+                GeneratorSource dump{dumpPath(actualPath)};
+                RewritingSource rewritten(rewrites, dump);
                 Path tmpPath = actualPath + ".tmp";
-                restorePath(tmpPath, *source);
+                restorePath(tmpPath, rewritten);
                 deletePath(actualPath);
                 movePath(tmpPath, actualPath);
 
@@ -2483,23 +2446,21 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
             rewriteOutput(outputRewrites);
             /* FIXME optimize and deduplicate with addToStore */
             std::string oldHashPart { scratchPath->hashPart() };
-            HashModuloSink caSink { outputHash.hashType, oldHashPart };
-            std::visit(overloaded {
-                [&](const TextIngestionMethod &) {
-                    readFileSource(actualPath)->drainInto(caSink);
+            auto input = std::visit(overloaded {
+                [&](const TextIngestionMethod &) -> GeneratorSource {
+                    return GeneratorSource(readFileSource(actualPath));
                 },
-                [&](const FileIngestionMethod & m2) {
+                [&](const FileIngestionMethod & m2) -> GeneratorSource {
                     switch (m2) {
                     case FileIngestionMethod::Recursive:
-                        dumpPath(actualPath, caSink);
-                        break;
+                        return GeneratorSource(dumpPath(actualPath));
                     case FileIngestionMethod::Flat:
-                        readFileSource(actualPath)->drainInto(caSink);
-                        break;
+                        return GeneratorSource(readFileSource(actualPath));
                     }
+                    assert(false);
                 },
             }, outputHash.method.raw);
-            auto got = caSink.finish().first;
+            auto got = computeHashModulo(outputHash.hashType, oldHashPart, input).first;
 
             auto optCA = ContentAddressWithReferences::fromPartsOpt(
                 outputHash.method,

src/libstore/build/local-derivation-goal.hh

@@ -178,7 +178,28 @@ struct LocalDerivationGoal : public DerivationGoal
 
     friend struct RestrictedStore;
 
-    using DerivationGoal::DerivationGoal;
+    /**
+     * Create a LocalDerivationGoal without an on-disk .drv file,
+     * possibly a platform-specific subclass
+     */
+    static std::shared_ptr<LocalDerivationGoal> makeLocalDerivationGoal(
+        const StorePath & drvPath,
+        const OutputsSpec & wantedOutputs,
+        Worker & worker,
+        BuildMode buildMode
+    );
+
+    /**
+     * Create a LocalDerivationGoal for an on-disk .drv file,
+     * possibly a platform-specific subclass
+     */
+    static std::shared_ptr<LocalDerivationGoal> makeLocalDerivationGoal(
+        const StorePath & drvPath,
+        const BasicDerivation & drv,
+        const OutputsSpec & wantedOutputs,
+        Worker & worker,
+        BuildMode buildMode
+    );
 
     virtual ~LocalDerivationGoal() noexcept(false) override;
 
@@ -282,7 +303,7 @@ struct LocalDerivationGoal : public DerivationGoal
      * Kill any processes running under the build user UID or in the
      * cgroup of the build.
      */
-    void killSandbox(bool getStats);
+    virtual void killSandbox(bool getStats);
 
     /**
      * Create alternative path calculated from but distinct from the
@@ -299,6 +320,16 @@ struct LocalDerivationGoal : public DerivationGoal
      * rewrites caught everything
      */
     StorePath makeFallbackPath(OutputNameView outputName);
+
+protected:
+    using DerivationGoal::DerivationGoal;
+
+    /**
+     * Execute the builder, replacing the current process.
+     * Generally this means an `execve` call.
+     */
+    virtual void execBuilder(std::string builder, Strings args, Strings envStrs);
+
 };
 
 }

src/libstore/build/substitution-goal.cc

@@ -217,6 +217,7 @@ void PathSubstitutionGoal::tryToRun()
     promise = std::promise<void>();
 
     thr = std::thread([this]() {
+        auto & fetchPath = subPath ? *subPath : storePath;
         try {
             ReceiveInterrupts receiveInterrupts;
 
@@ -226,10 +227,17 @@ void PathSubstitutionGoal::tryToRun()
             Activity act(*logger, actSubstitute, Logger::Fields{worker.store.printStorePath(storePath), sub->getUri()});
             PushActivity pact(act.id);
 
-            copyStorePath(*sub, worker.store,
-                subPath ? *subPath : storePath, repair, sub->isTrusted ? NoCheckSigs : CheckSigs);
+            copyStorePath(
+                *sub, worker.store, fetchPath, repair, sub->isTrusted ? NoCheckSigs : CheckSigs
+            );
 
             promise.set_value();
+        } catch (const EndOfFile &) {
+            promise.set_exception(std::make_exception_ptr(EndOfFile(
+                "NAR for '%s' fetched from '%s' is incomplete",
+                sub->printStorePath(fetchPath),
+                sub->getUri()
+            )));
         } catch (...) {
             promise.set_exception(std::current_exception());
         }

src/libstore/build/worker.cc

@@ -65,8 +65,8 @@ std::shared_ptr<DerivationGoal> Worker::makeDerivationGoal(const StorePath & drv
 {
     return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() -> std::shared_ptr<DerivationGoal> {
         return !dynamic_cast<LocalStore *>(&store)
-            ? std::make_shared</* */DerivationGoal>(drvPath, wantedOutputs, *this, buildMode)
-            : std::make_shared<LocalDerivationGoal>(drvPath, wantedOutputs, *this, buildMode);
+            ? std::make_shared<DerivationGoal>(drvPath, wantedOutputs, *this, buildMode)
+            : LocalDerivationGoal::makeLocalDerivationGoal(drvPath, wantedOutputs, *this, buildMode);
     });
 }
 
@@ -76,8 +76,8 @@ std::shared_ptr<DerivationGoal> Worker::makeBasicDerivationGoal(const StorePath
 {
     return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() -> std::shared_ptr<DerivationGoal> {
         return !dynamic_cast<LocalStore *>(&store)
-            ? std::make_shared</* */DerivationGoal>(drvPath, drv, wantedOutputs, *this, buildMode)
-            : std::make_shared<LocalDerivationGoal>(drvPath, drv, wantedOutputs, *this, buildMode);
+            ? std::make_shared<DerivationGoal>(drvPath, drv, wantedOutputs, *this, buildMode)
+            : LocalDerivationGoal::makeLocalDerivationGoal(drvPath, drv, wantedOutputs, *this, buildMode);
     });
 }
 

src/libstore/builtins/fetchurl.cc

@@ -32,23 +32,19 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
 
     auto fetch = [&](const std::string & url) {
 
-        auto source = sinkToSource([&](Sink & sink) {
+        /* No need to do TLS verification, because we check the hash of
+            the result anyway. */
+        FileTransferRequest request(url);
+        request.verifyTLS = false;
 
-            /* No need to do TLS verification, because we check the hash of
-               the result anyway. */
-            FileTransferRequest request(url);
-            request.verifyTLS = false;
-
-            auto decompressor = makeDecompressionSink(
-                unpack && mainUrl.ends_with(".xz") ? "xz" : "none", sink);
-            fileTransfer->download(std::move(request))->drainInto(*decompressor);
-            decompressor->finish();
-        });
+        auto raw = fileTransfer->download(std::move(request));
+        auto decompressor = makeDecompressionSource(
+            unpack && mainUrl.ends_with(".xz") ? "xz" : "none", *raw);
 
         if (unpack)
-            restorePath(storePath, *source);
+            restorePath(storePath, *decompressor);
         else
-            writeFile(storePath, *source);
+            writeFile(storePath, *decompressor);
 
         auto executable = drv.env.find("executable");
         if (executable != drv.env.end() && executable->second == "1") {

src/libstore/common-protocol-impl.hh

@@ -20,9 +20,9 @@ namespace nix {
     { \
         return LengthPrefixedProtoHelper<CommonProto, T >::read(store, conn); \
     } \
-    TEMPLATE void CommonProto::Serialise< T >::write(const Store & store, CommonProto::WriteConn conn, const T & t) \
+    TEMPLATE [[nodiscard]] WireFormatGenerator CommonProto::Serialise< T >::write(const Store & store, CommonProto::WriteConn conn, const T & t) \
     { \
-        LengthPrefixedProtoHelper<CommonProto, T >::write(store, conn, t); \
+        return LengthPrefixedProtoHelper<CommonProto, T >::write(store, conn, t); \
     }
 
 COMMON_USE_LENGTH_PREFIX_SERIALISER(template<typename T>, std::vector<T>)

src/libstore/common-protocol.cc

@@ -16,9 +16,9 @@ std::string CommonProto::Serialise<std::string>::read(const Store & store, Commo
     return readString(conn.from);
 }
 
-void CommonProto::Serialise<std::string>::write(const Store & store, CommonProto::WriteConn conn, const std::string & str)
+WireFormatGenerator CommonProto::Serialise<std::string>::write(const Store & store, CommonProto::WriteConn conn, const std::string & str)
 {
-    conn.to << str;
+    co_yield str;
 }
 
 
@@ -27,9 +27,9 @@ StorePath CommonProto::Serialise<StorePath>::read(const Store & store, CommonPro
     return store.parseStorePath(readString(conn.from));
 }
 
-void CommonProto::Serialise<StorePath>::write(const Store & store, CommonProto::WriteConn conn, const StorePath & storePath)
+WireFormatGenerator CommonProto::Serialise<StorePath>::write(const Store & store, CommonProto::WriteConn conn, const StorePath & storePath)
 {
-    conn.to << store.printStorePath(storePath);
+    co_yield store.printStorePath(storePath);
 }
 
 
@@ -38,9 +38,9 @@ ContentAddress CommonProto::Serialise<ContentAddress>::read(const Store & store,
     return ContentAddress::parse(readString(conn.from));
 }
 
-void CommonProto::Serialise<ContentAddress>::write(const Store & store, CommonProto::WriteConn conn, const ContentAddress & ca)
+WireFormatGenerator CommonProto::Serialise<ContentAddress>::write(const Store & store, CommonProto::WriteConn conn, const ContentAddress & ca)
 {
-    conn.to << renderContentAddress(ca);
+    co_yield renderContentAddress(ca);
 }
 
 
@@ -53,9 +53,9 @@ Realisation CommonProto::Serialise<Realisation>::read(const Store & store, Commo
     );
 }
 
-void CommonProto::Serialise<Realisation>::write(const Store & store, CommonProto::WriteConn conn, const Realisation & realisation)
+WireFormatGenerator CommonProto::Serialise<Realisation>::write(const Store & store, CommonProto::WriteConn conn, const Realisation & realisation)
 {
-    conn.to << realisation.toJSON().dump();
+    co_yield realisation.toJSON().dump();
 }
 
 
@@ -64,9 +64,9 @@ DrvOutput CommonProto::Serialise<DrvOutput>::read(const Store & store, CommonPro
     return DrvOutput::parse(readString(conn.from));
 }
 
-void CommonProto::Serialise<DrvOutput>::write(const Store & store, CommonProto::WriteConn conn, const DrvOutput & drvOutput)
+WireFormatGenerator CommonProto::Serialise<DrvOutput>::write(const Store & store, CommonProto::WriteConn conn, const DrvOutput & drvOutput)
 {
-    conn.to << drvOutput.to_string();
+    co_yield drvOutput.to_string();
 }
 
 
@@ -76,9 +76,11 @@ std::optional<StorePath> CommonProto::Serialise<std::optional<StorePath>>::read(
     return s == "" ? std::optional<StorePath> {} : store.parseStorePath(s);
 }
 
-void CommonProto::Serialise<std::optional<StorePath>>::write(const Store & store, CommonProto::WriteConn conn, const std::optional<StorePath> & storePathOpt)
+WireFormatGenerator CommonProto::Serialise<std::optional<StorePath>>::write(const Store & store, CommonProto::WriteConn conn, const std::optional<StorePath> & storePathOpt)
 {
-    conn.to << (storePathOpt ? store.printStorePath(*storePathOpt) : "");
+    return [](std::string s) -> WireFormatGenerator {
+        co_yield s;
+    }(storePathOpt ? store.printStorePath(*storePathOpt) : "");
 }
 
 
@@ -87,9 +89,11 @@ std::optional<ContentAddress> CommonProto::Serialise<std::optional<ContentAddres
     return ContentAddress::parseOpt(readString(conn.from));
 }
 
-void CommonProto::Serialise<std::optional<ContentAddress>>::write(const Store & store, CommonProto::WriteConn conn, const std::optional<ContentAddress> & caOpt)
+WireFormatGenerator CommonProto::Serialise<std::optional<ContentAddress>>::write(const Store & store, CommonProto::WriteConn conn, const std::optional<ContentAddress> & caOpt)
 {
-    conn.to << (caOpt ? renderContentAddress(*caOpt) : "");
+    return [](std::string s) -> WireFormatGenerator {
+        co_yield s;
+    }(caOpt ? renderContentAddress(*caOpt) : "");
 }
 
 }

src/libstore/common-protocol.hh

@@ -48,9 +48,10 @@ struct CommonProto
      * infer the type instead of having to write it down explicitly.
      */
     template<typename T>
-    static void write(const Store & store, WriteConn conn, const T & t)
+    [[nodiscard]]
+    static WireFormatGenerator write(const Store & store, WriteConn conn, const T & t)
     {
-        CommonProto::Serialise<T>::write(store, conn, t);
+        return CommonProto::Serialise<T>::write(store, conn, t);
     }
 };
 
@@ -58,7 +59,7 @@ struct CommonProto
     struct CommonProto::Serialise< T > \
     { \
         static T read(const Store & store, CommonProto::ReadConn conn); \
-        static void write(const Store & store, CommonProto::WriteConn conn, const T & str); \
+        [[nodiscard]] static WireFormatGenerator write(const Store & store, CommonProto::WriteConn conn, const T & str); \
     }
 
 template<>

src/libstore/daemon.cc

@@ -160,8 +160,7 @@ struct TunnelSink : Sink
     TunnelSink(Sink & to) : to(to) { }
     void operator () (std::string_view data)
     {
-        to << STDERR_WRITE;
-        writeString(data, to);
+        to << STDERR_WRITE << data;
     }
 };
 
@@ -292,18 +291,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         }
         auto res = store->queryValidPaths(paths, substitute);
         logger->stopWork();
-        WorkerProto::write(*store, wconn, res);
-        break;
-    }
-
-    case WorkerProto::Op::HasSubstitutes: {
-        auto path = store->parseStorePath(readString(from));
-        logger->startWork();
-        StorePathSet paths; // FIXME
-        paths.insert(path);
-        auto res = store->querySubstitutablePaths(paths);
-        logger->stopWork();
-        to << (res.count(path) != 0);
+        wconn.to << WorkerProto::write(*store, wconn, res);
         break;
     }
 
@@ -312,40 +300,78 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         logger->startWork();
         auto res = store->querySubstitutablePaths(paths);
         logger->stopWork();
-        WorkerProto::write(*store, wconn, res);
+        wconn.to << WorkerProto::write(*store, wconn, res);
+        break;
+    }
+
+    case WorkerProto::Op::HasSubstitutes: {
+        throw UnimplementedError("HasSubstitutes is not supported in Lix. This is not used if the declared server protocol is > 1.12 (Nix 1.0, 2012)");
         break;
     }
 
     case WorkerProto::Op::QueryPathHash: {
-        auto path = store->parseStorePath(readString(from));
-        logger->startWork();
-        auto hash = store->queryPathInfo(path)->narHash;
-        logger->stopWork();
-        to << hash.to_string(Base16, false);
+        throw UnimplementedError("QueryPathHash is not supported in Lix, client usages were removed in 2016 in e0204f8d462041387651af388074491fd0bf36d6");
+        break;
+    }
+
+    case WorkerProto::Op::QueryReferences: {
+        throw UnimplementedError("QueryReferences is not supported in Lix, client usages were removed in 2016 in e0204f8d462041387651af388074491fd0bf36d6");
+        break;
+    }
+
+    case WorkerProto::Op::QueryDeriver: {
+        throw UnimplementedError("QueryDeriver is not supported in Lix, client usages were removed in 2016 in e0204f8d462041387651af388074491fd0bf36d6");
+        break;
+    }
+
+    case WorkerProto::Op::ExportPath: {
+        throw UnimplementedError("ExportPath is not supported in Lix, client usage were removed in 2017 in 27dc76c1a5dbe654465245ff5f6bc22e2c8902da");
+        break;
+    }
+
+    case WorkerProto::Op::ImportPaths: {
+        throw UnimplementedError("ImportPaths is not supported in Lix. This is not used if the declared server protocol is >= 1.18 (Nix 2.0, 2016)");
         break;
     }
 
-    case WorkerProto::Op::QueryReferences:
     case WorkerProto::Op::QueryReferrers:
     case WorkerProto::Op::QueryValidDerivers:
     case WorkerProto::Op::QueryDerivationOutputs: {
         auto path = store->parseStorePath(readString(from));
         logger->startWork();
         StorePathSet paths;
-        if (op == WorkerProto::Op::QueryReferences)
-            for (auto & i : store->queryPathInfo(path)->references)
-                paths.insert(i);
-        else if (op == WorkerProto::Op::QueryReferrers)
-            store->queryReferrers(path, paths);
-        else if (op == WorkerProto::Op::QueryValidDerivers)
-            paths = store->queryValidDerivers(path);
-        else paths = store->queryDerivationOutputs(path);
+
+        #pragma GCC diagnostic push
+        #pragma GCC diagnostic ignored "-Wswitch-enum"
+        switch (op) {
+            case WorkerProto::Op::QueryReferrers: {
+                store->queryReferrers(path, paths);
+                break;
+            }
+            case WorkerProto::Op::QueryValidDerivers: {
+                paths = store->queryValidDerivers(path);
+                break;
+            }
+            case WorkerProto::Op::QueryDerivationOutputs: {
+                // Only sent if server presents proto version <= 1.21
+                REMOVE_AFTER_DROPPING_PROTO_MINOR(21);
+                paths = store->queryDerivationOutputs(path);
+                break;
+            }
+            default:
+                abort();
+                break;
+        }
+        #pragma GCC diagnostic pop
+
         logger->stopWork();
-        WorkerProto::write(*store, wconn, paths);
+        wconn.to << WorkerProto::write(*store, wconn, paths);
         break;
     }
 
     case WorkerProto::Op::QueryDerivationOutputNames: {
+        // Unused in CppNix >= 2.4 (removed in 045b07200c77bf1fe19c0a986aafb531e7e1ba54)
+        REMOVE_AFTER_DROPPING_PROTO_MINOR(31);
         auto path = store->parseStorePath(readString(from));
         logger->startWork();
         auto names = store->readDerivation(path).outputNames();
@@ -359,16 +385,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         logger->startWork();
         auto outputs = store->queryPartialDerivationOutputMap(path);
         logger->stopWork();
-        WorkerProto::write(*store, wconn, outputs);
-        break;
-    }
-
-    case WorkerProto::Op::QueryDeriver: {
-        auto path = store->parseStorePath(readString(from));
-        logger->startWork();
-        auto info = store->queryPathInfo(path);
-        logger->stopWork();
-        to << (info->deriver ? store->printStorePath(*info->deriver) : "");
+        wconn.to << WorkerProto::write(*store, wconn, outputs);
         break;
     }
 
@@ -415,7 +432,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
             }();
             logger->stopWork();
 
-            WorkerProto::Serialise<ValidPathInfo>::write(*store, wconn, *pathInfo);
+            wconn.to << WorkerProto::Serialise<ValidPathInfo>::write(*store, wconn, *pathInfo);
         } else {
             HashType hashAlgo;
             std::string baseName;
@@ -436,7 +453,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
                 hashAlgo = parseHashType(hashAlgoRaw);
             }
 
-            auto dumpSource = sinkToSource([&](Sink & saved) {
+            GeneratorSource dumpSource{[&]() -> WireFormatGenerator {
                 if (method == FileIngestionMethod::Recursive) {
                     /* We parse the NAR dump through into `saved` unmodified,
                        so why all this extra work? We still parse the NAR so
@@ -446,18 +463,33 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
                        command. (We don't trust `addToStoreFromDump` to not
                        eagerly consume the entire stream it's given, past the
                        length of the Nar. */
-                    copyNAR(from, saved);
+                    co_yield copyNAR(from);
                 } else {
                     /* Incrementally parse the NAR file, stripping the
                        metadata, and streaming the sole file we expect into
                        `saved`. */
-                    RetrieveRegularNARSink savedRegular { saved };
-                    parseDump(savedRegular, from);
-                    if (!savedRegular.regular) throw Error("regular file expected");
+                    auto parser = nar::parse(from);
+                    nar::File * file = nullptr;
+                    while (auto entry = parser.next()) {
+                        file = std::visit(
+                            overloaded{
+                                [](nar::MetadataString) -> nar::File * { return nullptr; },
+                                [](nar::MetadataRaw) -> nar::File * { return nullptr; },
+                                [](nar::File & f) -> nar::File * { return &f; },
+                                [](auto &) -> nar::File * { throw Error("regular file expected"); },
+                            },
+                            *entry
+                        );
+                        if (file) {
+                            break;
+                        }
+                    }
+                    assert(file); // should never fail unless the nar is empty, which would be invalid
+                    co_yield std::move(file->contents);
                 }
-            });
+            }()};
             logger->startWork();
-            auto path = store->addToStoreFromDump(*dumpSource, baseName, method, hashAlgo);
+            auto path = store->addToStoreFromDump(dumpSource, baseName, method, hashAlgo);
             logger->stopWork();
 
             to << store->printStorePath(path);
@@ -493,29 +525,6 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         break;
     }
 
-    case WorkerProto::Op::ExportPath: {
-        auto path = store->parseStorePath(readString(from));
-        readInt(from); // obsolete
-        logger->startWork();
-        TunnelSink sink(to);
-        store->exportPath(path, sink);
-        logger->stopWork();
-        to << 1;
-        break;
-    }
-
-    case WorkerProto::Op::ImportPaths: {
-        logger->startWork();
-        TunnelSource source(from, to);
-        auto paths = store->importPaths(source,
-            trusted ? NoCheckSigs : CheckSigs);
-        logger->stopWork();
-        Strings paths2;
-        for (auto & i : paths) paths2.push_back(store->printStorePath(i));
-        to << paths2;
-        break;
-    }
-
     case WorkerProto::Op::BuildPaths: {
         auto drvs = WorkerProto::Serialise<DerivedPaths>::read(*store, rconn);
         BuildMode mode = buildModeFromInteger(readInt(from));
@@ -554,7 +563,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         auto results = store->buildPathsWithResults(drvs, mode);
         logger->stopWork();
 
-        WorkerProto::write(*store, wconn, results);
+        wconn.to << WorkerProto::write(*store, wconn, results);
 
         break;
     }
@@ -632,7 +641,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
 
         auto res = store->buildDerivation(drvPath, drv, buildMode);
         logger->stopWork();
-        WorkerProto::write(*store, wconn, res);
+        wconn.to << WorkerProto::write(*store, wconn, res);
         break;
     }
 
@@ -666,8 +675,10 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         break;
     }
 
-    // Obsolete.
+    // Obsolete since 9947f1646a26b339fff2e02b77798e9841fac7f0 (included in CppNix 2.5.0).
     case WorkerProto::Op::SyncWithGC: {
+        // CppNix 2.5.0 is 32
+        REMOVE_AFTER_DROPPING_PROTO_MINOR(31);
         logger->startWork();
         logger->stopWork();
         to << 1;
@@ -764,7 +775,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         else {
             to << 1
                << (i->second.deriver ? store->printStorePath(*i->second.deriver) : "");
-            WorkerProto::write(*store, wconn, i->second.references);
+            wconn.to << WorkerProto::write(*store, wconn, i->second.references);
             to << i->second.downloadSize
                << i->second.narSize;
         }
@@ -787,7 +798,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         for (auto & i : infos) {
             to << store->printStorePath(i.first)
                << (i.second.deriver ? store->printStorePath(*i.second.deriver) : "");
-            WorkerProto::write(*store, wconn, i.second.references);
+            wconn.to << WorkerProto::write(*store, wconn, i.second.references);
             to << i.second.downloadSize << i.second.narSize;
         }
         break;
@@ -797,7 +808,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         logger->startWork();
         auto paths = store->queryAllValidPaths();
         logger->stopWork();
-        WorkerProto::write(*store, wconn, paths);
+        wconn.to << WorkerProto::write(*store, wconn, paths);
         break;
     }
 
@@ -814,7 +825,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         logger->stopWork();
         if (info) {
             to << 1;
-            WorkerProto::write(*store, wconn, static_cast<const UnkeyedValidPathInfo &>(*info));
+            wconn.to << WorkerProto::write(*store, wconn, static_cast<const UnkeyedValidPathInfo &>(*info));
         } else {
             to << 0;
         }
@@ -854,7 +865,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         auto path = store->parseStorePath(readString(from));
         logger->startWork();
         logger->stopWork();
-        dumpPath(store->toRealPath(path), to);
+        to << dumpPath(store->toRealPath(path));
         break;
     }
 
@@ -888,7 +899,6 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
 
         else {
             std::unique_ptr<Source> source;
-            StringSink saved;
             source = std::make_unique<TunnelSource>(from, to);
 
             logger->startWork();
@@ -910,9 +920,9 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         uint64_t downloadSize, narSize;
         store->queryMissing(targets, willBuild, willSubstitute, unknown, downloadSize, narSize);
         logger->stopWork();
-        WorkerProto::write(*store, wconn, willBuild);
-        WorkerProto::write(*store, wconn, willSubstitute);
-        WorkerProto::write(*store, wconn, unknown);
+        wconn.to << WorkerProto::write(*store, wconn, willBuild);
+        wconn.to << WorkerProto::write(*store, wconn, willSubstitute);
+        wconn.to << WorkerProto::write(*store, wconn, unknown);
         to << downloadSize << narSize;
         break;
     }
@@ -940,11 +950,11 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         if (GET_PROTOCOL_MINOR(clientVersion) < 31) {
             std::set<StorePath> outPaths;
             if (info) outPaths.insert(info->outPath);
-            WorkerProto::write(*store, wconn, outPaths);
+            wconn.to << WorkerProto::write(*store, wconn, outPaths);
         } else {
             std::set<Realisation> realisations;
             if (info) realisations.insert(*info);
-            WorkerProto::write(*store, wconn, realisations);
+            wconn.to << WorkerProto::write(*store, wconn, realisations);
         }
         break;
     }
@@ -1025,7 +1035,7 @@ void processConnection(
             ? store->isTrustedClient()
             : std::optional { NotTrusted };
         WorkerProto::WriteConn wconn {to, clientVersion};
-        WorkerProto::write(*store, wconn, temp);
+        wconn.to << WorkerProto::write(*store, wconn, temp);
     }
 
     /* Send startup error messages to the client. */

src/libstore/derivations.cc

@@ -994,7 +994,7 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr
             },
         }, i.second.raw);
     }
-    CommonProto::write(store,
+    out << CommonProto::write(store,
         CommonProto::WriteConn { .to = out },
         drv.inputSrcs);
     out << drv.platform << drv.builder << drv.args;

src/libstore/dummy-store.cc

@@ -63,7 +63,7 @@ struct DummyStore : public virtual DummyStoreConfig, public virtual Store
         RepairFlag repair) override
     { unsupported("addTextToStore"); }
 
-    void narFromPath(const StorePath & path, Sink & sink) override
+    WireFormatGenerator narFromPath(const StorePath & path) override
     { unsupported("narFromPath"); }
 
     std::shared_ptr<const Realisation> queryRealisationUncached(const DrvOutput &) override

src/libstore/export-import.cc

@@ -33,7 +33,7 @@ void Store::exportPath(const StorePath & path, Sink & sink)
     HashSink hashSink(htSHA256);
     TeeSink teeSink(sink, hashSink);
 
-    narFromPath(path, teeSink);
+    teeSink << narFromPath(path);
 
     /* Refuse to export paths that have changed.  This prevents
        filesystem corruption from spreading to other machines.
@@ -45,11 +45,10 @@ void Store::exportPath(const StorePath & path, Sink & sink)
 
     teeSink
         << exportMagic
-        << printStorePath(path);
-    CommonProto::write(*this,
-        CommonProto::WriteConn { .to = teeSink },
-        info->references);
-    teeSink
+        << printStorePath(path)
+        << CommonProto::write(*this,
+            CommonProto::WriteConn { .to = teeSink },
+            info->references)
         << (info->deriver ? printStorePath(*info->deriver) : "")
         << 0;
 }
@@ -64,7 +63,7 @@ StorePaths Store::importPaths(Source & source, CheckSigsFlag checkSigs)
 
         /* Extract the NAR from the source. */
         StringSink saved;
-        copyNAR(source, saved);
+        saved << copyNAR(source);
 
         uint32_t magic = readInt(source);
         if (magic != exportMagic)

src/libstore/globals.cc

@@ -429,7 +429,7 @@ void initLibStore() {
     /* On macOS, don't use the per-session TMPDIR (as set e.g. by
        sshd). This breaks build users because they don't have access
        to the TMPDIR, in particular in ‘nix-store --serve’. */
-    if (getEnv("TMPDIR").value_or("/tmp").starts_with("/var/folders/"))
+    if (defaultTempDir().starts_with("/var/folders/"))
         unsetenv("TMPDIR");
 #endif
 

src/libstore/globals.hh

@@ -582,16 +582,36 @@ public:
     Setting<std::string> sandboxShmSize{
         this, "50%", "sandbox-dev-shm-size",
         R"(
-          This option determines the maximum size of the `tmpfs` filesystem
-          mounted on `/dev/shm` in Linux sandboxes. For the format, see the
-          description of the `size` option of `tmpfs` in mount(8). The default
-          is `50%`.
+            *Linux only*
+
+            This option determines the maximum size of the `tmpfs` filesystem
+            mounted on `/dev/shm` in Linux sandboxes. For the format, see the
+            description of the `size` option of `tmpfs` in mount(8). The default
+            is `50%`.
         )"};
 
     Setting<Path> sandboxBuildDir{this, "/build", "sandbox-build-dir",
-        "The build directory inside the sandbox."};
+        R"(
+            *Linux only*
+
+            The build directory inside the sandbox.
+
+            This directory is backed by [`build-dir`](#conf-build-dir) on the host.
+        )"};
 #endif
 
+    Setting<std::optional<Path>> buildDir{this, std::nullopt, "build-dir",
+        R"(
+            The directory on the host, in which derivations' temporary build directories are created.
+
+            If not set, Nix will use the system temporary directory indicated by the `TMPDIR` environment variable.
+            Note that builds are often performed by the Nix daemon, so its `TMPDIR` is used, and not that of the Nix command line interface.
+
+            This is also the location where [`--keep-failed`](@docroot@/command-ref/opt-common.md#opt-keep-failed) leaves its files.
+
+            If Nix runs without sandbox, or if the platform does not support sandboxing with bind mounts (e.g. macOS), then the [`builder`](@docroot@/language/derivations.md#attr-builder)'s environment will contain this directory, instead of the virtual location [`sandbox-build-dir`](#conf-sandbox-build-dir).
+        )"};
+
     Setting<PathSet> allowedImpureHostPrefixes{this, {}, "allowed-impure-host-deps",
         "Which prefixes to allow derivations to ask for access to (primarily for Darwin)."};
 

src/libstore/http-binary-cache-store.cc

@@ -150,12 +150,12 @@ protected:
 
     }
 
-    void getFile(const std::string & path, Sink & sink) override
+    box_ptr<Source> getFile(const std::string & path) override
     {
         checkEnabled();
         auto request(makeRequest(path));
         try {
-            getFileTransfer()->download(std::move(request))->drainInto(sink);
+            return getFileTransfer()->download(std::move(request));
         } catch (FileTransferError & e) {
             if (e.error == FileTransfer::NotFound || e.error == FileTransfer::Forbidden)
                 throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache '%s'", path, getUri());

src/libstore/legacy-ssh-store.cc

@@ -185,7 +185,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor
                 << printStorePath(info.path)
                 << (info.deriver ? printStorePath(*info.deriver) : "")
                 << info.narHash.to_string(Base16, false);
-            ServeProto::write(*this, *conn, info.references);
+            conn->to << ServeProto::write(*this, *conn, info.references);
             conn->to
                 << info.registrationTime
                 << info.narSize
@@ -193,7 +193,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor
                 << info.sigs
                 << renderContentAddress(info.ca);
             try {
-                copyNAR(source, conn->to);
+                conn->to << copyNAR(source);
             } catch (...) {
                 conn->good = false;
                 throw;
@@ -206,7 +206,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor
                 << ServeProto::Command::ImportPaths
                 << 1;
             try {
-                copyNAR(source, conn->to);
+                conn->to << copyNAR(source);
             } catch (...) {
                 conn->good = false;
                 throw;
@@ -214,7 +214,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor
             conn->to
                 << exportMagic
                 << printStorePath(info.path);
-            ServeProto::write(*this, *conn, info.references);
+            conn->to << ServeProto::write(*this, *conn, info.references);
             conn->to
                 << (info.deriver ? printStorePath(*info.deriver) : "")
                 << 0
@@ -227,13 +227,15 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor
             throw Error("failed to add path '%s' to remote host '%s'", printStorePath(info.path), host);
     }
 
-    void narFromPath(const StorePath & path, Sink & sink) override
+    WireFormatGenerator narFromPath(const StorePath & path) override
     {
         auto conn(connections->get());
 
         conn->to << ServeProto::Command::DumpStorePath << printStorePath(path);
         conn->to.flush();
-        copyNAR(conn->from, sink);
+        return [] (auto conn) -> WireFormatGenerator {
+            co_yield copyNAR(conn->from);
+        }(std::move(conn));
     }
 
     std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override
@@ -364,7 +366,7 @@ public:
         conn->to
             << ServeProto::Command::QueryClosure
             << includeOutputs;
-        ServeProto::write(*this, *conn, paths);
+        conn->to << ServeProto::write(*this, *conn, paths);
         conn->to.flush();
 
         for (auto & i : ServeProto::Serialise<StorePathSet>::read(*this, *conn))
@@ -380,7 +382,7 @@ public:
             << ServeProto::Command::QueryValidPaths
             << false // lock
             << maybeSubstitute;
-        ServeProto::write(*this, *conn, paths);
+        conn->to << ServeProto::write(*this, *conn, paths);
         conn->to.flush();
 
         return ServeProto::Serialise<StorePathSet>::read(*this, *conn);

src/libstore/length-prefixed-protocol-helper.hh

@@ -7,6 +7,7 @@
  */
 
 #include "types.hh"
+#include "serialise.hh"
 
 namespace nix {
 
@@ -45,7 +46,7 @@ struct LengthPrefixedProtoHelper;
     struct LengthPrefixedProtoHelper< Inner, T > \
     { \
         static T read(const Store & store, typename Inner::ReadConn conn); \
-        static void write(const Store & store, typename Inner::WriteConn conn, const T & str); \
+        [[nodiscard]] static WireFormatGenerator write(const Store & store, typename Inner::WriteConn conn, const T & str); \
     private: \
         template<typename U> using S = typename Inner::template Serialise<U>; \
     }
@@ -78,13 +79,13 @@ LengthPrefixedProtoHelper<Inner, std::vector<T>>::read(
 }
 
 template<class Inner, typename T>
-void
+WireFormatGenerator
 LengthPrefixedProtoHelper<Inner, std::vector<T>>::write(
     const Store & store, typename Inner::WriteConn conn, const std::vector<T> & resSet)
 {
-    conn.to << resSet.size();
+    co_yield resSet.size();
     for (auto & key : resSet) {
-        S<T>::write(store, conn, key);
+        co_yield S<T>::write(store, conn, key);
     }
 }
 
@@ -102,13 +103,13 @@ LengthPrefixedProtoHelper<Inner, std::set<T>>::read(
 }
 
 template<class Inner, typename T>
-void
+WireFormatGenerator
 LengthPrefixedProtoHelper<Inner, std::set<T>>::write(
     const Store & store, typename Inner::WriteConn conn, const std::set<T> & resSet)
 {
-    conn.to << resSet.size();
+    co_yield resSet.size();
     for (auto & key : resSet) {
-        S<T>::write(store, conn, key);
+        co_yield S<T>::write(store, conn, key);
     }
 }
 
@@ -128,14 +129,14 @@ LengthPrefixedProtoHelper<Inner, std::map<K, V>>::read(
 }
 
 template<class Inner, typename K, typename V>
-void
+WireFormatGenerator
 LengthPrefixedProtoHelper<Inner, std::map<K, V>>::write(
     const Store & store, typename Inner::WriteConn conn, const std::map<K, V> & resMap)
 {
-    conn.to << resMap.size();
+    co_yield resMap.size();
     for (auto & i : resMap) {
-        S<K>::write(store, conn, i.first);
-        S<V>::write(store, conn, i.second);
+        co_yield S<K>::write(store, conn, i.first);
+        co_yield S<V>::write(store, conn, i.second);
     }
 }
 
@@ -150,13 +151,24 @@ LengthPrefixedProtoHelper<Inner, std::tuple<Ts...>>::read(
 }
 
 template<class Inner, typename... Ts>
-void
+WireFormatGenerator
 LengthPrefixedProtoHelper<Inner, std::tuple<Ts...>>::write(
     const Store & store, typename Inner::WriteConn conn, const std::tuple<Ts...> & res)
 {
-    std::apply([&]<typename... Us>(const Us &... args) {
-        (S<Us>::write(store, conn, args), ...);
-    }, res);
+    auto fullArgs = std::apply(
+        [&](auto &... rest) {
+            return std::tuple<const Store &, typename Inner::WriteConn &, const Ts &...>(
+                std::cref(store), conn, rest...
+            );
+        },
+        res
+    );
+    return std::apply(
+        []<typename... Us>(auto & store, auto conn, const Us &... args) -> WireFormatGenerator {
+            (co_yield S<Us>::write(store, conn, args), ...);
+        },
+        fullArgs
+    );
 }
 
 }

src/libstore/local-binary-cache-store.cc

@@ -68,10 +68,10 @@ protected:
         del.cancel();
     }
 
-    void getFile(const std::string & path, Sink & sink) override
+    box_ptr<Source> getFile(const std::string & path) override
     {
         try {
-            readFileSource(binaryCacheDir + "/" + path)->drainInto(sink);
+            return make_box_ptr<GeneratorSource>(readFileSource(binaryCacheDir + "/" + path));
         } catch (SysError & e) {
             if (e.errNo == ENOENT)
                 throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache", path);

src/libstore/local-fs-store.cc

@@ -78,11 +78,11 @@ ref<FSAccessor> LocalFSStore::getFSAccessor()
             std::dynamic_pointer_cast<LocalFSStore>(shared_from_this())));
 }
 
-void LocalFSStore::narFromPath(const StorePath & path, Sink & sink)
+WireFormatGenerator LocalFSStore::narFromPath(const StorePath & path)
 {
     if (!isValidPath(path))
         throw Error("path '%s' does not exist in store", printStorePath(path));
-    dumpPath(getRealStoreDir() + std::string(printStorePath(path), storeDir.size()), sink);
+    return dumpPath(getRealStoreDir() + std::string(printStorePath(path), storeDir.size()));
 }
 
 const std::string LocalFSStore::drvsLogDir = "drvs";

src/libstore/local-fs-store.hh

@@ -42,7 +42,7 @@ public:
 
     LocalFSStore(const Params & params);
 
-    void narFromPath(const StorePath & path, Sink & sink) override;
+    WireFormatGenerator narFromPath(const StorePath & path) override;
     ref<FSAccessor> getFSAccessor() override;
 
     /**

src/libstore/local-store.cc

@@ -1406,7 +1406,7 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name
             auto narHash = std::pair { hash, size };
             if (method != FileIngestionMethod::Recursive || hashAlgo != htSHA256) {
                 HashSink narSink { htSHA256 };
-                dumpPath(realPath, narSink);
+                narSink << dumpPath(realPath);
                 narHash = narSink.finish();
             }
 
@@ -1461,7 +1461,7 @@ StorePath LocalStore::addTextToStore(
             canonicalisePathMetaData(realPath, {});
 
             StringSink sink;
-            dumpString(s, sink);
+            sink << dumpString(s);
             auto narHash = hashString(htSHA256, sink.s);
 
             optimisePath(realPath, repair);
@@ -1601,7 +1601,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
 
                 auto hashSink = HashSink(info->narHash.type);
 
-                dumpPath(Store::toRealPath(i), hashSink);
+                hashSink << dumpPath(Store::toRealPath(i));
                 auto current = hashSink.finish();
 
                 if (info->narHash != nullHash && info->narHash != current.first) {
@@ -1887,25 +1887,23 @@ ContentAddress LocalStore::hashCAPath(
     const std::string_view pathHash
 )
 {
-    HashModuloSink caSink ( hashType, std::string(pathHash) );
-    std::visit(overloaded {
-        [&](const TextIngestionMethod &) {
-            readFileSource(path)->drainInto(caSink);
+    auto data = std::visit(overloaded {
+        [&](const TextIngestionMethod &) -> GeneratorSource {
+            return GeneratorSource(readFileSource(path));
         },
-        [&](const FileIngestionMethod & m2) {
+        [&](const FileIngestionMethod & m2) -> GeneratorSource {
             switch (m2) {
             case FileIngestionMethod::Recursive:
-                dumpPath(path, caSink);
-                break;
+                return GeneratorSource(dumpPath(path));
             case FileIngestionMethod::Flat:
-                readFileSource(path)->drainInto(caSink);
-                break;
+                return GeneratorSource(readFileSource(path));
             }
+            assert(false);
         },
     }, method.raw);
     return ContentAddress {
         .method = method,
-        .hash = caSink.finish().first,
+        .hash = computeHashModulo(hashType, std::string(pathHash), data).first,
     };
 }
 

src/libstore/make-content-addressed.cc

@@ -23,7 +23,7 @@ std::map<StorePath, StorePath> makeContentAddressed(
         std::string oldHashPart(path.hashPart());
 
         StringSink sink;
-        srcStore.narFromPath(path, sink);
+        sink << srcStore.narFromPath(path);
 
         StringMap rewrites;
 
@@ -43,10 +43,10 @@ std::map<StorePath, StorePath> makeContentAddressed(
 
         sink.s = rewriteStrings(sink.s, rewrites);
 
-        HashModuloSink hashModuloSink(htSHA256, oldHashPart);
-        hashModuloSink(sink.s);
-
-        auto narModuloHash = hashModuloSink.finish().first;
+        auto narModuloHash = [&] {
+            StringSource source{sink.s};
+            return computeHashModulo(htSHA256, oldHashPart, source).first;
+        }();
 
         ValidPathInfo info {
             dstStore,
@@ -61,15 +61,12 @@ std::map<StorePath, StorePath> makeContentAddressed(
 
         printInfo("rewriting '%s' to '%s'", pathS, dstStore.printStorePath(info.path));
 
-        StringSink sink2;
-        RewritingSink rsink2(oldHashPart, std::string(info.path.hashPart()), sink2);
-        rsink2(sink.s);
-        rsink2.flush();
+        const auto rewritten = rewriteStrings(sink.s, {{oldHashPart, std::string(info.path.hashPart())}});
 
-        info.narHash = hashString(htSHA256, sink2.s);
+        info.narHash = hashString(htSHA256, rewritten);
         info.narSize = sink.s.size();
 
-        StringSource source(sink2.s);
+        StringSource source(rewritten);
         dstStore.addToStore(info, source);
 
         remappings.insert_or_assign(std::move(path), std::move(info.path));

src/libstore/path-references.cc

@@ -61,7 +61,7 @@ StorePathSet scanForReferences(
     TeeSink sink { refsSink, toTee };
 
     /* Look for the hashes in the NAR dump of the path. */
-    dumpPath(path, sink);
+    sink << dumpPath(path);
 
     return refsSink.getResultPaths();
 }

src/libstore/platform.cc

@@ -1,4 +1,5 @@
 #include "local-store.hh"
+#include "build/local-derivation-goal.hh"
 
 #if __linux__
 #include "platform/linux.hh"
@@ -19,4 +20,43 @@ std::shared_ptr<LocalStore> LocalStore::makeLocalStore(const Params & params)
     return std::shared_ptr<LocalStore>(new FallbackLocalStore(params));
 #endif
 }
+
+std::shared_ptr<LocalDerivationGoal> LocalDerivationGoal::makeLocalDerivationGoal(
+    const StorePath & drvPath,
+    const OutputsSpec & wantedOutputs,
+    Worker & worker,
+    BuildMode buildMode
+)
+{
+#if __linux__
+    return std::make_shared<LinuxLocalDerivationGoal>(drvPath, wantedOutputs, worker, buildMode);
+#elif __APPLE__
+    return std::make_shared<DarwinLocalDerivationGoal>(drvPath, wantedOutputs, worker, buildMode);
+#else
+    return std::make_shared<FallbackLocalDerivationGoal>(drvPath, wantedOutputs, worker, buildMode);
+#endif
+}
+
+std::shared_ptr<LocalDerivationGoal> LocalDerivationGoal::makeLocalDerivationGoal(
+    const StorePath & drvPath,
+    const BasicDerivation & drv,
+    const OutputsSpec & wantedOutputs,
+    Worker & worker,
+    BuildMode buildMode
+)
+{
+#if __linux__
+    return std::make_shared<LinuxLocalDerivationGoal>(
+        drvPath, drv, wantedOutputs, worker, buildMode
+    );
+#elif __APPLE__
+    return std::make_shared<DarwinLocalDerivationGoal>(
+        drvPath, drv, wantedOutputs, worker, buildMode
+    );
+#else
+    return std::make_shared<FallbackLocalDerivationGoal>(
+        drvPath, drv, wantedOutputs, worker, buildMode
+    );
+#endif
+}
 }

src/libstore/platform/darwin.cc

@@ -6,6 +6,7 @@
 #include <sys/proc_info.h>
 #include <sys/sysctl.h>
 #include <libproc.h>
+#include <spawn.h>
 
 #include <regex>
 
@@ -220,4 +221,29 @@ void DarwinLocalStore::findPlatformRoots(UncheckedRoots & unchecked)
         }
     }
 }
+
+void DarwinLocalDerivationGoal::execBuilder(std::string builder, Strings args, Strings envStrs)
+{
+    posix_spawnattr_t attrp;
+
+    if (posix_spawnattr_init(&attrp))
+        throw SysError("failed to initialize builder");
+
+    if (posix_spawnattr_setflags(&attrp, POSIX_SPAWN_SETEXEC))
+        throw SysError("failed to initialize builder");
+
+    if (drv->platform == "aarch64-darwin") {
+        // Unset kern.curproc_arch_affinity so we can escape Rosetta
+        int affinity = 0;
+        sysctlbyname("kern.curproc_arch_affinity", NULL, NULL, &affinity, sizeof(affinity));
+
+        cpu_type_t cpu = CPU_TYPE_ARM64;
+        posix_spawnattr_setbinpref_np(&attrp, 1, &cpu, NULL);
+    } else if (drv->platform == "x86_64-darwin") {
+        cpu_type_t cpu = CPU_TYPE_X86_64;
+        posix_spawnattr_setbinpref_np(&attrp, 1, &cpu, NULL);
+    }
+
+    posix_spawn(NULL, builder.c_str(), NULL, &attrp, stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data());
+}
 }

src/libstore/platform/darwin.hh

@@ -1,6 +1,7 @@
 #pragma once
 ///@file
 
+#include "build/local-derivation-goal.hh"
 #include "gc-store.hh"
 #include "local-store.hh"
 
@@ -32,4 +33,19 @@ private:
     void findPlatformRoots(UncheckedRoots & unchecked) override;
 };
 
+/**
+ * Darwin-specific implementation of LocalDerivationGoal
+ */
+class DarwinLocalDerivationGoal : public LocalDerivationGoal
+{
+public:
+    using LocalDerivationGoal::LocalDerivationGoal;
+
+private:
+    /**
+     * Set process flags to enter or leave rosetta, then execute the builder
+     */
+    void execBuilder(std::string builder, Strings args, Strings envStrs) override;
+};
+
 }

src/libstore/platform/fallback.hh

@@ -1,6 +1,7 @@
 #pragma once
 ///@file
 
+#include "build/local-derivation-goal.hh"
 #include "local-store.hh"
 
 namespace nix {
@@ -28,4 +29,14 @@ public:
     }
 };
 
+/**
+ * Fallback platform implementation of LocalDerivationGoal
+ * Exists so we can make LocalDerivationGoal constructor protected
+ */
+class FallbackLocalDerivationGoal : public LocalDerivationGoal
+{
+public:
+    using LocalDerivationGoal::LocalDerivationGoal;
+};
+
 }

src/libstore/platform/linux.cc

@@ -1,3 +1,4 @@
+#include "cgroup.hh"
 #include "gc-store.hh"
 #include "signals.hh"
 #include "platform/linux.hh"
@@ -114,4 +115,17 @@ void LinuxLocalStore::findPlatformRoots(UncheckedRoots & unchecked)
     readFileRoots("/proc/sys/kernel/fbsplash", unchecked);
     readFileRoots("/proc/sys/kernel/poweroff_cmd", unchecked);
 }
+
+void LinuxLocalDerivationGoal::killSandbox(bool getStats)
+{
+    if (cgroup) {
+        auto stats = destroyCgroup(*cgroup);
+        if (getStats) {
+            buildResult.cpuUser = stats.cpuUser;
+            buildResult.cpuSystem = stats.cpuSystem;
+        }
+    } else {
+        LocalDerivationGoal::killSandbox(getStats);
+    }
+}
 }

src/libstore/platform/linux.hh

@@ -1,6 +1,7 @@
 #pragma once
 ///@file
 
+#include "build/local-derivation-goal.hh"
 #include "gc-store.hh"
 #include "local-store.hh"
 
@@ -32,4 +33,16 @@ private:
     void findPlatformRoots(UncheckedRoots & unchecked) override;
 };
 
+/**
+ * Linux-specific implementation of LocalDerivationGoal
+ */
+class LinuxLocalDerivationGoal : public LocalDerivationGoal
+{
+public:
+    using LocalDerivationGoal::LocalDerivationGoal;
+
+private:
+    void killSandbox(bool getStatus) override;
+};
+
 }

src/libstore/remote-fs-accessor.cc

@@ -97,7 +97,7 @@ std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_, boo
     }
 
     StringSink sink;
-    store->narFromPath(storePath, sink);
+    sink << store->narFromPath(storePath);
     return {addToCache(storePath.hashPart(), std::move(sink.s)), restPath};
 }
 

src/libstore/remote-store.cc

@@ -203,7 +203,7 @@ StorePathSet RemoteStore::queryValidPaths(const StorePathSet & paths, Substitute
 {
     auto conn(getConnection());
     conn->to << WorkerProto::Op::QueryValidPaths;
-    WorkerProto::write(*this, *conn, paths);
+    conn->to << WorkerProto::write(*this, *conn, paths);
     if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 27) {
         conn->to << maybeSubstitute;
     }
@@ -225,7 +225,7 @@ StorePathSet RemoteStore::querySubstitutablePaths(const StorePathSet & paths)
 {
     auto conn(getConnection());
     conn->to << WorkerProto::Op::QuerySubstitutablePaths;
-    WorkerProto::write(*this, *conn, paths);
+    conn->to << WorkerProto::write(*this, *conn, paths);
     conn.processStderr();
     return WorkerProto::Serialise<StorePathSet>::read(*this, *conn);
 }
@@ -243,9 +243,9 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S
         StorePathSet paths;
         for (auto & path : pathsMap)
             paths.insert(path.first);
-        WorkerProto::write(*this, *conn, paths);
+        conn->to << WorkerProto::write(*this, *conn, paths);
     } else
-        WorkerProto::write(*this, *conn, pathsMap);
+        conn->to << WorkerProto::write(*this, *conn, pathsMap);
     conn.processStderr();
     size_t count = readNum<size_t>(conn->from);
     for (size_t n = 0; n < count; n++) {
@@ -307,6 +307,7 @@ StorePathSet RemoteStore::queryDerivationOutputs(const StorePath & path)
     if (GET_PROTOCOL_MINOR(getProtocol()) >= 22) {
         return Store::queryDerivationOutputs(path);
     }
+    REMOVE_AFTER_DROPPING_PROTO_MINOR(21);
     auto conn(getConnection());
     conn->to << WorkerProto::Op::QueryDerivationOutputs << printStorePath(path);
     conn.processStderr();
@@ -336,6 +337,7 @@ std::map<std::string, std::optional<StorePath>> RemoteStore::queryPartialDerivat
             return outputs;
         }
     } else {
+        REMOVE_AFTER_DROPPING_PROTO_MINOR(21);
         auto & evalStore = evalStore_ ? *evalStore_ : *this;
         // Fallback for old daemon versions.
         // For floating-CA derivations (and their co-dependencies) this is an
@@ -375,7 +377,7 @@ ref<const ValidPathInfo> RemoteStore::addCAToStore(
             << WorkerProto::Op::AddToStore
             << name
             << caMethod.render(hashType);
-        WorkerProto::write(*this, *conn, references);
+        conn->to << WorkerProto::write(*this, *conn, references);
         conn->to << repair;
 
         // The dump source may invoke the store, so we need to make some room.
@@ -400,7 +402,7 @@ ref<const ValidPathInfo> RemoteStore::addCAToStore(
                         name, printHashType(hashType));
                 std::string s = dump.drain();
                 conn->to << WorkerProto::Op::AddTextToStore << name << s;
-                WorkerProto::write(*this, *conn, references);
+                conn->to << WorkerProto::write(*this, *conn, references);
                 conn.processStderr();
             },
             [&](const FileIngestionMethod & fim) -> void {
@@ -420,7 +422,7 @@ ref<const ValidPathInfo> RemoteStore::addCAToStore(
                             dump.drainInto(conn->to);
                         } else {
                             std::string contents = dump.drain();
-                            dumpString(contents, conn->to);
+                            conn->to << dumpString(contents);
                         }
                     }
                     conn.processStderr();
@@ -460,14 +462,14 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
              << printStorePath(info.path)
              << (info.deriver ? printStorePath(*info.deriver) : "")
              << info.narHash.to_string(Base16, false);
-    WorkerProto::write(*this, *conn, info.references);
+    conn->to << WorkerProto::write(*this, *conn, info.references);
     conn->to << info.registrationTime << info.narSize
              << info.ultimate << info.sigs << renderContentAddress(info.ca)
              << repair << !checkSigs;
 
     if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 23) {
         conn.withFramedSink([&](Sink & sink) {
-            copyNAR(source, sink);
+            sink << copyNAR(source);
         });
     } else {
         conn.processStderr(0, &source);
@@ -483,17 +485,25 @@ void RemoteStore::addMultipleToStore(
 {
     auto remoteVersion = getProtocol();
 
-    auto source = sinkToSource([&](Sink & sink) {
-        sink << pathsToCopy.size();
+    GeneratorSource source{[](auto self, auto & pathsToCopy, auto remoteVersion) -> WireFormatGenerator {
+        NullSink null; // TODO remove .to from WriteConn instead
+        co_yield pathsToCopy.size();
         for (auto & [pathInfo, pathSource] : pathsToCopy) {
-            WorkerProto::Serialise<ValidPathInfo>::write(*this,
-                 WorkerProto::WriteConn {sink, remoteVersion},
+            co_yield WorkerProto::Serialise<ValidPathInfo>::write(*self,
+                 WorkerProto::WriteConn {null, remoteVersion},
                  pathInfo);
-            pathSource->drainInto(sink);
+            try {
+                char buf[65536];
+                while (true) {
+                    const auto read = pathSource->read(buf, sizeof(buf));
+                    co_yield std::span{buf, read};
+                }
+            } catch (EndOfFile &) {
+            }
         }
-    });
+    }(this, pathsToCopy, remoteVersion)};
 
-    addMultipleToStore(*source, repair, checkSigs);
+    addMultipleToStore(source, repair, checkSigs);
 }
 
 void RemoteStore::addMultipleToStore(
@@ -530,10 +540,11 @@ void RemoteStore::registerDrvOutput(const Realisation & info)
     auto conn(getConnection());
     conn->to << WorkerProto::Op::RegisterDrvOutput;
     if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 31) {
+        REMOVE_AFTER_DROPPING_PROTO_MINOR(30);
         conn->to << info.id.to_string();
         conn->to << std::string(info.outPath.to_string());
     } else {
-        WorkerProto::write(*this, *conn, info);
+        conn->to << WorkerProto::write(*this, *conn, info);
     }
     conn.processStderr();
 }
@@ -594,7 +605,7 @@ void RemoteStore::buildPaths(const std::vector<DerivedPath> & drvPaths, BuildMod
 
     auto conn(getConnection());
     conn->to << WorkerProto::Op::BuildPaths;
-    WorkerProto::write(*this, *conn, drvPaths);
+    conn->to << WorkerProto::write(*this, *conn, drvPaths);
     conn->to << buildMode;
     conn.processStderr();
     readInt(conn->from);
@@ -612,11 +623,12 @@ std::vector<KeyedBuildResult> RemoteStore::buildPathsWithResults(
 
     if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 34) {
         conn->to << WorkerProto::Op::BuildPathsWithResults;
-        WorkerProto::write(*this, *conn, paths);
+        conn->to << WorkerProto::write(*this, *conn, paths);
         conn->to << buildMode;
         conn.processStderr();
         return WorkerProto::Serialise<std::vector<KeyedBuildResult>>::read(*this, *conn);
     } else {
+        REMOVE_AFTER_DROPPING_PROTO_MINOR(33);
         // Avoid deadlock.
         conn_.reset();
 
@@ -736,7 +748,7 @@ void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results)
 
     conn->to
         << WorkerProto::Op::CollectGarbage << options.action;
-    WorkerProto::write(*this, *conn, options.pathsToDelete);
+    conn->to << WorkerProto::write(*this, *conn, options.pathsToDelete);
     conn->to << options.ignoreLiveness
         << options.maxFreed
         /* removed options */
@@ -788,7 +800,7 @@ void RemoteStore::queryMissing(const std::vector<DerivedPath> & targets,
 {
     auto conn(getConnection());
     conn->to << WorkerProto::Op::QueryMissing;
-    WorkerProto::write(*this, *conn, targets);
+    conn->to << WorkerProto::write(*this, *conn, targets);
     conn.processStderr();
     willBuild = WorkerProto::Serialise<StorePathSet>::read(*this, *conn);
     willSubstitute = WorkerProto::Serialise<StorePathSet>::read(*this, *conn);
@@ -844,12 +856,14 @@ RemoteStore::Connection::~Connection()
     }
 }
 
-void RemoteStore::narFromPath(const StorePath & path, Sink & sink)
+WireFormatGenerator RemoteStore::narFromPath(const StorePath & path)
 {
     auto conn(connections->get());
     conn->to << WorkerProto::Op::NarFromPath << printStorePath(path);
     conn->processStderr();
-    copyNAR(conn->from, sink);
+    return [](auto conn) -> WireFormatGenerator {
+        co_yield copyNAR(conn->from);
+    }(std::move(conn));
 }
 
 ref<FSAccessor> RemoteStore::getFSAccessor()
@@ -893,7 +907,7 @@ std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source *
             if (!source) throw Error("no source");
             size_t len = readNum<size_t>(from);
             auto buf = std::make_unique<char[]>(len);
-            writeString({(const char *) buf.get(), source->read(buf.get(), len)}, to);
+            to << std::string_view((const char *) buf.get(), source->read(buf.get(), len));
             to.flush();
         }
 

src/libstore/remote-store.hh

@@ -183,7 +183,7 @@ protected:
 
     virtual ref<FSAccessor> getFSAccessor() override;
 
-    virtual void narFromPath(const StorePath & path, Sink & sink) override;
+    virtual WireFormatGenerator narFromPath(const StorePath & path) override;
 
 private:
 

src/libstore/s3-binary-cache-store.cc

@@ -455,7 +455,7 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual
             uploadFile(path, istream, mimeType, "");
     }
 
-    void getFile(const std::string & path, Sink & sink) override
+    box_ptr<Source> getFile(const std::string & path) override
     {
         stats.get++;
 
@@ -469,7 +469,11 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual
             printTalkative("downloaded 's3://%s/%s' (%d bytes) in %d ms",
                 bucketName, path, res.data->size(), res.durationMs);
 
-            sink(*res.data);
+            return make_box_ptr<GeneratorSource>(
+                [](std::string data) -> Generator<Bytes> {
+                    co_yield std::span{data.data(), data.size()};
+                }(std::move(*res.data))
+            );
         } else
             throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache '%s'", path, getUri());
     }

src/libstore/serve-protocol-impl.hh

@@ -20,9 +20,9 @@ namespace nix {
     { \
         return LengthPrefixedProtoHelper<ServeProto, T >::read(store, conn); \
     } \
-    TEMPLATE void ServeProto::Serialise< T >::write(const Store & store, ServeProto::WriteConn conn, const T & t) \
+    TEMPLATE [[nodiscard]] WireFormatGenerator ServeProto::Serialise< T >::write(const Store & store, ServeProto::WriteConn conn, const T & t) \
     { \
-        LengthPrefixedProtoHelper<ServeProto, T >::write(store, conn, t); \
+        return LengthPrefixedProtoHelper<ServeProto, T >::write(store, conn, t); \
     }
 
 SERVE_USE_LENGTH_PREFIX_SERIALISER(template<typename T>, std::vector<T>)
@@ -46,9 +46,10 @@ struct ServeProto::Serialise
         return CommonProto::Serialise<T>::read(store,
             CommonProto::ReadConn { .from = conn.from });
     }
-    static void write(const Store & store, ServeProto::WriteConn conn, const T & t)
+    [[nodiscard]]
+    static WireFormatGenerator write(const Store & store, ServeProto::WriteConn conn, const T & t)
     {
-        CommonProto::Serialise<T>::write(store,
+        return CommonProto::Serialise<T>::write(store,
             CommonProto::WriteConn { .to = conn.to },
             t);
     }

src/libstore/serve-protocol.cc

@@ -34,23 +34,22 @@ BuildResult ServeProto::Serialise<BuildResult>::read(const Store & store, ServeP
     return status;
 }
 
-void ServeProto::Serialise<BuildResult>::write(const Store & store, ServeProto::WriteConn conn, const BuildResult & status)
+WireFormatGenerator ServeProto::Serialise<BuildResult>::write(const Store & store, ServeProto::WriteConn conn, const BuildResult & status)
 {
-    conn.to
-        << status.status
-        << status.errorMsg;
+    co_yield status.status;
+    co_yield status.errorMsg;
 
-    if (GET_PROTOCOL_MINOR(conn.version) >= 3)
-        conn.to
-            << status.timesBuilt
-            << status.isNonDeterministic
-            << status.startTime
-            << status.stopTime;
+    if (GET_PROTOCOL_MINOR(conn.version) >= 3) {
+        co_yield status.timesBuilt;
+        co_yield status.isNonDeterministic;
+        co_yield status.startTime;
+        co_yield status.stopTime;
+    }
     if (GET_PROTOCOL_MINOR(conn.version) >= 6) {
         DrvOutputs builtOutputs;
         for (auto & [output, realisation] : status.builtOutputs)
             builtOutputs.insert_or_assign(realisation.id, realisation);
-        ServeProto::write(store, conn, builtOutputs);
+        co_yield ServeProto::write(store, conn, builtOutputs);
     }
 }
 
@@ -80,21 +79,19 @@ UnkeyedValidPathInfo ServeProto::Serialise<UnkeyedValidPathInfo>::read(const Sto
     return info;
 }
 
-void ServeProto::Serialise<UnkeyedValidPathInfo>::write(const Store & store, WriteConn conn, const UnkeyedValidPathInfo & info)
+WireFormatGenerator ServeProto::Serialise<UnkeyedValidPathInfo>::write(const Store & store, WriteConn conn, const UnkeyedValidPathInfo & info)
 {
-    conn.to
-        << (info.deriver ? store.printStorePath(*info.deriver) : "");
+    co_yield (info.deriver ? store.printStorePath(*info.deriver) : "");
 
-    ServeProto::write(store, conn, info.references);
+    co_yield ServeProto::write(store, conn, info.references);
     // !!! Maybe we want compression?
-    conn.to
-        << info.narSize // downloadSize, lie a little
-        << info.narSize;
-    if (GET_PROTOCOL_MINOR(conn.version) >= 4)
-        conn.to
-            << info.narHash.to_string(Base32, true)
-            << renderContentAddress(info.ca)
-            << info.sigs;
+    co_yield info.narSize; // downloadSize, lie a little
+    co_yield info.narSize;
+    if (GET_PROTOCOL_MINOR(conn.version) >= 4) {
+        co_yield info.narHash.to_string(Base32, true);
+        co_yield renderContentAddress(info.ca);
+        co_yield info.sigs;
+    }
 }
 
 }

src/libstore/serve-protocol.hh

@@ -8,6 +8,11 @@ namespace nix {
 #define SERVE_MAGIC_1 0x390c9deb
 #define SERVE_MAGIC_2 0x5452eecb
 
+// This must remain at 2.7 (Nix 2.18) forever in Lix, since the protocol
+// versioning is monotonic, so if we ever change it in the future, it will
+// break compatibility with any potential CppNix-originated protocol changes.
+//
+// Lix intends to replace this protocol entirely.
 #define SERVE_PROTOCOL_VERSION (2 << 8 | 7)
 #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00)
 #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff)
@@ -74,7 +79,7 @@ struct ServeProto
 #if 0
     {
         static T read(const Store & store, ReadConn conn);
-        static void write(const Store & store, WriteConn conn, const T & t);
+        static WireFormatGenerator write(const Store & store, WriteConn conn, const T & t);
     };
 #endif
 
@@ -83,9 +88,10 @@ struct ServeProto
      * infer the type instead of having to write it down explicitly.
      */
     template<typename T>
-    static void write(const Store & store, WriteConn conn, const T & t)
+    [[nodiscard]]
+    static WireFormatGenerator write(const Store & store, WriteConn conn, const T & t)
     {
-        ServeProto::Serialise<T>::write(store, conn, t);
+        return ServeProto::Serialise<T>::write(store, conn, t);
     }
 };
 
@@ -137,7 +143,7 @@ inline std::ostream & operator << (std::ostream & s, ServeProto::Command op)
     struct ServeProto::Serialise< T > \
     { \
         static T read(const Store & store, ServeProto::ReadConn conn); \
-        static void write(const Store & store, ServeProto::WriteConn conn, const T & t); \
+        [[nodiscard]] static WireFormatGenerator write(const Store & store, ServeProto::WriteConn conn, const T & t); \
     };
 
 template<>

src/libstore/ssh.cc

@@ -131,7 +131,7 @@ Path SSHMaster::startMaster()
 
     auto state(state_.lock());
 
-    if (state->sshMaster != -1) return state->socketPath;
+    if (state->sshMaster) return state->socketPath;
 
     state->socketPath = (Path) *state->tmpDir + "/ssh.sock";
 

src/libstore/store-api.cc

@@ -275,13 +275,11 @@ StorePath Store::addToStore(
     const StorePathSet & references)
 {
     Path srcPath(absPath(_srcPath));
-    auto source = sinkToSource([&](Sink & sink) {
-        if (method == FileIngestionMethod::Recursive)
-            dumpPath(srcPath, sink, filter);
-        else
-            readFileSource(srcPath)->drainInto(sink);
-    });
-    return addToStoreFromDump(*source, name, method, hashAlgo, repair, references);
+    auto source = GeneratorSource{
+        method == FileIngestionMethod::Recursive ? dumpPath(srcPath, filter).decay()
+                                                 : readFileSource(srcPath)
+    };
+    return addToStoreFromDump(source, name, method, hashAlgo, repair, references);
 }
 
 void Store::addMultipleToStore(
@@ -337,9 +335,9 @@ void Store::addMultipleToStore(
             info.ultimate = false;
 
             /* Make sure that the Source object is destroyed when
-               we're done. In particular, a SinkToSource object must
-               be destroyed to ensure that the destructors on its
-               stack frame are run; this includes
+               we're done. In particular, a coroutine object must
+               be destroyed to ensure that the destructors in its
+               state are run; this includes
                LegacySSHStore::narFromPath()'s connection lock. */
             auto source = std::move(source_);
 
@@ -425,13 +423,11 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath,
     /* Functionally, this means that fileSource will yield the content of
        srcPath. The fact that we use scratchpadSink as a temporary buffer here
        is an implementation detail. */
-    auto fileSource = sinkToSource([&](Sink & scratchpadSink) {
-        dumpPath(srcPath, scratchpadSink);
-    });
+    auto fileSource = GeneratorSource{dumpPath(srcPath)};
 
     /* tapped provides the same data as fileSource, but we also write all the
        information to narSink. */
-    TeeSource tapped { *fileSource, narSink };
+    TeeSource tapped { fileSource, narSink };
 
     ParseSink blank;
     auto & parseSink = method == FileIngestionMethod::Flat
@@ -466,10 +462,8 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath,
     info.narSize = narSize;
 
     if (!isValidPath(info.path)) {
-        auto source = sinkToSource([&](Sink & scratchpadSink) {
-            dumpPath(srcPath, scratchpadSink);
-        });
-        addToStore(info, *source);
+        auto source = GeneratorSource{dumpPath(srcPath)};
+        addToStore(info, source);
     }
 
     return info;
@@ -1065,18 +1059,17 @@ void copyStorePath(
         info = info2;
     }
 
-    auto source = sinkToSource([&](Sink & sink) {
-        LambdaSink progressSink([&, total = 0ULL](std::string_view data) mutable {
-            total += data.size();
+    GeneratorSource source{[](auto & act, auto & info, auto & srcStore, auto & storePath) -> WireFormatGenerator {
+        auto nar = srcStore.narFromPath(storePath);
+        uint64_t total = 0;
+        while (auto data = nar.next()) {
+            total += data->size();
             act.progress(total, info->narSize);
-        });
-        TeeSink tee { sink, progressSink };
-        srcStore.narFromPath(storePath, tee);
-    }, [&]() {
-           throw EndOfFile("NAR for '%s' fetched from '%s' is incomplete", srcStore.printStorePath(storePath), srcStore.getUri());
-    });
+            co_yield *data;
+        }
+    }(act, info, srcStore, storePath)};
 
-    dstStore.addToStore(*info, *source, repair, checkSigs);
+    dstStore.addToStore(*info, source, repair, checkSigs);
 }
 
 
@@ -1188,31 +1181,33 @@ std::map<StorePath, StorePath> copyPaths(
         ValidPathInfo infoForDst = *info;
         infoForDst.path = storePathForDst;
 
-        auto source =
-            sinkToSource([&srcStore, &dstStore, missingPath = missingPath, info = std::move(info)](Sink & sink) {
-                // We can reasonably assume that the copy will happen whenever we
-                // read the path, so log something about that at that point
-                auto srcUri = srcStore.getUri();
-                auto dstUri = dstStore.getUri();
-                auto storePathS = srcStore.printStorePath(missingPath);
-                Activity act(
-                    *logger,
-                    lvlInfo,
-                    actCopyPath,
-                    makeCopyPathMessage(srcUri, dstUri, storePathS),
-                    {storePathS, srcUri, dstUri}
-                );
-                PushActivity pact(act.id);
+        auto source = [](auto & srcStore, auto & dstStore, auto missingPath, auto info
+                      ) -> WireFormatGenerator {
+            // We can reasonably assume that the copy will happen whenever we
+            // read the path, so log something about that at that point
+            auto srcUri = srcStore.getUri();
+            auto dstUri = dstStore.getUri();
+            auto storePathS = srcStore.printStorePath(missingPath);
+            Activity act(
+                *logger,
+                lvlInfo,
+                actCopyPath,
+                makeCopyPathMessage(srcUri, dstUri, storePathS),
+                {storePathS, srcUri, dstUri}
+            );
+            PushActivity pact(act.id);
 
-                LambdaSink progressSink([&, total = 0ULL](std::string_view data) mutable {
-                    total += data.size();
-                    act.progress(total, info->narSize);
-                });
-                TeeSink tee{sink, progressSink};
-
-                srcStore.narFromPath(missingPath, tee);
-            });
-        pathsToCopy.push_back(std::pair{infoForDst, std::move(source)});
+            auto nar = srcStore.narFromPath(missingPath);
+            uint64_t total = 0;
+            while (auto data = nar.next()) {
+                total += data->size();
+                act.progress(total, info->narSize);
+                co_yield *data;
+            }
+        };
+        pathsToCopy.push_back(std::pair{
+            infoForDst, std::make_unique<GeneratorSource>(source(srcStore, dstStore, missingPath, info))
+        });
     }
 
     dstStore.addMultipleToStore(pathsToCopy, act, repair, checkSigs);

src/libstore/store-api.hh

@@ -577,9 +577,9 @@ public:
     { return registerDrvOutput(output); }
 
     /**
-     * Write a NAR dump of a store path.
+     * Generate a NAR dump of a store path.
      */
-    virtual void narFromPath(const StorePath & path, Sink & sink) = 0;
+    virtual WireFormatGenerator narFromPath(const StorePath & path) = 0;
 
     /**
      * For each path, if it's a derivation, build it.  Building a

src/libstore/uds-remote-store.hh

@@ -38,8 +38,8 @@ public:
     ref<FSAccessor> getFSAccessor() override
     { return LocalFSStore::getFSAccessor(); }
 
-    void narFromPath(const StorePath & path, Sink & sink) override
-    { LocalFSStore::narFromPath(path, sink); }
+    WireFormatGenerator narFromPath(const StorePath & path) override
+    { return LocalFSStore::narFromPath(path); }
 
     /**
      * Implementation of `IndirectRootStore::addIndirectRoot()` which

src/libstore/worker-protocol-impl.hh

@@ -20,9 +20,9 @@ namespace nix {
     { \
         return LengthPrefixedProtoHelper<WorkerProto, T >::read(store, conn); \
     } \
-    TEMPLATE void WorkerProto::Serialise< T >::write(const Store & store, WorkerProto::WriteConn conn, const T & t) \
+    TEMPLATE [[nodiscard]] WireFormatGenerator WorkerProto::Serialise< T >::write(const Store & store, WorkerProto::WriteConn conn, const T & t) \
     { \
-        LengthPrefixedProtoHelper<WorkerProto, T >::write(store, conn, t); \
+        return LengthPrefixedProtoHelper<WorkerProto, T >::write(store, conn, t); \
     }
 
 WORKER_USE_LENGTH_PREFIX_SERIALISER(template<typename T>, std::vector<T>)
@@ -46,9 +46,10 @@ struct WorkerProto::Serialise
         return CommonProto::Serialise<T>::read(store,
             CommonProto::ReadConn { .from = conn.from });
     }
-    static void write(const Store & store, WorkerProto::WriteConn conn, const T & t)
+    [[nodiscard]]
+    static WireFormatGenerator write(const Store & store, WorkerProto::WriteConn conn, const T & t)
     {
-        CommonProto::Serialise<T>::write(store,
+        return CommonProto::Serialise<T>::write(store,
             CommonProto::WriteConn { .to = conn.to },
             t);
     }

src/libstore/worker-protocol.cc

@@ -28,17 +28,17 @@ std::optional<TrustedFlag> WorkerProto::Serialise<std::optional<TrustedFlag>>::r
     }
 }
 
-void WorkerProto::Serialise<std::optional<TrustedFlag>>::write(const Store & store, WorkerProto::WriteConn conn, const std::optional<TrustedFlag> & optTrusted)
+WireFormatGenerator WorkerProto::Serialise<std::optional<TrustedFlag>>::write(const Store & store, WorkerProto::WriteConn conn, const std::optional<TrustedFlag> & optTrusted)
 {
     if (!optTrusted)
-        conn.to << (uint8_t)0;
+        co_yield (uint8_t)0;
     else {
         switch (*optTrusted) {
         case Trusted:
-            conn.to << (uint8_t)1;
+            co_yield (uint8_t)1;
             break;
         case NotTrusted:
-            conn.to << (uint8_t)2;
+            co_yield (uint8_t)2;
             break;
         default:
             assert(false);
@@ -57,23 +57,23 @@ DerivedPath WorkerProto::Serialise<DerivedPath>::read(const Store & store, Worke
     }
 }
 
-void WorkerProto::Serialise<DerivedPath>::write(const Store & store, WorkerProto::WriteConn conn, const DerivedPath & req)
+WireFormatGenerator WorkerProto::Serialise<DerivedPath>::write(const Store & store, WorkerProto::WriteConn conn, const DerivedPath & req)
 {
     if (GET_PROTOCOL_MINOR(conn.version) >= 30) {
-        conn.to << req.to_string_legacy(store);
+        co_yield req.to_string_legacy(store);
     } else {
         auto sOrDrvPath = StorePathWithOutputs::tryFromDerivedPath(req);
-        std::visit(overloaded {
-            [&](const StorePathWithOutputs & s) {
-                conn.to << s.to_string(store);
+        co_yield std::visit(overloaded {
+            [&](const StorePathWithOutputs & s) -> std::string {
+                return s.to_string(store);
             },
-            [&](const StorePath & drvPath) {
+            [&](const StorePath & drvPath) -> std::string {
                 throw Error("trying to request '%s', but daemon protocol %d.%d is too old (< 1.29) to request a derivation file",
                     store.printStorePath(drvPath),
                     GET_PROTOCOL_MAJOR(conn.version),
                     GET_PROTOCOL_MINOR(conn.version));
             },
-            [&](std::monostate) {
+            [&](std::monostate) -> std::string {
                 throw Error("wanted to build a derivation that is itself a build product, but protocols do not support that. Try upgrading the Nix implementation on the other end of this connection");
             },
         }, sOrDrvPath);
@@ -91,10 +91,10 @@ KeyedBuildResult WorkerProto::Serialise<KeyedBuildResult>::read(const Store & st
     };
 }
 
-void WorkerProto::Serialise<KeyedBuildResult>::write(const Store & store, WorkerProto::WriteConn conn, const KeyedBuildResult & res)
+WireFormatGenerator WorkerProto::Serialise<KeyedBuildResult>::write(const Store & store, WorkerProto::WriteConn conn, const KeyedBuildResult & res)
 {
-    WorkerProto::write(store, conn, res.path);
-    WorkerProto::write(store, conn, static_cast<const BuildResult &>(res));
+    co_yield WorkerProto::write(store, conn, res.path);
+    co_yield WorkerProto::write(store, conn, static_cast<const BuildResult &>(res));
 }
 
 
@@ -120,23 +120,21 @@ BuildResult WorkerProto::Serialise<BuildResult>::read(const Store & store, Worke
     return res;
 }
 
-void WorkerProto::Serialise<BuildResult>::write(const Store & store, WorkerProto::WriteConn conn, const BuildResult & res)
+WireFormatGenerator WorkerProto::Serialise<BuildResult>::write(const Store & store, WorkerProto::WriteConn conn, const BuildResult & res)
 {
-    conn.to
-        << res.status
-        << res.errorMsg;
+    co_yield res.status;
+    co_yield res.errorMsg;
     if (GET_PROTOCOL_MINOR(conn.version) >= 29) {
-        conn.to
-            << res.timesBuilt
-            << res.isNonDeterministic
-            << res.startTime
-            << res.stopTime;
+        co_yield res.timesBuilt;
+        co_yield res.isNonDeterministic;
+        co_yield res.startTime;
+        co_yield res.stopTime;
     }
     if (GET_PROTOCOL_MINOR(conn.version) >= 28) {
         DrvOutputs builtOutputs;
         for (auto & [output, realisation] : res.builtOutputs)
             builtOutputs.insert_or_assign(realisation.id, realisation);
-        WorkerProto::write(store, conn, builtOutputs);
+        co_yield WorkerProto::write(store, conn, builtOutputs);
     }
 }
 
@@ -150,10 +148,10 @@ ValidPathInfo WorkerProto::Serialise<ValidPathInfo>::read(const Store & store, R
     };
 }
 
-void WorkerProto::Serialise<ValidPathInfo>::write(const Store & store, WriteConn conn, const ValidPathInfo & pathInfo)
+WireFormatGenerator WorkerProto::Serialise<ValidPathInfo>::write(const Store & store, WriteConn conn, const ValidPathInfo & pathInfo)
 {
-    WorkerProto::write(store, conn, pathInfo.path);
-    WorkerProto::write(store, conn, static_cast<const UnkeyedValidPathInfo &>(pathInfo));
+    co_yield WorkerProto::write(store, conn, pathInfo.path);
+    co_yield WorkerProto::write(store, conn, static_cast<const UnkeyedValidPathInfo &>(pathInfo));
 }
 
 
@@ -173,18 +171,17 @@ UnkeyedValidPathInfo WorkerProto::Serialise<UnkeyedValidPathInfo>::read(const St
     return info;
 }
 
-void WorkerProto::Serialise<UnkeyedValidPathInfo>::write(const Store & store, WriteConn conn, const UnkeyedValidPathInfo & pathInfo)
+WireFormatGenerator WorkerProto::Serialise<UnkeyedValidPathInfo>::write(const Store & store, WriteConn conn, const UnkeyedValidPathInfo & pathInfo)
 {
-    conn.to
-        << (pathInfo.deriver ? store.printStorePath(*pathInfo.deriver) : "")
-        << pathInfo.narHash.to_string(Base16, false);
-    WorkerProto::write(store, conn, pathInfo.references);
-    conn.to << pathInfo.registrationTime << pathInfo.narSize;
+    co_yield (pathInfo.deriver ? store.printStorePath(*pathInfo.deriver) : "");
+    co_yield pathInfo.narHash.to_string(Base16, false);
+    co_yield WorkerProto::write(store, conn, pathInfo.references);
+    co_yield pathInfo.registrationTime;
+    co_yield pathInfo.narSize;
 
-    conn.to
-        << pathInfo.ultimate
-        << pathInfo.sigs
-        << renderContentAddress(pathInfo.ca);
+    co_yield pathInfo.ultimate;
+    co_yield pathInfo.sigs;
+    co_yield renderContentAddress(pathInfo.ca);
 }
 
 }

src/libstore/worker-protocol.hh

@@ -23,6 +23,10 @@ namespace nix {
 #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff)
 
 
+#define REMOVE_AFTER_DROPPING_PROTO_MINOR(protoMinor) \
+    static_assert(MIN_SUPPORTED_MINOR_WORKER_PROTO_VERSION <= (protoMinor))
+
+
 #define STDERR_NEXT  0x6f6c6d67
 #define STDERR_READ  0x64617461 // data needed from source
 #define STDERR_WRITE 0x64617416 // data for sink
@@ -118,7 +122,7 @@ struct WorkerProto
 #if 0
     {
         static T read(const Store & store, ReadConn conn);
-        static void write(const Store & store, WriteConn conn, const T & t);
+        static WireFormatGenerator write(const Store & store, WriteConn conn, const T & t);
     };
 #endif
 
@@ -127,39 +131,40 @@ struct WorkerProto
      * infer the type instead of having to write it down explicitly.
      */
     template<typename T>
-    static void write(const Store & store, WriteConn conn, const T & t)
+    [[nodiscard]]
+    static WireFormatGenerator write(const Store & store, WriteConn conn, const T & t)
     {
-        WorkerProto::Serialise<T>::write(store, conn, t);
+        return WorkerProto::Serialise<T>::write(store, conn, t);
     }
 };
 
 enum struct WorkerProto::Op : uint64_t
 {
     IsValidPath = 1,
-    HasSubstitutes = 3,
-    QueryPathHash = 4, // obsolete
-    QueryReferences = 5, // obsolete
+    HasSubstitutes = 3, // obsolete since 2012, stubbed to error
+    QueryPathHash = 4, // obsolete since 2016, stubbed to error
+    QueryReferences = 5, // obsolete since 2016, stubbed to error
     QueryReferrers = 6,
     AddToStore = 7,
-    AddTextToStore = 8, // obsolete since 1.25, Nix 3.0. Use WorkerProto::Op::AddToStore
+    AddTextToStore = 8, // obsolete since protocol 1.25, CppNix 2.4. Use WorkerProto::Op::AddToStore
     BuildPaths = 9,
     EnsurePath = 10,
     AddTempRoot = 11,
     AddIndirectRoot = 12,
-    SyncWithGC = 13,
+    SyncWithGC = 13, // obsolete since CppNix 2.5.0
     FindRoots = 14,
-    ExportPath = 16, // obsolete
-    QueryDeriver = 18, // obsolete
+    ExportPath = 16, // obsolete since 2017, stubbed to error
+    QueryDeriver = 18, // obsolete since 2016, stubbed to error
     SetOptions = 19,
     CollectGarbage = 20,
     QuerySubstitutablePathInfo = 21,
-    QueryDerivationOutputs = 22, // obsolete
+    QueryDerivationOutputs = 22, // obsolete since protocol 1.21, CppNix 2.4
     QueryAllValidPaths = 23,
-    QueryFailedPaths = 24,
-    ClearFailedPaths = 25,
+    QueryFailedPaths = 24, // obsolete, removed
+    ClearFailedPaths = 25, // obsolete, removed
     QueryPathInfo = 26,
-    ImportPaths = 27, // obsolete
-    QueryDerivationOutputNames = 28, // obsolete
+    ImportPaths = 27, // obsolete since 2016
+    QueryDerivationOutputNames = 28, // obsolete since CppNix 2.4
     QueryPathFromHashPart = 29,
     QuerySubstitutablePathInfos = 30,
     QueryValidPaths = 31,
@@ -215,7 +220,7 @@ inline std::ostream & operator << (std::ostream & s, WorkerProto::Op op)
     struct WorkerProto::Serialise< T > \
     { \
         static T read(const Store & store, WorkerProto::ReadConn conn); \
-        static void write(const Store & store, WorkerProto::WriteConn conn, const T & t); \
+        [[nodiscard]] static WireFormatGenerator write(const Store & store, WorkerProto::WriteConn conn, const T & t); \
     };
 
 template<>

src/libutil/archive.cc

@@ -1,5 +1,6 @@
 #include <cerrno>
 #include <algorithm>
+#include <string_view>
 #include <vector>
 #include <map>
 
@@ -13,6 +14,8 @@
 
 #include "archive.hh"
 #include "file-system.hh"
+#include "finally.hh"
+#include "serialise.hh"
 #include "config.hh"
 #include "logging.hh"
 #include "signals.hh"
@@ -40,10 +43,10 @@ static GlobalConfig::Register rArchiveSettings(&archiveSettings);
 PathFilter defaultPathFilter = [](const Path &) { return true; };
 
 
-static void dumpContents(const Path & path, off_t size,
-    Sink & sink)
+static WireFormatGenerator dumpContents(const Path & path, off_t size)
 {
-    sink << "contents" << size;
+    co_yield "contents";
+    co_yield size;
 
     AutoCloseFD fd{open(path.c_str(), O_RDONLY | O_CLOEXEC)};
     if (!fd) throw SysError("opening file '%1%'", path);
@@ -55,31 +58,35 @@ static void dumpContents(const Path & path, off_t size,
         auto n = std::min(left, buf.size());
         readFull(fd.get(), buf.data(), n);
         left -= n;
-        sink({buf.data(), n});
+        co_yield std::span{buf.data(), n};
     }
 
-    writePadding(size, sink);
+    co_yield SerializingTransform::padding(size);
 }
 
 
-static time_t dump(const Path & path, Sink & sink, PathFilter & filter)
+static WireFormatGenerator dump(const Path & path, time_t & mtime, PathFilter & filter)
 {
     checkInterrupt();
 
     auto st = lstat(path);
-    time_t result = st.st_mtime;
+    mtime = st.st_mtime;
 
-    sink << "(";
+    co_yield "(";
 
     if (S_ISREG(st.st_mode)) {
-        sink << "type" << "regular";
-        if (st.st_mode & S_IXUSR)
-            sink << "executable" << "";
-        dumpContents(path, st.st_size, sink);
+        co_yield "type";
+        co_yield "regular";
+        if (st.st_mode & S_IXUSR) {
+            co_yield "executable";
+            co_yield "";
+        }
+        co_yield dumpContents(path, st.st_size);
     }
 
     else if (S_ISDIR(st.st_mode)) {
-        sink << "type" << "directory";
+        co_yield "type";
+        co_yield "directory";
 
         /* If we're on a case-insensitive system like macOS, undo
            the case hack applied by restorePath(). */
@@ -101,41 +108,55 @@ static time_t dump(const Path & path, Sink & sink, PathFilter & filter)
 
         for (auto & i : unhacked)
             if (filter(path + "/" + i.first)) {
-                sink << "entry" << "(" << "name" << i.first << "node";
-                auto tmp_mtime = dump(path + "/" + i.second, sink, filter);
-                if (tmp_mtime > result) {
-                    result = tmp_mtime;
+                co_yield "entry";
+                co_yield "(";
+                co_yield "name";
+                co_yield i.first;
+                co_yield "node";
+                time_t tmp_mtime;
+                co_yield dump(path + "/" + i.second, tmp_mtime, filter);
+                if (tmp_mtime > mtime) {
+                    mtime = tmp_mtime;
                 }
-                sink << ")";
+                co_yield ")";
             }
     }
 
-    else if (S_ISLNK(st.st_mode))
-        sink << "type" << "symlink" << "target" << readLink(path);
+    else if (S_ISLNK(st.st_mode)) {
+        co_yield "type";
+        co_yield "symlink";
+        co_yield "target";
+        co_yield readLink(path);
+    }
 
     else throw Error("file '%1%' has an unsupported type", path);
 
-    sink << ")";
-
-    return result;
+    co_yield ")";
 }
 
 
-time_t dumpPathAndGetMtime(const Path & path, Sink & sink, PathFilter & filter)
+WireFormatGenerator dumpPathAndGetMtime(Path path, time_t & mtime, PathFilter & filter)
 {
-    sink << narVersionMagic1;
-    return dump(path, sink, filter);
+    co_yield narVersionMagic1;
+    co_yield dump(std::move(path), mtime, filter);
 }
 
-void dumpPath(const Path & path, Sink & sink, PathFilter & filter)
+WireFormatGenerator dumpPath(Path path, PathFilter & filter)
 {
-    dumpPathAndGetMtime(path, sink, filter);
+    time_t ignored;
+    co_yield dumpPathAndGetMtime(std::move(path), ignored, filter);
 }
 
 
-void dumpString(std::string_view s, Sink & sink)
+WireFormatGenerator dumpString(std::string_view s)
 {
-    sink << narVersionMagic1 << "(" << "type" << "regular" << "contents" << s << ")";
+    co_yield narVersionMagic1;
+    co_yield "(";
+    co_yield "type";
+    co_yield "regular";
+    co_yield "contents";
+    co_yield s;
+    co_yield ")";
 }
 
 
@@ -156,28 +177,6 @@ static void skipGeneric(Source & source)
 #endif
 
 
-static void parseContents(ParseSink & sink, Source & source, const Path & path)
-{
-    uint64_t size = readLongLong(source);
-
-    sink.preallocateContents(size);
-
-    uint64_t left = size;
-    std::array<char, 65536> buf;
-
-    while (left) {
-        checkInterrupt();
-        auto n = buf.size();
-        if ((uint64_t)n > left) n = left;
-        source(buf.data(), n);
-        sink.receiveContents({buf.data(), n});
-        left -= n;
-    }
-
-    readPadding(size, source);
-}
-
-
 struct CaseInsensitiveCompare
 {
     bool operator() (const std::string & a, const std::string & b) const
@@ -186,123 +185,204 @@ struct CaseInsensitiveCompare
     }
 };
 
+namespace nar {
 
-static void parse(ParseSink & sink, Source & source, const Path & path)
+static Generator<Entry> parseObject(Source & source, const Path & path)
 {
-    std::string s;
+#define EXPECT(raw, kind)                              \
+    do {                                               \
+        const auto s = readString(source);             \
+        if (s != raw) {                                \
+            throw badArchive("expected " kind " tag"); \
+        }                                              \
+        co_yield MetadataString{s};                    \
+    } while (0)
 
-    s = readString(source);
-    if (s != "(") throw badArchive("expected open tag");
+    EXPECT("(", "open");
+    EXPECT("type", "type");
 
-    enum { tpUnknown, tpRegular, tpDirectory, tpSymlink } type = tpUnknown;
+    checkInterrupt();
 
-    std::map<Path, int, CaseInsensitiveCompare> names;
+    const auto t = readString(source);
+    co_yield MetadataString{t};
 
-    while (1) {
-        checkInterrupt();
-
-        s = readString(source);
-
-        if (s == ")") {
-            break;
-        }
-
-        else if (s == "type") {
-            if (type != tpUnknown)
-                throw badArchive("multiple type fields");
-            std::string t = readString(source);
-
-            if (t == "regular") {
-                type = tpRegular;
-                sink.createRegularFile(path);
-            }
-
-            else if (t == "directory") {
-                sink.createDirectory(path);
-                type = tpDirectory;
-            }
-
-            else if (t == "symlink") {
-                type = tpSymlink;
-            }
-
-            else throw badArchive("unknown file type " + t);
-
-        }
-
-        else if (s == "contents" && type == tpRegular) {
-            parseContents(sink, source, path);
-            sink.closeRegularFile();
-        }
-
-        else if (s == "executable" && type == tpRegular) {
+    if (t == "regular") {
+        auto contentsOrFlag = readString(source);
+        co_yield MetadataString{contentsOrFlag};
+        const bool executable = contentsOrFlag == "executable";
+        if (executable) {
             auto s = readString(source);
-            if (s != "") throw badArchive("executable marker has non-empty value");
-            sink.isExecutable();
+            co_yield MetadataString{s};
+            if (s != "") {
+                throw badArchive("executable marker has non-empty value");
+            }
+            contentsOrFlag = readString(source);
+            co_yield MetadataString{contentsOrFlag};
         }
+        if (contentsOrFlag == "contents") {
+            uint64_t size = readLongLong(source);
+            co_yield MetadataRaw{SerializingTransform()(size)};
+            auto reader = [](Source & source, uint64_t left) -> Generator<Bytes> {
+                std::array<char, 65536> buf;
 
-        else if (s == "entry" && type == tpDirectory) {
-            std::string name, prevName;
-
-            s = readString(source);
-            if (s != "(") throw badArchive("expected open tag");
+                while (left) {
+                    checkInterrupt();
+                    auto n = std::min<uint64_t>(buf.size(), left);
+                    source(buf.data(), n);
+                    co_yield std::span{buf.data(), n};
+                    left -= n;
+                }
+            }(source, size);
+            co_yield File{path, executable, size, reader};
+            while (reader.next()) {
+                // ignore remainder of contents
+            }
+            readPadding(size, source);
+            co_yield MetadataRaw{SerializingTransform::padding(size)};
+        } else {
+            throw badArchive("file without contents found: " + path);
+        }
+    } else if (t == "directory") {
+        auto reader = [](Source & source, const Path & path) -> Generator<Entry> {
+            std::map<Path, int, CaseInsensitiveCompare> names;
+            std::string prevName;
 
             while (1) {
                 checkInterrupt();
 
-                s = readString(source);
-
-                if (s == ")") {
-                    break;
-                } else if (s == "name") {
-                    name = readString(source);
-                    if (name.empty() || name == "." || name == ".." || name.find('/') != std::string::npos || name.find((char) 0) != std::string::npos)
-                        throw Error("NAR contains invalid file name '%1%'", name);
-                    if (name <= prevName)
-                        throw Error("NAR directory is not sorted");
-                    prevName = name;
-                    if (archiveSettings.useCaseHack) {
-                        auto i = names.find(name);
-                        if (i != names.end()) {
-                            debug("case collision between '%1%' and '%2%'", i->first, name);
-                            name += caseHackSuffix;
-                            name += std::to_string(++i->second);
-                        } else
-                            names[name] = 0;
+                {
+                    const auto s = readString(source);
+                    co_yield MetadataString{s};
+                    if (s == ")") {
+                        break;
+                    } else if (s != "entry") {
+                        throw badArchive("expected entry tag");
                     }
-                } else if (s == "node") {
-                    if (name.empty()) throw badArchive("entry name missing");
-                    parse(sink, source, path + "/" + name);
-                } else
-                    throw badArchive("unknown field " + s);
+                    EXPECT("(", "open");
+                }
+
+                EXPECT("name", "name");
+                auto name = readString(source);
+                co_yield MetadataString{name};
+                if (name.empty() || name == "." || name == ".."
+                    || name.find('/') != std::string::npos
+                    || name.find((char) 0) != std::string::npos)
+                {
+                    throw Error("NAR contains invalid file name '%1%'", name);
+                }
+                if (name <= prevName) {
+                    throw Error("NAR directory is not sorted");
+                }
+                prevName = name;
+                if (archiveSettings.useCaseHack) {
+                    auto i = names.find(name);
+                    if (i != names.end()) {
+                        debug("case collision between '%1%' and '%2%'", i->first, name);
+                        name += caseHackSuffix;
+                        name += std::to_string(++i->second);
+                    } else {
+                        names[name] = 0;
+                    }
+                }
+
+                EXPECT("node", "node");
+                co_yield parseObject(source, path + "/" + name);
+                EXPECT(")", "close");
             }
+        }(source, path);
+        co_yield Directory{path, reader};
+        while (reader.next()) {
+            // ignore remaining entries
         }
-
-        else if (s == "target" && type == tpSymlink) {
-            std::string target = readString(source);
-            sink.createSymlink(path, target);
-        }
-
-        else
-            throw badArchive("unknown field " + s);
+        // directories are terminated already, don't try to read another ")"
+        co_return;
+    } else if (t == "symlink") {
+        EXPECT("target", "target");
+        std::string target = readString(source);
+        co_yield MetadataString{target};
+        co_yield Symlink{path, target};
+    } else {
+        throw badArchive("unknown file type " + t);
     }
+
+    EXPECT(")", "close");
+
+#undef EXPECT
 }
 
-
-void parseDump(ParseSink & sink, Source & source)
+Generator<Entry> parse(Source & source)
 {
     std::string version;
     try {
         version = readString(source, narVersionMagic1.size());
+        co_yield MetadataString{version};
     } catch (SerialisationError & e) {
         /* This generally means the integer at the start couldn't be
            decoded.  Ignore and throw the exception below. */
     }
     if (version != narVersionMagic1)
         throw badArchive("input doesn't look like a Nix archive");
-    parse(sink, source, "");
+    co_yield parseObject(source, "");
 }
 
+}
+
+
+static WireFormatGenerator restore(ParseSink & sink, Generator<nar::Entry> nar)
+{
+    while (auto entry = nar.next()) {
+        co_yield std::visit(
+            overloaded{
+                [](nar::MetadataString m) -> WireFormatGenerator {
+                    co_yield m.data;
+                },
+                [](nar::MetadataRaw r) -> WireFormatGenerator {
+                    co_yield r.raw;
+                },
+                [&](nar::File f) {
+                    return [](auto f, auto & sink) -> WireFormatGenerator {
+                        sink.createRegularFile(f.path);
+                        sink.preallocateContents(f.size);
+                        if (f.executable) {
+                            sink.isExecutable();
+                        }
+                        while (auto block = f.contents.next()) {
+                            sink.receiveContents(std::string_view{block->data(), block->size()});
+                            co_yield *block;
+                        }
+                        sink.closeRegularFile();
+                    }(std::move(f), sink);
+                },
+                [&](nar::Symlink sl) {
+                    return [](auto sl, auto & sink) -> WireFormatGenerator {
+                        sink.createSymlink(sl.path, sl.target);
+                        co_return;
+                    }(std::move(sl), sink);
+                },
+                [&](nar::Directory d) {
+                    return [](auto d, auto & sink) -> WireFormatGenerator {
+                        sink.createDirectory(d.path);
+                        return restore(sink, std::move(d.contents));
+                    }(std::move(d), sink);
+                },
+            },
+            std::move(*entry)
+        );
+    }
+}
+
+WireFormatGenerator parseAndCopyDump(ParseSink & sink, Source & source)
+{
+    return restore(sink, nar::parse(source));
+}
+
+void parseDump(ParseSink & sink, Source & source)
+{
+    auto parser = parseAndCopyDump(sink, source);
+    while (parser.next()) {
+        // ignore the actual item
+    }
+}
 
 struct RestoreSink : ParseSink
 {
@@ -377,16 +457,14 @@ void restorePath(const Path & path, Source & source)
 }
 
 
-void copyNAR(Source & source, Sink & sink)
+WireFormatGenerator copyNAR(Source & source)
 {
     // FIXME: if 'source' is the output of dumpPath() followed by EOF,
     // we should just forward all data directly without parsing.
 
-    ParseSink parseSink; /* null sink; just parse the NAR */
+    static ParseSink parseSink; /* null sink; just parse the NAR */
 
-    TeeSource wrapper { source, sink };
-
-    parseDump(parseSink, wrapper);
+    return parseAndCopyDump(parseSink, source);
 }
 
 

src/libutil/archive.hh

@@ -1,6 +1,7 @@
 #pragma once
 ///@file
 
+#include "generator.hh"
 #include "types.hh"
 #include "serialise.hh"
 #include "file-system.hh"
@@ -57,13 +58,13 @@ namespace nix {
  *   `+` denotes string concatenation.
  * ```
  */
-void dumpPath(const Path & path, Sink & sink,
+WireFormatGenerator dumpPath(Path path,
     PathFilter & filter = defaultPathFilter);
 
 /**
  * Same as dumpPath(), but returns the last modified date of the path.
  */
-time_t dumpPathAndGetMtime(const Path & path, Sink & sink,
+WireFormatGenerator dumpPathAndGetMtime(Path path, time_t & mtime,
     PathFilter & filter = defaultPathFilter);
 
 /**
@@ -71,7 +72,7 @@ time_t dumpPathAndGetMtime(const Path & path, Sink & sink,
  *
  * @param s Contents of the file.
  */
-void dumpString(std::string_view s, Sink & sink);
+WireFormatGenerator dumpString(std::string_view s);
 
 /**
  * \todo Fix this API, it sucks.
@@ -116,14 +117,64 @@ struct RetrieveRegularNARSink : ParseSink
     }
 };
 
+namespace nar {
+
+// FIXME: the Generator & are needed here because gcc 12 has a bug that copies
+// move-only values that are yielded through types without user-defined ctors,
+// such as File and Directory. gcc 13 fixes this, but even then it's easier to
+// use refs anyway because it lets us drain incompletely read entries of a nar
+// without resorting to Finally trickery. eventually this should be fixed with
+// a more reasonable format though that doesn't need quite this weird a parser
+
+struct MetadataString;
+struct MetadataRaw;
+struct File;
+struct Symlink;
+struct Directory;
+using Entry = std::variant<MetadataString, MetadataRaw, File, Symlink, Directory>;
+
+struct MetadataString
+{
+    std::string_view data;
+};
+
+struct MetadataRaw
+{
+    Bytes raw;
+};
+
+struct File
+{
+    const Path & path;
+    bool executable;
+    uint64_t size;
+    Generator<Bytes> & contents;
+};
+
+struct Symlink
+{
+    const Path & path, target;
+};
+
+struct Directory
+{
+    const Path & path;
+    Generator<Entry> & contents;
+};
+
+Generator<Entry> parse(Source & source);
+
+}
+
+WireFormatGenerator parseAndCopyDump(ParseSink & sink, Source & source);
 void parseDump(ParseSink & sink, Source & source);
 
 void restorePath(const Path & path, Source & source);
 
 /**
- * Read a NAR from 'source' and write it to 'sink'.
+ * Read a NAR from 'source' and return it as a generator.
  */
-void copyNAR(Source & source, Sink & sink);
+WireFormatGenerator copyNAR(Source & source);
 
 
 inline constexpr std::string_view narVersionMagic1 = "nix-archive-1";

src/libutil/compression.cc

@@ -198,31 +198,6 @@ std::string decompress(const std::string & method, std::string_view in)
     return filter->drain();
 }
 
-std::unique_ptr<FinishSink> makeDecompressionSink(const std::string & method, Sink & nextSink)
-{
-    if (method == "none" || method == "")
-        return std::make_unique<NoneSink>(nextSink);
-    else if (method == "br")
-        return sourceToSink([&](Source & source) {
-            BrotliDecompressionSource wrapped{source};
-            wrapped.drainInto(nextSink);
-            // special handling because sourceToSink is screwy: try
-            // to read the source one final time and fail when that
-            // succeeds (to reject trailing garbage in input data).
-            try {
-                char buf;
-                source(&buf, 1);
-                throw Error("garbage at end of brotli stream detected");
-            } catch (EndOfFile &) {
-            }
-        });
-    else
-        return sourceToSink([&](Source & source) {
-            auto decompressionSource = std::make_unique<ArchiveDecompressionSource>(source);
-            decompressionSource->drainInto(nextSink);
-        });
-}
-
 std::unique_ptr<Source> makeDecompressionSource(const std::string & method, Source & inner)
 {
     if (method == "none" || method == "") {

src/libutil/compression.hh

@@ -18,7 +18,6 @@ struct CompressionSink : BufferedSink, FinishSink
 
 std::string decompress(const std::string & method, std::string_view in);
 
-std::unique_ptr<FinishSink> makeDecompressionSink(const std::string & method, Sink & nextSink);
 std::unique_ptr<Source> makeDecompressionSource(const std::string & method, Source & inner);
 
 std::string compress(const std::string & method, std::string_view in, const bool parallel = false, int level = -1);

src/libutil/file-descriptor.cc

@@ -81,12 +81,12 @@ std::string drainFD(int fd, bool block, const size_t reserveSize)
     // the parser needs two extra bytes to append terminating characters, other users will
     // not care very much about the extra memory.
     StringSink sink(reserveSize + 2);
-    drainFD(fd, sink, block);
+    sink << drainFDSource(fd, block);
     return std::move(sink.s);
 }
 
 
-void drainFD(int fd, Sink & sink, bool block)
+Generator<Bytes> drainFDSource(int fd, bool block)
 {
     // silence GCC maybe-uninitialized warning in finally
     int saved = 0;
@@ -115,7 +115,7 @@ void drainFD(int fd, Sink & sink, bool block)
                 throw SysError("reading from file");
         }
         else if (rd == 0) break;
-        else sink({(char *) buf.data(), (size_t) rd});
+        else co_yield std::span{(char *) buf.data(), (size_t) rd};
     }
 }
 

src/libutil/file-descriptor.hh

@@ -2,6 +2,7 @@
 ///@file
 
 #include "error.hh"
+#include "generator.hh"
 
 namespace nix {
 
@@ -35,7 +36,7 @@ void writeFull(int fd, std::string_view s, bool allowInterrupts = true);
  */
 std::string drainFD(int fd, bool block = true, const size_t reserveSize=0);
 
-void drainFD(int fd, Sink & sink, bool block = true);
+Generator<Bytes> drainFDSource(int fd, bool block = true);
 
 class AutoCloseFD
 {

src/libutil/file-system.cc

@@ -289,17 +289,14 @@ std::string readFile(const Path & path)
 }
 
 
-box_ptr<Source> readFileSource(const Path & path)
+Generator<Bytes> readFileSource(const Path & path)
 {
     AutoCloseFD fd{open(path.c_str(), O_RDONLY | O_CLOEXEC)};
     if (!fd)
         throw SysError("opening file '%s'", path);
-
-    struct FileSource : FdSource {
-        AutoCloseFD fd;
-        explicit FileSource(AutoCloseFD fd) : FdSource(fd.get()), fd(std::move(fd)) {}
-    };
-    return make_box_ptr<FileSource>(std::move(fd));
+    return [](AutoCloseFD fd) -> Generator<Bytes> {
+        co_yield drainFDSource(fd.get());
+    }(std::move(fd));
 }
 
 
@@ -511,10 +508,14 @@ void AutoDelete::reset(const Path & p, bool recursive) {
 
 //////////////////////////////////////////////////////////////////////
 
+std::string defaultTempDir() {
+    return getEnvNonEmpty("TMPDIR").value_or("/tmp");
+}
+
 static Path tempName(Path tmpRoot, const Path & prefix, bool includePid,
     std::atomic<unsigned int> & counter)
 {
-    tmpRoot = canonPath(tmpRoot.empty() ? getEnv("TMPDIR").value_or("/tmp") : tmpRoot, true);
+    tmpRoot = canonPath(tmpRoot.empty() ? defaultTempDir() : tmpRoot, true);
     if (includePid)
         return fmt("%1%/%2%-%3%-%4%", tmpRoot, prefix, getpid(), counter++);
     else
@@ -554,7 +555,7 @@ Path createTempDir(const Path & tmpRoot, const Path & prefix,
 
 std::pair<AutoCloseFD, Path> createTempFile(const Path & prefix)
 {
-    Path tmpl(getEnv("TMPDIR").value_or("/tmp") + "/" + prefix + ".XXXXXX");
+    Path tmpl(defaultTempDir() + "/" + prefix + ".XXXXXX");
     // Strictly speaking, this is UB, but who cares...
     // FIXME: use O_TMPFILE.
     AutoCloseFD fd(mkstemp((char *) tmpl.c_str()));

src/libutil/file-system.hh

@@ -143,7 +143,7 @@ unsigned char getFileType(const Path & path);
  * Read the contents of a file into a string.
  */
 std::string readFile(const Path & path);
-box_ptr<Source> readFileSource(const Path & path);
+Generator<Bytes> readFileSource(const Path & path);
 
 /**
  * Write a string to a file.
@@ -258,6 +258,11 @@ Path createTempDir(const Path & tmpRoot = "", const Path & prefix = "nix",
  */
 std::pair<AutoCloseFD, Path> createTempFile(const Path & prefix = "nix");
 
+/**
+ * Return `TMPDIR`, or the default temporary directory if unset or empty.
+ */
+Path defaultTempDir();
+
 /**
  * Used in various places.
  */

src/libutil/generator.hh

@@ -0,0 +1,250 @@
+#pragma once
+///@file
+
+#include "types.hh"
+
+#include <coroutine>
+#include <exception>
+#include <optional>
+#include <utility>
+#include <variant>
+
+namespace nix {
+
+template<typename T, typename Transform>
+struct Generator;
+
+namespace _generator {
+
+template<typename T>
+struct promise_state;
+template<typename T>
+struct GeneratorBase;
+
+struct finished {};
+
+template<typename T>
+struct link
+{
+    std::coroutine_handle<> handle{};
+    promise_state<T> * state{};
+};
+
+struct failure
+{
+    std::exception_ptr e;
+};
+
+template<typename T>
+struct promise_state
+{
+    std::variant<T, link<T>, failure, finished> value{};
+    link<T> parent{};
+};
+
+template<typename T, typename Transform>
+struct promise : promise_state<T>
+{
+    using transform_t = std::conditional_t<std::is_void_v<Transform>, std::identity, Transform>;
+
+    transform_t convert;
+    std::optional<GeneratorBase<T>> inner;
+
+    Generator<T, Transform> get_return_object()
+    {
+        auto h = std::coroutine_handle<promise>::from_promise(*this);
+        return Generator<T, Transform>(GeneratorBase<T>(h, h.promise()));
+    }
+    std::suspend_always initial_suspend()
+    {
+        return {};
+    }
+    std::suspend_always final_suspend() noexcept
+    {
+        return {};
+    }
+    void unhandled_exception()
+    {
+        this->value = failure{std::current_exception()};
+    }
+
+    template<typename From>
+        requires requires(transform_t t, From && f) {
+            {
+                t(std::forward<From>(f))
+            } -> std::convertible_to<T>;
+        }
+    std::suspend_always yield_value(From && from)
+    {
+        this->value.template emplace<0>(convert(std::forward<From>(from)));
+        return {};
+    }
+
+    template<typename From>
+        requires requires(transform_t t, From && f) {
+            static_cast<Generator<T, void>>(t(std::forward<From>(f)));
+        }
+    std::suspend_always yield_value(From && from)
+    {
+        inner = static_cast<Generator<T, void>>(convert(std::forward<From>(from))).impl;
+        this->value = inner->active;
+        return {};
+    }
+
+    void return_void()
+    {
+        this->value = finished{};
+    }
+};
+
+template<typename T>
+struct GeneratorBase
+{
+    template<typename, typename>
+    friend struct Generator;
+    template<typename, typename>
+    friend struct promise;
+
+    // NOTE coroutine handles are LiteralType, own a memory resource (that may
+    // itself own unique resources), and are "typically TriviallyCopyable". we
+    // need to take special care to wrap this into a less footgunny interface.
+    GeneratorBase(GeneratorBase && other)
+    {
+        swap(other);
+    }
+
+    GeneratorBase & operator=(GeneratorBase && other)
+    {
+        GeneratorBase(std::move(other)).swap(*this);
+        return *this;
+    }
+
+    ~GeneratorBase()
+    {
+        if (h) {
+            h.destroy();
+        }
+    }
+
+    std::optional<T> next()
+    {
+        while (active.handle) {
+            active.handle.resume();
+            auto & p = *active.state;
+            auto result = std::visit(
+                overloaded{
+                    [&](finished) -> std::optional<T> {
+                        active = p.parent;
+                        return {};
+                    },
+                    [&](link<T> & inner) -> std::optional<T> {
+                        auto base = inner.state;
+                        while (base->parent.handle) {
+                            base = base->parent.state;
+                        }
+                        base->parent = active;
+                        active = inner;
+                        return {};
+                    },
+                    [&](T & value) -> std::optional<T> { return std::move(value); },
+                    [&](failure & f) -> std::optional<T> {
+                        active = {};
+                        std::rethrow_exception(f.e);
+                    },
+                },
+                p.value
+            );
+            if (result) {
+                return std::move(result);
+            }
+        }
+
+        return std::nullopt;
+    }
+
+protected:
+    std::coroutine_handle<> h{};
+    link<T> active{};
+
+    GeneratorBase(std::coroutine_handle<> h, promise_state<T> & state)
+        : h(h)
+        , active(h, &state)
+    {
+    }
+
+    void swap(GeneratorBase & other)
+    {
+        std::swap(h, other.h);
+        std::swap(active, other.active);
+    }
+};
+
+}
+
+/// Coroutine-based iterator modeled loosely on Rust [`std::iter::Iterator`][iter]
+/// interface. Like Rust's `Iterator` and unlike common C++ iterators, a Generator
+/// returns `std::optional<T>` values from its next() function, but unlike both it
+/// can also transform items produced within using a Transform function object the
+/// Generator holds before returning them via next(). To allow generator nesting a
+/// Transform may also return another Generator instance for any yielded value, in
+/// this case the new Generator will temporarily take priority over the previously
+/// running one and have its values returned until it is exhausted, then return to
+/// the previous Generator. This mechanism may nest Generator to arbitrary depths.
+///
+/// \tparam T item type
+/// \tparam Transform transform function object type, or `void` for no transform
+///
+/// [iter]: https://doc.rust-lang.org/stable/std/iter/trait.Iterator.html
+template<typename T, typename Transform = void>
+struct Generator
+{
+    template<typename, typename>
+    friend struct _generator::promise;
+    template<typename, typename>
+    friend struct Generator;
+
+    using promise_type = _generator::promise<T, Transform>;
+
+    Generator(const Generator &) = delete;
+    Generator & operator=(const Generator &) = delete;
+    Generator(Generator &&) = default;
+    Generator & operator=(Generator &&) = default;
+
+    /// If the coroutine held by the Generator has not finished, runs it until it
+    /// yields a value, throws and exception, or returns. If the coroutine yields
+    /// a value this value is passed to a persistent instance of `Transform` that
+    /// is held by the Generator, and the result of this call is returned. If the
+    /// coroutine throws an exception, or the Transform throws an exception while
+    /// processing an item, that exception is rethrown and the Generator will not
+    /// return any more non-`std::nullopt` values from next(). Once the contained
+    /// coroutine has completed or an exception has been thrown the Generator can
+    /// no longer return any valid values, only `std::nullopt`. Exceptions thrown
+    /// are thrown only once, further invocations of next() return `std::nullopt`.
+    ///
+    /// \returns `std::nullopt` if the coroutine has completed, or a value
+    std::optional<T> next()
+    {
+        return impl.next();
+    }
+
+    /// Type-erases the `Transform`.
+    ///
+    /// \return a new Generator with the `Transform` type-erased
+    Generator<T, void> decay() &&
+    {
+        return Generator<T, void>(std::move(impl));
+    }
+
+    /// \copydoc decay()
+    operator Generator<T, void>() &&
+    {
+        return std::move(*this).decay();
+    }
+
+private:
+    _generator::GeneratorBase<T> impl;
+
+    explicit Generator(_generator::GeneratorBase<T> b) : impl(std::move(b)) {}
+};
+
+}

src/libutil/hash.cc

@@ -324,7 +324,7 @@ Hash hashString(HashType ht, std::string_view s)
 Hash hashFile(HashType ht, const Path & path)
 {
     HashSink sink(ht);
-    readFileSource(path)->drainInto(sink);
+    sink << readFileSource(path);
     return sink.finish().first;
 }
 
@@ -370,7 +370,7 @@ HashResult hashPath(
     HashType ht, const Path & path, PathFilter & filter)
 {
     HashSink sink(ht);
-    dumpPath(path, sink, filter);
+    sink << dumpPath(path, filter);
     return sink.finish();
 }