gosh

Change-Id: I92dba73610fbdc9fc67ba6590e828a7bed869d28
Signed-off-by: Raito Bezarius <raito@lix.systems>

.editorconfig

@@ -33,3 +33,7 @@ max_line_length = 0
 [meson.build]
 indent_style = space
 indent_size = 2
+
+[*.json]
+indent_style = space
+indent_size = 4

.gitignore

@@ -39,3 +39,5 @@ buildtime.bin
 
 # Python compiled files from the code generators and test suite
 *.pyc
+
+**/.idea

doc/manual/change-authors.yml

@@ -66,6 +66,9 @@ delan:
   forgejo: delan
   github: delan
 
+delroth:
+  github: delroth
+
 detroyejr:
   display_name: Jonathan De Troye
   github: detroyejr
@@ -98,6 +101,9 @@ ian-h-chamberlain:
   forgejo: ian-h-chamberlain
   github: ian-h-chamberlain
 
+infinisil:
+  github: infinisil
+
 isabelroses:
   forgejo: isabelroses
   github: isabelroses
@@ -152,9 +158,17 @@ midnightveil:
   forgejo: midnightveil
   github: midnightveil
 
+nan-git:
+  display_name: NaN-git
+  github: NaN-git
+
 ncfavier:
   github: ncfavier
 
+not-my-profile:
+  display_name: Martin Fischer
+  github: not-my-profile
+
 p-e-meunier:
   display_name: Pierre-Etienne Meunier
   github: P-E-Meunier
@@ -226,6 +240,11 @@ vigress8:
   forgejo: vigress8
   github: vigress8
 
+vlinkz:
+  display_name: Victor Fuentes
+  forgejo: vlinkz
+  github: vlinkz
+
 winter:
   forgejo: winter
   github: winterqt

doc/manual/rl-next/arg-identifier.md

@@ -0,0 +1,16 @@
+---
+synopsis: First argument to `--arg`/`--argstr` must be a valid Nix identifier
+issues: [fj#496]
+category: "Breaking Changes"
+credits: [ma27]
+---
+
+The first argument to `--arg`/`--argstr` must be a valid Nix identifier, i.e.
+`nix-build --arg config.allowUnfree true` is now rejected.
+
+This is because that invocation is a false friend since it doesn't set
+`{ config = { allowUnfree = true; }; }`, but `{ "config.allowUnfree" = true; }`.
+
+The idea is to change the behavior to the latter in the long-term. For that,
+non-identifiers started giving a warning since 2.92 and are now rejected to give people
+who depend on that a chance to notice and potentially weigh in on the discussion.

doc/manual/rl-next/async-query.md

@@ -0,0 +1,20 @@
+---
+synopsis: "Improved susbtituter query speed"
+issues: []
+cls: []
+category: Improvements
+credits: [horrors]
+---
+
+The code used to query substituters for derivations has been rewritten slightly
+to take advantage of our asynchronous runtime. Such queries run for every build
+that could download from substituters and processes every derivation that isn't
+yet present on the local system. Previously Lix would use `http-connections` to
+limit query concurrency, even for modern caches that support HTTP/2 and have no
+limit on how many queries can be run concurrently on one single connection. Lix
+no longer does this, resulting in approximately 60% reduction in query time for
+medium-sized closures (e.g. NixOS system closures) during testing, although the
+exact number depends greatly on local network latency and generally improves as
+latency increases. Unlike previously setting `http-connections` to `1` or other
+low values no longer brings a massive penalty in query performance if the cache
+in use by the querying system supports HTTP/2 (as e.g. `cache.nixos.org` does).

doc/manual/rl-next/build-dir-mandatory.md

@@ -0,0 +1,12 @@
+---
+synopsis: "`build-dir` no longer defaults to `temp-dir`"
+cls: [3453]
+category: "Fixes"
+credits: [horrors]
+---
+
+The directory in which temporary build directories are created no longer defaults
+to the value of the `temp-dir` setting to avoid builders making their directories
+world-accessible. This behavior has been used to escape the build sandbox and can
+cause build impurities even when not used maliciously. We now default to `builds`
+in `NIX_STATE_DIR` (which is `/nix/var/nix/builds` in the default configuration).

doc/manual/rl-next/cgroups-delegation.md

@@ -0,0 +1,69 @@
+---
+synopsis: New cgroup delegation model
+issues: [fj#537, fj#77]
+cls: [3230]
+category: "Breaking Changes"
+credits: [raito, horrors, lheckemann]
+---
+
+Builds using cgroups (i.e. `use-cgroups = true` and the experimental feature
+`cgroups`) now always delegate a cgroup tree to the sandbox.
+
+Compared to the original C++ Nix project, our delegation includes the
+`subtree_control` file as well, which means that the sandbox can disable
+certain controllers in its own cgroup tree.
+
+This is a breaking change because this requires the Nix daemon to run with an
+already delegated cgroup tree by the service manager.
+
+## How to setup the cgroup tree with systemd?
+
+systemd offers knobs to perform the required setup using:
+
+```
+[Service]
+Delegate=yes
+DelegateSubtree=supervisor
+```
+
+These directives are now included in our systemd packaging.
+
+## What about using Nix as root without connecting to the daemon?
+
+Builds run as `root` without connecting to the daemon relying on the cgroup
+feature are now broken, i.e.
+
+```console
+# nix-build --use-cgroups --sandbox ... # will not work
+```
+
+Consider doing instead:
+
+```console
+# systemd-run --same-dir --wait -p Delegate=yes -p DelegateSubgroup=supervisor nix-build --use-cgroups ...
+```
+
+If you need to disable cgroups temporarily, remember that you can do
+`NIX_CONF='include /etc/nix/nix.conf\nuse-cgroups = false' nix-build ...` or
+`nix-build --no-use-cgroups ...`.
+
+## What about other service managers than systemd?
+
+systemd has a [documentation](https://systemd.io/CGROUP_DELEGATION/) on how to
+handle cgroup delegation from service management perspective.
+
+If your service manager adheres to systemd semantics, e.g. writing an extended
+attribute `user.delegate=1` on the delegated cgroup tree directory and moving
+the `nix-daemon` process inside a cgroup tree to respect the inner process
+rule, then, the feature will work as well.
+
+## Why is the cgroup feature still experimental?
+
+While the cgroup feature unlocks many use cases, its behavior and integration (e.g. user experience), especially at scale on build farms or in multi-tenant environments, are not yet fully matured. There’s also potential for deeper systemd integration (e.g. using slices and scopes) that has not been fully explored.
+
+To avoid locking in an unstable interface, we’re keeping the experimental flag until we have validated the feature across a broader range of scenarios, including but not limited to:
+
+* Nix as root
+* Hydra-style build farms
+* Forgejo CI runners
+* Shared remote builders

doc/manual/rl-next/deprecated-derivation-types.md

@@ -0,0 +1,16 @@
+---
+synopsis: Deprecation of CA derivations, dynamic derivations, and impure derivations
+issues: [fj#815]
+cls: []
+significance: significant
+category: Miscellany
+credits: []
+---
+
+Content-addressed derivations are now deprecated and slated for removal in Lix 2.94.
+We're doing this because the CA derivation system has been a known cause of problems
+and inconsistencies, is unmaintained, habitually makes improving the store code very
+difficult (or blocks such improvements outright), and is beset by a number of design
+flaws that in our opinion cannot be fixed without a full reimplementation from zero.
+Dynamic derivations and impure derivations are built on the CA derivation framework,
+and owing to this they too are deprecated and slated for removal in another release.

doc/manual/rl-next/fix-gc-links.md

@@ -0,0 +1,13 @@
+---
+synopsis: "nix-store --delete: always remove obsolete hardlinks"
+issues: []
+cls: [3188]
+category: Fixes
+credits: [lheckemann]
+---
+
+Deleting specific paths using `nix-store --delete` or `nix store
+delete` previously did not delete hard links created by `nix-store
+--optimise` even if they became obsolete, unless _all_ of the given
+paths were deleted successfully. Now, hard links are always cleaned
+up, even if some of the given paths could not be deleted.

doc/manual/rl-next/fix-gc-stats.md

@@ -0,0 +1,24 @@
+---
+synopsis: "Report GC statistics correctly"
+issues: []
+cls: [3188]
+category: Fixes
+credits: [lheckemann]
+---
+
+Deleting specific paths using `nix-store --delete` or `nix store delete` previously did
+not report statistics correctly when some of the paths could not be deleted, even if
+others were deleted:
+
+```
+$ nix store delete /nix/store/9bwryidal9q3g91cjm6xschfn4ikd82q-hello-2.12.1 --delete-closure -v
+finding garbage collector roots...
+deleting '/nix/store/9bwryidal9q3g91cjm6xschfn4ikd82q-hello-2.12.1'
+0 store paths deleted, 0.00 MiB freed
+error: Cannot delete some of the given paths because they are still alive. Paths not deleted:
+         k9bxzr1l92r5y6mihrkbpbr3fmc8qszx-libidn2-2.3.8
+         mbx9ii53lzjlrsnlrfmzpwm33ynljwdn-libunistring-1.3
+         rf8hcy6bldxdqc0g6q1dcka1vh47x69s-xgcc-14.2.1.20250322-libgcc
+         vbrdc5wgzn0w1zdp10xd2favkjn5fk7y-glibc-2.40-66
+       To find out why, use nix-store --query --roots and nix-store --query --referrers.
+```

doc/manual/rl-next/ignore-try-default.md

@@ -0,0 +1,24 @@
+---
+synopsis: Repl debugger uses `--ignore-try` by default
+issues: [lix#666]
+cls: [3488]
+category: Breaking Changes
+credits: [jade]
+---
+Previously, using the debugger meant that exceptions thrown in `builtins.tryEval` would trigger the debugger.
+
+However, this caught nixpkgs initialization code, which is unhelpful in the majority of cases, so we changed the default.
+
+To get the old behaviour, use `--no-ignore-try`.
+
+```
+$ nix repl --debugger --expr 'with import <nixpkgs> {}; pkgs.hello'
+Lix 2.94.0-dev-pre20250625-9a59106
+Type :? for help.
+error: file 'nixpkgs-overlays' was not found in the Nix search path (add it using $NIX_PATH or -I)
+
+This exception occurred in a 'tryEval' call. Use --ignore-try to skip these.
+
+Added 13 variables.
+nix-repl>
+```

doc/manual/rl-next/infallible-build-dirs.md

@@ -0,0 +1,25 @@
+---
+synopsis: "Fallback to safe temp dir when build-dir is unwritable"
+issues: [fj#876]
+cls: [3501]
+category: "Fixes"
+credits: ["raito", "horrors"]
+---
+
+Non-daemon builds started failing with a permission error after introducing the `build-dir` option:
+
+```
+$ nix build --store ~/scratch nixpkgs#hello --rebuild
+error: creating directory '/nix/var/nix/builds/nix-build-hello-2.12.2.drv-0': Permission denied
+```
+
+This happens because:
+
+1. These builds are not run via the daemon, which owns `/nix/var/nix/builds`.
+2. The user lacks permissions for that path.
+
+We considered making `build-dir` a store-level option and defaulting it to `<chroot-root>/nix/var/nix/builds` for chroot stores, but opted instead for a fallback: if the default fails, Nix now creates a safe build directory under `/tmp`.
+
+To avoid CVE-2025-52991, the fallback uses an extra path component between `/tmp` and the build dir.
+
+**Note**: this fallback clutters `/tmp` with build directories that are not cleaned up. To prevent this, explicitly set `build-dir` to a path managed by Lix, even for local workloads.

doc/manual/rl-next/integer-coercion.md

@@ -0,0 +1,47 @@
+---
+synopsis: Experimental integer coercion in interpolated strings
+issues: []
+cls: [3198]
+category: "Features"
+credits: [raito, delroth, horrors, winter]
+---
+
+Ever tried interpolating a port number in Lix and ended up with something like this?
+
+```nix
+"http://${config.network.host}:${builtins.toString config.network.port}/"
+```
+
+You're not alone. Thousands of Lix users suffer every day from excessive `builtins.toString` syndrome. It’s 2025, and we still have to cast integers to use them in strings.
+
+To address this, Lix introduces the **`coerce-integers`** experimental feature. When enabled, interpolated integers within `"${...}"` are automatically coerced to strings. This allows writing:
+
+```nix
+"http://${config.network.host}:${config.network.port}/"
+```
+
+without additional conversion.
+
+To enable the feature, you need to add `coerce-integers` to your set of experimental features.
+
+### Stabilization criteria
+
+The `coerce-integers` feature is experimental and limited strictly to string interpolation (`"${...}"`). Before stabilization, the following must hold:
+
+1. **Interpolation-only**
+   Coercion must not occur outside interpolation. Expressions like `"" + 42` must continue to fail.
+
+2. **Expectation that no explicit cast are being observed**
+   Cases observing explicit coercion (e.g., via `tryEval` gadget or similar) are expected not to be load-bearing in actual production code.
+
+### Timeline for stabilization
+
+If the feature proves safe and is widely adopted across typical usage (e.g., actual configurations in the wild turning on the flag, non-trivial out-of-tree projects using it), the experimental flag will be removed **after six months of active use or two Lix releases**, whichever is longer.
+
+This avoids locking the feature in experimental status indefinitely, as happened with Flakes, while allowing time for validation and ecosystem integration.
+
+### What about coercing floats or more?
+
+Coercion beyond integers -- such as for floats or other types -- is **not planned**, even under an experimental flag. Questions like "what is the canonical string representation of a float?" involve subtle and context-dependent trade-offs. Without a robust and principled mechanism to define and audit such behavior, introducing broader coercion risks setting unintended and hard-to-reverse precedents. The scope of `coerce-integers` is intentionally narrow and will remain so.
+
+In terms of outlook, a proposal like https://git.lix.systems/lix-project/lix/issues/835 could pave the way for a better solution.

doc/manual/rl-next/log-output-terminal-codes.md

@@ -0,0 +1,11 @@
+---
+synopsis: Fix handling of OSC codes in terminal output
+issues: [fj#160]
+cls: [3143]
+category: Fixes
+credits: [lilyball]
+---
+
+OSC codes in terminal output are now handled correctly, where OSC 8 (hyperlink) is preserved any
+time color codes are allowed and all other OSC codes are stripped out. This applies not only to
+output from build commands but also to rendered documentation in the REPL.

doc/manual/rl-next/mismatch-check-path.md

@@ -0,0 +1,14 @@
+---
+synopsis: Better debuggability on fixed-output hash mismatches
+issues: []
+cls: []
+category: Improvements
+credits: [lheckemann]
+---
+
+Fixed-output derivation hash mismatch error messages will now include the path that was
+produced unexpectedly, and this path will be registered as valid even if `--check`
+(`nix-store`, `nix-build`) or `--rebuild` (`nix build`) was passed. This makes comparing
+the expected path with the obtained path easier, and is useful for debugging when
+upstreams modify previously-published releases or when changes in fixed-output
+derivations' dependencies affect their output unexpectedly.

doc/manual/rl-next/nix-instantiate-raw.md

@@ -0,0 +1,12 @@
+---
+synopsis: "Add --raw flag to `nix-instantiate --eval` for unescaped output"
+issues: []
+prs: [gh#12119]
+cls: [2886]
+category: Improvements
+credits: [not-my-profile, infinisil, raito]
+---
+
+The `nix-instantiate --eval` command now supports a `--raw` flag. When used,
+the result must be coercible to a string (as with `${...}`) and is printed
+verbatim, without quotes or escaping.

doc/manual/rl-next/nix-store-ls-read-nar-listings.md

@@ -0,0 +1,12 @@
+---
+synopsis: Allow `nix store ls` to read nar listings from binary cache stores.
+issues: []
+cls: [3225]
+category: Improvements
+credits: [vlinkz]
+---
+
+The `nix store ls` command now supports reading `.ls` nar listings from binary cache stores.
+If a listing is detected for the store path being queried, the nar is no longer downloaded.
+These nar listings are available in binary cache stores where the `write-nar-listing` option is
+enabled, such as cache.nixos.org.

doc/manual/rl-next/pasta.md

@@ -0,0 +1,20 @@
+---
+synopsis: "Fixed output derivations can be run using `pasta` network isolation"
+cls: [3452]
+issues: [fj#285]
+category: "Breaking Changes"
+credits: [horrors, puck]
+---
+
+Fixed output derivations traditionally run in the host network namespace.
+On Linux this allows such derivations to communicate with other sandboxes
+or the host using the abstract Unix domains socket namespace; this hasn't
+been unproblematic in the past and has been used in two distinct exploits
+to break out of the sandbox. For this reason fixed output derivations can
+now run in a network namespace (provided by [`pasta`]), restricted to TCP
+and UDP communication with the rest of the world. When enabled this could
+be a breaking change and we classify it as such, even though we don't yet
+enable or require such isolation by default. We may enforce this in later
+releases of Lix once we have sufficient confidence that breakage is rare.
+
+[`pasta`]: https://passt.top/

doc/manual/rl-next/reliance-on-bash-for-ssh.md

@@ -0,0 +1,21 @@
+---
+synopsis: Remove reliance on Bash for remote stores via SSH
+issues: [fj#830, fj#805, fj#304]
+cls: [3159]
+category: "Fixes"
+credits: [raito]
+---
+
+The pre-flight `echo started` handshake -- added years ago to catch race conditions -- has been removed.
+
+After removal of connection sharing in Lix 2.93, it required a Bash-compatible shell and a standard `echo`, so it failed on:
+
+  * builders protected by `ForceCommand` wrappers (e.g. `nix-remote-build`),
+  * BusyBox / initrd images with no Bash,
+  * hosts using non-POSIX shells such as Nushell.
+
+The race the probe once addressed was tied to SSH connection-sharing -- since connection-sharing code has already been removed, the probe is now pointless.
+
+Real connection or protocol errors are now left to SSH/Nix to report directly.
+
+This is technically a breaking change if you had scripts that relied on the literal "started" which needs to be updated to rely on other signals, e.g., exit codes.

doc/manual/rl-next/remove-old-wires.md

@@ -0,0 +1,20 @@
+---
+synopsis: Remove support for daemon protocols before 2.18
+issues: []
+cls: [3249]
+significance: significant
+category: "Breaking Changes"
+credits: [horrors]
+---
+
+Support for daemon wire protocols belonging to Nix 2.18 or older have been
+removed. This impacts clients connecting to the local daemon socket or any
+remote builder configured using the `ssh-ng` protocol. Builders configured
+with the `ssh` protocol are still accessible from clients such as Nix 2.3.
+Additionally Lix will not be able to connect to an old daemon locally, and
+remote build connections to old daemons is likewise limited to `ssh` urls.
+
+We have decided to take this step because the old protocols are very badly
+tested (if at all), maintenance overhead is high, and a number of problems
+with their design makes it infeasible to remain backwards compatible while
+we move Lix to a more modern RPC mechanism with better versioning support.

doc/manual/rl-next/remove-xp-derivations.md

@@ -0,0 +1,17 @@
+---
+synopsis: Remove impure derivations and dynamic derivations
+issues: [fj#815]
+cls: [3210]
+significance: significant
+category: "Breaking Changes"
+credits: [horrors]
+---
+
+The `impure-derivations` and `dynamic-derivations` experimental feature have
+been removed.
+
+New impure or dynamic derivations cannot be created from this point forward, and
+any such pre-existing store derivations canot be read or built any more.
+Derivation outputs created by building such a derivation are still valid
+until garbage collected; existing store derivations can only be garbage
+collected.

doc/manual/rl-next/repl-overlays-pure.md

@@ -0,0 +1,20 @@
+---
+synopsis: "repl-overlays now work in the debugger for flakes"
+issues: [fj#777]
+cls: [3398]
+category: Fixes
+credits: [jade]
+---
+Due to a bug, it was previously not possible to use the debugger on flakes with repl-overlays, or with pure evaluation in general:
+
+```
+$ nix repl --pure-eval
+Lix 2.94.0-dev-pre20250617-87d99da
+Type :? for help.
+Loading 'repl-overlays'...
+error: access to absolute path '/Users/jade/.config/nix/repl.nix' is forbidden in pure eval mode (use '--impure' to override)
+```
+
+This is now fixed.
+The contents of the repl-overlays file itself (i.e. most typically the top level lambda in it) will be evaluated in impure mode.
+It may be necessary to use `builtins.seq` to force the impure operations to happen first if one wants to do impure operations inside a repl-overlays file in pure evaluation mode.

doc/manual/rl-next/symbol-value-reuse.md

@@ -0,0 +1,32 @@
+---
+synopsis: Symbols reuses once-allocated Value to reduce garbage collected allocations
+issues: []
+cls: [3308, 3300, 3314, 3310, 3312, 3313]
+category: Improvements
+credits: [raito, horrors, thubrecht, nan-git]
+---
+
+In the Lix evaluator, **symbols** represent immutable strings, like those used
+for attribute names.
+
+In evaluator design, such strings are typically [**interned**](https://en.wikipedia.org/wiki/String_interning), stored uniquely
+to save memory, and Lix inherits this approach from the original C++ codebase.
+
+However, some builtins, like `builtins.attrNames`, must return a `Value` type
+that can represent any Nix value (strings, integers, lists, etc.).
+
+Before this change, these builtins would create lists of `Value` objects by
+allocating them through the garbage collector, copying the symbol’s string
+content each time.
+
+This allocation is unnecessary if the interned symbols themselves also hold a
+`Value` representation allocated outside the garbage collector, since these
+live for the full duration of evaluation.
+
+As a result, this reduces the number of allocations, leading to:
+
+* A significant drop in maximum [resident set memory](https://en.wikipedia.org/wiki/Resident_set_size) (RSS), with some large-scale
+  tests showing up to 11% (about 500 MiB) savings in large colmena deployments.
+* A slight decrease in CPU usage during Nix evaluations.
+
+This change is inspired by https://github.com/NixOS/nix/pull/13258 but the approach is different.

doc/manual/rl-next/uid-range-requires-cgroups.md

@@ -0,0 +1,15 @@
+---
+synopsis: uid-range depends on cgroups
+issues: []
+cls: [3230]
+category: "Breaking Changes"
+credits: [raito, horrors]
+---
+
+`uid-range` builds now depends on `cgroups`, an experimental feature.
+
+`uid-range` builds already depended upon `auto-allocate-uids`, another experimental feature.
+
+The rationale for doing so is that `uid-range` provides a sandbox with many
+UIDs, this is useful for re-mapping them into a nested namespace, e.g. a
+container.

doc/manual/src/SUMMARY.md

@@ -20,7 +20,6 @@
   - [Basic Package Management](package-management/basic-package-mgmt.md)
   - [Profiles](package-management/profiles.md)
   - [Garbage Collection](package-management/garbage-collection.md)
-    - [Garbage Collector Roots](package-management/garbage-collector-roots.md)
   - [Sharing Packages Between Machines](package-management/sharing-packages.md)
     - [Serving a Nix store via HTTP](package-management/binary-cache-substituter.md)
     - [Copying Closures via SSH](package-management/copy-closure.md)

doc/manual/src/command-ref/nix-env/install.md

@@ -148,8 +148,8 @@ To copy the store path with symbolic name `gcc` from another profile:
 $ nix-env --install --from-profile /nix/var/nix/profiles/foo gcc
 ```
 
-To install a specific [store derivation] (typically created by
+To install a specific [store derivation](@docroot@/glossary.md#gloss-store-derivation)
-`nix-instantiate`):
+(typically created by `nix-instantiate`):
 
 ```console
 $ nix-env --install /nix/store/fibjb1bfbpm5mrsxc4mh2d8n37sxh91i-gcc-3.4.3.drv

doc/manual/src/command-ref/nix-instantiate.md

@@ -5,7 +5,7 @@
 # Synopsis
 
 `nix-instantiate`
-  [`--parse` | `--eval` [`--strict`] [`--json`] [`--xml`] ]
+  [`--parse` | `--eval` [`--strict`] [`--raw`] [`--json`] [`--xml`] ]
   [`--read-write-mode`]
   [`--arg` *name* *value*]
   [{`--attr`| `-A`} *attrPath*]
@@ -107,6 +107,11 @@ See that section for complete details (`nix-build --help`), but in summary, a pa
     > This option can cause non-termination, because lazy data
     > structures can be infinitely large.
 
+  - `--raw`
+    When used with `--eval`, the result must be coercible to a string, i.e.,
+    something that can be converted using `${...}`. The output is
+    printed exactly as-is, with no quotes, escaping, or trailing newline.
+
   - `--json`\
     When used with `--eval`, print the resulting value as an JSON
     representation of the abstract syntax tree rather than as a Nix expression.

doc/manual/src/command-ref/nix-store/realise.md

@@ -15,7 +15,6 @@ Each of *paths* is processed as follows:
   1. If it is not [valid], substitute the store derivation file itself.
   2. Realise its [output paths]:
     - Try to fetch from [substituters] the [store objects] associated with the output paths in the store derivation's [closure].
-      - With [content-addressed derivations] (experimental): Determine the output paths to realise by querying content-addressed realisation entries in the [Nix database].
     - For any store paths that cannot be substituted, produce the required store objects. This involves first realising all outputs of the derivation's dependencies and then running the derivation's [`builder`](@docroot@/language/derivations.md#attr-builder) executable. <!-- TODO: Link to build process page #8888 -->
 - Otherwise, and if the path is not already valid: Try to fetch the associated [store objects] in the path's [closure] from [substituters].
 
@@ -28,7 +27,6 @@ If no substitutes are available and no store derivation is given, realisation fa
 [store objects]: @docroot@/glossary.md#gloss-store-object
 [closure]: @docroot@/glossary.md#gloss-closure
 [substituters]: @docroot@/command-ref/conf-file.md#conf-substituters
-[content-addressed derivations]: @docroot@/contributing/experimental-features.md#xp-feature-ca-derivations
 [Nix database]: @docroot@/glossary.md#gloss-nix-database
 
 The resulting paths are printed on standard output.

doc/manual/src/contributing/hacking.md

@@ -39,7 +39,7 @@ $ nix-shell -A native-clangStdenvPackages
 
 ### Building from the development shell
 
-Run a clean build and test with `just clean build install test`.
+Run a clean build and test with `just clean setup build install test`.
 
 You can also run the unit tests and integration tests separately:
 

doc/manual/src/contributing/testing.md

@@ -449,9 +449,6 @@ I grepped `lix/` for `get[eE]nv\("` to find the mentions in Lix code.
 - `NIX_CLIENT_PACKAGE` - Runs the test suite against an alternate Nix client with the current daemon.
 
   **Expected value**: something like `/nix/store/...-nix-2.18.2`
-- `NIX_TESTS_CA_BY_DEFAULT` - Pass `__contentAddressed`, `outputHashMode` and `outputHashAlgo` to builds of some input-addressed derivations in the test suite.
-
-  **Expected value**: 1
 - `TEST_DATA` - Not an environment variable! This is used in repl characterization tests to refer to `tests/functional/repl_characterization/data`.
   More specifically, that path is replaced with the string `$TEST_DATA` in output for reproducibility.
 - `TEST_HOME` (output) - Set to the temporary directory that is set as `$HOME` inside the tests, underneath `$TEST_ROOT`.

doc/manual/src/glossary.md

@@ -41,12 +41,6 @@
 
   [realise]: #gloss-realise
 
-- [content-addressed derivation]{#gloss-content-addressed-derivation}
-
-  A derivation which has the
-  [`__contentAddressed`](./language/advanced-attributes.md#adv-attr-__contentAddressed)
-  attribute set to `true`.
-
 - [fixed-output derivation]{#gloss-fixed-output-derivation}
 
   A derivation which includes the
@@ -114,14 +108,13 @@
 - [input-addressed store object]{#gloss-input-addressed-store-object}
 
   A store object produced by building a
-  non-[content-addressed](#gloss-content-addressed-derivation),
   non-[fixed-output](#gloss-fixed-output-derivation)
   derivation.
 
 - [output-addressed store object]{#gloss-output-addressed-store-object}
 
   A [store object] whose [store path] is determined by its contents.
-  This includes derivations, the outputs of [content-addressed derivations](#gloss-content-addressed-derivation), and the outputs of [fixed-output derivations](#gloss-fixed-output-derivation).
+  This includes derivations and the outputs of [fixed-output derivations](#gloss-fixed-output-derivation).
 
 - [substitute]{#gloss-substitute}
 

doc/manual/src/language/advanced-attributes.md

@@ -209,15 +209,8 @@ Derivations can declare some infrequently used optional attributes.
 
   - [`__contentAddressed`]{#adv-attr-__contentAddressed}
     > **Warning**
-    > This attribute is part of an [experimental feature](@docroot@/contributing/experimental-features.md).
+    > This attribute is part of a removed [experimental feature](@docroot@/contributing/experimental-features.md).
-    >
+    > Setting this flag *will* cause eval errors.
-    > To use this attribute, you must enable the
-    > [`ca-derivations`](@docroot@/contributing/experimental-features.md#xp-feature-ca-derivations) experimental feature.
-    > For example, in [nix.conf](../command-ref/conf-file.md) you could add:
-    >
-    > ```
-    > extra-experimental-features = ca-derivations
-    > ```
 
     If this attribute is set to `true`, then the derivation
     outputs will be stored in a content-addressed location rather than the

doc/manual/src/package-management/garbage-collection.md

@@ -71,3 +71,62 @@ $ nix-collect-garbage -d
 ```
 
 is a quick and easy way to clean up your system.
+
+## Garbage Collector Roots
+
+### Explicit roots
+
+All store paths to which there are symlinks in the directory
+`prefix/nix/var/nix/gcroots` will be used as roots by the garbage
+collector. For instance, the following command makes the path
+`/nix/store/d718ef...-foo` a root of the collector:
+
+```console
+$ ln -s /nix/store/d718ef...-foo /nix/var/nix/gcroots/bar
+```
+
+That is, after this command, the garbage collector will not remove
+`/nix/store/d718ef...-foo` or any of its dependencies.
+
+Subdirectories of `prefix/nix/var/nix/gcroots` are also searched for
+symlinks.
+
+Symlinks may also point to paths outside the nix store. If the
+destination of the symlink is itself a symlink to a store path, it
+is also considered a root. This style of GC root is called an
+"indirect root", and is created by tools like `nix-build` to avoid
+garbage-collecting paths that are being used on-the-fly rather than
+installed in profiles.
+
+
+### In-use roots
+
+Lix will also perform a best-effort detection of paths that are in use
+by running processes when scanning for garbage collection roots, to
+avoid removing paths that are still needed by running processes.
+
+Exact details vary between platforms, but the following will generally
+be taken into account:
+
+- Executables in the store that are currently running;
+- Other files in the store that are mapped into a process's address space (e.g. shared libraries);
+- Files in the store to which processes have open handles;
+- Store paths found in processes' environment variables.
+
+Note that this detection is susceptible to missing paths that may still be in use for multiple reasons:
+
+- Time-of-check-to-time-of-use (TOCTTOU): new processes may appear
+  after Lix has enumerated the currently running processes, and will
+  not be taken into account;
+- Access privileges: if the garbage collection is not running as the
+  root user (this is typically the case for single-user
+  installations), it will not be able to scan processes belonging to
+  other users;
+- Other types of references: store paths may be stored in parts of the
+  filesystem (e.g. databases) or process memory (e.g. environment
+  variables changed since the start of the process) that Lix does not
+  scan.
+
+For this reason, it is recommended to create explicit roots whenever
+using store paths that aren't obtained from some existing explicit GC
+root.

doc/manual/src/package-management/garbage-collector-roots.md

@@ -1,18 +0,0 @@
-# Garbage Collector Roots
-
-The roots of the garbage collector are all store paths to which there
-are symlinks in the directory `prefix/nix/var/nix/gcroots`. For
-instance, the following command makes the path
-`/nix/store/d718ef...-foo` a root of the collector:
-
-```console
-$ ln -s /nix/store/d718ef...-foo /nix/var/nix/gcroots/bar
-```
-
-That is, after this command, the garbage collector will not remove
-`/nix/store/d718ef...-foo` or any of its dependencies.
-
-Subdirectories of `prefix/nix/var/nix/gcroots` are also searched for
-symlinks. Symlinks to non-store paths are followed and searched for
-roots, but symlinks to non-store paths *inside* the paths reached in
-that way are not followed to prevent infinite recursion.

doc/manual/src/protocols/derivation-aterm.md

@@ -2,18 +2,8 @@
 
 For historical reasons, [derivations](@docroot@/glossary.md#gloss-store-derivation) are stored on-disk in [ATerm](https://homepages.cwi.nl/~daybuild/daily-books/technology/aterm-guide/aterm-guide.html) format.
 
-Derivations are serialised in one of the following formats:
+Derivations are serialised in the following format:
 
-- ```
+```
-  Derive(...)
+Derive(...)
-  ```
+```
-
-  For all stable derivations.
-
-- ```
-  DrvWithVersion(<version-string>, ...)
-  ```
-
-  The only `version-string`s that are in use today are for [experimental features](@docroot@/contributing/experimental-features.md):
-
-  - `"xp-dyn-drv"` for the [`dynamic-derivations`](@docroot@/contributing/experimental-features.md#xp-feature-dynamic-derivations) experimental feature.

docker.nix

@@ -193,13 +193,11 @@ let
             in
             ''
               {
-                ${
+                ${lib.concatStringsSep "\n" (
-                  lib.concatStringsSep "\n" (
+                  builtins.map (output: ''
-                    builtins.map (output: ''
+                    ${output} = { outPath = "${lib.getOutput output drv}"; };
-                      ${output} = { outPath = "${lib.getOutput output drv}"; };
+                  '') outputs
-                    '') outputs
+                )}
-                  )
-                }
                 outputs = [ ${lib.concatStringsSep " " (builtins.map (x: "\"${x}\"") outputs)} ];
                 name = "${drv.name}";
                 outPath = "${drv}";
@@ -361,7 +359,8 @@ let
         "org.opencontainers.image.source" = "https://git.lix.systems/lix-project/lix";
         "org.opencontainers.image.vendor" = "Lix project";
         "org.opencontainers.image.version" = pkgs.nix.version;
-        "org.opencontainers.image.description" = "Minimal Lix container image, with some batteries included.";
+        "org.opencontainers.image.description" =
+          "Minimal Lix container image, with some batteries included.";
       } // lib.optionalAttrs (lixRevision != null) { "org.opencontainers.image.revision" = lixRevision; };
     };
 

flake.lock

@@ -16,6 +16,22 @@
         "type": "github"
       }
     },
+    "lowdown-src": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1633514407,
+        "narHash": "sha256-Dw32tiMjdK9t3ETl5fzGrutQTzh2rufgZV4A/BbxuD4=",
+        "owner": "kristapsdz",
+        "repo": "lowdown",
+        "rev": "d2c2b44ff6c27b936ec27358a2653caaef8f73b8",
+        "type": "github"
+      },
+      "original": {
+        "owner": "kristapsdz",
+        "repo": "lowdown",
+        "type": "github"
+      }
+    },
     "nix2container": {
       "flake": false,
       "locked": {
@@ -32,18 +48,44 @@
         "type": "github"
       }
     },
-    "nixpkgs": {
+    "nix_2_18": {
+      "inputs": {
+        "flake-compat": [
+          "flake-compat"
+        ],
+        "lowdown-src": "lowdown-src",
+        "nixpkgs": "nixpkgs",
+        "nixpkgs-regression": [
+          "nixpkgs-regression"
+        ]
+      },
       "locked": {
-        "lastModified": 1733348545,
+        "lastModified": 1730375271,
-        "narHash": "sha256-b4JrUmqT0vFNx42aEN9LTWOHomkTKL/ayLopflVf81U=",
+        "narHash": "sha256-RrOFlDGmRXcVRV2p2HqHGqvzGNyWoD0Dado/BNlJ1SI=",
         "owner": "NixOS",
-        "repo": "nixpkgs",
+        "repo": "nix",
-        "rev": "9ecb50d2fae8680be74c08bb0a995c5383747f89",
+        "rev": "0f665ff6779454f2117dcc32e44380cda7f45523",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
-        "ref": "nixos-24.11-small",
+        "ref": "2.18.9",
+        "repo": "nix",
+        "type": "github"
+      }
+    },
+    "nixpkgs": {
+      "locked": {
+        "lastModified": 1705033721,
+        "narHash": "sha256-K5eJHmL1/kev6WuqyqqbS1cdNnSidIZ3jeqJ7GbrYnQ=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "a1982c92d8980a0114372973cbdfe0a307f1bdea",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "nixos-23.05-small",
         "repo": "nixpkgs",
         "type": "github"
       }
@@ -64,6 +106,22 @@
         "type": "github"
       }
     },
+    "nixpkgs_2": {
+      "locked": {
+        "lastModified": 1749522908,
+        "narHash": "sha256-eWANkhWXFL1MmaxzsZ9bhLCNT8OVs7CC+OXaSDGlA8A=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "e5cb99555c45a13dcc5f1317462238530b0066b7",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "nixos-25.05-small",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
     "pre-commit-hooks": {
       "flake": false,
       "locked": {
@@ -84,7 +142,8 @@
       "inputs": {
         "flake-compat": "flake-compat",
         "nix2container": "nix2container",
-        "nixpkgs": "nixpkgs",
+        "nix_2_18": "nix_2_18",
+        "nixpkgs": "nixpkgs_2",
         "nixpkgs-regression": "nixpkgs-regression",
         "pre-commit-hooks": "pre-commit-hooks"
       }

flake.nix

@@ -2,8 +2,19 @@
   description = "Lix: A modern, delicious implementation of the Nix package manager";
 
   inputs = {
-    nixpkgs.url = "github:NixOS/nixpkgs/nixos-24.11-small";
+    nixpkgs.url = "github:NixOS/nixpkgs/nixos-25.05-small";
     nixpkgs-regression.url = "github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2";
+
+    # Required because Nix 2.18 is not in Nixpkgs ≥ 25.05 anymore.
+    nix_2_18 = {
+      url = "github:NixOS/nix/2.18.9";
+      # NOTE(Raito): this is not possible because patches on libseccomp does not apply anymore on this Nix.
+      # Let's keep the latest known nixpkgs useable with Nix 2.18 for our tests.
+      # inputs.nixpkgs.follows = "nixpkgs";
+      inputs.nixpkgs-regression.follows = "nixpkgs-regression";
+      inputs.flake-compat.follows = "flake-compat";
+    };
+
     pre-commit-hooks = {
       url = "github:cachix/git-hooks.nix";
       flake = false;
@@ -25,6 +36,7 @@
       nixpkgs-regression,
       pre-commit-hooks,
       nix2container,
+      nix_2_18,
       flake-compat,
     }:
 
@@ -163,6 +175,11 @@
         {
           nixStable = prev.nix;
 
+          # Nix 2.18 has been removed from Nixpkgs ≥ 25.05, so we need to reintroduce it ourselves for our tests.
+          nixVersions = prev.nixVersions // {
+            nix_2_18 = nix_2_18.outputs.packages.${currentStdenv.hostPlatform.system}.default;
+          };
+
           # Forward from the previous stage as we don’t want it to pick the lowdown override
           nixUnstable = prev.nixUnstable;
 
@@ -383,10 +400,12 @@
                 name = "nixpkgs-lib-tests";
                 paths =
                   [ testWithNix ]
-                  # NOTE: nixpkgs 24.11 is being ... *creative*, and requires this dance to override
+                  # NOTE: nixpkgs 25.05 is being ... *creative*, and requires this dance to override
                   # the evaluator used for the test. it will break again in the future, don't worry.
                   ++ lib.optionals pkgs.stdenv.isLinux [
-                    (pkgs.callPackage "${nixpkgs}/ci/eval" { nixVersions.nix_2_24 = nix; }).attrpathsSuperset
+                    ((pkgs.callPackage "${nixpkgs}/ci/eval" { nixVersions.latest = nix; }).attrpathsSuperset {
+                      evalSystem = system;
+                    })
                   ];
               }
             );
@@ -460,7 +479,10 @@
         }
         // (
           lib.optionalAttrs (builtins.elem system linux64BitSystems) {
-            nix-static = nixpkgsFor.${system}.static.nix;
+            # python doesn't work in static builds as of 2025-06-27
+            nix-static = nixpkgsFor.${system}.static.nix.overrideAttrs (_: {
+              doCheck = false;
+            });
             dockerImage =
               let
                 pkgs = nixpkgsFor.${system}.native;

justfile

@@ -41,6 +41,10 @@ test-unit *OPTIONS: (test "--suite" "check")
 # Run integration tests only
 test-integration *OPTIONS: install (test "--suite" "installcheck")
 
+# Run functional2 tests using pytest directly, allowing for additional arguments to be passed to pytest e.g. for more granular test selection
+test-functional2 *OPTIONS:
+    cd tests && python -m pytest -v {{ OPTIONS }} functional2
+
 alias clang-tidy := lint
 
 # Lint with `clang-tidy`

lix/legacy/build-remote.cc

@@ -84,7 +84,13 @@ static int main_build_remote(AsyncIoRoot & aio, std::string programName, Strings
 
         initPlugins();
 
-        auto store = aio.blockOn(openStore());
+        // FIXME this does not open a daemon connection for historical reasons.
+        // we may create a lot of build hook instances, and having each of them
+        // also create a daemon instance is inefficient and wasteful. in future
+        // versions of the build hook (where we don't need one hook process per
+        // build) we should change this to using a daemon connection, ideally a
+        // daemon connection provided by the parent via file descriptor passing
+        auto store = aio.blockOn(openStore(settings.storeUri, {}, AllowDaemon::Disallow));
 
         /* It would be more appropriate to use $XDG_RUNTIME_DIR, since
            that gets cleared on reboot, but it wouldn't work on macOS. */
@@ -324,7 +330,7 @@ connected:
             //
             // 2. Changing the `inputSrcs` set changes the associated
             //    output ids, which break CA derivations
-            if (!drv.inputDrvs.map.empty())
+            if (!drv.inputDrvs.empty())
                 drv.inputSrcs = store->parseStorePathSet(inputs);
             optResult = aio.blockOn(sshStore->buildDerivation(*drvPath, (const BasicDerivation &) drv));
             auto & result = *optResult;
@@ -336,7 +342,7 @@ connected:
             ));
             auto res = aio.blockOn(sshStore->buildPathsWithResults({
                 DerivedPath::Built {
-                    .drvPath = makeConstantStorePathRef(*drvPath),
+                    .drvPath = makeConstantStorePath(*drvPath),
                     .outputs = OutputsSpec::All {},
                 }
             }));
@@ -346,31 +352,11 @@ connected:
         }
 
 
-        auto outputHashes = aio.blockOn(staticOutputHashes(*store, drv));
-        std::set<Realisation> missingRealisations;
         StorePathSet missingPaths;
-        if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations) && !drv.type().hasKnownOutputPaths()) {
+        auto outputPaths = drv.outputsAndPaths(*store);
-            for (auto & outputName : wantedOutputs) {
+        for (auto & [outputName, outputPath] : outputPaths) {
-                auto thisOutputHash = outputHashes.at(outputName);
+            if (!aio.blockOn(store->isValidPath(outputPath.second)))
-                auto thisOutputId = DrvOutput{ thisOutputHash, outputName };
+                missingPaths.insert(outputPath.second);
-                if (!aio.blockOn(store->queryRealisation(thisOutputId))) {
-                    debug("missing output %s", outputName);
-                    assert(optResult);
-                    auto & result = *optResult;
-                    auto i = result.builtOutputs.find(outputName);
-                    assert(i != result.builtOutputs.end());
-                    auto & newRealisation = i->second;
-                    missingRealisations.insert(newRealisation);
-                    missingPaths.insert(newRealisation.outPath);
-                }
-            }
-        } else {
-            auto outputPaths = drv.outputsAndOptPaths(*store);
-            for (auto & [outputName, hopefullyOutputPath] : outputPaths) {
-                assert(hopefullyOutputPath.second);
-                if (!aio.blockOn(store->isValidPath(*hopefullyOutputPath.second)))
-                    missingPaths.insert(*hopefullyOutputPath.second);
-            }
         }
 
         if (!missingPaths.empty()) {
@@ -382,13 +368,6 @@ connected:
                 copyPaths(*sshStore, *store, missingPaths, NoRepair, NoCheckSigs, NoSubstitute)
             );
         }
-        // XXX: Should be done as part of `copyPaths`
-        for (auto & realisation : missingRealisations) {
-            // Should hold, because if the feature isn't enabled the set
-            // of missing realisations should be empty
-            experimentalFeatureSettings.require(Xp::CaDerivations);
-            aio.blockOn(store->registerDrvOutput(realisation));
-        }
 
         return 0;
     }

lix/legacy/nix-build.cc

@@ -355,7 +355,7 @@ static void main_nix_build(AsyncIoRoot & aio, std::string programName, Strings a
 
                 auto bashDrv = drv->requireDrvPath(*state);
                 pathsToBuild.push_back(DerivedPath::Built {
-                    .drvPath = makeConstantStorePathRef(bashDrv),
+                    .drvPath = makeConstantStorePath(bashDrv),
                     .outputs = OutputsSpec::Names {"out"},
                 });
                 pathsToCopy.insert(bashDrv);
@@ -368,22 +368,16 @@ static void main_nix_build(AsyncIoRoot & aio, std::string programName, Strings a
             }
         }
 
-        std::function<void(ref<SingleDerivedPath>, const DerivedPathMap<StringSet>::ChildNode &)> accumDerivedPath;
+        auto accumDerivedPath = [&](SingleDerivedPath::Opaque inputDrv, const StringSet & inputNode) {
-
+            if (!inputNode.empty())
-        accumDerivedPath = [&](ref<SingleDerivedPath> inputDrv, const DerivedPathMap<StringSet>::ChildNode & inputNode) {
-            if (!inputNode.value.empty())
                 pathsToBuild.push_back(DerivedPath::Built {
                     .drvPath = inputDrv,
-                    .outputs = OutputsSpec::Names { inputNode.value },
+                    .outputs = OutputsSpec::Names { inputNode },
                 });
-            for (const auto & [outputName, childNode] : inputNode.childMap)
-                accumDerivedPath(
-                    make_ref<SingleDerivedPath>(SingleDerivedPath::Built { inputDrv, outputName }),
-                    childNode);
         };
 
         // Build or fetch all dependencies of the derivation.
-        for (const auto & [inputDrv0, inputNode] : drv.inputDrvs.map) {
+        for (const auto & [inputDrv0, inputNode] : drv.inputDrvs) {
             // To get around lambda capturing restrictions in the
             // standard.
             const auto & inputDrv = inputDrv0;
@@ -392,7 +386,7 @@ static void main_nix_build(AsyncIoRoot & aio, std::string programName, Strings a
                         return !std::regex_search(store->printStorePath(inputDrv), regex::parse(exclude));
                     }))
             {
-                accumDerivedPath(makeConstantStorePathRef(inputDrv), inputNode);
+                accumDerivedPath(makeConstantStorePath(inputDrv), inputNode);
                 pathsToCopy.insert(inputDrv);
             }
         }
@@ -407,14 +401,8 @@ static void main_nix_build(AsyncIoRoot & aio, std::string programName, Strings a
 
         if (shellDrv) {
             auto shellDrvOutputs =
-                aio.blockOn(store->queryPartialDerivationOutputMap(shellDrv.value(), &*evalStore));
+                aio.blockOn(store->queryDerivationOutputMap(shellDrv.value(), &*evalStore));
-            shell = store->printStorePath(shellDrvOutputs.at("out").value()) + "/bin/bash";
+            shell = store->printStorePath(shellDrvOutputs.at("out")) + "/bin/bash";
-        }
-
-        if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) {
-            auto resolvedDrv = aio.blockOn(drv.tryResolve(*store));
-            assert(resolvedDrv && "Successfully resolved the derivation");
-            drv = *resolvedDrv;
         }
 
         // Set the environment.
@@ -455,20 +443,16 @@ static void main_nix_build(AsyncIoRoot & aio, std::string programName, Strings a
         if (env.count("__json")) {
             StorePathSet inputs;
 
-            std::function<void(const StorePath &, const DerivedPathMap<StringSet>::ChildNode &)> accumInputClosure;
+            auto accumInputClosure = [&](const StorePath & inputDrv, const StringSet & inputNode) {
-
-            accumInputClosure = [&](const StorePath & inputDrv, const DerivedPathMap<StringSet>::ChildNode & inputNode) {
                 auto outputs =
-                    aio.blockOn(store->queryPartialDerivationOutputMap(inputDrv, &*evalStore));
+                    aio.blockOn(store->queryDerivationOutputMap(inputDrv, &*evalStore));
-                for (auto & i : inputNode.value) {
+                for (auto & i : inputNode) {
                     auto o = outputs.at(i);
-                    aio.blockOn(store->computeFSClosure(*o, inputs));
+                    aio.blockOn(store->computeFSClosure(o, inputs));
                 }
-                for (const auto & [outputName, childNode] : inputNode.childMap)
-                    accumInputClosure(*outputs.at(outputName), childNode);
             };
 
-            for (const auto & [inputDrv, inputNode] : drv.inputDrvs.map)
+            for (const auto & [inputDrv, inputNode] : drv.inputDrvs)
                 accumInputClosure(inputDrv, inputNode);
 
             ParsedDerivation parsedDrv(drvInfo.requireDrvPath(*state), drv);
@@ -581,7 +565,7 @@ static void main_nix_build(AsyncIoRoot & aio, std::string programName, Strings a
                 throw Error("derivation '%s' lacks an 'outputName' attribute", store->printStorePath(drvPath));
 
             pathsToBuild.push_back(DerivedPath::Built{
-                .drvPath = makeConstantStorePathRef(drvPath),
+                .drvPath = makeConstantStorePath(drvPath),
                 .outputs = OutputsSpec::Names{outputName},
             });
             pathsToBuildOrdered.push_back({drvPath, {outputName}});
@@ -607,11 +591,9 @@ static void main_nix_build(AsyncIoRoot & aio, std::string programName, Strings a
                 drvPrefix += fmt("-%d", counter + 1);
 
             auto builtOutputs =
-                aio.blockOn(store->queryPartialDerivationOutputMap(drvPath, &*evalStore));
+                aio.blockOn(store->queryDerivationOutputMap(drvPath, &*evalStore));
 
-            auto maybeOutputPath = builtOutputs.at(outputName);
+            auto outputPath = builtOutputs.at(outputName);
-            assert(maybeOutputPath);
-            auto outputPath = *maybeOutputPath;
 
             if (auto store2 = store.try_cast_shared<LocalFSStore>()) {
                 std::string symlink = drvPrefix;

lix/legacy/nix-env.cc

@@ -495,7 +495,7 @@ static void printMissing(EvalState & state, DrvInfos & elems)
     for (auto & i : elems)
         if (auto drvPath = i.queryDrvPath(state))
             targets.emplace_back(DerivedPath::Built{
-                .drvPath = makeConstantStorePathRef(*drvPath),
+                .drvPath = makeConstantStorePath(*drvPath),
                 .outputs = OutputsSpec::All { },
             });
         else
@@ -792,7 +792,7 @@ static void opSet(Globals & globals, Strings opFlags, Strings opArgs)
     std::vector<DerivedPath> paths {
         drvPath
         ? (DerivedPath) (DerivedPath::Built {
-            .drvPath = makeConstantStorePathRef(*drvPath),
+            .drvPath = makeConstantStorePath(*drvPath),
             .outputs = OutputsSpec::All { },
         })
         : (DerivedPath) (DerivedPath::Opaque {

lix/legacy/nix-instantiate.cc

@@ -22,7 +22,7 @@ static Path gcRoot;
 static int rootNr = 0;
 
 
-enum OutputKind { okPlain, okXML, okJSON };
+enum OutputKind { okPlain, okRaw, okXML, okJSON };
 
 void processExpr(EvalState & state, const Strings & attrPaths,
     bool parseOnly, bool strict, Bindings & autoArgs,
@@ -48,7 +48,11 @@ void processExpr(EvalState & state, const Strings & attrPaths,
                 vRes = v;
             else
                 state.autoCallFunction(autoArgs, v, vRes, noPos);
-            if (output == okXML)
+            if (output == okRaw)
+                std::cout << *state.coerceToString(noPos, vRes, context, "while generating the nix-instantiate output", StringCoercionMode::Strict);
+                // We intentionally don't output a newline here. The default PS1 for Bash in NixOS starts with a newline
+                // and other interactive shells like Zsh are smart enough to print a missing newline before the prompt.
+            else if (output == okXML)
                 printValueAsXML(state, strict, location, vRes, std::cout, context, noPos);
             else if (output == okJSON) {
                 printValueAsJSON(state, strict, vRes, noPos, std::cout, context);
@@ -130,6 +134,8 @@ static int main_nix_instantiate(AsyncIoRoot & aio, std::string programName, Stri
                 gcRoot = getArg(*arg, arg, end);
             else if (*arg == "--indirect")
                 ;
+            else if (*arg == "--raw")
+                outputKind = okRaw;
             else if (*arg == "--xml")
                 outputKind = okXML;
             else if (*arg == "--json")

lix/legacy/nix-store.cc

@@ -33,25 +33,23 @@ namespace nix {
 using std::cin;
 using std::cout;
 
-
+typedef void (*Operation)(
-typedef void (* Operation) (AsyncIoRoot & aio, Strings opFlags, Strings opArgs);
+    std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, Strings opArgs
-
+);
 
 static Path gcRoot;
 static int rootNr = 0;
 static bool noOutput = false;
-static std::shared_ptr<Store> store;
 
-
+ref<LocalStore> ensureLocalStore(std::shared_ptr<Store> store)
-ref<LocalStore> ensureLocalStore()
 {
     auto store2 = std::dynamic_pointer_cast<LocalStore>(store);
     if (!store2) throw Error("you don't have sufficient rights to use this command");
     return ref<LocalStore>::unsafeFromPtr(store2);
 }
 
-
+static kj::Promise<Result<StorePath>>
-static kj::Promise<Result<StorePath>> useDeriver(const StorePath & path)
+useDeriver(std::shared_ptr<Store> store, const StorePath & path)
 try {
     if (path.isDerivation()) co_return path;
     auto info = TRY_AWAIT(store->queryPathInfo(path));
@@ -65,7 +63,8 @@ try {
 
 /* Realise the given path.  For a derivation that means build it; for
    other paths it means ensure their validity. */
-static kj::Promise<Result<PathSet>> realisePath(StorePathWithOutputs path, bool build = true)
+static kj::Promise<Result<PathSet>>
+realisePath(std::shared_ptr<Store> store, StorePathWithOutputs path, bool build = true)
 try {
     auto store2 = std::dynamic_pointer_cast<LocalFSStore>(store);
 
@@ -125,7 +124,8 @@ try {
 
 
 /* Realise the given paths. */
-static void opRealise(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
+static void
+opRealise(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
 {
     bool dryRun = false;
     BuildMode buildMode = bmNormal;
@@ -170,7 +170,7 @@ static void opRealise(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
 
     if (!ignoreUnknown)
         for (auto & i : paths) {
-            auto paths2 = aio.blockOn(realisePath(i, false));
+            auto paths2 = aio.blockOn(realisePath(store, i, false));
             if (!noOutput)
                 for (auto & j : paths2)
                     cout << fmt("%1%\n", j);
@@ -179,7 +179,7 @@ static void opRealise(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
 
 
 /* Add files to the Nix store and print the resulting paths. */
-static void opAdd(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
+static void opAdd(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
 {
     if (!opFlags.empty()) throw UsageError("unknown flag");
 
@@ -196,7 +196,8 @@ static void opAdd(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
 
 /* Preload the output of a fixed-output derivation into the Nix
    store. */
-static void opAddFixed(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
+static void
+opAddFixed(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
 {
     auto method = FileIngestionMethod::Flat;
 
@@ -222,7 +223,8 @@ static void opAddFixed(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
 
 
 /* Hack to support caching in `nix-prefetch-url'. */
-static void opPrintFixedPath(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
+static void
+opPrintFixedPath(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
 {
     auto method = FileIngestionMethod::Flat;
 
@@ -245,19 +247,20 @@ static void opPrintFixedPath(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
     })));
 }
 
-
+static kj::Promise<Result<StorePathSet>> maybeUseOutputs(
-static kj::Promise<Result<StorePathSet>> maybeUseOutputs(const StorePath & storePath, bool useOutput, bool forceRealise)
+    std::shared_ptr<Store> store, const StorePath & storePath, bool useOutput, bool forceRealise
+)
 try {
-    if (forceRealise) TRY_AWAIT(realisePath({storePath}));
+    if (forceRealise) {
+        TRY_AWAIT(realisePath(store, {storePath}));
+    }
     if (useOutput && storePath.isDerivation()) {
         auto drv = TRY_AWAIT(store->derivationFromPath(storePath));
         StorePathSet outputs;
         if (forceRealise)
             co_return TRY_AWAIT(store->queryDerivationOutputs(storePath));
-        for (auto & i : drv.outputsAndOptPaths(*store)) {
+        for (auto & i : drv.outputsAndPaths(*store)) {
-            if (!i.second.second)
+            outputs.insert(i.second.second);
-                throw UsageError("Cannot use output path of floating content-addressed derivation until we know what it is (e.g. by building it)");
-            outputs.insert(*i.second.second);
         }
         co_return outputs;
     }
@@ -270,8 +273,14 @@ try {
 /* Some code to print a tree representation of a derivation dependency
    graph.  Topological sorting is used to keep the tree relatively
    flat. */
-static void printTree(AsyncIoRoot & aio, const StorePath & path,
+static void printTree(
-    const std::string & firstPad, const std::string & tailPad, StorePathSet & done)
+    std::shared_ptr<Store> store,
+    AsyncIoRoot & aio,
+    const StorePath & path,
+    const std::string & firstPad,
+    const std::string & tailPad,
+    StorePathSet & done
+)
 {
     if (!done.insert(path).second) {
         cout << fmt("%s%s [...]\n", firstPad, store->printStorePath(path));
@@ -291,16 +300,21 @@ static void printTree(AsyncIoRoot & aio, const StorePath & path,
 
     for (const auto &[n, i] : enumerate(sorted)) {
         bool last = n + 1 == sorted.size();
-        printTree(aio, i,
+        printTree(
+            store,
+            aio,
+            i,
             tailPad + (last ? treeLast : treeConn),
             tailPad + (last ? treeNull : treeLine),
-            done);
+            done
+        );
     }
 }
 
 
 /* Perform various sorts of queries. */
-static void opQuery(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
+static void
+opQuery(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
 {
     enum QueryType
         { qOutputs, qRequisites, qReferences, qReferrers
@@ -351,7 +365,9 @@ static void opQuery(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
 
         case qOutputs: {
             for (auto & i : opArgs) {
-                auto outputs = aio.blockOn(maybeUseOutputs(store->followLinksToStorePath(i), true, forceRealise));
+                auto outputs = aio.blockOn(
+                    maybeUseOutputs(store, store->followLinksToStorePath(i), true, forceRealise)
+                );
                 for (auto & outputPath : outputs)
                     cout << fmt("%1%\n", store->printStorePath(outputPath));
             }
@@ -364,7 +380,9 @@ static void opQuery(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
         case qReferrersClosure: {
             StorePathSet paths;
             for (auto & i : opArgs) {
-                auto ps = aio.blockOn(maybeUseOutputs(store->followLinksToStorePath(i), useOutput, forceRealise));
+                auto ps = aio.blockOn(maybeUseOutputs(
+                    store, store->followLinksToStorePath(i), useOutput, forceRealise
+                ));
                 for (auto & j : ps) {
                     if (query == qRequisites) {
                         aio.blockOn(store->computeFSClosure(j, paths, false, includeOutputs));
@@ -415,7 +433,7 @@ static void opQuery(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
 
         case qBinding:
             for (auto & i : opArgs) {
-                auto path = aio.blockOn(useDeriver(store->followLinksToStorePath(i)));
+                auto path = aio.blockOn(useDeriver(store, store->followLinksToStorePath(i)));
                 Derivation drv = aio.blockOn(store->derivationFromPath(path));
                 StringPairs::iterator j = drv.env.find(bindingName);
                 if (j == drv.env.end())
@@ -428,7 +446,10 @@ static void opQuery(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
         case qHash:
         case qSize:
             for (auto & i : opArgs) {
-                for (auto & j : aio.blockOn(maybeUseOutputs(store->followLinksToStorePath(i), useOutput, forceRealise))) {
+                for (auto & j : aio.blockOn(maybeUseOutputs(
+                         store, store->followLinksToStorePath(i), useOutput, forceRealise
+                     )))
+                {
                     auto info = aio.blockOn(store->queryPathInfo(j));
                     if (query == qHash) {
                         assert(info->narHash.type == HashType::SHA256);
@@ -442,15 +463,19 @@ static void opQuery(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
         case qTree: {
             StorePathSet done;
             for (auto & i : opArgs)
-                printTree(aio, store->followLinksToStorePath(i), "", "", done);
+                printTree(store, aio, store->followLinksToStorePath(i), "", "", done);
             break;
         }
 
         case qGraph: {
             StorePathSet roots;
             for (auto & i : opArgs)
-                for (auto & j : aio.blockOn(maybeUseOutputs(store->followLinksToStorePath(i), useOutput, forceRealise)))
+                for (auto & j : aio.blockOn(maybeUseOutputs(
+                         store, store->followLinksToStorePath(i), useOutput, forceRealise
+                     )))
+                {
                     roots.insert(j);
+                }
             aio.blockOn(printDotGraph(ref<Store>::unsafeFromPtr(store), std::move(roots)));
             break;
         }
@@ -458,8 +483,12 @@ static void opQuery(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
         case qGraphML: {
             StorePathSet roots;
             for (auto & i : opArgs)
-                for (auto & j : aio.blockOn(maybeUseOutputs(store->followLinksToStorePath(i), useOutput, forceRealise)))
+                for (auto & j : aio.blockOn(maybeUseOutputs(
+                         store, store->followLinksToStorePath(i), useOutput, forceRealise
+                     )))
+                {
                     roots.insert(j);
+                }
             aio.blockOn(printGraphML(ref<Store>::unsafeFromPtr(store), std::move(roots)));
             break;
         }
@@ -473,8 +502,12 @@ static void opQuery(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
         case qRoots: {
             StorePathSet args;
             for (auto & i : opArgs)
-                for (auto & p : aio.blockOn(maybeUseOutputs(store->followLinksToStorePath(i), useOutput, forceRealise)))
+                for (auto & p : aio.blockOn(maybeUseOutputs(
+                         store, store->followLinksToStorePath(i), useOutput, forceRealise
+                     )))
+                {
                     args.insert(p);
+                }
 
             StorePathSet referrers;
             aio.blockOn(store->computeFSClosure(
@@ -494,8 +527,8 @@ static void opQuery(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
     }
 }
 
-
+static void
-static void opPrintEnv(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
+opPrintEnv(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
 {
     if (!opFlags.empty()) throw UsageError("unknown flag");
     if (opArgs.size() != 1) throw UsageError("'--print-env' requires one derivation store path");
@@ -520,8 +553,8 @@ static void opPrintEnv(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
     cout << "'\n";
 }
 
-
+static void
-static void opReadLog(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
+opReadLog(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
 {
     if (!opFlags.empty()) throw UsageError("unknown flag");
 
@@ -538,8 +571,8 @@ static void opReadLog(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
     }
 }
 
-
+static void
-static void opDumpDB(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
+opDumpDB(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
 {
     if (!opFlags.empty()) throw UsageError("unknown flag");
     if (!opArgs.empty()) {
@@ -554,8 +587,13 @@ static void opDumpDB(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
     }
 }
 
-
+static void registerValidity(
-static void registerValidity(AsyncIoRoot & aio, bool reregister, bool hashGiven, bool canonicalise)
+    std::shared_ptr<Store> store,
+    AsyncIoRoot & aio,
+    bool reregister,
+    bool hashGiven,
+    bool canonicalise
+)
 {
     ValidPathInfos infos;
 
@@ -578,20 +616,20 @@ static void registerValidity(AsyncIoRoot & aio, bool reregister, bool hashGiven,
         }
     }
 
-    aio.blockOn(ensureLocalStore()->registerValidPaths(infos));
+    aio.blockOn(ensureLocalStore(store)->registerValidPaths(infos));
 }
 
-
+static void
-static void opLoadDB(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
+opLoadDB(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
 {
     if (!opFlags.empty()) throw UsageError("unknown flag");
     if (!opArgs.empty())
         throw UsageError("no arguments expected");
-    registerValidity(aio, true, true, false);
+    registerValidity(store, aio, true, true, false);
 }
 
-
+static void
-static void opRegisterValidity(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
+opRegisterValidity(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
 {
     bool reregister = false; // !!! maybe this should be the default
     bool hashGiven = false;
@@ -603,11 +641,11 @@ static void opRegisterValidity(AsyncIoRoot & aio, Strings opFlags, Strings opArg
 
     if (!opArgs.empty()) throw UsageError("no arguments expected");
 
-    registerValidity(aio, reregister, hashGiven, true);
+    registerValidity(store, aio, reregister, hashGiven, true);
 }
 
-
+static void
-static void opCheckValidity(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
+opCheckValidity(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
 {
     bool printInvalid = false;
 
@@ -626,8 +664,7 @@ static void opCheckValidity(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
     }
 }
 
-
+static void opGC(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
-static void opGC(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
 {
     bool printRoots = false;
     GCOptions options;
@@ -673,7 +710,8 @@ static void opGC(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
 /* Remove paths from the Nix store if possible (i.e., if they do not
    have any remaining referrers and are not reachable from any GC
    roots). */
-static void opDelete(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
+static void
+opDelete(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
 {
     GCOptions options;
     options.action = GCOptions::gcDeleteSpecific;
@@ -703,7 +741,7 @@ static void opDelete(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
 
 
 /* Dump a path as a Nix archive.  The archive is written to stdout */
-static void opDump(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
+static void opDump(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
 {
     if (!opFlags.empty()) throw UsageError("unknown flag");
     if (opArgs.size() != 1) throw UsageError("only one argument allowed");
@@ -716,7 +754,8 @@ static void opDump(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
 
 
 /* Restore a value from a Nix archive.  The archive is read from stdin. */
-static void opRestore(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
+static void
+opRestore(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
 {
     if (!opFlags.empty()) throw UsageError("unknown flag");
     if (opArgs.size() != 1) throw UsageError("only one argument allowed");
@@ -725,8 +764,8 @@ static void opRestore(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
     restorePath(*opArgs.begin(), source);
 }
 
-
+static void
-static void opExport(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
+opExport(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
 {
     for (auto & i : opFlags)
         throw UsageError("unknown flag '%1%'", i);
@@ -741,8 +780,8 @@ static void opExport(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
     sink.flush();
 }
 
-
+static void
-static void opImport(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
+opImport(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
 {
     for (auto & i : opFlags)
         throw UsageError("unknown flag '%1%'", i);
@@ -758,7 +797,7 @@ static void opImport(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
 
 
 /* Initialise the Nix databases. */
-static void opInit(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
+static void opInit(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
 {
     if (!opFlags.empty()) throw UsageError("unknown flag");
     if (!opArgs.empty())
@@ -769,7 +808,8 @@ static void opInit(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
 
 
 /* Verify the consistency of the Nix environment. */
-static void opVerify(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
+static void
+opVerify(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
 {
     if (!opArgs.empty())
         throw UsageError("no arguments expected");
@@ -790,7 +830,8 @@ static void opVerify(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
 
 
 /* Verify whether the contents of the given store path have not changed. */
-static void opVerifyPath(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
+static void
+opVerifyPath(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
 {
     if (!opFlags.empty())
         throw UsageError("no flags expected");
@@ -802,7 +843,7 @@ static void opVerifyPath(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
         printMsg(lvlTalkative, "checking path '%s'...", store->printStorePath(path));
         auto info = aio.blockOn(store->queryPathInfo(path));
         HashSink sink(info->narHash.type);
-        aio.blockOn(store->narFromPath(path))->drainInto(sink);
+        aio.blockOn(aio.blockOn(store->narFromPath(path))->drainInto(sink));
         auto current = sink.finish();
         if (current.first != info->narHash) {
             printError("path '%s' was modified! expected hash '%s', got '%s'",
@@ -819,7 +860,8 @@ static void opVerifyPath(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
 
 /* Repair the contents of the given path by redownloading it using a
    substituter (if available). */
-static void opRepairPath(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
+static void
+opRepairPath(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
 {
     if (!opFlags.empty())
         throw UsageError("no flags expected");
@@ -830,7 +872,8 @@ static void opRepairPath(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
 
 /* Optimise the disk space usage of the Nix store by hard-linking
    files with the same contents. */
-static void opOptimise(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
+static void
+opOptimise(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
 {
     if (!opArgs.empty() || !opFlags.empty())
         throw UsageError("no arguments expected");
@@ -839,7 +882,8 @@ static void opOptimise(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
 }
 
 /* Serve the nix store in a way usable by a restricted ssh user. */
-static void opServe(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
+static void
+opServe(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
 {
     bool writeAllowed = false;
     for (auto & i : opFlags)
@@ -860,9 +904,11 @@ static void opServe(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
 
     ServeProto::ReadConn rconn {
         .from = in,
+        .store = *store,
         .version = clientVersion,
     };
     ServeProto::WriteConn wconn {
+        .store = *store,
         .version = clientVersion,
     };
 
@@ -909,7 +955,7 @@ static void opServe(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
             case ServeProto::Command::QueryValidPaths: {
                 bool lock = readInt(in);
                 bool substitute = readInt(in);
-                auto paths = ServeProto::Serialise<StorePathSet>::read(*store, rconn);
+                auto paths = ServeProto::Serialise<StorePathSet>::read(rconn);
                 if (lock && writeAllowed)
                     for (auto & path : paths)
                         aio.blockOn(store->addTempRoot(path));
@@ -919,18 +965,18 @@ static void opServe(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
                 }
 
                 auto valid = aio.blockOn(store->queryValidPaths(paths));
-                out << ServeProto::write(*store, wconn, valid);
+                out << ServeProto::write(wconn, valid);
                 break;
             }
 
             case ServeProto::Command::QueryPathInfos: {
-                auto paths = ServeProto::Serialise<StorePathSet>::read(*store, rconn);
+                auto paths = ServeProto::Serialise<StorePathSet>::read(rconn);
                 // !!! Maybe we want a queryPathInfos?
                 for (auto & i : paths) {
                     try {
                         auto info = aio.blockOn(store->queryPathInfo(i));
                         out << store->printStorePath(info->path);
-                        out << ServeProto::write(*store, wconn, static_cast<const UnkeyedValidPathInfo &>(*info));
+                        out << ServeProto::write(wconn, static_cast<const UnkeyedValidPathInfo &>(*info));
                     } catch (InvalidPath &) {
                     }
                 }
@@ -939,8 +985,8 @@ static void opServe(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
             }
 
             case ServeProto::Command::DumpStorePath:
-                aio.blockOn(store->narFromPath(store->parseStorePath(readString(in))))
+                aio.blockOn(aio.blockOn(store->narFromPath(store->parseStorePath(readString(in))))
-                    ->drainInto(out);
+                                ->drainInto(out));
                 break;
 
             case ServeProto::Command::ImportPaths: {
@@ -953,7 +999,7 @@ static void opServe(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
             case ServeProto::Command::ExportPaths: {
                 readInt(in); // obsolete
                 aio.blockOn(store->exportPaths(
-                    ServeProto::Serialise<StorePathSet>::read(*store, rconn), out
+                    ServeProto::Serialise<StorePathSet>::read(rconn), out
                 ));
                 break;
             }
@@ -992,7 +1038,7 @@ static void opServe(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
                 MonitorFdHup monitor(in.fd);
                 auto status = aio.blockOn(store->buildDerivation(drvPath, drv));
 
-                out << ServeProto::write(*store, wconn, status);
+                out << ServeProto::write(wconn, status);
                 break;
             }
 
@@ -1000,12 +1046,12 @@ static void opServe(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
                 bool includeOutputs = readInt(in);
                 StorePathSet closure;
                 aio.blockOn(store->computeFSClosure(
-                    ServeProto::Serialise<StorePathSet>::read(*store, rconn),
+                    ServeProto::Serialise<StorePathSet>::read(rconn),
                     closure,
                     false,
                     includeOutputs
                 ));
-                out << ServeProto::write(*store, wconn, closure);
+                out << ServeProto::write(wconn, closure);
                 break;
             }
 
@@ -1020,7 +1066,7 @@ static void opServe(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
                 };
                 if (deriver != "")
                     info.deriver = store->parseStorePath(deriver);
-                info.references = ServeProto::Serialise<StorePathSet>::read(*store, rconn);
+                info.references = ServeProto::Serialise<StorePathSet>::read(rconn);
                 in >> info.registrationTime >> info.narSize >> info.ultimate;
                 info.sigs = readStrings<StringSet>(in);
                 info.ca = ContentAddress::parseOpt(readString(in));
@@ -1049,8 +1095,9 @@ static void opServe(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
     }
 }
 
-
+static void opGenerateBinaryCacheKey(
-static void opGenerateBinaryCacheKey(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
+    std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, Strings opArgs
+)
 {
     for (auto & i : opFlags)
         throw UsageError("unknown flag '%1%'", i);
@@ -1068,8 +1115,8 @@ static void opGenerateBinaryCacheKey(AsyncIoRoot & aio, Strings opFlags, Strings
     writeFile(secretKeyFile, secretKey.to_string());
 }
 
-
+static void
-static void opVersion(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
+opVersion(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
 {
     printVersion("nix-store");
 }
@@ -1214,10 +1261,11 @@ static int main_nix_store(AsyncIoRoot & aio, std::string programName, Strings ar
         if (showHelp) showManPage("nix-store" + opName);
         if (!op) throw UsageError("no operation specified");
 
+        std::shared_ptr<Store> store;
         if (op != opDump && op != opRestore) /* !!! hack */
             store = aio.blockOn(openStore());
 
-        op(aio, std::move(opFlags), std::move(opArgs));
+        op(store, aio, std::move(opFlags), std::move(opArgs));
 
         return 0;
     }

lix/libcmd/built-path.cc

@@ -13,9 +13,9 @@ namespace nix {
     bool MY_TYPE ::operator COMPARATOR (const MY_TYPE & other) const \
     { \
         const MY_TYPE* me = this; \
-        auto fields1 = std::tie(*me->drvPath, me->FIELD); \
+        auto fields1 = std::tie(me->drvPath, me->FIELD); \
         me = &other; \
-        auto fields2 = std::tie(*me->drvPath, me->FIELD); \
+        auto fields2 = std::tie(me->drvPath, me->FIELD); \
         return fields1 COMPARATOR fields2; \
     }
 #define CMP(CHILD_TYPE, MY_TYPE, FIELD) \
@@ -23,10 +23,6 @@ namespace nix {
     CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, !=) \
     CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, <)
 
-#define FIELD_TYPE std::pair<std::string, StorePath>
-CMP(SingleBuiltPath, SingleBuiltPathBuilt, output)
-#undef FIELD_TYPE
-
 #define FIELD_TYPE std::map<std::string, StorePath>
 CMP(SingleBuiltPath, BuiltPathBuilt, outputs)
 #undef FIELD_TYPE
@@ -34,16 +30,6 @@ CMP(SingleBuiltPath, BuiltPathBuilt, outputs)
 #undef CMP
 #undef CMP_ONE
 
-StorePath SingleBuiltPath::outPath() const
-{
-    return std::visit(
-        overloaded{
-            [](const SingleBuiltPath::Opaque & p) { return p.path; },
-            [](const SingleBuiltPath::Built & b) { return b.output.second; },
-        }, raw()
-    );
-}
-
 StorePathSet BuiltPath::outPaths() const
 {
     return std::visit(
@@ -59,32 +45,10 @@ StorePathSet BuiltPath::outPaths() const
     );
 }
 
-SingleDerivedPath::Built SingleBuiltPath::Built::discardOutputPath() const
-{
-    return SingleDerivedPath::Built {
-        .drvPath = make_ref<SingleDerivedPath>(drvPath->discardOutputPath()),
-        .output = output.first,
-    };
-}
-
-SingleDerivedPath SingleBuiltPath::discardOutputPath() const
-{
-    return std::visit(
-        overloaded{
-            [](const SingleBuiltPath::Opaque & p) -> SingleDerivedPath {
-                return p;
-            },
-            [](const SingleBuiltPath::Built & b) -> SingleDerivedPath {
-                return b.discardOutputPath();
-            },
-        }, raw()
-    );
-}
-
 kj::Promise<Result<JSON>> BuiltPath::Built::toJSON(const Store & store) const
 try {
     JSON res;
-    res["drvPath"] = TRY_AWAIT(drvPath->toJSON(store));
+    res["drvPath"] = TRY_AWAIT(drvPath.toJSON(store));
     for (const auto & [outputName, outputPath] : outputs) {
         res["outputs"][outputName] = store.printStorePath(outputPath);
     }
@@ -93,36 +57,6 @@ try {
     co_return result::current_exception();
 }
 
-kj::Promise<Result<JSON>> SingleBuiltPath::Built::toJSON(const Store & store) const
-try {
-    JSON res;
-    res["drvPath"] = TRY_AWAIT(drvPath->toJSON(store));
-    auto & [outputName, outputPath] = output;
-    res["output"] = outputName;
-    res["outputPath"] = store.printStorePath(outputPath);
-    co_return res;
-} catch (...) {
-    co_return result::current_exception();
-}
-
-kj::Promise<Result<JSON>> SingleBuiltPath::toJSON(const Store & store) const
-try {
-    co_return TRY_AWAIT(std::visit([&](const auto & buildable) {
-        return buildable.toJSON(store);
-    }, raw()));
-} catch (...) {
-    co_return result::current_exception();
-}
-
-
-kj::Promise<Result<JSON>> BuiltPath::toJSON(const Store & store) const
-try {
-    co_return TRY_AWAIT(std::visit([&](const auto & buildable) {
-        return buildable.toJSON(store);
-    }, raw()));
-} catch (...) {
-    co_return result::current_exception();
-}
 
 kj::Promise<Result<RealisedPath::Set>> BuiltPath::toRealisedPaths(Store & store) const
 try {
@@ -141,24 +75,10 @@ try {
         [&](const BuiltPath::Built & p) -> kj::Promise<Result<void>> {
             try {
                 auto drvHashes = TRY_AWAIT(
-                    staticOutputHashes(store, TRY_AWAIT(store.readDerivation(p.drvPath->outPath())))
+                    staticOutputHashes(store, TRY_AWAIT(store.readDerivation(p.drvPath.path)))
                 );
                 for (auto& [outputName, outputPath] : p.outputs) {
-                    if (experimentalFeatureSettings.isEnabled(
+                    res.insert(outputPath);
-                                Xp::CaDerivations)) {
-                        auto drvOutput = get(drvHashes, outputName);
-                        if (!drvOutput)
-                            throw Error(
-                                "the derivation '%s' has unrealised output '%s' (derived-path.cc/toRealisedPaths)",
-                                store.printStorePath(p.drvPath->outPath()), outputName);
-                        auto thisRealisation = TRY_AWAIT(store.queryRealisation(
-                            DrvOutput{*drvOutput, outputName}));
-                        assert(thisRealisation);  // We’ve built it, so we must
-                                                  // have the realisation
-                        res.insert(*thisRealisation);
-                    } else {
-                        res.insert(outputPath);
-                    }
                 }
                 co_return result::success();
             } catch (...) {

lix/libcmd/built-path.hh

@@ -7,63 +7,15 @@
 
 namespace nix {
 
-struct SingleBuiltPath;
-
-struct SingleBuiltPathBuilt {
-    ref<SingleBuiltPath> drvPath;
-    std::pair<std::string, StorePath> output;
-
-    SingleDerivedPathBuilt discardOutputPath() const;
-
-    std::string to_string(const Store & store) const;
-    static SingleBuiltPathBuilt parse(const Store & store, std::string_view, std::string_view);
-    kj::Promise<Result<JSON>> toJSON(const Store & store) const;
-
-    DECLARE_CMP(SingleBuiltPathBuilt);
-};
-
-namespace built_path::detail {
-using SingleBuiltPathRaw = std::variant<
-    DerivedPathOpaque,
-    SingleBuiltPathBuilt
->;
-}
-
-struct SingleBuiltPath : built_path::detail::SingleBuiltPathRaw {
-    using Raw = built_path::detail::SingleBuiltPathRaw;
-    using Raw::Raw;
-
-    using Opaque = DerivedPathOpaque;
-    using Built = SingleBuiltPathBuilt;
-
-    inline const Raw & raw() const {
-        return static_cast<const Raw &>(*this);
-    }
-
-    StorePath outPath() const;
-
-    SingleDerivedPath discardOutputPath() const;
-
-    static SingleBuiltPath parse(const Store & store, std::string_view);
-    kj::Promise<Result<JSON>> toJSON(const Store & store) const;
-};
-
-static inline ref<SingleBuiltPath> staticDrv(StorePath drvPath)
-{
-    return make_ref<SingleBuiltPath>(SingleBuiltPath::Opaque { drvPath });
-}
-
 /**
  * A built derived path with hints in the form of optional concrete output paths.
  *
  * See 'BuiltPath' for more an explanation.
  */
 struct BuiltPathBuilt {
-    ref<SingleBuiltPath> drvPath;
+    DerivedPathOpaque drvPath;
     std::map<std::string, StorePath> outputs;
 
-    std::string to_string(const Store & store) const;
-    static BuiltPathBuilt parse(const Store & store, std::string_view, std::string_view);
     kj::Promise<Result<JSON>> toJSON(const Store & store) const;
 
     DECLARE_CMP(BuiltPathBuilt);
@@ -93,8 +45,6 @@ struct BuiltPath : built_path::detail::BuiltPathRaw {
 
     StorePathSet outPaths() const;
     kj::Promise<Result<RealisedPath::Set>> toRealisedPaths(Store & store) const;
-
-    kj::Promise<Result<JSON>> toJSON(const Store & store) const;
 };
 
 typedef std::vector<BuiltPath> BuiltPaths;

lix/libcmd/common-eval-args.cc

@@ -9,6 +9,7 @@
 #include "lix/libstore/store-api.hh"
 #include "lix/libcmd/command.hh"
 #include "lix/libutil/async.hh"
+#include "lix/libutil/error.hh"
 #include "lix/libutil/regex.hh"
 
 #include <regex>
@@ -16,31 +17,36 @@
 namespace nix {
 
 static std::regex const identifierRegex = regex::parse("^[A-Za-z_][A-Za-z0-9_'-]*$");
-static void warnInvalidNixIdentifier(const std::string & name)
+static void checkValidNixIdentifier(const std::string & name)
 {
     std::smatch match;
     if (!std::regex_match(name, match, identifierRegex)) {
-        warn("This Nix invocation specifies a value for argument '%s' which isn't a valid \
+        throw UsageError(
-Nix identifier. The project is considering to drop support for this \
+            "This invocation specifies a value for argument '%s' "
-or to require quotes around args that aren't valid Nix identifiers. \
+            "which isn't a valid Nix identifier. "
-If you depend on this behvior, please reach out in \
+            "The project is dropping support for this so that it's possible to make e.g. "
-https://git.lix.systems/lix-project/lix/issues/496 so we can discuss \
+            "'%s' evaluating to '%s' in the future. "
-your use-case.", name);
+            "If you depend on this behavior, please reach out in "
+            "<https://git.lix.systems/lix-project/lix/issues/496> so we can discuss your use-case.",
+            name,
+            "--arg config.allowUnfree true",
+            "{ config.allowUnfree = true; }"
+        );
     }
 }
 
 MixEvalArgs::MixEvalArgs()
 {
-    addFlag({
+    addFlag(
-        .longName = "arg",
+        {.longName = "arg",
-        .description = "Pass the value *expr* as the argument *name* to Nix functions.",
+         .description = "Pass the value *expr* as the argument *name* to Nix functions.",
-        .category = category,
+         .category = category,
-        .labels = {"name", "expr"},
+         .labels = {"name", "expr"},
-        .handler = {[&](std::string name, std::string expr) {
+         .handler = {[&](std::string name, std::string expr) {
-            warnInvalidNixIdentifier(name);
+             checkValidNixIdentifier(name);
-            autoArgs[name] = 'E' + expr;
+             autoArgs[name] = 'E' + expr;
-        }}
+         }}}
-    });
+    );
 
     addFlag({
         .longName = "argstr",
@@ -48,7 +54,7 @@ MixEvalArgs::MixEvalArgs()
         .category = category,
         .labels = {"name", "string"},
         .handler = {[&](std::string name, std::string s) {
-            warnInvalidNixIdentifier(name);
+            checkValidNixIdentifier(name);
             autoArgs[name] = 'S' + s;
         }},
     });

lix/libcmd/installable-attr-path.cc

@@ -76,7 +76,7 @@ DerivedPathsWithInfo InstallableAttrPath::toDerivedPaths(EvalState & state)
     for (auto & [drvPath, outputs] : byDrvPath)
         res.push_back({
             .path = DerivedPath::Built {
-                .drvPath = makeConstantStorePathRef(drvPath),
+                .drvPath = makeConstantStorePath(drvPath),
                 .outputs = outputs,
             },
             .info = make_ref<ExtraPathInfoValue>(ExtraPathInfoValue::Value {

lix/libcmd/installable-derived-path.cc

@@ -35,7 +35,7 @@ InstallableDerivedPath InstallableDerivedPath::parse(
             // Remove this prior to stabilizing the new CLI.
             if (storePath.isDerivation()) {
                 auto oldDerivedPath = DerivedPath::Built {
-                    .drvPath = makeConstantStorePathRef(storePath),
+                    .drvPath = makeConstantStorePath(storePath),
                     .outputs = OutputsSpec::All { },
                 };
                 warn(
@@ -48,8 +48,7 @@ InstallableDerivedPath InstallableDerivedPath::parse(
         },
         // If the user did use ^, we just do exactly what is written.
         [&](const ExtendedOutputsSpec::Explicit & outputSpec) -> DerivedPath {
-            auto drv = make_ref<SingleDerivedPath>(SingleDerivedPath::parse(*store, prefix));
+            auto drv = DerivedPathOpaque::parse(*store, prefix);
-            drvRequireExperiment(*drv);
             return DerivedPath::Built {
                 .drvPath = std::move(drv),
                 .outputs = outputSpec,

lix/libcmd/installable-flake.cc

@@ -98,7 +98,7 @@ DerivedPathsWithInfo InstallableFlake::toDerivedPaths(EvalState & state)
 
     return {{
         .path = DerivedPath::Built {
-            .drvPath = makeConstantStorePathRef(std::move(drvPath)),
+            .drvPath = makeConstantStorePath(std::move(drvPath)),
             .outputs = std::visit(overloaded {
                 [&](const ExtendedOutputsSpec::Default & d) -> OutputsSpec {
                     std::set<std::string> outputsToInstall;

lix/libcmd/installables.cc

@@ -524,36 +524,6 @@ ref<Installable> SourceExprCommand::parseInstallable(
     return installables.front();
 }
 
-static kj::Promise<Result<SingleBuiltPath>> getBuiltPath(ref<Store> evalStore, ref<Store> store, const SingleDerivedPath & b)
-try {
-    auto handlers = overloaded{
-        [&](const SingleDerivedPath::Opaque & bo) -> kj::Promise<Result<SingleBuiltPath>> {
-            return {SingleBuiltPath::Opaque { bo.path }};
-        },
-        // NOLINTNEXTLINE(cppcoreguidelines-avoid-capturing-lambda-coroutines)
-        [&](const SingleDerivedPath::Built & bfd) -> kj::Promise<Result<SingleBuiltPath>> {
-            try {
-                auto drvPath = TRY_AWAIT(getBuiltPath(evalStore, store, *bfd.drvPath));
-                // Resolving this instead of `bfd` will yield the same result, but avoid duplicative work.
-                SingleDerivedPath::Built truncatedBfd {
-                    .drvPath = makeConstantStorePathRef(drvPath.outPath()),
-                    .output = bfd.output,
-                };
-                auto outputPath = TRY_AWAIT(resolveDerivedPath(*store, truncatedBfd, &*evalStore));
-                co_return SingleBuiltPath::Built {
-                    .drvPath = make_ref<SingleBuiltPath>(std::move(drvPath)),
-                    .output = { bfd.output, outputPath },
-                };
-            } catch (...) {
-                co_return result::current_exception();
-            }
-        },
-    };
-    co_return TRY_AWAIT(std::visit(handlers, b.raw()));
-} catch (...) {
-    co_return result::current_exception();
-}
-
 std::vector<BuiltPathWithResult> Installable::build(
     EvalState & state,
     ref<Store> evalStore,
@@ -642,7 +612,7 @@ std::vector<std::pair<ref<Installable>, BuiltPathWithResult>> Installable::build
                             state.aio.blockOn(resolveDerivedPath(*store, bfd, &*evalStore));
                         res.push_back({aux.installable, {
                             .path = BuiltPath::Built {
-                                .drvPath = make_ref<SingleBuiltPath>(state.aio.blockOn(getBuiltPath(evalStore, store, *bfd.drvPath))),
+                                .drvPath = bfd.drvPath,
                                 .outputs = outputs,
                              },
                             .info = aux.info}});
@@ -674,7 +644,7 @@ std::vector<std::pair<ref<Installable>, BuiltPathWithResult>> Installable::build
                             outputs.emplace(outputName, realisation.outPath);
                         res.push_back({aux.installable, {
                             .path = BuiltPath::Built {
-                                .drvPath = make_ref<SingleBuiltPath>(state.aio.blockOn(getBuiltPath(evalStore, store, *bfd.drvPath))),
+                                .drvPath = bfd.drvPath,
                                 .outputs = outputs,
                             },
                             .info = aux.info,
@@ -789,7 +759,7 @@ StorePathSet Installable::toDerivations(
                         : throw Error("argument '%s' did not evaluate to a derivation", i->what()));
                 },
                 [&](const DerivedPath::Built & bfd) {
-                    drvPaths.insert(state.aio.blockOn(resolveDerivedPath(*store, *bfd.drvPath)));
+                    drvPaths.insert(bfd.drvPath.path);
                 },
             }, b.path.raw());
 

lix/libcmd/markdown.cc

@@ -3,12 +3,52 @@
 #include "lix/libutil/finally.hh"
 #include "lix/libutil/terminal.hh"
 
+#include <cstdlib>
+#include <iterator>
+#include <new>
+#include <regex>
 #include <sys/queue.h>
 #include <lowdown.h>
 
 namespace nix {
 
-std::string renderMarkdownToTerminal(std::string_view markdown)
+static const std::string DOCROOT = "@docroot@";
+static const std::string DOCROOT_URL = "https://docs.lix.systems/manual/lix/stable";
+
+static void processLinks(struct lowdown_node * node)
+{
+    if (node->type == LOWDOWN_LINK) {
+        struct lowdown_buf *link = &node->rndr_link.link;
+        if (link && link->size && std::string_view(link->data, link->size).starts_with(DOCROOT)) {
+            // link starts with @docroot@, replace that and check the path extension too.
+            static std::regex mdRewrite{"\\.md(#.*)?$"}; // NOLINT(lix-foreign-exceptions)
+
+            auto oldLink = std::string_view(link->data, link->size).substr(DOCROOT.size());
+            std::string newLink = DOCROOT_URL;
+            std::regex_replace(
+                std::back_inserter(newLink), oldLink.begin(), oldLink.end(), mdRewrite, ".html$1"
+            );
+            if (link->maxsize < newLink.size()) {
+                // the existing link buffer doesn't have enough space for the new string
+                char *newData;
+                if (!(newData = static_cast<char *>(std::realloc(link->data, newLink.size())))) {
+                    throw std::bad_alloc();
+                }
+                link->data = newData;
+                link->maxsize = newLink.size();
+            }
+            newLink.copy(link->data, newLink.size());
+            link->size = newLink.size();
+        }
+    } else {
+        // recurse into children
+        struct lowdown_node *child;
+        TAILQ_FOREACH(child, &node->children, entries)
+            processLinks(child);
+    }
+}
+
+std::string renderMarkdownToTerminal(std::string_view markdown, StandardOutputStream fileno)
 {
     int windowWidth = getWindowSize().second;
 
@@ -21,7 +61,7 @@ std::string renderMarkdownToTerminal(std::string_view markdown)
         .feat = LOWDOWN_COMMONMARK | LOWDOWN_FENCED | LOWDOWN_DEFLIST | LOWDOWN_TABLES,
         .oflags = LOWDOWN_TERM_NOLINK,
     };
-    if (!shouldANSI()) {
+    if (!shouldANSI(fileno)) {
         opts.oflags |= LOWDOWN_TERM_NOANSI;
     }
 
@@ -36,6 +76,8 @@ std::string renderMarkdownToTerminal(std::string_view markdown)
         throw Error("cannot parse Markdown document");
     Finally freeNode([&]() { lowdown_node_free(node); });
 
+    processLinks(node);
+
     auto renderer = lowdown_term_new(&opts);
     if (!renderer)
         throw Error("cannot allocate Markdown renderer");

lix/libcmd/markdown.hh

@@ -1,10 +1,11 @@
 #pragma once
 ///@file
 
+#include "lix/libutil/terminal.hh"
 #include "lix/libutil/types.hh"
 
 namespace nix {
 
-std::string renderMarkdownToTerminal(std::string_view markdown);
+std::string renderMarkdownToTerminal(std::string_view markdown, StandardOutputStream fileno = StandardOutputStream::Stdout);
 
 }

lix/libcmd/repl.cc

@@ -3,7 +3,6 @@
 #include <iostream>
 #include <cstdlib>
 #include <cstring>
-#include <climits>
 #include <string_view>
 
 #include "lix/libutil/box_ptr.hh"
@@ -254,7 +253,7 @@ void runNix(Path program, const Strings & args)
         .program = settings.nixBinDir+ "/" + program,
         .args = args,
         .environment = subprocessEnv,
-    }).wait();
+    }).waitAndCheck();
 
     return;
 }
@@ -672,7 +671,7 @@ ProcessLineResult NixRepl::processLine(std::string line)
 
         // runProgram redirects stdout to a StringSink,
         // using runProgram2 to allow editors to display their UI
-        runProgram2(RunOptions { .program = editor, .searchPath = true, .args = args }).wait();
+        runProgram2(RunOptions { .program = editor, .searchPath = true, .args = args }).waitAndCheck();
 
         // Reload right after exiting the editor if path is not in store
         // Store is immutable, so there could be no changes, so there's no need to reload
@@ -760,7 +759,7 @@ ProcessLineResult NixRepl::processLine(std::string line)
 
             state.aio.blockOn(evaluator.store->buildPaths({
                 DerivedPath::Built {
-                    .drvPath = makeConstantStorePathRef(drvPath),
+                    .drvPath = makeConstantStorePath(drvPath),
                     .outputs = OutputsSpec::All { },
                 },
             }));
@@ -1008,7 +1007,15 @@ Value * NixRepl::replOverlays()
     for (auto path : evalSettings.replOverlays.get()) {
         debug("Loading '%1%' path '%2%'...", "repl-overlays", path);
         SourcePath sourcePath((CanonPath(path)));
+
+        // XXX(jade): This is a somewhat unsatisfying solution to
+        // https://git.lix.systems/lix-project/lix/issues/777 which means that
+        // the top level item in the repl-overlays file (that is, the lambda)
+        // gets evaluated with pure eval off. This means that if you want to do
+        // impure eval stuff, you will have to force it with builtins.seq.
+        bool prevPureEval = evalSettings.pureEval.get();
         auto replInit = evalFile(sourcePath);
+        evalSettings.pureEval.setDefault(prevPureEval);
 
         if (!replInit->isLambda()) {
             evaluator.errors.make<TypeError>(
@@ -1056,19 +1063,24 @@ template<typename T, typename NameFn, typename ValueFn>
 void NixRepl::addToScope(T && things, NameFn nameFn, ValueFn valueFn)
 {
     size_t added = 0;
-    for (auto && thing : things) {
-        if (displ + 1 >= envSize)
-            throw Error("environment full; cannot add more variables");
 
-        const auto name = nameFn(thing);
+    staticEnv->vars.unsafe_insert_bulk([&] (auto & map) {
-        staticEnv->vars.emplace_back(name, displ);
+        auto oldSize = map.size();
-        env->values[displ++] = valueFn(thing);
+        for (auto && thing : things) {
-        varNames.emplace(evaluator.symbols[name]);
+            if (displ + 1 >= envSize)
-        added++;
+                throw Error("environment full; cannot add more variables");
-    }
+
+            const auto name = nameFn(thing);
+            map.emplace_back(name, displ);
+            env->values[displ++] = valueFn(thing);
+            varNames.emplace(evaluator.symbols[name]);
+            added++;
+        }
+        // safety: we sort the range that we inserted so that we don't have to push that
+        // invariant up to the caller
+        std::sort(map.begin() + oldSize, map.end());
+    });
 
-    staticEnv->sort();
-    staticEnv->deduplicate();
     if (added > 0) {
         notice("Added %1% variables.", added);
     }
@@ -1093,14 +1105,11 @@ void NixRepl::addVarToScope(const Symbol name, Value & v)
 {
     if (displ >= envSize)
         throw Error("environment full; cannot add more variables");
-    if (auto oldVar = staticEnv->find(name); oldVar != staticEnv->vars.end()) {
+    if (staticEnv->vars.insert_or_assign(name, displ).second) {
-        staticEnv->vars.erase(oldVar);
         notice("Updated %s.", evaluator.symbols[name]);
     } else {
         notice("Added %s.", evaluator.symbols[name]);
     }
-    staticEnv->vars.emplace_back(name, displ);
-    staticEnv->sort();
     env->values[displ++] = &v;
     varNames.emplace(evaluator.symbols[name]);
 }

lix/libexpr/builtins/fetchGit.md

@@ -48,6 +48,69 @@ attribute with the following attributes (all except `url` optional):
   With this argument being true, it's possible to load a `rev` from *any* `ref`
   (by default only `rev`s from the specified `ref` are supported).
 
+- `narHash`
+
+  If given, the source is first looked-up in the Nix store and the [substituters](@docroot@/command-ref/conf-file.md#conf-substituters), and only fetched if not available.
+
+The return value is an attrset containing the following keys:
+
+- `lastModified` (`integer`)
+
+  Unix timestamp of the last update.
+  This corresponds to the timestamp of the "committer" timestamp embedded in the fetched commit.
+
+- `lastModifiedDate` (`string`)
+
+  Textual representation of the `lastModified` timestamp in UTC (the timezone embedded in the git commit is discarded).
+
+- `outPath` (`string`)
+
+  Resulting store path of the fetch process.
+
+- `narHash` (`string`)
+
+  SRI representation of the hash of the `outPath`.
+
+- `rev` (`string`)
+
+  The full-length revision fetched from the remote.
+  For further information see the `rev` input parameter.
+  This will usually be the output of `git rev-parse <rev>` (or `ref` when no `rev` is provided as an input parameter).
+
+- `revCount` (`integer`)
+
+  Number of revisions in the history of the revision fetched.
+  For a repository with a single commit (the root) this number equals 1.
+  Fetches of shallow repositories report a value of 0.
+
+- `shortRev` (`string`)
+
+  A short representation of the `rev`.
+  This string is a *truncated* version of the `rev`.
+  It is of fixed length and therefore not guaranteed to be unique (unlike the output of `git rev-parse --short`).
+  Future versions of Lix may change the length of this string only as part of a breaking change.
+  For maximum reproducibility and interoperability it is recommended to not rely on this value and to truncate the returned `rev` to an appropriate value instead.
+
+- `submodules` (`boolean`)
+
+  Indicates whether submodules have been fetched.
+  If this value is set to `true`, any submodules are already checked out in the resulting `outPath`.
+
+A full example of the output:
+
+```nix
+{
+  lastModified = 1746827286;
+  lastModifiedDate = "20250509214806";
+  narHash = "sha256-qCRBy8Bbh5XhPalPkhonxNgfsbw3lP0UIXBLSrhxAvI=";
+  outPath = "/nix/store/2qdnzhzccspwm70mni7jkvrfkpwcb3jn-source";
+  rev = "dcb0a97000d50b2868ed4f8d9fd465c5a5b8eb3a";
+  revCount = 17845;
+  shortRev = "dcb0a97";
+  submodules = false;
+}
+```
+
 Here are some examples of how to use `fetchGit`.
 
   - To fetch a private repository over SSH:

lix/libexpr/builtins/outputOf.md

@@ -1,23 +0,0 @@
----
-name: outputOf
-args: [derivation-reference, output-name]
-experimentalFeature: dynamic-derivations
----
-Return the output path of a derivation, literally or using a placeholder if needed.
-
-If the derivation has a statically-known output path (i.e. the derivation output is input-addressed, or fixed content-addressed), the output path will just be returned.
-But if the derivation is content-addressed or if the derivation is itself not-statically produced (i.e. is the output of another derivation), a placeholder will be returned instead.
-
-*`derivation reference`* must be a string that may contain a regular store path to a derivation, or may be a placeholder reference. If the derivation is produced by a derivation, you must explicitly select `drv.outPath`.
-This primop can be chained arbitrarily deeply.
-For instance,
-
-```nix
-builtins.outputOf
-  (builtins.outputOf myDrv "out)
-  "out"
-```
-
-will return a placeholder for the output of the output of `myDrv`.
-
-This primop corresponds to the `^` sigil for derivable paths, e.g. as part of installable syntax on the command line.

lix/libexpr/eval-cache.cc

@@ -580,7 +580,7 @@ string_t AttrCursor::getStringWithContext(EvalState & state)
                             return d.drvPath;
                         },
                         [&](const NixStringContextElem::Built & b) -> const StorePath & {
-                            return b.drvPath->getBaseStorePath();
+                            return b.drvPath.path;
                         },
                         [&](const NixStringContextElem::Opaque & o) -> const StorePath & {
                             return o.path;

lix/libexpr/eval-inline.hh

@@ -73,7 +73,6 @@ void EvalState::forceValue(Value & v, const PosIdx pos)
         Expr & expr = *v.thunk.expr;
         try {
             v.mkBlackhole();
-            //checkInterrupt();
             expr.eval(*this, *env, v);
         } catch (...) {
             v.mkThunk(env, expr);

lix/libexpr/eval.cc

@@ -14,7 +14,6 @@
 #include "lix/libutil/types.hh"
 #include "lix/libstore/store-api.hh"
 #include "lix/libstore/derivations.hh"
-#include "lix/libstore/downstream-placeholder.hh"
 #include "lix/libexpr/gc-alloc.hh"
 #include "lix/libstore/filetransfer.hh"
 #include "lix/libexpr/function-trace.hh"
@@ -549,7 +548,7 @@ void EvalBuiltins::addConstant(const std::string & name, Value * v, Constant inf
             assert(info.type == gotType);
 
         /* Install value the base environment. */
-        staticEnv->vars.emplace_back(symbols.create(name), baseEnvDispl);
+        staticEnv->vars.insert_or_assign(symbols.create(name), baseEnvDispl);
         env.values[baseEnvDispl++] = v;
         env.values[0]->attrs->push_back(Attr(symbols.create(name2), v));
     }
@@ -584,7 +583,7 @@ Value * EvalBuiltins::addPrimOp(PrimOp && primOp)
 
     Value * v = mem.allocValue();
     v->mkPrimOp(new PrimOp(primOp));
-    staticEnv->vars.emplace_back(envName, baseEnvDispl);
+    staticEnv->vars.insert_or_assign(auto(envName), baseEnvDispl);
     env.values[baseEnvDispl++] = v;
     env.values[0]->attrs->push_back(Attr(symbols.create(primOp.name), v));
     return v;
@@ -888,29 +887,18 @@ void EvalPaths::mkStorePathString(const StorePath & p, Value & v)
 
 
 std::string EvalState::mkOutputStringRaw(
-    const SingleDerivedPath::Built & b,
+    const StorePath & staticOutputPath)
-    std::optional<StorePath> optStaticOutputPath,
-    const ExperimentalFeatureSettings & xpSettings)
 {
-    /* In practice, this is testing for the case of CA derivations, or
+    return ctx.store->printStorePath(staticOutputPath);
-       dynamic derivations. */
-    return optStaticOutputPath
-        ? ctx.store->printStorePath(std::move(*optStaticOutputPath))
-        /* Downstream we would substitute this for an actual path once
-           we build the floating CA derivation */
-        : DownstreamPlaceholder::fromSingleDerivedPathBuilt(b, xpSettings).render();
 }
 
 
 void EvalState::mkOutputString(
     Value & value,
     const SingleDerivedPath::Built & b,
-    std::optional<StorePath> optStaticOutputPath,
+    const StorePath & staticOutputPath)
-    const ExperimentalFeatureSettings & xpSettings)
 {
-    value.mkString(
+    value.mkString(mkOutputStringRaw(staticOutputPath), NixStringContext { b });
-        mkOutputStringRaw(b, optStaticOutputPath, xpSettings),
-        NixStringContext { b });
 }
 
 
@@ -922,19 +910,12 @@ std::string EvalState::mkSingleDerivedPathStringRaw(
             return ctx.store->printStorePath(o.path);
         },
         [&](const SingleDerivedPath::Built & b) {
-            auto optStaticOutputPath = std::visit(overloaded {
+            auto drv = aio.blockOn(ctx.store->readDerivation(b.drvPath.path));
-                [&](const SingleDerivedPath::Opaque & o) {
+            auto i = drv.outputs.find(b.output);
-                    auto drv = aio.blockOn(ctx.store->readDerivation(o.path));
+            if (i == drv.outputs.end())
-                    auto i = drv.outputs.find(b.output);
+                throw Error("derivation '%s' does not have output '%s'", b.drvPath.to_string(*ctx.store), b.output);
-                    if (i == drv.outputs.end())
+            auto staticOutputPath = i->second.path(*ctx.store, drv.name, b.output);
-                        throw Error("derivation '%s' does not have output '%s'", b.drvPath->to_string(*ctx.store), b.output);
+            return mkOutputStringRaw(staticOutputPath);
-                    return i->second.path(*ctx.store, drv.name, b.output);
-                },
-                [&](const SingleDerivedPath::Built & o) -> std::optional<StorePath> {
-                    return std::nullopt;
-                },
-            }, b.drvPath->raw());
-            return mkOutputStringRaw(b, optStaticOutputPath);
         }
     }, p.raw());
 }
@@ -1973,7 +1954,7 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
     NixInt n{0};
     NixFloat nf = 0;
 
-    bool first = !forceString;
+    bool first = !isInterpolation;
     ValueType firstType = nString;
 
     const auto str = [&] {
@@ -2036,12 +2017,17 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
                 state.ctx.errors.make<EvalError>("cannot add %1% to a float", showType(vTmp)).atPos(i_pos).withFrame(env, *this).debugThrow();
         } else {
             if (s.empty()) s.reserve(es.size());
+
+            /* If we are coercing inside of an interpolation, we may allow slightly more comfort by coercing things like integers. */
+            auto coercionMode = isInterpolation && featureSettings.isEnabled(Xp::CoerceIntegers)
+                ? StringCoercionMode::Interpolation : StringCoercionMode::Strict;
+
             /* skip canonization of first path, which would only be not
             canonized in the first place if it's coming from a ./${foo} type
             path */
             auto part = state.coerceToString(i_pos, vTmp, context,
                                              "while evaluating a path segment",
-                                             false, firstType == nString, !first);
+                                             coercionMode, firstType == nString, !first);
             sSize += part->size();
             s.emplace_back(std::move(part));
         }
@@ -2274,7 +2260,7 @@ bool EvalState::isDerivation(Value & v)
 
 
 std::optional<std::string> EvalState::tryAttrsToString(const PosIdx pos, Value & v,
-    NixStringContext & context, bool coerceMore, bool copyToStore)
+    NixStringContext & context, StringCoercionMode mode, bool copyToStore)
 {
     auto i = v.attrs->find(ctx.s.toString);
     if (i != v.attrs->end()) {
@@ -2283,7 +2269,7 @@ std::optional<std::string> EvalState::tryAttrsToString(const PosIdx pos, Value &
             callFunction(*i->value, v, v1, i->pos);
             return coerceToString(pos, v1, context,
                     "while evaluating the result of the `__toString` attribute",
-                    coerceMore, copyToStore).toOwned();
+                    mode, copyToStore).toOwned();
         } catch (EvalError & e) {
             e.addTrace(ctx.positions[pos], "while converting a set to string");
             throw;
@@ -2298,7 +2284,7 @@ BackedStringView EvalState::coerceToString(
     Value & v,
     NixStringContext & context,
     std::string_view errorCtx,
-    bool coerceMore,
+    StringCoercionMode mode,
     bool copyToStore,
     bool canonicalizePath)
 {
@@ -2322,7 +2308,7 @@ BackedStringView EvalState::coerceToString(
     }
 
     if (v.type() == nAttrs) {
-        auto maybeString = tryAttrsToString(pos, v, context, coerceMore, copyToStore);
+        auto maybeString = tryAttrsToString(pos, v, context, mode, copyToStore);
         if (maybeString)
             return std::move(*maybeString);
         auto i = v.attrs->find(ctx.s.outPath);
@@ -2336,24 +2322,29 @@ BackedStringView EvalState::coerceToString(
                 .debugThrow();
         }
         return coerceToString(pos, *i->value, context, errorCtx,
-                              coerceMore, copyToStore, canonicalizePath);
+                              mode, copyToStore, canonicalizePath);
     }
 
     if (v.type() == nExternal) {
         try {
-            return v.external->coerceToString(*this, pos, context, coerceMore, copyToStore);
+            return v.external->coerceToString(*this, pos, context, mode, copyToStore);
         } catch (Error & e) {
             e.addTrace(nullptr, errorCtx);
             throw;
         }
     }
 
-    if (coerceMore) {
+    /* Raito: Any addition to this mode is subject to extra scrutiny
+     * until we have better formatting tools. */
+    if (mode >= StringCoercionMode::Interpolation) {
+        if (v.type() == nInt) return std::to_string(v.integer.value);
+    }
+
+    if (mode >= StringCoercionMode::ToString) {
         /* Note that `false' is represented as an empty string for
            shell scripting convenience, just like `null'. */
         if (v.type() == nBool && v.boolean) return "1";
         if (v.type() == nBool && !v.boolean) return "";
-        if (v.type() == nInt) return std::to_string(v.integer.value);
         if (v.type() == nFloat) return std::to_string(v.fpoint);
         if (v.type() == nNull) return "";
 
@@ -2363,7 +2354,7 @@ BackedStringView EvalState::coerceToString(
                 try {
                     result += *coerceToString(pos, *v2, context,
                             "while evaluating one element of the list",
-                            coerceMore, copyToStore, canonicalizePath);
+                            mode, copyToStore, canonicalizePath);
                 } catch (Error & e) {
                     e.addTrace(ctx.positions[pos], errorCtx);
                     throw;
@@ -2420,7 +2411,7 @@ try {
 
 SourcePath EvalState::coerceToPath(const PosIdx pos, Value & v, NixStringContext & context, std::string_view errorCtx)
 {
-    auto path = coerceToString(pos, v, context, errorCtx, false, false, true).toOwned();
+    auto path = coerceToString(pos, v, context, errorCtx, StringCoercionMode::Strict, false, true).toOwned();
     if (path == "" || path[0] != '/')
         ctx.errors.make<EvalError>("string '%1%' doesn't represent an absolute path", path).withTrace(pos, errorCtx).debugThrow();
     return CanonPath(path);
@@ -2429,7 +2420,7 @@ SourcePath EvalState::coerceToPath(const PosIdx pos, Value & v, NixStringContext
 
 StorePath EvalState::coerceToStorePath(const PosIdx pos, Value & v, NixStringContext & context, std::string_view errorCtx)
 {
-    auto path = coerceToString(pos, v, context, errorCtx, false, false, true).toOwned();
+    auto path = coerceToString(pos, v, context, errorCtx, StringCoercionMode::Strict, false, true).toOwned();
     if (auto storePath = ctx.store->maybeParseStorePath(path))
         return *storePath;
     ctx.errors.make<EvalError>("path '%1%' is not in the Nix store", path).withTrace(pos, errorCtx).debugThrow();
@@ -2484,7 +2475,7 @@ SingleDerivedPath EvalState::coerceToSingleDerivedPath(const PosIdx pos, Value &
             [&](const SingleDerivedPath::Built & b) {
                 ctx.errors.make<EvalError>(
                     "string '%s' has context with the output '%s' from derivation '%s', but the string is not the right placeholder for this derivation output. It should be '%s'",
-                    s, b.output, b.drvPath->to_string(*ctx.store), sExpected)
+                    s, b.output, b.drvPath.to_string(*ctx.store), sExpected)
                     .withTrace(pos, errorCtx).debugThrow(always_progresses);
             }
         }, derivedPath.raw());
@@ -2891,7 +2882,7 @@ try {
 }
 
 
-std::string ExternalValueBase::coerceToString(EvalState & state, const PosIdx & pos, NixStringContext & context, bool copyMore, bool copyToStore) const
+std::string ExternalValueBase::coerceToString(EvalState & state, const PosIdx & pos, NixStringContext & context, StringCoercionMode mode, bool copyToStore) const
 {
     state.ctx.errors.make<TypeError>(
         "cannot coerce %1% to a string: %2%", showType(), *this

lix/libexpr/eval.hh

@@ -767,19 +767,18 @@ public:
     bool isDerivation(Value & v);
 
     std::optional<std::string> tryAttrsToString(const PosIdx pos, Value & v,
-        NixStringContext & context, bool coerceMore = false, bool copyToStore = true);
+        NixStringContext & context, StringCoercionMode mode = StringCoercionMode::Strict, bool copyToStore = true);
 
     /**
      * String coercion.
      *
      * Converts strings, paths and derivations to a
-     * string.  If `coerceMore` is set, also converts nulls, integers,
+     * string. If `copyToStore` is set,
-     * booleans and lists to a string.  If `copyToStore` is set,
      * referenced paths are copied to the Nix store as a side effect.
      */
     BackedStringView coerceToString(const PosIdx pos, Value & v, NixStringContext & context,
         std::string_view errorCtx,
-        bool coerceMore = false, bool copyToStore = true,
+        StringCoercionMode mode = StringCoercionMode::Strict, bool copyToStore = true,
         bool canonicalizePath = true);
 
     /**
@@ -804,8 +803,7 @@ public:
     /**
      * Coerce to `SingleDerivedPath`.
      *
-     * Must be a string which is either a literal store path or a
+     * Must be a string which is either a literal store path.
-     * "placeholder (see `DownstreamPlaceholder`).
      *
      * Even more importantly, the string context must be exactly one
      * element, which is either a `NixStringContextElem::Opaque` or
@@ -870,19 +868,13 @@ public:
      * @param b the drv whose output we are making a string for, and the
      * output
      *
-     * @param optStaticOutputPath Optional output path for that string.
+     * @param staticOutputPath Output path for that string.
-     * Must be passed if and only if output store object is
+     * Will be printed to form string.
-     * input-addressed or fixed output. Will be printed to form string
-     * if passed, otherwise a placeholder will be used (see
-     * `DownstreamPlaceholder`).
-     *
-     * @param xpSettings Stop-gap to avoid globals during unit tests.
      */
     void mkOutputString(
         Value & value,
         const SingleDerivedPath::Built & b,
-        std::optional<StorePath> optStaticOutputPath,
+        const StorePath & staticOutputPath);
-        const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
 
     /**
      * Create a string representing a `SingleDerivedPath`.
@@ -902,9 +894,7 @@ private:
      * string Value, which would also have a string context.
      */
     std::string mkOutputStringRaw(
-        const SingleDerivedPath::Built & b,
+        const StorePath & staticOutputPath);
-        std::optional<StorePath> optStaticOutputPath,
-        const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
 
     /**
      * Like `mkSingleDerivedPathStringRaw` but just creates a raw string

lix/libexpr/fetchurl.nix

@@ -34,6 +34,7 @@
   executable ? false,
   unpack ? false,
   name ? baseNameOf (toString url),
+  # still translates to __impure to trigger derivationStrict error checks.
   impure ? false,
 }:
 

lix/libexpr/flake/flake.cc

@@ -298,7 +298,7 @@ static Flake getFlake(
                 NixStringContext emptyContext = {};
                 flake.config.settings.emplace(
                     state.ctx.symbols[setting.name],
-                    state.coerceToString(setting.pos, *setting.value, emptyContext, "", false, true, true) .toOwned());
+                    state.coerceToString(setting.pos, *setting.value, emptyContext, "", StringCoercionMode::Strict, true, true) .toOwned());
             }
             else if (setting.value->type() == nInt)
                 flake.config.settings.emplace(

lix/libexpr/json-to-value.cc

@@ -7,6 +7,11 @@
 
 namespace nix {
 
+/*
+ * Used for `JSONObjectState`
+ */
+using ValueMap = GcMap<Symbol, Value *>;
+
 // for more information, refer to
 // https://github.com/nlohmann/json/blob/master/include/nlohmann/detail/input/json_sax.hpp
 class JSONSax : nlohmann::json_sax<JSON> {

lix/libexpr/meson.build

@@ -115,7 +115,6 @@ builtin_definitions = files(
   'builtins/mapAttrs.md',
   'builtins/match.md',
   'builtins/mul.md',
-  'builtins/outputOf.md',
   'builtins/parseDrvName.md',
   'builtins/parseFlakeRef.md',
   'builtins/partition.md',

lix/libexpr/nixexpr.cc

@@ -19,6 +19,12 @@ std::ostream & operator <<(std::ostream & str, const SymbolStr & symbol)
     return printIdentifier(str, s);
 }
 
+std::ostream & operator<<(std::ostream & str, const InternedSymbol & symbol)
+{
+    str << SymbolStr(symbol);
+    return str;
+}
+
 AttrName::AttrName(PosIdx pos, Symbol s) : pos(pos), symbol(s)
 {
 }
@@ -273,7 +279,7 @@ JSON ExprConcatStrings::toJSON(const SymbolTable & symbols) const
         parts.push_back(part->toJSON(symbols));
     return {
         {"_type", "ExprConcatStrings"},
-        {"forceString", forceString},
+        {"isInterpolation", isInterpolation},
         {"es", parts}
     };
 }
@@ -465,7 +471,7 @@ void VarBinder::visit(ExprVar & e, std::unique_ptr<Expr> & ptr)
         if (curEnv->isWith) {
             if (withLevel == -1) withLevel = level;
         } else {
-            auto i = curEnv->find(e.name);
+            auto i = curEnv->vars.find(e.name);
             if (i != curEnv->vars.end()) {
                 if (e.needsRoot && !curEnv->isRoot) {
                     throw ParseError({
@@ -523,9 +529,12 @@ std::shared_ptr<const StaticEnv> ExprAttrs::buildRecursiveEnv(const std::shared_
 {
     auto newEnv = std::make_shared<StaticEnv>(nullptr, env.get(), attrs.size());
 
-    Displacement displ = 0;
+    // safety: the attrs is already sorted
-    for (auto & i : attrs)
+    newEnv->vars.unsafe_insert_bulk([&] (auto & map) {
-        newEnv->vars.emplace_back(i.first, i.second.displ = displ++);
+        Displacement displ = 0;
+        for (auto & i : attrs)
+            map.emplace_back(i.first, i.second.displ = displ++);
+    });
     return newEnv;
 }
 
@@ -662,7 +671,7 @@ void VarBinder::visit(ExprPos & e, std::unique_ptr<Expr> & ptr)
 std::shared_ptr<const StaticEnv> SimplePattern::buildEnv(const StaticEnv * up)
 {
     auto newEnv = std::make_shared<StaticEnv>(nullptr, up, 1);
-    newEnv->vars.emplace_back(name, 0);
+    newEnv->vars.insert_or_assign(name, 0);
     return newEnv;
 }
 
@@ -677,12 +686,14 @@ std::shared_ptr<const StaticEnv> AttrsPattern::buildEnv(const StaticEnv * up)
 
     Displacement displ = 0;
 
-    if (name) newEnv->vars.emplace_back(name, displ++);
+    if (name) newEnv->vars.insert_or_assign(name, displ++);
 
-    for (auto & i : formals)
+    // safety: The formals are already sorted
-        newEnv->vars.emplace_back(i.name, displ++);
+    newEnv->vars.unsafe_insert_bulk([&] (auto & map) {
+        for (auto & i : formals)
+            map.emplace_back(i.name, displ++);
+    });
 
-    newEnv->sort();
     return newEnv;
 }
 

lix/libexpr/nixexpr.hh

@@ -11,6 +11,7 @@
 #include "lix/libexpr/eval-error.hh"
 #include "lix/libexpr/pos-idx.hh"
 #include "lix/libutil/strings.hh"
+#include "lix/libutil/linear-map.hh"
 
 namespace nix {
 
@@ -312,7 +313,7 @@ struct ExprAttrs
         AttrDef() { };
 
         template<typename T>
-        const T & chooseByKind(const T & plain, const T & inherited, const T & inheritedFrom) const
+        T chooseByKind(const T & plain, const T & inherited, const T & inheritedFrom) const
         {
             switch (kind) {
             case Kind::Plain:
@@ -572,10 +573,10 @@ MakeBinOp(ExprOpConcatLists, "++")
 
 struct ExprConcatStrings : Expr
 {
-    bool forceString;
+    bool isInterpolation;
     std::vector<std::pair<PosIdx, std::unique_ptr<Expr>>> es;
-    ExprConcatStrings(const PosIdx & pos, bool forceString, std::vector<std::pair<PosIdx, std::unique_ptr<Expr>>> es)
+    ExprConcatStrings(const PosIdx & pos, bool isInterpolation, std::vector<std::pair<PosIdx, std::unique_ptr<Expr>>> es)
-        : Expr(pos), forceString(forceString), es(std::move(es)) { };
+        : Expr(pos), isInterpolation(isInterpolation), es(std::move(es)) { };
     JSON toJSON(const SymbolTable & symbols) const override;
     void eval(EvalState & state, Env & env, Value & v) override;
     void accept(ExprVisitor & ev, std::unique_ptr<Expr> & ptr) override { ev.visit(*this, ptr); }
@@ -607,9 +608,7 @@ struct StaticEnv
     ExprWith * isWith;
     const StaticEnv * up;
 
-    // Note: these must be in sorted order.
+    LinearMap<Symbol, Displacement> vars;
-    typedef std::vector<std::pair<Symbol, Displacement>> Vars;
-    Vars vars;
 
     /* See ExprVar::needsRoot */
     bool isRoot = false;
@@ -617,31 +616,6 @@ struct StaticEnv
     StaticEnv(ExprWith * isWith, const StaticEnv * up, size_t expectedSize = 0) : isWith(isWith), up(up) {
         vars.reserve(expectedSize);
     };
-
-    void sort()
-    {
-        std::stable_sort(vars.begin(), vars.end(),
-            [](const Vars::value_type & a, const Vars::value_type & b) { return a.first < b.first; });
-    }
-
-    void deduplicate()
-    {
-        auto it = vars.begin(), jt = it, end = vars.end();
-        while (jt != end) {
-            *it = *jt++;
-            while (jt != end && it->first == jt->first) *it = *jt++;
-            it++;
-        }
-        vars.erase(it, end);
-    }
-
-    Vars::const_iterator find(Symbol name) const
-    {
-        Vars::value_type key(name, 0);
-        auto i = std::lower_bound(vars.begin(), vars.end(), key);
-        if (i != vars.end() && i->first == name) return i;
-        return vars.end();
-    }
 };
 
 

lix/libexpr/parser/grammar.hh

@@ -686,10 +686,12 @@ struct nothing : p::nothing<Rule> {
     static_assert(!std::is_base_of_v<semantic, Rule>);
 };
 
-
-
 template<typename Self, typename OpCtx, typename AttrPathT, typename ExprT>
-struct operator_semantics {
+struct operator_semantics
+{
+private:
+    operator_semantics() = default;
+    friend Self;
 public:
     struct has_attr : grammar::v1::op::has_attr {
         AttrPathT path;
@@ -775,5 +777,4 @@ public:
         return popExpr();
     }
 };
-
 }

lix/libexpr/primops.cc

@@ -1,6 +1,5 @@
 #include "lix/libutil/archive.hh"
 #include "lix/libstore/derivations.hh"
-#include "lix/libstore/downstream-placeholder.hh"
 #include "lix/libexpr/eval.hh"
 #include "lix/libexpr/eval-settings.hh"
 #include "lix/libexpr/extra-primops.hh"
@@ -37,6 +36,10 @@
 
 namespace nix {
 
+/*
+ * Used for `builtins.groupBy`
+ */
+using ValueVectorMap = std::map<Symbol, ValueVector>;
 
 /*************************************************************
  * Miscellaneous
@@ -58,7 +61,7 @@ StringMap EvalState::realiseContext(const NixStringContext & context)
                     .drvPath = b.drvPath,
                     .outputs = OutputsSpec::Names { b.output },
                 });
-                return ensureValid(b.drvPath->getBaseStorePath());
+                return ensureValid(b.drvPath.path);
             },
             [&](const NixStringContextElem::Opaque & o) {
                 auto ctxS = ctx.store->printStorePath(o.path);
@@ -93,18 +96,6 @@ StringMap EvalState::realiseContext(const NixStringContext & context)
         auto outputs = aio.blockOn(resolveDerivedPath(*ctx.buildStore, drv, &*ctx.store));
         for (auto & [outputName, outputPath] : outputs) {
             outputsToCopyAndAllow.insert(outputPath);
-
-            /* Get all the output paths corresponding to the placeholders we had */
-            if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) {
-                res.insert_or_assign(
-                    DownstreamPlaceholder::fromSingleDerivedPathBuilt(
-                        SingleDerivedPath::Built {
-                            .drvPath = drv.drvPath,
-                            .output = outputName,
-                        }).render(),
-                    ctx.buildStore->printStorePath(outputPath)
-                );
-            }
         }
     }
 
@@ -166,7 +157,7 @@ static void mkOutputString(
     state.mkOutputString(
         attrs.alloc(o.first),
         SingleDerivedPath::Built {
-            .drvPath = makeConstantStorePathRef(drvPath),
+            .drvPath = makeConstantStorePath(drvPath),
             .output = o.first,
         },
         o.second.path(*state.ctx.store, Derivation::nameFromPath(drvPath), o.first));
@@ -242,14 +233,14 @@ static void import(EvalState & state, Value & vPath, Value * vScope, Value & v)
                 nullptr, state.ctx.builtins.staticEnv.get(), vScope->attrs->size()
             );
 
-            unsigned int displ = 0;
+            staticEnv->vars.unsafe_insert_bulk([&] (auto & map) {
-            for (auto & attr : *vScope->attrs) {
+                unsigned int displ = 0;
-                staticEnv->vars.emplace_back(attr.name, displ);
+                for (auto & attr : *vScope->attrs) {
-                env->values[displ++] = attr.value;
+                    // safety: args[0]->attrs is already sorted.
-            }
+                    map.emplace_back(attr.name, displ);
-
+                    env->values[displ++] = attr.value;
-            // No need to call staticEnv.sort(), because
+                }
-            // args[0]->attrs is already sorted.
+            });
 
             debug("evaluating file '%1%'", path);
             Expr & e = state.ctx.parseExprFromFile(state.ctx.paths.resolveExprPath(path), staticEnv);
@@ -315,13 +306,13 @@ void prim_exec(EvalState & state, Value * * args, Value & v)
     NixStringContext context;
     auto program = state.coerceToString(noPos, *elems[0], context,
             "while evaluating the first element of the argument passed to builtins.exec",
-            false, false).toOwned();
+            StringCoercionMode::Strict, false).toOwned();
     Strings commandArgs;
     for (unsigned int i = 1; i < args[0]->listSize(); ++i) {
         commandArgs.push_back(
                 state.coerceToString(noPos, *elems[i], context,
                         "while evaluating an element of the argument passed to builtins.exec",
-                        false, false).toOwned());
+                        StringCoercionMode::Strict, false).toOwned());
     }
     try {
         auto _ = state.realiseContext(context); // FIXME: Handle CA derivations
@@ -610,7 +601,7 @@ static void prim_addErrorContext(EvalState & state, Value * * args, Value & v)
         NixStringContext context;
         auto message = state.coerceToString(noPos, *args[0], context,
                 "while evaluating the error message passed to builtins.addErrorContext",
-                false, false).toOwned();
+                StringCoercionMode::Strict, false).toOwned();
         e.addTrace(nullptr, HintFmt(message));
         throw;
     }
@@ -810,8 +801,6 @@ drvName, Bindings * attrs, Value & v)
 
     NixStringContext context;
 
-    bool contentAddressed = false;
-    bool isImpure = false;
     std::optional<std::string> outputHash;
     std::string outputHashAlgo;
     std::optional<ContentAddressMethod> ingestionMethod;
@@ -821,16 +810,13 @@ drvName, Bindings * attrs, Value & v)
 
     for (auto & i : attrs->lexicographicOrder(state.ctx.symbols)) {
         if (i->name == state.ctx.s.ignoreNulls) continue;
-        const std::string & key = state.ctx.symbols[i->name];
+        auto & key = state.ctx.symbols[i->name];
         vomit("processing attribute '%1%'", key);
 
         auto handleHashMode = [&](const std::string_view s, NeverAsync = {}) {
             if (s == "recursive") ingestionMethod = FileIngestionMethod::Recursive;
             else if (s == "flat") ingestionMethod = FileIngestionMethod::Flat;
-            else if (s == "text") {
+            else
-                experimentalFeatureSettings.require(Xp::DynamicDerivations);
-                ingestionMethod = TextIngestionMethod {};
-            } else
                 state.ctx.errors.make<EvalError>(
                     "invalid value '%s' for 'outputHashMode' attribute", s
                 ).debugThrow();
@@ -868,13 +854,13 @@ drvName, Bindings * attrs, Value & v)
             }
 
             if (i->name == state.ctx.s.contentAddressed && state.forceBool(*i->value, noPos, context_below)) {
-                contentAddressed = true;
+                state.ctx.errors.make<EvalError>("ca derivations are not supported in Lix")
-                experimentalFeatureSettings.require(Xp::CaDerivations);
+                    .debugThrow();
             }
 
             else if (i->name == state.ctx.s.impure && state.forceBool(*i->value, noPos, context_below)) {
-                isImpure = true;
+                state.ctx.errors.make<EvalError>("impure derivations are not supported in Lix")
-                experimentalFeatureSettings.require(Xp::ImpureDerivations);
+                    .debugThrow();
             }
 
             /* The `args' attribute is special: it supplies the
@@ -884,7 +870,7 @@ drvName, Bindings * attrs, Value & v)
                 for (auto elem : i->value->listItems()) {
                     auto s = state.coerceToString(noPos, *elem, context,
                             "while evaluating an element of the argument list",
-                            true).toOwned();
+                            StringCoercionMode::ToString).toOwned();
                     drv.args.push_back(s);
                 }
             }
@@ -933,7 +919,7 @@ drvName, Bindings * attrs, Value & v)
 
 
                 } else {
-                    auto s = state.coerceToString(noPos, *i->value, context, context_below, true).toOwned();
+                    auto s = state.coerceToString(noPos, *i->value, context, context_below, StringCoercionMode::ToString).toOwned();
                     drv.env.emplace(key, s);
                     if (i->name == state.ctx.s.builder) drv.builder = std::move(s);
                     else if (i->name == state.ctx.s.system) drv.platform = std::move(s);
@@ -975,13 +961,13 @@ drvName, Bindings * attrs, Value & v)
                 for (auto & j : refs) {
                     drv.inputSrcs.insert(j);
                     if (j.isDerivation()) {
-                        drv.inputDrvs.map[j].value =
+                        drv.inputDrvs[j] =
                             state.aio.blockOn(state.ctx.store->readDerivation(j)).outputNames();
                     }
                 }
             },
             [&](const NixStringContextElem::Built & b) {
-                drv.inputDrvs.ensureSlot(*b.drvPath).value.insert(b.output);
+                drv.inputDrvs[b.drvPath.path].insert(b.output);
             },
             [&](const NixStringContextElem::Opaque & o) {
                 drv.inputSrcs.insert(o.path);
@@ -999,15 +985,10 @@ drvName, Bindings * attrs, Value & v)
             .debugThrow();
 
     /* Check whether the derivation name is valid. */
-    if (isDerivation(drvName) &&
+    if (isDerivation(drvName)) {
-        !(ingestionMethod == ContentAddressMethod { TextIngestionMethod { } } &&
+        state.ctx.errors
-          outputs.size() == 1 &&
+            .make<EvalError>("derivation names are not allowed to end in '%s'", drvExtension)
-          *(outputs.begin()) == "out"))
+            .debugThrow();
-    {
-        state.ctx.errors.make<EvalError>(
-            "derivation names are allowed to end in '%s' only if they produce a single derivation file",
-            drvExtension
-        ).debugThrow();
     }
 
     if (outputHash) {
@@ -1035,31 +1016,6 @@ drvName, Bindings * attrs, Value & v)
         drv.outputs.insert_or_assign("out", std::move(dof));
     }
 
-    else if (contentAddressed || isImpure) {
-        if (contentAddressed && isImpure)
-            state.ctx.errors.make<EvalError>("derivation cannot be both content-addressed and impure")
-                .debugThrow();
-
-        auto ht = parseHashTypeOpt(outputHashAlgo).value_or(HashType::SHA256);
-        auto method = ingestionMethod.value_or(FileIngestionMethod::Recursive);
-
-        for (auto & i : outputs) {
-            drv.env[i] = hashPlaceholder(i);
-            if (isImpure)
-                drv.outputs.insert_or_assign(i,
-                    DerivationOutput::Impure {
-                        .method = method,
-                        .hashType = ht,
-                    });
-            else
-                drv.outputs.insert_or_assign(i,
-                    DerivationOutput::CAFloating {
-                        .method = method,
-                        .hashType = ht,
-                    });
-        }
-    }
-
     else {
         /* Compute a hash over the "masked" store derivation, which is
            the final one except that in the list of outputs, the
@@ -1070,34 +1026,25 @@ drvName, Bindings * attrs, Value & v)
         for (auto & i : outputs) {
             drv.env[i] = "";
             drv.outputs.insert_or_assign(i,
-                DerivationOutput::Deferred { });
+                DerivationOutput::InputAddressed { .path = StorePath::dummy });
         }
 
         auto hashModulo =
             state.aio.blockOn(hashDerivationModulo(*state.ctx.store, Derivation(drv), true));
-        switch (hashModulo.kind) {
+        for (auto & i : outputs) {
-        case DrvHash::Kind::Regular:
+            auto h = get(hashModulo.hashes, i);
-            for (auto & i : outputs) {
+            if (!h)
-                auto h = get(hashModulo.hashes, i);
+                state.ctx.errors.make<AssertionError>(
-                if (!h)
+                    "derivation produced no hash for output '%s'",
-                    state.ctx.errors.make<AssertionError>(
+                    i
-                        "derivation produced no hash for output '%s'",
+                ).debugThrow();
-                        i
+            auto outPath = state.ctx.store->makeOutputPath(i, *h, drvName);
-                    ).debugThrow();
+            drv.env[i] = state.ctx.store->printStorePath(outPath);
-                auto outPath = state.ctx.store->makeOutputPath(i, *h, drvName);
+            drv.outputs.insert_or_assign(
-                drv.env[i] = state.ctx.store->printStorePath(outPath);
+                i,
-                drv.outputs.insert_or_assign(
+                DerivationOutput::InputAddressed {
-                    i,
+                    .path = std::move(outPath),
-                    DerivationOutput::InputAddressed {
+                });
-                        .path = std::move(outPath),
-                    });
-            }
-            break;
-            ;
-        case DrvHash::Kind::Deferred:
-            for (auto & i : outputs) {
-                drv.outputs.insert_or_assign(i, DerivationOutput::Deferred {});
-            }
         }
     }
 
@@ -1232,7 +1179,7 @@ static void prim_baseNameOf(EvalState & state, Value * * args, Value & v)
     NixStringContext context;
     v.mkString(baseNameOf(*state.coerceToString(noPos, *args[0], context,
             "while evaluating the first argument passed to builtins.baseNameOf",
-            false, false)), context);
+            StringCoercionMode::Strict, false)), context);
 }
 
 /* Return the directory of the given path, i.e., everything before the
@@ -1248,7 +1195,7 @@ static void prim_dirOf(EvalState & state, Value * * args, Value & v)
         NixStringContext context;
         auto path = state.coerceToString(noPos, *args[0], context,
             "while evaluating the first argument passed to 'builtins.dirOf'",
-            false, false);
+            StringCoercionMode::Strict, false);
         auto dir = dirOf(*path);
         v.mkString(dir, context);
     }
@@ -1309,7 +1256,7 @@ static void prim_findFile(EvalState & state, Value * * args, Value & v)
         NixStringContext context;
         auto path = state.coerceToString(noPos, *i->value, context,
                 "while evaluating the `path` attribute of an element of the list passed to builtins.findFile",
-                false, false).toOwned();
+                StringCoercionMode::Strict, false).toOwned();
 
         try {
             auto rewrites = state.realiseContext(context);
@@ -1402,21 +1349,6 @@ static void prim_readDir(EvalState & state, Value * * args, Value & v)
     v.mkAttrs(attrs);
 }
 
-/* Extend single element string context with another output. */
-static void prim_outputOf(EvalState & state, Value * * args, Value & v)
-{
-    SingleDerivedPath drvPath = state.coerceToSingleDerivedPath(noPos, *args[0], "while evaluating the first argument to builtins.outputOf");
-
-    OutputNameView outputName = state.forceStringNoCtx(*args[1], noPos, "while evaluating the second argument to builtins.outputOf");
-
-    state.mkSingleDerivedPathString(
-        SingleDerivedPath::Built {
-            .drvPath = make_ref<SingleDerivedPath>(drvPath),
-            .output = std::string { outputName },
-        },
-        v);
-}
-
 /*************************************************************
  * Creating files
  *************************************************************/
@@ -1607,7 +1539,7 @@ static void prim_path(EvalState & state, Value * * args, Value & v)
     state.forceAttrs(*args[0], noPos, "while evaluating the argument passed to 'builtins.path'");
 
     for (auto & attr : *args[0]->attrs) {
-        auto n = state.ctx.symbols[attr.name];
+        auto & n = state.ctx.symbols[attr.name];
         if (n == "path")
             path.emplace(state.coerceToPath(attr.pos, *attr.value, context, "while evaluating the 'path' attribute passed to 'builtins.path'"));
         else if (attr.name == state.ctx.s.name)
@@ -1650,7 +1582,7 @@ static void prim_attrNames(EvalState & state, Value * * args, Value & v)
 
     size_t n = 0;
     for (auto & i : *args[0]->attrs)
-        (v.listElems()[n++] = state.ctx.mem.allocValue())->mkString(state.ctx.symbols[i.name]);
+        v.listElems()[n++] = const_cast<Value *>(state.ctx.symbols[i.name].toValuePtr());
 
     std::sort(v.listElems(), v.listElems() + n,
               [](Value * v1, Value * v2) { return strcmp(v1->string.s, v2->string.s) < 0; });
@@ -1949,9 +1881,8 @@ static void prim_mapAttrs(EvalState & state, Value * * args, Value & v)
     auto attrs = state.ctx.buildBindings(args[1]->attrs->size());
 
     for (auto & i : *args[1]->attrs) {
-        Value * vName = state.ctx.mem.allocValue();
         Value * vFun2 = state.ctx.mem.allocValue();
-        vName->mkString(state.ctx.symbols[i.name]);
+        auto vName = const_cast<Value *>(state.ctx.symbols[i.name].toValuePtr());
         vFun2->mkApp(args[0], vName);
         attrs.alloc(i.name).mkApp(vFun2, i.value);
     }
@@ -1997,8 +1928,7 @@ static void prim_zipAttrsWith(EvalState & state, Value * * args, Value & v)
     }
 
     for (auto & attr : *v.attrs) {
-        auto name = state.ctx.mem.allocValue();
+        auto name = const_cast<Value *>(state.ctx.symbols[attr.name].toValuePtr());
-        name->mkString(state.ctx.symbols[attr.name]);
         auto call1 = state.ctx.mem.allocValue();
         call1->mkApp(args[0], name);
         auto call2 = state.ctx.mem.allocValue();
@@ -2488,7 +2418,7 @@ static void prim_toString(EvalState & state, Value * * args, Value & v)
     NixStringContext context;
     auto s = state.coerceToString(noPos, *args[0], context,
             "while evaluating the first argument passed to builtins.toString",
-            true, false);
+            StringCoercionMode::ToString, false);
     v.mkString(*s, context);
 }
 
@@ -2870,7 +2800,6 @@ void EvalBuiltins::createBaseEnv(const SearchPath & searchPath, const Path & sto
        because attribute lookups expect it to be sorted. */
     env.values[0]->attrs->sort();
 
-    staticEnv->sort();
     staticEnv->isRoot = true;
 }
 

lix/libexpr/primops/context.cc

@@ -127,9 +127,7 @@ void prim_getContext(EvalState & state, Value * * args, Value & v)
                 contextInfos[std::move(d.drvPath)].allOutputs = true;
             },
             [&](NixStringContextElem::Built && b) {
-                // FIXME should eventually show string context as is, no
+                auto drvPath = b.drvPath.path;
-                // resolving here.
-                auto drvPath = state.aio.blockOn(resolveDerivedPath(*state.ctx.store, *b.drvPath));
                 contextInfos[std::move(drvPath)].outputs.emplace_back(std::move(b.output));
             },
             [&](NixStringContextElem::Opaque && o) {
@@ -219,7 +217,7 @@ static void prim_appendContext(EvalState & state, Value * * args, Value & v)
             for (auto elem : iter->value->listItems()) {
                 auto outputName = state.forceStringNoCtx(*elem, iter->pos, "while evaluating an output name within a string context");
                 context.emplace(NixStringContextElem::Built {
-                    .drvPath = makeConstantStorePathRef(namePath),
+                    .drvPath = makeConstantStorePath(namePath),
                     .output = std::string { outputName },
                 });
             }

lix/libexpr/primops/fetchMercurial.cc

@@ -22,7 +22,7 @@ static void prim_fetchMercurial(EvalState & state, Value * * args, Value & v)
             if (n == "url")
                 url = state.coerceToString(attr.pos, *attr.value, context,
                         "while evaluating the `url` attribute passed to builtins.fetchMercurial",
-                        false, false).toOwned();
+                        StringCoercionMode::Strict, false).toOwned();
             else if (n == "rev") {
                 // Ugly: unlike fetchGit, here the "rev" attribute can
                 // be both a revision or a branch/tag name.
@@ -44,7 +44,7 @@ static void prim_fetchMercurial(EvalState & state, Value * * args, Value & v)
     } else
         url = state.coerceToString(noPos, *args[0], context,
                 "while evaluating the first argument passed to builtins.fetchMercurial",
-                false, false).toOwned();
+                StringCoercionMode::Strict, false).toOwned();
 
     // FIXME: git externals probably can be used to bypass the URI
     // whitelist. Ah well.

lix/libexpr/primops/fetchTree.cc

@@ -139,7 +139,7 @@ static void fetchTree(
             if (attr.name == state.ctx.s.type) continue;
             state.forceValue(*attr.value, attr.pos);
             if (attr.value->type() == nPath || attr.value->type() == nString) {
-                auto s = state.coerceToString(attr.pos, *attr.value, context, "", false, false).toOwned();
+                auto s = state.coerceToString(attr.pos, *attr.value, context, "", StringCoercionMode::Strict, false).toOwned();
                 attrs.emplace(state.ctx.symbols[attr.name],
                     state.ctx.symbols[attr.name] == "url"
                     ? type == "git"
@@ -169,11 +169,18 @@ static void fetchTree(
                     "attribute 'name' isn’t supported in call to 'fetchTree'"
                 ).atPos(pos).debugThrow();
 
+        // HACK: When using `fetchGit`, locking with only the hash should happen
+        //       as we don't care about flake shenanigans about `lastModified`
+        if (type == "git" && attrs.contains("narHash")) {
+            using namespace std::literals::string_literals;
+            attrs["type"] = "\0git-locked"s;
+        }
+
         input = fetchers::Input::fromAttrs(std::move(attrs));
     } else {
         auto url = state.coerceToString(pos, *args[0], context,
                 "while evaluating the first argument passed to the fetcher",
-                false, false).toOwned();
+                StringCoercionMode::Strict, false).toOwned();
 
         if (type == "git") {
             fetchers::Attrs attrs;

lix/libexpr/print-ambiguous.hh

@@ -1,6 +1,7 @@
 #pragma once
 ///@file
 
+#include "lix/libexpr/symbol-table.hh"
 #include "lix/libexpr/value.hh"
 
 namespace nix {

lix/libexpr/print.cc

@@ -287,7 +287,7 @@ private:
 
             AttrVec sorted;
             for (auto & i : *v.attrs)
-                sorted.emplace_back(std::pair(state.ctx.symbols[i.name], i.value));
+                sorted.emplace_back(state.ctx.symbols[i.name], i.value);
 
             if (options.maxAttrs == std::numeric_limits<size_t>::max())
                 std::sort(sorted.begin(), sorted.end());

lix/libexpr/settings/ignore-try.md

@@ -2,7 +2,7 @@
 name: ignore-try
 internalName: ignoreExceptionsDuringTry
 type: bool
-default: false
+default: true
 ---
 If set to true, ignore exceptions inside 'tryEval' calls when evaluating nix expressions in
 debug mode (using the --debugger flag). By default the debugger will pause on all exceptions.

lix/libexpr/symbol-table.hh

@@ -6,6 +6,8 @@
 #include "lix/libutil/types.hh"
 #include "lix/libutil/chunked-vector.hh"
 
+#include "lix/libexpr/value.hh"
+
 namespace nix {
 
 /**
@@ -16,6 +18,7 @@ namespace nix {
 class SymbolStr
 {
     friend class SymbolTable;
+    friend class InternedSymbol;
 
 private:
     const std::string * s;
@@ -41,6 +44,59 @@ public:
     friend std::ostream & operator <<(std::ostream & os, const SymbolStr & symbol);
 };
 
+class InternedSymbol
+{
+private:
+    /*
+     * The type that actually stores the string contained inside of the Value.
+     */
+    std::string contents;
+
+    /*
+     * A value containing a string that can be immediately passed to the evaluator.
+     */
+    Value underlyingValue;
+
+public:
+    explicit InternedSymbol(std::string_view s)
+        : contents(s)
+        , underlyingValue(NewValueAs::string, contents.c_str(), nullptr)
+    {
+    }
+
+    InternedSymbol(InternedSymbol &&) = default;
+    InternedSymbol & operator=(InternedSymbol &&) = default;
+
+    KJ_DISALLOW_COPY(InternedSymbol);
+
+    operator SymbolStr() const
+    {
+        return SymbolStr(contents);
+    }
+
+    bool operator==(std::string_view s2) const
+    {
+        return contents == s2;
+    }
+
+    operator const std::string &() const
+    {
+        return contents;
+    }
+
+    operator std::string_view() const
+    {
+        return contents;
+    }
+
+    const Value * toValuePtr() const
+    {
+        return &underlyingValue;
+    }
+
+    friend std::ostream & operator<<(std::ostream & os, const InternedSymbol & symbol);
+};
+
 /**
  * Symbols have the property that they can be compared efficiently
  * (using an equality test), because the symbol table stores only one
@@ -72,13 +128,17 @@ public:
 class SymbolTable
 {
 private:
-    std::unordered_map<std::string_view, std::pair<const std::string *, uint32_t>> symbols;
+    /**
-    ChunkedVector<std::string, 8192> store{16};
+     * Map from string view (backed by ChunkedVector) -> offset into the store.
+     * ChunkedVector references are never invalidated.
+     */
+    std::unordered_map<std::string_view, uint32_t> symbols;
+    ChunkedVector<InternedSymbol, 8192> store{16};
 
 public:
 
     /**
-     * converts a string into a symbol.
+     * Converts a string into a symbol.
      */
     Symbol create(std::string_view s)
     {
@@ -88,18 +148,20 @@ public:
         // on the original implementation using unordered_set
         // FIXME: make this thread-safe.
         auto it = symbols.find(s);
-        if (it != symbols.end()) return Symbol(it->second.second + 1);
+        if (it != symbols.end()) {
+            return Symbol(it->second + 1);
+        }
 
-        const auto & [rawSym, idx] = store.add(std::string(s));
+        const auto & [rawSym, idx] = store.add(s);
-        symbols.emplace(rawSym, std::make_pair(&rawSym, idx));
+        symbols.emplace(rawSym, idx);
         return Symbol(idx + 1);
     }
 
-    SymbolStr operator[](Symbol s) const
+    const InternedSymbol & operator[](Symbol s) const
     {
         if (s.id == 0 || s.id > store.size())
             abort();
-        return SymbolStr(store[s.id - 1]);
+        return store[s.id - 1];
     }
 
     size_t size() const

lix/libexpr/value-to-json.cc

@@ -46,7 +46,7 @@ JSON printValueAsJSON(EvalState & state, bool strict,
             break;
 
         case nAttrs: {
-            auto maybeString = state.tryAttrsToString(pos, v, context, false, false);
+            auto maybeString = state.tryAttrsToString(pos, v, context, StringCoercionMode::Strict, false);
             if (maybeString) {
                 out = *maybeString;
                 break;

lix/libexpr/value.hh

@@ -8,8 +8,8 @@
 #include <span>
 
 #include "lix/libexpr/gc-alloc.hh"
-#include "lix/libexpr/symbol-table.hh"
 #include "lix/libexpr/value/context.hh"
+#include "lix/libutil/logging.hh"
 #include "lix/libutil/source-path.hh"
 #include "lix/libexpr/print-options.hh"
 #include "lix/libutil/checked-arithmetic.hh"
@@ -59,13 +59,32 @@ typedef enum {
     nExternal
 } ValueType;
 
+
+/**
+ * Modes of string coercion.
+ *
+ * Determines how permissive the coercion functions are when converting
+ * values to strings.
+ *
+ * - Strict: Only allow coercion of values that are already strings,
+ *   paths, or derivations.
+ * - Interpolation: Additionally allow coercion of unambiguously printable values in a string, for
+ *   now: integers. This mode is meant as a stopgap measure until we get better formatting tools.
+ * - ToString: Additionally allow coercion of integers, booleans, null,
+ *   and lists to strings.
+ */
+enum class StringCoercionMode {
+    Strict,
+    Interpolation,
+    ToString,
+};
+
 class Bindings;
 struct Env;
 struct Expr;
 struct ExprLambda;
 struct ExprBlackHole;
 struct PrimOp;
-class Symbol;
 class PosIdx;
 struct Pos;
 class StorePath;
@@ -106,7 +125,7 @@ class ExternalValueBase
      * Coerce the value to a string. Defaults to uncoercable, i.e. throws an
      * error.
      */
-    virtual std::string coerceToString(EvalState & state, const PosIdx & pos, NixStringContext & context, bool copyMore, bool copyToStore) const;
+    virtual std::string coerceToString(EvalState & state, const PosIdx & pos, NixStringContext & context, StringCoercionMode mode, bool copyToStore) const;
 
     /**
      * Compare to another value of the same type. Defaults to uncomparable,
@@ -839,8 +858,6 @@ public:
 };
 
 using ValueVector = GcVector<Value *>;
-using ValueMap = GcMap<Symbol, Value *>;
-using ValueVectorMap = std::map<Symbol, ValueVector>;
 
 /**
  * A value allocated in traceable memory.

lix/libexpr/value/context.cc

@@ -2,33 +2,10 @@
 
 namespace nix {
 
-NixStringContextElem NixStringContextElem::parse(
+NixStringContextElem NixStringContextElem::parse(std::string_view s0)
-    std::string_view s0,
-    const ExperimentalFeatureSettings & xpSettings)
 {
     std::string_view s = s0;
 
-    std::function<SingleDerivedPath()> parseRest;
-    parseRest = [&]() -> SingleDerivedPath {
-        // Case on whether there is a '!'
-        size_t index = s.find("!");
-        if (index == std::string_view::npos) {
-            return SingleDerivedPath::Opaque {
-                .path = StorePath { s },
-            };
-        } else {
-            std::string output { s.substr(0, index) };
-            // Advance string to parse after the '!'
-            s = s.substr(index + 1);
-            auto drv = make_ref<SingleDerivedPath>(parseRest());
-            drvRequireExperiment(*drv, xpSettings);
-            return SingleDerivedPath::Built {
-                .drvPath = std::move(drv),
-                .output = std::move(output),
-            };
-        }
-    };
-
     if (s.size() == 0) {
         throw BadNixStringContextElem(s0,
             "String context element should never be an empty string");
@@ -40,14 +17,20 @@ NixStringContextElem NixStringContextElem::parse(
         s = s.substr(1);
 
         // Find *second* '!'
-        if (s.find("!") == std::string_view::npos) {
+        size_t index = s.find("!");
+        if (index == std::string_view::npos) {
             throw BadNixStringContextElem(s0,
                 "String content element beginning with '!' should have a second '!'");
         }
 
-        return std::visit(
+        std::string output { s.substr(0, index) };
-            [&](auto x) -> NixStringContextElem { return std::move(x); },
+        // Advance string to parse after the '!'
-            parseRest());
+        s = s.substr(index + 1);
+        auto drv = SingleDerivedPath::Opaque{StorePath{s}};
+        return SingleDerivedPath::Built{
+            .drvPath = std::move(drv),
+            .output = std::move(output),
+        };
     }
     case '=': {
         return NixStringContextElem::DrvDeep {
@@ -60,9 +43,9 @@ NixStringContextElem NixStringContextElem::parse(
             throw BadNixStringContextElem(s0,
                 "String content element not beginning with '!' should not have a second '!'");
         }
-        return std::visit(
+        return SingleDerivedPath::Opaque{
-            [&](auto x) -> NixStringContextElem { return std::move(x); },
+            .path = StorePath{s},
-            parseRest());
+        };
     }
     }
 }
@@ -71,27 +54,15 @@ std::string NixStringContextElem::to_string() const
 {
     std::string res;
 
-    std::function<void(const SingleDerivedPath &)> toStringRest;
-    toStringRest = [&](auto & p) {
-        std::visit(overloaded {
-            [&](const SingleDerivedPath::Opaque & o) {
-                res += o.path.to_string();
-            },
-            [&](const SingleDerivedPath::Built & o) {
-                res += o.output;
-                res += '!';
-                toStringRest(*o.drvPath);
-            },
-        }, p.raw());
-    };
-
     std::visit(overloaded {
         [&](const NixStringContextElem::Built & b) {
             res += '!';
-            toStringRest(b);
+            res += b.output;
+            res += '!';
+            res += b.drvPath.path.to_string();
         },
         [&](const NixStringContextElem::Opaque & o) {
-            toStringRest(o);
+            res += o.path.to_string();
         },
         [&](const NixStringContextElem::DrvDeep & d) {
             res += '=';

lix/libexpr/value/context.hh

@@ -69,12 +69,8 @@ struct NixStringContextElem {
      * - ‘<path>’
      * - ‘=<path>’
      * - ‘!<name>!<path>’
-     *
-     * @param xpSettings Stop-gap to avoid globals during unit tests.
      */
-    static NixStringContextElem parse(
+    static NixStringContextElem parse(std::string_view s);
-        std::string_view s,
-        const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
     std::string to_string() const;
 };
 

lix/libfetchers/builtin-fetchers.hh

@@ -10,6 +10,7 @@ std::unique_ptr<InputScheme> makePathInputScheme();
 std::unique_ptr<InputScheme> makeFileInputScheme();
 std::unique_ptr<InputScheme> makeTarballInputScheme();
 std::unique_ptr<InputScheme> makeGitInputScheme();
+std::unique_ptr<InputScheme> makeGitLockedInputScheme();
 std::unique_ptr<InputScheme> makeMercurialInputScheme();
 std::unique_ptr<InputScheme> makeGitHubInputScheme();
 std::unique_ptr<InputScheme> makeGitLabInputScheme();

lix/libfetchers/fetchers.cc

@@ -22,6 +22,7 @@ void initLibFetchers()
     registerInputScheme(makeTarballInputScheme());
     registerInputScheme(makeFileInputScheme());
     registerInputScheme(makeGitInputScheme());
+    registerInputScheme(makeGitLockedInputScheme());
     registerInputScheme(makeMercurialInputScheme());
     registerInputScheme(makeGitHubInputScheme());
     registerInputScheme(makeGitLabInputScheme());
@@ -304,6 +305,24 @@ std::optional<time_t> Input::getLastModified() const
     return maybeGetIntAttr(attrs, "lastModified");
 }
 
+std::optional<Input> InputScheme::inputFromAttrs(const Attrs & attrs) const
+{
+    if (maybeGetStrAttr(attrs, "type") != schemeType()) return {};
+
+    Attrs finalAttrs = preprocessAttrs(attrs);
+
+    for (auto & [name, value] : finalAttrs)
+        // All attrs need to accept a `type` and `narHash` key, the rest is scheme-specific
+        if (name != "type" && name != "narHash" && !allowedAttrs().contains(name))
+            throw UnsupportedAttributeError("unsupported input attribute '%s' for the '%s' scheme", name, schemeType());
+
+    Input input;
+    input.attrs = finalAttrs;
+    return input;
+}
+
+
+
 ParsedURL InputScheme::toURL(const Input & input) const
 {
     throw Error("don't know how to convert input '%s' to a URL", attrsToJSON(input.attrs));

lix/libfetchers/fetchers.hh

@@ -2,6 +2,7 @@
 ///@file
 
 #include "lix/libstore/content-address.hh"
+#include "lix/libutil/error.hh"
 #include "lix/libutil/result.hh"
 #include "lix/libutil/types.hh"
 #include "lix/libutil/hash.hh"
@@ -19,6 +20,8 @@ namespace nix { class Store; }
 
 namespace nix::fetchers {
 
+MakeError(UnsupportedAttributeError, Error);
+
 struct Tree
 {
     Path actualPath;
@@ -125,7 +128,6 @@ public:
     std::optional<time_t> getLastModified() const;
 };
 
-
 /**
  * The InputScheme represents a type of fetcher.  Each fetcher
  * registers with nix at startup time.  When processing an input for a
@@ -142,7 +144,12 @@ struct InputScheme
 
     virtual std::optional<Input> inputFromURL(const ParsedURL & url, bool requireTree) const = 0;
 
-    virtual std::optional<Input> inputFromAttrs(const Attrs & attrs) const = 0;
+    virtual Attrs preprocessAttrs(const Attrs & attrs) const = 0;
+
+    // The scheme type, which is used to match attributes to a specific scheme
+    virtual std::string schemeType() const = 0;
+
+    virtual std::optional<Input> inputFromAttrs(const Attrs & attrs) const;
 
     virtual ParsedURL toURL(const Input & input) const;
 
@@ -174,6 +181,9 @@ struct InputScheme
     virtual bool isLockedByRev() const { return true; }
 
 protected:
+    // The set of allowed attributes for this specific fetcher
+    virtual const std::set<std::string> & allowedAttrs() const = 0;
+
     void emplaceURLQueryIntoAttrs(
         const ParsedURL & parsedURL,
         Attrs & attrs,

lix/libfetchers/git.cc

@@ -22,6 +22,7 @@
 #include <string.h>
 #include <sys/time.h>
 #include <sys/wait.h>
+#include <unistd.h>
 
 using namespace std::string_literals;
 
@@ -164,11 +165,19 @@ WorkdirInfo getWorkdirInfo(const Input & input, const Path & workdir)
 
     /* Check whether HEAD points to something that looks like a commit,
        since that is the refrence we want to use later on. */
-    auto result = runProgram(RunOptions {
+    auto result = runProgram(RunOptions{
         .program = "git",
-        .args = { "-C", workdir, "--git-dir", gitDir, "rev-parse", "--verify", "--no-revs", "HEAD^{commit}" },
+        .args =
+            {"-C",
+             workdir,
+             "--git-dir",
+             gitDir,
+             "rev-parse",
+             "--verify",
+             "--no-revs",
+             "HEAD^{commit}"},
         .environment = env,
-        .mergeStderrToStdout = true
+        .redirections = {{.dup = STDERR_FILENO, .from = STDOUT_FILENO}},
     });
     auto exitCode = WEXITSTATUS(result.first);
     auto errorMessage = result.second;
@@ -288,8 +297,28 @@ static std::optional<Path> resolveRefToCachePath(
     return std::nullopt;
 }
 
+static const std::set<std::string> allowedGitAttrs = {
+    "allRefs",
+    "dirtyRev",
+    "dirtyShortRev",
+    "lastModified",
+    "name",
+    "ref",
+    "rev",
+    "revCount",
+    "shallow",
+    "submodules",
+    "url",
+};
+
 struct GitInputScheme : InputScheme
 {
+    std::string schemeType() const override { return "git"; }
+
+    const std::set<std::string> & allowedAttrs() const override {
+        return allowedGitAttrs;
+    }
+
     std::optional<Input> inputFromURL(const ParsedURL & url, bool requireTree) const override
     {
         if (url.scheme != "git" &&
@@ -316,14 +345,7 @@ struct GitInputScheme : InputScheme
         return inputFromAttrs(attrs);
     }
 
-    std::optional<Input> inputFromAttrs(const Attrs & attrs) const override
+    Attrs preprocessAttrs(const Attrs & attrs) const override {
-    {
-        if (maybeGetStrAttr(attrs, "type") != "git") return {};
-
-        for (auto & [name, value] : attrs)
-            if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow" && name != "submodules" && name != "lastModified" && name != "revCount" && name != "narHash" && name != "allRefs" && name != "name" && name != "dirtyRev" && name != "dirtyShortRev")
-                throw Error("unsupported Git input attribute '%s'", name);
-
         parseURL(getStrAttr(attrs, "url"));
         maybeGetBoolAttr(attrs, "shallow");
         maybeGetBoolAttr(attrs, "submodules");
@@ -334,9 +356,7 @@ struct GitInputScheme : InputScheme
                 throw BadURL("invalid Git branch/tag name '%s'", *ref);
         }
 
-        Input input;
+        return attrs;
-        input.attrs = attrs;
-        return input;
     }
 
     ParsedURL toURL(const Input & input) const override
@@ -698,10 +718,12 @@ struct GitInputScheme : InputScheme
         AutoDelete delTmpDir(tmpDir, true);
         PathFilter filter = defaultPathFilter;
 
-        auto result = runProgram(RunOptions {
+        auto result = runProgram(RunOptions{
             .program = "git",
-            .args = { "-C", repoDir, "--git-dir", gitDir, "cat-file", "commit", input.getRev()->gitRev() },
+            .args =
-            .mergeStderrToStdout = true
+                {"-C", repoDir, "--git-dir", gitDir, "cat-file", "commit", input.getRev()->gitRev()
+                },
+            .redirections = {{.dup = STDERR_FILENO, .from = STDOUT_FILENO}},
         });
         if (WEXITSTATUS(result.first) == 128
             && result.second.find("bad file") != std::string::npos)
@@ -766,7 +788,7 @@ struct GitInputScheme : InputScheme
                 .args = { "-C", repoDir, "--git-dir", gitDir, "archive", input.getRev()->gitRev() },
                 .captureStdout = true,
             });
-            Finally const _wait([&] { proc.wait(); });
+            Finally const _wait([&] { proc.waitAndCheck(); });
 
             unpackTarfile(*proc.getStdout(), tmpDir);
         }
@@ -812,4 +834,22 @@ std::unique_ptr<InputScheme> makeGitInputScheme()
     return std::make_unique<GitInputScheme>();
 }
 
+struct GitLockedInputScheme : GitInputScheme {
+
+    std::string schemeType() const override {
+        using namespace std::literals::string_literals;
+        return "\0git-locked"s;
+    }
+
+    bool hasAllInfo(const Input & input) const override {
+        return true;
+    }
+
+};
+
+std::unique_ptr<InputScheme> makeGitLockedInputScheme()
+{
+    return std::make_unique<GitLockedInputScheme>();
+}
+
 }

lix/libfetchers/github.cc

@@ -1,7 +1,5 @@
 #include "lix/libfetchers/attrs.hh"
 #include "lix/libstore/filetransfer.hh"
-#include "lix/libfetchers/cache.hh"
-#include "lix/libstore/globals.hh"
 #include "lix/libfetchers/builtin-fetchers.hh"
 #include "lix/libstore/store-api.hh"
 #include "lix/libutil/async.hh"
@@ -24,19 +22,30 @@ struct DownloadUrl
     Headers headers;
 };
 
+static const std::set<std::string> allowedGitArchiveAttrs = {
+    "host",
+    "lastModified",
+    "owner",
+    "ref",
+    "repo",
+    "rev",
+};
+
 // A github, gitlab, or sourcehut host
 const static std::string hostRegexS = "[a-zA-Z0-9.-]*"; // FIXME: check
 std::regex hostRegex = regex::parse(hostRegexS, std::regex::ECMAScript);
 
 struct GitArchiveInputScheme : InputScheme
 {
-    virtual std::string type() const = 0;
+    const std::set<std::string> & allowedAttrs() const override {
+        return allowedGitArchiveAttrs;
+    }
 
     virtual std::optional<std::pair<std::string, std::string>> accessHeaderFromToken(const std::string & token) const = 0;
 
     std::optional<Input> inputFromURL(const ParsedURL & url, bool requireTree) const override
     {
-        if (url.scheme != type()) return {};
+        if (url.scheme != schemeType()) return {};
 
         auto path = tokenizeString<std::vector<std::string>>(url.path, "/");
 
@@ -63,7 +72,7 @@ struct GitArchiveInputScheme : InputScheme
             throw BadURL("URL '%s' is invalid", url.url);
 
         Attrs attrs;
-        attrs.emplace("type", type());
+        attrs.emplace("type", schemeType());
         attrs.emplace("owner", path[0]);
         attrs.emplace("repo", path[1]);
 
@@ -93,20 +102,18 @@ struct GitArchiveInputScheme : InputScheme
         return inputFromAttrs(attrs);
     }
 
-    std::optional<Input> inputFromAttrs(const Attrs & attrs) const override
+    Attrs preprocessAttrs(const Attrs & attrs) const override
     {
         // Attributes can contain refOrRev and it needs to be figured out
         // which one it is (see inputFromURL for when that may happen).
         // The correct one (ref or rev) will be written into finalAttrs and
         // it needs to be mutable for that.
         Attrs finalAttrs(attrs);
-        auto type_ = maybeGetStrAttr(finalAttrs, "type");
-        if (type_ != type()) return {};
 
         auto owner = getStrAttr(finalAttrs, "owner");
         auto repo = getStrAttr(finalAttrs, "repo");
 
-        auto url = fmt("%s:%s/%s", *type_, owner, repo);
+        auto url = fmt("%s:%s/%s", schemeType(), owner, repo);
         if (auto host = maybeGetStrAttr(finalAttrs, "host")) {
             if (!std::regex_match(*host, hostRegex)) {
                 throw BadURL("URL '%s' contains an invalid instance host", url);
@@ -132,15 +139,7 @@ struct GitArchiveInputScheme : InputScheme
             }
         }
 
-        for (auto & [name, value] : finalAttrs) {
+        return finalAttrs;
-            if (name != "type" && name != "owner" && name != "repo" && name != "ref" && name != "rev" && name != "narHash" && name != "lastModified" && name != "host") {
-                throw Error("unsupported input attribute '%s'", name);
-            }
-        }
-
-        Input input;
-        input.attrs = finalAttrs;
-        return input;
     }
 
     ParsedURL toURL(const Input & input) const override
@@ -154,7 +153,7 @@ struct GitArchiveInputScheme : InputScheme
         if (ref) path += "/" + *ref;
         if (rev) path += "/" + rev->to_string(Base::Base16, false);
         return ParsedURL {
-            .scheme = type(),
+            .scheme = schemeType(),
             .path = path,
         };
     }
@@ -239,7 +238,7 @@ struct GitArchiveInputScheme : InputScheme
 
 struct GitHubInputScheme : GitArchiveInputScheme
 {
-    std::string type() const override { return "github"; }
+    std::string schemeType() const override { return "github"; }
 
     std::optional<std::pair<std::string, std::string>> accessHeaderFromToken(const std::string & token) const override
     {
@@ -329,7 +328,7 @@ struct GitHubInputScheme : GitArchiveInputScheme
 
 struct GitLabInputScheme : GitArchiveInputScheme
 {
-    std::string type() const override { return "gitlab"; }
+    std::string schemeType() const override { return "gitlab"; }
 
     std::optional<std::pair<std::string, std::string>> accessHeaderFromToken(const std::string & token) const override
     {
@@ -405,7 +404,7 @@ struct GitLabInputScheme : GitArchiveInputScheme
 
 struct SourceHutInputScheme : GitArchiveInputScheme
 {
-    std::string type() const override { return "sourcehut"; }
+    std::string schemeType() const override { return "sourcehut"; }
 
     std::optional<std::pair<std::string, std::string>> accessHeaderFromToken(const std::string & token) const override
     {

lix/libfetchers/indirect.cc

@@ -8,8 +8,20 @@ namespace nix::fetchers {
 
 std::regex flakeRegex = regex::parse("[a-zA-Z][a-zA-Z0-9_-]*", std::regex::ECMAScript);
 
+static const std::set<std::string> allowedIndirectAttrs = {
+    "id",
+    "ref",
+    "rev",
+};
+
 struct IndirectInputScheme : InputScheme
 {
+    std::string schemeType() const override { return "indirect"; }
+
+    const std::set<std::string> & allowedAttrs() const override {
+        return allowedIndirectAttrs;
+    }
+
     std::optional<Input> inputFromURL(const ParsedURL & url, bool requireTree) const override
     {
         if (url.scheme != "flake") return {};
@@ -47,14 +59,7 @@ struct IndirectInputScheme : InputScheme
         return inputFromAttrs(attrs);
     }
 
-    std::optional<Input> inputFromAttrs(const Attrs & attrs) const override
+    Attrs preprocessAttrs(const Attrs & attrs) const override {
-    {
-        if (maybeGetStrAttr(attrs, "type") != "indirect") return {};
-
-        for (auto & [name, value] : attrs)
-            if (name != "type" && name != "id" && name != "ref" && name != "rev" && name != "narHash")
-                throw Error("unsupported indirect input attribute '%s'", name);
-
         auto id = getStrAttr(attrs, "id");
         if (!std::regex_match(id, flakeRegex))
             throw BadURL("'%s' is not a valid flake ID", id);
@@ -71,9 +76,17 @@ struct IndirectInputScheme : InputScheme
             }
         }
 
-        Input input;
+        return attrs;
-        input.direct = false;
+    }
-        input.attrs = attrs;
+
+    std::optional<Input> inputFromAttrs(const Attrs & attrs) const override
+    {
+        std::optional<Input> input = InputScheme::inputFromAttrs(attrs);
+
+        if (input) {
+            input->direct = false;
+        }
+
         return input;
     }
 

lix/libfetchers/mercurial.cc

@@ -44,8 +44,22 @@ static std::string runHg(const Strings & args)
     return res.second;
 }
 
+static const std::set<std::string> allowedMercurialAttrs = {
+    "name",
+    "ref",
+    "rev",
+    "revCount",
+    "url",
+};
+
 struct MercurialInputScheme : InputScheme
 {
+    std::string schemeType() const override { return "hg"; }
+
+    const std::set<std::string> & allowedAttrs() const override {
+        return allowedMercurialAttrs;
+    }
+
     std::optional<Input> inputFromURL(const ParsedURL & url, bool requireTree) const override
     {
         if (url.scheme != "hg+http" &&
@@ -67,14 +81,8 @@ struct MercurialInputScheme : InputScheme
         return inputFromAttrs(attrs);
     }
 
-    std::optional<Input> inputFromAttrs(const Attrs & attrs) const override
+    Attrs preprocessAttrs(const Attrs & attrs) const override
     {
-        if (maybeGetStrAttr(attrs, "type") != "hg") return {};
-
-        for (auto & [name, value] : attrs)
-            if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "revCount" && name != "narHash" && name != "name")
-                throw Error("unsupported Mercurial input attribute '%s'", name);
-
         parseURL(getStrAttr(attrs, "url"));
 
         if (auto ref = maybeGetStrAttr(attrs, "ref")) {
@@ -82,9 +90,7 @@ struct MercurialInputScheme : InputScheme
                 throw BadURL("invalid Mercurial branch/tag name '%s'", *ref);
         }
 
-        Input input;
+        return attrs;
-        input.attrs = attrs;
-        return input;
     }
 
     ParsedURL toURL(const Input & input) const override

lix/libfetchers/path.cc

@@ -6,8 +6,25 @@
 
 namespace nix::fetchers {
 
+/* Allow the user to pass in "fake" tree info
+   attributes. This is useful for making a pinned tree
+   work the same as the repository from which is exported
+   (e.g. path:/nix/store/...-source?lastModified=1585388205&rev=b0c285...). */
+static const std::set<std::string> allowedPathAttrs = {
+    "lastModified",
+    "path",
+    "rev",
+    "revCount",
+};
+
 struct PathInputScheme : InputScheme
 {
+    std::string schemeType() const override { return "path"; }
+
+    const std::set<std::string> & allowedAttrs() const override {
+        return allowedPathAttrs;
+    }
+
     std::optional<Input> inputFromURL(const ParsedURL & url, bool requireTree) const override
     {
         if (url.scheme != "path") return {};
@@ -34,26 +51,11 @@ struct PathInputScheme : InputScheme
         return input;
     }
 
-    std::optional<Input> inputFromAttrs(const Attrs & attrs) const override
+    Attrs preprocessAttrs(const Attrs & attrs) const override
     {
-        if (maybeGetStrAttr(attrs, "type") != "path") return {};
-
         getStrAttr(attrs, "path");
 
-        for (auto & [name, value] : attrs)
+        return attrs;
-            /* Allow the user to pass in "fake" tree info
-               attributes. This is useful for making a pinned tree
-               work the same as the repository from which is exported
-               (e.g. path:/nix/store/...-source?lastModified=1585388205&rev=b0c285...). */
-            if (name == "type" || name == "rev" || name == "revCount" || name == "lastModified" || name == "narHash" || name == "path")
-                // checked in Input::fromAttrs
-                ;
-            else
-                throw Error("unsupported path input attribute '%s'", name);
-
-        Input input;
-        input.attrs = attrs;
-        return input;
     }
 
     bool isLockedByRev() const override { return false; }

lix/libfetchers/tarball.cc

@@ -85,9 +85,9 @@ try {
     FileTransferResult res;
     std::string data;
     try {
-        auto [meta, content] = getFileTransfer()->download(url, headers);
+        auto [meta, content] = TRY_AWAIT(getFileTransfer()->download(url, headers));
         res = std::move(meta);
-        data = content->drain();
+        data = TRY_AWAIT(content->drain());
     } catch (FileTransferError & e) {
         if (cached) {
             warn("%s; using cached version", e.msg());
@@ -230,12 +230,25 @@ try {
     co_return result::current_exception();
 }
 
+// FIXME: some of these only apply to TarballInputScheme.
+static const std::set<std::string> allowedCurlAttrs = {
+    "lastModified",
+    "name",
+    "rev",
+    "revCount",
+    "unpack",
+    "url",
+};
+
 // An input scheme corresponding to a curl-downloadable resource.
 struct CurlInputScheme : InputScheme
 {
-    virtual const std::string inputType() const = 0;
     const std::set<std::string> transportUrlSchemes = {"file", "http", "https"};
 
+    const std::set<std::string> & allowedAttrs() const override {
+        return allowedCurlAttrs;
+    }
+
     bool hasTarballExtension(std::string_view path) const
     {
         return path.ends_with(".zip") || path.ends_with(".tar")
@@ -254,7 +267,7 @@ struct CurlInputScheme : InputScheme
         auto url = _url;
 
         Attrs attrs;
-        attrs.emplace("type", inputType());
+        attrs.emplace("type", schemeType());
 
         url.scheme = parseUrlScheme(url.scheme).transport;
 
@@ -264,24 +277,16 @@ struct CurlInputScheme : InputScheme
         return inputFromAttrs(attrs);
     }
 
-    std::optional<Input> inputFromAttrs(const Attrs & attrs) const override
+    Attrs preprocessAttrs(const Attrs & attrs) const override
     {
-        auto type = maybeGetStrAttr(attrs, "type");
-        if (type != inputType()) return {};
-
-        // FIXME: some of these only apply to TarballInputScheme.
-        std::set<std::string> allowedNames = {"type", "url", "narHash", "name", "unpack", "rev", "revCount", "lastModified"};
         for (auto & [name, value] : attrs)
-            if (!allowedNames.count(name))
+            if (name != "type" && name != "narHash" && !allowedAttrs().contains(name))
-                throw Error("unsupported %s input attribute '%s'. If you wanted to fetch a tarball with a query parameter, please use '{ type = \"tarball\"; url = \"...\"; }'", *type, name);
+                throw UnsupportedAttributeError("unsupported tarball input attribute '%s'. If you wanted to fetch a tarball with a query parameter, please use '{ type = \"tarball\"; url = \"...\"; }'", name);
 
-        Input input;
+        return attrs;
-        input.attrs = attrs;
-
-        //input.locked = (bool) maybeGetStrAttr(input.attrs, "hash");
-        return input;
     }
 
+
     ParsedURL toURL(const Input & input) const override
     {
         auto url = parseURL(getStrAttr(input.attrs, "url"));
@@ -303,14 +308,14 @@ struct CurlInputScheme : InputScheme
 
 struct FileInputScheme : CurlInputScheme
 {
-    const std::string inputType() const override { return "file"; }
+    std::string schemeType() const override { return "file"; }
 
     bool isValidURL(const ParsedURL & url, bool requireTree) const override
     {
         auto parsedUrlScheme = parseUrlScheme(url.scheme);
         return transportUrlSchemes.count(std::string(parsedUrlScheme.transport))
             && (parsedUrlScheme.application
-                ? parsedUrlScheme.application.value() == inputType()
+                ? parsedUrlScheme.application.value() == schemeType()
                 : (!requireTree && !hasTarballExtension(url.path)));
     }
 
@@ -328,7 +333,7 @@ struct FileInputScheme : CurlInputScheme
 
 struct TarballInputScheme : CurlInputScheme
 {
-    const std::string inputType() const override { return "tarball"; }
+    std::string schemeType() const override { return "tarball"; }
 
     bool isValidURL(const ParsedURL & url, bool requireTree) const override
     {
@@ -336,7 +341,7 @@ struct TarballInputScheme : CurlInputScheme
 
         return transportUrlSchemes.count(std::string(parsedUrlScheme.transport))
             && (parsedUrlScheme.application
-                ? parsedUrlScheme.application.value() == inputType()
+                ? parsedUrlScheme.application.value() == schemeType()
                 : (requireTree || hasTarballExtension(url.path)));
     }
 

lix/libmain/crash-handler.cc

@@ -1,11 +1,9 @@
 #include "lix/libmain/crash-handler.hh"
 #include "lix/libutil/error.hh"
-#include "lix/libutil/fmt.hh"
 #include "lix/libutil/logging.hh"
 
 #include <boost/core/demangle.hpp>
 #include <exception>
-#include <source_location>
 
 namespace nix {
 
@@ -23,38 +21,14 @@ void onTerminate()
         } else {
             logFatal("std::terminate() called without exception");
         }
-    } catch (const ForeignException & ex) {
-        asyncTrace = ex.asyncTrace();
-        logFatal(fmt("Exception: %s: %s", boost::core::demangle(ex.innerType.name()), ex.what()));
     } catch (const BaseException & ex) {
-        asyncTrace = ex.asyncTrace();
+        logException("Exception", ex);
-        logFatal(fmt("Exception: %s: %s", boost::core::demangle(typeid(ex).name()), ex.what()));
     } catch (const std::exception & ex) { // NOLINT(lix-foreign-exceptions)
-        logFatal(fmt("Exception: %s: %s", boost::core::demangle(typeid(ex).name()), ex.what()));
+        logException("Exception", ex);
     } catch (...) {
         logFatal("Unknown exception! Spooky.");
     }
 
-    logFatal("Stack trace:");
-    logFatal(getStackTrace());
-
-    if (asyncTrace && !asyncTrace->empty()) {
-        logFatal("Async task trace (probably incomplete):");
-        for (auto [i, frame] : enumerate(*asyncTrace)) {
-            logFatal(
-                fmt("#%i: %s (%s:%i:%i)",
-                    i,
-                    frame.location.function_name(),
-                    frame.location.file_name(),
-                    frame.location.line(),
-                    frame.location.column())
-            );
-            if (frame.description) {
-                logFatal(fmt("\t%s", *frame.description));
-            }
-        }
-    }
-
     std::abort();
 }
 }

lix/libmain/shared.cc

@@ -130,12 +130,6 @@ static void sigHandler(int signo) { }
 
 void initNix()
 {
-    // kj needs a signal for internal use. no system lix habitually runs on causes
-    // kj to actually *use* this signal, but better safe than sorry—and since some
-    // OSes (*cough* macos) don't support realtime signals we must use SIGUSR2 for
-    // this, thus "consuming" both USR signals. at some point we will change this.
-    kj::UnixEventPort::setReservedSignal(SIGUSR2);
-
     registerCrashHandler();
 
     /* Turn on buffering for cerr. */
@@ -155,9 +149,11 @@ void initNix()
     if (sigaction(SIGCHLD, &act, 0))
         throw SysError("resetting SIGCHLD");
 
-    /* Install a dummy SIGUSR1 handler for use with pthread_kill(). */
+    /* Install a dummy INTERRUPT_NOTIFY_SIGNAL handler for use with pthread_kill(). */
     act.sa_handler = sigHandler;
-    if (sigaction(SIGUSR1, &act, 0)) throw SysError("handling SIGUSR1");
+    if (sigaction(INTERRUPT_NOTIFY_SIGNAL, &act, 0)) {
+        throw SysError("handling interrupt notify signal %i", INTERRUPT_NOTIFY_SIGNAL);
+    }
 
 #if __APPLE__
     /* HACK: on darwin, we need can’t use sigprocmask with SIGWINCH.
@@ -191,7 +187,6 @@ void initNix()
     umask(0022);
 }
 
-
 LegacyArgs::LegacyArgs(AsyncIoRoot & aio, const std::string & programName,
     std::function<bool(Strings::iterator & arg, const Strings::iterator & end)> parseArg)
     : MixCommonArgs(programName), aio_(aio), parseArg(parseArg)

lix/libstore/binary-cache-store.cc

@@ -1,6 +1,7 @@
 #include "lix/libutil/archive.hh"
 #include "lix/libstore/binary-cache-store.hh"
 #include "lix/libutil/async-io.hh"
+#include "lix/libutil/async.hh"
 #include "lix/libutil/compression.hh"
 #include "lix/libstore/derivations.hh"
 #include "lix/libstore/fs-accessor.hh"
@@ -18,6 +19,7 @@
 #include "lix/libutil/strings.hh"
 
 #include <chrono>
+#include <functional>
 #include <regex>
 #include <fstream>
 #include <sstream>
@@ -38,9 +40,11 @@ kj::Promise<Result<void>> BinaryCacheStore::init()
 try {
     std::string cacheInfoFile = "nix-cache-info";
 
-    auto cacheInfo = getFileContents(cacheInfoFile);
+    auto cacheInfo = TRY_AWAIT(getFileContents(cacheInfoFile));
     if (!cacheInfo) {
-        upsertFile(cacheInfoFile, "StoreDir: " + config().storeDir + "\n", "text/x-nix-cache-info");
+        TRY_AWAIT(upsertFile(
+            cacheInfoFile, "StoreDir: " + config().storeDir + "\n", "text/x-nix-cache-info"
+        ));
     } else {
         for (auto & line : tokenizeString<Strings>(*cacheInfo, "\n")) {
             size_t colon= line.find(':');
@@ -63,20 +67,26 @@ try {
     co_return result::current_exception();
 }
 
-void BinaryCacheStore::upsertFile(const std::string & path,
+kj::Promise<Result<void>> BinaryCacheStore::upsertFile(
-    std::string && data,
+    const std::string & path, std::string && data, const std::string & mimeType
-    const std::string & mimeType)
+)
-{
+try {
-    upsertFile(path, std::make_shared<std::stringstream>(std::move(data)), mimeType);
+    TRY_AWAIT(upsertFile(path, std::make_shared<std::stringstream>(std::move(data)), mimeType));
+    co_return result::success();
+} catch (...) {
+    co_return result::current_exception();
 }
 
-std::optional<std::string> BinaryCacheStore::getFileContents(const std::string & path)
+kj::Promise<Result<std::optional<std::string>>>
-{
+BinaryCacheStore::getFileContents(const std::string & path)
+try {
     try {
-        return getFile(path)->drain();
+        co_return TRY_AWAIT(TRY_AWAIT(getFile(path))->drain());
     } catch (NoSuchBinaryCacheFile &) {
-        return std::nullopt;
+        co_return std::nullopt;
     }
+} catch (...) {
+    co_return result::current_exception();
 }
 
 std::string BinaryCacheStore::narInfoFileFor(const StorePath & storePath)
@@ -88,7 +98,7 @@ kj::Promise<Result<void>> BinaryCacheStore::writeNarInfo(ref<NarInfo> narInfo)
 try {
     auto narInfoFile = narInfoFileFor(narInfo->path);
 
-    upsertFile(narInfoFile, narInfo->to_string(*this), "text/x-nix-narinfo");
+    TRY_AWAIT(upsertFile(narInfoFile, narInfo->to_string(*this), "text/x-nix-narinfo"));
 
     {
         auto state_(co_await state.lock());
@@ -178,7 +188,21 @@ try {
             {"root", listNar(narIndex)},
         };
 
-        upsertFile(std::string(info.path.hashPart()) + ".ls", j.dump(), "application/json");
+        try {
+            TRY_AWAIT(
+                upsertFile(std::string(info.path.hashPart()) + ".ls", j.dump(), "application/json")
+            );
+        } catch (ForeignException & exc) {
+            if (exc.is<JSON::exception>()) {
+                warn(
+                    "Skipping NAR listing for path '%1%' due to serialization failure: %2%",
+                    printStorePath(narInfo->path),
+                    exc.what()
+                );
+            } else {
+                throw exc;
+            }
+        }
     }
 
     /* Optionally maintain an index of DWARF debug info files
@@ -198,20 +222,23 @@ try {
 
             ThreadPool threadPool("write debuginfo pool", 25);
 
-            auto doFile = [&](std::string member, std::string key, std::string target) {
+            auto doFile = [&](AsyncIoRoot & aio,
-                checkInterrupt();
+                              std::string member,
-
+                              std::string key,
+                              std::string target) {
                 JSON json;
                 json["archive"] = target;
                 json["member"] = member;
 
                 // FIXME: or should we overwrite? The previous link may point
                 // to a GC'ed file, so overwriting might be useful...
-                if (fileExists(key)) return;
+                if (aio.blockOn(fileExists(key))) {
+                    return;
+                }
 
                 printMsg(lvlTalkative, "creating debuginfo link from '%s' to '%s'", key, target);
 
-                upsertFile(key, json.dump(), "application/json");
+                aio.blockOn(upsertFile(key, json.dump(), "application/json"));
             };
 
             std::regex regex1 = regex::parse("^[0-9a-f]{2}$");
@@ -235,7 +262,9 @@ try {
                     std::string key = "debuginfo/" + buildId;
                     std::string target = "../" + narInfo->url;
 
-                    threadPool.enqueue(std::bind(doFile, std::string(debugPath, 1), key, target));
+                    threadPool.enqueueWithAio(std::bind(
+                        doFile, std::placeholders::_1, std::string(debugPath, 1), key, target
+                    ));
                 }
             }
 
@@ -244,13 +273,16 @@ try {
     }
 
     /* Atomically write the NAR file. */
-    if (repair || !fileExists(narInfo->url)) {
+    if (repair || !TRY_AWAIT(fileExists(narInfo->url))) {
         stats.narWrite++;
-        upsertFile(narInfo->url,
+        TRY_AWAIT(upsertFile(
+            narInfo->url,
             std::make_shared<std::fstream>(fnTemp, std::ios_base::in | std::ios_base::binary),
-            "application/x-nix-nar");
+            "application/x-nix-nar"
-    } else
+        ));
+    } else {
         stats.narWriteAverted++;
+    }
 
     stats.narWriteBytes += info.narSize;
     stats.narWriteCompressedBytes += fileSize;
@@ -331,7 +363,7 @@ try {
     // FIXME: this only checks whether a .narinfo with a matching hash
     // part exists. So ‘f4kb...-foo’ matches ‘f4kb...-bar’, even
     // though they shouldn't. Not easily fixed.
-    co_return fileExists(narInfoFileFor(storePath));
+    co_return TRY_AWAIT(fileExists(narInfoFileFor(storePath)));
 } catch (...) {
     co_return result::current_exception();
 }
@@ -350,34 +382,47 @@ try {
     co_return result::current_exception();
 }
 
-kj::Promise<Result<box_ptr<Source>>> BinaryCacheStore::narFromPath(const StorePath & storePath)
+kj::Promise<Result<box_ptr<AsyncInputStream>>>
+BinaryCacheStore::narFromPath(const StorePath & storePath)
 try {
+    struct NarFromPath : AsyncInputStream
+    {
+        Stats<std::atomic> & stats;
+        box_ptr<AsyncInputStream> decompressed;
+        uint64_t total;
+
+        NarFromPath(
+            Stats<std::atomic> & stats, const std::string & method, box_ptr<AsyncInputStream> file
+        )
+            : stats(stats)
+            , decompressed(makeDecompressionStream(method, std::move(file)))
+        {
+        }
+
+        kj::Promise<Result<size_t>> read(void * buffer, size_t size) override
+        {
+            return decompressed->read(buffer, size).then([&](auto r) {
+                if (r.has_value()) {
+                    if (r.value() > 0) {
+                        total += r.value();
+                    } else {
+                        stats.narRead++;
+                        // stats.narReadCompressedBytes += nar->size(); // FIXME
+                        stats.narReadBytes += total;
+                    }
+                }
+                return r;
+            });
+        }
+    };
+
     auto info_ = TRY_AWAIT(queryPathInfo(storePath)).try_cast<const NarInfo>();
     assert(info_ && "binary cache queryPathInfo didn't return a NarInfo");
     auto & info = *info_;
 
     try {
-        auto file = getFile(info->url);
+        auto file = TRY_AWAIT(getFile(info->url));
-        co_return make_box_ptr<GeneratorSource>(
+        co_return make_box_ptr<NarFromPath>(stats, info->compression, std::move(file));
-            [](auto info, auto file, auto & stats) -> WireFormatGenerator {
-                constexpr size_t buflen = 65536;
-                auto buf = std::make_unique<char[]>(buflen);
-                size_t total = 0;
-                auto decompressor = makeDecompressionSource(info->compression, *file);
-                try {
-                    while (true) {
-                        const auto len = decompressor->read(buf.get(), buflen);
-                        co_yield std::span{buf.get(), len};
-                        total += len;
-                    }
-                } catch (EndOfFile &) {
-                }
-
-                stats.narRead++;
-                // stats.narReadCompressedBytes += nar->size(); // FIXME
-                stats.narReadBytes += total;
-            }(std::move(info), std::move(file), stats)
-        );
     } catch (NoSuchBinaryCacheFile & e) {
         throw SubstituteGone(std::move(e.info()));
     }
@@ -396,7 +441,7 @@ try {
 
     auto narInfoFile = narInfoFileFor(storePath);
 
-    auto data = getFileContents(narInfoFile);
+    auto data = TRY_AWAIT(getFileContents(narInfoFile));
 
     if (!data) co_return result::success(nullptr);
 
@@ -505,32 +550,6 @@ try {
     co_return result::current_exception();
 }
 
-kj::Promise<Result<std::shared_ptr<const Realisation>>>
-BinaryCacheStore::queryRealisationUncached(const DrvOutput & id)
-try {
-    auto outputInfoFilePath = realisationsPrefix + "/" + id.to_string() + ".doi";
-
-    auto data = getFileContents(outputInfoFilePath);
-    if (!data) co_return result::success(nullptr);
-
-    auto realisation = Realisation::fromJSON(
-        json::parse(*data), outputInfoFilePath);
-    co_return std::make_shared<const Realisation>(realisation);
-} catch (...) {
-    co_return result::current_exception();
-}
-
-kj::Promise<Result<void>> BinaryCacheStore::registerDrvOutput(const Realisation& info)
-try {
-    if (diskCache)
-        diskCache->upsertRealisation(getUri(), info);
-    auto filePath = realisationsPrefix + "/" + info.id.to_string() + ".doi";
-    upsertFile(filePath, info.toJSON().dump(), "application/json");
-    co_return result::success();
-} catch (...) {
-    co_return result::current_exception();
-}
-
 ref<FSAccessor> BinaryCacheStore::getFSAccessor()
 {
     return make_ref<RemoteFSAccessor>(ref<Store>(*this), config().localNarCache);
@@ -563,7 +582,7 @@ try {
 
     debug("fetching build log from binary cache '%s/%s'", getUri(), logPath);
 
-    co_return getFileContents(logPath);
+    co_return TRY_AWAIT(getFileContents(logPath));
 } catch (...) {
     co_return result::current_exception();
 }
@@ -573,10 +592,11 @@ BinaryCacheStore::addBuildLog(const StorePath & drvPath, std::string_view log)
 try {
     assert(drvPath.isDerivation());
 
-    upsertFile(
+    TRY_AWAIT(upsertFile(
         "log/" + std::string(drvPath.to_string()),
         (std::string) log, // FIXME: don't copy
-        "text/plain; charset=utf-8");
+        "text/plain; charset=utf-8"
+    ));
     co_return result::success();
 } catch (...) {
     co_return result::current_exception();

lix/libstore/binary-cache-store.hh

@@ -73,23 +73,28 @@ public:
     BinaryCacheStoreConfig & config() override = 0;
     const BinaryCacheStoreConfig & config() const override = 0;
 
-    virtual bool fileExists(const std::string & path) = 0;
+    virtual kj::Promise<Result<bool>> fileExists(const std::string & path) = 0;
 
-    virtual void upsertFile(const std::string & path,
+    virtual kj::Promise<Result<void>> upsertFile(
+        const std::string & path,
         std::shared_ptr<std::basic_iostream<char>> istream,
-        const std::string & mimeType) = 0;
+        const std::string & mimeType
+    ) = 0;
 
-    void upsertFile(const std::string & path,
+    kj::Promise<Result<void>> upsertFile(
+        const std::string & path,
         // FIXME: use std::string_view
         std::string && data,
-        const std::string & mimeType);
+        const std::string & mimeType
+    );
 
     /**
      * Dump the contents of the specified file to a sink.
      */
-    virtual box_ptr<Source> getFile(const std::string & path) = 0;
+    virtual kj::Promise<Result<box_ptr<AsyncInputStream>>> getFile(const std::string & path) = 0;
 
-    virtual std::optional<std::string> getFileContents(const std::string & path);
+    virtual kj::Promise<Result<std::optional<std::string>>> getFileContents(const std::string & path
+    );
 
 public:
 
@@ -146,12 +151,7 @@ public:
         const StorePathSet & references,
         RepairFlag repair) override;
 
-    kj::Promise<Result<void>> registerDrvOutput(const Realisation & info) override;
+    kj::Promise<Result<box_ptr<AsyncInputStream>>> narFromPath(const StorePath & path) override;
-
-    kj::Promise<Result<std::shared_ptr<const Realisation>>>
-    queryRealisationUncached(const DrvOutput &) override;
-
-    kj::Promise<Result<box_ptr<Source>>> narFromPath(const StorePath & path) override;
 
     ref<FSAccessor> getFSAccessor() override;
 

lix/libstore/build/derivation-goal.cc

@@ -1,5 +1,6 @@
 #include "lix/libstore/build/derivation-goal.hh"
 #include "lix/libutil/async.hh"
+#include "lix/libutil/file-descriptor.hh"
 #include "lix/libutil/file-system.hh"
 #include "lix/libstore/build/hook-instance.hh"
 #include "lix/libstore/build/worker.hh"
@@ -10,7 +11,6 @@
 #include "lix/libstore/common-protocol-impl.hh" // IWYU pragma: keep
 #include "lix/libstore/local-store.hh" // TODO remove, along with remaining downcasts
 #include "lix/libstore/build/substitution-goal.hh"
-#include "lix/libstore/build/drv-output-substitution-goal.hh"
 #include "lix/libutil/result.hh"
 #include "lix/libutil/strings.hh"
 
@@ -74,7 +74,7 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath,
 {
     name = fmt(
         "building of '%s' from .drv file",
-        DerivedPath::Built { makeConstantStorePathRef(drvPath), wantedOutputs }.to_string(worker.store));
+        DerivedPath::Built { makeConstantStorePath(drvPath), wantedOutputs }.to_string(worker.store));
     trace("created");
 
     mcExpectedBuilds = worker.expectedBuilds.addTemporarily(1);
@@ -93,7 +93,7 @@ DerivationGoal::DerivationGoal(DrvHasRoot, const StorePath & drvPath, const Basi
 
     name = fmt(
         "building of '%s' from in-memory derivation",
-        DerivedPath::Built { makeConstantStorePathRef(drvPath), drv.outputNames() }.to_string(worker.store));
+        DerivedPath::Built { makeConstantStorePath(drvPath), drv.outputNames() }.to_string(worker.store));
     trace("created");
 
     mcExpectedBuilds = worker.expectedBuilds.addTemporarily(1);
@@ -199,8 +199,6 @@ try {
     /* Get the derivation. It is probably in the eval store, but it might be inthe main store:
 
          - Resolved derivation are resolved against main store realisations, and so must be stored there.
-
-         - Dynamic derivations are built, and so are found in the main store.
      */
     for (auto * drvStore : { &worker.evalStore, &worker.store }) {
         if (TRY_AWAIT(drvStore->isValidPath(drvPath))) {
@@ -222,34 +220,8 @@ try {
 
     parsedDrv = std::make_unique<ParsedDerivation>(drvPath, *drv);
 
-    if (!drv->type().hasKnownOutputPaths())
+    for (auto & i : drv->outputsAndPaths(worker.store))
-        experimentalFeatureSettings.require(Xp::CaDerivations);
+        TRY_AWAIT(worker.store.addTempRoot(i.second.second));
-
-    if (!drv->type().isPure()) {
-        experimentalFeatureSettings.require(Xp::ImpureDerivations);
-
-        for (auto & [outputName, output] : drv->outputs) {
-            auto randomPath = StorePath::random(outputPathName(drv->name, outputName));
-            assert(!TRY_AWAIT(worker.store.isValidPath(randomPath)));
-            initialOutputs.insert({
-                outputName,
-                InitialOutput {
-                    .wanted = true,
-                    .outputHash = impureOutputHash,
-                    .known = InitialOutputStatus {
-                        .path = randomPath,
-                        .status = PathStatus::Absent
-                    }
-                }
-            });
-        }
-
-        co_return co_await gaveUpOnSubstitution();
-    }
-
-    for (auto & i : drv->outputsAndOptPaths(worker.store))
-        if (i.second.second)
-            TRY_AWAIT(worker.store.addTempRoot(*i.second.second));
 
     auto outputHashes = TRY_AWAIT(staticOutputHashes(worker.evalStore, *drv));
     for (auto & [outputName, outputHash] : outputHashes)
@@ -277,14 +249,13 @@ try {
         if (parsedDrv->substitutesAllowed()) {
             for (auto & [outputName, status] : initialOutputs) {
                 if (!status.wanted) continue;
-                if (!status.known)
+                if (!status.known) {
-                    dependencies.add(
+                    // TODO remove somehow
-                        worker.goalFactory().makeDrvOutputSubstitutionGoal(
+                    throw Error(
-                            DrvOutput{status.outputHash, outputName},
+                        "congrats, you hit vestigial CA code. sigh.\n"
-                            buildMode == bmRepair ? Repair : NoRepair
+                        "please report a bug at https://git.lix.systems/lix-project/lix/issues"
-                        )
                     );
-                else {
+                } else {
                     auto * cap = getDerivationCA(*drv);
                     dependencies.add(worker.goalFactory().makePathSubstitutionGoal(
                         status.known->path,
@@ -309,8 +280,6 @@ kj::Promise<Result<Goal::WorkResult>> DerivationGoal::outputsSubstitutionTried()
 try {
     trace("all outputs substituted (maybe)");
 
-    assert(drv->type().isPure());
-
     if (nrFailed > 0 && nrFailed > nrNoSubstituters + nrIncompleteClosure && !settings.tryFallback)
     {
         co_return done(
@@ -397,34 +366,18 @@ try {
     /* The inputs must be built before we can build this goal. */
     inputDrvOutputs.clear();
     if (useDerivation) {
-        std::function<void(ref<SingleDerivedPath>, const DerivedPathMap<StringSet>::ChildNode &)> addWaiteeDerivedPath;
+        auto addWaiteeDerivedPath = [&](DerivedPathOpaque inputDrv, const StringSet & inputNode) {
-
+            if (!inputNode.empty())
-        addWaiteeDerivedPath = [&](ref<SingleDerivedPath> inputDrv, const DerivedPathMap<StringSet>::ChildNode & inputNode) {
-            if (!inputNode.value.empty())
                 dependencies.add(worker.goalFactory().makeGoal(
                     DerivedPath::Built {
                         .drvPath = inputDrv,
-                        .outputs = inputNode.value,
+                        .outputs = inputNode,
                     },
                     buildMode == bmRepair ? bmRepair : bmNormal));
-            for (const auto & [outputName, childNode] : inputNode.childMap)
-                addWaiteeDerivedPath(
-                    make_ref<SingleDerivedPath>(SingleDerivedPath::Built { inputDrv, outputName }),
-                    childNode);
         };
 
-        for (const auto & [inputDrvPath, inputNode] : dynamic_cast<Derivation *>(drv.get())->inputDrvs.map) {
+        for (const auto & [inputDrvPath, inputNode] : dynamic_cast<Derivation *>(drv.get())->inputDrvs) {
-            /* Ensure that pure, non-fixed-output derivations don't
+            addWaiteeDerivedPath(makeConstantStorePath(inputDrvPath), inputNode);
-               depend on impure derivations. */
-            if (experimentalFeatureSettings.isEnabled(Xp::ImpureDerivations) && drv->type().isPure() && !drv->type().isFixed()) {
-                auto inputDrv = TRY_AWAIT(worker.evalStore.readDerivation(inputDrvPath));
-                if (!inputDrv.type().isPure())
-                    throw Error("pure derivation '%s' depends on impure derivation '%s'",
-                        worker.store.printStorePath(drvPath),
-                        worker.store.printStorePath(inputDrvPath));
-            }
-
-            addWaiteeDerivedPath(makeConstantStorePathRef(inputDrvPath), inputNode);
         }
     }
 
@@ -460,8 +413,6 @@ try {
 
 kj::Promise<Result<Goal::WorkResult>> DerivationGoal::repairClosure() noexcept
 try {
-    assert(drv->type().isPure());
-
     /* If we're repairing, we now know that our own outputs are valid.
        Now check whether the other paths in the outputs closure are
        good.  If not, then start derivation goals for the derivations
@@ -488,10 +439,9 @@ try {
     for (auto & i : inputClosure)
         if (i.isDerivation()) {
             auto depOutputs =
-                TRY_AWAIT(worker.store.queryPartialDerivationOutputMap(i, &worker.evalStore));
+                TRY_AWAIT(worker.store.queryDerivationOutputMap(i, &worker.evalStore));
             for (auto & j : depOutputs)
-                if (j.second)
+                outputsToDrv.insert_or_assign(j.second, i);
-                    outputsToDrv.insert_or_assign(*j.second, i);
         }
 
     /* Check each path (slow!). */
@@ -507,7 +457,7 @@ try {
         else
             dependencies.add(worker.goalFactory().makeGoal(
                 DerivedPath::Built {
-                    .drvPath = makeConstantStorePathRef(drvPath2->second),
+                    .drvPath = makeConstantStorePath(drvPath2->second),
                     .outputs = OutputsSpec::All { },
                 },
                 bmRepair));
@@ -572,68 +522,8 @@ try {
     if (useDerivation) {
         auto & fullDrv = *dynamic_cast<Derivation *>(drv.get());
 
-        auto drvType = fullDrv.type();
-        bool resolveDrv = std::visit(overloaded {
-            [&](const DerivationType::InputAddressed & ia) {
-                /* must resolve if deferred. */
-                return ia.deferred;
-            },
-            [&](const DerivationType::ContentAddressed & ca) {
-                return !fullDrv.inputDrvs.map.empty() && (
-                    ca.fixed
-                    /* Can optionally resolve if fixed, which is good
-                       for avoiding unnecessary rebuilds. */
-                    ? experimentalFeatureSettings.isEnabled(Xp::CaDerivations)
-                    /* Must resolve if floating and there are any inputs
-                       drvs. */
-                    : true);
-            },
-            [&](const DerivationType::Impure &) {
-                return true;
-            }
-        }, drvType.raw);
-
-        if (resolveDrv && !fullDrv.inputDrvs.map.empty()) {
-            experimentalFeatureSettings.require(Xp::CaDerivations);
-
-            /* We are be able to resolve this derivation based on the
-               now-known results of dependencies. If so, we become a
-               stub goal aliasing that resolved derivation goal. */
-            std::optional attempt = TRY_AWAIT(fullDrv.tryResolve(worker.store, inputDrvOutputs));
-            if (!attempt) {
-              /* TODO (impure derivations-induced tech debt) (see below):
-                 The above attempt should have found it, but because we manage
-                 inputDrvOutputs statefully, sometimes it gets out of sync with
-                 the real source of truth (store). So we query the store
-                 directly if there's a problem. */
-              attempt = TRY_AWAIT(fullDrv.tryResolve(worker.store, &worker.evalStore));
-            }
-            assert(attempt);
-            Derivation drvResolved { std::move(*attempt) };
-
-            auto pathResolved = TRY_AWAIT(writeDerivation(worker.store, drvResolved));
-
-            auto msg = fmt("resolved derivation: '%s' -> '%s'",
-                worker.store.printStorePath(drvPath),
-                worker.store.printStorePath(pathResolved));
-            act = std::make_unique<Activity>(*logger, lvlInfo, actBuildWaiting, msg,
-                Logger::Fields {
-                       worker.store.printStorePath(drvPath),
-                       worker.store.printStorePath(pathResolved),
-                   });
-
-            auto dependency = worker.goalFactory().makeDerivationGoal(
-                pathResolved, wantedOutputs, buildMode);
-            resolvedDrvGoal = dependency.first;
-
-            TRY_AWAIT(waitForGoals(std::move(dependency)));
-            co_return co_await resolvedFinished();
-        }
-
-        std::function<kj::Promise<Result<void>>(const StorePath &, const DerivedPathMap<StringSet>::ChildNode &)> accumInputPaths;
-
         // NOLINTNEXTLINE(cppcoreguidelines-avoid-capturing-lambda-coroutines)
-        accumInputPaths = [&](const StorePath & depDrvPath, const DerivedPathMap<StringSet>::ChildNode & inputNode) -> kj::Promise<Result<void>> {
+        auto accumInputPaths = [&](const StorePath & depDrvPath, const StringSet & inputNode) -> kj::Promise<Result<void>> {
             try {
                 /* Add the relevant output closures of the input derivation
                    `i' as input paths.  Only add the closures of output paths
@@ -676,21 +566,19 @@ try {
                     }
                 };
 
-                for (auto & outputName : inputNode.value) {
+                for (auto & outputName : inputNode) {
                     TRY_AWAIT(
                         worker.store.computeFSClosure(TRY_AWAIT(getOutput(outputName)), inputPaths)
                     );
                 }
 
-                for (auto & [outputName, childNode] : inputNode.childMap)
-                    TRY_AWAIT(accumInputPaths(TRY_AWAIT(getOutput(outputName)), childNode));
                 co_return result::success();
             } catch (...) {
                 co_return result::current_exception();
             }
         };
 
-        for (auto & [depDrvPath, depNode] : fullDrv.inputDrvs.map)
+        for (auto & [depDrvPath, depNode] : fullDrv.inputDrvs)
             TRY_AWAIT(accumInputPaths(depDrvPath, depNode));
     }
 
@@ -735,8 +623,6 @@ retry:
        other goal can start a build, and if not, the main loop will sleep a few
        seconds and then retry this goal. */
     PathSet lockFiles;
-    /* FIXME: Should lock something like the drv itself so we don't build same
-       CA drv concurrently */
     if (dynamic_cast<LocalStore *>(&worker.store)) {
         /* If we aren't a local store, we might need to use the local store as
            a build remote, but that would cause a deadlock. */
@@ -745,13 +631,8 @@ retry:
         /* FIXME: find some way to lock for scheduling for the other stores so
            a forking daemon with --store still won't farm out redundant builds.
            */
-        for (auto & i : drv->outputsAndOptPaths(worker.store)) {
+        for (auto & i : drv->outputsAndPaths(worker.store)) {
-            if (i.second.second)
+            lockFiles.insert(worker.store.Store::toRealPath(i.second.second));
-                lockFiles.insert(worker.store.Store::toRealPath(*i.second.second));
-            else
-                lockFiles.insert(
-                    worker.store.Store::toRealPath(drvPath) + "." + i.first
-                );
         }
     }
 
@@ -1011,11 +892,11 @@ void runPostBuildHook(
         .program = settings.postBuildHook,
         .environment = hookEnvironment,
         .captureStdout = true,
-        .mergeStderrToStdout = true,
+        .redirections = {{.dup = STDERR_FILENO, .from = STDOUT_FILENO}},
     });
     Finally const _wait([&] {
         try {
-            proc.wait();
+            proc.waitAndCheck();
         } catch (nix::Error & e) {
             e.addTrace(nullptr,
                 "while running the post-build-hook %s for derivation %s",
@@ -1149,88 +1030,6 @@ try {
     co_return result::current_exception();
 }
 
-kj::Promise<Result<Goal::WorkResult>> DerivationGoal::resolvedFinished() noexcept
-try {
-    trace("resolved derivation finished");
-
-    assert(resolvedDrvGoal);
-    auto resolvedDrv = *resolvedDrvGoal->drv;
-    auto & resolvedResult = resolvedDrvGoal->buildResult;
-
-    SingleDrvOutputs builtOutputs;
-
-    if (resolvedResult.success()) {
-        auto resolvedHashes = TRY_AWAIT(staticOutputHashes(worker.store, resolvedDrv));
-
-        StorePathSet outputPaths;
-
-        for (auto & outputName : resolvedDrv.outputNames()) {
-            auto initialOutput = get(initialOutputs, outputName);
-            auto resolvedHash = get(resolvedHashes, outputName);
-            if ((!initialOutput) || (!resolvedHash))
-                throw Error(
-                    "derivation '%s' doesn't have expected output '%s' (derivation-goal.cc/resolvedFinished,resolve)",
-                    worker.store.printStorePath(drvPath), outputName);
-
-            // NOLINTNEXTLINE(cppcoreguidelines-avoid-capturing-lambda-coroutines)
-            auto realisation = TRY_AWAIT([&]() -> kj::Promise<Result<Realisation>> {
-                try {
-                    auto take1 = get(resolvedResult.builtOutputs, outputName);
-                    if (take1) co_return *take1;
-
-                    /* The above `get` should work. But sateful tracking of
-                       outputs in resolvedResult, this can get out of sync with the
-                       store, which is our actual source of truth. For now we just
-                       check the store directly if it fails. */
-                    auto take2 = TRY_AWAIT(
-                        worker.evalStore.queryRealisation(DrvOutput{*resolvedHash, outputName})
-                    );
-                    if (take2) co_return *take2;
-
-                    throw Error(
-                        "derivation '%s' doesn't have expected output '%s' (derivation-goal.cc/resolvedFinished,realisation)",
-                        worker.store.printStorePath(resolvedDrvGoal->drvPath), outputName);
-                } catch (...) {
-                    co_return result::current_exception();
-                }
-            }());
-
-            if (drv->type().isPure()) {
-                auto newRealisation = realisation;
-                newRealisation.id = DrvOutput { initialOutput->outputHash, outputName };
-                newRealisation.signatures.clear();
-                if (!drv->type().isFixed()) {
-                    auto & drvStore = TRY_AWAIT(worker.evalStore.isValidPath(drvPath))
-                        ? worker.evalStore
-                        : worker.store;
-                    newRealisation.dependentRealisations = TRY_AWAIT(
-                        drvOutputReferences(worker.store, *drv, realisation.outPath, &drvStore)
-                    );
-                }
-                signRealisation(newRealisation);
-                TRY_AWAIT(worker.store.registerDrvOutput(newRealisation));
-            }
-            outputPaths.insert(realisation.outPath);
-            builtOutputs.emplace(outputName, realisation);
-        }
-
-        runPostBuildHook(
-            worker.store,
-            *logger,
-            drvPath,
-            outputPaths
-        );
-    }
-
-    auto status = resolvedResult.status;
-    if (status == BuildResult::AlreadyValid)
-        status = BuildResult::ResolvesToAlreadyValid;
-
-    co_return done(status, std::move(builtOutputs));
-} catch (...) {
-    co_return result::current_exception();
-}
-
 HookReply DerivationGoal::tryBuildHook()
 {
     if (!worker.hook.available || !useDerivation) return HookReply::Decline{};
@@ -1241,13 +1040,10 @@ HookReply DerivationGoal::tryBuildHook()
     try {
 
         /* Send the request to the hook. */
-        worker.hook.instance->sink
+        *worker.hook.instance->sink << "try" << (slotToken.valid() ? 1 : 0) << drv->platform
-            << "try"
+                                    << worker.store.printStorePath(drvPath)
-            << (slotToken.valid() ? 1 : 0)
+                                    << parsedDrv->getRequiredSystemFeatures();
-            << drv->platform
+        worker.hook.instance->sink->flush();
-            << worker.store.printStorePath(drvPath)
-            << parsedDrv->getRequiredSystemFeatures();
-        worker.hook.instance->sink.flush();
 
         /* Read the first line of input, which should be a word indicating
            whether the hook wishes to perform the build. */
@@ -1310,7 +1106,7 @@ HookReply DerivationGoal::tryBuildHook()
 
     /* Tell the hook all the inputs that have to be copied to the
        remote system. */
-    hook->sink << CommonProto::write(worker.store, {}, inputPaths);
+    *hook->sink << CommonProto::write({worker.store}, inputPaths);
 
     /* Tell the hooks the missing outputs that have to be copied back
        from the remote system. */
@@ -1321,10 +1117,10 @@ HookReply DerivationGoal::tryBuildHook()
             if (buildMode != bmCheck && status.known && status.known->isValid()) continue;
             missingOutputs.insert(outputName);
         }
-        hook->sink << CommonProto::write(worker.store, {}, missingOutputs);
+        *hook->sink << CommonProto::write({worker.store}, missingOutputs);
     }
 
-    hook->sink = FdSink();
+    hook->sink = nullptr;
     hook->toHook.reset();
 
     /* Create the log file and pipe. */
@@ -1409,13 +1205,7 @@ struct DerivationGoal::InputStream final : private kj::AsyncObject
         : fd(fd)
         , observer(ep, fd, kj::UnixEventPort::FdObserver::OBSERVE_READ)
     {
-        int flags = fcntl(fd, F_GETFL);
+        makeNonBlocking(fd);
-        if (flags < 0) {
-            throw SysError("fcntl(F_GETFL) failed on fd %i", fd);
-        }
-        if (fcntl(fd, F_SETFL, flags | O_NONBLOCK) < 0) {
-            throw SysError("fcntl(F_SETFL) failed on fd %i", fd);
-        }
     }
 
     kj::Promise<std::string_view> read(kj::ArrayPtr<char> buffer)
@@ -1618,43 +1408,12 @@ void DerivationGoal::flushLine()
 }
 
 
-kj::Promise<Result<std::map<std::string, std::optional<StorePath>>>> DerivationGoal::queryPartialDerivationOutputMap()
-try {
-    assert(drv->type().isPure());
-    if (!useDerivation || drv->type().hasKnownOutputPaths()) {
-        std::map<std::string, std::optional<StorePath>> res;
-        for (auto & [name, output] : drv->outputs)
-            res.insert_or_assign(name, output.path(worker.store, drv->name, name));
-        co_return res;
-    } else {
-        for (auto * drvStore : {&worker.evalStore, &worker.store}) {
-            if (TRY_AWAIT(drvStore->isValidPath(drvPath))) {
-                co_return TRY_AWAIT(worker.store.queryPartialDerivationOutputMap(drvPath, drvStore)
-                );
-            }
-        }
-        assert(false);
-    }
-} catch (...) {
-    co_return result::current_exception();
-}
-
 kj::Promise<Result<OutputPathMap>> DerivationGoal::queryDerivationOutputMap()
 try {
-    assert(drv->type().isPure());
+    OutputPathMap res;
-    if (!useDerivation || drv->type().hasKnownOutputPaths()) {
+    for (auto & [name, output] : drv->outputsAndPaths(worker.store))
-        OutputPathMap res;
+        res.insert_or_assign(name, output.second);
-        for (auto & [name, output] : drv->outputsAndOptPaths(worker.store))
+    co_return res;
-            res.insert_or_assign(name, *output.second);
-        co_return res;
-    } else {
-        for (auto * drvStore : {&worker.evalStore, &worker.store}) {
-            if (TRY_AWAIT(drvStore->isValidPath(drvPath))) {
-                co_return TRY_AWAIT(worker.store.queryDerivationOutputMap(drvPath, drvStore));
-            }
-        }
-        assert(false);
-    }
 } catch (...) {
     co_return result::current_exception();
 }
@@ -1662,8 +1421,6 @@ try {
 
 kj::Promise<Result<std::pair<bool, SingleDrvOutputs>>> DerivationGoal::checkPathValidity()
 try {
-    if (!drv->type().isPure()) co_return { false, SingleDrvOutputs{} };
-
     bool checkHash = buildMode == bmRepair;
     auto wantedOutputsLeft = std::visit(overloaded {
         [&](const OutputsSpec::All &) {
@@ -1675,7 +1432,7 @@ try {
     }, wantedOutputs.raw);
     SingleDrvOutputs validOutputs;
 
-    for (auto & i : TRY_AWAIT(queryPartialDerivationOutputMap())) {
+    for (auto & i : TRY_AWAIT(queryDerivationOutputMap())) {
         auto initialOutput = get(initialOutputs, i.first);
         if (!initialOutput)
             // this is an invalid output, gets catched with (!wantedOutputsLeft.empty())
@@ -1684,37 +1441,16 @@ try {
         info.wanted = wantedOutputs.contains(i.first);
         if (info.wanted)
             wantedOutputsLeft.erase(i.first);
-        if (i.second) {
+        auto & outputPath = i.second;
-            auto outputPath = *i.second;
+        info.known = {
-            info.known = {
+            .path = outputPath,
-                .path = outputPath,
+            .status = !TRY_AWAIT(worker.store.isValidPath(outputPath))
-                .status = !TRY_AWAIT(worker.store.isValidPath(outputPath))
+                ? PathStatus::Absent
-                    ? PathStatus::Absent
+                : !checkHash || TRY_AWAIT(worker.pathContentsGood(outputPath))
-                    : !checkHash || TRY_AWAIT(worker.pathContentsGood(outputPath))
+                ? PathStatus::Valid
-                    ? PathStatus::Valid
+                : PathStatus::Corrupt,
-                    : PathStatus::Corrupt,
+        };
-            };
-        }
         auto drvOutput = DrvOutput{info.outputHash, i.first};
-        if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) {
-            if (auto real = TRY_AWAIT(worker.store.queryRealisation(drvOutput))) {
-                info.known = {
-                    .path = real->outPath,
-                    .status = PathStatus::Valid,
-                };
-            } else if (info.known && info.known->isValid()) {
-                // We know the output because it's a static output of the
-                // derivation, and the output path is valid, but we don't have
-                // its realisation stored (probably because it has been built
-                // without the `ca-derivations` experimental flag).
-                TRY_AWAIT(worker.store.registerDrvOutput(
-                    Realisation {
-                        drvOutput,
-                        info.known->path,
-                    }
-                ));
-            }
-        }
         if (info.known && info.known->isValid())
             validOutputs.emplace(i.first, Realisation { drvOutput, info.known->path });
     }
@@ -1811,13 +1547,13 @@ void DerivationGoal::waiteeDone(GoalPtr waitee)
 
     auto & fullDrv = *dynamic_cast<Derivation *>(drv.get());
 
-    auto * nodeP = fullDrv.inputDrvs.findSlot(DerivedPath::Opaque { .path = dg->drvPath });
+    auto * nodeP = get(fullDrv.inputDrvs, dg->drvPath);
     if (!nodeP) return;
-    auto & outputs = nodeP->value;
+    auto & outputs = *nodeP;
 
     for (auto & outputName : outputs) {
         auto buildResult = dg->buildResult.restrictTo(DerivedPath::Built {
-            .drvPath = makeConstantStorePathRef(dg->drvPath),
+            .drvPath = makeConstantStorePath(dg->drvPath),
             .outputs = OutputsSpec::Names { outputName },
         });
         if (buildResult.success()) {

lix/libstore/build/derivation-goal.hh

@@ -87,11 +87,6 @@ struct DerivationGoal : public Goal
     /** The path of the derivation. */
     StorePath drvPath;
 
-    /**
-     * The goal for the corresponding resolved derivation
-     */
-    std::shared_ptr<DerivationGoal> resolvedDrvGoal;
-
     /**
      * The specific outputs that we need to build.
      */
@@ -279,8 +274,6 @@ struct DerivationGoal : public Goal
     virtual kj::Promise<Result<WorkResult>> tryLocalBuild() noexcept;
     kj::Promise<Result<WorkResult>> buildDone() noexcept;
 
-    kj::Promise<Result<WorkResult>> resolvedFinished() noexcept;
-
     /**
      * Is the build hook willing to perform the build?
      */
@@ -299,11 +292,6 @@ struct DerivationGoal : public Goal
      */
     Path openLogFile();
 
-    /**
-     * Sign the newly built realisation if the store allows it
-     */
-    virtual void signRealisation(Realisation&) {}
-
     /**
      * Close the log file.
      */
@@ -338,11 +326,10 @@ protected:
 
 public:
     /**
-     * Wrappers around the corresponding Store methods that first consult the
+     * Wrappers around the corresponding Store method that first consults the
      * derivation.  This is currently needed because when there is no drv file
      * there also is no DB entry.
      */
-    kj::Promise<Result<std::map<std::string, std::optional<StorePath>>>> queryPartialDerivationOutputMap();
     kj::Promise<Result<OutputPathMap>> queryDerivationOutputMap();
 
     /**
@@ -380,8 +367,6 @@ public:
         return false;
     }
 
-    StorePathSet exportReferences(const StorePathSet & storePaths);
-
     JobCategory jobCategory() const override {
         return JobCategory::Build;
     };

lix/libstore/build/drv-output-substitution-goal.cc

@@ -1,164 +0,0 @@
-#include "lix/libstore/build/drv-output-substitution-goal.hh"
-#include "lix/libstore/build-result.hh"
-#include "lix/libutil/async.hh"
-#include "lix/libutil/finally.hh"
-#include "lix/libstore/build/worker.hh"
-#include "lix/libstore/build/substitution-goal.hh"
-#include "lix/libutil/signals.hh"
-#include <kj/array.h>
-#include <kj/async.h>
-#include <kj/vector.h>
-
-namespace nix {
-
-DrvOutputSubstitutionGoal::DrvOutputSubstitutionGoal(
-    const DrvOutput & id,
-    Worker & worker,
-    bool isDependency,
-    RepairFlag repair,
-    std::optional<ContentAddress> ca)
-    : Goal(worker, isDependency)
-    , id(id)
-{
-    name = fmt("substitution of '%s'", id.to_string());
-    trace("created");
-}
-
-
-kj::Promise<Result<Goal::WorkResult>> DrvOutputSubstitutionGoal::workImpl() noexcept
-try {
-    trace("init");
-
-    /* If the derivation already exists, we’re done */
-    if (TRY_AWAIT(worker.store.queryRealisation(id))) {
-        co_return WorkResult{ecSuccess};
-    }
-
-    subs = settings.useSubstitutes ? TRY_AWAIT(getDefaultSubstituters()) : std::list<ref<Store>>();
-    co_return co_await tryNext();
-} catch (...) {
-    co_return result::current_exception();
-}
-
-kj::Promise<Result<Goal::WorkResult>> DrvOutputSubstitutionGoal::tryNext() noexcept
-try {
-    trace("trying next substituter");
-
-    if (!slotToken.valid()) {
-        slotToken = co_await worker.substitutions.acquire();
-    }
-
-    maintainRunningSubstitutions = worker.runningSubstitutions.addTemporarily(1);
-
-    if (subs.size() == 0) {
-        /* None left.  Terminate this goal and let someone else deal
-           with it. */
-        debug("derivation output '%s' is required, but there is no substituter that can provide it", id.to_string());
-
-        if (substituterFailed) {
-            worker.failedSubstitutions++;
-        }
-
-        /* Hack: don't indicate failure if there were no substituters.
-           In that case the calling derivation should just do a
-           build. */
-        co_return WorkResult{substituterFailed ? ecFailed : ecNoSubstituters};
-    }
-
-    sub = subs.front();
-    subs.pop_front();
-
-    /* The async call to a curl download below can outlive `this` (if
-       some other error occurs), so it must not touch `this`. So put
-       the shared state in a separate refcounted object. */
-    downloadState = std::make_shared<DownloadState>();
-    auto pipe = kj::newPromiseAndCrossThreadFulfiller<void>();
-    downloadState->outPipe = kj::mv(pipe.fulfiller);
-
-    downloadState->result =
-        std::async(std::launch::async, [downloadState{downloadState}, id{id}, sub{sub}] {
-            Finally updateStats([&]() { downloadState->outPipe->fulfill(); });
-            ReceiveInterrupts receiveInterrupts;
-            AsyncIoRoot aio;
-            return aio.blockOn(sub->queryRealisation(id));
-        });
-
-    co_await pipe.promise;
-    co_return co_await realisationFetched();
-} catch (...) {
-    co_return result::current_exception();
-}
-
-kj::Promise<Result<Goal::WorkResult>> DrvOutputSubstitutionGoal::realisationFetched() noexcept
-try {
-    maintainRunningSubstitutions.reset();
-    slotToken = {};
-
-    try {
-        outputInfo = downloadState->result.get();
-    } catch (std::exception & e) { // NOLINT(lix-foreign-exceptions)
-        printError(e.what());
-        substituterFailed = true;
-    }
-
-    if (!outputInfo) {
-        co_return co_await tryNext();
-    }
-
-    kj::Vector<std::pair<GoalPtr, kj::Promise<Result<WorkResult>>>> dependencies;
-    for (const auto & [depId, depPath] : outputInfo->dependentRealisations) {
-        if (depId != id) {
-            if (auto localOutputInfo = TRY_AWAIT(worker.store.queryRealisation(depId));
-                localOutputInfo && localOutputInfo->outPath != depPath) {
-                warn(
-                    "substituter '%s' has an incompatible realisation for '%s', ignoring.\n"
-                    "Local:  %s\n"
-                    "Remote: %s",
-                    sub->getUri(),
-                    depId.to_string(),
-                    worker.store.printStorePath(localOutputInfo->outPath),
-                    worker.store.printStorePath(depPath)
-                );
-                co_return co_await tryNext();
-            }
-            dependencies.add(worker.goalFactory().makeDrvOutputSubstitutionGoal(depId));
-        }
-    }
-
-    dependencies.add(worker.goalFactory().makePathSubstitutionGoal(outputInfo->outPath));
-
-    if (!dependencies.empty()) {
-        TRY_AWAIT(waitForGoals(dependencies.releaseAsArray()));
-    }
-    co_return co_await outPathValid();
-} catch (...) {
-    co_return result::current_exception();
-}
-
-kj::Promise<Result<Goal::WorkResult>> DrvOutputSubstitutionGoal::outPathValid() noexcept
-try {
-    assert(outputInfo);
-    trace("output path substituted");
-
-    if (nrFailed > 0) {
-        debug("The output path of the derivation output '%s' could not be substituted", id.to_string());
-        co_return WorkResult{
-            nrNoSubstituters > 0 || nrIncompleteClosure > 0 ? ecIncompleteClosure : ecFailed,
-        };
-    }
-
-    TRY_AWAIT(worker.store.registerDrvOutput(*outputInfo));
-    co_return TRY_AWAIT(finished());
-} catch (...) {
-    co_return result::current_exception();
-}
-
-kj::Promise<Result<Goal::WorkResult>> DrvOutputSubstitutionGoal::finished() noexcept
-try {
-    trace("finished");
-    return {WorkResult{ecSuccess}};
-} catch (...) {
-    return {result::current_exception()};
-}
-
-}

lix/libstore/build/drv-output-substitution-goal.hh

@@ -1,80 +0,0 @@
-#pragma once
-///@file
-
-#include "lix/libutil/notifying-counter.hh"
-#include "lix/libstore/store-api.hh"
-#include "lix/libstore/build/goal.hh"
-#include "lix/libstore/realisation.hh"
-#include <future>
-
-namespace nix {
-
-class Worker;
-
-/**
- * Substitution of a derivation output.
- * This is done in three steps:
- * 1. Fetch the output info from a substituter
- * 2. Substitute the corresponding output path
- * 3. Register the output info
- */
-class DrvOutputSubstitutionGoal : public Goal {
-
-    /**
-     * The drv output we're trying to substitute
-     */
-    DrvOutput id;
-
-    /**
-     * The realisation corresponding to the given output id.
-     * Will be filled once we can get it.
-     */
-    std::shared_ptr<const Realisation> outputInfo;
-
-    /**
-     * The remaining substituters.
-     */
-    std::list<ref<Store>> subs;
-
-    /**
-     * The current substituter.
-     */
-    std::shared_ptr<Store> sub;
-
-    NotifyingCounter<uint64_t>::Bump maintainRunningSubstitutions;
-
-    struct DownloadState
-    {
-        kj::Own<kj::CrossThreadPromiseFulfiller<void>> outPipe;
-        std::future<std::shared_ptr<const Realisation>> result;
-    };
-
-    std::shared_ptr<DownloadState> downloadState;
-
-    /**
-     * Whether a substituter failed.
-     */
-    bool substituterFailed = false;
-
-public:
-    DrvOutputSubstitutionGoal(
-        const DrvOutput & id,
-        Worker & worker,
-        bool isDependency,
-        RepairFlag repair = NoRepair,
-        std::optional<ContentAddress> ca = std::nullopt
-    );
-
-    kj::Promise<Result<WorkResult>> tryNext() noexcept;
-    kj::Promise<Result<WorkResult>> realisationFetched() noexcept;
-    kj::Promise<Result<WorkResult>> outPathValid() noexcept;
-    kj::Promise<Result<WorkResult>> finished() noexcept;
-
-    kj::Promise<Result<WorkResult>> workImpl() noexcept override;
-
-    JobCategory jobCategory() const override {
-        return JobCategory::Substitution;
-    };
-};
-
-}

lix/libstore/build/entry-points.cc

@@ -93,7 +93,7 @@ try {
         }));
         auto & result = results.goals.begin()->second;
         co_return result.result.restrictTo(DerivedPath::Built {
-            .drvPath = makeConstantStorePathRef(drvPath),
+            .drvPath = makeConstantStorePath(drvPath),
             .outputs = OutputsSpec::All {},
         });
     } catch (Error & e) {
@@ -151,7 +151,7 @@ try {
                 Worker::Targets goals;
                 goals.emplace_back(gf.makeGoal(
                     DerivedPath::Built{
-                        .drvPath = makeConstantStorePathRef(*info->deriver),
+                        .drvPath = makeConstantStorePath(*info->deriver),
                         // FIXME: Should just build the specific output we need.
                         .outputs = OutputsSpec::All{},
                     },

lix/libstore/build/hook-instance.cc

@@ -72,12 +72,12 @@ HookInstance::HookInstance()
     toHook = std::move(toHook_.writeSide);
     builderOut = std::move(builderOut_.readSide);
 
-    sink = FdSink(toHook.get());
+    sink = std::make_unique<FdSink>(toHook.get());
     std::map<std::string, Config::SettingInfo> settings;
     globalConfig.getSettings(settings, true);
     for (auto & setting : settings)
-        sink << 1 << setting.first << setting.second.value;
+        *sink << 1 << setting.first << setting.second.value;
-    sink << 0;
+    *sink << 0;
 }
 
 

lix/libstore/build/hook-instance.hh

@@ -29,7 +29,7 @@ struct HookInstance
      */
     Pid pid;
 
-    FdSink sink;
+    std::unique_ptr<FdSink> sink;
 
     std::map<ActivityId, Activity> activities;
 

lix/libstore/build/local-derivation-goal.cc

@@ -13,10 +13,11 @@
 #include "lix/libutil/archive.hh"
 #include "lix/libstore/daemon.hh"
 #include "lix/libutil/regex.hh"
+#include "lix/libutil/file-descriptor.hh"
+#include "lix/libutil/file-system.hh"
 #include "lix/libutil/result.hh"
 #include "lix/libutil/topo-sort.hh"
 #include "lix/libutil/json.hh"
-#include "lix/libutil/cgroup.hh"
 #include "lix/libstore/build/personality.hh"
 #include "lix/libutil/namespaces.hh"
 #include "lix/libstore/build/child.hh"
@@ -25,6 +26,7 @@
 #include "lix/libutil/mount.hh"
 #include "lix/libutil/strings.hh"
 #include "lix/libutil/thread-name.hh"
+#include "platform/linux.hh"
 
 #include <cstddef>
 #include <exception>
@@ -403,58 +405,11 @@ void LocalDerivationGoal::cleanupPostOutputsRegisteredModeNonCheck()
 
 kj::Promise<Result<void>> LocalDerivationGoal::startBuilder()
 try {
-    if ((buildUser && buildUser->getUIDCount() != 1)
+#if !(__linux__)
-        #if __linux__
+    if (buildUser && buildUser->getUIDCount() != 1) {
-        || settings.useCgroups
-        #endif
-        )
-    {
-        #if __linux__
-        experimentalFeatureSettings.require(Xp::Cgroups);
-
-        auto cgroupFS = getCgroupFS();
-        if (!cgroupFS)
-            throw Error("cannot determine the cgroups file system");
-
-        auto ourCgroups = getCgroups("/proc/self/cgroup");
-        auto ourCgroup = ourCgroups[""];
-        if (ourCgroup == "")
-            throw Error("cannot determine cgroup name from /proc/self/cgroup");
-
-        auto ourCgroupPath = canonPath(*cgroupFS + "/" + ourCgroup);
-
-        if (!pathExists(ourCgroupPath))
-            throw Error("expected cgroup directory '%s'", ourCgroupPath);
-
-        static std::atomic<unsigned int> counter{0};
-
-        cgroup = buildUser
-            ? fmt("%s/nix-build-uid-%d", ourCgroupPath, buildUser->getUID())
-            : fmt("%s/nix-build-pid-%d-%d", ourCgroupPath, getpid(), counter++);
-
-        debug("using cgroup '%s'", *cgroup);
-
-        /* When using a build user, record the cgroup we used for that
-           user so that if we got interrupted previously, we can kill
-           any left-over cgroup first. */
-        if (buildUser) {
-            auto cgroupsDir = settings.nixStateDir + "/cgroups";
-            createDirs(cgroupsDir);
-
-            auto cgroupFile = fmt("%s/%d", cgroupsDir, buildUser->getUID());
-
-            if (pathExists(cgroupFile)) {
-                auto prevCgroup = readFile(cgroupFile);
-                destroyCgroup(prevCgroup);
-            }
-
-            writeFile(cgroupFile, *cgroup);
-        }
-
-        #else
         throw Error("cgroups are not supported on this platform");
-        #endif
     }
+#endif
 
     /* Make sure that no other processes are executing under the
        sandbox uids. This must be done before any chownToBuilder()
@@ -480,17 +435,55 @@ try {
         });
     }
 
-    /* Create a temporary directory where the build will take
+    try {
-       place. */
+        auto buildDir = worker.buildDirOverride.value_or(settings.buildDir.get());
-    tmpDir = createTempDir(
-        settings.buildDir.get().value_or(""),
-        "nix-build-" + std::string(drvPath.name()),
-        false,
-        false,
-        0700
-    );
 
-    chownToBuilder(tmpDir);
+        createDirs(buildDir);
+
+        /* Create a temporary directory where the build will take
+           place. */
+        tmpDir =
+            createTempDir(buildDir, "nix-build-" + std::string(drvPath.name()), false, false, 0700);
+    } catch (SysError & e) {
+        /*
+         * Fallback to the global tmpdir and create a safe space there
+         * only if it's a permission error.
+         */
+        if (e.errNo != EACCES) {
+            throw;
+        }
+
+        auto globalTmp = defaultTempDir();
+        createDirs(globalTmp);
+#if __APPLE__
+        /* macOS filesystem namespacing does not exist, to avoid breaking builds, we need to weaken
+         * the mode bits on the top-level directory. This avoids issues like
+         * https://github.com/NixOS/nix/pull/11031. */
+        constexpr int toplevelDirMode = 0755;
+#else
+        constexpr int toplevelDirMode = 0700;
+#endif
+        auto nixBuildsTmp = createTempDir(
+            globalTmp, fmt("nix-builds-%s", geteuid()), false, false, toplevelDirMode
+        );
+        warn(
+            "Failed to use the system-wide build directory '%s', falling back to a temporary "
+            "directory inside '%s'",
+            settings.buildDir.get(),
+            nixBuildsTmp
+        );
+        worker.buildDirOverride = nixBuildsTmp;
+        tmpDir = createTempDir(
+            nixBuildsTmp, "nix-build-" + std::string(drvPath.name()), false, false, 0700
+        );
+    }
+    /* The TOCTOU between the previous mkdir call and this open call is unavoidable due to
+     * POSIX semantics.*/
+    tmpDirFd = AutoCloseFD{open(tmpDir.c_str(), O_RDONLY | O_NOFOLLOW | O_DIRECTORY)};
+    if (!tmpDirFd)
+        throw SysError("failed to open the build temporary directory descriptor '%1%'", tmpDir);
+
+    chownToBuilder(tmpDirFd);
 
     for (auto & [outputName, status] : initialOutputs) {
         /* Set scratch path we'll actually use during the build.
@@ -823,7 +816,6 @@ try {
     co_return result::current_exception();
 }
 
-
 Pid LocalDerivationGoal::startChild(std::function<void()> openSlave) {
     return startProcess([&]() {
         openSlave();
@@ -858,8 +850,13 @@ void LocalDerivationGoal::initTmpDir() {
                 auto hash = hashString(HashType::SHA256, i.first);
                 std::string fn = ".attr-" + hash.to_string(Base::Base32, false);
                 Path p = tmpDir + "/" + fn;
-                writeFile(p, rewriteStrings(i.second, inputRewrites));
+                /* TODO(jade): we should have BorrowedFD instead of OwnedFD. */
-                chownToBuilder(p);
+                AutoCloseFD passAsFileFd{openat(tmpDirFd.get(), fn.c_str(), O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC | O_EXCL | O_NOFOLLOW, 0666)};
+                if (!passAsFileFd) {
+                    throw SysError("opening `passAsFile` file in the sandbox '%1%'", p);
+                }
+                writeFile(passAsFileFd, rewriteStrings(i.second, inputRewrites));
+                chownToBuilder(passAsFileFd);
                 env[i.first + "Path"] = tmpDirInSandbox + "/" + fn;
             }
         }
@@ -975,6 +972,13 @@ void LocalDerivationGoal::chownToBuilder(const Path & path)
         throw SysError("cannot change ownership of '%1%'", path);
 }
 
+void LocalDerivationGoal::chownToBuilder(const AutoCloseFD & fd)
+{
+    if (!buildUser) return;
+    if (fchown(fd.get(), buildUser->getUID(), buildUser->getGID()) == -1)
+        throw SysError("cannot change ownership of file '%1%'", fd.guessOrInventPath());
+}
+
 
 void LocalDerivationGoal::runChild()
 {
@@ -1105,7 +1109,7 @@ void LocalDerivationGoal::runChild()
                 /* N.B. it is realistic that these paths might not exist. It
                    happens when testing Nix building fixed-output derivations
                    within a pure derivation. */
-                for (auto & path : { "/etc/resolv.conf", "/etc/services", "/etc/hosts" })
+                for (auto & path : { "/etc/services", "/etc/hosts" })
                     if (pathExists(path)) {
                         // Copy the actual file, not the symlink, because we don't know where
                         // the symlink is pointing, and we don't want to chase down the entire
@@ -1126,6 +1130,11 @@ void LocalDerivationGoal::runChild()
                         copyFile(path, chrootRootDir + path, { .followSymlinks = true });
                     }
 
+                if (pathExists("/etc/resolv.conf")) {
+                    const auto resolvConf = rewriteResolvConf(readFile("/etc/resolv.conf"));
+                    writeFile(chrootRootDir + "/etc/resolv.conf", resolvConf);
+                }
+
                 if (settings.caFile != "" && pathExists(settings.caFile)) {
                     // For the same reasons as above, copy the CA certificates file too.
                     // It should be even less likely to change during the build than resolv.conf.
@@ -1253,6 +1262,36 @@ void LocalDerivationGoal::runChild()
             if (setuid(sandboxUid()) == -1)
                 throw SysError("setuid failed");
 
+            if (runPasta) {
+                // wait for the pasta interface to appear. pasta can't signal us when
+                // it's done setting up the namespace, so we have to wait for a while
+                AutoCloseFD fd(socket(PF_INET, SOCK_DGRAM, IPPROTO_IP));
+                if (!fd) throw SysError("cannot open IP socket");
+
+                struct ifreq ifr;
+                strcpy(ifr.ifr_name, LinuxLocalDerivationGoal::PASTA_NS_IFNAME);
+                // wait two minutes for the interface to appear. if it does not do so
+                // we are either grossly overloaded, or pasta startup failed somehow.
+                static constexpr int SINGLE_WAIT_US = 1000;
+                static constexpr int TOTAL_WAIT_US = 120'000'000;
+                for (unsigned tries = 0; ; tries++) {
+                    if (tries > TOTAL_WAIT_US / SINGLE_WAIT_US) {
+                        throw Error(
+                            "sandbox network setup timed out, please check daemon logs for "
+                            "possible error output."
+                        );
+                    } else if (ioctl(fd.get(), SIOCGIFFLAGS, &ifr) == 0) {
+                        if ((ifr.ifr_ifru.ifru_flags & IFF_UP) != 0) {
+                            break;
+                        }
+                    } else if (errno == ENODEV) {
+                        usleep(SINGLE_WAIT_US);
+                    } else {
+                        throw SysError("cannot get loopback interface flags");
+                    }
+                }
+            }
+
             setUser = false;
         }
 #endif
@@ -1424,6 +1463,8 @@ void LocalDerivationGoal::runChild()
 
         if (getEnv("_NIX_TEST_NO_SANDBOX") != "1") {
             Strings sandboxArgs;
+            sandboxArgs.push_back("_NIX_BUILD_TOP");
+            sandboxArgs.push_back(tmpDir);
             sandboxArgs.push_back("_GLOBAL_TMP_DIR");
             sandboxArgs.push_back(globalTmpDir);
             if (allowLocalNetworking) {
@@ -1739,14 +1780,14 @@ try {
             return res;
         };
 
-        auto newInfoFromCA = [&](const DerivationOutput::CAFloating outputHash) -> ValidPathInfo {
+        auto newInfoFromCA = [&](ContentAddressMethod method, HashType hashType) -> ValidPathInfo {
             auto st = get(outputStats, outputName);
             if (!st)
                 throw BuildError(
                     "output path %1% without valid stats info",
                     actualPath);
-            if (outputHash.method == ContentAddressMethod { FileIngestionMethod::Flat } ||
+            if (method == ContentAddressMethod { FileIngestionMethod::Flat } ||
-                outputHash.method == ContentAddressMethod { TextIngestionMethod {} })
+                method == ContentAddressMethod { TextIngestionMethod {} })
             {
                 /* The output path should be a regular file without execute permission. */
                 if (!S_ISREG(st->st_mode) || (st->st_mode & S_IXUSR) != 0)
@@ -1771,11 +1812,11 @@ try {
                     }
                     assert(false);
                 },
-            }, outputHash.method.raw);
+            }, method.raw);
-            auto got = computeHashModulo(outputHash.hashType, oldHashPart, input).first;
+            auto got = computeHashModulo(hashType, oldHashPart, input).first;
 
             auto optCA = ContentAddressWithReferences::fromPartsOpt(
-                outputHash.method,
+                method,
                 std::move(got),
                 rewriteRefs());
             if (!optCA) {
@@ -1840,10 +1881,7 @@ try {
                 movePath(actualPath, tmpOutput);
                 copyFile(tmpOutput, actualPath, { .deleteAfter = true });
 
-                auto newInfo0 = newInfoFromCA(DerivationOutput::CAFloating {
+                auto newInfo0 = newInfoFromCA(dof.ca.method, wanted.type);
-                    .method = dof.ca.method,
-                    .hashType = wanted.type,
-                });
 
                 /* Check wanted hash */
                 assert(newInfo0.ca);
@@ -1854,12 +1892,16 @@ try {
                     anyHashMismatchSeen = true;
                     // XXX: shameless layering violation hack that makes the hash mismatch error at least not utterly worthless
                     auto guessedUrl = getOr(drv->env, "urls", getOr(drv->env, "url", "(unknown)"));
-                    delayedException = std::make_exception_ptr(
+                    delayedException = std::make_exception_ptr(BuildError(
-                        BuildError("hash mismatch in fixed-output derivation '%s':\n likely URL: %s\n  specified: %s\n     got:    %s",
+                        "hash mismatch in fixed-output derivation '%s':\n    likely URL: %s\n     "
-                            worker.store.printStorePath(drvPath),
+                        "specified: %s\n           got: %s\n expected path: %s\n      got path: %s",
-                            guessedUrl,
+                        worker.store.printStorePath(drvPath),
-                            wanted.to_string(Base::SRI, true),
+                        guessedUrl,
-                            got.to_string(Base::SRI, true)));
+                        wanted.to_string(Base::SRI, true),
+                        got.to_string(Base::SRI, true),
+                        worker.store.printStorePath(dof.path(worker.store, drv->name, outputName)),
+                        worker.store.printStorePath(newInfo0.path)
+                    ));
                 }
                 if (!newInfo0.references.empty()) {
                     std::string references;
@@ -1879,24 +1921,6 @@ try {
 
                 return newInfo0;
             },
-
-            [&](const DerivationOutput::CAFloating & dof) {
-                return newInfoFromCA(dof);
-            },
-
-            [&](const DerivationOutput::Deferred &) -> ValidPathInfo {
-                // No derivation should reach that point without having been
-                // rewritten first
-                assert(false);
-            },
-
-            [&](const DerivationOutput::Impure & doi) {
-                return newInfoFromCA(DerivationOutput::CAFloating {
-                    .method = doi.method,
-                    .hashType = doi.hashType,
-                });
-            },
-
         }, output->raw);
 
         /* FIXME: set proper permissions in restorePath() so
@@ -1912,10 +1936,8 @@ try {
            floating CA derivations and hash-mismatching fixed-output
            derivations. */
         std::optional<PathLock> dynamicOutputLock;
-        auto optFixedPath = output->path(worker.store, drv->name, outputName);
+        auto fixedPath = output->path(worker.store, drv->name, outputName);
-        if (!optFixedPath ||
+        if (worker.store.printStorePath(fixedPath) != finalDestPath) {
-            worker.store.printStorePath(*optFixedPath) != finalDestPath)
-        {
             assert(newInfo.ca);
             dynamicOutputLock = TRY_AWAIT(lockPathAsync(worker.store.toRealPath(finalDestPath)));
         }
@@ -1926,9 +1948,10 @@ try {
                 /* Path already exists, need to replace it */
                 replaceValidPath(worker.store.toRealPath(finalDestPath), actualPath);
                 actualPath = worker.store.toRealPath(finalDestPath);
-            } else if (buildMode == bmCheck) {
+            } else if (buildMode == bmCheck && TRY_AWAIT(worker.store.isValidPath(newInfo.path))) {
-                /* Path already exists, and we want to compare, so we leave out
+                /* Path already exists, and we want to compare, so we
-                   new path in place. */
+                   don't replace the previously existing output with
+                   the new one. */
             } else if (TRY_AWAIT(worker.store.isValidPath(newInfo.path))) {
                 /* Path already exists because CA path produced by something
                    else. No moving needed. */
@@ -1943,8 +1966,12 @@ try {
 
         auto & localStore = getLocalStore();
 
-        if (buildMode == bmCheck) {
+        // Check determinism and run the diff hook for input-addressed
+        // paths if we're in check mode.
+        // TODO: implement this for content-addressed paths too.
+        if (buildMode == bmCheck && !newInfo.ca) {
 
+            // We can only do this if we have a previous output path to compare.
             if (!TRY_AWAIT(worker.store.isValidPath(newInfo.path))) continue;
             ValidPathInfo oldInfo(*TRY_AWAIT(worker.store.queryPathInfo(newInfo.path)));
             if (newInfo.narHash != oldInfo.narHash) {
@@ -2015,12 +2042,11 @@ try {
             }
             throw NotDeterministic(msg.str());
         }
-        /* In case of fixed-output derivations, if there are
+        /* In case of fixed-output derivations with hash mismatches,
-           mismatches on `--check` an error must be thrown as this is
+           we don't want to rethrow the exception until later so that
-           also a source for non-determinism. */
+           the unexpected path is still registered as valid. */
-        if (delayedException)
+        if (!delayedException)
-            std::rethrow_exception(delayedException);
+            co_return TRY_AWAIT(assertPathValidity());
-        co_return TRY_AWAIT(assertPathValidity());
     }
 
     /* Apply output checks. */
@@ -2061,12 +2087,6 @@ try {
             },
             .outPath = newInfo.path
         };
-        if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)
-            && drv->type().isPure())
-        {
-            signRealisation(thisRealisation);
-            TRY_AWAIT(worker.store.registerDrvOutput(thisRealisation));
-        }
         builtOutputs.emplace(outputName, thisRealisation);
     }
 
@@ -2075,11 +2095,6 @@ try {
     co_return result::current_exception();
 }
 
-void LocalDerivationGoal::signRealisation(Realisation & realisation)
-{
-    getLocalStore().signRealisation(realisation);
-}
-
 
 kj::Promise<Result<void>> LocalDerivationGoal::checkOutputs(const std::map<std::string, ValidPathInfo> & newlyBuiltOutputs, const std::map<std::string, StorePath> & alreadyRegisteredOutputs)
 try {