release: 2.93.4 "Bici Bici"

Release produced with releng/create_release.xsh

Change-Id: I24bd93272db7b333f509f8d77820dbf5db5339e3

.editorconfig

@@ -33,7 +33,3 @@ max_line_length = 0
 [meson.build]
 indent_style = space
 indent_size = 2
-
-[*.json]
-indent_style = space
-indent_size = 4

.gitignore

@@ -39,5 +39,3 @@ buildtime.bin
 
 # Python compiled files from the code generators and test suite
 *.pyc
-
-**/.idea

doc/manual/change-authors.yml

@@ -66,13 +66,13 @@ delan:
   forgejo: delan
   github: delan
 
-delroth:
-  github: delroth
-
 detroyejr:
   display_name: Jonathan De Troye
   github: detroyejr
 
+edef:
+  github: edef1c
+
 edolstra:
   display_name: Eelco Dolstra
   github: edolstra
@@ -101,9 +101,6 @@ ian-h-chamberlain:
   forgejo: ian-h-chamberlain
   github: ian-h-chamberlain
 
-infinisil:
-  github: infinisil
-
 isabelroses:
   forgejo: isabelroses
   github: isabelroses
@@ -158,17 +155,9 @@ midnightveil:
   forgejo: midnightveil
   github: midnightveil
 
-nan-git:
-  display_name: NaN-git
-  github: NaN-git
-
 ncfavier:
   github: ncfavier
 
-not-my-profile:
-  display_name: Martin Fischer
-  github: not-my-profile
-
 p-e-meunier:
   display_name: Pierre-Etienne Meunier
   github: P-E-Meunier
@@ -211,6 +200,9 @@ roberth:
   display_name: Robert Hensing
   github: roberth
 
+sandydoo:
+  github: sandydoo
+
 seppel3210:
   github: Seppel3210
 
@@ -240,11 +232,6 @@ vigress8:
   forgejo: vigress8
   github: vigress8
 
-vlinkz:
-  display_name: Victor Fuentes
-  forgejo: vlinkz
-  github: vlinkz
-
 winter:
   forgejo: winter
   github: winterqt

doc/manual/rl-next/arg-identifier.md

@@ -1,16 +0,0 @@
----
-synopsis: First argument to `--arg`/`--argstr` must be a valid Nix identifier
-issues: [fj#496]
-category: "Breaking Changes"
-credits: [ma27]
----
-
-The first argument to `--arg`/`--argstr` must be a valid Nix identifier, i.e.
-`nix-build --arg config.allowUnfree true` is now rejected.
-
-This is because that invocation is a false friend since it doesn't set
-`{ config = { allowUnfree = true; }; }`, but `{ "config.allowUnfree" = true; }`.
-
-The idea is to change the behavior to the latter in the long-term. For that,
-non-identifiers started giving a warning since 2.92 and are now rejected to give people
-who depend on that a chance to notice and potentially weigh in on the discussion.

doc/manual/rl-next/async-query.md

@@ -1,20 +0,0 @@
----
-synopsis: "Improved susbtituter query speed"
-issues: []
-cls: []
-category: Improvements
-credits: [horrors]
----
-
-The code used to query substituters for derivations has been rewritten slightly
-to take advantage of our asynchronous runtime. Such queries run for every build
-that could download from substituters and processes every derivation that isn't
-yet present on the local system. Previously Lix would use `http-connections` to
-limit query concurrency, even for modern caches that support HTTP/2 and have no
-limit on how many queries can be run concurrently on one single connection. Lix
-no longer does this, resulting in approximately 60% reduction in query time for
-medium-sized closures (e.g. NixOS system closures) during testing, although the
-exact number depends greatly on local network latency and generally improves as
-latency increases. Unlike previously setting `http-connections` to `1` or other
-low values no longer brings a massive penalty in query performance if the cache
-in use by the querying system supports HTTP/2 (as e.g. `cache.nixos.org` does).

doc/manual/rl-next/build-dir-mandatory.md

@@ -1,12 +0,0 @@
----
-synopsis: "`build-dir` no longer defaults to `temp-dir`"
-cls: [3453]
-category: "Fixes"
-credits: [horrors]
----
-
-The directory in which temporary build directories are created no longer defaults
-to the value of the `temp-dir` setting to avoid builders making their directories
-world-accessible. This behavior has been used to escape the build sandbox and can
-cause build impurities even when not used maliciously. We now default to `builds`
-in `NIX_STATE_DIR` (which is `/nix/var/nix/builds` in the default configuration).

doc/manual/rl-next/cgroups-delegation.md

@@ -1,69 +0,0 @@
----
-synopsis: New cgroup delegation model
-issues: [fj#537, fj#77]
-cls: [3230]
-category: "Breaking Changes"
-credits: [raito, horrors, lheckemann]
----
-
-Builds using cgroups (i.e. `use-cgroups = true` and the experimental feature
-`cgroups`) now always delegate a cgroup tree to the sandbox.
-
-Compared to the original C++ Nix project, our delegation includes the
-`subtree_control` file as well, which means that the sandbox can disable
-certain controllers in its own cgroup tree.
-
-This is a breaking change because this requires the Nix daemon to run with an
-already delegated cgroup tree by the service manager.
-
-## How to setup the cgroup tree with systemd?
-
-systemd offers knobs to perform the required setup using:
-
-```
-[Service]
-Delegate=yes
-DelegateSubtree=supervisor
-```
-
-These directives are now included in our systemd packaging.
-
-## What about using Nix as root without connecting to the daemon?
-
-Builds run as `root` without connecting to the daemon relying on the cgroup
-feature are now broken, i.e.
-
-```console
-# nix-build --use-cgroups --sandbox ... # will not work
-```
-
-Consider doing instead:
-
-```console
-# systemd-run --same-dir --wait -p Delegate=yes -p DelegateSubgroup=supervisor nix-build --use-cgroups ...
-```
-
-If you need to disable cgroups temporarily, remember that you can do
-`NIX_CONF='include /etc/nix/nix.conf\nuse-cgroups = false' nix-build ...` or
-`nix-build --no-use-cgroups ...`.
-
-## What about other service managers than systemd?
-
-systemd has a [documentation](https://systemd.io/CGROUP_DELEGATION/) on how to
-handle cgroup delegation from service management perspective.
-
-If your service manager adheres to systemd semantics, e.g. writing an extended
-attribute `user.delegate=1` on the delegated cgroup tree directory and moving
-the `nix-daemon` process inside a cgroup tree to respect the inner process
-rule, then, the feature will work as well.
-
-## Why is the cgroup feature still experimental?
-
-While the cgroup feature unlocks many use cases, its behavior and integration (e.g. user experience), especially at scale on build farms or in multi-tenant environments, are not yet fully matured. There’s also potential for deeper systemd integration (e.g. using slices and scopes) that has not been fully explored.
-
-To avoid locking in an unstable interface, we’re keeping the experimental flag until we have validated the feature across a broader range of scenarios, including but not limited to:
-
-* Nix as root
-* Hydra-style build farms
-* Forgejo CI runners
-* Shared remote builders

doc/manual/rl-next/deprecated-derivation-types.md

@@ -1,16 +0,0 @@
----
-synopsis: Deprecation of CA derivations, dynamic derivations, and impure derivations
-issues: [fj#815]
-cls: []
-significance: significant
-category: Miscellany
-credits: []
----
-
-Content-addressed derivations are now deprecated and slated for removal in Lix 2.94.
-We're doing this because the CA derivation system has been a known cause of problems
-and inconsistencies, is unmaintained, habitually makes improving the store code very
-difficult (or blocks such improvements outright), and is beset by a number of design
-flaws that in our opinion cannot be fixed without a full reimplementation from zero.
-Dynamic derivations and impure derivations are built on the CA derivation framework,
-and owing to this they too are deprecated and slated for removal in another release.

doc/manual/rl-next/fix-gc-links.md

@@ -1,13 +0,0 @@
----
-synopsis: "nix-store --delete: always remove obsolete hardlinks"
-issues: []
-cls: [3188]
-category: Fixes
-credits: [lheckemann]
----
-
-Deleting specific paths using `nix-store --delete` or `nix store
-delete` previously did not delete hard links created by `nix-store
---optimise` even if they became obsolete, unless _all_ of the given
-paths were deleted successfully. Now, hard links are always cleaned
-up, even if some of the given paths could not be deleted.

doc/manual/rl-next/fix-gc-stats.md

@@ -1,24 +0,0 @@
----
-synopsis: "Report GC statistics correctly"
-issues: []
-cls: [3188]
-category: Fixes
-credits: [lheckemann]
----
-
-Deleting specific paths using `nix-store --delete` or `nix store delete` previously did
-not report statistics correctly when some of the paths could not be deleted, even if
-others were deleted:
-
-```
-$ nix store delete /nix/store/9bwryidal9q3g91cjm6xschfn4ikd82q-hello-2.12.1 --delete-closure -v
-finding garbage collector roots...
-deleting '/nix/store/9bwryidal9q3g91cjm6xschfn4ikd82q-hello-2.12.1'
-0 store paths deleted, 0.00 MiB freed
-error: Cannot delete some of the given paths because they are still alive. Paths not deleted:
-         k9bxzr1l92r5y6mihrkbpbr3fmc8qszx-libidn2-2.3.8
-         mbx9ii53lzjlrsnlrfmzpwm33ynljwdn-libunistring-1.3
-         rf8hcy6bldxdqc0g6q1dcka1vh47x69s-xgcc-14.2.1.20250322-libgcc
-         vbrdc5wgzn0w1zdp10xd2favkjn5fk7y-glibc-2.40-66
-       To find out why, use nix-store --query --roots and nix-store --query --referrers.
-```

doc/manual/rl-next/ignore-try-default.md

@@ -1,24 +0,0 @@
----
-synopsis: Repl debugger uses `--ignore-try` by default
-issues: [lix#666]
-cls: [3488]
-category: Breaking Changes
-credits: [jade]
----
-Previously, using the debugger meant that exceptions thrown in `builtins.tryEval` would trigger the debugger.
-
-However, this caught nixpkgs initialization code, which is unhelpful in the majority of cases, so we changed the default.
-
-To get the old behaviour, use `--no-ignore-try`.
-
-```
-$ nix repl --debugger --expr 'with import <nixpkgs> {}; pkgs.hello'
-Lix 2.94.0-dev-pre20250625-9a59106
-Type :? for help.
-error: file 'nixpkgs-overlays' was not found in the Nix search path (add it using $NIX_PATH or -I)
-
-This exception occurred in a 'tryEval' call. Use --ignore-try to skip these.
-
-Added 13 variables.
-nix-repl>
-```

doc/manual/rl-next/infallible-build-dirs.md

@@ -1,25 +0,0 @@
----
-synopsis: "Fallback to safe temp dir when build-dir is unwritable"
-issues: [fj#876]
-cls: [3501]
-category: "Fixes"
-credits: ["raito", "horrors"]
----
-
-Non-daemon builds started failing with a permission error after introducing the `build-dir` option:
-
-```
-$ nix build --store ~/scratch nixpkgs#hello --rebuild
-error: creating directory '/nix/var/nix/builds/nix-build-hello-2.12.2.drv-0': Permission denied
-```
-
-This happens because:
-
-1. These builds are not run via the daemon, which owns `/nix/var/nix/builds`.
-2. The user lacks permissions for that path.
-
-We considered making `build-dir` a store-level option and defaulting it to `<chroot-root>/nix/var/nix/builds` for chroot stores, but opted instead for a fallback: if the default fails, Nix now creates a safe build directory under `/tmp`.
-
-To avoid CVE-2025-52991, the fallback uses an extra path component between `/tmp` and the build dir.
-
-**Note**: this fallback clutters `/tmp` with build directories that are not cleaned up. To prevent this, explicitly set `build-dir` to a path managed by Lix, even for local workloads.

doc/manual/rl-next/integer-coercion.md

@@ -1,47 +0,0 @@
----
-synopsis: Experimental integer coercion in interpolated strings
-issues: []
-cls: [3198]
-category: "Features"
-credits: [raito, delroth, horrors, winter]
----
-
-Ever tried interpolating a port number in Lix and ended up with something like this?
-
-```nix
-"http://${config.network.host}:${builtins.toString config.network.port}/"
-```
-
-You're not alone. Thousands of Lix users suffer every day from excessive `builtins.toString` syndrome. It’s 2025, and we still have to cast integers to use them in strings.
-
-To address this, Lix introduces the **`coerce-integers`** experimental feature. When enabled, interpolated integers within `"${...}"` are automatically coerced to strings. This allows writing:
-
-```nix
-"http://${config.network.host}:${config.network.port}/"
-```
-
-without additional conversion.
-
-To enable the feature, you need to add `coerce-integers` to your set of experimental features.
-
-### Stabilization criteria
-
-The `coerce-integers` feature is experimental and limited strictly to string interpolation (`"${...}"`). Before stabilization, the following must hold:
-
-1. **Interpolation-only**
-   Coercion must not occur outside interpolation. Expressions like `"" + 42` must continue to fail.
-
-2. **Expectation that no explicit cast are being observed**
-   Cases observing explicit coercion (e.g., via `tryEval` gadget or similar) are expected not to be load-bearing in actual production code.
-
-### Timeline for stabilization
-
-If the feature proves safe and is widely adopted across typical usage (e.g., actual configurations in the wild turning on the flag, non-trivial out-of-tree projects using it), the experimental flag will be removed **after six months of active use or two Lix releases**, whichever is longer.
-
-This avoids locking the feature in experimental status indefinitely, as happened with Flakes, while allowing time for validation and ecosystem integration.
-
-### What about coercing floats or more?
-
-Coercion beyond integers -- such as for floats or other types -- is **not planned**, even under an experimental flag. Questions like "what is the canonical string representation of a float?" involve subtle and context-dependent trade-offs. Without a robust and principled mechanism to define and audit such behavior, introducing broader coercion risks setting unintended and hard-to-reverse precedents. The scope of `coerce-integers` is intentionally narrow and will remain so.
-
-In terms of outlook, a proposal like https://git.lix.systems/lix-project/lix/issues/835 could pave the way for a better solution.

doc/manual/rl-next/log-output-terminal-codes.md

@@ -1,11 +0,0 @@
----
-synopsis: Fix handling of OSC codes in terminal output
-issues: [fj#160]
-cls: [3143]
-category: Fixes
-credits: [lilyball]
----
-
-OSC codes in terminal output are now handled correctly, where OSC 8 (hyperlink) is preserved any
-time color codes are allowed and all other OSC codes are stripped out. This applies not only to
-output from build commands but also to rendered documentation in the REPL.

doc/manual/rl-next/mismatch-check-path.md

@@ -1,14 +0,0 @@
----
-synopsis: Better debuggability on fixed-output hash mismatches
-issues: []
-cls: []
-category: Improvements
-credits: [lheckemann]
----
-
-Fixed-output derivation hash mismatch error messages will now include the path that was
-produced unexpectedly, and this path will be registered as valid even if `--check`
-(`nix-store`, `nix-build`) or `--rebuild` (`nix build`) was passed. This makes comparing
-the expected path with the obtained path easier, and is useful for debugging when
-upstreams modify previously-published releases or when changes in fixed-output
-derivations' dependencies affect their output unexpectedly.

doc/manual/rl-next/nix-instantiate-raw.md

@@ -1,12 +0,0 @@
----
-synopsis: "Add --raw flag to `nix-instantiate --eval` for unescaped output"
-issues: []
-prs: [gh#12119]
-cls: [2886]
-category: Improvements
-credits: [not-my-profile, infinisil, raito]
----
-
-The `nix-instantiate --eval` command now supports a `--raw` flag. When used,
-the result must be coercible to a string (as with `${...}`) and is printed
-verbatim, without quotes or escaping.

doc/manual/rl-next/nix-store-ls-read-nar-listings.md

@@ -1,12 +0,0 @@
----
-synopsis: Allow `nix store ls` to read nar listings from binary cache stores.
-issues: []
-cls: [3225]
-category: Improvements
-credits: [vlinkz]
----
-
-The `nix store ls` command now supports reading `.ls` nar listings from binary cache stores.
-If a listing is detected for the store path being queried, the nar is no longer downloaded.
-These nar listings are available in binary cache stores where the `write-nar-listing` option is
-enabled, such as cache.nixos.org.

doc/manual/rl-next/pasta.md

@@ -1,20 +0,0 @@
----
-synopsis: "Fixed output derivations can be run using `pasta` network isolation"
-cls: [3452]
-issues: [fj#285]
-category: "Breaking Changes"
-credits: [horrors, puck]
----
-
-Fixed output derivations traditionally run in the host network namespace.
-On Linux this allows such derivations to communicate with other sandboxes
-or the host using the abstract Unix domains socket namespace; this hasn't
-been unproblematic in the past and has been used in two distinct exploits
-to break out of the sandbox. For this reason fixed output derivations can
-now run in a network namespace (provided by [`pasta`]), restricted to TCP
-and UDP communication with the rest of the world. When enabled this could
-be a breaking change and we classify it as such, even though we don't yet
-enable or require such isolation by default. We may enforce this in later
-releases of Lix once we have sufficient confidence that breakage is rare.
-
-[`pasta`]: https://passt.top/

doc/manual/rl-next/reliance-on-bash-for-ssh.md

@@ -1,21 +0,0 @@
----
-synopsis: Remove reliance on Bash for remote stores via SSH
-issues: [fj#830, fj#805, fj#304]
-cls: [3159]
-category: "Fixes"
-credits: [raito]
----
-
-The pre-flight `echo started` handshake -- added years ago to catch race conditions -- has been removed.
-
-After removal of connection sharing in Lix 2.93, it required a Bash-compatible shell and a standard `echo`, so it failed on:
-
-  * builders protected by `ForceCommand` wrappers (e.g. `nix-remote-build`),
-  * BusyBox / initrd images with no Bash,
-  * hosts using non-POSIX shells such as Nushell.
-
-The race the probe once addressed was tied to SSH connection-sharing -- since connection-sharing code has already been removed, the probe is now pointless.
-
-Real connection or protocol errors are now left to SSH/Nix to report directly.
-
-This is technically a breaking change if you had scripts that relied on the literal "started" which needs to be updated to rely on other signals, e.g., exit codes.

doc/manual/rl-next/remove-old-wires.md

@@ -1,20 +0,0 @@
----
-synopsis: Remove support for daemon protocols before 2.18
-issues: []
-cls: [3249]
-significance: significant
-category: "Breaking Changes"
-credits: [horrors]
----
-
-Support for daemon wire protocols belonging to Nix 2.18 or older have been
-removed. This impacts clients connecting to the local daemon socket or any
-remote builder configured using the `ssh-ng` protocol. Builders configured
-with the `ssh` protocol are still accessible from clients such as Nix 2.3.
-Additionally Lix will not be able to connect to an old daemon locally, and
-remote build connections to old daemons is likewise limited to `ssh` urls.
-
-We have decided to take this step because the old protocols are very badly
-tested (if at all), maintenance overhead is high, and a number of problems
-with their design makes it infeasible to remain backwards compatible while
-we move Lix to a more modern RPC mechanism with better versioning support.

doc/manual/rl-next/remove-xp-derivations.md

@@ -1,17 +0,0 @@
----
-synopsis: Remove impure derivations and dynamic derivations
-issues: [fj#815]
-cls: [3210]
-significance: significant
-category: "Breaking Changes"
-credits: [horrors]
----
-
-The `impure-derivations` and `dynamic-derivations` experimental feature have
-been removed.
-
-New impure or dynamic derivations cannot be created from this point forward, and
-any such pre-existing store derivations canot be read or built any more.
-Derivation outputs created by building such a derivation are still valid
-until garbage collected; existing store derivations can only be garbage
-collected.

doc/manual/rl-next/repl-overlays-pure.md

@@ -1,20 +0,0 @@
----
-synopsis: "repl-overlays now work in the debugger for flakes"
-issues: [fj#777]
-cls: [3398]
-category: Fixes
-credits: [jade]
----
-Due to a bug, it was previously not possible to use the debugger on flakes with repl-overlays, or with pure evaluation in general:
-
-```
-$ nix repl --pure-eval
-Lix 2.94.0-dev-pre20250617-87d99da
-Type :? for help.
-Loading 'repl-overlays'...
-error: access to absolute path '/Users/jade/.config/nix/repl.nix' is forbidden in pure eval mode (use '--impure' to override)
-```
-
-This is now fixed.
-The contents of the repl-overlays file itself (i.e. most typically the top level lambda in it) will be evaluated in impure mode.
-It may be necessary to use `builtins.seq` to force the impure operations to happen first if one wants to do impure operations inside a repl-overlays file in pure evaluation mode.

doc/manual/rl-next/symbol-value-reuse.md

@@ -1,32 +0,0 @@
----
-synopsis: Symbols reuses once-allocated Value to reduce garbage collected allocations
-issues: []
-cls: [3308, 3300, 3314, 3310, 3312, 3313]
-category: Improvements
-credits: [raito, horrors, thubrecht, nan-git]
----
-
-In the Lix evaluator, **symbols** represent immutable strings, like those used
-for attribute names.
-
-In evaluator design, such strings are typically [**interned**](https://en.wikipedia.org/wiki/String_interning), stored uniquely
-to save memory, and Lix inherits this approach from the original C++ codebase.
-
-However, some builtins, like `builtins.attrNames`, must return a `Value` type
-that can represent any Nix value (strings, integers, lists, etc.).
-
-Before this change, these builtins would create lists of `Value` objects by
-allocating them through the garbage collector, copying the symbol’s string
-content each time.
-
-This allocation is unnecessary if the interned symbols themselves also hold a
-`Value` representation allocated outside the garbage collector, since these
-live for the full duration of evaluation.
-
-As a result, this reduces the number of allocations, leading to:
-
-* A significant drop in maximum [resident set memory](https://en.wikipedia.org/wiki/Resident_set_size) (RSS), with some large-scale
-  tests showing up to 11% (about 500 MiB) savings in large colmena deployments.
-* A slight decrease in CPU usage during Nix evaluations.
-
-This change is inspired by https://github.com/NixOS/nix/pull/13258 but the approach is different.

doc/manual/rl-next/uid-range-requires-cgroups.md

@@ -1,15 +0,0 @@
----
-synopsis: uid-range depends on cgroups
-issues: []
-cls: [3230]
-category: "Breaking Changes"
-credits: [raito, horrors]
----
-
-`uid-range` builds now depends on `cgroups`, an experimental feature.
-
-`uid-range` builds already depended upon `auto-allocate-uids`, another experimental feature.
-
-The rationale for doing so is that `uid-range` provides a sandbox with many
-UIDs, this is useful for re-mapping them into a nested namespace, e.g. a
-container.

doc/manual/src/SUMMARY.md

@@ -20,6 +20,7 @@
   - [Basic Package Management](package-management/basic-package-mgmt.md)
   - [Profiles](package-management/profiles.md)
   - [Garbage Collection](package-management/garbage-collection.md)
+    - [Garbage Collector Roots](package-management/garbage-collector-roots.md)
   - [Sharing Packages Between Machines](package-management/sharing-packages.md)
     - [Serving a Nix store via HTTP](package-management/binary-cache-substituter.md)
     - [Copying Closures via SSH](package-management/copy-closure.md)

doc/manual/src/command-ref/nix-env/install.md

@@ -148,8 +148,8 @@ To copy the store path with symbolic name `gcc` from another profile:
 $ nix-env --install --from-profile /nix/var/nix/profiles/foo gcc
 ```
 
-To install a specific [store derivation](@docroot@/glossary.md#gloss-store-derivation)
-(typically created by `nix-instantiate`):
+To install a specific [store derivation] (typically created by
+`nix-instantiate`):
 
 ```console
 $ nix-env --install /nix/store/fibjb1bfbpm5mrsxc4mh2d8n37sxh91i-gcc-3.4.3.drv

doc/manual/src/command-ref/nix-instantiate.md

@@ -5,7 +5,7 @@
 # Synopsis
 
 `nix-instantiate`
-  [`--parse` | `--eval` [`--strict`] [`--raw`] [`--json`] [`--xml`] ]
+  [`--parse` | `--eval` [`--strict`] [`--json`] [`--xml`] ]
   [`--read-write-mode`]
   [`--arg` *name* *value*]
   [{`--attr`| `-A`} *attrPath*]
@@ -107,11 +107,6 @@ See that section for complete details (`nix-build --help`), but in summary, a pa
     > This option can cause non-termination, because lazy data
     > structures can be infinitely large.
 
-  - `--raw`
-    When used with `--eval`, the result must be coercible to a string, i.e.,
-    something that can be converted using `${...}`. The output is
-    printed exactly as-is, with no quotes, escaping, or trailing newline.
-
   - `--json`\
     When used with `--eval`, print the resulting value as an JSON
     representation of the abstract syntax tree rather than as a Nix expression.

doc/manual/src/command-ref/nix-store/realise.md

@@ -15,6 +15,7 @@ Each of *paths* is processed as follows:
   1. If it is not [valid], substitute the store derivation file itself.
   2. Realise its [output paths]:
     - Try to fetch from [substituters] the [store objects] associated with the output paths in the store derivation's [closure].
+      - With [content-addressed derivations] (experimental): Determine the output paths to realise by querying content-addressed realisation entries in the [Nix database].
     - For any store paths that cannot be substituted, produce the required store objects. This involves first realising all outputs of the derivation's dependencies and then running the derivation's [`builder`](@docroot@/language/derivations.md#attr-builder) executable. <!-- TODO: Link to build process page #8888 -->
 - Otherwise, and if the path is not already valid: Try to fetch the associated [store objects] in the path's [closure] from [substituters].
 
@@ -27,6 +28,7 @@ If no substitutes are available and no store derivation is given, realisation fa
 [store objects]: @docroot@/glossary.md#gloss-store-object
 [closure]: @docroot@/glossary.md#gloss-closure
 [substituters]: @docroot@/command-ref/conf-file.md#conf-substituters
+[content-addressed derivations]: @docroot@/contributing/experimental-features.md#xp-feature-ca-derivations
 [Nix database]: @docroot@/glossary.md#gloss-nix-database
 
 The resulting paths are printed on standard output.

doc/manual/src/contributing/hacking.md

@@ -39,7 +39,7 @@ $ nix-shell -A native-clangStdenvPackages
 
 ### Building from the development shell
 
-Run a clean build and test with `just clean setup build install test`.
+Run a clean build and test with `just clean build install test`.
 
 You can also run the unit tests and integration tests separately:
 

doc/manual/src/contributing/testing.md

@@ -449,6 +449,9 @@ I grepped `lix/` for `get[eE]nv\("` to find the mentions in Lix code.
 - `NIX_CLIENT_PACKAGE` - Runs the test suite against an alternate Nix client with the current daemon.
 
   **Expected value**: something like `/nix/store/...-nix-2.18.2`
+- `NIX_TESTS_CA_BY_DEFAULT` - Pass `__contentAddressed`, `outputHashMode` and `outputHashAlgo` to builds of some input-addressed derivations in the test suite.
+
+  **Expected value**: 1
 - `TEST_DATA` - Not an environment variable! This is used in repl characterization tests to refer to `tests/functional/repl_characterization/data`.
   More specifically, that path is replaced with the string `$TEST_DATA` in output for reproducibility.
 - `TEST_HOME` (output) - Set to the temporary directory that is set as `$HOME` inside the tests, underneath `$TEST_ROOT`.

doc/manual/src/glossary.md

@@ -41,6 +41,12 @@
 
   [realise]: #gloss-realise
 
+- [content-addressed derivation]{#gloss-content-addressed-derivation}
+
+  A derivation which has the
+  [`__contentAddressed`](./language/advanced-attributes.md#adv-attr-__contentAddressed)
+  attribute set to `true`.
+
 - [fixed-output derivation]{#gloss-fixed-output-derivation}
 
   A derivation which includes the
@@ -108,13 +114,14 @@
 - [input-addressed store object]{#gloss-input-addressed-store-object}
 
   A store object produced by building a
+  non-[content-addressed](#gloss-content-addressed-derivation),
   non-[fixed-output](#gloss-fixed-output-derivation)
   derivation.
 
 - [output-addressed store object]{#gloss-output-addressed-store-object}
 
   A [store object] whose [store path] is determined by its contents.
-  This includes derivations and the outputs of [fixed-output derivations](#gloss-fixed-output-derivation).
+  This includes derivations, the outputs of [content-addressed derivations](#gloss-content-addressed-derivation), and the outputs of [fixed-output derivations](#gloss-fixed-output-derivation).
 
 - [substitute]{#gloss-substitute}
 

doc/manual/src/language/advanced-attributes.md

@@ -209,8 +209,15 @@ Derivations can declare some infrequently used optional attributes.
 
   - [`__contentAddressed`]{#adv-attr-__contentAddressed}
     > **Warning**
-    > This attribute is part of a removed [experimental feature](@docroot@/contributing/experimental-features.md).
-    > Setting this flag *will* cause eval errors.
+    > This attribute is part of an [experimental feature](@docroot@/contributing/experimental-features.md).
+    >
+    > To use this attribute, you must enable the
+    > [`ca-derivations`](@docroot@/contributing/experimental-features.md#xp-feature-ca-derivations) experimental feature.
+    > For example, in [nix.conf](../command-ref/conf-file.md) you could add:
+    >
+    > ```
+    > extra-experimental-features = ca-derivations
+    > ```
 
     If this attribute is set to `true`, then the derivation
     outputs will be stored in a content-addressed location rather than the

doc/manual/src/package-management/garbage-collection.md

@@ -71,62 +71,3 @@ $ nix-collect-garbage -d
 ```
 
 is a quick and easy way to clean up your system.
-
-## Garbage Collector Roots
-
-### Explicit roots
-
-All store paths to which there are symlinks in the directory
-`prefix/nix/var/nix/gcroots` will be used as roots by the garbage
-collector. For instance, the following command makes the path
-`/nix/store/d718ef...-foo` a root of the collector:
-
-```console
-$ ln -s /nix/store/d718ef...-foo /nix/var/nix/gcroots/bar
-```
-
-That is, after this command, the garbage collector will not remove
-`/nix/store/d718ef...-foo` or any of its dependencies.
-
-Subdirectories of `prefix/nix/var/nix/gcroots` are also searched for
-symlinks.
-
-Symlinks may also point to paths outside the nix store. If the
-destination of the symlink is itself a symlink to a store path, it
-is also considered a root. This style of GC root is called an
-"indirect root", and is created by tools like `nix-build` to avoid
-garbage-collecting paths that are being used on-the-fly rather than
-installed in profiles.
-
-
-### In-use roots
-
-Lix will also perform a best-effort detection of paths that are in use
-by running processes when scanning for garbage collection roots, to
-avoid removing paths that are still needed by running processes.
-
-Exact details vary between platforms, but the following will generally
-be taken into account:
-
-- Executables in the store that are currently running;
-- Other files in the store that are mapped into a process's address space (e.g. shared libraries);
-- Files in the store to which processes have open handles;
-- Store paths found in processes' environment variables.
-
-Note that this detection is susceptible to missing paths that may still be in use for multiple reasons:
-
-- Time-of-check-to-time-of-use (TOCTTOU): new processes may appear
-  after Lix has enumerated the currently running processes, and will
-  not be taken into account;
-- Access privileges: if the garbage collection is not running as the
-  root user (this is typically the case for single-user
-  installations), it will not be able to scan processes belonging to
-  other users;
-- Other types of references: store paths may be stored in parts of the
-  filesystem (e.g. databases) or process memory (e.g. environment
-  variables changed since the start of the process) that Lix does not
-  scan.
-
-For this reason, it is recommended to create explicit roots whenever
-using store paths that aren't obtained from some existing explicit GC
-root.

doc/manual/src/package-management/garbage-collector-roots.md

@@ -0,0 +1,18 @@
+# Garbage Collector Roots
+
+The roots of the garbage collector are all store paths to which there
+are symlinks in the directory `prefix/nix/var/nix/gcroots`. For
+instance, the following command makes the path
+`/nix/store/d718ef...-foo` a root of the collector:
+
+```console
+$ ln -s /nix/store/d718ef...-foo /nix/var/nix/gcroots/bar
+```
+
+That is, after this command, the garbage collector will not remove
+`/nix/store/d718ef...-foo` or any of its dependencies.
+
+Subdirectories of `prefix/nix/var/nix/gcroots` are also searched for
+symlinks. Symlinks to non-store paths are followed and searched for
+roots, but symlinks to non-store paths *inside* the paths reached in
+that way are not followed to prevent infinite recursion.

doc/manual/src/protocols/derivation-aterm.md

@@ -2,8 +2,18 @@
 
 For historical reasons, [derivations](@docroot@/glossary.md#gloss-store-derivation) are stored on-disk in [ATerm](https://homepages.cwi.nl/~daybuild/daily-books/technology/aterm-guide/aterm-guide.html) format.
 
-Derivations are serialised in the following format:
+Derivations are serialised in one of the following formats:
 
-```
-Derive(...)
-```
+- ```
+  Derive(...)
+  ```
+
+  For all stable derivations.
+
+- ```
+  DrvWithVersion(<version-string>, ...)
+  ```
+
+  The only `version-string`s that are in use today are for [experimental features](@docroot@/contributing/experimental-features.md):
+
+  - `"xp-dyn-drv"` for the [`dynamic-derivations`](@docroot@/contributing/experimental-features.md#xp-feature-dynamic-derivations) experimental feature.

doc/manual/src/release-notes/rl-2.93.md

@@ -1,4 +1,242 @@
 # Lix 2.93 "Bici Bici" (2025-05-09)
+# Lix 2.93.4 (2026-05-04)
+## Fixes
+
+- `build-dir` no longer defaults to `temp-dir` [cl/3453](https://gerrit.lix.systems/c/lix/+/3453)
+
+  The directory in which temporary build directories are created no longer defaults
+  to the value of the `temp-dir` setting to avoid builders making their directories
+  world-accessible. This behavior has been used to escape the build sandbox and can
+  cause build impurities even when not used maliciously. We now default to `builds`
+  in `NIX_STATE_DIR` (which is `/nix/var/nix/b` in the default configuration).
+
+  Many thanks to [eldritch horrors](https://git.lix.systems/pennae) for this.
+
+- Fix develop shells for derivations with escape codes [fj#991](https://git.lix.systems/lix-project/lix/issues/991) [cl/4154](https://gerrit.lix.systems/c/lix/+/4154) [cl/4155](https://gerrit.lix.systems/c/lix/+/4155)
+
+  ASCII control characters (including `\e`, used for ANSI escape codes) in derivation variables are now correctly escaped for `nix develop` and `nix print-dev-env`, instead of erroring.
+
+  Many thanks to [Qyriad](https://git.lix.systems/Qyriad) for this.
+
+- Fix nix develop for derivations that rejects dependencies with structured attrs [fj#997](https://git.lix.systems/lix-project/lix/issues/997) [cl/4182](https://gerrit.lix.systems/c/lix/+/4182) [cl/4214](https://gerrit.lix.systems/c/lix/+/4214)
+
+  For the sake of concision, we refer to `disallowedReferences` in what follows,
+  but all output checks were equally fixed:
+  `{dis,}allowed{References,Requisites}`.
+
+  Derivations can define *output checks* to reject unwanted dependencies, such as
+  interpreters like `bash` or compilers like `gcc`. This can be done in two ways:
+
+  * **Legacy style**: `disallowedReferences = [ ... ]` in the environment.
+  * **Structured attrs**: `outputChecks.<output>.disallowedReferences = [ ... ]`,
+    typically used in `__json`.
+
+  Only the structured form supports derivations with multiple outputs.
+
+  `nix develop` internally rewrites derivations to create development shells. It
+  relied on the legacy `disallowedReferences`, and failed to honor the structured
+  variant. This led to broken shells in cases where `bashInteractive` was
+  explicitly disallowed using structured output checks, e.g. `nix develop
+  nixpkgs#systemd` after the "bash-less NixOS" changes.
+
+  This fix teaches `nix develop` to respect structured output checks, restoring
+  support for such derivations.
+
+  Many thanks to [Raito Bezarius](https://git.lix.systems/raito) for this.
+
+- `nix-shell` default shell directory is not `/tmp` anymore for `$NIX_BUILD_TOP` [fj#940](https://git.lix.systems/lix-project/lix/issues/940)
+
+  Previously, Lix `nix-shell`s could exit non-zero status when `stdenv`'s `dumpVars` phase failed to write to `$NIX_BUILD_TOP/env-vars`, despite `dumpVars` being intended as a debugging aid.
+
+  This happens when `TMPDIR` is not set and defaults therefore to `/tmp`, resulting in a `/tmp/env-vars` global file that every `nix-shell` wants to write.
+
+  We fix this issue by reusing a pre-created, unique, and writable location, as the build top directory, avoiding shell exiting from write failures silently.
+
+  Many thanks to [Raito Bezarius](https://git.lix.systems/raito) for this.
+
+- Fix unsigned overflow leading to out-of-band write in the NAR parser [cl/5537](https://gerrit.lix.systems/c/lix/+/5537)
+
+  The NAR parser contained an unsigned integer overflow that could be used by an
+  attacker to write arbitrary data to an unknown memory location and possibly
+  achieve code execution. A successful attack on the system-wide Lix daemon
+  could lead to privilege escalation to root. Any process that involves NAR
+  serialization could trigger this issue, including (but not limited to)
+
+    - local user interaction, whether the users are trusted or untrusted
+    - malicious substituters sending malformed NARs
+    - remote builders sending malformed build results
+    - remote daemons sending malformed inputs when requesting remote builds
+
+  Successful attacks using this bug require ASLR weakening of some sort, whether
+  by architecture constraints (e.g. on 32 bit systems, where little randomization
+  is possible) or system configuration (e.g. low ASLR entropy when loading
+  libraries), and millions of attempts. Local attacks can be mounted in less than
+  an hour. Remote builds typically require a fresh SSH connection for each build
+  and are thus less susceptible. Only one attempt can be made by substituters for
+  every build using substituters, they are thus not a likely vector for attacks.
+
+  At the time of writing, MITRE has not assigned this a CVE yet.
+
+  Many thanks to [eldritch horrors](https://git.lix.systems/pennae), [Raito Bezarius](https://git.lix.systems/raito), [edef](https://github.com/edef1c), and [sandydoo](https://github.com/sandydoo) for this.
+
+
+
+
+# Lix 2.93.3 (2025-07-22)
+## Improvements
+
+- `--keep-failed` chowns the build directory to the user that request the build [cl/3678](https://gerrit.lix.systems/c/lix/+/3678)
+
+  Running a build with `--keep-failed` now chowns the temporary directory from the
+  builder user and group to the user that request the build if the build came from
+  a local user connected to the daemon. This makes inspecting failed derivations a
+  lot easier. On Linux the build directory made visible to the user will not be in
+  the same path as it was in the sandbox and continuing builds will usually break.
+
+  Many thanks to [eldritch horrors](https://git.lix.systems/pennae) for this.
+
+
+
+
+# Lix 2.93.2 (2025-06-30)
+## Fixes
+
+- Revert CVE-2025-52992 failed mitigation [fj#883](https://git.lix.systems/lix-project/lix/issues/883) [fj#887](https://git.lix.systems/lix-project/lix/issues/887) [cl/3444](https://gerrit.lix.systems/c/lix/+/3444) [cl/3528](https://gerrit.lix.systems/c/lix/+/3528)
+
+  Following the initial mitigation of **CVE-2025-52992** in `cl/3444`, we
+  received reports of **unexpected deletion of in-use store paths**.
+
+  Upon investigation, we found that the patch did **not correctly cancel all
+  automatic deleters**, resulting in potentially critical path loss during normal
+  operation.
+
+  Given the severity and time-sensitive nature of the situation ([see incident
+  report](https://lix.systems/blog/2025-06-27-lix-critical-bug/)), we evaluated
+  possible options to repair the behavior safely. However, we concluded that a
+  rushed fix would either
+
+  * **Overdelete**, i.e. breaking running systems, or,
+  * **Underdelete**, effectively **reopening CVE-2025-52992** while leaving
+    orphaned paths behind.
+
+  As **CVE-2025-52992 has no known exploit vector**, and correctness is critical
+  in the Lix project, we have **fully reverted the previous mitigations**.
+
+  The affected patches (`cl/3444`) have been rolled back for the time being.
+
+  Moving forward, the Lix team will rework this code path in a **long-term,
+  correctness-first fix** on the main branch. We will explore backporting it to
+  stable channels once its safety is assured.
+
+  We are deeply sorry for the stability incident and the Lix team remain
+  available for assisting you in recovering your systems.
+
+  Many thanks to [Raito Bezarius](https://git.lix.systems/raito) and [eldritch horrors](https://git.lix.systems/pennae) for this.
+
+- Fallback to safe temp dir when build-dir is unwritable [fj#876](https://git.lix.systems/lix-project/lix/issues/876) [cl/3501](https://gerrit.lix.systems/c/lix/+/3501)
+
+  Non-daemon builds started failing with a permission error after introducing the `build-dir` option:
+
+  ```
+  $ nix build --store ~/scratch nixpkgs#hello --rebuild
+  error: creating directory '/nix/var/nix/builds/nix-build-hello-2.12.2.drv-0': Permission denied
+  ```
+
+  This happens because:
+
+  1. These builds are not run via the daemon, which owns `/nix/var/nix/builds`.
+  2. The user lacks permissions for that path.
+
+  We considered making `build-dir` a store-level option and defaulting it to `<chroot-root>/nix/var/nix/builds` for chroot stores, but opted instead for a fallback: if the default fails, Nix now creates a safe build directory under `/tmp`.
+
+  To avoid CVE-2025-52991, the fallback uses an extra path component between `/tmp` and the build dir.
+
+  **Note**: this fallback clutters `/tmp` with build directories that are not cleaned up. To prevent this, explicitly set `build-dir` to a path managed by Lix, even for local workloads.
+
+  Many thanks to [Raito Bezarius](https://git.lix.systems/raito) and [eldritch horrors](https://git.lix.systems/pennae) for this.
+
+
+
+
+# Lix 2.93.1 (2025-06-23)
+## Breaking Changes
+
+- Fixed output derivations can be run using `pasta` network isolation [fj#285](https://git.lix.systems/lix-project/lix/issues/285) [cl/3442](https://gerrit.lix.systems/c/lix/+/3442)
+
+  Fixed output derivations traditionally run in the host network namespace.
+  On Linux this allows such derivations to communicate with other sandboxes
+  or the host using the abstract Unix domains socket namespace; this hasn't
+  been unproblematic in the past and has been used in two distinct exploits
+  to break out of the sandbox. For this reason fixed output derivations can
+  now run in a network namespace (provided by [`pasta`]), restricted to TCP
+  and UDP communication with the rest of the world. When enabled this could
+  be a breaking change and we classify it as such, even though we don't yet
+  enable or require such isolation by default. We may enforce this in later
+  releases of Lix once we have sufficient confidence that breakage is rare.
+
+  [`pasta`]: https://passt.top/
+
+  Many thanks to [eldritch horrors](https://git.lix.systems/pennae) and [puck](https://git.lix.systems/puck) for this.
+
+
+## Fixes
+
+- Always clean up scratch paths after derivations failed to build [cl/3444](https://gerrit.lix.systems/c/lix/+/3444)
+
+  Previously, scratch paths created during builds were not always cleaned up if
+  the derivation failed, potentially leaving behind unnecessary temporary files
+  or directories in the Nix store.
+
+  This fix ensures that such paths are consistently removed after a failed build,
+  improving Nix store hygiene, hardening Lix against mis-reuse of failed builds
+  scratch paths.
+
+  Many thanks to [Raito Bezarius](https://git.lix.systems/raito) and [eldritch horrors](https://git.lix.systems/pennae) for this.
+
+- `build-dir` no longer defaults to `temp-dir` [cl/3443](https://gerrit.lix.systems/c/lix/+/3443)
+
+  The directory in which temporary build directories are created no longer defaults
+  to the value of the `temp-dir` setting to avoid builders making their directories
+  world-accessible. This behavior has been used to escape the build sandbox and can
+  cause build impurities even when not used maliciously. We now default to `builds`
+  in `NIX_STATE_DIR` (which is `/nix/var/nix/builds` in the default configuration).
+
+  Many thanks to [eldritch horrors](https://git.lix.systems/pennae) for this.
+
+- Remove reliance on Bash for remote stores via SSH [fj#830](https://git.lix.systems/lix-project/lix/issues/830) [fj#805](https://git.lix.systems/lix-project/lix/issues/805) [fj#304](https://git.lix.systems/lix-project/lix/issues/304) [cl/3159](https://gerrit.lix.systems/c/lix/+/3159)
+
+  The pre-flight `echo started` handshake -- added years ago to catch race conditions -- has been removed.
+
+  After removal of connection sharing in Lix 2.93, it required a Bash-compatible shell and a standard `echo`, so it failed on:
+
+    * builders protected by `ForceCommand` wrappers (e.g. `nix-remote-build`),
+    * BusyBox / initrd images with no Bash,
+    * hosts using non-POSIX shells such as Nushell.
+
+  The race the probe once addressed was tied to SSH connection-sharing -- since connection-sharing code has already been removed, the probe is now pointless.
+
+  Real connection or protocol errors are now left to SSH/Nix to report directly.
+
+  This is technically a breaking change if you had scripts that relied on the literal "started" which needs to be updated to rely on other signals, e.g., exit codes.
+
+  Many thanks to [Raito Bezarius](https://git.lix.systems/raito) for this.
+
+
+## Miscellany
+
+- Deprecation of CA derivations, dynamic derivations, and impure derivations [fj#815](https://git.lix.systems/lix-project/lix/issues/815)
+
+  Content-addressed derivations are now deprecated and slated for removal in Lix 2.94.
+  We're doing this because the CA derivation system has been a known cause of problems
+  and inconsistencies, is unmaintained, habitually makes improving the store code very
+  difficult (or blocks such improvements outright), and is beset by a number of design
+  flaws that in our opinion cannot be fixed without a full reimplementation from zero.
+  Dynamic derivations and impure derivations are built on the CA derivation framework,
+  and owing to this they too are deprecated and slated for removal in another release.
+
+
+
+
 
 
 # Lix 2.93.0 (2025-05-09)

docker.nix

@@ -62,38 +62,37 @@ let
     ++ autoLayered
     ++ extraPkgs;
 
-  users =
-    {
+  users = {
 
-      root = {
-        uid = 0;
-        shell = "${pkgs.bashInteractive}/bin/bash";
-        home = "/root";
-        gid = 0;
-        groups = [ "root" ];
-        description = "System administrator";
-      };
+    root = {
+      uid = 0;
+      shell = "${pkgs.bashInteractive}/bin/bash";
+      home = "/root";
+      gid = 0;
+      groups = [ "root" ];
+      description = "System administrator";
+    };
 
-      nobody = {
-        uid = 65534;
-        shell = "${pkgs.shadow}/bin/nologin";
-        home = "/var/empty";
-        gid = 65534;
-        groups = [ "nobody" ];
-        description = "Unprivileged account (don't use!)";
+    nobody = {
+      uid = 65534;
+      shell = "${pkgs.shadow}/bin/nologin";
+      home = "/var/empty";
+      gid = 65534;
+      groups = [ "nobody" ];
+      description = "Unprivileged account (don't use!)";
+    };
+  }
+  // lib.listToAttrs (
+    map (n: {
+      name = "nixbld${toString n}";
+      value = {
+        uid = 30000 + n;
+        gid = 30000;
+        groups = [ "nixbld" ];
+        description = "Nix build user ${toString n}";
       };
-    }
-    // lib.listToAttrs (
-      map (n: {
-        name = "nixbld${toString n}";
-        value = {
-          uid = 30000 + n;
-          gid = 30000;
-          groups = [ "nixbld" ];
-          description = "Nix build user ${toString n}";
-        };
-      }) (lib.lists.range 1 32)
-    );
+    }) (lib.lists.range 1 32)
+  );
 
   groups = {
     root.gid = 0;
@@ -361,7 +360,8 @@ let
         "org.opencontainers.image.version" = pkgs.nix.version;
         "org.opencontainers.image.description" =
           "Minimal Lix container image, with some batteries included.";
-      } // lib.optionalAttrs (lixRevision != null) { "org.opencontainers.image.revision" = lixRevision; };
+      }
+      // lib.optionalAttrs (lixRevision != null) { "org.opencontainers.image.revision" = lixRevision; };
     };
 
     meta = {

flake.lock

@@ -108,11 +108,11 @@
     },
     "nixpkgs_2": {
       "locked": {
-        "lastModified": 1749522908,
-        "narHash": "sha256-eWANkhWXFL1MmaxzsZ9bhLCNT8OVs7CC+OXaSDGlA8A=",
+        "lastModified": 1757198069,
+        "narHash": "sha256-m3VUcOD4rTs8J7S+3dOjWMrAjw6RcITC3XYQ98zhEFs=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "e5cb99555c45a13dcc5f1317462238530b0066b7",
+        "rev": "0747026fc57ecb9c28901c7f7a2b5dc40e8af43c",
         "type": "github"
       },
       "original": {

flake.nix

@@ -175,8 +175,13 @@
         {
           nixStable = prev.nix;
 
-          # Nix 2.18 has been removed from Nixpkgs ≥ 25.05, so we need to reintroduce it ourselves for our tests.
           nixVersions = prev.nixVersions // {
+            nix_2_3 = prev.nixVersions.nix_2_3.overrideAttrs (old: {
+              meta = old.meta // {
+                knownVulnerabilities = [ ];
+              };
+            });
+            # Nix 2.18 has been removed from Nixpkgs ≥ 25.05, so we need to reintroduce it ourselves for our tests.
             nix_2_18 = nix_2_18.outputs.packages.${currentStdenv.hostPlatform.system}.default;
           };
 
@@ -214,6 +219,9 @@
             inherit versionSuffix officialRelease;
             stdenv = currentStdenv;
             busybox-sandbox-shell = final.busybox-sandbox-shell or final.default-busybox-sandbox-shell;
+            # See below
+            lowdown = final.lowdown_3_0;
+            lowdown-unsandboxed = final.lowdown_3_0.override { enableDarwinSandbox = false; };
           };
 
           lix-clang-tidy = final.callPackage ./subprojects/lix-clang-tidy { };
@@ -238,6 +246,30 @@
           boehmgc-nix = final.nix.passthru.boehmgc-nix;
           # And same thing for our build-release-notes package.
           build-release-notes = final.nix.passthru.build-release-notes;
+
+          # As soon as Nixpkgs updates to >= 3.0.0, change to lowdown_2_0!
+          # We don't change the default version in order to not change the hash
+          # of Nix/Lix from upstream Nixpkgs.
+          lowdown_3_0 =
+            assert lib.versionOlder prev.lowdown.version "3.0.0";
+            prev.lowdown.overrideAttrs (
+              finalAttrs: prevAttrs: {
+                version = "3.0.1";
+                src = final.fetchurl {
+                  url = "https://kristaps.bsd.lv/lowdown/snapshots/lowdown-${finalAttrs.version}.tar.gz";
+                  sha512 = "fe68e1b7ff23f3992398356d7aa9a330dfd7b72e22bea9a91eeef74182b209ecea0c9f3e2b2216e1a07b2358da2b746238ec9cbbdeebdd3551cef14dd2d79f46";
+                };
+
+                # no longer compiles with GNU make
+                nativeBuildInputs = prevAttrs.nativeBuildInputs ++ [ final.bmake ];
+                # dylib fixups on darwin are no longer necessary
+                postInstall = "";
+                # doesn't work on darwin due to disallowed nested sandboxes
+                doInstallCheck = prevAttrs.doInstallCheck && !(final.stdenv.hostPlatform.isDarwin);
+                doCheck = prevAttrs.doCheck && !(final.stdenv.hostPlatform.isDarwin);
+              }
+            );
+
         };
     in
     {
@@ -252,6 +284,15 @@
         # Binary package for various platforms.
         build = forAllSystems (system: self.packages.${system}.nix);
 
+        # Ensure support for lowdown < 3.0 doesn't regress for NixOS 25.11
+        build-lowdown_2_0.aarch64-linux = lib.genAttrs [ "aarch64-linux" ] (
+          system:
+          self.packages.${system}.nix.override {
+            lowdown = nixpkgsFor.${system}.native.lowdown;
+            lowdown-unsandboxed = nixpkgsFor.${system}.native.lowdown-unsandboxed;
+          }
+        );
+
         devShell = forAllSystems (system: {
           default = self.devShells.${system}.default;
           clang = self.devShells.${system}.native-clangStdenvPackages;
@@ -398,15 +439,16 @@
               in
               pkgs.symlinkJoin {
                 name = "nixpkgs-lib-tests";
-                paths =
-                  [ testWithNix ]
-                  # NOTE: nixpkgs 25.05 is being ... *creative*, and requires this dance to override
-                  # the evaluator used for the test. it will break again in the future, don't worry.
-                  ++ lib.optionals pkgs.stdenv.isLinux [
-                    ((pkgs.callPackage "${nixpkgs}/ci/eval" { nixVersions.latest = nix; }).attrpathsSuperset {
-                      evalSystem = system;
-                    })
-                  ];
+                paths = [
+                  testWithNix
+                ]
+                # NOTE: nixpkgs 25.05 is being ... *creative*, and requires this dance to override
+                # the evaluator used for the test. it will break again in the future, don't worry.
+                ++ lib.optionals pkgs.stdenv.isLinux [
+                  ((pkgs.callPackage "${nixpkgs}/ci/eval" { inherit nix; }).attrpathsSuperset {
+                    evalSystem = system;
+                  })
+                ];
               }
             );
           };
@@ -479,10 +521,7 @@
         }
         // (
           lib.optionalAttrs (builtins.elem system linux64BitSystems) {
-            # python doesn't work in static builds as of 2025-06-27
-            nix-static = nixpkgsFor.${system}.static.nix.overrideAttrs (_: {
-              doCheck = false;
-            });
+            nix-static = nixpkgsFor.${system}.static.nix;
             dockerImage =
               let
                 pkgs = nixpkgsFor.${system}.native;

justfile

@@ -41,10 +41,6 @@ test-unit *OPTIONS: (test "--suite" "check")
 # Run integration tests only
 test-integration *OPTIONS: install (test "--suite" "installcheck")
 
-# Run functional2 tests using pytest directly, allowing for additional arguments to be passed to pytest e.g. for more granular test selection
-test-functional2 *OPTIONS:
-    cd tests && python -m pytest -v {{ OPTIONS }} functional2
-
 alias clang-tidy := lint
 
 # Lint with `clang-tidy`

lix/legacy/build-remote.cc

@@ -84,13 +84,7 @@ static int main_build_remote(AsyncIoRoot & aio, std::string programName, Strings
 
         initPlugins();
 
-        // FIXME this does not open a daemon connection for historical reasons.
-        // we may create a lot of build hook instances, and having each of them
-        // also create a daemon instance is inefficient and wasteful. in future
-        // versions of the build hook (where we don't need one hook process per
-        // build) we should change this to using a daemon connection, ideally a
-        // daemon connection provided by the parent via file descriptor passing
-        auto store = aio.blockOn(openStore(settings.storeUri, {}, AllowDaemon::Disallow));
+        auto store = aio.blockOn(openStore());
 
         /* It would be more appropriate to use $XDG_RUNTIME_DIR, since
            that gets cleared on reboot, but it wouldn't work on macOS. */
@@ -330,7 +324,7 @@ connected:
             //
             // 2. Changing the `inputSrcs` set changes the associated
             //    output ids, which break CA derivations
-            if (!drv.inputDrvs.empty())
+            if (!drv.inputDrvs.map.empty())
                 drv.inputSrcs = store->parseStorePathSet(inputs);
             optResult = aio.blockOn(sshStore->buildDerivation(*drvPath, (const BasicDerivation &) drv));
             auto & result = *optResult;
@@ -342,7 +336,7 @@ connected:
             ));
             auto res = aio.blockOn(sshStore->buildPathsWithResults({
                 DerivedPath::Built {
-                    .drvPath = makeConstantStorePath(*drvPath),
+                    .drvPath = makeConstantStorePathRef(*drvPath),
                     .outputs = OutputsSpec::All {},
                 }
             }));
@@ -352,11 +346,31 @@ connected:
         }
 
 
+        auto outputHashes = aio.blockOn(staticOutputHashes(*store, drv));
+        std::set<Realisation> missingRealisations;
         StorePathSet missingPaths;
-        auto outputPaths = drv.outputsAndPaths(*store);
-        for (auto & [outputName, outputPath] : outputPaths) {
-            if (!aio.blockOn(store->isValidPath(outputPath.second)))
-                missingPaths.insert(outputPath.second);
+        if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations) && !drv.type().hasKnownOutputPaths()) {
+            for (auto & outputName : wantedOutputs) {
+                auto thisOutputHash = outputHashes.at(outputName);
+                auto thisOutputId = DrvOutput{ thisOutputHash, outputName };
+                if (!aio.blockOn(store->queryRealisation(thisOutputId))) {
+                    debug("missing output %s", outputName);
+                    assert(optResult);
+                    auto & result = *optResult;
+                    auto i = result.builtOutputs.find(outputName);
+                    assert(i != result.builtOutputs.end());
+                    auto & newRealisation = i->second;
+                    missingRealisations.insert(newRealisation);
+                    missingPaths.insert(newRealisation.outPath);
+                }
+            }
+        } else {
+            auto outputPaths = drv.outputsAndOptPaths(*store);
+            for (auto & [outputName, hopefullyOutputPath] : outputPaths) {
+                assert(hopefullyOutputPath.second);
+                if (!aio.blockOn(store->isValidPath(*hopefullyOutputPath.second)))
+                    missingPaths.insert(*hopefullyOutputPath.second);
+            }
         }
 
         if (!missingPaths.empty()) {
@@ -368,6 +382,13 @@ connected:
                 copyPaths(*sshStore, *store, missingPaths, NoRepair, NoCheckSigs, NoSubstitute)
             );
         }
+        // XXX: Should be done as part of `copyPaths`
+        for (auto & realisation : missingRealisations) {
+            // Should hold, because if the feature isn't enabled the set
+            // of missing realisations should be empty
+            experimentalFeatureSettings.require(Xp::CaDerivations);
+            aio.blockOn(store->registerDrvOutput(realisation));
+        }
 
         return 0;
     }

lix/legacy/nix-build.cc

@@ -187,7 +187,8 @@ static void main_nix_build(AsyncIoRoot & aio, std::string programName, Strings a
     if (packages && fromArgs)
         throw UsageError("'-p' and '-E' are mutually exclusive");
 
-    AutoDelete tmpDir(createTempDir("", myName));
+    AutoDelete tmpDir(createTempDir(myName));
+    AutoDelete buildTopTmpDir(createTempSubdir(tmpDir, "build-top"));
     if (outLink.empty())
         outLink = (Path) tmpDir + "/result";
 
@@ -355,7 +356,7 @@ static void main_nix_build(AsyncIoRoot & aio, std::string programName, Strings a
 
                 auto bashDrv = drv->requireDrvPath(*state);
                 pathsToBuild.push_back(DerivedPath::Built {
-                    .drvPath = makeConstantStorePath(bashDrv),
+                    .drvPath = makeConstantStorePathRef(bashDrv),
                     .outputs = OutputsSpec::Names {"out"},
                 });
                 pathsToCopy.insert(bashDrv);
@@ -368,16 +369,22 @@ static void main_nix_build(AsyncIoRoot & aio, std::string programName, Strings a
             }
         }
 
-        auto accumDerivedPath = [&](SingleDerivedPath::Opaque inputDrv, const StringSet & inputNode) {
-            if (!inputNode.empty())
+        std::function<void(ref<SingleDerivedPath>, const DerivedPathMap<StringSet>::ChildNode &)> accumDerivedPath;
+
+        accumDerivedPath = [&](ref<SingleDerivedPath> inputDrv, const DerivedPathMap<StringSet>::ChildNode & inputNode) {
+            if (!inputNode.value.empty())
                 pathsToBuild.push_back(DerivedPath::Built {
                     .drvPath = inputDrv,
-                    .outputs = OutputsSpec::Names { inputNode },
+                    .outputs = OutputsSpec::Names { inputNode.value },
                 });
+            for (const auto & [outputName, childNode] : inputNode.childMap)
+                accumDerivedPath(
+                    make_ref<SingleDerivedPath>(SingleDerivedPath::Built { inputDrv, outputName }),
+                    childNode);
         };
 
         // Build or fetch all dependencies of the derivation.
-        for (const auto & [inputDrv0, inputNode] : drv.inputDrvs) {
+        for (const auto & [inputDrv0, inputNode] : drv.inputDrvs.map) {
             // To get around lambda capturing restrictions in the
             // standard.
             const auto & inputDrv = inputDrv0;
@@ -386,7 +393,7 @@ static void main_nix_build(AsyncIoRoot & aio, std::string programName, Strings a
                         return !std::regex_search(store->printStorePath(inputDrv), regex::parse(exclude));
                     }))
             {
-                accumDerivedPath(makeConstantStorePath(inputDrv), inputNode);
+                accumDerivedPath(makeConstantStorePathRef(inputDrv), inputNode);
                 pathsToCopy.insert(inputDrv);
             }
         }
@@ -401,8 +408,14 @@ static void main_nix_build(AsyncIoRoot & aio, std::string programName, Strings a
 
         if (shellDrv) {
             auto shellDrvOutputs =
-                aio.blockOn(store->queryDerivationOutputMap(shellDrv.value(), &*evalStore));
-            shell = store->printStorePath(shellDrvOutputs.at("out")) + "/bin/bash";
+                aio.blockOn(store->queryPartialDerivationOutputMap(shellDrv.value(), &*evalStore));
+            shell = store->printStorePath(shellDrvOutputs.at("out").value()) + "/bin/bash";
+        }
+
+        if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) {
+            auto resolvedDrv = aio.blockOn(drv.tryResolve(*store));
+            assert(resolvedDrv && "Successfully resolved the derivation");
+            drv = *resolvedDrv;
         }
 
         // Set the environment.
@@ -419,7 +432,8 @@ static void main_nix_build(AsyncIoRoot & aio, std::string programName, Strings a
         }
 
         // Don't use defaultTempDir() here! We want to preserve the user's TMPDIR for the shell
-        env["NIX_BUILD_TOP"] = env["TMPDIR"] = env["TEMPDIR"] = env["TMP"] = env["TEMP"] = getEnvNonEmpty("TMPDIR").value_or("/tmp");
+        env["NIX_BUILD_TOP"] = env["TMPDIR"] = env["TEMPDIR"] = env["TMP"] = env["TEMP"] =
+            getEnvNonEmpty("TMPDIR").value_or(buildTopTmpDir);
         env["NIX_STORE"] = store->config().storeDir;
         env["NIX_BUILD_CORES"] = std::to_string(settings.buildCores);
 
@@ -443,16 +457,20 @@ static void main_nix_build(AsyncIoRoot & aio, std::string programName, Strings a
         if (env.count("__json")) {
             StorePathSet inputs;
 
-            auto accumInputClosure = [&](const StorePath & inputDrv, const StringSet & inputNode) {
+            std::function<void(const StorePath &, const DerivedPathMap<StringSet>::ChildNode &)> accumInputClosure;
+
+            accumInputClosure = [&](const StorePath & inputDrv, const DerivedPathMap<StringSet>::ChildNode & inputNode) {
                 auto outputs =
-                    aio.blockOn(store->queryDerivationOutputMap(inputDrv, &*evalStore));
-                for (auto & i : inputNode) {
+                    aio.blockOn(store->queryPartialDerivationOutputMap(inputDrv, &*evalStore));
+                for (auto & i : inputNode.value) {
                     auto o = outputs.at(i);
-                    aio.blockOn(store->computeFSClosure(o, inputs));
+                    aio.blockOn(store->computeFSClosure(*o, inputs));
                 }
+                for (const auto & [outputName, childNode] : inputNode.childMap)
+                    accumInputClosure(*outputs.at(outputName), childNode);
             };
 
-            for (const auto & [inputDrv, inputNode] : drv.inputDrvs)
+            for (const auto & [inputDrv, inputNode] : drv.inputDrvs.map)
                 accumInputClosure(inputDrv, inputNode);
 
             ParsedDerivation parsedDrv(drvInfo.requireDrvPath(*state), drv);
@@ -565,7 +583,7 @@ static void main_nix_build(AsyncIoRoot & aio, std::string programName, Strings a
                 throw Error("derivation '%s' lacks an 'outputName' attribute", store->printStorePath(drvPath));
 
             pathsToBuild.push_back(DerivedPath::Built{
-                .drvPath = makeConstantStorePath(drvPath),
+                .drvPath = makeConstantStorePathRef(drvPath),
                 .outputs = OutputsSpec::Names{outputName},
             });
             pathsToBuildOrdered.push_back({drvPath, {outputName}});
@@ -591,9 +609,11 @@ static void main_nix_build(AsyncIoRoot & aio, std::string programName, Strings a
                 drvPrefix += fmt("-%d", counter + 1);
 
             auto builtOutputs =
-                aio.blockOn(store->queryDerivationOutputMap(drvPath, &*evalStore));
+                aio.blockOn(store->queryPartialDerivationOutputMap(drvPath, &*evalStore));
 
-            auto outputPath = builtOutputs.at(outputName);
+            auto maybeOutputPath = builtOutputs.at(outputName);
+            assert(maybeOutputPath);
+            auto outputPath = *maybeOutputPath;
 
             if (auto store2 = store.try_cast_shared<LocalFSStore>()) {
                 std::string symlink = drvPrefix;

lix/legacy/nix-env.cc

@@ -495,7 +495,7 @@ static void printMissing(EvalState & state, DrvInfos & elems)
     for (auto & i : elems)
         if (auto drvPath = i.queryDrvPath(state))
             targets.emplace_back(DerivedPath::Built{
-                .drvPath = makeConstantStorePath(*drvPath),
+                .drvPath = makeConstantStorePathRef(*drvPath),
                 .outputs = OutputsSpec::All { },
             });
         else
@@ -792,7 +792,7 @@ static void opSet(Globals & globals, Strings opFlags, Strings opArgs)
     std::vector<DerivedPath> paths {
         drvPath
         ? (DerivedPath) (DerivedPath::Built {
-            .drvPath = makeConstantStorePath(*drvPath),
+            .drvPath = makeConstantStorePathRef(*drvPath),
             .outputs = OutputsSpec::All { },
         })
         : (DerivedPath) (DerivedPath::Opaque {

lix/legacy/nix-instantiate.cc

@@ -22,7 +22,7 @@ static Path gcRoot;
 static int rootNr = 0;
 
 
-enum OutputKind { okPlain, okRaw, okXML, okJSON };
+enum OutputKind { okPlain, okXML, okJSON };
 
 void processExpr(EvalState & state, const Strings & attrPaths,
     bool parseOnly, bool strict, Bindings & autoArgs,
@@ -48,11 +48,7 @@ void processExpr(EvalState & state, const Strings & attrPaths,
                 vRes = v;
             else
                 state.autoCallFunction(autoArgs, v, vRes, noPos);
-            if (output == okRaw)
-                std::cout << *state.coerceToString(noPos, vRes, context, "while generating the nix-instantiate output", StringCoercionMode::Strict);
-                // We intentionally don't output a newline here. The default PS1 for Bash in NixOS starts with a newline
-                // and other interactive shells like Zsh are smart enough to print a missing newline before the prompt.
-            else if (output == okXML)
+            if (output == okXML)
                 printValueAsXML(state, strict, location, vRes, std::cout, context, noPos);
             else if (output == okJSON) {
                 printValueAsJSON(state, strict, vRes, noPos, std::cout, context);
@@ -134,8 +130,6 @@ static int main_nix_instantiate(AsyncIoRoot & aio, std::string programName, Stri
                 gcRoot = getArg(*arg, arg, end);
             else if (*arg == "--indirect")
                 ;
-            else if (*arg == "--raw")
-                outputKind = okRaw;
             else if (*arg == "--xml")
                 outputKind = okXML;
             else if (*arg == "--json")

lix/legacy/nix-store.cc

@@ -33,23 +33,25 @@ namespace nix {
 using std::cin;
 using std::cout;
 
-typedef void (*Operation)(
-    std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, Strings opArgs
-);
+
+typedef void (* Operation) (AsyncIoRoot & aio, Strings opFlags, Strings opArgs);
+
 
 static Path gcRoot;
 static int rootNr = 0;
 static bool noOutput = false;
+static std::shared_ptr<Store> store;
 
-ref<LocalStore> ensureLocalStore(std::shared_ptr<Store> store)
+
+ref<LocalStore> ensureLocalStore()
 {
     auto store2 = std::dynamic_pointer_cast<LocalStore>(store);
     if (!store2) throw Error("you don't have sufficient rights to use this command");
     return ref<LocalStore>::unsafeFromPtr(store2);
 }
 
-static kj::Promise<Result<StorePath>>
-useDeriver(std::shared_ptr<Store> store, const StorePath & path)
+
+static kj::Promise<Result<StorePath>> useDeriver(const StorePath & path)
 try {
     if (path.isDerivation()) co_return path;
     auto info = TRY_AWAIT(store->queryPathInfo(path));
@@ -63,8 +65,7 @@ try {
 
 /* Realise the given path.  For a derivation that means build it; for
    other paths it means ensure their validity. */
-static kj::Promise<Result<PathSet>>
-realisePath(std::shared_ptr<Store> store, StorePathWithOutputs path, bool build = true)
+static kj::Promise<Result<PathSet>> realisePath(StorePathWithOutputs path, bool build = true)
 try {
     auto store2 = std::dynamic_pointer_cast<LocalFSStore>(store);
 
@@ -124,8 +125,7 @@ try {
 
 
 /* Realise the given paths. */
-static void
-opRealise(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
+static void opRealise(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
 {
     bool dryRun = false;
     BuildMode buildMode = bmNormal;
@@ -170,7 +170,7 @@ opRealise(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, Stri
 
     if (!ignoreUnknown)
         for (auto & i : paths) {
-            auto paths2 = aio.blockOn(realisePath(store, i, false));
+            auto paths2 = aio.blockOn(realisePath(i, false));
             if (!noOutput)
                 for (auto & j : paths2)
                     cout << fmt("%1%\n", j);
@@ -179,7 +179,7 @@ opRealise(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, Stri
 
 
 /* Add files to the Nix store and print the resulting paths. */
-static void opAdd(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
+static void opAdd(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
 {
     if (!opFlags.empty()) throw UsageError("unknown flag");
 
@@ -196,8 +196,7 @@ static void opAdd(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFla
 
 /* Preload the output of a fixed-output derivation into the Nix
    store. */
-static void
-opAddFixed(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
+static void opAddFixed(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
 {
     auto method = FileIngestionMethod::Flat;
 
@@ -223,8 +222,7 @@ opAddFixed(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, Str
 
 
 /* Hack to support caching in `nix-prefetch-url'. */
-static void
-opPrintFixedPath(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
+static void opPrintFixedPath(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
 {
     auto method = FileIngestionMethod::Flat;
 
@@ -247,20 +245,19 @@ opPrintFixedPath(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlag
     })));
 }
 
-static kj::Promise<Result<StorePathSet>> maybeUseOutputs(
-    std::shared_ptr<Store> store, const StorePath & storePath, bool useOutput, bool forceRealise
-)
+
+static kj::Promise<Result<StorePathSet>> maybeUseOutputs(const StorePath & storePath, bool useOutput, bool forceRealise)
 try {
-    if (forceRealise) {
-        TRY_AWAIT(realisePath(store, {storePath}));
-    }
+    if (forceRealise) TRY_AWAIT(realisePath({storePath}));
     if (useOutput && storePath.isDerivation()) {
         auto drv = TRY_AWAIT(store->derivationFromPath(storePath));
         StorePathSet outputs;
         if (forceRealise)
             co_return TRY_AWAIT(store->queryDerivationOutputs(storePath));
-        for (auto & i : drv.outputsAndPaths(*store)) {
-            outputs.insert(i.second.second);
+        for (auto & i : drv.outputsAndOptPaths(*store)) {
+            if (!i.second.second)
+                throw UsageError("Cannot use output path of floating content-addressed derivation until we know what it is (e.g. by building it)");
+            outputs.insert(*i.second.second);
         }
         co_return outputs;
     }
@@ -273,14 +270,8 @@ try {
 /* Some code to print a tree representation of a derivation dependency
    graph.  Topological sorting is used to keep the tree relatively
    flat. */
-static void printTree(
-    std::shared_ptr<Store> store,
-    AsyncIoRoot & aio,
-    const StorePath & path,
-    const std::string & firstPad,
-    const std::string & tailPad,
-    StorePathSet & done
-)
+static void printTree(AsyncIoRoot & aio, const StorePath & path,
+    const std::string & firstPad, const std::string & tailPad, StorePathSet & done)
 {
     if (!done.insert(path).second) {
         cout << fmt("%s%s [...]\n", firstPad, store->printStorePath(path));
@@ -300,21 +291,16 @@ static void printTree(
 
     for (const auto &[n, i] : enumerate(sorted)) {
         bool last = n + 1 == sorted.size();
-        printTree(
-            store,
-            aio,
-            i,
+        printTree(aio, i,
             tailPad + (last ? treeLast : treeConn),
             tailPad + (last ? treeNull : treeLine),
-            done
-        );
+            done);
     }
 }
 
 
 /* Perform various sorts of queries. */
-static void
-opQuery(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
+static void opQuery(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
 {
     enum QueryType
         { qOutputs, qRequisites, qReferences, qReferrers
@@ -365,9 +351,7 @@ opQuery(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, String
 
         case qOutputs: {
             for (auto & i : opArgs) {
-                auto outputs = aio.blockOn(
-                    maybeUseOutputs(store, store->followLinksToStorePath(i), true, forceRealise)
-                );
+                auto outputs = aio.blockOn(maybeUseOutputs(store->followLinksToStorePath(i), true, forceRealise));
                 for (auto & outputPath : outputs)
                     cout << fmt("%1%\n", store->printStorePath(outputPath));
             }
@@ -380,9 +364,7 @@ opQuery(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, String
         case qReferrersClosure: {
             StorePathSet paths;
             for (auto & i : opArgs) {
-                auto ps = aio.blockOn(maybeUseOutputs(
-                    store, store->followLinksToStorePath(i), useOutput, forceRealise
-                ));
+                auto ps = aio.blockOn(maybeUseOutputs(store->followLinksToStorePath(i), useOutput, forceRealise));
                 for (auto & j : ps) {
                     if (query == qRequisites) {
                         aio.blockOn(store->computeFSClosure(j, paths, false, includeOutputs));
@@ -433,7 +415,7 @@ opQuery(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, String
 
         case qBinding:
             for (auto & i : opArgs) {
-                auto path = aio.blockOn(useDeriver(store, store->followLinksToStorePath(i)));
+                auto path = aio.blockOn(useDeriver(store->followLinksToStorePath(i)));
                 Derivation drv = aio.blockOn(store->derivationFromPath(path));
                 StringPairs::iterator j = drv.env.find(bindingName);
                 if (j == drv.env.end())
@@ -446,10 +428,7 @@ opQuery(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, String
         case qHash:
         case qSize:
             for (auto & i : opArgs) {
-                for (auto & j : aio.blockOn(maybeUseOutputs(
-                         store, store->followLinksToStorePath(i), useOutput, forceRealise
-                     )))
-                {
+                for (auto & j : aio.blockOn(maybeUseOutputs(store->followLinksToStorePath(i), useOutput, forceRealise))) {
                     auto info = aio.blockOn(store->queryPathInfo(j));
                     if (query == qHash) {
                         assert(info->narHash.type == HashType::SHA256);
@@ -463,19 +442,15 @@ opQuery(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, String
         case qTree: {
             StorePathSet done;
             for (auto & i : opArgs)
-                printTree(store, aio, store->followLinksToStorePath(i), "", "", done);
+                printTree(aio, store->followLinksToStorePath(i), "", "", done);
             break;
         }
 
         case qGraph: {
             StorePathSet roots;
             for (auto & i : opArgs)
-                for (auto & j : aio.blockOn(maybeUseOutputs(
-                         store, store->followLinksToStorePath(i), useOutput, forceRealise
-                     )))
-                {
+                for (auto & j : aio.blockOn(maybeUseOutputs(store->followLinksToStorePath(i), useOutput, forceRealise)))
                     roots.insert(j);
-                }
             aio.blockOn(printDotGraph(ref<Store>::unsafeFromPtr(store), std::move(roots)));
             break;
         }
@@ -483,12 +458,8 @@ opQuery(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, String
         case qGraphML: {
             StorePathSet roots;
             for (auto & i : opArgs)
-                for (auto & j : aio.blockOn(maybeUseOutputs(
-                         store, store->followLinksToStorePath(i), useOutput, forceRealise
-                     )))
-                {
+                for (auto & j : aio.blockOn(maybeUseOutputs(store->followLinksToStorePath(i), useOutput, forceRealise)))
                     roots.insert(j);
-                }
             aio.blockOn(printGraphML(ref<Store>::unsafeFromPtr(store), std::move(roots)));
             break;
         }
@@ -502,12 +473,8 @@ opQuery(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, String
         case qRoots: {
             StorePathSet args;
             for (auto & i : opArgs)
-                for (auto & p : aio.blockOn(maybeUseOutputs(
-                         store, store->followLinksToStorePath(i), useOutput, forceRealise
-                     )))
-                {
+                for (auto & p : aio.blockOn(maybeUseOutputs(store->followLinksToStorePath(i), useOutput, forceRealise)))
                     args.insert(p);
-                }
 
             StorePathSet referrers;
             aio.blockOn(store->computeFSClosure(
@@ -527,8 +494,8 @@ opQuery(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, String
     }
 }
 
-static void
-opPrintEnv(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
+
+static void opPrintEnv(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
 {
     if (!opFlags.empty()) throw UsageError("unknown flag");
     if (opArgs.size() != 1) throw UsageError("'--print-env' requires one derivation store path");
@@ -553,8 +520,8 @@ opPrintEnv(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, Str
     cout << "'\n";
 }
 
-static void
-opReadLog(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
+
+static void opReadLog(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
 {
     if (!opFlags.empty()) throw UsageError("unknown flag");
 
@@ -571,8 +538,8 @@ opReadLog(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, Stri
     }
 }
 
-static void
-opDumpDB(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
+
+static void opDumpDB(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
 {
     if (!opFlags.empty()) throw UsageError("unknown flag");
     if (!opArgs.empty()) {
@@ -587,13 +554,8 @@ opDumpDB(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, Strin
     }
 }
 
-static void registerValidity(
-    std::shared_ptr<Store> store,
-    AsyncIoRoot & aio,
-    bool reregister,
-    bool hashGiven,
-    bool canonicalise
-)
+
+static void registerValidity(AsyncIoRoot & aio, bool reregister, bool hashGiven, bool canonicalise)
 {
     ValidPathInfos infos;
 
@@ -616,20 +578,20 @@ static void registerValidity(
         }
     }
 
-    aio.blockOn(ensureLocalStore(store)->registerValidPaths(infos));
+    aio.blockOn(ensureLocalStore()->registerValidPaths(infos));
 }
 
-static void
-opLoadDB(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
+
+static void opLoadDB(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
 {
     if (!opFlags.empty()) throw UsageError("unknown flag");
     if (!opArgs.empty())
         throw UsageError("no arguments expected");
-    registerValidity(store, aio, true, true, false);
+    registerValidity(aio, true, true, false);
 }
 
-static void
-opRegisterValidity(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
+
+static void opRegisterValidity(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
 {
     bool reregister = false; // !!! maybe this should be the default
     bool hashGiven = false;
@@ -641,11 +603,11 @@ opRegisterValidity(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFl
 
     if (!opArgs.empty()) throw UsageError("no arguments expected");
 
-    registerValidity(store, aio, reregister, hashGiven, true);
+    registerValidity(aio, reregister, hashGiven, true);
 }
 
-static void
-opCheckValidity(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
+
+static void opCheckValidity(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
 {
     bool printInvalid = false;
 
@@ -664,7 +626,8 @@ opCheckValidity(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags
     }
 }
 
-static void opGC(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
+
+static void opGC(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
 {
     bool printRoots = false;
     GCOptions options;
@@ -710,8 +673,7 @@ static void opGC(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlag
 /* Remove paths from the Nix store if possible (i.e., if they do not
    have any remaining referrers and are not reachable from any GC
    roots). */
-static void
-opDelete(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
+static void opDelete(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
 {
     GCOptions options;
     options.action = GCOptions::gcDeleteSpecific;
@@ -741,7 +703,7 @@ opDelete(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, Strin
 
 
 /* Dump a path as a Nix archive.  The archive is written to stdout */
-static void opDump(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
+static void opDump(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
 {
     if (!opFlags.empty()) throw UsageError("unknown flag");
     if (opArgs.size() != 1) throw UsageError("only one argument allowed");
@@ -754,8 +716,7 @@ static void opDump(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFl
 
 
 /* Restore a value from a Nix archive.  The archive is read from stdin. */
-static void
-opRestore(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
+static void opRestore(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
 {
     if (!opFlags.empty()) throw UsageError("unknown flag");
     if (opArgs.size() != 1) throw UsageError("only one argument allowed");
@@ -764,8 +725,8 @@ opRestore(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, Stri
     restorePath(*opArgs.begin(), source);
 }
 
-static void
-opExport(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
+
+static void opExport(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
 {
     for (auto & i : opFlags)
         throw UsageError("unknown flag '%1%'", i);
@@ -780,8 +741,8 @@ opExport(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, Strin
     sink.flush();
 }
 
-static void
-opImport(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
+
+static void opImport(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
 {
     for (auto & i : opFlags)
         throw UsageError("unknown flag '%1%'", i);
@@ -797,7 +758,7 @@ opImport(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, Strin
 
 
 /* Initialise the Nix databases. */
-static void opInit(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
+static void opInit(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
 {
     if (!opFlags.empty()) throw UsageError("unknown flag");
     if (!opArgs.empty())
@@ -808,8 +769,7 @@ static void opInit(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFl
 
 
 /* Verify the consistency of the Nix environment. */
-static void
-opVerify(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
+static void opVerify(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
 {
     if (!opArgs.empty())
         throw UsageError("no arguments expected");
@@ -830,8 +790,7 @@ opVerify(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, Strin
 
 
 /* Verify whether the contents of the given store path have not changed. */
-static void
-opVerifyPath(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
+static void opVerifyPath(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
 {
     if (!opFlags.empty())
         throw UsageError("no flags expected");
@@ -843,7 +802,7 @@ opVerifyPath(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, S
         printMsg(lvlTalkative, "checking path '%s'...", store->printStorePath(path));
         auto info = aio.blockOn(store->queryPathInfo(path));
         HashSink sink(info->narHash.type);
-        aio.blockOn(aio.blockOn(store->narFromPath(path))->drainInto(sink));
+        aio.blockOn(store->narFromPath(path))->drainInto(sink);
         auto current = sink.finish();
         if (current.first != info->narHash) {
             printError("path '%s' was modified! expected hash '%s', got '%s'",
@@ -860,8 +819,7 @@ opVerifyPath(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, S
 
 /* Repair the contents of the given path by redownloading it using a
    substituter (if available). */
-static void
-opRepairPath(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
+static void opRepairPath(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
 {
     if (!opFlags.empty())
         throw UsageError("no flags expected");
@@ -872,8 +830,7 @@ opRepairPath(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, S
 
 /* Optimise the disk space usage of the Nix store by hard-linking
    files with the same contents. */
-static void
-opOptimise(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
+static void opOptimise(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
 {
     if (!opArgs.empty() || !opFlags.empty())
         throw UsageError("no arguments expected");
@@ -882,8 +839,7 @@ opOptimise(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, Str
 }
 
 /* Serve the nix store in a way usable by a restricted ssh user. */
-static void
-opServe(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
+static void opServe(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
 {
     bool writeAllowed = false;
     for (auto & i : opFlags)
@@ -904,11 +860,9 @@ opServe(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, String
 
     ServeProto::ReadConn rconn {
         .from = in,
-        .store = *store,
         .version = clientVersion,
     };
     ServeProto::WriteConn wconn {
-        .store = *store,
         .version = clientVersion,
     };
 
@@ -955,7 +909,7 @@ opServe(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, String
             case ServeProto::Command::QueryValidPaths: {
                 bool lock = readInt(in);
                 bool substitute = readInt(in);
-                auto paths = ServeProto::Serialise<StorePathSet>::read(rconn);
+                auto paths = ServeProto::Serialise<StorePathSet>::read(*store, rconn);
                 if (lock && writeAllowed)
                     for (auto & path : paths)
                         aio.blockOn(store->addTempRoot(path));
@@ -965,18 +919,18 @@ opServe(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, String
                 }
 
                 auto valid = aio.blockOn(store->queryValidPaths(paths));
-                out << ServeProto::write(wconn, valid);
+                out << ServeProto::write(*store, wconn, valid);
                 break;
             }
 
             case ServeProto::Command::QueryPathInfos: {
-                auto paths = ServeProto::Serialise<StorePathSet>::read(rconn);
+                auto paths = ServeProto::Serialise<StorePathSet>::read(*store, rconn);
                 // !!! Maybe we want a queryPathInfos?
                 for (auto & i : paths) {
                     try {
                         auto info = aio.blockOn(store->queryPathInfo(i));
                         out << store->printStorePath(info->path);
-                        out << ServeProto::write(wconn, static_cast<const UnkeyedValidPathInfo &>(*info));
+                        out << ServeProto::write(*store, wconn, static_cast<const UnkeyedValidPathInfo &>(*info));
                     } catch (InvalidPath &) {
                     }
                 }
@@ -985,8 +939,8 @@ opServe(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, String
             }
 
             case ServeProto::Command::DumpStorePath:
-                aio.blockOn(aio.blockOn(store->narFromPath(store->parseStorePath(readString(in))))
-                                ->drainInto(out));
+                aio.blockOn(store->narFromPath(store->parseStorePath(readString(in))))
+                    ->drainInto(out);
                 break;
 
             case ServeProto::Command::ImportPaths: {
@@ -999,7 +953,7 @@ opServe(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, String
             case ServeProto::Command::ExportPaths: {
                 readInt(in); // obsolete
                 aio.blockOn(store->exportPaths(
-                    ServeProto::Serialise<StorePathSet>::read(rconn), out
+                    ServeProto::Serialise<StorePathSet>::read(*store, rconn), out
                 ));
                 break;
             }
@@ -1038,7 +992,7 @@ opServe(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, String
                 MonitorFdHup monitor(in.fd);
                 auto status = aio.blockOn(store->buildDerivation(drvPath, drv));
 
-                out << ServeProto::write(wconn, status);
+                out << ServeProto::write(*store, wconn, status);
                 break;
             }
 
@@ -1046,12 +1000,12 @@ opServe(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, String
                 bool includeOutputs = readInt(in);
                 StorePathSet closure;
                 aio.blockOn(store->computeFSClosure(
-                    ServeProto::Serialise<StorePathSet>::read(rconn),
+                    ServeProto::Serialise<StorePathSet>::read(*store, rconn),
                     closure,
                     false,
                     includeOutputs
                 ));
-                out << ServeProto::write(wconn, closure);
+                out << ServeProto::write(*store, wconn, closure);
                 break;
             }
 
@@ -1066,7 +1020,7 @@ opServe(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, String
                 };
                 if (deriver != "")
                     info.deriver = store->parseStorePath(deriver);
-                info.references = ServeProto::Serialise<StorePathSet>::read(rconn);
+                info.references = ServeProto::Serialise<StorePathSet>::read(*store, rconn);
                 in >> info.registrationTime >> info.narSize >> info.ultimate;
                 info.sigs = readStrings<StringSet>(in);
                 info.ca = ContentAddress::parseOpt(readString(in));
@@ -1095,9 +1049,8 @@ opServe(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, String
     }
 }
 
-static void opGenerateBinaryCacheKey(
-    std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, Strings opArgs
-)
+
+static void opGenerateBinaryCacheKey(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
 {
     for (auto & i : opFlags)
         throw UsageError("unknown flag '%1%'", i);
@@ -1115,8 +1068,8 @@ static void opGenerateBinaryCacheKey(
     writeFile(secretKeyFile, secretKey.to_string());
 }
 
-static void
-opVersion(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
+
+static void opVersion(AsyncIoRoot & aio, Strings opFlags, Strings opArgs)
 {
     printVersion("nix-store");
 }
@@ -1261,11 +1214,10 @@ static int main_nix_store(AsyncIoRoot & aio, std::string programName, Strings ar
         if (showHelp) showManPage("nix-store" + opName);
         if (!op) throw UsageError("no operation specified");
 
-        std::shared_ptr<Store> store;
         if (op != opDump && op != opRestore) /* !!! hack */
             store = aio.blockOn(openStore());
 
-        op(store, aio, std::move(opFlags), std::move(opArgs));
+        op(aio, std::move(opFlags), std::move(opArgs));
 
         return 0;
     }

lix/libcmd/built-path.cc

@@ -13,9 +13,9 @@ namespace nix {
     bool MY_TYPE ::operator COMPARATOR (const MY_TYPE & other) const \
     { \
         const MY_TYPE* me = this; \
-        auto fields1 = std::tie(me->drvPath, me->FIELD); \
+        auto fields1 = std::tie(*me->drvPath, me->FIELD); \
         me = &other; \
-        auto fields2 = std::tie(me->drvPath, me->FIELD); \
+        auto fields2 = std::tie(*me->drvPath, me->FIELD); \
         return fields1 COMPARATOR fields2; \
     }
 #define CMP(CHILD_TYPE, MY_TYPE, FIELD) \
@@ -23,6 +23,10 @@ namespace nix {
     CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, !=) \
     CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, <)
 
+#define FIELD_TYPE std::pair<std::string, StorePath>
+CMP(SingleBuiltPath, SingleBuiltPathBuilt, output)
+#undef FIELD_TYPE
+
 #define FIELD_TYPE std::map<std::string, StorePath>
 CMP(SingleBuiltPath, BuiltPathBuilt, outputs)
 #undef FIELD_TYPE
@@ -30,6 +34,16 @@ CMP(SingleBuiltPath, BuiltPathBuilt, outputs)
 #undef CMP
 #undef CMP_ONE
 
+StorePath SingleBuiltPath::outPath() const
+{
+    return std::visit(
+        overloaded{
+            [](const SingleBuiltPath::Opaque & p) { return p.path; },
+            [](const SingleBuiltPath::Built & b) { return b.output.second; },
+        }, raw()
+    );
+}
+
 StorePathSet BuiltPath::outPaths() const
 {
     return std::visit(
@@ -45,10 +59,32 @@ StorePathSet BuiltPath::outPaths() const
     );
 }
 
+SingleDerivedPath::Built SingleBuiltPath::Built::discardOutputPath() const
+{
+    return SingleDerivedPath::Built {
+        .drvPath = make_ref<SingleDerivedPath>(drvPath->discardOutputPath()),
+        .output = output.first,
+    };
+}
+
+SingleDerivedPath SingleBuiltPath::discardOutputPath() const
+{
+    return std::visit(
+        overloaded{
+            [](const SingleBuiltPath::Opaque & p) -> SingleDerivedPath {
+                return p;
+            },
+            [](const SingleBuiltPath::Built & b) -> SingleDerivedPath {
+                return b.discardOutputPath();
+            },
+        }, raw()
+    );
+}
+
 kj::Promise<Result<JSON>> BuiltPath::Built::toJSON(const Store & store) const
 try {
     JSON res;
-    res["drvPath"] = TRY_AWAIT(drvPath.toJSON(store));
+    res["drvPath"] = TRY_AWAIT(drvPath->toJSON(store));
     for (const auto & [outputName, outputPath] : outputs) {
         res["outputs"][outputName] = store.printStorePath(outputPath);
     }
@@ -57,6 +93,36 @@ try {
     co_return result::current_exception();
 }
 
+kj::Promise<Result<JSON>> SingleBuiltPath::Built::toJSON(const Store & store) const
+try {
+    JSON res;
+    res["drvPath"] = TRY_AWAIT(drvPath->toJSON(store));
+    auto & [outputName, outputPath] = output;
+    res["output"] = outputName;
+    res["outputPath"] = store.printStorePath(outputPath);
+    co_return res;
+} catch (...) {
+    co_return result::current_exception();
+}
+
+kj::Promise<Result<JSON>> SingleBuiltPath::toJSON(const Store & store) const
+try {
+    co_return TRY_AWAIT(std::visit([&](const auto & buildable) {
+        return buildable.toJSON(store);
+    }, raw()));
+} catch (...) {
+    co_return result::current_exception();
+}
+
+
+kj::Promise<Result<JSON>> BuiltPath::toJSON(const Store & store) const
+try {
+    co_return TRY_AWAIT(std::visit([&](const auto & buildable) {
+        return buildable.toJSON(store);
+    }, raw()));
+} catch (...) {
+    co_return result::current_exception();
+}
 
 kj::Promise<Result<RealisedPath::Set>> BuiltPath::toRealisedPaths(Store & store) const
 try {
@@ -75,10 +141,24 @@ try {
         [&](const BuiltPath::Built & p) -> kj::Promise<Result<void>> {
             try {
                 auto drvHashes = TRY_AWAIT(
-                    staticOutputHashes(store, TRY_AWAIT(store.readDerivation(p.drvPath.path)))
+                    staticOutputHashes(store, TRY_AWAIT(store.readDerivation(p.drvPath->outPath())))
                 );
                 for (auto& [outputName, outputPath] : p.outputs) {
-                    res.insert(outputPath);
+                    if (experimentalFeatureSettings.isEnabled(
+                                Xp::CaDerivations)) {
+                        auto drvOutput = get(drvHashes, outputName);
+                        if (!drvOutput)
+                            throw Error(
+                                "the derivation '%s' has unrealised output '%s' (derived-path.cc/toRealisedPaths)",
+                                store.printStorePath(p.drvPath->outPath()), outputName);
+                        auto thisRealisation = TRY_AWAIT(store.queryRealisation(
+                            DrvOutput{*drvOutput, outputName}));
+                        assert(thisRealisation);  // We’ve built it, so we must
+                                                  // have the realisation
+                        res.insert(*thisRealisation);
+                    } else {
+                        res.insert(outputPath);
+                    }
                 }
                 co_return result::success();
             } catch (...) {

lix/libcmd/built-path.hh

@@ -7,15 +7,63 @@
 
 namespace nix {
 
+struct SingleBuiltPath;
+
+struct SingleBuiltPathBuilt {
+    ref<SingleBuiltPath> drvPath;
+    std::pair<std::string, StorePath> output;
+
+    SingleDerivedPathBuilt discardOutputPath() const;
+
+    std::string to_string(const Store & store) const;
+    static SingleBuiltPathBuilt parse(const Store & store, std::string_view, std::string_view);
+    kj::Promise<Result<JSON>> toJSON(const Store & store) const;
+
+    DECLARE_CMP(SingleBuiltPathBuilt);
+};
+
+namespace built_path::detail {
+using SingleBuiltPathRaw = std::variant<
+    DerivedPathOpaque,
+    SingleBuiltPathBuilt
+>;
+}
+
+struct SingleBuiltPath : built_path::detail::SingleBuiltPathRaw {
+    using Raw = built_path::detail::SingleBuiltPathRaw;
+    using Raw::Raw;
+
+    using Opaque = DerivedPathOpaque;
+    using Built = SingleBuiltPathBuilt;
+
+    inline const Raw & raw() const {
+        return static_cast<const Raw &>(*this);
+    }
+
+    StorePath outPath() const;
+
+    SingleDerivedPath discardOutputPath() const;
+
+    static SingleBuiltPath parse(const Store & store, std::string_view);
+    kj::Promise<Result<JSON>> toJSON(const Store & store) const;
+};
+
+static inline ref<SingleBuiltPath> staticDrv(StorePath drvPath)
+{
+    return make_ref<SingleBuiltPath>(SingleBuiltPath::Opaque { drvPath });
+}
+
 /**
  * A built derived path with hints in the form of optional concrete output paths.
  *
  * See 'BuiltPath' for more an explanation.
  */
 struct BuiltPathBuilt {
-    DerivedPathOpaque drvPath;
+    ref<SingleBuiltPath> drvPath;
     std::map<std::string, StorePath> outputs;
 
+    std::string to_string(const Store & store) const;
+    static BuiltPathBuilt parse(const Store & store, std::string_view, std::string_view);
     kj::Promise<Result<JSON>> toJSON(const Store & store) const;
 
     DECLARE_CMP(BuiltPathBuilt);
@@ -45,6 +93,8 @@ struct BuiltPath : built_path::detail::BuiltPathRaw {
 
     StorePathSet outPaths() const;
     kj::Promise<Result<RealisedPath::Set>> toRealisedPaths(Store & store) const;
+
+    kj::Promise<Result<JSON>> toJSON(const Store & store) const;
 };
 
 typedef std::vector<BuiltPath> BuiltPaths;

lix/libcmd/common-eval-args.cc

@@ -9,7 +9,6 @@
 #include "lix/libstore/store-api.hh"
 #include "lix/libcmd/command.hh"
 #include "lix/libutil/async.hh"
-#include "lix/libutil/error.hh"
 #include "lix/libutil/regex.hh"
 
 #include <regex>
@@ -17,36 +16,31 @@
 namespace nix {
 
 static std::regex const identifierRegex = regex::parse("^[A-Za-z_][A-Za-z0-9_'-]*$");
-static void checkValidNixIdentifier(const std::string & name)
+static void warnInvalidNixIdentifier(const std::string & name)
 {
     std::smatch match;
     if (!std::regex_match(name, match, identifierRegex)) {
-        throw UsageError(
-            "This invocation specifies a value for argument '%s' "
-            "which isn't a valid Nix identifier. "
-            "The project is dropping support for this so that it's possible to make e.g. "
-            "'%s' evaluating to '%s' in the future. "
-            "If you depend on this behavior, please reach out in "
-            "<https://git.lix.systems/lix-project/lix/issues/496> so we can discuss your use-case.",
-            name,
-            "--arg config.allowUnfree true",
-            "{ config.allowUnfree = true; }"
-        );
+        warn("This Nix invocation specifies a value for argument '%s' which isn't a valid \
+Nix identifier. The project is considering to drop support for this \
+or to require quotes around args that aren't valid Nix identifiers. \
+If you depend on this behvior, please reach out in \
+https://git.lix.systems/lix-project/lix/issues/496 so we can discuss \
+your use-case.", name);
     }
 }
 
 MixEvalArgs::MixEvalArgs()
 {
-    addFlag(
-        {.longName = "arg",
-         .description = "Pass the value *expr* as the argument *name* to Nix functions.",
-         .category = category,
-         .labels = {"name", "expr"},
-         .handler = {[&](std::string name, std::string expr) {
-             checkValidNixIdentifier(name);
-             autoArgs[name] = 'E' + expr;
-         }}}
-    );
+    addFlag({
+        .longName = "arg",
+        .description = "Pass the value *expr* as the argument *name* to Nix functions.",
+        .category = category,
+        .labels = {"name", "expr"},
+        .handler = {[&](std::string name, std::string expr) {
+            warnInvalidNixIdentifier(name);
+            autoArgs[name] = 'E' + expr;
+        }}
+    });
 
     addFlag({
         .longName = "argstr",
@@ -54,7 +48,7 @@ MixEvalArgs::MixEvalArgs()
         .category = category,
         .labels = {"name", "string"},
         .handler = {[&](std::string name, std::string s) {
-            checkValidNixIdentifier(name);
+            warnInvalidNixIdentifier(name);
             autoArgs[name] = 'S' + s;
         }},
     });

lix/libcmd/installable-attr-path.cc

@@ -76,7 +76,7 @@ DerivedPathsWithInfo InstallableAttrPath::toDerivedPaths(EvalState & state)
     for (auto & [drvPath, outputs] : byDrvPath)
         res.push_back({
             .path = DerivedPath::Built {
-                .drvPath = makeConstantStorePath(drvPath),
+                .drvPath = makeConstantStorePathRef(drvPath),
                 .outputs = outputs,
             },
             .info = make_ref<ExtraPathInfoValue>(ExtraPathInfoValue::Value {

lix/libcmd/installable-derived-path.cc

@@ -35,7 +35,7 @@ InstallableDerivedPath InstallableDerivedPath::parse(
             // Remove this prior to stabilizing the new CLI.
             if (storePath.isDerivation()) {
                 auto oldDerivedPath = DerivedPath::Built {
-                    .drvPath = makeConstantStorePath(storePath),
+                    .drvPath = makeConstantStorePathRef(storePath),
                     .outputs = OutputsSpec::All { },
                 };
                 warn(
@@ -48,7 +48,8 @@ InstallableDerivedPath InstallableDerivedPath::parse(
         },
         // If the user did use ^, we just do exactly what is written.
         [&](const ExtendedOutputsSpec::Explicit & outputSpec) -> DerivedPath {
-            auto drv = DerivedPathOpaque::parse(*store, prefix);
+            auto drv = make_ref<SingleDerivedPath>(SingleDerivedPath::parse(*store, prefix));
+            drvRequireExperiment(*drv);
             return DerivedPath::Built {
                 .drvPath = std::move(drv),
                 .outputs = outputSpec,

lix/libcmd/installable-flake.cc

@@ -98,7 +98,7 @@ DerivedPathsWithInfo InstallableFlake::toDerivedPaths(EvalState & state)
 
     return {{
         .path = DerivedPath::Built {
-            .drvPath = makeConstantStorePath(std::move(drvPath)),
+            .drvPath = makeConstantStorePathRef(std::move(drvPath)),
             .outputs = std::visit(overloaded {
                 [&](const ExtendedOutputsSpec::Default & d) -> OutputsSpec {
                     std::set<std::string> outputsToInstall;

lix/libcmd/installables.cc

@@ -524,6 +524,36 @@ ref<Installable> SourceExprCommand::parseInstallable(
     return installables.front();
 }
 
+static kj::Promise<Result<SingleBuiltPath>> getBuiltPath(ref<Store> evalStore, ref<Store> store, const SingleDerivedPath & b)
+try {
+    auto handlers = overloaded{
+        [&](const SingleDerivedPath::Opaque & bo) -> kj::Promise<Result<SingleBuiltPath>> {
+            return {SingleBuiltPath::Opaque { bo.path }};
+        },
+        // NOLINTNEXTLINE(cppcoreguidelines-avoid-capturing-lambda-coroutines)
+        [&](const SingleDerivedPath::Built & bfd) -> kj::Promise<Result<SingleBuiltPath>> {
+            try {
+                auto drvPath = TRY_AWAIT(getBuiltPath(evalStore, store, *bfd.drvPath));
+                // Resolving this instead of `bfd` will yield the same result, but avoid duplicative work.
+                SingleDerivedPath::Built truncatedBfd {
+                    .drvPath = makeConstantStorePathRef(drvPath.outPath()),
+                    .output = bfd.output,
+                };
+                auto outputPath = TRY_AWAIT(resolveDerivedPath(*store, truncatedBfd, &*evalStore));
+                co_return SingleBuiltPath::Built {
+                    .drvPath = make_ref<SingleBuiltPath>(std::move(drvPath)),
+                    .output = { bfd.output, outputPath },
+                };
+            } catch (...) {
+                co_return result::current_exception();
+            }
+        },
+    };
+    co_return TRY_AWAIT(std::visit(handlers, b.raw()));
+} catch (...) {
+    co_return result::current_exception();
+}
+
 std::vector<BuiltPathWithResult> Installable::build(
     EvalState & state,
     ref<Store> evalStore,
@@ -612,7 +642,7 @@ std::vector<std::pair<ref<Installable>, BuiltPathWithResult>> Installable::build
                             state.aio.blockOn(resolveDerivedPath(*store, bfd, &*evalStore));
                         res.push_back({aux.installable, {
                             .path = BuiltPath::Built {
-                                .drvPath = bfd.drvPath,
+                                .drvPath = make_ref<SingleBuiltPath>(state.aio.blockOn(getBuiltPath(evalStore, store, *bfd.drvPath))),
                                 .outputs = outputs,
                              },
                             .info = aux.info}});
@@ -644,7 +674,7 @@ std::vector<std::pair<ref<Installable>, BuiltPathWithResult>> Installable::build
                             outputs.emplace(outputName, realisation.outPath);
                         res.push_back({aux.installable, {
                             .path = BuiltPath::Built {
-                                .drvPath = bfd.drvPath,
+                                .drvPath = make_ref<SingleBuiltPath>(state.aio.blockOn(getBuiltPath(evalStore, store, *bfd.drvPath))),
                                 .outputs = outputs,
                             },
                             .info = aux.info,
@@ -759,7 +789,7 @@ StorePathSet Installable::toDerivations(
                         : throw Error("argument '%s' did not evaluate to a derivation", i->what()));
                 },
                 [&](const DerivedPath::Built & bfd) {
-                    drvPaths.insert(bfd.drvPath.path);
+                    drvPaths.insert(state.aio.blockOn(resolveDerivedPath(*store, *bfd.drvPath)));
                 },
             }, b.path.raw());
 

lix/libcmd/markdown.cc

@@ -3,65 +3,44 @@
 #include "lix/libutil/finally.hh"
 #include "lix/libutil/terminal.hh"
 
-#include <cstdlib>
-#include <iterator>
-#include <new>
-#include <regex>
 #include <sys/queue.h>
 #include <lowdown.h>
 
 namespace nix {
 
-static const std::string DOCROOT = "@docroot@";
-static const std::string DOCROOT_URL = "https://docs.lix.systems/manual/lix/stable";
-
-static void processLinks(struct lowdown_node * node)
-{
-    if (node->type == LOWDOWN_LINK) {
-        struct lowdown_buf *link = &node->rndr_link.link;
-        if (link && link->size && std::string_view(link->data, link->size).starts_with(DOCROOT)) {
-            // link starts with @docroot@, replace that and check the path extension too.
-            static std::regex mdRewrite{"\\.md(#.*)?$"}; // NOLINT(lix-foreign-exceptions)
-
-            auto oldLink = std::string_view(link->data, link->size).substr(DOCROOT.size());
-            std::string newLink = DOCROOT_URL;
-            std::regex_replace(
-                std::back_inserter(newLink), oldLink.begin(), oldLink.end(), mdRewrite, ".html$1"
-            );
-            if (link->maxsize < newLink.size()) {
-                // the existing link buffer doesn't have enough space for the new string
-                char *newData;
-                if (!(newData = static_cast<char *>(std::realloc(link->data, newLink.size())))) {
-                    throw std::bad_alloc();
-                }
-                link->data = newData;
-                link->maxsize = newLink.size();
-            }
-            newLink.copy(link->data, newLink.size());
-            link->size = newLink.size();
-        }
-    } else {
-        // recurse into children
-        struct lowdown_node *child;
-        TAILQ_FOREACH(child, &node->children, entries)
-            processLinks(child);
-    }
-}
-
-std::string renderMarkdownToTerminal(std::string_view markdown, StandardOutputStream fileno)
+std::string renderMarkdownToTerminal(std::string_view markdown)
 {
     int windowWidth = getWindowSize().second;
+    size_t lowdown_cols = std::max(windowWidth - 5, 60);
 
-    struct lowdown_opts opts {
+    struct lowdown_opts opts{
         .type = LOWDOWN_TERM,
+#ifdef LOWDOWN_SEPARATE_TERM_OPTS
+        .term =
+            {
+                .cols = lowdown_cols,
+                .width = 0,
+                .hmargin = 0,
+                .hpadding = 4,
+                .vmargin = 0,
+                .centre = 0,
+            },
+        // maxdepth needs to be part of the ifdefs to match declaration order
         .maxdepth = 20,
-        .cols = (size_t) std::max(windowWidth - 5, 60),
+#else
+        .maxdepth = 20,
+        .cols = lowdown_cols,
         .hmargin = 0,
         .vmargin = 0,
+#endif /* LOWDOWN_SEPARATE_TERM_OPTS */
         .feat = LOWDOWN_COMMONMARK | LOWDOWN_FENCED | LOWDOWN_DEFLIST | LOWDOWN_TABLES,
+#ifdef LOWDOWN_CONSOLIDATED_OFLAGS
+        .oflags = LOWDOWN_NOLINK,
+#else
         .oflags = LOWDOWN_TERM_NOLINK,
+#endif /* LOWDOWN_CONSOLIDATED_OFLAGS */
     };
-    if (!shouldANSI(fileno)) {
+    if (!shouldANSI()) {
         opts.oflags |= LOWDOWN_TERM_NOANSI;
     }
 
@@ -76,8 +55,6 @@ std::string renderMarkdownToTerminal(std::string_view markdown, StandardOutputSt
         throw Error("cannot parse Markdown document");
     Finally freeNode([&]() { lowdown_node_free(node); });
 
-    processLinks(node);
-
     auto renderer = lowdown_term_new(&opts);
     if (!renderer)
         throw Error("cannot allocate Markdown renderer");

lix/libcmd/markdown.hh

@@ -1,11 +1,10 @@
 #pragma once
 ///@file
 
-#include "lix/libutil/terminal.hh"
 #include "lix/libutil/types.hh"
 
 namespace nix {
 
-std::string renderMarkdownToTerminal(std::string_view markdown, StandardOutputStream fileno = StandardOutputStream::Stdout);
+std::string renderMarkdownToTerminal(std::string_view markdown);
 
 }

lix/libcmd/repl.cc

@@ -3,6 +3,7 @@
 #include <iostream>
 #include <cstdlib>
 #include <cstring>
+#include <climits>
 #include <string_view>
 
 #include "lix/libutil/box_ptr.hh"
@@ -759,7 +760,7 @@ ProcessLineResult NixRepl::processLine(std::string line)
 
             state.aio.blockOn(evaluator.store->buildPaths({
                 DerivedPath::Built {
-                    .drvPath = makeConstantStorePath(drvPath),
+                    .drvPath = makeConstantStorePathRef(drvPath),
                     .outputs = OutputsSpec::All { },
                 },
             }));
@@ -1007,15 +1008,7 @@ Value * NixRepl::replOverlays()
     for (auto path : evalSettings.replOverlays.get()) {
         debug("Loading '%1%' path '%2%'...", "repl-overlays", path);
         SourcePath sourcePath((CanonPath(path)));
-
-        // XXX(jade): This is a somewhat unsatisfying solution to
-        // https://git.lix.systems/lix-project/lix/issues/777 which means that
-        // the top level item in the repl-overlays file (that is, the lambda)
-        // gets evaluated with pure eval off. This means that if you want to do
-        // impure eval stuff, you will have to force it with builtins.seq.
-        bool prevPureEval = evalSettings.pureEval.get();
         auto replInit = evalFile(sourcePath);
-        evalSettings.pureEval.setDefault(prevPureEval);
 
         if (!replInit->isLambda()) {
             evaluator.errors.make<TypeError>(
@@ -1063,24 +1056,19 @@ template<typename T, typename NameFn, typename ValueFn>
 void NixRepl::addToScope(T && things, NameFn nameFn, ValueFn valueFn)
 {
     size_t added = 0;
+    for (auto && thing : things) {
+        if (displ + 1 >= envSize)
+            throw Error("environment full; cannot add more variables");
 
-    staticEnv->vars.unsafe_insert_bulk([&] (auto & map) {
-        auto oldSize = map.size();
-        for (auto && thing : things) {
-            if (displ + 1 >= envSize)
-                throw Error("environment full; cannot add more variables");
-
-            const auto name = nameFn(thing);
-            map.emplace_back(name, displ);
-            env->values[displ++] = valueFn(thing);
-            varNames.emplace(evaluator.symbols[name]);
-            added++;
-        }
-        // safety: we sort the range that we inserted so that we don't have to push that
-        // invariant up to the caller
-        std::sort(map.begin() + oldSize, map.end());
-    });
+        const auto name = nameFn(thing);
+        staticEnv->vars.emplace_back(name, displ);
+        env->values[displ++] = valueFn(thing);
+        varNames.emplace(evaluator.symbols[name]);
+        added++;
+    }
 
+    staticEnv->sort();
+    staticEnv->deduplicate();
     if (added > 0) {
         notice("Added %1% variables.", added);
     }
@@ -1105,11 +1093,14 @@ void NixRepl::addVarToScope(const Symbol name, Value & v)
 {
     if (displ >= envSize)
         throw Error("environment full; cannot add more variables");
-    if (staticEnv->vars.insert_or_assign(name, displ).second) {
+    if (auto oldVar = staticEnv->find(name); oldVar != staticEnv->vars.end()) {
+        staticEnv->vars.erase(oldVar);
         notice("Updated %s.", evaluator.symbols[name]);
     } else {
         notice("Added %s.", evaluator.symbols[name]);
     }
+    staticEnv->vars.emplace_back(name, displ);
+    staticEnv->sort();
     env->values[displ++] = &v;
     varNames.emplace(evaluator.symbols[name]);
 }

lix/libexpr/builtins/fetchGit.md

@@ -48,69 +48,6 @@ attribute with the following attributes (all except `url` optional):
   With this argument being true, it's possible to load a `rev` from *any* `ref`
   (by default only `rev`s from the specified `ref` are supported).
 
-- `narHash`
-
-  If given, the source is first looked-up in the Nix store and the [substituters](@docroot@/command-ref/conf-file.md#conf-substituters), and only fetched if not available.
-
-The return value is an attrset containing the following keys:
-
-- `lastModified` (`integer`)
-
-  Unix timestamp of the last update.
-  This corresponds to the timestamp of the "committer" timestamp embedded in the fetched commit.
-
-- `lastModifiedDate` (`string`)
-
-  Textual representation of the `lastModified` timestamp in UTC (the timezone embedded in the git commit is discarded).
-
-- `outPath` (`string`)
-
-  Resulting store path of the fetch process.
-
-- `narHash` (`string`)
-
-  SRI representation of the hash of the `outPath`.
-
-- `rev` (`string`)
-
-  The full-length revision fetched from the remote.
-  For further information see the `rev` input parameter.
-  This will usually be the output of `git rev-parse <rev>` (or `ref` when no `rev` is provided as an input parameter).
-
-- `revCount` (`integer`)
-
-  Number of revisions in the history of the revision fetched.
-  For a repository with a single commit (the root) this number equals 1.
-  Fetches of shallow repositories report a value of 0.
-
-- `shortRev` (`string`)
-
-  A short representation of the `rev`.
-  This string is a *truncated* version of the `rev`.
-  It is of fixed length and therefore not guaranteed to be unique (unlike the output of `git rev-parse --short`).
-  Future versions of Lix may change the length of this string only as part of a breaking change.
-  For maximum reproducibility and interoperability it is recommended to not rely on this value and to truncate the returned `rev` to an appropriate value instead.
-
-- `submodules` (`boolean`)
-
-  Indicates whether submodules have been fetched.
-  If this value is set to `true`, any submodules are already checked out in the resulting `outPath`.
-
-A full example of the output:
-
-```nix
-{
-  lastModified = 1746827286;
-  lastModifiedDate = "20250509214806";
-  narHash = "sha256-qCRBy8Bbh5XhPalPkhonxNgfsbw3lP0UIXBLSrhxAvI=";
-  outPath = "/nix/store/2qdnzhzccspwm70mni7jkvrfkpwcb3jn-source";
-  rev = "dcb0a97000d50b2868ed4f8d9fd465c5a5b8eb3a";
-  revCount = 17845;
-  shortRev = "dcb0a97";
-  submodules = false;
-}
-```
-
 Here are some examples of how to use `fetchGit`.
 
   - To fetch a private repository over SSH:

lix/libexpr/builtins/outputOf.md

@@ -0,0 +1,23 @@
+---
+name: outputOf
+args: [derivation-reference, output-name]
+experimentalFeature: dynamic-derivations
+---
+Return the output path of a derivation, literally or using a placeholder if needed.
+
+If the derivation has a statically-known output path (i.e. the derivation output is input-addressed, or fixed content-addressed), the output path will just be returned.
+But if the derivation is content-addressed or if the derivation is itself not-statically produced (i.e. is the output of another derivation), a placeholder will be returned instead.
+
+*`derivation reference`* must be a string that may contain a regular store path to a derivation, or may be a placeholder reference. If the derivation is produced by a derivation, you must explicitly select `drv.outPath`.
+This primop can be chained arbitrarily deeply.
+For instance,
+
+```nix
+builtins.outputOf
+  (builtins.outputOf myDrv "out)
+  "out"
+```
+
+will return a placeholder for the output of the output of `myDrv`.
+
+This primop corresponds to the `^` sigil for derivable paths, e.g. as part of installable syntax on the command line.

lix/libexpr/eval-cache.cc

@@ -580,7 +580,7 @@ string_t AttrCursor::getStringWithContext(EvalState & state)
                             return d.drvPath;
                         },
                         [&](const NixStringContextElem::Built & b) -> const StorePath & {
-                            return b.drvPath.path;
+                            return b.drvPath->getBaseStorePath();
                         },
                         [&](const NixStringContextElem::Opaque & o) -> const StorePath & {
                             return o.path;

lix/libexpr/eval-inline.hh

@@ -73,6 +73,7 @@ void EvalState::forceValue(Value & v, const PosIdx pos)
         Expr & expr = *v.thunk.expr;
         try {
             v.mkBlackhole();
+            //checkInterrupt();
             expr.eval(*this, *env, v);
         } catch (...) {
             v.mkThunk(env, expr);

lix/libexpr/eval.cc

@@ -14,6 +14,7 @@
 #include "lix/libutil/types.hh"
 #include "lix/libstore/store-api.hh"
 #include "lix/libstore/derivations.hh"
+#include "lix/libstore/downstream-placeholder.hh"
 #include "lix/libexpr/gc-alloc.hh"
 #include "lix/libstore/filetransfer.hh"
 #include "lix/libexpr/function-trace.hh"
@@ -548,7 +549,7 @@ void EvalBuiltins::addConstant(const std::string & name, Value * v, Constant inf
             assert(info.type == gotType);
 
         /* Install value the base environment. */
-        staticEnv->vars.insert_or_assign(symbols.create(name), baseEnvDispl);
+        staticEnv->vars.emplace_back(symbols.create(name), baseEnvDispl);
         env.values[baseEnvDispl++] = v;
         env.values[0]->attrs->push_back(Attr(symbols.create(name2), v));
     }
@@ -583,7 +584,7 @@ Value * EvalBuiltins::addPrimOp(PrimOp && primOp)
 
     Value * v = mem.allocValue();
     v->mkPrimOp(new PrimOp(primOp));
-    staticEnv->vars.insert_or_assign(auto(envName), baseEnvDispl);
+    staticEnv->vars.emplace_back(envName, baseEnvDispl);
     env.values[baseEnvDispl++] = v;
     env.values[0]->attrs->push_back(Attr(symbols.create(primOp.name), v));
     return v;
@@ -887,18 +888,29 @@ void EvalPaths::mkStorePathString(const StorePath & p, Value & v)
 
 
 std::string EvalState::mkOutputStringRaw(
-    const StorePath & staticOutputPath)
+    const SingleDerivedPath::Built & b,
+    std::optional<StorePath> optStaticOutputPath,
+    const ExperimentalFeatureSettings & xpSettings)
 {
-    return ctx.store->printStorePath(staticOutputPath);
+    /* In practice, this is testing for the case of CA derivations, or
+       dynamic derivations. */
+    return optStaticOutputPath
+        ? ctx.store->printStorePath(std::move(*optStaticOutputPath))
+        /* Downstream we would substitute this for an actual path once
+           we build the floating CA derivation */
+        : DownstreamPlaceholder::fromSingleDerivedPathBuilt(b, xpSettings).render();
 }
 
 
 void EvalState::mkOutputString(
     Value & value,
     const SingleDerivedPath::Built & b,
-    const StorePath & staticOutputPath)
+    std::optional<StorePath> optStaticOutputPath,
+    const ExperimentalFeatureSettings & xpSettings)
 {
-    value.mkString(mkOutputStringRaw(staticOutputPath), NixStringContext { b });
+    value.mkString(
+        mkOutputStringRaw(b, optStaticOutputPath, xpSettings),
+        NixStringContext { b });
 }
 
 
@@ -910,12 +922,19 @@ std::string EvalState::mkSingleDerivedPathStringRaw(
             return ctx.store->printStorePath(o.path);
         },
         [&](const SingleDerivedPath::Built & b) {
-            auto drv = aio.blockOn(ctx.store->readDerivation(b.drvPath.path));
-            auto i = drv.outputs.find(b.output);
-            if (i == drv.outputs.end())
-                throw Error("derivation '%s' does not have output '%s'", b.drvPath.to_string(*ctx.store), b.output);
-            auto staticOutputPath = i->second.path(*ctx.store, drv.name, b.output);
-            return mkOutputStringRaw(staticOutputPath);
+            auto optStaticOutputPath = std::visit(overloaded {
+                [&](const SingleDerivedPath::Opaque & o) {
+                    auto drv = aio.blockOn(ctx.store->readDerivation(o.path));
+                    auto i = drv.outputs.find(b.output);
+                    if (i == drv.outputs.end())
+                        throw Error("derivation '%s' does not have output '%s'", b.drvPath->to_string(*ctx.store), b.output);
+                    return i->second.path(*ctx.store, drv.name, b.output);
+                },
+                [&](const SingleDerivedPath::Built & o) -> std::optional<StorePath> {
+                    return std::nullopt;
+                },
+            }, b.drvPath->raw());
+            return mkOutputStringRaw(b, optStaticOutputPath);
         }
     }, p.raw());
 }
@@ -1954,7 +1973,7 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
     NixInt n{0};
     NixFloat nf = 0;
 
-    bool first = !isInterpolation;
+    bool first = !forceString;
     ValueType firstType = nString;
 
     const auto str = [&] {
@@ -2017,17 +2036,12 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
                 state.ctx.errors.make<EvalError>("cannot add %1% to a float", showType(vTmp)).atPos(i_pos).withFrame(env, *this).debugThrow();
         } else {
             if (s.empty()) s.reserve(es.size());
-
-            /* If we are coercing inside of an interpolation, we may allow slightly more comfort by coercing things like integers. */
-            auto coercionMode = isInterpolation && featureSettings.isEnabled(Xp::CoerceIntegers)
-                ? StringCoercionMode::Interpolation : StringCoercionMode::Strict;
-
             /* skip canonization of first path, which would only be not
             canonized in the first place if it's coming from a ./${foo} type
             path */
             auto part = state.coerceToString(i_pos, vTmp, context,
                                              "while evaluating a path segment",
-                                             coercionMode, firstType == nString, !first);
+                                             false, firstType == nString, !first);
             sSize += part->size();
             s.emplace_back(std::move(part));
         }
@@ -2260,7 +2274,7 @@ bool EvalState::isDerivation(Value & v)
 
 
 std::optional<std::string> EvalState::tryAttrsToString(const PosIdx pos, Value & v,
-    NixStringContext & context, StringCoercionMode mode, bool copyToStore)
+    NixStringContext & context, bool coerceMore, bool copyToStore)
 {
     auto i = v.attrs->find(ctx.s.toString);
     if (i != v.attrs->end()) {
@@ -2269,7 +2283,7 @@ std::optional<std::string> EvalState::tryAttrsToString(const PosIdx pos, Value &
             callFunction(*i->value, v, v1, i->pos);
             return coerceToString(pos, v1, context,
                     "while evaluating the result of the `__toString` attribute",
-                    mode, copyToStore).toOwned();
+                    coerceMore, copyToStore).toOwned();
         } catch (EvalError & e) {
             e.addTrace(ctx.positions[pos], "while converting a set to string");
             throw;
@@ -2284,7 +2298,7 @@ BackedStringView EvalState::coerceToString(
     Value & v,
     NixStringContext & context,
     std::string_view errorCtx,
-    StringCoercionMode mode,
+    bool coerceMore,
     bool copyToStore,
     bool canonicalizePath)
 {
@@ -2308,7 +2322,7 @@ BackedStringView EvalState::coerceToString(
     }
 
     if (v.type() == nAttrs) {
-        auto maybeString = tryAttrsToString(pos, v, context, mode, copyToStore);
+        auto maybeString = tryAttrsToString(pos, v, context, coerceMore, copyToStore);
         if (maybeString)
             return std::move(*maybeString);
         auto i = v.attrs->find(ctx.s.outPath);
@@ -2322,29 +2336,24 @@ BackedStringView EvalState::coerceToString(
                 .debugThrow();
         }
         return coerceToString(pos, *i->value, context, errorCtx,
-                              mode, copyToStore, canonicalizePath);
+                              coerceMore, copyToStore, canonicalizePath);
     }
 
     if (v.type() == nExternal) {
         try {
-            return v.external->coerceToString(*this, pos, context, mode, copyToStore);
+            return v.external->coerceToString(*this, pos, context, coerceMore, copyToStore);
         } catch (Error & e) {
             e.addTrace(nullptr, errorCtx);
             throw;
         }
     }
 
-    /* Raito: Any addition to this mode is subject to extra scrutiny
-     * until we have better formatting tools. */
-    if (mode >= StringCoercionMode::Interpolation) {
-        if (v.type() == nInt) return std::to_string(v.integer.value);
-    }
-
-    if (mode >= StringCoercionMode::ToString) {
+    if (coerceMore) {
         /* Note that `false' is represented as an empty string for
            shell scripting convenience, just like `null'. */
         if (v.type() == nBool && v.boolean) return "1";
         if (v.type() == nBool && !v.boolean) return "";
+        if (v.type() == nInt) return std::to_string(v.integer.value);
         if (v.type() == nFloat) return std::to_string(v.fpoint);
         if (v.type() == nNull) return "";
 
@@ -2354,7 +2363,7 @@ BackedStringView EvalState::coerceToString(
                 try {
                     result += *coerceToString(pos, *v2, context,
                             "while evaluating one element of the list",
-                            mode, copyToStore, canonicalizePath);
+                            coerceMore, copyToStore, canonicalizePath);
                 } catch (Error & e) {
                     e.addTrace(ctx.positions[pos], errorCtx);
                     throw;
@@ -2411,7 +2420,7 @@ try {
 
 SourcePath EvalState::coerceToPath(const PosIdx pos, Value & v, NixStringContext & context, std::string_view errorCtx)
 {
-    auto path = coerceToString(pos, v, context, errorCtx, StringCoercionMode::Strict, false, true).toOwned();
+    auto path = coerceToString(pos, v, context, errorCtx, false, false, true).toOwned();
     if (path == "" || path[0] != '/')
         ctx.errors.make<EvalError>("string '%1%' doesn't represent an absolute path", path).withTrace(pos, errorCtx).debugThrow();
     return CanonPath(path);
@@ -2420,7 +2429,7 @@ SourcePath EvalState::coerceToPath(const PosIdx pos, Value & v, NixStringContext
 
 StorePath EvalState::coerceToStorePath(const PosIdx pos, Value & v, NixStringContext & context, std::string_view errorCtx)
 {
-    auto path = coerceToString(pos, v, context, errorCtx, StringCoercionMode::Strict, false, true).toOwned();
+    auto path = coerceToString(pos, v, context, errorCtx, false, false, true).toOwned();
     if (auto storePath = ctx.store->maybeParseStorePath(path))
         return *storePath;
     ctx.errors.make<EvalError>("path '%1%' is not in the Nix store", path).withTrace(pos, errorCtx).debugThrow();
@@ -2475,7 +2484,7 @@ SingleDerivedPath EvalState::coerceToSingleDerivedPath(const PosIdx pos, Value &
             [&](const SingleDerivedPath::Built & b) {
                 ctx.errors.make<EvalError>(
                     "string '%s' has context with the output '%s' from derivation '%s', but the string is not the right placeholder for this derivation output. It should be '%s'",
-                    s, b.output, b.drvPath.to_string(*ctx.store), sExpected)
+                    s, b.output, b.drvPath->to_string(*ctx.store), sExpected)
                     .withTrace(pos, errorCtx).debugThrow(always_progresses);
             }
         }, derivedPath.raw());
@@ -2882,7 +2891,7 @@ try {
 }
 
 
-std::string ExternalValueBase::coerceToString(EvalState & state, const PosIdx & pos, NixStringContext & context, StringCoercionMode mode, bool copyToStore) const
+std::string ExternalValueBase::coerceToString(EvalState & state, const PosIdx & pos, NixStringContext & context, bool copyMore, bool copyToStore) const
 {
     state.ctx.errors.make<TypeError>(
         "cannot coerce %1% to a string: %2%", showType(), *this

lix/libexpr/eval.hh

@@ -767,18 +767,19 @@ public:
     bool isDerivation(Value & v);
 
     std::optional<std::string> tryAttrsToString(const PosIdx pos, Value & v,
-        NixStringContext & context, StringCoercionMode mode = StringCoercionMode::Strict, bool copyToStore = true);
+        NixStringContext & context, bool coerceMore = false, bool copyToStore = true);
 
     /**
      * String coercion.
      *
      * Converts strings, paths and derivations to a
-     * string. If `copyToStore` is set,
+     * string.  If `coerceMore` is set, also converts nulls, integers,
+     * booleans and lists to a string.  If `copyToStore` is set,
      * referenced paths are copied to the Nix store as a side effect.
      */
     BackedStringView coerceToString(const PosIdx pos, Value & v, NixStringContext & context,
         std::string_view errorCtx,
-        StringCoercionMode mode = StringCoercionMode::Strict, bool copyToStore = true,
+        bool coerceMore = false, bool copyToStore = true,
         bool canonicalizePath = true);
 
     /**
@@ -803,7 +804,8 @@ public:
     /**
      * Coerce to `SingleDerivedPath`.
      *
-     * Must be a string which is either a literal store path.
+     * Must be a string which is either a literal store path or a
+     * "placeholder (see `DownstreamPlaceholder`).
      *
      * Even more importantly, the string context must be exactly one
      * element, which is either a `NixStringContextElem::Opaque` or
@@ -868,13 +870,19 @@ public:
      * @param b the drv whose output we are making a string for, and the
      * output
      *
-     * @param staticOutputPath Output path for that string.
-     * Will be printed to form string.
+     * @param optStaticOutputPath Optional output path for that string.
+     * Must be passed if and only if output store object is
+     * input-addressed or fixed output. Will be printed to form string
+     * if passed, otherwise a placeholder will be used (see
+     * `DownstreamPlaceholder`).
+     *
+     * @param xpSettings Stop-gap to avoid globals during unit tests.
      */
     void mkOutputString(
         Value & value,
         const SingleDerivedPath::Built & b,
-        const StorePath & staticOutputPath);
+        std::optional<StorePath> optStaticOutputPath,
+        const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
 
     /**
      * Create a string representing a `SingleDerivedPath`.
@@ -894,7 +902,9 @@ private:
      * string Value, which would also have a string context.
      */
     std::string mkOutputStringRaw(
-        const StorePath & staticOutputPath);
+        const SingleDerivedPath::Built & b,
+        std::optional<StorePath> optStaticOutputPath,
+        const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
 
     /**
      * Like `mkSingleDerivedPathStringRaw` but just creates a raw string

lix/libexpr/fetchurl.nix

@@ -34,7 +34,6 @@
   executable ? false,
   unpack ? false,
   name ? baseNameOf (toString url),
-  # still translates to __impure to trigger derivationStrict error checks.
   impure ? false,
 }:
 

lix/libexpr/flake/flake.cc

@@ -298,7 +298,7 @@ static Flake getFlake(
                 NixStringContext emptyContext = {};
                 flake.config.settings.emplace(
                     state.ctx.symbols[setting.name],
-                    state.coerceToString(setting.pos, *setting.value, emptyContext, "", StringCoercionMode::Strict, true, true) .toOwned());
+                    state.coerceToString(setting.pos, *setting.value, emptyContext, "", false, true, true) .toOwned());
             }
             else if (setting.value->type() == nInt)
                 flake.config.settings.emplace(

lix/libexpr/json-to-value.cc

@@ -7,11 +7,6 @@
 
 namespace nix {
 
-/*
- * Used for `JSONObjectState`
- */
-using ValueMap = GcMap<Symbol, Value *>;
-
 // for more information, refer to
 // https://github.com/nlohmann/json/blob/master/include/nlohmann/detail/input/json_sax.hpp
 class JSONSax : nlohmann::json_sax<JSON> {

lix/libexpr/meson.build

@@ -115,6 +115,7 @@ builtin_definitions = files(
   'builtins/mapAttrs.md',
   'builtins/match.md',
   'builtins/mul.md',
+  'builtins/outputOf.md',
   'builtins/parseDrvName.md',
   'builtins/parseFlakeRef.md',
   'builtins/partition.md',

lix/libexpr/nixexpr.cc

@@ -19,12 +19,6 @@ std::ostream & operator <<(std::ostream & str, const SymbolStr & symbol)
     return printIdentifier(str, s);
 }
 
-std::ostream & operator<<(std::ostream & str, const InternedSymbol & symbol)
-{
-    str << SymbolStr(symbol);
-    return str;
-}
-
 AttrName::AttrName(PosIdx pos, Symbol s) : pos(pos), symbol(s)
 {
 }
@@ -279,7 +273,7 @@ JSON ExprConcatStrings::toJSON(const SymbolTable & symbols) const
         parts.push_back(part->toJSON(symbols));
     return {
         {"_type", "ExprConcatStrings"},
-        {"isInterpolation", isInterpolation},
+        {"forceString", forceString},
         {"es", parts}
     };
 }
@@ -471,7 +465,7 @@ void VarBinder::visit(ExprVar & e, std::unique_ptr<Expr> & ptr)
         if (curEnv->isWith) {
             if (withLevel == -1) withLevel = level;
         } else {
-            auto i = curEnv->vars.find(e.name);
+            auto i = curEnv->find(e.name);
             if (i != curEnv->vars.end()) {
                 if (e.needsRoot && !curEnv->isRoot) {
                     throw ParseError({
@@ -529,12 +523,9 @@ std::shared_ptr<const StaticEnv> ExprAttrs::buildRecursiveEnv(const std::shared_
 {
     auto newEnv = std::make_shared<StaticEnv>(nullptr, env.get(), attrs.size());
 
-    // safety: the attrs is already sorted
-    newEnv->vars.unsafe_insert_bulk([&] (auto & map) {
-        Displacement displ = 0;
-        for (auto & i : attrs)
-            map.emplace_back(i.first, i.second.displ = displ++);
-    });
+    Displacement displ = 0;
+    for (auto & i : attrs)
+        newEnv->vars.emplace_back(i.first, i.second.displ = displ++);
     return newEnv;
 }
 
@@ -671,7 +662,7 @@ void VarBinder::visit(ExprPos & e, std::unique_ptr<Expr> & ptr)
 std::shared_ptr<const StaticEnv> SimplePattern::buildEnv(const StaticEnv * up)
 {
     auto newEnv = std::make_shared<StaticEnv>(nullptr, up, 1);
-    newEnv->vars.insert_or_assign(name, 0);
+    newEnv->vars.emplace_back(name, 0);
     return newEnv;
 }
 
@@ -686,14 +677,12 @@ std::shared_ptr<const StaticEnv> AttrsPattern::buildEnv(const StaticEnv * up)
 
     Displacement displ = 0;
 
-    if (name) newEnv->vars.insert_or_assign(name, displ++);
+    if (name) newEnv->vars.emplace_back(name, displ++);
 
-    // safety: The formals are already sorted
-    newEnv->vars.unsafe_insert_bulk([&] (auto & map) {
-        for (auto & i : formals)
-            map.emplace_back(i.name, displ++);
-    });
+    for (auto & i : formals)
+        newEnv->vars.emplace_back(i.name, displ++);
 
+    newEnv->sort();
     return newEnv;
 }
 

lix/libexpr/nixexpr.hh

@@ -11,7 +11,6 @@
 #include "lix/libexpr/eval-error.hh"
 #include "lix/libexpr/pos-idx.hh"
 #include "lix/libutil/strings.hh"
-#include "lix/libutil/linear-map.hh"
 
 namespace nix {
 
@@ -573,10 +572,10 @@ MakeBinOp(ExprOpConcatLists, "++")
 
 struct ExprConcatStrings : Expr
 {
-    bool isInterpolation;
+    bool forceString;
     std::vector<std::pair<PosIdx, std::unique_ptr<Expr>>> es;
-    ExprConcatStrings(const PosIdx & pos, bool isInterpolation, std::vector<std::pair<PosIdx, std::unique_ptr<Expr>>> es)
-        : Expr(pos), isInterpolation(isInterpolation), es(std::move(es)) { };
+    ExprConcatStrings(const PosIdx & pos, bool forceString, std::vector<std::pair<PosIdx, std::unique_ptr<Expr>>> es)
+        : Expr(pos), forceString(forceString), es(std::move(es)) { };
     JSON toJSON(const SymbolTable & symbols) const override;
     void eval(EvalState & state, Env & env, Value & v) override;
     void accept(ExprVisitor & ev, std::unique_ptr<Expr> & ptr) override { ev.visit(*this, ptr); }
@@ -608,7 +607,9 @@ struct StaticEnv
     ExprWith * isWith;
     const StaticEnv * up;
 
-    LinearMap<Symbol, Displacement> vars;
+    // Note: these must be in sorted order.
+    typedef std::vector<std::pair<Symbol, Displacement>> Vars;
+    Vars vars;
 
     /* See ExprVar::needsRoot */
     bool isRoot = false;
@@ -616,6 +617,31 @@ struct StaticEnv
     StaticEnv(ExprWith * isWith, const StaticEnv * up, size_t expectedSize = 0) : isWith(isWith), up(up) {
         vars.reserve(expectedSize);
     };
+
+    void sort()
+    {
+        std::stable_sort(vars.begin(), vars.end(),
+            [](const Vars::value_type & a, const Vars::value_type & b) { return a.first < b.first; });
+    }
+
+    void deduplicate()
+    {
+        auto it = vars.begin(), jt = it, end = vars.end();
+        while (jt != end) {
+            *it = *jt++;
+            while (jt != end && it->first == jt->first) *it = *jt++;
+            it++;
+        }
+        vars.erase(it, end);
+    }
+
+    Vars::const_iterator find(Symbol name) const
+    {
+        Vars::value_type key(name, 0);
+        auto i = std::lower_bound(vars.begin(), vars.end(), key);
+        if (i != vars.end() && i->first == name) return i;
+        return vars.end();
+    }
 };
 
 

lix/libexpr/primops.cc

@@ -1,5 +1,6 @@
 #include "lix/libutil/archive.hh"
 #include "lix/libstore/derivations.hh"
+#include "lix/libstore/downstream-placeholder.hh"
 #include "lix/libexpr/eval.hh"
 #include "lix/libexpr/eval-settings.hh"
 #include "lix/libexpr/extra-primops.hh"
@@ -36,10 +37,6 @@
 
 namespace nix {
 
-/*
- * Used for `builtins.groupBy`
- */
-using ValueVectorMap = std::map<Symbol, ValueVector>;
 
 /*************************************************************
  * Miscellaneous
@@ -61,7 +58,7 @@ StringMap EvalState::realiseContext(const NixStringContext & context)
                     .drvPath = b.drvPath,
                     .outputs = OutputsSpec::Names { b.output },
                 });
-                return ensureValid(b.drvPath.path);
+                return ensureValid(b.drvPath->getBaseStorePath());
             },
             [&](const NixStringContextElem::Opaque & o) {
                 auto ctxS = ctx.store->printStorePath(o.path);
@@ -96,6 +93,18 @@ StringMap EvalState::realiseContext(const NixStringContext & context)
         auto outputs = aio.blockOn(resolveDerivedPath(*ctx.buildStore, drv, &*ctx.store));
         for (auto & [outputName, outputPath] : outputs) {
             outputsToCopyAndAllow.insert(outputPath);
+
+            /* Get all the output paths corresponding to the placeholders we had */
+            if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) {
+                res.insert_or_assign(
+                    DownstreamPlaceholder::fromSingleDerivedPathBuilt(
+                        SingleDerivedPath::Built {
+                            .drvPath = drv.drvPath,
+                            .output = outputName,
+                        }).render(),
+                    ctx.buildStore->printStorePath(outputPath)
+                );
+            }
         }
     }
 
@@ -157,7 +166,7 @@ static void mkOutputString(
     state.mkOutputString(
         attrs.alloc(o.first),
         SingleDerivedPath::Built {
-            .drvPath = makeConstantStorePath(drvPath),
+            .drvPath = makeConstantStorePathRef(drvPath),
             .output = o.first,
         },
         o.second.path(*state.ctx.store, Derivation::nameFromPath(drvPath), o.first));
@@ -233,14 +242,14 @@ static void import(EvalState & state, Value & vPath, Value * vScope, Value & v)
                 nullptr, state.ctx.builtins.staticEnv.get(), vScope->attrs->size()
             );
 
-            staticEnv->vars.unsafe_insert_bulk([&] (auto & map) {
-                unsigned int displ = 0;
-                for (auto & attr : *vScope->attrs) {
-                    // safety: args[0]->attrs is already sorted.
-                    map.emplace_back(attr.name, displ);
-                    env->values[displ++] = attr.value;
-                }
-            });
+            unsigned int displ = 0;
+            for (auto & attr : *vScope->attrs) {
+                staticEnv->vars.emplace_back(attr.name, displ);
+                env->values[displ++] = attr.value;
+            }
+
+            // No need to call staticEnv.sort(), because
+            // args[0]->attrs is already sorted.
 
             debug("evaluating file '%1%'", path);
             Expr & e = state.ctx.parseExprFromFile(state.ctx.paths.resolveExprPath(path), staticEnv);
@@ -306,13 +315,13 @@ void prim_exec(EvalState & state, Value * * args, Value & v)
     NixStringContext context;
     auto program = state.coerceToString(noPos, *elems[0], context,
             "while evaluating the first element of the argument passed to builtins.exec",
-            StringCoercionMode::Strict, false).toOwned();
+            false, false).toOwned();
     Strings commandArgs;
     for (unsigned int i = 1; i < args[0]->listSize(); ++i) {
         commandArgs.push_back(
                 state.coerceToString(noPos, *elems[i], context,
                         "while evaluating an element of the argument passed to builtins.exec",
-                        StringCoercionMode::Strict, false).toOwned());
+                        false, false).toOwned());
     }
     try {
         auto _ = state.realiseContext(context); // FIXME: Handle CA derivations
@@ -601,7 +610,7 @@ static void prim_addErrorContext(EvalState & state, Value * * args, Value & v)
         NixStringContext context;
         auto message = state.coerceToString(noPos, *args[0], context,
                 "while evaluating the error message passed to builtins.addErrorContext",
-                StringCoercionMode::Strict, false).toOwned();
+                false, false).toOwned();
         e.addTrace(nullptr, HintFmt(message));
         throw;
     }
@@ -801,6 +810,8 @@ drvName, Bindings * attrs, Value & v)
 
     NixStringContext context;
 
+    bool contentAddressed = false;
+    bool isImpure = false;
     std::optional<std::string> outputHash;
     std::string outputHashAlgo;
     std::optional<ContentAddressMethod> ingestionMethod;
@@ -810,13 +821,16 @@ drvName, Bindings * attrs, Value & v)
 
     for (auto & i : attrs->lexicographicOrder(state.ctx.symbols)) {
         if (i->name == state.ctx.s.ignoreNulls) continue;
-        auto & key = state.ctx.symbols[i->name];
+        const std::string & key = state.ctx.symbols[i->name];
         vomit("processing attribute '%1%'", key);
 
         auto handleHashMode = [&](const std::string_view s, NeverAsync = {}) {
             if (s == "recursive") ingestionMethod = FileIngestionMethod::Recursive;
             else if (s == "flat") ingestionMethod = FileIngestionMethod::Flat;
-            else
+            else if (s == "text") {
+                experimentalFeatureSettings.require(Xp::DynamicDerivations);
+                ingestionMethod = TextIngestionMethod {};
+            } else
                 state.ctx.errors.make<EvalError>(
                     "invalid value '%s' for 'outputHashMode' attribute", s
                 ).debugThrow();
@@ -854,13 +868,13 @@ drvName, Bindings * attrs, Value & v)
             }
 
             if (i->name == state.ctx.s.contentAddressed && state.forceBool(*i->value, noPos, context_below)) {
-                state.ctx.errors.make<EvalError>("ca derivations are not supported in Lix")
-                    .debugThrow();
+                contentAddressed = true;
+                experimentalFeatureSettings.require(Xp::CaDerivations);
             }
 
             else if (i->name == state.ctx.s.impure && state.forceBool(*i->value, noPos, context_below)) {
-                state.ctx.errors.make<EvalError>("impure derivations are not supported in Lix")
-                    .debugThrow();
+                isImpure = true;
+                experimentalFeatureSettings.require(Xp::ImpureDerivations);
             }
 
             /* The `args' attribute is special: it supplies the
@@ -870,7 +884,7 @@ drvName, Bindings * attrs, Value & v)
                 for (auto elem : i->value->listItems()) {
                     auto s = state.coerceToString(noPos, *elem, context,
                             "while evaluating an element of the argument list",
-                            StringCoercionMode::ToString).toOwned();
+                            true).toOwned();
                     drv.args.push_back(s);
                 }
             }
@@ -919,7 +933,7 @@ drvName, Bindings * attrs, Value & v)
 
 
                 } else {
-                    auto s = state.coerceToString(noPos, *i->value, context, context_below, StringCoercionMode::ToString).toOwned();
+                    auto s = state.coerceToString(noPos, *i->value, context, context_below, true).toOwned();
                     drv.env.emplace(key, s);
                     if (i->name == state.ctx.s.builder) drv.builder = std::move(s);
                     else if (i->name == state.ctx.s.system) drv.platform = std::move(s);
@@ -961,13 +975,13 @@ drvName, Bindings * attrs, Value & v)
                 for (auto & j : refs) {
                     drv.inputSrcs.insert(j);
                     if (j.isDerivation()) {
-                        drv.inputDrvs[j] =
+                        drv.inputDrvs.map[j].value =
                             state.aio.blockOn(state.ctx.store->readDerivation(j)).outputNames();
                     }
                 }
             },
             [&](const NixStringContextElem::Built & b) {
-                drv.inputDrvs[b.drvPath.path].insert(b.output);
+                drv.inputDrvs.ensureSlot(*b.drvPath).value.insert(b.output);
             },
             [&](const NixStringContextElem::Opaque & o) {
                 drv.inputSrcs.insert(o.path);
@@ -985,10 +999,15 @@ drvName, Bindings * attrs, Value & v)
             .debugThrow();
 
     /* Check whether the derivation name is valid. */
-    if (isDerivation(drvName)) {
-        state.ctx.errors
-            .make<EvalError>("derivation names are not allowed to end in '%s'", drvExtension)
-            .debugThrow();
+    if (isDerivation(drvName) &&
+        !(ingestionMethod == ContentAddressMethod { TextIngestionMethod { } } &&
+          outputs.size() == 1 &&
+          *(outputs.begin()) == "out"))
+    {
+        state.ctx.errors.make<EvalError>(
+            "derivation names are allowed to end in '%s' only if they produce a single derivation file",
+            drvExtension
+        ).debugThrow();
     }
 
     if (outputHash) {
@@ -1016,6 +1035,31 @@ drvName, Bindings * attrs, Value & v)
         drv.outputs.insert_or_assign("out", std::move(dof));
     }
 
+    else if (contentAddressed || isImpure) {
+        if (contentAddressed && isImpure)
+            state.ctx.errors.make<EvalError>("derivation cannot be both content-addressed and impure")
+                .debugThrow();
+
+        auto ht = parseHashTypeOpt(outputHashAlgo).value_or(HashType::SHA256);
+        auto method = ingestionMethod.value_or(FileIngestionMethod::Recursive);
+
+        for (auto & i : outputs) {
+            drv.env[i] = hashPlaceholder(i);
+            if (isImpure)
+                drv.outputs.insert_or_assign(i,
+                    DerivationOutput::Impure {
+                        .method = method,
+                        .hashType = ht,
+                    });
+            else
+                drv.outputs.insert_or_assign(i,
+                    DerivationOutput::CAFloating {
+                        .method = method,
+                        .hashType = ht,
+                    });
+        }
+    }
+
     else {
         /* Compute a hash over the "masked" store derivation, which is
            the final one except that in the list of outputs, the
@@ -1026,25 +1070,34 @@ drvName, Bindings * attrs, Value & v)
         for (auto & i : outputs) {
             drv.env[i] = "";
             drv.outputs.insert_or_assign(i,
-                DerivationOutput::InputAddressed { .path = StorePath::dummy });
+                DerivationOutput::Deferred { });
         }
 
         auto hashModulo =
             state.aio.blockOn(hashDerivationModulo(*state.ctx.store, Derivation(drv), true));
-        for (auto & i : outputs) {
-            auto h = get(hashModulo.hashes, i);
-            if (!h)
-                state.ctx.errors.make<AssertionError>(
-                    "derivation produced no hash for output '%s'",
-                    i
-                ).debugThrow();
-            auto outPath = state.ctx.store->makeOutputPath(i, *h, drvName);
-            drv.env[i] = state.ctx.store->printStorePath(outPath);
-            drv.outputs.insert_or_assign(
-                i,
-                DerivationOutput::InputAddressed {
-                    .path = std::move(outPath),
-                });
+        switch (hashModulo.kind) {
+        case DrvHash::Kind::Regular:
+            for (auto & i : outputs) {
+                auto h = get(hashModulo.hashes, i);
+                if (!h)
+                    state.ctx.errors.make<AssertionError>(
+                        "derivation produced no hash for output '%s'",
+                        i
+                    ).debugThrow();
+                auto outPath = state.ctx.store->makeOutputPath(i, *h, drvName);
+                drv.env[i] = state.ctx.store->printStorePath(outPath);
+                drv.outputs.insert_or_assign(
+                    i,
+                    DerivationOutput::InputAddressed {
+                        .path = std::move(outPath),
+                    });
+            }
+            break;
+            ;
+        case DrvHash::Kind::Deferred:
+            for (auto & i : outputs) {
+                drv.outputs.insert_or_assign(i, DerivationOutput::Deferred {});
+            }
         }
     }
 
@@ -1179,7 +1232,7 @@ static void prim_baseNameOf(EvalState & state, Value * * args, Value & v)
     NixStringContext context;
     v.mkString(baseNameOf(*state.coerceToString(noPos, *args[0], context,
             "while evaluating the first argument passed to builtins.baseNameOf",
-            StringCoercionMode::Strict, false)), context);
+            false, false)), context);
 }
 
 /* Return the directory of the given path, i.e., everything before the
@@ -1195,7 +1248,7 @@ static void prim_dirOf(EvalState & state, Value * * args, Value & v)
         NixStringContext context;
         auto path = state.coerceToString(noPos, *args[0], context,
             "while evaluating the first argument passed to 'builtins.dirOf'",
-            StringCoercionMode::Strict, false);
+            false, false);
         auto dir = dirOf(*path);
         v.mkString(dir, context);
     }
@@ -1256,7 +1309,7 @@ static void prim_findFile(EvalState & state, Value * * args, Value & v)
         NixStringContext context;
         auto path = state.coerceToString(noPos, *i->value, context,
                 "while evaluating the `path` attribute of an element of the list passed to builtins.findFile",
-                StringCoercionMode::Strict, false).toOwned();
+                false, false).toOwned();
 
         try {
             auto rewrites = state.realiseContext(context);
@@ -1349,6 +1402,21 @@ static void prim_readDir(EvalState & state, Value * * args, Value & v)
     v.mkAttrs(attrs);
 }
 
+/* Extend single element string context with another output. */
+static void prim_outputOf(EvalState & state, Value * * args, Value & v)
+{
+    SingleDerivedPath drvPath = state.coerceToSingleDerivedPath(noPos, *args[0], "while evaluating the first argument to builtins.outputOf");
+
+    OutputNameView outputName = state.forceStringNoCtx(*args[1], noPos, "while evaluating the second argument to builtins.outputOf");
+
+    state.mkSingleDerivedPathString(
+        SingleDerivedPath::Built {
+            .drvPath = make_ref<SingleDerivedPath>(drvPath),
+            .output = std::string { outputName },
+        },
+        v);
+}
+
 /*************************************************************
  * Creating files
  *************************************************************/
@@ -1539,7 +1607,7 @@ static void prim_path(EvalState & state, Value * * args, Value & v)
     state.forceAttrs(*args[0], noPos, "while evaluating the argument passed to 'builtins.path'");
 
     for (auto & attr : *args[0]->attrs) {
-        auto & n = state.ctx.symbols[attr.name];
+        auto n = state.ctx.symbols[attr.name];
         if (n == "path")
             path.emplace(state.coerceToPath(attr.pos, *attr.value, context, "while evaluating the 'path' attribute passed to 'builtins.path'"));
         else if (attr.name == state.ctx.s.name)
@@ -1582,7 +1650,7 @@ static void prim_attrNames(EvalState & state, Value * * args, Value & v)
 
     size_t n = 0;
     for (auto & i : *args[0]->attrs)
-        v.listElems()[n++] = const_cast<Value *>(state.ctx.symbols[i.name].toValuePtr());
+        (v.listElems()[n++] = state.ctx.mem.allocValue())->mkString(state.ctx.symbols[i.name]);
 
     std::sort(v.listElems(), v.listElems() + n,
               [](Value * v1, Value * v2) { return strcmp(v1->string.s, v2->string.s) < 0; });
@@ -1881,8 +1949,9 @@ static void prim_mapAttrs(EvalState & state, Value * * args, Value & v)
     auto attrs = state.ctx.buildBindings(args[1]->attrs->size());
 
     for (auto & i : *args[1]->attrs) {
+        Value * vName = state.ctx.mem.allocValue();
         Value * vFun2 = state.ctx.mem.allocValue();
-        auto vName = const_cast<Value *>(state.ctx.symbols[i.name].toValuePtr());
+        vName->mkString(state.ctx.symbols[i.name]);
         vFun2->mkApp(args[0], vName);
         attrs.alloc(i.name).mkApp(vFun2, i.value);
     }
@@ -1928,7 +1997,8 @@ static void prim_zipAttrsWith(EvalState & state, Value * * args, Value & v)
     }
 
     for (auto & attr : *v.attrs) {
-        auto name = const_cast<Value *>(state.ctx.symbols[attr.name].toValuePtr());
+        auto name = state.ctx.mem.allocValue();
+        name->mkString(state.ctx.symbols[attr.name]);
         auto call1 = state.ctx.mem.allocValue();
         call1->mkApp(args[0], name);
         auto call2 = state.ctx.mem.allocValue();
@@ -2418,7 +2488,7 @@ static void prim_toString(EvalState & state, Value * * args, Value & v)
     NixStringContext context;
     auto s = state.coerceToString(noPos, *args[0], context,
             "while evaluating the first argument passed to builtins.toString",
-            StringCoercionMode::ToString, false);
+            true, false);
     v.mkString(*s, context);
 }
 
@@ -2800,6 +2870,7 @@ void EvalBuiltins::createBaseEnv(const SearchPath & searchPath, const Path & sto
        because attribute lookups expect it to be sorted. */
     env.values[0]->attrs->sort();
 
+    staticEnv->sort();
     staticEnv->isRoot = true;
 }
 

lix/libexpr/primops/context.cc

@@ -127,7 +127,9 @@ void prim_getContext(EvalState & state, Value * * args, Value & v)
                 contextInfos[std::move(d.drvPath)].allOutputs = true;
             },
             [&](NixStringContextElem::Built && b) {
-                auto drvPath = b.drvPath.path;
+                // FIXME should eventually show string context as is, no
+                // resolving here.
+                auto drvPath = state.aio.blockOn(resolveDerivedPath(*state.ctx.store, *b.drvPath));
                 contextInfos[std::move(drvPath)].outputs.emplace_back(std::move(b.output));
             },
             [&](NixStringContextElem::Opaque && o) {
@@ -217,7 +219,7 @@ static void prim_appendContext(EvalState & state, Value * * args, Value & v)
             for (auto elem : iter->value->listItems()) {
                 auto outputName = state.forceStringNoCtx(*elem, iter->pos, "while evaluating an output name within a string context");
                 context.emplace(NixStringContextElem::Built {
-                    .drvPath = makeConstantStorePath(namePath),
+                    .drvPath = makeConstantStorePathRef(namePath),
                     .output = std::string { outputName },
                 });
             }

lix/libexpr/primops/fetchMercurial.cc

@@ -22,7 +22,7 @@ static void prim_fetchMercurial(EvalState & state, Value * * args, Value & v)
             if (n == "url")
                 url = state.coerceToString(attr.pos, *attr.value, context,
                         "while evaluating the `url` attribute passed to builtins.fetchMercurial",
-                        StringCoercionMode::Strict, false).toOwned();
+                        false, false).toOwned();
             else if (n == "rev") {
                 // Ugly: unlike fetchGit, here the "rev" attribute can
                 // be both a revision or a branch/tag name.
@@ -44,7 +44,7 @@ static void prim_fetchMercurial(EvalState & state, Value * * args, Value & v)
     } else
         url = state.coerceToString(noPos, *args[0], context,
                 "while evaluating the first argument passed to builtins.fetchMercurial",
-                StringCoercionMode::Strict, false).toOwned();
+                false, false).toOwned();
 
     // FIXME: git externals probably can be used to bypass the URI
     // whitelist. Ah well.

lix/libexpr/primops/fetchTree.cc

@@ -139,7 +139,7 @@ static void fetchTree(
             if (attr.name == state.ctx.s.type) continue;
             state.forceValue(*attr.value, attr.pos);
             if (attr.value->type() == nPath || attr.value->type() == nString) {
-                auto s = state.coerceToString(attr.pos, *attr.value, context, "", StringCoercionMode::Strict, false).toOwned();
+                auto s = state.coerceToString(attr.pos, *attr.value, context, "", false, false).toOwned();
                 attrs.emplace(state.ctx.symbols[attr.name],
                     state.ctx.symbols[attr.name] == "url"
                     ? type == "git"
@@ -169,18 +169,11 @@ static void fetchTree(
                     "attribute 'name' isn’t supported in call to 'fetchTree'"
                 ).atPos(pos).debugThrow();
 
-        // HACK: When using `fetchGit`, locking with only the hash should happen
-        //       as we don't care about flake shenanigans about `lastModified`
-        if (type == "git" && attrs.contains("narHash")) {
-            using namespace std::literals::string_literals;
-            attrs["type"] = "\0git-locked"s;
-        }
-
         input = fetchers::Input::fromAttrs(std::move(attrs));
     } else {
         auto url = state.coerceToString(pos, *args[0], context,
                 "while evaluating the first argument passed to the fetcher",
-                StringCoercionMode::Strict, false).toOwned();
+                false, false).toOwned();
 
         if (type == "git") {
             fetchers::Attrs attrs;

lix/libexpr/primops/fromTOML.cc

@@ -6,82 +6,164 @@
 
 namespace nix {
 
-void prim_fromTOML(EvalState & state, Value * * args, Value & val)
+#if HAVE_TOML11_4
+
+/**
+ * This is what toml11 < 4.0 did when choosing the subsecond precision.
+ * TOML 1.0.0 spec doesn't define how sub-millisecond ranges should be handled and calls it
+ * implementation defined behavior. For a lack of a better choice we stick with what older versions
+ * of toml11 did [1].
+ *
+ * [1]:
+ * https://github.com/ToruNiina/toml11/blob/dcfe39a783a94e8d52c885e5883a6fbb21529019/toml/datetime.hpp#L282
+ */
+static size_t normalizeSubsecondPrecision(toml::local_time lt)
 {
-    auto toml = state.forceStringNoCtx(*args[0], noPos, "while evaluating the argument passed to builtins.fromTOML");
+    auto millis = lt.millisecond;
+    auto micros = lt.microsecond;
+    auto nanos = lt.nanosecond;
+    if (millis != 0 || micros != 0 || nanos != 0) {
+        if (micros != 0 || nanos != 0) {
+            if (nanos != 0) {
+                return 9;
+            }
+            return 6;
+        }
+        return 3;
+    }
+    return 0;
+}
+
+/**
+ * Normalize date/time formats to serialize to the same strings as versions prior to toml11 4.0.
+ *
+ * Several things to consider:
+ *
+ * 1. Sub-millisecond range is represented the same way as in toml11 versions prior to 4.0.
+ * Precision is rounded towards the next multiple of 3 or capped at 9 digits.
+ * 2. Seconds must be specified. This may become optional in (yet unreleased) TOML 1.1.0, but 1.0.0
+ * defined local time in terms of RFC3339 [1].
+ * 3. date-time separator (`t`, `T` or space ` `) is canonicalized to an upper T. This is compliant
+ * with RFC3339 [1] 5.6: > Applications that generate this format SHOULD use upper case letters.
+ *
+ * [1]: https://datatracker.ietf.org/doc/html/rfc3339#section-5.6
+ */
+static void normalizeDatetimeFormat(toml::value & t)
+{
+    if (t.is_local_datetime()) {
+        auto & ldt = t.as_local_datetime();
+        t.as_local_datetime_fmt() = {
+            .delimiter = toml::datetime_delimiter_kind::upper_T,
+            // https://datatracker.ietf.org/doc/html/rfc3339#section-5.6
+            .has_seconds = true, // Mandated by TOML 1.0.0
+            .subsecond_precision = normalizeSubsecondPrecision(ldt.time),
+        };
+        return;
+    }
+
+    if (t.is_offset_datetime()) {
+        auto & odt = t.as_offset_datetime();
+        t.as_offset_datetime_fmt() = {
+            .delimiter = toml::datetime_delimiter_kind::upper_T,
+            // https://datatracker.ietf.org/doc/html/rfc3339#section-5.6
+            .has_seconds = true, // Mandated by TOML 1.0.0
+            .subsecond_precision = normalizeSubsecondPrecision(odt.time),
+        };
+        return;
+    }
+
+    if (t.is_local_time()) {
+        auto & lt = t.as_local_time();
+        t.as_local_time_fmt() = {
+            .has_seconds = true, // Mandated by TOML 1.0.0
+            .subsecond_precision = normalizeSubsecondPrecision(lt),
+        };
+        return;
+    }
+}
+
+#endif
+
+void prim_fromTOML(EvalState & state, Value ** args, Value & val)
+{
+    auto toml = state.forceStringNoCtx(
+        *args[0], noPos, "while evaluating the argument passed to builtins.fromTOML"
+    );
 
     std::istringstream tomlStream(std::string{toml});
 
-    std::function<void(Value &, toml::value)> visit;
+    auto visit = [&](this const auto & self, Value & v, toml::value t) -> void {
+        switch (t.type()) {
+        case toml::value_t::table: {
+            auto table = toml::get<toml::table>(t);
+            auto attrs = state.ctx.buildBindings(table.size());
 
-    visit = [&](Value & v, toml::value t) {
+            for (auto & elem : table) {
+                self(attrs.alloc(elem.first), elem.second);
+            }
 
-        switch(t.type())
-        {
-            case toml::value_t::table:
-                {
-                    auto table = toml::get<toml::table>(t);
-
-                    size_t size = 0;
-                    for (auto & i : table) { (void) i; size++; }
-
-                    auto attrs = state.ctx.buildBindings(size);
-
-                    for(auto & elem : table)
-                        visit(attrs.alloc(elem.first), elem.second);
-
-                    v.mkAttrs(attrs);
-                }
-                break;;
-            case toml::value_t::array:
-                {
-                    auto array = toml::get<std::vector<toml::value>>(t);
-
-                    size_t size = array.size();
-                    v = state.ctx.mem.newList(size);
-                    for (size_t i = 0; i < size; ++i)
-                        visit(*(v.listElems()[i] = state.ctx.mem.allocValue()), array[i]);
-                }
-                break;;
-            case toml::value_t::boolean:
-                v.mkBool(toml::get<bool>(t));
-                break;;
-            case toml::value_t::integer:
-                v.mkInt(toml::get<int64_t>(t));
-                break;;
-            case toml::value_t::floating:
-                v.mkFloat(toml::get<NixFloat>(t));
-                break;;
-            case toml::value_t::string:
-                v.mkString(toml::get<std::string>(t));
-                break;;
-            case toml::value_t::local_datetime:
-            case toml::value_t::offset_datetime:
-            case toml::value_t::local_date:
-            case toml::value_t::local_time:
-                {
-                    if (experimentalFeatureSettings.isEnabled(Xp::ParseTomlTimestamps)) {
-                        auto attrs = state.ctx.buildBindings(2);
-                        attrs.alloc("_type").mkString("timestamp");
-                        std::ostringstream s;
-                        s << t;
-                        attrs.alloc("value").mkString(s.str());
-                        v.mkAttrs(attrs);
-                    } else {
-                        // NOLINTNEXTLINE(lix-foreign-exceptions)
-                        throw std::runtime_error("Dates and times are not supported");
-                    }
-                }
-                break;;
-            case toml::value_t::empty:
-                v.mkNull();
-                break;;
+            v.mkAttrs(attrs);
+        } break;
+        case toml::value_t::array: {
+            auto array = toml::get<std::vector<toml::value>>(t);
 
+            size_t size = array.size();
+            v = state.ctx.mem.newList(size);
+            for (size_t i = 0; i < size; ++i) {
+                self(*(v.listElems()[i] = state.ctx.mem.allocValue()), array[i]);
+            }
+        } break;
+        case toml::value_t::boolean:
+            v.mkBool(toml::get<bool>(t));
+            break;
+        case toml::value_t::integer:
+            v.mkInt(toml::get<int64_t>(t));
+            break;
+        case toml::value_t::floating:
+            v.mkFloat(toml::get<NixFloat>(t));
+            break;
+        case toml::value_t::string:
+            v.mkString(toml::get<std::string>(t));
+            break;
+        case toml::value_t::local_datetime:
+        case toml::value_t::offset_datetime:
+        case toml::value_t::local_date:
+        case toml::value_t::local_time: {
+            if (experimentalFeatureSettings.isEnabled(Xp::ParseTomlTimestamps)) {
+#if HAVE_TOML11_4
+                normalizeDatetimeFormat(t);
+#endif
+                auto attrs = state.ctx.buildBindings(2);
+                attrs.alloc("_type").mkString("timestamp");
+                std::ostringstream s;
+                s << t;
+                attrs.alloc("value").mkString(s.str());
+                v.mkAttrs(attrs);
+            } else {
+                // NOLINTNEXTLINE(lix-foreign-exceptions)
+                throw std::runtime_error("Dates and times are not supported");
+            }
+        } break;
+        case toml::value_t::empty:
+            v.mkNull();
+            break;
         }
     };
 
     try {
-        visit(val, toml::parse(tomlStream, "fromTOML" /* the "filename" */));
+        visit(
+            val,
+            toml::parse(
+                tomlStream,
+                "fromTOML" /* the "filename" */
+#if HAVE_TOML11_4
+                ,
+                toml::spec::v(
+                    1, 0, 0
+                ) // Be explicit that we are parsing TOML 1.0.0 without extensions
+#endif
+            )
+        );
     } catch (std::exception & e) { // NOLINT(lix-foreign-exceptions) // TODO: toml::syntax_error
         state.ctx.errors.make<EvalError>("while parsing TOML: %s", e.what()).debugThrow();
     }

lix/libexpr/print-ambiguous.hh

@@ -1,7 +1,6 @@
 #pragma once
 ///@file
 
-#include "lix/libexpr/symbol-table.hh"
 #include "lix/libexpr/value.hh"
 
 namespace nix {

lix/libexpr/print.cc

@@ -287,7 +287,7 @@ private:
 
             AttrVec sorted;
             for (auto & i : *v.attrs)
-                sorted.emplace_back(state.ctx.symbols[i.name], i.value);
+                sorted.emplace_back(std::pair(state.ctx.symbols[i.name], i.value));
 
             if (options.maxAttrs == std::numeric_limits<size_t>::max())
                 std::sort(sorted.begin(), sorted.end());

lix/libexpr/settings/ignore-try.md

@@ -2,7 +2,7 @@
 name: ignore-try
 internalName: ignoreExceptionsDuringTry
 type: bool
-default: true
+default: false
 ---
 If set to true, ignore exceptions inside 'tryEval' calls when evaluating nix expressions in
 debug mode (using the --debugger flag). By default the debugger will pause on all exceptions.

lix/libexpr/symbol-table.hh

@@ -6,8 +6,6 @@
 #include "lix/libutil/types.hh"
 #include "lix/libutil/chunked-vector.hh"
 
-#include "lix/libexpr/value.hh"
-
 namespace nix {
 
 /**
@@ -18,7 +16,6 @@ namespace nix {
 class SymbolStr
 {
     friend class SymbolTable;
-    friend class InternedSymbol;
 
 private:
     const std::string * s;
@@ -44,59 +41,6 @@ public:
     friend std::ostream & operator <<(std::ostream & os, const SymbolStr & symbol);
 };
 
-class InternedSymbol
-{
-private:
-    /*
-     * The type that actually stores the string contained inside of the Value.
-     */
-    std::string contents;
-
-    /*
-     * A value containing a string that can be immediately passed to the evaluator.
-     */
-    Value underlyingValue;
-
-public:
-    explicit InternedSymbol(std::string_view s)
-        : contents(s)
-        , underlyingValue(NewValueAs::string, contents.c_str(), nullptr)
-    {
-    }
-
-    InternedSymbol(InternedSymbol &&) = default;
-    InternedSymbol & operator=(InternedSymbol &&) = default;
-
-    KJ_DISALLOW_COPY(InternedSymbol);
-
-    operator SymbolStr() const
-    {
-        return SymbolStr(contents);
-    }
-
-    bool operator==(std::string_view s2) const
-    {
-        return contents == s2;
-    }
-
-    operator const std::string &() const
-    {
-        return contents;
-    }
-
-    operator std::string_view() const
-    {
-        return contents;
-    }
-
-    const Value * toValuePtr() const
-    {
-        return &underlyingValue;
-    }
-
-    friend std::ostream & operator<<(std::ostream & os, const InternedSymbol & symbol);
-};
-
 /**
  * Symbols have the property that they can be compared efficiently
  * (using an equality test), because the symbol table stores only one
@@ -128,17 +72,13 @@ public:
 class SymbolTable
 {
 private:
-    /**
-     * Map from string view (backed by ChunkedVector) -> offset into the store.
-     * ChunkedVector references are never invalidated.
-     */
-    std::unordered_map<std::string_view, uint32_t> symbols;
-    ChunkedVector<InternedSymbol, 8192> store{16};
+    std::unordered_map<std::string_view, std::pair<const std::string *, uint32_t>> symbols;
+    ChunkedVector<std::string, 8192> store{16};
 
 public:
 
     /**
-     * Converts a string into a symbol.
+     * converts a string into a symbol.
      */
     Symbol create(std::string_view s)
     {
@@ -148,20 +88,18 @@ public:
         // on the original implementation using unordered_set
         // FIXME: make this thread-safe.
         auto it = symbols.find(s);
-        if (it != symbols.end()) {
-            return Symbol(it->second + 1);
-        }
+        if (it != symbols.end()) return Symbol(it->second.second + 1);
 
-        const auto & [rawSym, idx] = store.add(s);
-        symbols.emplace(rawSym, idx);
+        const auto & [rawSym, idx] = store.add(std::string(s));
+        symbols.emplace(rawSym, std::make_pair(&rawSym, idx));
         return Symbol(idx + 1);
     }
 
-    const InternedSymbol & operator[](Symbol s) const
+    SymbolStr operator[](Symbol s) const
     {
         if (s.id == 0 || s.id > store.size())
             abort();
-        return store[s.id - 1];
+        return SymbolStr(store[s.id - 1]);
     }
 
     size_t size() const

lix/libexpr/value-to-json.cc

@@ -46,7 +46,7 @@ JSON printValueAsJSON(EvalState & state, bool strict,
             break;
 
         case nAttrs: {
-            auto maybeString = state.tryAttrsToString(pos, v, context, StringCoercionMode::Strict, false);
+            auto maybeString = state.tryAttrsToString(pos, v, context, false, false);
             if (maybeString) {
                 out = *maybeString;
                 break;

lix/libexpr/value.hh

@@ -8,8 +8,8 @@
 #include <span>
 
 #include "lix/libexpr/gc-alloc.hh"
+#include "lix/libexpr/symbol-table.hh"
 #include "lix/libexpr/value/context.hh"
-#include "lix/libutil/logging.hh"
 #include "lix/libutil/source-path.hh"
 #include "lix/libexpr/print-options.hh"
 #include "lix/libutil/checked-arithmetic.hh"
@@ -59,32 +59,13 @@ typedef enum {
     nExternal
 } ValueType;
 
-
-/**
- * Modes of string coercion.
- *
- * Determines how permissive the coercion functions are when converting
- * values to strings.
- *
- * - Strict: Only allow coercion of values that are already strings,
- *   paths, or derivations.
- * - Interpolation: Additionally allow coercion of unambiguously printable values in a string, for
- *   now: integers. This mode is meant as a stopgap measure until we get better formatting tools.
- * - ToString: Additionally allow coercion of integers, booleans, null,
- *   and lists to strings.
- */
-enum class StringCoercionMode {
-    Strict,
-    Interpolation,
-    ToString,
-};
-
 class Bindings;
 struct Env;
 struct Expr;
 struct ExprLambda;
 struct ExprBlackHole;
 struct PrimOp;
+class Symbol;
 class PosIdx;
 struct Pos;
 class StorePath;
@@ -125,7 +106,7 @@ class ExternalValueBase
      * Coerce the value to a string. Defaults to uncoercable, i.e. throws an
      * error.
      */
-    virtual std::string coerceToString(EvalState & state, const PosIdx & pos, NixStringContext & context, StringCoercionMode mode, bool copyToStore) const;
+    virtual std::string coerceToString(EvalState & state, const PosIdx & pos, NixStringContext & context, bool copyMore, bool copyToStore) const;
 
     /**
      * Compare to another value of the same type. Defaults to uncomparable,
@@ -858,6 +839,8 @@ public:
 };
 
 using ValueVector = GcVector<Value *>;
+using ValueMap = GcMap<Symbol, Value *>;
+using ValueVectorMap = std::map<Symbol, ValueVector>;
 
 /**
  * A value allocated in traceable memory.

lix/libexpr/value/context.cc

@@ -2,10 +2,33 @@
 
 namespace nix {
 
-NixStringContextElem NixStringContextElem::parse(std::string_view s0)
+NixStringContextElem NixStringContextElem::parse(
+    std::string_view s0,
+    const ExperimentalFeatureSettings & xpSettings)
 {
     std::string_view s = s0;
 
+    std::function<SingleDerivedPath()> parseRest;
+    parseRest = [&]() -> SingleDerivedPath {
+        // Case on whether there is a '!'
+        size_t index = s.find("!");
+        if (index == std::string_view::npos) {
+            return SingleDerivedPath::Opaque {
+                .path = StorePath { s },
+            };
+        } else {
+            std::string output { s.substr(0, index) };
+            // Advance string to parse after the '!'
+            s = s.substr(index + 1);
+            auto drv = make_ref<SingleDerivedPath>(parseRest());
+            drvRequireExperiment(*drv, xpSettings);
+            return SingleDerivedPath::Built {
+                .drvPath = std::move(drv),
+                .output = std::move(output),
+            };
+        }
+    };
+
     if (s.size() == 0) {
         throw BadNixStringContextElem(s0,
             "String context element should never be an empty string");
@@ -17,20 +40,14 @@ NixStringContextElem NixStringContextElem::parse(std::string_view s0)
         s = s.substr(1);
 
         // Find *second* '!'
-        size_t index = s.find("!");
-        if (index == std::string_view::npos) {
+        if (s.find("!") == std::string_view::npos) {
             throw BadNixStringContextElem(s0,
                 "String content element beginning with '!' should have a second '!'");
         }
 
-        std::string output { s.substr(0, index) };
-        // Advance string to parse after the '!'
-        s = s.substr(index + 1);
-        auto drv = SingleDerivedPath::Opaque{StorePath{s}};
-        return SingleDerivedPath::Built{
-            .drvPath = std::move(drv),
-            .output = std::move(output),
-        };
+        return std::visit(
+            [&](auto x) -> NixStringContextElem { return std::move(x); },
+            parseRest());
     }
     case '=': {
         return NixStringContextElem::DrvDeep {
@@ -43,9 +60,9 @@ NixStringContextElem NixStringContextElem::parse(std::string_view s0)
             throw BadNixStringContextElem(s0,
                 "String content element not beginning with '!' should not have a second '!'");
         }
-        return SingleDerivedPath::Opaque{
-            .path = StorePath{s},
-        };
+        return std::visit(
+            [&](auto x) -> NixStringContextElem { return std::move(x); },
+            parseRest());
     }
     }
 }
@@ -54,15 +71,27 @@ std::string NixStringContextElem::to_string() const
 {
     std::string res;
 
+    std::function<void(const SingleDerivedPath &)> toStringRest;
+    toStringRest = [&](auto & p) {
+        std::visit(overloaded {
+            [&](const SingleDerivedPath::Opaque & o) {
+                res += o.path.to_string();
+            },
+            [&](const SingleDerivedPath::Built & o) {
+                res += o.output;
+                res += '!';
+                toStringRest(*o.drvPath);
+            },
+        }, p.raw());
+    };
+
     std::visit(overloaded {
         [&](const NixStringContextElem::Built & b) {
             res += '!';
-            res += b.output;
-            res += '!';
-            res += b.drvPath.path.to_string();
+            toStringRest(b);
         },
         [&](const NixStringContextElem::Opaque & o) {
-            res += o.path.to_string();
+            toStringRest(o);
         },
         [&](const NixStringContextElem::DrvDeep & d) {
             res += '=';

lix/libexpr/value/context.hh

@@ -69,8 +69,12 @@ struct NixStringContextElem {
      * - ‘<path>’
      * - ‘=<path>’
      * - ‘!<name>!<path>’
+     *
+     * @param xpSettings Stop-gap to avoid globals during unit tests.
      */
-    static NixStringContextElem parse(std::string_view s);
+    static NixStringContextElem parse(
+        std::string_view s,
+        const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
     std::string to_string() const;
 };
 

lix/libfetchers/builtin-fetchers.hh

@@ -10,7 +10,6 @@ std::unique_ptr<InputScheme> makePathInputScheme();
 std::unique_ptr<InputScheme> makeFileInputScheme();
 std::unique_ptr<InputScheme> makeTarballInputScheme();
 std::unique_ptr<InputScheme> makeGitInputScheme();
-std::unique_ptr<InputScheme> makeGitLockedInputScheme();
 std::unique_ptr<InputScheme> makeMercurialInputScheme();
 std::unique_ptr<InputScheme> makeGitHubInputScheme();
 std::unique_ptr<InputScheme> makeGitLabInputScheme();

lix/libfetchers/fetchers.cc

@@ -22,7 +22,6 @@ void initLibFetchers()
     registerInputScheme(makeTarballInputScheme());
     registerInputScheme(makeFileInputScheme());
     registerInputScheme(makeGitInputScheme());
-    registerInputScheme(makeGitLockedInputScheme());
     registerInputScheme(makeMercurialInputScheme());
     registerInputScheme(makeGitHubInputScheme());
     registerInputScheme(makeGitLabInputScheme());
@@ -305,24 +304,6 @@ std::optional<time_t> Input::getLastModified() const
     return maybeGetIntAttr(attrs, "lastModified");
 }
 
-std::optional<Input> InputScheme::inputFromAttrs(const Attrs & attrs) const
-{
-    if (maybeGetStrAttr(attrs, "type") != schemeType()) return {};
-
-    Attrs finalAttrs = preprocessAttrs(attrs);
-
-    for (auto & [name, value] : finalAttrs)
-        // All attrs need to accept a `type` and `narHash` key, the rest is scheme-specific
-        if (name != "type" && name != "narHash" && !allowedAttrs().contains(name))
-            throw UnsupportedAttributeError("unsupported input attribute '%s' for the '%s' scheme", name, schemeType());
-
-    Input input;
-    input.attrs = finalAttrs;
-    return input;
-}
-
-
-
 ParsedURL InputScheme::toURL(const Input & input) const
 {
     throw Error("don't know how to convert input '%s' to a URL", attrsToJSON(input.attrs));

lix/libfetchers/fetchers.hh

@@ -2,7 +2,6 @@
 ///@file
 
 #include "lix/libstore/content-address.hh"
-#include "lix/libutil/error.hh"
 #include "lix/libutil/result.hh"
 #include "lix/libutil/types.hh"
 #include "lix/libutil/hash.hh"
@@ -20,8 +19,6 @@ namespace nix { class Store; }
 
 namespace nix::fetchers {
 
-MakeError(UnsupportedAttributeError, Error);
-
 struct Tree
 {
     Path actualPath;
@@ -128,6 +125,7 @@ public:
     std::optional<time_t> getLastModified() const;
 };
 
+
 /**
  * The InputScheme represents a type of fetcher.  Each fetcher
  * registers with nix at startup time.  When processing an input for a
@@ -144,12 +142,7 @@ struct InputScheme
 
     virtual std::optional<Input> inputFromURL(const ParsedURL & url, bool requireTree) const = 0;
 
-    virtual Attrs preprocessAttrs(const Attrs & attrs) const = 0;
-
-    // The scheme type, which is used to match attributes to a specific scheme
-    virtual std::string schemeType() const = 0;
-
-    virtual std::optional<Input> inputFromAttrs(const Attrs & attrs) const;
+    virtual std::optional<Input> inputFromAttrs(const Attrs & attrs) const = 0;
 
     virtual ParsedURL toURL(const Input & input) const;
 
@@ -181,9 +174,6 @@ struct InputScheme
     virtual bool isLockedByRev() const { return true; }
 
 protected:
-    // The set of allowed attributes for this specific fetcher
-    virtual const std::set<std::string> & allowedAttrs() const = 0;
-
     void emplaceURLQueryIntoAttrs(
         const ParsedURL & parsedURL,
         Attrs & attrs,

lix/libfetchers/git.cc

@@ -165,19 +165,11 @@ WorkdirInfo getWorkdirInfo(const Input & input, const Path & workdir)
 
     /* Check whether HEAD points to something that looks like a commit,
        since that is the refrence we want to use later on. */
-    auto result = runProgram(RunOptions{
+    auto result = runProgram(RunOptions {
         .program = "git",
-        .args =
-            {"-C",
-             workdir,
-             "--git-dir",
-             gitDir,
-             "rev-parse",
-             "--verify",
-             "--no-revs",
-             "HEAD^{commit}"},
+        .args = { "-C", workdir, "--git-dir", gitDir, "rev-parse", "--verify", "--no-revs", "HEAD^{commit}" },
         .environment = env,
-        .redirections = {{.dup = STDERR_FILENO, .from = STDOUT_FILENO}},
+        .redirections = {{.from = STDERR_FILENO, .to = STDOUT_FILENO}},
     });
     auto exitCode = WEXITSTATUS(result.first);
     auto errorMessage = result.second;
@@ -297,28 +289,8 @@ static std::optional<Path> resolveRefToCachePath(
     return std::nullopt;
 }
 
-static const std::set<std::string> allowedGitAttrs = {
-    "allRefs",
-    "dirtyRev",
-    "dirtyShortRev",
-    "lastModified",
-    "name",
-    "ref",
-    "rev",
-    "revCount",
-    "shallow",
-    "submodules",
-    "url",
-};
-
 struct GitInputScheme : InputScheme
 {
-    std::string schemeType() const override { return "git"; }
-
-    const std::set<std::string> & allowedAttrs() const override {
-        return allowedGitAttrs;
-    }
-
     std::optional<Input> inputFromURL(const ParsedURL & url, bool requireTree) const override
     {
         if (url.scheme != "git" &&
@@ -345,7 +317,14 @@ struct GitInputScheme : InputScheme
         return inputFromAttrs(attrs);
     }
 
-    Attrs preprocessAttrs(const Attrs & attrs) const override {
+    std::optional<Input> inputFromAttrs(const Attrs & attrs) const override
+    {
+        if (maybeGetStrAttr(attrs, "type") != "git") return {};
+
+        for (auto & [name, value] : attrs)
+            if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow" && name != "submodules" && name != "lastModified" && name != "revCount" && name != "narHash" && name != "allRefs" && name != "name" && name != "dirtyRev" && name != "dirtyShortRev")
+                throw Error("unsupported Git input attribute '%s'", name);
+
         parseURL(getStrAttr(attrs, "url"));
         maybeGetBoolAttr(attrs, "shallow");
         maybeGetBoolAttr(attrs, "submodules");
@@ -356,7 +335,9 @@ struct GitInputScheme : InputScheme
                 throw BadURL("invalid Git branch/tag name '%s'", *ref);
         }
 
-        return attrs;
+        Input input;
+        input.attrs = attrs;
+        return input;
     }
 
     ParsedURL toURL(const Input & input) const override
@@ -718,12 +699,10 @@ struct GitInputScheme : InputScheme
         AutoDelete delTmpDir(tmpDir, true);
         PathFilter filter = defaultPathFilter;
 
-        auto result = runProgram(RunOptions{
+        auto result = runProgram(RunOptions {
             .program = "git",
-            .args =
-                {"-C", repoDir, "--git-dir", gitDir, "cat-file", "commit", input.getRev()->gitRev()
-                },
-            .redirections = {{.dup = STDERR_FILENO, .from = STDOUT_FILENO}},
+            .args = { "-C", repoDir, "--git-dir", gitDir, "cat-file", "commit", input.getRev()->gitRev() },
+            .redirections = {{.from = STDERR_FILENO, .to = STDOUT_FILENO}},
         });
         if (WEXITSTATUS(result.first) == 128
             && result.second.find("bad file") != std::string::npos)
@@ -834,22 +813,4 @@ std::unique_ptr<InputScheme> makeGitInputScheme()
     return std::make_unique<GitInputScheme>();
 }
 
-struct GitLockedInputScheme : GitInputScheme {
-
-    std::string schemeType() const override {
-        using namespace std::literals::string_literals;
-        return "\0git-locked"s;
-    }
-
-    bool hasAllInfo(const Input & input) const override {
-        return true;
-    }
-
-};
-
-std::unique_ptr<InputScheme> makeGitLockedInputScheme()
-{
-    return std::make_unique<GitLockedInputScheme>();
-}
-
 }

lix/libfetchers/github.cc

@@ -1,5 +1,7 @@
 #include "lix/libfetchers/attrs.hh"
 #include "lix/libstore/filetransfer.hh"
+#include "lix/libfetchers/cache.hh"
+#include "lix/libstore/globals.hh"
 #include "lix/libfetchers/builtin-fetchers.hh"
 #include "lix/libstore/store-api.hh"
 #include "lix/libutil/async.hh"
@@ -22,30 +24,19 @@ struct DownloadUrl
     Headers headers;
 };
 
-static const std::set<std::string> allowedGitArchiveAttrs = {
-    "host",
-    "lastModified",
-    "owner",
-    "ref",
-    "repo",
-    "rev",
-};
-
 // A github, gitlab, or sourcehut host
 const static std::string hostRegexS = "[a-zA-Z0-9.-]*"; // FIXME: check
 std::regex hostRegex = regex::parse(hostRegexS, std::regex::ECMAScript);
 
 struct GitArchiveInputScheme : InputScheme
 {
-    const std::set<std::string> & allowedAttrs() const override {
-        return allowedGitArchiveAttrs;
-    }
+    virtual std::string type() const = 0;
 
     virtual std::optional<std::pair<std::string, std::string>> accessHeaderFromToken(const std::string & token) const = 0;
 
     std::optional<Input> inputFromURL(const ParsedURL & url, bool requireTree) const override
     {
-        if (url.scheme != schemeType()) return {};
+        if (url.scheme != type()) return {};
 
         auto path = tokenizeString<std::vector<std::string>>(url.path, "/");
 
@@ -72,7 +63,7 @@ struct GitArchiveInputScheme : InputScheme
             throw BadURL("URL '%s' is invalid", url.url);
 
         Attrs attrs;
-        attrs.emplace("type", schemeType());
+        attrs.emplace("type", type());
         attrs.emplace("owner", path[0]);
         attrs.emplace("repo", path[1]);
 
@@ -102,18 +93,20 @@ struct GitArchiveInputScheme : InputScheme
         return inputFromAttrs(attrs);
     }
 
-    Attrs preprocessAttrs(const Attrs & attrs) const override
+    std::optional<Input> inputFromAttrs(const Attrs & attrs) const override
     {
         // Attributes can contain refOrRev and it needs to be figured out
         // which one it is (see inputFromURL for when that may happen).
         // The correct one (ref or rev) will be written into finalAttrs and
         // it needs to be mutable for that.
         Attrs finalAttrs(attrs);
+        auto type_ = maybeGetStrAttr(finalAttrs, "type");
+        if (type_ != type()) return {};
 
         auto owner = getStrAttr(finalAttrs, "owner");
         auto repo = getStrAttr(finalAttrs, "repo");
 
-        auto url = fmt("%s:%s/%s", schemeType(), owner, repo);
+        auto url = fmt("%s:%s/%s", *type_, owner, repo);
         if (auto host = maybeGetStrAttr(finalAttrs, "host")) {
             if (!std::regex_match(*host, hostRegex)) {
                 throw BadURL("URL '%s' contains an invalid instance host", url);
@@ -139,7 +132,15 @@ struct GitArchiveInputScheme : InputScheme
             }
         }
 
-        return finalAttrs;
+        for (auto & [name, value] : finalAttrs) {
+            if (name != "type" && name != "owner" && name != "repo" && name != "ref" && name != "rev" && name != "narHash" && name != "lastModified" && name != "host") {
+                throw Error("unsupported input attribute '%s'", name);
+            }
+        }
+
+        Input input;
+        input.attrs = finalAttrs;
+        return input;
     }
 
     ParsedURL toURL(const Input & input) const override
@@ -153,7 +154,7 @@ struct GitArchiveInputScheme : InputScheme
         if (ref) path += "/" + *ref;
         if (rev) path += "/" + rev->to_string(Base::Base16, false);
         return ParsedURL {
-            .scheme = schemeType(),
+            .scheme = type(),
             .path = path,
         };
     }
@@ -238,7 +239,7 @@ struct GitArchiveInputScheme : InputScheme
 
 struct GitHubInputScheme : GitArchiveInputScheme
 {
-    std::string schemeType() const override { return "github"; }
+    std::string type() const override { return "github"; }
 
     std::optional<std::pair<std::string, std::string>> accessHeaderFromToken(const std::string & token) const override
     {
@@ -328,7 +329,7 @@ struct GitHubInputScheme : GitArchiveInputScheme
 
 struct GitLabInputScheme : GitArchiveInputScheme
 {
-    std::string schemeType() const override { return "gitlab"; }
+    std::string type() const override { return "gitlab"; }
 
     std::optional<std::pair<std::string, std::string>> accessHeaderFromToken(const std::string & token) const override
     {
@@ -404,7 +405,7 @@ struct GitLabInputScheme : GitArchiveInputScheme
 
 struct SourceHutInputScheme : GitArchiveInputScheme
 {
-    std::string schemeType() const override { return "sourcehut"; }
+    std::string type() const override { return "sourcehut"; }
 
     std::optional<std::pair<std::string, std::string>> accessHeaderFromToken(const std::string & token) const override
     {

lix/libfetchers/indirect.cc

@@ -8,20 +8,8 @@ namespace nix::fetchers {
 
 std::regex flakeRegex = regex::parse("[a-zA-Z][a-zA-Z0-9_-]*", std::regex::ECMAScript);
 
-static const std::set<std::string> allowedIndirectAttrs = {
-    "id",
-    "ref",
-    "rev",
-};
-
 struct IndirectInputScheme : InputScheme
 {
-    std::string schemeType() const override { return "indirect"; }
-
-    const std::set<std::string> & allowedAttrs() const override {
-        return allowedIndirectAttrs;
-    }
-
     std::optional<Input> inputFromURL(const ParsedURL & url, bool requireTree) const override
     {
         if (url.scheme != "flake") return {};
@@ -59,7 +47,14 @@ struct IndirectInputScheme : InputScheme
         return inputFromAttrs(attrs);
     }
 
-    Attrs preprocessAttrs(const Attrs & attrs) const override {
+    std::optional<Input> inputFromAttrs(const Attrs & attrs) const override
+    {
+        if (maybeGetStrAttr(attrs, "type") != "indirect") return {};
+
+        for (auto & [name, value] : attrs)
+            if (name != "type" && name != "id" && name != "ref" && name != "rev" && name != "narHash")
+                throw Error("unsupported indirect input attribute '%s'", name);
+
         auto id = getStrAttr(attrs, "id");
         if (!std::regex_match(id, flakeRegex))
             throw BadURL("'%s' is not a valid flake ID", id);
@@ -76,17 +71,9 @@ struct IndirectInputScheme : InputScheme
             }
         }
 
-        return attrs;
-    }
-
-    std::optional<Input> inputFromAttrs(const Attrs & attrs) const override
-    {
-        std::optional<Input> input = InputScheme::inputFromAttrs(attrs);
-
-        if (input) {
-            input->direct = false;
-        }
-
+        Input input;
+        input.direct = false;
+        input.attrs = attrs;
         return input;
     }
 

lix/libfetchers/mercurial.cc

@@ -44,22 +44,8 @@ static std::string runHg(const Strings & args)
     return res.second;
 }
 
-static const std::set<std::string> allowedMercurialAttrs = {
-    "name",
-    "ref",
-    "rev",
-    "revCount",
-    "url",
-};
-
 struct MercurialInputScheme : InputScheme
 {
-    std::string schemeType() const override { return "hg"; }
-
-    const std::set<std::string> & allowedAttrs() const override {
-        return allowedMercurialAttrs;
-    }
-
     std::optional<Input> inputFromURL(const ParsedURL & url, bool requireTree) const override
     {
         if (url.scheme != "hg+http" &&
@@ -81,8 +67,14 @@ struct MercurialInputScheme : InputScheme
         return inputFromAttrs(attrs);
     }
 
-    Attrs preprocessAttrs(const Attrs & attrs) const override
+    std::optional<Input> inputFromAttrs(const Attrs & attrs) const override
     {
+        if (maybeGetStrAttr(attrs, "type") != "hg") return {};
+
+        for (auto & [name, value] : attrs)
+            if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "revCount" && name != "narHash" && name != "name")
+                throw Error("unsupported Mercurial input attribute '%s'", name);
+
         parseURL(getStrAttr(attrs, "url"));
 
         if (auto ref = maybeGetStrAttr(attrs, "ref")) {
@@ -90,7 +82,9 @@ struct MercurialInputScheme : InputScheme
                 throw BadURL("invalid Mercurial branch/tag name '%s'", *ref);
         }
 
-        return attrs;
+        Input input;
+        input.attrs = attrs;
+        return input;
     }
 
     ParsedURL toURL(const Input & input) const override

lix/libfetchers/path.cc

@@ -6,25 +6,8 @@
 
 namespace nix::fetchers {
 
-/* Allow the user to pass in "fake" tree info
-   attributes. This is useful for making a pinned tree
-   work the same as the repository from which is exported
-   (e.g. path:/nix/store/...-source?lastModified=1585388205&rev=b0c285...). */
-static const std::set<std::string> allowedPathAttrs = {
-    "lastModified",
-    "path",
-    "rev",
-    "revCount",
-};
-
 struct PathInputScheme : InputScheme
 {
-    std::string schemeType() const override { return "path"; }
-
-    const std::set<std::string> & allowedAttrs() const override {
-        return allowedPathAttrs;
-    }
-
     std::optional<Input> inputFromURL(const ParsedURL & url, bool requireTree) const override
     {
         if (url.scheme != "path") return {};
@@ -51,11 +34,26 @@ struct PathInputScheme : InputScheme
         return input;
     }
 
-    Attrs preprocessAttrs(const Attrs & attrs) const override
+    std::optional<Input> inputFromAttrs(const Attrs & attrs) const override
     {
+        if (maybeGetStrAttr(attrs, "type") != "path") return {};
+
         getStrAttr(attrs, "path");
 
-        return attrs;
+        for (auto & [name, value] : attrs)
+            /* Allow the user to pass in "fake" tree info
+               attributes. This is useful for making a pinned tree
+               work the same as the repository from which is exported
+               (e.g. path:/nix/store/...-source?lastModified=1585388205&rev=b0c285...). */
+            if (name == "type" || name == "rev" || name == "revCount" || name == "lastModified" || name == "narHash" || name == "path")
+                // checked in Input::fromAttrs
+                ;
+            else
+                throw Error("unsupported path input attribute '%s'", name);
+
+        Input input;
+        input.attrs = attrs;
+        return input;
     }
 
     bool isLockedByRev() const override { return false; }

lix/libfetchers/tarball.cc

@@ -85,9 +85,9 @@ try {
     FileTransferResult res;
     std::string data;
     try {
-        auto [meta, content] = TRY_AWAIT(getFileTransfer()->download(url, headers));
+        auto [meta, content] = getFileTransfer()->download(url, headers);
         res = std::move(meta);
-        data = TRY_AWAIT(content->drain());
+        data = content->drain();
     } catch (FileTransferError & e) {
         if (cached) {
             warn("%s; using cached version", e.msg());
@@ -230,25 +230,12 @@ try {
     co_return result::current_exception();
 }
 
-// FIXME: some of these only apply to TarballInputScheme.
-static const std::set<std::string> allowedCurlAttrs = {
-    "lastModified",
-    "name",
-    "rev",
-    "revCount",
-    "unpack",
-    "url",
-};
-
 // An input scheme corresponding to a curl-downloadable resource.
 struct CurlInputScheme : InputScheme
 {
+    virtual const std::string inputType() const = 0;
     const std::set<std::string> transportUrlSchemes = {"file", "http", "https"};
 
-    const std::set<std::string> & allowedAttrs() const override {
-        return allowedCurlAttrs;
-    }
-
     bool hasTarballExtension(std::string_view path) const
     {
         return path.ends_with(".zip") || path.ends_with(".tar")
@@ -267,7 +254,7 @@ struct CurlInputScheme : InputScheme
         auto url = _url;
 
         Attrs attrs;
-        attrs.emplace("type", schemeType());
+        attrs.emplace("type", inputType());
 
         url.scheme = parseUrlScheme(url.scheme).transport;
 
@@ -277,16 +264,24 @@ struct CurlInputScheme : InputScheme
         return inputFromAttrs(attrs);
     }
 
-    Attrs preprocessAttrs(const Attrs & attrs) const override
+    std::optional<Input> inputFromAttrs(const Attrs & attrs) const override
     {
+        auto type = maybeGetStrAttr(attrs, "type");
+        if (type != inputType()) return {};
+
+        // FIXME: some of these only apply to TarballInputScheme.
+        std::set<std::string> allowedNames = {"type", "url", "narHash", "name", "unpack", "rev", "revCount", "lastModified"};
         for (auto & [name, value] : attrs)
-            if (name != "type" && name != "narHash" && !allowedAttrs().contains(name))
-                throw UnsupportedAttributeError("unsupported tarball input attribute '%s'. If you wanted to fetch a tarball with a query parameter, please use '{ type = \"tarball\"; url = \"...\"; }'", name);
+            if (!allowedNames.count(name))
+                throw Error("unsupported %s input attribute '%s'. If you wanted to fetch a tarball with a query parameter, please use '{ type = \"tarball\"; url = \"...\"; }'", *type, name);
 
-        return attrs;
+        Input input;
+        input.attrs = attrs;
+
+        //input.locked = (bool) maybeGetStrAttr(input.attrs, "hash");
+        return input;
     }
 
-
     ParsedURL toURL(const Input & input) const override
     {
         auto url = parseURL(getStrAttr(input.attrs, "url"));
@@ -308,14 +303,14 @@ struct CurlInputScheme : InputScheme
 
 struct FileInputScheme : CurlInputScheme
 {
-    std::string schemeType() const override { return "file"; }
+    const std::string inputType() const override { return "file"; }
 
     bool isValidURL(const ParsedURL & url, bool requireTree) const override
     {
         auto parsedUrlScheme = parseUrlScheme(url.scheme);
         return transportUrlSchemes.count(std::string(parsedUrlScheme.transport))
             && (parsedUrlScheme.application
-                ? parsedUrlScheme.application.value() == schemeType()
+                ? parsedUrlScheme.application.value() == inputType()
                 : (!requireTree && !hasTarballExtension(url.path)));
     }
 
@@ -333,7 +328,7 @@ struct FileInputScheme : CurlInputScheme
 
 struct TarballInputScheme : CurlInputScheme
 {
-    std::string schemeType() const override { return "tarball"; }
+    const std::string inputType() const override { return "tarball"; }
 
     bool isValidURL(const ParsedURL & url, bool requireTree) const override
     {
@@ -341,7 +336,7 @@ struct TarballInputScheme : CurlInputScheme
 
         return transportUrlSchemes.count(std::string(parsedUrlScheme.transport))
             && (parsedUrlScheme.application
-                ? parsedUrlScheme.application.value() == schemeType()
+                ? parsedUrlScheme.application.value() == inputType()
                 : (requireTree || hasTarballExtension(url.path)));
     }
 

lix/libmain/crash-handler.cc

@@ -1,9 +1,11 @@
 #include "lix/libmain/crash-handler.hh"
 #include "lix/libutil/error.hh"
+#include "lix/libutil/fmt.hh"
 #include "lix/libutil/logging.hh"
 
 #include <boost/core/demangle.hpp>
 #include <exception>
+#include <source_location>
 
 namespace nix {
 
@@ -21,14 +23,38 @@ void onTerminate()
         } else {
             logFatal("std::terminate() called without exception");
         }
+    } catch (const ForeignException & ex) {
+        asyncTrace = ex.asyncTrace();
+        logFatal(fmt("Exception: %s: %s", boost::core::demangle(ex.innerType.name()), ex.what()));
     } catch (const BaseException & ex) {
-        logException("Exception", ex);
+        asyncTrace = ex.asyncTrace();
+        logFatal(fmt("Exception: %s: %s", boost::core::demangle(typeid(ex).name()), ex.what()));
     } catch (const std::exception & ex) { // NOLINT(lix-foreign-exceptions)
-        logException("Exception", ex);
+        logFatal(fmt("Exception: %s: %s", boost::core::demangle(typeid(ex).name()), ex.what()));
     } catch (...) {
         logFatal("Unknown exception! Spooky.");
     }
 
+    logFatal("Stack trace:");
+    logFatal(getStackTrace());
+
+    if (asyncTrace && !asyncTrace->empty()) {
+        logFatal("Async task trace (probably incomplete):");
+        for (auto [i, frame] : enumerate(*asyncTrace)) {
+            logFatal(
+                fmt("#%i: %s (%s:%i:%i)",
+                    i,
+                    frame.location.function_name(),
+                    frame.location.file_name(),
+                    frame.location.line(),
+                    frame.location.column())
+            );
+            if (frame.description) {
+                logFatal(fmt("\t%s", *frame.description));
+            }
+        }
+    }
+
     std::abort();
 }
 }

lix/libmain/shared.cc

@@ -130,6 +130,12 @@ static void sigHandler(int signo) { }
 
 void initNix()
 {
+    // kj needs a signal for internal use. no system lix habitually runs on causes
+    // kj to actually *use* this signal, but better safe than sorry—and since some
+    // OSes (*cough* macos) don't support realtime signals we must use SIGUSR2 for
+    // this, thus "consuming" both USR signals. at some point we will change this.
+    kj::UnixEventPort::setReservedSignal(SIGUSR2);
+
     registerCrashHandler();
 
     /* Turn on buffering for cerr. */
@@ -149,11 +155,9 @@ void initNix()
     if (sigaction(SIGCHLD, &act, 0))
         throw SysError("resetting SIGCHLD");
 
-    /* Install a dummy INTERRUPT_NOTIFY_SIGNAL handler for use with pthread_kill(). */
+    /* Install a dummy SIGUSR1 handler for use with pthread_kill(). */
     act.sa_handler = sigHandler;
-    if (sigaction(INTERRUPT_NOTIFY_SIGNAL, &act, 0)) {
-        throw SysError("handling interrupt notify signal %i", INTERRUPT_NOTIFY_SIGNAL);
-    }
+    if (sigaction(SIGUSR1, &act, 0)) throw SysError("handling SIGUSR1");
 
 #if __APPLE__
     /* HACK: on darwin, we need can’t use sigprocmask with SIGWINCH.
@@ -187,6 +191,7 @@ void initNix()
     umask(0022);
 }
 
+
 LegacyArgs::LegacyArgs(AsyncIoRoot & aio, const std::string & programName,
     std::function<bool(Strings::iterator & arg, const Strings::iterator & end)> parseArg)
     : MixCommonArgs(programName), aio_(aio), parseArg(parseArg)

lix/libstore/binary-cache-store.cc

@@ -1,7 +1,6 @@
 #include "lix/libutil/archive.hh"
 #include "lix/libstore/binary-cache-store.hh"
 #include "lix/libutil/async-io.hh"
-#include "lix/libutil/async.hh"
 #include "lix/libutil/compression.hh"
 #include "lix/libstore/derivations.hh"
 #include "lix/libstore/fs-accessor.hh"
@@ -19,7 +18,6 @@
 #include "lix/libutil/strings.hh"
 
 #include <chrono>
-#include <functional>
 #include <regex>
 #include <fstream>
 #include <sstream>
@@ -40,11 +38,9 @@ kj::Promise<Result<void>> BinaryCacheStore::init()
 try {
     std::string cacheInfoFile = "nix-cache-info";
 
-    auto cacheInfo = TRY_AWAIT(getFileContents(cacheInfoFile));
+    auto cacheInfo = getFileContents(cacheInfoFile);
     if (!cacheInfo) {
-        TRY_AWAIT(upsertFile(
-            cacheInfoFile, "StoreDir: " + config().storeDir + "\n", "text/x-nix-cache-info"
-        ));
+        upsertFile(cacheInfoFile, "StoreDir: " + config().storeDir + "\n", "text/x-nix-cache-info");
     } else {
         for (auto & line : tokenizeString<Strings>(*cacheInfo, "\n")) {
             size_t colon= line.find(':');
@@ -67,26 +63,20 @@ try {
     co_return result::current_exception();
 }
 
-kj::Promise<Result<void>> BinaryCacheStore::upsertFile(
-    const std::string & path, std::string && data, const std::string & mimeType
-)
-try {
-    TRY_AWAIT(upsertFile(path, std::make_shared<std::stringstream>(std::move(data)), mimeType));
-    co_return result::success();
-} catch (...) {
-    co_return result::current_exception();
+void BinaryCacheStore::upsertFile(const std::string & path,
+    std::string && data,
+    const std::string & mimeType)
+{
+    upsertFile(path, std::make_shared<std::stringstream>(std::move(data)), mimeType);
 }
 
-kj::Promise<Result<std::optional<std::string>>>
-BinaryCacheStore::getFileContents(const std::string & path)
-try {
+std::optional<std::string> BinaryCacheStore::getFileContents(const std::string & path)
+{
     try {
-        co_return TRY_AWAIT(TRY_AWAIT(getFile(path))->drain());
+        return getFile(path)->drain();
     } catch (NoSuchBinaryCacheFile &) {
-        co_return std::nullopt;
+        return std::nullopt;
     }
-} catch (...) {
-    co_return result::current_exception();
 }
 
 std::string BinaryCacheStore::narInfoFileFor(const StorePath & storePath)
@@ -98,7 +88,7 @@ kj::Promise<Result<void>> BinaryCacheStore::writeNarInfo(ref<NarInfo> narInfo)
 try {
     auto narInfoFile = narInfoFileFor(narInfo->path);
 
-    TRY_AWAIT(upsertFile(narInfoFile, narInfo->to_string(*this), "text/x-nix-narinfo"));
+    upsertFile(narInfoFile, narInfo->to_string(*this), "text/x-nix-narinfo");
 
     {
         auto state_(co_await state.lock());
@@ -188,21 +178,7 @@ try {
             {"root", listNar(narIndex)},
         };
 
-        try {
-            TRY_AWAIT(
-                upsertFile(std::string(info.path.hashPart()) + ".ls", j.dump(), "application/json")
-            );
-        } catch (ForeignException & exc) {
-            if (exc.is<JSON::exception>()) {
-                warn(
-                    "Skipping NAR listing for path '%1%' due to serialization failure: %2%",
-                    printStorePath(narInfo->path),
-                    exc.what()
-                );
-            } else {
-                throw exc;
-            }
-        }
+        upsertFile(std::string(info.path.hashPart()) + ".ls", j.dump(), "application/json");
     }
 
     /* Optionally maintain an index of DWARF debug info files
@@ -214,6 +190,7 @@ try {
 
         auto * buildIdDir = std::get_if<nar_index::Directory>(&narIndex);
         for (auto subdir : { "lib", "debug", ".build-id" }) {
+            if (!buildIdDir) break;
             // get returns nullptr subdir does not exist, and std::get_if propagates it.
             buildIdDir = std::get_if<nar_index::Directory>(get(buildIdDir->contents, subdir));
         }
@@ -222,23 +199,20 @@ try {
 
             ThreadPool threadPool("write debuginfo pool", 25);
 
-            auto doFile = [&](AsyncIoRoot & aio,
-                              std::string member,
-                              std::string key,
-                              std::string target) {
+            auto doFile = [&](std::string member, std::string key, std::string target) {
+                checkInterrupt();
+
                 JSON json;
                 json["archive"] = target;
                 json["member"] = member;
 
                 // FIXME: or should we overwrite? The previous link may point
                 // to a GC'ed file, so overwriting might be useful...
-                if (aio.blockOn(fileExists(key))) {
-                    return;
-                }
+                if (fileExists(key)) return;
 
                 printMsg(lvlTalkative, "creating debuginfo link from '%s' to '%s'", key, target);
 
-                aio.blockOn(upsertFile(key, json.dump(), "application/json"));
+                upsertFile(key, json.dump(), "application/json");
             };
 
             std::regex regex1 = regex::parse("^[0-9a-f]{2}$");
@@ -262,9 +236,7 @@ try {
                     std::string key = "debuginfo/" + buildId;
                     std::string target = "../" + narInfo->url;
 
-                    threadPool.enqueueWithAio(std::bind(
-                        doFile, std::placeholders::_1, std::string(debugPath, 1), key, target
-                    ));
+                    threadPool.enqueue(std::bind(doFile, std::string(debugPath, 1), key, target));
                 }
             }
 
@@ -273,16 +245,13 @@ try {
     }
 
     /* Atomically write the NAR file. */
-    if (repair || !TRY_AWAIT(fileExists(narInfo->url))) {
+    if (repair || !fileExists(narInfo->url)) {
         stats.narWrite++;
-        TRY_AWAIT(upsertFile(
-            narInfo->url,
+        upsertFile(narInfo->url,
             std::make_shared<std::fstream>(fnTemp, std::ios_base::in | std::ios_base::binary),
-            "application/x-nix-nar"
-        ));
-    } else {
+            "application/x-nix-nar");
+    } else
         stats.narWriteAverted++;
-    }
 
     stats.narWriteBytes += info.narSize;
     stats.narWriteCompressedBytes += fileSize;
@@ -363,7 +332,7 @@ try {
     // FIXME: this only checks whether a .narinfo with a matching hash
     // part exists. So ‘f4kb...-foo’ matches ‘f4kb...-bar’, even
     // though they shouldn't. Not easily fixed.
-    co_return TRY_AWAIT(fileExists(narInfoFileFor(storePath)));
+    co_return fileExists(narInfoFileFor(storePath));
 } catch (...) {
     co_return result::current_exception();
 }
@@ -382,47 +351,34 @@ try {
     co_return result::current_exception();
 }
 
-kj::Promise<Result<box_ptr<AsyncInputStream>>>
-BinaryCacheStore::narFromPath(const StorePath & storePath)
+kj::Promise<Result<box_ptr<Source>>> BinaryCacheStore::narFromPath(const StorePath & storePath)
 try {
-    struct NarFromPath : AsyncInputStream
-    {
-        Stats<std::atomic> & stats;
-        box_ptr<AsyncInputStream> decompressed;
-        uint64_t total;
-
-        NarFromPath(
-            Stats<std::atomic> & stats, const std::string & method, box_ptr<AsyncInputStream> file
-        )
-            : stats(stats)
-            , decompressed(makeDecompressionStream(method, std::move(file)))
-        {
-        }
-
-        kj::Promise<Result<size_t>> read(void * buffer, size_t size) override
-        {
-            return decompressed->read(buffer, size).then([&](auto r) {
-                if (r.has_value()) {
-                    if (r.value() > 0) {
-                        total += r.value();
-                    } else {
-                        stats.narRead++;
-                        // stats.narReadCompressedBytes += nar->size(); // FIXME
-                        stats.narReadBytes += total;
-                    }
-                }
-                return r;
-            });
-        }
-    };
-
     auto info_ = TRY_AWAIT(queryPathInfo(storePath)).try_cast<const NarInfo>();
     assert(info_ && "binary cache queryPathInfo didn't return a NarInfo");
     auto & info = *info_;
 
     try {
-        auto file = TRY_AWAIT(getFile(info->url));
-        co_return make_box_ptr<NarFromPath>(stats, info->compression, std::move(file));
+        auto file = getFile(info->url);
+        co_return make_box_ptr<GeneratorSource>(
+            [](auto info, auto file, auto & stats) -> WireFormatGenerator {
+                constexpr size_t buflen = 65536;
+                auto buf = std::make_unique<char[]>(buflen);
+                size_t total = 0;
+                auto decompressor = makeDecompressionSource(info->compression, *file);
+                try {
+                    while (true) {
+                        const auto len = decompressor->read(buf.get(), buflen);
+                        co_yield std::span{buf.get(), len};
+                        total += len;
+                    }
+                } catch (EndOfFile &) {
+                }
+
+                stats.narRead++;
+                // stats.narReadCompressedBytes += nar->size(); // FIXME
+                stats.narReadBytes += total;
+            }(std::move(info), std::move(file), stats)
+        );
     } catch (NoSuchBinaryCacheFile & e) {
         throw SubstituteGone(std::move(e.info()));
     }
@@ -441,7 +397,7 @@ try {
 
     auto narInfoFile = narInfoFileFor(storePath);
 
-    auto data = TRY_AWAIT(getFileContents(narInfoFile));
+    auto data = getFileContents(narInfoFile);
 
     if (!data) co_return result::success(nullptr);
 
@@ -550,6 +506,32 @@ try {
     co_return result::current_exception();
 }
 
+kj::Promise<Result<std::shared_ptr<const Realisation>>>
+BinaryCacheStore::queryRealisationUncached(const DrvOutput & id)
+try {
+    auto outputInfoFilePath = realisationsPrefix + "/" + id.to_string() + ".doi";
+
+    auto data = getFileContents(outputInfoFilePath);
+    if (!data) co_return result::success(nullptr);
+
+    auto realisation = Realisation::fromJSON(
+        json::parse(*data), outputInfoFilePath);
+    co_return std::make_shared<const Realisation>(realisation);
+} catch (...) {
+    co_return result::current_exception();
+}
+
+kj::Promise<Result<void>> BinaryCacheStore::registerDrvOutput(const Realisation& info)
+try {
+    if (diskCache)
+        diskCache->upsertRealisation(getUri(), info);
+    auto filePath = realisationsPrefix + "/" + info.id.to_string() + ".doi";
+    upsertFile(filePath, info.toJSON().dump(), "application/json");
+    co_return result::success();
+} catch (...) {
+    co_return result::current_exception();
+}
+
 ref<FSAccessor> BinaryCacheStore::getFSAccessor()
 {
     return make_ref<RemoteFSAccessor>(ref<Store>(*this), config().localNarCache);
@@ -582,7 +564,7 @@ try {
 
     debug("fetching build log from binary cache '%s/%s'", getUri(), logPath);
 
-    co_return TRY_AWAIT(getFileContents(logPath));
+    co_return getFileContents(logPath);
 } catch (...) {
     co_return result::current_exception();
 }
@@ -592,11 +574,10 @@ BinaryCacheStore::addBuildLog(const StorePath & drvPath, std::string_view log)
 try {
     assert(drvPath.isDerivation());
 
-    TRY_AWAIT(upsertFile(
+    upsertFile(
         "log/" + std::string(drvPath.to_string()),
         (std::string) log, // FIXME: don't copy
-        "text/plain; charset=utf-8"
-    ));
+        "text/plain; charset=utf-8");
     co_return result::success();
 } catch (...) {
     co_return result::current_exception();

lix/libstore/binary-cache-store.hh

@@ -73,28 +73,23 @@ public:
     BinaryCacheStoreConfig & config() override = 0;
     const BinaryCacheStoreConfig & config() const override = 0;
 
-    virtual kj::Promise<Result<bool>> fileExists(const std::string & path) = 0;
+    virtual bool fileExists(const std::string & path) = 0;
 
-    virtual kj::Promise<Result<void>> upsertFile(
-        const std::string & path,
+    virtual void upsertFile(const std::string & path,
         std::shared_ptr<std::basic_iostream<char>> istream,
-        const std::string & mimeType
-    ) = 0;
+        const std::string & mimeType) = 0;
 
-    kj::Promise<Result<void>> upsertFile(
-        const std::string & path,
+    void upsertFile(const std::string & path,
         // FIXME: use std::string_view
         std::string && data,
-        const std::string & mimeType
-    );
+        const std::string & mimeType);
 
     /**
      * Dump the contents of the specified file to a sink.
      */
-    virtual kj::Promise<Result<box_ptr<AsyncInputStream>>> getFile(const std::string & path) = 0;
+    virtual box_ptr<Source> getFile(const std::string & path) = 0;
 
-    virtual kj::Promise<Result<std::optional<std::string>>> getFileContents(const std::string & path
-    );
+    virtual std::optional<std::string> getFileContents(const std::string & path);
 
 public:
 
@@ -151,7 +146,12 @@ public:
         const StorePathSet & references,
         RepairFlag repair) override;
 
-    kj::Promise<Result<box_ptr<AsyncInputStream>>> narFromPath(const StorePath & path) override;
+    kj::Promise<Result<void>> registerDrvOutput(const Realisation & info) override;
+
+    kj::Promise<Result<std::shared_ptr<const Realisation>>>
+    queryRealisationUncached(const DrvOutput &) override;
+
+    kj::Promise<Result<box_ptr<Source>>> narFromPath(const StorePath & path) override;
 
     ref<FSAccessor> getFSAccessor() override;
 

lix/libstore/build/derivation-goal.cc

@@ -1,6 +1,5 @@
 #include "lix/libstore/build/derivation-goal.hh"
 #include "lix/libutil/async.hh"
-#include "lix/libutil/file-descriptor.hh"
 #include "lix/libutil/file-system.hh"
 #include "lix/libstore/build/hook-instance.hh"
 #include "lix/libstore/build/worker.hh"
@@ -11,6 +10,7 @@
 #include "lix/libstore/common-protocol-impl.hh" // IWYU pragma: keep
 #include "lix/libstore/local-store.hh" // TODO remove, along with remaining downcasts
 #include "lix/libstore/build/substitution-goal.hh"
+#include "lix/libstore/build/drv-output-substitution-goal.hh"
 #include "lix/libutil/result.hh"
 #include "lix/libutil/strings.hh"
 
@@ -74,7 +74,7 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath,
 {
     name = fmt(
         "building of '%s' from .drv file",
-        DerivedPath::Built { makeConstantStorePath(drvPath), wantedOutputs }.to_string(worker.store));
+        DerivedPath::Built { makeConstantStorePathRef(drvPath), wantedOutputs }.to_string(worker.store));
     trace("created");
 
     mcExpectedBuilds = worker.expectedBuilds.addTemporarily(1);
@@ -93,7 +93,7 @@ DerivationGoal::DerivationGoal(DrvHasRoot, const StorePath & drvPath, const Basi
 
     name = fmt(
         "building of '%s' from in-memory derivation",
-        DerivedPath::Built { makeConstantStorePath(drvPath), drv.outputNames() }.to_string(worker.store));
+        DerivedPath::Built { makeConstantStorePathRef(drvPath), drv.outputNames() }.to_string(worker.store));
     trace("created");
 
     mcExpectedBuilds = worker.expectedBuilds.addTemporarily(1);
@@ -199,6 +199,8 @@ try {
     /* Get the derivation. It is probably in the eval store, but it might be inthe main store:
 
          - Resolved derivation are resolved against main store realisations, and so must be stored there.
+
+         - Dynamic derivations are built, and so are found in the main store.
      */
     for (auto * drvStore : { &worker.evalStore, &worker.store }) {
         if (TRY_AWAIT(drvStore->isValidPath(drvPath))) {
@@ -220,8 +222,34 @@ try {
 
     parsedDrv = std::make_unique<ParsedDerivation>(drvPath, *drv);
 
-    for (auto & i : drv->outputsAndPaths(worker.store))
-        TRY_AWAIT(worker.store.addTempRoot(i.second.second));
+    if (!drv->type().hasKnownOutputPaths())
+        experimentalFeatureSettings.require(Xp::CaDerivations);
+
+    if (!drv->type().isPure()) {
+        experimentalFeatureSettings.require(Xp::ImpureDerivations);
+
+        for (auto & [outputName, output] : drv->outputs) {
+            auto randomPath = StorePath::random(outputPathName(drv->name, outputName));
+            assert(!TRY_AWAIT(worker.store.isValidPath(randomPath)));
+            initialOutputs.insert({
+                outputName,
+                InitialOutput {
+                    .wanted = true,
+                    .outputHash = impureOutputHash,
+                    .known = InitialOutputStatus {
+                        .path = randomPath,
+                        .status = PathStatus::Absent
+                    }
+                }
+            });
+        }
+
+        co_return co_await gaveUpOnSubstitution();
+    }
+
+    for (auto & i : drv->outputsAndOptPaths(worker.store))
+        if (i.second.second)
+            TRY_AWAIT(worker.store.addTempRoot(*i.second.second));
 
     auto outputHashes = TRY_AWAIT(staticOutputHashes(worker.evalStore, *drv));
     for (auto & [outputName, outputHash] : outputHashes)
@@ -249,13 +277,14 @@ try {
         if (parsedDrv->substitutesAllowed()) {
             for (auto & [outputName, status] : initialOutputs) {
                 if (!status.wanted) continue;
-                if (!status.known) {
-                    // TODO remove somehow
-                    throw Error(
-                        "congrats, you hit vestigial CA code. sigh.\n"
-                        "please report a bug at https://git.lix.systems/lix-project/lix/issues"
+                if (!status.known)
+                    dependencies.add(
+                        worker.goalFactory().makeDrvOutputSubstitutionGoal(
+                            DrvOutput{status.outputHash, outputName},
+                            buildMode == bmRepair ? Repair : NoRepair
+                        )
                     );
-                } else {
+                else {
                     auto * cap = getDerivationCA(*drv);
                     dependencies.add(worker.goalFactory().makePathSubstitutionGoal(
                         status.known->path,
@@ -280,6 +309,8 @@ kj::Promise<Result<Goal::WorkResult>> DerivationGoal::outputsSubstitutionTried()
 try {
     trace("all outputs substituted (maybe)");
 
+    assert(drv->type().isPure());
+
     if (nrFailed > 0 && nrFailed > nrNoSubstituters + nrIncompleteClosure && !settings.tryFallback)
     {
         co_return done(
@@ -366,18 +397,34 @@ try {
     /* The inputs must be built before we can build this goal. */
     inputDrvOutputs.clear();
     if (useDerivation) {
-        auto addWaiteeDerivedPath = [&](DerivedPathOpaque inputDrv, const StringSet & inputNode) {
-            if (!inputNode.empty())
+        std::function<void(ref<SingleDerivedPath>, const DerivedPathMap<StringSet>::ChildNode &)> addWaiteeDerivedPath;
+
+        addWaiteeDerivedPath = [&](ref<SingleDerivedPath> inputDrv, const DerivedPathMap<StringSet>::ChildNode & inputNode) {
+            if (!inputNode.value.empty())
                 dependencies.add(worker.goalFactory().makeGoal(
                     DerivedPath::Built {
                         .drvPath = inputDrv,
-                        .outputs = inputNode,
+                        .outputs = inputNode.value,
                     },
                     buildMode == bmRepair ? bmRepair : bmNormal));
+            for (const auto & [outputName, childNode] : inputNode.childMap)
+                addWaiteeDerivedPath(
+                    make_ref<SingleDerivedPath>(SingleDerivedPath::Built { inputDrv, outputName }),
+                    childNode);
         };
 
-        for (const auto & [inputDrvPath, inputNode] : dynamic_cast<Derivation *>(drv.get())->inputDrvs) {
-            addWaiteeDerivedPath(makeConstantStorePath(inputDrvPath), inputNode);
+        for (const auto & [inputDrvPath, inputNode] : dynamic_cast<Derivation *>(drv.get())->inputDrvs.map) {
+            /* Ensure that pure, non-fixed-output derivations don't
+               depend on impure derivations. */
+            if (experimentalFeatureSettings.isEnabled(Xp::ImpureDerivations) && drv->type().isPure() && !drv->type().isFixed()) {
+                auto inputDrv = TRY_AWAIT(worker.evalStore.readDerivation(inputDrvPath));
+                if (!inputDrv.type().isPure())
+                    throw Error("pure derivation '%s' depends on impure derivation '%s'",
+                        worker.store.printStorePath(drvPath),
+                        worker.store.printStorePath(inputDrvPath));
+            }
+
+            addWaiteeDerivedPath(makeConstantStorePathRef(inputDrvPath), inputNode);
         }
     }
 
@@ -413,6 +460,8 @@ try {
 
 kj::Promise<Result<Goal::WorkResult>> DerivationGoal::repairClosure() noexcept
 try {
+    assert(drv->type().isPure());
+
     /* If we're repairing, we now know that our own outputs are valid.
        Now check whether the other paths in the outputs closure are
        good.  If not, then start derivation goals for the derivations
@@ -439,9 +488,10 @@ try {
     for (auto & i : inputClosure)
         if (i.isDerivation()) {
             auto depOutputs =
-                TRY_AWAIT(worker.store.queryDerivationOutputMap(i, &worker.evalStore));
+                TRY_AWAIT(worker.store.queryPartialDerivationOutputMap(i, &worker.evalStore));
             for (auto & j : depOutputs)
-                outputsToDrv.insert_or_assign(j.second, i);
+                if (j.second)
+                    outputsToDrv.insert_or_assign(*j.second, i);
         }
 
     /* Check each path (slow!). */
@@ -457,7 +507,7 @@ try {
         else
             dependencies.add(worker.goalFactory().makeGoal(
                 DerivedPath::Built {
-                    .drvPath = makeConstantStorePath(drvPath2->second),
+                    .drvPath = makeConstantStorePathRef(drvPath2->second),
                     .outputs = OutputsSpec::All { },
                 },
                 bmRepair));
@@ -522,8 +572,68 @@ try {
     if (useDerivation) {
         auto & fullDrv = *dynamic_cast<Derivation *>(drv.get());
 
+        auto drvType = fullDrv.type();
+        bool resolveDrv = std::visit(overloaded {
+            [&](const DerivationType::InputAddressed & ia) {
+                /* must resolve if deferred. */
+                return ia.deferred;
+            },
+            [&](const DerivationType::ContentAddressed & ca) {
+                return !fullDrv.inputDrvs.map.empty() && (
+                    ca.fixed
+                    /* Can optionally resolve if fixed, which is good
+                       for avoiding unnecessary rebuilds. */
+                    ? experimentalFeatureSettings.isEnabled(Xp::CaDerivations)
+                    /* Must resolve if floating and there are any inputs
+                       drvs. */
+                    : true);
+            },
+            [&](const DerivationType::Impure &) {
+                return true;
+            }
+        }, drvType.raw);
+
+        if (resolveDrv && !fullDrv.inputDrvs.map.empty()) {
+            experimentalFeatureSettings.require(Xp::CaDerivations);
+
+            /* We are be able to resolve this derivation based on the
+               now-known results of dependencies. If so, we become a
+               stub goal aliasing that resolved derivation goal. */
+            std::optional attempt = TRY_AWAIT(fullDrv.tryResolve(worker.store, inputDrvOutputs));
+            if (!attempt) {
+              /* TODO (impure derivations-induced tech debt) (see below):
+                 The above attempt should have found it, but because we manage
+                 inputDrvOutputs statefully, sometimes it gets out of sync with
+                 the real source of truth (store). So we query the store
+                 directly if there's a problem. */
+              attempt = TRY_AWAIT(fullDrv.tryResolve(worker.store, &worker.evalStore));
+            }
+            assert(attempt);
+            Derivation drvResolved { std::move(*attempt) };
+
+            auto pathResolved = TRY_AWAIT(writeDerivation(worker.store, drvResolved));
+
+            auto msg = fmt("resolved derivation: '%s' -> '%s'",
+                worker.store.printStorePath(drvPath),
+                worker.store.printStorePath(pathResolved));
+            act = std::make_unique<Activity>(*logger, lvlInfo, actBuildWaiting, msg,
+                Logger::Fields {
+                       worker.store.printStorePath(drvPath),
+                       worker.store.printStorePath(pathResolved),
+                   });
+
+            auto dependency = worker.goalFactory().makeDerivationGoal(
+                pathResolved, wantedOutputs, buildMode);
+            resolvedDrvGoal = dependency.first;
+
+            TRY_AWAIT(waitForGoals(std::move(dependency)));
+            co_return co_await resolvedFinished();
+        }
+
+        std::function<kj::Promise<Result<void>>(const StorePath &, const DerivedPathMap<StringSet>::ChildNode &)> accumInputPaths;
+
         // NOLINTNEXTLINE(cppcoreguidelines-avoid-capturing-lambda-coroutines)
-        auto accumInputPaths = [&](const StorePath & depDrvPath, const StringSet & inputNode) -> kj::Promise<Result<void>> {
+        accumInputPaths = [&](const StorePath & depDrvPath, const DerivedPathMap<StringSet>::ChildNode & inputNode) -> kj::Promise<Result<void>> {
             try {
                 /* Add the relevant output closures of the input derivation
                    `i' as input paths.  Only add the closures of output paths
@@ -566,19 +676,21 @@ try {
                     }
                 };
 
-                for (auto & outputName : inputNode) {
+                for (auto & outputName : inputNode.value) {
                     TRY_AWAIT(
                         worker.store.computeFSClosure(TRY_AWAIT(getOutput(outputName)), inputPaths)
                     );
                 }
 
+                for (auto & [outputName, childNode] : inputNode.childMap)
+                    TRY_AWAIT(accumInputPaths(TRY_AWAIT(getOutput(outputName)), childNode));
                 co_return result::success();
             } catch (...) {
                 co_return result::current_exception();
             }
         };
 
-        for (auto & [depDrvPath, depNode] : fullDrv.inputDrvs)
+        for (auto & [depDrvPath, depNode] : fullDrv.inputDrvs.map)
             TRY_AWAIT(accumInputPaths(depDrvPath, depNode));
     }
 
@@ -623,6 +735,8 @@ retry:
        other goal can start a build, and if not, the main loop will sleep a few
        seconds and then retry this goal. */
     PathSet lockFiles;
+    /* FIXME: Should lock something like the drv itself so we don't build same
+       CA drv concurrently */
     if (dynamic_cast<LocalStore *>(&worker.store)) {
         /* If we aren't a local store, we might need to use the local store as
            a build remote, but that would cause a deadlock. */
@@ -631,8 +745,13 @@ retry:
         /* FIXME: find some way to lock for scheduling for the other stores so
            a forking daemon with --store still won't farm out redundant builds.
            */
-        for (auto & i : drv->outputsAndPaths(worker.store)) {
-            lockFiles.insert(worker.store.Store::toRealPath(i.second.second));
+        for (auto & i : drv->outputsAndOptPaths(worker.store)) {
+            if (i.second.second)
+                lockFiles.insert(worker.store.Store::toRealPath(*i.second.second));
+            else
+                lockFiles.insert(
+                    worker.store.Store::toRealPath(drvPath) + "." + i.first
+                );
         }
     }
 
@@ -766,11 +885,7 @@ void replaceValidPath(const Path & storePath, const Path & tmpPath)
        we're repairing (say) Glibc, we end up with a broken system. */
     Path oldPath;
     if (pathExists(storePath)) {
-        do {
-            oldPath = makeTempPath(storePath, ".old");
-            // store paths are often directories so we can't just unlink() it
-            // let's make sure the path doesn't exist before we try to use it
-        } while (pathExists(oldPath));
+        oldPath = makeTempSiblingPath(storePath);
         movePath(storePath, oldPath);
     }
 
@@ -892,7 +1007,7 @@ void runPostBuildHook(
         .program = settings.postBuildHook,
         .environment = hookEnvironment,
         .captureStdout = true,
-        .redirections = {{.dup = STDERR_FILENO, .from = STDOUT_FILENO}},
+        .redirections = {{.from = STDERR_FILENO, .to = STDOUT_FILENO}},
     });
     Finally const _wait([&] {
         try {
@@ -1030,6 +1145,88 @@ try {
     co_return result::current_exception();
 }
 
+kj::Promise<Result<Goal::WorkResult>> DerivationGoal::resolvedFinished() noexcept
+try {
+    trace("resolved derivation finished");
+
+    assert(resolvedDrvGoal);
+    auto resolvedDrv = *resolvedDrvGoal->drv;
+    auto & resolvedResult = resolvedDrvGoal->buildResult;
+
+    SingleDrvOutputs builtOutputs;
+
+    if (resolvedResult.success()) {
+        auto resolvedHashes = TRY_AWAIT(staticOutputHashes(worker.store, resolvedDrv));
+
+        StorePathSet outputPaths;
+
+        for (auto & outputName : resolvedDrv.outputNames()) {
+            auto initialOutput = get(initialOutputs, outputName);
+            auto resolvedHash = get(resolvedHashes, outputName);
+            if ((!initialOutput) || (!resolvedHash))
+                throw Error(
+                    "derivation '%s' doesn't have expected output '%s' (derivation-goal.cc/resolvedFinished,resolve)",
+                    worker.store.printStorePath(drvPath), outputName);
+
+            // NOLINTNEXTLINE(cppcoreguidelines-avoid-capturing-lambda-coroutines)
+            auto realisation = TRY_AWAIT([&]() -> kj::Promise<Result<Realisation>> {
+                try {
+                    auto take1 = get(resolvedResult.builtOutputs, outputName);
+                    if (take1) co_return *take1;
+
+                    /* The above `get` should work. But sateful tracking of
+                       outputs in resolvedResult, this can get out of sync with the
+                       store, which is our actual source of truth. For now we just
+                       check the store directly if it fails. */
+                    auto take2 = TRY_AWAIT(
+                        worker.evalStore.queryRealisation(DrvOutput{*resolvedHash, outputName})
+                    );
+                    if (take2) co_return *take2;
+
+                    throw Error(
+                        "derivation '%s' doesn't have expected output '%s' (derivation-goal.cc/resolvedFinished,realisation)",
+                        worker.store.printStorePath(resolvedDrvGoal->drvPath), outputName);
+                } catch (...) {
+                    co_return result::current_exception();
+                }
+            }());
+
+            if (drv->type().isPure()) {
+                auto newRealisation = realisation;
+                newRealisation.id = DrvOutput { initialOutput->outputHash, outputName };
+                newRealisation.signatures.clear();
+                if (!drv->type().isFixed()) {
+                    auto & drvStore = TRY_AWAIT(worker.evalStore.isValidPath(drvPath))
+                        ? worker.evalStore
+                        : worker.store;
+                    newRealisation.dependentRealisations = TRY_AWAIT(
+                        drvOutputReferences(worker.store, *drv, realisation.outPath, &drvStore)
+                    );
+                }
+                signRealisation(newRealisation);
+                TRY_AWAIT(worker.store.registerDrvOutput(newRealisation));
+            }
+            outputPaths.insert(realisation.outPath);
+            builtOutputs.emplace(outputName, realisation);
+        }
+
+        runPostBuildHook(
+            worker.store,
+            *logger,
+            drvPath,
+            outputPaths
+        );
+    }
+
+    auto status = resolvedResult.status;
+    if (status == BuildResult::AlreadyValid)
+        status = BuildResult::ResolvesToAlreadyValid;
+
+    co_return done(status, std::move(builtOutputs));
+} catch (...) {
+    co_return result::current_exception();
+}
+
 HookReply DerivationGoal::tryBuildHook()
 {
     if (!worker.hook.available || !useDerivation) return HookReply::Decline{};
@@ -1040,10 +1237,13 @@ HookReply DerivationGoal::tryBuildHook()
     try {
 
         /* Send the request to the hook. */
-        *worker.hook.instance->sink << "try" << (slotToken.valid() ? 1 : 0) << drv->platform
-                                    << worker.store.printStorePath(drvPath)
-                                    << parsedDrv->getRequiredSystemFeatures();
-        worker.hook.instance->sink->flush();
+        worker.hook.instance->sink
+            << "try"
+            << (slotToken.valid() ? 1 : 0)
+            << drv->platform
+            << worker.store.printStorePath(drvPath)
+            << parsedDrv->getRequiredSystemFeatures();
+        worker.hook.instance->sink.flush();
 
         /* Read the first line of input, which should be a word indicating
            whether the hook wishes to perform the build. */
@@ -1106,7 +1306,7 @@ HookReply DerivationGoal::tryBuildHook()
 
     /* Tell the hook all the inputs that have to be copied to the
        remote system. */
-    *hook->sink << CommonProto::write({worker.store}, inputPaths);
+    hook->sink << CommonProto::write(worker.store, {}, inputPaths);
 
     /* Tell the hooks the missing outputs that have to be copied back
        from the remote system. */
@@ -1117,10 +1317,10 @@ HookReply DerivationGoal::tryBuildHook()
             if (buildMode != bmCheck && status.known && status.known->isValid()) continue;
             missingOutputs.insert(outputName);
         }
-        *hook->sink << CommonProto::write({worker.store}, missingOutputs);
+        hook->sink << CommonProto::write(worker.store, {}, missingOutputs);
     }
 
-    hook->sink = nullptr;
+    hook->sink = FdSink();
     hook->toHook.reset();
 
     /* Create the log file and pipe. */
@@ -1205,7 +1405,13 @@ struct DerivationGoal::InputStream final : private kj::AsyncObject
         : fd(fd)
         , observer(ep, fd, kj::UnixEventPort::FdObserver::OBSERVE_READ)
     {
-        makeNonBlocking(fd);
+        int flags = fcntl(fd, F_GETFL);
+        if (flags < 0) {
+            throw SysError("fcntl(F_GETFL) failed on fd %i", fd);
+        }
+        if (fcntl(fd, F_SETFL, flags | O_NONBLOCK) < 0) {
+            throw SysError("fcntl(F_SETFL) failed on fd %i", fd);
+        }
     }
 
     kj::Promise<std::string_view> read(kj::ArrayPtr<char> buffer)
@@ -1408,12 +1614,43 @@ void DerivationGoal::flushLine()
 }
 
 
+kj::Promise<Result<std::map<std::string, std::optional<StorePath>>>> DerivationGoal::queryPartialDerivationOutputMap()
+try {
+    assert(drv->type().isPure());
+    if (!useDerivation || drv->type().hasKnownOutputPaths()) {
+        std::map<std::string, std::optional<StorePath>> res;
+        for (auto & [name, output] : drv->outputs)
+            res.insert_or_assign(name, output.path(worker.store, drv->name, name));
+        co_return res;
+    } else {
+        for (auto * drvStore : {&worker.evalStore, &worker.store}) {
+            if (TRY_AWAIT(drvStore->isValidPath(drvPath))) {
+                co_return TRY_AWAIT(worker.store.queryPartialDerivationOutputMap(drvPath, drvStore)
+                );
+            }
+        }
+        assert(false);
+    }
+} catch (...) {
+    co_return result::current_exception();
+}
+
 kj::Promise<Result<OutputPathMap>> DerivationGoal::queryDerivationOutputMap()
 try {
-    OutputPathMap res;
-    for (auto & [name, output] : drv->outputsAndPaths(worker.store))
-        res.insert_or_assign(name, output.second);
-    co_return res;
+    assert(drv->type().isPure());
+    if (!useDerivation || drv->type().hasKnownOutputPaths()) {
+        OutputPathMap res;
+        for (auto & [name, output] : drv->outputsAndOptPaths(worker.store))
+            res.insert_or_assign(name, *output.second);
+        co_return res;
+    } else {
+        for (auto * drvStore : {&worker.evalStore, &worker.store}) {
+            if (TRY_AWAIT(drvStore->isValidPath(drvPath))) {
+                co_return TRY_AWAIT(worker.store.queryDerivationOutputMap(drvPath, drvStore));
+            }
+        }
+        assert(false);
+    }
 } catch (...) {
     co_return result::current_exception();
 }
@@ -1421,6 +1658,8 @@ try {
 
 kj::Promise<Result<std::pair<bool, SingleDrvOutputs>>> DerivationGoal::checkPathValidity()
 try {
+    if (!drv->type().isPure()) co_return { false, SingleDrvOutputs{} };
+
     bool checkHash = buildMode == bmRepair;
     auto wantedOutputsLeft = std::visit(overloaded {
         [&](const OutputsSpec::All &) {
@@ -1432,7 +1671,7 @@ try {
     }, wantedOutputs.raw);
     SingleDrvOutputs validOutputs;
 
-    for (auto & i : TRY_AWAIT(queryDerivationOutputMap())) {
+    for (auto & i : TRY_AWAIT(queryPartialDerivationOutputMap())) {
         auto initialOutput = get(initialOutputs, i.first);
         if (!initialOutput)
             // this is an invalid output, gets catched with (!wantedOutputsLeft.empty())
@@ -1441,16 +1680,37 @@ try {
         info.wanted = wantedOutputs.contains(i.first);
         if (info.wanted)
             wantedOutputsLeft.erase(i.first);
-        auto & outputPath = i.second;
-        info.known = {
-            .path = outputPath,
-            .status = !TRY_AWAIT(worker.store.isValidPath(outputPath))
-                ? PathStatus::Absent
-                : !checkHash || TRY_AWAIT(worker.pathContentsGood(outputPath))
-                ? PathStatus::Valid
-                : PathStatus::Corrupt,
-        };
+        if (i.second) {
+            auto outputPath = *i.second;
+            info.known = {
+                .path = outputPath,
+                .status = !TRY_AWAIT(worker.store.isValidPath(outputPath))
+                    ? PathStatus::Absent
+                    : !checkHash || TRY_AWAIT(worker.pathContentsGood(outputPath))
+                    ? PathStatus::Valid
+                    : PathStatus::Corrupt,
+            };
+        }
         auto drvOutput = DrvOutput{info.outputHash, i.first};
+        if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) {
+            if (auto real = TRY_AWAIT(worker.store.queryRealisation(drvOutput))) {
+                info.known = {
+                    .path = real->outPath,
+                    .status = PathStatus::Valid,
+                };
+            } else if (info.known && info.known->isValid()) {
+                // We know the output because it's a static output of the
+                // derivation, and the output path is valid, but we don't have
+                // its realisation stored (probably because it has been built
+                // without the `ca-derivations` experimental flag).
+                TRY_AWAIT(worker.store.registerDrvOutput(
+                    Realisation {
+                        drvOutput,
+                        info.known->path,
+                    }
+                ));
+            }
+        }
         if (info.known && info.known->isValid())
             validOutputs.emplace(i.first, Realisation { drvOutput, info.known->path });
     }
@@ -1547,13 +1807,13 @@ void DerivationGoal::waiteeDone(GoalPtr waitee)
 
     auto & fullDrv = *dynamic_cast<Derivation *>(drv.get());
 
-    auto * nodeP = get(fullDrv.inputDrvs, dg->drvPath);
+    auto * nodeP = fullDrv.inputDrvs.findSlot(DerivedPath::Opaque { .path = dg->drvPath });
     if (!nodeP) return;
-    auto & outputs = *nodeP;
+    auto & outputs = nodeP->value;
 
     for (auto & outputName : outputs) {
         auto buildResult = dg->buildResult.restrictTo(DerivedPath::Built {
-            .drvPath = makeConstantStorePath(dg->drvPath),
+            .drvPath = makeConstantStorePathRef(dg->drvPath),
             .outputs = OutputsSpec::Names { outputName },
         });
         if (buildResult.success()) {

lix/libstore/build/derivation-goal.hh

@@ -87,6 +87,11 @@ struct DerivationGoal : public Goal
     /** The path of the derivation. */
     StorePath drvPath;
 
+    /**
+     * The goal for the corresponding resolved derivation
+     */
+    std::shared_ptr<DerivationGoal> resolvedDrvGoal;
+
     /**
      * The specific outputs that we need to build.
      */
@@ -274,6 +279,8 @@ struct DerivationGoal : public Goal
     virtual kj::Promise<Result<WorkResult>> tryLocalBuild() noexcept;
     kj::Promise<Result<WorkResult>> buildDone() noexcept;
 
+    kj::Promise<Result<WorkResult>> resolvedFinished() noexcept;
+
     /**
      * Is the build hook willing to perform the build?
      */
@@ -292,6 +299,11 @@ struct DerivationGoal : public Goal
      */
     Path openLogFile();
 
+    /**
+     * Sign the newly built realisation if the store allows it
+     */
+    virtual void signRealisation(Realisation&) {}
+
     /**
      * Close the log file.
      */
@@ -326,10 +338,11 @@ protected:
 
 public:
     /**
-     * Wrappers around the corresponding Store method that first consults the
+     * Wrappers around the corresponding Store methods that first consult the
      * derivation.  This is currently needed because when there is no drv file
      * there also is no DB entry.
      */
+    kj::Promise<Result<std::map<std::string, std::optional<StorePath>>>> queryPartialDerivationOutputMap();
     kj::Promise<Result<OutputPathMap>> queryDerivationOutputMap();
 
     /**
@@ -367,6 +380,8 @@ public:
         return false;
     }
 
+    StorePathSet exportReferences(const StorePathSet & storePaths);
+
     JobCategory jobCategory() const override {
         return JobCategory::Build;
     };

lix/libstore/build/drv-output-substitution-goal.cc

@@ -0,0 +1,164 @@
+#include "lix/libstore/build/drv-output-substitution-goal.hh"
+#include "lix/libstore/build-result.hh"
+#include "lix/libutil/async.hh"
+#include "lix/libutil/finally.hh"
+#include "lix/libstore/build/worker.hh"
+#include "lix/libstore/build/substitution-goal.hh"
+#include "lix/libutil/signals.hh"
+#include <kj/array.h>
+#include <kj/async.h>
+#include <kj/vector.h>
+
+namespace nix {
+
+DrvOutputSubstitutionGoal::DrvOutputSubstitutionGoal(
+    const DrvOutput & id,
+    Worker & worker,
+    bool isDependency,
+    RepairFlag repair,
+    std::optional<ContentAddress> ca)
+    : Goal(worker, isDependency)
+    , id(id)
+{
+    name = fmt("substitution of '%s'", id.to_string());
+    trace("created");
+}
+
+
+kj::Promise<Result<Goal::WorkResult>> DrvOutputSubstitutionGoal::workImpl() noexcept
+try {
+    trace("init");
+
+    /* If the derivation already exists, we’re done */
+    if (TRY_AWAIT(worker.store.queryRealisation(id))) {
+        co_return WorkResult{ecSuccess};
+    }
+
+    subs = settings.useSubstitutes ? TRY_AWAIT(getDefaultSubstituters()) : std::list<ref<Store>>();
+    co_return co_await tryNext();
+} catch (...) {
+    co_return result::current_exception();
+}
+
+kj::Promise<Result<Goal::WorkResult>> DrvOutputSubstitutionGoal::tryNext() noexcept
+try {
+    trace("trying next substituter");
+
+    if (!slotToken.valid()) {
+        slotToken = co_await worker.substitutions.acquire();
+    }
+
+    maintainRunningSubstitutions = worker.runningSubstitutions.addTemporarily(1);
+
+    if (subs.size() == 0) {
+        /* None left.  Terminate this goal and let someone else deal
+           with it. */
+        debug("derivation output '%s' is required, but there is no substituter that can provide it", id.to_string());
+
+        if (substituterFailed) {
+            worker.failedSubstitutions++;
+        }
+
+        /* Hack: don't indicate failure if there were no substituters.
+           In that case the calling derivation should just do a
+           build. */
+        co_return WorkResult{substituterFailed ? ecFailed : ecNoSubstituters};
+    }
+
+    sub = subs.front();
+    subs.pop_front();
+
+    /* The async call to a curl download below can outlive `this` (if
+       some other error occurs), so it must not touch `this`. So put
+       the shared state in a separate refcounted object. */
+    downloadState = std::make_shared<DownloadState>();
+    auto pipe = kj::newPromiseAndCrossThreadFulfiller<void>();
+    downloadState->outPipe = kj::mv(pipe.fulfiller);
+
+    downloadState->result =
+        std::async(std::launch::async, [downloadState{downloadState}, id{id}, sub{sub}] {
+            Finally updateStats([&]() { downloadState->outPipe->fulfill(); });
+            ReceiveInterrupts receiveInterrupts;
+            AsyncIoRoot aio;
+            return aio.blockOn(sub->queryRealisation(id));
+        });
+
+    co_await pipe.promise;
+    co_return co_await realisationFetched();
+} catch (...) {
+    co_return result::current_exception();
+}
+
+kj::Promise<Result<Goal::WorkResult>> DrvOutputSubstitutionGoal::realisationFetched() noexcept
+try {
+    maintainRunningSubstitutions.reset();
+    slotToken = {};
+
+    try {
+        outputInfo = downloadState->result.get();
+    } catch (std::exception & e) { // NOLINT(lix-foreign-exceptions)
+        printError(e.what());
+        substituterFailed = true;
+    }
+
+    if (!outputInfo) {
+        co_return co_await tryNext();
+    }
+
+    kj::Vector<std::pair<GoalPtr, kj::Promise<Result<WorkResult>>>> dependencies;
+    for (const auto & [depId, depPath] : outputInfo->dependentRealisations) {
+        if (depId != id) {
+            if (auto localOutputInfo = TRY_AWAIT(worker.store.queryRealisation(depId));
+                localOutputInfo && localOutputInfo->outPath != depPath) {
+                warn(
+                    "substituter '%s' has an incompatible realisation for '%s', ignoring.\n"
+                    "Local:  %s\n"
+                    "Remote: %s",
+                    sub->getUri(),
+                    depId.to_string(),
+                    worker.store.printStorePath(localOutputInfo->outPath),
+                    worker.store.printStorePath(depPath)
+                );
+                co_return co_await tryNext();
+            }
+            dependencies.add(worker.goalFactory().makeDrvOutputSubstitutionGoal(depId));
+        }
+    }
+
+    dependencies.add(worker.goalFactory().makePathSubstitutionGoal(outputInfo->outPath));
+
+    if (!dependencies.empty()) {
+        TRY_AWAIT(waitForGoals(dependencies.releaseAsArray()));
+    }
+    co_return co_await outPathValid();
+} catch (...) {
+    co_return result::current_exception();
+}
+
+kj::Promise<Result<Goal::WorkResult>> DrvOutputSubstitutionGoal::outPathValid() noexcept
+try {
+    assert(outputInfo);
+    trace("output path substituted");
+
+    if (nrFailed > 0) {
+        debug("The output path of the derivation output '%s' could not be substituted", id.to_string());
+        co_return WorkResult{
+            nrNoSubstituters > 0 || nrIncompleteClosure > 0 ? ecIncompleteClosure : ecFailed,
+        };
+    }
+
+    TRY_AWAIT(worker.store.registerDrvOutput(*outputInfo));
+    co_return TRY_AWAIT(finished());
+} catch (...) {
+    co_return result::current_exception();
+}
+
+kj::Promise<Result<Goal::WorkResult>> DrvOutputSubstitutionGoal::finished() noexcept
+try {
+    trace("finished");
+    return {WorkResult{ecSuccess}};
+} catch (...) {
+    return {result::current_exception()};
+}
+
+}

lix/libstore/build/drv-output-substitution-goal.hh

@@ -0,0 +1,80 @@
+#pragma once
+///@file
+
+#include "lix/libutil/notifying-counter.hh"
+#include "lix/libstore/store-api.hh"
+#include "lix/libstore/build/goal.hh"
+#include "lix/libstore/realisation.hh"
+#include <future>
+
+namespace nix {
+
+class Worker;
+
+/**
+ * Substitution of a derivation output.
+ * This is done in three steps:
+ * 1. Fetch the output info from a substituter
+ * 2. Substitute the corresponding output path
+ * 3. Register the output info
+ */
+class DrvOutputSubstitutionGoal : public Goal {
+
+    /**
+     * The drv output we're trying to substitute
+     */
+    DrvOutput id;
+
+    /**
+     * The realisation corresponding to the given output id.
+     * Will be filled once we can get it.
+     */
+    std::shared_ptr<const Realisation> outputInfo;
+
+    /**
+     * The remaining substituters.
+     */
+    std::list<ref<Store>> subs;
+
+    /**
+     * The current substituter.
+     */
+    std::shared_ptr<Store> sub;
+
+    NotifyingCounter<uint64_t>::Bump maintainRunningSubstitutions;
+
+    struct DownloadState
+    {
+        kj::Own<kj::CrossThreadPromiseFulfiller<void>> outPipe;
+        std::future<std::shared_ptr<const Realisation>> result;
+    };
+
+    std::shared_ptr<DownloadState> downloadState;
+
+    /**
+     * Whether a substituter failed.
+     */
+    bool substituterFailed = false;
+
+public:
+    DrvOutputSubstitutionGoal(
+        const DrvOutput & id,
+        Worker & worker,
+        bool isDependency,
+        RepairFlag repair = NoRepair,
+        std::optional<ContentAddress> ca = std::nullopt
+    );
+
+    kj::Promise<Result<WorkResult>> tryNext() noexcept;
+    kj::Promise<Result<WorkResult>> realisationFetched() noexcept;
+    kj::Promise<Result<WorkResult>> outPathValid() noexcept;
+    kj::Promise<Result<WorkResult>> finished() noexcept;
+
+    kj::Promise<Result<WorkResult>> workImpl() noexcept override;
+
+    JobCategory jobCategory() const override {
+        return JobCategory::Substitution;
+    };
+};
+
+}

lix/libstore/build/entry-points.cc

@@ -93,7 +93,7 @@ try {
         }));
         auto & result = results.goals.begin()->second;
         co_return result.result.restrictTo(DerivedPath::Built {
-            .drvPath = makeConstantStorePath(drvPath),
+            .drvPath = makeConstantStorePathRef(drvPath),
             .outputs = OutputsSpec::All {},
         });
     } catch (Error & e) {
@@ -151,7 +151,7 @@ try {
                 Worker::Targets goals;
                 goals.emplace_back(gf.makeGoal(
                     DerivedPath::Built{
-                        .drvPath = makeConstantStorePath(*info->deriver),
+                        .drvPath = makeConstantStorePathRef(*info->deriver),
                         // FIXME: Should just build the specific output we need.
                         .outputs = OutputsSpec::All{},
                     },

lix/libstore/build/hook-instance.cc

@@ -72,12 +72,12 @@ HookInstance::HookInstance()
     toHook = std::move(toHook_.writeSide);
     builderOut = std::move(builderOut_.readSide);
 
-    sink = std::make_unique<FdSink>(toHook.get());
+    sink = FdSink(toHook.get());
     std::map<std::string, Config::SettingInfo> settings;
     globalConfig.getSettings(settings, true);
     for (auto & setting : settings)
-        *sink << 1 << setting.first << setting.second.value;
-    *sink << 0;
+        sink << 1 << setting.first << setting.second.value;
+    sink << 0;
 }
 
 

lix/libstore/build/hook-instance.hh

@@ -29,7 +29,7 @@ struct HookInstance
      */
     Pid pid;
 
-    std::unique_ptr<FdSink> sink;
+    FdSink sink;
 
     std::map<ActivityId, Activity> activities;