jade wip

Change-Id: I4d5a0bebeeefc35dac0bbc065c104c70c24aae17

.clang-tidy

@@ -8,16 +8,12 @@ Checks:
   - -bugprone-narrowing-conversions
     # kind of nonsense
   - -bugprone-easily-swappable-parameters
-    # too many warnings for now
-  - -bugprone-implicit-widening-of-multiplication-result
     # Lix's exception handling is Questionable
   - -bugprone-empty-catch
     # many warnings
   - -bugprone-unchecked-optional-access
     # many warnings, seems like a questionable lint
   - -bugprone-branch-clone
-    # extremely noisy before clang 19: https://github.com/llvm/llvm-project/issues/93959
-  - -bugprone-multi-level-implicit-pointer-conversion
     # we don't compile out our asserts
   - -bugprone-assert-side-effect
     # FIXME(jade): figure out if this warning is any good
@@ -29,9 +25,6 @@ Checks:
     # crimes must be appropriately declared as crimes
   - cppcoreguidelines-pro-type-cstyle-cast
   - lix-*
-    # This can not yet be applied to Lix itself since we need to do source
-    # reorganization so that lix/ include paths work.
-  - -lix-fixincludes
     # This lint is included as an example, but the lib function it replaces is
     # already gone.
   - -lix-hasprefixsuffix
@@ -40,3 +33,5 @@ Checks:
 CheckOptions:
   bugprone-reserved-identifier.AllowedIdentifiers: '__asan_default_options'
   bugprone-unused-return-value.AllowCastToVoid: true
+
+ExtraArgs: ["-Werror=unnecessary-virtual-specifier"]

doc/manual/anchors.jq

@@ -24,8 +24,8 @@ def map_contents_recursively(transformer):
 def process_command:
     .[0] as $context |
     .[1] as $body |
-    $body + {
-        sections: $body.sections | map(map_contents_recursively(if $context.renderer == "html" then transform_anchors_html else transform_anchors_strip end)),
-    };
+	# XXX FUTURE: drop sections once mdBook is at 0.5.0 or above in nixpkgs
+	$body | (.items? // .sections) |= map(map_contents_recursively(if $context.renderer == "html" then transform_anchors_html else transform_anchors_strip end))
+	;
 
 process_command

doc/manual/book.toml

@@ -22,20 +22,21 @@ fold.level = 30
 # not want to disable the links preprocessor entirely though because that requires
 # disabling *all* built-in preprocessors and selectively reenabling those we want.
 [preprocessor.substitute]
-command = "python3 doc/manual/substitute.py"
+command = "python3 substitute.py"
 before = ["anchors", "links"]
 
 [preprocessor.anchors]
 renderers = ["html"]
-command = "jq --from-file doc/manual/anchors.jq"
+command = "jq --from-file anchors.jq"
 
 [output.markdown]
 
-[output.linkcheck]
+# XXX FUTURE: may be reenabled once mdBook 0.5.0 or above and matching mdbook-linkchecker are in nixpkgs
+#[output.linkcheck]
 # no Internet during the build (in the sandbox)
-follow-web-links = false
+#follow-web-links = false
 
 # mdbook-linkcheck does not understand [foo]{#bar} style links, resulting in
 # excessive "Potential incomplete link" warnings. No other kind of warning was
 # produced at the time of writing.
-warning-policy = "ignore"
+#warning-policy = "ignore"

doc/manual/change-authors.yml

@@ -61,6 +61,11 @@ cole-h:
   display_name: Cole Helbling
   github: cole-h
 
+commentator2.0:
+  display_name: Commentator2.0 (Rutile)
+  forgejo: commentatorforall
+  github: CommentatorForAll
+
 delan:
   display_name: delan
   forgejo: delan
@@ -125,6 +130,11 @@ jade:
 just1602:
   forgejo: just1602
 
+k900:
+  display_name: K900
+  forgejo: K900
+  github: K900
+
 kasimeka:
   display_name: ورد
   forgejo: janw4ld
@@ -176,6 +186,11 @@ midnightveil:
   forgejo: midnightveil
   github: midnightveil
 
+milibopp:
+  display_name: Emilia Bopp
+  forgejo: milibopp
+  github: milibopp
+
 nan-git:
   display_name: NaN-git
   github: NaN-git

doc/manual/meson.build

@@ -42,8 +42,9 @@ manual = custom_target(
     '-c',
     '''
         @0@ @INPUT0@ @CURRENT_SOURCE_DIR@ > @DEPFILE@
+        cd @3@
+        @1@ build . -d @2@ | { grep -Fv "because fragment resolution isn't implemented" || :; }
         cd @SOURCE_ROOT@
-        @1@ build doc/manual -d @2@ | { grep -Fv "because fragment resolution isn't implemented" || :; }
         rm -rf @2@/manual
         mv @2@/html @2@/manual
         find @2@/manual -iname meson.build -delete
@@ -51,6 +52,7 @@ manual = custom_target(
       python.full_path(),
       mdbook.full_path(),
       meson.current_build_dir(),
+      meson.current_source_dir()
     ),
   ],
   input : [
@@ -81,7 +83,7 @@ manual = custom_target(
   depfile : 'manual.d',
   env : {
     'RUST_LOG': 'info',
-    'MDBOOK_SUBSTITUTE_SEARCH': meson.current_build_dir() / 'src',
+    'MANUAL_SUBSTITUTE_SEARCH': meson.current_build_dir() / 'src',
   },
 )
 manual_md = manual[1]

doc/manual/rl-next/added-shell-dir-length.md

@@ -0,0 +1,13 @@
+---
+synopsis: "Nix shells' $NIX_BUILD_TOP are shorter"
+cls: [4663]
+issues: [fj#1044]
+category: "Fixes"
+credits: [raito]
+---
+
+Following the changes in 2.94.0 to shorten build directory paths, aimed at [resolving UNIX domain socket length issues](https://gerrit.lix.systems/c/lix/+/4168/13) and [improving nix-shell](https://git.lix.systems/lix-project/lix/issues/940), we inadvertently introduced an excessively long path for the `$NIX_BUILD_TOP` environment variable used by Nix shells (their effective temporary `/build` directory).
+
+To fix this, we replaced the `build-top-$HASH` directory name with simply `build-top`, reducing these paths by at least 30 characters.
+
+We also added a test to ensure that Nix shells do not introduce more than 50 extra characters relative to their base directory (e.g., `/tmp` when `$TMPDIR` is not set).

doc/manual/rl-next/builtins-warn.md

@@ -0,0 +1,15 @@
+---
+synopsis: "Add `builtins.warn` for emitting warnings from Nix code"
+cls: [2248]
+category: "Features"
+credits: [milibopp, Qyriad]
+---
+
+Lix now has a builtin function for emitting warnings.
+Like `builtins.trace`, it takes two arguments: the message to emit, and the expression to return.
+_Unlike_ `builtins.trace`, `builtins.warn` requires the first argument — the message — to be a string.
+In the future we may extend `builtins.warn` to accept a more structured API.
+
+To go along with this, we also have two new config settings:
+- [`debugger-on-warn`](@docroot@/command-ref/conf-file.md#conf-debugger-on-warn), which, when used with `--debugger`, makes `builtins.warn` also function like [`builtins.break`](@docroot@/language/builtins.md#builtins-break).
+- [`abort-on-warn`](@docroot@/command-ref/conf-file.md#conf-abort-on-warn), which aborts evaluation entirely after the warning is emitted.

doc/manual/rl-next/deprecat-nix-path-shadow.md

@@ -0,0 +1,13 @@
+---
+synopsis: "Deprecate shadowing internal files through the Nix search path"
+issues: [998]
+cls: [4632]
+category: "Breaking Changes"
+credits: [thubrecht]
+---
+
+As Lix uses the path `<nix/fetchurl.nix>` for bootstrapping purposes, the ability to shadow it by adding `nix=/some/path` (or `/other/path` that contains a `nix` directory) to the search path is not desirable.
+
+To alleviate potential issues, Lix now emits a warning when the Nix search path contains potential shadows for internal files, which will be changed to an error in a future release.
+
+The warning can be disabled by enabling the deprecated feature `nix-path-shadow`.

doc/manual/rl-next/final-destination-warning.md

@@ -0,0 +1,10 @@
+---
+synopsis: "Warn instead of erroring when the final destination of a transfer changes in-flight"
+cls: [4641]
+issues: [fj#1004]
+category: "Miscellany"
+credits: [thubrecht]
+---
+
+Lix will now emit a warning during downloads where the final destination changes suddently mid-transfer instead of throwing an error.
+This transfer behavior has been known to happen very rarely while fetching from some CDNs.

doc/manual/rl-next/functional-lang-removal.md

@@ -0,0 +1,10 @@
+---
+synopsis: 'functional lang migration'
+issues: [lix#856]
+cls: [3213, 3214, 3215, 3224, 4092, 4093, 4094, 4095, 4096, 4097, 4098, 4099, 4100, 4101, 4102, 4103, 4104, 4105, 4106, 4107, 4108, 4109, 4110, 4111, 4112, 4113, 4114, 4115, 4116, 4117, 4122, 4123, 4269, 4270, 4271, 4272, 4273, 4274, 4347, 4348, 4349, 4350, 4351, 4352, 4569, 4570, 4571, 4572, 4573, 4574, 4596, 4597, 4598, 4599, 4600, 4601, 4602, 4603, 4604, 4605]
+category: Development
+credits: [piegames, commentator2.0]
+---
+We have done it! The functional/lang framework has now been fully migrated to functional2/lang.
+This means: no more `just clean` and `just install` mess and whatever because one removed a test.
+The lang test suite is also getting a face lift, with an improved folder structure and restructuring of many tests.

doc/manual/rl-next/machines-toml.md

@@ -0,0 +1,10 @@
+---
+synopsis: "Allow remote builders to be configured using TOML"
+cls: [4533]
+category: "Features"
+credits: [commentator2.0, Qyriad]
+---
+Lix now supports configuring remote builders using a TOML file instead of the old, very cursed and incomprehensible format.
+This comes with not only a human-understandable file, but also with better messages and error reports on misconfiguration.
+
+A more detailed Documentation can be found on the [distributed-builds](@docroot@/advanced-topics/distributed-builds.md) Wiki-page

doc/manual/rl-next/more-logs.md

@@ -0,0 +1,8 @@
+---
+synopsis: "Default to showing build logs in the new-style (nix3) CLI"
+cls: [4674]
+category: "Miscellany"
+credits: [k900]
+---
+
+Lix will now show logs by default, in addition to the progress bar, when invoked through the new-style "nix3" CLI (`nix build`, etc)

doc/manual/rl-next/move-nixdaemon-cache-dirs.md

@@ -0,0 +1,31 @@
+---
+synopsis: "Move /root/.cache/nix to /var/cache/nix by default"
+cls: [4671]
+issues: [fj#634]
+category: "Breaking Changes"
+credits: [raito]
+---
+
+By default, Lix attempts to locate a cache directory for its operations (such
+as the narinfo cache) by checking the value of `$XDG_CACHE_DIR`.
+
+However, since the Nix daemon is a system service, using `$XDG_CACHE_DIR` is
+not typical in this context.
+
+To address this, systemd provides a better solution. Specifically, when
+`CacheDirectory=` is set in the `[Service]` section of a systemd unit, it
+automatically sets the `$CACHE_DIRECTORY` environment variable and systemd will
+manage that cache directory for us.
+
+Now, our systemd unit includes `CacheDirectory=nix`, which sets the
+`$CACHE_DIRECTORY` and takes precedence over `$XDG_CACHE_DIR`.
+
+If the daemon is run under user units, systemd will automatically set
+`$XDG_CACHE_DIR`.
+
+If neither of these variables is set, Lix falls back to its default behavior.
+By default, Lix will try to find a cache directory for its various operations
+(e.g. narinfo cache) by looking into `$XDG_CACHE_DIR`.
+
+In summary, what was stored in `/root/.cache/nix` is now moved to
+`/var/cache/nix/nix`.

doc/manual/rl-next/nix-shell-level.md

@@ -0,0 +1,9 @@
+---
+synopsis: "Add an indication of nix-shell nesting depth"
+cls: [4657]
+issues: [fj#826]
+category: "Improvements"
+credits: [thubrecht]
+---
+
+When in a nix shell (either via a `nix-shell` or a `nix develop` invocation), a variable `NIX_SHELL_LEVEL` is exported to indicate the nesting depth of nix shells.

doc/manual/rl-next/print-derivations.md

@@ -0,0 +1,20 @@
+---
+synopsis: Derivations can now be printed in detail in `nix repl`
+cls: [3842]
+category: Improvements
+credits: [Lunaphied]
+---
+
+Traditionally derivations printed in the REPL would only print a formatted object
+representing the path of the derivation file it refers to. This makes inspecting
+the enhanced derivation attribute sets encountered from `mkDerivation` or similar
+wrappers more difficult. Even the `:p`/`:print` command would not elaborate attribute sets
+tagged as a derivation.
+
+With this change you can now use `:p`/`:print` to directly inspect a derivation
+by providing one as the top-level object. Derivation attribute sets will only be
+printed two levels deep and internal derivation attrsets will remain in unexpanded
+path form as before. `drvAttrs` will also be elided as these attributes are already
+present in the top-level attribute set of the derivation. These heuristics provide
+a balance between readability and functionality. When the `:p`/`:print` is omitted,
+a bare derivation is printed in the path format as before.

doc/manual/rl-next/remove-xp-fetch-closure-.md

@@ -0,0 +1,37 @@
+---
+synopsis: Remove `fetch-closure` experimental feature
+issues: [fj#1010]
+cls: [4595]
+category: "Breaking Changes"
+credits: [just1602]
+---
+
+The `fetch-closure` experimental feature has been removed.
+
+Outside of allowing the user to import closure from binary cache,
+`fetchClosure` also allow you to do the following:
+
+* rewrite non-CA path to CA
+* reject non-CA paths at fetching time
+* reject CA paths at fetching time
+
+Some people are using those mechanism to prevent users from having to build any
+package and force going via the declared cache or as a way to use ancient/old
+software without paying the evaluation cost of a second nixpkgs.
+
+Both use cases are somewhat of an antipattern in Nix semantics. If the user
+cannot fetch a program directly via the substituter mechanism and fall back to
+local build, this is a feature AND a misconfiguration. If the user cannot build
+certain derivations because they are too expensive, the build directives should
+pass `-j0` or similar.
+
+As for the second usecase, there's a different way to do it that also allows to
+have a way to reproduce the paths that are hardcoded in that file, perform
+`import (fetchurl "https://my-cache/${hashparts storepath}.drv")` rather, i.e.
+an IFD to a possibly well known name. The backend can generate them on the fly
+or once, and possess stable names.
+
+Finally, as for the non-CA → CA features, Lix removed ca-derivations.
+fetchClosure offers ca-derivations-like features which suffers from similar
+shortcomings albeit lessened. It only follows that we should rather deprecate
+and remove these capabilities.

doc/manual/rl-next/shells-honors-nix-log-fd.md

@@ -0,0 +1,14 @@
+---
+synopsis: "Shells supports $NIX_LOG_FD now"
+cls: [4694, 4695]
+issues: [fj#336]
+category: "Improvements"
+credits: [raito]
+---
+
+Lix's "debugging" shells (`nix3-develop` and `nix-shell`) now supports
+`$NIX_LOG_FD` environment variable.
+
+This means that [hook logging in
+stdenv](https://github.com/NixOS/nixpkgs/pull/310387) appears while debugging
+derivations via `nix3-develop` or `nix-shell`.

doc/manual/src/advanced-topics/distributed-builds.md

@@ -41,106 +41,17 @@ contains Nix.
 > If you are building via the Lix daemon (default on Linux and macOS), it is the Lix daemon user account (that is, `root`) that should have SSH access to a user (not necessarily `root`) on the remote machine.
 >
 > Furthermore, `root` needs to have the public host keys for the remote system in its `.ssh/known_hosts`.
-> To add them to `known_hosts` for root, do `ssh-keyscan USER@HOST | sudo tee -a ~root/.ssh/known_hosts`.
+> To add them to `known_hosts` for root, do `ssh-keyscan HOST | sudo tee -a ~root/.ssh/known_hosts`.
 >
 > If you can’t or don’t want to configure `root` to be able to access the remote machine, you can use a private Nix store instead by passing e.g. `--store ~/my-nix` when running a Nix command from the local machine.
 
+
+## Configuration
+
 The list of remote machines can be specified on the command line or in
-the Lix configuration file. The former is convenient for testing. For
-example, the following command allows you to build a derivation for
-`x86_64-darwin` on a Linux machine:
-
-```console
-$ uname
-Linux
-
-$ nix build --impure \
-  --expr '(with import <nixpkgs> { system = "x86_64-darwin"; }; runCommand "foo" {} "uname > $out")' \
-  --builders 'ssh://mac x86_64-darwin'
-[1/0/1 built, 0.0 MiB DL] building foo on ssh://mac
-
-$ cat ./result
-Darwin
-```
-
-It is possible to specify multiple builders separated by a semicolon or
-a newline, e.g.
-
-```console
-  --builders 'ssh://mac x86_64-darwin ; ssh://beastie x86_64-freebsd'
-```
-
-Each machine specification consists of the following elements, separated
-by spaces. Only the first element is required. To leave a field at its
-default, set it to `-`.
-
-1.  The URI of the remote store in the format
-    `ssh://[username@]hostname[?port=<port>]`, e.g. `ssh://nix@mac` or `ssh://mac`.
-    If the ssh server is not listening on port 22 (e.g. port 1337 in this case)
-    the URI would be `ssh://nix@mac?port=1337`
-    For backward compatibility, `ssh://` may be omitted. The hostname
-    may be an alias defined in your `~/.ssh/config`.
-
-2.  A comma-separated list of Nix platform type identifiers, such as
-    `x86_64-darwin`. It is possible for a machine to support multiple
-    platform types, e.g., `i686-linux,x86_64-linux`. If omitted, this
-    defaults to the local platform type.
-
-3.  The SSH identity file to be used to log in to the remote machine. If
-    omitted, SSH will use its regular identities.
-
-4.  The maximum number of builds that Lix will execute in parallel on
-    the machine. Typically this should be equal to the number of CPU
-    cores. For instance, the machine `itchy` in the example will execute
-    up to 8 builds in parallel.
-
-5.  The “speed factor”, indicating the relative speed of the machine. If
-    there are multiple machines of the right type, Lix will prefer the
-    fastest, taking load into account.
-
-6.  A comma-separated list of *supported features*. If a derivation has
-    the `requiredSystemFeatures` attribute, then Lix will only perform
-    the derivation on a machine that has the specified features. For
-    instance, the attribute
-
-    ```nix
-    requiredSystemFeatures = [ "kvm" ];
-    ```
-
-    will cause the build to be performed on a machine that has the `kvm`
-    feature.
-
-7.  A comma-separated list of *mandatory features*. A machine will only
-    be used to build a derivation if all of the machine’s mandatory
-    features appear in the derivation’s `requiredSystemFeatures`
-    attribute.
-
-8.  The (base64-encoded) public host key of the remote machine. If omitted, SSH
-    will use its regular known-hosts file. Specifically, the field is calculated
-    via `base64 -w0 /etc/ssh/ssh_host_ed25519_key.pub`.
-
-For example, the machine specification
-
-    nix@scratchy.labs.cs.uu.nl  i686-linux      /home/nix/.ssh/id_scratchy_auto        8 1 kvm
-    nix@itchy.labs.cs.uu.nl     i686-linux      /home/nix/.ssh/id_scratchy_auto        8 2
-    nix@poochie.labs.cs.uu.nl   i686-linux      /home/nix/.ssh/id_scratchy_auto        1 2 kvm benchmark
-
-specifies several machines that can perform `i686-linux` builds.
-However, `poochie` will only do builds that have the attribute
-
-```nix
-requiredSystemFeatures = [ "benchmark" ];
-```
-
-or
-
-```nix
-requiredSystemFeatures = [ "benchmark" "kvm" ];
-```
-
-`itchy` cannot do builds that require `kvm`, but `scratchy` does support
-such builds. For regular builds, `itchy` will be preferred over
-`scratchy` because it has a higher speed factor.
+the Lix configuration file. The former is convenient for testing.
+Additionally, there are two supported formats to configure remote builders:
+The legacy, "space"-separated format and starting with Lix 2.95.0, a TOML.
 
 Remote builders can also be configured in `nix.conf`, e.g.
 
@@ -159,3 +70,180 @@ option `builders-use-substitutes` in your local `nix.conf`.
 
 To build only on remote builders and disable building on the local
 machine, you can use the option `--max-jobs 0`.
+
+---
+
+Each machine specification consists of the following attributes.
+How those are combined within the configuration file differs for the formats, and will be explained further down.
+
+1.  `uri` (**required**)
+    The URI of the remote store in the format
+    `ssh[-ng]://[username@]hostname[?port=<port>]`, e.g. `ssh://nix@mac` or `ssh://mac`.
+    If the ssh server is not listening on port 22 (e.g. port 1337 in this case)
+    the URI would be `ssh[-ng]://nix@mac?port=1337`. The hostname
+    may be an alias defined in your `~/.ssh/config`.
+
+2.  `system-types` (**optional**)
+    A list of Nix platform type identifiers, such as
+    `x86_64-darwin`. It is possible for a machine to support multiple
+    platform types, e.g., `i686-linux` and `x86_64-linux`.
+
+    Defaults to the local platform type
+
+3.  `ssh-key` (**optional**)
+    The SSH identity file to be used to log in to the remote machine.
+
+    Defaults to SSHs regular identities.
+
+4.  `jobs` (**optional**)
+    The maximum number of builds that Lix will execute in parallel on
+    the machine. Typically, this should be equal to the number of CPU
+    cores divided by the cores within the target machines configuration, i.e. `jobs * cores ~= cpu cores`
+
+    Defaults to 1; must be a positive integer.
+
+5.  `speed-factor`
+    The “speed factor”, indicating the relative speed of the machine. If
+    there are multiple machines of the right type, Lix will prefer the
+    fastest, taking load into account.
+
+    Defaults to 1; must be a positive float.
+
+6.  `supported-features` (**optional**)
+    A list of *supported features*. If a derivation has
+    the `requiredSystemFeatures` attribute, then Lix will only schedule
+    the derivation on a machine that has the specified features. For
+    example, the attribute
+
+    ```nix
+    requiredSystemFeatures = [ "kvm" ];
+    ```
+
+    will cause the build to be performed on a machine that has the `kvm`
+    feature.
+
+    Defaults to an empty list.
+
+7.  `mandatory-features` (**optional**)
+    A list of *mandatory features*. A machine will only
+    be used to build a derivation if all the machine’s mandatory
+    features appear in the derivation’s `requiredSystemFeatures`
+    attribute.
+
+    Defaults to an empty list.
+
+8.  `ssh-public-host-key` (**optional**)
+    The public host key of the remote machine.
+
+    Defaults to basic ssh behavior (checking contests of the known-hosts file)
+
+
+### Using a TOML configuration
+
+Each machine is configured as an attribute within the map called `machines`.
+The attributes name is the machines name.
+Attributes can be in any order.
+
+For example:
+
+```toml
+version = 1
+
+[machines.andesite]
+uri = "ssh://lix@andesite.lix.systems" # toml also allows for comments
+system-types = ["i686-linux"]
+jobs = 8
+speed-factor = 1.0
+supported-features = ["kvm"]
+ssh-key = "/home/deepslate/.ssh/id_ed25519"
+
+[machines.diorite]
+uri = "ssh://lix@diorite.lix.systems"
+system-types = ["i686-linux"]
+jobs = 8
+speed-factor = 2.0
+ssh-key = "/home/deepslate/.ssh/id_ed25519"
+
+[machines.granite]
+uri = "ssh://lix@granite.lix.systems"
+system-types = ["i686-linux"]
+jobs = 1
+speed-factor = 2.0
+supported-features = ["kvm", "benchmark"]
+ssh-key = "/home/deepslate/.ssh/id_ed25519"
+
+[machines.legacy]
+uri = "ssh://nix@nix-15-11.nixos.org"
+enable = false
+
+```
+
+> **Note**
+>
+> If the version tag is omitted (e.g. in the CLI), it defaults to the latest version.
+> It is strongly recommended to always provide a version tag for configuration within files to avoid breakage.
+
+For testing purposes, one can also define a builder ad hoc on the CLI as follows:
+`--builders 'machines.andesite = {uri = "ssh://lix@andesite.lix.systems", jobs = 8}'`
+
+
+#### Special handling of fields
+ - `enable` (**optional**)
+    If set to false, the declared machine will not be loaded.
+    This allows one to statically disable machines.
+
+    Defaults to true
+
+### Using the legacy format
+> **Warning**
+>
+> This format is frozen and new features / configuration options will not be backported to this format.
+
+It is possible to specify multiple builders separated by a semicolon or
+a newline, e.g.
+
+```console
+  --builders 'ssh://mac x86_64-darwin ; ssh://beastie x86_64-freebsd'
+```
+
+Every machine specification consists of the elements listed in the section above, seperated by any amount of spaces or tabs.
+The Attributes need to be provided **in order** and without names.
+To leave a field at its default, set it to `-`.
+Lists are colon seperated, without additional spaces.
+
+```
+lix@andesite.lix.systems    i686-linux      /home/deepslate/.ssh/id_ed25519        8 1 kvm
+lix@diorite.lix.systems     i686-linux      /home/deepslate/.ssh/id_ed25519        8 2
+lix@granite.lix.systems     i686-linux      /home/deepslate/.ssh/id_ed25519        1 2 kvm benchmark
+```
+
+#### Special handling of fields
+- `uri`: Due to backward compatibility, the `ssh://` may be omitted for the store-uri.
+- `ssh-public-host-key`: The key must be provided encoded in base64. Specifically calculated via `base64 -w0 /etc/ssh/ssh_host_ed25519_key.pub`
+
+
+### Format detection
+
+At first, the given configuration is being parsed syntactically as a toml.
+If parsing fails and the given configuration contains a `"` the error is presented to the user, as those characters are necessary for TOML, but disallowed for the legacy format.
+Otherwise, parsing is retried using the legacy format.
+If non-syntactic errors are detected within the toml, the exception will always be shown to the user directly.
+
+
+## Builder selection
+The configuration(s) above specify several machines that can perform `i686-linux` builds.
+However, `granite` will only do builds that have the attribute
+
+```nix
+requiredSystemFeatures = [ "benchmark" ];
+```
+
+or
+
+```nix
+requiredSystemFeatures = [ "benchmark" "kvm" ];
+```
+
+`diorite` cannot do builds that require `kvm`, but `andesite` does support
+such builds. For regular builds, `diorite` will be preferred over
+`andesite` because it has a higher speed factor.

doc/manual/src/contributing/hacking.md

@@ -141,6 +141,21 @@ To inspect the canonical source of truth on what the state of the buildsystem co
 $ meson introspect
 ```
 
+#### LLD
+
+The development shell on Linux uses LLD by default for faster link times.
+This is set using `mesonFlags`, so to override it, you can simplify re-specify the linker to Meson:
+
+```bash
+$ just setup-custom -Dc_link_args=-fuse-ld=ld -Dcpp_link_args=-fuse-ld=ld
+```
+
+While using LLD, you may find it helpful to use ThinLTO for even further improvements to link times for incremental builds:
+
+```bash
+$ just setup-custom -Db_lto=true -Db_lto_mode=thin -Db_thinlto_cache=true
+```
+
 ## Sending changes to Gerrit for review {#sending-to-gerrit}
 
 We use Gerrit for all our code review in Lix.

doc/manual/src/contributing/testing.md

@@ -401,7 +401,6 @@ I grepped `lix/` for `get[eE]nv\("` to find the mentions in Lix code.
 
   **Expected value**: the path to an executable shell
 - `PRINT_PATH` - Undocumented. Used by `nix-prefetch-url` as an alternative form of `--print-path`. Why???
-- `_NIX_IN_TEST` - If present with any value, makes `fetchClosure` accept file URLs in addition to HTTP ones. Why is this not `_NIX_FORCE_HTTP`??
 
   Not used anywhere else.
 - `NIX_ALLOW_EVAL` - Used by eval-cache tests to block evaluation if set to `0`.

doc/manual/src/package-management/basic-package-mgmt.md

@@ -5,7 +5,7 @@
 FIXME(Lix): This section does not document the most common modern practices in terms of avoiding channels, pinning, declarative software installation (see flakey-profile or home-manager or NixOS), or using flakes, etc.
 It is, however, likely correct at a technical level.
 
-For more information on modern practices, see the [resources](https://lix.systems/resources) page on the Lix site.
+For more information on modern practices, see the [resources](https://wiki.lix.systems/books/lix-users/page/nix-resources) page on the Lix site.
 
 </div>
 

doc/manual/substitute.py

@@ -70,10 +70,15 @@ def do_include(content: str, relative_md_path: Path, source_root: Path, search_p
 
 def recursive_replace(data, book_root, search_path):
     match data:
+        # XXX FUTURE: drop sections once mdBook is at 0.5.0 or above in nixpkgs
         case {'sections': sections}:
             return data | dict(
                 sections = [recursive_replace(section, book_root, search_path) for section in sections],
             )
+        case {'items': items}:
+            return data | dict(
+                items = [recursive_replace(item, book_root, search_path) for item in items],
+            )
         case {'Chapter': chapter}:
             path_to_chapter = Path(chapter['path'])
             chapter_content = chapter['content']
@@ -119,10 +124,10 @@ def main():
     context, book = json.load(sys.stdin)
 
     # book_root is the directory where book contents leave (ie, src/)
-    book_root = Path(context['root']) / context['config']['book']['src']
+    book_root = Path(context['root']) / context['config']['book'].get('src', 'src')
 
     # includes pointing into @generated@ will look here
-    search_path = Path(os.environ['MDBOOK_SUBSTITUTE_SEARCH'])
+    search_path = Path(os.environ['MANUAL_SUBSTITUTE_SEARCH'])
 
     # Find @var@ in all parts of our recursive book structure.
     replaced_content = recursive_replace(book, book_root, search_path)

flake.nix

@@ -222,9 +222,13 @@
             busybox-sandbox-shell = final.busybox-sandbox-shell or final.default-busybox-sandbox-shell;
           };
 
-          lix-clang-tidy = final.callPackage ./subprojects/lix-clang-tidy { };
+          lix-clang-tidy = final.callPackage ./subprojects/lix-clang-tidy {
+            # FIXME: To be removed when switching to nixos-25.11-small
+            llvmPackages = final.llvmPackages_20;
+          };
 
           nix-eval-jobs = final.callPackage ./subprojects/nix-eval-jobs {
+            stdenv = currentStdenv;
             srcDir = ./subprojects/nix-eval-jobs;
           };
 
@@ -258,7 +262,9 @@
               sha512 = "2a4d0rqh8gkw4ca3gkzddp0hjpmmw74cbks8k0inhh0vizmgbn188zdv6m1kgmr019b99g7insli8js3ci1ji7y4n5nk704bswf3z3i";
             };
             nativeBuildInputs = prevAttrs.nativeBuildInputs ++ [ final.buildPackages.bmake ];
-            postInstall = lib.replaceStrings [ "lowdown.so.1" ] [ "lowdown.so.2" ] prevAttrs.postInstall;
+            postInstall = lib.replaceStrings [ "lowdown.so.1" ] [ "lowdown.so.2" ] (
+              prevAttrs.postInstall or ""
+            );
           });
 
           capnproto = prev.capnproto.overrideAttrs (old: {
@@ -427,17 +433,20 @@
               '';
 
             # clang-tidy run against the Lix codebase using the Lix clang-tidy plugin
-            clang-tidy =
+            clang-tidy = forAllSystems (
+              system:
               let
-                nixpkgs = nixpkgsFor.x86_64-linux.native;
-                inherit (nixpkgs) pkgs;
+                pkgs = nixpkgsFor.${system}.native;
               in
               pkgs.callPackage ./package.nix {
                 # Required since we don't support gcc stdenv
                 stdenv = pkgs.clangStdenv;
+                # FIXME: To be removed when switching to nixos-25.11-small
+                llvmPackages = pkgs.llvmPackages_20;
                 versionSuffix = "";
                 lintInsteadOfBuild = true;
-              };
+              }
+            );
 
             # Make sure that nix-env still produces the exact same result
             # on a particular version of Nixpkgs.
@@ -582,6 +591,8 @@
                 inherit stdenv versionSuffix;
                 busybox-sandbox-shell = pkgs.busybox-sandbox-shell or pkgs.default-busybox-sandbox;
                 internalApiDocs = false;
+                # Use LLD in the dev shell by default for faster link times.
+                useLld = stdenv.hostPlatform.isLinux;
               };
               pre-commit = self.hydraJobs.pre-commit.${pkgs.system} or { };
             in

lix/legacy/build-remote.cc

@@ -527,7 +527,7 @@ kj::Promise<void> AcceptedBuild::runImpl(RunContext context)
                 AIO().timeoutAfter(15 * kj::MINUTES, lockFileAsync(uploadLock.get(), ltWrite))
             );
             if (!result) {
-                printError("somebody is hogging the upload lock for '%s', continuing...");
+                printError("somebody is hogging the upload lock for '%s', continuing...", storeUri);
             }
         }
 

lix/legacy/nix-build.cc

@@ -25,6 +25,7 @@
 #include "lix/libutil/shlex.hh"
 #include "nix-build.hh"
 #include "lix/libstore/temporary-dir.hh"
+#include "lix/libutil/strings.hh"
 
 extern char * * environ __attribute__((weak)); // Man what even is this
 
@@ -191,7 +192,11 @@ static int main_nix_build(AsyncIoRoot & aio, std::string programName, Strings ar
         throw UsageError("'-p' and '-E' are mutually exclusive");
 
     AutoDelete tmpDir(createTempDir(myName));
-    AutoDelete buildTopTmpDir(createTempSubdir(tmpDir, "build-top"));
+    // NOTE: we assume there's no `build-top` directory created inside of `tmpDir` and we have
+    // ownership of this.
+    auto buildTopTmpDir = tmpDir + "/build-top";
+    createDirs(buildTopTmpDir);
+
     if (outLink.empty())
         outLink = (Path) tmpDir + "/result";
 
@@ -425,6 +430,18 @@ static int main_nix_build(AsyncIoRoot & aio, std::string programName, Strings ar
             env["__ETC_PROFILE_SOURCED"] = "1";
         }
 
+        // Set NIX_SHELL_LEVEL
+        env["NIX_SHELL_LEVEL"] = std::to_string(
+            getEnvNonEmpty("NIX_SHELL_LEVEL")
+                .and_then([](std::string lvl) { return string2Int<size_t>(lvl); })
+                .value_or(0)
+            + 1
+        );
+
+        // We re-export similarly to what occurs inside of a derivation goal `NIX_LOG_FD` to stderr.
+        // So that stdenv hooks that logs information can be observed inside this debugging tool.
+        env["NIX_LOG_FD"] = "2";
+
         // Don't use defaultTempDir() here! We want to preserve the user's TMPDIR for the shell
         env["NIX_BUILD_TOP"] = env["TMPDIR"] = env["TEMPDIR"] = env["TMP"] = env["TEMP"] =
             getEnvNonEmpty("TMPDIR").value_or(buildTopTmpDir);

lix/legacy/nix-collect-garbage.cc

@@ -64,7 +64,7 @@ static int main_nix_collect_garbage(AsyncIoRoot & aio, std::string programName,
     {
         bool removeOld = false;
 
-        GCOptions options;
+        GCOptions options = {.action = GCOptions::gcDeleteDead};
 
         LegacyArgs(aio, programName, [&](Strings::iterator & arg, const Strings::iterator & end) {
             if (*arg == "--help")
@@ -75,12 +75,13 @@ static int main_nix_collect_garbage(AsyncIoRoot & aio, std::string programName,
             else if (*arg == "--delete-older-than") {
                 removeOld = true;
                 deleteOlderThan = getArg(*arg, arg, end);
-            }
-            else if (*arg == "--dry-run") dryRun = true;
-            else if (*arg == "--max-freed")
+            } else if (*arg == "--dry-run") {
+                options.action = GCOptions::gcReturnDead;
+            } else if (*arg == "--max-freed") {
                 options.maxFreed = std::max(getIntArg<int64_t>(*arg, arg, end, true), (int64_t) 0);
-            else
+            } else {
                 return false;
+            }
             return true;
         }).parseCmdline(argv);
 
@@ -92,24 +93,12 @@ static int main_nix_collect_garbage(AsyncIoRoot & aio, std::string programName,
         }
 
         // Run the actual garbage collector.
-        if (!dryRun) {
-            options.action = GCOptions::gcDeleteDead;
-        } else {
-            options.action = GCOptions::gcReturnDead;
-        }
         auto store = aio.blockOn(openStore());
         auto & gcStore = require<GcStore>(*store);
         GCResults results;
-        PrintFreed freed(true, results);
+        PrintFreed freed(options.action, results);
         aio.blockOn(gcStore.collectGarbage(options, results));
 
-        if (dryRun) {
-            // Only print results for dry run; when !dryRun, paths will be printed as they're deleted.
-            for (auto & i : results.paths) {
-                printInfo("%s", Uncolored(i));
-            }
-        }
-
         return 0;
     }
 }

lix/legacy/nix-store.cc

@@ -711,12 +711,8 @@ static void opGC(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlag
     }
 
     else {
-        PrintFreed freed(options.action == GCOptions::gcDeleteDead, results);
+        PrintFreed freed(options.action, results);
         aio.blockOn(gcStore.collectGarbage(options, results));
-
-        if (options.action != GCOptions::gcDeleteDead)
-            for (auto & i : results.paths)
-                cout << i << std::endl;
     }
 }
 
@@ -749,7 +745,7 @@ opDelete(std::shared_ptr<Store> store, AsyncIoRoot & aio, Strings opFlags, Strin
     auto & gcStore = require<GcStore>(*store);
 
     GCResults results;
-    PrintFreed freed(true, results);
+    PrintFreed freed(options.action, results);
     aio.blockOn(gcStore.collectGarbage(options, results));
 }
 

lix/libcmd/command.cc

@@ -75,6 +75,15 @@ CopyCommand::CopyCommand()
     });
 }
 
+void CopyCommand::run()
+{
+    if (requireStore && srcUri.empty() && dstUri.empty()) {
+        throw UsageError("you must pass '--from' and/or '--to'");
+    }
+
+    StoreCommand::run();
+}
+
 ref<Store> CopyCommand::createStore(AsyncIoRoot & in)
 {
     return srcUri.empty() ? StoreCommand::createStore(in) : in.blockOn(openStore(srcUri));
@@ -82,9 +91,6 @@ ref<Store> CopyCommand::createStore(AsyncIoRoot & in)
 
 ref<Store> CopyCommand::getDstStore()
 {
-    if (srcUri.empty() && dstUri.empty())
-        throw UsageError("you must pass '--from' and/or '--to'");
-
     return aio().blockOn(dstUri.empty() ? openStore() : openStore(dstUri));
 }
 

lix/libcmd/command.hh

@@ -15,8 +15,6 @@ namespace nix {
 
 extern std::string programPath;
 
-extern char * * savedArgv;
-
 class EvalState;
 struct Pos;
 class Store;
@@ -56,9 +54,11 @@ private:
 struct CopyCommand : virtual StoreCommand
 {
     std::string srcUri, dstUri;
+    bool requireStore = true;
 
     CopyCommand();
 
+    void run() override;
     ref<Store> createStore(AsyncIoRoot & in) override;
 
     ref<Store> getDstStore();

lix/libcmd/legacy.hh

@@ -5,18 +5,32 @@
 #include <functional>
 #include <list>
 #include <map>
+#include <span>
 #include <string>
 
 namespace nix {
 
-typedef std::function<int(AsyncIoRoot &, std::string, std::list<std::string>)> MainFunction;
-
 struct LegacyCommandRegistry
 {
-    using LegacyCommandMap = std::map<std::string, MainFunction>;
+    typedef std::function<int(AsyncIoRoot &, std::string, std::list<std::string>)> MainFunction;
+    typedef std::function<
+        int(AsyncIoRoot &, std::string, std::list<std::string>, std::span<char *>)>
+        RawMainFunction;
+
+    using LegacyCommandMap = std::map<std::string, RawMainFunction>;
     static LegacyCommandMap * commands;
 
     static void add(const std::string & name, MainFunction fun)
+    {
+        addWithRaw(
+            name,
+            [fun](AsyncIoRoot & aio, std::string name, std::list<std::string> args, std::span<char *>) {
+                return fun(aio, name, args);
+            }
+        );
+    }
+
+    static void addWithRaw(const std::string & name, RawMainFunction fun)
     {
         if (!commands) commands = new LegacyCommandMap;
         (*commands)[name] = fun;

lix/libcmd/repl-interacter.cc

@@ -48,7 +48,7 @@ char ** copyCompletions(const StringSet& possible)
             if (vp) {
                 while (--ac >= 0)
                     free(vp[ac]);
-                free(vp);
+                free(static_cast<void *>(vp));
             }
             throw Error("allocation failure");
         }

lix/libcmd/repl-interacter.hh

@@ -46,7 +46,7 @@ public:
      *
      * This function logs but ignores errors from readline's write_history().
      */
-    virtual void writeHistory();
+    void writeHistory();
     virtual ~ReadlineLikeInteracter() override;
 };
 

lix/libcmd/repl.cc

@@ -209,12 +209,13 @@ struct NixRepl
 
     void printValue(std::ostream & str,
                               Value & v,
-                              unsigned int maxDepth = std::numeric_limits<unsigned int>::max())
+                              unsigned int maxDepth = std::numeric_limits<unsigned int>::max(),
+                              unsigned int derivationPathDepth = 0)
     {
         ::nix::printValue(state, str, v, PrintOptions {
             .ansiColors = true,
             .force = true,
-            .derivationPaths = true,
+            .derivationPathDepth = derivationPathDepth,
             .maxDepth = maxDepth,
             .prettyIndent = 2,
             .errors = ErrorPrintBehavior::ThrowTopLevel,
@@ -793,8 +794,10 @@ ProcessLineResult NixRepl::processLine(std::string line)
         evalString(arg, v);
         if (v.type() == nString) {
             std::cout << v.str();
+        } else if (v.type() == nAttrs && state.isDerivation(v)) {
+            printValue(std::cout, v, 2, 1);
         } else {
-            printValue(std::cout, v);
+            printValue(std::cout, v, std::numeric_limits<unsigned int>::max(), 0);
         }
         std::cout << std::endl;
     }

lix/libexpr/attr-set.hh

@@ -97,6 +97,8 @@ public:
         res.reserve(size_);
         for (Size n = 0; n < size_; n++)
             res.emplace_back(&attrs[n]);
+        // NOTE: this sort uses a total order, so the iteration over pointers is not an issue
+        // NOLINTNEXTLINE(bugprone-nondeterministic-pointer-iteration-order)
         std::sort(res.begin(), res.end(), [&](const Attr * a, const Attr * b) {
             std::string_view sa = symbols[a->name], sb = symbols[b->name];
             return sa < sb;
@@ -119,7 +121,6 @@ public:
 
 private:
     Bindings * bindings;
-    EvalMemory & mem;
     SymbolTable & symbols;
     Size capacity;
 
@@ -127,9 +128,8 @@ public:
     // needed by std::back_inserter
     using value_type = Attr;
 
-    BindingsBuilder(EvalMemory & mem, SymbolTable & symbols, Bindings * bindings, Size capacity)
+    BindingsBuilder(SymbolTable & symbols, Bindings * bindings, Size capacity)
         : bindings(bindings)
-        , mem(mem)
         , symbols(symbols)
         , capacity(capacity)
     {

lix/libexpr/builtins/addErrorContext.md

@@ -0,0 +1,96 @@
+---
+name: addErrorContext
+args: [message, expr]
+---
+
+This adds a `message` to be shown in the stacktrace in the event of
+a failure during the evaluation of `expr`.
+
+For example, if a file `err.nix` contains the following:
+
+```nix
+let
+  countDown =
+    n:
+    if n == 0 then
+      throw "kaboom"
+    else
+      builtins.addErrorContext "while counting down; n = ${toString n}" ("x" + countDown (n - 1));
+in
+countDown 2
+```
+
+Then, evaluating the file will give the following stack trace:
+
+```console
+$ nix-instantiate --show-trace err.nix
+error:
+       … from call site
+         at /home/plop/git.lix.systems/lix-project/lix/err.nix:9:1:
+            8| in
+            9| countDown 2
+             | ^
+           10|
+
+       … while calling 'countDown'
+         at /home/plop/git.lix.systems/lix-project/lix/err.nix:3:5:
+            2|   countDown =
+            3|     n:
+             |     ^
+            4|     if n == 0 then
+
+       … while calling the 'addErrorContext' builtin
+         at /home/plop/git.lix.systems/lix-project/lix/err.nix:7:7:
+            6|     else
+            7|       builtins.addErrorContext "while counting down; n = ${toString n}" ("x" + countDown (n - 1));
+             |       ^
+            8| in
+
+       … while counting down; n = 2
+
+       … from call site
+         at /home/plop/git.lix.systems/lix-project/lix/err.nix:7:80:
+            6|     else
+            7|       builtins.addErrorContext "while counting down; n = ${toString n}" ("x" + countDown (n - 1));
+             |                                                                                ^
+            8| in
+
+       … while calling 'countDown'
+         at /home/plop/git.lix.systems/lix-project/lix/err.nix:3:5:
+            2|   countDown =
+            3|     n:
+             |     ^
+            4|     if n == 0 then
+
+       … while calling the 'addErrorContext' builtin
+         at /home/plop/git.lix.systems/lix-project/lix/err.nix:7:7:
+            6|     else
+            7|       builtins.addErrorContext "while counting down; n = ${toString n}" ("x" + countDown (n - 1));
+             |       ^
+            8| in
+
+       … while counting down; n = 1
+
+       … from call site
+         at /home/plop/git.lix.systems/lix-project/lix/err.nix:7:80:
+            6|     else
+            7|       builtins.addErrorContext "while counting down; n = ${toString n}" ("x" + countDown (n - 1));
+             |                                                                                ^
+            8| in
+
+       … while calling 'countDown'
+         at /home/plop/git.lix.systems/lix-project/lix/err.nix:3:5:
+            2|   countDown =
+            3|     n:
+             |     ^
+            4|     if n == 0 then
+
+       … caused by explicit throw
+         at /home/plop/git.lix.systems/lix-project/lix/err.nix:5:7:
+            4|     if n == 0 then
+            5|       throw "kaboom"
+             |       ^
+            6|     else
+
+       error: kaboom
+```

lix/libexpr/builtins/appendContext.md

@@ -0,0 +1,6 @@
+---
+name: appendContext
+args: [s, ctx]
+---
+
+Appends the attribute set `ctx` as a context to the string `s`, see [`getContext`](#builtins-getContext) for details on the format of the context.

lix/libexpr/builtins/derivationStrict.md

@@ -0,0 +1,22 @@
+---
+name: derivationStrict
+args: [args]
+renameInGlobalScope: false
+---
+
+Constructs a [store derivation](../glossary.md#gloss-store-derivation) from the attribute set `args`
+(c.f. [derivation](#builtins-derivation). Unlike `derivation` the produced store derivation is placed
+in the store *immediately* when this builtin is called, while `derivation` may defer placing store
+derivations in the store until it is proven that they are used.
+
+It then returns a new attrset with *only* the following attributes:
+
+- `drvPath` containing the path of the store derivation;
+- For each output of the derivation (`out`, `dev`, etc): an attribute named after that output containing the output path
+
+> **Note**
+>
+> In contrast to [`builtins.derivation`](#builtins-derivation), this computes
+> the derivation set in a fully *strict* manner, i.e. the values of the attributes
+> directly computed, whereas using `builtins.derivation` will produce an attrset
+> whose values will be evaluated when they are used at a later point.

lix/libexpr/builtins/fetchClosure.md

@@ -1,72 +0,0 @@
----
-name: fetchClosure
-args: [args]
-experimentalFeature: fetch-closure
----
-Fetch a store path [closure](@docroot@/glossary.md#gloss-closure) from a binary cache, and return the store path as a string with context.
-
-This function can be invoked in three ways, that we will discuss in order of preference.
-
-**Fetch a content-addressed store path**
-
-Example:
-
-```nix
-builtins.fetchClosure {
-  fromStore = "https://cache.nixos.org";
-  fromPath = /nix/store/ldbhlwhh39wha58rm61bkiiwm6j7211j-git-2.33.1;
-}
-```
-
-This is the simplest invocation, and it does not require the user of the expression to configure [`trusted-public-keys`](@docroot@/command-ref/conf-file.md#conf-trusted-public-keys) to ensure their authenticity.
-
-If your store path is [input addressed](@docroot@/glossary.md#gloss-input-addressed-store-object) instead of content addressed, consider the other two invocations.
-
-**Fetch any store path and rewrite it to a fully content-addressed store path**
-
-Example:
-
-```nix
-builtins.fetchClosure {
-  fromStore = "https://cache.nixos.org";
-  fromPath = /nix/store/r2jd6ygnmirm2g803mksqqjm4y39yi6i-git-2.33.1;
-  toPath = /nix/store/ldbhlwhh39wha58rm61bkiiwm6j7211j-git-2.33.1;
-}
-```
-
-This example fetches `/nix/store/r2jd...` from the specified binary cache,
-and rewrites it into the content-addressed store path
-`/nix/store/ldbh...`.
-
-Like the previous example, no extra configuration or privileges are required.
-
-To find out the correct value for `toPath` given a `fromPath`,
-use [`nix store make-content-addressed`](@docroot@/command-ref/new-cli/nix3-store-make-content-addressed.md):
-
-```console
-# nix store make-content-addressed --from https://cache.nixos.org /nix/store/r2jd6ygnmirm2g803mksqqjm4y39yi6i-git-2.33.1
-rewrote '/nix/store/r2jd6ygnmirm2g803mksqqjm4y39yi6i-git-2.33.1' to '/nix/store/ldbhlwhh39wha58rm61bkiiwm6j7211j-git-2.33.1'
-```
-
-Alternatively, set `toPath = ""` and find the correct `toPath` in the error message.
-
-**Fetch an input-addressed store path as is**
-
-Example:
-
-```nix
-builtins.fetchClosure {
-  fromStore = "https://cache.nixos.org";
-  fromPath = /nix/store/r2jd6ygnmirm2g803mksqqjm4y39yi6i-git-2.33.1;
-  inputAddressed = true;
-}
-```
-
-It is possible to fetch an [input-addressed store path](@docroot@/glossary.md#gloss-input-addressed-store-object) and return it as is.
-However, this is the least preferred way of invoking `fetchClosure`, because it requires that the input-addressed paths are trusted by the Lix configuration.
-
-**`builtins.storePath`**
-
-`fetchClosure` is similar to [`builtins.storePath`](#builtins-storePath) in that it allows you to use a previously built store path in a Nix expression.
-However, `fetchClosure` is more reproducible because it specifies a binary cache from which the path can be fetched.
-Also, using content-addressed store paths does not require users to configure [`trusted-public-keys`](@docroot@/command-ref/conf-file.md#conf-trusted-public-keys) to ensure their authenticity.

lix/libexpr/builtins/fetchMercurial.md

@@ -0,0 +1,48 @@
+---
+name: fetchMercurial
+args: [args]
+renameInGlobalScope: false
+---
+
+Fetch a Mercurial repository. *args* can be a URL, in which case the default
+branch of the repo at that URL is fetched. Otherwise, it can be an
+attribute with the following attributes (all except `url` optional):
+
+- `url`
+
+  The URL of the repo.
+
+- `name` (default: `"source"`)
+
+  The name of the directory the repo should be exported to in the store.
+
+- `rev`
+
+  The revision to fetch.
+  This is typically a commit hash.
+
+  > **Note**
+  >
+  > Currently, `rev` can either contain a revision or a branch/tag name.
+
+The return value is an attrset containing the following keys:
+
+- `outPath` (`string`)
+
+  Resulting store path of the fetch process.
+
+- `branch` (`string`)
+
+  The branch of the fetch repository.
+
+- `rev` (`string`)
+
+  The revision that was fetched.
+
+- `revCount` (`int`)
+
+  The number of revsets for this branch.
+
+- `shortRev` (`string`)
+
+  The first *12* characters of `rev`.

lix/libexpr/builtins/scopedImport.md

@@ -0,0 +1,59 @@
+---
+name: scopedImport
+implementation: "[](EvalState & state, Value ** args, Value & v) { import(state, *args[1], args[0], v); }"
+args: [scope, path]
+renameInGlobalScope: false
+---
+
+> **Warning**
+>
+> This builtin's use is heavily discouraged, it has many drawbacks and may be removed
+> in a future version of Lix.
+
+Functions like [`import`](#builtins-import) with the exception that
+it takes a `scope`, which is a set of attributes to be added to the
+lexical scope of the expression.
+
+This essentially allows overriding the ambient builtin variables.
+
+For example, if `foo.nix` is a file containing the following content:
+
+```nix
+x
+```
+
+then the following expression
+
+```nix
+scopedImport { x = 1; } ./foo.nix
+```
+
+will evaluate to `1`.
+
+Another application is overriding builtin functions or constants, e.g. to
+trace all calls to `map`, one can do:
+
+```nix
+let
+  overrides = {
+    map = f: xs: builtins.trace "map called!" (map f xs);
+
+    # Ensure that our override gets propagated by calls to
+    # import/scopedImport.
+    import = fn: scopedImport overrides fn;
+
+    scopedImport = attrs: fn: scopedImport (overrides // attrs) fn;
+
+    # Also update ‘builtins’.
+    builtins = builtins // overrides;
+  };
+in scopedImport overrides ./bla.nix
+```
+
+Similarly, it can be used to extend the set of builtin functions.
+
+> **Warning**
+>
+> One of the downsides of `scopedImport` is that it bypasses the evaluation cache.
+> This means that importing a file multiple times will lead to multiple expensive
+> parsings and evaluations.

lix/libexpr/builtins/storePath.md

@@ -13,5 +13,3 @@ in a new path (e.g. `/nix/store/ld01dnzc…-source-source`).
 
 Not available in [pure evaluation mode](@docroot@/command-ref/conf-file.md#conf-pure-eval).
 Lix may change this, tracking issue: <https://git.lix.systems/lix-project/lix/issues/402>
-
-See also [`builtins.fetchClosure`](#builtins-fetchClosure).

lix/libexpr/builtins/unsafeDiscardStringContext.md

@@ -0,0 +1,33 @@
+---
+name: unsafeDiscardStringContext
+args: [s]
+---
+
+Returns a copy of the string `s` with all string context associated with `s` removed.
+Since string context is used for dependency tracking the returned string will also have
+*no dependencies* on store objects, even when the original string `s` had such dependencies.
+
+This function is mainly useful when discarding dependencies is explicitly required, e.g.
+to produce a string listing all inputs of a derivation without propagating these inputs
+as dependencies into all *users* of the listing. For example, the derivation `dep` in the
+following example will pull `hello` into its closure despite never using it while `nodep`
+will not:
+
+```nix
+dep = runCommand "dep" {
+  inherit hello;
+} "echo hello is at $hello >$out";
+
+nodep = runCommand "nodep" {
+  hello = builtins.unsafeDiscardStringContext hello;
+} "echo hello is at $hello >$out";
+```
+
+This behavior also makes this function unsafe: if `s` contains the path of a
+store object that is not present in the store then any use of `s` in a
+derivation tree will attempt to realize that path in the store, but no use
+of `unsafeDiscardStringContext s` will. This can lead to derivation outputs that
+refer to paths that were never created.
+
+Lix cannot determine whether such reference are safe or not and must pass
+this obligation to the user.

lix/libexpr/builtins/warn.md

@@ -0,0 +1,7 @@
+---
+name: warn
+args: [msg, e2]
+---
+Evaluate string *msg* and print it on standard error. Then return *e2*.
+This function is useful for warning about unexpected conditions without aborting evaluation.
+If the [`debugger-on-warn`](@docroot@/command-ref/conf-file.md#conf-debugger-on-trace) option is set to `true` and the `--debugger` flag is given, the interactive debugger will be started when `warn` is called (like [`break`](@docroot@/language/builtins.md#builtins-break)).

lix/libexpr/eval.cc

@@ -4,12 +4,14 @@
 #include "lix/libutil/archive.hh"
 #include "lix/libutil/ansicolor.hh"
 #include "lix/libutil/async.hh"
+#include "lix/libutil/deprecated-features.hh"
 #include "lix/libutil/error.hh"
 #include "lix/libutil/english.hh"
 #include "lix/libutil/fmt.hh"
 #include "lix/libexpr/primops.hh"
 #include "lix/libexpr/print-options.hh"
 #include "lix/libmain/shared.hh"
+#include "lix/libutil/logging.hh"
 #include "lix/libutil/suggestions.hh"
 #include "lix/libutil/types.hh"
 #include "lix/libstore/store-api.hh"
@@ -208,9 +210,9 @@ void initLibExpr()
        (resident) memory to be allocated.  This might be a problem on
        systems that don't overcommit. */
     if (!getEnv("GC_INITIAL_HEAP_SIZE")) {
-        int64_t size = 32 * 1024 * 1024;
+        int64_t size = 32l * 1024 * 1024;
 #if HAVE_SYSCONF && defined(_SC_PAGESIZE) && defined(_SC_PHYS_PAGES)
-        int64_t maxSize = 384 * 1024 * 1024;
+        int64_t maxSize = 384l * 1024 * 1024;
         int64_t pageSize = sysconf(_SC_PAGESIZE);
         int64_t pages = sysconf(_SC_PHYS_PAGES);
         if (pageSize != -1) {
@@ -289,14 +291,14 @@ EvalMemory::EvalMemory()
 {
     assert(libexprInitialised);
 #if HAVE_BOEHMGC
-    GC_add_roots(gcCache, gcCache + CACHES);
+    GC_add_roots(static_cast<void *>(gcCache), static_cast<void *>(gcCache + CACHES));
 #endif
 }
 
 EvalMemory::~EvalMemory()
 {
 #if HAVE_BOEHMGC
-    GC_remove_roots(gcCache, gcCache + CACHES);
+    GC_remove_roots(static_cast<void *>(gcCache), static_cast<void *>(gcCache + CACHES));
 #endif
 }
 
@@ -347,6 +349,61 @@ EvalPaths::EvalPaths(
                 allowPath(path);
         }
     }
+
+#if LIX_MAJOR >= 3 || (LIX_MAJOR == 2 && LIX_MINOR >= 96)
+#warning \
+    "The feature nix-path-shadow was deprecated in 2.95 with a warning, which needs to be turned into an error in 2.96"
+#endif
+    if (!featureSettings.isEnabled(DeprecatedFeature::NixPathShadow)) {
+        for (auto & [prefix, path] : searchPath_.elements) {
+            // Match on the 'nix' prefix
+            if (prefix.s == "nix") {
+                logWarning(
+                    {.msg = HintFmt(
+                         "The prefix '%s' is reserved for internal use by Lix in the Nix search "
+                         "path, its usage is deprecated and will be forbidden in the future.\n"
+                         "Use %s to silence this warning.\n"
+                         "This is due to adding '%s=%s' in the Nix search path, either through the "
+                         "environment variable '%s' or by passing the flag %s to the nix "
+                         "invocation.",
+                         "nix",
+                         "--extra-deprecated-features nix-path-shadow",
+                         prefix.s,
+                         path.s,
+                         "NIX_PATH",
+                         "-I"
+                     )}
+                );
+            } else
+                // Match prefixless paths that contain a `nix` directory
+                if (auto res =
+                        prefix.suffixIfPotentialMatch("nix").and_then([&](std::string_view s) {
+                            return aio.blockOn(resolveSearchPathPath(path))
+                                .and_then([&](std::string r) {
+                                    Path res = s.length() ? concatStrings(r, "/", s): r;
+
+                                    return pathExists(res) ? std::optional(res) : std::nullopt;
+                                });
+                        }))
+                {
+                    logWarning(
+                        {.msg = HintFmt(
+                             "Shadowing '%s' by configuring the nix-path is deprecated and "
+                             "will be forbidden in the future.\n"
+                             "Use %s to silence this warning.\n"
+                             "This is due to adding '%s' to the nix-path without a prefix, "
+                             "either by passing the flag '-I %s' to the nix invocation or by "
+                             "adding this path to the environment variable '%s'.",
+                             "<nix/...>",
+                             "--extra-deprecated-features nix-path-shadow",
+                             path.s,
+                             path.s,
+                             "NIX_PATH"
+                         )}
+                    );
+                }
+        }
+    }
 }
 
 Evaluator::Evaluator(
@@ -2924,6 +2981,15 @@ Expr & Evaluator::parseStdin()
     );
 }
 
+std::optional<DebugTrace const *> Evaluator::nextDebugTrace() const
+{
+    if (!debug) {
+        return std::nullopt;
+    }
+
+    return debug->traces().next();
+}
+
 
 kj::Promise<Result<EvalPaths::PathResult<SourcePath, ThrownError>>>
 EvalPaths::findFile(const std::string_view path)

lix/libexpr/eval.hh

@@ -216,7 +216,7 @@ public:
 
     BindingsBuilder buildBindings(SymbolTable & symbols, size_t capacity)
     {
-        return BindingsBuilder(*this, symbols, allocBindings(capacity), capacity);
+        return BindingsBuilder(symbols, allocBindings(capacity), capacity);
     }
 
     const Statistics getStats() const { return stats; }
@@ -568,6 +568,9 @@ public:
      */
     void evalLazily(Expr & e, Value & v);
 
+    /** If debugging is enabled, returns the next trace. Otherwise, std::nullopt. */
+    std::optional<DebugTrace const *> nextDebugTrace() const;
+
 private:
     Expr * parse(
         char * text,

lix/libexpr/extra-primops.hh

@@ -9,15 +9,17 @@ class EvalState;
 struct Value;
 
 void prim_addDrvOutputDependencies(EvalState & state, Value * * args, Value & v);
-void prim_fetchClosure(EvalState & state, Value * * args, Value & v);
 void prim_fetchTree(EvalState & state, Value * * args, Value & v);
 void prim_fetchGit(EvalState & state, Value * * args, Value & v);
+void prim_fetchMercurial(EvalState & state, Value ** args, Value & v);
 void prim_fetchTarball(EvalState & state, Value * * args, Value & v);
 void prim_fetchurl(EvalState & state, Value * * args, Value & v);
 void prim_fromTOML(EvalState & state, Value * * args, Value & v);
+void prim_appendContext(EvalState & state, Value ** args, Value & v);
 void prim_getContext(EvalState & state, Value * * args, Value & v);
 void prim_hasContext(EvalState & state, Value * * args, Value & v);
 void prim_unsafeDiscardOutputDependency(EvalState & state, Value * * args, Value & v);
+void prim_unsafeDiscardStringContext(EvalState & state, Value ** args, Value & v);
 
 namespace flake {
 

lix/libexpr/meson.build

@@ -15,10 +15,12 @@ subdir('flake')
 
 libexpr_setting_definitions = files(
   # keep-sorted start
+  'settings/abort-on-warn.md',
   'settings/allow-import-from-derivation.md',
   'settings/allow-unsafe-native-code-during-evaluation.md',
   'settings/allowed-uris.md',
   'settings/debugger-on-trace.md',
+  'settings/debugger-on-warn.md',
   'settings/eval-cache.md',
   'settings/eval-system.md',
   'settings/ignore-try.md',
@@ -51,8 +53,10 @@ builtin_definitions = files(
   'builtins/abort.md',
   'builtins/add.md',
   'builtins/addDrvOutputDependencies.md',
+  'builtins/addErrorContext.md',
   'builtins/all.md',
   'builtins/any.md',
+  'builtins/appendContext.md',
   'builtins/attrNames.md',
   'builtins/attrValues.md',
   'builtins/baseNameOf.md',
@@ -67,12 +71,13 @@ builtin_definitions = files(
   'builtins/concatMap.md',
   'builtins/concatStringsSep.md',
   'builtins/deepSeq.md',
+  'builtins/derivationStrict.md',
   'builtins/dirOf.md',
   'builtins/div.md',
   'builtins/elem.md',
   'builtins/elemAt.md',
-  'builtins/fetchClosure.md',
   'builtins/fetchGit.md',
+  'builtins/fetchMercurial.md',
   'builtins/fetchTarball.md',
   'builtins/fetchTree.md',
   'builtins/fetchurl.md',
@@ -126,6 +131,7 @@ builtin_definitions = files(
   'builtins/readFileType.md',
   'builtins/removeAttrs.md',
   'builtins/replaceStrings.md',
+  'builtins/scopedImport.md',
   'builtins/seq.md',
   'builtins/sort.md',
   'builtins/split.md',
@@ -146,7 +152,9 @@ builtin_definitions = files(
   'builtins/tryEval.md',
   'builtins/typeOf.md',
   'builtins/unsafeDiscardOutputDependency.md',
+  'builtins/unsafeDiscardStringContext.md',
   'builtins/unsafeGetAttrPos.md',
+  'builtins/warn.md',
   'builtins/zipAttrsWith.md',
   # keep-sorted end
 )
@@ -219,7 +227,6 @@ libexpr_sources = files(
   'parser/parser.cc',
   'primops.cc',
   'primops/context.cc',
-  'primops/fetchClosure.cc',
   'primops/fetchMercurial.cc',
   'primops/fetchTree.cc',
   'primops/fromTOML.cc',

lix/libexpr/nixexpr.cc

@@ -35,6 +35,24 @@ AttrName::AttrName(PosIdx pos, std::unique_ptr<Expr> e) : pos(pos), expr(std::mo
 {
 }
 
+static JSON stringToJSON(std::string_view s)
+{
+    try {
+        JSON value = s;
+        // dump it now to catch invalid utf8 strings early. this code path is not
+        // hot, so the extra memory allocation and encoding is not worth avoiding
+        (void) value.dump();
+        return value;
+    } catch (nlohmann::json::type_error & e) { // NOLINT(lix-foreign-exceptions)
+        if (e.id == 316) {
+            // invalid utf8 in string! serialize as byte array instead
+            return s | std::ranges::to<std::vector<unsigned char>>();
+        } else {
+            throw; // NOLINT(lix-foreign-exceptions)
+        }
+    }
+}
+
 JSON Expr::toJSON(const SymbolTable & symbols) const
 {
     abort();
@@ -57,7 +75,7 @@ JSON ExprLiteral::toJSON(const SymbolTable & symbols) const
             break;
         case nString:
             valueType = "String";
-            value = v.str();
+            value = stringToJSON(v.str());
             break;
         case nPath:
             valueType = "Path";
@@ -77,10 +95,7 @@ JSON ExprLiteral::toJSON(const SymbolTable & symbols) const
 
 JSON ExprVar::toJSON(const SymbolTable & symbols) const
 {
-    return {
-        {"_type", "ExprVar"},
-        {"value", symbols[name]}
-    };
+    return {{"_type", "ExprVar"}, {"value", stringToJSON(symbols[name])}};
 }
 
 JSON ExprInheritFrom::toJSON(SymbolTable const & symbols) const
@@ -116,6 +131,8 @@ void ExprAttrs::addBindingsToJSON(JSON & out, const SymbolTable & symbols) const
     typedef const decltype(attrs)::value_type * Attr;
     std::vector<Attr> sorted;
     for (auto & i : attrs) sorted.push_back(&i);
+    // NOTE: this sort uses a total order, so the iteration over pointers is not an issue
+    // NOLINTNEXTLINE(bugprone-nondeterministic-pointer-iteration-order)
     std::sort(sorted.begin(), sorted.end(), [&](Attr a, Attr b) {
         std::string_view sa = symbols[a->first], sb = symbols[b->first];
         return sa < sb;
@@ -124,11 +141,19 @@ void ExprAttrs::addBindingsToJSON(JSON & out, const SymbolTable & symbols) const
     for (auto & i : sorted) {
         switch (i->second.kind) {
         case AttrDef::Kind::Plain:
-            out["attrs"][std::string(symbols[i->first])] = i->second.e->toJSON(symbols);
-            break;
-        case AttrDef::Kind::Inherited:
-            out["inherit"][std::string(symbols[i->first])] = i->second.e->toJSON(symbols);
+        case AttrDef::Kind::Inherited: {
+            const auto key = i->second.kind == AttrDef::Kind::Plain ? "attrs" : "inherit";
+            auto name = stringToJSON(symbols[i->first]);
+            if (name.is_string()) {
+                out[key][name] = i->second.e->toJSON(symbols);
+            } else {
+                out[fmt("binary_%s", key)][key].push_back({
+                    {"name", name},
+                    {"value", i->second.e->toJSON(symbols)},
+                });
+            }
             break;
+        }
         case AttrDef::Kind::InheritedFrom: {
             auto & select = i->second.e->cast<ExprSelect>();
             auto & from = select.e->cast<ExprInheritFrom>();
@@ -148,7 +173,7 @@ void ExprAttrs::addBindingsToJSON(JSON & out, const SymbolTable & symbols) const
     for (const auto & [from, syms] : inheritsFrom) {
         JSON attrs = JSON::array();
         for (auto sym : syms)
-            attrs.push_back(symbols[sym]);
+            attrs.push_back(stringToJSON(symbols[sym]));
         out["inheritFrom"].push_back({
             {"from", inheritFromExprs[from]->toJSON(symbols)},
             {"attrs", attrs}
@@ -186,11 +211,13 @@ JSON ExprList::toJSON(const SymbolTable & symbols) const
 
 void SimplePattern::addBindingsToJSON(JSON & out, const SymbolTable & symbols) const
 {
+    // name must be alphanumeric
     out["arg"] = symbols[name];
 }
 
 void AttrsPattern::addBindingsToJSON(JSON & out, const SymbolTable & symbols) const
 {
+    // name must be alphanumeric
     if (name)
         out["arg"] = symbols[name];
 
@@ -198,6 +225,7 @@ void AttrsPattern::addBindingsToJSON(JSON & out, const SymbolTable & symbols) co
     // same expression being printed in two different ways depending on its
     // context. always use lexicographic ordering to avoid this.
     for (const Formal & i : lexicographicOrder(symbols)) {
+        // names must be alphanumeric
         if (i.def)
             out["formals"][std::string(symbols[i.name])] = i.def->toJSON(symbols);
         else
@@ -311,7 +339,7 @@ JSON printAttrPathToJson(const SymbolTable & symbols, const AttrPath & attrPath)
     JSON out = JSON::array();
     for (auto & i : attrPath) {
         if (i.symbol)
-            out.push_back(symbols[i.symbol]);
+            out.push_back(stringToJSON(symbols[i.symbol]));
         else
             out.push_back(i.expr->toJSON(symbols));
     }

lix/libexpr/nixexpr.hh

@@ -32,6 +32,11 @@ struct AttrName
     std::unique_ptr<Expr> expr;
     AttrName(PosIdx pos, Symbol s);
     AttrName(PosIdx pos, std::unique_ptr<Expr> e);
+
+    inline bool isDynamic()
+    {
+        return !symbol;
+    }
 };
 
 typedef std::vector<AttrName> AttrPath;

lix/libexpr/parser/grammar.hh

@@ -203,6 +203,16 @@ struct sep : sor<
 
 using seps = star<t::sep>;
 
+// Delimiters.
+namespace d {
+    // Terminators.
+    namespace t {
+        struct inherit : one<';'> { };
+        struct binding : one<';'> { };
+        struct attrs : one<'}'> { };
+    }
+}
+
 
 // marker for semantic rules. not handling one of these in an action that cares about
 // semantics is probably an error.
@@ -392,7 +402,7 @@ struct _attr {
     struct expr : semantic, seq<
         TAO_PEGTL_STRING("${"), seps,
         must<grammar::v1::expr>, seps,
-        must<one<'}'>>
+        must<d::t::attrs>
     > {};
 };
 struct attr : _attr, sor<
@@ -411,7 +421,7 @@ struct inherit : _inherit, seq<
     t::kw_inherit, seps,
     opt<one<'('>, seps, _inherit::from, seps, must<one<')'>>, seps>,
     opt<_inherit::attrs, seps>,
-    must<one<';'>>
+    must<d::t::inherit>
 > {};
 
 struct _binding {
@@ -430,7 +440,7 @@ struct bindings : opt<
     list<
         sor<
             inherit,
-            seq<binding, seps, must<one<';'>>>
+            seq<binding, seps, must<d::t::binding>>
         >,
         seps
     >
@@ -439,9 +449,9 @@ struct bindings : opt<
 struct repl_binding : binding {};
 
 struct repl_bindings : seq<
-    list<repl_binding, one<';'>, t::sep>,
+    list<repl_binding, d::t::binding, t::sep>,
     /* Optional semicolon at the end for convenience */
-    opt<seps, one<';'>>
+    opt<seps, d::t::binding>
 > {};
 
 struct op {
@@ -496,7 +506,7 @@ struct _expr {
         Init...,
         OpenMod<one<'{'>>, seps,
         bindings, seps,
-        must<one<'}'>>
+        must<d::t::attrs>
     > {};
 
     struct select;

lix/libexpr/parser/parser-impl1.inc.cc

@@ -56,6 +56,9 @@ error_message_for(grammar::v1::expr::select) = "expecting selection expression";
 error_message_for(grammar::v1::t::kw_then) = "expecting 'then'";
 error_message_for(grammar::v1::t::kw_else) = "expecting 'else'";
 error_message_for(grammar::v1::t::kw_in) = "expecting 'in'";
+error_message_for(grammar::v1::d::t::binding) = "expecting ';' to end binding";
+error_message_for(grammar::v1::d::t::inherit) = "expecting ';' to end 'inherit' bindings";
+error_message_for(grammar::v1::d::t::attrs) = "expecting '}' to end attribute set";
 
 struct SyntaxErrors
 {

lix/libexpr/parser/state.hh

@@ -6,7 +6,6 @@
 #include "lix/libutil/logging.hh"
 
 namespace nix::parser {
-
 struct IndStringLine {
     // String containing only the leading whitespace of the line. May be empty.
     std::string_view indentation;
@@ -40,6 +39,7 @@ struct State
     void badLineEndingFound(const PosIdx pos, bool warnOnly);
     void nulFound(const PosIdx pos);
     void addAttr(ExprAttrs * attrs, AttrPath && attrPath, std::unique_ptr<Expr> e, const PosIdx pos);
+    void mergeAttrs(AttrPath & attrPath, ExprSet * source, ExprSet * target);
     void validateLambdaAttrs(AttrsPattern & pattern, PosIdx pos = noPos);
     std::unique_ptr<Expr> stripIndentation(const PosIdx pos, std::vector<IndStringLine> && line);
 
@@ -132,81 +132,113 @@ inline void State::addAttr(ExprAttrs * attrs, AttrPath && attrPath, std::unique_
     AttrPath::iterator i;
     // All attrpaths have at least one attr
     assert(!attrPath.empty());
-    // Checking attrPath validity.
-    // ===========================
+
+    // Walk the attrpath up to the parent of the attribute we want to insert, moving `attrs` along
+    // and creating new empty intermediate attrsets as necessary.
     for (i = attrPath.begin(); i + 1 < attrPath.end(); i++) {
-        if (i->symbol) {
-            ExprAttrs::AttrDefs::iterator j = attrs->attrs.find(i->symbol);
-            if (j != attrs->attrs.end()) {
-                if (j->second.kind != ExprAttrs::AttrDef::Kind::Inherited) {
-                    ExprSet * attrs2 = dynamic_cast<ExprSet *>(j->second.e.get());
-                    if (!attrs2) {
-                        attrPath.erase(i + 1, attrPath.end());
-                        dupAttr(attrPath, pos, j->second.pos);
-                    }
-                    attrs = attrs2;
-                } else {
-                    attrPath.erase(i + 1, attrPath.end());
-                    dupAttr(attrPath, pos, j->second.pos);
-                }
-            } else {
-                auto next = attrs->attrs.emplace(std::piecewise_construct,
-                    std::tuple(i->symbol),
-                    std::tuple(std::make_unique<ExprSet>(), pos));
-                attrs = static_cast<ExprSet *>(next.first->second.e.get());
-            }
-        } else {
+        AttrName & attr = *i;
+
+        if (attr.isDynamic()) {
+            // Simply insert an empty attrset (but dynamic)
             auto & next = attrs->dynamicAttrs.emplace_back(std::move(i->expr), std::make_unique<ExprSet>(), pos);
             attrs = static_cast<ExprSet *>(next.valueExpr.get());
+        } else if (ExprAttrs::AttrDefs::iterator j = attrs->attrs.find(i->symbol);
+                   j != attrs->attrs.end())
+        {
+            // Try to walk down the next attribute, throw duplicate error if not possible
+            auto & [foundName, foundDef] = *j;
+            if (foundDef.kind == ExprAttrs::AttrDef::Kind::Inherited) {
+                attrPath.erase(i + 1, attrPath.end());
+                return dupAttr(attrPath, pos, foundDef.pos);
+            }
+            ExprSet * foundAttrs = dynamic_cast<ExprSet *>(foundDef.e.get());
+            if (!foundAttrs) {
+                attrPath.erase(i + 1, attrPath.end());
+                return dupAttr(attrPath, pos, foundDef.pos);
+            }
+            attrs = foundAttrs;
+        } else {
+            // Simply insert an empty attrset
+            auto next = attrs->attrs.emplace(
+                std::piecewise_construct,
+                std::tuple(attr.symbol),
+                std::tuple(std::make_unique<ExprSet>(), pos)
+            );
+            attrs = static_cast<ExprSet *>(next.first->second.e.get());
         }
     }
+
     // Expr insertion.
     // ==========================
-    if (i->symbol) {
-        ExprAttrs::AttrDefs::iterator j = attrs->attrs.find(i->symbol);
-        if (j != attrs->attrs.end()) {
-            // This attr path is already defined. However, if both
-            // e and the expr pointed by the attr path are two attribute sets,
-            // we want to merge them.
-            // Otherwise, throw an error.
-            auto * ae = dynamic_cast<ExprSet *>(e.get());
-            auto * jAttrs = dynamic_cast<ExprSet *>(j->second.e.get());
-            if (jAttrs && ae) {
-                if (ae->inheritFromExprs && !jAttrs->inheritFromExprs)
-                    jAttrs->inheritFromExprs = std::make_unique<std::list<std::unique_ptr<Expr>>>();
-                for (auto & ad : ae->attrs) {
-                    auto j2 = jAttrs->attrs.find(ad.first);
-                    if (j2 != jAttrs->attrs.end()) // Attr already defined in iAttrs, error.
-                        return dupAttr(ad.first, j2->second.pos, ad.second.pos);
-                    if (ad.second.kind == ExprAttrs::AttrDef::Kind::InheritedFrom) {
-                        auto & sel = dynamic_cast<ExprSelect &>(*ad.second.e);
-                        auto & from = dynamic_cast<ExprInheritFrom &>(*sel.e);
-                        from.displ += jAttrs->inheritFromExprs->size();
-                    }
-                    jAttrs->attrs.emplace(ad.first, std::move(ad.second));
-                }
-                std::ranges::move(ae->dynamicAttrs, std::back_inserter(jAttrs->dynamicAttrs));
-                if (ae->inheritFromExprs)
-                    std::ranges::move(*ae->inheritFromExprs, std::back_inserter(*jAttrs->inheritFromExprs));
-            } else {
-                dupAttr(attrPath, pos, j->second.pos);
+    AttrName & attr = *i;
+
+    if (attr.isDynamic()) {
+        attrs->dynamicAttrs.emplace_back(std::move(attr.expr), std::move(e), pos);
+    } else if (ExprAttrs::AttrDefs::iterator j = attrs->attrs.find(attr.symbol);
+               j != attrs->attrs.end())
+    {
+        // This attr path is already defined. However, if both
+        // e and the expr pointed by the attr path are two attribute sets,
+        // we want to merge them.
+        // Otherwise, throw an error.
+        auto & [foundName, foundDef] = *j;
+
+        auto * insertAttrs = dynamic_cast<ExprSet *>(e.get());
+        auto * foundAttrs = dynamic_cast<ExprSet *>(foundDef.e.get());
+        if (!foundAttrs || !insertAttrs) {
+            return dupAttr(attrPath, pos, foundDef.pos);
+        }
+        mergeAttrs(attrPath, insertAttrs, foundAttrs);
+    } else {
+        // This attr path is not defined. Let's create it.
+
+        // Before inserting new attrs, check for __override and throw an error
+        // (the error will initially be a warning to ease migration)
+        if (!featureSettings.isEnabled(Dep::RecSetOverrides) && attr.symbol == s.overrides) {
+            if (auto set = dynamic_cast<ExprSet *>(attrs); set && set->recursive) {
+                overridesFound(pos);
             }
-        } else {
-            // Before inserting new attrs, check for __override and throw an error
-            // (the error will initially be a warning to ease migration)
-            if (!featureSettings.isEnabled(Dep::RecSetOverrides) && i->symbol == s.overrides) {
-                if (auto set = dynamic_cast<ExprSet *>(attrs); set && set->recursive)
-                    overridesFound(pos);
+        }
+
+        e->setName(attr.symbol);
+        attrs->attrs.emplace(
+            std::piecewise_construct, std::tuple(attr.symbol), std::tuple(std::move(e), pos)
+        );
+    }
+}
+
+/* mutably merge source into target. attrPath is only for error messages */
+inline void State::mergeAttrs(AttrPath & attrPath, ExprSet * source, ExprSet * target)
+{
+    if (source->inheritFromExprs && !target->inheritFromExprs) {
+        target->inheritFromExprs = std::make_unique<std::list<std::unique_ptr<Expr>>>();
+    }
+    for (auto & [insertKey, insertDef] : source->attrs) {
+        if (auto collision = target->attrs.find(insertKey); collision != target->attrs.end()) {
+            // Attr already defined in target, recurse merge if possible otherwise error.
+            auto * collisionInsert = dynamic_cast<ExprSet *>(insertDef.e.get());
+            auto * collisionTarget = dynamic_cast<ExprSet *>(collision->second.e.get());
+            if (!collisionInsert || !collisionTarget) {
+                attrPath.push_back(AttrName(insertDef.pos, insertKey));
+                return dupAttr(attrPath, insertDef.pos, collision->second.pos);
             }
 
-            // This attr path is not defined. Let's create it.
-            e->setName(i->symbol);
-            attrs->attrs.emplace(std::piecewise_construct,
-                std::tuple(i->symbol),
-                std::tuple(std::move(e), pos));
+            // Push insertKey to the attrPath for error propagation (pop afterwards), then recurse
+            // merge
+            attrPath.push_back(AttrName(insertDef.pos, insertKey));
+            mergeAttrs(attrPath, collisionInsert, collisionTarget);
+            attrPath.pop_back();
         }
-    } else {
-        attrs->dynamicAttrs.emplace_back(std::move(i->expr), std::move(e), pos);
+        if (insertDef.kind == ExprAttrs::AttrDef::Kind::InheritedFrom) {
+            auto & sel = dynamic_cast<ExprSelect &>(*insertDef.e);
+            auto & from = dynamic_cast<ExprInheritFrom &>(*sel.e);
+            from.displ += target->inheritFromExprs->size();
+        }
+        target->attrs.emplace(insertKey, std::move(insertDef));
+    }
+    std::ranges::move(source->dynamicAttrs, std::back_inserter(target->dynamicAttrs));
+    if (source->inheritFromExprs) {
+        std::ranges::move(*source->inheritFromExprs, std::back_inserter(*target->inheritFromExprs));
     }
 }
 

lix/libexpr/primops.cc

@@ -251,13 +251,6 @@ static void import(EvalState & state, Value & vPath, Value * vScope, Value & v)
     }
 }
 
-static RegisterPrimOp primop_scopedImport(PrimOp{
-    {.name = "scopedImport",
-     .arity = 2,
-     .fun = [](EvalState & state, Value ** args, Value & v) { import(state, *args[1], args[0], v); }
-    }
-});
-
 static void prim_import(EvalState & state, Value * * args, Value & v)
 {
     import(state, *args[0], nullptr, v);
@@ -270,6 +263,11 @@ extern "C" typedef void (*ValueInitializer)(EvalState & state, Value & v);
 /* Load a ValueInitializer from a DSO and return whatever it initializes */
 void prim_importNative(EvalState & state, Value * * args, Value & v)
 {
+# if LIX_MAJOR > 2 || (LIX_MAJOR == 2 && LIX_MINOR >= 97)
+#pragma message ("Folks, we need to rip this out since we've reached 2.97 See fj#796 for more details")
+# endif
+    printTaggedWarning("builtins.importNative is deprecated and will be removed in Lix 2.97, please migrate away from it. You can browse issue #795 for more details.");
+
     auto path = realisePath(state, *args[0]);
 
     std::string sym(state.forceStringNoCtx(*args[1], noPos, "while evaluating the second argument passed to builtins.importNative"));
@@ -636,7 +634,7 @@ static void prim_genericClosure(EvalState & state, Value * * args, Value & v)
 
 static void prim_break(EvalState & state, Value * * args, Value & v)
 {
-    if (auto trace = state.ctx.debug ? state.ctx.debug->traces().next() : std::nullopt) {
+    if (auto const trace = state.ctx.nextDebugTrace()) {
         auto error = EvalError(ErrorInfo {
             .level = lvlInfo,
             .msg = HintFmt("breakpoint reached"),
@@ -680,12 +678,6 @@ static void prim_addErrorContext(EvalState & state, Value * * args, Value & v)
     }
 }
 
-static RegisterPrimOp primop_addErrorContext(PrimOp{{
-    .name = "__addErrorContext",
-    .arity = 2,
-    .fun = prim_addErrorContext,
-}});
-
 static void prim_ceil(EvalState & state, Value * * args, Value & v)
 {
     auto value = state.forceFloat(*args[0], noPos,
@@ -771,16 +763,46 @@ static void prim_trace(EvalState & state, Value * * args, Value & v)
         printError("trace: %1%", Uncolored(args[0]->str()));
     else
         printError("trace: %1%", Uncolored(ValuePrinter(state, *args[0])));
-    if (auto last = evalSettings.builtinsTraceDebugger && state.ctx.debug
-            ? state.ctx.debug->traces().next()
-            : std::nullopt)
-    {
-        state.ctx.debug->onEvalError(nullptr, (*last)->env, (*last)->expr);
+
+    if (evalSettings.debuggerOnTrace) {
+        if (auto const trace = state.ctx.nextDebugTrace()) {
+            state.ctx.debug->onEvalError(nullptr, (*trace)->env, (*trace)->expr);
+        }
     }
+
     state.forceValue(*args[1], noPos);
     v = *args[1];
 }
 
+static void prim_warn(EvalState & state, Value ** args, Value & v)
+{
+    // We only accept a string argument for now. The use case for pretty printing a value is covered
+    // by `trace`. By rejecting non-strings we allow future versions to add more features without
+    // breaking existing code.
+    auto const msg =
+        state.forceString(*args[0], noPos, "while evaluating message for builtins.warn");
+
+    printTaggedWarning("%s", Uncolored(msg));
+
+    if (evalSettings.abortOnWarn) {
+        state.ctx.errors.make<Abort>("evaluation aborted (abort-on-warn)")
+            .debugThrow();
+    }
+
+    if (evalSettings.debuggerOnWarn) {
+        if (auto const trace = state.ctx.nextDebugTrace()) {
+            auto const error = EvalError(ErrorInfo{
+                .level = lvlWarn,
+                .msg = HintFmt("builtins.warn reached"),
+            });
+
+            state.ctx.debug->onEvalError(&error, (*trace)->env, (*trace)->expr);
+        }
+    }
+
+    state.forceValue(*args[1], noPos);
+    v = *args[1];
+}
 
 /* Takes two arguments and evaluates to the second one. Used as the
  * builtins.traceVerbose implementation when --trace-verbose is not enabled
@@ -1226,12 +1248,6 @@ drvName, Bindings * attrs, Value & v)
     v.mkAttrs(result);
 }
 
-static RegisterPrimOp primop_derivationStrict(PrimOp{{
-    .name = "derivationStrict",
-    .arity = 1,
-    .fun = prim_derivationStrict,
-}});
-
 /* Return a placeholder string for the specified output that will be
    substituted by the corresponding output path at build time. For
    example, 'placeholder "out"' returns the string

lix/libexpr/primops/context.cc

@@ -7,20 +7,13 @@
 
 namespace nix {
 
-static void prim_unsafeDiscardStringContext(EvalState & state, Value * * args, Value & v)
+void prim_unsafeDiscardStringContext(EvalState & state, Value ** args, Value & v)
 {
     NixStringContext context;
     auto s = state.coerceToString(noPos, *args[0], context, "while evaluating the argument passed to builtins.unsafeDiscardStringContext");
     v.mkString(*s);
 }
 
-static RegisterPrimOp primop_unsafeDiscardStringContext({
-    .name = "__unsafeDiscardStringContext",
-    .arity = 1,
-    .fun = prim_unsafeDiscardStringContext
-});
-
-
 void prim_hasContext(EvalState & state, Value * * args, Value & v)
 {
     NixStringContext context;
@@ -165,7 +158,7 @@ void prim_getContext(EvalState & state, Value * * args, Value & v)
    See the commentary above unsafeGetContext for details of the
    context representation.
 */
-static void prim_appendContext(EvalState & state, Value * * args, Value & v)
+void prim_appendContext(EvalState & state, Value ** args, Value & v)
 {
     NixStringContext context;
     auto orig = state.forceString(*args[0], context, noPos, "while evaluating the first argument passed to builtins.appendContext");
@@ -241,11 +234,4 @@ static void prim_appendContext(EvalState & state, Value * * args, Value & v)
 
     v.mkString(orig, context);
 }
-
-static RegisterPrimOp primop_appendContext({
-    .name = "__appendContext",
-    .arity = 2,
-    .fun = prim_appendContext
-});
-
 }

lix/libexpr/primops/fetchClosure.cc

@@ -1,201 +0,0 @@
-#include "lix/libexpr/eval.hh"
-#include "lix/libexpr/extra-primops.hh"
-#include "lix/libstore/store-api.hh"
-#include "lix/libstore/make-content-addressed.hh"
-#include "lix/libutil/async.hh"
-#include "lix/libutil/url.hh"
-
-namespace nix {
-
-/**
- * Handler for the content addressed case.
- *
- * @param state Evaluator state and store to write to.
- * @param fromStore Store containing the path to rewrite.
- * @param fromPath Source path to be rewritten.
- * @param toPathMaybe Path to write the rewritten path to. If empty, the error shows the actual path.
- * @param v Return `Value`
- */
-static void runFetchClosureWithRewrite(EvalState & state, Store & fromStore, const StorePath & fromPath, const std::optional<StorePath> & toPathMaybe, Value &v) {
-
-    // establish toPath or throw
-
-    if (!toPathMaybe || !state.aio.blockOn(state.ctx.store->isValidPath(*toPathMaybe))) {
-        auto rewrittenPath =
-            state.aio.blockOn(makeContentAddressed(fromStore, *state.ctx.store, fromPath));
-        if (toPathMaybe && *toPathMaybe != rewrittenPath)
-            throw Error({
-                .msg = HintFmt("rewriting '%s' to content-addressed form yielded '%s', while '%s' was expected",
-                    state.ctx.store->printStorePath(fromPath),
-                    state.ctx.store->printStorePath(rewrittenPath),
-                    state.ctx.store->printStorePath(*toPathMaybe)),
-            });
-        if (!toPathMaybe)
-            throw Error({
-                .msg = HintFmt(
-                    "rewriting '%s' to content-addressed form yielded '%s'\n"
-                    "Use this value for the 'toPath' attribute passed to 'fetchClosure'",
-                    state.ctx.store->printStorePath(fromPath),
-                    state.ctx.store->printStorePath(rewrittenPath)),
-            });
-    }
-
-    auto toPath = *toPathMaybe;
-
-    // check and return
-
-    auto resultInfo = state.aio.blockOn(state.ctx.store->queryPathInfo(toPath));
-
-    if (!resultInfo->isContentAddressed(*state.ctx.store)) {
-        // We don't perform the rewriting when outPath already exists, as an optimisation.
-        // However, we can quickly detect a mistake if the toPath is input addressed.
-        throw Error({
-            .msg = HintFmt(
-                "The 'toPath' value '%s' is input-addressed, so it can't possibly be the result of rewriting to a content-addressed path.\n\n"
-                "Set 'toPath' to an empty string to make Lix report the correct content-addressed path.",
-                state.ctx.store->printStorePath(toPath)),
-        });
-    }
-
-    state.ctx.paths.mkStorePathString(toPath, v);
-}
-
-/**
- * Fetch the closure and make sure it's content addressed.
- */
-static void runFetchClosureWithContentAddressedPath(EvalState & state, Store & fromStore, const StorePath & fromPath, Value & v) {
-
-    if (!state.aio.blockOn(state.ctx.store->isValidPath(fromPath)))
-        state.aio.blockOn(copyClosure(fromStore, *state.ctx.store, RealisedPath::Set { fromPath }));
-
-    auto info = state.aio.blockOn(state.ctx.store->queryPathInfo(fromPath));
-
-    if (!info->isContentAddressed(*state.ctx.store)) {
-        throw Error({
-            .msg = HintFmt(
-                "The 'fromPath' value '%s' is input-addressed, but 'inputAddressed' is set to 'false' (default).\n\n"
-                "If you do intend to fetch an input-addressed store path, add\n\n"
-                "    inputAddressed = true;\n\n"
-                "to the 'fetchClosure' arguments.\n\n"
-                "Note that to ensure authenticity input-addressed store paths, users must configure a trusted binary cache public key on their systems. This is not needed for content-addressed paths.",
-                state.ctx.store->printStorePath(fromPath)),
-        });
-    }
-
-    state.ctx.paths.mkStorePathString(fromPath, v);
-}
-
-/**
- * Fetch the closure and make sure it's input addressed.
- */
-static void runFetchClosureWithInputAddressedPath(EvalState & state, Store & fromStore, const StorePath & fromPath, Value & v) {
-
-    if (!state.aio.blockOn(state.ctx.store->isValidPath(fromPath)))
-        state.aio.blockOn(copyClosure(fromStore, *state.ctx.store, RealisedPath::Set { fromPath }));
-
-    auto info = state.aio.blockOn(state.ctx.store->queryPathInfo(fromPath));
-
-    if (info->isContentAddressed(*state.ctx.store)) {
-        throw Error({
-            .msg = HintFmt(
-                "The store object referred to by 'fromPath' at '%s' is not input-addressed, but 'inputAddressed' is set to 'true'.\n\n"
-                "Remove the 'inputAddressed' attribute (it defaults to 'false') to expect 'fromPath' to be content-addressed",
-                state.ctx.store->printStorePath(fromPath)),
-        });
-    }
-
-    state.ctx.paths.mkStorePathString(fromPath, v);
-}
-
-typedef std::optional<StorePath> StorePathOrGap;
-
-void prim_fetchClosure(EvalState & state, Value * * args, Value & v)
-{
-    state.forceAttrs(*args[0], noPos, "while evaluating the argument passed to builtins.fetchClosure");
-
-    std::optional<std::string> fromStoreUrl;
-    std::optional<StorePath> fromPath;
-    std::optional<StorePathOrGap> toPath;
-    std::optional<bool> inputAddressedMaybe;
-
-    for (auto & attr : *args[0]->attrs()) {
-        const auto & attrName = state.ctx.symbols[attr.name];
-        auto attrHint = [&]() -> std::string {
-            return "while evaluating the '" + attrName + "' attribute passed to builtins.fetchClosure";
-        };
-
-        if (attrName == "fromPath") {
-            NixStringContext context;
-            fromPath = state.coerceToStorePath(attr.pos, attr.value, context, attrHint());
-        }
-
-        else if (attrName == "toPath") {
-            state.forceValue(attr.value, attr.pos);
-            bool isEmptyString = attr.value.type() == nString && attr.value.str().empty();
-            if (isEmptyString) {
-                toPath = StorePathOrGap {};
-            }
-            else {
-                NixStringContext context;
-                toPath = state.coerceToStorePath(attr.pos, attr.value, context, attrHint());
-            }
-        }
-
-        else if (attrName == "fromStore")
-            fromStoreUrl = state.forceStringNoCtx(attr.value, attr.pos, attrHint());
-
-        else if (attrName == "inputAddressed")
-            inputAddressedMaybe = state.forceBool(attr.value, attr.pos, attrHint());
-
-        else
-            throw Error({
-                .msg = HintFmt("attribute '%s' isn't supported in call to 'fetchClosure'", attrName),
-            });
-    }
-
-    if (!fromPath)
-        throw Error({
-            .msg = HintFmt("attribute '%s' is missing in call to 'fetchClosure'", "fromPath"),
-        });
-
-    bool inputAddressed = inputAddressedMaybe.value_or(false);
-
-    if (inputAddressed) {
-        if (toPath)
-            throw Error({
-                .msg = HintFmt("attribute '%s' is set to true, but '%s' is also set. Please remove one of them",
-                    "inputAddressed",
-                    "toPath"),
-            });
-    }
-
-    if (!fromStoreUrl)
-        throw Error({
-            .msg = HintFmt("attribute '%s' is missing in call to 'fetchClosure'", "fromStore"),
-        });
-
-    auto parsedURL = parseURL(*fromStoreUrl);
-
-    if (parsedURL.scheme != "http" &&
-        parsedURL.scheme != "https" &&
-        !(getEnv("_NIX_IN_TEST").has_value() && parsedURL.scheme == "file"))
-        throw Error({
-            .msg = HintFmt("'fetchClosure' only supports http:// and https:// stores"),
-        });
-
-    if (!parsedURL.query.empty())
-        throw Error({
-            .msg = HintFmt("'fetchClosure' does not support URL query parameters (in '%s')", *fromStoreUrl),
-        });
-
-    auto fromStore = state.aio.blockOn(openStore(parsedURL.to_string()));
-
-    if (toPath)
-        runFetchClosureWithRewrite(state, *fromStore, *fromPath, *toPath, v);
-    else if (inputAddressed)
-        runFetchClosureWithInputAddressedPath(state, *fromStore, *fromPath, v);
-    else
-        runFetchClosureWithContentAddressedPath(state, *fromStore, *fromPath, v);
-}
-
-}

lix/libexpr/primops/fetchMercurial.cc

@@ -5,7 +5,7 @@
 
 namespace nix {
 
-static void prim_fetchMercurial(EvalState & state, Value * * args, Value & v)
+void prim_fetchMercurial(EvalState & state, Value ** args, Value & v)
 {
     std::string url;
     std::optional<Hash> rev;
@@ -102,11 +102,4 @@ static void prim_fetchMercurial(EvalState & state, Value * * args, Value & v)
 
     state.ctx.paths.allowPath(tree.storePath);
 }
-
-static RegisterPrimOp r_fetchMercurial({
-    .name = "fetchMercurial",
-    .arity = 1,
-    .fun = prim_fetchMercurial
-});
-
 }

lix/libexpr/print-options.hh

@@ -48,10 +48,12 @@ struct PrintOptions
     bool force = false;
 
     /**
-     * If true and `force` is set, print derivations as
-     * `«derivation /nix/store/...»` instead of as attribute sets.
+     * Determines at what depth derivations start getting printed as paths.
+     * When `force` is true and our print depth is greater than or equal to
+     * this value, print derivations as `«derivation /nix/store/...»`
+     * instead of as attribute sets.
      */
-    bool derivationPaths = false;
+    size_t derivationPathDepth = std::numeric_limits<size_t>::max();
 
     /**
      * If true, track which values have been printed and skip them on

lix/libexpr/print.cc

@@ -281,7 +281,9 @@ private:
 
     void printAttrs(Value & v, size_t depth)
     {
-        if (options.force && options.derivationPaths && state.isDerivation(v)) {
+        bool shouldSimplifyDerivations = options.force && depth >= options.derivationPathDepth;
+        bool isDerivation = state.isDerivation(v);
+        if (shouldSimplifyDerivations && isDerivation) {
             printDerivation(v);
         } else if (seen && !v.attrs()->empty() && !seen->insert(v.attrs()).second) {
             printRepeated();
@@ -304,7 +306,7 @@ private:
             for (auto & i : sorted) {
                 printSpace(prettyPrint);
 
-                if (attrsPrinted >= options.maxAttrs) {
+                if (attrsPrinted >= options.maxAttrs ) {
                     printElided(sorted.size() - printedHere, "attribute", "attributes");
                     break;
                 }
@@ -324,6 +326,21 @@ private:
                 }
 
                 output << " = ";
+
+                // Elide repeated drvAttrs attribute.
+                if (!shouldSimplifyDerivations && isDerivation && i.first == "drvAttrs") {
+                    state.forceValue(i.second->value, noPos);
+                    if (i.second->value.type() == ValueType::nAttrs) {
+                        printElided(i.second->value.attrs()->size(), "attribute", "attributes");
+                    } else {
+                        print(i.second->value, depth + 1);
+                    }
+                    output << ";";
+                    attrsPrinted++;
+                    printedHere++;
+                    continue;
+                }
+
                 print(i.second->value, depth + 1);
                 output << ";";
                 attrsPrinted++;
@@ -582,7 +599,6 @@ public:
             seen.reset();
         }
 
-        ValuesSeen seen;
         print(v, 0);
     }
 };

lix/libexpr/settings/abort-on-warn.md

@@ -0,0 +1,10 @@
+---
+name: abort-on-warn
+internalName: abortOnWarn
+type: bool
+default: false
+---
+If set to true, [`builtins.warn`](@docroot@/language/builtins.md#builtins-warn) will throw an error when logging a warning.
+This will give you a stack trace that leads to the location of the warning.
+This is useful for finding information about warnings in third-party Nix code when you can not start the interactive debugger, such as when Nix is called from a non-interactive script. See [`debugger-on-warn`](#conf-debugger-on-warn).
+Currently, a stack trace can only be produced when the debugger is enabled, or when evaluation is aborted.

lix/libexpr/settings/debugger-on-trace.md

@@ -1,6 +1,6 @@
 ---
 name: debugger-on-trace
-internalName: builtinsTraceDebugger
+internalName: debuggerOnTrace
 type: bool
 default: false
 ---

lix/libexpr/settings/debugger-on-warn.md

@@ -0,0 +1,9 @@
+---
+name: debugger-on-warn
+internalName: debuggerOnWarn
+type: bool
+default: false
+---
+If set to true and the `--debugger` flag is given, [`builtins.warn`](@docroot@/language/builtins.md#builtins-warn) will enter the debugger like [`builtins.break`](@docroot@/language/builtins.md#builtins-break).
+This is useful for debugging warnings in third-party Nix code.
+Use [`debugger-on-trace`](#conf-debugger-on-trace) to also enter the debugger on legacy warnings that are logged with [`builtins.trace`](@docroot@/language/builtins.md#builtins-trace).

lix/libmain/shared.cc

@@ -4,6 +4,7 @@
 #include "lix/libstore/store-api.hh"
 #include "lix/libstore/gc-store.hh"
 #include "lix/libutil/c-calls.hh"
+#include "lix/libutil/logging.hh"
 #include "lix/libutil/result.hh"
 #include "lix/libutil/signals.hh"
 #include "lix/libmain/loggers.hh"
@@ -32,8 +33,6 @@
 
 namespace nix {
 
-char * * savedArgv;
-
 static bool gcWarning = true;
 
 void printGCWarning()
@@ -94,6 +93,8 @@ try {
         std::vector<const StorePath *> willSubstituteSorted = {};
         std::for_each(willSubstitute.begin(), willSubstitute.end(),
                    [&](const StorePath &p) { willSubstituteSorted.push_back(&p); });
+        // NOTE: this sort uses a total order, so the iteration over pointers is not an issue
+        // NOLINTNEXTLINE(bugprone-nondeterministic-pointer-iteration-order)
         std::sort(willSubstituteSorted.begin(), willSubstituteSorted.end(),
                   [](const StorePath *lhs, const StorePath *rhs) {
                     if (lhs->name() == rhs->name())
@@ -403,13 +404,34 @@ RunPager::~RunPager()
     }
 }
 
-
 PrintFreed::~PrintFreed()
 {
-    if (show)
-        std::cout << fmt("%d store paths deleted, %s freed\n",
-            results.paths.size(),
-            showBytes(results.bytesFreed));
-}
+    // When in dry-run mode, print the paths on stdout
+    if (action == GCOptions::gcReturnLive || action == GCOptions::gcReturnDead) {
+        for (auto & i : results.paths) {
+            logger->cout("%s", i);
+        };
+    }
 
+    switch (action) {
+    case GCOptions::gcReturnLive: {
+        notice("%1% store paths would be kept\n", results.paths.size());
+        break;
+    }
+    case GCOptions::gcReturnDead: {
+        notice("%1% store paths would be deleted\n", results.paths.size());
+        break;
+    }
+    case GCOptions::gcDeleteDead:
+    case GCOptions::gcDeleteSpecific:
+    case GCOptions::gcTryDeleteSpecific: {
+        notice(
+            "%1% store paths deleted, %2% freed\n",
+            results.paths.size(),
+            showBytes(results.bytesFreed)
+        );
+        break;
+    }
+    }
+}
 }

lix/libmain/shared.hh

@@ -1,6 +1,7 @@
 #pragma once
 ///@file
 
+#include "lix/libstore/gc-store.hh"
 #include "lix/libutil/args.hh"
 #include "lix/libutil/args/root.hh"
 #include "lix/libmain/common-args.hh"
@@ -99,10 +100,14 @@ struct GCResults;
 
 struct PrintFreed
 {
-    bool show;
+    GCOptions::GCAction action;
+
     const GCResults & results;
-    PrintFreed(bool show, const GCResults & results)
-        : show(show), results(results) { }
+    PrintFreed(GCOptions::GCAction action, const GCResults & results)
+        : action(action)
+        , results(results)
+    {
+    }
     ~PrintFreed();
 };
 

lix/libmain/stack.cc

@@ -48,7 +48,7 @@ void detectStackOverflow()
        requires an alternative stack, otherwise the signal cannot be
        delivered when we're out of stack space. */
     stack_t stack;
-    stack.ss_size = 4096 * 4 + MINSIGSTKSZ;
+    stack.ss_size = 4096ul * 4 + MINSIGSTKSZ;
     static auto stackBuf = std::make_unique<std::vector<char>>(stack.ss_size);
     stack.ss_sp = stackBuf->data();
     if (!stack.ss_sp) throw Error("cannot allocate alternative stack");

lix/libstore/build/local-derivation-goal.cc

@@ -227,13 +227,20 @@ retry:
     }
 
     auto & localStore = getLocalStore();
-    if (localStore.config().storeDir != localStore.config().realStoreDir.get()) {
-        #if __linux__
+    const bool sandboxFallbackAllowed = [&]() {
+        if (localStore.config().storeDir != localStore.config().realStoreDir.get()) {
+#if __linux__
+            if (!useChroot) {
+                printTaggedWarning("auto-enabling the sandbox due to using a diverted store");
+            }
             useChroot = true;
-        #else
+            return false;
+#else
             throw Error("building using a diverted store is not supported on this platform");
-        #endif
-    }
+#endif
+        }
+        return true;
+    }();
 
     if (useBuildUsers()) {
         if (!buildUser)
@@ -263,7 +270,9 @@ retry:
         if (!mountAndPidNamespacesSupported()) {
             if (!settings.sandboxFallback)
                 throw Error("this system does not support the kernel namespaces that are required for sandboxing; use '--no-sandbox' to disable sandboxing. Pass --debug for diagnostics on what is broken.");
-            debug("auto-disabling sandboxing because the prerequisite namespaces are not available");
+            if (!sandboxFallbackAllowed)
+                throw Error("Sandboxing is enabled due to using a diverted store, but this system does not support the kernel namespaces that are required. Pass --debug for diagnostics on what is broken.");
+            printTaggedWarning("auto-disabling sandboxing because the prerequisite namespaces are not available");
             useChroot = false;
         }
 
@@ -1825,15 +1834,18 @@ try {
             );
         }
 
+        outputGraph[scratchOutputs.at(name)] = StorePathSet{};
         std::visit(
             overloaded{/* Since we'll use the already installed versions of these, we
                                    can treat them as leaves and ignore any references they
                                    have. */
-                       [&](const AlreadyRegistered &) {
-                           outputGraph[scratchOutputs.at(name)] = StorePathSet{};
-                       },
+                       [&](const AlreadyRegistered &) {},
                        [&](const PerhapsNeedToRegister & refs) {
-                           outputGraph[scratchOutputs.at(name)] = refs.refs;
+                           for (auto & ref : refs.refs) {
+                               if (inverseOutputMap.find(ref) != inverseOutputMap.end()) {
+                                   outputGraph[scratchOutputs.at(name)].insert(ref);
+                               }
+                           }
                        }
             },
             *orifu
@@ -1844,10 +1856,8 @@ try {
         topoSort(outputsToSort, {[&](const std::string & name) {
                      StringSet dependencies;
                      for (auto & path : outputGraph.at(scratchOutputs.at(name))) {
-                         auto outputName = inverseOutputMap.find(path);
-                         if (outputName != inverseOutputMap.end()) {
-                             dependencies.insert(outputName->second);
-                         }
+                         auto outputName = inverseOutputMap.at(path);
+                         dependencies.insert(outputName);
                      }
                      return dependencies;
                  }});

lix/libstore/derivations.cc

@@ -142,6 +142,13 @@ static void expect(StringViewStream & str, std::string_view s)
     str.remaining.remove_prefix(s.size());
 }
 
+static void expectWithErrorMsg(StringViewStream & str, std::string_view s, const char * errorMsg)
+{
+    if (!str.remaining.starts_with(s)) {
+        throw FormatError(errorMsg);
+    }
+    str.remaining.remove_prefix(s.size());
+}
 
 /* Read a C-style string from stream `str'. */
 static BackedStringView parseString(StringViewStream & str)
@@ -276,13 +283,16 @@ Derivation parseDerivation(
     drv.name = name;
 
     StringViewStream str{s};
-    expect(str, "D");
+
+    const char * genericErrorMsg = "derivation does not start with 'Derive' or 'DrvWithVersion'";
+
+    expectWithErrorMsg(str, "D", genericErrorMsg);
     switch (str.peek()) {
     case 'e':
-        expect(str, "erive(");
+        expectWithErrorMsg(str, "erive(", genericErrorMsg);
         break;
     case 'r': {
-        expect(str, "rvWithVersion(");
+        expectWithErrorMsg(str, "rvWithVersion(", genericErrorMsg);
         auto versionS = parseString(str);
         throw FormatError("Unknown derivation ATerm format version '%s'", *versionS);
     }
@@ -342,7 +352,7 @@ Derivation parseDerivation(
  */
 static void printString(std::string & res, std::string_view s)
 {
-    boost::container::small_vector<char, 64 * 1024> buffer;
+    boost::container::small_vector<char, 64ul * 1024> buffer;
     buffer.reserve(s.size() * 2 + 2);
     char * buf = buffer.data();
     char * p = buf;

lix/libstore/filetransfer.cc

@@ -333,7 +333,7 @@ struct TransferItem
 
             // when the buffer is full (as determined by a historical magic value) we
             // pause the transfer and wait for the receiver to unpause it when ready.
-            if (successfulStatuses.count(getHTTPStatus()) && state->data.size() > 1024 * 1024) {
+            if (successfulStatuses.count(getHTTPStatus()) && state->data.size() > 1024ul * 1024) {
                 return CURL_WRITEFUNC_PAUSE;
             }
 
@@ -1093,15 +1093,6 @@ struct curlFileTransfer : public FileTransfer
             co_return result::current_exception();
         }
 
-        void throwChangedTarget(std::string_view what, std::string_view from, std::string_view to)
-        {
-            if (!from.empty() && from != to) {
-                throw FileTransferError(
-                    Misc, {}, "uri %s changed %s from %s to %s during transfer", uri, what, from, to
-                );
-            }
-        }
-
         kj::Promise<Result<void>> prepareRetry(
             const std::string & context,
             const std::chrono::milliseconds & waitTime,
@@ -1113,7 +1104,7 @@ struct curlFileTransfer : public FileTransfer
                     "%s; retrying from offset %d in %d ms (attempt %d/%d)",
                     Uncolored(context),
                     totalReceived,
-                    waitTime,
+                    waitTime.count(),
                     Uncolored(attempt),
                     Uncolored(tries)
                 );
@@ -1121,7 +1112,7 @@ struct curlFileTransfer : public FileTransfer
                 printTaggedWarning(
                     "%s; retrying in %d ms (attempt %d/%d)",
                     Uncolored(context),
-                    waitTime,
+                    waitTime.count(),
                     Uncolored(attempt),
                     Uncolored(tries)
                 );
@@ -1135,17 +1126,41 @@ struct curlFileTransfer : public FileTransfer
 
         kj::Promise<Result<void>> restartTransfer(const std::chrono::milliseconds & timeout)
         try {
+            auto onChange =
+                [&](std::string_view what, std::string_view from, std::string_view to, bool throw_
+                ) -> void {
+                if (!from.empty() && from != to) {
+                    FileTransferError e = FileTransferError(
+                        Misc,
+                        {},
+                        "uri %s changed %s from %s to %s during transfer",
+                        uri,
+                        what,
+                        from,
+                        to
+                    );
+
+                    if (throw_) {
+                        throw e;
+                    }
+
+                    logWarning(e.info());
+                }
+            };
+
             // use the effective URI of the previous transfer for retries. this avoids
             // some silent corruption if a redirect changes between starting and retry
             const auto & uri = metadata.effectiveUri.empty() ? this->uri : metadata.effectiveUri;
 
             auto newMeta = TRY_AWAIT(startTransfer(uri, timeout, totalReceived));
-            throwChangedTarget("final destination", metadata.effectiveUri, newMeta.effectiveUri);
-            throwChangedTarget("ETag", metadata.etag, newMeta.etag);
-            throwChangedTarget(
+
+            onChange("final destination", metadata.effectiveUri, newMeta.effectiveUri, false);
+            onChange("ETag", metadata.etag, newMeta.etag, true);
+            onChange(
                 "immutable url",
                 metadata.immutableUrl.value_or(""),
-                newMeta.immutableUrl.value_or("")
+                newMeta.immutableUrl.value_or(""),
+                true
             );
             co_return result::success();
         } catch (...) {

lix/libstore/gc.cc

@@ -18,6 +18,7 @@
 #include <kj/async.h>
 #include <queue>
 #include <regex>
+#include <thread>
 
 #include <errno.h>
 #include <fcntl.h>

lix/libstore/machines.cc

@@ -1,47 +1,16 @@
 #include "lix/libstore/machines.hh"
+
 #include "lix/libstore/globals.hh"
 #include "lix/libstore/store-api.hh"
 #include "lix/libutil/async.hh"
 #include "lix/libutil/strings.hh"
 
+#include <numeric>
 #include <algorithm>
 
-namespace nix {
+#include <toml.hpp>
 
-Machine::Machine(decltype(storeUri) storeUri,
-    decltype(systemTypes) systemTypes,
-    decltype(sshKey) sshKey,
-    decltype(maxJobs) maxJobs,
-    decltype(speedFactor) speedFactor,
-    decltype(supportedFeatures) supportedFeatures,
-    decltype(mandatoryFeatures) mandatoryFeatures,
-    decltype(sshPublicHostKey) sshPublicHostKey) :
-    storeUri(
-        // Backwards compatibility: if the URI is schemeless, is not a path,
-        // and is not one of the special store connection words, prepend
-        // ssh://.
-        storeUri.find("://") != std::string::npos
-        || storeUri.find("/") != std::string::npos
-        || storeUri == "auto"
-        || storeUri == "daemon"
-        || storeUri == "local"
-        || storeUri.starts_with("auto?")
-        || storeUri.starts_with("daemon?")
-        || storeUri.starts_with("local?")
-        || storeUri.starts_with("?")
-        ? storeUri
-        : "ssh://" + storeUri),
-    systemTypes(systemTypes),
-    sshKey(sshKey),
-    maxJobs(maxJobs),
-    speedFactor(speedFactor == 0.0f ? 1.0f : speedFactor),
-    supportedFeatures(supportedFeatures),
-    mandatoryFeatures(mandatoryFeatures),
-    sshPublicHostKey(sshPublicHostKey)
-{
-    if (speedFactor < 0.0)
-        throw UsageError("speed factor must be >= 0");
-}
+namespace nix {
 
 bool Machine::systemSupported(const std::string & system) const
 {
@@ -71,6 +40,7 @@ try {
 
     StoreConfig::Params storeParams;
     if (storeUri.starts_with("ssh://")) {
+        // Remote builds become flakey, when having more than one ssh connection
         storeParams["max-connections"] = "1";
     }
 
@@ -100,6 +70,8 @@ try {
     co_return result::current_exception();
 }
 
+namespace machines_legacy_parsing {
+
 static std::vector<std::string> expandBuilderLines(const std::string & builders)
 {
     std::vector<std::string> result;
@@ -166,15 +138,43 @@ static Machine parseBuilderLine(const std::string & line)
     if (!isSet(0))
         throw FormatError("bad machine specification: store URL was not found at the first column of a row: '%s'", line);
 
+    auto storeUri = tokens[0];
+    // Backwards compatibility: if the URI is schemeless, is not a path,
+    // and is not one of the special store connection words, prepend
+    // ssh://.
+    storeUri = storeUri.find("://") != std::string::npos || storeUri.find("/") != std::string::npos
+            || storeUri == "auto" || storeUri == "daemon" || storeUri == "local"
+            || storeUri.starts_with("auto?") || storeUri.starts_with("daemon?")
+            || storeUri.starts_with("local?") || storeUri.starts_with("?")
+        ? storeUri
+        : "ssh://" + storeUri;
+
+    auto systemTypes = isSet(1) ? tokenizeString<std::set<std::string>>(tokens[1], ",")
+                                : std::set<std::string>{settings.thisSystem};
+    auto sshKey = isSet(2) ? tokens[2] : "";
+    auto maxJobs = isSet(3) ? parseUnsignedIntField(3) : 1U;
+    auto speedFactor = isSet(4) ? parseFloatField(4) : 1.0f;
+
+    auto supportedFeatures =
+        isSet(5) ? tokenizeString<std::set<std::string>>(tokens[5], ",") : std::set<std::string>{};
+    auto mandatoryFeatures =
+        isSet(6) ? tokenizeString<std::set<std::string>>(tokens[6], ",") : std::set<std::string>{};
+    auto sshPublicHostKey = isSet(7) ? ensureBase64(7) : "";
+
+    speedFactor = speedFactor == 0.0f ? 1.0f : speedFactor;
+    if (speedFactor < 0.0) {
+        throw UsageError("speed factor must be >= 0");
+    }
+
     return {
-        tokens[0],
-        isSet(1) ? tokenizeString<std::set<std::string>>(tokens[1], ",") : std::set<std::string>{settings.thisSystem},
-        isSet(2) ? tokens[2] : "",
-        isSet(3) ? parseUnsignedIntField(3) : 1U,
-        isSet(4) ? parseFloatField(4) : 1.0f,
-        isSet(5) ? tokenizeString<std::set<std::string>>(tokens[5], ",") : std::set<std::string>{},
-        isSet(6) ? tokenizeString<std::set<std::string>>(tokens[6], ",") : std::set<std::string>{},
-        isSet(7) ? ensureBase64(7) : ""
+        storeUri,
+        systemTypes,
+        sshKey,
+        maxJobs,
+        speedFactor,
+        supportedFeatures,
+        mandatoryFeatures,
+        sshPublicHostKey
     };
 }
 
@@ -192,3 +192,287 @@ Machines getMachines()
 }
 
 }
+
+namespace machines_toml_parsing {
+
+static constexpr int MIN_VERSION = 1;
+static constexpr int LATEST_VERSION = 1;
+// Toml format:
+// [[machines.andesite]]
+// uri = "..."
+//
+// [[machines.diorite]]
+// ...
+
+template<typename T>
+static toml::result<T, std::string> parse(const toml::value & data, const std::string & key)
+{
+    try {
+        return toml::success(toml::get<T>(data.at(key)));
+    } catch (toml::type_error & e) { // NOLINT(lix-foreign-exceptions)
+        return toml::failure<std::string>({e.what()});
+    } catch (std::out_of_range & _) { // NOLINT(lix-foreign-exceptions)
+        const auto ei =
+            toml::make_error_info(fmt("%s must be present", key), data, "but was not set");
+        return toml::failure(toml::format_error(ei));
+    }
+}
+
+template<typename T>
+static toml::result<T, std::string>
+parse(const toml::value & data, const std::string & key, T defaultValue)
+{
+    if (!data.contains(key)) {
+        return toml::success(defaultValue);
+    }
+    try {
+        return toml::success(toml::get<T>(data.at(key)));
+    } catch (toml::type_error & e) { // NOLINT(lix-foreign-exceptions)
+        // invalid value
+        return toml::failure<std::string>({e.what()});
+    }
+}
+
+static const std::set<std::string> EXPECTED_KEYS = {
+    "uri",
+    "system-types",
+    "ssh-key",
+    "jobs",
+    "speed-factor",
+    "supported-features",
+    "mandatory-features",
+    "ssh-public-host-key",
+    "enable",
+};
+
+static toml::result<float, std::string> getSpeedFactor(const toml::value & data)
+{
+    if (data.contains("speed-factor")) {
+        auto sf = data.at("speed-factor");
+        if (sf.is_integer()) {
+            return toml::success(static_cast<float>(sf.as_integer()));
+        }
+        if (sf.is_floating()) {
+            return toml::success(static_cast<float>(sf.as_floating()));
+        }
+        return toml::failure(toml::format_error(toml::make_error_info(
+            "bad_cast to floating for `speed-factor`", sf, "Was neither an integer nor a float"
+        )));
+    }
+    return toml::success(1.0f);
+}
+
+static toml::result<Machine, std::vector<std::string>> parseMachine(const toml::value & data)
+{
+    std::vector<std::string> errs;
+
+    if (!data.is_table()) {
+        errs.push_back(toml::format_error(toml::make_error_info(
+            "Each machine must be a table", data, "This should be a table. Did you mean `.uri = `?"
+        )));
+        return toml::failure(errs);
+    }
+
+    // parsing
+    auto storeUri = parse<std::string>(data, "uri");
+    auto systemTypes = parse<std::vector<std::string>>(
+        data, "system-types", std::vector<std::string>{settings.thisSystem}
+    );
+    auto sshKey = parse<std::string>(data, "ssh-key", "");
+    auto maxJobs = parse<int>(data, "jobs", 1U);
+    auto speedFactor = getSpeedFactor(data);
+    auto supportedFeatures =
+        parse<std::vector<std::string>>(data, "supported-features", std::vector<std::string>{});
+    auto mandatoryFeatures =
+        parse<std::vector<std::string>>(data, "mandatory-features", std::vector<std::string>{});
+    auto sshPublicHostKey = parse<std::string>(data, "ssh-public-host-key", "");
+
+    // parsing validation
+    if (storeUri.is_err()) {
+        errs.push_back(storeUri.as_err());
+    }
+    if (systemTypes.is_err()) {
+        errs.push_back(systemTypes.as_err());
+    }
+    if (sshKey.is_err()) {
+        errs.push_back(sshKey.as_err());
+    }
+    if (maxJobs.is_err()) {
+        errs.push_back(maxJobs.as_err());
+    }
+    if (speedFactor.is_err()) {
+        errs.push_back(speedFactor.as_err());
+    }
+    if (supportedFeatures.is_err()) {
+        errs.push_back(supportedFeatures.as_err());
+    }
+    if (mandatoryFeatures.is_err()) {
+        errs.push_back(mandatoryFeatures.as_err());
+    }
+    if (sshPublicHostKey.is_err()) {
+        errs.push_back(sshPublicHostKey.as_err());
+    }
+
+    // value validation
+    if (maxJobs.is_ok() && maxJobs.as_ok() < 0) {
+        auto ei =
+            toml::make_error_info("jobs must be >= 0", data.at("jobs"), "but got negative value");
+        errs.push_back(toml::format_error(ei));
+    }
+
+    if (speedFactor.is_ok() && speedFactor.as_ok() < 0.0) {
+        auto ei = toml::make_error_info(
+            "speed factor must be >= 0", data.at("speed-factor"), "but got negative value"
+        );
+        errs.push_back(toml::format_error(ei));
+    }
+
+    for (const auto & [key, _] : data.as_table()) {
+        if (!EXPECTED_KEYS.contains(key)) {
+            errs.push_back(toml::format_error(toml::make_error_info(
+                fmt("unexpected key `%s`", key), data.at(key), "should not be present"
+            )));
+        }
+    }
+
+    if (!errs.empty()) {
+        return toml::failure(errs);
+    }
+    return toml::success<Machine>({
+        storeUri.unwrap(),
+        std::set(systemTypes.unwrap().begin(), systemTypes.unwrap().end()),
+        sshKey.unwrap(),
+        static_cast<unsigned>(maxJobs.unwrap()),
+        speedFactor.unwrap(),
+        std::set(supportedFeatures.unwrap().begin(), supportedFeatures.unwrap().end()),
+        std::set(mandatoryFeatures.unwrap().begin(), mandatoryFeatures.unwrap().end()),
+        base64Encode(sshPublicHostKey.unwrap()),
+    });
+}
+
+static toml::result<Machines, std::vector<std::string>> parseToml(const toml::value & data)
+{
+    auto const array_name = "machines";
+    std::vector<std::string> parserErrors;
+    Machines machines;
+    // Empty config
+    if (data.size() == 0) {
+        return toml::success<Machines>({});
+    }
+    if (!data.is_table()) {
+        parserErrors.push_back(
+            "Top level must be a table. This should never throw as this is required by the toml "
+            "SPEC"
+        );
+        return toml::failure(parserErrors);
+    }
+
+    if (auto config_version = parse<int>(data, "version", LATEST_VERSION); config_version.is_err())
+    {
+        parserErrors.push_back(config_version.as_err());
+    } else if (config_version.as_ok() < MIN_VERSION || config_version.as_ok() > LATEST_VERSION) {
+        parserErrors.push_back(
+            fmt("Unable to parse Machines of version %d, only versions between %d and %d are "
+                "supported.",
+                config_version.as_ok(),
+                MIN_VERSION,
+                LATEST_VERSION)
+        );
+    }
+    if (!parserErrors.empty()) {
+        return toml::failure(parserErrors);
+    }
+
+    auto & tbl = data.as_table();
+    std::string unexpected_keys;
+    for (auto it = tbl.begin(); it != tbl.end(); ++it) {
+        if (it->first == array_name || it->first == "version") {
+            // expected keys
+            continue;
+        }
+        unexpected_keys += ", " + it->first;
+    }
+    if (unexpected_keys.size()) {
+        parserErrors.push_back(fmt("unexpected keys found: %s", unexpected_keys.erase(0, 2)));
+    }
+
+    if (!data.at(array_name).is_table()) {
+        parserErrors.push_back(
+            fmt("Expected key `%s` to be a table of name -> machine configurations", array_name)
+        );
+        return toml::failure(parserErrors);
+    }
+
+    for (const auto & [name, machine] : data.at(array_name).as_table()) {
+        auto const res = parseMachine(machine);
+        if (res.is_err()) {
+            auto err = res.as_err();
+            parserErrors.push_back(fmt("for machine %s:", name));
+            parserErrors.insert(parserErrors.end(), err.begin(), err.end());
+            continue;
+        }
+        auto enable = parse<bool>(machine, "enable", true);
+        if (enable.is_ok()) {
+            if (enable.unwrap()) {
+                // Check if it hasn't been statically disabled
+                // But still throw parsing errors if it was
+                machines.push_back(res.unwrap());
+            }
+        } else {
+            parserErrors.push_back(enable.as_err());
+        }
+    }
+
+    if (!parserErrors.empty()) {
+        return toml::failure(parserErrors);
+    }
+    return toml::success(machines);
+}
+
+static std::optional<Machines> getMachines()
+{
+    toml::value data;
+    auto buildersStr = settings.builders.get();
+    try {
+        if (buildersStr.size() > 0 && buildersStr.at(0) == '@') {
+            data = toml::parse(buildersStr.substr(1));
+        } else {
+            data = toml::parse_str(settings.builders);
+        }
+    } catch (toml::syntax_error const & e) { // NOLINT(lix-foreign-exceptions)
+        if (toLower(buildersStr).contains("toml") || buildersStr.contains("\"")) {
+            // Yes, we are sure this is a TOML and no this shitty legacy format
+            // so we can safely throw the syntax error here
+            throw UsageError(fmt("invalid Machines TOML syntax: \n%s", e.what()));
+        }
+        return {};
+    } catch (toml::file_io_error const & _) { // NOLINT(lix-foreign-exceptions)
+        // sadly we have to do this otherwise we break the old format,
+        // which requires **silently ignoring** invalid files
+        return {};
+    }
+    auto const fromToml = parseToml(data);
+    if (fromToml.is_ok()) {
+        return fromToml.unwrap();
+    }
+
+    auto const & errs = fromToml.as_err();
+    std::string msg = "invalid Machines TOML:\n";
+    msg += concatStringsSep("\n", errs);
+
+    throw UsageError(msg);
+}
+
+}
+
+Machines getMachines()
+{
+    auto const toml_result = machines_toml_parsing::getMachines();
+    if (toml_result.has_value()) {
+        return toml_result.value();
+    }
+    debug("Trying again with legacy format");
+    return machines_legacy_parsing::getMachines();
+}
+
+}

lix/libstore/machines.hh

@@ -41,15 +41,6 @@ struct Machine {
      */
     bool mandatoryMet(const std::set<std::string> & features) const;
 
-    Machine(decltype(storeUri) storeUri,
-        decltype(systemTypes) systemTypes,
-        decltype(sshKey) sshKey,
-        decltype(maxJobs) maxJobs,
-        decltype(speedFactor) speedFactor,
-        decltype(supportedFeatures) supportedFeatures,
-        decltype(mandatoryFeatures) mandatoryFeatures,
-        decltype(sshPublicHostKey) sshPublicHostKey);
-
     kj::Promise<Result<std::pair<ref<Store>, Pipe>>> openStore() const;
 };
 

lix/libstore/meson.build

@@ -471,6 +471,6 @@ configure_file(
     'libdir' : libdir,
     'includedir' : includedir,
     'PACKAGE_VERSION' : meson.project_version(),
-    'AWS_SDK_IF_FOUND' : aws_sdk.found() ? 'aws-cpp-sdk-core aws-cpp-sdk-s3 aws-cpp-std-transfer' : '',
+    'AWS_SDK_IF_FOUND' : aws_sdk.found() ? 'aws-cpp-sdk-core aws-cpp-sdk-s3 aws-cpp-sdk-transfer' : '',
   },
 )

lix/libstore/path-tree.cc

@@ -265,7 +265,7 @@ static std::map<StorePath, Node> mkGraph(
 
     for (auto & node : graph_data) {
         for (auto & ref : node.second.dependencies) {
-            graph_data.find(ref)->second.dependents.insert(node.first);
+            graph_data.at(ref).dependents.insert(node.first);
         }
     }
 

lix/libstore/profiles.cc

@@ -253,7 +253,7 @@ time_t parseOlderThanTimeSpec(std::string_view timeSpec)
 
     time_t curTime = time(0);
     auto strDays = timeSpec.substr(0, timeSpec.size() - 1);
-    auto days = string2Int<int>(strDays);
+    auto days = string2Int<time_t>(strDays);
 
     if (!days || *days < 1)
         throw UsageError("invalid number of days specifier '%1%'", timeSpec);

lix/libstore/s3-binary-cache-store.cc

@@ -195,8 +195,8 @@ ref<Aws::Client::ClientConfiguration> S3Helper::makeConfig(
     if (!endpoint.empty()) {
         res->endpointOverride = endpoint;
     }
-    res->requestTimeoutMs = 600 * 1000;
-    res->connectTimeoutMs = 5 * 1000;
+    res->requestTimeoutMs = 600l * 1000;
+    res->connectTimeoutMs = 5l * 1000;
     res->retryStrategy = std::make_shared<RetryStrategy>();
     res->caFile = settings.caFile;
     // Use the system proxy env-vars in curl for s3, which is off by default for some reason
@@ -312,8 +312,11 @@ struct S3BinaryCacheStoreConfig final : BinaryCacheStoreConfig
         "Whether to use multi-part uploads."};
 
     const Setting<uint64_t> bufferSize{
-        this, 5 * 1024 * 1024, "buffer-size",
-        "Size (in bytes) of each part in multi-part uploads."};
+        this,
+        5ul * 1024 * 1024,
+        "buffer-size",
+        "Size (in bytes) of each part in multi-part uploads."
+    };
 
     const std::string name() override { return "S3 Binary Cache Store"; }
 

lix/libstore/store-api.cc

@@ -1338,7 +1338,19 @@ try {
             Derivation::nameFromPath(drvPath)
         );
     } catch (FormatError & e) {
-        throw Error("error parsing derivation '%s': %s", store.printStorePath(drvPath), e.msg());
+        auto drvPathS = store.printStorePath(drvPath);
+        throw Error(
+            fmt("error parsing derivation '%1%': %2%\n"
+                "This can occur when the derivation is corrupted.\n"
+                "You can check this with `nix-store --verify-path %1%` and possibly repair with "
+                "`nix-store --repair-path %1%`.\n"
+                "In case the repair cannot be done, `nix-store --delete %1%` may be able "
+                "to remove the broken path.\n"
+                "We would appreciate a bug report at "
+                "https://git.lix.systems/lix-project/lix/issues if you think this is a bug.",
+                drvPathS,
+                e.msg())
+        );
     }
 } catch (...) {
     co_return result::current_exception();
@@ -1537,7 +1549,10 @@ try {
             try {
                 (*lk)->push_back(TRY_AWAIT(openStore(uri)));
             } catch (Error & e) {
-                logWarning(e.info());
+                logWarning(
+                    {.msg = HintFmt("Failed to setup the substituter at URI '%s': %s", uri, e.msg())
+                    }
+                );
             }
         }
 

lix/libutil/async-io.hh

@@ -121,7 +121,7 @@ public:
     {
     }
 
-    AsyncBufferedInputStream(AsyncInputStream & inner, size_t bufSize = 32 * 1024)
+    AsyncBufferedInputStream(AsyncInputStream & inner, size_t bufSize = 32ul * 1024)
         : AsyncBufferedInputStream(inner, make_ref<IoBuffer>(bufSize))
     {
     }
@@ -173,7 +173,7 @@ public:
     {
     }
 
-    AsyncBufferedOutputStream(AsyncOutputStream & inner, size_t bufSize = 32 * 1024)
+    AsyncBufferedOutputStream(AsyncOutputStream & inner, size_t bufSize = 32ul * 1024)
         : AsyncBufferedOutputStream(inner, make_ref<IoBuffer>(bufSize))
     {
     }

lix/libutil/compression.cc

@@ -16,7 +16,6 @@
 #include <archive.h>
 #include <archive_entry.h>
 #include <cerrno>
-#include <chrono>
 #include <cstdio>
 #include <cstring>
 
@@ -38,7 +37,7 @@ static const int COMPRESSION_LEVEL_DEFAULT = -1;
 // Don't feed brotli too much at once.
 struct ChunkedCompressionSink : CompressionSink
 {
-    uint8_t outbuf[32 * 1024];
+    uint8_t outbuf[32ul * 1024];
 
     void writeUnbuffered(std::string_view data) override
     {
@@ -63,13 +62,15 @@ struct ArchiveDecompressionSource : Source
         struct archive_entry * ae;
         if (!archive) {
             archive = std::make_unique<TarArchive>(*src, true);
-            this->archive->check(archive_read_next_header(this->archive->archive, &ae),
-                "failed to read header (%s)");
-            if (archive_filter_count(this->archive->archive) < 2) {
+            this->archive->check(
+                archive_read_next_header(this->archive->archive.get(), &ae),
+                "failed to read header (%s)"
+            );
+            if (archive_filter_count(this->archive->archive.get()) < 2) {
                 throw CompressionError("input compression not recognized");
             }
         }
-        ssize_t result = archive_read_data(this->archive->archive, data, len);
+        ssize_t result = archive_read_data(this->archive->archive.get(), data, len);
         if (result > 0) return result;
         if (result == 0) {
             throw EndOfFile("reached end of compressed file");
@@ -167,7 +168,7 @@ struct NoneSink : CompressionSink
 
 struct BrotliDecompressionSource : Source
 {
-    static constexpr size_t BUF_SIZE = 32 * 1024;
+    static constexpr size_t BUF_SIZE = 32ul * 1024;
     std::unique_ptr<char[]> buf;
     size_t avail_in = 0;
     const uint8_t * next_in;

lix/libutil/deprecated-features/nix-path-shadow.md

@@ -0,0 +1,6 @@
+---
+name: nix-path-shadow
+internalName: NixPathShadow
+---
+
+Allows shadowing `<nix/fetchurl.nix>` by configuration of the [*nix path*](@docroot@/language/builtin-constants.html#builtins-nixPath) to a value containing `nix=/some/path`.

lix/libutil/experimental-features/fetch-closure.md

@@ -1,5 +0,0 @@
----
-name: fetch-closure
-internalName: FetchClosure
----
-Enable the use of the [`fetchClosure`](@docroot@/language/builtins.md#builtins-fetchClosure) built-in function in the Nix language.

lix/libutil/file-descriptor.cc

@@ -133,7 +133,7 @@ Generator<Bytes> drainFDSource(int fd, bool block)
         }
     });
 
-    std::array<unsigned char, 64 * 1024> buf;
+    std::array<unsigned char, 64ul * 1024> buf;
     while (1) {
         checkInterrupt();
         ssize_t rd = read(fd, buf.data(), buf.size());

lix/libutil/file-system.cc

@@ -443,7 +443,7 @@ void writeFile(const Path & path, Source & source, mode_t mode)
 {
     AutoCloseFD fd = openForWrite(path, mode);
 
-    std::vector<char> buf(64 * 1024);
+    std::vector<char> buf(64ul * 1024);
 
     try {
         while (true) {
@@ -463,7 +463,7 @@ void writeFileExcl(const Path & path, Source & source, mode_t mode)
 {
     AutoCloseFD fd = openForWriteExcl(path, mode);
 
-    std::vector<char> buf(64 * 1024);
+    std::vector<char> buf(64ul * 1024);
 
     try {
         while (true) {
@@ -483,7 +483,7 @@ kj::Promise<Result<void>> writeFile(const Path & path, AsyncInputStream & source
 try {
     AutoCloseFD fd = openForWrite(path, mode);
 
-    std::vector<char> buf(64 * 1024);
+    std::vector<char> buf(64ul * 1024);
 
     try {
         while (true) {

lix/libutil/hash.cc

@@ -191,9 +191,8 @@ Hash::Hash(std::string_view rest, HashType type, bool isSRI)
         };
 
         for (unsigned int i = 0; i < hashSize; i++) {
-            hash[i] =
-                parseHexDigit(rest[i * 2]) << 4
-                | parseHexDigit(rest[i * 2 + 1]);
+            const size_t j = i << 1;
+            hash[i] = parseHexDigit(rest[j]) << 4 | parseHexDigit(rest[j + 1]);
         }
     }
 

lix/libutil/io-buffer.hh

@@ -19,7 +19,7 @@ class IoBuffer
     std::unique_ptr<char[]> buffer;
 
 public:
-    explicit IoBuffer(size_t bufSize = 32 * 1024) : bufSize(bufSize) {}
+    explicit IoBuffer(size_t bufSize = 32ul * 1024) : bufSize(bufSize) {}
 
     size_t size() const
     {

lix/libutil/logging-rpc.cc

@@ -107,8 +107,8 @@ struct RpcLogger : Logger
         }
         buffer->sizeEstimate += sizeof(e) + extraSize;
         buffer->items.emplace_back(std::move(e));
-        return buffer->sizeEstimate >= 1024 * 1024 ? BufferState::NeedsFlush
-                                                   : BufferState::HasSpace;
+        return buffer->sizeEstimate >= static_cast<size_t>(1024 * 1024) ? BufferState::NeedsFlush
+                                                                        : BufferState::HasSpace;
     }
 
     BufferState log(Verbosity lvl, std::string_view s) override

lix/libutil/meson.build

@@ -158,7 +158,6 @@ experimental_feature_definitions = files(
   'experimental-features/cgroups.md',
   'experimental-features/coerce-integers.md',
   'experimental-features/daemon-trust-override.md',
-  'experimental-features/fetch-closure.md',
   'experimental-features/flake-self-attrs.md',
   'experimental-features/flakes.md',
   'experimental-features/lix-custom-sub-commands.md',
@@ -173,6 +172,7 @@ deprecated_feature_definitions = files(
   # keep-sorted start
   'deprecated-features/ancient-let.md',
   'deprecated-features/cr-line-endings.md',
+  'deprecated-features/nix-path-shadow.md',
   'deprecated-features/nul-bytes.md',
   'deprecated-features/rec-set-overrides.md',
   'deprecated-features/shadow-internal-symbols.md',

lix/libutil/processes.cc

@@ -214,7 +214,7 @@ Pid startProcess(std::function<void()> fun, const ProcessOptions & options)
         // Not supported, since then we don't know when to free the stack.
         assert(!(options.cloneFlags & CLONE_VM));
 
-        size_t stackSize = 1 * 1024 * 1024;
+        size_t stackSize = 1ul * 1024 * 1024;
         auto stack = static_cast<char *>(mmap(0, stackSize,
             PROT_WRITE | PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0));
         if (stack == MAP_FAILED) throw SysError("allocating stack");

lix/libutil/serialise.hh

@@ -53,7 +53,7 @@ struct BufferedSink : virtual Sink
 {
     ref<IoBuffer> buffer;
 
-    BufferedSink(size_t bufSize = 32 * 1024) : buffer(make_ref<IoBuffer>(bufSize)) {}
+    BufferedSink(size_t bufSize = 32ul * 1024) : buffer(make_ref<IoBuffer>(bufSize)) {}
     explicit BufferedSink(ref<IoBuffer> buffer) : buffer(std::move(buffer)) {}
 
     void operator () (std::string_view data) override;
@@ -105,7 +105,7 @@ struct BufferedSource : Source
 {
     ref<IoBuffer> buffer;
 
-    BufferedSource(size_t bufSize = 32 * 1024) : buffer(make_ref<IoBuffer>(bufSize)) {}
+    BufferedSource(size_t bufSize = 32ul * 1024) : buffer(make_ref<IoBuffer>(bufSize)) {}
     explicit BufferedSource(ref<IoBuffer> buffer) : buffer(std::move(buffer)) {}
 
     size_t read(char * data, size_t len) override;

lix/libutil/tarfile.cc

@@ -1,9 +1,11 @@
 #include <archive.h>
 #include <archive_entry.h>
 #include <kj/async.h>
+#include <map>
 
 #include "async-io.hh"
 #include "file-descriptor.hh"
+#include "lix/libstore/temporary-dir.hh"
 #include "lix/libutil/c-calls.hh"
 #include "lix/libutil/charptr-cast.hh"
 #include "lix/libutil/file-system.hh"
@@ -45,67 +47,110 @@ void TarArchive::check(int err, const std::string & reason)
     if (err == ARCHIVE_EOF)
         throw EndOfFile("reached end of archive");
     else if (err != ARCHIVE_OK)
-        throw Error(reason, archive_error_string(this->archive));
+        throw Error(reason, archive_error_string(this->archive.get()));
 }
 
-TarArchive::TarArchive(Source & source, bool raw) : buffer(65536)
+TarArchive::TarArchive(Source & source, bool raw)
+    : archive{archive_read_new(), archive_read_free}
+    , source(&source)
+    , buffer(65536)
 {
-    this->archive = archive_read_new();
-    this->source = &source;
-
     if (!raw) {
-        archive_read_support_filter_all(archive);
-        archive_read_support_format_all(archive);
+        archive_read_support_filter_all(archive.get());
+        archive_read_support_format_all(archive.get());
     } else {
-        archive_read_support_filter_all(archive);
-        archive_read_support_format_raw(archive);
-        archive_read_support_format_empty(archive);
+        archive_read_support_filter_all(archive.get());
+        archive_read_support_format_raw(archive.get());
+        archive_read_support_format_empty(archive.get());
     }
-    archive_read_set_option(archive, nullptr, "mac-ext", nullptr);
-    check(archive_read_open(archive, (void *)this, callback_open, callback_read, callback_close), "Failed to open archive (%s)");
+    archive_read_set_option(archive.get(), nullptr, "mac-ext", nullptr);
+    check(
+        archive_read_open(
+            archive.get(), (void *) this, callback_open, callback_read, callback_close
+        ),
+        "Failed to open archive (%s)"
+    );
 }
 
-
-TarArchive::TarArchive(const Path & path)
+TarArchive::TarArchive(const Path & path) : archive{archive_read_new(), archive_read_free}
 {
-    this->archive = archive_read_new();
-
-    archive_read_support_filter_all(archive);
-    archive_read_support_format_all(archive);
-    archive_read_set_option(archive, nullptr, "mac-ext", nullptr);
+    archive_read_support_filter_all(archive.get());
+    archive_read_support_format_all(archive.get());
+    archive_read_set_option(archive.get(), nullptr, "mac-ext", nullptr);
     check(
-        archive_read_open_filename(archive, requireCString(path), 16384),
+        archive_read_open_filename(archive.get(), requireCString(path), 16384),
         "failed to open archive: %s"
     );
 }
 
 void TarArchive::close()
 {
-    check(archive_read_close(this->archive), "Failed to close archive (%s)");
+    check(archive_read_close(this->archive.get()), "Failed to close archive (%s)");
 }
 
-TarArchive::~TarArchive()
+namespace {
+
+struct IndexEntry
 {
-    if (this->archive) archive_read_free(this->archive);
+    struct IndexDirectory
+    {
+        std::map<std::string, IndexEntry> entries;
+    };
+
+    struct IndexFile
+    {
+        uint64_t start;
+        uint64_t size;
+    };
+
+    std::variant<IndexDirectory, IndexFile> innerData;
+    int mode;
+};
+
+/** Extract a TAR archive into an in-memory index with data stored on disk,
+ * because fundamentally all we have to do when importing archives to the store
+ * is to sort the entries and then send it into a NAR serializer.
+ *
+ * This deals with macOS performance problems by not materializing the files to
+ * disk in the first place.
+ *
+ * https://git.lix.systems/lix-project/lix/issues/1072
+ */
+struct InMemoryIndex
+{
+    Path tempDir;
+    AutoDelete tempDirDeleter;
+    AutoCloseFD backingFile;
+    std::ofstream writer;
+
+    IndexEntry topLevel;
+
+    InMemoryIndex() : tempDir(createTempDir()), tempDirDeleter(tempDir, true)
+    {
+        backingFile = sys::open(tempDir + "/extracted", O_RDWR | O_TRUNC | O_CLOEXEC);
+    }
+};
+
 }
 
 static void extract_archive(TarArchive & archive, const Path & destDir)
 {
     requireCString(destDir);
 
-    int flags = ARCHIVE_EXTRACT_TIME
-        | ARCHIVE_EXTRACT_SECURE_SYMLINKS
-        | ARCHIVE_EXTRACT_SECURE_NODOTDOT;
+    int flags = ARCHIVE_EXTRACT_TIME | ARCHIVE_EXTRACT_SECURE_SYMLINKS
+        | ARCHIVE_EXTRACT_SECURE_NODOTDOT | ARCHIVE_EXTRACT_NO_HFS_COMPRESSION;
 
     for (;;) {
         struct archive_entry * entry;
-        int r = archive_read_next_header(archive.archive, &entry);
+        int r = archive_read_next_header(archive.archive.get(), &entry);
         if (r == ARCHIVE_EOF) break;
         auto name = archive_entry_pathname(entry);
         if (!name)
-            throw Error("cannot get archive member name: %s", archive_error_string(archive.archive));
+            throw Error(
+                "cannot get archive member name: %s", archive_error_string(archive.archive.get())
+            );
         if (r == ARCHIVE_WARN)
-            printTaggedWarning("%1%", Uncolored(archive_error_string(archive.archive)));
+            printTaggedWarning("%1%", Uncolored(archive_error_string(archive.archive.get())));
         else
             archive.check(r);
 
@@ -125,7 +170,7 @@ static void extract_archive(TarArchive & archive, const Path & destDir)
                 (destDir + "/" + original_hardlink).c_str());
         }
 
-        archive.check(archive_read_extract(archive.archive, entry, flags));
+        archive.check(archive_read_extract(archive.archive.get(), entry, flags));
     }
 
     archive.close();

lix/libutil/tarfile.hh

@@ -8,7 +8,7 @@
 namespace nix {
 
 struct TarArchive {
-    struct archive * archive;
+    std::unique_ptr<struct archive, decltype(&archive_read_free)> archive;
     Source * source;
     std::vector<unsigned char> buffer;
 
@@ -22,8 +22,6 @@ struct TarArchive {
     TarArchive(const TarArchive &) = delete;
 
     void close();
-
-    ~TarArchive();
 };
 
 kj::Promise<Result<void>> unpackTarfile(AsyncInputStream & source, const Path & destDir);

lix/libutil/types.hh

@@ -11,6 +11,14 @@
 #include <span>
 #include <stdint.h> // IWYU pragma: keep (this is used literally everywhere)
 
+// NOTE: This check is load-bearing, c.f.
+//       https://git.lix.systems/lix-project/lix/commit/96a36833084f66b79ba778e719862b8219557773
+//       It is put here so that every consumer of our headers are prohibited from
+//       compiling with NDEBUG as some asserts are used in inline code
+#ifdef NDEBUG
+#error "Lix may not be built with assertions disabled (i.e. with -DNDEBUG)."
+#endif
+
 namespace nix {
 
 typedef std::list<std::string> Strings;

lix/libutil/users.cc

@@ -73,8 +73,24 @@ Path getHome()
 
 Path getCacheDir()
 {
-    auto cacheDir = getEnv("XDG_CACHE_HOME");
-    return cacheDir ? *cacheDir : getHome() + "/.cache";
+    // We follow systemd semantics here:
+    // https://www.freedesktop.org/software/systemd/man/latest/systemd.exec.html#RuntimeDirectory=
+    static auto cacheDir = [] {
+        auto userCacheDir = getEnv("XDG_CACHE_HOME");
+        auto serviceCacheDir = getEnv("CACHE_DIRECTORY");
+
+        if (serviceCacheDir) {
+            return *serviceCacheDir;
+        }
+
+        if (userCacheDir) {
+            return *userCacheDir;
+        }
+
+        return getHome() + "/.cache";
+    }();
+
+    return cacheDir;
 }
 
 

lix/meson.build

@@ -1,7 +1,17 @@
+# Cursed, but I don't think there's another way to get this environment variable.
+lix_suffix = run_command('bash', '-c', 'echo -n "$VERSION_SUFFIX"', check : true).stdout().strip()
+lix_version_parts = meson.project_version().split('.')
+lix_major = lix_version_parts[0]
+lix_minor = lix_version_parts[1]
+lix_patch = lix_version_parts[2].replace(lix_suffix, '')
+
 config_h = configure_file(
   configuration : {
     'PACKAGE_NAME': '"' + meson.project_name() + '"',
     'PACKAGE_VERSION': '"' + meson.project_version() + '"',
+    'LIX_MAJOR': lix_major,
+    'LIX_MINOR': lix_minor,
+    'LIX_PATCH': lix_patch,
     'PACKAGE_TARNAME': '"' + meson.project_name() + '"',
     'PACKAGE_STRING': '"' + meson.project_name() + ' ' + meson.project_version() + '"',
     'HAVE_STRUCT_DIRENT_D_TYPE': 1, # FIXME: actually check this for solaris

lix/nix/daemon.cc

@@ -55,7 +55,6 @@
 #endif
 
 static constexpr int SUBDAEMON_CONNECTION_FD = 0;
-static constexpr int SUBDAEMON_SETTINGS_FD = 3;
 
 namespace nix {
 
@@ -346,9 +345,6 @@ try {
                 peer.pidKnown ? fmt("pid %1%", peer.pid) : "unknown peer"
             );
 
-            Pipe settings;
-            settings.create();
-
             // Fork a child to handle the connection. make sure it's called with
             // argv0 `nix-daemon` so we don't try to run `nix --for` when called
             // from more modern scripts that assume nix-command being available.
@@ -363,11 +359,7 @@ try {
                         fmt("%1%", int(verbosity)),
                     },
                 .dieWithParent = false,
-                .redirections =
-                    {
-                        {.dup = SUBDAEMON_CONNECTION_FD, .from = remote.get()},
-                        {.dup = SUBDAEMON_SETTINGS_FD, .from = settings.readSide.get()},
-                    }
+                .redirections = {{.dup = SUBDAEMON_CONNECTION_FD, .from = remote.get()}}
             };
             if (forceTrustClientOpt) {
                 options.args.push_back(
@@ -375,15 +367,6 @@ try {
                 );
             }
             runProgram2(options).release();
-
-            FdSink sink(settings.writeSide.get());
-            std::map<std::string, Config::SettingInfo> overriddenSettings;
-            globalConfig.getSettings(overriddenSettings, true);
-            for (auto & setting : overriddenSettings) {
-                sink << 1 << setting.first << setting.second.value;
-            }
-            sink << 0;
-            sink.flush();
         } catch (Error & error) {
             auto ei = error.info();
             // FIXME: add to trace?
@@ -417,21 +400,6 @@ static void daemonInstance(AsyncIoRoot & aio, std::optional<TrustedFlag> forceTr
         forceTrustClientOpt ? " by override" : ""
     );
 
-    {
-        FdSource source(SUBDAEMON_SETTINGS_FD);
-
-        /* Read the parent's settings. */
-        while (readNum<unsigned>(source)) {
-            auto name = readString(source);
-            auto value = readString(source);
-            settings.set(name, value);
-        }
-
-        if (close(SUBDAEMON_SETTINGS_FD) < 0) {
-            throw SysError("preparing subdaemon connection");
-        }
-    }
-
     //  Background the daemon.
     if (setsid() == -1) {
         throw SysError("creating a new session");

lix/nix/develop.cc

@@ -257,6 +257,10 @@ try {
         output.second = DerivationOutput::InputAddressed { .path = StorePath::dummy };
         drv.env[output.first] = "";
     }
+
+    // outputs default to [ "out" ]
+    drv.env.try_emplace("outputs", "out");
+
     auto hashesModulo = TRY_AWAIT(hashDerivationModulo(*evalStore, drv, true));
 
     for (auto & output : drv.outputs) {
@@ -352,10 +356,17 @@ struct Common : InstallableCommand, MixProfile
 
         buildEnvironment.toBash(out, ignoreVars);
 
+        // Add NIX_SHELL_LEVEL
+        out << "printf -v NIX_SHELL_LEVEL \"%d\" \"${NIX_SHELL_LEVEL:-0}\" || NIX_SHELL_LEVEL=0\n";
+        out << "export NIX_SHELL_LEVEL=$(( NIX_SHELL_LEVEL + 1 ))\n";
+
         for (auto & var : savedVars)
             out << fmt("%s=\"$%s${nix_saved_%s:+:$nix_saved_%s}\"\n", var, var, var, var);
 
         out << "export NIX_BUILD_TOP=\"$(mktemp -d -t nix-shell.XXXXXX)\"\n";
+        // We re-export similarly to what occurs inside of a derivation goal `NIX_LOG_FD` to stderr.
+        // So that stdenv hooks that logs information can be observed inside this debugging tool.
+        out << "export NIX_LOG_FD=2\n";
         for (auto & i : {"TMP", "TMPDIR", "TEMP", "TEMPDIR"})
             out << fmt("export %s=\"$NIX_BUILD_TOP\"\n", i);
 

lix/nix/diff-closures.cc

@@ -152,7 +152,7 @@ void renderDiffInfo(
     const std::string_view indent)
 {
     for (auto & [name, item] : diff) {
-        auto showDelta = std::abs(item.sizeDelta) >= 8 * 1024;
+        auto showDelta = std::abs(item.sizeDelta) >= 8l * 1024;
 
         std::vector<std::string> line;
         if (!item.removedVersions.empty() || !item.addedVersions.empty())

lix/nix/eval.cc

@@ -110,7 +110,7 @@ struct CmdEval : MixJSON, InstallableCommand, MixReadOnlyOption
                     v,
                     PrintOptions{
                         .force = true,
-                        .derivationPaths = true,
+                        .derivationPathDepth = 0,
                         .errors = ErrorPrintBehavior::ThrowTopLevel,
                     }
                 )

lix/nix/main.cc

@@ -455,8 +455,6 @@ void registerNixHelp()
 
 int mainWrapped(AsyncIoRoot & aio, int argc, char ** argv)
 {
-    savedArgv = argv;
-
     /* The chroot helper needs to be run before any threads have been
        started. */
     if (argc > 0 && argv[0] == chrootHelperName) {
@@ -493,14 +491,19 @@ int mainWrapped(AsyncIoRoot & aio, int argc, char ** argv)
         registerLegacyCommands();
         auto legacy = (*LegacyCommandRegistry::commands)[programName];
         if (legacy) {
-            return legacy(aio, std::string(baseNameOf(argv[0])), Strings(argv + 1, argv + argc));
+            return legacy(
+                aio,
+                std::string(baseNameOf(argv[0])),
+                Strings(argv + 1, argv + argc),
+                {argv + 1, argv + argc}
+            );
         }
     }
 
     evalSettings.pureEval.setDefault(true);
 
-    setLogFormat(LogFormat::bar);
-    settings.verboseBuild = false;
+    setLogFormat(LogFormat::barWithLogs);
+
     // FIXME: stop messing about with log verbosity depending on if it is interactive use
     if (isatty(STDERR_FILENO)) {
         verbosity = lvlNotice;
@@ -521,8 +524,7 @@ int mainWrapped(AsyncIoRoot & aio, int argc, char ** argv)
 
     if (argc == 2 && std::string(argv[1]) == "__dump-language") {
         experimentalFeatureSettings.experimentalFeatures.override(ExperimentalFeatures{}
-            | Xp::Flakes
-            | Xp::FetchClosure);
+            | Xp::Flakes);
         evalSettings.pureEval.override(false);
         Evaluator state(aio, {}, aio.blockOn(openStore("dummy://")));
         auto res = JSON::object();
@@ -646,7 +648,6 @@ int mainWrapped(AsyncIoRoot & aio, int argc, char ** argv)
 
     return 0;
 }
-
 }
 
 int main(int argc, char * * argv)
@@ -658,7 +659,7 @@ int main(int argc, char * * argv)
 
     // Increase the default stack size for the evaluator and for
     // libstdc++'s std::regex.
-    nix::setStackSize(64 * 1024 * 1024);
+    nix::setStackSize(64ul * 1024 * 1024);
 
     return nix::handleExceptions(argv[0], [&]() {
         nix::AsyncIoRoot aio;

lix/nix/make-content-addressed.cc

@@ -12,6 +12,7 @@ struct CmdMakeContentAddressed : virtual CopyCommand, virtual StorePathsCommand,
     CmdMakeContentAddressed()
     {
         realiseMode = Realise::Outputs;
+        requireStore = false;
     }
 
     std::string description() override
@@ -28,7 +29,7 @@ struct CmdMakeContentAddressed : virtual CopyCommand, virtual StorePathsCommand,
 
     void run(ref<Store> srcStore, StorePaths && storePaths) override
     {
-        auto dstStore = aio().blockOn(dstUri.empty() ? openStore() : openStore(dstUri));
+        auto dstStore = getDstStore();
 
         auto remappings = aio().blockOn(makeContentAddressed(*srcStore, *dstStore,
             StorePathSet(storePaths.begin(), storePaths.end())));

lix/nix/run.cc

@@ -150,6 +150,17 @@ struct CmdShell : InstallablesCommand, MixEnvironment
         auto unixPathString = concatStringsSep(":", unixPath);
         (void) sys::setenv("PATH", unixPathString, 1);
         (void) sys::setenv("IN_NIX_SHELL", ignoreEnvironment ? "pure" : "impure", 1);
+        // Set NIX_SHELL_LEVEL
+        (void) sys::setenv(
+            "NIX_SHELL_LEVEL",
+            std::to_string(
+                getEnvNonEmpty("NIX_SHELL_LEVEL")
+                    .and_then([](std::string lvl) { return string2Int<size_t>(lvl); })
+                    .value_or(0)
+                + 1
+            ),
+            1
+        );
 
         Strings args;
         for (auto & arg : command) args.push_back(arg);
@@ -230,6 +241,7 @@ void registerNixRun()
 
 void chrootHelper(int argc, char * * argv)
 {
+#if __linux__
     int p = 1;
     std::string storeDir = argv[p++];
     std::string realStoreDir = argv[p++];
@@ -239,7 +251,6 @@ void chrootHelper(int argc, char * * argv)
     while (p < argc)
         args.push_back(argv[p++]);
 
-#if __linux__
     uid_t uid = getuid();
     uid_t gid = getgid();
 
@@ -310,6 +321,7 @@ void chrootHelper(int argc, char * * argv)
     throw SysError("unable to exec '%s'", cmd);
 
 #else
+    std::string storeDir = argv[1];
     throw Error("mounting the Nix store on '%s' is not supported on this platform", storeDir);
 #endif
 }

lix/nix/store-delete.cc

@@ -58,7 +58,7 @@ struct CmdStoreDelete : StorePathsCommand
         }
 
         GCResults results;
-        PrintFreed freed(true, results);
+        PrintFreed freed(options.action, results);
         aio().blockOn(gcStore.collectGarbage(options, results));
     }
 };