chore(libmain): license it under LGPL-2.1-only

Progress towards REUSE compliance.

Change-Id: I2991234cfd193773a377d2f1735b9974918b8138
Signed-off-by: Raito Bezarius <raito@lix.systems>

.envrc

@@ -1,9 +1,5 @@
 # shellcheck shell=bash
 source_env_if_exists .envrc.local
-# TODO: `use flake .#native-clangStdenvPackages` on macOS?
-use flake ".#${LIX_SHELL_VARIANT:-default}" "${LIX_SHELL_EXTRA_ARGS[@]}"
-export MAKEFLAGS="$MAKEFLAGS -e"
-if [[ -n "$NIX_BUILD_CORES" ]]; then
-    export MAKEFLAGS="$MAKEFLAGS -j $NIX_BUILD_CORES"
-fi
+# Use native-clangStdenvPackages to get clangd by default.
+use flake ".#${LIX_SHELL_VARIANT:-native-clangStdenvPackages}" "${LIX_SHELL_EXTRA_ARGS[@]}"
 export GTEST_BRIEF=1

.this-is-lix

@@ -0,0 +1 @@
+This is a file used by the dev shell shellHook in package.nix to check that this is actually a Lix repo before installing git hooks. Its contents have no meaning.

doc/manual/change-authors.yml

@@ -3,6 +3,10 @@
 #
 # It's used for crediting people accurately in release notes. The release notes
 # script will link to forgejo, then to GitHub if forgejo is not present.
+#
+# When adding someone from outside the Lix project, you generally want to simply link their GitHub profile without adding a display name unless they are well-known in the community by that display name.
+#
+# See doc/manual/src/contributing/hacking.md for more documentation on this file's format and typical usage.
 9999years:
   display_name: wiggles
   forgejo: rbt
@@ -40,6 +44,11 @@ cole-h:
   display_name: Cole Helbling
   github: cole-h
 
+delan:
+  display_name: delan
+  forgejo: delan
+  github: delan
+
 edolstra:
   display_name: Eelco Dolstra
   github: edolstra
@@ -60,6 +69,10 @@ jade:
   forgejo: jade
   github: lf-
 
+kloenk:
+  forgejo: kloenk
+  github: kloenk
+
 lovesegfault:
   github: lovesegfault
 
@@ -83,6 +96,10 @@ puck:
   forgejo: puck
   github: puckipedia
 
+quantumjump:
+  display_name: Quantum Jump
+  github: QuantumBJump
+
 r-vdp:
   github: r-vdp
 

doc/manual/meson.build

@@ -110,6 +110,7 @@ manual = custom_target(
     builtins_md,
     builtin_constants_md,
     rl_next_generated,
+    nix,
   ],
   output : [
     'manual',
@@ -186,6 +187,7 @@ foreach command : nix_nested_manpages
       ],
       input : [
         manual_md,
+        nix,
       ],
       output : command[0] + '-' + page + '.1',
       install : true,
@@ -298,6 +300,7 @@ foreach page : nix3_manpages
     input : [
       'render-manpage.sh',
       manual_md,
+      nix,
     ],
     output : page + '.1',
     install : true,
@@ -341,6 +344,7 @@ foreach entry : nix_manpages
       'render-manpage.sh',
       manual_md,
       entry.get(3, []),
+      nix,
     ],
     output : '@0@.@1@'.format(entry[0], entry[1]),
     install : true,

doc/manual/process-includes.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/usr/bin/env bash
 
 set -euo pipefail
 

doc/manual/render-manpage.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/usr/bin/env bash
 
 set -euo pipefail
 

doc/manual/rl-next/better-attrpath-errors.md

@@ -0,0 +1,90 @@
+---
+synopsis: "Trace which part of a `foo.bar.baz` expression errors"
+cls: 1505, 1506
+credits: Qyriad
+category: Improvements
+---
+
+Previously, if an attribute path selection expression like `linux_4_9.meta.description` it wouldn't show you which one of those parts in the attribute path, or even that that line of code is what caused evaluation of the failing expression.
+The previous error looks like this:
+
+```
+pkgs.linuxKernel.kernels.linux_4_9.meta.description
+
+error:
+       … while evaluating the attribute 'linuxKernel.kernels.linux_4_9.meta.description'
+         at /nix/store/dk2rpyb6ndvfbf19bkb2plcz5y3k8i5v-source/pkgs/top-level/linux-kernels.nix:278:5:
+          277|   } // lib.optionalAttrs config.allowAliases {
+          278|     linux_4_9 = throw "linux 4.9 was removed because it will reach its end of life within 22.11";
+             |     ^
+          279|     linux_4_14 = throw "linux 4.14 was removed because it will reach its end of life within 23.11";
+
+       … while calling the 'throw' builtin
+         at /nix/store/dk2rpyb6ndvfbf19bkb2plcz5y3k8i5v-source/pkgs/top-level/linux-kernels.nix:278:17:
+          277|   } // lib.optionalAttrs config.allowAliases {
+          278|     linux_4_9 = throw "linux 4.9 was removed because it will reach its end of life within 22.11";
+             |                 ^
+          279|     linux_4_14 = throw "linux 4.14 was removed because it will reach its end of life within 23.11";
+
+       error: linux 4.9 was removed because it will reach its end of life within 22.11
+```
+
+Now, the error will look like this:
+
+```
+pkgs.linuxKernel.kernels.linux_4_9.meta.description
+
+error:
+       … while evaluating the attribute 'linuxKernel.kernels.linux_4_9.meta.description'
+         at /nix/store/dk2rpyb6ndvfbf19bkb2plcz5y3k8i5v-source/pkgs/top-level/linux-kernels.nix:278:5:
+          277|   } // lib.optionalAttrs config.allowAliases {
+          278|     linux_4_9 = throw "linux 4.9 was removed because it will reach its end of life within 22.11";
+             |     ^
+          279|     linux_4_14 = throw "linux 4.14 was removed because it will reach its end of life within 23.11";
+
+       … while evaluating 'pkgs.linuxKernel.kernels.linux_4_9' to select 'meta' on it
+         at «string»:1:1:
+            1| pkgs.linuxKernel.kernels.linux_4_9.meta.description
+             | ^
+
+       … caused by explicit throw
+         at /nix/store/dk2rpyb6ndvfbf19bkb2plcz5y3k8i5v-source/pkgs/top-level/linux-kernels.nix:278:17:
+          277|   } // lib.optionalAttrs config.allowAliases {
+          278|     linux_4_9 = throw "linux 4.9 was removed because it will reach its end of life within 22.11";
+             |                 ^
+          279|     linux_4_14 = throw "linux 4.14 was removed because it will reach its end of life within 23.11";
+
+       error: linux 4.9 was removed because it will reach its end of life within 22.11
+```
+
+Not only does the line of code that referenced the failing attribute show up in the trace, it also tells you that it was specifically the `linux_4_9` part that failed.
+
+This includes if the failing part is a top-level binding:
+
+```
+let
+  inherit (pkgs.linuxKernel.kernels) linux_4_9;
+in linux_4_9.meta.description
+error:
+       … while evaluating 'linux_4_9' to select 'meta.description' on it
+         at «string»:3:4:
+            2|   inherit (pkgs.linuxKernel.kernels) linux_4_9;
+            3| in linux_4_9.meta.description
+             |    ^
+
+       … while evaluating the attribute 'linux_4_9'
+         at /nix/store/dk2rpyb6ndvfbf19bkb2plcz5y3k8i5v-source/pkgs/top-level/linux-kernels.nix:278:5:
+          277|   } // lib.optionalAttrs config.allowAliases {
+          278|     linux_4_9 = throw "linux 4.9 was removed because it will reach its end of life within 22.11";
+             |     ^
+          279|     linux_4_14 = throw "linux 4.14 was removed because it will reach its end of life within 23.11";
+
+       … caused by explicit throw
+         at /nix/store/dk2rpyb6ndvfbf19bkb2plcz5y3k8i5v-source/pkgs/top-level/linux-kernels.nix:278:17:
+          277|   } // lib.optionalAttrs config.allowAliases {
+          278|     linux_4_9 = throw "linux 4.9 was removed because it will reach its end of life within 22.11";
+             |                 ^
+          279|     linux_4_14 = throw "linux 4.14 was removed because it will reach its end of life within 23.11";
+
+       error: linux 4.9 was removed because it will reach its end of life within 22.11
+```

doc/manual/rl-next/build-dir.md

@@ -0,0 +1,12 @@
+---
+synopsis: "Add a `build-dir` setting to set the backing directory for builds"
+cls: 1514
+credits: [roberth, tomberek]
+category: Improvements
+---
+
+`build-dir` can now be set in the Nix configuration to choose the backing directory for the build sandbox.
+This can be useful on systems with `/tmp` on tmpfs, or simply to relocate large builds to another disk.
+
+Also, `XDG_RUNTIME_DIR` is no longer considered when selecting the default temporary directory,
+as it's not intended to be used for large amounts of data.

doc/manual/rl-next/distinguish-throw-errors.md

@@ -0,0 +1,70 @@
+---
+synopsis: "Distinguish between explicit throws and errors that happened while evaluating a throw"
+cls: 1511
+credits: Qyriad
+category: Improvements
+---
+
+Previously, errors caused by an expression like `throw "invalid argument"` were treated like an error that happened simply while some builtin function was being called:
+
+```
+let
+  throwMsg = p: throw "${p} isn't the right package";
+in throwMsg "linuz"
+
+error:
+       … while calling the 'throw' builtin
+         at «string»:2:17:
+            1| let
+            2|   throwMsg = p: throw "${p} isn't the right package";
+             |                 ^
+            3| in throwMsg "linuz"
+
+       error: linuz isn't the right package
+```
+
+But the error didn't just happen "while" calling the `throw` builtin — it's a throw error!
+Now it looks like this:
+
+```
+let
+  throwMsg = p: throw "${p} isn't the right package";
+in throwMsg "linuz"
+
+error:
+       … caused by explicit throw
+         at «string»:2:17:
+            1| let
+            2|   throwMsg = p: throw "${p} isn't the right package";
+             |                 ^
+            3| in throwMsg "linuz"
+
+       error: linuz isn't the right package
+```
+
+This also means that incorrect usage of `throw` or errors evaluating its arguments are easily distinguishable from explicit throws:
+
+```
+let
+  throwMsg = p: throw "${p} isn't the right package";
+in throwMsg { attrs = "error when coerced in string interpolation"; }
+
+error:
+       … while calling the 'throw' builtin
+         at «string»:2:17:
+            1| let
+            2|   throwMsg = p: throw "${p} isn't the right package";
+             |                 ^
+            3| in throwMsg { attrs = "error when coerced in string interpolation"; }
+
+       … while evaluating a path segment
+         at «string»:2:24:
+            1| let
+            2|   throwMsg = p: throw "${p} isn't the right package";
+             |                        ^
+            3| in throwMsg { attrs = "error when coerced in string interpolation"; }
+
+       error: cannot coerce a set to a string: { attrs = "error when coerced in string interpolation"; }
+```
+
+Here, instead of an actual thrown error, a type error happens first (trying to coerce an attribute set to a string), but that type error happened *while* calling `throw`.

doc/manual/rl-next/fix-gc-dry-run.md

@@ -0,0 +1,29 @@
+---
+synopsis: Fix nix-collect-garbage --dry-run
+issues: [fj#432]
+cls: [1566]
+category: Fixes
+credits: [quantumjump]
+---
+
+`nix-collect-garbage --dry-run` did not previously give any output - it simply
+exited without even checking to see what paths would be deleted.
+
+```
+$ nix-collect-garbage --dry-run
+$
+```
+
+We updated the behaviour of the flag such that instead it prints out how many
+paths it *would* delete, but doesn't actually delete them.
+
+```
+$ nix-collect-garbage --dry-run
+finding garbage collector roots...
+determining live/dead paths...
+...
+<nix store paths>
+...
+2670 store paths deleted, 0.00MiB freed
+$
+```

doc/manual/rl-next/fod-failure-includes-url.md

@@ -0,0 +1,16 @@
+---
+synopsis: "Hash mismatch diagnostics for fixed-output derivations include the URL"
+cls: [1536]
+credits: [jade]
+category: Improvements
+---
+
+Now, when building fixed-output derivations, Lix will guess the URL that was used in the derivation using the `url` or `urls` properties in the derivation environment.
+This is a layering violation but making these diagnostics tractable when there are multiple instances of the `AAAA` hash is too significant of an improvement to pass it up.
+
+```
+error: hash mismatch in fixed-output derivation '/nix/store/sjfw324j4533lwnpmr5z4icpb85r63ai-x1.drv':
+        likely URL: https://meow.puppy.forge/puppy.tar.gz
+         specified: sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=
+            got:    sha256-a1Qvp3FOOkWpL9kFHgugU1ok5UtRPSu+NwCZKbbaEro=
+```

doc/manual/rl-next/multiline-log-format.md

@@ -0,0 +1,14 @@
+---
+synopsis: Add log formats `multiline` and `multiline-with-logs`
+cls: [1369]
+credits: [kloenk]
+category: Improvements
+---
+
+Added two new log formats (`multiline` and `multiline-with-logs`) that display
+current activities below each other for better visibility.
+
+These formats attempt to use the maximum available lines
+(defaulting to 25 if unable to determine) and print up to that many lines.
+The status bar is displayed as the first line, with each subsequent
+activity on its own line.

doc/manual/rl-next/nix-copy-is-fast.md

@@ -0,0 +1,12 @@
+---
+synopsis: "`nix copy` is now several times faster at `querying info about /nix/store/...`"
+cls: [1462]
+issues: [fj#366]
+credits: [jade]
+category: Fixes
+---
+
+We fixed a locking bug that serialized `querying info about /nix/store/...`
+onto just one thread such that it was eating `O(paths to copy * latency)` time
+while setting up to copy paths to s3 and other stores. It is now `nproc` times
+faster.

doc/manual/rl-next/old-protocol-removal.md

@@ -0,0 +1,21 @@
+---
+synopsis: "Lix no longer speaks the Nix remote-build worker protocol to clients or servers older than CppNix 2.3"
+cls: [1207, 1208, 1206, 1205, 1204, 1203, 1479]
+issues: [fj#325]
+credits: [jade]
+category: Breaking Changes
+---
+
+CppNix 2.3 was released in 2019, and is the new oldest supported version. We
+will increase our support baseline in the future up to a final version of CppNix
+2.18 (which may happen soon given that it is the only still-packaged and thus
+still-tested >2.3 version), but this step already removes a significant amount
+of dead, untested, code paths.
+
+Lix speaks the same version of the protocol as CppNix 2.18 and that fact will
+never change in the future; the Lix plans to replace the protocol for evolution
+will entail a complete incompatible replacement that will be supported in
+parallel with the old protocol. Lix will thus retain remote build compatibility
+with CppNix as long as CppNix maintains protocol compatibility with 2.18, and
+as long as Lix retains legacy protocol support (which will likely be a long
+time given that we plan to convert it to a frozen-in-time shim).

doc/manual/rl-next/registry-add-shorthand-only.md

@@ -0,0 +1,10 @@
+---
+synopsis: "`nix registry add` now requires a shorthand flakeref on the 'from' side"
+cls: 1494
+credits: delan
+category: Improvements
+---
+
+The 'from' argument must now be a shorthand flakeref like `nixpkgs` or `nixpkgs/nixos-20.03`, making it harder to accidentally swap the 'from' and 'to' arguments.
+
+Registry entries that map from other flake URLs can still be specified in registry.json, the `nix.registry` option in NixOS, or the `--override-flake` option in the CLI, but they are not guaranteed to work correctly.

doc/manual/rl-next/reject-flake-config.md

@@ -0,0 +1,9 @@
+---
+synopsis: Allow automatic rejection of configuration options from flakes
+cls: [1541]
+credits: [alois31]
+category: Improvements
+---
+
+Setting `accept-flake-config` to `false` now respects user choice by automatically rejecting configuration options set by flakes.
+The old behaviour of asking each time is still available (and default) by setting it to the special value `ask`.

doc/manual/rl-next/repl-complete-colon.md

@@ -0,0 +1,8 @@
+---
+synopsis: "`nix repl` now allows tab-completing the special repl :colon commands"
+cls: 1367
+credits: Qyriad
+category: Improvements
+---
+
+The REPL (`nix repl`) supports pressing `<TAB>` to complete a partial expression, but now also supports completing the special :colon commands as well (`:b`, `:edit`, `:doc`, etc), if the line starts with a colon.

doc/manual/rl-next/sanitizers.md

@@ -0,0 +1,10 @@
+---
+synopsis: "Lix now supports building with UndefinedBehaviorSanitizer"
+cls: [1483]
+credits: [jade]
+category: Development
+---
+
+You can now build Lix with the configuration option `-Db_sanitize=undefined` and it will both work and pass tests. AddressSanitizer support is also coming soon.
+
+For a list of undefined behaviour fixed by sanitizer usage, see [the gerrit topic "undefined-behaviour"](https://gerrit.lix.systems/q/topic:%22undefined-behaviour%22).

doc/manual/src/SUMMARY.md

@@ -197,7 +197,7 @@
 - [Release Notes](release-notes/release-notes.md)
   - [Upcoming release](release-notes/rl-next.md)
   <!-- RELENG-AUTO-INSERTION-MARKER (see releng/release_notes.py) -->
-  - [Lix 2.90 (2024-07-10)](release-notes/rl-2.90.md)
+  - [Lix 2.90 (FIXME date)](release-notes/rl-2.90.md)
   - [Nix 2.18 (2023-09-20)](release-notes/rl-2.18.md)
   - [Nix 2.17 (2023-07-24)](release-notes/rl-2.17.md)
   - [Nix 2.16 (2023-05-31)](release-notes/rl-2.16.md)

doc/manual/src/command-ref/opt-common.md

@@ -78,6 +78,16 @@ Most commands in Lix accept the following command-line options:
 
     Display the raw logs, with the progress bar at the bottom.
 
+  - `multiline`
+
+    Display a progress bar during the builds and in the lines below that one line per activity.
+
+
+  - `multiline-with-logs`
+
+    Displayes the raw logs, with a progress bar and activities each in a new line at the bottom.
+
+
 - <span id="opt-no-build-output">[`--no-build-output`](#opt-no-build-output)</span> / `-Q`
 
   By default, output written by builders to standard output and standard error is echoed to the Lix command's standard error.

doc/manual/src/contributing/hacking.md

@@ -39,17 +39,19 @@ $ nix-shell -A native-clangStdenvPackages
 
 ### Building from the development shell
 
-As always you may run [stdenv's phases by name](https://nixos.org/manual/nixpkgs/unstable/#sec-building-stdenv-package-in-nix-shell), e.g.:
+You can build and test Lix with just:
 
 ```bash
-$ configurePhase
-$ buildPhase
-$ checkPhase
-$ installPhase
-$ installCheckPhase
+$ just setup
+$ just build
+$ just test --suite=check
+$ just install
+$ just test --suite=installcheck
 ```
 
-To build manually, however, use the following:
+(Check and installcheck may both be done after install, allowing you to omit the --suite argument entirely, but this is the order package.nix runs them in.)
+
+You can also build Lix manually:
 
 ```bash
 $ meson setup ./build "--prefix=$out" $mesonFlags
@@ -64,9 +66,7 @@ $ meson install -C build
 $ meson test -C build --suite=installcheck
 ```
 
-(Check and installcheck may both be done after install, allowing you to omit the --suite argument entirely, but this is the order package.nix runs them in.)
-
-This will install Lix to `$PWD/outputs`, the `/bin` of which is prepended to PATH in the development shells.
+In both cases, Lix will be installed to `$PWD/outputs`, the `/bin` of which is prepended to PATH in the development shells.
 
 If the tests fail and Meson helpfully has no output for why, use the `--print-error-logs` option to `meson test`.
 
@@ -168,8 +168,26 @@ or for Nix with the [`flakes`] and [`nix-command`] experimental features enabled
 $ nix build .#packages.aarch64-linux.default
 ```
 
-Cross-compiled builds are available for ARMv6 (`armv6l-linux`) and ARMv7 (`armv7l-linux`).
-Add more [system types](#system-type) to `crossSystems` in `flake.nix` to bootstrap Nix on unsupported platforms.
+### Cross compiling using the Lix flake
+
+Lix can also be easily cross compiled to the following arbitrarily-chosen system doubles, which can be useful for bootstrapping Lix on new platforms.
+These are specified in `crossSystems` in `flake.nix`; feel free to submit changes to add new ones if they are useful to you.
+
+- `armv6l-linux`
+- `armv7l-linux`
+- `riscv64-linux`
+
+For example, to cross-compile Lix for `armv6l-linux` from another Linux, use the following:
+
+```console
+$ nix build .#nix-armv6l-linux
+```
+
+It's also possible to cross-compile a tarball of binaries suitable for the Lix installer, for example, for `riscv64-linux`:
+
+```console
+$ nix build .#nix-riscv64-linux.passthru.binaryTarball
+```
 
 ### Building for multiple platforms at once
 
@@ -282,9 +300,8 @@ Regular markdown files used for the manual have a base path of their own and the
 
 ## API documentation
 
-Doxygen API documentation is [available
-online](https://hydra.nixos.org/job/nix/master/internal-api-docs/latest/download-by-type/doc/internal-api-docs). You
-can also build and view it yourself:
+Doxygen API documentation will be available online in the future ([tracking issue](https://git.lix.systems/lix-project/lix/issues/422)).
+You can also build and view it yourself:
 
 ```console
 # nix build .#hydraJobs.internal-api-docs
@@ -294,44 +311,50 @@ can also build and view it yourself:
 or inside a `nix develop` shell by running:
 
 ```bash
+$ meson configure build -Dinternal-api-docs=enabled
 $ meson compile -C build internal-api-docs
 $ xdg-open ./outputs/doc/share/doc/nix/internal-api/html/index.html
 ```
 
 ## Coverage analysis
 
-A coverage analysis report is [available
-online](https://hydra.nixos.org/job/nix/master/coverage/latest/download-by-type/report/coverage). You
-can build it yourself:
+A coverage analysis report will be available online in the future (FIXME(lix-hydra)).
+You can build it yourself:
 
 ```
 # nix build .#hydraJobs.coverage
 # xdg-open ./result/coverage/index.html
 ```
 
-Metrics about the change in line/function coverage over time are also
-[available](https://hydra.nixos.org/job/nix/master/coverage#tabs-charts).
+Metrics about the change in line/function coverage over time will be available in the future (FIXME(lix-hydra)).
 
 ## Add a release note
 
 `doc/manual/rl-next` contains release notes entries for all unreleased changes.
 
 User-visible changes should come with a release note.
+Developer-facing changes should have a release note in the Development category if they are significant and if developers should know about them.
 
 ### Add an entry
 
-Here's what a complete entry looks like. The file name is not incorporated in the document.
+Here's what a complete entry looks like.
+The file name is not incorporated in the final document, and is generally a super brief summary of the change synopsis.
 
-```
+```markdown
 ---
 synopsis: Basically a title
 # 1234 or gh#1234 will refer to CppNix GitHub, fj#1234 will refer to a Lix forgejo issue.
 issues: [1234, fj#1234]
-# Use this *only* if there is a CppNix pull request associated with this change
+# Use this *only* if there is a CppNix pull request associated with this change.
 prs: 1238
-# List of Lix Gerrit changelist numbers; if there is an associated Lix GitHub
-# PR, just put in the Gerrit CL number.
+# List of Lix Gerrit changelist numbers.
+# If there is an associated Lix GitHub PR, just put in the Gerrit CL number.
 cls: [123]
+# Heading that this release note will appear under.
+category: Breaking Changes
+# Add a credit mention in the bottom of the release note.
+# your-name is used as a key into doc/manual/change-authors.yml for metadata
+credits: [your-name]
 ---
 
 Here's one or more paragraphs that describe the change.
@@ -346,6 +369,31 @@ Significant changes should add the following header, which moves them to the top
 significance: significant
 ```
 
+The following categories of release notes are supported (see `maintainers/build-release-notes.py`):
+- Breaking Changes
+- Features
+- Improvements
+- Fixes
+- Packaging
+- Development
+- Miscellany
+
+The `credits` field, if present, gives credit to the author of the patch in the release notes with a message like "Many thanks to (your-name) for this" and linking to GitHub or Forgejo profiles if listed.
+
+If you are forward-porting a change from CppNix, please credit the original author, and optionally credit yourself.
+When adding credits metadata for people external to the project and deciding whether to put in a `display_name`, consider what they are generally known as in the community; even if you know their full name (e.g. from their GitHub profile), we suggest only adding it as a display name if that is what they go by in the community.
+There are multiple reasons we follow this practice, but it boils down to privacy and consent: we would rather not capture full names that are not widely used in the community without the consent of the parties involved, even if they are publicly available.
+As of this writing, the entries with full names as `display_name` are either members of the CppNix team or people who added them themselves.
+
+The names specified in `credits` are used as keys to look up the authorship info in `doc/manual/change-authors.yml`.
+The only mandatory part is that every key appearing in `credits` has an entry present in `change-authors.yml`.
+All of the following properties are optional; you can specify `{}` as the metadata if you want a simple non-hyperlinked mention.
+The following properties are supported:
+
+- `display_name`: display name used in place of the key when showing names, if present.
+- `forgejo`: Forgejo username. The name in the release notes will be a link to this, if present.
+- `github`: GitHub username, used if `forgejo` is not set, again making a link.
+
 ### Build process
 
 Releases have a precomputed `rl-MAJOR.MINOR.md`, and no `rl-next.md`.

doc/manual/src/contributing/testing.md

@@ -433,9 +433,6 @@ I grepped `src/` for `get[eE]nv\("` to find the mentions in Lix code.
 - `NIX_PROFILE` - Selects which profile `nix-env` will operate on. Documented elsewhere.
 - `NIX_SSHOPTS` - Options passed to `ssh(1)` when using a ssh remote store.
   Incorrectly documented on `nix-copy-closure` which is *surely* not the only place they are used??
-- `_NIX_TEST_NO_LSOF` - Used on non-Linux, non-macOS platforms to disable using `lsof` when finding gc roots.
-
-  Since https://git.lix.systems/lix-project/lix/issues/156 was fixed, this should probably just be removed as it was a bad workaround for a macOS issue.
 - `_NIX_TEST_GC_SYNC_1` - Path to a pipe that is used to block the GC briefly to validate invariants from the test suite.
 - `_NIX_TEST_GC_SYNC_2` - Path to a pipe that is used to block the GC briefly to validate invariants from the test suite.
 - `_NIX_TEST_FREE_SPACE_FILE` - Path to a file containing a decimal number with the free space that the GC is to believe it has.

doc/manual/src/release-notes/rl-2.90.md

@@ -1,7 +1,7 @@
-# Lix 2.90 "Vanilla Ice Cream" (2024-07-10)
+# Lix 2.90 "Vanilla Ice Cream" (FIXME date)
 
 
-# Lix 2.90.0 (2024-07-10)
+# Lix 2.90.0 (FIXME date)
 
 ## Breaking Changes
 - Deprecate the online flake registries and vendor the default registry [fj#183](https://git.lix.systems/lix-project/lix/issues/183) [fj#110](https://git.lix.systems/lix-project/lix/issues/110) [fj#116](https://git.lix.systems/lix-project/lix/issues/116) [#8953](https://github.com/NixOS/nix/issues/8953) [#9087](https://github.com/NixOS/nix/issues/9087) [cl/1127](https://gerrit.lix.systems/c/lix/+/1127)

flake.nix

@@ -84,6 +84,8 @@
       ];
       systems = linuxSystems ++ darwinSystems;
 
+      # If you add something here, please update the list in doc/manual/src/contributing/hacking.md.
+      # Thanks~
       crossSystems = [
         "armv6l-linux"
         "armv7l-linux"
@@ -195,14 +197,19 @@
             busybox-sandbox-shell = final.busybox-sandbox-shell or final.default-busybox-sandbox-shell;
           };
 
+          pegtl = final.nix.passthru.pegtl;
+
           # Export the patched version of boehmgc that Lix uses into the overlay
           # for consumers of this flake.
-          boehmgc-nix = final.nix.boehmgc-nix;
+          boehmgc-nix = final.nix.passthru.boehmgc-nix;
           # And same thing for our build-release-notes package.
-          build-release-notes = final.nix.build-release-notes;
+          build-release-notes = final.nix.passthru.build-release-notes;
         };
     in
     {
+      # for repl debugging
+      inherit self;
+
       # A Nixpkgs overlay that overrides the 'nix' and
       # 'nix.perl-bindings' packages.
       overlays.default = overlayFor (p: p.stdenv);
@@ -233,6 +240,11 @@
           }
         );
 
+        # Completion tests for the Nix REPL.
+        repl-completion = forAllSystems (
+          system: nixpkgsFor.${system}.native.callPackage ./tests/repl-completion.nix { }
+        );
+
         # Perl bindings for various platforms.
         perlBindings = forAllSystems (system: nixpkgsFor.${system}.native.nix.passthru.perl-bindings);
 
@@ -330,6 +342,7 @@
           rl-next = self.hydraJobs.rl-next.${system}.user;
           # Will be empty attr set on i686-linux, and filtered out by forAvailableSystems.
           pre-commit = self.hydraJobs.pre-commit.${system};
+          repl-completion = self.hydraJobs.repl-completion.${system};
         }
         // (lib.optionalAttrs (builtins.elem system linux64BitSystems)) {
           dockerImage = self.hydraJobs.dockerImage.${system};
@@ -379,7 +392,7 @@
               nix = pkgs.callPackage ./package.nix {
                 inherit stdenv officialRelease versionSuffix;
                 busybox-sandbox-shell = pkgs.busybox-sandbox-shell or pkgs.default-busybox-sandbox;
-                internalApiDocs = true;
+                internalApiDocs = false;
               };
               pre-commit = self.hydraJobs.pre-commit.${pkgs.system} or { };
             in

justfile

@@ -9,8 +9,8 @@ clean:
     rm -rf build
 
 # Prepare meson for building
-setup:
-    meson setup build --prefix="$PWD/outputs/out"
+setup *OPTIONS:
+    meson setup build --prefix="$PWD/outputs/out" $mesonFlags {{ OPTIONS }}
 
 # Build lix
 build *OPTIONS:

maintainers/build-release-notes.py

@@ -20,6 +20,8 @@ SIGNIFICANCECES = {
 
 # This is just hardcoded for better validation. If you think there should be
 # more of them, feel free to add more.
+#
+# Please update doc/manual/src/contributing/hacking.md if you do. Thanks~
 CATEGORIES = [
     'Breaking Changes',
     'Features',

maintainers/buildtime_report.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/usr/bin/env bash
 
 # Generates a report of build time based on a meson build using -ftime-trace in
 # Clang.

meson.build

@@ -142,6 +142,16 @@ else
   cpp_pch = []
 endif
 
+# gcc 12 is known to miscompile some coroutine-based code quite horribly,
+# causing (among other things) copies of move-only objects and the double
+# frees one would expect when the objects are unique_ptrs. these problems
+# often show up as memory corruption when nesting generators (since we do
+# treat generators like owned memory) and will cause inexplicable crashs.
+assert(
+  cxx.get_id() != 'gcc' or cxx.version().version_compare('>=13'),
+  'GCC 12 and earlier are known to miscompile lix coroutines, use GCC 13 or clang.'
+)
+
 
 # Translate some historical and Mesony CPU names to Lixy CPU names.
 # FIXME(Qyriad): the 32-bit x86 code is not tested right now, because cross compilation for Lix
@@ -167,6 +177,7 @@ message('canonical Nix system name:', host_system)
 
 is_linux = host_machine.system() == 'linux'
 is_darwin = host_machine.system() == 'darwin'
+is_freebsd = host_machine.system() == 'freebsd'
 is_x64 = host_machine.cpu_family() == 'x86_64'
 
 # Per-platform arguments that you should probably pass to shared_module() invocations.
@@ -222,7 +233,8 @@ brotli = [
 
 openssl = dependency('libcrypto', 'openssl', required : true)
 
-aws_sdk = dependency('aws-cpp-sdk-core', required : false)
+# FIXME: confirm we actually support such old versions of aws-sdk-cpp
+aws_sdk = dependency('aws-cpp-sdk-core', required : false, version : '>=1.8')
 aws_sdk_transfer = dependency('aws-cpp-sdk-transfer', required : aws_sdk.found(), fallback : ['aws_sdk', 'aws_cpp_sdk_transfer_dep'])
 if aws_sdk.found()
   # The AWS pkg-config adds -std=c++11.
@@ -234,12 +246,6 @@ if aws_sdk.found()
     links : true,
     sources : true,
   )
-  s = aws_sdk.version().split('.')
-  configdata += {
-    'AWS_VERSION_MAJOR': s[0].to_int(),
-    'AWS_VERSION_MINOR': s[1].to_int(),
-    'AWS_VERSION_PATCH': s[2].to_int(),
-  }
   aws_sdk_transfer = aws_sdk_transfer.partial_dependency(
     compile_args : false,
     includes : true,
@@ -291,6 +297,14 @@ gtest = [
 
 toml11 = dependency('toml11', version : '>=3.7.0', required : true, method : 'cmake')
 
+pegtl = dependency(
+  'pegtl',
+  version : '>=3.2.7',
+  required : true,
+  method : 'cmake',
+  modules : [ 'taocpp::pegtl' ],
+)
+
 nlohmann_json = dependency('nlohmann_json', required : true)
 
 # lix-doc is a Rust project provided via buildInputs and unfortunately doesn't have any way to be detected.
@@ -339,8 +353,6 @@ endif
 # that busybox sh won't run busybox applets as builtins (which would break our sandbox).
 
 lsof = find_program('lsof', native : true)
-bison = find_program('bison', native : true)
-flex = find_program('flex', native : true)
 
 # This is how Nix does generated headers...
 # other instances of header generation use a very similar command.
@@ -430,6 +442,7 @@ add_project_arguments(
   '-Wimplicit-fallthrough',
   '-Werror=switch',
   '-Werror=switch-enum',
+  '-Werror=unused-result',
   '-Wdeprecated-copy',
   '-Wignored-qualifiers',
   # Enable assertions in libstdc++ by default. Harmless on libc++. Benchmarked
@@ -441,7 +454,9 @@ add_project_arguments(
   language : 'cpp',
 )
 
-if cxx.get_id() in ['gcc', 'clang']
+# We turn off the production UBSan if the slower dev UBSan is requested, to
+# give better diagnostics.
+if cxx.get_id() in ['gcc', 'clang'] and 'undefined' not in get_option('b_sanitize')
   # 2024-03-24: jade benchmarked the default sanitize reporting in clang and got
   # a regression of about 10% on hackage-packages.nix with clang. So we are trapping instead.
   #
@@ -456,12 +471,23 @@ if cxx.get_id() in ['gcc', 'clang']
   add_project_arguments(sanitize_args, language: 'cpp')
   add_project_link_arguments(sanitize_args, language: 'cpp')
 endif
+# Clang's default of -no-shared-libsan on Linux causes link errors; on macOS it defaults to shared.
+# GCC defaults to shared libsan so is fine.
+if cxx.get_id() == 'clang' and get_option('b_sanitize') != ''
+  add_project_link_arguments('-shared-libsan', language : 'cpp')
+endif
 
 add_project_link_arguments('-pthread', language : 'cpp')
 if cxx.get_linker_id() in ['ld.bfd', 'ld.gold']
   add_project_link_arguments('-Wl,--no-copy-dt-needed-entries', language : 'cpp')
 endif
 
+if is_freebsd
+  # FreeBSD's `environ` is defined in `crt1.o`, not `libc.so`,
+  # so the linker thinks it's undefined
+  add_project_link_arguments('-Wl,-z,undefs', language: 'cpp')
+endif
+
 # Generate Chromium tracing files for each compiled file, which enables
 # maintainers/buildtime_report.sh BUILD-DIR to simply work in clang builds.
 #

meson/cleanup-install.bash

@@ -1,50 +0,0 @@
-#!/usr/bin/env bash
-# Meson will call this with an absolute path to Bash.
-# The shebang is just for convenience.
-
-# The parser and lexer tab are generated via custom Meson targets in src/libexpr/meson.build,
-# but Meson doesn't support marking only part of a target for install. The generation creates
-# both headers (parser-tab.hh, lexer-tab.hh) and source files (parser-tab.cc, lexer-tab.cc),
-# and we definitely want the former installed, but not the latter. This script is added to
-# Meson's install steps to correct this, as the logic for it is just complex enough to
-# warrant separate and careful handling, because both Meson's configured include directory
-# may or may not be an absolute path, and DESTDIR may or may not be set at all, but can't be
-# manipulated in Meson logic.
-
-set -euo pipefail
-
-echo "cleanup-install: removing Meson-placed C++ sources from dest includedir"
-
-if [[ "${1/--help/}" != "$1" ]]; then
-	echo "cleanup-install: this script should only be called from the Meson build system"
-	exit 1
-fi
-
-# Ensure the includedir was passed as the first argument
-# (set -u will make this fail otherwise).
-includedir="$1"
-# And then ensure that first argument is a directory that exists.
-if ! [[ -d "$1" ]]; then
-	echo "cleanup-install: this script should only be called from the Meson build system"
-	echo "argv[1] (${1@Q}) is not a directory"
-	exit 2
-fi
-
-# If DESTDIR environment variable is set, prepend it to the include dir.
-# Unfortunately, we cannot do this on the Meson side. We do have an environment variable
-# `MESON_INSTALL_DESTDIR_PREFIX`, but that will not refer to the include directory if
-# includedir has been set separately, which Lix's split-output derivation does.
-# We also cannot simply do an inline bash conditional like "${DESTDIR:=}" or similar,
-# because we need to specifically *join* DESTDIR and includedir with a slash, and *not*
-# have a slash if DESTDIR isn't set at all, since $includedir could be a relative directory.
-# Finally, DESTDIR is only available to us as an environment variable in these install scripts,
-# not in Meson logic.
-# Therefore, our best option is to have Meson pass this script the configured includedir,
-# and perform this dance with it and $DESTDIR.
-if [[ -n "${DESTDIR:-}" ]]; then
-	includedir="$DESTDIR/$includedir"
-fi
-
-# Intentionally not using -f.
-# If these files don't exist then our assumptions have been violated and we should fail.
-rm -v "$includedir/lix/libexpr/parser-tab.cc" "$includedir/lix/libexpr/lexer-tab.cc"

misc/pegtl.nix

@@ -0,0 +1,23 @@
+{
+  stdenv,
+  cmake,
+  ninja,
+  fetchFromGitHub,
+}:
+
+stdenv.mkDerivation {
+  pname = "pegtl";
+  version = "3.2.7";
+
+  src = fetchFromGitHub {
+    repo = "PEGTL";
+    owner = "taocpp";
+    rev = "refs/tags/3.2.7";
+    hash = "sha256-IV5YNGE4EWVrmg2Sia/rcU8jCuiBynQGJM6n3DCWTQU=";
+  };
+
+  nativeBuildInputs = [
+    cmake
+    ninja
+  ];
+}

package.nix

@@ -10,17 +10,16 @@
   boehmgc-nix ? __forDefaults.boehmgc-nix,
   boehmgc,
   nlohmann_json,
-  bison,
   build-release-notes ? __forDefaults.build-release-notes,
   boost,
   brotli,
   bzip2,
+  callPackage,
   cmake,
   curl,
   doxygen,
   editline-lix ? __forDefaults.editline-lix,
   editline,
-  flex,
   git,
   gtest,
   jq,
@@ -36,6 +35,7 @@
   meson,
   ninja,
   openssl,
+  pegtl ? __forDefaults.pegtl,
   pkg-config,
   python3,
   rapidcheck,
@@ -76,8 +76,10 @@
       configureFlags = prev.configureFlags or [ ] ++ [ (lib.enableFeature true "sigstop") ];
     });
 
-    lix-doc = pkgs.callPackage ./lix-doc/package.nix { };
-    build-release-notes = pkgs.callPackage ./maintainers/build-release-notes.nix { };
+    lix-doc = callPackage ./lix-doc/package.nix { };
+    build-release-notes = callPackage ./maintainers/build-release-notes.nix { };
+
+    pegtl = callPackage ./misc/pegtl.nix { };
   },
 }:
 let
@@ -210,8 +212,6 @@ stdenv.mkDerivation (finalAttrs: {
 
   nativeBuildInputs =
     [
-      bison
-      flex
       python3
       meson
       ninja
@@ -250,6 +250,7 @@ stdenv.mkDerivation (finalAttrs: {
       libsodium
       toml11
       lix-doc
+      pegtl
     ]
     ++ lib.optionals hostPlatform.isLinux [
       libseccomp
@@ -304,6 +305,9 @@ stdenv.mkDerivation (finalAttrs: {
       else
         appendToVar configureFlags "--disable-tests"
       fi
+
+      # Fix up /usr/bin/env shebangs relied on by the build
+      patchShebangs --build tests/ doc/manual/
     '';
 
   mesonBuildType = "debugoptimized";
@@ -379,7 +383,12 @@ stdenv.mkDerivation (finalAttrs: {
   # Export the patched version of boehmgc.
   # flake.nix exports that into its overlay.
   passthru = {
-    inherit (__forDefaults) boehmgc-nix editline-lix build-release-notes;
+    inherit (__forDefaults)
+      boehmgc-nix
+      editline-lix
+      build-release-notes
+      pegtl
+      ;
 
     inherit officialRelease;
 
@@ -392,6 +401,7 @@ stdenv.mkDerivation (finalAttrs: {
         bashInteractive,
         clang-tools,
         clangbuildanalyzer,
+        doxygen,
         glibcLocales,
         just,
         llvmPackages,
@@ -457,6 +467,10 @@ stdenv.mkDerivation (finalAttrs: {
               skopeo
               just
               nixfmt
+              # Included above when internalApiDocs is true, but we set that to
+              # false intentionally to save dev build time.
+              # To build them in a dev shell, you can set -Dinternal-api-docs=enabled when configuring.
+              doxygen
               # Load-bearing order. Must come before clang-unwrapped below, but after clang_tools above.
               stdenv.cc
             ]
@@ -476,7 +490,7 @@ stdenv.mkDerivation (finalAttrs: {
               # https://git.lix.systems/lix-project/lix/src/commit/7575db522e9008685c4009423398f6900a16bcce/src/nix/develop.cc#L240-L241
               # this is, of course, absurd.
               if [[ $name != lix-shell-env && $name != lix-shell-env-env ]]; then
-                return;
+                return
               fi
 
               PATH=$prefix/bin:$PATH
@@ -486,6 +500,11 @@ stdenv.mkDerivation (finalAttrs: {
               # Make bash completion work.
               XDG_DATA_DIRS+=:$out/share
 
+              if [[ ! -f ./.this-is-lix ]]; then
+                echo "Dev shell not started from inside a Lix repo, skipping repo setup" >&2
+                return
+              fi
+
               ${lib.optionalString (pre-commit-checks ? shellHook) pre-commit-checks.shellHook}
               # Allow `touch .nocontribmsg` to turn this notice off.
               if ! [[ -f .nocontribmsg ]]; then

perl/meson.build

@@ -65,5 +65,6 @@ if cxx.get_linker_id() in ['ld.bfd', 'ld.gold']
 endif
 
 libstore = dependency('lixstore', 'lix-store', required : true)
+libutil = dependency('lixutil', 'lix-util', required : true)
 
 subdir('lib/Nix')

src/libcmd/built-path.cc

@@ -1,3 +1,7 @@
+// SPDX-FileCopyrightText: 2024 Nix and Lix Authors
+//
+// SPDX-License-Identifier: LGPL-2.1-only
+
 #include "built-path.hh"
 #include "derivations.hh"
 #include "store-api.hh"

src/libcmd/built-path.hh

@@ -1,3 +1,7 @@
+// SPDX-FileCopyrightText: 2024 Nix and Lix Authors
+//
+// SPDX-License-Identifier: LGPL-2.1-only
+
 #pragma once
 ///@file
 #include "derived-path.hh"

src/libcmd/cmd-profiles.cc

@@ -1,3 +1,7 @@
+// SPDX-FileCopyrightText: 2024 Nix and Lix Authors
+//
+// SPDX-License-Identifier: LGPL-2.1-only
+
 #include <set>
 
 #include "cmd-profiles.hh"
@@ -244,7 +248,7 @@ StorePath ProfileManifest::build(ref<Store> store)
 
     /* Add the symlink tree to the store. */
     StringSink sink;
-    dumpPath(tempDir, sink);
+    sink << dumpPath(tempDir);
 
     auto narHash = hashString(htSHA256, sink.s);
 

src/libcmd/cmd-profiles.hh

@@ -1,3 +1,7 @@
+// SPDX-FileCopyrightText: 2024 Nix and Lix Authors
+//
+// SPDX-License-Identifier: LGPL-2.1-only
+
 #pragma once
 ///@file
 

src/libcmd/command-installable-value.cc

@@ -1,11 +0,0 @@
-#include "command-installable-value.hh"
-
-namespace nix {
-
-void InstallableValueCommand::run(ref<Store> store, ref<Installable> installable)
-{
-    auto installableValue = InstallableValue::require(installable);
-    run(store, installableValue);
-}
-
-}

src/libcmd/command-installable-value.hh

@@ -1,23 +0,0 @@
-#pragma once
-///@file
-
-#include "installable-value.hh"
-#include "command.hh"
-
-namespace nix {
-
-/**
- * An InstallableCommand where the single positional argument must be an
- * InstallableValue in particular.
- */
-struct InstallableValueCommand : InstallableCommand
-{
-    /**
-     * Entry point to this command
-     */
-    virtual void run(ref<Store> store, ref<InstallableValue> installable) = 0;
-
-    void run(ref<Store> store, ref<Installable> installable) override;
-};
-
-}

src/libcmd/command.cc

@@ -1,3 +1,7 @@
+// SPDX-FileCopyrightText: 2024 Nix and Lix Authors
+//
+// SPDX-License-Identifier: LGPL-2.1-only
+
 #include "command.hh"
 #include "store-api.hh"
 #include "local-fs-store.hh"

src/libcmd/command.hh

@@ -1,3 +1,7 @@
+// SPDX-FileCopyrightText: 2024 Nix and Lix Authors
+//
+// SPDX-License-Identifier: LGPL-2.1-only
+
 #pragma once
 ///@file
 

src/libcmd/common-eval-args.cc

@@ -1,3 +1,7 @@
+// SPDX-FileCopyrightText: 2024 Nix and Lix Authors
+//
+// SPDX-License-Identifier: LGPL-2.1-only
+
 #include "eval-settings.hh"
 #include "common-eval-args.hh"
 #include "shared.hh"

src/libcmd/common-eval-args.hh

@@ -1,3 +1,7 @@
+// SPDX-FileCopyrightText: 2024 Nix and Lix Authors
+//
+// SPDX-License-Identifier: LGPL-2.1-only
+
 #pragma once
 ///@file
 

src/libcmd/editor-for.cc

@@ -1,3 +1,7 @@
+// SPDX-FileCopyrightText: 2024 Nix and Lix Authors
+//
+// SPDX-License-Identifier: LGPL-2.1-only
+
 #include "editor-for.hh"
 #include "environment-variables.hh"
 #include "source-path.hh"

src/libcmd/editor-for.hh

@@ -1,3 +1,7 @@
+// SPDX-FileCopyrightText: 2024 Nix and Lix Authors
+//
+// SPDX-License-Identifier: LGPL-2.1-only
+
 #pragma once
 ///@file
 

src/libcmd/installable-attr-path.cc

@@ -1,3 +1,7 @@
+// SPDX-FileCopyrightText: 2024 Nix and Lix Authors
+//
+// SPDX-License-Identifier: LGPL-2.1-only
+
 #include "globals.hh"
 #include "installable-attr-path.hh"
 #include "outputs-spec.hh"

src/libcmd/installable-attr-path.hh

@@ -1,3 +1,7 @@
+// SPDX-FileCopyrightText: 2024 Nix and Lix Authors
+//
+// SPDX-License-Identifier: LGPL-2.1-only
+
 #pragma once
 ///@file
 

src/libcmd/installable-derived-path.cc

@@ -1,3 +1,7 @@
+// SPDX-FileCopyrightText: 2024 Nix and Lix Authors
+//
+// SPDX-License-Identifier: LGPL-2.1-only
+
 #include "installable-derived-path.hh"
 #include "derivations.hh"
 

src/libcmd/installable-derived-path.hh

@@ -1,3 +1,7 @@
+// SPDX-FileCopyrightText: 2024 Nix and Lix Authors
+//
+// SPDX-License-Identifier: LGPL-2.1-only
+
 #pragma once
 ///@file
 

src/libcmd/installable-flake.cc

@@ -1,3 +1,7 @@
+// SPDX-FileCopyrightText: 2024 Nix and Lix Authors
+//
+// SPDX-License-Identifier: LGPL-2.1-only
+
 #include "globals.hh"
 #include "installable-flake.hh"
 #include "installable-derived-path.hh"

src/libcmd/installable-flake.hh

@@ -1,3 +1,7 @@
+// SPDX-FileCopyrightText: 2024 Nix and Lix Authors
+//
+// SPDX-License-Identifier: LGPL-2.1-only
+
 #pragma once
 ///@file
 

src/libcmd/installable-value.cc

@@ -1,3 +1,7 @@
+// SPDX-FileCopyrightText: 2024 Nix and Lix Authors
+//
+// SPDX-License-Identifier: LGPL-2.1-only
+
 #include "installable-value.hh"
 #include "eval-cache.hh"
 #include "fetch-to-store.hh"

src/libcmd/installable-value.hh

@@ -1,3 +1,7 @@
+// SPDX-FileCopyrightText: 2024 Nix and Lix Authors
+//
+// SPDX-License-Identifier: LGPL-2.1-only
+
 #pragma once
 ///@file
 

src/libcmd/installables.cc

@@ -1,3 +1,7 @@
+// SPDX-FileCopyrightText: 2024 Nix and Lix Authors
+//
+// SPDX-License-Identifier: LGPL-2.1-only
+
 #include "globals.hh"
 #include "installables.hh"
 #include "installable-derived-path.hh"
@@ -214,7 +218,7 @@ void SourceExprCommand::completeInstallable(AddCompletions & completions, std::s
 
             evalSettings.pureEval = false;
             auto state = getEvalState();
-            Expr *e = state->parseExprFromFile(
+            Expr & e = state->parseExprFromFile(
                 resolveExprPath(state->checkSourcePath(lookupFileArg(*state, *file)))
                 );
 
@@ -393,13 +397,10 @@ ref<eval_cache::EvalCache> openEvalCache(
     EvalState & state,
     std::shared_ptr<flake::LockedFlake> lockedFlake)
 {
-    auto fingerprint = lockedFlake->getFingerprint();
-    return make_ref<nix::eval_cache::EvalCache>(
-        evalSettings.useEvalCache && evalSettings.pureEval
-            ? std::optional { std::cref(fingerprint) }
-            : std::nullopt,
-        state,
-        [&state, lockedFlake]()
+    auto fingerprint = evalSettings.useEvalCache && evalSettings.pureEval
+        ? std::make_optional(lockedFlake->getFingerprint())
+        : std::nullopt;
+    auto rootLoader = [&state, lockedFlake]()
         {
             /* For testing whether the evaluation cache is
                complete. */
@@ -415,7 +416,17 @@ ref<eval_cache::EvalCache> openEvalCache(
             assert(aOutputs);
 
             return aOutputs->value;
-        });
+        };
+
+    if (fingerprint) {
+        auto search = state.evalCaches.find(fingerprint.value());
+        if (search == state.evalCaches.end()) {
+            search = state.evalCaches.emplace(fingerprint.value(), make_ref<nix::eval_cache::EvalCache>(fingerprint, state, rootLoader)).first;
+        }
+        return search->second;
+    } else {
+        return make_ref<nix::eval_cache::EvalCache>(std::nullopt, state, rootLoader);
+    }
 }
 
 Installables SourceExprCommand::parseInstallables(
@@ -434,13 +445,13 @@ Installables SourceExprCommand::parseInstallables(
         auto vFile = state->allocValue();
 
         if (file == "-") {
-            auto e = state->parseStdin();
+            auto & e = state->parseStdin();
             state->eval(e, *vFile);
         }
         else if (file)
             state->evalFile(lookupFileArg(*state, *file), *vFile);
         else {
-            auto e = state->parseExprFromString(*expr, state->rootPath(CanonPath::fromCwd()));
+            auto & e = state->parseExprFromString(*expr, state->rootPath(CanonPath::fromCwd()));
             state->eval(e, *vFile);
         }
 

src/libcmd/installables.hh

@@ -1,3 +1,7 @@
+// SPDX-FileCopyrightText: 2024 Nix and Lix Authors
+//
+// SPDX-License-Identifier: LGPL-2.1-only
+
 #pragma once
 ///@file
 

src/libcmd/legacy.cc

@@ -1,3 +1,7 @@
+// SPDX-FileCopyrightText: 2024 Nix and Lix Authors
+//
+// SPDX-License-Identifier: LGPL-2.1-only
+
 #include "legacy.hh"
 
 namespace nix {

src/libcmd/legacy.hh

@@ -1,3 +1,7 @@
+// SPDX-FileCopyrightText: 2024 Nix and Lix Authors
+//
+// SPDX-License-Identifier: LGPL-2.1-only
+
 #pragma once
 ///@file
 

src/libcmd/markdown.cc

@@ -1,3 +1,7 @@
+// SPDX-FileCopyrightText: 2024 Nix and Lix Authors
+//
+// SPDX-License-Identifier: LGPL-2.1-only
+
 #include "markdown.hh"
 #include "error.hh"
 #include "finally.hh"

src/libcmd/markdown.hh

@@ -1,3 +1,7 @@
+// SPDX-FileCopyrightText: 2024 Nix and Lix Authors
+//
+// SPDX-License-Identifier: LGPL-2.1-only
+
 #pragma once
 ///@file
 

src/libcmd/meson.build

@@ -1,6 +1,9 @@
+# SPDX-FileCopyrightText: 2024 Nix and Lix Authors
+#
+# SPDX-License-Identifier: LGPL-2.1-only
+
 libcmd_sources = files(
   'built-path.cc',
-  'command-installable-value.cc',
   'cmd-profiles.cc',
   'command.cc',
   'common-eval-args.cc',
@@ -18,7 +21,6 @@ libcmd_sources = files(
 
 libcmd_headers = files(
   'built-path.hh',
-  'command-installable-value.hh',
   'cmd-profiles.hh',
   'command.hh',
   'common-eval-args.hh',

src/libcmd/repl-interacter.cc

@@ -1,3 +1,7 @@
+// SPDX-FileCopyrightText: 2024 Nix and Lix Authors
+//
+// SPDX-License-Identifier: LGPL-2.1-only
+
 #include "error.hh"
 #include "file-system.hh"
 #include "logging.hh"

src/libcmd/repl-interacter.hh

@@ -1,3 +1,7 @@
+// SPDX-FileCopyrightText: 2024 Nix and Lix Authors
+//
+// SPDX-License-Identifier: LGPL-2.1-only
+
 #pragma once
 /// @file
 

src/libcmd/repl-overlays.nix

@@ -1,3 +1,7 @@
+# SPDX-FileCopyrightText: 2024 Nix and Lix Authors
+#
+# SPDX-License-Identifier: LGPL-2.1-only
+
 info: initial: functions:
 let
   final = builtins.foldl' (prev: function: prev // (function info final prev)) initial functions;

src/libcmd/repl.cc

@@ -1,7 +1,14 @@
+// SPDX-FileCopyrightText: 2024 Nix and Lix Authors
+//
+// SPDX-License-Identifier: LGPL-2.1-only
+
+#include <cstdio>
+#include <editline.h>
 #include <iostream>
 #include <cstdlib>
 #include <cstring>
 #include <climits>
+#include <string_view>
 
 #include "box_ptr.hh"
 #include "repl-interacter.hh"
@@ -29,7 +36,6 @@
 #include "local-fs-store.hh"
 #include "signals.hh"
 #include "print.hh"
-#include "progress-bar.hh"
 #include "gc-small-vector.hh"
 #include "users.hh"
 
@@ -79,6 +85,8 @@ enum class ProcessLineResult {
     PromptAgain,
 };
 
+using namespace std::literals::string_view_literals;
+
 struct NixRepl
     : AbstractNixRepl
     , detail::ReplCompleterMixin
@@ -86,6 +94,35 @@ struct NixRepl
     , gc
     #endif
 {
+    /* clang-format: off */
+    static constexpr std::array COMMANDS = {
+        "add"sv, "a"sv,
+        "load"sv, "l"sv,
+        "load-flake"sv, "lf"sv,
+        "reload"sv, "r"sv,
+        "edit"sv, "e"sv,
+        "t"sv,
+        "u"sv,
+        "b"sv,
+        "bl"sv,
+        "i"sv,
+        "sh"sv,
+        "log"sv,
+        "print"sv, "p"sv,
+        "quit"sv, "q"sv,
+        "doc"sv,
+        "te"sv,
+    };
+
+    static constexpr std::array DEBUG_COMMANDS = {
+        "env"sv,
+        "bt"sv, "backtrace"sv,
+        "st"sv,
+        "c"sv, "continue"sv,
+        "s"sv, "step"sv,
+    };
+    /* clang-format: on */
+
     size_t debugTraceIndex;
 
     Strings loadedFiles;
@@ -122,7 +159,7 @@ struct NixRepl
     void reloadFiles();
     void addAttrsToScope(Value & attrs);
     void addVarToScope(const Symbol name, Value & v);
-    Expr * parseString(std::string s);
+    Expr & parseString(std::string s);
     void evalString(std::string s, Value & v);
     void loadDebugTraceEnv(DebugTrace & dt);
 
@@ -209,8 +246,7 @@ NixRepl::NixRepl(const SearchPath & searchPath, nix::ref<Store> store, ref<EvalS
 {
 }
 
-void runNix(Path program, const Strings & args,
-    const std::optional<std::string> & input = {})
+void runNix(Path program, const Strings & args)
 {
     auto subprocessEnv = getEnv();
     subprocessEnv["NIX_CONFIG"] = globalConfig.toKeyValue();
@@ -219,8 +255,7 @@ void runNix(Path program, const Strings & args,
         .program = settings.nixBinDir+ "/" + program,
         .args = args,
         .environment = subprocessEnv,
-        .input = input,
-    });
+    }).wait();
 
     return;
 }
@@ -268,7 +303,7 @@ ReplExitStatus NixRepl::mainLoop()
 
     /* Stop the progress bar because it interferes with the display of
        the repl. */
-    stopProgressBar();
+    logger->pause();
 
     std::string input;
 
@@ -323,6 +358,36 @@ StringSet NixRepl::completePrefix(const std::string & prefix)
 {
     StringSet completions;
 
+    // We should only complete colon commands if there's a colon at the beginning,
+    // but editline (for... whatever reason) doesn't *give* us the colon in the
+    // completion callback. If the user types :rel<TAB>, `prefix` will only be `rel`.
+    // Luckily, editline provides a global variable for its current buffer, so we can
+    // check for the presence of a colon there.
+    if (rl_line_buffer != nullptr && rl_line_buffer[0] == ':') {
+        for (auto const & colonCmd : this->COMMANDS) {
+            if (colonCmd.starts_with(prefix)) {
+                completions.insert(std::string(colonCmd));
+            }
+        }
+
+        if (state->debugRepl) {
+            for (auto const & colonCmd : this->DEBUG_COMMANDS) {
+                if (colonCmd.starts_with(prefix)) {
+                    completions.insert(std::string(colonCmd));
+                }
+            }
+        }
+
+        // If there were : command completions, then we should only return those,
+        // because otherwise this is not valid Nix syntax.
+        // However if we didn't get any completions, then this could be something
+        // like `:b pkgs.hel<TAB>`, in which case we should do expression completion
+        // as normal.
+        if (!completions.empty()) {
+            return completions;
+        }
+    }
+
     size_t start = prefix.find_last_of(" \n\r\t(){}[]");
     std::string prev, cur;
     if (start == std::string::npos) {
@@ -365,9 +430,9 @@ StringSet NixRepl::completePrefix(const std::string & prefix)
             auto expr = cur.substr(0, dot);
             auto cur2 = cur.substr(dot + 1);
 
-            Expr * e = parseString(expr);
+            Expr & e = parseString(expr);
             Value v;
-            e->eval(*state, *env, v);
+            e.eval(*state, *env, v);
             state->forceAttrs(v, noPos, "while evaluating an attrset for the purpose of completion (this error should not be displayed; file an issue?)");
 
             for (auto & i : *v.attrs) {
@@ -589,7 +654,7 @@ ProcessLineResult NixRepl::processLine(std::string line)
 
         // runProgram redirects stdout to a StringSink,
         // using runProgram2 to allow editors to display their UI
-        runProgram2(RunOptions { .program = editor, .searchPath = true, .args = args });
+        runProgram2(RunOptions { .program = editor, .searchPath = true, .args = args }).wait();
 
         // Reload right after exiting the editor
         state->resetFileCache();
@@ -622,9 +687,10 @@ ProcessLineResult NixRepl::processLine(std::string line)
             // TODO: this only shows a progress bar for explicitly initiated builds,
             // not eval-time fetching or builds performed for IFD.
             // But we can't just show it everywhere, since that would erase partial output from evaluation.
-            startProgressBar();
+            logger->resetProgress();
+            logger->resume();
             Finally stopLogger([&]() {
-                stopProgressBar();
+                logger->pause();
             });
 
             state->store->buildPaths({
@@ -758,7 +824,7 @@ ProcessLineResult NixRepl::processLine(std::string line)
             line[p + 1] != '=' &&
             isVarName(name = removeWhitespace(line.substr(0, p))))
         {
-            Expr * e = parseString(line.substr(p + 1));
+            Expr & e = parseString(line.substr(p + 1));
             Value & v(*state->allocValue());
             v.mkThunk(env, e);
             addVarToScope(state->symbols.create(name), v);
@@ -883,7 +949,7 @@ Value * NixRepl::getReplOverlaysEvalFunction()
     auto code =
         #include "repl-overlays.nix.gen.hh"
         ;
-    auto expr = state->parseExprFromString(
+    auto & expr = state->parseExprFromString(
         code,
         SourcePath(evalReplInitFilesPath),
         state->staticBaseEnv
@@ -991,7 +1057,7 @@ Value * NixRepl::bindingsToAttrs()
 }
 
 
-Expr * NixRepl::parseString(std::string s)
+Expr & NixRepl::parseString(std::string s)
 {
     return state->parseExprFromString(std::move(s), state->rootPath(CanonPath::fromCwd()), staticEnv);
 }
@@ -999,16 +1065,16 @@ Expr * NixRepl::parseString(std::string s)
 
 void NixRepl::evalString(std::string s, Value & v)
 {
-    Expr * e = parseString(s);
-    e->eval(*state, *env, v);
+    Expr & e = parseString(s);
+    e.eval(*state, *env, v);
     state->forceValue(v, v.determinePos(noPos));
 }
 
 Value * NixRepl::evalFile(SourcePath & path)
 {
-    auto expr = state->parseExprFromFile(path, staticEnv);
+    auto & expr = state->parseExprFromFile(path, staticEnv);
     Value * result(state->allocValue());
-    expr->eval(*state, *env, *result);
+    expr.eval(*state, *env, *result);
     state->forceValue(*result, result->determinePos(noPos));
     return result;
 }

src/libcmd/repl.hh

@@ -1,3 +1,7 @@
+// SPDX-FileCopyrightText: 2024 Nix and Lix Authors
+//
+// SPDX-License-Identifier: LGPL-2.1-only
+
 #pragma once
 ///@file
 

src/libexpr/attr-path.cc

@@ -1,3 +1,7 @@
+// SPDX-FileCopyrightText: 2024 Nix and Lix Authors
+//
+// SPDX-License-Identifier: LGPL-2.1-only
+
 #include "attr-path.hh"
 #include "eval-inline.hh"
 

src/libexpr/attr-path.hh

@@ -1,3 +1,7 @@
+// SPDX-FileCopyrightText: 2024 Nix and Lix Authors
+//
+// SPDX-License-Identifier: LGPL-2.1-only
+
 #pragma once
 ///@file
 

src/libexpr/attr-set.cc

@@ -1,3 +1,7 @@
+// SPDX-FileCopyrightText: 2024 Nix and Lix Authors
+//
+// SPDX-License-Identifier: LGPL-2.1-only
+
 #include "attr-set.hh"
 #include "eval-inline.hh"
 

src/libexpr/attr-set.hh

@@ -1,3 +1,7 @@
+// SPDX-FileCopyrightText: 2024 Nix and Lix Authors
+//
+// SPDX-License-Identifier: LGPL-2.1-only
+
 #pragma once
 ///@file
 

src/libexpr/eval-cache.cc

@@ -1,3 +1,7 @@
+// SPDX-FileCopyrightText: 2024 Nix and Lix Authors
+//
+// SPDX-License-Identifier: LGPL-2.1-only
+
 #include "eval-cache.hh"
 #include "sqlite.hh"
 #include "eval.hh"

src/libexpr/eval-cache.hh

@@ -1,3 +1,7 @@
+// SPDX-FileCopyrightText: 2024 Nix and Lix Authors
+//
+// SPDX-License-Identifier: LGPL-2.1-only
+
 #pragma once
 ///@file
 

src/libexpr/eval-error.cc

@@ -1,3 +1,7 @@
+// SPDX-FileCopyrightText: 2024 Nix and Lix Authors
+//
+// SPDX-License-Identifier: LGPL-2.1-only
+
 #include "eval-error.hh"
 #include "eval.hh"
 #include "value.hh"

src/libexpr/eval-error.hh

@@ -1,3 +1,7 @@
+// SPDX-FileCopyrightText: 2024 Nix and Lix Authors
+//
+// SPDX-License-Identifier: LGPL-2.1-only
+
 #pragma once
 ///@file
 
@@ -47,12 +51,16 @@ MakeError(MissingArgumentError, EvalError);
 MakeError(RestrictedPathError, Error);
 MakeError(InfiniteRecursionError, EvalError);
 
+/**
+ * Represents an exception due to an invalid path; that is, it does not exist.
+ * It corresponds to `!Store::validPath()`.
+ */
 struct InvalidPathError : public EvalError
 {
 public:
     Path path;
     InvalidPathError(EvalState & state, const Path & path)
-        : EvalError(state, "path '%s' is not valid", path)
+        : EvalError(state, "path '%s' did not exist in the store during evaluation", path)
     {
     }
 };

src/libexpr/eval-inline.hh

@@ -1,3 +1,7 @@
+// SPDX-FileCopyrightText: 2024 Nix and Lix Authors
+//
+// SPDX-License-Identifier: LGPL-2.1-only
+
 #pragma once
 ///@file
 
@@ -86,11 +90,11 @@ void EvalState::forceValue(Value & v, const PosIdx pos)
 {
     if (v.isThunk()) {
         Env * env = v.thunk.env;
-        Expr * expr = v.thunk.expr;
+        Expr & expr = *v.thunk.expr;
         try {
             v.mkBlackhole();
             //checkInterrupt();
-            expr->eval(*this, *env, v);
+            expr.eval(*this, *env, v);
         } catch (...) {
             v.mkThunk(env, expr);
             tryFixupBlackHolePos(v, pos);

src/libexpr/eval-settings.cc

@@ -1,3 +1,7 @@
+// SPDX-FileCopyrightText: 2024 Nix and Lix Authors
+//
+// SPDX-License-Identifier: LGPL-2.1-only
+
 #include "file-system.hh"
 #include "globals.hh"
 #include "profiles.hh"
@@ -63,11 +67,9 @@ Strings EvalSettings::getDefaultNixPath()
         }
     };
 
-    if (!evalSettings.restrictEval && !evalSettings.pureEval) {
-        add(getNixDefExpr() + "/channels");
-        add(rootChannelsDir() + "/nixpkgs", "nixpkgs");
-        add(rootChannelsDir());
-    }
+    add(getNixDefExpr() + "/channels");
+    add(rootChannelsDir() + "/nixpkgs", "nixpkgs");
+    add(rootChannelsDir());
 
     return res;
 }

src/libexpr/eval-settings.hh

@@ -1,3 +1,7 @@
+// SPDX-FileCopyrightText: 2024 Nix and Lix Authors
+//
+// SPDX-License-Identifier: LGPL-2.1-only
+
 #pragma once
 ///@file
 #include "config.hh"
@@ -15,9 +19,21 @@ struct EvalSettings : Config
     static std::string resolvePseudoUrl(std::string_view url);
 
     Setting<bool> enableNativeCode{this, false, "allow-unsafe-native-code-during-evaluation", R"(
-        Whether builtin functions that allow executing native code should be enabled.
+        Enable built-in functions that allow executing native code.
 
-        In particular, this adds the `importNative` and `exec` builtins.
+        In particular, this adds:
+        - `builtins.importNative` *path* *symbol*
+
+          Runs function with *symbol* from a dynamic shared object (DSO) at *path*.
+          This may be used to add new builtins to the Nix language.
+          The procedure must have the following signature:
+          ```cpp
+          extern "C" typedef void (*ValueInitialiser) (EvalState & state, Value & v);
+          ```
+
+        - `builtins.exec` *arguments*
+
+          Execute a program, where *arguments* are specified as a list of strings, and parse its output as a Nix expression.
     )"};
 
     Setting<Strings> nixPath{
@@ -63,8 +79,17 @@ struct EvalSettings : Config
         R"(
           Pure evaluation mode ensures that the result of Nix expressions is fully determined by explicitly declared inputs, and not influenced by external state:
 
-          - Restrict file system and network access to files specified by cryptographic hash
-          - Disable [`builtins.currentSystem`](@docroot@/language/builtin-constants.md#builtins-currentSystem) and [`builtins.currentTime`](@docroot@/language/builtin-constants.md#builtins-currentTime)
+          - File system and network access is restricted to accesses to immutable data only:
+            - Path literals relative to the home directory like `~/lix` are rejected at parse time.
+            - Access to absolute paths that did not result from Nix language evaluation is rejected when such paths are given as parameters to builtins like, for example, [`builtins.readFile`](@docroot@/language/builtins.md#builtins-readFile).
+
+              Access is nonetheless allowed to (absolute) paths in the Nix store that are returned by builtins like [`builtins.filterSource`](@docroot@/language/builtins.md#builtins-filterSource), [`builtins.fetchTarball`](@docroot@/language/builtins.md#builtins-fetchTarball) and similar.
+            - Impure fetches such as not specifying a commit ID for `builtins.fetchGit` or not specifying a hash for `builtins.fetchTarball` are rejected.
+            - In flakes, access to relative paths outside of the root of the flake's source tree (often, a git repository) is rejected.
+          - The evaluator ignores `NIX_PATH`, `-I` and the `nix-path` setting. Thus, [`builtins.nixPath`](@docroot@/language/builtin-constants.md#builtins-nixPath) is an empty list.
+          - The builtins [`builtins.currentSystem`](@docroot@/language/builtin-constants.md#builtins-currentSystem) and [`builtins.currentTime`](@docroot@/language/builtin-constants.md#builtins-currentTime) are absent from `builtins`.
+          - [`builtins.getEnv`](@docroot@/language/builtin-constants.md#builtins-currentSystem) always returns empty string for any variable.
+          - [`builtins.storePath`](@docroot@/language/builtins.md#builtins-storePath) throws an error (Lix may change this, tracking issue: <https://git.lix.systems/lix-project/lix/issues/402>)
         )"
         };
 
@@ -86,6 +111,7 @@ struct EvalSettings : Config
           allowed to access `https://github.com/NixOS/patchelf.git`.
         )"};
 
+
     Setting<bool> traceFunctionCalls{this, false, "trace-function-calls",
         R"(
           If set to `true`, the Nix evaluator will trace every function call.

src/libexpr/eval.cc

@@ -1,9 +1,14 @@
+// SPDX-FileCopyrightText: 2024 Nix and Lix Authors
+//
+// SPDX-License-Identifier: LGPL-2.1-only
+
 #include "eval.hh"
 #include "eval-settings.hh"
 #include "hash.hh"
 #include "primops.hh"
 #include "print-options.hh"
 #include "shared.hh"
+#include "suggestions.hh"
 #include "types.hh"
 #include "store-api.hh"
 #include "derivations.hh"
@@ -17,7 +22,6 @@
 #include "gc-small-vector.hh"
 #include "fetch-to-store.hh"
 #include "flake/flakeref.hh"
-#include "parser-tab.hh"
 
 #include <algorithm>
 #include <chrono>
@@ -418,7 +422,7 @@ EvalState::EvalState(
     }
 
     if (evalSettings.restrictEval || evalSettings.pureEval) {
-        allowedPaths = PathSet();
+        allowedPaths = std::optional(PathSet());
 
         for (auto & i : searchPath.elements) {
             auto r = resolveSearchPathPath(i.path);
@@ -950,14 +954,14 @@ void EvalState::mkList(Value & v, size_t size)
 
 unsigned long nrThunks = 0;
 
-static inline void mkThunk(Value & v, Env & env, Expr * expr)
+static inline void mkThunk(Value & v, Env & env, Expr & expr)
 {
     v.mkThunk(&env, expr);
     nrThunks++;
 }
 
 
-void EvalState::mkThunk_(Value & v, Expr * expr)
+void EvalState::mkThunk_(Value & v, Expr & expr)
 {
     mkThunk(v, baseEnv, expr);
 }
@@ -1058,7 +1062,7 @@ void EvalState::mkSingleDerivedPathString(
 Value * Expr::maybeThunk(EvalState & state, Env & env)
 {
     Value * v = state.allocValue();
-    mkThunk(*v, env, this);
+    mkThunk(*v, env, *this);
     return v;
 }
 
@@ -1122,7 +1126,7 @@ void EvalState::evalFile(const SourcePath & path_, Value & v, bool mustBeTrivial
         e = j->second;
 
     if (!e)
-        e = parseExprFromFile(checkSourcePath(resolvedPath));
+        e = &parseExprFromFile(checkSourcePath(resolvedPath));
 
     cacheFile(path, resolvedPath, e, v, mustBeTrivial);
 }
@@ -1159,7 +1163,7 @@ void EvalState::cacheFile(
         if (mustBeTrivial &&
             !(dynamic_cast<ExprAttrs *>(e)))
             error<EvalError>("file '%s' must be an attribute set", path).debugThrow();
-        eval(e, v);
+        eval(*e, v);
     } catch (Error & e) {
         addErrorTrace(e, "while evaluating the file '%1%':", resolvedPath.to_string());
         throw;
@@ -1170,23 +1174,23 @@ void EvalState::cacheFile(
 }
 
 
-void EvalState::eval(Expr * e, Value & v)
+void EvalState::eval(Expr & e, Value & v)
 {
-    e->eval(*this, baseEnv, v);
+    e.eval(*this, baseEnv, v);
 }
 
 
-inline bool EvalState::evalBool(Env & env, Expr * e, const PosIdx pos, std::string_view errorCtx)
+inline bool EvalState::evalBool(Env & env, Expr & e, const PosIdx pos, std::string_view errorCtx)
 {
     try {
         Value v;
-        e->eval(*this, env, v);
+        e.eval(*this, env, v);
         if (v.type() != nBool)
             error<TypeError>(
                  "expected a Boolean but found %1%: %2%",
                  showType(v),
                  ValuePrinter(*this, v, errorPrintOptions)
-             ).atPos(pos).withFrame(env, *e).debugThrow();
+             ).atPos(pos).withFrame(env, e).debugThrow();
         return v.boolean;
     } catch (Error & e) {
         e.addTrace(positions[pos], errorCtx);
@@ -1195,16 +1199,16 @@ inline bool EvalState::evalBool(Env & env, Expr * e, const PosIdx pos, std::stri
 }
 
 
-inline void EvalState::evalAttrs(Env & env, Expr * e, Value & v, const PosIdx pos, std::string_view errorCtx)
+inline void EvalState::evalAttrs(Env & env, Expr & e, Value & v, const PosIdx pos, std::string_view errorCtx)
 {
     try {
-        e->eval(*this, env, v);
+        e.eval(*this, env, v);
         if (v.type() != nAttrs)
             error<TypeError>(
                 "expected a set but found %1%: %2%",
                 showType(v),
                 ValuePrinter(*this, v, errorPrintOptions)
-            ).withFrame(env, *e).debugThrow();
+            ).withFrame(env, e).debugThrow();
     } catch (Error & e) {
         e.addTrace(positions[pos], errorCtx);
         throw;
@@ -1247,7 +1251,7 @@ Env * ExprAttrs::buildInheritFromEnv(EvalState & state, Env & up)
     inheritEnv.up = &up;
 
     Displacement displ = 0;
-    for (auto from : *inheritFromExprs)
+    for (auto & from : *inheritFromExprs)
         inheritEnv.values[displ++] = from->maybeThunk(state, up);
 
     return &inheritEnv;
@@ -1277,7 +1281,7 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v)
             Value * vAttr;
             if (hasOverrides && i.second.kind != AttrDef::Kind::Inherited) {
                 vAttr = state.allocValue();
-                mkThunk(*vAttr, *i.second.chooseByKind(&env2, &env, inheritEnv), i.second.e);
+                mkThunk(*vAttr, *i.second.chooseByKind(&env2, &env, inheritEnv), *i.second.e);
             } else
                 vAttr = i.second.e->maybeThunk(state, *i.second.chooseByKind(&env2, &env, inheritEnv));
             env2.values[displ++] = vAttr;
@@ -1426,11 +1430,26 @@ static std::string showAttrPath(EvalState & state, Env & env, const AttrPath & a
 
 void ExprSelect::eval(EvalState & state, Env & env, Value & v)
 {
-    Value vTmp;
-    PosIdx pos2;
-    Value * vAttrs = &vTmp;
+    Value vFirst;
 
-    e->eval(state, env, vTmp);
+    // Pointer to the current attrset Value in this select chain.
+    Value * vCurrent = &vFirst;
+    // Position for the current attrset Value in this select chain.
+    PosIdx posCurrent;
+
+    try {
+        e->eval(state, env, vFirst);
+    } catch (Error & e) {
+        assert(this->e != nullptr);
+        state.addErrorTrace(
+            e,
+            getPos(),
+            "while evaluating '%s' to select '%s' on it",
+            ExprPrinter(state, *this->e),
+            showAttrPath(state.symbols, this->attrPath)
+        );
+        throw;
+    }
 
     try {
         auto dts = state.debugRepl
@@ -1443,48 +1462,112 @@ void ExprSelect::eval(EvalState & state, Env & env, Value & v)
                 showAttrPath(state, env, attrPath))
             : nullptr;
 
-        for (auto & i : attrPath) {
+        for (auto const & [partIdx, currentAttrName] : enumerate(attrPath)) {
             state.nrLookups++;
-            Bindings::iterator j;
-            auto name = getName(i, state, env);
-            if (def) {
-                state.forceValue(*vAttrs, pos);
-                if (vAttrs->type() != nAttrs ||
-                    (j = vAttrs->attrs->find(name)) == vAttrs->attrs->end())
-                {
-                    def->eval(state, env, v);
+
+            Symbol const name = getName(currentAttrName, state, env);
+
+            // For formatting errors, which should be done only when needed.
+            auto partsSoFar = [&]() -> std::string {
+                std::stringstream ss;
+                // We start with the base thing this ExprSelect is selecting on.
+                assert(this->e != nullptr);
+                this->e->show(state.symbols, ss);
+
+                // Then grab each part of the attr path up to this one.
+                assert(partIdx < attrPath.size());
+                std::span<AttrName> const parts(
+                    attrPath.begin(),
+                    attrPath.begin() + partIdx
+                );
+
+                // And convert them to strings and join them.
+                for (auto const & part : parts) {
+                    auto const partName = getName(part, state, env);
+                    ss << "." << state.symbols[partName];
+                }
+
+                return ss.str();
+            };
+
+            try {
+                state.forceValue(*vCurrent, pos);
+            } catch (Error & e) {
+                state.addErrorTrace(
+                    e,
+                    getPos(),
+                    "while evaluating '%s' to select '%s' on it",
+                    partsSoFar(),
+                    state.symbols[name]
+                );
+                throw;
+            }
+
+            if (vCurrent->type() != nAttrs) {
+
+                // If we have an `or` provided default,
+                // then this is allowed to not be an attrset.
+                if (def != nullptr) {
+                    this->def->eval(state, env, v);
                     return;
                 }
-            } else {
-                state.forceAttrs(*vAttrs, pos, "while selecting an attribute");
-                if ((j = vAttrs->attrs->find(name)) == vAttrs->attrs->end()) {
-                    std::set<std::string> allAttrNames;
-                    for (auto & attr : *vAttrs->attrs)
-                        allAttrNames.insert(state.symbols[attr.name]);
-                    auto suggestions = Suggestions::bestMatches(allAttrNames, state.symbols[name]);
-                    state.error<EvalError>("attribute '%1%' missing", state.symbols[name])
-                        .atPos(pos).withSuggestions(suggestions).withFrame(env, *this).debugThrow();
-                }
+
+                // Otherwise, we must type error.
+                state.error<TypeError>(
+                    "expected a set but found %s: %s",
+                    showType(*vCurrent),
+                    ValuePrinter(state, *vCurrent, errorPrintOptions)
+                ).addTrace(
+                    pos,
+                    HintFmt("while selecting '%s' on '%s'", state.symbols[name], partsSoFar())
+                ).debugThrow();
             }
-            vAttrs = j->value;
-            pos2 = j->pos;
-            if (state.countCalls) state.attrSelects[pos2]++;
+
+            // Now that we know this is actually an attrset, try to find an attr
+            // with the selected name.
+            Bindings::iterator attrIt = vCurrent->attrs->find(name);
+            if (attrIt == vCurrent->attrs->end()) {
+
+                // If we have an `or` provided default, then we'll use that.
+                if (def != nullptr) {
+                    this->def->eval(state, env, v);
+                    return;
+                }
+
+                // Otherwise, missing attr error.
+                std::set<std::string> allAttrNames;
+                for (auto const & attr : *vCurrent->attrs) {
+                    allAttrNames.insert(state.symbols[attr.name]);
+                }
+                auto suggestions = Suggestions::bestMatches(allAttrNames, state.symbols[name]);
+                state.error<EvalError>("attribute '%s' missing", state.symbols[name])
+                    .atPos(pos)
+                    .withSuggestions(suggestions)
+                    .withFrame(env, *this)
+                    .debugThrow();
+            }
+
+            // If we're here, then we successfully found the attribute.
+            // Set our currently operated-on attrset to this one, and keep going.
+            vCurrent = attrIt->value;
+            posCurrent = attrIt->pos;
+            if (state.countCalls) state.attrSelects[posCurrent]++;
         }
 
-        state.forceValue(*vAttrs, (pos2 ? pos2 : this->pos ) );
+        state.forceValue(*vCurrent, (posCurrent ? posCurrent : this->pos));
 
     } catch (Error & e) {
-        if (pos2) {
-            auto pos2r = state.positions[pos2];
+        if (posCurrent) {
+            auto pos2r = state.positions[posCurrent];
             auto origin = std::get_if<SourcePath>(&pos2r.origin);
             if (!(origin && *origin == state.derivationInternal))
-                state.addErrorTrace(e, pos2, "while evaluating the attribute '%1%'",
+                state.addErrorTrace(e, posCurrent, "while evaluating the attribute '%1%'",
                     showAttrPath(state, env, attrPath));
         }
         throw;
     }
 
-    v = *vAttrs;
+    v = *vCurrent;
 }
 
 
@@ -1533,6 +1616,66 @@ public:
 };
 };
 
+/** Currently these each just take one, but maybe in the future we could have diagnostics
+ * for all unexpected and missing arguments?
+ */
+struct FormalsMatch
+{
+    std::vector<Symbol> missing;
+    std::vector<Symbol> unexpected;
+};
+
+/** Matchup an attribute argument set to a lambda's formal arguments,
+ * or return what arguments were required but not given, or given but not allowed.
+ * (currently returns only one, for each).
+ */
+FormalsMatch matchupFormals(EvalState & state, Env & env, Displacement & displ, ExprLambda const & lambda, Bindings & attrs)
+{
+    size_t attrsUsed = 0;
+
+    for (auto const & formal : lambda.formals->formals) {
+
+        // The attribute whose name matches the name of the formal we're matching up, if it exists.
+        Attr const * matchingArg = attrs.get(formal.name);
+        if (matchingArg) {
+            attrsUsed += 1;
+            env.values[displ] = matchingArg->value;
+            displ += 1;
+
+            // We're done here. Move on to the next formal.
+            continue;
+        }
+
+        // The argument for this formal wasn't given.
+        // If the formal has a default, use it.
+        if (formal.def) {
+            env.values[displ] = formal.def->maybeThunk(state, env);
+            displ += 1;
+        } else {
+            // Otherwise, let our caller know what was missing.
+            return FormalsMatch{
+                .missing = {formal.name},
+            };
+        }
+    }
+
+    // Check for unexpected extra arguments.
+    if (!lambda.formals->ellipsis && attrsUsed != attrs.size()) {
+        // Return the first unexpected argument.
+        for (Attr const & attr : attrs) {
+            if (!lambda.formals->has(attr.name)) {
+                return FormalsMatch{
+                    .unexpected = {attr.name},
+                };
+            }
+        }
+
+        abort(); // unreachable.
+    }
+
+    return FormalsMatch{};
+}
+
 void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & vRes, const PosIdx pos)
 {
     if (callDepth > evalSettings.maxCallDepth)
@@ -1586,53 +1729,42 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value &
                 if (lambda.arg)
                     env2.values[displ++] = args[0];
 
-                /* For each formal argument, get the actual argument.  If
-                   there is no matching actual argument but the formal
-                   argument has a default, use the default. */
-                size_t attrsUsed = 0;
-                for (auto & i : lambda.formals->formals) {
-                    auto j = args[0]->attrs->get(i.name);
-                    if (!j) {
-                        if (!i.def) {
-                            error<TypeError>("function '%1%' called without required argument '%2%'",
-                                             (lambda.name ? std::string(symbols[lambda.name]) : "anonymous lambda"),
-                                             symbols[i.name])
-                                    .atPos(lambda.pos)
-                                    .withTrace(pos, "from call site")
-                                    .withFrame(*fun.lambda.env, lambda)
-                                    .debugThrow();
-                        }
-                        env2.values[displ++] = i.def->maybeThunk(*this, env2);
-                    } else {
-                        attrsUsed++;
-                        env2.values[displ++] = j->value;
+                ///* For each formal argument, get the actual argument.  If
+                //   there is no matching actual argument but the formal
+                //   argument has a default, use the default. */
+                auto const formalsMatch = matchupFormals(
+                    *this,
+                    env2,
+                    displ,
+                    lambda,
+                    *args[0]->attrs
+                );
+                for (auto const & missingArg : formalsMatch.missing) {
+                    auto const missing = symbols[missingArg];
+                    error<TypeError>("function '%s' called without required argument '%s'", lambda.getName(symbols), missing)
+                        .atPos(lambda.pos)
+                        .withTrace(pos, "from call site")
+                        .withFrame(*fun.lambda.env, lambda)
+                        .debugThrow();
+                }
+                for (auto const & unexpectedArg : formalsMatch.unexpected) {
+                    auto const unex = symbols[unexpectedArg];
+                    std::set<std::string> formalNames;
+                    for (auto const & formal : lambda.formals->formals) {
+                        formalNames.insert(symbols[formal.name]);
                     }
+                    auto sug = Suggestions::bestMatches(formalNames, unex);
+                    error<TypeError>("function '%s' called with unexpected argument '%s'", lambda.getName(symbols), unex)
+                        .atPos(lambda.pos)
+                        .withTrace(pos, "from call site")
+                        .withSuggestions(sug)
+                        .withFrame(*fun.lambda.env, lambda)
+                        .debugThrow();
                 }
 
-                /* Check that each actual argument is listed as a formal
-                   argument (unless the attribute match specifies a `...'). */
-                if (!lambda.formals->ellipsis && attrsUsed != args[0]->attrs->size()) {
-                    /* Nope, so show the first unexpected argument to the
-                       user. */
-                    for (auto & i : *args[0]->attrs)
-                        if (!lambda.formals->has(i.name)) {
-                            std::set<std::string> formalNames;
-                            for (auto & formal : lambda.formals->formals)
-                                formalNames.insert(symbols[formal.name]);
-                            auto suggestions = Suggestions::bestMatches(formalNames, symbols[i.name]);
-                            error<TypeError>("function '%1%' called with unexpected argument '%2%'",
-                                             (lambda.name ? std::string(symbols[lambda.name]) : "anonymous lambda"),
-                                             symbols[i.name])
-                                .atPos(lambda.pos)
-                                .withTrace(pos, "from call site")
-                                .withSuggestions(suggestions)
-                                .withFrame(*fun.lambda.env, lambda)
-                                .debugThrow();
-                        }
-                    abort(); // can't happen
-                }
             }
 
+
             nrFunctionCalls++;
             if (countCalls) incrFunctionCall(&lambda);
 
@@ -1642,9 +1774,7 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value &
                     ? makeDebugTraceStacker(
                         *this, *lambda.body, env2, positions[lambda.pos],
                         "while calling %s",
-                        lambda.name
-                        ? concatStrings("'", symbols[lambda.name], "'")
-                        : "anonymous lambda")
+                        lambda.getQuotedName(symbols))
                     : nullptr;
 
                 lambda.body->eval(*this, env2, vCur);
@@ -1654,9 +1784,7 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value &
                         e,
                         lambda.pos,
                         "while calling %s",
-                        lambda.name
-                        ? concatStrings("'", symbols[lambda.name], "'")
-                        : "anonymous lambda");
+                        lambda.getQuotedName(symbols));
                     if (pos) addErrorTrace(e, pos, "from call site");
                 }
                 throw;
@@ -1683,6 +1811,15 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value &
 
                 try {
                     fn->fun(*this, vCur.determinePos(noPos), args, vCur);
+                } catch (ThrownError & e) {
+                    // Distinguish between an error that simply happened while "throw"
+                    // was being evaluated and an explicit thrown error.
+                    if (fn->name == "throw") {
+                        addErrorTrace(e, pos, "caused by explicit %s", "throw");
+                    } else {
+                        addErrorTrace(e, pos, "while calling the '%s' builtin", fn->name);
+                    }
+                    throw;
                 } catch (Error & e) {
                     addErrorTrace(e, pos, "while calling the '%1%' builtin", fn->name);
                     throw;
@@ -1872,13 +2009,13 @@ void ExprWith::eval(EvalState & state, Env & env, Value & v)
 void ExprIf::eval(EvalState & state, Env & env, Value & v)
 {
     // We cheat in the parser, and pass the position of the condition as the position of the if itself.
-    (state.evalBool(env, cond, pos, "while evaluating a branch condition") ? then : else_)->eval(state, env, v);
+    (state.evalBool(env, *cond, pos, "while evaluating a branch condition") ? *then : *else_).eval(state, env, v);
 }
 
 
 void ExprAssert::eval(EvalState & state, Env & env, Value & v)
 {
-    if (!state.evalBool(env, cond, pos, "in the condition of the assert statement")) {
+    if (!state.evalBool(env, *cond, pos, "in the condition of the assert statement")) {
         std::ostringstream out;
         cond->show(state.symbols, out);
         state.error<AssertionError>("assertion '%1%' failed", out.str()).atPos(pos).withFrame(env, *this).debugThrow();
@@ -1889,7 +2026,7 @@ void ExprAssert::eval(EvalState & state, Env & env, Value & v)
 
 void ExprOpNot::eval(EvalState & state, Env & env, Value & v)
 {
-    v.mkBool(!state.evalBool(env, e, getPos(), "in the argument of the not operator")); // XXX: FIXME: !
+    v.mkBool(!state.evalBool(env, *e, getPos(), "in the argument of the not operator")); // XXX: FIXME: !
 }
 
 
@@ -1911,27 +2048,27 @@ void ExprOpNEq::eval(EvalState & state, Env & env, Value & v)
 
 void ExprOpAnd::eval(EvalState & state, Env & env, Value & v)
 {
-    v.mkBool(state.evalBool(env, e1, pos, "in the left operand of the AND (&&) operator") && state.evalBool(env, e2, pos, "in the right operand of the AND (&&) operator"));
+    v.mkBool(state.evalBool(env, *e1, pos, "in the left operand of the AND (&&) operator") && state.evalBool(env, *e2, pos, "in the right operand of the AND (&&) operator"));
 }
 
 
 void ExprOpOr::eval(EvalState & state, Env & env, Value & v)
 {
-    v.mkBool(state.evalBool(env, e1, pos, "in the left operand of the OR (||) operator") || state.evalBool(env, e2, pos, "in the right operand of the OR (||) operator"));
+    v.mkBool(state.evalBool(env, *e1, pos, "in the left operand of the OR (||) operator") || state.evalBool(env, *e2, pos, "in the right operand of the OR (||) operator"));
 }
 
 
 void ExprOpImpl::eval(EvalState & state, Env & env, Value & v)
 {
-    v.mkBool(!state.evalBool(env, e1, pos, "in the left operand of the IMPL (->) operator") || state.evalBool(env, e2, pos, "in the right operand of the IMPL (->) operator"));
+    v.mkBool(!state.evalBool(env, *e1, pos, "in the left operand of the IMPL (->) operator") || state.evalBool(env, *e2, pos, "in the right operand of the IMPL (->) operator"));
 }
 
 
 void ExprOpUpdate::eval(EvalState & state, Env & env, Value & v)
 {
     Value v1, v2;
-    state.evalAttrs(env, e1, v1, pos, "in the left operand of the update (//) operator");
-    state.evalAttrs(env, e2, v2, pos, "in the right operand of the update (//) operator");
+    state.evalAttrs(env, *e1, v1, pos, "in the left operand of the update (//) operator");
+    state.evalAttrs(env, *e2, v2, pos, "in the right operand of the update (//) operator");
 
     state.nrOpUpdates++;
 
@@ -2035,10 +2172,10 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
     };
 
     // List of returned strings. References to these Values must NOT be persisted.
-    SmallTemporaryValueVector<conservativeStackReservation> values(es->size());
+    SmallTemporaryValueVector<conservativeStackReservation> values(es.size());
     Value * vTmpP = values.data();
 
-    for (auto & [i_pos, i] : *es) {
+    for (auto & [i_pos, i] : es) {
         Value & vTmp = *vTmpP++;
         i->eval(state, env, vTmp);
 
@@ -2068,7 +2205,7 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
             } else
                 state.error<EvalError>("cannot add %1% to a float", showType(vTmp)).atPos(i_pos).withFrame(env, *this).debugThrow();
         } else {
-            if (s.empty()) s.reserve(es->size());
+            if (s.empty()) s.reserve(es.size());
             /* skip canonization of first path, which would only be not
             canonized in the first place if it's coming from a ./${foo} type
             path */
@@ -2752,39 +2889,39 @@ SourcePath resolveExprPath(SourcePath path)
 }
 
 
-Expr * EvalState::parseExprFromFile(const SourcePath & path)
+Expr & EvalState::parseExprFromFile(const SourcePath & path)
 {
     return parseExprFromFile(path, staticBaseEnv);
 }
 
 
-Expr * EvalState::parseExprFromFile(const SourcePath & path, std::shared_ptr<StaticEnv> & staticEnv)
+Expr & EvalState::parseExprFromFile(const SourcePath & path, std::shared_ptr<StaticEnv> & staticEnv)
 {
     auto buffer = path.readFile();
     // readFile hopefully have left some extra space for terminators
     buffer.append("\0\0", 2);
-    return parse(buffer.data(), buffer.size(), Pos::Origin(path), path.parent(), staticEnv);
+    return *parse(buffer.data(), buffer.size(), Pos::Origin(path), path.parent(), staticEnv);
 }
 
 
-Expr * EvalState::parseExprFromString(std::string s_, const SourcePath & basePath, std::shared_ptr<StaticEnv> & staticEnv)
+Expr & EvalState::parseExprFromString(std::string s_, const SourcePath & basePath, std::shared_ptr<StaticEnv> & staticEnv)
 {
     // NOTE this method (and parseStdin) must take care to *fully copy* their input
     // into their respective Pos::Origin until the parser stops overwriting its input
     // data.
     auto s = make_ref<std::string>(s_);
     s_.append("\0\0", 2);
-    return parse(s_.data(), s_.size(), Pos::String{.source = s}, basePath, staticEnv);
+    return *parse(s_.data(), s_.size(), Pos::String{.source = s}, basePath, staticEnv);
 }
 
 
-Expr * EvalState::parseExprFromString(std::string s, const SourcePath & basePath)
+Expr & EvalState::parseExprFromString(std::string s, const SourcePath & basePath)
 {
     return parseExprFromString(std::move(s), basePath, staticBaseEnv);
 }
 
 
-Expr * EvalState::parseStdin()
+Expr & EvalState::parseStdin()
 {
     // NOTE this method (and parseExprFromString) must take care to *fully copy* their
     // input into their respective Pos::Origin until the parser stops overwriting its
@@ -2794,7 +2931,7 @@ Expr * EvalState::parseStdin()
     // drainFD should have left some extra space for terminators
     auto s = make_ref<std::string>(buffer);
     buffer.append("\0\0", 2);
-    return parse(buffer.data(), buffer.size(), Pos::Stdin{.source = s}, rootPath(CanonPath::fromCwd()), staticBaseEnv);
+    return *parse(buffer.data(), buffer.size(), Pos::Stdin{.source = s}, rootPath(CanonPath::fromCwd()), staticBaseEnv);
 }
 
 
@@ -2883,21 +3020,6 @@ std::optional<std::string> EvalState::resolveSearchPathPath(const SearchPath::Pa
 }
 
 
-Expr * EvalState::parse(
-    char * text,
-    size_t length,
-    Pos::Origin origin,
-    const SourcePath & basePath,
-    std::shared_ptr<StaticEnv> & staticEnv)
-{
-    auto result = parseExprFromBuf(text, length, origin, basePath, symbols, positions, exprSymbols);
-
-    result->bindVars(*this, staticEnv);
-
-    return result;
-}
-
-
 std::string ExternalValueBase::coerceToString(EvalState & state, const PosIdx & pos, NixStringContext & context, bool copyMore, bool copyToStore) const
 {
     state.error<TypeError>(

src/libexpr/eval.hh

@@ -1,3 +1,7 @@
+// SPDX-FileCopyrightText: 2024 Nix and Lix Authors
+//
+// SPDX-License-Identifier: LGPL-2.1-only
+
 #pragma once
 ///@file
 
@@ -9,14 +13,13 @@
 #include "symbol-table.hh"
 #include "config.hh"
 #include "experimental-features.hh"
-#include "input-accessor.hh"
 #include "search-path.hh"
 #include "repl-exit-status.hh"
 
+#include <gc/gc_allocator.h>
 #include <map>
 #include <optional>
 #include <unordered_map>
-#include <mutex>
 #include <functional>
 
 namespace nix {
@@ -33,12 +36,31 @@ class EvalState;
 class StorePath;
 struct SingleDerivedPath;
 enum RepairFlag : bool;
+struct MemoryInputAccessor;
+namespace eval_cache {
+    class EvalCache;
+}
+
+/** Alias for std::map which uses boehmgc's allocator conditional on us actually
+ * using boehmgc in this build.
+ */
+#if HAVE_BOEHMGC
+    template<typename KeyT, typename ValueT>
+    using GcMap = std::map<
+        KeyT,
+        ValueT,
+        std::less<KeyT>,
+        traceable_allocator<std::pair<KeyT const, ValueT>>
+    >;
+#else
+    using GcMap = std::map<KeyT, ValueT>
+#endif
 
 
 /**
  * Function that implements a primop.
  */
-typedef void (* PrimOpFun) (EvalState & state, const PosIdx pos, Value * * args, Value & v);
+using PrimOpImpl = void(EvalState & state, PosIdx pos, Value ** args, Value & v);
 
 /**
  * Info about a primitive operation, and its implementation
@@ -72,7 +94,7 @@ struct PrimOp
     /**
      * Implementation of the primop.
      */
-    std::function<std::remove_pointer<PrimOpFun>::type> fun;
+    std::function<PrimOpImpl> fun;
 
     /**
      * Optional experimental for this to be gated on.
@@ -111,11 +133,7 @@ struct Constant
     bool impureOnly = false;
 };
 
-#if HAVE_BOEHMGC
-    typedef std::map<std::string, Value *, std::less<std::string>, traceable_allocator<std::pair<const std::string, Value *> > > ValMap;
-#else
-    typedef std::map<std::string, Value *> ValMap;
-#endif
+using ValMap = GcMap<std::string, Value *>;
 
 struct Env
 {
@@ -210,7 +228,7 @@ public:
     /**
      * Debugger
      */
-    ReplExitStatus (* debugRepl)(ref<EvalState> es, const ValMap & extraEnv);
+    std::function<ReplExitStatus(ref<EvalState> es, ValMap const & extraEnv)> debugRepl;
     bool debugStop;
     bool inDebugger = false;
     int trylevel;
@@ -234,6 +252,11 @@ public:
         return *new EvalErrorBuilder<T>(*this, args...);
     }
 
+    /**
+     * A cache for evaluation caches, so as to reuse the same root value if possible
+     */
+    std::map<const Hash, ref<eval_cache::EvalCache>> evalCaches;
+
 private:
 
     /* Cache for calls to addToStore(); maps source paths to the store
@@ -243,21 +266,13 @@ private:
     /**
      * A cache from path names to parse trees.
      */
-#if HAVE_BOEHMGC
-    typedef std::map<SourcePath, Expr *, std::less<SourcePath>, traceable_allocator<std::pair<const SourcePath, Expr *>>> FileParseCache;
-#else
-    typedef std::map<SourcePath, Expr *> FileParseCache;
-#endif
+    using FileParseCache = GcMap<SourcePath, Expr *>;
     FileParseCache fileParseCache;
 
     /**
      * A cache from path names to values.
      */
-#if HAVE_BOEHMGC
-    typedef std::map<SourcePath, Value, std::less<SourcePath>, traceable_allocator<std::pair<const SourcePath, Value>>> FileEvalCache;
-#else
-    typedef std::map<SourcePath, Value> FileEvalCache;
-#endif
+    using FileEvalCache = GcMap<SourcePath, Value>;
     FileEvalCache fileEvalCache;
 
     SearchPath searchPath;
@@ -340,16 +355,16 @@ public:
     /**
      * Parse a Nix expression from the specified file.
      */
-    Expr * parseExprFromFile(const SourcePath & path);
-    Expr * parseExprFromFile(const SourcePath & path, std::shared_ptr<StaticEnv> & staticEnv);
+    Expr & parseExprFromFile(const SourcePath & path);
+    Expr & parseExprFromFile(const SourcePath & path, std::shared_ptr<StaticEnv> & staticEnv);
 
     /**
      * Parse a Nix expression from the specified string.
      */
-    Expr * parseExprFromString(std::string s, const SourcePath & basePath, std::shared_ptr<StaticEnv> & staticEnv);
-    Expr * parseExprFromString(std::string s, const SourcePath & basePath);
+    Expr & parseExprFromString(std::string s, const SourcePath & basePath, std::shared_ptr<StaticEnv> & staticEnv);
+    Expr & parseExprFromString(std::string s, const SourcePath & basePath);
 
-    Expr * parseStdin();
+    Expr & parseStdin();
 
     /**
      * Evaluate an expression read from the given file to normal
@@ -390,15 +405,15 @@ public:
      *
      * @param [out] v The resulting is stored here.
      */
-    void eval(Expr * e, Value & v);
+    void eval(Expr & e, Value & v);
 
     /**
      * Evaluation the expression, then verify that it has the expected
      * type.
      */
-    inline bool evalBool(Env & env, Expr * e);
-    inline bool evalBool(Env & env, Expr * e, const PosIdx pos, std::string_view errorCtx);
-    inline void evalAttrs(Env & env, Expr * e, Value & v, const PosIdx pos, std::string_view errorCtx);
+    inline bool evalBool(Env & env, Expr & e);
+    inline bool evalBool(Env & env, Expr & e, const PosIdx pos, std::string_view errorCtx);
+    inline void evalAttrs(Env & env, Expr & e, Value & v, const PosIdx pos, std::string_view errorCtx);
 
     /**
      * If `v` is a thunk, enter it and overwrite `v` with the result
@@ -619,7 +634,7 @@ public:
     }
 
     void mkList(Value & v, size_t length);
-    void mkThunk_(Value & v, Expr * expr);
+    void mkThunk_(Value & v, Expr & expr);
     void mkPos(Value & v, PosIdx pos);
 
     /**
@@ -728,15 +743,15 @@ private:
 
     bool countCalls;
 
-    typedef std::map<std::string, size_t> PrimOpCalls;
+    using PrimOpCalls = std::map<std::string, size_t>;
     PrimOpCalls primOpCalls;
 
-    typedef std::map<ExprLambda *, size_t> FunctionCalls;
+    using FunctionCalls = std::map<ExprLambda *, size_t>;
     FunctionCalls functionCalls;
 
     void incrFunctionCall(ExprLambda * fun);
 
-    typedef std::map<PosIdx, size_t> AttrSelects;
+    using AttrSelects = std::map<PosIdx, size_t>;
     AttrSelects attrSelects;
 
     friend struct ExprOpUpdate;
@@ -778,7 +793,7 @@ std::string showType(const Value & v);
  */
 SourcePath resolveExprPath(SourcePath path);
 
-static const std::string corepkgsPrefix{"/__corepkgs__/"};
+static constexpr std::string_view corepkgsPrefix{"/__corepkgs__/"};
 
 
 }

src/libexpr/fetchurl.nix

@@ -1,3 +1,7 @@
+# SPDX-FileCopyrightText: 2024 Nix and Lix Authors
+#
+# SPDX-License-Identifier: LGPL-2.1-only
+
 {
   system ? "", # obsolete
   url,

src/libexpr/flake/call-flake.nix

@@ -1,3 +1,7 @@
+# SPDX-FileCopyrightText: 2024 Nix and Lix Authors
+#
+# SPDX-License-Identifier: LGPL-2.1-only
+
 lockFileStr: rootSrc: rootSubdir:
 
 let

src/libexpr/flake/config.cc

@@ -1,3 +1,7 @@
+// SPDX-FileCopyrightText: 2024 Nix and Lix Authors
+//
+// SPDX-License-Identifier: LGPL-2.1-only
+
 #include "flake.hh"
 #include "logging.hh"
 #include "users.hh"
@@ -51,30 +55,46 @@ void ConfigFile::apply()
         else
             assert(false);
 
-        if (!whitelist.count(baseName) && !nix::fetchSettings.acceptFlakeConfig) {
-            bool trusted = false;
-            auto trustedList = readTrustedList();
-            auto tlname = get(trustedList, name);
-            if (auto saved = tlname ? get(*tlname, valueS) : nullptr) {
-                trusted = *saved;
-                printInfo("Using saved setting for '%s = %s' from ~/.local/share/nix/trusted-settings.json.", name, valueS);
-            } else {
-                // FIXME: filter ANSI escapes, newlines, \r, etc.
-                if (std::tolower(logger->ask(fmt("do you want to allow configuration setting '%s' to be set to '" ANSI_RED "%s" ANSI_NORMAL "' (y/N)?", name, valueS)).value_or('n')) == 'y') {
-                    trusted = true;
-                }
-                if (std::tolower(logger->ask(fmt("do you want to permanently mark this value as %s (y/N)?",  trusted ? "trusted": "untrusted" )).value_or('n')) == 'y') {
-                    trustedList[name][valueS] = trusted;
-                    writeTrustedList(trustedList);
-                }
+        bool trusted = whitelist.count(baseName);
+        if (!trusted) {
+            switch (nix::fetchSettings.acceptFlakeConfig) {
+            case AcceptFlakeConfig::True: {
+                trusted = true;
+                break;
             }
-            if (!trusted) {
-                warn("ignoring untrusted flake configuration setting '%s'.\nPass '%s' to trust it", name, "--accept-flake-config");
-                continue;
+            case AcceptFlakeConfig::Ask: {
+                auto trustedList = readTrustedList();
+                auto tlname = get(trustedList, name);
+                if (auto saved = tlname ? get(*tlname, valueS) : nullptr) {
+                    trusted = *saved;
+                    printInfo("Using saved setting for '%s = %s' from ~/.local/share/nix/trusted-settings.json.", name, valueS);
+                } else {
+                    // FIXME: filter ANSI escapes, newlines, \r, etc.
+                    if (std::tolower(logger->ask(fmt("Do you want to allow configuration setting '%s' to be set to '" ANSI_RED "%s" ANSI_NORMAL "' (y/N)? This may allow the flake to gain root, see the nix.conf manual page.", name, valueS)).value_or('n')) == 'y') {
+                        trusted = true;
+                    } else {
+                        warn("you can set '%s' to '%b' to automatically reject configuration options supplied by flakes", "accept-flake-config", false);
+                    }
+                    if (std::tolower(logger->ask(fmt("do you want to permanently mark this value as %s (y/N)?",  trusted ? "trusted": "untrusted" )).value_or('n')) == 'y') {
+                        trustedList[name][valueS] = trusted;
+                        writeTrustedList(trustedList);
+                    }
+                }
+                break;
+            }
+            case nix::AcceptFlakeConfig::False: {
+                trusted = false;
+                break;
+            };
             }
         }
 
-        globalConfig.set(name, valueS);
+        if (trusted) {
+            debug("accepting trusted flake configuration setting '%s'", name);
+            globalConfig.set(name, valueS);
+        } else {
+            warn("ignoring untrusted flake configuration setting '%s', pass '%s' to trust it (may allow the flake to gain root, see the nix.conf manual page)", name, "--accept-flake-config");
+        }
     }
 }
 

src/libexpr/flake/flake.cc

@@ -1,3 +1,7 @@
+// SPDX-FileCopyrightText: 2024 Nix and Lix Authors
+//
+// SPDX-License-Identifier: LGPL-2.1-only
+
 #include "flake.hh"
 #include "eval.hh"
 #include "eval-settings.hh"

src/libexpr/flake/flake.hh

@@ -1,3 +1,7 @@
+// SPDX-FileCopyrightText: 2024 Nix and Lix Authors
+//
+// SPDX-License-Identifier: LGPL-2.1-only
+
 #pragma once
 ///@file
 

src/libexpr/flake/flakeref.cc

@@ -1,3 +1,7 @@
+// SPDX-FileCopyrightText: 2024 Nix and Lix Authors
+//
+// SPDX-License-Identifier: LGPL-2.1-only
+
 #include "flakeref.hh"
 #include "store-api.hh"
 #include "url.hh"
@@ -85,8 +89,8 @@ std::pair<FlakeRef, std::string> parseFlakeRefWithFragment(
         + "(?:#(" + queryRegex + "))?",
         std::regex::ECMAScript);
 
-    static std::regex flakeRegex(
-        "((" + flakeIdRegexS + ")(?:/(?:" + refAndOrRevRegex + "))?)"
+    static std::regex flakeShorthandRegex(
+        flakeShorthandRegexS
         + "(?:#(" + queryRegex + "))?",
         std::regex::ECMAScript);
 
@@ -95,7 +99,7 @@ std::pair<FlakeRef, std::string> parseFlakeRefWithFragment(
     /* Check if 'url' is a flake ID. This is an abbreviated syntax for
        'flake:<flake-id>?ref=<ref>&rev=<rev>'. */
 
-    if (std::regex_match(url, match, flakeRegex)) {
+    if (std::regex_match(url, match, flakeShorthandRegex)) {
         auto parsedURL = ParsedURL{
             .url = url,
             .base = "flake:" + match.str(1),

src/libexpr/flake/flakeref.hh

@@ -1,3 +1,7 @@
+// SPDX-FileCopyrightText: 2024 Nix and Lix Authors
+//
+// SPDX-License-Identifier: LGPL-2.1-only
+
 #pragma once
 ///@file
 

src/libexpr/flake/lockfile.cc

@@ -1,3 +1,7 @@
+// SPDX-FileCopyrightText: 2024 Nix and Lix Authors
+//
+// SPDX-License-Identifier: LGPL-2.1-only
+
 #include "lockfile.hh"
 #include "store-api.hh"
 #include "url-parts.hh"

src/libexpr/flake/lockfile.hh

@@ -1,3 +1,7 @@
+// SPDX-FileCopyrightText: 2024 Nix and Lix Authors
+//
+// SPDX-License-Identifier: LGPL-2.1-only
+
 #pragma once
 ///@file
 

src/libexpr/flake/meson.build

@@ -1,3 +1,7 @@
+# SPDX-FileCopyrightText: 2024 Nix and Lix Authors
+#
+# SPDX-License-Identifier: LGPL-2.1-only
+
 libexpr_generated_headers += custom_target(
   command : [ 'bash', '-c', 'echo \'R"__NIX_STR(\' | cat - @INPUT@ && echo \')__NIX_STR"\'' ],
   input : 'call-flake.nix',

src/libexpr/function-trace.cc

@@ -1,3 +1,7 @@
+// SPDX-FileCopyrightText: 2024 Nix and Lix Authors
+//
+// SPDX-License-Identifier: LGPL-2.1-only
+
 #include "function-trace.hh"
 #include "logging.hh"
 

src/libexpr/function-trace.hh

@@ -1,3 +1,7 @@
+// SPDX-FileCopyrightText: 2024 Nix and Lix Authors
+//
+// SPDX-License-Identifier: LGPL-2.1-only
+
 #pragma once
 ///@file
 

src/libexpr/gc-small-vector.hh

@@ -1,3 +1,7 @@
+// SPDX-FileCopyrightText: 2024 Nix and Lix Authors
+//
+// SPDX-License-Identifier: LGPL-2.1-only
+
 #pragma once
 ///@file
 

src/libexpr/get-drvs.cc

@@ -1,3 +1,7 @@
+// SPDX-FileCopyrightText: 2024 Nix and Lix Authors
+//
+// SPDX-License-Identifier: LGPL-2.1-only
+
 #include "get-drvs.hh"
 #include "eval-inline.hh"
 #include "derivations.hh"
@@ -45,7 +49,7 @@ DrvInfo::DrvInfo(EvalState & state, ref<Store> store, const std::string & drvPat
 }
 
 
-std::string DrvInfo::queryName() const
+std::string DrvInfo::queryName()
 {
     if (name == "" && attrs) {
         auto i = attrs->find(state->sName);
@@ -56,7 +60,7 @@ std::string DrvInfo::queryName() const
 }
 
 
-std::string DrvInfo::querySystem() const
+std::string DrvInfo::querySystem()
 {
     if (system == "" && attrs) {
         auto i = attrs->find(state->sSystem);
@@ -66,7 +70,7 @@ std::string DrvInfo::querySystem() const
 }
 
 
-std::optional<StorePath> DrvInfo::queryDrvPath() const
+std::optional<StorePath> DrvInfo::queryDrvPath()
 {
     if (!drvPath && attrs) {
         Bindings::iterator i = attrs->find(state->sDrvPath);
@@ -80,7 +84,7 @@ std::optional<StorePath> DrvInfo::queryDrvPath() const
 }
 
 
-StorePath DrvInfo::requireDrvPath() const
+StorePath DrvInfo::requireDrvPath()
 {
     if (auto drvPath = queryDrvPath())
         return *drvPath;
@@ -88,7 +92,7 @@ StorePath DrvInfo::requireDrvPath() const
 }
 
 
-StorePath DrvInfo::queryOutPath() const
+StorePath DrvInfo::queryOutPath()
 {
     if (!outPath && attrs) {
         Bindings::iterator i = attrs->find(state->sOutPath);
@@ -164,7 +168,7 @@ DrvInfo::Outputs DrvInfo::queryOutputs(bool withPaths, bool onlyOutputsToInstall
 }
 
 
-std::string DrvInfo::queryOutputName() const
+std::string DrvInfo::queryOutputName()
 {
     if (outputName == "" && attrs) {
         Bindings::iterator i = attrs->find(state->sOutputName);

src/libexpr/get-drvs.hh

@@ -1,3 +1,7 @@
+// SPDX-FileCopyrightText: 2024 Nix and Lix Authors
+//
+// SPDX-License-Identifier: LGPL-2.1-only
+
 #pragma once
 ///@file
 
@@ -19,11 +23,11 @@ public:
 private:
     EvalState * state;
 
-    mutable std::string name;
-    mutable std::string system;
-    mutable std::optional<std::optional<StorePath>> drvPath;
-    mutable std::optional<StorePath> outPath;
-    mutable std::string outputName;
+    std::string name;
+    std::string system;
+    std::optional<std::optional<StorePath>> drvPath;
+    std::optional<StorePath> outPath;
+    std::string outputName;
     Outputs outputs;
 
     /**
@@ -47,12 +51,12 @@ public:
     DrvInfo(EvalState & state, std::string attrPath, Bindings * attrs);
     DrvInfo(EvalState & state, ref<Store> store, const std::string & drvPathWithOutputs);
 
-    std::string queryName() const;
-    std::string querySystem() const;
-    std::optional<StorePath> queryDrvPath() const;
-    StorePath requireDrvPath() const;
-    StorePath queryOutPath() const;
-    std::string queryOutputName() const;
+    std::string queryName();
+    std::string querySystem();
+    std::optional<StorePath> queryDrvPath();
+    StorePath requireDrvPath();
+    StorePath queryOutPath();
+    std::string queryOutputName();
     /**
      * Return the unordered map of output names to (optional) output paths.
      * The "outputs to install" are determined by `meta.outputsToInstall`.

src/libexpr/imported-drv-to-derivation.nix

@@ -1,3 +1,7 @@
+# SPDX-FileCopyrightText: 2024 Nix and Lix Authors
+#
+# SPDX-License-Identifier: LGPL-2.1-only
+
 attrs@{
   drvPath,
   outputs,

src/libexpr/json-to-value.cc

@@ -1,3 +1,7 @@
+// SPDX-FileCopyrightText: 2024 Nix and Lix Authors
+//
+// SPDX-License-Identifier: LGPL-2.1-only
+
 #include "json-to-value.hh"
 #include "value.hh"
 #include "eval.hh"

src/libexpr/json-to-value.hh

@@ -1,3 +1,7 @@
+// SPDX-FileCopyrightText: 2024 Nix and Lix Authors
+//
+// SPDX-License-Identifier: LGPL-2.1-only
+
 #pragma once
 ///@file
 

src/libexpr/lexer.l

@@ -1,296 +0,0 @@
-%option reentrant bison-bridge bison-locations
-%option align
-%option noyywrap
-%option never-interactive
-%option stack
-%option nodefault
-%option nounput noyy_top_state
-
-
-%s DEFAULT
-%x STRING
-%x IND_STRING
-%x INPATH
-%x INPATH_SLASH
-%x PATH_START
-
-
-%{
-#ifdef __clang__
-#pragma clang diagnostic ignored "-Wunneeded-internal-declaration"
-#endif
-
-// yacc generates code that uses unannotated fallthrough.
-#pragma GCC diagnostic ignored "-Wimplicit-fallthrough"
-#ifdef __clang__
-#pragma clang diagnostic ignored "-Wimplicit-fallthrough"
-#endif
-
-#include "nixexpr.hh"
-#include "parser-tab.hh"
-#include "strings.hh"
-
-using namespace nix;
-
-namespace nix {
-
-#define CUR_POS state->at(*yylloc)
-
-static void initLoc(YYLTYPE * loc)
-{
-    loc->first_line = loc->last_line = 0;
-    loc->first_column = loc->last_column = 0;
-}
-
-static void adjustLoc(YYLTYPE * loc, const char * s, size_t len)
-{
-    loc->stash();
-
-    loc->first_column = loc->last_column;
-    loc->last_column += len;
-}
-
-
-// we make use of the fact that the parser receives a private copy of the input
-// string and can munge around in it.
-static StringToken unescapeStr(SymbolTable & symbols, char * s, size_t length)
-{
-    char * result = s;
-    char * t = s;
-    char c;
-    // the input string is terminated with *two* NULs, so we can safely take
-    // *one* character after the one being checked against.
-    while ((c = *s++)) {
-        if (c == '\\') {
-            c = *s++;
-            if (c == 'n') *t = '\n';
-            else if (c == 'r') *t = '\r';
-            else if (c == 't') *t = '\t';
-            else *t = c;
-        }
-        else if (c == '\r') {
-            /* Normalise CR and CR/LF into LF. */
-            *t = '\n';
-            if (*s == '\n') s++; /* cr/lf */
-        }
-        else *t = c;
-        t++;
-    }
-    return {result, size_t(t - result)};
-}
-
-
-}
-
-#define YY_USER_INIT initLoc(yylloc)
-#define YY_USER_ACTION adjustLoc(yylloc, yytext, yyleng);
-
-#define PUSH_STATE(state) yy_push_state(state, yyscanner)
-#define POP_STATE() yy_pop_state(yyscanner)
-
-%}
-
-
-ANY         .|\n
-ID          [a-zA-Z\_][a-zA-Z0-9\_\'\-]*
-INT         [0-9]+
-FLOAT       (([1-9][0-9]*\.[0-9]*)|(0?\.[0-9]+))([Ee][+-]?[0-9]+)?
-PATH_CHAR   [a-zA-Z0-9\.\_\-\+]
-PATH        {PATH_CHAR}*(\/{PATH_CHAR}+)+\/?
-PATH_SEG    {PATH_CHAR}*\/
-HPATH       \~(\/{PATH_CHAR}+)+\/?
-HPATH_START \~\/
-SPATH       \<{PATH_CHAR}+(\/{PATH_CHAR}+)*\>
-URI         [a-zA-Z][a-zA-Z0-9\+\-\.]*\:[a-zA-Z0-9\%\/\?\:\@\&\=\+\$\,\-\_\.\!\~\*\']+
-
-
-%%
-
-
-if          { return IF; }
-then        { return THEN; }
-else        { return ELSE; }
-assert      { return ASSERT; }
-with        { return WITH; }
-let         { return LET; }
-in          { return IN; }
-rec         { return REC; }
-inherit     { return INHERIT; }
-or          { return OR_KW; }
-\.\.\.      { return ELLIPSIS; }
-
-\=\=        { return EQ; }
-\!\=        { return NEQ; }
-\<\=        { return LEQ; }
-\>\=        { return GEQ; }
-\&\&        { return AND; }
-\|\|        { return OR; }
-\-\>        { return IMPL; }
-\/\/        { return UPDATE; }
-\+\+        { return CONCAT; }
-
-{ID}        { yylval->id = {yytext, (size_t) yyleng}; return ID; }
-{INT}       { errno = 0;
-              std::optional<int64_t> numMay = string2Int<int64_t>(yytext);
-              if (numMay.has_value()) {
-                  yylval->n = *numMay;
-              } else {
-                  throw ParseError(ErrorInfo{
-                      .msg = HintFmt("invalid integer '%1%'", yytext),
-                      .pos = state->positions[CUR_POS],
-                  });
-              }
-              return INT;
-            }
-{FLOAT}     { errno = 0;
-              yylval->nf = strtod(yytext, 0);
-              if (errno != 0)
-                  throw ParseError(ErrorInfo{
-                      .msg = HintFmt("invalid float '%1%'", yytext),
-                      .pos = state->positions[CUR_POS],
-                  });
-              return FLOAT;
-            }
-
-\$\{        { PUSH_STATE(DEFAULT); return DOLLAR_CURLY; }
-
-\}          { /* State INITIAL only exists at the bottom of the stack and is
-                 used as a marker. DEFAULT replaces it everywhere else.
-                 Popping when in INITIAL state causes an empty stack exception,
-                 so don't */
-              if (YYSTATE != INITIAL)
-                POP_STATE();
-              return '}';
-            }
-\{          { PUSH_STATE(DEFAULT); return '{'; }
-
-\"          { PUSH_STATE(STRING); return '"'; }
-<STRING>([^\$\"\\]|\$[^\{\"\\]|\\{ANY}|\$\\{ANY})*\$/\" |
-<STRING>([^\$\"\\]|\$[^\{\"\\]|\\{ANY}|\$\\{ANY})+ {
-                /* It is impossible to match strings ending with '$' with one
-                   regex because trailing contexts are only valid at the end
-                   of a rule. (A sane but undocumented limitation.) */
-                yylval->str = unescapeStr(state->symbols, yytext, yyleng);
-                return STR;
-              }
-<STRING>\$\{  { PUSH_STATE(DEFAULT); return DOLLAR_CURLY; }
-<STRING>\"    { POP_STATE(); return '"'; }
-<STRING>\$|\\|\$\\ {
-                /* This can only occur when we reach EOF, otherwise the above
-                   (...|\$[^\{\"\\]|\\.|\$\\.)+ would have triggered.
-                   This is technically invalid, but we leave the problem to the
-                   parser who fails with exact location. */
-                return EOF;
-              }
-
-\'\'(\ *\n)?     { PUSH_STATE(IND_STRING); return IND_STRING_OPEN; }
-<IND_STRING>([^\$\']|\$[^\{\']|\'[^\'\$])+ {
-                   yylval->str = {yytext, (size_t) yyleng, true};
-                   return IND_STR;
-                 }
-<IND_STRING>\'\'\$ |
-<IND_STRING>\$   {
-                   yylval->str = {"$", 1};
-                   return IND_STR;
-                 }
-<IND_STRING>\'\'\' {
-                   yylval->str = {"''", 2};
-                   return IND_STR;
-                 }
-<IND_STRING>\'\'\\{ANY} {
-                   yylval->str = unescapeStr(state->symbols, yytext + 2, yyleng - 2);
-                   return IND_STR;
-                 }
-<IND_STRING>\$\{ { PUSH_STATE(DEFAULT); return DOLLAR_CURLY; }
-<IND_STRING>\'\' { POP_STATE(); return IND_STRING_CLOSE; }
-<IND_STRING>\'   {
-                   yylval->str = {"'", 1};
-                   return IND_STR;
-                 }
-
-{PATH_SEG}\$\{ |
-{HPATH_START}\$\{ {
-  PUSH_STATE(PATH_START);
-  yyless(0);
-  yylloc->unstash();
-}
-
-<PATH_START>{PATH_SEG} {
-  POP_STATE();
-  PUSH_STATE(INPATH_SLASH);
-  yylval->path = {yytext, (size_t) yyleng};
-  return PATH;
-}
-
-<PATH_START>{HPATH_START} {
-  POP_STATE();
-  PUSH_STATE(INPATH_SLASH);
-  yylval->path = {yytext, (size_t) yyleng};
-  return HPATH;
-}
-
-{PATH} {
-  if (yytext[yyleng-1] == '/')
-    PUSH_STATE(INPATH_SLASH);
-  else
-    PUSH_STATE(INPATH);
-  yylval->path = {yytext, (size_t) yyleng};
-  return PATH;
-}
-{HPATH} {
-  if (yytext[yyleng-1] == '/')
-    PUSH_STATE(INPATH_SLASH);
-  else
-    PUSH_STATE(INPATH);
-  yylval->path = {yytext, (size_t) yyleng};
-  return HPATH;
-}
-
-<INPATH,INPATH_SLASH>\$\{ {
-  POP_STATE();
-  PUSH_STATE(INPATH);
-  PUSH_STATE(DEFAULT);
-  return DOLLAR_CURLY;
-}
-<INPATH,INPATH_SLASH>{PATH}|{PATH_SEG}|{PATH_CHAR}+ {
-  POP_STATE();
-  if (yytext[yyleng-1] == '/')
-      PUSH_STATE(INPATH_SLASH);
-  else
-      PUSH_STATE(INPATH);
-  yylval->str = {yytext, (size_t) yyleng};
-  return STR;
-}
-<INPATH>{ANY} |
-<INPATH><<EOF>> {
-  /* if we encounter a non-path character we inform the parser that the path has
-     ended with a PATH_END token and re-parse this character in the default
-     context (it may be ')', ';', or something of that sort) */
-  POP_STATE();
-  yyless(0);
-  yylloc->unstash();
-  return PATH_END;
-}
-
-<INPATH_SLASH>{ANY} |
-<INPATH_SLASH><<EOF>> {
-  throw ParseError(ErrorInfo{
-      .msg = HintFmt("path has a trailing slash"),
-      .pos = state->positions[CUR_POS],
-  });
-}
-
-{SPATH}     { yylval->path = {yytext, (size_t) yyleng}; return SPATH; }
-{URI}       { yylval->uri = {yytext, (size_t) yyleng}; return URI; }
-
-[ \t\r\n]+    /* eat up whitespace */
-\#[^\r\n]*    /* single-line comments */
-\/\*([^*]|\*+[^*/])*\*+\/  /* long comments */
-
-{ANY}       {
-              /* Don't return a negative number, as this will cause
-                 Bison to stop parsing without an error. */
-              return (unsigned char) yytext[0];
-            }
-
-%%

src/libexpr/meson.build

@@ -1,53 +1,6 @@
-parser_tab = custom_target(
-  input : 'parser.y',
-  output : [
-    'parser-tab.cc',
-    'parser-tab.hh',
-  ],
-  command : [
-    'bison',
-    '-v',
-    '-o',
-    '@OUTPUT0@',
-    '@INPUT@',
-    '-d',
-  ],
-  # NOTE(Qyriad): Meson doesn't support installing only part of a custom target, so we add
-  # an install script below which removes parser-tab.cc.
-  install : true,
-  install_dir : includedir / 'lix/libexpr',
-)
-
-lexer_tab = custom_target(
-  input : [
-    'lexer.l',
-    parser_tab,
-  ],
-  output : [
-    'lexer-tab.cc',
-    'lexer-tab.hh',
-  ],
-  command : [
-    'flex',
-    '--outfile',
-    '@OUTPUT0@',
-    '--header-file=' + '@OUTPUT1@',
-    '@INPUT0@',
-  ],
-  # NOTE(Qyriad): Meson doesn't support installing only part of a custom target, so we add
-  # an install script below which removes lexer-tab.cc.
-  install : true,
-  install_dir : includedir / 'lix/libexpr',
-)
-
-# TODO(Qyriad): When the parser and lexer are rewritten this should be removed.
-# NOTE(Qyriad): We do this this way instead of an inline bash or rm command
-# due to subtleties in Meson. Check the comments in cleanup-install.bash for details.
-meson.add_install_script(
-  bash,
-  meson.project_source_root() / 'meson/cleanup-install.bash',
-  '@0@'.format(includedir),
-)
+# SPDX-FileCopyrightText: 2024 Nix and Lix Authors
+#
+# SPDX-License-Identifier: LGPL-2.1-only
 
 libexpr_generated_headers = [
   gen_header.process('primops/derivation.nix', preserve_path_from : meson.current_source_dir()),
@@ -75,6 +28,7 @@ libexpr_sources = files(
   'get-drvs.cc',
   'json-to-value.cc',
   'nixexpr.cc',
+  'parser/parser.cc',
   'paths.cc',
   'primops.cc',
   'print-ambiguous.cc',
@@ -110,7 +64,9 @@ libexpr_headers = files(
   'get-drvs.hh',
   'json-to-value.hh',
   'nixexpr.hh',
-  'parser-state.hh',
+  'parser/change_head.hh',
+  'parser/grammar.hh',
+  'parser/state.hh',
   'pos-idx.hh',
   'pos-table.hh',
   'primops.hh',
@@ -129,8 +85,6 @@ libexpr_headers = files(
 libexpr = library(
   'lixexpr',
   libexpr_sources,
-  parser_tab,
-  lexer_tab,
   libexpr_generated_headers,
   dependencies : [
     liblixutil,

src/libexpr/nixexpr.cc

@@ -1,3 +1,7 @@
+// SPDX-FileCopyrightText: 2024 Nix and Lix Authors
+//
+// SPDX-License-Identifier: LGPL-2.1-only
+
 #include "nixexpr.hh"
 #include "derivations.hh"
 #include "eval.hh"
@@ -50,6 +54,13 @@ void ExprVar::show(const SymbolTable & symbols, std::ostream & str) const
     str << symbols[name];
 }
 
+void ExprInheritFrom::show(SymbolTable const & symbols, std::ostream & str) const
+{
+    str << "(/* expanded inherit (expr) */ ";
+    fromExpr->show(symbols, str);
+    str << ")";
+}
+
 void ExprSelect::show(const SymbolTable & symbols, std::ostream & str) const
 {
     str << "(";
@@ -79,7 +90,7 @@ void ExprAttrs::showBindings(const SymbolTable & symbols, std::ostream & str) co
         return sa < sb;
     });
     std::vector<Symbol> inherits;
-    std::map<ExprInheritFrom *, std::vector<Symbol>> inheritsFrom;
+    std::map<Displacement, std::vector<Symbol>> inheritsFrom;
     for (auto & i : sorted) {
         switch (i->second.kind) {
         case AttrDef::Kind::Plain:
@@ -90,7 +101,7 @@ void ExprAttrs::showBindings(const SymbolTable & symbols, std::ostream & str) co
         case AttrDef::Kind::InheritedFrom: {
             auto & select = dynamic_cast<ExprSelect &>(*i->second.e);
             auto & from = dynamic_cast<ExprInheritFrom &>(*select.e);
-            inheritsFrom[&from].push_back(i->first);
+            inheritsFrom[from.displ].push_back(i->first);
             break;
         }
         }
@@ -102,7 +113,7 @@ void ExprAttrs::showBindings(const SymbolTable & symbols, std::ostream & str) co
     }
     for (const auto & [from, syms] : inheritsFrom) {
         str << "inherit (";
-        (*inheritFromExprs)[from->displ]->show(symbols, str);
+        (*inheritFromExprs)[from]->show(symbols, str);
         str << ")";
         for (auto sym : syms) str << " " << symbols[sym];
         str << "; ";
@@ -151,7 +162,7 @@ void ExprLambda::show(const SymbolTable & symbols, std::ostream & str) const
         // the natural Symbol ordering is by creation time, which can lead to the
         // same expression being printed in two different ways depending on its
         // context. always use lexicographic ordering to avoid this.
-        for (auto & i : formals->lexicographicOrder(symbols)) {
+        for (const Formal & i : formals->lexicographicOrder(symbols)) {
             if (first) first = false; else str << ", ";
             str << symbols[i.name];
             if (i.def) {
@@ -176,7 +187,7 @@ void ExprCall::show(const SymbolTable & symbols, std::ostream & str) const
 {
     str << '(';
     fun->show(symbols, str);
-    for (auto e : args) {
+    for (auto & e : args) {
         str <<  ' ';
         e->show(symbols, str);
     }
@@ -231,7 +242,7 @@ void ExprConcatStrings::show(const SymbolTable & symbols, std::ostream & str) co
 {
     bool first = true;
     str << "(";
-    for (auto & i : *es) {
+    for (auto & i : es) {
         if (first) first = false; else str << " + ";
         i.second->show(symbols, str);
     }
@@ -375,7 +386,7 @@ std::shared_ptr<const StaticEnv> ExprAttrs::bindInheritSources(
     // not even *have* an expr that grabs anything from this env since it's fully
     // invisible, but the evaluator does not allow for this yet.
     auto inner = std::make_shared<StaticEnv>(nullptr, env.get(), 0);
-    for (auto from : *inheritFromExprs)
+    for (auto & from : *inheritFromExprs)
         from->bindVars(es, env);
 
     return inner;
@@ -462,7 +473,7 @@ void ExprCall::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> &
         es.exprEnvs.insert(std::make_pair(this, env));
 
     fun->bindVars(es, env);
-    for (auto e : args)
+    for (auto & e : args)
         e->bindVars(es, env);
 }
 
@@ -547,7 +558,7 @@ void ExprConcatStrings::bindVars(EvalState & es, const std::shared_ptr<const Sta
     if (es.debugRepl)
         es.exprEnvs.insert(std::make_pair(this, env));
 
-    for (auto & i : *this->es)
+    for (auto & i : this->es)
         i.second->bindVars(es, env);
 }
 

src/libexpr/nixexpr.hh

@@ -1,3 +1,7 @@
+// SPDX-FileCopyrightText: 2024 Nix and Lix Authors
+//
+// SPDX-License-Identifier: LGPL-2.1-only
+
 #pragma once
 ///@file
 
@@ -28,9 +32,9 @@ struct StaticEnv;
 struct AttrName
 {
     Symbol symbol;
-    Expr * expr;
+    std::unique_ptr<Expr> expr;
     AttrName(Symbol s) : symbol(s) {};
-    AttrName(Expr * e) : expr(e) {};
+    AttrName(std::unique_ptr<Expr> e) : expr(std::move(e)) {};
 };
 
 typedef std::vector<AttrName> AttrPath;
@@ -42,12 +46,21 @@ std::string showAttrPath(const SymbolTable & symbols, const AttrPath & attrPath)
 
 struct Expr
 {
+protected:
+    Expr(Expr &&) = default;
+    Expr & operator=(Expr &&) = default;
+
+public:
     struct AstSymbols {
         Symbol sub, lessThan, mul, div, or_, findFile, nixPath, body;
     };
 
 
+    Expr() = default;
+    Expr(const Expr &) = delete;
+    Expr & operator=(const Expr &) = delete;
     virtual ~Expr() { };
+
     virtual void show(const SymbolTable & symbols, std::ostream & str) const;
     virtual void bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> & env);
     virtual void eval(EvalState & state, Env & env, Value & v);
@@ -135,32 +148,46 @@ struct ExprVar : Expr
  */
 struct ExprInheritFrom : ExprVar
 {
-    ExprInheritFrom(PosIdx pos, Displacement displ): ExprVar(pos, {})
+    ref<Expr> fromExpr;
+
+    ExprInheritFrom(PosIdx pos, Displacement displ, ref<Expr> fromExpr)
+          : ExprVar(pos, {}), fromExpr(fromExpr)
     {
         this->level = 0;
         this->displ = displ;
         this->fromWith = nullptr;
     }
 
-    void bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> & env);
+    void show(SymbolTable const & symbols, std::ostream & str) const override;
+    void bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> & env) override;
 };
 
 struct ExprSelect : Expr
 {
     PosIdx pos;
-    Expr * e, * def;
+
+    /** The expression attributes are being selected on. e.g. `foo` in `foo.bar.baz`. */
+    std::unique_ptr<Expr> e;
+
+    /** A default value specified with `or`, if the selected attr doesn't exist.
+     * e.g. `bix` in `foo.bar.baz or bix`
+     */
+    std::unique_ptr<Expr> def;
+
+    /** The path of attributes being selected. e.g. `bar.baz` in `foo.bar.baz.` */
     AttrPath attrPath;
-    ExprSelect(const PosIdx & pos, Expr * e, AttrPath attrPath, Expr * def) : pos(pos), e(e), def(def), attrPath(std::move(attrPath)) { };
-    ExprSelect(const PosIdx & pos, Expr * e, Symbol name) : pos(pos), e(e), def(0) { attrPath.push_back(AttrName(name)); };
+
+    ExprSelect(const PosIdx & pos, std::unique_ptr<Expr> e, AttrPath attrPath, std::unique_ptr<Expr> def) : pos(pos), e(std::move(e)), def(std::move(def)), attrPath(std::move(attrPath)) { };
+    ExprSelect(const PosIdx & pos, std::unique_ptr<Expr> e, Symbol name) : pos(pos), e(std::move(e)) { attrPath.push_back(AttrName(name)); };
     PosIdx getPos() const override { return pos; }
     COMMON_METHODS
 };
 
 struct ExprOpHasAttr : Expr
 {
-    Expr * e;
+    std::unique_ptr<Expr> e;
     AttrPath attrPath;
-    ExprOpHasAttr(Expr * e, AttrPath attrPath) : e(e), attrPath(std::move(attrPath)) { };
+    ExprOpHasAttr(std::unique_ptr<Expr> e, AttrPath attrPath) : e(std::move(e)), attrPath(std::move(attrPath)) { };
     PosIdx getPos() const override { return e->getPos(); }
     COMMON_METHODS
 };
@@ -180,11 +207,11 @@ struct ExprAttrs : Expr
         };
 
         Kind kind;
-        Expr * e;
+        std::unique_ptr<Expr> e;
         PosIdx pos;
         Displacement displ; // displacement
-        AttrDef(Expr * e, const PosIdx & pos, Kind kind = Kind::Plain)
-            : kind(kind), e(e), pos(pos) { };
+        AttrDef(std::unique_ptr<Expr> e, const PosIdx & pos, Kind kind = Kind::Plain)
+            : kind(kind), e(std::move(e)), pos(pos) { };
         AttrDef() { };
 
         template<typename T>
@@ -203,12 +230,12 @@ struct ExprAttrs : Expr
     };
     typedef std::map<Symbol, AttrDef> AttrDefs;
     AttrDefs attrs;
-    std::unique_ptr<std::vector<Expr *>> inheritFromExprs;
+    std::unique_ptr<std::vector<ref<Expr>>> inheritFromExprs;
     struct DynamicAttrDef {
-        Expr * nameExpr, * valueExpr;
+        std::unique_ptr<Expr> nameExpr, valueExpr;
         PosIdx pos;
-        DynamicAttrDef(Expr * nameExpr, Expr * valueExpr, const PosIdx & pos)
-            : nameExpr(nameExpr), valueExpr(valueExpr), pos(pos) { };
+        DynamicAttrDef(std::unique_ptr<Expr> nameExpr, std::unique_ptr<Expr> valueExpr, const PosIdx & pos)
+            : nameExpr(std::move(nameExpr)), valueExpr(std::move(valueExpr)), pos(pos) { };
     };
     typedef std::vector<DynamicAttrDef> DynamicAttrDefs;
     DynamicAttrDefs dynamicAttrs;
@@ -225,7 +252,7 @@ struct ExprAttrs : Expr
 
 struct ExprList : Expr
 {
-    std::vector<Expr *> elems;
+    std::vector<std::unique_ptr<Expr>> elems;
     ExprList() { };
     COMMON_METHODS
     Value * maybeThunk(EvalState & state, Env & env) override;
@@ -240,7 +267,7 @@ struct Formal
 {
     PosIdx pos;
     Symbol name;
-    Expr * def;
+    std::unique_ptr<Expr> def;
 };
 
 /** Attribute set destructuring in arguments of a lambda, if present */
@@ -257,9 +284,9 @@ struct Formals
         return it != formals.end() && it->name == arg;
     }
 
-    std::vector<Formal> lexicographicOrder(const SymbolTable & symbols) const
+    std::vector<std::reference_wrapper<const Formal>> lexicographicOrder(const SymbolTable & symbols) const
     {
-        std::vector<Formal> result(formals.begin(), formals.end());
+        std::vector<std::reference_wrapper<const Formal>> result(formals.begin(), formals.end());
         std::sort(result.begin(), result.end(),
             [&] (const Formal & a, const Formal & b) {
                 std::string_view sa = symbols[a.name], sb = symbols[b.name];
@@ -283,30 +310,55 @@ struct ExprLambda : Expr
     Symbol arg;
     /** Formals are present when the lambda destructures an attr set as
      * argument, with or without ellipsis */
-    Formals * formals;
-    Expr * body;
-    ExprLambda(PosIdx pos, Symbol arg, Formals * formals, Expr * body)
-        : pos(pos), arg(arg), formals(formals), body(body)
+    std::unique_ptr<Formals> formals;
+    std::unique_ptr<Expr> body;
+    ExprLambda(PosIdx pos, Symbol arg, std::unique_ptr<Formals> formals, std::unique_ptr<Expr> body)
+        : pos(pos), arg(arg), formals(std::move(formals)), body(std::move(body))
     {
     };
-    ExprLambda(PosIdx pos, Formals * formals, Expr * body)
-        : pos(pos), formals(formals), body(body)
+    ExprLambda(PosIdx pos, std::unique_ptr<Formals> formals, std::unique_ptr<Expr> body)
+        : pos(pos), formals(std::move(formals)), body(std::move(body))
     {
     }
     void setName(Symbol name) override;
     std::string showNamePos(const EvalState & state) const;
     inline bool hasFormals() const { return formals != nullptr; }
     PosIdx getPos() const override { return pos; }
+
+    /** Returns the name of the lambda,
+     * or "anonymous lambda" if it doesn't have one.
+     */
+    inline std::string getName(SymbolTable const & symbols) const
+    {
+        if (this->name) {
+            return symbols[this->name];
+        }
+
+        return "anonymous lambda";
+    }
+
+    /** Returns the name of the lambda in single quotes,
+     * or "anonymous lambda" if it doesn't have one.
+     */
+    inline std::string getQuotedName(SymbolTable const & symbols) const
+    {
+        if (this->name) {
+            return concatStrings("'", symbols[this->name], "'");
+        }
+
+        return "anonymous lambda";
+    }
+
     COMMON_METHODS
 };
 
 struct ExprCall : Expr
 {
-    Expr * fun;
-    std::vector<Expr *> args;
+    std::unique_ptr<Expr> fun;
+    std::vector<std::unique_ptr<Expr>> args;
     PosIdx pos;
-    ExprCall(const PosIdx & pos, Expr * fun, std::vector<Expr *> && args)
-        : fun(fun), args(args), pos(pos)
+    ExprCall(const PosIdx & pos, std::unique_ptr<Expr> fun, std::vector<std::unique_ptr<Expr>> && args)
+        : fun(std::move(fun)), args(std::move(args)), pos(pos)
     { }
     PosIdx getPos() const override { return pos; }
     COMMON_METHODS
@@ -314,19 +366,19 @@ struct ExprCall : Expr
 
 struct ExprLet : Expr
 {
-    ExprAttrs * attrs;
-    Expr * body;
-    ExprLet(ExprAttrs * attrs, Expr * body) : attrs(attrs), body(body) { };
+    std::unique_ptr<ExprAttrs> attrs;
+    std::unique_ptr<Expr> body;
+    ExprLet(std::unique_ptr<ExprAttrs> attrs, std::unique_ptr<Expr> body) : attrs(std::move(attrs)), body(std::move(body)) { };
     COMMON_METHODS
 };
 
 struct ExprWith : Expr
 {
     PosIdx pos;
-    Expr * attrs, * body;
+    std::unique_ptr<Expr> attrs, body;
     size_t prevWith;
     ExprWith * parentWith;
-    ExprWith(const PosIdx & pos, Expr * attrs, Expr * body) : pos(pos), attrs(attrs), body(body) { };
+    ExprWith(const PosIdx & pos, std::unique_ptr<Expr> attrs, std::unique_ptr<Expr> body) : pos(pos), attrs(std::move(attrs)), body(std::move(body)) { };
     PosIdx getPos() const override { return pos; }
     COMMON_METHODS
 };
@@ -334,8 +386,8 @@ struct ExprWith : Expr
 struct ExprIf : Expr
 {
     PosIdx pos;
-    Expr * cond, * then, * else_;
-    ExprIf(const PosIdx & pos, Expr * cond, Expr * then, Expr * else_) : pos(pos), cond(cond), then(then), else_(else_) { };
+    std::unique_ptr<Expr> cond, then, else_;
+    ExprIf(const PosIdx & pos, std::unique_ptr<Expr> cond, std::unique_ptr<Expr> then, std::unique_ptr<Expr> else_) : pos(pos), cond(std::move(cond)), then(std::move(then)), else_(std::move(else_)) { };
     PosIdx getPos() const override { return pos; }
     COMMON_METHODS
 };
@@ -343,16 +395,16 @@ struct ExprIf : Expr
 struct ExprAssert : Expr
 {
     PosIdx pos;
-    Expr * cond, * body;
-    ExprAssert(const PosIdx & pos, Expr * cond, Expr * body) : pos(pos), cond(cond), body(body) { };
+    std::unique_ptr<Expr> cond, body;
+    ExprAssert(const PosIdx & pos, std::unique_ptr<Expr> cond, std::unique_ptr<Expr> body) : pos(pos), cond(std::move(cond)), body(std::move(body)) { };
     PosIdx getPos() const override { return pos; }
     COMMON_METHODS
 };
 
 struct ExprOpNot : Expr
 {
-    Expr * e;
-    ExprOpNot(Expr * e) : e(e) { };
+    std::unique_ptr<Expr> e;
+    ExprOpNot(std::unique_ptr<Expr> e) : e(std::move(e)) { };
     PosIdx getPos() const override { return e->getPos(); }
     COMMON_METHODS
 };
@@ -361,9 +413,9 @@ struct ExprOpNot : Expr
     struct name : Expr \
     { \
         PosIdx pos; \
-        Expr * e1, * e2; \
-        name(Expr * e1, Expr * e2) : e1(e1), e2(e2) { }; \
-        name(const PosIdx & pos, Expr * e1, Expr * e2) : pos(pos), e1(e1), e2(e2) { }; \
+        std::unique_ptr<Expr> e1, e2; \
+        name(std::unique_ptr<Expr> e1, std::unique_ptr<Expr> e2) : e1(std::move(e1)), e2(std::move(e2)) { }; \
+        name(const PosIdx & pos, std::unique_ptr<Expr> e1, std::unique_ptr<Expr> e2) : pos(pos), e1(std::move(e1)), e2(std::move(e2)) { }; \
         void show(const SymbolTable & symbols, std::ostream & str) const override \
         { \
             str << "("; e1->show(symbols, str); str << " " s " "; e2->show(symbols, str); str << ")"; \
@@ -388,9 +440,9 @@ struct ExprConcatStrings : Expr
 {
     PosIdx pos;
     bool forceString;
-    std::vector<std::pair<PosIdx, Expr *>> * es;
-    ExprConcatStrings(const PosIdx & pos, bool forceString, std::vector<std::pair<PosIdx, Expr *>> * es)
-        : pos(pos), forceString(forceString), es(es) { };
+    std::vector<std::pair<PosIdx, std::unique_ptr<Expr>>> es;
+    ExprConcatStrings(const PosIdx & pos, bool forceString, std::vector<std::pair<PosIdx, std::unique_ptr<Expr>>> es)
+        : pos(pos), forceString(forceString), es(std::move(es)) { };
     PosIdx getPos() const override { return pos; }
     COMMON_METHODS
 };

src/libexpr/parser.y

@@ -1,452 +0,0 @@
-%glr-parser
-%define api.pure
-%locations
-%define parse.error verbose
-%defines
-/* %no-lines */
-%parse-param { void * scanner }
-%parse-param { nix::ParserState * state }
-%lex-param { void * scanner }
-%lex-param { nix::ParserState * state }
-%expect 1
-%expect-rr 1
-
-%code requires {
-
-#ifndef BISON_HEADER
-#define BISON_HEADER
-
-#include <variant>
-
-#include "finally.hh"
-#include "users.hh"
-
-#include "nixexpr.hh"
-#include "eval.hh"
-#include "eval-settings.hh"
-#include "globals.hh"
-#include "parser-state.hh"
-
-#define YYLTYPE ::nix::ParserLocation
-#define YY_DECL int yylex \
-    (YYSTYPE * yylval_param, YYLTYPE * yylloc_param, yyscan_t yyscanner, nix::ParserState * state)
-
-namespace nix {
-
-Expr * parseExprFromBuf(
-    char * text,
-    size_t length,
-    Pos::Origin origin,
-    const SourcePath & basePath,
-    SymbolTable & symbols,
-    PosTable & positions,
-    const Expr::AstSymbols & astSymbols);
-
-}
-
-#endif
-
-}
-
-%{
-
-#include "parser-tab.hh"
-#include "lexer-tab.hh"
-
-YY_DECL;
-
-using namespace nix;
-
-#define CUR_POS state->at(*yylocp)
-
-
-void yyerror(YYLTYPE * loc, yyscan_t scanner, ParserState * state, const char * error)
-{
-    if (std::string_view(error).starts_with("syntax error, unexpected end of file")) {
-        loc->first_column = loc->last_column;
-        loc->first_line = loc->last_line;
-    }
-    throw ParseError({
-        .msg = HintFmt(error),
-        .pos = state->positions[state->at(*loc)]
-    });
-}
-
-
-%}
-
-%union {
-  // !!! We're probably leaking stuff here.
-  nix::Expr * e;
-  nix::ExprList * list;
-  nix::ExprAttrs * attrs;
-  nix::Formals * formals;
-  nix::Formal * formal;
-  nix::NixInt n;
-  nix::NixFloat nf;
-  nix::StringToken id; // !!! -> Symbol
-  nix::StringToken path;
-  nix::StringToken uri;
-  nix::StringToken str;
-  std::vector<nix::AttrName> * attrNames;
-  std::vector<std::pair<nix::AttrName, nix::PosIdx>> * inheritAttrs;
-  std::vector<std::pair<nix::PosIdx, nix::Expr *>> * string_parts;
-  std::vector<std::pair<nix::PosIdx, std::variant<nix::Expr *, nix::StringToken>>> * ind_string_parts;
-}
-
-%type <e> start expr expr_function expr_if expr_op
-%type <e> expr_select expr_simple expr_app
-%type <list> expr_list
-%type <attrs> binds
-%type <formals> formals
-%type <formal> formal
-%type <attrNames> attrpath
-%type <inheritAttrs> attrs
-%type <string_parts> string_parts_interpolated
-%type <ind_string_parts> ind_string_parts
-%type <e> path_start string_parts string_attr
-%type <id> attr
-%token <id> ID
-%token <str> STR IND_STR
-%token <n> INT
-%token <nf> FLOAT
-%token <path> PATH HPATH SPATH PATH_END
-%token <uri> URI
-%token IF THEN ELSE ASSERT WITH LET IN REC INHERIT EQ NEQ AND OR IMPL OR_KW
-%token DOLLAR_CURLY /* == ${ */
-%token IND_STRING_OPEN IND_STRING_CLOSE
-%token ELLIPSIS
-
-%right IMPL
-%left OR
-%left AND
-%nonassoc EQ NEQ
-%nonassoc '<' '>' LEQ GEQ
-%right UPDATE
-%left NOT
-%left '+' '-'
-%left '*' '/'
-%right CONCAT
-%nonassoc '?'
-%nonassoc NEGATE
-
-%%
-
-start: expr { state->result = $1; };
-
-expr: expr_function;
-
-expr_function
-  : ID ':' expr_function
-    { $$ = new ExprLambda(CUR_POS, state->symbols.create($1), 0, $3); }
-  | '{' formals '}' ':' expr_function
-    { $$ = new ExprLambda(CUR_POS, state->validateFormals($2), $5); }
-  | '{' formals '}' '@' ID ':' expr_function
-    {
-      auto arg = state->symbols.create($5);
-      $$ = new ExprLambda(CUR_POS, arg, state->validateFormals($2, CUR_POS, arg), $7);
-    }
-  | ID '@' '{' formals '}' ':' expr_function
-    {
-      auto arg = state->symbols.create($1);
-      $$ = new ExprLambda(CUR_POS, arg, state->validateFormals($4, CUR_POS, arg), $7);
-    }
-  | ASSERT expr ';' expr_function
-    { $$ = new ExprAssert(CUR_POS, $2, $4); }
-  | WITH expr ';' expr_function
-    { $$ = new ExprWith(CUR_POS, $2, $4); }
-  | LET binds IN expr_function
-    { if (!$2->dynamicAttrs.empty())
-        throw ParseError({
-            .msg = HintFmt("dynamic attributes not allowed in let"),
-            .pos = state->positions[CUR_POS]
-        });
-      $$ = new ExprLet($2, $4);
-    }
-  | expr_if
-  ;
-
-expr_if
-  : IF expr THEN expr ELSE expr { $$ = new ExprIf(CUR_POS, $2, $4, $6); }
-  | expr_op
-  ;
-
-expr_op
-  : '!' expr_op %prec NOT { $$ = new ExprOpNot($2); }
-  | '-' expr_op %prec NEGATE { $$ = new ExprCall(CUR_POS, new ExprVar(state->s.sub), {new ExprInt(0), $2}); }
-  | expr_op EQ expr_op { $$ = new ExprOpEq($1, $3); }
-  | expr_op NEQ expr_op { $$ = new ExprOpNEq($1, $3); }
-  | expr_op '<' expr_op { $$ = new ExprCall(state->at(@2), new ExprVar(state->s.lessThan), {$1, $3}); }
-  | expr_op LEQ expr_op { $$ = new ExprOpNot(new ExprCall(state->at(@2), new ExprVar(state->s.lessThan), {$3, $1})); }
-  | expr_op '>' expr_op { $$ = new ExprCall(state->at(@2), new ExprVar(state->s.lessThan), {$3, $1}); }
-  | expr_op GEQ expr_op { $$ = new ExprOpNot(new ExprCall(state->at(@2), new ExprVar(state->s.lessThan), {$1, $3})); }
-  | expr_op AND expr_op { $$ = new ExprOpAnd(state->at(@2), $1, $3); }
-  | expr_op OR expr_op { $$ = new ExprOpOr(state->at(@2), $1, $3); }
-  | expr_op IMPL expr_op { $$ = new ExprOpImpl(state->at(@2), $1, $3); }
-  | expr_op UPDATE expr_op { $$ = new ExprOpUpdate(state->at(@2), $1, $3); }
-  | expr_op '?' attrpath { $$ = new ExprOpHasAttr($1, std::move(*$3)); delete $3; }
-  | expr_op '+' expr_op
-    { $$ = new ExprConcatStrings(state->at(@2), false, new std::vector<std::pair<PosIdx, Expr *> >({{state->at(@1), $1}, {state->at(@3), $3}})); }
-  | expr_op '-' expr_op { $$ = new ExprCall(state->at(@2), new ExprVar(state->s.sub), {$1, $3}); }
-  | expr_op '*' expr_op { $$ = new ExprCall(state->at(@2), new ExprVar(state->s.mul), {$1, $3}); }
-  | expr_op '/' expr_op { $$ = new ExprCall(state->at(@2), new ExprVar(state->s.div), {$1, $3}); }
-  | expr_op CONCAT expr_op { $$ = new ExprOpConcatLists(state->at(@2), $1, $3); }
-  | expr_app
-  ;
-
-expr_app
-  : expr_app expr_select {
-      if (auto e2 = dynamic_cast<ExprCall *>($1)) {
-          e2->args.push_back($2);
-          $$ = $1;
-      } else
-          $$ = new ExprCall(CUR_POS, $1, {$2});
-  }
-  | expr_select
-  ;
-
-expr_select
-  : expr_simple '.' attrpath
-    { $$ = new ExprSelect(CUR_POS, $1, std::move(*$3), nullptr); delete $3; }
-  | expr_simple '.' attrpath OR_KW expr_select
-    { $$ = new ExprSelect(CUR_POS, $1, std::move(*$3), $5); delete $3; }
-  | /* Backwards compatibility: because Nixpkgs has a rarely used
-       function named ‘or’, allow stuff like ‘map or [...]’. */
-    expr_simple OR_KW
-    { $$ = new ExprCall(CUR_POS, $1, {new ExprVar(CUR_POS, state->s.or_)}); }
-  | expr_simple
-  ;
-
-expr_simple
-  : ID {
-      std::string_view s = "__curPos";
-      if ($1.l == s.size() && strncmp($1.p, s.data(), s.size()) == 0)
-          $$ = new ExprPos(CUR_POS);
-      else
-          $$ = new ExprVar(CUR_POS, state->symbols.create($1));
-  }
-  | INT { $$ = new ExprInt($1); }
-  | FLOAT { $$ = new ExprFloat($1); }
-  | '"' string_parts '"' { $$ = $2; }
-  | IND_STRING_OPEN ind_string_parts IND_STRING_CLOSE {
-      $$ = state->stripIndentation(CUR_POS, std::move(*$2));
-      delete $2;
-  }
-  | path_start PATH_END
-  | path_start string_parts_interpolated PATH_END {
-      $2->insert($2->begin(), {state->at(@1), $1});
-      $$ = new ExprConcatStrings(CUR_POS, false, $2);
-  }
-  | SPATH {
-      std::string path($1.p + 1, $1.l - 2);
-      $$ = new ExprCall(CUR_POS,
-          new ExprVar(state->s.findFile),
-          {new ExprVar(state->s.nixPath),
-           new ExprString(std::move(path))});
-  }
-  | URI {
-      static bool noURLLiterals = experimentalFeatureSettings.isEnabled(Xp::NoUrlLiterals);
-      if (noURLLiterals)
-          throw ParseError({
-              .msg = HintFmt("URL literals are disabled"),
-              .pos = state->positions[CUR_POS]
-          });
-      $$ = new ExprString(std::string($1));
-  }
-  | '(' expr ')' { $$ = $2; }
-  /* Let expressions `let {..., body = ...}' are just desugared
-     into `(rec {..., body = ...}).body'. */
-  | LET '{' binds '}'
-    { $3->recursive = true; $$ = new ExprSelect(noPos, $3, state->s.body); }
-  | REC '{' binds '}'
-    { $3->recursive = true; $$ = $3; }
-  | '{' binds '}'
-    { $$ = $2; }
-  | '[' expr_list ']' { $$ = $2; }
-  ;
-
-string_parts
-  : STR { $$ = new ExprString(std::string($1)); }
-  | string_parts_interpolated { $$ = new ExprConcatStrings(CUR_POS, true, $1); }
-  | { $$ = new ExprString(""); }
-  ;
-
-string_parts_interpolated
-  : string_parts_interpolated STR
-  { $$ = $1; $1->emplace_back(state->at(@2), new ExprString(std::string($2))); }
-  | string_parts_interpolated DOLLAR_CURLY expr '}' { $$ = $1; $1->emplace_back(state->at(@2), $3); }
-  | DOLLAR_CURLY expr '}' { $$ = new std::vector<std::pair<PosIdx, Expr *>>; $$->emplace_back(state->at(@1), $2); }
-  | STR DOLLAR_CURLY expr '}' {
-      $$ = new std::vector<std::pair<PosIdx, Expr *>>;
-      $$->emplace_back(state->at(@1), new ExprString(std::string($1)));
-      $$->emplace_back(state->at(@2), $3);
-    }
-  ;
-
-path_start
-  : PATH {
-    Path path(absPath({$1.p, $1.l}, state->basePath.path.abs()));
-    /* add back in the trailing '/' to the first segment */
-    if ($1.p[$1.l-1] == '/' && $1.l > 1)
-      path += "/";
-    $$ = new ExprPath(path);
-  }
-  | HPATH {
-    if (evalSettings.pureEval) {
-        throw Error(
-            "the path '%s' can not be resolved in pure mode",
-            std::string_view($1.p, $1.l)
-        );
-    }
-    Path path(getHome() + std::string($1.p + 1, $1.l - 1));
-    $$ = new ExprPath(path);
-  }
-  ;
-
-ind_string_parts
-  : ind_string_parts IND_STR { $$ = $1; $1->emplace_back(state->at(@2), $2); }
-  | ind_string_parts DOLLAR_CURLY expr '}' { $$ = $1; $1->emplace_back(state->at(@2), $3); }
-  | { $$ = new std::vector<std::pair<PosIdx, std::variant<Expr *, StringToken>>>; }
-  ;
-
-binds
-  : binds attrpath '=' expr ';' { $$ = $1; state->addAttr($$, std::move(*$2), $4, state->at(@2)); delete $2; }
-  | binds INHERIT attrs ';'
-    { $$ = $1;
-      for (auto & [i, iPos] : *$3) {
-          if ($$->attrs.find(i.symbol) != $$->attrs.end())
-              state->dupAttr(i.symbol, iPos, $$->attrs[i.symbol].pos);
-          $$->attrs.emplace(
-              i.symbol,
-              ExprAttrs::AttrDef(new ExprVar(iPos, i.symbol), iPos, ExprAttrs::AttrDef::Kind::Inherited));
-      }
-      delete $3;
-    }
-  | binds INHERIT '(' expr ')' attrs ';'
-    { $$ = $1;
-      if (!$$->inheritFromExprs)
-          $$->inheritFromExprs = std::make_unique<std::vector<Expr *>>();
-      $$->inheritFromExprs->push_back($4);
-      auto from = new nix::ExprInheritFrom(state->at(@4), $$->inheritFromExprs->size() - 1);
-      for (auto & [i, iPos] : *$6) {
-          if ($$->attrs.find(i.symbol) != $$->attrs.end())
-              state->dupAttr(i.symbol, iPos, $$->attrs[i.symbol].pos);
-          $$->attrs.emplace(
-              i.symbol,
-              ExprAttrs::AttrDef(
-                  new ExprSelect(iPos, from, i.symbol),
-                  iPos,
-                  ExprAttrs::AttrDef::Kind::InheritedFrom));
-      }
-      delete $6;
-    }
-  | { $$ = new ExprAttrs(state->at(@0)); }
-  ;
-
-attrs
-  : attrs attr { $$ = $1; $1->emplace_back(AttrName(state->symbols.create($2)), state->at(@2)); }
-  | attrs string_attr
-    { $$ = $1;
-      ExprString * str = dynamic_cast<ExprString *>($2);
-      if (str) {
-          $$->emplace_back(AttrName(state->symbols.create(str->s)), state->at(@2));
-          delete str;
-      } else
-          throw ParseError({
-              .msg = HintFmt("dynamic attributes not allowed in inherit"),
-              .pos = state->positions[state->at(@2)]
-          });
-    }
-  | { $$ = new std::vector<std::pair<AttrName, PosIdx>>; }
-  ;
-
-attrpath
-  : attrpath '.' attr { $$ = $1; $1->push_back(AttrName(state->symbols.create($3))); }
-  | attrpath '.' string_attr
-    { $$ = $1;
-      ExprString * str = dynamic_cast<ExprString *>($3);
-      if (str) {
-          $$->push_back(AttrName(state->symbols.create(str->s)));
-          delete str;
-      } else
-          $$->push_back(AttrName($3));
-    }
-  | attr { $$ = new std::vector<AttrName>; $$->push_back(AttrName(state->symbols.create($1))); }
-  | string_attr
-    { $$ = new std::vector<AttrName>;
-      ExprString *str = dynamic_cast<ExprString *>($1);
-      if (str) {
-          $$->push_back(AttrName(state->symbols.create(str->s)));
-          delete str;
-      } else
-          $$->push_back(AttrName($1));
-    }
-  ;
-
-attr
-  : ID
-  | OR_KW { $$ = {"or", 2}; }
-  ;
-
-string_attr
-  : '"' string_parts '"' { $$ = $2; }
-  | DOLLAR_CURLY expr '}' { $$ = $2; }
-  ;
-
-expr_list
-  : expr_list expr_select { $$ = $1; $1->elems.push_back($2); /* !!! dangerous */ }
-  | { $$ = new ExprList; }
-  ;
-
-formals
-  : formal ',' formals
-    { $$ = $3; $$->formals.emplace_back(*$1); delete $1; }
-  | formal
-    { $$ = new Formals; $$->formals.emplace_back(*$1); $$->ellipsis = false; delete $1; }
-  |
-    { $$ = new Formals; $$->ellipsis = false; }
-  | ELLIPSIS
-    { $$ = new Formals; $$->ellipsis = true; }
-  ;
-
-formal
-  : ID { $$ = new Formal{CUR_POS, state->symbols.create($1), 0}; }
-  | ID '?' expr { $$ = new Formal{CUR_POS, state->symbols.create($1), $3}; }
-  ;
-
-%%
-
-#include "eval.hh"
-
-
-namespace nix {
-
-Expr * parseExprFromBuf(
-    char * text,
-    size_t length,
-    Pos::Origin origin,
-    const SourcePath & basePath,
-    SymbolTable & symbols,
-    PosTable & positions,
-    const Expr::AstSymbols & astSymbols)
-{
-    yyscan_t scanner;
-    ParserState state {
-        .symbols = symbols,
-        .positions = positions,
-        .basePath = basePath,
-        .origin = positions.addOrigin(origin, length),
-        .s = astSymbols,
-    };
-
-    yylex_init(&scanner);
-    Finally _destroy([&] { yylex_destroy(scanner); });
-
-    yy_scan_buffer(text, length, scanner);
-    yyparse(scanner, &state);
-
-    return state.result;
-}
-
-
-}