Import C bindings from NixOS/nix#8699

Co-authored-by: Yorick van Pelt <yorick@yorickvanpelt.nl>
Co-authored-by: José Luis Lafuente <jose.lafuente@tweag.io>
Change-Id: I48aa262d880aa75aa5b230553f663c35c283d5f6

.github/workflows/labels.yml

@@ -18,7 +18,7 @@ jobs:
     runs-on: ubuntu-latest
     if: github.repository_owner == 'NixOS'
     steps:
-    - uses: actions/labeler@v5
+    - uses: actions/labeler@v4
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
         sync-labels: false

.gitignore

@@ -21,12 +21,14 @@ perl/Makefile.config
 /doc/manual/language.json
 /doc/manual/xp-features.json
 /doc/manual/src/SUMMARY.md
+/doc/manual/src/SUMMARY-rl-next.md
 /doc/manual/src/command-ref/new-cli
 /doc/manual/src/command-ref/conf-file.md
 /doc/manual/src/command-ref/experimental-features-shortlist.md
 /doc/manual/src/contributing/experimental-feature-descriptions.md
 /doc/manual/src/language/builtins.md
 /doc/manual/src/language/builtin-constants.md
+/doc/manual/src/release-notes/rl-next.md
 
 # /scripts/
 /scripts/nix-profile.sh
@@ -41,13 +43,16 @@ perl/Makefile.config
 /src/libexpr/parser-tab.hh
 /src/libexpr/parser-tab.output
 /src/libexpr/nix.tbl
+/src/libexpr/tests
 /tests/unit/libexpr/libnixexpr-tests
 
 # /src/libstore/
 *.gen.*
+/src/libstore/tests
 /tests/unit/libstore/libnixstore-tests
 
 # /src/libutil/
+/src/libutil/tests
 /tests/unit/libutil/libnixutil-tests
 
 /src/nix/nix
@@ -90,6 +95,7 @@ perl/Makefile.config
 /tests/functional/ca/config.nix
 /tests/functional/dyn-drv/config.nix
 /tests/functional/repl-result-out
+/tests/functional/debugger-test-out
 /tests/functional/test-libstoreconsumer/test-libstoreconsumer
 
 # /tests/functional/lang/
@@ -137,8 +143,12 @@ compile_commands.json
 nix-rust/target
 
 result
+result-*
 
 .vscode/
 
 # clangd and possibly more
 .cache/
+
+# Mac OS
+.DS_Store

.version

@@ -1 +1 @@
-2.18.1
+2.90.0

Makefile

@@ -1,3 +1,9 @@
+include mk/build-dir.mk
+
+-include $(buildprefix)Makefile.config
+clean-files += $(buildprefix)Makefile.config
+
+ifeq ($(ENABLE_BUILD), yes)
 makefiles = \
   mk/precompiled-headers.mk \
   local.mk \
@@ -15,17 +21,19 @@ makefiles = \
   misc/zsh/local.mk \
   misc/systemd/local.mk \
   misc/launchd/local.mk \
-  misc/upstart/local.mk \
-  doc/manual/local.mk \
-  doc/internal-api/local.mk
+  misc/upstart/local.mk
+endif
 
--include Makefile.config
-
-ifeq ($(tests), yes)
+ifeq ($(ENABLE_BUILD)_$(ENABLE_TESTS), yes_yes)
+UNIT_TEST_ENV = _NIX_TEST_UNIT_DATA=unit-test-data
 makefiles += \
   tests/unit/libutil/local.mk \
   tests/unit/libutil-support/local.mk \
-  tests/unit/libstore/local.mk \
+  tests/unit/libstore/local.mk
+endif
+
+ifeq ($(ENABLE_TESTS), yes)
+makefiles += \
   tests/unit/libstore-support/local.mk \
   tests/unit/libexpr/local.mk \
   tests/unit/libexpr-support/local.mk \
@@ -39,6 +47,16 @@ makefiles += \
   mk/disable-tests.mk
 endif
 
+# Some makefiles require access to built programs and must be included late.
+makefiles-late =
+
+ifeq ($(ENABLE_BUILD), yes)
+makefiles-late += doc/manual/local.mk
+makefiles-late += doc/internal-api/local.mk
+endif
+
+# Miscellaneous global Flags
+
 OPTIMIZE = 1
 
 ifeq ($(OPTIMIZE), 1)
@@ -50,4 +68,4 @@ endif
 
 include mk/lib.mk
 
-GLOBAL_CXXFLAGS += -g -Wall -include config.h -std=c++2a -I src
+GLOBAL_CXXFLAGS += -g -Wall -Wimplicit-fallthrough -include $(buildprefix)config.h -std=c++2a -I src

Makefile.config.in

@@ -46,5 +46,6 @@ sandbox_shell = @sandbox_shell@
 storedir = @storedir@
 sysconfdir = @sysconfdir@
 system = @system@
-tests = @tests@
+ENABLE_BUILD = @ENABLE_BUILD@
+ENABLE_TESTS = @ENABLE_TESTS@
 internal_api_docs = @internal_api_docs@

boehmgc-coroutine-sp-fallback.diff

@@ -59,12 +59,18 @@ index b5d71e62..aed7b0bf 100644
      GC_bool found_me = FALSE;
      size_t nthreads = 0;
      int i;
-@@ -851,6 +853,31 @@ GC_INNER void GC_push_all_stacks(void)
+@@ -851,6 +853,37 @@ GC_INNER void GC_push_all_stacks(void)
            hi = p->altstack + p->altstack_size;
            /* FIXME: Need to scan the normal stack too, but how ? */
            /* FIXME: Assume stack grows down */
 +        } else {
-+          if (pthread_getattr_np(p->id, &pattr)) {
++#ifdef HAVE_PTHREAD_ATTR_GET_NP
++          if (!pthread_attr_init(&pattr)
++              || !pthread_attr_get_np(p->id, &pattr))
++#else /* HAVE_PTHREAD_GETATTR_NP */
++          if (pthread_getattr_np(p->id, &pattr))
++#endif
++          {
 +            ABORT("GC_push_all_stacks: pthread_getattr_np failed!");
 +          }
 +          if (pthread_attr_getstacksize(&pattr, &stack_limit)) {

boehmgc-traceable_allocator-public.diff

@@ -0,0 +1,12 @@
+diff --git a/include/gc_allocator.h b/include/gc_allocator.h
+index 597c7f13..587286be 100644
+--- a/include/gc_allocator.h
++++ b/include/gc_allocator.h
+@@ -312,6 +312,7 @@ public:
+ 
+ template<>
+ class traceable_allocator<void> {
++public:
+   typedef size_t      size_type;
+   typedef ptrdiff_t   difference_type;
+   typedef void*       pointer;

bootstrap.sh

@@ -1,4 +0,0 @@
-#! /bin/sh -e
-rm -f aclocal.m4
-mkdir -p config
-exec autoreconf -vfi

configure.ac

@@ -47,6 +47,10 @@ AC_DEFINE_UNQUOTED(SYSTEM, ["$system"], [platform identifier ('cpu-os')])
 # State should be stored in /nix/var, unless the user overrides it explicitly.
 test "$localstatedir" = '${prefix}/var' && localstatedir=/nix/var
 
+# Assign a default value to C{,XX}FLAGS as the default configure script sets them
+# to -O2 otherwise, which we don't want to have hardcoded
+CFLAGS=${CFLAGS-""}
+CXXFLAGS=${CXXFLAGS-""}
 
 AC_PROG_CC
 AC_PROG_CXX
@@ -68,6 +72,9 @@ case "$host_os" in
 esac
 
 
+ENSURE_NO_GCC_BUG_80431
+
+
 # Check for pubsetbuf.
 AC_MSG_CHECKING([for pubsetbuf])
 AC_LANG_PUSH(C++)
@@ -152,12 +159,17 @@ if test "x$GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC" = xyes; then
     LDFLAGS="-latomic $LDFLAGS"
 fi
 
+# Running the functional tests without building Nix is useful for testing
+# different pre-built versions of Nix against each other.
+AC_ARG_ENABLE(build, AS_HELP_STRING([--disable-build],[Do not build nix]),
+  ENABLE_BUILD=$enableval, ENABLE_BUILD=yes)
+AC_SUBST(ENABLE_BUILD)
 # Building without tests is useful for bootstrapping with a smaller footprint
 # or running the tests in a separate derivation. Otherwise, we do compile and
 # run them.
 AC_ARG_ENABLE(tests, AS_HELP_STRING([--disable-tests],[Do not build the tests]),
-  tests=$enableval, tests=yes)
-AC_SUBST(tests)
+  ENABLE_TESTS=$enableval, ENABLE_TESTS=yes)
+AC_SUBST(ENABLE_TESTS)
 
 # Building without API docs is the default as Nix' C++ interfaces are internal and unstable.
 AC_ARG_ENABLE(internal_api_docs, AS_HELP_STRING([--enable-internal-api-docs],[Build API docs for Nix's internal unstable C++ interfaces]),
@@ -289,10 +301,10 @@ if test "$gc" = yes; then
 fi
 
 
-if test "$tests" = yes; then
+if test "$ENABLE_TESTS" = yes; then
 
 # Look for gtest.
-PKG_CHECK_MODULES([GTEST], [gtest_main])
+PKG_CHECK_MODULES([GTEST], [gtest_main gmock_main])
 
 
 # Look for rapidcheck.

doc/internal-api/doxygen.cfg.in

@@ -58,6 +58,23 @@ INPUT                  = \
   src/nix-env \
   src/nix-store
 
+# If the MACRO_EXPANSION tag is set to YES, doxygen will expand all macro names
+# in the source code. If set to NO, only conditional compilation will be
+# performed. Macro expansion can be done in a controlled way by setting
+# EXPAND_ONLY_PREDEF to YES.
+# The default value is: NO.
+# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
+
+MACRO_EXPANSION        = YES
+
+# If the EXPAND_ONLY_PREDEF and MACRO_EXPANSION tags are both set to YES then
+# the macro expansion is limited to the macros specified with the PREDEFINED and
+# EXPAND_AS_DEFINED tags.
+# The default value is: NO.
+# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
+
+EXPAND_ONLY_PREDEF     = YES
+
 # The INCLUDE_PATH tag can be used to specify one or more directories that
 # contain include files that are not input files but should be processed by the
 # preprocessor. Note that the INCLUDE_PATH is not recursive, so the setting of
@@ -65,3 +82,16 @@ INPUT                  = \
 # This tag requires that the tag SEARCH_INCLUDES is set to YES.
 
 INCLUDE_PATH           = @RAPIDCHECK_HEADERS@
+
+# If the MACRO_EXPANSION and EXPAND_ONLY_PREDEF tags are set to YES then this
+# tag can be used to specify a list of macro names that should be expanded. The
+# macro definition that is found in the sources will be used. Use the PREDEFINED
+# tag if you want to use a different macro definition that overrules the
+# definition found in the source code.
+# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
+
+EXPAND_AS_DEFINED      = \
+  DECLARE_COMMON_SERIALISER \
+  DECLARE_WORKER_SERIALISER \
+  DECLARE_SERVE_SERIALISER \
+  LENGTH_PREFIXED_PROTO_HELPER

doc/manual/generate-manpage.nix

@@ -70,6 +70,7 @@ let
         * [`${command} ${name}`](./${appendName filename name}.md) - ${subcmd.description}
       '';
 
+      # TODO: move this confusing special case out of here when implementing #8496
       maybeStoreDocs = optionalString
         (details ? doc)
         (replaceStrings ["@stores@"] [storeDocs] details.doc);
@@ -78,6 +79,10 @@ let
         # Options
 
         ${showOptions details.flags toplevel.flags}
+
+        > **Note**
+        >
+        > See [`man nix.conf`](@docroot@/command-ref/conf-file.md#command-line-flags) for overriding configuration settings with command line flags.
       '';
 
       showOptions = options: commonOptions:
@@ -150,7 +155,7 @@ let
             To use this store, you need to make sure the corresponding experimental feature,
             [`${experimentalFeature}`](@docroot@/contributing/experimental-features.md#xp-feature-${experimentalFeature}),
             is enabled.
-            For example, include the following in [`nix.conf`](#):
+            For example, include the following in [`nix.conf`](@docroot@/command-ref/conf-file.md):
 
             ```
             extra-experimental-features = ${experimentalFeature}

doc/manual/local.mk

@@ -1,5 +1,10 @@
 ifeq ($(doc_generate),yes)
 
+# The version of Nix used to generate the doc. Can also be
+# `$(nix_INSTALL_PATH)` or just `nix` (to grap ambient from the `PATH`),
+# if one prefers.
+doc_nix = $(nix_PATH)
+
 MANUAL_SRCS := \
 	$(call rwildcard, $(d)/src, *.md) \
 	$(call rwildcard, $(d)/src, */*.md)
@@ -32,7 +37,7 @@ dummy-env = env -i \
 	NIX_STATE_DIR=/dummy \
 	NIX_CONFIG='cores = 0'
 
-nix-eval = $(dummy-env) $(bindir)/nix eval --experimental-features nix-command -I nix/corepkgs=corepkgs --store dummy:// --impure --raw
+nix-eval = $(dummy-env) $(doc_nix) eval --experimental-features nix-command -I nix/corepkgs=corepkgs --store dummy:// --impure --raw
 
 # re-implement mdBook's include directive to make it usable for terminal output and for proper @docroot@ substitution
 define process-includes
@@ -92,58 +97,76 @@ $(d)/nix-profiles.5: $(d)/src/command-ref/files/profiles.md
 	$(trace-gen) lowdown -sT man --nroff-nolinks -M section=5 $^.tmp -o $@
 	@rm $^.tmp
 
-$(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md
+$(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/SUMMARY-rl-next.md $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md
 	@cp $< $@
 	@$(call process-includes,$@,$@)
 
-$(d)/src/command-ref/new-cli: $(d)/nix.json $(d)/utils.nix $(d)/generate-manpage.nix $(bindir)/nix
+$(d)/src/command-ref/new-cli: $(d)/nix.json $(d)/utils.nix $(d)/generate-manpage.nix $(doc_nix)
 	@rm -rf $@ $@.tmp
 	$(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-manpage.nix true (builtins.readFile $<)'
 	@mv $@.tmp $@
 
-$(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/utils.nix $(d)/src/command-ref/conf-file-prefix.md $(d)/src/command-ref/experimental-features-shortlist.md $(bindir)/nix
+$(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/utils.nix $(d)/src/command-ref/conf-file-prefix.md $(d)/src/command-ref/experimental-features-shortlist.md $(doc_nix)
 	@cat doc/manual/src/command-ref/conf-file-prefix.md > $@.tmp
 	$(trace-gen) $(nix-eval) --expr '(import doc/manual/utils.nix).showSettings { inlineHTML = true; } (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp;
 	@mv $@.tmp $@
 
-$(d)/nix.json: $(bindir)/nix
-	$(trace-gen) $(dummy-env) $(bindir)/nix __dump-cli > $@.tmp
+$(d)/nix.json: $(doc_nix)
+	$(trace-gen) $(dummy-env) $(doc_nix) __dump-cli > $@.tmp
 	@mv $@.tmp $@
 
-$(d)/conf-file.json: $(bindir)/nix
-	$(trace-gen) $(dummy-env) $(bindir)/nix show-config --json --experimental-features nix-command > $@.tmp
+$(d)/conf-file.json: $(doc_nix)
+	$(trace-gen) $(dummy-env) $(doc_nix) show-config --json --experimental-features nix-command > $@.tmp
 	@mv $@.tmp $@
 
-$(d)/src/contributing/experimental-feature-descriptions.md: $(d)/xp-features.json $(d)/utils.nix $(d)/generate-xp-features.nix $(bindir)/nix
+$(d)/src/contributing/experimental-feature-descriptions.md: $(d)/xp-features.json $(d)/utils.nix $(d)/generate-xp-features.nix $(doc_nix)
 	@rm -rf $@ $@.tmp
 	$(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-xp-features.nix (builtins.fromJSON (builtins.readFile $<))'
 	@mv $@.tmp $@
 
-$(d)/src/command-ref/experimental-features-shortlist.md: $(d)/xp-features.json $(d)/utils.nix $(d)/generate-xp-features-shortlist.nix $(bindir)/nix
+$(d)/src/command-ref/experimental-features-shortlist.md: $(d)/xp-features.json $(d)/utils.nix $(d)/generate-xp-features-shortlist.nix $(doc_nix)
 	@rm -rf $@ $@.tmp
 	$(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-xp-features-shortlist.nix (builtins.fromJSON (builtins.readFile $<))'
 	@mv $@.tmp $@
 
-$(d)/xp-features.json: $(bindir)/nix
-	$(trace-gen) $(dummy-env) NIX_PATH=nix/corepkgs=corepkgs $(bindir)/nix __dump-xp-features > $@.tmp
+$(d)/xp-features.json: $(doc_nix)
+	$(trace-gen) $(dummy-env) NIX_PATH=nix/corepkgs=corepkgs $(doc_nix) __dump-xp-features > $@.tmp
 	@mv $@.tmp $@
 
-$(d)/src/language/builtins.md: $(d)/language.json $(d)/generate-builtins.nix $(d)/src/language/builtins-prefix.md $(bindir)/nix
+$(d)/src/language/builtins.md: $(d)/language.json $(d)/generate-builtins.nix $(d)/src/language/builtins-prefix.md $(doc_nix)
 	@cat doc/manual/src/language/builtins-prefix.md > $@.tmp
 	$(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-builtins.nix (builtins.fromJSON (builtins.readFile $<)).builtins' >> $@.tmp;
 	@cat doc/manual/src/language/builtins-suffix.md >> $@.tmp
 	@mv $@.tmp $@
 
-$(d)/src/language/builtin-constants.md: $(d)/language.json $(d)/generate-builtin-constants.nix $(d)/src/language/builtin-constants-prefix.md $(bindir)/nix
+$(d)/src/language/builtin-constants.md: $(d)/language.json $(d)/generate-builtin-constants.nix $(d)/src/language/builtin-constants-prefix.md $(doc_nix)
 	@cat doc/manual/src/language/builtin-constants-prefix.md > $@.tmp
 	$(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-builtin-constants.nix (builtins.fromJSON (builtins.readFile $<)).constants' >> $@.tmp;
 	@cat doc/manual/src/language/builtin-constants-suffix.md >> $@.tmp
 	@mv $@.tmp $@
 
-$(d)/language.json: $(bindir)/nix
-	$(trace-gen) $(dummy-env) NIX_PATH=nix/corepkgs=corepkgs $(bindir)/nix __dump-language > $@.tmp
+$(d)/language.json: $(doc_nix)
+	$(trace-gen) $(dummy-env) NIX_PATH=nix/corepkgs=corepkgs $(doc_nix) __dump-language > $@.tmp
 	@mv $@.tmp $@
 
+# Generate "Upcoming release" notes (or clear it and remove from menu)
+$(d)/src/release-notes/rl-next.md: $(d)/rl-next $(d)/rl-next/*
+	@if type -p changelog-d > /dev/null; then \
+		echo "  GEN   " $@; \
+		changelog-d doc/manual/rl-next > $@; \
+	else \
+		echo "  NULL  " $@; \
+		true > $@; \
+	fi
+
+$(d)/src/SUMMARY-rl-next.md: $(d)/src/release-notes/rl-next.md
+	$(trace-gen) true
+	@if [ -s $< ]; then \
+		echo '  - [Upcoming release](release-notes/rl-next.md)' > $@; \
+	else \
+	  true > $@; \
+	fi
+
 # Generate the HTML manual.
 .PHONY: manual-html
 manual-html: $(docdir)/manual/index.html
@@ -173,7 +196,7 @@ doc/manual/generated/man1/nix3-manpages: $(d)/src/command-ref/new-cli
 	done
 	@touch $@
 
-$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md $(d)/src/language/builtin-constants.md
+$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md $(d)/src/language/builtin-constants.md $(d)/src/release-notes/rl-next.md
 	$(trace-gen) \
 		tmp="$$(mktemp -d)"; \
 		cp -r doc/manual "$$tmp"; \

doc/manual/rl-next/better-errors-in-nix-repl.md

@@ -0,0 +1,40 @@
+---
+synopsis: Concise error printing in `nix repl`
+prs: 9928
+---
+
+Previously, if an element of a list or attribute set threw an error while
+evaluating, `nix repl` would print the entire error (including source location
+information) inline. This output was clumsy and difficult to parse:
+
+```
+nix-repl> { err = builtins.throw "uh oh!"; }
+{ err = «error:
+       … while calling the 'throw' builtin
+         at «string»:1:9:
+            1| { err = builtins.throw "uh oh!"; }
+             |         ^
+
+       error: uh oh!»; }
+```
+
+Now, only the error message is displayed, making the output much more readable.
+```
+nix-repl> { err = builtins.throw "uh oh!"; }
+{ err = «error: uh oh!»; }
+```
+
+However, if the whole expression being evaluated throws an error, source
+locations and (if applicable) a stack trace are printed, just like you'd expect:
+
+```
+nix-repl> builtins.throw "uh oh!"
+error:
+       … while calling the 'throw' builtin
+         at «string»:1:1:
+            1| builtins.throw "uh oh!"
+             | ^
+
+       error: uh oh!
+```
+

doc/manual/rl-next/config

@@ -0,0 +1,2 @@
+organization: NixOS
+repository: nix

doc/manual/rl-next/debugger-locals-for-let-expressions.md

@@ -0,0 +1,9 @@
+---
+synopsis: "`--debugger` can now access bindings from `let` expressions"
+prs: 9918
+issues: 8827.
+---
+
+Breakpoints and errors in the bindings of a `let` expression can now access
+those bindings in the debugger. Previously, only the body of `let` expressions
+could access those bindings.

doc/manual/rl-next/drv-string-parse-hang.md

@@ -0,0 +1,6 @@
+---
+synopsis: Fix handling of truncated `.drv` files.
+prs: 9673
+---
+
+Previously a `.drv` that was truncated in the middle of a string would case nix to enter an infinite loop, eventually exhausting all memory and crashing.

doc/manual/rl-next/empty-search-regex.md

@@ -0,0 +1,8 @@
+synopsis: Disallow empty search regex in `nix search`
+prs: #9481
+description: {
+
+[`nix search`](@docroot@/command-ref/new-cli/nix3-search.md) now requires a search regex to be passed. To show all packages, use `^`.
+
+}
+

doc/manual/rl-next/enter-debugger-more-reliably-in-let-and-calls.md

@@ -0,0 +1,25 @@
+---
+synopsis: The `--debugger` will start more reliably in `let` expressions and function calls
+prs: 9917
+issues: 6649
+---
+
+Previously, if you attempted to evaluate this file with the debugger:
+
+```nix
+let
+  a = builtins.trace "before inner break" (
+    builtins.break "hello"
+  );
+  b = builtins.trace "before outer break" (
+    builtins.break a
+  );
+in
+  b
+```
+
+Nix would correctly enter the debugger at `builtins.break a`, but if you asked
+it to `:continue`, it would skip over the `builtins.break "hello"` expression
+entirely.
+
+Now, Nix will correctly enter the debugger at both breakpoints.

doc/manual/rl-next/env-size-reduction.md

@@ -0,0 +1,7 @@
+---
+synopsis: Reduce eval memory usage and wall time
+prs: 9658
+---
+
+Reduce the size of the `Env` struct used in the evaluator by a pointer, or 8 bytes on most modern machines.
+This reduces memory usage during eval by around 2% and wall time by around 3%.

doc/manual/rl-next/eval-system.md

@@ -0,0 +1,12 @@
+---
+synopsis: Add new `eval-system` setting
+prs: 4093
+---
+
+Add a new `eval-system` option.
+Unlike `system`, it just overrides the value of `builtins.currentSystem`.
+This is more useful than overriding `system`, because you can build these derivations on remote builders which can work on the given system.
+In contrast, `system` also effects scheduling which will cause Nix to build those derivations locally even if that doesn't make sense.
+
+`eval-system` only takes effect if it is non-empty.
+If empty (the default) `system` is used as before, so there is no breakage.

doc/manual/rl-next/nix-env-json-drv-path.md

@@ -0,0 +1,6 @@
+---
+synopsis: Fix `nix-env --query --drv-path --json`
+prs: 9257
+---
+
+Fixed a bug where `nix-env --query` ignored `--drv-path` when `--json` was set.

doc/manual/rl-next/nix-flake-check-logs-actions.md

@@ -0,0 +1,33 @@
+---
+synopsis: "`nix flake check` logs the checks"
+issues: 8882
+prs: 8893
+---
+
+`nix flake check` now logs the checks it runs and the derivations it evaluates:
+
+```
+$ nix flake check -v
+evaluating flake...
+checking flake output 'checks'...
+checking derivation 'checks.aarch64-darwin.ghciwatch-tests'...
+derivation evaluated to /nix/store/nh7dlvsrhds4cxl91mvgj4h5cbq6skmq-ghciwatch-test-0.3.0.drv
+checking derivation 'checks.aarch64-darwin.ghciwatch-clippy'...
+derivation evaluated to /nix/store/9cb5a6wmp6kf6hidqw9wphidvb8bshym-ghciwatch-clippy-0.3.0.drv
+checking derivation 'checks.aarch64-darwin.ghciwatch-doc'...
+derivation evaluated to /nix/store/8brdd3jbawfszpbs7vdpsrhy80as1il8-ghciwatch-doc-0.3.0.drv
+checking derivation 'checks.aarch64-darwin.ghciwatch-fmt'...
+derivation evaluated to /nix/store/wjhs0l1njl5pyji53xlmfjrlya0wmz8p-ghciwatch-fmt-0.3.0.drv
+checking derivation 'checks.aarch64-darwin.ghciwatch-audit'...
+derivation evaluated to /nix/store/z0mps8dyj2ds7c0fn0819y5h5611033z-ghciwatch-audit-0.3.0.drv
+checking flake output 'packages'...
+checking derivation 'packages.aarch64-darwin.default'...
+derivation evaluated to /nix/store/41abbdyglw5x9vcsvd89xan3ydjf8d7r-ghciwatch-0.3.0.drv
+checking flake output 'apps'...
+checking flake output 'devShells'...
+checking derivation 'devShells.aarch64-darwin.default'...
+derivation evaluated to /nix/store/bc935gz7dylzmcpdb5cczr8gngv8pmdb-nix-shell.drv
+running 5 flake checks...
+warning: The check omitted these incompatible systems: aarch64-linux, x86_64-darwin, x86_64-linux
+Use '--all-systems' to check all.
+```

doc/manual/rl-next/print-value-in-coercion-error.md

@@ -0,0 +1,24 @@
+---
+synopsis: Coercion errors include the failing value
+issues: #561
+prs: #9754
+---
+
+The `error: cannot coerce a <TYPE> to a string` message now includes the value
+which caused the error.
+
+Before:
+
+```
+       error: cannot coerce a set to a string
+```
+
+After:
+
+```
+       error: cannot coerce a set to a string: { aesSupport = «thunk»;
+       avx2Support = «thunk»; avx512Support = «thunk»; avxSupport = «thunk»;
+       canExecute = «thunk»; config = «thunk»; darwinArch = «thunk»; darwinMinVersion
+       = «thunk»; darwinMinVersionVariable = «thunk»; darwinPlatform = «thunk»; «84
+       attributes elided»}
+```

doc/manual/rl-next/print-value-in-type-error.md

@@ -0,0 +1,23 @@
+---
+synopsis: Type errors include the failing value
+issues: #561
+prs: #9753
+---
+
+In errors like `value is an integer while a list was expected`, the message now
+includes the failing value.
+
+Before:
+
+```
+       error: value is a set while a string was expected
+```
+
+After:
+
+```
+       error: expected a string but found a set: { ghc810 = «thunk»;
+       ghc8102Binary = «thunk»; ghc8107 = «thunk»; ghc8107Binary = «thunk»;
+       ghc865Binary = «thunk»; ghc90 = «thunk»; ghc902 = «thunk»; ghc92 = «thunk»;
+       ghc924Binary = «thunk»; ghc925 = «thunk»;  «17 attributes elided»}
+```

doc/manual/rl-next/reduce-debugger-clutter.md

@@ -0,0 +1,37 @@
+---
+synopsis: "Visual clutter in `--debugger` is reduced"
+prs: 9919
+---
+
+Before:
+```
+info: breakpoint reached
+
+
+Starting REPL to allow you to inspect the current state of the evaluator.
+
+Welcome to Nix 2.20.0pre20231222_dirty. Type :? for help.
+
+nix-repl> :continue
+error: uh oh
+
+
+Starting REPL to allow you to inspect the current state of the evaluator.
+
+Welcome to Nix 2.20.0pre20231222_dirty. Type :? for help.
+
+nix-repl>
+```
+
+After:
+
+```
+info: breakpoint reached
+
+Nix 2.20.0pre20231222_dirty debugger
+Type :? for help.
+nix-repl> :continue
+error: uh oh
+
+nix-repl>
+```

doc/manual/rl-next/source-location-in-while-evaluating-attribute.md

@@ -0,0 +1,23 @@
+---
+synopsis: "In the debugger, `while evaluating the attribute` errors now include position information"
+prs: 9915
+---
+
+Before:
+
+```
+0: while evaluating the attribute 'python311.pythonForBuild.pkgs'
+0x600001522598
+```
+
+After:
+
+```
+0: while evaluating the attribute 'python311.pythonForBuild.pkgs'
+/nix/store/hg65h51xnp74ikahns9hyf3py5mlbbqq-source/overrides/default.nix:132:27
+
+   131|
+   132|       bootstrappingBase = pkgs.${self.python.pythonAttr}.pythonForBuild.pkgs;
+      |                           ^
+   133|     in
+```

doc/manual/rl-next/source-positions-in-errors.md

@@ -0,0 +1,42 @@
+---
+synopsis: Source locations are printed more consistently in errors
+issues: 561
+prs: 9555
+---
+
+Source location information is now included in error messages more
+consistently. Given this code:
+
+```nix
+let
+  attr = {foo = "bar";};
+  key = {};
+in
+  attr.${key}
+```
+
+Previously, Nix would show this unhelpful message when attempting to evaluate
+it:
+
+```
+error:
+       … while evaluating an attribute name
+
+       error: value is a set while a string was expected
+```
+
+Now, the error message displays where the problematic value was found:
+
+```
+error:
+       … while evaluating an attribute name
+
+         at bad.nix:4:11:
+
+            3|   key = {};
+            4| in attr.${key}
+             |           ^
+            5|
+
+       error: expected a string but found a set: { }
+```

doc/manual/rl-next/stack-overflow-segfaults.md

@@ -0,0 +1,32 @@
+---
+synopsis: Some stack overflow segfaults are fixed
+issues: 9616
+prs: 9617
+---
+
+The number of nested function calls has been restricted, to detect and report
+infinite function call recursions. The default maximum call depth is 10,000 and
+can be set with [the `max-call-depth`
+option](@docroot@/command-ref/conf-file.md#conf-max-call-depth).
+
+This fixes segfaults or the following unhelpful error message in many cases:
+
+    error: stack overflow (possible infinite recursion)
+
+Before:
+
+```
+$ nix-instantiate --eval --expr '(x: x x) (x: x x)'
+Segmentation fault: 11
+```
+
+After:
+
+```
+$ nix-instantiate --eval --expr '(x: x x) (x: x x)'
+error: stack overflow
+
+       at «string»:1:14:
+            1| (x: x x) (x: x x)
+             |              ^
+```

doc/manual/rl-next/with-error-reporting.md

@@ -0,0 +1,31 @@
+---
+synopsis: Better error reporting for `with` expressions
+prs: 9658
+---
+
+`with` expressions using non-attrset values to resolve variables are now reported with proper positions.
+
+Previously an incorrect `with` expression would report no position at all, making it hard to determine where the error originated:
+
+```
+nix-repl> with 1; a
+error:
+       … <borked>
+
+         at «none»:0: (source not available)
+
+       error: value is an integer while a set was expected
+```
+
+Now position information is preserved and reported as with most other errors:
+
+```
+nix-repl> with 1; a
+error:
+       … while evaluating the first subexpression of a with expression
+         at «string»:1:1:
+            1| with 1; a
+             | ^
+
+       error: expected a set but found an integer: 1
+```

doc/manual/src/SUMMARY.md.in

@@ -109,6 +109,7 @@
   - [CLI guideline](contributing/cli-guideline.md)
   - [C++ style guide](contributing/cxx.md)
 - [Release Notes](release-notes/release-notes.md)
+{{#include ./SUMMARY-rl-next.md}}
   - [Release 2.18 (2023-09-20)](release-notes/rl-2.18.md)
   - [Release 2.17 (2023-07-24)](release-notes/rl-2.17.md)
   - [Release 2.16 (2023-05-31)](release-notes/rl-2.16.md)

doc/manual/src/command-ref/nix-instantiate.md

@@ -35,13 +35,51 @@ standard input.
 
   - `--parse`\
     Just parse the input files, and print their abstract syntax trees on
-    standard output in ATerm format.
+    standard output as a Nix expression.
 
   - `--eval`\
     Just parse and evaluate the input files, and print the resulting
     values on standard output. No instantiation of store derivations
     takes place.
 
+    > **Warning**
+    >
+    > This option produces output which can be parsed as a Nix expression which
+    > will produce a different result than the input expression when evaluated.
+    > For example, these two Nix expressions print the same result despite
+    > having different meaning:
+    >
+    > ```console
+    > $ nix-instantiate --eval --expr '{ a = {}; }'
+    > { a = <CODE>; }
+    > $ nix-instantiate --eval --expr '{ a = <CODE>; }'
+    > { a = <CODE>; }
+    > ```
+    >
+    > For human-readable output, `nix eval` (experimental) is more informative:
+    >
+    > ```console
+    > $ nix-instantiate --eval --expr 'a: a'
+    > <LAMBDA>
+    > $ nix eval --expr 'a: a'
+    > «lambda @ «string»:1:1»
+    > ```
+    >
+    > For machine-readable output, the `--xml` option produces unambiguous
+    > output:
+    >
+    > ```console
+    > $ nix-instantiate --eval --xml --expr '{ foo = <CODE>; }'
+    > <?xml version='1.0' encoding='utf-8'?>
+    > <expr>
+    >   <attrs>
+    >     <attr column="3" line="1" name="foo">
+    >       <unevaluated />
+    >     </attr>
+    >   </attrs>
+    > </expr>
+    > ```
+
   - `--find-file`\
     Look up the given files in Nix’s search path (as specified by the
     `NIX_PATH` environment variable). If found, print the corresponding
@@ -61,11 +99,11 @@ standard input.
 
   - `--json`\
     When used with `--eval`, print the resulting value as an JSON
-    representation of the abstract syntax tree rather than as an ATerm.
+    representation of the abstract syntax tree rather than as a Nix expression.
 
   - `--xml`\
     When used with `--eval`, print the resulting value as an XML
-    representation of the abstract syntax tree rather than as an ATerm.
+    representation of the abstract syntax tree rather than as a Nix expression.
     The schema is the same as that used by the [`toXML`
     built-in](../language/builtins.md).
 
@@ -133,28 +171,24 @@ $ nix-instantiate --eval --xml --expr '1 + 2'
 The difference between non-strict and strict evaluation:
 
 ```console
-$ nix-instantiate --eval --xml --expr 'rec { x = "foo"; y = x; }'
-...
-  <attr name="x">
-    <string value="foo" />
-  </attr>
-  <attr name="y">
-    <unevaluated />
-  </attr>
-...
-```
+$ nix-instantiate --eval --xml --expr '{ x = {}; }'
+<?xml version='1.0' encoding='utf-8'?>
+<expr>
+  <attrs>
+    <attr column="3" line="1" name="x">
+      <unevaluated />
+    </attr>
+  </attrs>
+</expr>
 
-Note that `y` is left unevaluated (the XML representation doesn’t
-attempt to show non-normal forms).
-
-```console
-$ nix-instantiate --eval --xml --strict --expr 'rec { x = "foo"; y = x; }'
-...
-  <attr name="x">
-    <string value="foo" />
-  </attr>
-  <attr name="y">
-    <string value="foo" />
-  </attr>
-...
+$ nix-instantiate --eval --xml --strict --expr '{ x = {}; }'
+<?xml version='1.0' encoding='utf-8'?>
+<expr>
+  <attrs>
+    <attr column="3" line="1" name="x">
+      <attrs>
+      </attrs>
+    </attr>
+  </attrs>
+</expr>
 ```

doc/manual/src/command-ref/nix-shell.md

@@ -235,14 +235,14 @@ package like Terraform:
 
 ```bash
 #! /usr/bin/env nix-shell
-#! nix-shell -i bash --packages "terraform.withPlugins (plugins: [ plugins.openstack ])"
+#! nix-shell -i bash --packages 'terraform.withPlugins (plugins: [ plugins.openstack ])'
 
 terraform apply
 ```
 
 > **Note**
 >
-> You must use double quotes (`"`) when passing a simple Nix expression
+> You must use single or double quotes (`'`, `"`) when passing a simple Nix expression
 > in a nix-shell shebang.
 
 Finally, using the merging of multiple nix-shell shebangs the following
@@ -251,7 +251,7 @@ branch):
 
 ```haskell
 #! /usr/bin/env nix-shell
-#! nix-shell -i runghc --packages "haskellPackages.ghcWithPackages (ps: [ps.download-curl ps.tagsoup])"
+#! nix-shell -i runghc --packages 'haskellPackages.ghcWithPackages (ps: [ps.download-curl ps.tagsoup])'
 #! nix-shell -I nixpkgs=https://github.com/NixOS/nixpkgs/archive/nixos-20.03.tar.gz
 
 import Network.Curl.Download

doc/manual/src/command-ref/opt-common.md

@@ -203,3 +203,7 @@ Most Nix commands accept the following command-line options:
   Fix corrupted or missing store paths by redownloading or rebuilding them.
   Note that this is slow because it requires computing a cryptographic hash of the contents of every path in the closure of the build.
   Also note the warning under `nix-store --repair-path`.
+
+> **Note**
+>
+> See [`man nix.conf`](@docroot@/command-ref/conf-file.md#command-line-flags) for overriding configuration settings with command line flags.

doc/manual/src/contributing/hacking.md

@@ -31,7 +31,7 @@ This shell also adds `./outputs/bin/nix` to your `$PATH` so you can run `nix` im
 To get a shell with one of the other [supported compilation environments](#compilation-environments):
 
 ```console
-$ nix develop .#native-clang11StdenvPackages
+$ nix develop .#native-clangStdenvPackages
 ```
 
 > **Note**
@@ -42,8 +42,8 @@ $ nix develop .#native-clang11StdenvPackages
 To build Nix itself in this shell:
 
 ```console
-[nix-shell]$ ./bootstrap.sh
-[nix-shell]$ ./configure $configureFlags --prefix=$(pwd)/outputs/out
+[nix-shell]$ autoreconfPhase
+[nix-shell]$ configurePhase
 [nix-shell]$ make -j $NIX_BUILD_CORES
 ```
 
@@ -75,7 +75,7 @@ $ nix-shell
 To get a shell with one of the other [supported compilation environments](#compilation-environments):
 
 ```console
-$ nix-shell --attr devShells.x86_64-linux.native-clang11StdenvPackages
+$ nix-shell --attr devShells.x86_64-linux.native-clangStdenvPackages
 ```
 
 > **Note**
@@ -86,7 +86,7 @@ $ nix-shell --attr devShells.x86_64-linux.native-clang11StdenvPackages
 To build Nix itself in this shell:
 
 ```console
-[nix-shell]$ ./bootstrap.sh
+[nix-shell]$ autoreconfPhase
 [nix-shell]$ ./configure $configureFlags --prefix=$(pwd)/outputs/out
 [nix-shell]$ make -j $NIX_BUILD_CORES
 ```
@@ -146,6 +146,31 @@ $ nix build .#packages.aarch64-linux.default
 Cross-compiled builds are available for ARMv6 (`armv6l-linux`) and ARMv7 (`armv7l-linux`).
 Add more [system types](#system-type) to `crossSystems` in `flake.nix` to bootstrap Nix on unsupported platforms.
 
+### Building for multiple platforms at once
+
+It is useful to perform multiple cross and native builds on the same source tree,
+for example to ensure that better support for one platform doesn't break the build for another.
+In order to facilitate this, Nix has some support for being built out of tree – that is, placing build artefacts in a different directory than the source code:
+
+1. Create a directory for the build, e.g.
+
+   ```bash
+   mkdir build
+   ```
+
+2. Run the configure script from that directory, e.g.
+
+   ```bash
+   cd build
+   ../configure <configure flags>
+   ```
+
+3. Run make from the source directory, but with the build directory specified, e.g.
+
+   ```bash
+   make builddir=build <make flags>
+   ```
+
 ## System type
 
 Nix uses a string with he following format to identify the *system type* or *platform* it runs on:
@@ -285,3 +310,40 @@ can build it yourself:
 
 Metrics about the change in line/function coverage over time are also
 [available](https://hydra.nixos.org/job/nix/master/coverage#tabs-charts).
+
+## Add a release note
+
+`doc/manual/rl-next` contains release notes entries for all unreleased changes.
+
+User-visible changes should come with a release note.
+
+### Add an entry
+
+Here's what a complete entry looks like. The file name is not incorporated in the document.
+
+```
+---
+synopsis: Basically a title
+issues: 1234
+prs: 1238
+---
+
+Here's one or more paragraphs that describe the change.
+
+- It's markdown
+- Add references to the manual using @docroot@
+```
+
+Significant changes should add the following header, which moves them to the top.
+
+```
+significance: significant
+```
+
+<!-- Keep an eye on https://codeberg.org/fgaz/changelog-d/issues/1 -->
+See also the [format documentation](https://github.com/haskell/cabal/blob/master/CONTRIBUTING.md#changelog).
+
+### Build process
+
+Releases have a precomputed `rl-MAJOR.MINOR.md`, and no `rl-next.md`.
+Set `buildUnreleasedNotes = true;` in `flake.nix` to build the release notes on the fly.

doc/manual/src/contributing/testing.md

@@ -2,10 +2,48 @@
 
 ## Unit-tests
 
-The unit-tests for each Nix library (`libexpr`, `libstore`, etc..) are defined
-under `tests/unit/{library_name}/tests` using the
-[googletest](https://google.github.io/googletest/) and
-[rapidcheck](https://github.com/emil-e/rapidcheck) frameworks.
+The unit tests are defined using the [googletest] and [rapidcheck] frameworks.
+
+[googletest]: https://google.github.io/googletest/
+[rapidcheck]: https://github.com/emil-e/rapidcheck
+
+### Source and header layout
+
+> An example of some files, demonstrating much of what is described below
+>
+> ```
+> src
+> ├── libexpr
+> │   ├── value/context.hh
+> │   ├── value/context.cc
+> │   │
+> │   …
+>     └── tests
+> │       ├── value/context.hh
+> │       ├── value/context.cc
+> │       │
+> │       …
+> │
+> ├── unit-test-data
+> │   ├── libstore
+> │   │   ├── worker-protocol/content-address.bin
+> │   │   …
+> │   …
+> …
+> ```
+
+The unit tests for each Nix library (`libnixexpr`, `libnixstore`, etc..) live inside a directory `src/${library_shortname}/tests` within the directory for the library (`src/${library_shortname}`).
+
+The data is in `unit-test-data`, with one subdir per library, with the same name as where the code goes.
+For example, `libnixstore` code is in `src/libstore`, and its test data is in `unit-test-data/libstore`.
+The path to the `unit-test-data` directory is passed to the unit test executable with the environment variable `_NIX_TEST_UNIT_DATA`.
+
+> **Note**
+> Due to the way googletest works, downstream unit test executables will actually include and re-run upstream library tests.
+> Therefore it is important that the same value for `_NIX_TEST_UNIT_DATA` be used with the tests for each library.
+> That is why we have the test data nested within a single `unit-test-data` directory.
+
+### Running tests
 
 You can run the whole testsuite with `make check`, or the tests for a specific component with `make libfoo-tests_RUN`.
 Finer-grained filtering is also possible using the [--gtest_filter](https://google.github.io/googletest/advanced.html#running-a-subset-of-the-tests) command-line option, or the `GTEST_FILTER` environment variable.
@@ -22,6 +60,24 @@ It is important that these testing libraries don't contain any actual tests them
 On some platforms they would be run as part of every test executable that uses them, which is redundant.
 On other platforms they wouldn't be run at all.
 
+### Characterization testing
+
+See [below](#characterization-testing-1) for a broader discussion of characterization testing.
+
+Like with the functional characterization, `_NIX_TEST_ACCEPT=1` is also used.
+For example:
+```shell-session
+$ _NIX_TEST_ACCEPT=1 make libstore-tests-exe_RUN
+...
+[  SKIPPED ] WorkerProtoTest.string_read
+[  SKIPPED ] WorkerProtoTest.string_write
+[  SKIPPED ] WorkerProtoTest.storePath_read
+[  SKIPPED ] WorkerProtoTest.storePath_write
+...
+```
+will regenerate the "golden master" expected result for the `libnixstore` characterization tests.
+The characterization tests will mark themselves "skipped" since they regenerated the expected result instead of actually testing anything.
+
 ## Functional tests
 
 The functional tests reside under the `tests/functional` directory and are listed in `tests/functional/local.mk`.

doc/manual/src/installation/building-source.md

@@ -3,7 +3,7 @@
 After cloning Nix's Git repository, issue the following commands:
 
 ```console
-$ ./bootstrap.sh
+$ autoreconf -vfi
 $ ./configure options...
 $ make
 $ make install

doc/manual/src/language/advanced-attributes.md

@@ -261,6 +261,9 @@ Derivations can declare some infrequently used optional attributes.
     useful for very trivial derivations (such as `writeText` in Nixpkgs)
     that are cheaper to build than to substitute from a binary cache.
 
+    You may disable the effects of this attibute by enabling the
+    `always-allow-substitutes` configuration option in Nix.
+
     > **Note**
     >
     > You need to have a builder configured which satisfies the
@@ -271,18 +274,21 @@ Derivations can declare some infrequently used optional attributes.
 
   - [`__structuredAttrs`]{#adv-attr-structuredAttrs}\
     If the special attribute `__structuredAttrs` is set to `true`, the other derivation
-    attributes are serialised in JSON format and made available to the
-    builder via the file `.attrs.json` in the builder’s temporary
-    directory. This obviates the need for [`passAsFile`](#adv-attr-passAsFile) since JSON files
-    have no size restrictions, unlike process environments.
+    attributes are serialised into a file in JSON format. The environment variable
+    `NIX_ATTRS_JSON_FILE` points to the exact location of that file both in a build
+    and a [`nix-shell`](../command-ref/nix-shell.md). This obviates the need for
+    [`passAsFile`](#adv-attr-passAsFile) since JSON files have no size restrictions,
+    unlike process environments.
 
     It also makes it possible to tweak derivation settings in a structured way; see
     [`outputChecks`](#adv-attr-outputChecks) for example.
 
     As a convenience to Bash builders,
-    Nix writes a script named `.attrs.sh` to the builder’s directory
-    that initialises shell variables corresponding to all attributes
-    that are representable in Bash. This includes non-nested
+    Nix writes a script that initialises shell variables
+    corresponding to all attributes that are representable in Bash. The
+    environment variable `NIX_ATTRS_SH_FILE` points to the exact
+    location of the script, both in a build and a
+    [`nix-shell`](../command-ref/nix-shell.md). This includes non-nested
     (associative) arrays. For example, the attribute `hardening.format = true`
     ends up as the Bash associative array element `${hardening[format]}`.
 

doc/manual/src/language/derivations.md

@@ -123,6 +123,11 @@ The builder is executed as follows:
     
       - `NIX_STORE` is set to the path of the top-level Nix store
         directory (typically, `/nix/store`).
+
+      - `NIX_ATTRS_JSON_FILE` & `NIX_ATTRS_SH_FILE` if `__structuredAttrs`
+        is set to `true` for the dervation. A detailed explanation of this
+        behavior can be found in the
+        [section about structured attrs](./advanced-attributes.md#adv-attr-structuredAttrs).
     
       - For each output declared in `outputs`, the corresponding
         environment variable is set to point to the intended path in the

doc/manual/src/release-notes/rl-next.md

@@ -1 +0,0 @@
-# Release X.Y (202?-??-??)

flake.lock

@@ -16,34 +16,18 @@
         "type": "github"
       }
     },
-    "lowdown-src": {
-      "flake": false,
-      "locked": {
-        "lastModified": 1633514407,
-        "narHash": "sha256-Dw32tiMjdK9t3ETl5fzGrutQTzh2rufgZV4A/BbxuD4=",
-        "owner": "kristapsdz",
-        "repo": "lowdown",
-        "rev": "d2c2b44ff6c27b936ec27358a2653caaef8f73b8",
-        "type": "github"
-      },
-      "original": {
-        "owner": "kristapsdz",
-        "repo": "lowdown",
-        "type": "github"
-      }
-    },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1700748986,
-        "narHash": "sha256-/nqLrNU297h3PCw4QyDpZKZEUHmialJdZW2ceYFobds=",
+        "lastModified": 1709884566,
+        "narHash": "sha256-NSYJg2sfdO/XS3L8XN/59Zhzn0dqWm7XtVnKI2mHq3w=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "9ba29e2346bc542e9909d1021e8fd7d4b3f64db0",
+        "rev": "2be119add7b37dc535da2dd4cba68e2cf8d1517e",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
-        "ref": "nixos-23.05-small",
+        "ref": "nixos-23.11-small",
         "repo": "nixpkgs",
         "type": "github"
       }
@@ -67,7 +51,6 @@
     "root": {
       "inputs": {
         "flake-compat": "flake-compat",
-        "lowdown-src": "lowdown-src",
         "nixpkgs": "nixpkgs",
         "nixpkgs-regression": "nixpkgs-regression"
       }

flake.nix

@@ -1,18 +1,21 @@
 {
   description = "The purely functional package manager";
 
-  inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.05-small";
+  inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.11-small";
   inputs.nixpkgs-regression.url = "github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2";
-  inputs.lowdown-src = { url = "github:kristapsdz/lowdown"; flake = false; };
   inputs.flake-compat = { url = "github:edolstra/flake-compat"; flake = false; };
 
-  outputs = { self, nixpkgs, nixpkgs-regression, lowdown-src, flake-compat }:
+  outputs = { self, nixpkgs, nixpkgs-regression, flake-compat }:
 
     let
       inherit (nixpkgs) lib;
+      inherit (lib) fileset;
 
       officialRelease = true;
 
+      # Set to true to build the release notes for the next release.
+      buildUnreleasedNotes = false;
+
       version = lib.fileContents ./.version + versionSuffix;
       versionSuffix =
         if officialRelease
@@ -25,9 +28,12 @@
       darwinSystems = [ "x86_64-darwin" "aarch64-darwin" ];
       systems = linuxSystems ++ darwinSystems;
 
-      crossSystems = [ "armv6l-linux" "armv7l-linux" ];
+      crossSystems = [
+        "armv6l-linux" "armv7l-linux"
+        "x86_64-freebsd13" "x86_64-netbsd"
+      ];
 
-      stdenvs = [ "gccStdenv" "clangStdenv" "clang11Stdenv" "stdenv" "libcxxStdenv" "ccacheStdenv" ];
+      stdenvs = [ "gccStdenv" "clangStdenv" "stdenv" "libcxxStdenv" "ccacheStdenv" ];
 
       forAllSystems = lib.genAttrs systems;
 
@@ -42,54 +48,23 @@
             })
             stdenvs);
 
-      # Experimental fileset library: https://github.com/NixOS/nixpkgs/pull/222981
-      # Not an "idiomatic" flake input because:
-      #  - Propagation to dependent locks: https://github.com/NixOS/nix/issues/7730
-      #  - Subflake would download redundant and huge parent flake
-      #  - No git tree hash support: https://github.com/NixOS/nix/issues/6044
-      inherit (import (builtins.fetchTarball { url = "https://github.com/NixOS/nix/archive/1bdcd7fc8a6a40b2e805bad759b36e64e911036b.tar.gz"; sha256 = "sha256:14ljlpdsp4x7h1fkhbmc4bd3vsqnx8zdql4h3037wh09ad6a0893"; }))
-        fileset;
-
-      baseFiles =
-        # .gitignore has already been processed, so any changes in it are irrelevant
-        # at this point. It is not represented verbatim for test purposes because
-        # that would interfere with repo semantics.
-        fileset.fileFilter (f: f.name != ".gitignore") ./.;
-
-      nixSrc = fileset.toSource {
-        root = ./.;
-        fileset = fileset.intersect baseFiles (fileset.unions [
-          ./.version
-          ./boehmgc-coroutine-sp-fallback.diff
-          ./bootstrap.sh
-          ./configure.ac
-          ./doc
-          ./local.mk
-          ./m4
-          ./Makefile
-          ./Makefile.config.in
-          ./misc
-          ./mk
-          ./precompiled-headers.h
-          ./src
-          ./tests/functional
-          ./tests/unit
-          ./COPYING
-          ./scripts/local.mk
-          (fileset.fileFilter (f: lib.strings.hasPrefix "nix-profile" f.name) ./scripts)
-          # TODO: do we really need README.md? It doesn't seem used in the build.
-          ./README.md
-        ]);
-      };
-
       # Memoize nixpkgs for different platforms for efficiency.
       nixpkgsFor = forAllSystems
         (system: let
           make-pkgs = crossSystem: stdenv: import nixpkgs {
-            inherit system crossSystem;
+            localSystem = {
+              inherit system;
+            };
+            crossSystem = if crossSystem == null then null else {
+              system = crossSystem;
+            } // lib.optionalAttrs (crossSystem == "x86_64-freebsd13") {
+              useLLVM = true;
+            };
             overlays = [
               (overlayFor (p: p.${stdenv}))
             ];
+
+            config.permittedInsecurePackages = [ "nix-2.13.6" ];
           };
           stdenvs = forAllStdenvs (make-pkgs null);
           native = stdenvs.stdenvPackages;
@@ -99,112 +74,6 @@
           cross = forAllCrossSystems (crossSystem: make-pkgs crossSystem "stdenv");
         });
 
-      commonDeps =
-        { pkgs
-        , isStatic ? pkgs.stdenv.hostPlatform.isStatic
-        }:
-        with pkgs; rec {
-        # Use "busybox-sandbox-shell" if present,
-        # if not (legacy) fallback and hope it's sufficient.
-        sh = pkgs.busybox-sandbox-shell or (busybox.override {
-          useMusl = true;
-          enableStatic = true;
-          enableMinimal = true;
-          extraConfig = ''
-            CONFIG_FEATURE_FANCY_ECHO y
-            CONFIG_FEATURE_SH_MATH y
-            CONFIG_FEATURE_SH_MATH_64 y
-
-            CONFIG_ASH y
-            CONFIG_ASH_OPTIMIZE_FOR_SIZE y
-
-            CONFIG_ASH_ALIAS y
-            CONFIG_ASH_BASH_COMPAT y
-            CONFIG_ASH_CMDCMD y
-            CONFIG_ASH_ECHO y
-            CONFIG_ASH_GETOPTS y
-            CONFIG_ASH_INTERNAL_GLOB y
-            CONFIG_ASH_JOB_CONTROL y
-            CONFIG_ASH_PRINTF y
-            CONFIG_ASH_TEST y
-          '';
-        });
-
-        configureFlags =
-          lib.optionals stdenv.isLinux [
-            "--with-boost=${boost}/lib"
-            "--with-sandbox-shell=${sh}/bin/busybox"
-          ]
-          ++ lib.optionals (stdenv.isLinux && !(isStatic && stdenv.system == "aarch64-linux")) [
-            "LDFLAGS=-fuse-ld=gold"
-          ];
-
-        testConfigureFlags = [
-          "RAPIDCHECK_HEADERS=${lib.getDev rapidcheck}/extras/gtest/include"
-        ];
-
-        internalApiDocsConfigureFlags = [
-          "--enable-internal-api-docs"
-        ];
-
-        nativeBuildDeps =
-          [
-            buildPackages.bison
-            buildPackages.flex
-            (lib.getBin buildPackages.lowdown-nix)
-            buildPackages.mdbook
-            buildPackages.mdbook-linkcheck
-            buildPackages.autoconf-archive
-            buildPackages.autoreconfHook
-            buildPackages.pkg-config
-
-            # Tests
-            buildPackages.git
-            buildPackages.mercurial # FIXME: remove? only needed for tests
-            buildPackages.jq # Also for custom mdBook preprocessor.
-          ]
-          ++ lib.optionals stdenv.hostPlatform.isLinux [(buildPackages.util-linuxMinimal or buildPackages.utillinuxMinimal)];
-
-        buildDeps =
-          [ curl
-            bzip2 xz brotli editline
-            openssl sqlite
-            libarchive
-            boost
-            lowdown-nix
-          ]
-          ++ lib.optionals stdenv.isLinux [libseccomp]
-          ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium
-          ++ lib.optional stdenv.hostPlatform.isx86_64 libcpuid;
-
-        checkDeps = [
-          gtest
-          rapidcheck
-        ];
-
-        internalApiDocsDeps = [
-          buildPackages.doxygen
-        ];
-
-        awsDeps = lib.optional (stdenv.isLinux || stdenv.isDarwin)
-          (aws-sdk-cpp.override {
-            apis = ["s3" "transfer"];
-            customMemoryManagement = false;
-          });
-
-        propagatedDeps =
-          [ ((boehmgc.override {
-              enableLargeConfig = true;
-            }).overrideAttrs(o: {
-              patches = (o.patches or []) ++ [
-                ./boehmgc-coroutine-sp-fallback.diff
-              ];
-            })
-            )
-            nlohmann_json
-          ];
-      };
-
       installScriptFor = systems:
         with nixpkgsFor.x86_64-linux.native;
         runCommand "installer-script"
@@ -239,37 +108,40 @@
             echo "file installer $out/install" >> $out/nix-support/hydra-build-products
           '';
 
-      testNixVersions = pkgs: client: daemon: with commonDeps { inherit pkgs; }; with pkgs.lib; pkgs.stdenv.mkDerivation {
+      testNixVersions = pkgs: client: daemon: let
+        nix = pkgs.callPackage ./package.nix {
+          pname =
+            "nix-tests"
+            + lib.optionalString
+            (lib.versionAtLeast daemon.version "2.4pre20211005" &&
+            lib.versionAtLeast client.version "2.4pre20211005")
+            "-${client.version}-against-${daemon.version}";
+
+            inherit fileset;
+        };
+      in nix.overrideAttrs (prevAttrs: {
         NIX_DAEMON_PACKAGE = daemon;
         NIX_CLIENT_PACKAGE = client;
-        name =
-          "nix-tests"
-          + optionalString
-            (versionAtLeast daemon.version "2.4pre20211005" &&
-             versionAtLeast client.version "2.4pre20211005")
-            "-${client.version}-against-${daemon.version}";
-        inherit version;
 
-        src = nixSrc;
-
-        VERSION_SUFFIX = versionSuffix;
-
-        nativeBuildInputs = nativeBuildDeps;
-        buildInputs = buildDeps ++ awsDeps ++ checkDeps;
-        propagatedBuildInputs = propagatedDeps;
-
-        enableParallelBuilding = true;
-
-        configureFlags = testConfigureFlags; # otherwise configure fails
         dontBuild = true;
         doInstallCheck = true;
 
+        configureFlags = [
+          "RAPIDCHECK_HEADERS=${lib.getDev pkgs.rapidcheck}/extras/gtest/include"
+          "--disable-build"
+        ];
+
         installPhase = ''
           mkdir -p $out
         '';
 
-        installCheckPhase = "make installcheck -j$NIX_BUILD_CORES -l$NIX_BUILD_CORES";
-      };
+        installCheckPhase = (lib.optionalString pkgs.stdenv.hostPlatform.isDarwin ''
+          export OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES
+        '') + ''
+          mkdir -p src/nix-channel
+          make installcheck -j$NIX_BUILD_CORES -l$NIX_BUILD_CORES
+        '';
+      });
 
       binaryTarball = nix: pkgs:
         let
@@ -346,165 +218,55 @@
           '';
 
       overlayFor = getStdenv: final: prev:
-        let currentStdenv = getStdenv final; in
         {
           nixStable = prev.nix;
 
           # Forward from the previous stage as we don’t want it to pick the lowdown override
           nixUnstable = prev.nixUnstable;
 
-          nix =
-          with final;
-          with commonDeps {
-            inherit pkgs;
-            inherit (currentStdenv.hostPlatform) isStatic;
-          };
-          let
-            canRunInstalled = currentStdenv.buildPlatform.canExecute currentStdenv.hostPlatform;
-          in currentStdenv.mkDerivation (finalAttrs: {
-            name = "nix-${version}";
-            inherit version;
+          changelog-d = final.buildPackages.callPackage ./misc/changelog-d.nix { };
+          boehmgc-nix = (final.boehmgc.override {
+            enableLargeConfig = true;
+          }).overrideAttrs (o: {
+            patches = (o.patches or [ ]) ++ [
+              ./boehmgc-coroutine-sp-fallback.diff
 
-            src = nixSrc;
-            VERSION_SUFFIX = versionSuffix;
-
-            outputs = [ "out" "dev" "doc" ];
-
-            nativeBuildInputs = nativeBuildDeps;
-            buildInputs = buildDeps
-              # There have been issues building these dependencies
-              ++ lib.optionals (currentStdenv.hostPlatform == currentStdenv.buildPlatform) awsDeps
-              ++ lib.optionals finalAttrs.doCheck checkDeps;
-
-            propagatedBuildInputs = propagatedDeps;
-
-            disallowedReferences = [ boost ];
-
-            preConfigure = lib.optionalString (! currentStdenv.hostPlatform.isStatic)
-              ''
-                # Copy libboost_context so we don't get all of Boost in our closure.
-                # https://github.com/NixOS/nixpkgs/issues/45462
-                mkdir -p $out/lib
-                cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*} $out/lib
-                rm -f $out/lib/*.a
-                ${lib.optionalString currentStdenv.hostPlatform.isLinux ''
-                  chmod u+w $out/lib/*.so.*
-                  patchelf --set-rpath $out/lib:${currentStdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.*
-                ''}
-                ${lib.optionalString currentStdenv.hostPlatform.isDarwin ''
-                  for LIB in $out/lib/*.dylib; do
-                    chmod u+w $LIB
-                    install_name_tool -id $LIB $LIB
-                    install_name_tool -delete_rpath ${boost}/lib/ $LIB || true
-                  done
-                  install_name_tool -change ${boost}/lib/libboost_system.dylib $out/lib/libboost_system.dylib $out/lib/libboost_thread.dylib
-                ''}
-              '';
-
-            configureFlags = configureFlags ++
-              [ "--sysconfdir=/etc" ] ++
-              lib.optional stdenv.hostPlatform.isStatic "--enable-embedded-sandbox-shell" ++
-              [ (lib.enableFeature finalAttrs.doCheck "tests") ] ++
-              lib.optionals finalAttrs.doCheck testConfigureFlags ++
-              lib.optional (!canRunInstalled) "--disable-doc-gen";
-
-            enableParallelBuilding = true;
-
-            makeFlags = "profiledir=$(out)/etc/profile.d PRECOMPILE_HEADERS=1";
-
-            doCheck = true;
-
-            installFlags = "sysconfdir=$(out)/etc";
-
-            postInstall = ''
-              mkdir -p $doc/nix-support
-              echo "doc manual $doc/share/doc/nix/manual" >> $doc/nix-support/hydra-build-products
-              ${lib.optionalString currentStdenv.hostPlatform.isStatic ''
-              mkdir -p $out/nix-support
-              echo "file binary-dist $out/bin/nix" >> $out/nix-support/hydra-build-products
-              ''}
-              ${lib.optionalString currentStdenv.isDarwin ''
-              install_name_tool \
-                -change ${boost}/lib/libboost_context.dylib \
-                $out/lib/libboost_context.dylib \
-                $out/lib/libnixutil.dylib
-              ''}
-            '';
-
-            doInstallCheck = finalAttrs.doCheck;
-            installCheckFlags = "sysconfdir=$(out)/etc";
-            installCheckTarget = "installcheck"; # work around buggy detection in stdenv
-
-            separateDebugInfo = !currentStdenv.hostPlatform.isStatic;
-
-            strictDeps = true;
-
-            hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie";
-
-            passthru.perl-bindings = with final; perl.pkgs.toPerlModule (currentStdenv.mkDerivation {
-              name = "nix-perl-${version}";
-
-              src = self;
-
-              nativeBuildInputs =
-                [ buildPackages.autoconf-archive
-                  buildPackages.autoreconfHook
-                  buildPackages.pkg-config
-                ];
-
-              buildInputs =
-                [ nix
-                  curl
-                  bzip2
-                  xz
-                  pkgs.perl
-                  boost
-                ]
-                ++ lib.optional (currentStdenv.isLinux || currentStdenv.isDarwin) libsodium
-                ++ lib.optional currentStdenv.isDarwin darwin.apple_sdk.frameworks.Security;
-
-              configureFlags = [
-                "--with-dbi=${perlPackages.DBI}/${pkgs.perl.libPrefix}"
-                "--with-dbd-sqlite=${perlPackages.DBDSQLite}/${pkgs.perl.libPrefix}"
-              ];
-
-              enableParallelBuilding = true;
-
-              postUnpack = "sourceRoot=$sourceRoot/perl";
-            });
-
-            meta.platforms = lib.platforms.unix;
+              # https://github.com/ivmai/bdwgc/pull/586
+              ./boehmgc-traceable_allocator-public.diff
+            ];
           });
 
-          lowdown-nix = with final; currentStdenv.mkDerivation rec {
-            name = "lowdown-0.9.0";
+          default-busybox-sandbox-shell = final.busybox.override {
+            useMusl = true;
+            enableStatic = true;
+            enableMinimal = true;
+            extraConfig = ''
+              CONFIG_FEATURE_FANCY_ECHO y
+              CONFIG_FEATURE_SH_MATH y
+              CONFIG_FEATURE_SH_MATH_64 y
 
-            src = lowdown-src;
+              CONFIG_ASH y
+              CONFIG_ASH_OPTIMIZE_FOR_SIZE y
 
-            outputs = [ "out" "bin" "dev" ];
-
-            nativeBuildInputs = [ buildPackages.which ];
-
-            configurePhase = ''
-                ${if (currentStdenv.isDarwin && currentStdenv.isAarch64) then "echo \"HAVE_SANDBOX_INIT=false\" > configure.local" else ""}
-                ./configure \
-                  PREFIX=${placeholder "dev"} \
-                  BINDIR=${placeholder "bin"}/bin
+              CONFIG_ASH_ALIAS y
+              CONFIG_ASH_BASH_COMPAT y
+              CONFIG_ASH_CMDCMD y
+              CONFIG_ASH_ECHO y
+              CONFIG_ASH_GETOPTS y
+              CONFIG_ASH_INTERNAL_GLOB y
+              CONFIG_ASH_JOB_CONTROL y
+              CONFIG_ASH_PRINTF y
+              CONFIG_ASH_TEST y
             '';
           };
-        };
 
-      nixos-lib = import (nixpkgs + "/nixos/lib") { };
-
-      # https://nixos.org/manual/nixos/unstable/index.html#sec-calling-nixos-tests
-      runNixOSTestFor = system: test: nixos-lib.runTest {
-        imports = [ test ];
-        hostPkgs = nixpkgsFor.${system}.native;
-        defaults = {
-          nixpkgs.pkgs = nixpkgsFor.${system}.native;
+          nix = final.callPackage ./package.nix {
+            inherit (final) stdenv;
+            inherit versionSuffix fileset;
+            boehmgc = final.boehmgc-nix;
+            busybox-sandbox-shell = final.busybox-sandbox-shell or final.default-busybox-sandbox-shell;
+          };
         };
-        _module.args.nixpkgs = nixpkgs;
-      };
 
     in {
       # A Nixpkgs overlay that overrides the 'nix' and
@@ -516,21 +278,6 @@
         # Binary package for various platforms.
         build = forAllSystems (system: self.packages.${system}.nix);
 
-        buildStatic = lib.genAttrs linux64BitSystems (system: self.packages.${system}.nix-static);
-
-        buildCross = forAllCrossSystems (crossSystem:
-          lib.genAttrs ["x86_64-linux"] (system: self.packages.${system}."nix-${crossSystem}"));
-
-        buildNoGc = forAllSystems (system: self.packages.${system}.nix.overrideAttrs (a: { configureFlags = (a.configureFlags or []) ++ ["--enable-gc=no"];}));
-
-        buildNoTests = forAllSystems (system:
-          self.packages.${system}.nix.overrideAttrs (a: {
-            doCheck =
-              assert ! a?dontCheck;
-              false;
-          })
-        );
-
         # Perl bindings for various platforms.
         perlBindings = forAllSystems (system: nixpkgsFor.${system}.native.nix.perl-bindings);
 
@@ -539,126 +286,80 @@
         # the installation script.
         binaryTarball = forAllSystems (system: binaryTarball nixpkgsFor.${system}.native.nix nixpkgsFor.${system}.native);
 
-        binaryTarballCross = lib.genAttrs ["x86_64-linux"] (system:
-          forAllCrossSystems (crossSystem:
-            binaryTarball
-              self.packages.${system}."nix-${crossSystem}"
-              nixpkgsFor.${system}.cross.${crossSystem}));
-
-        # The first half of the installation script. This is uploaded
-        # to https://nixos.org/nix/install. It downloads the binary
-        # tarball for the user's system and calls the second half of the
-        # installation script.
-        installerScript = installScriptFor [ "x86_64-linux" "i686-linux" "aarch64-linux" "x86_64-darwin" "aarch64-darwin" "armv6l-linux" "armv7l-linux" ];
-        installerScriptForGHA = installScriptFor [ "x86_64-linux" "x86_64-darwin" "armv6l-linux" "armv7l-linux"];
-
         # docker image with Nix inside
         dockerImage = lib.genAttrs linux64BitSystems (system: self.packages.${system}.dockerImage);
 
-        # Line coverage analysis.
-        coverage =
-          with nixpkgsFor.x86_64-linux.native;
-          with commonDeps { inherit pkgs; };
-
-          releaseTools.coverageAnalysis {
-            name = "nix-coverage-${version}";
-
-            src = nixSrc;
-
-            configureFlags = testConfigureFlags;
-
-            enableParallelBuilding = true;
-
-            nativeBuildInputs = nativeBuildDeps;
-            buildInputs = buildDeps ++ propagatedDeps ++ awsDeps ++ checkDeps;
-
-            dontInstall = false;
-
-            doInstallCheck = true;
-            installCheckTarget = "installcheck"; # work around buggy detection in stdenv
-
-            lcovFilter = [ "*/boost/*" "*-tab.*" ];
-
-            hardeningDisable = ["fortify"];
-
-            NIX_CFLAGS_COMPILE = "-DCOVERAGE=1";
-          };
-
         # API docs for Nix's unstable internal C++ interfaces.
-        internal-api-docs =
-          with nixpkgsFor.x86_64-linux.native;
-          with commonDeps { inherit pkgs; };
+        internal-api-docs = let
+          nixpkgs = nixpkgsFor.x86_64-linux.native;
+          inherit (nixpkgs) pkgs;
 
-          stdenv.mkDerivation {
+          nix = pkgs.callPackage ./package.nix {
+            inherit versionSuffix fileset officialRelease buildUnreleasedNotes;
+            inherit (pkgs) changelog-d;
+            boehmgc = pkgs.boehmgc-nix;
+            busybox-sandbox-shell = pkgs.busybox-sandbox-shell;
+          };
+        in
+          nix.overrideAttrs (prev: {
             pname = "nix-internal-api-docs";
-            inherit version;
 
-            src = nixSrc;
+            outputs = [ "out" ];
+            separateDebugInfo = false;
 
-            configureFlags = testConfigureFlags ++ internalApiDocsConfigureFlags;
+            nativeBuildInputs = prev.nativeBuildInputs ++ [ pkgs.doxygen ];
 
-            nativeBuildInputs = nativeBuildDeps;
-            buildInputs = buildDeps ++ propagatedDeps
-              ++ awsDeps ++ checkDeps ++ internalApiDocsDeps;
+            # Depropagate the build inputs for the docs build.
+            propagatedBuildInputs = [ ];
+            buildInputs = prev.buildInputs ++ [ pkgs.buildPackages.doxygen ] ++ [
+              pkgs.gtest
+              pkgs.rapidcheck
+            ] ++ prev.propagatedBuildInputs;
 
-            dontBuild = true;
+            configureFlags = prev.configureFlags ++ [
+              "--enable-internal-api-docs"
+              "RAPIDCHECK_HEADERS=${lib.getDev pkgs.rapidcheck}/extras/gtest/include"
+            ];
 
             installTargets = [ "internal-api-html" ];
 
+            # Convince the package.nix logic that we're building,
+            # but don't actually run the build phase.
+            env.dontBuild = true;
+
+            doCheck = false;
+            doInstallCheck = false;
+
             postInstall = ''
               mkdir -p $out/nix-support
-              echo "doc internal-api-docs $out/share/doc/nix/internal-api/html" >> $out/nix-support/hydra-build-products
+              echo "doc internal-api-docs $out/share/doc/nix/internal-api/html" > $out/nix-support/hydra-build-products
+              rm $out/lib -rf
             '';
-          };
+          });
 
         # System tests.
-        tests.authorization = runNixOSTestFor "x86_64-linux" ./tests/nixos/authorization.nix;
+        tests = import ./tests/nixos { inherit lib nixpkgs nixpkgsFor; } // {
 
-        tests.remoteBuilds = runNixOSTestFor "x86_64-linux" ./tests/nixos/remote-builds.nix;
+          # Make sure that nix-env still produces the exact same result
+          # on a particular version of Nixpkgs.
+          evalNixpkgs =
+            with nixpkgsFor.x86_64-linux.native;
+            runCommand "eval-nixos" { buildInputs = [ nix ]; }
+              ''
+                type -p nix-env
+                # Note: we're filtering out nixos-install-tools because https://github.com/NixOS/nixpkgs/pull/153594#issuecomment-1020530593.
+                time nix-env --store dummy:// -f ${nixpkgs-regression} -qaP --drv-path | sort | grep -v nixos-install-tools > packages
+                [[ $(sha1sum < packages | cut -c1-40) = ff451c521e61e4fe72bdbe2d0ca5d1809affa733 ]]
+                mkdir $out
+              '';
 
-        tests.nix-copy-closure = runNixOSTestFor "x86_64-linux" ./tests/nixos/nix-copy-closure.nix;
-
-        tests.nix-copy = runNixOSTestFor "x86_64-linux" ./tests/nixos/nix-copy.nix;
-
-        tests.nssPreload = runNixOSTestFor "x86_64-linux" ./tests/nixos/nss-preload.nix;
-
-        tests.githubFlakes = runNixOSTestFor "x86_64-linux" ./tests/nixos/github-flakes.nix;
-
-        tests.sourcehutFlakes = runNixOSTestFor "x86_64-linux" ./tests/nixos/sourcehut-flakes.nix;
-
-        tests.tarballFlakes = runNixOSTestFor "x86_64-linux" ./tests/nixos/tarball-flakes.nix;
-
-        tests.containers = runNixOSTestFor "x86_64-linux" ./tests/nixos/containers/containers.nix;
-
-        tests.setuid = lib.genAttrs
-          ["i686-linux" "x86_64-linux"]
-          (system: runNixOSTestFor system ./tests/nixos/setuid.nix);
-
-
-        # Make sure that nix-env still produces the exact same result
-        # on a particular version of Nixpkgs.
-        tests.evalNixpkgs =
-          with nixpkgsFor.x86_64-linux.native;
-          runCommand "eval-nixos" { buildInputs = [ nix ]; }
-            ''
-              type -p nix-env
-              # Note: we're filtering out nixos-install-tools because https://github.com/NixOS/nixpkgs/pull/153594#issuecomment-1020530593.
-              time nix-env --store dummy:// -f ${nixpkgs-regression} -qaP --drv-path | sort | grep -v nixos-install-tools > packages
-              [[ $(sha1sum < packages | cut -c1-40) = ff451c521e61e4fe72bdbe2d0ca5d1809affa733 ]]
-              mkdir $out
-            '';
-
-        tests.nixpkgsLibTests =
-          forAllSystems (system:
-            import (nixpkgs + "/lib/tests/release.nix")
-              { pkgs = nixpkgsFor.${system}.native;
-                nixVersions = [ self.packages.${system}.nix ];
-              }
-          );
-
-        metrics.nixpkgs = import "${nixpkgs-regression}/pkgs/top-level/metrics.nix" {
-          pkgs = nixpkgsFor.x86_64-linux.native;
-          nixpkgs = nixpkgs-regression;
+          nixpkgsLibTests =
+            forAllSystems (system:
+              import (nixpkgs + "/lib/tests/release.nix")
+                { pkgs = nixpkgsFor.${system}.native;
+                  nixVersions = [ self.packages.${system}.nix ];
+                }
+            );
         };
 
         installTests = forAllSystems (system:
@@ -674,12 +375,6 @@
             # `NIX_DAEMON_SOCKET_PATH` which is required for the tests to work
             # againstLatestStable = testNixVersions pkgs pkgs.nix pkgs.nixStable;
           } "touch $out");
-
-        installerTests = import ./tests/installer {
-          binaryTarballs = self.hydraJobs.binaryTarball;
-          inherit nixpkgsFor;
-        };
-
       };
 
       checks = forAllSystems (system: {
@@ -687,6 +382,11 @@
         perlBindings = self.hydraJobs.perlBindings.${system};
         installTests = self.hydraJobs.installTests.${system};
         nixpkgsLibTests = self.hydraJobs.tests.nixpkgsLibTests.${system};
+        rl-next =
+          let pkgs = nixpkgsFor.${system}.native;
+          in pkgs.buildPackages.runCommand "test-rl-next-release-notes" { } ''
+          LANG=C.UTF-8 ${pkgs.changelog-d}/bin/changelog-d ${./doc/manual/rl-next} >$out
+        '';
       } // (lib.optionalAttrs (builtins.elem system linux64BitSystems)) {
         dockerImage = self.hydraJobs.dockerImage.${system};
       });
@@ -725,27 +425,27 @@
 
       devShells = let
         makeShell = pkgs: stdenv:
-          with commonDeps { inherit pkgs; };
-          stdenv.mkDerivation {
-            name = "nix";
+          let
+            nix = pkgs.callPackage ./package.nix {
+              inherit stdenv versionSuffix fileset;
+              boehmgc = pkgs.boehmgc-nix;
+              busybox-sandbox-shell = pkgs.busybox-sandbox-shell or pkgs.default-busybox-sandbox;
+            };
+          in
+            nix.overrideAttrs (prev: {
+              nativeBuildInputs = prev.nativeBuildInputs
+                ++ lib.optional (stdenv.cc.isClang && !stdenv.buildPlatform.isDarwin) pkgs.buildPackages.bear
+                ++ lib.optional
+                  (stdenv.cc.isClang && stdenv.hostPlatform == stdenv.buildPlatform)
+                  pkgs.buildPackages.clang-tools;
+                src = null;
 
-            outputs = [ "out" "dev" "doc" ];
+              doInstallCheck = false;
+              doCheck = false;
+              installFlags = "sysconfdir=$(out)/etc";
+              strictDeps = false;
 
-            nativeBuildInputs = nativeBuildDeps
-                                ++ (lib.optionals stdenv.cc.isClang [ pkgs.bear pkgs.clang-tools ]);
-
-            buildInputs = buildDeps ++ propagatedDeps
-              ++ awsDeps ++ checkDeps ++ internalApiDocsDeps;
-
-            configureFlags = configureFlags
-              ++ testConfigureFlags ++ internalApiDocsConfigureFlags;
-
-            enableParallelBuilding = true;
-
-            installFlags = "sysconfdir=$(out)/etc";
-
-            shellHook =
-              ''
+              shellHook = ''
                 PATH=$prefix/bin:$PATH
                 unset PYTHONPATH
                 export MANPATH=$out/share/man:$MANPATH
@@ -753,7 +453,7 @@
                 # Make bash completion work.
                 XDG_DATA_DIRS+=:$out/share
               '';
-          };
+            });
         in
         forAllSystems (system:
           let

local.mk

@@ -1,5 +1,3 @@
-clean-files += Makefile.config
-
 GLOBAL_CXXFLAGS += -Wno-deprecated-declarations -Werror=switch
 # Allow switch-enum to be overridden for files that do not support it, usually because of dependency headers.
 ERROR_SWITCH_ENUM = -Werror=switch-enum

m4/gcc_bug_80431.m4

@@ -0,0 +1,64 @@
+# Ensure that this bug is not present in the C++ toolchain we are using.
+#
+# URL for bug: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=80431
+#
+# The test program is from that issue, with only a slight modification
+# to set an exit status instead of printing strings.
+AC_DEFUN([ENSURE_NO_GCC_BUG_80431],
+[
+  AC_MSG_CHECKING([that GCC bug 80431 is fixed])
+  AC_LANG_PUSH(C++)
+  AC_RUN_IFELSE(
+    [AC_LANG_PROGRAM(
+      [[
+        #include <cstdio>
+
+        static bool a = true;
+        static bool b = true;
+
+        struct Options { };
+
+        struct Option
+        {
+            Option(Options * options)
+            {
+                a = false;
+            }
+
+            ~Option()
+            {
+                b = false;
+            }
+        };
+
+        struct MyOptions : Options { };
+
+        struct MyOptions2 : virtual MyOptions
+        {
+            Option foo{this};
+        };
+      ]],
+      [[
+        {
+            MyOptions2 opts;
+        }
+        return (a << 1) | b;
+      ]])],
+    [status_80431=0],
+    [status_80431=$?],
+    [
+      # Assume we're bug-free when cross-compiling
+    ])
+  AC_LANG_POP(C++)
+  AS_CASE([$status_80431],
+    [0],[
+      AC_MSG_RESULT(yes)
+    ],
+    [2],[
+      AC_MSG_RESULT(no)
+      AC_MSG_ERROR(Cannot build Nix with C++ compiler with this bug)
+    ],
+    [
+      AC_MSG_RESULT(unexpected result $status_80431: not expected failure with bug, ignoring)
+    ])
+])

maintainers/release-notes

@@ -0,0 +1,179 @@
+#!/usr/bin/env nix-shell
+#!nix-shell -i bash ../shell.nix -I nixpkgs=channel:nixos-unstable-small
+#                                   ^^^^^^^
+#                Only used for bash. shell.nix goes to the flake.
+
+# --- CONFIGURATION ---
+
+# This does double duty for
+# - including rl-next
+# - marking where to insert new links (right after)
+SUMMARY_MARKER_LINE='{{#include ./SUMMARY-rl-next.md}}'
+
+# --- LIB ---
+
+log() {
+  echo 1>&2 "release-notes:" "$@"
+}
+logcmd() {
+  local cmd="$1"
+  shift
+  logcmd2 "$cmd" "${*@Q}" "$cmd" "$@"
+}
+logcmd2() {
+  local fakecmd="$1"
+  local fakeargs="$2"
+  shift
+  shift
+  printf 1>&2 "release-notes: \033[34;1m$fakecmd\033[0m "
+  echo "$fakeargs" 1>&2
+  "$@"
+}
+die() {
+  # ANSI red
+  printf 1>&2 "release-notes: \033[31;1merror:\033[0m"
+  echo 1>&2 "" "$@"
+  exit 1
+}
+confirm() {
+  local answer
+  echo 1>&2 "$@" "[y/n]"
+  read -r answer
+  case "$answer" in
+    y|Y|yes|Yes|YES)
+      return 0
+      ;;
+    n|N|no|No|NO)
+      return 1
+      ;;
+    *)
+      echo 1>&2 "please answer y or n"
+      confirm "$@"
+      ;;
+  esac
+}
+report_done() {
+  logcmd2 "git" "show" git -c pager.show=false show
+  printf 1>&2 "release-notes: \033[32;1mdone\033[0m\n"
+}
+
+# --- PARSE ARGS ---
+
+if [[ $# -gt 0 ]]; then
+  die "Release notes takes no arguments, but make sure to set VERSION."
+fi
+
+# --- CHECKS ---
+
+if [[ ! -e flake.nix ]] || [[ ! -e .git ]]; then
+  die "must run in repo root"
+  exit 1
+fi
+
+# repo must be clean
+if ! git diff --quiet; then
+  die "repo is dirty, please commit or stash changes"
+fi
+
+if ! git diff --quiet --cached; then
+  die "repo has staged changes, please commit or stash them"
+fi
+
+if ! grep "$SUMMARY_MARKER_LINE" doc/manual/src/SUMMARY.md.in >/dev/null; then
+  # would have been nice to catch this early, but won't be worth the extra infra
+  die "SUMMARY.md.in is missing the marker line '$SUMMARY_MARKER_LINE', which would be used for inserting a new release notes page. Please fix the script."
+fi
+
+if [[ ! -n "${VERSION:-}" ]]; then
+  die "please set the VERSION environment variable before invoking this script"
+  exit 1
+fi
+
+# version_major_minor:      MAJOR.MINOR
+# version_full:             MAJOR.MINOR.PATCH
+# IS_PATCH:                 true if this is a patch release; append instead of create
+if grep -E '^[0-9]+\.[0-9]+$' <<< "$VERSION" >/dev/null; then
+  log 'is minor'
+  IS_PATCH=false
+  version_full="$VERSION.0"
+  version_major_minor="$VERSION"
+elif grep -E '^[0-9]+\.[0-9]+\.0$' <<< "$VERSION" >/dev/null; then
+  log 'is minor (.0)'
+  IS_PATCH=false
+  version_full="$VERSION"
+  version_major_minor="$(echo "$VERSION" | sed -e 's/\.0$//')"
+elif grep -E '^[0-9]+\.[0-9]+\.[0-9]+$' <<< "$VERSION" >/dev/null; then
+  log 'is patch'
+  IS_PATCH=true
+  version_full="$VERSION"
+  version_major_minor="$(echo "$VERSION" | sed -e 's/\.[0-9]*$//')"
+else
+  die "VERSION must be MAJOR.MINOR[.PATCH], where each is a number, e.g. 2.20 or 2.20.1 (VERSION was set to $VERSION)"
+fi
+
+unset VERSION
+
+log "version_major_minor=$version_major_minor"
+log "version_full=$version_full"
+log "IS_PATCH=$IS_PATCH"
+
+basename=rl-${version_major_minor}.md
+file=doc/manual/src/release-notes/$basename
+
+if ! $IS_PATCH; then
+  if [[ -e $file ]]; then
+    die "release notes file $file already exists. If you'd like to make a minor release, pass a patch version, e.g. 2.20.1"
+  fi
+fi
+
+# --- DEFAULTS ---
+
+if [[ ! -n "${DATE:-}" ]]; then
+  DATE="$(date +%Y-%m-%d)"
+  log "DATE not set, using $DATE"
+fi
+
+case "$DATE" in
+  [0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9])
+    ;;
+  *)
+    die "DATE must be YYYY-MM-DD, e.g. 2021-12-31 (DATE was set to $DATE)"
+    ;;
+esac
+
+# --- DO THE WORK ---
+
+# menu
+title="Release $version_major_minor ($DATE)"
+# section on page
+section_title="Release $version_full ($DATE)"
+
+(
+  # TODO add minor number, and append?
+  echo "# $section_title"
+  echo
+  changelog-d doc/manual/rl-next | sed -e 's/ *$//'
+) | tee -a $file
+
+log "Wrote $file"
+
+if ! $IS_PATCH; then
+  NEW_SUMMARY_LINE="  - [$title](release-notes/$basename)"
+
+  # find the marker line, insert new link after it
+  escaped_marker="$(echo "$SUMMARY_MARKER_LINE" | sed -e 's/\//\\\//g' -e 's/ /\\ /g')"
+  escaped_line="$(echo "$NEW_SUMMARY_LINE" | sed -e 's/\//\\\//g' -e 's/ /\\ /g')"
+  logcmd sed -i -e "/$escaped_marker/a $escaped_line" doc/manual/src/SUMMARY.md.in
+fi
+
+for f in doc/manual/rl-next/*.md; do
+  if [[ config != "$(basename $f)" ]]; then
+    logcmd git rm $f
+  fi
+done
+
+logcmd git add $file doc/manual/src/SUMMARY.md.in
+logcmd git status
+logcmd git commit -m "release notes: $version_full"
+
+report_done

maintainers/release-process.md

@@ -24,34 +24,23 @@ release:
 * In a checkout of the Nix repo, make sure you're on `master` and run
   `git pull`.
 
-* Move the contents of `doc/manual/src/release-notes/rl-next.md`
-  (except the first line) to
-  `doc/manual/src/release-notes/rl-$VERSION.md` (where `$VERSION` is
-  the contents of `.version` *without* the patch level, e.g. `2.12`
-  rather than `2.12.0`).
-
-* Add a header to `doc/manual/src/release-notes/rl-$VERSION.md` like
-
-  ```
-  # Release 2.12 (2022-12-06)
-  ```
-
-* Proof-read / edit / rearrange the release notes. Breaking changes
-  and highlights should go to the top.
-
-* Add a link to the release notes to `doc/manual/src/SUMMARY.md.in`
-  (*not* `SUMMARY.md`), e.g.
-
-  ```
-  - [Release 2.12 (2022-12-06)](release-notes/rl-2.12.md)
-  ```
-
-* Run
+* Compile the release notes by running
 
   ```console
   $ git checkout -b release-notes
-  $ git add doc/manual/src/release-notes/rl-$VERSION.md
-  $ git commit -a -m 'Release notes'
+  $ VERSION=X.YY ./maintainers/release-notes
+  ```
+
+  where `X.YY` is *without* the patch level, e.g. `2.12` rather than ~~`2.12.0`~~.
+
+  A commit is created.
+
+* Proof-read / edit / rearrange the release notes if needed. Breaking changes
+  and highlights should go to the top.
+
+* Push.
+
+  ```console
   $ git push --set-upstream $REMOTE release-notes
   ```
 
@@ -67,15 +56,17 @@ release:
   $ git checkout -b $VERSION-maintenance
   ```
 
-* Mark the release as stable:
+* Mark the release as official:
 
   ```console
-  $ git cherry-pick f673551e71942a52b6d7ae66af8b67140904a76a
+  $ sed -e 's/officialRelease = false;/officialRelease = true;/' -i flake.nix
   ```
 
   This removes the link to `rl-next.md` from the manual and sets
   `officialRelease = true` in `flake.nix`.
 
+* Commit
+
 * Push the release branch:
 
   ```console
@@ -159,6 +150,30 @@ release:
 
 ## Creating a point release
 
+* Checkout.
+
+  ```console
+  $ git checkout XX.YY-maintenance
+  ```
+
+* Determine the next patch version.
+
+  ```console
+  $ export VERSION=XX.YY.ZZ
+  ```
+
+* Update release notes.
+
+  ```console
+  $ ./maintainers/release-notes
+  ```
+
+* Push.
+
+  ```console
+  $ git push
+  ```
+
 * Wait for the desired evaluation of the maintenance jobset to finish
   building.
 

meson.build

@@ -0,0 +1,222 @@
+project('lix', 'cpp',
+  version : run_command('bash', '-c', 'echo -n $(cat ./.version)$VERSION_SUFFIX', check : true).stdout().strip(),
+  default_options : [
+    'cpp_std=c++20',
+    'warning_level=1',
+    'debug=true',
+    'optimization=3',
+  ],
+)
+
+cxx = meson.get_compiler('cpp')
+
+host_system = host_machine.cpu_family() + '-' + host_machine.system()
+message('canonical Nix system name:', host_system)
+
+all_sources = { }
+all_deps = { }
+
+deps = [ ]
+configdata = { }
+
+#
+# Dependencies
+#
+
+boehm = dependency('bdw-gc', required : get_option('gc'))
+if boehm.found()
+  deps += boehm
+endif
+configdata += {
+  'HAVE_BOEHMGC': boehm.found().to_int(),
+}
+
+boost = dependency('boost', required : true, modules : ['context', 'coroutine', 'container'])
+deps += boost
+
+cpuid = dependency('libcpuid', 'cpuid', required : get_option('cpuid'))
+configdata += {
+  'HAVE_LIBCPUID': cpuid.found().to_int(),
+  }
+deps += cpuid
+
+seccomp = dependency('libseccomp', 'seccomp', required : get_option('seccomp-sandboxing'))
+configdata += {
+  'HAVE_SECCOMP': seccomp.found().to_int(),
+  }
+
+libarchive = dependency('libarchive', required : true)
+deps += libarchive
+
+brotli = [
+  dependency('libbrotlicommon', required : true),
+  dependency('libbrotlidec', required : true),
+  dependency('libbrotlienc', required : true),
+  ]
+deps += brotli
+
+openssl = dependency('libcrypto', 'openssl', required : true)
+deps += openssl
+
+aws_sdk = dependency('aws-cpp-sdk-core', required : true)
+if aws_sdk.found()
+  # The AWS pkg-config adds -std=c++11.
+  aws_sdk = aws_sdk.partial_dependency(
+    compile_args : false,
+    includes : true,
+    link_args : true,
+    links : true,
+    sources : true,
+  )
+  deps += aws_sdk
+  s = aws_sdk.version().split('.')
+  configdata += {
+    'AWS_VERSION_MAJOR': s[0].to_int(),
+    'AWS_VERSION_MINOR': s[1].to_int(),
+    'AWS_VERSION_PATCH': s[2].to_int(),
+  }
+  aws_sdk_transfer = dependency('aws-cpp-sdk-transfer', required : true).partial_dependency(
+    compile_args : false,
+    includes : true,
+    link_args : true,
+    links : true,
+    sources : true,
+  )
+endif
+aws_s3 = dependency('aws-cpp-sdk-s3', required : true)
+if aws_s3.found()
+  # The AWS pkg-config adds -std=c++11.
+  aws_s3 = aws_s3.partial_dependency(
+    compile_args : false,
+    includes : true,
+    link_args : true,
+    links : true,
+    sources : true,
+  )
+  deps += aws_s3
+endif
+
+sqlite = dependency('sqlite3', 'sqlite', version : '>=3.6.19', required : true)
+deps += sqlite
+
+sodium = dependency('sodium', 'libsodium', required : true)
+deps += sodium
+
+curl = dependency('libcurl', 'curl', required : true)
+deps += curl
+
+editline = dependency('libeditline', 'editline', version : '>=1.14', required : true)
+deps += editline
+
+lowdown = dependency('lowdown', version : '>=0.9.0', required : true)
+deps += lowdown
+
+#
+# Build-time tools
+#
+bash = find_program('bash')
+lsof = find_program('lsof')
+
+# This is how Nix does generated headers...
+# FIXME(Qyriad): do we really need to use the shell for this?
+gen_header = generator(
+  bash,
+  arguments : [
+    '-c',
+    'echo \'R"foo(\' | cat - @INPUT@ && echo \')foo"\'',
+  ],
+  capture : true,
+  output : '@PLAINNAME@.gen.hh',
+)
+
+#
+# Configuration
+#
+
+# FIXME(Qyriad): at least do this in the build directory
+run_command('ln', '-s', 'bla', 'tmp_link', check : true)
+can_link_symlink = run_command('ln', 'tmp_link', 'tmp_link2', check : false).returncode() == 0
+run_command('rm', '-f', 'tmp_link', 'tmp_link2', check : true)
+message('possible to create a link to a symlink:', can_link_symlink)
+configdata += { 'CAN_LINK_SYMLINK': can_link_symlink.to_int() }
+
+check_headers = [
+  # I can't use dictionaries here as they can only contain identifier-strings.
+  ['aws/s3/S3Client.h', aws_s3]
+]
+
+foreach headerspec : check_headers
+  key = headerspec[0]
+  value = headerspec[1]
+  define_name = 'HAVE_' + key.underscorify().to_upper()
+  define_value = cxx.check_header(key, dependencies : value).to_int()
+  configdata += { define_name: define_value }
+endforeach
+
+check_funcs = [
+  'lchown',
+  'lutimes',
+  'pipe2',
+  'posix_fallocate',
+  'statvfs',
+  'strsignal',
+  'sysconf',
+]
+
+foreach funcspec : check_funcs
+  define_name = 'HAVE_' + funcspec.underscorify().to_upper()
+  define_value = cxx.has_function(funcspec).to_int()
+  configdata += {
+    define_name: define_value,
+  }
+endforeach
+
+has_pubsetbuf = cxx.compiles('''
+  #include <iostream>
+  using namespace std;
+  static char buf[1024];
+  decltype(cerr.rdbuf()->pubsetbuf(buf, sizeof(buf))) _;
+''')
+message('have function "\x1b[1mstd::basic_streambuf::pubsetbuf\x1b[0m" :', has_pubsetbuf)
+configdata += {
+  'HAVE_PUBSETBUF': has_pubsetbuf.to_int(),
+}
+
+configdata += {
+  'ENABLE_S3': aws_s3.found().to_int(),
+}
+
+configure_file(
+  configuration : {
+    'PACKAGE_NAME': '"' + meson.project_name() + '"',
+    'PACKAGE_VERSION': '"' + meson.project_version() + '"',
+    'PACKAGE_TARNAME': '"' + meson.project_name() + '"',
+    'PACKAGE_STRING': '"' + meson.project_name() + ' ' + meson.project_version() + '"',
+    'HAVE_STRUCT_DIRENT_D_TYPE': 1, # FIXME: actually check this for solaris
+    'SYSTEM': '"' + host_system + '"',
+  } + configdata,
+  output : 'config.h',
+)
+
+add_project_arguments(
+  # TODO(Qyriad): Yes this is how the autoconf+Make system did it.
+  # I would love to remove this.
+  '-include', 'config.h',
+  # TODO(Qyriad): would love to remove these
+  '-Wno-deprecated-declarations',
+  '-Wno-unused-parameter',
+  '-Wno-missing-field-initializers',
+  '-Wno-deprecated-copy',
+  '-pthread',
+  #'-fPIC',
+  language : 'cpp',
+)
+
+# FIXME(Qyriad): only if not Darwin, Solaris, or FreeBSD
+# (...so only if Linux?)
+add_project_link_arguments(
+  '-Wl,--no-copy-dt-needed-entries',
+  language : 'cpp',
+)
+
+subdir('src')

meson.options

@@ -0,0 +1,24 @@
+# vim: filetype=meson
+option('gc', type : 'feature',
+  description : 'enable garbage collection in the Nix expression evaluator (requires Boehm GC)',
+)
+# TODO(Qyriad): is this feature maintained?
+option('embedded-sandbox-shell', type : 'feature',
+  description : 'include the sandbox shell in the Nix bindary',
+)
+
+option('cpuid', type : 'feature',
+  description : 'determine microarchitecture levels with libcpuid (only relevant on x86_64)',
+)
+
+option('seccomp-sandboxing', type : 'feature',
+  description : 'build support for seccomp sandboxing (recommended unless your arch doesn\'t support libseccomp, only relevant on Linux)',
+)
+
+option('store-dir', type : 'string', value : '/nix/store',
+  description : 'path of the Nix store',
+)
+
+option('sandbox-shell', type : 'string',
+  description : 'path to a statically-linked shell to use as /bin/sh in sandboxes (usually busybox)',
+)

misc/changelog-d.cabal.nix

@@ -0,0 +1,31 @@
+{ mkDerivation, aeson, base, bytestring, cabal-install-parsers
+, Cabal-syntax, containers, directory, filepath, frontmatter
+, generic-lens-lite, lib, mtl, optparse-applicative, parsec, pretty
+, regex-applicative, text, pkgs
+}:
+let rev = "f30f6969e9cd8b56242309639d58acea21c99d06";
+in
+mkDerivation {
+  pname = "changelog-d";
+  version = "0.1";
+  src = pkgs.fetchurl {
+    name = "changelog-d-${rev}.tar.gz";
+    url = "https://codeberg.org/roberth/changelog-d/archive/${rev}.tar.gz";
+    hash = "sha256-8a2+i5u7YoszAgd5OIEW0eYUcP8yfhtoOIhLJkylYJ4=";
+  } // { inherit rev; };
+  isLibrary = false;
+  isExecutable = true;
+  libraryHaskellDepends = [
+    aeson base bytestring cabal-install-parsers Cabal-syntax containers
+    directory filepath frontmatter generic-lens-lite mtl parsec pretty
+    regex-applicative text
+  ];
+  executableHaskellDepends = [
+    base bytestring Cabal-syntax directory filepath
+    optparse-applicative
+  ];
+  doHaddock = false;
+  description = "Concatenate changelog entries into a single one";
+  license = lib.licenses.gpl3Plus;
+  mainProgram = "changelog-d";
+}

misc/changelog-d.nix

@@ -0,0 +1,31 @@
+# Taken temporarily from <nixpkgs/pkgs/by-name/ch/changelog-d/package.nix>
+{
+  callPackage,
+  lib,
+  haskell,
+  haskellPackages,
+}:
+
+let
+  hsPkg = haskellPackages.callPackage ./changelog-d.cabal.nix { };
+
+  addCompletions = haskellPackages.generateOptparseApplicativeCompletions ["changelog-d"];
+
+  haskellModifications =
+    lib.flip lib.pipe [
+      addCompletions
+      haskell.lib.justStaticExecutables
+    ];
+
+  mkDerivationOverrides = finalAttrs: oldAttrs: {
+
+    version = oldAttrs.version + "-git-${lib.strings.substring 0 7 oldAttrs.src.rev}";
+
+    meta = oldAttrs.meta // {
+      homepage = "https://codeberg.org/roberth/changelog-d";
+      maintainers = [ lib.maintainers.roberth ];
+    };
+
+  };
+in
+  (haskellModifications hsPkg).overrideAttrs mkDerivationOverrides

mk/build-dir.mk

@@ -0,0 +1,10 @@
+# Initialise support for build directories.
+builddir ?=
+
+ifdef builddir
+  buildprefix = $(builddir)/
+  buildprefixrel = $(builddir)
+else
+  buildprefix =
+  buildprefixrel = .
+endif

mk/install-dirs.mk

@@ -0,0 +1,11 @@
+# Default installation paths.
+prefix ?= /usr/local
+libdir ?= $(prefix)/lib
+bindir ?= $(prefix)/bin
+libexecdir ?= $(prefix)/libexec
+datadir ?= $(prefix)/share
+localstatedir ?= $(prefix)/var
+sysconfdir ?= $(prefix)/etc
+mandir ?= $(prefix)/share/man
+
+DESTDIR ?=

mk/lib.mk

@@ -43,27 +43,6 @@ define newline
 endef
 
 
-# Default installation paths.
-prefix ?= /usr/local
-libdir ?= $(prefix)/lib
-bindir ?= $(prefix)/bin
-libexecdir ?= $(prefix)/libexec
-datadir ?= $(prefix)/share
-localstatedir ?= $(prefix)/var
-sysconfdir ?= $(prefix)/etc
-mandir ?= $(prefix)/share/man
-
-
-# Initialise support for build directories.
-builddir ?=
-
-ifdef builddir
-  buildprefix = $(builddir)/
-else
-  buildprefix =
-endif
-
-
 # Pass -fPIC if we're building dynamic libraries.
 BUILD_SHARED_LIBS ?= 1
 
@@ -94,6 +73,8 @@ ifeq ($(BUILD_DEBUG), 1)
 endif
 
 
+include mk/build-dir.mk
+include mk/install-dirs.mk
 include mk/functions.mk
 include mk/tracing.mk
 include mk/clean.mk
@@ -112,7 +93,7 @@ define include-sub-makefile
   include $(1)
 endef
 
-$(foreach mf, $(makefiles), $(eval $(call include-sub-makefile, $(mf))))
+$(foreach mf, $(makefiles), $(eval $(call include-sub-makefile,$(mf))))
 
 
 # Instantiate stuff.
@@ -132,6 +113,10 @@ $(foreach test-group, $(install-tests-groups), \
     $(eval $(call run-install-test,$(test))) \
     $(eval $(test-group).test-group: $(test).test)))
 
+# Include makefiles requiring built programs.
+$(foreach mf, $(makefiles-late), $(eval $(call include-sub-makefile,$(mf))))
+
+
 $(foreach file, $(man-pages), $(eval $(call install-data-in, $(file), $(mandir)/man$(patsubst .%,%,$(suffix $(file))))))
 
 

mk/programs.mk

@@ -87,6 +87,6 @@ define build-program
   # Phony target to run this program (typically as a dependency of 'check').
   .PHONY: $(1)_RUN
   $(1)_RUN: $$($(1)_PATH)
-	$(trace-test) $$($(1)_PATH)
+	$(trace-test) $$(UNIT_TEST_ENV) $$($(1)_PATH)
 
 endef

mk/templates.mk

@@ -10,10 +10,10 @@ endef
 
 ifneq ($(MAKECMDGOALS), clean)
 
-%.h: %.h.in
-	$(trace-gen) rm -f $@ && ./config.status --quiet --header=$@
+$(buildprefix)%.h: %.h.in
+	$(trace-gen) rm -f $@ && cd $(buildprefixrel) && ./config.status --quiet --header=$(@:$(buildprefix)%=%)
 
-%: %.in
-	$(trace-gen) rm -f $@ && ./config.status --quiet --file=$@
+$(buildprefix)%: %.in
+	$(trace-gen) rm -f $@ && cd $(buildprefixrel) && ./config.status --quiet --file=$(@:$(buildprefix)%=%)
 
 endif

package.nix

@@ -0,0 +1,258 @@
+{
+  pkgs,
+  lib,
+  stdenv,
+  autoconf-archive,
+  autoreconfHook,
+  aws-sdk-cpp,
+  boehmgc,
+  nlohmann_json,
+  bison,
+  changelog-d,
+  cmake,
+  boost,
+  brotli,
+  bzip2,
+  curl,
+  editline,
+  fileset,
+  flex,
+  git,
+  gtest,
+  jq,
+  libarchive,
+  libcpuid,
+  libseccomp,
+  libsodium,
+  lsof,
+  lowdown,
+  mdbook,
+  mdbook-linkcheck,
+  mercurial,
+  meson,
+  ninja,
+  openssl,
+  pkg-config,
+  rapidcheck,
+  sqlite,
+  util-linuxMinimal ? utillinuxMinimal,
+  utillinuxMinimal ? null,
+  xz,
+
+  busybox-sandbox-shell,
+
+  pname ? "nix",
+  versionSuffix ? "",
+  officialRelease ? true,
+  # Set to true to build the release notes for the next release.
+  buildUnreleasedNotes ? false,
+
+  # Not a real argument, just the only way to approximate let-binding some
+  # stuff for argument defaults.
+  __forDefaults ? {
+    canRunInstalled = stdenv.buildPlatform.canExecute stdenv.hostPlatform;
+  },
+}: let
+  inherit (__forDefaults) canRunInstalled;
+
+  version = lib.fileContents ./.version + versionSuffix;
+
+  aws-sdk-cpp-nix = aws-sdk-cpp.override {
+    apis = [ "s3" "transfer" ];
+    customMemoryManagement = false;
+  };
+
+  testConfigureFlags = [
+    "RAPIDCHECK_HEADERS=${lib.getDev rapidcheck}/extras/gtest/include"
+  ];
+
+  # .gitignore has already been processed, so any changes in it are irrelevant
+  # at this point. It is not represented verbatim for test purposes because
+  # that would interfere with repo semantics.
+  baseFiles = fileset.fileFilter (f: f.name != ".gitignore") ./.;
+
+  configureFiles = fileset.unions [
+    ./.version
+    ./configure.ac
+    ./m4
+    # TODO: do we really need README.md? It doesn't seem used in the build.
+    ./README.md
+  ];
+
+  topLevelBuildFiles = fileset.unions [
+    ./local.mk
+    ./Makefile
+    ./Makefile.config.in
+    ./mk
+  ];
+
+ functionalTestFiles = fileset.unions [
+    ./tests/functional
+    ./tests/unit
+    (fileset.fileFilter (f: lib.strings.hasPrefix "nix-profile" f.name) ./scripts)
+ ];
+
+in stdenv.mkDerivation (finalAttrs: {
+  inherit pname version;
+
+  src = fileset.toSource {
+    root = ./.;
+    fileset = fileset.intersection baseFiles (fileset.unions ([
+      configureFiles
+      topLevelBuildFiles
+      functionalTestFiles
+      ./unit-test-data
+    ] ++ lib.optionals finalAttrs.doBuild [
+      ./boehmgc-coroutine-sp-fallback.diff
+      ./doc
+      ./misc
+      ./precompiled-headers.h
+      ./src
+      ./COPYING
+      ./scripts/local.mk
+    ]));
+  };
+
+  VERSION_SUFFIX = versionSuffix;
+
+  outputs = [ "out" ]
+    ++ lib.optionals finalAttrs.doBuild [ "dev" "doc" ];
+
+  # dontBuild isn't specified most of the time, where it is implicitly false.
+  doBuild = !(finalAttrs.dontBuild or false);
+
+  nativeBuildInputs = [
+    bison
+    flex
+  ] ++ [
+    (lib.getBin lowdown)
+    mdbook
+    mdbook-linkcheck
+    autoconf-archive
+    autoreconfHook
+    pkg-config
+
+    # Tests
+    git
+    mercurial
+    jq
+    cmake
+    meson
+    ninja
+    lsof
+  ] ++ lib.optional stdenv.hostPlatform.isLinux util-linuxMinimal
+    ++ lib.optional (!officialRelease && buildUnreleasedNotes) changelog-d;
+
+  buildInputs = [
+    curl
+    bzip2
+    xz
+    brotli
+    editline
+    openssl
+    sqlite
+    libarchive
+    boost
+    lowdown
+    libsodium
+  ]
+    ++ lib.optionals stdenv.isLinux [ libseccomp ]
+    ++ lib.optional stdenv.hostPlatform.isx86_64 libcpuid
+    # There have been issues building these dependencies
+    ++ lib.optional (stdenv.hostPlatform == stdenv.buildPlatform) aws-sdk-cpp-nix
+  ;
+
+  checkInputs = [
+    gtest
+    rapidcheck
+  ];
+
+  propagatedBuildInputs = [
+    boehmgc
+    nlohmann_json
+  ];
+
+  disallowedReferences = [
+    boost
+  ];
+
+  # Needed for Meson to find Boost.
+  # https://github.com/NixOS/nixpkgs/issues/86131.
+  env.BOOST_INCLUDEDIR = "${lib.getDev boost}/include";
+  env.BOOST_LIBRARYDIR = "${lib.getLib boost}/lib";
+
+  preConfigure = lib.optionalString (finalAttrs.doBuild && (! stdenv.hostPlatform.isStatic)) ''
+      # Copy libboost_context so we don't get all of Boost in our closure.
+      # https://github.com/NixOS/nixpkgs/issues/45462
+      mkdir -p $out/lib
+      cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*} $out/lib
+      rm -f $out/lib/*.a
+      ${lib.optionalString stdenv.hostPlatform.isLinux ''
+        chmod u+w $out/lib/*.so.*
+        patchelf --set-rpath $out/lib:${stdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.*
+      ''}
+      ${lib.optionalString stdenv.hostPlatform.isDarwin ''
+        for LIB in $out/lib/*.dylib; do
+          chmod u+w $LIB
+          install_name_tool -id $LIB $LIB
+          install_name_tool -delete_rpath ${boost}/lib/ $LIB || true
+        done
+        install_name_tool -change ${boost}/lib/libboost_system.dylib $out/lib/libboost_system.dylib $out/lib/libboost_thread.dylib
+      ''}
+  '';
+
+  configureFlags = lib.optionals stdenv.isLinux [
+    "--with-boost=${boost}/lib"
+    "--with-sandbox-shell=${busybox-sandbox-shell}/bin/busybox"
+  ] ++ lib.optionals (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) [
+    "LDFLAGS=-fuse-ld=gold"
+  ] ++ [ "--sysconfdir=/etc" ]
+    ++ lib.optional stdenv.hostPlatform.isStatic "--enable-embedded-sandbox-shell"
+    ++ [ (lib.enableFeature finalAttrs.doCheck "tests") ]
+    ++ lib.optionals finalAttrs.doCheck testConfigureFlags
+    ++ lib.optional (!canRunInstalled) "--disable-doc-gen"
+  ;
+
+  enableParallelBuilding = true;
+
+  makeFlags = "profiledir=$(out)/etc/profile.d PRECOMPILE_HEADERS=1";
+
+  doCheck = true;
+
+  installFlags = "sysconfdir=$(out)/etc";
+
+  postInstall = ''
+      mkdir -p $doc/nix-support
+      echo "doc manual $doc/share/doc/nix/manual" >> $doc/nix-support/hydra-build-products
+      ${lib.optionalString stdenv.hostPlatform.isStatic ''
+      mkdir -p $out/nix-support
+      echo "file binary-dist $out/bin/nix" >> $out/nix-support/hydra-build-products
+      ''}
+      ${lib.optionalString stdenv.isDarwin ''
+      install_name_tool \
+        -change ${boost}/lib/libboost_context.dylib \
+        $out/lib/libboost_context.dylib \
+        $out/lib/libnixutil.dylib
+      ''}
+  '';
+
+  doInstallCheck = finalAttrs.doCheck;
+  installCheckFlags = "sysconfdir=$(out)/etc";
+  installCheckTarget = "installcheck"; # work around buggy detection in stdenv
+
+  preInstallCheck = lib.optionalString stdenv.hostPlatform.isDarwin ''
+    export OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES
+  '';
+
+  separateDebugInfo = !stdenv.hostPlatform.isStatic && finalAttrs.doBuild;
+
+  strictDeps = true;
+
+  hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie";
+
+  meta.platforms = lib.platforms.unix;
+
+  passthru.perl-bindings = pkgs.callPackage ./perl {
+    inherit fileset stdenv;
+  };
+})

perl/Makefile

@@ -1,6 +1,12 @@
 makefiles = local.mk
 
-GLOBAL_CXXFLAGS += -g -Wall -std=c++2a -I ../src
+GLOBAL_CXXFLAGS += -g -Wall -std=c++2a
+
+# A convenience for concurrent development of Nix and its Perl bindings.
+# Not needed in a standalone build of the Perl bindings.
+ifneq ("$(wildcard ../src)", "")
+  GLOBAL_CXXFLAGS += -I ../src
+endif
 
 -include Makefile.config
 

perl/default.nix

@@ -0,0 +1,51 @@
+{ lib, fileset
+, stdenv
+, perl, perlPackages
+, autoconf-archive, autoreconfHook, pkg-config
+, nix, curl, bzip2, xz, boost, libsodium, darwin
+}:
+
+perl.pkgs.toPerlModule (stdenv.mkDerivation {
+  name = "nix-perl-${nix.version}";
+
+  src = fileset.toSource {
+    root = ../.;
+    fileset = fileset.unions [
+      ../.version
+      ../m4
+      ../mk
+      ./MANIFEST
+      ./Makefile
+      ./Makefile.config.in
+      ./configure.ac
+      ./lib
+      ./local.mk
+    ];
+  };
+
+  nativeBuildInputs =
+    [ autoconf-archive
+      autoreconfHook
+      pkg-config
+    ];
+
+  buildInputs =
+    [ nix
+      curl
+      bzip2
+      xz
+      perl
+      boost
+    ]
+    ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium
+    ++ lib.optional stdenv.isDarwin darwin.apple_sdk.frameworks.Security;
+
+  configureFlags = [
+    "--with-dbi=${perlPackages.DBI}/${perl.libPrefix}"
+    "--with-dbd-sqlite=${perlPackages.DBDSQLite}/${perl.libPrefix}"
+  ];
+
+  enableParallelBuilding = true;
+
+  postUnpack = "sourceRoot=$sourceRoot/perl";
+})

scripts/install-darwin-multi-user.sh

@@ -3,11 +3,13 @@
 set -eu
 set -o pipefail
 
+# System specific settings
+export NIX_FIRST_BUILD_UID="${NIX_FIRST_BUILD_UID:-301}"
+export NIX_BUILD_USER_NAME_TEMPLATE="_nixbld%d"
+
 readonly NIX_DAEMON_DEST=/Library/LaunchDaemons/org.nixos.nix-daemon.plist
 # create by default; set 0 to DIY, use a symlink, etc.
 readonly NIX_VOLUME_CREATE=${NIX_VOLUME_CREATE:-1} # now default
-NIX_FIRST_BUILD_UID="301"
-NIX_BUILD_USER_NAME_TEMPLATE="_nixbld%d"
 
 # caution: may update times on / if not run as normal non-root user
 read_only_root() {
@@ -100,7 +102,7 @@ poly_extra_try_me_commands() {
 poly_configure_nix_daemon_service() {
     task "Setting up the nix-daemon LaunchDaemon"
     _sudo "to set up the nix-daemon as a LaunchDaemon" \
-          /usr/bin/install -m -rw-r--r-- "/nix/var/nix/profiles/default$NIX_DAEMON_DEST" "$NIX_DAEMON_DEST"
+          /usr/bin/install -m "u=rw,go=r" "/nix/var/nix/profiles/default$NIX_DAEMON_DEST" "$NIX_DAEMON_DEST"
 
     _sudo "to load the LaunchDaemon plist for nix-daemon" \
           launchctl load /Library/LaunchDaemons/org.nixos.nix-daemon.plist

scripts/install-multi-user.sh

@@ -25,9 +25,9 @@ readonly RED='\033[31m'
 readonly NIX_USER_COUNT=${NIX_USER_COUNT:-32}
 readonly NIX_BUILD_GROUP_ID="${NIX_BUILD_GROUP_ID:-30000}"
 readonly NIX_BUILD_GROUP_NAME="nixbld"
-# darwin installer needs to override these
-NIX_FIRST_BUILD_UID="${NIX_FIRST_BUILD_UID:-30001}"
-NIX_BUILD_USER_NAME_TEMPLATE="nixbld%d"
+# each system specific installer must set these:
+#   NIX_FIRST_BUILD_UID
+#   NIX_BUILD_USER_NAME_TEMPLATE
 # Please don't change this. We don't support it, because the
 # default shell profile that comes with Nix doesn't support it.
 readonly NIX_ROOT="/nix"
@@ -699,6 +699,12 @@ EOF
     fi
 }
 
+check_required_system_specific_settings() {
+    if [ -z "${NIX_FIRST_BUILD_UID+x}" ] || [ -z "${NIX_BUILD_USER_NAME_TEMPLATE+x}" ]; then
+        failure "Internal error: System specific installer for $(uname) ($1) does not export required settings."
+    fi
+}
+
 welcome_to_nix() {
     local -r NIX_UID_RANGES="${NIX_FIRST_BUILD_UID}..$((NIX_FIRST_BUILD_UID + NIX_USER_COUNT - 1))"
     local -r RANGE_TEXT=$(echo -ne "${BLUE}(uids [${NIX_UID_RANGES}])${ESC}")
@@ -718,7 +724,9 @@ manager. This will happen in a few stages:
    if you are ready to continue.
 
 3. Create the system users ${RANGE_TEXT} and groups ${GROUP_TEXT}
-   that the Nix daemon uses to run builds.
+   that the Nix daemon uses to run builds. To create system users
+   in a different range, exit and run this tool again with
+   NIX_FIRST_BUILD_UID set.
 
 4. Perform the basic installation of the Nix files daemon.
 
@@ -960,13 +968,16 @@ main() {
     if is_os_darwin; then
         # shellcheck source=./install-darwin-multi-user.sh
         . "$EXTRACTED_NIX_PATH/install-darwin-multi-user.sh"
+        check_required_system_specific_settings "install-darwin-multi-user.sh"
     elif is_os_linux; then
         # shellcheck source=./install-systemd-multi-user.sh
         . "$EXTRACTED_NIX_PATH/install-systemd-multi-user.sh" # most of this works on non-systemd distros also
+        check_required_system_specific_settings "install-systemd-multi-user.sh"
     else
         failure "Sorry, I don't know what to do on $(uname)"
     fi
 
+
     welcome_to_nix
 
     if ! is_root; then

scripts/install-systemd-multi-user.sh

@@ -3,6 +3,10 @@
 set -eu
 set -o pipefail
 
+# System specific settings
+export NIX_FIRST_BUILD_UID="${NIX_FIRST_BUILD_UID:-30001}"
+export NIX_BUILD_USER_NAME_TEMPLATE="nixbld%d"
+
 readonly SERVICE_SRC=/lib/systemd/system/nix-daemon.service
 readonly SERVICE_DEST=/etc/systemd/system/nix-daemon.service
 

scripts/nix-profile-daemon.fish.in

@@ -19,6 +19,14 @@ set __ETC_PROFILE_NIX_SOURCED 1
 
 set --export NIX_PROFILES "@localstatedir@/nix/profiles/default $HOME/.nix-profile"
 
+# Populate bash completions, .desktop files, etc
+if test -z "$XDG_DATA_DIRS"
+    # According to XDG spec the default is /usr/local/share:/usr/share, don't set something that prevents that default
+    set --export XDG_DATA_DIRS "/usr/local/share:/usr/share:/nix/var/nix/profiles/default/share"
+else
+    set --export XDG_DATA_DIRS "$XDG_DATA_DIRS:/nix/var/nix/profiles/default/share"
+end
+
 # Set $NIX_SSL_CERT_FILE so that Nixpkgs applications like curl work.
 if test -n "$NIX_SSH_CERT_FILE"
   : # Allow users to override the NIX_SSL_CERT_FILE

scripts/nix-profile-daemon.sh.in

@@ -30,6 +30,14 @@ fi
 
 export NIX_PROFILES="@localstatedir@/nix/profiles/default $NIX_LINK"
 
+# Populate bash completions, .desktop files, etc
+if [ -z "${XDG_DATA_DIRS-}" ]; then
+    # According to XDG spec the default is /usr/local/share:/usr/share, don't set something that prevents that default
+    export XDG_DATA_DIRS="/usr/local/share:/usr/share:$NIX_LINK/share:/nix/var/nix/profiles/default/share"
+else
+    export XDG_DATA_DIRS="$XDG_DATA_DIRS:$NIX_LINK/share:/nix/var/nix/profiles/default/share"
+fi
+
 # Set $NIX_SSL_CERT_FILE so that Nixpkgs applications like curl work.
 if [ -n "${NIX_SSL_CERT_FILE:-}" ]; then
     : # Allow users to override the NIX_SSL_CERT_FILE

scripts/nix-profile.fish.in

@@ -20,6 +20,14 @@ if test -n "$HOME" && test -n "$USER"
     # This part should be kept in sync with nixpkgs:nixos/modules/programs/environment.nix
     set --export NIX_PROFILES "@localstatedir@/nix/profiles/default $HOME/.nix-profile"
 
+    # Populate bash completions, .desktop files, etc
+    if test -z "$XDG_DATA_DIRS"
+        # According to XDG spec the default is /usr/local/share:/usr/share, don't set something that prevents that default
+        set --export XDG_DATA_DIRS "/usr/local/share:/usr/share:$NIX_LINK/share:/nix/var/nix/profiles/default/share"
+    else
+        set --export XDG_DATA_DIRS "$XDG_DATA_DIRS:$NIX_LINK/share:/nix/var/nix/profiles/default/share"
+    end
+
     # Set $NIX_SSL_CERT_FILE so that Nixpkgs applications like curl work.
     if test -n "$NIX_SSH_CERT_FILE"
         : # Allow users to override the NIX_SSL_CERT_FILE

scripts/nix-profile.sh.in

@@ -32,6 +32,14 @@ if [ -n "$HOME" ] && [ -n "$USER" ]; then
     # This part should be kept in sync with nixpkgs:nixos/modules/programs/environment.nix
     export NIX_PROFILES="@localstatedir@/nix/profiles/default $NIX_LINK"
 
+    # Populate bash completions, .desktop files, etc
+    if [ -z "${XDG_DATA_DIRS-}" ]; then
+        # According to XDG spec the default is /usr/local/share:/usr/share, don't set something that prevents that default
+        export XDG_DATA_DIRS="/usr/local/share:/usr/share:$NIX_LINK/share:/nix/var/nix/profiles/default/share"
+    else
+        export XDG_DATA_DIRS="$XDG_DATA_DIRS:$NIX_LINK/share:/nix/var/nix/profiles/default/share"
+    fi
+
     # Set $NIX_SSL_CERT_FILE so that Nixpkgs applications like curl work.
     if [ -e /etc/ssl/certs/ca-certificates.crt ]; then # NixOS, Ubuntu, Debian, Gentoo, Arch
         export NIX_SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt

src/build-remote/build-remote.cc

@@ -137,11 +137,8 @@ static int main_build_remote(int argc, char * * argv)
                 for (auto & m : machines) {
                     debug("considering building on remote machine '%s'", m.storeUri);
 
-                    if (m.enabled
-                        && (neededSystem == "builtin"
-                            || std::find(m.systemTypes.begin(),
-                                m.systemTypes.end(),
-                                neededSystem) != m.systemTypes.end()) &&
+                    if (m.enabled &&
+                        m.systemSupported(neededSystem) &&
                         m.allSupported(requiredFeatures) &&
                         m.mandatoryMet(requiredFeatures))
                     {
@@ -205,7 +202,7 @@ static int main_build_remote(int argc, char * * argv)
                         else
                             drvstr = "<unknown>";
 
-                        auto error = hintformat(errorText);
+                        auto error = HintFmt(errorText);
                         error
                             % drvstr
                             % neededSystem
@@ -214,7 +211,7 @@ static int main_build_remote(int argc, char * * argv)
 
                         for (auto & m : machines)
                             error
-                                % concatStringsSep<std::vector<std::string>>(", ", m.systemTypes)
+                                % concatStringsSep<StringSet>(", ", m.systemTypes)
                                 % m.maxJobs
                                 % concatStringsSep<StringSet>(", ", m.supportedFeatures)
                                 % concatStringsSep<StringSet>(", ", m.mandatoryFeatures);

src/libcmd/built-path.cc

@@ -12,9 +12,9 @@ namespace nix {
     bool MY_TYPE ::operator COMPARATOR (const MY_TYPE & other) const \
     { \
         const MY_TYPE* me = this; \
-        auto fields1 = std::make_tuple<const CHILD_TYPE &, const FIELD_TYPE &>(*me->drvPath, me->FIELD); \
+        auto fields1 = std::tie(*me->drvPath, me->FIELD); \
         me = &other; \
-        auto fields2 = std::make_tuple<const CHILD_TYPE &, const FIELD_TYPE &>(*me->drvPath, me->FIELD); \
+        auto fields2 = std::tie(*me->drvPath, me->FIELD); \
         return fields1 COMPARATOR fields2; \
     }
 #define CMP(CHILD_TYPE, MY_TYPE, FIELD) \

src/libcmd/command.cc

@@ -98,7 +98,7 @@ EvalCommand::EvalCommand()
 EvalCommand::~EvalCommand()
 {
     if (evalState)
-        evalState->printStats();
+        evalState->maybePrintStats();
 }
 
 ref<Store> EvalCommand::getEvalStore()
@@ -175,7 +175,7 @@ void BuiltPathsCommand::run(ref<Store> store, Installables && installables)
             throw UsageError("'--all' does not expect arguments");
         // XXX: Only uses opaque paths, ignores all the realisations
         for (auto & p : store->queryAllValidPaths())
-            paths.push_back(BuiltPath::Opaque{p});
+            paths.emplace_back(BuiltPath::Opaque{p});
     } else {
         paths = Installable::toBuiltPaths(getEvalStore(), store, realiseMode, operateOn, installables);
         if (recursive) {
@@ -188,7 +188,7 @@ void BuiltPathsCommand::run(ref<Store> store, Installables && installables)
             }
             store->computeFSClosure(pathsRoots, pathsClosure);
             for (auto & path : pathsClosure)
-                paths.push_back(BuiltPath::Opaque{path});
+                paths.emplace_back(BuiltPath::Opaque{path});
         }
     }
 

src/libcmd/command.hh

@@ -34,21 +34,28 @@ struct NixMultiCommand : virtual MultiCommand, virtual Command
 // For the overloaded run methods
 #pragma GCC diagnostic ignored "-Woverloaded-virtual"
 
-/* A command that requires a Nix store. */
+/**
+ * A command that requires a \ref Store "Nix store".
+ */
 struct StoreCommand : virtual Command
 {
     StoreCommand();
     void run() override;
     ref<Store> getStore();
     virtual ref<Store> createStore();
+    /**
+     * Main entry point, with a `Store` provided
+     */
     virtual void run(ref<Store>) = 0;
 
 private:
     std::shared_ptr<Store> _store;
 };
 
-/* A command that copies something between `--from` and `--to`
-   stores. */
+/**
+ * A command that copies something between `--from` and `--to` \ref
+ * Store stores.
+ */
 struct CopyCommand : virtual StoreCommand
 {
     std::string srcUri, dstUri;
@@ -60,6 +67,9 @@ struct CopyCommand : virtual StoreCommand
     ref<Store> getDstStore();
 };
 
+/**
+ * A command that needs to evaluate Nix language expressions.
+ */
 struct EvalCommand : virtual StoreCommand, MixEvalArgs
 {
     bool startReplOnEvalErrors = false;
@@ -79,6 +89,10 @@ private:
     std::shared_ptr<EvalState> evalState;
 };
 
+/**
+ * A mixin class for commands that process flakes, adding a few standard
+ * flake-related options/flags.
+ */
 struct MixFlakeOptions : virtual Args, EvalCommand
 {
     flake::LockFlags lockFlags;
@@ -87,6 +101,14 @@ struct MixFlakeOptions : virtual Args, EvalCommand
 
     MixFlakeOptions();
 
+    /**
+     * The completion for some of these flags depends on the flake(s) in
+     * question.
+     *
+     * This method should be implemented to gather all flakerefs the
+     * command is operating with (presumably specified via some other
+     * arguments) so that the completions for these flags can use them.
+     */
     virtual std::vector<std::string> getFlakesForCompletion()
     { return {}; }
 
@@ -112,15 +134,29 @@ struct SourceExprCommand : virtual Args, MixFlakeOptions
 
     virtual Strings getDefaultFlakeAttrPathPrefixes();
 
+    /**
+     * Complete an installable from the given prefix.
+     */
     void completeInstallable(std::string_view prefix);
 };
 
+/**
+ * A mixin class for commands that need a read-only flag.
+ *
+ * What exactly is "read-only" is unspecified, but it will usually be
+ * the \ref Store "Nix store".
+ */
 struct MixReadOnlyOption : virtual Args
 {
     MixReadOnlyOption();
 };
 
-/* Like InstallablesCommand but the installables are not loaded */
+/**
+ * Like InstallablesCommand but the installables are not loaded.
+ *
+ * This is needed by `CmdRepl` which wants to load (and reload) the
+ * installables itself.
+ */
 struct RawInstallablesCommand : virtual Args, SourceExprCommand
 {
     RawInstallablesCommand();
@@ -129,7 +165,7 @@ struct RawInstallablesCommand : virtual Args, SourceExprCommand
 
     void run(ref<Store> store) override;
 
-    // FIXME make const after CmdRepl's override is fixed up
+    // FIXME make const after `CmdRepl`'s override is fixed up
     virtual void applyDefaultInstallables(std::vector<std::string> & rawInstallables);
 
     bool readFromStdIn = false;
@@ -140,8 +176,11 @@ private:
 
     std::vector<std::string> rawInstallables;
 };
-/* A command that operates on a list of "installables", which can be
-   store paths, attribute paths, Nix expressions, etc. */
+
+/**
+ * A command that operates on a list of "installables", which can be
+ * store paths, attribute paths, Nix expressions, etc.
+ */
 struct InstallablesCommand : RawInstallablesCommand
 {
     virtual void run(ref<Store> store, Installables && installables) = 0;
@@ -149,7 +188,9 @@ struct InstallablesCommand : RawInstallablesCommand
     void run(ref<Store> store, std::vector<std::string> && rawInstallables) override;
 };
 
-/* A command that operates on exactly one "installable" */
+/**
+ * A command that operates on exactly one "installable".
+ */
 struct InstallableCommand : virtual Args, SourceExprCommand
 {
     InstallableCommand();
@@ -175,7 +216,12 @@ struct MixOperateOnOptions : virtual Args
     MixOperateOnOptions();
 };
 
-/* A command that operates on zero or more store paths. */
+/**
+ * A command that operates on zero or more extant store paths.
+ *
+ * If the argument the user passes is a some sort of recipe for a path
+ * not yet built, it must be built first.
+ */
 struct BuiltPathsCommand : InstallablesCommand, virtual MixOperateOnOptions
 {
 private:
@@ -207,7 +253,9 @@ struct StorePathsCommand : public BuiltPathsCommand
     void run(ref<Store> store, BuiltPaths && paths) override;
 };
 
-/* A command that operates on exactly one store path. */
+/**
+ * A command that operates on exactly one store path.
+ */
 struct StorePathCommand : public StorePathsCommand
 {
     virtual void run(ref<Store> store, const StorePath & storePath) = 0;
@@ -215,7 +263,9 @@ struct StorePathCommand : public StorePathsCommand
     void run(ref<Store> store, StorePaths && storePaths) override;
 };
 
-/* A helper class for registering commands globally. */
+/**
+ * A helper class for registering \ref Command commands globally.
+ */
 struct RegisterCommand
 {
     typedef std::map<std::vector<std::string>, std::function<ref<Command>()>> Commands;
@@ -271,7 +321,11 @@ struct MixEnvironment : virtual Args {
 
     MixEnvironment();
 
-    /* Modify global environ based on ignoreEnvironment, keep, and unset. It's expected that exec will be called before this class goes out of scope, otherwise environ will become invalid. */
+    /***
+     * Modify global environ based on `ignoreEnvironment`, `keep`, and
+     * `unset`. It's expected that exec will be called before this class
+     * goes out of scope, otherwise `environ` will become invalid.
+     */
     void setEnviron();
 };
 

src/libcmd/editor-for.cc

@@ -1,5 +1,6 @@
 #include "util.hh"
 #include "editor-for.hh"
+#include "source-path.hh"
 
 namespace nix {
 

src/libcmd/editor-for.hh

@@ -2,7 +2,7 @@
 ///@file
 
 #include "types.hh"
-#include "input-accessor.hh"
+#include "source-path.hh"
 
 namespace nix {
 

src/libcmd/installable-flake.cc

@@ -47,7 +47,7 @@ Value * InstallableFlake::getFlakeOutputs(EvalState & state, const flake::Locked
     auto aOutputs = vFlake->attrs->get(state.symbols.create("outputs"));
     assert(aOutputs);
 
-    state.forceValue(*aOutputs->value, [&]() { return aOutputs->value->determinePos(noPos); });
+    state.forceValue(*aOutputs->value, aOutputs->value->determinePos(noPos));
 
     return aOutputs->value;
 }

src/libcmd/installable-value.cc

@@ -1,5 +1,6 @@
 #include "installable-value.hh"
 #include "eval-cache.hh"
+#include "fetch-to-store.hh"
 
 namespace nix {
 
@@ -44,7 +45,7 @@ ref<InstallableValue> InstallableValue::require(ref<Installable> installable)
 std::optional<DerivedPathWithInfo> InstallableValue::trySinglePathToDerivedPaths(Value & v, const PosIdx pos, std::string_view errorCtx)
 {
     if (v.type() == nPath) {
-        auto storePath = v.path().fetchToStore(state->store);
+        auto storePath = fetchToStore(*state->store, v.path());
         return {{
             .path = DerivedPath::Opaque {
                 .path = std::move(storePath),

src/libcmd/installables.cc

@@ -669,12 +669,12 @@ BuiltPaths Installable::toBuiltPaths(
 
         BuiltPaths res;
         for (auto & drvPath : Installable::toDerivations(store, installables, true))
-            res.push_back(BuiltPath::Opaque{drvPath});
+            res.emplace_back(BuiltPath::Opaque{drvPath});
         return res;
     }
 }
 
-StorePathSet Installable::toStorePaths(
+StorePathSet Installable::toStorePathSet(
     ref<Store> evalStore,
     ref<Store> store,
     Realise mode, OperateOn operateOn,
@@ -688,13 +688,27 @@ StorePathSet Installable::toStorePaths(
     return outPaths;
 }
 
+StorePaths Installable::toStorePaths(
+    ref<Store> evalStore,
+    ref<Store> store,
+    Realise mode, OperateOn operateOn,
+    const Installables & installables)
+{
+    StorePaths outPaths;
+    for (auto & path : toBuiltPaths(evalStore, store, mode, operateOn, installables)) {
+        auto thisOutPaths = path.outPaths();
+        outPaths.insert(outPaths.end(), thisOutPaths.begin(), thisOutPaths.end());
+    }
+    return outPaths;
+}
+
 StorePath Installable::toStorePath(
     ref<Store> evalStore,
     ref<Store> store,
     Realise mode, OperateOn operateOn,
     ref<Installable> installable)
 {
-    auto paths = toStorePaths(evalStore, store, mode, operateOn, {installable});
+    auto paths = toStorePathSet(evalStore, store, mode, operateOn, {installable});
 
     if (paths.size() != 1)
         throw Error("argument '%s' should evaluate to one store path", installable->what());

src/libcmd/installables.hh

@@ -166,7 +166,14 @@ struct Installable
         const Installables & installables,
         BuildMode bMode = bmNormal);
 
-    static std::set<StorePath> toStorePaths(
+    static std::set<StorePath> toStorePathSet(
+        ref<Store> evalStore,
+        ref<Store> store,
+        Realise mode,
+        OperateOn operateOn,
+        const Installables & installables);
+
+    static std::vector<StorePath> toStorePaths(
         ref<Store> evalStore,
         ref<Store> store,
         Realise mode,

src/libcmd/local.mk

@@ -12,4 +12,4 @@ libcmd_LDFLAGS = $(EDITLINE_LIBS) $(LOWDOWN_LIBS) -pthread
 
 libcmd_LIBS = libstore libutil libexpr libmain libfetchers
 
-$(eval $(call install-file-in, $(d)/nix-cmd.pc, $(libdir)/pkgconfig, 0644))
+$(eval $(call install-file-in, $(buildprefix)$(d)/nix-cmd.pc, $(libdir)/pkgconfig, 0644))

src/libcmd/meson.build

@@ -0,0 +1,35 @@
+libcmd_sources = files(
+  'built-path.cc',
+  'command-installable-value.cc',
+  'command.cc',
+  'common-eval-args.cc',
+  'editor-for.cc',
+  'installable-attr-path.cc',
+  'installable-derived-path.cc',
+  'installable-flake.cc',
+  'installable-value.cc',
+  'installables.cc',
+  'legacy.cc',
+  'markdown.cc',
+  'repl.cc',
+)
+
+libcmd = library(
+  'nixcmd',
+  libcmd_sources,
+  dependencies : [
+    liblixutil,
+    liblixstore,
+    liblixexpr,
+    liblixfetchers,
+    liblixmain,
+    boehm,
+    editline,
+    lowdown,
+  ],
+)
+
+liblixcmd = declare_dependency(
+  include_directories : '.',
+  link_with : libcmd,
+)

src/libcmd/repl.cc

@@ -40,7 +40,6 @@ extern "C" {
 #include "finally.hh"
 #include "markdown.hh"
 #include "local-fs-store.hh"
-#include "progress-bar.hh"
 #include "print.hh"
 
 #if HAVE_BOEHMGC
@@ -91,9 +90,17 @@ struct NixRepl
     void evalString(std::string s, Value & v);
     void loadDebugTraceEnv(DebugTrace & dt);
 
-    typedef std::set<Value *> ValuesSeen;
-    std::ostream & printValue(std::ostream & str, Value & v, unsigned int maxDepth);
-    std::ostream & printValue(std::ostream & str, Value & v, unsigned int maxDepth, ValuesSeen & seen);
+    void printValue(std::ostream & str,
+                              Value & v,
+                              unsigned int maxDepth = std::numeric_limits<unsigned int>::max())
+    {
+        ::nix::printValue(*state, str, v, PrintOptions {
+            .ansiColors = true,
+            .force = true,
+            .derivationPaths = true,
+            .maxDepth = maxDepth
+        });
+    }
 };
 
 std::string removeWhitespace(std::string s)
@@ -110,7 +117,7 @@ NixRepl::NixRepl(const SearchPath & searchPath, nix::ref<Store> store, ref<EvalS
     : AbstractNixRepl(state)
     , debugTraceIndex(0)
     , getValues(getValues)
-    , staticEnv(new StaticEnv(false, state->staticBaseEnv.get()))
+    , staticEnv(new StaticEnv(nullptr, state->staticBaseEnv.get()))
     , historyFile(getDataDir() + "/nix/repl-history")
 {
 }
@@ -219,10 +226,10 @@ static std::ostream & showDebugTrace(std::ostream & out, const PosTable & positi
     // prefer direct pos, but if noPos then try the expr.
     auto pos = dt.pos
         ? dt.pos
-        : static_cast<std::shared_ptr<AbstractPos>>(positions[dt.expr.getPos() ? dt.expr.getPos() : noPos]);
+        : positions[dt.expr.getPos() ? dt.expr.getPos() : noPos];
 
     if (pos) {
-        out << pos;
+        out << *pos;
         if (auto loc = pos->getCodeLines()) {
             out << "\n";
             printCodeLines(out, "", *pos, *loc);
@@ -233,10 +240,19 @@ static std::ostream & showDebugTrace(std::ostream & out, const PosTable & positi
     return out;
 }
 
+static bool isFirstRepl = true;
+
 void NixRepl::mainLoop()
 {
-    std::string error = ANSI_RED "error:" ANSI_NORMAL " ";
-    notice("Welcome to Nix " + nixVersion + ". Type :? for help.\n");
+    if (isFirstRepl) {
+        std::string_view debuggerNotice = "";
+        if (state->debugRepl) {
+            debuggerNotice = " debugger";
+        }
+        notice("Nix %1%%2%\nType :? for help.", nixVersion, debuggerNotice);
+    }
+
+    isFirstRepl = false;
 
     loadFiles();
 
@@ -259,13 +275,11 @@ void NixRepl::mainLoop()
     rl_set_list_possib_func(listPossibleCallback);
 #endif
 
-    /* Stop the progress bar because it interferes with the display of
-       the repl. */
-    stopProgressBar();
-
     std::string input;
 
     while (true) {
+        // Hide the progress bar while waiting for user input, so that it won't interfere.
+        logger->pause();
         // When continuing input from previous lines, don't print a prompt, just align to the same
         // number of chars as the prompt.
         if (!getLine(input, input.empty() ? "nix-repl> " : "          ")) {
@@ -275,6 +289,7 @@ void NixRepl::mainLoop()
             logger->cout("");
             break;
         }
+        logger->resume();
         try {
             if (!removeWhitespace(input).empty() && !processLine(input)) return;
         } catch (ParseError & e) {
@@ -333,7 +348,6 @@ bool NixRepl::getLine(std::string & input, const std::string & prompt)
     };
 
     setupSignals();
-    Finally resetTerminal([&]() { rl_deprep_terminal(); });
     char * s = readline(prompt.c_str());
     Finally doFree([&]() { free(s); });
     restoreSignals();
@@ -413,8 +427,6 @@ StringSet NixRepl::completePrefix(const std::string & prefix)
             // Quietly ignore parse errors.
         } catch (EvalError & e) {
             // Quietly ignore evaluation errors.
-        } catch (UndefinedVarError & e) {
-            // Quietly ignore undefined variable errors.
         } catch (BadURL & e) {
             // Quietly ignore BadURL flake-related errors.
         }
@@ -707,7 +719,8 @@ bool NixRepl::processLine(std::string line)
     else if (command == ":p" || command == ":print") {
         Value v;
         evalString(arg, v);
-        printValue(std::cout, v, 1000000000) << std::endl;
+        printValue(std::cout, v);
+        std::cout << std::endl;
     }
 
     else if (command == ":q" || command == ":quit") {
@@ -769,7 +782,8 @@ bool NixRepl::processLine(std::string line)
         } else {
             Value v;
             evalString(line, v);
-            printValue(std::cout, v, 1) << std::endl;
+            printValue(std::cout, v, 1);
+            std::cout << std::endl;
         }
     }
 
@@ -887,145 +901,7 @@ void NixRepl::evalString(std::string s, Value & v)
 {
     Expr * e = parseString(s);
     e->eval(*state, *env, v);
-    state->forceValue(v, [&]() { return v.determinePos(noPos); });
-}
-
-
-std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int maxDepth)
-{
-    ValuesSeen seen;
-    return printValue(str, v, maxDepth, seen);
-}
-
-
-
-
-// FIXME: lot of cut&paste from Nix's eval.cc.
-std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int maxDepth, ValuesSeen & seen)
-{
-    str.flush();
-    checkInterrupt();
-
-    state->forceValue(v, [&]() { return v.determinePos(noPos); });
-
-    switch (v.type()) {
-
-    case nInt:
-        str << ANSI_CYAN << v.integer << ANSI_NORMAL;
-        break;
-
-    case nBool:
-        str << ANSI_CYAN;
-        printLiteralBool(str, v.boolean);
-        str << ANSI_NORMAL;
-        break;
-
-    case nString:
-        str << ANSI_WARNING;
-        printLiteralString(str, v.string.s);
-        str << ANSI_NORMAL;
-        break;
-
-    case nPath:
-        str << ANSI_GREEN << v.path().to_string() << ANSI_NORMAL; // !!! escaping?
-        break;
-
-    case nNull:
-        str << ANSI_CYAN "null" ANSI_NORMAL;
-        break;
-
-    case nAttrs: {
-        seen.insert(&v);
-
-        bool isDrv = state->isDerivation(v);
-
-        if (isDrv) {
-            str << "«derivation ";
-            Bindings::iterator i = v.attrs->find(state->sDrvPath);
-            NixStringContext context;
-            if (i != v.attrs->end())
-                str << state->store->printStorePath(state->coerceToStorePath(i->pos, *i->value, context, "while evaluating the drvPath of a derivation"));
-            else
-                str << "???";
-            str << "»";
-        }
-
-        else if (maxDepth > 0) {
-            str << "{ ";
-
-            typedef std::map<std::string, Value *> Sorted;
-            Sorted sorted;
-            for (auto & i : *v.attrs)
-                sorted.emplace(state->symbols[i.name], i.value);
-
-            for (auto & i : sorted) {
-                printAttributeName(str, i.first);
-                str << " = ";
-                if (seen.count(i.second))
-                    str << "«repeated»";
-                else
-                    try {
-                        printValue(str, *i.second, maxDepth - 1, seen);
-                    } catch (AssertionError & e) {
-                        str << ANSI_RED "«error: " << e.msg() << "»" ANSI_NORMAL;
-                    }
-                str << "; ";
-            }
-
-            str << "}";
-        } else
-            str << "{ ... }";
-
-        break;
-    }
-
-    case nList:
-        seen.insert(&v);
-
-        str << "[ ";
-        if (maxDepth > 0)
-            for (auto elem : v.listItems()) {
-                if (seen.count(elem))
-                    str << "«repeated»";
-                else
-                    try {
-                        printValue(str, *elem, maxDepth - 1, seen);
-                    } catch (AssertionError & e) {
-                        str << ANSI_RED "«error: " << e.msg() << "»" ANSI_NORMAL;
-                    }
-                str << " ";
-            }
-        else
-            str << "... ";
-        str << "]";
-        break;
-
-    case nFunction:
-        if (v.isLambda()) {
-            std::ostringstream s;
-            s << state->positions[v.lambda.fun->pos];
-            str << ANSI_BLUE "«lambda @ " << filterANSIEscapes(s.str()) << "»" ANSI_NORMAL;
-        } else if (v.isPrimOp()) {
-            str << ANSI_MAGENTA "«primop»" ANSI_NORMAL;
-        } else if (v.isPrimOpApp()) {
-            str << ANSI_BLUE "«primop-app»" ANSI_NORMAL;
-        } else {
-            abort();
-        }
-        break;
-
-    case nFloat:
-        str << v.fpoint;
-        break;
-
-    case nThunk:
-    case nExternal:
-    default:
-        str << ANSI_RED "«unknown»" ANSI_NORMAL;
-        break;
-    }
-
-    return str;
+    state->forceValue(v, v.determinePos(noPos));
 }
 
 

src/libexpr/attr-path.cc

@@ -66,10 +66,10 @@ std::pair<Value *, PosIdx> findAlongAttrPath(EvalState & state, const std::strin
         if (!attrIndex) {
 
             if (v->type() != nAttrs)
-                throw TypeError(
+                state.error<TypeError>(
                     "the expression selected by the selection path '%1%' should be a set but is %2%",
                     attrPath,
-                    showType(*v));
+                    showType(*v)).debugThrow();
             if (attr.empty())
                 throw Error("empty attribute name in selection path '%1%'", attrPath);
 
@@ -89,10 +89,10 @@ std::pair<Value *, PosIdx> findAlongAttrPath(EvalState & state, const std::strin
         else {
 
             if (!v->isList())
-                throw TypeError(
+                state.error<TypeError>(
                     "the expression selected by the selection path '%1%' should be a list but is %2%",
                     attrPath,
-                    showType(*v));
+                    showType(*v)).debugThrow();
             if (*attrIndex >= v->listSize())
                 throw AttrPathNotFound("list index %1% in selection path '%2%' is out of range", *attrIndex, attrPath);
 

src/libexpr/c/meson.build

@@ -0,0 +1,31 @@
+pkg = import('pkgconfig')
+
+libexprc_sources = files(
+  'nix_api_expr.cc',
+  'nix_api_external.cc',
+  'nix_api_value.cc',
+)
+
+all_sources += {
+  'libexprc': libexprc_sources
+}
+
+libexprc = library(
+  'nix-expr-c',
+  libexprc_sources,
+  dependencies : [
+    liblixutil,
+    liblixstore,
+    liblixexpr,
+    liblixutilc,
+    liblixstorec,
+    boehm,
+  ],
+)
+
+pkg.generate(libexprc, requires: [ 'nix-store-c' ])
+
+liblixexprc = declare_dependency(
+  include_directories : include_directories('.'),
+  link_with : libexprc,
+)

src/libexpr/c/nix_api_expr.cc

@@ -0,0 +1,178 @@
+#include <cstring>
+#include <iostream>
+#include <stdexcept>
+#include <string>
+
+#include "config.hh"
+#include "eval.hh"
+#include "globals.hh"
+#include "util.hh"
+
+#include "nix_api_expr.h"
+#include "nix_api_expr_internal.h"
+#include "nix_api_store.h"
+#include "nix_api_store_internal.h"
+#include "nix_api_util.h"
+#include "nix_api_util_internal.h"
+
+#ifdef HAVE_BOEHMGC
+#include <mutex>
+#define GC_INCLUDE_NEW 1
+#include "gc_cpp.h"
+#endif
+
+nix_err nix_libexpr_init(nix_c_context * context)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    {
+        auto ret = nix_libutil_init(context);
+        if (ret != NIX_OK)
+            return ret;
+    }
+    {
+        auto ret = nix_libstore_init(context);
+        if (ret != NIX_OK)
+            return ret;
+    }
+    try {
+        nix::initGC();
+    }
+    NIXC_CATCH_ERRS
+}
+
+nix_err nix_expr_eval_from_string(
+    nix_c_context * context, EvalState * state, const char * expr, const char * path, Value * value)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        nix::Expr * parsedExpr = state->state.parseExprFromString(expr, state->state.rootPath(nix::CanonPath(path)));
+        state->state.eval(parsedExpr, *(nix::Value *) value);
+        state->state.forceValue(*(nix::Value *) value, nix::noPos);
+    }
+    NIXC_CATCH_ERRS
+}
+
+nix_err nix_value_call(nix_c_context * context, EvalState * state, Value * fn, Value * arg, Value * value)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        state->state.callFunction(*(nix::Value *) fn, *(nix::Value *) arg, *(nix::Value *) value, nix::noPos);
+        state->state.forceValue(*(nix::Value *) value, nix::noPos);
+    }
+    NIXC_CATCH_ERRS
+}
+
+nix_err nix_value_force(nix_c_context * context, EvalState * state, Value * value)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        state->state.forceValue(*(nix::Value *) value, nix::noPos);
+    }
+    NIXC_CATCH_ERRS
+}
+
+nix_err nix_value_force_deep(nix_c_context * context, EvalState * state, Value * value)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        state->state.forceValueDeep(*(nix::Value *) value);
+    }
+    NIXC_CATCH_ERRS
+}
+
+EvalState * nix_state_create(nix_c_context * context, const char ** searchPath_c, Store * store)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        nix::Strings searchPath;
+        if (searchPath_c != nullptr)
+            for (size_t i = 0; searchPath_c[i] != nullptr; i++)
+                searchPath.push_back(searchPath_c[i]);
+
+        return new EvalState{nix::EvalState(nix::SearchPath::parse(searchPath), store->ptr)};
+    }
+    NIXC_CATCH_ERRS_NULL
+}
+
+void nix_state_free(EvalState * state)
+{
+    delete state;
+}
+
+#ifdef HAVE_BOEHMGC
+std::unordered_map<
+    const void *,
+    unsigned int,
+    std::hash<const void *>,
+    std::equal_to<const void *>,
+    traceable_allocator<std::pair<const void * const, unsigned int>>>
+    nix_refcounts;
+
+std::mutex nix_refcount_lock;
+
+nix_err nix_gc_incref(nix_c_context * context, const void * p)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        std::scoped_lock lock(nix_refcount_lock);
+        auto f = nix_refcounts.find(p);
+        if (f != nix_refcounts.end()) {
+            f->second++;
+        } else {
+            nix_refcounts[p] = 1;
+        }
+    }
+    NIXC_CATCH_ERRS
+}
+
+nix_err nix_gc_decref(nix_c_context * context, const void * p)
+{
+
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        std::scoped_lock lock(nix_refcount_lock);
+        auto f = nix_refcounts.find(p);
+        if (f != nix_refcounts.end()) {
+            if (--f->second == 0)
+                nix_refcounts.erase(f);
+        } else
+            throw std::runtime_error("nix_gc_decref: object was not referenced");
+    }
+    NIXC_CATCH_ERRS
+}
+
+void nix_gc_now()
+{
+    GC_gcollect();
+}
+
+#else
+void nix_gc_incref(nix_c_context * context, const void *)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    return NIX_OK;
+}
+void nix_gc_decref(nix_c_context * context, const void *)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    return NIX_OK;
+}
+void nix_gc_now() {}
+#endif
+
+void nix_gc_register_finalizer(void * obj, void * cd, void (*finalizer)(void * obj, void * cd))
+{
+#ifdef HAVE_BOEHMGC
+    GC_REGISTER_FINALIZER(obj, finalizer, cd, 0, 0);
+#endif
+}

src/libexpr/c/nix_api_expr.h

@@ -0,0 +1,213 @@
+#ifndef NIX_API_EXPR_H
+#define NIX_API_EXPR_H
+/** @defgroup libexpr libexpr
+ * @brief Bindings to the Nix language evaluator
+ *
+ * Example (without error handling):
+ * @code{.c}
+ * int main() {
+ *    nix_libexpr_init(NULL);
+ *
+ *    Store* store = nix_store_open(NULL, "dummy", NULL);
+ *    EvalState* state = nix_state_create(NULL, NULL, store); // empty nix path
+ *    Value *value = nix_alloc_value(NULL, state);
+ *
+ *    nix_expr_eval_from_string(NULL, state, "builtins.nixVersion", ".", value);
+ *    nix_value_force(NULL, state, value);
+ *    printf("nix version: %s\n", nix_get_string(NULL, value));
+ *
+ *    nix_gc_decref(NULL, value);
+ *    nix_state_free(state);
+ *    nix_store_free(store);
+ *    return 0;
+ *    }
+ * @endcode
+ * @{
+ */
+/** @file
+ * @brief Main entry for the libexpr C bindings
+ */
+
+#include "nix_api_store.h"
+#include "nix_api_util.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+// cffi start
+
+// Type definitions
+/**
+ * @brief Represents a state of the Nix language evaluator.
+ *
+ * Multiple states can be created for multi-threaded
+ * operation.
+ * @struct EvalState
+ * @see nix_state_create
+ */
+typedef struct EvalState EvalState; // nix::EvalState
+/**
+ * @brief Represents a value in the Nix language.
+ *
+ * Owned by the garbage collector.
+ * @struct Value
+ * @see value_manip
+ */
+typedef void Value; // nix::Value
+
+// Function prototypes
+/**
+ * @brief Initialize the Nix language evaluator.
+ *
+ * This function must be called at least once,
+ * at some point before constructing a EvalState for the first time.
+ * This function can be called multiple times, and is idempotent.
+ *
+ * @param[out] context Optional, stores error information
+ * @return NIX_OK if the initialization was successful, an error code otherwise.
+ */
+nix_err nix_libexpr_init(nix_c_context * context);
+
+/**
+ * @brief Parses and evaluates a Nix expression from a string.
+ *
+ * @param[out] context Optional, stores error information
+ * @param[in] state The state of the evaluation.
+ * @param[in] expr The Nix expression to parse.
+ * @param[in] path The file path to associate with the expression.
+ * This is required for expressions that contain relative paths (such as `./.`) that are resolved relative to the given
+ * directory.
+ * @param[out] value The result of the evaluation. You must allocate this
+ * yourself.
+ * @return NIX_OK if the evaluation was successful, an error code otherwise.
+ */
+nix_err nix_expr_eval_from_string(
+    nix_c_context * context, EvalState * state, const char * expr, const char * path, Value * value);
+
+/**
+ * @brief Calls a Nix function with an argument.
+ *
+ * @param[out] context Optional, stores error information
+ * @param[in] state The state of the evaluation.
+ * @param[in] fn The Nix function to call.
+ * @param[in] arg The argument to pass to the function.
+ * @param[out] value The result of the function call.
+ * @return NIX_OK if the function call was successful, an error code otherwise.
+ */
+nix_err nix_value_call(nix_c_context * context, EvalState * state, Value * fn, Value * arg, Value * value);
+
+/**
+ * @brief Forces the evaluation of a Nix value.
+ *
+ * The Nix interpreter is lazy, and not-yet-evaluated Values can be
+ * of type NIX_TYPE_THUNK instead of their actual value.
+ *
+ * This function converts these Values into their final type.
+ *
+ * @note You don't need this function for basic API usage, since all functions
+ * that return a value call it for you. The only place you will see a
+ * NIX_TYPE_THUNK is in the arguments that are passed to a PrimOp function
+ * you supplied to nix_alloc_primop.
+ *
+ * @param[out] context Optional, stores error information
+ * @param[in] state The state of the evaluation.
+ * @param[in,out] value The Nix value to force.
+ * @post value is not of type NIX_TYPE_THUNK
+ * @return NIX_OK if the force operation was successful, an error code
+ * otherwise.
+ */
+nix_err nix_value_force(nix_c_context * context, EvalState * state, Value * value);
+
+/**
+ * @brief Forces the deep evaluation of a Nix value.
+ *
+ * Recursively calls nix_value_force
+ *
+ * @see nix_value_force
+ * @warning Calling this function on a recursive data structure will cause a
+ * stack overflow.
+ * @param[out] context Optional, stores error information
+ * @param[in] state The state of the evaluation.
+ * @param[in,out] value The Nix value to force.
+ * @return NIX_OK if the deep force operation was successful, an error code
+ * otherwise.
+ */
+nix_err nix_value_force_deep(nix_c_context * context, EvalState * state, Value * value);
+
+/**
+ * @brief Create a new Nix language evaluator state.
+ *
+ * @param[out] context Optional, stores error information
+ * @param[in] searchPath Array of strings corresponding to entries in NIX_PATH.
+ * @param[in] store The Nix store to use.
+ * @return A new Nix state or NULL on failure.
+ */
+EvalState * nix_state_create(nix_c_context * context, const char ** searchPath, Store * store);
+
+/**
+ * @brief Frees a Nix state.
+ *
+ * Does not fail.
+ *
+ * @param[in] state The state to free.
+ */
+void nix_state_free(EvalState * state);
+
+/** @addtogroup GC
+ * @brief Reference counting and garbage collector operations
+ *
+ * The Nix language evaluator uses a garbage collector. To ease C interop, we implement
+ * a reference counting scheme, where objects will be deallocated
+ * when there are no references from the Nix side, and the reference count kept
+ * by the C API reaches `0`.
+ *
+ * Functions returning a garbage-collected object will automatically increase
+ * the refcount for you. You should make sure to call `nix_gc_decref` when
+ * you're done with a value returned by the evaluator.
+ * @{
+ */
+/**
+ * @brief Increment the garbage collector reference counter for the given object.
+ *
+ * The Nix language evaluator C API keeps track of alive objects by reference counting.
+ * When you're done with a refcounted pointer, call nix_gc_decref().
+ *
+ * @param[out] context Optional, stores error information
+ * @param[in] object The object to keep alive
+ */
+nix_err nix_gc_incref(nix_c_context * context, const void * object);
+/**
+ * @brief Decrement the garbage collector reference counter for the given object
+ *
+ * @param[out] context Optional, stores error information
+ * @param[in] object The object to stop referencing
+ */
+nix_err nix_gc_decref(nix_c_context * context, const void * object);
+
+/**
+ * @brief Trigger the garbage collector manually
+ *
+ * You should not need to do this, but it can be useful for debugging.
+ */
+void nix_gc_now();
+
+/**
+ * @brief Register a callback that gets called when the object is garbage
+ * collected.
+ * @note Objects can only have a single finalizer. This function overwrites existing values
+ * silently.
+ * @param[in] obj the object to watch
+ * @param[in] cd the data to pass to the finalizer
+ * @param[in] finalizer the callback function, called with obj and cd
+ */
+void nix_gc_register_finalizer(void * obj, void * cd, void (*finalizer)(void * obj, void * cd));
+
+/** @} */
+// cffi end
+#ifdef __cplusplus
+}
+#endif
+
+/** @} */
+
+#endif // NIX_API_EXPR_H

src/libexpr/c/nix_api_expr_internal.h

@@ -0,0 +1,64 @@
+#ifndef NIX_API_EXPR_INTERNAL_H
+#define NIX_API_EXPR_INTERNAL_H
+
+#include "eval.hh"
+#include "attr-set.hh"
+#include "nix_api_value.h"
+
+class CListBuilder
+{
+private:
+    std::vector<nix::Value *> values;
+
+public:
+    CListBuilder(size_t capacity)
+    {
+        values.reserve(capacity);
+    }
+
+    void push_back(nix::Value * value)
+    {
+        values.push_back(value);
+    }
+
+    Value * finish(nix::EvalState * state, nix::Value * list)
+    {
+        state->mkList(*list, values.size());
+        for (size_t n = 0; n < list->listSize(); ++n) {
+            list->listElems()[n] = values[n];
+        }
+        return list;
+    }
+};
+
+struct EvalState
+{
+    nix::EvalState state;
+};
+
+struct BindingsBuilder
+{
+    nix::BindingsBuilder builder;
+};
+
+struct ListBuilder
+{
+    CListBuilder builder;
+};
+
+struct nix_string_return
+{
+    std::string str;
+};
+
+struct nix_printer
+{
+    std::ostream & s;
+};
+
+struct nix_string_context
+{
+    nix::NixStringContext & ctx;
+};
+
+#endif // NIX_API_EXPR_INTERNAL_H

src/libexpr/c/nix_api_external.cc

@@ -0,0 +1,198 @@
+#include "attr-set.hh"
+#include "config.hh"
+#include "eval.hh"
+#include "gc/gc.h"
+#include "globals.hh"
+#include "value.hh"
+
+#include "nix_api_expr.h"
+#include "nix_api_expr_internal.h"
+#include "nix_api_external.h"
+#include "nix_api_util.h"
+#include "nix_api_util_internal.h"
+#include "nix_api_value.h"
+#include "value/context.hh"
+
+#include <nlohmann/json.hpp>
+
+#ifdef HAVE_BOEHMGC
+#define GC_INCLUDE_NEW 1
+#include "gc_cpp.h"
+#endif
+
+void nix_set_string_return(nix_string_return * str, const char * c)
+{
+    str->str = c;
+}
+
+nix_err nix_external_print(nix_c_context * context, nix_printer * printer, const char * c)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        printer->s << c;
+    }
+    NIXC_CATCH_ERRS
+}
+
+nix_err nix_external_add_string_context(nix_c_context * context, nix_string_context * ctx, const char * c)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto r = nix::NixStringContextElem::parse(c);
+        ctx->ctx.insert(r);
+    }
+    NIXC_CATCH_ERRS
+}
+
+class NixCExternalValue : public nix::ExternalValueBase
+{
+    NixCExternalValueDesc & desc;
+    void * v;
+
+public:
+    NixCExternalValue(NixCExternalValueDesc & desc, void * v)
+        : desc(desc)
+        , v(v){};
+    void * get_ptr()
+    {
+        return v;
+    }
+    /**
+     * Print out the value
+     */
+    virtual std::ostream & print(std::ostream & str) const override
+    {
+        nix_printer p{str};
+        desc.print(v, &p);
+        return str;
+    }
+
+    /**
+     * Return a simple string describing the type
+     */
+    virtual std::string showType() const override
+    {
+        nix_string_return res;
+        desc.showType(v, &res);
+        return std::move(res.str);
+    }
+
+    /**
+     * Return a string to be used in builtins.typeOf
+     */
+    virtual std::string typeOf() const override
+    {
+        nix_string_return res;
+        desc.typeOf(v, &res);
+        return std::move(res.str);
+    }
+
+    /**
+     * Coerce the value to a string.
+     */
+    virtual std::string coerceToString(
+        nix::EvalState & state,
+        const nix::PosIdx & pos,
+        nix::NixStringContext & context,
+        bool copyMore,
+        bool copyToStore) const override
+    {
+        if (!desc.coerceToString) {
+            return nix::ExternalValueBase::coerceToString(state, pos, context, copyMore, copyToStore);
+        }
+        nix_string_context ctx{context};
+        nix_string_return res{""};
+        // todo: pos, errors
+        desc.coerceToString(v, &ctx, copyMore, copyToStore, &res);
+        if (res.str.empty()) {
+            return nix::ExternalValueBase::coerceToString(state, pos, context, copyMore, copyToStore);
+        }
+        return std::move(res.str);
+    }
+
+    /**
+     * Compare to another value of the same type.
+     */
+    virtual bool operator==(const ExternalValueBase & b) const override
+    {
+        if (!desc.equal) {
+            return false;
+        }
+        auto r = dynamic_cast<const NixCExternalValue *>(&b);
+        if (!r)
+            return false;
+        return desc.equal(v, r->v);
+    }
+
+    /**
+     * Print the value as JSON.
+     */
+    virtual nlohmann::json printValueAsJSON(
+        nix::EvalState & state, bool strict, nix::NixStringContext & context, bool copyToStore = true) const override
+    {
+        if (!desc.printValueAsJSON) {
+            return nix::ExternalValueBase::printValueAsJSON(state, strict, context, copyToStore);
+        }
+        nix_string_context ctx{context};
+        nix_string_return res{""};
+        desc.printValueAsJSON(v, (EvalState *) &state, strict, &ctx, copyToStore, &res);
+        if (res.str.empty()) {
+            return nix::ExternalValueBase::printValueAsJSON(state, strict, context, copyToStore);
+        }
+        return nlohmann::json::parse(res.str);
+    }
+
+    /**
+     * Print the value as XML.
+     */
+    virtual void printValueAsXML(
+        nix::EvalState & state,
+        bool strict,
+        bool location,
+        nix::XMLWriter & doc,
+        nix::NixStringContext & context,
+        nix::PathSet & drvsSeen,
+        const nix::PosIdx pos) const override
+    {
+        if (!desc.printValueAsXML) {
+            return nix::ExternalValueBase::printValueAsXML(state, strict, location, doc, context, drvsSeen, pos);
+        }
+        nix_string_context ctx{context};
+        desc.printValueAsXML(
+            v, (EvalState *) &state, strict, location, &doc, &ctx, &drvsSeen,
+            *reinterpret_cast<const uint32_t *>(&pos));
+    }
+
+    virtual ~NixCExternalValue() override{};
+};
+
+ExternalValue * nix_create_external_value(nix_c_context * context, NixCExternalValueDesc * desc, void * v)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto ret = new
+#ifdef HAVE_BOEHMGC
+            (GC)
+#endif
+                NixCExternalValue(*desc, v);
+        nix_gc_incref(nullptr, ret);
+        return (ExternalValue *) ret;
+    }
+    NIXC_CATCH_ERRS_NULL
+}
+
+void * nix_get_external_value_content(nix_c_context * context, ExternalValue * b)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto r = dynamic_cast<NixCExternalValue *>((nix::ExternalValueBase *) b);
+        if (r)
+            return r->get_ptr();
+        return nullptr;
+    }
+    NIXC_CATCH_ERRS_NULL
+}

src/libexpr/c/nix_api_external.h

@@ -0,0 +1,196 @@
+#ifndef NIX_API_EXTERNAL_H
+#define NIX_API_EXTERNAL_H
+/** @ingroup libexpr
+ * @addtogroup Externals
+ * @brief Deal with external values
+ * @{
+ */
+/** @file
+ * @brief libexpr C bindings dealing with external values
+ */
+
+#include "nix_api_expr.h"
+#include "nix_api_util.h"
+#include "nix_api_value.h"
+#include "stdbool.h"
+#include "stddef.h"
+#include "stdint.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+// cffi start
+
+/**
+ * @brief Represents a string owned by the Nix language evaluator.
+ * @see nix_set_owned_string
+ */
+typedef struct nix_string_return nix_string_return;
+/**
+ * @brief Wraps a stream that can output multiple string pieces.
+ */
+typedef struct nix_printer nix_printer;
+/**
+ * @brief A list of string context items
+ */
+typedef struct nix_string_context nix_string_context;
+
+/**
+ * @brief Sets the contents of a nix_string_return
+ *
+ * Copies the passed string.
+ * @param[out] str the nix_string_return to write to
+ * @param[in]  c   The string to copy
+ */
+void nix_set_string_return(nix_string_return * str, const char * c);
+
+/**
+ * Print to the nix_printer
+ *
+ * @param[out] context Optional, stores error information
+ * @param printer The nix_printer to print to
+ * @param[in] str The string to print
+ * @returns NIX_OK if everything worked
+ */
+nix_err nix_external_print(nix_c_context * context, nix_printer * printer, const char * str);
+
+/**
+ * Add string context to the nix_string_context object
+ * @param[out] context Optional, stores error information
+ * @param[out] string_context The nix_string_context to add to
+ * @param[in] c The context string to add
+ * @returns NIX_OK if everything worked
+ */
+nix_err nix_external_add_string_context(nix_c_context * context, nix_string_context * string_context, const char * c);
+
+/**
+ * @brief Definition for a class of external values
+ *
+ * Create and implement one of these, then pass it to nix_create_external_value
+ * Make sure to keep it alive while the external value lives.
+ *
+ * Optional functions can be set to NULL
+ *
+ * @see nix_create_external_value
+ */
+typedef struct NixCExternalValueDesc
+{
+    /**
+     * @brief Called when printing the external value
+     *
+     * @param[in] self the void* passed to nix_create_external_value
+     * @param[out] printer The printer to print to, pass to nix_external_print
+     */
+    void (*print)(void * self, nix_printer * printer);
+    /**
+     * @brief Called on :t
+     * @param[in] self the void* passed to nix_create_external_value
+     * @param[out] res the return value
+     */
+    void (*showType)(void * self, nix_string_return * res);
+    /**
+     * @brief Called on `builtins.typeOf`
+     * @param self the void* passed to nix_create_external_value
+     * @param[out] res the return value
+     */
+    void (*typeOf)(void * self, nix_string_return * res);
+    /**
+     * @brief Called on "${str}" and builtins.toString.
+     *
+     * The latter with coerceMore=true
+     * Optional, the default is to throw an error.
+     * @param[in] self the void* passed to nix_create_external_value
+     * @param[out] c writable string context for the resulting string
+     * @param[in] coerceMore boolean, try to coerce to strings in more cases
+     * instead of throwing an error
+     * @param[in] copyToStore boolean, whether to copy referenced paths to store
+     * or keep them as-is
+     * @param[out] res the return value. Not touching this, or setting it to the
+     * empty string, will make the conversion throw an error.
+     */
+    void (*coerceToString)(
+        void * self, nix_string_context * c, int coerceMore, int copyToStore, nix_string_return * res);
+    /**
+     * @brief Try to compare two external values
+     *
+     * Optional, the default is always false.
+     * If the other object was not a Nix C external value, this comparison will
+     * also return false
+     * @param[in] self the void* passed to nix_create_external_value
+     * @param[in] other the void* passed to the other object's
+     * nix_create_external_value
+     * @returns true if the objects are deemed to be equal
+     */
+    int (*equal)(void * self, void * other);
+    /**
+     * @brief Convert the external value to json
+     *
+     * Optional, the default is to throw an error
+     * @param[in] self the void* passed to nix_create_external_value
+     * @param[in] state The evaluator state
+     * @param[in] strict boolean Whether to force the value before printing
+     * @param[out] c writable string context for the resulting string
+     * @param[in] copyToStore whether to copy referenced paths to store or keep
+     * them as-is
+     * @param[out] res the return value. Gets parsed as JSON. Not touching this,
+     * or setting it to the empty string, will make the conversion throw an error.
+     */
+    void (*printValueAsJSON)(
+        void * self, EvalState *, bool strict, nix_string_context * c, bool copyToStore, nix_string_return * res);
+    /**
+     * @brief Convert the external value to XML
+     *
+     * Optional, the default is to throw an error
+     * @todo The mechanisms for this call are incomplete. There are no C
+     *       bindings to work with XML, pathsets and positions.
+     * @param[in] self the void* passed to nix_create_external_value
+     * @param[in] state The evaluator state
+     * @param[in] strict boolean Whether to force the value before printing
+     * @param[in] location boolean Whether to include position information in the
+     * xml
+     * @param[out] doc XML document to output to
+     * @param[out] c writable string context for the resulting string
+     * @param[in,out] drvsSeen a path set to avoid duplicating derivations
+     * @param[in] pos The position of the call.
+     */
+    void (*printValueAsXML)(
+        void * self,
+        EvalState *,
+        int strict,
+        int location,
+        void * doc,
+        nix_string_context * c,
+        void * drvsSeen,
+        int pos);
+} NixCExternalValueDesc;
+
+/**
+ * @brief Create an external value, that can be given to nix_init_external
+ *
+ * Owned by the GC. Use nix_gc_decref when you're done with the pointer.
+ *
+ * @param[out] context Optional, stores error information
+ * @param[in] desc a NixCExternalValueDesc, you should keep this alive as long
+ * as the ExternalValue lives
+ * @param[in] v the value to store
+ * @returns external value, owned by the garbage collector
+ * @see nix_init_external
+ */
+ExternalValue * nix_create_external_value(nix_c_context * context, NixCExternalValueDesc * desc, void * v);
+
+/**
+ * @brief Extract the pointer from a nix c external value.
+ * @param[out] context Optional, stores error information
+ * @param[in] b The external value
+ * @returns The pointer, or null if the external value was not from nix c.
+ * @see nix_get_external
+ */
+void * nix_get_external_value_content(nix_c_context * context, ExternalValue * b);
+
+// cffi end
+#ifdef __cplusplus
+}
+#endif
+/** @} */
+
+#endif // NIX_API_EXTERNAL_H

src/libexpr/c/nix_api_value.cc

@@ -0,0 +1,521 @@
+#include "attr-set.hh"
+#include "config.hh"
+#include "eval.hh"
+#include "gc/gc.h"
+#include "globals.hh"
+#include "primops.hh"
+#include "value.hh"
+
+#include "nix_api_expr.h"
+#include "nix_api_expr_internal.h"
+#include "nix_api_util.h"
+#include "nix_api_util_internal.h"
+#include "nix_api_value.h"
+
+#ifdef HAVE_BOEHMGC
+#define GC_INCLUDE_NEW 1
+#include "gc_cpp.h"
+#endif
+
+// Helper function to throw an exception if value is null
+static const nix::Value & check_value_not_null(const Value * value)
+{
+    if (!value) {
+        throw std::runtime_error("Value is null");
+    }
+    return *((const nix::Value *) value);
+}
+
+static nix::Value & check_value_not_null(Value * value)
+{
+    if (!value) {
+        throw std::runtime_error("Value is null");
+    }
+    return *((nix::Value *) value);
+}
+
+/**
+ * Helper function to convert calls from nix into C API.
+ *
+ * Deals with errors and converts arguments from C++ into C types.
+ */
+static void nix_c_primop_wrapper(
+    PrimOpFun f, void * userdata, nix::EvalState & state, const nix::PosIdx pos, nix::Value ** args, nix::Value & v)
+{
+    nix_c_context ctx;
+    f(userdata, &ctx, (EvalState *) &state, (Value **) args, (Value *) &v);
+    /* TODO: In the future, this should throw different errors depending on the error code */
+    if (ctx.last_err_code != NIX_OK)
+        state.error<nix::EvalError>("Error from builtin function: %s", *ctx.last_err).atPos(pos).debugThrow();
+}
+
+PrimOp * nix_alloc_primop(
+    nix_c_context * context,
+    PrimOpFun fun,
+    int arity,
+    const char * name,
+    const char ** args,
+    const char * doc,
+    void * user_data)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        using namespace std::placeholders;
+        auto p = new
+#ifdef HAVE_BOEHMGC
+            (GC)
+#endif
+                nix::PrimOp{
+                    .name = name,
+                    .args = {},
+                    .arity = (size_t) arity,
+                    .doc = doc,
+                    .fun = std::bind(nix_c_primop_wrapper, fun, user_data, _1, _2, _3, _4)};
+        if (args)
+            for (size_t i = 0; args[i]; i++)
+                p->args.emplace_back(*args);
+        nix_gc_incref(nullptr, p);
+        return (PrimOp *) p;
+    }
+    NIXC_CATCH_ERRS_NULL
+}
+
+nix_err nix_register_primop(nix_c_context * context, PrimOp * primOp)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        nix::RegisterPrimOp r(std::move(*((nix::PrimOp *) primOp)));
+    }
+    NIXC_CATCH_ERRS
+}
+
+Value * nix_alloc_value(nix_c_context * context, EvalState * state)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        Value * res = state->state.allocValue();
+        nix_gc_incref(nullptr, res);
+        return res;
+    }
+    NIXC_CATCH_ERRS_NULL
+}
+
+ValueType nix_get_type(nix_c_context * context, const Value * value)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        using namespace nix;
+        switch (v.type()) {
+        case nThunk:
+            return NIX_TYPE_THUNK;
+        case nInt:
+            return NIX_TYPE_INT;
+        case nFloat:
+            return NIX_TYPE_FLOAT;
+        case nBool:
+            return NIX_TYPE_BOOL;
+        case nString:
+            return NIX_TYPE_STRING;
+        case nPath:
+            return NIX_TYPE_PATH;
+        case nNull:
+            return NIX_TYPE_NULL;
+        case nAttrs:
+            return NIX_TYPE_ATTRS;
+        case nList:
+            return NIX_TYPE_LIST;
+        case nFunction:
+            return NIX_TYPE_FUNCTION;
+        case nExternal:
+            return NIX_TYPE_EXTERNAL;
+        }
+        return NIX_TYPE_NULL;
+    }
+    NIXC_CATCH_ERRS_RES(NIX_TYPE_NULL);
+}
+
+const char * nix_get_typename(nix_c_context * context, const Value * value)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        auto s = nix::showType(v);
+        return strdup(s.c_str());
+    }
+    NIXC_CATCH_ERRS_NULL
+}
+
+bool nix_get_bool(nix_c_context * context, const Value * value)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        assert(v.type() == nix::nBool);
+        return v.boolean;
+    }
+    NIXC_CATCH_ERRS_RES(false);
+}
+
+const char * nix_get_string(nix_c_context * context, const Value * value)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        assert(v.type() == nix::nString);
+        return v.string.s;
+    }
+    NIXC_CATCH_ERRS_NULL
+}
+
+const char * nix_get_path_string(nix_c_context * context, const Value * value)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        assert(v.type() == nix::nPath);
+        return v._path;
+    }
+    NIXC_CATCH_ERRS_NULL
+}
+
+unsigned int nix_get_list_size(nix_c_context * context, const Value * value)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        assert(v.type() == nix::nList);
+        return v.listSize();
+    }
+    NIXC_CATCH_ERRS_RES(0);
+}
+
+unsigned int nix_get_attrs_size(nix_c_context * context, const Value * value)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        assert(v.type() == nix::nAttrs);
+        return v.attrs->size();
+    }
+    NIXC_CATCH_ERRS_RES(0);
+}
+
+double nix_get_float(nix_c_context * context, const Value * value)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        assert(v.type() == nix::nFloat);
+        return v.fpoint;
+    }
+    NIXC_CATCH_ERRS_RES(NAN);
+}
+
+int64_t nix_get_int(nix_c_context * context, const Value * value)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        assert(v.type() == nix::nInt);
+        return v.integer;
+    }
+    NIXC_CATCH_ERRS_RES(0);
+}
+
+ExternalValue * nix_get_external(nix_c_context * context, Value * value)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        assert(v.type() == nix::nExternal);
+        return (ExternalValue *) v.external;
+    }
+    NIXC_CATCH_ERRS_NULL;
+}
+
+Value * nix_get_list_byidx(nix_c_context * context, const Value * value, EvalState * state, unsigned int ix)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        assert(v.type() == nix::nList);
+        auto * p = v.listElems()[ix];
+        nix_gc_incref(nullptr, p);
+        state->state.forceValue(*p, nix::noPos);
+        return (Value *) p;
+    }
+    NIXC_CATCH_ERRS_NULL
+}
+
+Value * nix_get_attr_byname(nix_c_context * context, const Value * value, EvalState * state, const char * name)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        assert(v.type() == nix::nAttrs);
+        nix::Symbol s = state->state.symbols.create(name);
+        auto attr = v.attrs->get(s);
+        if (attr) {
+            nix_gc_incref(nullptr, attr->value);
+            state->state.forceValue(*attr->value, nix::noPos);
+            return attr->value;
+        }
+        nix_set_err_msg(context, NIX_ERR_KEY, "missing attribute");
+        return nullptr;
+    }
+    NIXC_CATCH_ERRS_NULL
+}
+
+bool nix_has_attr_byname(nix_c_context * context, const Value * value, EvalState * state, const char * name)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        assert(v.type() == nix::nAttrs);
+        nix::Symbol s = state->state.symbols.create(name);
+        auto attr = v.attrs->get(s);
+        if (attr)
+            return true;
+        return false;
+    }
+    NIXC_CATCH_ERRS_RES(false);
+}
+
+Value *
+nix_get_attr_byidx(nix_c_context * context, const Value * value, EvalState * state, unsigned int i, const char ** name)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        const nix::Attr & a = (*v.attrs)[i];
+        *name = ((const std::string &) (state->state.symbols[a.name])).c_str();
+        nix_gc_incref(nullptr, a.value);
+        state->state.forceValue(*a.value, nix::noPos);
+        return a.value;
+    }
+    NIXC_CATCH_ERRS_NULL
+}
+
+const char * nix_get_attr_name_byidx(nix_c_context * context, const Value * value, EvalState * state, unsigned int i)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        const nix::Attr & a = (*v.attrs)[i];
+        return ((const std::string &) (state->state.symbols[a.name])).c_str();
+    }
+    NIXC_CATCH_ERRS_NULL
+}
+
+nix_err nix_init_bool(nix_c_context * context, Value * value, bool b)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        v.mkBool(b);
+    }
+    NIXC_CATCH_ERRS
+}
+
+// todo string context
+nix_err nix_init_string(nix_c_context * context, Value * value, const char * str)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        v.mkString(std::string_view(str));
+    }
+    NIXC_CATCH_ERRS
+}
+
+nix_err nix_init_path_string(nix_c_context * context, EvalState * s, Value * value, const char * str)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        v.mkPath(s->state.rootPath(nix::CanonPath(str)));
+    }
+    NIXC_CATCH_ERRS
+}
+
+nix_err nix_init_float(nix_c_context * context, Value * value, double d)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        v.mkFloat(d);
+    }
+    NIXC_CATCH_ERRS
+}
+
+nix_err nix_init_int(nix_c_context * context, Value * value, int64_t i)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        v.mkInt(i);
+    }
+    NIXC_CATCH_ERRS
+}
+
+nix_err nix_init_null(nix_c_context * context, Value * value)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        v.mkNull();
+    }
+    NIXC_CATCH_ERRS
+}
+
+nix_err nix_init_external(nix_c_context * context, Value * value, ExternalValue * val)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        auto r = (nix::ExternalValueBase *) val;
+        v.mkExternal(r);
+    }
+    NIXC_CATCH_ERRS
+}
+
+ListBuilder * nix_make_list_builder(nix_c_context * context, EvalState * state, size_t capacity)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto builder = CListBuilder(capacity);
+        return new
+#if HAVE_BOEHMGC
+            (NoGC)
+#endif
+                ListBuilder{std::move(builder)};
+    }
+    NIXC_CATCH_ERRS_NULL
+}
+
+nix_err nix_list_builder_insert(nix_c_context * context, ListBuilder * list_builder, Value * value)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        list_builder->builder.push_back((nix::Value *) value);
+    }
+    NIXC_CATCH_ERRS
+}
+
+void nix_list_builder_free(ListBuilder * bb)
+{
+#if HAVE_BOEHMGC
+    GC_FREE(bb);
+#else
+    delete bb;
+#endif
+}
+
+nix_err nix_make_list(nix_c_context * context, EvalState * s, ListBuilder * list_builder, Value * value)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        list_builder->builder.finish(&(s->state), &v);
+    }
+    NIXC_CATCH_ERRS
+}
+
+nix_err nix_init_primop(nix_c_context * context, Value * value, PrimOp * p)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        v.mkPrimOp((nix::PrimOp *) p);
+    }
+    NIXC_CATCH_ERRS
+}
+
+nix_err nix_copy_value(nix_c_context * context, Value * value, Value * source)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        auto & s = check_value_not_null(source);
+        v = s;
+    }
+    NIXC_CATCH_ERRS
+}
+
+nix_err nix_make_attrs(nix_c_context * context, Value * value, BindingsBuilder * b)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        v.mkAttrs(b->builder);
+    }
+    NIXC_CATCH_ERRS
+}
+
+BindingsBuilder * nix_make_bindings_builder(nix_c_context * context, EvalState * state, size_t capacity)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto bb = state->state.buildBindings(capacity);
+        return new
+#if HAVE_BOEHMGC
+            (NoGC)
+#endif
+                BindingsBuilder{std::move(bb)};
+    }
+    NIXC_CATCH_ERRS_NULL
+}
+
+nix_err nix_bindings_builder_insert(nix_c_context * context, BindingsBuilder * bb, const char * name, Value * value)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        nix::Symbol s = bb->builder.state.symbols.create(name);
+        bb->builder.insert(s, &v);
+    }
+    NIXC_CATCH_ERRS
+}
+
+void nix_bindings_builder_free(BindingsBuilder * bb)
+{
+#if HAVE_BOEHMGC
+    GC_FREE((nix::BindingsBuilder *) bb);
+#else
+    delete (nix::BindingsBuilder *) bb;
+#endif
+}

src/libexpr/c/nix_api_value.h

@@ -0,0 +1,433 @@
+#ifndef NIX_API_VALUE_H
+#define NIX_API_VALUE_H
+
+/** @addtogroup libexpr
+ * @{
+ */
+/** @file
+ * @brief libexpr C bindings dealing with values
+ */
+
+#include "nix_api_util.h"
+#include "stdbool.h"
+#include "stddef.h"
+#include "stdint.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+// cffi start
+
+// Type definitions
+typedef enum {
+    NIX_TYPE_THUNK,
+    NIX_TYPE_INT,
+    NIX_TYPE_FLOAT,
+    NIX_TYPE_BOOL,
+    NIX_TYPE_STRING,
+    NIX_TYPE_PATH,
+    NIX_TYPE_NULL,
+    NIX_TYPE_ATTRS,
+    NIX_TYPE_LIST,
+    NIX_TYPE_FUNCTION,
+    NIX_TYPE_EXTERNAL
+} ValueType;
+
+// forward declarations
+typedef void Value;
+typedef struct EvalState EvalState;
+// type defs
+/** @brief Stores an under-construction set of bindings
+ * @ingroup value_manip
+ *
+ * Do not reuse.
+ * @see nix_make_bindings_builder, nix_bindings_builder_free, nix_make_attrs
+ * @see nix_bindings_builder_insert
+ */
+typedef struct BindingsBuilder BindingsBuilder;
+
+/** @brief Stores an under-construction list
+ * @ingroup value_manip
+ *
+ * Do not reuse.
+ * @see nix_make_list_builder, nix_list_builder_free, nix_make_list
+ * @see nix_list_builder_insert
+ */
+typedef struct ListBuilder ListBuilder;
+
+/** @brief PrimOp function
+ * @ingroup primops
+ *
+ * Owned by the GC
+ * @see nix_alloc_primop, nix_init_primop
+ */
+typedef struct PrimOp PrimOp;
+/** @brief External Value
+ * @ingroup Externals
+ *
+ * Owned by the GC
+ */
+typedef struct ExternalValue ExternalValue;
+
+/** @defgroup primops
+ * @brief Create your own primops
+ * @{
+ */
+/** @brief Function pointer for primops
+ * When you want to return an error, call nix_set_err_msg(context, NIX_ERR_UNKNOWN, "your error message here").
+ *
+ * @param[in] user_data Arbitrary data that was initially supplied to nix_alloc_primop
+ * @param[out] context Stores error information.
+ * @param[in] state Evaluator state
+ * @param[in] args list of arguments. Note that these can be thunks and should be forced using nix_value_force before
+ * use.
+ * @param[out] ret return value
+ * @see nix_alloc_primop, nix_init_primop
+ */
+typedef void (*PrimOpFun)(void * user_data, nix_c_context * context, EvalState * state, Value ** args, Value * ret);
+
+/** @brief Allocate a PrimOp
+ *
+ * Owned by the garbage collector.
+ * Use nix_gc_decref() when you're done with the returned PrimOp.
+ *
+ * @param[out] context Optional, stores error information
+ * @param[in] fun callback
+ * @param[in] arity expected number of function arguments
+ * @param[in] name function name
+ * @param[in] args array of argument names, NULL-terminated
+ * @param[in] doc optional, documentation for this primop
+ * @param[in] user_data optional, arbitrary data, passed to the callback when it's called
+ * @return primop, or null in case of errors
+ * @see nix_init_primop
+ */
+PrimOp * nix_alloc_primop(
+    nix_c_context * context,
+    PrimOpFun fun,
+    int arity,
+    const char * name,
+    const char ** args,
+    const char * doc,
+    void * user_data);
+
+/** @brief add a primop to the `builtins` attribute set
+ *
+ * Only applies to States created after this call.
+ *
+ * Moves your PrimOp content into the global evaluator
+ * registry, meaning your input PrimOp pointer is no longer usable.
+ * You are free to remove your references to it,
+ * after which it will be garbage collected.
+ *
+ * @param[out] context Optional, stores error information
+ * @return primop, or null in case of errors
+ *
+ */
+nix_err nix_register_primop(nix_c_context * context, PrimOp * primOp);
+/** @} */
+
+// Function prototypes
+
+/** @brief Allocate a Nix value
+ *
+ * Owned by the GC. Use nix_gc_decref() when you're done with the pointer
+ * @param[out] context Optional, stores error information
+ * @param[in] state nix evaluator state
+ * @return value, or null in case of errors
+ *
+ */
+
+Value * nix_alloc_value(nix_c_context * context, EvalState * state);
+/** @addtogroup value_manip Manipulating values
+ * @brief Functions to inspect and change Nix language values, represented by Value.
+ * @{
+ */
+/** @name Getters
+ */
+/**@{*/
+/** @brief Get value type
+ * @param[out] context Optional, stores error information
+ * @param[in] value Nix value to inspect
+ * @return type of nix value
+ */
+
+ValueType nix_get_type(nix_c_context * context, const Value * value);
+/** @brief Get type name of value as defined in the evaluator
+ * @param[out] context Optional, stores error information
+ * @param[in] value Nix value to inspect
+ * @return type name, owned string
+ * @todo way to free the result
+ */
+
+const char * nix_get_typename(nix_c_context * context, const Value * value);
+
+/** @brief Get boolean value
+ * @param[out] context Optional, stores error information
+ * @param[in] value Nix value to inspect
+ * @return true or false, error info via context
+ */
+bool nix_get_bool(nix_c_context * context, const Value * value);
+/** @brief Get string
+ * @param[out] context Optional, stores error information
+ * @param[in] value Nix value to inspect
+ * @return string
+ * @return NULL in case of error.
+ */
+const char * nix_get_string(nix_c_context * context, const Value * value);
+
+/** @brief Get path as string
+ * @param[out] context Optional, stores error information
+ * @param[in] value Nix value to inspect
+ * @return string
+ * @return NULL in case of error.
+ */
+const char * nix_get_path_string(nix_c_context * context, const Value * value);
+
+/** @brief Get the length of a list
+ * @param[out] context Optional, stores error information
+ * @param[in] value Nix value to inspect
+ * @return length of list, error info via context
+ */
+unsigned int nix_get_list_size(nix_c_context * context, const Value * value);
+
+/** @brief Get the element count of an attrset
+ * @param[out] context Optional, stores error information
+ * @param[in] value Nix value to inspect
+ * @return attrset element count, error info via context
+ */
+unsigned int nix_get_attrs_size(nix_c_context * context, const Value * value);
+
+/** @brief Get float value in 64 bits
+ * @param[out] context Optional, stores error information
+ * @param[in] value Nix value to inspect
+ * @return float contents, error info via context
+ */
+double nix_get_float(nix_c_context * context, const Value * value);
+
+/** @brief Get int value
+ * @param[out] context Optional, stores error information
+ * @param[in] value Nix value to inspect
+ * @return int contents, error info via context
+ */
+int64_t nix_get_int(nix_c_context * context, const Value * value);
+
+/** @brief Get external reference
+ * @param[out] context Optional, stores error information
+ * @param[in] value Nix value to inspect
+ * @return reference to external, NULL in case of error
+ */
+ExternalValue * nix_get_external(nix_c_context * context, Value *);
+
+/** @brief Get the ix'th element of a list
+ *
+ * Owned by the GC. Use nix_gc_decref when you're done with the pointer
+ * @param[out] context Optional, stores error information
+ * @param[in] value Nix value to inspect
+ * @param[in] state nix evaluator state
+ * @param[in] ix list element to get
+ * @return value, NULL in case of errors
+ */
+Value * nix_get_list_byidx(nix_c_context * context, const Value * value, EvalState * state, unsigned int ix);
+
+/** @brief Get an attr by name
+ *
+ * Owned by the GC. Use nix_gc_decref when you're done with the pointer
+ * @param[out] context Optional, stores error information
+ * @param[in] value Nix value to inspect
+ * @param[in] state nix evaluator state
+ * @param[in] name attribute name
+ * @return value, NULL in case of errors
+ */
+Value * nix_get_attr_byname(nix_c_context * context, const Value * value, EvalState * state, const char * name);
+
+/** @brief Check if an attribute name exists on a value
+ * @param[out] context Optional, stores error information
+ * @param[in] value Nix value to inspect
+ * @param[in] state nix evaluator state
+ * @param[in] name attribute name
+ * @return value, error info via context
+ */
+bool nix_has_attr_byname(nix_c_context * context, const Value * value, EvalState * state, const char * name);
+
+/** @brief Get an attribute by index in the sorted bindings
+ *
+ * Also gives you the name.
+ *
+ * Owned by the GC. Use nix_gc_decref when you're done with the pointer
+ * @param[out] context Optional, stores error information
+ * @param[in] value Nix value to inspect
+ * @param[in] state nix evaluator state
+ * @param[in] i attribute index
+ * @param[out] name will store a pointer to the attribute name
+ * @return value, NULL in case of errors
+ */
+Value *
+nix_get_attr_byidx(nix_c_context * context, const Value * value, EvalState * state, unsigned int i, const char ** name);
+
+/** @brief Get an attribute name by index in the sorted bindings
+ *
+ * Useful when you want the name but want to avoid evaluation.
+ *
+ * Owned by the nix EvalState
+ * @param[out] context Optional, stores error information
+ * @param[in] value Nix value to inspect
+ * @param[in] state nix evaluator state
+ * @param[in] i attribute index
+ * @return name, NULL in case of errors
+ */
+const char * nix_get_attr_name_byidx(nix_c_context * context, const Value * value, EvalState * state, unsigned int i);
+
+/**@}*/
+/** @name Initializers
+ *
+ * Values are typically "returned" by initializing already allocated memory that serves as the return value.
+ * For this reason, the construction of values is not tied their allocation.
+ * Nix is a language with immutable values. Respect this property by only initializing Values once; and only initialize
+ * Values that are meant to be initialized by you. Failing to adhere to these rules may lead to undefined behavior.
+ */
+/**@{*/
+/** @brief Set boolean value
+ * @param[out] context Optional, stores error information
+ * @param[out] value Nix value to modify
+ * @param[in] b the boolean value
+ * @return error code, NIX_OK on success.
+ */
+nix_err nix_init_bool(nix_c_context * context, Value * value, bool b);
+
+/** @brief Set a string
+ * @param[out] context Optional, stores error information
+ * @param[out] value Nix value to modify
+ * @param[in] str the string, copied
+ * @return error code, NIX_OK on success.
+ */
+nix_err nix_init_string(nix_c_context * context, Value * value, const char * str);
+
+/** @brief Set a path
+ * @param[out] context Optional, stores error information
+ * @param[out] value Nix value to modify
+ * @param[in] str the path string, copied
+ * @return error code, NIX_OK on success.
+ */
+nix_err nix_init_path_string(nix_c_context * context, EvalState * s, Value * value, const char * str);
+
+/** @brief Set a float
+ * @param[out] context Optional, stores error information
+ * @param[out] value Nix value to modify
+ * @param[in] d the float, 64-bits
+ * @return error code, NIX_OK on success.
+ */
+nix_err nix_init_float(nix_c_context * context, Value * value, double d);
+
+/** @brief Set an int
+ * @param[out] context Optional, stores error information
+ * @param[out] value Nix value to modify
+ * @param[in] i the int
+ * @return error code, NIX_OK on success.
+ */
+
+nix_err nix_init_int(nix_c_context * context, Value * value, int64_t i);
+/** @brief Set null
+ * @param[out] context Optional, stores error information
+ * @param[out] value Nix value to modify
+ * @return error code, NIX_OK on success.
+ */
+
+nix_err nix_init_null(nix_c_context * context, Value * value);
+/** @brief Set an external value
+ * @param[out] context Optional, stores error information
+ * @param[out] value Nix value to modify
+ * @param[in] val the external value to set. Will be GC-referenced by the value.
+ * @return error code, NIX_OK on success.
+ */
+nix_err nix_init_external(nix_c_context * context, Value * value, ExternalValue * val);
+
+/** @brief Create a list from a list builder
+ * @param[out] context Optional, stores error information
+ * @param[in] list_builder list builder to use. Make sure to unref this afterwards.
+ * @param[out] value Nix value to modify
+ * @return error code, NIX_OK on success.
+ */
+nix_err nix_make_list(nix_c_context * context, EvalState * s, ListBuilder * list_builder, Value * value);
+
+/** @brief Create a list builder
+ * @param[out] context Optional, stores error information
+ * @param[in] state nix evaluator state
+ * @param[in] capacity how many bindings you'll add. Don't exceed.
+ * @return owned reference to a list builder. Make sure to unref when you're done.
+ */
+ListBuilder * nix_make_list_builder(nix_c_context * context, EvalState * state, size_t capacity);
+
+/** @brief Insert bindings into a builder
+ * @param[out] context Optional, stores error information
+ * @param[in] list_builder ListBuilder to insert into
+ * @param[in] value value to insert
+ * @return error code, NIX_OK on success.
+ */
+nix_err nix_list_builder_insert(nix_c_context * context, ListBuilder * list_builder, Value * value);
+
+/** @brief Free a list builder
+ *
+ * Does not fail.
+ * @param[in] builder the builder to free
+ */
+void nix_list_builder_free(ListBuilder * builder);
+
+/** @brief Create an attribute set from a bindings builder
+ * @param[out] context Optional, stores error information
+ * @param[out] value Nix value to modify
+ * @param[in] b bindings builder to use. Make sure to unref this afterwards.
+ * @return error code, NIX_OK on success.
+ */
+nix_err nix_make_attrs(nix_c_context * context, Value * value, BindingsBuilder * b);
+
+/** @brief Set primop
+ * @param[out] context Optional, stores error information
+ * @param[out] value Nix value to modify
+ * @param[in] op primop, will be gc-referenced by the value
+ * @see nix_alloc_primop
+ * @return error code, NIX_OK on success.
+ */
+nix_err nix_init_primop(nix_c_context * context, Value * value, PrimOp * op);
+/** @brief Copy from another value
+ * @param[out] context Optional, stores error information
+ * @param[out] value Nix value to modify
+ * @param[in] source value to copy from
+ * @return error code, NIX_OK on success.
+ */
+nix_err nix_copy_value(nix_c_context * context, Value * value, Value * source);
+/**@}*/
+
+/** @brief Create a bindings builder
+* @param[out] context Optional, stores error information
+* @param[in] state nix evaluator state
+* @param[in] capacity how many bindings you'll add. Don't exceed.
+* @return owned reference to a bindings builder. Make sure to unref when you're
+done.
+*/
+BindingsBuilder * nix_make_bindings_builder(nix_c_context * context, EvalState * state, size_t capacity);
+
+/** @brief Insert bindings into a builder
+ * @param[out] context Optional, stores error information
+ * @param[in] builder BindingsBuilder to insert into
+ * @param[in] name attribute name, copied into the symbol store
+ * @param[in] value value to give the binding
+ * @return error code, NIX_OK on success.
+ */
+nix_err
+nix_bindings_builder_insert(nix_c_context * context, BindingsBuilder * builder, const char * name, Value * value);
+
+/** @brief Free a bindings builder
+ *
+ * Does not fail.
+ * @param[in] builder the builder to free
+ */
+void nix_bindings_builder_free(BindingsBuilder * builder);
+/**@}*/
+
+// cffi end
+#ifdef __cplusplus
+}
+#endif
+
+/** @} */
+#endif // NIX_API_VALUE_H

src/libexpr/eval-cache.cc

@@ -490,7 +490,7 @@ std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(Symbol name, bool forceErro
                         if (forceErrors)
                             debug("reevaluating failed cached attribute '%s'", getAttrPathStr(name));
                         else
-                            throw CachedEvalError("cached failure of attribute '%s'", getAttrPathStr(name));
+                            throw CachedEvalError(root->state, "cached failure of attribute '%s'", getAttrPathStr(name));
                     } else
                         return std::make_shared<AttrCursor>(root,
                             std::make_pair(shared_from_this(), name), nullptr, std::move(attr));
@@ -499,7 +499,7 @@ std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(Symbol name, bool forceErro
                 // evaluate to see whether 'name' exists
             } else
                 return nullptr;
-                //throw TypeError("'%s' is not an attribute set", getAttrPathStr());
+                //error<TypeError>("'%s' is not an attribute set", getAttrPathStr()).debugThrow();
         }
     }
 
@@ -507,7 +507,7 @@ std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(Symbol name, bool forceErro
 
     if (v.type() != nAttrs)
         return nullptr;
-        //throw TypeError("'%s' is not an attribute set", getAttrPathStr());
+        //error<TypeError>("'%s' is not an attribute set", getAttrPathStr()).debugThrow();
 
     auto attr = v.attrs->get(name);
 
@@ -573,14 +573,14 @@ std::string AttrCursor::getString()
                 debug("using cached string attribute '%s'", getAttrPathStr());
                 return s->first;
             } else
-                root->state.error("'%s' is not a string", getAttrPathStr()).debugThrow<TypeError>();
+                root->state.error<TypeError>("'%s' is not a string", getAttrPathStr()).debugThrow();
         }
     }
 
     auto & v = forceValue();
 
     if (v.type() != nString && v.type() != nPath)
-        root->state.error("'%s' is not a string but %s", getAttrPathStr()).debugThrow<TypeError>();
+        root->state.error<TypeError>("'%s' is not a string but %s", getAttrPathStr()).debugThrow();
 
     return v.type() == nString ? v.string.s : v.path().to_string();
 }
@@ -615,7 +615,7 @@ string_t AttrCursor::getStringWithContext()
                     return *s;
                 }
             } else
-                root->state.error("'%s' is not a string", getAttrPathStr()).debugThrow<TypeError>();
+                root->state.error<TypeError>("'%s' is not a string", getAttrPathStr()).debugThrow();
         }
     }
 
@@ -629,7 +629,7 @@ string_t AttrCursor::getStringWithContext()
     else if (v.type() == nPath)
         return {v.path().to_string(), {}};
     else
-        root->state.error("'%s' is not a string but %s", getAttrPathStr()).debugThrow<TypeError>();
+        root->state.error<TypeError>("'%s' is not a string but %s", getAttrPathStr()).debugThrow();
 }
 
 bool AttrCursor::getBool()
@@ -642,14 +642,14 @@ bool AttrCursor::getBool()
                 debug("using cached Boolean attribute '%s'", getAttrPathStr());
                 return *b;
             } else
-                root->state.error("'%s' is not a Boolean", getAttrPathStr()).debugThrow<TypeError>();
+                root->state.error<TypeError>("'%s' is not a Boolean", getAttrPathStr()).debugThrow();
         }
     }
 
     auto & v = forceValue();
 
     if (v.type() != nBool)
-        root->state.error("'%s' is not a Boolean", getAttrPathStr()).debugThrow<TypeError>();
+        root->state.error<TypeError>("'%s' is not a Boolean", getAttrPathStr()).debugThrow();
 
     return v.boolean;
 }
@@ -664,14 +664,14 @@ NixInt AttrCursor::getInt()
                 debug("using cached integer attribute '%s'", getAttrPathStr());
                 return i->x;
             } else
-                throw TypeError("'%s' is not an integer", getAttrPathStr());
+                root->state.error<TypeError>("'%s' is not an integer", getAttrPathStr()).debugThrow();
         }
     }
 
     auto & v = forceValue();
 
     if (v.type() != nInt)
-        throw TypeError("'%s' is not an integer", getAttrPathStr());
+        root->state.error<TypeError>("'%s' is not an integer", getAttrPathStr()).debugThrow();
 
     return v.integer;
 }
@@ -686,7 +686,7 @@ std::vector<std::string> AttrCursor::getListOfStrings()
                 debug("using cached list of strings attribute '%s'", getAttrPathStr());
                 return *l;
             } else
-                throw TypeError("'%s' is not a list of strings", getAttrPathStr());
+                root->state.error<TypeError>("'%s' is not a list of strings", getAttrPathStr()).debugThrow();
         }
     }
 
@@ -696,7 +696,7 @@ std::vector<std::string> AttrCursor::getListOfStrings()
     root->state.forceValue(v, noPos);
 
     if (v.type() != nList)
-        throw TypeError("'%s' is not a list", getAttrPathStr());
+        root->state.error<TypeError>("'%s' is not a list", getAttrPathStr()).debugThrow();
 
     std::vector<std::string> res;
 
@@ -719,14 +719,14 @@ std::vector<Symbol> AttrCursor::getAttrs()
                 debug("using cached attrset attribute '%s'", getAttrPathStr());
                 return *attrs;
             } else
-                root->state.error("'%s' is not an attribute set", getAttrPathStr()).debugThrow<TypeError>();
+                root->state.error<TypeError>("'%s' is not an attribute set", getAttrPathStr()).debugThrow();
         }
     }
 
     auto & v = forceValue();
 
     if (v.type() != nAttrs)
-        root->state.error("'%s' is not an attribute set", getAttrPathStr()).debugThrow<TypeError>();
+        root->state.error<TypeError>("'%s' is not an attribute set", getAttrPathStr()).debugThrow();
 
     std::vector<Symbol> attrs;
     for (auto & attr : *getValue().attrs)

src/libexpr/eval-cache.hh

@@ -10,8 +10,6 @@
 
 namespace nix::eval_cache {
 
-MakeError(CachedEvalError, EvalError);
-
 struct AttrDb;
 class AttrCursor;
 

src/libexpr/eval-error.cc

@@ -0,0 +1,113 @@
+#include "eval-error.hh"
+#include "eval.hh"
+#include "value.hh"
+
+namespace nix {
+
+template<class T>
+EvalErrorBuilder<T> & EvalErrorBuilder<T>::withExitStatus(unsigned int exitStatus)
+{
+    error.withExitStatus(exitStatus);
+    return *this;
+}
+
+template<class T>
+EvalErrorBuilder<T> & EvalErrorBuilder<T>::atPos(PosIdx pos)
+{
+    error.err.pos = error.state.positions[pos];
+    return *this;
+}
+
+template<class T>
+EvalErrorBuilder<T> & EvalErrorBuilder<T>::atPos(Value & value, PosIdx fallback)
+{
+    return atPos(value.determinePos(fallback));
+}
+
+template<class T>
+EvalErrorBuilder<T> & EvalErrorBuilder<T>::withTrace(PosIdx pos, const std::string_view text)
+{
+    error.err.traces.push_front(
+        Trace{.pos = error.state.positions[pos], .hint = HintFmt(std::string(text)), .frame = false});
+    return *this;
+}
+
+template<class T>
+EvalErrorBuilder<T> & EvalErrorBuilder<T>::withFrameTrace(PosIdx pos, const std::string_view text)
+{
+    error.err.traces.push_front(
+        Trace{.pos = error.state.positions[pos], .hint = HintFmt(std::string(text)), .frame = true});
+    return *this;
+}
+
+template<class T>
+EvalErrorBuilder<T> & EvalErrorBuilder<T>::withSuggestions(Suggestions & s)
+{
+    error.err.suggestions = s;
+    return *this;
+}
+
+template<class T>
+EvalErrorBuilder<T> & EvalErrorBuilder<T>::withFrame(const Env & env, const Expr & expr)
+{
+    // NOTE: This is abusing side-effects.
+    // TODO: check compatibility with nested debugger calls.
+    // TODO: What side-effects??
+    error.state.debugTraces.push_front(DebugTrace{
+        .pos = error.state.positions[expr.getPos()],
+        .expr = expr,
+        .env = env,
+        .hint = HintFmt("Fake frame for debugging purposes"),
+        .isError = true});
+    return *this;
+}
+
+template<class T>
+EvalErrorBuilder<T> & EvalErrorBuilder<T>::addTrace(PosIdx pos, HintFmt hint, bool frame)
+{
+    error.addTrace(error.state.positions[pos], hint, frame);
+    return *this;
+}
+
+template<class T>
+template<typename... Args>
+EvalErrorBuilder<T> &
+EvalErrorBuilder<T>::addTrace(PosIdx pos, std::string_view formatString, const Args &... formatArgs)
+{
+
+    addTrace(error.state.positions[pos], HintFmt(std::string(formatString), formatArgs...));
+    return *this;
+}
+
+template<class T>
+void EvalErrorBuilder<T>::debugThrow()
+{
+    if (error.state.debugRepl && !error.state.debugTraces.empty()) {
+        const DebugTrace & last = error.state.debugTraces.front();
+        const Env * env = &last.env;
+        const Expr * expr = &last.expr;
+        error.state.runDebugRepl(&error, *env, *expr);
+    }
+
+    // `EvalState` is the only class that can construct an `EvalErrorBuilder`,
+    // and it does so in dynamic storage. This is the final method called on
+    // any such instance and must delete itself before throwing the underlying
+    // error.
+    auto error = std::move(this->error);
+    delete this;
+
+    throw error;
+}
+
+template class EvalErrorBuilder<EvalError>;
+template class EvalErrorBuilder<AssertionError>;
+template class EvalErrorBuilder<ThrownError>;
+template class EvalErrorBuilder<Abort>;
+template class EvalErrorBuilder<TypeError>;
+template class EvalErrorBuilder<UndefinedVarError>;
+template class EvalErrorBuilder<MissingArgumentError>;
+template class EvalErrorBuilder<InfiniteRecursionError>;
+template class EvalErrorBuilder<CachedEvalError>;
+template class EvalErrorBuilder<InvalidPathError>;
+
+}

src/libexpr/eval-error.hh

@@ -0,0 +1,105 @@
+#pragma once
+
+#include <algorithm>
+
+#include "error.hh"
+#include "pos-idx.hh"
+
+namespace nix {
+
+struct Env;
+struct Expr;
+struct Value;
+
+class EvalState;
+template<class T>
+class EvalErrorBuilder;
+
+class EvalError : public Error
+{
+    template<class T>
+    friend class EvalErrorBuilder;
+public:
+    EvalState & state;
+
+    EvalError(EvalState & state, ErrorInfo && errorInfo)
+        : Error(errorInfo)
+        , state(state)
+    {
+    }
+
+    template<typename... Args>
+    explicit EvalError(EvalState & state, const std::string & formatString, const Args &... formatArgs)
+        : Error(formatString, formatArgs...)
+        , state(state)
+    {
+    }
+};
+
+MakeError(ParseError, Error);
+MakeError(AssertionError, EvalError);
+MakeError(ThrownError, AssertionError);
+MakeError(Abort, EvalError);
+MakeError(TypeError, EvalError);
+MakeError(UndefinedVarError, EvalError);
+MakeError(MissingArgumentError, EvalError);
+MakeError(RestrictedPathError, Error);
+MakeError(CachedEvalError, EvalError);
+MakeError(InfiniteRecursionError, EvalError);
+
+struct InvalidPathError : public EvalError
+{
+public:
+    Path path;
+    InvalidPathError(EvalState & state, const Path & path)
+        : EvalError(state, "path '%s' is not valid", path)
+    {
+    }
+};
+
+/**
+ * `EvalErrorBuilder`s may only be constructed by `EvalState`. The `debugThrow`
+ * method must be the final method in any such `EvalErrorBuilder` usage, and it
+ * handles deleting the object.
+ */
+template<class T>
+class EvalErrorBuilder final
+{
+    friend class EvalState;
+
+    template<typename... Args>
+    explicit EvalErrorBuilder(EvalState & state, const Args &... args)
+        : error(T(state, args...))
+    {
+    }
+
+public:
+    T error;
+
+    [[nodiscard, gnu::noinline]] EvalErrorBuilder<T> & withExitStatus(unsigned int exitStatus);
+
+    [[nodiscard, gnu::noinline]] EvalErrorBuilder<T> & atPos(PosIdx pos);
+
+    [[nodiscard, gnu::noinline]] EvalErrorBuilder<T> & atPos(Value & value, PosIdx fallback = noPos);
+
+    [[nodiscard, gnu::noinline]] EvalErrorBuilder<T> & withTrace(PosIdx pos, const std::string_view text);
+
+    [[nodiscard, gnu::noinline]] EvalErrorBuilder<T> & withFrameTrace(PosIdx pos, const std::string_view text);
+
+    [[nodiscard, gnu::noinline]] EvalErrorBuilder<T> & withSuggestions(Suggestions & s);
+
+    [[nodiscard, gnu::noinline]] EvalErrorBuilder<T> & withFrame(const Env & e, const Expr & ex);
+
+    [[nodiscard, gnu::noinline]] EvalErrorBuilder<T> & addTrace(PosIdx pos, HintFmt hint, bool frame = false);
+
+    template<typename... Args>
+    [[nodiscard, gnu::noinline]] EvalErrorBuilder<T> &
+    addTrace(PosIdx pos, std::string_view formatString, const Args &... formatArgs);
+
+    /**
+     * Delete the `EvalErrorBuilder` and throw the underlying exception.
+     */
+    [[gnu::noinline, gnu::noreturn]] void debugThrow();
+};
+
+}

src/libexpr/eval-inline.hh

@@ -1,7 +1,9 @@
 #pragma once
 ///@file
 
+#include "print.hh"
 #include "eval.hh"
+#include "eval-error.hh"
 
 namespace nix {
 
@@ -73,8 +75,6 @@ Env & EvalState::allocEnv(size_t size)
 #endif
         env = (Env *) allocBytes(sizeof(Env) + size * sizeof(Value *));
 
-    env->type = Env::Plain;
-
     /* We assume that env->values has been cleared by the allocator; maybeThunk() and lookupVar fromWith expect this. */
 
     return *env;
@@ -83,13 +83,6 @@ Env & EvalState::allocEnv(size_t size)
 
 [[gnu::always_inline]]
 void EvalState::forceValue(Value & v, const PosIdx pos)
-{
-    forceValue(v, [&]() { return pos; });
-}
-
-
-template<typename Callable>
-void EvalState::forceValue(Value & v, Callable getPos)
 {
     if (v.isThunk()) {
         Env * env = v.thunk.env;
@@ -100,13 +93,12 @@ void EvalState::forceValue(Value & v, Callable getPos)
             expr->eval(*this, *env, v);
         } catch (...) {
             v.mkThunk(env, expr);
+            tryFixupBlackHolePos(v, pos);
             throw;
         }
     }
     else if (v.isApp())
-        callFunction(*v.app.left, *v.app.right, v, noPos);
-    else if (v.isBlackhole())
-        error("infinite recursion encountered").atPos(getPos()).template debugThrow<EvalError>();
+        callFunction(*v.app.left, *v.app.right, v, pos);
 }
 
 
@@ -121,10 +113,14 @@ template <typename Callable>
 [[gnu::always_inline]]
 inline void EvalState::forceAttrs(Value & v, Callable getPos, std::string_view errorCtx)
 {
-    forceValue(v, noPos);
+    PosIdx pos = getPos();
+    forceValue(v, pos);
     if (v.type() != nAttrs) {
-        PosIdx pos = getPos();
-        error("value is %1% while a set was expected", showType(v)).withTrace(pos, errorCtx).debugThrow<TypeError>();
+        error<TypeError>(
+            "expected a set but found %1%: %2%",
+            showType(v),
+            ValuePrinter(*this, v, errorPrintOptions)
+        ).withTrace(pos, errorCtx).debugThrow();
     }
 }
 
@@ -132,9 +128,13 @@ inline void EvalState::forceAttrs(Value & v, Callable getPos, std::string_view e
 [[gnu::always_inline]]
 inline void EvalState::forceList(Value & v, const PosIdx pos, std::string_view errorCtx)
 {
-    forceValue(v, noPos);
+    forceValue(v, pos);
     if (!v.isList()) {
-        error("value is %1% while a list was expected", showType(v)).withTrace(pos, errorCtx).debugThrow<TypeError>();
+        error<TypeError>(
+            "expected a list but found %1%: %2%",
+            showType(v),
+            ValuePrinter(*this, v, errorPrintOptions)
+        ).withTrace(pos, errorCtx).debugThrow();
     }
 }
 

src/libexpr/eval-settings.cc

@@ -88,6 +88,12 @@ std::string EvalSettings::resolvePseudoUrl(std::string_view url)
         return std::string(url);
 }
 
+const std::string & EvalSettings::getCurrentSystem()
+{
+    const auto & evalSystem = currentSystem.get();
+    return evalSystem != "" ? evalSystem : settings.thisSystem.get();
+}
+
 EvalSettings evalSettings;
 
 static GlobalConfig::Register rEvalSettings(&evalSettings);

src/libexpr/eval-settings.hh

@@ -25,6 +25,26 @@ struct EvalSettings : Config
           [`builtins.nixPath`](@docroot@/language/builtin-constants.md#builtins-nixPath).
         )"};
 
+    Setting<std::string> currentSystem{
+        this, "", "eval-system",
+        R"(
+          This option defines
+          [`builtins.currentSystem`](@docroot@/language/builtin-constants.md#builtins-currentSystem)
+          in the Nix language if it is set as a non-empty string.
+          Otherwise, if it is defined as the empty string (the default), the value of the
+          [`system` ](#conf-system)
+          configuration setting is used instead.
+
+          Unlike `system`, this setting does not change what kind of derivations can be built locally.
+          This is useful for evaluating Nix code on one system to produce derivations to be built on another type of system.
+        )"};
+
+    /**
+     * Implements the `eval-system` vs `system` defaulting logic
+     * described for `eval-system`.
+     */
+    const std::string & getCurrentSystem();
+
     Setting<bool> restrictEval{
         this, false, "restrict-eval",
         R"(
@@ -91,6 +111,9 @@ struct EvalSettings : Config
 
     Setting<bool> traceVerbose{this, false, "trace-verbose",
         "Whether `builtins.traceVerbose` should trace its first argument when evaluated."};
+
+    Setting<unsigned int> maxCallDepth{this, 10000, "max-call-depth",
+        "The maximum function call depth to allow before erroring."};
 };
 
 extern EvalSettings evalSettings;

src/libexpr/eval.hh

@@ -2,6 +2,7 @@
 ///@file
 
 #include "attr-set.hh"
+#include "eval-error.hh"
 #include "types.hh"
 #include "value.hh"
 #include "nixexpr.hh"
@@ -18,6 +19,12 @@
 
 namespace nix {
 
+/**
+ * We put a limit on primop arity because it lets us use a fixed size array on
+ * the stack. 8 is already an impractical number of arguments. Use an attrset
+ * argument for such overly complicated functions.
+ */
+constexpr size_t maxPrimOpArity = 8;
 
 class Store;
 class EvalState;
@@ -29,7 +36,7 @@ enum RepairFlag : bool;
 /**
  * Function that implements a primop.
  */
-typedef void (* PrimOpFun) (EvalState & state, const PosIdx pos, Value * * args, Value & v);
+typedef std::function<void (EvalState &, const PosIdx, Value **, Value &)> PrimOpFun;
 
 /**
  * Info about a primitive operation, and its implementation
@@ -69,8 +76,16 @@ struct PrimOp
      * Optional experimental for this to be gated on.
      */
     std::optional<ExperimentalFeature> experimentalFeature;
+
+    /**
+     * Validity check to be performed by functions that introduce primops,
+     * such as RegisterPrimOp() and Value::mkPrimOp().
+     */
+    void check();
 };
 
+std::ostream & operator<<(std::ostream & output, PrimOp & primOp);
+
 /**
  * Info about a constant
  */
@@ -103,11 +118,6 @@ struct Constant
 struct Env
 {
     Env * up;
-    /**
-     * Number of of levels up to next `with` environment
-     */
-    unsigned short prevWith:14;
-    enum { Plain = 0, HasWithExpr, HasWithAttrs } type:2;
     Value * values[0];
 };
 
@@ -119,7 +129,7 @@ std::unique_ptr<ValMap> mapStaticEnvBindings(const SymbolTable & st, const Stati
 void copyContext(const Value & v, NixStringContext & context);
 
 
-std::string printValue(const EvalState & state, const Value & v);
+std::string printValue(EvalState & state, Value & v);
 std::ostream & operator << (std::ostream & os, const ValueType t);
 
 
@@ -134,52 +144,13 @@ struct RegexCache;
 std::shared_ptr<RegexCache> makeRegexCache();
 
 struct DebugTrace {
-    std::shared_ptr<AbstractPos> pos;
+    std::shared_ptr<Pos> pos;
     const Expr & expr;
     const Env & env;
-    hintformat hint;
+    HintFmt hint;
     bool isError;
 };
 
-void debugError(Error * e, Env & env, Expr & expr);
-
-class ErrorBuilder
-{
-    private:
-        EvalState & state;
-        ErrorInfo info;
-
-        ErrorBuilder(EvalState & s, ErrorInfo && i): state(s), info(i) { }
-
-    public:
-        template<typename... Args>
-        [[nodiscard, gnu::noinline]]
-        static ErrorBuilder * create(EvalState & s, const Args & ... args)
-        {
-            return new ErrorBuilder(s, ErrorInfo { .msg = hintfmt(args...) });
-        }
-
-        [[nodiscard, gnu::noinline]]
-        ErrorBuilder & atPos(PosIdx pos);
-
-        [[nodiscard, gnu::noinline]]
-        ErrorBuilder & withTrace(PosIdx pos, const std::string_view text);
-
-        [[nodiscard, gnu::noinline]]
-        ErrorBuilder & withFrameTrace(PosIdx pos, const std::string_view text);
-
-        [[nodiscard, gnu::noinline]]
-        ErrorBuilder & withSuggestions(Suggestions & s);
-
-        [[nodiscard, gnu::noinline]]
-        ErrorBuilder & withFrame(const Env & e, const Expr & ex);
-
-        template<class ErrorType>
-        [[gnu::noinline, gnu::noreturn]]
-        void debugThrow();
-};
-
-
 class EvalState : public std::enable_shared_from_this<EvalState>
 {
 public:
@@ -197,6 +168,8 @@ public:
         sPrefix,
         sOutputSpecified;
 
+    const Expr::AstSymbols exprSymbols;
+
     /**
      * If set, force copying files to the Nix store even if they
      * already exist there.
@@ -211,6 +184,11 @@ public:
 
     Bindings emptyBindings;
 
+    /**
+     * Empty list constant.
+     */
+    Value vEmptyList;
+
     const SourcePath derivationInternal;
 
     /**
@@ -246,39 +224,11 @@ public:
 
     void runDebugRepl(const Error * error, const Env & env, const Expr & expr);
 
-    template<class E>
-    [[gnu::noinline, gnu::noreturn]]
-    void debugThrowLastTrace(E && error)
-    {
-        debugThrow(error, nullptr, nullptr);
-    }
-
-    template<class E>
-    [[gnu::noinline, gnu::noreturn]]
-    void debugThrow(E && error, const Env * env, const Expr * expr)
-    {
-        if (debugRepl && ((env && expr) || !debugTraces.empty())) {
-            if (!env || !expr) {
-                const DebugTrace & last = debugTraces.front();
-                env = &last.env;
-                expr = &last.expr;
-            }
-            runDebugRepl(&error, *env, *expr);
-        }
-
-        throw std::move(error);
-    }
-
-    // This is dangerous, but gets in line with the idea that error creation and
-    // throwing should not allocate on the stack of hot functions.
-    // as long as errors are immediately thrown, it works.
-    ErrorBuilder * errorBuilder;
-
-    template<typename... Args>
+    template<class T, typename... Args>
     [[nodiscard, gnu::noinline]]
-    ErrorBuilder & error(const Args & ... args) {
-        errorBuilder = ErrorBuilder::create(*this, args...);
-        return *errorBuilder;
+    EvalErrorBuilder<T> & error(const Args & ... args) {
+        // `EvalErrorBuilder::debugThrow` performs the corresponding `delete`.
+        return *new EvalErrorBuilder<T>(*this, args...);
     }
 
 private:
@@ -455,8 +405,7 @@ public:
      */
     inline void forceValue(Value & v, const PosIdx pos);
 
-    template <typename Callable>
-    inline void forceValue(Value & v, Callable getPos);
+    void tryFixupBlackHolePos(Value & v, PosIdx pos);
 
     /**
      * Force a value, then recursively force list elements and
@@ -618,6 +567,11 @@ private:
         const SourcePath & basePath,
         std::shared_ptr<StaticEnv> & staticEnv);
 
+    /**
+     * Current Nix call stack depth, used with `max-call-depth` setting to throw stack overflow hopefully before we run out of system stack.
+     */
+    size_t callDepth = 0;
+
 public:
 
     /**
@@ -709,9 +663,25 @@ public:
     void concatLists(Value & v, size_t nrLists, Value * * lists, const PosIdx pos, std::string_view errorCtx);
 
     /**
-     * Print statistics.
+     * Print statistics, if enabled.
+     *
+     * Performs a full memory GC before printing the statistics, so that the
+     * GC statistics are more accurate.
      */
-    void printStats();
+    void maybePrintStats();
+
+    /**
+     * Print statistics, unconditionally, cheaply, without performing a GC first.
+     */
+    void printStatistics();
+
+    /**
+     * Perform a full memory garbage collection - not incremental.
+     *
+     * @return true if Nix was built with GC and a GC was performed, false if not.
+     *              The return value is currently not thread safe - just the return value.
+     */
+    bool fullGC();
 
     /**
      * Realise the given context, and return a mapping from the placeholders
@@ -804,23 +774,8 @@ std::string showType(const Value & v);
  */
 SourcePath resolveExprPath(SourcePath path);
 
-struct InvalidPathError : EvalError
-{
-    Path path;
-    InvalidPathError(const Path & path);
-#ifdef EXCEPTION_NEEDS_THROW_SPEC
-    ~InvalidPathError() throw () { };
-#endif
-};
-
 static const std::string corepkgsPrefix{"/__corepkgs__/"};
 
-template<class ErrorType>
-void ErrorBuilder::debugThrow()
-{
-    // NOTE: We always use the -LastTrace version as we push the new trace in withFrame()
-    state.debugThrowLastTrace(ErrorType(info));
-}
 
 }
 

src/libexpr/flake/flake.cc

@@ -140,15 +140,15 @@ static FlakeInput parseFlakeInput(EvalState & state,
                         attrs.emplace(state.symbols[attr.name], (long unsigned int)attr.value->integer);
                         break;
                     default:
-                        throw TypeError("flake input attribute '%s' is %s while a string, Boolean, or integer is expected",
-                            state.symbols[attr.name], showType(*attr.value));
+                        state.error<TypeError>("flake input attribute '%s' is %s while a string, Boolean, or integer is expected",
+                            state.symbols[attr.name], showType(*attr.value)).debugThrow();
                 }
                 #pragma GCC diagnostic pop
             }
         } catch (Error & e) {
             e.addTrace(
                 state.positions[attr.pos],
-                hintfmt("while evaluating flake attribute '%s'", state.symbols[attr.name]));
+                HintFmt("while evaluating flake attribute '%s'", state.symbols[attr.name]));
             throw;
         }
     }
@@ -157,7 +157,7 @@ static FlakeInput parseFlakeInput(EvalState & state,
         try {
             input.ref = FlakeRef::fromAttrs(attrs);
         } catch (Error & e) {
-            e.addTrace(state.positions[pos], hintfmt("while evaluating flake input"));
+            e.addTrace(state.positions[pos], HintFmt("while evaluating flake input"));
             throw;
         }
     else {
@@ -205,8 +205,16 @@ static Flake getFlake(
     auto [sourceInfo, resolvedRef, lockedRef] = fetchOrSubstituteTree(
         state, originalRef, allowLookup, flakeCache);
 
+    // We need to guard against symlink attacks, but before we start doing
+    // filesystem operations we should make sure there's a flake.nix in the
+    // first place.
+    auto unsafeFlakeDir = sourceInfo.actualPath + "/" + lockedRef.subdir;
+    auto unsafeFlakeFile = unsafeFlakeDir + "/flake.nix";
+    if (!pathExists(unsafeFlakeFile))
+        throw Error("source tree referenced by '%s' does not contain a '%s/flake.nix' file", lockedRef, lockedRef.subdir);
+
     // Guard against symlink attacks.
-    auto flakeDir = canonPath(sourceInfo.actualPath + "/" + lockedRef.subdir, true);
+    auto flakeDir = canonPath(unsafeFlakeDir, true);
     auto flakeFile = canonPath(flakeDir + "/flake.nix", true);
     if (!isInDir(flakeFile, sourceInfo.actualPath))
         throw Error("'flake.nix' file of flake '%s' escapes from '%s'",
@@ -219,9 +227,6 @@ static Flake getFlake(
         .sourceInfo = std::make_shared<fetchers::Tree>(std::move(sourceInfo))
     };
 
-    if (!pathExists(flakeFile))
-        throw Error("source tree referenced by '%s' does not contain a '%s/flake.nix' file", lockedRef, lockedRef.subdir);
-
     Value vInfo;
     state.evalFile(CanonPath(flakeFile), vInfo, true); // FIXME: symlink attack
 
@@ -283,15 +288,15 @@ static Flake getFlake(
                 std::vector<std::string> ss;
                 for (auto elem : setting.value->listItems()) {
                     if (elem->type() != nString)
-                        throw TypeError("list element in flake configuration setting '%s' is %s while a string is expected",
-                            state.symbols[setting.name], showType(*setting.value));
+                        state.error<TypeError>("list element in flake configuration setting '%s' is %s while a string is expected",
+                            state.symbols[setting.name], showType(*setting.value)).debugThrow();
                     ss.emplace_back(state.forceStringNoCtx(*elem, setting.pos, ""));
                 }
                 flake.config.settings.emplace(state.symbols[setting.name], ss);
             }
             else
-                throw TypeError("flake configuration setting '%s' is %s",
-                    state.symbols[setting.name], showType(*setting.value));
+                state.error<TypeError>("flake configuration setting '%s' is %s",
+                    state.symbols[setting.name], showType(*setting.value)).debugThrow();
         }
     }
 
@@ -852,11 +857,11 @@ static void prim_flakeRefToString(
             attrs.emplace(state.symbols[attr.name],
                           std::string(attr.value->str()));
         } else {
-            state.error(
+            state.error<EvalError>(
                 "flake reference attribute sets may only contain integers, Booleans, "
                 "and strings, but attribute '%s' is %s",
                 state.symbols[attr.name],
-                showType(*attr.value)).debugThrow<EvalError>();
+                showType(*attr.value)).debugThrow();
         }
     }
     auto flakeRef = FlakeRef::fromAttrs(attrs);

src/libexpr/flake/lockfile.cc

@@ -89,7 +89,7 @@ LockFile::LockFile(const nlohmann::json & json, const Path & path)
                 std::string inputKey = i.value();
                 auto k = nodeMap.find(inputKey);
                 if (k == nodeMap.end()) {
-                    auto nodes = json["nodes"];
+                    auto & nodes = json["nodes"];
                     auto jsonNode2 = nodes.find(inputKey);
                     if (jsonNode2 == nodes.end())
                         throw Error("lock file references missing node '%s'", inputKey);

src/libexpr/gc-small-vector.hh

@@ -0,0 +1,42 @@
+#pragma once
+
+#include <boost/container/small_vector.hpp>
+
+#if HAVE_BOEHMGC
+
+#include <gc/gc.h>
+#include <gc/gc_cpp.h>
+#include <gc/gc_allocator.h>
+
+#endif
+
+namespace nix {
+
+struct Value;
+
+/**
+ * A GC compatible vector that may used a reserved portion of `nItems` on the stack instead of allocating on the heap.
+ */
+#if HAVE_BOEHMGC
+template <typename T, size_t nItems>
+using SmallVector = boost::container::small_vector<T, nItems, traceable_allocator<T>>;
+#else
+template <typename T, size_t nItems>
+using SmallVector = boost::container::small_vector<T, nItems>;
+#endif
+
+/**
+ * A vector of value pointers. See `SmallVector`.
+ */
+template <size_t nItems>
+using SmallValueVector = SmallVector<Value *, nItems>;
+
+/**
+ * A vector of values that must not be referenced after the vector is destroyed.
+ *
+ * See also `SmallValueVector`.
+ */
+template <size_t nItems>
+using SmallTemporaryValueVector = SmallVector<Value, nItems>;
+
+}

src/libexpr/get-drvs.cc

@@ -50,7 +50,7 @@ std::string DrvInfo::queryName() const
 {
     if (name == "" && attrs) {
         auto i = attrs->find(state->sName);
-        if (i == attrs->end()) throw TypeError("derivation name missing");
+        if (i == attrs->end()) state->error<TypeError>("derivation name missing").debugThrow();
         name = state->forceStringNoCtx(*i->value, noPos, "while evaluating the 'name' attribute of a derivation");
     }
     return name;
@@ -199,7 +199,7 @@ StringSet DrvInfo::queryMetaNames()
 
 bool DrvInfo::checkMeta(Value & v)
 {
-    state->forceValue(v, [&]() { return v.determinePos(noPos); });
+    state->forceValue(v, v.determinePos(noPos));
     if (v.type() == nList) {
         for (auto elem : v.listItems())
             if (!checkMeta(*elem)) return false;
@@ -305,7 +305,7 @@ static bool getDerivation(EvalState & state, Value & v,
     bool ignoreAssertionFailures)
 {
     try {
-        state.forceValue(v, [&]() { return v.determinePos(noPos); });
+        state.forceValue(v, v.determinePos(noPos));
         if (!state.isDerivation(v)) return true;
 
         /* Remove spurious duplicates (e.g., a set like `rec { x =
@@ -397,7 +397,8 @@ static void getDerivations(EvalState & state, Value & vIn,
         }
     }
 
-    else throw TypeError("expression does not evaluate to a derivation (or a set or list of those)");
+    else
+        state.error<TypeError>("expression does not evaluate to a derivation (or a set or list of those)").debugThrow();
 }
 
 

src/libexpr/json-to-value.cc

@@ -1,4 +1,6 @@
 #include "json-to-value.hh"
+#include "value.hh"
+#include "eval.hh"
 
 #include <variant>
 #include <nlohmann/json.hpp>
@@ -159,7 +161,7 @@ public:
     }
 
     bool parse_error(std::size_t, const std::string&, const nlohmann::detail::exception& ex) {
-        throw JSONParseError(ex.what());
+        throw JSONParseError("%s", ex.what());
     }
 };